diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java index d78da3fa..facbc2f9 100644 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java @@ -14,6 +14,7 @@ import com.mongodb.client.gridfs.GridFSFindIterable; import com.mongodb.client.gridfs.model.GridFSFile; import com.mongodb.client.model.Filters; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.DateUtils; import org.bson.conversions.Bson; @@ -67,6 +68,7 @@ public class GridFsService extends AbstractDFsService { @Override public void download(DownloadRequest downloadRequest) throws IOException { GridFSBucket bucket = getBucket(downloadRequest.getFileLocation().getBucket()); + FileUtils.forceMkdirParent(downloadRequest.getTarget()); try (GridFSDownloadStream gis = bucket.openDownloadStream(downloadRequest.getFileLocation().getName()); BufferedOutputStream bos = new BufferedOutputStream(Files.newOutputStream(downloadRequest.getTarget().toPath())) ) { @@ -128,7 +130,7 @@ public class GridFsService extends AbstractDFsService { return environment.getProperty(SPRING_MONGO_DB_CONFIG_KEY); } - private void initMongo(String uri) { + void initMongo(String uri) { log.info("[GridFsService] mongoDB uri: {}", uri); if (StringUtils.isEmpty(uri)) { log.warn("[GridFsService] uri is empty, GridFsService is off now!"); @@ -137,9 +139,14 @@ public class GridFsService extends AbstractDFsService { ConnectionString connectionString = new ConnectionString(uri); mongoClient = MongoClients.create(connectionString); - db = mongoClient.getDatabase(Optional.ofNullable(connectionString.getDatabase()).orElse("pj")); - log.info("[GridFsService] turn on mongodb GridFs as storage layer."); + if (StringUtils.isEmpty(connectionString.getDatabase())) { + log.warn("[GridFsService] can't find database info from uri, will use [powerjob] as default, please make sure you have created the database 'powerjob'"); + } + + db = mongoClient.getDatabase(Optional.ofNullable(connectionString.getDatabase()).orElse("powerjob")); + + log.info("[GridFsService] initialize MongoDB and GridFS successfully, will use mongodb GridFs as storage layer."); } @Override diff --git a/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AbstractDfsServiceTest.java b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AbstractDfsServiceTest.java new file mode 100644 index 00000000..bd15b482 --- /dev/null +++ b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AbstractDfsServiceTest.java @@ -0,0 +1,82 @@ +package tech.powerjob.server.persistence.storage.impl; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; +import tech.powerjob.common.serialize.JsonUtils; +import tech.powerjob.server.common.utils.OmsFileUtils; +import tech.powerjob.server.extension.dfs.*; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.util.Optional; + +/** + * AbstractDfsServiceTest + * + * @author tjq + * @since 2023/7/30 + */ +@Slf4j +public abstract class AbstractDfsServiceTest { + + abstract protected Optional fetchService(); + + @Test + void testBaseFileOperation() throws Exception { + + Optional aliOssServiceOpt = fetchService(); + if (!aliOssServiceOpt.isPresent()) { + return; + } + + DFsService aliOssService = aliOssServiceOpt.get(); + + String content = "wlcgyqsl"; + + String temporarySourcePath = OmsFileUtils.genTemporaryWorkPath() + "source.txt"; + String temporaryDownloadPath = OmsFileUtils.genTemporaryWorkPath() + "download.txt"; + + log.info("[testBaseFileOperation] temporarySourcePath: {}", temporarySourcePath); + File sourceFile = new File(temporarySourcePath); + FileUtils.forceMkdirParent(sourceFile); + OmsFileUtils.string2File(content, sourceFile); + + FileLocation fileLocation = new FileLocation().setBucket("pj_test").setName("testAliOss.txt"); + + StoreRequest storeRequest = new StoreRequest() + .setFileLocation(fileLocation) + .setLocalFile(sourceFile); + + // 存储 + aliOssService.store(storeRequest); + + // 读取 meta + Optional metaOpt = aliOssService.fetchFileMeta(fileLocation); + assert metaOpt.isPresent(); + + log.info("[testBaseFileOperation] file meta: {}", JsonUtils.toJSONString(metaOpt.get())); + + // 下载 + log.info("[testBaseFileOperation] temporaryDownloadPath: {}", temporaryDownloadPath); + File downloadFile = new File(temporaryDownloadPath); + DownloadRequest downloadRequest = new DownloadRequest() + .setFileLocation(fileLocation) + .setTarget(downloadFile); + aliOssService.download(downloadRequest); + + String downloadFileContent = FileUtils.readFileToString(downloadFile, StandardCharsets.UTF_8); + log.info("[testBaseFileOperation] download content: {}", downloadFileContent); + assert downloadFileContent.equals(content); + } + + @Test + void testFileNotExist() throws Exception { + Optional aliOssServiceOpt = fetchService(); + if (!aliOssServiceOpt.isPresent()) { + return; + } + Optional metaOpt = aliOssServiceOpt.get().fetchFileMeta(new FileLocation().setBucket("tjq").setName("yhz")); + assert !metaOpt.isPresent(); + } +} diff --git a/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AliOssServiceTest.java b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AliOssServiceTest.java index cbcdae13..94167731 100644 --- a/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AliOssServiceTest.java +++ b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AliOssServiceTest.java @@ -3,18 +3,9 @@ package tech.powerjob.server.persistence.storage.impl; import com.aliyun.oss.common.utils.AuthUtils; import com.aliyun.oss.common.utils.StringUtils; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.exception.ExceptionUtils; -import org.junit.jupiter.api.Test; -import tech.powerjob.common.serialize.JsonUtils; -import tech.powerjob.server.common.utils.OmsFileUtils; -import tech.powerjob.server.extension.dfs.DownloadRequest; -import tech.powerjob.server.extension.dfs.FileLocation; -import tech.powerjob.server.extension.dfs.FileMeta; -import tech.powerjob.server.extension.dfs.StoreRequest; +import tech.powerjob.server.extension.dfs.DFsService; -import java.io.File; -import java.nio.charset.StandardCharsets; import java.util.Optional; @@ -25,74 +16,16 @@ import java.util.Optional; * @since 2023/7/30 */ @Slf4j -class AliOssServiceTest { +class AliOssServiceTest extends AbstractDfsServiceTest { private static final String BUCKET = "power-job"; - @Test - void testBaseFileOperation() throws Exception { - - Optional aliOssServiceOpt = fetchService(); - if (!aliOssServiceOpt.isPresent()) { - return; - } - - AliOssService aliOssService = aliOssServiceOpt.get(); - - String content = "wlcgyqsl"; - - String temporarySourcePath = OmsFileUtils.genTemporaryWorkPath() + "source.txt"; - String temporaryDownloadPath = OmsFileUtils.genTemporaryWorkPath() + "download.txt"; - - log.info("[testBaseFileOperation] temporarySourcePath: {}", temporarySourcePath); - File sourceFile = new File(temporarySourcePath); - FileUtils.forceMkdirParent(sourceFile); - OmsFileUtils.string2File(content, sourceFile); - - FileLocation fileLocation = new FileLocation().setBucket("pj_test").setName("testAliOss.txt"); - - StoreRequest storeRequest = new StoreRequest() - .setFileLocation(fileLocation) - .setLocalFile(sourceFile); - - // 存储 - aliOssService.store(storeRequest); - - // 读取 meta - Optional metaOpt = aliOssService.fetchFileMeta(fileLocation); - assert metaOpt.isPresent(); - - log.info("[testBaseFileOperation] file meta: {}", JsonUtils.toJSONString(metaOpt.get())); - - // 下载 - log.info("[testBaseFileOperation] temporaryDownloadPath: {}", temporaryDownloadPath); - File downloadFile = new File(temporaryDownloadPath); - DownloadRequest downloadRequest = new DownloadRequest() - .setFileLocation(fileLocation) - .setTarget(downloadFile); - aliOssService.download(downloadRequest); - - String downloadFileContent = FileUtils.readFileToString(downloadFile, StandardCharsets.UTF_8); - log.info("[testBaseFileOperation] download content: {}", downloadFileContent); - assert downloadFileContent.equals(content); - } - - @Test - void testFileNotExist() throws Exception { - Optional aliOssServiceOpt = fetchService(); - if (!aliOssServiceOpt.isPresent()) { - return; - } - Optional metaOpt = aliOssServiceOpt.get().fetchFileMeta(new FileLocation().setBucket("tjq").setName("yhz")); - assert !metaOpt.isPresent(); - } - /** * 依赖阿里云账号密码测试，为了保证单测在其他环境也能通过，如果发现不存在配置则直接跳过 * @return AliOssService */ - private Optional fetchService() { - + @Override + protected Optional fetchService() { String accessKeyId = StringUtils.trim(System.getenv(AuthUtils.ACCESS_KEY_ENV_VAR)); String secretAccessKey = StringUtils.trim(System.getenv(AuthUtils.SECRET_KEY_ENV_VAR)); diff --git a/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/GridFsServiceTest.java b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/GridFsServiceTest.java index 8212748c..1abfba2d 100644 --- a/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/GridFsServiceTest.java +++ b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/GridFsServiceTest.java @@ -1,6 +1,14 @@ package tech.powerjob.server.persistence.storage.impl; -import static org.junit.jupiter.api.Assertions.*; +import com.alibaba.fastjson.JSONObject; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import tech.powerjob.server.extension.dfs.DFsService; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.util.Optional; /** * test GridFS @@ -8,6 +16,29 @@ import static org.junit.jupiter.api.Assertions.*; * @author tjq * @since 2023/7/30 */ -class GridFsServiceTest { +@Slf4j +class GridFsServiceTest extends AbstractDfsServiceTest { + @Override + protected Optional fetchService() { + + try { + // 后续本地测试，密钥相关的内容统一存入 .powerjob_test 中，方便管理 + String content = FileUtils.readFileToString(new File(System.getProperty("user.home").concat("/.powerjob_test")), StandardCharsets.UTF_8); + if (StringUtils.isNotEmpty(content)) { + JSONObject jsonObject = JSONObject.parseObject(content); + Object mongoUri = jsonObject.get("mongoUri"); + if (mongoUri != null) { + GridFsService gridFsService = new GridFsService(); + gridFsService.initMongo(String.valueOf(mongoUri)); + + return Optional.of(gridFsService); + } + } + } catch (Exception e) { + e.printStackTrace(); + log.warn("[GridFsServiceTest] fetch mongo config failed, skip!"); + } + return Optional.empty(); + } } \ No newline at end of file