test: [storageExt] finished gridfs service's test

powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java

@@ -14,6 +14,7 @@ import com.mongodb.client.gridfs.GridFSFindIterable;
 import com.mongodb.client.gridfs.model.GridFSFile;
 import com.mongodb.client.model.Filters;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.time.DateUtils;
 import org.bson.conversions.Bson;
@@ -67,6 +68,7 @@ public class GridFsService extends AbstractDFsService {
     @Override
     public void download(DownloadRequest downloadRequest) throws IOException {
         GridFSBucket bucket = getBucket(downloadRequest.getFileLocation().getBucket());
+        FileUtils.forceMkdirParent(downloadRequest.getTarget());
         try (GridFSDownloadStream gis = bucket.openDownloadStream(downloadRequest.getFileLocation().getName());
              BufferedOutputStream bos = new BufferedOutputStream(Files.newOutputStream(downloadRequest.getTarget().toPath()))
         ) {
@@ -128,7 +130,7 @@ public class GridFsService extends AbstractDFsService {
         return environment.getProperty(SPRING_MONGO_DB_CONFIG_KEY);
     }
 
-    private void initMongo(String uri) {
+    void initMongo(String uri) {
         log.info("[GridFsService] mongoDB uri: {}", uri);
         if (StringUtils.isEmpty(uri)) {
             log.warn("[GridFsService] uri is empty, GridFsService is off now!");
@@ -137,9 +139,14 @@ public class GridFsService extends AbstractDFsService {
 
         ConnectionString connectionString = new ConnectionString(uri);
         mongoClient = MongoClients.create(connectionString);
-        db = mongoClient.getDatabase(Optional.ofNullable(connectionString.getDatabase()).orElse("pj"));
 
-        log.info("[GridFsService] turn on mongodb GridFs as storage layer.");
+        if (StringUtils.isEmpty(connectionString.getDatabase())) {
+            log.warn("[GridFsService] can't find database info from uri, will use [powerjob] as default, please make sure you have created the database 'powerjob'");
+        }
+
+        db = mongoClient.getDatabase(Optional.ofNullable(connectionString.getDatabase()).orElse("powerjob"));
+
+        log.info("[GridFsService] initialize MongoDB and GridFS successfully, will use mongodb GridFs as storage layer.");
     }
 
     @Override

powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AbstractDfsServiceTest.java

@@ -0,0 +1,82 @@
+package tech.powerjob.server.persistence.storage.impl;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
+import org.junit.jupiter.api.Test;
+import tech.powerjob.common.serialize.JsonUtils;
+import tech.powerjob.server.common.utils.OmsFileUtils;
+import tech.powerjob.server.extension.dfs.*;
+
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
+
+/**
+ * AbstractDfsServiceTest
+ *
+ * @author tjq
+ * @since 2023/7/30
+ */
+@Slf4j
+public abstract class AbstractDfsServiceTest {
+
+    abstract protected Optional<DFsService> fetchService();
+
+    @Test
+    void testBaseFileOperation() throws Exception {
+
+        Optional<DFsService> aliOssServiceOpt = fetchService();
+        if (!aliOssServiceOpt.isPresent()) {
+            return;
+        }
+
+        DFsService aliOssService = aliOssServiceOpt.get();
+
+        String content = "wlcgyqsl";
+
+        String temporarySourcePath = OmsFileUtils.genTemporaryWorkPath() + "source.txt";
+        String temporaryDownloadPath = OmsFileUtils.genTemporaryWorkPath() + "download.txt";
+
+        log.info("[testBaseFileOperation] temporarySourcePath: {}", temporarySourcePath);
+        File sourceFile = new File(temporarySourcePath);
+        FileUtils.forceMkdirParent(sourceFile);
+        OmsFileUtils.string2File(content, sourceFile);
+
+        FileLocation fileLocation = new FileLocation().setBucket("pj_test").setName("testAliOss.txt");
+
+        StoreRequest storeRequest = new StoreRequest()
+                .setFileLocation(fileLocation)
+                .setLocalFile(sourceFile);
+
+        // 存储
+        aliOssService.store(storeRequest);
+
+        // 读取 meta
+        Optional<FileMeta> metaOpt = aliOssService.fetchFileMeta(fileLocation);
+        assert metaOpt.isPresent();
+
+        log.info("[testBaseFileOperation] file meta: {}", JsonUtils.toJSONString(metaOpt.get()));
+
+        // 下载
+        log.info("[testBaseFileOperation] temporaryDownloadPath: {}", temporaryDownloadPath);
+        File downloadFile = new File(temporaryDownloadPath);
+        DownloadRequest downloadRequest = new DownloadRequest()
+                .setFileLocation(fileLocation)
+                .setTarget(downloadFile);
+        aliOssService.download(downloadRequest);
+
+        String downloadFileContent = FileUtils.readFileToString(downloadFile, StandardCharsets.UTF_8);
+        log.info("[testBaseFileOperation] download content: {}", downloadFileContent);
+        assert downloadFileContent.equals(content);
+    }
+
+    @Test
+    void testFileNotExist() throws Exception {
+        Optional<DFsService> aliOssServiceOpt = fetchService();
+        if (!aliOssServiceOpt.isPresent()) {
+            return;
+        }
+        Optional<FileMeta> metaOpt = aliOssServiceOpt.get().fetchFileMeta(new FileLocation().setBucket("tjq").setName("yhz"));
+        assert !metaOpt.isPresent();
+    }
+}

powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AliOssServiceTest.java

@@ -3,18 +3,9 @@ package tech.powerjob.server.persistence.storage.impl;
 import com.aliyun.oss.common.utils.AuthUtils;
 import com.aliyun.oss.common.utils.StringUtils;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.exception.ExceptionUtils;
-import org.junit.jupiter.api.Test;
+import tech.powerjob.server.extension.dfs.DFsService;
-import tech.powerjob.common.serialize.JsonUtils;
-import tech.powerjob.server.common.utils.OmsFileUtils;
-import tech.powerjob.server.extension.dfs.DownloadRequest;
-import tech.powerjob.server.extension.dfs.FileLocation;
-import tech.powerjob.server.extension.dfs.FileMeta;
-import tech.powerjob.server.extension.dfs.StoreRequest;
 
-import java.io.File;
-import java.nio.charset.StandardCharsets;
 import java.util.Optional;
 
 
@@ -25,74 +16,16 @@ import java.util.Optional;
  * @since 2023/7/30
  */
 @Slf4j
-class AliOssServiceTest {
+class AliOssServiceTest extends AbstractDfsServiceTest {
 
     private static final String BUCKET = "power-job";
 
-    @Test
-    void testBaseFileOperation() throws Exception {
-
-        Optional<AliOssService> aliOssServiceOpt = fetchService();
-        if (!aliOssServiceOpt.isPresent()) {
-            return;
-        }
-
-        AliOssService aliOssService = aliOssServiceOpt.get();
-
-        String content = "wlcgyqsl";
-
-        String temporarySourcePath = OmsFileUtils.genTemporaryWorkPath() + "source.txt";
-        String temporaryDownloadPath = OmsFileUtils.genTemporaryWorkPath() + "download.txt";
-
-        log.info("[testBaseFileOperation] temporarySourcePath: {}", temporarySourcePath);
-        File sourceFile = new File(temporarySourcePath);
-        FileUtils.forceMkdirParent(sourceFile);
-        OmsFileUtils.string2File(content, sourceFile);
-
-        FileLocation fileLocation = new FileLocation().setBucket("pj_test").setName("testAliOss.txt");
-
-        StoreRequest storeRequest = new StoreRequest()
-                .setFileLocation(fileLocation)
-                .setLocalFile(sourceFile);
-
-        // 存储
-        aliOssService.store(storeRequest);
-
-        // 读取 meta
-        Optional<FileMeta> metaOpt = aliOssService.fetchFileMeta(fileLocation);
-        assert metaOpt.isPresent();
-
-        log.info("[testBaseFileOperation] file meta: {}", JsonUtils.toJSONString(metaOpt.get()));
-
-        // 下载
-        log.info("[testBaseFileOperation] temporaryDownloadPath: {}", temporaryDownloadPath);
-        File downloadFile = new File(temporaryDownloadPath);
-        DownloadRequest downloadRequest = new DownloadRequest()
-                .setFileLocation(fileLocation)
-                .setTarget(downloadFile);
-        aliOssService.download(downloadRequest);
-
-        String downloadFileContent = FileUtils.readFileToString(downloadFile, StandardCharsets.UTF_8);
-        log.info("[testBaseFileOperation] download content: {}", downloadFileContent);
-        assert downloadFileContent.equals(content);
-    }
-
-    @Test
-    void testFileNotExist() throws Exception {
-        Optional<AliOssService> aliOssServiceOpt = fetchService();
-        if (!aliOssServiceOpt.isPresent()) {
-            return;
-        }
-        Optional<FileMeta> metaOpt = aliOssServiceOpt.get().fetchFileMeta(new FileLocation().setBucket("tjq").setName("yhz"));
-        assert !metaOpt.isPresent();
-    }
-
     /**
      * 依赖阿里云账号密码测试，为了保证单测在其他环境也能通过，如果发现不存在配置则直接跳过
      * @return AliOssService
      */
-    private Optional<AliOssService> fetchService() {
+    @Override
-
+    protected Optional<DFsService> fetchService() {
         String accessKeyId = StringUtils.trim(System.getenv(AuthUtils.ACCESS_KEY_ENV_VAR));
         String secretAccessKey = StringUtils.trim(System.getenv(AuthUtils.SECRET_KEY_ENV_VAR));
 

powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/GridFsServiceTest.java

@@ -1,6 +1,14 @@
 package tech.powerjob.server.persistence.storage.impl;
 
-import static org.junit.jupiter.api.Assertions.*;
+import com.alibaba.fastjson.JSONObject;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import tech.powerjob.server.extension.dfs.DFsService;
+
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
 
 /**
  * test GridFS
@@ -8,6 +16,29 @@ import static org.junit.jupiter.api.Assertions.*;
  * @author tjq
  * @since 2023/7/30
  */
-class GridFsServiceTest {
+@Slf4j
+class GridFsServiceTest extends AbstractDfsServiceTest {
 
+    @Override
+    protected Optional<DFsService> fetchService() {
+
+        try {
+            // 后续本地测试，密钥相关的内容统一存入 .powerjob_test 中，方便管理
+            String content = FileUtils.readFileToString(new File(System.getProperty("user.home").concat("/.powerjob_test")), StandardCharsets.UTF_8);
+            if (StringUtils.isNotEmpty(content)) {
+                JSONObject jsonObject = JSONObject.parseObject(content);
+                Object mongoUri = jsonObject.get("mongoUri");
+                if (mongoUri != null) {
+                    GridFsService gridFsService = new GridFsService();
+                    gridFsService.initMongo(String.valueOf(mongoUri));
+
+                    return Optional.of(gridFsService);
+                }
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            log.warn("[GridFsServiceTest] fetch mongo config failed, skip!");
+        }
+        return Optional.empty();
+    }
 }