mirror of
https://github.com/PowerJob/PowerJob.git
synced 2025-07-17 00:00:04 +08:00
test: [storageExt] finished gridfs service's test
This commit is contained in:
tjq
parent
1c70bbc670
commit
d03247ea03
@ -14,6 +14,7 @@ import com.mongodb.client.gridfs.GridFSFindIterable;
|
||||
import com.mongodb.client.gridfs.model.GridFSFile;
|
||||
import com.mongodb.client.model.Filters;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.lang3.time.DateUtils;
|
||||
import org.bson.conversions.Bson;
|
||||
@ -67,6 +68,7 @@ public class GridFsService extends AbstractDFsService {
|
||||
@Override
|
||||
public void download(DownloadRequest downloadRequest) throws IOException {
|
||||
GridFSBucket bucket = getBucket(downloadRequest.getFileLocation().getBucket());
|
||||
FileUtils.forceMkdirParent(downloadRequest.getTarget());
|
||||
try (GridFSDownloadStream gis = bucket.openDownloadStream(downloadRequest.getFileLocation().getName());
|
||||
BufferedOutputStream bos = new BufferedOutputStream(Files.newOutputStream(downloadRequest.getTarget().toPath()))
|
||||
) {
|
||||
@ -128,7 +130,7 @@ public class GridFsService extends AbstractDFsService {
|
||||
return environment.getProperty(SPRING_MONGO_DB_CONFIG_KEY);
|
||||
}
|
||||
|
||||
private void initMongo(String uri) {
|
||||
void initMongo(String uri) {
|
||||
log.info("[GridFsService] mongoDB uri: {}", uri);
|
||||
if (StringUtils.isEmpty(uri)) {
|
||||
log.warn("[GridFsService] uri is empty, GridFsService is off now!");
|
||||
@ -137,9 +139,14 @@ public class GridFsService extends AbstractDFsService {
|
||||
|
||||
ConnectionString connectionString = new ConnectionString(uri);
|
||||
mongoClient = MongoClients.create(connectionString);
|
||||
db = mongoClient.getDatabase(Optional.ofNullable(connectionString.getDatabase()).orElse("pj"));
|
||||
|
||||
log.info("[GridFsService] turn on mongodb GridFs as storage layer.");
|
||||
if (StringUtils.isEmpty(connectionString.getDatabase())) {
|
||||
log.warn("[GridFsService] can't find database info from uri, will use [powerjob] as default, please make sure you have created the database 'powerjob'");
|
||||
}
|
||||
|
||||
db = mongoClient.getDatabase(Optional.ofNullable(connectionString.getDatabase()).orElse("powerjob"));
|
||||
|
||||
log.info("[GridFsService] initialize MongoDB and GridFS successfully, will use mongodb GridFs as storage layer.");
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@ -0,0 +1,82 @@
|
||||
package tech.powerjob.server.persistence.storage.impl;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import tech.powerjob.common.serialize.JsonUtils;
|
||||
import tech.powerjob.server.common.utils.OmsFileUtils;
|
||||
import tech.powerjob.server.extension.dfs.*;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* AbstractDfsServiceTest
|
||||
*
|
||||
* @author tjq
|
||||
* @since 2023/7/30
|
||||
*/
|
||||
@Slf4j
|
||||
public abstract class AbstractDfsServiceTest {
|
||||
|
||||
abstract protected Optional<DFsService> fetchService();
|
||||
|
||||
@Test
|
||||
void testBaseFileOperation() throws Exception {
|
||||
|
||||
Optional<DFsService> aliOssServiceOpt = fetchService();
|
||||
if (!aliOssServiceOpt.isPresent()) {
|
||||
return;
|
||||
}
|
||||
|
||||
DFsService aliOssService = aliOssServiceOpt.get();
|
||||
|
||||
String content = "wlcgyqsl";
|
||||
|
||||
String temporarySourcePath = OmsFileUtils.genTemporaryWorkPath() + "source.txt";
|
||||
String temporaryDownloadPath = OmsFileUtils.genTemporaryWorkPath() + "download.txt";
|
||||
|
||||
log.info("[testBaseFileOperation] temporarySourcePath: {}", temporarySourcePath);
|
||||
File sourceFile = new File(temporarySourcePath);
|
||||
FileUtils.forceMkdirParent(sourceFile);
|
||||
OmsFileUtils.string2File(content, sourceFile);
|
||||
|
||||
FileLocation fileLocation = new FileLocation().setBucket("pj_test").setName("testAliOss.txt");
|
||||
|
||||
StoreRequest storeRequest = new StoreRequest()
|
||||
.setFileLocation(fileLocation)
|
||||
.setLocalFile(sourceFile);
|
||||
|
||||
// 存储
|
||||
aliOssService.store(storeRequest);
|
||||
|
||||
// 读取 meta
|
||||
Optional<FileMeta> metaOpt = aliOssService.fetchFileMeta(fileLocation);
|
||||
assert metaOpt.isPresent();
|
||||
|
||||
log.info("[testBaseFileOperation] file meta: {}", JsonUtils.toJSONString(metaOpt.get()));
|
||||
|
||||
// 下载
|
||||
log.info("[testBaseFileOperation] temporaryDownloadPath: {}", temporaryDownloadPath);
|
||||
File downloadFile = new File(temporaryDownloadPath);
|
||||
DownloadRequest downloadRequest = new DownloadRequest()
|
||||
.setFileLocation(fileLocation)
|
||||
.setTarget(downloadFile);
|
||||
aliOssService.download(downloadRequest);
|
||||
|
||||
String downloadFileContent = FileUtils.readFileToString(downloadFile, StandardCharsets.UTF_8);
|
||||
log.info("[testBaseFileOperation] download content: {}", downloadFileContent);
|
||||
assert downloadFileContent.equals(content);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testFileNotExist() throws Exception {
|
||||
Optional<DFsService> aliOssServiceOpt = fetchService();
|
||||
if (!aliOssServiceOpt.isPresent()) {
|
||||
return;
|
||||
}
|
||||
Optional<FileMeta> metaOpt = aliOssServiceOpt.get().fetchFileMeta(new FileLocation().setBucket("tjq").setName("yhz"));
|
||||
assert !metaOpt.isPresent();
|
||||
}
|
||||
}
|
||||
@ -3,18 +3,9 @@ package tech.powerjob.server.persistence.storage.impl;
|
||||
import com.aliyun.oss.common.utils.AuthUtils;
|
||||
import com.aliyun.oss.common.utils.StringUtils;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.lang3.exception.ExceptionUtils;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import tech.powerjob.common.serialize.JsonUtils;
|
||||
import tech.powerjob.server.common.utils.OmsFileUtils;
|
||||
import tech.powerjob.server.extension.dfs.DownloadRequest;
|
||||
import tech.powerjob.server.extension.dfs.FileLocation;
|
||||
import tech.powerjob.server.extension.dfs.FileMeta;
|
||||
import tech.powerjob.server.extension.dfs.StoreRequest;
|
||||
import tech.powerjob.server.extension.dfs.DFsService;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Optional;
|
||||
|
||||
|
||||
@ -25,74 +16,16 @@ import java.util.Optional;
|
||||
* @since 2023/7/30
|
||||
*/
|
||||
@Slf4j
|
||||
class AliOssServiceTest {
|
||||
class AliOssServiceTest extends AbstractDfsServiceTest {
|
||||
|
||||
private static final String BUCKET = "power-job";
|
||||
|
||||
@Test
|
||||
void testBaseFileOperation() throws Exception {
|
||||
|
||||
Optional<AliOssService> aliOssServiceOpt = fetchService();
|
||||
if (!aliOssServiceOpt.isPresent()) {
|
||||
return;
|
||||
}
|
||||
|
||||
AliOssService aliOssService = aliOssServiceOpt.get();
|
||||
|
||||
String content = "wlcgyqsl";
|
||||
|
||||
String temporarySourcePath = OmsFileUtils.genTemporaryWorkPath() + "source.txt";
|
||||
String temporaryDownloadPath = OmsFileUtils.genTemporaryWorkPath() + "download.txt";
|
||||
|
||||
log.info("[testBaseFileOperation] temporarySourcePath: {}", temporarySourcePath);
|
||||
File sourceFile = new File(temporarySourcePath);
|
||||
FileUtils.forceMkdirParent(sourceFile);
|
||||
OmsFileUtils.string2File(content, sourceFile);
|
||||
|
||||
FileLocation fileLocation = new FileLocation().setBucket("pj_test").setName("testAliOss.txt");
|
||||
|
||||
StoreRequest storeRequest = new StoreRequest()
|
||||
.setFileLocation(fileLocation)
|
||||
.setLocalFile(sourceFile);
|
||||
|
||||
// 存储
|
||||
aliOssService.store(storeRequest);
|
||||
|
||||
// 读取 meta
|
||||
Optional<FileMeta> metaOpt = aliOssService.fetchFileMeta(fileLocation);
|
||||
assert metaOpt.isPresent();
|
||||
|
||||
log.info("[testBaseFileOperation] file meta: {}", JsonUtils.toJSONString(metaOpt.get()));
|
||||
|
||||
// 下载
|
||||
log.info("[testBaseFileOperation] temporaryDownloadPath: {}", temporaryDownloadPath);
|
||||
File downloadFile = new File(temporaryDownloadPath);
|
||||
DownloadRequest downloadRequest = new DownloadRequest()
|
||||
.setFileLocation(fileLocation)
|
||||
.setTarget(downloadFile);
|
||||
aliOssService.download(downloadRequest);
|
||||
|
||||
String downloadFileContent = FileUtils.readFileToString(downloadFile, StandardCharsets.UTF_8);
|
||||
log.info("[testBaseFileOperation] download content: {}", downloadFileContent);
|
||||
assert downloadFileContent.equals(content);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testFileNotExist() throws Exception {
|
||||
Optional<AliOssService> aliOssServiceOpt = fetchService();
|
||||
if (!aliOssServiceOpt.isPresent()) {
|
||||
return;
|
||||
}
|
||||
Optional<FileMeta> metaOpt = aliOssServiceOpt.get().fetchFileMeta(new FileLocation().setBucket("tjq").setName("yhz"));
|
||||
assert !metaOpt.isPresent();
|
||||
}
|
||||
|
||||
/**
|
||||
* 依赖阿里云账号密码测试，为了保证单测在其他环境也能通过，如果发现不存在配置则直接跳过
|
||||
* @return AliOssService
|
||||
*/
|
||||
private Optional<AliOssService> fetchService() {
|
||||
|
||||
@Override
|
||||
protected Optional<DFsService> fetchService() {
|
||||
String accessKeyId = StringUtils.trim(System.getenv(AuthUtils.ACCESS_KEY_ENV_VAR));
|
||||
String secretAccessKey = StringUtils.trim(System.getenv(AuthUtils.SECRET_KEY_ENV_VAR));
|
||||
|
||||
|
||||
@ -1,6 +1,14 @@
|
||||
package tech.powerjob.server.persistence.storage.impl;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import tech.powerjob.server.extension.dfs.DFsService;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* test GridFS
|
||||
@ -8,6 +16,29 @@ import static org.junit.jupiter.api.Assertions.*;
|
||||
* @author tjq
|
||||
* @since 2023/7/30
|
||||
*/
|
||||
class GridFsServiceTest {
|
||||
@Slf4j
|
||||
class GridFsServiceTest extends AbstractDfsServiceTest {
|
||||
|
||||
@Override
|
||||
protected Optional<DFsService> fetchService() {
|
||||
|
||||
try {
|
||||
// 后续本地测试，密钥相关的内容统一存入 .powerjob_test 中，方便管理
|
||||
String content = FileUtils.readFileToString(new File(System.getProperty("user.home").concat("/.powerjob_test")), StandardCharsets.UTF_8);
|
||||
if (StringUtils.isNotEmpty(content)) {
|
||||
JSONObject jsonObject = JSONObject.parseObject(content);
|
||||
Object mongoUri = jsonObject.get("mongoUri");
|
||||
if (mongoUri != null) {
|
||||
GridFsService gridFsService = new GridFsService();
|
||||
gridFsService.initMongo(String.valueOf(mongoUri));
|
||||
|
||||
return Optional.of(gridFsService);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
log.warn("[GridFsServiceTest] fetch mongo config failed, skip!");
|
||||
}
|
||||
return Optional.empty();
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user