From 8de1c479710527c143825ab14133d24a1c764456 Mon Sep 17 00:00:00 2001 From: yw <2185022909@qq.com> Date: Wed, 23 Aug 2023 09:37:14 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=AD=98=E5=82=A8?= =?UTF-8?q?=E6=89=A9=E5=B1=95-Minio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- powerjob-server/pom.xml | 8 +- .../powerjob-server-persistence/pom.xml | 4 + .../storage/StorageConfiguration.java | 11 +- .../storage/impl/MinioOssService.java | 234 ++++++++++++++++++ 4 files changed, 252 insertions(+), 5 deletions(-) create mode 100644 powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/MinioOssService.java diff --git a/powerjob-server/pom.xml b/powerjob-server/pom.xml index cc097c96..1815ec51 100644 --- a/powerjob-server/pom.xml +++ b/powerjob-server/pom.xml @@ -55,6 +55,7 @@ 4.3.5 1.6.14 3.17.1 + 8.5.2 4.4 @@ -113,7 +114,12 @@ aliyun-sdk-oss ${aliyun-sdk-oss.version} - + + + io.minio + minio + ${minio.version} + org.apache.commons diff --git a/powerjob-server/powerjob-server-persistence/pom.xml b/powerjob-server/powerjob-server-persistence/pom.xml index e51b54a9..d467dc63 100644 --- a/powerjob-server/powerjob-server-persistence/pom.xml +++ b/powerjob-server/powerjob-server-persistence/pom.xml @@ -40,6 +40,10 @@ com.aliyun.oss aliyun-sdk-oss + + io.minio + minio + \ No newline at end of file diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/StorageConfiguration.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/StorageConfiguration.java index 6b5627fe..890fbeba 100644 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/StorageConfiguration.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/StorageConfiguration.java @@ -4,10 +4,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; import tech.powerjob.server.extension.dfs.DFsService; -import tech.powerjob.server.persistence.storage.impl.AliOssService; -import tech.powerjob.server.persistence.storage.impl.EmptyDFsService; -import tech.powerjob.server.persistence.storage.impl.GridFsService; -import tech.powerjob.server.persistence.storage.impl.MySqlSeriesDfsService; +import tech.powerjob.server.persistence.storage.impl.*; /** * 初始化内置的存储服务 @@ -36,6 +33,12 @@ public class StorageConfiguration { return new AliOssService(); } + @Bean + @Conditional(MinioOssService.MinioOssCondition.class) + public DFsService initMinioOssFs() { + return new MinioOssService(); + } + @Bean @Conditional(EmptyDFsService.EmptyCondition.class) public DFsService initEmptyDfs() { diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/MinioOssService.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/MinioOssService.java new file mode 100644 index 00000000..fee449ca --- /dev/null +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/MinioOssService.java @@ -0,0 +1,234 @@ +package tech.powerjob.server.persistence.storage.impl; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import io.minio.*; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Conditional; +import org.springframework.core.env.Environment; +import tech.powerjob.server.common.spring.condition.PropertyAndOneBeanCondition; +import tech.powerjob.server.extension.dfs.*; +import tech.powerjob.server.persistence.storage.AbstractDFsService; + +import javax.annotation.Priority; +import java.nio.file.Files; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * MINIO support + * 配置项： + * oms.storage.dfs.minio.endpoint + * oms.storage.dfs.minio.bucketName + * oms.storage.dfs.minio.accessKey + * oms.storage.dfs.minio.secretKey + * + * @author xinyi + * @since 2023/8/21 + */ +@Slf4j +@Priority(value = Integer.MAX_VALUE - 1) +@Conditional(MinioOssService.MinioOssCondition.class) +public class MinioOssService extends AbstractDFsService { + + private static final String TYPE_MINIO = "minio"; + private static final String KEY_ENDPOINT = "endpoint"; + private static final String KEY_BUCKET_NAME = "bucketName"; + private static final String ACCESS_KEY = "accessKey"; + private static final String SECRET_KEY = "secretKey"; + private MinioClient minioClient; + private String bucket; + private static final String NO_SUCH_KEY = "NoSuchKey"; + + @Override + public void store(StoreRequest storeRequest) { + try { + minioClient.uploadObject(UploadObjectArgs.builder() + .bucket(this.bucket) + .object(parseFileName(storeRequest.getFileLocation())) + .filename(storeRequest.getLocalFile().getPath()) + .contentType(Files.probeContentType(storeRequest.getLocalFile().toPath())) + .build()); + } catch (Throwable t) { + ExceptionUtils.rethrow(t); + } + } + + @Override + public void download(DownloadRequest downloadRequest) { + try { + FileUtils.forceMkdirParent(downloadRequest.getTarget()); + // 下载文件 + minioClient.downloadObject( + DownloadObjectArgs.builder() + .bucket(this.bucket) + .object(parseFileName(downloadRequest.getFileLocation())) + .filename(downloadRequest.getTarget().getAbsolutePath()) + .build()); + } catch (Throwable t) { + ExceptionUtils.rethrow(t); + } + } + + /** + * 获取文件元 + * + * @param fileLocation 文件位置 + */ + @Override + public Optional fetchFileMeta(FileLocation fileLocation) { + try { + StatObjectResponse stat = minioClient.statObject(StatObjectArgs.builder() + .bucket(this.bucket) + .object(parseFileName(fileLocation)) + .build()); + return Optional.ofNullable(stat).map(minioStat -> { + + Map metaInfo = Maps.newHashMap(); + if (stat.userMetadata() != null) { + metaInfo.putAll(stat.userMetadata()); + } + return new FileMeta() + .setLastModifiedTime(Date.from(stat.lastModified().toInstant())) + .setLength(stat.size()) + .setMetaInfo(metaInfo); + }); + } catch (Exception oe) { + String errorCode = oe.getMessage(); + if (NO_SUCH_KEY.equalsIgnoreCase(errorCode)) { + return Optional.empty(); + } + ExceptionUtils.rethrow(oe); + } + return Optional.empty(); + } + + private static String parseFileName(FileLocation fileLocation) { + return String.format("%s/%s", fileLocation.getBucket(), fileLocation.getName()); + } + + /** + * 清理过期文件 + * + * @param bucket 桶名 + * @param days 日期 + */ + @Override + public void cleanExpiredFiles(String bucket, int days) { + /* + 使用Minio的管理界面或Minio客户端命令行工具设置对象的生命周期规则。在生命周期规则中定义文件的过期时间。Minio将自动根据设置的规则删除过期的文件。 + */ + } + + /** + * 释放连接 + */ + @Override + public void destroy() { + //minioClient.close(); + } + + /** + * 初始化minio + * + * @param applicationContext / + */ + @Override + protected void init(ApplicationContext applicationContext) { + Environment environment = applicationContext.getEnvironment(); + + String endpoint = fetchProperty(environment, TYPE_MINIO, KEY_ENDPOINT); + String bucketName = fetchProperty(environment, TYPE_MINIO, KEY_BUCKET_NAME); + String accessKey = fetchProperty(environment, TYPE_MINIO, ACCESS_KEY); + String secretKey = fetchProperty(environment, TYPE_MINIO, SECRET_KEY); + + try { + initOssClient(endpoint, bucketName, accessKey, secretKey); + } catch (Exception e) { + ExceptionUtils.rethrow(e); + } + } + + /** + * 创建minio连接并且创建桶 + * + * @param endpoint 端口 + * @param bucketName 桶名 + * @param accessKey 访问密钥 + * @param secretKey 秘密密钥 + */ + public void initOssClient(String endpoint, String bucketName, String accessKey, String secretKey) { + log.info("[Minio] init OSS by config: endpoint={}, bucketName={}, accessKey={}, secretKey={}", endpoint, bucketName, accessKey, secretKey); + if (StringUtils.isEmpty(bucketName)) { + throw new IllegalArgumentException("'oms.storage.dfs.minio.bucketName' can't be empty, please creat a bucket in minio oss console then config it to powerjob"); + } + this.bucket = bucketName; + minioClient = MinioClient.builder().endpoint(endpoint).credentials(accessKey, secretKey).build(); + createBucket(bucketName); + log.info("[Minio] initialize OSS successfully!"); + } + + /** + * 创建 bucket + * + * @param bucketName 桶名 + */ + @SneakyThrows(Exception.class) + public void createBucket(String bucketName) { + if (!bucketExists(bucketName)) { + minioClient.makeBucket(MakeBucketArgs.builder() + .bucket(bucketName).build()); + } + String policy = "{\n" + + " \"Version\": \"2012-10-17\",\n" + + " \"Statement\": [\n" + + " {\n" + + " \"Action\": [\n" + + " \"s3:GetObject\"\n" + + " ],\n" + + " \"Effect\": \"Allow\",\n" + + " \"Principal\": {\n" + + " \"AWS\": [\n" + + " \"*\"\n" + + " ]\n" + + " },\n" + + " \"Resource\": [\n" + + " \"arn:aws:s3:::" + bucketName + "/*\"\n" + + " ]\n" + + " }\n" + + " ]\n" + + "}"; + minioClient.setBucketPolicy(SetBucketPolicyArgs.builder().bucket(bucketName).config(policy).build()); + } + + /** + * 判断 bucket是否存在 + * + * @param bucketName: 桶名 + * @return boolean + */ + @SneakyThrows(Exception.class) + public boolean bucketExists(String bucketName) { + return minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build()); + } + + public static class MinioOssCondition extends PropertyAndOneBeanCondition { + + @Override + protected List anyConfigKey() { + return Lists.newArrayList("oms.storage.dfs.minio.endpoint"); + } + + @Override + protected Class beanType() { + return DFsService.class; + } + } +} From 360f105c0146016d7def5e97b59d880148384285 Mon Sep 17 00:00:00 2001 From: tjq Date: Thu, 31 Aug 2023 23:51:50 +0800 Subject: [PATCH 2/2] docs: add second qq group --- README_zhCN.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README_zhCN.md b/README_zhCN.md index c49ed0ee..41820938 100644 --- a/README_zhCN.md +++ b/README_zhCN.md @@ -69,4 +69,6 @@ PowerJob 的设计目标为企业级的分布式任务调度平台，即成为 * 欢迎共同参与本项目的贡献，PR和Issue都大大滴欢迎（求求了）～ * 觉得还不错的话，可以点个Star支持一下哦～ =￣ω￣= * 联系方式@KFCFans -> `tengjiqi@gmail.com` -* 用户交流QQ群（因广告信息泛滥，加群需要验证，请认真填写申请原因）：487453839 \ No newline at end of file +* 用户交流QQ群（因广告信息泛滥，加群需要验证，请认真填写申请原因）： + * 一群（已满）：487453839 + * 二群：834937813 \ No newline at end of file