From 1adc25308f82c4f41348b04f61c950685387e59f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E5=AE=B6=E4=BC=9F?= Date: Fri, 30 Jun 2023 15:22:32 +0800 Subject: [PATCH 01/24] =?UTF-8?q?=E8=8E=B7=E5=8F=96=E4=B8=BB=E6=9C=BA?= =?UTF-8?q?=E7=BD=91=E7=BB=9C=E8=BF=9E=E6=8E=A5=E4=BF=A1=E6=81=AF=E6=97=B6?= =?UTF-8?q?=E6=8C=89=E7=B4=A2=E5=BC=95=E4=BB=8E=E5=B0=8F=E5=88=B0=E5=A4=A7?= =?UTF-8?q?=E6=8E=92=E5=BA=8F=EF=BC=8C=E4=BC=98=E5=85=88=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E7=B4=A2=E5=BC=95=E5=80=BC=E5=B0=8F=E7=9A=84=E7=BD=91=E7=BB=9C?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/tech/powerjob/common/utils/NetUtils.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/powerjob-common/src/main/java/tech/powerjob/common/utils/NetUtils.java b/powerjob-common/src/main/java/tech/powerjob/common/utils/NetUtils.java index 1b84ab4b..9a0d88a3 100644 --- a/powerjob-common/src/main/java/tech/powerjob/common/utils/NetUtils.java +++ b/powerjob-common/src/main/java/tech/powerjob/common/utils/NetUtils.java @@ -155,6 +155,9 @@ public class NetUtils { log.warn("[Net] findNetworkInterface failed", e); } + // sort by interface index, the smaller is preferred. + validNetworkInterfaces.sort(Comparator.comparingInt(NetworkInterface::getIndex)); + // Try to find the preferred one for (NetworkInterface networkInterface : validNetworkInterfaces) { if (isPreferredNetworkInterface(networkInterface)) { From 8aaa6020820f403323fb7b67b0f4b5bf70b547ab Mon Sep 17 00:00:00 2001 From: archieself Date: Wed, 5 Jul 2023 10:49:37 +0800 Subject: [PATCH 02/24] fix-[#676] Set notifyUserIds to null when empty the notify info of a job. --- .../server/core/service/impl/job/JobServiceImpl.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/service/impl/job/JobServiceImpl.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/service/impl/job/JobServiceImpl.java index 29db1439..e76345b3 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/service/impl/job/JobServiceImpl.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/service/impl/job/JobServiceImpl.java @@ -89,8 +89,12 @@ public class JobServiceImpl implements JobService { fillDefaultValue(jobInfoDO); // 转化报警用户列表 - if (!CollectionUtils.isEmpty(request.getNotifyUserIds())) { - jobInfoDO.setNotifyUserIds(SJ.COMMA_JOINER.join(request.getNotifyUserIds())); + if (request.getNotifyUserIds() != null) { + if (request.getNotifyUserIds().size() == 0) { + jobInfoDO.setNotifyUserIds(null); + } else { + jobInfoDO.setNotifyUserIds(SJ.COMMA_JOINER.join(request.getNotifyUserIds())); + } } LifeCycle lifecycle = Optional.ofNullable(request.getLifeCycle()).orElse(LifeCycle.EMPTY_LIFE_CYCLE); jobInfoDO.setLifecycle(JSON.toJSONString(lifecycle)); From 67a22e8b7e714d44c3be414e5baf4293712928af Mon Sep 17 00:00:00 2001 From: tjq Date: Wed, 12 Jul 2023 20:45:38 +0800 Subject: [PATCH 03/24] feat: add log for ContainerService --- .../powerjob/common/exception/ImpossibleException.java | 10 ++++++++++ .../server/core/container/ContainerService.java | 10 +++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 powerjob-common/src/main/java/tech/powerjob/common/exception/ImpossibleException.java diff --git a/powerjob-common/src/main/java/tech/powerjob/common/exception/ImpossibleException.java b/powerjob-common/src/main/java/tech/powerjob/common/exception/ImpossibleException.java new file mode 100644 index 00000000..39803498 --- /dev/null +++ b/powerjob-common/src/main/java/tech/powerjob/common/exception/ImpossibleException.java @@ -0,0 +1,10 @@ +package tech.powerjob.common.exception; + +/** + * ImpossibleException + * + * @author tjq + * @since 2023/7/12 + */ +public class ImpossibleException extends RuntimeException { +} diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/container/ContainerService.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/container/ContainerService.java index 35de0310..59008698 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/container/ContainerService.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/container/ContainerService.java @@ -26,6 +26,7 @@ import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; import org.springframework.web.multipart.MultipartFile; import tech.powerjob.common.OmsConstant; +import tech.powerjob.common.exception.ImpossibleException; import tech.powerjob.common.model.DeployedContainerInfo; import tech.powerjob.common.model.GitRepoInfo; import tech.powerjob.common.request.ServerDeployContainerRequest; @@ -149,6 +150,8 @@ public class ContainerService { */ public String uploadContainerJarFile(MultipartFile file) throws IOException { + log.info("[ContainerService] start to uploadContainerJarFile, fileName={},size={}", file.getName(), file.getSize()); + String workerDirStr = OmsFileUtils.genTemporaryWorkPath(); String tmpFileStr = workerDirStr + "tmp.jar"; @@ -175,9 +178,14 @@ public class ContainerService { } FileUtils.moveFile(tmpFile, finalFile); + log.info("[ContainerService] uploadContainerJarFile successfully,md5={}", md5); return md5; - }finally { + } catch (Throwable t) { + log.error("[ContainerService] uploadContainerJarFile failed!", t); + ExceptionUtils.rethrow(t); + throw new ImpossibleException(); + } finally { CommonUtils.executeIgnoreException(() -> FileUtils.forceDelete(workerDir)); } } From 7318fed73aa3e52af41b8340cee01704daa06cf0 Mon Sep 17 00:00:00 2001 From: tjq Date: Sat, 15 Jul 2023 21:38:56 +0800 Subject: [PATCH 04/24] feat: support non-LAN communication(worker side) --- .../tech/powerjob/common/PowerJobDKey.java | 9 +++++- .../powerjob/common/utils/PropertyUtils.java | 32 +++++++++++++++++++ .../remote/framework/base/Address.java | 6 +++- .../remote/transporter/ProtocolInfo.java | 22 +++++++++++-- .../impl/PowerTransportService.java | 2 +- .../tech/powerjob/worker/PowerJobWorker.java | 16 +++++++--- 6 files changed, 77 insertions(+), 10 deletions(-) create mode 100644 powerjob-common/src/main/java/tech/powerjob/common/utils/PropertyUtils.java diff --git a/powerjob-common/src/main/java/tech/powerjob/common/PowerJobDKey.java b/powerjob-common/src/main/java/tech/powerjob/common/PowerJobDKey.java index aa27ea48..92c3a353 100644 --- a/powerjob-common/src/main/java/tech/powerjob/common/PowerJobDKey.java +++ b/powerjob-common/src/main/java/tech/powerjob/common/PowerJobDKey.java @@ -5,7 +5,6 @@ import java.net.NetworkInterface; /** * 通过 JVM 启动参数传入的配置信息 * - * * @author tjq * @since 2020/8/8 */ @@ -16,7 +15,15 @@ public class PowerJobDKey { */ public static final String PREFERRED_NETWORK_INTERFACE = "powerjob.network.interface.preferred"; + /** + * 绑定地址，一般填写本机网卡地址 + */ public static final String BIND_LOCAL_ADDRESS = "powerjob.network.local.address"; + /** + * 外部地址，可选，默认与绑定地址相同。当存在 NAT 等场景时可通过单独传递外部地址来实现通讯 + */ + public static final String NT_EXTERNAL_ADDRESS = "powerjob.network.external.address"; + public static final String NT_EXTERNAL_PORT = "powerjob.network.external.port"; /** * Java regular expressions for network interfaces that will be ignored. diff --git a/powerjob-common/src/main/java/tech/powerjob/common/utils/PropertyUtils.java b/powerjob-common/src/main/java/tech/powerjob/common/utils/PropertyUtils.java new file mode 100644 index 00000000..c1458586 --- /dev/null +++ b/powerjob-common/src/main/java/tech/powerjob/common/utils/PropertyUtils.java @@ -0,0 +1,32 @@ +package tech.powerjob.common.utils; + +import org.apache.commons.lang3.StringUtils; + +/** + * PropertyUtils + * + * @author tjq + * @since 2023/7/15 + */ +public class PropertyUtils { + + public static String readProperty(String key, String defaultValue) { + // 从启动参数读取 + String property = System.getProperty(key); + if (StringUtils.isNotEmpty(property)) { + return property; + } + + // 从 ENV 读取 + property= System.getenv(key); + if (StringUtils.isNotEmpty(property)) { + return property; + } + // 部分操作系统不兼容 a.b.c 的环境变量，转换为 a_b_c 再取一次，即 PowerJob 支持 2 种类型的环境变量 key + property = System.getenv(key.replaceAll("\\.", "_")); + if (StringUtils.isNotEmpty(property)) { + return property; + } + return defaultValue; + } +} diff --git a/powerjob-remote/powerjob-remote-framework/src/main/java/tech/powerjob/remote/framework/base/Address.java b/powerjob-remote/powerjob-remote-framework/src/main/java/tech/powerjob/remote/framework/base/Address.java index 5541f111..873b9eeb 100644 --- a/powerjob-remote/powerjob-remote-framework/src/main/java/tech/powerjob/remote/framework/base/Address.java +++ b/powerjob-remote/powerjob-remote-framework/src/main/java/tech/powerjob/remote/framework/base/Address.java @@ -20,7 +20,7 @@ public class Address implements Serializable { private int port; public String toFullAddress() { - return String.format("%s:%d", host, port); + return toFullAddress(host, port); } public static Address fromIpv4(String ipv4) { @@ -30,6 +30,10 @@ public class Address implements Serializable { .setPort(Integer.parseInt(split[1])); } + public static String toFullAddress(String host, int port) { + return String.format("%s:%d", host, port); + } + @Override public String toString() { return toFullAddress(); diff --git a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/transporter/ProtocolInfo.java b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/transporter/ProtocolInfo.java index 0802e811..c27d939c 100644 --- a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/transporter/ProtocolInfo.java +++ b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/transporter/ProtocolInfo.java @@ -3,6 +3,9 @@ package tech.powerjob.server.remote.transporter; import lombok.Getter; import lombok.Setter; import lombok.ToString; +import tech.powerjob.common.PowerJobDKey; +import tech.powerjob.common.utils.PropertyUtils; +import tech.powerjob.remote.framework.base.Address; import tech.powerjob.remote.framework.transporter.Transporter; /** @@ -20,11 +23,26 @@ public class ProtocolInfo { private String address; + /** + * 外部地址，当存在 NAT 等场景时，需要下发该地址到 worker + */ + private String externalAddress; + private transient Transporter transporter; - public ProtocolInfo(String protocol, String address, Transporter transporter) { + public ProtocolInfo(String protocol, String host, int port, Transporter transporter) { this.protocol = protocol; - this.address = address; this.transporter = transporter; + + this.address = Address.toFullAddress(host, port); + + // 处理外部地址 + String externalAddress = PropertyUtils.readProperty(PowerJobDKey.NT_EXTERNAL_ADDRESS, host); + + // 考虑到不同协议 port 理论上不一样，server 需要为每个单独的端口配置映射，规则为 powerjob.network.external.port.${协议}，比如 powerjob.network.external.port.http + String externalPortByProtocolKey = PowerJobDKey.NT_EXTERNAL_PORT.concat(".").concat(protocol.toLowerCase()); + // 大部分用户只使用一种协议，在此处做兼容处理降低答疑量和提高易用性（如果用户有多种协议，只有被转发的协议能成功通讯） + String externalPort = PropertyUtils.readProperty(externalPortByProtocolKey, PropertyUtils.readProperty(PowerJobDKey.NT_EXTERNAL_PORT, String.valueOf(port))); + this.externalAddress = Address.toFullAddress(externalAddress, Integer.parseInt(externalPort)); } } diff --git a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/transporter/impl/PowerTransportService.java b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/transporter/impl/PowerTransportService.java index d5d2f624..331a83ad 100644 --- a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/transporter/impl/PowerTransportService.java +++ b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/transporter/impl/PowerTransportService.java @@ -120,7 +120,7 @@ public class PowerTransportService implements TransportService, InitializingBean log.info("[PowerTransportService] start RemoteEngine[type={},address={}] successfully", protocol, address); this.engines.add(re); - this.protocolName2Info.put(protocol, new ProtocolInfo(protocol, address.toFullAddress(), engineOutput.getTransporter())); + this.protocolName2Info.put(protocol, new ProtocolInfo(protocol, address.getHost(), address.getPort(), engineOutput.getTransporter())); } @Override diff --git a/powerjob-worker/src/main/java/tech/powerjob/worker/PowerJobWorker.java b/powerjob-worker/src/main/java/tech/powerjob/worker/PowerJobWorker.java index 61ea7271..bbe682d2 100644 --- a/powerjob-worker/src/main/java/tech/powerjob/worker/PowerJobWorker.java +++ b/powerjob-worker/src/main/java/tech/powerjob/worker/PowerJobWorker.java @@ -3,12 +3,14 @@ package tech.powerjob.worker; import com.google.common.base.Stopwatch; import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; +import tech.powerjob.common.PowerJobDKey; import tech.powerjob.common.exception.PowerJobException; import tech.powerjob.common.response.ResultDTO; import tech.powerjob.common.serialize.JsonUtils; import tech.powerjob.common.utils.CommonUtils; import tech.powerjob.common.utils.HttpUtils; import tech.powerjob.common.utils.NetUtils; +import tech.powerjob.common.utils.PropertyUtils; import tech.powerjob.remote.framework.base.Address; import tech.powerjob.remote.framework.base.ServerType; import tech.powerjob.remote.framework.engine.EngineConfig; @@ -79,9 +81,13 @@ public class PowerJobWorker { log.warn("[PowerJobWorker] using TestMode now, it's dangerous if this is production env."); } - // 初始化元数据 - String workerAddress = NetUtils.getLocalHost() + ":" + config.getPort(); - workerRuntime.setWorkerAddress(workerAddress); + // 初始化网络数据，区别对待上报地址和本机绑定地址（对外统一使用上报地址） + String localBindIp = NetUtils.getLocalHost(); + int localBindPort = config.getPort(); + String externalIp = PropertyUtils.readProperty(PowerJobDKey.NT_EXTERNAL_ADDRESS, localBindIp); + String externalPort = PropertyUtils.readProperty(PowerJobDKey.NT_EXTERNAL_PORT, String.valueOf(localBindPort)); + log.info("[PowerJobWorker] [ADDRESS_INFO] localBindIp: {}, localBindPort: {}; externalIp: {}, externalPort: {}", localBindIp, localBindPort, externalIp, externalPort); + workerRuntime.setWorkerAddress(Address.toFullAddress(externalIp, Integer.parseInt(externalPort))); // 初始化 线程池 final ExecutorManager executorManager = new ExecutorManager(workerRuntime.getWorkerConfig()); @@ -100,7 +106,7 @@ public class PowerJobWorker { EngineConfig engineConfig = new EngineConfig() .setType(config.getProtocol().name()) .setServerType(ServerType.WORKER) - .setBindAddress(new Address().setHost(NetUtils.getLocalHost()).setPort(config.getPort())) + .setBindAddress(new Address().setHost(localBindIp).setPort(localBindPort)) .setActorList(Lists.newArrayList(taskTrackerActor, processorTrackerActor, workerActor)); EngineOutput engineOutput = remoteEngine.start(engineConfig); @@ -115,7 +121,7 @@ public class PowerJobWorker { log.info("[PowerJobWorker] PowerJobRemoteEngine initialized successfully."); // 初始化日志系统 - OmsLogHandler omsLogHandler = new OmsLogHandler(workerAddress, workerRuntime.getTransporter(), serverDiscoveryService); + OmsLogHandler omsLogHandler = new OmsLogHandler(workerRuntime.getWorkerAddress(), workerRuntime.getTransporter(), serverDiscoveryService); workerRuntime.setOmsLogHandler(omsLogHandler); // 初始化存储 From d3140d05017ce3c6fc7ddc597c113ba72e9b305a Mon Sep 17 00:00:00 2001 From: tjq Date: Sat, 15 Jul 2023 22:22:38 +0800 Subject: [PATCH 05/24] feat: support non-LAN communication(server side) --- .../election/ServerElectionService.java | 22 ++++++++++--------- .../remote/transporter/ProtocolInfo.java | 6 +++++ 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/election/ServerElectionService.java b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/election/ServerElectionService.java index b2a86791..0c384d6a 100644 --- a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/election/ServerElectionService.java +++ b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/election/ServerElectionService.java @@ -60,9 +60,11 @@ public class ServerElectionService { final String currentServer = request.getCurrentServer(); // 如果是本机，就不需要查数据库那么复杂的操作了，直接返回成功 Optional localProtocolInfoOpt = Optional.ofNullable(transportService.allProtocols().get(request.getProtocol())); - if (localProtocolInfoOpt.isPresent() && localProtocolInfoOpt.get().getAddress().equals(currentServer)) { - log.debug("[ServerElectionService] this server[{}] is worker's current server, skip check", currentServer); - return currentServer; + if (localProtocolInfoOpt.isPresent()) { + if (localProtocolInfoOpt.get().getExternalAddress().equals(currentServer) || localProtocolInfoOpt.get().getAddress().equals(currentServer)) { + log.info("[ServerElection] this server[{}] is worker[appId={}]'s current server, skip check", currentServer, request.getAppId()); + return currentServer; + } } } return getServer0(request); @@ -110,13 +112,13 @@ public class ServerElectionService { // 篡位，如果本机存在协议，则作为Server调度该 worker final ProtocolInfo targetProtocolInfo = transportService.allProtocols().get(protocol); if (targetProtocolInfo != null) { - // 注意，写入 AppInfoDO#currentServer 的永远是 default 的地址，仅在返回的时候特殊处理为协议地址 + // 注意，写入 AppInfoDO#currentServer 的永远是 default 的绑定地址，仅在返回的时候特殊处理为协议地址 appInfo.setCurrentServer(transportService.defaultProtocol().getAddress()); appInfo.setGmtModified(new Date()); appInfoRepository.saveAndFlush(appInfo); log.info("[ServerElection] this server({}) become the new server for app(appId={}).", appInfo.getCurrentServer(), appId); - return targetProtocolInfo.getAddress(); + return targetProtocolInfo.getExternalAddress(); } }catch (Exception e) { log.error("[ServerElection] write new server to db failed for app {}.", appName, e); @@ -129,10 +131,10 @@ public class ServerElectionService { /** * 判断指定server是否存活 - * @param serverAddress 需要检测的server地址 + * @param serverAddress 需要检测的server地址（绑定的内网地址） * @param downServerCache 缓存，防止多次发送PING（这个QPS其实还蛮爆表的...） * @param protocol 协议，用于返回指定的地址 - * @return null or address + * @return null or address（外部地址） */ private String activeAddress(String serverAddress, Set downServerCache, String protocol) { @@ -156,9 +158,9 @@ public class ServerElectionService { final JSONObject protocolInfo = JsonUtils.parseObject(response.getData(), JSONObject.class).getJSONObject(protocol); if (protocolInfo != null) { downServerCache.remove(serverAddress); - final String protocolAddress = protocolInfo.toJavaObject(ProtocolInfo.class).getAddress(); - log.info("[ServerElection] server[{}] is active, it will be the master, final protocol address={}", serverAddress, protocolAddress); - return protocolAddress; + ProtocolInfo remoteProtocol = protocolInfo.toJavaObject(ProtocolInfo.class); + log.info("[ServerElection] server[{}] is active, it will be the master, final protocol={}", serverAddress, remoteProtocol); + return remoteProtocol.getExternalAddress(); } else { log.warn("[ServerElection] server[{}] is active but don't have target protocol", serverAddress); } diff --git a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/transporter/ProtocolInfo.java b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/transporter/ProtocolInfo.java index c27d939c..1832a7e0 100644 --- a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/transporter/ProtocolInfo.java +++ b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/transporter/ProtocolInfo.java @@ -30,6 +30,12 @@ public class ProtocolInfo { private transient Transporter transporter; + /** + * 序列化需要，必须存在无参构造方法！严禁删除 + */ + public ProtocolInfo() { + } + public ProtocolInfo(String protocol, String host, int port, Transporter transporter) { this.protocol = protocol; this.transporter = transporter; From 5e9935fed439ad47a7865be1e9cae77a6b10457d Mon Sep 17 00:00:00 2001 From: tjq Date: Sun, 16 Jul 2023 17:55:01 +0800 Subject: [PATCH 06/24] feat: [storageExt] define DFsService --- .../server/extension/dfs/DFsService.java | 51 +++++++++++++++++++ .../server/extension/dfs/DownloadRequest.java | 20 ++++++++ .../server/extension/dfs/FileLocation.java | 23 +++++++++ .../server/extension/dfs/FileMeta.java | 26 ++++++++++ .../server/extension/dfs/StoreRequest.java | 20 ++++++++ 5 files changed, 140 insertions(+) create mode 100644 powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/DFsService.java create mode 100644 powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/DownloadRequest.java create mode 100644 powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/FileLocation.java create mode 100644 powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/FileMeta.java create mode 100644 powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/StoreRequest.java diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/DFsService.java b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/DFsService.java new file mode 100644 index 00000000..b51e9b52 --- /dev/null +++ b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/DFsService.java @@ -0,0 +1,51 @@ +package tech.powerjob.server.extension.dfs; + +import java.io.IOException; +import java.util.Optional; + +/** + * 分布式文件服务 + * + * @author tjq + * @since 2023/7/16 + */ +public interface DFsService { + + /** + * 存储文件 + * @param storeRequest 存储请求 + * @throws IOException 异常 + */ + void store(StoreRequest storeRequest) throws IOException; + + /** + * 下载文件 + * @param downloadRequest 文件下载请求 + * @throws IOException 异常 + */ + void download(DownloadRequest downloadRequest) throws IOException; + + /** + * 删除文件 + * @param fileLocation 文件位置 + * @throws IOException 异常 + */ + void delete(FileLocation fileLocation) throws IOException; + + /** + * 获取文件元信息 + * @param fileLocation 文件位置 + * @return 存在则返回文件元信息 + * @throws IOException 异常 + */ + Optional fetchFileMeta(FileLocation fileLocation) throws IOException; + + /** + * 清理 powerjob 认为“过期”的文件 + * 部分存储系统自带生命周期管理（如阿里云OSS，则不需要单独实现该方法） + * @param bucket bucket + * @param days 天数，需要清理超过 X 天的文件 + */ + default void cleanExpiredFiles(String bucket, int days) { + } +} diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/DownloadRequest.java b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/DownloadRequest.java new file mode 100644 index 00000000..70efd4fa --- /dev/null +++ b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/DownloadRequest.java @@ -0,0 +1,20 @@ +package tech.powerjob.server.extension.dfs; + +import lombok.Data; + +import java.io.File; +import java.io.Serializable; + +/** + * download request + * + * @author tjq + * @since 2023/7/16 + */ +@Data +public class DownloadRequest implements Serializable { + + private transient File target; + + private FileLocation fileLocation; +} diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/FileLocation.java b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/FileLocation.java new file mode 100644 index 00000000..d58a928e --- /dev/null +++ b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/FileLocation.java @@ -0,0 +1,23 @@ +package tech.powerjob.server.extension.dfs; + +import lombok.Data; + +/** + * 文件路径 + * + * @author tjq + * @since 2023/7/16 + */ +@Data +public class FileLocation { + + /** + * 存储桶 + */ + private String bucket; + + /** + * 名称 + */ + private String name; +} diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/FileMeta.java b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/FileMeta.java new file mode 100644 index 00000000..6e15155b --- /dev/null +++ b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/FileMeta.java @@ -0,0 +1,26 @@ +package tech.powerjob.server.extension.dfs; + +import lombok.Data; + +import java.util.Map; +import java.util.Objects; + +/** + * FileMeta + * + * @author tjq + * @since 2023/7/16 + */ +@Data +public class FileMeta { + + /** + * 文件大小 + */ + private final long length; + + /** + * 元数据 + */ + private Map metaInfo; +} diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/StoreRequest.java b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/StoreRequest.java new file mode 100644 index 00000000..b07581e3 --- /dev/null +++ b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/StoreRequest.java @@ -0,0 +1,20 @@ +package tech.powerjob.server.extension.dfs; + +import lombok.Data; + +import java.io.File; +import java.io.Serializable; + +/** + * StoreRequest + * + * @author tjq + * @since 2023/7/16 + */ +@Data +public class StoreRequest implements Serializable { + + private transient File localFile; + + private FileLocation fileLocation; +} From fc57226d3a937a52ee67ad95150aef8fe9116b5b Mon Sep 17 00:00:00 2001 From: tjq Date: Sun, 16 Jul 2023 18:14:20 +0800 Subject: [PATCH 07/24] refactor: optimize extension package --- .../server/core}/alarm/AlarmCenter.java | 20 +++++++--- .../alarm/impl/DingTalkAlarmService.java | 10 ++--- .../core}/alarm/impl/DingTalkUtils.java | 2 +- .../core}/alarm/impl/MailAlarmService.java | 12 +++--- .../core}/alarm/impl/WebHookAlarmService.java | 10 ++--- .../core}/alarm/module/JobInstanceAlarm.java | 3 +- .../alarm/module/WorkflowInstanceAlarm.java | 3 +- .../server/core/instance/InstanceManager.java | 4 +- .../core/lock}/DatabaseLockService.java | 2 +- .../workflow/WorkflowInstanceManager.java | 4 +- .../powerjob/server/extension/Alarmable.java | 17 --------- .../alarm/module => alarm}/Alarm.java | 2 +- .../server/extension/alarm/AlarmTarget.java | 37 +++++++++++++++++++ .../server/extension/alarm/Alarmable.java | 14 +++++++ .../filter}/DesignatedWorkerFilter.java | 2 +- .../filter}/DisconnectedWorkerFilter.java | 2 +- .../filter}/SystemMetricsWorkerFilter.java | 2 +- .../powerjob/server/test/DingTalkTest.java | 5 +-- 18 files changed, 97 insertions(+), 54 deletions(-) rename powerjob-server/{powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl => powerjob-server-core/src/main/java/tech/powerjob/server/core}/alarm/AlarmCenter.java (66%) rename powerjob-server/{powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl => powerjob-server-core/src/main/java/tech/powerjob/server/core}/alarm/impl/DingTalkAlarmService.java (92%) rename powerjob-server/{powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl => powerjob-server-core/src/main/java/tech/powerjob/server/core}/alarm/impl/DingTalkUtils.java (98%) rename powerjob-server/{powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl => powerjob-server-core/src/main/java/tech/powerjob/server/core}/alarm/impl/MailAlarmService.java (77%) rename powerjob-server/{powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl => powerjob-server-core/src/main/java/tech/powerjob/server/core}/alarm/impl/WebHookAlarmService.java (84%) rename powerjob-server/{powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl => powerjob-server-core/src/main/java/tech/powerjob/server/core}/alarm/module/JobInstanceAlarm.java (93%) rename powerjob-server/{powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl => powerjob-server-core/src/main/java/tech/powerjob/server/core}/alarm/module/WorkflowInstanceAlarm.java (92%) rename powerjob-server/{powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl => powerjob-server-core/src/main/java/tech/powerjob/server/core/lock}/DatabaseLockService.java (97%) delete mode 100644 powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/Alarmable.java rename powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/{defaultimpl/alarm/module => alarm}/Alarm.java (94%) create mode 100644 powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/alarm/AlarmTarget.java create mode 100644 powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/alarm/Alarmable.java rename powerjob-server/{powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/workerfilter => powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter}/DesignatedWorkerFilter.java (94%) rename powerjob-server/{powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/workerfilter => powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter}/DisconnectedWorkerFilter.java (92%) rename powerjob-server/{powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/workerfilter => powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter}/SystemMetricsWorkerFilter.java (93%) diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/alarm/AlarmCenter.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/AlarmCenter.java similarity index 66% rename from powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/alarm/AlarmCenter.java rename to powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/AlarmCenter.java index 118e5d41..cb837fdc 100644 --- a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/alarm/AlarmCenter.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/AlarmCenter.java @@ -1,9 +1,10 @@ -package tech.powerjob.server.extension.defaultimpl.alarm; +package tech.powerjob.server.core.alarm; -import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Component; -import tech.powerjob.server.extension.defaultimpl.alarm.module.Alarm; -import tech.powerjob.server.extension.Alarmable; +import tech.powerjob.server.extension.alarm.Alarm; +import tech.powerjob.server.extension.alarm.AlarmTarget; +import tech.powerjob.server.extension.alarm.Alarmable; import tech.powerjob.server.persistence.remote.model.UserInfoDO; import com.google.common.collect.Lists; import com.google.common.collect.Queues; @@ -12,6 +13,7 @@ import lombok.extern.slf4j.Slf4j; import java.util.List; import java.util.concurrent.*; +import java.util.stream.Collectors; /** * 报警服务 @@ -41,10 +43,18 @@ public class AlarmCenter { public void alarmFailed(Alarm alarm, List targetUserList) { POOL.execute(() -> BEANS.forEach(alarmable -> { try { - alarmable.onFailed(alarm, targetUserList); + alarmable.onFailed(alarm, targetUserList.stream().map(AlarmCenter::convertUserInfo2AlarmTarget).collect(Collectors.toList())); }catch (Exception e) { log.warn("[AlarmCenter] alarm failed.", e); } })); } + + private static AlarmTarget convertUserInfo2AlarmTarget(UserInfoDO userInfoDO) { + AlarmTarget alarmTarget = new AlarmTarget(); + BeanUtils.copyProperties(userInfoDO, alarmTarget); + + alarmTarget.setName(userInfoDO.getUsername()); + return alarmTarget; + } } diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/alarm/impl/DingTalkAlarmService.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/impl/DingTalkAlarmService.java similarity index 92% rename from powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/alarm/impl/DingTalkAlarmService.java rename to powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/impl/DingTalkAlarmService.java index 57eb4d35..029c3f56 100644 --- a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/alarm/impl/DingTalkAlarmService.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/impl/DingTalkAlarmService.java @@ -1,4 +1,4 @@ -package tech.powerjob.server.extension.defaultimpl.alarm.impl; +package tech.powerjob.server.core.alarm.impl; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; @@ -14,9 +14,9 @@ import tech.powerjob.common.exception.PowerJobException; import tech.powerjob.common.utils.NetUtils; import tech.powerjob.server.common.PowerJobServerConfigKey; import tech.powerjob.server.common.SJ; -import tech.powerjob.server.extension.Alarmable; -import tech.powerjob.server.extension.defaultimpl.alarm.module.Alarm; -import tech.powerjob.server.persistence.remote.model.UserInfoDO; +import tech.powerjob.server.extension.alarm.AlarmTarget; +import tech.powerjob.server.extension.alarm.Alarmable; +import tech.powerjob.server.extension.alarm.Alarm; import javax.annotation.PostConstruct; import java.util.List; @@ -46,7 +46,7 @@ public class DingTalkAlarmService implements Alarmable { private static final String EMPTY_TAG = "EMPTY"; @Override - public void onFailed(Alarm alarm, List targetUserList) { + public void onFailed(Alarm alarm, List targetUserList) { if (dingTalkUtils == null) { return; } diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/alarm/impl/DingTalkUtils.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/impl/DingTalkUtils.java similarity index 98% rename from powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/alarm/impl/DingTalkUtils.java rename to powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/impl/DingTalkUtils.java index e2cd77f0..a6002cb9 100644 --- a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/alarm/impl/DingTalkUtils.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/impl/DingTalkUtils.java @@ -1,4 +1,4 @@ -package tech.powerjob.server.extension.defaultimpl.alarm.impl; +package tech.powerjob.server.core.alarm.impl; import com.dingtalk.api.DefaultDingTalkClient; import com.dingtalk.api.DingTalkClient; diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/alarm/impl/MailAlarmService.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/impl/MailAlarmService.java similarity index 77% rename from powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/alarm/impl/MailAlarmService.java rename to powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/impl/MailAlarmService.java index d9438879..dbbf1a0d 100644 --- a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/alarm/impl/MailAlarmService.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/impl/MailAlarmService.java @@ -1,10 +1,10 @@ -package tech.powerjob.server.extension.defaultimpl.alarm.impl; +package tech.powerjob.server.core.alarm.impl; import org.springframework.beans.factory.annotation.Value; import org.apache.commons.lang3.StringUtils; -import tech.powerjob.server.persistence.remote.model.UserInfoDO; -import tech.powerjob.server.extension.defaultimpl.alarm.module.Alarm; -import tech.powerjob.server.extension.Alarmable; +import tech.powerjob.server.extension.alarm.AlarmTarget; +import tech.powerjob.server.extension.alarm.Alarm; +import tech.powerjob.server.extension.alarm.Alarmable; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; @@ -36,7 +36,7 @@ public class MailAlarmService implements Alarmable { private String from; @Override - public void onFailed(Alarm alarm, List targetUserList) { + public void onFailed(Alarm alarm, List targetUserList) { if (CollectionUtils.isEmpty(targetUserList) || javaMailSender == null || StringUtils.isEmpty(from)) { return; } @@ -44,7 +44,7 @@ public class MailAlarmService implements Alarmable { SimpleMailMessage sm = new SimpleMailMessage(); try { sm.setFrom(from); - sm.setTo(targetUserList.stream().map(UserInfoDO::getEmail).filter(Objects::nonNull).toArray(String[]::new)); + sm.setTo(targetUserList.stream().map(AlarmTarget::getEmail).filter(Objects::nonNull).toArray(String[]::new)); sm.setSubject(alarm.fetchTitle()); sm.setText(alarm.fetchContent()); diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/alarm/impl/WebHookAlarmService.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/impl/WebHookAlarmService.java similarity index 84% rename from powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/alarm/impl/WebHookAlarmService.java rename to powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/impl/WebHookAlarmService.java index 7c46dc2f..f8f550f4 100644 --- a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/alarm/impl/WebHookAlarmService.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/impl/WebHookAlarmService.java @@ -1,11 +1,11 @@ -package tech.powerjob.server.extension.defaultimpl.alarm.impl; +package tech.powerjob.server.core.alarm.impl; import com.alibaba.fastjson.JSONObject; import tech.powerjob.common.OmsConstant; import tech.powerjob.common.utils.HttpUtils; -import tech.powerjob.server.persistence.remote.model.UserInfoDO; -import tech.powerjob.server.extension.defaultimpl.alarm.module.Alarm; -import tech.powerjob.server.extension.Alarmable; +import tech.powerjob.server.extension.alarm.AlarmTarget; +import tech.powerjob.server.extension.alarm.Alarm; +import tech.powerjob.server.extension.alarm.Alarmable; import lombok.extern.slf4j.Slf4j; import okhttp3.MediaType; import okhttp3.RequestBody; @@ -29,7 +29,7 @@ public class WebHookAlarmService implements Alarmable { private static final String HTTPS_PROTOCOL_PREFIX = "https://"; @Override - public void onFailed(Alarm alarm, List targetUserList) { + public void onFailed(Alarm alarm, List targetUserList) { if (CollectionUtils.isEmpty(targetUserList)) { return; } diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/alarm/module/JobInstanceAlarm.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/module/JobInstanceAlarm.java similarity index 93% rename from powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/alarm/module/JobInstanceAlarm.java rename to powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/module/JobInstanceAlarm.java index 4924966e..5c2d907a 100644 --- a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/alarm/module/JobInstanceAlarm.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/module/JobInstanceAlarm.java @@ -1,6 +1,7 @@ -package tech.powerjob.server.extension.defaultimpl.alarm.module; +package tech.powerjob.server.core.alarm.module; import lombok.Data; +import tech.powerjob.server.extension.alarm.Alarm; /** * 任务执行失败告警对象 diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/alarm/module/WorkflowInstanceAlarm.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/module/WorkflowInstanceAlarm.java similarity index 92% rename from powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/alarm/module/WorkflowInstanceAlarm.java rename to powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/module/WorkflowInstanceAlarm.java index 9639dc54..bac2946b 100644 --- a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/alarm/module/WorkflowInstanceAlarm.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/module/WorkflowInstanceAlarm.java @@ -1,7 +1,8 @@ -package tech.powerjob.server.extension.defaultimpl.alarm.module; +package tech.powerjob.server.core.alarm.module; import tech.powerjob.common.model.PEWorkflowDAG; import lombok.Data; +import tech.powerjob.server.extension.alarm.Alarm; /** * 工作流执行失败告警对象 diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceManager.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceManager.java index 07f02e94..274a8a5b 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceManager.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceManager.java @@ -17,8 +17,8 @@ import tech.powerjob.server.common.timewheel.holder.HashedWheelTimerHolder; import tech.powerjob.server.common.utils.SpringUtils; import tech.powerjob.server.core.service.UserService; import tech.powerjob.server.core.workflow.WorkflowInstanceManager; -import tech.powerjob.server.extension.defaultimpl.alarm.AlarmCenter; -import tech.powerjob.server.extension.defaultimpl.alarm.module.JobInstanceAlarm; +import tech.powerjob.server.core.alarm.AlarmCenter; +import tech.powerjob.server.core.alarm.module.JobInstanceAlarm; import tech.powerjob.server.persistence.remote.model.InstanceInfoDO; import tech.powerjob.server.persistence.remote.model.JobInfoDO; import tech.powerjob.server.persistence.remote.model.UserInfoDO; diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/DatabaseLockService.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/lock/DatabaseLockService.java similarity index 97% rename from powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/DatabaseLockService.java rename to powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/lock/DatabaseLockService.java index 15a3d91e..e7b93913 100644 --- a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/DatabaseLockService.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/lock/DatabaseLockService.java @@ -1,4 +1,4 @@ -package tech.powerjob.server.extension.defaultimpl; +package tech.powerjob.server.core.lock; import tech.powerjob.common.utils.CommonUtils; import tech.powerjob.common.utils.NetUtils; diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/workflow/WorkflowInstanceManager.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/workflow/WorkflowInstanceManager.java index 68535a32..001c3e12 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/workflow/WorkflowInstanceManager.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/workflow/WorkflowInstanceManager.java @@ -26,8 +26,8 @@ import tech.powerjob.server.core.service.UserService; import tech.powerjob.server.core.service.WorkflowNodeHandleService; import tech.powerjob.server.core.uid.IdGenerateService; import tech.powerjob.server.core.workflow.algorithm.WorkflowDAGUtils; -import tech.powerjob.server.extension.defaultimpl.alarm.AlarmCenter; -import tech.powerjob.server.extension.defaultimpl.alarm.module.WorkflowInstanceAlarm; +import tech.powerjob.server.core.alarm.AlarmCenter; +import tech.powerjob.server.core.alarm.module.WorkflowInstanceAlarm; import tech.powerjob.server.persistence.remote.model.*; import tech.powerjob.server.persistence.remote.repository.JobInfoRepository; import tech.powerjob.server.persistence.remote.repository.WorkflowInfoRepository; diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/Alarmable.java b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/Alarmable.java deleted file mode 100644 index 0cec4398..00000000 --- a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/Alarmable.java +++ /dev/null @@ -1,17 +0,0 @@ -package tech.powerjob.server.extension; - -import tech.powerjob.server.persistence.remote.model.UserInfoDO; -import tech.powerjob.server.extension.defaultimpl.alarm.module.Alarm; - -import java.util.List; - -/** - * 报警接口 - * - * @author tjq - * @since 2020/4/19 - */ -public interface Alarmable { - - void onFailed(Alarm alarm, List targetUserList); -} diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/alarm/module/Alarm.java b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/alarm/Alarm.java similarity index 94% rename from powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/alarm/module/Alarm.java rename to powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/alarm/Alarm.java index 76708113..6d76986b 100644 --- a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/alarm/module/Alarm.java +++ b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/alarm/Alarm.java @@ -1,4 +1,4 @@ -package tech.powerjob.server.extension.defaultimpl.alarm.module; +package tech.powerjob.server.extension.alarm; import com.alibaba.fastjson.JSONObject; import tech.powerjob.common.OmsConstant; diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/alarm/AlarmTarget.java b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/alarm/AlarmTarget.java new file mode 100644 index 00000000..6512f6f7 --- /dev/null +++ b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/alarm/AlarmTarget.java @@ -0,0 +1,37 @@ +package tech.powerjob.server.extension.alarm; + +import lombok.Data; + +import java.io.Serializable; +import java.util.Map; +import java.util.Objects; + +/** + * 报警目标 + * + * @author tjq + * @since 2023/7/16 + */ +@Data +public class AlarmTarget implements Serializable { + + private String name; + /** + * 手机号 + */ + private String phone; + /** + * 邮箱地址 + */ + private String email; + /** + * webHook + */ + private String webHook; + /** + * 扩展字段 + */ + private String extra; + + private Map attributes; +} diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/alarm/Alarmable.java b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/alarm/Alarmable.java new file mode 100644 index 00000000..7ede922e --- /dev/null +++ b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/alarm/Alarmable.java @@ -0,0 +1,14 @@ +package tech.powerjob.server.extension.alarm; + +import java.util.List; + +/** + * 报警接口 + * + * @author tjq + * @since 2020/4/19 + */ +public interface Alarmable { + + void onFailed(Alarm alarm, List alarmTargets); +} diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/workerfilter/DesignatedWorkerFilter.java b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/DesignatedWorkerFilter.java similarity index 94% rename from powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/workerfilter/DesignatedWorkerFilter.java rename to powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/DesignatedWorkerFilter.java index 0cb98396..26656ae6 100644 --- a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/workerfilter/DesignatedWorkerFilter.java +++ b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/DesignatedWorkerFilter.java @@ -1,4 +1,4 @@ -package tech.powerjob.server.extension.defaultimpl.workerfilter; +package tech.powerjob.server.remote.worker.filter; import com.google.common.collect.Sets; import lombok.extern.slf4j.Slf4j; diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/workerfilter/DisconnectedWorkerFilter.java b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/DisconnectedWorkerFilter.java similarity index 92% rename from powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/workerfilter/DisconnectedWorkerFilter.java rename to powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/DisconnectedWorkerFilter.java index de87a910..ff8482d9 100644 --- a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/workerfilter/DisconnectedWorkerFilter.java +++ b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/DisconnectedWorkerFilter.java @@ -1,4 +1,4 @@ -package tech.powerjob.server.extension.defaultimpl.workerfilter; +package tech.powerjob.server.remote.worker.filter; import tech.powerjob.server.extension.WorkerFilter; import tech.powerjob.server.persistence.remote.model.JobInfoDO; diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/workerfilter/SystemMetricsWorkerFilter.java b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/SystemMetricsWorkerFilter.java similarity index 93% rename from powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/workerfilter/SystemMetricsWorkerFilter.java rename to powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/SystemMetricsWorkerFilter.java index 0402604d..f88bbdd5 100644 --- a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/defaultimpl/workerfilter/SystemMetricsWorkerFilter.java +++ b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/SystemMetricsWorkerFilter.java @@ -1,4 +1,4 @@ -package tech.powerjob.server.extension.defaultimpl.workerfilter; +package tech.powerjob.server.remote.worker.filter; import tech.powerjob.common.model.SystemMetrics; import tech.powerjob.server.extension.WorkerFilter; diff --git a/powerjob-server/powerjob-server-starter/src/test/java/tech/powerjob/server/test/DingTalkTest.java b/powerjob-server/powerjob-server-starter/src/test/java/tech/powerjob/server/test/DingTalkTest.java index 9195ff1c..2b7883a2 100644 --- a/powerjob-server/powerjob-server-starter/src/test/java/tech/powerjob/server/test/DingTalkTest.java +++ b/powerjob-server/powerjob-server-starter/src/test/java/tech/powerjob/server/test/DingTalkTest.java @@ -1,12 +1,9 @@ package tech.powerjob.server.test; import org.junit.jupiter.api.Disabled; -import tech.powerjob.server.extension.defaultimpl.alarm.impl.DingTalkUtils; -import com.google.common.collect.Lists; +import tech.powerjob.server.core.alarm.impl.DingTalkUtils; import org.junit.jupiter.api.Test; -import java.util.List; - /** * 测试钉钉消息工具 * From 236d0a7f3b22e46094bdbbfb8024de1552f66ba7 Mon Sep 17 00:00:00 2001 From: tjq Date: Sun, 30 Jul 2023 12:00:45 +0800 Subject: [PATCH 08/24] feat: [storageExt] Unified File Storage Solution with DFsService --- powerjob-server/pom.xml | 14 +- .../core/container/ContainerService.java | 51 ++++-- .../core/instance/InstanceLogService.java | 36 ++--- .../server/core/scheduler/CleanService.java | 27 ++-- .../powerjob-server-extension/pom.xml | 4 - .../server/extension/dfs/DFsService.java | 7 - .../server/extension/dfs/DownloadRequest.java | 2 + .../server/extension/dfs/FileLocation.java | 2 + .../server/extension/dfs/FileMeta.java | 7 +- .../server/extension/dfs/StoreRequest.java | 2 + .../powerjob-server-persistence/pom.xml | 9 ++ .../persistence/mongodb/GridFsManager.java | 153 ------------------ .../storage/AbstractDFsService.java | 37 +++++ .../server/persistence/storage/Constants.java | 15 ++ .../storage/impl/AliOssService.java | 45 ++++++ .../storage/impl/EmptyDFsService.java | 38 +++++ .../storage/impl/GridFsService.java | 141 ++++++++++++++++ .../worker/WorkerClusterQueryService.java | 3 +- .../worker/filter/DesignatedWorkerFilter.java | 1 - .../filter/DisconnectedWorkerFilter.java | 1 - .../filter/SystemMetricsWorkerFilter.java | 1 - .../remote/worker/filter}/WorkerFilter.java | 2 +- .../tech/powerjob/server/test/GridFsTest.java | 61 ------- .../tech/powerjob/server/test/OmsLogTest.java | 56 ------- 24 files changed, 372 insertions(+), 343 deletions(-) delete mode 100644 powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/mongodb/GridFsManager.java create mode 100644 powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/AbstractDFsService.java create mode 100644 powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/Constants.java create mode 100644 powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/AliOssService.java create mode 100644 powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/EmptyDFsService.java create mode 100644 powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java rename powerjob-server/{powerjob-server-extension/src/main/java/tech/powerjob/server/extension => powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter}/WorkerFilter.java (91%) delete mode 100644 powerjob-server/powerjob-server-starter/src/test/java/tech/powerjob/server/test/GridFsTest.java delete mode 100644 powerjob-server/powerjob-server-starter/src/test/java/tech/powerjob/server/test/OmsLogTest.java diff --git a/powerjob-server/pom.xml b/powerjob-server/pom.xml index df3bd5fe..0aac969f 100644 --- a/powerjob-server/pom.xml +++ b/powerjob-server/pom.xml @@ -35,6 +35,7 @@ 11.5.0.0 42.2.14 2.1.214 + 4.10.2 2.11.2 5.7.0.202003110725-r @@ -97,6 +98,14 @@ powerjob-server-starter ${project.version} + + + + org.mongodb + mongodb-driver-sync + ${mongodb-driver-sync.version} + + @@ -188,11 +197,6 @@ spring-boot-starter-data-jpa ${springboot.version} - - org.springframework.boot - spring-boot-starter-data-mongodb - ${springboot.version} - org.springframework.boot spring-boot-starter-actuator diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/container/ContainerService.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/container/ContainerService.java index 59008698..acb2fd0b 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/container/ContainerService.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/container/ContainerService.java @@ -41,9 +41,10 @@ import tech.powerjob.server.common.constants.SwitchableStatus; import tech.powerjob.server.common.module.WorkerInfo; import tech.powerjob.server.common.utils.OmsFileUtils; import tech.powerjob.server.extension.LockService; -import tech.powerjob.server.persistence.mongodb.GridFsManager; +import tech.powerjob.server.extension.dfs.*; import tech.powerjob.server.persistence.remote.model.ContainerInfoDO; import tech.powerjob.server.persistence.remote.repository.ContainerInfoRepository; +import tech.powerjob.server.persistence.storage.Constants; import tech.powerjob.server.remote.server.redirector.DesignateServer; import tech.powerjob.server.remote.transporter.impl.ServerURLFactory; import tech.powerjob.server.remote.transporter.TransportService; @@ -75,7 +76,7 @@ public class ContainerService { @Resource private ContainerInfoRepository containerInfoRepository; @Resource - private GridFsManager gridFsManager; + private DFsService dFsService; @Resource private TransportService transportService; @@ -167,8 +168,10 @@ public class ContainerService { String md5 = OmsFileUtils.md5(tmpFile); String fileName = genContainerJarName(md5); - // 上传到 mongoDB，这兄弟耗时也有点小严重，导致这个接口整体比较慢...不过也没必要开线程去处理 - gridFsManager.store(tmpFile, GridFsManager.CONTAINER_BUCKET, fileName); + // 上传到 DFS，这兄弟耗时也有点小严重，导致这个接口整体比较慢...不过也没必要开线程去处理 + FileLocation fl = new FileLocation().setBucket(Constants.CONTAINER_BUCKET).setName(fileName); + StoreRequest storeRequest = new StoreRequest().setLocalFile(tmpFile).setFileLocation(fl); + dFsService.store(storeRequest); // 将文件拷贝到正确的路径 String finalFileStr = OmsFileUtils.genContainerJarPath() + fileName; @@ -204,9 +207,17 @@ public class ContainerService { if (localFile.exists()) { return localFile; } - if (gridFsManager.available()) { - downloadJarFromGridFS(fileName, localFile); + + FileLocation fileLocation = new FileLocation().setBucket(Constants.CONTAINER_BUCKET).setName(fileName); + try { + Optional fileMetaOpt = dFsService.fetchFileMeta(fileLocation); + if (fileMetaOpt.isPresent()) { + dFsService.download(new DownloadRequest().setFileLocation(fileLocation).setTarget(localFile)); + } + } catch (Exception e) { + log.warn("[ContainerService] fetchContainerJarFile from dsf failed, version: {}", version, e); } + return localFile; } @@ -412,12 +423,14 @@ public class ContainerService { String jarFileName = genContainerJarName(container.getVersion()); - if (!gridFsManager.exists(GridFsManager.CONTAINER_BUCKET, jarFileName)) { - remote.sendText("SYSTEM: can't find the jar resource in remote, maybe this is a new version, start to upload new version."); - gridFsManager.store(jarWithDependency, GridFsManager.CONTAINER_BUCKET, jarFileName); - remote.sendText("SYSTEM: upload to GridFS successfully~"); - }else { + FileLocation dfsFL = new FileLocation().setBucket(Constants.CONTAINER_BUCKET).setName(jarFileName); + Optional dfsMetaOpt = dFsService.fetchFileMeta(dfsFL); + if (dfsMetaOpt.isPresent()) { remote.sendText("SYSTEM: find the jar resource in remote successfully, so it's no need to upload anymore."); + } else { + remote.sendText("SYSTEM: can't find the jar resource in remote, maybe this is a new version, start to upload new version."); + dFsService.store(new StoreRequest().setFileLocation(dfsFL).setLocalFile(jarWithDependency)); + remote.sendText("SYSTEM: upload to GridFS successfully~"); } // 将文件从临时工作目录移动到正式目录 @@ -463,13 +476,19 @@ public class ContainerService { if (targetFile.exists()) { return; } - if (!gridFsManager.exists(GridFsManager.CONTAINER_BUCKET, mongoFileName)) { - log.warn("[ContainerService] can't find container's jar file({}) in gridFS.", mongoFileName); - return; - } + try { + + FileLocation dfsFL = new FileLocation().setBucket(Constants.CONTAINER_BUCKET).setName(mongoFileName); + Optional dfsMetaOpt = dFsService.fetchFileMeta(dfsFL); + if (!dfsMetaOpt.isPresent()) { + log.warn("[ContainerService] can't find container's jar file({}) in gridFS.", mongoFileName); + return; + } + FileUtils.forceMkdirParent(targetFile); - gridFsManager.download(targetFile, GridFsManager.CONTAINER_BUCKET, mongoFileName); + + dFsService.download(new DownloadRequest().setTarget(targetFile).setFileLocation(dfsFL)); }catch (Exception e) { CommonUtils.executeIgnoreException(() -> FileUtils.forceDelete(targetFile)); ExceptionUtils.rethrow(e); diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceLogService.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceLogService.java index 5adfe8d7..a12f05d0 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceLogService.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceLogService.java @@ -10,17 +10,17 @@ import tech.powerjob.common.utils.CommonUtils; import tech.powerjob.common.utils.NetUtils; import tech.powerjob.common.utils.SegmentLock; import tech.powerjob.server.common.constants.PJThreadPool; +import tech.powerjob.server.extension.dfs.*; +import tech.powerjob.server.persistence.storage.Constants; import tech.powerjob.server.remote.server.redirector.DesignateServer; import tech.powerjob.server.common.utils.OmsFileUtils; import tech.powerjob.server.persistence.StringPage; import tech.powerjob.server.persistence.remote.model.JobInfoDO; import tech.powerjob.server.persistence.local.LocalInstanceLogDO; import tech.powerjob.server.persistence.local.LocalInstanceLogRepository; -import tech.powerjob.server.persistence.mongodb.GridFsManager; import com.google.common.base.Stopwatch; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.collect.Queues; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.exception.ExceptionUtils; @@ -37,6 +37,7 @@ import javax.annotation.Resource; import java.io.*; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.*; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -58,7 +59,7 @@ public class InstanceLogService { private InstanceMetadataService instanceMetadataService; @Resource - private GridFsManager gridFsManager; + private DFsService dFsService; /** * 本地数据库操作bean */ @@ -214,14 +215,16 @@ public class InstanceLogService { // 先持久化到本地文件 File stableLogFile = genStableLogFile(instanceId); // 将文件推送到 MongoDB - if (gridFsManager.available()) { - try { - gridFsManager.store(stableLogFile, GridFsManager.LOG_BUCKET, genMongoFileName(instanceId)); - log.info("[InstanceLog-{}] push local instanceLogs to mongoDB succeed, using: {}.", instanceId, sw.stop()); - }catch (Exception e) { - log.warn("[InstanceLog-{}] push local instanceLogs to mongoDB failed.", instanceId, e); - } + + FileLocation dfsFL = new FileLocation().setBucket(Constants.LOG_BUCKET).setName(genMongoFileName(instanceId)); + + try { + dFsService.store(new StoreRequest().setLocalFile(stableLogFile).setFileLocation(dfsFL)); + log.info("[InstanceLog-{}] push local instanceLogs to mongoDB succeed, using: {}.", instanceId, sw.stop()); + }catch (Exception e) { + log.warn("[InstanceLog-{}] push local instanceLogs to mongoDB failed.", instanceId, e); } + }catch (Exception e) { log.warn("[InstanceLog-{}] sync local instanceLogs failed.", instanceId, e); } @@ -291,17 +294,14 @@ public class InstanceLogService { } }else { - if (!gridFsManager.available()) { - OmsFileUtils.string2File("SYSTEM: There is no local log for this task now, you need to use mongoDB to store the past logs.", f); - return f; - } - - // 否则从 mongoDB 拉取数据（对应后期查询的情况） - if (!gridFsManager.exists(GridFsManager.LOG_BUCKET, genMongoFileName(instanceId))) { + FileLocation dfl = new FileLocation().setBucket(Constants.LOG_BUCKET).setName(genMongoFileName(instanceId)); + Optional dflMetaOpt = dFsService.fetchFileMeta(dfl); + if (!dflMetaOpt.isPresent()) { OmsFileUtils.string2File("SYSTEM: There is no online log for this job instance.", f); return f; } - gridFsManager.download(f, GridFsManager.LOG_BUCKET, genMongoFileName(instanceId)); + + dFsService.download(new DownloadRequest().setTarget(f).setFileLocation(dfl)); } return f; }catch (Exception e) { diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/scheduler/CleanService.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/scheduler/CleanService.java index 06037106..9900fb94 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/scheduler/CleanService.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/scheduler/CleanService.java @@ -13,10 +13,11 @@ import tech.powerjob.common.enums.WorkflowInstanceStatus; import tech.powerjob.server.common.constants.PJThreadPool; import tech.powerjob.server.common.utils.OmsFileUtils; import tech.powerjob.server.extension.LockService; -import tech.powerjob.server.persistence.mongodb.GridFsManager; +import tech.powerjob.server.extension.dfs.DFsService; import tech.powerjob.server.persistence.remote.repository.InstanceInfoRepository; import tech.powerjob.server.persistence.remote.repository.WorkflowInstanceInfoRepository; import tech.powerjob.server.persistence.remote.repository.WorkflowNodeInfoRepository; +import tech.powerjob.server.persistence.storage.Constants; import tech.powerjob.server.remote.worker.WorkerClusterManagerService; import java.io.File; @@ -32,7 +33,7 @@ import java.util.Date; @Service public class CleanService { - private final GridFsManager gridFsManager; + private final DFsService dFsService; private final InstanceInfoRepository instanceInfoRepository; @@ -57,12 +58,12 @@ public class CleanService { private static final String HISTORY_DELETE_LOCK = "history_delete_lock"; - public CleanService(GridFsManager gridFsManager, InstanceInfoRepository instanceInfoRepository, WorkflowInstanceInfoRepository workflowInstanceInfoRepository, + public CleanService(DFsService dFsService, InstanceInfoRepository instanceInfoRepository, WorkflowInstanceInfoRepository workflowInstanceInfoRepository, WorkflowNodeInfoRepository workflowNodeInfoRepository, LockService lockService, @Value("${oms.instanceinfo.retention}") int instanceInfoRetentionDay, @Value("${oms.container.retention.local}") int localContainerRetentionDay, @Value("${oms.container.retention.remote}") int remoteContainerRetentionDay) { - this.gridFsManager = gridFsManager; + this.dFsService = dFsService; this.instanceInfoRepository = instanceInfoRepository; this.workflowInstanceInfoRepository = workflowInstanceInfoRepository; this.workflowNodeInfoRepository = workflowNodeInfoRepository; @@ -106,8 +107,8 @@ public class CleanService { // 删除无用节点 cleanWorkflowNodeInfo(); // 删除 GridFS 过期文件 - cleanRemote(GridFsManager.LOG_BUCKET, instanceInfoRetentionDay); - cleanRemote(GridFsManager.CONTAINER_BUCKET, remoteContainerRetentionDay); + cleanRemote(Constants.LOG_BUCKET, instanceInfoRetentionDay); + cleanRemote(Constants.CONTAINER_BUCKET, remoteContainerRetentionDay); } finally { lockService.unlock(HISTORY_DELETE_LOCK); } @@ -152,15 +153,13 @@ public class CleanService { log.info("[CleanService] won't clean up bucket({}) because of offset day <= 0.", bucketName); return; } - if (gridFsManager.available()) { - Stopwatch stopwatch = Stopwatch.createStarted(); - try { - gridFsManager.deleteBefore(bucketName, day); - }catch (Exception e) { - log.warn("[CleanService] clean remote bucket({}) failed.", bucketName, e); - } - log.info("[CleanService] clean remote bucket({}) successfully, using {}.", bucketName, stopwatch.stop()); + Stopwatch stopwatch = Stopwatch.createStarted(); + try { + dFsService.cleanExpiredFiles(bucketName, day); + }catch (Exception e) { + log.warn("[CleanService] clean remote bucket({}) failed.", bucketName, e); } + log.info("[CleanService] clean remote bucket({}) successfully, using {}.", bucketName, stopwatch.stop()); } @VisibleForTesting diff --git a/powerjob-server/powerjob-server-extension/pom.xml b/powerjob-server/powerjob-server-extension/pom.xml index aa65f78d..89954a8b 100644 --- a/powerjob-server/powerjob-server-extension/pom.xml +++ b/powerjob-server/powerjob-server-extension/pom.xml @@ -19,10 +19,6 @@ - - tech.powerjob - powerjob-server-persistence - \ No newline at end of file diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/DFsService.java b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/DFsService.java index b51e9b52..fdb77b6b 100644 --- a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/DFsService.java +++ b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/DFsService.java @@ -25,13 +25,6 @@ public interface DFsService { */ void download(DownloadRequest downloadRequest) throws IOException; - /** - * 删除文件 - * @param fileLocation 文件位置 - * @throws IOException 异常 - */ - void delete(FileLocation fileLocation) throws IOException; - /** * 获取文件元信息 * @param fileLocation 文件位置 diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/DownloadRequest.java b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/DownloadRequest.java index 70efd4fa..e3288f36 100644 --- a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/DownloadRequest.java +++ b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/DownloadRequest.java @@ -1,6 +1,7 @@ package tech.powerjob.server.extension.dfs; import lombok.Data; +import lombok.experimental.Accessors; import java.io.File; import java.io.Serializable; @@ -12,6 +13,7 @@ import java.io.Serializable; * @since 2023/7/16 */ @Data +@Accessors(chain = true) public class DownloadRequest implements Serializable { private transient File target; diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/FileLocation.java b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/FileLocation.java index d58a928e..13132876 100644 --- a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/FileLocation.java +++ b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/FileLocation.java @@ -1,6 +1,7 @@ package tech.powerjob.server.extension.dfs; import lombok.Data; +import lombok.experimental.Accessors; /** * 文件路径 @@ -9,6 +10,7 @@ import lombok.Data; * @since 2023/7/16 */ @Data +@Accessors(chain = true) public class FileLocation { /** diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/FileMeta.java b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/FileMeta.java index 6e15155b..9fb7656a 100644 --- a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/FileMeta.java +++ b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/FileMeta.java @@ -1,9 +1,9 @@ package tech.powerjob.server.extension.dfs; import lombok.Data; +import lombok.experimental.Accessors; import java.util.Map; -import java.util.Objects; /** * FileMeta @@ -12,15 +12,16 @@ import java.util.Objects; * @since 2023/7/16 */ @Data +@Accessors(chain = true) public class FileMeta { /** * 文件大小 */ - private final long length; + private long length; /** * 元数据 */ - private Map metaInfo; + private Map metaInfo; } diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/StoreRequest.java b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/StoreRequest.java index b07581e3..1933343c 100644 --- a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/StoreRequest.java +++ b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/StoreRequest.java @@ -1,6 +1,7 @@ package tech.powerjob.server.extension.dfs; import lombok.Data; +import lombok.experimental.Accessors; import java.io.File; import java.io.Serializable; @@ -12,6 +13,7 @@ import java.io.Serializable; * @since 2023/7/16 */ @Data +@Accessors(chain = true) public class StoreRequest implements Serializable { private transient File localFile; diff --git a/powerjob-server/powerjob-server-persistence/pom.xml b/powerjob-server/powerjob-server-persistence/pom.xml index c5494ae9..a1d526b2 100644 --- a/powerjob-server/powerjob-server-persistence/pom.xml +++ b/powerjob-server/powerjob-server-persistence/pom.xml @@ -23,10 +23,19 @@ tech.powerjob powerjob-server-common + + tech.powerjob + powerjob-server-extension + tech.powerjob powerjob-server-monitor + + + org.mongodb + mongodb-driver-sync + \ No newline at end of file diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/mongodb/GridFsManager.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/mongodb/GridFsManager.java deleted file mode 100644 index d275a0fd..00000000 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/mongodb/GridFsManager.java +++ /dev/null @@ -1,153 +0,0 @@ -package tech.powerjob.server.persistence.mongodb; - -import com.google.common.base.Stopwatch; -import com.google.common.collect.Maps; -import com.mongodb.client.MongoDatabase; -import com.mongodb.client.gridfs.GridFSBucket; -import com.mongodb.client.gridfs.GridFSBuckets; -import com.mongodb.client.gridfs.GridFSDownloadStream; -import com.mongodb.client.gridfs.GridFSFindIterable; -import com.mongodb.client.gridfs.model.GridFSFile; -import com.mongodb.client.model.Filters; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.time.DateUtils; -import org.bson.conversions.Bson; -import org.bson.types.ObjectId; -import org.springframework.beans.factory.InitializingBean; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.env.Environment; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.stereotype.Service; -import tech.powerjob.server.common.PowerJobServerConfigKey; - -import java.io.*; -import java.util.Date; -import java.util.Map; -import java.util.function.Consumer; - -/** - * GridFS 操作助手 - * - * @author tjq - * @since 2020/5/18 - */ -@Slf4j -@Service -public class GridFsManager implements InitializingBean { - - private final Environment environment; - - private final MongoDatabase db; - - private boolean available; - - private final Map bucketCache = Maps.newConcurrentMap(); - - public static final String LOG_BUCKET = "log"; - - public static final String CONTAINER_BUCKET = "container"; - - public GridFsManager(Environment environment, @Autowired(required = false) MongoTemplate mongoTemplate) { - this.environment = environment; - if (mongoTemplate != null) { - this.db = mongoTemplate.getDb(); - } else { - this.db = null; - } - } - - /** - * 是否可用 - * @return true：可用；false：不可用 - */ - public boolean available() { - return available; - } - - /** - * 存储文件到 GridFS - * @param localFile 本地文件 - * @param bucketName 桶名称 - * @param fileName GirdFS中的文件名称 - * @throws IOException 异常 - */ - public void store(File localFile, String bucketName, String fileName) throws IOException { - if (available()) { - GridFSBucket bucket = getBucket(bucketName); - try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(localFile))) { - bucket.uploadFromStream(fileName, bis); - } - } - } - - /** - * 从 GridFS 下载文件 - * @param targetFile 下载的目标文件（本地文件） - * @param bucketName 桶名称 - * @param fileName GirdFS中的文件名称 - * @throws IOException 异常 - */ - public void download(File targetFile, String bucketName, String fileName) throws IOException { - if (available()) { - GridFSBucket bucket = getBucket(bucketName); - try (GridFSDownloadStream gis = bucket.openDownloadStream(fileName); - BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(targetFile)) - ) { - byte[] buffer = new byte[1024]; - int bytes = 0; - while ((bytes = gis.read(buffer)) != -1) { - bos.write(buffer, 0, bytes); - } - bos.flush(); - } - } - } - - /** - * 删除几天前的文件 - * @param bucketName 桶名称 - * @param day 日期偏移量，单位 天 - */ - public void deleteBefore(String bucketName, int day) { - - Stopwatch sw = Stopwatch.createStarted(); - - Date date = DateUtils.addDays(new Date(), -day); - GridFSBucket bucket = getBucket(bucketName); - Bson filter = Filters.lt("uploadDate", date); - - // 循环删除性能很差？我猜你肯定没看过官方实现[狗头]：org.springframework.data.mongodb.gridfs.GridFsTemplate.delete - bucket.find(filter).forEach((Consumer) gridFSFile -> { - ObjectId objectId = gridFSFile.getObjectId(); - try { - bucket.delete(objectId); - log.info("[GridFsManager] deleted {}#{}", bucketName, objectId); - }catch (Exception e) { - log.error("[GridFsManager] deleted {}#{} failed.", bucketName, objectId, e); - } - }); - log.info("[GridFsManager] clean bucket({}) successfully, delete all files before {}, using {}.", bucketName, date, sw.stop()); - } - - public boolean exists(String bucketName, String fileName) { - GridFSBucket bucket = getBucket(bucketName); - GridFSFindIterable files = bucket.find(Filters.eq("filename", fileName)); - try { - GridFSFile first = files.first(); - return first != null; - }catch (Exception ignore) { - } - return false; - } - - private GridFSBucket getBucket(String bucketName) { - return bucketCache.computeIfAbsent(bucketName, ignore -> GridFSBuckets.create(db, bucketName)); - } - - @Override - public void afterPropertiesSet() throws Exception { - String enable = environment.getProperty(PowerJobServerConfigKey.MONGODB_ENABLE, Boolean.FALSE.toString()); - available = Boolean.TRUE.toString().equals(enable) && db != null; - log.info("[GridFsManager] available: {}, db: {}", available, db); - } -} diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/AbstractDFsService.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/AbstractDFsService.java new file mode 100644 index 00000000..a6d6ec56 --- /dev/null +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/AbstractDFsService.java @@ -0,0 +1,37 @@ +package tech.powerjob.server.persistence.storage; + +import org.springframework.beans.factory.InitializingBean; +import org.springframework.core.env.Environment; +import tech.powerjob.server.extension.dfs.DFsService; + +import javax.annotation.Resource; + +/** + * AbstractDFsService + * + * @author tjq + * @since 2023/7/28 + */ +public abstract class AbstractDFsService implements DFsService, InitializingBean { + + @Resource + protected Environment environment; + + protected boolean active = false; + + protected static final String PROPERTY_KEY = "oms.storage.dfs"; + + protected boolean active() { + return active; + } + + protected void turnOn() { + active = true; + } + + protected static String fetchProperty(Environment environment, String dfsType, String key) { + String pKey = String.format("%s.%s.%s", PROPERTY_KEY, dfsType, key); + return environment.getProperty(pKey); + } + +} diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/Constants.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/Constants.java new file mode 100644 index 00000000..dd7fcd69 --- /dev/null +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/Constants.java @@ -0,0 +1,15 @@ +package tech.powerjob.server.persistence.storage; + +/** + * Constants + * + * @author tjq + * @since 2023/7/30 + */ +public class Constants { + + public static final String LOG_BUCKET = "log"; + + public static final String CONTAINER_BUCKET = "container"; + +} diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/AliOssService.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/AliOssService.java new file mode 100644 index 00000000..8ab4d997 --- /dev/null +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/AliOssService.java @@ -0,0 +1,45 @@ +package tech.powerjob.server.persistence.storage.impl; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import tech.powerjob.server.extension.dfs.*; +import tech.powerjob.server.persistence.storage.AbstractDFsService; + +import java.io.IOException; +import java.util.Optional; + +/** + * Alibaba OSS support + * 海量、安全、低成本、高可靠的云存储服务 + * + * @author tjq + * @since 2023/7/30 + */ +@Slf4j +@Service +@ConditionalOnProperty(name = {"oms.storage.dfs.alioss.uri"}, matchIfMissing = false) +@ConditionalOnMissingBean(DFsService.class) +public class AliOssService extends AbstractDFsService { + + @Override + public void afterPropertiesSet() throws Exception { + + } + + @Override + public void store(StoreRequest storeRequest) throws IOException { + + } + + @Override + public void download(DownloadRequest downloadRequest) throws IOException { + + } + + @Override + public Optional fetchFileMeta(FileLocation fileLocation) throws IOException { + return Optional.empty(); + } +} diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/EmptyDFsService.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/EmptyDFsService.java new file mode 100644 index 00000000..2ca193cf --- /dev/null +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/EmptyDFsService.java @@ -0,0 +1,38 @@ +package tech.powerjob.server.persistence.storage.impl; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Service; +import tech.powerjob.server.extension.dfs.*; + +import java.io.IOException; +import java.util.Optional; + +/** + * EmptyDFsService + * + * @author tjq + * @since 2023/7/30 + */ +@Service +@Order(value = Ordered.LOWEST_PRECEDENCE) +@ConditionalOnMissingBean(DFsService.class) +public class EmptyDFsService implements DFsService, { + + public EmptyDFsService() { + } + + @Override + public void store(StoreRequest storeRequest) throws IOException { + } + + @Override + public void download(DownloadRequest downloadRequest) throws IOException { + } + + @Override + public Optional fetchFileMeta(FileLocation fileLocation) throws IOException { + return Optional.empty(); + } +} diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java new file mode 100644 index 00000000..7c1e131d --- /dev/null +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java @@ -0,0 +1,141 @@ +package tech.powerjob.server.persistence.storage.impl; + +import com.google.common.base.Stopwatch; +import com.google.common.collect.Maps; +import com.mongodb.ConnectionString; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.gridfs.GridFSBucket; +import com.mongodb.client.gridfs.GridFSBuckets; +import com.mongodb.client.gridfs.GridFSDownloadStream; +import com.mongodb.client.gridfs.GridFSFindIterable; +import com.mongodb.client.gridfs.model.GridFSFile; +import com.mongodb.client.model.Filters; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateUtils; +import org.bson.conversions.Bson; +import org.bson.types.ObjectId; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Service; +import tech.powerjob.server.extension.dfs.*; +import tech.powerjob.server.persistence.storage.AbstractDFsService; + +import java.io.*; +import java.nio.file.Files; +import java.util.Date; +import java.util.Map; +import java.util.Optional; + +/** + * 使用 MongoDB GridFS 作为底层存储 + * 配置用法：oms.storage.dfs.mongodb.uri=mongodb+srv://zqq:No1Bug2Please3!@cluster0.wie54.gcp.mongodb.net/powerjob_daily?retryWrites=true&w=majority + * + * @author tjq + * @since 2023/7/28 + */ +@Slf4j +@Service +@ConditionalOnProperty(name = {"oms.storage.dfs.mongodb.uri", "spring.data.mongodb.uri"}, matchIfMissing = false) +@ConditionalOnMissingBean(DFsService.class) +public class GridFsService extends AbstractDFsService implements InitializingBean { + + private MongoDatabase db; + private final Map bucketCache = Maps.newConcurrentMap(); + private static final String TYPE_MONGO = "mongodb"; + + private static final String SPRING_MONGO_DB_CONFIG_KEY = "spring.data.mongodb.uri"; + + @Override + public void store(StoreRequest storeRequest) throws IOException { + GridFSBucket bucket = getBucket(storeRequest.getFileLocation().getBucket()); + try (BufferedInputStream bis = new BufferedInputStream(Files.newInputStream(storeRequest.getLocalFile().toPath()))) { + bucket.uploadFromStream(storeRequest.getFileLocation().getName(), bis); + } + } + + @Override + public void download(DownloadRequest downloadRequest) throws IOException { + GridFSBucket bucket = getBucket(downloadRequest.getFileLocation().getBucket()); + try (GridFSDownloadStream gis = bucket.openDownloadStream(downloadRequest.getFileLocation().getName()); + BufferedOutputStream bos = new BufferedOutputStream(Files.newOutputStream(downloadRequest.getTarget().toPath())) + ) { + byte[] buffer = new byte[1024]; + int bytes = 0; + while ((bytes = gis.read(buffer)) != -1) { + bos.write(buffer, 0, bytes); + } + bos.flush(); + } + } + + @Override + public Optional fetchFileMeta(FileLocation fileLocation) throws IOException { + GridFSBucket bucket = getBucket(fileLocation.getBucket()); + GridFSFindIterable files = bucket.find(Filters.eq("filename", fileLocation.getName())); + GridFSFile first = files.first(); + if (first == null) { + return Optional.empty(); + } + return Optional.of(new FileMeta() + .setLength(first.getLength()) + .setMetaInfo(first.getMetadata())); + } + + @Override + public void cleanExpiredFiles(String bucketName, int days) { + Stopwatch sw = Stopwatch.createStarted(); + + Date date = DateUtils.addDays(new Date(), -days); + GridFSBucket bucket = getBucket(bucketName); + Bson filter = Filters.lt("uploadDate", date); + + // 循环删除性能很差？我猜你肯定没看过官方实现[狗头]：org.springframework.data.mongodb.gridfs.GridFsTemplate.delete + bucket.find(filter).forEach(gridFSFile -> { + ObjectId objectId = gridFSFile.getObjectId(); + try { + bucket.delete(objectId); + log.info("[GridFsService] deleted {}#{}", bucketName, objectId); + }catch (Exception e) { + log.error("[GridFsService] deleted {}#{} failed.", bucketName, objectId, e); + } + }); + log.info("[GridFsService] clean bucket({}) successfully, delete all files before {}, using {}.", bucketName, date, sw.stop()); + } + + @Override + public void afterPropertiesSet() throws Exception { + String uri = parseMongoUri(environment); + log.info("[GridFsService] mongoDB uri: {}", uri); + if (StringUtils.isEmpty(uri)) { + log.warn("[GridFsService] uri is empty, GridFsService is off now!"); + return; + } + + ConnectionString connectionString = new ConnectionString(uri); + MongoClient mongoClient = MongoClients.create(connectionString); + db = mongoClient.getDatabase(Optional.ofNullable(connectionString.getDatabase()).orElse("pj")); + + turnOn(); + + log.info("[GridFsService] turn on mongodb GridFs as storage layer."); + } + + private GridFSBucket getBucket(String bucketName) { + return bucketCache.computeIfAbsent(bucketName, ignore -> GridFSBuckets.create(db, bucketName)); + } + + static String parseMongoUri(Environment environment) { + // 优先从新的规则读取 + String uri = fetchProperty(environment, TYPE_MONGO, "uri"); + if (StringUtils.isNotEmpty(uri)) { + return uri; + } + // 兼容 4.3.3 前的逻辑，读取 SpringMongoDB 配置 + return environment.getProperty(SPRING_MONGO_DB_CONFIG_KEY); + } +} diff --git a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/WorkerClusterQueryService.java b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/WorkerClusterQueryService.java index 2cbef095..62e4d320 100644 --- a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/WorkerClusterQueryService.java +++ b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/WorkerClusterQueryService.java @@ -2,12 +2,11 @@ package tech.powerjob.server.remote.worker; import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import tech.powerjob.common.enums.DispatchStrategy; import tech.powerjob.common.model.DeployedContainerInfo; import tech.powerjob.server.common.module.WorkerInfo; -import tech.powerjob.server.extension.WorkerFilter; +import tech.powerjob.server.remote.worker.filter.WorkerFilter; import tech.powerjob.server.persistence.remote.model.JobInfoDO; import tech.powerjob.server.remote.server.redirector.DesignateServer; diff --git a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/DesignatedWorkerFilter.java b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/DesignatedWorkerFilter.java index 26656ae6..c03a9f9f 100644 --- a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/DesignatedWorkerFilter.java +++ b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/DesignatedWorkerFilter.java @@ -6,7 +6,6 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import tech.powerjob.server.common.SJ; import tech.powerjob.server.common.module.WorkerInfo; -import tech.powerjob.server.extension.WorkerFilter; import tech.powerjob.server.persistence.remote.model.JobInfoDO; import java.util.Set; diff --git a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/DisconnectedWorkerFilter.java b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/DisconnectedWorkerFilter.java index ff8482d9..5cdfb9a8 100644 --- a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/DisconnectedWorkerFilter.java +++ b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/DisconnectedWorkerFilter.java @@ -1,6 +1,5 @@ package tech.powerjob.server.remote.worker.filter; -import tech.powerjob.server.extension.WorkerFilter; import tech.powerjob.server.persistence.remote.model.JobInfoDO; import tech.powerjob.server.common.module.WorkerInfo; import lombok.extern.slf4j.Slf4j; diff --git a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/SystemMetricsWorkerFilter.java b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/SystemMetricsWorkerFilter.java index f88bbdd5..2270189a 100644 --- a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/SystemMetricsWorkerFilter.java +++ b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/SystemMetricsWorkerFilter.java @@ -1,7 +1,6 @@ package tech.powerjob.server.remote.worker.filter; import tech.powerjob.common.model.SystemMetrics; -import tech.powerjob.server.extension.WorkerFilter; import tech.powerjob.server.persistence.remote.model.JobInfoDO; import tech.powerjob.server.common.module.WorkerInfo; import lombok.extern.slf4j.Slf4j; diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/WorkerFilter.java b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/WorkerFilter.java similarity index 91% rename from powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/WorkerFilter.java rename to powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/WorkerFilter.java index 5df4214d..35e41586 100644 --- a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/WorkerFilter.java +++ b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/WorkerFilter.java @@ -1,4 +1,4 @@ -package tech.powerjob.server.extension; +package tech.powerjob.server.remote.worker.filter; import tech.powerjob.server.persistence.remote.model.JobInfoDO; import tech.powerjob.server.common.module.WorkerInfo; diff --git a/powerjob-server/powerjob-server-starter/src/test/java/tech/powerjob/server/test/GridFsTest.java b/powerjob-server/powerjob-server-starter/src/test/java/tech/powerjob/server/test/GridFsTest.java deleted file mode 100644 index 729dc2f2..00000000 --- a/powerjob-server/powerjob-server-starter/src/test/java/tech/powerjob/server/test/GridFsTest.java +++ /dev/null @@ -1,61 +0,0 @@ -package tech.powerjob.server.test; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import tech.powerjob.server.persistence.mongodb.GridFsManager; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit4.SpringRunner; - -import javax.annotation.Resource; -import java.io.File; -import java.io.IOException; - -/** - * GridFS 测试 - * - * @author tjq - * @since 2020/5/18 - */ -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -public class GridFsTest { - - @Resource - private GridFsManager gridFsManager; - - @Test - @Disabled - public void testStore() throws IOException { - /** - File file = new File("/Users/tjq/Desktop/DistributeCompute/oms-template-origin.zip"); - gridFsManager.store(file, "test", "test.zip"); - **/ - } - - @Test - @Disabled - public void testDownload() throws IOException { - /** - File file = new File("/Users/tjq/Desktop/tmp/test-download.zip"); - gridFsManager.download(file, "test", "test.zip"); - **/ - } - - @Test - @Disabled - public void testDelete() { - /** - gridFsManager.deleteBefore("fs", 0); - **/ - } - - @Test - @Disabled - public void testExists() { - /** - System.out.println(gridFsManager.exists("test", "test.zip")); - System.out.println(gridFsManager.exists("test", "oms-sql.sql")); - **/ - } - -} diff --git a/powerjob-server/powerjob-server-starter/src/test/java/tech/powerjob/server/test/OmsLogTest.java b/powerjob-server/powerjob-server-starter/src/test/java/tech/powerjob/server/test/OmsLogTest.java deleted file mode 100644 index 2fa7a093..00000000 --- a/powerjob-server/powerjob-server-starter/src/test/java/tech/powerjob/server/test/OmsLogTest.java +++ /dev/null @@ -1,56 +0,0 @@ -package tech.powerjob.server.test; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import tech.powerjob.server.common.utils.OmsFileUtils; -import tech.powerjob.server.persistence.mongodb.GridFsManager; -import tech.powerjob.server.core.scheduler.CleanService; -import com.mongodb.client.gridfs.model.GridFSFile; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.data.mongodb.core.query.Criteria; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.data.mongodb.gridfs.GridFsTemplate; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit4.SpringRunner; - -import javax.annotation.Resource; -import java.util.Date; -import java.util.function.Consumer; - -/** - * 在线日志测试 - * - * @author tjq - * @since 2020/5/11 - */ - -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@Disabled -public class OmsLogTest { - - @Resource - private CleanService cleanService; - @Resource - private GridFsTemplate gridFsTemplate; - - @Test - public void testLocalLogCleaner() { - cleanService.cleanLocal(OmsFileUtils.genLogDirPath(), 0); - } - - @Test - public void testRemoteLogCleaner() { - cleanService.cleanRemote(GridFsManager.LOG_BUCKET, 0); - } - - @Test - public void testGridFsQuery() { - Query mongoQuery = Query.query(Criteria.where("uploadDate").gt(new Date())); - gridFsTemplate.find(mongoQuery).forEach(new Consumer() { - @Override - public void accept(GridFSFile gridFSFile) { - System.out.println(gridFSFile.getFilename()); - } - }); - } -} From f0514ac65f7df34bd9be3d06dd3f17627eb452a7 Mon Sep 17 00:00:00 2001 From: tjq Date: Sun, 30 Jul 2023 14:39:57 +0800 Subject: [PATCH 09/24] feat: [storageExt] support alicloud oss --- powerjob-server/pom.xml | 9 +- .../server/extension/dfs/FileMeta.java | 5 + .../powerjob-server-persistence/pom.xml | 4 + .../storage/AbstractDFsService.java | 2 +- .../storage/impl/AliOssService.java | 127 +++++++++++++++++- .../storage/impl/GridFsService.java | 35 ++--- 6 files changed, 164 insertions(+), 18 deletions(-) diff --git a/powerjob-server/pom.xml b/powerjob-server/pom.xml index 0aac969f..6c762e93 100644 --- a/powerjob-server/pom.xml +++ b/powerjob-server/pom.xml @@ -54,6 +54,7 @@ 4.3.3 4.3.3 1.6.14 + 3.17.1 @@ -99,12 +100,18 @@ ${project.version} - + org.mongodb mongodb-driver-sync ${mongodb-driver-sync.version} + + + com.aliyun.oss + aliyun-sdk-oss + ${aliyun-sdk-oss.version} + diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/FileMeta.java b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/FileMeta.java index 9fb7656a..077701aa 100644 --- a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/FileMeta.java +++ b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/FileMeta.java @@ -3,6 +3,7 @@ package tech.powerjob.server.extension.dfs; import lombok.Data; import lombok.experimental.Accessors; +import java.util.Date; import java.util.Map; /** @@ -19,6 +20,10 @@ public class FileMeta { * 文件大小 */ private long length; + /** + * 最后修改时间 + */ + private Date lastModifiedTime; /** * 元数据 diff --git a/powerjob-server/powerjob-server-persistence/pom.xml b/powerjob-server/powerjob-server-persistence/pom.xml index a1d526b2..d476400b 100644 --- a/powerjob-server/powerjob-server-persistence/pom.xml +++ b/powerjob-server/powerjob-server-persistence/pom.xml @@ -36,6 +36,10 @@ org.mongodb mongodb-driver-sync + + com.aliyun.oss + aliyun-sdk-oss + \ No newline at end of file diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/AbstractDFsService.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/AbstractDFsService.java index a6d6ec56..99b2c72a 100644 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/AbstractDFsService.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/AbstractDFsService.java @@ -29,7 +29,7 @@ public abstract class AbstractDFsService implements DFsService, InitializingBean active = true; } - protected static String fetchProperty(Environment environment, String dfsType, String key) { + protected String fetchProperty(String dfsType, String key) { String pKey = String.format("%s.%s.%s", PROPERTY_KEY, dfsType, key); return environment.getProperty(pKey); } diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/AliOssService.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/AliOssService.java index 8ab4d997..321dc8b6 100644 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/AliOssService.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/AliOssService.java @@ -1,6 +1,17 @@ package tech.powerjob.server.persistence.storage.impl; +import com.aliyun.oss.*; +import com.aliyun.oss.common.auth.CredentialsProvider; +import com.aliyun.oss.common.auth.CredentialsProviderFactory; +import com.aliyun.oss.common.auth.DefaultCredentialProvider; +import com.aliyun.oss.model.DownloadFileRequest; +import com.aliyun.oss.model.ObjectMetadata; +import com.aliyun.oss.model.PutObjectRequest; +import com.google.common.collect.Maps; +import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; @@ -8,38 +19,152 @@ import tech.powerjob.server.extension.dfs.*; import tech.powerjob.server.persistence.storage.AbstractDFsService; import java.io.IOException; +import java.util.Map; import java.util.Optional; /** * Alibaba OSS support * 海量、安全、低成本、高可靠的云存储服务 + * 配置项： + * oms.storage.dfs.alioss.endpoint + * oms.storage.dfs.alioss.credential_type + * oms.storage.dfs.alioss.ak + * oms.storage.dfs.alioss.sk + * oms.storage.dfs.alioss.token * * @author tjq * @since 2023/7/30 */ @Slf4j @Service -@ConditionalOnProperty(name = {"oms.storage.dfs.alioss.uri"}, matchIfMissing = false) +@ConditionalOnProperty(name = {"oms.storage.dfs.alioss.endpoint"}, matchIfMissing = false) @ConditionalOnMissingBean(DFsService.class) public class AliOssService extends AbstractDFsService { + private static final String TYPE_ALI_OSS = "alioss"; + + private static final String KEY_ENDPOINT = "endpoint"; + private static final String KEY_CREDENTIAL_TYPE = "credential_type"; + private static final String KEY_AK = "ak"; + private static final String KEY_SK = "sk"; + private static final String KEY_TOKEN = "token"; + + private OSS oss; + + private static final int DOWNLOAD_PART_SIZE = 10240; + @Override public void afterPropertiesSet() throws Exception { + String endpoint = fetchProperty(TYPE_ALI_OSS, KEY_ENDPOINT); + String ct = fetchProperty(TYPE_ALI_OSS, KEY_CREDENTIAL_TYPE); + String ak = fetchProperty(TYPE_ALI_OSS, KEY_AK); + String sk = fetchProperty(TYPE_ALI_OSS, KEY_SK); + String token = fetchProperty(TYPE_ALI_OSS, KEY_TOKEN); + + initOssClient(endpoint, ct, ak, sk, token); } @Override public void store(StoreRequest storeRequest) throws IOException { + FileLocation dfl = storeRequest.getFileLocation(); + ObjectMetadata objectMetadata = new ObjectMetadata(); + + PutObjectRequest putObjectRequest = new PutObjectRequest(dfl.getBucket(), dfl.getName(), storeRequest.getLocalFile(), objectMetadata); + oss.putObject(putObjectRequest); } @Override public void download(DownloadRequest downloadRequest) throws IOException { + FileLocation dfl = downloadRequest.getFileLocation(); + DownloadFileRequest downloadFileRequest = new DownloadFileRequest(dfl.getBucket(), dfl.getName(), downloadRequest.getTarget().getAbsolutePath(), DOWNLOAD_PART_SIZE); + try { + oss.downloadFile(downloadFileRequest); + } catch (Throwable t) { + ExceptionUtils.rethrow(t); + } } @Override public Optional fetchFileMeta(FileLocation fileLocation) throws IOException { + try { + ObjectMetadata objectMetadata = oss.getObjectMetadata(fileLocation.getBucket(), fileLocation.getName()); + return Optional.ofNullable(objectMetadata).map(ossM -> { + + Map metaInfo = Maps.newHashMap(); + metaInfo.putAll(ossM.getRawMetadata()); + if (ossM.getUserMetadata() != null) { + metaInfo.putAll(ossM.getUserMetadata()); + } + + return new FileMeta() + .setLastModifiedTime(ossM.getLastModified()) + .setLength(ossM.getContentLength()) + .setMetaInfo(metaInfo); + }); + } catch (OSSException oe) { + // TODO: 判断文件不存在 + } return Optional.empty(); } + + void initOssClient(String endpoint, String mode, String ak, String sk, String token) throws Exception { + + log.info("[AliOssService] init OSS by config: {},{},{},{},{}", endpoint, mode, ak, sk, token); + + CredentialsProvider credentialsProvider; + CredentialType credentialType = CredentialType.parse(mode); + switch (credentialType) { + case PWD: + credentialsProvider = new DefaultCredentialProvider(ak, sk, token); + break; + case SYSTEM_PROPERTY: + credentialsProvider = CredentialsProviderFactory.newSystemPropertiesCredentialsProvider(); + break; + default: + credentialsProvider = CredentialsProviderFactory.newEnvironmentVariableCredentialsProvider(); + } + + this.oss = new OSSClientBuilder().build(endpoint, credentialsProvider); + log.info("[AliOssService] initialize OSS successfully!"); + } + + + @AllArgsConstructor + enum CredentialType { + /** + * 从环境读取 + */ + ENV("env"), + /** + * 系统配置 + */ + SYSTEM_PROPERTY("sys"), + /** + * 从账号密码读取 + */ + PWD("pwd") + ; + + private final String code; + + /** + * parse credential type + * @param mode oms.storage.dfs.alioss.credential_type + * @return CredentialType + */ + public static CredentialType parse(String mode) { + + for (CredentialType credentialType : values()) { + if (StringUtils.equalsIgnoreCase(credentialType.code, mode)) { + return credentialType; + } + } + + return PWD; + } + + } } diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java index 7c1e131d..68e973f8 100644 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java @@ -83,6 +83,7 @@ public class GridFsService extends AbstractDFsService implements InitializingBea } return Optional.of(new FileMeta() .setLength(first.getLength()) + .setLastModifiedTime(first.getUploadDate()) .setMetaInfo(first.getMetadata())); } @@ -109,7 +110,25 @@ public class GridFsService extends AbstractDFsService implements InitializingBea @Override public void afterPropertiesSet() throws Exception { - String uri = parseMongoUri(environment); + String uri = parseMongoUri(); + initMongo(uri); + } + + private GridFSBucket getBucket(String bucketName) { + return bucketCache.computeIfAbsent(bucketName, ignore -> GridFSBuckets.create(db, bucketName)); + } + + private String parseMongoUri() { + // 优先从新的规则读取 + String uri = fetchProperty(TYPE_MONGO, "uri"); + if (StringUtils.isNotEmpty(uri)) { + return uri; + } + // 兼容 4.3.3 前的逻辑，读取 SpringMongoDB 配置 + return environment.getProperty(SPRING_MONGO_DB_CONFIG_KEY); + } + + private void initMongo(String uri) { log.info("[GridFsService] mongoDB uri: {}", uri); if (StringUtils.isEmpty(uri)) { log.warn("[GridFsService] uri is empty, GridFsService is off now!"); @@ -124,18 +143,4 @@ public class GridFsService extends AbstractDFsService implements InitializingBea log.info("[GridFsService] turn on mongodb GridFs as storage layer."); } - - private GridFSBucket getBucket(String bucketName) { - return bucketCache.computeIfAbsent(bucketName, ignore -> GridFSBuckets.create(db, bucketName)); - } - - static String parseMongoUri(Environment environment) { - // 优先从新的规则读取 - String uri = fetchProperty(environment, TYPE_MONGO, "uri"); - if (StringUtils.isNotEmpty(uri)) { - return uri; - } - // 兼容 4.3.3 前的逻辑，读取 SpringMongoDB 配置 - return environment.getProperty(SPRING_MONGO_DB_CONFIG_KEY); - } } From b251df4c35b886c6ce63dd71bbda8bf120c372e6 Mon Sep 17 00:00:00 2001 From: tjq Date: Sun, 30 Jul 2023 15:35:18 +0800 Subject: [PATCH 10/24] feat: [storageExt] support alicloud oss and add some test code --- .../storage/AbstractDFsService.java | 10 -- .../storage/impl/AliOssService.java | 47 +++++-- .../storage/impl/EmptyDFsService.java | 2 +- .../storage/impl/GridFsService.java | 2 - .../storage/impl/AliOssServiceTest.java | 116 ++++++++++++++++++ 5 files changed, 156 insertions(+), 21 deletions(-) create mode 100644 powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AliOssServiceTest.java diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/AbstractDFsService.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/AbstractDFsService.java index 99b2c72a..3bbdd83c 100644 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/AbstractDFsService.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/AbstractDFsService.java @@ -17,18 +17,8 @@ public abstract class AbstractDFsService implements DFsService, InitializingBean @Resource protected Environment environment; - protected boolean active = false; - protected static final String PROPERTY_KEY = "oms.storage.dfs"; - protected boolean active() { - return active; - } - - protected void turnOn() { - active = true; - } - protected String fetchProperty(String dfsType, String key) { String pKey = String.format("%s.%s.%s", PROPERTY_KEY, dfsType, key); return environment.getProperty(pKey); diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/AliOssService.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/AliOssService.java index 321dc8b6..7bcd65f1 100644 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/AliOssService.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/AliOssService.java @@ -9,7 +9,9 @@ import com.aliyun.oss.model.ObjectMetadata; import com.aliyun.oss.model.PutObjectRequest; import com.google.common.collect.Maps; import lombok.AllArgsConstructor; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; @@ -27,6 +29,7 @@ import java.util.Optional; * 海量、安全、低成本、高可靠的云存储服务 * 配置项： * oms.storage.dfs.alioss.endpoint + * oms.storage.dfs.alioss.bucket * oms.storage.dfs.alioss.credential_type * oms.storage.dfs.alioss.ak * oms.storage.dfs.alioss.sk @@ -44,34 +47,38 @@ public class AliOssService extends AbstractDFsService { private static final String TYPE_ALI_OSS = "alioss"; private static final String KEY_ENDPOINT = "endpoint"; + private static final String KEY_BUCKET = "bucket"; private static final String KEY_CREDENTIAL_TYPE = "credential_type"; private static final String KEY_AK = "ak"; private static final String KEY_SK = "sk"; private static final String KEY_TOKEN = "token"; private OSS oss; + private String bucket; private static final int DOWNLOAD_PART_SIZE = 10240; + private static final String NO_SUCH_KEY = "NoSuchKey"; + @Override public void afterPropertiesSet() throws Exception { String endpoint = fetchProperty(TYPE_ALI_OSS, KEY_ENDPOINT); + String bkt = fetchProperty(TYPE_ALI_OSS, KEY_BUCKET); String ct = fetchProperty(TYPE_ALI_OSS, KEY_CREDENTIAL_TYPE); String ak = fetchProperty(TYPE_ALI_OSS, KEY_AK); String sk = fetchProperty(TYPE_ALI_OSS, KEY_SK); String token = fetchProperty(TYPE_ALI_OSS, KEY_TOKEN); - initOssClient(endpoint, ct, ak, sk, token); + initOssClient(endpoint, bkt, ct, ak, sk, token); } @Override public void store(StoreRequest storeRequest) throws IOException { - FileLocation dfl = storeRequest.getFileLocation(); ObjectMetadata objectMetadata = new ObjectMetadata(); - PutObjectRequest putObjectRequest = new PutObjectRequest(dfl.getBucket(), dfl.getName(), storeRequest.getLocalFile(), objectMetadata); + PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, parseFileName(storeRequest.getFileLocation()), storeRequest.getLocalFile(), objectMetadata); oss.putObject(putObjectRequest); } @@ -79,8 +86,9 @@ public class AliOssService extends AbstractDFsService { public void download(DownloadRequest downloadRequest) throws IOException { FileLocation dfl = downloadRequest.getFileLocation(); - DownloadFileRequest downloadFileRequest = new DownloadFileRequest(dfl.getBucket(), dfl.getName(), downloadRequest.getTarget().getAbsolutePath(), DOWNLOAD_PART_SIZE); + DownloadFileRequest downloadFileRequest = new DownloadFileRequest(bucket, parseFileName(dfl), downloadRequest.getTarget().getAbsolutePath(), DOWNLOAD_PART_SIZE); try { + FileUtils.forceMkdirParent(downloadRequest.getTarget()); oss.downloadFile(downloadFileRequest); } catch (Throwable t) { ExceptionUtils.rethrow(t); @@ -90,7 +98,7 @@ public class AliOssService extends AbstractDFsService { @Override public Optional fetchFileMeta(FileLocation fileLocation) throws IOException { try { - ObjectMetadata objectMetadata = oss.getObjectMetadata(fileLocation.getBucket(), fileLocation.getName()); + ObjectMetadata objectMetadata = oss.getObjectMetadata(bucket, parseFileName(fileLocation)); return Optional.ofNullable(objectMetadata).map(ossM -> { Map metaInfo = Maps.newHashMap(); @@ -105,14 +113,28 @@ public class AliOssService extends AbstractDFsService { .setMetaInfo(metaInfo); }); } catch (OSSException oe) { - // TODO: 判断文件不存在 + String errorCode = oe.getErrorCode(); + if (NO_SUCH_KEY.equalsIgnoreCase(errorCode)) { + return Optional.empty(); + } + ExceptionUtils.rethrow(oe); } return Optional.empty(); } - void initOssClient(String endpoint, String mode, String ak, String sk, String token) throws Exception { + private static String parseFileName(FileLocation fileLocation) { + return String.format("%s/%s", fileLocation.getBucket(), fileLocation.getName()); + } - log.info("[AliOssService] init OSS by config: {},{},{},{},{}", endpoint, mode, ak, sk, token); + void initOssClient(String endpoint, String bucket, String mode, String ak, String sk, String token) throws Exception { + + log.info("[AliOssService] init OSS by config: endpoint={},bucket={},credentialType={},ak={},sk={},token={}", endpoint, bucket, mode, ak, sk, token); + + if (StringUtils.isEmpty(bucket)) { + throw new IllegalArgumentException("'oms.storage.dfs.alioss.bucket' can't be empty, please creat a bucket in aliyun oss console then config it to powerjob"); + } + + this.bucket = bucket; CredentialsProvider credentialsProvider; CredentialType credentialType = CredentialType.parse(mode); @@ -131,7 +153,16 @@ public class AliOssService extends AbstractDFsService { log.info("[AliOssService] initialize OSS successfully!"); } + @Override + public void cleanExpiredFiles(String bucket, int days) { + /* + 阿里云 OSS 自带生命周期管理，请参考文档进行配置，代码层面不进行实现（浪费服务器资源）https://help.aliyun.com/zh/oss/user-guide/overview-54 + 阿里云 OSS 自带生命周期管理，请参考文档进行配置，代码层面不进行实现（浪费服务器资源）https://help.aliyun.com/zh/oss/user-guide/overview-54 + 阿里云 OSS 自带生命周期管理，请参考文档进行配置，代码层面不进行实现（浪费服务器资源）https://help.aliyun.com/zh/oss/user-guide/overview-54 + */ + } + @Getter @AllArgsConstructor enum CredentialType { /** diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/EmptyDFsService.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/EmptyDFsService.java index 2ca193cf..c0b54d7f 100644 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/EmptyDFsService.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/EmptyDFsService.java @@ -18,7 +18,7 @@ import java.util.Optional; @Service @Order(value = Ordered.LOWEST_PRECEDENCE) @ConditionalOnMissingBean(DFsService.class) -public class EmptyDFsService implements DFsService, { +public class EmptyDFsService implements DFsService { public EmptyDFsService() { } diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java index 68e973f8..7bb01464 100644 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java @@ -139,8 +139,6 @@ public class GridFsService extends AbstractDFsService implements InitializingBea MongoClient mongoClient = MongoClients.create(connectionString); db = mongoClient.getDatabase(Optional.ofNullable(connectionString.getDatabase()).orElse("pj")); - turnOn(); - log.info("[GridFsService] turn on mongodb GridFs as storage layer."); } } diff --git a/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AliOssServiceTest.java b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AliOssServiceTest.java new file mode 100644 index 00000000..cbcdae13 --- /dev/null +++ b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AliOssServiceTest.java @@ -0,0 +1,116 @@ +package tech.powerjob.server.persistence.storage.impl; + +import com.aliyun.oss.common.utils.AuthUtils; +import com.aliyun.oss.common.utils.StringUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.junit.jupiter.api.Test; +import tech.powerjob.common.serialize.JsonUtils; +import tech.powerjob.server.common.utils.OmsFileUtils; +import tech.powerjob.server.extension.dfs.DownloadRequest; +import tech.powerjob.server.extension.dfs.FileLocation; +import tech.powerjob.server.extension.dfs.FileMeta; +import tech.powerjob.server.extension.dfs.StoreRequest; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.util.Optional; + + +/** + * test AliOSS + * + * @author tjq + * @since 2023/7/30 + */ +@Slf4j +class AliOssServiceTest { + + private static final String BUCKET = "power-job"; + + @Test + void testBaseFileOperation() throws Exception { + + Optional aliOssServiceOpt = fetchService(); + if (!aliOssServiceOpt.isPresent()) { + return; + } + + AliOssService aliOssService = aliOssServiceOpt.get(); + + String content = "wlcgyqsl"; + + String temporarySourcePath = OmsFileUtils.genTemporaryWorkPath() + "source.txt"; + String temporaryDownloadPath = OmsFileUtils.genTemporaryWorkPath() + "download.txt"; + + log.info("[testBaseFileOperation] temporarySourcePath: {}", temporarySourcePath); + File sourceFile = new File(temporarySourcePath); + FileUtils.forceMkdirParent(sourceFile); + OmsFileUtils.string2File(content, sourceFile); + + FileLocation fileLocation = new FileLocation().setBucket("pj_test").setName("testAliOss.txt"); + + StoreRequest storeRequest = new StoreRequest() + .setFileLocation(fileLocation) + .setLocalFile(sourceFile); + + // 存储 + aliOssService.store(storeRequest); + + // 读取 meta + Optional metaOpt = aliOssService.fetchFileMeta(fileLocation); + assert metaOpt.isPresent(); + + log.info("[testBaseFileOperation] file meta: {}", JsonUtils.toJSONString(metaOpt.get())); + + // 下载 + log.info("[testBaseFileOperation] temporaryDownloadPath: {}", temporaryDownloadPath); + File downloadFile = new File(temporaryDownloadPath); + DownloadRequest downloadRequest = new DownloadRequest() + .setFileLocation(fileLocation) + .setTarget(downloadFile); + aliOssService.download(downloadRequest); + + String downloadFileContent = FileUtils.readFileToString(downloadFile, StandardCharsets.UTF_8); + log.info("[testBaseFileOperation] download content: {}", downloadFileContent); + assert downloadFileContent.equals(content); + } + + @Test + void testFileNotExist() throws Exception { + Optional aliOssServiceOpt = fetchService(); + if (!aliOssServiceOpt.isPresent()) { + return; + } + Optional metaOpt = aliOssServiceOpt.get().fetchFileMeta(new FileLocation().setBucket("tjq").setName("yhz")); + assert !metaOpt.isPresent(); + } + + /** + * 依赖阿里云账号密码测试，为了保证单测在其他环境也能通过，如果发现不存在配置则直接跳过 + * @return AliOssService + */ + private Optional fetchService() { + + String accessKeyId = StringUtils.trim(System.getenv(AuthUtils.ACCESS_KEY_ENV_VAR)); + String secretAccessKey = StringUtils.trim(System.getenv(AuthUtils.SECRET_KEY_ENV_VAR)); + + String bucket = Optional.ofNullable(System.getenv("POWERJOB_OSS_BUEKCT")).orElse(BUCKET); + + log.info("[AliOssServiceTest] ak: {}, sk: {}", accessKeyId, secretAccessKey); + + if (org.apache.commons.lang3.StringUtils.isAnyEmpty(accessKeyId, secretAccessKey)) { + return Optional.empty(); + } + + try { + AliOssService aliOssService = new AliOssService(); + aliOssService.initOssClient("oss-cn-beijing.aliyuncs.com", bucket, AliOssService.CredentialType.ENV.getCode(), null, null, null); + return Optional.of(aliOssService); + } catch (Exception e) { + ExceptionUtils.rethrow(e); + } + return Optional.empty(); + } +} \ No newline at end of file From 1c70bbc6706dff8c88704430197b47557a3ee804 Mon Sep 17 00:00:00 2001 From: tjq Date: Sun, 30 Jul 2023 21:14:18 +0800 Subject: [PATCH 11/24] feat: [storageExt] use PropertyAndOneBeanCondition to control multi impl --- .../PropertyAndOneBeanCondition.java | 95 +++++++++++++++++++ .../server/common/spring/package-info.java | 7 ++ .../storage/AbstractDFsService.java | 25 +++-- .../storage/StorageConfiguration.java | 37 ++++++++ .../storage/impl/AliOssService.java | 68 +++++++++---- .../storage/impl/EmptyDFsService.java | 41 ++++++-- .../storage/impl/GridFsService.java | 59 ++++++++---- .../storage/impl/GridFsServiceTest.java | 13 +++ 8 files changed, 290 insertions(+), 55 deletions(-) create mode 100644 powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/spring/condition/PropertyAndOneBeanCondition.java create mode 100644 powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/spring/package-info.java create mode 100644 powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/StorageConfiguration.java create mode 100644 powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/GridFsServiceTest.java diff --git a/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/spring/condition/PropertyAndOneBeanCondition.java b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/spring/condition/PropertyAndOneBeanCondition.java new file mode 100644 index 00000000..3b7e993c --- /dev/null +++ b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/spring/condition/PropertyAndOneBeanCondition.java @@ -0,0 +1,95 @@ +package tech.powerjob.server.common.spring.condition; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.NoSuchBeanDefinitionException; +import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.env.Environment; +import org.springframework.core.type.AnnotatedTypeMetadata; +import tech.powerjob.common.utils.CollectionUtils; + +import java.util.List; + +/** + * PropertyAndOneBeanCondition + * 存在多个接口实现时的唯一规则 + * + * @author tjq + * @since 2023/7/30 + */ +@Slf4j +public abstract class PropertyAndOneBeanCondition implements Condition { + + /** + * 配置中存在任意一个 Key 即可加载该 Bean，空代表不校验 + * @return Keys + */ + protected abstract List anyConfigKey(); + + /** + * Bean 唯一性校验，空代表不校验 + * @return beanType + */ + protected abstract Class beanType(); + + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + + boolean anyCfgExist = checkAnyConfigExist(context); + log.info("[PropertyAndOneBeanCondition] [{}] check any config exist result with keys={}: {}", thisName(), anyConfigKey(), anyCfgExist); + if (!anyCfgExist) { + return false; + } + + Class beanType = beanType(); + if (beanType == null) { + return true; + } + boolean exist = checkBeanExist(context); + log.info("[PropertyAndOneBeanCondition] [{}] bean of type[{}] exist check result: {}", thisName(), beanType.getSimpleName(), exist); + if (exist) { + log.info("[PropertyAndOneBeanCondition] [{}] bean of type[{}] already exist, skip load!", thisName(), beanType.getSimpleName()); + return false; + } + return true; + } + + private boolean checkAnyConfigExist(ConditionContext context) { + Environment environment = context.getEnvironment(); + + List keys = anyConfigKey(); + + if (CollectionUtils.isEmpty(keys)) { + return true; + } + + // 判断前缀是否符合，任意满足即可 + for (String key : keys) { + if (StringUtils.isNotEmpty(environment.getProperty(key))) { + return true; + } + } + + return false; + } + + private boolean checkBeanExist(ConditionContext context) { + + ConfigurableListableBeanFactory beanFactory = context.getBeanFactory(); + if (beanFactory == null) { + return false; + } + try { + beanFactory.getBean(beanType()); + return true; + } catch (NoSuchBeanDefinitionException ignore) { + return false; + } + } + + private String thisName() { + return this.getClass().getSimpleName(); + } +} diff --git a/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/spring/package-info.java b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/spring/package-info.java new file mode 100644 index 00000000..c354be2f --- /dev/null +++ b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/spring/package-info.java @@ -0,0 +1,7 @@ +/** + * Spring 通用能力包 + * + * @author tjq + * @since 2023/7/30 + */ +package tech.powerjob.server.common.spring; \ No newline at end of file diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/AbstractDFsService.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/AbstractDFsService.java index 3bbdd83c..4ce3c72e 100644 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/AbstractDFsService.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/AbstractDFsService.java @@ -1,27 +1,38 @@ package tech.powerjob.server.persistence.storage; -import org.springframework.beans.factory.InitializingBean; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; import org.springframework.core.env.Environment; import tech.powerjob.server.extension.dfs.DFsService; -import javax.annotation.Resource; - /** * AbstractDFsService * * @author tjq * @since 2023/7/28 */ -public abstract class AbstractDFsService implements DFsService, InitializingBean { +@Slf4j +public abstract class AbstractDFsService implements DFsService, ApplicationContextAware, DisposableBean { - @Resource - protected Environment environment; + public AbstractDFsService() { + log.info("[DFsService] invoke [{}]'s constructor", this.getClass().getName()); + } + + abstract protected void init(ApplicationContext applicationContext); protected static final String PROPERTY_KEY = "oms.storage.dfs"; - protected String fetchProperty(String dfsType, String key) { + protected static String fetchProperty(Environment environment, String dfsType, String key) { String pKey = String.format("%s.%s.%s", PROPERTY_KEY, dfsType, key); return environment.getProperty(pKey); } + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + log.info("[DFsService] invoke [{}]'s setApplicationContext", this.getClass().getName()); + init(applicationContext); + } } diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/StorageConfiguration.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/StorageConfiguration.java new file mode 100644 index 00000000..f99767e4 --- /dev/null +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/StorageConfiguration.java @@ -0,0 +1,37 @@ +package tech.powerjob.server.persistence.storage; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; +import tech.powerjob.server.extension.dfs.DFsService; +import tech.powerjob.server.persistence.storage.impl.AliOssService; +import tech.powerjob.server.persistence.storage.impl.EmptyDFsService; +import tech.powerjob.server.persistence.storage.impl.GridFsService; + +/** + * Description + * + * @author tjq + * @since 2023/7/30 + */ +@Configuration +public class StorageConfiguration { + + @Bean + @Conditional(GridFsService.GridFsCondition.class) + public DFsService initGridFs() { + return new GridFsService(); + } + + @Bean + @Conditional(AliOssService.AliOssCondition.class) + public DFsService initAliOssFs() { + return new AliOssService(); + } + + @Bean + @Conditional(EmptyDFsService.EmptyCondition.class) + public DFsService initEmptyDfs() { + return new EmptyDFsService(); + } +} diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/AliOssService.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/AliOssService.java index 7bcd65f1..7ae36243 100644 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/AliOssService.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/AliOssService.java @@ -1,12 +1,15 @@ package tech.powerjob.server.persistence.storage.impl; -import com.aliyun.oss.*; +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClientBuilder; +import com.aliyun.oss.OSSException; import com.aliyun.oss.common.auth.CredentialsProvider; import com.aliyun.oss.common.auth.CredentialsProviderFactory; import com.aliyun.oss.common.auth.DefaultCredentialProvider; import com.aliyun.oss.model.DownloadFileRequest; import com.aliyun.oss.model.ObjectMetadata; import com.aliyun.oss.model.PutObjectRequest; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import lombok.AllArgsConstructor; import lombok.Getter; @@ -14,13 +17,16 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Service; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Conditional; +import org.springframework.core.env.Environment; import tech.powerjob.server.extension.dfs.*; import tech.powerjob.server.persistence.storage.AbstractDFsService; +import tech.powerjob.server.common.spring.condition.PropertyAndOneBeanCondition; +import javax.annotation.Priority; import java.io.IOException; +import java.util.List; import java.util.Map; import java.util.Optional; @@ -39,9 +45,8 @@ import java.util.Optional; * @since 2023/7/30 */ @Slf4j -@Service -@ConditionalOnProperty(name = {"oms.storage.dfs.alioss.endpoint"}, matchIfMissing = false) -@ConditionalOnMissingBean(DFsService.class) +@Priority(value = Integer.MAX_VALUE - 1) +@Conditional(AliOssService.AliOssCondition.class) public class AliOssService extends AbstractDFsService { private static final String TYPE_ALI_OSS = "alioss"; @@ -60,19 +65,6 @@ public class AliOssService extends AbstractDFsService { private static final String NO_SUCH_KEY = "NoSuchKey"; - @Override - public void afterPropertiesSet() throws Exception { - - String endpoint = fetchProperty(TYPE_ALI_OSS, KEY_ENDPOINT); - String bkt = fetchProperty(TYPE_ALI_OSS, KEY_BUCKET); - String ct = fetchProperty(TYPE_ALI_OSS, KEY_CREDENTIAL_TYPE); - String ak = fetchProperty(TYPE_ALI_OSS, KEY_AK); - String sk = fetchProperty(TYPE_ALI_OSS, KEY_SK); - String token = fetchProperty(TYPE_ALI_OSS, KEY_TOKEN); - - initOssClient(endpoint, bkt, ct, ak, sk, token); - } - @Override public void store(StoreRequest storeRequest) throws IOException { @@ -162,6 +154,29 @@ public class AliOssService extends AbstractDFsService { */ } + @Override + public void destroy() throws Exception { + oss.shutdown(); + } + + @Override + protected void init(ApplicationContext applicationContext) { + Environment environment = applicationContext.getEnvironment(); + + String endpoint = fetchProperty(environment, TYPE_ALI_OSS, KEY_ENDPOINT); + String bkt = fetchProperty(environment, TYPE_ALI_OSS, KEY_BUCKET); + String ct = fetchProperty(environment, TYPE_ALI_OSS, KEY_CREDENTIAL_TYPE); + String ak = fetchProperty(environment, TYPE_ALI_OSS, KEY_AK); + String sk = fetchProperty(environment, TYPE_ALI_OSS, KEY_SK); + String token = fetchProperty(environment, TYPE_ALI_OSS, KEY_TOKEN); + + try { + initOssClient(endpoint, bkt, ct, ak, sk, token); + } catch (Exception e) { + ExceptionUtils.rethrow(e); + } + } + @Getter @AllArgsConstructor enum CredentialType { @@ -198,4 +213,17 @@ public class AliOssService extends AbstractDFsService { } } + + public static class AliOssCondition extends PropertyAndOneBeanCondition { + + @Override + protected List anyConfigKey() { + return Lists.newArrayList("oms.storage.dfs.alioss.endpoint"); + } + + @Override + protected Class beanType() { + return DFsService.class; + } + } } diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/EmptyDFsService.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/EmptyDFsService.java index c0b54d7f..9f464898 100644 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/EmptyDFsService.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/EmptyDFsService.java @@ -1,12 +1,14 @@ package tech.powerjob.server.persistence.storage.impl; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.core.Ordered; -import org.springframework.core.annotation.Order; -import org.springframework.stereotype.Service; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Conditional; import tech.powerjob.server.extension.dfs.*; +import tech.powerjob.server.persistence.storage.AbstractDFsService; +import tech.powerjob.server.common.spring.condition.PropertyAndOneBeanCondition; +import javax.annotation.Priority; import java.io.IOException; +import java.util.List; import java.util.Optional; /** @@ -15,13 +17,10 @@ import java.util.Optional; * @author tjq * @since 2023/7/30 */ -@Service -@Order(value = Ordered.LOWEST_PRECEDENCE) -@ConditionalOnMissingBean(DFsService.class) -public class EmptyDFsService implements DFsService { +@Priority(value = Integer.MAX_VALUE) +@Conditional(EmptyDFsService.EmptyCondition.class) +public class EmptyDFsService extends AbstractDFsService { - public EmptyDFsService() { - } @Override public void store(StoreRequest storeRequest) throws IOException { @@ -35,4 +34,26 @@ public class EmptyDFsService implements DFsService { public Optional fetchFileMeta(FileLocation fileLocation) throws IOException { return Optional.empty(); } + + @Override + public void destroy() throws Exception { + } + + @Override + protected void init(ApplicationContext applicationContext) { + + } + + + public static class EmptyCondition extends PropertyAndOneBeanCondition { + @Override + protected List anyConfigKey() { + return null; + } + + @Override + protected Class beanType() { + return DFsService.class; + } + } } diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java index 7bb01464..d78da3fa 100644 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java @@ -1,6 +1,7 @@ package tech.powerjob.server.persistence.storage.impl; import com.google.common.base.Stopwatch; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.mongodb.ConnectionString; import com.mongodb.client.MongoClient; @@ -17,17 +18,20 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.DateUtils; import org.bson.conversions.Bson; import org.bson.types.ObjectId; -import org.springframework.beans.factory.InitializingBean; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Conditional; import org.springframework.core.env.Environment; -import org.springframework.stereotype.Service; import tech.powerjob.server.extension.dfs.*; import tech.powerjob.server.persistence.storage.AbstractDFsService; +import tech.powerjob.server.common.spring.condition.PropertyAndOneBeanCondition; -import java.io.*; +import javax.annotation.Priority; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.IOException; import java.nio.file.Files; import java.util.Date; +import java.util.List; import java.util.Map; import java.util.Optional; @@ -39,15 +43,17 @@ import java.util.Optional; * @since 2023/7/28 */ @Slf4j -@Service -@ConditionalOnProperty(name = {"oms.storage.dfs.mongodb.uri", "spring.data.mongodb.uri"}, matchIfMissing = false) -@ConditionalOnMissingBean(DFsService.class) -public class GridFsService extends AbstractDFsService implements InitializingBean { +@Priority(value = Integer.MAX_VALUE - 10) +@Conditional(GridFsService.GridFsCondition.class) +public class GridFsService extends AbstractDFsService { + private MongoClient mongoClient; private MongoDatabase db; private final Map bucketCache = Maps.newConcurrentMap(); private static final String TYPE_MONGO = "mongodb"; + private static final String KEY_URI = "uri"; + private static final String SPRING_MONGO_DB_CONFIG_KEY = "spring.data.mongodb.uri"; @Override @@ -108,19 +114,13 @@ public class GridFsService extends AbstractDFsService implements InitializingBea log.info("[GridFsService] clean bucket({}) successfully, delete all files before {}, using {}.", bucketName, date, sw.stop()); } - @Override - public void afterPropertiesSet() throws Exception { - String uri = parseMongoUri(); - initMongo(uri); - } - private GridFSBucket getBucket(String bucketName) { return bucketCache.computeIfAbsent(bucketName, ignore -> GridFSBuckets.create(db, bucketName)); } - private String parseMongoUri() { + private String parseMongoUri(Environment environment) { // 优先从新的规则读取 - String uri = fetchProperty(TYPE_MONGO, "uri"); + String uri = fetchProperty(environment, TYPE_MONGO, KEY_URI); if (StringUtils.isNotEmpty(uri)) { return uri; } @@ -136,9 +136,32 @@ public class GridFsService extends AbstractDFsService implements InitializingBea } ConnectionString connectionString = new ConnectionString(uri); - MongoClient mongoClient = MongoClients.create(connectionString); + mongoClient = MongoClients.create(connectionString); db = mongoClient.getDatabase(Optional.ofNullable(connectionString.getDatabase()).orElse("pj")); log.info("[GridFsService] turn on mongodb GridFs as storage layer."); } + + @Override + public void destroy() throws Exception { + mongoClient.close(); + } + + @Override + protected void init(ApplicationContext applicationContext) { + String uri = parseMongoUri(applicationContext.getEnvironment()); + initMongo(uri); + } + + public static class GridFsCondition extends PropertyAndOneBeanCondition { + @Override + protected List anyConfigKey() { + return Lists.newArrayList("spring.data.mongodb.uri", "oms.storage.dfs.mongo.uri"); + } + + @Override + protected Class beanType() { + return DFsService.class; + } + } } diff --git a/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/GridFsServiceTest.java b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/GridFsServiceTest.java new file mode 100644 index 00000000..8212748c --- /dev/null +++ b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/GridFsServiceTest.java @@ -0,0 +1,13 @@ +package tech.powerjob.server.persistence.storage.impl; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * test GridFS + * + * @author tjq + * @since 2023/7/30 + */ +class GridFsServiceTest { + +} \ No newline at end of file From d03247ea0382bd54d7df18cf01e850c31377af7e Mon Sep 17 00:00:00 2001 From: tjq Date: Sun, 30 Jul 2023 21:59:35 +0800 Subject: [PATCH 12/24] test: [storageExt] finished gridfs service's test --- .../storage/impl/GridFsService.java | 13 ++- .../storage/impl/AbstractDfsServiceTest.java | 82 +++++++++++++++++++ .../storage/impl/AliOssServiceTest.java | 75 +---------------- .../storage/impl/GridFsServiceTest.java | 35 +++++++- 4 files changed, 129 insertions(+), 76 deletions(-) create mode 100644 powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AbstractDfsServiceTest.java diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java index d78da3fa..facbc2f9 100644 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java @@ -14,6 +14,7 @@ import com.mongodb.client.gridfs.GridFSFindIterable; import com.mongodb.client.gridfs.model.GridFSFile; import com.mongodb.client.model.Filters; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.DateUtils; import org.bson.conversions.Bson; @@ -67,6 +68,7 @@ public class GridFsService extends AbstractDFsService { @Override public void download(DownloadRequest downloadRequest) throws IOException { GridFSBucket bucket = getBucket(downloadRequest.getFileLocation().getBucket()); + FileUtils.forceMkdirParent(downloadRequest.getTarget()); try (GridFSDownloadStream gis = bucket.openDownloadStream(downloadRequest.getFileLocation().getName()); BufferedOutputStream bos = new BufferedOutputStream(Files.newOutputStream(downloadRequest.getTarget().toPath())) ) { @@ -128,7 +130,7 @@ public class GridFsService extends AbstractDFsService { return environment.getProperty(SPRING_MONGO_DB_CONFIG_KEY); } - private void initMongo(String uri) { + void initMongo(String uri) { log.info("[GridFsService] mongoDB uri: {}", uri); if (StringUtils.isEmpty(uri)) { log.warn("[GridFsService] uri is empty, GridFsService is off now!"); @@ -137,9 +139,14 @@ public class GridFsService extends AbstractDFsService { ConnectionString connectionString = new ConnectionString(uri); mongoClient = MongoClients.create(connectionString); - db = mongoClient.getDatabase(Optional.ofNullable(connectionString.getDatabase()).orElse("pj")); - log.info("[GridFsService] turn on mongodb GridFs as storage layer."); + if (StringUtils.isEmpty(connectionString.getDatabase())) { + log.warn("[GridFsService] can't find database info from uri, will use [powerjob] as default, please make sure you have created the database 'powerjob'"); + } + + db = mongoClient.getDatabase(Optional.ofNullable(connectionString.getDatabase()).orElse("powerjob")); + + log.info("[GridFsService] initialize MongoDB and GridFS successfully, will use mongodb GridFs as storage layer."); } @Override diff --git a/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AbstractDfsServiceTest.java b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AbstractDfsServiceTest.java new file mode 100644 index 00000000..bd15b482 --- /dev/null +++ b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AbstractDfsServiceTest.java @@ -0,0 +1,82 @@ +package tech.powerjob.server.persistence.storage.impl; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; +import tech.powerjob.common.serialize.JsonUtils; +import tech.powerjob.server.common.utils.OmsFileUtils; +import tech.powerjob.server.extension.dfs.*; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.util.Optional; + +/** + * AbstractDfsServiceTest + * + * @author tjq + * @since 2023/7/30 + */ +@Slf4j +public abstract class AbstractDfsServiceTest { + + abstract protected Optional fetchService(); + + @Test + void testBaseFileOperation() throws Exception { + + Optional aliOssServiceOpt = fetchService(); + if (!aliOssServiceOpt.isPresent()) { + return; + } + + DFsService aliOssService = aliOssServiceOpt.get(); + + String content = "wlcgyqsl"; + + String temporarySourcePath = OmsFileUtils.genTemporaryWorkPath() + "source.txt"; + String temporaryDownloadPath = OmsFileUtils.genTemporaryWorkPath() + "download.txt"; + + log.info("[testBaseFileOperation] temporarySourcePath: {}", temporarySourcePath); + File sourceFile = new File(temporarySourcePath); + FileUtils.forceMkdirParent(sourceFile); + OmsFileUtils.string2File(content, sourceFile); + + FileLocation fileLocation = new FileLocation().setBucket("pj_test").setName("testAliOss.txt"); + + StoreRequest storeRequest = new StoreRequest() + .setFileLocation(fileLocation) + .setLocalFile(sourceFile); + + // 存储 + aliOssService.store(storeRequest); + + // 读取 meta + Optional metaOpt = aliOssService.fetchFileMeta(fileLocation); + assert metaOpt.isPresent(); + + log.info("[testBaseFileOperation] file meta: {}", JsonUtils.toJSONString(metaOpt.get())); + + // 下载 + log.info("[testBaseFileOperation] temporaryDownloadPath: {}", temporaryDownloadPath); + File downloadFile = new File(temporaryDownloadPath); + DownloadRequest downloadRequest = new DownloadRequest() + .setFileLocation(fileLocation) + .setTarget(downloadFile); + aliOssService.download(downloadRequest); + + String downloadFileContent = FileUtils.readFileToString(downloadFile, StandardCharsets.UTF_8); + log.info("[testBaseFileOperation] download content: {}", downloadFileContent); + assert downloadFileContent.equals(content); + } + + @Test + void testFileNotExist() throws Exception { + Optional aliOssServiceOpt = fetchService(); + if (!aliOssServiceOpt.isPresent()) { + return; + } + Optional metaOpt = aliOssServiceOpt.get().fetchFileMeta(new FileLocation().setBucket("tjq").setName("yhz")); + assert !metaOpt.isPresent(); + } +} diff --git a/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AliOssServiceTest.java b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AliOssServiceTest.java index cbcdae13..94167731 100644 --- a/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AliOssServiceTest.java +++ b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AliOssServiceTest.java @@ -3,18 +3,9 @@ package tech.powerjob.server.persistence.storage.impl; import com.aliyun.oss.common.utils.AuthUtils; import com.aliyun.oss.common.utils.StringUtils; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.exception.ExceptionUtils; -import org.junit.jupiter.api.Test; -import tech.powerjob.common.serialize.JsonUtils; -import tech.powerjob.server.common.utils.OmsFileUtils; -import tech.powerjob.server.extension.dfs.DownloadRequest; -import tech.powerjob.server.extension.dfs.FileLocation; -import tech.powerjob.server.extension.dfs.FileMeta; -import tech.powerjob.server.extension.dfs.StoreRequest; +import tech.powerjob.server.extension.dfs.DFsService; -import java.io.File; -import java.nio.charset.StandardCharsets; import java.util.Optional; @@ -25,74 +16,16 @@ import java.util.Optional; * @since 2023/7/30 */ @Slf4j -class AliOssServiceTest { +class AliOssServiceTest extends AbstractDfsServiceTest { private static final String BUCKET = "power-job"; - @Test - void testBaseFileOperation() throws Exception { - - Optional aliOssServiceOpt = fetchService(); - if (!aliOssServiceOpt.isPresent()) { - return; - } - - AliOssService aliOssService = aliOssServiceOpt.get(); - - String content = "wlcgyqsl"; - - String temporarySourcePath = OmsFileUtils.genTemporaryWorkPath() + "source.txt"; - String temporaryDownloadPath = OmsFileUtils.genTemporaryWorkPath() + "download.txt"; - - log.info("[testBaseFileOperation] temporarySourcePath: {}", temporarySourcePath); - File sourceFile = new File(temporarySourcePath); - FileUtils.forceMkdirParent(sourceFile); - OmsFileUtils.string2File(content, sourceFile); - - FileLocation fileLocation = new FileLocation().setBucket("pj_test").setName("testAliOss.txt"); - - StoreRequest storeRequest = new StoreRequest() - .setFileLocation(fileLocation) - .setLocalFile(sourceFile); - - // 存储 - aliOssService.store(storeRequest); - - // 读取 meta - Optional metaOpt = aliOssService.fetchFileMeta(fileLocation); - assert metaOpt.isPresent(); - - log.info("[testBaseFileOperation] file meta: {}", JsonUtils.toJSONString(metaOpt.get())); - - // 下载 - log.info("[testBaseFileOperation] temporaryDownloadPath: {}", temporaryDownloadPath); - File downloadFile = new File(temporaryDownloadPath); - DownloadRequest downloadRequest = new DownloadRequest() - .setFileLocation(fileLocation) - .setTarget(downloadFile); - aliOssService.download(downloadRequest); - - String downloadFileContent = FileUtils.readFileToString(downloadFile, StandardCharsets.UTF_8); - log.info("[testBaseFileOperation] download content: {}", downloadFileContent); - assert downloadFileContent.equals(content); - } - - @Test - void testFileNotExist() throws Exception { - Optional aliOssServiceOpt = fetchService(); - if (!aliOssServiceOpt.isPresent()) { - return; - } - Optional metaOpt = aliOssServiceOpt.get().fetchFileMeta(new FileLocation().setBucket("tjq").setName("yhz")); - assert !metaOpt.isPresent(); - } - /** * 依赖阿里云账号密码测试，为了保证单测在其他环境也能通过，如果发现不存在配置则直接跳过 * @return AliOssService */ - private Optional fetchService() { - + @Override + protected Optional fetchService() { String accessKeyId = StringUtils.trim(System.getenv(AuthUtils.ACCESS_KEY_ENV_VAR)); String secretAccessKey = StringUtils.trim(System.getenv(AuthUtils.SECRET_KEY_ENV_VAR)); diff --git a/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/GridFsServiceTest.java b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/GridFsServiceTest.java index 8212748c..1abfba2d 100644 --- a/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/GridFsServiceTest.java +++ b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/GridFsServiceTest.java @@ -1,6 +1,14 @@ package tech.powerjob.server.persistence.storage.impl; -import static org.junit.jupiter.api.Assertions.*; +import com.alibaba.fastjson.JSONObject; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import tech.powerjob.server.extension.dfs.DFsService; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.util.Optional; /** * test GridFS @@ -8,6 +16,29 @@ import static org.junit.jupiter.api.Assertions.*; * @author tjq * @since 2023/7/30 */ -class GridFsServiceTest { +@Slf4j +class GridFsServiceTest extends AbstractDfsServiceTest { + @Override + protected Optional fetchService() { + + try { + // 后续本地测试，密钥相关的内容统一存入 .powerjob_test 中，方便管理 + String content = FileUtils.readFileToString(new File(System.getProperty("user.home").concat("/.powerjob_test")), StandardCharsets.UTF_8); + if (StringUtils.isNotEmpty(content)) { + JSONObject jsonObject = JSONObject.parseObject(content); + Object mongoUri = jsonObject.get("mongoUri"); + if (mongoUri != null) { + GridFsService gridFsService = new GridFsService(); + gridFsService.initMongo(String.valueOf(mongoUri)); + + return Optional.of(gridFsService); + } + } + } catch (Exception e) { + e.printStackTrace(); + log.warn("[GridFsServiceTest] fetch mongo config failed, skip!"); + } + return Optional.empty(); + } } \ No newline at end of file From c3ce46aee9976fe5179493d91f37be75aff65e39 Mon Sep 17 00:00:00 2001 From: tjq Date: Mon, 31 Jul 2023 23:45:52 +0800 Subject: [PATCH 13/24] refactor: optimize alarm code --- powerjob-server/pom.xml | 9 +++ .../powerjob-server-common/pom.xml | 7 ++ .../server/common/utils/TestUtils.java | 42 +++++++++++ .../server/core/alarm/AlarmCenter.java | 28 +++---- .../server/core/alarm/AlarmUtils.java | 35 +++++++++ .../core/alarm/module/JobInstanceAlarm.java | 2 + .../server/core/instance/InstanceManager.java | 3 +- .../workflow/WorkflowInstanceManager.java | 3 +- .../server/extension/alarm/AlarmTarget.java | 2 + .../storage/impl/GridFsService.java | 2 +- .../storage/impl/GridFsServiceTest.java | 30 +++----- .../server/web/controller/TestController.java | 75 +++++++++++++++++++ .../resources/application-daily.properties | 3 +- .../main/resources/application-pre.properties | 3 +- .../resources/application-product.properties | 3 +- 15 files changed, 199 insertions(+), 48 deletions(-) create mode 100644 powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/utils/TestUtils.java create mode 100644 powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/AlarmUtils.java create mode 100644 powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/web/controller/TestController.java diff --git a/powerjob-server/pom.xml b/powerjob-server/pom.xml index 6c762e93..a75f4158 100644 --- a/powerjob-server/pom.xml +++ b/powerjob-server/pom.xml @@ -55,6 +55,7 @@ 4.3.3 1.6.14 3.17.1 + 4.4 @@ -113,6 +114,14 @@ ${aliyun-sdk-oss.version} + + + org.apache.commons + commons-collections4 + ${commons-collections4.version} + + + diff --git a/powerjob-server/powerjob-server-common/pom.xml b/powerjob-server/powerjob-server-common/pom.xml index bbff066f..2cd23f6c 100644 --- a/powerjob-server/powerjob-server-common/pom.xml +++ b/powerjob-server/powerjob-server-common/pom.xml @@ -18,4 +18,11 @@ 8 + + + org.apache.commons + commons-collections4 + + + \ No newline at end of file diff --git a/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/utils/TestUtils.java b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/utils/TestUtils.java new file mode 100644 index 00000000..1440b6e0 --- /dev/null +++ b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/utils/TestUtils.java @@ -0,0 +1,42 @@ +package tech.powerjob.server.common.utils; + +import com.alibaba.fastjson.JSONObject; +import com.google.common.collect.Maps; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +/** + * 开发团队专用测试工具 + * + * @author tjq + * @since 2023/7/31 + */ +public class TestUtils { + + private static final String TEST_CONFIG_NAME = "/.powerjob_test"; + + public static final String KEY_PHONE_NUMBER = "phone"; + + public static final String KEY_MONGO_URI = "mongoUri"; + + /** + * 获取本地的测试配置，主要用于存放一些密钥 + * @return 测试配置 + */ + public static Map fetchTestConfig() { + try { + // 后续本地测试，密钥相关的内容统一存入 .powerjob_test 中，方便管理 + String content = FileUtils.readFileToString(new File(System.getProperty("user.home").concat(TEST_CONFIG_NAME)), StandardCharsets.UTF_8); + if (StringUtils.isNotEmpty(content)) { + return JSONObject.parseObject(content); + } + } catch (Exception ignore) { + } + return Maps.newHashMap(); + } + +} diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/AlarmCenter.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/AlarmCenter.java index cb837fdc..bde075a9 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/AlarmCenter.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/AlarmCenter.java @@ -1,19 +1,19 @@ package tech.powerjob.server.core.alarm; -import org.springframework.beans.BeanUtils; -import org.springframework.stereotype.Component; -import tech.powerjob.server.extension.alarm.Alarm; -import tech.powerjob.server.extension.alarm.AlarmTarget; -import tech.powerjob.server.extension.alarm.Alarmable; -import tech.powerjob.server.persistence.remote.model.UserInfoDO; import com.google.common.collect.Lists; import com.google.common.collect.Queues; import com.google.common.util.concurrent.ThreadFactoryBuilder; import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import tech.powerjob.server.extension.alarm.Alarm; +import tech.powerjob.server.extension.alarm.AlarmTarget; +import tech.powerjob.server.extension.alarm.Alarmable; import java.util.List; -import java.util.concurrent.*; -import java.util.stream.Collectors; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** * 报警服务 @@ -40,21 +40,13 @@ public class AlarmCenter { }); } - public void alarmFailed(Alarm alarm, List targetUserList) { + public void alarmFailed(Alarm alarm, List alarmTargets) { POOL.execute(() -> BEANS.forEach(alarmable -> { try { - alarmable.onFailed(alarm, targetUserList.stream().map(AlarmCenter::convertUserInfo2AlarmTarget).collect(Collectors.toList())); + alarmable.onFailed(alarm, alarmTargets); }catch (Exception e) { log.warn("[AlarmCenter] alarm failed.", e); } })); } - - private static AlarmTarget convertUserInfo2AlarmTarget(UserInfoDO userInfoDO) { - AlarmTarget alarmTarget = new AlarmTarget(); - BeanUtils.copyProperties(userInfoDO, alarmTarget); - - alarmTarget.setName(userInfoDO.getUsername()); - return alarmTarget; - } } diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/AlarmUtils.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/AlarmUtils.java new file mode 100644 index 00000000..2db55b42 --- /dev/null +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/AlarmUtils.java @@ -0,0 +1,35 @@ +package tech.powerjob.server.core.alarm; + +import org.springframework.beans.BeanUtils; +import tech.powerjob.common.utils.CollectionUtils; +import tech.powerjob.server.extension.alarm.AlarmTarget; +import tech.powerjob.server.persistence.remote.model.UserInfoDO; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +/** + * AlarmUtils + * + * @author tjq + * @since 2023/7/31 + */ +public class AlarmUtils { + + public static List convertUserInfoList2AlarmTargetList(List userInfoDOS) { + if (CollectionUtils.isEmpty(userInfoDOS)) { + return Collections.emptyList(); + } + return userInfoDOS.stream().map(AlarmUtils::convertUserInfo2AlarmTarget).collect(Collectors.toList()); + } + + public static AlarmTarget convertUserInfo2AlarmTarget(UserInfoDO userInfoDO) { + AlarmTarget alarmTarget = new AlarmTarget(); + BeanUtils.copyProperties(userInfoDO, alarmTarget); + + alarmTarget.setName(userInfoDO.getUsername()); + return alarmTarget; + } + +} diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/module/JobInstanceAlarm.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/module/JobInstanceAlarm.java index 5c2d907a..b4d4f400 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/module/JobInstanceAlarm.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/module/JobInstanceAlarm.java @@ -1,6 +1,7 @@ package tech.powerjob.server.core.alarm.module; import lombok.Data; +import lombok.experimental.Accessors; import tech.powerjob.server.extension.alarm.Alarm; /** @@ -10,6 +11,7 @@ import tech.powerjob.server.extension.alarm.Alarm; * @since 2020/4/30 */ @Data +@Accessors(chain = true) public class JobInstanceAlarm implements Alarm { /** * 应用ID diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceManager.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceManager.java index 274a8a5b..36511e46 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceManager.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceManager.java @@ -15,6 +15,7 @@ import tech.powerjob.remote.framework.base.URL; import tech.powerjob.server.common.module.WorkerInfo; import tech.powerjob.server.common.timewheel.holder.HashedWheelTimerHolder; import tech.powerjob.server.common.utils.SpringUtils; +import tech.powerjob.server.core.alarm.AlarmUtils; import tech.powerjob.server.core.service.UserService; import tech.powerjob.server.core.workflow.WorkflowInstanceManager; import tech.powerjob.server.core.alarm.AlarmCenter; @@ -239,7 +240,7 @@ public class InstanceManager implements TransportServiceAware { if (!StringUtils.isEmpty(alertContent)) { content.setResult(alertContent); } - alarmCenter.alarmFailed(content, userList); + alarmCenter.alarmFailed(content, AlarmUtils.convertUserInfoList2AlarmTargetList(userList)); } @Override diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/workflow/WorkflowInstanceManager.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/workflow/WorkflowInstanceManager.java index 001c3e12..2c6cdd78 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/workflow/WorkflowInstanceManager.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/workflow/WorkflowInstanceManager.java @@ -20,6 +20,7 @@ import tech.powerjob.common.serialize.JsonUtils; import tech.powerjob.common.utils.CommonUtils; import tech.powerjob.server.common.constants.SwitchableStatus; import tech.powerjob.server.common.utils.SpringUtils; +import tech.powerjob.server.core.alarm.AlarmUtils; import tech.powerjob.server.core.helper.StatusMappingHelper; import tech.powerjob.server.core.lock.UseCacheLock; import tech.powerjob.server.core.service.UserService; @@ -458,7 +459,7 @@ public class WorkflowInstanceManager { content.setResult(result); List userList = userService.fetchNotifyUserList(wfInfo.getNotifyUserIds()); - alarmCenter.alarmFailed(content, userList); + alarmCenter.alarmFailed(content, AlarmUtils.convertUserInfoList2AlarmTargetList(userList)); }); } catch (Exception ignore) { // ignore diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/alarm/AlarmTarget.java b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/alarm/AlarmTarget.java index 6512f6f7..982ba398 100644 --- a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/alarm/AlarmTarget.java +++ b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/alarm/AlarmTarget.java @@ -1,6 +1,7 @@ package tech.powerjob.server.extension.alarm; import lombok.Data; +import lombok.experimental.Accessors; import java.io.Serializable; import java.util.Map; @@ -13,6 +14,7 @@ import java.util.Objects; * @since 2023/7/16 */ @Data +@Accessors(chain = true) public class AlarmTarget implements Serializable { private String name; diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java index facbc2f9..c5c4f70c 100644 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java @@ -163,7 +163,7 @@ public class GridFsService extends AbstractDFsService { public static class GridFsCondition extends PropertyAndOneBeanCondition { @Override protected List anyConfigKey() { - return Lists.newArrayList("spring.data.mongodb.uri", "oms.storage.dfs.mongo.uri"); + return Lists.newArrayList("spring.data.mongodb.uri", "oms.storage.dfs.mongodb.uri"); } @Override diff --git a/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/GridFsServiceTest.java b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/GridFsServiceTest.java index 1abfba2d..ba0d7c2e 100644 --- a/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/GridFsServiceTest.java +++ b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/GridFsServiceTest.java @@ -1,13 +1,9 @@ package tech.powerjob.server.persistence.storage.impl; -import com.alibaba.fastjson.JSONObject; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; +import tech.powerjob.server.common.utils.TestUtils; import tech.powerjob.server.extension.dfs.DFsService; -import java.io.File; -import java.nio.charset.StandardCharsets; import java.util.Optional; /** @@ -22,23 +18,15 @@ class GridFsServiceTest extends AbstractDfsServiceTest { @Override protected Optional fetchService() { - try { - // 后续本地测试，密钥相关的内容统一存入 .powerjob_test 中，方便管理 - String content = FileUtils.readFileToString(new File(System.getProperty("user.home").concat("/.powerjob_test")), StandardCharsets.UTF_8); - if (StringUtils.isNotEmpty(content)) { - JSONObject jsonObject = JSONObject.parseObject(content); - Object mongoUri = jsonObject.get("mongoUri"); - if (mongoUri != null) { - GridFsService gridFsService = new GridFsService(); - gridFsService.initMongo(String.valueOf(mongoUri)); + Object mongoUri = TestUtils.fetchTestConfig().get(TestUtils.KEY_MONGO_URI); - return Optional.of(gridFsService); - } - } - } catch (Exception e) { - e.printStackTrace(); - log.warn("[GridFsServiceTest] fetch mongo config failed, skip!"); + if (mongoUri == null) { + log.info("[GridFsServiceTest] mongoUri is null, skip load!"); + return Optional.empty(); } - return Optional.empty(); + + GridFsService gridFsService = new GridFsService(); + gridFsService.initMongo(String.valueOf(mongoUri)); + return Optional.of(gridFsService); } } \ No newline at end of file diff --git a/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/web/controller/TestController.java b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/web/controller/TestController.java new file mode 100644 index 00000000..c7fea8be --- /dev/null +++ b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/web/controller/TestController.java @@ -0,0 +1,75 @@ +package tech.powerjob.server.web.controller; + +import com.google.common.collect.Lists; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.MapUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import tech.powerjob.common.serialize.JsonUtils; +import tech.powerjob.common.utils.CollectionUtils; +import tech.powerjob.server.common.utils.TestUtils; +import tech.powerjob.server.core.alarm.AlarmCenter; +import tech.powerjob.server.core.alarm.module.JobInstanceAlarm; +import tech.powerjob.server.extension.alarm.AlarmTarget; + +import javax.annotation.Resource; +import java.util.Map; + +/** + * 开发团队专用（或者 PRO 用户用来做自检也可以 lol） + * 测试某些强依赖运行时环境的组件，如 Mail 告警等 + * + * @author tjq + * @since 2023/7/31 + */ +@Slf4j +@RestController +@RequestMapping("/test") +public class TestController { + + @Value("${server.port}") + private int port; + + @Resource + private AlarmCenter alarmCenter; + + @RequestMapping("/io") + public Map io(@RequestBody Map input) { + log.info("[TestController] input: {}", JsonUtils.toJSONString(input)); + return input; + } + + @GetMapping("/check") + public void check() { + Map testConfig = TestUtils.fetchTestConfig(); + if (CollectionUtils.isEmpty(testConfig)) { + log.info("[TestController] testConfig not exist, skip check!"); + return; + } + + log.info("[TestController] testConfig: {}", JsonUtils.toJSONString(testConfig)); + + testAlarmCenter(); + } + + void testAlarmCenter() { + JobInstanceAlarm jobInstanceAlarm = new JobInstanceAlarm().setAppId(277).setJobId(1).setInstanceId(2) + .setJobName("test-alarm").setJobParams("jobParams").setInstanceParams("instanceParams") + .setExecuteType(1).setFinishedTime(System.currentTimeMillis()); + + AlarmTarget target = new AlarmTarget().setName("ald").setPhone("208140").setExtra("extra") + .setPhone(MapUtils.getString(TestUtils.fetchTestConfig(), TestUtils.KEY_PHONE_NUMBER)) + .setEmail("tjq@zju.edu.cn") + .setWebHook(localUrlPath().concat("/test/io")); + + log.info("[TestController] start to testAlarmCenter, target: {}", target); + alarmCenter.alarmFailed(jobInstanceAlarm, Lists.newArrayList(target)); + } + + private String localUrlPath() { + return String.format("http://127.0.0.1:%d", port); + } +} diff --git a/powerjob-server/powerjob-server-starter/src/main/resources/application-daily.properties b/powerjob-server/powerjob-server-starter/src/main/resources/application-daily.properties index 693186e6..e61cd400 100644 --- a/powerjob-server/powerjob-server-starter/src/main/resources/application-daily.properties +++ b/powerjob-server/powerjob-server-starter/src/main/resources/application-daily.properties @@ -11,8 +11,7 @@ spring.datasource.core.minimum-idle=5 ####### MongoDB properties(Non-core configuration properties) ####### ####### delete mongodb config to disable mongodb ####### -#oms.mongodb.enable=true -#spring.data.mongodb.uri=mongodb+srv://zqq:No1Bug2Please3!@cluster0.wie54.gcp.mongodb.net/powerjob_daily?retryWrites=true&w=majority +#oms.storage.dfs.mongodb.uri=mongodb+srv://zqq:No1Bug2Please3!@cluster0.wie54.gcp.mongodb.net/powerjob_daily?retryWrites=true&w=majority ####### Email properties(Non-core configuration properties) ####### ####### Delete the following code to disable the mail ####### diff --git a/powerjob-server/powerjob-server-starter/src/main/resources/application-pre.properties b/powerjob-server/powerjob-server-starter/src/main/resources/application-pre.properties index f08f673b..a19491e8 100644 --- a/powerjob-server/powerjob-server-starter/src/main/resources/application-pre.properties +++ b/powerjob-server/powerjob-server-starter/src/main/resources/application-pre.properties @@ -11,8 +11,7 @@ spring.datasource.core.minimum-idle=5 ####### MongoDB properties(Non-core configuration properties) ####### ####### delete mongodb config to disable mongodb ####### -oms.mongodb.enable=true -spring.data.mongodb.uri=mongodb://remotehost:27017/powerjob-pre +oms.storage.dfs.mongodb.uri=mongodb://remotehost:27017/powerjob-pre ####### Email properties(Non-core configuration properties) ####### ####### Delete the following code to disable the mail ####### diff --git a/powerjob-server/powerjob-server-starter/src/main/resources/application-product.properties b/powerjob-server/powerjob-server-starter/src/main/resources/application-product.properties index a3ed4fe1..53e47d17 100644 --- a/powerjob-server/powerjob-server-starter/src/main/resources/application-product.properties +++ b/powerjob-server/powerjob-server-starter/src/main/resources/application-product.properties @@ -11,8 +11,7 @@ spring.datasource.core.minimum-idle=5 ####### MongoDB properties(Non-core configuration properties) ####### ####### delete mongodb config to disable mongodb ####### -oms.mongodb.enable=true -spring.data.mongodb.uri=mongodb://localhost:27017/powerjob-product +oms.storage.dfs.mongodb.uri=mongodb://localhost:27017/powerjob-product ####### Email properties(Non-core configuration properties) ####### ####### Delete the following code to disable the mail ####### From dd3291663732dd334b9029ed99c354e9cc68a635 Mon Sep 17 00:00:00 2001 From: tjq Date: Fri, 4 Aug 2023 22:20:01 +0800 Subject: [PATCH 14/24] docs: add SECURITY.md #698 --- SECURITY.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..2ada85b7 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,4 @@ +# Security notices relating to PowerJob + +Please disclose any security issues or vulnerabilities found through [Tidelift's coordinated disclosure system](https://tidelift.com/security) or to the maintainers privately(tengjiqi@gmail.com). + From 570ea0487b14c3cce6dcb792b97cc60c3da32171 Mon Sep 17 00:00:00 2001 From: tjq Date: Sun, 6 Aug 2023 19:25:47 +0800 Subject: [PATCH 15/24] feat: output vertx request error --- .../tech/powerjob/remote/http/vertx/VertxTransporter.java | 7 ++++++- powerjob-server/docker/Dockerfile | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/powerjob-remote/powerjob-remote-impl-http/src/main/java/tech/powerjob/remote/http/vertx/VertxTransporter.java b/powerjob-remote/powerjob-remote-impl-http/src/main/java/tech/powerjob/remote/http/vertx/VertxTransporter.java index 862e1ce0..ddbd4be1 100644 --- a/powerjob-remote/powerjob-remote-impl-http/src/main/java/tech/powerjob/remote/http/vertx/VertxTransporter.java +++ b/powerjob-remote/powerjob-remote-impl-http/src/main/java/tech/powerjob/remote/http/vertx/VertxTransporter.java @@ -10,6 +10,8 @@ import io.vertx.core.http.HttpClientResponse; import io.vertx.core.http.HttpMethod; import io.vertx.core.http.RequestOptions; import io.vertx.core.json.JsonObject; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; import tech.powerjob.common.PowerSerializable; import tech.powerjob.remote.framework.base.RemotingException; import tech.powerjob.remote.framework.base.URL; @@ -25,6 +27,7 @@ import java.util.concurrent.CompletionStage; * @author tjq * @since 2023/1/1 */ +@Slf4j public class VertxTransporter implements Transporter { private final HttpClient httpClient; @@ -90,6 +93,8 @@ public class VertxTransporter implements Transporter { return Future.succeededFuture(x.toJsonObject().mapTo(clz)); }); - }).toCompletionStage(); + }) + .onFailure(t -> log.warn("[VertxTransporter] post to url[{}] failed,msg: {}", url, ExceptionUtils.getMessage(t))) + .toCompletionStage(); } } diff --git a/powerjob-server/docker/Dockerfile b/powerjob-server/docker/Dockerfile index 933d495b..606dd711 100644 --- a/powerjob-server/docker/Dockerfile +++ b/powerjob-server/docker/Dockerfile @@ -3,8 +3,8 @@ FROM adoptopenjdk:8-jdk-hotspot # 维护者 MAINTAINER tengjiqi@gmail.com # 下载并安装 maven -RUN curl -O https://mirrors.tuna.tsinghua.edu.cn/apache/maven/maven-3/3.9.2/binaries/apache-maven-3.9.2-bin.tar.gz -RUN tar -zxvf apache-maven-3.9.2-bin.tar.gz && mv apache-maven-3.9.2 /opt/powerjob-maven && rm -rf apache-maven-3.9.2-bin.tar.gz +RUN curl -O https://mirrors.aliyun.com/apache/maven/maven-3/3.9.4/binaries/apache-maven-3.9.4-bin.tar.gz +RUN tar -zxvf apache-maven-3.9.4-bin.tar.gz && mv apache-maven-3.9.4 /opt/powerjob-maven && rm -rf apache-maven-3.9.4-bin.tar.gz # 替换 maven 配置文件 RUN rm -rf /opt/powerjob-maven/conf/settings.xml COPY settings.xml /opt/powerjob-maven/conf/settings.xml From df5e259e54b069dd67bb6c20a99a5ca764ae999f Mon Sep 17 00:00:00 2001 From: tjq Date: Sun, 6 Aug 2023 20:17:40 +0800 Subject: [PATCH 16/24] chore: change main version to 4.3.4 --- others/dev/publish_docker.sh | 2 +- pom.xml | 2 +- powerjob-client/pom.xml | 6 +++--- powerjob-common/pom.xml | 4 ++-- powerjob-official-processors/pom.xml | 6 +++--- powerjob-remote/pom.xml | 2 +- powerjob-remote/powerjob-remote-benchmark/pom.xml | 6 +++--- powerjob-remote/powerjob-remote-framework/pom.xml | 6 +++--- powerjob-remote/powerjob-remote-impl-akka/pom.xml | 6 +++--- powerjob-remote/powerjob-remote-impl-http/pom.xml | 6 +++--- powerjob-server/pom.xml | 10 +++++----- powerjob-server/powerjob-server-common/pom.xml | 2 +- powerjob-server/powerjob-server-core/pom.xml | 2 +- powerjob-server/powerjob-server-extension/pom.xml | 2 +- powerjob-server/powerjob-server-migrate/pom.xml | 2 +- powerjob-server/powerjob-server-monitor/pom.xml | 2 +- powerjob-server/powerjob-server-persistence/pom.xml | 2 +- powerjob-server/powerjob-server-remote/pom.xml | 2 +- powerjob-server/powerjob-server-starter/pom.xml | 2 +- powerjob-worker-agent/pom.xml | 8 ++++---- powerjob-worker-samples/pom.xml | 8 ++++---- powerjob-worker-spring-boot-starter/pom.xml | 6 +++--- powerjob-worker/pom.xml | 12 ++++++------ 23 files changed, 53 insertions(+), 53 deletions(-) diff --git a/others/dev/publish_docker.sh b/others/dev/publish_docker.sh index ee295e3d..7b3da88e 100755 --- a/others/dev/publish_docker.sh +++ b/others/dev/publish_docker.sh @@ -90,7 +90,7 @@ if [ "$startup" = "y" ] || [ "$startup" = "Y" ]; then echo "================== 准备启动 powerjob-server ==================" docker run -d \ --name powerjob-server \ - -p 7700:7700 -p 10086:10086 -p 5001:5005 -p 10001:10000 \ + -p 7700:7700 -p 10086:10086 -p 10010:10010 -p 5001:5005 -p 10001:10000 \ -e JVMOPTIONS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=10000 -Dcom.sun.management.jmxremote.rmi.port=10000 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false" \ -e PARAMS="--spring.profiles.active=pre" \ -e TZ="Asia/Shanghai" \ diff --git a/pom.xml b/pom.xml index f946cf27..6008d9c4 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ tech.powerjob powerjob - 4.3.3 + 4.3.4 pom powerjob http://www.powerjob.tech diff --git a/powerjob-client/pom.xml b/powerjob-client/pom.xml index dfacf993..fea0a769 100644 --- a/powerjob-client/pom.xml +++ b/powerjob-client/pom.xml @@ -5,18 +5,18 @@ powerjob tech.powerjob - 4.3.3 + 4.3.4 4.0.0 powerjob-client - 4.3.3 + 4.3.4 jar 5.9.1 1.2.83 - 4.3.3 + 4.3.4 3.2.4 diff --git a/powerjob-common/pom.xml b/powerjob-common/pom.xml index 179357d0..a168f772 100644 --- a/powerjob-common/pom.xml +++ b/powerjob-common/pom.xml @@ -5,12 +5,12 @@ powerjob tech.powerjob - 4.3.3 + 4.3.4 4.0.0 powerjob-common - 4.3.3 + 4.3.4 jar diff --git a/powerjob-official-processors/pom.xml b/powerjob-official-processors/pom.xml index c4cc43a7..f3088354 100644 --- a/powerjob-official-processors/pom.xml +++ b/powerjob-official-processors/pom.xml @@ -5,12 +5,12 @@ powerjob tech.powerjob - 4.3.3 + 4.3.4 4.0.0 powerjob-official-processors - 4.3.3 + 4.3.4 jar @@ -20,7 +20,7 @@ 5.9.1 1.2.9 - 4.3.3 + 4.3.4 5.2.9.RELEASE 2.1.214 8.0.28 diff --git a/powerjob-remote/pom.xml b/powerjob-remote/pom.xml index e15ded3d..dbc9800c 100644 --- a/powerjob-remote/pom.xml +++ b/powerjob-remote/pom.xml @@ -5,7 +5,7 @@ powerjob tech.powerjob - 4.3.3 + 4.3.4 4.0.0 pom diff --git a/powerjob-remote/powerjob-remote-benchmark/pom.xml b/powerjob-remote/powerjob-remote-benchmark/pom.xml index de40102e..081a1082 100644 --- a/powerjob-remote/powerjob-remote-benchmark/pom.xml +++ b/powerjob-remote/powerjob-remote-benchmark/pom.xml @@ -5,7 +5,7 @@ powerjob-remote tech.powerjob - 4.3.3 + 4.3.4 4.0.0 @@ -21,8 +21,8 @@ 1.2.9 2.7.4 - 4.3.3 - 4.3.3 + 4.3.4 + 4.3.4 3.9.0 4.2.9 diff --git a/powerjob-remote/powerjob-remote-framework/pom.xml b/powerjob-remote/powerjob-remote-framework/pom.xml index 3735d44a..e2e86d76 100644 --- a/powerjob-remote/powerjob-remote-framework/pom.xml +++ b/powerjob-remote/powerjob-remote-framework/pom.xml @@ -5,11 +5,11 @@ powerjob-remote tech.powerjob - 4.3.3 + 4.3.4 4.0.0 - 4.3.3 + 4.3.4 powerjob-remote-framework @@ -17,7 +17,7 @@ 8 UTF-8 - 4.3.3 + 4.3.4 0.10.2 diff --git a/powerjob-remote/powerjob-remote-impl-akka/pom.xml b/powerjob-remote/powerjob-remote-impl-akka/pom.xml index 20248943..f511e5be 100644 --- a/powerjob-remote/powerjob-remote-impl-akka/pom.xml +++ b/powerjob-remote/powerjob-remote-impl-akka/pom.xml @@ -5,19 +5,19 @@ powerjob-remote tech.powerjob - 4.3.3 + 4.3.4 4.0.0 powerjob-remote-impl-akka - 4.3.3 + 4.3.4 8 8 UTF-8 - 4.3.3 + 4.3.4 2.6.13 diff --git a/powerjob-remote/powerjob-remote-impl-http/pom.xml b/powerjob-remote/powerjob-remote-impl-http/pom.xml index f0f2dcee..6a58f504 100644 --- a/powerjob-remote/powerjob-remote-impl-http/pom.xml +++ b/powerjob-remote/powerjob-remote-impl-http/pom.xml @@ -5,12 +5,12 @@ powerjob-remote tech.powerjob - 4.3.3 + 4.3.4 4.0.0 powerjob-remote-impl-http - 4.3.3 + 4.3.4 8 @@ -18,7 +18,7 @@ UTF-8 4.3.7 - 4.3.3 + 4.3.4 diff --git a/powerjob-server/pom.xml b/powerjob-server/pom.xml index a75f4158..fa515f27 100644 --- a/powerjob-server/pom.xml +++ b/powerjob-server/pom.xml @@ -5,12 +5,12 @@ powerjob tech.powerjob - 4.3.3 + 4.3.4 4.0.0 powerjob-server - 4.3.3 + 4.3.4 pom @@ -50,9 +50,9 @@ 3.0.10 9.1.6 - 4.3.3 - 4.3.3 - 4.3.3 + 4.3.4 + 4.3.4 + 4.3.4 1.6.14 3.17.1 4.4 diff --git a/powerjob-server/powerjob-server-common/pom.xml b/powerjob-server/powerjob-server-common/pom.xml index 2cd23f6c..946f8b24 100644 --- a/powerjob-server/powerjob-server-common/pom.xml +++ b/powerjob-server/powerjob-server-common/pom.xml @@ -5,7 +5,7 @@ powerjob-server tech.powerjob - 4.3.3 + 4.3.4 ../pom.xml 4.0.0 diff --git a/powerjob-server/powerjob-server-core/pom.xml b/powerjob-server/powerjob-server-core/pom.xml index 3473f27a..c85431a9 100644 --- a/powerjob-server/powerjob-server-core/pom.xml +++ b/powerjob-server/powerjob-server-core/pom.xml @@ -5,7 +5,7 @@ powerjob-server tech.powerjob - 4.3.3 + 4.3.4 ../pom.xml 4.0.0 diff --git a/powerjob-server/powerjob-server-extension/pom.xml b/powerjob-server/powerjob-server-extension/pom.xml index 89954a8b..6f90723c 100644 --- a/powerjob-server/powerjob-server-extension/pom.xml +++ b/powerjob-server/powerjob-server-extension/pom.xml @@ -5,7 +5,7 @@ powerjob-server tech.powerjob - 4.3.3 + 4.3.4 ../pom.xml 4.0.0 diff --git a/powerjob-server/powerjob-server-migrate/pom.xml b/powerjob-server/powerjob-server-migrate/pom.xml index 6d245066..f4f54b1c 100644 --- a/powerjob-server/powerjob-server-migrate/pom.xml +++ b/powerjob-server/powerjob-server-migrate/pom.xml @@ -5,7 +5,7 @@ powerjob-server tech.powerjob - 4.3.3 + 4.3.4 ../pom.xml 4.0.0 diff --git a/powerjob-server/powerjob-server-monitor/pom.xml b/powerjob-server/powerjob-server-monitor/pom.xml index 6158c254..ebe7c793 100644 --- a/powerjob-server/powerjob-server-monitor/pom.xml +++ b/powerjob-server/powerjob-server-monitor/pom.xml @@ -5,7 +5,7 @@ powerjob-server tech.powerjob - 4.3.3 + 4.3.4 ../pom.xml 4.0.0 diff --git a/powerjob-server/powerjob-server-persistence/pom.xml b/powerjob-server/powerjob-server-persistence/pom.xml index d476400b..48b275de 100644 --- a/powerjob-server/powerjob-server-persistence/pom.xml +++ b/powerjob-server/powerjob-server-persistence/pom.xml @@ -5,7 +5,7 @@ powerjob-server tech.powerjob - 4.3.3 + 4.3.4 ../pom.xml 4.0.0 diff --git a/powerjob-server/powerjob-server-remote/pom.xml b/powerjob-server/powerjob-server-remote/pom.xml index e5040c98..6a43bc30 100644 --- a/powerjob-server/powerjob-server-remote/pom.xml +++ b/powerjob-server/powerjob-server-remote/pom.xml @@ -5,7 +5,7 @@ powerjob-server tech.powerjob - 4.3.3 + 4.3.4 ../pom.xml 4.0.0 diff --git a/powerjob-server/powerjob-server-starter/pom.xml b/powerjob-server/powerjob-server-starter/pom.xml index d1a954dc..d7b40670 100644 --- a/powerjob-server/powerjob-server-starter/pom.xml +++ b/powerjob-server/powerjob-server-starter/pom.xml @@ -5,7 +5,7 @@ powerjob-server tech.powerjob - 4.3.3 + 4.3.4 ../pom.xml 4.0.0 diff --git a/powerjob-worker-agent/pom.xml b/powerjob-worker-agent/pom.xml index d125c0bb..e8aba22e 100644 --- a/powerjob-worker-agent/pom.xml +++ b/powerjob-worker-agent/pom.xml @@ -5,24 +5,24 @@ powerjob tech.powerjob - 4.3.3 + 4.3.4 4.0.0 powerjob-worker-agent - 4.3.3 + 4.3.4 jar - 4.3.3 + 4.3.4 1.2.9 4.3.2 5.3.23 2.3.4.RELEASE - 4.3.3 + 4.3.4 8.0.28 diff --git a/powerjob-worker-samples/pom.xml b/powerjob-worker-samples/pom.xml index cc616a8f..b3decf98 100644 --- a/powerjob-worker-samples/pom.xml +++ b/powerjob-worker-samples/pom.xml @@ -5,18 +5,18 @@ powerjob tech.powerjob - 4.3.3 + 4.3.4 4.0.0 powerjob-worker-samples - 4.3.3 + 4.3.4 2.7.4 - 4.3.3 + 4.3.4 1.2.83 - 4.3.3 + 4.3.4 true diff --git a/powerjob-worker-spring-boot-starter/pom.xml b/powerjob-worker-spring-boot-starter/pom.xml index 50c6314c..73f81b69 100644 --- a/powerjob-worker-spring-boot-starter/pom.xml +++ b/powerjob-worker-spring-boot-starter/pom.xml @@ -5,16 +5,16 @@ powerjob tech.powerjob - 4.3.3 + 4.3.4 4.0.0 powerjob-worker-spring-boot-starter - 4.3.3 + 4.3.4 jar - 4.3.3 + 4.3.4 2.7.4 diff --git a/powerjob-worker/pom.xml b/powerjob-worker/pom.xml index 312ac0b7..3297a182 100644 --- a/powerjob-worker/pom.xml +++ b/powerjob-worker/pom.xml @@ -5,12 +5,12 @@ powerjob tech.powerjob - 4.3.3 + 4.3.4 4.0.0 powerjob-worker - 4.3.3 + 4.3.4 jar @@ -21,10 +21,10 @@ 1.2.9 - 4.3.3 - 4.3.3 - 4.3.3 - 4.3.3 + 4.3.4 + 4.3.4 + 4.3.4 + 4.3.4 From 09b15dfbc1c33af7bec8e53535796007ce371022 Mon Sep 17 00:00:00 2001 From: tjq Date: Sun, 6 Aug 2023 21:03:41 +0800 Subject: [PATCH 17/24] feat: optimize online log cache time(60 -> 10) --- .../core/instance/InstanceLogService.java | 45 ++++++++++--------- .../processors/MapReduceProcessorDemo.java | 3 +- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceLogService.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceLogService.java index a12f05d0..42c67d0c 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceLogService.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceLogService.java @@ -1,23 +1,5 @@ package tech.powerjob.server.core.instance; -import org.springframework.core.task.AsyncTaskExecutor; -import org.springframework.core.task.TaskExecutor; -import tech.powerjob.common.enums.LogLevel; -import tech.powerjob.common.OmsConstant; -import tech.powerjob.common.enums.TimeExpressionType; -import tech.powerjob.common.model.InstanceLogContent; -import tech.powerjob.common.utils.CommonUtils; -import tech.powerjob.common.utils.NetUtils; -import tech.powerjob.common.utils.SegmentLock; -import tech.powerjob.server.common.constants.PJThreadPool; -import tech.powerjob.server.extension.dfs.*; -import tech.powerjob.server.persistence.storage.Constants; -import tech.powerjob.server.remote.server.redirector.DesignateServer; -import tech.powerjob.server.common.utils.OmsFileUtils; -import tech.powerjob.server.persistence.StringPage; -import tech.powerjob.server.persistence.remote.model.JobInfoDO; -import tech.powerjob.server.persistence.local.LocalInstanceLogDO; -import tech.powerjob.server.persistence.local.LocalInstanceLogRepository; import com.google.common.base.Stopwatch; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -27,18 +9,37 @@ import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.commons.lang3.time.FastDateFormat; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.task.AsyncTaskExecutor; import org.springframework.scheduling.annotation.Async; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; import org.springframework.util.CollectionUtils; +import tech.powerjob.common.OmsConstant; +import tech.powerjob.common.enums.LogLevel; +import tech.powerjob.common.enums.TimeExpressionType; +import tech.powerjob.common.model.InstanceLogContent; +import tech.powerjob.common.utils.CommonUtils; +import tech.powerjob.common.utils.NetUtils; +import tech.powerjob.common.utils.SegmentLock; +import tech.powerjob.server.common.constants.PJThreadPool; +import tech.powerjob.server.common.utils.OmsFileUtils; +import tech.powerjob.server.extension.dfs.*; +import tech.powerjob.server.persistence.StringPage; +import tech.powerjob.server.persistence.local.LocalInstanceLogDO; +import tech.powerjob.server.persistence.local.LocalInstanceLogRepository; +import tech.powerjob.server.persistence.remote.model.JobInfoDO; +import tech.powerjob.server.persistence.storage.Constants; +import tech.powerjob.server.remote.server.redirector.DesignateServer; import javax.annotation.Resource; import java.io.*; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.*; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -91,9 +92,9 @@ public class InstanceLogService { */ private static final int MAX_LINE_COUNT = 100; /** - * 过期时间 + * 更新中的日志缓存时间 */ - private static final long EXPIRE_INTERVAL_MS = 60000; + private static final long LOG_CACHE_TIME = 10000; /** * 提交日志记录，持久化到本地数据库中 @@ -248,7 +249,7 @@ public class InstanceLogService { return localTransactionTemplate.execute(status -> { File f = new File(path); // 如果文件存在且有效，则不再重新构建日志文件（这个判断也需要放在锁内，否则构建到一半的文件会被返回） - if (f.exists() && (System.currentTimeMillis() - f.lastModified()) < EXPIRE_INTERVAL_MS) { + if (f.exists() && (System.currentTimeMillis() - f.lastModified()) < LOG_CACHE_TIME) { return f; } try { diff --git a/powerjob-worker-samples/src/main/java/tech/powerjob/samples/processors/MapReduceProcessorDemo.java b/powerjob-worker-samples/src/main/java/tech/powerjob/samples/processors/MapReduceProcessorDemo.java index 215a95a0..2b77c370 100644 --- a/powerjob-worker-samples/src/main/java/tech/powerjob/samples/processors/MapReduceProcessorDemo.java +++ b/powerjob-worker-samples/src/main/java/tech/powerjob/samples/processors/MapReduceProcessorDemo.java @@ -17,6 +17,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import java.util.List; +import java.util.Optional; import java.util.concurrent.ThreadLocalRandom; /** @@ -40,7 +41,7 @@ public class MapReduceProcessorDemo implements MapReduceProcessor { log.info("taskContext:{}", JsonUtils.toJSONString(context)); // 根据控制台参数获取MR批次及子任务大小 - final JSONObject jobParams = JSONObject.parseObject(context.getJobParams()); + final JSONObject jobParams = Optional.ofNullable(context.getJobParams()).map(JSONObject::parseObject).orElse(new JSONObject()); Integer batchSize = (Integer) jobParams.getOrDefault("batchSize", 100); Integer batchNum = (Integer) jobParams.getOrDefault("batchNum", 10); From c50a3edebff7d03744c41cad580df8166b072098 Mon Sep 17 00:00:00 2001 From: tjq Date: Thu, 10 Aug 2023 00:16:40 +0800 Subject: [PATCH 18/24] feat: [storageExt] MySqlSeriesDfsService --- .../storage/impl/MySqlSeriesDfsService.java | 136 ++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/MySqlSeriesDfsService.java diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/MySqlSeriesDfsService.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/MySqlSeriesDfsService.java new file mode 100644 index 00000000..744b79c0 --- /dev/null +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/MySqlSeriesDfsService.java @@ -0,0 +1,136 @@ +package tech.powerjob.server.persistence.storage.impl; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import lombok.Data; +import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.springframework.context.ApplicationContext; +import org.springframework.core.env.Environment; +import tech.powerjob.server.extension.dfs.DownloadRequest; +import tech.powerjob.server.extension.dfs.FileLocation; +import tech.powerjob.server.extension.dfs.FileMeta; +import tech.powerjob.server.extension.dfs.StoreRequest; +import tech.powerjob.server.persistence.storage.AbstractDFsService; + +import javax.sql.DataSource; +import java.io.IOException; +import java.util.Optional; + +/** + * MySQL 特性类似的数据库存储 + * + * @author tjq + * @since 2023/8/9 + */ +@Slf4j +public class MySqlSeriesDfsService extends AbstractDFsService { + + private DataSource dataSource; + + private static final String TYPE_MYSQL = "mysql_series"; + + private static final String KEY_DRIVER_NAME = "driver"; + private static final String KEY_URL = "url"; + private static final String KEY_USERNAME = "username"; + + private static final String KEY_PASSWORD = "password"; + + private static final String CREATE_TABLE_SQL = "CREATE TABLE\n" + + "IF\n" + + "\tNOT EXISTS powerjob_files (\n" + + "\t\t`id` BIGINT NOT NULL AUTO_INCREMENT COMMENT 'ID',\n" + + "\t\t`gmt_create` DATETIME NOT NULL COMMENT '创建时间',\n" + + "\t\t`gmt_modified` DATETIME COMMENT '更新时间',\n" + + "\t\t`name` VARCHAR ( 255 ) NOT NULL COMMENT '文件名称',\n" + + "\t\t`bucket` VARCHAR ( 255 ) NOT NULL COMMENT '分桶',\n" + + "\t\t`extra` VARCHAR ( 255 ) NOT NULL COMMENT '其他信息',\n" + + "\t\t`version` VARCHAR ( 255 ) NOT NULL COMMENT '版本',\n" + + "\t\t`data` LONGBLOB NOT NULL COMMENT '文件内容',\n" + + "\tPRIMARY KEY ( id ) \n" + + "\t);"; + + @Override + public void store(StoreRequest storeRequest) throws IOException { + + } + + @Override + public void download(DownloadRequest downloadRequest) throws IOException { + + } + + @Override + public Optional fetchFileMeta(FileLocation fileLocation) throws IOException { + return Optional.empty(); + } + + @Override + protected void init(ApplicationContext applicationContext) { + + Environment env = applicationContext.getEnvironment(); + + MySQLProperty mySQLProperty = new MySQLProperty() + .setDriver(fetchProperty(env, TYPE_MYSQL, KEY_DRIVER_NAME)) + .setUrl(fetchProperty(env, TYPE_MYSQL, KEY_URL)) + .setUsername(fetchProperty(env, TYPE_MYSQL, KEY_USERNAME)) + .setPassword(fetchProperty(env, TYPE_MYSQL, KEY_PASSWORD)); + + try { + initDatabase(mySQLProperty); + initTable(mySQLProperty); + } catch (Exception e) { + log.error("[MySqlSeriesDfsService] init datasource failed!", e); + ExceptionUtils.rethrow(e); + } + } + + private void initDatabase(MySQLProperty property) { + + log.info("[MySqlSeriesDfsService] init datasource by config: {}", property); + + HikariConfig config = new HikariConfig(); + + config.setDriverClassName(property.driver); + config.setJdbcUrl(property.url); + config.setUsername(property.username); + config.setPassword(property.password); + + config.setAutoCommit(true); + // 池中最小空闲连接数量 + config.setMinimumIdle(2); + // 池中最大连接数量 + config.setMaximumPoolSize(32); + + dataSource = new HikariDataSource(config); + } + + private void initTable(MySQLProperty property) throws Exception { + dataSource.getConnection().createStatement().execute(CREATE_TABLE_SQL); + } + + @Override + public void destroy() throws Exception { + } + + @Data + @Accessors(chain = true) + static class MySQLProperty { + private String driver; + private String url; + private String username; + private String password; + } + + public static void main(String[] args) throws Exception { + MySQLProperty mySQLProperty = new MySQLProperty() + .setDriver("com.mysql.cj.jdbc.Driver") + .setUrl("jdbc:mysql://localhost:3306/powerjob-daily?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai") + .setUsername("root") + .setPassword("No1Bug2Please3!"); + MySqlSeriesDfsService mySqlSeriesDfsService = new MySqlSeriesDfsService(); + mySqlSeriesDfsService.initDatabase(mySQLProperty); + mySqlSeriesDfsService.initTable(mySQLProperty); + } +} From 37a62549db9c71e712499338896db6ef98bd79cc Mon Sep 17 00:00:00 2001 From: tjq Date: Thu, 10 Aug 2023 23:59:07 +0800 Subject: [PATCH 19/24] feat: [storageExt] MySqlSeriesDfsService --- .../powerjob/common/serialize/JsonUtils.java | 17 ++ .../tech/powerjob/common/utils/NetUtils.java | 15 ++ .../server/extension/dfs/FileLocation.java | 11 +- .../storage/AbstractDFsService.java | 3 + .../storage/impl/MySqlSeriesDfsService.java | 240 ++++++++++++++++-- .../storage/impl/AbstractDfsServiceTest.java | 5 +- .../impl/MySqlSeriesDfsServiceTest.java | 42 +++ 7 files changed, 305 insertions(+), 28 deletions(-) create mode 100644 powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/MySqlSeriesDfsServiceTest.java diff --git a/powerjob-common/src/main/java/tech/powerjob/common/serialize/JsonUtils.java b/powerjob-common/src/main/java/tech/powerjob/common/serialize/JsonUtils.java index 085da18c..0480357a 100644 --- a/powerjob-common/src/main/java/tech/powerjob/common/serialize/JsonUtils.java +++ b/powerjob-common/src/main/java/tech/powerjob/common/serialize/JsonUtils.java @@ -8,9 +8,12 @@ import com.fasterxml.jackson.databind.json.JsonMapper; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; +import tech.powerjob.common.exception.ImpossibleException; import tech.powerjob.common.exception.PowerJobException; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; /** * JSON工具类 @@ -27,6 +30,8 @@ public class JsonUtils { .configure(JsonParser.Feature.IGNORE_UNDEFINED, true) .build(); + private static final TypeReference> MAP_TYPE_REFERENCE = new TypeReference> () {}; + private JsonUtils(){ } @@ -67,6 +72,18 @@ public class JsonUtils { return JSON_MAPPER.readValue(json, clz); } + public static Map parseMap(String json) { + if (StringUtils.isEmpty(json)) { + return new HashMap<>(); + } + try { + return JSON_MAPPER.readValue(json, MAP_TYPE_REFERENCE); + } catch (Exception e) { + ExceptionUtils.rethrow(e); + } + throw new ImpossibleException(); + } + public static T parseObject(byte[] b, Class clz) throws IOException { return JSON_MAPPER.readValue(b, clz); } diff --git a/powerjob-common/src/main/java/tech/powerjob/common/utils/NetUtils.java b/powerjob-common/src/main/java/tech/powerjob/common/utils/NetUtils.java index 9a0d88a3..b6a3fe93 100644 --- a/powerjob-common/src/main/java/tech/powerjob/common/utils/NetUtils.java +++ b/powerjob-common/src/main/java/tech/powerjob/common/utils/NetUtils.java @@ -56,6 +56,21 @@ public class NetUtils { return ThreadLocalRandom.current().nextInt(RND_PORT_START, RND_PORT_END); } + /** + * 检测某个 IP 端口是否可用 + * @param ip IP + * @param port 端口 + * @return 是否可用 + */ + public static boolean checkIpPortAvailable(String ip, int port) { + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(ip, port), 1000); + return true; + } catch (Exception e) { + return false; + } + } + /** * 获取本机 IP 地址 * diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/FileLocation.java b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/FileLocation.java index 13132876..0eda207d 100644 --- a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/FileLocation.java +++ b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/dfs/FileLocation.java @@ -1,6 +1,7 @@ package tech.powerjob.server.extension.dfs; -import lombok.Data; +import lombok.Getter; +import lombok.Setter; import lombok.experimental.Accessors; /** @@ -9,7 +10,8 @@ import lombok.experimental.Accessors; * @author tjq * @since 2023/7/16 */ -@Data +@Getter +@Setter @Accessors(chain = true) public class FileLocation { @@ -22,4 +24,9 @@ public class FileLocation { * 名称 */ private String name; + + @Override + public String toString() { + return String.format("%s.%s", bucket, name); + } } diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/AbstractDFsService.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/AbstractDFsService.java index 4ce3c72e..edb52a8c 100644 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/AbstractDFsService.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/AbstractDFsService.java @@ -17,6 +17,8 @@ import tech.powerjob.server.extension.dfs.DFsService; @Slf4j public abstract class AbstractDFsService implements DFsService, ApplicationContextAware, DisposableBean { + protected ApplicationContext applicationContext; + public AbstractDFsService() { log.info("[DFsService] invoke [{}]'s constructor", this.getClass().getName()); } @@ -32,6 +34,7 @@ public abstract class AbstractDFsService implements DFsService, ApplicationConte @Override public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + this.applicationContext = applicationContext; log.info("[DFsService] invoke [{}]'s setApplicationContext", this.getClass().getName()); init(applicationContext); } diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/MySqlSeriesDfsService.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/MySqlSeriesDfsService.java index 744b79c0..4b3ea930 100644 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/MySqlSeriesDfsService.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/MySqlSeriesDfsService.java @@ -1,68 +1,228 @@ package tech.powerjob.server.persistence.storage.impl; +import com.google.common.base.Stopwatch; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import lombok.Data; import lombok.experimental.Accessors; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Conditional; import org.springframework.core.env.Environment; -import tech.powerjob.server.extension.dfs.DownloadRequest; -import tech.powerjob.server.extension.dfs.FileLocation; -import tech.powerjob.server.extension.dfs.FileMeta; -import tech.powerjob.server.extension.dfs.StoreRequest; +import tech.powerjob.common.serialize.JsonUtils; +import tech.powerjob.server.common.constants.SwitchableStatus; +import tech.powerjob.server.common.spring.condition.PropertyAndOneBeanCondition; +import tech.powerjob.server.extension.dfs.*; import tech.powerjob.server.persistence.storage.AbstractDFsService; +import javax.annotation.Priority; import javax.sql.DataSource; +import java.io.BufferedInputStream; import java.io.IOException; +import java.nio.file.Files; +import java.sql.*; +import java.util.List; +import java.util.Map; import java.util.Optional; /** * MySQL 特性类似的数据库存储 + * PS1. 大文件上传可能会报 max_allowed_packet 不足，可根据参数放开数据库限制 set global max_allowed_packet = 500*1024*1024 + * PS1. 官方基于 MySQL 测试，其他数据库使用前请自测，敬请谅解！ + * PS2. 数据库并不适合大规模的文件存储，该扩展仅适用于简单业务，大型业务场景请选择其他存储方案（OSS、MongoDB等） + * ********************* 配置项 ********************* + * oms.storage.dfs.mysql_series.driver + * oms.storage.dfs.mysql_series.url + * oms.storage.dfs.mysql_series.username + * oms.storage.dfs.mysql_series.password + * oms.storage.dfs.mysql_series.auto_create_table + * oms.storage.dfs.mysql_series.table_name * * @author tjq * @since 2023/8/9 */ @Slf4j +@Priority(value = Integer.MAX_VALUE - 2) +@Conditional(MySqlSeriesDfsService.MySqlSeriesCondition.class) public class MySqlSeriesDfsService extends AbstractDFsService { private DataSource dataSource; private static final String TYPE_MYSQL = "mysql_series"; + /** + * 数据库驱动，MYSQL8 为 com.mysql.cj.jdbc.Driver + */ private static final String KEY_DRIVER_NAME = "driver"; + /** + * 数据库地址，比如 jdbc:mysql://localhost:3306/powerjob-daily?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai + */ private static final String KEY_URL = "url"; + /** + * 数据库账号，比如 root + */ private static final String KEY_USERNAME = "username"; - + /** + * 数据库密码 + */ private static final String KEY_PASSWORD = "password"; + /** + * 是否自动建表 + */ + private static final String KEY_AUTO_CREATE_TABLE = "auto_create_table"; + /** + * 表名 + */ + private static final String KEY_TABLE_NAME = "table_name"; + + /* ********************* SQL region ********************* */ + + private static final String DEFAULT_TABLE_NAME = "powerjob_files"; private static final String CREATE_TABLE_SQL = "CREATE TABLE\n" + "IF\n" + - "\tNOT EXISTS powerjob_files (\n" + + "\tNOT EXISTS %s (\n" + "\t\t`id` BIGINT NOT NULL AUTO_INCREMENT COMMENT 'ID',\n" + + "\t\t`bucket` VARCHAR ( 255 ) NOT NULL COMMENT '分桶',\n" + + "\t\t`name` VARCHAR ( 255 ) NOT NULL COMMENT '文件名称',\n" + + "\t\t`version` VARCHAR ( 255 ) NOT NULL COMMENT '版本',\n" + + "\t\t`meta` VARCHAR ( 255 ) COMMENT '元数据',\n" + + "\t\t`length` BIGINT NOT NULL COMMENT '长度',\n" + + "\t\t`status` INT NOT NULL COMMENT '状态',\n" + + "\t\t`data` LONGBLOB NOT NULL COMMENT '文件内容',\n" + + "\t\t`extra` VARCHAR ( 255 ) COMMENT '其他信息',\n" + "\t\t`gmt_create` DATETIME NOT NULL COMMENT '创建时间',\n" + "\t\t`gmt_modified` DATETIME COMMENT '更新时间',\n" + - "\t\t`name` VARCHAR ( 255 ) NOT NULL COMMENT '文件名称',\n" + - "\t\t`bucket` VARCHAR ( 255 ) NOT NULL COMMENT '分桶',\n" + - "\t\t`extra` VARCHAR ( 255 ) NOT NULL COMMENT '其他信息',\n" + - "\t\t`version` VARCHAR ( 255 ) NOT NULL COMMENT '版本',\n" + - "\t\t`data` LONGBLOB NOT NULL COMMENT '文件内容',\n" + "\tPRIMARY KEY ( id ) \n" + "\t);"; + private static final String INSERT_SQL = "insert into %s(bucket, name, version, meta, length, status, data, extra, gmt_create, gmt_modified) values (?,?,?,?,?,?,?,?,?,?);"; + + private static final String DELETE_SQL = "DELETE FROM %s"; + + private static final String QUERY_FULL_SQL = "select * from %s"; + + private static final String QUERY_META_SQL = "select bucket, name, version, meta, length, status, extra, gmt_create, gmt_modified from %s"; + + private void deleteByLocation(FileLocation fileLocation) { + String dSQLPrefix = fullSQL(DELETE_SQL); + + try (Connection con = dataSource.getConnection()) { + + String dSQL = dSQLPrefix.concat(whereSQL(fileLocation)); + + con.createStatement().executeUpdate(dSQL); + + } catch (Exception e) { + log.error("[MySqlSeriesDfsService] deleteByLocation [{}] failed!", fileLocation); + } + + } + @Override public void store(StoreRequest storeRequest) throws IOException { + Stopwatch sw = Stopwatch.createStarted(); + String insertSQL = fullSQL(INSERT_SQL); + + FileLocation fileLocation = storeRequest.getFileLocation(); + + // 覆盖写，写之前先删除 + deleteByLocation(fileLocation); + + Map meta = Maps.newHashMap(); + meta.put("_local_file_path_", storeRequest.getLocalFile().getAbsolutePath()); + + Date date = new Date(System.currentTimeMillis()); + + try (Connection con = dataSource.getConnection()) { + PreparedStatement pst = con.prepareStatement(insertSQL); + + pst.setString(1, fileLocation.getBucket()); + pst.setString(2, fileLocation.getName()); + pst.setString(3, "mu"); + pst.setString(4, JsonUtils.toJSONString(meta)); + pst.setLong(5, storeRequest.getLocalFile().length()); + pst.setInt(6, SwitchableStatus.ENABLE.getV()); + pst.setBlob(7, new BufferedInputStream(Files.newInputStream(storeRequest.getLocalFile().toPath()))); + pst.setString(8, null); + pst.setDate(9, date); + pst.setDate(10, date); + + pst.execute(); + + log.info("[MySqlSeriesDfsService] store [{}] successfully, cost: {}", fileLocation, sw); + + } catch (Exception e) { + log.error("[MySqlSeriesDfsService] store [{}] failed!", fileLocation); + ExceptionUtils.rethrow(e); + } } @Override public void download(DownloadRequest downloadRequest) throws IOException { + Stopwatch sw = Stopwatch.createStarted(); + String querySQL = fullSQL(QUERY_FULL_SQL); + + FileLocation fileLocation = downloadRequest.getFileLocation(); + + FileUtils.forceMkdirParent(downloadRequest.getTarget()); + + try (Connection con = dataSource.getConnection()) { + + ResultSet resultSet = con.createStatement().executeQuery(querySQL.concat(whereSQL(fileLocation))); + + boolean exist = resultSet.next(); + + if (!exist) { + log.warn("[MySqlSeriesDfsService] download file[{}] failed due to not exits!", fileLocation); + return; + } + + Blob dataBlob = resultSet.getBlob("data"); + FileUtils.copyInputStreamToFile(new BufferedInputStream(dataBlob.getBinaryStream()), downloadRequest.getTarget()); + + log.info("[MySqlSeriesDfsService] download [{}] successfully, cost: {}", fileLocation, sw); + + } catch (Exception e) { + log.error("[MySqlSeriesDfsService] download file [{}] failed!", fileLocation, e); + ExceptionUtils.rethrow(e); + } + } @Override public Optional fetchFileMeta(FileLocation fileLocation) throws IOException { + + String querySQL = fullSQL(QUERY_META_SQL); + + try (Connection con = dataSource.getConnection()) { + + ResultSet resultSet = con.createStatement().executeQuery(querySQL.concat(whereSQL(fileLocation))); + + boolean exist = resultSet.next(); + + if (!exist) { + return Optional.empty(); + } + + FileMeta fileMeta = new FileMeta() + .setLength(resultSet.getLong("length")) + .setLastModifiedTime(resultSet.getDate("gmt_modified")) + .setMetaInfo(JsonUtils.parseMap(resultSet.getString("meta"))); + return Optional.of(fileMeta); + + } catch (Exception e) { + log.error("[MySqlSeriesDfsService] fetchFileMeta [{}] failed!", fileLocation); + ExceptionUtils.rethrow(e); + } + return Optional.empty(); } @@ -75,7 +235,9 @@ public class MySqlSeriesDfsService extends AbstractDFsService { .setDriver(fetchProperty(env, TYPE_MYSQL, KEY_DRIVER_NAME)) .setUrl(fetchProperty(env, TYPE_MYSQL, KEY_URL)) .setUsername(fetchProperty(env, TYPE_MYSQL, KEY_USERNAME)) - .setPassword(fetchProperty(env, TYPE_MYSQL, KEY_PASSWORD)); + .setPassword(fetchProperty(env, TYPE_MYSQL, KEY_PASSWORD)) + .setAutoCreateTable(Boolean.TRUE.toString().equalsIgnoreCase(fetchProperty(env, TYPE_MYSQL, KEY_AUTO_CREATE_TABLE))) + ; try { initDatabase(mySQLProperty); @@ -86,7 +248,7 @@ public class MySqlSeriesDfsService extends AbstractDFsService { } } - private void initDatabase(MySQLProperty property) { + void initDatabase(MySQLProperty property) { log.info("[MySqlSeriesDfsService] init datasource by config: {}", property); @@ -106,8 +268,35 @@ public class MySqlSeriesDfsService extends AbstractDFsService { dataSource = new HikariDataSource(config); } - private void initTable(MySQLProperty property) throws Exception { - dataSource.getConnection().createStatement().execute(CREATE_TABLE_SQL); + void initTable(MySQLProperty property) throws Exception { + + if (property.autoCreateTable) { + + String createTableSQL = fullSQL(CREATE_TABLE_SQL); + + log.info("[MySqlSeriesDfsService] use create table SQL: {}", createTableSQL); + try (Connection connection = dataSource.getConnection()) { + connection.createStatement().execute(createTableSQL); + log.info("[MySqlSeriesDfsService] auto create table successfully!"); + } + } + } + + private String fullSQL(String sql) { + return String.format(sql, parseTableName()); + } + + private String parseTableName() { + // 误删，兼容本地 unit test + if (applicationContext == null) { + return DEFAULT_TABLE_NAME; + } + String tableName = fetchProperty(applicationContext.getEnvironment(), TYPE_MYSQL, KEY_TABLE_NAME); + return StringUtils.isEmpty(tableName) ? DEFAULT_TABLE_NAME : tableName; + } + + private static String whereSQL(FileLocation fileLocation) { + return String.format(" where bucket='%s' AND name='%s' ", fileLocation.getBucket(), fileLocation.getName()); } @Override @@ -121,16 +310,19 @@ public class MySqlSeriesDfsService extends AbstractDFsService { private String url; private String username; private String password; + + private boolean autoCreateTable; } - public static void main(String[] args) throws Exception { - MySQLProperty mySQLProperty = new MySQLProperty() - .setDriver("com.mysql.cj.jdbc.Driver") - .setUrl("jdbc:mysql://localhost:3306/powerjob-daily?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai") - .setUsername("root") - .setPassword("No1Bug2Please3!"); - MySqlSeriesDfsService mySqlSeriesDfsService = new MySqlSeriesDfsService(); - mySqlSeriesDfsService.initDatabase(mySQLProperty); - mySqlSeriesDfsService.initTable(mySQLProperty); + public static class MySqlSeriesCondition extends PropertyAndOneBeanCondition { + @Override + protected List anyConfigKey() { + return Lists.newArrayList("oms.storage.dfs.mysql_series.url"); + } + + @Override + protected Class beanType() { + return DFsService.class; + } } } diff --git a/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AbstractDfsServiceTest.java b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AbstractDfsServiceTest.java index bd15b482..8b5a4dc9 100644 --- a/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AbstractDfsServiceTest.java +++ b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AbstractDfsServiceTest.java @@ -10,6 +10,7 @@ import tech.powerjob.server.extension.dfs.*; import java.io.File; import java.nio.charset.StandardCharsets; import java.util.Optional; +import java.util.concurrent.ThreadLocalRandom; /** * AbstractDfsServiceTest @@ -32,7 +33,7 @@ public abstract class AbstractDfsServiceTest { DFsService aliOssService = aliOssServiceOpt.get(); - String content = "wlcgyqsl"; + String content = "wlcgyqsl".concat(String.valueOf(ThreadLocalRandom.current().nextLong())); String temporarySourcePath = OmsFileUtils.genTemporaryWorkPath() + "source.txt"; String temporaryDownloadPath = OmsFileUtils.genTemporaryWorkPath() + "download.txt"; @@ -42,7 +43,7 @@ public abstract class AbstractDfsServiceTest { FileUtils.forceMkdirParent(sourceFile); OmsFileUtils.string2File(content, sourceFile); - FileLocation fileLocation = new FileLocation().setBucket("pj_test").setName("testAliOss.txt"); + FileLocation fileLocation = new FileLocation().setBucket("pj_test").setName(String.format("test_%d.txt", ThreadLocalRandom.current().nextLong())); StoreRequest storeRequest = new StoreRequest() .setFileLocation(fileLocation) diff --git a/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/MySqlSeriesDfsServiceTest.java b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/MySqlSeriesDfsServiceTest.java new file mode 100644 index 00000000..d14116fe --- /dev/null +++ b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/MySqlSeriesDfsServiceTest.java @@ -0,0 +1,42 @@ +package tech.powerjob.server.persistence.storage.impl; + +import tech.powerjob.common.utils.NetUtils; +import tech.powerjob.server.extension.dfs.DFsService; + +import java.util.Optional; + +/** + * MySqlSeriesDfsServiceTest + * + * @author tjq + * @since 2023/8/10 + */ +class MySqlSeriesDfsServiceTest extends AbstractDfsServiceTest { + + @Override + protected Optional fetchService() { + + boolean dbAvailable = NetUtils.checkIpPortAvailable("127.0.0.1", 3306); + if (dbAvailable) { + MySqlSeriesDfsService mySqlSeriesDfsService = new MySqlSeriesDfsService(); + + try { + + MySqlSeriesDfsService.MySQLProperty mySQLProperty = new MySqlSeriesDfsService.MySQLProperty() + .setDriver("com.mysql.cj.jdbc.Driver") + .setUrl("jdbc:mysql://localhost:3306/powerjob-daily?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai") + .setUsername("root") + .setAutoCreateTable(true) + .setPassword("No1Bug2Please3!"); + mySqlSeriesDfsService.initDatabase(mySQLProperty); + mySqlSeriesDfsService.initTable(mySQLProperty); + + return Optional.of(mySqlSeriesDfsService); + } catch (Exception e) { + e.printStackTrace(); + } + } + + return Optional.empty(); + } +} \ No newline at end of file From ad08406d0be6f8797e61f6ad57d64b34feb1de51 Mon Sep 17 00:00:00 2001 From: tjq Date: Sun, 13 Aug 2023 16:11:10 +0800 Subject: [PATCH 20/24] feat: [storageExt] finished MySqlSeriesDfsService --- .../storage/StorageConfiguration.java | 7 ++++ .../storage/impl/MySqlSeriesDfsService.java | 32 ++++++++++++++----- .../storage/impl/AbstractDfsServiceTest.java | 7 +++- 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/StorageConfiguration.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/StorageConfiguration.java index f99767e4..2e3abd82 100644 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/StorageConfiguration.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/StorageConfiguration.java @@ -7,6 +7,7 @@ import tech.powerjob.server.extension.dfs.DFsService; import tech.powerjob.server.persistence.storage.impl.AliOssService; import tech.powerjob.server.persistence.storage.impl.EmptyDFsService; import tech.powerjob.server.persistence.storage.impl.GridFsService; +import tech.powerjob.server.persistence.storage.impl.MySqlSeriesDfsService; /** * Description @@ -23,6 +24,12 @@ public class StorageConfiguration { return new GridFsService(); } + @Bean + @Conditional(MySqlSeriesDfsService.MySqlSeriesCondition.class) + public DFsService initDbFs() { + return new MySqlSeriesDfsService(); + } + @Bean @Conditional(AliOssService.AliOssCondition.class) public DFsService initAliOssFs() { diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/MySqlSeriesDfsService.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/MySqlSeriesDfsService.java index 4b3ea930..ca8348a8 100644 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/MySqlSeriesDfsService.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/MySqlSeriesDfsService.java @@ -11,10 +11,12 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.commons.lang3.time.DateUtils; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Conditional; import org.springframework.core.env.Environment; import tech.powerjob.common.serialize.JsonUtils; +import tech.powerjob.common.utils.CommonUtils; import tech.powerjob.server.common.constants.SwitchableStatus; import tech.powerjob.server.common.spring.condition.PropertyAndOneBeanCondition; import tech.powerjob.server.extension.dfs.*; @@ -103,25 +105,25 @@ public class MySqlSeriesDfsService extends AbstractDFsService { private static final String INSERT_SQL = "insert into %s(bucket, name, version, meta, length, status, data, extra, gmt_create, gmt_modified) values (?,?,?,?,?,?,?,?,?,?);"; - private static final String DELETE_SQL = "DELETE FROM %s"; + private static final String DELETE_SQL = "DELETE FROM %s "; private static final String QUERY_FULL_SQL = "select * from %s"; private static final String QUERY_META_SQL = "select bucket, name, version, meta, length, status, extra, gmt_create, gmt_modified from %s"; + private void deleteByLocation(FileLocation fileLocation) { String dSQLPrefix = fullSQL(DELETE_SQL); + String dSQL = dSQLPrefix.concat(whereSQL(fileLocation)); + executeDelete(dSQL); + } + private void executeDelete(String sql) { try (Connection con = dataSource.getConnection()) { - - String dSQL = dSQLPrefix.concat(whereSQL(fileLocation)); - - con.createStatement().executeUpdate(dSQL); - + con.createStatement().executeUpdate(sql); } catch (Exception e) { - log.error("[MySqlSeriesDfsService] deleteByLocation [{}] failed!", fileLocation); + log.error("[MySqlSeriesDfsService] executeDelete failed, sql: {}", sql); } - } @Override @@ -226,6 +228,20 @@ public class MySqlSeriesDfsService extends AbstractDFsService { return Optional.empty(); } + @Override + public void cleanExpiredFiles(String bucket, int days) { + + // 虽然官方提供了服务端删除的能力，依然强烈建议用户直接在数据库层面配置清理事件！！！ + + String dSQLPrefix = fullSQL(DELETE_SQL); + final long targetTs = DateUtils.addDays(new Date(System.currentTimeMillis()), -days).getTime(); + final String targetDeleteTime = CommonUtils.formatTime(targetTs); + log.info("[MySqlSeriesDfsService] start to cleanExpiredFiles, targetDeleteTime: {}", targetDeleteTime); + String fSQL = dSQLPrefix.concat(String.format(" where gmt_modified < '%s'", targetDeleteTime)); + log.info("[MySqlSeriesDfsService] cleanExpiredFiles SQL: {}", fSQL); + executeDelete(fSQL); + } + @Override protected void init(ApplicationContext applicationContext) { diff --git a/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AbstractDfsServiceTest.java b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AbstractDfsServiceTest.java index 8b5a4dc9..25974864 100644 --- a/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AbstractDfsServiceTest.java +++ b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AbstractDfsServiceTest.java @@ -21,6 +21,8 @@ import java.util.concurrent.ThreadLocalRandom; @Slf4j public abstract class AbstractDfsServiceTest { + private static final String BUCKET = "pj_test"; + abstract protected Optional fetchService(); @Test @@ -43,7 +45,7 @@ public abstract class AbstractDfsServiceTest { FileUtils.forceMkdirParent(sourceFile); OmsFileUtils.string2File(content, sourceFile); - FileLocation fileLocation = new FileLocation().setBucket("pj_test").setName(String.format("test_%d.txt", ThreadLocalRandom.current().nextLong())); + FileLocation fileLocation = new FileLocation().setBucket(BUCKET).setName(String.format("test_%d.txt", ThreadLocalRandom.current().nextLong())); StoreRequest storeRequest = new StoreRequest() .setFileLocation(fileLocation) @@ -69,6 +71,9 @@ public abstract class AbstractDfsServiceTest { String downloadFileContent = FileUtils.readFileToString(downloadFile, StandardCharsets.UTF_8); log.info("[testBaseFileOperation] download content: {}", downloadFileContent); assert downloadFileContent.equals(content); + + // 定时清理，只是执行，不校验 + aliOssService.cleanExpiredFiles(BUCKET, 3); } @Test From 73ebe83c051e770e70d6145460c884bd80d3eb8e Mon Sep 17 00:00:00 2001 From: tjq Date: Sun, 13 Aug 2023 16:31:12 +0800 Subject: [PATCH 21/24] feat: code review problem modification, ready to release 4.3.4 --- .../server/persistence/storage/StorageConfiguration.java | 2 +- .../server/remote/server/election/ServerElectionService.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/StorageConfiguration.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/StorageConfiguration.java index 2e3abd82..6b5627fe 100644 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/StorageConfiguration.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/StorageConfiguration.java @@ -10,7 +10,7 @@ import tech.powerjob.server.persistence.storage.impl.GridFsService; import tech.powerjob.server.persistence.storage.impl.MySqlSeriesDfsService; /** - * Description + * 初始化内置的存储服务 * * @author tjq * @since 2023/7/30 diff --git a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/election/ServerElectionService.java b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/election/ServerElectionService.java index 0c384d6a..e6febdfe 100644 --- a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/election/ServerElectionService.java +++ b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/election/ServerElectionService.java @@ -160,7 +160,8 @@ public class ServerElectionService { downServerCache.remove(serverAddress); ProtocolInfo remoteProtocol = protocolInfo.toJavaObject(ProtocolInfo.class); log.info("[ServerElection] server[{}] is active, it will be the master, final protocol={}", serverAddress, remoteProtocol); - return remoteProtocol.getExternalAddress(); + // 4.3.3 升级 4.3.4 过程中，未升级的 server 还不存在 externalAddress，需要使用 address 兼容 + return Optional.ofNullable(remoteProtocol.getExternalAddress()).orElse(remoteProtocol.getAddress()); } else { log.warn("[ServerElection] server[{}] is active but don't have target protocol", serverAddress); } From 89b35c8495a2b6bb8395d7e41bceb1ea4b272b29 Mon Sep 17 00:00:00 2001 From: tjq Date: Sun, 13 Aug 2023 18:00:20 +0800 Subject: [PATCH 22/24] feat: [officialProcessor] add VerificationProcessor --- .../impl/VerificationProcessor.java | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 powerjob-official-processors/src/main/java/tech/powerjob/official/processors/impl/VerificationProcessor.java diff --git a/powerjob-official-processors/src/main/java/tech/powerjob/official/processors/impl/VerificationProcessor.java b/powerjob-official-processors/src/main/java/tech/powerjob/official/processors/impl/VerificationProcessor.java new file mode 100644 index 00000000..394908d4 --- /dev/null +++ b/powerjob-official-processors/src/main/java/tech/powerjob/official/processors/impl/VerificationProcessor.java @@ -0,0 +1,154 @@ +package tech.powerjob.official.processors.impl; + +import com.alibaba.fastjson.JSONObject; +import lombok.*; +import org.apache.commons.lang3.RandomStringUtils; +import tech.powerjob.common.exception.PowerJobException; +import tech.powerjob.official.processors.CommonBasicProcessor; +import tech.powerjob.official.processors.util.CommonUtils; +import tech.powerjob.worker.core.processor.ProcessResult; +import tech.powerjob.worker.core.processor.TaskContext; +import tech.powerjob.worker.core.processor.TaskResult; +import tech.powerjob.worker.core.processor.sdk.BroadcastProcessor; +import tech.powerjob.worker.core.processor.sdk.MapReduceProcessor; +import tech.powerjob.worker.log.OmsLogger; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ThreadLocalRandom; + +/** + * 功能验证用处理器，帮助用户快速验证想要测试的功能 + * + * @author tjq + * @since 2023/8/13 + */ +public class VerificationProcessor extends CommonBasicProcessor implements MapReduceProcessor, BroadcastProcessor { + + @Override + public ProcessResult preProcess(TaskContext context) throws Exception { + return new ProcessResult(true, "preProcess successfully!"); + } + + @Override + protected ProcessResult process0(TaskContext taskContext) throws Exception { + + final OmsLogger omsLogger = taskContext.getOmsLogger(); + + final String paramsStr = CommonUtils.parseParams(taskContext); + final VerificationParam verificationParam = JSONObject.parseObject(paramsStr, VerificationParam.class); + + final Mode mode = Mode.of(verificationParam.getMode()); + + switch (mode) { + case ERROR: + return new ProcessResult(false, "EXECUTE_FAILED_DUE_TO_CONFIG"); + case EXCEPTION: + throw new PowerJobException("exception for test"); + case TIMEOUT: + final Long sleepMs = Optional.ofNullable(verificationParam.getSleepMs()).orElse(3600000L); + Thread.sleep(sleepMs); + return new ProcessResult(true, "AFTER_SLEEP_" + sleepMs); + case MR: + if (isRootTask()) { + final int batchNum = Optional.ofNullable(verificationParam.getBatchNum()).orElse(10); + final int batchSize = Optional.ofNullable(verificationParam.getBatchSize()).orElse(100); + omsLogger.info("[VerificationProcessor] start root task~"); + List subTasks = new ArrayList<>(); + for (int a = 0; a < batchNum; a++) { + for (int b = 0; b < batchSize; b++) { + int x = a * batchSize + b; + subTasks.add(new TestSubTask("task_" + x, x)); + } + map(subTasks, "MAP_TEST_TASK_" + a); + omsLogger.info("[VerificationProcessor] [{}] map one batch successfully~", batchNum); + subTasks.clear(); + } + omsLogger.info("[VerificationProcessor] all map successfully!"); + return new ProcessResult(true, "MAP_SUCCESS"); + } else { + final Double successRate = Optional.ofNullable(verificationParam.getSubTaskSuccessRate()).orElse(0.5); + final double rd = ThreadLocalRandom.current().nextDouble(0, 1); + boolean success = rd <= successRate; + return new ProcessResult(success, String.format("taskId_%s_success_%s", taskContext.getTaskId(), success)); + } + } + + return new ProcessResult(true, "EXECUTE_SUCCESSFULLY_" + RandomStringUtils.randomAlphanumeric(Optional.ofNullable(verificationParam.getResponseSize()).orElse(10))); + } + + @Override + public ProcessResult reduce(TaskContext context, List taskResults) { + return new ProcessResult(true, "REDUCE_SUCCESS"); + } + + enum Mode { + /** + * 常规模式，直接返回响应 + */ + BASE, + /** + * 超时，sleep 一段时间测试超时控制 + */ + TIMEOUT, + /** + * 测试执行失败，响应返回 success = false + */ + ERROR, + /** + * 测试执行异常，抛出异常 + */ + EXCEPTION, + /** + * MapReduce + */ + MR + ; + + public static Mode of(String v) { + for (Mode m : values()) { + if (m.name().equalsIgnoreCase(v)) { + return m; + } + } + return Mode.BASE; + } + } + + @Data + public static class VerificationParam implements Serializable { + /** + * 验证模式 + */ + private String mode; + /** + * 休眠时间，用于验证超时 + */ + private Long sleepMs; + /** + * 【MR】批次大小，用于验证 MapReduce + */ + private Integer batchSize; + /** + * 【MR】batchNum + */ + private Integer batchNum; + /** + * 【MR】子任务成功率 + */ + private Double subTaskSuccessRate; + + private Integer responseSize; + } + + @Getter + @ToString + @NoArgsConstructor + @AllArgsConstructor + public static class TestSubTask { + private String taskName; + private int id; + } +} From c08b4f18584b623436f2887f6df74ddefb177e37 Mon Sep 17 00:00:00 2001 From: tjq Date: Sun, 13 Aug 2023 21:30:58 +0800 Subject: [PATCH 23/24] fix: timeout bug #678 --- .../official/processors/impl/VerificationProcessor.java | 6 +++++- .../tech/powerjob/worker/core/tracker/task/TaskTracker.java | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/powerjob-official-processors/src/main/java/tech/powerjob/official/processors/impl/VerificationProcessor.java b/powerjob-official-processors/src/main/java/tech/powerjob/official/processors/impl/VerificationProcessor.java index 394908d4..9a882dc0 100644 --- a/powerjob-official-processors/src/main/java/tech/powerjob/official/processors/impl/VerificationProcessor.java +++ b/powerjob-official-processors/src/main/java/tech/powerjob/official/processors/impl/VerificationProcessor.java @@ -76,7 +76,10 @@ public class VerificationProcessor extends CommonBasicProcessor implements MapRe } } - return new ProcessResult(true, "EXECUTE_SUCCESSFULLY_" + RandomStringUtils.randomAlphanumeric(Optional.ofNullable(verificationParam.getResponseSize()).orElse(10))); + + String randomMsg = RandomStringUtils.randomAlphanumeric(Optional.ofNullable(verificationParam.getResponseSize()).orElse(10)); + omsLogger.info("generate random string: {}", randomMsg); + return new ProcessResult(true, "EXECUTE_SUCCESSFULLY_" + randomMsg); } @Override @@ -87,6 +90,7 @@ public class VerificationProcessor extends CommonBasicProcessor implements MapRe enum Mode { /** * 常规模式，直接返回响应 + * {"mode":"BASE","responseSize":12} */ BASE, /** diff --git a/powerjob-worker/src/main/java/tech/powerjob/worker/core/tracker/task/TaskTracker.java b/powerjob-worker/src/main/java/tech/powerjob/worker/core/tracker/task/TaskTracker.java index a1edebc4..58a6bc3a 100644 --- a/powerjob-worker/src/main/java/tech/powerjob/worker/core/tracker/task/TaskTracker.java +++ b/powerjob-worker/src/main/java/tech/powerjob/worker/core/tracker/task/TaskTracker.java @@ -73,6 +73,7 @@ public abstract class TaskTracker { instanceInfo.setThreadConcurrency(req.getThreadConcurrency()); instanceInfo.setTaskRetryNum(req.getTaskRetryNum()); instanceInfo.setLogConfig(req.getLogConfig()); + instanceInfo.setInstanceTimeoutMS(req.getInstanceTimeoutMS()); // 特殊处理超时时间 if (instanceInfo.getInstanceTimeoutMS() <= 0) { From 15fa1abd915dd439517c3a13ca22a57cf4535c54 Mon Sep 17 00:00:00 2001 From: tjq Date: Sun, 13 Aug 2023 22:29:31 +0800 Subject: [PATCH 24/24] feat: Complete all testing and ready for release --- .../impl/VerificationProcessor.java | 80 ++++++++++++++++--- 1 file changed, 70 insertions(+), 10 deletions(-) diff --git a/powerjob-official-processors/src/main/java/tech/powerjob/official/processors/impl/VerificationProcessor.java b/powerjob-official-processors/src/main/java/tech/powerjob/official/processors/impl/VerificationProcessor.java index 9a882dc0..4588c22e 100644 --- a/powerjob-official-processors/src/main/java/tech/powerjob/official/processors/impl/VerificationProcessor.java +++ b/powerjob-official-processors/src/main/java/tech/powerjob/official/processors/impl/VerificationProcessor.java @@ -1,9 +1,11 @@ package tech.powerjob.official.processors.impl; import com.alibaba.fastjson.JSONObject; +import com.google.common.collect.Lists; import lombok.*; import org.apache.commons.lang3.RandomStringUtils; import tech.powerjob.common.exception.PowerJobException; +import tech.powerjob.common.utils.NetUtils; import tech.powerjob.official.processors.CommonBasicProcessor; import tech.powerjob.official.processors.util.CommonUtils; import tech.powerjob.worker.core.processor.ProcessResult; @@ -27,11 +29,6 @@ import java.util.concurrent.ThreadLocalRandom; */ public class VerificationProcessor extends CommonBasicProcessor implements MapReduceProcessor, BroadcastProcessor { - @Override - public ProcessResult preProcess(TaskContext context) throws Exception { - return new ProcessResult(true, "preProcess successfully!"); - } - @Override protected ProcessResult process0(TaskContext taskContext) throws Exception { @@ -44,13 +41,25 @@ public class VerificationProcessor extends CommonBasicProcessor implements MapRe switch (mode) { case ERROR: - return new ProcessResult(false, "EXECUTE_FAILED_DUE_TO_CONFIG"); + return new ProcessResult(false, "EXECUTE_FAILED_FOR_TEST"); case EXCEPTION: throw new PowerJobException("exception for test"); case TIMEOUT: final Long sleepMs = Optional.ofNullable(verificationParam.getSleepMs()).orElse(3600000L); Thread.sleep(sleepMs); return new ProcessResult(true, "AFTER_SLEEP_" + sleepMs); + case RETRY: + int currentRetryTimes = taskContext.getCurrentRetryTimes(); + int maxRetryTimes = taskContext.getMaxRetryTimes(); + omsLogger.info("[Retry] currentRetryTimes: {}, maxRetryTimes: {}", currentRetryTimes, maxRetryTimes); + if (currentRetryTimes < maxRetryTimes) { + Thread.sleep(100); + omsLogger.info("[Retry] currentRetryTimes[{}] < maxRetryTimes[{}], return failed status!", currentRetryTimes, maxRetryTimes); + return new ProcessResult(false, "FAILED_UNTIL_LAST_RETRY_" + currentRetryTimes); + } else { + omsLogger.info("[Retry] last retry, return success status!"); + return new ProcessResult(true, "RETRY_SUCCESSFULLY!"); + } case MR: if (isRootTask()) { final int batchNum = Optional.ofNullable(verificationParam.getBatchNum()).orElse(10); @@ -69,10 +78,14 @@ public class VerificationProcessor extends CommonBasicProcessor implements MapRe omsLogger.info("[VerificationProcessor] all map successfully!"); return new ProcessResult(true, "MAP_SUCCESS"); } else { + String taskId = taskContext.getTaskId(); final Double successRate = Optional.ofNullable(verificationParam.getSubTaskSuccessRate()).orElse(0.5); final double rd = ThreadLocalRandom.current().nextDouble(0, 1); boolean success = rd <= successRate; - return new ProcessResult(success, String.format("taskId_%s_success_%s", taskContext.getTaskId(), success)); + long processCost = ThreadLocalRandom.current().nextLong(277); + Thread.sleep(processCost); + omsLogger.info("[VerificationProcessor] [MR] taskId:{}, processCost: {}, success:{}", taskId, processCost, success); + return new ProcessResult(success, RandomStringUtils.randomAlphanumeric(3)); } } @@ -84,9 +97,47 @@ public class VerificationProcessor extends CommonBasicProcessor implements MapRe @Override public ProcessResult reduce(TaskContext context, List taskResults) { - return new ProcessResult(true, "REDUCE_SUCCESS"); + List successTaskIds = Lists.newArrayList(); + List failedTaskIds = Lists.newArrayList(); + StringBuilder sb = new StringBuilder(); + taskResults.forEach(taskResult -> { + sb.append("tId:").append(taskResult.getTaskId()).append(";") + .append("tSuc:").append(taskResult.isSuccess()).append(";") + .append("tRes:").append(taskResult.getResult()); + if (taskResult.isSuccess()) { + successTaskIds.add(taskResult.getTaskId()); + } else { + failedTaskIds.add(taskResult.getTaskId()); + } + }); + + context.getOmsLogger().info("[Reduce] [summary] successTaskNum: {}, failedTaskNum: {}, successRate: {}", + successTaskIds.size(), failedTaskIds.size(), 1.0 * successTaskIds.size() / (successTaskIds.size() + failedTaskIds.size())); + context.getOmsLogger().info("[Reduce] successTaskIds: {}", successTaskIds); + context.getOmsLogger().info("[Reduce] failedTaskIds: {}", failedTaskIds); + + return new ProcessResult(true, sb.toString()); } + /* ************************** 广播任务部分 ************************** */ + + @Override + public ProcessResult preProcess(TaskContext context) throws Exception { + context.getOmsLogger().info("start to preProcess, current worker IP is {}.", NetUtils.getLocalHost()); + return new ProcessResult(true, "preProcess successfully!"); + } + + @Override + public ProcessResult postProcess(TaskContext context, List taskResults) throws Exception { + OmsLogger omsLogger = context.getOmsLogger(); + omsLogger.info("start to postProcess, current worker IP is {}.", NetUtils.getLocalHost()); + omsLogger.info("====== All Node's Process Result ======"); + taskResults.forEach(r -> omsLogger.info("taskId:{},success:{},result:{}", r.getTaskId(), r.isSuccess(), r.getResult())); + return new ProcessResult(true, "postProcess successfully!"); + } + + /* ************************** 广播任务部分 ************************** */ + enum Mode { /** * 常规模式，直接返回响应 @@ -95,20 +146,29 @@ public class VerificationProcessor extends CommonBasicProcessor implements MapRe BASE, /** * 超时，sleep 一段时间测试超时控制 + * {"mode":"TIMEOUT","sleepMs":3600000} */ TIMEOUT, /** * 测试执行失败，响应返回 success = false + * {"mode":"ERROR"} */ ERROR, /** * 测试执行异常，抛出异常 + * {"mode":"EXCEPTION"} */ EXCEPTION, /** - * MapReduce + * MapReduce，需要控制台配置为 MapReduce 执行模式 + * {"mode":"MR","batchNum": 10, "batchSize": 20,"subTaskSuccessRate":0.7} */ - MR + MR, + /** + * 重试后成功，JOB 配置 Task 重试次数 + * {"mode":"EXCEPTION"} + */ + RETRY ; public static Mode of(String v) {