From d6abe79c17b1edb8c42d6b46bd5ec9ad70ef0ca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=B1=E5=85=AB?= Date: Sun, 26 Jul 2020 16:29:47 +0800 Subject: [PATCH 1/6] [dev] valid appName --- .../service/workflow/WorkflowInstanceManager.java | 2 +- .../server/web/controller/AppInfoController.java | 1 + .../server/web/request/ModifyAppInfoRequest.java | 10 ++++++++++ .../src/main/resources/application.properties | 2 ++ .../github/kfcfans/powerjob/server/test/UtilsTest.java | 9 +++++++++ 5 files changed, 23 insertions(+), 1 deletion(-) diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/workflow/WorkflowInstanceManager.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/workflow/WorkflowInstanceManager.java index b5cfbd56..955c831b 100644 --- a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/workflow/WorkflowInstanceManager.java +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/workflow/WorkflowInstanceManager.java @@ -204,7 +204,7 @@ public class WorkflowInstanceManager { node.setStatus(status.getV()); node.setResult(result); - log.debug("[Workflow-{}|{}] node(jobId={},instanceId={}) finished in workflowInstance, status={},result={}", wfId, wfInstanceId, node.getJobId(), instanceId, status.name(), result); + log.info("[Workflow-{}|{}] node(jobId={},instanceId={}) finished in workflowInstance, status={},result={}", wfId, wfInstanceId, node.getJobId(), instanceId, status.name(), result); } if (InstanceStatus.generalizedRunningStatus.contains(node.getStatus())) { diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/web/controller/AppInfoController.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/web/controller/AppInfoController.java index eb3f463d..5123c123 100644 --- a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/web/controller/AppInfoController.java +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/web/controller/AppInfoController.java @@ -41,6 +41,7 @@ public class AppInfoController { @PostMapping("/save") public ResultDTO saveAppInfo(@RequestBody ModifyAppInfoRequest req) { + req.valid(); AppInfoDO appInfoDO; Long id = req.getId(); diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/web/request/ModifyAppInfoRequest.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/web/request/ModifyAppInfoRequest.java index aeacc5db..77ea5da3 100644 --- a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/web/request/ModifyAppInfoRequest.java +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/web/request/ModifyAppInfoRequest.java @@ -1,6 +1,9 @@ package com.github.kfcfans.powerjob.server.web.request; +import com.github.kfcfans.powerjob.common.OmsException; +import com.github.kfcfans.powerjob.common.utils.CommonUtils; import lombok.Data; +import org.apache.commons.lang3.StringUtils; /** * 修改应用信息请求 @@ -14,4 +17,11 @@ public class ModifyAppInfoRequest { private Long id; private String appName; private String password; + + public void valid() { + CommonUtils.requireNonNull(appName, "appName can't be empty"); + if (StringUtils.containsWhitespace(appName)) { + throw new OmsException("appName can't contains white space!"); + } + } } diff --git a/powerjob-server/src/main/resources/application.properties b/powerjob-server/src/main/resources/application.properties index a9383fbe..781cd407 100644 --- a/powerjob-server/src/main/resources/application.properties +++ b/powerjob-server/src/main/resources/application.properties @@ -2,6 +2,7 @@ server.port=7700 spring.profiles.active=daily +spring.main.banner-mode=log spring.jpa.open-in-view=false spring.data.mongodb.repositories.type=none @@ -17,6 +18,7 @@ spring.datasource.druid.stat-view-servlet.url-pattern=/druid/* spring.datasource.druid.filter.stat.enabled=true spring.datasource.druid.filter.stat.log-slow-sql=true spring.datasource.druid.filter.stat.slow-sql-millis=5000 +spring.datasource.druid.stat-view-servlet.allow=* spring.datasource.druid.stat-view-servlet.enabled=true spring.datasource.druid.stat-view-servlet.login-username=powerjob spring.datasource.druid.stat-view-servlet.login-password=powerjob diff --git a/powerjob-server/src/test/java/com/github/kfcfans/powerjob/server/test/UtilsTest.java b/powerjob-server/src/test/java/com/github/kfcfans/powerjob/server/test/UtilsTest.java index 0703432f..8350e6e5 100644 --- a/powerjob-server/src/test/java/com/github/kfcfans/powerjob/server/test/UtilsTest.java +++ b/powerjob-server/src/test/java/com/github/kfcfans/powerjob/server/test/UtilsTest.java @@ -5,6 +5,7 @@ import com.github.kfcfans.powerjob.server.common.utils.timewheel.HashedWheelTime import com.github.kfcfans.powerjob.server.common.utils.timewheel.TimerFuture; import com.github.kfcfans.powerjob.server.common.utils.timewheel.TimerTask; import com.google.common.collect.Lists; +import org.apache.commons.lang3.StringUtils; import org.junit.Test; import java.util.Date; @@ -82,4 +83,12 @@ public class UtilsTest { public void testTZ() { System.out.println(TimeZone.getDefault()); } + + @Test + public void testStringUtils() { + String goodAppName = "powerjob-server"; + String appName = "powerjob-server "; + System.out.println(StringUtils.containsWhitespace(goodAppName)); + System.out.println(StringUtils.containsWhitespace(appName)); + } } From d2cfb6e3971169b41b6cba879a363b02e6a9bfd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=B1=E5=85=AB?= Date: Sun, 26 Jul 2020 20:03:48 +0800 Subject: [PATCH 2/6] [fix] fix the bug of frequent job can't execute when maxInstanceNum is 0 (report by @dudiao) --- .../common/utils/timewheel/HashedWheelTimer.java | 2 +- .../powerjob/server/service/DispatchService.java | 10 ++++++---- .../src/main/resources/application-daily.properties | 2 +- .../core/tracker/task/FrequentTaskTracker.java | 12 +++++++----- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/common/utils/timewheel/HashedWheelTimer.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/common/utils/timewheel/HashedWheelTimer.java index 646eebfb..26036070 100644 --- a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/common/utils/timewheel/HashedWheelTimer.java +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/common/utils/timewheel/HashedWheelTimer.java @@ -184,7 +184,6 @@ public class HashedWheelTimer implements Timer { log.warn("[HashedWheelTimer] timerFuture.totalTicks < currentTick, please fix the bug"); } - timerFuture.status = HashedWheelTimerFuture.RUNNING; try { // 提交执行 runTask(timerFuture); @@ -202,6 +201,7 @@ public class HashedWheelTimer implements Timer { } private void runTask(HashedWheelTimerFuture timerFuture) { + timerFuture.status = HashedWheelTimerFuture.RUNNING; if (taskProcessPool == null) { timerFuture.timerTask.run(); }else { diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/DispatchService.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/DispatchService.java index 299d0794..3c621821 100644 --- a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/DispatchService.java +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/DispatchService.java @@ -78,13 +78,15 @@ public class DispatchService { long current = System.currentTimeMillis(); // 0 代表不限制在线任务，还能省去一次 DB 查询 - if (jobInfo.getMaxInstanceNum() > 0) { + Integer maxInstanceNum = jobInfo.getMaxInstanceNum(); + if (maxInstanceNum > 0) { + // 这个 runningInstanceCount 已经包含了本 instance long runningInstanceCount = instanceInfoRepository.countByJobIdAndStatusIn(jobId, Lists.newArrayList(WAITING_WORKER_RECEIVE.getV(), RUNNING.getV())); // 超出最大同时运行限制，不执行调度 - if (runningInstanceCount > jobInfo.getMaxInstanceNum()) { - String result = String.format(SystemInstanceResult.TOO_MUCH_INSTANCE, runningInstanceCount, jobInfo.getMaxInstanceNum()); - log.warn("[Dispatcher-{}|{}] cancel dispatch job due to too much instance(num={}) is running.", jobId, instanceId, runningInstanceCount); + if (runningInstanceCount > maxInstanceNum) { + String result = String.format(SystemInstanceResult.TOO_MUCH_INSTANCE, runningInstanceCount, maxInstanceNum); + log.warn("[Dispatcher-{}|{}] cancel dispatch job due to too much instance is running ({} > {}).", jobId, instanceId, runningInstanceCount, maxInstanceNum); instanceInfoRepository.update4TriggerFailed(instanceId, FAILED.getV(), currentRunningTimes, current, current, RemoteConstant.EMPTY_ADDRESS, result, dbInstanceParams, now); instanceManager.processFinishedInstance(instanceId, wfInstanceId, FAILED, result); diff --git a/powerjob-server/src/main/resources/application-daily.properties b/powerjob-server/src/main/resources/application-daily.properties index a0e57d82..ed4040e0 100644 --- a/powerjob-server/src/main/resources/application-daily.properties +++ b/powerjob-server/src/main/resources/application-daily.properties @@ -3,7 +3,7 @@ logging.config=classpath:logback-dev.xml ####### 外部数据库配置（需要用户更改为自己的数据库配置） ####### spring.datasource.druid.driver-class-name=com.mysql.cj.jdbc.Driver -spring.datasource.druid.url=jdbc:mysql://remotehost:3306/powerjob-daily?useUnicode=true&characterEncoding=UTF-8 +spring.datasource.druid.url=jdbc:mysql://localhost:3306/powerjob-daily?useUnicode=true&characterEncoding=UTF-8 spring.datasource.druid.username=root spring.datasource.druid.password=No1Bug2Please3! spring.datasource.druid.initial-size=5 diff --git a/powerjob-worker/src/main/java/com/github/kfcfans/powerjob/worker/core/tracker/task/FrequentTaskTracker.java b/powerjob-worker/src/main/java/com/github/kfcfans/powerjob/worker/core/tracker/task/FrequentTaskTracker.java index 3e484b84..00cd28cc 100644 --- a/powerjob-worker/src/main/java/com/github/kfcfans/powerjob/worker/core/tracker/task/FrequentTaskTracker.java +++ b/powerjob-worker/src/main/java/com/github/kfcfans/powerjob/worker/core/tracker/task/FrequentTaskTracker.java @@ -171,11 +171,13 @@ public class FrequentTaskTracker extends TaskTracker { newRootTask.setLastReportTime(-1L); // 判断是否超出最大执行实例数 - if (timeExpressionType == TimeExpressionType.FIX_RATE) { - if (subInstanceId2TimeHolder.size() > maxInstanceNum) { - log.warn("[TaskTracker-{}] cancel to launch the subInstance({}) due to too much subInstance is running.", instanceId, subInstanceId); - processFinishedSubInstance(subInstanceId, false, "TOO_MUCH_INSTANCE"); - return; + if (maxInstanceNum > 0) { + if (timeExpressionType == TimeExpressionType.FIX_RATE) { + if (subInstanceId2TimeHolder.size() > maxInstanceNum) { + log.warn("[TaskTracker-{}] cancel to launch the subInstance({}) due to too much subInstance is running.", instanceId, subInstanceId); + processFinishedSubInstance(subInstanceId, false, "TOO_MUCH_INSTANCE"); + return; + } } } From 4faad700b9c3d296e6f8d25cc3e0ebcff200aff0 Mon Sep 17 00:00:00 2001 From: songyinyin Date: Sun, 26 Jul 2020 20:17:04 +0800 Subject: [PATCH 3/6] [feature] Add powerjob-worker-starter --- pom.xml | 1 + powerjob-worker-starter/pom.xml | 45 ++++++++++++++++ .../PowerJobAutoConfiguration.java | 52 +++++++++++++++++++ .../autoconfigure/PowerJobProperties.java | 44 ++++++++++++++++ .../spring-configuration-metadata.json | 51 ++++++++++++++++++ .../main/resources/META-INF/spring.factories | 2 + .../PowerJobAutoConfigurationTest.java | 22 ++++++++ .../src/test/resources/application.properties | 2 + 8 files changed, 219 insertions(+) create mode 100644 powerjob-worker-starter/pom.xml create mode 100644 powerjob-worker-starter/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobAutoConfiguration.java create mode 100644 powerjob-worker-starter/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobProperties.java create mode 100644 powerjob-worker-starter/src/main/resources/META-INF/spring-configuration-metadata.json create mode 100644 powerjob-worker-starter/src/main/resources/META-INF/spring.factories create mode 100644 powerjob-worker-starter/src/test/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobAutoConfigurationTest.java create mode 100644 powerjob-worker-starter/src/test/resources/application.properties diff --git a/pom.xml b/pom.xml index 99022ea4..484a8834 100644 --- a/pom.xml +++ b/pom.xml @@ -41,6 +41,7 @@ powerjob-common powerjob-client powerjob-worker-samples + powerjob-worker-starter powerjob-worker-agent diff --git a/powerjob-worker-starter/pom.xml b/powerjob-worker-starter/pom.xml new file mode 100644 index 00000000..8f281119 --- /dev/null +++ b/powerjob-worker-starter/pom.xml @@ -0,0 +1,45 @@ + + + + powerjob + com.github.kfcfans + 1.0.0 + + + 4.0.0 + powerjob-worker-starter + 3.2.0 + jar + + + 3.2.0 + 2.2.6.RELEASE + + + + + + com.github.kfcfans + powerjob-worker + ${powerjob.worker.version} + + + + org.springframework.boot + spring-boot-starter + ${springboot.version} + compile + + + + org.springframework.boot + spring-boot-starter-test + ${springboot.version} + test + + + + + \ No newline at end of file diff --git a/powerjob-worker-starter/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobAutoConfiguration.java b/powerjob-worker-starter/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobAutoConfiguration.java new file mode 100644 index 00000000..14391bcb --- /dev/null +++ b/powerjob-worker-starter/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobAutoConfiguration.java @@ -0,0 +1,52 @@ +package com.github.kfcfans.powerjob.worker.autoconfigure; + +import com.github.kfcfans.powerjob.worker.OhMyWorker; +import com.github.kfcfans.powerjob.worker.common.OhMyConfig; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.util.StringUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * PowerJob 自动装配 + * + * @author songyinyin + * @date 2020/7/26 16:37 + */ +@Configuration +@EnableConfigurationProperties(PowerJobProperties.class) +public class PowerJobAutoConfiguration { + + @Bean + @ConditionalOnMissingBean + public OhMyWorker initPowerJob(PowerJobProperties properties) { + + // 服务器HTTP地址（端口号为 server.port，而不是 ActorSystem port），请勿添加任何前缀（http://） + List serverAddress = StringUtils.hasText(properties.getServerAddress()) + ? Arrays.asList(properties.getServerAddress().split(",")) + : Collections.singletonList("127.0.0.1:7700"); + + // 1. 创建配置文件 + OhMyConfig config = new OhMyConfig(); + // 可以不显示设置，默认值 27777 + config.setPort(properties.getAkkaPort()); + // appName，需要提前在控制台注册，否则启动报错 + config.setAppName(properties.getAppName()); + config.setServerAddress(serverAddress); + // 如果没有大型 Map/MapReduce 的需求，建议使用内存来加速计算 + // 有大型 Map/MapReduce 需求，可能产生大量子任务（Task）的场景，请使用 DISK，否则妥妥的 OutOfMemeory + config.setStoreStrategy(properties.getStoreStrategy()); + // 启动测试模式，true情况下，不再尝试连接 server 并验证appName + config.setEnableTestMode(properties.isEnableTestMode()); + + // 2. 创建 Worker 对象，设置配置文件 + OhMyWorker ohMyWorker = new OhMyWorker(); + ohMyWorker.setConfig(config); + return ohMyWorker; + } +} diff --git a/powerjob-worker-starter/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobProperties.java b/powerjob-worker-starter/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobProperties.java new file mode 100644 index 00000000..220f260d --- /dev/null +++ b/powerjob-worker-starter/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobProperties.java @@ -0,0 +1,44 @@ +package com.github.kfcfans.powerjob.worker.autoconfigure; + +import com.github.kfcfans.powerjob.common.RemoteConstant; +import com.github.kfcfans.powerjob.worker.common.constants.StoreStrategy; +import com.github.kfcfans.powerjob.worker.core.processor.ProcessResult; +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * PowerJob 配置项 + * + * @author songyinyin + * @date 2020/7/26 16:37 + */ +@Data +@ConfigurationProperties(prefix = "powerjob") +public class PowerJobProperties { + /** + * 应用名称，需要提前在控制台注册，否则启动报错 + */ + private String appName; + /** + * 启动 akka 端口 + */ + private int akkaPort = RemoteConstant.DEFAULT_WORKER_PORT; + /** + * 调度服务器地址，ip:port 或 域名，多个用英文逗号分隔 + */ + private String serverAddress = "127.0.0.1:7700"; + /** + * 本地持久化方式，默认使用磁盘 + */ + private StoreStrategy storeStrategy = StoreStrategy.DISK; + /** + * 最大返回值长度，超过会被截断 + * {@link ProcessResult}#msg 的最大长度 + */ + private int maxResultLength = 8096; + /** + * 启动测试模式，true情况下，不再尝试连接 server 并验证appName。 + * true -> 用于本地写单元测试调试； false -> 默认值，标准模式 + */ + private boolean enableTestMode = false; +} diff --git a/powerjob-worker-starter/src/main/resources/META-INF/spring-configuration-metadata.json b/powerjob-worker-starter/src/main/resources/META-INF/spring-configuration-metadata.json new file mode 100644 index 00000000..6a012148 --- /dev/null +++ b/powerjob-worker-starter/src/main/resources/META-INF/spring-configuration-metadata.json @@ -0,0 +1,51 @@ +{ + "groups": [ + { + "name": "powerjob", + "type": "com.github.kfcfans.powerjob.worker.autoconfigure.PowerJobProperties", + "sourceType": "com.github.kfcfans.powerjob.worker.autoconfigure.PowerJobProperties" + } + ], + "properties": [ + { + "name": "powerjob.app-name", + "type": "java.lang.String", + "description": "应用名称，需要提前在控制台注册，否则启动报错", + "sourceType": "com.github.kfcfans.powerjob.worker.autoconfigure.PowerJobProperties" + }, + { + "name": "powerjob.max-result-length", + "type": "java.lang.Integer", + "description": "最大返回值长度，超过会被截断 {@link ProcessResult}#msg 的最大长度", + "sourceType": "com.github.kfcfans.powerjob.worker.autoconfigure.PowerJobProperties", + "defaultValue": 8096 + }, + { + "name": "powerjob.akka-port", + "type": "java.lang.Integer", + "description": "启动 akka 端口", + "sourceType": "com.github.kfcfans.powerjob.worker.autoconfigure.PowerJobProperties" + }, + { + "name": "powerjob.server-address", + "type": "java.lang.String", + "description": "调度服务器地址，ip:port 或 域名，多个用英文逗号分隔", + "sourceType": "com.github.kfcfans.powerjob.worker.autoconfigure.PowerJobProperties", + "defaultValue": "127.0.0.1:7700" + }, + { + "name": "powerjob.store-strategy", + "type": "com.github.kfcfans.powerjob.worker.common.constants.StoreStrategy", + "description": "本地持久化方式，默认使用磁盘", + "sourceType": "com.github.kfcfans.powerjob.worker.autoconfigure.PowerJobProperties" + }, + { + "name": "powerjob.enable-test-mode", + "type": "java.lang.Boolean", + "description": "启动测试模式，true情况下，不再尝试连接 server 并验证appName。true -> 用于本地写单元测试调试； false -> 默认值，标准模式", + "sourceType": "com.github.kfcfans.powerjob.worker.autoconfigure.PowerJobProperties", + "defaultValue": false + } + ], + "hints": [] +} \ No newline at end of file diff --git a/powerjob-worker-starter/src/main/resources/META-INF/spring.factories b/powerjob-worker-starter/src/main/resources/META-INF/spring.factories new file mode 100644 index 00000000..19eea33f --- /dev/null +++ b/powerjob-worker-starter/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + com.github.kfcfans.powerjob.worker.autoconfigure.PowerJobAutoConfiguration \ No newline at end of file diff --git a/powerjob-worker-starter/src/test/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobAutoConfigurationTest.java b/powerjob-worker-starter/src/test/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobAutoConfigurationTest.java new file mode 100644 index 00000000..dd90d395 --- /dev/null +++ b/powerjob-worker-starter/src/test/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobAutoConfigurationTest.java @@ -0,0 +1,22 @@ +package com.github.kfcfans.powerjob.worker.autoconfigure; + +import com.github.kfcfans.powerjob.worker.OhMyWorker; +import org.junit.Assert; +import org.junit.jupiter.api.Test; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Configuration; + +@Configuration +@EnableAutoConfiguration +class PowerJobAutoConfigurationTest { + + @Test + void testAutoConfiguration() { + ConfigurableApplicationContext run = SpringApplication.run(PowerJobAutoConfigurationTest.class); + OhMyWorker worker = run.getBean(OhMyWorker.class); + Assert.assertNotNull(worker); + } + +} diff --git a/powerjob-worker-starter/src/test/resources/application.properties b/powerjob-worker-starter/src/test/resources/application.properties new file mode 100644 index 00000000..fbcb42fa --- /dev/null +++ b/powerjob-worker-starter/src/test/resources/application.properties @@ -0,0 +1,2 @@ + +powerjob.enable-test-mode=true \ No newline at end of file From 8d8b8d504a240ea106b54231287e5cebf06b3152 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=B1=E5=85=AB?= Date: Sun, 26 Jul 2020 20:38:24 +0800 Subject: [PATCH 4/6] [opt] optimize worker-starter --- powerjob-worker-starter/pom.xml | 4 ++-- .../autoconfigure/PowerJobAutoConfiguration.java | 14 ++++++-------- .../worker/autoconfigure/PowerJobProperties.java | 4 ++-- .../META-INF/spring-configuration-metadata.json | 5 ++--- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/powerjob-worker-starter/pom.xml b/powerjob-worker-starter/pom.xml index 8f281119..43264e62 100644 --- a/powerjob-worker-starter/pom.xml +++ b/powerjob-worker-starter/pom.xml @@ -10,11 +10,11 @@ 4.0.0 powerjob-worker-starter - 3.2.0 + 3.2.1 jar - 3.2.0 + 3.2.1 2.2.6.RELEASE diff --git a/powerjob-worker-starter/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobAutoConfiguration.java b/powerjob-worker-starter/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobAutoConfiguration.java index 14391bcb..6f6000c4 100644 --- a/powerjob-worker-starter/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobAutoConfiguration.java +++ b/powerjob-worker-starter/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobAutoConfiguration.java @@ -1,22 +1,21 @@ package com.github.kfcfans.powerjob.worker.autoconfigure; +import com.github.kfcfans.powerjob.common.utils.CommonUtils; import com.github.kfcfans.powerjob.worker.OhMyWorker; import com.github.kfcfans.powerjob.worker.common.OhMyConfig; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.util.StringUtils; import java.util.Arrays; -import java.util.Collections; import java.util.List; /** * PowerJob 自动装配 * * @author songyinyin - * @date 2020/7/26 16:37 + * @since 2020/7/26 16:37 */ @Configuration @EnableConfigurationProperties(PowerJobProperties.class) @@ -27,19 +26,18 @@ public class PowerJobAutoConfiguration { public OhMyWorker initPowerJob(PowerJobProperties properties) { // 服务器HTTP地址（端口号为 server.port，而不是 ActorSystem port），请勿添加任何前缀（http://） - List serverAddress = StringUtils.hasText(properties.getServerAddress()) - ? Arrays.asList(properties.getServerAddress().split(",")) - : Collections.singletonList("127.0.0.1:7700"); + CommonUtils.requireNonNull(properties.getServerAddress(), "serverAddress can't be empty!"); + List serverAddress = Arrays.asList(properties.getServerAddress().split(",")); // 1. 创建配置文件 OhMyConfig config = new OhMyConfig(); - // 可以不显示设置，默认值 27777 + // 可以不显式设置，默认值 27777 config.setPort(properties.getAkkaPort()); // appName，需要提前在控制台注册，否则启动报错 config.setAppName(properties.getAppName()); config.setServerAddress(serverAddress); // 如果没有大型 Map/MapReduce 的需求，建议使用内存来加速计算 - // 有大型 Map/MapReduce 需求，可能产生大量子任务（Task）的场景，请使用 DISK，否则妥妥的 OutOfMemeory + // 有大型 Map/MapReduce 需求，可能产生大量子任务（Task）的场景，请使用 DISK，否则妥妥的 OutOfMemory config.setStoreStrategy(properties.getStoreStrategy()); // 启动测试模式，true情况下，不再尝试连接 server 并验证appName config.setEnableTestMode(properties.isEnableTestMode()); diff --git a/powerjob-worker-starter/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobProperties.java b/powerjob-worker-starter/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobProperties.java index 220f260d..b8cf30b9 100644 --- a/powerjob-worker-starter/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobProperties.java +++ b/powerjob-worker-starter/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobProperties.java @@ -10,7 +10,7 @@ import org.springframework.boot.context.properties.ConfigurationProperties; * PowerJob 配置项 * * @author songyinyin - * @date 2020/7/26 16:37 + * @since 2020/7/26 16:37 */ @Data @ConfigurationProperties(prefix = "powerjob") @@ -26,7 +26,7 @@ public class PowerJobProperties { /** * 调度服务器地址，ip:port 或 域名，多个用英文逗号分隔 */ - private String serverAddress = "127.0.0.1:7700"; + private String serverAddress; /** * 本地持久化方式，默认使用磁盘 */ diff --git a/powerjob-worker-starter/src/main/resources/META-INF/spring-configuration-metadata.json b/powerjob-worker-starter/src/main/resources/META-INF/spring-configuration-metadata.json index 6a012148..1fbedb0a 100644 --- a/powerjob-worker-starter/src/main/resources/META-INF/spring-configuration-metadata.json +++ b/powerjob-worker-starter/src/main/resources/META-INF/spring-configuration-metadata.json @@ -29,9 +29,8 @@ { "name": "powerjob.server-address", "type": "java.lang.String", - "description": "调度服务器地址，ip:port 或 域名，多个用英文逗号分隔", - "sourceType": "com.github.kfcfans.powerjob.worker.autoconfigure.PowerJobProperties", - "defaultValue": "127.0.0.1:7700" + "description": "调度服务器地址，ip:port 或 域名，多值用英文逗号分隔", + "sourceType": "com.github.kfcfans.powerjob.worker.autoconfigure.PowerJobProperties" }, { "name": "powerjob.store-strategy", From 97f1bbca6cf955cc4c5ba9754f5ad0868fbf75ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=B1=E5=85=AB?= Date: Sun, 26 Jul 2020 22:18:29 +0800 Subject: [PATCH 5/6] [opt] rollback database, use Hikari again --- powerjob-server/pom.xml | 13 --------- .../config/MultiDatasourceConfig.java | 29 +++++++++---------- .../resources/application-daily.properties | 16 +++++----- .../main/resources/application-pre.properties | 14 ++++----- .../resources/application-product.properties | 14 ++++----- .../src/main/resources/application.properties | 12 +------- 6 files changed, 34 insertions(+), 64 deletions(-) diff --git a/powerjob-server/pom.xml b/powerjob-server/pom.xml index 464668af..dc745cf1 100644 --- a/powerjob-server/pom.xml +++ b/powerjob-server/pom.xml @@ -24,7 +24,6 @@ 3.0.1 3.6 1.2.68 - 1.1.23 true @@ -79,18 +78,6 @@ spring-boot-starter-data-jpa ${springboot.version} - - - com.alibaba - druid-spring-boot-starter - ${druid.starter.version} - - - org.springframework.boot - spring-boot-autoconfigure - - - org.springframework.boot spring-boot-starter-data-mongodb diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/config/MultiDatasourceConfig.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/config/MultiDatasourceConfig.java index 53550268..62f55b99 100644 --- a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/config/MultiDatasourceConfig.java +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/config/MultiDatasourceConfig.java @@ -1,8 +1,9 @@ package com.github.kfcfans.powerjob.server.persistence.config; -import com.alibaba.druid.pool.DruidDataSource; -import com.alibaba.druid.spring.boot.autoconfigure.DruidDataSourceBuilder; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.jdbc.DataSourceBuilder; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; @@ -20,28 +21,26 @@ public class MultiDatasourceConfig { private static final String H2_DRIVER_CLASS_NAME = "org.h2.Driver"; private static final String H2_JDBC_URL = "jdbc:h2:file:~/powerjob-server/h2/powerjob_server_db"; - private static final int H2_INITIAL_SIZE = 4; private static final int H2_MIN_SIZE = 4; private static final int H2_MAX_ACTIVE_SIZE = 10; - private static final String H2_DATASOURCE_NAME = "localDatasource"; @Primary @Bean("omsCoreDatasource") - @ConfigurationProperties(prefix = "spring.datasource.druid") + @ConfigurationProperties(prefix = "spring.datasource.core") public DataSource initOmsCoreDatasource() { - return DruidDataSourceBuilder.create().build(); + return DataSourceBuilder.create().build(); } @Bean("omsLocalDatasource") public DataSource initOmsLocalDatasource() { - DruidDataSource ds = new DruidDataSource(); - ds.setDriverClassName(H2_DRIVER_CLASS_NAME); - ds.setUrl(H2_JDBC_URL); - ds.setInitialSize(H2_INITIAL_SIZE); - ds.setMinIdle(H2_MIN_SIZE); - ds.setMaxActive(H2_MAX_ACTIVE_SIZE); - ds.setName(H2_DATASOURCE_NAME); - ds.setTestWhileIdle(false); - return ds; + HikariConfig config = new HikariConfig(); + config.setDriverClassName(H2_DRIVER_CLASS_NAME); + config.setJdbcUrl(H2_JDBC_URL); + config.setAutoCommit(true); + // 池中最小空闲连接数量 + config.setMinimumIdle(H2_MIN_SIZE); + // 池中最大连接数量 + config.setMaximumPoolSize(H2_MAX_ACTIVE_SIZE); + return new HikariDataSource(config); } } diff --git a/powerjob-server/src/main/resources/application-daily.properties b/powerjob-server/src/main/resources/application-daily.properties index ed4040e0..cb7778b6 100644 --- a/powerjob-server/src/main/resources/application-daily.properties +++ b/powerjob-server/src/main/resources/application-daily.properties @@ -2,17 +2,15 @@ oms.env=DAILY logging.config=classpath:logback-dev.xml ####### 外部数据库配置（需要用户更改为自己的数据库配置） ####### -spring.datasource.druid.driver-class-name=com.mysql.cj.jdbc.Driver -spring.datasource.druid.url=jdbc:mysql://localhost:3306/powerjob-daily?useUnicode=true&characterEncoding=UTF-8 -spring.datasource.druid.username=root -spring.datasource.druid.password=No1Bug2Please3! -spring.datasource.druid.initial-size=5 -spring.datasource.druid.max-active=10 -spring.datasource.druid.test-while-idle=false -spring.datasource.druid.name=remoteDatasource +spring.datasource.core.driver-class-name=com.mysql.cj.jdbc.Driver +spring.datasource.core.jdbc-url=jdbc:mysql://localhost:3306/powerjob-daily?useUnicode=true&characterEncoding=UTF-8 +spring.datasource.core.username=root +spring.datasource.core.password=No1Bug2Please3! +spring.datasource.core.hikari.maximum-pool-size=20 +spring.datasource.core.hikari.minimum-idle=5 ####### mongoDB配置，非核心依赖，可移除 ####### -spring.data.mongodb.uri=mongodb://remotehost:27017/powerjob-daily +spring.data.mongodb.uri=mongodb://localhost:27017/powerjob-daily ####### 邮件配置（启用邮件报警则需要） ####### spring.mail.host=smtp.163.com diff --git a/powerjob-server/src/main/resources/application-pre.properties b/powerjob-server/src/main/resources/application-pre.properties index 2825e6d9..f2d33b88 100644 --- a/powerjob-server/src/main/resources/application-pre.properties +++ b/powerjob-server/src/main/resources/application-pre.properties @@ -2,14 +2,12 @@ oms.env=PRE logging.config=classpath:logback-product.xml ####### 数据库配置 ####### -spring.datasource.druid.driver-class-name=com.mysql.cj.jdbc.Driver -spring.datasource.druid.url=jdbc:mysql://remotehost:3306/powerjob-pre?useUnicode=true&characterEncoding=UTF-8 -spring.datasource.druid.username=root -spring.datasource.druid.password=No1Bug2Please3! -spring.datasource.druid.initial-size=5 -spring.datasource.druid.max-active=20 -spring.datasource.druid.test-while-idle=false -spring.datasource.druid.name=remoteDatasource +spring.datasource.core.driver-class-name=com.mysql.cj.jdbc.Driver +spring.datasource.core.jdbc-url=jdbc:mysql://remotehost:3306/powerjob-pre?useUnicode=true&characterEncoding=UTF-8 +spring.datasource.core.username=root +spring.datasource.core.password=No1Bug2Please3! +spring.datasource.core.hikari.maximum-pool-size=20 +spring.datasource.core.hikari.minimum-idle=5 ####### mongoDB配置，非核心依赖，可移除 ####### spring.data.mongodb.uri=mongodb://remotehost:27017/powerjob-pre diff --git a/powerjob-server/src/main/resources/application-product.properties b/powerjob-server/src/main/resources/application-product.properties index 5215c7aa..0a4e2556 100644 --- a/powerjob-server/src/main/resources/application-product.properties +++ b/powerjob-server/src/main/resources/application-product.properties @@ -2,14 +2,12 @@ oms.env=PRODUCT logging.config=classpath:logback-product.xml ####### 数据库配置 ####### -spring.datasource.druid.driver-class-name=com.mysql.cj.jdbc.Driver -spring.datasource.druid.url=jdbc:mysql://localhost:3306/powerjob-product?useUnicode=true&characterEncoding=UTF-8 -spring.datasource.druid.username=root -spring.datasource.druid.password=No1Bug2Please3! -spring.datasource.druid.initial-size=5 -spring.datasource.druid.max-active=20 -spring.datasource.druid.test-while-idle=false -spring.datasource.druid.name=remoteDatasource +spring.datasource.core.driver-class-name=com.mysql.cj.jdbc.Driver +spring.datasource.core.jdbc-url=jdbc:mysql://localhost:3306/powerjob-product?useUnicode=true&characterEncoding=UTF-8 +spring.datasource.core.username=root +spring.datasource.core.password=No1Bug2Please3! +spring.datasource.core.hikari.maximum-pool-size=20 +spring.datasource.core.hikari.minimum-idle=5 ####### mongoDB配置，非核心依赖，可移除 ####### spring.data.mongodb.uri=mongodb://localhost:27017/powerjob-product diff --git a/powerjob-server/src/main/resources/application.properties b/powerjob-server/src/main/resources/application.properties index 781cd407..6dc9fd8b 100644 --- a/powerjob-server/src/main/resources/application.properties +++ b/powerjob-server/src/main/resources/application.properties @@ -5,6 +5,7 @@ spring.profiles.active=daily spring.main.banner-mode=log spring.jpa.open-in-view=false spring.data.mongodb.repositories.type=none +logging.level.org.mongodb=warn # 文件上传配置 spring.servlet.multipart.enabled=true @@ -12,17 +13,6 @@ spring.servlet.multipart.file-size-threshold=0 spring.servlet.multipart.max-file-size=209715200 spring.servlet.multipart.max-request-size=209715200 -# druid 监控配置（监控地址 /druid，登陆账号和密码默认都是 powerjob） -spring.datasource.druid.filters=stat -spring.datasource.druid.stat-view-servlet.url-pattern=/druid/* -spring.datasource.druid.filter.stat.enabled=true -spring.datasource.druid.filter.stat.log-slow-sql=true -spring.datasource.druid.filter.stat.slow-sql-millis=5000 -spring.datasource.druid.stat-view-servlet.allow=* -spring.datasource.druid.stat-view-servlet.enabled=true -spring.datasource.druid.stat-view-servlet.login-username=powerjob -spring.datasource.druid.stat-view-servlet.login-password=powerjob - ###### PowerJob 自身配置（该配置只允许存在于 application.properties 文件中） ###### # akka ActorSystem 服务端口 oms.akka.port=10086 From 5974da1ae6552b17a703741d342eb9561d4274d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=B1=E5=85=AB?= Date: Sun, 26 Jul 2020 22:31:50 +0800 Subject: [PATCH 6/6] [opt] change name to spring-boot-starter --- pom.xml | 4 ++-- .../pom.xml | 2 +- .../worker/autoconfigure/PowerJobAutoConfiguration.java | 0 .../powerjob/worker/autoconfigure/PowerJobProperties.java | 0 .../resources/META-INF/spring-configuration-metadata.json | 0 .../src/main/resources/META-INF/spring.factories | 0 .../worker/autoconfigure/PowerJobAutoConfigurationTest.java | 0 .../src/test/resources/application.properties | 0 8 files changed, 3 insertions(+), 3 deletions(-) rename {powerjob-worker-starter => powerjob-worker-spring-boot-starter}/pom.xml (95%) rename {powerjob-worker-starter => powerjob-worker-spring-boot-starter}/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobAutoConfiguration.java (100%) rename {powerjob-worker-starter => powerjob-worker-spring-boot-starter}/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobProperties.java (100%) rename {powerjob-worker-starter => powerjob-worker-spring-boot-starter}/src/main/resources/META-INF/spring-configuration-metadata.json (100%) rename {powerjob-worker-starter => powerjob-worker-spring-boot-starter}/src/main/resources/META-INF/spring.factories (100%) rename {powerjob-worker-starter => powerjob-worker-spring-boot-starter}/src/test/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobAutoConfigurationTest.java (100%) rename {powerjob-worker-starter => powerjob-worker-spring-boot-starter}/src/test/resources/application.properties (100%) diff --git a/pom.xml b/pom.xml index 484a8834..24f4c25b 100644 --- a/pom.xml +++ b/pom.xml @@ -40,9 +40,9 @@ powerjob-server powerjob-common powerjob-client - powerjob-worker-samples - powerjob-worker-starter powerjob-worker-agent + powerjob-worker-spring-boot-starter + powerjob-worker-samples diff --git a/powerjob-worker-starter/pom.xml b/powerjob-worker-spring-boot-starter/pom.xml similarity index 95% rename from powerjob-worker-starter/pom.xml rename to powerjob-worker-spring-boot-starter/pom.xml index 43264e62..1e7eb819 100644 --- a/powerjob-worker-starter/pom.xml +++ b/powerjob-worker-spring-boot-starter/pom.xml @@ -9,7 +9,7 @@ 4.0.0 - powerjob-worker-starter + powerjob-worker-spring-boot-starter 3.2.1 jar diff --git a/powerjob-worker-starter/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobAutoConfiguration.java b/powerjob-worker-spring-boot-starter/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobAutoConfiguration.java similarity index 100% rename from powerjob-worker-starter/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobAutoConfiguration.java rename to powerjob-worker-spring-boot-starter/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobAutoConfiguration.java diff --git a/powerjob-worker-starter/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobProperties.java b/powerjob-worker-spring-boot-starter/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobProperties.java similarity index 100% rename from powerjob-worker-starter/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobProperties.java rename to powerjob-worker-spring-boot-starter/src/main/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobProperties.java diff --git a/powerjob-worker-starter/src/main/resources/META-INF/spring-configuration-metadata.json b/powerjob-worker-spring-boot-starter/src/main/resources/META-INF/spring-configuration-metadata.json similarity index 100% rename from powerjob-worker-starter/src/main/resources/META-INF/spring-configuration-metadata.json rename to powerjob-worker-spring-boot-starter/src/main/resources/META-INF/spring-configuration-metadata.json diff --git a/powerjob-worker-starter/src/main/resources/META-INF/spring.factories b/powerjob-worker-spring-boot-starter/src/main/resources/META-INF/spring.factories similarity index 100% rename from powerjob-worker-starter/src/main/resources/META-INF/spring.factories rename to powerjob-worker-spring-boot-starter/src/main/resources/META-INF/spring.factories diff --git a/powerjob-worker-starter/src/test/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobAutoConfigurationTest.java b/powerjob-worker-spring-boot-starter/src/test/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobAutoConfigurationTest.java similarity index 100% rename from powerjob-worker-starter/src/test/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobAutoConfigurationTest.java rename to powerjob-worker-spring-boot-starter/src/test/java/com/github/kfcfans/powerjob/worker/autoconfigure/PowerJobAutoConfigurationTest.java diff --git a/powerjob-worker-starter/src/test/resources/application.properties b/powerjob-worker-spring-boot-starter/src/test/resources/application.properties similarity index 100% rename from powerjob-worker-starter/src/test/resources/application.properties rename to powerjob-worker-spring-boot-starter/src/test/resources/application.properties