From c3ce46aee9976fe5179493d91f37be75aff65e39 Mon Sep 17 00:00:00 2001 From: tjq Date: Mon, 31 Jul 2023 23:45:52 +0800 Subject: [PATCH] refactor: optimize alarm code --- powerjob-server/pom.xml | 9 +++ .../powerjob-server-common/pom.xml | 7 ++ .../server/common/utils/TestUtils.java | 42 +++++++++++ .../server/core/alarm/AlarmCenter.java | 28 +++---- .../server/core/alarm/AlarmUtils.java | 35 +++++++++ .../core/alarm/module/JobInstanceAlarm.java | 2 + .../server/core/instance/InstanceManager.java | 3 +- .../workflow/WorkflowInstanceManager.java | 3 +- .../server/extension/alarm/AlarmTarget.java | 2 + .../storage/impl/GridFsService.java | 2 +- .../storage/impl/GridFsServiceTest.java | 30 +++----- .../server/web/controller/TestController.java | 75 +++++++++++++++++++ .../resources/application-daily.properties | 3 +- .../main/resources/application-pre.properties | 3 +- .../resources/application-product.properties | 3 +- 15 files changed, 199 insertions(+), 48 deletions(-) create mode 100644 powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/utils/TestUtils.java create mode 100644 powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/AlarmUtils.java create mode 100644 powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/web/controller/TestController.java diff --git a/powerjob-server/pom.xml b/powerjob-server/pom.xml index 6c762e93..a75f4158 100644 --- a/powerjob-server/pom.xml +++ b/powerjob-server/pom.xml @@ -55,6 +55,7 @@ 4.3.3 1.6.14 3.17.1 + 4.4 @@ -113,6 +114,14 @@ ${aliyun-sdk-oss.version} + + + org.apache.commons + commons-collections4 + ${commons-collections4.version} + + + diff --git a/powerjob-server/powerjob-server-common/pom.xml b/powerjob-server/powerjob-server-common/pom.xml index bbff066f..2cd23f6c 100644 --- a/powerjob-server/powerjob-server-common/pom.xml +++ b/powerjob-server/powerjob-server-common/pom.xml @@ -18,4 +18,11 @@ 8 + + + org.apache.commons + commons-collections4 + + + \ No newline at end of file diff --git a/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/utils/TestUtils.java b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/utils/TestUtils.java new file mode 100644 index 00000000..1440b6e0 --- /dev/null +++ b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/utils/TestUtils.java @@ -0,0 +1,42 @@ +package tech.powerjob.server.common.utils; + +import com.alibaba.fastjson.JSONObject; +import com.google.common.collect.Maps; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +/** + * 开发团队专用测试工具 + * + * @author tjq + * @since 2023/7/31 + */ +public class TestUtils { + + private static final String TEST_CONFIG_NAME = "/.powerjob_test"; + + public static final String KEY_PHONE_NUMBER = "phone"; + + public static final String KEY_MONGO_URI = "mongoUri"; + + /** + * 获取本地的测试配置，主要用于存放一些密钥 + * @return 测试配置 + */ + public static Map fetchTestConfig() { + try { + // 后续本地测试，密钥相关的内容统一存入 .powerjob_test 中，方便管理 + String content = FileUtils.readFileToString(new File(System.getProperty("user.home").concat(TEST_CONFIG_NAME)), StandardCharsets.UTF_8); + if (StringUtils.isNotEmpty(content)) { + return JSONObject.parseObject(content); + } + } catch (Exception ignore) { + } + return Maps.newHashMap(); + } + +} diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/AlarmCenter.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/AlarmCenter.java index cb837fdc..bde075a9 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/AlarmCenter.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/AlarmCenter.java @@ -1,19 +1,19 @@ package tech.powerjob.server.core.alarm; -import org.springframework.beans.BeanUtils; -import org.springframework.stereotype.Component; -import tech.powerjob.server.extension.alarm.Alarm; -import tech.powerjob.server.extension.alarm.AlarmTarget; -import tech.powerjob.server.extension.alarm.Alarmable; -import tech.powerjob.server.persistence.remote.model.UserInfoDO; import com.google.common.collect.Lists; import com.google.common.collect.Queues; import com.google.common.util.concurrent.ThreadFactoryBuilder; import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import tech.powerjob.server.extension.alarm.Alarm; +import tech.powerjob.server.extension.alarm.AlarmTarget; +import tech.powerjob.server.extension.alarm.Alarmable; import java.util.List; -import java.util.concurrent.*; -import java.util.stream.Collectors; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; /** * 报警服务 @@ -40,21 +40,13 @@ public class AlarmCenter { }); } - public void alarmFailed(Alarm alarm, List targetUserList) { + public void alarmFailed(Alarm alarm, List alarmTargets) { POOL.execute(() -> BEANS.forEach(alarmable -> { try { - alarmable.onFailed(alarm, targetUserList.stream().map(AlarmCenter::convertUserInfo2AlarmTarget).collect(Collectors.toList())); + alarmable.onFailed(alarm, alarmTargets); }catch (Exception e) { log.warn("[AlarmCenter] alarm failed.", e); } })); } - - private static AlarmTarget convertUserInfo2AlarmTarget(UserInfoDO userInfoDO) { - AlarmTarget alarmTarget = new AlarmTarget(); - BeanUtils.copyProperties(userInfoDO, alarmTarget); - - alarmTarget.setName(userInfoDO.getUsername()); - return alarmTarget; - } } diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/AlarmUtils.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/AlarmUtils.java new file mode 100644 index 00000000..2db55b42 --- /dev/null +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/AlarmUtils.java @@ -0,0 +1,35 @@ +package tech.powerjob.server.core.alarm; + +import org.springframework.beans.BeanUtils; +import tech.powerjob.common.utils.CollectionUtils; +import tech.powerjob.server.extension.alarm.AlarmTarget; +import tech.powerjob.server.persistence.remote.model.UserInfoDO; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +/** + * AlarmUtils + * + * @author tjq + * @since 2023/7/31 + */ +public class AlarmUtils { + + public static List convertUserInfoList2AlarmTargetList(List userInfoDOS) { + if (CollectionUtils.isEmpty(userInfoDOS)) { + return Collections.emptyList(); + } + return userInfoDOS.stream().map(AlarmUtils::convertUserInfo2AlarmTarget).collect(Collectors.toList()); + } + + public static AlarmTarget convertUserInfo2AlarmTarget(UserInfoDO userInfoDO) { + AlarmTarget alarmTarget = new AlarmTarget(); + BeanUtils.copyProperties(userInfoDO, alarmTarget); + + alarmTarget.setName(userInfoDO.getUsername()); + return alarmTarget; + } + +} diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/module/JobInstanceAlarm.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/module/JobInstanceAlarm.java index 5c2d907a..b4d4f400 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/module/JobInstanceAlarm.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/alarm/module/JobInstanceAlarm.java @@ -1,6 +1,7 @@ package tech.powerjob.server.core.alarm.module; import lombok.Data; +import lombok.experimental.Accessors; import tech.powerjob.server.extension.alarm.Alarm; /** @@ -10,6 +11,7 @@ import tech.powerjob.server.extension.alarm.Alarm; * @since 2020/4/30 */ @Data +@Accessors(chain = true) public class JobInstanceAlarm implements Alarm { /** * 应用ID diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceManager.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceManager.java index 274a8a5b..36511e46 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceManager.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceManager.java @@ -15,6 +15,7 @@ import tech.powerjob.remote.framework.base.URL; import tech.powerjob.server.common.module.WorkerInfo; import tech.powerjob.server.common.timewheel.holder.HashedWheelTimerHolder; import tech.powerjob.server.common.utils.SpringUtils; +import tech.powerjob.server.core.alarm.AlarmUtils; import tech.powerjob.server.core.service.UserService; import tech.powerjob.server.core.workflow.WorkflowInstanceManager; import tech.powerjob.server.core.alarm.AlarmCenter; @@ -239,7 +240,7 @@ public class InstanceManager implements TransportServiceAware { if (!StringUtils.isEmpty(alertContent)) { content.setResult(alertContent); } - alarmCenter.alarmFailed(content, userList); + alarmCenter.alarmFailed(content, AlarmUtils.convertUserInfoList2AlarmTargetList(userList)); } @Override diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/workflow/WorkflowInstanceManager.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/workflow/WorkflowInstanceManager.java index 001c3e12..2c6cdd78 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/workflow/WorkflowInstanceManager.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/workflow/WorkflowInstanceManager.java @@ -20,6 +20,7 @@ import tech.powerjob.common.serialize.JsonUtils; import tech.powerjob.common.utils.CommonUtils; import tech.powerjob.server.common.constants.SwitchableStatus; import tech.powerjob.server.common.utils.SpringUtils; +import tech.powerjob.server.core.alarm.AlarmUtils; import tech.powerjob.server.core.helper.StatusMappingHelper; import tech.powerjob.server.core.lock.UseCacheLock; import tech.powerjob.server.core.service.UserService; @@ -458,7 +459,7 @@ public class WorkflowInstanceManager { content.setResult(result); List userList = userService.fetchNotifyUserList(wfInfo.getNotifyUserIds()); - alarmCenter.alarmFailed(content, userList); + alarmCenter.alarmFailed(content, AlarmUtils.convertUserInfoList2AlarmTargetList(userList)); }); } catch (Exception ignore) { // ignore diff --git a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/alarm/AlarmTarget.java b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/alarm/AlarmTarget.java index 6512f6f7..982ba398 100644 --- a/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/alarm/AlarmTarget.java +++ b/powerjob-server/powerjob-server-extension/src/main/java/tech/powerjob/server/extension/alarm/AlarmTarget.java @@ -1,6 +1,7 @@ package tech.powerjob.server.extension.alarm; import lombok.Data; +import lombok.experimental.Accessors; import java.io.Serializable; import java.util.Map; @@ -13,6 +14,7 @@ import java.util.Objects; * @since 2023/7/16 */ @Data +@Accessors(chain = true) public class AlarmTarget implements Serializable { private String name; diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java index facbc2f9..c5c4f70c 100644 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java @@ -163,7 +163,7 @@ public class GridFsService extends AbstractDFsService { public static class GridFsCondition extends PropertyAndOneBeanCondition { @Override protected List anyConfigKey() { - return Lists.newArrayList("spring.data.mongodb.uri", "oms.storage.dfs.mongo.uri"); + return Lists.newArrayList("spring.data.mongodb.uri", "oms.storage.dfs.mongodb.uri"); } @Override diff --git a/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/GridFsServiceTest.java b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/GridFsServiceTest.java index 1abfba2d..ba0d7c2e 100644 --- a/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/GridFsServiceTest.java +++ b/powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/GridFsServiceTest.java @@ -1,13 +1,9 @@ package tech.powerjob.server.persistence.storage.impl; -import com.alibaba.fastjson.JSONObject; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; +import tech.powerjob.server.common.utils.TestUtils; import tech.powerjob.server.extension.dfs.DFsService; -import java.io.File; -import java.nio.charset.StandardCharsets; import java.util.Optional; /** @@ -22,23 +18,15 @@ class GridFsServiceTest extends AbstractDfsServiceTest { @Override protected Optional fetchService() { - try { - // 后续本地测试，密钥相关的内容统一存入 .powerjob_test 中，方便管理 - String content = FileUtils.readFileToString(new File(System.getProperty("user.home").concat("/.powerjob_test")), StandardCharsets.UTF_8); - if (StringUtils.isNotEmpty(content)) { - JSONObject jsonObject = JSONObject.parseObject(content); - Object mongoUri = jsonObject.get("mongoUri"); - if (mongoUri != null) { - GridFsService gridFsService = new GridFsService(); - gridFsService.initMongo(String.valueOf(mongoUri)); + Object mongoUri = TestUtils.fetchTestConfig().get(TestUtils.KEY_MONGO_URI); - return Optional.of(gridFsService); - } - } - } catch (Exception e) { - e.printStackTrace(); - log.warn("[GridFsServiceTest] fetch mongo config failed, skip!"); + if (mongoUri == null) { + log.info("[GridFsServiceTest] mongoUri is null, skip load!"); + return Optional.empty(); } - return Optional.empty(); + + GridFsService gridFsService = new GridFsService(); + gridFsService.initMongo(String.valueOf(mongoUri)); + return Optional.of(gridFsService); } } \ No newline at end of file diff --git a/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/web/controller/TestController.java b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/web/controller/TestController.java new file mode 100644 index 00000000..c7fea8be --- /dev/null +++ b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/web/controller/TestController.java @@ -0,0 +1,75 @@ +package tech.powerjob.server.web.controller; + +import com.google.common.collect.Lists; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.MapUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import tech.powerjob.common.serialize.JsonUtils; +import tech.powerjob.common.utils.CollectionUtils; +import tech.powerjob.server.common.utils.TestUtils; +import tech.powerjob.server.core.alarm.AlarmCenter; +import tech.powerjob.server.core.alarm.module.JobInstanceAlarm; +import tech.powerjob.server.extension.alarm.AlarmTarget; + +import javax.annotation.Resource; +import java.util.Map; + +/** + * 开发团队专用（或者 PRO 用户用来做自检也可以 lol） + * 测试某些强依赖运行时环境的组件，如 Mail 告警等 + * + * @author tjq + * @since 2023/7/31 + */ +@Slf4j +@RestController +@RequestMapping("/test") +public class TestController { + + @Value("${server.port}") + private int port; + + @Resource + private AlarmCenter alarmCenter; + + @RequestMapping("/io") + public Map io(@RequestBody Map input) { + log.info("[TestController] input: {}", JsonUtils.toJSONString(input)); + return input; + } + + @GetMapping("/check") + public void check() { + Map testConfig = TestUtils.fetchTestConfig(); + if (CollectionUtils.isEmpty(testConfig)) { + log.info("[TestController] testConfig not exist, skip check!"); + return; + } + + log.info("[TestController] testConfig: {}", JsonUtils.toJSONString(testConfig)); + + testAlarmCenter(); + } + + void testAlarmCenter() { + JobInstanceAlarm jobInstanceAlarm = new JobInstanceAlarm().setAppId(277).setJobId(1).setInstanceId(2) + .setJobName("test-alarm").setJobParams("jobParams").setInstanceParams("instanceParams") + .setExecuteType(1).setFinishedTime(System.currentTimeMillis()); + + AlarmTarget target = new AlarmTarget().setName("ald").setPhone("208140").setExtra("extra") + .setPhone(MapUtils.getString(TestUtils.fetchTestConfig(), TestUtils.KEY_PHONE_NUMBER)) + .setEmail("tjq@zju.edu.cn") + .setWebHook(localUrlPath().concat("/test/io")); + + log.info("[TestController] start to testAlarmCenter, target: {}", target); + alarmCenter.alarmFailed(jobInstanceAlarm, Lists.newArrayList(target)); + } + + private String localUrlPath() { + return String.format("http://127.0.0.1:%d", port); + } +} diff --git a/powerjob-server/powerjob-server-starter/src/main/resources/application-daily.properties b/powerjob-server/powerjob-server-starter/src/main/resources/application-daily.properties index 693186e6..e61cd400 100644 --- a/powerjob-server/powerjob-server-starter/src/main/resources/application-daily.properties +++ b/powerjob-server/powerjob-server-starter/src/main/resources/application-daily.properties @@ -11,8 +11,7 @@ spring.datasource.core.minimum-idle=5 ####### MongoDB properties(Non-core configuration properties) ####### ####### delete mongodb config to disable mongodb ####### -#oms.mongodb.enable=true -#spring.data.mongodb.uri=mongodb+srv://zqq:No1Bug2Please3!@cluster0.wie54.gcp.mongodb.net/powerjob_daily?retryWrites=true&w=majority +#oms.storage.dfs.mongodb.uri=mongodb+srv://zqq:No1Bug2Please3!@cluster0.wie54.gcp.mongodb.net/powerjob_daily?retryWrites=true&w=majority ####### Email properties(Non-core configuration properties) ####### ####### Delete the following code to disable the mail ####### diff --git a/powerjob-server/powerjob-server-starter/src/main/resources/application-pre.properties b/powerjob-server/powerjob-server-starter/src/main/resources/application-pre.properties index f08f673b..a19491e8 100644 --- a/powerjob-server/powerjob-server-starter/src/main/resources/application-pre.properties +++ b/powerjob-server/powerjob-server-starter/src/main/resources/application-pre.properties @@ -11,8 +11,7 @@ spring.datasource.core.minimum-idle=5 ####### MongoDB properties(Non-core configuration properties) ####### ####### delete mongodb config to disable mongodb ####### -oms.mongodb.enable=true -spring.data.mongodb.uri=mongodb://remotehost:27017/powerjob-pre +oms.storage.dfs.mongodb.uri=mongodb://remotehost:27017/powerjob-pre ####### Email properties(Non-core configuration properties) ####### ####### Delete the following code to disable the mail ####### diff --git a/powerjob-server/powerjob-server-starter/src/main/resources/application-product.properties b/powerjob-server/powerjob-server-starter/src/main/resources/application-product.properties index a3ed4fe1..53e47d17 100644 --- a/powerjob-server/powerjob-server-starter/src/main/resources/application-product.properties +++ b/powerjob-server/powerjob-server-starter/src/main/resources/application-product.properties @@ -11,8 +11,7 @@ spring.datasource.core.minimum-idle=5 ####### MongoDB properties(Non-core configuration properties) ####### ####### delete mongodb config to disable mongodb ####### -oms.mongodb.enable=true -spring.data.mongodb.uri=mongodb://localhost:27017/powerjob-product +oms.storage.dfs.mongodb.uri=mongodb://localhost:27017/powerjob-product ####### Email properties(Non-core configuration properties) ####### ####### Delete the following code to disable the mail #######