From 5080796c6fadc8fe11128533fb4f38bf241d8e40 Mon Sep 17 00:00:00 2001 From: tjq Date: Tue, 6 Sep 2022 00:26:04 +0800 Subject: [PATCH 01/30] feat: add monitor module --- powerjob-server/pom.xml | 6 +++ .../powerjob-server-monitor/pom.xml | 28 +++++++++++++ .../tech/powerjob/server/monitor/Event.java | 22 +++++++++++ .../tech/powerjob/server/monitor/Monitor.java | 16 ++++++++ .../monitor/monitors/ServerMonitor.java | 39 +++++++++++++++++++ .../monitor/monitors/impl/LogMonitor.java | 22 +++++++++++ .../powerjob-server-starter/pom.xml | 4 ++ 7 files changed, 137 insertions(+) create mode 100644 powerjob-server/powerjob-server-monitor/pom.xml create mode 100644 powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/Event.java create mode 100644 powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/Monitor.java create mode 100644 powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/ServerMonitor.java create mode 100644 powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/impl/LogMonitor.java diff --git a/powerjob-server/pom.xml b/powerjob-server/pom.xml index 08e6b04d..268dba77 100644 --- a/powerjob-server/pom.xml +++ b/powerjob-server/pom.xml @@ -21,6 +21,7 @@ powerjob-server-extension powerjob-server-migrate powerjob-server-core + powerjob-server-monitor @@ -56,6 +57,11 @@ powerjob-server-common ${project.version} + + tech.powerjob + powerjob-server-monitor + ${project.version} + tech.powerjob powerjob-server-extension diff --git a/powerjob-server/powerjob-server-monitor/pom.xml b/powerjob-server/powerjob-server-monitor/pom.xml new file mode 100644 index 00000000..dce0a058 --- /dev/null +++ b/powerjob-server/powerjob-server-monitor/pom.xml @@ -0,0 +1,28 @@ + + + + powerjob-server + tech.powerjob + 4.1.0 + ../pom.xml + + 4.0.0 + + powerjob-server-monitor + ${project.parent.version} + + + 8 + 8 + + + + + tech.powerjob + powerjob-server-common + + + + \ No newline at end of file diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/Event.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/Event.java new file mode 100644 index 00000000..e62a304f --- /dev/null +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/Event.java @@ -0,0 +1,22 @@ +package tech.powerjob.server.monitor; + +/** + * 监控事件 + * + * @author tjq + * @since 2022/9/6 + */ +public interface Event { + + /** + * 监控事件的类型 + * @return 监控类型 + */ + String type(); + + /** + * 监控事件的内容 + * @return 监控事件的内容 + */ + String message(); +} diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/Monitor.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/Monitor.java new file mode 100644 index 00000000..7d3a4de2 --- /dev/null +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/Monitor.java @@ -0,0 +1,16 @@ +package tech.powerjob.server.monitor; + +/** + * 监视器 + * + * @author tjq + * @since 2022/9/6 + */ +public interface Monitor { + /** + * 记录监控事件 + * 请注意该方法务必异步不阻塞！！！ + * @param event 事件 + */ + void record(Event event); +} diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/ServerMonitor.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/ServerMonitor.java new file mode 100644 index 00000000..62f30035 --- /dev/null +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/ServerMonitor.java @@ -0,0 +1,39 @@ +package tech.powerjob.server.monitor.monitors; + +import com.google.common.collect.Lists; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import tech.powerjob.server.monitor.Event; +import tech.powerjob.server.monitor.Monitor; + +import java.util.List; + +/** + * 服务端监视器 + * + * @author tjq + * @since 2022/9/6 + */ +@Slf4j +@Component +public class ServerMonitor implements Monitor { + + private final List monitors = Lists.newLinkedList(); + + @Autowired + public ServerMonitor(List monitors) { + monitors.forEach(m -> { + if (m == this) { + return; + } + log.info("[ServerMonitor] register monitor: {}", m.getClass().getName()); + this.monitors.add(m); + }); + } + + @Override + public void record(Event event) { + monitors.forEach(m -> m.record(event)); + } +} diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/impl/LogMonitor.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/impl/LogMonitor.java new file mode 100644 index 00000000..79e433bb --- /dev/null +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/impl/LogMonitor.java @@ -0,0 +1,22 @@ +package tech.powerjob.server.monitor.monitors.impl; + +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import tech.powerjob.server.monitor.Event; +import tech.powerjob.server.monitor.Monitor; + +/** + * 系统默认实现——基于日志的监控监视器 + * 需要接入方自行基于类 ELK 系统采集 + * + * @author tjq + * @since 2022/9/6 + */ +@Component +public class LogMonitor implements Monitor { + + @Override + public void record(Event event) { + LoggerFactory.getLogger(event.type()).info(event.message()); + } +} diff --git a/powerjob-server/powerjob-server-starter/pom.xml b/powerjob-server/powerjob-server-starter/pom.xml index c1fb09f1..b6e95a20 100644 --- a/powerjob-server/powerjob-server-starter/pom.xml +++ b/powerjob-server/powerjob-server-starter/pom.xml @@ -31,6 +31,10 @@ tech.powerjob powerjob-server-common + + tech.powerjob + powerjob-server-monitor + tech.powerjob powerjob-server-persistence From a0cc5670d4b204270b464115e1d6b0fae577f774 Mon Sep 17 00:00:00 2001 From: tjq Date: Tue, 6 Sep 2022 01:54:10 +0800 Subject: [PATCH 02/30] feat: define DatabaseEvent --- .../java/tech/powerjob/server/common/SJ.java | 2 + .../monitor/events/db/DatabaseEvent.java | 48 ++++++++++++++++ .../monitor/events/db/DatabaseEventType.java | 13 +++++ .../monitor/events/db/DatabaseMonitor.java | 20 +++++++ .../events/db/DatabaseMonitorAspect.java | 56 +++++++++++++++++++ .../powerjob-server-persistence/pom.xml | 4 ++ .../logback-config/powerjob_monitor.xml | 34 +++++++++++ .../src/main/resources/logback-product.xml | 3 + 8 files changed, 180 insertions(+) create mode 100644 powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseEvent.java create mode 100644 powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseEventType.java create mode 100644 powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseMonitor.java create mode 100644 powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseMonitorAspect.java create mode 100644 powerjob-server/powerjob-server-starter/src/main/resources/logback-config/powerjob_monitor.xml diff --git a/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/SJ.java b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/SJ.java index 443300b2..89a89730 100644 --- a/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/SJ.java +++ b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/SJ.java @@ -14,4 +14,6 @@ public class SJ { public static final Splitter COMMA_SPLITTER = Splitter.on(","); public static final Joiner COMMA_JOINER = Joiner.on(","); + public static final Joiner MONITOR_JOINER = Joiner.on("|").useForNull("-"); + } diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseEvent.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseEvent.java new file mode 100644 index 00000000..f1046031 --- /dev/null +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseEvent.java @@ -0,0 +1,48 @@ +package tech.powerjob.server.monitor.events.db; + +import lombok.Setter; +import lombok.experimental.Accessors; +import tech.powerjob.server.common.SJ; +import tech.powerjob.server.monitor.Event; + +/** + * 数据库操作事件 + * + * @author tjq + * @since 2022/9/6 + */ +@Setter +@Accessors(chain = true) +public class DatabaseEvent implements Event { + + private DatabaseEventType type; + + private String serviceName; + + private String methodName; + + private Status status; + + private Integer rows; + + private long cost; + + private String errorMsg; + + private String extra; + + public enum Status { + SUCCESS, + FAILED + } + + @Override + public String type() { + return "MONITOR_LOGGER_DB_OPERATION"; + } + + @Override + public String message() { + return SJ.MONITOR_JOINER.join(type, serviceName, methodName, status, rows, cost, errorMsg, extra); + } +} diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseEventType.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseEventType.java new file mode 100644 index 00000000..5f0054fe --- /dev/null +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseEventType.java @@ -0,0 +1,13 @@ +package tech.powerjob.server.monitor.events.db; + +/** + * DatabaseEventType + * + * @author tjq + * @since 2022/9/6 + */ +public enum DatabaseEventType { + H2, + CORE, + MONGO +} diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseMonitor.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseMonitor.java new file mode 100644 index 00000000..e0d83037 --- /dev/null +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseMonitor.java @@ -0,0 +1,20 @@ +package tech.powerjob.server.monitor.events.db; + +import java.lang.annotation.*; + +/** + * 数据库监控注解 + * + * @author tjq + * @since 2022/9/6 + */ +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Documented +public @interface DatabaseMonitor { + /** + * 类型，比如 H2 / CORE / MONGO + * @return 类型 + */ + DatabaseEventType type(); +} diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseMonitorAspect.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseMonitorAspect.java new file mode 100644 index 00000000..c229d614 --- /dev/null +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseMonitorAspect.java @@ -0,0 +1,56 @@ +package tech.powerjob.server.monitor.events.db; + +import lombok.extern.slf4j.Slf4j; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.springframework.stereotype.Component; +import tech.powerjob.server.monitor.monitors.ServerMonitor; + +import javax.annotation.Resource; + +/** + * 监控切面 + * + * @author tjq + * @since 2022/9/6 + */ +@Slf4j +@Aspect +@Component +public class DatabaseMonitorAspect { + + @Resource + private ServerMonitor serverMonitor; + + @Around(value = "@annotation(databaseMonitor))") + public Object execute(ProceedingJoinPoint point, DatabaseMonitor databaseMonitor) throws Throwable { + + final String className = point.getTarget().getClass().getSimpleName(); + final String methodName = point.getSignature().getName(); + + DatabaseEvent event = new DatabaseEvent().setType(databaseMonitor.type()) + .setServiceName(className) + .setMethodName(methodName) + .setStatus(DatabaseEvent.Status.SUCCESS); + + long startTs = System.currentTimeMillis(); + try { + final Object ret = point.proceed(); + event.setRows(parseEffectRows(ret)); + return ret; + } catch (Throwable t) { + + long cost = System.currentTimeMillis() - startTs; + event.setCost(cost).setErrorMsg(t.getMessage()).setStatus(DatabaseEvent.Status.FAILED); + serverMonitor.record(event); + + throw t; + } + } + + private static Integer parseEffectRows(Object ret) { + // TODO: 计算影响行数，可能需要小改下 DAO 层， + return null; + } +} diff --git a/powerjob-server/powerjob-server-persistence/pom.xml b/powerjob-server/powerjob-server-persistence/pom.xml index 444fac74..b26d81d8 100644 --- a/powerjob-server/powerjob-server-persistence/pom.xml +++ b/powerjob-server/powerjob-server-persistence/pom.xml @@ -23,6 +23,10 @@ tech.powerjob powerjob-server-common + + tech.powerjob + powerjob-server-monitor + \ No newline at end of file diff --git a/powerjob-server/powerjob-server-starter/src/main/resources/logback-config/powerjob_monitor.xml b/powerjob-server/powerjob-server-starter/src/main/resources/logback-config/powerjob_monitor.xml new file mode 100644 index 00000000..89c1bcb4 --- /dev/null +++ b/powerjob-server/powerjob-server-starter/src/main/resources/logback-config/powerjob_monitor.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + ${MONITOR_LOG_PATH}/database.log + + ${MONITOR_LOG_PATTERN} + UTF-8 + + + ${MONITOR_LOG_PATH}/database.log.${ROTATE_PATTERN} + 3 + 500MB + 1500MB + + + + 512 + 0 + true + + + + + + + + \ No newline at end of file diff --git a/powerjob-server/powerjob-server-starter/src/main/resources/logback-product.xml b/powerjob-server/powerjob-server-starter/src/main/resources/logback-product.xml index b1d2b1f4..80be04f6 100644 --- a/powerjob-server/powerjob-server-starter/src/main/resources/logback-product.xml +++ b/powerjob-server/powerjob-server-starter/src/main/resources/logback-product.xml @@ -14,6 +14,9 @@ --> + + + ${LOG_PATH}/powerjob-server-error.log From e23825c399dbf0d2070371b1dcc21457ac769fff Mon Sep 17 00:00:00 2001 From: tjq Date: Thu, 8 Sep 2022 23:59:17 +0800 Subject: [PATCH 03/30] feat: use package aop --- .../monitor/events/db/DatabaseEvent.java | 2 +- .../monitor/events/db/DatabaseEventType.java | 13 ----------- .../monitor/events/db/DatabaseMonitor.java | 20 ---------------- .../events/db/DatabaseMonitorAspect.java | 23 ++++++++++++------- .../monitor/events/db/DatabaseType.java | 22 ++++++++++++++++++ 5 files changed, 38 insertions(+), 42 deletions(-) delete mode 100644 powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseEventType.java delete mode 100644 powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseMonitor.java create mode 100644 powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseType.java diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseEvent.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseEvent.java index f1046031..1980cf54 100644 --- a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseEvent.java +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseEvent.java @@ -15,7 +15,7 @@ import tech.powerjob.server.monitor.Event; @Accessors(chain = true) public class DatabaseEvent implements Event { - private DatabaseEventType type; + private DatabaseType type; private String serviceName; diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseEventType.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseEventType.java deleted file mode 100644 index 5f0054fe..00000000 --- a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseEventType.java +++ /dev/null @@ -1,13 +0,0 @@ -package tech.powerjob.server.monitor.events.db; - -/** - * DatabaseEventType - * - * @author tjq - * @since 2022/9/6 - */ -public enum DatabaseEventType { - H2, - CORE, - MONGO -} diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseMonitor.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseMonitor.java deleted file mode 100644 index e0d83037..00000000 --- a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseMonitor.java +++ /dev/null @@ -1,20 +0,0 @@ -package tech.powerjob.server.monitor.events.db; - -import java.lang.annotation.*; - -/** - * 数据库监控注解 - * - * @author tjq - * @since 2022/9/6 - */ -@Target(ElementType.TYPE) -@Retention(RetentionPolicy.RUNTIME) -@Documented -public @interface DatabaseMonitor { - /** - * 类型，比如 H2 / CORE / MONGO - * @return 类型 - */ - DatabaseEventType type(); -} diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseMonitorAspect.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseMonitorAspect.java index c229d614..a7d5e490 100644 --- a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseMonitorAspect.java +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseMonitorAspect.java @@ -23,13 +23,21 @@ public class DatabaseMonitorAspect { @Resource private ServerMonitor serverMonitor; - @Around(value = "@annotation(databaseMonitor))") - public Object execute(ProceedingJoinPoint point, DatabaseMonitor databaseMonitor) throws Throwable { + @Around("execution(* tech.powerjob.server.persistence.remote.repository..*.*(..))") + public Object monitorCoreDB(ProceedingJoinPoint joinPoint) throws Throwable { + return wrapperMonitor(joinPoint, DatabaseType.CORE); + } + @Around("execution(* tech.powerjob.server.persistence.local..*.*(..))") + public Object monitorLocalDB(ProceedingJoinPoint joinPoint) throws Throwable { + return wrapperMonitor(joinPoint, DatabaseType.LOCAL); + } + + private Object wrapperMonitor(ProceedingJoinPoint point, DatabaseType type) throws Throwable { final String className = point.getTarget().getClass().getSimpleName(); final String methodName = point.getSignature().getName(); - DatabaseEvent event = new DatabaseEvent().setType(databaseMonitor.type()) + DatabaseEvent event = new DatabaseEvent().setType(type) .setServiceName(className) .setMethodName(methodName) .setStatus(DatabaseEvent.Status.SUCCESS); @@ -40,12 +48,11 @@ public class DatabaseMonitorAspect { event.setRows(parseEffectRows(ret)); return ret; } catch (Throwable t) { - - long cost = System.currentTimeMillis() - startTs; - event.setCost(cost).setErrorMsg(t.getMessage()).setStatus(DatabaseEvent.Status.FAILED); - serverMonitor.record(event); - + event.setErrorMsg(t.getMessage()).setStatus(DatabaseEvent.Status.FAILED); throw t; + } finally { + long cost = System.currentTimeMillis() - startTs; + serverMonitor.record(event.setCost(cost)); } } diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseType.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseType.java new file mode 100644 index 00000000..673e16b0 --- /dev/null +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseType.java @@ -0,0 +1,22 @@ +package tech.powerjob.server.monitor.events.db; + +/** + * DatabaseEventType + * + * @author tjq + * @since 2022/9/6 + */ +public enum DatabaseType { + /** + * 本地存储库，H2 + */ + LOCAL, + /** + * 远程核心库 + */ + CORE, + /** + * 扩展库 + */ + EXTRA +} From cca9c5421e851b735f59c2e8a177c10e456c39a9 Mon Sep 17 00:00:00 2001 From: tjq Date: Fri, 9 Sep 2022 00:09:22 +0800 Subject: [PATCH 04/30] feat: finished jpa base monitor --- .../java/tech/powerjob/server/common/utils/AOPUtils.java | 5 +++++ .../server/monitor/events/db/DatabaseMonitorAspect.java | 6 ++++-- .../main/java/tech/powerjob/server/web/WebLogAspect.java | 5 +++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/utils/AOPUtils.java b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/utils/AOPUtils.java index c1edeb09..3d06dd64 100644 --- a/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/utils/AOPUtils.java +++ b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/utils/AOPUtils.java @@ -2,6 +2,7 @@ package tech.powerjob.server.common.utils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.exception.ExceptionUtils; +import org.aspectj.lang.JoinPoint; import org.aspectj.lang.ProceedingJoinPoint; import org.aspectj.lang.Signature; import org.aspectj.lang.reflect.MethodSignature; @@ -27,6 +28,10 @@ public class AOPUtils { private static final ExpressionParser parser = new SpelExpressionParser(); private static final ParameterNameDiscoverer discoverer = new LocalVariableTableParameterNameDiscoverer(); + public static String parseRealClassName(JoinPoint joinPoint) { + return joinPoint.getSignature().getDeclaringType().getSimpleName(); + } + public static Method parseMethod(ProceedingJoinPoint joinPoint) { Signature pointSignature = joinPoint.getSignature(); if (!(pointSignature instanceof MethodSignature)) { diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseMonitorAspect.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseMonitorAspect.java index a7d5e490..b4eaefc7 100644 --- a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseMonitorAspect.java +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseMonitorAspect.java @@ -4,7 +4,9 @@ import lombok.extern.slf4j.Slf4j; import org.aspectj.lang.ProceedingJoinPoint; import org.aspectj.lang.annotation.Around; import org.aspectj.lang.annotation.Aspect; +import org.springframework.aop.support.AopUtils; import org.springframework.stereotype.Component; +import tech.powerjob.server.common.utils.AOPUtils; import tech.powerjob.server.monitor.monitors.ServerMonitor; import javax.annotation.Resource; @@ -34,11 +36,11 @@ public class DatabaseMonitorAspect { } private Object wrapperMonitor(ProceedingJoinPoint point, DatabaseType type) throws Throwable { - final String className = point.getTarget().getClass().getSimpleName(); + String classNameMini = AOPUtils.parseRealClassName(point); final String methodName = point.getSignature().getName(); DatabaseEvent event = new DatabaseEvent().setType(type) - .setServiceName(className) + .setServiceName(classNameMini) .setMethodName(methodName) .setStatus(DatabaseEvent.Status.SUCCESS); diff --git a/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/web/WebLogAspect.java b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/web/WebLogAspect.java index 10aa456f..33b5812e 100644 --- a/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/web/WebLogAspect.java +++ b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/web/WebLogAspect.java @@ -13,6 +13,7 @@ import org.springframework.stereotype.Component; import org.springframework.web.context.request.RequestContextHolder; import org.springframework.web.context.request.ServletRequestAttributes; import org.springframework.web.multipart.MultipartFile; +import tech.powerjob.server.common.utils.AOPUtils; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -60,8 +61,8 @@ public class WebLogAspect { } HttpServletRequest request = requestAttributes.getRequest(); - String[] classNameSplit = joinPoint.getSignature().getDeclaringTypeName().split("\\."); - String classNameMini = classNameSplit[classNameSplit.length - 1]; + + String classNameMini = AOPUtils.parseRealClassName(joinPoint); String classMethod = classNameMini + "." + joinPoint.getSignature().getName(); // 排除特殊类 From 2a76e7d043c99914ca77fd22d3815df1fdba259b Mon Sep 17 00:00:00 2001 From: tjq Date: Fri, 9 Sep 2022 00:22:27 +0800 Subject: [PATCH 05/30] feat: monitor db effect rows --- .../monitor}/DatabaseMonitorAspect.java | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) rename powerjob-server/{powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db => powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/monitor}/DatabaseMonitorAspect.java (75%) diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseMonitorAspect.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/monitor/DatabaseMonitorAspect.java similarity index 75% rename from powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseMonitorAspect.java rename to powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/monitor/DatabaseMonitorAspect.java index b4eaefc7..9bc51e4a 100644 --- a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/db/DatabaseMonitorAspect.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/monitor/DatabaseMonitorAspect.java @@ -1,15 +1,19 @@ -package tech.powerjob.server.monitor.events.db; +package tech.powerjob.server.persistence.monitor; import lombok.extern.slf4j.Slf4j; import org.aspectj.lang.ProceedingJoinPoint; import org.aspectj.lang.annotation.Around; import org.aspectj.lang.annotation.Aspect; -import org.springframework.aop.support.AopUtils; +import org.springframework.data.domain.Slice; import org.springframework.stereotype.Component; import tech.powerjob.server.common.utils.AOPUtils; +import tech.powerjob.server.monitor.events.db.DatabaseEvent; +import tech.powerjob.server.monitor.events.db.DatabaseType; import tech.powerjob.server.monitor.monitors.ServerMonitor; import javax.annotation.Resource; +import java.util.Collection; +import java.util.Optional; /** * 监控切面 @@ -59,6 +63,20 @@ public class DatabaseMonitorAspect { } private static Integer parseEffectRows(Object ret) { + + if (ret instanceof Collection) { + return ((Collection) ret).size(); + } + if (ret instanceof Number) { + return ((Number) ret).intValue(); + } + if (ret instanceof Optional) { + return ((Optional) ret).isPresent() ? 1 : 0; + } + if (ret instanceof Slice) { + return ((Slice) ret).getSize(); + } + // TODO: 计算影响行数，可能需要小改下 DAO 层， return null; } From 29a50ed89a113ca8fd9d282c134988cdbd5a4f15 Mon Sep 17 00:00:00 2001 From: tjq Date: Fri, 9 Sep 2022 00:27:13 +0800 Subject: [PATCH 06/30] feat: optimize parseEffectRows --- .../persistence/monitor/DatabaseMonitorAspect.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/monitor/DatabaseMonitorAspect.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/monitor/DatabaseMonitorAspect.java index 9bc51e4a..392bae15 100644 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/monitor/DatabaseMonitorAspect.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/monitor/DatabaseMonitorAspect.java @@ -64,20 +64,21 @@ public class DatabaseMonitorAspect { private static Integer parseEffectRows(Object ret) { - if (ret instanceof Collection) { - return ((Collection) ret).size(); - } if (ret instanceof Number) { return ((Number) ret).intValue(); } if (ret instanceof Optional) { return ((Optional) ret).isPresent() ? 1 : 0; } + if (ret instanceof Collection) { + return ((Collection) ret).size(); + } if (ret instanceof Slice) { return ((Slice) ret).getSize(); } - // TODO: 计算影响行数，可能需要小改下 DAO 层， - return null; + // TODO: 直接返回对象的方法全部改成 Optional + + return ret == null ? 0 : 1; } } From ce369b3e30cd3dc2eb95716f3cc2420cfecf59f8 Mon Sep 17 00:00:00 2001 From: tjq Date: Fri, 9 Sep 2022 00:31:54 +0800 Subject: [PATCH 07/30] feat: optimize parseEffectRows --- .../server/persistence/monitor/DatabaseMonitorAspect.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/monitor/DatabaseMonitorAspect.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/monitor/DatabaseMonitorAspect.java index 392bae15..71aef266 100644 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/monitor/DatabaseMonitorAspect.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/monitor/DatabaseMonitorAspect.java @@ -14,6 +14,7 @@ import tech.powerjob.server.monitor.monitors.ServerMonitor; import javax.annotation.Resource; import java.util.Collection; import java.util.Optional; +import java.util.stream.Stream; /** * 监控切面 @@ -64,6 +65,8 @@ public class DatabaseMonitorAspect { private static Integer parseEffectRows(Object ret) { + // 从性能角度考虑，最高频场景放在最前面判断 + if (ret instanceof Number) { return ((Number) ret).intValue(); } @@ -77,6 +80,9 @@ public class DatabaseMonitorAspect { return ((Slice) ret).getSize(); } + if (ret instanceof Stream) { + return null; + } // TODO: 直接返回对象的方法全部改成 Optional return ret == null ? 0 : 1; From 519213ad4a560570f4b895d1be9f394d4f03dd3e Mon Sep 17 00:00:00 2001 From: tjq Date: Fri, 9 Sep 2022 00:41:50 +0800 Subject: [PATCH 08/30] feat: add SlowLockEvent --- .../server/core/lock/UseCacheLockAspect.java | 18 +++++++++- .../monitor/events/lock/SlowLockEvent.java | 33 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/lock/SlowLockEvent.java diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/lock/UseCacheLockAspect.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/lock/UseCacheLockAspect.java index 631036e2..f51809fb 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/lock/UseCacheLockAspect.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/lock/UseCacheLockAspect.java @@ -11,7 +11,10 @@ import org.aspectj.lang.annotation.Aspect; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; import tech.powerjob.server.common.utils.AOPUtils; +import tech.powerjob.server.monitor.events.lock.SlowLockEvent; +import tech.powerjob.server.monitor.monitors.ServerMonitor; +import javax.annotation.Resource; import java.lang.reflect.Method; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -29,6 +32,9 @@ import java.util.concurrent.locks.ReentrantLock; @Order(1) public class UseCacheLockAspect { + @Resource + private ServerMonitor serverMonitor; + private final Map> lockContainer = Maps.newConcurrentMap(); private static final long SLOW_THRESHOLD = 100; @@ -53,8 +59,18 @@ public class UseCacheLockAspect { try { long timeCost = System.currentTimeMillis() - start; if (timeCost > SLOW_THRESHOLD) { + + final SlowLockEvent slowLockEvent = new SlowLockEvent() + .setLocKType(useCacheLock.type()) + .setLockKey(String.valueOf(key)) + .setCallerService(method.getDeclaringClass().getSimpleName()) + .setCallerMethod(method.getName()) + .setCost(timeCost); + + serverMonitor.record(slowLockEvent); + log.warn("[UseSegmentLockAspect] wait lock for method({}#{}) cost {} ms! key = '{}', args = {}, ", method.getDeclaringClass().getSimpleName(), method.getName(), timeCost, - useCacheLock.key(), + key, JSON.toJSONString(point.getArgs())); } return point.proceed(); diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/lock/SlowLockEvent.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/lock/SlowLockEvent.java new file mode 100644 index 00000000..16527214 --- /dev/null +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/lock/SlowLockEvent.java @@ -0,0 +1,33 @@ +package tech.powerjob.server.monitor.events.lock; + +import lombok.Setter; +import lombok.experimental.Accessors; +import tech.powerjob.server.common.SJ; +import tech.powerjob.server.monitor.Event; + +/** + * 长时间等待锁事件 + * + * @author tjq + * @since 2022/9/9 + */ +@Setter +@Accessors(chain = true) +public class SlowLockEvent implements Event { + + private String locKType; + private String lockKey; + private String callerService; + private String callerMethod; + private long cost; + + @Override + public String type() { + return "MONITOR_LOGGER_SLOW_LOCK"; + } + + @Override + public String message() { + return SJ.MONITOR_JOINER.join(locKType, lockKey, callerService, callerMethod, cost); + } +} From b6de5aa56373920c178d6295d24f70bd85de838f Mon Sep 17 00:00:00 2001 From: tjq Date: Fri, 9 Sep 2022 22:45:05 +0800 Subject: [PATCH 09/30] feat: add WorkerHeartbeatEvent --- .../core/handler/WorkerRequestHandler.java | 9 +++++ .../events/handler/WorkerHeartbeatEvent.java | 38 +++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/handler/WorkerHeartbeatEvent.java diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/WorkerRequestHandler.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/WorkerRequestHandler.java index ed5787a6..df8ab472 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/WorkerRequestHandler.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/WorkerRequestHandler.java @@ -19,6 +19,8 @@ import tech.powerjob.server.core.handler.impl.WorkerRequestHttpHandler; import tech.powerjob.server.core.instance.InstanceLogService; import tech.powerjob.server.core.instance.InstanceManager; import tech.powerjob.server.core.workflow.WorkflowInstanceManager; +import tech.powerjob.server.monitor.events.handler.WorkerHeartbeatEvent; +import tech.powerjob.server.monitor.monitors.ServerMonitor; import tech.powerjob.server.persistence.remote.model.ContainerInfoDO; import tech.powerjob.server.persistence.remote.model.JobInfoDO; import tech.powerjob.server.persistence.remote.repository.ContainerInfoRepository; @@ -45,6 +47,8 @@ import java.util.stream.Collectors; @Component public class WorkerRequestHandler { + @Resource + private ServerMonitor serverMonitor; @Resource private Environment environment; @Resource @@ -74,6 +78,11 @@ public class WorkerRequestHandler { * @param heartbeat 心跳包 */ public void onReceiveWorkerHeartbeat(WorkerHeartbeat heartbeat) { + + WorkerHeartbeatEvent event = new WorkerHeartbeatEvent(); + BeanUtils.copyProperties(heartbeat, event); + serverMonitor.record(event.setScore(heartbeat.getSystemMetrics().getScore())); + WorkerClusterManagerService.updateStatus(heartbeat); } diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/handler/WorkerHeartbeatEvent.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/handler/WorkerHeartbeatEvent.java new file mode 100644 index 00000000..542c23d7 --- /dev/null +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/handler/WorkerHeartbeatEvent.java @@ -0,0 +1,38 @@ +package tech.powerjob.server.monitor.events.handler; + +import lombok.Setter; +import lombok.experimental.Accessors; +import tech.powerjob.server.common.SJ; +import tech.powerjob.server.monitor.Event; + +/** + * worker 心跳事件监控 + * + * @author tjq + * @since 2022/9/9 + */ +@Setter +@Accessors(chain = true) +public class WorkerHeartbeatEvent implements Event { + + private String appName; + + private String version; + + private String protocol; + + private String tag; + private String workerAddress; + + private Integer score; + + @Override + public String type() { + return "MONITOR_LOGGER_WORKER_HEART_BEAT"; + } + + @Override + public String message() { + return SJ.MONITOR_JOINER.join(appName, version, protocol, tag, workerAddress, score); + } +} From eaf6dcad4fee6e0988142c50fc496fb984439614 Mon Sep 17 00:00:00 2001 From: tjq Date: Sat, 10 Sep 2022 08:30:18 +0800 Subject: [PATCH 10/30] feat: add appId in WorkerHeartbeatEvent --- .../handler/TtReportInstanceStatusEvent.java | 26 +++++++++++++++++++ .../events/handler/WorkerHeartbeatEvent.java | 7 +++-- 2 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/handler/TtReportInstanceStatusEvent.java diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/handler/TtReportInstanceStatusEvent.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/handler/TtReportInstanceStatusEvent.java new file mode 100644 index 00000000..567762b4 --- /dev/null +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/handler/TtReportInstanceStatusEvent.java @@ -0,0 +1,26 @@ +package tech.powerjob.server.monitor.events.handler; + +import lombok.Setter; +import lombok.experimental.Accessors; +import tech.powerjob.server.monitor.Event; + +/** + * TaskTrackerReportInstanceStatus + * + * @author tjq + * @since 2022/9/9 + */ +@Setter +@Accessors(chain = true) +public class TtReportInstanceStatusEvent implements Event { + + @Override + public String type() { + return "MONITOR_LOGGER_TT_REPORT_STATUS"; + } + + @Override + public String message() { + return null; + } +} diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/handler/WorkerHeartbeatEvent.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/handler/WorkerHeartbeatEvent.java index 542c23d7..9dd3cff0 100644 --- a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/handler/WorkerHeartbeatEvent.java +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/handler/WorkerHeartbeatEvent.java @@ -16,7 +16,10 @@ import tech.powerjob.server.monitor.Event; public class WorkerHeartbeatEvent implements Event { private String appName; - + /** + * 虽然和 AppName 冗余，但考虑到其他日志使用 appId 监控，此处可方便潜在的其他处理 + */ + private Long appId; private String version; private String protocol; @@ -33,6 +36,6 @@ public class WorkerHeartbeatEvent implements Event { @Override public String message() { - return SJ.MONITOR_JOINER.join(appName, version, protocol, tag, workerAddress, score); + return SJ.MONITOR_JOINER.join(appName, appId, version, protocol, tag, workerAddress, score); } } From 22522c099dae54543d01bd2df64a2f08beabb6d8 Mon Sep 17 00:00:00 2001 From: tjq Date: Sat, 10 Sep 2022 08:32:32 +0800 Subject: [PATCH 11/30] feat: status report append appId info --- .../common/request/TaskTrackerReportInstanceStatusReq.java | 6 ++++++ .../worker/core/tracker/task/CommonTaskTracker.java | 1 + .../worker/core/tracker/task/FrequentTaskTracker.java | 4 ++-- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/powerjob-common/src/main/java/tech/powerjob/common/request/TaskTrackerReportInstanceStatusReq.java b/powerjob-common/src/main/java/tech/powerjob/common/request/TaskTrackerReportInstanceStatusReq.java index 55cf865d..8347d6ac 100644 --- a/powerjob-common/src/main/java/tech/powerjob/common/request/TaskTrackerReportInstanceStatusReq.java +++ b/powerjob-common/src/main/java/tech/powerjob/common/request/TaskTrackerReportInstanceStatusReq.java @@ -15,6 +15,12 @@ import java.util.Map; @Data public class TaskTrackerReportInstanceStatusReq implements PowerSerializable { + /** + * 追加上报自己的 appId + * 方便后续的监控日志埋点 + */ + private Long appId; + private Long jobId; private Long instanceId; diff --git a/powerjob-worker/src/main/java/tech/powerjob/worker/core/tracker/task/CommonTaskTracker.java b/powerjob-worker/src/main/java/tech/powerjob/worker/core/tracker/task/CommonTaskTracker.java index 2a188a54..1cf442b5 100644 --- a/powerjob-worker/src/main/java/tech/powerjob/worker/core/tracker/task/CommonTaskTracker.java +++ b/powerjob-worker/src/main/java/tech/powerjob/worker/core/tracker/task/CommonTaskTracker.java @@ -158,6 +158,7 @@ public class CommonTaskTracker extends TaskTracker { log.debug("[TaskTracker-{}] status check result: {}", instanceId, holder); TaskTrackerReportInstanceStatusReq req = new TaskTrackerReportInstanceStatusReq(); + req.setAppId(workerRuntime.getAppId()); req.setJobId(instanceInfo.getJobId()); req.setInstanceId(instanceId); req.setWfInstanceId(instanceInfo.getWfInstanceId()); diff --git a/powerjob-worker/src/main/java/tech/powerjob/worker/core/tracker/task/FrequentTaskTracker.java b/powerjob-worker/src/main/java/tech/powerjob/worker/core/tracker/task/FrequentTaskTracker.java index a4e79f83..071477d5 100644 --- a/powerjob-worker/src/main/java/tech/powerjob/worker/core/tracker/task/FrequentTaskTracker.java +++ b/powerjob-worker/src/main/java/tech/powerjob/worker/core/tracker/task/FrequentTaskTracker.java @@ -7,14 +7,13 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.util.concurrent.ThreadFactoryBuilder; import lombok.Data; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.util.StringUtils; -import tech.powerjob.common.exception.PowerJobException; import tech.powerjob.common.enums.ExecuteType; import tech.powerjob.common.enums.InstanceStatus; import tech.powerjob.common.enums.TimeExpressionType; +import tech.powerjob.common.exception.PowerJobException; import tech.powerjob.common.model.AlarmConfig; import tech.powerjob.common.model.InstanceDetail; import tech.powerjob.common.request.ServerScheduleJobReq; @@ -339,6 +338,7 @@ public class FrequentTaskTracker extends TaskTracker { } TaskTrackerReportInstanceStatusReq req = new TaskTrackerReportInstanceStatusReq(); + req.setAppId(workerRuntime.getAppId()); req.setJobId(instanceInfo.getJobId()); req.setInstanceId(instanceId); req.setReportTime(System.currentTimeMillis()); From 22db37cad98d01b1da2f59dd12f28d835aebe065 Mon Sep 17 00:00:00 2001 From: tjq Date: Sat, 10 Sep 2022 08:40:29 +0800 Subject: [PATCH 12/30] feat: optimize instanceMetadataCacheSize --- .../powerjob/server/core/instance/InstanceMetadataService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceMetadataService.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceMetadataService.java index 4304a74b..79a320e1 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceMetadataService.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceMetadataService.java @@ -33,13 +33,14 @@ public class InstanceMetadataService implements InitializingBean { @Value("${oms.instance.metadata.cache.size}") private int instanceMetadataCacheSize; - private static final int CACHE_CONCURRENCY_LEVEL = 4; + private static final int CACHE_CONCURRENCY_LEVEL = 16; @Override public void afterPropertiesSet() throws Exception { instanceId2JobInfoCache = CacheBuilder.newBuilder() .concurrencyLevel(CACHE_CONCURRENCY_LEVEL) .maximumSize(instanceMetadataCacheSize) + .softValues() .build(); } From dfd1fd069b8d99ff343caeeac544a05d2b3891ec Mon Sep 17 00:00:00 2001 From: tjq Date: Sat, 10 Sep 2022 09:27:57 +0800 Subject: [PATCH 13/30] fix: invalid random when JobInfo's maxWorkerCount is a small value #449 --- .../powerjob/server/core/DispatchService.java | 16 +--------------- .../remote/worker/WorkerClusterQueryService.java | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/DispatchService.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/DispatchService.java index 44d36d60..881c5061 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/DispatchService.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/DispatchService.java @@ -146,9 +146,8 @@ public class DispatchService { // 构造任务调度请求 ServerScheduleJobReq req = constructServerScheduleJobReq(jobInfo, instanceInfo, workerIpList); - // 发送请求（不可靠，需要一个后台线程定期轮询状态） - WorkerInfo taskTracker = selectTaskTracker(jobInfo, suitableWorkers); + WorkerInfo taskTracker = suitableWorkers.get(0); String taskTrackerAddress = taskTracker.getAddress(); transportService.tell(Protocol.of(taskTracker.getProtocol()), taskTrackerAddress, req); @@ -196,17 +195,4 @@ public class DispatchService { req.setThreadConcurrency(jobInfo.getConcurrency()); return req; } - - private WorkerInfo selectTaskTracker(JobInfoDO jobInfo, List workerInfos) { - DispatchStrategy dispatchStrategy = DispatchStrategy.of(jobInfo.getDispatchStrategy()); - switch (dispatchStrategy) { - case HEALTH_FIRST: - return workerInfos.get(0); - case RANDOM: - return workerInfos.get(ThreadLocalRandom.current().nextInt(workerInfos.size())); - default: - } - // impossible, indian java - return workerInfos.get(0); - } } diff --git a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/WorkerClusterQueryService.java b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/WorkerClusterQueryService.java index da555c0b..04002cb7 100644 --- a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/WorkerClusterQueryService.java +++ b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/WorkerClusterQueryService.java @@ -4,6 +4,7 @@ import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import tech.powerjob.common.enums.DispatchStrategy; import tech.powerjob.common.model.DeployedContainerInfo; import tech.powerjob.server.common.module.WorkerInfo; import tech.powerjob.server.extension.WorkerFilter; @@ -44,8 +45,17 @@ public class WorkerClusterQueryService { workers.removeIf(workerInfo -> filterWorker(workerInfo, jobInfo)); - // 按健康度排序 - workers.sort((o1, o2) -> o2.getSystemMetrics().calculateScore() - o1.getSystemMetrics().calculateScore()); + DispatchStrategy dispatchStrategy = DispatchStrategy.of(jobInfo.getDispatchStrategy()); + switch (dispatchStrategy) { + case RANDOM: + Collections.shuffle(workers); + break; + case HEALTH_FIRST: + workers.sort((o1, o2) -> o2.getSystemMetrics().calculateScore() - o1.getSystemMetrics().calculateScore()); + break; + default: + // do nothing + } // 限定集群大小（0代表不限制） if (!workers.isEmpty() && jobInfo.getMaxWorkerCount() > 0 && workers.size() > jobInfo.getMaxWorkerCount()) { From ac1b1fe0c8cf1ebffed5aeb72ab282901550d331 Mon Sep 17 00:00:00 2001 From: tjq Date: Sat, 10 Sep 2022 09:55:26 +0800 Subject: [PATCH 14/30] fix: limit worker num failed in map/mapreduce job #450 --- .../common/request/ServerScheduleJobReq.java | 5 ++ .../powerjob/server/core/DispatchService.java | 1 + .../core/ha/ProcessorTrackerStatusHolder.java | 54 +++++++++++++++++-- .../worker/core/tracker/task/TaskTracker.java | 15 +++--- 4 files changed, 65 insertions(+), 10 deletions(-) diff --git a/powerjob-common/src/main/java/tech/powerjob/common/request/ServerScheduleJobReq.java b/powerjob-common/src/main/java/tech/powerjob/common/request/ServerScheduleJobReq.java index 93093236..92fef38e 100644 --- a/powerjob-common/src/main/java/tech/powerjob/common/request/ServerScheduleJobReq.java +++ b/powerjob-common/src/main/java/tech/powerjob/common/request/ServerScheduleJobReq.java @@ -20,6 +20,11 @@ public class ServerScheduleJobReq implements PowerSerializable { */ private List allWorkerAddress; + /** + * 最大机器数量 + */ + private Integer maxWorkerCount; + /* *********************** 任务相关属性 *********************** */ /** diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/DispatchService.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/DispatchService.java index 881c5061..db0183f7 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/DispatchService.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/DispatchService.java @@ -181,6 +181,7 @@ public class DispatchService { } req.setInstanceId(instanceInfo.getInstanceId()); req.setAllWorkerAddress(finalWorkersIpList); + req.setMaxWorkerCount(jobInfo.getMaxWorkerCount()); // 设置工作流ID req.setWfInstanceId(instanceInfo.getWfInstanceId()); diff --git a/powerjob-worker/src/main/java/tech/powerjob/worker/core/ha/ProcessorTrackerStatusHolder.java b/powerjob-worker/src/main/java/tech/powerjob/worker/core/ha/ProcessorTrackerStatusHolder.java index 1a67c378..b5721b91 100644 --- a/powerjob-worker/src/main/java/tech/powerjob/worker/core/ha/ProcessorTrackerStatusHolder.java +++ b/powerjob-worker/src/main/java/tech/powerjob/worker/core/ha/ProcessorTrackerStatusHolder.java @@ -17,10 +17,15 @@ import java.util.Map; @Slf4j public class ProcessorTrackerStatusHolder { + private final Long instanceId; + private final Integer maxWorkerCount; // ProcessorTracker的address(IP:Port) -> 状态 private final Map address2Status; - public ProcessorTrackerStatusHolder(List allWorkerAddress) { + public ProcessorTrackerStatusHolder(Long instanceId, Integer maxWorkerCount, List allWorkerAddress) { + + this.instanceId = instanceId; + this.maxWorkerCount = maxWorkerCount; address2Status = Maps.newConcurrentMap(); allWorkerAddress.forEach(address -> { @@ -38,7 +43,7 @@ public class ProcessorTrackerStatusHolder { public ProcessorTrackerStatus getProcessorTrackerStatus(String address) { // remove 前突然收到了 PT 心跳同时立即被派发才可能出现这种情况，0.001% 概率 return address2Status.computeIfAbsent(address, ignore -> { - log.warn("[ProcessorTrackerStatusHolder] unregistered worker: {}", address); + log.warn("[PTStatusHolder-{}] unregistered worker: {}", instanceId, address); ProcessorTrackerStatus processorTrackerStatus = new ProcessorTrackerStatus(); processorTrackerStatus.init(address); return processorTrackerStatus; @@ -90,9 +95,9 @@ public class ProcessorTrackerStatusHolder { /** * 注册新的执行节点 * @param address 新的执行节点地址 - * @return true: 注册成功 / false：已存在 + * @return true: register successfully / false: already exists */ - public boolean register(String address) { + private boolean registerOne(String address) { ProcessorTrackerStatus pts = address2Status.get(address); if (pts != null) { return false; @@ -100,9 +105,50 @@ public class ProcessorTrackerStatusHolder { pts = new ProcessorTrackerStatus(); pts.init(address); address2Status.put(address, pts); + log.info("[PTStatusHolder-{}] register new worker: {}", instanceId, address); return true; } + public void register(List workerIpList) { + if (endlessWorkerNum()) { + workerIpList.forEach(this::registerOne); + return; + } + List availableProcessorTrackers = getAvailableProcessorTrackers(); + int currentWorkerSize = availableProcessorTrackers.size(); + int needMoreNum = maxWorkerCount - currentWorkerSize; + if (needMoreNum <= 0) { + return; + } + + log.info("[PTStatusHolder-{}] currentWorkerSize: {}, needMoreNum: {}", instanceId, currentWorkerSize, needMoreNum); + + for (String newIp : workerIpList) { + boolean success = registerOne(newIp); + if (success) { + needMoreNum --; + } + if (needMoreNum <= 0) { + return; + } + } + } + + /** + * 检查是否需要动态加载新的执行器 + * @return check need more workers + */ + public boolean checkNeedMoreWorker() { + if (endlessWorkerNum()) { + return true; + } + return getAvailableProcessorTrackers().size() < maxWorkerCount; + } + + private boolean endlessWorkerNum() { + return maxWorkerCount == null || maxWorkerCount == 0; + } + public void remove(List addressList) { addressList.forEach(address2Status::remove); } diff --git a/powerjob-worker/src/main/java/tech/powerjob/worker/core/tracker/task/TaskTracker.java b/powerjob-worker/src/main/java/tech/powerjob/worker/core/tracker/task/TaskTracker.java index fdf35fa9..492b8d7b 100644 --- a/powerjob-worker/src/main/java/tech/powerjob/worker/core/tracker/task/TaskTracker.java +++ b/powerjob-worker/src/main/java/tech/powerjob/worker/core/tracker/task/TaskTracker.java @@ -124,7 +124,7 @@ public abstract class TaskTracker { // 保护性操作 instanceInfo.setThreadConcurrency(Math.max(1, instanceInfo.getThreadConcurrency())); - this.ptStatusHolder = new ProcessorTrackerStatusHolder(req.getAllWorkerAddress()); + this.ptStatusHolder = new ProcessorTrackerStatusHolder(instanceId, req.getMaxWorkerCount(), req.getAllWorkerAddress()); this.taskPersistenceService = workerRuntime.getTaskPersistenceService(); this.finished = new AtomicBoolean(false); // 只有工作流中的任务允许向工作流中追加上下文数据 @@ -562,6 +562,13 @@ public abstract class TaskTracker { protected class WorkerDetector implements Runnable { @Override public void run() { + + boolean needMoreWorker = ptStatusHolder.checkNeedMoreWorker(); + log.info("[TaskTracker-{}] checkNeedMoreWorker: {}", instanceId, needMoreWorker); + if (!needMoreWorker) { + return; + } + String serverPath = AkkaUtils.getServerActorPath(workerRuntime.getServerDiscoveryService().getCurrentServerAddress()); if (StringUtils.isEmpty(serverPath)) { log.warn("[TaskTracker-{}] no server available, won't start worker detective!", instanceId); @@ -575,11 +582,7 @@ public abstract class TaskTracker { } try { List workerList = JsonUtils.parseObject(response.getData(), new TypeReference>() {}); - workerList.forEach(address -> { - if (ptStatusHolder.register(address)) { - log.info("[TaskTracker-{}] detective new worker: {}", instanceId, address); - } - }); + ptStatusHolder.register(workerList); } catch (Exception e) { log.warn("[TaskTracker-{}] detective failed!", instanceId, e); } From 48ac4460143b922936c6f18f4b5d37c67e1f25b9 Mon Sep 17 00:00:00 2001 From: tjq Date: Sat, 10 Sep 2022 23:41:48 +0800 Subject: [PATCH 15/30] feat: define MonitorService to inject monitor context --- .../core/handler/WorkerRequestHandler.java | 6 +-- .../server/core/lock/UseCacheLockAspect.java | 6 +-- .../tech/powerjob/server/monitor/Monitor.java | 6 +++ .../server/monitor/MonitorContext.java | 24 ++++++++++ .../server/monitor/MonitorService.java | 11 +++++ .../server/monitor/monitors/LogMonitor.java | 36 ++++++++++++++ .../monitor/monitors/ServerMonitor.java | 39 --------------- .../monitor/monitors/impl/LogMonitor.java | 22 --------- .../monitor/DatabaseMonitorAspect.java | 6 +-- .../initializer/PowerJobMonitorService.java | 48 +++++++++++++++++++ .../logback-config/powerjob_monitor.xml | 2 +- 11 files changed, 135 insertions(+), 71 deletions(-) create mode 100644 powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/MonitorContext.java create mode 100644 powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/MonitorService.java create mode 100644 powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/LogMonitor.java delete mode 100644 powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/ServerMonitor.java delete mode 100644 powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/impl/LogMonitor.java create mode 100644 powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/initializer/PowerJobMonitorService.java diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/WorkerRequestHandler.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/WorkerRequestHandler.java index df8ab472..47be383e 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/WorkerRequestHandler.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/WorkerRequestHandler.java @@ -19,8 +19,8 @@ import tech.powerjob.server.core.handler.impl.WorkerRequestHttpHandler; import tech.powerjob.server.core.instance.InstanceLogService; import tech.powerjob.server.core.instance.InstanceManager; import tech.powerjob.server.core.workflow.WorkflowInstanceManager; +import tech.powerjob.server.monitor.MonitorService; import tech.powerjob.server.monitor.events.handler.WorkerHeartbeatEvent; -import tech.powerjob.server.monitor.monitors.ServerMonitor; import tech.powerjob.server.persistence.remote.model.ContainerInfoDO; import tech.powerjob.server.persistence.remote.model.JobInfoDO; import tech.powerjob.server.persistence.remote.repository.ContainerInfoRepository; @@ -48,7 +48,7 @@ import java.util.stream.Collectors; public class WorkerRequestHandler { @Resource - private ServerMonitor serverMonitor; + private MonitorService monitorService; @Resource private Environment environment; @Resource @@ -81,7 +81,7 @@ public class WorkerRequestHandler { WorkerHeartbeatEvent event = new WorkerHeartbeatEvent(); BeanUtils.copyProperties(heartbeat, event); - serverMonitor.record(event.setScore(heartbeat.getSystemMetrics().getScore())); + monitorService.monitor(event.setScore(heartbeat.getSystemMetrics().getScore())); WorkerClusterManagerService.updateStatus(heartbeat); } diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/lock/UseCacheLockAspect.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/lock/UseCacheLockAspect.java index f51809fb..aed87933 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/lock/UseCacheLockAspect.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/lock/UseCacheLockAspect.java @@ -11,8 +11,8 @@ import org.aspectj.lang.annotation.Aspect; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; import tech.powerjob.server.common.utils.AOPUtils; +import tech.powerjob.server.monitor.MonitorService; import tech.powerjob.server.monitor.events.lock.SlowLockEvent; -import tech.powerjob.server.monitor.monitors.ServerMonitor; import javax.annotation.Resource; import java.lang.reflect.Method; @@ -33,7 +33,7 @@ import java.util.concurrent.locks.ReentrantLock; public class UseCacheLockAspect { @Resource - private ServerMonitor serverMonitor; + private MonitorService monitorService; private final Map> lockContainer = Maps.newConcurrentMap(); @@ -67,7 +67,7 @@ public class UseCacheLockAspect { .setCallerMethod(method.getName()) .setCost(timeCost); - serverMonitor.record(slowLockEvent); + monitorService.monitor(slowLockEvent); log.warn("[UseSegmentLockAspect] wait lock for method({}#{}) cost {} ms! key = '{}', args = {}, ", method.getDeclaringClass().getSimpleName(), method.getName(), timeCost, key, diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/Monitor.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/Monitor.java index 7d3a4de2..e53f7f5d 100644 --- a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/Monitor.java +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/Monitor.java @@ -7,6 +7,12 @@ package tech.powerjob.server.monitor; * @since 2022/9/6 */ public interface Monitor { + + /** + * 全局上下文绑定 & 初始化 + * @param monitorContext 日志上下文 + */ + void init(MonitorContext monitorContext); /** * 记录监控事件 * 请注意该方法务必异步不阻塞！！！ diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/MonitorContext.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/MonitorContext.java new file mode 100644 index 00000000..63549192 --- /dev/null +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/MonitorContext.java @@ -0,0 +1,24 @@ +package tech.powerjob.server.monitor; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import lombok.experimental.Accessors; + +import java.io.Serializable; + +/** + * 日志全局上下文 + * + * @author tjq + * @since 2022/9/10 + */ +@Getter +@Setter +@ToString +@Accessors(chain = true) +public class MonitorContext implements Serializable { + private long serverId; + private String serverAddress; + +} diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/MonitorService.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/MonitorService.java new file mode 100644 index 00000000..0fa1465f --- /dev/null +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/MonitorService.java @@ -0,0 +1,11 @@ +package tech.powerjob.server.monitor; + +/** + * 对外暴露的监控服务 + * + * @author tjq + * @since 2022/9/10 + */ +public interface MonitorService { + void monitor(Event event); +} diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/LogMonitor.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/LogMonitor.java new file mode 100644 index 00000000..8ffed245 --- /dev/null +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/LogMonitor.java @@ -0,0 +1,36 @@ +package tech.powerjob.server.monitor.monitors; + +import org.slf4j.LoggerFactory; +import org.slf4j.MDC; +import org.springframework.stereotype.Component; +import tech.powerjob.server.monitor.Event; +import tech.powerjob.server.monitor.Monitor; +import tech.powerjob.server.monitor.MonitorContext; + +/** + * 系统默认实现——基于日志的监控监视器 + * 需要接入方自行基于类 ELK 系统采集 + * + * @author tjq + * @since 2022/9/6 + */ +@Component +public class LogMonitor implements Monitor { + + private MonitorContext monitorContext; + + private static final String MDC_KEY_SERVER_ID = "serverId"; + private static final String MDC_KEY_SERVER_ADDRESS = "serverAddress"; + + @Override + public void init(MonitorContext monitorContext) { + this.monitorContext = monitorContext; + } + + @Override + public void record(Event event) { + MDC.put(MDC_KEY_SERVER_ID, String.valueOf(monitorContext.getServerId())); + MDC.put(MDC_KEY_SERVER_ADDRESS, monitorContext.getServerAddress()); + LoggerFactory.getLogger(event.type()).info(event.message()); + } +} diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/ServerMonitor.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/ServerMonitor.java deleted file mode 100644 index 62f30035..00000000 --- a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/ServerMonitor.java +++ /dev/null @@ -1,39 +0,0 @@ -package tech.powerjob.server.monitor.monitors; - -import com.google.common.collect.Lists; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import tech.powerjob.server.monitor.Event; -import tech.powerjob.server.monitor.Monitor; - -import java.util.List; - -/** - * 服务端监视器 - * - * @author tjq - * @since 2022/9/6 - */ -@Slf4j -@Component -public class ServerMonitor implements Monitor { - - private final List monitors = Lists.newLinkedList(); - - @Autowired - public ServerMonitor(List monitors) { - monitors.forEach(m -> { - if (m == this) { - return; - } - log.info("[ServerMonitor] register monitor: {}", m.getClass().getName()); - this.monitors.add(m); - }); - } - - @Override - public void record(Event event) { - monitors.forEach(m -> m.record(event)); - } -} diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/impl/LogMonitor.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/impl/LogMonitor.java deleted file mode 100644 index 79e433bb..00000000 --- a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/impl/LogMonitor.java +++ /dev/null @@ -1,22 +0,0 @@ -package tech.powerjob.server.monitor.monitors.impl; - -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; -import tech.powerjob.server.monitor.Event; -import tech.powerjob.server.monitor.Monitor; - -/** - * 系统默认实现——基于日志的监控监视器 - * 需要接入方自行基于类 ELK 系统采集 - * - * @author tjq - * @since 2022/9/6 - */ -@Component -public class LogMonitor implements Monitor { - - @Override - public void record(Event event) { - LoggerFactory.getLogger(event.type()).info(event.message()); - } -} diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/monitor/DatabaseMonitorAspect.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/monitor/DatabaseMonitorAspect.java index 71aef266..cd8649b3 100644 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/monitor/DatabaseMonitorAspect.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/monitor/DatabaseMonitorAspect.java @@ -7,9 +7,9 @@ import org.aspectj.lang.annotation.Aspect; import org.springframework.data.domain.Slice; import org.springframework.stereotype.Component; import tech.powerjob.server.common.utils.AOPUtils; +import tech.powerjob.server.monitor.MonitorService; import tech.powerjob.server.monitor.events.db.DatabaseEvent; import tech.powerjob.server.monitor.events.db.DatabaseType; -import tech.powerjob.server.monitor.monitors.ServerMonitor; import javax.annotation.Resource; import java.util.Collection; @@ -28,7 +28,7 @@ import java.util.stream.Stream; public class DatabaseMonitorAspect { @Resource - private ServerMonitor serverMonitor; + private MonitorService monitorService; @Around("execution(* tech.powerjob.server.persistence.remote.repository..*.*(..))") public Object monitorCoreDB(ProceedingJoinPoint joinPoint) throws Throwable { @@ -59,7 +59,7 @@ public class DatabaseMonitorAspect { throw t; } finally { long cost = System.currentTimeMillis() - startTs; - serverMonitor.record(event.setCost(cost)); + monitorService.monitor(event.setCost(cost)); } } diff --git a/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/initializer/PowerJobMonitorService.java b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/initializer/PowerJobMonitorService.java new file mode 100644 index 00000000..8cebd969 --- /dev/null +++ b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/initializer/PowerJobMonitorService.java @@ -0,0 +1,48 @@ +package tech.powerjob.server.initializer; + +import com.google.common.collect.Lists; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import tech.powerjob.server.monitor.Event; +import tech.powerjob.server.monitor.Monitor; +import tech.powerjob.server.monitor.MonitorContext; +import tech.powerjob.server.monitor.MonitorService; +import tech.powerjob.server.remote.server.ServerInfoService; + +import javax.annotation.Resource; +import java.util.List; + +/** + * PowerJob 服务端监控 + * + * @author tjq + * @since 2022/9/10 + */ +@Slf4j +@Component +public class PowerJobMonitorService implements MonitorService { + + @Resource + private ServerInfoService service; + + private final List monitors = Lists.newLinkedList(); + + @Autowired + public PowerJobMonitorService(List monitors) { + + MonitorContext monitorContext = new MonitorContext().setServerId(service.getServerId()).setServerAddress(service.getServerIp()); + log.info("[MonitorService] use monitor context: {}", monitorContext); + + monitors.forEach(m -> { + log.info("[MonitorService] register monitor: {}", m.getClass().getName()); + m.init(monitorContext); + this.monitors.add(m); + }); + } + + @Override + public void monitor(Event event) { + monitors.forEach(m -> m.record(event)); + } +} diff --git a/powerjob-server/powerjob-server-starter/src/main/resources/logback-config/powerjob_monitor.xml b/powerjob-server/powerjob-server-starter/src/main/resources/logback-config/powerjob_monitor.xml index 89c1bcb4..397c29c8 100644 --- a/powerjob-server/powerjob-server-starter/src/main/resources/logback-config/powerjob_monitor.xml +++ b/powerjob-server/powerjob-server-starter/src/main/resources/logback-config/powerjob_monitor.xml @@ -4,7 +4,7 @@ - + From 614349370abdbb28903f608e4943ebcdb67d897a Mon Sep 17 00:00:00 2001 From: tjq Date: Sun, 11 Sep 2022 17:14:00 +0800 Subject: [PATCH 16/30] feat: refactor worker request handler and add monitor --- .../core/handler/AbWorkerRequestHandler.java | 161 ++++++++++++++++ .../core/handler/IWorkerRequestHandler.java | 48 +++++ .../core/handler/WorkerRequestHandler.java | 180 ------------------ .../handler/WorkerRequestHandlerHolder.java | 28 +++ .../handler/WorkerRequestHandlerImpl.java | 67 +++++++ .../server/core/handler/impl/Initializer.java | 26 +++ .../impl/WorkerRequestAkkaHandler.java | 12 +- .../impl/WorkerRequestHttpHandler.java | 8 +- .../handler/TtReportInstanceStatusEvent.java | 26 --- .../w2s/TtReportInstanceStatusEvent.java | 46 +++++ .../WorkerHeartbeatEvent.java | 9 +- .../events/w2s/WorkerLogReportEvent.java | 42 ++++ 12 files changed, 434 insertions(+), 219 deletions(-) create mode 100644 powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/AbWorkerRequestHandler.java create mode 100644 powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/IWorkerRequestHandler.java delete mode 100644 powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/WorkerRequestHandler.java create mode 100644 powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/WorkerRequestHandlerHolder.java create mode 100644 powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/WorkerRequestHandlerImpl.java create mode 100644 powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/impl/Initializer.java delete mode 100644 powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/handler/TtReportInstanceStatusEvent.java create mode 100644 powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/w2s/TtReportInstanceStatusEvent.java rename powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/{handler => w2s}/WorkerHeartbeatEvent.java (81%) create mode 100644 powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/w2s/WorkerLogReportEvent.java diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/AbWorkerRequestHandler.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/AbWorkerRequestHandler.java new file mode 100644 index 00000000..81b61758 --- /dev/null +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/AbWorkerRequestHandler.java @@ -0,0 +1,161 @@ +package tech.powerjob.server.core.handler; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.springframework.beans.BeanUtils; +import org.springframework.core.env.Environment; +import tech.powerjob.common.request.*; +import tech.powerjob.common.response.AskResponse; +import tech.powerjob.common.serialize.JsonUtils; +import tech.powerjob.common.utils.NetUtils; +import tech.powerjob.server.common.constants.SwitchableStatus; +import tech.powerjob.server.common.module.WorkerInfo; +import tech.powerjob.server.common.utils.SpringUtils; +import tech.powerjob.server.monitor.MonitorService; +import tech.powerjob.server.monitor.events.w2s.TtReportInstanceStatusEvent; +import tech.powerjob.server.monitor.events.w2s.WorkerHeartbeatEvent; +import tech.powerjob.server.monitor.events.w2s.WorkerLogReportEvent; +import tech.powerjob.server.persistence.remote.model.ContainerInfoDO; +import tech.powerjob.server.persistence.remote.model.JobInfoDO; +import tech.powerjob.server.persistence.remote.repository.ContainerInfoRepository; +import tech.powerjob.server.persistence.remote.repository.JobInfoRepository; +import tech.powerjob.server.remote.worker.WorkerClusterQueryService; + +import javax.annotation.Resource; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.RejectedExecutionException; +import java.util.stream.Collectors; + +/** + * wrapper monitor for IWorkerRequestHandler + * + * @author tjq + * @since 2022/9/11 + */ +@Slf4j +public abstract class AbWorkerRequestHandler implements IWorkerRequestHandler { + + @Resource + protected MonitorService monitorService; + @Resource + protected Environment environment; + @Resource + protected ContainerInfoRepository containerInfoRepository; + @Resource + private WorkerClusterQueryService workerClusterQueryService; + + protected abstract void processWorkerHeartbeat0(WorkerHeartbeat heartbeat, WorkerHeartbeatEvent event); + + protected abstract Optional processTaskTrackerReportInstanceStatus0(TaskTrackerReportInstanceStatusReq req, TtReportInstanceStatusEvent event) throws Exception; + + protected abstract void processWorkerLogReport0(WorkerLogReportReq req, WorkerLogReportEvent event); + + + @Override + public void processWorkerHeartbeat(WorkerHeartbeat heartbeat) { + long startMs = System.currentTimeMillis(); + WorkerHeartbeatEvent event = new WorkerHeartbeatEvent() + .setAppName(heartbeat.getAppName()) + .setAppId(heartbeat.getAppId()) + .setVersion(heartbeat.getVersion()) + .setProtocol(heartbeat.getProtocol()) + .setTag(heartbeat.getTag()) + .setWorkerAddress(heartbeat.getWorkerAddress()) + .setDelayMs(startMs - heartbeat.getHeartbeatTime()) + .setScore(heartbeat.getSystemMetrics().getScore()); + processWorkerHeartbeat0(heartbeat, event); + monitorService.monitor(event); + } + + @Override + public Optional processTaskTrackerReportInstanceStatus(TaskTrackerReportInstanceStatusReq req) { + long startMs = System.currentTimeMillis(); + TtReportInstanceStatusEvent event = new TtReportInstanceStatusEvent() + .setAppId(req.getAppId()) + .setJobId(req.getJobId()) + .setInstanceId(req.getInstanceId()) + .setWfInstanceId(req.getWfInstanceId()) + .setInstanceStatus(req.getInstanceStatus()) + .setDelayMs(startMs - req.getReportTime()) + .setServerProcessStatus(TtReportInstanceStatusEvent.Status.SUCCESS); + try { + return processTaskTrackerReportInstanceStatus0(req, event); + } catch (Exception e) { + event.setServerProcessStatus(TtReportInstanceStatusEvent.Status.FAILED); + log.error("[WorkerRequestHandler] processTaskTrackerReportInstanceStatus failed for request: {}", req, e); + return Optional.of(AskResponse.failed(ExceptionUtils.getMessage(e))); + } finally { + event.setServerProcessCost(System.currentTimeMillis() - startMs); + monitorService.monitor(event); + } + } + + @Override + public void processWorkerLogReport(WorkerLogReportReq req) { + + long startMs = System.currentTimeMillis(); + WorkerLogReportEvent event = new WorkerLogReportEvent() + .setWorkerAddress(req.getWorkerAddress()); + try { + processWorkerLogReport0(req, event); + event.setStatus(WorkerLogReportEvent.Status.SUCCESS); + } catch (RejectedExecutionException re) { + event.setStatus(WorkerLogReportEvent.Status.REJECTED); + } catch (Throwable t) { + event.setStatus(WorkerLogReportEvent.Status.EXCEPTION); + log.warn("[WorkerRequestHandler] process worker report failed!", t); + } finally { + event.setServerCost(System.currentTimeMillis() - startMs); + monitorService.monitor(event); + } + } + + @Override + public AskResponse processWorkerQueryExecutorCluster(WorkerQueryExecutorClusterReq req) { + AskResponse askResponse; + + Long jobId = req.getJobId(); + Long appId = req.getAppId(); + + JobInfoRepository jobInfoRepository = SpringUtils.getBean(JobInfoRepository.class); + Optional jobInfoOpt = jobInfoRepository.findById(jobId); + if (jobInfoOpt.isPresent()) { + JobInfoDO jobInfo = jobInfoOpt.get(); + if (!jobInfo.getAppId().equals(appId)) { + askResponse = AskResponse.failed("Permission Denied!"); + }else { + List sortedAvailableWorker = workerClusterQueryService.getSuitableWorkers(jobInfo) + .stream().map(WorkerInfo::getAddress).collect(Collectors.toList()); + askResponse = AskResponse.succeed(sortedAvailableWorker); + } + }else { + askResponse = AskResponse.failed("can't find jobInfo by jobId: " + jobId); + } + return askResponse; + } + + @Override + public AskResponse processWorkerNeedDeployContainer(WorkerNeedDeployContainerRequest req) { + String port = environment.getProperty("local.server.port"); + + Optional containerInfoOpt = containerInfoRepository.findById(req.getContainerId()); + AskResponse askResponse = new AskResponse(); + if (!containerInfoOpt.isPresent() || containerInfoOpt.get().getStatus() != SwitchableStatus.ENABLE.getV()) { + askResponse.setSuccess(false); + askResponse.setMessage("can't find container by id: " + req.getContainerId()); + }else { + ContainerInfoDO containerInfo = containerInfoOpt.get(); + askResponse.setSuccess(true); + + ServerDeployContainerRequest dpReq = new ServerDeployContainerRequest(); + BeanUtils.copyProperties(containerInfo, dpReq); + dpReq.setContainerId(containerInfo.getId()); + String downloadURL = String.format("http://%s:%s/container/downloadJar?version=%s", NetUtils.getLocalHost(), port, containerInfo.getVersion()); + dpReq.setDownloadURL(downloadURL); + + askResponse.setData(JsonUtils.toBytes(dpReq)); + } + return askResponse; + } +} diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/IWorkerRequestHandler.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/IWorkerRequestHandler.java new file mode 100644 index 00000000..6fe30e70 --- /dev/null +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/IWorkerRequestHandler.java @@ -0,0 +1,48 @@ +package tech.powerjob.server.core.handler; + +import tech.powerjob.common.request.*; +import tech.powerjob.common.response.AskResponse; + +import java.util.Optional; + +/** + * 定义 server 与 worker 之间需要处理的协议 + * + * @author tjq + * @since 2022/9/10 + */ +public interface IWorkerRequestHandler { + + /** + * 处理 worker 上报的心跳信息 + * @param heartbeat 心跳信息 + */ + void processWorkerHeartbeat(WorkerHeartbeat heartbeat); + + /** + * 处理 TaskTracker 的任务实例上报 + * @param req 上报请求 + * @return 响应信息 + */ + Optional processTaskTrackerReportInstanceStatus(TaskTrackerReportInstanceStatusReq req); + + /** + * 处理 worker 查询执行器集群 + * @param req 请求 + * @return cluster info + */ + AskResponse processWorkerQueryExecutorCluster(WorkerQueryExecutorClusterReq req); + + /** + * 处理 worker 日志推送请求 + * @param req 请求 + */ + void processWorkerLogReport(WorkerLogReportReq req); + + /** + * 处理 worker 的容器部署请求 + * @param request 请求 + * @return 容器部署信息 + */ + AskResponse processWorkerNeedDeployContainer(WorkerNeedDeployContainerRequest request); +} diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/WorkerRequestHandler.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/WorkerRequestHandler.java deleted file mode 100644 index 47be383e..00000000 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/WorkerRequestHandler.java +++ /dev/null @@ -1,180 +0,0 @@ -package tech.powerjob.server.core.handler; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.BeanUtils; -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; -import tech.powerjob.common.RemoteConstant; -import tech.powerjob.common.enums.InstanceStatus; -import tech.powerjob.common.request.*; -import tech.powerjob.common.response.AskResponse; -import tech.powerjob.common.serialize.JsonUtils; -import tech.powerjob.common.utils.NetUtils; -import tech.powerjob.server.common.constants.SwitchableStatus; -import tech.powerjob.server.common.module.WorkerInfo; -import tech.powerjob.server.common.utils.SpringUtils; -import tech.powerjob.server.core.handler.impl.WorkerRequestAkkaHandler; -import tech.powerjob.server.core.handler.impl.WorkerRequestHttpHandler; -import tech.powerjob.server.core.instance.InstanceLogService; -import tech.powerjob.server.core.instance.InstanceManager; -import tech.powerjob.server.core.workflow.WorkflowInstanceManager; -import tech.powerjob.server.monitor.MonitorService; -import tech.powerjob.server.monitor.events.handler.WorkerHeartbeatEvent; -import tech.powerjob.server.persistence.remote.model.ContainerInfoDO; -import tech.powerjob.server.persistence.remote.model.JobInfoDO; -import tech.powerjob.server.persistence.remote.repository.ContainerInfoRepository; -import tech.powerjob.server.persistence.remote.repository.JobInfoRepository; -import tech.powerjob.server.remote.transport.starter.AkkaStarter; -import tech.powerjob.server.remote.transport.starter.VertXStarter; -import tech.powerjob.server.remote.worker.WorkerClusterManagerService; -import tech.powerjob.server.remote.worker.WorkerClusterQueryService; - -import javax.annotation.PostConstruct; -import javax.annotation.Resource; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; - -/** - * receive and process worker's request - * - * @author tjq - * @since 2021/2/8 - */ -@Slf4j -@Component -public class WorkerRequestHandler { - - @Resource - private MonitorService monitorService; - @Resource - private Environment environment; - @Resource - private InstanceManager instanceManager; - @Resource - private WorkflowInstanceManager workflowInstanceManager; - @Resource - private InstanceLogService instanceLogService; - @Resource - private ContainerInfoRepository containerInfoRepository; - - @Resource - private WorkerClusterQueryService workerClusterQueryService; - - private static WorkerRequestHandler workerRequestHandler; - - @PostConstruct - public void initHandler() { - // init akka - AkkaStarter.actorSystem.actorOf(WorkerRequestAkkaHandler.defaultProps(), RemoteConstant.SERVER_ACTOR_NAME); - // init vert.x - VertXStarter.vertx.deployVerticle(new WorkerRequestHttpHandler()); - } - - /** - * 处理 Worker 的心跳请求 - * @param heartbeat 心跳包 - */ - public void onReceiveWorkerHeartbeat(WorkerHeartbeat heartbeat) { - - WorkerHeartbeatEvent event = new WorkerHeartbeatEvent(); - BeanUtils.copyProperties(heartbeat, event); - monitorService.monitor(event.setScore(heartbeat.getSystemMetrics().getScore())); - - WorkerClusterManagerService.updateStatus(heartbeat); - } - - /** - * 处理 instance 状态 - * @param req 任务实例的状态上报请求 - */ - public Optional onReceiveTaskTrackerReportInstanceStatusReq(TaskTrackerReportInstanceStatusReq req) throws ExecutionException { - // 2021/02/05 如果是工作流中的实例先尝试更新上下文信息，再更新实例状态，这里一定不会有异常 - if (req.getWfInstanceId() != null && !CollectionUtils.isEmpty(req.getAppendedWfContext())) { - // 更新工作流上下文信息 - workflowInstanceManager.updateWorkflowContext(req.getWfInstanceId(),req.getAppendedWfContext()); - } - - instanceManager.updateStatus(req); - - // 结束状态（成功/失败）需要回复消息 - if (InstanceStatus.FINISHED_STATUS.contains(req.getInstanceStatus())) { - return Optional.of(AskResponse.succeed(null)); - } - return Optional.empty(); - } - - /** - * 处理OMS在线日志请求 - * @param req 日志请求 - */ - public void onReceiveWorkerLogReportReq(WorkerLogReportReq req) { - // 这个效率应该不会拉垮吧...也就是一些判断 + Map#get 吧... - instanceLogService.submitLogs(req.getWorkerAddress(), req.getInstanceLogContents()); - } - - /** - * 处理 Worker容器部署请求 - * @param req 容器部署请求 - */ - public AskResponse onReceiveWorkerNeedDeployContainerRequest(WorkerNeedDeployContainerRequest req) { - - String port = environment.getProperty("local.server.port"); - - Optional containerInfoOpt = containerInfoRepository.findById(req.getContainerId()); - AskResponse askResponse = new AskResponse(); - if (!containerInfoOpt.isPresent() || containerInfoOpt.get().getStatus() != SwitchableStatus.ENABLE.getV()) { - askResponse.setSuccess(false); - askResponse.setMessage("can't find container by id: " + req.getContainerId()); - }else { - ContainerInfoDO containerInfo = containerInfoOpt.get(); - askResponse.setSuccess(true); - - ServerDeployContainerRequest dpReq = new ServerDeployContainerRequest(); - BeanUtils.copyProperties(containerInfo, dpReq); - dpReq.setContainerId(containerInfo.getId()); - String downloadURL = String.format("http://%s:%s/container/downloadJar?version=%s", NetUtils.getLocalHost(), port, containerInfo.getVersion()); - dpReq.setDownloadURL(downloadURL); - - askResponse.setData(JsonUtils.toBytes(dpReq)); - } - return askResponse; - } - - /** - * 处理 worker 请求获取当前任务所有处理器节点的请求 - * @param req jobId + appId - */ - public AskResponse onReceiveWorkerQueryExecutorClusterReq(WorkerQueryExecutorClusterReq req) { - - AskResponse askResponse; - - Long jobId = req.getJobId(); - Long appId = req.getAppId(); - - JobInfoRepository jobInfoRepository = SpringUtils.getBean(JobInfoRepository.class); - Optional jobInfoOpt = jobInfoRepository.findById(jobId); - if (jobInfoOpt.isPresent()) { - JobInfoDO jobInfo = jobInfoOpt.get(); - if (!jobInfo.getAppId().equals(appId)) { - askResponse = AskResponse.failed("Permission Denied!"); - }else { - List sortedAvailableWorker = workerClusterQueryService.getSuitableWorkers(jobInfo) - .stream().map(WorkerInfo::getAddress).collect(Collectors.toList()); - askResponse = AskResponse.succeed(sortedAvailableWorker); - } - }else { - askResponse = AskResponse.failed("can't find jobInfo by jobId: " + jobId); - } - return askResponse; - } - - public static WorkerRequestHandler getWorkerRequestHandler() { - if (workerRequestHandler == null) { - workerRequestHandler = SpringUtils.getBean(WorkerRequestHandler.class); - } - return workerRequestHandler; - } -} diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/WorkerRequestHandlerHolder.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/WorkerRequestHandlerHolder.java new file mode 100644 index 00000000..ac259e03 --- /dev/null +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/WorkerRequestHandlerHolder.java @@ -0,0 +1,28 @@ +package tech.powerjob.server.core.handler; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; + +/** + * WorkerRequestHandlerHolder + * + * @author tjq + * @since 2022/9/11 + */ +@Component +public class WorkerRequestHandlerHolder { + + private static IWorkerRequestHandler workerRequestHandler; + + + public static IWorkerRequestHandler fetchWorkerRequestHandler() { + return workerRequestHandler; + } + + @Autowired + public void setWorkerRequestHandler(IWorkerRequestHandler workerRequestHandler) { + WorkerRequestHandlerHolder.workerRequestHandler = workerRequestHandler; + } +} diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/WorkerRequestHandlerImpl.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/WorkerRequestHandlerImpl.java new file mode 100644 index 00000000..77e573be --- /dev/null +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/WorkerRequestHandlerImpl.java @@ -0,0 +1,67 @@ +package tech.powerjob.server.core.handler; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; +import tech.powerjob.common.enums.InstanceStatus; +import tech.powerjob.common.request.TaskTrackerReportInstanceStatusReq; +import tech.powerjob.common.request.WorkerHeartbeat; +import tech.powerjob.common.request.WorkerLogReportReq; +import tech.powerjob.common.response.AskResponse; +import tech.powerjob.server.core.instance.InstanceLogService; +import tech.powerjob.server.core.instance.InstanceManager; +import tech.powerjob.server.core.workflow.WorkflowInstanceManager; +import tech.powerjob.server.monitor.events.w2s.TtReportInstanceStatusEvent; +import tech.powerjob.server.monitor.events.w2s.WorkerHeartbeatEvent; +import tech.powerjob.server.monitor.events.w2s.WorkerLogReportEvent; +import tech.powerjob.server.remote.worker.WorkerClusterManagerService; + +import javax.annotation.Resource; +import java.util.Optional; + +/** + * receive and process worker's request + * + * @author tjq + * @since 2022/9/11 + */ +@Slf4j +@Component +public class WorkerRequestHandlerImpl extends AbWorkerRequestHandler { + + @Resource + private InstanceManager instanceManager; + @Resource + private WorkflowInstanceManager workflowInstanceManager; + @Resource + private InstanceLogService instanceLogService; + + @Override + protected void processWorkerHeartbeat0(WorkerHeartbeat heartbeat, WorkerHeartbeatEvent event) { + WorkerClusterManagerService.updateStatus(heartbeat); + } + + @Override + protected Optional processTaskTrackerReportInstanceStatus0(TaskTrackerReportInstanceStatusReq req, TtReportInstanceStatusEvent event) throws Exception { + // 2021/02/05 如果是工作流中的实例先尝试更新上下文信息，再更新实例状态，这里一定不会有异常 + if (req.getWfInstanceId() != null && !CollectionUtils.isEmpty(req.getAppendedWfContext())) { + // 更新工作流上下文信息 + workflowInstanceManager.updateWorkflowContext(req.getWfInstanceId(),req.getAppendedWfContext()); + } + + instanceManager.updateStatus(req); + + // 结束状态（成功/失败）需要回复消息 + if (InstanceStatus.FINISHED_STATUS.contains(req.getInstanceStatus())) { + return Optional.of(AskResponse.succeed(null)); + } + return Optional.empty(); + } + + @Override + protected void processWorkerLogReport0(WorkerLogReportReq req, WorkerLogReportEvent event) { + // 这个效率应该不会拉垮吧...也就是一些判断 + Map#get 吧... + instanceLogService.submitLogs(req.getWorkerAddress(), req.getInstanceLogContents()); + } +} diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/impl/Initializer.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/impl/Initializer.java new file mode 100644 index 00000000..77951c32 --- /dev/null +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/impl/Initializer.java @@ -0,0 +1,26 @@ +package tech.powerjob.server.core.handler.impl; + +import org.springframework.stereotype.Component; +import tech.powerjob.common.RemoteConstant; +import tech.powerjob.server.remote.transport.starter.AkkaStarter; +import tech.powerjob.server.remote.transport.starter.VertXStarter; + +import javax.annotation.PostConstruct; + +/** + * 初始化器 + * + * @author tjq + * @since 2022/9/11 + */ +@Component +public class Initializer { + + @PostConstruct + public void initHandler() { + // init akka + AkkaStarter.actorSystem.actorOf(WorkerRequestAkkaHandler.defaultProps(), RemoteConstant.SERVER_ACTOR_NAME); + // init vert.x + VertXStarter.vertx.deployVerticle(new WorkerRequestHttpHandler()); + } +} diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/impl/WorkerRequestAkkaHandler.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/impl/WorkerRequestAkkaHandler.java index 99aa0cd3..147b9df4 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/impl/WorkerRequestAkkaHandler.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/impl/WorkerRequestAkkaHandler.java @@ -10,7 +10,7 @@ import lombok.extern.slf4j.Slf4j; import java.util.Optional; -import static tech.powerjob.server.core.handler.WorkerRequestHandler.getWorkerRequestHandler; +import static tech.powerjob.server.core.handler.WorkerRequestHandlerHolder.fetchWorkerRequestHandler; /** * 处理 Worker 请求 @@ -42,9 +42,9 @@ public class WorkerRequestAkkaHandler extends AbstractActor { @Override public Receive createReceive() { return receiveBuilder() - .match(WorkerHeartbeat.class, hb -> getWorkerRequestHandler().onReceiveWorkerHeartbeat(hb)) + .match(WorkerHeartbeat.class, hb -> fetchWorkerRequestHandler().processWorkerHeartbeat(hb)) .match(TaskTrackerReportInstanceStatusReq.class, this::onReceiveTaskTrackerReportInstanceStatusReq) - .match(WorkerLogReportReq.class, req -> getWorkerRequestHandler().onReceiveWorkerLogReportReq(req)) + .match(WorkerLogReportReq.class, req -> fetchWorkerRequestHandler().processWorkerLogReport(req)) .match(WorkerNeedDeployContainerRequest.class, this::onReceiveWorkerNeedDeployContainerRequest) .match(WorkerQueryExecutorClusterReq.class, this::onReceiveWorkerQueryExecutorClusterReq) .matchAny(obj -> log.warn("[WorkerRequestAkkaHandler] receive unknown request: {}.", obj)) @@ -71,7 +71,7 @@ public class WorkerRequestAkkaHandler extends AbstractActor { private void onReceiveTaskTrackerReportInstanceStatusReq(TaskTrackerReportInstanceStatusReq req) { try { - Optional askResponseOpt = getWorkerRequestHandler().onReceiveTaskTrackerReportInstanceStatusReq(req); + Optional askResponseOpt = fetchWorkerRequestHandler().processTaskTrackerReportInstanceStatus(req); if (askResponseOpt.isPresent()) { getSender().tell(AskResponse.succeed(null), getSelf()); } @@ -85,7 +85,7 @@ public class WorkerRequestAkkaHandler extends AbstractActor { * @param req 容器部署请求 */ private void onReceiveWorkerNeedDeployContainerRequest(WorkerNeedDeployContainerRequest req) { - getSender().tell(getWorkerRequestHandler().onReceiveWorkerNeedDeployContainerRequest(req), getSelf()); + getSender().tell(fetchWorkerRequestHandler().processWorkerNeedDeployContainer(req), getSelf()); } /** @@ -94,7 +94,7 @@ public class WorkerRequestAkkaHandler extends AbstractActor { */ private void onReceiveWorkerQueryExecutorClusterReq(WorkerQueryExecutorClusterReq req) { - getSender().tell(getWorkerRequestHandler().onReceiveWorkerQueryExecutorClusterReq(req), getSelf()); + getSender().tell(fetchWorkerRequestHandler().processWorkerQueryExecutorCluster(req), getSelf()); } } diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/impl/WorkerRequestHttpHandler.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/impl/WorkerRequestHttpHandler.java index 615dab2c..3749158f 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/impl/WorkerRequestHttpHandler.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/impl/WorkerRequestHttpHandler.java @@ -21,7 +21,7 @@ import org.apache.commons.lang3.exception.ExceptionUtils; import java.util.Properties; -import static tech.powerjob.server.core.handler.WorkerRequestHandler.getWorkerRequestHandler; +import static tech.powerjob.server.core.handler.WorkerRequestHandlerHolder.fetchWorkerRequestHandler; /** * WorkerRequestHandler @@ -46,14 +46,14 @@ public class WorkerRequestHttpHandler extends AbstractVerticle { router.post(ProtocolConstant.SERVER_PATH_HEARTBEAT) .handler(ctx -> { WorkerHeartbeat heartbeat = ctx.getBodyAsJson().mapTo(WorkerHeartbeat.class); - getWorkerRequestHandler().onReceiveWorkerHeartbeat(heartbeat); + fetchWorkerRequestHandler().processWorkerHeartbeat(heartbeat); success(ctx); }); router.post(ProtocolConstant.SERVER_PATH_STATUS_REPORT) .blockingHandler(ctx -> { TaskTrackerReportInstanceStatusReq req = ctx.getBodyAsJson().mapTo(TaskTrackerReportInstanceStatusReq.class); try { - getWorkerRequestHandler().onReceiveTaskTrackerReportInstanceStatusReq(req); + fetchWorkerRequestHandler().processTaskTrackerReportInstanceStatus(req); out(ctx, AskResponse.succeed(null)); } catch (Exception e) { log.error("[WorkerRequestHttpHandler] update instance status failed for request: {}.", req, e); @@ -63,7 +63,7 @@ public class WorkerRequestHttpHandler extends AbstractVerticle { router.post(ProtocolConstant.SERVER_PATH_LOG_REPORT) .blockingHandler(ctx -> { WorkerLogReportReq req = ctx.getBodyAsJson().mapTo(WorkerLogReportReq.class); - getWorkerRequestHandler().onReceiveWorkerLogReportReq(req); + fetchWorkerRequestHandler().processWorkerLogReport(req); success(ctx); }); server.requestHandler(router).listen(port); diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/handler/TtReportInstanceStatusEvent.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/handler/TtReportInstanceStatusEvent.java deleted file mode 100644 index 567762b4..00000000 --- a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/handler/TtReportInstanceStatusEvent.java +++ /dev/null @@ -1,26 +0,0 @@ -package tech.powerjob.server.monitor.events.handler; - -import lombok.Setter; -import lombok.experimental.Accessors; -import tech.powerjob.server.monitor.Event; - -/** - * TaskTrackerReportInstanceStatus - * - * @author tjq - * @since 2022/9/9 - */ -@Setter -@Accessors(chain = true) -public class TtReportInstanceStatusEvent implements Event { - - @Override - public String type() { - return "MONITOR_LOGGER_TT_REPORT_STATUS"; - } - - @Override - public String message() { - return null; - } -} diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/w2s/TtReportInstanceStatusEvent.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/w2s/TtReportInstanceStatusEvent.java new file mode 100644 index 00000000..6eef0faf --- /dev/null +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/w2s/TtReportInstanceStatusEvent.java @@ -0,0 +1,46 @@ +package tech.powerjob.server.monitor.events.w2s; + +import lombok.Setter; +import lombok.experimental.Accessors; +import tech.powerjob.server.common.SJ; +import tech.powerjob.server.monitor.Event; + +/** + * TaskTrackerReportInstanceStatus + * + * @author tjq + * @since 2022/9/9 + */ +@Setter +@Accessors(chain = true) +public class TtReportInstanceStatusEvent implements Event { + + private Long appId; + private Long jobId; + private Long instanceId; + + private Long wfInstanceId; + + private int instanceStatus; + + private Long delayMs; + + private Status serverProcessStatus; + + private Long serverProcessCost; + + public enum Status { + SUCCESS, + FAILED + } + + @Override + public String type() { + return "MONITOR_LOGGER_TT_REPORT_STATUS"; + } + + @Override + public String message() { + return SJ.MONITOR_JOINER.join(appId, jobId, instanceId, wfInstanceId, instanceStatus, delayMs, serverProcessStatus, serverProcessCost); + } +} diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/handler/WorkerHeartbeatEvent.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/w2s/WorkerHeartbeatEvent.java similarity index 81% rename from powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/handler/WorkerHeartbeatEvent.java rename to powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/w2s/WorkerHeartbeatEvent.java index 9dd3cff0..d2686dd8 100644 --- a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/handler/WorkerHeartbeatEvent.java +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/w2s/WorkerHeartbeatEvent.java @@ -1,4 +1,4 @@ -package tech.powerjob.server.monitor.events.handler; +package tech.powerjob.server.monitor.events.w2s; import lombok.Setter; import lombok.experimental.Accessors; @@ -26,7 +26,10 @@ public class WorkerHeartbeatEvent implements Event { private String tag; private String workerAddress; - + /** + * worker 上报时间与 server 之间的延迟 + */ + private long delayMs; private Integer score; @Override @@ -36,6 +39,6 @@ public class WorkerHeartbeatEvent implements Event { @Override public String message() { - return SJ.MONITOR_JOINER.join(appName, appId, version, protocol, tag, workerAddress, score); + return SJ.MONITOR_JOINER.join(appName, appId, version, protocol, tag, workerAddress, delayMs, score); } } diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/w2s/WorkerLogReportEvent.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/w2s/WorkerLogReportEvent.java new file mode 100644 index 00000000..89bec99d --- /dev/null +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/w2s/WorkerLogReportEvent.java @@ -0,0 +1,42 @@ +package tech.powerjob.server.monitor.events.w2s; + +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; +import tech.powerjob.server.common.SJ; +import tech.powerjob.server.monitor.Event; + +/** + * description + * + * @author tjq + * @since 2022/9/11 + */ +@Setter +@Accessors(chain = true) +public class WorkerLogReportEvent implements Event { + + private String workerAddress; + + private long logSize; + + private Status status; + + private long serverCost; + + public enum Status { + SUCCESS, + REJECTED, + EXCEPTION + } + + @Override + public String type() { + return "MONITOR_LOGGER_WORKER_LOG_REPORT"; + } + + @Override + public String message() { + return SJ.MONITOR_JOINER.join(workerAddress, logSize, status, serverCost); + } +} From 3869b115ce1500c366ed1c034d6f3f7fc5a5dc52 Mon Sep 17 00:00:00 2001 From: tjq Date: Mon, 12 Sep 2022 09:36:29 +0800 Subject: [PATCH 17/30] feat: optimize RejectedExecutionHandlerFactory --- .../common/RejectedExecutionHandlerFactory.java | 11 ++++------- .../tech/powerjob/server/config/ThreadPoolConfig.java | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/RejectedExecutionHandlerFactory.java b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/RejectedExecutionHandlerFactory.java index e7ea7cc0..e0727763 100644 --- a/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/RejectedExecutionHandlerFactory.java +++ b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/RejectedExecutionHandlerFactory.java @@ -21,10 +21,9 @@ public class RejectedExecutionHandlerFactory { * @param source log name * @return A handler for tasks that cannot be executed by ThreadPool */ - public static RejectedExecutionHandler newReject(String source) { + public static RejectedExecutionHandler newDiscard(String source) { return (r, p) -> { - log.error("[{}] ThreadPool[{}] overload, the task[{}] will be dropped!", source, p, r); - log.warn("[{}] Maybe you need to adjust the ThreadPool config!", source); + log.error("[{}] ThreadPool[{}] overload, the task[{}] will be Discard, Maybe you need to adjust the ThreadPool config!", source, p, r); }; } @@ -35,8 +34,7 @@ public class RejectedExecutionHandlerFactory { */ public static RejectedExecutionHandler newCallerRun(String source) { return (r, p) -> { - log.warn("[{}] ThreadPool[{}] overload, the task[{}] will run by caller thread!", source, p, r); - log.warn("[{}] Maybe you need to adjust the ThreadPool config!", source); + log.error("[{}] ThreadPool[{}] overload, the task[{}] will run by caller thread, Maybe you need to adjust the ThreadPool config!", source, p, r); if (!p.isShutdown()) { r.run(); } @@ -50,8 +48,7 @@ public class RejectedExecutionHandlerFactory { */ public static RejectedExecutionHandler newThreadRun(String source) { return (r, p) -> { - log.warn("[{}] ThreadPool[{}] overload, the task[{}] will run by a new thread!", source, p, r); - log.warn("[{}] Maybe you need to adjust the ThreadPool config!", source); + log.error("[{}] ThreadPool[{}] overload, the task[{}] will run by a new thread!, Maybe you need to adjust the ThreadPool config!", source, p, r); if (!p.isShutdown()) { String threadName = source + "-T-" + COUNTER.getAndIncrement(); log.info("[{}] create new thread[{}] to run job", source, threadName); diff --git a/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/config/ThreadPoolConfig.java b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/config/ThreadPoolConfig.java index b9f42e86..ba97cae9 100644 --- a/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/config/ThreadPoolConfig.java +++ b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/config/ThreadPoolConfig.java @@ -46,7 +46,7 @@ public class ThreadPoolConfig { executor.setQueueCapacity(8192); executor.setKeepAliveSeconds(60); executor.setThreadNamePrefix("omsBackgroundPool-"); - executor.setRejectedExecutionHandler(RejectedExecutionHandlerFactory.newReject("PowerJobBackgroundPool")); + executor.setRejectedExecutionHandler(RejectedExecutionHandlerFactory.newDiscard("PowerJobBackgroundPool")); return executor; } From 3b73a750e6b2d9ca19fe39d3c83b191b8d69b069 Mon Sep 17 00:00:00 2001 From: tjq Date: Mon, 12 Sep 2022 10:45:56 +0800 Subject: [PATCH 18/30] feat: server async process log report to prevent timeout #432 --- .../RejectedExecutionHandlerFactory.java | 14 +++++++++ .../holder/HashedWheelTimerHolder.java | 3 +- .../holder/InstanceTimeWheelService.java | 5 ++-- .../core/handler/AbWorkerRequestHandler.java | 2 -- .../core/instance/InstanceLogService.java | 29 +++++++++++-------- .../server/core/instance/InstanceManager.java | 2 +- .../events/w2s/WorkerLogReportEvent.java | 4 +-- .../server/config/ThreadPoolConfig.java | 17 +++++++++-- 8 files changed, 52 insertions(+), 24 deletions(-) diff --git a/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/RejectedExecutionHandlerFactory.java b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/RejectedExecutionHandlerFactory.java index e0727763..6e0e1527 100644 --- a/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/RejectedExecutionHandlerFactory.java +++ b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/RejectedExecutionHandlerFactory.java @@ -2,6 +2,7 @@ package tech.powerjob.server.common; import lombok.extern.slf4j.Slf4j; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.RejectedExecutionHandler; import java.util.concurrent.atomic.AtomicLong; @@ -16,6 +17,19 @@ public class RejectedExecutionHandlerFactory { private static final AtomicLong COUNTER = new AtomicLong(); + /** + * 拒绝执行，抛出 RejectedExecutionException + * @param source name for log + * @return A handler for tasks that cannot be executed by ThreadPool + */ + public static RejectedExecutionHandler newAbort(String source) { + return (r, e) -> { + log.error("[{}] ThreadPool[{}] overload, the task[{}] will be Abort, Maybe you need to adjust the ThreadPool config!", source, e, r); + throw new RejectedExecutionException("Task " + r.toString() + + " rejected from " + source); + }; + } + /** * 直接丢弃该任务 * @param source log name diff --git a/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/timewheel/holder/HashedWheelTimerHolder.java b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/timewheel/holder/HashedWheelTimerHolder.java index 0d9556ba..5d73a46a 100644 --- a/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/timewheel/holder/HashedWheelTimerHolder.java +++ b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/timewheel/holder/HashedWheelTimerHolder.java @@ -1,6 +1,7 @@ package tech.powerjob.server.common.timewheel.holder; import tech.powerjob.server.common.timewheel.HashedWheelTimer; +import tech.powerjob.server.common.timewheel.Timer; /** * 时间轮单例 @@ -11,7 +12,7 @@ import tech.powerjob.server.common.timewheel.HashedWheelTimer; public class HashedWheelTimerHolder { // 非精确时间轮，每 5S 走一格 - public static final HashedWheelTimer INACCURATE_TIMER = new HashedWheelTimer(5, 16, 0); + public static final Timer INACCURATE_TIMER = new HashedWheelTimer(5000, 16, 0); private HashedWheelTimerHolder() { } diff --git a/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/timewheel/holder/InstanceTimeWheelService.java b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/timewheel/holder/InstanceTimeWheelService.java index a6453916..35d06d4c 100644 --- a/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/timewheel/holder/InstanceTimeWheelService.java +++ b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/timewheel/holder/InstanceTimeWheelService.java @@ -1,6 +1,7 @@ package tech.powerjob.server.common.timewheel.holder; import tech.powerjob.server.common.timewheel.HashedWheelTimer; +import tech.powerjob.server.common.timewheel.Timer; import tech.powerjob.server.common.timewheel.TimerFuture; import tech.powerjob.server.common.timewheel.TimerTask; import com.google.common.collect.Maps; @@ -19,9 +20,9 @@ public class InstanceTimeWheelService { private static final Map CARGO = Maps.newConcurrentMap(); // 精确调度时间轮，每 1MS 走一格 - private static final HashedWheelTimer TIMER = new HashedWheelTimer(1, 4096, Runtime.getRuntime().availableProcessors() * 4); + private static final Timer TIMER = new HashedWheelTimer(1, 4096, Runtime.getRuntime().availableProcessors() * 4); // 非精确调度时间轮，用于处理高延迟任务，每 10S 走一格 - private static final HashedWheelTimer SLOW_TIMER = new HashedWheelTimer(10000, 12, 0); + private static final Timer SLOW_TIMER = new HashedWheelTimer(10000, 12, 0); // 支持取消的时间间隔，低于该阈值则不会放进 CARGO private static final long MIN_INTERVAL_MS = 1000; diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/AbWorkerRequestHandler.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/AbWorkerRequestHandler.java index 81b61758..2a240912 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/AbWorkerRequestHandler.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/AbWorkerRequestHandler.java @@ -94,7 +94,6 @@ public abstract class AbWorkerRequestHandler implements IWorkerRequestHandler { @Override public void processWorkerLogReport(WorkerLogReportReq req) { - long startMs = System.currentTimeMillis(); WorkerLogReportEvent event = new WorkerLogReportEvent() .setWorkerAddress(req.getWorkerAddress()); try { @@ -106,7 +105,6 @@ public abstract class AbWorkerRequestHandler implements IWorkerRequestHandler { event.setStatus(WorkerLogReportEvent.Status.EXCEPTION); log.warn("[WorkerRequestHandler] process worker report failed!", t); } finally { - event.setServerCost(System.currentTimeMillis() - startMs); monitorService.monitor(event); } } diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceLogService.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceLogService.java index f24f776c..4ffc21d0 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceLogService.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceLogService.java @@ -51,6 +51,9 @@ public class InstanceLogService { @Value("${server.port}") private int port; + @Resource + private Executor omsLocalDbPool; + @Resource private InstanceMetadataService instanceMetadataService; @Resource @@ -99,20 +102,22 @@ public class InstanceLogService { */ public void submitLogs(String workerAddress, List logs) { - List logList = logs.stream().map(x -> { - instanceId2LastReportTime.put(x.getInstanceId(), System.currentTimeMillis()); + omsLocalDbPool.execute(() -> { + List logList = logs.stream().map(x -> { + instanceId2LastReportTime.put(x.getInstanceId(), System.currentTimeMillis()); - LocalInstanceLogDO y = new LocalInstanceLogDO(); - BeanUtils.copyProperties(x, y); - y.setWorkerAddress(workerAddress); - return y; - }).collect(Collectors.toList()); + LocalInstanceLogDO y = new LocalInstanceLogDO(); + BeanUtils.copyProperties(x, y); + y.setWorkerAddress(workerAddress); + return y; + }).collect(Collectors.toList()); - try { - CommonUtils.executeWithRetry0(() -> localInstanceLogRepository.saveAll(logList)); - }catch (Exception e) { - log.warn("[InstanceLogService] persistent instance logs failed, these logs will be dropped: {}.", logs, e); - } + try { + CommonUtils.executeWithRetry0(() -> localInstanceLogRepository.saveAll(logList)); + }catch (Exception e) { + log.warn("[InstanceLogService] persistent instance logs failed, these logs will be dropped: {}.", logs, e); + } + }); } /** diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceManager.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceManager.java index a4751c0b..351fbf9f 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceManager.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceManager.java @@ -191,7 +191,7 @@ public class InstanceManager { log.info("[Instance-{}] process finished, final status is {}.", instanceId, status.name()); // 上报日志数据 - HashedWheelTimerHolder.INACCURATE_TIMER.schedule(() -> instanceLogService.sync(instanceId), 15, TimeUnit.SECONDS); + HashedWheelTimerHolder.INACCURATE_TIMER.schedule(() -> instanceLogService.sync(instanceId), 60, TimeUnit.SECONDS); // workflow 特殊处理 if (wfInstanceId != null) { diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/w2s/WorkerLogReportEvent.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/w2s/WorkerLogReportEvent.java index 89bec99d..37c8a59d 100644 --- a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/w2s/WorkerLogReportEvent.java +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/w2s/WorkerLogReportEvent.java @@ -22,8 +22,6 @@ public class WorkerLogReportEvent implements Event { private Status status; - private long serverCost; - public enum Status { SUCCESS, REJECTED, @@ -37,6 +35,6 @@ public class WorkerLogReportEvent implements Event { @Override public String message() { - return SJ.MONITOR_JOINER.join(workerAddress, logSize, status, serverCost); + return SJ.MONITOR_JOINER.join(workerAddress, logSize, status); } } diff --git a/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/config/ThreadPoolConfig.java b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/config/ThreadPoolConfig.java index ba97cae9..75949b90 100644 --- a/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/config/ThreadPoolConfig.java +++ b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/config/ThreadPoolConfig.java @@ -34,7 +34,7 @@ public class ThreadPoolConfig { executor.setQueueCapacity(0); executor.setKeepAliveSeconds(60); executor.setThreadNamePrefix("omsTimingPool-"); - executor.setRejectedExecutionHandler(RejectedExecutionHandlerFactory.newThreadRun("PowerJobTiming")); + executor.setRejectedExecutionHandler(RejectedExecutionHandlerFactory.newThreadRun("PJ-TIMING")); return executor; } @@ -46,7 +46,18 @@ public class ThreadPoolConfig { executor.setQueueCapacity(8192); executor.setKeepAliveSeconds(60); executor.setThreadNamePrefix("omsBackgroundPool-"); - executor.setRejectedExecutionHandler(RejectedExecutionHandlerFactory.newDiscard("PowerJobBackgroundPool")); + executor.setRejectedExecutionHandler(RejectedExecutionHandlerFactory.newDiscard("PJ-BACKGROUND")); + return executor; + } + + @Bean("omsLocalDbPool") + public Executor initOmsLocalDbPool() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(Runtime.getRuntime().availableProcessors() / 2); + executor.setMaxPoolSize(Runtime.getRuntime().availableProcessors() / 2); + executor.setQueueCapacity(2048); + executor.setThreadNamePrefix("omsLocalDbPool-"); + executor.setRejectedExecutionHandler(RejectedExecutionHandlerFactory.newAbort("PJ-LOCAL-DB")); return executor; } @@ -55,7 +66,7 @@ public class ThreadPoolConfig { public TaskScheduler taskScheduler() { ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler(); scheduler.setPoolSize(Runtime.getRuntime().availableProcessors()); - scheduler.setThreadNamePrefix("PowerJobSchedulePool-"); + scheduler.setThreadNamePrefix("PJ-WS-"); scheduler.setDaemon(true); return scheduler; } From d531bf3a22ad90cf8941a2fc7f86a3fa783e33e7 Mon Sep 17 00:00:00 2001 From: tjq Date: Mon, 12 Sep 2022 11:07:05 +0800 Subject: [PATCH 19/30] feat: optimize threadpool config --- .../server/common/constants/PJThreadPool.java | 26 ++++++++++++++ .../core/instance/InstanceLogService.java | 35 +++++++++---------- .../server/core/scheduler/CleanService.java | 3 +- .../scheduler/InstanceStatusCheckService.java | 3 +- .../core/scheduler/PowerScheduleService.java | 3 +- .../server/config/ThreadPoolConfig.java | 23 ++++++------ 6 files changed, 59 insertions(+), 34 deletions(-) create mode 100644 powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/constants/PJThreadPool.java diff --git a/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/constants/PJThreadPool.java b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/constants/PJThreadPool.java new file mode 100644 index 00000000..ef8b2b06 --- /dev/null +++ b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/constants/PJThreadPool.java @@ -0,0 +1,26 @@ +package tech.powerjob.server.common.constants; + +/** + * 线程池 + * + * @author tjq + * @since 2022/9/12 + */ +public class PJThreadPool { + + /** + * 定时调度用线程池 + */ + public static final String TIMING_POOL = "PowerJobTimingPool"; + + /** + * 后台任务异步线程池 + */ + public static final String BACKGROUND_POOL = "PowerJobBackgroundPool"; + + /** + * 本地数据库专用线程池 + */ + public static final String LOCAL_DB_POOL = "PowerJobLocalDbPool"; + +} diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceLogService.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceLogService.java index 4ffc21d0..dea809b4 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceLogService.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceLogService.java @@ -7,6 +7,7 @@ import tech.powerjob.common.model.InstanceLogContent; import tech.powerjob.common.utils.CommonUtils; import tech.powerjob.common.utils.NetUtils; import tech.powerjob.common.utils.SegmentLock; +import tech.powerjob.server.common.constants.PJThreadPool; import tech.powerjob.server.remote.server.redirector.DesignateServer; import tech.powerjob.server.common.utils.OmsFileUtils; import tech.powerjob.server.persistence.StringPage; @@ -51,9 +52,6 @@ public class InstanceLogService { @Value("${server.port}") private int port; - @Resource - private Executor omsLocalDbPool; - @Resource private InstanceMetadataService instanceMetadataService; @Resource @@ -100,24 +98,23 @@ public class InstanceLogService { * @param workerAddress 上报机器地址 * @param logs 任务实例运行时日志 */ + @Async(value = PJThreadPool.LOCAL_DB_POOL) public void submitLogs(String workerAddress, List logs) { - omsLocalDbPool.execute(() -> { - List logList = logs.stream().map(x -> { - instanceId2LastReportTime.put(x.getInstanceId(), System.currentTimeMillis()); + List logList = logs.stream().map(x -> { + instanceId2LastReportTime.put(x.getInstanceId(), System.currentTimeMillis()); - LocalInstanceLogDO y = new LocalInstanceLogDO(); - BeanUtils.copyProperties(x, y); - y.setWorkerAddress(workerAddress); - return y; - }).collect(Collectors.toList()); + LocalInstanceLogDO y = new LocalInstanceLogDO(); + BeanUtils.copyProperties(x, y); + y.setWorkerAddress(workerAddress); + return y; + }).collect(Collectors.toList()); - try { - CommonUtils.executeWithRetry0(() -> localInstanceLogRepository.saveAll(logList)); - }catch (Exception e) { - log.warn("[InstanceLogService] persistent instance logs failed, these logs will be dropped: {}.", logs, e); - } - }); + try { + CommonUtils.executeWithRetry0(() -> localInstanceLogRepository.saveAll(logList)); + }catch (Exception e) { + log.warn("[InstanceLogService] persistent instance logs failed, these logs will be dropped: {}.", logs, e); + } } /** @@ -208,7 +205,7 @@ public class InstanceLogService { * 将本地的任务实例运行日志同步到 mongoDB 存储，在任务执行结束后异步执行 * @param instanceId 任务实例ID */ - @Async("omsBackgroundPool") + @Async(PJThreadPool.BACKGROUND_POOL) public void sync(Long instanceId) { Stopwatch sw = Stopwatch.createStarted(); @@ -350,7 +347,7 @@ public class InstanceLogService { } - @Async("omsTimingPool") + @Async(PJThreadPool.TIMING_POOL) @Scheduled(fixedDelay = 120000) public void timingCheck() { diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/scheduler/CleanService.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/scheduler/CleanService.java index 03ba4ffb..d9c9e137 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/scheduler/CleanService.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/scheduler/CleanService.java @@ -2,6 +2,7 @@ package tech.powerjob.server.core.scheduler; import tech.powerjob.common.enums.InstanceStatus; import tech.powerjob.common.enums.WorkflowInstanceStatus; +import tech.powerjob.server.common.constants.PJThreadPool; import tech.powerjob.server.common.utils.OmsFileUtils; import tech.powerjob.server.persistence.remote.repository.InstanceInfoRepository; import tech.powerjob.server.persistence.remote.repository.WorkflowInstanceInfoRepository; @@ -62,7 +63,7 @@ public class CleanService { private static final String HISTORY_DELETE_LOCK = "history_delete_lock"; - @Async("omsTimingPool") + @Async(PJThreadPool.TIMING_POOL) @Scheduled(cron = CLEAN_TIME_EXPRESSION) public void timingClean() { diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/scheduler/InstanceStatusCheckService.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/scheduler/InstanceStatusCheckService.java index 4ca3e327..b324923e 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/scheduler/InstanceStatusCheckService.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/scheduler/InstanceStatusCheckService.java @@ -4,6 +4,7 @@ import tech.powerjob.common.enums.InstanceStatus; import tech.powerjob.common.SystemInstanceResult; import tech.powerjob.common.enums.TimeExpressionType; import tech.powerjob.common.enums.WorkflowInstanceStatus; +import tech.powerjob.server.common.constants.PJThreadPool; import tech.powerjob.server.common.constants.SwitchableStatus; import tech.powerjob.server.remote.transport.starter.AkkaStarter; import tech.powerjob.server.persistence.remote.model.*; @@ -59,7 +60,7 @@ public class InstanceStatusCheckService { @Resource private WorkflowInstanceInfoRepository workflowInstanceInfoRepository; - @Async("omsTimingPool") + @Async(PJThreadPool.TIMING_POOL) @Scheduled(fixedDelay = 10000) public void timingStatusCheck() { Stopwatch stopwatch = Stopwatch.createStarted(); diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/scheduler/PowerScheduleService.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/scheduler/PowerScheduleService.java index 7e46c452..862acf8e 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/scheduler/PowerScheduleService.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/scheduler/PowerScheduleService.java @@ -3,6 +3,7 @@ package tech.powerjob.server.core.scheduler; import tech.powerjob.common.enums.InstanceStatus; import tech.powerjob.common.enums.TimeExpressionType; import tech.powerjob.common.model.LifeCycle; +import tech.powerjob.server.common.constants.PJThreadPool; import tech.powerjob.server.remote.transport.starter.AkkaStarter; import tech.powerjob.server.common.constants.SwitchableStatus; import tech.powerjob.server.persistence.remote.model.AppInfoDO; @@ -73,7 +74,7 @@ public class PowerScheduleService { private static final long SCHEDULE_RATE = 15000; - @Async("omsTimingPool") + @Async(PJThreadPool.TIMING_POOL) @Scheduled(fixedDelay = SCHEDULE_RATE) public void timingSchedule() { diff --git a/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/config/ThreadPoolConfig.java b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/config/ThreadPoolConfig.java index 75949b90..f3ce629e 100644 --- a/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/config/ThreadPoolConfig.java +++ b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/config/ThreadPoolConfig.java @@ -1,5 +1,6 @@ package tech.powerjob.server.config; +import org.springframework.core.task.TaskExecutor; import tech.powerjob.server.common.RejectedExecutionHandlerFactory; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Bean; @@ -8,14 +9,12 @@ import org.springframework.scheduling.TaskScheduler; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; +import tech.powerjob.server.common.constants.PJThreadPool; import java.util.concurrent.*; /** * 公用线程池配置 - * omsTimingPool：用于执行定时任务的线程池 - * omsBackgroundPool：用于执行后台任务的线程池，这类任务对时间不敏感，慢慢执行细水长流即可 - * taskScheduler：用于定时调度的线程池 * * @author tjq * @since 2020/4/28 @@ -25,7 +24,7 @@ import java.util.concurrent.*; @Configuration public class ThreadPoolConfig { - @Bean("omsTimingPool") + @Bean(PJThreadPool.TIMING_POOL) public Executor getTimingPool() { ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); executor.setCorePoolSize(Runtime.getRuntime().availableProcessors()); @@ -33,31 +32,31 @@ public class ThreadPoolConfig { // use SynchronousQueue executor.setQueueCapacity(0); executor.setKeepAliveSeconds(60); - executor.setThreadNamePrefix("omsTimingPool-"); - executor.setRejectedExecutionHandler(RejectedExecutionHandlerFactory.newThreadRun("PJ-TIMING")); + executor.setThreadNamePrefix("PJ-TIMING-"); + executor.setRejectedExecutionHandler(RejectedExecutionHandlerFactory.newThreadRun(PJThreadPool.TIMING_POOL)); return executor; } - @Bean("omsBackgroundPool") + @Bean(PJThreadPool.BACKGROUND_POOL) public Executor initBackgroundPool() { ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); executor.setCorePoolSize(Runtime.getRuntime().availableProcessors() * 8); executor.setMaxPoolSize(Runtime.getRuntime().availableProcessors() * 16); executor.setQueueCapacity(8192); executor.setKeepAliveSeconds(60); - executor.setThreadNamePrefix("omsBackgroundPool-"); - executor.setRejectedExecutionHandler(RejectedExecutionHandlerFactory.newDiscard("PJ-BACKGROUND")); + executor.setThreadNamePrefix("PJ-BG-"); + executor.setRejectedExecutionHandler(RejectedExecutionHandlerFactory.newDiscard(PJThreadPool.BACKGROUND_POOL)); return executor; } - @Bean("omsLocalDbPool") + @Bean(PJThreadPool.LOCAL_DB_POOL) public Executor initOmsLocalDbPool() { ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); executor.setCorePoolSize(Runtime.getRuntime().availableProcessors() / 2); executor.setMaxPoolSize(Runtime.getRuntime().availableProcessors() / 2); executor.setQueueCapacity(2048); - executor.setThreadNamePrefix("omsLocalDbPool-"); - executor.setRejectedExecutionHandler(RejectedExecutionHandlerFactory.newAbort("PJ-LOCAL-DB")); + executor.setThreadNamePrefix("PJ-LOCALDB-"); + executor.setRejectedExecutionHandler(RejectedExecutionHandlerFactory.newAbort(PJThreadPool.LOCAL_DB_POOL)); return executor; } From 2db0f05feb20c765a3db1007a766e9bf61a84971 Mon Sep 17 00:00:00 2001 From: tjq Date: Mon, 12 Sep 2022 11:33:13 +0800 Subject: [PATCH 20/30] feat: optimize thread pool config --- .../server/core/instance/InstanceLogService.java | 13 ++++++------- .../powerjob/server/config/ThreadPoolConfig.java | 7 ++++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceLogService.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceLogService.java index dea809b4..9f75587a 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceLogService.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/instance/InstanceLogService.java @@ -1,5 +1,7 @@ package tech.powerjob.server.core.instance; +import org.springframework.core.task.AsyncTaskExecutor; +import org.springframework.core.task.TaskExecutor; import tech.powerjob.common.enums.LogLevel; import tech.powerjob.common.OmsConstant; import tech.powerjob.common.enums.TimeExpressionType; @@ -68,7 +70,9 @@ public class InstanceLogService { * 本地维护了在线日志的任务实例ID */ private final Map instanceId2LastReportTime = Maps.newConcurrentMap(); - private final ExecutorService workerPool; + + @Resource(name = PJThreadPool.BACKGROUND_POOL) + private AsyncTaskExecutor powerJobBackgroundPool; /** * 分段锁 @@ -88,11 +92,6 @@ public class InstanceLogService { */ private static final long EXPIRE_INTERVAL_MS = 60000; - public InstanceLogService() { - int coreSize = Runtime.getRuntime().availableProcessors(); - workerPool = new ThreadPoolExecutor(coreSize, coreSize, 1, TimeUnit.MINUTES, Queues.newLinkedBlockingQueue()); - } - /** * 提交日志记录，持久化到本地数据库中 * @param workerAddress 上报机器地址 @@ -192,7 +191,7 @@ public class InstanceLogService { * @return 异步结果 */ private Future prepareLogFile(long instanceId) { - return workerPool.submit(() -> { + return powerJobBackgroundPool.submit(() -> { // 在线日志还在不断更新，需要使用本地数据库中的数据 if (instanceId2LastReportTime.containsKey(instanceId)) { return genTemporaryLogFile(instanceId); diff --git a/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/config/ThreadPoolConfig.java b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/config/ThreadPoolConfig.java index f3ce629e..99b61ade 100644 --- a/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/config/ThreadPoolConfig.java +++ b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/config/ThreadPoolConfig.java @@ -1,5 +1,6 @@ package tech.powerjob.server.config; +import org.springframework.core.task.AsyncTaskExecutor; import org.springframework.core.task.TaskExecutor; import tech.powerjob.server.common.RejectedExecutionHandlerFactory; import lombok.extern.slf4j.Slf4j; @@ -25,7 +26,7 @@ import java.util.concurrent.*; public class ThreadPoolConfig { @Bean(PJThreadPool.TIMING_POOL) - public Executor getTimingPool() { + public TaskExecutor getTimingPool() { ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); executor.setCorePoolSize(Runtime.getRuntime().availableProcessors()); executor.setMaxPoolSize(Runtime.getRuntime().availableProcessors() * 4); @@ -38,7 +39,7 @@ public class ThreadPoolConfig { } @Bean(PJThreadPool.BACKGROUND_POOL) - public Executor initBackgroundPool() { + public AsyncTaskExecutor initBackgroundPool() { ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); executor.setCorePoolSize(Runtime.getRuntime().availableProcessors() * 8); executor.setMaxPoolSize(Runtime.getRuntime().availableProcessors() * 16); @@ -50,7 +51,7 @@ public class ThreadPoolConfig { } @Bean(PJThreadPool.LOCAL_DB_POOL) - public Executor initOmsLocalDbPool() { + public TaskExecutor initOmsLocalDbPool() { ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); executor.setCorePoolSize(Runtime.getRuntime().availableProcessors() / 2); executor.setMaxPoolSize(Runtime.getRuntime().availableProcessors() / 2); From 5450ac00dba34900076b123a26b6c623bc666bac Mon Sep 17 00:00:00 2001 From: tjq Date: Mon, 12 Sep 2022 11:58:56 +0800 Subject: [PATCH 21/30] feat: optimize WorkerLogReportEvent --- .../server/core/handler/AbWorkerRequestHandler.java | 3 ++- .../monitor/events/w2s/WorkerLogReportEvent.java | 10 +++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/AbWorkerRequestHandler.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/AbWorkerRequestHandler.java index 2a240912..9ef62dae 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/AbWorkerRequestHandler.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/AbWorkerRequestHandler.java @@ -95,7 +95,8 @@ public abstract class AbWorkerRequestHandler implements IWorkerRequestHandler { public void processWorkerLogReport(WorkerLogReportReq req) { WorkerLogReportEvent event = new WorkerLogReportEvent() - .setWorkerAddress(req.getWorkerAddress()); + .setWorkerAddress(req.getWorkerAddress()) + .setLogNum(req.getInstanceLogContents().size()); try { processWorkerLogReport0(req, event); event.setStatus(WorkerLogReportEvent.Status.SUCCESS); diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/w2s/WorkerLogReportEvent.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/w2s/WorkerLogReportEvent.java index 37c8a59d..fe12778c 100644 --- a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/w2s/WorkerLogReportEvent.java +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/w2s/WorkerLogReportEvent.java @@ -18,6 +18,14 @@ public class WorkerLogReportEvent implements Event { private String workerAddress; + /** + * 日志条数 + */ + private long logNum; + + /** + * 日志大小，用于统计 IO 压力 + */ private long logSize; private Status status; @@ -35,6 +43,6 @@ public class WorkerLogReportEvent implements Event { @Override public String message() { - return SJ.MONITOR_JOINER.join(workerAddress, logSize, status); + return SJ.MONITOR_JOINER.join(workerAddress, logNum, logSize, status); } } From 74f70cd58b59ad37d8a6ba894f466a805bf61bca Mon Sep 17 00:00:00 2001 From: tjq Date: Mon, 12 Sep 2022 12:56:12 +0800 Subject: [PATCH 22/30] feat: support serverInfoAware --- .../server/common/aware/PowerJobAware.java | 10 ++++ .../server/common/aware/ServerInfoAware.java | 14 ++++++ .../server/common/module/ServerInfo.java | 19 ++++++++ .../server/core/uid/IdGenerateService.java | 4 +- .../tech/powerjob/server/monitor/Monitor.java | 3 +- .../server/monitor/MonitorContext.java | 24 ---------- .../monitor}/PowerJobMonitorService.java | 15 +----- .../server/monitor/monitors/LogMonitor.java | 23 ++++++--- .../remote/server/self/ServerInfoService.java | 15 ++++++ .../ServerInfoServiceImpl.java} | 47 +++++++++---------- .../powerjob/server/config/SwaggerConfig.java | 4 +- .../support/ServerInfoAwareProcessor.java | 29 ++++++++++++ .../web/controller/SystemInfoController.java | 6 +-- .../server/web/response/SystemOverviewVO.java | 11 +---- 14 files changed, 135 insertions(+), 89 deletions(-) create mode 100644 powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/aware/PowerJobAware.java create mode 100644 powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/aware/ServerInfoAware.java create mode 100644 powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/module/ServerInfo.java delete mode 100644 powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/MonitorContext.java rename powerjob-server/{powerjob-server-starter/src/main/java/tech/powerjob/server/initializer => powerjob-server-monitor/src/tech/powerjob/server/monitor}/PowerJobMonitorService.java (55%) create mode 100644 powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/self/ServerInfoService.java rename powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/{ServerInfoService.java => self/ServerInfoServiceImpl.java} (82%) create mode 100644 powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/support/ServerInfoAwareProcessor.java diff --git a/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/aware/PowerJobAware.java b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/aware/PowerJobAware.java new file mode 100644 index 00000000..fc124fa9 --- /dev/null +++ b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/aware/PowerJobAware.java @@ -0,0 +1,10 @@ +package tech.powerjob.server.common.aware; + +/** + * PowerJobAware + * + * @author tjq + * @since 2022/9/12 + */ +public interface PowerJobAware { +} diff --git a/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/aware/ServerInfoAware.java b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/aware/ServerInfoAware.java new file mode 100644 index 00000000..57706f4b --- /dev/null +++ b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/aware/ServerInfoAware.java @@ -0,0 +1,14 @@ +package tech.powerjob.server.common.aware; + +import tech.powerjob.server.common.module.ServerInfo; + +/** + * notify server info + * + * @author tjq + * @since 2022/9/12 + */ +public interface ServerInfoAware extends PowerJobAware { + + void setServerInfo(ServerInfo serverInfo); +} diff --git a/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/module/ServerInfo.java b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/module/ServerInfo.java new file mode 100644 index 00000000..d2c004fa --- /dev/null +++ b/powerjob-server/powerjob-server-common/src/main/java/tech/powerjob/server/common/module/ServerInfo.java @@ -0,0 +1,19 @@ +package tech.powerjob.server.common.module; + +import lombok.Data; + +/** + * current server info + * + * @author tjq + * @since 2022/9/12 + */ +@Data +public class ServerInfo { + + private Long id; + + private String ip; + + private String version = "UNKNOWN"; +} diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/uid/IdGenerateService.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/uid/IdGenerateService.java index eae67869..2f42e087 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/uid/IdGenerateService.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/uid/IdGenerateService.java @@ -1,9 +1,9 @@ package tech.powerjob.server.core.uid; -import tech.powerjob.server.remote.server.ServerInfoService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import tech.powerjob.server.remote.server.self.ServerInfoService; /** * 唯一ID生成服务，使用 Twitter snowflake 算法 @@ -23,7 +23,7 @@ public class IdGenerateService { @Autowired public IdGenerateService(ServerInfoService serverInfoService) { - long id = serverInfoService.getServerId(); + long id = serverInfoService.fetchServiceInfo().getId(); snowFlakeIdGenerator = new SnowFlakeIdGenerator(DATA_CENTER_ID, id); log.info("[IdGenerateService] initialize IdGenerateService successfully, ID:{}", id); } diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/Monitor.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/Monitor.java index e53f7f5d..6df7030d 100644 --- a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/Monitor.java +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/Monitor.java @@ -10,9 +10,8 @@ public interface Monitor { /** * 全局上下文绑定 & 初始化 - * @param monitorContext 日志上下文 */ - void init(MonitorContext monitorContext); + void init(); /** * 记录监控事件 * 请注意该方法务必异步不阻塞！！！ diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/MonitorContext.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/MonitorContext.java deleted file mode 100644 index 63549192..00000000 --- a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/MonitorContext.java +++ /dev/null @@ -1,24 +0,0 @@ -package tech.powerjob.server.monitor; - -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; -import lombok.experimental.Accessors; - -import java.io.Serializable; - -/** - * 日志全局上下文 - * - * @author tjq - * @since 2022/9/10 - */ -@Getter -@Setter -@ToString -@Accessors(chain = true) -public class MonitorContext implements Serializable { - private long serverId; - private String serverAddress; - -} diff --git a/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/initializer/PowerJobMonitorService.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/PowerJobMonitorService.java similarity index 55% rename from powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/initializer/PowerJobMonitorService.java rename to powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/PowerJobMonitorService.java index 8cebd969..90caaa5e 100644 --- a/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/initializer/PowerJobMonitorService.java +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/PowerJobMonitorService.java @@ -1,16 +1,10 @@ -package tech.powerjob.server.initializer; +package tech.powerjob.server.monitor; import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import tech.powerjob.server.monitor.Event; -import tech.powerjob.server.monitor.Monitor; -import tech.powerjob.server.monitor.MonitorContext; -import tech.powerjob.server.monitor.MonitorService; -import tech.powerjob.server.remote.server.ServerInfoService; -import javax.annotation.Resource; import java.util.List; /** @@ -23,20 +17,13 @@ import java.util.List; @Component public class PowerJobMonitorService implements MonitorService { - @Resource - private ServerInfoService service; - private final List monitors = Lists.newLinkedList(); @Autowired public PowerJobMonitorService(List monitors) { - MonitorContext monitorContext = new MonitorContext().setServerId(service.getServerId()).setServerAddress(service.getServerIp()); - log.info("[MonitorService] use monitor context: {}", monitorContext); - monitors.forEach(m -> { log.info("[MonitorService] register monitor: {}", m.getClass().getName()); - m.init(monitorContext); this.monitors.add(m); }); } diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/LogMonitor.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/LogMonitor.java index 8ffed245..45f598b7 100644 --- a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/LogMonitor.java +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/LogMonitor.java @@ -3,9 +3,10 @@ package tech.powerjob.server.monitor.monitors; import org.slf4j.LoggerFactory; import org.slf4j.MDC; import org.springframework.stereotype.Component; +import tech.powerjob.server.common.aware.ServerInfoAware; +import tech.powerjob.server.common.module.ServerInfo; import tech.powerjob.server.monitor.Event; import tech.powerjob.server.monitor.Monitor; -import tech.powerjob.server.monitor.MonitorContext; /** * 系统默认实现——基于日志的监控监视器 @@ -15,22 +16,30 @@ import tech.powerjob.server.monitor.MonitorContext; * @since 2022/9/6 */ @Component -public class LogMonitor implements Monitor { +public class LogMonitor implements Monitor, ServerInfoAware { - private MonitorContext monitorContext; + /** + * server 启动依赖 DB，DB会被 monitor，因此最初的几条 log serverInfo 一定为空，在此处简单防空 + */ + private ServerInfo serverInfo = new ServerInfo(); private static final String MDC_KEY_SERVER_ID = "serverId"; private static final String MDC_KEY_SERVER_ADDRESS = "serverAddress"; + @Override - public void init(MonitorContext monitorContext) { - this.monitorContext = monitorContext; + public void init() { } @Override public void record(Event event) { - MDC.put(MDC_KEY_SERVER_ID, String.valueOf(monitorContext.getServerId())); - MDC.put(MDC_KEY_SERVER_ADDRESS, monitorContext.getServerAddress()); + MDC.put(MDC_KEY_SERVER_ID, String.valueOf(serverInfo.getId())); + MDC.put(MDC_KEY_SERVER_ADDRESS, serverInfo.getIp()); LoggerFactory.getLogger(event.type()).info(event.message()); } + + @Override + public void setServerInfo(ServerInfo serverInfo) { + this.serverInfo = serverInfo; + } } diff --git a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/self/ServerInfoService.java b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/self/ServerInfoService.java new file mode 100644 index 00000000..a01b8c0b --- /dev/null +++ b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/self/ServerInfoService.java @@ -0,0 +1,15 @@ +package tech.powerjob.server.remote.server.self; + +import tech.powerjob.server.common.module.ServerInfo; + +/** + * ServerInfoService + * + * @author tjq + * @since 2022/9/12 + */ +public interface ServerInfoService { + + ServerInfo fetchServiceInfo(); + +} diff --git a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/ServerInfoService.java b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/self/ServerInfoServiceImpl.java similarity index 82% rename from powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/ServerInfoService.java rename to powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/self/ServerInfoServiceImpl.java index 6b83a007..5159da83 100644 --- a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/ServerInfoService.java +++ b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/self/ServerInfoServiceImpl.java @@ -1,10 +1,11 @@ -package tech.powerjob.server.remote.server; +package tech.powerjob.server.remote.server.self; import org.apache.commons.lang3.StringUtils; import org.springframework.boot.info.BuildProperties; import tech.powerjob.common.exception.PowerJobException; import tech.powerjob.common.utils.CommonUtils; import tech.powerjob.common.utils.NetUtils; +import tech.powerjob.server.common.module.ServerInfo; import tech.powerjob.server.extension.LockService; import tech.powerjob.server.persistence.remote.model.ServerInfoDO; import tech.powerjob.server.persistence.remote.repository.ServerInfoRepository; @@ -28,37 +29,25 @@ import java.util.stream.Collectors; */ @Slf4j @Service -public class ServerInfoService { +public class ServerInfoServiceImpl implements ServerInfoService { - private final String ip; - private final long serverId; + private final ServerInfo serverInfo; private final ServerInfoRepository serverInfoRepository; - private String version = "UNKNOWN"; - private static final long MAX_SERVER_CLUSTER_SIZE = 10000; private static final String SERVER_INIT_LOCK = "server_init_lock"; private static final int SERVER_INIT_LOCK_MAX_TIME = 15000; - public long getServerId() { - return serverId; - } - - public String getServerIp() { - return ip; - } - - public String getServerVersion() { - return version; - } - @Autowired - public ServerInfoService(LockService lockService, ServerInfoRepository serverInfoRepository) { + public ServerInfoServiceImpl(LockService lockService, ServerInfoRepository serverInfoRepository) { - this.ip = NetUtils.getLocalHost(); + this.serverInfo = new ServerInfo(); + + String ip = NetUtils.getLocalHost(); + serverInfo.setIp(ip); this.serverInfoRepository = serverInfoRepository; Stopwatch sw = Stopwatch.createStarted(); @@ -80,10 +69,11 @@ public class ServerInfoService { } if (server.getId() < MAX_SERVER_CLUSTER_SIZE) { - this.serverId = server.getId(); + serverInfo.setId(server.getId()); } else { - this.serverId = retryServerId(); - serverInfoRepository.updateIdByIp(this.serverId, ip); + long retryServerId = retryServerId(); + serverInfo.setId(retryServerId); + serverInfoRepository.updateIdByIp(retryServerId, ip); } } catch (Exception e) { @@ -93,12 +83,12 @@ public class ServerInfoService { lockService.unlock(SERVER_INIT_LOCK); } - log.info("[ServerInfoService] ip:{}, id:{}, cost:{}", ip, serverId, sw); + log.info("[ServerInfoService] ip:{}, id:{}, cost:{}", ip, serverInfo.getId(), sw); } @Scheduled(fixedRate = 15000, initialDelay = 15000) public void heartbeat() { - serverInfoRepository.updateGmtModifiedByIp(ip, new Date()); + serverInfoRepository.updateGmtModifiedByIp(serverInfo.getIp(), new Date()); } @@ -142,7 +132,12 @@ public class ServerInfoService { } String pomVersion = buildProperties.getVersion(); if (StringUtils.isNotBlank(pomVersion)) { - version = pomVersion; + serverInfo.setVersion(pomVersion); } } + + @Override + public ServerInfo fetchServiceInfo() { + return serverInfo; + } } diff --git a/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/config/SwaggerConfig.java b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/config/SwaggerConfig.java index 77bb4310..fac926c5 100644 --- a/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/config/SwaggerConfig.java +++ b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/config/SwaggerConfig.java @@ -9,7 +9,7 @@ import springfox.documentation.spi.DocumentationType; import springfox.documentation.spring.web.plugins.Docket; import springfox.documentation.swagger2.annotations.EnableSwagger2; import tech.powerjob.server.common.PowerJobServerConfigKey; -import tech.powerjob.server.remote.server.ServerInfoService; +import tech.powerjob.server.remote.server.self.ServerInfoService; import javax.annotation.Resource; @@ -39,7 +39,7 @@ public class SwaggerConfig { .description("Distributed scheduling and computing framework.") .license("Apache Licence 2") .termsOfServiceUrl("https://github.com/PowerJob/PowerJob") - .version(serverInfoService.getServerVersion()) + .version(serverInfoService.fetchServiceInfo().getVersion()) .build(); return new Docket(DocumentationType.SWAGGER_2) diff --git a/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/support/ServerInfoAwareProcessor.java b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/support/ServerInfoAwareProcessor.java new file mode 100644 index 00000000..6a4aa54a --- /dev/null +++ b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/support/ServerInfoAwareProcessor.java @@ -0,0 +1,29 @@ +package tech.powerjob.server.support; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import tech.powerjob.server.common.aware.ServerInfoAware; +import tech.powerjob.server.common.module.ServerInfo; +import tech.powerjob.server.remote.server.self.ServerInfoService; + +import java.util.List; + +/** + * ServerInfoAwareProcessor + * + * @author tjq + * @since 2022/9/12 + */ +@Slf4j +@Component +public class ServerInfoAwareProcessor { + + public ServerInfoAwareProcessor(ServerInfoService serverInfoService, List awareList) { + final ServerInfo serverInfo = serverInfoService.fetchServiceInfo(); + log.info("[ServerInfoAwareProcessor] current server info: {}", serverInfo); + awareList.forEach(aware -> { + aware.setServerInfo(serverInfo); + log.info("[ServerInfoAwareProcessor] set ServerInfo for: {} successfully", aware); + }); + } +} diff --git a/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/web/controller/SystemInfoController.java b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/web/controller/SystemInfoController.java index 6d747129..9dda6ea2 100644 --- a/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/web/controller/SystemInfoController.java +++ b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/web/controller/SystemInfoController.java @@ -4,9 +4,10 @@ import tech.powerjob.common.enums.InstanceStatus; import tech.powerjob.common.OmsConstant; import tech.powerjob.common.response.ResultDTO; import tech.powerjob.server.common.constants.SwitchableStatus; +import tech.powerjob.server.common.module.ServerInfo; import tech.powerjob.server.persistence.remote.repository.InstanceInfoRepository; import tech.powerjob.server.persistence.remote.repository.JobInfoRepository; -import tech.powerjob.server.remote.server.ServerInfoService; +import tech.powerjob.server.remote.server.self.ServerInfoService; import tech.powerjob.server.remote.worker.WorkerClusterQueryService; import tech.powerjob.server.common.module.WorkerInfo; import tech.powerjob.server.web.response.SystemOverviewVO; @@ -70,8 +71,7 @@ public class SystemInfoController { // 服务器时间 overview.setServerTime(DateFormatUtils.format(new Date(), OmsConstant.TIME_PATTERN)); - SystemOverviewVO.CurrentServerInfo info = new SystemOverviewVO.CurrentServerInfo(serverInfoService.getServerId(), serverInfoService.getServerIp(), serverInfoService.getServerVersion()); - overview.setCurrentServerInfo(info); + overview.setServerInfo(serverInfoService.fetchServiceInfo()); return ResultDTO.success(overview); } diff --git a/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/web/response/SystemOverviewVO.java b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/web/response/SystemOverviewVO.java index 3bc86786..c7b94500 100644 --- a/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/web/response/SystemOverviewVO.java +++ b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/web/response/SystemOverviewVO.java @@ -3,6 +3,7 @@ package tech.powerjob.server.web.response; import lombok.AllArgsConstructor; import lombok.Data; import lombok.Getter; +import tech.powerjob.server.common.module.ServerInfo; /** * 系统概览 @@ -21,13 +22,5 @@ public class SystemOverviewVO { // 服务器时间 private String serverTime; - private CurrentServerInfo currentServerInfo; - - @Getter - @AllArgsConstructor - public static class CurrentServerInfo { - private final long id; - private final String ip; - private final String version; - } + private ServerInfo serverInfo; } From 1b9d8331a19e2d69f1f67ea4d613cf57fae5ad54 Mon Sep 17 00:00:00 2001 From: tjq Date: Mon, 12 Sep 2022 21:00:01 +0800 Subject: [PATCH 23/30] feat: optimize DatabaseMonitorAspect --- .../server/persistence/monitor/DatabaseMonitorAspect.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/monitor/DatabaseMonitorAspect.java b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/monitor/DatabaseMonitorAspect.java index cd8649b3..6d5adb49 100644 --- a/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/monitor/DatabaseMonitorAspect.java +++ b/powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/monitor/DatabaseMonitorAspect.java @@ -55,7 +55,7 @@ public class DatabaseMonitorAspect { event.setRows(parseEffectRows(ret)); return ret; } catch (Throwable t) { - event.setErrorMsg(t.getMessage()).setStatus(DatabaseEvent.Status.FAILED); + event.setErrorMsg(t.getClass().getSimpleName()).setStatus(DatabaseEvent.Status.FAILED); throw t; } finally { long cost = System.currentTimeMillis() - startTs; From a77ba8084e01c621c4183afc025baad3ae99bc62 Mon Sep 17 00:00:00 2001 From: tjq Date: Mon, 12 Sep 2022 21:09:47 +0800 Subject: [PATCH 24/30] feat: config logback for monitor --- .../server/core/lock/UseCacheLockAspect.java | 3 +- .../monitor/events/lock/SlowLockEvent.java | 10 +- .../logback-config/powerjob_monitor.xml | 102 +++++++++++++++++- 3 files changed, 110 insertions(+), 5 deletions(-) diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/lock/UseCacheLockAspect.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/lock/UseCacheLockAspect.java index aed87933..11b4591c 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/lock/UseCacheLockAspect.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/lock/UseCacheLockAspect.java @@ -61,7 +61,8 @@ public class UseCacheLockAspect { if (timeCost > SLOW_THRESHOLD) { final SlowLockEvent slowLockEvent = new SlowLockEvent() - .setLocKType(useCacheLock.type()) + .setType(SlowLockEvent.Type.LOCAL) + .setLockType(useCacheLock.type()) .setLockKey(String.valueOf(key)) .setCallerService(method.getDeclaringClass().getSimpleName()) .setCallerMethod(method.getName()) diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/lock/SlowLockEvent.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/lock/SlowLockEvent.java index 16527214..9211c90b 100644 --- a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/lock/SlowLockEvent.java +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/lock/SlowLockEvent.java @@ -15,12 +15,18 @@ import tech.powerjob.server.monitor.Event; @Accessors(chain = true) public class SlowLockEvent implements Event { - private String locKType; + private Type type; + private String lockType; private String lockKey; private String callerService; private String callerMethod; private long cost; + public enum Type { + LOCAL, + DB + } + @Override public String type() { return "MONITOR_LOGGER_SLOW_LOCK"; @@ -28,6 +34,6 @@ public class SlowLockEvent implements Event { @Override public String message() { - return SJ.MONITOR_JOINER.join(locKType, lockKey, callerService, callerMethod, cost); + return SJ.MONITOR_JOINER.join(type, lockType, lockKey, callerService, callerMethod, cost); } } diff --git a/powerjob-server/powerjob-server-starter/src/main/resources/logback-config/powerjob_monitor.xml b/powerjob-server/powerjob-server-starter/src/main/resources/logback-config/powerjob_monitor.xml index 397c29c8..c3a49f8e 100644 --- a/powerjob-server/powerjob-server-starter/src/main/resources/logback-config/powerjob_monitor.xml +++ b/powerjob-server/powerjob-server-starter/src/main/resources/logback-config/powerjob_monitor.xml @@ -6,7 +6,7 @@ - + ${MONITOR_LOG_PATH}/database.log @@ -29,6 +29,104 @@ - + + + + + ${MONITOR_LOG_PATH}/tt_status_report.log + + ${MONITOR_LOG_PATTERN} + UTF-8 + + + ${MONITOR_LOG_PATH}/tt_status_report.log.${ROTATE_PATTERN} + 3 + 500MB + 1500MB + + + + 512 + 0 + true + + + + + + + + + ${MONITOR_LOG_PATH}/worker_heartbeat.log + + ${MONITOR_LOG_PATTERN} + UTF-8 + + + ${MONITOR_LOG_PATH}/worker_heartbeat.log.${ROTATE_PATTERN} + 3 + 500MB + 1500MB + + + + 512 + 0 + true + + + + + + + + + + ${MONITOR_LOG_PATH}/worker_log_report.log + + ${MONITOR_LOG_PATTERN} + UTF-8 + + + ${MONITOR_LOG_PATH}/worker_log_report.log.${ROTATE_PATTERN} + 3 + 500MB + 1500MB + + + + 512 + 0 + true + + + + + + + + + + ${MONITOR_LOG_PATH}/lock.log + + ${MONITOR_LOG_PATTERN} + UTF-8 + + + ${MONITOR_LOG_PATH}/lock.log.${ROTATE_PATTERN} + 3 + 500MB + 1500MB + + + + 512 + 0 + true + + + + + \ No newline at end of file From fadf2ce14e6b2075bdb965872748206a8bf4d8e9 Mon Sep 17 00:00:00 2001 From: tjq Date: Mon, 12 Sep 2022 21:16:00 +0800 Subject: [PATCH 25/30] feat: optimize monitor logger config --- .../logback-config/powerjob_monitor.xml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/powerjob-server/powerjob-server-starter/src/main/resources/logback-config/powerjob_monitor.xml b/powerjob-server/powerjob-server-starter/src/main/resources/logback-config/powerjob_monitor.xml index c3a49f8e..d357a833 100644 --- a/powerjob-server/powerjob-server-starter/src/main/resources/logback-config/powerjob_monitor.xml +++ b/powerjob-server/powerjob-server-starter/src/main/resources/logback-config/powerjob_monitor.xml @@ -16,8 +16,8 @@ ${MONITOR_LOG_PATH}/database.log.${ROTATE_PATTERN} 3 - 500MB - 1500MB + 200MB + 1000MB @@ -41,8 +41,8 @@ ${MONITOR_LOG_PATH}/tt_status_report.log.${ROTATE_PATTERN} 3 - 500MB - 1500MB + 200MB + 1000MB @@ -65,8 +65,8 @@ ${MONITOR_LOG_PATH}/worker_heartbeat.log.${ROTATE_PATTERN} 3 - 500MB - 1500MB + 200MB + 1000MB @@ -90,8 +90,8 @@ ${MONITOR_LOG_PATH}/worker_log_report.log.${ROTATE_PATTERN} 3 - 500MB - 1500MB + 200MB + 1000MB @@ -115,8 +115,8 @@ ${MONITOR_LOG_PATH}/lock.log.${ROTATE_PATTERN} 3 - 500MB - 1500MB + 200MB + 1000MB From e6d32c9a052311642c1a838c752e9ecce5c7a51b Mon Sep 17 00:00:00 2001 From: tjq Date: Mon, 12 Sep 2022 21:18:40 +0800 Subject: [PATCH 26/30] fix: worker heartbeat use wrong thread pool --- .../server/remote/server/self/ServerInfoServiceImpl.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/self/ServerInfoServiceImpl.java b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/self/ServerInfoServiceImpl.java index 5159da83..94bb12ca 100644 --- a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/self/ServerInfoServiceImpl.java +++ b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/self/ServerInfoServiceImpl.java @@ -2,9 +2,11 @@ package tech.powerjob.server.remote.server.self; import org.apache.commons.lang3.StringUtils; import org.springframework.boot.info.BuildProperties; +import org.springframework.scheduling.annotation.Async; import tech.powerjob.common.exception.PowerJobException; import tech.powerjob.common.utils.CommonUtils; import tech.powerjob.common.utils.NetUtils; +import tech.powerjob.server.common.constants.PJThreadPool; import tech.powerjob.server.common.module.ServerInfo; import tech.powerjob.server.extension.LockService; import tech.powerjob.server.persistence.remote.model.ServerInfoDO; @@ -86,6 +88,7 @@ public class ServerInfoServiceImpl implements ServerInfoService { log.info("[ServerInfoService] ip:{}, id:{}, cost:{}", ip, serverInfo.getId(), sw); } + @Async(PJThreadPool.TIMING_POOL) @Scheduled(fixedRate = 15000, initialDelay = 15000) public void heartbeat() { serverInfoRepository.updateGmtModifiedByIp(serverInfo.getIp(), new Date()); From fd562d8ea00d067a5978502b901562b06b1637f3 Mon Sep 17 00:00:00 2001 From: tjq Date: Mon, 12 Sep 2022 21:26:01 +0800 Subject: [PATCH 27/30] fix: worker heartbeat use wrong thread pool --- .../powerjob/server/remote/server/self/ServerInfoService.java | 4 ++++ .../server/remote/server/self/ServerInfoServiceImpl.java | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/self/ServerInfoService.java b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/self/ServerInfoService.java index a01b8c0b..5c799a7a 100644 --- a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/self/ServerInfoService.java +++ b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/self/ServerInfoService.java @@ -10,6 +10,10 @@ import tech.powerjob.server.common.module.ServerInfo; */ public interface ServerInfoService { + /** + * fetch current server info + * @return ServerInfo + */ ServerInfo fetchServiceInfo(); } diff --git a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/self/ServerInfoServiceImpl.java b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/self/ServerInfoServiceImpl.java index 94bb12ca..eb94b58b 100644 --- a/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/self/ServerInfoServiceImpl.java +++ b/powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/self/ServerInfoServiceImpl.java @@ -88,7 +88,6 @@ public class ServerInfoServiceImpl implements ServerInfoService { log.info("[ServerInfoService] ip:{}, id:{}, cost:{}", ip, serverInfo.getId(), sw); } - @Async(PJThreadPool.TIMING_POOL) @Scheduled(fixedRate = 15000, initialDelay = 15000) public void heartbeat() { serverInfoRepository.updateGmtModifiedByIp(serverInfo.getIp(), new Date()); From 5b223d23ad321b90628da99c68787911681f7edf Mon Sep 17 00:00:00 2001 From: tjq Date: Mon, 12 Sep 2022 21:52:26 +0800 Subject: [PATCH 28/30] feat: optimize server monitor event --- .../powerjob/server/core/handler/AbWorkerRequestHandler.java | 3 ++- .../server/core/handler/impl/WorkerRequestAkkaHandler.java | 2 +- .../server/monitor/events/w2s/TtReportInstanceStatusEvent.java | 3 ++- .../src/main/resources/oms-server.akka.conf | 3 ++- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/AbWorkerRequestHandler.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/AbWorkerRequestHandler.java index 9ef62dae..f1b68eec 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/AbWorkerRequestHandler.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/AbWorkerRequestHandler.java @@ -4,6 +4,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.beans.BeanUtils; import org.springframework.core.env.Environment; +import tech.powerjob.common.enums.InstanceStatus; import tech.powerjob.common.request.*; import tech.powerjob.common.response.AskResponse; import tech.powerjob.common.serialize.JsonUtils; @@ -76,7 +77,7 @@ public abstract class AbWorkerRequestHandler implements IWorkerRequestHandler { .setJobId(req.getJobId()) .setInstanceId(req.getInstanceId()) .setWfInstanceId(req.getWfInstanceId()) - .setInstanceStatus(req.getInstanceStatus()) + .setInstanceStatus(InstanceStatus.of(req.getInstanceStatus())) .setDelayMs(startMs - req.getReportTime()) .setServerProcessStatus(TtReportInstanceStatusEvent.Status.SUCCESS); try { diff --git a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/impl/WorkerRequestAkkaHandler.java b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/impl/WorkerRequestAkkaHandler.java index 147b9df4..26f18be0 100644 --- a/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/impl/WorkerRequestAkkaHandler.java +++ b/powerjob-server/powerjob-server-core/src/main/java/tech/powerjob/server/core/handler/impl/WorkerRequestAkkaHandler.java @@ -24,7 +24,7 @@ public class WorkerRequestAkkaHandler extends AbstractActor { public static Props defaultProps(){ return Props.create(WorkerRequestAkkaHandler.class) - .withDispatcher("akka.worker-request-actor-dispatcher") + .withDispatcher("akka.w-r-c-d") .withRouter( new RoundRobinPool(Runtime.getRuntime().availableProcessors() * 4) .withResizer(new DefaultResizer( diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/w2s/TtReportInstanceStatusEvent.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/w2s/TtReportInstanceStatusEvent.java index 6eef0faf..d43e7db0 100644 --- a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/w2s/TtReportInstanceStatusEvent.java +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/events/w2s/TtReportInstanceStatusEvent.java @@ -2,6 +2,7 @@ package tech.powerjob.server.monitor.events.w2s; import lombok.Setter; import lombok.experimental.Accessors; +import tech.powerjob.common.enums.InstanceStatus; import tech.powerjob.server.common.SJ; import tech.powerjob.server.monitor.Event; @@ -21,7 +22,7 @@ public class TtReportInstanceStatusEvent implements Event { private Long wfInstanceId; - private int instanceStatus; + private InstanceStatus instanceStatus; private Long delayMs; diff --git a/powerjob-server/powerjob-server-starter/src/main/resources/oms-server.akka.conf b/powerjob-server/powerjob-server-starter/src/main/resources/oms-server.akka.conf index 7e2274cd..95afe889 100644 --- a/powerjob-server/powerjob-server-starter/src/main/resources/oms-server.akka.conf +++ b/powerjob-server/powerjob-server-starter/src/main/resources/oms-server.akka.conf @@ -25,7 +25,8 @@ akka { } } - worker-request-actor-dispatcher { + # worker-request-core-dispatcher + w-r-c-d { # Dispatcher is the name of the event-based dispatcher type = Dispatcher # What kind of ExecutionService to use From a32d7cebb5fb7ef9135d6442c2afcc0fe3be3e17 Mon Sep 17 00:00:00 2001 From: tjq Date: Mon, 12 Sep 2022 22:10:07 +0800 Subject: [PATCH 29/30] feat: optimize server monitor event --- .../src/tech/powerjob/server/monitor/monitors/LogMonitor.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/LogMonitor.java b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/LogMonitor.java index 45f598b7..050e5ed7 100644 --- a/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/LogMonitor.java +++ b/powerjob-server/powerjob-server-monitor/src/tech/powerjob/server/monitor/monitors/LogMonitor.java @@ -24,7 +24,6 @@ public class LogMonitor implements Monitor, ServerInfoAware { private ServerInfo serverInfo = new ServerInfo(); private static final String MDC_KEY_SERVER_ID = "serverId"; - private static final String MDC_KEY_SERVER_ADDRESS = "serverAddress"; @Override @@ -34,7 +33,6 @@ public class LogMonitor implements Monitor, ServerInfoAware { @Override public void record(Event event) { MDC.put(MDC_KEY_SERVER_ID, String.valueOf(serverInfo.getId())); - MDC.put(MDC_KEY_SERVER_ADDRESS, serverInfo.getIp()); LoggerFactory.getLogger(event.type()).info(event.message()); } From 5acb8f82e79c59702cdd112cdb13ebbc5a55e4a4 Mon Sep 17 00:00:00 2001 From: tjq Date: Mon, 12 Sep 2022 22:59:00 +0800 Subject: [PATCH 30/30] feat: optimize OmsLocalDbPool config --- .../java/tech/powerjob/server/config/ThreadPoolConfig.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/config/ThreadPoolConfig.java b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/config/ThreadPoolConfig.java index 99b61ade..899c182e 100644 --- a/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/config/ThreadPoolConfig.java +++ b/powerjob-server/powerjob-server-starter/src/main/java/tech/powerjob/server/config/ThreadPoolConfig.java @@ -53,8 +53,9 @@ public class ThreadPoolConfig { @Bean(PJThreadPool.LOCAL_DB_POOL) public TaskExecutor initOmsLocalDbPool() { ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); - executor.setCorePoolSize(Runtime.getRuntime().availableProcessors() / 2); - executor.setMaxPoolSize(Runtime.getRuntime().availableProcessors() / 2); + int tSize = Math.max(1, Runtime.getRuntime().availableProcessors() / 2); + executor.setCorePoolSize(tSize); + executor.setMaxPoolSize(tSize); executor.setQueueCapacity(2048); executor.setThreadNamePrefix("PJ-LOCALDB-"); executor.setRejectedExecutionHandler(RejectedExecutionHandlerFactory.newAbort(PJThreadPool.LOCAL_DB_POOL));