From 9822e8a10fe4203e1a05fb79dd4e9391d2c0f501 Mon Sep 17 00:00:00 2001 From: tanwenhai Date: Tue, 15 Dec 2020 15:27:57 +0800 Subject: [PATCH 1/7] postgresql --- .../server/persistence/core/model/InstanceInfoDO.java | 3 +++ .../powerjob/server/persistence/core/model/JobInfoDO.java | 2 ++ .../server/persistence/core/model/WorkflowInfoDO.java | 2 ++ .../server/persistence/core/model/WorkflowInstanceInfoDO.java | 4 ++++ 4 files changed, 11 insertions(+) diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/InstanceInfoDO.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/InstanceInfoDO.java index 5af69945..b0d7599d 100644 --- a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/InstanceInfoDO.java +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/InstanceInfoDO.java @@ -5,6 +5,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import org.hibernate.annotations.GenericGenerator; +import org.hibernate.annotations.Type; import javax.persistence.*; import java.util.Date; @@ -36,6 +37,7 @@ public class InstanceInfoDO { // 任务实例参数 @Lob @Column + @Type(type ="org.hibernate.type.StringType") private String instanceParams; // 该任务实例的类型，普通/工作流（InstanceType） @@ -49,6 +51,7 @@ public class InstanceInfoDO { // 执行结果（允许存储稍大的结果） @Lob @Column + @Type(type ="org.hibernate.type.StringType") private String result; // 预计触发时间 private Long expectedTriggerTime; diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/JobInfoDO.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/JobInfoDO.java index b5d2b171..5bfe66b7 100644 --- a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/JobInfoDO.java +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/JobInfoDO.java @@ -5,6 +5,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import org.hibernate.annotations.GenericGenerator; +import org.hibernate.annotations.Type; import javax.persistence.*; import java.util.Date; @@ -52,6 +53,7 @@ public class JobInfoDO { // 执行器信息（可能需要存储整个脚本文件） @Lob @Column + @Type(type ="org.hibernate.type.StringType") private String processorInfo; /* ************************** 运行时配置 ************************** */ diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/WorkflowInfoDO.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/WorkflowInfoDO.java index f3c5d560..3bcb6ac1 100644 --- a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/WorkflowInfoDO.java +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/WorkflowInfoDO.java @@ -4,6 +4,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import org.hibernate.annotations.GenericGenerator; +import org.hibernate.annotations.Type; import javax.persistence.*; import java.util.Date; @@ -35,6 +36,7 @@ public class WorkflowInfoDO { // 工作流的DAG图信息（点线式DAG的json） @Lob @Column + @Type(type ="org.hibernate.type.StringType") private String peDAG; /* ************************** 定时参数 ************************** */ diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/WorkflowInstanceInfoDO.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/WorkflowInstanceInfoDO.java index e6f483c4..2dcb9a1c 100644 --- a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/WorkflowInstanceInfoDO.java +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/WorkflowInstanceInfoDO.java @@ -4,6 +4,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import org.hibernate.annotations.GenericGenerator; +import org.hibernate.annotations.Type; import javax.persistence.*; import java.util.Date; @@ -39,13 +40,16 @@ public class WorkflowInstanceInfoDO { // 工作流启动参数 @Lob @Column + @Type(type ="org.hibernate.type.StringType") private String wfInitParams; @Lob @Column + @Type(type ="org.hibernate.type.StringType") private String dag; @Lob @Column + @Type(type ="org.hibernate.type.StringType") private String result; // 预计触发时间 From be43839d730d8ba945614179ba175a1f430c826b Mon Sep 17 00:00:00 2001 From: tanwenhai Date: Tue, 15 Dec 2020 15:29:06 +0800 Subject: [PATCH 2/7] actuator endpoint --- powerjob-server/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/powerjob-server/pom.xml b/powerjob-server/pom.xml index 68036a10..080e85d1 100644 --- a/powerjob-server/pom.xml +++ b/powerjob-server/pom.xml @@ -119,6 +119,11 @@ spring-boot-starter-data-mongodb ${springboot.version} + + org.springframework.boot + spring-boot-starter-actuator + ${springboot.version} + org.springframework.boot spring-boot-starter-mail From e311e7c50062115c390c16966c5a8d294eb964b6 Mon Sep 17 00:00:00 2001 From: tanwenhai Date: Tue, 15 Dec 2020 16:11:02 +0800 Subject: [PATCH 3/7] id generate --- .../core/repository/ServerInfoRepository.java | 9 ++++ .../server/service/id/IdGenerateService.java | 52 ++++++++++++++----- .../resources/application-daily.properties | 17 ++---- 3 files changed, 51 insertions(+), 27 deletions(-) diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/repository/ServerInfoRepository.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/repository/ServerInfoRepository.java index 615933f2..7f0b70ba 100644 --- a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/repository/ServerInfoRepository.java +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/repository/ServerInfoRepository.java @@ -2,6 +2,11 @@ package com.github.kfcfans.powerjob.server.persistence.core.repository; import com.github.kfcfans.powerjob.server.persistence.core.model.ServerInfoDO; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Lock; +import org.springframework.data.jpa.repository.Query; + +import javax.persistence.LockModeType; +import java.util.List; /** * 服务器信息 数据操作层 @@ -11,4 +16,8 @@ import org.springframework.data.jpa.repository.JpaRepository; */ public interface ServerInfoRepository extends JpaRepository { ServerInfoDO findByIp(String ip); + + @Query("select t from ServerInfoDO as t order by t.id asc") + @Lock(LockModeType.PESSIMISTIC_WRITE) + List findAllAndLockTable(); } diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/id/IdGenerateService.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/id/IdGenerateService.java index e56462fa..bf3f0ee7 100644 --- a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/id/IdGenerateService.java +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/id/IdGenerateService.java @@ -3,9 +3,23 @@ package com.github.kfcfans.powerjob.server.service.id; import com.github.kfcfans.powerjob.common.utils.NetUtils; import com.github.kfcfans.powerjob.server.persistence.core.model.ServerInfoDO; import com.github.kfcfans.powerjob.server.persistence.core.repository.ServerInfoRepository; +import com.google.common.base.Throwables; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.jdbc.support.JdbcUtils; import org.springframework.stereotype.Service; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.support.TransactionTemplate; +import org.springframework.util.Assert; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.List; +import java.util.Objects; +import java.util.function.Consumer; /** * 唯一ID生成服务，使用 Twitter snowflake 算法 @@ -24,20 +38,32 @@ public class IdGenerateService { private static final int DATA_CENTER_ID = 0; @Autowired - public IdGenerateService(ServerInfoRepository serverInfoRepository) { + public IdGenerateService(ServerInfoRepository serverInfoRepository, + @Qualifier("coreTransactionManager") PlatformTransactionManager platformTransactionManager) { + TransactionTemplate transactionTemplate = new TransactionTemplate(platformTransactionManager); + // 服务频繁重启，数据库id自增值过大，使用表锁获取机器id + snowFlakeIdGenerator = transactionTemplate.execute(action -> { + List serverInfos = serverInfoRepository.findAllAndLockTable(); + String ip = NetUtils.getLocalHost(); + ServerInfoDO server = serverInfoRepository.findByIp(ip); + Long id = null; + if (server == null) { + ServerInfoDO newServerInfo = new ServerInfoDO(ip); + serverInfoRepository.saveAndFlush(newServerInfo); + id = serverInfos.size() + 1L; + } else { + for (int i = 0, len = serverInfos.size(); i < len; i++) { + if (Objects.equals(serverInfos.get(i).getId(), server.getId())) { + id = i + 1L; + break; + } + } + } + Assert.notNull(id, "[IdGenerateService] init snowflake error, id is null"); + log.info("[IdGenerateService] init snowflake for server(address={}) by machineId({}).", ip, id); - String ip = NetUtils.getLocalHost(); - ServerInfoDO server = serverInfoRepository.findByIp(ip); - - if (server == null) { - ServerInfoDO newServerInfo = new ServerInfoDO(ip); - server = serverInfoRepository.saveAndFlush(newServerInfo); - } - - Long id = server.getId(); - snowFlakeIdGenerator = new SnowFlakeIdGenerator(DATA_CENTER_ID, id); - - log.info("[IdGenerateService] init snowflake for server(address={}) by machineId({}).", ip, id); + return new SnowFlakeIdGenerator(DATA_CENTER_ID, id); + }); } /** diff --git a/powerjob-server/src/main/resources/application-daily.properties b/powerjob-server/src/main/resources/application-daily.properties index 18bb67ef..32d7da66 100644 --- a/powerjob-server/src/main/resources/application-daily.properties +++ b/powerjob-server/src/main/resources/application-daily.properties @@ -3,28 +3,17 @@ logging.config=classpath:logback-dev.xml ####### 外部数据库配置（需要用户更改为自己的数据库配置） ####### spring.datasource.core.driver-class-name=com.mysql.cj.jdbc.Driver -spring.datasource.core.jdbc-url=jdbc:mysql://localhost:3306/powerjob-daily?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai +spring.datasource.core.jdbc-url=jdbc:mysql://localhost:3306/powerjob?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai spring.datasource.core.username=root -spring.datasource.core.password=No1Bug2Please3! +spring.datasource.core.password=123456 spring.datasource.core.hikari.maximum-pool-size=20 spring.datasource.core.hikari.minimum-idle=5 ####### mongoDB配置，非核心依赖，通过配置 oms.mongodb.enable=false 来关闭 ####### -oms.mongodb.enable=true +oms.mongodb.enable=false spring.data.mongodb.uri=mongodb+srv://zqq:No1Bug2Please3!@cluster0.wie54.gcp.mongodb.net/powerjob_daily?retryWrites=true&w=majority -####### 邮件配置（不需要邮件报警可以删除以下配置来避免报错） ####### -spring.mail.host=smtp.163.com -spring.mail.username=zqq@163.com -spring.mail.password=GOFZPNARMVKCGONV -spring.mail.properties.mail.smtp.auth=true -spring.mail.properties.mail.smtp.starttls.enable=true -spring.mail.properties.mail.smtp.starttls.required=true -####### 钉钉报警配置（不需要钉钉报警可以删除以下配置来避免报错） ####### -oms.alarm.ding.app-key=dingauqwkvxxnqskknfv -oms.alarm.ding.app-secret=XWrEPdAZMPgJeFtHuL0LH73LRj-74umF2_0BFcoXMfvnX0pCQvt0rpb1JOJU_HLl -oms.alarm.ding.agent-id=847044348 ####### 资源清理配置 ####### oms.instanceinfo.retention=1 From ca6b02d19f9b6bdba892a2560288e858e1da3fcd Mon Sep 17 00:00:00 2001 From: tanwenhai Date: Tue, 15 Dec 2020 16:14:18 +0800 Subject: [PATCH 4/7] reset --- .../resources/application-daily.properties | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/powerjob-server/src/main/resources/application-daily.properties b/powerjob-server/src/main/resources/application-daily.properties index 32d7da66..9b1e2b7a 100644 --- a/powerjob-server/src/main/resources/application-daily.properties +++ b/powerjob-server/src/main/resources/application-daily.properties @@ -3,17 +3,28 @@ logging.config=classpath:logback-dev.xml ####### 外部数据库配置（需要用户更改为自己的数据库配置） ####### spring.datasource.core.driver-class-name=com.mysql.cj.jdbc.Driver -spring.datasource.core.jdbc-url=jdbc:mysql://localhost:3306/powerjob?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai +spring.datasource.core.jdbc-url=jdbc:mysql://localhost:3306/powerjob-daily?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai spring.datasource.core.username=root -spring.datasource.core.password=123456 +spring.datasource.core.password=No1Bug2Please3! spring.datasource.core.hikari.maximum-pool-size=20 spring.datasource.core.hikari.minimum-idle=5 ####### mongoDB配置，非核心依赖，通过配置 oms.mongodb.enable=false 来关闭 ####### -oms.mongodb.enable=false +oms.mongodb.enable=true spring.data.mongodb.uri=mongodb+srv://zqq:No1Bug2Please3!@cluster0.wie54.gcp.mongodb.net/powerjob_daily?retryWrites=true&w=majority +####### 邮件配置（不需要邮件报警可以删除以下配置来避免报错） ####### +spring.mail.host=smtp.163.com +spring.mail.username=zqq@163.com +spring.mail.password=GOFZPNARMVKCGONV +spring.mail.properties.mail.smtp.auth=true +spring.mail.properties.mail.smtp.starttls.enable=true +spring.mail.properties.mail.smtp.starttls.required=true +####### 钉钉报警配置（不需要钉钉报警可以删除以下配置来避免报错） ####### +oms.alarm.ding.app-key=dingauqwkvxxnqskknfv +oms.alarm.ding.app-secret=XWrEPdAZMPgJeFtHuL0LH73LRj-74umF2_0BFcoXMfvnX0pCQvt0rpb1JOJU_HLl +oms.alarm.ding.agent-id=847044348 ####### 资源清理配置 ####### oms.instanceinfo.retention=1 @@ -21,7 +32,4 @@ oms.container.retention.local=1 oms.container.retention.remote=-1 ####### 缓存配置 ####### -oms.instance.metadata.cache.size=1024 - -####### 精确获取 server 的百分比，0～100，100代表每次 worker 获取 server 都会进行完整的探活流程，不存在脑裂问题，但有性能开销 ####### -oms.accurate.select.server.percentage = 50 \ No newline at end of file +oms.instance.metadata.cache.size=1024 \ No newline at end of file From 4eb53990f404bf957c003f6ccc9f065e5c8f0c24 Mon Sep 17 00:00:00 2001 From: tanwenhai Date: Wed, 16 Dec 2020 17:05:29 +0800 Subject: [PATCH 5/7] daily config --- .../src/main/resources/application-daily.properties | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/powerjob-server/src/main/resources/application-daily.properties b/powerjob-server/src/main/resources/application-daily.properties index 9b1e2b7a..18bb67ef 100644 --- a/powerjob-server/src/main/resources/application-daily.properties +++ b/powerjob-server/src/main/resources/application-daily.properties @@ -32,4 +32,7 @@ oms.container.retention.local=1 oms.container.retention.remote=-1 ####### 缓存配置 ####### -oms.instance.metadata.cache.size=1024 \ No newline at end of file +oms.instance.metadata.cache.size=1024 + +####### 精确获取 server 的百分比，0～100，100代表每次 worker 获取 server 都会进行完整的探活流程，不存在脑裂问题，但有性能开销 ####### +oms.accurate.select.server.percentage = 50 \ No newline at end of file From 35f0991f031ee047c80e097e81c452e4d125222d Mon Sep 17 00:00:00 2001 From: tanwenhai Date: Wed, 16 Dec 2020 18:35:56 +0800 Subject: [PATCH 6/7] TypeDef --- .../server/persistence/core/model/InstanceInfoDO.java | 4 ++-- .../server/persistence/core/model/JobInfoDO.java | 2 +- .../server/persistence/core/model/TypeDefConstant.java | 9 +++++++++ .../server/persistence/core/model/WorkflowInfoDO.java | 2 +- .../persistence/core/model/WorkflowInstanceInfoDO.java | 6 +++--- .../server/persistence/core/model/package-info.java | 7 +++++++ 6 files changed, 23 insertions(+), 7 deletions(-) create mode 100644 powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/TypeDefConstant.java create mode 100644 powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/package-info.java diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/InstanceInfoDO.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/InstanceInfoDO.java index b0d7599d..a956830c 100644 --- a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/InstanceInfoDO.java +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/InstanceInfoDO.java @@ -37,7 +37,7 @@ public class InstanceInfoDO { // 任务实例参数 @Lob @Column - @Type(type ="org.hibernate.type.StringType") + @Type(type = TypeDefConstant.STRING_TYPE) private String instanceParams; // 该任务实例的类型，普通/工作流（InstanceType） @@ -51,7 +51,7 @@ public class InstanceInfoDO { // 执行结果（允许存储稍大的结果） @Lob @Column - @Type(type ="org.hibernate.type.StringType") + @Type(type = TypeDefConstant.STRING_TYPE) private String result; // 预计触发时间 private Long expectedTriggerTime; diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/JobInfoDO.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/JobInfoDO.java index 5bfe66b7..ddccbf7a 100644 --- a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/JobInfoDO.java +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/JobInfoDO.java @@ -53,7 +53,7 @@ public class JobInfoDO { // 执行器信息（可能需要存储整个脚本文件） @Lob @Column - @Type(type ="org.hibernate.type.StringType") + @Type(type = TypeDefConstant.STRING_TYPE) private String processorInfo; /* ************************** 运行时配置 ************************** */ diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/TypeDefConstant.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/TypeDefConstant.java new file mode 100644 index 00000000..fbe4b199 --- /dev/null +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/TypeDefConstant.java @@ -0,0 +1,9 @@ +package com.github.kfcfans.powerjob.server.persistence.core.model; + +/** + * @see package-info.java + * @author user + */ +public final class TypeDefConstant { + public static final String STRING_TYPE = "string-type"; +} diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/WorkflowInfoDO.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/WorkflowInfoDO.java index 3bcb6ac1..ee8c53a8 100644 --- a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/WorkflowInfoDO.java +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/WorkflowInfoDO.java @@ -36,7 +36,7 @@ public class WorkflowInfoDO { // 工作流的DAG图信息（点线式DAG的json） @Lob @Column - @Type(type ="org.hibernate.type.StringType") + @Type(type = TypeDefConstant.STRING_TYPE) private String peDAG; /* ************************** 定时参数 ************************** */ diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/WorkflowInstanceInfoDO.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/WorkflowInstanceInfoDO.java index 2dcb9a1c..d6290205 100644 --- a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/WorkflowInstanceInfoDO.java +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/WorkflowInstanceInfoDO.java @@ -40,16 +40,16 @@ public class WorkflowInstanceInfoDO { // 工作流启动参数 @Lob @Column - @Type(type ="org.hibernate.type.StringType") + @Type(type = TypeDefConstant.STRING_TYPE) private String wfInitParams; @Lob @Column - @Type(type ="org.hibernate.type.StringType") + @Type(type = TypeDefConstant.STRING_TYPE) private String dag; @Lob @Column - @Type(type ="org.hibernate.type.StringType") + @Type(type = TypeDefConstant.STRING_TYPE) private String result; // 预计触发时间 diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/package-info.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/package-info.java new file mode 100644 index 00000000..543fd8a0 --- /dev/null +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/core/model/package-info.java @@ -0,0 +1,7 @@ +@TypeDefs({ + @TypeDef(name = TypeDefConstant.STRING_TYPE, typeClass = org.hibernate.type.StringType.class) +}) +package com.github.kfcfans.powerjob.server.persistence.core.model; + +import org.hibernate.annotations.TypeDef; +import org.hibernate.annotations.TypeDefs; \ No newline at end of file From 76e5a41881c1f401c65fc5c0df68980d5d539a16 Mon Sep 17 00:00:00 2001 From: tanwenhai Date: Thu, 17 Dec 2020 11:12:54 +0800 Subject: [PATCH 7/7] serverIdProvider interface and implements --- .../service/id/DefaultServerIdProvider.java | 39 +++++++++++++++++ .../server/service/id/IdGenerateService.java | 43 +++---------------- .../server/service/id/ServerIdProvider.java | 12 ++++++ .../id/ServerIdProviderConfiguration.java | 25 +++++++++++ .../id/StatefulSetServerIdProvider.java | 30 +++++++++++++ 5 files changed, 111 insertions(+), 38 deletions(-) create mode 100644 powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/id/DefaultServerIdProvider.java create mode 100644 powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/id/ServerIdProvider.java create mode 100644 powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/id/ServerIdProviderConfiguration.java create mode 100644 powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/id/StatefulSetServerIdProvider.java diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/id/DefaultServerIdProvider.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/id/DefaultServerIdProvider.java new file mode 100644 index 00000000..1c018370 --- /dev/null +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/id/DefaultServerIdProvider.java @@ -0,0 +1,39 @@ +package com.github.kfcfans.powerjob.server.service.id; + +import com.github.kfcfans.powerjob.common.utils.NetUtils; +import com.github.kfcfans.powerjob.server.persistence.core.model.ServerInfoDO; +import com.github.kfcfans.powerjob.server.persistence.core.repository.ServerInfoRepository; + +/** + * @author user + */ +public class DefaultServerIdProvider implements ServerIdProvider { + private final ServerInfoRepository serverInfoRepository; + + private volatile Long id; + + public DefaultServerIdProvider(ServerInfoRepository serverInfoRepository) { + this.serverInfoRepository = serverInfoRepository; + } + + @Override + public long serverId() { + if (id == null) { + synchronized (this) { + if (id == null) { + String ip = NetUtils.getLocalHost(); + ServerInfoDO server = serverInfoRepository.findByIp(ip); + + if (server == null) { + ServerInfoDO newServerInfo = new ServerInfoDO(ip); + server = serverInfoRepository.saveAndFlush(newServerInfo); + } + + id = server.getId(); + } + } + } + + return id; + } +} diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/id/IdGenerateService.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/id/IdGenerateService.java index bf3f0ee7..c334d01b 100644 --- a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/id/IdGenerateService.java +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/id/IdGenerateService.java @@ -3,23 +3,10 @@ package com.github.kfcfans.powerjob.server.service.id; import com.github.kfcfans.powerjob.common.utils.NetUtils; import com.github.kfcfans.powerjob.server.persistence.core.model.ServerInfoDO; import com.github.kfcfans.powerjob.server.persistence.core.repository.ServerInfoRepository; -import com.google.common.base.Throwables; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.jdbc.support.JdbcUtils; import org.springframework.stereotype.Service; -import org.springframework.transaction.PlatformTransactionManager; -import org.springframework.transaction.annotation.Transactional; -import org.springframework.transaction.support.TransactionTemplate; -import org.springframework.util.Assert; - -import javax.sql.DataSource; -import java.sql.Connection; -import java.sql.SQLException; -import java.util.List; -import java.util.Objects; -import java.util.function.Consumer; /** * 唯一ID生成服务，使用 Twitter snowflake 算法 @@ -38,32 +25,12 @@ public class IdGenerateService { private static final int DATA_CENTER_ID = 0; @Autowired - public IdGenerateService(ServerInfoRepository serverInfoRepository, - @Qualifier("coreTransactionManager") PlatformTransactionManager platformTransactionManager) { - TransactionTemplate transactionTemplate = new TransactionTemplate(platformTransactionManager); - // 服务频繁重启，数据库id自增值过大，使用表锁获取机器id - snowFlakeIdGenerator = transactionTemplate.execute(action -> { - List serverInfos = serverInfoRepository.findAllAndLockTable(); - String ip = NetUtils.getLocalHost(); - ServerInfoDO server = serverInfoRepository.findByIp(ip); - Long id = null; - if (server == null) { - ServerInfoDO newServerInfo = new ServerInfoDO(ip); - serverInfoRepository.saveAndFlush(newServerInfo); - id = serverInfos.size() + 1L; - } else { - for (int i = 0, len = serverInfos.size(); i < len; i++) { - if (Objects.equals(serverInfos.get(i).getId(), server.getId())) { - id = i + 1L; - break; - } - } - } - Assert.notNull(id, "[IdGenerateService] init snowflake error, id is null"); - log.info("[IdGenerateService] init snowflake for server(address={}) by machineId({}).", ip, id); + public IdGenerateService(@Qualifier("serverIdProvider") ServerIdProvider serverIdProvider) { + long id = serverIdProvider.serverId(); + snowFlakeIdGenerator = new SnowFlakeIdGenerator(DATA_CENTER_ID, id); + String ip = NetUtils.getLocalHost(); - return new SnowFlakeIdGenerator(DATA_CENTER_ID, id); - }); + log.info("[IdGenerateService] init snowflake for server(address={}) by machineId({}).", ip, id); } /** diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/id/ServerIdProvider.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/id/ServerIdProvider.java new file mode 100644 index 00000000..f7fcbd2e --- /dev/null +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/id/ServerIdProvider.java @@ -0,0 +1,12 @@ +package com.github.kfcfans.powerjob.server.service.id; + +/** + * @author user + */ +public interface ServerIdProvider { + /** + * get number for IdGenerateService + * @return serverId + */ + long serverId(); +} diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/id/ServerIdProviderConfiguration.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/id/ServerIdProviderConfiguration.java new file mode 100644 index 00000000..e2e6f5c4 --- /dev/null +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/id/ServerIdProviderConfiguration.java @@ -0,0 +1,25 @@ +package com.github.kfcfans.powerjob.server.service.id; + +import com.github.kfcfans.powerjob.server.persistence.core.repository.ServerInfoRepository; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * @author user + */ +@Configuration +public class ServerIdProviderConfiguration { + @ConditionalOnProperty(prefix = "oms.server-id", name = "provider", havingValue = "hostname") + @Bean(name = "serverIdProvider") + public ServerIdProvider statefulSetServerIdProvider() { + return new StatefulSetServerIdProvider(); + } + + @ConditionalOnMissingBean(ServerIdProvider.class) + @Bean(name = "serverIdProvider") + public ServerIdProvider defaultServerIdProvider(ServerInfoRepository serverInfoRepository) { + return new DefaultServerIdProvider(serverInfoRepository); + } +} diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/id/StatefulSetServerIdProvider.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/id/StatefulSetServerIdProvider.java new file mode 100644 index 00000000..9e796e36 --- /dev/null +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/id/StatefulSetServerIdProvider.java @@ -0,0 +1,30 @@ +package com.github.kfcfans.powerjob.server.service.id; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @author user + */ +public class StatefulSetServerIdProvider implements ServerIdProvider { + /** + * xxx-1,aa-bb-2 + */ + private static final Pattern HOSTNAME_PATTERN = Pattern.compile("^.*-([0-9]+)$"); + + @Override + public long serverId() { + try { + String hostname = InetAddress.getLocalHost().getHostName(); + Matcher matcher = HOSTNAME_PATTERN.matcher(hostname); + if (matcher.matches()) { + return Long.parseLong(matcher.group(1)); + } + throw new RuntimeException(String.format("hostname=%s not match %s", hostname, HOSTNAME_PATTERN.toString())); + } catch (UnknownHostException e) { + throw new RuntimeException(e); + } + } +}