diff --git a/oh-my-scheduler-server/src/main/java/com/github/kfcfans/oms/server/persistence/core/model/ContainerInfoDO.java b/oh-my-scheduler-server/src/main/java/com/github/kfcfans/oms/server/persistence/core/model/ContainerInfoDO.java index 77544d28..777cbc61 100644 --- a/oh-my-scheduler-server/src/main/java/com/github/kfcfans/oms/server/persistence/core/model/ContainerInfoDO.java +++ b/oh-my-scheduler-server/src/main/java/com/github/kfcfans/oms/server/persistence/core/model/ContainerInfoDO.java @@ -13,7 +13,7 @@ import java.util.Date; */ @Data @Entity -@Table(name = "container_info", uniqueConstraints = {@UniqueConstraint(name = "containerNameUK", columnNames = {"containerName"})}) +@Table(name = "container_info", indexes = {@Index(columnList = "appId")}) public class ContainerInfoDO { @Id diff --git a/oh-my-scheduler-server/src/main/java/com/github/kfcfans/oms/server/persistence/core/repository/InstanceInfoRepository.java b/oh-my-scheduler-server/src/main/java/com/github/kfcfans/oms/server/persistence/core/repository/InstanceInfoRepository.java index 307b8081..92692963 100644 --- a/oh-my-scheduler-server/src/main/java/com/github/kfcfans/oms/server/persistence/core/repository/InstanceInfoRepository.java +++ b/oh-my-scheduler-server/src/main/java/com/github/kfcfans/oms/server/persistence/core/repository/InstanceInfoRepository.java @@ -74,4 +74,10 @@ public interface InstanceInfoRepository extends JpaRepository findByJobIdInAndStatusIn(List jobIds, List status); + // 删除历史数据，JPA自带的删除居然是根据ID循环删，2000条数据删了几秒，也太拉垮了吧... + // 结果只能用 int 接受 + @Modifying + @Transactional + @Query(value = "delete from instance_info where gmt_modified < ?1", nativeQuery = true) + int deleteAllByGmtModifiedBefore(Date time); } diff --git a/oh-my-scheduler-server/src/main/java/com/github/kfcfans/oms/server/persistence/local/LocalInstanceLogRepository.java b/oh-my-scheduler-server/src/main/java/com/github/kfcfans/oms/server/persistence/local/LocalInstanceLogRepository.java index 994d1f50..539ce87f 100644 --- a/oh-my-scheduler-server/src/main/java/com/github/kfcfans/oms/server/persistence/local/LocalInstanceLogRepository.java +++ b/oh-my-scheduler-server/src/main/java/com/github/kfcfans/oms/server/persistence/local/LocalInstanceLogRepository.java @@ -24,6 +24,7 @@ public interface LocalInstanceLogRepository extends JpaRepository instanceIds, Long t); diff --git a/oh-my-scheduler-server/src/main/java/com/github/kfcfans/oms/server/service/timing/CleanService.java b/oh-my-scheduler-server/src/main/java/com/github/kfcfans/oms/server/service/timing/CleanService.java index 6146bc95..cdbefbb4 100644 --- a/oh-my-scheduler-server/src/main/java/com/github/kfcfans/oms/server/service/timing/CleanService.java +++ b/oh-my-scheduler-server/src/main/java/com/github/kfcfans/oms/server/service/timing/CleanService.java @@ -1,11 +1,13 @@ package com.github.kfcfans.oms.server.service.timing; import com.github.kfcfans.oms.server.common.utils.OmsFileUtils; +import com.github.kfcfans.oms.server.persistence.core.repository.InstanceInfoRepository; import com.github.kfcfans.oms.server.persistence.mongodb.GridFsManager; import com.github.kfcfans.oms.server.service.ha.WorkerManagerService; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Stopwatch; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.time.DateUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Async; import org.springframework.scheduling.annotation.Scheduled; @@ -13,6 +15,7 @@ import org.springframework.stereotype.Service; import javax.annotation.Resource; import java.io.File; +import java.util.Date; /** * CCO（Chief Clean Officer） @@ -26,6 +29,8 @@ public class CleanService { @Resource private GridFsManager gridFsManager; + @Resource + private InstanceInfoRepository instanceInfoRepository; @Value("${oms.log.retention.local}") private int localLogRetentionDay; @@ -36,6 +41,9 @@ public class CleanService { @Value("${oms.container.retention.remote}") private int remoteContainerRetentionDay; + @Value("${oms.instanceinfo.retention}") + private int instanceInfoRetentionDay; + private static final int TEMPORARY_RETENTION_DAY = 3; // 每天凌晨3点定时清理 @@ -48,6 +56,8 @@ public class CleanService { WorkerManagerService.releaseContainerInfos(); + cleanInstanceLog(); + cleanLocal(OmsFileUtils.genLogDirPath(), localLogRetentionDay); cleanLocal(OmsFileUtils.genContainerJarPath(), localContainerRetentionDay); cleanLocal(OmsFileUtils.genTemporaryPath(), TEMPORARY_RETENTION_DAY); @@ -106,4 +116,15 @@ public class CleanService { } } + @VisibleForTesting + public void cleanInstanceLog() { + try { + Date t = DateUtils.addDays(new Date(), -instanceInfoRetentionDay); + int num = instanceInfoRepository.deleteAllByGmtModifiedBefore(t); + log.info("[CleanService] deleted {} instanceInfo records whose modify time before {}.", num, t); + }catch (Exception e) { + log.warn("[CleanService] clean instanceInfo failed.", e); + } + } + } diff --git a/oh-my-scheduler-server/src/main/resources/application-daily.properties b/oh-my-scheduler-server/src/main/resources/application-daily.properties index 9cd0c105..efb61ca3 100644 --- a/oh-my-scheduler-server/src/main/resources/application-daily.properties +++ b/oh-my-scheduler-server/src/main/resources/application-daily.properties @@ -20,8 +20,9 @@ spring.mail.properties.mail.smtp.auth=true spring.mail.properties.mail.smtp.starttls.enable=true spring.mail.properties.mail.smtp.starttls.required=true -####### 日志保留天数，单位天 ####### +####### 资源清理配置 ####### oms.log.retention.local=0 oms.log.retention.remote=0 oms.container.retention.local=0 -oms.container.retention.remote=0 \ No newline at end of file +oms.container.retention.remote=0 +oms.instanceinfo.retention=0 \ No newline at end of file diff --git a/oh-my-scheduler-server/src/main/resources/application-pre.properties b/oh-my-scheduler-server/src/main/resources/application-pre.properties index 5d1ccf8a..3a2c5cee 100644 --- a/oh-my-scheduler-server/src/main/resources/application-pre.properties +++ b/oh-my-scheduler-server/src/main/resources/application-pre.properties @@ -20,8 +20,9 @@ spring.mail.properties.mail.smtp.auth=true spring.mail.properties.mail.smtp.starttls.enable=true spring.mail.properties.mail.smtp.starttls.required=true -####### 日志保留天数，单位天 ####### +####### 资源清理配置 ####### oms.log.retention.local=3 oms.log.retention.remote=3 oms.container.retention.local=3 -oms.container.retention.remote=3 \ No newline at end of file +oms.container.retention.remote=3 +oms.instanceinfo.retention=3 \ No newline at end of file diff --git a/oh-my-scheduler-server/src/main/resources/application-product.properties b/oh-my-scheduler-server/src/main/resources/application-product.properties index 9d19ea0c..3b41ce8e 100644 --- a/oh-my-scheduler-server/src/main/resources/application-product.properties +++ b/oh-my-scheduler-server/src/main/resources/application-product.properties @@ -20,8 +20,9 @@ spring.mail.properties.mail.smtp.auth=true spring.mail.properties.mail.smtp.starttls.enable=true spring.mail.properties.mail.smtp.starttls.required=true -####### 日志保留天数，单位天 ####### +####### 资源清理配置 ####### oms.log.retention.local=7 oms.log.retention.remote=7 oms.container.retention.local=7 -oms.container.retention.remote=7 \ No newline at end of file +oms.container.retention.remote=7 +oms.instanceinfo.retention=3 \ No newline at end of file diff --git a/oh-my-scheduler-server/src/test/java/com/github/kfcfans/oms/server/test/ServiceTest.java b/oh-my-scheduler-server/src/test/java/com/github/kfcfans/oms/server/test/ServiceTest.java index 2f6862b6..4248943e 100644 --- a/oh-my-scheduler-server/src/test/java/com/github/kfcfans/oms/server/test/ServiceTest.java +++ b/oh-my-scheduler-server/src/test/java/com/github/kfcfans/oms/server/test/ServiceTest.java @@ -2,15 +2,13 @@ package com.github.kfcfans.oms.server.test; import com.github.kfcfans.oms.server.service.id.IdGenerateService; import com.github.kfcfans.oms.server.service.lock.LockService; -import org.assertj.core.util.Lists; +import com.github.kfcfans.oms.server.service.timing.CleanService; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit4.SpringRunner; import javax.annotation.Resource; -import java.util.List; /** * 服务测试 @@ -18,7 +16,7 @@ import java.util.List; * @author tjq * @since 2020/4/2 */ -@ActiveProfiles("daily") +//@ActiveProfiles("daily") @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) public class ServiceTest { @@ -27,6 +25,8 @@ public class ServiceTest { private LockService lockService; @Resource private IdGenerateService idGenerateService; + @Resource + private CleanService cleanService; @Test public void testLockService() { @@ -42,4 +42,9 @@ public class ServiceTest { System.out.println(idGenerateService.allocate()); } + @Test + public void testCleanInstanceInfo() { + cleanService.cleanInstanceLog(); + } + } diff --git a/oh-my-scheduler-server/src/test/resources/application.properties b/oh-my-scheduler-server/src/test/resources/application.properties index 61cdfea3..6047d1a7 100644 --- a/oh-my-scheduler-server/src/test/resources/application.properties +++ b/oh-my-scheduler-server/src/test/resources/application.properties @@ -32,4 +32,5 @@ oms.alarm.bean.names=omsDefaultMailAlarmService oms.log.retention.local=0 oms.log.retention.remote=0 oms.container.retention.local=0 -oms.container.retention.remote=0 \ No newline at end of file +oms.container.retention.remote=0 +oms.instanceinfo.retention=0; \ No newline at end of file