mirror of
https://github.com/PowerJob/PowerJob.git
synced 2025-07-17 00:00:04 +08:00
[dev] add index for table ContainerInfo and add instanceInfo timing clean
This commit is contained in:
parent
d999f18f93
commit
c213fca31d
@ -13,7 +13,7 @@ import java.util.Date;
|
||||
*/
|
||||
@Data
|
||||
@Entity
|
||||
@Table(name = "container_info", uniqueConstraints = {@UniqueConstraint(name = "containerNameUK", columnNames = {"containerName"})})
|
||||
@Table(name = "container_info", indexes = {@Index(columnList = "appId")})
|
||||
public class ContainerInfoDO {
|
||||
|
||||
@Id
|
||||
|
@ -74,4 +74,10 @@ public interface InstanceInfoRepository extends JpaRepository<InstanceInfoDO, Lo
|
||||
@Query(value = "select job_id from instance_info where job_id in ?1 and status in ?2", nativeQuery = true)
|
||||
List<Long> findByJobIdInAndStatusIn(List<Long> jobIds, List<Integer> status);
|
||||
|
||||
// 删除历史数据,JPA自带的删除居然是根据ID循环删,2000条数据删了几秒,也太拉垮了吧...
|
||||
// 结果只能用 int 接受
|
||||
@Modifying
|
||||
@Transactional
|
||||
@Query(value = "delete from instance_info where gmt_modified < ?1", nativeQuery = true)
|
||||
int deleteAllByGmtModifiedBefore(Date time);
|
||||
}
|
||||
|
@ -24,6 +24,7 @@ public interface LocalInstanceLogRepository extends JpaRepository<LocalInstanceL
|
||||
@Transactional
|
||||
long deleteByInstanceId(Long instanceId);
|
||||
|
||||
@Modifying
|
||||
@Transactional
|
||||
@CanIgnoreReturnValue
|
||||
long deleteByInstanceIdInAndLogTimeLessThan(List<Long> instanceIds, Long t);
|
||||
|
@ -1,11 +1,13 @@
|
||||
package com.github.kfcfans.oms.server.service.timing;
|
||||
|
||||
import com.github.kfcfans.oms.server.common.utils.OmsFileUtils;
|
||||
import com.github.kfcfans.oms.server.persistence.core.repository.InstanceInfoRepository;
|
||||
import com.github.kfcfans.oms.server.persistence.mongodb.GridFsManager;
|
||||
import com.github.kfcfans.oms.server.service.ha.WorkerManagerService;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Stopwatch;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.time.DateUtils;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.scheduling.annotation.Async;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
@ -13,6 +15,7 @@ import org.springframework.stereotype.Service;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.io.File;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* CCO(Chief Clean Officer)
|
||||
@ -26,6 +29,8 @@ public class CleanService {
|
||||
|
||||
@Resource
|
||||
private GridFsManager gridFsManager;
|
||||
@Resource
|
||||
private InstanceInfoRepository instanceInfoRepository;
|
||||
|
||||
@Value("${oms.log.retention.local}")
|
||||
private int localLogRetentionDay;
|
||||
@ -36,6 +41,9 @@ public class CleanService {
|
||||
@Value("${oms.container.retention.remote}")
|
||||
private int remoteContainerRetentionDay;
|
||||
|
||||
@Value("${oms.instanceinfo.retention}")
|
||||
private int instanceInfoRetentionDay;
|
||||
|
||||
private static final int TEMPORARY_RETENTION_DAY = 3;
|
||||
|
||||
// 每天凌晨3点定时清理
|
||||
@ -48,6 +56,8 @@ public class CleanService {
|
||||
|
||||
WorkerManagerService.releaseContainerInfos();
|
||||
|
||||
cleanInstanceLog();
|
||||
|
||||
cleanLocal(OmsFileUtils.genLogDirPath(), localLogRetentionDay);
|
||||
cleanLocal(OmsFileUtils.genContainerJarPath(), localContainerRetentionDay);
|
||||
cleanLocal(OmsFileUtils.genTemporaryPath(), TEMPORARY_RETENTION_DAY);
|
||||
@ -106,4 +116,15 @@ public class CleanService {
|
||||
}
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public void cleanInstanceLog() {
|
||||
try {
|
||||
Date t = DateUtils.addDays(new Date(), -instanceInfoRetentionDay);
|
||||
int num = instanceInfoRepository.deleteAllByGmtModifiedBefore(t);
|
||||
log.info("[CleanService] deleted {} instanceInfo records whose modify time before {}.", num, t);
|
||||
}catch (Exception e) {
|
||||
log.warn("[CleanService] clean instanceInfo failed.", e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -20,8 +20,9 @@ spring.mail.properties.mail.smtp.auth=true
|
||||
spring.mail.properties.mail.smtp.starttls.enable=true
|
||||
spring.mail.properties.mail.smtp.starttls.required=true
|
||||
|
||||
####### 日志保留天数,单位天 #######
|
||||
####### 资源清理配置 #######
|
||||
oms.log.retention.local=0
|
||||
oms.log.retention.remote=0
|
||||
oms.container.retention.local=0
|
||||
oms.container.retention.remote=0
|
||||
oms.container.retention.remote=0
|
||||
oms.instanceinfo.retention=0
|
@ -20,8 +20,9 @@ spring.mail.properties.mail.smtp.auth=true
|
||||
spring.mail.properties.mail.smtp.starttls.enable=true
|
||||
spring.mail.properties.mail.smtp.starttls.required=true
|
||||
|
||||
####### 日志保留天数,单位天 #######
|
||||
####### 资源清理配置 #######
|
||||
oms.log.retention.local=3
|
||||
oms.log.retention.remote=3
|
||||
oms.container.retention.local=3
|
||||
oms.container.retention.remote=3
|
||||
oms.container.retention.remote=3
|
||||
oms.instanceinfo.retention=3
|
@ -20,8 +20,9 @@ spring.mail.properties.mail.smtp.auth=true
|
||||
spring.mail.properties.mail.smtp.starttls.enable=true
|
||||
spring.mail.properties.mail.smtp.starttls.required=true
|
||||
|
||||
####### 日志保留天数,单位天 #######
|
||||
####### 资源清理配置 #######
|
||||
oms.log.retention.local=7
|
||||
oms.log.retention.remote=7
|
||||
oms.container.retention.local=7
|
||||
oms.container.retention.remote=7
|
||||
oms.container.retention.remote=7
|
||||
oms.instanceinfo.retention=3
|
@ -2,15 +2,13 @@ package com.github.kfcfans.oms.server.test;
|
||||
|
||||
import com.github.kfcfans.oms.server.service.id.IdGenerateService;
|
||||
import com.github.kfcfans.oms.server.service.lock.LockService;
|
||||
import org.assertj.core.util.Lists;
|
||||
import com.github.kfcfans.oms.server.service.timing.CleanService;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.test.context.ActiveProfiles;
|
||||
import org.springframework.test.context.junit4.SpringRunner;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 服务测试
|
||||
@ -18,7 +16,7 @@ import java.util.List;
|
||||
* @author tjq
|
||||
* @since 2020/4/2
|
||||
*/
|
||||
@ActiveProfiles("daily")
|
||||
//@ActiveProfiles("daily")
|
||||
@RunWith(SpringRunner.class)
|
||||
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
|
||||
public class ServiceTest {
|
||||
@ -27,6 +25,8 @@ public class ServiceTest {
|
||||
private LockService lockService;
|
||||
@Resource
|
||||
private IdGenerateService idGenerateService;
|
||||
@Resource
|
||||
private CleanService cleanService;
|
||||
|
||||
@Test
|
||||
public void testLockService() {
|
||||
@ -42,4 +42,9 @@ public class ServiceTest {
|
||||
System.out.println(idGenerateService.allocate());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCleanInstanceInfo() {
|
||||
cleanService.cleanInstanceLog();
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -32,4 +32,5 @@ oms.alarm.bean.names=omsDefaultMailAlarmService
|
||||
oms.log.retention.local=0
|
||||
oms.log.retention.remote=0
|
||||
oms.container.retention.local=0
|
||||
oms.container.retention.remote=0
|
||||
oms.container.retention.remote=0
|
||||
oms.instanceinfo.retention=0;
|
Loading…
x
Reference in New Issue
Block a user