[dev] add index for table ContainerInfo and add instanceInfo timing clean

This commit is contained in:
tjq 2020-05-22 23:02:35 +08:00
parent d999f18f93
commit c213fca31d
9 changed files with 49 additions and 12 deletions

View File

@ -13,7 +13,7 @@ import java.util.Date;
*/
@Data
@Entity
@Table(name = "container_info", uniqueConstraints = {@UniqueConstraint(name = "containerNameUK", columnNames = {"containerName"})})
@Table(name = "container_info", indexes = {@Index(columnList = "appId")})
public class ContainerInfoDO {
@Id

View File

@ -74,4 +74,10 @@ public interface InstanceInfoRepository extends JpaRepository<InstanceInfoDO, Lo
@Query(value = "select job_id from instance_info where job_id in ?1 and status in ?2", nativeQuery = true)
List<Long> findByJobIdInAndStatusIn(List<Long> jobIds, List<Integer> status);
// 删除历史数据JPA自带的删除居然是根据ID循环删2000条数据删了几秒也太拉垮了吧...
// 结果只能用 int 接受
@Modifying
@Transactional
@Query(value = "delete from instance_info where gmt_modified < ?1", nativeQuery = true)
int deleteAllByGmtModifiedBefore(Date time);
}

View File

@ -24,6 +24,7 @@ public interface LocalInstanceLogRepository extends JpaRepository<LocalInstanceL
@Transactional
long deleteByInstanceId(Long instanceId);
@Modifying
@Transactional
@CanIgnoreReturnValue
long deleteByInstanceIdInAndLogTimeLessThan(List<Long> instanceIds, Long t);

View File

@ -1,11 +1,13 @@
package com.github.kfcfans.oms.server.service.timing;
import com.github.kfcfans.oms.server.common.utils.OmsFileUtils;
import com.github.kfcfans.oms.server.persistence.core.repository.InstanceInfoRepository;
import com.github.kfcfans.oms.server.persistence.mongodb.GridFsManager;
import com.github.kfcfans.oms.server.service.ha.WorkerManagerService;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Stopwatch;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.time.DateUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.Scheduled;
@ -13,6 +15,7 @@ import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.io.File;
import java.util.Date;
/**
* CCOChief Clean Officer
@ -26,6 +29,8 @@ public class CleanService {
@Resource
private GridFsManager gridFsManager;
@Resource
private InstanceInfoRepository instanceInfoRepository;
@Value("${oms.log.retention.local}")
private int localLogRetentionDay;
@ -36,6 +41,9 @@ public class CleanService {
@Value("${oms.container.retention.remote}")
private int remoteContainerRetentionDay;
@Value("${oms.instanceinfo.retention}")
private int instanceInfoRetentionDay;
private static final int TEMPORARY_RETENTION_DAY = 3;
// 每天凌晨3点定时清理
@ -48,6 +56,8 @@ public class CleanService {
WorkerManagerService.releaseContainerInfos();
cleanInstanceLog();
cleanLocal(OmsFileUtils.genLogDirPath(), localLogRetentionDay);
cleanLocal(OmsFileUtils.genContainerJarPath(), localContainerRetentionDay);
cleanLocal(OmsFileUtils.genTemporaryPath(), TEMPORARY_RETENTION_DAY);
@ -106,4 +116,15 @@ public class CleanService {
}
}
@VisibleForTesting
public void cleanInstanceLog() {
try {
Date t = DateUtils.addDays(new Date(), -instanceInfoRetentionDay);
int num = instanceInfoRepository.deleteAllByGmtModifiedBefore(t);
log.info("[CleanService] deleted {} instanceInfo records whose modify time before {}.", num, t);
}catch (Exception e) {
log.warn("[CleanService] clean instanceInfo failed.", e);
}
}
}

View File

@ -20,8 +20,9 @@ spring.mail.properties.mail.smtp.auth=true
spring.mail.properties.mail.smtp.starttls.enable=true
spring.mail.properties.mail.smtp.starttls.required=true
####### 日志保留天数,单位天 #######
####### 资源清理配置 #######
oms.log.retention.local=0
oms.log.retention.remote=0
oms.container.retention.local=0
oms.container.retention.remote=0
oms.container.retention.remote=0
oms.instanceinfo.retention=0

View File

@ -20,8 +20,9 @@ spring.mail.properties.mail.smtp.auth=true
spring.mail.properties.mail.smtp.starttls.enable=true
spring.mail.properties.mail.smtp.starttls.required=true
####### 日志保留天数,单位天 #######
####### 资源清理配置 #######
oms.log.retention.local=3
oms.log.retention.remote=3
oms.container.retention.local=3
oms.container.retention.remote=3
oms.container.retention.remote=3
oms.instanceinfo.retention=3

View File

@ -20,8 +20,9 @@ spring.mail.properties.mail.smtp.auth=true
spring.mail.properties.mail.smtp.starttls.enable=true
spring.mail.properties.mail.smtp.starttls.required=true
####### 日志保留天数,单位天 #######
####### 资源清理配置 #######
oms.log.retention.local=7
oms.log.retention.remote=7
oms.container.retention.local=7
oms.container.retention.remote=7
oms.container.retention.remote=7
oms.instanceinfo.retention=3

View File

@ -2,15 +2,13 @@ package com.github.kfcfans.oms.server.test;
import com.github.kfcfans.oms.server.service.id.IdGenerateService;
import com.github.kfcfans.oms.server.service.lock.LockService;
import org.assertj.core.util.Lists;
import com.github.kfcfans.oms.server.service.timing.CleanService;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.junit4.SpringRunner;
import javax.annotation.Resource;
import java.util.List;
/**
* 服务测试
@ -18,7 +16,7 @@ import java.util.List;
* @author tjq
* @since 2020/4/2
*/
@ActiveProfiles("daily")
//@ActiveProfiles("daily")
@RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
public class ServiceTest {
@ -27,6 +25,8 @@ public class ServiceTest {
private LockService lockService;
@Resource
private IdGenerateService idGenerateService;
@Resource
private CleanService cleanService;
@Test
public void testLockService() {
@ -42,4 +42,9 @@ public class ServiceTest {
System.out.println(idGenerateService.allocate());
}
@Test
public void testCleanInstanceInfo() {
cleanService.cleanInstanceLog();
}
}

View File

@ -32,4 +32,5 @@ oms.alarm.bean.names=omsDefaultMailAlarmService
oms.log.retention.local=0
oms.log.retention.remote=0
oms.container.retention.local=0
oms.container.retention.remote=0
oms.container.retention.remote=0
oms.instanceinfo.retention=0;