From 0e77a23e76e0e30f6839cfe4d814350fefb13e57 Mon Sep 17 00:00:00 2001 From: ocean23 <43363687@qq.com> Date: Fri, 1 Jan 2021 14:17:54 +0800 Subject: [PATCH 1/5] =?UTF-8?q?fix=E5=B9=B6=E5=8F=91=E6=83=85=E5=86=B5?= =?UTF-8?q?=E4=B8=8Bserver=E7=AB=AF=E5=88=A0=E9=99=A4=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E6=97=B6=E6=95=B0=E6=8D=AE=E5=B7=B2=E7=BB=8F=E8=A2=AB=E5=85=B6?= =?UTF-8?q?=E4=BB=96server=E5=88=A0=E9=99=A4=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../persistence/mongodb/GridFsManager.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/mongodb/GridFsManager.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/mongodb/GridFsManager.java index 71d7d8b8..da4f8449 100644 --- a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/mongodb/GridFsManager.java +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/mongodb/GridFsManager.java @@ -1,6 +1,7 @@ package com.github.kfcfans.powerjob.server.persistence.mongodb; import com.github.kfcfans.powerjob.server.common.PowerJobServerConfigKey; +import com.github.kfcfans.powerjob.server.service.lock.LockService; import com.google.common.base.Stopwatch; import com.google.common.collect.Maps; import com.mongodb.client.MongoDatabase; @@ -38,6 +39,8 @@ public class GridFsManager implements InitializingBean { @Resource private Environment environment; + @Resource + private LockService lockService; private MongoDatabase db; private boolean available; @@ -107,9 +110,22 @@ public class GridFsManager implements InitializingBean { * @param day 日期偏移量，单位 天 */ public void deleteBefore(String bucketName, int day) { + String deleteFsLock = "deleteFsLock"; + // 只要第一个server抢到锁其他server就会返回，所以锁10分钟应该足够了 + boolean lock = lockService.lock(deleteFsLock, 10 * 60 * 1000); + if (!lock) { + log.info("[GridFsManager] deleted task is running, it's ok to return."); + return; + } + try{ + deleteHistoryFile(bucketName, day); + }finally { + lockService.unlock(deleteFsLock); + } + } + private void deleteHistoryFile(String bucketName, int day) { Stopwatch sw = Stopwatch.createStarted(); - Date date = DateUtils.addDays(new Date(), -day); GridFSBucket bucket = getBucket(bucketName); Bson filter = Filters.lt("uploadDate", date); From ead9f08e5285ff3ee8bc9b5c0a181662637ad0f5 Mon Sep 17 00:00:00 2001 From: ocean23 <43363687@qq.com> Date: Sat, 2 Jan 2021 12:38:19 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=E5=9B=9E=E6=BB=9AGridFsManager=E7=9A=84?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E6=8A=8A=E9=94=81=E5=8A=A0=E5=9C=A8CleanServ?= =?UTF-8?q?ice=E4=B8=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../persistence/mongodb/GridFsManager.java | 18 +----------------- .../server/service/timing/CleanService.java | 12 ++++++++++++ 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/mongodb/GridFsManager.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/mongodb/GridFsManager.java index da4f8449..71d7d8b8 100644 --- a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/mongodb/GridFsManager.java +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/persistence/mongodb/GridFsManager.java @@ -1,7 +1,6 @@ package com.github.kfcfans.powerjob.server.persistence.mongodb; import com.github.kfcfans.powerjob.server.common.PowerJobServerConfigKey; -import com.github.kfcfans.powerjob.server.service.lock.LockService; import com.google.common.base.Stopwatch; import com.google.common.collect.Maps; import com.mongodb.client.MongoDatabase; @@ -39,8 +38,6 @@ public class GridFsManager implements InitializingBean { @Resource private Environment environment; - @Resource - private LockService lockService; private MongoDatabase db; private boolean available; @@ -110,22 +107,9 @@ public class GridFsManager implements InitializingBean { * @param day 日期偏移量，单位 天 */ public void deleteBefore(String bucketName, int day) { - String deleteFsLock = "deleteFsLock"; - // 只要第一个server抢到锁其他server就会返回，所以锁10分钟应该足够了 - boolean lock = lockService.lock(deleteFsLock, 10 * 60 * 1000); - if (!lock) { - log.info("[GridFsManager] deleted task is running, it's ok to return."); - return; - } - try{ - deleteHistoryFile(bucketName, day); - }finally { - lockService.unlock(deleteFsLock); - } - } - private void deleteHistoryFile(String bucketName, int day) { Stopwatch sw = Stopwatch.createStarted(); + Date date = DateUtils.addDays(new Date(), -day); GridFSBucket bucket = getBucket(bucketName); Bson filter = Filters.lt("uploadDate", date); diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/timing/CleanService.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/timing/CleanService.java index f180ce3d..837917fe 100644 --- a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/timing/CleanService.java +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/timing/CleanService.java @@ -7,6 +7,7 @@ import com.github.kfcfans.powerjob.server.persistence.core.repository.InstanceIn import com.github.kfcfans.powerjob.server.persistence.core.repository.WorkflowInstanceInfoRepository; import com.github.kfcfans.powerjob.server.persistence.mongodb.GridFsManager; import com.github.kfcfans.powerjob.server.service.ha.WorkerManagerService; +import com.github.kfcfans.powerjob.server.service.lock.LockService; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Stopwatch; import lombok.extern.slf4j.Slf4j; @@ -36,6 +37,8 @@ public class CleanService { private InstanceInfoRepository instanceInfoRepository; @Resource private WorkflowInstanceInfoRepository workflowInstanceInfoRepository; + @Resource + private LockService lockService; @Value("${oms.instanceinfo.retention}") private int instanceInfoRetentionDay; @@ -113,11 +116,20 @@ public class CleanService { return; } if (gridFsManager.available()) { + String deleteFsLock = "deleteFsLock"; + // 只要第一个server抢到锁其他server就会返回，所以锁10分钟应该足够了 + boolean lock = lockService.lock(deleteFsLock, 10 * 60 * 1000); + if (!lock) { + log.info("[GridFsManager] deleted task is running, it's ok to return."); + return; + } Stopwatch stopwatch = Stopwatch.createStarted(); try { gridFsManager.deleteBefore(bucketName, day); }catch (Exception e) { log.warn("[CleanService] clean remote bucket({}) failed.", bucketName, e); + }finally { + lockService.unlock(deleteFsLock); } log.info("[CleanService] clean remote bucket({}) successfully, using {}.", bucketName, stopwatch.stop()); } From 05181c34ecca4fde1976442b30440bff7105de2c Mon Sep 17 00:00:00 2001 From: ocean23 <43363687@qq.com> Date: Sat, 2 Jan 2021 12:39:36 +0800 Subject: [PATCH 3/5] =?UTF-8?q?=E5=9B=9E=E6=BB=9AGridFsManager=E7=9A=84?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E6=8A=8A=E9=94=81=E5=8A=A0=E5=9C=A8CleanServ?= =?UTF-8?q?ice=E4=B8=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kfcfans/powerjob/server/service/timing/CleanService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/timing/CleanService.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/timing/CleanService.java index 837917fe..889fb207 100644 --- a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/timing/CleanService.java +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/timing/CleanService.java @@ -120,7 +120,7 @@ public class CleanService { // 只要第一个server抢到锁其他server就会返回，所以锁10分钟应该足够了 boolean lock = lockService.lock(deleteFsLock, 10 * 60 * 1000); if (!lock) { - log.info("[GridFsManager] deleted task is running, it's ok to return."); + log.info("[GridFsManager] deleted task is running, just return."); return; } Stopwatch stopwatch = Stopwatch.createStarted(); From 0daa097d20e8a657f8b3c2b9056a5989453ac893 Mon Sep 17 00:00:00 2001 From: ocean23 <43363687@qq.com> Date: Sat, 2 Jan 2021 13:28:55 +0800 Subject: [PATCH 4/5] =?UTF-8?q?=E6=A0=B9=E6=8D=AE=E6=84=8F=E8=A7=81?= =?UTF-8?q?=E9=87=8D=E6=9E=84=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../server/service/timing/CleanService.java | 42 ++++++++++++------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/timing/CleanService.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/timing/CleanService.java index 889fb207..274a5f59 100644 --- a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/timing/CleanService.java +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/timing/CleanService.java @@ -54,6 +54,8 @@ public class CleanService { // 每天凌晨3点定时清理 private static final String CLEAN_TIME_EXPRESSION = "0 0 3 * * ?"; + private static final String HISTORY_DELETE_LOCK = "history_delete"; + @Async("omsTimingPool") @Scheduled(cron = CLEAN_TIME_EXPRESSION) @@ -62,18 +64,35 @@ public class CleanService { // 释放本地缓存 WorkerManagerService.cleanUp(); - // 删除数据库运行记录 - cleanInstanceLog(); - cleanWorkflowInstanceLog(); - // 释放磁盘空间 cleanLocal(OmsFileUtils.genLogDirPath(), instanceInfoRetentionDay); cleanLocal(OmsFileUtils.genContainerJarPath(), localContainerRetentionDay); cleanLocal(OmsFileUtils.genTemporaryPath(), TEMPORARY_RETENTION_DAY); - // 删除 GridFS 过期文件 - cleanRemote(GridFsManager.LOG_BUCKET, instanceInfoRetentionDay); - cleanRemote(GridFsManager.CONTAINER_BUCKET, remoteContainerRetentionDay); + // 删除数据库历史的数据 + cleanOneServer(); + } + + /** + * 只能一台server清理的操作统一到这里执行 + */ + private void cleanOneServer() { + // 只要第一个server抢到锁其他server就会返回，所以锁10分钟应该足够了 + boolean lock = lockService.lock(HISTORY_DELETE_LOCK, 10 * 60 * 1000); + if (!lock) { + log.info("[GridFsManager] deleted task is running, just return."); + return; + } + try { + // 删除数据库运行记录 + cleanInstanceLog(); + cleanWorkflowInstanceLog(); + // 删除 GridFS 过期文件 + cleanRemote(GridFsManager.LOG_BUCKET, instanceInfoRetentionDay); + cleanRemote(GridFsManager.CONTAINER_BUCKET, remoteContainerRetentionDay); + }finally { + lockService.unlock(HISTORY_DELETE_LOCK); + } } @VisibleForTesting @@ -116,20 +135,11 @@ public class CleanService { return; } if (gridFsManager.available()) { - String deleteFsLock = "deleteFsLock"; - // 只要第一个server抢到锁其他server就会返回，所以锁10分钟应该足够了 - boolean lock = lockService.lock(deleteFsLock, 10 * 60 * 1000); - if (!lock) { - log.info("[GridFsManager] deleted task is running, just return."); - return; - } Stopwatch stopwatch = Stopwatch.createStarted(); try { gridFsManager.deleteBefore(bucketName, day); }catch (Exception e) { log.warn("[CleanService] clean remote bucket({}) failed.", bucketName, e); - }finally { - lockService.unlock(deleteFsLock); } log.info("[CleanService] clean remote bucket({}) successfully, using {}.", bucketName, stopwatch.stop()); } From 3776d4ad843585178926ae2748d4583b8cead1b8 Mon Sep 17 00:00:00 2001 From: ocean23 <43363687@qq.com> Date: Sat, 2 Jan 2021 13:30:46 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kfcfans/powerjob/server/service/timing/CleanService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/timing/CleanService.java b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/timing/CleanService.java index 274a5f59..71f80475 100644 --- a/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/timing/CleanService.java +++ b/powerjob-server/src/main/java/com/github/kfcfans/powerjob/server/service/timing/CleanService.java @@ -80,7 +80,7 @@ public class CleanService { // 只要第一个server抢到锁其他server就会返回，所以锁10分钟应该足够了 boolean lock = lockService.lock(HISTORY_DELETE_LOCK, 10 * 60 * 1000); if (!lock) { - log.info("[GridFsManager] deleted task is running, just return."); + log.info("task is already running, just return."); return; } try {