[dev] support instance status query

This commit is contained in:
tjq 2020-06-24 11:29:19 +08:00
parent 53a99241e0
commit 44445901da
13 changed files with 131 additions and 71 deletions

View File

@ -2,14 +2,14 @@
Navicat Premium Data Transfer
Source Server Type : MySQL
Source Server Version : 50724
Source Schema : oms-product
Source Server Version : 80020
Source Schema : powerjob-product
Target Server Type : MySQL
Target Server Version : 50724
Target Server Version : 80020
File Encoding : 65001
Date: 07/06/2020 11:11:47
Date: 23/06/2020 22:30:06
*/
SET NAMES utf8mb4;
@ -20,130 +20,131 @@ SET FOREIGN_KEY_CHECKS = 0;
-- ----------------------------
DROP TABLE IF EXISTS `app_info`;
CREATE TABLE `app_info` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`id` bigint NOT NULL AUTO_INCREMENT,
`app_name` varchar(255) DEFAULT NULL,
`current_server` varchar(255) DEFAULT NULL,
`description` varchar(255) DEFAULT NULL,
`gmt_create` datetime(6) DEFAULT NULL,
`gmt_modified` datetime(6) DEFAULT NULL,
`password` varchar(255) DEFAULT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `appNameUK` (`app_name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-- ----------------------------
-- Table structure for container_info
-- ----------------------------
DROP TABLE IF EXISTS `container_info`;
CREATE TABLE `container_info` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`app_id` bigint(20) DEFAULT NULL,
`id` bigint NOT NULL AUTO_INCREMENT,
`app_id` bigint DEFAULT NULL,
`container_name` varchar(255) DEFAULT NULL,
`gmt_create` datetime(6) DEFAULT NULL,
`gmt_modified` datetime(6) DEFAULT NULL,
`last_deploy_time` datetime(6) DEFAULT NULL,
`source_info` varchar(255) DEFAULT NULL,
`source_type` int(11) DEFAULT NULL,
`status` int(11) DEFAULT NULL,
`source_type` int DEFAULT NULL,
`status` int DEFAULT NULL,
`version` varchar(255) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `IDX8hixyaktlnwil2w9up6b0p898` (`app_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-- ----------------------------
-- Table structure for instance_info
-- ----------------------------
DROP TABLE IF EXISTS `instance_info`;
CREATE TABLE `instance_info` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`actual_trigger_time` bigint(20) DEFAULT NULL,
`app_id` bigint(20) DEFAULT NULL,
`expected_trigger_time` bigint(20) DEFAULT NULL,
`finished_time` bigint(20) DEFAULT NULL,
`id` bigint NOT NULL AUTO_INCREMENT,
`actual_trigger_time` bigint DEFAULT NULL,
`app_id` bigint DEFAULT NULL,
`expected_trigger_time` bigint DEFAULT NULL,
`finished_time` bigint DEFAULT NULL,
`gmt_create` datetime(6) DEFAULT NULL,
`gmt_modified` datetime(6) DEFAULT NULL,
`instance_id` bigint(20) DEFAULT NULL,
`instance_id` bigint DEFAULT NULL,
`instance_params` text,
`job_id` bigint(20) DEFAULT NULL,
`job_id` bigint DEFAULT NULL,
`last_report_time` bigint DEFAULT NULL,
`result` text,
`running_times` bigint(20) DEFAULT NULL,
`status` int(11) DEFAULT NULL,
`running_times` bigint DEFAULT NULL,
`status` int DEFAULT NULL,
`task_tracker_address` varchar(255) DEFAULT NULL,
`type` int(11) DEFAULT NULL,
`wf_instance_id` bigint(20) DEFAULT NULL,
`type` int DEFAULT NULL,
`wf_instance_id` bigint DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `IDX5b1nhpe5je7gc5s1ur200njr7` (`job_id`),
KEY `IDXjnji5lrr195kswk6f7mfhinrs` (`app_id`),
KEY `IDXa98hq3yu0l863wuotdjl7noum` (`instance_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-- ----------------------------
-- Table structure for job_info
-- ----------------------------
DROP TABLE IF EXISTS `job_info`;
CREATE TABLE `job_info` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`app_id` bigint(20) DEFAULT NULL,
`concurrency` int(11) DEFAULT NULL,
`id` bigint NOT NULL AUTO_INCREMENT,
`app_id` bigint DEFAULT NULL,
`concurrency` int DEFAULT NULL,
`designated_workers` varchar(255) DEFAULT NULL,
`execute_type` int(11) DEFAULT NULL,
`execute_type` int DEFAULT NULL,
`gmt_create` datetime(6) DEFAULT NULL,
`gmt_modified` datetime(6) DEFAULT NULL,
`instance_retry_num` int(11) DEFAULT NULL,
`instance_time_limit` bigint(20) DEFAULT NULL,
`instance_retry_num` int DEFAULT NULL,
`instance_time_limit` bigint DEFAULT NULL,
`job_description` varchar(255) DEFAULT NULL,
`job_name` varchar(255) DEFAULT NULL,
`job_params` varchar(255) DEFAULT NULL,
`max_instance_num` int(11) DEFAULT NULL,
`max_worker_count` int(11) DEFAULT NULL,
`max_instance_num` int DEFAULT NULL,
`max_worker_count` int DEFAULT NULL,
`min_cpu_cores` double NOT NULL,
`min_disk_space` double NOT NULL,
`min_memory_space` double NOT NULL,
`next_trigger_time` bigint(20) DEFAULT NULL,
`next_trigger_time` bigint DEFAULT NULL,
`notify_user_ids` varchar(255) DEFAULT NULL,
`processor_info` text,
`processor_type` int(11) DEFAULT NULL,
`status` int(11) DEFAULT NULL,
`task_retry_num` int(11) DEFAULT NULL,
`processor_type` int DEFAULT NULL,
`status` int DEFAULT NULL,
`task_retry_num` int DEFAULT NULL,
`time_expression` varchar(255) DEFAULT NULL,
`time_expression_type` int(11) DEFAULT NULL,
`time_expression_type` int DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `IDXk2xprmn3lldmlcb52i36udll1` (`app_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-- ----------------------------
-- Table structure for oms_lock
-- ----------------------------
DROP TABLE IF EXISTS `oms_lock`;
CREATE TABLE `oms_lock` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`id` bigint NOT NULL AUTO_INCREMENT,
`gmt_create` datetime(6) DEFAULT NULL,
`gmt_modified` datetime(6) DEFAULT NULL,
`lock_name` varchar(255) DEFAULT NULL,
`max_lock_time` bigint(20) DEFAULT NULL,
`max_lock_time` bigint DEFAULT NULL,
`ownerip` varchar(255) DEFAULT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `lockNameUK` (`lock_name`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-- ----------------------------
-- Table structure for server_info
-- ----------------------------
DROP TABLE IF EXISTS `server_info`;
CREATE TABLE `server_info` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`id` bigint NOT NULL AUTO_INCREMENT,
`gmt_create` datetime(6) DEFAULT NULL,
`gmt_modified` datetime(6) DEFAULT NULL,
`ip` varchar(255) DEFAULT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `UKtk8ytgpl7mpukhnvhbl82kgvy` (`ip`)
) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4;
) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-- ----------------------------
-- Table structure for user_info
-- ----------------------------
DROP TABLE IF EXISTS `user_info`;
CREATE TABLE `user_info` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`id` bigint NOT NULL AUTO_INCREMENT,
`email` varchar(255) DEFAULT NULL,
`gmt_create` datetime(6) DEFAULT NULL,
`gmt_modified` datetime(6) DEFAULT NULL,
@ -151,47 +152,47 @@ CREATE TABLE `user_info` (
`phone` varchar(255) DEFAULT NULL,
`username` varchar(255) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-- ----------------------------
-- Table structure for workflow_info
-- ----------------------------
DROP TABLE IF EXISTS `workflow_info`;
CREATE TABLE `workflow_info` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`app_id` bigint(20) DEFAULT NULL,
`id` bigint NOT NULL AUTO_INCREMENT,
`app_id` bigint DEFAULT NULL,
`gmt_create` datetime(6) DEFAULT NULL,
`gmt_modified` datetime(6) DEFAULT NULL,
`max_wf_instance_num` int(11) DEFAULT NULL,
`next_trigger_time` bigint(20) DEFAULT NULL,
`max_wf_instance_num` int DEFAULT NULL,
`next_trigger_time` bigint DEFAULT NULL,
`notify_user_ids` varchar(255) DEFAULT NULL,
`pedag` text,
`status` int(11) DEFAULT NULL,
`status` int DEFAULT NULL,
`time_expression` varchar(255) DEFAULT NULL,
`time_expression_type` int(11) DEFAULT NULL,
`time_expression_type` int DEFAULT NULL,
`wf_description` varchar(255) DEFAULT NULL,
`wf_name` varchar(255) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `IDX7uo5w0e3beeho3fnx9t7eiol3` (`app_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-- ----------------------------
-- Table structure for workflow_instance_info
-- ----------------------------
DROP TABLE IF EXISTS `workflow_instance_info`;
CREATE TABLE `workflow_instance_info` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`actual_trigger_time` bigint(20) DEFAULT NULL,
`app_id` bigint(20) DEFAULT NULL,
`id` bigint NOT NULL AUTO_INCREMENT,
`actual_trigger_time` bigint DEFAULT NULL,
`app_id` bigint DEFAULT NULL,
`dag` text,
`finished_time` bigint(20) DEFAULT NULL,
`finished_time` bigint DEFAULT NULL,
`gmt_create` datetime(6) DEFAULT NULL,
`gmt_modified` datetime(6) DEFAULT NULL,
`result` text,
`status` int(11) DEFAULT NULL,
`wf_instance_id` bigint(20) DEFAULT NULL,
`workflow_id` bigint(20) DEFAULT NULL,
`status` int DEFAULT NULL,
`wf_instance_id` bigint DEFAULT NULL,
`workflow_id` bigint DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
SET FOREIGN_KEY_CHECKS = 1;

View File

@ -34,6 +34,6 @@ echo "================== 准备启动 powerjob-agent =================="
serverIP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' powerjob-server)
serverAddress="$serverIP:7700"
echo "使用的Server地址$serverAddress"
docker run -d -e PARAMS="--app powerjob-agent-test --server $serverAddress" -p 27777:27777 --name powerjob-agent -v ~/docker/powerjob-agent:/root tjqq/powerjob-agent:latest
docker run -d -e PARAMS="--app powerjob-agent-test --server $serverAddress" -p 27778:27777 --name powerjob-agent2 -v ~/docker/powerjob-agent2:/root tjqq/powerjob-agent:latest
docker run -d --restart=always -e PARAMS="--app powerjob-agent-test --server $serverAddress" -p 27777:27777 --name powerjob-agent -v ~/docker/powerjob-agent:/root tjqq/powerjob-agent:latest
docker run -d --restart=always -e PARAMS="--app powerjob-agent-test --server $serverAddress" -p 27778:27777 --name powerjob-agent2 -v ~/docker/powerjob-agent2:/root tjqq/powerjob-agent:latest

View File

@ -1,4 +1,4 @@
# 基础镜像(支持 amd64 & arm64
# 基础镜像(支持 amd64 & arm64based on Ubuntu 18.04.4 LTS
FROM adoptopenjdk:8-jdk-hotspot
# 维护者
MAINTAINER tengjiqi@gmail.com
@ -11,8 +11,9 @@ COPY settings.xml /opt/powerjob-maven/conf/settings.xml
# 设置 maven 环境变量maven invoker 读取该变量调用 maven
ENV M2_HOME=/opt/powerjob-maven
# 设置时区Debian专用方法
# 设置时区
ENV TZ=Asia/Shanghai
RUN ln -sf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
# 设置其他环境变量
ENV APP_NAME=powerjob-server

View File

@ -49,7 +49,7 @@ public class ClusterStatusHolder {
Long oldTime = address2ActiveTime.getOrDefault(workerAddress, -1L);
if (heartbeatTime < oldTime) {
log.warn("[ClusterStatusHolder-{}] receive the old heartbeat: {}.", appName, heartbeat);
log.warn("[ClusterStatusHolder-{}] receive the expired heartbeat from {}, serverTime: {}, heartTime: {}", appName, heartbeat.getWorkerAddress(), System.currentTimeMillis(), heartbeat.getHeartbeatTime());
return;
}

View File

@ -1,5 +1,6 @@
package com.github.kfcfans.powerjob.server.web.controller;
import com.github.kfcfans.powerjob.common.InstanceStatus;
import com.github.kfcfans.powerjob.common.model.InstanceDetail;
import com.github.kfcfans.powerjob.common.response.ResultDTO;
import com.github.kfcfans.powerjob.server.akka.OhMyServer;
@ -21,6 +22,7 @@ import org.springframework.data.domain.Example;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.util.StringUtils;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
@ -112,6 +114,10 @@ public class InstanceController {
BeanUtils.copyProperties(request, queryEntity);
queryEntity.setType(request.getType().getV());
if (!StringUtils.isEmpty(request.getStatus())) {
queryEntity.setStatus(InstanceStatus.valueOf(request.getStatus()).getV());
}
Page<InstanceInfoDO> pageResult = instanceInfoRepository.findAll(Example.of(queryEntity), pageable);
return ResultDTO.success(convertPage(pageResult));
}

View File

@ -3,6 +3,7 @@ package com.github.kfcfans.powerjob.server.web.controller;
import akka.actor.ActorSelection;
import akka.pattern.Patterns;
import com.github.kfcfans.powerjob.common.InstanceStatus;
import com.github.kfcfans.powerjob.common.OmsConstant;
import com.github.kfcfans.powerjob.common.RemoteConstant;
import com.github.kfcfans.powerjob.common.model.SystemMetrics;
import com.github.kfcfans.powerjob.common.response.AskResponse;
@ -18,6 +19,7 @@ import com.github.kfcfans.powerjob.server.web.response.SystemOverviewVO;
import com.github.kfcfans.powerjob.server.web.response.WorkerStatusVO;
import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.springframework.util.StringUtils;
import org.springframework.web.bind.annotation.GetMapping;
@ -103,8 +105,10 @@ public class SystemInfoController {
Date date = DateUtils.addDays(new Date(), -1);
overview.setFailedInstanceCount(instanceInfoRepository.countByAppIdAndStatusAndGmtCreateAfter(appId, InstanceStatus.FAILED.getV(), date));
// 服务器时区
overview.setTimezone(TimeZone.getDefault().getDisplayName());
// 服务器时间
overview.setServerTime(System.currentTimeMillis());
overview.setServerTime(DateFormatUtils.format(new Date(), OmsConstant.TIME_PATTERN));
return ResultDTO.success(overview);
}

View File

@ -1,5 +1,6 @@
package com.github.kfcfans.powerjob.server.web.controller;
import com.github.kfcfans.powerjob.common.WorkflowInstanceStatus;
import com.github.kfcfans.powerjob.common.response.ResultDTO;
import com.github.kfcfans.powerjob.server.persistence.PageResult;
import com.github.kfcfans.powerjob.server.persistence.core.model.WorkflowInstanceInfoDO;
@ -13,6 +14,7 @@ import org.springframework.data.domain.Example;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.util.StringUtils;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
@ -54,6 +56,11 @@ public class WorkflowInstanceController {
WorkflowInstanceInfoDO queryEntity = new WorkflowInstanceInfoDO();
BeanUtils.copyProperties(req, queryEntity);
if (!StringUtils.isEmpty(req.getStatus())) {
queryEntity.setStatus(WorkflowInstanceStatus.valueOf(req.getStatus()).getV());
}
Page<WorkflowInstanceInfoDO> ps = workflowInstanceInfoRepository.findAll(Example.of(queryEntity), pageable);
return ResultDTO.success(convertPage(ps));

View File

@ -24,4 +24,6 @@ public class QueryInstanceRequest {
private Long instanceId;
private Long jobId;
private Long wfInstanceId;
private String status;
}

View File

@ -21,4 +21,6 @@ public class QueryWorkflowInstanceRequest {
// 查询条件NORMAL/WORKFLOW
private Long wfInstanceId;
private Long workflowId;
private String status;
}

View File

@ -13,6 +13,8 @@ public class SystemOverviewVO {
private long jobCount;
private long runningInstanceCount;
private long failedInstanceCount;
// 服务器时区
private String timezone;
// 服务器时间
private long serverTime;
private String serverTime;
}

View File

@ -0,0 +1,34 @@
package com.github.kfcfans.powerjob.server.test;
import com.github.kfcfans.powerjob.common.OmsConstant;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.junit.jupiter.api.Test;
import java.util.Date;
import java.util.TimeZone;
/**
* 时区问题测试
*
* @author tjq
* @since 2020/6/24
*/
public class TimezoneTest {
@Test
public void testTimeZone() {
Date now = new Date();
System.out.println(now.toString());
System.out.println("timestamp before GMT: " + System.currentTimeMillis());
TimeZone.setDefault(TimeZone.getTimeZone("GMT"));
TimeZone timeZone = TimeZone.getDefault();
System.out.println(timeZone.getDisplayName());
System.out.println(new Date());
System.out.println(DateFormatUtils.format(new Date(), OmsConstant.TIME_PATTERN));
System.out.println("timestamp after GMT: " + System.currentTimeMillis());
}
}

View File

@ -3,6 +3,7 @@ FROM adoptopenjdk:8-jdk-hotspot
MAINTAINER tengjiqi@gmail.com
# 设置时区
ENV TZ=Asia/Shanghai
RUN ln -sf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
ENV APP_NAME=powerjob-worker-agent
ENV PARAMS=""

View File

@ -62,18 +62,18 @@ public class ServerDiscoveryService {
}
if (StringUtils.isEmpty(result)) {
log.warn("[OMS-ServerDiscoveryService] can't find any available server, this worker has been quarantined.");
log.warn("[OmsServerDiscovery] can't find any available server, this worker has been quarantined.");
// Server 高可用的前提下连续失败多次说明该节点与外界失联Server已经将秒级任务转移到其他Worker需要杀死本地的任务
if (FAILED_COUNT++ > MAX_FAILED_COUNT) {
log.error("[OMS-ServerDiscoveryService] can't find any available server for 3 consecutive times, It's time to kill all frequent job in this worker.");
log.warn("[OmsServerDiscovery] can't find any available server for 3 consecutive times, It's time to kill all frequent job in this worker.");
List<Long> frequentInstanceIds = TaskTrackerPool.getAllFrequentTaskTrackerKeys();
if (!CollectionUtils.isEmpty(frequentInstanceIds)) {
frequentInstanceIds.forEach(instanceId -> {
TaskTracker taskTracker = TaskTrackerPool.remove(instanceId);
taskTracker.destroy();
log.warn("[OMS-ServerDiscoveryService] kill frequent instance(instanceId={}) due to can't find any available server.", instanceId);
log.warn("[OmsServerDiscovery] kill frequent instance(instanceId={}) due to can't find any available server.", instanceId);
});
}
@ -83,7 +83,7 @@ public class ServerDiscoveryService {
}else {
// 重置失败次数
FAILED_COUNT = 0;
log.debug("[OMS-ServerDiscoveryService] current server is {}.", result);
log.debug("[OmsServerDiscovery] current server is {}.", result);
return result;
}
}