feat: Implement external job import logging functionality with independent log files
This commit is contained in:
@@ -1,3 +1,7 @@
|
|||||||
|
# 外部岗位 Excel 全量同步运行日志。每次执行创建一份独立文件。
|
||||||
|
external-job-import:
|
||||||
|
run-log-directory: /data/code/shz-backend/logs/external-job-import
|
||||||
|
|
||||||
# 项目相关配置
|
# 项目相关配置
|
||||||
ruoyi:
|
ruoyi:
|
||||||
# 名称
|
# 名称
|
||||||
|
|||||||
@@ -130,6 +130,8 @@ external-job-import:
|
|||||||
internal-token: 8124757d8c37881b4b3d86c60332ca97aa97e6fd6a68db595bb38a1c72bc19dc
|
internal-token: 8124757d8c37881b4b3d86c60332ca97aa97e6fd6a68db595bb38a1c72bc19dc
|
||||||
connect-timeout-millis: 10000
|
connect-timeout-millis: 10000
|
||||||
read-timeout-millis: 120000
|
read-timeout-millis: 120000
|
||||||
|
# Quartz 每次全量同步生成独立 UTF-8 日志;相对路径以应用工作目录为基准。
|
||||||
|
run-log-directory: logs/external-job-import
|
||||||
max-invalid-row-ratio: 0.05
|
max-invalid-row-ratio: 0.05
|
||||||
max-delete-ratio: 0.30
|
max-delete-ratio: 0.30
|
||||||
required-sources:
|
required-sources:
|
||||||
|
|||||||
@@ -6,14 +6,18 @@ import com.ruoyi.cms.service.ICompanyService;
|
|||||||
import com.ruoyi.cms.service.IESJobSearchService;
|
import com.ruoyi.cms.service.IESJobSearchService;
|
||||||
import com.ruoyi.cms.service.IJobService;
|
import com.ruoyi.cms.service.IJobService;
|
||||||
import com.ruoyi.cms.service.IJobTitleSuggestService;
|
import com.ruoyi.cms.service.IJobTitleSuggestService;
|
||||||
|
import com.ruoyi.cms.externalimport.ExternalJobImportProperties;
|
||||||
|
import com.ruoyi.cms.externalimport.ExternalJobImportResult;
|
||||||
|
import com.ruoyi.cms.externalimport.ExternalJobImportRunLog;
|
||||||
import com.ruoyi.cms.externalimport.IExternalJobImportService;
|
import com.ruoyi.cms.externalimport.IExternalJobImportService;
|
||||||
import com.ruoyi.common.utils.spring.SpringUtils;
|
import com.ruoyi.common.utils.spring.SpringUtils;
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.slf4j.Logger;
|
||||||
import org.springframework.stereotype.Component;
|
import org.slf4j.LoggerFactory;
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
|
|
||||||
|
|
||||||
public class JobCron {
|
public class JobCron {
|
||||||
|
private static final Logger LOGGER = LoggerFactory.getLogger(JobCron.class);
|
||||||
|
|
||||||
public void isHot(){
|
public void isHot(){
|
||||||
SpringUtils.getBean(JobMapper.class).isHot();
|
SpringUtils.getBean(JobMapper.class).isHot();
|
||||||
}
|
}
|
||||||
@@ -39,9 +43,43 @@ public class JobCron {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* 外部 Excel 全量快照同步。
|
* 外部 Excel 全量快照同步。
|
||||||
* Quartz 调用目标:jobCron.importLatestExternalSnapshot()
|
* Quartz 调用目标:com.ruoyi.cms.cron.JobCron.importLatestExternalSnapshot()
|
||||||
*/
|
*/
|
||||||
public void importLatestExternalSnapshot(){
|
public void importLatestExternalSnapshot(){
|
||||||
SpringUtils.getBean(IExternalJobImportService.class).importLatest();
|
ExternalJobImportProperties properties = SpringUtils.getBean(ExternalJobImportProperties.class);
|
||||||
|
ExternalJobImportRunLog runLog = ExternalJobImportRunLog.open(properties.getRunLogDirectory());
|
||||||
|
LOGGER.info("外部岗位全量同步开始,runId={},独立日志={}", runLog.getRunId(), runLog.getFile());
|
||||||
|
try {
|
||||||
|
runLog.info("开始执行外部岗位 Excel 全量快照同步");
|
||||||
|
ExternalJobImportResult result = SpringUtils.getBean(IExternalJobImportService.class).importLatest();
|
||||||
|
runLog.info("执行完成;status=" + result.getStatus()
|
||||||
|
+ ";batchId=" + safe(result.getBatchId())
|
||||||
|
+ ";totalRows=" + result.getTotalRows()
|
||||||
|
+ ";validRows=" + result.getValidRows()
|
||||||
|
+ ";invalidRows=" + result.getInvalidRows()
|
||||||
|
+ ";duplicateRows=" + result.getDuplicateRows()
|
||||||
|
+ ";insertCount=" + result.getInsertCount()
|
||||||
|
+ ";updateCount=" + result.getUpdateCount()
|
||||||
|
+ ";keepCount=" + result.getKeepCount()
|
||||||
|
+ ";restoreCount=" + result.getRestoreCount()
|
||||||
|
+ ";deleteCount=" + result.getDeleteCount()
|
||||||
|
+ ";message=" + safe(result.getMessage()));
|
||||||
|
LOGGER.info("外部岗位全量同步结束,runId={},status={},batchId={}",
|
||||||
|
runLog.getRunId(), result.getStatus(), result.getBatchId());
|
||||||
|
} catch (RuntimeException ex) {
|
||||||
|
runLog.error("执行失败", ex);
|
||||||
|
LOGGER.error("外部岗位全量同步失败,runId={},独立日志={}", runLog.getRunId(), runLog.getFile(), ex);
|
||||||
|
throw ex;
|
||||||
|
} catch (Error error) {
|
||||||
|
runLog.error("执行发生 JVM 错误", error);
|
||||||
|
LOGGER.error("外部岗位全量同步发生 JVM 错误,runId={},独立日志={}", runLog.getRunId(), runLog.getFile(), error);
|
||||||
|
throw error;
|
||||||
|
} finally {
|
||||||
|
runLog.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String safe(String value) {
|
||||||
|
return value == null ? "" : value.replace('\r', ' ').replace('\n', ' ');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,6 +21,8 @@ public class ExternalJobImportProperties {
|
|||||||
private String internalToken;
|
private String internalToken;
|
||||||
private int connectTimeoutMillis = 10000;
|
private int connectTimeoutMillis = 10000;
|
||||||
private int readTimeoutMillis = 120000;
|
private int readTimeoutMillis = 120000;
|
||||||
|
/** 每次 Quartz 同步生成独立运行日志的目录;相对路径基于应用工作目录。 */
|
||||||
|
private String runLogDirectory = "logs/external-job-import";
|
||||||
private double maxInvalidRowRatio = 0.05D;
|
private double maxInvalidRowRatio = 0.05D;
|
||||||
private double maxDeleteRatio = 0.30D;
|
private double maxDeleteRatio = 0.30D;
|
||||||
private Set<String> requiredSources = new LinkedHashSet<>();
|
private Set<String> requiredSources = new LinkedHashSet<>();
|
||||||
|
|||||||
@@ -0,0 +1,107 @@
|
|||||||
|
package com.ruoyi.cms.externalimport;
|
||||||
|
|
||||||
|
import java.io.BufferedWriter;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.StringWriter;
|
||||||
|
import java.io.UncheckedIOException;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.nio.file.Paths;
|
||||||
|
import java.nio.file.StandardOpenOption;
|
||||||
|
import java.time.LocalDateTime;
|
||||||
|
import java.time.format.DateTimeFormatter;
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
|
/** 每一次外部岗位全量同步各自持有的 UTF-8 运行日志。 */
|
||||||
|
public final class ExternalJobImportRunLog implements AutoCloseable {
|
||||||
|
private static final DateTimeFormatter FILE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss-SSS");
|
||||||
|
private static final DateTimeFormatter LINE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS");
|
||||||
|
|
||||||
|
private final String runId;
|
||||||
|
private final Path file;
|
||||||
|
private final BufferedWriter writer;
|
||||||
|
private boolean closed;
|
||||||
|
|
||||||
|
private ExternalJobImportRunLog(String runId, Path file, BufferedWriter writer) {
|
||||||
|
this.runId = runId;
|
||||||
|
this.file = file;
|
||||||
|
this.writer = writer;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static ExternalJobImportRunLog open(String configuredDirectory) {
|
||||||
|
String directory = isBlank(configuredDirectory) ? "logs/external-job-import" : configuredDirectory.trim();
|
||||||
|
LocalDateTime startedAt = LocalDateTime.now();
|
||||||
|
String runId = UUID.randomUUID().toString().replace("-", "");
|
||||||
|
Path targetDirectory = Paths.get(directory).toAbsolutePath().normalize();
|
||||||
|
Path file = targetDirectory.resolve("external-job-import-" + FILE_TIME_FORMATTER.format(startedAt) + "-" + runId + ".log");
|
||||||
|
try {
|
||||||
|
Files.createDirectories(targetDirectory);
|
||||||
|
BufferedWriter writer = Files.newBufferedWriter(file, StandardCharsets.UTF_8,
|
||||||
|
StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE);
|
||||||
|
ExternalJobImportRunLog runLog = new ExternalJobImportRunLog(runId, file, writer);
|
||||||
|
runLog.info("运行日志已创建;开始时间=" + LINE_TIME_FORMATTER.format(startedAt));
|
||||||
|
return runLog;
|
||||||
|
} catch (IOException ex) {
|
||||||
|
throw new UncheckedIOException("无法创建外部岗位同步运行日志: " + file, ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getRunId() {
|
||||||
|
return runId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Path getFile() {
|
||||||
|
return file;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void info(String message) {
|
||||||
|
write("INFO", message);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void error(String message, Throwable throwable) {
|
||||||
|
write("ERROR", message);
|
||||||
|
if (throwable != null) {
|
||||||
|
StringWriter stackTrace = new StringWriter();
|
||||||
|
throwable.printStackTrace(new java.io.PrintWriter(stackTrace));
|
||||||
|
writeRaw(stackTrace.toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private synchronized void write(String level, String message) {
|
||||||
|
writeRaw(LINE_TIME_FORMATTER.format(LocalDateTime.now()) + " [" + level + "] [runId=" + runId + "] "
|
||||||
|
+ (message == null ? "" : message) + System.lineSeparator());
|
||||||
|
}
|
||||||
|
|
||||||
|
private synchronized void writeRaw(String content) {
|
||||||
|
if (closed) {
|
||||||
|
throw new IllegalStateException("运行日志已关闭: " + file);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
writer.write(content);
|
||||||
|
if (!content.endsWith(System.lineSeparator())) {
|
||||||
|
writer.newLine();
|
||||||
|
}
|
||||||
|
writer.flush();
|
||||||
|
} catch (IOException ex) {
|
||||||
|
throw new UncheckedIOException("无法写入外部岗位同步运行日志: " + file, ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public synchronized void close() {
|
||||||
|
if (closed) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
closed = true;
|
||||||
|
try {
|
||||||
|
writer.close();
|
||||||
|
} catch (IOException ex) {
|
||||||
|
throw new UncheckedIOException("无法关闭外部岗位同步运行日志: " + file, ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static boolean isBlank(String value) {
|
||||||
|
return value == null || value.trim().isEmpty();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
package com.ruoyi.cms.externalimport;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNotEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
class ExternalJobImportRunLogTest {
|
||||||
|
@Test
|
||||||
|
void createsIndependentUtf8LogForEachRun() throws Exception {
|
||||||
|
Path directory = Files.createTempDirectory("external-job-import-run-log-");
|
||||||
|
Path firstFile = null;
|
||||||
|
Path secondFile = null;
|
||||||
|
try {
|
||||||
|
try (ExternalJobImportRunLog first = ExternalJobImportRunLog.open(directory.toString())) {
|
||||||
|
first.info("第一份任务日志");
|
||||||
|
firstFile = first.getFile();
|
||||||
|
}
|
||||||
|
try (ExternalJobImportRunLog second = ExternalJobImportRunLog.open(directory.toString())) {
|
||||||
|
second.error("第二份任务日志异常", new IllegalStateException("test failure"));
|
||||||
|
secondFile = second.getFile();
|
||||||
|
}
|
||||||
|
|
||||||
|
assertTrue(Files.exists(firstFile));
|
||||||
|
assertTrue(Files.exists(secondFile));
|
||||||
|
assertNotEquals(firstFile, secondFile);
|
||||||
|
assertTrue(new String(Files.readAllBytes(firstFile), StandardCharsets.UTF_8).contains("第一份任务日志"));
|
||||||
|
assertTrue(new String(Files.readAllBytes(secondFile), StandardCharsets.UTF_8).contains("test failure"));
|
||||||
|
} finally {
|
||||||
|
if (firstFile != null) {
|
||||||
|
Files.deleteIfExists(firstFile);
|
||||||
|
}
|
||||||
|
if (secondFile != null) {
|
||||||
|
Files.deleteIfExists(secondFile);
|
||||||
|
}
|
||||||
|
Files.deleteIfExists(directory);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user