zkr/shz-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Implement external job import functionality

Browse Source 
- Add ExternalJobSourceAliasRequest class for managing source alias requests.
- Create ExternalUploadClient to interact with external Excel upload system.
- Introduce ExternalUploadFileMetadata to represent metadata of uploaded files.
- Define IExternalJobImportService interface for external job import operations.
- Update JobDataTrendServiceImpl to support dynamic source code filtering.
- Modify PublicJobFairMapper.xml to ensure proper query handling.
- Add unit tests for ExternalJobExcelParser and ExternalJobFingerprint.
- Create script for applying external job import migrations.
- Implement SQL migration for external job import, including new tables and columns.
This commit is contained in:
lapuda
2026-07-21 14:56:47 +08:00
parent fab6718a83
commit d206b25fcf
24 changed files with 2312 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
ruoyi-bussiness/src/main/java/com/ruoyi/cms/controller/cms/ExternalJobImportController.java Normal file
View File

@@ -0,0 +1,67 @@
package com.ruoyi.cms.controller.cms;
import com.ruoyi.cms.externalimport.ExternalJobImportResult;
import com.ruoyi.cms.externalimport.ExternalJobSourceAliasRequest;
import com.ruoyi.cms.externalimport.IExternalJobImportService;
import com.ruoyi.common.annotation.Log;
import com.ruoyi.common.core.controller.BaseController;
import com.ruoyi.common.core.domain.AjaxResult;
import com.ruoyi.common.enums.BusinessType;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
import java.util.Map;
/** 外部岗位导入批次监控和管理员手动触发接口。 */
@RestController
@RequestMapping("/cms/externalJobImport")
@Api(tags = "后台：外部岗位导入")
public class ExternalJobImportController extends BaseController {
private final IExternalJobImportService externalJobImportService;
public ExternalJobImportController(IExternalJobImportService externalJobImportService) {
this.externalJobImportService = externalJobImportService;
}
@GetMapping("/batches")
@ApiOperation("查询外部岗位导入批次")
@PreAuthorize("@ss.hasPermi('cms:recruit:monitor:list')")
public AjaxResult batches(@RequestParam(required = false) String status,
@RequestParam(required = false) Integer limit) {
List<Map<String, Object>> batches = externalJobImportService.listBatches(status, limit);
return success(batches);
}
@PostMapping("/run")
@ApiOperation("手动拉取并导入最新外部岗位 Excel")
@PreAuthorize("@ss.hasPermi('cms:recruit:import:manual')")
@Log(title = "手动导入外部岗位 Excel", businessType = BusinessType.IMPORT)
public AjaxResult run() {
ExternalJobImportResult result = externalJobImportService.importLatest();
return success(result);
}
@GetMapping("/sources")
@ApiOperation("查询外部 Excel 来源别名")
@PreAuthorize("@ss.hasPermi('cms:recruit:source:list')")
public AjaxResult sources() {
return success(externalJobImportService.listSourceAliases());
}
@PostMapping("/sources")
@ApiOperation("新增或修改外部 Excel 来源别名")
@PreAuthorize("@ss.hasPermi('cms:recruit:source:edit')")
@Log(title = "维护外部岗位来源别名", businessType = BusinessType.UPDATE)
public AjaxResult saveSource(@RequestBody ExternalJobSourceAliasRequest request) {
externalJobImportService.saveSourceAlias(request);
return success();
}
}

6
ruoyi-bussiness/src/main/java/com/ruoyi/cms/controller/cms/JobDataStorageDetectionController.java
View File

@@ -2,7 +2,6 @@ package com.ruoyi.cms.controller.cms;
import com.ruoyi.cms.domain.JobDataStorageDetection;
import com.ruoyi.cms.service.JobDataStorageDetectionService;
import com.ruoyi.common.annotation.Anonymous;
import com.ruoyi.common.annotation.Log;
import com.ruoyi.common.core.controller.BaseController;
import com.ruoyi.common.core.domain.AjaxResult;
@@ -10,6 +9,7 @@ import com.ruoyi.common.core.page.TableDataInfo;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.*;
import java.util.List;
@@ -20,7 +20,6 @@ import static com.ruoyi.common.enums.BusinessType.QUERY;
@RestController
@RequestMapping(value = "/cms/storageDetection")
@Api(tags = "后台：岗位数据入库检测")
@Anonymous
public class JobDataStorageDetectionController extends BaseController {
@Autowired
@@ -29,6 +28,7 @@ public class JobDataStorageDetectionController extends BaseController {
@Log(title = "查询岗位数据入库监测列表", businessType = QUERY)
@ApiOperation("查询岗位数据入库监测列表")
@GetMapping(value = "/getList")
@PreAuthorize("@ss.hasPermi('cms:recruit:monitor:list')")
public TableDataInfo getList(JobDataStorageDetection jobDataStorageDetection) {
startPage();
List<JobDataStorageDetection> list = jobDataStorageDetectionService.getList(jobDataStorageDetection);
@@ -38,6 +38,7 @@ public class JobDataStorageDetectionController extends BaseController {
@Log(title = "查询岗位数据入库监测详情", businessType = QUERY)
@ApiOperation("查询岗位数据入库监测详情")
@GetMapping(value = "/getSingle/{detectionId}")
@PreAuthorize("@ss.hasPermi('cms:recruit:monitor:list')")
public AjaxResult getSingle(@PathVariable String detectionId) {
return AjaxResult.success(jobDataStorageDetectionService.getSingle(detectionId));
}
@@ -45,6 +46,7 @@ public class JobDataStorageDetectionController extends BaseController {
@Log(title = "保存岗位数据入库监测记录", businessType = INSERT)
@ApiOperation("保存岗位数据入库监测记录")
@PostMapping(value = "/save")
@PreAuthorize("@ss.hasPermi('cms:recruit:monitor:edit')")
public AjaxResult save(@RequestBody JobDataStorageDetection jobDataStorageDetection) {
try {
jobDataStorageDetectionService.saveData(jobDataStorageDetection);

6
ruoyi-bussiness/src/main/java/com/ruoyi/cms/controller/cms/JobDataTrendController.java
View File

@@ -9,6 +9,7 @@ import com.ruoyi.common.enums.BusinessType;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
@@ -29,13 +30,14 @@ public class JobDataTrendController extends BaseController {
private IJobDataTrendService jobDataTrendService;
/**
* 查询全网采集趋势列表 (website_name = "total")
* 查询全网或单来源采集趋势列表。sourceCode 为空时查询全网（website_name = "total"）。
*/
@GetMapping("/list")
@ApiOperation("获取全网采集趋势数据(Total)")
@Log(title = "获取全网采集趋势数据(Total)", businessType = BusinessType.QUERY)
@PreAuthorize("@ss.hasPermi('cms:recruit:trend:list')")
public AjaxResult list(@RequestParam Map<String, String> params) {
List<JobDataTrend> list = jobDataTrendService.selectTrendTotalList(params);
return success(list);
}
}
}

10
ruoyi-bussiness/src/main/java/com/ruoyi/cms/controller/cms/WebsiteManagementController.java
View File

@@ -2,7 +2,6 @@ package com.ruoyi.cms.controller.cms;
import com.ruoyi.cms.domain.WebsiteManagement;
import com.ruoyi.cms.service.WebsiteManagementService;
import com.ruoyi.common.annotation.Anonymous;
import com.ruoyi.common.annotation.Log;
import com.ruoyi.common.core.controller.BaseController;
import com.ruoyi.common.core.domain.AjaxResult;
@@ -10,6 +9,7 @@ import com.ruoyi.common.core.page.TableDataInfo;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
@@ -20,7 +20,6 @@ import static com.ruoyi.common.enums.BusinessType.*;
@RestController
@RequestMapping("/cms/website")
@Api(tags = "后台：岗位信息来源管理")
@Anonymous
public class WebsiteManagementController extends BaseController {
@Autowired
@@ -29,6 +28,7 @@ public class WebsiteManagementController extends BaseController {
@Log(title = "查询岗位信息来源网站列表", businessType = QUERY)
@ApiOperation("查询岗位信息来源网站列表")
@GetMapping(value = "/getList")
@PreAuthorize("@ss.hasPermi('cms:recruit:source:list')")
public TableDataInfo getList(WebsiteManagement websiteManagement) {
startPage();
List<WebsiteManagement> list = websiteManagementService.getList(websiteManagement);
@@ -38,6 +38,7 @@ public class WebsiteManagementController extends BaseController {
@Log(title = "查询岗位信息来源网站详情", businessType = QUERY)
@ApiOperation("查询岗位信息来源网站详情")
@GetMapping(value = "/getSingle/{websiteId}")
@PreAuthorize("@ss.hasPermi('cms:recruit:source:list')")
public AjaxResult getSingle(@PathVariable Long websiteId) {
return AjaxResult.success(websiteManagementService.getById(websiteId));
}
@@ -45,6 +46,7 @@ public class WebsiteManagementController extends BaseController {
@Log(title = "岗位信息来源网站下拉选", businessType = QUERY)
@ApiOperation("岗位信息来源网站下拉选")
@GetMapping(value = "/getSelectOptions")
@PreAuthorize("@ss.hasPermi('cms:recruit:source:list')")
public AjaxResult getSelectOptions() {
WebsiteManagement websiteManagement = new WebsiteManagement();
websiteManagement.setDelFlag("0");
@@ -54,6 +56,7 @@ public class WebsiteManagementController extends BaseController {
@Log(title = "保存岗位信息来源网站信息", businessType = INSERT)
@ApiOperation("保存岗位信息来源网站信息")
@PostMapping(value = "/save")
@PreAuthorize("@ss.hasPermi('cms:recruit:source:edit')")
public AjaxResult save(@RequestBody WebsiteManagement websiteManagement) {
return toAjax(websiteManagementService.save(websiteManagement));
}
@@ -61,6 +64,7 @@ public class WebsiteManagementController extends BaseController {
@Log(title = "修改岗位信息来源网站信息", businessType = UPDATE)
@ApiOperation("修改岗位信息来源网站信息")
@PostMapping(value = "/update")
@PreAuthorize("@ss.hasPermi('cms:recruit:source:edit')")
public AjaxResult update(@RequestBody WebsiteManagement websiteManagement) {
return toAjax(websiteManagementService.updateById(websiteManagement));
}
@@ -68,6 +72,7 @@ public class WebsiteManagementController extends BaseController {
@Log(title = "删除岗位信息来源网站信息", businessType = DELETE)
@ApiOperation("删除岗位信息来源网站信息")
@PostMapping(value = "/delete")
@PreAuthorize("@ss.hasPermi('cms:recruit:source:edit')")
public AjaxResult delete(@RequestBody WebsiteManagement websiteManagement) {
websiteManagement.setDelFlag("2");
return toAjax(websiteManagementService.removeById(websiteManagement));
@@ -76,6 +81,7 @@ public class WebsiteManagementController extends BaseController {
@Log(title = "导入岗位信息来源网站信息", businessType = IMPORT)
@ApiOperation("导入岗位信息来源网站信息")
@PostMapping(value = "/import")
@PreAuthorize("@ss.hasPermi('cms:recruit:source:edit')")
public AjaxResult importData(@RequestParam(value = "file") MultipartFile file,@RequestParam(value = "dataType")String dataType) throws Exception {
return websiteManagementService.importData(file,dataType);
}

9
ruoyi-bussiness/src/main/java/com/ruoyi/cms/cron/JobCron.java
View File

@@ -6,6 +6,7 @@ import com.ruoyi.cms.service.ICompanyService;
import com.ruoyi.cms.service.IESJobSearchService;
import com.ruoyi.cms.service.IJobService;
import com.ruoyi.cms.service.IJobTitleSuggestService;
import com.ruoyi.cms.externalimport.IExternalJobImportService;
import com.ruoyi.common.utils.spring.SpringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@@ -35,4 +36,12 @@ public class JobCron {
public void rebuildJobTitleSuggest(){
SpringUtils.getBean(IJobTitleSuggestService.class).rebuildCache();
}
/**
* 外部 Excel 全量快照同步。
* Quartz 调用目标：jobCron.importLatestExternalSnapshot()
*/
public void importLatestExternalSnapshot(){
SpringUtils.getBean(IExternalJobImportService.class).importLatest();
}
}

27
ruoyi-bussiness/src/main/java/com/ruoyi/cms/domain/Job.java
View File

@@ -17,6 +17,7 @@ import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import java.math.BigDecimal;
import java.util.Date;
import java.util.List;
/**
@@ -146,6 +147,32 @@ public class Job extends BaseEntity
@ApiModelProperty("岗位链接")
private String jobUrl;
@ApiModelProperty("是否外部 Excel 导入岗位（0否，1是）")
private String externalImportFlag;
@ApiModelProperty("外部来源稳定编码")
private String externalSourceCode;
@ApiModelProperty("外部 Excel 原始来源名称")
private String externalSourceName;
@ApiModelProperty("外部岗位身份指纹")
private String jobFingerprint;
@ApiModelProperty("外部岗位内容指纹")
private String jobContentFingerprint;
@ApiModelProperty("最近一次外部导入批次")
private String externalBatchId;
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
@ApiModelProperty("外部岗位首次出现时间")
private Date externalFirstSeenAt;
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
@ApiModelProperty("外部岗位最近出现时间")
private Date externalLastSeenAt;
@ApiModelProperty("jobRow对应id")
private Long rowId;

64
ruoyi-bussiness/src/main/java/com/ruoyi/cms/externalimport/ExternalJobCandidate.java Normal file
View File

@@ -0,0 +1,64 @@
package com.ruoyi.cms.externalimport;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
/**
* Excel 单行在进入数据库前的规范化模型。
* 字段名不直接暴露给 API，避免将外部 Excel 表头绑定为内部契约。
*/
@Data
public class ExternalJobCandidate {
private int excelRowNumber;
private String sourceName;
private String sourceCode;
private String jobTitle;
private String description;
private String educationRaw;
private String educationCode;
private String experienceRaw;
private String experienceCode;
private String companyName;
private String location;
private String jobAddress;
private String areaCode;
private String salaryLow;
private String salaryHigh;
private String salaryText;
private String postingDate;
private String collectDate;
private String vacanciesText;
private String jobUrl;
private String jobCategory;
private String companyDescription;
private String industryType;
private String industrySub;
private String companyScale;
private String companyScaleCode;
private String companyNature;
private String companyNatureCode;
private String contactPerson;
private String jobFingerprint;
private String contentFingerprint;
private List<String> errors = new ArrayList<>();
private List<String> warnings = new ArrayList<>();
public boolean isValid() {
return errors.isEmpty();
}
public int completenessScore() {
int score = 0;
String[] values = {description, educationRaw, experienceRaw, location, jobAddress,
vacanciesText, companyDescription, industryType, industrySub, companyScale,
companyNature, contactPerson};
for (String value : values) {
if (value != null && !value.trim().isEmpty()) {
score++;
}
}
return score;
}
}

141
ruoyi-bussiness/src/main/java/com/ruoyi/cms/externalimport/ExternalJobExcelParser.java Normal file
View File

@@ -0,0 +1,141 @@
package com.ruoyi.cms.externalimport;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.DateUtil;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import java.io.IOException;
import java.io.InputStream;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* 解析外部岗位模板。此解析器明确处理 Excel 日期单元格，替代旧导入逻辑中日期全部丢失的行为。
*/
public class ExternalJobExcelParser {
private static final String[] REQUIRED_HEADERS = {
"Aca112", "Acb22a", "SalaryLow", "SalaryHight", "Aae397", "AAB004", "ORG", "ACE760", "Collect_time"
};
private final DataFormatter formatter = new DataFormatter();
private final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
public List<ExternalJobCandidate> parse(InputStream inputStream) throws IOException {
try (Workbook workbook = WorkbookFactory.create(inputStream)) {
if (workbook.getNumberOfSheets() == 0) {
throw new IllegalArgumentException("Excel 文件没有工作表");
}
Sheet sheet = workbook.getSheetAt(0);
Row headerRow = sheet.getRow(0);
if (headerRow == null) {
throw new IllegalArgumentException("Excel 文件缺少表头");
}
Map<String, Integer> headerIndexes = headers(headerRow);
for (String required : REQUIRED_HEADERS) {
if (!headerIndexes.containsKey(required)) {
throw new IllegalArgumentException("Excel 缺少必需表头: " + required);
}
}
List<ExternalJobCandidate> result = new ArrayList<>();
for (int index = 1; index <= sheet.getLastRowNum(); index++) {
Row row = sheet.getRow(index);
if (row == null || isBlank(row)) {
continue;
}
ExternalJobCandidate candidate = new ExternalJobCandidate();
candidate.setExcelRowNumber(index + 1);
candidate.setJobCategory(value(row, headerIndexes, "Std_class"));
candidate.setJobTitle(value(row, headerIndexes, "Aca112"));
candidate.setDescription(value(row, headerIndexes, "Acb22a"));
candidate.setEducationRaw(value(row, headerIndexes, "Aac011"));
candidate.setVacanciesText(firstNonBlank(value(row, headerIndexes, "Acb240"), value(row, headerIndexes, "Recruit_Num")));
candidate.setSalaryText(value(row, headerIndexes, "Salary"));
candidate.setSalaryLow(value(row, headerIndexes, "SalaryLow"));
candidate.setSalaryHigh(value(row, headerIndexes, "SalaryHight"));
candidate.setPostingDate(dateValue(row, headerIndexes, "Aae397"));
candidate.setCompanyName(value(row, headerIndexes, "AAB004"));
candidate.setJobAddress(value(row, headerIndexes, "AAE006"));
candidate.setSourceName(value(row, headerIndexes, "ORG"));
candidate.setJobUrl(value(row, headerIndexes, "ACE760"));
candidate.setContactPerson(value(row, headerIndexes, "AAE004"));
candidate.setExperienceRaw(value(row, headerIndexes, "Experience"));
candidate.setCompanyDescription(value(row, headerIndexes, "AAB092"));
candidate.setIndustryType(value(row, headerIndexes, "IndustryType"));
candidate.setIndustrySub(value(row, headerIndexes, "IndustrySub"));
candidate.setCompanyNature(firstNonBlank(value(row, headerIndexes, "AAB019_OK"), value(row, headerIndexes, "AAB019")));
candidate.setCompanyScale(firstNonBlank(value(row, headerIndexes, "Num_OK"), value(row, headerIndexes, "Num_employers")));
candidate.setCollectDate(dateValue(row, headerIndexes, "Collect_time"));
String county = value(row, headerIndexes, "County");
String city = value(row, headerIndexes, "City");
candidate.setLocation(firstNonBlank(candidate.getJobAddress(), county, city));
result.add(candidate);
}
return result;
} catch (IllegalArgumentException ex) {
throw ex;
} catch (Exception ex) {
throw new IOException("解析 Excel 失败", ex);
}
}
private Map<String, Integer> headers(Row row) {
Map<String, Integer> result = new HashMap<>();
for (Cell cell : row) {
String value = formatter.formatCellValue(cell);
if (value != null && !value.trim().isEmpty()) {
result.put(value.trim(), cell.getColumnIndex());
}
}
return result;
}
private String value(Row row, Map<String, Integer> headers, String header) {
Integer column = headers.get(header);
if (column == null) {
return "";
}
Cell cell = row.getCell(column);
return cell == null ? "" : formatter.formatCellValue(cell).trim();
}
private String dateValue(Row row, Map<String, Integer> headers, String header) {
Integer column = headers.get(header);
if (column == null) {
return "";
}
Cell cell = row.getCell(column);
if (cell == null) {
return "";
}
if (DateUtil.isCellDateFormatted(cell)) {
Date value = cell.getDateCellValue();
return value == null ? "" : dateFormat.format(value);
}
return formatter.formatCellValue(cell).trim();
}
private boolean isBlank(Row row) {
for (Cell cell : row) {
if (!formatter.formatCellValue(cell).trim().isEmpty()) {
return false;
}
}
return true;
}
private String firstNonBlank(String... values) {
for (String value : values) {
if (value != null && !value.trim().isEmpty()) {
return value.trim();
}
}
return "";
}
}

107
ruoyi-bussiness/src/main/java/com/ruoyi/cms/externalimport/ExternalJobFingerprint.java Normal file
View File

@@ -0,0 +1,107 @@
package com.ruoyi.cms.externalimport;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.text.Normalizer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
/** 岗位身份/内容指纹计算，保持纯函数以便单元测试。 */
public final class ExternalJobFingerprint {
private static final String VERSION = "FP_V1";
private ExternalJobFingerprint() {
}
public static String identity(ExternalJobCandidate candidate) {
String normalizedUrl = normalizeUrl(candidate.getJobUrl());
if (!normalizedUrl.isEmpty()) {
return sha256(VERSION + "|" + normalizeText(candidate.getSourceCode()) + "|" + normalizedUrl);
}
return sha256(VERSION + "|" + normalizeText(candidate.getSourceCode()) + "|"
+ normalizeText(candidate.getCompanyName()) + "|" + normalizeText(candidate.getJobTitle()) + "|"
+ normalizeText(candidate.getLocation()) + "|" + normalizeText(candidate.getSalaryLow()) + "|"
+ normalizeText(candidate.getSalaryHigh()));
}
public static String content(ExternalJobCandidate candidate) {
return sha256(normalizeText(candidate.getJobTitle()) + "|" + normalizeText(candidate.getDescription()) + "|"
+ normalizeText(candidate.getSalaryLow()) + "|" + normalizeText(candidate.getSalaryHigh()) + "|"
+ normalizeText(candidate.getEducationCode()) + "|" + normalizeText(candidate.getExperienceCode()) + "|"
+ normalizeText(candidate.getVacanciesText()) + "|" + normalizeText(candidate.getJobAddress()) + "|"
+ normalizeText(candidate.getLocation()) + "|" + normalizeText(candidate.getAreaCode()) + "|"
+ normalizeText(candidate.getJobCategory()) + "|" + normalizeText(candidate.getCompanyName()) + "|"
+ normalizeUrl(candidate.getJobUrl()) + "|" + normalizeText(candidate.getPostingDate()));
}
public static String normalizeText(String value) {
if (value == null) {
return "";
}
return Normalizer.normalize(value, Normalizer.Form.NFKC)
.trim()
.replaceAll("\\s+", " ")
.toLowerCase(Locale.ROOT);
}
public static String normalizeUrl(String value) {
if (value == null || value.trim().isEmpty()) {
return "";
}
try {
URI uri = new URI(value.trim());
String scheme = uri.getScheme() == null ? "https" : uri.getScheme().toLowerCase(Locale.ROOT);
String host = uri.getHost();
if (host == null || host.trim().isEmpty()) {
return "";
}
String path = uri.getRawPath() == null || uri.getRawPath().isEmpty() ? "/" : uri.getRawPath();
if (path.length() > 1 && path.endsWith("/")) {
path = path.substring(0, path.length() - 1);
}
List<String> queryParts = new ArrayList<>();
if (uri.getRawQuery() != null && !uri.getRawQuery().isEmpty()) {
for (String part : uri.getRawQuery().split("&")) {
String key = part.split("=", 2)[0].toLowerCase(Locale.ROOT);
if (!key.startsWith("utm_") && !"spm".equals(key) && !"from".equals(key)) {
queryParts.add(part);
}
}
}
Collections.sort(queryParts);
return scheme + "://" + host.toLowerCase(Locale.ROOT) + path
+ (queryParts.isEmpty() ? "" : "?" + join(queryParts, "&"));
} catch (URISyntaxException ex) {
return "";
}
}
private static String join(List<String> values, String delimiter) {
StringBuilder builder = new StringBuilder();
for (String value : values) {
if (builder.length() > 0) {
builder.append(delimiter);
}
builder.append(value);
}
return builder.toString();
}
private static String sha256(String value) {
try {
byte[] bytes = MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8));
StringBuilder builder = new StringBuilder(64);
for (byte item : bytes) {
builder.append(String.format("%02x", item));
}
return builder.toString();
} catch (NoSuchAlgorithmException ex) {
throw new IllegalStateException("SHA-256 is unavailable", ex);
}
}
}

27
ruoyi-bussiness/src/main/java/com/ruoyi/cms/externalimport/ExternalJobImportProperties.java Normal file
View File

@@ -0,0 +1,27 @@
package com.ruoyi.cms.externalimport;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
import java.util.LinkedHashSet;
import java.util.Set;
/**
* 外部岗位全量快照导入配置。
*
* 令牌只能由环境专用 yml 或密钥管理系统提供，禁止提交真实值。
*/
@Data
@Component
@ConfigurationProperties(prefix = "external-job-import")
public class ExternalJobImportProperties {
private boolean enabled = false;
private String latestMetadataUrl;
private String internalToken;
private int connectTimeoutMillis = 10000;
private int readTimeoutMillis = 120000;
private double maxInvalidRowRatio = 0.05D;
private double maxDeleteRatio = 0.30D;
private Set<String> requiredSources = new LinkedHashSet<>();
}

20
ruoyi-bussiness/src/main/java/com/ruoyi/cms/externalimport/ExternalJobImportResult.java Normal file
View File

@@ -0,0 +1,20 @@
package com.ruoyi.cms.externalimport;
import lombok.Data;
/** 一个成功或被拦截的导入批次摘要。 */
@Data
public class ExternalJobImportResult {
private String batchId;
private String status;
private int totalRows;
private int validRows;
private int invalidRows;
private int duplicateRows;
private int insertCount;
private int updateCount;
private int keepCount;
private int restoreCount;
private int deleteCount;
private String message;
}

667
ruoyi-bussiness/src/main/java/com/ruoyi/cms/externalimport/ExternalJobImportServiceImpl.java Normal file
View File

@@ -0,0 +1,667 @@
package com.ruoyi.cms.externalimport;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigDecimal;
import java.sql.Timestamp;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
/**
* 外部岗位全量快照发布服务。
*
* 本服务完全独立于旧的 JobServiceImpl.importRow：不清空 row_work，且只删除带外部标记的岗位。
*/
@Service
public class ExternalJobImportServiceImpl implements IExternalJobImportService {
private static final String SYSTEM_USER = "external-import";
private final ExternalJobImportProperties properties;
private final ExternalUploadClient uploadClient;
private final JdbcTemplate jdbcTemplate;
private final ObjectMapper objectMapper;
private final ExternalJobExcelParser parser = new ExternalJobExcelParser();
public ExternalJobImportServiceImpl(ExternalJobImportProperties properties,
ExternalUploadClient uploadClient,
JdbcTemplate jdbcTemplate,
ObjectMapper objectMapper) {
this.properties = properties;
this.uploadClient = uploadClient;
this.jdbcTemplate = jdbcTemplate;
this.objectMapper = objectMapper;
}
@Override
@Transactional(rollbackFor = Exception.class)
public ExternalJobImportResult importLatest() {
if (!properties.isEnabled()) {
throw new IllegalStateException("external-job-import.enabled 未开启");
}
try {
ExternalUploadFileMetadata metadata = uploadClient.latest();
if (hasSuccessfulSha256(metadata.getSha256())) {
ExternalJobImportResult result = new ExternalJobImportResult();
result.setStatus("SKIPPED");
result.setMessage("该 SHA-256 文件已经成功导入，已跳过");
return result;
}
try (ExternalUploadClient.Download download = uploadClient.download(metadata)) {
return importStream(metadata, download.getInputStream());
}
} catch (IOException ex) {
throw new IllegalStateException("拉取外部岗位 Excel 失败", ex);
}
}
private ExternalJobImportResult importStream(ExternalUploadFileMetadata metadata, InputStream inputStream) throws IOException {
String batchId = UUID.randomUUID().toString().replace("-", "");
List<ExternalJobCandidate> parsedRows = parser.parse(inputStream);
ExternalJobImportResult result = new ExternalJobImportResult();
result.setBatchId(batchId);
result.setTotalRows(parsedRows.size());
insertBatch(batchId, metadata, "VALIDATING", result);
Map<String, String> sourceCodeCache = new HashMap<>();
Map<String, String> dictionaryCodeCache = new HashMap<>();
for (ExternalJobCandidate candidate : parsedRows) {
validateAndNormalize(candidate, sourceCodeCache, dictionaryCodeCache);
}
int invalidRows = countInvalid(parsedRows);
result.setInvalidRows(invalidRows);
if (parsedRows.isEmpty()) {
blockBatch(batchId, result, "Excel 中没有有效数据行");
return result;
}
if ((double) invalidRows / parsedRows.size() > properties.getMaxInvalidRowRatio()) {
persistRows(batchId, parsedRows, Collections.<String, Long>emptyMap(), Collections.<String, ExternalJobCandidate>emptyMap());
blockBatch(batchId, result, "无效行比例超过阈值: " + properties.getMaxInvalidRowRatio());
return result;
}
Map<String, ExternalJobCandidate> current = deduplicate(parsedRows, result);
Set<String> sourceScope = sources(current.values());
if (current.isEmpty()) {
persistRows(batchId, parsedRows, Collections.<String, Long>emptyMap(), current);
blockBatch(batchId, result, "没有可发布的有效岗位");
return result;
}
if (!sourceScope.containsAll(properties.getRequiredSources())) {
persistRows(batchId, parsedRows, Collections.<String, Long>emptyMap(), current);
blockBatch(batchId, result, "本批次缺少必需来源: " + missingSources(sourceScope));
return result;
}
Map<String, ExistingJob> previous = activeExternalJobs(sourceScope);
int prospectiveDeletes = previous.size() - intersectionSize(previous.keySet(), current.keySet());
if (!previous.isEmpty() && (double) prospectiveDeletes / previous.size() > properties.getMaxDeleteRatio()) {
persistRows(batchId, parsedRows, Collections.<String, Long>emptyMap(), current);
blockBatch(batchId, result, "拟删除比例 " + prospectiveDeletes + "/" + previous.size() + " 超过阈值: " + properties.getMaxDeleteRatio());
return result;
}
Map<String, Long> fingerprintToJobId = new HashMap<>();
for (ExternalJobCandidate candidate : current.values()) {
ExistingJob existing = previous.remove(candidate.getJobFingerprint());
Long companyId = resolveCompany(candidate);
if (existing != null) {
if (candidate.getContentFingerprint().equals(existing.contentFingerprint)) {
touchExistingJob(existing.jobId, batchId);
result.setKeepCount(result.getKeepCount() + 1);
} else {
updateExistingJob(existing.jobId, candidate, companyId, batchId);
result.setUpdateCount(result.getUpdateCount() + 1);
}
fingerprintToJobId.put(candidate.getJobFingerprint(), existing.jobId);
continue;
}
Long deletedJobId = deletedExternalJob(candidate.getJobFingerprint());
if (deletedJobId != null) {
restoreJob(deletedJobId, candidate, companyId, batchId);
result.setRestoreCount(result.getRestoreCount() + 1);
fingerprintToJobId.put(candidate.getJobFingerprint(), deletedJobId);
} else {
Long jobId = insertJob(candidate, companyId, batchId);
result.setInsertCount(result.getInsertCount() + 1);
fingerprintToJobId.put(candidate.getJobFingerprint(), jobId);
}
}
for (ExistingJob absent : previous.values()) {
logicallyDeleteJob(absent.jobId, batchId);
result.setDeleteCount(result.getDeleteCount() + 1);
}
persistRows(batchId, parsedRows, fingerprintToJobId, current);
recordMonitoring(batchId, parsedRows, current.values(), result);
result.setValidRows(current.size());
result.setStatus("SUCCESS");
result.setMessage("外部岗位全量快照发布成功");
updateBatch(batchId, "SUCCESS", result, null);
return result;
}
@Override
public List<Map<String, Object>> listBatches(String status, Integer limit) {
int safeLimit = limit == null ? 50 : Math.max(1, Math.min(limit, 200));
if (isBlank(status)) {
return jdbcTemplate.queryForList("SELECT * FROM job_import_batch ORDER BY created_at DESC LIMIT ?", safeLimit);
}
return jdbcTemplate.queryForList("SELECT * FROM job_import_batch WHERE status = ? ORDER BY created_at DESC LIMIT ?", status, safeLimit);
}
@Override
public List<Map<String, Object>> listSourceAliases() {
return jdbcTemplate.queryForList("SELECT alias.*, website.website_name, website.website_url "
+ "FROM job_import_source_alias alias "
+ "LEFT JOIN website_management website ON website.website_id = alias.website_id "
+ "ORDER BY alias.is_active DESC, alias.alias_name");
}
@Override
public void saveSourceAlias(ExternalJobSourceAliasRequest request) {
if (request == null || isBlank(request.getAliasName()) || isBlank(request.getSourceCode())) {
throw new IllegalArgumentException("aliasName 和 sourceCode 为必填项");
}
String active = "0".equals(request.getIsActive()) ? "0" : "1";
String snapshotRequired = "0".equals(request.getSnapshotRequired()) ? "0" : "1";
jdbcTemplate.update("INSERT INTO job_import_source_alias (alias_name, source_code, website_id, is_active, snapshot_required, remark, create_time, update_time) "
+ "VALUES (?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) "
+ "ON CONFLICT (alias_name) DO UPDATE SET source_code=EXCLUDED.source_code, website_id=EXCLUDED.website_id, "
+ "is_active=EXCLUDED.is_active, snapshot_required=EXCLUDED.snapshot_required, remark=EXCLUDED.remark, update_time=CURRENT_TIMESTAMP",
request.getAliasName().trim(), request.getSourceCode().trim(), request.getWebsiteId(), active, snapshotRequired, emptyToNull(request.getRemark()));
}
private void validateAndNormalize(ExternalJobCandidate candidate, Map<String, String> sourceCodeCache,
Map<String, String> dictionaryCodeCache) {
candidate.setJobTitle(trim(candidate.getJobTitle()));
candidate.setCompanyName(trim(candidate.getCompanyName()));
candidate.setDescription(trim(candidate.getDescription()));
candidate.setJobUrl(trim(candidate.getJobUrl()));
candidate.setSourceName(trim(candidate.getSourceName()));
if (isBlank(candidate.getJobTitle())) {
candidate.getErrors().add("岗位名称 Aca112 为空");
}
if (isBlank(candidate.getCompanyName())) {
candidate.getErrors().add("企业名称 AAB004 为空");
}
if (isBlank(candidate.getDescription())) {
candidate.getErrors().add("岗位描述 Acb22a 为空");
}
if (isBlank(candidate.getSourceName())) {
candidate.getErrors().add("来源 ORG 为空");
} else {
String sourceCode = sourceCodeCache.get(candidate.getSourceName());
if (sourceCode == null) {
List<String> values = jdbcTemplate.queryForList(
"SELECT source_code FROM job_import_source_alias WHERE alias_name = ? AND is_active = '1'", String.class, candidate.getSourceName());
sourceCode = values.size() == 1 ? values.get(0) : "";
sourceCodeCache.put(candidate.getSourceName(), sourceCode);
}
if (isBlank(sourceCode)) {
candidate.getErrors().add("来源未在 job_import_source_alias 中启用: " + candidate.getSourceName());
} else {
candidate.setSourceCode(sourceCode);
}
}
if (isBlank(candidate.getJobUrl()) || isBlank(ExternalJobFingerprint.normalizeUrl(candidate.getJobUrl()))) {
candidate.getErrors().add("岗位链接 ACE760 为空或格式无效");
}
Long low = parseMoney(candidate.getSalaryLow());
Long high = parseMoney(candidate.getSalaryHigh());
if (low == null || high == null) {
candidate.getErrors().add("薪资字段必须为数字");
} else if (low > high) {
candidate.getErrors().add("最低薪资不能大于最高薪资");
}
if (!isDate(candidate.getPostingDate())) {
candidate.getErrors().add("发布日期 Aae397 无法解析");
}
if (!isDate(candidate.getCollectDate())) {
candidate.getErrors().add("采集日期 Collect_time 无法解析");
}
candidate.setEducationCode(educationCode(candidate.getEducationRaw(), candidate));
candidate.setExperienceCode(experienceCode(candidate.getExperienceRaw(), candidate));
candidate.setAreaCode(areaCode(candidate.getLocation(), candidate));
candidate.setCompanyScaleCode(dictionaryCode("scale", candidate.getCompanyScale(), candidate, dictionaryCodeCache));
candidate.setCompanyNatureCode(dictionaryCode("company_nature", candidate.getCompanyNature(), candidate, dictionaryCodeCache));
if (isBlank(candidate.getJobAddress())) {
candidate.getWarnings().add("岗位详细地址为空，已使用区县或城市作为岗位地点");
}
if (candidate.isValid()) {
candidate.setJobFingerprint(ExternalJobFingerprint.identity(candidate));
candidate.setContentFingerprint(ExternalJobFingerprint.content(candidate));
}
}
private Map<String, ExternalJobCandidate> deduplicate(List<ExternalJobCandidate> candidates, ExternalJobImportResult result) {
Map<String, ExternalJobCandidate> current = new LinkedHashMap<>();
for (ExternalJobCandidate candidate : candidates) {
if (!candidate.isValid()) {
continue;
}
ExternalJobCandidate before = current.get(candidate.getJobFingerprint());
if (before == null) {
current.put(candidate.getJobFingerprint(), candidate);
} else {
result.setDuplicateRows(result.getDuplicateRows() + 1);
before.getWarnings().add("批内重复指纹，保留信息更完整的一行");
if (isBetter(candidate, before)) {
current.put(candidate.getJobFingerprint(), candidate);
}
}
}
return current;
}
private boolean isBetter(ExternalJobCandidate left, ExternalJobCandidate right) {
int completeness = Integer.compare(left.completenessScore(), right.completenessScore());
if (completeness != 0) {
return completeness > 0;
}
return trim(left.getCollectDate()).compareTo(trim(right.getCollectDate())) > 0;
}
private Map<String, ExistingJob> activeExternalJobs(Set<String> sourceCodes) {
if (sourceCodes.isEmpty()) {
return Collections.emptyMap();
}
String placeholders = placeholders(sourceCodes.size());
List<Object> args = new ArrayList<Object>(sourceCodes);
List<ExistingJob> jobs = jdbcTemplate.query(
"SELECT job_id, job_fingerprint, job_content_fingerprint FROM job "
+ "WHERE external_import_flag = '1' AND del_flag = '0' "
+ "AND external_source_code IN (" + placeholders + ")",
(rs, rowNum) -> new ExistingJob(rs.getLong("job_id"), rs.getString("job_fingerprint"), rs.getString("job_content_fingerprint")),
args.toArray());
Map<String, ExistingJob> result = new HashMap<>();
for (ExistingJob job : jobs) {
result.put(job.fingerprint, job);
}
return result;
}
private Long deletedExternalJob(String fingerprint) {
List<Long> ids = jdbcTemplate.queryForList(
"SELECT job_id FROM job WHERE external_import_flag = '1' AND del_flag = '2' "
+ "AND job_fingerprint = ? ORDER BY update_time DESC LIMIT 1", Long.class, fingerprint);
return ids.isEmpty() ? null : ids.get(0);
}
private Long resolveCompany(ExternalJobCandidate candidate) {
List<Long> ids = jdbcTemplate.queryForList(
"SELECT company_id FROM company WHERE del_flag = '0' AND LOWER(BTRIM(name)) = LOWER(BTRIM(?))", Long.class, candidate.getCompanyName());
if (ids.size() > 1) {
candidate.getWarnings().add("匹配到多个同名企业，岗位未绑定 company_id");
return null;
}
if (ids.size() == 1) {
Long companyId = ids.get(0);
jdbcTemplate.update("UPDATE company SET "
+ "location = CASE WHEN NULLIF(BTRIM(location), '') IS NULL THEN ? ELSE location END, "
+ "description = CASE WHEN NULLIF(BTRIM(description), '') IS NULL THEN ? ELSE description END, "
+ "industry = CASE WHEN NULLIF(BTRIM(industry), '') IS NULL THEN ? ELSE industry END, "
+ "scale = CASE WHEN NULLIF(BTRIM(scale), '') IS NULL THEN ? ELSE scale END, "
+ "company_nature = CASE WHEN NULLIF(BTRIM(company_nature), '') IS NULL THEN ? ELSE company_nature END, "
+ "contact_person = CASE WHEN NULLIF(BTRIM(contact_person), '') IS NULL THEN ? ELSE contact_person END, "
+ "update_by = ?, update_time = CURRENT_TIMESTAMP WHERE company_id = ?",
emptyToNull(candidate.getLocation()), emptyToNull(candidate.getCompanyDescription()), emptyToNull(firstNonBlank(candidate.getIndustrySub(), candidate.getIndustryType())),
emptyToNull(candidate.getCompanyScaleCode()), emptyToNull(candidate.getCompanyNatureCode()), emptyToNull(candidate.getContactPerson()), SYSTEM_USER, companyId);
return companyId;
}
Long companyId = jdbcTemplate.queryForObject(
"INSERT INTO company (name, location, industry, scale, company_nature, description, contact_person, del_flag, create_by, create_time, company_status) "
+ "VALUES (?, ?, ?, ?, ?, ?, ?, '0', ?, CURRENT_TIMESTAMP, '0') RETURNING company_id",
Long.class, candidate.getCompanyName(), emptyToNull(candidate.getLocation()), emptyToNull(firstNonBlank(candidate.getIndustrySub(), candidate.getIndustryType())),
emptyToNull(candidate.getCompanyScaleCode()), emptyToNull(candidate.getCompanyNatureCode()), emptyToNull(candidate.getCompanyDescription()), emptyToNull(candidate.getContactPerson()), SYSTEM_USER);
return companyId;
}
private Long insertJob(ExternalJobCandidate candidate, Long companyId, String batchId) {
return jdbcTemplate.queryForObject("INSERT INTO job (job_title, min_salary, max_salary, education, experience, company_name, "
+ "job_location, job_location_area_code, posting_date, vacancies, \"view\", company_id, is_hot, is_urgent, apply_num, "
+ "description, is_publish, data_source, job_url, job_category, job_address, job_status, del_flag, create_by, create_time, "
+ "external_import_flag, external_source_code, external_source_name, job_fingerprint, job_content_fingerprint, "
+ "external_batch_id, external_first_seen_at, external_last_seen_at) "
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, '0', '0', '0', ?, 1, ?, ?, ?, ?, '0', '0', ?, CURRENT_TIMESTAMP, "
+ "'1', ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) RETURNING job_id",
Long.class,
candidate.getJobTitle(), parseMoney(candidate.getSalaryLow()), parseMoney(candidate.getSalaryHigh()), candidate.getEducationCode(), candidate.getExperienceCode(),
candidate.getCompanyName(), emptyToNull(candidate.getLocation()), emptyToNull(candidate.getAreaCode()), timestamp(candidate.getPostingDate()), vacancies(candidate.getVacanciesText()),
companyId, candidate.getDescription(), candidate.getSourceName(), candidate.getJobUrl(), emptyToNull(candidate.getJobCategory()), emptyToNull(candidate.getJobAddress()),
SYSTEM_USER, candidate.getSourceCode(), candidate.getSourceName(), candidate.getJobFingerprint(), candidate.getContentFingerprint(), batchId);
}
private void updateExistingJob(Long jobId, ExternalJobCandidate candidate, Long companyId, String batchId) {
jdbcTemplate.update("UPDATE job SET job_title=?, min_salary=?, max_salary=?, education=?, experience=?, company_name=?, "
+ "job_location=?, job_location_area_code=?, posting_date=?, vacancies=?, company_id=?, description=?, is_publish=1, "
+ "data_source=?, job_url=?, job_category=?, job_address=?, job_status='0', del_flag='0', update_by=?, update_time=CURRENT_TIMESTAMP, "
+ "external_source_code=?, external_source_name=?, job_content_fingerprint=?, external_batch_id=?, external_last_seen_at=CURRENT_TIMESTAMP "
+ "WHERE job_id=?",
candidate.getJobTitle(), parseMoney(candidate.getSalaryLow()), parseMoney(candidate.getSalaryHigh()), candidate.getEducationCode(), candidate.getExperienceCode(),
candidate.getCompanyName(), emptyToNull(candidate.getLocation()), emptyToNull(candidate.getAreaCode()), timestamp(candidate.getPostingDate()), vacancies(candidate.getVacanciesText()),
companyId, candidate.getDescription(), candidate.getSourceName(), candidate.getJobUrl(), emptyToNull(candidate.getJobCategory()), emptyToNull(candidate.getJobAddress()),
SYSTEM_USER, candidate.getSourceCode(), candidate.getSourceName(), candidate.getContentFingerprint(), batchId, jobId);
}
private void restoreJob(Long jobId, ExternalJobCandidate candidate, Long companyId, String batchId) {
updateExistingJob(jobId, candidate, companyId, batchId);
}
private void touchExistingJob(Long jobId, String batchId) {
jdbcTemplate.update("UPDATE job SET external_batch_id=?, external_last_seen_at=CURRENT_TIMESTAMP, update_by=?, update_time=CURRENT_TIMESTAMP WHERE job_id=?",
batchId, SYSTEM_USER, jobId);
}
private void logicallyDeleteJob(Long jobId, String batchId) {
jdbcTemplate.update("UPDATE job SET del_flag='2', job_status='1', is_publish=0, external_batch_id=?, update_by=?, update_time=CURRENT_TIMESTAMP WHERE job_id=?",
batchId, SYSTEM_USER, jobId);
}
private void insertBatch(String batchId, ExternalUploadFileMetadata metadata, String status, ExternalJobImportResult result) {
jdbcTemplate.update("INSERT INTO job_import_batch (batch_id, file_id, file_name, file_sha256, snapshot_scope, fingerprint_version, status, total_rows, "
+ "valid_rows, invalid_rows, duplicate_rows, insert_count, update_count, keep_count, restore_count, delete_count, created_at, updated_at) "
+ "VALUES (?, ?, ?, ?, ?, 'FP_V1', ?, ?, 0, 0, 0, 0, 0, 0, 0, 0, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)",
batchId, metadata.getFileId(), metadata.getFilename(), metadata.getSha256(), join(metadata.getSourceScope()), status, result.getTotalRows());
}
private void updateBatch(String batchId, String status, ExternalJobImportResult result, String error) {
jdbcTemplate.update("UPDATE job_import_batch SET status=?, valid_rows=?, invalid_rows=?, duplicate_rows=?, insert_count=?, update_count=?, keep_count=?, "
+ "restore_count=?, delete_count=?, error_summary=?, completed_at=CASE WHEN ? IN ('SUCCESS','FAILED','BLOCKED') THEN CURRENT_TIMESTAMP ELSE completed_at END, "
+ "updated_at=CURRENT_TIMESTAMP WHERE batch_id=?",
status, result.getValidRows(), result.getInvalidRows(), result.getDuplicateRows(), result.getInsertCount(), result.getUpdateCount(), result.getKeepCount(),
result.getRestoreCount(), result.getDeleteCount(), error, status, batchId);
}
private void blockBatch(String batchId, ExternalJobImportResult result, String message) {
result.setStatus("BLOCKED");
result.setMessage(message);
updateBatch(batchId, "BLOCKED", result, message);
}
private void persistRows(String batchId, List<ExternalJobCandidate> candidates, Map<String, Long> fingerprintToJobId,
Map<String, ExternalJobCandidate> publishedCandidates) {
Set<String> retained = fingerprintToJobId.keySet();
for (ExternalJobCandidate candidate : candidates) {
boolean canonicalRow = candidate.isValid() && publishedCandidates.get(candidate.getJobFingerprint()) == candidate;
String validationStatus = candidate.isValid() ? (canonicalRow ? "VALID" : "DUPLICATE") : "INVALID";
String resultStatus = canonicalRow && retained.contains(candidate.getJobFingerprint()) ? "PUBLISHED" : candidate.isValid() ? "SKIPPED" : "FAILED";
Long jobId = canonicalRow ? fingerprintToJobId.get(candidate.getJobFingerprint()) : null;
jdbcTemplate.update("INSERT INTO job_import_row (batch_id, excel_row_number, source_name, source_code, job_fingerprint, content_fingerprint, "
+ "validation_status, result_status, error_message, raw_payload, job_id, created_at) "
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)",
batchId, candidate.getExcelRowNumber(), candidate.getSourceName(), candidate.getSourceCode(), candidate.getJobFingerprint(), candidate.getContentFingerprint(),
validationStatus, resultStatus, join(candidate.getErrors()), rawPayload(candidate), jobId);
}
}
/** 将批次结果同步到现有的入库监测与采集趋势表，避免后台手工补录。 */
private void recordMonitoring(String batchId, List<ExternalJobCandidate> parsedRows,
Collection<ExternalJobCandidate> publishedRows,
ExternalJobImportResult result) {
String summary = "新增" + result.getInsertCount() + "个，更新" + result.getUpdateCount()
+ "个，保留" + result.getKeepCount() + "个，恢复" + result.getRestoreCount()
+ "个，下线" + result.getDeleteCount() + "个，失败" + result.getInvalidRows() + "个";
jdbcTemplate.update("INSERT INTO job_data_storage_detection (detection_id, storage_date, storage_number, storage_result, failed_reason, del_flag) "
+ "VALUES (?, CURRENT_TIMESTAMP, ?, ?, ?, '0')",
batchId, publishedRows.size(), summary, result.getInvalidRows() == 0 ? null : "存在字段校验失败行，请查看 job_import_row");
jdbcTemplate.update("INSERT INTO job_data_trend (delete_count, insert_count, fail_count, storage_time, website_name, del_flag) "
+ "VALUES (?, ?, ?, CURRENT_TIMESTAMP, 'total', '0')",
result.getDeleteCount(), result.getInsertCount() + result.getRestoreCount(), result.getInvalidRows() + result.getDuplicateRows());
Map<String, Integer> successBySource = new HashMap<>();
Map<String, Integer> failedBySource = new HashMap<>();
Map<String, String> sourceNames = new HashMap<>();
for (ExternalJobCandidate candidate : publishedRows) {
increment(successBySource, candidate.getSourceCode());
sourceNames.put(candidate.getSourceCode(), candidate.getSourceName());
}
for (ExternalJobCandidate candidate : parsedRows) {
if (!candidate.isValid()) {
increment(failedBySource, candidate.getSourceCode());
if (!isBlank(candidate.getSourceCode())) {
sourceNames.put(candidate.getSourceCode(), candidate.getSourceName());
}
}
}
Set<String> sources = new HashSet<>();
sources.addAll(successBySource.keySet());
sources.addAll(failedBySource.keySet());
for (String sourceCode : sources) {
if (isBlank(sourceCode)) {
continue;
}
List<Long> websiteIds = jdbcTemplate.queryForList(
"SELECT website_id FROM job_import_source_alias WHERE source_code=? AND is_active='1' LIMIT 1", Long.class, sourceCode);
Long websiteId = websiteIds.isEmpty() ? null : websiteIds.get(0);
int success = value(successBySource, sourceCode);
int failed = value(failedBySource, sourceCode);
jdbcTemplate.update("INSERT INTO job_data_storage_detail (detection_id, website_id, website_name, storage_time, success_number, failed_number, storage_detail, failed_reason) "
+ "VALUES (?, ?, ?, CURRENT_TIMESTAMP, ?, ?, ?, ?)",
batchId, websiteId, sourceNames.get(sourceCode), success, failed,
"外部导入批次 " + batchId, failed == 0 ? null : "存在字段校验失败行");
jdbcTemplate.update("INSERT INTO job_data_trend (delete_count, insert_count, fail_count, storage_time, website_name, del_flag) "
+ "VALUES (0, ?, ?, CURRENT_TIMESTAMP, ?, '0')", success, failed, sourceCode);
}
}
private String rawPayload(ExternalJobCandidate candidate) {
try {
return objectMapper.writeValueAsString(candidate);
} catch (JsonProcessingException ex) {
return "{\"excelRowNumber\":" + candidate.getExcelRowNumber() + "}";
}
}
private void increment(Map<String, Integer> values, String key) {
if (!isBlank(key)) {
values.put(key, value(values, key) + 1);
}
}
private int value(Map<String, Integer> values, String key) {
Integer value = values.get(key);
return value == null ? 0 : value;
}
private boolean hasSuccessfulSha256(String sha256) {
Integer count = jdbcTemplate.queryForObject("SELECT COUNT(1) FROM job_import_batch WHERE file_sha256=? AND status='SUCCESS'", Integer.class, sha256);
return count != null && count > 0;
}
private int countInvalid(Collection<ExternalJobCandidate> candidates) {
int result = 0;
for (ExternalJobCandidate candidate : candidates) {
if (!candidate.isValid()) {
result++;
}
}
return result;
}
private Set<String> sources(Collection<ExternalJobCandidate> candidates) {
Set<String> result = new HashSet<>();
for (ExternalJobCandidate candidate : candidates) {
result.add(candidate.getSourceCode());
}
return result;
}
private int intersectionSize(Set<String> first, Set<String> second) {
int count = 0;
for (String value : first) {
if (second.contains(value)) {
count++;
}
}
return count;
}
private String missingSources(Set<String> scope) {
Set<String> missing = new HashSet<>(properties.getRequiredSources());
missing.removeAll(scope);
return join(missing);
}
private String educationCode(String raw, ExternalJobCandidate candidate) {
String value = trim(raw);
if ("不限".equals(value) || "学历不限".equals(value)) return "-1";
if ("初中及以下".equals(value) || "高中以下".equals(value)) return "0";
if ("中专/中技".equals(value) || "中专/技校".equals(value) || "中技/中专".equals(value)) return "1";
if ("高中".equals(value)) return "2";
if ("大专".equals(value)) return "3";
if ("本科".equals(value) || "统招本科".equals(value)) return "4";
if ("硕士".equals(value)) return "5";
if ("博士".equals(value)) return "6";
candidate.getWarnings().add("学历未映射，按不限处理: " + value);
return "-1";
}
private String experienceCode(String raw, ExternalJobCandidate candidate) {
String value = trim(raw);
if ("不限".equals(value) || "经验不限".equals(value)) return "0";
if ("实习生".equals(value)) return "1";
if ("应届毕业生".equals(value)) return "2";
if ("1年以下".equals(value) || "1年".equals(value)) return "3";
if ("1-3年".equals(value) || "1-3年工作经验".equals(value) || "2年".equals(value) || "3年".equals(value)) return "4";
if ("3-5年".equals(value)) return "5";
if ("5-10年".equals(value)) return "6";
if ("10年以上".equals(value)) return "7";
candidate.getWarnings().add("经验未映射，按经验不限处理: " + value);
return "0";
}
private String areaCode(String location, ExternalJobCandidate candidate) {
if (isBlank(location)) {
return null;
}
List<String> values = jdbcTemplate.queryForList("SELECT dict_value FROM bussiness_dict_data WHERE dict_type='area' AND dict_label=? AND status='0' LIMIT 1", String.class, location);
if (values.isEmpty()) {
candidate.getWarnings().add("未匹配到区域字典: " + location);
return null;
}
return values.get(0);
}
private String dictionaryCode(String type, String raw, ExternalJobCandidate candidate, Map<String, String> cache) {
if (isBlank(raw)) {
return null;
}
String label = trim(raw);
String cacheKey = type + "|" + label;
if (cache.containsKey(cacheKey)) {
return emptyToNull(cache.get(cacheKey));
}
List<String> values = jdbcTemplate.queryForList(
"SELECT dict_value FROM bussiness_dict_data WHERE dict_type=? AND dict_label=? AND status='0' LIMIT 1",
String.class, type, label);
if (values.isEmpty() && "scale".equals(type) && !label.endsWith("人")) {
values = jdbcTemplate.queryForList(
"SELECT dict_value FROM bussiness_dict_data WHERE dict_type=? AND dict_label=? AND status='0' LIMIT 1",
String.class, type, label + "人");
}
String code = values.isEmpty() ? "" : values.get(0);
cache.put(cacheKey, code);
if (isBlank(code)) {
candidate.getWarnings().add("未匹配到 " + type + " 字典: " + label);
return null;
}
return code;
}
private Long parseMoney(String value) {
try {
if (isBlank(value)) return null;
return new BigDecimal(trim(value)).longValueExact();
} catch (Exception ex) {
return null;
}
}
private Long vacancies(String value) {
if (isBlank(value)) return -1L;
String normalized = value.replace("人", "").replace("以上", "").trim();
if (normalized.contains("-")) {
normalized = normalized.substring(normalized.lastIndexOf('-') + 1).trim();
}
try {
return new BigDecimal(normalized.replaceAll("[^0-9.]", "")).longValue();
} catch (Exception ex) {
return -1L;
}
}
private Timestamp timestamp(String value) {
return Timestamp.valueOf(LocalDate.parse(value).atStartOfDay());
}
private boolean isDate(String value) {
try {
LocalDate.parse(value);
return true;
} catch (Exception ex) {
return false;
}
}
private String placeholders(int size) {
List<String> values = new ArrayList<>();
for (int i = 0; i < size; i++) values.add("?");
return join(values);
}
private String trim(String value) {
return value == null ? "" : value.trim();
}
private String firstNonBlank(String first, String second) {
return !isBlank(first) ? first : second;
}
private String emptyToNull(String value) {
return isBlank(value) ? null : value.trim();
}
private boolean isBlank(String value) {
return value == null || value.trim().isEmpty();
}
private String join(Collection<String> values) {
StringBuilder builder = new StringBuilder();
for (String value : values) {
if (isBlank(value)) continue;
if (builder.length() > 0) builder.append(",");
builder.append(value);
}
return builder.toString();
}
private static final class ExistingJob {
private final Long jobId;
private final String fingerprint;
private final String contentFingerprint;
private ExistingJob(Long jobId, String fingerprint, String contentFingerprint) {
this.jobId = jobId;
this.fingerprint = fingerprint;
this.contentFingerprint = contentFingerprint;
}
}
}

14
ruoyi-bussiness/src/main/java/com/ruoyi/cms/externalimport/ExternalJobSourceAliasRequest.java Normal file
View File

@@ -0,0 +1,14 @@
package com.ruoyi.cms.externalimport;
import lombok.Data;
/** 外部 Excel ORG 到系统稳定来源编码的后台维护请求。 */
@Data
public class ExternalJobSourceAliasRequest {
private String aliasName;
private String sourceCode;
private Long websiteId;
private String isActive = "1";
private String snapshotRequired = "1";
private String remark;
}

102
ruoyi-bussiness/src/main/java/com/ruoyi/cms/externalimport/ExternalUploadClient.java Normal file
View File

@@ -0,0 +1,102 @@
package com.ruoyi.cms.externalimport;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.stereotype.Component;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
/** 与外部 Excel 上传系统交互的最小内部客户端。 */
@Component
public class ExternalUploadClient {
private final ExternalJobImportProperties properties;
private final ObjectMapper objectMapper;
public ExternalUploadClient(ExternalJobImportProperties properties, ObjectMapper objectMapper) {
this.properties = properties;
this.objectMapper = objectMapper;
}
public ExternalUploadFileMetadata latest() throws IOException {
if (isBlank(properties.getLatestMetadataUrl())) {
throw new IllegalStateException("external-job-import.latest-metadata-url 未配置");
}
HttpURLConnection connection = open(properties.getLatestMetadataUrl());
try {
requireOk(connection, "获取最新外部 Excel 元数据失败");
JsonNode root = objectMapper.readTree(connection.getInputStream());
JsonNode data = root.path("data");
if (data.isMissingNode() || data.isNull()) {
throw new IOException("外部上传服务没有返回可导入文件");
}
ExternalUploadFileMetadata metadata = objectMapper.treeToValue(data, ExternalUploadFileMetadata.class);
if (isBlank(metadata.getFileId()) || isBlank(metadata.getSha256()) || isBlank(metadata.getDownloadUrl())) {
throw new IOException("外部上传服务返回的文件元数据不完整");
}
metadata.setDownloadUrl(resolve(properties.getLatestMetadataUrl(), metadata.getDownloadUrl()));
return metadata;
} finally {
connection.disconnect();
}
}
public Download download(ExternalUploadFileMetadata metadata) throws IOException {
HttpURLConnection connection = open(metadata.getDownloadUrl());
requireOk(connection, "下载外部 Excel 文件失败");
return new Download(connection, new BufferedInputStream(connection.getInputStream()));
}
private HttpURLConnection open(String target) throws IOException {
HttpURLConnection connection = (HttpURLConnection) new URL(target).openConnection();
connection.setRequestMethod("GET");
connection.setConnectTimeout(properties.getConnectTimeoutMillis());
connection.setReadTimeout(properties.getReadTimeoutMillis());
connection.setRequestProperty("Accept", "application/json, application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
if (!isBlank(properties.getInternalToken())) {
connection.setRequestProperty("X-Internal-Token", properties.getInternalToken());
}
return connection;
}
private void requireOk(HttpURLConnection connection, String message) throws IOException {
int status = connection.getResponseCode();
if (status != HttpURLConnection.HTTP_OK) {
throw new IOException(message + "，HTTP 状态码: " + status);
}
}
private String resolve(String base, String value) throws IOException {
return new URL(new URL(base), value).toExternalForm();
}
private boolean isBlank(String value) {
return value == null || value.trim().isEmpty();
}
public static final class Download implements AutoCloseable {
private final HttpURLConnection connection;
private final InputStream inputStream;
private Download(HttpURLConnection connection, InputStream inputStream) {
this.connection = connection;
this.inputStream = inputStream;
}
public InputStream getInputStream() {
return inputStream;
}
@Override
public void close() throws IOException {
try {
inputStream.close();
} finally {
connection.disconnect();
}
}
}
}

22
ruoyi-bussiness/src/main/java/com/ruoyi/cms/externalimport/ExternalUploadFileMetadata.java Normal file
View File

@@ -0,0 +1,22 @@
package com.ruoyi.cms.externalimport;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
/** 外部上传服务返回的一个可导入文件版本。 */
@Data
public class ExternalUploadFileMetadata {
@JsonProperty("file_id")
private String fileId;
private String filename;
private String sha256;
@JsonProperty("download_url")
private String downloadUrl;
@JsonProperty("upload_time")
private String uploadTime;
@JsonProperty("source_scope")
private List<String> sourceScope = new ArrayList<>();
}

14
ruoyi-bussiness/src/main/java/com/ruoyi/cms/externalimport/IExternalJobImportService.java Normal file
View File

@@ -0,0 +1,14 @@
package com.ruoyi.cms.externalimport;
import java.util.List;
import java.util.Map;
public interface IExternalJobImportService {
ExternalJobImportResult importLatest();
List<Map<String, Object>> listBatches(String status, Integer limit);
List<Map<String, Object>> listSourceAliases();
void saveSourceAlias(ExternalJobSourceAliasRequest request);
}

8
ruoyi-bussiness/src/main/java/com/ruoyi/cms/service/impl/JobDataTrendServiceImpl.java
View File

@@ -85,7 +85,9 @@ public class JobDataTrendServiceImpl extends ServiceImpl<JobDataTrendMapper, Job
// ================= Step 2: 查询数据库原始数据 =================
LambdaQueryWrapper<JobDataTrend> lqw = Wrappers.lambdaQuery();
lqw.eq(JobDataTrend::getWebsiteName, "total");
String sourceCode = params != null && StringUtils.isNotEmpty(params.get("sourceCode"))
? params.get("sourceCode") : "total";
lqw.eq(JobDataTrend::getWebsiteName, sourceCode);
// 查询范围： >= Start 00:00:00 且 <= End 23:59:59
// 注意：数据库存的是 Date (带时分秒)，这里转一下
lqw.ge(JobDataTrend::getStorageTime, Date.from(start.atStartOfDay(ZoneId.systemDefault()).toInstant()));
@@ -133,7 +135,7 @@ public class JobDataTrendServiceImpl extends ServiceImpl<JobDataTrendMapper, Job
} else {
// 2. 如果当天没数据，创建一个全0对象
JobDataTrend emptyData = new JobDataTrend();
emptyData.setWebsiteName("total");
emptyData.setWebsiteName(sourceCode);
emptyData.setInsertCount(0L);
emptyData.setDeleteCount(0L);
emptyData.setFailCount(0L);
@@ -146,4 +148,4 @@ public class JobDataTrendServiceImpl extends ServiceImpl<JobDataTrendMapper, Job
// 返回结果（此时 resultList 已经是按时间正序排列的：从旧到新）
return resultList;
}
}
}

2
ruoyi-bussiness/src/main/resources/mapper/rc/PublicJobFairMapper.xml
View File

@@ -92,7 +92,7 @@
<where>
del_flag = '0'
<if test="query.jobFairTitle != null and query.jobFairTitle != ''">
and job_fair_title like concat('%', #{query.jobFairTitle}, '%')
and job_fair_title like concat('%', cast(#{query.jobFairTitle, jdbcType=VARCHAR} as varchar), '%')
</if>
<if test="query.jobFairType != null and query.jobFairType != ''">
and job_fair_type = #{query.jobFairType}

56
ruoyi-bussiness/src/test/java/com/ruoyi/cms/externalimport/ExternalJobExcelParserTest.java Normal file
View File

@@ -0,0 +1,56 @@
package com.ruoyi.cms.externalimport;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.CellStyle;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.junit.jupiter.api.Test;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.time.LocalDate;
import java.util.Date;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
class ExternalJobExcelParserTest {
@Test
void parsesExcelDatesAndLocationFallback() throws Exception {
byte[] data = workbookBytes();
List<ExternalJobCandidate> rows = new ExternalJobExcelParser().parse(new ByteArrayInputStream(data));
assertEquals(1, rows.size());
ExternalJobCandidate row = rows.get(0);
assertEquals("2026-07-14", row.getPostingDate());
assertEquals("2026-07-20", row.getCollectDate());
assertEquals("石河子市", row.getLocation());
assertEquals("Java 工程师", row.getJobTitle());
}
private byte[] workbookBytes() throws Exception {
String[] headers = {"Aca112", "Acb22a", "SalaryLow", "SalaryHight", "Aae397", "AAB004", "ORG", "ACE760", "Collect_time", "City"};
try (XSSFWorkbook workbook = new XSSFWorkbook(); ByteArrayOutputStream output = new ByteArrayOutputStream()) {
Row header = workbook.createSheet("合并数据").createRow(0);
for (int i = 0; i < headers.length; i++) {
header.createCell(i).setCellValue(headers[i]);
}
Row data = workbook.getSheetAt(0).createRow(1);
CellStyle dateStyle = workbook.createCellStyle();
dateStyle.setDataFormat(workbook.getCreationHelper().createDataFormat().getFormat("yyyy-mm-dd"));
data.createCell(0).setCellValue("Java 工程师");
data.createCell(1).setCellValue("负责平台开发");
data.createCell(2).setCellValue(5000);
data.createCell(3).setCellValue(10000);
data.createCell(4).setCellValue(Date.from(LocalDate.of(2026, 7, 14).atStartOfDay(java.time.ZoneId.systemDefault()).toInstant()));
data.getCell(4).setCellStyle(dateStyle);
data.createCell(5).setCellValue("示例企业");
data.createCell(6).setCellValue("智联招聘");
data.createCell(7).setCellValue("https://www.zhaopin.com/job/42");
data.createCell(8).setCellValue(Date.from(LocalDate.of(2026, 7, 20).atStartOfDay(java.time.ZoneId.systemDefault()).toInstant()));
data.getCell(8).setCellStyle(dateStyle);
data.createCell(9).setCellValue("石河子市");
workbook.write(output);
return output.toByteArray();
}
}
}

42
ruoyi-bussiness/src/test/java/com/ruoyi/cms/externalimport/ExternalJobFingerprintTest.java Normal file
View File

@@ -0,0 +1,42 @@
package com.ruoyi.cms.externalimport;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
class ExternalJobFingerprintTest {
@Test
void identityFingerprintIgnoresTrackingParametersAndMutableContent() {
ExternalJobCandidate first = candidate("https://www.zhaopin.com/job/42/?utm_source=test&from=feed", "5000", "岗位描述一");
ExternalJobCandidate changed = candidate("https://www.zhaopin.com/job/42?utm_campaign=summer", "7000", "岗位描述二");
assertEquals(ExternalJobFingerprint.identity(first), ExternalJobFingerprint.identity(changed));
assertNotEquals(ExternalJobFingerprint.content(first), ExternalJobFingerprint.content(changed));
}
@Test
void identityFingerprintSeparatesDifferentSources() {
ExternalJobCandidate zhaopin = candidate("https://www.zhaopin.com/job/42", "5000", "岗位描述");
ExternalJobCandidate liepin = candidate("https://www.zhaopin.com/job/42", "5000", "岗位描述");
liepin.setSourceCode("liepin");
assertNotEquals(ExternalJobFingerprint.identity(zhaopin), ExternalJobFingerprint.identity(liepin));
}
private ExternalJobCandidate candidate(String url, String salary, String description) {
ExternalJobCandidate candidate = new ExternalJobCandidate();
candidate.setSourceCode("zhaopin");
candidate.setCompanyName("石河子示例企业");
candidate.setJobTitle("Java 工程师");
candidate.setLocation("石河子市");
candidate.setJobUrl(url);
candidate.setSalaryLow(salary);
candidate.setSalaryHigh("10000");
candidate.setDescription(description);
candidate.setEducationCode("3");
candidate.setExperienceCode("4");
candidate.setPostingDate("2026-07-20");
return candidate;
}
}