lsd/ks
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

添加互联网抓取岗位上传功能——未完成

Browse Source
This commit is contained in:
sh
2026-06-24 12:24:11 +08:00
parent 59530c228d
commit 6e27560077
4 changed files with 102 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
ruoyi-bussiness/src/main/java/com/ruoyi/cms/controller/cms/CmsJobController.java
View File

@@ -16,6 +16,7 @@ import com.ruoyi.cms.util.excel.JobExcelListener;
import com.ruoyi.cms.util.sensitiveWord.SensitiveWordChecker;
import com.ruoyi.common.annotation.Anonymous;
import com.ruoyi.common.annotation.Log;
import com.ruoyi.common.config.RuoYiConfig;
import com.ruoyi.common.constant.SM4Constants;
import com.ruoyi.common.core.controller.BaseController;
import com.ruoyi.common.core.domain.AjaxResult;
@@ -29,6 +30,7 @@ import com.ruoyi.common.utils.SM4Utils;
import com.ruoyi.common.utils.SecurityUtils;
import com.ruoyi.common.utils.StringUtils;
import com.ruoyi.common.utils.bean.BeanUtils;
import com.ruoyi.common.utils.file.FileUploadUtils;
import com.ruoyi.common.utils.ip.IpUtils;
import com.ruoyi.common.utils.poi.ExcelUtil;
import io.swagger.annotations.Api;
@@ -703,4 +705,33 @@ public class CmsJobController extends BaseController
rspData.setTotal(total > 200 ? 200 : total);
return rspData;
}
/**
* 通用上传请求（单个）
*/
@PostMapping("/uploadFileHlwgwzq")
@PreAuthorize("@ss.hasPermi('bussiness:job:uploadFileHlwgwzq')")
@ApiOperation("互联网岗位抓取批量上传")
public AjaxResult uploadFileHlwgwzq(@RequestParam("file") MultipartFile file) throws Exception {
if (file.isEmpty()) {
return AjaxResult.error("上传文件不能为空");
}
String fileName = file.getOriginalFilename();
if (fileName == null || !fileName.endsWith(".xlsx") && !fileName.endsWith(".xls")) {
return AjaxResult.error("请上传Excel格式的文件");
}
// 上传文件路径
String filePath = RuoYiConfig.getUploadPath();
// 上传并返回新文件名称
try {
fileName = FileUploadUtils.upload(filePath, file);
} catch (IOException e) {
logger.error("读取文件出错", e);
return error("文件上传失败：" + e.getMessage());
}
String path = filePath + File.separator + fileName.replace("profile/upload/", "");
jobService.importRow(path);
return success();
}
}

4
ruoyi-bussiness/src/main/java/com/ruoyi/cms/mapper/JobMapper.java
View File

@@ -1,5 +1,6 @@
package com.ruoyi.cms.mapper;
import com.baomidou.mybatisplus.annotation.InterceptorIgnore;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import java.util.Date;
@@ -73,4 +74,7 @@ public interface JobMapper extends BaseMapper<Job>
void updateJobDown(List<Long> list);
List<Notice> selectUserRecommendJobNotic(Date tenMinutesAgo);
@InterceptorIgnore(blockAttack = "true")
Long delRowWork();
}

66
ruoyi-bussiness/src/main/java/com/ruoyi/cms/service/impl/JobServiceImpl.java
View File

@@ -780,6 +780,10 @@ public class JobServiceImpl extends ServiceImpl<JobMapper,Job> implements IJobSe
public void importRow(String path) {
try {
logger.info("开始导入Excel数据...");
//先清理旧数据
Long oldDataNum = jobMapper.delRowWork();
logger.info("删除旧的row-work..." + oldDataNum);
logger.info("文件路径path..." + path);
// 读取Excel文件中的数据
List<RowWork> rowWorks = ExcelToObject.readExcelToObjects(path, RowWork.class);
logger.info("从Excel中读取到 {} 条数据", rowWorks.size());
@@ -790,6 +794,14 @@ public class JobServiceImpl extends ServiceImpl<JobMapper,Job> implements IJobSe
orgDataMap.put("58同城", new ArrayList<>());
orgDataMap.put("前程无忧", new ArrayList<>());
orgDataMap.put("猎聘网", new ArrayList<>());
orgDataMap.put("新疆人才网", new ArrayList<>());
orgDataMap.put("今日招聘", new ArrayList<>());
orgDataMap.put("网隆招聘网", new ArrayList<>());
orgDataMap.put("建筑人才网", new ArrayList<>());
orgDataMap.put("工厂人才网", new ArrayList<>());
orgDataMap.put("技工人才网", new ArrayList<>());
orgDataMap.put("一览英才网", new ArrayList<>());
orgDataMap.put("电器人才网", new ArrayList<>());
// 遍历数据，按ORG分类，每类最多取20000条
for (RowWork rowWork : rowWorks) {
@@ -806,11 +818,28 @@ public class JobServiceImpl extends ServiceImpl<JobMapper,Job> implements IJobSe
int zlCount = orgDataMap.get("智联招聘").size();
int tcCount = orgDataMap.get("58同城").size();
int qcCount = orgDataMap.get("前程无忧").size();
int lpCount = orgDataMap.get("前程无忧").size();
int lpCount = orgDataMap.get("猎聘网").size();
int xjrcCount = orgDataMap.get("新疆人才网").size();
int jrzpCount = orgDataMap.get("今日招聘").size();
int wlzpCount = orgDataMap.get("网隆招聘网").size();
int jzrcCount = orgDataMap.get("建筑人才网").size();
int gcrcCount = orgDataMap.get("工厂人才网").size();
int jgrcCount = orgDataMap.get("技工人才网").size();
int ylycCount = orgDataMap.get("一览英才网").size();
int dqrcCount = orgDataMap.get("电器人才网").size();
logger.info("智联招聘数据量: {}", zlCount);
logger.info("58同城数据量: {}", tcCount);
logger.info("前程无忧数据量: {}", qcCount);
logger.info("猎聘网数据量: {}", lpCount);
logger.info("新疆人才网: {}", xjrcCount);
logger.info("今日招聘: {}", jrzpCount);
logger.info("网隆招聘网: {}", wlzpCount);
logger.info("建筑人才网: {}", jzrcCount);
logger.info("工厂人才网: {}", gcrcCount);
logger.info("技工人才网: {}", jgrcCount);
logger.info("一览英才网: {}", ylycCount);
logger.info("电器人才网: {}", dqrcCount);
// 合并所有数据（智联招聘 + 58同城 + 前程无忧）
List<RowWork> allData = new ArrayList<>();
@@ -818,6 +847,14 @@ public class JobServiceImpl extends ServiceImpl<JobMapper,Job> implements IJobSe
allData.addAll(orgDataMap.get("58同城"));
allData.addAll(orgDataMap.get("前程无忧"));
allData.addAll(orgDataMap.get("猎聘网"));
allData.addAll(orgDataMap.get("新疆人才网"));
allData.addAll(orgDataMap.get("今日招聘"));
allData.addAll(orgDataMap.get("网隆招聘网"));
allData.addAll(orgDataMap.get("建筑人才网"));
allData.addAll(orgDataMap.get("工厂人才网"));
allData.addAll(orgDataMap.get("技工人才网"));
allData.addAll(orgDataMap.get("一览英才网"));
allData.addAll(orgDataMap.get("电器人才网"));
// 批量插入数据
int batchSize = 50;
@@ -849,7 +886,8 @@ public class JobServiceImpl extends ServiceImpl<JobMapper,Job> implements IJobSe
logger.error("批量插入剩余数据失败: " + e.getMessage(), e);
}
}
//插入岗位表
// importData();
logger.info("数据导入完成，总共插入 {} 条数据", totalInserted);
} catch (Exception e) {
logger.error("导入数据过程中发生异常: " + e.getMessage(), e);
@@ -910,12 +948,19 @@ public class JobServiceImpl extends ServiceImpl<JobMapper,Job> implements IJobSe
logger.info("生成唯一标识符: {}", string);
LocalDateTime now = LocalDateTime.now();
String formattedDate = now.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
Job jobOne = null;
for (RowWork rowWork : rowWorks) {
try {
Job job = new Job();
if(StringUtil.isEmptyOrNull(rowWork.getAca112())){
continue;
}
//去重
jobOne = jobMapper.selectOne(Wrappers.lambdaQuery(Job.class).eq(Job::getJobTitle, rowWork.getAca112()).eq(Job::getMinSalary, Long.valueOf(rowWork.getSalaryLow())).eq(Job::getMaxSalary, rowWork.getSalaryHight())
.eq(Job::getCompanyName, rowWork.getAAB004()).eq(Job::getJobLocation, rowWork.getAAE006()).orderByDesc(Job::getCreateTime).last("LIMIT 1"));
if (jobOne != null) {
continue;
}
job.setJobTitle(rowWork.getAca112());
job.setMinSalary(Long.valueOf(rowWork.getSalaryLow()));
job.setMaxSalary(Long.valueOf(rowWork.getSalaryHight()));
@@ -936,7 +981,15 @@ public class JobServiceImpl extends ServiceImpl<JobMapper,Job> implements IJobSe
if (rowWork.getAcb240() == null) {
job.setVacancies(-1L);
} else {
job.setVacancies(Long.valueOf(rowWork.getAcb240().replace("人", "")));
String v = rowWork.getAcb240().replace("人", "").replace("以上", "");
if (v.contains("若干")) {
job.setVacancies(20L);
}else if (org.apache.commons.lang3.StringUtils.isNotEmpty(v) && v.contains("-")) {
v = v.split("-")[1];
job.setVacancies(org.apache.commons.lang3.StringUtils.isNotEmpty(v) ? Long.valueOf(v) : -1L);
}else {
job.setVacancies(org.apache.commons.lang3.StringUtils.isNotEmpty(v) ? Long.valueOf(v) : -1L);
}
}
if (rowWork.getLatitude() != null && !rowWork.getLatitude().isEmpty()) {
job.setLatitude(new BigDecimal(rowWork.getLatitude()));
@@ -960,6 +1013,7 @@ public class JobServiceImpl extends ServiceImpl<JobMapper,Job> implements IJobSe
job.setCreateBy(string);
job.setCreateTime(formattedDate);
job.setRowId(Long.valueOf(rowWork.getId()));
job.setJobStatus(StringUtil.HIRE_LY);//上架
jobBatch.add(job);
@@ -983,6 +1037,12 @@ public class JobServiceImpl extends ServiceImpl<JobMapper,Job> implements IJobSe
}
logger.info("数据导入完成，总共插入 {} 条公司数据和 {} 条职位数据", totalCompanyInserted, totalJobInserted);
Job job = jobMapper.selectOne(Wrappers.lambdaQuery(Job.class).eq(Job::getCreateTime, formattedDate).last("LIMIT 1"));
if (job != null) {
logger.info("存在新插入的数据，日期为：" + formattedDate);
} else {
logger.info("不存在新插入的数据：");
}
} catch (Exception e) {
logger.error("导入数据过程中发生异常: {}", e.getMessage(), e);
throw new RuntimeException(e);

4
ruoyi-bussiness/src/main/resources/mapper/app/JobMapper.xml
View File

@@ -431,4 +431,8 @@
WHERE j.create_time >= #{tenMinutesAgo} AND j.is_publish='1' AND j.job_status='0' AND um.notice_id IS NULL
</select>
<delete id="delRowWork">
delete from row_work
</delete>
</mapper>