添加互联网抓取岗位上传功能——未完成

ruoyi-bussiness/src/main/java/com/ruoyi/cms/controller/cms/CmsJobController.java

@@ -16,6 +16,7 @@ import com.ruoyi.cms.util.excel.JobExcelListener;
 import com.ruoyi.cms.util.sensitiveWord.SensitiveWordChecker;
 import com.ruoyi.common.annotation.Anonymous;
 import com.ruoyi.common.annotation.Log;
+import com.ruoyi.common.config.RuoYiConfig;
 import com.ruoyi.common.constant.SM4Constants;
 import com.ruoyi.common.core.controller.BaseController;
 import com.ruoyi.common.core.domain.AjaxResult;
@@ -29,6 +30,7 @@ import com.ruoyi.common.utils.SM4Utils;
 import com.ruoyi.common.utils.SecurityUtils;
 import com.ruoyi.common.utils.StringUtils;
 import com.ruoyi.common.utils.bean.BeanUtils;
+import com.ruoyi.common.utils.file.FileUploadUtils;
 import com.ruoyi.common.utils.ip.IpUtils;
 import com.ruoyi.common.utils.poi.ExcelUtil;
 import io.swagger.annotations.Api;
@@ -703,4 +705,33 @@ public class CmsJobController extends BaseController
         rspData.setTotal(total > 200 ? 200 : total);
         return rspData;
     }
+
+    /**
+     * 通用上传请求（单个）
+     */
+    @PostMapping("/uploadFileHlwgwzq")
+    @PreAuthorize("@ss.hasPermi('bussiness:job:uploadFileHlwgwzq')")
+    @ApiOperation("互联网岗位抓取批量上传")
+    public AjaxResult uploadFileHlwgwzq(@RequestParam("file") MultipartFile file) throws Exception {
+        if (file.isEmpty()) {
+            return AjaxResult.error("上传文件不能为空");
+        }
+        String fileName = file.getOriginalFilename();
+        if (fileName == null || !fileName.endsWith(".xlsx") && !fileName.endsWith(".xls")) {
+            return AjaxResult.error("请上传Excel格式的文件");
+        }
+
+        // 上传文件路径
+        String filePath = RuoYiConfig.getUploadPath();
+        // 上传并返回新文件名称
+        try {
+            fileName = FileUploadUtils.upload(filePath, file);
+        } catch (IOException e) {
+            logger.error("读取文件出错", e);
+            return error("文件上传失败：" + e.getMessage());
+        }
+        String path = filePath + File.separator + fileName.replace("profile/upload/", "");
+        jobService.importRow(path);
+        return success();
+    }
 }

ruoyi-bussiness/src/main/java/com/ruoyi/cms/mapper/JobMapper.java

@@ -1,5 +1,6 @@
 package com.ruoyi.cms.mapper;
 
+import com.baomidou.mybatisplus.annotation.InterceptorIgnore;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 
 import java.util.Date;
@@ -73,4 +74,7 @@ public interface JobMapper extends BaseMapper<Job>
     void updateJobDown(List<Long> list);
 
     List<Notice> selectUserRecommendJobNotic(Date tenMinutesAgo);
+
+    @InterceptorIgnore(blockAttack = "true")
+    Long delRowWork();
 }

ruoyi-bussiness/src/main/java/com/ruoyi/cms/service/impl/JobServiceImpl.java

@@ -780,6 +780,10 @@ public class JobServiceImpl extends ServiceImpl<JobMapper,Job> implements IJobSe
     public void importRow(String path) {
         try {
             logger.info("开始导入Excel数据...");
+            //先清理旧数据
+            Long oldDataNum = jobMapper.delRowWork();
+            logger.info("删除旧的row-work..." + oldDataNum);
+            logger.info("文件路径path..." + path);
             // 读取Excel文件中的数据
             List<RowWork> rowWorks = ExcelToObject.readExcelToObjects(path, RowWork.class);
             logger.info("从Excel中读取到 {} 条数据", rowWorks.size());
@@ -790,6 +794,14 @@ public class JobServiceImpl extends ServiceImpl<JobMapper,Job> implements IJobSe
             orgDataMap.put("58同城", new ArrayList<>());
             orgDataMap.put("前程无忧", new ArrayList<>());
             orgDataMap.put("猎聘网", new ArrayList<>());
+            orgDataMap.put("新疆人才网", new ArrayList<>());
+            orgDataMap.put("今日招聘", new ArrayList<>());
+            orgDataMap.put("网隆招聘网", new ArrayList<>());
+            orgDataMap.put("建筑人才网", new ArrayList<>());
+            orgDataMap.put("工厂人才网", new ArrayList<>());
+            orgDataMap.put("技工人才网", new ArrayList<>());
+            orgDataMap.put("一览英才网", new ArrayList<>());
+            orgDataMap.put("电器人才网", new ArrayList<>());
 
             // 遍历数据，按ORG分类，每类最多取20000条
             for (RowWork rowWork : rowWorks) {
@@ -806,11 +818,28 @@ public class JobServiceImpl extends ServiceImpl<JobMapper,Job> implements IJobSe
             int zlCount = orgDataMap.get("智联招聘").size();
             int tcCount = orgDataMap.get("58同城").size();
             int qcCount = orgDataMap.get("前程无忧").size();
-            int lpCount = orgDataMap.get("前程无忧").size();
+            int lpCount = orgDataMap.get("猎聘网").size();
+            int xjrcCount = orgDataMap.get("新疆人才网").size();
+            int jrzpCount = orgDataMap.get("今日招聘").size();
+            int wlzpCount = orgDataMap.get("网隆招聘网").size();
+            int jzrcCount = orgDataMap.get("建筑人才网").size();
+            int gcrcCount = orgDataMap.get("工厂人才网").size();
+            int jgrcCount = orgDataMap.get("技工人才网").size();
+            int ylycCount = orgDataMap.get("一览英才网").size();
+            int dqrcCount = orgDataMap.get("电器人才网").size();
+
             logger.info("智联招聘数据量: {}", zlCount);
             logger.info("58同城数据量: {}", tcCount);
             logger.info("前程无忧数据量: {}", qcCount);
             logger.info("猎聘网数据量: {}", lpCount);
+            logger.info("新疆人才网: {}", xjrcCount);
+            logger.info("今日招聘: {}", jrzpCount);
+            logger.info("网隆招聘网: {}", wlzpCount);
+            logger.info("建筑人才网: {}", jzrcCount);
+            logger.info("工厂人才网: {}", gcrcCount);
+            logger.info("技工人才网: {}", jgrcCount);
+            logger.info("一览英才网: {}", ylycCount);
+            logger.info("电器人才网: {}", dqrcCount);
 
             // 合并所有数据（智联招聘 + 58同城 + 前程无忧）
             List<RowWork> allData = new ArrayList<>();
@@ -818,6 +847,14 @@ public class JobServiceImpl extends ServiceImpl<JobMapper,Job> implements IJobSe
             allData.addAll(orgDataMap.get("58同城"));
             allData.addAll(orgDataMap.get("前程无忧"));
             allData.addAll(orgDataMap.get("猎聘网"));
+            allData.addAll(orgDataMap.get("新疆人才网"));
+            allData.addAll(orgDataMap.get("今日招聘"));
+            allData.addAll(orgDataMap.get("网隆招聘网"));
+            allData.addAll(orgDataMap.get("建筑人才网"));
+            allData.addAll(orgDataMap.get("工厂人才网"));
+            allData.addAll(orgDataMap.get("技工人才网"));
+            allData.addAll(orgDataMap.get("一览英才网"));
+            allData.addAll(orgDataMap.get("电器人才网"));
 
             // 批量插入数据
             int batchSize = 50;
@@ -849,7 +886,8 @@ public class JobServiceImpl extends ServiceImpl<JobMapper,Job> implements IJobSe
                     logger.error("批量插入剩余数据失败: " + e.getMessage(), e);
                 }
             }
-
+            //插入岗位表
+//            importData();
             logger.info("数据导入完成，总共插入 {} 条数据", totalInserted);
         } catch (Exception e) {
             logger.error("导入数据过程中发生异常: " + e.getMessage(), e);
@@ -910,12 +948,19 @@ public class JobServiceImpl extends ServiceImpl<JobMapper,Job> implements IJobSe
             logger.info("生成唯一标识符: {}", string);
             LocalDateTime now = LocalDateTime.now();
             String formattedDate = now.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));
+            Job jobOne = null;
             for (RowWork rowWork : rowWorks) {
                 try {
                     Job job = new Job();
                     if(StringUtil.isEmptyOrNull(rowWork.getAca112())){
                         continue;
                     }
+                    //去重
+                    jobOne = jobMapper.selectOne(Wrappers.lambdaQuery(Job.class).eq(Job::getJobTitle, rowWork.getAca112()).eq(Job::getMinSalary, Long.valueOf(rowWork.getSalaryLow())).eq(Job::getMaxSalary, rowWork.getSalaryHight())
+                            .eq(Job::getCompanyName, rowWork.getAAB004()).eq(Job::getJobLocation, rowWork.getAAE006()).orderByDesc(Job::getCreateTime).last("LIMIT 1"));
+                    if (jobOne != null) {
+                        continue;
+                    }
                     job.setJobTitle(rowWork.getAca112());
                     job.setMinSalary(Long.valueOf(rowWork.getSalaryLow()));
                     job.setMaxSalary(Long.valueOf(rowWork.getSalaryHight()));
@@ -936,7 +981,15 @@ public class JobServiceImpl extends ServiceImpl<JobMapper,Job> implements IJobSe
                     if (rowWork.getAcb240() == null) {
                         job.setVacancies(-1L);
                     } else {
-                        job.setVacancies(Long.valueOf(rowWork.getAcb240().replace("人", "")));
+                        String v = rowWork.getAcb240().replace("人", "").replace("以上", "");
+                        if (v.contains("若干")) {
+                            job.setVacancies(20L);
+                        }else if (org.apache.commons.lang3.StringUtils.isNotEmpty(v) && v.contains("-")) {
+                            v = v.split("-")[1];
+                            job.setVacancies(org.apache.commons.lang3.StringUtils.isNotEmpty(v) ? Long.valueOf(v) : -1L);
+                        }else {
+                            job.setVacancies(org.apache.commons.lang3.StringUtils.isNotEmpty(v) ? Long.valueOf(v) : -1L);
+                        }
                     }
                     if (rowWork.getLatitude() != null && !rowWork.getLatitude().isEmpty()) {
                         job.setLatitude(new BigDecimal(rowWork.getLatitude()));
@@ -960,6 +1013,7 @@ public class JobServiceImpl extends ServiceImpl<JobMapper,Job> implements IJobSe
                     job.setCreateBy(string);
                     job.setCreateTime(formattedDate);
                     job.setRowId(Long.valueOf(rowWork.getId()));
+                    job.setJobStatus(StringUtil.HIRE_LY);//上架
 
                     jobBatch.add(job);
 
@@ -983,6 +1037,12 @@ public class JobServiceImpl extends ServiceImpl<JobMapper,Job> implements IJobSe
             }
 
             logger.info("数据导入完成，总共插入 {} 条公司数据和 {} 条职位数据", totalCompanyInserted, totalJobInserted);
+            Job job = jobMapper.selectOne(Wrappers.lambdaQuery(Job.class).eq(Job::getCreateTime, formattedDate).last("LIMIT 1"));
+            if (job != null) {
+                logger.info("存在新插入的数据，日期为：" + formattedDate);
+            } else {
+                logger.info("不存在新插入的数据：");
+            }
         } catch (Exception e) {
             logger.error("导入数据过程中发生异常: {}", e.getMessage(), e);
             throw new RuntimeException(e);

ruoyi-bussiness/src/main/resources/mapper/app/JobMapper.xml

@@ -431,4 +431,8 @@
         WHERE j.create_time >= #{tenMinutesAgo} AND j.is_publish='1' AND j.job_status='0' AND um.notice_id IS NULL
     </select>
 
+    <delete id="delRowWork">
+        delete from row_work
+    </delete>
+
 </mapper>