rabbitmq

job_crawler/app/services/crawler.py

@@ -4,7 +4,7 @@ import logging
 from typing import Dict, Optional
 from concurrent.futures import ThreadPoolExecutor
 from app.services.api_client import api_client
-from app.services.kafka_service import kafka_service
+from app.services.rabbitmq_service import rabbitmq_service
 from app.services.progress_store import progress_store
 from app.utils import is_within_days
 from app.models import JobData
@@ -134,21 +134,20 @@ class TaskCrawler:
             aae397 = raw.get("aae397", "")
             collect_time = raw.get("Collect_time", "")
             if is_within_days(aae397, collect_time, self.filter_days):
-                job = JobData.from_raw(raw)
-                job.task_id = self.task_id
+                job = JobData.from_raw(raw, self.task_id)
                 filtered_jobs.append(job)
         
         valid_count = len(filtered_jobs)
         expired_count = len(data_list) - valid_count
         self._total_filtered += valid_count
         
-        # 立即发送到Kafka
+        # 立即发送到RabbitMQ
         produced = 0
         if filtered_jobs:
-            produced = kafka_service.produce_batch(filtered_jobs)
+            produced = rabbitmq_service.produce_batch(filtered_jobs)
             self._total_produced += produced
         
-        logger.info(f"[{self.task_name}] offset={offset}, 获取={len(data_list)}, 有效={valid_count}, 过期={expired_count}, 发送Kafka={produced}")
+        logger.info(f"[{self.task_name}] offset={offset}, 获取={len(data_list)}, 有效={valid_count}, 过期={expired_count}, 发送MQ={produced}")
         
         return valid_count
     
@@ -236,7 +235,7 @@ class CrawlerManager:
             return crawler.get_status() if crawler else {}
         return {
             "tasks": [c.get_status() for c in self._crawlers.values()],
-            "kafka_lag": kafka_service.get_lag(),
+            "queue_size": rabbitmq_service.get_queue_size(),
             "running_count": sum(1 for c in self._crawlers.values() if c.is_running)
         }