ORIGIN DATA

job_crawler/app/services/crawler.py

@@ -7,7 +7,6 @@ from app.services.api_client import api_client
 from app.services.kafka_service import kafka_service
 from app.services.progress_store import progress_store
 from app.utils import is_within_days
-from app.models import JobData
 from app.core.config import settings
 
 logger = logging.getLogger(__name__)
@@ -128,24 +127,24 @@ class TaskCrawler:
             logger.info(f"[{self.task_name}] offset={offset}, 返回数据为空")
             return 0
         
-        # 过滤数据
-        filtered_jobs = []
+        # 过滤数据，保留原始内容
+        filtered_raw_data = []
         for raw in data_list:
             aae397 = raw.get("aae397", "")
             collect_time = raw.get("Collect_time", "")
             if is_within_days(aae397, collect_time, self.filter_days):
-                job = JobData.from_raw(raw)
-                job.task_id = self.task_id
-                filtered_jobs.append(job)
+                # 添加 task_id 到原始数据
+                raw["task_id"] = self.task_id
+                filtered_raw_data.append(raw)
         
-        valid_count = len(filtered_jobs)
+        valid_count = len(filtered_raw_data)
         expired_count = len(data_list) - valid_count
         self._total_filtered += valid_count
         
-        # 立即发送到Kafka
+        # 立即发送到Kafka（原始数据）
         produced = 0
-        if filtered_jobs:
-            produced = kafka_service.produce_batch(filtered_jobs)
+        if filtered_raw_data:
+            produced = kafka_service.produce_batch_raw(filtered_raw_data)
             self._total_produced += produced
         
         logger.info(f"[{self.task_name}] offset={offset}, 获取={len(data_list)}, 有效={valid_count}, 过期={expired_count}, 发送Kafka={produced}")