zkr/ocups-kafka
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
李顺东
2026-01-15 23:00:22 +08:00
parent 67128c616f
commit 37af630893
1 changed files with 6 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
job_crawler/app/services/crawler.py
View File

@@ -11,11 +11,14 @@ from app.core.config import settings
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# 任务完成后重新执行的延迟时间（秒）
RESCHEDULE_DELAY_SECONDS = 3600 # 1小时
class TaskCrawler: class TaskCrawler:
"""单个任务采集器 - 从后往前采集""" """单个任务采集器 - 从后往前采集"""
def __init__(self, task_config): def __init__(self, task_config, on_complete_callback=None):
self.task_id = task_config.id self.task_id = task_config.id
self.task_name = task_config.name or task_config.id self.task_name = task_config.name or task_config.id
self.batch_size = settings.api.batch_size self.batch_size = settings.api.batch_size
@@ -24,6 +27,8 @@ class TaskCrawler:
self._running = False self._running = False
self._total_filtered = 0 self._total_filtered = 0
self._total_produced = 0 self._total_produced = 0
self._on_complete_callback = on_complete_callback # 完成回调
self._scheduled_task: Optional[asyncio.Task] = None # 延迟调度任务
@property @property
def is_running(self) -> bool: def is_running(self) -> bool: