feat(job_crawler): initialize job crawler service with kafka integration
- Add technical documentation (技术方案.md) with system architecture and design details - Create FastAPI application structure with modular organization (api, core, models, services, utils) - Implement job data crawler service with incremental collection from third-party API - Add Kafka service integration with Docker Compose configuration for message queue - Create data models for job listings, progress tracking, and API responses - Implement REST API endpoints for data consumption (/consume, /status) and task management - Add progress persistence layer using SQLite for tracking collection offsets - Implement date filtering logic to extract data published within 7 days - Create API client service for third-party data source integration - Add configuration management with environment-based settings - Include Docker support with Dockerfile and docker-compose.yml for containerized deployment - Add logging configuration and utility functions for date parsing - Include requirements.txt with all Python dependencies and README documentation
This commit is contained in:
39
job_crawler/config/config.yml.docker
Normal file
39
job_crawler/config/config.yml.docker
Normal file
@@ -0,0 +1,39 @@
|
||||
# Docker环境配置文件
|
||||
# 复制此文件为 config.yml 并修改账号密码
|
||||
|
||||
# 应用配置
|
||||
app:
|
||||
name: job-crawler
|
||||
version: 1.0.0
|
||||
debug: false
|
||||
|
||||
# 八爪鱼API配置
|
||||
api:
|
||||
base_url: https://openapi.bazhuayu.com
|
||||
username: "your_username"
|
||||
password: "your_password"
|
||||
batch_size: 100
|
||||
# 多任务配置
|
||||
tasks:
|
||||
- id: "00f3b445-d8ec-44e8-88b2-4b971a228b1e"
|
||||
name: "青岛招聘数据"
|
||||
enabled: true
|
||||
- id: "task-id-2"
|
||||
name: "任务2"
|
||||
enabled: false
|
||||
|
||||
# Kafka配置(Docker内部网络)
|
||||
kafka:
|
||||
bootstrap_servers: kafka:29092
|
||||
topic: job_data
|
||||
consumer_group: job_consumer_group
|
||||
|
||||
# 采集配置
|
||||
crawler:
|
||||
interval: 300
|
||||
filter_days: 7
|
||||
max_workers: 5
|
||||
|
||||
# 数据库配置
|
||||
database:
|
||||
path: /app/data/crawl_progress.db
|
||||
Reference in New Issue
Block a user