docs(job_crawler): add deployment guide and scripts for Linux/Mac/Windows
- Add comprehensive DEPLOY.md with quick start instructions for all platforms - Add deploy.sh script for Linux/Mac with build, up, down, restart, logs, status, and clean commands - Add deploy.bat script for Windows with equivalent deployment commands - Include manual deployment steps using docker and docker-compose - Document configuration setup and environment variables - Add production environment recommendations for external Kafka, data persistence, and logging - Include troubleshooting section for common deployment issues - Provide health check and service status verification commands
This commit is contained in:
167
job_crawler/DEPLOY.md
Normal file
167
job_crawler/DEPLOY.md
Normal file
@@ -0,0 +1,167 @@
|
||||
# 部署指南
|
||||
|
||||
## 快速部署
|
||||
|
||||
### Linux/Mac
|
||||
|
||||
```bash
|
||||
# 添加执行权限
|
||||
chmod +x deploy.sh
|
||||
|
||||
# 构建镜像
|
||||
./deploy.sh build
|
||||
|
||||
# 启动服务
|
||||
./deploy.sh up
|
||||
|
||||
# 查看日志
|
||||
./deploy.sh logs
|
||||
```
|
||||
|
||||
### Windows
|
||||
|
||||
```cmd
|
||||
REM 构建镜像
|
||||
deploy.bat build
|
||||
|
||||
REM 启动服务
|
||||
deploy.bat up
|
||||
|
||||
REM 查看日志
|
||||
deploy.bat logs
|
||||
```
|
||||
|
||||
## 部署命令
|
||||
|
||||
| 命令 | 说明 |
|
||||
|------|------|
|
||||
| `build` | 构建Docker镜像 |
|
||||
| `up` | 启动所有服务（Kafka + App） |
|
||||
| `down` | 停止所有服务 |
|
||||
| `restart` | 重启应用服务 |
|
||||
| `logs` | 查看应用日志 |
|
||||
| `status` | 查看服务状态 |
|
||||
| `clean` | 清理无用镜像 |
|
||||
|
||||
## 手动部署
|
||||
|
||||
### 1. 构建镜像
|
||||
|
||||
```bash
|
||||
docker build -t job-crawler:latest .
|
||||
```
|
||||
|
||||
### 2. 推送到私有仓库（可选）
|
||||
|
||||
```bash
|
||||
# 打标签
|
||||
docker tag job-crawler:latest your-registry.com/job-crawler:latest
|
||||
|
||||
# 推送
|
||||
docker push your-registry.com/job-crawler:latest
|
||||
```
|
||||
|
||||
### 3. 启动服务
|
||||
|
||||
```bash
|
||||
# 使用 docker-compose（推荐）
|
||||
docker-compose up -d
|
||||
|
||||
# 或单独运行容器
|
||||
docker run -d \
|
||||
--name job-crawler \
|
||||
-p 8000:8000 \
|
||||
-v $(pwd)/config:/app/config:ro \
|
||||
-v job-data:/app/data \
|
||||
-e CONFIG_PATH=/app/config/config.yml \
|
||||
job-crawler:latest
|
||||
```
|
||||
|
||||
## 配置说明
|
||||
|
||||
### 1. 复制配置模板
|
||||
|
||||
```bash
|
||||
cp config/config.yml.docker config/config.yml
|
||||
```
|
||||
|
||||
### 2. 编辑配置文件
|
||||
|
||||
```yaml
|
||||
api:
|
||||
username: "your_username" # 修改为你的账号
|
||||
password: "your_password" # 修改为你的密码
|
||||
tasks:
|
||||
- id: "your-task-id" # 修改为你的任务ID
|
||||
name: "任务名称"
|
||||
enabled: true
|
||||
```
|
||||
|
||||
## 生产环境建议
|
||||
|
||||
### 1. 使用外部Kafka
|
||||
|
||||
修改 `config/config.yml`:
|
||||
|
||||
```yaml
|
||||
kafka:
|
||||
bootstrap_servers: your-kafka-host:9092
|
||||
```
|
||||
|
||||
修改 `docker-compose.yml`，移除 zookeeper 和 kafka 服务。
|
||||
|
||||
### 2. 数据持久化
|
||||
|
||||
确保挂载数据卷：
|
||||
|
||||
```yaml
|
||||
volumes:
|
||||
- ./data:/app/data # 本地目录
|
||||
# 或使用命名卷
|
||||
- app_data:/app/data
|
||||
```
|
||||
|
||||
### 3. 日志收集
|
||||
|
||||
```bash
|
||||
# 查看日志
|
||||
docker-compose logs -f app
|
||||
|
||||
# 导出日志
|
||||
docker-compose logs app > app.log
|
||||
```
|
||||
|
||||
### 4. 健康检查
|
||||
|
||||
```bash
|
||||
# 检查服务健康
|
||||
curl http://localhost:8000/health
|
||||
|
||||
# 检查采集状态
|
||||
curl http://localhost:8000/status
|
||||
```
|
||||
|
||||
## 常见问题
|
||||
|
||||
### Q: Kafka连接失败
|
||||
|
||||
检查 `config.yml` 中的 `kafka.bootstrap_servers`:
|
||||
- Docker内部: `kafka:29092`
|
||||
- 本地开发: `localhost:9092`
|
||||
|
||||
### Q: 配置文件未生效
|
||||
|
||||
确保配置文件正确挂载：
|
||||
|
||||
```bash
|
||||
docker exec job-crawler cat /app/config/config.yml
|
||||
```
|
||||
|
||||
### Q: 数据丢失
|
||||
|
||||
检查数据卷是否正确挂载：
|
||||
|
||||
```bash
|
||||
docker volume ls
|
||||
docker volume inspect job_crawler_app_data
|
||||
```
|
||||
85
job_crawler/deploy.bat
Normal file
85
job_crawler/deploy.bat
Normal file
@@ -0,0 +1,85 @@
|
||||
@echo off
|
||||
REM Windows 部署脚本
|
||||
setlocal enabledelayedexpansion
|
||||
|
||||
set IMAGE_NAME=job-crawler
|
||||
set IMAGE_TAG=latest
|
||||
set CONTAINER_NAME=job-crawler
|
||||
|
||||
if "%1"=="" goto help
|
||||
if "%1"=="build" goto build
|
||||
if "%1"=="up" goto up
|
||||
if "%1"=="down" goto down
|
||||
if "%1"=="restart" goto restart
|
||||
if "%1"=="logs" goto logs
|
||||
if "%1"=="status" goto status
|
||||
if "%1"=="clean" goto clean
|
||||
if "%1"=="help" goto help
|
||||
goto help
|
||||
|
||||
:build
|
||||
echo [INFO] 构建镜像: %IMAGE_NAME%:%IMAGE_TAG%
|
||||
docker build -t %IMAGE_NAME%:%IMAGE_TAG% .
|
||||
echo [INFO] 构建完成
|
||||
goto end
|
||||
|
||||
:up
|
||||
echo [INFO] 启动服务...
|
||||
if not exist "config\config.yml" (
|
||||
echo [WARN] 配置文件不存在，从模板复制
|
||||
copy config\config.yml.docker config\config.yml
|
||||
echo [WARN] 请编辑 config\config.yml 填入账号密码后重新运行
|
||||
goto end
|
||||
)
|
||||
docker-compose up -d
|
||||
echo [INFO] 服务已启动
|
||||
echo [INFO] API地址: http://localhost:8000
|
||||
echo [INFO] API文档: http://localhost:8000/docs
|
||||
goto end
|
||||
|
||||
:down
|
||||
echo [INFO] 停止服务...
|
||||
docker-compose down
|
||||
echo [INFO] 服务已停止
|
||||
goto end
|
||||
|
||||
:restart
|
||||
echo [INFO] 重启应用服务...
|
||||
docker-compose restart app
|
||||
echo [INFO] 重启完成
|
||||
goto end
|
||||
|
||||
:logs
|
||||
docker-compose logs -f --tail=100 app
|
||||
goto end
|
||||
|
||||
:status
|
||||
echo === 容器状态 ===
|
||||
docker-compose ps
|
||||
echo.
|
||||
echo === 采集状态 ===
|
||||
curl -s http://localhost:8000/status
|
||||
goto end
|
||||
|
||||
:clean
|
||||
echo [INFO] 清理无用镜像...
|
||||
docker image prune -f
|
||||
echo [INFO] 清理完成
|
||||
goto end
|
||||
|
||||
:help
|
||||
echo 用法: deploy.bat [命令]
|
||||
echo.
|
||||
echo 命令:
|
||||
echo build 构建镜像
|
||||
echo up 启动所有服务
|
||||
echo down 停止所有服务
|
||||
echo restart 重启应用服务
|
||||
echo logs 查看日志
|
||||
echo status 查看服务状态
|
||||
echo clean 清理无用镜像
|
||||
echo help 显示帮助
|
||||
goto end
|
||||
|
||||
:end
|
||||
endlocal
|
||||
123
job_crawler/deploy.sh
Normal file
123
job_crawler/deploy.sh
Normal file
@@ -0,0 +1,123 @@
|
||||
#!/bin/bash
|
||||
# 部署脚本
|
||||
|
||||
set -e
|
||||
|
||||
# 配置
|
||||
IMAGE_NAME="job-crawler"
|
||||
IMAGE_TAG="latest"
|
||||
CONTAINER_NAME="job-crawler"
|
||||
REGISTRY="" # 私有仓库地址，如: registry.example.com
|
||||
|
||||
# 颜色输出
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
|
||||
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
||||
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
|
||||
|
||||
# 显示帮助
|
||||
show_help() {
|
||||
echo "用法: ./deploy.sh [命令]"
|
||||
echo ""
|
||||
echo "命令:"
|
||||
echo " build 构建镜像"
|
||||
echo " push 推送镜像到仓库"
|
||||
echo " up 启动所有服务 (docker-compose)"
|
||||
echo " down 停止所有服务"
|
||||
echo " restart 重启应用服务"
|
||||
echo " logs 查看日志"
|
||||
echo " status 查看服务状态"
|
||||
echo " clean 清理无用镜像"
|
||||
echo " help 显示帮助"
|
||||
}
|
||||
|
||||
# 构建镜像
|
||||
build() {
|
||||
log_info "构建镜像: ${IMAGE_NAME}:${IMAGE_TAG}"
|
||||
docker build -t ${IMAGE_NAME}:${IMAGE_TAG} .
|
||||
log_info "构建完成"
|
||||
}
|
||||
|
||||
# 推送镜像
|
||||
push() {
|
||||
if [ -z "$REGISTRY" ]; then
|
||||
log_error "请先配置 REGISTRY 变量"
|
||||
exit 1
|
||||
fi
|
||||
FULL_IMAGE="${REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}"
|
||||
log_info "推送镜像: ${FULL_IMAGE}"
|
||||
docker tag ${IMAGE_NAME}:${IMAGE_TAG} ${FULL_IMAGE}
|
||||
docker push ${FULL_IMAGE}
|
||||
log_info "推送完成"
|
||||
}
|
||||
|
||||
# 启动服务
|
||||
up() {
|
||||
log_info "启动服务..."
|
||||
|
||||
# 检查配置文件
|
||||
if [ ! -f "config/config.yml" ]; then
|
||||
log_warn "配置文件不存在，从模板复制"
|
||||
cp config/config.yml.docker config/config.yml
|
||||
log_warn "请编辑 config/config.yml 填入账号密码后重新运行"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
docker-compose up -d
|
||||
log_info "服务已启动"
|
||||
log_info "API地址: http://localhost:8000"
|
||||
log_info "API文档: http://localhost:8000/docs"
|
||||
}
|
||||
|
||||
# 停止服务
|
||||
down() {
|
||||
log_info "停止服务..."
|
||||
docker-compose down
|
||||
log_info "服务已停止"
|
||||
}
|
||||
|
||||
# 重启应用
|
||||
restart() {
|
||||
log_info "重启应用服务..."
|
||||
docker-compose restart app
|
||||
log_info "重启完成"
|
||||
}
|
||||
|
||||
# 查看日志
|
||||
logs() {
|
||||
docker-compose logs -f --tail=100 app
|
||||
}
|
||||
|
||||
# 查看状态
|
||||
status() {
|
||||
echo "=== 容器状态 ==="
|
||||
docker-compose ps
|
||||
echo ""
|
||||
echo "=== 采集状态 ==="
|
||||
curl -s http://localhost:8000/status 2>/dev/null | python -m json.tool || echo "服务未启动"
|
||||
}
|
||||
|
||||
# 清理
|
||||
clean() {
|
||||
log_info "清理无用镜像..."
|
||||
docker image prune -f
|
||||
log_info "清理完成"
|
||||
}
|
||||
|
||||
# 主逻辑
|
||||
case "${1:-help}" in
|
||||
build) build ;;
|
||||
push) push ;;
|
||||
up) up ;;
|
||||
down) down ;;
|
||||
restart) restart ;;
|
||||
logs) logs ;;
|
||||
status) status ;;
|
||||
clean) clean ;;
|
||||
help) show_help ;;
|
||||
*) show_help ;;
|
||||
esac
|
||||
Reference in New Issue
Block a user