main:支持 Worker 模式运行并优化任务管理
变更内容: - 在 `Dockerfile` 和 `docker-compose.yml` 中添加 Worker 模式支持,包含运行模式 `RUN_MODE` 的配置。 - 更新 API 路由,改为将任务入队处理,并由 Worker 执行。 - 在 JobManager 中新增任务队列及分布式锁功能,支持任务的入队、出队、执行控制以及重试机制。 - 添加全局并发控制逻辑,避免任务超额运行。 - 扩展单元测试,覆盖任务队列、锁机制和并发控制的各类场景。 - 在 Serverless 配置中分别为 API 和 Worker 添加独立服务定义。 提升任务调度灵活性,增强系统可靠性与扩展性。
This commit is contained in:
@@ -30,9 +30,15 @@ USER appuser
|
||||
# 暴露端口
|
||||
EXPOSE 8000
|
||||
|
||||
# 健康检查
|
||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
||||
CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/healthz')"
|
||||
# 运行模式:api(默认)或 worker
|
||||
ENV RUN_MODE=api
|
||||
|
||||
# 启动命令
|
||||
CMD ["uvicorn", "functional_scaffold.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
# 健康检查(仅对 API 模式有效)
|
||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
||||
CMD if [ "$RUN_MODE" = "api" ]; then python -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/healthz')"; else exit 0; fi
|
||||
|
||||
# 启动脚本
|
||||
COPY --chown=appuser:appuser deployment/entrypoint.sh /app/entrypoint.sh
|
||||
RUN chmod +x /app/entrypoint.sh
|
||||
|
||||
CMD ["/app/entrypoint.sh"]
|
||||
@@ -11,6 +11,7 @@ services:
|
||||
- APP_ENV=development
|
||||
- LOG_LEVEL=INFO
|
||||
- METRICS_ENABLED=true
|
||||
- RUN_MODE=api
|
||||
# Redis 指标存储配置
|
||||
- REDIS_HOST=redis
|
||||
- REDIS_PORT=6379
|
||||
@@ -38,6 +39,38 @@ services:
|
||||
retries: 3
|
||||
start_period: 5s
|
||||
|
||||
# Worker 服务 - 处理异步任务
|
||||
worker:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: deployment/Dockerfile
|
||||
environment:
|
||||
- APP_ENV=development
|
||||
- LOG_LEVEL=INFO
|
||||
- METRICS_ENABLED=true
|
||||
- RUN_MODE=worker
|
||||
# Redis 配置
|
||||
- REDIS_HOST=redis
|
||||
- REDIS_PORT=6379
|
||||
- REDIS_DB=0
|
||||
# Worker 配置
|
||||
- WORKER_POLL_INTERVAL=1.0
|
||||
- MAX_CONCURRENT_JOBS=10
|
||||
- JOB_MAX_RETRIES=3
|
||||
- JOB_EXECUTION_TIMEOUT=300
|
||||
volumes:
|
||||
- ../src:/app/src
|
||||
- ../config:/app/config
|
||||
labels:
|
||||
logging: "promtail"
|
||||
logging_jobname: "functional-scaffold-worker"
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
replicas: 2
|
||||
|
||||
# Redis - 用于集中式指标存储
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
|
||||
@@ -17,7 +17,7 @@ Resources:
|
||||
prime-checker:
|
||||
Type: 'Aliyun::Serverless::Function'
|
||||
Properties:
|
||||
Description: '质数判断算法服务'
|
||||
Description: '质数判断算法服务(API)'
|
||||
Runtime: custom-container
|
||||
MemorySize: 512
|
||||
Timeout: 60
|
||||
@@ -25,11 +25,14 @@ Resources:
|
||||
CAPort: 8000
|
||||
CustomContainerConfig:
|
||||
Image: 'registry.cn-hangzhou.aliyuncs.com/your-namespace/functional-scaffold:latest'
|
||||
Command: '["uvicorn", "functional_scaffold.main:app", "--host", "0.0.0.0", "--port", "8000"]'
|
||||
Command: '["/app/entrypoint.sh"]'
|
||||
EnvironmentVariables:
|
||||
APP_ENV: production
|
||||
LOG_LEVEL: INFO
|
||||
METRICS_ENABLED: 'true'
|
||||
RUN_MODE: api
|
||||
REDIS_HOST: 'r-xxxxx.redis.rds.aliyuncs.com'
|
||||
REDIS_PORT: '6379'
|
||||
Events:
|
||||
httpTrigger:
|
||||
Type: HTTP
|
||||
@@ -38,3 +41,32 @@ Resources:
|
||||
Methods:
|
||||
- GET
|
||||
- POST
|
||||
job-worker:
|
||||
Type: 'Aliyun::Serverless::Function'
|
||||
Properties:
|
||||
Description: '异步任务 Worker'
|
||||
Runtime: custom-container
|
||||
MemorySize: 512
|
||||
Timeout: 900
|
||||
InstanceConcurrency: 1
|
||||
CustomContainerConfig:
|
||||
Image: 'registry.cn-hangzhou.aliyuncs.com/your-namespace/functional-scaffold:latest'
|
||||
Command: '["/app/entrypoint.sh"]'
|
||||
EnvironmentVariables:
|
||||
APP_ENV: production
|
||||
LOG_LEVEL: INFO
|
||||
METRICS_ENABLED: 'true'
|
||||
RUN_MODE: worker
|
||||
REDIS_HOST: 'r-xxxxx.redis.rds.aliyuncs.com'
|
||||
REDIS_PORT: '6379'
|
||||
WORKER_POLL_INTERVAL: '1.0'
|
||||
MAX_CONCURRENT_JOBS: '5'
|
||||
JOB_MAX_RETRIES: '3'
|
||||
JOB_EXECUTION_TIMEOUT: '300'
|
||||
Events:
|
||||
timerTrigger:
|
||||
Type: Timer
|
||||
Properties:
|
||||
CronExpression: '0 */1 * * * *'
|
||||
Enable: true
|
||||
Payload: '{}'
|
||||
|
||||
Reference in New Issue
Block a user