Add LandSAR cluster worker deployment

This commit is contained in:
2026-06-24 14:10:58 +08:00
parent 71c524967c
commit da09ba05cb
88 changed files with 9300 additions and 3011 deletions
+15 -2
View File
@@ -3,7 +3,7 @@ import os
import socket
import uuid
import time
from typing import Set
from typing import Optional, Set
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy import func
@@ -26,6 +26,15 @@ IDL_JOB_TYPES = {
}
def _parse_allowed_job_types(raw_value: str) -> Optional[Set[str]]:
values = {
part.strip().upper()
for part in str(raw_value or "").replace(";", ",").split(",")
if part.strip()
}
return values or None
def _default_worker_id() -> str:
host = socket.gethostname()
pid = os.getpid()
@@ -129,6 +138,7 @@ async def run_worker_loop(
concurrency = max(1, int(concurrency))
sem = asyncio.Semaphore(concurrency)
active: Set[asyncio.Task] = set()
allowed_job_types = _parse_allowed_job_types(getattr(settings, "JOB_WORKER_ALLOWED_TYPES", ""))
job_heartbeat_interval = float(settings.JOB_WORKER_JOB_HEARTBEAT_INTERVAL)
stale_recover_interval = float(settings.JOB_WORKER_STALE_RECOVER_INTERVAL)
stale_running_seconds = int(settings.JOB_WORKER_STALE_RUNNING_SECONDS)
@@ -187,7 +197,10 @@ async def run_worker_loop(
print(f"[WARN] worker poll: {exc}")
if len(active) < concurrency:
job = await job_queue_service.claim_next_job(worker_id)
job = await job_queue_service.claim_next_job(
worker_id,
allowed_job_types=allowed_job_types,
)
if job:
task = asyncio.create_task(_wrap(job))
active.add(task)