Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Checkpoint production workflow updates

Browse Source
This commit is contained in:
Harmon
2026-06-30 15:25:29 +08:00
parent 19ae3ec37f
commit 9c80b95385
66 changed files with 7639 additions and 267 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/routers/tasks_runtime.py
+264 -1
View File
@@ -2,16 +2,19 @@ from __future__ import annotations
import asyncio
import json
from datetime import datetime, timedelta
from typing import List, Optional
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from fastapi.responses import StreamingResponse
from pydantic import BaseModel
from sqlalchemy import case, func, select
from .. import database
from ..auth_service import SESSION_COOKIE_NAME, get_user_by_session_token
from ..auth_utils import verify_password
from ..models import AuthUserORM, TaskInfo
from ..config import settings
from ..models import AuthUserORM, SystemJobORM, SystemTaskORM, SystemWorkerHeartbeatORM, TaskInfo
from ..services.dinsar_production_service import dinsar_production_service
from ..services.task_service import (
TASK_ACTIVE_DEFAULT_LIMIT,
@@ -47,6 +50,73 @@ def _split_csv_param(raw: Optional[str]) -> List[str]:
return values
def _dt(value):
return value.isoformat() if value else None
def _task_payload(task: SystemTaskORM) -> dict:
return TaskInfo.model_validate(task).model_dump(mode="json")
def _worker_note(worker: SystemWorkerHeartbeatORM) -> dict:
try:
parsed = json.loads(str(worker.note or "") or "{}")
return parsed if isinstance(parsed, dict) else {}
except Exception:
return {}
def _worker_concurrency(worker: SystemWorkerHeartbeatORM) -> int:
note = _worker_note(worker)
try:
return max(1, int(note.get("concurrency") or 1))
except (TypeError, ValueError):
return 1
def _job_payload(job: SystemJobORM, task_by_id: dict[str, SystemTaskORM]) -> dict:
task = task_by_id.get(str(job.task_id or ""))
return {
"job_id": job.job_id,
"job_type": job.job_type,
"status": job.status,
"priority": int(job.priority or 0),
"attempts": int(job.attempts or 0),
"max_attempts": int(job.max_attempts or 0),
"task_id": job.task_id,
"task_type": task.task_type if task else None,
"task_name": task.task_name if task else None,
"task_status": task.status if task else None,
"task_progress": int(task.progress or 0) if task else None,
"task_message": task.message if task else None,
"workflow_run_id": job.workflow_run_id,
"workflow_step_id": job.workflow_step_id,
"locked_by": job.locked_by,
"locked_at": _dt(job.locked_at),
"heartbeat_at": _dt(job.heartbeat_at),
"next_run_at": _dt(job.next_run_at),
"created_at": _dt(job.created_at),
"started_at": _dt(job.started_at),
"finished_at": _dt(job.finished_at),
"last_error": job.last_error,
}
def _worker_payload(worker: SystemWorkerHeartbeatORM, active_job_count: int, concurrency: int) -> dict:
note = _worker_note(worker)
return {
"worker_id": worker.worker_id,
"hostname": worker.hostname,
"pid": worker.pid,
"note": worker.note,
"concurrency": concurrency,
"allowed_job_types": note.get("allowed_job_types") if isinstance(note.get("allowed_job_types"), list) else [],
"started_at": _dt(worker.started_at),
"last_seen": _dt(worker.last_seen),
"active_job_count": active_job_count,
}
@router.get("/tasks/active", response_model=List[TaskInfo])
async def get_active_tasks(limit: int = TASK_ACTIVE_DEFAULT_LIMIT, offset: int = 0):
safe_limit = min(TASK_ACTIVE_MAX_LIMIT, max(1, int(limit or TASK_ACTIVE_DEFAULT_LIMIT)))
@@ -73,6 +143,199 @@ async def get_recent_tasks(
return [TaskInfo.model_validate(task) for task in orm_tasks]
@router.get("/tasks/runtime-summary")
async def get_task_runtime_summary(limit: int = TASK_ACTIVE_DEFAULT_LIMIT, offset: int = 0):
safe_limit = min(TASK_ACTIVE_MAX_LIMIT, max(1, int(limit or TASK_ACTIVE_DEFAULT_LIMIT)))
safe_offset = min(TASK_QUERY_MAX_OFFSET, max(0, int(offset or 0)))
active_job_statuses = ["READY", "RETRY", "RUNNING"]
scan_job_types = {
"SCAN_DATA",
"SCAN_DINSAR",
"SCAN_ASSET_INVENTORY",
"AUDIT_SOURCE_ARCHIVE_INTEGRITY",
"GF3_SARSCAPE_SYNC",
"GF3_QUICKLOOK_WEBP",
}
worker_timeout = max(5, int(getattr(settings, "JOB_WORKER_HEALTH_TIMEOUT", 60) or 60))
worker_threshold = datetime.utcnow() - timedelta(seconds=worker_timeout)
configured_concurrency = max(1, int(getattr(settings, "JOB_WORKER_CONCURRENCY", 1) or 1))
async with _new_session() as db:
active_tasks = await task_service.get_active_tasks(limit=safe_limit, offset=safe_offset, db=db)
workers_result = await db.execute(
select(SystemWorkerHeartbeatORM)
.where(SystemWorkerHeartbeatORM.last_seen >= worker_threshold)
.order_by(SystemWorkerHeartbeatORM.last_seen.desc())
)
active_workers = workers_result.scalars().all()
active_worker_ids = {str(worker.worker_id) for worker in active_workers}
running_by_worker_result = await db.execute(
select(SystemJobORM.locked_by, func.count(SystemJobORM.id))
.where(SystemJobORM.status == "RUNNING")
.group_by(SystemJobORM.locked_by)
)
running_by_worker = {
str(worker_id or ""): int(count or 0)
for worker_id, count in running_by_worker_result.all()
}
status_counts_result = await db.execute(
select(SystemJobORM.status, func.count(SystemJobORM.id))
.where(SystemJobORM.status.in_(active_job_statuses))
.group_by(SystemJobORM.status)
)
job_status_counts = {
"READY": 0,
"RETRY": 0,
"RUNNING": 0,
}
for status, count in status_counts_result.all():
job_status_counts[str(status or "").upper()] = int(count or 0)
status_rank = case(
(SystemJobORM.status == "RUNNING", 0),
(SystemJobORM.status == "RETRY", 1),
else_=2,
)
jobs_result = await db.execute(
select(SystemJobORM)
.where(SystemJobORM.status.in_(active_job_statuses))
.order_by(status_rank, SystemJobORM.priority.desc(), SystemJobORM.id.asc())
.offset(safe_offset)
.limit(safe_limit)
)
active_jobs = jobs_result.scalars().all()
task_ids = {
str(task.task_id)
for task in active_tasks
if task.task_id
}
task_ids.update(
str(job.task_id)
for job in active_jobs
if job.task_id
)
task_by_id: dict[str, SystemTaskORM] = {}
if task_ids:
task_result = await db.execute(
select(SystemTaskORM).where(SystemTaskORM.task_id.in_(sorted(task_ids)))
)
task_by_id = {
str(task.task_id): task
for task in task_result.scalars().all()
if task.task_id
}
worker_concurrency_by_id = {
str(worker.worker_id): _worker_concurrency(worker)
for worker in active_workers
}
total_slots = sum(worker_concurrency_by_id.values())
busy_slots = sum(
count
for worker_id, count in running_by_worker.items()
if worker_id in active_worker_ids
)
running_count = int(job_status_counts.get("RUNNING") or 0)
stale_running_count = max(0, running_count - busy_slots)
queued_count = int(job_status_counts.get("READY") or 0) + int(job_status_counts.get("RETRY") or 0)
task_items = [_task_payload(task) for task in active_tasks]
job_items = [_job_payload(job, task_by_id) for job in active_jobs]
scan_jobs = [
item for item in job_items
if str(item.get("job_type") or "").upper() in scan_job_types
]
scan_tasks = [
item for item in task_items
if str(item.get("task_type") or "").upper() in scan_job_types
]
return {
"timestamp": datetime.utcnow().isoformat() + "Z",
"worker": {
"ok": len(active_workers) > 0,
"worker_count": len(active_workers),
"configured_concurrency": configured_concurrency,
"total_slots": total_slots,
"busy_slots": busy_slots,
"idle_slots": max(0, total_slots - busy_slots),
"timeout_seconds": worker_timeout,
"stale_running_job_count": stale_running_count,
"workers": [
_worker_payload(
worker,
running_by_worker.get(str(worker.worker_id), 0),
worker_concurrency_by_id.get(str(worker.worker_id), 1),
)
for worker in active_workers
],
},
"jobs": {
"active_count": running_count + queued_count,
"running_count": running_count,
"queued_count": queued_count,
"ready_count": int(job_status_counts.get("READY") or 0),
"retry_count": int(job_status_counts.get("RETRY") or 0),
"items": job_items,
},
"tasks": {
"active_count": len(task_items),
"running_count": sum(1 for item in task_items if item.get("status") == "RUNNING"),
"pending_count": sum(1 for item in task_items if item.get("status") == "PENDING"),
"items": task_items,
},
"scan": {
"active_task_count": len(scan_tasks),
"active_job_count": len(scan_jobs),
"running_job_count": sum(1 for item in scan_jobs if item.get("status") == "RUNNING"),
"queued_job_count": sum(1 for item in scan_jobs if item.get("status") in {"READY", "RETRY"}),
"tasks": scan_tasks,
"jobs": scan_jobs,
},
}
@router.get("/tasks/runtime-summary/stream")
async def stream_task_runtime_summary(request: Request):
token = request.cookies.get(SESSION_COOKIE_NAME)
if not token:
raise HTTPException(status_code=401, detail="Authentication required.")
async with _new_session() as db:
user = await get_user_by_session_token(db, token)
if not user:
raise HTTPException(status_code=401, detail="Authentication required.")
async def event_generator():
while True:
if await request.is_disconnected():
break
try:
summary = await get_task_runtime_summary(
limit=TASK_ACTIVE_MAX_LIMIT,
offset=0,
)
yield f"data: {json.dumps(summary)}\n\n"
except Exception:
yield "data: {}\n\n"
await asyncio.sleep(3)
return StreamingResponse(
event_generator(),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
"X-Accel-Buffering": "no",
"Connection": "keep-alive",
},
)
@router.get("/tasks/active/stream")
async def stream_active_tasks(request: Request):
token = request.cookies.get(SESSION_COOKIE_NAME)