Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Checkpoint production workflow updates

Browse Source
This commit is contained in:
Harmon
2026-06-30 15:25:29 +08:00
parent 19ae3ec37f
commit 9c80b95385
66 changed files with 7639 additions and 267 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/routers/__init__.py
+2
View File
@@ -14,6 +14,7 @@ from . import (
hazard,
health,
idl,
landsar_lt1_production,
license,
logs,
monitor,
@@ -55,6 +56,7 @@ def include_all_routers(router: APIRouter) -> None:
router.include_router(dinsar.router)
router.include_router(dinsar_products.router)
router.include_router(dinsar_production.router)
router.include_router(landsar_lt1_production.router)
router.include_router(sbas_insar_production.router)
router.include_router(sbas_insar_products.router)
router.include_router(timeseries_production.router)
backend/app/routers/idl.py
+23 -79
View File
@@ -16,7 +16,6 @@ from .dependencies import _require_admin, _get_current_user, _validate_export_pa
from ..models import AuthUserORM
from ..services import envi_service
from ..services.job_queue_service import job_queue_service
from ..services.result_catalog_service import result_catalog_service
from ..services.task_service import task_service
router = APIRouter()
@@ -61,38 +60,6 @@ class SarscapeSbasInspectRequest(BaseModel):
timeout_seconds: Optional[int] = Field(default=120, ge=10, le=600)
def _normalize_existing_dir(path: Optional[str]) -> Optional[str]:
text = str(path or "").strip()
if not text:
return None
normalized = os.path.normpath(os.path.abspath(text))
if not os.path.isdir(normalized):
return None
return normalized
def _dedupe_publish_roots(*paths: Optional[str]) -> list[str]:
ordered: list[str] = []
for raw_path in paths:
normalized = _normalize_existing_dir(raw_path)
if not normalized:
continue
if any(
normalized == existing or normalized.startswith(existing + os.sep)
for existing in ordered
):
continue
ordered = [
existing
for existing in ordered
if not existing.startswith(normalized + os.sep)
]
ordered.append(normalized)
return ordered
# ---------------------------------------------------------------------------
# Job queue helper
# ---------------------------------------------------------------------------
@@ -292,7 +259,7 @@ async def get_task_overview_endpoint(
return result
@router.post("/idl/extract-disp")
@router.post("/idl/extract-disp", status_code=202)
async def extract_disp_endpoint(
request: ExtractDispRequest,
admin_user: AuthUserORM = Depends(_require_admin),
@@ -303,52 +270,29 @@ async def extract_disp_endpoint(
if request.dest_dir:
_validate_export_path(request.dest_dir, "dest_dir")
try:
result = await asyncio.to_thread(
envi_service.extract_disp_results, request.root_dir, request.dest_dir
payload = {
"root_dir": request.root_dir,
"dest_dir": request.dest_dir,
}
task_id = await task_service.create_task(
"EXTRACT_DINSAR_PRODUCTS",
"D-InSAR 结果提取与登记",
params=payload,
db=db,
)
job_id = await job_queue_service.create_job(
"EXTRACT_DINSAR_PRODUCTS",
payload=payload,
task_id=task_id,
db=db,
)
await db.commit()
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
raise HTTPException(status_code=409, detail=str(exc)) from exc
publish_roots = _dedupe_publish_roots(result.get("target_dir"))
catalog_status: Dict[str, Any] = {
"attempted": False,
"status": "skipped",
"source_directories": publish_roots,
"message": "catalog publish skipped",
return {
"queued": True,
"task_id": task_id,
"job_id": job_id,
"message": "D-InSAR 结果提取与登记任务已入队",
}
if publish_roots:
try:
catalog_status["attempted"] = True
publish_result = await result_catalog_service.publish_from_sources(
db,
publish_roots,
)
rebuild_result = None
if int(publish_result.get("processed", 0) or 0) > 0:
rebuild_result = await result_catalog_service.rebuild_catalog(
db,
full_rebuild=True,
)
catalog_status = {
"attempted": True,
"status": "ok",
"source_directories": publish_roots,
"publish": publish_result,
"rebuild": rebuild_result,
"message": (
"catalog published and rebuilt"
if rebuild_result is not None
else "catalog publish finished with no rebuild needed"
),
}
except Exception as exc:
await db.rollback()
catalog_status = {
"attempted": True,
"status": "error",
"source_directories": publish_roots,
"message": str(exc),
}
result["catalog"] = catalog_status
return result
backend/app/routers/landsar_lt1_production.py
+485
View File
@@ -0,0 +1,485 @@
from __future__ import annotations
import os
from typing import Any, Dict, List, Optional
from fastapi import APIRouter, Depends, HTTPException, Request
from fastapi.responses import FileResponse
from pydantic import BaseModel, Field, field_validator
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from ..database import get_db
from ..models import AuthUserORM, RadarDataORM, SARSceneGeoORM
from ..services.job_handlers import JOB_TYPE_SAR_SCENE_PREPROCESS
from ..services.job_queue_service import job_queue_service
from ..services.landsar_lt1_production_service import landsar_lt1_production_service
from ..services.task_service import task_service
from ..utils import normalize_satellite_family
from .dependencies import _add_operation_audit_log, _get_current_user, _require_admin
router = APIRouter()
STATIC_ASSET_CACHE_HEADERS = {"Cache-Control": "public, max-age=31536000, immutable"}
class LandsarLt1ImageProductionRequest(BaseModel):
source_asset_ids: List[int] = Field(default_factory=list)
radar_data_ids: List[int] = Field(default_factory=list)
mode: str = "scene"
task_name: Optional[str] = None
@field_validator("mode")
@classmethod
def _validate_mode(cls, value):
mode = str(value or "scene").strip().lower()
if mode == "stack":
mode = "batch"
if mode not in {"scene", "batch"}:
raise ValueError("mode must be scene or batch")
return mode
def _dedupe_positive_ids(values: List[int]) -> List[int]:
result: List[int] = []
for value in values or []:
try:
parsed = int(value)
except (TypeError, ValueError):
continue
if parsed > 0 and parsed not in result:
result.append(parsed)
return result
def _scene_product_marker(scene: SARSceneGeoORM) -> Dict[str, Any]:
return {
"scene_id": scene.id,
"radar_data_id": scene.radar_data_id,
"product_id": f"sar_scene_geo:{scene.id}",
"product_family": "lt1_analysis_ready_geotiff",
"engine_code": scene.analysis_engine,
"profile_code": scene.analysis_profile,
"analysis_tif_path": scene.analysis_tif_path,
"analysis_dir": scene.analysis_dir,
"analysis_preview_path": scene.analysis_preview_path,
"status": scene.status,
"published_at": scene.updated_at.isoformat() if scene.updated_at else None,
}
def _scene_asset_items(scene: SARSceneGeoORM) -> List[Dict[str, Any]]:
candidates = [
(1, "analysis_tif", "analysis_ready.tif", scene.analysis_tif_path, "image/tiff", True),
(2, "preview", "preview.png", scene.analysis_preview_path, "image/png", False),
]
metadata = scene.analysis_metadata_json if isinstance(scene.analysis_metadata_json, dict) else {}
manifest_path = str(metadata.get("manifest_path") or "").strip()
if manifest_path:
candidates.append((3, "manifest", "manifest.json", manifest_path, "application/json", False))
if scene.analysis_dir:
quality_path = os.path.join(scene.analysis_dir, "quality.json")
candidates.append((4, "quality", "quality.json", quality_path, "application/json", False))
assets: List[Dict[str, Any]] = []
for asset_id, role, name, path, media_type, primary in candidates:
if not path:
continue
assets.append(
{
"id": asset_id,
"role": role,
"name": name,
"relative_path": os.path.basename(path),
"absolute_path": path,
"format": os.path.splitext(path)[1].lower().lstrip(".") or None,
"media_type": media_type,
"is_required": primary,
"is_primary": primary,
"exists": os.path.isfile(path),
"file_size": os.path.getsize(path) if os.path.isfile(path) else None,
}
)
return assets
async def _resolve_lt1_radars_for_request(
db: AsyncSession,
request: LandsarLt1ImageProductionRequest,
) -> List[RadarDataORM]:
source_asset_ids = _dedupe_positive_ids(request.source_asset_ids)
radar_data_ids = _dedupe_positive_ids(request.radar_data_ids)
filters = []
if radar_data_ids:
filters.append(RadarDataORM.id.in_(radar_data_ids))
if source_asset_ids:
filters.append(RadarDataORM.source_product_ref_id.in_(source_asset_ids))
if not filters:
return []
result = await db.execute(select(RadarDataORM).where(*([filters[0]] if len(filters) == 1 else [filters[0] | filters[1]])))
radars = list(result.scalars().all())
unique: Dict[int, RadarDataORM] = {}
for radar in radars:
if not radar.id:
continue
family = normalize_satellite_family(radar.satellite_family or radar.satellite)
if str(family or "").upper() != "LT1":
continue
unique[int(radar.id)] = radar
return [unique[key] for key in sorted(unique.keys())]
async def _produced_radars_for_request(
db: AsyncSession,
request: LandsarLt1ImageProductionRequest,
) -> Dict[int, dict]:
radars = await _resolve_lt1_radars_for_request(db, request)
radar_ids = [int(item.id) for item in radars if item.id]
if not radar_ids:
return {}
result = await db.execute(
select(SARSceneGeoORM).where(
SARSceneGeoORM.radar_data_id.in_(radar_ids),
SARSceneGeoORM.status == "DONE",
SARSceneGeoORM.analysis_tif_path.isnot(None),
SARSceneGeoORM.analysis_engine == "lt_gamma",
SARSceneGeoORM.analysis_profile == "lt1_gamma_geocoded_mli",
)
)
return {int(scene.radar_data_id): _scene_product_marker(scene) for scene in result.scalars().all()}
async def _active_radars_for_request(
db: AsyncSession,
request: LandsarLt1ImageProductionRequest,
) -> Dict[int, dict]:
radars = await _resolve_lt1_radars_for_request(db, request)
radar_ids = [int(item.id) for item in radars if item.id]
if not radar_ids:
return {}
result = await db.execute(
select(SARSceneGeoORM).where(
SARSceneGeoORM.radar_data_id.in_(radar_ids),
SARSceneGeoORM.status.in_(["PENDING", "RUNNING"]),
)
)
return {int(scene.radar_data_id): _scene_product_marker(scene) for scene in result.scalars().all()}
def _already_produced_blocker(produced: Dict[int, dict]) -> str:
first_id = sorted(produced.keys())[0]
marker = produced[first_id] or {}
product_id = marker.get("product_id") or "unknown"
return f"Radar data {first_id} already has an analysis-ready GeoTIFF: {product_id}"
def _active_blocker(active: Dict[int, dict]) -> str:
first_id = sorted(active.keys())[0]
marker = active[first_id] or {}
return f"Radar data {first_id} already has an active GeoTIFF production task (scene_id={marker.get('scene_id')})."
@router.get("/landsar-lt1-production/capabilities")
async def get_landsar_lt1_capabilities(
current_user: AuthUserORM = Depends(_get_current_user),
):
_ = current_user
legacy = landsar_lt1_production_service.check_capabilities()
return {
"catalog_name": "sar_scene_geo",
"supported_profiles": ["lt1_gamma_geocoded_mli"],
"engine": "lt_gamma",
"available": True,
"status": "configured",
"message": "LT-1 image production uses the existing Gamma single-scene pipeline: multilook, geocode, and analysis-ready GeoTIFF registration.",
"legacy_landsar_import": legacy,
}
@router.post("/landsar-lt1-production/preview")
async def preview_landsar_lt1_production(
request: LandsarLt1ImageProductionRequest,
current_user: AuthUserORM = Depends(_get_current_user),
db: AsyncSession = Depends(get_db),
):
_ = current_user
blockers: List[str] = []
warnings: List[str] = []
radars = await _resolve_lt1_radars_for_request(db, request)
if not radars:
blockers.append("No LT-1 radar records were resolved from the selected source assets.")
if request.mode == "scene" and len(radars) != 1:
blockers.append("Scene mode requires exactly one LT-1 source asset.")
if request.mode == "batch" and len(radars) < 1:
blockers.append("Batch mode requires at least one LT-1 source asset.")
produced = await _produced_radars_for_request(db, request)
if produced:
blockers.append(_already_produced_blocker(produced))
active = await _active_radars_for_request(db, request)
if active:
blockers.append(_active_blocker(active))
if request.mode == "batch":
warnings.append("Batch mode submits one independent geocoded GeoTIFF task per scene; it does not build a D-InSAR stack.")
preview = {
"allow_submit": not blockers,
"blockers": blockers,
"warnings": warnings,
"mode": request.mode,
"profile_code": "lt1_gamma_geocoded_mli",
"engine": "lt_gamma",
"scene_count": len(radars),
"source_asset_count": len(_dedupe_positive_ids(request.source_asset_ids)),
"radar_data_count": len(radars),
"produced_radars": produced,
"active_radars": active,
"scenes": [
{
"radar_data_id": radar.id,
"source_asset_id": radar.source_product_ref_id,
"satellite": radar.satellite,
"imaging_date": radar.imaging_date,
"imaging_mode": radar.imaging_mode,
"polarization": radar.polarization,
"file_path": radar.file_path,
}
for radar in radars
],
}
return preview
@router.post("/landsar-lt1-production/run", status_code=202)
async def queue_landsar_lt1_production(
request: LandsarLt1ImageProductionRequest,
http_request: Request,
db: AsyncSession = Depends(get_db),
admin_user: AuthUserORM = Depends(_require_admin),
):
_ = admin_user
preview = await preview_landsar_lt1_production(request, current_user=admin_user, db=db)
if preview.get("blockers"):
raise HTTPException(status_code=400, detail={"blockers": preview.get("blockers")})
queued: List[Dict[str, Any]] = []
radars = await _resolve_lt1_radars_for_request(db, request)
for radar in radars:
result = await db.execute(
select(SARSceneGeoORM)
.where(SARSceneGeoORM.radar_data_id == int(radar.id))
.with_for_update(skip_locked=True)
)
scene = result.scalar_one_or_none()
if scene and scene.status in ("PENDING", "RUNNING"):
raise HTTPException(status_code=409, detail=f"Radar data {radar.id} already has an active GeoTIFF production task.")
if scene and scene.status == "DONE" and scene.analysis_tif_path:
raise HTTPException(status_code=409, detail=f"Radar data {radar.id} already has an analysis-ready GeoTIFF.")
if not scene:
scene = SARSceneGeoORM(radar_data_id=int(radar.id), status="PENDING")
db.add(scene)
await db.flush()
else:
scene.status = "PENDING"
scene.error_msg = None
await db.flush()
scene_id = int(scene.id)
await db.commit()
payload = {
"scene_id": scene_id,
"radar_data_id": int(radar.id),
"engine": "lt_gamma",
"source_asset_id": radar.source_product_ref_id,
"requested_from": "landsar_lt1_production",
}
task_label = request.task_name or radar.product_unique_id or radar.unique_id or f"radar_id={radar.id}"
task_type = f"LT1_SCENE_GEOTIFF_{scene_id}"
try:
task_id = await task_service.create_task(
task_type,
f"LT-1 geocoded GeoTIFF: {task_label}",
params=payload,
)
job_id = await job_queue_service.create_job(
JOB_TYPE_SAR_SCENE_PREPROCESS,
payload=payload,
task_id=task_id,
max_attempts=3,
)
except Exception as exc:
failed_scene = await db.get(SARSceneGeoORM, scene_id)
if failed_scene and failed_scene.status == "PENDING":
failed_scene.status = "FAILED"
failed_scene.error_msg = "Job queue failed"
await db.commit()
raise HTTPException(status_code=409 if "conflict" in str(exc).lower() else 400, detail=str(exc)) from exc
queued.append(
{
"task_id": task_id,
"job_id": job_id,
"scene_id": scene_id,
"radar_data_id": int(radar.id),
"source_asset_id": radar.source_product_ref_id,
}
)
await _add_operation_audit_log(
db,
request=http_request,
action="lt1_geotiff_production_queued",
resource="landsar-lt1-production/run",
detail={
"queued": queued,
"mode": request.mode,
"scene_count": len(queued),
},
)
await db.commit()
return {
"message": "LT-1 geocoded GeoTIFF production job queued.",
"task_id": queued[0]["task_id"] if len(queued) == 1 else None,
"job_id": queued[0]["job_id"] if len(queued) == 1 else None,
"queued": queued,
"preview": preview,
}
@router.get("/landsar-lt1-production/products")
async def list_landsar_lt1_products(
limit: int = 100,
offset: int = 0,
status: Optional[str] = None,
query: Optional[str] = None,
current_user: AuthUserORM = Depends(_get_current_user),
db: AsyncSession = Depends(get_db),
):
_ = current_user
safe_limit = max(1, min(500, int(limit or 100)))
safe_offset = max(0, int(offset or 0))
filters = [
SARSceneGeoORM.analysis_engine == "lt_gamma",
SARSceneGeoORM.analysis_profile == "lt1_gamma_geocoded_mli",
]
if status:
filters.append(SARSceneGeoORM.status == str(status).strip().upper())
if query:
like = f"%{str(query).strip()}%"
filters.append(RadarDataORM.product_unique_id.ilike(like) | RadarDataORM.unique_id.ilike(like) | RadarDataORM.file_path.ilike(like))
total_result = await db.execute(
select(func.count(SARSceneGeoORM.id))
.join(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id)
.where(*filters)
)
total = int(total_result.scalar_one() or 0)
result = await db.execute(
select(SARSceneGeoORM, RadarDataORM)
.join(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id)
.where(*filters)
.order_by(SARSceneGeoORM.updated_at.desc().nullslast(), SARSceneGeoORM.id.desc())
.limit(safe_limit)
.offset(safe_offset)
)
items = []
for scene, radar in result.all():
marker = _scene_product_marker(scene)
items.append(
{
"id": scene.id,
"product_id": marker["product_id"],
"catalog_name": "sar_scene_geo",
"product_family": "lt1_analysis_ready_geotiff",
"product_type": "analysis_ready_geotiff",
"display_name": radar.product_unique_id or radar.unique_id or f"radar_id={radar.id}",
"task_name": "",
"profile_code": scene.analysis_profile,
"engine_code": scene.analysis_engine,
"status": scene.status,
"health_status": "OK" if scene.status == "DONE" and scene.analysis_tif_path else "PENDING",
"publish_dir": scene.analysis_dir,
"manifest_path": (scene.analysis_metadata_json or {}).get("manifest_path") if isinstance(scene.analysis_metadata_json, dict) else None,
"native_output_dir": scene.analysis_dir,
"primary_asset_path": scene.analysis_tif_path,
"summary": {
"scene_count": 1,
"radar_data_id": radar.id,
"source_asset_ids": [radar.source_product_ref_id] if radar.source_product_ref_id else [],
"imaging_date": radar.imaging_date,
"polarization": radar.polarization,
"pixel_size_m": scene.pixel_size_m,
"backscatter_unit": scene.analysis_backscatter_unit,
},
"tags": {"engine": scene.analysis_engine, "profile": scene.analysis_profile},
"produced_at": scene.updated_at.isoformat() if scene.updated_at else None,
"published_at": scene.updated_at.isoformat() if scene.updated_at else None,
"registered_at": scene.created_at.isoformat() if scene.created_at else None,
}
)
return {"total": total, "limit": safe_limit, "offset": safe_offset, "items": items}
@router.get("/landsar-lt1-production/products/{product_db_id}")
async def get_landsar_lt1_product_detail(
product_db_id: int,
current_user: AuthUserORM = Depends(_get_current_user),
db: AsyncSession = Depends(get_db),
):
_ = current_user
result = await db.execute(
select(SARSceneGeoORM, RadarDataORM)
.join(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id)
.where(
SARSceneGeoORM.id == product_db_id,
SARSceneGeoORM.analysis_engine == "lt_gamma",
SARSceneGeoORM.analysis_profile == "lt1_gamma_geocoded_mli",
)
)
row = result.first()
if row is None:
raise HTTPException(status_code=404, detail="LT-1 geocoded GeoTIFF product not found")
scene, radar = row
marker = _scene_product_marker(scene)
detail = {
"id": scene.id,
"product_id": marker["product_id"],
"catalog_name": "sar_scene_geo",
"product_family": "lt1_analysis_ready_geotiff",
"product_type": "analysis_ready_geotiff",
"display_name": radar.product_unique_id or radar.unique_id or f"radar_id={radar.id}",
"profile_code": scene.analysis_profile,
"engine_code": scene.analysis_engine,
"status": scene.status,
"publish_dir": scene.analysis_dir,
"primary_asset_path": scene.analysis_tif_path,
"summary": {
"scene_count": 1,
"radar_data_id": radar.id,
"source_asset_ids": [radar.source_product_ref_id] if radar.source_product_ref_id else [],
"imaging_date": radar.imaging_date,
"polarization": radar.polarization,
"pixel_size_m": scene.pixel_size_m,
"backscatter_unit": scene.analysis_backscatter_unit,
},
"assets": _scene_asset_items(scene),
}
return detail
@router.get("/landsar-lt1-production/products/{product_db_id}/assets/{asset_id}")
async def get_landsar_lt1_product_asset(
product_db_id: int,
asset_id: int,
current_user: AuthUserORM = Depends(_get_current_user),
db: AsyncSession = Depends(get_db),
):
_ = current_user
scene = await db.get(SARSceneGeoORM, product_db_id)
if scene is None or scene.analysis_engine != "lt_gamma" or scene.analysis_profile != "lt1_gamma_geocoded_mli":
raise HTTPException(status_code=404, detail="LT-1 geocoded GeoTIFF product not found")
asset = next((item for item in _scene_asset_items(scene) if int(item["id"]) == int(asset_id)), None)
if asset is None:
raise HTTPException(status_code=404, detail="LT-1 geocoded GeoTIFF asset not found")
absolute_path = str(asset.get("absolute_path") or "")
if not absolute_path or not os.path.isfile(absolute_path):
raise HTTPException(status_code=404, detail="Asset file not found")
return FileResponse(
absolute_path,
media_type=str(asset.get("media_type") or "application/octet-stream"),
filename=str(asset.get("name") or os.path.basename(absolute_path)),
headers=STATIC_ASSET_CACHE_HEADERS,
)
backend/app/routers/pairing.py
+52 -6
View File
@@ -26,7 +26,10 @@ from ..models import (
TimeseriesStackPlanORM,
)
from ..services.pairing_cache_service import pairing_cache_service
from ..services.job_handlers import JOB_TYPE_PAIRING_CACHE_REBUILD
from ..services.job_queue_service import job_queue_service
from ..services.spatial_service import spatial_service
from ..services.task_service import task_service
from .dependencies import (
_parse_aoi_from_files,
_parse_aoi_geojson_form_value,
@@ -38,6 +41,49 @@ logger = logging.getLogger(__name__)
router = APIRouter()
async def _queue_pairing_cache_job(
*,
db: AsyncSession,
mode: str,
force_full: bool = False,
) -> Dict[str, object]:
full_rebuild = mode in {"full", "full_rebuild"} or force_full
task_name = (
"D-InSAR pairing cache full rebuild"
if full_rebuild
else "D-InSAR pairing cache dirty reconcile"
)
payload = {
"mode": "full_rebuild" if full_rebuild else "auto_reconcile",
"force_full": bool(full_rebuild),
}
try:
task_id = await task_service.create_task(
JOB_TYPE_PAIRING_CACHE_REBUILD,
task_name,
params=payload,
db=db,
)
job_id = await job_queue_service.create_job(
JOB_TYPE_PAIRING_CACHE_REBUILD,
payload=payload,
max_attempts=1,
task_id=task_id,
db=db,
)
await db.commit()
except ValueError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
return {
"ok": True,
"queued": True,
"mode": payload["mode"],
"task_id": task_id,
"job_id": job_id,
"message": f"{task_name} queued",
}
def get_pairing_request_from_form(
time_baseline_min: int = Form(1),
time_baseline_max: int = Form(30),
@@ -130,26 +176,26 @@ async def get_pairing_health_endpoint(
return await pairing_cache_service.get_admin_summary(db)
@router.post("/pairing/rebuild-cache")
@router.post("/pairing/rebuild-cache", status_code=202)
async def rebuild_pairing_cache_endpoint(
db: AsyncSession = Depends(get_db),
current_user: AuthUserORM = Depends(_require_admin),
):
_ = current_user
return await pairing_cache_service.rebuild_metric_cache(db, commit=True)
return await _queue_pairing_cache_job(db=db, mode="full_rebuild", force_full=True)
@router.post("/pairing/reconcile-dirty")
@router.post("/pairing/reconcile-dirty", status_code=202)
async def reconcile_dirty_pairing_endpoint(
force_full: bool = Query(False),
db: AsyncSession = Depends(get_db),
current_user: AuthUserORM = Depends(_require_admin),
):
_ = current_user
return await pairing_cache_service.reconcile_dirty_scenes(
db,
return await _queue_pairing_cache_job(
db=db,
mode="full_rebuild" if force_full else "auto_reconcile",
force_full=force_full,
commit=True,
)
backend/app/routers/radar.py
+84 -14
View File
@@ -23,6 +23,7 @@ from ..models import (
RadarDataORM,
RadarDataPage,
RadarPreviewStatusInfo,
SARSceneGeoORM,
ScanRequest,
)
from ..services.data_service import data_service
@@ -154,6 +155,51 @@ def _normalize_list_pagination(limit: int, offset: int) -> Tuple[int, int]:
return safe_limit, safe_offset
def _lt1_image_marker(scene: SARSceneGeoORM) -> Dict[str, Any]:
return {
"scene_id": scene.id,
"radar_data_id": scene.radar_data_id,
"product_id": f"sar_scene_geo:{scene.id}",
"product_family": "lt1_analysis_ready_geotiff",
"engine_code": scene.analysis_engine,
"profile_code": scene.analysis_profile,
"analysis_tif_path": scene.analysis_tif_path,
"analysis_dir": scene.analysis_dir,
"analysis_preview_path": scene.analysis_preview_path,
"status": scene.status,
"published_at": scene.updated_at.isoformat() if scene.updated_at else None,
}
async def _decorate_lt1_landsar_status(db: AsyncSession, items: List[RadarDataORM]) -> List[RadarData]:
payloads = [RadarData.model_validate(item) for item in items]
radar_ids = [
int(item.id)
for item in items
if item.id and str(item.satellite_family or item.satellite or "").upper().replace("-", "") in {"LT1", "LT"}
]
if not radar_ids:
return payloads
result = await db.execute(
select(SARSceneGeoORM).where(
SARSceneGeoORM.radar_data_id.in_(radar_ids),
SARSceneGeoORM.status == "DONE",
SARSceneGeoORM.analysis_tif_path.isnot(None),
SARSceneGeoORM.analysis_engine == "lt_gamma",
SARSceneGeoORM.analysis_profile == "lt1_gamma_geocoded_mli",
)
)
produced = {int(scene.radar_data_id): _lt1_image_marker(scene) for scene in result.scalars().all()}
for payload in payloads:
marker = produced.get(int(payload.id or 0))
if marker:
payload.lt1_image_produced = True
payload.lt1_image_product = marker
payload.lt1_landsar_produced = True
payload.lt1_landsar_product = marker
return payloads
def _normalize_optional_text(value: Optional[str]) -> Optional[str]:
if value is None:
return None
@@ -256,6 +302,24 @@ def _build_radar_preview_status(
)
def _build_cached_radar_preview_status(record: RadarDataORM) -> RadarPreviewStatusInfo:
raw_cache_path, geo_cache_path = _radar_preview_paths(record)
preview_cache_path = str(record.preview_cache_path or "")
has_geo_cache = (
os.path.exists(geo_cache_path)
or bool(preview_cache_path and os.path.exists(preview_cache_path))
)
has_raw_cache = os.path.exists(raw_cache_path)
cached_status = str(record.preview_cache_status or "NONE").upper()
source_found = cached_status in {"READY", "FAILED"} or has_geo_cache or has_raw_cache
return _build_radar_preview_status(
record=record,
source_found=source_found,
has_geo_cache=has_geo_cache,
has_raw_cache=has_raw_cache,
)
async def _build_radar_preview_cache(
record: RadarDataORM,
db: AsyncSession,
@@ -510,10 +574,13 @@ async def search_radar_data_endpoint(
limit: int = Form(500),
offset: int = Form(0),
satellite: Optional[str] = Form(None),
satellite_family: Optional[str] = Form(None),
source_format: Optional[str] = Form(None),
satellite_mode: Optional[str] = Form(None),
receiving_station: Optional[str] = Form(None),
imaging_mode: Optional[str] = Form(None),
orbit_circle: Optional[str] = Form(None),
relative_orbit: Optional[str] = Form(None),
acquisition_time_utc: Optional[str] = Form(None),
product_type: Optional[str] = Form(None),
polarization: Optional[str] = Form(None),
@@ -536,10 +603,13 @@ async def search_radar_data_endpoint(
n_satellite_list: Optional[List[str]] = None
if n_satellite_raw and "," in n_satellite_raw:
n_satellite_list = [s.strip() for s in n_satellite_raw.split(",") if s.strip()]
n_satellite_family = _normalize_optional_text(satellite_family)
n_source_format = _normalize_optional_text(source_format)
n_satellite_mode = _normalize_optional_text(satellite_mode)
n_receiving_station = _normalize_optional_text(receiving_station)
n_imaging_mode = _normalize_optional_text(imaging_mode)
n_orbit_circle = _normalize_optional_text(orbit_circle)
n_relative_orbit = _normalize_optional_text(relative_orbit)
n_acquisition_time = _normalize_optional_text(acquisition_time_utc)
n_product_type = _normalize_optional_text(product_type)
n_polarization = _normalize_optional_text(polarization)
@@ -584,6 +654,10 @@ async def search_radar_data_endpoint(
filters.append(RadarDataORM.satellite.in_(n_satellite_list))
elif n_satellite_raw:
filters.append(RadarDataORM.satellite.ilike(f"%{n_satellite_raw}%"))
if n_satellite_family:
filters.append(func.upper(RadarDataORM.satellite_family) == n_satellite_family.upper())
if n_source_format:
filters.append(func.upper(RadarDataORM.source_format) == n_source_format.upper())
if n_satellite_mode:
filters.append(RadarDataORM.satellite_mode.ilike(f"%{n_satellite_mode}%"))
if n_receiving_station:
@@ -592,6 +666,8 @@ async def search_radar_data_endpoint(
filters.append(RadarDataORM.imaging_mode.ilike(f"%{n_imaging_mode}%"))
if n_orbit_circle:
filters.append(RadarDataORM.orbit_circle.ilike(f"%{n_orbit_circle}%"))
if n_relative_orbit:
filters.append(RadarDataORM.relative_orbit.ilike(f"%{n_relative_orbit}%"))
if n_acquisition_time:
filters.append(RadarDataORM.acquisition_time_utc.ilike(f"%{n_acquisition_time}%"))
if n_product_type:
@@ -606,10 +682,11 @@ async def search_radar_data_endpoint(
filters.append(RadarDataORM.orbit_direction.ilike(f"%{n_orbit_direction}%"))
if has_orbit_data is not None:
filters.append(RadarDataORM.has_orbit_data == has_orbit_data)
normalized_imaging_date = func.replace(RadarDataORM.imaging_date, "-", "")
if n_date_from:
filters.append(RadarDataORM.imaging_date >= n_date_from)
filters.append(normalized_imaging_date >= n_date_from.replace("-", ""))
if n_date_to:
filters.append(RadarDataORM.imaging_date <= n_date_to)
filters.append(normalized_imaging_date <= n_date_to.replace("-", ""))
if resolved_aoi_wkt:
aoi_geom = func.ST_GeomFromText(resolved_aoi_wkt, 4326)
filters.append(ST_Intersects(RadarDataORM.geom, aoi_geom))
@@ -630,8 +707,9 @@ async def search_radar_data_endpoint(
result = await db.execute(data_stmt)
items = result.scalars().all()
decorated_items = await _decorate_lt1_landsar_status(db, items)
return RadarDataSearchPageResponse(
items=items,
items=decorated_items,
total=total,
limit=limit,
offset=offset,
@@ -663,8 +741,9 @@ async def get_all_data_endpoint(
.limit(limit)
)
items = result.scalars().all()
decorated_items = await _decorate_lt1_landsar_status(db, items)
return RadarDataPage(
items=items,
items=decorated_items,
total=total,
limit=limit,
offset=offset,
@@ -724,16 +803,7 @@ async def get_radar_preview_status_endpoint(data_id: int, db: AsyncSession = Dep
if _is_gf3_native_preview_record(record):
return _build_gf3_native_preview_status(record)
raw_cache_path, geo_cache_path = _radar_preview_paths(record)
has_geo_cache = os.path.exists(geo_cache_path)
has_raw_cache = os.path.exists(raw_cache_path)
source_found = bool(await asyncio.to_thread(data_service.find_radar_preview_source, record.file_path))
return _build_radar_preview_status(
record=record,
source_found=source_found,
has_geo_cache=has_geo_cache,
has_raw_cache=has_raw_cache,
)
return _build_cached_radar_preview_status(record)
@router.post("/radar-data/{data_id}/rebuild-preview-cache", response_model=RadarPreviewStatusInfo)
backend/app/routers/tasks_runtime.py
+264 -1
View File
@@ -2,16 +2,19 @@ from __future__ import annotations
import asyncio
import json
from datetime import datetime, timedelta
from typing import List, Optional
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from fastapi.responses import StreamingResponse
from pydantic import BaseModel
from sqlalchemy import case, func, select
from .. import database
from ..auth_service import SESSION_COOKIE_NAME, get_user_by_session_token
from ..auth_utils import verify_password
from ..models import AuthUserORM, TaskInfo
from ..config import settings
from ..models import AuthUserORM, SystemJobORM, SystemTaskORM, SystemWorkerHeartbeatORM, TaskInfo
from ..services.dinsar_production_service import dinsar_production_service
from ..services.task_service import (
TASK_ACTIVE_DEFAULT_LIMIT,
@@ -47,6 +50,73 @@ def _split_csv_param(raw: Optional[str]) -> List[str]:
return values
def _dt(value):
return value.isoformat() if value else None
def _task_payload(task: SystemTaskORM) -> dict:
return TaskInfo.model_validate(task).model_dump(mode="json")
def _worker_note(worker: SystemWorkerHeartbeatORM) -> dict:
try:
parsed = json.loads(str(worker.note or "") or "{}")
return parsed if isinstance(parsed, dict) else {}
except Exception:
return {}
def _worker_concurrency(worker: SystemWorkerHeartbeatORM) -> int:
note = _worker_note(worker)
try:
return max(1, int(note.get("concurrency") or 1))
except (TypeError, ValueError):
return 1
def _job_payload(job: SystemJobORM, task_by_id: dict[str, SystemTaskORM]) -> dict:
task = task_by_id.get(str(job.task_id or ""))
return {
"job_id": job.job_id,
"job_type": job.job_type,
"status": job.status,
"priority": int(job.priority or 0),
"attempts": int(job.attempts or 0),
"max_attempts": int(job.max_attempts or 0),
"task_id": job.task_id,
"task_type": task.task_type if task else None,
"task_name": task.task_name if task else None,
"task_status": task.status if task else None,
"task_progress": int(task.progress or 0) if task else None,
"task_message": task.message if task else None,
"workflow_run_id": job.workflow_run_id,
"workflow_step_id": job.workflow_step_id,
"locked_by": job.locked_by,
"locked_at": _dt(job.locked_at),
"heartbeat_at": _dt(job.heartbeat_at),
"next_run_at": _dt(job.next_run_at),
"created_at": _dt(job.created_at),
"started_at": _dt(job.started_at),
"finished_at": _dt(job.finished_at),
"last_error": job.last_error,
}
def _worker_payload(worker: SystemWorkerHeartbeatORM, active_job_count: int, concurrency: int) -> dict:
note = _worker_note(worker)
return {
"worker_id": worker.worker_id,
"hostname": worker.hostname,
"pid": worker.pid,
"note": worker.note,
"concurrency": concurrency,
"allowed_job_types": note.get("allowed_job_types") if isinstance(note.get("allowed_job_types"), list) else [],
"started_at": _dt(worker.started_at),
"last_seen": _dt(worker.last_seen),
"active_job_count": active_job_count,
}
@router.get("/tasks/active", response_model=List[TaskInfo])
async def get_active_tasks(limit: int = TASK_ACTIVE_DEFAULT_LIMIT, offset: int = 0):
safe_limit = min(TASK_ACTIVE_MAX_LIMIT, max(1, int(limit or TASK_ACTIVE_DEFAULT_LIMIT)))
@@ -73,6 +143,199 @@ async def get_recent_tasks(
return [TaskInfo.model_validate(task) for task in orm_tasks]
@router.get("/tasks/runtime-summary")
async def get_task_runtime_summary(limit: int = TASK_ACTIVE_DEFAULT_LIMIT, offset: int = 0):
safe_limit = min(TASK_ACTIVE_MAX_LIMIT, max(1, int(limit or TASK_ACTIVE_DEFAULT_LIMIT)))
safe_offset = min(TASK_QUERY_MAX_OFFSET, max(0, int(offset or 0)))
active_job_statuses = ["READY", "RETRY", "RUNNING"]
scan_job_types = {
"SCAN_DATA",
"SCAN_DINSAR",
"SCAN_ASSET_INVENTORY",
"AUDIT_SOURCE_ARCHIVE_INTEGRITY",
"GF3_SARSCAPE_SYNC",
"GF3_QUICKLOOK_WEBP",
}
worker_timeout = max(5, int(getattr(settings, "JOB_WORKER_HEALTH_TIMEOUT", 60) or 60))
worker_threshold = datetime.utcnow() - timedelta(seconds=worker_timeout)
configured_concurrency = max(1, int(getattr(settings, "JOB_WORKER_CONCURRENCY", 1) or 1))
async with _new_session() as db:
active_tasks = await task_service.get_active_tasks(limit=safe_limit, offset=safe_offset, db=db)
workers_result = await db.execute(
select(SystemWorkerHeartbeatORM)
.where(SystemWorkerHeartbeatORM.last_seen >= worker_threshold)
.order_by(SystemWorkerHeartbeatORM.last_seen.desc())
)
active_workers = workers_result.scalars().all()
active_worker_ids = {str(worker.worker_id) for worker in active_workers}
running_by_worker_result = await db.execute(
select(SystemJobORM.locked_by, func.count(SystemJobORM.id))
.where(SystemJobORM.status == "RUNNING")
.group_by(SystemJobORM.locked_by)
)
running_by_worker = {
str(worker_id or ""): int(count or 0)
for worker_id, count in running_by_worker_result.all()
}
status_counts_result = await db.execute(
select(SystemJobORM.status, func.count(SystemJobORM.id))
.where(SystemJobORM.status.in_(active_job_statuses))
.group_by(SystemJobORM.status)
)
job_status_counts = {
"READY": 0,
"RETRY": 0,
"RUNNING": 0,
}
for status, count in status_counts_result.all():
job_status_counts[str(status or "").upper()] = int(count or 0)
status_rank = case(
(SystemJobORM.status == "RUNNING", 0),
(SystemJobORM.status == "RETRY", 1),
else_=2,
)
jobs_result = await db.execute(
select(SystemJobORM)
.where(SystemJobORM.status.in_(active_job_statuses))
.order_by(status_rank, SystemJobORM.priority.desc(), SystemJobORM.id.asc())
.offset(safe_offset)
.limit(safe_limit)
)
active_jobs = jobs_result.scalars().all()
task_ids = {
str(task.task_id)
for task in active_tasks
if task.task_id
}
task_ids.update(
str(job.task_id)
for job in active_jobs
if job.task_id
)
task_by_id: dict[str, SystemTaskORM] = {}
if task_ids:
task_result = await db.execute(
select(SystemTaskORM).where(SystemTaskORM.task_id.in_(sorted(task_ids)))
)
task_by_id = {
str(task.task_id): task
for task in task_result.scalars().all()
if task.task_id
}
worker_concurrency_by_id = {
str(worker.worker_id): _worker_concurrency(worker)
for worker in active_workers
}
total_slots = sum(worker_concurrency_by_id.values())
busy_slots = sum(
count
for worker_id, count in running_by_worker.items()
if worker_id in active_worker_ids
)
running_count = int(job_status_counts.get("RUNNING") or 0)
stale_running_count = max(0, running_count - busy_slots)
queued_count = int(job_status_counts.get("READY") or 0) + int(job_status_counts.get("RETRY") or 0)
task_items = [_task_payload(task) for task in active_tasks]
job_items = [_job_payload(job, task_by_id) for job in active_jobs]
scan_jobs = [
item for item in job_items
if str(item.get("job_type") or "").upper() in scan_job_types
]
scan_tasks = [
item for item in task_items
if str(item.get("task_type") or "").upper() in scan_job_types
]
return {
"timestamp": datetime.utcnow().isoformat() + "Z",
"worker": {
"ok": len(active_workers) > 0,
"worker_count": len(active_workers),
"configured_concurrency": configured_concurrency,
"total_slots": total_slots,
"busy_slots": busy_slots,
"idle_slots": max(0, total_slots - busy_slots),
"timeout_seconds": worker_timeout,
"stale_running_job_count": stale_running_count,
"workers": [
_worker_payload(
worker,
running_by_worker.get(str(worker.worker_id), 0),
worker_concurrency_by_id.get(str(worker.worker_id), 1),
)
for worker in active_workers
],
},
"jobs": {
"active_count": running_count + queued_count,
"running_count": running_count,
"queued_count": queued_count,
"ready_count": int(job_status_counts.get("READY") or 0),
"retry_count": int(job_status_counts.get("RETRY") or 0),
"items": job_items,
},
"tasks": {
"active_count": len(task_items),
"running_count": sum(1 for item in task_items if item.get("status") == "RUNNING"),
"pending_count": sum(1 for item in task_items if item.get("status") == "PENDING"),
"items": task_items,
},
"scan": {
"active_task_count": len(scan_tasks),
"active_job_count": len(scan_jobs),
"running_job_count": sum(1 for item in scan_jobs if item.get("status") == "RUNNING"),
"queued_job_count": sum(1 for item in scan_jobs if item.get("status") in {"READY", "RETRY"}),
"tasks": scan_tasks,
"jobs": scan_jobs,
},
}
@router.get("/tasks/runtime-summary/stream")
async def stream_task_runtime_summary(request: Request):
token = request.cookies.get(SESSION_COOKIE_NAME)
if not token:
raise HTTPException(status_code=401, detail="Authentication required.")
async with _new_session() as db:
user = await get_user_by_session_token(db, token)
if not user:
raise HTTPException(status_code=401, detail="Authentication required.")
async def event_generator():
while True:
if await request.is_disconnected():
break
try:
summary = await get_task_runtime_summary(
limit=TASK_ACTIVE_MAX_LIMIT,
offset=0,
)
yield f"data: {json.dumps(summary)}\n\n"
except Exception:
yield "data: {}\n\n"
await asyncio.sleep(3)
return StreamingResponse(
event_generator(),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
"X-Accel-Buffering": "no",
"Connection": "keep-alive",
},
)
@router.get("/tasks/active/stream")
async def stream_active_tasks(request: Request):
token = request.cookies.get(SESSION_COOKIE_NAME)