feat: engineer SBAS timeseries production workflow

2026-04-29 14:43:31 +08:00
parent dace8b20f6
commit 4c0d1f2c2b
54 changed files with 5843 additions and 201 deletions
@@ -158,6 +158,7 @@ AOI_UPLOAD_MAX_TOTAL_BYTES = max(
    AOI_UPLOAD_MAX_SINGLE_FILE_BYTES,
 )
 AOI_UPLOAD_STREAM_CHUNK_BYTES = 1024 * 1024
+_SHAPEFILE_READ_LOCK = asyncio.Lock()

 # ---------------------------------------------------------------------------
 # Region index caches
@@ -805,6 +806,20 @@ def _parse_aoi_geojson_form_value(aoi_geojson: Optional[str]) -> Optional[Tuple[
    return merged_geometry.wkt, feature_collection


+def _read_aoi_shapefile_with_restore_shx(shp_path: str):
+    import geopandas as gpd
+
+    previous_restore_shx = os.environ.get("SHAPE_RESTORE_SHX")
+    os.environ["SHAPE_RESTORE_SHX"] = "YES"
+    try:
+        return gpd.read_file(shp_path, engine="pyogrio")
+    finally:
+        if previous_restore_shx is None:
+            os.environ.pop("SHAPE_RESTORE_SHX", None)
+        else:
+            os.environ["SHAPE_RESTORE_SHX"] = previous_restore_shx
+
+
 async def _parse_aoi_from_files(files: Optional[List[UploadFile]]) -> Optional[Tuple[str, Dict[str, Any]]]:
    if not files:
        return None
@@ -869,9 +884,17 @@ async def _parse_aoi_from_files(files: Optional[List[UploadFile]]) -> Optional[T
                    geojson_payload = json.loads(Path(dest_path).read_text(encoding="gbk"))

        if shp_path:
-            import geopandas as gpd
-
-            gdf = await asyncio.to_thread(gpd.read_file, shp_path, engine="pyogrio")
+            try:
+                async with _SHAPEFILE_READ_LOCK:
+                    gdf = await asyncio.to_thread(_read_aoi_shapefile_with_restore_shx, shp_path)
+            except Exception as exc:
+                raise HTTPException(
+                    status_code=400,
+                    detail=(
+                        "AOI Shapefile 读取失败。系统已尝试自动恢复缺失的 .shx 索引；"
+                        f"请确认已上传 .shp/.dbf/.prj/.shx 或可恢复的标准 Shapefile。原始错误: {exc}"
+                    ),
+                ) from exc
            if gdf.crs and gdf.crs.to_epsg() != 4326:
                gdf = gdf.to_crs(epsg=4326)
            feature_collection = json.loads(gdf.to_json())
@@ -16,6 +16,11 @@ from ..models import (
    PairingResponse,
    PsRequest,
    RadarData,
+    TimeseriesStackPlan,
+    TimeseriesStackPlanDetail,
+    TimeseriesStackPlanItem,
+    TimeseriesStackPlanItemORM,
+    TimeseriesStackPlanORM,
 )
 from ..services.pairing_cache_service import pairing_cache_service
 from ..services.spatial_service import spatial_service
@@ -177,6 +182,37 @@ async def get_pairing_network_run_endpoint(
    }


+@router.get("/timeseries-plans/{plan_id}", response_model=TimeseriesStackPlanDetail)
+async def get_timeseries_stack_plan_endpoint(
+    plan_id: str,
+    db: AsyncSession = Depends(get_db),
+    current_user: AuthUserORM = Depends(_require_admin),
+):
+    _ = current_user
+    normalized_plan_id = str(plan_id or "").strip()
+    if not normalized_plan_id:
+        raise HTTPException(status_code=400, detail="plan_id is required.")
+
+    plan_result = await db.execute(
+        select(TimeseriesStackPlanORM).where(TimeseriesStackPlanORM.plan_id == normalized_plan_id)
+    )
+    plan = plan_result.scalar_one_or_none()
+    if plan is None:
+        raise HTTPException(status_code=404, detail="Timeseries stack plan not found.")
+
+    items_result = await db.execute(
+        select(TimeseriesStackPlanItemORM)
+        .where(TimeseriesStackPlanItemORM.plan_ref_id == plan.id)
+        .order_by(TimeseriesStackPlanItemORM.scene_rank.asc(), TimeseriesStackPlanItemORM.id.asc())
+    )
+    payload = TimeseriesStackPlan.model_validate(plan).model_dump()
+    payload["items"] = [
+        TimeseriesStackPlanItem.model_validate(item)
+        for item in items_result.scalars().all()
+    ]
+    return TimeseriesStackPlanDetail.model_validate(payload)
+
+
@router.post("/find-pairs", response_model=PairingResponse)
 async def find_pairs_endpoint(
    params: PairingRequest = Depends(get_pairing_request_from_form),
@@ -1,8 +1,10 @@
 from __future__ import annotations

+import json
+import os
 import uuid
 from datetime import datetime
-from typing import List, Optional
+from typing import Any, Dict, List, Optional

 from fastapi import APIRouter, Depends, HTTPException, Request
 from pydantic import BaseModel, Field, field_validator
@@ -23,6 +25,8 @@ from ..models import (
    PsTaskItemORM,
    RadarData,
    RadarPair,
+    TimeseriesStackPlanItemORM,
+    TimeseriesStackPlanORM,
 )
 from .dependencies import (
    _add_operation_audit_log,
@@ -109,7 +113,9 @@ class DinsarBatchCreateRequest(BaseModel):
 class PsBatchCreateRequest(BaseModel):
    name: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH)
    direction: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH)
+    plan_id: Optional[str] = Field(default=None, max_length=64)
    stack: List[RadarData]
+    planning_context: Optional[Dict[str, Any]] = None

    @field_validator("stack")
    @classmethod
@@ -118,6 +124,8 @@ class PsBatchCreateRequest(BaseModel):
            raise ValueError(
                f"stack exceeds max item count ({TASK_BATCH_MAX_ITEMS})."
            )
+        if len(value) < 3:
+            raise ValueError("SBAS timeseries batch requires at least 3 scenes.")
        return value


@@ -126,6 +134,57 @@ class BatchItemUpdateRequest(BaseModel):
    remark: Optional[str] = Field(default=None, max_length=BATCH_REMARK_MAX_LENGTH)


+def _normalize_lookup_key(value: Optional[str]) -> str:
+    text = str(value or "").strip()
+    if not text:
+        return ""
+    return os.path.normcase(os.path.normpath(text))
+
+
+def _build_plan_context(
+    plan: TimeseriesStackPlanORM,
+    plan_items: List[TimeseriesStackPlanItemORM],
+) -> Dict[str, Any]:
+    request_params = plan.request_params_json if isinstance(plan.request_params_json, dict) else {}
+    ordered_items = sorted(
+        plan_items,
+        key=lambda item: (int(item.scene_rank or 0), int(item.id or 0)),
+    )
+    scenes = [
+        {
+            "plan_item_id": item.id,
+            "scene_id": item.radar_data_ref_id,
+            "scene_rank": item.scene_rank,
+            "scene_file_path": item.file_path,
+            "scene_imaging_date": item.imaging_date,
+            "scene_satellite": item.satellite,
+            "scene_imaging_mode": item.imaging_mode,
+            "scene_polarization": item.polarization,
+            "selection_meta": item.selection_meta_json if isinstance(item.selection_meta_json, dict) else None,
+        }
+        for item in ordered_items
+    ]
+    return {
+        "source": "timeseries_stack_plan",
+        "plan_id": plan.plan_id,
+        "strategy": plan.strategy,
+        "direction": plan.direction,
+        "scene_count": int(plan.scene_count or len(scenes)),
+        "stack_key": plan.stack_key,
+        "group_key": plan.group_key,
+        "request_hash": plan.request_hash,
+        "aoi_summary": plan.aoi_summary_json if isinstance(plan.aoi_summary_json, dict) else None,
+        "initial_overlap_threshold": request_params.get("initial_overlap_threshold"),
+        "final_overlap_threshold": request_params.get("final_overlap_threshold"),
+        "stack_dates": [
+            str(item.imaging_date).strip()
+            for item in ordered_items
+            if str(item.imaging_date or "").strip()
+        ],
+        "scenes": scenes,
+    }
+
+
@router.post("/task-batches/dinsar", response_model=DinsarTaskBatch)
 async def create_dinsar_batch_endpoint(
    request: DinsarBatchCreateRequest,
@@ -303,12 +362,85 @@ async def create_ps_batch_endpoint(
    if not request.stack:
        raise HTTPException(status_code=400, detail="No PS items provided.")

+    request_plan_id = (
+        request.planning_context.get("plan_id")
+        if isinstance(request.planning_context, dict)
+        else None
+    )
+    explicit_plan_id = str(request.plan_id or request_plan_id or "").strip() or None
+    inferred_plan_ids = sorted(
+        {
+            str(item.stack_plan_id or "").strip()
+            for item in request.stack
+            if str(item.stack_plan_id or "").strip()
+        }
+    )
+    if len(inferred_plan_ids) > 1:
+        raise HTTPException(status_code=400, detail="PS stack items belong to multiple stack plans.")
+    if explicit_plan_id and inferred_plan_ids and explicit_plan_id != inferred_plan_ids[0]:
+        raise HTTPException(status_code=400, detail="request.plan_id does not match stack scene plan metadata.")
+
+    effective_plan_id = explicit_plan_id or (inferred_plan_ids[0] if inferred_plan_ids else None)
+    plan: Optional[TimeseriesStackPlanORM] = None
+    plan_items: List[TimeseriesStackPlanItemORM] = []
+    plan_item_by_id: Dict[int, TimeseriesStackPlanItemORM] = {}
+    plan_item_by_scene_id: Dict[int, TimeseriesStackPlanItemORM] = {}
+    plan_item_by_path: Dict[str, TimeseriesStackPlanItemORM] = {}
+    planning_context = request.planning_context if isinstance(request.planning_context, dict) else None
+
+    if effective_plan_id:
+        plan_result = await db.execute(
+            select(TimeseriesStackPlanORM).where(TimeseriesStackPlanORM.plan_id == effective_plan_id)
+        )
+        plan = plan_result.scalar_one_or_none()
+        if plan is None:
+            raise HTTPException(status_code=404, detail=f"Timeseries stack plan not found: {effective_plan_id}")
+        if (
+            str(request.direction or "").strip()
+            and str(plan.direction or "").strip()
+            and str(request.direction).strip().upper() != str(plan.direction).strip().upper()
+        ):
+            raise HTTPException(status_code=400, detail="request.direction does not match the referenced stack plan.")
+
+        items_result = await db.execute(
+            select(TimeseriesStackPlanItemORM)
+            .where(TimeseriesStackPlanItemORM.plan_ref_id == plan.id)
+            .order_by(TimeseriesStackPlanItemORM.scene_rank.asc(), TimeseriesStackPlanItemORM.id.asc())
+        )
+        plan_items = items_result.scalars().all()
+        plan_item_by_id = {int(item.id): item for item in plan_items if item.id is not None}
+        plan_item_by_scene_id = {
+            int(item.radar_data_ref_id): item
+            for item in plan_items
+            if item.radar_data_ref_id is not None
+        }
+        plan_item_by_path = {
+            _normalize_lookup_key(item.file_path): item
+            for item in plan_items
+            if _normalize_lookup_key(item.file_path)
+        }
+        if not planning_context:
+            planning_context = _build_plan_context(plan, plan_items)
+        else:
+            merged_context = {
+                **_build_plan_context(plan, plan_items),
+                **planning_context,
+            }
+            if "scenes" not in planning_context:
+                merged_context["scenes"] = _build_plan_context(plan, plan_items).get("scenes") or []
+            planning_context = merged_context
+
    batch_id = str(uuid.uuid4())
    batch_name = request.name or f"PS_{(request.direction or 'STACK')}_{datetime.utcnow().strftime('%Y%m%d_%H%M%S')}"
    batch = PsTaskBatchORM(
        batch_id=batch_id,
        name=batch_name,
        direction=request.direction,
+        plan_id=plan.plan_id if plan is not None else effective_plan_id,
+        plan_strategy=(
+            (plan.strategy if plan is not None else None)
+            or (planning_context or {}).get("strategy")
+        ),
        status="PENDING",
        total_items=len(request.stack),
        completed_items=0,
@@ -316,14 +448,49 @@ async def create_ps_batch_endpoint(
    db.add(batch)

    for img in request.stack:
+        matched_plan_item: Optional[TimeseriesStackPlanItemORM] = None
+        if img.stack_plan_item_id is not None and int(img.stack_plan_item_id) in plan_item_by_id:
+            matched_plan_item = plan_item_by_id[int(img.stack_plan_item_id)]
+        elif img.id is not None and int(img.id) in plan_item_by_scene_id:
+            matched_plan_item = plan_item_by_scene_id[int(img.id)]
+        else:
+            matched_plan_item = plan_item_by_path.get(_normalize_lookup_key(img.file_path))
+        if batch.plan_id and matched_plan_item is None:
+            raise HTTPException(
+                status_code=400,
+                detail=f"PS stack scene is not part of referenced stack plan: {img.file_path}",
+            )
+
+        remark_payload = None
+        if planning_context:
+            planning_summary = {
+                key: value
+                for key, value in planning_context.items()
+                if key != "scenes"
+            }
+            remark_payload = {
+                **planning_summary,
+                "plan_id": batch.plan_id,
+                "plan_item_id": int(matched_plan_item.id) if matched_plan_item and matched_plan_item.id is not None else None,
+                "scene_id": img.id,
+                "scene_file_path": img.file_path,
+                "scene_imaging_date": img.imaging_date,
+                "scene_satellite": img.satellite,
+            }
        item = PsTaskItemORM(
            batch_id=batch_id,
+            plan_item_ref_id=(
+                int(matched_plan_item.id)
+                if matched_plan_item is not None and matched_plan_item.id is not None
+                else None
+            ),
            file_path=img.file_path,
            satellite=img.satellite,
            imaging_date=img.imaging_date,
            polarization=img.polarization,
            has_orbit_data=bool(img.has_orbit_data),
            status="PENDING",
+            remark=json.dumps(remark_payload, ensure_ascii=False) if remark_payload else None,
        )
        db.add(item)

@@ -332,7 +499,14 @@ async def create_ps_batch_endpoint(
        request=http_request,
        action="batch_created",
        resource=f"task-batches/ps/{batch_id}",
-        detail={"batch_name": batch_name, "items": len(request.stack), "direction": request.direction},
+        detail={
+            "batch_name": batch_name,
+            "items": len(request.stack),
+            "direction": request.direction,
+            "plan_id": batch.plan_id,
+            "plan_strategy": batch.plan_strategy,
+            "planning_context": planning_context,
+        },
    )
    await db.commit()
    await db.refresh(batch)
@@ -31,6 +31,45 @@ class TimeseriesRunCreateRequest(BaseModel):
        return text


+class TimeseriesWslCheckRequest(BaseModel):
+    distro: Optional[str] = Field(default=None, max_length=128)
+    smoke_test: bool = Field(default=False)
+
+    @field_validator("distro", mode="before")
+    @classmethod
+    def _normalize_distro(cls, value: Optional[str]) -> Optional[str]:
+        if value is None:
+            return None
+        text = str(value).strip()
+        return text or None
+
+
+class TimeseriesPreflightRequest(BaseModel):
+    batch_id: str = Field(..., description="PS batch id")
+    reference_date: Optional[str] = Field(default=None, pattern=r"^\d{8}$|^$")
+    water_mask_mode: str = Field(default="synthetic_fallback", max_length=64)
+
+    @field_validator("batch_id", mode="before")
+    @classmethod
+    def _validate_batch_id(cls, value: str) -> str:
+        text = str(value or "").strip()
+        if not text:
+            raise ValueError("batch_id is required")
+        return text
+
+
+class TimeseriesRetryStepRequest(BaseModel):
+    step_id: str = Field(..., max_length=128)
+
+    @field_validator("step_id", mode="before")
+    @classmethod
+    def _validate_step_id(cls, value: str) -> str:
+        text = str(value or "").strip()
+        if not text:
+            raise ValueError("step_id is required")
+        return text
+
+
@router.post("/runs", status_code=202)
 async def create_timeseries_run(
    request: TimeseriesRunCreateRequest,
@@ -53,6 +92,39 @@ async def create_timeseries_run(
        raise HTTPException(status_code=status_code, detail=message) from exc


+@router.post("/wsl-check")
+async def run_timeseries_wsl_check(
+    request: TimeseriesWslCheckRequest,
+    current_user: AuthUserORM = Depends(_require_admin),
+):
+    _ = current_user
+    try:
+        return await timeseries_service.get_runtime_report(
+            distro=request.distro,
+            smoke_test=request.smoke_test,
+        )
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+
+@router.post("/preflight")
+async def run_timeseries_preflight(
+    request: TimeseriesPreflightRequest,
+    current_user: AuthUserORM = Depends(_require_admin),
+    db: AsyncSession = Depends(get_db),
+):
+    _ = current_user
+    try:
+        return await timeseries_service.get_preflight_report(
+            batch_id=request.batch_id,
+            reference_date=request.reference_date,
+            water_mask_mode=request.water_mask_mode,
+            db=db,
+        )
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+
@router.get("/runs")
 async def list_timeseries_runs(
    limit: int = 50,
@@ -75,3 +147,25 @@ async def get_timeseries_run_detail(
    if detail is None:
        raise HTTPException(status_code=404, detail="Timeseries run not found")
    return detail
+
+
+@router.post("/runs/{run_id}/retry-step", status_code=202)
+async def retry_timeseries_run_step(
+    run_id: str,
+    request: TimeseriesRetryStepRequest,
+    current_user: AuthUserORM = Depends(_require_admin),
+    db: AsyncSession = Depends(get_db),
+):
+    _ = current_user
+    try:
+        return await timeseries_service.retry_step(
+            run_id,
+            step_id=request.step_id,
+            db=db,
+        )
+    except ValueError as exc:
+        message = str(exc)
+        status_code = 409 if "cannot be retried" in message or "running steps" in message else 400
+        if "not found" in message:
+            status_code = 404
+        raise HTTPException(status_code=status_code, detail=message) from exc