Add SARscape SBAS prepared stack workflow

This commit is contained in:
2026-04-30 09:42:01 +08:00
parent 4c0d1f2c2b
commit 6696fe90fa
27 changed files with 4915 additions and 95 deletions
+2
View File
@@ -56,6 +56,8 @@ nginx/temp/
experiments/**/scratch/ experiments/**/scratch/
backend/quality_model.pkl backend/quality_model.pkl
/_par_win_*.xls /_par_win_*.xls
/IDL*.tmp
/env_*.xyz
.codex_tmp/ .codex_tmp/
# Large local datasets / installers # Large local datasets / installers
+15
View File
@@ -301,8 +301,13 @@ class Settings(BaseSettings):
TIMESERIES_MINTPY_SBAS_SCRIPT: str = "" TIMESERIES_MINTPY_SBAS_SCRIPT: str = ""
TIMESERIES_EXPORT_PUBLISH_SCRIPT: str = "" TIMESERIES_EXPORT_PUBLISH_SCRIPT: str = ""
TIMESERIES_STACK_WORKFLOW: str = "interferogram" TIMESERIES_STACK_WORKFLOW: str = "interferogram"
TIMESERIES_DEFAULT_PROCESSOR_CODE: str = "isce2_stack_mintpy"
TIMESERIES_WSL_STEP_TIMEOUT_SECONDS: int = 7200 TIMESERIES_WSL_STEP_TIMEOUT_SECONDS: int = 7200
TIMESERIES_ALLOW_SYNTHETIC_WATER_MASK: bool = True TIMESERIES_ALLOW_SYNTHETIC_WATER_MASK: bool = True
SARSCAPE_SBAS_PARAMETER_TEMPLATE_PATH: str = ""
SARSCAPE_SBAS_ALLOW_EXECUTION: bool = False
SARSCAPE_SBAS_DISCOVERY_TIMEOUT_SECONDS: int = 120
SARSCAPE_SBAS_STEP_TIMEOUT_SECONDS: int = 21600
@model_validator(mode="after") @model_validator(mode="after")
def _set_path_defaults(self) -> "Settings": def _set_path_defaults(self) -> "Settings":
@@ -601,6 +606,16 @@ class Settings(BaseSettings):
"export_mintpy_publish_products_unified_env_ubuntu2404.sh", "export_mintpy_publish_products_unified_env_ubuntu2404.sh",
), ),
) )
if not self.SARSCAPE_SBAS_PARAMETER_TEMPLATE_PATH:
object.__setattr__(
self,
"SARSCAPE_SBAS_PARAMETER_TEMPLATE_PATH",
os.path.join(
backend_dir,
"templates",
"sarscape_sbas_parameter_template.example.json",
),
)
return self return self
@staticmethod @staticmethod
+1
View File
@@ -34,6 +34,7 @@ MIGRATION_FILES = [
"005_pairing_task_trace.sql", "005_pairing_task_trace.sql",
"006_result_pairing_trace.sql", "006_result_pairing_trace.sql",
"007_timeseries_stack_plan_trace.sql", "007_timeseries_stack_plan_trace.sql",
"008_timeseries_stack_plan_edges.sql",
] ]
+4 -2
View File
@@ -17,6 +17,7 @@ from .orm import (
PairingNetworkEdgeORM, PairingNetworkEdgeORM,
TimeseriesStackPlanORM, TimeseriesStackPlanORM,
TimeseriesStackPlanItemORM, TimeseriesStackPlanItemORM,
TimeseriesStackPlanEdgeORM,
HazardPointORM, HazardPointORM,
SystemTaskORM, SystemTaskORM,
TaskLogORM, TaskLogORM,
@@ -63,6 +64,7 @@ from .schemas import (
PsRequest, PsRequest,
TimeseriesStackPlan, TimeseriesStackPlan,
TimeseriesStackPlanItem, TimeseriesStackPlanItem,
TimeseriesStackPlanEdge,
TimeseriesStackPlanDetail, TimeseriesStackPlanDetail,
TaskInfo, TaskInfo,
AuthUserInfo, AuthUserInfo,
@@ -89,7 +91,7 @@ __all__ = [
"ResultIssueORM", "ResultCatalogStateORM", "ResultIssueORM", "ResultCatalogStateORM",
"PairingCacheStateORM", "PairingDirtySceneORM", "PairingMetricCacheORM", "PairingCacheStateORM", "PairingDirtySceneORM", "PairingMetricCacheORM",
"PairingNetworkRunORM", "PairingNetworkEdgeORM", "PairingNetworkRunORM", "PairingNetworkEdgeORM",
"TimeseriesStackPlanORM", "TimeseriesStackPlanItemORM", "TimeseriesStackPlanORM", "TimeseriesStackPlanItemORM", "TimeseriesStackPlanEdgeORM",
"SystemTaskORM", "TaskLogORM", "SystemJobORM", "ScanStateORM", "SystemTaskORM", "TaskLogORM", "SystemJobORM", "ScanStateORM",
"ManagedRootORM", "ScanCursorORM", "PathInventoryORM", "ManagedRootORM", "ScanCursorORM", "PathInventoryORM",
"WorkflowDefORM", "WorkflowRunORM", "WorkflowStepORM", "WorkflowArtifactORM", "WorkflowDefORM", "WorkflowRunORM", "WorkflowStepORM", "WorkflowArtifactORM",
@@ -105,7 +107,7 @@ __all__ = [
"HazardPoint", "DinsarResult", "ScanRequest", "ManagedRootInfo", "ScanCursorInfo", "HazardPoint", "DinsarResult", "ScanRequest", "ManagedRootInfo", "ScanCursorInfo",
"RadarData", "RadarDataPage", "DinsarResultPage", "RadarData", "RadarDataPage", "DinsarResultPage",
"PairingRequest", "RadarPair", "PairingResponse", "PairingRequest", "RadarPair", "PairingResponse",
"PsRequest", "TimeseriesStackPlan", "TimeseriesStackPlanItem", "TimeseriesStackPlanDetail", "TaskInfo", "PsRequest", "TimeseriesStackPlan", "TimeseriesStackPlanItem", "TimeseriesStackPlanEdge", "TimeseriesStackPlanDetail", "TaskInfo",
"AuthUserInfo", "AuthAuditLogInfo", "RadarPreviewStatusInfo", "AuthUserInfo", "AuthAuditLogInfo", "RadarPreviewStatusInfo",
"DinsarTaskBatch", "DinsarTaskItem", "PsTaskBatch", "PsTaskItem", "PsTimeseriesRun", "DinsarTaskBatch", "DinsarTaskItem", "PsTaskBatch", "PsTaskItem", "PsTimeseriesRun",
"WaterDetectRequest", "WaterDetectResponse", "WaterDetectRequest", "WaterDetectResponse",
+82
View File
@@ -467,6 +467,11 @@ class TimeseriesStackPlanORM(Base):
back_populates="plan", back_populates="plan",
cascade="all, delete-orphan", cascade="all, delete-orphan",
) )
edges = relationship(
"TimeseriesStackPlanEdgeORM",
back_populates="plan",
cascade="all, delete-orphan",
)
__table_args__ = ( __table_args__ = (
Index("idx_timeseries_stack_plans_direction_created", "direction", "created_at"), Index("idx_timeseries_stack_plans_direction_created", "direction", "created_at"),
@@ -508,6 +513,83 @@ class TimeseriesStackPlanItemORM(Base):
) )
class TimeseriesStackPlanEdgeORM(Base):
__tablename__ = "timeseries_stack_plan_edges"
id = Column(Integer, primary_key=True, autoincrement=True)
plan_ref_id = Column(
Integer,
ForeignKey("timeseries_stack_plans.id", ondelete="CASCADE"),
index=True,
nullable=False,
)
master_plan_item_ref_id = Column(
Integer,
ForeignKey("timeseries_stack_plan_items.id", ondelete="SET NULL"),
index=True,
nullable=True,
)
slave_plan_item_ref_id = Column(
Integer,
ForeignKey("timeseries_stack_plan_items.id", ondelete="SET NULL"),
index=True,
nullable=True,
)
metric_cache_ref_id = Column(
Integer,
ForeignKey("pairing_metric_cache.id", ondelete="SET NULL"),
index=True,
nullable=True,
)
master_scene_ref_id = Column(
Integer,
ForeignKey("radar_data.id", ondelete="SET NULL"),
index=True,
nullable=True,
)
slave_scene_ref_id = Column(
Integer,
ForeignKey("radar_data.id", ondelete="SET NULL"),
index=True,
nullable=True,
)
edge_rank = Column(Integer, nullable=False, default=0)
master_imaging_date = Column(String(8), index=True, nullable=True)
slave_imaging_date = Column(String(8), index=True, nullable=True)
temporal_baseline_days = Column(Integer, nullable=True)
spatial_baseline_meters = Column(Float, nullable=True)
perpendicular_baseline_meters = Column(Float, nullable=True)
scene_overlap_ratio = Column(Float, nullable=True)
pair_aoi_overlap_ratio = Column(Float, nullable=True)
selection_reason = Column(String(64), nullable=True)
selection_score = Column(Float, nullable=True)
selection_meta_json = Column(JSON, nullable=True)
enabled = Column(Boolean, nullable=False, default=True)
created_at = Column(DateTime, server_default=func.now(), nullable=False)
plan = relationship("TimeseriesStackPlanORM", back_populates="edges")
master_plan_item = relationship("TimeseriesStackPlanItemORM", foreign_keys=[master_plan_item_ref_id])
slave_plan_item = relationship("TimeseriesStackPlanItemORM", foreign_keys=[slave_plan_item_ref_id])
metric_cache = relationship("PairingMetricCacheORM")
master_scene = relationship("RadarDataORM", foreign_keys=[master_scene_ref_id])
slave_scene = relationship("RadarDataORM", foreign_keys=[slave_scene_ref_id])
__table_args__ = (
UniqueConstraint(
"plan_ref_id",
"edge_rank",
name="uq_timeseries_plan_edges_plan_rank",
),
Index("idx_timeseries_plan_edges_plan_enabled", "plan_ref_id", "enabled"),
Index(
"idx_timeseries_plan_edges_plan_scenes",
"plan_ref_id",
"master_scene_ref_id",
"slave_scene_ref_id",
),
)
class HazardPointORM(Base): class HazardPointORM(Base):
__tablename__ = 'hazard_points' __tablename__ = 'hazard_points'
+33
View File
@@ -203,6 +203,8 @@ class RadarData(BaseModel):
stack_coverage_consistency_ratio: Optional[float] = None stack_coverage_consistency_ratio: Optional[float] = None
stack_threshold_satisfied: Optional[bool] = None stack_threshold_satisfied: Optional[bool] = None
stack_selection_mode: Optional[str] = None stack_selection_mode: Optional[str] = None
stack_network_edge_count: Optional[int] = None
stack_network_warnings: Optional[List[str]] = None
model_config = ConfigDict(from_attributes=True) model_config = ConfigDict(from_attributes=True)
@@ -353,6 +355,11 @@ class PsRequest(BaseModel):
"""PS-InSAR 时序分析数据准备的请求模型。""" """PS-InSAR 时序分析数据准备的请求模型。"""
initial_overlap_threshold: float = Field(default=0.3, ge=0.0, le=1.0) initial_overlap_threshold: float = Field(default=0.3, ge=0.0, le=1.0)
final_overlap_threshold: float = Field(default=0.95, ge=0.0, le=1.0) final_overlap_threshold: float = Field(default=0.95, ge=0.0, le=1.0)
time_baseline_min: int = Field(default=1, ge=0, le=3650)
time_baseline_max: int = Field(default=90, ge=1, le=3650)
spatial_baseline_max_meters: int = Field(default=3000, ge=0, le=100000)
network_overlap_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
num_connections: int = Field(default=1, ge=1, le=10)
class TimeseriesStackPlanItem(BaseModel): class TimeseriesStackPlanItem(BaseModel):
@@ -393,8 +400,34 @@ class TimeseriesStackPlan(BaseModel):
model_config = ConfigDict(from_attributes=True) model_config = ConfigDict(from_attributes=True)
class TimeseriesStackPlanEdge(BaseModel):
id: int
plan_ref_id: int
master_plan_item_ref_id: Optional[int] = None
slave_plan_item_ref_id: Optional[int] = None
metric_cache_ref_id: Optional[int] = None
master_scene_ref_id: Optional[int] = None
slave_scene_ref_id: Optional[int] = None
edge_rank: int
master_imaging_date: Optional[str] = None
slave_imaging_date: Optional[str] = None
temporal_baseline_days: Optional[int] = None
spatial_baseline_meters: Optional[float] = None
perpendicular_baseline_meters: Optional[float] = None
scene_overlap_ratio: Optional[float] = None
pair_aoi_overlap_ratio: Optional[float] = None
selection_reason: Optional[str] = None
selection_score: Optional[float] = None
selection_meta_json: Optional[Dict[str, Any]] = None
enabled: bool = True
created_at: datetime
model_config = ConfigDict(from_attributes=True)
class TimeseriesStackPlanDetail(TimeseriesStackPlan): class TimeseriesStackPlanDetail(TimeseriesStackPlan):
items: List[TimeseriesStackPlanItem] = Field(default_factory=list) items: List[TimeseriesStackPlanItem] = Field(default_factory=list)
edges: List[TimeseriesStackPlanEdge] = Field(default_factory=list)
class TaskInfo(BaseModel): class TaskInfo(BaseModel):
+23 -1
View File
@@ -3,7 +3,7 @@ from __future__ import annotations
import asyncio import asyncio
import os import os
import re as _re import re as _re
from typing import Any, Dict, Optional from typing import Any, Dict, List, Optional
from fastapi import APIRouter, Depends, HTTPException from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel, Field from pydantic import BaseModel, Field
@@ -55,6 +55,12 @@ class ExtractDispRequest(BaseModel):
dest_dir: Optional[str] = None dest_dir: Optional[str] = None
class SarscapeSbasInspectRequest(BaseModel):
task_names: Optional[List[str]] = None
include_parameters: bool = False
timeout_seconds: Optional[int] = Field(default=120, ge=10, le=600)
def _normalize_existing_dir(path: Optional[str]) -> Optional[str]: def _normalize_existing_dir(path: Optional[str]) -> Optional[str]:
text = str(path or "").strip() text = str(path or "").strip()
if not text: if not text:
@@ -162,6 +168,22 @@ async def inspect_dinsar_endpoint(
return envi_service.inspect_dinsar(request.root_dir) return envi_service.inspect_dinsar(request.root_dir)
@router.post("/idl/inspect/sarscape-sbas")
async def inspect_sarscape_sbas_endpoint(
request: SarscapeSbasInspectRequest,
admin_user: AuthUserORM = Depends(_require_admin),
):
_ = admin_user
try:
return envi_service.inspect_sarscape_sbas_tasks_subprocess(
request.task_names,
timeout_seconds=request.timeout_seconds or 120,
include_parameters=bool(request.include_parameters),
)
except Exception as exc:
raise HTTPException(status_code=500, detail=str(exc)) from exc
@router.post("/idl/jobs/import") @router.post("/idl/jobs/import")
async def run_import_job_endpoint( async def run_import_job_endpoint(
request: ImportJobRequest, request: ImportJobRequest,
+22 -1
View File
@@ -18,6 +18,8 @@ from ..models import (
RadarData, RadarData,
TimeseriesStackPlan, TimeseriesStackPlan,
TimeseriesStackPlanDetail, TimeseriesStackPlanDetail,
TimeseriesStackPlanEdge,
TimeseriesStackPlanEdgeORM,
TimeseriesStackPlanItem, TimeseriesStackPlanItem,
TimeseriesStackPlanItemORM, TimeseriesStackPlanItemORM,
TimeseriesStackPlanORM, TimeseriesStackPlanORM,
@@ -89,11 +91,21 @@ def get_pairing_request_from_form(
def get_ps_request_from_form( def get_ps_request_from_form(
initial_overlap_threshold: float = Form(0.3), initial_overlap_threshold: float = Form(0.3),
final_overlap_threshold: float = Form(0.95) final_overlap_threshold: float = Form(0.95),
time_baseline_min: int = Form(1),
time_baseline_max: int = Form(90),
spatial_baseline_max_meters: int = Form(3000),
network_overlap_threshold: float = Form(0.5),
num_connections: int = Form(1),
) -> PsRequest: ) -> PsRequest:
return PsRequest( return PsRequest(
initial_overlap_threshold=initial_overlap_threshold, initial_overlap_threshold=initial_overlap_threshold,
final_overlap_threshold=final_overlap_threshold, final_overlap_threshold=final_overlap_threshold,
time_baseline_min=time_baseline_min,
time_baseline_max=time_baseline_max,
spatial_baseline_max_meters=spatial_baseline_max_meters,
network_overlap_threshold=network_overlap_threshold,
num_connections=num_connections,
) )
@@ -205,11 +217,20 @@ async def get_timeseries_stack_plan_endpoint(
.where(TimeseriesStackPlanItemORM.plan_ref_id == plan.id) .where(TimeseriesStackPlanItemORM.plan_ref_id == plan.id)
.order_by(TimeseriesStackPlanItemORM.scene_rank.asc(), TimeseriesStackPlanItemORM.id.asc()) .order_by(TimeseriesStackPlanItemORM.scene_rank.asc(), TimeseriesStackPlanItemORM.id.asc())
) )
edges_result = await db.execute(
select(TimeseriesStackPlanEdgeORM)
.where(TimeseriesStackPlanEdgeORM.plan_ref_id == plan.id)
.order_by(TimeseriesStackPlanEdgeORM.edge_rank.asc(), TimeseriesStackPlanEdgeORM.id.asc())
)
payload = TimeseriesStackPlan.model_validate(plan).model_dump() payload = TimeseriesStackPlan.model_validate(plan).model_dump()
payload["items"] = [ payload["items"] = [
TimeseriesStackPlanItem.model_validate(item) TimeseriesStackPlanItem.model_validate(item)
for item in items_result.scalars().all() for item in items_result.scalars().all()
] ]
payload["edges"] = [
TimeseriesStackPlanEdge.model_validate(edge)
for edge in edges_result.scalars().all()
]
return TimeseriesStackPlanDetail.model_validate(payload) return TimeseriesStackPlanDetail.model_validate(payload)
+49 -4
View File
@@ -25,6 +25,7 @@ from ..models import (
PsTaskItemORM, PsTaskItemORM,
RadarData, RadarData,
RadarPair, RadarPair,
TimeseriesStackPlanEdgeORM,
TimeseriesStackPlanItemORM, TimeseriesStackPlanItemORM,
TimeseriesStackPlanORM, TimeseriesStackPlanORM,
) )
@@ -144,6 +145,7 @@ def _normalize_lookup_key(value: Optional[str]) -> str:
def _build_plan_context( def _build_plan_context(
plan: TimeseriesStackPlanORM, plan: TimeseriesStackPlanORM,
plan_items: List[TimeseriesStackPlanItemORM], plan_items: List[TimeseriesStackPlanItemORM],
plan_edges: Optional[List[TimeseriesStackPlanEdgeORM]] = None,
) -> Dict[str, Any]: ) -> Dict[str, Any]:
request_params = plan.request_params_json if isinstance(plan.request_params_json, dict) else {} request_params = plan.request_params_json if isinstance(plan.request_params_json, dict) else {}
ordered_items = sorted( ordered_items = sorted(
@@ -164,6 +166,10 @@ def _build_plan_context(
} }
for item in ordered_items for item in ordered_items
] ]
ordered_edges = sorted(
list(plan_edges or []),
key=lambda item: (int(item.edge_rank or 0), int(item.id or 0)),
)
return { return {
"source": "timeseries_stack_plan", "source": "timeseries_stack_plan",
"plan_id": plan.plan_id, "plan_id": plan.plan_id,
@@ -176,12 +182,41 @@ def _build_plan_context(
"aoi_summary": plan.aoi_summary_json if isinstance(plan.aoi_summary_json, dict) else None, "aoi_summary": plan.aoi_summary_json if isinstance(plan.aoi_summary_json, dict) else None,
"initial_overlap_threshold": request_params.get("initial_overlap_threshold"), "initial_overlap_threshold": request_params.get("initial_overlap_threshold"),
"final_overlap_threshold": request_params.get("final_overlap_threshold"), "final_overlap_threshold": request_params.get("final_overlap_threshold"),
"time_baseline_min": request_params.get("time_baseline_min"),
"time_baseline_max": request_params.get("time_baseline_max"),
"spatial_baseline_max_meters": request_params.get("spatial_baseline_max_meters"),
"network_overlap_threshold": request_params.get("network_overlap_threshold"),
"num_connections": request_params.get("num_connections"),
"network_edge_count": len(ordered_edges),
"stack_dates": [ "stack_dates": [
str(item.imaging_date).strip() str(item.imaging_date).strip()
for item in ordered_items for item in ordered_items
if str(item.imaging_date or "").strip() if str(item.imaging_date or "").strip()
], ],
"scenes": scenes, "scenes": scenes,
"network_edges": [
{
"edge_id": item.id,
"edge_rank": item.edge_rank,
"master_plan_item_ref_id": item.master_plan_item_ref_id,
"slave_plan_item_ref_id": item.slave_plan_item_ref_id,
"metric_cache_ref_id": item.metric_cache_ref_id,
"master_scene_ref_id": item.master_scene_ref_id,
"slave_scene_ref_id": item.slave_scene_ref_id,
"master_imaging_date": item.master_imaging_date,
"slave_imaging_date": item.slave_imaging_date,
"temporal_baseline_days": item.temporal_baseline_days,
"spatial_baseline_meters": item.spatial_baseline_meters,
"perpendicular_baseline_meters": item.perpendicular_baseline_meters,
"scene_overlap_ratio": item.scene_overlap_ratio,
"pair_aoi_overlap_ratio": item.pair_aoi_overlap_ratio,
"selection_reason": item.selection_reason,
"selection_score": item.selection_score,
"enabled": bool(item.enabled),
"selection_meta": item.selection_meta_json if isinstance(item.selection_meta_json, dict) else None,
}
for item in ordered_edges
],
} }
@@ -383,6 +418,7 @@ async def create_ps_batch_endpoint(
effective_plan_id = explicit_plan_id or (inferred_plan_ids[0] if inferred_plan_ids else None) effective_plan_id = explicit_plan_id or (inferred_plan_ids[0] if inferred_plan_ids else None)
plan: Optional[TimeseriesStackPlanORM] = None plan: Optional[TimeseriesStackPlanORM] = None
plan_items: List[TimeseriesStackPlanItemORM] = [] plan_items: List[TimeseriesStackPlanItemORM] = []
plan_edges: List[TimeseriesStackPlanEdgeORM] = []
plan_item_by_id: Dict[int, TimeseriesStackPlanItemORM] = {} plan_item_by_id: Dict[int, TimeseriesStackPlanItemORM] = {}
plan_item_by_scene_id: Dict[int, TimeseriesStackPlanItemORM] = {} plan_item_by_scene_id: Dict[int, TimeseriesStackPlanItemORM] = {}
plan_item_by_path: Dict[str, TimeseriesStackPlanItemORM] = {} plan_item_by_path: Dict[str, TimeseriesStackPlanItemORM] = {}
@@ -408,6 +444,12 @@ async def create_ps_batch_endpoint(
.order_by(TimeseriesStackPlanItemORM.scene_rank.asc(), TimeseriesStackPlanItemORM.id.asc()) .order_by(TimeseriesStackPlanItemORM.scene_rank.asc(), TimeseriesStackPlanItemORM.id.asc())
) )
plan_items = items_result.scalars().all() plan_items = items_result.scalars().all()
edges_result = await db.execute(
select(TimeseriesStackPlanEdgeORM)
.where(TimeseriesStackPlanEdgeORM.plan_ref_id == plan.id)
.order_by(TimeseriesStackPlanEdgeORM.edge_rank.asc(), TimeseriesStackPlanEdgeORM.id.asc())
)
plan_edges = edges_result.scalars().all()
plan_item_by_id = {int(item.id): item for item in plan_items if item.id is not None} plan_item_by_id = {int(item.id): item for item in plan_items if item.id is not None}
plan_item_by_scene_id = { plan_item_by_scene_id = {
int(item.radar_data_ref_id): item int(item.radar_data_ref_id): item
@@ -419,15 +461,18 @@ async def create_ps_batch_endpoint(
for item in plan_items for item in plan_items
if _normalize_lookup_key(item.file_path) if _normalize_lookup_key(item.file_path)
} }
plan_context = _build_plan_context(plan, plan_items, plan_edges)
if not planning_context: if not planning_context:
planning_context = _build_plan_context(plan, plan_items) planning_context = plan_context
else: else:
merged_context = { merged_context = {
**_build_plan_context(plan, plan_items), **plan_context,
**planning_context, **planning_context,
} }
if "scenes" not in planning_context: if "scenes" not in planning_context:
merged_context["scenes"] = _build_plan_context(plan, plan_items).get("scenes") or [] merged_context["scenes"] = plan_context.get("scenes") or []
if "network_edges" not in planning_context:
merged_context["network_edges"] = plan_context.get("network_edges") or []
planning_context = merged_context planning_context = merged_context
batch_id = str(uuid.uuid4()) batch_id = str(uuid.uuid4())
@@ -466,7 +511,7 @@ async def create_ps_batch_endpoint(
planning_summary = { planning_summary = {
key: value key: value
for key, value in planning_context.items() for key, value in planning_context.items()
if key != "scenes" if key not in {"scenes", "network_edges"}
} }
remark_payload = { remark_payload = {
**planning_summary, **planning_summary,
@@ -20,6 +20,8 @@ class TimeseriesRunCreateRequest(BaseModel):
run_name: Optional[str] = Field(default=None, max_length=255) run_name: Optional[str] = Field(default=None, max_length=255)
reference_date: Optional[str] = Field(default=None, pattern=r"^\d{8}$|^$") reference_date: Optional[str] = Field(default=None, pattern=r"^\d{8}$|^$")
water_mask_mode: str = Field(default="synthetic_fallback", max_length=64) water_mask_mode: str = Field(default="synthetic_fallback", max_length=64)
processor_code: str = Field(default="isce2_stack_mintpy", max_length=64)
execution_mode: Optional[str] = Field(default=None, max_length=32)
notes: Optional[str] = Field(default=None, max_length=1000) notes: Optional[str] = Field(default=None, max_length=1000)
@field_validator("batch_id", mode="before") @field_validator("batch_id", mode="before")
@@ -58,6 +60,21 @@ class TimeseriesPreflightRequest(BaseModel):
return text return text
class SarscapeSbasPreflightRequest(BaseModel):
batch_id: str = Field(..., description="PS batch id")
reference_date: Optional[str] = Field(default=None, pattern=r"^\d{8}$|^$")
include_task_discovery: bool = True
discovery_timeout_seconds: int = Field(default=120, ge=10, le=600)
@field_validator("batch_id", mode="before")
@classmethod
def _validate_batch_id(cls, value: str) -> str:
text = str(value or "").strip()
if not text:
raise ValueError("batch_id is required")
return text
class TimeseriesRetryStepRequest(BaseModel): class TimeseriesRetryStepRequest(BaseModel):
step_id: str = Field(..., max_length=128) step_id: str = Field(..., max_length=128)
@@ -82,6 +99,8 @@ async def create_timeseries_run(
run_name=request.run_name, run_name=request.run_name,
reference_date=request.reference_date, reference_date=request.reference_date,
water_mask_mode=request.water_mask_mode, water_mask_mode=request.water_mask_mode,
processor_code=request.processor_code,
execution_mode=request.execution_mode,
notes=request.notes, notes=request.notes,
created_by=getattr(current_user, "username", None), created_by=getattr(current_user, "username", None),
db=db, db=db,
@@ -125,6 +144,25 @@ async def run_timeseries_preflight(
raise HTTPException(status_code=400, detail=str(exc)) from exc raise HTTPException(status_code=400, detail=str(exc)) from exc
@router.post("/sarscape-sbas/preflight")
async def run_sarscape_sbas_preflight(
request: SarscapeSbasPreflightRequest,
current_user: AuthUserORM = Depends(_require_admin),
db: AsyncSession = Depends(get_db),
):
_ = current_user
try:
return await timeseries_service.get_sarscape_sbas_preflight_report(
batch_id=request.batch_id,
reference_date=request.reference_date,
include_task_discovery=request.include_task_discovery,
discovery_timeout_seconds=request.discovery_timeout_seconds,
db=db,
)
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@router.get("/runs") @router.get("/runs")
async def list_timeseries_runs( async def list_timeseries_runs(
limit: int = 50, limit: int = 50,
@@ -149,6 +187,19 @@ async def get_timeseries_run_detail(
return detail return detail
@router.get("/runs/{run_id}/prepared-stack")
async def get_timeseries_run_prepared_stack(
run_id: str,
current_user: AuthUserORM = Depends(_get_current_user),
db: AsyncSession = Depends(get_db),
):
_ = current_user
summary = await timeseries_service.get_prepared_stack_summary(db, run_id=run_id)
if summary is None:
raise HTTPException(status_code=404, detail="Timeseries run not found")
return summary
@router.post("/runs/{run_id}/retry-step", status_code=202) @router.post("/runs/{run_id}/retry-step", status_code=202)
async def retry_timeseries_run_step( async def retry_timeseries_run_step(
run_id: str, run_id: str,
+20 -2
View File
@@ -25,9 +25,12 @@ def _parse_args() -> argparse.Namespace:
) )
parser.add_argument( parser.add_argument(
"--workflow", "--workflow",
required=True, required=False,
choices=["import", "dinsar", "dinsar_custom"], choices=["import", "dinsar", "dinsar_custom"],
) )
parser.add_argument("--inspect-sarscape-sbas", action="store_true")
parser.add_argument("--include-parameters", action="store_true")
parser.add_argument("--task-name", action="append", default=[])
parser.add_argument("--root-dir", required=False) parser.add_argument("--root-dir", required=False)
parser.add_argument("--task-dir", required=False) parser.add_argument("--task-dir", required=False)
parser.add_argument("--output-dir", required=False) parser.add_argument("--output-dir", required=False)
@@ -44,7 +47,22 @@ def main() -> int:
ensure_project_env_loaded() ensure_project_env_loaded()
args = _parse_args() args = _parse_args()
try: try:
from .envi_service import run_single_task_workflow, run_workflow from .envi_service import (
inspect_sarscape_sbas_tasks,
run_single_task_workflow,
run_workflow,
)
if args.inspect_sarscape_sbas:
record = inspect_sarscape_sbas_tasks(
args.task_name or None,
include_parameters=bool(args.include_parameters),
)
print(json.dumps(record, ensure_ascii=False))
return 0 if record.get("ok") else 2
if not args.workflow:
raise ValueError("--workflow is required unless --inspect-sarscape-sbas is used.")
if args.task_dir: if args.task_dir:
if not args.output_dir: if not args.output_dir:
+558 -20
View File
@@ -13,6 +13,7 @@ import subprocess
import sys import sys
import time import time
import defusedxml.ElementTree as ET import defusedxml.ElementTree as ET
from contextlib import contextmanager
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError
from datetime import datetime from datetime import datetime
from glob import glob from glob import glob
@@ -67,13 +68,43 @@ def get_envi_runner_python() -> str:
return os.path.normpath(sys.executable) return os.path.normpath(sys.executable)
def get_envi_taskengine_cwd() -> str:
"""Dedicated cwd for envipyengine/taskengine temp files.
SARscape can create zero-byte env_*.xyz and IDL*.tmp files in the current
working directory. Keep those files under runtime instead of the repo root.
"""
base_dir = _to_local_path(
getattr(settings, "IDL_WORKER_RUNTIME_DIR", "")
or os.path.join(_BACKEND_DIR, "runtime", "idl_worker")
)
cwd = os.path.join(base_dir, "envi_cwd")
os.makedirs(cwd, exist_ok=True)
return os.path.normpath(os.path.abspath(cwd))
def get_envi_custom_code_dir() -> str:
envi_root = _envi_install_root()
if not envi_root:
return ""
candidates = [
os.path.join(envi_root, "user_custom_code"),
os.path.join(envi_root, "custom_code"),
]
for path in candidates:
if os.path.isdir(path):
return os.path.normpath(os.path.abspath(path))
return ""
def get_envi_runner_cwd() -> str: def get_envi_runner_cwd() -> str:
return os.path.normpath(os.path.abspath(type(settings).PROJECT_ROOT)) return get_envi_taskengine_cwd()
def get_envi_runner_env() -> Dict[str, str]: def get_envi_runner_env() -> Dict[str, str]:
env = os.environ.copy() env = os.environ.copy()
project_root = get_envi_runner_cwd() project_root = os.path.normpath(os.path.abspath(type(settings).PROJECT_ROOT))
taskengine_cwd = get_envi_taskengine_cwd()
existing = [part for part in str(env.get("PYTHONPATH") or "").split(os.pathsep) if str(part).strip()] existing = [part for part in str(env.get("PYTHONPATH") or "").split(os.pathsep) if str(part).strip()]
ordered = [project_root, *existing] ordered = [project_root, *existing]
deduped: List[str] = [] deduped: List[str] = []
@@ -88,9 +119,36 @@ def get_envi_runner_env() -> Dict[str, str]:
seen.add(key) seen.add(key)
deduped.append(str(raw_path)) deduped.append(str(raw_path))
env["PYTHONPATH"] = os.pathsep.join(deduped) env["PYTHONPATH"] = os.pathsep.join(deduped)
env["TEMP"] = taskengine_cwd
env["TMP"] = taskengine_cwd
env["IDL_TMPDIR"] = taskengine_cwd
custom_code_dir = get_envi_custom_code_dir()
if custom_code_dir:
env["ENVI_CUSTOM_CODE"] = custom_code_dir
return env return env
@contextmanager
def _envi_taskengine_runtime_context():
"""Run in-process ENVI calls from the dedicated runtime cwd."""
target_cwd = get_envi_taskengine_cwd()
old_cwd = os.getcwd()
old_env = {name: os.environ.get(name) for name in ("TEMP", "TMP", "IDL_TMPDIR")}
os.environ["TEMP"] = target_cwd
os.environ["TMP"] = target_cwd
os.environ["IDL_TMPDIR"] = target_cwd
try:
os.chdir(target_cwd)
yield target_cwd
finally:
os.chdir(old_cwd)
for name, value in old_env.items():
if value is None:
os.environ.pop(name, None)
else:
os.environ[name] = value
def build_envi_runner_command(*args: Any) -> List[str]: def build_envi_runner_command(*args: Any) -> List[str]:
command = [ command = [
get_envi_runner_python(), get_envi_runner_python(),
@@ -101,6 +159,79 @@ def build_envi_runner_command(*args: Any) -> List[str]:
return command return command
def _list_taskengine_pids() -> set[int]:
"""Return taskengine.exe PIDs on Windows without importing optional deps."""
if os.name != "nt":
return set()
try:
completed = subprocess.run(
[
"powershell.exe",
"-NoProfile",
"-Command",
"Get-Process taskengine -ErrorAction SilentlyContinue | ForEach-Object { $_.Id }",
],
capture_output=True,
text=True,
timeout=5,
check=False,
)
except Exception:
return set()
pids: set[int] = set()
for line in str(completed.stdout or "").splitlines():
raw = line.strip()
if raw.isdigit():
pids.add(int(raw))
return pids
def _stop_taskengine_pids(pids: set[int]) -> List[int]:
"""Stop specific taskengine.exe PIDs; avoids killing pre-existing sessions."""
stopped: List[int] = []
if os.name != "nt":
return stopped
for pid in sorted(pids):
try:
subprocess.run(
[
"powershell.exe",
"-NoProfile",
"-Command",
f"Stop-Process -Id {int(pid)} -Force -ErrorAction SilentlyContinue",
],
capture_output=True,
text=True,
timeout=10,
check=False,
)
stopped.append(int(pid))
except Exception:
continue
return stopped
def _cleanup_new_taskengine_processes(existing_pids: set[int]) -> Dict[str, Any]:
"""Best-effort cleanup for taskengine.exe children spawned by a timed-out runner."""
existing = set(existing_pids or set())
first_targets = _list_taskengine_pids() - existing
stopped = _stop_taskengine_pids(first_targets)
# taskengine can take a moment to detach from the runner process. Re-check once.
time.sleep(1)
second_targets = _list_taskengine_pids() - existing
stopped.extend(pid for pid in _stop_taskengine_pids(second_targets) if pid not in stopped)
time.sleep(1)
remaining = sorted(_list_taskengine_pids() - existing)
return {
"taskengine_cleanup_attempted": True,
"taskengine_stopped_pids": sorted(set(stopped)),
"taskengine_remaining_new_pids": remaining,
}
def probe_envi_runner() -> Dict[str, Any]: def probe_envi_runner() -> Dict[str, Any]:
python_path = get_envi_runner_python() python_path = get_envi_runner_python()
project_root = get_envi_runner_cwd() project_root = get_envi_runner_cwd()
@@ -238,6 +369,39 @@ import threading
_ENVI_GLOBAL_LOCK = threading.Lock() _ENVI_GLOBAL_LOCK = threading.Lock()
SARSCAPE_SBAS_NATIVE_WORKFLOW_CANDIDATES = [
"wf_sbas",
"wf_esbas",
]
SARSCAPE_SBAS_SUPPORT_TASK_CANDIDATES = [
"SARscape_setting_output_folders",
"SARsLoadPreferences",
"SARsImportSarSelector",
"SARscapeSuggestLooks",
"SARscapeEnviuriToShape",
]
SARSCAPE_SBAS_STACK_TASK_CANDIDATES = [
"SARsInSARStackSBASGenerateConnectionGraph",
"SARsInSARStackSBASInterferogramGeneration",
"SARsInSARStackSBASInversionStep1",
"SARsInSARStackSBASInversionStep2",
"SARsInSARStackSBASGeocode",
"SARsInSARStackSBASVariogram",
"SARsInSARStackESBASInterferogramGeneration",
"SARsInSARStackESBASInversion",
"SARsInSARStackESBASGeocode",
"SARsInSARConnectionGraphESBAS",
]
SARSCAPE_SBAS_TASK_CANDIDATES = [
*SARSCAPE_SBAS_NATIVE_WORKFLOW_CANDIDATES,
*SARSCAPE_SBAS_SUPPORT_TASK_CANDIDATES,
*SARSCAPE_SBAS_STACK_TASK_CANDIDATES,
]
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Progress file for subprocess ↔ job handler communication # Progress file for subprocess ↔ job handler communication
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -417,26 +581,32 @@ def execute_envi_task(task_name: str, parameters: Dict[str, Any]) -> Dict[str, A
) from exc ) from exc
with _ENVI_GLOBAL_LOCK: with _ENVI_GLOBAL_LOCK:
engine = Engine("ENVI") with _envi_taskengine_runtime_context():
task = engine.task(task_name) engine = Engine("ENVI")
task = engine.task(task_name)
existing_taskengine_pids = _list_taskengine_pids()
# Run with timeout to handle envipyengine hangs # Run with timeout to handle envipyengine hangs
with ThreadPoolExecutor(max_workers=1) as pool: with ThreadPoolExecutor(max_workers=1) as pool:
future = pool.submit(task.execute, parameters) future = pool.submit(task.execute, parameters)
try:
result = future.result(timeout=_ENVI_TASK_TIMEOUT)
except FuturesTimeoutError:
try: try:
import subprocess as _sp result = future.result(timeout=_ENVI_TASK_TIMEOUT)
_sp.run(["taskkill", "/F", "/IM", "taskengine.exe"], except FuturesTimeoutError:
capture_output=True, timeout=10) try:
print("[WARN] execute_envi_task: killed taskengine after timeout") cleanup = _cleanup_new_taskengine_processes(existing_taskengine_pids)
except Exception as _exc: stopped = cleanup.get("taskengine_stopped_pids") or []
print(f"[WARN] execute_envi_task: taskengine cleanup failed — {_exc}") remaining = cleanup.get("taskengine_remaining_new_pids") or []
raise RuntimeError( print(
f"Task {task_name} timed out after {_ENVI_TASK_TIMEOUT}s " "[WARN] execute_envi_task: task timed out; "
f"(envipyengine hung). Output files may still exist." f"stopped_new_taskengine_pids={stopped}; "
) f"remaining_new_taskengine_pids={remaining}"
)
except Exception as _exc:
print(f"[WARN] execute_envi_task: taskengine cleanup failed — {_exc}")
raise RuntimeError(
f"Task {task_name} timed out after {_ENVI_TASK_TIMEOUT}s "
f"(envipyengine hung). Output files may still exist."
)
# taskengine returns {"outputParameters": {...}, ...} # taskengine returns {"outputParameters": {...}, ...}
return result.get("outputParameters", result) return result.get("outputParameters", result)
@@ -453,6 +623,374 @@ def _unwrap_sarscapedata(value: Any) -> Any:
return value return value
def _configured_sarscape_sbas_task_candidates() -> List[str]:
configured = str(_read_env("SARSCAPE_SBAS_TASK_NAMES", "") or "").strip()
if not configured:
return list(SARSCAPE_SBAS_TASK_CANDIDATES)
names: List[str] = []
for raw in configured.replace(";", ",").split(","):
name = raw.strip()
if name and name not in names:
names.append(name)
return names or list(SARSCAPE_SBAS_TASK_CANDIDATES)
def _envi_install_root() -> str:
executable = _to_local_path(IDL_EXECUTABLE)
if not executable:
return ""
return os.path.abspath(os.path.join(os.path.dirname(executable), "..", "..", ".."))
def _static_envi_task_template_path(task_name: str) -> str:
name = str(task_name or "").strip()
if not name:
return ""
envi_root = _envi_install_root()
if not envi_root:
return ""
candidates = [
os.path.join(envi_root, "user_custom_code", f"{name}.task"),
os.path.join(envi_root, "resource", "templates", "tasks", "SARscape", f"{name}.task"),
os.path.join(envi_root, "resource", "templates", "tasks", f"{name}.task"),
]
for path in candidates:
if os.path.isfile(path):
return path
return ""
def _json_safe_parameter(value: Any) -> Any:
if isinstance(value, dict):
return {str(key): _json_safe_parameter(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return [_json_safe_parameter(item) for item in value]
if value is None or isinstance(value, (str, int, float, bool)):
return value
return str(value)
def _summarize_task_parameters(raw_parameters: Any) -> Dict[str, Any]:
safe_parameters = _json_safe_parameter(raw_parameters)
input_names: List[str] = []
output_names: List[str] = []
required_input_names: List[str] = []
if isinstance(safe_parameters, dict):
iterable = safe_parameters.values()
elif isinstance(safe_parameters, list):
iterable = safe_parameters
else:
iterable = []
for item in iterable:
if not isinstance(item, dict):
continue
name = str(item.get("name") or item.get("NAME") or "").strip()
direction = str(item.get("direction") or item.get("DIRECTION") or "").strip().lower()
required = bool(item.get("required") or item.get("REQUIRED"))
if not name:
continue
if direction == "input":
input_names.append(name)
if required:
required_input_names.append(name)
elif direction == "output":
output_names.append(name)
return {
"parameter_count": (
len(safe_parameters)
if isinstance(safe_parameters, (dict, list))
else 0
),
"input_names": input_names,
"required_input_names": required_input_names,
"output_names": output_names,
"parameters": safe_parameters,
}
def list_envi_tasks() -> Dict[str, Any]:
"""List ENVI task names without instantiating individual task parameters."""
result: Dict[str, Any] = {
"ok": False,
"engine": "envipyengine",
"task_count": 0,
"tasks": [],
"error": None,
}
try:
_ensure_envipyengine_config()
from envipyengine import Engine
except ImportError:
result["error"] = (
"envipyengine is not installed. Install it with: pip install envipyengine"
)
return result
with _ENVI_GLOBAL_LOCK:
with _envi_taskengine_runtime_context():
try:
names = Engine("ENVI").tasks()
except Exception as exc:
result["error"] = str(exc)
return result
result["tasks"] = [str(name) for name in names]
result["task_count"] = len(result["tasks"])
result["ok"] = True
return result
def discover_sarscape_sbas_tasks() -> Dict[str, Any]:
"""Discover installed SARscape SBAS/E-SBAS task names by filtering Engine.tasks()."""
report = list_envi_tasks()
task_names = list(report.get("tasks") or [])
keywords = (
"StackSBAS",
"StackESBAS",
"ConnectionGraphESBAS",
)
explicit_names = set(SARSCAPE_SBAS_NATIVE_WORKFLOW_CANDIDATES) | set(SARSCAPE_SBAS_SUPPORT_TASK_CANDIDATES)
matches = [
name
for name in task_names
if (
str(name) in explicit_names
or (
str(name).startswith("SARsInSAR")
and any(keyword.lower() in str(name).lower() for keyword in keywords)
)
)
]
static_task_files: Dict[str, str] = {}
for name in SARSCAPE_SBAS_TASK_CANDIDATES:
path = _static_envi_task_template_path(name)
if path:
static_task_files[name] = path
if name not in matches:
matches.append(name)
preferred_order = [
*SARSCAPE_SBAS_NATIVE_WORKFLOW_CANDIDATES,
*SARSCAPE_SBAS_SUPPORT_TASK_CANDIDATES,
"SARsInSARStackSBASGenerateConnectionGraph",
"SARsInSARStackSBASInterferogramGeneration",
"SARsInSARStackSBASInversionStep1",
"SARsInSARStackSBASInversionStep2",
"SARsInSARStackSBASGeocode",
"SARsInSARStackSBASVariogram",
"SARsInSARStackESBASInterferogramGeneration",
"SARsInSARStackESBASInversion",
"SARsInSARStackESBASGeocode",
"SARsInSARConnectionGraphESBAS",
]
ordered: List[str] = []
for name in preferred_order:
if name in matches and name not in ordered:
ordered.append(name)
for name in sorted(matches):
if name not in ordered:
ordered.append(name)
return {
"ok": bool(report.get("ok")) and bool(ordered),
"engine": report.get("engine"),
"task_count": int(report.get("task_count") or 0),
"sarscape_sbas_task_count": len(ordered),
"sarscape_sbas_tasks": ordered,
"static_task_files": {
name: static_task_files[name]
for name in ordered
if name in static_task_files
},
"error": report.get("error"),
}
def inspect_envi_tasks(task_names: List[str]) -> Dict[str, Any]:
"""Inspect ENVI/SARscape tasks without executing them."""
started_at = _utc_now_text()
deduped_names: List[str] = []
for raw_name in task_names:
name = str(raw_name or "").strip()
if name and name not in deduped_names:
deduped_names.append(name)
result: Dict[str, Any] = {
"ok": False,
"engine": "envipyengine",
"started_at": started_at,
"finished_at": None,
"task_count": len(deduped_names),
"available_count": 0,
"missing_count": 0,
"tasks": [],
"error": None,
}
if not deduped_names:
result["error"] = "No task names provided."
result["finished_at"] = _utc_now_text()
return result
try:
_ensure_envipyengine_config()
from envipyengine import Engine
except ImportError as exc:
result["error"] = (
"envipyengine is not installed. Install it with: pip install envipyengine"
)
result["finished_at"] = _utc_now_text()
return result
with _ENVI_GLOBAL_LOCK:
with _envi_taskengine_runtime_context():
try:
engine = Engine("ENVI")
except Exception as exc:
result["error"] = f"Failed to initialize ENVI engine: {exc}"
result["finished_at"] = _utc_now_text()
return result
for task_name in deduped_names:
item: Dict[str, Any] = {
"name": task_name,
"available": False,
"error": None,
"parameter_count": 0,
"input_names": [],
"required_input_names": [],
"output_names": [],
"parameters": [],
}
try:
task = engine.task(task_name)
summary = _summarize_task_parameters(getattr(task, "parameters", []))
item.update(summary)
item["available"] = True
except Exception as exc:
item["error"] = str(exc)
result["tasks"].append(item)
result["available_count"] = sum(1 for item in result["tasks"] if item.get("available"))
result["missing_count"] = sum(1 for item in result["tasks"] if not item.get("available"))
result["ok"] = result["available_count"] > 0
result["finished_at"] = _utc_now_text()
return result
def inspect_sarscape_sbas_tasks(
task_names: Optional[List[str]] = None,
*,
include_parameters: bool = False,
) -> Dict[str, Any]:
"""Inspect likely SARscape SBAS/E-SBAS task names for the installed version."""
status = get_status()
discovery = discover_sarscape_sbas_tasks()
names = task_names or list(discovery.get("sarscape_sbas_tasks") or _configured_sarscape_sbas_task_candidates())
if include_parameters:
task_report = inspect_envi_tasks(names)
else:
discovered_set = set(discovery.get("sarscape_sbas_tasks") or [])
task_report = {
"ok": bool(discovery.get("ok")),
"engine": "envipyengine",
"task_count": len(names),
"available_count": sum(1 for name in names if name in discovered_set),
"missing_count": sum(1 for name in names if name not in discovered_set),
"tasks": [
{
"name": name,
"available": name in discovered_set,
"error": None if name in discovered_set else "Task name not listed by Engine.tasks().",
"parameter_count": None,
"input_names": [],
"required_input_names": [],
"output_names": [],
"parameters": [],
}
for name in names
],
"error": discovery.get("error"),
}
task_report["status"] = {
"idl_installed": status.get("idl_installed"),
"idl_executable": status.get("idl_executable"),
"runner_ready": status.get("runner_ready"),
"runner_python": status.get("runner_python"),
"runner_message": status.get("runner_message"),
"dem_base_file": status.get("dem_base_file"),
"dem_exists": status.get("dem_exists"),
}
task_report["candidate_source"] = (
"SARSCAPE_SBAS_TASK_NAMES"
if str(_read_env("SARSCAPE_SBAS_TASK_NAMES", "") or "").strip()
else "engine_task_list"
)
task_report["include_parameters"] = bool(include_parameters)
task_report["discovery"] = discovery
task_report["ready_for_pipeline_design"] = bool(task_report.get("ok"))
return task_report
def inspect_sarscape_sbas_tasks_subprocess(
task_names: Optional[List[str]] = None,
*,
timeout_seconds: int = 120,
include_parameters: bool = False,
) -> Dict[str, Any]:
"""Run SARscape SBAS task inspection through the isolated ENVI runner."""
command = build_envi_runner_command("--inspect-sarscape-sbas")
if include_parameters:
command.append("--include-parameters")
for name in task_names or []:
if str(name or "").strip():
command.extend(["--task-name", str(name).strip()])
existing_taskengine_pids = _list_taskengine_pids()
try:
completed = subprocess.run(
command,
cwd=get_envi_runner_cwd(),
env=get_envi_runner_env(),
capture_output=True,
text=True,
timeout=max(10, int(timeout_seconds or 120)),
check=False,
)
except subprocess.TimeoutExpired as exc:
cleanup = _cleanup_new_taskengine_processes(existing_taskengine_pids)
stdout_text = str(exc.stdout or "").strip()
stderr_text = str(exc.stderr or "").strip()
return {
"ok": False,
"returncode": None,
"timeout": True,
"timeout_seconds": max(10, int(timeout_seconds or 120)),
"stdout": stdout_text[:2000],
"stderr": stderr_text[:2000],
"error": (
"SARscape SBAS task inspection timed out. "
"Use lightweight discovery without include_parameters, or provide a manually verified task template."
),
"runner_command": command,
**cleanup,
}
stdout_text = str(completed.stdout or "").strip()
stderr_text = str(completed.stderr or "").strip()
try:
payload = json.loads(stdout_text) if stdout_text else {}
except Exception:
payload = {}
payload.setdefault("returncode", int(completed.returncode))
payload.setdefault("stdout", stdout_text[:2000])
payload.setdefault("stderr", stderr_text[:2000])
payload["runner_command"] = command
if completed.returncode != 0:
payload["ok"] = False
payload.setdefault("error", stderr_text or stdout_text or f"returncode={completed.returncode}")
return payload
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Import workflow # Import workflow
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
+71
View File
@@ -43,6 +43,8 @@ from .timeseries_service import (
JOB_TYPE_TIMESERIES_REGISTER_PRODUCT, JOB_TYPE_TIMESERIES_REGISTER_PRODUCT,
JOB_TYPE_TIMESERIES_RUN_ISCE2_STACK, JOB_TYPE_TIMESERIES_RUN_ISCE2_STACK,
JOB_TYPE_TIMESERIES_RUN_MINTPY_SBAS, JOB_TYPE_TIMESERIES_RUN_MINTPY_SBAS,
JOB_TYPE_TIMESERIES_RUN_SARSCAPE_SBAS,
JOB_TYPE_TIMESERIES_SARSCAPE_PREFLIGHT,
JOB_TYPE_TIMESERIES_STACK_PREP, JOB_TYPE_TIMESERIES_STACK_PREP,
JOB_TYPE_TIMESERIES_EXPORT_PUBLISH, JOB_TYPE_TIMESERIES_EXPORT_PUBLISH,
timeseries_service, timeseries_service,
@@ -3550,6 +3552,73 @@ async def _handle_timeseries_run_mintpy_sbas(job: SystemJobORM) -> None:
raise raise
async def _handle_timeseries_sarscape_preflight(job: SystemJobORM) -> None:
if not job.task_id:
raise ValueError("TIMESERIES_SARSCAPE_PREFLIGHT requires task_id for progress tracking.")
payload = job.payload or {}
run_id = str(payload.get("run_id") or "").strip()
if not run_id:
raise ValueError("TIMESERIES_SARSCAPE_PREFLIGHT requires run_id payload.")
async with AsyncSessionLocal() as db:
try:
await task_service.update_task(
job.task_id,
progress=45,
message="Building SARscape SBAS processor manifest...",
db=db,
)
result = await timeseries_service.build_sarscape_processor_preflight(run_id, db=db)
ready_text = "ready" if result.get("ready_for_execution") else "planning_only"
is_preflight_only = str(result.get("execution_mode") or "").strip() == "preflight_only"
await task_service.update_task(
job.task_id,
status="COMPLETED" if is_preflight_only else None,
progress=100 if is_preflight_only else 55,
message=(
f"SARscape SBAS preflight complete: state={ready_text} "
f"manifest={result.get('processor_manifest_path')}"
),
db=db,
)
except Exception as exc:
await timeseries_service.mark_run_failed(run_id, str(exc), db=db)
raise
async def _handle_timeseries_run_sarscape_sbas(job: SystemJobORM) -> None:
if not job.task_id:
raise ValueError("TIMESERIES_RUN_SARSCAPE_SBAS requires task_id for progress tracking.")
payload = job.payload or {}
run_id = str(payload.get("run_id") or "").strip()
if not run_id:
raise ValueError("TIMESERIES_RUN_SARSCAPE_SBAS requires run_id payload.")
async with AsyncSessionLocal() as db:
try:
await task_service.update_task(
job.task_id,
progress=90,
message="Running SARscape SBAS pipeline...",
db=db,
)
async with engine_lock_service.acquire("sarscape_sbas_timeseries"):
result = await timeseries_service.run_sarscape_sbas(run_id, db=db)
await task_service.update_task(
job.task_id,
status="COMPLETED",
progress=100,
message=(
f"SARscape SBAS complete: tasks={result.get('task_count', 0)} "
f"report={result.get('report_path')}"
),
db=db,
)
except Exception as exc:
await timeseries_service.mark_run_failed(run_id, str(exc), db=db)
raise
async def _handle_timeseries_export_publish(job: SystemJobORM) -> None: async def _handle_timeseries_export_publish(job: SystemJobORM) -> None:
if not job.task_id: if not job.task_id:
raise ValueError("TIMESERIES_EXPORT_PUBLISH requires task_id for progress tracking.") raise ValueError("TIMESERIES_EXPORT_PUBLISH requires task_id for progress tracking.")
@@ -3648,6 +3717,8 @@ _HANDLERS = {
JOB_TYPE_TIMESERIES_MATERIALIZE: _handle_timeseries_materialize, JOB_TYPE_TIMESERIES_MATERIALIZE: _handle_timeseries_materialize,
JOB_TYPE_TIMESERIES_RUN_ISCE2_STACK: _handle_timeseries_run_isce2_stack, JOB_TYPE_TIMESERIES_RUN_ISCE2_STACK: _handle_timeseries_run_isce2_stack,
JOB_TYPE_TIMESERIES_RUN_MINTPY_SBAS: _handle_timeseries_run_mintpy_sbas, JOB_TYPE_TIMESERIES_RUN_MINTPY_SBAS: _handle_timeseries_run_mintpy_sbas,
JOB_TYPE_TIMESERIES_SARSCAPE_PREFLIGHT: _handle_timeseries_sarscape_preflight,
JOB_TYPE_TIMESERIES_RUN_SARSCAPE_SBAS: _handle_timeseries_run_sarscape_sbas,
JOB_TYPE_TIMESERIES_EXPORT_PUBLISH: _handle_timeseries_export_publish, JOB_TYPE_TIMESERIES_EXPORT_PUBLISH: _handle_timeseries_export_publish,
JOB_TYPE_TIMESERIES_REGISTER_PRODUCT: _handle_timeseries_register_product, JOB_TYPE_TIMESERIES_REGISTER_PRODUCT: _handle_timeseries_register_product,
JOB_TYPE_REBUILD_PSINSAR_CATALOG: _handle_rebuild_psinsar_catalog, JOB_TYPE_REBUILD_PSINSAR_CATALOG: _handle_rebuild_psinsar_catalog,
+7 -1
View File
@@ -17,7 +17,13 @@ from .. import database
from ..config import settings from ..config import settings
from ..models import SystemWorkerHeartbeatORM from ..models import SystemWorkerHeartbeatORM
IDL_JOB_TYPES = {"IDL_RUN_IMPORT", "IDL_RUN_DINSAR", "WATER_GEOCODE", "WATER_FLOOD"} IDL_JOB_TYPES = {
"IDL_RUN_IMPORT",
"IDL_RUN_DINSAR",
"WATER_GEOCODE",
"WATER_FLOOD",
"TIMESERIES_RUN_SARSCAPE_SBAS",
}
def _default_worker_id() -> str: def _default_worker_id() -> str:
@@ -0,0 +1,681 @@
from __future__ import annotations
import hashlib
import json
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional
from ..config import settings
from . import envi_service
PROCESSOR_CODE = "sarscape_sbas"
ENGINE_CODE = "sarscape"
PREPARED_STACK_SCHEMA = "insar.prepared-sbas-stack/v1"
NATIVE_WORKFLOW_TASK = "wf_sbas"
NATIVE_ESBAS_WORKFLOW_TASK = "wf_esbas"
TEMPLATE_STRATEGY_NATIVE = "native_workflow_metatask"
TEMPLATE_STRATEGY_EXPLICIT = "explicit_stack_tasks"
SUPPORTED_TEMPLATE_STRATEGIES = {
TEMPLATE_STRATEGY_NATIVE,
TEMPLATE_STRATEGY_EXPLICIT,
}
REQUIRED_STACK_TASKS = [
"SARsInSARStackSBASGenerateConnectionGraph",
"SARsInSARStackSBASInterferogramGeneration",
"SARsInSARStackSBASInversionStep1",
"SARsInSARStackSBASInversionStep2",
"SARsInSARStackSBASGeocode",
]
REQUIRED_TASKS = REQUIRED_STACK_TASKS
OPTIONAL_TASKS = [
NATIVE_WORKFLOW_TASK,
NATIVE_ESBAS_WORKFLOW_TASK,
"SARscape_setting_output_folders",
"SARsLoadPreferences",
"SARsImportSarSelector",
"SARscapeSuggestLooks",
"SARscapeEnviuriToShape",
"SARsInSARStackSBASVariogram",
"SARsInSARStackESBASInterferogramGeneration",
"SARsInSARStackESBASInversion",
"SARsInSARStackESBASGeocode",
"SARsInSARConnectionGraphESBAS",
]
PIPELINE_PHASES = [
{
"phase_id": "connection_graph",
"task_name": "SARsInSARStackSBASGenerateConnectionGraph",
"purpose": "Build or ingest the SBAS connection graph.",
},
{
"phase_id": "interferogram_generation",
"task_name": "SARsInSARStackSBASInterferogramGeneration",
"purpose": "Generate interferograms for the selected SBAS graph.",
},
{
"phase_id": "inversion_step1",
"task_name": "SARsInSARStackSBASInversionStep1",
"purpose": "Run SARscape SBAS inversion step 1.",
},
{
"phase_id": "inversion_step2",
"task_name": "SARsInSARStackSBASInversionStep2",
"purpose": "Run SARscape SBAS inversion step 2.",
},
{
"phase_id": "geocode_export",
"task_name": "SARsInSARStackSBASGeocode",
"purpose": "Geocode velocity, displacement, and quality outputs.",
},
{
"phase_id": "variogram_optional",
"task_name": "SARsInSARStackSBASVariogram",
"purpose": "Optional SARscape variogram/quality analysis.",
"optional": True,
},
]
REQUIRED_RESULT_ROLES = [
"stack_manifest",
"processor_manifest",
"selected_network_edges",
"velocity_product",
"timeseries_product",
"temporal_coherence",
"geocoded_raster",
"preview_png",
"logs",
]
def default_parameter_template_path() -> str:
configured = str(getattr(settings, "SARSCAPE_SBAS_PARAMETER_TEMPLATE_PATH", "") or "").strip()
if configured:
return configured
return str(
Path(__file__).resolve().parents[2]
/ "templates"
/ "sarscape_sbas_parameter_template.example.json"
)
def _utcnow_iso() -> str:
return datetime.utcnow().replace(microsecond=0).isoformat() + "Z"
def _canonical_json(payload: Dict[str, Any]) -> str:
return json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
def _sha256_payload(payload: Dict[str, Any]) -> str:
return hashlib.sha256(_canonical_json(payload).encode("utf-8")).hexdigest()
def _available_task_names(discovery_report: Optional[Dict[str, Any]]) -> set[str]:
if not isinstance(discovery_report, dict):
return set()
names: set[str] = set()
for item in discovery_report.get("tasks") or []:
if isinstance(item, dict) and bool(item.get("available")):
name = str(item.get("name") or "").strip()
if name:
names.add(name)
discovered = discovery_report.get("discovery") or {}
for name in discovered.get("sarscape_sbas_tasks") or []:
text = str(name or "").strip()
if text:
names.add(text)
return names
def _numeric_values(items: List[Dict[str, Any]], key: str) -> List[float]:
values: List[float] = []
for item in items:
try:
if item.get(key) is not None:
values.append(float(item.get(key)))
except Exception:
continue
return values
def load_parameter_template(parameter_template_path: Optional[str] = None) -> Dict[str, Any]:
path = str(parameter_template_path or default_parameter_template_path() or "").strip()
result: Dict[str, Any] = {
"path": path or None,
"exists": False,
"readable": False,
"schema": None,
"validated": False,
"execution_strategy": TEMPLATE_STRATEGY_NATIVE,
"native_workflow_task": None,
"task_count": 0,
"missing_required_tasks": list(REQUIRED_STACK_TASKS),
"tasks_without_parameters": [],
"errors": [],
"template": None,
}
if not path:
result["errors"].append("SARscape SBAS parameter template path is empty.")
return result
template_file = Path(path)
result["exists"] = template_file.is_file()
if not template_file.is_file():
result["errors"].append(f"SARscape SBAS parameter template not found: {path}")
return result
try:
payload = json.loads(template_file.read_text(encoding="utf-8"))
except Exception as exc:
result["errors"].append(f"Failed to read SARscape SBAS parameter template: {exc}")
return result
if not isinstance(payload, dict):
result["errors"].append("SARscape SBAS parameter template must be a JSON object.")
return result
raw_strategy = str(payload.get("execution_strategy") or TEMPLATE_STRATEGY_NATIVE).strip()
execution_strategy = (
raw_strategy if raw_strategy in SUPPORTED_TEMPLATE_STRATEGIES else TEMPLATE_STRATEGY_NATIVE
)
native_workflow = payload.get("native_workflow") if isinstance(payload.get("native_workflow"), dict) else {}
native_workflow_task = str(native_workflow.get("task_name") or NATIVE_WORKFLOW_TASK).strip()
native_workflow_parameters = native_workflow.get("parameters")
tasks = payload.get("tasks") if isinstance(payload.get("tasks"), list) else []
task_names = {
str(item.get("task_name") or "").strip()
for item in tasks
if isinstance(item, dict) and str(item.get("task_name") or "").strip()
}
missing_required = [name for name in REQUIRED_STACK_TASKS if name not in task_names]
tasks_without_parameters = [
str(item.get("task_name") or item.get("phase_id") or "<unnamed>")
for item in tasks
if isinstance(item, dict)
and bool(item.get("enabled", True))
and not isinstance(item.get("parameters"), dict)
]
result.update(
{
"readable": True,
"schema": payload.get("schema"),
"validated": bool(payload.get("validated")),
"execution_strategy": execution_strategy,
"native_workflow_task": native_workflow_task,
"task_count": len(tasks),
"missing_required_tasks": missing_required,
"tasks_without_parameters": tasks_without_parameters,
"template": payload,
}
)
if str(payload.get("schema") or "") != "insar.sarscape-sbas-template/v1":
result["errors"].append("Unsupported SARscape SBAS parameter template schema.")
if raw_strategy not in SUPPORTED_TEMPLATE_STRATEGIES:
result["errors"].append(
"Unsupported SARscape SBAS execution_strategy: " + (raw_strategy or "<empty>")
)
if execution_strategy == TEMPLATE_STRATEGY_NATIVE:
if not native_workflow_task:
result["errors"].append("Native SARscape workflow task name is empty.")
if not isinstance(native_workflow_parameters, dict):
result["errors"].append("Native SARscape workflow parameters must be a JSON object.")
if execution_strategy == TEMPLATE_STRATEGY_EXPLICIT and missing_required:
result["errors"].append("Template is missing required tasks: " + ", ".join(missing_required))
if execution_strategy == TEMPLATE_STRATEGY_EXPLICIT and tasks_without_parameters:
result["errors"].append("Template tasks without parameters object: " + ", ".join(tasks_without_parameters))
if not payload.get("validated"):
result["errors"].append("Template is not marked validated=true.")
return result
def summarize_network_edges(network_edges: List[Dict[str, Any]]) -> Dict[str, Any]:
enabled_edges = [item for item in network_edges if bool(item.get("enabled", True))]
temporal = _numeric_values(enabled_edges, "temporal_baseline_days")
spatial = _numeric_values(enabled_edges, "spatial_baseline_meters")
overlap = _numeric_values(enabled_edges, "pair_aoi_overlap_ratio")
return {
"edge_count": len(network_edges),
"enabled_edge_count": len(enabled_edges),
"temporal_baseline_days": {
"min": min(temporal) if temporal else None,
"max": max(temporal) if temporal else None,
},
"spatial_baseline_meters": {
"min": min(spatial) if spatial else None,
"max": max(spatial) if spatial else None,
},
"pair_aoi_overlap_ratio": {
"min": min(overlap) if overlap else None,
"max": max(overlap) if overlap else None,
},
}
def build_processor_manifest(
stack_manifest: Dict[str, Any],
*,
discovery_report: Optional[Dict[str, Any]] = None,
parameter_template_path: Optional[str] = None,
) -> Dict[str, Any]:
"""Build the SARscape SBAS processor contract without executing ENVI tasks."""
scenes = stack_manifest.get("scenes") if isinstance(stack_manifest.get("scenes"), list) else []
network_edges = (
stack_manifest.get("network_edges")
if isinstance(stack_manifest.get("network_edges"), list)
else []
)
template_status = load_parameter_template(parameter_template_path)
template = template_status.get("template") if isinstance(template_status.get("template"), dict) else {}
template_strategy = str(
template_status.get("execution_strategy") or TEMPLATE_STRATEGY_NATIVE
).strip()
native_workflow_task = str(
template_status.get("native_workflow_task") or NATIVE_WORKFLOW_TASK
).strip()
available_tasks = _available_task_names(discovery_report)
missing_stack_tasks = [name for name in REQUIRED_STACK_TASKS if name not in available_tasks]
missing_native_tasks = [native_workflow_task] if native_workflow_task not in available_tasks else []
missing_required_tasks = (
missing_native_tasks
if template_strategy == TEMPLATE_STRATEGY_NATIVE
else missing_stack_tasks
)
template_path = str(template_status.get("path") or "").strip()
template_exists = bool(template_status.get("exists"))
template_validated = bool(template_status.get("validated")) and not template_status.get("errors")
execution_enabled = bool(getattr(settings, "SARSCAPE_SBAS_ALLOW_EXECUTION", False))
native_workflow_available = not missing_native_tasks
explicit_stack_available = not missing_stack_tasks
blockers: List[str] = []
if len(scenes) < 3:
blockers.append("SARscape SBAS requires at least 3 stack scenes.")
if not network_edges:
blockers.append("No SBAS network_edges are present in the stack manifest.")
if missing_required_tasks:
blockers.append(
"Missing required SARscape SBAS tasks for "
f"{template_strategy}: " + ", ".join(missing_required_tasks)
)
if not template_exists:
blockers.append(
"SARscape SBAS parameter template is not configured. "
"Live task.parameters is intentionally not used because it can hang taskengine."
)
elif not template_validated:
blockers.extend(str(item) for item in (template_status.get("errors") or []))
if not execution_enabled:
blockers.append("SARSCAPE_SBAS_ALLOW_EXECUTION is false; SARscape SBAS production execution is disabled.")
parameter_template_state = (
"validated"
if template_validated
else ("configured_unvalidated" if template_exists else "required")
)
task_sequence = [
{
"phase_id": "native_wf_sbas",
"task_name": native_workflow_task,
"purpose": "Run SARscape's installed end-to-end SBAS metatask.",
"available": native_workflow_available,
"required": template_strategy == TEMPLATE_STRATEGY_NATIVE,
"parameter_template_status": parameter_template_state,
"has_template_parameters": isinstance(
(template.get("native_workflow") or {}).get("parameters")
if isinstance(template.get("native_workflow"), dict)
else None,
dict,
),
"supports_system_selected_edges": False,
"ready": (
native_workflow_available
and template_strategy == TEMPLATE_STRATEGY_NATIVE
and template_validated
and execution_enabled
),
}
]
template_tasks = {
str(item.get("task_name") or "").strip(): item
for item in (template.get("tasks") or [])
if isinstance(item, dict)
}
for phase in PIPELINE_PHASES:
task_name = str(phase["task_name"])
optional = bool(phase.get("optional", False))
template_task = template_tasks.get(task_name) or {}
has_template_parameters = isinstance(template_task.get("parameters"), dict)
task_sequence.append(
{
**phase,
"available": task_name in available_tasks,
"required": not optional,
"template_phase_id": template_task.get("phase_id"),
"parameter_template_status": parameter_template_state,
"has_template_parameters": has_template_parameters,
"ready": (
(task_name in available_tasks or optional)
and template_strategy == TEMPLATE_STRATEGY_EXPLICIT
and template_validated
and execution_enabled
),
}
)
return {
"schema": "insar.sarscape-sbas-processor/v1",
"created_at_utc": _utcnow_iso(),
"engine_code": ENGINE_CODE,
"processor_code": PROCESSOR_CODE,
"execution_enabled": execution_enabled,
"ready_for_pipeline_design": bool(discovery_report and discovery_report.get("ok")),
"ready_for_execution": not blockers,
"blockers": blockers,
"stack_manifest_checksum": _sha256_payload(stack_manifest),
"stack_manifest_summary": {
"schema": stack_manifest.get("schema"),
"prepared_stack_schema": stack_manifest.get("prepared_stack_schema"),
"prepared_stack_id": stack_manifest.get("prepared_stack_id"),
"manifest_role": stack_manifest.get("manifest_role"),
"batch_id": stack_manifest.get("batch_id"),
"plan_id": stack_manifest.get("plan_id"),
"plan_strategy": stack_manifest.get("plan_strategy"),
"reference_date": stack_manifest.get("reference_date"),
"scene_count": len(scenes),
"stack_key": stack_manifest.get("stack_key"),
"group_key": stack_manifest.get("group_key"),
},
"network_summary": summarize_network_edges(network_edges),
"execution_strategy": template_strategy,
"execution_strategies": {
TEMPLATE_STRATEGY_NATIVE: {
"preferred": template_strategy == TEMPLATE_STRATEGY_NATIVE,
"task_name": native_workflow_task,
"available": native_workflow_available,
"required_tasks": [native_workflow_task],
"missing_tasks": missing_native_tasks,
"supports_system_selected_edges": False,
"graph_policy": "SARscape wf_sbas builds the connection graph internally; system network_edges are retained for audit and comparison.",
},
TEMPLATE_STRATEGY_EXPLICIT: {
"preferred": template_strategy == TEMPLATE_STRATEGY_EXPLICIT,
"available": explicit_stack_available,
"required_tasks": list(REQUIRED_STACK_TASKS),
"missing_tasks": missing_stack_tasks,
"supports_system_selected_edges": "not_verified",
"graph_policy": "Explicit task chaining can expose the connection graph step, but direct injection of the system-selected edge list still needs SARscape parameter validation.",
},
},
"required_tasks": [native_workflow_task] if template_strategy == TEMPLATE_STRATEGY_NATIVE else list(REQUIRED_STACK_TASKS),
"required_stack_tasks": list(REQUIRED_STACK_TASKS),
"optional_tasks": list(OPTIONAL_TASKS),
"available_tasks": sorted(available_tasks),
"missing_required_tasks": missing_required_tasks,
"parameter_template": {
"path": template_path or None,
"exists": template_exists,
"readable": bool(template_status.get("readable")),
"validated": bool(template_status.get("validated")),
"execution_strategy": template_strategy,
"native_workflow_task": native_workflow_task,
"task_count": int(template_status.get("task_count") or 0),
"errors": template_status.get("errors") or [],
"source": "manual_sarscape_template",
},
"task_sequence": task_sequence,
"input_contract": {
"required_manifest_role_for_execution": "prepared_sbas_stack",
"prepared_stack_schema": PREPARED_STACK_SCHEMA,
"production_input_policy": "prepared_stack_manifest_only",
"scene_path_fields": ["folder_path", "tiff_path", "meta_path"],
"network_edge_source": "stack_manifest.network_edges",
"dem_source": "IDL_DINSAR_DEM_BASE_FILE",
"orbit_source": "ORBIT_POOL_ENVI",
},
"result_contract": {
"catalog_name": "psinsar",
"required_roles": list(REQUIRED_RESULT_ROLES),
"publish_manifest_schema": "psinsar.publish.v2",
},
"notes": [
"This manifest is a planning contract only; it does not execute SARscape tasks.",
"Execution must use checked-in SARscape parameter templates, not live task.parameters.",
"The native wf_sbas strategy is the preferred first integration path on this workstation.",
],
}
def build_preflight_report(
stack_manifest: Dict[str, Any],
*,
include_task_discovery: bool = True,
discovery_timeout_seconds: int = 120,
parameter_template_path: Optional[str] = None,
) -> Dict[str, Any]:
status = envi_service.get_status()
discovery_report: Optional[Dict[str, Any]] = None
if include_task_discovery:
discovery_report = envi_service.inspect_sarscape_sbas_tasks_subprocess(
timeout_seconds=discovery_timeout_seconds,
include_parameters=False,
)
processor_manifest = build_processor_manifest(
stack_manifest,
discovery_report=discovery_report,
parameter_template_path=parameter_template_path,
)
env_blockers: List[str] = []
if not status.get("idl_installed"):
env_blockers.append("IDL/ENVI executable is not installed or not configured.")
if not status.get("runner_ready"):
env_blockers.append("ENVI runner is not ready: " + str(status.get("runner_message") or "unknown"))
if not status.get("dem_exists"):
env_blockers.append("SARscape DEM base file is missing: " + str(status.get("dem_base_file") or ""))
if discovery_report is not None and not discovery_report.get("ok"):
env_blockers.append("SARscape SBAS task discovery failed: " + str(discovery_report.get("error") or "unknown"))
all_blockers = [*env_blockers, *(processor_manifest.get("blockers") or [])]
return {
"schema": "insar.sarscape-sbas-preflight/v1",
"created_at_utc": _utcnow_iso(),
"engine_code": ENGINE_CODE,
"processor_code": PROCESSOR_CODE,
"ready_for_pipeline_design": bool(
status.get("idl_installed")
and status.get("runner_ready")
and (discovery_report is None or discovery_report.get("ok"))
),
"ready_for_execution": not all_blockers,
"blockers": all_blockers,
"environment": status,
"task_discovery": discovery_report,
"processor_manifest": processor_manifest,
}
def _resolve_template_value(value: Any, context: Dict[str, Any]) -> Any:
if isinstance(value, str):
text = value.strip()
if text in context:
return context[text]
resolved = value
for key, replacement in context.items():
if key in resolved and isinstance(replacement, (str, int, float, bool)):
resolved = resolved.replace(key, str(replacement))
return resolved
if isinstance(value, list):
return [_resolve_template_value(item, context) for item in value]
if isinstance(value, dict):
return {
str(key): _resolve_template_value(item, context)
for key, item in value.items()
}
return value
def _scene_input_uris(scenes: List[Dict[str, Any]]) -> List[str]:
uris: List[str] = []
for item in scenes:
for key in ("meta_path", "tiff_path", "folder_path"):
text = str(item.get(key) or "").strip()
if text:
uris.append(text)
break
return uris
def execute_template_workflow(
stack_manifest: Dict[str, Any],
*,
work_root: str,
selected_manifest_path: str,
timeout_seconds: Optional[int] = None,
) -> Dict[str, Any]:
"""Execute a validated SARscape SBAS template.
This path is intentionally gated by SARSCAPE_SBAS_ALLOW_EXECUTION and
template validated=true. The default checked-in template is not executable.
"""
if stack_manifest.get("prepared_stack_schema") != PREPARED_STACK_SCHEMA:
raise ValueError(
f"SARscape SBAS execution requires a prepared stack manifest ({PREPARED_STACK_SCHEMA})."
)
if not str(stack_manifest.get("prepared_stack_id") or "").strip():
raise ValueError("SARscape SBAS execution requires prepared_stack_id.")
discovery_report = envi_service.inspect_sarscape_sbas_tasks_subprocess(
timeout_seconds=int(getattr(settings, "SARSCAPE_SBAS_DISCOVERY_TIMEOUT_SECONDS", 120) or 120),
include_parameters=False,
)
processor_manifest = build_processor_manifest(
stack_manifest,
discovery_report=discovery_report,
)
if not processor_manifest.get("ready_for_execution"):
blockers = "; ".join(str(item) for item in (processor_manifest.get("blockers") or []))
raise ValueError("SARscape SBAS execution is not ready: " + (blockers or "unknown blocker"))
template_status = load_parameter_template()
template = template_status.get("template") if isinstance(template_status.get("template"), dict) else {}
tasks = [item for item in (template.get("tasks") or []) if isinstance(item, dict)]
output_root = Path(work_root) / "sarscape_sbas"
output_root.mkdir(parents=True, exist_ok=True)
artifacts = stack_manifest.get("artifacts") if isinstance(stack_manifest.get("artifacts"), dict) else {}
prepared_edges_path = str(artifacts.get("selected_network_edges_path_windows") or "").strip()
if prepared_edges_path:
network_edges_path = Path(prepared_edges_path)
if not network_edges_path.is_file():
raise FileNotFoundError(f"Prepared selected_network_edges.json not found: {network_edges_path}")
else:
network_edges_path = output_root / "selected_network_edges.json"
network_edges_path.write_text(
json.dumps(stack_manifest.get("network_edges") or [], ensure_ascii=False, indent=2),
encoding="utf-8",
)
scenes = stack_manifest.get("scenes") if isinstance(stack_manifest.get("scenes"), list) else []
context: Dict[str, Any] = {
"${work_root}": str(work_root),
"${output_root}": str(output_root),
"${selected_stack_manifest}": str(selected_manifest_path),
"${selected_network_edges}": str(network_edges_path),
"${scene_meta_paths}": [
str(item.get("meta_path"))
for item in scenes
if str(item.get("meta_path") or "").strip()
],
"${scene_input_uris}": _scene_input_uris(scenes),
"${scene_folder_paths}": [
str(item.get("folder_path"))
for item in scenes
if str(item.get("folder_path") or "").strip()
],
"${selection_params}": stack_manifest.get("selection_params") or {},
"${dem_sarscapedata}": envi_service._build_sarscapedata(envi_service.DEM_BASE_FILE), # noqa: SLF001
}
executed: List[Dict[str, Any]] = []
previous_outputs: Dict[str, Any] = {}
execution_strategy = str(template.get("execution_strategy") or TEMPLATE_STRATEGY_NATIVE).strip()
if execution_strategy not in SUPPORTED_TEMPLATE_STRATEGIES:
execution_strategy = TEMPLATE_STRATEGY_NATIVE
if execution_strategy == TEMPLATE_STRATEGY_NATIVE:
native_workflow = template.get("native_workflow") if isinstance(template.get("native_workflow"), dict) else {}
task_name = str(native_workflow.get("task_name") or NATIVE_WORKFLOW_TASK).strip()
if not task_name:
raise ValueError("Native SARscape SBAS workflow task_name is empty.")
phase_id = str(native_workflow.get("phase_id") or "native_wf_sbas").strip()
phase_output_dir = output_root / phase_id
phase_output_dir.mkdir(parents=True, exist_ok=True)
phase_context = {
**context,
"${phase_id}": phase_id,
"${phase_output_dir}": str(phase_output_dir),
"${previous_outputs}": previous_outputs,
}
parameters = _resolve_template_value(native_workflow.get("parameters") or {}, phase_context)
result = envi_service.execute_envi_task(task_name, parameters)
previous_outputs[phase_id] = result
executed.append(
{
"phase_id": phase_id,
"task_name": task_name,
"output_dir": str(phase_output_dir),
"output_keys": sorted((result or {}).keys()) if isinstance(result, dict) else [],
}
)
tasks = []
for item in tasks:
if not bool(item.get("enabled", True)):
continue
task_name = str(item.get("task_name") or "").strip()
phase_id = str(item.get("phase_id") or task_name).strip()
if not task_name:
raise ValueError(f"SARscape template task is missing task_name: {phase_id}")
phase_output_dir = output_root / phase_id
phase_output_dir.mkdir(parents=True, exist_ok=True)
phase_context = {
**context,
"${phase_id}": phase_id,
"${phase_output_dir}": str(phase_output_dir),
"${previous_outputs}": previous_outputs,
}
parameters = _resolve_template_value(item.get("parameters") or {}, phase_context)
result = envi_service.execute_envi_task(task_name, parameters)
previous_outputs[phase_id] = result
executed.append(
{
"phase_id": phase_id,
"task_name": task_name,
"output_dir": str(phase_output_dir),
"output_keys": sorted((result or {}).keys()) if isinstance(result, dict) else [],
}
)
return {
"schema": "insar.sarscape-sbas-execution/v1",
"created_at_utc": _utcnow_iso(),
"processor_code": PROCESSOR_CODE,
"execution_strategy": execution_strategy,
"prepared_stack_id": stack_manifest.get("prepared_stack_id"),
"work_root": str(work_root),
"output_root": str(output_root),
"selected_network_edges_path": str(network_edges_path),
"task_count": len(executed),
"executed_tasks": executed,
"processor_manifest": processor_manifest,
}
def write_processor_manifest(path: str | Path, manifest: Dict[str, Any]) -> str:
target = Path(path)
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
return str(target)
+204
View File
@@ -35,6 +35,7 @@ from ..models import (
RadarDataORM, RadarDataORM,
RadarPair, RadarPair,
ResultProductORM, ResultProductORM,
TimeseriesStackPlanEdgeORM,
TimeseriesStackPlanItemORM, TimeseriesStackPlanItemORM,
TimeseriesStackPlanORM, TimeseriesStackPlanORM,
) )
@@ -404,6 +405,138 @@ class SpatialService:
"stack_dates": stack_dates, "stack_dates": stack_dates,
} }
async def _build_timeseries_network_edges(
self,
db: AsyncSession,
scenes: List[RadarDataORM],
params: PsRequest,
*,
aoi_wkt: Optional[str],
selection_mode: Optional[str],
) -> Tuple[List[Dict[str, Any]], List[str]]:
scene_ids = [int(item.id) for item in scenes if item.id is not None]
if len(scene_ids) < 2:
return [], []
master_alias = aliased(RadarDataORM)
slave_alias = aliased(RadarDataORM)
stmt = (
select(PairingMetricCacheORM, master_alias, slave_alias)
.join(master_alias, master_alias.id == PairingMetricCacheORM.master_scene_ref_id)
.join(slave_alias, slave_alias.id == PairingMetricCacheORM.slave_scene_ref_id)
.where(
PairingMetricCacheORM.metric_version == pairing_state_service.metric_version,
PairingMetricCacheORM.status == "READY",
PairingMetricCacheORM.master_scene_ref_id.in_(scene_ids),
PairingMetricCacheORM.slave_scene_ref_id.in_(scene_ids),
PairingMetricCacheORM.time_baseline_days >= params.time_baseline_min,
PairingMetricCacheORM.time_baseline_days <= params.time_baseline_max,
PairingMetricCacheORM.spatial_baseline_meters <= params.spatial_baseline_max_meters,
PairingMetricCacheORM.scene_overlap_ratio >= params.network_overlap_threshold,
)
.order_by(
PairingMetricCacheORM.master_imaging_date.asc(),
PairingMetricCacheORM.slave_imaging_date.asc(),
PairingMetricCacheORM.time_baseline_days.asc(),
PairingMetricCacheORM.spatial_baseline_meters.asc(),
func.coalesce(PairingMetricCacheORM.scene_overlap_ratio, 0).desc(),
PairingMetricCacheORM.id.asc(),
)
)
result = await db.execute(stmt)
candidate_pool: List[dict] = []
for metric_row, master_row, slave_row in result.all():
candidate_pool.append(
{
"metric_cache_ref_id": int(metric_row.id),
"pair_uid": metric_row.pair_uid,
"master_scene_uid": metric_row.master_scene_uid,
"slave_scene_uid": metric_row.slave_scene_uid,
"master": RadarData.model_validate(master_row),
"slave": RadarData.model_validate(slave_row),
"days": int(metric_row.time_baseline_days or 0),
"dist": float(metric_row.spatial_baseline_meters or 0.0),
"overlap_ratio": float(metric_row.scene_overlap_ratio or 0.0),
}
)
warnings: List[str] = []
if not candidate_pool:
warnings.append(
"No pairing_metric_cache edges matched the time-series SBAS network thresholds."
)
return [], warnings
candidate_scene_ids = {
int(candidate[role].id)
for candidate in candidate_pool
for role in ("master", "slave")
if candidate.get(role) is not None
}
missing_scene_count = len(set(scene_ids) - candidate_scene_ids)
if missing_scene_count > 0:
warnings.append(
f"{missing_scene_count} selected scenes have no metric-cache edge under the current SBAS thresholds."
)
pairing_params = PairingRequest(
time_baseline_min=params.time_baseline_min,
time_baseline_max=params.time_baseline_max,
overlap_threshold=params.network_overlap_threshold,
spatial_baseline_max_meters=params.spatial_baseline_max_meters,
coverage_diversity_penalty=0.3,
require_same_imaging_mode=True,
require_same_polarization=True,
strategy="sbas",
num_connections=params.num_connections,
)
selected_candidates, strategy_warnings = self._apply_sbas_strategy(
candidate_pool,
pairing_params,
aoi_wkt=aoi_wkt,
)
warnings.extend(strategy_warnings)
edges: List[Dict[str, Any]] = []
for edge_rank, candidate in enumerate(self._sorted_candidates(selected_candidates), start=1):
master = candidate["master"]
slave = candidate["slave"]
edges.append(
{
"edge_rank": edge_rank,
"metric_cache_ref_id": candidate.get("metric_cache_ref_id"),
"master_scene_ref_id": int(master.id),
"slave_scene_ref_id": int(slave.id),
"master_imaging_date": master.imaging_date,
"slave_imaging_date": slave.imaging_date,
"temporal_baseline_days": int(candidate.get("days") or 0),
"spatial_baseline_meters": float(candidate.get("dist") or 0.0),
"scene_overlap_ratio": float(candidate.get("overlap_ratio") or 0.0),
"selection_reason": candidate.get("selection_reason"),
"selection_score": (
float(candidate["selection_score"])
if candidate.get("selection_score") is not None
else None
),
"selection_meta_json": {
"source": "pairing_metric_cache",
"selection_mode": selection_mode,
"pair_uid": candidate.get("pair_uid"),
"metric_version": pairing_state_service.metric_version,
"time_baseline_min": params.time_baseline_min,
"time_baseline_max": params.time_baseline_max,
"spatial_baseline_max_meters": params.spatial_baseline_max_meters,
"network_overlap_threshold": params.network_overlap_threshold,
"num_connections": params.num_connections,
},
"enabled": True,
}
)
if not edges:
warnings.append("SBAS strategy did not select any network edges for this stack.")
return edges, warnings
async def _persist_timeseries_stack_plan( async def _persist_timeseries_stack_plan(
self, self,
db: AsyncSession, db: AsyncSession,
@@ -416,6 +549,8 @@ class SpatialService:
coverage_consistency_ratio: Optional[float] = None, coverage_consistency_ratio: Optional[float] = None,
threshold_satisfied: Optional[bool] = None, threshold_satisfied: Optional[bool] = None,
selection_mode: Optional[str] = None, selection_mode: Optional[str] = None,
network_edges: Optional[List[Dict[str, Any]]] = None,
network_warnings: Optional[List[str]] = None,
) -> Dict[str, Any]: ) -> Dict[str, Any]:
request_payload = params.model_dump(exclude_none=True) request_payload = params.model_dump(exclude_none=True)
aoi_hash = self._stable_sha1(aoi_wkt) if aoi_wkt else None aoi_hash = self._stable_sha1(aoi_wkt) if aoi_wkt else None
@@ -447,6 +582,9 @@ class SpatialService:
sorted_scenes = sorted(scenes, key=lambda item: str(item.imaging_date or "")) sorted_scenes = sorted(scenes, key=lambda item: str(item.imaging_date or ""))
scene_payloads: List[RadarData] = [] scene_payloads: List[RadarData] = []
plan_item_by_scene_id: Dict[int, TimeseriesStackPlanItemORM] = {}
safe_network_edges = list(network_edges or [])
safe_network_warnings = [str(item) for item in (network_warnings or []) if str(item).strip()]
for rank, item in enumerate(sorted_scenes, start=1): for rank, item in enumerate(sorted_scenes, start=1):
plan_item = TimeseriesStackPlanItemORM( plan_item = TimeseriesStackPlanItemORM(
plan_ref_id=plan.id, plan_ref_id=plan.id,
@@ -469,6 +607,8 @@ class SpatialService:
"coverage_consistency_ratio": coverage_consistency_ratio, "coverage_consistency_ratio": coverage_consistency_ratio,
"threshold_satisfied": threshold_satisfied, "threshold_satisfied": threshold_satisfied,
"selection_mode": selection_mode, "selection_mode": selection_mode,
"network_edge_count": len(safe_network_edges),
"network_warnings": safe_network_warnings,
"orbit_direction": item.orbit_direction, "orbit_direction": item.orbit_direction,
"satellite_family": self._normalize_timeseries_satellite_family(item), "satellite_family": self._normalize_timeseries_satellite_family(item),
"bbox": [item.min_lon, item.min_lat, item.max_lon, item.max_lat], "bbox": [item.min_lon, item.min_lat, item.max_lon, item.max_lat],
@@ -477,6 +617,8 @@ class SpatialService:
) )
db.add(plan_item) db.add(plan_item)
await db.flush() await db.flush()
if item.id is not None:
plan_item_by_scene_id[int(item.id)] = plan_item
scene_payloads.append( scene_payloads.append(
RadarData.model_validate(item).model_copy( RadarData.model_validate(item).model_copy(
update={ update={
@@ -490,14 +632,61 @@ class SpatialService:
"stack_coverage_consistency_ratio": coverage_consistency_ratio, "stack_coverage_consistency_ratio": coverage_consistency_ratio,
"stack_threshold_satisfied": threshold_satisfied, "stack_threshold_satisfied": threshold_satisfied,
"stack_selection_mode": selection_mode, "stack_selection_mode": selection_mode,
"stack_network_edge_count": len(safe_network_edges),
"stack_network_warnings": safe_network_warnings,
} }
) )
) )
for edge_payload in safe_network_edges:
master_scene_id = edge_payload.get("master_scene_ref_id")
slave_scene_id = edge_payload.get("slave_scene_ref_id")
master_plan_item = (
plan_item_by_scene_id.get(int(master_scene_id))
if master_scene_id is not None
else None
)
slave_plan_item = (
plan_item_by_scene_id.get(int(slave_scene_id))
if slave_scene_id is not None
else None
)
edge = TimeseriesStackPlanEdgeORM(
plan_ref_id=plan.id,
master_plan_item_ref_id=(
int(master_plan_item.id)
if master_plan_item is not None and master_plan_item.id is not None
else None
),
slave_plan_item_ref_id=(
int(slave_plan_item.id)
if slave_plan_item is not None and slave_plan_item.id is not None
else None
),
metric_cache_ref_id=edge_payload.get("metric_cache_ref_id"),
master_scene_ref_id=master_scene_id,
slave_scene_ref_id=slave_scene_id,
edge_rank=int(edge_payload.get("edge_rank") or 0),
master_imaging_date=edge_payload.get("master_imaging_date"),
slave_imaging_date=edge_payload.get("slave_imaging_date"),
temporal_baseline_days=edge_payload.get("temporal_baseline_days"),
spatial_baseline_meters=edge_payload.get("spatial_baseline_meters"),
perpendicular_baseline_meters=edge_payload.get("perpendicular_baseline_meters"),
scene_overlap_ratio=edge_payload.get("scene_overlap_ratio"),
pair_aoi_overlap_ratio=edge_payload.get("pair_aoi_overlap_ratio"),
selection_reason=edge_payload.get("selection_reason"),
selection_score=edge_payload.get("selection_score"),
selection_meta_json=edge_payload.get("selection_meta_json"),
enabled=bool(edge_payload.get("enabled", True)),
)
db.add(edge)
return { return {
"plan_id": plan.plan_id, "plan_id": plan.plan_id,
"group_key": identity.get("group_key"), "group_key": identity.get("group_key"),
"stack_key": identity.get("stack_key"), "stack_key": identity.get("stack_key"),
"edge_count": len(safe_network_edges),
"network_warnings": safe_network_warnings,
"scenes": scene_payloads, "scenes": scene_payloads,
} }
@@ -1286,6 +1475,19 @@ class SpatialService:
if len(final_stack) >= 3: if len(final_stack) >= 3:
final_stack.sort(key=lambda x: str(x.imaging_date or "")) final_stack.sort(key=lambda x: str(x.imaging_date or ""))
direction = group_key[0] direction = group_key[0]
network_edges, network_warnings = await self._build_timeseries_network_edges(
db,
final_stack,
params,
aoi_wkt=aoi_wkt,
selection_mode=selection_mode,
)
logger.info(
"timeseries stack planning: group=%s network_edges=%s network_warnings=%s",
self._format_timeseries_group_label(group_key),
len(network_edges),
len(network_warnings),
)
persisted_plan = await self._persist_timeseries_stack_plan( persisted_plan = await self._persist_timeseries_stack_plan(
db, db,
direction=direction, direction=direction,
@@ -1296,6 +1498,8 @@ class SpatialService:
coverage_consistency_ratio=consistency_ratio, coverage_consistency_ratio=consistency_ratio,
threshold_satisfied=threshold_satisfied, threshold_satisfied=threshold_satisfied,
selection_mode=selection_mode, selection_mode=selection_mode,
network_edges=network_edges,
network_warnings=network_warnings,
) )
result_key = persisted_plan.get("group_key") or self._format_timeseries_group_label(group_key) result_key = persisted_plan.get("group_key") or self._format_timeseries_group_label(group_key)
if result_key in final_results: if result_key in final_results:
File diff suppressed because it is too large Load Diff
+1
View File
@@ -173,6 +173,7 @@ class WorkflowService:
step.outputs = outputs step.outputs = outputs
await self._advance_ready_steps(run_id, db) await self._advance_ready_steps(run_id, db)
await db.flush()
await self.enqueue_ready_steps(run_id, db=db) await self.enqueue_ready_steps(run_id, db=db)
if gen_db: if gen_db:
@@ -0,0 +1,36 @@
-- Persist selected SBAS graph edges for time-series stack plans.
CREATE TABLE IF NOT EXISTS timeseries_stack_plan_edges (
id SERIAL PRIMARY KEY,
plan_ref_id INTEGER NOT NULL REFERENCES timeseries_stack_plans(id) ON DELETE CASCADE,
master_plan_item_ref_id INTEGER NULL REFERENCES timeseries_stack_plan_items(id) ON DELETE SET NULL,
slave_plan_item_ref_id INTEGER NULL REFERENCES timeseries_stack_plan_items(id) ON DELETE SET NULL,
metric_cache_ref_id INTEGER NULL REFERENCES pairing_metric_cache(id) ON DELETE SET NULL,
master_scene_ref_id INTEGER NULL REFERENCES radar_data(id) ON DELETE SET NULL,
slave_scene_ref_id INTEGER NULL REFERENCES radar_data(id) ON DELETE SET NULL,
edge_rank INTEGER NOT NULL DEFAULT 0,
master_imaging_date VARCHAR(8) NULL,
slave_imaging_date VARCHAR(8) NULL,
temporal_baseline_days INTEGER NULL,
spatial_baseline_meters DOUBLE PRECISION NULL,
perpendicular_baseline_meters DOUBLE PRECISION NULL,
scene_overlap_ratio DOUBLE PRECISION NULL,
pair_aoi_overlap_ratio DOUBLE PRECISION NULL,
selection_reason VARCHAR(64) NULL,
selection_score DOUBLE PRECISION NULL,
selection_meta_json JSON NULL,
enabled BOOLEAN NOT NULL DEFAULT TRUE,
created_at TIMESTAMP NOT NULL DEFAULT NOW()
);
CREATE UNIQUE INDEX IF NOT EXISTS uq_timeseries_plan_edges_plan_rank
ON timeseries_stack_plan_edges (plan_ref_id, edge_rank);
CREATE INDEX IF NOT EXISTS idx_timeseries_plan_edges_plan_enabled
ON timeseries_stack_plan_edges (plan_ref_id, enabled);
CREATE INDEX IF NOT EXISTS idx_timeseries_plan_edges_plan_scenes
ON timeseries_stack_plan_edges (plan_ref_id, master_scene_ref_id, slave_scene_ref_id);
CREATE INDEX IF NOT EXISTS idx_timeseries_plan_edges_metric_cache
ON timeseries_stack_plan_edges (metric_cache_ref_id);
@@ -0,0 +1,398 @@
{
"schema": "insar.sarscape-sbas-template/v1",
"template_name": "SARscape SBAS native wf_sbas integration template",
"sarscape_version_hint": "ENVI 5.6 / SARscape taskengine, .task version 5.3",
"validated": false,
"execution_strategy": "native_workflow_metatask",
"source": {
"method": "static_task_json",
"extractor": "scripts/extract_sarscape_sbas_task_templates.py",
"envi_root": "C:\\Program Files\\Harris\\ENVI56",
"workflow_task_file": "C:\\Program Files\\Harris\\ENVI56\\user_custom_code\\wf_sbas.task",
"available_task_count_on_reference_machine": 17
},
"notes": [
"This file is a checked-in contract and is intentionally not executable until validated=true.",
"The preferred first execution strategy is SARscape's native wf_sbas metatask.",
"wf_sbas includes import, preferences, suggested looks, SBAS connection graph, interferogram generation, inversion step 1, inversion step 2, geocode, and shape export in its embedded DAG.",
"Static wf_sbas.task contains a DAG parameter with an embedded default, but live taskengine QueryTask exposes 17 parameters and does not require the caller to pass DAG.",
"wf_sbas builds the connection graph internally. The system-selected network_edges remain the authoritative planning and audit graph until explicit stack-task graph injection is verified.",
"Do not use live task.parameters for stack SBAS tasks on this workstation; it can hang taskengine.exe."
],
"macros": {
"${work_root}": "Run work root directory.",
"${output_root}": "SARscape SBAS output root directory.",
"${phase_output_dir}": "Output directory for the current phase.",
"${selected_stack_manifest}": "Selected stack manifest JSON path.",
"${selected_network_edges}": "Selected network edges JSON path.",
"${scene_input_uris}": "Ordered source scene input URI list, preferring scene meta_path.",
"${scene_meta_paths}": "Ordered source scene metadata paths.",
"${scene_folder_paths}": "Ordered source scene folders.",
"${selection_params}": "SBAS network selection parameters from stack_manifest.selection_params.",
"${dem_sarscapedata}": "DEM SARSCAPEDATA object built from IDL_DINSAR_DEM_BASE_FILE.",
"${previous_outputs}": "Outputs from earlier explicit task phases."
},
"native_workflow": {
"phase_id": "native_wf_sbas",
"task_name": "wf_sbas",
"source_task_file": "C:\\Program Files\\Harris\\ENVI56\\user_custom_code\\wf_sbas.task",
"parameters": {
"INPUT_FILE_LIST": "${scene_input_uris}",
"SARSCAPE_PREFERENCE": "Use actual preferences",
"DEM_SARSCAPEDATA": "${dem_sarscapedata}",
"OUTPUT_FOLDER": "${output_root}",
"GEOCODE_RG_GRID_SIZE": 10.0,
"ESTIMATE_RESIDUAL_HEIGHT": true,
"DISPLACEMENT_MODEL_TYPE": "linear"
},
"parameter_schema_summary": [
{
"name": "INPUT_FILE_LIST",
"type": "ENVIURI",
"dimensions": "[*]",
"direction": "input",
"required": true
},
{
"name": "SARSCAPE_PREFERENCE",
"type": "STRING",
"direction": "input",
"required": false,
"default": "Use actual preferences"
},
{
"name": "DEM_SARSCAPEDATA",
"type": "SARSCAPEDATA",
"direction": "input",
"required": false
},
{
"name": "REFINEMENT_GCP_FILE_NAME",
"type": "ENVIURI",
"direction": "input",
"required": false
},
{
"name": "OUTPUT_FOLDER",
"type": "ENVIURI",
"direction": "input",
"required": false
},
{
"name": "GEOCODE_RG_GRID_SIZE",
"type": "DOUBLE",
"direction": "input",
"required": false
},
{
"name": "ESTIMATE_RESIDUAL_HEIGHT",
"type": "BOOLEAN",
"direction": "input",
"required": false,
"default": true
},
{
"name": "DISPLACEMENT_MODEL_TYPE",
"type": "STRING",
"direction": "input",
"required": false,
"choices": [
"no_displacement",
"linear",
"quadratic",
"cubic",
"linear_periodic"
]
},
{
"name": "OUTPUT_ENVI_CARTOGRAPHIC_SYSTEM",
"type": "ENVICOORDSYS",
"direction": "input",
"required": false
},
{
"name": "DAG",
"type": "ENVIMETATASKDAG",
"direction": "input",
"required": false,
"provided_by_task_default": true,
"static_task_only": true,
"live_taskengine_querytask_exposes": false
}
],
"output_parameters": [
"OUTPUT_SHAPES",
"DISPLACEMENT_SARSCAPEDATA",
"DEM_OUT_SARSCAPEDATA",
"CORRECTION_H_SARSCAPEDATA",
"COHERENCE_SARSCAPEDATA",
"ALOS_SARSCAPEDATA",
"ILOS_SARSCAPEDATA",
"VELOCITY_SARSCAPEDATA"
],
"dag_summary": [
{
"node_id": "task_10",
"task_name": "SARscape_setting_output_folders",
"external_input": {
"output_folder": "OUTPUT_FOLDER"
},
"static_input": {
"sub_1_folder": "imported_data"
}
},
{
"node_id": "task_6",
"task_name": "SARsLoadPreferences",
"external_input": {
"sarscape_preference": "SARSCAPE_PREFERENCE"
}
},
{
"node_id": "task_8",
"task_name": "SARsImportSarSelector",
"external_input": {
"input_file_list": "INPUT_FILE_LIST"
},
"internal_input": {
"root_uri_for_output": "task_10.root_uri_1_for_output"
},
"static_input": {
"cross_copolarization": "ONLY_COPOL_POL"
}
},
{
"node_id": "elementExtractor_1",
"task_name": "ENVIEXTRACTELEMENTSFROMARRAYTASK",
"internal_input": {
"input_array": "task_8.output_sarscapedata"
},
"static_input": {
"indices": [
0
]
}
},
{
"node_id": "task_9",
"task_name": "SARscapeSuggestLooks",
"external_input": {
"grid_size_for_suggested_looks": "GEOCODE_RG_GRID_SIZE"
},
"internal_input": {
"reference_data": "elementExtractor_1.output_element"
}
},
{
"node_id": "task_1",
"task_name": "SARsInSARStackSBASGenerateConnectionGraph",
"internal_input": {
"input_sarscapedata": "task_8.output_sarscapedata",
"root_uri_for_output": "task_10.root_uri_for_output"
}
},
{
"node_id": "task_2",
"task_name": "SARsInSARStackSBASInterferogramGeneration",
"external_input": {
"dem_sarscapedata": "DEM_SARSCAPEDATA"
},
"internal_input": {
"auxiliary_file_name": "task_1.auxiliary_processing_info_file",
"az_looks_nbr": "task_9.looks_az",
"rg_looks_nbr": "task_9.looks_rg"
}
},
{
"node_id": "task_3",
"task_name": "SARsInSARStackSBASInversionStep1",
"external_input": {
"refinement_gcp_file_name": "REFINEMENT_GCP_FILE_NAME",
"estimate_residual_height": "ESTIMATE_RESIDUAL_HEIGHT",
"displacement_model_type": "DISPLACEMENT_MODEL_TYPE"
},
"internal_input": {
"auxiliary_file_name": "task_2.auxiliary_processing_info_file"
}
},
{
"node_id": "task_4",
"task_name": "SARsInSARStackSBASInversionStep2",
"external_input": {
"refinement_gcp_file_name": "REFINEMENT_GCP_FILE_NAME"
},
"internal_input": {
"auxiliary_file_name": "task_3.auxiliary_processing_info_file"
}
},
{
"node_id": "task_5",
"task_name": "SARsInSARStackSBASGeocode",
"external_input": {
"geocode_rg_grid_size": "GEOCODE_RG_GRID_SIZE",
"geocode_az_grid_size": "GEOCODE_RG_GRID_SIZE",
"output_envi_cartographic_system": "OUTPUT_ENVI_CARTOGRAPHIC_SYSTEM",
"refinement_gcp_file_name": "REFINEMENT_GCP_FILE_NAME",
"dem_sarscapedata": "DEM_SARSCAPEDATA"
},
"internal_input": {
"auxiliary_file_name": "task_4.auxiliary_processing_info_file"
},
"output": {
"displacement_sarscapedata": "DISPLACEMENT_SARSCAPEDATA",
"dem_out_sarscapedata": "DEM_OUT_SARSCAPEDATA",
"correction_h_sarscapedata": "CORRECTION_H_SARSCAPEDATA",
"coherence_sarscapedata": "COHERENCE_SARSCAPEDATA",
"alos_sarscapedata": "ALOS_SARSCAPEDATA",
"ilos_sarscapedata": "ILOS_SARSCAPEDATA",
"velocity_sarscapedata": "VELOCITY_SARSCAPEDATA"
}
},
{
"node_id": "task_7",
"task_name": "SARscapeEnviuriToShape",
"external_input": {
"arcgis_output_folder": "OUTPUT_FOLDER"
},
"internal_input": {
"input_data": "task_5.output_sbas_shapes"
},
"output": {
"output_shapes": "OUTPUT_SHAPES"
}
}
]
},
"explicit_stack_task_chain": {
"status": "available_but_not_validated_for_system_edge_injection",
"required_tasks": [
"SARsInSARStackSBASGenerateConnectionGraph",
"SARsInSARStackSBASInterferogramGeneration",
"SARsInSARStackSBASInversionStep1",
"SARsInSARStackSBASInversionStep2",
"SARsInSARStackSBASGeocode"
],
"optional_tasks": [
"SARsInSARStackSBASVariogram"
],
"chaining_rule": "Each phase consumes the previous phase AUXILIARY_PROCESSING_INFO_FILE as AUXILIARY_FILE_NAME.",
"open_issue": "Need SARscape-supported method to force or import the system-selected network_edges instead of allowing GenerateConnectionGraph to rebuild the graph."
},
"tasks": [
{
"phase_id": "connection_graph",
"task_name": "SARsInSARStackSBASGenerateConnectionGraph",
"enabled": false,
"source_task_file": "C:\\Program Files\\Harris\\ENVI56\\resource\\templates\\tasks\\SARscape\\SARsInSARStackSBASGenerateConnectionGraph.task",
"required_inputs": [
"INPUT_SARSCAPEDATA"
],
"outputs": [
"OUT_TRIGGERING_EXECUTION_OPTION",
"AUXILIARY_PROCESSING_INFO_FILE"
],
"parameter_count": 18,
"parameters": {}
},
{
"phase_id": "interferogram_generation",
"task_name": "SARsInSARStackSBASInterferogramGeneration",
"enabled": false,
"source_task_file": "C:\\Program Files\\Harris\\ENVI56\\resource\\templates\\tasks\\SARscape\\SARsInSARStackSBASInterferogramGeneration.task",
"required_inputs": [
"AUXILIARY_FILE_NAME"
],
"outputs": [
"OUT_TRIGGERING_EXECUTION_OPTION",
"AUXILIARY_PROCESSING_INFO_FILE"
],
"parameter_count": 36,
"parameters": {}
},
{
"phase_id": "inversion_step1",
"task_name": "SARsInSARStackSBASInversionStep1",
"enabled": false,
"source_task_file": "C:\\Program Files\\Harris\\ENVI56\\resource\\templates\\tasks\\SARscape\\SARsInSARStackSBASInversionStep1.task",
"required_inputs": [
"AUXILIARY_FILE_NAME"
],
"outputs": [
"OUT_TRIGGERING_EXECUTION_OPTION",
"AUXILIARY_PROCESSING_INFO_FILE"
],
"parameter_count": 24,
"parameters": {}
},
{
"phase_id": "inversion_step2",
"task_name": "SARsInSARStackSBASInversionStep2",
"enabled": false,
"source_task_file": "C:\\Program Files\\Harris\\ENVI56\\resource\\templates\\tasks\\SARscape\\SARsInSARStackSBASInversionStep2.task",
"required_inputs": [
"AUXILIARY_FILE_NAME"
],
"outputs": [
"OUT_TRIGGERING_EXECUTION_OPTION",
"AUXILIARY_PROCESSING_INFO_FILE"
],
"parameter_count": 19,
"parameters": {}
},
{
"phase_id": "geocode_export",
"task_name": "SARsInSARStackSBASGeocode",
"enabled": false,
"source_task_file": "C:\\Program Files\\Harris\\ENVI56\\resource\\templates\\tasks\\SARscape\\SARsInSARStackSBASGeocode.task",
"required_inputs": [
"AUXILIARY_FILE_NAME"
],
"outputs": [
"OUT_TRIGGERING_EXECUTION_OPTION",
"OUTPUT_SBAS_DIRECTORY",
"DISPLACEMENT_SARSCAPEDATA",
"DEM_OUT_SARSCAPEDATA",
"CORRECTION_H_SARSCAPEDATA",
"COHERENCE_SARSCAPEDATA",
"ALOS_SARSCAPEDATA",
"ILOS_SARSCAPEDATA",
"VELOCITY_SARSCAPEDATA",
"OUTPUT_SBAS_SHAPES"
],
"parameter_count": 44,
"parameters": {}
},
{
"phase_id": "variogram_optional",
"task_name": "SARsInSARStackSBASVariogram",
"enabled": false,
"optional": true,
"source_task_file": "C:\\Program Files\\Harris\\ENVI56\\resource\\templates\\tasks\\SARscape\\SARsInSARStackSBASVariogram.task",
"required_inputs": [
"AUXILIARY_FILE_NAME"
],
"outputs": [
"OUT_TRIGGERING_EXECUTION_OPTION"
],
"parameter_count": 21,
"parameters": {}
}
],
"expected_outputs": {
"velocity_product": [
"VELOCITY_SARSCAPEDATA"
],
"timeseries_product": [
"DISPLACEMENT_SARSCAPEDATA",
"ALOS_SARSCAPEDATA",
"ILOS_SARSCAPEDATA"
],
"temporal_coherence": [
"COHERENCE_SARSCAPEDATA"
],
"geocoded_raster": [
"OUTPUT_SBAS_DIRECTORY"
],
"preview_png": []
}
}
@@ -0,0 +1,586 @@
# Time-Series SBAS And SARscape Integration Design
## 1. Problem Statement
The current time-series route can find and run a scene stack, but the system does not yet treat SBAS as a first-class production input. The main gaps are:
- `find-ps-timeseries` returns scenes, not a durable SBAS network.
- `PsTaskBatch` is used as a production input even though it is a thin list of paths.
- Planning context is partly duplicated in `PsTaskItem.remark`.
- `copy-ps-stack` copies source folders, but does not create a stack-level production package.
- The managed time-series runner reconstructs input state at run time.
- SARscape is currently integrated only as a D-InSAR pair processor.
The design goal is to make one immutable stack manifest the source of truth for every SBAS run, then let ISCE2/MintPy and SARscape consume the same contract.
## 2. Target Workflow
```text
AOI + filters
-> time-series stack search
-> SBAS network plan
-> user review and commit
-> immutable stack package
-> processor workflow
-> publish bundle
-> psinsar catalog
```
The stack plan and the production package are separate states. A plan is a previewable proposal; a package is a committed production input.
## 3. Planning Contract
### 3.1 Search API
Add or evolve the current `find-ps-timeseries` route toward:
```text
POST /timeseries/plans/search
```
Core request fields:
- AOI source: uploaded shapefile, region geometry, or GeoJSON.
- Scene compatibility filters: satellite, orbit direction, imaging mode, polarization, date range.
- Scene thresholds: `initial_overlap_threshold`, `final_overlap_threshold`.
- Network thresholds: `time_baseline_min`, `time_baseline_max`, `spatial_baseline_max_meters`, later `perpendicular_baseline_max_meters`.
- Network policy: `strategy`, `num_connections`, `reference_image_id`.
- Processor hint: optional `processor_target`, for example `isce2_stack_mintpy` or `sarscape_sbas`.
### 3.2 Plan Tables
Existing:
- `timeseries_stack_plans`
- `timeseries_stack_plan_items`
New:
- `timeseries_stack_plan_edges`
The edge table stores the selected SBAS graph:
- plan reference
- master/slave plan item references
- master/slave radar scene references
- optional `pairing_metric_cache` reference
- temporal baseline
- spatial/perpendicular baseline
- scene overlap ratio
- AOI pair overlap ratio
- selection reason and score
- enabled flag
This lets the system answer: which pairs were selected, why were they selected, and what graph was actually submitted.
## 4. Production Input Package
Committed production input is represented by a prepared stack manifest. In the
current backend this file is:
```text
backend/runtime/timeseries_work/<run_id>/input/selected_stack_manifest.json
```
This file is not the same thing as a `TimeseriesStackPlan`. The plan is the
candidate pool and audit graph. The prepared stack is the smaller frozen set
submitted to a processor.
Schema:
```json
{
"schema": "insar.timeseries-stack/v1",
"prepared_stack_schema": "insar.prepared-sbas-stack/v1",
"manifest_role": "prepared_sbas_stack",
"mode": "sbas",
"plan_id": "tsp_...",
"prepared_stack_id": "pss_...",
"source_plan_id": "tsp_...",
"source_batch_id": "...",
"processor_code": "sarscape_sbas",
"aoi": {},
"candidate_pool_source": {},
"selection_params": {},
"scenes": [],
"network_edges": [],
"reference_date": "YYYYMMDD",
"production_contract": {
"input_policy": "prepared_stack_only",
"catalog_scan_allowed_after_prepare": false,
"scene_selection_frozen": true
},
"artifacts": {
"selected_network_edges_path_windows": "..."
},
"prepared_stack_validation": {},
"prepared_at_utc": "...",
"manifest_checksum": "..."
}
```
Rules:
- A production run consumes the prepared manifest, not `PsTaskItem.remark` and
not a fresh scan of the full radar catalog.
- The manifest is immutable after `prepare` completes, except for explicit
retry/re-prepare workflows.
- Processor-specific materialization is recorded in a separate processor manifest.
- Source data copying must include the manifest and graph.
### 4.1 Layered SBAS Input Model
The production model is now four layers:
1. Full radar inventory
- The long-lived scene catalog and pairing metric cache.
- It can be large and dirty/rebuilt over time.
2. Candidate time-series pool
- `TimeseriesStackPlanORM`, plan items, and plan edges.
- This is the large pool selected by AOI, date, orbit, baseline, overlap,
and network policy.
- It records why each scene and edge was selected.
3. Prepared SBAS stack
- `selected_stack_manifest.json` with
`prepared_stack_schema=insar.prepared-sbas-stack/v1`.
- Contains only the frozen scenes for this run.
- Writes `input/selected_network_edges.json` as a standalone artifact.
- Records validation results for scene files, graph count/date consistency,
DEM availability when required, and the no-catalog-scan production policy.
4. Processor execution
- SARscape `wf_sbas` consumes the prepared scene stack.
- System `network_edges` are mandatory as the planning/audit graph, but the
native `wf_sbas` path may rebuild the executable graph internally.
- When SARscape's actual graph can be extracted, it should be saved as
`actual_network_edges.json` and compared with `selected_network_edges.json`.
Backend enforcement:
- `prepare_run()` creates the prepared stack contract and validates it.
- `build_sarscape_processor_preflight()` refuses non-prepared manifests.
- `run_sarscape_sbas()` refuses non-prepared manifests and missing
`selected_network_edges.json`.
- `execute_template_workflow()` in the SARscape service has a second guard so
lower-level execution cannot accidentally run from a candidate pool.
## 5. Processor Boundary
Introduce a time-series processor interface:
```text
TimeseriesProcessor
check_available()
preflight(manifest)
build_workflow(run)
prepare_inputs(run)
execute_step(run, step_id)
export_publish_bundle(run)
```
Processor codes:
- `isce2_stack_mintpy`
- `sarscape_sbas`
The existing `timeseries_service` can remain the orchestration service, but processor-specific logic should move behind this interface.
## 6. SARscape SBAS Processor
SARscape SBAS should be a stack-level processor, not an extension of the D-InSAR pair engine.
Suggested steps:
1. `sarscape_preflight`
- Check ENVI, SARscape, taskengine, license, DEM, orbit pool, and output roots.
- Enumerate available SARscape SBAS/E-SBAS task names via `envipyengine`.
2. `sarscape_import`
- Import LT-1 scenes.
- Write `sarscape_import_manifest.json`.
3. `sarscape_connection_graph`
- Prefer the system-selected `network_edges`.
- If SARscape internally rebuilds the graph, export the actual graph as `actual_network_edges.json`.
4. `sarscape_interferogram_generation`
5. `sarscape_inversion`
- Generate time-series, velocity, coherence, and quality products.
6. `sarscape_geocode_export`
7. `export_publish_bundle`
8. `register_psinsar_product`
## 7. Result Contract
One SBAS run registers one `psinsar` product bundle.
Required bundle roles:
- stack manifest
- processor manifest
- selected network edges
- actual network edges if processor modified them
- velocity product
- time-series product
- temporal coherence or equivalent quality product
- geocoded rasters
- quicklooks
- logs
- processor reports
- product manifest
The catalog registers the publish manifest, not the transient work directory.
## 8. Delivery Phases
### Phase 1: Planning Boundary
- Stop auto-creating PS batches after search.
- Persist `TimeseriesStackPlanEdge`.
- Return edges from `/timeseries-plans/{plan_id}`.
- Add network thresholds to `PsRequest` with backward-compatible defaults.
### Phase 2: Manifest Boundary
- Add committed stack package creation.
- Generate immutable `stack_manifest.json`.
- Make the existing ISCE2/MintPy route consume the manifest.
### Phase 3: SARscape Discovery
- Add a SARscape SBAS task verifier script.
- Capture task names and required parameters per installed SARscape version.
- Add `sarscape_sbas` preflight endpoint.
Initial implementation points:
- `scripts/verify_sarscape_sbas_tasks.py`
- `POST /idl/inspect/sarscape-sbas`
- `POST /timeseries-production/sarscape-sbas/preflight`
- `python -m backend.app.services.envi_runner_cli --inspect-sarscape-sbas`
These entry points must stay read-only. They instantiate ENVI task definitions
and inspect parameters, but do not execute SBAS processing.
The time-series SARscape preflight endpoint builds a processor manifest from
the committed PS batch/stack plan context. It reports the selected network
edges, the SARscape task sequence, required publish roles, and current blockers.
At this phase it must return `ready_for_pipeline_design=true` when ENVI/SARscape
is discoverable, but `ready_for_execution=false` until a checked-in parameter
template and job handler are implemented.
Current implementation status:
- `sarscape_sbas` is a selectable time-series processor.
- The production UI defaults to `ENVI/SARscape SBAS` with `Preflight only`.
- `POST /timeseries-production/runs` accepts `processor_code` and
`execution_mode`.
- SARscape runs use workflow `psinsar_sarscape_sbas_chain`.
- Preflight-only SARscape runs execute `prepare` plus
`sarscape_processor_preflight`, then complete the task without launching the
long SARscape stack execution.
- Full execution is gated by `SARSCAPE_SBAS_ALLOW_EXECUTION=true` and a
`validated=true` parameter template at
`SARSCAPE_SBAS_PARAMETER_TEMPLATE_PATH`.
- The checked-in template at
`backend/templates/sarscape_sbas_parameter_template.example.json` is a
placeholder contract and is intentionally not executable.
Observed on the target workstation:
- Lightweight `Engine.tasks()` discovery succeeds.
- Static `.task` extraction succeeds without starting taskengine. The extractor is:
- `scripts/extract_sarscape_sbas_task_templates.py`
- The installed SARscape exposes native workflow metatasks:
- `wf_sbas`
- `wf_esbas`
- `wf_sbas` is an ENVI metatask at
`C:\Program Files\Harris\ENVI56\user_custom_code\wf_sbas.task`.
It is not listed by `Engine.tasks()` on this workstation, but
`Engine("ENVI").task("wf_sbas")` can instantiate it successfully. Discovery
therefore combines `Engine.tasks()` with static `.task` file detection.
It contains an embedded 11-node DAG:
- `SARscape_setting_output_folders`
- `SARsLoadPreferences`
- `SARsImportSarSelector`
- `ENVIEXTRACTELEMENTSFROMARRAYTASK`
- `SARscapeSuggestLooks`
- `SARsInSARStackSBASGenerateConnectionGraph`
- `SARsInSARStackSBASInterferogramGeneration`
- `SARsInSARStackSBASInversionStep1`
- `SARsInSARStackSBASInversionStep2`
- `SARsInSARStackSBASGeocode`
- `SARscapeEnviuriToShape`
- The static `wf_sbas.task` file contains 18 parameter entries including the
embedded `DAG` default. Live taskengine `QueryTask` exposes 17 callable
parameters; it does not require the caller to pass `DAG`.
- The core production inputs are:
- `INPUT_FILE_LIST`
- `SARSCAPE_PREFERENCE`
- `DEM_SARSCAPEDATA`
- `OUTPUT_FOLDER`
- `GEOCODE_RG_GRID_SIZE`
- `ESTIMATE_RESIDUAL_HEIGHT`
- `DISPLACEMENT_MODEL_TYPE`
- `OUTPUT_ENVI_CARTOGRAPHIC_SYSTEM`
- `wf_sbas` returns SBAS product handles:
- `DISPLACEMENT_SARSCAPEDATA`
- `DEM_OUT_SARSCAPEDATA`
- `CORRECTION_H_SARSCAPEDATA`
- `COHERENCE_SARSCAPEDATA`
- `ALOS_SARSCAPEDATA`
- `ILOS_SARSCAPEDATA`
- `VELOCITY_SARSCAPEDATA`
- `OUTPUT_SHAPES`
- The installed SARscape also exposes these stack tasks:
- `SARsInSARStackSBASGenerateConnectionGraph`
- `SARsInSARStackSBASInterferogramGeneration`
- `SARsInSARStackSBASInversionStep1`
- `SARsInSARStackSBASInversionStep2`
- `SARsInSARStackSBASGeocode`
- `SARsInSARStackSBASVariogram`
- `SARsInSARStackESBASInterferogramGeneration`
- `SARsInSARStackESBASInversion`
- `SARsInSARStackESBASGeocode`
- `SARsInSARConnectionGraphESBAS`
- Reading `.parameters` for stack SBAS tasks can hang taskengine. Parameter
discovery must therefore be optional, subprocess-isolated, and timeout-bound.
Processor implementation should use a checked-in task template or SARscape
help/SML-derived parameter contract rather than relying on live parameter
introspection at run time.
- Timeout cleanup must remove only taskengine processes spawned by the timed-out
inspection subprocess. Existing user-launched ENVI/taskengine sessions should
not be killed by name.
- SARscape/taskengine can create zero-byte `env_*.xyz` and `IDL*.tmp` files in
the process current working directory. ENVI runner cwd and temp variables must
point at `backend/runtime/idl_worker/envi_cwd`, not the repository root.
Root-level `env_*.xyz` and `IDL*.tmp` are disposable taskengine leftovers.
### Phase 3.5: SARscape Native Workflow Strategy
The short-term production strategy is to integrate SARscape through `wf_sbas`.
This is the lowest-risk ENVI/SARscape path because SARscape already wires import,
connection graph generation, interferogram generation, inversion, geocoding, and
shape export in one metatask DAG.
The backend template contract now supports two execution strategies:
- `native_workflow_metatask`
- Preferred first implementation.
- Executes `wf_sbas` once with the committed stack manifest converted into
`INPUT_FILE_LIST`, configured DEM, output folder, and basic SBAS options.
- Does not directly consume the system-selected `network_edges`.
- Requires post-run extraction of SARscape's actual connection graph for audit.
- `explicit_stack_tasks`
- Future controllable implementation.
- Executes `SARsInSARStackSBASGenerateConnectionGraph`,
`InterferogramGeneration`, `InversionStep1`, `InversionStep2`, and
`Geocode` as separate tasks.
- May allow tighter control of graph settings, but direct injection of the
system-selected edge list is not verified yet.
Current rule:
- `network_edges` remain mandatory in the stack manifest because they are the
system planning decision and task-dispatch audit record.
- When using `wf_sbas`, SARscape may rebuild the graph internally. The output
bundle must therefore contain both:
- `selected_network_edges.json`
- `actual_network_edges.json`, when it can be extracted from SARscape outputs
Current code points:
- `backend/app/services/envi_service.py`
- Discovers `wf_sbas`, `wf_esbas`, support tasks, and stack tasks.
- Cleans up only newly spawned `taskengine.exe` PIDs on timeout.
- Runs subprocess and in-process envipyengine calls from
`backend/runtime/idl_worker/envi_cwd` so taskengine temp files do not pollute
the project root.
- `backend/app/services/sarscape_sbas_service.py`
- Builds processor manifests with `execution_strategy`.
- Reports both native and explicit strategy availability.
- Requires `insar.prepared-sbas-stack/v1` before execution.
- Executes `native_workflow_metatask` only when the template is validated and
execution is explicitly enabled.
- `backend/app/services/timeseries_service.py`
- Treats `TimeseriesStackPlan` as the candidate pool.
- Creates `selected_stack_manifest.json` as the prepared stack in
`prepare_run()`.
- Writes `input/selected_network_edges.json` before SARscape preflight or
execution.
- Refuses SARscape preflight/execution when the prepared stack validation
fails.
- `backend/templates/sarscape_sbas_parameter_template.example.json`
- Records the `wf_sbas` parameter contract and DAG summary.
- Keeps `validated=false` until a controlled run validates parameters and
output capture.
- `scripts/extract_sarscape_sbas_task_templates.py`
- Regenerates the static parameter report from installed `.task` files.
Open engineering items:
- Confirm `wf_sbas.INPUT_FILE_LIST` accepts the same LT-1 `*.meta.xml` list used
by current SARscape import tasks.
- Confirm whether `DAG` must be passed explicitly or SARscape uses the embedded
default from `wf_sbas.task`.
- Locate SARscape's written connection graph or auxiliary processing file and
convert it into `actual_network_edges.json`.
- Map `VELOCITY_SARSCAPEDATA`, `DISPLACEMENT_SARSCAPEDATA`,
`COHERENCE_SARSCAPEDATA`, and `OUTPUT_SHAPES` into the unified `psinsar`
publish bundle.
- Decide later whether to invest in `explicit_stack_tasks` for strict graph
injection, depending on whether SARscape exposes a supported graph import or
connection-list parameter.
Smoke test on 2026-04-30:
- Applied the non-destructive `008_timeseries_stack_plan_edges.sql` migration.
- Backfilled two edges for test plan `tsp_d89bfc5bded744e6bf9b60c1` from
`pairing_metric_cache` because the plan was created before the edge table
existed.
- Ran SARscape SBAS preflight for batch
`e240a63a-5941-4a86-8aae-182a6bc95dae`.
- Result:
- `scene_count=3`
- `network_edge_count=2`
- `ready_for_pipeline_design=true`
- `ready_for_execution=false`
- `execution_strategy=native_workflow_metatask`
- `missing_required_tasks=[]`
- blockers are only `Template is not marked validated=true` and
`SARSCAPE_SBAS_ALLOW_EXECUTION is false`.
- Created a `preflight_only` run
`b7c2df45-a891-4ff7-b106-013e8d285fbd` and executed its `prepare` plus
`sarscape_processor_preflight` steps. This wrote
`selected_stack_manifest.json` and `sarscape_sbas_processor_manifest.json`
without launching the full SARscape SBAS pipeline.
- Dispatch verification for workflow
`eeaf1d82-7268-490c-9fb5-911a00a475c6` exposed a real workflow bug:
`workflow_service.mark_step_completed()` advanced downstream steps to
`READY`, but the database session has `autoflush=False`, so the immediate
`enqueue_ready_steps()` query did not see the new `READY` status.
`sarscape_processor_preflight` therefore stayed `READY` without a job.
- Fixed the dispatcher by flushing after `_advance_ready_steps()` and before
`enqueue_ready_steps()`.
- Verified the dispatcher fix in a rollback-only two-step workflow regression
check: completing step `a` immediately advanced step `b` to `RUNNING` and
created its queued job.
- Re-ran the controlled dispatch path for only this workflow:
- `TIMESERIES_PREPARE`: `COMPLETED`
- `TIMESERIES_SARSCAPE_PREFLIGHT`: `COMPLETED`
- workflow status: `COMPLETED`
- task status: `COMPLETED`, progress `100`
- run status: `PREPARED`
- no `TIMESERIES_RUN_SARSCAPE_SBAS` job or `run_sarscape_sbas` step was
created because execution mode was `preflight_only`.
- Root-level taskengine leftovers after the run:
- `env_*.xyz`: `0`
- `IDL*.tmp`: `0`
ENVI status now reports runner cwd as
`backend/runtime/idl_worker/envi_cwd`.
Parameter template validation on 2026-04-30:
- Initial live `Engine("ENVI").task("wf_sbas")` parameter inspection failed
with `ENVITASK: No task matches: wf_sbas`, even though the static
`wf_sbas.task` file was present.
- Root cause: SARscape installed `wf_sbas.task` under
`C:\Program Files\Harris\ENVI56\user_custom_code`, while taskengine only
auto-loads deployed custom tasks from `ENVI_CUSTOM_CODE`, the ENVI
`custom_code` directory, the application user directory, or IDL packages.
- Backend runner now sets `ENVI_CUSTOM_CODE` to the discovered SARscape
`user_custom_code` directory. This is process-local to the runner and does
not modify the machine-level environment.
- After the fix, live `wf_sbas` parameter inspection succeeds:
- `available=true`
- `parameter_count=17`
- required inputs: `INPUT_FILE_LIST`
- outputs: `OUTPUT_SHAPES`, `DISPLACEMENT_SARSCAPEDATA`,
`DEM_OUT_SARSCAPEDATA`, `CORRECTION_H_SARSCAPEDATA`,
`COHERENCE_SARSCAPEDATA`, `ALOS_SARSCAPEDATA`,
`ILOS_SARSCAPEDATA`, `VELOCITY_SARSCAPEDATA`
- Added repeatable validation script:
`scripts/validate_sarscape_sbas_template.py`.
- Validation report:
`backend/runtime/sarscape_sbas_template_validation_latest.json`.
- Current 3-scene validation result:
- `ok=true`
- validation scope: template contract only, no `task.execute()`
- manifest scene count: `3`
- network edge count: `2`
- `INPUT_FILE_LIST_count=3`
- scene `meta_path`, `tiff_path`, and folders all exist
- DEM base, `.sml`, and `.hdr` all exist
- remaining execution gate issue: checked-in template is still
`validated=false`
Prepared stack boundary implementation on 2026-04-30:
- Added `prepared_stack_schema=insar.prepared-sbas-stack/v1` to
`selected_stack_manifest.json`.
- Added `prepared_stack_id`, `source_plan_id`, `source_batch_id`,
`candidate_pool_source`, and `production_contract`.
- Added `input/selected_network_edges.json` as the frozen planning/audit graph
artifact.
- Added prepared stack validation for:
- scene count and dates
- required scene folder, TIFF, and metadata XML paths
- zero-size source files
- network edge count and edge date consistency
- SARscape DEM dependency when SARscape is the selected processor
- missing `selected_network_edges.json`
- SARscape processor preflight and execution now reject manifests that are not
prepared stacks. The lower-level SARscape executor repeats this guard before
calling any ENVI task.
Prepared stack UI/API update on 2026-04-30:
- Added read-only backend summary endpoint:
`GET /timeseries-production/runs/{run_id}/prepared-stack`.
- The endpoint reads only existing run artifacts and does not trigger catalog
scans, preflight, or SARscape execution.
- The summary reports:
- prepared stack state
- `prepared_stack_id`
- manifest and selected network edge artifact paths
- scene count and network edge count
- prepared stack validation result
- SARscape processor manifest readiness and blockers
- `TimeseriesProductionPanel` now shows a dedicated `Prepared SBAS Stack`
section in run details.
- The SARscape preflight card now states that batch preflight is against the
candidate pool, while production freezes a prepared stack before processor
execution.
- `usePairingLogic` now marks created PS batches as candidate time-series pools
in the planning context and logs that production will freeze a prepared SBAS
stack during `prepare`.
### Phase 4: SARscape Execution
- Implement the SARscape SBAS processor steps.
- Serialize taskengine execution through the existing ENVI lock.
- Persist step manifests and logs.
Initial execution skeleton is in place:
- `TIMESERIES_SARSCAPE_PREFLIGHT`
- `TIMESERIES_RUN_SARSCAPE_SBAS`
- `backend/app/services/sarscape_sbas_service.py`
The execution handler resolves template macros and calls `execute_envi_task`
only after the template is readable, structurally valid, marked
`validated=true`, required tasks are discoverable, and execution is explicitly
enabled.
### Phase 5: Unified Result Management
- Normalize ISCE2/MintPy and SARscape outputs into the same publish bundle roles.
- Keep processor-specific files as secondary assets.
- Show products by role in the UI, not by processor-specific filenames.
+213 -18
View File
@@ -4,9 +4,11 @@ import { getPsBatches } from './api/taskBatches';
import { useBatchStore } from './store'; import { useBatchStore } from './store';
import { import {
createTimeseriesRun, createTimeseriesRun,
getTimeseriesPreparedStack,
getTimeseriesRunDetail, getTimeseriesRunDetail,
listTimeseriesRuns, listTimeseriesRuns,
retryTimeseriesStep, retryTimeseriesStep,
runSarscapeSbasPreflight,
runTimeseriesPreflight, runTimeseriesPreflight,
runTimeseriesWslCheck, runTimeseriesWslCheck,
} from './api/timeseriesProduction'; } from './api/timeseriesProduction';
@@ -40,6 +42,15 @@ const STATUS_COLOR = {
PUBLISHED: '#16a34a', PUBLISHED: '#16a34a',
}; };
const PREPARED_STACK_STATE = {
not_prepared: { label: 'Not prepared', color: '#64748b' },
manifest_unreadable: { label: 'Manifest unreadable', color: '#dc2626' },
prepared_invalid: { label: 'Prepared invalid', color: '#dc2626' },
prepared: { label: 'Prepared', color: '#16a34a' },
processor_blocked: { label: 'Processor blocked', color: '#d97706' },
ready_for_execution: { label: 'Ready for execution', color: '#15803d' },
};
function formatDateTime(value) { function formatDateTime(value) {
if (!value) return '-'; if (!value) return '-';
try { try {
@@ -49,6 +60,36 @@ function formatDateTime(value) {
} }
} }
function StateBadge({ value }) {
const state = PREPARED_STACK_STATE[value] || { label: value || 'Unknown', color: '#64748b' };
return (
<span
style={{
display: 'inline-flex',
alignItems: 'center',
gap: 6,
padding: '2px 10px',
borderRadius: 999,
background: `${state.color}14`,
color: state.color,
fontSize: 12,
fontWeight: 600,
}}
>
<span
style={{
width: 7,
height: 7,
borderRadius: '50%',
background: state.color,
display: 'inline-block',
}}
/>
{state.label}
</span>
);
}
function StatusPill({ value }) { function StatusPill({ value }) {
const color = STATUS_COLOR[value] || '#64748b'; const color = STATUS_COLOR[value] || '#64748b';
return ( return (
@@ -153,6 +194,7 @@ export default function TimeseriesProductionPanel({ readOnly = false, onJobQueue
const [selectedBatchId, setSelectedBatchId] = useState(''); const [selectedBatchId, setSelectedBatchId] = useState('');
const [selectedRunId, setSelectedRunId] = useState(''); const [selectedRunId, setSelectedRunId] = useState('');
const [selectedRunDetail, setSelectedRunDetail] = useState(null); const [selectedRunDetail, setSelectedRunDetail] = useState(null);
const [preparedStackSummary, setPreparedStackSummary] = useState(null);
const [loading, setLoading] = useState(false); const [loading, setLoading] = useState(false);
const [detailLoading, setDetailLoading] = useState(false); const [detailLoading, setDetailLoading] = useState(false);
const [submitting, setSubmitting] = useState(false); const [submitting, setSubmitting] = useState(false);
@@ -160,6 +202,8 @@ export default function TimeseriesProductionPanel({ readOnly = false, onJobQueue
const [runName, setRunName] = useState(''); const [runName, setRunName] = useState('');
const [referenceDate, setReferenceDate] = useState(''); const [referenceDate, setReferenceDate] = useState('');
const [waterMaskMode, setWaterMaskMode] = useState('synthetic_fallback'); const [waterMaskMode, setWaterMaskMode] = useState('synthetic_fallback');
const [processorCode, setProcessorCode] = useState('sarscape_sbas');
const [executionMode, setExecutionMode] = useState('preflight_only');
const [notes, setNotes] = useState(''); const [notes, setNotes] = useState('');
const [wslChecking, setWslChecking] = useState(false); const [wslChecking, setWslChecking] = useState(false);
const [wslReport, setWslReport] = useState(null); const [wslReport, setWslReport] = useState(null);
@@ -205,12 +249,19 @@ export default function TimeseriesProductionPanel({ readOnly = false, onJobQueue
const loadRunDetail = useCallback(async runId => { const loadRunDetail = useCallback(async runId => {
if (!runId) { if (!runId) {
setSelectedRunDetail(null); setSelectedRunDetail(null);
setPreparedStackSummary(null);
return; return;
} }
setDetailLoading(true); setDetailLoading(true);
try { try {
const detail = await getTimeseriesRunDetail(runId); const [detail, stackSummary] = await Promise.all([
getTimeseriesRunDetail(runId),
getTimeseriesPreparedStack(runId).catch(error => ({
error: error?.response?.data?.detail || error.message || 'Prepared stack summary load failed',
})),
]);
setSelectedRunDetail(detail); setSelectedRunDetail(detail);
setPreparedStackSummary(stackSummary);
} catch (error) { } catch (error) {
setSelectedRunDetail({ setSelectedRunDetail({
error: error?.response?.data?.detail || error.message || '运行详情加载失败', error: error?.response?.data?.detail || error.message || '运行详情加载失败',
@@ -231,6 +282,7 @@ export default function TimeseriesProductionPanel({ readOnly = false, onJobQueue
message: error?.response?.data?.detail || error.message || 'WSL 检查失败', message: error?.response?.data?.detail || error.message || 'WSL 检查失败',
checks: [], checks: [],
}); });
setPreparedStackSummary(null);
} finally { } finally {
setWslChecking(false); setWslChecking(false);
} }
@@ -265,18 +317,29 @@ export default function TimeseriesProductionPanel({ readOnly = false, onJobQueue
} }
setPreflightLoading(true); setPreflightLoading(true);
try { try {
const report = await runTimeseriesPreflight({ const basePayload = {
batch_id: selectedBatchId, batch_id: selectedBatchId,
reference_date: referenceDate.trim() || null, reference_date: referenceDate.trim() || null,
water_mask_mode: waterMaskMode, };
}); const report = processorCode === 'sarscape_sbas'
? await runSarscapeSbasPreflight({
...basePayload,
include_task_discovery: true,
discovery_timeout_seconds: 120,
})
: await runTimeseriesPreflight({
...basePayload,
water_mask_mode: waterMaskMode,
});
setPreflightReport(report); setPreflightReport(report);
} catch (error) { } catch (error) {
const detail = error?.response?.data?.detail || error.message || '预检失败'; const detail = error?.response?.data?.detail || error.message || '预检失败';
setPreflightReport({ setPreflightReport({
overall_ok: false, overall_ok: false,
ready_for_pipeline_design: false,
batch_id: selectedBatchId, batch_id: selectedBatchId,
errors: [detail], errors: [detail],
blockers: [detail],
warnings: [], warnings: [],
checks: [], checks: [],
summary: {}, summary: {},
@@ -284,7 +347,7 @@ export default function TimeseriesProductionPanel({ readOnly = false, onJobQueue
} finally { } finally {
setPreflightLoading(false); setPreflightLoading(false);
} }
}, [referenceDate, selectedBatchId, waterMaskMode]); }, [processorCode, referenceDate, selectedBatchId, waterMaskMode]);
useEffect(() => { useEffect(() => {
loadBatches(); loadBatches();
@@ -312,7 +375,7 @@ export default function TimeseriesProductionPanel({ readOnly = false, onJobQueue
useEffect(() => { useEffect(() => {
setPreflightReport(null); setPreflightReport(null);
}, [selectedBatchId, referenceDate, waterMaskMode]); }, [selectedBatchId, referenceDate, waterMaskMode, processorCode, executionMode]);
const handleSubmit = async () => { const handleSubmit = async () => {
if (!selectedBatchId) { if (!selectedBatchId) {
@@ -327,6 +390,8 @@ export default function TimeseriesProductionPanel({ readOnly = false, onJobQueue
run_name: runName.trim() || null, run_name: runName.trim() || null,
reference_date: referenceDate.trim() || null, reference_date: referenceDate.trim() || null,
water_mask_mode: waterMaskMode, water_mask_mode: waterMaskMode,
processor_code: processorCode,
execution_mode: executionMode,
notes: notes.trim() || null, notes: notes.trim() || null,
}); });
setMessage(`运行已入队:${result.run_id} / task=${result.task_id}`); setMessage(`运行已入队:${result.run_id} / task=${result.task_id}`);
@@ -343,12 +408,29 @@ export default function TimeseriesProductionPanel({ readOnly = false, onJobQueue
const runData = selectedRunDetail?.run || null; const runData = selectedRunDetail?.run || null;
const linkedProduct = selectedRunDetail?.product || null; const linkedProduct = selectedRunDetail?.product || null;
const workflowSteps = selectedRunDetail?.workflow?.steps || []; const workflowSteps = selectedRunDetail?.workflow?.steps || [];
const isSarscapePreflight = preflightReport?.schema === 'insar.sarscape-sbas-preview/v1';
const preflightChecks = Array.isArray(preflightReport?.checks) ? preflightReport.checks : []; const preflightChecks = Array.isArray(preflightReport?.checks) ? preflightReport.checks : [];
const preflightErrors = Array.isArray(preflightReport?.errors) ? preflightReport.errors : []; const preflightErrors = Array.isArray(preflightReport?.errors)
? preflightReport.errors
: (Array.isArray(preflightReport?.blockers) ? preflightReport.blockers : []);
const preflightWarnings = Array.isArray(preflightReport?.warnings) ? preflightReport.warnings : []; const preflightWarnings = Array.isArray(preflightReport?.warnings) ? preflightReport.warnings : [];
const preflightSummary = preflightReport?.summary || {}; const preflightSummary = preflightReport?.summary || preflightReport?.stack_manifest || {};
const preflightOk = isSarscapePreflight
? !!preflightReport?.ready_for_pipeline_design
: !!preflightReport?.overall_ok;
const runPreflightQuality = runData?.quality_summary_json?.preflight || null; const runPreflightQuality = runData?.quality_summary_json?.preflight || null;
const runPublishValidation = runData?.quality_summary_json?.publish_validation || null; const runPublishValidation = runData?.quality_summary_json?.publish_validation || null;
const preparedStack = preparedStackSummary && !preparedStackSummary.error ? preparedStackSummary : null;
const preparedValidation = preparedStack?.validation || runData?.input_snapshot_json?.prepared_stack_validation || null;
const preparedBlockers = Array.isArray(preparedStack?.blockers)
? preparedStack.blockers
: (Array.isArray(preparedValidation?.blockers) ? preparedValidation.blockers : []);
const preparedWarnings = Array.isArray(preparedStack?.warnings)
? preparedStack.warnings
: (Array.isArray(preparedValidation?.warnings) ? preparedValidation.warnings : []);
const processorManifest = preparedStack?.processor_manifest || null;
const processorBlockers = Array.isArray(processorManifest?.blockers) ? processorManifest.blockers : [];
const showPreparedStack = !!(preparedStack || preparedStackSummary?.error || runData?.summary_json?.prepared_stack_id);
return ( return (
<div style={{ padding: '16px 0', width: '100%' }}> <div style={{ padding: '16px 0', width: '100%' }}>
@@ -382,10 +464,9 @@ export default function TimeseriesProductionPanel({ readOnly = false, onJobQueue
borderRadius: 6, borderRadius: 6,
}} }}
> >
当前接入实现为 SBAS现阶段已连通完整八步链路preparestack_prep_initialmaterialize 当前生产入口采用分层 SBAS 模型时序配对先形成候选大池提交 run 后由 prepare 冻结 prepared SBAS 小栈
stack_prep_refreshrun_isce2_stackrun_mintpy_sbasexport_publish_bundle ENVI/SARscape SBAS 后续只读取 prepared manifest selected_network_edges 审计图不再重新扫描全量数据
register_psinsar_product提交后系统会依次生成选栈 manifest物化 LT-1 SLC执行 ISCE2 ISCE2 + MintPy 路径仍沿用 stack_prepmaterializestackMintPypublishregister 链路
stack运行 MintPy SBAS导出 publish bundle并把结果注册进时序InSAR catalog
</div> </div>
{wslReport && ( {wslReport && (
<div <div
@@ -466,6 +547,34 @@ export default function TimeseriesProductionPanel({ readOnly = false, onJobQueue
<option value="local">local</option> <option value="local">local</option>
</select> </select>
</div> </div>
<div>
<div style={{ fontSize: 12, color: '#64748b', marginBottom: 4 }}>Processor</div>
<select
value={processorCode}
onChange={event => {
const next = event.target.value;
setProcessorCode(next);
setExecutionMode(next === 'sarscape_sbas' ? 'preflight_only' : 'full');
}}
disabled={readOnly || submitting}
style={{ width: '100%', padding: '6px 8px', borderRadius: 6, border: '1px solid #cbd5e1' }}
>
<option value="sarscape_sbas">ENVI/SARscape SBAS</option>
<option value="isce2_stack_mintpy">ISCE2 + MintPy</option>
</select>
</div>
<div>
<div style={{ fontSize: 12, color: '#64748b', marginBottom: 4 }}>Execution</div>
<select
value={executionMode}
onChange={event => setExecutionMode(event.target.value)}
disabled={readOnly || submitting}
style={{ width: '100%', padding: '6px 8px', borderRadius: 6, border: '1px solid #cbd5e1' }}
>
<option value="preflight_only">Preflight only</option>
<option value="full">Full execution</option>
</select>
</div>
</div> </div>
<div style={{ marginTop: 10 }}> <div style={{ marginTop: 10 }}>
@@ -542,16 +651,16 @@ export default function TimeseriesProductionPanel({ readOnly = false, onJobQueue
marginTop: 12, marginTop: 12,
padding: '10px 12px', padding: '10px 12px',
borderRadius: 6, borderRadius: 6,
border: `1px solid ${preflightReport.overall_ok ? '#bbf7d0' : '#fecaca'}`, border: `1px solid ${preflightOk ? '#bbf7d0' : '#fecaca'}`,
background: preflightReport.overall_ok ? '#f0fdf4' : '#fef2f2', background: preflightOk ? '#f0fdf4' : '#fef2f2',
}} }}
> >
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', gap: 12, marginBottom: 8, flexWrap: 'wrap' }}> <div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', gap: 12, marginBottom: 8, flexWrap: 'wrap' }}>
<div style={{ display: 'flex', alignItems: 'center', gap: 8, flexWrap: 'wrap' }}> <div style={{ display: 'flex', alignItems: 'center', gap: 8, flexWrap: 'wrap' }}>
<strong style={{ fontSize: 13, color: preflightReport.overall_ok ? '#166534' : '#991b1b' }}> <strong style={{ fontSize: 13, color: preflightOk ? '#166534' : '#991b1b' }}>
{preflightReport.overall_ok ? '预检通过' : '预检发现问题'} {preflightOk ? '预检通过' : '预检发现问题'}
</strong> </strong>
<QualityBadge ok={!!preflightReport.overall_ok} okLabel="可提交" failLabel="需处理" /> <QualityBadge ok={preflightOk} okLabel="可提交" failLabel="需处理" />
</div> </div>
<div style={{ fontSize: 11, color: '#475569' }}> <div style={{ fontSize: 11, color: '#475569' }}>
错误 {preflightErrors.length} / 告警 {preflightWarnings.length} 错误 {preflightErrors.length} / 告警 {preflightWarnings.length}
@@ -561,12 +670,16 @@ export default function TimeseriesProductionPanel({ readOnly = false, onJobQueue
<div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(180px, 1fr))', gap: 8, marginBottom: 8 }}> <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(180px, 1fr))', gap: 8, marginBottom: 8 }}>
<div style={{ padding: '8px 10px', borderRadius: 6, background: '#fff', border: '1px solid #e2e8f0', fontSize: 12 }}> <div style={{ padding: '8px 10px', borderRadius: 6, background: '#fff', border: '1px solid #e2e8f0', fontSize: 12 }}>
<div style={{ color: '#64748b', marginBottom: 4 }}>有效参考日期</div> <div style={{ color: '#64748b', marginBottom: 4 }}>有效参考日期</div>
<strong>{preflightReport.reference_date_effective || '-'}</strong> <strong>{preflightReport.reference_date_effective || preflightReport.reference_date || '-'}</strong>
</div> </div>
<div style={{ padding: '8px 10px', borderRadius: 6, background: '#fff', border: '1px solid #e2e8f0', fontSize: 12 }}> <div style={{ padding: '8px 10px', borderRadius: 6, background: '#fff', border: '1px solid #e2e8f0', fontSize: 12 }}>
<div style={{ color: '#64748b', marginBottom: 4 }}>场景规模</div> <div style={{ color: '#64748b', marginBottom: 4 }}>场景规模</div>
<strong>{preflightSummary.scene_count || 0} </strong> <strong>{preflightSummary.scene_count || 0} </strong>
</div> </div>
<div style={{ padding: '8px 10px', borderRadius: 6, background: '#fff', border: '1px solid #e2e8f0', fontSize: 12 }}>
<div style={{ color: '#64748b', marginBottom: 4 }}>Network edges</div>
<strong>{preflightReport.network_edge_count ?? preflightSummary.network_edge_count ?? 0}</strong>
</div>
<div style={{ padding: '8px 10px', borderRadius: 6, background: '#fff', border: '1px solid #e2e8f0', fontSize: 12 }}> <div style={{ padding: '8px 10px', borderRadius: 6, background: '#fff', border: '1px solid #e2e8f0', fontSize: 12 }}>
<div style={{ color: '#64748b', marginBottom: 4 }}>Stack Key</div> <div style={{ color: '#64748b', marginBottom: 4 }}>Stack Key</div>
<strong style={{ wordBreak: 'break-all' }}>{preflightSummary.stack_key || '-'}</strong> <strong style={{ wordBreak: 'break-all' }}>{preflightSummary.stack_key || '-'}</strong>
@@ -582,12 +695,19 @@ export default function TimeseriesProductionPanel({ readOnly = false, onJobQueue
<div><strong>Plan Strategy:</strong>{preflightReport.plan_strategy || preflightSummary.plan_strategy || '-'}</div> <div><strong>Plan Strategy:</strong>{preflightReport.plan_strategy || preflightSummary.plan_strategy || '-'}</div>
<div><strong>批次</strong>{preflightReport.batch_name || preflightReport.batch_id || '-'}</div> <div><strong>批次</strong>{preflightReport.batch_name || preflightReport.batch_id || '-'}</div>
<div><strong>批次状态</strong>{preflightReport.batch_status || '-'}</div> <div><strong>批次状态</strong>{preflightReport.batch_status || '-'}</div>
<div><strong>Processor:</strong>{preflightReport.processor_manifest?.processor_code || processorCode || '-'}</div>
<div><strong>水体掩膜</strong>{preflightReport.water_mask_mode || '-'}</div> <div><strong>水体掩膜</strong>{preflightReport.water_mask_mode || '-'}</div>
<div><strong>分组</strong>{preflightSummary.group_key || '-'}</div> <div><strong>分组</strong>{preflightSummary.group_key || '-'}</div>
<div><strong>源目录</strong>{preflightSummary.source_root_windows || '-'}</div> <div><strong>源目录</strong>{preflightSummary.source_root_windows || '-'}</div>
<div><strong>日期列表</strong>{(preflightSummary.stack_dates || []).join(', ') || '-'}</div> <div><strong>日期列表</strong>{(preflightSummary.stack_dates || []).join(', ') || '-'}</div>
</div> </div>
{isSarscapePreflight && (
<div style={{ marginTop: 8, padding: '8px 10px', borderRadius: 6, background: '#fff', border: '1px solid #bfdbfe', fontSize: 12, color: '#1e3a8a', lineHeight: 1.6 }}>
当前预检针对候选批次/候选图提交 run prepare 步骤会冻结一个 prepared SBAS stackSARscape 后续只读取这个 prepared manifest不再重新访问全量数据池
</div>
)}
{preflightErrors.length > 0 && ( {preflightErrors.length > 0 && (
<div style={{ marginTop: 8, padding: '8px 10px', borderRadius: 6, background: '#fff', border: '1px solid #fecaca', fontSize: 12, color: '#991b1b' }}> <div style={{ marginTop: 8, padding: '8px 10px', borderRadius: 6, background: '#fff', border: '1px solid #fecaca', fontSize: 12, color: '#991b1b' }}>
<div style={{ fontWeight: 600, marginBottom: 4 }}>错误</div> <div style={{ fontWeight: 600, marginBottom: 4 }}>错误</div>
@@ -692,6 +812,7 @@ export default function TimeseriesProductionPanel({ readOnly = false, onJobQueue
</span> </span>
</div> </div>
<div style={{ fontSize: 11, color: '#64748b' }}> <div style={{ fontSize: 11, color: '#64748b' }}>
{item.processor_code || '-'} /
{item.reference_date || '-'} / {item.stack_size || 0} / {formatDateTime(item.created_at)} {item.reference_date || '-'} / {item.stack_size || 0} / {formatDateTime(item.created_at)}
</div> </div>
</button> </button>
@@ -716,6 +837,7 @@ export default function TimeseriesProductionPanel({ readOnly = false, onJobQueue
</div> </div>
<div><strong>Stack Plan:</strong>{runData?.plan_id || '-'}</div> <div><strong>Stack Plan:</strong>{runData?.plan_id || '-'}</div>
<div><strong>Plan Strategy:</strong>{runData?.plan_strategy || '-'}</div> <div><strong>Plan Strategy:</strong>{runData?.plan_strategy || '-'}</div>
<div><strong>Processor:</strong>{runData?.processor_code || '-'} / {runData?.engine_code || '-'}</div>
<div><strong>运行标识</strong>{runData?.run_id || '-'}</div> <div><strong>运行标识</strong>{runData?.run_id || '-'}</div>
<div><strong>批次标识</strong>{runData?.batch_id || '-'}</div> <div><strong>批次标识</strong>{runData?.batch_id || '-'}</div>
<div><strong>参考日期</strong>{runData?.reference_date || '-'}</div> <div><strong>参考日期</strong>{runData?.reference_date || '-'}</div>
@@ -731,6 +853,79 @@ export default function TimeseriesProductionPanel({ readOnly = false, onJobQueue
<div><strong>创建时间</strong>{formatDateTime(runData?.created_at)}</div> <div><strong>创建时间</strong>{formatDateTime(runData?.created_at)}</div>
<div><strong>结束时间</strong>{formatDateTime(runData?.ended_at)}</div> <div><strong>结束时间</strong>{formatDateTime(runData?.ended_at)}</div>
<div><strong>输入日期</strong>{(runData?.input_snapshot_json?.stack_dates || []).join(', ') || '-'}</div> <div><strong>输入日期</strong>{(runData?.input_snapshot_json?.stack_dates || []).join(', ') || '-'}</div>
{showPreparedStack && (
<div style={{ marginTop: 8, paddingTop: 8, borderTop: '1px dashed #cbd5e1' }}>
<div style={{ display: 'flex', alignItems: 'center', justifyContent: 'space-between', gap: 8, marginBottom: 6, flexWrap: 'wrap' }}>
<strong>Prepared SBAS Stack</strong>
{preparedStackSummary?.error ? (
<StateBadge value="manifest_unreadable" />
) : (
<StateBadge value={preparedStack?.state || (runData?.summary_json?.prepared_stack_id ? 'prepared' : 'not_prepared')} />
)}
</div>
{preparedStackSummary?.error ? (
<div style={{ padding: '8px 10px', borderRadius: 6, background: '#fef2f2', border: '1px solid #fecaca', color: '#991b1b' }}>
{preparedStackSummary.error}
</div>
) : (
<>
<div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(170px, 1fr))', gap: 8 }}>
<div style={{ padding: '8px 10px', borderRadius: 6, background: '#f8fafc', border: '1px solid #e2e8f0' }}>
<div style={{ color: '#64748b', marginBottom: 4 }}>Prepared ID</div>
<strong style={{ wordBreak: 'break-all' }}>{preparedStack?.prepared_stack_id || runData?.summary_json?.prepared_stack_id || '-'}</strong>
</div>
<div style={{ padding: '8px 10px', borderRadius: 6, background: '#f8fafc', border: '1px solid #e2e8f0' }}>
<div style={{ color: '#64748b', marginBottom: 4 }}>Validation</div>
<QualityBadge ok={!!(preparedValidation?.ok ?? preparedStack?.prepared)} okLabel="OK" failLabel="Blocked" />
</div>
<div style={{ padding: '8px 10px', borderRadius: 6, background: '#f8fafc', border: '1px solid #e2e8f0' }}>
<div style={{ color: '#64748b', marginBottom: 4 }}>Scenes</div>
<strong>{preparedStack?.scene_count ?? runData?.stack_size ?? 0}</strong>
</div>
<div style={{ padding: '8px 10px', borderRadius: 6, background: '#f8fafc', border: '1px solid #e2e8f0' }}>
<div style={{ color: '#64748b', marginBottom: 4 }}>Network edges</div>
<strong>{preparedStack?.network_edge_count ?? runData?.input_snapshot_json?.network_edge_count ?? 0}</strong>
</div>
</div>
<div style={{ marginTop: 8, lineHeight: 1.7 }}>
<div><strong>Schema:</strong>{preparedStack?.prepared_stack_schema || runData?.summary_json?.prepared_stack_schema || '-'}</div>
<div><strong>Source plan:</strong>{preparedStack?.source_plan_id || runData?.plan_id || '-'}</div>
<div><strong>Source batch:</strong>{preparedStack?.source_batch_id || runData?.batch_id || '-'}</div>
<div><strong>Prepared manifest:</strong>{preparedStack?.manifest_path_windows || runData?.manifest_path_windows || '-'}</div>
<div><strong>Selected network edges:</strong>{preparedStack?.selected_network_edges_path_windows || runData?.input_snapshot_json?.selected_network_edges_path_windows || '-'}</div>
<div><strong>Policy:</strong>{preparedStack?.production_contract?.input_policy || '-'} / catalog_scan_after_prepare={String(preparedStack?.production_contract?.catalog_scan_allowed_after_prepare ?? false)}</div>
</div>
{processorManifest && (
<div style={{ marginTop: 8, padding: '8px 10px', borderRadius: 6, background: processorManifest.ready_for_execution ? '#f0fdf4' : '#fffbeb', border: `1px solid ${processorManifest.ready_for_execution ? '#bbf7d0' : '#fde68a'}` }}>
<div style={{ display: 'flex', justifyContent: 'space-between', gap: 8, alignItems: 'center', flexWrap: 'wrap' }}>
<strong>SARscape processor</strong>
<QualityBadge ok={!!processorManifest.ready_for_execution} okLabel="Executable" failLabel="Blocked" />
</div>
<div style={{ marginTop: 4 }}><strong>Strategy:</strong>{processorManifest.execution_strategy || '-'}</div>
<div><strong>Template:</strong>{processorManifest.parameter_template?.validated ? 'validated' : 'not validated'}</div>
<div><strong>Execution enabled:</strong>{String(!!processorManifest.execution_enabled)}</div>
</div>
)}
{(preparedBlockers.length > 0 || processorBlockers.length > 0) && (
<div style={{ marginTop: 8, padding: '8px 10px', borderRadius: 6, background: '#fff7ed', border: '1px solid #fed7aa', color: '#9a3412' }}>
<div style={{ fontWeight: 600, marginBottom: 4 }}>Blockers</div>
{[...preparedBlockers, ...processorBlockers].map((item, index) => (
<div key={`prepared-blocker-${index}`}>{item}</div>
))}
</div>
)}
{preparedWarnings.length > 0 && (
<div style={{ marginTop: 8, padding: '8px 10px', borderRadius: 6, background: '#fffbeb', border: '1px solid #fde68a', color: '#92400e' }}>
<div style={{ fontWeight: 600, marginBottom: 4 }}>Warnings</div>
{preparedWarnings.map((item, index) => (
<div key={`prepared-warning-${index}`}>{item}</div>
))}
</div>
)}
</>
)}
</div>
)}
<div> <div>
<strong>轨道摘要</strong> <strong>轨道摘要</strong>
<div style={{ marginTop: 4 }}> <div style={{ marginTop: 4 }}>
+6
View File
@@ -9,11 +9,17 @@ export const runTimeseriesWslCheck = (payload = {}) =>
export const runTimeseriesPreflight = payload => export const runTimeseriesPreflight = payload =>
apiClient.post('/timeseries-production/preflight', payload).then(r => r.data); apiClient.post('/timeseries-production/preflight', payload).then(r => r.data);
export const runSarscapeSbasPreflight = payload =>
apiClient.post('/timeseries-production/sarscape-sbas/preflight', payload).then(r => r.data);
export const listTimeseriesRuns = (params = {}) => export const listTimeseriesRuns = (params = {}) =>
apiClient.get('/timeseries-production/runs', { params }).then(r => r.data); apiClient.get('/timeseries-production/runs', { params }).then(r => r.data);
export const getTimeseriesRunDetail = runId => export const getTimeseriesRunDetail = runId =>
apiClient.get(`/timeseries-production/runs/${encodeURIComponent(runId)}`).then(r => r.data); apiClient.get(`/timeseries-production/runs/${encodeURIComponent(runId)}`).then(r => r.data);
export const getTimeseriesPreparedStack = runId =>
apiClient.get(`/timeseries-production/runs/${encodeURIComponent(runId)}/prepared-stack`).then(r => r.data);
export const retryTimeseriesStep = (runId, payload) => export const retryTimeseriesStep = (runId, payload) =>
apiClient.post(`/timeseries-production/runs/${encodeURIComponent(runId)}/retry-step`, payload).then(r => r.data); apiClient.post(`/timeseries-production/runs/${encodeURIComponent(runId)}/retry-step`, payload).then(r => r.data);
+6 -3
View File
@@ -63,6 +63,8 @@ export default function usePairingLogic({
source: planId ? 'timeseries_stack_plan' : 'find_ps_timeseries', source: planId ? 'timeseries_stack_plan' : 'find_ps_timeseries',
plan_id: planId, plan_id: planId,
strategy: 'sbas_stack', strategy: 'sbas_stack',
pool_role: 'candidate_timeseries_pool',
production_contract: 'prepare_run_will_freeze_prepared_sbas_stack',
direction: batchDirection, direction: batchDirection,
display_group: direction, display_group: direction,
scene_count: stack.length, scene_count: stack.length,
@@ -70,6 +72,8 @@ export default function usePairingLogic({
stack_key: firstScene.stack_key || null, stack_key: firstScene.stack_key || null,
initial_overlap_threshold: psParams?.initial_overlap_threshold ?? null, initial_overlap_threshold: psParams?.initial_overlap_threshold ?? null,
final_overlap_threshold: psParams?.final_overlap_threshold ?? null, final_overlap_threshold: psParams?.final_overlap_threshold ?? null,
network_edge_count: firstScene.stack_network_edge_count ?? null,
network_warnings: firstScene.stack_network_warnings ?? [],
stack_dates: stack.map(item => item.imaging_date).filter(Boolean), stack_dates: stack.map(item => item.imaging_date).filter(Boolean),
}; };
try { try {
@@ -85,6 +89,7 @@ export default function usePairingLogic({
addLog('info', `时序批次已关联候选栈计划 ${planId}`); addLog('info', `时序批次已关联候选栈计划 ${planId}`);
} }
addLog('success', `已创建时序批次: ${batchId || batchDirection}`); addLog('success', `已创建时序批次: ${batchId || batchDirection}`);
addLog('info', '当前批次是候选时序池;正式生产会先执行 prepare,冻结 prepared SBAS 小栈后再进入处理器。');
if (batchId && sendToProduction) { if (batchId && sendToProduction) {
setBatchTab('ps'); setBatchTab('ps');
setSelectedBatchId(batchId); setSelectedBatchId(batchId);
@@ -293,9 +298,7 @@ export default function usePairingLogic({
if (Object.keys(processedResults).length > 0) { if (Object.keys(processedResults).length > 0) {
addLog('success', `成功找到 ${Object.keys(processedResults).length} 个时序InSAR候选栈。`); addLog('success', `成功找到 ${Object.keys(processedResults).length} 个时序InSAR候选栈。`);
setLeftPanelTab('ps_results'); setLeftPanelTab('ps_results');
for (const [direction, stack] of Object.entries(processedResults)) { addLog('info', '候选栈仅作为预览结果保留;需要生产时请手动保存批次或送入生产。');
await createPsBatch(direction, stack, { focusAfterCreate: false });
}
} else { } else {
addLog('info', '在给定的AOI和阈值下,未找到满足 SBAS 至少 3 景要求的时序影像栈。'); addLog('info', '在给定的AOI和阈值下,未找到满足 SBAS 至少 3 景要求的时序影像栈。');
setLeftPanelTab('ps_results'); setLeftPanelTab('ps_results');
@@ -0,0 +1,255 @@
"""Extract SARscape SBAS task template metadata from installed .task files.
This script is intentionally file-based. It does not start ENVI, taskengine, or
envipyengine, so it is safe to use on workstations where live
task.parameters inspection can hang.
Examples:
python scripts/extract_sarscape_sbas_task_templates.py --json
python scripts/extract_sarscape_sbas_task_templates.py --template --output tmp_sarscape_sbas_template.json
"""
from __future__ import annotations
import argparse
import json
import os
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional
DEFAULT_ENVI_ROOT = Path(os.environ.get("SARSCAPE_ENVI_ROOT", r"C:\Program Files\Harris\ENVI56"))
NATIVE_WORKFLOW_TASKS = ["wf_sbas", "wf_esbas"]
SUPPORT_TASKS = [
"SARscape_setting_output_folders",
"SARsLoadPreferences",
"SARsImportSarSelector",
"SARscapeSuggestLooks",
"SARscapeEnviuriToShape",
]
STACK_TASKS = [
"SARsInSARStackSBASGenerateConnectionGraph",
"SARsInSARStackSBASInterferogramGeneration",
"SARsInSARStackSBASInversionStep1",
"SARsInSARStackSBASInversionStep2",
"SARsInSARStackSBASGeocode",
"SARsInSARStackSBASVariogram",
]
ESBAS_TASKS = [
"SARsInSARConnectionGraphESBAS",
"SARsInSARStackESBASInterferogramGeneration",
"SARsInSARStackESBASInversion",
"SARsInSARStackESBASGeocode",
]
DEFAULT_TASKS = [
*NATIVE_WORKFLOW_TASKS,
*SUPPORT_TASKS,
*STACK_TASKS,
*ESBAS_TASKS,
]
def _parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Read SARscape SBAS .task files and emit a static parameter report."
)
parser.add_argument("--envi-root", default=str(DEFAULT_ENVI_ROOT), help="ENVI install root.")
parser.add_argument("--task", action="append", default=[], help="Task name to extract. May be repeated.")
parser.add_argument("--json", action="store_true", help="Print the extraction report as JSON.")
parser.add_argument("--template", action="store_true", help="Print a backend template skeleton.")
parser.add_argument("--output", default="", help="Optional output file for JSON/template output.")
return parser.parse_args()
def _candidate_paths(envi_root: Path, task_name: str) -> Iterable[Path]:
if task_name in NATIVE_WORKFLOW_TASKS:
yield envi_root / "user_custom_code" / f"{task_name}.task"
yield envi_root / "resource" / "templates" / "tasks" / "SARscape" / f"{task_name}.task"
yield envi_root / "resource" / "templates" / "tasks" / f"{task_name}.task"
yield envi_root / "user_custom_code" / f"{task_name}.task"
def _read_task(envi_root: Path, task_name: str) -> Dict[str, Any]:
for path in _candidate_paths(envi_root, task_name):
if path.is_file():
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except UnicodeDecodeError:
payload = json.loads(path.read_text(encoding="utf-8-sig"))
return {"ok": True, "path": str(path), "payload": payload}
return {"ok": False, "path": None, "payload": None, "error": "task file not found"}
def _choice_list(value: Any) -> Optional[Dict[str, Any]]:
if isinstance(value, dict):
return {str(key): item for key, item in value.items()}
return None
def _normalize_parameter(item: Dict[str, Any]) -> Dict[str, Any]:
parameter_type = str(item.get("parameterType") or "").strip()
required = bool(item.get("required")) or parameter_type.lower() == "required"
default = item.get("defaultValue", item.get("default", item.get("value")))
normalized: Dict[str, Any] = {
"name": str(item.get("name") or "").strip(),
"keyword": str(item.get("keyword") or item.get("name") or "").strip(),
"display_name": str(item.get("displayName") or item.get("display_name") or "").strip(),
"data_type": str(item.get("dataType") or item.get("type") or "").strip(),
"direction": str(item.get("direction") or "").strip().lower(),
"required": required,
}
if parameter_type:
normalized["parameter_type"] = parameter_type
if default is not None:
normalized["default"] = default
choices = _choice_list(item.get("choiceList") or item.get("choice_list"))
if choices:
normalized["choice_list"] = choices
description = str(item.get("description") or "").strip()
if description:
normalized["description"] = description
return normalized
def _dag_summary(payload: Dict[str, Any]) -> List[Dict[str, Any]]:
parameters = payload.get("parameters") if isinstance(payload.get("parameters"), list) else []
dag_param = next((item for item in parameters if item.get("name") == "DAG"), None)
dag = dag_param.get("default") if isinstance(dag_param, dict) else None
if not isinstance(dag, dict):
return []
summary: List[Dict[str, Any]] = []
for node_id, node in dag.items():
if not isinstance(node, dict):
continue
task_name = node.get("name")
if isinstance(task_name, dict):
task_name = task_name.get("base_class") or "<inline_task>"
summary.append(
{
"node_id": str(node_id),
"task_name": str(task_name or ""),
"external_input": node.get("external_input") or {},
"internal_input": node.get("internal_input") or {},
"static_input": node.get("static_input") or {},
"output": node.get("output") or {},
}
)
return summary
def build_report(envi_root: Path, task_names: List[str]) -> Dict[str, Any]:
tasks: List[Dict[str, Any]] = []
missing: List[str] = []
for task_name in task_names:
raw = _read_task(envi_root, task_name)
if not raw["ok"]:
missing.append(task_name)
tasks.append({"name": task_name, "available": False, "error": raw.get("error")})
continue
payload = raw["payload"] if isinstance(raw["payload"], dict) else {}
parameters = payload.get("parameters") if isinstance(payload.get("parameters"), list) else []
normalized_params = [
_normalize_parameter(item)
for item in parameters
if isinstance(item, dict) and str(item.get("name") or "").strip()
]
tasks.append(
{
"name": str(payload.get("name") or task_name),
"available": True,
"path": raw["path"],
"version": payload.get("version") or payload.get("revision"),
"display_name": payload.get("displayName") or payload.get("display_name"),
"base_class": payload.get("baseClass") or payload.get("base_class"),
"parameter_count": len(normalized_params),
"required_inputs": [
item["name"]
for item in normalized_params
if item.get("required") and item.get("direction") == "input"
],
"outputs": [
item["name"]
for item in normalized_params
if item.get("direction") == "output"
],
"parameters": normalized_params,
"dag": _dag_summary(payload),
}
)
return {
"schema": "insar.sarscape-task-template-extract/v1",
"generated_at_utc": datetime.utcnow().replace(microsecond=0).isoformat() + "Z",
"envi_root": str(envi_root),
"task_count": len(tasks),
"available_count": sum(1 for item in tasks if item.get("available")),
"missing": missing,
"tasks": tasks,
}
def build_template(report: Dict[str, Any]) -> Dict[str, Any]:
by_name = {str(item.get("name") or ""): item for item in report.get("tasks") or []}
wf_sbas = by_name.get("wf_sbas") or {}
stack_tasks = [by_name.get(name) or {"name": name, "available": False} for name in STACK_TASKS]
return {
"schema": "insar.sarscape-sbas-template/v1",
"template_name": "SARscape SBAS native wf_sbas template skeleton",
"sarscape_version_hint": "Extracted from installed ENVI/SARscape .task files",
"validated": False,
"execution_strategy": "native_workflow_metatask",
"source_report_schema": report.get("schema"),
"source_envi_root": report.get("envi_root"),
"native_workflow": {
"phase_id": "native_wf_sbas",
"task_name": "wf_sbas",
"source_task_file": wf_sbas.get("path"),
"parameters": {
"INPUT_FILE_LIST": "${scene_input_uris}",
"SARSCAPE_PREFERENCE": "Use actual preferences",
"DEM_SARSCAPEDATA": "${dem_sarscapedata}",
"OUTPUT_FOLDER": "${output_root}",
"GEOCODE_RG_GRID_SIZE": 10.0,
"ESTIMATE_RESIDUAL_HEIGHT": True,
"DISPLACEMENT_MODEL_TYPE": "linear",
},
"parameter_schema": wf_sbas.get("parameters") or [],
"dag": wf_sbas.get("dag") or [],
},
"tasks": [
{
"phase_id": str(item.get("name") or ""),
"task_name": str(item.get("name") or ""),
"enabled": False,
"source_task_file": item.get("path"),
"required_inputs": item.get("required_inputs") or [],
"outputs": item.get("outputs") or [],
"parameter_schema": item.get("parameters") or [],
"parameters": {},
}
for item in stack_tasks
],
}
def main() -> int:
args = _parse_args()
envi_root = Path(args.envi_root)
task_names = args.task or DEFAULT_TASKS
report = build_report(envi_root, task_names)
payload = build_template(report) if args.template else report
text = json.dumps(payload, ensure_ascii=False, indent=2)
if args.output:
output_path = Path(args.output)
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(text + "\n", encoding="utf-8")
if args.json or args.template or not args.output:
print(text)
return 0 if report.get("available_count") else 1
if __name__ == "__main__":
raise SystemExit(main())
+328
View File
@@ -0,0 +1,328 @@
"""Validate the SARscape SBAS parameter template without executing SBAS.
The validation scope is intentionally limited to the template contract:
- load the checked-in SARscape SBAS template
- inspect the native wf_sbas task parameters through taskengine
- resolve template macros against a selected stack manifest
- verify required inputs, parameter names, basic types, and source paths
It does not call task.execute() and does not run SARscape processing.
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List
def _repo_root() -> Path:
return Path(__file__).resolve().parents[1]
def _parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Validate SARscape SBAS template parameters without executing SARscape."
)
parser.add_argument(
"--stack-manifest",
required=True,
help="Path to selected_stack_manifest.json.",
)
parser.add_argument(
"--template",
default="",
help="Optional SARscape SBAS template path. Defaults to configured template.",
)
parser.add_argument(
"--output",
default="",
help="Optional output JSON report path.",
)
parser.add_argument(
"--timeout",
type=int,
default=120,
help="Task inspection timeout in seconds.",
)
parser.add_argument(
"--skip-live",
action="store_true",
help="Skip live wf_sbas parameter inspection.",
)
return parser.parse_args()
def _read_json(path: Path) -> Dict[str, Any]:
with path.open("r", encoding="utf-8") as fp:
payload = json.load(fp)
if not isinstance(payload, dict):
raise ValueError(f"JSON root must be an object: {path}")
return payload
def _path_exists(path_text: str) -> bool:
text = str(path_text or "").strip()
return bool(text) and os.path.exists(os.path.normpath(text))
def _dem_exists(dem: Dict[str, Any]) -> Dict[str, Any]:
url = str(dem.get("url") or "").replace("/", os.sep)
aux = [str(item or "").replace("/", os.sep) for item in dem.get("auxiliary_url") or []]
return {
"url": dem.get("url"),
"url_exists": _path_exists(url),
"auxiliary_url": dem.get("auxiliary_url") or [],
"auxiliary_exists": [_path_exists(item) for item in aux],
}
def _scene_path_report(scenes: List[Dict[str, Any]]) -> Dict[str, Any]:
rows = []
for index, scene in enumerate(scenes):
meta_path = str(scene.get("meta_path") or "").strip()
tiff_path = str(scene.get("tiff_path") or "").strip()
folder_path = str(scene.get("folder_path") or "").strip()
rows.append(
{
"index": index,
"imaging_date": scene.get("imaging_date"),
"meta_path": meta_path,
"meta_exists": _path_exists(meta_path),
"tiff_path": tiff_path,
"tiff_exists": _path_exists(tiff_path),
"folder_path": folder_path,
"folder_exists": _path_exists(folder_path),
}
)
return {
"scene_count": len(scenes),
"missing_meta_count": sum(1 for item in rows if not item["meta_exists"]),
"missing_tiff_count": sum(1 for item in rows if not item["tiff_exists"]),
"missing_folder_count": sum(1 for item in rows if not item["folder_exists"]),
"scenes": rows,
}
def _validate_resolved_parameters(
parameters: Dict[str, Any],
live_input_names: List[str],
live_required_inputs: List[str],
live_choice_lists: Dict[str, List[Any]],
) -> List[str]:
issues: List[str] = []
live_input_set = set(live_input_names)
for key in parameters:
if live_input_set and key not in live_input_set:
issues.append(f"Template parameter is not a live wf_sbas input: {key}")
for key in live_required_inputs:
value = parameters.get(key)
if value is None or value == "" or value == []:
issues.append(f"Required live wf_sbas input is missing or empty: {key}")
input_files = parameters.get("INPUT_FILE_LIST")
if not isinstance(input_files, list) or not input_files:
issues.append("INPUT_FILE_LIST must resolve to a non-empty list.")
elif any(not isinstance(item, str) or not item.strip() for item in input_files):
issues.append("INPUT_FILE_LIST contains an empty or non-string item.")
output_folder = parameters.get("OUTPUT_FOLDER")
if output_folder is not None and not isinstance(output_folder, str):
issues.append("OUTPUT_FOLDER must resolve to a string path.")
dem = parameters.get("DEM_SARSCAPEDATA")
if dem is not None:
if not isinstance(dem, dict):
issues.append("DEM_SARSCAPEDATA must resolve to a SARSCAPEDATA object.")
elif dem.get("factory") != "ENVISARscapedata":
issues.append("DEM_SARSCAPEDATA.factory must be ENVISARscapedata.")
if "GEOCODE_RG_GRID_SIZE" in parameters and not isinstance(
parameters.get("GEOCODE_RG_GRID_SIZE"),
(int, float),
):
issues.append("GEOCODE_RG_GRID_SIZE must be numeric.")
if "ESTIMATE_RESIDUAL_HEIGHT" in parameters and not isinstance(
parameters.get("ESTIMATE_RESIDUAL_HEIGHT"),
bool,
):
issues.append("ESTIMATE_RESIDUAL_HEIGHT must be boolean.")
for key, choices in live_choice_lists.items():
if key in parameters and choices and parameters[key] not in choices:
issues.append(f"{key} is not in live choice list: {parameters[key]}")
return issues
def main() -> int:
repo_root = _repo_root()
if str(repo_root) not in sys.path:
sys.path.insert(0, str(repo_root))
from backend.app.config import ensure_project_env_loaded
from backend.app.services import envi_service
from backend.app.services.sarscape_sbas_service import (
NATIVE_WORKFLOW_TASK,
_resolve_template_value,
_scene_input_uris,
default_parameter_template_path,
load_parameter_template,
)
ensure_project_env_loaded()
args = _parse_args()
manifest_path = Path(args.stack_manifest).resolve()
template_path = Path(args.template).resolve() if args.template else Path(default_parameter_template_path()).resolve()
stack_manifest = _read_json(manifest_path)
template_status = load_parameter_template(str(template_path))
template = template_status.get("template") if isinstance(template_status.get("template"), dict) else {}
native_workflow = template.get("native_workflow") if isinstance(template.get("native_workflow"), dict) else {}
task_name = str(native_workflow.get("task_name") or NATIVE_WORKFLOW_TASK).strip()
live_report: Dict[str, Any] = {"skipped": True}
live_task: Dict[str, Any] = {}
if not args.skip_live:
live_report = envi_service.inspect_sarscape_sbas_tasks_subprocess(
[task_name],
include_parameters=True,
timeout_seconds=max(10, int(args.timeout or 120)),
)
live_task = next(
(
item
for item in live_report.get("tasks") or []
if str(item.get("name") or "") == task_name
),
{},
)
scenes = stack_manifest.get("scenes") if isinstance(stack_manifest.get("scenes"), list) else []
work_root = Path(stack_manifest.get("proposed_scratch_windows") or manifest_path.parents[1]).resolve()
output_root = work_root / "sarscape_sbas_template_validation_output"
network_edges_path = output_root / "selected_network_edges.json"
context = {
"${work_root}": str(work_root),
"${output_root}": str(output_root),
"${selected_stack_manifest}": str(manifest_path),
"${selected_network_edges}": str(network_edges_path),
"${scene_meta_paths}": [
str(item.get("meta_path"))
for item in scenes
if str(item.get("meta_path") or "").strip()
],
"${scene_input_uris}": _scene_input_uris(scenes),
"${scene_folder_paths}": [
str(item.get("folder_path"))
for item in scenes
if str(item.get("folder_path") or "").strip()
],
"${selection_params}": stack_manifest.get("selection_params") or {},
"${dem_sarscapedata}": envi_service._build_sarscapedata(envi_service.DEM_BASE_FILE), # noqa: SLF001
}
resolved_parameters = _resolve_template_value(native_workflow.get("parameters") or {}, context)
live_parameters = live_task.get("parameters") if isinstance(live_task.get("parameters"), list) else []
live_choice_lists = {
str(item.get("name")): list(item.get("choice_list") or [])
for item in live_parameters
if isinstance(item, dict) and isinstance(item.get("choice_list"), list)
}
issues: List[str] = []
execution_gate_issues: List[str] = []
for item in template_status.get("errors") or []:
text = str(item)
if text == "Template is not marked validated=true.":
execution_gate_issues.append(text)
else:
issues.append(text)
if not bool(template_status.get("readable")):
issues.append("Template is not readable.")
if not args.skip_live:
if not bool(live_report.get("ok")):
issues.append("Live wf_sbas parameter inspection failed.")
if not bool(live_task.get("available")):
issues.append("Live wf_sbas task is not available to taskengine.")
issues.extend(
_validate_resolved_parameters(
resolved_parameters,
list(live_task.get("input_names") or []),
list(live_task.get("required_input_names") or []),
live_choice_lists,
)
)
scene_report = _scene_path_report(scenes)
if scene_report["missing_meta_count"]:
issues.append("One or more scene meta_path files are missing.")
dem_report = _dem_exists(resolved_parameters.get("DEM_SARSCAPEDATA") or {})
if not dem_report["url_exists"] and not all(dem_report["auxiliary_exists"]):
issues.append("DEM SARSCAPEDATA path or auxiliary files are missing.")
report = {
"schema": "insar.sarscape-sbas-template-validation/v1",
"created_at_utc": datetime.utcnow().replace(microsecond=0).isoformat() + "Z",
"ok": not issues,
"validation_scope": "template_contract_only_no_task_execute",
"issues": issues,
"execution_gate_issues": execution_gate_issues,
"template": {
"path": str(template_path),
"validated_flag": bool(template_status.get("validated")),
"execution_strategy": template_status.get("execution_strategy"),
"native_workflow_task": task_name,
"errors": template_status.get("errors") or [],
},
"live_task": {
"skipped": bool(args.skip_live),
"ok": bool(live_report.get("ok")) if not args.skip_live else None,
"available": bool(live_task.get("available")) if live_task else None,
"parameter_count": live_task.get("parameter_count"),
"input_names": live_task.get("input_names") or [],
"required_input_names": live_task.get("required_input_names") or [],
"output_names": live_task.get("output_names") or [],
"error": live_task.get("error"),
},
"environment": {
"runner_cwd": envi_service.get_envi_runner_cwd(),
"envi_custom_code": envi_service.get_envi_runner_env().get("ENVI_CUSTOM_CODE"),
"dem_base_file": envi_service.DEM_BASE_FILE,
},
"stack_manifest": {
"path": str(manifest_path),
"scene_count": len(scenes),
"network_edge_count": len(stack_manifest.get("network_edges") or []),
"reference_date": stack_manifest.get("reference_date"),
"processor_code": stack_manifest.get("processor_code"),
},
"resolved_parameters": {
"keys": sorted(resolved_parameters.keys()),
"INPUT_FILE_LIST_count": len(resolved_parameters.get("INPUT_FILE_LIST") or []),
"OUTPUT_FOLDER": resolved_parameters.get("OUTPUT_FOLDER"),
"GEOCODE_RG_GRID_SIZE": resolved_parameters.get("GEOCODE_RG_GRID_SIZE"),
"ESTIMATE_RESIDUAL_HEIGHT": resolved_parameters.get("ESTIMATE_RESIDUAL_HEIGHT"),
"DISPLACEMENT_MODEL_TYPE": resolved_parameters.get("DISPLACEMENT_MODEL_TYPE"),
"DEM_SARSCAPEDATA": dem_report,
},
"scene_paths": scene_report,
}
if args.output:
output_path = Path(args.output).resolve()
else:
output_path = repo_root / "backend" / "runtime" / "sarscape_sbas_template_validation_latest.json"
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(report, ensure_ascii=False, indent=2))
return 0 if report["ok"] else 1
if __name__ == "__main__":
raise SystemExit(main())
+100
View File
@@ -0,0 +1,100 @@
"""Inspect likely SARscape SBAS/E-SBAS ENVI task names.
This script does not execute processing tasks. It only asks envipyengine to
instantiate task definitions and read their parameters.
Examples:
python scripts/verify_sarscape_sbas_tasks.py
python scripts/verify_sarscape_sbas_tasks.py --task SARsInSARStackSBASGenerateConnectionGraph --parameters
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
def _repo_root() -> Path:
return Path(__file__).resolve().parents[1]
def _parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Inspect SARscape SBAS/E-SBAS ENVI task availability."
)
parser.add_argument(
"--task",
action="append",
default=[],
help="Task name to inspect. May be repeated. Defaults to built-in candidates.",
)
parser.add_argument(
"--json",
action="store_true",
help="Print the full JSON report.",
)
parser.add_argument(
"--parameters",
action="store_true",
help="Also inspect task parameters. This can be slow for some SARscape tasks.",
)
parser.add_argument(
"--timeout",
type=int,
default=120,
help="Timeout in seconds when --parameters is used.",
)
return parser.parse_args()
def main() -> int:
repo_root = _repo_root()
if str(repo_root) not in sys.path:
sys.path.insert(0, str(repo_root))
from backend.app.config import ensure_project_env_loaded
from backend.app.services.envi_service import (
inspect_sarscape_sbas_tasks,
inspect_sarscape_sbas_tasks_subprocess,
)
ensure_project_env_loaded()
args = _parse_args()
if args.parameters:
report = inspect_sarscape_sbas_tasks_subprocess(
args.task or None,
include_parameters=True,
timeout_seconds=max(10, int(args.timeout or 120)),
)
else:
report = inspect_sarscape_sbas_tasks(args.task or None)
if args.json:
print(json.dumps(report, ensure_ascii=False, indent=2))
else:
print("SARscape SBAS/E-SBAS task inspection")
print(f"ok: {report.get('ok')}")
print(f"candidate_source: {report.get('candidate_source')}")
print(f"include_parameters: {report.get('include_parameters')}")
print(f"available: {report.get('available_count')} / {report.get('task_count')}")
error = str(report.get("error") or "").strip()
if error:
print(f"error: {error}")
print()
for item in report.get("tasks") or []:
status = "OK" if item.get("available") else "MISS"
print(f"[{status}] {item.get('name')}")
if item.get("available"):
required = item.get("required_input_names") or []
outputs = item.get("output_names") or []
print(f" required inputs: {required}")
print(f" outputs: {outputs}")
else:
print(f" error: {item.get('error')}")
return 0 if report.get("ok") else 1
if __name__ == "__main__":
raise SystemExit(main())