Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve pairing planning and statistics visibility

Browse Source
This commit is contained in:
Harmon
2026-07-07 01:04:47 +08:00
parent 2da2121829
commit 36a406ae49
22 changed files with 2085 additions and 459 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/models/schemas.py
+9 -2
View File
@@ -284,7 +284,7 @@ class PairingRequest(BaseModel):
slave_date_to: Optional[str] = Field(default=None, pattern=r'^\d{8}$|^$')
# === 配对策略（新增） ===
strategy: str = Field(default="dinsar_production", pattern=r'^dinsar_production$')
strategy: str = Field(default="dinsar_production", pattern=r'^(dinsar_production|dinsar_province_coverage)$')
num_connections: int = Field(default=1, ge=1, le=10)
reference_image_id: Optional[int] = None
@@ -307,7 +307,8 @@ class PairingRequest(BaseModel):
normalized['overlap_threshold'] = normalized['pair_footprint_overlap_min_ratio']
if normalized.get('footprint_center_distance_max_meters') not in (None, ''):
normalized['spatial_baseline_max_meters'] = normalized['footprint_center_distance_max_meters']
normalized['strategy'] = 'dinsar_production'
if normalized.get('strategy') not in {'dinsar_production', 'dinsar_province_coverage'}:
normalized['strategy'] = 'dinsar_production'
return normalized
@field_validator(
@@ -390,6 +391,11 @@ class RadarPair(BaseModel):
selection_strategy: Optional[str] = None
selection_score: Optional[float] = None
selection_reason: Optional[str] = None
coverage_rank: Optional[int] = None
aoi_new_area_ratio: Optional[float] = None
aoi_pair_area_ratio: Optional[float] = None
aoi_coverage_ratio_after: Optional[float] = None
effective_coverage_geojson: Optional[Dict[str, Any]] = None
time_baseline_days: int
spatial_baseline_meters: float
scene_center_distance_meters: Optional[float] = None
@@ -416,6 +422,7 @@ class PairingResponse(BaseModel):
network_run_id: Optional[str] = None
candidate_count: int = 0
selected_edge_count: int = 0
coverage: Optional[Dict[str, Any]] = None
class PsRequest(BaseModel):
backend/app/routers/ops_maintenance.py
+1 -6
View File
@@ -17,12 +17,7 @@ router = APIRouter(prefix="/ops-maintenance", tags=["ops-maintenance"])
class CleanupRequest(BaseModel):
confirm: bool = False
delete_task_records: bool = True
delete_logs: bool = True
delete_production_records: bool = True
delete_result_products: bool = True
delete_production_dirs: bool = True
delete_task_pool_dir: bool = True
delete_landsar_work_dir: bool = True
@router.get("/tasks")
backend/app/routers/pairing.py
+61
View File
@@ -329,6 +329,7 @@ async def find_pairs_endpoint(
network_run_id=pairing_metadata.get("network_run_id"),
candidate_count=int(pairing_metadata.get("candidate_count") or 0),
selected_edge_count=int(pairing_metadata.get("selected_edge_count") or 0),
coverage=None,
)
except Exception as e:
if isinstance(e, HTTPException):
@@ -339,6 +340,66 @@ async def find_pairs_endpoint(
raise HTTPException(status_code=500, detail="处理 AOI 或查找干涉对时发生错误，请查看后端日志")
@router.post("/pairing/coverage-plan", response_model=PairingResponse)
async def build_coverage_pairing_plan_endpoint(
params: PairingRequest = Depends(get_pairing_request_from_form),
target_date_from: str = Form(...),
target_date_to: str = Form(...),
extension_days: int = Form(15),
max_pairs: int = Form(200),
target_coverage_ratio: float = Form(0.98),
min_new_coverage_ratio: float = Form(0.0005),
files: Optional[List[UploadFile]] = File(None),
aoi_geojson: Optional[str] = Form(None),
require_orbit_data: bool = Form(True),
db: AsyncSession = Depends(get_db),
):
try:
resolved_aoi = await _parse_aoi_from_files(files)
if resolved_aoi is None:
resolved_aoi = _parse_aoi_geojson_form_value(aoi_geojson)
if files and resolved_aoi is None:
raise HTTPException(status_code=400, detail="Uploaded files must include a valid SHP or GeoJSON AOI.")
aoi_wkt = resolved_aoi[0] if resolved_aoi else None
response_aoi_geojson = resolved_aoi[1] if resolved_aoi else None
if not aoi_wkt:
raise HTTPException(status_code=400, detail="区域空间覆盖规划必须选择行政区或上传 AOI。")
coverage_params = params.model_copy(update={"strategy": "dinsar_province_coverage"})
pairs, runtime_warnings, pairing_metadata = await spatial_service.find_dinsar_coverage_pairs(
db,
coverage_params,
target_date_from=target_date_from,
target_date_to=target_date_to,
extension_days=extension_days,
max_pairs=max_pairs,
target_coverage_ratio=target_coverage_ratio,
min_new_coverage_ratio=min_new_coverage_ratio,
aoi_wkt=aoi_wkt,
require_orbit_data=require_orbit_data,
)
return PairingResponse(
pairs=pairs,
aoi_geojson=response_aoi_geojson,
warnings=runtime_warnings,
fallback_used=bool(pairing_metadata.get("fallback_used")),
degraded=bool(pairing_metadata.get("degraded")),
policy_version=pairing_metadata.get("policy_version"),
network_run_id=pairing_metadata.get("network_run_id"),
candidate_count=int(pairing_metadata.get("candidate_count") or 0),
selected_edge_count=int(pairing_metadata.get("selected_edge_count") or 0),
coverage=pairing_metadata.get("coverage"),
)
except Exception as e:
if isinstance(e, HTTPException):
raise e
if isinstance(e, (RuntimeError, ValueError)):
raise HTTPException(status_code=409, detail=str(e))
logger.exception("Failed to build D-InSAR coverage pairing plan")
raise HTTPException(status_code=500, detail="Failed to build D-InSAR coverage pairing plan.")
@router.post("/find-ps-timeseries", response_model=Dict[str, List[RadarData]])
async def find_ps_timeseries_endpoint(
params: PsRequest = Depends(get_ps_request_from_form),
backend/app/routers/stats.py
+221 -52
View File
@@ -11,6 +11,8 @@ from typing import Any, Dict, Optional
logger = logging.getLogger(__name__)
SOURCE_METADATA_MANAGED_FORMATS = ("LT1_ARCHIVE", "S1_ZIP")
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import distinct, func, text
from sqlalchemy.ext.asyncio import AsyncSession
@@ -51,7 +53,6 @@ from ..services.data_service import data_service
from ..services.dinsar_read_service import dinsar_read_service
from ..services.pairing_state_service import pairing_state_service
from ..services.admin_region_lookup_service import _build_region_path, _load_region_records
from ..utils import find_xml_file
from . import dependencies as _deps
from .dependencies import _get_current_user
@@ -80,6 +81,10 @@ def _ratio(numerator: int, denominator: int) -> float:
return round(float(numerator) / float(denominator), 4)
def _is_source_metadata_managed_format(value: Any) -> bool:
return str(value or "").strip().upper() in SOURCE_METADATA_MANAGED_FORMATS
def _family_label(value: Any) -> str:
text = str(value or "").strip().upper()
if text in {"LT1", "LT-1", "LUTAN", "LUTAN1"}:
@@ -102,6 +107,14 @@ def _month_from_yyyymmdd(value: Any) -> Optional[str]:
return None
def _compact_yyyymmdd(value: Any) -> Optional[str]:
text = str(value or "").strip()
digits = "".join(ch for ch in text if ch.isdigit())
if len(digits) >= 8:
return digits[:8]
return None
def _month_from_datetime(value: Any) -> Optional[str]:
if not value:
return None
@@ -594,16 +607,36 @@ async def get_statistics_dashboard(
db,
select(func.count(SourceProductAssetORM.id)).where(SourceProductAssetORM.is_active == True),
)
metadata_source_total = await _scalar_count(
db,
select(func.count(SourceProductAssetORM.id)).where(
SourceProductAssetORM.is_active == True,
SourceProductAssetORM.source_format.in_(SOURCE_METADATA_MANAGED_FORMATS),
),
)
radar_total = await _scalar_count(db, select(func.count(RadarDataORM.id)))
metadata_asset_total = await _scalar_count(
db,
select(func.count(distinct(SourceMetadataDocumentORM.source_asset_id))),
select(func.count(distinct(SourceMetadataDocumentORM.source_asset_id))).where(
SourceMetadataDocumentORM.source_format.in_(SOURCE_METADATA_MANAGED_FORMATS),
),
)
metadata_doc_total = await _scalar_count(
db,
select(func.count(SourceMetadataDocumentORM.id)).where(
SourceMetadataDocumentORM.source_format.in_(SOURCE_METADATA_MANAGED_FORMATS),
),
)
geometry_total = await _scalar_count(
db,
select(func.count(SARSceneGeometryProfileORM.id)).where(
SARSceneGeometryProfileORM.source_format.in_(SOURCE_METADATA_MANAGED_FORMATS),
),
)
metadata_doc_total = await _scalar_count(db, select(func.count(SourceMetadataDocumentORM.id)))
geometry_total = await _scalar_count(db, select(func.count(SARSceneGeometryProfileORM.id)))
geometry_ready = await _scalar_count(
db,
select(func.count(SARSceneGeometryProfileORM.id)).where(
SARSceneGeometryProfileORM.source_format.in_(SOURCE_METADATA_MANAGED_FORMATS),
SARSceneGeometryProfileORM.metadata_quality == "READY",
SARSceneGeometryProfileORM.production_readiness == "READY",
),
@@ -633,6 +666,7 @@ async def get_statistics_dashboard(
for family, source_format, parse_status, count in source_group_rows.all():
family_label = _family_label(family)
status_label = _status_label(parse_status)
format_label = str(source_format or "UNKNOWN")
count_int = _safe_int(count)
family_bucket = source_by_family_map.setdefault(
family_label,
@@ -649,7 +683,6 @@ async def get_statistics_dashboard(
family_bucket["ready_count"] += count_int
else:
family_bucket["issue_count"] += count_int
format_label = str(source_format or "UNKNOWN")
family_bucket["formats"][format_label] = family_bucket["formats"].get(format_label, 0) + count_int
source_by_format.append(
{
@@ -673,19 +706,22 @@ async def get_statistics_dashboard(
geometry_rows = await db.execute(
select(
SARSceneGeometryProfileORM.satellite_family,
SARSceneGeometryProfileORM.source_format,
SARSceneGeometryProfileORM.metadata_quality,
SARSceneGeometryProfileORM.production_readiness,
func.count(SARSceneGeometryProfileORM.id),
)
.where(SARSceneGeometryProfileORM.source_format.in_(SOURCE_METADATA_MANAGED_FORMATS))
.group_by(
SARSceneGeometryProfileORM.satellite_family,
SARSceneGeometryProfileORM.source_format,
SARSceneGeometryProfileORM.metadata_quality,
SARSceneGeometryProfileORM.production_readiness,
)
.order_by(SARSceneGeometryProfileORM.satellite_family)
)
geometry_by_family_map: dict[str, dict[str, Any]] = {}
for family, metadata_quality, production_readiness, count in geometry_rows.all():
for family, source_format, metadata_quality, production_readiness, count in geometry_rows.all():
family_label = _family_label(family)
count_int = _safe_int(count)
bucket = geometry_by_family_map.setdefault(
@@ -1075,6 +1111,82 @@ async def get_statistics_dashboard(
issue_by_severity[severity_label] = issue_by_severity.get(severity_label, 0) + count_int
issue_by_code[code_label] = issue_by_code.get(code_label, 0) + count_int
missing_orbit_issue_rows = await db.execute(
select(AssetInventoryIssueORM.metadata_json)
.where(AssetInventoryIssueORM.status == "OPEN")
.where(AssetInventoryIssueORM.issue_code == "scene_missing_orbit")
)
orbit_missing_satellite_map: dict[str, dict[str, Any]] = {}
orbit_missing_top_dates: dict[tuple[str, str], int] = {}
orbit_missing_unknown_date_scene_count = 0
for (metadata_json,) in missing_orbit_issue_rows.all():
metadata = metadata_json if isinstance(metadata_json, dict) else {}
satellite = str(metadata.get("satellite") or "UNKNOWN").strip().upper() or "UNKNOWN"
ymd = _compact_yyyymmdd(metadata.get("imaging_date") or metadata.get("acquisition_start_time_utc"))
month = _month_from_yyyymmdd(ymd) if ymd else None
satellite_bucket = orbit_missing_satellite_map.setdefault(
satellite,
{
"satellite": satellite,
"affected_scene_count": 0,
"dates": defaultdict(int),
"months": defaultdict(lambda: {"affected_scene_count": 0, "dates": set()}),
"unknown_date_scene_count": 0,
},
)
satellite_bucket["affected_scene_count"] += 1
if not ymd:
satellite_bucket["unknown_date_scene_count"] += 1
orbit_missing_unknown_date_scene_count += 1
continue
satellite_bucket["dates"][ymd] += 1
orbit_missing_top_dates[(satellite, ymd)] = orbit_missing_top_dates.get((satellite, ymd), 0) + 1
if month:
month_bucket = satellite_bucket["months"][month]
month_bucket["affected_scene_count"] += 1
month_bucket["dates"].add(ymd)
orbit_missing_by_satellite: list[dict[str, Any]] = []
for satellite, bucket in sorted(orbit_missing_satellite_map.items()):
date_items = [
{"date": date_text, "affected_scene_count": _safe_int(count)}
for date_text, count in sorted(bucket["dates"].items())
]
month_items = [
{
"month": month,
"affected_scene_count": _safe_int(month_bucket["affected_scene_count"]),
"missing_orbit_date_count": len(month_bucket["dates"]),
}
for month, month_bucket in sorted(bucket["months"].items())
]
orbit_missing_by_satellite.append(
{
"satellite": satellite,
"affected_scene_count": _safe_int(bucket["affected_scene_count"]),
"missing_orbit_date_count": len(date_items),
"unknown_date_scene_count": _safe_int(bucket["unknown_date_scene_count"]),
"first_missing_date": date_items[0]["date"] if date_items else None,
"last_missing_date": date_items[-1]["date"] if date_items else None,
"months": month_items,
"dates": date_items,
}
)
orbit_missing_summary = {
"affected_scene_count": sum(item["affected_scene_count"] for item in orbit_missing_by_satellite),
"missing_orbit_date_count": sum(item["missing_orbit_date_count"] for item in orbit_missing_by_satellite),
"unknown_date_scene_count": orbit_missing_unknown_date_scene_count,
"by_satellite": orbit_missing_by_satellite,
"top_dates": [
{"satellite": satellite, "date": date_text, "affected_scene_count": _safe_int(count)}
for (satellite, date_text), count in sorted(
orbit_missing_top_dates.items(),
key=lambda item: (-item[1], item[0][0], item[0][1]),
)[:20]
],
}
inventory_state_rows = await db.execute(
select(
AssetInventoryStateORM.inventory_type,
@@ -1114,16 +1226,24 @@ async def get_statistics_dashboard(
avg_duration_seconds = round(sum(duration_seconds) / len(duration_seconds), 1) if duration_seconds else None
selected_orbit_rate = _ratio(selected_orbit_bindings, orbit_required_total)
geometry_ready_rate = _ratio(geometry_ready, source_total)
metadata_ready_rate = _ratio(metadata_asset_total, source_total)
geometry_ready_rate = _ratio(geometry_ready, geometry_total)
metadata_ready_rate = _ratio(metadata_asset_total, metadata_source_total)
result_ready_total = sum(item["ready_count"] for item in results_by_catalog)
metadata_missing_count = max(0, metadata_source_total - metadata_asset_total)
geometry_not_ready_count = max(0, geometry_total - geometry_ready)
orbit_missing_gap_count = max(0, orbit_required_total - selected_orbit_bindings)
non_orbit_open_issue_total = sum(
count
for code, count in issue_by_code.items()
if code != "scene_missing_orbit"
)
risk_count = (
max(0, source_total - metadata_asset_total)
+ max(0, source_total - geometry_ready)
+ max(0, orbit_required_total - selected_orbit_bindings)
metadata_missing_count
+ geometry_not_ready_count
+ orbit_missing_gap_count
+ result_assets_missing
+ open_issue_total
+ non_orbit_open_issue_total
)
kpis = [
@@ -1140,7 +1260,7 @@ async def get_statistics_dashboard(
"label": "元数据入库率",
"value": round(metadata_ready_rate * 100, 1),
"unit": "%",
"note": f"{metadata_asset_total}/{source_total} 景已提取 XML/元数据",
"note": f"{metadata_asset_total}/{metadata_source_total} 景已提取 XML/元数据",
"tone": "success" if metadata_ready_rate >= 0.98 else "warning",
},
{
@@ -1148,7 +1268,7 @@ async def get_statistics_dashboard(
"label": "几何画像可用率",
"value": round(geometry_ready_rate * 100, 1),
"unit": "%",
"note": f"{geometry_ready}/{source_total} 景可用于覆盖统计",
"note": f"{geometry_ready}/{geometry_total} 景可用于覆盖统计",
"tone": "success" if geometry_ready_rate >= 0.95 else "warning",
},
{
@@ -1172,7 +1292,7 @@ async def get_statistics_dashboard(
"label": "待关注项",
"value": risk_count,
"unit": "项",
"note": f"开放问题 {open_issue_total}，缺失成果资产 {result_assets_missing}",
"note": f"缺精轨 {orbit_missing_gap_count}，开放问题 {open_issue_total}，缺失成果资产 {result_assets_missing}",
"tone": "danger" if risk_count else "success",
},
]
@@ -1186,20 +1306,23 @@ async def get_statistics_dashboard(
"source_by_family": source_by_family,
"source_by_format": source_by_format,
"source_by_month": source_by_month,
"metadata_source_total": metadata_source_total,
"metadata_asset_total": metadata_asset_total,
"metadata_doc_total": metadata_doc_total,
"metadata_missing_count": metadata_missing_count,
"metadata_ready_rate": metadata_ready_rate,
"geometry_total": geometry_total,
"geometry_ready": geometry_ready,
"geometry_not_ready_count": geometry_not_ready_count,
"geometry_ready_rate": geometry_ready_rate,
"geometry_by_family": geometry_by_family,
"preview_ready": preview_ready,
"pipeline": [
{"key": "source", "label": "源资产登记", "value": source_total, "rate": 1.0},
{"key": "metadata", "label": "元数据入库", "value": metadata_asset_total, "rate": metadata_ready_rate},
{"key": "geometry", "label": "几何画像", "value": geometry_total, "rate": _ratio(geometry_total, source_total)},
{"key": "ready", "label": "可生产画像", "value": geometry_ready, "rate": geometry_ready_rate},
{"key": "preview", "label": "预览缓存", "value": preview_ready, "rate": _ratio(preview_ready, radar_total)},
{"key": "source", "label": "源资产登记", "value": source_total, "denominator": source_total, "rate": 1.0},
{"key": "metadata", "label": "元数据入库", "value": metadata_asset_total, "denominator": metadata_source_total, "rate": metadata_ready_rate},
{"key": "geometry", "label": "几何画像", "value": geometry_total, "denominator": metadata_source_total, "rate": _ratio(geometry_total, metadata_source_total)},
{"key": "ready", "label": "可生产画像", "value": geometry_ready, "denominator": geometry_total, "rate": geometry_ready_rate},
{"key": "preview", "label": "预览缓存", "value": preview_ready, "denominator": radar_total, "rate": _ratio(preview_ready, radar_total)},
],
},
"orbit": {
@@ -1208,6 +1331,8 @@ async def get_statistics_dashboard(
"orbit_required_total": orbit_required_total,
"selected_bindings": selected_orbit_bindings,
"matched_bindings": matched_orbit_bindings,
"missing_scene_count": orbit_missing_gap_count,
"missing_summary": orbit_missing_summary,
"selected_rate": selected_orbit_rate,
},
"coverage": {
@@ -1242,6 +1367,9 @@ async def get_statistics_dashboard(
"issues": {
"issue_total": issue_total,
"open_issue_total": open_issue_total,
"risk_count": risk_count,
"non_orbit_open_issue_total": non_orbit_open_issue_total,
"orbit_missing_gap_count": orbit_missing_gap_count,
"by_severity": [
{"severity": severity, "count": count}
for severity, count in sorted(issue_by_severity.items(), key=lambda kv: (-kv[1], kv[0]))
@@ -1361,12 +1489,16 @@ async def get_statistics(
"db_ready_and_cache_exists_count": 0,
"db_ready_but_cache_missing_count": 0,
}
source_xml_consistency = {
source_metadata_consistency = {
"total_records_count": 0,
"xml_detected_count": 0,
"xml_missing_count": 0,
"xml_parsed_ok_count": 0,
"xml_detected_but_unparsed_count": 0,
"metadata_document_asset_count": 0,
"metadata_document_count": 0,
"metadata_document_missing_count": 0,
"metadata_parsed_ok_count": 0,
"metadata_detected_but_unparsed_count": 0,
"geometry_ready_count": 0,
"geometry_missing_count": 0,
"geometry_not_ready_count": 0,
}
try:
@@ -1399,7 +1531,6 @@ async def get_statistics(
)
source_rows = source_rows_res.all()
source_preview_consistency["total_records_count"] = len(source_rows)
source_xml_consistency["total_records_count"] = len(source_rows)
for (
unique_id,
@@ -1415,7 +1546,6 @@ async def get_statistics(
) in source_rows:
if not file_path:
source_preview_consistency["preview_missing_count"] += 1
source_xml_consistency["xml_missing_count"] += 1
continue
cache_key = unique_id or file_path
@@ -1443,34 +1573,64 @@ async def get_statistics(
else:
source_preview_consistency["db_ready_but_cache_missing_count"] += 1
scene_dir = file_path if os.path.isdir(file_path) else os.path.dirname(file_path)
xml_path = find_xml_file(scene_dir) if scene_dir else None
has_xml = bool(xml_path and os.path.exists(xml_path))
if has_xml:
source_xml_consistency["xml_detected_count"] += 1
parsed_ok = any(
value is not None and value != ""
for value in [
scene_center_lon,
scene_center_lat,
acquisition_time_utc,
satellite_mode,
receiving_station,
product_level,
product_unique_id,
]
)
if parsed_ok:
source_xml_consistency["xml_parsed_ok_count"] += 1
else:
source_xml_consistency["xml_detected_but_unparsed_count"] += 1
else:
source_xml_consistency["xml_missing_count"] += 1
active_source_count = await _scalar_count(
db,
select(func.count(SourceProductAssetORM.id)).where(
SourceProductAssetORM.is_active == True,
SourceProductAssetORM.source_format.in_(SOURCE_METADATA_MANAGED_FORMATS),
),
)
metadata_asset_count = await _scalar_count(
db,
select(func.count(distinct(SourceMetadataDocumentORM.source_asset_id))).where(
SourceMetadataDocumentORM.source_format.in_(SOURCE_METADATA_MANAGED_FORMATS),
),
)
metadata_doc_count = await _scalar_count(
db,
select(func.count(SourceMetadataDocumentORM.id)).where(
SourceMetadataDocumentORM.source_format.in_(SOURCE_METADATA_MANAGED_FORMATS),
),
)
metadata_parse_issue_count = await _scalar_count(
db,
select(func.count(SourceMetadataDocumentORM.id)).where(
SourceMetadataDocumentORM.source_format.in_(SOURCE_METADATA_MANAGED_FORMATS),
SourceMetadataDocumentORM.parse_status != "OK",
),
)
geometry_ready_count = await _scalar_count(
db,
select(func.count(SARSceneGeometryProfileORM.id)).where(
SARSceneGeometryProfileORM.source_format.in_(SOURCE_METADATA_MANAGED_FORMATS),
SARSceneGeometryProfileORM.metadata_quality == "READY",
SARSceneGeometryProfileORM.production_readiness == "READY",
),
)
geometry_total_count = await _scalar_count(
db,
select(func.count(SARSceneGeometryProfileORM.id)).where(
SARSceneGeometryProfileORM.source_format.in_(SOURCE_METADATA_MANAGED_FORMATS),
),
)
source_metadata_consistency.update(
{
"total_records_count": active_source_count,
"metadata_document_asset_count": metadata_asset_count,
"metadata_document_count": metadata_doc_count,
"metadata_document_missing_count": max(0, active_source_count - metadata_asset_count),
"metadata_parsed_ok_count": max(0, metadata_doc_count - metadata_parse_issue_count),
"metadata_detected_but_unparsed_count": metadata_parse_issue_count,
"geometry_ready_count": geometry_ready_count,
"geometry_missing_count": max(0, active_source_count - geometry_total_count),
"geometry_not_ready_count": max(0, geometry_total_count - geometry_ready_count),
}
)
except Exception as e:
logger.warning("统计源数据时发生错误 (可能是表不存在): %s", e)
source_preview_consistency["error"] = str(e)
source_xml_consistency["error"] = str(e)
source_metadata_consistency["error"] = str(e)
# 4. AI 质量统计
labeled_good_count = sum(1 for record in dinsar_records if record.product.user_label == 1)
@@ -1756,7 +1916,16 @@ async def get_statistics(
"with_orbit_data_count": with_orbit_data_count,
},
"source_preview_consistency": source_preview_consistency,
"source_xml_consistency": source_xml_consistency,
"source_metadata_consistency": source_metadata_consistency,
"source_xml_consistency": {
"deprecated": True,
"replacement": "source_metadata_consistency",
"total_records_count": source_metadata_consistency.get("total_records_count", 0),
"xml_detected_count": source_metadata_consistency.get("metadata_document_asset_count", 0),
"xml_missing_count": 0,
"xml_parsed_ok_count": source_metadata_consistency.get("metadata_parsed_ok_count", 0),
"xml_detected_but_unparsed_count": 0,
},
"water_geo_consistency": water_geo_consistency,
"pairing_consistency": pairing_consistency,
"by_satellite": by_satellite,
backend/app/services/dinsar_production_service.py
+62 -46
View File
@@ -80,6 +80,16 @@ TERMINAL_ITEM_STATUSES = {
_SAFE_POINTER_RE = re.compile(r"[^0-9A-Za-z._-]+")
def _item_status_counts(items: List[DinsarProductionRunItemORM]) -> Dict[str, int]:
counts: Dict[str, int] = {}
for item in items:
status = str(item.status or "").strip().upper()
if not status:
continue
counts[status] = counts.get(status, 0) + 1
return counts
def _task_type_for_engine(engine_code: str) -> str:
normalized = str(engine_code or "").strip().lower()
if normalized == "sarscape":
@@ -997,53 +1007,59 @@ class DinsarProductionService:
)
for item in items_result.scalars().all():
items_by_run_id.setdefault(item.run_id, []).append(item)
def serialize_run(run: DinsarProductionRunORM) -> Dict[str, Any]:
run_items = items_by_run_id.get(run.run_id, [])
item_counts = _item_status_counts(run_items)
return {
"run_id": run.run_id,
"product_family": run.product_family,
"engine": run.engine_code,
"profile_code": run.profile_code,
"status": _public_run_status(run.status),
"raw_status": run.status,
"started_at": _safe_epoch(run.started_at or run.created_at),
"ended_at": _safe_epoch(run.ended_at),
"task_id": run.task_id,
"workflow_run_id": run.workflow_run_id,
"root_dir": run.source_root,
"publish_root_dir": run.publish_root_dir,
"message": run.latest_message,
"summary_json": run.summary_json if isinstance(run.summary_json, dict) else {},
"total_items": max(int(run.total_items or 0), len(run_items)),
"completed_items": item_counts.get(RUN_ITEM_STATUS_COMPLETED, int(run.completed_items or 0)),
"running_items": item_counts.get(RUN_ITEM_STATUS_RUNNING, 0),
"pending_items": item_counts.get(RUN_ITEM_STATUS_PENDING, 0),
"failed_items": item_counts.get(RUN_ITEM_STATUS_FAILED, int(run.failed_items or 0)),
"skipped_items": item_counts.get(RUN_ITEM_STATUS_SKIPPED, int(run.skipped_items or 0)),
"cancelled_items": item_counts.get(RUN_ITEM_STATUS_CANCELLED, 0),
"items": [
{
"task_name": item.task_name,
"task_alias": item.task_alias,
"pair_key": item.pair_key,
"status": item.status,
"current_step": item.current_step,
"latest_run_key": item.latest_run_key,
"latest_output_dir": item.latest_output_dir,
"latest_manifest_path": item.latest_manifest_path,
"last_error": item.last_error,
"paths": _build_output_paths(
engine_code=run.engine_code,
item=item,
run_key=str(item.latest_run_key or ""),
output_dir=str(item.latest_output_dir or _execution_dir(item, str(item.latest_run_key or ""))),
manifest_path=item.latest_manifest_path,
)
if item.latest_run_key
else {},
}
for item in run_items[:5]
],
}
return {
"runs": [
{
"run_id": run.run_id,
"product_family": run.product_family,
"engine": run.engine_code,
"profile_code": run.profile_code,
"status": _public_run_status(run.status),
"raw_status": run.status,
"started_at": _safe_epoch(run.started_at or run.created_at),
"ended_at": _safe_epoch(run.ended_at),
"task_id": run.task_id,
"workflow_run_id": run.workflow_run_id,
"root_dir": run.source_root,
"publish_root_dir": run.publish_root_dir,
"message": run.latest_message,
"summary_json": run.summary_json if isinstance(run.summary_json, dict) else {},
"total_items": run.total_items,
"completed_items": run.completed_items,
"failed_items": run.failed_items,
"skipped_items": run.skipped_items,
"items": [
{
"task_name": item.task_name,
"task_alias": item.task_alias,
"pair_key": item.pair_key,
"status": item.status,
"current_step": item.current_step,
"latest_run_key": item.latest_run_key,
"latest_output_dir": item.latest_output_dir,
"latest_manifest_path": item.latest_manifest_path,
"last_error": item.last_error,
"paths": _build_output_paths(
engine_code=run.engine_code,
item=item,
run_key=str(item.latest_run_key or ""),
output_dir=str(item.latest_output_dir or _execution_dir(item, str(item.latest_run_key or ""))),
manifest_path=item.latest_manifest_path,
)
if item.latest_run_key
else {},
}
for item in items_by_run_id.get(run.run_id, [])[:5]
],
}
for run in runs
],
"runs": [serialize_run(run) for run in runs],
"total": total,
}
backend/app/services/ops_maintenance_service.py
+82 -240
View File
@@ -4,9 +4,9 @@ import os
import shutil
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional
from typing import Any, Dict, List, Optional
from sqlalchemy import delete, func, or_, select
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from ..config import settings
@@ -14,25 +14,18 @@ from ..models import (
DinsarProductionExecutionORM,
DinsarProductionRunItemORM,
DinsarProductionRunORM,
DinsarResultORM,
ResultProductORM,
SystemJobORM,
SystemTaskORM,
TaskLogORM,
WorkflowArtifactORM,
WorkflowRunORM,
WorkflowStepORM,
)
from .dinsar_production_service import dinsar_production_service
TERMINAL_TASK_STATUSES = {"COMPLETED", "FAILED", "PARTIAL_SUCCESS", "CANCELLED"}
MAINTENANCE_LIST_STATUSES = {"FAILED", "PARTIAL_SUCCESS", "CANCELLED", "PENDING", "RUNNING"}
MAINTENANCE_LIST_STATUSES = {"COMPLETED", "FAILED", "PARTIAL_SUCCESS", "CANCELLED", "PENDING", "RUNNING"}
ACTIVE_TASK_STATUSES = {"PENDING", "RUNNING"}
ACTIVE_JOB_STATUSES = {"READY", "RETRY", "RUNNING"}
DINSAR_TASK_TYPES = {"LANDSAR_RUN", "LANDSAR_CLUSTER_RUN", "PYINT_RUN", "IDL_RUN_DINSAR"}
SUPPORTED_CLEANUP_TASK_TYPES = DINSAR_TASK_TYPES | {"COPY_DATA"}
DEFAULT_TASK_TYPES = SUPPORTED_CLEANUP_TASK_TYPES | {"PAIRING_CACHE_REBUILD"}
SUPPORTED_CLEANUP_TASK_TYPES = {"LANDSAR_RUN", "LANDSAR_CLUSTER_RUN"}
DEFAULT_TASK_TYPES = DINSAR_TASK_TYPES | {"COPY_DATA", "PAIRING_CACHE_REBUILD"}
LANDSAR_WORK_TERMINAL_EXECUTION_STATUSES = {"COMPLETED", "FAILED", "CANCELLED"}
def _utcnow_naive() -> datetime:
@@ -68,8 +61,15 @@ def _path_exists(path: str) -> bool:
return bool(path) and os.path.exists(path)
def _safe_count(rows: Iterable[Any]) -> int:
return len(list(rows))
def _is_within_path(path: str, parent: str) -> bool:
path_text = _normalize_path_text(path)
parent_text = _normalize_path_text(parent)
if not path_text or not parent_text:
return False
try:
return os.path.commonpath([os.path.abspath(path_text), os.path.abspath(parent_text)]) == os.path.abspath(parent_text)
except ValueError:
return False
class OpsMaintenanceService:
@@ -108,11 +108,12 @@ class OpsMaintenanceService:
if item.get("issue_level") in {"warning", "danger"}
or _norm_status(item.get("status")) in {"FAILED", "PARTIAL_SUCCESS", "CANCELLED"}
]
visible_items = items if status_filter else abnormal_items
return {
"items": abnormal_items,
"items": visible_items,
"limit": safe_limit,
"offset": safe_offset,
"returned": len(abnormal_items),
"returned": len(visible_items),
}
async def diagnose_task(self, db: AsyncSession, task_id: str) -> Optional[Dict[str, Any]]:
@@ -124,19 +125,11 @@ class OpsMaintenanceService:
item_counts: Dict[str, int] = {}
execution_counts: Dict[str, int] = {}
disk_paths: List[Dict[str, Any]] = []
products: List[Dict[str, Any]] = []
if run is not None:
item_counts = await self._status_counts(db, DinsarProductionRunItemORM, run.run_id)
execution_counts = await self._status_counts(db, DinsarProductionExecutionORM, run.run_id)
disk_paths.extend(await self._collect_run_disk_paths(db, run))
products = await self._collect_result_products(db, run)
related_tasks = await self._related_copy_tasks_for_run(db, run) if run is not None else []
copy_dest = self._copy_task_dest_dir(task)
if copy_dest:
disk_paths.append(self._path_payload(copy_dest, "task_pool"))
recent_logs = await self._recent_logs(db, task.task_id)
findings, cleanup_supported, cleanup_blockers = self._diagnose_findings(
task=task,
@@ -152,8 +145,6 @@ class OpsMaintenanceService:
"production_run": self._run_payload(run) if run else None,
"production_item_counts": item_counts,
"production_execution_counts": execution_counts,
"result_products": products,
"related_tasks": [self._task_payload(item) for item in related_tasks],
"recent_logs": recent_logs,
"disk_paths": disk_paths,
"diagnosis": {
@@ -179,15 +170,17 @@ class OpsMaintenanceService:
if task_status in ACTIVE_TASK_STATUSES:
blockers.append("任务仍处于活动状态，不能清理。")
db_counts = await self._cleanup_db_counts(db, diagnosis)
disk_deletes = self._cleanup_disk_targets(diagnosis)
has_existing_target = any(item.get("exists") and item.get("allowed") for item in disk_deletes)
if not has_existing_target:
blockers.append("未发现可清理的 LandSAR_WORK_ROOT/run_* 目录。")
blocked = bool(blockers) or not cleanup_supported
return {
"task_id": task_id,
"blocked": blocked,
"blockers": blockers,
"cleanup_supported": cleanup_supported and not blocked,
"database_deletes": db_counts,
"database_deletes": {},
"disk_deletes": disk_deletes,
}
@@ -204,82 +197,17 @@ class OpsMaintenanceService:
if preview.get("blocked"):
raise ValueError("; ".join(preview.get("blockers") or ["清理被阻止。"]))
deleted_db: Dict[str, int] = {}
task = await self._get_task(db, task_id)
if task is None:
return None
jobs = await self._get_jobs(db, task_id)
run = await self._get_production_run_for_task(db, task, jobs)
delete_logs = bool(options.get("delete_logs", True))
delete_task_records = bool(options.get("delete_task_records", True))
delete_production_records = bool(options.get("delete_production_records", True))
delete_result_products = bool(options.get("delete_result_products", True))
delete_task_pool_dir = bool(options.get("delete_task_pool_dir", True))
related_copy_tasks = await self._related_copy_tasks_for_run(db, run) if run is not None else []
if delete_result_products and run is not None:
products = await self._result_product_orms_for_run(db, run)
product_ids = [item.product_id for item in products]
compat_ids = await self._compat_ids_for_products(db, product_ids)
if product_ids:
result = await db.execute(delete(ResultProductORM).where(ResultProductORM.product_id.in_(product_ids)))
deleted_db["result_products"] = int(result.rowcount or 0)
if compat_ids:
result = await db.execute(delete(DinsarResultORM).where(DinsarResultORM.id.in_(compat_ids)))
deleted_db["dinsar_results"] = int(result.rowcount or 0)
if delete_production_records and run is not None:
deleted_run = await dinsar_production_service.delete_run_record(run.run_id, db=db)
deleted_db["dinsar_production_runs"] = 1 if deleted_run else 0
deleted_db["dinsar_production_run_items"] = int(preview["database_deletes"].get("dinsar_production_run_items", 0))
deleted_db["dinsar_production_executions"] = int(preview["database_deletes"].get("dinsar_production_executions", 0))
deleted_db["system_jobs"] = int(preview["database_deletes"].get("system_jobs", 0))
deleted_db["system_tasks"] = 1
deleted_db["task_logs"] = int(preview["database_deletes"].get("task_logs", 0))
if delete_task_pool_dir:
related_deleted = await self._delete_related_tasks(db, related_copy_tasks)
for key, value in related_deleted.items():
deleted_db[key] = int(deleted_db.get(key, 0)) + int(value or 0)
else:
if delete_logs:
result = await db.execute(delete(TaskLogORM).where(TaskLogORM.task_id == task_id))
deleted_db["task_logs"] = int(result.rowcount or 0)
if delete_task_records:
result = await db.execute(delete(SystemJobORM).where(SystemJobORM.task_id == task_id))
deleted_db["system_jobs"] = int(result.rowcount or 0)
result = await db.execute(delete(SystemTaskORM).where(SystemTaskORM.task_id == task_id))
deleted_db["system_tasks"] = int(result.rowcount or 0)
await db.commit()
disk_result = await self._delete_disk_targets(preview, options)
return {
"task_id": task_id,
"deleted_database": deleted_db,
"deleted_database": {},
"deleted_disk": disk_result,
}
async def _delete_related_tasks(
self,
db: AsyncSession,
tasks: List[SystemTaskORM],
) -> Dict[str, int]:
task_ids = [task.task_id for task in tasks if task.task_id]
if not task_ids:
return {}
result = await db.execute(delete(TaskLogORM).where(TaskLogORM.task_id.in_(task_ids)))
logs = int(result.rowcount or 0)
result = await db.execute(delete(SystemJobORM).where(SystemJobORM.task_id.in_(task_ids)))
jobs = int(result.rowcount or 0)
result = await db.execute(delete(SystemTaskORM).where(SystemTaskORM.task_id.in_(task_ids)))
task_count = int(result.rowcount or 0)
await db.commit()
return {
"related_task_logs": logs,
"related_system_jobs": jobs,
"related_system_tasks": task_count,
}
async def _get_task(self, db: AsyncSession, task_id: str) -> Optional[SystemTaskORM]:
result = await db.execute(select(SystemTaskORM).where(SystemTaskORM.task_id == str(task_id or "").strip()))
return result.scalar_one_or_none()
@@ -383,87 +311,33 @@ class OpsMaintenanceService:
async def _collect_run_disk_paths(self, db: AsyncSession, run: DinsarProductionRunORM) -> List[Dict[str, Any]]:
paths: Dict[str, Dict[str, Any]] = {}
if run.source_root:
paths[_normalize_path_text(run.source_root)] = self._path_payload(run.source_root, "task_pool")
result = await db.execute(select(DinsarProductionExecutionORM).where(DinsarProductionExecutionORM.run_id == run.run_id))
for execution in result.scalars().all():
if execution.output_dir:
publish_dir = self._publish_package_dir(execution.output_dir)
paths[publish_dir] = self._path_payload(publish_dir, "production_result")
log_path = dinsar_production_service.read_run_log(run.run_id, max_bytes=1).get("path")
if log_path:
paths[_normalize_path_text(log_path)] = self._path_payload(log_path, "run_log")
landsar_work_dir = self._landsar_work_dir_for_execution(run, execution)
if landsar_work_dir:
payload = self._path_payload(landsar_work_dir, "landsar_work")
payload["run_key"] = execution.run_key
payload["execution_status"] = execution.status
paths[_normalize_path_text(landsar_work_dir).lower()] = payload
return list(paths.values())
async def _collect_result_products(self, db: AsyncSession, run: DinsarProductionRunORM) -> List[Dict[str, Any]]:
products = await self._result_product_orms_for_run(db, run)
return [
{
"product_id": item.product_id,
"display_name": item.display_name,
"status": item.status,
"health_status": item.health_status,
"publish_dir": item.publish_dir,
"manifest_path": item.manifest_path,
}
for item in products
]
async def _result_product_orms_for_run(self, db: AsyncSession, run: DinsarProductionRunORM) -> List[ResultProductORM]:
result = await db.execute(select(DinsarProductionExecutionORM).where(DinsarProductionExecutionORM.run_id == run.run_id))
dirs = [self._publish_package_dir(item.output_dir) for item in result.scalars().all() if item.output_dir]
clauses = []
for path in dirs:
clauses.append(ResultProductORM.publish_dir == path)
clauses.append(ResultProductORM.native_output_dir.like(path + "%"))
clauses.append(ResultProductORM.manifest_path.like(path + "%"))
clauses.append(ResultProductORM.primary_asset_path.like(path + "%"))
if not clauses:
return []
products = await db.execute(select(ResultProductORM).where(or_(*clauses)))
by_id: Dict[str, ResultProductORM] = {}
for product in products.scalars().all():
by_id[product.product_id] = product
return list(by_id.values())
async def _compat_ids_for_products(self, db: AsyncSession, product_ids: List[str]) -> List[int]:
if not product_ids:
return []
result = await db.execute(select(DinsarResultORM).where(DinsarResultORM.compat_product_id.in_(product_ids)))
return [int(item.id) for item in result.scalars().all()]
def _publish_package_dir(self, output_dir: str) -> str:
normalized = _normalize_path_text(output_dir)
marker = os.sep + "runs" + os.sep
if marker.lower() in normalized.lower():
lower = normalized.lower()
index = lower.index(marker.lower())
return normalized[:index]
return normalized
def _copy_task_dest_dir(self, task: SystemTaskORM) -> str:
params = task.params if isinstance(task.params, dict) else {}
return _normalize_path_text(params.get("dest_dir")) if params.get("dest_dir") else ""
async def _related_copy_tasks_for_run(
def _landsar_work_dir_for_execution(
self,
db: AsyncSession,
run: Optional[DinsarProductionRunORM],
) -> List[SystemTaskORM]:
source_root = _normalize_path_text(run.source_root if run is not None else "")
if not source_root:
return []
result = await db.execute(
select(SystemTaskORM)
.where(SystemTaskORM.task_type == "COPY_DATA")
.order_by(SystemTaskORM.updated_at.desc(), SystemTaskORM.id.desc())
.limit(1000)
)
tasks = []
for task in result.scalars().all():
if _normalize_path_text(self._copy_task_dest_dir(task)).lower() == source_root.lower():
tasks.append(task)
return tasks
run: DinsarProductionRunORM,
execution: DinsarProductionExecutionORM,
) -> str:
if _norm_status(run.engine_code) != "LANDSAR":
return ""
if _norm_status(run.status) in ACTIVE_TASK_STATUSES:
return ""
if _norm_status(execution.status) not in LANDSAR_WORK_TERMINAL_EXECUTION_STATUSES:
return ""
work_root = _normalize_path_text(settings.LANDSAR_WORK_ROOT)
run_key = str(execution.run_key or "").strip()
if not work_root or not run_key or Path(run_key).name != run_key or not run_key.startswith("run_"):
return ""
candidate = _normalize_path_text(os.path.join(work_root, run_key))
return candidate if self._is_landsar_work_delete_path(candidate) else ""
def _diagnose_findings(
self,
@@ -481,9 +355,9 @@ class OpsMaintenanceService:
if status == "FAILED":
findings.append("任务已失败，需要人工确认后清理。")
elif status == "PARTIAL_SUCCESS":
findings.append("任务部分成功，清理前请确认保留策略。")
findings.append("任务部分成功，可按需清理 LandSAR 工作目录。")
elif status == "CANCELLED":
findings.append("任务已取消，可按需清理残留记录和目录。")
findings.append("任务已取消，可按需清理 LandSAR 工作目录。")
elif status in ACTIVE_TASK_STATUSES:
blockers.append("任务仍处于活动状态。")
@@ -507,58 +381,29 @@ class OpsMaintenanceService:
missing_paths = [item for item in disk_paths if item.get("path") and not item.get("exists")]
if missing_paths:
findings.append(f"有 {len(missing_paths)} 个登记路径已不存在。")
landsar_work_paths = [
item
for item in disk_paths
if item.get("kind") == "landsar_work" and item.get("exists")
]
if landsar_work_paths:
findings.append(f"发现 {len(landsar_work_paths)} 个可清理的 LandSAR 工作目录。")
cleanup_supported = _norm_status(task.task_type) in SUPPORTED_CLEANUP_TASK_TYPES and not blockers
cleanup_supported = (
_norm_status(task.task_type) in SUPPORTED_CLEANUP_TASK_TYPES
and not blockers
and bool(landsar_work_paths)
)
return findings, cleanup_supported, blockers
async def _cleanup_db_counts(self, db: AsyncSession, diagnosis: Dict[str, Any]) -> Dict[str, int]:
task_id = diagnosis["task"]["task_id"]
run = diagnosis.get("production_run") or {}
run_id = run.get("run_id")
workflow_run_id = run.get("workflow_run_id")
counts = {
"system_tasks": await self._count(db, select(func.count()).select_from(SystemTaskORM).where(SystemTaskORM.task_id == task_id)),
"system_jobs": await self._count(db, select(func.count()).select_from(SystemJobORM).where(SystemJobORM.task_id == task_id)),
"task_logs": await self._count(db, select(func.count()).select_from(TaskLogORM).where(TaskLogORM.task_id == task_id)),
"related_system_tasks": 0,
"related_system_jobs": 0,
"related_task_logs": 0,
"dinsar_production_runs": 0,
"dinsar_production_run_items": 0,
"dinsar_production_executions": 0,
"result_products": len(diagnosis.get("result_products") or []),
"dinsar_results": 0,
"workflow_runs": 0,
"workflow_steps": 0,
"workflow_artifacts": 0,
}
if run_id:
counts["dinsar_production_runs"] = await self._count(db, select(func.count()).select_from(DinsarProductionRunORM).where(DinsarProductionRunORM.run_id == run_id))
counts["dinsar_production_run_items"] = await self._count(db, select(func.count()).select_from(DinsarProductionRunItemORM).where(DinsarProductionRunItemORM.run_id == run_id))
counts["dinsar_production_executions"] = await self._count(db, select(func.count()).select_from(DinsarProductionExecutionORM).where(DinsarProductionExecutionORM.run_id == run_id))
run_obj = await self._get_run_by_id(db, run_id)
related_tasks = await self._related_copy_tasks_for_run(db, run_obj)
related_task_ids = [item.task_id for item in related_tasks if item.task_id]
counts["related_system_tasks"] = len(related_task_ids)
if related_task_ids:
counts["related_system_jobs"] = await self._count(db, select(func.count()).select_from(SystemJobORM).where(SystemJobORM.task_id.in_(related_task_ids)))
counts["related_task_logs"] = await self._count(db, select(func.count()).select_from(TaskLogORM).where(TaskLogORM.task_id.in_(related_task_ids)))
if workflow_run_id:
counts["workflow_runs"] = await self._count(db, select(func.count()).select_from(WorkflowRunORM).where(WorkflowRunORM.run_id == workflow_run_id))
counts["workflow_steps"] = await self._count(db, select(func.count()).select_from(WorkflowStepORM).where(WorkflowStepORM.run_id == workflow_run_id))
counts["workflow_artifacts"] = await self._count(db, select(func.count()).select_from(WorkflowArtifactORM).where(WorkflowArtifactORM.run_id == workflow_run_id))
return counts
async def _get_run_by_id(self, db: AsyncSession, run_id: str) -> Optional[DinsarProductionRunORM]:
result = await db.execute(select(DinsarProductionRunORM).where(DinsarProductionRunORM.run_id == run_id))
return result.scalar_one_or_none()
async def _count(self, db: AsyncSession, stmt: Any) -> int:
return int((await db.execute(stmt)).scalar_one() or 0)
def _cleanup_disk_targets(self, diagnosis: Dict[str, Any]) -> List[Dict[str, Any]]:
targets: Dict[str, Dict[str, Any]] = {}
for item in diagnosis.get("disk_paths") or []:
if item.get("kind") != "landsar_work":
continue
path = _normalize_path_text(item.get("path"))
if not path:
continue
@@ -581,29 +426,26 @@ class OpsMaintenanceService:
normalized = _normalize_path_text(path)
if not normalized:
return False
roots = [
settings.DINSAR_TASK_POOL_ROOT,
settings.DINSAR_PRODUCT_DIR,
os.path.join(settings.PROJECT_ROOT, "backend", "runtime", "dinsar_production"),
]
return self._is_landsar_work_delete_path(normalized)
def _is_landsar_work_delete_path(self, path: str) -> bool:
work_root = _normalize_path_text(settings.LANDSAR_WORK_ROOT)
normalized = _normalize_path_text(path)
if not work_root or not normalized:
return False
root_full = os.path.abspath(work_root)
full = os.path.abspath(normalized)
for root in roots:
root_text = _normalize_path_text(root)
if not root_text:
continue
root_full = os.path.abspath(root_text)
if full == root_full:
return False
try:
if os.path.commonpath([full, root_full]) == root_full:
return True
except ValueError:
continue
return False
if full == root_full or not _is_within_path(full, root_full):
return False
try:
relative = os.path.relpath(full, root_full)
except ValueError:
return False
parts = [part for part in relative.split(os.sep) if part]
return len(parts) == 1 and parts[0].startswith("run_")
async def _delete_disk_targets(self, preview: Dict[str, Any], options: Dict[str, bool]) -> Dict[str, Any]:
delete_production_dirs = bool(options.get("delete_production_dirs", True))
delete_task_pool_dir = bool(options.get("delete_task_pool_dir", True))
delete_landsar_work_dir = bool(options.get("delete_landsar_work_dir", True))
deleted: List[str] = []
missing: List[str] = []
skipped: List[str] = []
@@ -611,13 +453,13 @@ class OpsMaintenanceService:
for item in preview.get("disk_deletes") or []:
kind = item.get("kind")
path = _normalize_path_text(item.get("path"))
if kind != "landsar_work":
skipped.append(path)
continue
if not item.get("allowed"):
skipped.append(path)
continue
if kind == "task_pool" and not delete_task_pool_dir:
skipped.append(path)
continue
if kind == "production_result" and not delete_production_dirs:
if not delete_landsar_work_dir:
skipped.append(path)
continue
if not _path_exists(path):
backend/app/services/spatial_service.py
+436 -5
View File
@@ -7,8 +7,10 @@ import hashlib
import json
import logging
import math
import re
import uuid
from collections import defaultdict
from datetime import datetime, timedelta
from itertools import combinations
from typing import Any, Dict, List, Optional, Tuple
@@ -20,7 +22,7 @@ from sqlalchemy.orm import aliased
from geoalchemy2 import Geography
from geoalchemy2.shape import to_shape
from geoalchemy2.functions import ST_Intersects, ST_Intersection, ST_Area, ST_Centroid, ST_Covers
from shapely.geometry import Polygon
from shapely.geometry import Polygon, mapping, shape
from shapely.ops import unary_union
from ..models import (
@@ -196,6 +198,112 @@ class SpatialService:
}
return result_pairs, warnings, metadata
async def find_dinsar_coverage_pairs(
self,
db: AsyncSession,
params: PairingRequest,
*,
target_date_from: str,
target_date_to: str,
extension_days: int = 15,
max_pairs: int = 200,
target_coverage_ratio: float = 0.98,
min_new_coverage_ratio: float = 0.0005,
aoi_wkt: Optional[str] = None,
require_orbit_data: bool = True,
) -> Tuple[List[RadarPair], List[str], Dict[str, Any]]:
warnings: List[str] = []
target_start = self._parse_yyyymmdd(target_date_from, field_name="target_date_from")
target_end = self._parse_yyyymmdd(target_date_to, field_name="target_date_to")
if target_end < target_start:
raise ValueError("target_date_to must be greater than or equal to target_date_from.")
safe_extension_days = max(0, min(180, int(extension_days or 0)))
safe_max_pairs = max(1, min(5000, int(max_pairs or 200)))
safe_target_coverage_ratio = max(0.0, min(1.0, float(target_coverage_ratio or 0.98)))
safe_min_new_coverage_ratio = max(0.0, min(1.0, float(min_new_coverage_ratio or 0.0)))
query_start = target_start - timedelta(days=safe_extension_days)
query_end = target_end + timedelta(days=safe_extension_days)
effective_params = self._normalize_pairing_request(params).model_copy(
update={
"master_date_from": self._format_yyyymmdd(query_start),
"master_date_to": self._format_yyyymmdd(query_end),
"slave_date_from": self._format_yyyymmdd(query_start),
"slave_date_to": self._format_yyyymmdd(query_end),
"strategy": "dinsar_province_coverage",
}
)
pairing_status = await pairing_state_service.get_pairing_system_status(db)
cache_status = str(pairing_status.get("status") or "UNINITIALIZED")
scene_count = int(pairing_status.get("scene_count") or 0)
pair_count = int(pairing_status.get("pair_count") or 0)
degraded = bool(pairing_status.get("needs_rebuild"))
if cache_status in {"FAILED", "UNINITIALIZED", "ERROR"} or (scene_count > 1 and pair_count == 0):
raise RuntimeError(
"Pairing candidate cache is not available. Repair or rebuild the pairing foundation first."
)
if degraded:
warnings.append(
f"Pairing foundation status is {cache_status}; coverage plan uses current cached candidates."
)
candidate_pool = await self._query_pairing_metric_cache(
db,
effective_params,
aoi_wkt=aoi_wkt,
require_orbit_data=require_orbit_data,
)
selected_candidates, coverage_meta, strategy_warnings = self._apply_province_coverage_strategy(
candidate_pool,
target_start=target_start,
target_end=target_end,
query_start=query_start,
query_end=query_end,
max_pairs=safe_max_pairs,
target_coverage_ratio=safe_target_coverage_ratio,
min_new_coverage_ratio=safe_min_new_coverage_ratio,
aoi_wkt=aoi_wkt,
)
warnings.extend(strategy_warnings)
if not selected_candidates:
warnings.extend(
await self._build_empty_pairing_diagnostics(
db,
effective_params,
aoi_wkt=aoi_wkt,
require_orbit_data=require_orbit_data,
)
)
for candidate in selected_candidates:
candidate["selection_strategy"] = "dinsar_province_coverage"
self._ensure_candidate_identity(candidate)
network_run_id = await self._persist_network_run(
db,
params=effective_params,
aoi_wkt=aoi_wkt,
require_orbit_data=require_orbit_data,
warnings=warnings,
candidate_pool=candidate_pool,
selected_candidates=selected_candidates,
)
await self._attach_dinsar_production_summaries(db, selected_candidates)
result_pairs = self._generate_task_names(self._build_radar_pairs(selected_candidates))
metadata = {
"fallback_used": False,
"degraded": degraded,
"policy_version": PAIRING_POLICY_VERSION,
"network_run_id": network_run_id,
"candidate_count": len(candidate_pool),
"selected_edge_count": len(result_pairs),
"coverage": coverage_meta,
}
return result_pairs, warnings, metadata
def _normalize_pairing_request(self, params: PairingRequest) -> PairingRequest:
updates: Dict[str, Any] = {}
@@ -408,6 +516,27 @@ class SpatialService:
selection_strategy=candidate.get("selection_strategy"),
selection_score=float(selection_score) if selection_score is not None else None,
selection_reason=candidate.get("selection_reason"),
coverage_rank=(
int(candidate["coverage_rank"])
if candidate.get("coverage_rank") is not None
else None
),
aoi_new_area_ratio=(
float(candidate["aoi_new_area_ratio"])
if candidate.get("aoi_new_area_ratio") is not None
else None
),
aoi_pair_area_ratio=(
float(candidate["aoi_pair_area_ratio"])
if candidate.get("aoi_pair_area_ratio") is not None
else None
),
aoi_coverage_ratio_after=(
float(candidate["aoi_coverage_ratio_after"])
if candidate.get("aoi_coverage_ratio_after") is not None
else None
),
effective_coverage_geojson=candidate.get("effective_coverage_geojson"),
time_baseline_days=int(candidate["days"]),
spatial_baseline_meters=float(candidate["dist"]),
scene_center_distance_meters=float(
@@ -701,6 +830,253 @@ class SpatialService:
output.append(item)
return output
def _apply_province_coverage_strategy(
self,
candidate_pool: List[dict],
*,
target_start: datetime,
target_end: datetime,
query_start: datetime,
query_end: datetime,
max_pairs: int,
target_coverage_ratio: float,
min_new_coverage_ratio: float,
aoi_wkt: Optional[str],
) -> Tuple[List[dict], Dict[str, Any], List[str]]:
warnings: List[str] = []
target_days = self._date_set(target_start, target_end)
covered_days: set[datetime] = set()
selected: List[dict] = []
remaining = [candidate for candidate in candidate_pool if self._candidate_date_window(candidate) is not None]
seen_metric_ids: set[int] = set()
aoi_poly = self._parse_optional_aoi_polygon(aoi_wkt)
geometry_cache: Dict[int, Any] = {}
selected_coverage = Polygon()
aoi_area = float(aoi_poly.area or 0.0) if aoi_poly is not None else 0.0
stop_reason = "no_more_gain"
max_pairs_reached = False
while remaining and len(selected) < max_pairs:
best_candidate = None
best_score: Optional[Tuple[float, float, float, float, float, float, str]] = None
current_aoi_coverage_ratio = (
float(selected_coverage.area or 0.0) / aoi_area
if aoi_poly is not None and aoi_area > 0
else None
)
if current_aoi_coverage_ratio is not None and current_aoi_coverage_ratio >= target_coverage_ratio:
stop_reason = "target_coverage_reached"
break
for candidate in remaining:
window = self._candidate_date_window(candidate)
if window is None:
continue
candidate_days = self._date_set(*window) & target_days
new_days = candidate_days - covered_days
new_area = 0.0
candidate_area = 0.0
if aoi_poly is not None:
candidate_geom = self._get_candidate_intersection_geom(
candidate,
aoi_poly=aoi_poly,
geometry_cache=geometry_cache,
)
if candidate_geom is not None and not candidate_geom.is_empty:
candidate_area = float(candidate_geom.area or 0.0)
new_area = float(candidate_geom.difference(selected_coverage).area or 0.0)
if aoi_poly is not None:
new_area_ratio = (new_area / aoi_area) if aoi_area > 0 else 0.0
if new_area_ratio <= 0:
continue
elif not new_days:
continue
quality_score = float(candidate.get("dinsar_quality_score") or 0.0)
overlap = float(candidate.get("overlap_ratio") or 0.0)
temporal_days = float(candidate.get("days") or 0.0)
distance = float(candidate.get("scene_center_distance_meters") or candidate.get("dist") or 0.0)
if aoi_poly is not None and aoi_area > 0:
score = (
new_area_ratio,
candidate_area / aoi_area,
float(len(new_days)) / max(1, len(target_days)),
overlap,
quality_score,
-temporal_days - (distance / 1000000.0),
str(candidate.get("pair_uid") or ""),
)
else:
score = (
float(len(new_days)),
float(len(candidate_days)),
overlap,
quality_score,
0.0,
-temporal_days - (distance / 1000000.0),
str(candidate.get("pair_uid") or ""),
)
if best_score is None or score > best_score:
best_candidate = candidate
best_score = score
if best_candidate is None:
stop_reason = "no_candidate_adds_coverage"
break
if aoi_poly is None and covered_days == target_days:
stop_reason = "target_time_reached"
break
window = self._candidate_date_window(best_candidate)
candidate_days = self._date_set(*window) & target_days if window else set()
new_days = candidate_days - covered_days
candidate_geom = self._get_candidate_intersection_geom(
best_candidate,
aoi_poly=aoi_poly,
geometry_cache=geometry_cache,
) if aoi_poly is not None else None
new_area = 0.0
candidate_area = 0.0
if candidate_geom is not None and not candidate_geom.is_empty:
candidate_area = float(candidate_geom.area or 0.0)
new_area = float(candidate_geom.difference(selected_coverage).area or 0.0)
new_area_ratio = (new_area / aoi_area) if aoi_area > 0 else 0.0
if aoi_poly is not None and len(selected) > 0 and new_area_ratio < min_new_coverage_ratio:
stop_reason = "marginal_gain_below_threshold"
break
best_candidate["selection_reason"] = "province_coverage_new_days"
if aoi_poly is not None and new_area > 0:
best_candidate["selection_reason"] = "province_aoi_new_area"
best_candidate["selection_score"] = float(best_score[0] if best_score else len(new_days))
best_candidate["coverage_rank"] = len(selected) + 1
best_candidate["coverage_new_days"] = len(new_days)
best_candidate["coverage_total_days"] = len(candidate_days)
best_candidate["aoi_new_area_ratio"] = new_area_ratio if aoi_area > 0 else None
best_candidate["aoi_pair_area_ratio"] = (candidate_area / aoi_area) if aoi_area > 0 else None
next_selected_coverage = selected_coverage
if candidate_geom is not None and not candidate_geom.is_empty:
next_selected_coverage = unary_union([selected_coverage, candidate_geom])
best_candidate["aoi_coverage_ratio_after"] = (
float(next_selected_coverage.area or 0.0) / aoi_area
if aoi_area > 0 else None
)
best_candidate["effective_coverage_geojson"] = self._geometry_to_geojson(candidate_geom)
best_candidate["target_coverage_ratio_after"] = (
len(covered_days | candidate_days) / max(1, len(target_days))
)
selected.append(best_candidate)
covered_days |= candidate_days
selected_coverage = next_selected_coverage
seen_metric_ids.add(int(best_candidate.get("metric_cache_ref_id") or 0))
remaining = [
candidate for candidate in remaining
if int(candidate.get("metric_cache_ref_id") or 0) not in seen_metric_ids
]
if remaining and len(selected) >= max_pairs:
max_pairs_reached = True
stop_reason = "max_pairs_reached"
uncovered_ranges = self._date_ranges_from_days(target_days - covered_days)
if uncovered_ranges:
warnings.append(
"Coverage plan did not fully cover the requested time range. "
f"Uncovered ranges: {', '.join(f'{item[0]}~{item[1]}' for item in uncovered_ranges[:6])}"
)
if max_pairs_reached:
warnings.append(
f"Coverage planning reached task limit max_pairs={max_pairs}; increase the limit for better spatial coverage."
)
selected_area = float(selected_coverage.area or 0.0) if aoi_poly is not None else 0.0
aoi_coverage_ratio = (selected_area / aoi_area) if aoi_area > 0 else None
if aoi_coverage_ratio is not None and aoi_coverage_ratio < target_coverage_ratio:
warnings.append(
f"AOI spatial coverage is {aoi_coverage_ratio:.1%}; target is {target_coverage_ratio:.1%}."
)
temporal_ratio = len(covered_days) / max(1, len(target_days))
coverage_meta = {
"strategy": "dinsar_province_coverage",
"optimization_goal": "minimize_pair_count_for_aoi_spatial_coverage",
"greedy_rule": "select_pair_with_largest_new_aoi_intersection_area_each_step",
"stop_reason": stop_reason,
"max_pairs": max_pairs,
"target_coverage_ratio": round(target_coverage_ratio, 6),
"min_new_coverage_ratio": round(min_new_coverage_ratio, 6),
"coverage_basis": "aoi_spatial" if aoi_coverage_ratio is not None else "temporal",
"target_date_from": self._format_yyyymmdd(target_start),
"target_date_to": self._format_yyyymmdd(target_end),
"query_date_from": self._format_yyyymmdd(query_start),
"query_date_to": self._format_yyyymmdd(query_end),
"target_day_count": len(target_days),
"covered_day_count": len(covered_days),
"temporal_coverage_ratio": round(temporal_ratio, 6),
"aoi_coverage_ratio": round(aoi_coverage_ratio, 6) if aoi_coverage_ratio is not None else None,
"coverage_ratio": round(aoi_coverage_ratio if aoi_coverage_ratio is not None else temporal_ratio, 6),
"uncovered_ranges": [
{"date_from": start, "date_to": end}
for start, end in uncovered_ranges
],
"selected_pair_count": len(selected),
"candidate_count": len(candidate_pool),
}
return selected, coverage_meta, warnings
def _candidate_date_window(self, candidate: dict) -> Optional[Tuple[datetime, datetime]]:
master = candidate.get("master")
slave = candidate.get("slave")
master_date = self._try_parse_yyyymmdd(getattr(master, "imaging_date", None))
slave_date = self._try_parse_yyyymmdd(getattr(slave, "imaging_date", None))
if master_date is None or slave_date is None:
return None
return (master_date, slave_date) if master_date <= slave_date else (slave_date, master_date)
def _sort_coverage_candidates(self, candidates: List[dict]) -> List[dict]:
return sorted(
candidates,
key=lambda item: (
str(getattr(item.get("master"), "imaging_date", "") or ""),
str(getattr(item.get("slave"), "imaging_date", "") or ""),
-float(item.get("selection_score") or 0.0),
str(item.get("pair_uid") or ""),
),
)
def _date_set(self, start: datetime, end: datetime) -> set[datetime]:
if end < start:
return set()
return {start + timedelta(days=offset) for offset in range((end - start).days + 1)}
def _date_ranges_from_days(self, days: set[datetime]) -> List[Tuple[str, str]]:
if not days:
return []
ordered = sorted(days)
ranges: List[Tuple[datetime, datetime]] = []
start = previous = ordered[0]
for day in ordered[1:]:
if day == previous + timedelta(days=1):
previous = day
continue
ranges.append((start, previous))
start = previous = day
ranges.append((start, previous))
return [(self._format_yyyymmdd(start), self._format_yyyymmdd(end)) for start, end in ranges]
def _parse_yyyymmdd(self, value: str, *, field_name: str) -> datetime:
parsed = self._try_parse_yyyymmdd(value)
if parsed is None:
raise ValueError(f"{field_name} must be YYYYMMDD.")
return parsed
def _try_parse_yyyymmdd(self, value: Any) -> Optional[datetime]:
text_value = str(value or "").strip()
if not re.match(r"^\d{8}$", text_value):
return None
try:
return datetime.strptime(text_value, "%Y%m%d")
except ValueError:
return None
def _format_yyyymmdd(self, value: datetime) -> str:
return value.strftime("%Y%m%d")
async def _persist_network_run(
self,
db: AsyncSession,
@@ -797,6 +1173,13 @@ class SpatialService:
if candidate.get("pair_aoi_overlap_ratio") is not None
else None
),
"coverage_rank": candidate.get("coverage_rank"),
"coverage_new_days": candidate.get("coverage_new_days"),
"coverage_total_days": candidate.get("coverage_total_days"),
"aoi_new_area_ratio": candidate.get("aoi_new_area_ratio"),
"aoi_pair_area_ratio": candidate.get("aoi_pair_area_ratio"),
"aoi_coverage_ratio_after": candidate.get("aoi_coverage_ratio_after"),
"target_coverage_ratio_after": candidate.get("target_coverage_ratio_after"),
}
def _stable_sha1(self, value: Any) -> str:
@@ -949,7 +1332,12 @@ class SpatialService:
warnings.extend(strategy_warnings)
edges: List[Dict[str, Any]] = []
for edge_rank, candidate in enumerate(self._sorted_candidates(selected_candidates), start=1):
ordered_candidates = (
selected_candidates
if params.strategy == "dinsar_province_coverage"
else self._sorted_candidates(selected_candidates)
)
for edge_rank, candidate in enumerate(ordered_candidates, start=1):
master = candidate["master"]
slave = candidate["slave"]
edges.append(
@@ -1820,9 +2208,14 @@ class SpatialService:
return geometry_cache[cache_key]
try:
master_poly = Polygon(candidate["master"].coverage_polygon)
slave_poly = Polygon(candidate["slave"].coverage_polygon)
if master_poly.is_empty or slave_poly.is_empty:
master_poly = self._coverage_polygon_to_shape(getattr(candidate["master"], "coverage_polygon", None))
slave_poly = self._coverage_polygon_to_shape(getattr(candidate["slave"], "coverage_polygon", None))
if (
master_poly is None
or slave_poly is None
or master_poly.is_empty
or slave_poly.is_empty
):
geometry_cache[cache_key] = None
return None
pair_geom = master_poly.intersection(slave_poly)
@@ -1834,6 +2227,44 @@ class SpatialService:
geometry_cache[cache_key] = None
return None
def _coverage_polygon_to_shape(self, coverage_polygon: Any):
if not coverage_polygon:
return None
try:
if isinstance(coverage_polygon, dict):
if coverage_polygon.get("type") == "Feature":
geometry = coverage_polygon.get("geometry")
if not geometry:
return None
geom = shape(geometry)
else:
geom = shape(coverage_polygon)
elif isinstance(coverage_polygon, list):
points = []
for point in coverage_polygon:
if isinstance(point, (list, tuple)) and len(point) >= 2:
lon = float(point[0])
lat = float(point[1])
points.append((lon, lat))
if len(points) < 3:
return None
geom = Polygon(points)
else:
return None
if geom.is_empty or not geom.is_valid:
return None
return geom
except Exception:
return None
def _geometry_to_geojson(self, geometry: Any) -> Optional[Dict[str, Any]]:
try:
if geometry is None or geometry.is_empty:
return None
return mapping(geometry)
except Exception:
return None
def _score_sbas_candidate(
self,
candidate: dict,