Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve pairing planning and statistics visibility

Browse Source
This commit is contained in:
Harmon
2026-07-07 01:04:47 +08:00
parent 2da2121829
commit 36a406ae49
22 changed files with 2085 additions and 459 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/spatial_service.py
+436 -5
View File
@@ -7,8 +7,10 @@ import hashlib
import json
import logging
import math
import re
import uuid
from collections import defaultdict
from datetime import datetime, timedelta
from itertools import combinations
from typing import Any, Dict, List, Optional, Tuple
@@ -20,7 +22,7 @@ from sqlalchemy.orm import aliased
from geoalchemy2 import Geography
from geoalchemy2.shape import to_shape
from geoalchemy2.functions import ST_Intersects, ST_Intersection, ST_Area, ST_Centroid, ST_Covers
from shapely.geometry import Polygon
from shapely.geometry import Polygon, mapping, shape
from shapely.ops import unary_union
from ..models import (
@@ -196,6 +198,112 @@ class SpatialService:
}
return result_pairs, warnings, metadata
async def find_dinsar_coverage_pairs(
self,
db: AsyncSession,
params: PairingRequest,
*,
target_date_from: str,
target_date_to: str,
extension_days: int = 15,
max_pairs: int = 200,
target_coverage_ratio: float = 0.98,
min_new_coverage_ratio: float = 0.0005,
aoi_wkt: Optional[str] = None,
require_orbit_data: bool = True,
) -> Tuple[List[RadarPair], List[str], Dict[str, Any]]:
warnings: List[str] = []
target_start = self._parse_yyyymmdd(target_date_from, field_name="target_date_from")
target_end = self._parse_yyyymmdd(target_date_to, field_name="target_date_to")
if target_end < target_start:
raise ValueError("target_date_to must be greater than or equal to target_date_from.")
safe_extension_days = max(0, min(180, int(extension_days or 0)))
safe_max_pairs = max(1, min(5000, int(max_pairs or 200)))
safe_target_coverage_ratio = max(0.0, min(1.0, float(target_coverage_ratio or 0.98)))
safe_min_new_coverage_ratio = max(0.0, min(1.0, float(min_new_coverage_ratio or 0.0)))
query_start = target_start - timedelta(days=safe_extension_days)
query_end = target_end + timedelta(days=safe_extension_days)
effective_params = self._normalize_pairing_request(params).model_copy(
update={
"master_date_from": self._format_yyyymmdd(query_start),
"master_date_to": self._format_yyyymmdd(query_end),
"slave_date_from": self._format_yyyymmdd(query_start),
"slave_date_to": self._format_yyyymmdd(query_end),
"strategy": "dinsar_province_coverage",
}
)
pairing_status = await pairing_state_service.get_pairing_system_status(db)
cache_status = str(pairing_status.get("status") or "UNINITIALIZED")
scene_count = int(pairing_status.get("scene_count") or 0)
pair_count = int(pairing_status.get("pair_count") or 0)
degraded = bool(pairing_status.get("needs_rebuild"))
if cache_status in {"FAILED", "UNINITIALIZED", "ERROR"} or (scene_count > 1 and pair_count == 0):
raise RuntimeError(
"Pairing candidate cache is not available. Repair or rebuild the pairing foundation first."
)
if degraded:
warnings.append(
f"Pairing foundation status is {cache_status}; coverage plan uses current cached candidates."
)
candidate_pool = await self._query_pairing_metric_cache(
db,
effective_params,
aoi_wkt=aoi_wkt,
require_orbit_data=require_orbit_data,
)
selected_candidates, coverage_meta, strategy_warnings = self._apply_province_coverage_strategy(
candidate_pool,
target_start=target_start,
target_end=target_end,
query_start=query_start,
query_end=query_end,
max_pairs=safe_max_pairs,
target_coverage_ratio=safe_target_coverage_ratio,
min_new_coverage_ratio=safe_min_new_coverage_ratio,
aoi_wkt=aoi_wkt,
)
warnings.extend(strategy_warnings)
if not selected_candidates:
warnings.extend(
await self._build_empty_pairing_diagnostics(
db,
effective_params,
aoi_wkt=aoi_wkt,
require_orbit_data=require_orbit_data,
)
)
for candidate in selected_candidates:
candidate["selection_strategy"] = "dinsar_province_coverage"
self._ensure_candidate_identity(candidate)
network_run_id = await self._persist_network_run(
db,
params=effective_params,
aoi_wkt=aoi_wkt,
require_orbit_data=require_orbit_data,
warnings=warnings,
candidate_pool=candidate_pool,
selected_candidates=selected_candidates,
)
await self._attach_dinsar_production_summaries(db, selected_candidates)
result_pairs = self._generate_task_names(self._build_radar_pairs(selected_candidates))
metadata = {
"fallback_used": False,
"degraded": degraded,
"policy_version": PAIRING_POLICY_VERSION,
"network_run_id": network_run_id,
"candidate_count": len(candidate_pool),
"selected_edge_count": len(result_pairs),
"coverage": coverage_meta,
}
return result_pairs, warnings, metadata
def _normalize_pairing_request(self, params: PairingRequest) -> PairingRequest:
updates: Dict[str, Any] = {}
@@ -408,6 +516,27 @@ class SpatialService:
selection_strategy=candidate.get("selection_strategy"),
selection_score=float(selection_score) if selection_score is not None else None,
selection_reason=candidate.get("selection_reason"),
coverage_rank=(
int(candidate["coverage_rank"])
if candidate.get("coverage_rank") is not None
else None
),
aoi_new_area_ratio=(
float(candidate["aoi_new_area_ratio"])
if candidate.get("aoi_new_area_ratio") is not None
else None
),
aoi_pair_area_ratio=(
float(candidate["aoi_pair_area_ratio"])
if candidate.get("aoi_pair_area_ratio") is not None
else None
),
aoi_coverage_ratio_after=(
float(candidate["aoi_coverage_ratio_after"])
if candidate.get("aoi_coverage_ratio_after") is not None
else None
),
effective_coverage_geojson=candidate.get("effective_coverage_geojson"),
time_baseline_days=int(candidate["days"]),
spatial_baseline_meters=float(candidate["dist"]),
scene_center_distance_meters=float(
@@ -701,6 +830,253 @@ class SpatialService:
output.append(item)
return output
def _apply_province_coverage_strategy(
self,
candidate_pool: List[dict],
*,
target_start: datetime,
target_end: datetime,
query_start: datetime,
query_end: datetime,
max_pairs: int,
target_coverage_ratio: float,
min_new_coverage_ratio: float,
aoi_wkt: Optional[str],
) -> Tuple[List[dict], Dict[str, Any], List[str]]:
warnings: List[str] = []
target_days = self._date_set(target_start, target_end)
covered_days: set[datetime] = set()
selected: List[dict] = []
remaining = [candidate for candidate in candidate_pool if self._candidate_date_window(candidate) is not None]
seen_metric_ids: set[int] = set()
aoi_poly = self._parse_optional_aoi_polygon(aoi_wkt)
geometry_cache: Dict[int, Any] = {}
selected_coverage = Polygon()
aoi_area = float(aoi_poly.area or 0.0) if aoi_poly is not None else 0.0
stop_reason = "no_more_gain"
max_pairs_reached = False
while remaining and len(selected) < max_pairs:
best_candidate = None
best_score: Optional[Tuple[float, float, float, float, float, float, str]] = None
current_aoi_coverage_ratio = (
float(selected_coverage.area or 0.0) / aoi_area
if aoi_poly is not None and aoi_area > 0
else None
)
if current_aoi_coverage_ratio is not None and current_aoi_coverage_ratio >= target_coverage_ratio:
stop_reason = "target_coverage_reached"
break
for candidate in remaining:
window = self._candidate_date_window(candidate)
if window is None:
continue
candidate_days = self._date_set(*window) & target_days
new_days = candidate_days - covered_days
new_area = 0.0
candidate_area = 0.0
if aoi_poly is not None:
candidate_geom = self._get_candidate_intersection_geom(
candidate,
aoi_poly=aoi_poly,
geometry_cache=geometry_cache,
)
if candidate_geom is not None and not candidate_geom.is_empty:
candidate_area = float(candidate_geom.area or 0.0)
new_area = float(candidate_geom.difference(selected_coverage).area or 0.0)
if aoi_poly is not None:
new_area_ratio = (new_area / aoi_area) if aoi_area > 0 else 0.0
if new_area_ratio <= 0:
continue
elif not new_days:
continue
quality_score = float(candidate.get("dinsar_quality_score") or 0.0)
overlap = float(candidate.get("overlap_ratio") or 0.0)
temporal_days = float(candidate.get("days") or 0.0)
distance = float(candidate.get("scene_center_distance_meters") or candidate.get("dist") or 0.0)
if aoi_poly is not None and aoi_area > 0:
score = (
new_area_ratio,
candidate_area / aoi_area,
float(len(new_days)) / max(1, len(target_days)),
overlap,
quality_score,
-temporal_days - (distance / 1000000.0),
str(candidate.get("pair_uid") or ""),
)
else:
score = (
float(len(new_days)),
float(len(candidate_days)),
overlap,
quality_score,
0.0,
-temporal_days - (distance / 1000000.0),
str(candidate.get("pair_uid") or ""),
)
if best_score is None or score > best_score:
best_candidate = candidate
best_score = score
if best_candidate is None:
stop_reason = "no_candidate_adds_coverage"
break
if aoi_poly is None and covered_days == target_days:
stop_reason = "target_time_reached"
break
window = self._candidate_date_window(best_candidate)
candidate_days = self._date_set(*window) & target_days if window else set()
new_days = candidate_days - covered_days
candidate_geom = self._get_candidate_intersection_geom(
best_candidate,
aoi_poly=aoi_poly,
geometry_cache=geometry_cache,
) if aoi_poly is not None else None
new_area = 0.0
candidate_area = 0.0
if candidate_geom is not None and not candidate_geom.is_empty:
candidate_area = float(candidate_geom.area or 0.0)
new_area = float(candidate_geom.difference(selected_coverage).area or 0.0)
new_area_ratio = (new_area / aoi_area) if aoi_area > 0 else 0.0
if aoi_poly is not None and len(selected) > 0 and new_area_ratio < min_new_coverage_ratio:
stop_reason = "marginal_gain_below_threshold"
break
best_candidate["selection_reason"] = "province_coverage_new_days"
if aoi_poly is not None and new_area > 0:
best_candidate["selection_reason"] = "province_aoi_new_area"
best_candidate["selection_score"] = float(best_score[0] if best_score else len(new_days))
best_candidate["coverage_rank"] = len(selected) + 1
best_candidate["coverage_new_days"] = len(new_days)
best_candidate["coverage_total_days"] = len(candidate_days)
best_candidate["aoi_new_area_ratio"] = new_area_ratio if aoi_area > 0 else None
best_candidate["aoi_pair_area_ratio"] = (candidate_area / aoi_area) if aoi_area > 0 else None
next_selected_coverage = selected_coverage
if candidate_geom is not None and not candidate_geom.is_empty:
next_selected_coverage = unary_union([selected_coverage, candidate_geom])
best_candidate["aoi_coverage_ratio_after"] = (
float(next_selected_coverage.area or 0.0) / aoi_area
if aoi_area > 0 else None
)
best_candidate["effective_coverage_geojson"] = self._geometry_to_geojson(candidate_geom)
best_candidate["target_coverage_ratio_after"] = (
len(covered_days | candidate_days) / max(1, len(target_days))
)
selected.append(best_candidate)
covered_days |= candidate_days
selected_coverage = next_selected_coverage
seen_metric_ids.add(int(best_candidate.get("metric_cache_ref_id") or 0))
remaining = [
candidate for candidate in remaining
if int(candidate.get("metric_cache_ref_id") or 0) not in seen_metric_ids
]
if remaining and len(selected) >= max_pairs:
max_pairs_reached = True
stop_reason = "max_pairs_reached"
uncovered_ranges = self._date_ranges_from_days(target_days - covered_days)
if uncovered_ranges:
warnings.append(
"Coverage plan did not fully cover the requested time range. "
f"Uncovered ranges: {', '.join(f'{item[0]}~{item[1]}' for item in uncovered_ranges[:6])}"
)
if max_pairs_reached:
warnings.append(
f"Coverage planning reached task limit max_pairs={max_pairs}; increase the limit for better spatial coverage."
)
selected_area = float(selected_coverage.area or 0.0) if aoi_poly is not None else 0.0
aoi_coverage_ratio = (selected_area / aoi_area) if aoi_area > 0 else None
if aoi_coverage_ratio is not None and aoi_coverage_ratio < target_coverage_ratio:
warnings.append(
f"AOI spatial coverage is {aoi_coverage_ratio:.1%}; target is {target_coverage_ratio:.1%}."
)
temporal_ratio = len(covered_days) / max(1, len(target_days))
coverage_meta = {
"strategy": "dinsar_province_coverage",
"optimization_goal": "minimize_pair_count_for_aoi_spatial_coverage",
"greedy_rule": "select_pair_with_largest_new_aoi_intersection_area_each_step",
"stop_reason": stop_reason,
"max_pairs": max_pairs,
"target_coverage_ratio": round(target_coverage_ratio, 6),
"min_new_coverage_ratio": round(min_new_coverage_ratio, 6),
"coverage_basis": "aoi_spatial" if aoi_coverage_ratio is not None else "temporal",
"target_date_from": self._format_yyyymmdd(target_start),
"target_date_to": self._format_yyyymmdd(target_end),
"query_date_from": self._format_yyyymmdd(query_start),
"query_date_to": self._format_yyyymmdd(query_end),
"target_day_count": len(target_days),
"covered_day_count": len(covered_days),
"temporal_coverage_ratio": round(temporal_ratio, 6),
"aoi_coverage_ratio": round(aoi_coverage_ratio, 6) if aoi_coverage_ratio is not None else None,
"coverage_ratio": round(aoi_coverage_ratio if aoi_coverage_ratio is not None else temporal_ratio, 6),
"uncovered_ranges": [
{"date_from": start, "date_to": end}
for start, end in uncovered_ranges
],
"selected_pair_count": len(selected),
"candidate_count": len(candidate_pool),
}
return selected, coverage_meta, warnings
def _candidate_date_window(self, candidate: dict) -> Optional[Tuple[datetime, datetime]]:
master = candidate.get("master")
slave = candidate.get("slave")
master_date = self._try_parse_yyyymmdd(getattr(master, "imaging_date", None))
slave_date = self._try_parse_yyyymmdd(getattr(slave, "imaging_date", None))
if master_date is None or slave_date is None:
return None
return (master_date, slave_date) if master_date <= slave_date else (slave_date, master_date)
def _sort_coverage_candidates(self, candidates: List[dict]) -> List[dict]:
return sorted(
candidates,
key=lambda item: (
str(getattr(item.get("master"), "imaging_date", "") or ""),
str(getattr(item.get("slave"), "imaging_date", "") or ""),
-float(item.get("selection_score") or 0.0),
str(item.get("pair_uid") or ""),
),
)
def _date_set(self, start: datetime, end: datetime) -> set[datetime]:
if end < start:
return set()
return {start + timedelta(days=offset) for offset in range((end - start).days + 1)}
def _date_ranges_from_days(self, days: set[datetime]) -> List[Tuple[str, str]]:
if not days:
return []
ordered = sorted(days)
ranges: List[Tuple[datetime, datetime]] = []
start = previous = ordered[0]
for day in ordered[1:]:
if day == previous + timedelta(days=1):
previous = day
continue
ranges.append((start, previous))
start = previous = day
ranges.append((start, previous))
return [(self._format_yyyymmdd(start), self._format_yyyymmdd(end)) for start, end in ranges]
def _parse_yyyymmdd(self, value: str, *, field_name: str) -> datetime:
parsed = self._try_parse_yyyymmdd(value)
if parsed is None:
raise ValueError(f"{field_name} must be YYYYMMDD.")
return parsed
def _try_parse_yyyymmdd(self, value: Any) -> Optional[datetime]:
text_value = str(value or "").strip()
if not re.match(r"^\d{8}$", text_value):
return None
try:
return datetime.strptime(text_value, "%Y%m%d")
except ValueError:
return None
def _format_yyyymmdd(self, value: datetime) -> str:
return value.strftime("%Y%m%d")
async def _persist_network_run(
self,
db: AsyncSession,
@@ -797,6 +1173,13 @@ class SpatialService:
if candidate.get("pair_aoi_overlap_ratio") is not None
else None
),
"coverage_rank": candidate.get("coverage_rank"),
"coverage_new_days": candidate.get("coverage_new_days"),
"coverage_total_days": candidate.get("coverage_total_days"),
"aoi_new_area_ratio": candidate.get("aoi_new_area_ratio"),
"aoi_pair_area_ratio": candidate.get("aoi_pair_area_ratio"),
"aoi_coverage_ratio_after": candidate.get("aoi_coverage_ratio_after"),
"target_coverage_ratio_after": candidate.get("target_coverage_ratio_after"),
}
def _stable_sha1(self, value: Any) -> str:
@@ -949,7 +1332,12 @@ class SpatialService:
warnings.extend(strategy_warnings)
edges: List[Dict[str, Any]] = []
for edge_rank, candidate in enumerate(self._sorted_candidates(selected_candidates), start=1):
ordered_candidates = (
selected_candidates
if params.strategy == "dinsar_province_coverage"
else self._sorted_candidates(selected_candidates)
)
for edge_rank, candidate in enumerate(ordered_candidates, start=1):
master = candidate["master"]
slave = candidate["slave"]
edges.append(
@@ -1820,9 +2208,14 @@ class SpatialService:
return geometry_cache[cache_key]
try:
master_poly = Polygon(candidate["master"].coverage_polygon)
slave_poly = Polygon(candidate["slave"].coverage_polygon)
if master_poly.is_empty or slave_poly.is_empty:
master_poly = self._coverage_polygon_to_shape(getattr(candidate["master"], "coverage_polygon", None))
slave_poly = self._coverage_polygon_to_shape(getattr(candidate["slave"], "coverage_polygon", None))
if (
master_poly is None
or slave_poly is None
or master_poly.is_empty
or slave_poly.is_empty
):
geometry_cache[cache_key] = None
return None
pair_geom = master_poly.intersection(slave_poly)
@@ -1834,6 +2227,44 @@ class SpatialService:
geometry_cache[cache_key] = None
return None
def _coverage_polygon_to_shape(self, coverage_polygon: Any):
if not coverage_polygon:
return None
try:
if isinstance(coverage_polygon, dict):
if coverage_polygon.get("type") == "Feature":
geometry = coverage_polygon.get("geometry")
if not geometry:
return None
geom = shape(geometry)
else:
geom = shape(coverage_polygon)
elif isinstance(coverage_polygon, list):
points = []
for point in coverage_polygon:
if isinstance(point, (list, tuple)) and len(point) >= 2:
lon = float(point[0])
lat = float(point[1])
points.append((lon, lat))
if len(points) < 3:
return None
geom = Polygon(points)
else:
return None
if geom.is_empty or not geom.is_valid:
return None
return geom
except Exception:
return None
def _geometry_to_geojson(self, geometry: Any) -> Optional[Dict[str, Any]]:
try:
if geometry is None or geometry.is_empty:
return None
return mapping(geometry)
except Exception:
return None
def _score_sbas_candidate(
self,
candidate: dict,