Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: improve D-InSAR pairing and distribution

Browse Source
This commit is contained in:
Harmon
2026-05-09 14:59:20 +08:00
parent 69d15bffcb
commit 5c73e2c1c2
28 changed files with 1480 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/copier.py
+173 -10
View File
@@ -1,6 +1,8 @@
import os
import shutil
import asyncio
import tempfile
import zipfile
from datetime import datetime
from typing import List, Tuple, Optional, Dict, Any
@@ -34,14 +36,128 @@ async def _log_and_update(task_id: str, message: str, progress: Optional[int] =
def find_dinsar_source_to_copy(path: str) -> str:
"""
Find D-InSAR source path.
Prefer the envi_import subfolder when present and non-empty.
D-InSAR pairing/distribution works on the raw source product directory.
"""
envi_path = os.path.join(path, "envi_import")
if os.path.isdir(envi_path) and os.listdir(envi_path):
return envi_path
return path
def _resolve_orbit_dest_path(
orbit_dir: str,
role: str,
source_path: str,
used_dest_paths: Dict[str, str],
) -> str:
base_name = os.path.basename(source_path)
dest_path = os.path.join(orbit_dir, base_name)
dest_key = os.path.normcase(os.path.abspath(dest_path))
source_key = os.path.normcase(os.path.abspath(source_path))
if dest_key not in used_dest_paths or used_dest_paths[dest_key] == source_key:
return dest_path
role_path = os.path.join(orbit_dir, f"{role}_{base_name}")
role_key = os.path.normcase(os.path.abspath(role_path))
if role_key not in used_dest_paths or used_dest_paths[role_key] == source_key:
return role_path
stem, ext = os.path.splitext(base_name)
counter = 2
while True:
numbered_path = os.path.join(orbit_dir, f"{role}_{stem}_{counter}{ext}")
numbered_key = os.path.normcase(os.path.abspath(numbered_path))
if numbered_key not in used_dest_paths:
return numbered_path
counter += 1
async def _copy_dinsar_orbit_files(
task_id: str,
item: Dict[str, Any],
task_dir: str,
include_orbit_files: bool,
) -> List[Dict[str, Any]]:
if not include_orbit_files:
return []
orbit_dir = os.path.join(task_dir, "orbit")
copied_by_source: Dict[str, str] = {}
used_dest_paths: Dict[str, str] = {}
orbit_entries: List[Dict[str, Any]] = []
for role, key in (
("master", "master_orbit_file_path"),
("slave", "slave_orbit_file_path"),
):
raw_path = item.get(key)
if not raw_path:
await _log_and_update(task_id, f" -> {role} orbit missing in catalog metadata")
orbit_entries.append(
{
"role": role,
"source_path": None,
"copied": False,
"reason": "missing_orbit_path",
}
)
continue
source_path = os.path.normpath(os.path.abspath(str(raw_path)))
if not os.path.isfile(source_path):
await _log_and_update(task_id, f" -> {role} orbit file not found: {source_path}")
orbit_entries.append(
{
"role": role,
"source_path": source_path,
"copied": False,
"reason": "source_file_not_found",
}
)
continue
await asyncio.to_thread(os.makedirs, orbit_dir, exist_ok=True)
source_key = os.path.normcase(source_path)
if source_key in copied_by_source:
dest_path = copied_by_source[source_key]
else:
dest_path = _resolve_orbit_dest_path(orbit_dir, role, source_path, used_dest_paths)
await asyncio.to_thread(shutil.copy2, source_path, dest_path)
copied_by_source[source_key] = dest_path
used_dest_paths[os.path.normcase(os.path.abspath(dest_path))] = source_key
orbit_entries.append(
{
"role": role,
"source_path": source_path,
"relative_path": os.path.relpath(dest_path, start=task_dir),
"copied": True,
}
)
return orbit_entries
def _zip_task_directory(task_dir: str, zip_path: str) -> None:
parent_dir = os.path.dirname(task_dir)
temp_zip_path = f"{zip_path}.tmp"
try:
if os.path.exists(temp_zip_path):
os.remove(temp_zip_path)
with zipfile.ZipFile(temp_zip_path, "w", compression=zipfile.ZIP_DEFLATED) as archive:
for root, dirs, files in os.walk(task_dir):
rel_root = os.path.relpath(root, start=parent_dir)
for dirname in dirs:
arcname = os.path.join(rel_root, dirname).replace(os.sep, "/") + "/"
archive.writestr(arcname, "")
for filename in files:
file_path = os.path.join(root, filename)
arcname = os.path.join(rel_root, filename).replace(os.sep, "/")
archive.write(file_path, arcname)
os.replace(temp_zip_path, zip_path)
finally:
if os.path.exists(temp_zip_path):
try:
os.remove(temp_zip_path)
except OSError:
pass
async def run_ps_copy_items(task_id: str, items: List[Dict[str, Any]], dest_dir: str) -> None:
try:
await task_service.start_task(task_id, message="Starting PS-InSAR copy task...")
@@ -124,10 +240,25 @@ async def run_ps_copy_items(task_id: str, items: List[Dict[str, Any]], dest_dir:
raise CopyTaskExecutionError(fail_msg) from e
async def run_dinsar_copy_items(task_id: str, items: List[Dict[str, Any]], dest_dir: str) -> None:
async def run_dinsar_copy_items(
task_id: str,
items: List[Dict[str, Any]],
dest_dir: str,
*,
include_orbit_files: bool = False,
export_zip: bool = False,
) -> None:
try:
await task_service.start_task(task_id, message="Starting D-InSAR copy task...")
await _log_and_update(task_id, f"D-InSAR copy started. Dest: {dest_dir}")
await _log_and_update(
task_id,
(
"D-InSAR copy options: "
f"include_orbit_files={include_orbit_files}, "
f"export_zip={export_zip}"
),
)
if not os.path.exists(dest_dir):
try:
@@ -175,11 +306,22 @@ async def run_dinsar_copy_items(task_id: str, items: List[Dict[str, Any]], dest_
await _log_and_update(task_id, f"[{i}/{total}] Processing: {task_name}")
task_dir = os.path.join(dest_dir, task_alias)
master_dir = os.path.join(task_dir, "master")
slave_dir = os.path.join(task_dir, "slave")
staging_root: Optional[str] = None
try:
if export_zip:
staging_root = await asyncio.to_thread(
tempfile.mkdtemp,
prefix="._dinsar_zip_",
dir=dest_dir,
)
task_dir = os.path.join(staging_root, task_alias)
zip_path = os.path.join(dest_dir, f"{task_alias}.zip")
else:
task_dir = os.path.join(dest_dir, task_alias)
zip_path = None
master_dir = os.path.join(task_dir, "master")
slave_dir = os.path.join(task_dir, "slave")
master_src_path = find_dinsar_source_to_copy(master_path)
if not os.path.exists(master_src_path):
await _log_and_update(task_id, f" -> Missing master: {master_src_path}")
@@ -194,6 +336,12 @@ async def run_dinsar_copy_items(task_id: str, items: List[Dict[str, Any]], dest_
await asyncio.to_thread(shutil.copytree, master_src_path, master_dir, dirs_exist_ok=True)
await asyncio.to_thread(shutil.copytree, slave_src_path, slave_dir, dirs_exist_ok=True)
orbit_entries = await _copy_dinsar_orbit_files(
task_id,
item,
task_dir,
include_orbit_files,
)
await asyncio.to_thread(
write_pair_metadata,
task_dir,
@@ -213,6 +361,12 @@ async def run_dinsar_copy_items(task_id: str, items: List[Dict[str, Any]], dest_
"slave_polarization": item.get("slave_polarization"),
"time_baseline_days": item.get("time_baseline_days"),
"spatial_baseline_meters": item.get("spatial_baseline_meters"),
"scene_center_distance_meters": item.get("scene_center_distance_meters"),
"package_format": "zip" if export_zip else "folder",
"include_orbit_files": bool(include_orbit_files),
"master_orbit_file_path": item.get("master_orbit_file_path"),
"slave_orbit_file_path": item.get("slave_orbit_file_path"),
"orbit_files": orbit_entries,
"scene_pair_uid": item.get("scene_pair_uid") or item.get("pair_uid"),
"pair_uid": item.get("pair_uid") or item.get("scene_pair_uid"),
"network_run_id": item.get("network_run_id"),
@@ -222,6 +376,9 @@ async def run_dinsar_copy_items(task_id: str, items: List[Dict[str, Any]], dest_
"copied_at": datetime.utcnow().isoformat(timespec="seconds") + "Z",
},
)
if export_zip and zip_path:
await asyncio.to_thread(_zip_task_directory, task_dir, zip_path)
await _log_and_update(task_id, f" -> ZIP: {zip_path}")
await _log_and_update(task_id, " -> Success")
success_count += 1
@@ -231,8 +388,14 @@ async def run_dinsar_copy_items(task_id: str, items: List[Dict[str, Any]], dest_
except Exception as e:
await _log_and_update(task_id, f" -> Failed: {e}")
failed_count += 1
finally:
if staging_root:
await asyncio.to_thread(shutil.rmtree, staging_root, ignore_errors=True)
final_msg = f"D-InSAR copy finished. Success {success_count}, Failed {failed_count}"
final_msg = (
f"D-InSAR copy finished. Mode {'zip' if export_zip else 'folder'}. "
f"Success {success_count}, Failed {failed_count}"
)
await _log_and_update(task_id, final_msg, progress=100)
if failed_count > 0:
await task_service.update_task(task_id, status="FAILED", message=final_msg, progress=100)
backend/app/db_maintenance.py
+1
View File
@@ -35,6 +35,7 @@ MIGRATION_FILES = [
"006_result_pairing_trace.sql",
"007_timeseries_stack_plan_trace.sql",
"008_timeseries_stack_plan_edges.sql",
"009_raw_source_pairing_fields.sql",
]
backend/app/dinsar_engines/isce2_engine.py
+1
View File
@@ -1334,6 +1334,7 @@ class Isce2Engine(DinsarEngine):
"slave_polarization": pair_meta.get("slave_polarization"),
"time_baseline_days": pair_meta.get("time_baseline_days"),
"spatial_baseline_meters": pair_meta.get("spatial_baseline_meters"),
"scene_center_distance_meters": pair_meta.get("scene_center_distance_meters"),
"scene_pair_uid": pair_meta.get("scene_pair_uid") or pair_meta.get("pair_uid"),
"pair_uid": pair_meta.get("pair_uid") or pair_meta.get("scene_pair_uid"),
"network_run_id": pair_meta.get("network_run_id"),
backend/app/dinsar_engines/pyint_engine.py
+1
View File
@@ -1290,6 +1290,7 @@ class PyintEngine(DinsarEngine):
"slave_polarization": pair_meta.get("slave_polarization"),
"time_baseline_days": pair_meta.get("time_baseline_days") or time_baseline_days,
"spatial_baseline_meters": pair_meta.get("spatial_baseline_meters"),
"scene_center_distance_meters": pair_meta.get("scene_center_distance_meters"),
"scene_pair_uid": pair_meta.get("scene_pair_uid") or pair_meta.get("pair_uid"),
"pair_uid": pair_meta.get("pair_uid") or pair_meta.get("scene_pair_uid"),
"network_run_id": pair_meta.get("network_run_id"),
backend/app/models/orm.py
+20 -2
View File
@@ -33,8 +33,17 @@ class RadarDataORM(Base):
scene_center_lat = Column(Float, nullable=True)
acquisition_time_utc = Column(String, nullable=True)
product_type = Column(String, nullable=True)
source_product_token = Column(String, nullable=True)
image_data_type = Column(String, nullable=True)
image_data_format = Column(String, nullable=True)
product_variant = Column(String, nullable=True)
product_level = Column(String, nullable=True)
product_unique_id = Column(String, nullable=True)
satellite_family = Column(String, index=True, nullable=True)
look_direction = Column(String, index=True, nullable=True)
geocoded_flag = Column(Boolean, nullable=True)
insar_source_ready = Column(Boolean, nullable=False, default=False, server_default="false")
insar_source_reason = Column(Text, nullable=True)
file_path = Column(String, unique=True)
has_orbit_data = Column(Boolean)
orbit_file_path = Column(String, nullable=True)
@@ -179,6 +188,7 @@ class DinsarProductProfileORM(Base):
orbit_direction = Column(String, index=True, nullable=True)
time_baseline_days = Column(Integer, index=True, nullable=True)
spatial_baseline_meters = Column(Float, index=True, nullable=True)
scene_center_distance_meters = Column(Float, index=True, nullable=True)
grid_size_m = Column(Float, nullable=True)
radar_wavelength = Column(Float, nullable=True)
@@ -287,7 +297,7 @@ class PairingCacheStateORM(Base):
id = Column(Integer, primary_key=True, autoincrement=True)
cache_scope = Column(String(32), unique=True, index=True, nullable=False, default="global")
metric_version = Column(String(32), nullable=False, default="2026.04.v1")
metric_version = Column(String(32), nullable=False, default="2026.05.raw.v1")
status = Column(String(16), index=True, nullable=False, default="DIRTY")
scene_count = Column(Integer, nullable=False, default=0)
pair_count = Column(Integer, nullable=False, default=0)
@@ -338,23 +348,30 @@ class PairingMetricCacheORM(Base):
master_scene_uid = Column(String, index=True, nullable=False)
slave_scene_uid = Column(String, index=True, nullable=False)
pair_uid = Column(String, index=True, nullable=False)
metric_version = Column(String(32), index=True, nullable=False, default="2026.04.v1")
metric_version = Column(String(32), index=True, nullable=False, default="2026.05.raw.v1")
orientation_rule_version = Column(String(32), nullable=False, default="date_then_scene_uid_v1")
time_baseline_days = Column(Integer, index=True, nullable=True)
spatial_baseline_meters = Column(Float, index=True, nullable=True)
scene_center_distance_meters = Column(Float, index=True, nullable=True)
scene_overlap_ratio = Column(Float, index=True, nullable=True)
orbit_direction = Column(String, index=True, nullable=True)
same_satellite = Column(Boolean, nullable=False, default=True)
same_satellite_family = Column(Boolean, nullable=False, default=True, server_default="true")
same_look_direction = Column(Boolean, nullable=False, default=True, server_default="true")
same_imaging_mode = Column(Boolean, nullable=False, default=True)
same_polarization = Column(Boolean, nullable=False, default=True)
master_imaging_date = Column(String(8), index=True, nullable=True)
slave_imaging_date = Column(String(8), index=True, nullable=True)
master_satellite = Column(String, index=True, nullable=True)
slave_satellite = Column(String, index=True, nullable=True)
master_satellite_family = Column(String, index=True, nullable=True)
slave_satellite_family = Column(String, index=True, nullable=True)
master_imaging_mode = Column(String, nullable=True)
slave_imaging_mode = Column(String, nullable=True)
master_polarization = Column(String, nullable=True)
slave_polarization = Column(String, nullable=True)
master_look_direction = Column(String, nullable=True)
slave_look_direction = Column(String, nullable=True)
master_file_path = Column(String, nullable=True)
slave_file_path = Column(String, nullable=True)
status = Column(String(16), index=True, nullable=False, default="READY")
@@ -927,6 +944,7 @@ class DinsarTaskItemORM(Base):
slave_polarization = Column(String, nullable=True)
time_baseline_days = Column(Integer, nullable=True)
spatial_baseline_meters = Column(Float, nullable=True)
scene_center_distance_meters = Column(Float, nullable=True)
status = Column(String, index=True, nullable=False, default="PENDING")
remark = Column(Text, nullable=True)
backend/app/models/schemas.py
+30 -1
View File
@@ -5,7 +5,7 @@ Pydantic Schema 定义。
from datetime import datetime
from typing import Any, Dict, List, Optional, Tuple, Union
from pydantic import BaseModel, ConfigDict, Field, computed_field, field_validator
from pydantic import BaseModel, ConfigDict, Field, computed_field, field_validator, model_validator
from ..config import read_int_env
@@ -178,8 +178,17 @@ class RadarData(BaseModel):
scene_center_lat: Optional[float] = None
acquisition_time_utc: Optional[str] = None
product_type: Optional[str] = None
source_product_token: Optional[str] = None
image_data_type: Optional[str] = None
image_data_format: Optional[str] = None
product_variant: Optional[str] = None
product_level: Optional[str] = None
product_unique_id: Optional[str] = None
satellite_family: Optional[str] = None
look_direction: Optional[str] = None
geocoded_flag: Optional[bool] = None
insar_source_ready: bool = False
insar_source_reason: Optional[str] = None
file_path: str
has_orbit_data: bool
orbit_file_path: Optional[str] = None
@@ -242,6 +251,9 @@ class PairingRequest(BaseModel):
require_same_imaging_mode: bool = True
require_same_polarization: bool = True
aoi_overlap_threshold: Optional[float] = Field(default=None, ge=0.0, le=1.0)
max_temporal_baseline_days: Optional[int] = Field(default=None, ge=1, le=3650)
pair_footprint_overlap_min_ratio: Optional[float] = Field(default=None, ge=0.0, le=1.0)
footprint_center_distance_max_meters: Optional[int] = Field(default=None, ge=0, le=100000)
# === 双池日期（新增） ===
master_date_from: Optional[str] = Field(default=None, pattern=r'^\d{8}$|^$')
@@ -261,6 +273,20 @@ class PairingRequest(BaseModel):
# === 向后兼容（保留） ===
start_date: Optional[str] = Field(default=None, pattern=r'^\d{8}$|^$')
@model_validator(mode='before')
@classmethod
def _apply_aliases(cls, data):
if not isinstance(data, dict):
return data
normalized = dict(data)
if normalized.get('max_temporal_baseline_days') not in (None, ''):
normalized['time_baseline_max'] = normalized['max_temporal_baseline_days']
if normalized.get('pair_footprint_overlap_min_ratio') not in (None, ''):
normalized['overlap_threshold'] = normalized['pair_footprint_overlap_min_ratio']
if normalized.get('footprint_center_distance_max_meters') not in (None, ''):
normalized['spatial_baseline_max_meters'] = normalized['footprint_center_distance_max_meters']
return normalized
@field_validator(
'master_date_from',
'master_date_to',
@@ -336,6 +362,7 @@ class RadarPair(BaseModel):
selection_reason: Optional[str] = None
time_baseline_days: int
spatial_baseline_meters: float
scene_center_distance_meters: Optional[float] = None
class PairingResponse(BaseModel):
@@ -413,6 +440,7 @@ class TimeseriesStackPlanEdge(BaseModel):
slave_imaging_date: Optional[str] = None
temporal_baseline_days: Optional[int] = None
spatial_baseline_meters: Optional[float] = None
scene_center_distance_meters: Optional[float] = None
perpendicular_baseline_meters: Optional[float] = None
scene_overlap_ratio: Optional[float] = None
pair_aoi_overlap_ratio: Optional[float] = None
@@ -518,6 +546,7 @@ class DinsarTaskItem(BaseModel):
slave_polarization: Optional[str] = None
time_baseline_days: Optional[int] = None
spatial_baseline_meters: Optional[float] = None
scene_center_distance_meters: Optional[float] = None
status: str
remark: Optional[str] = None
created_at: datetime
backend/app/routers/pairing.py
+6
View File
@@ -42,6 +42,9 @@ def get_pairing_request_from_form(
time_baseline_max: int = Form(90),
overlap_threshold: float = Form(0.5),
spatial_baseline_max_meters: int = Form(3000),
max_temporal_baseline_days: Optional[int] = Form(None),
pair_footprint_overlap_min_ratio: Optional[float] = Form(None),
footprint_center_distance_max_meters: Optional[int] = Form(None),
coverage_diversity_penalty: float = Form(0.3),
require_same_imaging_mode: bool = Form(True),
require_same_polarization: bool = Form(True),
@@ -72,6 +75,9 @@ def get_pairing_request_from_form(
time_baseline_max=time_baseline_max,
overlap_threshold=overlap_threshold,
spatial_baseline_max_meters=spatial_baseline_max_meters,
max_temporal_baseline_days=max_temporal_baseline_days,
pair_footprint_overlap_min_ratio=pair_footprint_overlap_min_ratio,
footprint_center_distance_max_meters=footprint_center_distance_max_meters,
coverage_diversity_penalty=coverage_diversity_penalty,
require_same_imaging_mode=require_same_imaging_mode,
require_same_polarization=require_same_polarization,
backend/app/routers/task_batches.py
+5
View File
@@ -266,6 +266,11 @@ async def create_dinsar_batch_endpoint(
slave_polarization=slave.polarization,
time_baseline_days=pair.time_baseline_days,
spatial_baseline_meters=pair.spatial_baseline_meters,
scene_center_distance_meters=(
pair.scene_center_distance_meters
if pair.scene_center_distance_meters is not None
else pair.spatial_baseline_meters
),
status="PENDING",
)
db.add(item)
backend/app/routers/tools.py
+8
View File
@@ -33,6 +33,8 @@ class CopyBatchRequest(BaseModel):
batch_id: str = Field(max_length=COPY_BATCH_TEXT_MAX_LENGTH)
dest_dir: str = Field(max_length=COPY_BATCH_TEXT_MAX_LENGTH)
copy_statuses: Optional[List[str]] = None
include_orbit_files: bool = False
export_zip: bool = False
@field_validator("batch_id", "dest_dir", mode="before")
@classmethod
@@ -138,6 +140,8 @@ async def copy_dinsar_pairs_endpoint(
"file_type": "DINSAR_PAIRS",
"batch_id": request.batch_id,
"copy_statuses": copy_statuses,
"include_orbit_files": bool(request.include_orbit_files),
"export_zip": bool(request.export_zip),
}
task_id = await task_service.create_task("COPY_DATA", f"D-InSAR 数据分发: {request.dest_dir}", params=params)
@@ -146,6 +150,8 @@ async def copy_dinsar_pairs_endpoint(
"dest_dir": request.dest_dir,
"batch_id": request.batch_id,
"copy_statuses": copy_statuses,
"include_orbit_files": bool(request.include_orbit_files),
"export_zip": bool(request.export_zip),
}
await job_queue_service.create_job("COPY_DATA", payload=payload, task_id=task_id)
await _add_operation_audit_log(
@@ -158,6 +164,8 @@ async def copy_dinsar_pairs_endpoint(
"batch_id": request.batch_id,
"dest_dir": request.dest_dir,
"copy_statuses": copy_statuses,
"include_orbit_files": bool(request.include_orbit_files),
"export_zip": bool(request.export_zip),
},
)
await db.commit()
backend/app/services/data_service.py
+113 -14
View File
@@ -33,7 +33,13 @@ from ..config import settings
from ..models import (
RadarDataORM, RadarData, DinsarResultORM, HazardPointORM, HazardPoint, ScanStateORM
)
from ..utils import get_parser, RADAR_PARSERS, find_xml_file, parse_xml_metadata
from ..utils import (
get_parser,
RADAR_PARSERS,
find_xml_file,
normalize_satellite_family,
parse_xml_metadata,
)
from .task_service import task_service
from .image_service import image_service
from .orbit_converter import get_source_orbit_inventory, sync_orbit_pools
@@ -50,7 +56,7 @@ def _safe_mtime(path: str) -> float:
_DATE_RE = re.compile(r"^\d{8}$")
def _valid_imaging_date(value: str) -> bool:
def _valid_imaging_date(value: Optional[str]) -> bool:
return bool(value and _DATE_RE.match(value))
@@ -63,6 +69,62 @@ def _extract_date_from_text(value: Optional[str]) -> Optional[str]:
return None
def _text_or_none(value: Any) -> Optional[str]:
if value is None:
return None
text_value = str(value).strip()
return text_value or None
def _bool_or_none(value: Any) -> Optional[bool]:
if value is None or isinstance(value, bool):
return value
text_value = str(value).strip().lower()
if text_value in {"1", "true", "yes", "y"}:
return True
if text_value in {"0", "false", "no", "n"}:
return False
return None
def _build_insar_source_readiness(
meta: Dict[str, Any],
coverage_polygon: Optional[List[Tuple[float, float]]],
) -> Tuple[bool, Optional[str]]:
reasons: List[str] = []
if not coverage_polygon or len(coverage_polygon) < 3:
reasons.append("missing_footprint")
if not _valid_imaging_date(_text_or_none(meta.get("imaging_date"))):
reasons.append("missing_date")
for field_name, reason in (
("orbit_direction", "missing_orbit_direction"),
("imaging_mode", "missing_imaging_mode"),
("polarization", "missing_polarization"),
("satellite_family", "missing_satellite_family"),
):
if not _text_or_none(meta.get(field_name)):
reasons.append(reason)
geocoded_flag = _bool_or_none(meta.get("geocoded_flag"))
if geocoded_flag is True:
reasons.append("geocoded_product")
complex_tokens = {
_text_or_none(meta.get("image_data_type")),
_text_or_none(meta.get("product_type")),
_text_or_none(meta.get("source_product_token")),
_text_or_none(meta.get("product_variant")),
}
normalized_tokens = {str(token).strip().upper() for token in complex_tokens if token}
is_complex_source = bool(normalized_tokens.intersection({"COMPLEX", "SLC", "SSC"}))
if not is_complex_source:
reasons.append("not_complex_source")
if reasons:
return False, ";".join(reasons)
return True, None
def _iter_dirs(root: str, last_mtime: float):
stack = [root]
while stack:
@@ -586,20 +648,31 @@ class DataService:
continue
merged_meta[key] = value
satellite = merged_meta.get("satellite")
imaging_date = merged_meta.get("imaging_date")
imaging_mode = merged_meta.get("imaging_mode")
polarization = merged_meta.get("polarization")
orbit_direction = merged_meta.get("orbit_direction")
satellite_mode = merged_meta.get("satellite_mode")
receiving_station = merged_meta.get("receiving_station")
orbit_circle = merged_meta.get("orbit_circle")
satellite = _text_or_none(merged_meta.get("satellite"))
imaging_date = _text_or_none(merged_meta.get("imaging_date"))
imaging_mode = _text_or_none(merged_meta.get("imaging_mode"))
polarization = _text_or_none(merged_meta.get("polarization"))
orbit_direction = _text_or_none(merged_meta.get("orbit_direction"))
satellite_mode = _text_or_none(merged_meta.get("satellite_mode"))
receiving_station = _text_or_none(merged_meta.get("receiving_station"))
orbit_circle = _text_or_none(merged_meta.get("orbit_circle"))
scene_center_lon = merged_meta.get("scene_center_lon")
scene_center_lat = merged_meta.get("scene_center_lat")
acquisition_time_utc = merged_meta.get("acquisition_time_utc")
product_type = merged_meta.get("product_type")
product_level = merged_meta.get("product_level")
product_unique_id = merged_meta.get("product_unique_id")
acquisition_time_utc = _text_or_none(merged_meta.get("acquisition_time_utc"))
product_type = _text_or_none(merged_meta.get("product_type"))
source_product_token = _text_or_none(merged_meta.get("source_product_token"))
image_data_type = _text_or_none(merged_meta.get("image_data_type"))
image_data_format = _text_or_none(merged_meta.get("image_data_format"))
product_variant = _text_or_none(merged_meta.get("product_variant"))
product_level = _text_or_none(merged_meta.get("product_level"))
product_unique_id = _text_or_none(merged_meta.get("product_unique_id"))
satellite_family = normalize_satellite_family(
_text_or_none(merged_meta.get("satellite_family")) or satellite
)
look_direction = _text_or_none(merged_meta.get("look_direction"))
if look_direction:
look_direction = look_direction.upper()
geocoded_flag = _bool_or_none(merged_meta.get("geocoded_flag"))
if not satellite:
continue
@@ -635,9 +708,27 @@ class DataService:
scene_center_lon = scene_center_lon if scene_center_lon is not None else poly.centroid.x
scene_center_lat = scene_center_lat if scene_center_lat is not None else poly.centroid.y
readiness_meta = {
"satellite_family": satellite_family,
"imaging_date": imaging_date,
"imaging_mode": imaging_mode,
"orbit_direction": orbit_direction,
"polarization": polarization,
"product_type": product_type,
"source_product_token": source_product_token,
"image_data_type": image_data_type,
"product_variant": product_variant,
"geocoded_flag": geocoded_flag,
}
insar_source_ready, insar_source_reason = _build_insar_source_readiness(
readiness_meta,
coverage_polygon,
)
data_to_upsert = {
"unique_id": unique_id,
"satellite": satellite,
"satellite_family": satellite_family,
"imaging_date": imaging_date,
"imaging_mode": imaging_mode,
"orbit_direction": orbit_direction,
@@ -649,8 +740,16 @@ class DataService:
"scene_center_lat": scene_center_lat,
"acquisition_time_utc": acquisition_time_utc,
"product_type": product_type,
"source_product_token": source_product_token,
"image_data_type": image_data_type,
"image_data_format": image_data_format,
"product_variant": product_variant,
"product_level": product_level,
"product_unique_id": product_unique_id,
"look_direction": look_direction,
"geocoded_flag": geocoded_flag,
"insar_source_ready": insar_source_ready,
"insar_source_reason": insar_source_reason,
"file_path": radar_folder_path,
"has_orbit_data": has_orbit_data,
"orbit_file_path": orbit_file_path,
backend/app/services/envi_service.py
+1
View File
@@ -530,6 +530,7 @@ def _write_envi_run_sidecar(
"slave_polarization": pair_meta.get("slave_polarization"),
"time_baseline_days": pair_meta.get("time_baseline_days"),
"spatial_baseline_meters": pair_meta.get("spatial_baseline_meters"),
"scene_center_distance_meters": pair_meta.get("scene_center_distance_meters"),
"scene_pair_uid": pair_meta.get("scene_pair_uid") or pair_meta.get("pair_uid"),
"pair_uid": pair_meta.get("pair_uid") or pair_meta.get("scene_pair_uid"),
"network_run_id": pair_meta.get("network_run_id"),
backend/app/services/job_handlers.py
+38 -3
View File
@@ -19,7 +19,7 @@ from sqlalchemy import select
from .. import database
from ..config import settings
from ..models import SystemJobORM, DinsarResultORM, HazardPointORM, DinsarTaskItemORM, PsTaskItemORM, SARSceneGeoORM, FloodDetectionORM, WaterDetectionORM, GF3ProcessingORM, AiDiagnosisORM
from ..models import SystemJobORM, DinsarResultORM, HazardPointORM, DinsarTaskItemORM, PsTaskItemORM, RadarDataORM, SARSceneGeoORM, FloodDetectionORM, WaterDetectionORM, GF3ProcessingORM, AiDiagnosisORM
from ..scheduler import scan_data_job
from .data_service import data_service
from .dinsar_compat_service import dinsar_compat_service
@@ -287,6 +287,8 @@ async def _handle_copy_data(job: SystemJobORM) -> None:
dest_dir = payload.get("dest_dir")
batch_id = payload.get("batch_id")
copy_statuses = _normalize_copy_statuses(payload.get("copy_statuses"))
include_orbit_files = bool(payload.get("include_orbit_files"))
export_zip = bool(payload.get("export_zip"))
if not batch_id:
raise ValueError("COPY_DATA requires batch_id payload.")
@@ -318,6 +320,26 @@ async def _handle_copy_data(job: SystemJobORM) -> None:
.where(DinsarTaskItemORM.status.in_(copy_statuses))
.order_by(DinsarTaskItemORM.id.asc())
)
task_items = result.scalars().all()
orbit_by_path: Dict[str, Optional[str]] = {}
if include_orbit_files:
scene_paths = [
str(path)
for item in task_items
for path in (item.master_path, item.slave_path)
if path
]
if scene_paths:
scene_result = await db.execute(
select(RadarDataORM.file_path, RadarDataORM.orbit_file_path).where(
RadarDataORM.file_path.in_(scene_paths)
)
)
orbit_by_path = {
os.path.normcase(os.path.normpath(str(file_path))): orbit_path
for file_path, orbit_path in scene_result.all()
if file_path
}
items = [
{
"task_name": item.task_name,
@@ -335,6 +357,13 @@ async def _handle_copy_data(job: SystemJobORM) -> None:
"slave_polarization": item.slave_polarization,
"time_baseline_days": item.time_baseline_days,
"spatial_baseline_meters": item.spatial_baseline_meters,
"scene_center_distance_meters": item.scene_center_distance_meters,
"master_orbit_file_path": orbit_by_path.get(
os.path.normcase(os.path.normpath(str(item.master_path)))
) if include_orbit_files and item.master_path else None,
"slave_orbit_file_path": orbit_by_path.get(
os.path.normcase(os.path.normpath(str(item.slave_path)))
) if include_orbit_files and item.slave_path else None,
"scene_pair_uid": item.scene_pair_uid,
"pair_uid": item.scene_pair_uid,
"network_run_id": item.network_run_id,
@@ -342,13 +371,19 @@ async def _handle_copy_data(job: SystemJobORM) -> None:
"policy_version": item.policy_version,
"selection_strategy": item.selection_strategy,
}
for item in result.scalars().all()
for item in task_items
]
if not items:
raise ValueError(
f"No D-InSAR items matched copy statuses: {', '.join(copy_statuses)}"
)
await run_dinsar_copy_items(job.task_id, items, dest_dir)
await run_dinsar_copy_items(
job.task_id,
items,
dest_dir,
include_orbit_files=include_orbit_files,
export_zip=export_zip,
)
return
raise ValueError(f"Unknown COPY_DATA file_type: {file_type}")
backend/app/services/pairing_cache_service.py
+87 -3
View File
@@ -28,6 +28,38 @@ def _scene_uid_expr(alias: str) -> str:
)
def _satellite_family_expr(alias: str) -> str:
compact = (
f"upper(replace(replace(replace(COALESCE({alias}.satellite, ''), '-', ''), '_', ''), ' ', ''))"
)
return (
f"COALESCE(NULLIF({alias}.satellite_family, ''), "
f"CASE "
f"WHEN {compact} IN ('LT1', 'LT1A', 'LT1B', 'LUTAN1', 'LUTAN1A', 'LUTAN1B') THEN 'LT1' "
f"WHEN {compact} IN ('S1', 'S1A', 'S1B', 'SENTINEL1', 'SENTINEL1A', 'SENTINEL1B') THEN 'S1' "
f"WHEN NULLIF({alias}.satellite, '') IS NOT NULL THEN upper({alias}.satellite) "
f"ELSE NULL END)"
)
def _same_satellite_family_expr(left_alias: str, right_alias: str) -> str:
left_family = _satellite_family_expr(left_alias)
right_family = _satellite_family_expr(right_alias)
return (
f"(NULLIF({left_family}, '') IS NOT NULL "
f"AND NULLIF({right_family}, '') IS NOT NULL "
f"AND {left_family} = {right_family})"
)
def _same_look_direction_expr(left_alias: str, right_alias: str) -> str:
return (
f"(NULLIF({left_alias}.look_direction, '') IS NULL "
f"OR NULLIF({right_alias}.look_direction, '') IS NULL "
f"OR {left_alias}.look_direction = {right_alias}.look_direction)"
)
def _orientation_is_left_master_expr(left_alias: str, right_alias: str) -> str:
left_uid = _scene_uid_expr(left_alias)
right_uid = _scene_uid_expr(right_alias)
@@ -48,6 +80,9 @@ def _hard_constraints_expr(left_alias: str, right_alias: str) -> str:
f"AND {left_alias}.orbit_direction IS NOT NULL "
f"AND {right_alias}.orbit_direction IS NOT NULL "
f"AND {left_alias}.orbit_direction = {right_alias}.orbit_direction "
f"AND COALESCE({left_alias}.insar_source_ready, false) "
f"AND COALESCE({right_alias}.insar_source_ready, false) "
f"AND { _same_look_direction_expr(left_alias, right_alias) } "
f"AND ST_Intersects({left_alias}.geom, {right_alias}.geom)"
)
@@ -55,6 +90,9 @@ def _hard_constraints_expr(left_alias: str, right_alias: str) -> str:
def _full_rebuild_insert_sql() -> str:
master_uid = _scene_uid_expr("m")
slave_uid = _scene_uid_expr("s")
center_distance = "ST_DistanceSphere(ST_Centroid(m.geom), ST_Centroid(s.geom))::double precision"
master_family = _satellite_family_expr("m")
slave_family = _satellite_family_expr("s")
return f"""
INSERT INTO pairing_metric_cache (
master_scene_ref_id,
@@ -66,19 +104,26 @@ def _full_rebuild_insert_sql() -> str:
orientation_rule_version,
time_baseline_days,
spatial_baseline_meters,
scene_center_distance_meters,
scene_overlap_ratio,
orbit_direction,
same_satellite,
same_satellite_family,
same_look_direction,
same_imaging_mode,
same_polarization,
master_imaging_date,
slave_imaging_date,
master_satellite,
slave_satellite,
master_satellite_family,
slave_satellite_family,
master_imaging_mode,
slave_imaging_mode,
master_polarization,
slave_polarization,
master_look_direction,
slave_look_direction,
master_file_path,
slave_file_path,
status,
@@ -93,13 +138,16 @@ def _full_rebuild_insert_sql() -> str:
:metric_version AS metric_version,
:orientation_rule_version AS orientation_rule_version,
ABS(to_date(s.imaging_date, 'YYYYMMDD') - to_date(m.imaging_date, 'YYYYMMDD')) AS time_baseline_days,
ST_DistanceSphere(ST_Centroid(m.geom), ST_Centroid(s.geom))::double precision AS spatial_baseline_meters,
{center_distance} AS spatial_baseline_meters,
{center_distance} AS scene_center_distance_meters,
(
ST_Area(ST_Intersection(m.geom, s.geom)::geography) /
NULLIF(GREATEST(ST_Area(m.geom::geography), ST_Area(s.geom::geography)), 0)
)::double precision AS scene_overlap_ratio,
m.orbit_direction,
(m.satellite IS NOT NULL AND s.satellite IS NOT NULL AND m.satellite = s.satellite) AS same_satellite,
{ _same_satellite_family_expr('m', 's') } AS same_satellite_family,
{ _same_look_direction_expr('m', 's') } AS same_look_direction,
(
NULLIF(m.imaging_mode, '') IS NOT NULL
AND NULLIF(s.imaging_mode, '') IS NOT NULL
@@ -114,10 +162,14 @@ def _full_rebuild_insert_sql() -> str:
s.imaging_date AS slave_imaging_date,
m.satellite AS master_satellite,
s.satellite AS slave_satellite,
{master_family} AS master_satellite_family,
{slave_family} AS slave_satellite_family,
m.imaging_mode AS master_imaging_mode,
s.imaging_mode AS slave_imaging_mode,
m.polarization AS master_polarization,
s.polarization AS slave_polarization,
m.look_direction AS master_look_direction,
s.look_direction AS slave_look_direction,
m.file_path AS master_file_path,
s.file_path AS slave_file_path,
'READY' AS status,
@@ -133,6 +185,9 @@ def _incremental_insert_sql() -> str:
dirty_uid = _scene_uid_expr("d")
other_uid = _scene_uid_expr("o")
dirty_is_master = _orientation_is_left_master_expr("d", "o")
center_distance = "ST_DistanceSphere(ST_Centroid(d.geom), ST_Centroid(o.geom))::double precision"
dirty_family = _satellite_family_expr("d")
other_family = _satellite_family_expr("o")
return f"""
INSERT INTO pairing_metric_cache (
master_scene_ref_id,
@@ -144,19 +199,26 @@ def _incremental_insert_sql() -> str:
orientation_rule_version,
time_baseline_days,
spatial_baseline_meters,
scene_center_distance_meters,
scene_overlap_ratio,
orbit_direction,
same_satellite,
same_satellite_family,
same_look_direction,
same_imaging_mode,
same_polarization,
master_imaging_date,
slave_imaging_date,
master_satellite,
slave_satellite,
master_satellite_family,
slave_satellite_family,
master_imaging_mode,
slave_imaging_mode,
master_polarization,
slave_polarization,
master_look_direction,
slave_look_direction,
master_file_path,
slave_file_path,
status,
@@ -176,13 +238,16 @@ def _incremental_insert_sql() -> str:
:metric_version AS metric_version,
:orientation_rule_version AS orientation_rule_version,
ABS(to_date(o.imaging_date, 'YYYYMMDD') - to_date(d.imaging_date, 'YYYYMMDD')) AS time_baseline_days,
ST_DistanceSphere(ST_Centroid(d.geom), ST_Centroid(o.geom))::double precision AS spatial_baseline_meters,
{center_distance} AS spatial_baseline_meters,
{center_distance} AS scene_center_distance_meters,
(
ST_Area(ST_Intersection(d.geom, o.geom)::geography) /
NULLIF(GREATEST(ST_Area(d.geom::geography), ST_Area(o.geom::geography)), 0)
)::double precision AS scene_overlap_ratio,
d.orbit_direction,
(d.satellite IS NOT NULL AND o.satellite IS NOT NULL AND d.satellite = o.satellite) AS same_satellite,
{ _same_satellite_family_expr('d', 'o') } AS same_satellite_family,
{ _same_look_direction_expr('d', 'o') } AS same_look_direction,
(
NULLIF(d.imaging_mode, '') IS NOT NULL
AND NULLIF(o.imaging_mode, '') IS NOT NULL
@@ -197,10 +262,14 @@ def _incremental_insert_sql() -> str:
CASE WHEN {dirty_is_master} THEN o.imaging_date ELSE d.imaging_date END AS slave_imaging_date,
CASE WHEN {dirty_is_master} THEN d.satellite ELSE o.satellite END AS master_satellite,
CASE WHEN {dirty_is_master} THEN o.satellite ELSE d.satellite END AS slave_satellite,
CASE WHEN {dirty_is_master} THEN {dirty_family} ELSE {other_family} END AS master_satellite_family,
CASE WHEN {dirty_is_master} THEN {other_family} ELSE {dirty_family} END AS slave_satellite_family,
CASE WHEN {dirty_is_master} THEN d.imaging_mode ELSE o.imaging_mode END AS master_imaging_mode,
CASE WHEN {dirty_is_master} THEN o.imaging_mode ELSE d.imaging_mode END AS slave_imaging_mode,
CASE WHEN {dirty_is_master} THEN d.polarization ELSE o.polarization END AS master_polarization,
CASE WHEN {dirty_is_master} THEN o.polarization ELSE d.polarization END AS slave_polarization,
CASE WHEN {dirty_is_master} THEN d.look_direction ELSE o.look_direction END AS master_look_direction,
CASE WHEN {dirty_is_master} THEN o.look_direction ELSE d.look_direction END AS slave_look_direction,
CASE WHEN {dirty_is_master} THEN d.file_path ELSE o.file_path END AS master_file_path,
CASE WHEN {dirty_is_master} THEN o.file_path ELSE d.file_path END AS slave_file_path,
'READY' AS status,
@@ -228,7 +297,11 @@ class PairingCacheService:
return state
async def _count_pair_rows(self, db: AsyncSession) -> int:
result = await db.execute(select(func.count(PairingMetricCacheORM.id)))
result = await db.execute(
select(func.count(PairingMetricCacheORM.id)).where(
PairingMetricCacheORM.metric_version == pairing_state_service.metric_version
)
)
return int(result.scalar_one() or 0)
async def _count_scene_rows(self, db: AsyncSession) -> int:
@@ -382,6 +455,17 @@ class PairingCacheService:
pair_count = await self._count_pair_rows(db)
scene_count = await self._count_scene_rows(db)
if dirty_scene_count == 0 and self._should_full_rebuild(
dirty_scene_count=dirty_scene_count,
scene_count=scene_count,
pair_count=pair_count,
force_full=force_full,
):
result = await self.rebuild_metric_cache(db, commit=commit)
result["trigger_dirty_scene_count"] = dirty_scene_count
result["forced"] = force_full
return result
if dirty_scene_count == 0:
summary = await self._finalize_state_success(db, full_rebuild=False)
if commit:
backend/app/services/pairing_state_service.py
+12 -3
View File
@@ -18,7 +18,7 @@ from ..models import (
PAIRING_CACHE_SCOPE_GLOBAL = "global"
DEFAULT_PAIRING_METRIC_VERSION = "2026.04.v1"
DEFAULT_PAIRING_METRIC_VERSION = "2026.05.raw.v1"
PAIRING_ORIENTATION_RULE_VERSION = "date_then_scene_uid_v1"
@@ -66,7 +66,11 @@ class PairingStateService:
return int(result.scalar_one() or 0)
async def _count_metric_cache_rows(self, db: AsyncSession) -> int:
result = await db.execute(select(func.count(PairingMetricCacheORM.id)))
result = await db.execute(
select(func.count(PairingMetricCacheORM.id)).where(
PairingMetricCacheORM.metric_version == self.metric_version
)
)
return int(result.scalar_one() or 0)
async def _get_global_state(self, db: AsyncSession) -> Optional[PairingCacheStateORM]:
@@ -135,7 +139,12 @@ class PairingStateService:
await db.flush()
created = True
state.metric_version = state.metric_version or self.metric_version
if state.metric_version != self.metric_version:
state.metric_version = self.metric_version
state.status = "DIRTY"
state.last_error = None
else:
state.metric_version = state.metric_version or self.metric_version
state.scene_count = scene_count
state.pair_count = metric_cache_count
state.dirty_scene_count = dirty_scene_count
backend/app/services/result_catalog_service.py
+22
View File
@@ -149,6 +149,11 @@ def _build_pairing_trace_payload(
task_network_edge_id = getattr(task_item, "network_edge_id", None) if task_item is not None else None
task_policy_version = getattr(task_item, "policy_version", None) if task_item is not None else None
task_selection_strategy = getattr(task_item, "selection_strategy", None) if task_item is not None else None
task_scene_center_distance = (
getattr(task_item, "scene_center_distance_meters", None)
if task_item is not None
else None
)
candidate_network_edge_id = _coerce_optional_int(candidate_meta.get("network_edge_id"))
task_network_edge_id = _coerce_optional_int(task_network_edge_id)
@@ -176,6 +181,11 @@ def _build_pairing_trace_payload(
candidate_meta.get("selection_strategy"),
task_selection_strategy,
),
"scene_center_distance_meters": (
candidate_meta.get("scene_center_distance_meters")
if candidate_meta.get("scene_center_distance_meters") is not None
else task_scene_center_distance
),
}
return {
key: value
@@ -240,6 +250,7 @@ def _resolve_candidate_identity(candidate: Dict[str, Any]) -> Dict[str, Any]:
"slave_polarization",
"time_baseline_days",
"spatial_baseline_meters",
"scene_center_distance_meters",
):
resolved[field] = run_meta.get(field)
if resolved[field] in (None, ""):
@@ -608,8 +619,11 @@ class ResultCatalogService:
"slave_polarization": metric.slave_polarization,
"time_baseline_days": metric.time_baseline_days,
"spatial_baseline_meters": metric.spatial_baseline_meters,
"scene_center_distance_meters": metric.scene_center_distance_meters,
"scene_overlap_ratio": metric.scene_overlap_ratio,
"same_satellite": metric.same_satellite,
"same_satellite_family": metric.same_satellite_family,
"same_look_direction": metric.same_look_direction,
"same_imaging_mode": metric.same_imaging_mode,
"same_polarization": metric.same_polarization,
"status": metric.status,
@@ -721,6 +735,12 @@ class ResultCatalogService:
"orbit_direction": None,
"time_baseline_days": getattr(task_item, "time_baseline_days", None) or candidate_meta.get("time_baseline_days"),
"spatial_baseline_meters": getattr(task_item, "spatial_baseline_meters", None) or candidate_meta.get("spatial_baseline_meters"),
"scene_center_distance_meters": (
getattr(task_item, "scene_center_distance_meters", None)
or candidate_meta.get("scene_center_distance_meters")
or getattr(task_item, "spatial_baseline_meters", None)
or candidate_meta.get("spatial_baseline_meters")
),
"grid_size_m": profile_params.get("target_grid_size_m") or profile_params.get("geocoding_pixel_size_m"),
"radar_wavelength": profile_params.get("wavelength"),
"orbit_clip_margin": profile_params.get("orbit_margin_sec"),
@@ -1138,6 +1158,7 @@ class ResultCatalogService:
orbit_direction=profile_payload.get("orbit_direction"),
time_baseline_days=profile_payload.get("time_baseline_days"),
spatial_baseline_meters=profile_payload.get("spatial_baseline_meters"),
scene_center_distance_meters=profile_payload.get("scene_center_distance_meters"),
grid_size_m=profile_payload.get("grid_size_m"),
radar_wavelength=profile_payload.get("radar_wavelength"),
orbit_clip_margin=profile_payload.get("orbit_clip_margin"),
@@ -1532,6 +1553,7 @@ class ResultCatalogService:
"orbit_direction": profile.orbit_direction,
"time_baseline_days": profile.time_baseline_days,
"spatial_baseline_meters": profile.spatial_baseline_meters,
"scene_center_distance_meters": profile.scene_center_distance_meters,
"grid_size_m": profile.grid_size_m,
"radar_wavelength": profile.radar_wavelength,
"orbit_clip_margin": profile.orbit_clip_margin,
backend/app/services/spatial_service.py
+114 -21
View File
@@ -14,7 +14,7 @@ from typing import Any, Dict, List, Optional, Tuple
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from sqlalchemy import and_, cast, func
from sqlalchemy import and_, cast, func, or_
from sqlalchemy.orm import aliased
from geoalchemy2 import Geography
@@ -43,7 +43,7 @@ from .dinsar_naming import build_pair_key, build_task_alias, ensure_unique_task_
from .pairing_state_service import pairing_state_service
PAIRING_POLICY_VERSION = "2026.04.phase3.v1"
PAIRING_POLICY_VERSION = "2026.05.raw-source.v1"
PAIRING_WARNING_CANDIDATE_THRESHOLD = 3000
logger = logging.getLogger(__name__)
@@ -155,6 +155,10 @@ class SpatialService:
) -> List[dict]:
master_alias = aliased(RadarDataORM)
slave_alias = aliased(RadarDataORM)
center_distance_expr = func.coalesce(
PairingMetricCacheORM.scene_center_distance_meters,
PairingMetricCacheORM.spatial_baseline_meters,
)
stmt = (
select(PairingMetricCacheORM, master_alias, slave_alias)
@@ -165,8 +169,9 @@ class SpatialService:
PairingMetricCacheORM.status == "READY",
PairingMetricCacheORM.time_baseline_days >= params.time_baseline_min,
PairingMetricCacheORM.time_baseline_days <= params.time_baseline_max,
PairingMetricCacheORM.spatial_baseline_meters <= params.spatial_baseline_max_meters,
center_distance_expr <= params.spatial_baseline_max_meters,
PairingMetricCacheORM.scene_overlap_ratio >= params.overlap_threshold,
PairingMetricCacheORM.same_look_direction.is_(True),
)
)
@@ -177,7 +182,7 @@ class SpatialService:
)
if not params.cross_satellite_pairing:
stmt = stmt.where(PairingMetricCacheORM.same_satellite.is_(True))
stmt = stmt.where(PairingMetricCacheORM.same_satellite_family.is_(True))
if params.require_same_imaging_mode:
stmt = stmt.where(PairingMetricCacheORM.same_imaging_mode.is_(True))
@@ -186,9 +191,20 @@ class SpatialService:
stmt = stmt.where(PairingMetricCacheORM.same_polarization.is_(True))
if params.allowed_satellites:
allowed_satellites = [
str(item).strip().upper()
for item in params.allowed_satellites
if str(item).strip()
]
stmt = stmt.where(
master_alias.satellite.in_(params.allowed_satellites),
slave_alias.satellite.in_(params.allowed_satellites),
or_(
func.upper(master_alias.satellite).in_(allowed_satellites),
func.upper(master_alias.satellite_family).in_(allowed_satellites),
),
or_(
func.upper(slave_alias.satellite).in_(allowed_satellites),
func.upper(slave_alias.satellite_family).in_(allowed_satellites),
),
)
if params.master_date_from:
@@ -220,12 +236,18 @@ class SpatialService:
PairingMetricCacheORM.master_imaging_date.asc(),
PairingMetricCacheORM.slave_imaging_date.asc(),
func.coalesce(PairingMetricCacheORM.scene_overlap_ratio, 0).desc(),
center_distance_expr.asc(),
PairingMetricCacheORM.pair_uid.asc(),
)
result = await db.execute(stmt)
candidate_pool: List[dict] = []
for metric_row, master_row, slave_row in result.all():
center_distance = float(
metric_row.scene_center_distance_meters
if metric_row.scene_center_distance_meters is not None
else (metric_row.spatial_baseline_meters or 0)
)
candidate_pool.append(
{
"metric_cache_ref_id": int(metric_row.id),
@@ -235,7 +257,8 @@ class SpatialService:
"master": RadarData.model_validate(master_row),
"slave": RadarData.model_validate(slave_row),
"days": int(metric_row.time_baseline_days or 0),
"dist": float(metric_row.spatial_baseline_meters or 0),
"dist": center_distance,
"scene_center_distance_meters": center_distance,
"overlap_ratio": float(metric_row.scene_overlap_ratio or 0),
}
)
@@ -270,6 +293,11 @@ class SpatialService:
selection_reason=candidate.get("selection_reason"),
time_baseline_days=int(candidate["days"]),
spatial_baseline_meters=float(candidate["dist"]),
scene_center_distance_meters=float(
candidate.get("scene_center_distance_meters")
if candidate.get("scene_center_distance_meters") is not None
else candidate.get("dist") or 0
),
)
)
return result_pairs
@@ -356,6 +384,12 @@ class SpatialService:
"pair_uid": candidate.get("pair_uid"),
"time_baseline_days": int(candidate.get("days") or 0),
"spatial_baseline_meters": float(candidate.get("dist") or 0.0),
"scene_center_distance_meters": float(
candidate.get("scene_center_distance_meters")
if candidate.get("scene_center_distance_meters") is not None
else candidate.get("dist") or 0.0
),
"legacy_spatial_baseline_field": "scene_center_distance_meters",
"scene_overlap_ratio": float(candidate.get("overlap_ratio") or 0.0),
}
@@ -420,6 +454,10 @@ class SpatialService:
master_alias = aliased(RadarDataORM)
slave_alias = aliased(RadarDataORM)
center_distance_expr = func.coalesce(
PairingMetricCacheORM.scene_center_distance_meters,
PairingMetricCacheORM.spatial_baseline_meters,
)
stmt = (
select(PairingMetricCacheORM, master_alias, slave_alias)
.join(master_alias, master_alias.id == PairingMetricCacheORM.master_scene_ref_id)
@@ -431,14 +469,15 @@ class SpatialService:
PairingMetricCacheORM.slave_scene_ref_id.in_(scene_ids),
PairingMetricCacheORM.time_baseline_days >= params.time_baseline_min,
PairingMetricCacheORM.time_baseline_days <= params.time_baseline_max,
PairingMetricCacheORM.spatial_baseline_meters <= params.spatial_baseline_max_meters,
center_distance_expr <= params.spatial_baseline_max_meters,
PairingMetricCacheORM.scene_overlap_ratio >= params.network_overlap_threshold,
PairingMetricCacheORM.same_look_direction.is_(True),
)
.order_by(
PairingMetricCacheORM.master_imaging_date.asc(),
PairingMetricCacheORM.slave_imaging_date.asc(),
PairingMetricCacheORM.time_baseline_days.asc(),
PairingMetricCacheORM.spatial_baseline_meters.asc(),
center_distance_expr.asc(),
func.coalesce(PairingMetricCacheORM.scene_overlap_ratio, 0).desc(),
PairingMetricCacheORM.id.asc(),
)
@@ -447,6 +486,11 @@ class SpatialService:
candidate_pool: List[dict] = []
for metric_row, master_row, slave_row in result.all():
center_distance = float(
metric_row.scene_center_distance_meters
if metric_row.scene_center_distance_meters is not None
else (metric_row.spatial_baseline_meters or 0.0)
)
candidate_pool.append(
{
"metric_cache_ref_id": int(metric_row.id),
@@ -456,7 +500,8 @@ class SpatialService:
"master": RadarData.model_validate(master_row),
"slave": RadarData.model_validate(slave_row),
"days": int(metric_row.time_baseline_days or 0),
"dist": float(metric_row.spatial_baseline_meters or 0.0),
"dist": center_distance,
"scene_center_distance_meters": center_distance,
"overlap_ratio": float(metric_row.scene_overlap_ratio or 0.0),
}
)
@@ -511,6 +556,11 @@ class SpatialService:
"slave_imaging_date": slave.imaging_date,
"temporal_baseline_days": int(candidate.get("days") or 0),
"spatial_baseline_meters": float(candidate.get("dist") or 0.0),
"scene_center_distance_meters": float(
candidate.get("scene_center_distance_meters")
if candidate.get("scene_center_distance_meters") is not None
else candidate.get("dist") or 0.0
),
"scene_overlap_ratio": float(candidate.get("overlap_ratio") or 0.0),
"selection_reason": candidate.get("selection_reason"),
"selection_score": (
@@ -523,6 +573,11 @@ class SpatialService:
"selection_mode": selection_mode,
"pair_uid": candidate.get("pair_uid"),
"metric_version": pairing_state_service.metric_version,
"scene_center_distance_meters": float(
candidate.get("scene_center_distance_meters")
if candidate.get("scene_center_distance_meters") is not None
else candidate.get("dist") or 0.0
),
"time_baseline_min": params.time_baseline_min,
"time_baseline_max": params.time_baseline_max,
"spatial_baseline_max_meters": params.spatial_baseline_max_meters,
@@ -703,18 +758,44 @@ class SpatialService:
if params.strategy == "sbas":
return self._apply_sbas_strategy(candidate_pool, params, aoi_wkt=aoi_wkt)
if params.strategy == "sequential":
return self._apply_sequential_strategy(candidate_pool, params.num_connections)
return self._apply_sequential_strategy(candidate_pool, params.num_connections, params)
if params.strategy == "star":
return self._apply_star_strategy(candidate_pool, params.reference_image_id)
return self._apply_all_strategy(candidate_pool)
return self._apply_star_strategy(candidate_pool, params.reference_image_id, params)
return self._apply_all_strategy(candidate_pool, params)
def _apply_all_strategy(self, candidate_pool: List[dict]) -> Tuple[List[dict], List[str]]:
def _score_pair_candidate(self, candidate: dict, params: PairingRequest) -> float:
max_time = max(float(params.time_baseline_max or 1), 1.0)
max_center = max(float(params.spatial_baseline_max_meters or 1), 1.0)
time_score = 1.0 - min(float(candidate.get("days") or 0) / max_time, 1.0)
center_score = 1.0 - min(float(candidate.get("dist") or 0) / max_center, 1.0)
overlap_score = min(max(float(candidate.get("overlap_ratio") or 0), 0.0), 1.0)
source_score = 1.0 if (
bool(getattr(candidate.get("master"), "insar_source_ready", False))
and bool(getattr(candidate.get("slave"), "insar_source_ready", False))
) else 0.0
orbit_score = 1.0 if (
bool(getattr(candidate.get("master"), "has_orbit_data", False))
and bool(getattr(candidate.get("slave"), "has_orbit_data", False))
) else 0.0
return (
0.25 * time_score
+ 0.20 * center_score
+ 0.35 * overlap_score
+ 0.15 * source_score
+ 0.05 * orbit_score
)
def _apply_all_strategy(
self,
candidate_pool: List[dict],
params: PairingRequest,
) -> Tuple[List[dict], List[str]]:
return (
[
{
**candidate,
"selection_reason": "all_candidate",
"selection_score": float(candidate.get("overlap_ratio") or 0),
"selection_score": self._score_pair_candidate(candidate, params),
}
for candidate in self._sorted_candidates(candidate_pool)
],
@@ -725,6 +806,7 @@ class SpatialService:
self,
candidate_pool: List[dict],
num_connections: int,
params: PairingRequest,
) -> Tuple[List[dict], List[str]]:
"""
Sequential: 按稳定时间序列排序，每景连接后续 N 景。
@@ -759,7 +841,7 @@ class SpatialService:
{
**candidate,
"selection_reason": "sequential_neighbor",
"selection_score": float(candidate.get("overlap_ratio") or 0),
"selection_score": self._score_pair_candidate(candidate, params),
}
)
picked_count += 1
@@ -770,6 +852,7 @@ class SpatialService:
self,
candidate_pool: List[dict],
reference_image_id: Optional[int],
params: PairingRequest,
) -> Tuple[List[dict], List[str]]:
"""
Star: 参考像固定为 master。
@@ -820,7 +903,7 @@ class SpatialService:
{
**candidate,
"selection_reason": "star_reference_master",
"selection_score": float(candidate.get("overlap_ratio") or 0),
"selection_score": self._score_pair_candidate(candidate, params),
"is_reference_edge": True,
"reference_image_id": int(reference_image_id),
}
@@ -1136,6 +1219,14 @@ class SpatialService:
time_score = 1.0 - min(float(candidate.get("days") or 0) / max_time, 1.0)
spatial_score = 1.0 - min(float(candidate.get("dist") or 0) / max_space, 1.0)
overlap_score = min(max(float(candidate.get("overlap_ratio") or 0), 0.0), 1.0)
source_score = 1.0 if (
bool(getattr(candidate.get("master"), "insar_source_ready", False))
and bool(getattr(candidate.get("slave"), "insar_source_ready", False))
) else 0.0
orbit_score = 1.0 if (
bool(getattr(candidate.get("master"), "has_orbit_data", False))
and bool(getattr(candidate.get("slave"), "has_orbit_data", False))
) else 0.0
aoi_gain = 0.0
redundancy_penalty = 0.0
@@ -1153,10 +1244,12 @@ class SpatialService:
redundancy_penalty = max(0.0, min(overlap_area / candidate_area, 1.0))
return (
0.35 * time_score
+ 0.20 * spatial_score
0.30 * time_score
+ 0.15 * spatial_score
+ 0.30 * overlap_score
+ 0.15 * aoi_gain
+ 0.10 * aoi_gain
+ 0.10 * source_score
+ 0.05 * orbit_score
- float(params.coverage_diversity_penalty or 0.0) * redundancy_penalty
)
@@ -1588,7 +1681,7 @@ class SpatialService:
slave: RadarDataORM
) -> float:
"""
Calculate spatial baseline in meters using PostGIS sphere distance.
Calculate footprint center distance in meters using PostGIS sphere distance.
"""
master_alias = RadarDataORM.__table__.alias("master")
slave_alias = RadarDataORM.__table__.alias("slave")
backend/app/utils.py
+38 -3
View File
@@ -33,8 +33,15 @@ def _radar_meta_base() -> Dict[str, Any]:
"scene_center_lat": None,
"acquisition_time_utc": None,
"product_type": None,
"source_product_token": None,
"image_data_type": None,
"image_data_format": None,
"product_variant": None,
"product_level": None,
"product_unique_id": None,
"satellite_family": None,
"look_direction": None,
"geocoded_flag": None,
}
@@ -69,6 +76,20 @@ def _parse_coord_token(value: Optional[str]) -> Optional[float]:
return None
def normalize_satellite_family(value: Optional[str]) -> Optional[str]:
raw = str(value or "").strip().upper()
if not raw:
return None
compact = raw.replace("-", "").replace("_", "").replace(" ", "")
if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}:
return "LT1"
if compact in {"S1", "S1A", "S1B", "SENTINEL1", "SENTINEL1A", "SENTINEL1B"}:
return "S1"
if compact in {"GF3", "GAOFEN3"}:
return "GF3"
return raw
def parse_s1_radar_filename(folder_name: str) -> Optional[Dict[str, Any]]:
"""
Parses key info from a Sentinel-1 radar data folder name.
@@ -81,8 +102,11 @@ def parse_s1_radar_filename(folder_name: str) -> Optional[Dict[str, Any]]:
meta = _radar_meta_base()
meta["satellite"] = parts[0]
meta["satellite_family"] = normalize_satellite_family(parts[0])
meta["imaging_date"] = parts[4].split('T')[0]
meta["imaging_mode"] = parts[1]
meta["source_product_token"] = parts[2]
meta["product_type"] = parts[2]
polarization = parts[3] # e.g., '1SDV' -> 'DV' is dual-pol VV/VH
meta["polarization"] = polarization[2:] if len(polarization) > 2 else polarization
return meta
@@ -121,6 +145,7 @@ def parse_lt1_radar_filename(folder_name: str) -> Optional[Dict[str, Any]]:
meta = _radar_meta_base()
meta["satellite"] = parts[0]
meta["satellite_family"] = normalize_satellite_family(parts[0])
if len(parts) > 1:
meta["satellite_mode"] = parts[1]
if len(parts) > 2:
@@ -137,6 +162,7 @@ def parse_lt1_radar_filename(folder_name: str) -> Optional[Dict[str, Any]]:
meta["imaging_date"] = _extract_date_yyyymmdd(parts[7])
meta["acquisition_time_utc"] = parts[7]
if len(parts) > 8:
meta["source_product_token"] = parts[8]
meta["product_type"] = parts[8]
if len(parts) > 9:
meta["polarization"] = parts[9]
@@ -182,6 +208,7 @@ def parse_gf3_l2_dirname(folder_name: str) -> Optional[Dict[str, Any]]:
meta = _radar_meta_base()
meta["satellite"] = "GF3"
meta["satellite_family"] = normalize_satellite_family("GF3")
parts = name.split("_")
# Try to extract date: first 8-digit segment
@@ -437,14 +464,19 @@ def parse_xml_metadata(
])
# --- Product Type / Level / Unique ID ---
product_type = _get_first_text([
image_data_type = _get_first_text([
f".//{ns_prefix}imageDataInfo/{ns_prefix}imageDataType",
f".//{ns_prefix}orderInfo/{ns_prefix}productVariant",
f".//{ns_prefix}imageDataInfo/{ns_prefix}imageDataFormat",
"//*[local-name()='imageDataInfo']/*[local-name()='imageDataType']",
])
product_variant = _get_first_text([
f".//{ns_prefix}orderInfo/{ns_prefix}productVariant",
"//*[local-name()='orderInfo']/*[local-name()='productVariant']",
])
image_data_format = _get_first_text([
f".//{ns_prefix}imageDataInfo/{ns_prefix}imageDataFormat",
"//*[local-name()='imageDataInfo']/*[local-name()='imageDataFormat']",
])
product_type = image_data_type or product_variant or image_data_format
product_level = _get_first_text([
f".//{ns_prefix}generalHeader/{ns_prefix}itemName",
"//*[local-name()='generalHeader']/*[local-name()='itemName']",
@@ -527,6 +559,9 @@ def parse_xml_metadata(
"scene_center_lat": scene_center_lat,
"acquisition_time_utc": acquisition_time_utc,
"product_type": product_type,
"image_data_type": image_data_type,
"image_data_format": image_data_format,
"product_variant": product_variant,
"product_level": product_level,
"product_unique_id": product_unique_id,
"look_direction": look_direction,