diff --git a/backend/app/copier.py b/backend/app/copier.py index 8a92a7a..225eb1d 100644 --- a/backend/app/copier.py +++ b/backend/app/copier.py @@ -1,6 +1,8 @@ import os import shutil import asyncio +import tempfile +import zipfile from datetime import datetime from typing import List, Tuple, Optional, Dict, Any @@ -34,14 +36,128 @@ async def _log_and_update(task_id: str, message: str, progress: Optional[int] = def find_dinsar_source_to_copy(path: str) -> str: """ Find D-InSAR source path. - Prefer the envi_import subfolder when present and non-empty. + D-InSAR pairing/distribution works on the raw source product directory. """ - envi_path = os.path.join(path, "envi_import") - if os.path.isdir(envi_path) and os.listdir(envi_path): - return envi_path return path +def _resolve_orbit_dest_path( + orbit_dir: str, + role: str, + source_path: str, + used_dest_paths: Dict[str, str], +) -> str: + base_name = os.path.basename(source_path) + dest_path = os.path.join(orbit_dir, base_name) + dest_key = os.path.normcase(os.path.abspath(dest_path)) + source_key = os.path.normcase(os.path.abspath(source_path)) + if dest_key not in used_dest_paths or used_dest_paths[dest_key] == source_key: + return dest_path + + role_path = os.path.join(orbit_dir, f"{role}_{base_name}") + role_key = os.path.normcase(os.path.abspath(role_path)) + if role_key not in used_dest_paths or used_dest_paths[role_key] == source_key: + return role_path + + stem, ext = os.path.splitext(base_name) + counter = 2 + while True: + numbered_path = os.path.join(orbit_dir, f"{role}_{stem}_{counter}{ext}") + numbered_key = os.path.normcase(os.path.abspath(numbered_path)) + if numbered_key not in used_dest_paths: + return numbered_path + counter += 1 + + +async def _copy_dinsar_orbit_files( + task_id: str, + item: Dict[str, Any], + task_dir: str, + include_orbit_files: bool, +) -> List[Dict[str, Any]]: + if not include_orbit_files: + return [] + + orbit_dir = os.path.join(task_dir, "orbit") + copied_by_source: Dict[str, str] = {} + used_dest_paths: Dict[str, str] = {} + orbit_entries: List[Dict[str, Any]] = [] + for role, key in ( + ("master", "master_orbit_file_path"), + ("slave", "slave_orbit_file_path"), + ): + raw_path = item.get(key) + if not raw_path: + await _log_and_update(task_id, f" -> {role} orbit missing in catalog metadata") + orbit_entries.append( + { + "role": role, + "source_path": None, + "copied": False, + "reason": "missing_orbit_path", + } + ) + continue + + source_path = os.path.normpath(os.path.abspath(str(raw_path))) + if not os.path.isfile(source_path): + await _log_and_update(task_id, f" -> {role} orbit file not found: {source_path}") + orbit_entries.append( + { + "role": role, + "source_path": source_path, + "copied": False, + "reason": "source_file_not_found", + } + ) + continue + + await asyncio.to_thread(os.makedirs, orbit_dir, exist_ok=True) + source_key = os.path.normcase(source_path) + if source_key in copied_by_source: + dest_path = copied_by_source[source_key] + else: + dest_path = _resolve_orbit_dest_path(orbit_dir, role, source_path, used_dest_paths) + await asyncio.to_thread(shutil.copy2, source_path, dest_path) + copied_by_source[source_key] = dest_path + used_dest_paths[os.path.normcase(os.path.abspath(dest_path))] = source_key + + orbit_entries.append( + { + "role": role, + "source_path": source_path, + "relative_path": os.path.relpath(dest_path, start=task_dir), + "copied": True, + } + ) + return orbit_entries + + +def _zip_task_directory(task_dir: str, zip_path: str) -> None: + parent_dir = os.path.dirname(task_dir) + temp_zip_path = f"{zip_path}.tmp" + try: + if os.path.exists(temp_zip_path): + os.remove(temp_zip_path) + with zipfile.ZipFile(temp_zip_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for root, dirs, files in os.walk(task_dir): + rel_root = os.path.relpath(root, start=parent_dir) + for dirname in dirs: + arcname = os.path.join(rel_root, dirname).replace(os.sep, "/") + "/" + archive.writestr(arcname, "") + for filename in files: + file_path = os.path.join(root, filename) + arcname = os.path.join(rel_root, filename).replace(os.sep, "/") + archive.write(file_path, arcname) + os.replace(temp_zip_path, zip_path) + finally: + if os.path.exists(temp_zip_path): + try: + os.remove(temp_zip_path) + except OSError: + pass + + async def run_ps_copy_items(task_id: str, items: List[Dict[str, Any]], dest_dir: str) -> None: try: await task_service.start_task(task_id, message="Starting PS-InSAR copy task...") @@ -124,10 +240,25 @@ async def run_ps_copy_items(task_id: str, items: List[Dict[str, Any]], dest_dir: raise CopyTaskExecutionError(fail_msg) from e -async def run_dinsar_copy_items(task_id: str, items: List[Dict[str, Any]], dest_dir: str) -> None: +async def run_dinsar_copy_items( + task_id: str, + items: List[Dict[str, Any]], + dest_dir: str, + *, + include_orbit_files: bool = False, + export_zip: bool = False, +) -> None: try: await task_service.start_task(task_id, message="Starting D-InSAR copy task...") await _log_and_update(task_id, f"D-InSAR copy started. Dest: {dest_dir}") + await _log_and_update( + task_id, + ( + "D-InSAR copy options: " + f"include_orbit_files={include_orbit_files}, " + f"export_zip={export_zip}" + ), + ) if not os.path.exists(dest_dir): try: @@ -175,11 +306,22 @@ async def run_dinsar_copy_items(task_id: str, items: List[Dict[str, Any]], dest_ await _log_and_update(task_id, f"[{i}/{total}] Processing: {task_name}") - task_dir = os.path.join(dest_dir, task_alias) - master_dir = os.path.join(task_dir, "master") - slave_dir = os.path.join(task_dir, "slave") - + staging_root: Optional[str] = None try: + if export_zip: + staging_root = await asyncio.to_thread( + tempfile.mkdtemp, + prefix="._dinsar_zip_", + dir=dest_dir, + ) + task_dir = os.path.join(staging_root, task_alias) + zip_path = os.path.join(dest_dir, f"{task_alias}.zip") + else: + task_dir = os.path.join(dest_dir, task_alias) + zip_path = None + master_dir = os.path.join(task_dir, "master") + slave_dir = os.path.join(task_dir, "slave") + master_src_path = find_dinsar_source_to_copy(master_path) if not os.path.exists(master_src_path): await _log_and_update(task_id, f" -> Missing master: {master_src_path}") @@ -194,6 +336,12 @@ async def run_dinsar_copy_items(task_id: str, items: List[Dict[str, Any]], dest_ await asyncio.to_thread(shutil.copytree, master_src_path, master_dir, dirs_exist_ok=True) await asyncio.to_thread(shutil.copytree, slave_src_path, slave_dir, dirs_exist_ok=True) + orbit_entries = await _copy_dinsar_orbit_files( + task_id, + item, + task_dir, + include_orbit_files, + ) await asyncio.to_thread( write_pair_metadata, task_dir, @@ -213,6 +361,12 @@ async def run_dinsar_copy_items(task_id: str, items: List[Dict[str, Any]], dest_ "slave_polarization": item.get("slave_polarization"), "time_baseline_days": item.get("time_baseline_days"), "spatial_baseline_meters": item.get("spatial_baseline_meters"), + "scene_center_distance_meters": item.get("scene_center_distance_meters"), + "package_format": "zip" if export_zip else "folder", + "include_orbit_files": bool(include_orbit_files), + "master_orbit_file_path": item.get("master_orbit_file_path"), + "slave_orbit_file_path": item.get("slave_orbit_file_path"), + "orbit_files": orbit_entries, "scene_pair_uid": item.get("scene_pair_uid") or item.get("pair_uid"), "pair_uid": item.get("pair_uid") or item.get("scene_pair_uid"), "network_run_id": item.get("network_run_id"), @@ -222,6 +376,9 @@ async def run_dinsar_copy_items(task_id: str, items: List[Dict[str, Any]], dest_ "copied_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", }, ) + if export_zip and zip_path: + await asyncio.to_thread(_zip_task_directory, task_dir, zip_path) + await _log_and_update(task_id, f" -> ZIP: {zip_path}") await _log_and_update(task_id, " -> Success") success_count += 1 @@ -231,8 +388,14 @@ async def run_dinsar_copy_items(task_id: str, items: List[Dict[str, Any]], dest_ except Exception as e: await _log_and_update(task_id, f" -> Failed: {e}") failed_count += 1 + finally: + if staging_root: + await asyncio.to_thread(shutil.rmtree, staging_root, ignore_errors=True) - final_msg = f"D-InSAR copy finished. Success {success_count}, Failed {failed_count}" + final_msg = ( + f"D-InSAR copy finished. Mode {'zip' if export_zip else 'folder'}. " + f"Success {success_count}, Failed {failed_count}" + ) await _log_and_update(task_id, final_msg, progress=100) if failed_count > 0: await task_service.update_task(task_id, status="FAILED", message=final_msg, progress=100) diff --git a/backend/app/db_maintenance.py b/backend/app/db_maintenance.py index 1096973..4647b75 100644 --- a/backend/app/db_maintenance.py +++ b/backend/app/db_maintenance.py @@ -35,6 +35,7 @@ MIGRATION_FILES = [ "006_result_pairing_trace.sql", "007_timeseries_stack_plan_trace.sql", "008_timeseries_stack_plan_edges.sql", + "009_raw_source_pairing_fields.sql", ] diff --git a/backend/app/dinsar_engines/isce2_engine.py b/backend/app/dinsar_engines/isce2_engine.py index 845d778..831b733 100644 --- a/backend/app/dinsar_engines/isce2_engine.py +++ b/backend/app/dinsar_engines/isce2_engine.py @@ -1334,6 +1334,7 @@ class Isce2Engine(DinsarEngine): "slave_polarization": pair_meta.get("slave_polarization"), "time_baseline_days": pair_meta.get("time_baseline_days"), "spatial_baseline_meters": pair_meta.get("spatial_baseline_meters"), + "scene_center_distance_meters": pair_meta.get("scene_center_distance_meters"), "scene_pair_uid": pair_meta.get("scene_pair_uid") or pair_meta.get("pair_uid"), "pair_uid": pair_meta.get("pair_uid") or pair_meta.get("scene_pair_uid"), "network_run_id": pair_meta.get("network_run_id"), diff --git a/backend/app/dinsar_engines/pyint_engine.py b/backend/app/dinsar_engines/pyint_engine.py index 5aea18d..5b03d06 100644 --- a/backend/app/dinsar_engines/pyint_engine.py +++ b/backend/app/dinsar_engines/pyint_engine.py @@ -1290,6 +1290,7 @@ class PyintEngine(DinsarEngine): "slave_polarization": pair_meta.get("slave_polarization"), "time_baseline_days": pair_meta.get("time_baseline_days") or time_baseline_days, "spatial_baseline_meters": pair_meta.get("spatial_baseline_meters"), + "scene_center_distance_meters": pair_meta.get("scene_center_distance_meters"), "scene_pair_uid": pair_meta.get("scene_pair_uid") or pair_meta.get("pair_uid"), "pair_uid": pair_meta.get("pair_uid") or pair_meta.get("scene_pair_uid"), "network_run_id": pair_meta.get("network_run_id"), diff --git a/backend/app/models/orm.py b/backend/app/models/orm.py index 6c8923e..7906c7f 100644 --- a/backend/app/models/orm.py +++ b/backend/app/models/orm.py @@ -33,8 +33,17 @@ class RadarDataORM(Base): scene_center_lat = Column(Float, nullable=True) acquisition_time_utc = Column(String, nullable=True) product_type = Column(String, nullable=True) + source_product_token = Column(String, nullable=True) + image_data_type = Column(String, nullable=True) + image_data_format = Column(String, nullable=True) + product_variant = Column(String, nullable=True) product_level = Column(String, nullable=True) product_unique_id = Column(String, nullable=True) + satellite_family = Column(String, index=True, nullable=True) + look_direction = Column(String, index=True, nullable=True) + geocoded_flag = Column(Boolean, nullable=True) + insar_source_ready = Column(Boolean, nullable=False, default=False, server_default="false") + insar_source_reason = Column(Text, nullable=True) file_path = Column(String, unique=True) has_orbit_data = Column(Boolean) orbit_file_path = Column(String, nullable=True) @@ -179,6 +188,7 @@ class DinsarProductProfileORM(Base): orbit_direction = Column(String, index=True, nullable=True) time_baseline_days = Column(Integer, index=True, nullable=True) spatial_baseline_meters = Column(Float, index=True, nullable=True) + scene_center_distance_meters = Column(Float, index=True, nullable=True) grid_size_m = Column(Float, nullable=True) radar_wavelength = Column(Float, nullable=True) @@ -287,7 +297,7 @@ class PairingCacheStateORM(Base): id = Column(Integer, primary_key=True, autoincrement=True) cache_scope = Column(String(32), unique=True, index=True, nullable=False, default="global") - metric_version = Column(String(32), nullable=False, default="2026.04.v1") + metric_version = Column(String(32), nullable=False, default="2026.05.raw.v1") status = Column(String(16), index=True, nullable=False, default="DIRTY") scene_count = Column(Integer, nullable=False, default=0) pair_count = Column(Integer, nullable=False, default=0) @@ -338,23 +348,30 @@ class PairingMetricCacheORM(Base): master_scene_uid = Column(String, index=True, nullable=False) slave_scene_uid = Column(String, index=True, nullable=False) pair_uid = Column(String, index=True, nullable=False) - metric_version = Column(String(32), index=True, nullable=False, default="2026.04.v1") + metric_version = Column(String(32), index=True, nullable=False, default="2026.05.raw.v1") orientation_rule_version = Column(String(32), nullable=False, default="date_then_scene_uid_v1") time_baseline_days = Column(Integer, index=True, nullable=True) spatial_baseline_meters = Column(Float, index=True, nullable=True) + scene_center_distance_meters = Column(Float, index=True, nullable=True) scene_overlap_ratio = Column(Float, index=True, nullable=True) orbit_direction = Column(String, index=True, nullable=True) same_satellite = Column(Boolean, nullable=False, default=True) + same_satellite_family = Column(Boolean, nullable=False, default=True, server_default="true") + same_look_direction = Column(Boolean, nullable=False, default=True, server_default="true") same_imaging_mode = Column(Boolean, nullable=False, default=True) same_polarization = Column(Boolean, nullable=False, default=True) master_imaging_date = Column(String(8), index=True, nullable=True) slave_imaging_date = Column(String(8), index=True, nullable=True) master_satellite = Column(String, index=True, nullable=True) slave_satellite = Column(String, index=True, nullable=True) + master_satellite_family = Column(String, index=True, nullable=True) + slave_satellite_family = Column(String, index=True, nullable=True) master_imaging_mode = Column(String, nullable=True) slave_imaging_mode = Column(String, nullable=True) master_polarization = Column(String, nullable=True) slave_polarization = Column(String, nullable=True) + master_look_direction = Column(String, nullable=True) + slave_look_direction = Column(String, nullable=True) master_file_path = Column(String, nullable=True) slave_file_path = Column(String, nullable=True) status = Column(String(16), index=True, nullable=False, default="READY") @@ -927,6 +944,7 @@ class DinsarTaskItemORM(Base): slave_polarization = Column(String, nullable=True) time_baseline_days = Column(Integer, nullable=True) spatial_baseline_meters = Column(Float, nullable=True) + scene_center_distance_meters = Column(Float, nullable=True) status = Column(String, index=True, nullable=False, default="PENDING") remark = Column(Text, nullable=True) diff --git a/backend/app/models/schemas.py b/backend/app/models/schemas.py index a5bc8ff..b547853 100644 --- a/backend/app/models/schemas.py +++ b/backend/app/models/schemas.py @@ -5,7 +5,7 @@ Pydantic Schema 定义。 from datetime import datetime from typing import Any, Dict, List, Optional, Tuple, Union -from pydantic import BaseModel, ConfigDict, Field, computed_field, field_validator +from pydantic import BaseModel, ConfigDict, Field, computed_field, field_validator, model_validator from ..config import read_int_env @@ -178,8 +178,17 @@ class RadarData(BaseModel): scene_center_lat: Optional[float] = None acquisition_time_utc: Optional[str] = None product_type: Optional[str] = None + source_product_token: Optional[str] = None + image_data_type: Optional[str] = None + image_data_format: Optional[str] = None + product_variant: Optional[str] = None product_level: Optional[str] = None product_unique_id: Optional[str] = None + satellite_family: Optional[str] = None + look_direction: Optional[str] = None + geocoded_flag: Optional[bool] = None + insar_source_ready: bool = False + insar_source_reason: Optional[str] = None file_path: str has_orbit_data: bool orbit_file_path: Optional[str] = None @@ -242,6 +251,9 @@ class PairingRequest(BaseModel): require_same_imaging_mode: bool = True require_same_polarization: bool = True aoi_overlap_threshold: Optional[float] = Field(default=None, ge=0.0, le=1.0) + max_temporal_baseline_days: Optional[int] = Field(default=None, ge=1, le=3650) + pair_footprint_overlap_min_ratio: Optional[float] = Field(default=None, ge=0.0, le=1.0) + footprint_center_distance_max_meters: Optional[int] = Field(default=None, ge=0, le=100000) # === 双池日期（新增） === master_date_from: Optional[str] = Field(default=None, pattern=r'^\d{8}$|^$') @@ -261,6 +273,20 @@ class PairingRequest(BaseModel): # === 向后兼容（保留） === start_date: Optional[str] = Field(default=None, pattern=r'^\d{8}$|^$') + @model_validator(mode='before') + @classmethod + def _apply_aliases(cls, data): + if not isinstance(data, dict): + return data + normalized = dict(data) + if normalized.get('max_temporal_baseline_days') not in (None, ''): + normalized['time_baseline_max'] = normalized['max_temporal_baseline_days'] + if normalized.get('pair_footprint_overlap_min_ratio') not in (None, ''): + normalized['overlap_threshold'] = normalized['pair_footprint_overlap_min_ratio'] + if normalized.get('footprint_center_distance_max_meters') not in (None, ''): + normalized['spatial_baseline_max_meters'] = normalized['footprint_center_distance_max_meters'] + return normalized + @field_validator( 'master_date_from', 'master_date_to', @@ -336,6 +362,7 @@ class RadarPair(BaseModel): selection_reason: Optional[str] = None time_baseline_days: int spatial_baseline_meters: float + scene_center_distance_meters: Optional[float] = None class PairingResponse(BaseModel): @@ -413,6 +440,7 @@ class TimeseriesStackPlanEdge(BaseModel): slave_imaging_date: Optional[str] = None temporal_baseline_days: Optional[int] = None spatial_baseline_meters: Optional[float] = None + scene_center_distance_meters: Optional[float] = None perpendicular_baseline_meters: Optional[float] = None scene_overlap_ratio: Optional[float] = None pair_aoi_overlap_ratio: Optional[float] = None @@ -518,6 +546,7 @@ class DinsarTaskItem(BaseModel): slave_polarization: Optional[str] = None time_baseline_days: Optional[int] = None spatial_baseline_meters: Optional[float] = None + scene_center_distance_meters: Optional[float] = None status: str remark: Optional[str] = None created_at: datetime diff --git a/backend/app/routers/pairing.py b/backend/app/routers/pairing.py index d5028b6..bff6d41 100644 --- a/backend/app/routers/pairing.py +++ b/backend/app/routers/pairing.py @@ -42,6 +42,9 @@ def get_pairing_request_from_form( time_baseline_max: int = Form(90), overlap_threshold: float = Form(0.5), spatial_baseline_max_meters: int = Form(3000), + max_temporal_baseline_days: Optional[int] = Form(None), + pair_footprint_overlap_min_ratio: Optional[float] = Form(None), + footprint_center_distance_max_meters: Optional[int] = Form(None), coverage_diversity_penalty: float = Form(0.3), require_same_imaging_mode: bool = Form(True), require_same_polarization: bool = Form(True), @@ -72,6 +75,9 @@ def get_pairing_request_from_form( time_baseline_max=time_baseline_max, overlap_threshold=overlap_threshold, spatial_baseline_max_meters=spatial_baseline_max_meters, + max_temporal_baseline_days=max_temporal_baseline_days, + pair_footprint_overlap_min_ratio=pair_footprint_overlap_min_ratio, + footprint_center_distance_max_meters=footprint_center_distance_max_meters, coverage_diversity_penalty=coverage_diversity_penalty, require_same_imaging_mode=require_same_imaging_mode, require_same_polarization=require_same_polarization, diff --git a/backend/app/routers/task_batches.py b/backend/app/routers/task_batches.py index e7bd462..58d3213 100644 --- a/backend/app/routers/task_batches.py +++ b/backend/app/routers/task_batches.py @@ -266,6 +266,11 @@ async def create_dinsar_batch_endpoint( slave_polarization=slave.polarization, time_baseline_days=pair.time_baseline_days, spatial_baseline_meters=pair.spatial_baseline_meters, + scene_center_distance_meters=( + pair.scene_center_distance_meters + if pair.scene_center_distance_meters is not None + else pair.spatial_baseline_meters + ), status="PENDING", ) db.add(item) diff --git a/backend/app/routers/tools.py b/backend/app/routers/tools.py index 630e134..d33672b 100644 --- a/backend/app/routers/tools.py +++ b/backend/app/routers/tools.py @@ -33,6 +33,8 @@ class CopyBatchRequest(BaseModel): batch_id: str = Field(max_length=COPY_BATCH_TEXT_MAX_LENGTH) dest_dir: str = Field(max_length=COPY_BATCH_TEXT_MAX_LENGTH) copy_statuses: Optional[List[str]] = None + include_orbit_files: bool = False + export_zip: bool = False @field_validator("batch_id", "dest_dir", mode="before") @classmethod @@ -138,6 +140,8 @@ async def copy_dinsar_pairs_endpoint( "file_type": "DINSAR_PAIRS", "batch_id": request.batch_id, "copy_statuses": copy_statuses, + "include_orbit_files": bool(request.include_orbit_files), + "export_zip": bool(request.export_zip), } task_id = await task_service.create_task("COPY_DATA", f"D-InSAR 数据分发: {request.dest_dir}", params=params) @@ -146,6 +150,8 @@ async def copy_dinsar_pairs_endpoint( "dest_dir": request.dest_dir, "batch_id": request.batch_id, "copy_statuses": copy_statuses, + "include_orbit_files": bool(request.include_orbit_files), + "export_zip": bool(request.export_zip), } await job_queue_service.create_job("COPY_DATA", payload=payload, task_id=task_id) await _add_operation_audit_log( @@ -158,6 +164,8 @@ async def copy_dinsar_pairs_endpoint( "batch_id": request.batch_id, "dest_dir": request.dest_dir, "copy_statuses": copy_statuses, + "include_orbit_files": bool(request.include_orbit_files), + "export_zip": bool(request.export_zip), }, ) await db.commit() diff --git a/backend/app/services/data_service.py b/backend/app/services/data_service.py index 4b3c4c0..b754ecd 100644 --- a/backend/app/services/data_service.py +++ b/backend/app/services/data_service.py @@ -33,7 +33,13 @@ from ..config import settings from ..models import ( RadarDataORM, RadarData, DinsarResultORM, HazardPointORM, HazardPoint, ScanStateORM ) -from ..utils import get_parser, RADAR_PARSERS, find_xml_file, parse_xml_metadata +from ..utils import ( + get_parser, + RADAR_PARSERS, + find_xml_file, + normalize_satellite_family, + parse_xml_metadata, +) from .task_service import task_service from .image_service import image_service from .orbit_converter import get_source_orbit_inventory, sync_orbit_pools @@ -50,7 +56,7 @@ def _safe_mtime(path: str) -> float: _DATE_RE = re.compile(r"^\d{8}$") -def _valid_imaging_date(value: str) -> bool: +def _valid_imaging_date(value: Optional[str]) -> bool: return bool(value and _DATE_RE.match(value)) @@ -63,6 +69,62 @@ def _extract_date_from_text(value: Optional[str]) -> Optional[str]: return None +def _text_or_none(value: Any) -> Optional[str]: + if value is None: + return None + text_value = str(value).strip() + return text_value or None + + +def _bool_or_none(value: Any) -> Optional[bool]: + if value is None or isinstance(value, bool): + return value + text_value = str(value).strip().lower() + if text_value in {"1", "true", "yes", "y"}: + return True + if text_value in {"0", "false", "no", "n"}: + return False + return None + + +def _build_insar_source_readiness( + meta: Dict[str, Any], + coverage_polygon: Optional[List[Tuple[float, float]]], +) -> Tuple[bool, Optional[str]]: + reasons: List[str] = [] + if not coverage_polygon or len(coverage_polygon) < 3: + reasons.append("missing_footprint") + if not _valid_imaging_date(_text_or_none(meta.get("imaging_date"))): + reasons.append("missing_date") + for field_name, reason in ( + ("orbit_direction", "missing_orbit_direction"), + ("imaging_mode", "missing_imaging_mode"), + ("polarization", "missing_polarization"), + ("satellite_family", "missing_satellite_family"), + ): + if not _text_or_none(meta.get(field_name)): + reasons.append(reason) + + geocoded_flag = _bool_or_none(meta.get("geocoded_flag")) + if geocoded_flag is True: + reasons.append("geocoded_product") + + complex_tokens = { + _text_or_none(meta.get("image_data_type")), + _text_or_none(meta.get("product_type")), + _text_or_none(meta.get("source_product_token")), + _text_or_none(meta.get("product_variant")), + } + normalized_tokens = {str(token).strip().upper() for token in complex_tokens if token} + is_complex_source = bool(normalized_tokens.intersection({"COMPLEX", "SLC", "SSC"})) + if not is_complex_source: + reasons.append("not_complex_source") + + if reasons: + return False, ";".join(reasons) + return True, None + + def _iter_dirs(root: str, last_mtime: float): stack = [root] while stack: @@ -586,20 +648,31 @@ class DataService: continue merged_meta[key] = value - satellite = merged_meta.get("satellite") - imaging_date = merged_meta.get("imaging_date") - imaging_mode = merged_meta.get("imaging_mode") - polarization = merged_meta.get("polarization") - orbit_direction = merged_meta.get("orbit_direction") - satellite_mode = merged_meta.get("satellite_mode") - receiving_station = merged_meta.get("receiving_station") - orbit_circle = merged_meta.get("orbit_circle") + satellite = _text_or_none(merged_meta.get("satellite")) + imaging_date = _text_or_none(merged_meta.get("imaging_date")) + imaging_mode = _text_or_none(merged_meta.get("imaging_mode")) + polarization = _text_or_none(merged_meta.get("polarization")) + orbit_direction = _text_or_none(merged_meta.get("orbit_direction")) + satellite_mode = _text_or_none(merged_meta.get("satellite_mode")) + receiving_station = _text_or_none(merged_meta.get("receiving_station")) + orbit_circle = _text_or_none(merged_meta.get("orbit_circle")) scene_center_lon = merged_meta.get("scene_center_lon") scene_center_lat = merged_meta.get("scene_center_lat") - acquisition_time_utc = merged_meta.get("acquisition_time_utc") - product_type = merged_meta.get("product_type") - product_level = merged_meta.get("product_level") - product_unique_id = merged_meta.get("product_unique_id") + acquisition_time_utc = _text_or_none(merged_meta.get("acquisition_time_utc")) + product_type = _text_or_none(merged_meta.get("product_type")) + source_product_token = _text_or_none(merged_meta.get("source_product_token")) + image_data_type = _text_or_none(merged_meta.get("image_data_type")) + image_data_format = _text_or_none(merged_meta.get("image_data_format")) + product_variant = _text_or_none(merged_meta.get("product_variant")) + product_level = _text_or_none(merged_meta.get("product_level")) + product_unique_id = _text_or_none(merged_meta.get("product_unique_id")) + satellite_family = normalize_satellite_family( + _text_or_none(merged_meta.get("satellite_family")) or satellite + ) + look_direction = _text_or_none(merged_meta.get("look_direction")) + if look_direction: + look_direction = look_direction.upper() + geocoded_flag = _bool_or_none(merged_meta.get("geocoded_flag")) if not satellite: continue @@ -635,9 +708,27 @@ class DataService: scene_center_lon = scene_center_lon if scene_center_lon is not None else poly.centroid.x scene_center_lat = scene_center_lat if scene_center_lat is not None else poly.centroid.y + readiness_meta = { + "satellite_family": satellite_family, + "imaging_date": imaging_date, + "imaging_mode": imaging_mode, + "orbit_direction": orbit_direction, + "polarization": polarization, + "product_type": product_type, + "source_product_token": source_product_token, + "image_data_type": image_data_type, + "product_variant": product_variant, + "geocoded_flag": geocoded_flag, + } + insar_source_ready, insar_source_reason = _build_insar_source_readiness( + readiness_meta, + coverage_polygon, + ) + data_to_upsert = { "unique_id": unique_id, "satellite": satellite, + "satellite_family": satellite_family, "imaging_date": imaging_date, "imaging_mode": imaging_mode, "orbit_direction": orbit_direction, @@ -649,8 +740,16 @@ class DataService: "scene_center_lat": scene_center_lat, "acquisition_time_utc": acquisition_time_utc, "product_type": product_type, + "source_product_token": source_product_token, + "image_data_type": image_data_type, + "image_data_format": image_data_format, + "product_variant": product_variant, "product_level": product_level, "product_unique_id": product_unique_id, + "look_direction": look_direction, + "geocoded_flag": geocoded_flag, + "insar_source_ready": insar_source_ready, + "insar_source_reason": insar_source_reason, "file_path": radar_folder_path, "has_orbit_data": has_orbit_data, "orbit_file_path": orbit_file_path, diff --git a/backend/app/services/envi_service.py b/backend/app/services/envi_service.py index 89dd456..d7383ab 100644 --- a/backend/app/services/envi_service.py +++ b/backend/app/services/envi_service.py @@ -530,6 +530,7 @@ def _write_envi_run_sidecar( "slave_polarization": pair_meta.get("slave_polarization"), "time_baseline_days": pair_meta.get("time_baseline_days"), "spatial_baseline_meters": pair_meta.get("spatial_baseline_meters"), + "scene_center_distance_meters": pair_meta.get("scene_center_distance_meters"), "scene_pair_uid": pair_meta.get("scene_pair_uid") or pair_meta.get("pair_uid"), "pair_uid": pair_meta.get("pair_uid") or pair_meta.get("scene_pair_uid"), "network_run_id": pair_meta.get("network_run_id"), diff --git a/backend/app/services/job_handlers.py b/backend/app/services/job_handlers.py index 348338e..1f7e450 100644 --- a/backend/app/services/job_handlers.py +++ b/backend/app/services/job_handlers.py @@ -19,7 +19,7 @@ from sqlalchemy import select from .. import database from ..config import settings -from ..models import SystemJobORM, DinsarResultORM, HazardPointORM, DinsarTaskItemORM, PsTaskItemORM, SARSceneGeoORM, FloodDetectionORM, WaterDetectionORM, GF3ProcessingORM, AiDiagnosisORM +from ..models import SystemJobORM, DinsarResultORM, HazardPointORM, DinsarTaskItemORM, PsTaskItemORM, RadarDataORM, SARSceneGeoORM, FloodDetectionORM, WaterDetectionORM, GF3ProcessingORM, AiDiagnosisORM from ..scheduler import scan_data_job from .data_service import data_service from .dinsar_compat_service import dinsar_compat_service @@ -287,6 +287,8 @@ async def _handle_copy_data(job: SystemJobORM) -> None: dest_dir = payload.get("dest_dir") batch_id = payload.get("batch_id") copy_statuses = _normalize_copy_statuses(payload.get("copy_statuses")) + include_orbit_files = bool(payload.get("include_orbit_files")) + export_zip = bool(payload.get("export_zip")) if not batch_id: raise ValueError("COPY_DATA requires batch_id payload.") @@ -318,6 +320,26 @@ async def _handle_copy_data(job: SystemJobORM) -> None: .where(DinsarTaskItemORM.status.in_(copy_statuses)) .order_by(DinsarTaskItemORM.id.asc()) ) + task_items = result.scalars().all() + orbit_by_path: Dict[str, Optional[str]] = {} + if include_orbit_files: + scene_paths = [ + str(path) + for item in task_items + for path in (item.master_path, item.slave_path) + if path + ] + if scene_paths: + scene_result = await db.execute( + select(RadarDataORM.file_path, RadarDataORM.orbit_file_path).where( + RadarDataORM.file_path.in_(scene_paths) + ) + ) + orbit_by_path = { + os.path.normcase(os.path.normpath(str(file_path))): orbit_path + for file_path, orbit_path in scene_result.all() + if file_path + } items = [ { "task_name": item.task_name, @@ -335,6 +357,13 @@ async def _handle_copy_data(job: SystemJobORM) -> None: "slave_polarization": item.slave_polarization, "time_baseline_days": item.time_baseline_days, "spatial_baseline_meters": item.spatial_baseline_meters, + "scene_center_distance_meters": item.scene_center_distance_meters, + "master_orbit_file_path": orbit_by_path.get( + os.path.normcase(os.path.normpath(str(item.master_path))) + ) if include_orbit_files and item.master_path else None, + "slave_orbit_file_path": orbit_by_path.get( + os.path.normcase(os.path.normpath(str(item.slave_path))) + ) if include_orbit_files and item.slave_path else None, "scene_pair_uid": item.scene_pair_uid, "pair_uid": item.scene_pair_uid, "network_run_id": item.network_run_id, @@ -342,13 +371,19 @@ async def _handle_copy_data(job: SystemJobORM) -> None: "policy_version": item.policy_version, "selection_strategy": item.selection_strategy, } - for item in result.scalars().all() + for item in task_items ] if not items: raise ValueError( f"No D-InSAR items matched copy statuses: {', '.join(copy_statuses)}" ) - await run_dinsar_copy_items(job.task_id, items, dest_dir) + await run_dinsar_copy_items( + job.task_id, + items, + dest_dir, + include_orbit_files=include_orbit_files, + export_zip=export_zip, + ) return raise ValueError(f"Unknown COPY_DATA file_type: {file_type}") diff --git a/backend/app/services/pairing_cache_service.py b/backend/app/services/pairing_cache_service.py index d175b6a..bd6c3d3 100644 --- a/backend/app/services/pairing_cache_service.py +++ b/backend/app/services/pairing_cache_service.py @@ -28,6 +28,38 @@ def _scene_uid_expr(alias: str) -> str: ) +def _satellite_family_expr(alias: str) -> str: + compact = ( + f"upper(replace(replace(replace(COALESCE({alias}.satellite, ''), '-', ''), '_', ''), ' ', ''))" + ) + return ( + f"COALESCE(NULLIF({alias}.satellite_family, ''), " + f"CASE " + f"WHEN {compact} IN ('LT1', 'LT1A', 'LT1B', 'LUTAN1', 'LUTAN1A', 'LUTAN1B') THEN 'LT1' " + f"WHEN {compact} IN ('S1', 'S1A', 'S1B', 'SENTINEL1', 'SENTINEL1A', 'SENTINEL1B') THEN 'S1' " + f"WHEN NULLIF({alias}.satellite, '') IS NOT NULL THEN upper({alias}.satellite) " + f"ELSE NULL END)" + ) + + +def _same_satellite_family_expr(left_alias: str, right_alias: str) -> str: + left_family = _satellite_family_expr(left_alias) + right_family = _satellite_family_expr(right_alias) + return ( + f"(NULLIF({left_family}, '') IS NOT NULL " + f"AND NULLIF({right_family}, '') IS NOT NULL " + f"AND {left_family} = {right_family})" + ) + + +def _same_look_direction_expr(left_alias: str, right_alias: str) -> str: + return ( + f"(NULLIF({left_alias}.look_direction, '') IS NULL " + f"OR NULLIF({right_alias}.look_direction, '') IS NULL " + f"OR {left_alias}.look_direction = {right_alias}.look_direction)" + ) + + def _orientation_is_left_master_expr(left_alias: str, right_alias: str) -> str: left_uid = _scene_uid_expr(left_alias) right_uid = _scene_uid_expr(right_alias) @@ -48,6 +80,9 @@ def _hard_constraints_expr(left_alias: str, right_alias: str) -> str: f"AND {left_alias}.orbit_direction IS NOT NULL " f"AND {right_alias}.orbit_direction IS NOT NULL " f"AND {left_alias}.orbit_direction = {right_alias}.orbit_direction " + f"AND COALESCE({left_alias}.insar_source_ready, false) " + f"AND COALESCE({right_alias}.insar_source_ready, false) " + f"AND { _same_look_direction_expr(left_alias, right_alias) } " f"AND ST_Intersects({left_alias}.geom, {right_alias}.geom)" ) @@ -55,6 +90,9 @@ def _hard_constraints_expr(left_alias: str, right_alias: str) -> str: def _full_rebuild_insert_sql() -> str: master_uid = _scene_uid_expr("m") slave_uid = _scene_uid_expr("s") + center_distance = "ST_DistanceSphere(ST_Centroid(m.geom), ST_Centroid(s.geom))::double precision" + master_family = _satellite_family_expr("m") + slave_family = _satellite_family_expr("s") return f""" INSERT INTO pairing_metric_cache ( master_scene_ref_id, @@ -66,19 +104,26 @@ def _full_rebuild_insert_sql() -> str: orientation_rule_version, time_baseline_days, spatial_baseline_meters, + scene_center_distance_meters, scene_overlap_ratio, orbit_direction, same_satellite, + same_satellite_family, + same_look_direction, same_imaging_mode, same_polarization, master_imaging_date, slave_imaging_date, master_satellite, slave_satellite, + master_satellite_family, + slave_satellite_family, master_imaging_mode, slave_imaging_mode, master_polarization, slave_polarization, + master_look_direction, + slave_look_direction, master_file_path, slave_file_path, status, @@ -93,13 +138,16 @@ def _full_rebuild_insert_sql() -> str: :metric_version AS metric_version, :orientation_rule_version AS orientation_rule_version, ABS(to_date(s.imaging_date, 'YYYYMMDD') - to_date(m.imaging_date, 'YYYYMMDD')) AS time_baseline_days, - ST_DistanceSphere(ST_Centroid(m.geom), ST_Centroid(s.geom))::double precision AS spatial_baseline_meters, + {center_distance} AS spatial_baseline_meters, + {center_distance} AS scene_center_distance_meters, ( ST_Area(ST_Intersection(m.geom, s.geom)::geography) / NULLIF(GREATEST(ST_Area(m.geom::geography), ST_Area(s.geom::geography)), 0) )::double precision AS scene_overlap_ratio, m.orbit_direction, (m.satellite IS NOT NULL AND s.satellite IS NOT NULL AND m.satellite = s.satellite) AS same_satellite, + { _same_satellite_family_expr('m', 's') } AS same_satellite_family, + { _same_look_direction_expr('m', 's') } AS same_look_direction, ( NULLIF(m.imaging_mode, '') IS NOT NULL AND NULLIF(s.imaging_mode, '') IS NOT NULL @@ -114,10 +162,14 @@ def _full_rebuild_insert_sql() -> str: s.imaging_date AS slave_imaging_date, m.satellite AS master_satellite, s.satellite AS slave_satellite, + {master_family} AS master_satellite_family, + {slave_family} AS slave_satellite_family, m.imaging_mode AS master_imaging_mode, s.imaging_mode AS slave_imaging_mode, m.polarization AS master_polarization, s.polarization AS slave_polarization, + m.look_direction AS master_look_direction, + s.look_direction AS slave_look_direction, m.file_path AS master_file_path, s.file_path AS slave_file_path, 'READY' AS status, @@ -133,6 +185,9 @@ def _incremental_insert_sql() -> str: dirty_uid = _scene_uid_expr("d") other_uid = _scene_uid_expr("o") dirty_is_master = _orientation_is_left_master_expr("d", "o") + center_distance = "ST_DistanceSphere(ST_Centroid(d.geom), ST_Centroid(o.geom))::double precision" + dirty_family = _satellite_family_expr("d") + other_family = _satellite_family_expr("o") return f""" INSERT INTO pairing_metric_cache ( master_scene_ref_id, @@ -144,19 +199,26 @@ def _incremental_insert_sql() -> str: orientation_rule_version, time_baseline_days, spatial_baseline_meters, + scene_center_distance_meters, scene_overlap_ratio, orbit_direction, same_satellite, + same_satellite_family, + same_look_direction, same_imaging_mode, same_polarization, master_imaging_date, slave_imaging_date, master_satellite, slave_satellite, + master_satellite_family, + slave_satellite_family, master_imaging_mode, slave_imaging_mode, master_polarization, slave_polarization, + master_look_direction, + slave_look_direction, master_file_path, slave_file_path, status, @@ -176,13 +238,16 @@ def _incremental_insert_sql() -> str: :metric_version AS metric_version, :orientation_rule_version AS orientation_rule_version, ABS(to_date(o.imaging_date, 'YYYYMMDD') - to_date(d.imaging_date, 'YYYYMMDD')) AS time_baseline_days, - ST_DistanceSphere(ST_Centroid(d.geom), ST_Centroid(o.geom))::double precision AS spatial_baseline_meters, + {center_distance} AS spatial_baseline_meters, + {center_distance} AS scene_center_distance_meters, ( ST_Area(ST_Intersection(d.geom, o.geom)::geography) / NULLIF(GREATEST(ST_Area(d.geom::geography), ST_Area(o.geom::geography)), 0) )::double precision AS scene_overlap_ratio, d.orbit_direction, (d.satellite IS NOT NULL AND o.satellite IS NOT NULL AND d.satellite = o.satellite) AS same_satellite, + { _same_satellite_family_expr('d', 'o') } AS same_satellite_family, + { _same_look_direction_expr('d', 'o') } AS same_look_direction, ( NULLIF(d.imaging_mode, '') IS NOT NULL AND NULLIF(o.imaging_mode, '') IS NOT NULL @@ -197,10 +262,14 @@ def _incremental_insert_sql() -> str: CASE WHEN {dirty_is_master} THEN o.imaging_date ELSE d.imaging_date END AS slave_imaging_date, CASE WHEN {dirty_is_master} THEN d.satellite ELSE o.satellite END AS master_satellite, CASE WHEN {dirty_is_master} THEN o.satellite ELSE d.satellite END AS slave_satellite, + CASE WHEN {dirty_is_master} THEN {dirty_family} ELSE {other_family} END AS master_satellite_family, + CASE WHEN {dirty_is_master} THEN {other_family} ELSE {dirty_family} END AS slave_satellite_family, CASE WHEN {dirty_is_master} THEN d.imaging_mode ELSE o.imaging_mode END AS master_imaging_mode, CASE WHEN {dirty_is_master} THEN o.imaging_mode ELSE d.imaging_mode END AS slave_imaging_mode, CASE WHEN {dirty_is_master} THEN d.polarization ELSE o.polarization END AS master_polarization, CASE WHEN {dirty_is_master} THEN o.polarization ELSE d.polarization END AS slave_polarization, + CASE WHEN {dirty_is_master} THEN d.look_direction ELSE o.look_direction END AS master_look_direction, + CASE WHEN {dirty_is_master} THEN o.look_direction ELSE d.look_direction END AS slave_look_direction, CASE WHEN {dirty_is_master} THEN d.file_path ELSE o.file_path END AS master_file_path, CASE WHEN {dirty_is_master} THEN o.file_path ELSE d.file_path END AS slave_file_path, 'READY' AS status, @@ -228,7 +297,11 @@ class PairingCacheService: return state async def _count_pair_rows(self, db: AsyncSession) -> int: - result = await db.execute(select(func.count(PairingMetricCacheORM.id))) + result = await db.execute( + select(func.count(PairingMetricCacheORM.id)).where( + PairingMetricCacheORM.metric_version == pairing_state_service.metric_version + ) + ) return int(result.scalar_one() or 0) async def _count_scene_rows(self, db: AsyncSession) -> int: @@ -382,6 +455,17 @@ class PairingCacheService: pair_count = await self._count_pair_rows(db) scene_count = await self._count_scene_rows(db) + if dirty_scene_count == 0 and self._should_full_rebuild( + dirty_scene_count=dirty_scene_count, + scene_count=scene_count, + pair_count=pair_count, + force_full=force_full, + ): + result = await self.rebuild_metric_cache(db, commit=commit) + result["trigger_dirty_scene_count"] = dirty_scene_count + result["forced"] = force_full + return result + if dirty_scene_count == 0: summary = await self._finalize_state_success(db, full_rebuild=False) if commit: diff --git a/backend/app/services/pairing_state_service.py b/backend/app/services/pairing_state_service.py index d7e5b7f..7b3cafc 100644 --- a/backend/app/services/pairing_state_service.py +++ b/backend/app/services/pairing_state_service.py @@ -18,7 +18,7 @@ from ..models import ( PAIRING_CACHE_SCOPE_GLOBAL = "global" -DEFAULT_PAIRING_METRIC_VERSION = "2026.04.v1" +DEFAULT_PAIRING_METRIC_VERSION = "2026.05.raw.v1" PAIRING_ORIENTATION_RULE_VERSION = "date_then_scene_uid_v1" @@ -66,7 +66,11 @@ class PairingStateService: return int(result.scalar_one() or 0) async def _count_metric_cache_rows(self, db: AsyncSession) -> int: - result = await db.execute(select(func.count(PairingMetricCacheORM.id))) + result = await db.execute( + select(func.count(PairingMetricCacheORM.id)).where( + PairingMetricCacheORM.metric_version == self.metric_version + ) + ) return int(result.scalar_one() or 0) async def _get_global_state(self, db: AsyncSession) -> Optional[PairingCacheStateORM]: @@ -135,7 +139,12 @@ class PairingStateService: await db.flush() created = True - state.metric_version = state.metric_version or self.metric_version + if state.metric_version != self.metric_version: + state.metric_version = self.metric_version + state.status = "DIRTY" + state.last_error = None + else: + state.metric_version = state.metric_version or self.metric_version state.scene_count = scene_count state.pair_count = metric_cache_count state.dirty_scene_count = dirty_scene_count diff --git a/backend/app/services/result_catalog_service.py b/backend/app/services/result_catalog_service.py index 35958ec..6f90c83 100644 --- a/backend/app/services/result_catalog_service.py +++ b/backend/app/services/result_catalog_service.py @@ -149,6 +149,11 @@ def _build_pairing_trace_payload( task_network_edge_id = getattr(task_item, "network_edge_id", None) if task_item is not None else None task_policy_version = getattr(task_item, "policy_version", None) if task_item is not None else None task_selection_strategy = getattr(task_item, "selection_strategy", None) if task_item is not None else None + task_scene_center_distance = ( + getattr(task_item, "scene_center_distance_meters", None) + if task_item is not None + else None + ) candidate_network_edge_id = _coerce_optional_int(candidate_meta.get("network_edge_id")) task_network_edge_id = _coerce_optional_int(task_network_edge_id) @@ -176,6 +181,11 @@ def _build_pairing_trace_payload( candidate_meta.get("selection_strategy"), task_selection_strategy, ), + "scene_center_distance_meters": ( + candidate_meta.get("scene_center_distance_meters") + if candidate_meta.get("scene_center_distance_meters") is not None + else task_scene_center_distance + ), } return { key: value @@ -240,6 +250,7 @@ def _resolve_candidate_identity(candidate: Dict[str, Any]) -> Dict[str, Any]: "slave_polarization", "time_baseline_days", "spatial_baseline_meters", + "scene_center_distance_meters", ): resolved[field] = run_meta.get(field) if resolved[field] in (None, ""): @@ -608,8 +619,11 @@ class ResultCatalogService: "slave_polarization": metric.slave_polarization, "time_baseline_days": metric.time_baseline_days, "spatial_baseline_meters": metric.spatial_baseline_meters, + "scene_center_distance_meters": metric.scene_center_distance_meters, "scene_overlap_ratio": metric.scene_overlap_ratio, "same_satellite": metric.same_satellite, + "same_satellite_family": metric.same_satellite_family, + "same_look_direction": metric.same_look_direction, "same_imaging_mode": metric.same_imaging_mode, "same_polarization": metric.same_polarization, "status": metric.status, @@ -721,6 +735,12 @@ class ResultCatalogService: "orbit_direction": None, "time_baseline_days": getattr(task_item, "time_baseline_days", None) or candidate_meta.get("time_baseline_days"), "spatial_baseline_meters": getattr(task_item, "spatial_baseline_meters", None) or candidate_meta.get("spatial_baseline_meters"), + "scene_center_distance_meters": ( + getattr(task_item, "scene_center_distance_meters", None) + or candidate_meta.get("scene_center_distance_meters") + or getattr(task_item, "spatial_baseline_meters", None) + or candidate_meta.get("spatial_baseline_meters") + ), "grid_size_m": profile_params.get("target_grid_size_m") or profile_params.get("geocoding_pixel_size_m"), "radar_wavelength": profile_params.get("wavelength"), "orbit_clip_margin": profile_params.get("orbit_margin_sec"), @@ -1138,6 +1158,7 @@ class ResultCatalogService: orbit_direction=profile_payload.get("orbit_direction"), time_baseline_days=profile_payload.get("time_baseline_days"), spatial_baseline_meters=profile_payload.get("spatial_baseline_meters"), + scene_center_distance_meters=profile_payload.get("scene_center_distance_meters"), grid_size_m=profile_payload.get("grid_size_m"), radar_wavelength=profile_payload.get("radar_wavelength"), orbit_clip_margin=profile_payload.get("orbit_clip_margin"), @@ -1532,6 +1553,7 @@ class ResultCatalogService: "orbit_direction": profile.orbit_direction, "time_baseline_days": profile.time_baseline_days, "spatial_baseline_meters": profile.spatial_baseline_meters, + "scene_center_distance_meters": profile.scene_center_distance_meters, "grid_size_m": profile.grid_size_m, "radar_wavelength": profile.radar_wavelength, "orbit_clip_margin": profile.orbit_clip_margin, diff --git a/backend/app/services/spatial_service.py b/backend/app/services/spatial_service.py index 29fa0b0..2b746c4 100644 --- a/backend/app/services/spatial_service.py +++ b/backend/app/services/spatial_service.py @@ -14,7 +14,7 @@ from typing import Any, Dict, List, Optional, Tuple from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select -from sqlalchemy import and_, cast, func +from sqlalchemy import and_, cast, func, or_ from sqlalchemy.orm import aliased from geoalchemy2 import Geography @@ -43,7 +43,7 @@ from .dinsar_naming import build_pair_key, build_task_alias, ensure_unique_task_ from .pairing_state_service import pairing_state_service -PAIRING_POLICY_VERSION = "2026.04.phase3.v1" +PAIRING_POLICY_VERSION = "2026.05.raw-source.v1" PAIRING_WARNING_CANDIDATE_THRESHOLD = 3000 logger = logging.getLogger(__name__) @@ -155,6 +155,10 @@ class SpatialService: ) -> List[dict]: master_alias = aliased(RadarDataORM) slave_alias = aliased(RadarDataORM) + center_distance_expr = func.coalesce( + PairingMetricCacheORM.scene_center_distance_meters, + PairingMetricCacheORM.spatial_baseline_meters, + ) stmt = ( select(PairingMetricCacheORM, master_alias, slave_alias) @@ -165,8 +169,9 @@ class SpatialService: PairingMetricCacheORM.status == "READY", PairingMetricCacheORM.time_baseline_days >= params.time_baseline_min, PairingMetricCacheORM.time_baseline_days <= params.time_baseline_max, - PairingMetricCacheORM.spatial_baseline_meters <= params.spatial_baseline_max_meters, + center_distance_expr <= params.spatial_baseline_max_meters, PairingMetricCacheORM.scene_overlap_ratio >= params.overlap_threshold, + PairingMetricCacheORM.same_look_direction.is_(True), ) ) @@ -177,7 +182,7 @@ class SpatialService: ) if not params.cross_satellite_pairing: - stmt = stmt.where(PairingMetricCacheORM.same_satellite.is_(True)) + stmt = stmt.where(PairingMetricCacheORM.same_satellite_family.is_(True)) if params.require_same_imaging_mode: stmt = stmt.where(PairingMetricCacheORM.same_imaging_mode.is_(True)) @@ -186,9 +191,20 @@ class SpatialService: stmt = stmt.where(PairingMetricCacheORM.same_polarization.is_(True)) if params.allowed_satellites: + allowed_satellites = [ + str(item).strip().upper() + for item in params.allowed_satellites + if str(item).strip() + ] stmt = stmt.where( - master_alias.satellite.in_(params.allowed_satellites), - slave_alias.satellite.in_(params.allowed_satellites), + or_( + func.upper(master_alias.satellite).in_(allowed_satellites), + func.upper(master_alias.satellite_family).in_(allowed_satellites), + ), + or_( + func.upper(slave_alias.satellite).in_(allowed_satellites), + func.upper(slave_alias.satellite_family).in_(allowed_satellites), + ), ) if params.master_date_from: @@ -220,12 +236,18 @@ class SpatialService: PairingMetricCacheORM.master_imaging_date.asc(), PairingMetricCacheORM.slave_imaging_date.asc(), func.coalesce(PairingMetricCacheORM.scene_overlap_ratio, 0).desc(), + center_distance_expr.asc(), PairingMetricCacheORM.pair_uid.asc(), ) result = await db.execute(stmt) candidate_pool: List[dict] = [] for metric_row, master_row, slave_row in result.all(): + center_distance = float( + metric_row.scene_center_distance_meters + if metric_row.scene_center_distance_meters is not None + else (metric_row.spatial_baseline_meters or 0) + ) candidate_pool.append( { "metric_cache_ref_id": int(metric_row.id), @@ -235,7 +257,8 @@ class SpatialService: "master": RadarData.model_validate(master_row), "slave": RadarData.model_validate(slave_row), "days": int(metric_row.time_baseline_days or 0), - "dist": float(metric_row.spatial_baseline_meters or 0), + "dist": center_distance, + "scene_center_distance_meters": center_distance, "overlap_ratio": float(metric_row.scene_overlap_ratio or 0), } ) @@ -270,6 +293,11 @@ class SpatialService: selection_reason=candidate.get("selection_reason"), time_baseline_days=int(candidate["days"]), spatial_baseline_meters=float(candidate["dist"]), + scene_center_distance_meters=float( + candidate.get("scene_center_distance_meters") + if candidate.get("scene_center_distance_meters") is not None + else candidate.get("dist") or 0 + ), ) ) return result_pairs @@ -356,6 +384,12 @@ class SpatialService: "pair_uid": candidate.get("pair_uid"), "time_baseline_days": int(candidate.get("days") or 0), "spatial_baseline_meters": float(candidate.get("dist") or 0.0), + "scene_center_distance_meters": float( + candidate.get("scene_center_distance_meters") + if candidate.get("scene_center_distance_meters") is not None + else candidate.get("dist") or 0.0 + ), + "legacy_spatial_baseline_field": "scene_center_distance_meters", "scene_overlap_ratio": float(candidate.get("overlap_ratio") or 0.0), } @@ -420,6 +454,10 @@ class SpatialService: master_alias = aliased(RadarDataORM) slave_alias = aliased(RadarDataORM) + center_distance_expr = func.coalesce( + PairingMetricCacheORM.scene_center_distance_meters, + PairingMetricCacheORM.spatial_baseline_meters, + ) stmt = ( select(PairingMetricCacheORM, master_alias, slave_alias) .join(master_alias, master_alias.id == PairingMetricCacheORM.master_scene_ref_id) @@ -431,14 +469,15 @@ class SpatialService: PairingMetricCacheORM.slave_scene_ref_id.in_(scene_ids), PairingMetricCacheORM.time_baseline_days >= params.time_baseline_min, PairingMetricCacheORM.time_baseline_days <= params.time_baseline_max, - PairingMetricCacheORM.spatial_baseline_meters <= params.spatial_baseline_max_meters, + center_distance_expr <= params.spatial_baseline_max_meters, PairingMetricCacheORM.scene_overlap_ratio >= params.network_overlap_threshold, + PairingMetricCacheORM.same_look_direction.is_(True), ) .order_by( PairingMetricCacheORM.master_imaging_date.asc(), PairingMetricCacheORM.slave_imaging_date.asc(), PairingMetricCacheORM.time_baseline_days.asc(), - PairingMetricCacheORM.spatial_baseline_meters.asc(), + center_distance_expr.asc(), func.coalesce(PairingMetricCacheORM.scene_overlap_ratio, 0).desc(), PairingMetricCacheORM.id.asc(), ) @@ -447,6 +486,11 @@ class SpatialService: candidate_pool: List[dict] = [] for metric_row, master_row, slave_row in result.all(): + center_distance = float( + metric_row.scene_center_distance_meters + if metric_row.scene_center_distance_meters is not None + else (metric_row.spatial_baseline_meters or 0.0) + ) candidate_pool.append( { "metric_cache_ref_id": int(metric_row.id), @@ -456,7 +500,8 @@ class SpatialService: "master": RadarData.model_validate(master_row), "slave": RadarData.model_validate(slave_row), "days": int(metric_row.time_baseline_days or 0), - "dist": float(metric_row.spatial_baseline_meters or 0.0), + "dist": center_distance, + "scene_center_distance_meters": center_distance, "overlap_ratio": float(metric_row.scene_overlap_ratio or 0.0), } ) @@ -511,6 +556,11 @@ class SpatialService: "slave_imaging_date": slave.imaging_date, "temporal_baseline_days": int(candidate.get("days") or 0), "spatial_baseline_meters": float(candidate.get("dist") or 0.0), + "scene_center_distance_meters": float( + candidate.get("scene_center_distance_meters") + if candidate.get("scene_center_distance_meters") is not None + else candidate.get("dist") or 0.0 + ), "scene_overlap_ratio": float(candidate.get("overlap_ratio") or 0.0), "selection_reason": candidate.get("selection_reason"), "selection_score": ( @@ -523,6 +573,11 @@ class SpatialService: "selection_mode": selection_mode, "pair_uid": candidate.get("pair_uid"), "metric_version": pairing_state_service.metric_version, + "scene_center_distance_meters": float( + candidate.get("scene_center_distance_meters") + if candidate.get("scene_center_distance_meters") is not None + else candidate.get("dist") or 0.0 + ), "time_baseline_min": params.time_baseline_min, "time_baseline_max": params.time_baseline_max, "spatial_baseline_max_meters": params.spatial_baseline_max_meters, @@ -703,18 +758,44 @@ class SpatialService: if params.strategy == "sbas": return self._apply_sbas_strategy(candidate_pool, params, aoi_wkt=aoi_wkt) if params.strategy == "sequential": - return self._apply_sequential_strategy(candidate_pool, params.num_connections) + return self._apply_sequential_strategy(candidate_pool, params.num_connections, params) if params.strategy == "star": - return self._apply_star_strategy(candidate_pool, params.reference_image_id) - return self._apply_all_strategy(candidate_pool) + return self._apply_star_strategy(candidate_pool, params.reference_image_id, params) + return self._apply_all_strategy(candidate_pool, params) - def _apply_all_strategy(self, candidate_pool: List[dict]) -> Tuple[List[dict], List[str]]: + def _score_pair_candidate(self, candidate: dict, params: PairingRequest) -> float: + max_time = max(float(params.time_baseline_max or 1), 1.0) + max_center = max(float(params.spatial_baseline_max_meters or 1), 1.0) + time_score = 1.0 - min(float(candidate.get("days") or 0) / max_time, 1.0) + center_score = 1.0 - min(float(candidate.get("dist") or 0) / max_center, 1.0) + overlap_score = min(max(float(candidate.get("overlap_ratio") or 0), 0.0), 1.0) + source_score = 1.0 if ( + bool(getattr(candidate.get("master"), "insar_source_ready", False)) + and bool(getattr(candidate.get("slave"), "insar_source_ready", False)) + ) else 0.0 + orbit_score = 1.0 if ( + bool(getattr(candidate.get("master"), "has_orbit_data", False)) + and bool(getattr(candidate.get("slave"), "has_orbit_data", False)) + ) else 0.0 + return ( + 0.25 * time_score + + 0.20 * center_score + + 0.35 * overlap_score + + 0.15 * source_score + + 0.05 * orbit_score + ) + + def _apply_all_strategy( + self, + candidate_pool: List[dict], + params: PairingRequest, + ) -> Tuple[List[dict], List[str]]: return ( [ { **candidate, "selection_reason": "all_candidate", - "selection_score": float(candidate.get("overlap_ratio") or 0), + "selection_score": self._score_pair_candidate(candidate, params), } for candidate in self._sorted_candidates(candidate_pool) ], @@ -725,6 +806,7 @@ class SpatialService: self, candidate_pool: List[dict], num_connections: int, + params: PairingRequest, ) -> Tuple[List[dict], List[str]]: """ Sequential: 按稳定时间序列排序，每景连接后续 N 景。 @@ -759,7 +841,7 @@ class SpatialService: { **candidate, "selection_reason": "sequential_neighbor", - "selection_score": float(candidate.get("overlap_ratio") or 0), + "selection_score": self._score_pair_candidate(candidate, params), } ) picked_count += 1 @@ -770,6 +852,7 @@ class SpatialService: self, candidate_pool: List[dict], reference_image_id: Optional[int], + params: PairingRequest, ) -> Tuple[List[dict], List[str]]: """ Star: 参考像固定为 master。 @@ -820,7 +903,7 @@ class SpatialService: { **candidate, "selection_reason": "star_reference_master", - "selection_score": float(candidate.get("overlap_ratio") or 0), + "selection_score": self._score_pair_candidate(candidate, params), "is_reference_edge": True, "reference_image_id": int(reference_image_id), } @@ -1136,6 +1219,14 @@ class SpatialService: time_score = 1.0 - min(float(candidate.get("days") or 0) / max_time, 1.0) spatial_score = 1.0 - min(float(candidate.get("dist") or 0) / max_space, 1.0) overlap_score = min(max(float(candidate.get("overlap_ratio") or 0), 0.0), 1.0) + source_score = 1.0 if ( + bool(getattr(candidate.get("master"), "insar_source_ready", False)) + and bool(getattr(candidate.get("slave"), "insar_source_ready", False)) + ) else 0.0 + orbit_score = 1.0 if ( + bool(getattr(candidate.get("master"), "has_orbit_data", False)) + and bool(getattr(candidate.get("slave"), "has_orbit_data", False)) + ) else 0.0 aoi_gain = 0.0 redundancy_penalty = 0.0 @@ -1153,10 +1244,12 @@ class SpatialService: redundancy_penalty = max(0.0, min(overlap_area / candidate_area, 1.0)) return ( - 0.35 * time_score - + 0.20 * spatial_score + 0.30 * time_score + + 0.15 * spatial_score + 0.30 * overlap_score - + 0.15 * aoi_gain + + 0.10 * aoi_gain + + 0.10 * source_score + + 0.05 * orbit_score - float(params.coverage_diversity_penalty or 0.0) * redundancy_penalty ) @@ -1588,7 +1681,7 @@ class SpatialService: slave: RadarDataORM ) -> float: """ - Calculate spatial baseline in meters using PostGIS sphere distance. + Calculate footprint center distance in meters using PostGIS sphere distance. """ master_alias = RadarDataORM.__table__.alias("master") slave_alias = RadarDataORM.__table__.alias("slave") diff --git a/backend/app/utils.py b/backend/app/utils.py index 1875558..5d4835e 100644 --- a/backend/app/utils.py +++ b/backend/app/utils.py @@ -33,8 +33,15 @@ def _radar_meta_base() -> Dict[str, Any]: "scene_center_lat": None, "acquisition_time_utc": None, "product_type": None, + "source_product_token": None, + "image_data_type": None, + "image_data_format": None, + "product_variant": None, "product_level": None, "product_unique_id": None, + "satellite_family": None, + "look_direction": None, + "geocoded_flag": None, } @@ -69,6 +76,20 @@ def _parse_coord_token(value: Optional[str]) -> Optional[float]: return None +def normalize_satellite_family(value: Optional[str]) -> Optional[str]: + raw = str(value or "").strip().upper() + if not raw: + return None + compact = raw.replace("-", "").replace("_", "").replace(" ", "") + if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}: + return "LT1" + if compact in {"S1", "S1A", "S1B", "SENTINEL1", "SENTINEL1A", "SENTINEL1B"}: + return "S1" + if compact in {"GF3", "GAOFEN3"}: + return "GF3" + return raw + + def parse_s1_radar_filename(folder_name: str) -> Optional[Dict[str, Any]]: """ Parses key info from a Sentinel-1 radar data folder name. @@ -81,8 +102,11 @@ def parse_s1_radar_filename(folder_name: str) -> Optional[Dict[str, Any]]: meta = _radar_meta_base() meta["satellite"] = parts[0] + meta["satellite_family"] = normalize_satellite_family(parts[0]) meta["imaging_date"] = parts[4].split('T')[0] meta["imaging_mode"] = parts[1] + meta["source_product_token"] = parts[2] + meta["product_type"] = parts[2] polarization = parts[3] # e.g., '1SDV' -> 'DV' is dual-pol VV/VH meta["polarization"] = polarization[2:] if len(polarization) > 2 else polarization return meta @@ -121,6 +145,7 @@ def parse_lt1_radar_filename(folder_name: str) -> Optional[Dict[str, Any]]: meta = _radar_meta_base() meta["satellite"] = parts[0] + meta["satellite_family"] = normalize_satellite_family(parts[0]) if len(parts) > 1: meta["satellite_mode"] = parts[1] if len(parts) > 2: @@ -137,6 +162,7 @@ def parse_lt1_radar_filename(folder_name: str) -> Optional[Dict[str, Any]]: meta["imaging_date"] = _extract_date_yyyymmdd(parts[7]) meta["acquisition_time_utc"] = parts[7] if len(parts) > 8: + meta["source_product_token"] = parts[8] meta["product_type"] = parts[8] if len(parts) > 9: meta["polarization"] = parts[9] @@ -182,6 +208,7 @@ def parse_gf3_l2_dirname(folder_name: str) -> Optional[Dict[str, Any]]: meta = _radar_meta_base() meta["satellite"] = "GF3" + meta["satellite_family"] = normalize_satellite_family("GF3") parts = name.split("_") # Try to extract date: first 8-digit segment @@ -437,14 +464,19 @@ def parse_xml_metadata( ]) # --- Product Type / Level / Unique ID --- - product_type = _get_first_text([ + image_data_type = _get_first_text([ f".//{ns_prefix}imageDataInfo/{ns_prefix}imageDataType", - f".//{ns_prefix}orderInfo/{ns_prefix}productVariant", - f".//{ns_prefix}imageDataInfo/{ns_prefix}imageDataFormat", "//*[local-name()='imageDataInfo']/*[local-name()='imageDataType']", + ]) + product_variant = _get_first_text([ + f".//{ns_prefix}orderInfo/{ns_prefix}productVariant", "//*[local-name()='orderInfo']/*[local-name()='productVariant']", + ]) + image_data_format = _get_first_text([ + f".//{ns_prefix}imageDataInfo/{ns_prefix}imageDataFormat", "//*[local-name()='imageDataInfo']/*[local-name()='imageDataFormat']", ]) + product_type = image_data_type or product_variant or image_data_format product_level = _get_first_text([ f".//{ns_prefix}generalHeader/{ns_prefix}itemName", "//*[local-name()='generalHeader']/*[local-name()='itemName']", @@ -527,6 +559,9 @@ def parse_xml_metadata( "scene_center_lat": scene_center_lat, "acquisition_time_utc": acquisition_time_utc, "product_type": product_type, + "image_data_type": image_data_type, + "image_data_format": image_data_format, + "product_variant": product_variant, "product_level": product_level, "product_unique_id": product_unique_id, "look_direction": look_direction, diff --git a/backend/migrations/009_raw_source_pairing_fields.sql b/backend/migrations/009_raw_source_pairing_fields.sql new file mode 100644 index 0000000..530dbe8 --- /dev/null +++ b/backend/migrations/009_raw_source_pairing_fields.sql @@ -0,0 +1,185 @@ +-- Migration: Raw source pairing readiness and center-distance metrics +-- Version: 9.0 +-- Date: 2026-05-09 +-- Purpose: Pair D-InSAR candidates from raw complex source products without requiring prebuilt SLC/envi_import folders. + +ALTER TABLE IF EXISTS radar_data + ADD COLUMN IF NOT EXISTS satellite_family VARCHAR NULL; + +ALTER TABLE IF EXISTS radar_data + ADD COLUMN IF NOT EXISTS source_product_token VARCHAR NULL; + +ALTER TABLE IF EXISTS radar_data + ADD COLUMN IF NOT EXISTS image_data_type VARCHAR NULL; + +ALTER TABLE IF EXISTS radar_data + ADD COLUMN IF NOT EXISTS image_data_format VARCHAR NULL; + +ALTER TABLE IF EXISTS radar_data + ADD COLUMN IF NOT EXISTS product_variant VARCHAR NULL; + +ALTER TABLE IF EXISTS radar_data + ADD COLUMN IF NOT EXISTS look_direction VARCHAR NULL; + +ALTER TABLE IF EXISTS radar_data + ADD COLUMN IF NOT EXISTS geocoded_flag BOOLEAN NULL; + +ALTER TABLE IF EXISTS radar_data + ADD COLUMN IF NOT EXISTS insar_source_ready BOOLEAN NOT NULL DEFAULT FALSE; + +ALTER TABLE IF EXISTS radar_data + ADD COLUMN IF NOT EXISTS insar_source_reason TEXT NULL; + +ALTER TABLE IF EXISTS pairing_metric_cache + ADD COLUMN IF NOT EXISTS scene_center_distance_meters DOUBLE PRECISION NULL; + +ALTER TABLE IF EXISTS pairing_metric_cache + ADD COLUMN IF NOT EXISTS same_satellite_family BOOLEAN NOT NULL DEFAULT TRUE; + +ALTER TABLE IF EXISTS pairing_metric_cache + ADD COLUMN IF NOT EXISTS same_look_direction BOOLEAN NOT NULL DEFAULT TRUE; + +ALTER TABLE IF EXISTS pairing_metric_cache + ADD COLUMN IF NOT EXISTS master_satellite_family VARCHAR NULL; + +ALTER TABLE IF EXISTS pairing_metric_cache + ADD COLUMN IF NOT EXISTS slave_satellite_family VARCHAR NULL; + +ALTER TABLE IF EXISTS pairing_metric_cache + ADD COLUMN IF NOT EXISTS master_look_direction VARCHAR NULL; + +ALTER TABLE IF EXISTS pairing_metric_cache + ADD COLUMN IF NOT EXISTS slave_look_direction VARCHAR NULL; + +ALTER TABLE IF EXISTS dinsar_task_items + ADD COLUMN IF NOT EXISTS scene_center_distance_meters DOUBLE PRECISION NULL; + +ALTER TABLE IF EXISTS dinsar_product_profiles + ADD COLUMN IF NOT EXISTS scene_center_distance_meters DOUBLE PRECISION NULL; + +CREATE INDEX IF NOT EXISTS idx_radar_data_satellite_family + ON radar_data (satellite_family); + +CREATE INDEX IF NOT EXISTS idx_radar_data_look_direction + ON radar_data (look_direction); + +CREATE INDEX IF NOT EXISTS idx_radar_data_insar_source_ready + ON radar_data (insar_source_ready); + +CREATE INDEX IF NOT EXISTS idx_pairing_metric_cache_center_distance + ON pairing_metric_cache (scene_center_distance_meters); + +CREATE INDEX IF NOT EXISTS idx_pairing_metric_cache_same_family + ON pairing_metric_cache (same_satellite_family); + +UPDATE radar_data +SET + satellite_family = COALESCE( + NULLIF(satellite_family, ''), + CASE + WHEN upper(replace(replace(replace(COALESCE(satellite, ''), '-', ''), '_', ''), ' ', '')) IN + ('LT1', 'LT1A', 'LT1B', 'LUTAN1', 'LUTAN1A', 'LUTAN1B') + THEN 'LT1' + WHEN upper(replace(replace(replace(COALESCE(satellite, ''), '-', ''), '_', ''), ' ', '')) IN + ('S1', 'S1A', 'S1B', 'SENTINEL1', 'SENTINEL1A', 'SENTINEL1B') + THEN 'S1' + WHEN NULLIF(satellite, '') IS NOT NULL + THEN upper(satellite) + ELSE NULL + END + ), + source_product_token = COALESCE( + NULLIF(source_product_token, ''), + CASE + WHEN split_part(regexp_replace(COALESCE(file_path, ''), '^.*[\\/]', ''), '_', 1) LIKE 'LT1%%' + THEN NULLIF(split_part(regexp_replace(COALESCE(file_path, ''), '^.*[\\/]', ''), '_', 9), '') + WHEN split_part(regexp_replace(COALESCE(file_path, ''), '^.*[\\/]', ''), '_', 1) LIKE 'S1%%' + THEN NULLIF(split_part(regexp_replace(COALESCE(file_path, ''), '^.*[\\/]', ''), '_', 3), '') + ELSE NULL + END + ), + image_data_type = COALESCE(NULLIF(image_data_type, ''), NULLIF(product_type, '')) +WHERE satellite_family IS NULL + OR satellite_family = '' + OR source_product_token IS NULL + OR source_product_token = '' + OR image_data_type IS NULL + OR image_data_type = ''; + +UPDATE radar_data +SET + insar_source_ready = ( + geom IS NOT NULL + AND imaging_date ~ '^[0-9]{8}$' + AND NULLIF(orbit_direction, '') IS NOT NULL + AND NULLIF(imaging_mode, '') IS NOT NULL + AND NULLIF(polarization, '') IS NOT NULL + AND NULLIF(satellite_family, '') IS NOT NULL + AND geocoded_flag IS DISTINCT FROM TRUE + AND ( + upper(COALESCE(NULLIF(image_data_type, ''), NULLIF(product_type, ''), '')) = 'COMPLEX' + OR upper(COALESCE(NULLIF(source_product_token, ''), '')) IN ('SLC', 'SSC') + OR upper(COALESCE(NULLIF(product_variant, ''), '')) IN ('SLC', 'SSC') + ) + ), + insar_source_reason = CASE + WHEN ( + geom IS NOT NULL + AND imaging_date ~ '^[0-9]{8}$' + AND NULLIF(orbit_direction, '') IS NOT NULL + AND NULLIF(imaging_mode, '') IS NOT NULL + AND NULLIF(polarization, '') IS NOT NULL + AND NULLIF(satellite_family, '') IS NOT NULL + AND geocoded_flag IS DISTINCT FROM TRUE + AND ( + upper(COALESCE(NULLIF(image_data_type, ''), NULLIF(product_type, ''), '')) = 'COMPLEX' + OR upper(COALESCE(NULLIF(source_product_token, ''), '')) IN ('SLC', 'SSC') + OR upper(COALESCE(NULLIF(product_variant, ''), '')) IN ('SLC', 'SSC') + ) + ) + THEN NULL + ELSE concat_ws( + ';', + CASE WHEN geom IS NULL THEN 'missing_footprint' END, + CASE WHEN imaging_date IS NULL OR imaging_date !~ '^[0-9]{8}$' THEN 'missing_date' END, + CASE WHEN NULLIF(orbit_direction, '') IS NULL THEN 'missing_orbit_direction' END, + CASE WHEN NULLIF(imaging_mode, '') IS NULL THEN 'missing_imaging_mode' END, + CASE WHEN NULLIF(polarization, '') IS NULL THEN 'missing_polarization' END, + CASE WHEN NULLIF(satellite_family, '') IS NULL THEN 'missing_satellite_family' END, + CASE WHEN geocoded_flag IS TRUE THEN 'geocoded_product' END, + CASE WHEN NOT ( + upper(COALESCE(NULLIF(image_data_type, ''), NULLIF(product_type, ''), '')) = 'COMPLEX' + OR upper(COALESCE(NULLIF(source_product_token, ''), '')) IN ('SLC', 'SSC') + OR upper(COALESCE(NULLIF(product_variant, ''), '')) IN ('SLC', 'SSC') + ) THEN 'not_complex_source' END + ) + END; + +UPDATE pairing_metric_cache pmc +SET + scene_center_distance_meters = COALESCE(pmc.scene_center_distance_meters, pmc.spatial_baseline_meters), + master_satellite_family = COALESCE(pmc.master_satellite_family, m.satellite_family), + slave_satellite_family = COALESCE(pmc.slave_satellite_family, s.satellite_family), + master_look_direction = COALESCE(pmc.master_look_direction, m.look_direction), + slave_look_direction = COALESCE(pmc.slave_look_direction, s.look_direction), + same_satellite_family = ( + NULLIF(COALESCE(m.satellite_family, m.satellite), '') IS NOT NULL + AND NULLIF(COALESCE(s.satellite_family, s.satellite), '') IS NOT NULL + AND COALESCE(m.satellite_family, m.satellite) = COALESCE(s.satellite_family, s.satellite) + ), + same_look_direction = ( + NULLIF(m.look_direction, '') IS NULL + OR NULLIF(s.look_direction, '') IS NULL + OR m.look_direction = s.look_direction + ) +FROM radar_data m, radar_data s +WHERE pmc.master_scene_ref_id = m.id + AND pmc.slave_scene_ref_id = s.id; + +UPDATE pairing_cache_state +SET + metric_version = '2026.05.raw.v1', + status = CASE WHEN status = 'REBUILDING' THEN status ELSE 'DIRTY' END, + last_error = NULL, + updated_at = NOW() +WHERE metric_version IS DISTINCT FROM '2026.05.raw.v1'; diff --git a/docs/DINSAR_PAIRING_DISTRIBUTION_LOGIC_20260508.md b/docs/DINSAR_PAIRING_DISTRIBUTION_LOGIC_20260508.md new file mode 100644 index 0000000..8982a22 --- /dev/null +++ b/docs/DINSAR_PAIRING_DISTRIBUTION_LOGIC_20260508.md @@ -0,0 +1,553 @@ +# D-InSAR 配对与分发逻辑梳理 + +更新时间：2026-05-09 + +本文按当前代码实现梳理 D-InSAR 从“雷达数据入库”到“配对规划”、“批次保存”、“数据分发”和“多引擎生产执行”的主链路。重点依据源码，而不是早期设计文档。 + +## 1. 总览 + +当前 D-InSAR 链路分为两层： + +1. 配对规划层：把 `radar_data` 中的影像先预计算成 `pairing_metric_cache` 候选边，再按用户阈值和策略筛选，最后固化为一次 `pairing_network_runs` 和若干 `pairing_network_edges`。 +2. 分发执行层：配对结果可保存为 `dinsar_task_batches/items`，再复制成 `Task_*/master`、`Task_*/slave` 生产目录；生产面板再以这个根目录提交到 SARscape、ISCE2 或 PyINT/Gamma 引擎，由 DB job queue 和 worker 执行。 + +核心入口： + +- 配对 API：[backend/app/routers/pairing.py](../backend/app/routers/pairing.py) +- 配对服务：[backend/app/services/spatial_service.py](../backend/app/services/spatial_service.py) +- 配对缓存：[backend/app/services/pairing_cache_service.py](../backend/app/services/pairing_cache_service.py) +- 批次 API：[backend/app/routers/task_batches.py](../backend/app/routers/task_batches.py) +- 数据分发 API：[backend/app/routers/tools.py](../backend/app/routers/tools.py) +- 数据复制执行：[backend/app/copier.py](../backend/app/copier.py) +- 生产提交 API：[backend/app/routers/dinsar_production.py](../backend/app/routers/dinsar_production.py) +- 生产运行状态：[backend/app/services/dinsar_production_service.py](../backend/app/services/dinsar_production_service.py) +- job 队列和 worker：[backend/app/services/job_queue_service.py](../backend/app/services/job_queue_service.py)、[backend/app/services/job_worker.py](../backend/app/services/job_worker.py) + +## 2. 数据入库与配对缓存失效 + +雷达数据扫描在 [backend/app/services/data_service.py](../backend/app/services/data_service.py) 中写入或更新 `radar_data`。每个 scene 使用 `unique_id` 做 upsert；如果发现新 scene 或补齐了轨道文件，会调用 `pairing_state_service.mark_scenes_dirty()` 或 `mark_global_dirty()`。 + +配对缓存状态由 [backend/app/services/pairing_state_service.py](../backend/app/services/pairing_state_service.py) 管理： + +- 全局状态表：`pairing_cache_state` +- 待重算 scene 表：`pairing_dirty_scenes` +- 当前指标版本：`2026.05.raw.v1` +- 当前 master/slave 定向规则：`date_then_scene_uid_v1` + +应用启动时会调用 `bootstrap_pairing_cache_state()`，但不会自动全量重建候选边。缓存如果是 `DIRTY`，配对仍可返回旧缓存结果并给 warning；如果是 `FAILED`、`UNINITIALIZED`、`ERROR`，或 scene 数大于 1 但候选边为 0，`/find-pairs` 会拒绝并提示先修复缓存。 + +## 3. 候选边缓存构建 + +候选边缓存由 [pairing_cache_service.py](../backend/app/services/pairing_cache_service.py) 写入 `pairing_metric_cache`。 + +全量重建逻辑： + +- 删除全部 `pairing_metric_cache` +- 从 `radar_data m JOIN radar_data s` 重新生成候选边 +- 只保留满足硬约束的 pair： + - `m.id <> s.id` + - 两景都有 `geom` + - `imaging_date` 是 8 位日期 + - 两景都有 `orbit_direction` 且方向一致 + - 两景都是可用于 InSAR 的原始复数源：`insar_source_ready = true` + - 如果两景都有 `look_direction`，要求视向一致 + - 几何相交 `ST_Intersects` + - 按 `date_then_scene_uid_v1` 只保留一个方向，避免 A-B 和 B-A 双向重复 + +写入的主要指标： + +- `time_baseline_days`：两景日期差的绝对值 +- `scene_center_distance_meters`：两景 footprint 质心的球面距离 +- `spatial_baseline_meters`：兼容旧 API 的历史字段；新缓存中暂存同一个 footprint 中心距，不能解释为 SAR 空间/垂直基线 +- `scene_overlap_ratio`：两景交集面积 / 两景较大 footprint 面积 +- `same_satellite` +- `same_satellite_family`：同一卫星族，例如 LT1A/LT1B 归为 `LT1` +- `same_look_direction` +- `same_imaging_mode` +- `same_polarization` +- `pair_uid = md5(master_scene_uid + '|' + slave_scene_uid)` + +增量重算逻辑： + +- 如果 dirty scene 数过多、占比过高、或缓存为空，会转全量重建 +- 否则删除涉及 dirty scene 的缓存边 +- 对每个 dirty scene 与其他 scene 重新计算边 +- resolved 对应 dirty rows + +阈值： + +- dirty scene 数量达到 64 触发全量重建 +- dirty scene 占 scene 总数比例达到 25% 触发全量重建 + +## 4. `/find-pairs` 配对查询 + +前端在 [frontend/src/hooks/usePairingLogic.js](../frontend/src/hooks/usePairingLogic.js) 中把配对参数、AOI 文件或行政区 GeoJSON 组装为 `FormData`，提交到 `POST /api/find-pairs`。 + +后端入口是 [pairing.py](../backend/app/routers/pairing.py)： + +- 解析配对参数为 `PairingRequest` +- 解析 AOI：支持上传 Shapefile 或传入 GeoJSON +- 调用 `spatial_service.find_dinsar_pairs()` +- 返回 `PairingResponse`，包含 pairs、warnings、`network_run_id`、`policy_version`、候选数和入选边数 + +`PairingRequest` 在 [backend/app/models/schemas.py](../backend/app/models/schemas.py) 中定义，主要参数包括： + +- `time_baseline_min/max` +- `overlap_threshold` +- `spatial_baseline_max_meters` +- `coverage_diversity_penalty` +- `require_same_imaging_mode` +- `require_same_polarization` +- `aoi_overlap_threshold` +- master/slave 日期范围 +- `strategy`: `all | sbas | sequential | star` +- `num_connections` +- `reference_image_id` +- `allowed_satellites` +- `cross_satellite_pairing` +- `start_date` 兼容旧参数 + +## 5. 候选池过滤条件 + +`spatial_service._query_pairing_metric_cache()` 只查询缓存表，不再实时两两计算。基础过滤条件： + +- `metric_version == 2026.05.raw.v1` +- `status == READY` +- `time_baseline_days` 在请求范围内 +- `scene_center_distance_meters <= spatial_baseline_max_meters` +- `scene_overlap_ratio >= overlap_threshold` +- `same_look_direction = true` +- 如果 `require_orbit_data = true`，master 和 slave 都要有精轨 +- 默认要求同卫星族；除非 `cross_satellite_pairing = true` +- 默认要求成像模式一致、极化一致 +- 如果 `allowed_satellites` 不为空，master/slave 的卫星名或卫星族都必须在列表内 +- 如果传入 master/slave 日期范围，分别约束 `master_imaging_date` 和 `slave_imaging_date` +- 如果有 AOI，master/slave footprint 都要与 AOI 相交 +- 如果 `aoi_overlap_threshold` 有值，master/slave 各自覆盖 AOI 的比例都要达标 + +排序默认按： + +1. master 日期升序 +2. slave 日期升序 +3. overlap 降序 +4. pair_uid 升序 + +## 6. 配对策略 + +策略选择在 `spatial_service._apply_strategy()`。 + +### 6.1 all + +`all` 策略不再做网络抽稀，直接返回过滤后的全部候选边。每条边的： + +- `selection_reason = all_candidate` +- `selection_score` 综合时间基线、footprint 中心距、重叠率、源数据可用性和精轨状态 + +### 6.2 sequential + +`sequential` 策略先从候选池提取 scene，按稳定时间键排序： + +- 优先 `acquisition_time_utc` +- 否则 `imaging_date` +- 再按 scene_uid 和 id 打平同日多景 + +然后每个 scene 向后寻找最多 `num_connections` 个有候选边的后继 scene。不存在于候选池的边不会被补造。 + +输出边： + +- `selection_reason = sequential_neighbor` +- `selection_score` 综合时间基线、footprint 中心距、重叠率、源数据可用性和精轨状态 + +### 6.3 star + +`star` 策略要求参考影像固定作为 master。 + +如果用户未指定 `reference_image_id`，系统会在时间序列中找靠近中位位置、且能作为 master 的 scene 自动作为参考影像。注意当前实现不会把 slave 侧边反转为 master 侧边；如果参考影像在候选边中只能出现在 slave 侧，这些边会被跳过并给 warning。 + +输出边： + +- `selection_reason = star_reference_master` +- `is_reference_edge = true` +- `reference_image_id` 写入 edge meta + +### 6.4 sbas + +`sbas` 策略用于构造小基线网络，流程是： + +1. 按时间顺序先选相邻 scene 的候选边，形成时间骨架。 +2. 如果网络有多个连通分量，优先选能连接分量的候选边。 +3. 继续补低度数节点，直到达到目标连接数或达到最大边数。 +4. 如果无法形成完整连通图，或存在 0 度/低度数节点，返回 warning。 + +关键参数： + +- `min_degree = min(max(1, num_connections), scene_count - 1)` +- `max_degree = min(max(min_degree + 2, 3), scene_count - 1)` +- `max_edges = min(candidate_count, max(scene_count - 1, scene_count * min_degree))` + +候选边评分： + +```text +score = + 0.30 * time_score ++ 0.15 * center_distance_score ++ 0.30 * overlap_score ++ 0.10 * aoi_gain ++ 0.10 * source_ready_score ++ 0.05 * orbit_score +- coverage_diversity_penalty * redundancy_penalty +``` + +其中 `aoi_gain` 和 `redundancy_penalty` 基于 master/slave 交集几何计算；如果有 AOI，会先把交集裁到 AOI 范围。 + +## 7. 网络运行留痕 + +每次 `/find-pairs` 都会创建一条 `pairing_network_runs`： + +- `network_run_id = pnr_` +- `strategy` +- `policy_version = 2026.05.raw-source.v1` +- `request_hash` +- 请求参数 JSON +- AOI hash 和 summary +- 候选边数量、入选边数量、warning 数量 + +每条入选边写入 `pairing_network_edges`： + +- 指向 `pairing_metric_cache` +- `edge_rank` +- `selection_reason` +- `selection_score` +- `selection_meta_json` +- `is_reference_edge` + +之后 `RadarPair` 响应会携带： + +- `pair_key` +- `pair_uid` +- `metric_cache_ref_id` +- `network_run_id` +- `network_edge_id` +- `policy_version` +- `selection_strategy` +- `selection_score` +- `selection_reason` +- `scene_center_distance_meters` +- `task_name/task_alias` + +`task_alias` 由 [dinsar_naming.py](../backend/app/services/dinsar_naming.py) 生成，格式是 `Task_YYYYMMDD_YYYYMMDD`；同名时追加 `_1`、`_2` 保证唯一。 + +## 8. 批次保存 + +前端找到 pairs 后，用户勾选结果并调用 `createDinsarBatch()`，提交到 `POST /api/task-batches/dinsar`。 + +后端 [task_batches.py](../backend/app/routers/task_batches.py) 会创建： + +- `dinsar_task_batches` +- `dinsar_task_items` + +每条 item 会保存： + +- `task_name/task_alias` +- `pair_key` +- `scene_pair_uid` +- `network_run_id` +- `network_edge_id` +- `policy_version` +- `selection_strategy` +- master/slave 文件路径 +- master/slave 卫星、日期、成像模式、极化 +- 时间基线、footprint 中心距 +- 人工审核状态，默认 `PENDING` + +前端批次面板可把 item 状态改成： + +- `PENDING` +- `IN_PROGRESS` +- `COMPLETED` +- `FAILED` + +数据分发默认只复制 `COMPLETED` 状态的条目。 + +## 9. 数据分发到 Task 目录 + +数据分发入口是 `POST /api/tools/copy-dinsar-pairs`，代码在 [tools.py](../backend/app/routers/tools.py)。 + +请求参数： + +- `batch_id` +- `dest_dir` +- `copy_statuses`，为空时默认 `["COMPLETED"]` +- `include_orbit_files`，默认 `false`；为 `true` 时把 master/slave 精轨复制到 Task 内的 `orbit/` +- `export_zip`，默认 `false`；为 `true` 时每个 Task 输出为一个 `.zip` 包 + +后端动作： + +1. 校验目标路径。 +2. 创建 `SystemTask`，类型为 `COPY_DATA`。 +3. 创建 `SystemJob`，job_type 也是 `COPY_DATA`。 +4. worker 领取 job 后进入 `job_handlers._handle_copy_data()`。 +5. `_handle_copy_data()` 根据 `batch_id` 查询 `dinsar_task_items`，只取 `copy_statuses` 命中的条目。 +6. 调用 [backend/app/copier.py](../backend/app/copier.py) 的 `run_dinsar_copy_items()`。 + +`run_dinsar_copy_items()` 对每个 item 执行： + +- 文件夹模式目标目录：`//` +- zip 模式目标文件：`/.zip` +- master 目录：`/master` +- slave 目录：`/slave` +- 如果启用 `include_orbit_files`，从 `radar_data.orbit_file_path` 找 master/slave 精轨并复制到 `/orbit/` +- 直接复制配对时保存的原始产品目录；D-InSAR 分发不再优先使用 `envi_import/` +- 使用 `shutil.copytree(..., dirs_exist_ok=True)` 复制 master/slave +- 写入 `/.dinsar_pair.json` + +`.dinsar_pair.json` 是后续生产追踪的关键 sidecar，包含： + +- `pair_key` +- `task_name/task_alias` +- master/slave 原始路径和元数据 +- `time_baseline_days` +- `spatial_baseline_meters` +- `scene_center_distance_meters` +- `package_format` +- `include_orbit_files` +- `orbit_files` +- `scene_pair_uid/pair_uid` +- `network_run_id` +- `network_edge_id` +- `policy_version` +- `selection_strategy` +- `copied_at` + +当前实现不会清空已有 Task 目录，而是合并复制；如果目标已有旧文件，需要人工确认目录状态。 + +## 10. 生产提交与运行分发 + +生产入口是 `POST /api/dinsar-production/run`，前端在 [frontend/src/DinsarProductionPanel.jsx](../frontend/src/DinsarProductionPanel.jsx) 手动输入“根目录或单个任务目录”并选择引擎/模板。 + +支持引擎来自 [backend/app/dinsar_engines/registry.py](../backend/app/dinsar_engines/registry.py)： + +- `sarscape` +- `isce2` +- `pyint` +- `landsar`，目前预留，不进入 D-InSAR queued production 主链路 + +提交流程： + +1. 校验 engine 是否注册且可用。 +2. 校验 profile 是否属于该 engine。 +3. 对 ISCE2/PyINT 调用 engine 的 `validate_root_dir()` 和 `normalize_extra()`。 +4. PyINT 会额外做输入资产预检。 +5. 当前 SARscape、ISCE2、PyINT 都走 managed production run。 +6. 调用 `dinsar_production_service.create_run()`。 + +`create_run()` 做的事情： + +- 根据引擎映射 task_type： + - SARscape -> `IDL_RUN_DINSAR` + - ISCE2 -> `ISCE2_RUN` + - PyINT/Gamma -> `PYINT_RUN` +- 扫描 root 下的 `Task_*` 目录，或把 root 本身当单个 Task 目录 +- 从 `.dinsar_pair.json` 解析 pair identity；如果没有 sidecar，则按目录名和路径生成 fallback +- 根据 `rerun_mode` 跳过已有 current pointer 的完成项 +- 创建 `SystemTask` +- 创建 `dinsar_production_runs` +- 创建 `dinsar_production_run_items` +- 创建一个 workflow run，只有一个 step：`execute_items` +- workflow step 入队为 `SystemJob` + +注意：生产面板目前不直接从 `dinsar_task_batches` 选择批次。实际串联方式是：先在“分发”面板把批次复制到生产根目录，再在“生产”面板提交这个根目录。 + +## 11. worker 与执行控制 + +后台 worker 在 [job_worker.py](../backend/app/services/job_worker.py)： + +- 周期性 `claim_next_job()` +- DB 查询使用 `FOR UPDATE SKIP LOCKED` +- 按 `priority DESC, id ASC` 领取 `READY/RETRY` job +- 支持 worker heartbeat +- 支持 stale RUNNING job 恢复为 RETRY 或 FAILED +- `run_worker_loop()` 参数支持 job 级并发，但默认并发为 1 + +`SystemTask` 在 [task_service.py](../backend/app/services/task_service.py) 管理： + +- 创建任务时会检查同一 `task_type` 是否已有 `PENDING/RUNNING` +- PostgreSQL 下使用 advisory lock 防止并发创建同类任务 +- 因此同一类生产任务天然串行提交 + +workflow 在 [workflow_service.py](../backend/app/services/workflow_service.py)： + +- 创建 workflow run 和 steps +- 没有依赖的 step 立即入队 +- job 完成后 mark step completed +- step 全部终态后 workflow run 完成 + +## 12. 各引擎生产控制器 + +job handler 在 [job_handlers.py](../backend/app/services/job_handlers.py)。 + +### 12.1 SARscape + +`_handle_idl_run_dinsar()` 如果 payload 有 `production_run_id`，会进入 `_run_dinsar_production_controller()`。 + +执行特点： + +- 使用 `engine_lock_service.acquire("envi_taskengine")`，保证 ENVI/SARscape taskengine 串行 +- 对 run item 逐个执行 +- 每个 item 创建一个 `DinsarProductionExecution` +- 调用 `build_envi_runner_command()` 启动 runner +- 运行结束后规范化输出目录 +- 写 `execution_manifest.json` +- 写 `current/__.json` +- 标记 item completed/failed/cancelled +- 成功输出目录会进入 `result_catalog_service.publish_from_sources()` + +### 12.2 ISCE2 与 PyINT/Gamma + +`_handle_isce2_run()` 和 `_handle_pyint_run()` 在 managed 模式下都进入 `_run_wsl_dinsar_production_controller()`。 + +执行特点： + +- 使用 `engine_lock_service.acquire(f"wsl_dinsar_{engine_code}")` +- 每个 item 构造独立 managed run 目录： + - run dir + - native dir + - workflow dir + - export dir + - orbit output dir +- 构造 `RunRequest` 调用 engine 的 `run()` +- engine 返回 `primary_file`、`source_files`、`native_output_dir` +- 校验 primary output 存在 +- 写 `execution_manifest.json` +- 写 current pointer +- 标记 item 状态 +- 发布成功包，并对结果 catalog 做 rebuild + +一个 production run 内部 item 是串行执行的。多个 worker 可以领取不同 job，但同类任务创建限制和 engine lock 会进一步限制实际并发。 + +## 13. 结果发布与追踪 + +生产完成后会生成标准包结构，并由 result catalog 接管。`execution_manifest.json` 中保留： + +- `run_id` +- `task_id` +- `engine_code` +- `profile_code` +- `runtime_id` +- `task_name/task_alias` +- `pair_key` +- `pair_uid` +- `network_run_id` +- `network_edge_id` +- `policy_version` +- `selection_strategy` +- `source_task_dir` +- `results_root_dir` +- `publish_root_dir` +- `primary_file` +- `source_files` +- `metrics` + +catalog 注册逻辑在 [backend/app/services/result_catalog_service.py](../backend/app/services/result_catalog_service.py) 中会继续把 pairing trace 字段写到结果产品，便于从结果反查配对网络。 + +## 14. 关键表关系 + +配对规划： + +- `radar_data` +- `pairing_cache_state` +- `pairing_dirty_scenes` +- `pairing_metric_cache` +- `pairing_network_runs` +- `pairing_network_edges` + +人工批次： + +- `dinsar_task_batches` +- `dinsar_task_items` + +后台任务： + +- `system_tasks` +- `task_logs` +- `system_jobs` +- `system_worker_heartbeats` +- `workflow_runs` +- `workflow_steps` + +生产执行： + +- `dinsar_production_runs` +- `dinsar_production_run_items` +- `dinsar_production_executions` + +## 15. 常用 API 链路 + +配对健康和修复： + +- `GET /api/pairing/health` +- `POST /api/pairing/rebuild-cache` +- `POST /api/pairing/reconcile-dirty?force_full=false` + +配对规划： + +- `POST /api/find-pairs` +- `GET /api/pairing/networks/{network_run_id}` + +批次： + +- `POST /api/task-batches/dinsar` +- `GET /api/task-batches/dinsar` +- `GET /api/task-batches/dinsar/{batch_id}/items` +- `PATCH /api/task-batches/dinsar/items/{item_id}` +- `PATCH /api/task-batches/dinsar/{batch_id}/complete-all` + +数据分发： + +- `POST /api/tools/copy-dinsar-pairs` +- `GET /api/tools/copy-status/{task_id}` + +生产： + +- `GET /api/dinsar-production/engines` +- `POST /api/dinsar-production/engines/pyint/preview-input-assets` +- `POST /api/dinsar-production/run` +- `GET /api/dinsar-production/runs` + +## 16. 当前实现边界 + +1. 配对查询完全依赖 `pairing_metric_cache`。缓存未初始化、失败、或 scene 足够但 pair 为 0 时不会降级实时计算。 +2. `scene_center_distance_meters` 是 footprint 质心距离；`spatial_baseline_meters` 仅为旧 API 兼容字段，不是 SAR 几何中的垂直基线。 +3. master/slave 方向在缓存层已经固定为“日期优先、scene_uid 次之”。`star` 策略不会把参考影像位于 slave 的边翻转。 +4. `aoi_overlap_threshold` 约束的是每一景对 AOI 的覆盖比例，不是 pair 交集对 AOI 的覆盖比例。 +5. 数据分发默认只复制 `COMPLETED` 状态 item；如果用户没有在批次面板审核或一键完成，分发可能没有条目。 +6. 数据分发使用 `dirs_exist_ok=True` 合并复制，不会自动清理目标旧内容。 +7. 生产提交和批次保存之间没有数据库级直接引用；生产侧通过 `Task_*` 目录和 `.dinsar_pair.json` sidecar 重新恢复 pair trace。 +8. 每个 production run 内部 item 串行执行；job worker 可并发，但 task_type 冲突检查和 engine lock 会限制同类引擎并发。 +9. `landsar` 已注册为 engine，但当前 `/dinsar-production/run` 仅对 SARscape、ISCE2、PyINT 建立 queued production 主链路。 + +## 17. 推荐排查路径 + +配对为空： + +1. 查 `GET /api/pairing/health` +2. 看 `pair_count`、`dirty_scene_count`、`status` +3. 必要时执行 `POST /api/pairing/reconcile-dirty` 或 `POST /api/pairing/rebuild-cache` +4. 放宽 `time_baseline_max`、`spatial_baseline_max_meters`、`overlap_threshold` +5. 检查 `insar_source_ready`、`require_orbit_data`、同卫星族、同视向、同模式、同极化约束 + +分发为空： + +1. 查 batch item 是否存在 +2. 查 item 状态是否命中 `copy_statuses`，默认只取 `COMPLETED` +3. 查 master/slave 源路径是否存在 +4. 查目标目录是否已有旧文件影响判断 + +生产未执行： + +1. 查 `system_tasks` 状态和 task logs +2. 查 `system_jobs` 是否 READY/RUNNING/FAILED +3. 查 worker heartbeat +4. 查 engine lock 是否被长任务持有 +5. 查生产根目录是否包含有效 `Task_*/master`、`Task_*/slave` +6. 对 PyINT 先跑输入资产预检 diff --git a/docs/INDEX.md b/docs/INDEX.md index 3cdc2ea..5dcf1fe 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -60,6 +60,8 @@ ## 4. 配对与前端导航 +- [DINSAR_PAIRING_DISTRIBUTION_LOGIC_20260508.md](DINSAR_PAIRING_DISTRIBUTION_LOGIC_20260508.md) + 2026-05-08 源码走读记录，梳理 D-InSAR 配对缓存、策略筛选、批次保存、数据分发和生产 worker 执行链路。 - [PAIRING_ENHANCEMENT_DESIGN.md](PAIRING_ENHANCEMENT_DESIGN.md) - [FRONTEND_NAVIGATION_ARCHITECTURE.md](FRONTEND_NAVIGATION_ARCHITECTURE.md) diff --git a/docs/PAIRING_ENHANCEMENT_DESIGN.md b/docs/PAIRING_ENHANCEMENT_DESIGN.md index 6c7ce0d..ac5a2c7 100644 --- a/docs/PAIRING_ENHANCEMENT_DESIGN.md +++ b/docs/PAIRING_ENHANCEMENT_DESIGN.md @@ -55,8 +55,8 @@ **参数**： - `time_baseline_min/max`：时间基线范围（天） -- `spatial_baseline_max_meters`：空间基线上限（米） -- `overlap_threshold`：重叠率阈值 +- `spatial_baseline_max_meters`：footprint 中心距上限（米，兼容字段名保留） +- `overlap_threshold`：两景 footprint 最小重叠率（兼容字段名保留） - `coverage_diversity_penalty`：覆盖多样性惩罚因子 **配对逻辑**： @@ -214,7 +214,7 @@ if allowed_satellites: 不同卫星需要不同的配对参数： -| 卫星 | 典型时间基线 | 典型空间基线 | 波长 | 备注 | +| 卫星 | 典型时间基线 | 典型 footprint 中心距上限 | 波长 | 备注 | |---|---|---|---|---| | LT-1 | 30~90 天 | < 3000 m | L 波段 | 当前系统 | | Sentinel-1 | 6~12 天 | < 150 m | C 波段 | 高重访频率 | @@ -415,8 +415,8 @@ async def find_dinsar_pairs( │ ☑ 使用双池模式（不勾选则主辅池合并） │ ├─────────────────────────────────────────────────┤ │ 时间基线: [1] ~ [90] 天 │ -│ 空间基线上限: [3000] 米 │ -│ 重叠率阈值: [0.5] │ +│ footprint 中心距上限: [3000] 米 │ +│ 两景 footprint 最小重叠率: [0.5] │ │ 覆盖多样性惩罚: [0.3] │ │ │ │ ☑ 成像模式一致 ☑ 极化一致 ☑ 仅精轨影像 │ @@ -513,7 +513,7 @@ async def find_dinsar_pairs( ## 十、未来扩展 -1. **基线网络可视化**：时间-空间基线散点图（D3.js / ECharts） +1. **基线网络可视化**：时间-中心距散点图（D3.js / ECharts） 2. **配对质量评分**：根据相干性、大气条件预估配对质量 3. **自动参数推荐**：基于历史配对结果的机器学习推荐 4. **批量配对模板**：保存常用配对参数为模板 @@ -528,7 +528,7 @@ async def find_dinsar_pairs( | 主影像 | Master / Reference | 配对中的参考影像 | | 辅影像 | Slave / Secondary | 配对中的从属影像 | | 时间基线 | Temporal Baseline | 两景影像的时间间隔 | -| 空间基线 | Spatial Baseline | 两景影像的空间距离 | +| footprint 中心距 | Footprint Center Distance | 两景影像 footprint 的中心距离 | | 短基线子集 | SBAS (Small Baseline Subset) | 配对策略之一 | | 星型配对 | Star Graph | 单主影像配对策略 | | 顺序配对 | Sequential Pairing | 时间顺序配对策略 | diff --git a/frontend/src/DataCopierPanel.jsx b/frontend/src/DataCopierPanel.jsx index c8ac3f2..ef7c3d2 100644 --- a/frontend/src/DataCopierPanel.jsx +++ b/frontend/src/DataCopierPanel.jsx @@ -13,9 +13,11 @@ const BATCH_API_MAX_PAGES = 200; const DataCopierPanel = ({ apiEndpoint, readOnly = false, onJobQueued }) => { const { t } = useI18n(); - const [activeTab, setActiveTab] = useState('ps'); + const [activeTab, setActiveTab] = useState('dinsar'); const [destDir, setDestDir] = useState(''); const [copyStatuses, setCopyStatuses] = useState(['COMPLETED']); + const [includeDinsarOrbitFiles, setIncludeDinsarOrbitFiles] = useState(false); + const [dinsarExportZip, setDinsarExportZip] = useState(false); const [batches, setBatches] = useState([]); const [selectedBatchId, setSelectedBatchId] = useState(''); const [isUploading, setIsUploading] = useState(false); @@ -117,11 +119,16 @@ const DataCopierPanel = ({ apiEndpoint, readOnly = false, onJobQueued }) => { : `${apiEndpoint}/tools/copy-dinsar-pairs`; try { - const response = await axios.post(endpoint, { + const payload = { batch_id: selectedBatchId, dest_dir: destDir, copy_statuses: copyStatuses, - }, { withCredentials: true }); + }; + if (activeTab === 'dinsar') { + payload.include_orbit_files = includeDinsarOrbitFiles; + payload.export_zip = dinsarExportZip; + } + const response = await axios.post(endpoint, payload, { withCredentials: true }); const taskId = response.data.task_id; setTaskId(taskId); @@ -178,6 +185,42 @@ const DataCopierPanel = ({ apiEndpoint, readOnly = false, onJobQueued }) => { 当前账号为只读模式，无法发起复制任务。 )} + {activeTab === 'dinsar' && ( +
+ +
+ + +
+
+ 未勾选 ZIP 时直接导出 Task 文件夹；勾选后每个 Task 输出一个 .zip。 +
+
+ )}
diff --git a/frontend/src/components/DinsarCatalogPanel.jsx b/frontend/src/components/DinsarCatalogPanel.jsx index 707a83b..e565b51 100644 --- a/frontend/src/components/DinsarCatalogPanel.jsx +++ b/frontend/src/components/DinsarCatalogPanel.jsx @@ -476,7 +476,7 @@ export default function DinsarCatalogPanel({ - +
空间范围
@@ -521,7 +521,7 @@ export default function DinsarCatalogPanel({ - +
)} diff --git a/frontend/src/components/PairingModal.jsx b/frontend/src/components/PairingModal.jsx index ee5a2ae..fbb769b 100644 --- a/frontend/src/components/PairingModal.jsx +++ b/frontend/src/components/PairingModal.jsx @@ -11,23 +11,23 @@ const STRATEGY_DESCRIPTIONS = { title: '全部配对（默认）', description: '列出所有满足约束条件的候选干涉对，由用户自行筛选。', details: [ - '• 系统遍历所有影像组合，保留满足时间基线、空间基线和重叠率阈值的配对', + '• 系统遍历所有影像组合，保留满足时间基线、footprint 中心距和两景 footprint 最小重叠率的配对', '• 结果按时间排序，用户可在配对列表中逐一勾选或取消', '• 适用于研究型场景，需要精确控制每一对干涉组合', '• 配对数量可能较多，建议配合 AOI 和日期范围缩小结果' ], - params: '参数：时间基线范围、空间基线上限、最小重叠率' + params: '参数：时间基线范围、中心距上限、两景 footprint 最小重叠率' }, sbas: { title: 'SBAS (短基线子集)', description: '基于短基线原则的配对策略，通过覆盖优化算法自动筛选配对。', details: [ - '• 优先选择时间和空间基线都较短的配对', + '• 优先选择时间间隔和 footprint 中心距都较短的配对', '• 通过覆盖优化算法，去除冗余配对，确保时间序列连续性', '• 适用于大范围、长时间序列的形变监测', '• 配对数量会比"全部配对"少，但覆盖更均匀' ], - params: '参数：时间基线、空间基线、重叠率阈值、覆盖多样性惩罚' + params: '参数：时间基线、中心距、两景 footprint 最小重叠率、覆盖多样性惩罚' }, sequential: { title: 'Sequential (顺序配对)', @@ -321,24 +321,24 @@ function PairingModal({ )} - {/* 基线和重叠率约束 */} + {/* 时间、中心距和重叠率约束 */}
setPairingParams({...pairingParams, time_baseline_min: parseInt(e.target.value) || 0})} />
- + setPairingParams({...pairingParams, time_baseline_max: parseInt(e.target.value) || 90})} />
- + setPairingParams({...pairingParams, overlap_threshold: parseFloat(e.target.value) || 0})} />
- + setPairingParams({...pairingParams, spatial_baseline_max_meters: parseInt(e.target.value) || 3000})} />
diff --git a/frontend/src/components/panels/PairListRow.jsx b/frontend/src/components/panels/PairListRow.jsx index 482f2f6..723f9d7 100644 --- a/frontend/src/components/panels/PairListRow.jsx +++ b/frontend/src/components/panels/PairListRow.jsx @@ -7,6 +7,7 @@ function PairListRow({ onVisualizePair, onTogglePairVisibility, }) { + const centerDistance = pair.scene_center_distance_meters ?? pair.spatial_baseline_meters; return (
  • {pair.task_name}
    时基: {pair.time_baseline_days}d - 空基: {pair.spatial_baseline_meters.toFixed(2)}m + 中心距: {Number(centerDistance || 0).toFixed(2)}m