diff --git a/.env.example b/.env.example index 5c91a73..64acea0 100644 --- a/.env.example +++ b/.env.example @@ -55,9 +55,12 @@ ALLOWED_EXPORT_DIRS= # 源数据目录 # ----------------------------------------------------------------------------- UNPACK_SOURCE_DIRS=D:\Archives +SOURCE_PRODUCT_DIRS=D:\LuTan1_Image_Pool;D:\Sentinel1_Image_Pool_ZIP +SENTINEL1_STORAGE_DIRS=D:\Sentinel1_Image_Pool INSAR_STORAGE_DIRS=D:\LuTan1_Image_Pool MONITOR_RADAR_DIRS=D:\LuTan1_Image_Pool MONITOR_DINSAR_DIRS=D:\DInSARResult +ORBIT_SOURCE_DIRS=D:\LT1_data_lsarorbit;D:\Sentinel1_EOF_Pool MONITOR_ORBIT_DIR=D:\LT1_data_lsarorbit GF3_SOURCE_DIRS=D:\GF3_L1A_Image diff --git a/backend/app/config.py b/backend/app/config.py index 37d0c9f..f5a8ab0 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -167,6 +167,9 @@ class Settings(BaseSettings): DB_SCHEMA_RESET_CONFIRM: bool = False UNPACK_SOURCE_DIRS: str = "" + SOURCE_PRODUCT_DIRS: str = "" + SENTINEL1_STORAGE_DIRS: str = "" + ORBIT_SOURCE_DIRS: str = "" INSAR_STORAGE_DIRS: str = "" MONITOR_RADAR_DIRS: str = "" MONITOR_DINSAR_DIRS: str = "" @@ -859,6 +862,13 @@ def validate_runtime_config() -> dict[str, Any]: _check_path(label="SRTM_DEM_DIR", value=settings.SRTM_DEM_DIR, errors=errors, warnings=warnings, expect_file=False) _check_path(label="WATER_RESULTS_DIR", value=settings.WATER_RESULTS_DIR, errors=errors, warnings=warnings, expect_file=False) _check_path(label="MONITOR_ORBIT_DIR", value=settings.MONITOR_ORBIT_DIR, errors=errors, warnings=warnings, expect_file=False) + for label, value in ( + ("SOURCE_PRODUCT_DIRS", settings.SOURCE_PRODUCT_DIRS), + ("SENTINEL1_STORAGE_DIRS", settings.SENTINEL1_STORAGE_DIRS), + ("ORBIT_SOURCE_DIRS", settings.ORBIT_SOURCE_DIRS), + ): + for item in split_env_paths(value): + _check_path(label=label, value=item, errors=errors, warnings=warnings, expect_file=False) _check_path(label="ORBIT_POOL_ENVI", value=settings.ORBIT_POOL_ENVI, errors=errors, warnings=warnings, expect_file=False) _check_path(label="ORBIT_POOL_ISCE2", value=settings.ORBIT_POOL_ISCE2, errors=errors, warnings=warnings, expect_file=False) _check_path(label="RESULT_PUBLISH_ROOT", value=settings.RESULT_PUBLISH_ROOT, errors=errors, warnings=warnings, expect_file=False) diff --git a/backend/app/db_maintenance.py b/backend/app/db_maintenance.py index 4647b75..db97927 100644 --- a/backend/app/db_maintenance.py +++ b/backend/app/db_maintenance.py @@ -36,6 +36,7 @@ MIGRATION_FILES = [ "007_timeseries_stack_plan_trace.sql", "008_timeseries_stack_plan_edges.sql", "009_raw_source_pairing_fields.sql", + "010_source_orbit_asset_inventory.sql", ] diff --git a/backend/app/dinsar_engines/isce2_engine.py b/backend/app/dinsar_engines/isce2_engine.py index 831b733..3337091 100644 --- a/backend/app/dinsar_engines/isce2_engine.py +++ b/backend/app/dinsar_engines/isce2_engine.py @@ -10,6 +10,7 @@ from typing import Any, Dict, List from ..config import get_env_text, read_bool_env, settings from .base import DinsarEngine, EngineAvailability, EngineProfile, RunRequest, RunResult +from ..utils import normalize_satellite_family from ..services.dinsar_naming import ( PAIR_META_FILENAME, build_fallback_pair_key, @@ -728,7 +729,14 @@ class Isce2Engine(DinsarEngine): task_name = os.path.basename(os.path.normpath(task_dir)) pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {} task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name - pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(task_alias, task_dir) + satellite_family = normalize_satellite_family( + pair_meta.get("master_satellite") or pair_meta.get("slave_satellite") + ) + pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key( + task_alias, + task_dir, + satellite_family=satellite_family, + ) pointer_path = os.path.join( settings.DINSAR_PRODUCT_DIR, pair_key, @@ -1085,7 +1093,14 @@ class Isce2Engine(DinsarEngine): task_name = os.path.basename(os.path.normpath(task_dir)) pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {} task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name - pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(task_alias, task_dir) + satellite_family = normalize_satellite_family( + pair_meta.get("master_satellite") or pair_meta.get("slave_satellite") + ) + pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key( + task_alias, + task_dir, + satellite_family=satellite_family, + ) output_root = self._output_root or os.path.join(task_dir, "isce2_output") run_dir = managed_run_dir_override or os.path.normpath( os.path.join(output_root, pair_key, "runs", run_key) diff --git a/backend/app/dinsar_engines/pyint_engine.py b/backend/app/dinsar_engines/pyint_engine.py index 5b03d06..e3f8ae2 100644 --- a/backend/app/dinsar_engines/pyint_engine.py +++ b/backend/app/dinsar_engines/pyint_engine.py @@ -43,6 +43,7 @@ from ..services.pyint_service import ( REFLATTEN_MODEL_CHOICES, TARGET_GRID_SIZE_MAX_M, TARGET_GRID_SIZE_MIN_M, + build_profile_project_name, build_project_name, calculate_dem_oversampling, calculate_looks_from_task_dir, @@ -290,13 +291,13 @@ class PyintEngine(DinsarEngine): ) return _windows_path_to_wsl_mount(str(local_script)) + def _pipeline_script_for_profile(self, profile: str) -> str: + script_name = "run_s1_pyint_pipeline.py" if str(profile or "").strip() == "s1_gamma_dinsar" else "run_lt1_pyint_pipeline.py" + local_script = Path(__file__).resolve().parent.parent / "pyint_pipeline" / script_name + return _windows_path_to_wsl_mount(str(local_script)) + def get_profiles(self) -> List[EngineProfile]: - return [ - EngineProfile( - code="lt1_gamma_dinsar", - label="LT-1 Gamma D-InSAR", - description="Use PyINT + Gamma in WSL for single-pair LT-1 D-InSAR processing.", - params_schema={ + shared_schema = { "force": { "label": "强制重跑", "type": "boolean", @@ -459,7 +460,19 @@ class PyintEngine(DinsarEngine): "section": "Execution", "description": "关闭后不导出地理编码结果。", }, - }, + } + return [ + EngineProfile( + code="lt1_gamma_dinsar", + label="LT-1 Gamma D-InSAR", + description="Use PyINT + Gamma in WSL for single-pair LT-1 D-InSAR processing.", + params_schema=shared_schema, + ), + EngineProfile( + code="s1_gamma_dinsar", + label="Sentinel-1 Gamma D-InSAR", + description="Use PyINT + Gamma in WSL for single-pair Sentinel-1 D-InSAR processing.", + params_schema=shared_schema, ), ] @@ -694,7 +707,7 @@ class PyintEngine(DinsarEngine): error="PyINT is disabled.", ) - if request.profile != "lt1_gamma_dinsar": + if request.profile not in {"lt1_gamma_dinsar", "s1_gamma_dinsar"}: return RunResult( success=False, engine_code=self.engine_code, @@ -859,7 +872,11 @@ class PyintEngine(DinsarEngine): else os.path.join(run_dir, "native") ) template_root = os.path.normpath(os.path.join(self._template_root, pair_key, run_key)) - project_name = build_project_name(pair_key, run_key) + project_name = build_profile_project_name( + task_identity.get("satellite_family"), + pair_key, + run_key, + ) project_dir = os.path.join(work_run_root, project_name) # Keep input assets outside the run root because the WSL pipeline may delete run_root on --force. input_assets_dir = os.path.join(self._work_root, pair_key, "input_assets", run_key) @@ -1103,7 +1120,7 @@ class PyintEngine(DinsarEngine): ) cmd_parts = [ - f"{quote_shell(self._python)} {quote_shell(self._pipeline_script)} {quote_shell(wsl_task_dir)}", + f"{quote_shell(self._python)} {quote_shell(self._pipeline_script_for_profile(request.profile))} {quote_shell(wsl_task_dir)}", f"--project-dir {quote_shell(wsl_project_dir)}", f"--template-root {quote_shell(wsl_template_root)}", f"--output-dir {quote_shell(wsl_output_dir)}", @@ -1486,6 +1503,9 @@ class PyintEngine(DinsarEngine): @staticmethod def _discover_archives(task_dir: str) -> Dict[str, List[str]]: - from ..services.pyint_service import discover_lt1_archives + from ..services.pyint_service import discover_lt1_archives, discover_s1_scene_sources, infer_task_identity + task_identity = infer_task_identity(task_dir) + if str(task_identity.get("satellite_family") or "").strip().upper() == "S1": + return discover_s1_scene_sources(task_dir) return discover_lt1_archives(task_dir) diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index 1d1821d..2ea15ea 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -26,6 +26,12 @@ from .orm import ( ManagedRootORM, ScanCursorORM, PathInventoryORM, + SourceProductAssetORM, + OrbitAssetORM, + SceneOrbitBindingORM, + OrbitAssetDerivativeORM, + AssetInventoryStateORM, + AssetInventoryIssueORM, WorkflowDefORM, WorkflowRunORM, WorkflowStepORM, @@ -94,6 +100,8 @@ __all__ = [ "TimeseriesStackPlanORM", "TimeseriesStackPlanItemORM", "TimeseriesStackPlanEdgeORM", "SystemTaskORM", "TaskLogORM", "SystemJobORM", "ScanStateORM", "ManagedRootORM", "ScanCursorORM", "PathInventoryORM", + "SourceProductAssetORM", "OrbitAssetORM", "SceneOrbitBindingORM", + "OrbitAssetDerivativeORM", "AssetInventoryStateORM", "AssetInventoryIssueORM", "WorkflowDefORM", "WorkflowRunORM", "WorkflowStepORM", "WorkflowArtifactORM", "SystemWorkerHeartbeatORM", "DinsarTaskBatchORM", "DinsarTaskItemORM", diff --git a/backend/app/models/orm.py b/backend/app/models/orm.py index 7906c7f..984c508 100644 --- a/backend/app/models/orm.py +++ b/backend/app/models/orm.py @@ -41,6 +41,32 @@ class RadarDataORM(Base): product_unique_id = Column(String, nullable=True) satellite_family = Column(String, index=True, nullable=True) look_direction = Column(String, index=True, nullable=True) + acquisition_start_time_utc = Column(DateTime, nullable=True, index=True) + acquisition_stop_time_utc = Column(DateTime, nullable=True) + absolute_orbit = Column(String, nullable=True, index=True) + relative_orbit = Column(String, nullable=True, index=True) + source_format = Column(String(32), nullable=True, index=True) + source_product_ref_id = Column( + Integer, + ForeignKey("source_product_assets.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + source_archive_asset_id = Column( + Integer, + ForeignKey("source_product_assets.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + selected_orbit_asset_id = Column( + Integer, + ForeignKey("orbit_assets.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + orbit_binding_status = Column(String(32), nullable=False, default="UNBOUND", server_default="UNBOUND", index=True) + orbit_binding_reason = Column(Text, nullable=True) + metadata_json = Column(JSON, nullable=True) geocoded_flag = Column(Boolean, nullable=True) insar_source_ready = Column(Boolean, nullable=False, default=False, server_default="false") insar_source_reason = Column(Text, nullable=True) @@ -63,6 +89,10 @@ class RadarDataORM(Base): preview_cache_updated_at = Column(DateTime, nullable=True) preview_cache_error = Column(Text, nullable=True) + source_product_asset = relationship("SourceProductAssetORM", foreign_keys=[source_product_ref_id]) + source_archive_asset = relationship("SourceProductAssetORM", foreign_keys=[source_archive_asset_id]) + selected_orbit_asset = relationship("OrbitAssetORM", foreign_keys=[selected_orbit_asset_id]) + class DinsarResultORM(Base): __tablename__ = 'dinsar_results' @@ -742,6 +772,11 @@ class ManagedRootORM(Base): back_populates="root", cascade="all, delete-orphan", ) + asset_inventory_states = relationship( + "AssetInventoryStateORM", + back_populates="root", + cascade="all, delete-orphan", + ) __table_args__ = ( Index("idx_managed_roots_role_enabled", "root_role", "enabled"), @@ -809,6 +844,215 @@ class PathInventoryORM(Base): ) +class SourceProductAssetORM(Base): + __tablename__ = "source_product_assets" + + id = Column(Integer, primary_key=True, autoincrement=True) + asset_uid = Column(String(128), unique=True, index=True, nullable=False) + logical_product_uid = Column(String(128), index=True, nullable=True) + satellite_family = Column(String(32), index=True, nullable=True) + satellite = Column(String(32), index=True, nullable=True) + source_format = Column(String(32), index=True, nullable=False) + product_type = Column(String(64), nullable=True) + product_level = Column(String(64), nullable=True) + imaging_mode = Column(String(64), nullable=True) + polarization = Column(String(64), nullable=True) + absolute_orbit = Column(String(64), index=True, nullable=True) + relative_orbit = Column(String(64), index=True, nullable=True) + orbit_direction = Column(String(32), index=True, nullable=True) + acquisition_start_time_utc = Column(DateTime, index=True, nullable=True) + acquisition_stop_time_utc = Column(DateTime, nullable=True) + imaging_date = Column(String(8), index=True, nullable=True) + root_ref_id = Column(Integer, ForeignKey("managed_roots.id", ondelete="SET NULL"), index=True, nullable=True) + root_path = Column(String, nullable=True) + file_path = Column(String, unique=True, index=True, nullable=False) + archive_path = Column(String, nullable=True) + path_kind = Column(String(24), nullable=True) + file_name = Column(String(255), nullable=True) + file_stem = Column(String(255), nullable=True) + file_ext = Column(String(32), nullable=True) + size_bytes = Column(BigInteger, nullable=True) + mtime_epoch = Column(Float, nullable=True) + checksum_sha256 = Column(String(64), nullable=True) + checksum_status = Column(String(32), nullable=False, default="NOT_COMPUTED", server_default="NOT_COMPUTED") + parser_name = Column(String(64), nullable=True) + parser_version = Column(String(32), nullable=True) + parse_status = Column(String(32), nullable=False, default="PENDING", server_default="PENDING", index=True) + parse_error = Column(Text, nullable=True) + parsed_at = Column(DateTime, nullable=True) + metadata_json = Column(JSON, nullable=True) + is_active = Column(Boolean, nullable=False, default=True, server_default="true", index=True) + missing_since = Column(DateTime, nullable=True) + created_at = Column(DateTime, server_default=func.now(), nullable=False) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + + root = relationship("ManagedRootORM") + + __table_args__ = ( + Index("idx_source_product_assets_family_date", "satellite_family", "imaging_date"), + Index("idx_source_product_assets_logical_product", "logical_product_uid"), + Index("idx_source_product_assets_root_active", "root_ref_id", "is_active"), + ) + + +class OrbitAssetORM(Base): + __tablename__ = "orbit_assets" + + id = Column(Integer, primary_key=True, autoincrement=True) + orbit_uid = Column(String(128), unique=True, index=True, nullable=False) + satellite_family = Column(String(32), index=True, nullable=True) + satellite = Column(String(32), index=True, nullable=True) + orbit_type = Column(String(64), index=True, nullable=False) + native_format = Column(String(32), index=True, nullable=False) + quality_class = Column(String(32), index=True, nullable=False, default="unknown", server_default="unknown") + root_ref_id = Column(Integer, ForeignKey("managed_roots.id", ondelete="SET NULL"), index=True, nullable=True) + root_path = Column(String, nullable=True) + file_path = Column(String, unique=True, index=True, nullable=False) + file_name = Column(String(255), nullable=True) + file_stem = Column(String(255), nullable=True) + file_ext = Column(String(32), nullable=True) + size_bytes = Column(BigInteger, nullable=True) + mtime_epoch = Column(Float, nullable=True) + checksum_sha256 = Column(String(64), nullable=True) + checksum_status = Column(String(32), nullable=False, default="NOT_COMPUTED", server_default="NOT_COMPUTED") + validity_start_time_utc = Column(DateTime, index=True, nullable=True) + validity_stop_time_utc = Column(DateTime, index=True, nullable=True) + generation_time_utc = Column(DateTime, nullable=True) + published_time_utc = Column(DateTime, nullable=True) + parser_name = Column(String(64), nullable=True) + parser_version = Column(String(32), nullable=True) + parse_status = Column(String(32), nullable=False, default="PENDING", server_default="PENDING", index=True) + parse_error = Column(Text, nullable=True) + parsed_at = Column(DateTime, nullable=True) + metadata_json = Column(JSON, nullable=True) + is_active = Column(Boolean, nullable=False, default=True, server_default="true", index=True) + missing_since = Column(DateTime, nullable=True) + created_at = Column(DateTime, server_default=func.now(), nullable=False) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + + root = relationship("ManagedRootORM") + + __table_args__ = ( + Index("idx_orbit_assets_family_sat_window", "satellite_family", "satellite", "validity_start_time_utc", "validity_stop_time_utc"), + Index("idx_orbit_assets_root_active", "root_ref_id", "is_active"), + ) + + +class SceneOrbitBindingORM(Base): + __tablename__ = "scene_orbit_bindings" + + id = Column(Integer, primary_key=True, autoincrement=True) + radar_data_id = Column(Integer, ForeignKey("radar_data.id", ondelete="CASCADE"), index=True, nullable=False) + orbit_asset_id = Column(Integer, ForeignKey("orbit_assets.id", ondelete="CASCADE"), index=True, nullable=False) + binding_role = Column(String(32), nullable=False, default="primary_orbit", server_default="primary_orbit") + match_status = Column(String(32), nullable=False, default="CANDIDATE", server_default="CANDIDATE", index=True) + selection_status = Column(String(32), nullable=False, default="CANDIDATE", server_default="CANDIDATE", index=True) + selection_rank = Column(Integer, nullable=True) + priority_score = Column(Float, nullable=True) + coverage_margin_before_seconds = Column(Float, nullable=True) + coverage_margin_after_seconds = Column(Float, nullable=True) + match_rule_version = Column(String(64), nullable=True) + match_reason = Column(Text, nullable=True) + selected_at = Column(DateTime, nullable=True) + metadata_json = Column(JSON, nullable=True) + created_at = Column(DateTime, server_default=func.now(), nullable=False) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + + radar_data = relationship("RadarDataORM") + orbit_asset = relationship("OrbitAssetORM") + + __table_args__ = ( + UniqueConstraint("radar_data_id", "orbit_asset_id", "binding_role", name="uq_scene_orbit_binding_role"), + Index("idx_scene_orbit_bindings_scene_selected", "radar_data_id", "selection_status"), + ) + + +class OrbitAssetDerivativeORM(Base): + __tablename__ = "orbit_asset_derivatives" + + id = Column(Integer, primary_key=True, autoincrement=True) + orbit_asset_id = Column(Integer, ForeignKey("orbit_assets.id", ondelete="CASCADE"), index=True, nullable=False) + engine_code = Column(String(32), index=True, nullable=False) + derivative_format = Column(String(32), nullable=False) + derivative_role = Column(String(64), nullable=True) + pool_path = Column(String, index=True, nullable=False) + size_bytes = Column(BigInteger, nullable=True) + mtime_epoch = Column(Float, nullable=True) + checksum_sha256 = Column(String(64), nullable=True) + generation_status = Column(String(32), nullable=False, default="PENDING", server_default="PENDING", index=True) + generation_error = Column(Text, nullable=True) + generated_at = Column(DateTime, nullable=True) + metadata_json = Column(JSON, nullable=True) + created_at = Column(DateTime, server_default=func.now(), nullable=False) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + + orbit_asset = relationship("OrbitAssetORM") + + __table_args__ = ( + UniqueConstraint("orbit_asset_id", "engine_code", "derivative_format", "pool_path", name="uq_orbit_asset_derivative_pool_path"), + Index("idx_orbit_asset_derivatives_asset_engine", "orbit_asset_id", "engine_code"), + ) + + +class AssetInventoryStateORM(Base): + __tablename__ = "asset_inventory_states" + + id = Column(Integer, primary_key=True, autoincrement=True) + root_ref_id = Column(Integer, ForeignKey("managed_roots.id", ondelete="CASCADE"), index=True, nullable=False) + inventory_type = Column(String(32), index=True, nullable=False) + root_path = Column(String, nullable=False) + scan_mode = Column(String(32), nullable=False, default="file_pool", server_default="file_pool") + status = Column(String(32), nullable=False, default="NEVER_SCANNED", server_default="NEVER_SCANNED", index=True) + last_scan_started_at = Column(DateTime, nullable=True) + last_scan_finished_at = Column(DateTime, nullable=True) + last_seen_entry_count = Column(Integer, nullable=True) + last_asset_count = Column(Integer, nullable=True) + last_issue_count = Column(Integer, nullable=True) + parser_version = Column(String(32), nullable=True) + needs_rescan = Column(Boolean, nullable=False, default=True, server_default="true", index=True) + last_error = Column(Text, nullable=True) + metadata_json = Column(JSON, nullable=True) + created_at = Column(DateTime, server_default=func.now(), nullable=False) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + + root = relationship("ManagedRootORM", back_populates="asset_inventory_states") + + __table_args__ = ( + UniqueConstraint("root_ref_id", "inventory_type", name="uq_asset_inventory_state_root_type"), + Index("idx_asset_inventory_states_type_status", "inventory_type", "status"), + ) + + +class AssetInventoryIssueORM(Base): + __tablename__ = "asset_inventory_issues" + + id = Column(Integer, primary_key=True, autoincrement=True) + root_ref_id = Column(Integer, ForeignKey("managed_roots.id", ondelete="SET NULL"), index=True, nullable=True) + inventory_type = Column(String(32), index=True, nullable=False) + asset_ref_id = Column(Integer, ForeignKey("source_product_assets.id", ondelete="SET NULL"), index=True, nullable=True) + radar_data_id = Column(Integer, ForeignKey("radar_data.id", ondelete="SET NULL"), index=True, nullable=True) + orbit_asset_id = Column(Integer, ForeignKey("orbit_assets.id", ondelete="SET NULL"), index=True, nullable=True) + severity = Column(String(16), nullable=False, default="warning", server_default="warning", index=True) + issue_code = Column(String(64), index=True, nullable=False) + issue_message = Column(Text, nullable=True) + source_path = Column(String, nullable=True) + status = Column(String(16), nullable=False, default="OPEN", server_default="OPEN", index=True) + first_seen_at = Column(DateTime, server_default=func.now(), nullable=False) + last_seen_at = Column(DateTime, server_default=func.now(), nullable=False) + resolved_at = Column(DateTime, nullable=True) + metadata_json = Column(JSON, nullable=True) + + root = relationship("ManagedRootORM") + source_asset = relationship("SourceProductAssetORM", foreign_keys=[asset_ref_id]) + radar_data = relationship("RadarDataORM") + orbit_asset = relationship("OrbitAssetORM") + + __table_args__ = ( + Index("idx_asset_inventory_issues_open", "status", "severity"), + Index("idx_asset_inventory_issues_root_type", "root_ref_id", "inventory_type"), + ) + + class WorkflowDefORM(Base): """Workflow definition (DAG template).""" __tablename__ = "workflow_defs" diff --git a/backend/app/models/schemas.py b/backend/app/models/schemas.py index 35d4179..ac8a51a 100644 --- a/backend/app/models/schemas.py +++ b/backend/app/models/schemas.py @@ -192,6 +192,17 @@ class RadarData(BaseModel): product_unique_id: Optional[str] = None satellite_family: Optional[str] = None look_direction: Optional[str] = None + acquisition_start_time_utc: Optional[datetime] = None + acquisition_stop_time_utc: Optional[datetime] = None + absolute_orbit: Optional[str] = None + relative_orbit: Optional[str] = None + source_format: Optional[str] = None + source_product_ref_id: Optional[int] = None + source_archive_asset_id: Optional[int] = None + selected_orbit_asset_id: Optional[int] = None + orbit_binding_status: str = "UNBOUND" + orbit_binding_reason: Optional[str] = None + metadata_json: Optional[Dict[str, Any]] = None geocoded_flag: Optional[bool] = None insar_source_ready: bool = False insar_source_reason: Optional[str] = None diff --git a/backend/app/pyint_pipeline/run_s1_pyint_pipeline.py b/backend/app/pyint_pipeline/run_s1_pyint_pipeline.py new file mode 100644 index 0000000..78c376e --- /dev/null +++ b/backend/app/pyint_pipeline/run_s1_pyint_pipeline.py @@ -0,0 +1,651 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +import math +import os +import re +import shutil +import sys +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List + +try: + from .run_lt1_pyint_pipeline import ( + DEFAULT_DERAMP_MODE, + DEFAULT_REFERENCE_MODE, + DEFAULT_REFLATTEN_AZIMUTH_STEP, + DEFAULT_REFLATTEN_COH_THRESHOLD, + DEFAULT_REFLATTEN_FALLBACK_COH_THRESHOLD, + DEFAULT_REFLATTEN_MODEL, + DEFAULT_REFLATTEN_RANGE_STEP, + assert_output_sanity, + assert_required_outputs, + calculate_dem_oversampling, + collect_expected_outputs, + collect_output_sanity_checks, + collect_stage_error_logs, + copy_native_outputs, + ensure_directory, + export_standard_products, + format_gamma_number, + hardlink_or_copy, + inspect_prepared_dem_path, + load_json_file, + load_pair_meta, + load_shell_environment, + normalize_date_text, + parse_args, + require_task_layout, + rerun_pair_product_stages, + run_gamma_reflatten, + run_logged, + safe_rmtree, + validate_unit_interval, + write_ifgram_list, + write_text, + write_wrapper_scripts, + ) +except ImportError: + from run_lt1_pyint_pipeline import ( + DEFAULT_DERAMP_MODE, + DEFAULT_REFERENCE_MODE, + DEFAULT_REFLATTEN_AZIMUTH_STEP, + DEFAULT_REFLATTEN_COH_THRESHOLD, + DEFAULT_REFLATTEN_FALLBACK_COH_THRESHOLD, + DEFAULT_REFLATTEN_MODEL, + DEFAULT_REFLATTEN_RANGE_STEP, + assert_output_sanity, + assert_required_outputs, + calculate_dem_oversampling, + collect_expected_outputs, + collect_output_sanity_checks, + collect_stage_error_logs, + copy_native_outputs, + ensure_directory, + export_standard_products, + format_gamma_number, + hardlink_or_copy, + inspect_prepared_dem_path, + load_json_file, + load_pair_meta, + load_shell_environment, + normalize_date_text, + parse_args, + require_task_layout, + rerun_pair_product_stages, + run_gamma_reflatten, + run_logged, + safe_rmtree, + validate_unit_interval, + write_ifgram_list, + write_text, + write_wrapper_scripts, + ) + + +_WINDOWS_DRIVE_RE = re.compile(r"^[A-Za-z]:[\\/]") + + +def _normalize_runtime_path(value: Any) -> Path: + text = str(value or "").strip() + if not text: + return Path() + normalized = text.replace("\\", "/") + if normalized.startswith("/mnt/") or normalized.startswith("/"): + return Path(normalized).resolve() + if _WINDOWS_DRIVE_RE.match(text): + drive = text[0].lower() + tail = text[2:].replace("\\", "/").lstrip("/") + return Path(f"/mnt/{drive}/{tail}").resolve() + return Path(normalized).resolve() + + +def _build_s1_template_text( + *, + project_name: str, + satellite: str, + master_date: str, + range_looks: int, + azimuth_looks: int, + target_grid_size_m: int, + dem_lat_ovr: float, + dem_lon_ovr: float, + unwrap_coh_threshold: float, + geo_interp: str, + atmcor: bool, + atmcor_use_for_disp: bool, + reflatten: bool, + reflatten_model: str, + reflatten_coh_threshold: float, + parallel_workers: int, + unwrap: bool, + geocode: bool, + dem_mode: str, + fabdem_root: str, + prepared_dem_path: str, + opentopo_dem_type: str, + opentopo_api_key: str, +) -> str: + prepared_dem = inspect_prepared_dem_path(prepared_dem_path) if dem_mode == "prepared_file" else {} + lines = [ + f"# Auto-generated for {project_name}", + f"satelite={satellite}", + f"masterDate={master_date}", + f"range_looks={int(range_looks)}", + f"azimuth_looks={int(azimuth_looks)}", + f"target_grid_size_m={int(target_grid_size_m or 0)}", + f"dem_lat_ovr={format_gamma_number(dem_lat_ovr)}", + f"dem_lon_ovr={format_gamma_number(dem_lon_ovr)}", + "download_data=0", + "raw2slc_all=1", + f"raw2slc_all_parallel={int(parallel_workers)}", + "extract_burst_all=1", + f"extract_all_parallel={int(parallel_workers)}", + "coreg_all=1", + f"coreg_all_parallel={int(parallel_workers)}", + "select_pairs=0", + "diff_all=1", + f"diff_all_parallel={int(parallel_workers)}", + "pot_all=0", + f"pot_all_parallel={int(parallel_workers)}", + f"unwrap_all={1 if unwrap else 0}", + f"unwrap_all_parallel={int(parallel_workers)}", + f"unwrapThreshold={format_gamma_number(unwrap_coh_threshold)}", + "make_mask=1", + "auto_unw=1", + "r_refer=-", + "a_refer=-", + f"atmcor_all={1 if atmcor else 0}", + f"atmcor_all_parallel={int(parallel_workers)}", + f"atmcor_use_for_disp={1 if (atmcor and atmcor_use_for_disp) else 0}", + f"reflatten={1 if reflatten else 0}", + f"reflatten_model={str(reflatten_model or DEFAULT_REFLATTEN_MODEL).strip().lower()}", + f"reflatten_coh_threshold={format_gamma_number(reflatten_coh_threshold)}", + f"geocode_all={1 if geocode else 0}", + f"geocode_all_parallel={int(parallel_workers)}", + f"geo_interp={str(geo_interp or '0').strip()}", + "gacos_correction=0", + "load_data=0", + "geocode_products=hyp3,licsbas", + "start_swath=1", + "end_swath=3", + "start_burst=1", + "end_burst=20", + ] + if dem_mode == "local_fabdem" and fabdem_root: + lines.append(f"fabdem_dir={fabdem_root}") + else: + lines.append("fabdem_dir=-") + if dem_mode == "prepared_file" and prepared_dem.get("kind") == "gamma_ready": + lines.append(f"DEM={prepared_dem['direct_dem_path']}") + if dem_mode == "prepared_file" and prepared_dem.get("kind") == "source_dem": + lines.append(f"prepared_dem_source={prepared_dem['source_dem_path']}") + else: + lines.append("prepared_dem_source=-") + if dem_mode == "opentopo": + lines.append(f"opentopo_dem_type={opentopo_dem_type or 'SRTMGL1'}") + lines.append(f"opentopo_api_key={opentopo_api_key or '-'}") + else: + lines.append("opentopo_dem_type=-") + lines.append("opentopo_api_key=-") + return "\n".join(lines) + "\n" + + +def _resolve_s1_inputs(input_assets_payload: Dict[str, Any]) -> Dict[str, Any]: + task_source = input_assets_payload.get("task_source") or {} + prod = task_source.get("production_inputs") or {} + orbits = input_assets_payload.get("orbits") or {} + master_scene = _normalize_runtime_path( + ((prod.get("master_scene") or {}).get("staged_path") or (prod.get("master_scene") or {}).get("path") or (prod.get("master_zip") or {}).get("staged_path") or (prod.get("master_zip") or {}).get("path") or "") + ) + slave_scene = _normalize_runtime_path( + ((prod.get("slave_scene") or {}).get("staged_path") or (prod.get("slave_scene") or {}).get("path") or (prod.get("slave_zip") or {}).get("staged_path") or (prod.get("slave_zip") or {}).get("path") or "") + ) + master_eof = _normalize_runtime_path( + ((orbits.get("master") or {}).get("staged_path") or (orbits.get("master") or {}).get("path") or "") + ) + slave_eof = _normalize_runtime_path( + ((orbits.get("slave") or {}).get("staged_path") or (orbits.get("slave") or {}).get("path") or "") + ) + return { + "master_scene": master_scene, + "slave_scene": slave_scene, + "master_eof": master_eof, + "slave_eof": slave_eof, + } + + +def main() -> int: + args = parse_args() + + task_dir = Path(args.task_dir).resolve() + project_dir = Path(args.project_dir).resolve() + run_root = project_dir.parent + template_root = Path(args.template_root).resolve() + output_dir = Path(args.output_dir).resolve() + pyint_home = Path(args.pyint_home).resolve() + pyint_app_script = Path(args.pyint_app_script).resolve() + dem_root = Path(args.dem_root).resolve() + input_assets_dir = Path(args.input_assets_dir).resolve() if args.input_assets_dir else None + input_assets_json = Path(args.input_assets_json).resolve() if args.input_assets_json else None + input_assets_payload = load_json_file(input_assets_json) + dem_mode = str(args.dem_mode or "local_fabdem").strip().lower() or "local_fabdem" + prepared_dem_info = inspect_prepared_dem_path(args.prepared_dem_path) if dem_mode == "prepared_file" else {} + dem_oversampling = calculate_dem_oversampling( + dem_resolution_m=float(args.dem_resolution_m), + target_grid_size_m=float(args.target_grid_size_m or 0), + dem_lat_ovr=float(args.dem_lat_ovr or 0.0), + dem_lon_ovr=float(args.dem_lon_ovr or 0.0), + ) + unwrap_coh_threshold = validate_unit_interval(args.unwrap_coh_threshold, "--unwrap-coh-threshold") + coherence_mask_threshold = validate_unit_interval(args.coherence_mask_threshold, "--coherence-mask-threshold") + reference_mode = str(args.reference_mode or DEFAULT_REFERENCE_MODE).strip().lower() or DEFAULT_REFERENCE_MODE + deramp_mode = str(args.deramp_mode or DEFAULT_DERAMP_MODE).strip().lower() or DEFAULT_DERAMP_MODE + reference_coh_threshold = validate_unit_interval(args.reference_coh_threshold, "--reference-coh-threshold") + deramp_coh_threshold = validate_unit_interval(args.deramp_coh_threshold, "--deramp-coh-threshold") + geo_interp = str(args.geo_interp or "1").strip() + if geo_interp not in {"0", "1"}: + raise ValueError("--geo-interp must be 0 or 1.") + gamma_nodata_value = float(args.gamma_nodata_value) + if not math.isfinite(gamma_nodata_value): + raise ValueError("--gamma-nodata-value must be a finite number.") + reflatten_model = str(args.reflatten_model or DEFAULT_REFLATTEN_MODEL).strip().lower() + if reflatten_model == "linear": + reflatten_model = "plane" + if reflatten_model not in {"plane", "quadratic"}: + raise ValueError("--reflatten-model must be plane or quadratic.") + reflatten_coh_threshold = validate_unit_interval(args.reflatten_coh_threshold, "--reflatten-coh-threshold") + reflatten_fallback_coh_threshold = validate_unit_interval( + args.reflatten_fallback_coh_threshold, + "--reflatten-fallback-coh-threshold", + ) + reflatten_range_step = max(1, int(args.reflatten_range_step or DEFAULT_REFLATTEN_RANGE_STEP)) + reflatten_azimuth_step = max(1, int(args.reflatten_azimuth_step or DEFAULT_REFLATTEN_AZIMUTH_STEP)) + + require_task_layout(task_dir) + if not pyint_app_script.is_file(): + raise FileNotFoundError(f"pyintApp.py not found: {pyint_app_script}") + if not input_assets_json or not input_assets_json.is_file(): + raise RuntimeError("Sentinel-1 PyINT runner requires --input-assets-json.") + if dem_mode == "prepared_file" and not prepared_dem_info.get("kind"): + raise RuntimeError( + "Prepared DEM mode requires either a Gamma DEM with .par, " + "or a source DEM with .xml/.hdr/.vrt sidecars." + ) + + if args.force: + safe_rmtree(run_root) + safe_rmtree(template_root) + safe_rmtree(output_dir) + + if run_root.exists(): + raise RuntimeError(f"PyINT run root already exists, rerun with --force: {run_root}") + + pair_meta = load_pair_meta(task_dir) + s1_inputs = _resolve_s1_inputs(input_assets_payload) + master_scene = s1_inputs["master_scene"] + slave_scene = s1_inputs["slave_scene"] + master_eof = s1_inputs["master_eof"] + slave_eof = s1_inputs["slave_eof"] + if not (master_scene.is_file() or master_scene.is_dir()): + raise FileNotFoundError(f"Sentinel-1 master scene not found: {master_scene}") + if not (slave_scene.is_file() or slave_scene.is_dir()): + raise FileNotFoundError(f"Sentinel-1 slave scene not found: {slave_scene}") + if not master_eof.is_file(): + raise FileNotFoundError(f"Sentinel-1 master EOF not found: {master_eof}") + if not slave_eof.is_file(): + raise FileNotFoundError(f"Sentinel-1 slave EOF not found: {slave_eof}") + + master_date = normalize_date_text(args.master_date) or normalize_date_text(pair_meta.get("master_imaging_date")) + slave_date = normalize_date_text(args.slave_date) or normalize_date_text(pair_meta.get("slave_imaging_date")) + if not master_date or not slave_date: + raise RuntimeError("Unable to determine master/slave dates from pair metadata.") + + pair_name = f"{master_date}-{slave_date}" + task_alias = str(args.task_alias or pair_meta.get("task_alias") or task_dir.name).strip() or task_dir.name + pair_key = str(args.pair_key or pair_meta.get("pair_key") or "").strip() + time_baseline_days = int(args.time_baseline_days or pair_meta.get("time_baseline_days") or 0) + satellite = str(pair_meta.get("master_satellite") or "S1A").strip().upper() or "S1A" + + ensure_directory(run_root) + ensure_directory(template_root) + ensure_directory(output_dir) + ensure_directory(dem_root) + + pyint_scripts_dir = pyint_home / "pyint" + wrappers_dir = ensure_directory(run_root / "wrappers") + write_wrapper_scripts( + wrappers_dir=wrappers_dir, + pyint_home=pyint_home, + python_cmd=args.python, + gamma_env_script=args.gamma_env_script, + ) + for shim_name in ("python", "python3"): + shim_path = wrappers_dir / shim_name + write_text( + shim_path, + "\n".join( + [ + "#!/usr/bin/env bash", + f"exec '{args.python}' \"$@\"", + "", + ] + ), + ) + shim_path.chmod(shim_path.stat().st_mode | 0o111) + + template_path = write_text( + template_root / f"{args.project_name}.template", + _build_s1_template_text( + project_name=args.project_name, + satellite=satellite, + master_date=master_date, + range_looks=args.range_looks, + azimuth_looks=args.azimuth_looks, + target_grid_size_m=int(args.target_grid_size_m or 0), + dem_lat_ovr=dem_oversampling["dem_lat_ovr"], + dem_lon_ovr=dem_oversampling["dem_lon_ovr"], + unwrap_coh_threshold=unwrap_coh_threshold, + geo_interp=geo_interp, + atmcor=bool(args.atmcor), + atmcor_use_for_disp=bool(args.atmcor_use_for_disp), + reflatten=bool(args.reflatten), + reflatten_model=reflatten_model, + reflatten_coh_threshold=reflatten_coh_threshold, + parallel_workers=args.parallel_workers, + unwrap=bool(args.unwrap), + geocode=bool(args.geocode), + dem_mode=dem_mode, + fabdem_root=str(args.fabdem_root or "").strip(), + prepared_dem_path=str(args.prepared_dem_path or "").strip(), + opentopo_dem_type=str(args.opentopo_dem_type or "SRTMGL1").strip(), + opentopo_api_key=str(args.opentopo_api_key or "").strip(), + ), + ) + + scratch_root = ensure_directory(project_dir.parent) + archive_materialization: List[Dict[str, str]] = [] + env = os.environ.copy() + if args.gamma_env_script: + env.update(load_shell_environment(args.gamma_env_script, env)) + opod_dir = project_dir / "OPOD" + env.update( + { + "SCRATCHDIR": str(scratch_root), + "TEMPLATEDIR": str(template_root), + "DEMDIR": str(dem_root), + "OPOD_DIR": str(opod_dir), + "PATH": f"{wrappers_dir}:{pyint_scripts_dir}:{env.get('PATH', '')}", + "PYTHONPATH": f"{pyint_home}:{env.get('PYTHONPATH', '')}", + "PYINT_LT1_PRECISE_ORBIT_ENABLED": "false", + } + ) + + generate_stdout = run_root / "pyint_generate.stdout.log" + generate_stderr = run_root / "pyint_generate.stderr.log" + generate_result = run_logged( + [str(wrappers_dir / "pyintApp.py"), "-g", args.project_name], + env=env, + cwd=scratch_root, + stdout_path=generate_stdout, + stderr_path=generate_stderr, + ) + if generate_result.returncode != 0: + raise RuntimeError( + f"pyintApp.py -g failed with rc={generate_result.returncode}: " + f"{(generate_result.stderr or generate_result.stdout or '').strip()}" + ) + + pyint_project_dir = project_dir + ensure_directory(pyint_project_dir) + opod_dir = ensure_directory(opod_dir) + download_dir = ensure_directory(pyint_project_dir / "DOWNLOAD") + ifgram_list_path = write_ifgram_list(pyint_project_dir / "ifgram_list.txt", master_date, slave_date, time_baseline_days) + for role, src_path in (("master", master_scene), ("slave", slave_scene)): + target_path = download_dir / src_path.name + if src_path.is_dir(): + if target_path.exists(): + if target_path.is_dir(): + safe_rmtree(target_path) + else: + target_path.unlink() + shutil.copytree(src_path, target_path) + op = "copied_dir" + else: + op = hardlink_or_copy(src_path, target_path) + archive_materialization.append( + { + "role": role, + "source": str(src_path), + "target": str(target_path), + "operation": op, + } + ) + for role, src_path in (("master_orbit", master_eof), ("slave_orbit", slave_eof)): + target_path = opod_dir / src_path.name + op = hardlink_or_copy(src_path, target_path) + archive_materialization.append( + { + "role": role, + "source": str(src_path), + "target": str(target_path), + "operation": op, + } + ) + + run_stdout = run_root / "pyint.stdout.log" + run_stderr = run_root / "pyint.stderr.log" + run_started_at = datetime.utcnow().isoformat(timespec="seconds") + "Z" + run_result = run_logged( + [str(wrappers_dir / "pyintApp.py"), args.project_name], + env=env, + cwd=scratch_root, + stdout_path=run_stdout, + stderr_path=run_stderr, + ) + if run_result.returncode != 0: + stage_error_logs = collect_stage_error_logs(pyint_project_dir) + detail_text = (run_result.stderr or run_result.stdout or "").strip() + if stage_error_logs: + log_text = ", ".join(f"{name}={path}" for name, path in stage_error_logs.items()) + detail_text = f"{detail_text}\nStage logs: {log_text}" if detail_text else f"Stage logs: {log_text}" + raise RuntimeError(f"pyintApp.py failed with rc={run_result.returncode}: {detail_text}") + + repair_summary: Dict[str, Any] = { + "attempted": False, + "attempt_count": 0, + "max_attempts": 1, + } + outputs = collect_expected_outputs(pyint_project_dir, pair_name, args.range_looks) + try: + assert_required_outputs(outputs, unwrap=bool(args.unwrap), geocode=bool(args.geocode)) + output_sanity_checks = collect_output_sanity_checks(outputs, unwrap=bool(args.unwrap), geocode=bool(args.geocode)) + assert_output_sanity(output_sanity_checks) + except RuntimeError as exc: + repair_summary = rerun_pair_product_stages( + project_name=args.project_name, + project_dir=pyint_project_dir, + run_root=run_root, + scratch_root=scratch_root, + env=env, + pair_name=pair_name, + master_date=master_date, + slave_date=slave_date, + range_looks=args.range_looks, + unwrap=bool(args.unwrap), + atmcor=bool(args.atmcor), + geocode=bool(args.geocode), + ) + outputs = collect_expected_outputs(pyint_project_dir, pair_name, args.range_looks) + assert_required_outputs(outputs, unwrap=bool(args.unwrap), geocode=bool(args.geocode)) + output_sanity_checks = collect_output_sanity_checks(outputs, unwrap=bool(args.unwrap), geocode=bool(args.geocode)) + assert_output_sanity(output_sanity_checks) + + stage_error_logs = collect_stage_error_logs(pyint_project_dir) + reflatten_summary: Dict[str, Any] = { + "enabled": bool(args.reflatten), + "applied": False, + "reason": "", + "model": reflatten_model, + "coherence_threshold": reflatten_coh_threshold, + "fallback_coherence_threshold": reflatten_fallback_coh_threshold, + "range_step": reflatten_range_step, + "azimuth_step": reflatten_azimuth_step, + } + if bool(args.reflatten) and bool(args.unwrap): + reflatten_summary = run_gamma_reflatten( + project_dir=pyint_project_dir, + run_root=run_root, + output_dir=output_dir, + outputs=outputs, + pair_name=pair_name, + master_date=master_date, + range_looks=args.range_looks, + env=env, + model=reflatten_model, + coherence_threshold=reflatten_coh_threshold, + fallback_coherence_threshold=reflatten_fallback_coh_threshold, + range_step=reflatten_range_step, + azimuth_step=reflatten_azimuth_step, + geo_interp=geo_interp, + ) + outputs = collect_expected_outputs(pyint_project_dir, pair_name, args.range_looks) + output_sanity_checks = collect_output_sanity_checks(outputs, unwrap=bool(args.unwrap), geocode=bool(args.geocode)) + assert_output_sanity(output_sanity_checks) + elif bool(args.reflatten): + reflatten_summary["reason"] = "unwrap disabled" + else: + reflatten_summary["reason"] = "disabled" + + copied_paths = copy_native_outputs( + project_dir=pyint_project_dir, + output_dir=output_dir, + pair_name=pair_name, + template_path=template_path, + ifgram_list_path=ifgram_list_path, + stdout_path=run_stdout, + stderr_path=run_stderr, + ) + standard_products = ( + export_standard_products( + project_dir=pyint_project_dir, + output_dir=output_dir, + pair_name=pair_name, + master_date=master_date, + range_looks=args.range_looks, + azimuth_looks=args.azimuth_looks, + target_grid_size_m=int(args.target_grid_size_m or 0), + coherence_mask_threshold=coherence_mask_threshold, + reference_mode=reference_mode, + reference_coh_threshold=reference_coh_threshold, + deramp_mode=deramp_mode, + deramp_coh_threshold=deramp_coh_threshold, + atmcor_enabled=bool(args.atmcor), + atmcor_use_for_disp=bool(args.atmcor_use_for_disp), + reflatten_summary=reflatten_summary, + gamma_nodata_value=gamma_nodata_value, + outputs=outputs, + env=env, + run_root=run_root, + ) + if bool(args.geocode) + else {"enabled": False, "reason": "geocode disabled"} + ) + + summary = { + "ok": True, + "task_dir": str(task_dir), + "task_alias": task_alias, + "pair_key": pair_key, + "project_name": args.project_name, + "project_dir": str(pyint_project_dir), + "run_root": str(run_root), + "template_root": str(template_root), + "output_dir": str(output_dir), + "pyint_home": str(pyint_home), + "pyint_app_script": str(pyint_app_script), + "gamma_env_script": args.gamma_env_script, + "dem": { + "mode": dem_mode, + "dem_root": str(dem_root), + "fabdem_root": str(args.fabdem_root or "").strip(), + "prepared_dem_path": str(args.prepared_dem_path or "").strip(), + "prepared_dem_kind": str(prepared_dem_info.get("kind") or ""), + "configured_resolution_m": float(args.dem_resolution_m), + "oversampling": dem_oversampling, + "opentopo_dem_type": str(args.opentopo_dem_type or "SRTMGL1").strip(), + "opentopo_api_key_configured": bool(str(args.opentopo_api_key or "").strip()), + }, + "orbit_policy": "require_eof", + "precise_orbit_bridge": {"enabled": False, "mode": "not_applicable"}, + "input_assets_dir": str(input_assets_dir) if input_assets_dir else "", + "input_assets_json": str(input_assets_json) if input_assets_json else "", + "input_assets": input_assets_payload, + "master_date": master_date, + "slave_date": slave_date, + "pair_name": pair_name, + "time_baseline_days": time_baseline_days, + "target_grid_size_m": int(args.target_grid_size_m or 0), + "range_looks": int(args.range_looks), + "azimuth_looks": int(args.azimuth_looks), + "dem_resolution_m": float(args.dem_resolution_m), + "dem_oversampling": dem_oversampling, + "unwrap_coh_threshold": unwrap_coh_threshold, + "coherence_quality_threshold": coherence_mask_threshold, + "reference_mode": reference_mode, + "reference_coh_threshold": reference_coh_threshold, + "deramp_mode": deramp_mode, + "deramp_coh_threshold": deramp_coh_threshold, + "gamma_nodata_value": gamma_nodata_value, + "geo_interp": geo_interp, + "atmcor": bool(args.atmcor), + "atmcor_use_for_disp": bool(args.atmcor_use_for_disp), + "reflatten": bool(args.reflatten), + "reflatten_model": reflatten_model, + "reflatten_coh_threshold": reflatten_coh_threshold, + "reflatten_fallback_coh_threshold": reflatten_fallback_coh_threshold, + "reflatten_range_step": reflatten_range_step, + "reflatten_azimuth_step": reflatten_azimuth_step, + "parallel_workers": int(args.parallel_workers), + "unwrap": bool(args.unwrap), + "geocode": bool(args.geocode), + "archives": { + "master": [str(master_scene)], + "slave": [str(slave_scene)], + }, + "orbit_files": { + "master": str(master_eof), + "slave": str(slave_eof), + }, + "archive_materialization": archive_materialization, + "workspace_outputs": outputs, + "output_sanity_checks": output_sanity_checks, + "output_repair": repair_summary, + "reflatten_summary": reflatten_summary, + "copied_outputs": copied_paths, + "standard_products": standard_products, + "logs": { + "generate_stdout": str(generate_stdout), + "generate_stderr": str(generate_stderr), + "run_stdout": str(run_stdout), + "run_stderr": str(run_stderr), + "stage_error_logs": stage_error_logs, + }, + "started_at": run_started_at, + "finished_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + } + + summary_path = output_dir / "pyint_run_summary.json" + write_text(summary_path, json.dumps(summary, ensure_ascii=True, indent=2) + "\n") + print(json.dumps(summary, ensure_ascii=True, indent=2)) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as exc: + print(str(exc), file=sys.stderr) + raise diff --git a/backend/app/routers/__init__.py b/backend/app/routers/__init__.py index 62f4759..1574eed 100644 --- a/backend/app/routers/__init__.py +++ b/backend/app/routers/__init__.py @@ -5,6 +5,7 @@ from fastapi import APIRouter from . import ( ai, aoi, + assets, auth, dinsar, dinsar_products, @@ -43,6 +44,7 @@ def include_all_routers(router: APIRouter) -> None: router.include_router(unpack.router) router.include_router(monitor.router) router.include_router(orbit.router) + router.include_router(assets.router) router.include_router(root_registry.router) router.include_router(radar.router) router.include_router(aoi.router) diff --git a/backend/app/routers/assets.py b/backend/app/routers/assets.py new file mode 100644 index 0000000..84a9742 --- /dev/null +++ b/backend/app/routers/assets.py @@ -0,0 +1,222 @@ +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from fastapi import APIRouter, Depends, HTTPException, Query +from pydantic import BaseModel, Field +from sqlalchemy.ext.asyncio import AsyncSession + +from ..database import get_db +from ..models import AuthUserORM +from ..models.orm import SourceProductAssetORM +from ..services.asset_inventory_service import asset_inventory_service +from ..services.job_queue_service import job_queue_service +from ..services.task_service import task_service +from .dependencies import _get_current_user, _require_admin + + +router = APIRouter(prefix="/assets", tags=["assets"]) + + +class AssetScanRequest(BaseModel): + inventory_types: List[str] = Field(default_factory=list) + root_ids: List[int] = Field(default_factory=list) + bind_orbits: bool = True + + +class S1UnpackRequest(BaseModel): + target_root: Optional[str] = None + overwrite: bool = False + min_disk_space_gb: Optional[float] = Field(default=None, ge=0) + delete_archive: Optional[bool] = None + + +class S1BatchUnpackRequest(BaseModel): + target_root: Optional[str] = None + overwrite: bool = False + min_disk_space_gb: Optional[float] = Field(default=None, ge=0) + delete_archive: Optional[bool] = None + scan_before_unpack: bool = True + + +@router.get("/inventory/status") +async def get_asset_inventory_status( + current_user: AuthUserORM = Depends(_get_current_user), + db: AsyncSession = Depends(get_db), +): + _ = current_user + return await asset_inventory_service.get_status(db) + + +@router.post("/inventory/scan", status_code=202) +async def run_asset_inventory_scan( + request: Optional[AssetScanRequest] = None, + admin_user: AuthUserORM = Depends(_require_admin), +): + _ = admin_user + payload = (request or AssetScanRequest()).model_dump() + try: + task_id = await task_service.create_task( + "SCAN_ASSET_INVENTORY", + "Source/orbit asset inventory scan", + params=payload, + ) + job_id = await job_queue_service.create_job( + "SCAN_ASSET_INVENTORY", + payload=payload, + task_id=task_id, + ) + return {"message": "Asset inventory scan queued", "task_id": task_id, "job_id": job_id} + except ValueError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + + +@router.post("/inventory/scan-now") +async def run_asset_inventory_scan_now( + request: Optional[AssetScanRequest] = None, + admin_user: AuthUserORM = Depends(_require_admin), + db: AsyncSession = Depends(get_db), +): + _ = admin_user + payload = (request or AssetScanRequest()).model_dump() + return await asset_inventory_service.scan_configured_roots( + db, + inventory_types=payload.get("inventory_types") or None, + root_ids=payload.get("root_ids") or None, + bind_orbits=bool(payload.get("bind_orbits", True)), + ) + + +@router.get("/sources") +async def list_source_product_assets( + satellite_family: Optional[str] = None, + satellite: Optional[str] = None, + source_format: Optional[str] = None, + parse_status: Optional[str] = None, + include_inactive: bool = False, + limit: int = Query(200, ge=1, le=1000), + offset: int = Query(0, ge=0), + current_user: AuthUserORM = Depends(_get_current_user), + db: AsyncSession = Depends(get_db), +): + _ = current_user + return await asset_inventory_service.list_source_products( + db, + satellite_family=satellite_family, + satellite=satellite, + source_format=source_format, + parse_status=parse_status, + include_inactive=include_inactive, + limit=limit, + offset=offset, + ) + + +@router.get("/orbits") +async def list_orbit_assets( + satellite_family: Optional[str] = None, + satellite: Optional[str] = None, + orbit_type: Optional[str] = None, + parse_status: Optional[str] = None, + include_inactive: bool = False, + limit: int = Query(200, ge=1, le=1000), + offset: int = Query(0, ge=0), + current_user: AuthUserORM = Depends(_get_current_user), + db: AsyncSession = Depends(get_db), +): + _ = current_user + return await asset_inventory_service.list_orbits( + db, + satellite_family=satellite_family, + satellite=satellite, + orbit_type=orbit_type, + parse_status=parse_status, + include_inactive=include_inactive, + limit=limit, + offset=offset, + ) + + +@router.get("/issues") +async def list_asset_inventory_issues( + status: str = "OPEN", + severity: Optional[str] = None, + issue_code: Optional[str] = None, + limit: int = Query(200, ge=1, le=1000), + offset: int = Query(0, ge=0), + current_user: AuthUserORM = Depends(_get_current_user), + db: AsyncSession = Depends(get_db), +): + _ = current_user + return await asset_inventory_service.list_issues( + db, + status=status, + severity=severity, + issue_code=issue_code, + limit=limit, + offset=offset, + ) + + +@router.post("/sources/{asset_id}/unpack-sentinel1") +async def unpack_sentinel1_source_asset( + asset_id: int, + request: Optional[S1UnpackRequest] = None, + admin_user: AuthUserORM = Depends(_require_admin), + db: AsyncSession = Depends(get_db), +): + _ = admin_user + asset = await db.get(SourceProductAssetORM, asset_id) + if asset is None: + raise HTTPException(status_code=404, detail="Source product asset not found.") + if asset.source_format != "S1_ZIP": + raise HTTPException(status_code=400, detail="Only Sentinel-1 ZIP assets can be unpacked by this endpoint.") + + request_data = request or S1UnpackRequest() + payload = { + "asset_id": asset_id, + "target_root": request_data.target_root, + "overwrite": bool(request_data.overwrite), + } + if request_data.min_disk_space_gb is not None: + payload["min_disk_space_gb"] = request_data.min_disk_space_gb + if request_data.delete_archive is not None: + payload["delete_archive"] = request_data.delete_archive + try: + task_id = await task_service.create_task( + "UNPACK_SENTINEL1", + "Sentinel-1 unpack", + params=payload, + ) + job_id = await job_queue_service.create_job( + "UNPACK_SENTINEL1", + payload=payload, + task_id=task_id, + ) + except ValueError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + await db.commit() + return {"message": "Sentinel-1 unpack task queued", "task_id": task_id, "job_id": job_id} + + +@router.post("/inventory/unpack-sentinel1", status_code=202) +async def run_sentinel1_unpack_batch( + request: Optional[S1BatchUnpackRequest] = None, + admin_user: AuthUserORM = Depends(_require_admin), +): + _ = admin_user + payload = (request or S1BatchUnpackRequest()).model_dump() + try: + task_id = await task_service.create_task( + "UNPACK_SENTINEL1", + "Sentinel-1 batch unpack", + params=payload, + ) + job_id = await job_queue_service.create_job( + "UNPACK_SENTINEL1", + payload=payload, + task_id=task_id, + ) + return {"message": "Sentinel-1 batch unpack task queued", "task_id": task_id, "job_id": job_id} + except ValueError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc diff --git a/backend/app/routers/task_batches.py b/backend/app/routers/task_batches.py index a132c27..772d9a2 100644 --- a/backend/app/routers/task_batches.py +++ b/backend/app/routers/task_batches.py @@ -99,9 +99,15 @@ def _normalize_list_pagination(limit: int, offset: int) -> tuple[int, int]: class DinsarBatchSceneCreate(BaseModel): file_path: str = Field(min_length=1) satellite: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH) + satellite_family: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH) imaging_date: Optional[str] = Field(default=None, max_length=32) imaging_mode: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH) polarization: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH) + orbit_direction: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH) + relative_orbit: Optional[str] = Field(default=None, max_length=64) + absolute_orbit: Optional[str] = Field(default=None, max_length=64) + has_orbit_data: Optional[bool] = None + orbit_file_path: Optional[str] = Field(default=None, max_length=4096) model_config = ConfigDict(extra="ignore") diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index 95380f5..dd0e7ae 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -14,6 +14,10 @@ MONITOR_CONFIG = { "radar_dirs": split_env_paths(settings.MONITOR_RADAR_DIRS), "orbit_dir": settings.MONITOR_ORBIT_DIR, "dinsar_dirs": split_env_paths(settings.MONITOR_DINSAR_DIRS), + # Sentinel-1 链路 + "s1_source_dirs": split_env_paths(settings.SOURCE_PRODUCT_DIRS), + "s1_storage_dirs": split_env_paths(settings.SENTINEL1_STORAGE_DIRS), + "s1_orbit_dirs": split_env_paths(settings.ORBIT_SOURCE_DIRS), # GF3 链路 "gf3_source_dirs": split_env_paths(settings.GF3_SOURCE_DIRS), "gf3_storage_dirs": split_env_paths(settings.GF3_STORAGE_DIRS), diff --git a/backend/app/services/asset_inventory_service.py b/backend/app/services/asset_inventory_service.py new file mode 100644 index 0000000..9cb9634 --- /dev/null +++ b/backend/app/services/asset_inventory_service.py @@ -0,0 +1,2410 @@ +from __future__ import annotations + +import asyncio +import hashlib +import os +import re +import shutil +import zipfile +from datetime import datetime, timedelta +from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple + +from geoalchemy2.shape import from_shape +from lxml import etree +from shapely.geometry import Polygon +from sqlalchemy import and_, delete, func, or_, select, update +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.ext.asyncio import AsyncSession + +from .. import database +from ..config import settings +from ..models import ( + AssetInventoryIssueORM, + AssetInventoryStateORM, + ManagedRootORM, + OrbitAssetORM, + RadarDataORM, + SceneOrbitBindingORM, + SourceProductAssetORM, +) +from ..utils import ( + find_xml_file, + normalize_satellite_family, + parse_lt1_radar_filename, + parse_xml_metadata, +) +from .pairing_state_service import pairing_state_service +from .task_service import task_service + + +PARSER_VERSION = "asset_inventory_v1" +S1_ORBIT_MATCH_RULE_VERSION = "s1_orbit_window_v1" +LT1_ORBIT_MATCH_RULE_VERSION = "lt1_orbit_day_v1" + +_WINDOWS_DRIVE_RE = re.compile(r"^[a-zA-Z]:[\\/]") +_S1_SOURCE_RE = re.compile( + r"^(?PS1[A-Z])_" + r"(?P[A-Z0-9]+)_" + r"(?P[A-Z0-9]+)_+" + r"(?P[0-9A-Z]{4})_" + r"(?P\d{8}T\d{6}(?:\.\d+)?)_" + r"(?P\d{8}T\d{6}(?:\.\d+)?)_" + r"(?P\d+)_" + r"(?P[0-9A-F]+)_" + r"(?P[0-9A-F]+)" + r"(?:\.SAFE|\.zip)?$", + re.IGNORECASE, +) +_S1_EOF_RE = re.compile( + r"^(?PS1[A-Z])_OPER_" + r"(?PAUX_[A-Z0-9]+)_" + r"(?P[A-Z0-9]+)_" + r"(?P\d{8}T\d{6})_" + r"V(?P\d{8}T\d{6})_" + r"(?P\d{8}T\d{6})\.EOF$", + re.IGNORECASE, +) +_LT1_ORBIT_RE = re.compile( + r"^(?PLT1[A-Z]?)_GpsData_GAS_C_(?P\d{8})\.txt$", + re.IGNORECASE, +) + + +def _parse_bool(value: Any, default: bool = False) -> bool: + if value is None: + return default + return str(value).strip().lower() in {"1", "true", "yes", "on"} + + +def _parse_float(value: Any, default: float = 0.0) -> float: + try: + return float(str(value).strip()) + except (TypeError, ValueError): + return default + + +def _configured_sentinel1_storage_dirs() -> List[str]: + values = settings.SENTINEL1_STORAGE_DIRS or settings.SOURCE_PRODUCT_DIRS + paths = [ + _normalize_path(item) + for item in str(values or "").replace(";", ",").split(",") + if str(item or "").strip() + ] + if not paths: + paths = [_normalize_path(os.path.join(settings.BACKEND_DIR, "runtime", "sentinel1_safe"))] + return paths + + +def _configured_sentinel1_archive_dirs() -> List[str]: + values = settings.SOURCE_PRODUCT_DIRS + paths = [ + _normalize_path(item) + for item in str(values or "").replace(";", ",").split(",") + if str(item or "").strip() + ] + deduped: List[str] = [] + for path in paths: + if path and path not in deduped: + deduped.append(path) + return deduped + + +def _target_root_for_s1_archive(archive_path: str, target_root: Optional[str] = None) -> str: + requested = _normalize_path(target_root or "") + if requested: + return requested + + storage_dirs = _configured_sentinel1_storage_dirs() + source_dirs = [ + _normalize_path(item) + for item in str(settings.SOURCE_PRODUCT_DIRS or "").replace(";", ",").split(",") + if str(item or "").strip() + ] + if len(storage_dirs) > 1 and source_dirs and len(source_dirs) == len(storage_dirs): + archive_norm = os.path.normcase(_normalize_path(archive_path)) + matches: List[Tuple[int, int]] = [] + for index, source_dir in enumerate(source_dirs): + source_norm = os.path.normcase(_normalize_path(source_dir)) + if archive_norm == source_norm or archive_norm.startswith(source_norm + os.sep): + matches.append((len(source_norm), index)) + if matches: + _, best_index = max(matches) + return storage_dirs[best_index] + return storage_dirs[0] + + +def _new_session() -> AsyncSession: + if database.AsyncSessionLocal is None: + database.init_db() + if database.AsyncSessionLocal is None: + raise RuntimeError("Database session factory is not initialized.") + return database.AsyncSessionLocal() + + +def _utcnow() -> datetime: + return datetime.utcnow() + + +def _normalize_path(path: str) -> str: + text = str(path or "").strip() + if not text: + return "" + if text.startswith("\\\\"): + return os.path.normpath(text) + if _WINDOWS_DRIVE_RE.match(text): + return os.path.normpath(text) + if text.startswith("/"): + return text.replace("\\", "/") + return os.path.normpath(os.path.abspath(text)) + + +def _path_kind(path: str) -> str: + text = str(path or "").strip() + if text.startswith("\\\\"): + return "unc" + if _WINDOWS_DRIVE_RE.match(text): + return "windows" + if text.startswith("/mnt/"): + return "wsl_mount" + if text.startswith("/"): + return "posix" + return "relative" + + +def _stat_path(path: str) -> Dict[str, Optional[float]]: + try: + stat = os.stat(path) + return { + "size_bytes": int(stat.st_size), + "mtime_epoch": float(stat.st_mtime), + "ctime_epoch": float(stat.st_ctime), + } + except OSError: + return {"size_bytes": None, "mtime_epoch": None, "ctime_epoch": None} + + +def _asset_uid(prefix: str, path: str) -> str: + digest = hashlib.sha1(_normalize_path(path).lower().encode("utf-8", errors="ignore")).hexdigest() + return f"{prefix}:{digest[:32]}" + + +def _strip_known_suffix(name: str) -> str: + lower = name.lower() + if lower.endswith(".zip"): + return name[:-4] + if lower.endswith(".safe"): + return name[:-5] + return name + + +def _parse_datetime_token(value: Optional[str]) -> Optional[datetime]: + text = str(value or "").strip() + if not text: + return None + if text.startswith("UTC="): + text = text[4:] + text = text.rstrip("Z") + try: + if "-" in text and "." in text: + return datetime.strptime(text, "%Y-%m-%dT%H:%M:%S.%f") + if "-" in text: + return datetime.strptime(text, "%Y-%m-%dT%H:%M:%S") + if "." in text: + return datetime.strptime(text, "%Y%m%dT%H%M%S.%f") + return datetime.strptime(text, "%Y%m%dT%H%M%S") + except ValueError: + return None + + +def _date_start_stop(date_yyyymmdd: str) -> Tuple[Optional[datetime], Optional[datetime]]: + try: + start = datetime.strptime(date_yyyymmdd, "%Y%m%d") + except ValueError: + return None, None + return start, start + timedelta(days=1) + + +def _json_safe(value: Any) -> Any: + if isinstance(value, datetime): + return value.isoformat() + if isinstance(value, tuple): + return [_json_safe(item) for item in value] + if isinstance(value, list): + return [_json_safe(item) for item in value] + if isinstance(value, dict): + return {str(key): _json_safe(item) for key, item in value.items()} + return value + + +def _xml_parser() -> etree.XMLParser: + return etree.XMLParser( + resolve_entities=False, + load_dtd=False, + no_network=True, + huge_tree=False, + recover=False, + ) + + +def _local_name(element: etree._Element) -> str: + try: + return etree.QName(element).localname + except Exception: + return str(element.tag).split("}")[-1] + + +def _first_text_by_local_name(root: etree._Element, names: Sequence[str]) -> Optional[str]: + wanted = {name.lower() for name in names} + for element in root.iter(): + if _local_name(element).lower() in wanted: + text = (element.text or "").strip() + if text: + return text + return None + + +def _texts_by_local_name(root: etree._Element, name: str) -> List[str]: + wanted = name.lower() + values: List[str] = [] + for element in root.iter(): + if _local_name(element).lower() != wanted: + continue + text = (element.text or "").strip() + if text and text not in values: + values.append(text) + return values + + +def _s1_polygon_from_coordinates(text: Optional[str]) -> Optional[List[Tuple[float, float]]]: + if not text: + return None + points: List[Tuple[float, float]] = [] + for token in re.split(r"\s+", text.strip()): + if not token: + continue + parts = [part for part in re.split(r"[,;]", token) if part] + if len(parts) < 2: + continue + try: + first = float(parts[0]) + second = float(parts[1]) + except ValueError: + continue + + # Sentinel-1 manifest gml:coordinates commonly stores lat,lon. + if abs(first) > 90.0 and abs(second) <= 90.0: + lon, lat = first, second + else: + lon, lat = second, first + points.append((lon, lat)) + + if len(points) < 3: + return None + if points[0] != points[-1]: + points.append(points[0]) + return points + + +def _bbox_from_polygon(points: Optional[List[Tuple[float, float]]]) -> Optional[Tuple[float, float, float, float]]: + if not points or len(points) < 3: + return None + lons = [float(point[0]) for point in points] + lats = [float(point[1]) for point in points] + return min(lons), min(lats), max(lons), max(lats) + + +def _centroid_from_polygon(points: Optional[List[Tuple[float, float]]]) -> Tuple[Optional[float], Optional[float]]: + if not points or len(points) < 3: + return None, None + try: + poly = Polygon(points) + if not poly.is_valid: + poly = poly.buffer(0) + if poly.is_empty: + return None, None + return float(poly.centroid.x), float(poly.centroid.y) + except Exception: + return None, None + + +def _parse_s1_source_name(name: str) -> Optional[Dict[str, Any]]: + base = _strip_known_suffix(os.path.basename(name or "")) + match = _S1_SOURCE_RE.match(base) + if not match: + return None + + start_time = _parse_datetime_token(match.group("start")) + stop_time = _parse_datetime_token(match.group("stop")) + class_token = match.group("class").upper() + polarization = class_token[-2:] if len(class_token) >= 2 else class_token + absolute_orbit = match.group("absolute_orbit").lstrip("0") or match.group("absolute_orbit") + + return { + "logical_product_uid": base, + "satellite": match.group("satellite").upper(), + "satellite_family": "S1", + "source_format": "S1_ZIP" if name.lower().endswith(".zip") else "S1_SAFE_DIR", + "product_type": match.group("product").upper(), + "product_level": "L1", + "imaging_mode": match.group("mode").upper(), + "polarization": polarization, + "absolute_orbit": absolute_orbit, + "acquisition_start_time_utc": start_time, + "acquisition_stop_time_utc": stop_time, + "imaging_date": match.group("start")[:8], + "source_product_token": class_token, + "metadata": { + "filename_datatake": match.group("datatake").upper(), + "filename_product_uid": match.group("product_uid").upper(), + "filename_absolute_orbit": match.group("absolute_orbit"), + "filename_class_token": class_token, + }, + } + + +def _parse_s1_manifest_bytes(data: bytes) -> Dict[str, Any]: + root = etree.fromstring(data, parser=_xml_parser()) + start_time = _parse_datetime_token(_first_text_by_local_name(root, ["startTime"])) + stop_time = _parse_datetime_token(_first_text_by_local_name(root, ["stopTime"])) + product_type = _first_text_by_local_name(root, ["productType"]) + mode = _first_text_by_local_name(root, ["mode"]) + orbit_direction = _first_text_by_local_name(root, ["pass"]) + polarizations = _texts_by_local_name(root, "transmitterReceiverPolarisation") + absolute_orbit = _first_text_by_local_name(root, ["orbitNumber"]) + relative_orbit = _first_text_by_local_name(root, ["relativeOrbitNumber"]) + coordinates = _first_text_by_local_name(root, ["coordinates"]) + coverage_polygon = _s1_polygon_from_coordinates(coordinates) + + values: Dict[str, Any] = { + "manifest_start_time": start_time, + "manifest_stop_time": stop_time, + "manifest_product_type": product_type.strip().upper() if product_type else None, + "manifest_mode": mode.strip().upper() if mode else None, + "manifest_orbit_direction": orbit_direction.strip().upper() if orbit_direction else None, + "manifest_polarizations": [item.strip().upper() for item in polarizations if item.strip()], + "manifest_absolute_orbit": absolute_orbit.strip() if absolute_orbit else None, + "manifest_relative_orbit": relative_orbit.strip() if relative_orbit else None, + "coverage_polygon": coverage_polygon, + } + return {key: value for key, value in values.items() if value not in (None, "", [])} + + +def _parse_s1_zip_manifest(path: str) -> Dict[str, Any]: + with zipfile.ZipFile(path) as archive: + manifest_name = next( + (name for name in archive.namelist() if name.lower().endswith("/manifest.safe") or name.lower() == "manifest.safe"), + None, + ) + if not manifest_name: + return {"manifest_parse_status": "MISSING"} + return { + "manifest_parse_status": "OK", + "manifest_path": manifest_name, + **_parse_s1_manifest_bytes(archive.read(manifest_name)), + } + + +def _parse_s1_safe_manifest(path: str) -> Dict[str, Any]: + manifest_path = os.path.join(path, "manifest.safe") + if not os.path.isfile(manifest_path): + return {"manifest_parse_status": "MISSING"} + with open(manifest_path, "rb") as stream: + return { + "manifest_parse_status": "OK", + "manifest_path": manifest_path, + **_parse_s1_manifest_bytes(stream.read()), + } + + +def _parse_s1_eof_header(path: str) -> Dict[str, Any]: + try: + root = etree.parse(path, parser=_xml_parser()).getroot() + except Exception as exc: + return {"header_parse_status": "FAILED", "header_parse_error": str(exc)} + + def _time(name: str) -> Optional[datetime]: + return _parse_datetime_token(_first_text_by_local_name(root, [name])) + + mission = _first_text_by_local_name(root, ["Mission"]) + file_type = _first_text_by_local_name(root, ["File_Type"]) + return { + "header_parse_status": "OK", + "header_mission": mission, + "header_file_type": file_type, + "header_validity_start": _time("Validity_Start"), + "header_validity_stop": _time("Validity_Stop"), + "header_creation_date": _time("Creation_Date"), + } + + +def _parse_source_entry(path: str, root: ManagedRootORM) -> Optional[Dict[str, Any]]: + path = _normalize_path(path) + name = os.path.basename(path) + lower_name = name.lower() + stat = _stat_path(path) + now = _utcnow() + + if lower_name.endswith(".zip") and name.upper().startswith("S1"): + name_meta = _parse_s1_source_name(name) + if not name_meta: + return None + parse_status = "OK" + parse_error = None + manifest_meta: Dict[str, Any] = {} + try: + manifest_meta = _parse_s1_zip_manifest(path) + except Exception as exc: + parse_status = "PARTIAL" + parse_error = str(exc) + manifest_meta = {"manifest_parse_status": "FAILED", "manifest_parse_error": str(exc)} + return _build_s1_source_asset(path, root, name_meta, manifest_meta, stat, parse_status, parse_error, now) + + if lower_name.endswith(".safe") and os.path.isdir(path) and name.upper().startswith("S1"): + name_meta = _parse_s1_source_name(name) + if not name_meta: + return None + name_meta["source_format"] = "S1_SAFE_DIR" + parse_status = "OK" + parse_error = None + manifest_meta = {} + try: + manifest_meta = _parse_s1_safe_manifest(path) + except Exception as exc: + parse_status = "PARTIAL" + parse_error = str(exc) + manifest_meta = {"manifest_parse_status": "FAILED", "manifest_parse_error": str(exc)} + return _build_s1_source_asset(path, root, name_meta, manifest_meta, stat, parse_status, parse_error, now) + + if os.path.isdir(path) and name.upper().startswith("LT1"): + parsed = parse_lt1_radar_filename(name) + if not parsed: + return None + coverage_polygon = None + xml_meta: Dict[str, Any] = {} + xml_path = find_xml_file(path) + if xml_path: + try: + coverage_polygon, parsed_xml = parse_xml_metadata(xml_path) + if parsed_xml: + xml_meta = parsed_xml + except Exception as exc: + xml_meta = {"xml_parse_error": str(exc), "xml_path": xml_path} + return _build_lt1_source_asset(path, root, parsed, xml_meta, coverage_polygon, stat, now) + + return None + + +def _build_s1_source_asset( + path: str, + root: ManagedRootORM, + name_meta: Dict[str, Any], + manifest_meta: Dict[str, Any], + stat: Dict[str, Optional[float]], + parse_status: str, + parse_error: Optional[str], + now: datetime, +) -> Dict[str, Any]: + metadata = dict(name_meta.get("metadata") or {}) + metadata.update(manifest_meta) + coverage_polygon = manifest_meta.get("coverage_polygon") + centroid_lon, centroid_lat = _centroid_from_polygon(coverage_polygon) + bbox = _bbox_from_polygon(coverage_polygon) + metadata.update( + { + "coverage_polygon": coverage_polygon, + "coverage_bbox": bbox, + "scene_center_lon": centroid_lon, + "scene_center_lat": centroid_lat, + } + ) + + manifest_pols = manifest_meta.get("manifest_polarizations") or [] + if manifest_pols: + metadata["polarization_channels"] = manifest_pols + + return { + "asset_uid": _asset_uid("source", path), + "logical_product_uid": name_meta.get("logical_product_uid"), + "satellite_family": "S1", + "satellite": name_meta.get("satellite"), + "source_format": name_meta.get("source_format") or "S1_ZIP", + "product_type": manifest_meta.get("manifest_product_type") or name_meta.get("product_type"), + "product_level": name_meta.get("product_level"), + "imaging_mode": manifest_meta.get("manifest_mode") or name_meta.get("imaging_mode"), + "polarization": name_meta.get("polarization"), + "absolute_orbit": manifest_meta.get("manifest_absolute_orbit") or name_meta.get("absolute_orbit"), + "relative_orbit": manifest_meta.get("manifest_relative_orbit"), + "orbit_direction": manifest_meta.get("manifest_orbit_direction"), + "acquisition_start_time_utc": manifest_meta.get("manifest_start_time") or name_meta.get("acquisition_start_time_utc"), + "acquisition_stop_time_utc": manifest_meta.get("manifest_stop_time") or name_meta.get("acquisition_stop_time_utc"), + "imaging_date": name_meta.get("imaging_date"), + "root_ref_id": root.id, + "root_path": root.path, + "file_path": path, + "archive_path": path if path.lower().endswith(".zip") else None, + "path_kind": _path_kind(path), + "file_name": os.path.basename(path), + "file_stem": _strip_known_suffix(os.path.basename(path)), + "file_ext": os.path.splitext(path)[1].lower(), + "size_bytes": stat.get("size_bytes"), + "mtime_epoch": stat.get("mtime_epoch"), + "checksum_status": "NOT_COMPUTED", + "parser_name": "sentinel1_source_manifest", + "parser_version": PARSER_VERSION, + "parse_status": parse_status, + "parse_error": parse_error, + "parsed_at": now, + "metadata_json": _json_safe(metadata), + "is_active": True, + "missing_since": None, + "updated_at": now, + } + + +def _build_lt1_source_asset( + path: str, + root: ManagedRootORM, + parsed: Dict[str, Any], + xml_meta: Dict[str, Any], + coverage_polygon: Optional[List[Tuple[float, float]]], + stat: Dict[str, Optional[float]], + now: datetime, +) -> Dict[str, Any]: + metadata = dict(parsed) + metadata.update({key: value for key, value in xml_meta.items() if value not in (None, "")}) + metadata["coverage_polygon"] = coverage_polygon + metadata["coverage_bbox"] = _bbox_from_polygon(coverage_polygon) + centroid_lon, centroid_lat = _centroid_from_polygon(coverage_polygon) + metadata["scene_center_lon"] = parsed.get("scene_center_lon") if parsed.get("scene_center_lon") is not None else centroid_lon + metadata["scene_center_lat"] = parsed.get("scene_center_lat") if parsed.get("scene_center_lat") is not None else centroid_lat + satellite = parsed.get("satellite") + imaging_date = parsed.get("imaging_date") + + return { + "asset_uid": _asset_uid("source", path), + "logical_product_uid": os.path.basename(path), + "satellite_family": normalize_satellite_family(satellite), + "satellite": satellite, + "source_format": "LT1_DIR", + "product_type": parsed.get("product_type"), + "product_level": parsed.get("product_level"), + "imaging_mode": parsed.get("imaging_mode"), + "polarization": parsed.get("polarization"), + "absolute_orbit": parsed.get("orbit_circle"), + "relative_orbit": None, + "orbit_direction": xml_meta.get("orbit_direction") or parsed.get("orbit_direction"), + "acquisition_start_time_utc": None, + "acquisition_stop_time_utc": None, + "imaging_date": imaging_date, + "root_ref_id": root.id, + "root_path": root.path, + "file_path": path, + "archive_path": None, + "path_kind": _path_kind(path), + "file_name": os.path.basename(path), + "file_stem": os.path.basename(path), + "file_ext": "", + "size_bytes": stat.get("size_bytes"), + "mtime_epoch": stat.get("mtime_epoch"), + "checksum_status": "NOT_COMPUTED", + "parser_name": "lt1_source_directory", + "parser_version": PARSER_VERSION, + "parse_status": "OK", + "parse_error": None, + "parsed_at": now, + "metadata_json": _json_safe(metadata), + "is_active": True, + "missing_since": None, + "updated_at": now, + } + + +def _parse_orbit_entry(path: str, root: ManagedRootORM) -> Optional[Dict[str, Any]]: + path = _normalize_path(path) + name = os.path.basename(path) + lower_name = name.lower() + stat = _stat_path(path) + now = _utcnow() + + if lower_name.endswith(".eof") and name.upper().startswith("S1"): + match = _S1_EOF_RE.match(name) + if not match: + return None + header_meta = _parse_s1_eof_header(path) + orbit_type = match.group("orbit_type").upper() + valid_start = header_meta.get("header_validity_start") or _parse_datetime_token(match.group("valid_start")) + valid_stop = header_meta.get("header_validity_stop") or _parse_datetime_token(match.group("valid_stop")) + generation_time = header_meta.get("header_creation_date") or _parse_datetime_token(match.group("generation")) + parse_status = "OK" if header_meta.get("header_parse_status") != "FAILED" else "PARTIAL" + quality_class = "precise" if orbit_type == "AUX_POEORB" else "restituted" if orbit_type == "AUX_RESORB" else "unknown" + metadata = { + "filename_provider": match.group("provider").upper(), + "filename_generation_time": match.group("generation"), + "filename_validity_start": match.group("valid_start"), + "filename_validity_stop": match.group("valid_stop"), + **header_meta, + } + return { + "orbit_uid": _asset_uid("orbit", path), + "satellite_family": "S1", + "satellite": match.group("satellite").upper(), + "orbit_type": orbit_type, + "native_format": "EOF", + "quality_class": quality_class, + "root_ref_id": root.id, + "root_path": root.path, + "file_path": path, + "file_name": name, + "file_stem": os.path.splitext(name)[0], + "file_ext": ".eof", + "size_bytes": stat.get("size_bytes"), + "mtime_epoch": stat.get("mtime_epoch"), + "checksum_status": "NOT_COMPUTED", + "validity_start_time_utc": valid_start, + "validity_stop_time_utc": valid_stop, + "generation_time_utc": generation_time, + "published_time_utc": None, + "parser_name": "sentinel1_eof", + "parser_version": PARSER_VERSION, + "parse_status": parse_status, + "parse_error": header_meta.get("header_parse_error"), + "parsed_at": now, + "metadata_json": _json_safe(metadata), + "is_active": True, + "missing_since": None, + "updated_at": now, + } + + if lower_name.endswith(".txt"): + match = _LT1_ORBIT_RE.match(name) + if not match: + return None + date_text = match.group("date") + valid_start, valid_stop = _date_start_stop(date_text) + return { + "orbit_uid": _asset_uid("orbit", path), + "satellite_family": "LT1", + "satellite": match.group("satellite").upper(), + "orbit_type": "GPSDATA_GAS_C", + "native_format": "TXT", + "quality_class": "precise", + "root_ref_id": root.id, + "root_path": root.path, + "file_path": path, + "file_name": name, + "file_stem": os.path.splitext(name)[0], + "file_ext": ".txt", + "size_bytes": stat.get("size_bytes"), + "mtime_epoch": stat.get("mtime_epoch"), + "checksum_status": "NOT_COMPUTED", + "validity_start_time_utc": valid_start, + "validity_stop_time_utc": valid_stop, + "generation_time_utc": None, + "published_time_utc": None, + "parser_name": "lt1_gps_txt", + "parser_version": PARSER_VERSION, + "parse_status": "OK", + "parse_error": None, + "parsed_at": now, + "metadata_json": {"orbit_date": date_text}, + "is_active": True, + "missing_since": None, + "updated_at": now, + } + + return None + + +def _iter_source_candidates(root_path: str) -> Iterable[str]: + stack = [_normalize_path(root_path)] + while stack: + current = stack.pop() + try: + with os.scandir(current) as iterator: + for entry in iterator: + try: + if entry.is_dir(follow_symlinks=False): + name_upper = entry.name.upper() + if name_upper.startswith("S1") and entry.name.lower().endswith(".safe"): + yield _normalize_path(entry.path) + continue + if name_upper.startswith("LT1") and parse_lt1_radar_filename(entry.name): + yield _normalize_path(entry.path) + continue + stack.append(entry.path) + except OSError: + continue + except OSError: + continue + + +def _iter_s1_zip_candidates(root_path: str) -> Iterable[str]: + stack = [_normalize_path(root_path)] + while stack: + current = stack.pop() + try: + with os.scandir(current) as iterator: + for entry in iterator: + try: + if entry.is_dir(follow_symlinks=False): + stack.append(entry.path) + elif entry.is_file(follow_symlinks=False): + if entry.name.upper().startswith("S1") and entry.name.lower().endswith(".zip"): + yield _normalize_path(entry.path) + except OSError: + continue + except OSError: + continue + + +def _iter_orbit_candidates(root_path: str) -> Iterable[str]: + stack = [_normalize_path(root_path)] + while stack: + current = stack.pop() + try: + with os.scandir(current) as iterator: + for entry in iterator: + try: + if entry.is_dir(follow_symlinks=False): + stack.append(entry.path) + elif entry.is_file(follow_symlinks=False): + lower = entry.name.lower() + if lower.endswith(".eof") or lower.endswith(".txt"): + yield _normalize_path(entry.path) + except OSError: + continue + except OSError: + continue + + +def _collect_source_assets(root: ManagedRootORM) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], int]: + rows: List[Dict[str, Any]] = [] + issues: List[Dict[str, Any]] = [] + entry_count = 0 + for path in _iter_source_candidates(root.path): + entry_count += 1 + try: + row = _parse_source_entry(path, root) + except Exception as exc: + row = None + issues.append( + { + "severity": "warning", + "issue_code": "source_parse_failed", + "issue_message": str(exc), + "source_path": path, + } + ) + if row is None: + continue + rows.append(row) + if row.get("parse_status") in {"FAILED", "PARTIAL"}: + issues.append( + { + "severity": "warning", + "issue_code": "source_parse_partial" if row.get("parse_status") == "PARTIAL" else "source_parse_failed", + "issue_message": row.get("parse_error"), + "source_path": row.get("file_path"), + } + ) + return rows, issues, entry_count + + +def _collect_orbit_assets(root: ManagedRootORM) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], int]: + rows: List[Dict[str, Any]] = [] + issues: List[Dict[str, Any]] = [] + entry_count = 0 + for path in _iter_orbit_candidates(root.path): + try: + row = _parse_orbit_entry(path, root) + except Exception as exc: + row = None + issues.append( + { + "severity": "warning", + "issue_code": "orbit_parse_failed", + "issue_message": str(exc), + "source_path": path, + } + ) + if row is None: + continue + entry_count += 1 + rows.append(row) + if row.get("parse_status") in {"FAILED", "PARTIAL"}: + issues.append( + { + "severity": "warning", + "issue_code": "orbit_parse_partial" if row.get("parse_status") == "PARTIAL" else "orbit_parse_failed", + "issue_message": row.get("parse_error"), + "source_path": row.get("file_path"), + } + ) + return rows, issues, entry_count + + +def _insar_source_ready(row: Dict[str, Any], coverage_polygon: Optional[List[Tuple[float, float]]]) -> Tuple[bool, Optional[str]]: + reasons: List[str] = [] + if not coverage_polygon or len(coverage_polygon) < 3: + reasons.append("missing_footprint") + if not row.get("imaging_date"): + reasons.append("missing_date") + if not row.get("imaging_mode"): + reasons.append("missing_imaging_mode") + if not row.get("polarization"): + reasons.append("missing_polarization") + if str(row.get("product_type") or "").upper() not in {"SLC", "SSC"}: + reasons.append("not_complex_source") + if reasons: + return False, ";".join(reasons) + return True, None + + +class AssetInventoryService: + async def _progress(self, task_id: Optional[str], message: str, progress: int) -> None: + if not task_id: + return + await task_service.update_task(task_id, message=message, progress=max(0, min(100, int(progress)))) + + async def _get_scan_roots( + self, + db: AsyncSession, + *, + inventory_types: Optional[Sequence[str]] = None, + root_ids: Optional[Sequence[int]] = None, + ) -> List[ManagedRootORM]: + type_set = {str(item or "").strip().lower() for item in (inventory_types or []) if str(item or "").strip()} + roles: List[str] = [] + if not type_set or "source_product" in type_set or "source" in type_set: + roles.append("source_product_pool") + if not type_set or "orbit_asset" in type_set or "orbit" in type_set: + roles.append("orbit_asset_pool") + stmt = ( + select(ManagedRootORM) + .where(ManagedRootORM.enabled == True) # noqa: E712 + .where(ManagedRootORM.root_role.in_(roles)) + .order_by(ManagedRootORM.root_role.asc(), ManagedRootORM.id.asc()) + ) + if root_ids: + stmt = stmt.where(ManagedRootORM.id.in_([int(item) for item in root_ids])) + result = await db.execute(stmt) + return result.scalars().all() + + async def scan_configured_roots( + self, + db: Optional[AsyncSession] = None, + *, + inventory_types: Optional[Sequence[str]] = None, + root_ids: Optional[Sequence[int]] = None, + bind_orbits: bool = True, + task_id: Optional[str] = None, + ) -> Dict[str, Any]: + generated_session = db is None + if generated_session: + db = _new_session() + assert db is not None + + try: + await self._progress(task_id, "Preparing source/orbit asset scan...", 2) + roots = await self._get_scan_roots(db, inventory_types=inventory_types, root_ids=root_ids) + results: List[Dict[str, Any]] = [] + totals = { + "source_roots": 0, + "orbit_roots": 0, + "source_assets": 0, + "orbit_assets": 0, + "issues": 0, + "inaccessible_roots": 0, + } + + total_roots = len(roots) + for index, root in enumerate(roots, start=1): + progress = 5 + int((index - 1) / max(1, total_roots) * 75) + await self._progress(task_id, f"Scanning {root.display_name}: {root.path}", progress) + if root.root_role == "source_product_pool": + result = await self.scan_source_root(db, root) + totals["source_roots"] += 1 + totals["source_assets"] += int(result.get("asset_count") or 0) + elif root.root_role == "orbit_asset_pool": + result = await self.scan_orbit_root(db, root) + totals["orbit_roots"] += 1 + totals["orbit_assets"] += int(result.get("asset_count") or 0) + else: + continue + totals["issues"] += int(result.get("issue_count") or 0) + if result.get("status") == "INACCESSIBLE": + totals["inaccessible_roots"] += 1 + results.append(result) + await db.commit() + + binding_summary: Dict[str, Any] = {} + if bind_orbits: + await self._progress(task_id, "Binding scenes to precise orbit assets...", 88) + binding_summary = await self.bind_scene_orbits(db) + await db.commit() + + summary = { + "message": "Asset inventory scan completed", + "root_count": total_roots, + **totals, + "binding": binding_summary, + "results": results, + } + await self._progress(task_id, "Asset inventory scan completed", 100) + return summary + except Exception: + if db is not None: + await db.rollback() + raise + finally: + if generated_session and db is not None: + await db.close() + + async def scan_source_root(self, db: AsyncSession, root: ManagedRootORM) -> Dict[str, Any]: + started_at = _utcnow() + state = await self._ensure_state(db, root, "source_product", started_at) + if not os.path.isdir(root.path): + await self._finish_state( + db, + state, + status="INACCESSIBLE", + started_at=started_at, + entry_count=0, + asset_count=0, + issue_count=1, + error=f"Source product root is not accessible: {root.path}", + ) + await self._replace_root_issues( + db, + root, + "source_product", + [ + { + "severity": "error", + "issue_code": "root_inaccessible", + "issue_message": f"Source product root is not accessible: {root.path}", + "source_path": root.path, + } + ], + ) + return {"root_id": root.id, "inventory_type": "source_product", "status": "INACCESSIBLE", "asset_count": 0, "issue_count": 1} + + rows, issues, entry_count = await asyncio.to_thread(_collect_source_assets, root) + seen_paths = [row["file_path"] for row in rows] + now = _utcnow() + for row in rows: + stmt = pg_insert(SourceProductAssetORM).values(row) + excluded = stmt.excluded + stmt = stmt.on_conflict_do_update( + index_elements=["file_path"], + set_={ + "asset_uid": excluded.asset_uid, + "logical_product_uid": excluded.logical_product_uid, + "satellite_family": excluded.satellite_family, + "satellite": excluded.satellite, + "source_format": excluded.source_format, + "product_type": excluded.product_type, + "product_level": excluded.product_level, + "imaging_mode": excluded.imaging_mode, + "polarization": excluded.polarization, + "absolute_orbit": excluded.absolute_orbit, + "relative_orbit": excluded.relative_orbit, + "orbit_direction": excluded.orbit_direction, + "acquisition_start_time_utc": excluded.acquisition_start_time_utc, + "acquisition_stop_time_utc": excluded.acquisition_stop_time_utc, + "imaging_date": excluded.imaging_date, + "root_ref_id": excluded.root_ref_id, + "root_path": excluded.root_path, + "archive_path": excluded.archive_path, + "path_kind": excluded.path_kind, + "file_name": excluded.file_name, + "file_stem": excluded.file_stem, + "file_ext": excluded.file_ext, + "size_bytes": excluded.size_bytes, + "mtime_epoch": excluded.mtime_epoch, + "checksum_status": excluded.checksum_status, + "parser_name": excluded.parser_name, + "parser_version": excluded.parser_version, + "parse_status": excluded.parse_status, + "parse_error": excluded.parse_error, + "parsed_at": excluded.parsed_at, + "metadata_json": excluded.metadata_json, + "is_active": True, + "missing_since": None, + "updated_at": now, + }, + ) + await db.execute(stmt) + await db.flush() + + asset_ids_by_path: Dict[str, int] = {} + if seen_paths: + result = await db.execute( + select(SourceProductAssetORM.file_path, SourceProductAssetORM.id).where(SourceProductAssetORM.file_path.in_(seen_paths)) + ) + asset_ids_by_path = {str(path): int(asset_id) for path, asset_id in result.all()} + await self._upsert_radar_records_for_source_assets(db, rows, asset_ids_by_path) + + await self._mark_missing_source_assets(db, root, seen_paths, now) + await self._replace_root_issues(db, root, "source_product", issues) + await self._finish_state( + db, + state, + status="OK" if not any(item.get("severity") == "error" for item in issues) else "WARNING", + started_at=started_at, + entry_count=entry_count, + asset_count=len(rows), + issue_count=len(issues), + error=None, + ) + return { + "root_id": root.id, + "root_path": root.path, + "inventory_type": "source_product", + "status": state.status, + "entry_count": entry_count, + "asset_count": len(rows), + "issue_count": len(issues), + } + + async def scan_orbit_root(self, db: AsyncSession, root: ManagedRootORM) -> Dict[str, Any]: + started_at = _utcnow() + state = await self._ensure_state(db, root, "orbit_asset", started_at) + if not os.path.isdir(root.path): + await self._finish_state( + db, + state, + status="INACCESSIBLE", + started_at=started_at, + entry_count=0, + asset_count=0, + issue_count=1, + error=f"Orbit asset root is not accessible: {root.path}", + ) + await self._replace_root_issues( + db, + root, + "orbit_asset", + [ + { + "severity": "error", + "issue_code": "root_inaccessible", + "issue_message": f"Orbit asset root is not accessible: {root.path}", + "source_path": root.path, + } + ], + ) + return {"root_id": root.id, "inventory_type": "orbit_asset", "status": "INACCESSIBLE", "asset_count": 0, "issue_count": 1} + + rows, issues, entry_count = await asyncio.to_thread(_collect_orbit_assets, root) + seen_paths = [row["file_path"] for row in rows] + now = _utcnow() + for row in rows: + stmt = pg_insert(OrbitAssetORM).values(row) + excluded = stmt.excluded + stmt = stmt.on_conflict_do_update( + index_elements=["file_path"], + set_={ + "orbit_uid": excluded.orbit_uid, + "satellite_family": excluded.satellite_family, + "satellite": excluded.satellite, + "orbit_type": excluded.orbit_type, + "native_format": excluded.native_format, + "quality_class": excluded.quality_class, + "root_ref_id": excluded.root_ref_id, + "root_path": excluded.root_path, + "file_name": excluded.file_name, + "file_stem": excluded.file_stem, + "file_ext": excluded.file_ext, + "size_bytes": excluded.size_bytes, + "mtime_epoch": excluded.mtime_epoch, + "checksum_status": excluded.checksum_status, + "validity_start_time_utc": excluded.validity_start_time_utc, + "validity_stop_time_utc": excluded.validity_stop_time_utc, + "generation_time_utc": excluded.generation_time_utc, + "published_time_utc": excluded.published_time_utc, + "parser_name": excluded.parser_name, + "parser_version": excluded.parser_version, + "parse_status": excluded.parse_status, + "parse_error": excluded.parse_error, + "parsed_at": excluded.parsed_at, + "metadata_json": excluded.metadata_json, + "is_active": True, + "missing_since": None, + "updated_at": now, + }, + ) + await db.execute(stmt) + + await self._mark_missing_orbit_assets(db, root, seen_paths, now) + await self._replace_root_issues(db, root, "orbit_asset", issues) + await self._finish_state( + db, + state, + status="OK" if not any(item.get("severity") == "error" for item in issues) else "WARNING", + started_at=started_at, + entry_count=entry_count, + asset_count=len(rows), + issue_count=len(issues), + error=None, + ) + return { + "root_id": root.id, + "root_path": root.path, + "inventory_type": "orbit_asset", + "status": state.status, + "entry_count": entry_count, + "asset_count": len(rows), + "issue_count": len(issues), + } + + async def _find_source_asset_root( + self, + db: AsyncSession, + target_path: str, + ) -> Optional[ManagedRootORM]: + target_norm = os.path.normcase(_normalize_path(target_path)) + result = await db.execute( + select(ManagedRootORM) + .where(ManagedRootORM.enabled == True) # noqa: E712 + .where(ManagedRootORM.root_role == "source_product_pool") + .order_by(func.length(ManagedRootORM.path).desc()) + ) + for root in result.scalars().all(): + root_norm = os.path.normcase(_normalize_path(root.path)) + if target_norm == root_norm or target_norm.startswith(root_norm + os.sep): + return root + return None + + async def ensure_source_root_for_path( + self, + db: AsyncSession, + root_path: str, + *, + source_ref: str = "SENTINEL1_STORAGE_DIRS", + ) -> ManagedRootORM: + from .root_registry_service import root_registry_service + + root = await self._find_source_asset_root(db, root_path) + if root is not None: + return root + + await root_registry_service.sync_from_settings(db) + root = await self._find_source_asset_root(db, root_path) + if root is not None: + return root + + normalized = _normalize_path(root_path) + root_code = f"source_product_pool__sentinel1_storage_{hashlib.sha1(normalized.encode('utf-8')).hexdigest()[:12]}" + root = ManagedRootORM( + root_code=root_code, + root_role="source_product_pool", + display_name="Sentinel-1 Storage Pool", + path=normalized, + path_kind=_path_kind(normalized), + source_kind="env", + source_ref=source_ref, + scan_mode="file_pool", + enabled=True, + exists_flag=os.path.exists(normalized), + metadata_json={ + "env_var": source_ref, + "created_by": "sentinel1_unpack", + }, + ) + db.add(root) + await db.flush() + return root + + async def scan_source_path_after_unpack( + self, + db: AsyncSession, + source_path: str, + *, + bind_orbits: bool = True, + ) -> Dict[str, Any]: + path = _normalize_path(source_path) + root = await self._find_source_asset_root(db, path) + if root is None: + return { + "scanned": False, + "reason": "Sentinel-1 storage directory is not registered as a source product pool.", + "source_path": path, + } + + row = await asyncio.to_thread(_parse_source_entry, path, root) + if row is None: + return { + "scanned": False, + "reason": "Unpacked Sentinel-1 SAFE could not be parsed.", + "source_path": path, + "root_id": root.id, + } + + now = _utcnow() + stmt = pg_insert(SourceProductAssetORM).values(row) + excluded = stmt.excluded + stmt = stmt.on_conflict_do_update( + index_elements=["file_path"], + set_={ + "asset_uid": excluded.asset_uid, + "logical_product_uid": excluded.logical_product_uid, + "satellite_family": excluded.satellite_family, + "satellite": excluded.satellite, + "source_format": excluded.source_format, + "product_type": excluded.product_type, + "product_level": excluded.product_level, + "imaging_mode": excluded.imaging_mode, + "polarization": excluded.polarization, + "absolute_orbit": excluded.absolute_orbit, + "relative_orbit": excluded.relative_orbit, + "orbit_direction": excluded.orbit_direction, + "acquisition_start_time_utc": excluded.acquisition_start_time_utc, + "acquisition_stop_time_utc": excluded.acquisition_stop_time_utc, + "imaging_date": excluded.imaging_date, + "root_ref_id": excluded.root_ref_id, + "root_path": excluded.root_path, + "archive_path": excluded.archive_path, + "path_kind": excluded.path_kind, + "file_name": excluded.file_name, + "file_stem": excluded.file_stem, + "file_ext": excluded.file_ext, + "size_bytes": excluded.size_bytes, + "mtime_epoch": excluded.mtime_epoch, + "checksum_status": excluded.checksum_status, + "parser_name": excluded.parser_name, + "parser_version": excluded.parser_version, + "parse_status": excluded.parse_status, + "parse_error": excluded.parse_error, + "parsed_at": excluded.parsed_at, + "metadata_json": excluded.metadata_json, + "is_active": True, + "missing_since": None, + "updated_at": now, + }, + ) + await db.execute(stmt) + await db.flush() + + result = await db.execute( + select(SourceProductAssetORM.id).where(SourceProductAssetORM.file_path == path) + ) + asset_id = result.scalar_one_or_none() + radar_data_id = None + if asset_id is not None: + await self._upsert_radar_records_for_source_assets(db, [row], {path: int(asset_id)}) + radar_result = await db.execute( + select(RadarDataORM.id).where(RadarDataORM.file_path == path) + ) + radar_data_id = radar_result.scalar_one_or_none() + binding_summary: Dict[str, Any] = {} + if bind_orbits and radar_data_id is not None: + binding_summary = await self.bind_scene_orbits(db, radar_data_ids=[int(radar_data_id)]) + return { + "scanned": True, + "root_id": root.id, + "root_path": root.path, + "asset_id": asset_id, + "radar_data_id": radar_data_id, + "parse_status": row.get("parse_status"), + "binding": binding_summary, + } + + def unpack_sentinel1_archive( + self, + archive_path: str, + *, + target_root: Optional[str] = None, + overwrite: bool = False, + min_disk_space_gb: Optional[float] = None, + tmp_suffix: Optional[str] = None, + delete_archive: Optional[bool] = None, + progress_callback: Optional[Callable[[int, str], None]] = None, + log_callback: Optional[Callable[[str, str], None]] = None, + ) -> Dict[str, Any]: + archive = _normalize_path(archive_path) + if not os.path.isfile(archive): + raise FileNotFoundError(archive) + if not archive.lower().endswith(".zip"): + raise ValueError("Only Sentinel-1 ZIP archives can be unpacked.") + + target_dir = _target_root_for_s1_archive(archive, target_root) + os.makedirs(target_dir, exist_ok=True) + tmp_suffix_text = str(tmp_suffix or os.getenv("UNPACK_TMP_SUFFIX") or ".unpack_tmp").strip() or ".unpack_tmp" + min_free_gb = min_disk_space_gb + if min_free_gb is None: + min_free_gb = _parse_float(os.getenv("UNPACK_MIN_DISK_SPACE_GB"), 50.0) + should_delete_archive = ( + bool(delete_archive) + if delete_archive is not None + else _parse_bool(os.getenv("UNPACK_DELETE_ARCHIVE"), False) + ) + + def _log(level: str, message: str) -> None: + if log_callback: + log_callback(level, message) + + def _progress(progress: int, message: str) -> None: + if progress_callback: + progress_callback(progress, message) + + _progress(3, "Reading Sentinel-1 ZIP manifest...") + with zipfile.ZipFile(archive) as zip_obj: + names = zip_obj.namelist() + if not names: + raise ValueError("ZIP archive is empty.") + safe_dirs = { + name.split("/", 1)[0] + for name in names + if "/" in name and name.split("/", 1)[0].lower().endswith(".safe") + } + if len(safe_dirs) != 1: + raise ValueError("Expected exactly one top-level .SAFE directory in Sentinel-1 ZIP.") + safe_name = next(iter(safe_dirs)) + output_safe_dir = _normalize_path(os.path.join(target_dir, safe_name)) + tmp_dir = output_safe_dir + tmp_suffix_text + lock_path = output_safe_dir + ".unpacking" + target_root_abs = os.path.abspath(target_dir) + output_abs = os.path.abspath(output_safe_dir) + if not output_abs.startswith(target_root_abs + os.sep): + raise ValueError("Unsafe Sentinel-1 target path.") + for member in names: + member_target = os.path.abspath(os.path.join(target_dir, member)) + if not member_target.startswith(target_root_abs + os.sep): + raise ValueError(f"Unsafe ZIP member path: {member}") + required_bytes = sum(max(0, int(info.file_size or 0)) for info in zip_obj.infolist()) + + _, _, free_bytes = shutil.disk_usage(target_dir) + min_free_bytes = int(float(min_free_gb or 0) * (1024 ** 3)) + if free_bytes - required_bytes < min_free_bytes: + raise OSError( + "Sentinel-1 storage has insufficient free space: " + f"needed {required_bytes / (1024 ** 3):.2f} GB, " + f"free {free_bytes / (1024 ** 3):.2f} GB, " + f"min free after {float(min_free_gb or 0):.2f} GB" + ) + + if os.path.exists(output_safe_dir): + if not overwrite: + return { + "status": "EXISTS", + "archive_path": archive, + "target_root": target_dir, + "safe_dir": output_safe_dir, + "extracted": False, + "member_count": len(names), + } + shutil.rmtree(output_safe_dir) + if os.path.exists(tmp_dir): + shutil.rmtree(tmp_dir) + if os.path.exists(lock_path): + raise OSError(f"Sentinel-1 unpack lock exists: {lock_path}") + + os.makedirs(tmp_dir, exist_ok=True) + with open(lock_path, "w", encoding="utf-8") as stream: + stream.write(_utcnow().isoformat()) + + try: + total_members = len(names) + for index, member in enumerate(names, start=1): + if index % 100 == 0 or index == total_members: + pct = 5 + int(index / max(1, total_members) * 85) + _progress(pct, f"Extracting Sentinel-1 SAFE ({index}/{total_members})") + rel_member = member.split("/", 1)[1] if "/" in member else "" + if not rel_member: + continue + destination = os.path.abspath(os.path.join(tmp_dir, rel_member)) + if not destination.startswith(os.path.abspath(tmp_dir) + os.sep): + raise ValueError(f"Unsafe ZIP member path: {member}") + info = zip_obj.getinfo(member) + if info.is_dir(): + os.makedirs(destination, exist_ok=True) + continue + os.makedirs(os.path.dirname(destination), exist_ok=True) + with zip_obj.open(info, "r") as source, open(destination, "wb") as target: + shutil.copyfileobj(source, target, length=1024 * 1024) + + if not os.listdir(tmp_dir): + raise OSError("Extracted SAFE directory is empty.") + os.replace(tmp_dir, output_safe_dir) + finally: + if os.path.exists(lock_path): + try: + os.remove(lock_path) + except OSError: + pass + if os.path.exists(tmp_dir): + try: + shutil.rmtree(tmp_dir) + except OSError: + pass + + if should_delete_archive: + os.remove(archive) + _log("INFO", f"Deleted Sentinel-1 ZIP after unpack: {archive}") + + _progress(92, "Sentinel-1 SAFE extracted.") + return { + "status": "EXTRACTED", + "archive_path": archive, + "target_root": target_dir, + "safe_dir": output_safe_dir, + "extracted": True, + "member_count": len(names), + } + + async def run_sentinel1_unpack_task(self, task_id: str, payload: Optional[Dict[str, Any]] = None) -> None: + payload = payload if isinstance(payload, dict) else {} + asset_id = payload.get("asset_id") + if not asset_id: + raise ValueError("asset_id is required.") + + await task_service.start_task(task_id, message="Sentinel-1 unpack started") + + async with _new_session() as db: + asset = await db.get(SourceProductAssetORM, int(asset_id)) + if asset is None: + raise ValueError("Source product asset not found.") + if asset.source_format != "S1_ZIP": + raise ValueError("Only Sentinel-1 ZIP assets can be unpacked.") + + archive_path = asset.file_path + target_root = payload.get("target_root") or None + overwrite = bool(payload.get("overwrite", False)) + min_disk_space_gb = payload.get("min_disk_space_gb") + delete_archive = payload.get("delete_archive") if "delete_archive" in payload else None + tmp_suffix = payload.get("tmp_suffix") or os.getenv("UNPACK_TMP_SUFFIX") or ".unpack_tmp" + + loop = asyncio.get_running_loop() + + def _log(level: str, message: str) -> None: + async def _add() -> None: + await task_service.add_log(task_id, level, message) + + asyncio.run_coroutine_threadsafe(_add(), loop) + + def _progress(progress: int, message: str) -> None: + async def _update() -> None: + await task_service.update_task(task_id, progress=progress, message=message) + + asyncio.run_coroutine_threadsafe(_update(), loop) + + result = await asyncio.to_thread( + self.unpack_sentinel1_archive, + archive_path, + target_root=target_root, + overwrite=overwrite, + min_disk_space_gb=min_disk_space_gb, + tmp_suffix=tmp_suffix, + delete_archive=delete_archive, + progress_callback=_progress, + log_callback=_log, + ) + + if result.get("target_root"): + await self.ensure_source_root_for_path(db, str(result["target_root"])) + await db.commit() + + metadata = dict(asset.metadata_json or {}) + metadata["last_unpacked_safe_dir"] = result.get("safe_dir") + metadata["last_unpacked_target_root"] = result.get("target_root") + metadata["last_unpacked_at"] = _utcnow().isoformat() + metadata["last_unpacked_status"] = result.get("status") + asset.metadata_json = metadata + await db.commit() + + scan_summary: Dict[str, Any] = {} + if result.get("safe_dir"): + await task_service.update_task(task_id, progress=94, message="Scanning unpacked Sentinel-1 SAFE...") + scan_summary = await self.scan_source_path_after_unpack(db, str(result["safe_dir"]), bind_orbits=True) + await db.commit() + await task_service.add_log(task_id, "INFO", f"Sentinel-1 unpack result: {result}") + if scan_summary: + await task_service.add_log(task_id, "INFO", f"Sentinel-1 SAFE inventory scan: {scan_summary}") + + await task_service.update_task( + task_id, + status="COMPLETED", + progress=100, + message=( + "Sentinel-1 unpack complete: " + f"{os.path.basename(str(result.get('safe_dir') or '')) or result.get('status')}" + ), + ) + + async def run_sentinel1_unpack_batch_task(self, task_id: str, payload: Optional[Dict[str, Any]] = None) -> None: + payload = payload if isinstance(payload, dict) else {} + overwrite = bool(payload.get("overwrite", False)) + min_disk_space_gb = payload.get("min_disk_space_gb") + delete_archive = payload.get("delete_archive") if "delete_archive" in payload else None + target_root = payload.get("target_root") or None + scan_before_unpack = bool(payload.get("scan_before_unpack", True)) + + await task_service.start_task(task_id, message="Sentinel-1 batch unpack started") + + async with _new_session() as db: + if scan_before_unpack: + await task_service.update_task(task_id, progress=5, message="Refreshing Sentinel-1 inventory...") + await self.scan_configured_roots( + db, + inventory_types=["source_product", "orbit_asset"], + bind_orbits=True, + task_id=task_id, + ) + await db.commit() + + archive_dirs = _configured_sentinel1_archive_dirs() + archives: List[Dict[str, Any]] = [] + seen_archives: set[str] = set() + for archive_dir in archive_dirs: + if not archive_dir or not os.path.isdir(archive_dir): + continue + for archive_path in _iter_s1_zip_candidates(archive_dir): + normalized_path = _normalize_path(archive_path) + if not normalized_path or normalized_path in seen_archives: + continue + seen_archives.add(normalized_path) + name_meta = _parse_s1_source_name(os.path.basename(normalized_path)) or {} + archives.append( + { + "file_path": normalized_path, + "logical_product_uid": name_meta.get("logical_product_uid"), + } + ) + archives.sort(key=lambda item: (str(item.get("logical_product_uid") or ""), str(item.get("file_path") or ""))) + if not archives: + raise ValueError("No Sentinel-1 ZIP archives were found in SOURCE_PRODUCT_DIRS.") + + loop = asyncio.get_running_loop() + + def _log(level: str, message: str) -> None: + async def _add() -> None: + await task_service.add_log(task_id, level, message) + + asyncio.run_coroutine_threadsafe(_add(), loop) + + processed = 0 + skipped = 0 + failed = 0 + total = len(archives) + + for index, archive_item in enumerate(archives, start=1): + archive_path = str(archive_item.get("file_path") or "") + logical_product_uid = str(archive_item.get("logical_product_uid") or "").strip() + asset_name = os.path.basename(archive_path or logical_product_uid or f"archive-{index}") + await task_service.update_task( + task_id, + progress=10 + int((index - 1) / max(1, total) * 80), + message=f"Processing Sentinel-1 archive {index}/{total}: {asset_name}", + ) + + if await self._s1_zip_has_unpacked_safe( + db, + { + "source_format": "S1_ZIP", + "logical_product_uid": logical_product_uid, + }, + ) and not overwrite: + skipped += 1 + await task_service.add_log(task_id, "INFO", f"Skipping already unpacked Sentinel-1 archive: {archive_path}") + continue + + try: + result = await asyncio.to_thread( + self.unpack_sentinel1_archive, + archive_path, + target_root=target_root, + overwrite=overwrite, + min_disk_space_gb=min_disk_space_gb, + delete_archive=delete_archive, + log_callback=_log, + ) + if result.get("target_root"): + await self.ensure_source_root_for_path(db, str(result["target_root"])) + await db.commit() + + scan_summary: Dict[str, Any] = {} + if result.get("safe_dir"): + await task_service.update_task( + task_id, + progress=10 + int((index - 1) / max(1, total) * 80) + 3, + message=f"Scanning unpacked Sentinel-1 SAFE {index}/{total}...", + ) + scan_summary = await self.scan_source_path_after_unpack(db, str(result["safe_dir"]), bind_orbits=True) + await db.commit() + + await task_service.add_log(task_id, "INFO", f"Sentinel-1 unpack result: {result}") + if scan_summary: + await task_service.add_log(task_id, "INFO", f"Sentinel-1 SAFE inventory scan: {scan_summary}") + if result.get("status") == "EXISTS": + skipped += 1 + else: + processed += 1 + except Exception as exc: + failed += 1 + await task_service.add_log(task_id, "ERROR", f"Sentinel-1 archive unpack failed: {archive_path} -> {exc}") + + status = "COMPLETED" if failed < total else "FAILED" + message = ( + "Sentinel-1 batch unpack complete: " + f"processed={processed}, skipped={skipped}, failed={failed}, total={total}" + ) + await task_service.update_task(task_id, status=status, progress=100, message=message) + + async def _ensure_state( + self, + db: AsyncSession, + root: ManagedRootORM, + inventory_type: str, + started_at: datetime, + ) -> AssetInventoryStateORM: + result = await db.execute( + select(AssetInventoryStateORM).where( + AssetInventoryStateORM.root_ref_id == root.id, + AssetInventoryStateORM.inventory_type == inventory_type, + ) + ) + state = result.scalar_one_or_none() + if state is None: + state = AssetInventoryStateORM( + root_ref_id=root.id, + inventory_type=inventory_type, + root_path=root.path, + scan_mode=root.scan_mode, + status="RUNNING", + parser_version=PARSER_VERSION, + needs_rescan=True, + metadata_json={"created_by": "asset_inventory_scan"}, + ) + db.add(state) + await db.flush() + state.status = "RUNNING" + state.root_path = root.path + state.scan_mode = root.scan_mode + state.last_scan_started_at = started_at + state.last_error = None + state.parser_version = PARSER_VERSION + state.updated_at = started_at + return state + + async def _finish_state( + self, + db: AsyncSession, + state: AssetInventoryStateORM, + *, + status: str, + started_at: datetime, + entry_count: int, + asset_count: int, + issue_count: int, + error: Optional[str], + ) -> None: + state.status = status + state.last_scan_started_at = started_at + state.last_scan_finished_at = _utcnow() + state.last_seen_entry_count = int(entry_count) + state.last_asset_count = int(asset_count) + state.last_issue_count = int(issue_count) + state.parser_version = PARSER_VERSION + state.needs_rescan = status not in {"OK", "WARNING"} + state.last_error = error + state.updated_at = _utcnow() + db.add(state) + + async def _replace_root_issues( + self, + db: AsyncSession, + root: ManagedRootORM, + inventory_type: str, + issues: Sequence[Dict[str, Any]], + ) -> None: + now = _utcnow() + await db.execute( + update(AssetInventoryIssueORM) + .where( + AssetInventoryIssueORM.root_ref_id == root.id, + AssetInventoryIssueORM.inventory_type == inventory_type, + AssetInventoryIssueORM.status == "OPEN", + ) + .values(status="RESOLVED", resolved_at=now, last_seen_at=now) + ) + for issue in issues: + db.add( + AssetInventoryIssueORM( + root_ref_id=root.id, + inventory_type=inventory_type, + severity=str(issue.get("severity") or "warning").lower(), + issue_code=str(issue.get("issue_code") or "unknown"), + issue_message=issue.get("issue_message"), + source_path=issue.get("source_path"), + status="OPEN", + first_seen_at=now, + last_seen_at=now, + metadata_json=issue.get("metadata_json"), + ) + ) + + async def _mark_missing_source_assets( + self, + db: AsyncSession, + root: ManagedRootORM, + seen_paths: Sequence[str], + now: datetime, + ) -> None: + stmt = update(SourceProductAssetORM).where(SourceProductAssetORM.root_ref_id == root.id) + if seen_paths: + stmt = stmt.where(SourceProductAssetORM.file_path.notin_(list(seen_paths))) + await db.execute(stmt.values(is_active=False, missing_since=now, updated_at=now)) + + async def _mark_missing_orbit_assets( + self, + db: AsyncSession, + root: ManagedRootORM, + seen_paths: Sequence[str], + now: datetime, + ) -> None: + stmt = update(OrbitAssetORM).where(OrbitAssetORM.root_ref_id == root.id) + if seen_paths: + stmt = stmt.where(OrbitAssetORM.file_path.notin_(list(seen_paths))) + await db.execute(stmt.values(is_active=False, missing_since=now, updated_at=now)) + + async def _upsert_radar_records_for_source_assets( + self, + db: AsyncSession, + rows: Sequence[Dict[str, Any]], + asset_ids_by_path: Dict[str, int], + ) -> None: + dirty_scene_ids: List[int] = [] + for row in rows: + metadata = dict(row.get("metadata_json") or {}) + coverage_polygon = metadata.get("coverage_polygon") + if not coverage_polygon or len(coverage_polygon) < 3: + continue + family = normalize_satellite_family(row.get("satellite_family") or row.get("satellite")) + if family not in {"S1", "LT1"}: + continue + + bbox = _bbox_from_polygon(coverage_polygon) + if not bbox: + continue + try: + poly = Polygon(coverage_polygon) + if not poly.is_valid: + poly = poly.buffer(0) + if poly.is_empty: + continue + except Exception: + continue + + asset_id = asset_ids_by_path.get(str(row.get("file_path"))) + if not asset_id: + continue + if await self._s1_zip_has_unpacked_safe(db, row): + continue + archive_asset_id = await self._resolve_archive_asset_id_for_source_row(db, row, asset_id) + center_lon, center_lat = _centroid_from_polygon(coverage_polygon) + metadata_center_lon = metadata.get("scene_center_lon") + metadata_center_lat = metadata.get("scene_center_lat") + if metadata_center_lon is not None: + center_lon = metadata_center_lon + if metadata_center_lat is not None: + center_lat = metadata_center_lat + + ready, reason = _insar_source_ready(row, coverage_polygon) + radar_values = { + "satellite": row.get("satellite") or "", + "satellite_family": family, + "imaging_date": row.get("imaging_date") or "", + "imaging_mode": row.get("imaging_mode") or "", + "orbit_direction": row.get("orbit_direction"), + "polarization": row.get("polarization") or "", + "satellite_mode": metadata.get("satellite_mode"), + "receiving_station": metadata.get("receiving_station"), + "orbit_circle": row.get("absolute_orbit"), + "scene_center_lon": center_lon, + "scene_center_lat": center_lat, + "acquisition_time_utc": ( + row.get("acquisition_start_time_utc").isoformat() + if row.get("acquisition_start_time_utc") + else None + ), + "product_type": row.get("product_type"), + "source_product_token": metadata.get("filename_class_token") or metadata.get("source_product_token"), + "image_data_type": "COMPLEX", + "image_data_format": "ZIP" if row.get("source_format") == "S1_ZIP" else "DIRECTORY", + "product_variant": metadata.get("product_variant"), + "product_level": row.get("product_level"), + "product_unique_id": row.get("logical_product_uid"), + "look_direction": metadata.get("look_direction"), + "acquisition_start_time_utc": row.get("acquisition_start_time_utc"), + "acquisition_stop_time_utc": row.get("acquisition_stop_time_utc"), + "absolute_orbit": row.get("absolute_orbit"), + "relative_orbit": row.get("relative_orbit"), + "source_format": row.get("source_format"), + "source_product_ref_id": asset_id, + "source_archive_asset_id": archive_asset_id, + "metadata_json": _json_safe(metadata), + "geocoded_flag": False, + "insar_source_ready": ready, + "insar_source_reason": reason, + "file_path": row.get("file_path"), + "coverage_polygon": coverage_polygon, + "geom": from_shape(poly, srid=4326), + "min_lon": bbox[0], + "min_lat": bbox[1], + "max_lon": bbox[2], + "max_lat": bbox[3], + } + + result = await db.execute( + self._radar_record_match_stmt(row, asset_id, archive_asset_id) + ) + existing = result.scalar_one_or_none() + if existing is None: + db.add( + RadarDataORM( + unique_id=f"asset:{row.get('asset_uid')}", + has_orbit_data=False, + orbit_binding_status="UNBOUND", + is_envi_processed=False, + **radar_values, + ) + ) + else: + before_orbit_id = existing.selected_orbit_asset_id + for key, value in radar_values.items(): + setattr(existing, key, value) + if not existing.orbit_binding_status: + existing.orbit_binding_status = "UNBOUND" + db.add(existing) + if existing.id is not None and before_orbit_id != existing.selected_orbit_asset_id: + dirty_scene_ids.append(int(existing.id)) + + await db.flush() + if dirty_scene_ids: + await pairing_state_service.mark_scenes_dirty(db, scene_ids=dirty_scene_ids, reason="asset_inventory_source_update", commit=False) + + async def _s1_zip_has_unpacked_safe(self, db: AsyncSession, row: Dict[str, Any]) -> bool: + if row.get("source_format") != "S1_ZIP": + return False + logical_uid = str(row.get("logical_product_uid") or "").strip() + if not logical_uid: + return False + result = await db.execute( + select(func.count(SourceProductAssetORM.id)).where( + SourceProductAssetORM.satellite_family == "S1", + SourceProductAssetORM.source_format == "S1_SAFE_DIR", + SourceProductAssetORM.logical_product_uid == logical_uid, + SourceProductAssetORM.is_active == True, # noqa: E712 + ) + ) + return int(result.scalar_one() or 0) > 0 + + async def _resolve_archive_asset_id_for_source_row( + self, + db: AsyncSession, + row: Dict[str, Any], + fallback_asset_id: int, + ) -> Optional[int]: + if row.get("source_format") == "S1_ZIP": + return fallback_asset_id + if row.get("source_format") != "S1_SAFE_DIR": + return None + logical_uid = str(row.get("logical_product_uid") or "").strip() + if not logical_uid: + return None + result = await db.execute( + select(SourceProductAssetORM.id).where( + SourceProductAssetORM.satellite_family == "S1", + SourceProductAssetORM.source_format == "S1_ZIP", + SourceProductAssetORM.logical_product_uid == logical_uid, + SourceProductAssetORM.is_active == True, # noqa: E712 + ) + ) + return result.scalar_one_or_none() + + def _radar_record_match_stmt( + self, + row: Dict[str, Any], + asset_id: int, + archive_asset_id: Optional[int], + ): + logical_uid = str(row.get("logical_product_uid") or "").strip() + clauses = [ + RadarDataORM.file_path == row.get("file_path"), + RadarDataORM.unique_id == f"asset:{row.get('asset_uid')}", + ] + if archive_asset_id is not None: + clauses.append(RadarDataORM.source_archive_asset_id == int(archive_asset_id)) + if row.get("source_format") == "S1_SAFE_DIR" and logical_uid: + clauses.append( + and_( + RadarDataORM.satellite_family == "S1", + RadarDataORM.product_unique_id == logical_uid, + ) + ) + elif row.get("source_format") == "S1_ZIP" and logical_uid: + clauses.append( + and_( + RadarDataORM.satellite_family == "S1", + RadarDataORM.product_unique_id == logical_uid, + RadarDataORM.source_archive_asset_id == int(asset_id), + ) + ) + return select(RadarDataORM).where(or_(*clauses)) + + async def bind_scene_orbits( + self, + db: AsyncSession, + radar_data_ids: Optional[Sequence[int]] = None, + ) -> Dict[str, Any]: + now = _utcnow() + scene_stmt = select(RadarDataORM).where( + RadarDataORM.satellite_family.in_(["S1", "LT1"]), + RadarDataORM.source_product_ref_id.is_not(None), + ) + if radar_data_ids: + scene_stmt = scene_stmt.where(RadarDataORM.id.in_([int(item) for item in radar_data_ids])) + scene_result = await db.execute(scene_stmt) + scenes = scene_result.scalars().all() + issue_stmt = update(AssetInventoryIssueORM).where( + AssetInventoryIssueORM.issue_code.in_(["scene_missing_orbit", "scene_ambiguous_orbit"]), + AssetInventoryIssueORM.status == "OPEN", + ) + if radar_data_ids: + issue_stmt = issue_stmt.where(AssetInventoryIssueORM.radar_data_id.in_([int(item) for item in radar_data_ids])) + await db.execute(issue_stmt.values(status="RESOLVED", resolved_at=now, last_seen_at=now)) + if scenes: + await db.execute(delete(SceneOrbitBindingORM).where(SceneOrbitBindingORM.radar_data_id.in_([scene.id for scene in scenes if scene.id]))) + + matched = 0 + missing = 0 + candidate_count = 0 + dirty_scene_ids: List[int] = [] + for scene in scenes: + candidates = await self._find_orbit_candidates(db, scene) + if not candidates: + scene.has_orbit_data = False + scene.orbit_file_path = None + scene.selected_orbit_asset_id = None + scene.orbit_binding_status = "MISSING" + scene.orbit_binding_reason = "No active orbit asset covers the scene acquisition window." + missing += 1 + db.add( + AssetInventoryIssueORM( + inventory_type="orbit_asset", + radar_data_id=scene.id, + severity="warning", + issue_code="scene_missing_orbit", + issue_message=scene.orbit_binding_reason, + source_path=scene.file_path, + status="OPEN", + first_seen_at=now, + last_seen_at=now, + metadata_json={ + "satellite": scene.satellite, + "imaging_date": scene.imaging_date, + "acquisition_start_time_utc": scene.acquisition_start_time_utc.isoformat() + if scene.acquisition_start_time_utc + else None, + }, + ) + ) + if scene.id is not None: + dirty_scene_ids.append(int(scene.id)) + continue + + candidate_count += len(candidates) + selected = candidates[0] + for rank, (orbit, score, reason, margins, rule_version) in enumerate(candidates, start=1): + db.add( + SceneOrbitBindingORM( + radar_data_id=scene.id, + orbit_asset_id=orbit.id, + binding_role="primary_orbit", + match_status="MATCHED", + selection_status="SELECTED" if rank == 1 else "CANDIDATE", + selection_rank=rank, + priority_score=score, + coverage_margin_before_seconds=margins[0], + coverage_margin_after_seconds=margins[1], + match_rule_version=rule_version, + match_reason=reason, + selected_at=now if rank == 1 else None, + ) + ) + selected_orbit = selected[0] + scene.has_orbit_data = True + scene.orbit_file_path = selected_orbit.file_path + scene.selected_orbit_asset_id = selected_orbit.id + scene.orbit_binding_status = "MATCHED" + scene.orbit_binding_reason = selected[2] + db.add(scene) + matched += 1 + if scene.id is not None: + dirty_scene_ids.append(int(scene.id)) + + if len(candidates) > 1 and abs(float(candidates[0][1]) - float(candidates[1][1])) < 0.001: + db.add( + AssetInventoryIssueORM( + inventory_type="orbit_asset", + radar_data_id=scene.id, + orbit_asset_id=selected_orbit.id, + severity="warning", + issue_code="scene_ambiguous_orbit", + issue_message="Multiple orbit assets have equivalent selection priority.", + source_path=scene.file_path, + status="OPEN", + first_seen_at=now, + last_seen_at=now, + ) + ) + + if dirty_scene_ids: + await pairing_state_service.mark_scenes_dirty( + db, + scene_ids=sorted(set(dirty_scene_ids)), + reason="asset_inventory_orbit_binding", + commit=False, + ) + return { + "scene_count": len(scenes), + "matched_count": matched, + "missing_count": missing, + "candidate_count": candidate_count, + } + + async def _find_orbit_candidates( + self, + db: AsyncSession, + scene: RadarDataORM, + ) -> List[Tuple[OrbitAssetORM, float, str, Tuple[Optional[float], Optional[float]], str]]: + family = normalize_satellite_family(scene.satellite_family or scene.satellite) + satellite = str(scene.satellite or "").upper() + if family == "S1": + scene_start = scene.acquisition_start_time_utc + scene_stop = scene.acquisition_stop_time_utc or scene_start + if not scene_start or not scene_stop: + return [] + result = await db.execute( + select(OrbitAssetORM).where( + OrbitAssetORM.is_active == True, # noqa: E712 + OrbitAssetORM.satellite_family == "S1", + OrbitAssetORM.satellite == satellite, + OrbitAssetORM.validity_start_time_utc <= scene_start, + OrbitAssetORM.validity_stop_time_utc >= scene_stop, + ) + ) + rows = result.scalars().all() + candidates = [] + for orbit in rows: + before = (scene_start - orbit.validity_start_time_utc).total_seconds() if orbit.validity_start_time_utc else None + after = (orbit.validity_stop_time_utc - scene_stop).total_seconds() if orbit.validity_stop_time_utc else None + quality_score = 1000.0 if orbit.quality_class == "precise" else 500.0 if orbit.quality_class == "restituted" else 100.0 + margin_score = min(before or 0.0, after or 0.0) / 100000.0 + generation_score = (orbit.generation_time_utc.timestamp() / 1000000000.0) if orbit.generation_time_utc else 0.0 + score = quality_score + margin_score + generation_score + reason = ( + f"{orbit.orbit_type} covers scene window " + f"{scene_start.isoformat()} to {scene_stop.isoformat()}" + ) + candidates.append((orbit, score, reason, (before, after), S1_ORBIT_MATCH_RULE_VERSION)) + return sorted(candidates, key=lambda item: item[1], reverse=True) + + if family == "LT1": + if not scene.imaging_date: + return [] + day_start, day_stop = _date_start_stop(str(scene.imaging_date)) + if not day_start or not day_stop: + return [] + result = await db.execute( + select(OrbitAssetORM).where( + OrbitAssetORM.is_active == True, # noqa: E712 + OrbitAssetORM.satellite_family == "LT1", + OrbitAssetORM.satellite == satellite, + OrbitAssetORM.validity_start_time_utc <= day_start, + OrbitAssetORM.validity_stop_time_utc >= day_stop, + ) + ) + rows = result.scalars().all() + candidates = [] + for orbit in rows: + score = 1000.0 + reason = f"LT1 orbit date matches scene imaging_date {scene.imaging_date}" + candidates.append((orbit, score, reason, (0.0, 0.0), LT1_ORBIT_MATCH_RULE_VERSION)) + return sorted(candidates, key=lambda item: item[1], reverse=True) + + return [] + + async def get_status(self, db: AsyncSession) -> Dict[str, Any]: + state_rows = ( + await db.execute( + select(AssetInventoryStateORM) + .join(ManagedRootORM, AssetInventoryStateORM.root_ref_id == ManagedRootORM.id) + .order_by(AssetInventoryStateORM.inventory_type.asc(), AssetInventoryStateORM.root_path.asc()) + ) + ).scalars().all() + source_count = int( + ( + await db.execute( + select(func.count(SourceProductAssetORM.id)).where( + SourceProductAssetORM.is_active == True, # noqa: E712 + SourceProductAssetORM.source_format != "S1_ZIP", + ) + ) + ).scalar_one() + or 0 + ) + orbit_count = int((await db.execute(select(func.count(OrbitAssetORM.id)).where(OrbitAssetORM.is_active == True))).scalar_one() or 0) # noqa: E712 + binding_count = int((await db.execute(select(func.count(SceneOrbitBindingORM.id)).where(SceneOrbitBindingORM.selection_status == "SELECTED"))).scalar_one() or 0) + open_issue_count = int((await db.execute(select(func.count(AssetInventoryIssueORM.id)).where(AssetInventoryIssueORM.status == "OPEN"))).scalar_one() or 0) + + return { + "source_asset_count": source_count, + "orbit_asset_count": orbit_count, + "selected_binding_count": binding_count, + "open_issue_count": open_issue_count, + "states": [ + { + "id": row.id, + "root_ref_id": row.root_ref_id, + "inventory_type": row.inventory_type, + "root_path": row.root_path, + "scan_mode": row.scan_mode, + "status": row.status, + "last_scan_started_at": row.last_scan_started_at, + "last_scan_finished_at": row.last_scan_finished_at, + "last_seen_entry_count": row.last_seen_entry_count, + "last_asset_count": row.last_asset_count, + "last_issue_count": row.last_issue_count, + "parser_version": row.parser_version, + "needs_rescan": bool(row.needs_rescan), + "last_error": row.last_error, + } + for row in state_rows + ], + } + + async def list_source_products( + self, + db: AsyncSession, + *, + satellite_family: Optional[str] = None, + satellite: Optional[str] = None, + source_format: Optional[str] = None, + parse_status: Optional[str] = None, + include_inactive: bool = False, + limit: int = 200, + offset: int = 0, + ) -> Dict[str, Any]: + safe_limit = max(1, min(int(limit or 200), 1000)) + safe_offset = max(0, int(offset or 0)) + filters = [] + if not include_inactive: + filters.append(SourceProductAssetORM.is_active == True) # noqa: E712 + filters.append(SourceProductAssetORM.source_format != "S1_ZIP") + if satellite_family: + filters.append(SourceProductAssetORM.satellite_family == satellite_family.upper()) + if satellite: + filters.append(SourceProductAssetORM.satellite == satellite.upper()) + if source_format: + filters.append(SourceProductAssetORM.source_format == source_format.upper()) + if parse_status: + filters.append(SourceProductAssetORM.parse_status == parse_status.upper()) + stmt = select(SourceProductAssetORM) + count_stmt = select(func.count(SourceProductAssetORM.id)) + for item in filters: + stmt = stmt.where(item) + count_stmt = count_stmt.where(item) + total = int((await db.execute(count_stmt)).scalar_one() or 0) + rows = ( + await db.execute( + stmt.order_by(SourceProductAssetORM.acquisition_start_time_utc.desc().nullslast(), SourceProductAssetORM.id.desc()) + .offset(safe_offset) + .limit(safe_limit) + ) + ).scalars().all() + return { + "items": [self._source_asset_payload(row) for row in rows], + "total": total, + "limit": safe_limit, + "offset": safe_offset, + "has_more": safe_offset + len(rows) < total, + } + + async def list_orbits( + self, + db: AsyncSession, + *, + satellite_family: Optional[str] = None, + satellite: Optional[str] = None, + orbit_type: Optional[str] = None, + parse_status: Optional[str] = None, + include_inactive: bool = False, + limit: int = 200, + offset: int = 0, + ) -> Dict[str, Any]: + safe_limit = max(1, min(int(limit or 200), 1000)) + safe_offset = max(0, int(offset or 0)) + filters = [] + if not include_inactive: + filters.append(OrbitAssetORM.is_active == True) # noqa: E712 + if satellite_family: + filters.append(OrbitAssetORM.satellite_family == satellite_family.upper()) + if satellite: + filters.append(OrbitAssetORM.satellite == satellite.upper()) + if orbit_type: + filters.append(OrbitAssetORM.orbit_type == orbit_type.upper()) + if parse_status: + filters.append(OrbitAssetORM.parse_status == parse_status.upper()) + stmt = select(OrbitAssetORM) + count_stmt = select(func.count(OrbitAssetORM.id)) + for item in filters: + stmt = stmt.where(item) + count_stmt = count_stmt.where(item) + total = int((await db.execute(count_stmt)).scalar_one() or 0) + rows = ( + await db.execute( + stmt.order_by(OrbitAssetORM.validity_start_time_utc.desc().nullslast(), OrbitAssetORM.id.desc()) + .offset(safe_offset) + .limit(safe_limit) + ) + ).scalars().all() + return { + "items": [self._orbit_asset_payload(row) for row in rows], + "total": total, + "limit": safe_limit, + "offset": safe_offset, + "has_more": safe_offset + len(rows) < total, + } + + async def list_issues( + self, + db: AsyncSession, + *, + status: str = "OPEN", + severity: Optional[str] = None, + issue_code: Optional[str] = None, + limit: int = 200, + offset: int = 0, + ) -> Dict[str, Any]: + safe_limit = max(1, min(int(limit or 200), 1000)) + safe_offset = max(0, int(offset or 0)) + filters = [] + if status: + filters.append(AssetInventoryIssueORM.status == status.upper()) + if severity: + filters.append(AssetInventoryIssueORM.severity == severity.lower()) + if issue_code: + filters.append(AssetInventoryIssueORM.issue_code == issue_code) + stmt = select(AssetInventoryIssueORM) + count_stmt = select(func.count(AssetInventoryIssueORM.id)) + for item in filters: + stmt = stmt.where(item) + count_stmt = count_stmt.where(item) + total = int((await db.execute(count_stmt)).scalar_one() or 0) + rows = ( + await db.execute( + stmt.order_by(AssetInventoryIssueORM.last_seen_at.desc(), AssetInventoryIssueORM.id.desc()) + .offset(safe_offset) + .limit(safe_limit) + ) + ).scalars().all() + return { + "items": [ + { + "id": row.id, + "root_ref_id": row.root_ref_id, + "inventory_type": row.inventory_type, + "asset_ref_id": row.asset_ref_id, + "radar_data_id": row.radar_data_id, + "orbit_asset_id": row.orbit_asset_id, + "severity": row.severity, + "issue_code": row.issue_code, + "issue_message": row.issue_message, + "source_path": row.source_path, + "status": row.status, + "first_seen_at": row.first_seen_at, + "last_seen_at": row.last_seen_at, + "resolved_at": row.resolved_at, + "metadata_json": row.metadata_json, + } + for row in rows + ], + "total": total, + "limit": safe_limit, + "offset": safe_offset, + "has_more": safe_offset + len(rows) < total, + } + + def _source_asset_payload(self, row: SourceProductAssetORM) -> Dict[str, Any]: + return { + "id": row.id, + "asset_uid": row.asset_uid, + "logical_product_uid": row.logical_product_uid, + "satellite_family": row.satellite_family, + "satellite": row.satellite, + "source_format": row.source_format, + "product_type": row.product_type, + "product_level": row.product_level, + "imaging_mode": row.imaging_mode, + "polarization": row.polarization, + "absolute_orbit": row.absolute_orbit, + "relative_orbit": row.relative_orbit, + "orbit_direction": row.orbit_direction, + "acquisition_start_time_utc": row.acquisition_start_time_utc, + "acquisition_stop_time_utc": row.acquisition_stop_time_utc, + "imaging_date": row.imaging_date, + "root_ref_id": row.root_ref_id, + "root_path": row.root_path, + "file_path": row.file_path, + "archive_path": row.archive_path, + "size_bytes": row.size_bytes, + "mtime_epoch": row.mtime_epoch, + "checksum_status": row.checksum_status, + "parser_name": row.parser_name, + "parser_version": row.parser_version, + "parse_status": row.parse_status, + "parse_error": row.parse_error, + "parsed_at": row.parsed_at, + "metadata_json": row.metadata_json, + "is_active": bool(row.is_active), + "missing_since": row.missing_since, + "created_at": row.created_at, + "updated_at": row.updated_at, + } + + def _orbit_asset_payload(self, row: OrbitAssetORM) -> Dict[str, Any]: + return { + "id": row.id, + "orbit_uid": row.orbit_uid, + "satellite_family": row.satellite_family, + "satellite": row.satellite, + "orbit_type": row.orbit_type, + "native_format": row.native_format, + "quality_class": row.quality_class, + "root_ref_id": row.root_ref_id, + "root_path": row.root_path, + "file_path": row.file_path, + "file_name": row.file_name, + "size_bytes": row.size_bytes, + "mtime_epoch": row.mtime_epoch, + "checksum_status": row.checksum_status, + "validity_start_time_utc": row.validity_start_time_utc, + "validity_stop_time_utc": row.validity_stop_time_utc, + "generation_time_utc": row.generation_time_utc, + "published_time_utc": row.published_time_utc, + "parser_name": row.parser_name, + "parser_version": row.parser_version, + "parse_status": row.parse_status, + "parse_error": row.parse_error, + "parsed_at": row.parsed_at, + "metadata_json": row.metadata_json, + "is_active": bool(row.is_active), + "missing_since": row.missing_since, + "created_at": row.created_at, + "updated_at": row.updated_at, + } + + +asset_inventory_service = AssetInventoryService() diff --git a/backend/app/services/data_service.py b/backend/app/services/data_service.py index b754ecd..8cc7e6a 100644 --- a/backend/app/services/data_service.py +++ b/backend/app/services/data_service.py @@ -238,7 +238,7 @@ def _chunked(items: List[Any], size: int): _RADAR_PREVIEW_EXTENSIONS = (".jpg", ".jpeg", ".png", ".webp", ".bmp", ".tif", ".tiff") -_RADAR_PREVIEW_KEYWORDS = ("quicklook", "preview", "browse", "thumbnail", "thumb", "overview") +_RADAR_PREVIEW_KEYWORDS = ("quicklook", "quick-look", "preview", "browse", "thumbnail", "thumb", "overview") _RADAR_CACHE_NAME_RE = re.compile(r"[^A-Za-z0-9._-]+") @@ -396,7 +396,18 @@ class DataService: continue path = os.path.join(root, name) + root_lower = root.lower() keyword_score = 0 if any(key in lower_name for key in _RADAR_PREVIEW_KEYWORDS) else 1 + if lower_name == "quick-look.png": + keyword_score = -3 + elif lower_name == "quicklook.png": + keyword_score = -2 + elif lower_name.startswith("quick-look.") or lower_name.startswith("quicklook."): + keyword_score = min(keyword_score, -1) + if f"{os.sep}preview" in root_lower: + keyword_score -= 1 + if f"{os.sep}icons" in root_lower: + keyword_score += 2 ext_score = 0 if lower_name.endswith((".jpg", ".jpeg")) else 1 try: size_score = -os.path.getsize(path) diff --git a/backend/app/services/dinsar_naming.py b/backend/app/services/dinsar_naming.py index ee6183c..cdc1929 100644 --- a/backend/app/services/dinsar_naming.py +++ b/backend/app/services/dinsar_naming.py @@ -7,6 +7,8 @@ import re from datetime import datetime from typing import Any, Dict, Optional +from ..utils import normalize_satellite_family + PAIR_META_FILENAME = ".dinsar_pair.json" RUN_META_FILENAME = ".dinsar_run.json" @@ -48,6 +50,7 @@ def build_pair_key( slave_path: Any, master_date: Any = None, slave_date: Any = None, + satellite_family: Any = None, ) -> str: master_date_text = _normalize_date(master_date) slave_date_text = _normalize_date(slave_date) @@ -60,17 +63,23 @@ def build_pair_key( ] ) digest = hashlib.sha1(payload.encode("utf-8", errors="ignore")).hexdigest()[:10] - return f"lt1_{master_date_text}_{slave_date_text}_{digest}" + family = str(normalize_satellite_family(satellite_family) or "").strip().lower() + if not family: + family = "pair" + return f"{family}_{master_date_text}_{slave_date_text}_{digest}" -def build_fallback_pair_key(task_alias: Any, source_hint: Any = None) -> str: +def build_fallback_pair_key(task_alias: Any, source_hint: Any = None, satellite_family: Any = None) -> str: alias = str(task_alias or "").strip() or "Task_unknown_unknown" parts = alias.split("_") master_date = parts[1] if len(parts) > 2 else "unknown" slave_date = parts[2] if len(parts) > 2 else "unknown" payload = "||".join([alias, normalize_path(source_hint)]) digest = hashlib.sha1(payload.encode("utf-8", errors="ignore")).hexdigest()[:10] - return f"lt1_{_normalize_date(master_date)}_{_normalize_date(slave_date)}_{digest}" + family = str(normalize_satellite_family(satellite_family) or "").strip().lower() + if not family: + family = "pair" + return f"{family}_{_normalize_date(master_date)}_{_normalize_date(slave_date)}_{digest}" def build_run_key( diff --git a/backend/app/services/envi_service.py b/backend/app/services/envi_service.py index d7383ab..54756c1 100644 --- a/backend/app/services/envi_service.py +++ b/backend/app/services/envi_service.py @@ -21,6 +21,7 @@ from typing import Any, Dict, List, Optional from ..config import get_env_text, settings from ..process_utils import is_any_process_running +from ..utils import normalize_satellite_family from .dinsar_naming import ( PAIR_META_FILENAME, build_fallback_pair_key, @@ -480,7 +481,14 @@ def _utc_now_text() -> str: def _resolve_dinsar_pair_identity(task_dir: str, task_name: str) -> tuple[str, str, Dict[str, Any]]: pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {} task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name - pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(task_alias, task_dir) + satellite_family = normalize_satellite_family( + pair_meta.get("master_satellite") or pair_meta.get("slave_satellite") + ) + pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key( + task_alias, + task_dir, + satellite_family=satellite_family, + ) return task_alias, pair_key, pair_meta diff --git a/backend/app/services/health_service.py b/backend/app/services/health_service.py index 69870a0..dc92526 100644 --- a/backend/app/services/health_service.py +++ b/backend/app/services/health_service.py @@ -1,7 +1,7 @@ import os import asyncio from datetime import datetime, timedelta -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional import httpx from sqlalchemy import and_, func, literal, or_, select, text @@ -11,9 +11,14 @@ from ..config import read_int_env, settings, split_env_paths from ..db_maintenance import inspect_database_structure from ..models import ( AiDiagnosisORM, + AssetInventoryIssueORM, + AssetInventoryStateORM, DinsarResultORM, + OrbitAssetORM, ResultCatalogStateORM, ResultProductORM, + SceneOrbitBindingORM, + SourceProductAssetORM, SystemWorkerHeartbeatORM, ) from ..idl_service import get_idl_status @@ -399,6 +404,51 @@ def _sanitize_pairing_system_status(payload: Dict[str, Any]) -> Dict[str, Any]: } +def _sanitize_asset_inventory_status(payload: Dict[str, Any]) -> Dict[str, Any]: + source_roots = payload.get("source_roots", {}) or {} + orbit_roots = payload.get("orbit_roots", {}) or {} + source_assets = payload.get("source_assets", {}) or {} + orbit_assets = payload.get("orbit_assets", {}) or {} + bindings = payload.get("bindings", {}) or {} + issues = payload.get("issues", {}) or {} + return { + "ok": bool(payload.get("ok")), + "source_roots": { + "configured_count": int(source_roots.get("configured_count") or 0), + "accessible_count": int(source_roots.get("accessible_count") or 0), + "needs_rescan_count": int(source_roots.get("needs_rescan_count") or 0), + }, + "orbit_roots": { + "configured_count": int(orbit_roots.get("configured_count") or 0), + "accessible_count": int(orbit_roots.get("accessible_count") or 0), + "needs_rescan_count": int(orbit_roots.get("needs_rescan_count") or 0), + }, + "source_assets": { + "total_count": int(source_assets.get("total_count") or 0), + "lt1_count": int(source_assets.get("lt1_count") or 0), + "s1_count": int(source_assets.get("s1_count") or 0), + "parse_failed_count": int(source_assets.get("parse_failed_count") or 0), + }, + "orbit_assets": { + "total_count": int(orbit_assets.get("total_count") or 0), + "lt1_count": int(orbit_assets.get("lt1_count") or 0), + "s1_count": int(orbit_assets.get("s1_count") or 0), + "parse_failed_count": int(orbit_assets.get("parse_failed_count") or 0), + }, + "bindings": { + "scene_count": int(bindings.get("scene_count") or 0), + "matched_count": int(bindings.get("matched_count") or 0), + "missing_count": int(bindings.get("missing_count") or 0), + "ambiguous_count": int(bindings.get("ambiguous_count") or 0), + }, + "issues": { + "open_count": int(issues.get("open_count") or 0), + "error_count": int(issues.get("error_count") or 0), + "warning_count": int(issues.get("warning_count") or 0), + }, + } + + def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]: database = payload.get("database", {}) or {} worker = payload.get("worker", {}) or {} @@ -411,6 +461,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]: dinsar_bridge = payload.get("dinsar_bridge", {}) or {} source_roots = payload.get("source_roots", {}) or {} product_packages = payload.get("product_packages", {}) or {} + asset_inventory = payload.get("asset_inventory", {}) or {} wsl_runtime = payload.get("wsl_runtime", {}) or {} pairing_system = payload.get("pairing_system", {}) or {} idl = payload.get("idl", {}) or {} @@ -423,6 +474,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]: sanitized_dinsar_bridge = _sanitize_bridge_status(dinsar_bridge) sanitized_source_roots = _sanitize_source_roots_status(source_roots) sanitized_product_packages = _sanitize_product_package_status(product_packages) + sanitized_asset_inventory = _sanitize_asset_inventory_status(asset_inventory) sanitized_wsl_runtime = _sanitize_wsl_runtime_status(wsl_runtime) sanitized_pairing_system = _sanitize_pairing_system_status(pairing_system) @@ -451,6 +503,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]: "dinsar_bridge": sanitized_dinsar_bridge, "source_roots": sanitized_source_roots, "product_packages": sanitized_product_packages, + "asset_inventory": sanitized_asset_inventory, "wsl_runtime": sanitized_wsl_runtime, "pairing_system": sanitized_pairing_system, "idl": { @@ -905,6 +958,221 @@ async def _check_product_packages() -> Dict[str, Any]: return status +async def _check_asset_inventory() -> Dict[str, Any]: + status: Dict[str, Any] = { + "ok": False, + "source_roots": { + "configured_count": 0, + "accessible_count": 0, + "inaccessible_count": 0, + "needs_rescan_count": 0, + "items": [], + }, + "orbit_roots": { + "configured_count": 0, + "accessible_count": 0, + "inaccessible_count": 0, + "needs_rescan_count": 0, + "items": [], + }, + "source_assets": { + "total_count": 0, + "lt1_count": 0, + "s1_count": 0, + "parse_failed_count": 0, + "by_family": {}, + }, + "orbit_assets": { + "total_count": 0, + "lt1_count": 0, + "s1_count": 0, + "parse_failed_count": 0, + "by_family": {}, + }, + "bindings": { + "scene_count": 0, + "matched_count": 0, + "missing_count": 0, + "ambiguous_count": 0, + }, + "issues": { + "open_count": 0, + "error_count": 0, + "warning_count": 0, + "by_code": {}, + }, + "error": None, + } + + source_paths: List[str] = [] + for value in ( + settings.SOURCE_PRODUCT_DIRS, + settings.INSAR_STORAGE_DIRS, + settings.MONITOR_RADAR_DIRS, + ): + for path in split_env_paths(value): + if path not in source_paths: + source_paths.append(path) + + orbit_paths: List[str] = [] + for value in ( + settings.ORBIT_SOURCE_DIRS, + settings.MONITOR_ORBIT_DIR, + ): + for path in split_env_paths(value): + if path not in orbit_paths: + orbit_paths.append(path) + + for path in source_paths: + item = _probe_directory_status(path) + item["role"] = "source_product_pool" + status["source_roots"]["items"].append(item) + for path in orbit_paths: + item = _probe_directory_status(path) + item["role"] = "orbit_asset_pool" + status["orbit_roots"]["items"].append(item) + + for key in ("source_roots", "orbit_roots"): + root_status = status[key] + root_status["configured_count"] = len(root_status["items"]) + root_status["accessible_count"] = sum(1 for item in root_status["items"] if item.get("accessible")) + root_status["inaccessible_count"] = root_status["configured_count"] - root_status["accessible_count"] + + try: + session_factory = _get_session_factory() + async with session_factory() as db: + state_rows = ( + await db.execute( + select( + AssetInventoryStateORM.inventory_type, + func.count(AssetInventoryStateORM.id), + ) + .where(AssetInventoryStateORM.needs_rescan == True) # noqa: E712 + .group_by(AssetInventoryStateORM.inventory_type) + ) + ).all() + for inventory_type, count in state_rows: + key = "orbit_roots" if str(inventory_type or "").lower().startswith("orbit") else "source_roots" + status[key]["needs_rescan_count"] += int(count or 0) + + source_family_rows = ( + await db.execute( + select( + SourceProductAssetORM.satellite_family, + func.count(SourceProductAssetORM.id), + ) + .where( + SourceProductAssetORM.is_active == True, # noqa: E712 + SourceProductAssetORM.source_format != "S1_ZIP", + ) + .group_by(SourceProductAssetORM.satellite_family) + ) + ).all() + for family, count in source_family_rows: + family_key = str(family or "unknown").strip().upper() or "unknown" + value = int(count or 0) + status["source_assets"]["by_family"][family_key] = value + status["source_assets"]["total_count"] += value + status["source_assets"]["lt1_count"] = int(status["source_assets"]["by_family"].get("LT1", 0)) + status["source_assets"]["s1_count"] = int(status["source_assets"]["by_family"].get("S1", 0)) + source_parse_failed = await db.execute( + select(func.count(SourceProductAssetORM.id)).where( + SourceProductAssetORM.parse_status == "FAILED", + SourceProductAssetORM.source_format != "S1_ZIP", + ) + ) + status["source_assets"]["parse_failed_count"] = int(source_parse_failed.scalar_one() or 0) + + orbit_family_rows = ( + await db.execute( + select( + OrbitAssetORM.satellite_family, + func.count(OrbitAssetORM.id), + ) + .where(OrbitAssetORM.is_active == True) # noqa: E712 + .group_by(OrbitAssetORM.satellite_family) + ) + ).all() + for family, count in orbit_family_rows: + family_key = str(family or "unknown").strip().upper() or "unknown" + value = int(count or 0) + status["orbit_assets"]["by_family"][family_key] = value + status["orbit_assets"]["total_count"] += value + status["orbit_assets"]["lt1_count"] = int(status["orbit_assets"]["by_family"].get("LT1", 0)) + status["orbit_assets"]["s1_count"] = int(status["orbit_assets"]["by_family"].get("S1", 0)) + orbit_parse_failed = await db.execute( + select(func.count(OrbitAssetORM.id)).where(OrbitAssetORM.parse_status == "FAILED") + ) + status["orbit_assets"]["parse_failed_count"] = int(orbit_parse_failed.scalar_one() or 0) + + scene_count = await db.execute(select(func.count(SceneOrbitBindingORM.radar_data_id.distinct()))) + status["bindings"]["scene_count"] = int(scene_count.scalar_one() or 0) + selected_count = await db.execute( + select(func.count(SceneOrbitBindingORM.id)).where(SceneOrbitBindingORM.selection_status == "SELECTED") + ) + status["bindings"]["matched_count"] = int(selected_count.scalar_one() or 0) + missing_count = await db.execute( + select(func.count(AssetInventoryIssueORM.id)).where( + AssetInventoryIssueORM.status == "OPEN", + AssetInventoryIssueORM.issue_code == "scene_missing_orbit", + ) + ) + status["bindings"]["missing_count"] = int(missing_count.scalar_one() or 0) + ambiguous_count = await db.execute( + select(func.count(AssetInventoryIssueORM.id)).where( + AssetInventoryIssueORM.status == "OPEN", + AssetInventoryIssueORM.issue_code == "scene_ambiguous_orbit", + ) + ) + status["bindings"]["ambiguous_count"] = int(ambiguous_count.scalar_one() or 0) + + issue_rows = ( + await db.execute( + select( + AssetInventoryIssueORM.severity, + func.count(AssetInventoryIssueORM.id), + ) + .where(AssetInventoryIssueORM.status == "OPEN") + .group_by(AssetInventoryIssueORM.severity) + ) + ).all() + for severity, count in issue_rows: + severity_key = str(severity or "warning").strip().lower() or "warning" + value = int(count or 0) + status["issues"]["open_count"] += value + if severity_key == "error": + status["issues"]["error_count"] += value + elif severity_key == "warning": + status["issues"]["warning_count"] += value + + issue_code_rows = ( + await db.execute( + select( + AssetInventoryIssueORM.issue_code, + func.count(AssetInventoryIssueORM.id), + ) + .where(AssetInventoryIssueORM.status == "OPEN") + .group_by(AssetInventoryIssueORM.issue_code) + ) + ).all() + status["issues"]["by_code"] = { + str(code or "unknown"): int(count or 0) + for code, count in issue_code_rows + } + + status["ok"] = ( + status["source_roots"]["inaccessible_count"] == 0 + and status["orbit_roots"]["inaccessible_count"] == 0 + and status["source_assets"]["parse_failed_count"] == 0 + and status["orbit_assets"]["parse_failed_count"] == 0 + and status["issues"]["error_count"] == 0 + ) + except Exception as exc: + status["error"] = str(exc) + + return status + + async def _check_wsl_runtime() -> Dict[str, Any]: status = { "ok": False, @@ -989,6 +1257,7 @@ async def get_health_status( dinsar_bridge_status = await _check_dinsar_bridge() source_roots_status = await _check_source_roots() product_packages_status = await _check_product_packages() + asset_inventory_status = await _check_asset_inventory() wsl_runtime_status = await _check_wsl_runtime() pairing_system_status = await pairing_state_service.get_pairing_system_status() engines_status = {"ok": None, "overall": None, "engines": []} @@ -1005,6 +1274,7 @@ async def get_health_status( dinsar_bridge_status.get("ok"), source_roots_status.get("ok"), product_packages_status.get("ok"), + asset_inventory_status.get("ok"), wsl_runtime_status.get("ok"), pairing_system_status.get("ok"), (not settings.TIMESERIES_ENABLED) or timeseries_result_catalog_status.get("ok"), @@ -1028,6 +1298,7 @@ async def get_health_status( "dinsar_bridge": dinsar_bridge_status, "source_roots": source_roots_status, "product_packages": product_packages_status, + "asset_inventory": asset_inventory_status, "wsl_runtime": wsl_runtime_status, "pairing_system": pairing_system_status, "idl": { diff --git a/backend/app/services/job_handlers.py b/backend/app/services/job_handlers.py index 987691a..d4822e2 100644 --- a/backend/app/services/job_handlers.py +++ b/backend/app/services/job_handlers.py @@ -22,6 +22,7 @@ from ..config import settings from ..models import SystemJobORM, DinsarResultORM, HazardPointORM, DinsarTaskItemORM, PsTaskItemORM, RadarDataORM, SARSceneGeoORM, FloodDetectionORM, WaterDetectionORM, GF3ProcessingORM, AiDiagnosisORM from ..scheduler import scan_data_job from .data_service import data_service +from .asset_inventory_service import asset_inventory_service from .dinsar_compat_service import dinsar_compat_service from .dinsar_naming import build_run_key from .dinsar_production_service import dinsar_production_service @@ -67,6 +68,7 @@ JOB_TYPE_SCAN_DATA = "SCAN_DATA" JOB_TYPE_SCAN_DINSAR = "SCAN_DINSAR" JOB_TYPE_COPY_DATA = "COPY_DATA" JOB_TYPE_UNPACK = "UNPACK_ARCHIVES" +JOB_TYPE_UNPACK_SENTINEL1 = "UNPACK_SENTINEL1" JOB_TYPE_AI_TRAIN = "AI_TRAIN" JOB_TYPE_AI_PREDICT = "AI_PREDICT" JOB_TYPE_AI_ANALYZE = "AI_ANALYZE" @@ -85,6 +87,7 @@ JOB_TYPE_PYINT_RUN = "PYINT_RUN" JOB_TYPE_PUBLISH_DINSAR_PRODUCTS = "PUBLISH_DINSAR_PRODUCTS" JOB_TYPE_REBUILD_DINSAR_CATALOG = "REBUILD_DINSAR_CATALOG" JOB_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG" +JOB_TYPE_SCAN_ASSET_INVENTORY = "SCAN_ASSET_INVENTORY" COPY_ALLOWED_STATUSES = {"PENDING", "IN_PROGRESS", "COMPLETED", "FAILED"} @@ -225,6 +228,34 @@ async def _handle_scan_data(job: SystemJobORM) -> None: await _run_scan_data_custom(job.task_id, payload) +async def _handle_scan_asset_inventory(job: SystemJobORM) -> None: + if not job.task_id: + raise ValueError("SCAN_ASSET_INVENTORY requires task_id for progress tracking.") + payload = job.payload or {} + await task_service.start_task(job.task_id, message="Source/orbit asset inventory scan started") + async with AsyncSessionLocal() as db: + result = await asset_inventory_service.scan_configured_roots( + db, + inventory_types=payload.get("inventory_types") or None, + root_ids=payload.get("root_ids") or None, + bind_orbits=bool(payload.get("bind_orbits", True)), + task_id=job.task_id, + ) + await task_service.update_task( + job.task_id, + status="COMPLETED", + progress=100, + message=( + "Asset inventory scan completed: " + f"sources={result.get('source_assets', 0)}, " + f"orbits={result.get('orbit_assets', 0)}, " + f"matched={((result.get('binding') or {}).get('matched_count', 0))}, " + f"missing={((result.get('binding') or {}).get('missing_count', 0))}" + ), + db=db, + ) + + def _resolve_hazard_shp_path() -> str: base_dir = settings.HAZARD_POINTS_DIR filename = settings.HAZARD_POINTS_FILENAME @@ -418,6 +449,16 @@ async def _handle_unpack_archives(job: SystemJobORM) -> None: await run_unpack_task(job.task_id) +async def _handle_unpack_sentinel1(job: SystemJobORM) -> None: + if not job.task_id: + raise ValueError("UNPACK_SENTINEL1 requires task_id for progress tracking.") + payload = job.payload or {} + if payload.get("asset_id"): + await asset_inventory_service.run_sentinel1_unpack_task(job.task_id, payload) + return + await asset_inventory_service.run_sentinel1_unpack_batch_task(job.task_id, payload) + + async def _handle_ai_train(job: SystemJobORM) -> None: if not job.task_id: raise ValueError("AI_TRAIN requires task_id for progress tracking.") @@ -3849,6 +3890,7 @@ async def _handle_rebuild_psinsar_catalog(job: SystemJobORM) -> None: _HANDLERS = { JOB_TYPE_SCAN_DATA: _handle_scan_data, + JOB_TYPE_SCAN_ASSET_INVENTORY: _handle_scan_asset_inventory, JOB_TYPE_SCAN_DINSAR: _handle_scan_dinsar, JOB_TYPE_PUBLISH_DINSAR_PRODUCTS: _handle_publish_dinsar_products_clean, JOB_TYPE_REBUILD_DINSAR_CATALOG: _handle_rebuild_dinsar_catalog_clean, @@ -3864,6 +3906,7 @@ _HANDLERS = { JOB_TYPE_REBUILD_PSINSAR_CATALOG: _handle_rebuild_psinsar_catalog, JOB_TYPE_COPY_DATA: _handle_copy_data, JOB_TYPE_UNPACK: _handle_unpack_archives, + JOB_TYPE_UNPACK_SENTINEL1: _handle_unpack_sentinel1, JOB_TYPE_AI_TRAIN: _handle_ai_train, JOB_TYPE_AI_PREDICT: _handle_ai_predict, JOB_TYPE_AI_ANALYZE: _handle_ai_analyze, diff --git a/backend/app/services/pairing_cache_service.py b/backend/app/services/pairing_cache_service.py index bd6c3d3..659fbc2 100644 --- a/backend/app/services/pairing_cache_service.py +++ b/backend/app/services/pairing_cache_service.py @@ -36,7 +36,7 @@ def _satellite_family_expr(alias: str) -> str: f"COALESCE(NULLIF({alias}.satellite_family, ''), " f"CASE " f"WHEN {compact} IN ('LT1', 'LT1A', 'LT1B', 'LUTAN1', 'LUTAN1A', 'LUTAN1B') THEN 'LT1' " - f"WHEN {compact} IN ('S1', 'S1A', 'S1B', 'SENTINEL1', 'SENTINEL1A', 'SENTINEL1B') THEN 'S1' " + f"WHEN {compact} IN ('S1', 'S1A', 'S1B', 'S1C', 'SENTINEL1', 'SENTINEL1A', 'SENTINEL1B', 'SENTINEL1C') THEN 'S1' " f"WHEN NULLIF({alias}.satellite, '') IS NOT NULL THEN upper({alias}.satellite) " f"ELSE NULL END)" ) diff --git a/backend/app/services/pyint_input_assets_service.py b/backend/app/services/pyint_input_assets_service.py index d374cb1..8745348 100644 --- a/backend/app/services/pyint_input_assets_service.py +++ b/backend/app/services/pyint_input_assets_service.py @@ -10,11 +10,14 @@ from typing import Any, Dict, List from ..config import settings from .orbit_converter import get_source_orbit_inventory from .pyint_service import ( + discover_s1_scene_sources, discover_lt1_archives, infer_scene_date_from_archives, infer_task_identity, validate_pyint_root_dir, ) +from .asset_inventory_service import _configured_sentinel1_archive_dirs, _parse_s1_source_name +from ..utils import normalize_satellite_family VALID_DEM_MODES = {"local_fabdem", "opentopo", "prepared_file"} @@ -62,6 +65,198 @@ def _infer_satellite_from_archives(paths: List[str]) -> str: return "" +def _normalize_s1_satellite(value: Any) -> str: + text = str(value or "").strip().upper().replace("-", "").replace("_", "") + if text in {"S1A", "S1B", "S1C"}: + return text + return "" + + +def _find_s1_zip_by_logical_uid(logical_uid: str) -> str: + logical = str(logical_uid or "").strip() + if not logical: + return "" + expected_name = logical if logical.lower().endswith(".zip") else f"{logical}.zip" + for root in _configured_sentinel1_archive_dirs(): + if not root or not os.path.isdir(root): + continue + direct_candidate = os.path.join(root, expected_name) + if os.path.isfile(direct_candidate): + return _normalize_path(direct_candidate) + for current_root, _, files in os.walk(root): + if expected_name in files: + return _normalize_path(os.path.join(current_root, expected_name)) + return "" + + +def _is_s1_safe_dir(path: str) -> bool: + normalized = _normalize_path(path) + if not normalized or not os.path.isdir(normalized): + return False + return os.path.isfile(os.path.join(normalized, "manifest.safe")) + + +def _resolve_s1_scene_input( + *, + role: str, + scene_path: str, + pair_meta: Dict[str, Any], +) -> Dict[str, Any]: + normalized_scene = _normalize_path(scene_path) + scene_name = os.path.basename(normalized_scene.rstrip("\\/")) + parsed = _parse_s1_source_name(scene_name) or {} + logical_uid = str(parsed.get("logical_product_uid") or "").strip() + resolved_path = "" + input_kind = "" + resolution_method = "" + + if normalized_scene.lower().endswith(".zip") and os.path.isfile(normalized_scene): + resolved_path = normalized_scene + input_kind = "zip" + resolution_method = "task_or_pair_meta" + elif _is_s1_safe_dir(normalized_scene): + resolved_path = normalized_scene + input_kind = "safe_dir" + resolution_method = "task_or_pair_meta" + elif normalized_scene.lower().endswith(".safe") and os.path.isdir(normalized_scene): + resolved_path = normalized_scene + input_kind = "safe_dir" + resolution_method = "task_or_pair_meta" + else: + sibling_zip = "" + if normalized_scene.lower().endswith(".safe"): + sibling_zip = normalized_scene[:-5] + ".zip" + if sibling_zip and os.path.isfile(sibling_zip): + resolved_path = _normalize_path(sibling_zip) + input_kind = "zip" + resolution_method = "safe_sibling_zip" + elif logical_uid: + looked_up_zip = _find_s1_zip_by_logical_uid(logical_uid) + if looked_up_zip: + resolved_path = looked_up_zip + input_kind = "zip" + resolution_method = "source_pool_lookup" + + expected_name = scene_name + if input_kind == "zip": + if not expected_name.lower().endswith(".zip"): + expected_name = f"{logical_uid}.zip" if logical_uid else os.path.basename(resolved_path) + elif input_kind == "safe_dir": + if not expected_name.lower().endswith(".safe"): + expected_name = f"{logical_uid}.SAFE" if logical_uid else os.path.basename(resolved_path) + + satellite = _normalize_s1_satellite(pair_meta.get(f"{role}_satellite")) or _normalize_s1_satellite(parsed.get("satellite")) + date_text = str(pair_meta.get(f"{role}_imaging_date") or parsed.get("imaging_date") or "").strip() + return { + "role": role, + "scene_path": normalized_scene, + "scene_name": scene_name, + "logical_product_uid": logical_uid, + "satellite": satellite, + "date": date_text, + "resolved": bool(resolved_path), + "path": resolved_path, + "input_kind": input_kind, + "resolution_method": resolution_method, + "expected_name": expected_name, + "staged_path": "", + } + + +def _resolve_s1_zip_file( + *, + role: str, + scene_path: str, + pair_meta: Dict[str, Any], +) -> Dict[str, Any]: + normalized_scene = _normalize_path(scene_path) + scene_name = os.path.basename(normalized_scene) + parsed = _parse_s1_source_name(scene_name) or {} + logical_uid = str(parsed.get("logical_product_uid") or "").strip() + direct_zip = "" + if normalized_scene.lower().endswith(".zip") and os.path.isfile(normalized_scene): + direct_zip = normalized_scene + elif normalized_scene.lower().endswith(".safe"): + sibling_zip = normalized_scene[:-5] + ".zip" + if os.path.isfile(sibling_zip): + direct_zip = _normalize_path(sibling_zip) + if not direct_zip and logical_uid: + direct_zip = _find_s1_zip_by_logical_uid(logical_uid) + + satellite = _normalize_s1_satellite(pair_meta.get(f"{role}_satellite")) or _normalize_s1_satellite(parsed.get("satellite")) + date_text = str(pair_meta.get(f"{role}_imaging_date") or parsed.get("imaging_date") or "").strip() + return { + "role": role, + "scene_path": normalized_scene, + "scene_name": scene_name, + "logical_product_uid": logical_uid, + "satellite": satellite, + "date": date_text, + "resolved": bool(direct_zip), + "path": direct_zip, + "resolution_method": "scene_or_source_pool_lookup" if direct_zip else "", + "expected_name": f"{logical_uid}.zip" if logical_uid else "", + "staged_path": "", + } + + +def _resolve_s1_orbit_file( + *, + role: str, + pair_meta: Dict[str, Any], + task_dir: str, +) -> Dict[str, Any]: + direct_path = _normalize_path(pair_meta.get(f"{role}_orbit_file_path")) + if direct_path and os.path.isfile(direct_path): + return { + "role": role, + "resolved": True, + "path": direct_path, + "resolution_method": "pair_meta", + "expected_name": os.path.basename(direct_path), + "satellite": _normalize_s1_satellite(pair_meta.get(f"{role}_satellite")), + "date": str(pair_meta.get(f"{role}_imaging_date") or "").strip(), + "staged_path": "", + } + + role_orbit_dir = os.path.join(task_dir, "orbit") + satellite = _normalize_s1_satellite(pair_meta.get(f"{role}_satellite")) + date_text = str(pair_meta.get(f"{role}_imaging_date") or "").strip() + candidates: List[str] = [] + if os.path.isdir(role_orbit_dir): + for entry in os.scandir(role_orbit_dir): + if not entry.is_file(): + continue + if not entry.name.lower().endswith(".eof"): + continue + if satellite and satellite not in entry.name.upper(): + continue + candidates.append(_normalize_path(entry.path)) + candidates.sort() + if candidates: + return { + "role": role, + "resolved": True, + "path": candidates[0], + "resolution_method": "task_orbit_dir", + "expected_name": os.path.basename(candidates[0]), + "satellite": satellite, + "date": date_text, + "staged_path": "", + } + return { + "role": role, + "resolved": False, + "path": "", + "resolution_method": "", + "expected_name": "", + "satellite": satellite, + "date": date_text, + "staged_path": "", + "error": f"{role} Sentinel-1 EOF 缺失", + } + + def _get_dem_mode() -> str: raw_mode = str(getattr(settings, "PYINT_DEM_MODE", "local_fabdem") or "local_fabdem").strip().lower() if raw_mode not in VALID_DEM_MODES: @@ -569,6 +764,427 @@ def resolve_pyint_task_input_assets( } +def build_pyint_input_preview(root_dir: str, num_to_process: int = 0) -> Dict[str, Any]: + validation = validate_pyint_root_dir(root_dir, num_to_process) + dem_summary = get_pyint_dem_summary() + orbit_context = get_pyint_orbit_context() + + warnings: List[str] = list(dem_summary.get("warnings") or []) + blockers: List[str] = list(dem_summary.get("blockers") or []) + task_summaries: List[Dict[str, Any]] = [] + resolved_task_count = 0 + missing_task_count = 0 + effective_orbit_policy = _get_orbit_policy() + + for task_dir in validation.get("task_dirs", []) or []: + task_summary = resolve_pyint_task_input_assets( + task_dir, + dem_summary=dem_summary, + orbit_context=orbit_context, + ) + task_summaries.append(task_summary) + task_orbit_policy = str(((task_summary.get("input_assets") or {}).get("orbits") or {}).get("policy") or "").strip() + if task_orbit_policy: + effective_orbit_policy = task_orbit_policy + if task_summary.get("warnings"): + warnings.extend( + f"{task_summary['task_alias']}: {item}" + for item in task_summary["warnings"] + ) + if task_summary.get("blockers"): + blockers.extend( + f"{task_summary['task_alias']}: {item}" + for item in task_summary["blockers"] + ) + if task_summary["input_assets"]["orbits"]["missing_count"] == 0: + resolved_task_count += 1 + else: + missing_task_count += 1 + + allow_submit = not blockers + precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary() + return { + "root_dir": validation["root_dir"], + "mode": validation["mode"], + "task_count": len(task_summaries), + "selected_task_count": len(task_summaries), + "allow_submit": allow_submit, + "warnings": warnings, + "blockers": blockers, + "invalid_candidates": validation.get("invalid_candidates", []), + "dem": dem_summary, + "orbits": { + "policy": effective_orbit_policy, + "pool_root": orbit_context.get("pool_root", ""), + "pool_exists": bool(orbit_context.get("pool_exists")), + "resolved_task_count": resolved_task_count, + "missing_task_count": missing_task_count, + "duplicate_count": int(orbit_context.get("duplicate_count", 0) or 0), + "warnings": list(orbit_context.get("warnings") or []), + }, + "precise_orbit_bridge": precise_orbit_bridge, + "tasks": task_summaries, + } + + +def summarize_preview_blockers(preview: Dict[str, Any], limit: int = 8) -> str: + blockers = [str(item).strip() for item in (preview.get("blockers") or []) if str(item).strip()] + if not blockers: + return "" + if len(blockers) <= limit: + return "; ".join(blockers) + return "; ".join(blockers[:limit]) + f"; 其余 {len(blockers) - limit} 项已省略" + + +def materialize_pyint_input_assets( + *, + task_summary: Dict[str, Any], + input_assets_dir: str, + project_name: str = "", +) -> Dict[str, Any]: + input_assets_dir = _normalize_path(input_assets_dir) + os.makedirs(input_assets_dir, exist_ok=True) + + record_enabled = bool(getattr(settings, "PYINT_RECORD_INPUT_ASSETS", True)) + orbits_dir = os.path.join(input_assets_dir, "orbits") + dem_dir = os.path.join(input_assets_dir, "dem") + downloads_dir = os.path.join(input_assets_dir, "downloads") + if record_enabled: + os.makedirs(orbits_dir, exist_ok=True) + os.makedirs(dem_dir, exist_ok=True) + os.makedirs(downloads_dir, exist_ok=True) + + manifest = _copy_json_safe(task_summary.get("input_assets") or {}) + manifest["generated_at"] = _utc_now_text() + manifest["task_name"] = task_summary.get("task_name") + manifest["task_alias"] = task_summary.get("task_alias") + manifest["pair_key"] = task_summary.get("pair_key") + manifest["task_dir"] = task_summary.get("task_dir") + manifest["allow_submit"] = bool(task_summary.get("allow_submit")) + manifest["warnings"] = list(task_summary.get("warnings") or []) + manifest["blockers"] = list(task_summary.get("blockers") or []) + + dem_summary = manifest.get("dem") or {} + if project_name: + dem_summary["resolved_output_dir"] = os.path.join(_normalize_path(settings.PYINT_DEM_ROOT), project_name) + manifest["dem"] = dem_summary + + orbits_summary = manifest.get("orbits") or {} + staged_count = 0 + precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary() + should_stage_orbits = record_enabled and ( + str(orbits_summary.get("policy") or "").strip().lower() == "stage_txt" + or precise_orbit_bridge.get("enabled") + or str((manifest.get("task_source") or {}).get("satellite_family") or "").strip().upper() == "S1" + ) + if should_stage_orbits: + for role in ("master", "slave"): + orbit_item = orbits_summary.get(role) or {} + orbit_path = _normalize_path(orbit_item.get("path")) + expected_name = str(orbit_item.get("expected_name") or "").strip() + if not orbit_item.get("resolved") or not orbit_path or not expected_name: + continue + target_path = os.path.join(orbits_dir, expected_name) + if not os.path.exists(target_path): + shutil.copy2(orbit_path, target_path) + orbit_item["staged_path"] = target_path + orbit_item["stage_operation"] = "copied" + orbit_item["stage_reason"] = "precise_orbit_bridge" if precise_orbit_bridge.get("enabled") else "stage_txt_policy" + staged_count += 1 + orbits_summary[role] = orbit_item + manifest["orbits"] = orbits_summary + + download_staged_count = 0 + task_source = manifest.get("task_source") or {} + if record_enabled and str(task_source.get("satellite_family") or "").strip().upper() == "S1": + production_inputs = task_source.get("production_inputs") or {} + for role_key in ("master_scene", "slave_scene", "master_zip", "slave_zip"): + scene_item = production_inputs.get(role_key) or {} + scene_path = _normalize_path(scene_item.get("path")) + expected_name = str(scene_item.get("expected_name") or os.path.basename(scene_path) or "").strip() + if not scene_item.get("resolved") or not scene_path or not expected_name: + continue + input_kind = str(scene_item.get("input_kind") or "").strip().lower() + if input_kind == "safe_dir" or os.path.isdir(scene_path): + scene_item["staged_path"] = scene_path + scene_item["stage_operation"] = "source_reference" + production_inputs[role_key] = scene_item + continue + target_path = os.path.join(downloads_dir, expected_name) + if not os.path.exists(target_path): + shutil.copy2(scene_path, target_path) + scene_item["staged_path"] = target_path + scene_item["stage_operation"] = "copied" + production_inputs[role_key] = scene_item + download_staged_count += 1 + task_source["production_inputs"] = production_inputs + manifest["task_source"] = task_source + + materialized = { + "input_assets_dir": input_assets_dir, + "record_enabled": record_enabled, + "orbits_dir": orbits_dir if record_enabled else "", + "dem_dir": dem_dir if record_enabled else "", + "downloads_dir": downloads_dir if record_enabled else "", + "orbits_staged_count": staged_count, + "downloads_staged_count": download_staged_count, + "task_manifest_path": "", + "dem_summary_path": "", + "orbit_summary_path": "", + "input_assets": manifest, + } + + if not record_enabled: + return materialized + + task_manifest_path = os.path.join(input_assets_dir, "task_manifest.json") + dem_summary_path = os.path.join(dem_dir, "dem_summary.json") + orbit_summary_path = os.path.join(orbits_dir, "orbit_summary.json") + + with open(task_manifest_path, "w", encoding="utf-8") as fp: + json.dump(manifest, fp, ensure_ascii=False, indent=2) + fp.write("\n") + with open(dem_summary_path, "w", encoding="utf-8") as fp: + json.dump(dem_summary, fp, ensure_ascii=False, indent=2) + fp.write("\n") + with open(orbit_summary_path, "w", encoding="utf-8") as fp: + json.dump(orbits_summary, fp, ensure_ascii=False, indent=2) + fp.write("\n") + + materialized.update( + { + "task_manifest_path": task_manifest_path, + "dem_summary_path": dem_summary_path, + "orbit_summary_path": orbit_summary_path, + } + ) + return materialized + + +def resolve_pyint_task_input_assets( + task_dir: str, + *, + dem_summary: Dict[str, Any] | None = None, + orbit_context: Dict[str, Any] | None = None, +) -> Dict[str, Any]: + task_dir = _normalize_path(task_dir) + task_identity = infer_task_identity(task_dir) + pair_meta = task_identity["pair_meta"] + satellite_family = str(task_identity.get("satellite_family") or "").strip().upper() + + warnings: List[str] = [] + blockers: List[str] = [] + + if satellite_family == "S1": + scene_sources = discover_s1_scene_sources(task_dir) + master_archives = list(scene_sources.get("master", []) or []) + slave_archives = list(scene_sources.get("slave", []) or []) + master_date = task_identity["master_date"] or infer_scene_date_from_archives(master_archives) + slave_date = task_identity["slave_date"] or infer_scene_date_from_archives(slave_archives) + master_satellite = _normalize_s1_satellite(pair_meta.get("master_satellite")) or _normalize_s1_satellite(task_identity.get("master_satellite")) + slave_satellite = _normalize_s1_satellite(pair_meta.get("slave_satellite")) or _normalize_s1_satellite(task_identity.get("slave_satellite")) + + if not master_archives: + blockers.append("master/ 未识别到 Sentinel-1 SAFE 源目录。") + if not slave_archives: + blockers.append("slave/ 未识别到 Sentinel-1 SAFE 源目录。") + if not master_date: + blockers.append("未能识别主影像日期。") + if not slave_date: + blockers.append("未能识别从影像日期。") + + master_scene = _resolve_s1_scene_input( + role="master", + scene_path=master_archives[0] if master_archives else "", + pair_meta=pair_meta, + ) + slave_scene = _resolve_s1_scene_input( + role="slave", + scene_path=slave_archives[0] if slave_archives else "", + pair_meta=pair_meta, + ) + master_orbit = _resolve_s1_orbit_file( + role="master", + pair_meta=pair_meta, + task_dir=task_dir, + ) + slave_orbit = _resolve_s1_orbit_file( + role="slave", + pair_meta=pair_meta, + task_dir=task_dir, + ) + + if not master_scene.get("resolved"): + blockers.append("master Sentinel-1 源场景缺失。") + if not slave_scene.get("resolved"): + blockers.append("slave Sentinel-1 源场景缺失。") + if not master_orbit.get("resolved"): + blockers.append(str(master_orbit.get("error") or "master Sentinel-1 EOF 缺失")) + if not slave_orbit.get("resolved"): + blockers.append(str(slave_orbit.get("error") or "slave Sentinel-1 EOF 缺失")) + + task_source = { + "task_dir": task_dir, + "task_name": task_identity["task_name"], + "task_alias": task_identity["task_alias"], + "pair_key": task_identity["pair_key"], + "satellite_family": "S1", + "master_date": master_date, + "slave_date": slave_date, + "master_satellite": master_satellite, + "slave_satellite": slave_satellite, + "archives": { + "master": master_archives, + "slave": slave_archives, + }, + "production_inputs": { + "master_scene": master_scene, + "slave_scene": slave_scene, + }, + } + orbits_summary = { + "policy": "require_eof", + "pool_root": "", + "pool_exists": True, + "master": master_orbit, + "slave": slave_orbit, + "resolved_count": int(bool(master_orbit.get("resolved"))) + int(bool(slave_orbit.get("resolved"))), + "missing_count": int(not master_orbit.get("resolved")) + int(not slave_orbit.get("resolved")), + "warnings": [], + "stage_mode": "copy", + "precise_orbit_bridge": { + "enabled": False, + "mode": "not_applicable", + "strict": True, + }, + } + else: + archives = discover_lt1_archives(task_dir) + master_archives = list(archives.get("master", []) or []) + slave_archives = list(archives.get("slave", []) or []) + master_date = task_identity["master_date"] or infer_scene_date_from_archives(master_archives) + slave_date = task_identity["slave_date"] or infer_scene_date_from_archives(slave_archives) + master_satellite = _normalize_lt1_satellite(pair_meta.get("master_satellite")) or _infer_satellite_from_archives(master_archives) + slave_satellite = _normalize_lt1_satellite(pair_meta.get("slave_satellite")) or _infer_satellite_from_archives(slave_archives) + + if not master_archives: + blockers.append("master/ 未发现 LT-1 原始输入（LT1*.tar.gz 或 LT1*.tiff）。") + if not slave_archives: + blockers.append("slave/ 未发现 LT-1 原始输入（LT1*.tar.gz 或 LT1*.tiff）。") + if not master_date: + blockers.append("未能识别主影像日期。") + if not slave_date: + blockers.append("未能识别从影像日期。") + + orbit_policy = _get_orbit_policy() + orbit_context = orbit_context or get_pyint_orbit_context() + orbit_pool_root = orbit_context.get("pool_root", "") + orbit_pool_exists = bool(orbit_context.get("pool_exists")) + orbit_files = orbit_context.get("files", {}) or {} + + orbit_warnings: List[str] = [] + if orbit_context.get("warnings"): + orbit_warnings.extend(str(item) for item in orbit_context["warnings"] if item) + + master_orbit = _resolve_orbit_file( + role="master", + satellite=master_satellite, + date_text=master_date, + pool_root=orbit_pool_root, + orbit_files=orbit_files, + ) + slave_orbit = _resolve_orbit_file( + role="slave", + satellite=slave_satellite, + date_text=slave_date, + pool_root=orbit_pool_root, + orbit_files=orbit_files, + ) + + for orbit_item in (master_orbit, slave_orbit): + if orbit_item.get("resolved"): + continue + message = str(orbit_item.get("error") or f"{orbit_item.get('role')} orbit missing").strip() + if orbit_policy == "validate_only": + orbit_warnings.append(message) + else: + blockers.append(message) + + if not orbit_pool_root: + if orbit_policy == "validate_only": + orbit_warnings.append("轨道池未配置，当前仅记录警告。") + else: + blockers.append("轨道池未配置。") + elif not orbit_pool_exists: + if orbit_policy == "validate_only": + orbit_warnings.append(f"轨道池目录不可用: {orbit_pool_root}") + else: + blockers.append(f"轨道池目录不可用: {orbit_pool_root}") + + warnings.extend(orbit_warnings) + + precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary() + task_source = { + "task_dir": task_dir, + "task_name": task_identity["task_name"], + "task_alias": task_identity["task_alias"], + "pair_key": task_identity["pair_key"], + "satellite_family": "LT1", + "master_date": master_date, + "slave_date": slave_date, + "master_satellite": master_satellite, + "slave_satellite": slave_satellite, + "archives": { + "master": master_archives, + "slave": slave_archives, + }, + } + orbits_summary = { + "policy": orbit_policy, + "pool_root": orbit_pool_root, + "pool_exists": orbit_pool_exists, + "master": master_orbit, + "slave": slave_orbit, + "resolved_count": int(bool(master_orbit.get("resolved"))) + int(bool(slave_orbit.get("resolved"))), + "missing_count": int(not master_orbit.get("resolved")) + int(not slave_orbit.get("resolved")), + "warnings": orbit_warnings, + "stage_mode": "copy" if orbit_policy == "stage_txt" or precise_orbit_bridge.get("enabled") else "none", + "precise_orbit_bridge": precise_orbit_bridge, + } + + dem_payload = _copy_json_safe(dem_summary or get_pyint_dem_summary()) + allow_submit = not blockers and bool(dem_payload.get("allow_submit", True)) + return { + "task_name": task_identity["task_name"], + "task_alias": task_identity["task_alias"], + "pair_key": task_identity["pair_key"], + "task_dir": task_dir, + "master_date": master_date, + "slave_date": slave_date, + "master_satellite": master_satellite, + "slave_satellite": slave_satellite, + "satellite_family": satellite_family or normalize_satellite_family(master_satellite or slave_satellite), + "archive_counts": { + "master": len(master_archives), + "slave": len(slave_archives), + }, + "warnings": warnings, + "blockers": blockers, + "allow_submit": allow_submit, + "task_source": task_source, + "dem": dem_payload, + "orbit_resolution": { + "master": master_orbit, + "slave": slave_orbit, + }, + "input_assets": { + "task_source": task_source, + "dem": dem_payload, + "orbits": orbits_summary, + }, + } + + def build_pyint_input_preview(root_dir: str, num_to_process: int = 0) -> Dict[str, Any]: validation = validate_pyint_root_dir(root_dir, num_to_process) dem_summary = get_pyint_dem_summary() @@ -626,106 +1242,3 @@ def build_pyint_input_preview(root_dir: str, num_to_process: int = 0) -> Dict[st "precise_orbit_bridge": precise_orbit_bridge, "tasks": task_summaries, } - - -def summarize_preview_blockers(preview: Dict[str, Any], limit: int = 8) -> str: - blockers = [str(item).strip() for item in (preview.get("blockers") or []) if str(item).strip()] - if not blockers: - return "" - if len(blockers) <= limit: - return "; ".join(blockers) - return "; ".join(blockers[:limit]) + f"; 其余 {len(blockers) - limit} 项已省略" - - -def materialize_pyint_input_assets( - *, - task_summary: Dict[str, Any], - input_assets_dir: str, - project_name: str = "", -) -> Dict[str, Any]: - input_assets_dir = _normalize_path(input_assets_dir) - os.makedirs(input_assets_dir, exist_ok=True) - - record_enabled = bool(getattr(settings, "PYINT_RECORD_INPUT_ASSETS", True)) - orbits_dir = os.path.join(input_assets_dir, "orbits") - dem_dir = os.path.join(input_assets_dir, "dem") - if record_enabled: - os.makedirs(orbits_dir, exist_ok=True) - os.makedirs(dem_dir, exist_ok=True) - - manifest = _copy_json_safe(task_summary.get("input_assets") or {}) - manifest["generated_at"] = _utc_now_text() - manifest["task_name"] = task_summary.get("task_name") - manifest["task_alias"] = task_summary.get("task_alias") - manifest["pair_key"] = task_summary.get("pair_key") - manifest["task_dir"] = task_summary.get("task_dir") - manifest["allow_submit"] = bool(task_summary.get("allow_submit")) - manifest["warnings"] = list(task_summary.get("warnings") or []) - manifest["blockers"] = list(task_summary.get("blockers") or []) - - dem_summary = manifest.get("dem") or {} - if project_name: - dem_summary["resolved_output_dir"] = os.path.join(_normalize_path(settings.PYINT_DEM_ROOT), project_name) - manifest["dem"] = dem_summary - - orbits_summary = manifest.get("orbits") or {} - staged_count = 0 - precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary() - should_stage_orbits = record_enabled and ( - str(orbits_summary.get("policy") or "").strip().lower() == "stage_txt" - or precise_orbit_bridge.get("enabled") - ) - if should_stage_orbits: - for role in ("master", "slave"): - orbit_item = orbits_summary.get(role) or {} - orbit_path = _normalize_path(orbit_item.get("path")) - expected_name = str(orbit_item.get("expected_name") or "").strip() - if not orbit_item.get("resolved") or not orbit_path or not expected_name: - continue - target_path = os.path.join(orbits_dir, expected_name) - if not os.path.exists(target_path): - shutil.copy2(orbit_path, target_path) - orbit_item["staged_path"] = target_path - orbit_item["stage_operation"] = "copied" - orbit_item["stage_reason"] = "precise_orbit_bridge" if precise_orbit_bridge.get("enabled") else "stage_txt_policy" - staged_count += 1 - orbits_summary[role] = orbit_item - manifest["orbits"] = orbits_summary - - materialized = { - "input_assets_dir": input_assets_dir, - "record_enabled": record_enabled, - "orbits_dir": orbits_dir if record_enabled else "", - "dem_dir": dem_dir if record_enabled else "", - "orbits_staged_count": staged_count, - "task_manifest_path": "", - "dem_summary_path": "", - "orbit_summary_path": "", - "input_assets": manifest, - } - - if not record_enabled: - return materialized - - task_manifest_path = os.path.join(input_assets_dir, "task_manifest.json") - dem_summary_path = os.path.join(dem_dir, "dem_summary.json") - orbit_summary_path = os.path.join(orbits_dir, "orbit_summary.json") - - with open(task_manifest_path, "w", encoding="utf-8") as fp: - json.dump(manifest, fp, ensure_ascii=False, indent=2) - fp.write("\n") - with open(dem_summary_path, "w", encoding="utf-8") as fp: - json.dump(dem_summary, fp, ensure_ascii=False, indent=2) - fp.write("\n") - with open(orbit_summary_path, "w", encoding="utf-8") as fp: - json.dump(orbits_summary, fp, ensure_ascii=False, indent=2) - fp.write("\n") - - materialized.update( - { - "task_manifest_path": task_manifest_path, - "dem_summary_path": dem_summary_path, - "orbit_summary_path": orbit_summary_path, - } - ) - return materialized diff --git a/backend/app/services/pyint_service.py b/backend/app/services/pyint_service.py index dc6b9fa..e853d5f 100644 --- a/backend/app/services/pyint_service.py +++ b/backend/app/services/pyint_service.py @@ -13,10 +13,12 @@ from typing import Any, Dict, Iterable, List, Optional from ..config import get_env_text, read_bool_env, settings from .dinsar_naming import PAIR_META_FILENAME, build_fallback_pair_key, find_json_sidecar +from ..utils import normalize_satellite_family from .wsl_service import run_wsl_exec LT1_INPUT_GLOBS = ("LT1*.tar.gz", "LT1*.tiff") +S1_INPUT_GLOBS = ("S1*.zip",) DEFAULT_RANGE_LOOKS = 2 DEFAULT_AZIMUTH_LOOKS = 2 DEFAULT_DEM_RESOLUTION_M = 30.0 @@ -334,6 +336,15 @@ def build_project_name(pair_key: str, run_key: str) -> str: return slugify_text(f"{pair_key}_{run_key}", default="pyint_project", max_len=120) +def build_profile_project_name(satellite_family: Any, pair_key: str, run_key: str) -> str: + family = str(normalize_satellite_family(satellite_family) or "").strip().upper() + if family == "S1": + return slugify_text(f"s1_{pair_key}_{run_key}", default="s1_pyint_project", max_len=120) + if family == "LT1": + return slugify_text(f"lt1_{pair_key}_{run_key}", default="lt1_pyint_project", max_len=120) + return build_project_name(pair_key, run_key) + + def windows_path_to_wsl_mount(path: str) -> str: text = str(path or "").strip().strip('"').strip("'") if not text: @@ -377,6 +388,22 @@ def discover_lt1_archives(task_dir: str) -> Dict[str, List[str]]: return result +def discover_s1_scene_sources(task_dir: str) -> Dict[str, List[str]]: + task_path = Path(os.path.normpath(os.path.abspath(str(task_dir or "").strip()))) + pair_meta = find_json_sidecar(str(task_path), PAIR_META_FILENAME, max_levels=0) or {} + result: Dict[str, List[str]] = {"master": [], "slave": []} + for role in ("master", "slave"): + explicit_path = str(pair_meta.get(f"{role}_path") or "").strip() + role_dir = task_path / role + candidates: List[str] = [] + if explicit_path: + candidates.append(str(Path(explicit_path).resolve())) + if not candidates and role_dir.is_dir() and (role_dir / "manifest.safe").is_file(): + candidates.append(str(role_dir.resolve())) + result[role] = sorted(set(candidates)) + return result + + def infer_scene_date_from_archives(paths: Iterable[str]) -> str: dates = { date_text @@ -393,7 +420,14 @@ def infer_task_identity(task_dir: str) -> Dict[str, Any]: task_name = os.path.basename(os.path.normpath(task_dir)) pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {} task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name - pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(task_alias, task_dir) + master_satellite = str(pair_meta.get("master_satellite") or "").strip().upper() + slave_satellite = str(pair_meta.get("slave_satellite") or "").strip().upper() + satellite_family = normalize_satellite_family(master_satellite or slave_satellite) + pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key( + task_alias, + task_dir, + satellite_family=satellite_family, + ) master_date = normalize_date_text(pair_meta.get("master_imaging_date")) slave_date = normalize_date_text(pair_meta.get("slave_imaging_date")) return { @@ -403,6 +437,9 @@ def infer_task_identity(task_dir: str) -> Dict[str, Any]: "pair_meta": pair_meta, "master_date": master_date, "slave_date": slave_date, + "master_satellite": master_satellite, + "slave_satellite": slave_satellite, + "satellite_family": satellite_family, } diff --git a/backend/app/services/result_catalog_service.py b/backend/app/services/result_catalog_service.py index 6f90c83..c8c3446 100644 --- a/backend/app/services/result_catalog_service.py +++ b/backend/app/services/result_catalog_service.py @@ -39,6 +39,7 @@ from .dinsar_naming import ( build_fallback_pair_key, find_json_sidecar, ) +from ..utils import normalize_satellite_family from .dinsar_result_layout_service import ( RUN_CURRENT_DIRNAME, RUN_NATIVE_DIRNAME, @@ -203,10 +204,16 @@ def _resolve_candidate_identity(candidate: Dict[str, Any]) -> Dict[str, Any]: pair_meta.get("task_alias"), candidate.get("task_name"), ) or "Task_unknown_unknown" + satellite_family = normalize_satellite_family( + pair_meta.get("master_satellite") + or pair_meta.get("slave_satellite") + or run_meta.get("master_satellite") + or run_meta.get("slave_satellite") + ) pair_key = _first_text( run_meta.get("pair_key"), pair_meta.get("pair_key"), - ) or build_fallback_pair_key(task_alias, source_dir) + ) or build_fallback_pair_key(task_alias, source_dir, satellite_family=satellite_family) run_key = _first_text(run_meta.get("run_key")) or ( "legacy_" + _stable_digest(candidate.get("engine_code"), pair_key, source_dir, candidate["primary_file"], length=16) ) diff --git a/backend/app/services/root_registry_service.py b/backend/app/services/root_registry_service.py index 7d08b97..03d8716 100644 --- a/backend/app/services/root_registry_service.py +++ b/backend/app/services/root_registry_service.py @@ -10,7 +10,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from .. import database from ..config import settings, split_env_paths -from ..models import ManagedRootORM, PathInventoryORM, ScanCursorORM +from ..models import AssetInventoryStateORM, ManagedRootORM, PathInventoryORM, ScanCursorORM _SLUG_RE = re.compile(r"[^a-z0-9]+") @@ -87,6 +87,15 @@ def _cursor_type_for_scan_mode(scan_mode: str) -> str: return mapping.get(str(scan_mode or "").strip().lower(), "directory_walk") +def _asset_inventory_type_for_root_role(root_role: str) -> Optional[str]: + role = str(root_role or "").strip().lower() + if role == "source_product_pool": + return "source_product" + if role == "orbit_asset_pool": + return "orbit_asset" + return None + + def _iter_multi_root_specs( *, env_var: str, @@ -169,6 +178,36 @@ def _build_root_specs_from_settings() -> List[RootSpec]: scan_mode="archive_walk", ) ) + source_product_paths = split_env_paths(settings.SOURCE_PRODUCT_DIRS) + if not source_product_paths: + source_product_paths = ( + split_env_paths(settings.INSAR_STORAGE_DIRS) + + split_env_paths(settings.MONITOR_RADAR_DIRS) + ) + specs.extend( + _iter_multi_root_specs( + env_var="SOURCE_PRODUCT_DIRS", + paths=source_product_paths, + root_role="source_product_pool", + display_prefix="Source Product Pool", + scan_mode="file_pool", + ) + ) + source_product_path_set = {_normalize_root_path(path) for path in source_product_paths} + sentinel1_storage_paths = [ + path + for path in split_env_paths(settings.SENTINEL1_STORAGE_DIRS) + if _normalize_root_path(path) not in source_product_path_set + ] + specs.extend( + _iter_multi_root_specs( + env_var="SENTINEL1_STORAGE_DIRS", + paths=sentinel1_storage_paths, + root_role="source_product_pool", + display_prefix="Sentinel-1 Storage Pool", + scan_mode="file_pool", + ) + ) specs.extend( _iter_multi_root_specs( env_var="INSAR_STORAGE_DIRS", @@ -214,6 +253,18 @@ def _build_root_specs_from_settings() -> List[RootSpec]: scan_mode="scene_directory", ) ) + orbit_source_paths = split_env_paths(settings.ORBIT_SOURCE_DIRS) + if not orbit_source_paths: + orbit_source_paths = split_env_paths(settings.MONITOR_ORBIT_DIR) + specs.extend( + _iter_multi_root_specs( + env_var="ORBIT_SOURCE_DIRS", + paths=orbit_source_paths, + root_role="orbit_asset_pool", + display_prefix="Orbit Asset Pool", + scan_mode="file_pool", + ) + ) specs.extend( _iter_single_root_specs( env_var="MONITOR_ORBIT_DIR", @@ -374,6 +425,43 @@ class RootRegistryService: changed = True return "updated" if changed else None + async def _ensure_asset_inventory_state(self, db: AsyncSession, root: ManagedRootORM) -> Optional[str]: + inventory_type = _asset_inventory_type_for_root_role(root.root_role) + if not inventory_type: + return None + + result = await db.execute( + select(AssetInventoryStateORM).where( + AssetInventoryStateORM.root_ref_id == root.id, + AssetInventoryStateORM.inventory_type == inventory_type, + ) + ) + state = result.scalar_one_or_none() + if state is None: + state = AssetInventoryStateORM( + root_ref_id=root.id, + inventory_type=inventory_type, + root_path=root.path, + scan_mode=root.scan_mode, + status="NEVER_SCANNED", + needs_rescan=True, + metadata_json={ + "root_role": root.root_role, + "created_by": "root_registry_sync", + }, + ) + db.add(state) + return "created" + + changed = False + if state.root_path != root.path: + state.root_path = root.path + changed = True + if state.scan_mode != root.scan_mode: + state.scan_mode = root.scan_mode + changed = True + return "updated" if changed else None + async def sync_from_settings(self, db: Optional[AsyncSession] = None) -> Dict[str, Any]: generated_session = db is None if generated_session: @@ -394,6 +482,8 @@ class RootRegistryService: disabled = 0 cursor_created = 0 cursor_updated = 0 + inventory_state_created = 0 + inventory_state_updated = 0 synced_codes: set[str] = set() for spec in specs: @@ -448,6 +538,12 @@ class RootRegistryService: elif cursor_change == "updated": cursor_updated += 1 + inventory_state_change = await self._ensure_asset_inventory_state(db, row) + if inventory_state_change == "created": + inventory_state_created += 1 + elif inventory_state_change == "updated": + inventory_state_updated += 1 + for row in existing_rows: if row.root_code in synced_codes: continue @@ -464,6 +560,7 @@ class RootRegistryService: "updated": updated, "disabled": disabled, "cursor_created_or_updated": cursor_created + cursor_updated, + "asset_inventory_state_created_or_updated": inventory_state_created + inventory_state_updated, "summary": summary, } except Exception: diff --git a/backend/app/services/spatial_service.py b/backend/app/services/spatial_service.py index 3508404..30e8912 100644 --- a/backend/app/services/spatial_service.py +++ b/backend/app/services/spatial_service.py @@ -14,7 +14,7 @@ from typing import Any, Dict, List, Optional, Tuple from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select -from sqlalchemy import and_, cast, func, or_ +from sqlalchemy import and_, case, cast, func, or_ from sqlalchemy.orm import aliased from geoalchemy2 import Geography @@ -43,11 +43,60 @@ from .dinsar_naming import build_pair_key, build_task_alias, ensure_unique_task_ from .pairing_state_service import pairing_state_service -PAIRING_POLICY_VERSION = "2026.05.raw-source.v1" +PAIRING_POLICY_VERSION = "2026.05.raw-source.v2" PAIRING_WARNING_CANDIDATE_THRESHOLD = 3000 logger = logging.getLogger(__name__) +def _normalized_satellite_family_expr(alias): + compact_satellite = func.upper( + func.replace( + func.replace( + func.replace(func.coalesce(alias.satellite, ""), "-", ""), + "_", + "", + ), + " ", + "", + ) + ) + inferred_family = case( + ( + compact_satellite.in_( + ["LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"] + ), + "LT1", + ), + ( + compact_satellite.in_( + [ + "S1", + "S1A", + "S1B", + "S1C", + "SENTINEL1", + "SENTINEL1A", + "SENTINEL1B", + "SENTINEL1C", + ] + ), + "S1", + ), + else_=func.upper(alias.satellite), + ) + return func.coalesce(func.nullif(func.upper(alias.satellite_family), ""), inferred_family) + + +def _same_relative_orbit_expr(left_alias, right_alias): + left_relative_orbit = func.upper(func.trim(func.coalesce(left_alias.relative_orbit, ""))) + right_relative_orbit = func.upper(func.trim(func.coalesce(right_alias.relative_orbit, ""))) + return and_( + left_relative_orbit != "", + right_relative_orbit != "", + left_relative_orbit == right_relative_orbit, + ) + + class SpatialService: """ 纯 PostGIS 空间计算服务 @@ -155,6 +204,8 @@ class SpatialService: ) -> List[dict]: master_alias = aliased(RadarDataORM) slave_alias = aliased(RadarDataORM) + master_family_expr = _normalized_satellite_family_expr(master_alias) + slave_family_expr = _normalized_satellite_family_expr(slave_alias) center_distance_expr = func.coalesce( PairingMetricCacheORM.scene_center_distance_meters, PairingMetricCacheORM.spatial_baseline_meters, @@ -192,6 +243,14 @@ class SpatialService: if params.require_same_polarization: stmt = stmt.where(PairingMetricCacheORM.same_polarization.is_(True)) + stmt = stmt.where( + or_( + master_family_expr != "S1", + slave_family_expr != "S1", + _same_relative_orbit_expr(master_alias, slave_alias), + ) + ) + if params.allowed_satellites: allowed_satellites = [ str(item).strip().upper() @@ -284,6 +343,7 @@ class SpatialService: slave.file_path, master.imaging_date, slave.imaging_date, + master.satellite_family or slave.satellite_family or master.satellite or slave.satellite, ), pair_uid=candidate.get("pair_uid"), metric_cache_ref_id=candidate.get("metric_cache_ref_id"), @@ -1281,7 +1341,7 @@ class SpatialService: compact = raw_satellite.replace("-", "").replace("_", "").replace(" ", "") if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}: return "LT1" - if compact in {"S1", "S1A", "S1B", "SENTINEL1", "SENTINEL1A", "SENTINEL1B"}: + if compact in {"S1", "S1A", "S1B", "S1C", "SENTINEL1", "SENTINEL1A", "SENTINEL1B", "SENTINEL1C"}: return "S1" return raw_satellite or "UNKNOWN" diff --git a/backend/app/utils.py b/backend/app/utils.py index 5d4835e..4905d88 100644 --- a/backend/app/utils.py +++ b/backend/app/utils.py @@ -1,4 +1,6 @@ import os +import re +from datetime import datetime from typing import Optional, Tuple, List, Callable, Dict, Any from lxml import etree @@ -18,7 +20,7 @@ def _create_secure_xml_parser() -> etree.XMLParser: recover=False, ) -# --- Sentinel-1 (S1A/S1B) Parsers --- +# --- Sentinel-1 (S1A/S1B/S1C) Parsers --- def _radar_meta_base() -> Dict[str, Any]: return { @@ -83,7 +85,7 @@ def normalize_satellite_family(value: Optional[str]) -> Optional[str]: compact = raw.replace("-", "").replace("_", "").replace(" ", "") if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}: return "LT1" - if compact in {"S1", "S1A", "S1B", "SENTINEL1", "SENTINEL1A", "SENTINEL1B"}: + if compact in {"S1", "S1A", "S1B", "S1C", "SENTINEL1", "SENTINEL1A", "SENTINEL1B", "SENTINEL1C"}: return "S1" if compact in {"GF3", "GAOFEN3"}: return "GF3" @@ -96,19 +98,44 @@ def parse_s1_radar_filename(folder_name: str) -> Optional[Dict[str, Any]]: Example: S1A_IW_SLC__1SDV_20250101T104105_... Returns a metadata dict. """ - parts = folder_name.split('_') - if len(parts) < 5 or not parts[0].startswith('S1'): + name = os.path.basename(str(folder_name or "").strip()) + if name.lower().endswith(".zip"): + name = name[:-4] + if name.lower().endswith(".safe"): + name = name[:-5] + match = re.match( + r"^(?PS1[A-Z])_" + r"(?P[A-Z0-9]+)_" + r"(?P[A-Z0-9]+)_+" + r"(?P[0-9A-Z]{4})_" + r"(?P\d{8}T\d{6}(?:\.\d+)?)_" + r"(?P\d{8}T\d{6}(?:\.\d+)?)_" + r"(?P\d+)_" + r"(?P[0-9A-F]+)_" + r"(?P[0-9A-F]+)$", + name, + flags=re.IGNORECASE, + ) + if not match: return None - + meta = _radar_meta_base() - meta["satellite"] = parts[0] - meta["satellite_family"] = normalize_satellite_family(parts[0]) - meta["imaging_date"] = parts[4].split('T')[0] - meta["imaging_mode"] = parts[1] - meta["source_product_token"] = parts[2] - meta["product_type"] = parts[2] - polarization = parts[3] # e.g., '1SDV' -> 'DV' is dual-pol VV/VH - meta["polarization"] = polarization[2:] if len(polarization) > 2 else polarization + meta["satellite"] = match.group("satellite").upper() + meta["satellite_family"] = normalize_satellite_family(meta["satellite"]) + meta["imaging_date"] = match.group("start")[:8] + meta["imaging_mode"] = match.group("mode").upper() + meta["source_product_token"] = match.group("class").upper() + meta["product_type"] = match.group("product").upper() + meta["product_level"] = "L1" + polarization = match.group("class").upper() # e.g. 1SDV -> DV + meta["polarization"] = polarization[-2:] if len(polarization) > 2 else polarization + meta["orbit_circle"] = match.group("absolute_orbit").lstrip("0") or match.group("absolute_orbit") + meta["product_unique_id"] = name + try: + start_time = datetime.strptime(match.group("start").split(".")[0], "%Y%m%dT%H%M%S") + meta["acquisition_time_utc"] = start_time.isoformat() + except ValueError: + meta["acquisition_time_utc"] = match.group("start") return meta diff --git a/backend/migrations/009_raw_source_pairing_fields.sql b/backend/migrations/009_raw_source_pairing_fields.sql index 530dbe8..f0183e6 100644 --- a/backend/migrations/009_raw_source_pairing_fields.sql +++ b/backend/migrations/009_raw_source_pairing_fields.sql @@ -81,7 +81,7 @@ SET ('LT1', 'LT1A', 'LT1B', 'LUTAN1', 'LUTAN1A', 'LUTAN1B') THEN 'LT1' WHEN upper(replace(replace(replace(COALESCE(satellite, ''), '-', ''), '_', ''), ' ', '')) IN - ('S1', 'S1A', 'S1B', 'SENTINEL1', 'SENTINEL1A', 'SENTINEL1B') + ('S1', 'S1A', 'S1B', 'S1C', 'SENTINEL1', 'SENTINEL1A', 'SENTINEL1B', 'SENTINEL1C') THEN 'S1' WHEN NULLIF(satellite, '') IS NOT NULL THEN upper(satellite) diff --git a/backend/migrations/010_source_orbit_asset_inventory.sql b/backend/migrations/010_source_orbit_asset_inventory.sql new file mode 100644 index 0000000..b8e3283 --- /dev/null +++ b/backend/migrations/010_source_orbit_asset_inventory.sql @@ -0,0 +1,371 @@ +-- Migration: Source product and orbit asset inventory +-- Version: 10.0 +-- Date: 2026-05-12 +-- Purpose: Add first-class source product assets, orbit assets, scene-orbit bindings, +-- inventory state, and radar_data compatibility fields. + +ALTER TABLE IF EXISTS radar_data + ADD COLUMN IF NOT EXISTS source_product_ref_id INTEGER NULL; + +ALTER TABLE IF EXISTS radar_data + ADD COLUMN IF NOT EXISTS source_archive_asset_id INTEGER NULL; + +ALTER TABLE IF EXISTS radar_data + ADD COLUMN IF NOT EXISTS selected_orbit_asset_id INTEGER NULL; + +ALTER TABLE IF EXISTS radar_data + ADD COLUMN IF NOT EXISTS orbit_binding_status VARCHAR(32) NOT NULL DEFAULT 'UNBOUND'; + +ALTER TABLE IF EXISTS radar_data + ADD COLUMN IF NOT EXISTS orbit_binding_reason TEXT NULL; + +ALTER TABLE IF EXISTS radar_data + ADD COLUMN IF NOT EXISTS acquisition_start_time_utc TIMESTAMP NULL; + +ALTER TABLE IF EXISTS radar_data + ADD COLUMN IF NOT EXISTS acquisition_stop_time_utc TIMESTAMP NULL; + +ALTER TABLE IF EXISTS radar_data + ADD COLUMN IF NOT EXISTS absolute_orbit VARCHAR NULL; + +ALTER TABLE IF EXISTS radar_data + ADD COLUMN IF NOT EXISTS relative_orbit VARCHAR NULL; + +ALTER TABLE IF EXISTS radar_data + ADD COLUMN IF NOT EXISTS source_format VARCHAR(32) NULL; + +ALTER TABLE IF EXISTS radar_data + ADD COLUMN IF NOT EXISTS metadata_json JSON NULL; + +CREATE TABLE IF NOT EXISTS source_product_assets ( + id SERIAL PRIMARY KEY, + asset_uid VARCHAR(128) NOT NULL UNIQUE, + logical_product_uid VARCHAR(128) NULL, + satellite_family VARCHAR(32) NULL, + satellite VARCHAR(32) NULL, + source_format VARCHAR(32) NOT NULL, + product_type VARCHAR(64) NULL, + product_level VARCHAR(64) NULL, + imaging_mode VARCHAR(64) NULL, + polarization VARCHAR(64) NULL, + absolute_orbit VARCHAR(64) NULL, + relative_orbit VARCHAR(64) NULL, + orbit_direction VARCHAR(32) NULL, + acquisition_start_time_utc TIMESTAMP NULL, + acquisition_stop_time_utc TIMESTAMP NULL, + imaging_date VARCHAR(8) NULL, + root_ref_id INTEGER NULL REFERENCES managed_roots(id) ON DELETE SET NULL, + root_path VARCHAR NULL, + file_path VARCHAR NOT NULL UNIQUE, + archive_path VARCHAR NULL, + path_kind VARCHAR(24) NULL, + file_name VARCHAR(255) NULL, + file_stem VARCHAR(255) NULL, + file_ext VARCHAR(32) NULL, + size_bytes BIGINT NULL, + mtime_epoch DOUBLE PRECISION NULL, + checksum_sha256 VARCHAR(64) NULL, + checksum_status VARCHAR(32) NOT NULL DEFAULT 'NOT_COMPUTED', + parser_name VARCHAR(64) NULL, + parser_version VARCHAR(32) NULL, + parse_status VARCHAR(32) NOT NULL DEFAULT 'PENDING', + parse_error TEXT NULL, + parsed_at TIMESTAMP NULL, + metadata_json JSON NULL, + is_active BOOLEAN NOT NULL DEFAULT TRUE, + missing_since TIMESTAMP NULL, + created_at TIMESTAMP NOT NULL DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS orbit_assets ( + id SERIAL PRIMARY KEY, + orbit_uid VARCHAR(128) NOT NULL UNIQUE, + satellite_family VARCHAR(32) NULL, + satellite VARCHAR(32) NULL, + orbit_type VARCHAR(64) NOT NULL, + native_format VARCHAR(32) NOT NULL, + quality_class VARCHAR(32) NOT NULL DEFAULT 'unknown', + root_ref_id INTEGER NULL REFERENCES managed_roots(id) ON DELETE SET NULL, + root_path VARCHAR NULL, + file_path VARCHAR NOT NULL UNIQUE, + file_name VARCHAR(255) NULL, + file_stem VARCHAR(255) NULL, + file_ext VARCHAR(32) NULL, + size_bytes BIGINT NULL, + mtime_epoch DOUBLE PRECISION NULL, + checksum_sha256 VARCHAR(64) NULL, + checksum_status VARCHAR(32) NOT NULL DEFAULT 'NOT_COMPUTED', + validity_start_time_utc TIMESTAMP NULL, + validity_stop_time_utc TIMESTAMP NULL, + generation_time_utc TIMESTAMP NULL, + published_time_utc TIMESTAMP NULL, + parser_name VARCHAR(64) NULL, + parser_version VARCHAR(32) NULL, + parse_status VARCHAR(32) NOT NULL DEFAULT 'PENDING', + parse_error TEXT NULL, + parsed_at TIMESTAMP NULL, + metadata_json JSON NULL, + is_active BOOLEAN NOT NULL DEFAULT TRUE, + missing_since TIMESTAMP NULL, + created_at TIMESTAMP NOT NULL DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS scene_orbit_bindings ( + id SERIAL PRIMARY KEY, + radar_data_id INTEGER NOT NULL REFERENCES radar_data(id) ON DELETE CASCADE, + orbit_asset_id INTEGER NOT NULL REFERENCES orbit_assets(id) ON DELETE CASCADE, + binding_role VARCHAR(32) NOT NULL DEFAULT 'primary_orbit', + match_status VARCHAR(32) NOT NULL DEFAULT 'CANDIDATE', + selection_status VARCHAR(32) NOT NULL DEFAULT 'CANDIDATE', + selection_rank INTEGER NULL, + priority_score DOUBLE PRECISION NULL, + coverage_margin_before_seconds DOUBLE PRECISION NULL, + coverage_margin_after_seconds DOUBLE PRECISION NULL, + match_rule_version VARCHAR(64) NULL, + match_reason TEXT NULL, + selected_at TIMESTAMP NULL, + metadata_json JSON NULL, + created_at TIMESTAMP NOT NULL DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW(), + CONSTRAINT uq_scene_orbit_binding_role UNIQUE (radar_data_id, orbit_asset_id, binding_role) +); + +CREATE TABLE IF NOT EXISTS orbit_asset_derivatives ( + id SERIAL PRIMARY KEY, + orbit_asset_id INTEGER NOT NULL REFERENCES orbit_assets(id) ON DELETE CASCADE, + engine_code VARCHAR(32) NOT NULL, + derivative_format VARCHAR(32) NOT NULL, + derivative_role VARCHAR(64) NULL, + pool_path VARCHAR NOT NULL, + size_bytes BIGINT NULL, + mtime_epoch DOUBLE PRECISION NULL, + checksum_sha256 VARCHAR(64) NULL, + generation_status VARCHAR(32) NOT NULL DEFAULT 'PENDING', + generation_error TEXT NULL, + generated_at TIMESTAMP NULL, + metadata_json JSON NULL, + created_at TIMESTAMP NOT NULL DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW(), + CONSTRAINT uq_orbit_asset_derivative_pool_path UNIQUE (orbit_asset_id, engine_code, derivative_format, pool_path) +); + +CREATE TABLE IF NOT EXISTS asset_inventory_states ( + id SERIAL PRIMARY KEY, + root_ref_id INTEGER NOT NULL REFERENCES managed_roots(id) ON DELETE CASCADE, + inventory_type VARCHAR(32) NOT NULL, + root_path VARCHAR NOT NULL, + scan_mode VARCHAR(32) NOT NULL DEFAULT 'file_pool', + status VARCHAR(32) NOT NULL DEFAULT 'NEVER_SCANNED', + last_scan_started_at TIMESTAMP NULL, + last_scan_finished_at TIMESTAMP NULL, + last_seen_entry_count INTEGER NULL, + last_asset_count INTEGER NULL, + last_issue_count INTEGER NULL, + parser_version VARCHAR(32) NULL, + needs_rescan BOOLEAN NOT NULL DEFAULT TRUE, + last_error TEXT NULL, + metadata_json JSON NULL, + created_at TIMESTAMP NOT NULL DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW(), + CONSTRAINT uq_asset_inventory_state_root_type UNIQUE (root_ref_id, inventory_type) +); + +CREATE TABLE IF NOT EXISTS asset_inventory_issues ( + id SERIAL PRIMARY KEY, + root_ref_id INTEGER NULL REFERENCES managed_roots(id) ON DELETE SET NULL, + inventory_type VARCHAR(32) NOT NULL, + asset_ref_id INTEGER NULL REFERENCES source_product_assets(id) ON DELETE SET NULL, + radar_data_id INTEGER NULL REFERENCES radar_data(id) ON DELETE SET NULL, + orbit_asset_id INTEGER NULL REFERENCES orbit_assets(id) ON DELETE SET NULL, + severity VARCHAR(16) NOT NULL DEFAULT 'warning', + issue_code VARCHAR(64) NOT NULL, + issue_message TEXT NULL, + source_path VARCHAR NULL, + status VARCHAR(16) NOT NULL DEFAULT 'OPEN', + first_seen_at TIMESTAMP NOT NULL DEFAULT NOW(), + last_seen_at TIMESTAMP NOT NULL DEFAULT NOW(), + resolved_at TIMESTAMP NULL, + metadata_json JSON NULL +); + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint WHERE conname = 'fk_radar_data_source_product_ref_id' + ) THEN + ALTER TABLE radar_data + ADD CONSTRAINT fk_radar_data_source_product_ref_id + FOREIGN KEY (source_product_ref_id) + REFERENCES source_product_assets(id) + ON DELETE SET NULL; + END IF; + + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint WHERE conname = 'fk_radar_data_source_archive_asset_id' + ) THEN + ALTER TABLE radar_data + ADD CONSTRAINT fk_radar_data_source_archive_asset_id + FOREIGN KEY (source_archive_asset_id) + REFERENCES source_product_assets(id) + ON DELETE SET NULL; + END IF; + + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint WHERE conname = 'fk_radar_data_selected_orbit_asset_id' + ) THEN + ALTER TABLE radar_data + ADD CONSTRAINT fk_radar_data_selected_orbit_asset_id + FOREIGN KEY (selected_orbit_asset_id) + REFERENCES orbit_assets(id) + ON DELETE SET NULL; + END IF; +END $$; + +CREATE INDEX IF NOT EXISTS idx_radar_data_source_product_ref + ON radar_data (source_product_ref_id); + +CREATE INDEX IF NOT EXISTS idx_radar_data_source_archive_asset + ON radar_data (source_archive_asset_id); + +CREATE INDEX IF NOT EXISTS idx_radar_data_selected_orbit_asset + ON radar_data (selected_orbit_asset_id); + +CREATE INDEX IF NOT EXISTS idx_radar_data_orbit_binding_status + ON radar_data (orbit_binding_status); + +CREATE INDEX IF NOT EXISTS idx_radar_data_acquisition_start + ON radar_data (acquisition_start_time_utc); + +CREATE INDEX IF NOT EXISTS idx_radar_data_absolute_orbit + ON radar_data (absolute_orbit); + +CREATE INDEX IF NOT EXISTS idx_radar_data_relative_orbit + ON radar_data (relative_orbit); + +CREATE INDEX IF NOT EXISTS idx_radar_data_source_format + ON radar_data (source_format); + +CREATE INDEX IF NOT EXISTS idx_source_product_assets_asset_uid + ON source_product_assets (asset_uid); + +CREATE INDEX IF NOT EXISTS idx_source_product_assets_family_date + ON source_product_assets (satellite_family, imaging_date); + +CREATE INDEX IF NOT EXISTS idx_source_product_assets_satellite + ON source_product_assets (satellite); + +CREATE INDEX IF NOT EXISTS idx_source_product_assets_source_format + ON source_product_assets (source_format); + +CREATE INDEX IF NOT EXISTS idx_source_product_assets_parse_status + ON source_product_assets (parse_status); + +CREATE INDEX IF NOT EXISTS idx_source_product_assets_root_active + ON source_product_assets (root_ref_id, is_active); + +CREATE INDEX IF NOT EXISTS idx_source_product_assets_logical_product + ON source_product_assets (logical_product_uid); + +CREATE INDEX IF NOT EXISTS idx_source_product_assets_file_path + ON source_product_assets (file_path); + +CREATE INDEX IF NOT EXISTS idx_orbit_assets_orbit_uid + ON orbit_assets (orbit_uid); + +CREATE INDEX IF NOT EXISTS idx_orbit_assets_family_sat_window + ON orbit_assets (satellite_family, satellite, validity_start_time_utc, validity_stop_time_utc); + +CREATE INDEX IF NOT EXISTS idx_orbit_assets_orbit_type + ON orbit_assets (orbit_type); + +CREATE INDEX IF NOT EXISTS idx_orbit_assets_native_format + ON orbit_assets (native_format); + +CREATE INDEX IF NOT EXISTS idx_orbit_assets_quality_class + ON orbit_assets (quality_class); + +CREATE INDEX IF NOT EXISTS idx_orbit_assets_parse_status + ON orbit_assets (parse_status); + +CREATE INDEX IF NOT EXISTS idx_orbit_assets_root_active + ON orbit_assets (root_ref_id, is_active); + +CREATE INDEX IF NOT EXISTS idx_orbit_assets_file_path + ON orbit_assets (file_path); + +CREATE INDEX IF NOT EXISTS idx_scene_orbit_bindings_radar + ON scene_orbit_bindings (radar_data_id); + +CREATE INDEX IF NOT EXISTS idx_scene_orbit_bindings_orbit + ON scene_orbit_bindings (orbit_asset_id); + +CREATE INDEX IF NOT EXISTS idx_scene_orbit_bindings_match_status + ON scene_orbit_bindings (match_status); + +CREATE INDEX IF NOT EXISTS idx_scene_orbit_bindings_selection_status + ON scene_orbit_bindings (selection_status); + +CREATE INDEX IF NOT EXISTS idx_scene_orbit_bindings_scene_selected + ON scene_orbit_bindings (radar_data_id, selection_status); + +CREATE INDEX IF NOT EXISTS idx_orbit_asset_derivatives_asset_engine + ON orbit_asset_derivatives (orbit_asset_id, engine_code); + +CREATE INDEX IF NOT EXISTS idx_orbit_asset_derivatives_pool_path + ON orbit_asset_derivatives (pool_path); + +CREATE INDEX IF NOT EXISTS idx_orbit_asset_derivatives_generation_status + ON orbit_asset_derivatives (generation_status); + +CREATE INDEX IF NOT EXISTS idx_asset_inventory_states_type_status + ON asset_inventory_states (inventory_type, status); + +CREATE INDEX IF NOT EXISTS idx_asset_inventory_states_needs_rescan + ON asset_inventory_states (needs_rescan); + +CREATE INDEX IF NOT EXISTS idx_asset_inventory_issues_open + ON asset_inventory_issues (status, severity); + +CREATE INDEX IF NOT EXISTS idx_asset_inventory_issues_root_type + ON asset_inventory_issues (root_ref_id, inventory_type); + +CREATE INDEX IF NOT EXISTS idx_asset_inventory_issues_asset + ON asset_inventory_issues (asset_ref_id); + +CREATE INDEX IF NOT EXISTS idx_asset_inventory_issues_orbit + ON asset_inventory_issues (orbit_asset_id); + +CREATE INDEX IF NOT EXISTS idx_asset_inventory_issues_radar + ON asset_inventory_issues (radar_data_id); + +UPDATE radar_data +SET + orbit_binding_status = CASE + WHEN has_orbit_data IS TRUE AND NULLIF(orbit_file_path, '') IS NOT NULL THEN 'MATCHED' + WHEN has_orbit_data IS TRUE THEN 'MATCHED' + ELSE COALESCE(NULLIF(orbit_binding_status, ''), 'UNBOUND') + END +WHERE orbit_binding_status IS NULL + OR orbit_binding_status = '' + OR (has_orbit_data IS TRUE AND orbit_binding_status = 'UNBOUND'); + +UPDATE radar_data +SET absolute_orbit = COALESCE(NULLIF(absolute_orbit, ''), NULLIF(orbit_circle, '')) +WHERE absolute_orbit IS NULL OR absolute_orbit = ''; + +UPDATE radar_data +SET source_format = COALESCE( + NULLIF(source_format, ''), + CASE + WHEN upper(replace(replace(replace(COALESCE(satellite, ''), '-', ''), '_', ''), ' ', '')) LIKE 'S1%%' + AND lower(COALESCE(file_path, '')) LIKE '%%.zip' THEN 'S1_ZIP' + WHEN upper(replace(replace(replace(COALESCE(satellite, ''), '-', ''), '_', ''), ' ', '')) LIKE 'S1%%' + AND lower(COALESCE(file_path, '')) LIKE '%%.safe' THEN 'S1_SAFE_DIR' + WHEN upper(replace(replace(replace(COALESCE(satellite, ''), '-', ''), '_', ''), ' ', '')) LIKE 'LT1%%' THEN 'LT1_DIR' + WHEN upper(replace(replace(replace(COALESCE(satellite, ''), '-', ''), '_', ''), ' ', '')) LIKE 'GF3%%' THEN 'GF3_DIR' + ELSE NULL + END +) +WHERE source_format IS NULL OR source_format = ''; diff --git a/docs/INDEX.md b/docs/INDEX.md index e93e2f1..3c95ab6 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -60,6 +60,8 @@ ## 4. 配对与前端导航 +- [SENTINEL1_SOURCE_ORBIT_ASSET_DESIGN_20260512.md](SENTINEL1_SOURCE_ORBIT_ASSET_DESIGN_20260512.md) + Sentinel-1 / LT-1 源数据与精密轨道资产层设计，定义统一源产品库存、轨道资产、scene 绑定、启动自维护和健康检查边界。 - [SENTINEL1_SYSTEM_ENHANCEMENT_MASTER_PLAN_20260510.md](SENTINEL1_SYSTEM_ENHANCEMENT_MASTER_PLAN_20260510.md) Sentinel-1 系统增强主维护文档，汇总数据管理、精轨、配对、任务分发、Gamma/PyINT 生产、结果管理和分阶段实施边界。 - [SENTINEL1_DATA_MANAGEMENT_ADAPTATION_PLAN_20260510.md](SENTINEL1_DATA_MANAGEMENT_ADAPTATION_PLAN_20260510.md) @@ -125,4 +127,4 @@ - 已归档的 `项目汇报.md` - 已归档的各类 `*_EXPERIMENT_*` / `*_PROGRESS_*` / `*_TODO_*` -最后更新：2026-04-26 +最后更新：2026-05-12 diff --git a/docs/SENTINEL1_SOURCE_ORBIT_ASSET_DESIGN_20260512.md b/docs/SENTINEL1_SOURCE_ORBIT_ASSET_DESIGN_20260512.md new file mode 100644 index 0000000..73500a9 --- /dev/null +++ b/docs/SENTINEL1_SOURCE_ORBIT_ASSET_DESIGN_20260512.md @@ -0,0 +1,579 @@ +# Sentinel-1 / LT-1 源数据与精密轨道资产设计 + +日期：2026-05-12 + +本文定义源数据管理和精密轨道管理的下一阶段底座设计。目标是把 Sentinel-1 和 LT-1 放到同等位置和能力上，而不是把 Sentinel-1 临时塞进现有 LT-1 扫描和轨道字段里。 + +本文只覆盖源数据、精密轨道、资产库存、轨道绑定、启动自维护和健康检查。D-InSAR 配对、任务分发和生产 profile 不在本阶段实现范围内，但后续应消费这里建立的资产与绑定结果。 + +## 1. 目标 + +本阶段要完成： + +1. 统一管理 LT-1、Sentinel-1 的源产品资产。 +2. 统一管理 LT-1 原生精轨和 Sentinel-1 EOF 精轨。 +3. 让 `radar_data` 成为统一 scene 业务入口，而不是唯一资产库存表。 +4. 建立 scene 与精轨的候选、选中和异常状态。 +5. 把新 schema 纳入启动自维护、数据库 schema 检查和健康检查。 +6. 保持现有 LT-1、GF-3、水体、配对、结果 catalog 的兼容运行。 + +本阶段暂不做： + +1. 不新增 Sentinel-1 D-InSAR 生产链路。 +2. 不改造现有 `lt1_gamma_dinsar` 或 `lt1_stripmap` 为多卫星 profile。 +3. 不在启动时扫描大目录或解析大型 ZIP。 +4. 不把 Sentinel-1 EOF 送入 LT-1 txt 到 XML 转换链路。 + +## 2. 现有约束 + +当前系统启动链路中，`backend/app/main.py` 的 lifespan 会先调用 `ensure_database_ready()`，再初始化数据库会话、同步 managed roots、同步 manifest catalog、启动 pairing cache 状态。因此新资产层必须满足： + +- ORM 是 schema 事实来源。新增表和字段必须进入 ORM。 +- SQL migration 是约束、索引、幂等修补的事实来源。新增 migration 必须加入 `backend/app/db_maintenance.py` 的 `MIGRATION_FILES`。 +- migration 必须可重复运行，使用 `CREATE TABLE IF NOT EXISTS`、`ALTER TABLE ... ADD COLUMN IF NOT EXISTS`、`CREATE INDEX IF NOT EXISTS`。 +- 启动自维护只保证 schema 和轻量状态存在，不做 ZIP 解包、不扫大目录、不重建全量绑定。 +- 健康检查必须能报告资产层状态，不能只看旧的 `MONITOR_RADAR_DIRS` 和 `MONITOR_ORBIT_DIR` 是否存在。 + +现有兼容字段需要保留： + +- `radar_data.has_orbit_data` +- `radar_data.orbit_file_path` +- `radar_data.file_path` +- `radar_data.coverage_polygon` +- `radar_data.geom` + +这些字段继续服务旧接口和旧前端，但由新的资产与绑定结果回填。 + +## 3. 领域模型 + +新的数据模型分成五类。 + +### 3.1 源产品资产 + +`source_product_assets` 记录物理源产品。一个资产是一份实际存在的文件或目录，例如： + +- LT-1 scene 目录 +- LT-1 原始压缩包或 tiff +- Sentinel-1 `.zip` +- Sentinel-1 `.SAFE` 目录 +- GF-3 现有处理输入或输出目录，后续可接入同一库存 + +关键字段建议： + +```text +id +asset_uid +logical_product_uid +satellite_family +satellite +source_format +product_type +product_level +imaging_mode +polarization +absolute_orbit +relative_orbit +orbit_direction +acquisition_start_time_utc +acquisition_stop_time_utc +imaging_date +root_path +file_path +archive_path +path_kind +file_name +file_stem +file_ext +size_bytes +mtime_epoch +checksum_sha256 +checksum_status +parser_name +parser_version +parse_status +parse_error +parsed_at +metadata_json +is_active +missing_since +created_at +updated_at +``` + +字段语义： + +- `asset_uid` 是物理资产唯一键，优先由规范化绝对路径和大小/mtime 派生。 +- `logical_product_uid` 是同一个遥感产品的逻辑身份，用于把 Sentinel-1 ZIP 和解包后的 SAFE 关联起来。 +- `source_format` 使用稳定枚举，例如 `LT1_DIR`、`LT1_ARCHIVE`、`S1_ZIP`、`S1_SAFE_DIR`、`GF3_DIR`。 +- `archive_path` 用于记录生产更偏好的原始归档路径，例如 Sentinel-1 ZIP。 +- `metadata_json` 保存传感器专有字段，不把所有 Sentinel-1 annotation 字段拆成列。 + +### 3.2 Scene 业务入口 + +`radar_data` 仍然是前端检索、地图展示、水体、配对和后续生产的 scene 入口。它需要补充资产引用和 Sentinel-1/LT-1 通用字段： + +```text +source_product_ref_id +source_archive_asset_id +selected_orbit_asset_id +orbit_binding_status +orbit_binding_reason +acquisition_start_time_utc +acquisition_stop_time_utc +absolute_orbit +relative_orbit +source_format +metadata_json +``` + +字段语义： + +- `source_product_ref_id` 指向用于生成该 scene 元数据的主资产。 +- `source_archive_asset_id` 指向后续分发或生产优先使用的原始资产。Sentinel-1 通常是 ZIP。 +- `selected_orbit_asset_id` 指向当前选中的精轨资产。 +- `orbit_binding_status` 标识 `UNBOUND`、`MATCHED`、`MISSING`、`AMBIGUOUS`、`ERROR`。 +- 旧字段 `has_orbit_data` 和 `orbit_file_path` 从 `selected_orbit_asset_id` 兼容回填。 + +### 3.3 精密轨道资产 + +`orbit_assets` 记录原生轨道文件。它不记录派生到 ENVI/ISCE2 池里的文件。 + +关键字段建议： + +```text +id +orbit_uid +satellite_family +satellite +orbit_type +native_format +quality_class +root_path +file_path +file_name +file_stem +file_ext +size_bytes +mtime_epoch +checksum_sha256 +checksum_status +validity_start_time_utc +validity_stop_time_utc +generation_time_utc +published_time_utc +parser_name +parser_version +parse_status +parse_error +parsed_at +metadata_json +is_active +missing_since +created_at +updated_at +``` + +枚举建议： + +- `satellite_family`: `LT1`、`S1`、`GF3` +- `native_format`: `LT1_TXT`、`S1_EOF` +- `orbit_type`: `LT1_GPS`、`AUX_POEORB`、`AUX_RESORB` +- `quality_class`: `precise`、`restituted`、`predicted`、`unknown` + +Sentinel-1 EOF 的匹配逻辑基于： + +```text +orbit.satellite == scene.satellite +and orbit.validity_start_time_utc <= scene.acquisition_time <= orbit.validity_stop_time_utc +``` + +LT-1 第一阶段可继续复用现有日期匹配逻辑，后续如果轨道文件能提供明确有效期，也应收敛到相同的时间窗模型。 + +### 3.4 Scene 与轨道绑定 + +`scene_orbit_bindings` 记录每个 scene 的候选轨道和最终选择。 + +关键字段建议： + +```text +id +radar_data_id +orbit_asset_id +binding_role +match_status +selection_status +selection_rank +priority_score +coverage_margin_before_seconds +coverage_margin_after_seconds +match_rule_version +match_reason +selected_at +metadata_json +created_at +updated_at +``` + +字段语义： + +- `binding_role` 第一阶段使用 `primary_orbit`。 +- `match_status`: `CANDIDATE`、`MATCHED`、`REJECTED`、`STALE`、`ERROR`。 +- `selection_status`: `SELECTED`、`CANDIDATE`、`NOT_SELECTED`。 +- Sentinel-1 选择优先级：POEORB 优先于 RESORB；覆盖余量更大优先；generation time 更新优先。 +- LT-1 选择优先级：现有可用轨道优先；坏源精轨不得入选；派生池同步失败时保留资产但标注异常。 + +### 3.5 轨道派生产物 + +`orbit_asset_derivatives` 记录从原生轨道资产生成或同步到引擎池的文件。 + +关键字段建议： + +```text +id +orbit_asset_id +engine_code +derivative_format +derivative_role +pool_path +size_bytes +mtime_epoch +checksum_sha256 +generation_status +generation_error +generated_at +metadata_json +created_at +updated_at +``` + +用途： + +- LT-1 txt 同步到 ENVI pool。 +- LT-1 txt 转换成 ISCE2 XML。 +- Sentinel-1 EOF 第一阶段通常不转换，但后续可以记录 staging 到 OPOD 目录的结果。 + +### 3.6 库存状态与问题 + +为避免健康检查每次扫大目录，新增轻量状态表： + +`asset_inventory_states` + +```text +id +root_ref_id +inventory_type +root_path +scan_mode +status +last_scan_started_at +last_scan_finished_at +last_seen_entry_count +last_asset_count +last_issue_count +parser_version +needs_rescan +last_error +metadata_json +created_at +updated_at +``` + +`asset_inventory_issues` + +```text +id +root_ref_id +inventory_type +asset_ref_id +radar_data_id +orbit_asset_id +severity +issue_code +issue_message +source_path +status +first_seen_at +last_seen_at +resolved_at +metadata_json +``` + +典型 issue： + +- `source_path_missing` +- `source_parse_failed` +- `orbit_parse_failed` +- `duplicate_logical_product` +- `scene_missing_orbit` +- `scene_ambiguous_orbit` +- `selected_orbit_missing_file` +- `lt1_derivative_generation_failed` + +## 4. Root 与配置 + +建议新增通用配置，同时保留旧配置作为兼容入口： + +```text +SOURCE_PRODUCT_DIRS= +ORBIT_SOURCE_DIRS= +``` + +兼容关系： + +- `INSAR_STORAGE_DIRS` 和 `MONITOR_RADAR_DIRS` 继续生效，并作为 `SOURCE_PRODUCT_DIRS` 的兼容来源。 +- `MONITOR_ORBIT_DIR` 继续生效，并作为 `ORBIT_SOURCE_DIRS` 的兼容来源。 +- `ORBIT_POOL_ENVI`、`ORBIT_POOL_ISCE2` 仍然是派生轨道池，不作为原生轨道资产源。 + +`root_registry_service` 需要增加或调整 root role： + +- `source_product_pool` +- `legacy_scan_root_radar` +- `orbit_asset_pool` +- `orbit_pool_envi` +- `orbit_pool_isce2` + +每个 root 的 `metadata_json` 可以记录： + +```json +{ + "supported_families": ["LT1", "S1"], + "discovery_patterns": ["LT1*", "S1*.zip", "*.SAFE", "*.EOF"], + "imported_from": "settings" +} +``` + +对用户当前样本，推荐后续配置形态是： + +```text +SOURCE_PRODUCT_DIRS=D:\LuTan1_Image_Pool;D:\Sentinel1_Image_Pool_ZIP +ORBIT_SOURCE_DIRS=D:\LT1_data_lsarorbit;D:\Sentinel1_EOF_Pool +``` + +## 5. Parser 设计 + +新增解析器按资产类型组织： + +- `LT1SourceParser` +- `Sentinel1ZipParser` +- `Sentinel1SafeParser` +- `GF3SourceParser` +- `LT1OrbitParser` +- `Sentinel1EofParser` + +Sentinel-1 ZIP 解析原则： + +- 不全量解压。 +- 使用 Python `zipfile` 读取 `manifest.safe` 和 annotation XML。 +- 只读取必要 XML 文件和 quicklook 候选，不读取 measurement 大文件。 +- 文件名提供粗字段，XML 提供 relative orbit、升降轨、footprint、swath、极化细节。 +- `S1A`、`S1B`、`S1C` 都归一化为 `satellite_family = S1`。 + +Sentinel-1 EOF 解析原则： + +- 优先从文件名解析 mission、orbit type、generation time、validity start、validity stop。 +- 文件名不可靠时再读取 EOF XML 内容。 +- `AUX_POEORB` 标为 precise，`AUX_RESORB` 标为 restituted。 + +LT-1 解析原则： + +- 现有文件名和 XML 解析逻辑保留。 +- 现有坏源精轨检测和 quarantine 逻辑保留，但结果写入 `orbit_assets` 和 `asset_inventory_issues`。 +- 现有 `sync_orbit_pools()` 逐步下沉为 `orbit_asset_derivatives` 的生成步骤。 + +## 6. 扫描与绑定流程 + +### 6.1 源产品扫描 + +流程： + +1. 从 managed roots 找到 source product roots。 +2. 发现候选文件或目录。 +3. upsert `source_product_assets`。 +4. 调用对应 parser。 +5. 生成或更新 `radar_data`。 +6. 更新 `asset_inventory_states`。 +7. 记录解析失败或重复产品到 `asset_inventory_issues`。 + +`radar_data.unique_id` 的生成应优先使用稳定逻辑身份： + +```text +satellite_family + satellite + product_unique_id +``` + +如果缺少产品唯一号，再回退到规范化路径。 + +### 6.2 轨道扫描 + +流程： + +1. 从 managed roots 找到 orbit asset roots。 +2. 发现 LT-1 txt 和 Sentinel-1 EOF。 +3. upsert `orbit_assets`。 +4. 调用轨道 parser。 +5. LT-1 轨道执行坏源检测。 +6. 记录 parse status 和 inventory issues。 + +### 6.3 轨道绑定 + +流程： + +1. 找出新增或变更的 scene 与 orbit asset。 +2. 按卫星族调用绑定规则。 +3. 写入候选 `scene_orbit_bindings`。 +4. 选出 `selection_status = SELECTED` 的绑定。 +5. 回填 `radar_data.selected_orbit_asset_id`、`orbit_binding_status`、`has_orbit_data`、`orbit_file_path`。 +6. 对无轨道或多候选未决的 scene 写入 issue。 + +绑定规则版本需要显式记录，例如： + +```text +lt1_orbit_binding.v1 +s1_eof_window_binding.v1 +``` + +## 7. 启动自维护 + +新增资产层后，启动自维护必须按以下方式运行： + +1. `ensure_database_ready()` 创建 PostGIS 扩展。 +2. ORM `create_all()` 创建新增表和新增列。 +3. migration `010_source_orbit_asset_inventory.sql` 补齐索引、约束、兼容列、视图或必要函数。 +4. `root_registry_service.sync_from_settings()` 同步新旧 root。 +5. 启动阶段只初始化 `asset_inventory_states` 的空状态，不做真实扫描。 +6. 健康检查报告 `needs_rescan = true`，由用户手动触发扫描任务。 + +注意事项： + +- 不允许启动时遍历 `D:\Sentinel1_Image_Pool_ZIP` 这类大目录并打开 ZIP。 +- 不允许启动时自动删除资产记录。文件消失时标记 `is_active = false` 和 `missing_since`。 +- schema mismatch 必须在启动日志中可见，健康检查必须能展示缺表、缺列和资产库存异常。 + +## 8. 健康检查 + +`health_service` 增加 `asset_inventory` 检查项。 + +建议返回结构： + +```json +{ + "ok": true, + "source_roots": { + "configured_count": 2, + "accessible_count": 2, + "needs_rescan_count": 1 + }, + "orbit_roots": { + "configured_count": 2, + "accessible_count": 2, + "needs_rescan_count": 1 + }, + "source_assets": { + "total_count": 0, + "lt1_count": 0, + "s1_count": 0, + "parse_failed_count": 0 + }, + "orbit_assets": { + "total_count": 0, + "lt1_count": 0, + "s1_count": 0, + "parse_failed_count": 0 + }, + "bindings": { + "scene_count": 0, + "matched_count": 0, + "missing_count": 0, + "ambiguous_count": 0 + }, + "issues": { + "open_count": 0, + "error_count": 0, + "warning_count": 0 + } +} +``` + +整体健康状态建议： + +- schema 不完整：`ok = false` +- root 不可访问：`ok = false` +- 有 error 级 issue：`ok = false` +- 仅 `needs_rescan` 或 warning issue：`ok = true`，但显示 degraded/warning + +## 9. API 与前端边界 + +本阶段后端需要提供： + +- 源产品资产列表、详情、重扫。 +- 轨道资产列表、详情、重扫。 +- scene 的轨道绑定详情。 +- 资产库存健康摘要。 +- 手动触发源产品扫描、轨道扫描、轨道绑定重建。 + +前端数据管理页面需要显示： + +- satellite family: LT1、S1、GF3 +- source format: LT1_DIR、S1_ZIP、S1_SAFE_DIR +- relative orbit、absolute orbit +- acquisition start/stop +- orbit binding status +- selected orbit asset +- parser status 和 issue 摘要 + +轨道管理页面需要显示： + +- 原生轨道资产列表。 +- LT-1 与 Sentinel-1 的轨道类型。 +- validity window。 +- 派生产物状态。 +- 绑定到多少 scene。 +- 解析失败或缺文件问题。 + +## 10. 兼容与迁移策略 + +迁移后，现有业务继续读 `radar_data`。新资产层提供更完整的 provenance。 + +兼容规则： + +- 旧扫描入口可以保留，但应逐步改为调用资产扫描服务。 +- `radar_data.file_path` 继续保存主 scene 路径。 +- `radar_data.orbit_file_path` 继续保存选中原生轨道路径。 +- `radar_data.has_orbit_data` 继续表示是否存在选中轨道。 +- 现有 pairing cache 可以暂时继续使用旧字段，后续再改为读取 `relative_orbit` 和资产绑定状态。 + +数据回填： + +1. 对已有 `radar_data` 记录创建 `source_product_assets`。 +2. 对已有 `orbit_file_path` 创建 `orbit_assets`。 +3. 生成 `scene_orbit_bindings`。 +4. 确认兼容字段和新绑定结果一致。 + +## 11. 验收标准 + +以当前样本池作为第一轮验收： + +- `D:\Sentinel1_Image_Pool_ZIP` 中 29 个 Sentinel-1 ZIP 能登记为 `source_product_assets`。 +- S1A 与 S1C 都归入 `satellite_family = S1`。 +- 每个 ZIP 能生成或更新一条 `radar_data` scene。 +- `D:\Sentinel1_EOF_Pool` 中 24 个 EOF 能登记为 `orbit_assets`。 +- 29 个 Sentinel-1 scene 全部能通过 validity window 绑定到 EOF。 +- `has_orbit_data = true` 和 `orbit_file_path` 能兼容回填。 +- LT-1 现有源数据扫描和精轨同步不退化。 +- 启动后数据库健康检查不出现 schema mismatch。 +- 健康检查能展示源产品、轨道资产、绑定状态和 issue 统计。 + +## 12. 实施顺序 + +建议按以下顺序开工： + +1. Schema 与 ORM：新增资产表、绑定表、状态表，扩展 `radar_data`。 +2. Migration 与启动自维护：新增 `010_source_orbit_asset_inventory.sql`，加入 `MIGRATION_FILES`。 +3. 健康检查：新增 `asset_inventory` 状态，不依赖真实扫描也能返回稳定结构。 +4. Root registry：支持通用 `SOURCE_PRODUCT_DIRS`、`ORBIT_SOURCE_DIRS`，兼容旧配置。 +5. Parser 与库存扫描：先实现 Sentinel-1 ZIP、Sentinel-1 EOF、LT-1 现有源数据和轨道。 +6. 轨道绑定服务：实现 LT-1 日期绑定和 Sentinel-1 EOF 时间窗绑定。 +7. 兼容回填：更新 `radar_data.has_orbit_data`、`orbit_file_path`。 +8. API 与前端：展示资产、轨道、绑定和 issue。 +9. 回归测试：用 Sentinel-1 样本池和现有 LT-1 数据池验证。 + +这一路径先把底座做稳，再接 D-InSAR 配对和 `s1_gamma_dinsar` 生产 profile。 diff --git a/frontend/package-lock.json b/frontend/package-lock.json index a2aa4c7..ca79e8e 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -31,7 +31,7 @@ "eslint-plugin-react-hooks": "^5.2.0", "eslint-plugin-react-refresh": "^0.4.20", "globals": "^16.4.0", - "vite": "npm:rolldown-vite@7.1.12" + "vite": "^7.1.12" } }, "node_modules/@babel/code-frame": { @@ -316,38 +316,446 @@ "node": ">=6.9.0" } }, - "node_modules/@emnapi/core": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.5.0.tgz", - "integrity": "sha512-sbP8GzB1WDzacS8fgNPpHlp6C9VZe+SJP3F90W9rLemaQj2PzIuTEl1qDOYQf58YIpyjViI24y9aPWCjEzY2cg==", + "node_modules/@esbuild/aix-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz", + "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==", + "cpu": [ + "ppc64" + ], "dev": true, "license": "MIT", "optional": true, - "dependencies": { - "@emnapi/wasi-threads": "1.1.0", - "tslib": "^2.4.0" + "os": [ + "aix" + ], + "engines": { + "node": ">=18" } }, - "node_modules/@emnapi/runtime": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.5.0.tgz", - "integrity": "sha512-97/BJ3iXHww3djw6hYIfErCZFee7qCtrneuLa20UXFCOTCfBM2cvQHjWJ2EG0s0MtdNwInarqCTz35i4wWXHsQ==", + "node_modules/@esbuild/android-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz", + "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==", + "cpu": [ + "arm" + ], "dev": true, "license": "MIT", "optional": true, - "dependencies": { - "tslib": "^2.4.0" + "os": [ + "android" + ], + "engines": { + "node": ">=18" } }, - "node_modules/@emnapi/wasi-threads": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.1.0.tgz", - "integrity": "sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==", + "node_modules/@esbuild/android-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz", + "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==", + "cpu": [ + "arm64" + ], "dev": true, "license": "MIT", "optional": true, - "dependencies": { - "tslib": "^2.4.0" + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz", + "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz", + "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz", + "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz", + "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz", + "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz", + "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz", + "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz", + "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz", + "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz", + "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz", + "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz", + "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz", + "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz", + "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz", + "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz", + "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz", + "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz", + "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz", + "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz", + "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz", + "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz", + "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz", + "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" } }, "node_modules/@eslint-community/eslint-utils": { @@ -612,43 +1020,31 @@ "integrity": "sha512-M5UknZPHRu3DEDWoipU6sE8PdkZ6Z/S+v4dD+Ke8IaNlpdSQah50lz1KtcFBa2vsdOnwbbnxJwVM4wty6udA5w==", "license": "MIT" }, - "node_modules/@napi-rs/wasm-runtime": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.0.5.tgz", - "integrity": "sha512-TBr9Cf9onSAS2LQ2+QHx6XcC6h9+RIzJgbqG3++9TUZSH204AwEy5jg3BTQ0VATsyoGj4ee49tN/y6rvaOOtcg==", + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-beta.38", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.38.tgz", + "integrity": "sha512-N/ICGKleNhA5nc9XXQG/kkKHJ7S55u0x0XUJbbkmdCnFuoRkM1Il12q9q0eX19+M7KKUEPw/daUPIRnxhcxAIw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.60.3.tgz", + "integrity": "sha512-x35CNW/ANXG3hE/EZpRU8MXX1JDN86hBb2wMGAtltkz7pc6cxgjpy1OMMfDosOQ+2hWqIkag/fGok1Yady9nGw==", + "cpu": [ + "arm" + ], "dev": true, "license": "MIT", "optional": true, - "dependencies": { - "@emnapi/core": "^1.5.0", - "@emnapi/runtime": "^1.5.0", - "@tybys/wasm-util": "^0.10.1" - } + "os": [ + "android" + ] }, - "node_modules/@oxc-project/runtime": { - "version": "0.90.0", - "resolved": "https://registry.npmjs.org/@oxc-project/runtime/-/runtime-0.90.0.tgz", - "integrity": "sha512-TfWn2tT97Weq1/1kTc+6ZeQ3TTj8350HoovtWaUYkX1nie7ONBqeMvudpluj4rmt2jc+l1QsBV/U70Oqsv1S4A==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@oxc-project/types": { - "version": "0.90.0", - "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.90.0.tgz", - "integrity": "sha512-fWvaufWUcLtm/OBKcNmxUkR0kQW5ZKAF0t03BXPqdzpxmnVCmSKzvUDRCOKnSagSfNzG/3ZdKpComH3GMy881g==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/Boshen" - } - }, - "node_modules/@rolldown/binding-android-arm64": { - "version": "1.0.0-beta.39", - "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-beta.39.tgz", - "integrity": "sha512-mjraAJQ3VRLPb3BUgVigHvmAYhiBpEeSM0dhvaO6XHtJ0k1o9Ng1Z6Qvlp4/1wDiUf7a10L5c3yleoGZ2r0Maw==", + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.60.3.tgz", + "integrity": "sha512-xw3xtkDApIOGayehp2+Rz4zimfkaX65r4t47iy+ymQB2G4iJCBBfj0ogVg5jpvjpn8UWn/+q9tprxleYeNp3Hw==", "cpu": [ "arm64" ], @@ -657,15 +1053,12 @@ "optional": true, "os": [ "android" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } + ] }, - "node_modules/@rolldown/binding-darwin-arm64": { - "version": "1.0.0-beta.39", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.0-beta.39.tgz", - "integrity": "sha512-tnuiLq9vd08KsZeFkFgzCXVKsTgSZGn+YBQjHSEiUvXJy5pfUf82X/YyLCG8P6I+WDd2cgrcLilMBQPZgaNwkg==", + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.60.3.tgz", + "integrity": "sha512-vo6Y5Qfpx7/5EaamIwi0WqW2+zfiusVihKatLvtN1VFVy3D13uERk/6gZLU1UiHRL6fDXqj/ELIeVRGnvcTE1g==", "cpu": [ "arm64" ], @@ -674,15 +1067,12 @@ "optional": true, "os": [ "darwin" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } + ] }, - "node_modules/@rolldown/binding-darwin-x64": { - "version": "1.0.0-beta.39", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.0-beta.39.tgz", - "integrity": "sha512-wLFoB3ZM4AoeBlsP0eVbPzWfkEgvmnibMQEKUgWRfJnKhUWiSxl0kGdSw1fNYdX3KAqIeA5gPJNvSJmf6g5S3Q==", + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.60.3.tgz", + "integrity": "sha512-D+0QGcZhBzTN82weOnsSlY7V7+RMmPuF1CkbxyMAGE8+ZHeUjyb76ZiWmBlCu//AQQONvxcqRbwZTajZKqjuOw==", "cpu": [ "x64" ], @@ -691,15 +1081,26 @@ "optional": true, "os": [ "darwin" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } + ] }, - "node_modules/@rolldown/binding-freebsd-x64": { - "version": "1.0.0-beta.39", - "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.0-beta.39.tgz", - "integrity": "sha512-wzFZlixF9VMbyi++rHCU4Cy72SH11aBNnkadmvwTAbokwjYHi8NqxQ3/Lx00c700N6kwwuiTsbcGt5DEA9aROw==", + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.60.3.tgz", + "integrity": "sha512-6HnvHCT7fDyj6R0Ph7A6x8dQS/S38MClRWeDLqc0MdfWkxjiu1HSDYrdPhqSILzjTIC/pnXbbJbo+ft+gy/9hQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.60.3.tgz", + "integrity": "sha512-KHLgC3WKlUYW3ShFKnnosZDOJ0xjg9zp7au3sIm2bs/tGBeC2ipmvRh/N7JKi0t9Ue20C0dpEshi8WUubg+cnA==", "cpu": [ "x64" ], @@ -708,66 +1109,233 @@ "optional": true, "os": [ "freebsd" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } + ] }, - "node_modules/@rolldown/binding-linux-arm-gnueabihf": { - "version": "1.0.0-beta.39", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.0-beta.39.tgz", - "integrity": "sha512-eVnZcwGbje1uwdFjeQZQ6918RHgGIK7iTC+AoDsgetgAXQmQpnuWYQ9OWa5oTHNQyCkZbMfiHKgpkUPpceMecw==", + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.60.3.tgz", + "integrity": "sha512-DV6fJoxEYWJOvaZIsok7KrYl0tPvga5OZ2yvKHNNYyk/2roMLqQAbGhr78EQ5YhHpnhLKJD3S1WFusAkmUuV5g==", "cpu": [ "arm" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } + ] }, - "node_modules/@rolldown/binding-linux-arm64-gnu": { - "version": "1.0.0-beta.39", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.0-beta.39.tgz", - "integrity": "sha512-Td96iRQA0nmRZM6kJ3+LDDKWLh4bl0zqeR+IYxXwPZBw4iXSREzXrcZ3QqgFHqnXPgryIJEW1U1Ebh2xf+b2UA==", + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.60.3.tgz", + "integrity": "sha512-mQKoJAzvuOs6F+TZybQO4GOTSMUu7v0WdxEk24krQ/uUxXoPTtHjuaUuPmFhtBcM4K0ons8nrE3JyhTuCFtT/w==", + "cpu": [ + "arm" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.60.3.tgz", + "integrity": "sha512-Whjj2qoiJ6+OOJMGptTYazaJvjOJm+iKHpXQM1P3LzGjt7Ff++Tp7nH4N8J/BUA7R9IHfDyx4DJIflifwnbmIA==", "cpu": [ "arm64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } + ] }, - "node_modules/@rolldown/binding-linux-arm64-musl": { - "version": "1.0.0-beta.39", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.0-beta.39.tgz", - "integrity": "sha512-bcSIh1TFUoPcexJH+gO1sE6wpSR0j3UpWBnjAwyM1PRKfjtqN4R9Du90ofH5KsR/A35FT3eP4mdnhMDTd5Yt+A==", + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.60.3.tgz", + "integrity": "sha512-4YTNHKqGng5+yiZt3mg77nmyuCfmNfX4fPmyUapBcIk+BdwSwmCWGXOUxhXbBEkFHtoN5boLj/5NON+u5QC9tg==", "cpu": [ "arm64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } + ] }, - "node_modules/@rolldown/binding-linux-x64-gnu": { - "version": "1.0.0-beta.39", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.0-beta.39.tgz", - "integrity": "sha512-tYEcZdVGovEemh7ELr+VUoezGkuBgRZYvDHHW/HVIw9LQW5HKLtBIGLzFlOfu/Lq5b9FlDKl+lrY6weviaNnKw==", + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.60.3.tgz", + "integrity": "sha512-SU3kNlhkpI4UqlUc2VXPGK9o886ZsSeGfMAX2ba2b8DKmMXq4AL7KUrkSWVbb7koVqx41Yczx6dx5PNargIrEA==", + "cpu": [ + "loong64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.60.3.tgz", + "integrity": "sha512-6lDLl5h4TXpB1mTf2rQWnAk/LcXrx9vBfu/DT5TIPhvMhRWaZ5MxkIc8u4lJAmBo6klTe1ywXIUHFjylW505sg==", + "cpu": [ + "loong64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.60.3.tgz", + "integrity": "sha512-BMo8bOw8evlup/8G+cj5xWtPyp93xPdyoSN16Zy90Q2QZ0ZYRhCt6ZJSwbrRzG9HApFabjwj2p25TUPDWrhzqQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.60.3.tgz", + "integrity": "sha512-E0L8X1dZN1/Rph+5VPF6Xj2G7JJvMACVXtamTJIDrVI44Y3K+G8gQaMEAavbqCGTa16InptiVrX6eM6pmJ+7qA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.60.3.tgz", + "integrity": "sha512-oZJ/WHaVfHUiRAtmTAeo3DcevNsVvH8mbvodjZy7D5QKvCefO371SiKRpxoDcCxB3PTRTLayWBkvmDQKTcX/sw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.60.3.tgz", + "integrity": "sha512-Dhbyh7j9FybM3YaTgaHmVALwA8AkUwTPccyCQ79TG9AJUsMQqgN1DDEZNr4+QUfwiWvLDumW5vdwzoeUF+TNxQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.60.3.tgz", + "integrity": "sha512-cJd1X5XhHHlltkaypz1UcWLA8AcoIi1aWhsvaWDskD1oz2eKCypnqvTQ8ykMNI0RSmm7NkTdSqSSD7zM0xa6Ig==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.60.3.tgz", + "integrity": "sha512-DAZDBHQfG2oQuhY7mc6I3/qB4LU2fQCjRvxbDwd/Jdvb9fypP4IJ4qmtu6lNjes6B531AI8cg1aKC2di97bUxA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.60.3.tgz", + "integrity": "sha512-cRxsE8c13mZOh3vP+wLDxpQBRrOHDIGOWyDL93Sy0Ga8y515fBcC2pjUfFwUe5T7tqvTvWbCpg1URM/AXdWIXA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.60.3.tgz", + "integrity": "sha512-QaWcIgRxqEdQdhJqW4DJctsH6HCmo5vHxY0krHSX4jMtOqfzC+dqDGuHM87bu4H8JBeibWx7jFz+h6/4C8wA5Q==", "cpu": [ "x64" ], @@ -775,33 +1343,13 @@ "license": "MIT", "optional": true, "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } + "openbsd" + ] }, - "node_modules/@rolldown/binding-linux-x64-musl": { - "version": "1.0.0-beta.39", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.0-beta.39.tgz", - "integrity": "sha512-xf9QdMC+qwQxtFAty/9RxgCLFdp9pFl09g86hxGPzlzCtHUjd+BmeUnUTXvVC8CHJLWECLQbFP6/233XHG0blA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-openharmony-arm64": { - "version": "1.0.0-beta.39", - "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.0-beta.39.tgz", - "integrity": "sha512-QCvN02VpE6zFYry0zAU+29D5+O9tJELNt+OjuCubilZdD/S8xFdho7qBJaa3YhFYyA9cReOMVH8Z8b3yWb4hcA==", + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.60.3.tgz", + "integrity": "sha512-AaXwSvUi3QIPtroAUw1t5yHGIyqKEXwH54WUocFolZhpGDruJcs8c+xPNDRn4XiQsS7MEwnYsHW2l0MBLDMkWg==", "cpu": [ "arm64" ], @@ -810,32 +1358,12 @@ "optional": true, "os": [ "openharmony" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } + ] }, - "node_modules/@rolldown/binding-wasm32-wasi": { - "version": "1.0.0-beta.39", - "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.0-beta.39.tgz", - "integrity": "sha512-LFgshxApyBNiBHFVpun7tPrIQ4TvxW0f/endC5C4RzEHu7mxexBCQEkO5XrZ42Cr5DUY+ERNbkfNTUv+vVCaxQ==", - "cpu": [ - "wasm32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@napi-rs/wasm-runtime": "^1.0.5" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@rolldown/binding-win32-arm64-msvc": { - "version": "1.0.0-beta.39", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.0-beta.39.tgz", - "integrity": "sha512-Mykirawg+s1e0uzVSEFhUBTShvXrOghPnyuLYkCfw8gzy8bMYiJuxsAfcopzZIIAVOHeSblJoiA/e7gYFjg8HA==", + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.60.3.tgz", + "integrity": "sha512-65LAKM/bAWDqKNEelHlcHvm2V+Vfb8C6INFxQXRHCvaVN1rJfwr4NvdP4FyzUaLqWfaCGaadf6UbTm8xJeYfEg==", "cpu": [ "arm64" ], @@ -844,15 +1372,12 @@ "optional": true, "os": [ "win32" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } + ] }, - "node_modules/@rolldown/binding-win32-ia32-msvc": { - "version": "1.0.0-beta.39", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-1.0.0-beta.39.tgz", - "integrity": "sha512-4PQJfWx7mdzXbAa4y+3OSSo911BZyJ/Is4pJKiwcGUqtvY66MX7BqlNWMr9QAozArAGE2knDubLqCQwZpK631w==", + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.60.3.tgz", + "integrity": "sha512-EEM2gyhBF5MFnI6vMKdX1LAosE627RGBzIoGMdLloPZkXrUN0Ckqgr2Qi8+J3zip/8NVVro3/FjB+tjhZUgUHA==", "cpu": [ "ia32" ], @@ -861,15 +1386,12 @@ "optional": true, "os": [ "win32" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } + ] }, - "node_modules/@rolldown/binding-win32-x64-msvc": { - "version": "1.0.0-beta.39", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.0-beta.39.tgz", - "integrity": "sha512-0zmmPOWbFfp1g9ofieimHwhuclZMcib0HL52Q+JTRpOHChI2f83TtH3duKWtAaxqhLUndTr/Z5sxzb+G2FNL9g==", + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.60.3.tgz", + "integrity": "sha512-E5Eb5H/DpxaoXH++Qkv28RcUJboMopmdDUALBczvHMf7hNIxaDZqwY5lK12UK1BHacSmvupoEWGu+n993Z0y1A==", "cpu": [ "x64" ], @@ -878,28 +1400,21 @@ "optional": true, "os": [ "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.60.3.tgz", + "integrity": "sha512-hPt/bgL5cE+Qp+/TPHBqptcAgPzgj46mPcg/16zNUmbQk0j+mOEQV/+Lqu8QRtDV3Ek95Q6FeFITpuhl6OTsAA==", + "cpu": [ + "x64" ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/pluginutils": { - "version": "1.0.0-beta.38", - "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.38.tgz", - "integrity": "sha512-N/ICGKleNhA5nc9XXQG/kkKHJ7S55u0x0XUJbbkmdCnFuoRkM1Il12q9q0eX19+M7KKUEPw/daUPIRnxhcxAIw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@tybys/wasm-util": { - "version": "0.10.1", - "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz", - "integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==", "dev": true, "license": "MIT", "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } + "os": [ + "win32" + ] }, "node_modules/@types/babel__core": { "version": "7.20.5", @@ -1126,16 +1641,6 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/ansis": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/ansis/-/ansis-4.2.0.tgz", - "integrity": "sha512-HqZ5rWlFjGiV0tDm3UxxgNRqsOTniqoKZu0pIAfh7TZQMGuZK+hH0drySty0si0QXj1ieop4+SkSfPZBPPkHig==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=14" - } - }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -1525,16 +2030,6 @@ "node": ">=6" } }, - "node_modules/detect-libc": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.1.tgz", - "integrity": "sha512-ecqj/sy1jcK1uWrwpR67UhYrIFQ+5WlGxth34WquCbamhFA6hkkwiu37o6J5xCHdo1oixJRfVRw+ywV+Hq/0Aw==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=8" - } - }, "node_modules/devlop": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", @@ -1614,6 +2109,48 @@ "node": ">= 0.4" } }, + "node_modules/esbuild": { + "version": "0.25.12", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz", + "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.12", + "@esbuild/android-arm": "0.25.12", + "@esbuild/android-arm64": "0.25.12", + "@esbuild/android-x64": "0.25.12", + "@esbuild/darwin-arm64": "0.25.12", + "@esbuild/darwin-x64": "0.25.12", + "@esbuild/freebsd-arm64": "0.25.12", + "@esbuild/freebsd-x64": "0.25.12", + "@esbuild/linux-arm": "0.25.12", + "@esbuild/linux-arm64": "0.25.12", + "@esbuild/linux-ia32": "0.25.12", + "@esbuild/linux-loong64": "0.25.12", + "@esbuild/linux-mips64el": "0.25.12", + "@esbuild/linux-ppc64": "0.25.12", + "@esbuild/linux-riscv64": "0.25.12", + "@esbuild/linux-s390x": "0.25.12", + "@esbuild/linux-x64": "0.25.12", + "@esbuild/netbsd-arm64": "0.25.12", + "@esbuild/netbsd-x64": "0.25.12", + "@esbuild/openbsd-arm64": "0.25.12", + "@esbuild/openbsd-x64": "0.25.12", + "@esbuild/openharmony-arm64": "0.25.12", + "@esbuild/sunos-x64": "0.25.12", + "@esbuild/win32-arm64": "0.25.12", + "@esbuild/win32-ia32": "0.25.12", + "@esbuild/win32-x64": "0.25.12" + } + }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -2410,245 +2947,6 @@ "node": ">= 0.8.0" } }, - "node_modules/lightningcss": { - "version": "1.30.1", - "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.30.1.tgz", - "integrity": "sha512-xi6IyHML+c9+Q3W0S4fCQJOym42pyurFiJUHEcEyHS0CeKzia4yZDEsLlqOFykxOdHpNy0NmvVO31vcSqAxJCg==", - "dev": true, - "license": "MPL-2.0", - "dependencies": { - "detect-libc": "^2.0.3" - }, - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - }, - "optionalDependencies": { - "lightningcss-darwin-arm64": "1.30.1", - "lightningcss-darwin-x64": "1.30.1", - "lightningcss-freebsd-x64": "1.30.1", - "lightningcss-linux-arm-gnueabihf": "1.30.1", - "lightningcss-linux-arm64-gnu": "1.30.1", - "lightningcss-linux-arm64-musl": "1.30.1", - "lightningcss-linux-x64-gnu": "1.30.1", - "lightningcss-linux-x64-musl": "1.30.1", - "lightningcss-win32-arm64-msvc": "1.30.1", - "lightningcss-win32-x64-msvc": "1.30.1" - } - }, - "node_modules/lightningcss-darwin-arm64": { - "version": "1.30.1", - "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.30.1.tgz", - "integrity": "sha512-c8JK7hyE65X1MHMN+Viq9n11RRC7hgin3HhYKhrMyaXflk5GVplZ60IxyoVtzILeKr+xAJwg6zK6sjTBJ0FKYQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-darwin-x64": { - "version": "1.30.1", - "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.30.1.tgz", - "integrity": "sha512-k1EvjakfumAQoTfcXUcHQZhSpLlkAuEkdMBsI/ivWw9hL+7FtilQc0Cy3hrx0AAQrVtQAbMI7YjCgYgvn37PzA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-freebsd-x64": { - "version": "1.30.1", - "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.30.1.tgz", - "integrity": "sha512-kmW6UGCGg2PcyUE59K5r0kWfKPAVy4SltVeut+umLCFoJ53RdCUWxcRDzO1eTaxf/7Q2H7LTquFHPL5R+Gjyig==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-arm-gnueabihf": { - "version": "1.30.1", - "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.30.1.tgz", - "integrity": "sha512-MjxUShl1v8pit+6D/zSPq9S9dQ2NPFSQwGvxBCYaBYLPlCWuPh9/t1MRS8iUaR8i+a6w7aps+B4N0S1TYP/R+Q==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-arm64-gnu": { - "version": "1.30.1", - "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.30.1.tgz", - "integrity": "sha512-gB72maP8rmrKsnKYy8XUuXi/4OctJiuQjcuqWNlJQ6jZiWqtPvqFziskH3hnajfvKB27ynbVCucKSm2rkQp4Bw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-arm64-musl": { - "version": "1.30.1", - "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.30.1.tgz", - "integrity": "sha512-jmUQVx4331m6LIX+0wUhBbmMX7TCfjF5FoOH6SD1CttzuYlGNVpA7QnrmLxrsub43ClTINfGSYyHe2HWeLl5CQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-x64-gnu": { - "version": "1.30.1", - "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.30.1.tgz", - "integrity": "sha512-piWx3z4wN8J8z3+O5kO74+yr6ze/dKmPnI7vLqfSqI8bccaTGY5xiSGVIJBDd5K5BHlvVLpUB3S2YCfelyJ1bw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-x64-musl": { - "version": "1.30.1", - "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.30.1.tgz", - "integrity": "sha512-rRomAK7eIkL+tHY0YPxbc5Dra2gXlI63HL+v1Pdi1a3sC+tJTcFrHX+E86sulgAXeI7rSzDYhPSeHHjqFhqfeQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-win32-arm64-msvc": { - "version": "1.30.1", - "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.30.1.tgz", - "integrity": "sha512-mSL4rqPi4iXq5YVqzSsJgMVFENoa4nGTT/GjO2c0Yl9OuQfPsIfncvLrEW6RbbB24WtZ3xP/2CCmI3tNkNV4oA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-win32-x64-msvc": { - "version": "1.30.1", - "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.30.1.tgz", - "integrity": "sha512-PVqXh48wh4T53F/1CCu8PIPCxLzWyCnn/9T5W1Jpmdy5h9Cwd+0YQS6/LwhHXSafuc61/xg9Lv5OrCby6a++jg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -3673,47 +3971,51 @@ "node": ">=4" } }, - "node_modules/rolldown": { - "version": "1.0.0-beta.39", - "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.0-beta.39.tgz", - "integrity": "sha512-05bTT0CJU9dvCRC0Uc4zwB79W5N9MV9OG/Inyx8KNE2pSrrApJoWxEEArW6rmjx113HIx5IreCoTjzLfgvXTdg==", + "node_modules/rollup": { + "version": "4.60.3", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.60.3.tgz", + "integrity": "sha512-pAQK9HalE84QSm4Po3EmWIZPd3FnjkShVkiMlz1iligWYkWQ7wHYd1PF/T7QZ5TVSD6uSTon5gBVMSM4JfBV+A==", "dev": true, "license": "MIT", "dependencies": { - "@oxc-project/types": "=0.90.0", - "@rolldown/pluginutils": "1.0.0-beta.39", - "ansis": "^4.0.0" + "@types/estree": "1.0.8" }, "bin": { - "rolldown": "bin/cli.mjs" + "rollup": "dist/bin/rollup" }, "engines": { - "node": "^20.19.0 || >=22.12.0" + "node": ">=18.0.0", + "npm": ">=8.0.0" }, "optionalDependencies": { - "@rolldown/binding-android-arm64": "1.0.0-beta.39", - "@rolldown/binding-darwin-arm64": "1.0.0-beta.39", - "@rolldown/binding-darwin-x64": "1.0.0-beta.39", - "@rolldown/binding-freebsd-x64": "1.0.0-beta.39", - "@rolldown/binding-linux-arm-gnueabihf": "1.0.0-beta.39", - "@rolldown/binding-linux-arm64-gnu": "1.0.0-beta.39", - "@rolldown/binding-linux-arm64-musl": "1.0.0-beta.39", - "@rolldown/binding-linux-x64-gnu": "1.0.0-beta.39", - "@rolldown/binding-linux-x64-musl": "1.0.0-beta.39", - "@rolldown/binding-openharmony-arm64": "1.0.0-beta.39", - "@rolldown/binding-wasm32-wasi": "1.0.0-beta.39", - "@rolldown/binding-win32-arm64-msvc": "1.0.0-beta.39", - "@rolldown/binding-win32-ia32-msvc": "1.0.0-beta.39", - "@rolldown/binding-win32-x64-msvc": "1.0.0-beta.39" + "@rollup/rollup-android-arm-eabi": "4.60.3", + "@rollup/rollup-android-arm64": "4.60.3", + "@rollup/rollup-darwin-arm64": "4.60.3", + "@rollup/rollup-darwin-x64": "4.60.3", + "@rollup/rollup-freebsd-arm64": "4.60.3", + "@rollup/rollup-freebsd-x64": "4.60.3", + "@rollup/rollup-linux-arm-gnueabihf": "4.60.3", + "@rollup/rollup-linux-arm-musleabihf": "4.60.3", + "@rollup/rollup-linux-arm64-gnu": "4.60.3", + "@rollup/rollup-linux-arm64-musl": "4.60.3", + "@rollup/rollup-linux-loong64-gnu": "4.60.3", + "@rollup/rollup-linux-loong64-musl": "4.60.3", + "@rollup/rollup-linux-ppc64-gnu": "4.60.3", + "@rollup/rollup-linux-ppc64-musl": "4.60.3", + "@rollup/rollup-linux-riscv64-gnu": "4.60.3", + "@rollup/rollup-linux-riscv64-musl": "4.60.3", + "@rollup/rollup-linux-s390x-gnu": "4.60.3", + "@rollup/rollup-linux-x64-gnu": "4.60.3", + "@rollup/rollup-linux-x64-musl": "4.60.3", + "@rollup/rollup-openbsd-x64": "4.60.3", + "@rollup/rollup-openharmony-arm64": "4.60.3", + "@rollup/rollup-win32-arm64-msvc": "4.60.3", + "@rollup/rollup-win32-ia32-msvc": "4.60.3", + "@rollup/rollup-win32-x64-gnu": "4.60.3", + "@rollup/rollup-win32-x64-msvc": "4.60.3", + "fsevents": "~2.3.2" } }, - "node_modules/rolldown/node_modules/@rolldown/pluginutils": { - "version": "1.0.0-beta.39", - "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.39.tgz", - "integrity": "sha512-GkTtNCV8ObWbq3LrJStPBv9jkRPct8WlwotVjx3aU0RwfH3LyheixWK9Zhaj22C4EQj/TJxYyetoX+uOn/MWKw==", - "dev": true, - "license": "MIT" - }, "node_modules/scheduler": { "version": "0.26.0", "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.26.0.tgz", @@ -3877,14 +4179,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "dev": true, - "license": "0BSD", - "optional": true - }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -4064,19 +4358,17 @@ } }, "node_modules/vite": { - "name": "rolldown-vite", "version": "7.1.12", - "resolved": "https://registry.npmjs.org/rolldown-vite/-/rolldown-vite-7.1.12.tgz", - "integrity": "sha512-JREtUS+Lpa3s5Ha3ajf2F4LMS4BFxlVjpGz0k0ZR8rV3ZO3tzk5hukqyi9yRBcrvnTUg/BEForyCDahALFYAZA==", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.1.12.tgz", + "integrity": "sha512-ZWyE8YXEXqJrrSLvYgrRP7p62OziLW7xI5HYGWFzOvupfAlrLvURSzv/FyGyy0eidogEM3ujU+kUG1zuHgb6Ug==", "dev": true, "license": "MIT", "dependencies": { - "@oxc-project/runtime": "0.90.0", + "esbuild": "^0.25.0", "fdir": "^6.5.0", - "lightningcss": "^1.30.1", "picomatch": "^4.0.3", "postcss": "^8.5.6", - "rolldown": "1.0.0-beta.39", + "rollup": "^4.43.0", "tinyglobby": "^0.2.15" }, "bin": { @@ -4093,9 +4385,9 @@ }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", - "esbuild": "^0.25.0", "jiti": ">=1.21.0", "less": "^4.0.0", + "lightningcss": "^1.21.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", @@ -4108,15 +4400,15 @@ "@types/node": { "optional": true }, - "esbuild": { - "optional": true - }, "jiti": { "optional": true }, "less": { "optional": true }, + "lightningcss": { + "optional": true + }, "sass": { "optional": true }, diff --git a/frontend/package.json b/frontend/package.json index 6b49178..554325b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -11,7 +11,6 @@ }, "dependencies": { "axios": "^1.13.0", - "zustand": "^5.0.5", "chart.js": "^4.5.0", "chartjs-adapter-date-fns": "^3.0.0", "date-fns": "^4.1.0", @@ -21,7 +20,8 @@ "react": "^19.1.1", "react-chartjs-2": "^5.3.0", "react-dom": "^19.1.1", - "react-markdown": "^10.1.0" + "react-markdown": "^10.1.0", + "zustand": "^5.0.5" }, "devDependencies": { "@eslint/js": "^9.36.0", @@ -33,9 +33,6 @@ "eslint-plugin-react-hooks": "^5.2.0", "eslint-plugin-react-refresh": "^0.4.20", "globals": "^16.4.0", - "vite": "npm:rolldown-vite@7.1.12" - }, - "overrides": { - "vite": "npm:rolldown-vite@7.1.12" + "vite": "^7.1.12" } } diff --git a/frontend/src/App.css b/frontend/src/App.css index e7ff02b..3daffb6 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -2400,6 +2400,237 @@ input[type="checkbox"] { margin-top: 6px; } +.asset-inventory-panel { + padding: 12px; + display: flex; + flex-direction: column; + gap: 12px; + min-width: 0; +} + +.asset-toolbar { + display: flex; + justify-content: space-between; + gap: 10px; + align-items: flex-start; +} + +.asset-toolbar h3 { + margin: 0; + font-size: 16px; +} + +.asset-toolbar p { + margin: 4px 0 0; + color: var(--color-text-muted); + font-size: 12px; +} + +.asset-actions { + display: flex; + flex-wrap: wrap; + justify-content: flex-end; + gap: 6px; +} + +.asset-actions select, +.asset-actions button { + height: 30px; + font-size: 12px; +} + +.asset-message { + border: 1px solid var(--color-border); + background: var(--color-accent-soft); + color: var(--color-accent-strong); + border-radius: 6px; + padding: 8px 10px; + font-size: 12px; + word-break: break-word; +} + +.asset-message--error { + background: #fee2e2; + color: #991b1b; + border-color: #fecaca; +} + +.asset-metrics { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 8px; +} + +.asset-metric { + border: 1px solid var(--color-border); + border-radius: 8px; + padding: 8px; + background: #f8fafc; +} + +.asset-metric span, +.asset-metric small { + display: block; + color: var(--color-text-muted); + font-size: 11px; +} + +.asset-metric strong { + display: block; + margin: 2px 0; + font-size: 20px; + color: var(--color-text-primary); +} + +.asset-root-strip { + display: flex; + flex-direction: column; + gap: 6px; +} + +.asset-root-item { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + gap: 8px; + align-items: center; + border: 1px solid var(--color-border); + border-radius: 6px; + padding: 7px 8px; + background: #fff; +} + +.asset-root-item strong, +.asset-root-item span { + display: block; + min-width: 0; +} + +.asset-root-item span { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + color: var(--color-text-muted); + font-size: 11px; +} + +.asset-tabbar { + display: flex; + gap: 6px; +} + +.asset-tabbar button { + flex: 1; + font-size: 12px; + padding: 7px 6px; +} + +.asset-tabbar button.active-tab { + background: var(--color-accent); + color: #fff; + border-color: var(--color-accent-strong); +} + +.asset-table-wrap { + overflow-x: auto; + border: 1px solid var(--color-border); + border-radius: 8px; + background: #fff; +} + +.asset-table { + width: 100%; + min-width: 760px; + border-collapse: collapse; + font-size: 12px; +} + +.asset-table th, +.asset-table td { + border-bottom: 1px solid var(--color-border); + padding: 7px 8px; + text-align: left; + vertical-align: top; +} + +.asset-table th { + background: #f8fafc; + color: var(--color-text-secondary); + font-weight: 700; +} + +.asset-table td small { + display: block; + margin-top: 2px; + color: var(--color-text-muted); + max-width: 280px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.asset-inline-action { + min-width: 46px; + height: 26px; + padding: 0 8px; + font-size: 12px; + line-height: 1; +} + +.asset-action-placeholder { + color: var(--color-text-muted); + font-size: 12px; +} + +.asset-badge { + display: inline-flex; + align-items: center; + max-width: 100%; + border-radius: 999px; + padding: 2px 7px; + font-size: 11px; + font-weight: 700; + border: 1px solid var(--color-border); + white-space: nowrap; +} + +.asset-badge--ok { + background: #dcfce7; + color: #166534; + border-color: #bbf7d0; +} + +.asset-badge--warn { + background: #fef3c7; + color: #92400e; + border-color: #fde68a; +} + +.asset-badge--bad { + background: #fee2e2; + color: #991b1b; + border-color: #fecaca; +} + +.asset-badge--neutral { + background: #f1f5f9; + color: #475569; +} + +.asset-pager { + display: flex; + align-items: center; + justify-content: flex-end; + gap: 8px; + padding: 8px; + color: var(--color-text-muted); + font-size: 12px; +} + +.asset-pager button { + padding: 5px 9px; + font-size: 12px; +} + @media (max-width: 1200px) { .top-status-bar { grid-template-columns: 1fr; diff --git a/frontend/src/AssetInventoryPanel.jsx b/frontend/src/AssetInventoryPanel.jsx new file mode 100644 index 0000000..df3e067 --- /dev/null +++ b/frontend/src/AssetInventoryPanel.jsx @@ -0,0 +1,270 @@ +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { + getAssetInventoryStatus, + listAssetIssues, + listOrbitAssets, + listSourceAssets, + scanAssetInventory, +} from './api/assets'; + +const PAGE_SIZE = 100; + +const fmtDateTime = (value) => { + if (!value) return '-'; + const date = new Date(value); + if (Number.isNaN(date.getTime())) return String(value); + return date.toLocaleString(); +}; + +const fmtBytes = (value) => { + const n = Number(value); + if (!Number.isFinite(n) || n <= 0) return '-'; + if (n >= 1024 ** 3) return `${(n / (1024 ** 3)).toFixed(2)} GB`; + if (n >= 1024 ** 2) return `${(n / (1024 ** 2)).toFixed(1)} MB`; + if (n >= 1024) return `${(n / 1024).toFixed(1)} KB`; + return `${n} B`; +}; + +const StatusBadge = ({ value }) => { + const text = String(value || '-'); + const status = text.toUpperCase(); + const tone = status === 'OK' || status === 'MATCHED' || status === 'SELECTED' + ? 'ok' + : status === 'WARNING' || status === 'OPEN' || status === 'MISSING' + ? 'warn' + : status === 'FAILED' || status === 'INACCESSIBLE' || status === 'ERROR' + ? 'bad' + : 'neutral'; + return {text}; +}; + +const Metric = ({ label, value, hint }) => ( +
+ {label} + {value ?? 0} + {hint ? {hint} : null} +
+); + +export default function AssetInventoryPanel({ readOnly = false, onTaskStart }) { + const [status, setStatus] = useState(null); + const [sources, setSources] = useState({ items: [], total: 0, offset: 0, has_more: false }); + const [orbits, setOrbits] = useState({ items: [], total: 0, offset: 0, has_more: false }); + const [issues, setIssues] = useState({ items: [], total: 0, offset: 0, has_more: false }); + const [activeTab, setActiveTab] = useState('sources'); + const [family, setFamily] = useState('all'); + const [loading, setLoading] = useState(false); + const [scanLoading, setScanLoading] = useState(false); + const [message, setMessage] = useState(''); + const [error, setError] = useState(''); + + const familyParam = useMemo(() => (family === 'all' ? undefined : family), [family]); + + const refresh = useCallback(async ({ sourceOffset = 0, orbitOffset = 0, issueOffset = 0 } = {}) => { + setLoading(true); + setError(''); + try { + const [nextStatus, nextSources, nextOrbits, nextIssues] = await Promise.all([ + getAssetInventoryStatus(), + listSourceAssets({ satellite_family: familyParam, limit: PAGE_SIZE, offset: sourceOffset }), + listOrbitAssets({ satellite_family: familyParam, limit: PAGE_SIZE, offset: orbitOffset }), + listAssetIssues({ status: 'OPEN', limit: PAGE_SIZE, offset: issueOffset }), + ]); + setStatus(nextStatus); + setSources(nextSources); + setOrbits(nextOrbits); + setIssues(nextIssues); + } catch (err) { + setError(err?.response?.data?.detail || err.message || '加载资产库存失败'); + } finally { + setLoading(false); + } + }, [familyParam]); + + useEffect(() => { + refresh(); + }, [refresh]); + + const handleScan = async () => { + if (readOnly || scanLoading) return; + setScanLoading(true); + setMessage(''); + setError(''); + try { + const result = await scanAssetInventory({ inventory_types: [], root_ids: [], bind_orbits: true }); + setMessage(`资产扫描任务已入队: ${result.task_id}`); + onTaskStart?.(result.task_id, '源数据/精轨资产扫描已入队', { + taskType: 'SCAN_ASSET_INVENTORY', + nonBlocking: true, + }); + } catch (err) { + setError(err?.response?.data?.detail || err.message || '启动资产扫描失败'); + } finally { + setScanLoading(false); + } + }; + + const states = status?.states || []; + const sourceRoots = states.filter(item => item.inventory_type === 'source_product'); + const orbitRoots = states.filter(item => item.inventory_type === 'orbit_asset'); + + const renderPager = (data, onPage) => ( +
+ + {data.offset + 1}-{data.offset + data.items.length} / {data.total} + +
+ ); + + return ( +
+
+
+

源数据与精轨资产

+

Sentinel-1 与 LT-1 的源产品、精密轨道和绑定状态

+
+
+ + + +
+
+ + {error ?
{error}
: null} + {message ?
{message}
: null} + +
+ + + + +
+ +
+ {states.map((item) => ( +
+
+ {item.inventory_type === 'source_product' ? '源数据池' : '精轨池'} + {item.root_path} +
+ +
+ ))} +
+ +
+ + + +
+ + {activeTab === 'sources' && ( +
+ + + + + + + + + + + + + + {sources.items.map(item => { + return ( + + + + + + + + + + ); + })} + +
卫星日期/时间产品轨道状态动作文件
{item.satellite}{item.satellite_family}{item.imaging_date}{fmtDateTime(item.acquisition_start_time_utc)}{item.source_format}{item.imaging_mode} / {item.polarization}{item.relative_orbit || '-'}abs {item.absolute_orbit || '-'} + - + {item.file_name || item.logical_product_uid}{fmtBytes(item.size_bytes)}
+ {renderPager(sources, (offset) => refresh({ sourceOffset: offset, orbitOffset: orbits.offset, issueOffset: issues.offset }))} +
+ )} + + {activeTab === 'orbits' && ( +
+ + + + + + + + + + + + + {orbits.items.map(item => ( + + + + + + + + + ))} + +
卫星类型有效期质量状态文件
{item.satellite}{item.satellite_family}{item.orbit_type}{item.native_format}{fmtDateTime(item.validity_start_time_utc)}{fmtDateTime(item.validity_stop_time_utc)}{item.quality_class}{item.file_name}{fmtBytes(item.size_bytes)}
+ {renderPager(orbits, (offset) => refresh({ sourceOffset: sources.offset, orbitOffset: offset, issueOffset: issues.offset }))} +
+ )} + + {activeTab === 'issues' && ( +
+ + + + + + + + + + + + {issues.items.map(item => ( + + + + + + + + ))} + +
级别代码对象说明时间
{item.issue_code}{item.inventory_type}{item.source_path || `radar ${item.radar_data_id || '-'}`}{item.issue_message || '-'}{fmtDateTime(item.last_seen_at)}
+ {renderPager(issues, (offset) => refresh({ sourceOffset: sources.offset, orbitOffset: orbits.offset, issueOffset: offset }))} +
+ )} +
+ ); +} diff --git a/frontend/src/DataMonitorPanel.jsx b/frontend/src/DataMonitorPanel.jsx index 6bf6965..6adaddd 100644 --- a/frontend/src/DataMonitorPanel.jsx +++ b/frontend/src/DataMonitorPanel.jsx @@ -1,6 +1,7 @@ import React, { useEffect, useRef, useState } from 'react'; import './App.css'; import { useI18n } from './i18n/I18nContext'; +import { getAssetInventoryStatus, scanAssetInventory, unpackSentinel1Batch } from './api/assets'; const DEFAULT_MONITOR_CONFIG = { radar_dirs: [], @@ -8,6 +9,9 @@ const DEFAULT_MONITOR_CONFIG = { dinsar_dirs: [], gf3_source_dirs: [], gf3_storage_dirs: [], + s1_source_dirs: [], + s1_storage_dirs: [], + s1_orbit_dirs: [], }; const DEFAULT_UNPACK_CONFIG = { @@ -42,6 +46,7 @@ const parseUnpackRunValue = (rawValue, label) => { }; const formatList = (list) => (Array.isArray(list) && list.length ? list.join('; ') : '未配置'); +const normalizeComparePath = (value) => String(value || '').trim().replace(/\\/g, '/').replace(/\/+$/, '').toLowerCase(); const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled = true }) => { const { t } = useI18n(); @@ -50,8 +55,6 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled const [logs, setLogs] = useState([]); const [activeTasks, setActiveTasks] = useState([]); const [unpackConfig, setUnpackConfig] = useState(DEFAULT_UNPACK_CONFIG); - const [loading, setLoading] = useState(false); - const [message, setMessage] = useState(''); const [unpackLoading, setUnpackLoading] = useState(false); const [unpackMessage, setUnpackMessage] = useState(''); const [showUnpackDialog, setShowUnpackDialog] = useState(false); @@ -59,7 +62,11 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled const [unpackDialogError, setUnpackDialogError] = useState(''); const [unpackTaskId, setUnpackTaskId] = useState(''); const [unpackTaskTerminal, setUnpackTaskTerminal] = useState(false); + const [s1Loading, setS1Loading] = useState(false); + const [s1ScanLoading, setS1ScanLoading] = useState(false); + const [s1Message, setS1Message] = useState(''); const [gf3Loading, setGf3Loading] = useState(false); + const [gf3ProcessLoading, setGf3ProcessLoading] = useState(false); const [gf3Message, setGf3Message] = useState(''); const logEndRef = useRef(null); @@ -76,6 +83,7 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled const unpackActiveTask = displayActiveTasks.find((task) => task.task_id === unpackTaskId || task.task_type === 'UNPACK_ARCHIVES' ); + const s1ActiveTask = displayActiveTasks.find((task) => task.task_type === 'UNPACK_SENTINEL1'); useEffect(() => { if (!enabled) { @@ -102,6 +110,9 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled ...data, radar_dirs: toArray(data?.radar_dirs), dinsar_dirs: toArray(data?.dinsar_dirs), + s1_source_dirs: toArray(data?.s1_source_dirs), + s1_storage_dirs: toArray(data?.s1_storage_dirs), + s1_orbit_dirs: toArray(data?.s1_orbit_dirs), gf3_source_dirs: toArray(data?.gf3_source_dirs), gf3_storage_dirs: toArray(data?.gf3_storage_dirs), }); @@ -267,51 +278,134 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled const hasRadarDirs = config.radar_dirs.length > 0; const hasOrbitDir = typeof config.orbit_dir === 'string' && config.orbit_dir.trim() !== ''; - const hasDinsarDirs = config.dinsar_dirs.length > 0; + const hasS1SourceDirs = config.s1_source_dirs.length > 0; + const hasS1StorageDirs = config.s1_storage_dirs.length > 0; + const hasS1OrbitDirs = config.s1_orbit_dirs.length > 0; const hasGf3SourceDirs = config.gf3_source_dirs.length > 0; const hasGf3StorageDirs = config.gf3_storage_dirs.length > 0; const canRunRadar = !readOnly && configLoaded && hasRadarDirs; const canRunOrbit = !readOnly && configLoaded && hasOrbitDir; - const canRunDinsar = !readOnly && configLoaded && hasDinsarDirs; + const canRunS1 = !readOnly && configLoaded && (hasS1SourceDirs || hasS1StorageDirs || hasS1OrbitDirs); + const canRunS1Scan = !readOnly && configLoaded && hasS1SourceDirs; + const canRunS1OrbitScan = !readOnly && configLoaded && hasS1OrbitDirs; const canRunGf3Scan = !readOnly && configLoaded && hasGf3StorageDirs; const canRunGf3Process = !readOnly && configLoaded && hasGf3SourceDirs; const canOpenUnpackDialog = !readOnly && unpackConfig.source_dirs.length > 0; - const handleRunNow = async (target) => { + const handleS1Run = async () => { if (readOnly) { - setMessage('当前账户为只读模式，无法触发扫描。'); + setS1Message('当前账户为只读模式，无法触发 Sentinel-1 任务。'); return; } - - setLoading(true); - const targetMap = { - radar: 'LT-1 数据', - orbit: '精轨数据', - dinsar: 'D-InSAR 结果', - gf3: 'GF3 数据', - }; - setMessage(`正在触发${targetMap[target] || '全部'}手动扫描...`); - + setS1Loading(true); + setS1Message('Sentinel-1 任务启动中...'); try { - const url = target ? `${apiEndpoint}/monitor/run-now?target=${target}` : `${apiEndpoint}/monitor/run-now`; - const res = await fetch(url, { + const res = await unpackSentinel1Batch({ + scan_before_unpack: true, + overwrite: false, + }); + setS1Message(res.message || 'Sentinel-1 任务已启动'); + if (onTaskStart) { + onTaskStart(res.task_id, 'Sentinel-1 任务已启动。', { + nonBlocking: true, + taskType: 'UNPACK_SENTINEL1', + }); + } + } catch (err) { + setS1Message(`失败：${err?.response?.data?.detail || err.message || '未知错误'}`); + } finally { + setS1Loading(false); + } + }; + + const handleS1Scan = async () => { + if (readOnly) { + setS1Message('当前账户为只读模式，无法触发 Sentinel-1 扫描。'); + return; + } + setS1ScanLoading(true); + setS1Message('Sentinel-1 源数据扫描启动中...'); + try { + const inventoryStatus = await getAssetInventoryStatus(); + const sourcePathSet = new Set(config.s1_source_dirs.map(normalizeComparePath)); + const rootIds = (inventoryStatus?.states || []) + .filter((item) => item?.inventory_type === 'source_product' && sourcePathSet.has(normalizeComparePath(item?.root_path))) + .map((item) => item.root_ref_id) + .filter((value, index, array) => value && array.indexOf(value) === index); + const res = await scanAssetInventory({ + inventory_types: ['source_product'], + root_ids: rootIds, + bind_orbits: true, + }); + setS1Message(res.message || 'Sentinel-1 源数据扫描任务已启动'); + onTaskStart?.(res.task_id, 'Sentinel-1 源数据扫描任务已启动。', { + nonBlocking: true, + taskType: 'SCAN_ASSET_INVENTORY', + }); + } catch (err) { + setS1Message(`失败：${err?.response?.data?.detail || err.message || '未知错误'}`); + } finally { + setS1ScanLoading(false); + } + }; + + const handleS1OrbitScan = async () => { + if (readOnly) { + setS1Message('当前账户为只读模式，无法触发 Sentinel-1 精轨扫描。'); + return; + } + setS1ScanLoading(true); + setS1Message('Sentinel-1 精轨扫描启动中...'); + try { + const inventoryStatus = await getAssetInventoryStatus(); + const orbitPathSet = new Set(config.s1_orbit_dirs.map(normalizeComparePath)); + const rootIds = (inventoryStatus?.states || []) + .filter((item) => item?.inventory_type === 'orbit_asset' && orbitPathSet.has(normalizeComparePath(item?.root_path))) + .map((item) => item.root_ref_id) + .filter((value, index, array) => value && array.indexOf(value) === index); + const res = await scanAssetInventory({ + inventory_types: ['orbit_asset'], + root_ids: rootIds, + bind_orbits: true, + }); + setS1Message(res.message || 'Sentinel-1 精轨扫描任务已启动'); + onTaskStart?.(res.task_id, 'Sentinel-1 精轨扫描任务已启动。', { + nonBlocking: true, + taskType: 'SCAN_ASSET_INVENTORY', + }); + } catch (err) { + setS1Message(`失败：${err?.response?.data?.detail || err.message || '未知错误'}`); + } finally { + setS1ScanLoading(false); + } + }; + + const handleGf3BatchProcess = async () => { + if (readOnly) { + setGf3Message('当前账户为只读模式，无法触发 GF3 预处理。'); + return; + } + setGf3ProcessLoading(true); + setGf3Message('GF3 预处理启动中...'); + try { + const res = await fetch(`${apiEndpoint}/monitor/gf3-process`, { method: 'POST', credentials: 'include', }); const data = await parseJsonSafe(res, {}); if (res.ok) { - setMessage(data.message || '扫描任务已启动。'); + setGf3Message(data.message || 'GF3 批量处理任务已启动'); if (onTaskStart) { - onTaskStart(data.task_id, `已触发${targetMap[target] || '全部'}手动扫描...`); + onTaskStart(data.task_id, 'GF3 L1A→L2 批量处理已启动。'); } } else { - setMessage(`触发失败: ${data.detail || '未知错误'}`); + setGf3Message(`失败：${data.detail || '未知错误'}`); } } catch (err) { - setMessage(`触发失败: ${err.message}`); + setGf3Message(`失败：${err.message || '未知错误'}`); } finally { - setLoading(false); + setGf3ProcessLoading(false); } }; @@ -397,29 +491,79 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled } }; - const handleGf3BatchProcess = async () => { + const handleRadarScan = async () => { if (readOnly) { - setGf3Message('当前账户为只读模式，无法触发 GF3 处理。'); + setUnpackMessage('当前账户为只读模式，无法触发扫描。'); return; } - setGf3Loading(true); - setGf3Message('GF3 批量处理启动中...'); + setUnpackMessage('LT-1 扫描启动中...'); try { - const res = await fetch(`${apiEndpoint}/monitor/gf3-process`, { + const res = await fetch(`${apiEndpoint}/monitor/run-now?target=radar`, { method: 'POST', credentials: 'include', }); const data = await parseJsonSafe(res, {}); if (res.ok) { - setGf3Message(data.message || 'GF3 批量处理任务已启动'); + setUnpackMessage(data.message || 'LT-1 扫描任务已启动'); if (onTaskStart) { - onTaskStart(data.task_id, 'GF3 L1A→L2 批量处理已启动。'); + onTaskStart(data.task_id, '已触发 LT-1 手动扫描...'); + } + } else { + setUnpackMessage(`失败：${data.detail || '未知错误'}`); + } + } catch (err) { + setUnpackMessage(`失败：${err.message || '未知错误'}`); + } + }; + + const handleOrbitScan = async () => { + if (readOnly) { + setUnpackMessage('当前账户为只读模式，无法触发扫描。'); + return; + } + setUnpackMessage('精轨扫描启动中...'); + try { + const res = await fetch(`${apiEndpoint}/monitor/run-now?target=orbit`, { + method: 'POST', + credentials: 'include', + }); + const data = await parseJsonSafe(res, {}); + if (res.ok) { + setUnpackMessage(data.message || '精轨扫描任务已启动'); + if (onTaskStart) { + onTaskStart(data.task_id, '已触发精轨手动扫描...'); + } + } else { + setUnpackMessage(`失败：${data.detail || '未知错误'}`); + } + } catch (err) { + setUnpackMessage(`失败：${err.message || '未知错误'}`); + } + }; + + const handleGf3Scan = async () => { + if (readOnly) { + setGf3Message('当前账户为只读模式，无法触发扫描。'); + return; + } + setGf3Loading(true); + setGf3Message('GF3 扫描启动中...'); + try { + const res = await fetch(`${apiEndpoint}/monitor/run-now?target=gf3`, { + method: 'POST', + credentials: 'include', + }); + const data = await parseJsonSafe(res, {}); + if (res.ok) { + setGf3Message(data.message || 'GF3 扫描任务已启动'); + if (onTaskStart) { + onTaskStart(data.task_id, '已触发 GF3 手动扫描...'); } } else { setGf3Message(`失败：${data.detail || '未知错误'}`); } } catch (err) { - setGf3Message(`失败：${err.message}`); + setGf3Message(`失败：${err.message || '未知错误'}`); } finally { setGf3Loading(false); } @@ -437,17 +581,6 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled const rowStyle = { display: 'flex', gap: '8px' }; const gridStyle = { display: 'grid', rowGap: '6px', fontSize: '0.9em', color: 'var(--color-text-secondary)' }; - const scanBtnStyle = (canRun) => ({ - flex: 1, - padding: '8px 5px', - backgroundColor: 'var(--color-info)', - color: 'white', - border: 'none', - borderRadius: '4px', - cursor: loading || !canRun ? 'not-allowed' : 'pointer', - fontSize: '0.85em', - }); - const actionBtnStyle = (isLoading, isDisabled) => ({ padding: '6px 10px', backgroundColor: 'var(--color-accent)', @@ -495,6 +628,9 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
LT-1 存储{formatList(config.radar_dirs)}
LT-1 精轨{config.orbit_dir || '未配置'}
+
S1 源数据{formatList(config.s1_source_dirs)}
+
S1 存储{formatList(config.s1_storage_dirs)}
+
S1 精轨{formatList(config.s1_orbit_dirs)}
GF3 来源{formatList(config.gf3_source_dirs)}
GF3 存储{formatList(config.gf3_storage_dirs)}
D-InSAR 结果{formatList(config.dinsar_dirs)}
@@ -502,14 +638,14 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
-
LT-1 归档解包
+
LT-1 归档解包 / 扫描
来源目录{formatList(unpackConfig.source_dirs)}
LT-1 存储{formatList(unpackConfig.insar_storage_dirs)}
单次上限{unpackConfig.max_files_per_run > 0 ? `${unpackConfig.max_files_per_run} 个压缩包` : '不限'}
最长运行{unpackConfig.max_runtime_minutes > 0 ? `${unpackConfig.max_runtime_minutes} 分钟` : '不限'}
-
+
+ +
-
GF3 L1A → L2 处理
+
Sentinel-1 解包 / 扫描
+
+
S1 源数据{formatList(config.s1_source_dirs)}
+
S1 存储{formatList(config.s1_storage_dirs)}
+
S1 精轨{formatList(config.s1_orbit_dirs)}
+
+
+ + + +
+ {s1ActiveTask ? (s1ActiveTask.message || 'Sentinel-1 任务运行中...') : s1Message} +
+
+
+ +
+
GF3 归档预处理
L1A 来源{formatList(config.gf3_source_dirs)}
L2 存储{formatList(config.gf3_storage_dirs)}
@@ -538,10 +729,17 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
+
- -
-
活动任务
- {displayActiveTasks.length === 0 ? ( -
当前无活动任务。
- ) : ( -
- {displayActiveTasks.slice(0, 4).map((task) => ( -
-
- {task.task_type} - {task.progress}% -
-
-
-
-
{t(task.message || '')}
-
- ))} -
- )} -
- -
-

实时日志

-
- {displayLogs.length === 0 ? ( -
暂无日志...
- ) : ( - displayLogs.map((log, index) => ( -
{t(log)}
- )) - )} -
-
-
-
- - - - +
实时日志
+
+ {displayLogs.length === 0 ? ( +
暂无日志...
+ ) : ( + displayLogs.map((log, index) => ( +
{t(log)}
+ )) + )} +
- {message &&
{message}
}
{showUnpackDialog && ( diff --git a/frontend/src/DinsarProductionPanel.jsx b/frontend/src/DinsarProductionPanel.jsx index 2ac7220..9d3562d 100644 --- a/frontend/src/DinsarProductionPanel.jsx +++ b/frontend/src/DinsarProductionPanel.jsx @@ -2,6 +2,7 @@ import { deleteRunLog, deleteRunRecord, getRunLog, listEngines, listRuns, previewPyintInputAssets, submitRun } from './api/dinsarProduction'; import { clearTaskLogs, deleteTaskLog, deleteTaskRecord, getActiveTasks, getRecentTasks, getTaskLogs } from './api/tasks'; +import { formatSatelliteFamilyLabel, inferSatelliteFamilyFromResultLike } from './utils/satelliteFamily'; const card = { background: '#fff', @@ -160,6 +161,9 @@ function taskToRunRow(task) { completed_items: null, failed_items: null, skipped_items: null, + master_satellite: task?.master_satellite || '', + slave_satellite: task?.slave_satellite || '', + pair_key: task?.pair_key || '', }; } @@ -1375,12 +1379,12 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) }} >
-
-
PyINT 输入资产预检
-
- 提交前检查 Task_* 结构、DEM 策略和 LT-1 轨道是否齐备。即使不手动预检，后端提交时也会做同样校验。 +
+
PyINT 输入资产预检
+
+ 提交前检查 Task_* 结构、DEM 策略以及生产所需源数据和轨道文件是否齐备。即使不手动预检，后端提交时也会做同样校验。 +
-
+ ); + })()}
diff --git a/frontend/src/components/app/AppSidePanel.jsx b/frontend/src/components/app/AppSidePanel.jsx index 6a393fb..5bdb65a 100644 --- a/frontend/src/components/app/AppSidePanel.jsx +++ b/frontend/src/components/app/AppSidePanel.jsx @@ -14,6 +14,7 @@ import { getLeftTabLabel } from '../../utils/appUiHelpers'; import { PanelLoadingBody, PanelLoadingPanel } from './AppLoadingFallbacks'; const LazyDataMonitorPanel = lazy(() => import('../../DataMonitorPanel')); +const LazyAssetInventoryPanel = lazy(() => import('../../AssetInventoryPanel')); const LazyDataCopierPanel = lazy(() => import('../../DataCopierPanel')); const LazyIDLAutomationPanel = lazy(() => import('../../IDLAutomationPanel')); const LazyHazardPointPanel = lazy(() => import('../../HazardPointPanel')); @@ -237,6 +238,17 @@ export default function AppSidePanel({
)} + {leftPanelTab === 'asset_inventory' && ( +
+ }> + + +
+ )} + {leftPanelTab === 'pairing' && ( }> formatYmd(value, language)) : null; const engineMeta = getDinsarEngineMeta(result.engine_code); + const satelliteFamily = inferSatelliteFamilyFromResultLike(result); const hasTrace = !!( result.selection_strategy || result.network_run_id || @@ -45,6 +47,14 @@ function DinsarResultRow({ > {engineMeta.shortLabel} + {satelliteFamily && ( + + {formatSatelliteFamilyLabel(satelliteFamily)} + + )} {result.ai_score !== null && ( 0.7 ? 'good' : (result.ai_score < 0.4 ? 'bad' : 'medium')}`} diff --git a/frontend/src/config/appConstants.js b/frontend/src/config/appConstants.js index c0cc6f0..a580f3a 100644 --- a/frontend/src/config/appConstants.js +++ b/frontend/src/config/appConstants.js @@ -145,7 +145,7 @@ export const LEFT_GROUP_SECTIONS = { }; export const LEFT_GROUP_TABS = { - data: ['ingest', 'data', 'hazard'], + data: ['ingest', 'asset_inventory', 'data', 'hazard'], production_planning: LEFT_GROUP_SECTIONS.production_planning.flatMap(section => section.tabs), production_management: [PRODUCTION_WORKSPACE_TAB], insar_analysis: LEFT_GROUP_SECTIONS.insar_analysis.flatMap(section => section.tabs), @@ -180,6 +180,7 @@ export const FULL_WIDTH_LEFT_TABS = new Set([ export const ADMIN_ONLY_TABS = new Set([ 'ingest', + 'asset_inventory', 'pairing', 'pairs', 'ps_results', @@ -199,6 +200,7 @@ export const BATCH_API_MAX_PAGES = 200; export const SATELLITE_GROUPS = [ { key: 'LT-1', label: 'LT-1', prefixes: ['LT1'] }, + { key: 'S1', label: 'Sentinel-1', prefixes: ['S1'] }, { key: 'GF-3', label: 'GF-3', prefixes: ['GF3'] }, ]; diff --git a/frontend/src/hooks/useDinsarOperations.js b/frontend/src/hooks/useDinsarOperations.js index cfe2400..c94be9d 100644 --- a/frontend/src/hooks/useDinsarOperations.js +++ b/frontend/src/hooks/useDinsarOperations.js @@ -11,7 +11,7 @@ import { normalizePagePayload } from '../utils/appHelpers'; import { normalizeTaskStatus } from '../utils/appUiHelpers'; import { DEFAULT_LIST_PAGE_SIZE } from '../config/appConstants'; -const NON_BLOCKING_TASK_TYPES = new Set(['UNPACK_ARCHIVES', 'COPY_DATA']); +const NON_BLOCKING_TASK_TYPES = new Set(['UNPACK_ARCHIVES', 'UNPACK_SENTINEL1', 'SCAN_ASSET_INVENTORY', 'COPY_DATA']); export default function useDinsarOperations({ onCleanupDinsarLayers, diff --git a/frontend/src/hooks/useGlobalTaskControl.js b/frontend/src/hooks/useGlobalTaskControl.js index afb7765..728a86e 100644 --- a/frontend/src/hooks/useGlobalTaskControl.js +++ b/frontend/src/hooks/useGlobalTaskControl.js @@ -2,7 +2,7 @@ import { useCallback, useEffect, useRef, useState } from 'react'; import apiClient from '../api/client'; import { normalizeTaskStatus } from '../utils/appUiHelpers'; -const NON_BLOCKING_TASK_TYPES = new Set(['UNPACK_ARCHIVES', 'COPY_DATA']); +const NON_BLOCKING_TASK_TYPES = new Set(['UNPACK_ARCHIVES', 'UNPACK_SENTINEL1', 'SCAN_ASSET_INVENTORY', 'COPY_DATA']); const isTaskNonBlocking = (taskId, taskType, nonBlockingTaskIds = []) => ( NON_BLOCKING_TASK_TYPES.has(String(taskType || '').toUpperCase()) diff --git a/frontend/src/hooks/useRadarSearch.js b/frontend/src/hooks/useRadarSearch.js index 74ef2e5..31cb9d4 100644 --- a/frontend/src/hooks/useRadarSearch.js +++ b/frontend/src/hooks/useRadarSearch.js @@ -59,6 +59,19 @@ export default function useRadarSearch({ clearRadarSearchResults, clearRadarMapLayers, }) { + const getSatelliteCatalog = () => { + const satellites = useRadarStore.getState().radarSearchOptions?.satellite; + return Array.isArray(satellites) ? satellites.filter(Boolean) : []; + }; + + const getSatellitesForGroup = (groupKey, satellites = getSatelliteCatalog()) => { + const group = SATELLITE_GROUPS.find((item) => item.key === groupKey); + if (!group) return []; + return satellites.filter((sat) => + group.prefixes.some((prefix) => String(sat || '').startsWith(prefix)) + ); + }; + const fetchRadarImagingDates = useCallback(async () => { try { const response = await apiClient.get('/radar-data/imaging-dates'); @@ -74,13 +87,31 @@ export default function useRadarSearch({ try { setRadarSearchOptionsLoading(true); const params = {}; - if (Array.isArray(satelliteFilter) && satelliteFilter.length > 0) { - params.satellite = satelliteFilter; + const storeState = useRadarStore.getState(); + const satelliteCatalog = getSatelliteCatalog(); + const hasExplicitSatelliteFilter = Array.isArray(satelliteFilter); + let resolvedSatelliteFilter = hasExplicitSatelliteFilter + ? satelliteFilter.filter(Boolean) + : []; + + if (!hasExplicitSatelliteFilter) { + const groupKey = storeState.selectedSatelliteGroup; + if (groupKey && groupKey !== 'all') { + resolvedSatelliteFilter = getSatellitesForGroup(groupKey, satelliteCatalog); + } + } + + if (resolvedSatelliteFilter.length > 0) { + params.satellite = resolvedSatelliteFilter; } const response = await apiClient.get('/radar-data/search/options', { params }); const payload = response?.data && typeof response.data === 'object' ? response.data : {}; + const payloadSatellites = Array.isArray(payload.satellite) ? payload.satellite.filter(Boolean) : []; + const nextSatelliteCatalog = satelliteCatalog.length > payloadSatellites.length + ? satelliteCatalog + : payloadSatellites; setRadarSearchOptions({ - satellite: Array.isArray(payload.satellite) ? payload.satellite : [], + satellite: nextSatelliteCatalog, satellite_mode: Array.isArray(payload.satellite_mode) ? payload.satellite_mode : [], receiving_station: Array.isArray(payload.receiving_station) ? payload.receiving_station : [], imaging_mode: Array.isArray(payload.imaging_mode) ? payload.imaging_mode : [], @@ -119,14 +150,9 @@ export default function useRadarSearch({ orbit_direction: '', })); if (groupKey === 'all') { - fetchRadarSearchOptions(); + fetchRadarSearchOptions([]); } else { - const group = SATELLITE_GROUPS.find((g) => g.key === groupKey); - if (!group) return; - const allSatellites = useRadarStore.getState().radarSearchOptions.satellite; - const matched = allSatellites.filter((sat) => - group.prefixes.some((prefix) => sat.startsWith(prefix)) - ); + const matched = getSatellitesForGroup(groupKey); if (matched.length > 0) { fetchRadarSearchOptions(matched); } @@ -265,15 +291,9 @@ export default function useRadarSearch({ const applyRadarSearch = useCallback(async () => { const draftWithSatelliteGroup = { ...radarSearchDraft }; if (selectedSatelliteGroup && selectedSatelliteGroup !== 'all') { - const group = SATELLITE_GROUPS.find((g) => g.key === selectedSatelliteGroup); - if (group) { - const allSatellites = useRadarStore.getState().radarSearchOptions.satellite; - const matched = allSatellites.filter((sat) => - group.prefixes.some((prefix) => sat.startsWith(prefix)) - ); - if (matched.length > 0) { - draftWithSatelliteGroup.satellite = matched.join(','); - } + const matched = getSatellitesForGroup(selectedSatelliteGroup); + if (matched.length > 0) { + draftWithSatelliteGroup.satellite = matched.join(','); } } const normalizedCriteria = normalizeRadarSearchCriteria(draftWithSatelliteGroup, RADAR_SEARCH_DEFAULTS); @@ -349,7 +369,7 @@ export default function useRadarSearch({ radarSearchRequestSeqRef.current += 1; setHasRadarSearched(false); clearRadarSearchResults({ limit: radarPagination.limit }); - fetchRadarSearchOptions(); + fetchRadarSearchOptions([]); addLog('info', '已清除检索条件，请点击"搜索"或"搜索全部"获取数据。'); }, [ radarPagination.limit, radarSearchRequestSeqRef, @@ -377,7 +397,7 @@ export default function useRadarSearch({ setRadarSearchRegionError(''); setRadarSearchAoiToken(''); setSelectedSatelliteGroup('all'); - fetchRadarSearchOptions(); + fetchRadarSearchOptions([]); setHasRadarSearched(true); setIsLoading(true); diff --git a/frontend/src/store/pairingStore.js b/frontend/src/store/pairingStore.js index d8d05b4..35ffa37 100644 --- a/frontend/src/store/pairingStore.js +++ b/frontend/src/store/pairingStore.js @@ -20,8 +20,8 @@ export const usePairingStore = create((set) => ({ spatial_baseline_max_meters: 3000, limit_footprint_center_distance: false, coverage_diversity_penalty: 0.3, - require_same_imaging_mode: false, - require_same_polarization: false, + require_same_imaging_mode: true, + require_same_polarization: true, aoi_overlap_threshold: 0, start_date: '', // === 新增字段 === @@ -32,7 +32,7 @@ export const usePairingStore = create((set) => ({ strategy: 'all', num_connections: 1, reference_image_id: null, - allowed_satellites: ['LT1A', 'LT1B'], + allowed_satellites: null, cross_satellite_pairing: false, }, pairingAlert: { warnings: [], fallbackUsed: false }, diff --git a/frontend/src/utils/appUiHelpers.js b/frontend/src/utils/appUiHelpers.js index 21c5016..41e586d 100644 --- a/frontend/src/utils/appUiHelpers.js +++ b/frontend/src/utils/appUiHelpers.js @@ -33,6 +33,8 @@ export const getLeftTabLabel = (tabKey, metrics = {}) => { switch (tabKey) { case 'ingest': return '入库监控'; + case 'asset_inventory': + return '资产库存'; case 'data': return '数据列表'; case 'hazard': diff --git a/frontend/src/utils/satelliteFamily.js b/frontend/src/utils/satelliteFamily.js new file mode 100644 index 0000000..03f9ad5 --- /dev/null +++ b/frontend/src/utils/satelliteFamily.js @@ -0,0 +1,33 @@ +export function normalizeSatelliteFamily(value) { + const raw = String(value || '').trim().toUpperCase(); + if (!raw) return ''; + const compact = raw.replace(/[-_\s]+/g, ''); + if (['LT1', 'LT1A', 'LT1B', 'LUTAN1', 'LUTAN1A', 'LUTAN1B'].includes(compact)) return 'LT1'; + if (['S1', 'S1A', 'S1B', 'S1C', 'SENTINEL1', 'SENTINEL1A', 'SENTINEL1B', 'SENTINEL1C'].includes(compact)) return 'S1'; + if (['GF3', 'GAOFEN3'].includes(compact)) return 'GF3'; + return raw; +} + +export function inferSatelliteFamilyFromResultLike(item) { + const direct = normalizeSatelliteFamily( + item?.satellite_family + || item?.master_satellite + || item?.slave_satellite + || item?.satellite + ); + if (direct) return direct; + + const pairKey = String(item?.pair_key || '').trim().toLowerCase(); + if (pairKey.startsWith('s1_')) return 'S1'; + if (pairKey.startsWith('lt1_')) return 'LT1'; + if (pairKey.startsWith('gf3_')) return 'GF3'; + return ''; +} + +export function formatSatelliteFamilyLabel(value) { + const family = normalizeSatelliteFamily(value); + if (family === 'S1') return 'Sentinel-1'; + if (family === 'LT1') return 'LT-1'; + if (family === 'GF3') return 'GF3'; + return family || '-'; +} diff --git a/third_party/PyINT/pyint/coreg_gamma_all.py b/third_party/PyINT/pyint/coreg_gamma_all.py index cf7dd4d..c794987 100644 --- a/third_party/PyINT/pyint/coreg_gamma_all.py +++ b/third_party/PyINT/pyint/coreg_gamma_all.py @@ -158,8 +158,11 @@ def main(argv): if not os.path.isdir(rslcDir): os.mkdir(rslcDir) ensure_master_rslc(projectName) - if 'S1' in projectName: cmd_command = 'coreg_s1_gamma.py' - else: cmd_command = 'coreg_gamma.py' + template_satellite = str(ut.update_template(templateDir + "/" + projectName + ".template").get('satelite', '') or '') + if str(template_satellite).startswith('S1') or 'S1' in projectName: + cmd_command = 'coreg_s1_gamma.py' + else: + cmd_command = 'coreg_gamma.py' err_txt = scratchDir + '/' + projectName + '/coreg_gamma_all.err' if os.path.isfile(err_txt): os.remove(err_txt) diff --git a/third_party/PyINT/pyint/coreg_s1_gamma.py b/third_party/PyINT/pyint/coreg_s1_gamma.py index 97ebd10..6aa2e34 100644 --- a/third_party/PyINT/pyint/coreg_s1_gamma.py +++ b/third_party/PyINT/pyint/coreg_s1_gamma.py @@ -231,9 +231,9 @@ def main(argv): # if os.path.isfile(HGTSIM): os.remove(HGTSIM) else: print('The SLC has already coregister Done') - sys.exit(1) + sys.exit(0) print("Coregister TOP SLC image to the reference TOPS image is done !!") - sys.exit(1) + sys.exit(0) if __name__ == '__main__': main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/down2slc_cat_sen.py b/third_party/PyINT/pyint/down2slc_cat_sen.py index 20e331c..d5f3fef 100644 --- a/third_party/PyINT/pyint/down2slc_cat_sen.py +++ b/third_party/PyINT/pyint/down2slc_cat_sen.py @@ -14,6 +14,7 @@ import time import glob import argparse import csv +from pathlib import Path from pyint import _utils as ut @@ -29,10 +30,11 @@ def get_s1_time(raw_file): return times def get_satellite(raw_file): - if 'S1A_IW_SLC_' in raw_file: - s0 = 'A' + name = os.path.basename(str(raw_file)) + if name.startswith('S1') and len(name) >= 3: + s0 = name[2] else: - s0 = 'B' + s0 = 'A' return s0 @@ -86,8 +88,8 @@ def main(argv): #if not os.path.isdir(opod_dir): # os.mkdir(opod_dir) - call_str = " eof --save-dir " + opod_dir + " -p " + down_dir - os.system(call_str) + if opod_dir and not os.path.isdir(opod_dir): + os.makedirs(opod_dir, exist_ok=True) if not os.path.isdir(slc_dir): os.mkdir(slc_dir) @@ -99,9 +101,6 @@ def main(argv): t_date = 't_' + date - call_str = 'ls ' + down_dir + '/S1*' + date + '* > ' + t_date - os.system(call_str) - start_swath = templateDict['start_swath'] end_swath = templateDict['end_swath'] @@ -119,13 +118,23 @@ def main(argv): # k_swath = '-' k_swath = ut.get_sardata_swath(start_swath,end_swath) - raw_files = ut.read_txt2list(t_date) + raw_files = sorted(glob.glob(down_dir + '/S1*' + date + '*.zip') + glob.glob(down_dir + '/S1*' + date + '*.SAFE')) + if not raw_files: + raise FileNotFoundError('No Sentinel-1 scenes found for date ' + date) + with open(t_date, 'w', encoding='utf-8') as f: + for raw_file in raw_files: + f.write(str(raw_file) + '\n') satellite = get_satellite(str(raw_files[0])) #orbit_file = ut.download_s1_orbit(date,opod_dir,satellite=satellite) - raw_file_list = glob.glob(down_dir + '/S1*' + date + '*.zip') + raw_file_list = glob.glob(down_dir + '/S1*' + date + '*.zip') + glob.glob(down_dir + '/S1*' + date + '*.SAFE') file_num=len(raw_files) - call_str = 'S1_BURST_tab_from_zipfile.py 3 --zip_ref_list ' + t_date + ' --zip_list ' + t_date - os.system(call_str) + if raw_files and os.path.isdir(str(raw_files[0])): + call_str = 'S1_BURST_tab_from_zipfile.py 3 --dir_ref_list ' + t_date + ' --dir_list ' + t_date + else: + call_str = 'S1_BURST_tab_from_zipfile.py 3 --zip_ref_list ' + t_date + ' --zip_list ' + t_date + rc = os.system(call_str) + if rc != 0: + raise RuntimeError('S1_BURST_tab_from_zipfile.py failed with rc=' + str(rc)) for kk in range(file_num): zipfile_ref=str(raw_files[kk]) @@ -135,22 +144,39 @@ def main(argv): #os.system(call_str) #call_str = 'S1_import_SLC_from_zipfiles ' + t_date + ' ' + burst_number_table_ref + ' vv 0 ' + k_swath + ' ' + opod_dir + ' 1 1 ' call_str = 'read_S1_TOPS_SLC.py ' + zipfile_ref + ' --burst_sel ' + burst_number_table_ref + ' --pol vv --root_name ' + date + ' --sw_start ' + start_swath + ' --swn ' + end_swath + ' --OPOD_dir ' + opod_dir - os.system(call_str) - os.chdir(work_dir) - call_str = "rename 's/vv.iw1.slc/iw1_" + str(kk)+".slc/g' *" - os.system(call_str) - call_str = "rename 's/vv.iw2.slc/iw2_" + str(kk)+".slc/g' *" - os.system(call_str) - call_str = "rename 's/vv.iw3.slc/iw3_" + str(kk)+".slc/g' *" - os.system(call_str) + rc = os.system(call_str) + if rc != 0: + raise RuntimeError('read_S1_TOPS_SLC.py failed with rc=' + str(rc)) + os.chdir(work_dir) + rename_rules = [ + ('vv.iw1.slc', 'iw1_' + str(kk) + '.slc'), + ('vv.iw2.slc', 'iw2_' + str(kk) + '.slc'), + ('vv.iw3.slc', 'iw3_' + str(kk) + '.slc'), + ('vv.SLC_tab', 'SLC_tab'), + ('vv.slc', 'slc'), + ('tops_par', 'TOPS_par'), + ] + for src_token, dst_token in rename_rules: + for candidate in list(Path(work_dir).iterdir()): + if src_token not in candidate.name: + continue + target = candidate.with_name(candidate.name.replace(src_token, dst_token)) + if target != candidate and not target.exists(): + candidate.rename(target) SLC_Tab = work_dir + '/' + date + '_SLC_Tab' +str(kk) SLC_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc')) SLC_par_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc.par')) - TOP_par_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc.tops_par')) + TOP_par_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc.TOPS_par')) + if not TOP_par_list: + TOP_par_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc.tops_par')) + if (not SLC_list or not SLC_par_list or not TOP_par_list + or len(SLC_list) != len(SLC_par_list) + or len(SLC_list) != len(TOP_par_list)): + raise RuntimeError('Sentinel-1 concatenated SLC generation did not produce expected IW products for ' + date) cat_str = 'touch slc_list slc_par_list top_par_list' - os.system(call_str) + os.system(cat_str) list_num=len(SLC_list) for tt in range(list_num): call_str = 'echo ' + SLC_list[tt] + ' >> slc_list' @@ -160,7 +186,9 @@ def main(argv): call_str = 'echo ' + TOP_par_list[tt] + ' >> top_par_list' os.system(call_str) call_str = ' paste slc_list slc_par_list top_par_list > ' + SLC_Tab - os.system(call_str) + rc = os.system(call_str) + if rc != 0: + raise RuntimeError('paste SLC tab failed with rc=' + str(rc)) call_str = 'rm slc_list slc_par_list top_par_list' os.system(call_str) SLC_list = sorted(glob.glob(work_dir + '/*iw*.slc')) @@ -188,23 +216,42 @@ def main(argv): SLC_list = sorted(glob.glob(work_dir + '/*IW*.slc')) SLC_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.par')) TOP_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.TOPS_par')) + if not TOP_par_list: + TOP_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.tops_par')) #call_str = 'rm *iw* *vv.SLC_tab' #os.system(call_str) if os.path.isfile(SLC_Tab): os.remove(SLC_Tab) + if (not SLC_list or not SLC_par_list or not TOP_par_list + or len(SLC_list) != len(SLC_par_list) + or len(SLC_list) != len(TOP_par_list)): + raise RuntimeError('Sentinel-1 concatenated final SLC products are incomplete for ' + date) + for kk in range(len(SLC_list)): call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' >> ' + SLC_Tab - os.system(call_str) + rc = os.system(call_str) + if rc != 0: + raise RuntimeError('failed to append concatenated SLC tab with rc=' + str(rc)) BURST = SLC_par_list[kk].replace('slc.par','burst.par') call_str = 'SLC_burst_corners ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' > ' +BURST - os.system(call_str) + rc = os.system(call_str) + if rc != 0: + raise RuntimeError('SLC_burst_corners failed with rc=' + str(rc)) call_str = "echo 'SLC has already down' >down2slc.dat" os.system(call_str) + + TSLC = work_dir + '/' + date + '.slc' + TSLCPar = work_dir + '/' + date + '.slc.par' + if not os.path.isfile(TSLC) or not os.path.isfile(TSLCPar): + call_str = 'SLC_mosaic_ScanSAR ' + SLC_Tab + ' ' + TSLC + ' ' + TSLCPar + ' 10 2' + rc = os.system(call_str) + if rc != 0: + raise RuntimeError('SLC_mosaic_ScanSAR failed with rc=' + str(rc)) print("Down to SLC for %s is done! " % date) - sys.exit(1) + sys.exit(0) if __name__ == '__main__': main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/down2slc_sen.py b/third_party/PyINT/pyint/down2slc_sen.py index 953fe87..959e921 100644 --- a/third_party/PyINT/pyint/down2slc_sen.py +++ b/third_party/PyINT/pyint/down2slc_sen.py @@ -13,6 +13,7 @@ import getopt import time import glob import argparse +from pathlib import Path from pyint import _utils as ut @@ -23,10 +24,11 @@ def get_s1_date(raw_file): return date def get_satellite(raw_file): - if 'S1A_IW_SLC_' in raw_file: - s0 = 'A' + name = os.path.basename(str(raw_file)) + if name.startswith('S1') and len(name) >= 3: + s0 = name[2] else: - s0 = 'B' + s0 = 'A' return s0 @@ -88,16 +90,13 @@ def main(argv): # os.mkdir(opod_dir) - call_str = " eof --save-dir " + opod_dir + " -p " + down_dir - os.system(call_str) + if opod_dir and not os.path.isdir(opod_dir): + os.makedirs(opod_dir, exist_ok=True) os.chdir(work_dir) t_date = 't_' + date - call_str = 'ls ' + down_dir + '/S1*' + date + '*.zip > ' + t_date - os.system(call_str) - start_swath = templateDict['start_swath'] end_swath = templateDict['end_swath'] @@ -115,53 +114,87 @@ def main(argv): # k_swath = '-' k_swath = ut.get_sardata_swath(start_swath,end_swath) - raw_files = ut.read_txt2list(t_date) + raw_files = sorted(glob.glob(down_dir + '/S1*' + date + '*.zip') + glob.glob(down_dir + '/S1*' + date + '*.SAFE')) + if not raw_files: + raise FileNotFoundError('No Sentinel-1 scenes found for date ' + date) + with open(t_date, 'w', encoding='utf-8') as f: + for raw_file in raw_files: + f.write(str(raw_file) + '\n') satellite = get_satellite(str(raw_files[0])) #orbit_file = ut.download_s1_orbit(date,opod_dir,satellite=satellite) zipfile_ref=str(raw_files[0]) outfile_name=zipfile_ref.split('/')[-1].split('.')[0] burst_number_table_ref=outfile_name + '.BURST_tab' - call_str = 'S1_BURST_tab_from_zipfile.py 3 --zip_ref_list ' + t_date + ' --zip_list ' + t_date - os.system(call_str) + if raw_files and os.path.isdir(str(raw_files[0])): + call_str = 'S1_BURST_tab_from_zipfile.py 3 --dir_ref_list ' + t_date + ' --dir_list ' + t_date + else: + call_str = 'S1_BURST_tab_from_zipfile.py 3 --zip_ref_list ' + t_date + ' --zip_list ' + t_date + rc = os.system(call_str) + if rc != 0: + raise RuntimeError('S1_BURST_tab_from_zipfile.py failed with rc=' + str(rc)) # call_str = 'S1_import_SLC_from_zipfiles ' + t_date + ' ' + burst_number_table_ref + ' vv 0 ' + k_swath call_str = 'read_S1_TOPS_SLC.py ' + zipfile_ref + ' --burst_sel ' + burst_number_table_ref + ' --pol vv --root_name ' + date + ' --sw_start ' + start_swath + ' --swn ' + end_swath + ' --OPOD_dir ' + opod_dir - os.system(call_str) - + rc = os.system(call_str) + if rc != 0: + raise RuntimeError('read_S1_TOPS_SLC.py failed with rc=' + str(rc)) + os.chdir(work_dir) - - call_str = "rename 's/vv.iw1.slc/IW1.slc/g' *" - #call_str = "rename vv.slc.iw1 IW1.slc * " - os.system(call_str) - call_str = "rename 's/vv.iw2.slc/IW2.slc/g' *" - #call_str = "rename vv.slc.iw2 IW2.slc * " - os.system(call_str) - call_str = "rename 's/vv.iw3.slc/IW3.slc/g' *" - #call_str = "rename vv.slc.iw3 IW3.slc * " - os.system(call_str) - call_str = "rename 's/tops_par/TOPS_par/g' *.tops_par " - os.system(call_str) + + rename_rules = [ + ('vv.iw1.slc', 'IW1.slc'), + ('vv.iw2.slc', 'IW2.slc'), + ('vv.iw3.slc', 'IW3.slc'), + ('vv.SLC_tab', 'SLC_tab'), + ('vv.slc', 'slc'), + ('tops_par', 'TOPS_par'), + ] + for src_token, dst_token in rename_rules: + for candidate in list(Path(work_dir).iterdir()): + if src_token not in candidate.name: + continue + target = candidate.with_name(candidate.name.replace(src_token, dst_token)) + if target != candidate and not target.exists(): + candidate.rename(target) SLC_Tab = work_dir + '/' + date+'_SLC_Tab' SLC_list = sorted(glob.glob(work_dir + '/*IW*.slc')) SLC_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.par')) TOP_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.TOPS_par')) + if not TOP_par_list: + TOP_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.tops_par')) + if (not SLC_list or not SLC_par_list or not TOP_par_list + or len(SLC_list) != len(SLC_par_list) + or len(SLC_list) != len(TOP_par_list)): + raise RuntimeError('Sentinel-1 SLC generation did not produce expected IW SLC products for ' + date) if os.path.isfile(SLC_Tab): os.remove(SLC_Tab) for kk in range(len(SLC_list)): call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' >> ' + SLC_Tab - os.system(call_str) + rc = os.system(call_str) + if rc != 0: + raise RuntimeError('failed to append SLC tab with rc=' + str(rc)) BURST = SLC_par_list[kk].replace('slc.par','burst.par') call_str = 'SLC_burst_corners ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' > ' +BURST - os.system(call_str) + rc = os.system(call_str) + if rc != 0: + raise RuntimeError('SLC_burst_corners failed with rc=' + str(rc)) call_str = "echo 'SLC has already down' >down2slc.dat" os.system(call_str) + + TSLC = work_dir + '/' + date + '.slc' + TSLCPar = work_dir + '/' + date + '.slc.par' + if not os.path.isfile(TSLC) or not os.path.isfile(TSLCPar): + call_str = 'SLC_mosaic_ScanSAR ' + SLC_Tab + ' ' + TSLC + ' ' + TSLCPar + ' 10 2' + rc = os.system(call_str) + if rc != 0: + raise RuntimeError('SLC_mosaic_ScanSAR failed with rc=' + str(rc)) print("Down to SLC for %s is done! " % date) - sys.exit(1) + sys.exit(0) if __name__ == '__main__': main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/down2slc_sen_all.py b/third_party/PyINT/pyint/down2slc_sen_all.py index 15c7800..a80afab 100644 --- a/third_party/PyINT/pyint/down2slc_sen_all.py +++ b/third_party/PyINT/pyint/down2slc_sen_all.py @@ -32,9 +32,27 @@ def work(data0): stderr = p.stderr if type(stderr) == bytes: - aa=stderr.decode("utf-8") + aa=stderr.decode("utf-8", errors="replace") else: aa = stderr + if type(stdout) == bytes: + bb=stdout.decode("utf-8", errors="replace") + else: + bb = stdout + if p.returncode != 0: + detail_parts = [] + if bb: + detail_parts.append(bb) + if aa: + detail_parts.append(aa) + detail = '\n'.join(part.strip() for part in detail_parts if part and part.strip()) + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + with open(err_txt, 'a') as f: + f.write(str0) + if detail: + f.write(detail) + f.write('\n') + raise RuntimeError(str0.strip() + ' failed with rc=' + str(p.returncode) + ('\n' + detail if detail else '')) if aa: str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' @@ -83,7 +101,7 @@ def main(argv): projectDir = scratchDir + '/' + projectName downDir = scratchDir + '/' + projectName + "/DOWNLOAD" slcDir = scratchDir + '/' + projectName + "/SLC" - raw_file_list = glob.glob(downDir + '/S1*.zip') + raw_file_list = glob.glob(downDir + '/S1*.zip') + glob.glob(downDir + '/S1*.SAFE') templateDir = os.getenv('TEMPLATEDIR') templateFile = templateDir + "/" + projectName + ".template" templateDict=ut.update_template(templateFile) @@ -139,12 +157,15 @@ def main(argv): k00 = 0 if k00==0: data_para.append(data0) - ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + results = ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + failures = [str(item) for item in results if isinstance(item, Exception)] + if failures: + raise RuntimeError('\n\n'.join(failures)) os.chdir(downDir) print("Down to SLC for project %s is done! " % projectName) ut.print_process_time(start_time, time.time()) - sys.exit(1) + sys.exit(0) if __name__ == '__main__': main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/extract_s1_bursts.py b/third_party/PyINT/pyint/extract_s1_bursts.py index 2f03583..33482b5 100644 --- a/third_party/PyINT/pyint/extract_s1_bursts.py +++ b/third_party/PyINT/pyint/extract_s1_bursts.py @@ -291,7 +291,7 @@ def main(argv): iw3 = Sdate + '.IW3.slc*' call_str = 'rm '+ iw1 + ' ' + iw2 + ' '+ iw3 os.system(call_str) - sys.exit(1) + sys.exit(0) if __name__ == '__main__': main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/extract_s1_bursts_all.py b/third_party/PyINT/pyint/extract_s1_bursts_all.py index 2a48bf8..2f271b4 100644 --- a/third_party/PyINT/pyint/extract_s1_bursts_all.py +++ b/third_party/PyINT/pyint/extract_s1_bursts_all.py @@ -26,9 +26,27 @@ def work(data0): stderr = p.stderr if type(stderr) == bytes: - aa=stderr.decode("utf-8") + aa=stderr.decode("utf-8", errors="replace") else: aa = stderr + if type(stdout) == bytes: + bb=stdout.decode("utf-8", errors="replace") + else: + bb = stdout + if p.returncode != 0: + detail_parts = [] + if bb: + detail_parts.append(bb) + if aa: + detail_parts.append(aa) + detail = '\n'.join(part.strip() for part in detail_parts if part and part.strip()) + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + with open(err_file, 'a') as f: + f.write(str0) + if detail: + f.write(detail) + f.write('\n') + raise RuntimeError(str0.strip() + ' failed with rc=' + str(p.returncode) + ('\n' + detail if detail else '')) if aa: str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' @@ -75,22 +93,31 @@ def main(argv): projectDir = scratchDir + '/' + projectName slcDir = scratchDir + '/' + projectName + '/SLC' slc_list = [os.path.basename(fname) for fname in sorted(glob.glob(slcDir + '/*'))] + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + master_date = templateDict['masterDate'] err_txt = scratchDir + '/' + projectName + '/extract_s1_bursts_all.err' if os.path.isfile(err_txt): os.remove(err_txt) data_para = [] for i in range(len(slc_list)): + if slc_list[i] == master_date: + continue cmd0 = ['extract_s1_bursts.py',projectName,slc_list[i]] data0 = [cmd0,err_txt] data_para.append(data0) - ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + results = ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + failures = [str(item) for item in results if isinstance(item, Exception)] + if failures: + raise RuntimeError('\n\n'.join(failures)) print("Extract TOPS bursts for project %s is done! " % projectName) ut.print_process_time(start_time, time.time()) - sys.exit(1) + sys.exit(0) if __name__ == '__main__': main(sys.argv[:]) - \ No newline at end of file + diff --git a/third_party/PyINT/pyint/phase2los.py b/third_party/PyINT/pyint/phase2los.py index 3cff4db..952bd65 100644 --- a/third_party/PyINT/pyint/phase2los.py +++ b/third_party/PyINT/pyint/phase2los.py @@ -83,7 +83,7 @@ def main(argv): wavelength = 0.0312283810417 elif satelite == 'TSX': wavelength = 0.03106657823461874 - elif satelite == 'S1A': + elif str(satelite).startswith('S1'): wavelength = 0.0554657647 elif satelite == 'ALOS2': wavelength = 0.2424525 @@ -106,4 +106,3 @@ def main(argv): if __name__ == '__main__': main(sys.argv[:]) - diff --git a/third_party/PyINT/pyint/phase2los_all.py b/third_party/PyINT/pyint/phase2los_all.py index 38f9ad2..6da1023 100644 --- a/third_party/PyINT/pyint/phase2los_all.py +++ b/third_party/PyINT/pyint/phase2los_all.py @@ -105,7 +105,7 @@ def main(argv): print("Generate differential interferograms for project %s is done! " % projectName) ut.print_process_time(start_time, time.time()) - sys.exit(1) + sys.exit(0) if __name__ == '__main__': main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/pyintApp.py b/third_party/PyINT/pyint/pyintApp.py index 1ad8c15..d259209 100644 --- a/third_party/PyINT/pyint/pyintApp.py +++ b/third_party/PyINT/pyint/pyintApp.py @@ -210,7 +210,7 @@ def main(argv): if templateDict['raw2slc_all'] == '1': # only for S1 data now print('Start to convert downloaded-raw data into SLC ...') print('Number of processor: %s' % str(templateDict['raw2slc_all_parallel'])) - if satelite=='S1A': + if str(satelite).startswith('S1'): call_str = 'down2slc_sen_all.py ' + projectName + ' --parallel ' + templateDict['raw2slc_all_parallel'] _run_or_raise(call_str, 'raw2slc_s1') elif satelite=='ALOS': diff --git a/third_party/PyINT/pyint/slc_sen_cat.py b/third_party/PyINT/pyint/slc_sen_cat.py index e1ff019..cd4009b 100644 --- a/third_party/PyINT/pyint/slc_sen_cat.py +++ b/third_party/PyINT/pyint/slc_sen_cat.py @@ -159,7 +159,7 @@ def main(argv): call_str = 'SLC_burst_corners ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' > ' +BURST os.system(call_str) print("Down to SLC for %s is done! " % date) - sys.exit(1) + sys.exit(0) if __name__ == '__main__': main(sys.argv[:])