From 71c524967cd6d53f630ec98e45463cda5b5e7a65 Mon Sep 17 00:00:00 2001 From: Harmon Date: Sun, 21 Jun 2026 12:30:21 +0800 Subject: [PATCH] Refactor local InSAR asset and production workflows --- .env.example | 44 +- backend/app/ai_service.py | 101 +- backend/app/config.py | 117 +- backend/app/copier.py | 305 ++- backend/app/db_maintenance.py | 2 + backend/app/dinsar_engines/landsar_engine.py | 511 +++- backend/app/models/__init__.py | 5 +- backend/app/models/orm.py | 94 +- backend/app/models/schemas.py | 30 +- backend/app/routers/ai.py | 18 +- backend/app/routers/assets.py | 57 +- backend/app/routers/dinsar_production.py | 89 + backend/app/routers/monitor.py | 529 +++- backend/app/routers/orbit.py | 28 +- backend/app/routers/pairing.py | 10 +- backend/app/routers/radar.py | 65 + backend/app/routers/tools.py | 96 +- backend/app/scheduler.py | 7 +- .../app/services/asset_inventory_service.py | 2191 ++++++++++++++++- backend/app/services/data_service.py | 244 +- .../services/gf3_native_inventory_service.py | 229 +- .../app/services/gf3_standardize_service.py | 746 +++++- backend/app/services/job_handlers.py | 291 +-- backend/app/services/landsar_sbas_service.py | 77 +- backend/app/services/orbit_converter.py | 54 +- backend/app/services/pairing_cache_service.py | 96 + backend/app/services/pairing_state_service.py | 2 +- backend/app/services/root_registry_service.py | 9 - backend/app/services/spatial_service.py | 649 ++++- backend/app/services/task_service.py | 3 + backend/app/services/unpack_service.py | 2 +- backend/app/utils.py | 80 +- .../011_source_metadata_documents.sql | 138 ++ .../012_source_archive_integrity.sql | 20 + docs/DEPLOYMENT.md | 19 +- ...ASK_POOL_THREE_ENGINE_REFACTOR_20260614.md | 2 +- ..._ALGORITHM_ENGINEERING_HANDOFF_20260602.md | 2 +- docs/FRONTEND_NAVIGATION_ARCHITECTURE.md | 239 +- ...SCAPE_NATIVE_TO_GEOTIFF_DESIGN_20260530.md | 59 +- docs/INDEX.md | 16 +- ...NDSAR_DEM_PREPARATION_CONTRACT_20260618.md | 63 + ...MA_DINSAR_DIAGNOSIS_DEPLOYMENT_20260620.md | 97 + ...CISE_ORBIT_PRODUCTION_CONTRACT_20260617.md | 109 + docs/SBAS_INSAR_CURRENT_WORKFLOW.md | 2 +- ...NEL1_SOURCE_ORBIT_ASSET_DESIGN_20260512.md | 2 +- ...SOURCE_ARCHIVE_INTEGRITY_AUDIT_20260620.md | 67 + ...NSOR_LOCAL_PRODUCTION_CONTRACT_20260616.md | 121 + ...ARCHIVE_AND_MATERIALIZE_DESIGN_20260615.md | 255 +- frontend/src/AiAnalysisPanel.jsx | 47 +- frontend/src/App.css | 236 +- frontend/src/App.jsx | 24 +- frontend/src/AssetInventoryPanel.jsx | 73 +- frontend/src/DataCopierPanel.jsx | 260 +- frontend/src/DataMonitorPanel.jsx | 1272 ++++------ frontend/src/DinsarProductionPanel.jsx | 211 +- frontend/src/HealthCheckPanel.jsx | 157 +- frontend/src/ProductionWorkspace.jsx | 242 +- frontend/src/SbasInsarProductionPanel.jsx | 92 +- frontend/src/api/assets.js | 3 + frontend/src/api/dinsarProduction.js | 3 + frontend/src/components/MiniCoverageMap.jsx | 219 ++ frontend/src/components/PairingModal.jsx | 711 +++--- frontend/src/components/UnifiedDatePicker.jsx | 40 + frontend/src/components/app/AppOverlays.jsx | 19 +- frontend/src/components/app/AppSidePanel.jsx | 100 +- .../src/components/panels/PairListRow.jsx | 89 +- .../src/components/panels/RadarDataRow.jsx | 2 +- frontend/src/config/appConstants.js | 140 +- frontend/src/config/taskUiPolicies.js | 19 +- frontend/src/hooks/useBatchOperations.js | 9 +- frontend/src/hooks/useDinsarOperations.js | 34 +- frontend/src/hooks/usePairingLogic.js | 206 +- frontend/src/hooks/useRadarSearch.js | 13 +- frontend/src/panels/BatchPanel.jsx | 90 +- frontend/src/panels/DinsarAnalysisPanel.jsx | 78 + frontend/src/panels/PairPlanningPanel.jsx | 43 +- frontend/src/panels/PairingPanel.jsx | 6 +- frontend/src/panels/PairsListPanel.jsx | 104 +- frontend/src/panels/RadarDataPanel.jsx | 2 +- frontend/src/store/pairingStore.js | 8 +- frontend/src/utils/appUiHelpers.js | 22 +- scripts/build_archive_preview_caches.py | 62 + scripts/migrate_source_archives_to_zip.py | 594 +++++ scripts/prepare_landsar_dem_int16.py | 136 +- .../repack_unpacked_sources_to_archives.py | 189 ++ scripts/repair_lt1_archive_metadata.py | 292 +++ scripts/reset_lt1_s1_source_catalog.py | 332 +++ scripts/unpack_archives_parallel.py | 11 +- 88 files changed, 11165 insertions(+), 3017 deletions(-) create mode 100644 backend/migrations/011_source_metadata_documents.sql create mode 100644 backend/migrations/012_source_archive_integrity.sql create mode 100644 docs/LANDSAR_DEM_PREPARATION_CONTRACT_20260618.md create mode 100644 docs/OLLAMA_DINSAR_DIAGNOSIS_DEPLOYMENT_20260620.md create mode 100644 docs/PRECISE_ORBIT_PRODUCTION_CONTRACT_20260617.md create mode 100644 docs/SOURCE_ARCHIVE_INTEGRITY_AUDIT_20260620.md create mode 100644 docs/THREE_SENSOR_LOCAL_PRODUCTION_CONTRACT_20260616.md create mode 100644 frontend/src/components/MiniCoverageMap.jsx create mode 100644 frontend/src/panels/DinsarAnalysisPanel.jsx create mode 100644 scripts/build_archive_preview_caches.py create mode 100644 scripts/migrate_source_archives_to_zip.py create mode 100644 scripts/repack_unpacked_sources_to_archives.py create mode 100644 scripts/repair_lt1_archive_metadata.py create mode 100644 scripts/reset_lt1_s1_source_catalog.py diff --git a/.env.example b/.env.example index fd8b0fd..a675db8 100644 --- a/.env.example +++ b/.env.example @@ -58,33 +58,34 @@ ALLOWED_EXPORT_DIRS= # ----------------------------------------------------------------------------- # 源数据目录 # ----------------------------------------------------------------------------- -UNPACK_SOURCE_DIRS=D:\Archives +UNPACK_SOURCE_DIRS= TASK_POOL_ROOT=D:\Task_Pool DINSAR_TASK_POOL_ROOT=D:\Task_Pool\DInSAR SBAS_TASK_POOL_ROOT=D:\Task_Pool\SBAS -GF3_TASK_POOL_ROOT=D:\Task_Pool\GF3 -SOURCE_PRODUCT_DIRS=D:\LuTan1_Image_Pool;D:\Sentinel1_Image_Pool_ZIP;\\DESKTOP-N16HJ84\InSAR_Storage_2\LuTan-1\Archive;\\DESKTOP-N16HJ84\InSAR_Storage_2\Sentinel-1\Archive -SENTINEL1_STORAGE_DIRS=D:\Sentinel1_Image_Pool -INSAR_STORAGE_DIRS=D:\LuTan1_Image_Pool -MONITOR_RADAR_DIRS=D:\LuTan1_Image_Pool -MONITOR_DINSAR_DIRS=D:\DInSARResult -ORBIT_SOURCE_DIRS=D:\LT1_data_lsarorbit;D:\Sentinel1_EOF_Pool;\\DESKTOP-N16HJ84\InSAR_Storage_2\Orbit\LuTan-1;\\DESKTOP-N16HJ84\InSAR_Storage_2\Orbit\Sentinel-1 +DATA_DISTRIBUTION_ROOT=D:\Task_Pool\Data_Distribution +GF3_TASK_POOL_ROOT=D:\GaoFen3_Pool\task_pool +SOURCE_PRODUCT_DIRS=D:\LuTan1_Image_Pool_Zip;D:\Sentinel1_Image_Pool_ZIP +SENTINEL1_STORAGE_DIRS= +INSAR_STORAGE_DIRS= +MONITOR_RADAR_DIRS= +MONITOR_DINSAR_DIRS=D:\production_results\dinsar +ORBIT_SOURCE_DIRS=D:\LT1_data_lsarorbit;D:\Sentinel1_EOF_Pool MONITOR_ORBIT_DIR=D:\LT1_data_lsarorbit -GF3_ARCHIVE_SOURCE_DIRS=\\DESKTOP-N16HJ84\InSAR_Storage_1\GaoFen-3 +GF3_ARCHIVE_SOURCE_DIRS=D:\GaoFen3_Pool\archives GF3_ARCHIVE_EXTS=.zip,.tar,.tar.gz,.tgz -GF3_UNPACK_DELETE_ARCHIVE=true +GF3_UNPACK_DELETE_ARCHIVE=false GF3_LEGACY_GDAL_ENABLED=false GF3_SOURCE_DIRS= -GF3_SARSCAPE_NATIVE_DIRS=D:\production_results\gf3\sarscape_native -GF3_STORAGE_DIRS=D:\production_results\gf3\standard_l2 -GF3_SARSCAPE_RUNTIME_DIR=D:\production_runtime\gf3\sarscape_runtime +GF3_SARSCAPE_NATIVE_DIRS=D:\GaoFen3_Pool\native_geo +GF3_STORAGE_DIRS=D:\GaoFen3_Pool\catalog +GF3_SARSCAPE_RUNTIME_DIR=D:\GaoFen3_Pool\task_pool\sarscape_runtime GF3_SARSCAPE_WRAPPER_EXE=D:\Code\Insar_management_system_v2\third_party\GF3_L1A_To_L2_pipeline\dist\windows\gf3wrapper.exe GF3_SARSCAPE_IDLRT_PATH=C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlrt.exe GF3_SARSCAPE_DEM_PATH=D:\DEM\COPDEM_GLO30_China_4326_DEM GF3_SARSCAPE_POLARIZATIONS=HH,HV GF3_SARSCAPE_KEEP_EXTRACTED=true -GF3_SARSCAPE_AUTO_STANDARDIZE=true +GF3_SARSCAPE_AUTO_STANDARDIZE=false GF3_SARSCAPE_CLEAN_AFTER_SUCCESS=true GF3_SARSCAPE_PRODUCE_TIMEOUT_SECONDS=0 @@ -110,7 +111,7 @@ RESULT_CATALOG_AUTO_REBUILD_ON_STARTUP=true # 轨道池 # ----------------------------------------------------------------------------- ORBIT_POOL_ENVI=D:\orbit_pools\envi -ORBIT_POOL_ISCE2=D:\orbit_pools\isce2 +ORBIT_POOL_ISCE2= ORBIT_POOL_LANDSAR= ORBIT_QUARANTINE_DIR= @@ -174,7 +175,7 @@ PYINT_RUNTIME_ID=gamma_pyint_runtime_v1 # ----------------------------------------------------------------------------- # ISCE2 D-InSAR # ----------------------------------------------------------------------------- -ISCE2_ENABLED=true +ISCE2_ENABLED=false ISCE2_WSL_DISTRO=Ubuntu-24.04 ISCE2_PYTHON=/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python ISCE2_PROFILE=lt1_stripmap @@ -211,12 +212,13 @@ LANDSAR_AUTH_SERVER_EXE=D:\Code\Insar_management_system_v2\third_party\LandSAR\t LANDSAR_AUTH_SERVER_AUTO_START=true LANDSAR_AUTH_SERVER_HOST=127.0.0.1 LANDSAR_AUTH_SERVER_PORT=6666 -LANDSAR_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84 +# Prepared global Int16 GeoTIFF source; LandSAR jobs crop task-level DEM files from it before execution. +LANDSAR_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape_global_int16.tif LANDSAR_DINSAR_TIMEOUT_SECONDS=43200 LANDSAR_SBAS_ENABLED=true LANDSAR_SBAS_WORK_ROOT=D:\LandSAR_Work\sbas LANDSAR_SBAS_PRODUCT_ROOT=D:\production_results\timeseries\sbas_landsar -LANDSAR_SBAS_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84 +LANDSAR_SBAS_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape_global_int16.tif LANDSAR_SBAS_SOURCE_ROOTS=D:\LandSAR_Work LANDSAR_SBAS_TIMEOUT_SECONDS=172800 LANDSAR_SBAS_MIN_SCENES=3 @@ -283,7 +285,7 @@ GAMMA_SBAS_WORK_ROOT=D:\Task_Pool\SBAS GAMMA_SBAS_PRODUCT_ROOT=D:\production_results\timeseries\sbas GAMMA_SBAS_TRIAL_ROOT=D:\production_runtime\gamma_ipta_trials GAMMA_SBAS_SCRIPT_TEMPLATE_ROOT=D:\Code\Insar_management_system_v2\backend\templates\gamma_sbas -GAMMA_SBAS_SOURCE_ROOTS=D:\LuTan1_Image_Pool +GAMMA_SBAS_SOURCE_ROOTS=D:\Task_Pool\source_materialized\lutan1 GAMMA_SBAS_ORBIT_ROOTS=D:\orbit_pools\envi GAMMA_SBAS_DEM_PATH=D:\DEM\HeiLongJiang10M_DEM.tif GAMMA_SBAS_DEFAULT_RLKS=8 @@ -305,7 +307,7 @@ TIMESERIES_ENV_NAME=insar_wsl_v1 TIMESERIES_PYTHON=/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python TIMESERIES_WORK_ROOT=D:\production_runtime\timeseries_work TIMESERIES_DEM_PATH=D:\SRTM30m\SRTMDEM_RSP_SARscape.wgs84 -TIMESERIES_ORBIT_POOL_ISCE2=D:\orbit_pools\isce2 +TIMESERIES_ORBIT_POOL_ISCE2= TIMESERIES_EXPERIMENT_ROOT= TIMESERIES_STACK_PREP_SCRIPT= TIMESERIES_MATERIALIZE_SCRIPT= @@ -359,7 +361,7 @@ UNPACK_SCAN_WORKERS=4 UNPACK_EXTRACT_WORKERS=4 UNPACK_MAX_FILES_PER_RUN=100 UNPACK_MAX_RUNTIME_MINUTES=360 -UNPACK_DELETE_ARCHIVE=true +UNPACK_DELETE_ARCHIVE=false UNPACK_TMP_SUFFIX=.unpack_tmp diff --git a/backend/app/ai_service.py b/backend/app/ai_service.py index 3eaebce..ba02059 100644 --- a/backend/app/ai_service.py +++ b/backend/app/ai_service.py @@ -13,6 +13,7 @@ from sklearn.preprocessing import StandardScaler from scipy.stats import entropy from concurrent.futures import ProcessPoolExecutor import functools +from typing import List, Optional from .config import settings @@ -224,25 +225,74 @@ def get_model_info() -> dict: OLLAMA_BASE_URL = settings.OLLAMA_BASE_URL OLLAMA_API_URL = settings.OLLAMA_API_URL DEFAULT_VLM_MODEL = settings.DEFAULT_VLM_MODEL +VLM_MODEL_MARKERS = ( + "qwen3-vl", + "qwen2-vl", + "minicpm-v", + "llama3.2-vision", + "llava", + "vision", + "-vl", + "_vl", +) -async def _get_available_vlm_model() -> str: +def _normalize_model_name(model_name: Optional[str]) -> Optional[str]: + normalized = str(model_name or "").strip() + return normalized or None + +def is_likely_vlm_model(model_name: Optional[str]) -> bool: + lower = str(model_name or "").strip().lower() + return bool(lower and any(marker in lower for marker in VLM_MODEL_MARKERS)) + +async def get_ollama_models(timeout: float = 2.0) -> List[str]: + """Return model names reported by the local Ollama service.""" + async with httpx.AsyncClient(timeout=timeout) as client: + resp = await client.get(f"{OLLAMA_BASE_URL.rstrip('/')}/api/tags") + resp.raise_for_status() + return [ + str(model.get("name") or "").strip() + for model in resp.json().get("models", []) + if str(model.get("name") or "").strip() + ] + +async def get_ollama_vlm_models(timeout: float = 2.0) -> List[str]: + """Return installed Ollama models whose names indicate image-input support.""" + return [ + model_name + for model_name in await get_ollama_models(timeout=timeout) + if is_likely_vlm_model(model_name) + ] + +async def _get_available_vlm_model(preferred_model: Optional[str] = None) -> str: """自动检测本地可用的 VLM 模型""" + preferred = _normalize_model_name(preferred_model) try: - async with httpx.AsyncClient(timeout=2.0) as client: - resp = await client.get(f"{OLLAMA_BASE_URL}/api/tags") - if resp.status_code == 200: - models = [m['name'] for m in resp.json().get('models', [])] - # 优先级：qwen3-vl > qwen2-vl > minicpm-v > 任何包含 vl 的模型 - for target in ["qwen3-vl:8b", "qwen2-vl", "minicpm-v"]: - for m in models: - if target in m: return m - for m in models: - if "vl" in m.lower(): return m - except: + models = await get_ollama_models(timeout=2.0) + if preferred and is_likely_vlm_model(preferred): + if preferred in models: + return preferred + for model in models: + if is_likely_vlm_model(model) and (model.startswith(preferred) or preferred in model): + return model + # 优先级：qwen3-vl > qwen2-vl > minicpm-v > 任何包含 vl/vision/llava 的模型 + for target in ["qwen3-vl", "qwen2-vl", "minicpm-v", "llama3.2-vision", "llava"]: + for model in models: + if target in model.lower(): + return model + for model in models: + if is_likely_vlm_model(model): + return model + except Exception: pass return DEFAULT_VLM_MODEL -async def analyze_map_with_vlm(images_base64: list, prompt: str, progress_callback=None) -> str: +async def analyze_map_with_vlm( + images_base64: list, + prompt: str, + progress_callback=None, + model_name: Optional[str] = None, + raise_on_error: bool = False, +) -> str: """ 使用本地 Ollama 部署的多模态大模型分析地图截图。 已改为一次性返回模式，以提高连接稳定性。 @@ -250,10 +300,10 @@ async def analyze_map_with_vlm(images_base64: list, prompt: str, progress_callba if not images_base64: return "未接收到有效的地图截图。" - model_name = await _get_available_vlm_model() + resolved_model_name = await _get_available_vlm_model(model_name) payload = { - "model": model_name, + "model": resolved_model_name, "prompt": prompt, "images": [img.split(",")[1] if "," in img else img for img in images_base64], "stream": False, # 关闭流式传输，改为一次性返回 @@ -282,9 +332,11 @@ async def analyze_map_with_vlm(images_base64: list, prompt: str, progress_callba final_output += f"> [!NOTE] 思考过程\n> {full_thinking}\n\n" final_output += full_response - return final_output.strip() if final_output else f"模型 ({model_name}) 未返回任何内容。" + return final_output.strip() if final_output else f"模型 ({resolved_model_name}) 未返回任何内容。" except Exception as e: + if raise_on_error: + raise RuntimeError(f"Ollama VLM request failed: {str(e)}") from e return f"AI 分析过程中发生错误: {str(e)}" async def generate_dinsar_diagnosis( @@ -293,12 +345,13 @@ async def generate_dinsar_diagnosis( date_str: str, quality_context: str, hazard_info: str, - progress_callback=None + progress_callback=None, + model_name: Optional[str] = None, ) -> str: """ 针对 VLM 优化的 D-InSAR 专家诊断逻辑。 """ - model_name = await _get_available_vlm_model() + resolved_model_name = await _get_available_vlm_model(model_name) prompt = ( f"你是一位拥有 20 年经验的资深 InSAR 地质灾害解译专家。请根据提供的 D-InSAR 形变图及背景信息，撰写一份专业的诊断报告。\n\n" @@ -320,22 +373,22 @@ async def generate_dinsar_diagnosis( f"- 使用 Markdown 格式，语言严谨、专业，严禁幻觉。\n" f"- 报告末尾必须包含以下加粗文字：\n" f"**--- 免责声明 ---**\n" - f"**本报告由 AI 自动生成（模型：{model_name}），仅供科研参考，不具备法律效力。**" + f"**本报告由 AI 自动生成（模型：{resolved_model_name}），仅供科研参考，不具备法律效力。**" ) - return await analyze_map_with_vlm(images_base64, prompt, progress_callback=progress_callback) + return await analyze_map_with_vlm(images_base64, prompt, progress_callback=progress_callback, model_name=resolved_model_name) -async def warm_up_vlm() -> bool: +async def warm_up_vlm(model_name: Optional[str] = None) -> bool: """ 预热 VLM 模型，将其加载至显存。 发送一个轻量级请求以触发模型冷启动。 返回 True 表示成功，False 表示失败。 """ - # 预热时直接使用探测到的模型 - model_name = await _get_available_vlm_model() + # 预热时直接使用指定模型或探测到的模型 + resolved_model_name = await _get_available_vlm_model(model_name) payload = { - "model": model_name, + "model": resolved_model_name, "prompt": "hi", "stream": False, "keep_alive": "30m" diff --git a/backend/app/config.py b/backend/app/config.py index 1984965..6ea2e9a 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -101,6 +101,18 @@ def _default_task_pool_root(project_root: str) -> str: return os.path.join(normalized_root, "Task_Pool") +def _default_gf3_data_root(project_root: str) -> str: + normalized_root = os.path.normpath(project_root) + drive, _tail = os.path.splitdrive(normalized_root) + if drive: + return os.path.join(drive + os.sep, "GaoFen3_Pool") + return os.path.join(normalized_root, "GaoFen3_Pool") + + +def _default_gf3_task_pool_root(project_root: str) -> str: + return os.path.join(_default_gf3_data_root(project_root), "task_pool") + + def _default_runtime_dir(project_root: str, *parts: str) -> str: return os.path.join(_default_runtime_root(project_root), *parts) @@ -199,6 +211,7 @@ class Settings(BaseSettings): DINSAR_TASK_POOL_ROOT: str = "" SBAS_TASK_POOL_ROOT: str = "" GF3_TASK_POOL_ROOT: str = "" + DATA_DISTRIBUTION_ROOT: str = "" SOURCE_PRODUCT_DIRS: str = "" SENTINEL1_STORAGE_DIRS: str = "" ORBIT_SOURCE_DIRS: str = "" @@ -239,7 +252,7 @@ class Settings(BaseSettings): GF3_GEO_DEM_PATH: str = "" GF3_ARCHIVE_SOURCE_DIRS: str = "" GF3_ARCHIVE_EXTS: str = ".zip,.tar,.tar.gz,.tgz" - GF3_UNPACK_DELETE_ARCHIVE: bool = True + GF3_UNPACK_DELETE_ARCHIVE: bool = False GF3_LEGACY_GDAL_ENABLED: bool = False GF3_SOURCE_DIRS: str = "" GF3_SARSCAPE_NATIVE_DIRS: str = "" @@ -250,7 +263,7 @@ class Settings(BaseSettings): GF3_SARSCAPE_DEM_PATH: str = "" GF3_SARSCAPE_POLARIZATIONS: str = "HH,HV" GF3_SARSCAPE_KEEP_EXTRACTED: bool = True - GF3_SARSCAPE_AUTO_STANDARDIZE: bool = True + GF3_SARSCAPE_AUTO_STANDARDIZE: bool = False GF3_SARSCAPE_CLEAN_AFTER_SUCCESS: bool = True GF3_SARSCAPE_PRODUCE_TIMEOUT_SECONDS: int = 0 @@ -460,7 +473,9 @@ class Settings(BaseSettings): if not self.SBAS_TASK_POOL_ROOT: object.__setattr__(self, "SBAS_TASK_POOL_ROOT", os.path.join(self.TASK_POOL_ROOT, "SBAS")) if not self.GF3_TASK_POOL_ROOT: - object.__setattr__(self, "GF3_TASK_POOL_ROOT", os.path.join(self.TASK_POOL_ROOT, "GF3")) + object.__setattr__(self, "GF3_TASK_POOL_ROOT", _default_gf3_task_pool_root(project_root)) + if not self.DATA_DISTRIBUTION_ROOT: + object.__setattr__(self, "DATA_DISTRIBUTION_ROOT", os.path.join(self.TASK_POOL_ROOT, "Data_Distribution")) if not self.SAR_ANALYSIS_READY_ROOT: object.__setattr__( self, @@ -484,25 +499,25 @@ class Settings(BaseSettings): object.__setattr__( self, "GF3_ARCHIVE_SOURCE_DIRS", - os.path.join(_default_input_root(project_root), "gf3", "archives"), + os.path.join(_default_gf3_data_root(project_root), "archives"), ) if not self.GF3_SARSCAPE_NATIVE_DIRS: object.__setattr__( self, "GF3_SARSCAPE_NATIVE_DIRS", - os.path.join(_default_result_publish_root(project_root), "gf3", "sarscape_native"), + os.path.join(_default_gf3_data_root(project_root), "native_geo"), ) if not self.GF3_STORAGE_DIRS: object.__setattr__( self, "GF3_STORAGE_DIRS", - os.path.join(_default_result_publish_root(project_root), "gf3", "standard_l2"), + os.path.join(_default_gf3_data_root(project_root), "catalog"), ) if not self.GF3_SARSCAPE_RUNTIME_DIR: object.__setattr__( self, "GF3_SARSCAPE_RUNTIME_DIR", - _default_runtime_dir(project_root, "gf3", "sarscape_runtime"), + os.path.join(self.GF3_TASK_POOL_ROOT, "sarscape_runtime"), ) if not self.ORBIT_QUARANTINE_DIR and self.MONITOR_ORBIT_DIR: object.__setattr__( @@ -835,7 +850,8 @@ class Settings(BaseSettings): if "lutan" in item.lower() or "lt1" in item.lower() ] lt1_roots = list(dict.fromkeys(lt1_roots)) - object.__setattr__(self, "GAMMA_SBAS_SOURCE_ROOTS", ";".join(lt1_roots) or r"D:\LuTan1_Image_Pool") + materialized_lt1_root = os.path.join(self.TASK_POOL_ROOT, "source_materialized", "lutan1") + object.__setattr__(self, "GAMMA_SBAS_SOURCE_ROOTS", materialized_lt1_root) if not self.GAMMA_SBAS_ORBIT_ROOTS: object.__setattr__(self, "GAMMA_SBAS_ORBIT_ROOTS", self.PYINT_ORBIT_POOL_TXT or self.ORBIT_POOL_ENVI) object.__setattr__(self, "GAMMA_SBAS_DEFAULT_RLKS", max(1, int(self.GAMMA_SBAS_DEFAULT_RLKS or 8))) @@ -1014,6 +1030,7 @@ class Settings(BaseSettings): os.makedirs(settings.DINSAR_TASK_POOL_ROOT, exist_ok=True) os.makedirs(settings.SBAS_TASK_POOL_ROOT, exist_ok=True) os.makedirs(settings.GF3_TASK_POOL_ROOT, exist_ok=True) + os.makedirs(settings.DATA_DISTRIBUTION_ROOT, exist_ok=True) for path in split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS): os.makedirs(path, exist_ok=True) for path in split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS): @@ -1100,6 +1117,24 @@ def _is_wsl_posix_path(value: str) -> bool: return text.startswith("/home/") or text.startswith("/mnt/") +def _is_unc_path(value: str) -> bool: + return str(value or "").strip().strip('"').strip("'").startswith("\\\\") + + +def _enforce_local_runtime_paths( + *, + pairs: list[tuple[str, str]], + errors: list[str], +) -> None: + for label, raw_value in pairs: + raw_text = str(raw_value or "") + values = split_env_paths(raw_text) if (";" in raw_text or "," in raw_text) else [raw_text] + for item in values: + text = str(item or "").strip().strip('"').strip("'") + if text and _is_unc_path(text): + errors.append(f"{label} points to UNC path {text}; active production paths must be local.") + + def _check_path( *, label: str, @@ -1171,21 +1206,66 @@ def validate_runtime_config() -> dict[str, Any]: else: info.append("DATABASE_URL 已配置。") + _enforce_local_runtime_paths( + pairs=[ + ("UNPACK_SOURCE_DIRS", settings.UNPACK_SOURCE_DIRS), + ("SOURCE_PRODUCT_DIRS", settings.SOURCE_PRODUCT_DIRS), + ("SENTINEL1_STORAGE_DIRS", settings.SENTINEL1_STORAGE_DIRS), + ("ORBIT_SOURCE_DIRS", settings.ORBIT_SOURCE_DIRS), + ("INSAR_STORAGE_DIRS", settings.INSAR_STORAGE_DIRS), + ("MONITOR_RADAR_DIRS", settings.MONITOR_RADAR_DIRS), + ("MONITOR_DINSAR_DIRS", settings.MONITOR_DINSAR_DIRS), + ("MONITOR_ORBIT_DIR", settings.MONITOR_ORBIT_DIR), + ("TASK_POOL_ROOT", settings.TASK_POOL_ROOT), + ("DINSAR_TASK_POOL_ROOT", settings.DINSAR_TASK_POOL_ROOT), + ("SBAS_TASK_POOL_ROOT", settings.SBAS_TASK_POOL_ROOT), + ("GF3_TASK_POOL_ROOT", settings.GF3_TASK_POOL_ROOT), + ("DATA_DISTRIBUTION_ROOT", settings.DATA_DISTRIBUTION_ROOT), + ("ORBIT_POOL_ENVI", settings.ORBIT_POOL_ENVI), + ("ORBIT_POOL_ISCE2", settings.ORBIT_POOL_ISCE2), + ("ORBIT_POOL_LANDSAR", settings.ORBIT_POOL_LANDSAR), + ("PYINT_ORBIT_POOL_TXT", settings.PYINT_ORBIT_POOL_TXT), + ("PYINT_WORK_ROOT", settings.PYINT_WORK_ROOT), + ("PYINT_OUTPUT_ROOT", settings.PYINT_OUTPUT_ROOT), + ("LANDSAR_WORK_ROOT", settings.LANDSAR_WORK_ROOT), + ("RESULT_PUBLISH_ROOT", settings.RESULT_PUBLISH_ROOT), + ("DINSAR_PRODUCT_DIR", settings.DINSAR_PRODUCT_DIR), + ("TIMESERIES_PRODUCT_DIR", settings.TIMESERIES_PRODUCT_DIR), + ("SAR_ANALYSIS_READY_ROOT", settings.SAR_ANALYSIS_READY_ROOT), + ("SAR_ANALYSIS_WORK_ROOT", settings.SAR_ANALYSIS_WORK_ROOT), + ("GAMMA_SBAS_WORK_ROOT", settings.GAMMA_SBAS_WORK_ROOT), + ("GAMMA_SBAS_PRODUCT_ROOT", settings.GAMMA_SBAS_PRODUCT_ROOT), + ("GAMMA_SBAS_SOURCE_ROOTS", settings.GAMMA_SBAS_SOURCE_ROOTS), + ("GAMMA_SBAS_ORBIT_ROOTS", settings.GAMMA_SBAS_ORBIT_ROOTS), + ("GF3_ARCHIVE_SOURCE_DIRS", settings.GF3_ARCHIVE_SOURCE_DIRS), + ("GF3_SOURCE_DIRS", settings.GF3_SOURCE_DIRS), + ("GF3_SARSCAPE_NATIVE_DIRS", settings.GF3_SARSCAPE_NATIVE_DIRS), + ("GF3_STORAGE_DIRS", settings.GF3_STORAGE_DIRS), + ("GF3_SARSCAPE_RUNTIME_DIR", settings.GF3_SARSCAPE_RUNTIME_DIR), + ], + errors=errors, + ) + _check_path(label="PYTHON_PATH", value=settings.PYTHON_PATH, errors=errors, warnings=warnings, expect_file=True) _check_path(label="NGINX_PATH", value=settings.NGINX_PATH, errors=errors, warnings=warnings, required=True, expect_file=True) _check_path(label="LICENSE_PATH", value=settings.LICENSE_PATH, errors=errors, warnings=warnings, expect_file=True) _check_path(label="IDL_EXECUTABLE", value=settings.IDL_EXECUTABLE, errors=errors, warnings=warnings, expect_file=True) _check_path(label="IDL_WORKBENCH_PATH", value=settings.IDL_WORKBENCH_PATH, errors=errors, warnings=warnings, expect_file=True) _check_path(label="GF3_GEO_DEM_PATH", value=settings.GF3_GEO_DEM_PATH, errors=errors, warnings=warnings, expect_file=True) - _check_path(label="GF3_SARSCAPE_WRAPPER_EXE", value=settings.GF3_SARSCAPE_WRAPPER_EXE, errors=errors, warnings=warnings, expect_file=True) - _check_path(label="GF3_SARSCAPE_IDLRT_PATH", value=settings.GF3_SARSCAPE_IDLRT_PATH, errors=errors, warnings=warnings, expect_file=True) - _check_path( - label="GF3_SARSCAPE_DEM_PATH", - value=(settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH), - errors=errors, - warnings=warnings, - expect_file=True, + gf3_local_sarscape_enabled = bool( + settings.GF3_LEGACY_GDAL_ENABLED + or settings.GF3_SARSCAPE_AUTO_STANDARDIZE ) + if gf3_local_sarscape_enabled: + _check_path(label="GF3_SARSCAPE_WRAPPER_EXE", value=settings.GF3_SARSCAPE_WRAPPER_EXE, errors=errors, warnings=warnings, expect_file=True) + _check_path(label="GF3_SARSCAPE_IDLRT_PATH", value=settings.GF3_SARSCAPE_IDLRT_PATH, errors=errors, warnings=warnings, expect_file=True) + _check_path( + label="GF3_SARSCAPE_DEM_PATH", + value=(settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH), + errors=errors, + warnings=warnings, + expect_file=True, + ) _check_path(label="SRTM_DEM_DIR", value=settings.SRTM_DEM_DIR, errors=errors, warnings=warnings, expect_file=False) _check_path(label="WATER_RESULTS_DIR", value=settings.WATER_RESULTS_DIR, errors=errors, warnings=warnings, expect_file=False) _check_path(label="SAR_ANALYSIS_READY_ROOT", value=settings.SAR_ANALYSIS_READY_ROOT, errors=errors, warnings=warnings, expect_file=False) @@ -1213,9 +1293,6 @@ def validate_runtime_config() -> dict[str, Any]: _check_path(label="RESULT_QUARANTINE_ROOT", value=settings.RESULT_QUARANTINE_ROOT, errors=errors, warnings=warnings, expect_file=False) for label, raw_value in ( - ("UNPACK_SOURCE_DIRS", settings.UNPACK_SOURCE_DIRS), - ("INSAR_STORAGE_DIRS", settings.INSAR_STORAGE_DIRS), - ("MONITOR_RADAR_DIRS", settings.MONITOR_RADAR_DIRS), ("MONITOR_DINSAR_DIRS", settings.MONITOR_DINSAR_DIRS), ("GF3_SARSCAPE_NATIVE_DIRS", settings.GF3_SARSCAPE_NATIVE_DIRS), ("GF3_STORAGE_DIRS", settings.GF3_STORAGE_DIRS), @@ -1230,7 +1307,7 @@ def validate_runtime_config() -> dict[str, Any]: gf3_archive_dirs = split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS) if not gf3_archive_dirs: - warnings.append("GF3_ARCHIVE_SOURCE_DIRS 未配置；无法触发 GF3 SARscape 生产，只能扫描已有原生结果。") + warnings.append("GF3_ARCHIVE_SOURCE_DIRS 未配置；GF3 本机生产已停用，原始归档仅用于本地资产追踪。") for item in gf3_archive_dirs: _check_path(label="GF3_ARCHIVE_SOURCE_DIRS", value=item, errors=errors, warnings=warnings, expect_file=False) diff --git a/backend/app/copier.py b/backend/app/copier.py index aa7c70d..920d7f2 100644 --- a/backend/app/copier.py +++ b/backend/app/copier.py @@ -6,11 +6,12 @@ import tarfile import zipfile import json import hashlib +import re from datetime import datetime from typing import List, Tuple, Optional, Dict, Any from .services.task_service import task_service -from .services.dinsar_naming import write_pair_metadata +from .services.dinsar_naming import build_task_alias, write_pair_metadata # --- Core Logic --- @@ -44,7 +45,20 @@ def find_dinsar_source_to_copy(path: str) -> str: return path +def _resolve_dinsar_task_names(item: Dict[str, Any]) -> Tuple[str, str]: + task_name = str(item.get("task_name") or item.get("task_alias") or "").strip() + task_alias = str(item.get("task_alias") or "").strip() + if not task_alias: + task_alias = build_task_alias(item.get("master_imaging_date"), item.get("slave_imaging_date")) + if task_alias == "Task_unknown_unknown" and task_name: + task_alias = task_name + if not task_name: + task_name = task_alias or "task" + return task_name or "task", task_alias or task_name or "task" + + _DINSAR_ARCHIVE_SUFFIXES = (".tar.gz", ".tgz", ".zip", ".tar") +_LT1_RASTER_SUFFIXES = (".tif", ".tiff") def _is_supported_archive(path: str) -> bool: @@ -52,6 +66,243 @@ def _is_supported_archive(path: str) -> bool: return any(lower.endswith(suffix) for suffix in _DINSAR_ARCHIVE_SUFFIXES) +def _is_lt1_scene_file_name(filename: str) -> bool: + return str(filename or "").lower().startswith("lt1") + + +def _is_lt1_meta_name(filename: str) -> bool: + lower = str(filename or "").lower() + return _is_lt1_scene_file_name(lower) and lower.endswith(".meta.xml") + + +def _is_lt1_raster_name(filename: str) -> bool: + lower = str(filename or "").lower() + return _is_lt1_scene_file_name(lower) and lower.endswith(_LT1_RASTER_SUFFIXES) + + +def _extract_yyyymmdd_from_name(filename: str) -> str: + match = re.search(r"(20\d{6})", str(filename or "")) + return match.group(1) if match else "" + + +def _iter_lt1_scene_files(root_dir: str) -> List[str]: + normalized = os.path.normpath(os.path.abspath(str(root_dir or ""))) + if not os.path.isdir(normalized): + return [] + files: List[str] = [] + for current_root, _, filenames in os.walk(normalized): + for filename in filenames: + if _is_lt1_scene_file_name(filename): + files.append(os.path.join(current_root, filename)) + return sorted(files, key=lambda item: os.path.normcase(os.path.relpath(item, normalized))) + + +def _link_or_copy_file(source_path: str, dest_path: str) -> str: + source = os.path.normpath(os.path.abspath(str(source_path or ""))) + dest = os.path.normpath(os.path.abspath(str(dest_path or ""))) + if not os.path.isfile(source): + raise FileNotFoundError(source) + os.makedirs(os.path.dirname(dest), exist_ok=True) + if os.path.exists(dest): + try: + if os.path.samefile(source, dest): + return "exists" + except OSError: + pass + if os.path.isdir(dest): + raise IsADirectoryError(dest) + os.remove(dest) + try: + os.link(source, dest) + return "linked" + except OSError: + shutil.copy2(source, dest) + return "copied" + + +def _list_direct_files(directory: str) -> List[str]: + if not os.path.isdir(directory): + return [] + return sorted( + os.path.join(directory, entry.name) + for entry in os.scandir(directory) + if entry.is_file() + ) + + +def _flatten_lt1_side_inputs(side_dir: str) -> Dict[str, Any]: + normalized = os.path.normpath(os.path.abspath(str(side_dir or ""))) + summary: Dict[str, Any] = { + "side_dir": normalized, + "lt1_source_file_count": 0, + "linked": 0, + "copied": 0, + "exists": 0, + "direct_meta_count": 0, + "direct_raster_count": 0, + } + if not os.path.isdir(normalized): + summary["status"] = "missing_side_dir" + return summary + + source_files = _iter_lt1_scene_files(normalized) + summary["lt1_source_file_count"] = len(source_files) + for source in source_files: + dest = os.path.join(normalized, os.path.basename(source)) + action = _link_or_copy_file(source, dest) + if action in {"linked", "copied", "exists"}: + summary[action] = int(summary.get(action, 0)) + 1 + + direct_files = _list_direct_files(normalized) + summary["direct_meta_count"] = sum(1 for path in direct_files if _is_lt1_meta_name(os.path.basename(path))) + summary["direct_raster_count"] = sum(1 for path in direct_files if _is_lt1_raster_name(os.path.basename(path))) + summary["status"] = "ready" if summary["direct_meta_count"] and summary["direct_raster_count"] else "no_lt1_direct_pair" + return summary + + +def _select_lt1_meta_raster(side_dir: str, expected_date: Any = None) -> Dict[str, str]: + expected = re.sub(r"\D", "", str(expected_date or ""))[:8] + direct_files = _list_direct_files(side_dir) + metas = [path for path in direct_files if _is_lt1_meta_name(os.path.basename(path))] + rasters = [path for path in direct_files if _is_lt1_raster_name(os.path.basename(path))] + + def sort_key(path: str) -> tuple[int, str]: + name = os.path.basename(path) + date_mismatch = 0 if expected and expected in name else 1 if expected else 0 + return date_mismatch, name.lower() + + metas.sort(key=sort_key) + rasters.sort(key=sort_key) + return { + "meta": metas[0] if metas else "", + "raster": rasters[0] if rasters else "", + } + + +def _prepare_landsar_input_data( + task_dir: str, + master_selection: Dict[str, str], + slave_selection: Dict[str, str], +) -> Dict[str, Any]: + input_dir = os.path.join(task_dir, "Input_Data") + copied: List[Dict[str, Any]] = [] + for role, selection in (("master", master_selection), ("slave", slave_selection)): + for kind, source in (("meta", selection.get("meta")), ("raster", selection.get("raster"))): + if not source: + return { + "status": "missing_selected_lt1_file", + "input_data_dir": input_dir, + "role": role, + "kind": kind, + "copied": copied, + } + dest = os.path.join(input_dir, os.path.basename(source)) + action = _link_or_copy_file(source, dest) + copied.append( + { + "role": role, + "kind": kind, + "source_path": source, + "relative_path": os.path.relpath(dest, start=task_dir), + "action": action, + } + ) + return { + "status": "ready", + "input_data_dir": input_dir, + "copied": copied, + } + + +def _landsar_input_data_ready(input_data_dir: str) -> bool: + if not os.path.isdir(input_data_dir): + return False + by_date: Dict[str, Dict[str, bool]] = {} + for path in _list_direct_files(input_data_dir): + name = os.path.basename(path) + if not _is_lt1_scene_file_name(name): + continue + lower_name = name.lower() + if lower_name.endswith(".meta.xml") or lower_name.endswith("_check.xml"): + continue + if lower_name.endswith(".xml") and "_slc" not in lower_name: + continue + date_text = _extract_yyyymmdd_from_name(name) + if not date_text: + continue + entry = by_date.setdefault(date_text, {"meta": False, "raster": False}) + if lower_name.endswith(".xml"): + entry["xml"] = True + elif _is_lt1_raster_name(name): + entry["raster"] = True + return sum(1 for item in by_date.values() if item.get("xml") and item.get("raster")) >= 2 + + +def _lt1_side_direct_raw_ready(side_dir: str) -> bool: + direct_files = _list_direct_files(side_dir) + return ( + any(_is_lt1_meta_name(os.path.basename(path)) for path in direct_files) + and any(_is_lt1_raster_name(os.path.basename(path)) for path in direct_files) + ) + + +def _prepare_dinsar_engine_inputs(task_dir: str, item: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + normalized = os.path.normpath(os.path.abspath(str(task_dir or ""))) + payload = dict(item or {}) + master_dir = os.path.join(normalized, "master") + slave_dir = os.path.join(normalized, "slave") + master_summary = _flatten_lt1_side_inputs(master_dir) + slave_summary = _flatten_lt1_side_inputs(slave_dir) + summary: Dict[str, Any] = { + "master": master_summary, + "slave": slave_summary, + "landsar": {"status": "skipped_non_lt1"}, + } + + has_lt1 = bool(master_summary.get("lt1_source_file_count")) or bool(slave_summary.get("lt1_source_file_count")) + if not has_lt1: + return summary + + master_selection = _select_lt1_meta_raster(master_dir, payload.get("master_imaging_date")) + slave_selection = _select_lt1_meta_raster(slave_dir, payload.get("slave_imaging_date")) + summary["selected"] = { + "master": { + "meta": os.path.relpath(master_selection["meta"], start=normalized) if master_selection.get("meta") else "", + "raster": os.path.relpath(master_selection["raster"], start=normalized) if master_selection.get("raster") else "", + }, + "slave": { + "meta": os.path.relpath(slave_selection["meta"], start=normalized) if slave_selection.get("meta") else "", + "raster": os.path.relpath(slave_selection["raster"], start=normalized) if slave_selection.get("raster") else "", + }, + } + if not all(master_selection.values()) or not all(slave_selection.values()): + summary["landsar"] = {"status": "missing_lt1_meta_or_raster"} + return summary + + summary["landsar"] = { + "status": "raw_ready_for_import", + "input_data_policy": "LandSAR 100016 import creates native landsar_input at run time.", + } + return summary + + +def _normalize_source_bundle_archive_path(source_path: str) -> str: + normalized = os.path.normpath(os.path.abspath(str(source_path or ""))) + if not os.path.exists(normalized): + raise FileNotFoundError(normalized) + if os.path.isdir(normalized): + raise ValueError( + "D-InSAR source bundle distribution requires source archive files; " + f"unpacked directories are retired: {normalized}" + ) + if not os.path.isfile(normalized) or not _is_supported_archive(normalized): + raise ValueError( + "D-InSAR source bundle distribution requires .zip/.tar.gz/.tgz/.tar source archives: " + f"{normalized}" + ) + return normalized + + def _safe_archive_member_name(member_name: str, archive_path: str) -> str: name = str(member_name or "").replace("\\", "/").strip("/") if not name or name.startswith("../") or "/../" in f"/{name}/": @@ -104,10 +355,20 @@ def _extract_archive_to_dir(archive_path: str, dest_dir: str) -> int: raise ValueError(f"Unsupported archive format: {archive_path}") -def _materialize_dinsar_source(source_path: str, dest_dir: str) -> Dict[str, Any]: +def _materialize_dinsar_source( + source_path: str, + dest_dir: str, + *, + require_archive: bool = False, +) -> Dict[str, Any]: normalized = os.path.normpath(os.path.abspath(str(source_path or ""))) if not os.path.exists(normalized): raise FileNotFoundError(normalized) + if require_archive and (not os.path.isfile(normalized) or not _is_supported_archive(normalized)): + raise ValueError( + "D-InSAR production preparation requires source archive files; " + f"rebuild the batch from LT-1/Sentinel-1 archive assets: {normalized}" + ) if os.path.isdir(normalized): shutil.copytree(normalized, dest_dir, dirs_exist_ok=True) @@ -266,9 +527,16 @@ def _directory_has_entries(path: str) -> bool: def _is_existing_dinsar_folder_complete(task_dir: str) -> bool: + master_dir = os.path.join(task_dir, "master") + slave_dir = os.path.join(task_dir, "slave") + input_data_dir = os.path.join(task_dir, "Input_Data") + has_lt1_inputs = bool(_iter_lt1_scene_files(master_dir) or _iter_lt1_scene_files(slave_dir)) + lt1_raw_ready = _lt1_side_direct_raw_ready(master_dir) and _lt1_side_direct_raw_ready(slave_dir) + input_data_ready = (not has_lt1_inputs) or lt1_raw_ready or _landsar_input_data_ready(input_data_dir) return ( - _directory_has_entries(os.path.join(task_dir, "master")) - and _directory_has_entries(os.path.join(task_dir, "slave")) + _directory_has_entries(master_dir) + and _directory_has_entries(slave_dir) + and input_data_ready ) @@ -532,8 +800,7 @@ async def run_dinsar_source_bundle_items( candidate_tasks: List[Dict[str, Any]] = [] for item in items: - task_name = item.get("task_name") or item.get("task_alias") or "task" - task_alias = item.get("task_alias") or task_name + task_name, task_alias = _resolve_dinsar_task_names(item) master_path = item.get("master_path") slave_path = item.get("slave_path") if not master_path or not slave_path: @@ -749,7 +1016,7 @@ async def run_dinsar_source_bundle_items( ) def ensure_scene_entry(scene_path: str, role_item: Dict[str, Any], prefix: str) -> Dict[str, Any]: - source_path = os.path.normpath(os.path.abspath(str(scene_path))) + source_path = _normalize_source_bundle_archive_path(scene_path) source_key = os.path.normcase(source_path) relative_path = _bundle_relative_scene_path(source_path) existing = scene_entries_by_source.get(source_key) or scene_entries_by_relative.get(relative_path) @@ -1100,8 +1367,7 @@ async def run_dinsar_copy_items( tasks: List[Dict[str, Any]] = [] for item in items: - task_name = item.get("task_name") or item.get("task_alias") or "task" - task_alias = item.get("task_alias") or task_name + task_name, task_alias = _resolve_dinsar_task_names(item) master_path = item.get("master_path") slave_path = item.get("slave_path") if not master_path or not slave_path: @@ -1195,11 +1461,13 @@ async def run_dinsar_copy_items( _materialize_dinsar_source, master_src_path, master_dir, + require_archive=True, ) slave_materialization = await asyncio.to_thread( _materialize_dinsar_source, slave_src_path, slave_dir, + require_archive=True, ) source_materialization = { "master": { @@ -1211,6 +1479,19 @@ async def run_dinsar_copy_items( "target_relative_path": "slave", }, } + engine_inputs = await asyncio.to_thread( + _prepare_dinsar_engine_inputs, + task_dir, + item, + ) + source_materialization["engine_inputs"] = engine_inputs + landsar_status = str(engine_inputs.get("landsar", {}).get("status") or "") + if landsar_status == "raw_ready_for_import": + await _log_and_update(task_id, " -> LT-1 raw inputs ready; LandSAR will run 100016 import at production time") + elif landsar_status == "skipped_non_lt1": + await _log_and_update(task_id, " -> Skipped LandSAR input preparation (non-LT1 source)") + else: + await _log_and_update(task_id, f" -> LandSAR raw inputs not ready: {landsar_status or 'unknown'}") orbit_entries = await _copy_dinsar_orbit_files( task_id, item, @@ -1235,10 +1516,8 @@ async def run_dinsar_copy_items( await _log_and_update(task_id, f" -> ZIP: {zip_path}") elif not export_zip: if os.path.exists(final_task_dir): - raise CopyTaskExecutionError( - "Destination folder already exists and cannot be atomically replaced: " - f"{final_task_dir}" - ) + await asyncio.to_thread(shutil.rmtree, final_task_dir) + await _log_and_update(task_id, f" -> Replaced incomplete existing Task: {final_task_dir}") await asyncio.to_thread(os.replace, task_dir, final_task_dir) await _log_and_update(task_id, f" -> Folder: {final_task_dir}") diff --git a/backend/app/db_maintenance.py b/backend/app/db_maintenance.py index 035c34a..2619613 100644 --- a/backend/app/db_maintenance.py +++ b/backend/app/db_maintenance.py @@ -40,6 +40,8 @@ MIGRATION_FILES = [ "008_timeseries_stack_plan_edges.sql", "009_raw_source_pairing_fields.sql", "010_source_orbit_asset_inventory.sql", + "011_source_metadata_documents.sql", + "012_source_archive_integrity.sql", ] diff --git a/backend/app/dinsar_engines/landsar_engine.py b/backend/app/dinsar_engines/landsar_engine.py index 0b35acd..4bc6113 100644 --- a/backend/app/dinsar_engines/landsar_engine.py +++ b/backend/app/dinsar_engines/landsar_engine.py @@ -9,18 +9,22 @@ can run in the service process without desktop dependencies. """ from __future__ import annotations +import hashlib import json +import math import os import queue import re import shutil import socket import subprocess +import tempfile import threading import time from datetime import datetime from pathlib import Path from typing import Any, Dict, Iterable, List, Optional +import xml.etree.ElementTree as ET from ..config import get_env_text, read_bool_env, settings from ..services.dinsar_naming import ( @@ -42,8 +46,10 @@ SUPPORTED_PROFILES = {"lt1_dinsar", "standard"} _PROJECT_ROOT = Path(__file__).resolve().parents[3] _DATE_RE = re.compile(r"(?:^|[_-])((?:19|20)\d{6})(?:[_-]|$)") +_CENTER_RE = re.compile(r"(?:^|_)E([+-]?\d+(?:\.\d+)?)_N([+-]?\d+(?:\.\d+)?)(?:_|$)", re.IGNORECASE) _SAFE_NAME_RE = re.compile(r"[^0-9A-Za-z._-]+") _SUCCESS_RE = re.compile(r"(success|成功)", re.IGNORECASE) +_DEFAULT_DEM_CROP_MARGIN_DEGREES = 0.35 _DEFAULT_PARAM_VALUES: Dict[str, Any] = { "dem_file_type": 0, @@ -271,6 +277,33 @@ def _extract_date(name: str) -> str: return fallback.group(1) if fallback else "" +def _safe_path_name(value: Any, fallback: str = "item") -> str: + text = _SAFE_NAME_RE.sub("_", str(value or "").strip()).strip("._-") + return text or fallback + + +def _extract_center_from_name(name: str) -> Optional[tuple[float, float]]: + match = _CENTER_RE.search(name or "") + if not match: + return None + try: + return float(match.group(1)), float(match.group(2)) + except (TypeError, ValueError): + return None + + +def _extract_center_from_path(path: str) -> Optional[tuple[float, float]]: + source = _norm_path(path) + if not source: + return None + candidates = [os.path.basename(source), os.path.basename(os.path.dirname(source))] + for candidate in candidates: + center = _extract_center_from_name(candidate) + if center: + return center + return None + + def _collect_tail(text: str, max_chars: int = 4000) -> str: content = str(text or "") if len(content) <= max_chars: @@ -296,6 +329,13 @@ def _summarize_landsar_failure(stdout_text: str, stage: str, return_code: int) - return f"{prefix}: LandSAR license dongle login failed{status_text}." if "cann't find 'config.csv'" in lowered or "can't find 'config.csv'" in lowered: return f"{prefix}: LandSAR config.csv is missing. Run versionControl.exe once from LANDSAR_HOME." + if "cannot operate regis module" in lowered or "invalid parameters" in lowered: + return ( + f"{prefix}: registration module failed. " + "LandSAR could not obtain a valid coregistration solution for this pair." + ) + if "invalid data type" in lowered or "access window out of range" in lowered or "subterrain phase failed" in lowered: + return f"{prefix}: DEM/sub-terrain processing failed. Check DEM coverage, data type, and LandSAR-readable format." lines = [line.strip() for line in content.splitlines() if line.strip()] error_lines = [ line @@ -418,6 +458,348 @@ def _looks_like_raw_task_dir(task_dir: str) -> bool: ) +def _find_lt1_scene_token(directory: str) -> str: + source_dir = _norm_path(directory) + if not os.path.isdir(source_dir): + return "" + for entry in os.scandir(source_dir): + if entry.is_file(): + match = re.match(r"^(LT1[AB])_", entry.name, re.IGNORECASE) + if match: + return match.group(1).upper() + return "" + + +def _infer_lt1_import_sat_mode(master_dir: str, slave_dir: str) -> str: + master_sat = _find_lt1_scene_token(master_dir) + slave_sat = _find_lt1_scene_token(slave_dir) + if master_sat and slave_sat and master_sat != slave_sat: + return "BIST" + return "MONO" + + +def _read_dem_bounds(dem_path: str) -> Optional[tuple[float, float, float, float]]: + path = _norm_path(dem_path) + if not path or not os.path.isfile(path): + return None + try: + import rasterio # type: ignore + + with rasterio.open(path) as dataset: + bounds = dataset.bounds + return float(bounds.left), float(bounds.bottom), float(bounds.right), float(bounds.top) + except Exception: + return None + + +def _scene_centers_outside_dem( + parsed_pair: Dict[str, Any], + dem_path: str, + *, + margin_degrees: float = 0.25, +) -> List[str]: + bounds = _read_dem_bounds(dem_path) + if not bounds: + return [] + left, bottom, right, top = bounds + blockers: List[str] = [] + for role, key in (("master", "master_xml"), ("slave", "slave_xml")): + source_path = str(parsed_pair.get(key) or "") + center = _extract_center_from_path(source_path) + if not center: + continue + lon, lat = center + if lon < left + margin_degrees or lon > right - margin_degrees or lat < bottom + margin_degrees or lat > top - margin_degrees: + blockers.append( + f"{role} center E{lon:.3f}/N{lat:.3f} is outside or too close to DEM bounds " + f"E{left:.3f}-{right:.3f}, N{bottom:.3f}-{top:.3f}" + ) + return blockers + + +def _xml_local_name(tag: str) -> str: + return str(tag or "").rsplit("}", 1)[-1].lower() + + +def _xml_float_text(value: Any) -> Optional[float]: + try: + text = str(value or "").strip() + if not text: + return None + return float(text) + except (TypeError, ValueError): + return None + + +def _xml_child_float_by_local_names(element: ET.Element, names: Iterable[str]) -> Optional[float]: + wanted = {str(name).lower() for name in names} + for child in element.iter(): + if child is element: + continue + if _xml_local_name(child.tag) in wanted: + value = _xml_float_text(child.text) + if value is not None: + return value + return None + + +def _extract_scene_lonlat_points_from_xml(xml_path: str) -> List[tuple[float, float]]: + source = _norm_path(xml_path) + if not source or not os.path.isfile(source): + return [] + try: + root = ET.parse(source).getroot() + except Exception: + return [] + + points: List[tuple[float, float]] = [] + for element in root.iter(): + if _xml_local_name(element.tag) != "scenecornercoord": + continue + lon = _xml_child_float_by_local_names(element, ("lon", "longitude")) + lat = _xml_child_float_by_local_names(element, ("lat", "latitude")) + if lon is not None and lat is not None: + points.append((lon, lat)) + if points: + return points + + for element in root.iter(): + if _xml_local_name(element.tag) != "scenecentercoord": + continue + lon = _xml_child_float_by_local_names(element, ("lon", "longitude")) + lat = _xml_child_float_by_local_names(element, ("lat", "latitude")) + if lon is not None and lat is not None: + return [(lon, lat)] + return [] + + +def _bbox_from_points(points: Iterable[tuple[float, float]]) -> Optional[tuple[float, float, float, float]]: + cleaned = [ + (float(lon), float(lat)) + for lon, lat in points + if math.isfinite(float(lon)) and math.isfinite(float(lat)) + ] + if not cleaned: + return None + lons = [item[0] for item in cleaned] + lats = [item[1] for item in cleaned] + return min(lons), min(lats), max(lons), max(lats) + + +def _expand_lonlat_bbox( + bbox: tuple[float, float, float, float], + margin_degrees: float, +) -> tuple[float, float, float, float]: + margin = max(0.0, float(margin_degrees or 0.0)) + left, bottom, right, top = bbox + return ( + max(-180.0, float(left) - margin), + max(-90.0, float(bottom) - margin), + min(180.0, float(right) + margin), + min(90.0, float(top) + margin), + ) + + +def derive_landsar_dem_bbox_from_xml_paths( + xml_paths: Iterable[str], + *, + fallback_paths: Iterable[str] = (), +) -> Optional[tuple[float, float, float, float]]: + points: List[tuple[float, float]] = [] + for xml_path in xml_paths: + points.extend(_extract_scene_lonlat_points_from_xml(str(xml_path or ""))) + bbox = _bbox_from_points(points) + if bbox: + return bbox + + fallback_points: List[tuple[float, float]] = [] + for path in fallback_paths: + center = _extract_center_from_path(str(path or "")) + if center: + fallback_points.append(center) + return _bbox_from_points(fallback_points) + + +def derive_landsar_pair_dem_bbox(parsed_pair: Dict[str, Any]) -> Optional[tuple[float, float, float, float]]: + return derive_landsar_dem_bbox_from_xml_paths( + [ + str(parsed_pair.get("master_xml") or ""), + str(parsed_pair.get("slave_xml") or ""), + ], + fallback_paths=[ + str(parsed_pair.get("master_xml") or parsed_pair.get("master_tif") or ""), + str(parsed_pair.get("slave_xml") or parsed_pair.get("slave_tif") or ""), + ], + ) + + +def _align_raster_window(window: Any, width: int, height: int) -> Any: + from rasterio.windows import Window + + col_off = max(0, int(math.floor(window.col_off))) + row_off = max(0, int(math.floor(window.row_off))) + col_stop = min(width, int(math.ceil(window.col_off + window.width))) + row_stop = min(height, int(math.ceil(window.row_off + window.height))) + if col_stop <= col_off or row_stop <= row_off: + raise ValueError("DEM crop bbox does not overlap source DEM") + return Window(col_off, row_off, col_stop - col_off, row_stop - row_off) + + +def _iter_raster_windows(width: int, height: int, block_size: int = 2048) -> Iterable[Any]: + from rasterio.windows import Window + + step = max(256, int(block_size or 2048)) + for row in range(0, int(height), step): + h = min(step, int(height) - row) + for col in range(0, int(width), step): + w = min(step, int(width) - col) + yield Window(col, row, w, h) + + +def _format_bbox_key(bbox: tuple[float, float, float, float]) -> str: + return ",".join(f"{value:.8f}" for value in bbox) + + +def prepare_landsar_dem_crop( + source_dem_path: str, + crop_root: str, + bbox: tuple[float, float, float, float], + *, + label: str = "task", + margin_degrees: float = _DEFAULT_DEM_CROP_MARGIN_DEGREES, + block_size: int = 2048, +) -> Dict[str, Any]: + source = _norm_path(source_dem_path) + if not source or not os.path.isfile(source): + raise FileNotFoundError(f"LandSAR DEM source file is missing: {source or ''}") + if not bbox: + raise ValueError("LandSAR DEM crop bbox is empty") + + expanded_bbox = _expand_lonlat_bbox(bbox, margin_degrees) + crop_dir = Path(_norm_path(crop_root)) + crop_dir.mkdir(parents=True, exist_ok=True) + safe_label = _safe_path_name(label, "task") + key = hashlib.sha1(f"{source}|{_format_bbox_key(expanded_bbox)}".encode("utf-8", errors="ignore")).hexdigest()[:16] + target = crop_dir / f"{safe_label}_{key}_dem.tif" + manifest = target.with_suffix(target.suffix + ".json") + + if target.is_file() and manifest.is_file(): + try: + payload = json.loads(manifest.read_text(encoding="utf-8")) + except Exception: + payload = {} + return { + "source_dem_path": source, + "dem_path": str(target), + "bbox": list(bbox), + "expanded_bbox": list(expanded_bbox), + "reused": True, + "manifest_path": str(manifest), + **({"bounds": payload.get("bounds")} if payload.get("bounds") else {}), + } + + try: + import rasterio # type: ignore + from rasterio.transform import array_bounds + from rasterio.windows import Window, from_bounds + except Exception as exc: + raise RuntimeError("rasterio is required to crop LandSAR DEMs") from exc + + temp_path: Optional[Path] = None + with rasterio.open(source) as src: + source_dtype = str(src.dtypes[0]).lower() + if source_dtype != "int16": + raise ValueError(f"LandSAR DEM source must be a prepared Int16 GeoTIFF, got dtype={src.dtypes[0]}: {source}") + bounds = src.bounds + left, bottom, right, top = expanded_bbox + if left < bounds.left or right > bounds.right or bottom < bounds.bottom or top > bounds.top: + raise ValueError( + "LandSAR DEM crop bbox is outside source DEM bounds: " + f"bbox=E{left:.6f}-{right:.6f},N{bottom:.6f}-{top:.6f}; " + f"source=E{bounds.left:.6f}-{bounds.right:.6f},N{bounds.bottom:.6f}-{bounds.top:.6f}" + ) + + source_window = _align_raster_window(from_bounds(*expanded_bbox, transform=src.transform), src.width, src.height) + source_window = Window( + int(source_window.col_off), + int(source_window.row_off), + int(source_window.width), + int(source_window.height), + ) + transform = src.window_transform(source_window) + crop_bounds = array_bounds(int(source_window.height), int(source_window.width), transform) + + profile = src.profile.copy() + profile.update( + driver="GTiff", + height=int(source_window.height), + width=int(source_window.width), + count=1, + dtype=src.dtypes[0], + crs=src.crs, + transform=transform, + nodata=src.nodata, + compress="NONE", + BIGTIFF="YES", + interleave="band", + ) + profile.pop("photometric", None) + profile.pop("predictor", None) + if int(source_window.width) >= 512 and int(source_window.height) >= 512: + profile.update(tiled=True, blockxsize=512, blockysize=512) + else: + profile.pop("blockxsize", None) + profile.pop("blockysize", None) + profile["tiled"] = False + + try: + with tempfile.NamedTemporaryFile( + prefix=f"{target.stem}.", + suffix=".tmp.tif", + dir=str(crop_dir), + delete=False, + ) as tmp: + temp_path = Path(tmp.name) + + with rasterio.open(temp_path, "w", **profile) as dst: + for rel_window in _iter_raster_windows(int(source_window.width), int(source_window.height), block_size): + src_window = Window( + source_window.col_off + rel_window.col_off, + source_window.row_off + rel_window.row_off, + rel_window.width, + rel_window.height, + ) + dst.write(src.read(1, window=src_window, masked=False), 1, window=rel_window) + os.replace(temp_path, target) + temp_path = None + payload = { + "source_dem_path": source, + "dem_path": str(target), + "bbox": list(bbox), + "expanded_bbox": list(expanded_bbox), + "bounds": [float(value) for value in crop_bounds], + "width": int(source_window.width), + "height": int(source_window.height), + "dtype": src.dtypes[0], + "nodata": src.nodata, + "margin_degrees": float(margin_degrees), + } + manifest.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") + finally: + if temp_path and temp_path.exists(): + temp_path.unlink(missing_ok=True) + + return { + "source_dem_path": source, + "dem_path": str(target), + "bbox": list(bbox), + "expanded_bbox": list(expanded_bbox), + "bounds": [float(value) for value in crop_bounds], + "reused": False, + "manifest_path": str(manifest), + } + + def parse_lt1_slc_pair(input_data_dir: str) -> Optional[Dict[str, Any]]: """Return the first chronological LT-1 SLC pair from Input_Data.""" @@ -432,6 +814,10 @@ def parse_lt1_slc_pair(input_data_dir: str) -> Optional[Dict[str, Any]]: lower_name = entry.name.lower() if not lower_name.endswith((".xml", ".tif", ".tiff")): continue + if lower_name.endswith(".meta.xml") or lower_name.endswith("_check.xml"): + continue + if lower_name.endswith(".xml") and "_slc" not in lower_name: + continue date_text = _extract_date(entry.name) if not date_text: continue @@ -830,6 +1216,9 @@ class LandsarEngine(DinsarEngine): "type": "string", "default": self._default_dem_path, "section": "输入数据", + "readonly": True, + "readonly_label": "服务器固定", + "include_in_payload": False, "description": "传给 LandSAR 200014 D-InSAR 模块的外部参考 DEM，建议使用已验证可用的 GeoTIFF。", }, "az_looks": { @@ -1178,14 +1567,14 @@ class LandsarEngine(DinsarEngine): ) extra = self.normalize_extra(request.extra) - dem_path = _norm_path(extra.get("dem_path") or self._default_dem_path) - if not dem_path or not os.path.isfile(dem_path): + dem_source_path = _norm_path(extra.get("dem_path") or self._default_dem_path) + if not dem_source_path or not os.path.isfile(dem_source_path): return RunResult( success=False, engine_code=self.engine_code, profile=request.profile, job_id=request.job_id, - error=f"LandSAR DEM file is missing: {dem_path or ''}", + error=f"LandSAR DEM source file is missing: {dem_source_path or ''}", ) if _coerce_bool(extra.get("do_atmosphere")): gacos_path = _norm_path(extra.get("gacos_file")) @@ -1341,6 +1730,85 @@ class LandsarEngine(DinsarEngine): continue task_alias, pair_key, pair_meta = self._resolve_task_identity(task_dir, task_name, parsed_pair) + dem_blockers = _scene_centers_outside_dem(parsed_pair, dem_source_path) + if dem_blockers: + pairs_failed += 1 + error_text = "LandSAR DEM coverage preflight failed: " + "; ".join(dem_blockers) + emit_progress("pair_finished", pair_index=pair_index, pair_total=len(task_dirs), success=False, error=error_text) + task_results.append( + self._build_task_result( + task_name=task_name, + task_alias=task_alias, + pair_key=pair_key, + run_key=run_key, + task_dir=task_dir, + run_dir=run_dir, + native_output_dir=native_output_dir, + landsar_output_dir=landsar_output_dir, + command=command, + success=False, + returncode=-2, + error=error_text, + stdout_tail="", + param_file="", + dem_source_path=dem_source_path, + dem_path=effective_dem_path, + dem_crop=dem_crop_info, + ) + ) + continue + + effective_dem_path = dem_source_path + dem_crop_info: Dict[str, Any] = {} + try: + dem_bbox = derive_landsar_pair_dem_bbox(parsed_pair) + if not dem_bbox: + raise ValueError("cannot derive DEM crop bbox from LT-1 XML corner coordinates") + dem_crop_info = prepare_landsar_dem_crop( + dem_source_path, + os.path.join(native_output_dir, "dem_crop"), + dem_bbox, + label=task_alias, + ) + effective_dem_path = str(dem_crop_info.get("dem_path") or dem_source_path) + emit_progress( + "log", + pair_index=pair_index, + pair_total=len(task_dirs), + task_name=task_name, + task_alias=task_alias, + pair_key=pair_key, + level="INFO", + source="dem", + message=( + f"LandSAR DEM crop ready: {effective_dem_path} " + f"from {dem_source_path}" + ), + ) + except Exception as exc: + pairs_failed += 1 + error_text = f"LandSAR DEM crop failed: {exc}" + emit_progress("pair_finished", pair_index=pair_index, pair_total=len(task_dirs), success=False, error=error_text) + task_results.append( + self._build_task_result( + task_name=task_name, + task_alias=task_alias, + pair_key=pair_key, + run_key=run_key, + task_dir=task_dir, + run_dir=run_dir, + native_output_dir=native_output_dir, + landsar_output_dir=landsar_output_dir, + command=command, + success=False, + returncode=-2, + error=error_text, + stdout_tail="", + param_file="", + ) + ) + continue + param_values = {**_DEFAULT_PARAM_VALUES} param_values.update({key: value for key, value in extra.items() if key in _DEFAULT_PARAM_VALUES}) param_file = _generate_dinsar_param_file( @@ -1349,7 +1817,7 @@ class LandsarEngine(DinsarEngine): master_tif=parsed_pair["master_tif"], slave_xml=parsed_pair["slave_xml"], slave_tif=parsed_pair["slave_tif"], - dem_path=dem_path, + dem_path=effective_dem_path, output_dir=landsar_output_dir, params=param_values, ) @@ -1375,7 +1843,7 @@ class LandsarEngine(DinsarEngine): pair_key=pair_key, level="INFO", source="dem", - message=dem_path, + message=effective_dem_path, ) rc, stdout_text, timed_out = self._run_console( @@ -1461,7 +1929,9 @@ class LandsarEngine(DinsarEngine): request=request, pair_meta=pair_meta_payload, params=param_values, - dem_path=dem_path, + dem_path=effective_dem_path, + dem_source_path=dem_source_path, + dem_crop=dem_crop_info, landsar_output_dir=landsar_output_dir, primary_file=primary_file, source_files=source_files, @@ -1510,18 +1980,20 @@ class LandsarEngine(DinsarEngine): param_file=param_file, raw_primary_file=primary_raw, raw_coherence_file=coherence_raw, + dem_source_path=dem_source_path, + dem_path=effective_dem_path, + dem_crop=dem_crop_info, ) ) invalid_candidates = validation.get("invalid_candidates", []) pairs_failed += len(invalid_candidates) - overall_success = pairs_processed > 0 and pairs_failed == 0 - if pairs_processed > 0 and pairs_failed > 0: - overall_success = False + overall_success = pairs_processed > 0 or (pairs_processed == 0 and pairs_failed == 0) + run_status = "COMPLETED" if pairs_failed == 0 else ("PARTIAL" if pairs_processed > 0 else "FAILED") error = None if not overall_success: failed_names = [item.get("task_alias") or item.get("task_name") for item in task_results if not item.get("success")] - error = f"LandSAR run failed: {', '.join(failed_names[:10])}" if failed_names else "LandSAR run failed." + error = f"All LandSAR tasks failed: {', '.join(failed_names[:10])}" if failed_names else "LandSAR run failed." return RunResult( success=overall_success, @@ -1534,6 +2006,7 @@ class LandsarEngine(DinsarEngine): error=error, detail={ "mode": validation["mode"], + "run_status": run_status, "task_count": len(task_dirs), "selected_tasks": [item.get("task_alias") or item.get("task_name") for item in task_results], "invalid_candidates": invalid_candidates, @@ -1541,7 +2014,8 @@ class LandsarEngine(DinsarEngine): "run_key": run_key, "started_at": run_started_at_text, "timeout_seconds": timeout, - "dem_path": dem_path, + "dem_path": dem_source_path, + "dem_role": "global_prepared_source", "console_path": console_path, }, ) @@ -1674,13 +2148,14 @@ class LandsarEngine(DinsarEngine): "error": "Task directory has neither valid Input_Data nor valid master/slave raw LT-1 folders.", } + sat_mode = _infer_lt1_import_sat_mode(master_dir, slave_dir) param_file = _generate_import_param_file( os.path.join(export_dir, f"{IMPORT_PROID}.txt"), master_dir=master_dir, slave_dir=slave_dir, export_dir=export_dir, import_method="dir", - sat_mode="BIST", + sat_mode=sat_mode, read_xml=True, read_slc=True, export_to_new=True, @@ -1695,7 +2170,7 @@ class LandsarEngine(DinsarEngine): pair_key=pair_key, level="INFO", source="import", - message=f"LandSAR import 100016 -> {export_dir}", + message=f"LandSAR import 100016 ({sat_mode}) -> {export_dir}", ) rc, stdout_text, timed_out = self._run_console( command, @@ -1832,6 +2307,8 @@ class LandsarEngine(DinsarEngine): source_files: List[str], returncode: int, started_at: str, + dem_source_path: str = "", + dem_crop: Optional[Dict[str, Any]] = None, ) -> None: payload = { "run_key": run_key, @@ -1850,6 +2327,8 @@ class LandsarEngine(DinsarEngine): "params": { **dict(params or {}), "dem_path": dem_path, + "dem_source_path": dem_source_path or dem_path, + "dem_crop": dem_crop or {}, }, "metrics": { "returncode": returncode, @@ -1902,6 +2381,9 @@ class LandsarEngine(DinsarEngine): param_file: str = "", raw_primary_file: str = "", raw_coherence_file: str = "", + dem_source_path: str = "", + dem_path: str = "", + dem_crop: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: command_text = command if isinstance(command, str) else " ".join(str(part) for part in command) return { @@ -1919,6 +2401,9 @@ class LandsarEngine(DinsarEngine): "raw_primary_file": _norm_path(raw_primary_file) if raw_primary_file else "", "raw_coherence_file": _norm_path(raw_coherence_file) if raw_coherence_file else "", "param_file": _norm_path(param_file) if param_file else "", + "dem_source_path": _norm_path(dem_source_path) if dem_source_path else "", + "dem_path": _norm_path(dem_path) if dem_path else "", + "dem_crop": dem_crop or {}, "validation": validation or {}, "layout": layout or {}, "command": command_text, diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index 1a9d4aa..72494b8 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -27,6 +27,8 @@ from .orm import ( ScanCursorORM, PathInventoryORM, SourceProductAssetORM, + SourceMetadataDocumentORM, + SARSceneGeometryProfileORM, OrbitAssetORM, SceneOrbitBindingORM, OrbitAssetDerivativeORM, @@ -103,7 +105,8 @@ __all__ = [ "TimeseriesStackPlanORM", "TimeseriesStackPlanItemORM", "TimeseriesStackPlanEdgeORM", "SystemTaskORM", "TaskLogORM", "SystemJobORM", "ScanStateORM", "ManagedRootORM", "ScanCursorORM", "PathInventoryORM", - "SourceProductAssetORM", "OrbitAssetORM", "SceneOrbitBindingORM", + "SourceProductAssetORM", "SourceMetadataDocumentORM", "SARSceneGeometryProfileORM", + "OrbitAssetORM", "SceneOrbitBindingORM", "OrbitAssetDerivativeORM", "AssetInventoryStateORM", "AssetInventoryIssueORM", "WorkflowDefORM", "WorkflowRunORM", "WorkflowStepORM", "WorkflowArtifactORM", "SystemWorkerHeartbeatORM", diff --git a/backend/app/models/orm.py b/backend/app/models/orm.py index 5167eb4..2f173e0 100644 --- a/backend/app/models/orm.py +++ b/backend/app/models/orm.py @@ -3,7 +3,7 @@ SQLAlchemy ORM 模型定义。 所有数据库表对应的 ORM 类均在此文件中定义。 """ from sqlalchemy import ( - Column, Integer, BigInteger, String, Boolean, Float, JSON, Text, + Column, Integer, BigInteger, String, Boolean, Float, JSON, Text, LargeBinary, DateTime, func, ForeignKey, UniqueConstraint, Index, ) from sqlalchemy.orm import relationship @@ -384,7 +384,15 @@ class PairingMetricCacheORM(Base): spatial_baseline_meters = Column(Float, index=True, nullable=True) scene_center_distance_meters = Column(Float, index=True, nullable=True) scene_overlap_ratio = Column(Float, index=True, nullable=True) + pair_aoi_overlap_ratio = Column(Float, nullable=True) orbit_direction = Column(String, index=True, nullable=True) + same_relative_orbit = Column(Boolean, nullable=False, default=False, server_default="false", index=True) + master_relative_orbit = Column(String(64), nullable=True) + slave_relative_orbit = Column(String(64), nullable=True) + dinsar_quality_tier = Column(String(16), nullable=False, default="C", server_default="C", index=True) + dinsar_quality_score = Column(Float, nullable=True) + dinsar_readiness = Column(String(32), nullable=False, default="CANDIDATE", server_default="CANDIDATE", index=True) + dinsar_reasons_json = Column(JSON, nullable=True) same_satellite = Column(Boolean, nullable=False, default=True) same_satellite_family = Column(Boolean, nullable=False, default=True, server_default="true") same_look_direction = Column(Boolean, nullable=False, default=True, server_default="true") @@ -875,6 +883,12 @@ class SourceProductAssetORM(Base): mtime_epoch = Column(Float, nullable=True) checksum_sha256 = Column(String(64), nullable=True) checksum_status = Column(String(32), nullable=False, default="NOT_COMPUTED", server_default="NOT_COMPUTED") + archive_integrity_status = Column(String(32), nullable=False, default="NOT_CHECKED", server_default="NOT_CHECKED", index=True) + archive_integrity_method = Column(String(64), nullable=True) + archive_integrity_checked_at = Column(DateTime, nullable=True) + archive_integrity_error = Column(Text, nullable=True) + archive_integrity_version = Column(String(32), nullable=True) + archive_integrity_member_count = Column(Integer, nullable=True) parser_name = Column(String(64), nullable=True) parser_version = Column(String(32), nullable=True) parse_status = Column(String(32), nullable=False, default="PENDING", server_default="PENDING", index=True) @@ -895,6 +909,84 @@ class SourceProductAssetORM(Base): ) +class SourceMetadataDocumentORM(Base): + __tablename__ = "source_metadata_documents" + + id = Column(Integer, primary_key=True, autoincrement=True) + source_asset_id = Column(Integer, ForeignKey("source_product_assets.id", ondelete="CASCADE"), index=True, nullable=False) + radar_data_id = Column(Integer, ForeignKey("radar_data.id", ondelete="SET NULL"), index=True, nullable=True) + satellite_family = Column(String(32), index=True, nullable=True) + source_format = Column(String(32), index=True, nullable=True) + document_type = Column(String(32), index=True, nullable=False) + member_path = Column(String, nullable=False) + content_sha256 = Column(String(64), index=True, nullable=False) + content_encoding = Column(String(16), nullable=False, default="gzip", server_default="gzip") + content_bytes = Column(LargeBinary, nullable=False) + content_size_bytes = Column(BigInteger, nullable=True) + archive_path = Column(String, nullable=True) + archive_mtime = Column(Float, nullable=True) + parser_version = Column(String(32), nullable=True) + parse_status = Column(String(32), nullable=False, default="OK", server_default="OK", index=True) + parse_error = Column(Text, nullable=True) + extracted_at = Column(DateTime, nullable=True) + created_at = Column(DateTime, server_default=func.now(), nullable=False) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + + source_asset = relationship("SourceProductAssetORM") + radar_data = relationship("RadarDataORM") + + __table_args__ = ( + UniqueConstraint("source_asset_id", "document_type", "member_path", name="uq_source_metadata_document_member"), + Index("idx_source_metadata_documents_asset_type", "source_asset_id", "document_type"), + Index("idx_source_metadata_documents_radar_type", "radar_data_id", "document_type"), + ) + + +class SARSceneGeometryProfileORM(Base): + __tablename__ = "sar_scene_geometry_profiles" + + id = Column(Integer, primary_key=True, autoincrement=True) + source_asset_id = Column(Integer, ForeignKey("source_product_assets.id", ondelete="CASCADE"), unique=True, index=True, nullable=False) + radar_data_id = Column(Integer, ForeignKey("radar_data.id", ondelete="CASCADE"), unique=True, index=True, nullable=True) + satellite_family = Column(String(32), index=True, nullable=True) + satellite = Column(String(32), index=True, nullable=True) + source_format = Column(String(32), index=True, nullable=True) + imaging_mode = Column(String(64), index=True, nullable=True) + polarization = Column(String(64), index=True, nullable=True) + orbit_direction = Column(String(32), index=True, nullable=True) + look_direction = Column(String(32), index=True, nullable=True) + absolute_orbit = Column(String(64), index=True, nullable=True) + relative_orbit = Column(String(64), index=True, nullable=True) + acquisition_start_time_utc = Column(DateTime, index=True, nullable=True) + acquisition_stop_time_utc = Column(DateTime, nullable=True) + scene_center_lon = Column(Float, nullable=True) + scene_center_lat = Column(Float, nullable=True) + footprint_geom = Column(Geometry("POLYGON", srid=4326), index=True, nullable=True) + footprint_polygon = Column(JSON, nullable=True) + swath_summary_json = Column(JSON, nullable=True) + burst_summary_json = Column(JSON, nullable=True) + incidence_angle_min = Column(Float, nullable=True) + incidence_angle_max = Column(Float, nullable=True) + doppler_summary_json = Column(JSON, nullable=True) + state_vector_summary_json = Column(JSON, nullable=True) + metadata_quality = Column(String(32), nullable=False, default="UNKNOWN", server_default="UNKNOWN", index=True) + production_readiness = Column(String(32), nullable=False, default="UNKNOWN", server_default="UNKNOWN", index=True) + readiness_reasons_json = Column(JSON, nullable=True) + parser_version = Column(String(32), nullable=True) + parsed_at = Column(DateTime, nullable=True) + created_at = Column(DateTime, server_default=func.now(), nullable=False) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + + source_asset = relationship("SourceProductAssetORM") + radar_data = relationship("RadarDataORM") + + __table_args__ = ( + Index("idx_sar_scene_geometry_profiles_family_date", "satellite_family", "acquisition_start_time_utc"), + Index("idx_sar_scene_geometry_profiles_track", "satellite_family", "relative_orbit", "orbit_direction"), + Index("idx_sar_scene_geometry_profiles_readiness", "production_readiness", "metadata_quality"), + ) + + class OrbitAssetORM(Base): __tablename__ = "orbit_assets" diff --git a/backend/app/models/schemas.py b/backend/app/models/schemas.py index 716d190..eb8b0c7 100644 --- a/backend/app/models/schemas.py +++ b/backend/app/models/schemas.py @@ -261,10 +261,10 @@ class PairingRequest(BaseModel): """D-InSAR 配对请求的参数模型（增强版 v2.0）""" # === 时空约束（保留） === time_baseline_min: int = Field(default=1, ge=0, le=3650) - time_baseline_max: int = Field(default=90, ge=1, le=3650) + time_baseline_max: int = Field(default=30, ge=1, le=3650) overlap_threshold: float = Field(default=0.5, ge=0.0, le=1.0) - spatial_baseline_max_meters: int = Field(default=3000, ge=0, le=PAIRING_CENTER_DISTANCE_MAX_METERS) - limit_footprint_center_distance: bool = False + spatial_baseline_max_meters: int = Field(default=5000, ge=0, le=PAIRING_CENTER_DISTANCE_MAX_METERS) + limit_footprint_center_distance: bool = True coverage_diversity_penalty: float = Field(default=0.3, ge=0.0, le=1.0) require_same_imaging_mode: bool = True require_same_polarization: bool = True @@ -280,7 +280,7 @@ class PairingRequest(BaseModel): slave_date_to: Optional[str] = Field(default=None, pattern=r'^\d{8}$|^$') # === 配对策略（新增） === - strategy: str = Field(default="all", pattern=r'^(all|sbas|sequential|star)$') + strategy: str = Field(default="dinsar_production", pattern=r'^dinsar_production$') num_connections: int = Field(default=1, ge=1, le=10) reference_image_id: Optional[int] = None @@ -303,6 +303,7 @@ class PairingRequest(BaseModel): normalized['overlap_threshold'] = normalized['pair_footprint_overlap_min_ratio'] if normalized.get('footprint_center_distance_max_meters') not in (None, ''): normalized['spatial_baseline_max_meters'] = normalized['footprint_center_distance_max_meters'] + normalized['strategy'] = 'dinsar_production' return normalized @field_validator( @@ -343,8 +344,15 @@ class PairingRequest(BaseModel): return None if not isinstance(value, list): return value - normalized = [str(item).strip() for item in value if str(item).strip()] - return normalized or None + normalized = [] + for item in value: + compact = str(item).strip().upper().replace("-", "").replace("_", "").replace(" ", "") + if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}: + normalized.append("LT1") + elif compact in {"S1", "S1A", "S1B", "S1C", "SENTINEL1", "SENTINEL1A", "SENTINEL1B", "SENTINEL1C"}: + normalized.append("S1") + deduped = list(dict.fromkeys(normalized)) + return deduped or ["__UNSUPPORTED_DINSAR_FAMILY__"] @field_validator('master_date_to') @classmethod @@ -381,6 +389,16 @@ class RadarPair(BaseModel): time_baseline_days: int spatial_baseline_meters: float scene_center_distance_meters: Optional[float] = None + scene_overlap_ratio: Optional[float] = None + pair_aoi_overlap_ratio: Optional[float] = None + dinsar_quality_tier: Optional[str] = None + dinsar_quality_score: Optional[float] = None + dinsar_readiness: Optional[str] = None + dinsar_reasons: Optional[List[str]] = None + same_relative_orbit: Optional[bool] = None + master_relative_orbit: Optional[str] = None + slave_relative_orbit: Optional[str] = None + production_summary: Optional[Dict[str, Any]] = None class PairingResponse(BaseModel): diff --git a/backend/app/routers/ai.py b/backend/app/routers/ai.py index 4b7262f..ca404e6 100644 --- a/backend/app/routers/ai.py +++ b/backend/app/routers/ai.py @@ -15,7 +15,9 @@ from typing import List, Optional from ..ai_service import ( analyze_map_with_vlm, + get_ollama_models, get_model_info, + is_likely_vlm_model, is_model_trained, predict_quality, train_quality_model, @@ -146,12 +148,12 @@ async def get_ai_status(db: AsyncSession = Depends(get_db)): counts = await dinsar_read_service.get_ai_status_counts(db) ollama_online = False + ollama_models: List[str] = [] + ollama_vlm_models: List[str] = [] try: - import httpx - async with httpx.AsyncClient(timeout=1.0) as client: - ollama_base = settings.OLLAMA_BASE_URL - response = await client.get(f"{ollama_base.rstrip('/')}/api/tags") - ollama_online = response.status_code == 200 + ollama_models = await get_ollama_models(timeout=1.0) + ollama_vlm_models = [model for model in ollama_models if is_likely_vlm_model(model)] + ollama_online = True except Exception: ollama_online = False @@ -161,7 +163,11 @@ async def get_ai_status(db: AsyncSession = Depends(get_db)): "labeled_count": counts["labeled_count"], "good_count": counts["good_count"], "bad_count": counts["bad_count"], - "ollama_online": ollama_online + "ollama_online": ollama_online, + "ollama_models": ollama_models, + "ollama_vlm_models": ollama_vlm_models, + "ollama_base_url": settings.OLLAMA_BASE_URL, + "default_vlm_model": settings.DEFAULT_VLM_MODEL, } diff --git a/backend/app/routers/assets.py b/backend/app/routers/assets.py index 5cfce0d..ab9bb2f 100644 --- a/backend/app/routers/assets.py +++ b/backend/app/routers/assets.py @@ -22,21 +22,29 @@ router = APIRouter(prefix="/assets", tags=["assets"]) class AssetScanRequest(BaseModel): inventory_types: List[str] = Field(default_factory=list) root_ids: List[int] = Field(default_factory=list) + families: List[str] = Field(default_factory=list) bind_orbits: bool = True + build_previews: bool = True + + +class ArchiveIntegrityAuditRequest(BaseModel): + families: List[str] = Field(default_factory=list) + source_formats: List[str] = Field(default_factory=list) + asset_ids: List[int] = Field(default_factory=list) + force: bool = False + limit: Optional[int] = Field(default=None, ge=0) class S1UnpackRequest(BaseModel): target_root: Optional[str] = None overwrite: bool = False min_disk_space_gb: Optional[float] = Field(default=None, ge=0) - delete_archive: Optional[bool] = None class S1BatchUnpackRequest(BaseModel): target_root: Optional[str] = None overwrite: bool = False min_disk_space_gb: Optional[float] = Field(default=None, ge=0) - delete_archive: Optional[bool] = None scan_before_unpack: bool = True @@ -89,7 +97,50 @@ async def run_asset_inventory_scan_now( db, inventory_types=payload.get("inventory_types") or None, root_ids=payload.get("root_ids") or None, + families=payload.get("families") or None, bind_orbits=bool(payload.get("bind_orbits", True)), + build_previews=bool(payload.get("build_previews", True)), + ) + + +@router.post("/inventory/archive-integrity-audit", status_code=202) +async def run_archive_integrity_audit( + request: Optional[ArchiveIntegrityAuditRequest] = None, + admin_user: AuthUserORM = Depends(_require_admin), +): + _ = admin_user + payload = (request or ArchiveIntegrityAuditRequest()).model_dump() + try: + task_id = await task_service.create_task( + "AUDIT_SOURCE_ARCHIVE_INTEGRITY", + "Source archive integrity audit", + params=payload, + ) + job_id = await job_queue_service.create_job( + "AUDIT_SOURCE_ARCHIVE_INTEGRITY", + payload=payload, + task_id=task_id, + ) + return {"message": "Source archive integrity audit queued", "task_id": task_id, "job_id": job_id} + except ValueError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + + +@router.post("/inventory/archive-integrity-audit-now") +async def run_archive_integrity_audit_now( + request: Optional[ArchiveIntegrityAuditRequest] = None, + admin_user: AuthUserORM = Depends(_require_admin), + db: AsyncSession = Depends(get_db), +): + _ = admin_user + payload = (request or ArchiveIntegrityAuditRequest()).model_dump() + return await asset_inventory_service.audit_source_archive_integrity( + db, + families=payload.get("families") or None, + source_formats=payload.get("source_formats") or None, + asset_ids=payload.get("asset_ids") or None, + force=bool(payload.get("force", False)), + limit=payload.get("limit"), ) @@ -186,8 +237,6 @@ async def unpack_sentinel1_source_asset( } if request_data.min_disk_space_gb is not None: payload["min_disk_space_gb"] = request_data.min_disk_space_gb - if request_data.delete_archive is not None: - payload["delete_archive"] = request_data.delete_archive try: task_id = await task_service.create_task( "UNPACK_SENTINEL1", diff --git a/backend/app/routers/dinsar_production.py b/backend/app/routers/dinsar_production.py index 047a348..f266a4a 100644 --- a/backend/app/routers/dinsar_production.py +++ b/backend/app/routers/dinsar_production.py @@ -2,7 +2,9 @@ from __future__ import annotations import asyncio +import os import re +from datetime import datetime from typing import Any, Dict, Literal, Optional from fastapi import APIRouter, Depends, HTTPException @@ -65,6 +67,43 @@ class PreviewInputAssetsRequest(BaseModel): num_to_process: int = Field(default=0, ge=0, description="How many tasks to preview; 0 means all") +def _is_dinsar_task_dir(path: str) -> bool: + return os.path.isdir(os.path.join(path, "master")) and os.path.isdir(os.path.join(path, "slave")) + + +def _is_landsar_task_dir(path: str) -> bool: + return os.path.isdir(os.path.join(path, "Input_Data")) or _is_dinsar_task_dir(path) + + +def _count_candidate_tasks(root_dir: str) -> Dict[str, Any]: + try: + if _is_dinsar_task_dir(root_dir) or _is_landsar_task_dir(root_dir): + return {"task_count": 1, "invalid_child_count": 0, "mode": "single_task"} + task_count = 0 + invalid_child_count = 0 + with os.scandir(root_dir) as entries: + for entry in entries: + if not entry.is_dir() or entry.name.startswith("._"): + continue + child_path = entry.path + if _is_dinsar_task_dir(child_path) or _is_landsar_task_dir(child_path): + task_count += 1 + elif entry.name.lower().startswith("task_"): + invalid_child_count += 1 + return { + "task_count": task_count, + "invalid_child_count": invalid_child_count, + "mode": "task_root", + } + except OSError as exc: + return { + "task_count": 0, + "invalid_child_count": 0, + "mode": "unreadable", + "error": str(exc), + } + + def _get_registry(): from ..dinsar_engines import registry @@ -136,6 +175,56 @@ async def run_wsl_check( return report.to_dict() +@router.get("/task-roots") +async def list_task_roots( + current_user: AuthUserORM = Depends(_get_current_user), +): + _ = current_user + root_dir = os.path.normpath(os.path.abspath(str(settings.DINSAR_TASK_POOL_ROOT or "").strip())) + root_exists = bool(root_dir and os.path.isdir(root_dir)) + items = [] + child_dir_count = 0 + if root_exists: + try: + with os.scandir(root_dir) as entries: + child_dirs = sorted( + [entry for entry in entries if entry.is_dir() and not entry.name.startswith("._")], + key=lambda entry: entry.name.lower(), + ) + except OSError as exc: + raise HTTPException(status_code=500, detail=f"无法读取 D-InSAR Task_Pool: {exc}") from exc + child_dir_count = len(child_dirs) + + for entry in child_dirs: + path = os.path.normpath(entry.path) + stat = entry.stat() + summary = _count_candidate_tasks(path) + task_count = int(summary.get("task_count") or 0) + invalid_child_count = int(summary.get("invalid_child_count") or 0) + items.append( + { + "name": entry.name, + "path": path, + "task_count": task_count, + "invalid_child_count": invalid_child_count, + "valid": task_count > 0, + "mode": summary.get("mode"), + "error": summary.get("error"), + "mtime": stat.st_mtime, + "updated_at": datetime.fromtimestamp(stat.st_mtime).isoformat(timespec="seconds"), + } + ) + + items.sort(key=lambda item: str(item.get("name") or "").lower()) + return { + "root": root_dir, + "root_exists": root_exists, + "items": items, + "count": len(items), + "child_dir_count": child_dir_count, + } + + @router.post("/engines/pyint/preview-input-assets") async def preview_pyint_input_assets( req: PreviewInputAssetsRequest, diff --git a/backend/app/routers/monitor.py b/backend/app/routers/monitor.py index ef78abb..e42ee5e 100644 --- a/backend/app/routers/monitor.py +++ b/backend/app/routers/monitor.py @@ -1,9 +1,12 @@ from __future__ import annotations -from typing import List, Optional +import os +import shutil +from typing import Any, Iterable, List, Optional from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException from pydantic import BaseModel, Field +from sqlalchemy import delete, func from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select @@ -17,6 +20,15 @@ from .dependencies import _require_admin router = APIRouter() MONITOR_ALLOWED_TARGETS = {"radar", "orbit", "dinsar", "gf3"} +MONITOR_SCAN_TASK_TYPES = { + "SCAN_ASSET_INVENTORY", + "AUDIT_SOURCE_ARCHIVE_INTEGRITY", + "SCAN_DATA", + "SCAN_DINSAR", + "GF3_SARSCAPE_SYNC", + "GF3_QUICKLOOK_WEBP", +} +MONITOR_TERMINAL_TASK_STATUSES = {"COMPLETED", "FAILED", "CANCELLED"} MONITOR_LOG_DEFAULT_LIMIT = read_int_env( "MONITOR_LOG_DEFAULT_LIMIT", 50, @@ -37,10 +49,272 @@ MONITOR_LOG_MAX_OFFSET = read_int_env( ) +def _path_kind(path: str) -> str: + text = str(path or "").strip() + if text.startswith("\\\\"): + return "unc" + drive, _tail = os.path.splitdrive(os.path.normpath(text)) + if drive: + return "windows" + if text.startswith("/mnt/"): + return "wsl_mount" + if text.startswith("/"): + return "posix" + return "relative" + + +def _nearest_existing_path(path: str) -> str: + candidate = os.path.normpath(str(path or "").strip()) + while candidate and not os.path.exists(candidate): + parent = os.path.dirname(candidate) + if not parent or parent == candidate: + break + candidate = parent + return candidate + + +def _storage_status(label: str, path: str, role: str) -> dict[str, Any]: + text = str(path or "").strip() + item: dict[str, Any] = { + "label": label, + "role": role, + "path": text, + "path_kind": _path_kind(text), + "exists": bool(text and os.path.exists(text)), + "probe_path": "", + "total_gb": None, + "used_gb": None, + "free_gb": None, + "free_ratio": None, + "status": "missing" if text else "empty", + "message": "", + } + if not text: + item["message"] = "Path is not configured." + return item + if item["path_kind"] == "unc": + item["status"] = "blocked" + item["message"] = "UNC paths are not allowed for active local production." + return item + + probe = _nearest_existing_path(text) + if not probe or not os.path.exists(probe): + item["message"] = "No existing parent path found." + return item + item["probe_path"] = probe + try: + total, used, free = shutil.disk_usage(probe) + except OSError as exc: + item["status"] = "error" + item["message"] = str(exc) + return item + + gb = 1024 ** 3 + free_ratio = float(free) / float(total or 1) + item.update( + { + "total_gb": round(total / gb, 2), + "used_gb": round(used / gb, 2), + "free_gb": round(free / gb, 2), + "free_ratio": round(free_ratio, 4), + "status": "ok", + "message": "", + } + ) + if not item["exists"]: + item["status"] = "missing" + item["message"] = f"Path does not exist; disk usage probed from {probe}." + elif free_ratio < 0.10: + item["status"] = "critical" + item["message"] = "Free space is below 10%." + elif free_ratio < 0.20: + item["status"] = "warning" + item["message"] = "Free space is below 20%." + return item + + +def _add_storage_roots(rows: list[dict[str, str]], label: str, paths: Iterable[str], role: str) -> None: + for path in paths: + text = str(path or "").strip() + if text: + rows.append({"label": label, "path": text, "role": role}) + + +def _storage_group_identity(item: dict[str, Any]) -> tuple[str, str, str]: + path = str(item.get("path") or "").strip() + path_kind = str(item.get("path_kind") or "") + if path_kind == "windows": + base = str(item.get("probe_path") or path).strip() + drive, _tail = os.path.splitdrive(os.path.normpath(base)) + if drive: + drive = drive.upper() + volume_path = f"{drive}\\" + return f"windows:{drive}", volume_path, f"本机磁盘 {volume_path}" + if path_kind == "unc": + key = os.path.normcase(os.path.normpath(path)) + return f"unc:{key}", path, "UNC path" + if path_kind in {"posix", "wsl_mount"}: + base = str(item.get("probe_path") or path or "/").strip() + root = "/mnt/" + base.split("/")[2] if base.startswith("/mnt/") and len(base.split("/")) > 2 else "/" + return f"{path_kind}:{root}", root, root + key = os.path.normcase(os.path.normpath(path)) + return f"path:{key}", path, str(item.get("label") or item.get("role") or path or "Storage") + + +def _capacity_status(free_ratio: Any) -> str: + if free_ratio is None: + return "missing" + try: + ratio = float(free_ratio) + except (TypeError, ValueError): + return "missing" + if ratio < 0.10: + return "critical" + if ratio < 0.20: + return "warning" + return "ok" + + +def _group_storage_statuses(items: list[dict[str, Any]]) -> list[dict[str, Any]]: + groups: dict[str, dict[str, Any]] = {} + order: list[str] = [] + + for item in items: + group_key, volume_path, volume_label = _storage_group_identity(item) + group = groups.get(group_key) + if group is None: + group = { + "label": volume_label, + "role": "storage_volume", + "path": volume_path, + "path_kind": item.get("path_kind"), + "exists": False, + "probe_path": item.get("probe_path") or "", + "total_gb": None, + "used_gb": None, + "free_gb": None, + "free_ratio": None, + "status": "missing", + "message": "", + "paths": [], + "configured_path_count": 0, + "existing_path_count": 0, + "missing_path_count": 0, + "blocked_path_count": 0, + "error_path_count": 0, + } + groups[group_key] = group + order.append(group_key) + + group["configured_path_count"] += 1 + if item.get("exists"): + group["existing_path_count"] += 1 + if item.get("status") == "missing": + group["missing_path_count"] += 1 + elif item.get("status") == "blocked": + group["blocked_path_count"] += 1 + elif item.get("status") == "error": + group["error_path_count"] += 1 + + group["paths"].append( + { + "label": item.get("label"), + "role": item.get("role"), + "path": item.get("path"), + "exists": item.get("exists"), + "status": item.get("status"), + "message": item.get("message"), + } + ) + + if item.get("total_gb") is not None: + group["exists"] = True + group["probe_path"] = item.get("probe_path") or group["probe_path"] + group["total_gb"] = item.get("total_gb") + group["used_gb"] = item.get("used_gb") + group["free_gb"] = item.get("free_gb") + group["free_ratio"] = item.get("free_ratio") + + result: list[dict[str, Any]] = [] + for key in order: + group = groups[key] + status = _capacity_status(group.get("free_ratio")) + if group["blocked_path_count"]: + status = "blocked" + elif group["error_path_count"]: + status = "error" + elif group["missing_path_count"] and status == "ok": + status = "partial" + group["status"] = status + + messages = [f"{group['configured_path_count']} 个配置路径"] + if group["missing_path_count"]: + messages.append(f"{group['missing_path_count']} 个路径缺失") + if group["blocked_path_count"]: + messages.append(f"{group['blocked_path_count']} 个 UNC 被禁用") + if group["error_path_count"]: + messages.append(f"{group['error_path_count']} 个探测失败") + if status == "critical": + messages.append("剩余空间低于 10%") + elif status == "warning" and not group["missing_path_count"]: + messages.append("剩余空间低于 20%") + group["message"] = "；".join(messages) + "。" + result.append(group) + return result + + +def _collect_storage_roots() -> list[dict[str, Any]]: + rows: list[dict[str, str]] = [] + _add_storage_roots(rows, "Task_Pool", [settings.TASK_POOL_ROOT], "task_pool") + _add_storage_roots(rows, "D-InSAR Task_Pool", [settings.DINSAR_TASK_POOL_ROOT], "dinsar_task_pool") + _add_storage_roots(rows, "SBAS Task_Pool", [settings.SBAS_TASK_POOL_ROOT], "sbas_task_pool") + _add_storage_roots(rows, "Data distribution root", [settings.DATA_DISTRIBUTION_ROOT], "data_distribution") + source_paths = split_env_paths(settings.SOURCE_PRODUCT_DIRS) + source_path_keys = {os.path.normcase(os.path.normpath(path)) for path in source_paths} + s1_storage_paths = [ + path + for path in split_env_paths(settings.SENTINEL1_STORAGE_DIRS) + if os.path.normcase(os.path.normpath(path)) not in source_path_keys + ] + _add_storage_roots(rows, "LT/S1 local source pools", source_paths, "source_local") + _add_storage_roots(rows, "Sentinel-1 local source pool", s1_storage_paths, "source_local") + _add_storage_roots(rows, "Orbit source pool", split_env_paths(settings.ORBIT_SOURCE_DIRS), "orbit_source") + _add_storage_roots(rows, "LT-1 radar scan pool", split_env_paths(settings.MONITOR_RADAR_DIRS), "lt1_storage") + _add_storage_roots(rows, "D-InSAR product root", [settings.DINSAR_PRODUCT_DIR], "dinsar_product") + _add_storage_roots(rows, "GF3 native _geo", split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS), "gf3_native") + _add_storage_roots(rows, "GF3 standard/index", split_env_paths(settings.GF3_STORAGE_DIRS), "gf3_storage") + _add_storage_roots(rows, "GF3 task/runtime pool", [settings.GF3_TASK_POOL_ROOT], "gf3_task_pool") + _add_storage_roots(rows, "GF3 SARscape runtime", [settings.GF3_SARSCAPE_RUNTIME_DIR], "gf3_runtime") + _add_storage_roots(rows, "Result publish root", [settings.RESULT_PUBLISH_ROOT], "result_publish") + _add_storage_roots(rows, "SBAS work root", [settings.GAMMA_SBAS_WORK_ROOT], "sbas_work") + _add_storage_roots(rows, "SBAS product root", [settings.GAMMA_SBAS_PRODUCT_ROOT], "sbas_product") + _add_storage_roots(rows, "SAR analysis ready", [settings.SAR_ANALYSIS_READY_ROOT], "analysis_ready") + _add_storage_roots(rows, "SAR analysis work", [settings.SAR_ANALYSIS_WORK_ROOT], "analysis_work") + + seen: set[str] = set() + result: list[dict[str, Any]] = [] + for row in rows: + key = os.path.normcase(os.path.normpath(row["path"])) + if key in seen: + continue + seen.add(key) + result.append(_storage_status(row["label"], row["path"], row["role"])) + return _group_storage_statuses(result) + + class MonitorConfig(BaseModel): radar_dirs: List[str] = [] orbit_dir: Optional[str] = None + orbit_source_dirs: List[str] = [] + orbit_production_txt_pool: Optional[str] = None dinsar_dirs: List[str] = [] + dinsar_product_dir: Optional[str] = None + sbas_product_root: Optional[str] = None + task_pool_root: Optional[str] = None + dinsar_task_pool_root: Optional[str] = None + sbas_task_pool_root: Optional[str] = None + gf3_task_pool_root: Optional[str] = None + data_distribution_root: Optional[str] = None gf3_archive_source_dirs: List[str] = [] gf3_source_dirs: List[str] = [] gf3_legacy_gdal_enabled: bool = False @@ -53,6 +327,7 @@ class MonitorConfig(BaseModel): gf3_sarscape_polarizations: Optional[str] = None gf3_sarscape_auto_standardize: bool = True gf3_sarscape_clean_after_success: bool = True + storage_roots: List[dict[str, Any]] = [] # Manual-only: config is read from .env @@ -70,6 +345,13 @@ class GF3UnpackRunRequest(BaseModel): class GF3SarscapeSyncRequest(BaseModel): force: bool = False register: bool = True + quicklook_only: bool = False + native_dirs: List[str] = [] + + +class GF3QuicklookWebpRequest(BaseModel): + force: bool = False + max_records: Optional[int] = Field(default=None, ge=0) class GF3SarscapeProduceRequest(BaseModel): @@ -91,7 +373,7 @@ class GF3SarscapeCleanRequest(BaseModel): @router.post("/monitor/config") async def update_monitor_config(config: MonitorConfig): """ - 更新数据监控配置（雷达、精轨和Dinsar结果）。 + 鏇存柊鏁版嵁鐩戞帶閰嶇疆锛堥浄杈俱€佺簿杞ㄥ拰Dinsar缁撴灉锛夈€? """ raise HTTPException( status_code=403, @@ -102,8 +384,8 @@ async def update_monitor_config(config: MonitorConfig): @router.post("/monitor/run-now") async def run_monitor_now(target: Optional[str] = None, background_tasks: BackgroundTasks = None, admin_user: AuthUserORM = Depends(_require_admin)): """ - 手动触发一次监控任务（扫描所有配置的目录）。 - target: 'radar', 'orbit', 'dinsar' 或 None (全部) + 鎵嬪姩瑙﹀彂涓€娆＄洃鎺т换鍔★紙鎵弿鎵€鏈夐厤缃殑鐩綍锛夈€? + target: 'radar', 'orbit', 'dinsar' 鎴?None (鍏ㄩ儴) """ normalized_target = (target or "").strip().lower() or None if normalized_target and normalized_target not in MONITOR_ALLOWED_TARGETS: @@ -127,7 +409,7 @@ async def run_monitor_now(target: Optional[str] = None, background_tasks: Backgr raise HTTPException(status_code=400, detail="Monitor paths are not configured in .env.") task_type = "SCAN_DATA" if normalized_target in ["radar", "orbit", "gf3", None] else "SCAN_DINSAR" - task_name = f"手动触发扫描 ({normalized_target or '全部'})" + task_name = f"Manual scan ({normalized_target or 'all'})" try: task_id = await task_service.create_task(task_type, task_name) @@ -135,7 +417,7 @@ async def run_monitor_now(target: Optional[str] = None, background_tasks: Backgr payload = {"dirs": dinsar_dirs} job_type = "SCAN_DINSAR" elif normalized_target == "gf3": - # GF3 扫描：用 gf3_storage_dirs 作为 radar_dirs + # GF3 scan uses gf3_storage_dirs as radar_dirs. payload = { "radar_dirs": gf3_storage_dirs, "target": "radar", @@ -150,7 +432,7 @@ async def run_monitor_now(target: Optional[str] = None, background_tasks: Backgr await job_queue_service.create_job(job_type, payload=payload, task_id=task_id) return { - "message": f"已触发{normalized_target or '全部'}手动扫描任务（已进入队列）", + "message": f"Manual scan queued for {normalized_target or 'all'}.", "task_id": task_id } except ValueError as e: @@ -160,14 +442,14 @@ async def run_monitor_now(target: Optional[str] = None, background_tasks: Backgr @router.post("/monitor/gf3-process") async def run_gf3_batch_process(admin_user: AuthUserORM = Depends(_require_admin)): """ - 批量 GF3 L1A→L2 处理：扫描 GF3_SOURCE_DIRS，过滤已处理的，逐个处理并自动入库。 + Batch GF3 legacy L1A to L2 processing. """ if not settings.GF3_LEGACY_GDAL_ENABLED: raise HTTPException( status_code=409, detail=( "Legacy GF3 Python/GDAL preprocessing is disabled. " - "Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly." + "Use GF3 native _geo result registration or set GF3_LEGACY_GDAL_ENABLED=true explicitly." ), ) gf3_source_dirs = MONITOR_CONFIG.get("gf3_source_dirs") or [] @@ -175,7 +457,7 @@ async def run_gf3_batch_process(admin_user: AuthUserORM = Depends(_require_admin raise HTTPException(status_code=400, detail="GF3_SOURCE_DIRS is not configured.") task_type = "GF3_BATCH_PROCESS" - task_name = "GF3 批量 L1A→L2 处理" + task_name = "GF3 legacy batch process" try: task_id = await task_service.create_task(task_type, task_name) @@ -185,7 +467,7 @@ async def run_gf3_batch_process(admin_user: AuthUserORM = Depends(_require_admin task_id=task_id, ) return { - "message": "GF3 批量处理任务已提交", + "message": "GF3 batch process task submitted", "task_id": task_id, } except ValueError as e: @@ -198,147 +480,100 @@ async def run_gf3_sarscape_sync( admin_user: AuthUserORM = Depends(_require_admin), ): """ - 扫描 GF3 SARscape 原生 _geo 二进制池，转换为标准 GeoTIFF，并登记入库。 + Scan GF3 SARscape native _geo outputs. """ - gf3_native_dirs = MONITOR_CONFIG.get("gf3_sarscape_native_dirs") or [] + options = request_data or GF3SarscapeSyncRequest() + gf3_native_dirs = options.native_dirs or MONITOR_CONFIG.get("gf3_sarscape_native_dirs") or [] gf3_storage_dirs = MONITOR_CONFIG.get("gf3_storage_dirs") or [] if not gf3_native_dirs: raise HTTPException(status_code=400, detail="GF3_SARSCAPE_NATIVE_DIRS is not configured.") if not gf3_storage_dirs: raise HTTPException(status_code=400, detail="GF3_STORAGE_DIRS is not configured.") - options = request_data or GF3SarscapeSyncRequest() task_type = "GF3_SARSCAPE_SYNC" - task_name = "GF3 SARscape 原生结果标准化" + task_name = "GF3 SARscape native result inventory" if options.quicklook_only else "GF3 SARscape native standardize" payload = { "native_dirs": gf3_native_dirs, "storage_root": gf3_storage_dirs[0], "force": bool(options.force), "register": bool(options.register), + "quicklook_only": bool(options.quicklook_only), } try: task_id = await task_service.create_task(task_type, task_name, params=payload) await job_queue_service.create_job(task_type, payload=payload, task_id=task_id) return { - "message": "GF3 SARscape 原生结果标准化任务已提交", + "message": ( + "GF3 SARscape native result inventory task submitted" + if options.quicklook_only + else "GF3 SARscape native standardize task submitted" + ), "task_id": task_id, } except ValueError as e: raise HTTPException(status_code=409, detail=str(e)) +@router.post("/monitor/gf3-quicklook-webp", status_code=202) +async def run_gf3_quicklook_webp( + request_data: GF3QuicklookWebpRequest | None = None, + admin_user: AuthUserORM = Depends(_require_admin), +): + """ + Generate local WebP cache files from registered GF3 SARscape native _geo records. + """ + options = request_data or GF3QuicklookWebpRequest() + task_type = "GF3_QUICKLOOK_WEBP" + task_name = "GF3 native _geo WebP cache" + gf3_native_dirs = MONITOR_CONFIG.get("gf3_sarscape_native_dirs") or [] + if not gf3_native_dirs: + raise HTTPException(status_code=400, detail="GF3_SARSCAPE_NATIVE_DIRS is not configured.") + payload = { + "force": bool(options.force), + "max_records": int(options.max_records or 0), + "native_dirs": gf3_native_dirs, + } + + try: + task_id = await task_service.create_task(task_type, task_name, params=payload) + await job_queue_service.create_job(task_type, payload=payload, task_id=task_id) + return { + "message": "GF3 native _geo WebP cache task submitted", + "task_id": task_id, + } + except ValueError as e: + raise HTTPException(status_code=409, detail=str(e)) + + + @router.post("/monitor/gf3-sarscape-produce", status_code=202) async def run_gf3_sarscape_produce( request_data: GF3SarscapeProduceRequest | None = None, admin_user: AuthUserORM = Depends(_require_admin), ): """ - Run GF3 raw archives through the SARscape wrapper, standardize outputs, and optionally clean intermediates. + GF3 SARscape production is disabled on this management machine. """ - gf3_archive_source_dirs = MONITOR_CONFIG.get("gf3_archive_source_dirs") or [] - gf3_native_dirs = MONITOR_CONFIG.get("gf3_sarscape_native_dirs") or [] - gf3_storage_dirs = MONITOR_CONFIG.get("gf3_storage_dirs") or [] - if not gf3_archive_source_dirs: - raise HTTPException(status_code=400, detail="GF3_ARCHIVE_SOURCE_DIRS is not configured.") - if not gf3_native_dirs: - raise HTTPException(status_code=400, detail="GF3_SARSCAPE_NATIVE_DIRS is not configured.") - if not gf3_storage_dirs: - raise HTTPException(status_code=400, detail="GF3_STORAGE_DIRS is not configured.") - if not settings.GF3_SARSCAPE_WRAPPER_EXE: - raise HTTPException(status_code=400, detail="GF3_SARSCAPE_WRAPPER_EXE is not configured.") - if not (settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH): - raise HTTPException(status_code=400, detail="GF3_SARSCAPE_DEM_PATH or GF3_GEO_DEM_PATH is not configured.") - - options = request_data or GF3SarscapeProduceRequest() - selected_dates = [] - for raw_date in options.selected_dates or []: - text = str(raw_date or "").strip() - if not text: - continue - normalized = text.replace("-", "").replace("_", "") - if len(normalized) != 8 or not normalized.isdigit(): - raise HTTPException(status_code=400, detail=f"Invalid GF3 scene date: {raw_date}") - if normalized not in selected_dates: - selected_dates.append(normalized) - task_type = "GF3_SARSCAPE_PRODUCE" - task_name = "GF3 SARscape production" - auto_standardize = settings.GF3_SARSCAPE_AUTO_STANDARDIZE if options.auto_standardize is None else bool(options.auto_standardize) - clean_after_success = settings.GF3_SARSCAPE_CLEAN_AFTER_SUCCESS if options.clean_after_success is None else bool(options.clean_after_success) - payload = { - "source_dirs": gf3_archive_source_dirs, - "native_dirs": gf3_native_dirs, - "native_root": gf3_native_dirs[0], - "storage_root": gf3_storage_dirs[0], - "wrapper_exe": settings.GF3_SARSCAPE_WRAPPER_EXE, - "idlrt_path": settings.GF3_SARSCAPE_IDLRT_PATH, - "dem_path": settings.GF3_SARSCAPE_DEM_PATH or settings.GF3_GEO_DEM_PATH, - "polarizations": settings.GF3_SARSCAPE_POLARIZATIONS, - "archive_exts": split_env_paths(settings.GF3_ARCHIVE_EXTS), - "max_archives_per_run": int(options.max_archives_per_run or 0), - "selected_dates": selected_dates, - "local_staging_root": settings.GF3_TASK_POOL_ROOT, - "timeout_seconds": int(settings.GF3_SARSCAPE_PRODUCE_TIMEOUT_SECONDS or 0), - "keep_extracted": bool(settings.GF3_SARSCAPE_KEEP_EXTRACTED), - "auto_standardize": bool(auto_standardize), - "clean_after_success": bool(clean_after_success), - "force_standardize": bool(options.force_standardize), - "register": bool(options.register), - "cleanup_require_standardized": True, - "cleanup_dry_run": bool(options.cleanup_dry_run), - } - - try: - task_id = await task_service.create_task(task_type, task_name, params=payload) - await job_queue_service.create_job(task_type, payload=payload, task_id=task_id) - return { - "message": ( - f"GF3 SARscape production task submitted for {', '.join(selected_dates)}" - if selected_dates - else "GF3 SARscape production task submitted" - ), - "task_id": task_id, - } - except ValueError as e: - raise HTTPException(status_code=409, detail=str(e)) + raise HTTPException( + status_code=409, + detail=( + "GF3 SARscape production is disabled on this management machine. " + "Run GF3 production on the SARscape host, copy completed _geo results to local " + "GF3_SARSCAPE_NATIVE_DIRS, then use GF3 native result registration and GF3 _geo WebP generation." + ), + ) @router.get("/monitor/gf3-sarscape-dates") async def list_gf3_sarscape_dates(admin_user: AuthUserORM = Depends(_require_admin)): """ - List available GF3 SARscape source dates from configured raw archive roots. + GF3 SARscape production date selection is disabled on this management machine. """ - from ..services.gf3_sarscape_production_service import discover_gf3_sarscape_inputs - - gf3_archive_source_dirs = MONITOR_CONFIG.get("gf3_archive_source_dirs") or [] - if not gf3_archive_source_dirs: - raise HTTPException(status_code=400, detail="GF3_ARCHIVE_SOURCE_DIRS is not configured.") - - discovery = discover_gf3_sarscape_inputs( - gf3_archive_source_dirs, - archive_exts=split_env_paths(settings.GF3_ARCHIVE_EXTS), + raise HTTPException( + status_code=409, + detail="GF3 SARscape production date selection is disabled. Register local _geo native results instead.", ) - by_date: dict[str, dict[str, object]] = {} - undated = 0 - for item in discovery.get("inputs") or []: - scene_name = str(item.get("scene_name") or "") - date_text = str(item.get("scene_date") or "") - if not date_text: - undated += 1 - continue - bucket = by_date.setdefault(date_text, {"date": date_text, "scene_count": 0, "scenes": []}) - bucket["scene_count"] = int(bucket.get("scene_count") or 0) + 1 - scenes = bucket.get("scenes") - if isinstance(scenes, list) and len(scenes) < 20: - scenes.append(scene_name) - - dates = sorted(by_date.values(), key=lambda item: str(item.get("date") or ""), reverse=True) - return { - "dates": dates, - "input_count": discovery.get("input_count") or 0, - "undated_count": undated, - "missing_roots": discovery.get("missing_roots") or [], - } @router.post("/monitor/gf3-sarscape-clean", status_code=202) @@ -394,14 +629,14 @@ async def run_gf3_unpack( admin_user: AuthUserORM = Depends(_require_admin), ): """ - 将 GF3 压缩包池解包到 GF3_SOURCE_DIRS，作为后续 L1A→L2 预处理输入。 + Unpack GF3 archives into GF3_SOURCE_DIRS for the legacy pipeline. """ if not settings.GF3_LEGACY_GDAL_ENABLED: raise HTTPException( status_code=409, detail=( "Legacy GF3 archive unpack is disabled. " - "Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly." + "Use GF3 native _geo result registration or set GF3_LEGACY_GDAL_ENABLED=true explicitly." ), ) gf3_archive_source_dirs = MONITOR_CONFIG.get("gf3_archive_source_dirs") or [] @@ -416,7 +651,7 @@ async def run_gf3_unpack( max_files = max(0, int(request_data.max_files_per_run)) task_type = "GF3_UNPACK" - task_name = "GF3 压缩包解包" + task_name = "GF3 archive unpack" payload = { "source_dirs": gf3_archive_source_dirs, "target_dirs": gf3_source_dirs, @@ -429,7 +664,7 @@ async def run_gf3_unpack( task_id = await task_service.create_task(task_type, task_name, params=payload) await job_queue_service.create_job(task_type, payload=payload, task_id=task_id) return { - "message": "GF3 解包任务已提交", + "message": "GF3 unpack task submitted", "task_id": task_id, } except ValueError as e: @@ -439,9 +674,24 @@ async def run_gf3_unpack( @router.get("/monitor/status") async def get_monitor_status(): """ - 获取当前监控状态。 + 鑾峰彇褰撳墠鐩戞帶鐘舵€併€? """ - return MONITOR_CONFIG + config = dict(MONITOR_CONFIG) + config.update( + { + "task_pool_root": settings.TASK_POOL_ROOT, + "dinsar_task_pool_root": settings.DINSAR_TASK_POOL_ROOT, + "sbas_task_pool_root": settings.SBAS_TASK_POOL_ROOT, + "gf3_task_pool_root": settings.GF3_TASK_POOL_ROOT, + "data_distribution_root": settings.DATA_DISTRIBUTION_ROOT, + "dinsar_product_dir": settings.DINSAR_PRODUCT_DIR, + "sbas_product_root": settings.GAMMA_SBAS_PRODUCT_ROOT, + "orbit_source_dirs": split_env_paths(settings.ORBIT_SOURCE_DIRS) or split_env_paths(settings.MONITOR_ORBIT_DIR), + "orbit_production_txt_pool": settings.ORBIT_POOL_ENVI, + "storage_roots": _collect_storage_roots(), + } + ) + return config @router.get("/monitor/logs") @@ -451,7 +701,7 @@ async def get_monitor_logs( db: AsyncSession = Depends(get_db), ): """ - 获取最新的监控日志 (来自任务日志表)。 + 鑾峰彇鏈€鏂扮殑鐩戞帶鏃ュ織 (鏉ヨ嚜浠诲姟鏃ュ織琛?銆? """ safe_limit = min(MONITOR_LOG_MAX_LIMIT, max(1, int(limit or MONITOR_LOG_DEFAULT_LIMIT))) safe_offset = min(MONITOR_LOG_MAX_OFFSET, max(0, int(offset or 0))) @@ -478,3 +728,42 @@ async def get_monitor_logs( "count": len(logs), "logs": logs, } + + +@router.delete("/monitor/scan-task-history") +async def clear_monitor_scan_task_history( + admin_user: AuthUserORM = Depends(_require_admin), + db: AsyncSession = Depends(get_db), +): + """ + Clear finished scan task records and their logs from the monitor panel. + + Running and pending tasks are intentionally preserved. + """ + task_rows = await db.execute( + select(SystemTaskORM.task_id) + .where(SystemTaskORM.task_type.in_(MONITOR_SCAN_TASK_TYPES)) + .where(SystemTaskORM.status.in_(MONITOR_TERMINAL_TASK_STATUSES)) + ) + task_ids = [str(task_id) for task_id in task_rows.scalars().all() if task_id] + if not task_ids: + return { + "deleted_task_count": 0, + "deleted_log_count": 0, + "preserved_active": True, + } + + log_count_result = await db.execute( + select(func.count(TaskLogORM.id)).where(TaskLogORM.task_id.in_(task_ids)) + ) + deleted_log_count = int(log_count_result.scalar_one() or 0) + + await db.execute(delete(TaskLogORM).where(TaskLogORM.task_id.in_(task_ids))) + task_delete_result = await db.execute(delete(SystemTaskORM).where(SystemTaskORM.task_id.in_(task_ids))) + await db.commit() + + return { + "deleted_task_count": int(task_delete_result.rowcount or len(task_ids)), + "deleted_log_count": deleted_log_count, + "preserved_active": True, + } diff --git a/backend/app/routers/orbit.py b/backend/app/routers/orbit.py index 9d190b4..5cf6965 100644 --- a/backend/app/routers/orbit.py +++ b/backend/app/routers/orbit.py @@ -35,6 +35,8 @@ class OrbitPoolActionRequest(BaseModel): async def _build_orbit_database_stats( db: AsyncSession, pool_inventory: Dict[str, Any], + *, + isce2_enabled: bool = False, ) -> Dict[str, Any]: total_radar_count = ( await db.execute(select(func.count(RadarDataORM.id))) @@ -94,9 +96,9 @@ async def _build_orbit_database_stats( } envi_stems = set(pool_inventory["envi"]["files"].keys()) - isce2_stems = set(pool_inventory["isce2"]["files"].keys()) + isce2_stems = set(pool_inventory["isce2"]["files"].keys()) if isce2_enabled else set() missing_in_envi = sorted(db_expected_stems - envi_stems) - missing_in_isce2 = sorted(db_expected_stems - isce2_stems) + missing_in_isce2 = sorted(db_expected_stems - isce2_stems) if isce2_enabled else [] return { "total_radar_count": int(total_radar_count or 0), @@ -110,6 +112,7 @@ async def _build_orbit_database_stats( "db_expected_stem_count": len(db_expected_stems), "stems_missing_in_envi_count": len(missing_in_envi), "stems_missing_in_isce2_count": len(missing_in_isce2), + "isce2_enabled": bool(isce2_enabled), "sample_missing_in_envi": missing_in_envi[:20], "sample_missing_in_isce2": missing_in_isce2[:20], "path_errors": db_path_errors, @@ -123,26 +126,27 @@ async def get_orbit_status( ): """Return source, pool, consistency, and database orbit status.""" source_dir = settings.MONITOR_ORBIT_DIR + isce2_pool = settings.ORBIT_POOL_ISCE2 if settings.ISCE2_ENABLED else "" source_stats = await asyncio.to_thread(scan_orbit_dir, source_dir) pool_inventory = await asyncio.to_thread( get_orbit_pool_inventory, settings.ORBIT_POOL_ENVI, - settings.ORBIT_POOL_ISCE2, + isce2_pool, True, ) consistency = await asyncio.to_thread( check_orbit_consistency, settings.ORBIT_POOL_ENVI, - settings.ORBIT_POOL_ISCE2, + isce2_pool, ) source_gap_summary = await asyncio.to_thread( summarize_source_orbit_gaps, source_dir, settings.ORBIT_POOL_ENVI, - settings.ORBIT_POOL_ISCE2, + isce2_pool, settings.ORBIT_QUARANTINE_DIR, ) - database_stats = await _build_orbit_database_stats(db, pool_inventory) + database_stats = await _build_orbit_database_stats(db, pool_inventory, isce2_enabled=bool(settings.ISCE2_ENABLED)) return { "orbit_root": source_stats.orbit_root, @@ -184,7 +188,8 @@ async def get_orbit_status( "errors": pool_inventory["envi"]["errors"], }, "isce2": { - "path": settings.ORBIT_POOL_ISCE2, + "path": isce2_pool, + "enabled": bool(settings.ISCE2_ENABLED), "total": pool_inventory["isce2"]["total"], "duplicate_count": pool_inventory["isce2"]["duplicate_count"], "errors": pool_inventory["isce2"]["errors"], @@ -207,25 +212,28 @@ async def sync_orbit_pool_action( Check pool consistency, or repair missing entries when repair=true. """ if payload and payload.quarantine_bad: + isce2_pool = settings.ORBIT_POOL_ISCE2 if settings.ISCE2_ENABLED else "" return await asyncio.to_thread( quarantine_bad_orbits, settings.MONITOR_ORBIT_DIR, settings.ORBIT_POOL_ENVI, - settings.ORBIT_POOL_ISCE2, + isce2_pool, settings.ORBIT_QUARANTINE_DIR, ) if payload and payload.repair: + isce2_pool = settings.ORBIT_POOL_ISCE2 if settings.ISCE2_ENABLED else "" return await asyncio.to_thread( repair_orbit_pools, settings.MONITOR_ORBIT_DIR, settings.ORBIT_POOL_ENVI, - settings.ORBIT_POOL_ISCE2, + isce2_pool, settings.ORBIT_POOL_LANDSAR, ) + isce2_pool = settings.ORBIT_POOL_ISCE2 if settings.ISCE2_ENABLED else "" return await asyncio.to_thread( check_orbit_consistency, settings.ORBIT_POOL_ENVI, - settings.ORBIT_POOL_ISCE2, + isce2_pool, ) diff --git a/backend/app/routers/pairing.py b/backend/app/routers/pairing.py index d4b6541..b9d04d6 100644 --- a/backend/app/routers/pairing.py +++ b/backend/app/routers/pairing.py @@ -40,10 +40,10 @@ router = APIRouter() def get_pairing_request_from_form( time_baseline_min: int = Form(1), - time_baseline_max: int = Form(90), + time_baseline_max: int = Form(30), overlap_threshold: float = Form(0.5), - spatial_baseline_max_meters: int = Form(3000), - limit_footprint_center_distance: bool = Form(False), + spatial_baseline_max_meters: int = Form(5000), + limit_footprint_center_distance: bool = Form(True), max_temporal_baseline_days: Optional[int] = Form(None), pair_footprint_overlap_min_ratio: Optional[float] = Form(None), footprint_center_distance_max_meters: Optional[int] = Form(None), @@ -57,7 +57,7 @@ def get_pairing_request_from_form( master_date_to: Optional[str] = Form(None), slave_date_from: Optional[str] = Form(None), slave_date_to: Optional[str] = Form(None), - strategy: str = Form("all"), + strategy: str = Form("dinsar_production"), num_connections: int = Form(1), reference_image_id: Optional[int] = Form(None), allowed_satellites: Optional[str] = Form(None), # JSON string @@ -91,7 +91,7 @@ def get_pairing_request_from_form( master_date_to=master_date_to, slave_date_from=slave_date_from, slave_date_to=slave_date_to, - strategy=strategy, + strategy="dinsar_production", num_connections=num_connections, reference_image_id=reference_image_id, allowed_satellites=satellites_list, diff --git a/backend/app/routers/radar.py b/backend/app/routers/radar.py index 7b8ffca..df559a4 100644 --- a/backend/app/routers/radar.py +++ b/backend/app/routers/radar.py @@ -41,6 +41,7 @@ from .dependencies import ( router = APIRouter() logger = logging.getLogger(__name__) +GF3_NATIVE_PREVIEW_SOURCE_FORMAT = "GF3_SARSCAPE_NATIVE_PREVIEW" LIST_QUERY_MAX_LIMIT = read_int_env( "LIST_QUERY_MAX_LIMIT", 2000, @@ -188,6 +189,38 @@ def _radar_preview_paths(record: RadarDataORM) -> Tuple[str, str]: return raw_cache_path, geo_cache_path +def _is_gf3_native_preview_record(record: RadarDataORM) -> bool: + return str(record.source_format or "") == GF3_NATIVE_PREVIEW_SOURCE_FORMAT + + +def _build_gf3_native_preview_status(record: RadarDataORM) -> RadarPreviewStatusInfo: + preview_path = str(record.preview_cache_path or "") + has_native_cache = ( + (record.preview_cache_status or "NONE") == "READY" + and preview_path.lower().endswith(".webp") + and os.path.exists(preview_path) + ) + metadata = record.metadata_json or {} + source_path = str(metadata.get("default_native_path") or "") + source_found = bool(source_path and os.path.exists(source_path)) + return RadarPreviewStatusInfo( + radar_id=record.id, + status="READY" if has_native_cache else (record.preview_cache_status or "NONE"), + cache_version=record.preview_cache_version, + cache_updated_at=record.preview_cache_updated_at, + has_geo_cache=has_native_cache, + has_raw_cache=has_native_cache, + source_found=source_found, + fallback_in_use=False, + message=( + "GF3 native _geo WebP cache is available." + if has_native_cache + else "GF3 native _geo WebP cache has not been generated." + ), + error=None if has_native_cache else record.preview_cache_error, + ) + + def _build_radar_preview_status( record: RadarDataORM, source_found: bool, @@ -228,6 +261,9 @@ async def _build_radar_preview_cache( db: AsyncSession, force: bool = False, ) -> RadarPreviewStatusInfo: + if _is_gf3_native_preview_record(record): + return _build_gf3_native_preview_status(record) + raw_cache_path, geo_cache_path = _radar_preview_paths(record) has_geo_cache = os.path.exists(geo_cache_path) has_raw_cache = os.path.exists(raw_cache_path) @@ -355,7 +391,33 @@ async def _get_cached_radar_preview(data_id: int, db: AsyncSession): if not record: raise HTTPException(status_code=404, detail=f"ID为 {data_id} 的源数据不存在。") + if _is_gf3_native_preview_record(record): + preview_path = str(record.preview_cache_path or "") + if ( + (record.preview_cache_status or "NONE") == "READY" + and preview_path.lower().endswith(".webp") + and os.path.exists(preview_path) + ): + return FileResponse( + preview_path, + media_type="image/webp", + headers={"Cache-Control": "public, max-age=31536000"}, + ) + raise HTTPException(status_code=404, detail="GF3 native _geo WebP cache has not been generated.") + raw_cache_path, geo_cache_path = _radar_preview_paths(record) + if ( + (record.preview_cache_status or "NONE") == "READY" + and record.preview_cache_path + and str(record.preview_cache_path).lower().endswith(".webp") + and os.path.exists(record.preview_cache_path) + ): + return FileResponse( + record.preview_cache_path, + media_type="image/webp", + headers={"Cache-Control": "public, max-age=31536000"}, + ) + if os.path.exists(geo_cache_path): return FileResponse( geo_cache_path, @@ -664,6 +726,9 @@ async def get_radar_preview_status_endpoint(data_id: int, db: AsyncSession = Dep if not record: raise HTTPException(status_code=404, detail=f"ID为 {data_id} 的源数据不存在。") + if _is_gf3_native_preview_record(record): + return _build_gf3_native_preview_status(record) + raw_cache_path, geo_cache_path = _radar_preview_paths(record) has_geo_cache = os.path.exists(geo_cache_path) has_raw_cache = os.path.exists(raw_cache_path) diff --git a/backend/app/routers/tools.py b/backend/app/routers/tools.py index e4e4701..df08b5c 100644 --- a/backend/app/routers/tools.py +++ b/backend/app/routers/tools.py @@ -1,12 +1,14 @@ from __future__ import annotations +import os +import re from typing import List, Optional from fastapi import APIRouter, Depends, HTTPException, Request from pydantic import BaseModel, Field, field_validator from sqlalchemy.ext.asyncio import AsyncSession -from ..config import read_int_env +from ..config import read_int_env, settings from ..database import get_db from ..services.job_queue_service import job_queue_service from ..services.task_service import TASK_LOG_DEFAULT_LIMIT, TASK_LOG_MAX_LIMIT, TASK_QUERY_MAX_OFFSET, task_service @@ -33,12 +35,14 @@ COPY_BATCH_MAX_COPY_ITEMS = read_int_env( minimum=1, maximum=200000, ) -COPY_DINSAR_PACKAGE_MODES = {"task_folder", "task_zip", "source_bundle"} +COPY_DINSAR_PACKAGE_MODES = {"task_folder", "source_bundle"} +_COPY_TARGET_NAME_RE = re.compile(r"[^0-9A-Za-z\u4e00-\u9fff._-]+") class CopyBatchRequest(BaseModel): batch_id: str = Field(max_length=COPY_BATCH_TEXT_MAX_LENGTH) - dest_dir: str = Field(max_length=COPY_BATCH_TEXT_MAX_LENGTH) + dest_dir: str = Field(default="", max_length=COPY_BATCH_TEXT_MAX_LENGTH) + target_name: Optional[str] = Field(default=None, max_length=255) copy_statuses: Optional[List[str]] = None include_orbit_files: bool = False export_zip: bool = False @@ -46,7 +50,7 @@ class CopyBatchRequest(BaseModel): skip_existing: bool = True max_items: Optional[int] = None - @field_validator("batch_id", "dest_dir", mode="before") + @field_validator("batch_id", mode="before") @classmethod def _normalize_required_text(cls, value): normalized = str(value or "").strip() @@ -54,6 +58,17 @@ class CopyBatchRequest(BaseModel): raise ValueError("Field must not be empty.") return normalized + @field_validator("dest_dir", mode="before") + @classmethod + def _normalize_optional_dest_dir(cls, value): + return str(value or "").strip() + + @field_validator("target_name", mode="before") + @classmethod + def _normalize_optional_target_name(cls, value): + normalized = str(value or "").strip() + return normalized or None + @field_validator("copy_statuses", mode="before") @classmethod def _validate_copy_statuses_length(cls, value): @@ -88,6 +103,10 @@ class CopyBatchRequest(BaseModel): @classmethod def _normalize_package_mode(cls, value): normalized = str(value or "task_folder").strip().lower() + if normalized == "task_zip": + normalized = "task_folder" + if normalized in {"bundle", "dedupe_source"}: + normalized = "source_bundle" if normalized not in COPY_DINSAR_PACKAGE_MODES: raise ValueError( f"package_mode must be one of: {sorted(COPY_DINSAR_PACKAGE_MODES)}." @@ -115,6 +134,28 @@ def _normalize_copy_batch_statuses(copy_statuses: Optional[List[str]]) -> List[s return normalized or ["COMPLETED"] +def _safe_copy_target_name(value: Optional[str]) -> str: + raw = str(value or "").strip() + if not raw: + raise HTTPException(status_code=400, detail="target_name 不能为空") + if os.path.isabs(raw) or os.path.splitdrive(raw)[0] or "\\" in raw or "/" in raw: + raise HTTPException(status_code=400, detail="target_name 只能是任务名，不能包含路径") + normalized = _COPY_TARGET_NAME_RE.sub("_", raw).strip("._ ") + if not normalized: + raise HTTPException(status_code=400, detail="target_name 不合法") + if normalized.upper() in {"CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "LPT1", "LPT2", "LPT3"}: + raise HTTPException(status_code=400, detail="target_name 是 Windows 保留名称") + return normalized[:120] + + +def _server_copy_destination(package_mode: str, target_name: Optional[str]) -> str: + safe_name = _safe_copy_target_name(target_name) + root = settings.DINSAR_TASK_POOL_ROOT if package_mode == "task_folder" else settings.DATA_DISTRIBUTION_ROOT + if not root: + raise HTTPException(status_code=500, detail="服务器目标根目录未配置") + return os.path.normpath(os.path.join(root, safe_name)) + + @router.post("/tools/copy-ps-stack") async def copy_ps_stack_endpoint( request: CopyBatchRequest, @@ -124,20 +165,26 @@ async def copy_ps_stack_endpoint( """ Start PS-InSAR copy task from a batch. """ - _validate_export_path(request.dest_dir, "dest_dir") try: copy_statuses = _normalize_copy_batch_statuses(request.copy_statuses) + if request.target_name: + dest_dir = _server_copy_destination("source_bundle", request.target_name) + else: + dest_dir = request.dest_dir + _validate_export_path(dest_dir, "dest_dir") params = { - "dest_dir": request.dest_dir, + "dest_dir": dest_dir, + "target_name": request.target_name, "file_type": "PS_STACK", "batch_id": request.batch_id, "copy_statuses": copy_statuses, } - task_id = await task_service.create_task("COPY_DATA", f"PS数据分发: {request.dest_dir}", params=params) + task_id = await task_service.create_task("COPY_DATA", f"PS数据分发: {request.target_name or dest_dir}", params=params) payload = { "file_type": "PS_STACK", - "dest_dir": request.dest_dir, + "dest_dir": dest_dir, + "target_name": request.target_name, "batch_id": request.batch_id, "copy_statuses": copy_statuses, } @@ -150,7 +197,8 @@ async def copy_ps_stack_endpoint( detail={ "task_id": task_id, "batch_id": request.batch_id, - "dest_dir": request.dest_dir, + "dest_dir": dest_dir, + "target_name": request.target_name, "copy_statuses": copy_statuses, }, ) @@ -169,32 +217,41 @@ async def copy_dinsar_pairs_endpoint( """ Start D-InSAR copy task from a batch. """ - _validate_export_path(request.dest_dir, "dest_dir") try: copy_statuses = _normalize_copy_batch_statuses(request.copy_statuses) package_mode = request.package_mode - if bool(request.export_zip) and package_mode == "task_folder": - package_mode = "task_zip" + if request.target_name: + dest_dir = _server_copy_destination(package_mode, request.target_name) + else: + dest_dir = request.dest_dir + _validate_export_path(dest_dir, "dest_dir") params = { - "dest_dir": request.dest_dir, + "dest_dir": dest_dir, + "target_name": request.target_name, "file_type": "DINSAR_PAIRS", "batch_id": request.batch_id, "copy_statuses": copy_statuses, "include_orbit_files": bool(request.include_orbit_files), - "export_zip": package_mode == "task_zip", + "export_zip": False, "package_mode": package_mode, "skip_existing": bool(request.skip_existing), "max_items": request.max_items, } - task_id = await task_service.create_task("COPY_DATA", f"D-InSAR 数据分发: {request.dest_dir}", params=params) + task_name = ( + f"D-InSAR 生产数据准备: {request.target_name or dest_dir}" + if package_mode == "task_folder" + else f"D-InSAR 数据分发: {request.target_name or dest_dir}" + ) + task_id = await task_service.create_task("COPY_DATA", task_name, params=params) payload = { "file_type": "DINSAR_PAIRS", - "dest_dir": request.dest_dir, + "dest_dir": dest_dir, + "target_name": request.target_name, "batch_id": request.batch_id, "copy_statuses": copy_statuses, "include_orbit_files": bool(request.include_orbit_files), - "export_zip": package_mode == "task_zip", + "export_zip": False, "package_mode": package_mode, "skip_existing": bool(request.skip_existing), "max_items": request.max_items, @@ -208,10 +265,11 @@ async def copy_dinsar_pairs_endpoint( detail={ "task_id": task_id, "batch_id": request.batch_id, - "dest_dir": request.dest_dir, + "dest_dir": dest_dir, + "target_name": request.target_name, "copy_statuses": copy_statuses, "include_orbit_files": bool(request.include_orbit_files), - "export_zip": package_mode == "task_zip", + "export_zip": False, "package_mode": package_mode, "skip_existing": bool(request.skip_existing), "max_items": request.max_items, diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index e5405e7..fde8970 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -13,8 +13,11 @@ MONITOR_CONFIG = { # LT-1 链路 "radar_dirs": split_env_paths(settings.MONITOR_RADAR_DIRS), "orbit_dir": settings.MONITOR_ORBIT_DIR, - "dinsar_dirs": split_env_paths(settings.MONITOR_DINSAR_DIRS), - # Sentinel-1 链路 + "dinsar_dirs": split_env_paths(settings.MONITOR_DINSAR_DIRS) or [settings.DINSAR_PRODUCT_DIR], + "task_pool_root": settings.TASK_POOL_ROOT, + "dinsar_task_pool_root": settings.DINSAR_TASK_POOL_ROOT, + "sbas_task_pool_root": settings.SBAS_TASK_POOL_ROOT, + # LT-1 / Sentinel-1 source inventory "s1_source_dirs": split_env_paths(settings.SOURCE_PRODUCT_DIRS), "s1_storage_dirs": split_env_paths(settings.SENTINEL1_STORAGE_DIRS), "s1_orbit_dirs": split_env_paths(settings.ORBIT_SOURCE_DIRS), diff --git a/backend/app/services/asset_inventory_service.py b/backend/app/services/asset_inventory_service.py index 649ca1e..8514529 100644 --- a/backend/app/services/asset_inventory_service.py +++ b/backend/app/services/asset_inventory_service.py @@ -1,7 +1,9 @@ from __future__ import annotations import asyncio +import gzip import hashlib +import math import os import re import shutil @@ -9,12 +11,12 @@ import tarfile import zipfile from datetime import datetime, timedelta from pathlib import PurePosixPath -from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple +from typing import Any, Awaitable, Callable, Dict, Iterable, List, Optional, Sequence, Tuple from geoalchemy2.shape import from_shape from lxml import etree from shapely.geometry import Polygon -from sqlalchemy import and_, delete, func, or_, select, update +from sqlalchemy import and_, case, delete, func, or_, select, update from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.ext.asyncio import AsyncSession @@ -25,8 +27,11 @@ from ..models import ( AssetInventoryStateORM, ManagedRootORM, OrbitAssetORM, + OrbitAssetDerivativeORM, RadarDataORM, + SARSceneGeometryProfileORM, SceneOrbitBindingORM, + SourceMetadataDocumentORM, SourceProductAssetORM, ) from ..utils import ( @@ -37,12 +42,19 @@ from ..utils import ( parse_xml_metadata, ) from .pairing_state_service import pairing_state_service +from .data_service import DataService +from .image_service import image_service +from .orbit_converter import sync_orbit_pools from .task_service import task_service -PARSER_VERSION = "asset_inventory_v1" +PARSER_VERSION = "asset_inventory_v2" +ARCHIVE_INTEGRITY_VERSION = "archive_integrity_v1" S1_ORBIT_MATCH_RULE_VERSION = "s1_orbit_window_v1" LT1_ORBIT_MATCH_RULE_VERSION = "lt1_orbit_day_v1" +ASSET_SCAN_LOG_INTERVAL = 100 +ASSET_SCAN_DETAILED_PARSE_LOG_LIMIT = 200 +ARCHIVE_INTEGRITY_LOG_INTERVAL = 10 _WINDOWS_DRIVE_RE = re.compile(r"^[a-zA-Z]:[\\/]") _S1_SOURCE_RE = re.compile( @@ -138,6 +150,32 @@ def _target_root_for_s1_archive(archive_path: str, target_root: Optional[str] = return storage_dirs[0] +def _task_pool_materialize_root(source_format: str) -> str: + base = _normalize_path(getattr(settings, "TASK_POOL_ROOT", "") or "") + if not base: + base = _normalize_path(os.path.join(settings.BACKEND_DIR, "runtime", "task_pool")) + folder = { + "S1_ZIP": "sentinel1", + "S1_SAFE_DIR": "sentinel1", + "LT1_ARCHIVE": "lutan1", + "GF3_ARCHIVE": "gf3", + }.get(str(source_format or "").upper(), "source") + return _normalize_path(os.path.join(base, "source_materialized", folder)) + + +def _source_ref_for_materialized_root(path: str) -> str: + normalized = os.path.normcase(_normalize_path(path)) + task_root = os.path.normcase(_normalize_path(getattr(settings, "TASK_POOL_ROOT", "") or "")) + if task_root and (normalized == task_root or normalized.startswith(task_root + os.sep)): + return "TASK_POOL_ROOT" + storage_roots = _configured_sentinel1_storage_dirs() + for storage_root in storage_roots: + storage_norm = os.path.normcase(_normalize_path(storage_root)) + if storage_norm and (normalized == storage_norm or normalized.startswith(storage_norm + os.sep)): + return "SENTINEL1_STORAGE_DIRS" + return "SOURCE_PRODUCT_DIRS" + + def _new_session() -> AsyncSession: if database.AsyncSessionLocal is None: database.init_db() @@ -176,6 +214,13 @@ def _path_kind(path: str) -> str: return "relative" +def _ensure_local_runtime_path(path: str, label: str) -> str: + normalized = _normalize_path(path) + if _path_kind(normalized) == "unc": + raise ValueError(f"{label} cannot use UNC path for active production: {normalized}") + return normalized + + def _stat_path(path: str) -> Dict[str, Optional[float]]: try: stat = os.stat(path) @@ -188,6 +233,15 @@ def _stat_path(path: str) -> Dict[str, Optional[float]]: return {"size_bytes": None, "mtime_epoch": None, "ctime_epoch": None} +def _activity_progress(progress_start: int, progress_end: int, count: int) -> int: + start = max(0, min(100, int(progress_start))) + end = max(start, min(100, int(progress_end))) + collect_end = max(start, end - 6) + if count <= 0 or collect_end <= start: + return start + return min(collect_end, start + 1 + int(count // 50)) + + def _asset_uid(prefix: str, path: str) -> str: digest = hashlib.sha1(_normalize_path(path).lower().encode("utf-8", errors="ignore")).hexdigest() return f"{prefix}:{digest[:32]}" @@ -207,6 +261,153 @@ def _strip_known_suffix(name: str) -> str: return name +def _file_ext_for_path(path: str) -> str: + name = os.path.basename(str(path or "")) + lower = name.lower() + for suffix in (".tar.gz", ".tgz", ".zip", ".tar", ".safe", ".eof", ".txt"): + if lower.endswith(suffix): + return suffix + ext = os.path.splitext(name)[1].lower() + return ext[:32] if ext else "" + + +def _ordered_closed_polygon(points: Sequence[Any]) -> Optional[List[Tuple[float, float]]]: + unique: List[Tuple[float, float]] = [] + for point in points or []: + try: + lon = float(point[0]) + lat = float(point[1]) + except (TypeError, ValueError, IndexError): + continue + current = (lon, lat) + if unique and abs(unique[-1][0] - lon) < 1e-12 and abs(unique[-1][1] - lat) < 1e-12: + continue + if unique and abs(unique[0][0] - lon) < 1e-12 and abs(unique[0][1] - lat) < 1e-12: + continue + if current not in unique: + unique.append(current) + if len(unique) < 3: + return None + + candidates: List[List[Tuple[float, float]]] = [] + candidates.append(unique) + if len(unique) == 4: + center_lon = sum(point[0] for point in unique) / len(unique) + center_lat = sum(point[1] for point in unique) / len(unique) + candidates.insert( + 0, + sorted( + unique, + key=lambda point: math.atan2(point[1] - center_lat, point[0] - center_lon), + ), + ) + + for candidate in candidates: + ring = list(candidate) + if ring[0] != ring[-1]: + ring.append(ring[0]) + try: + polygon = Polygon(ring) + if polygon.is_valid and not polygon.is_empty and polygon.area > 0: + return ring + except Exception: + continue + return None + + +def _closed_polygon_if_valid(points: Sequence[Any]) -> Optional[List[Tuple[float, float]]]: + ring: List[Tuple[float, float]] = [] + for point in points or []: + try: + ring.append((float(point[0]), float(point[1]))) + except (TypeError, ValueError, IndexError): + return None + if len(ring) < 3: + return None + if ring[0] != ring[-1]: + ring.append(ring[0]) + try: + polygon = Polygon(ring) + if polygon.is_valid and not polygon.is_empty and polygon.area > 0: + return ring + except Exception: + return None + return None + + +def _ordered_closed_polygon_from_corners(corners: Sequence[Dict[str, Any]]) -> Optional[List[Tuple[float, float]]]: + entries = [ + item + for item in (corners or []) + if item.get("lon") is not None and item.get("lat") is not None + ] + if len(entries) < 3: + return None + + by_name = { + str(item.get("name") or "").strip().lower(): item + for item in entries + if str(item.get("name") or "").strip() + } + name_order = ["bottomleft", "bottomright", "topright", "topleft"] + if all(name in by_name for name in name_order): + ordered = [(by_name[name]["lon"], by_name[name]["lat"]) for name in name_order] + valid = _closed_polygon_if_valid(ordered) + if valid: + return valid + + ref_entries = [ + item + for item in entries + if item.get("ref_row") is not None and item.get("ref_col") is not None + ] + if len(ref_entries) >= 4: + min_row = min(float(item["ref_row"]) for item in ref_entries) + max_row = max(float(item["ref_row"]) for item in ref_entries) + min_col = min(float(item["ref_col"]) for item in ref_entries) + max_col = max(float(item["ref_col"]) for item in ref_entries) + targets = [(min_row, min_col), (min_row, max_col), (max_row, max_col), (max_row, min_col)] + remaining = list(ref_entries) + ordered_entries: List[Dict[str, Any]] = [] + for target_row, target_col in targets: + chosen = min( + remaining, + key=lambda item: ( + abs(float(item["ref_row"]) - target_row) + abs(float(item["ref_col"]) - target_col), + str(item.get("name") or ""), + ), + ) + ordered_entries.append(chosen) + remaining.remove(chosen) + ordered = [(item["lon"], item["lat"]) for item in ordered_entries] + valid = _closed_polygon_if_valid(ordered) + if valid: + return valid + + return _ordered_closed_polygon([(item["lon"], item["lat"]) for item in entries]) + + +def _root_supported_families(root: ManagedRootORM) -> List[str]: + text = " ".join( + str(value or "") + for value in ( + root.path, + root.root_code, + root.root_role, + root.display_name, + root.source_ref, + ) + ).lower() + families: List[str] = [] + if "lutan" in text or "lt1" in text or "lt-1" in text: + families.append("LT1") + if "sentinel" in text or "sentinel1" in text or "eof" in text or "safe" in text: + families.append("S1") + if "gaofen" in text or "gf3" in text: + families.append("GF3") + return families + + def _has_archive_suffix(name: str, suffixes: Sequence[str]) -> bool: lower = str(name or "").lower() return any(lower.endswith(suffix) for suffix in suffixes) @@ -356,6 +557,139 @@ def _extract_archive_to_dir(archive_path: str, target_dir: str, *, overwrite: bo shutil.rmtree(tmp_dir, ignore_errors=True) +def _archive_integrity_supported(source_format: Any, path: str) -> bool: + normalized_format = str(source_format or "").upper() + if normalized_format not in {"LT1_ARCHIVE", "S1_ZIP", "GF3_ARCHIVE"}: + return False + ext = _file_ext_for_path(path) + if normalized_format == "S1_ZIP": + return ext == ".zip" + return ext in {".tar.gz", ".tgz", ".tar", ".zip"} + + +def _truncate_error_text(value: Any, limit: int = 1000) -> Optional[str]: + if value is None: + return None + text = str(value) + if len(text) <= limit: + return text + return text[: limit - 3] + "..." + + +def _check_zip_archive_integrity(path: str) -> Dict[str, Any]: + method = "zip_testzip" + with zipfile.ZipFile(path, "r") as archive: + infos = archive.infolist() + member_count = 0 + total_uncompressed = 0 + for info in infos: + _safe_archive_member_name(info.filename, path) + if info.is_dir(): + continue + member_count += 1 + total_uncompressed += int(info.file_size or 0) + bad_member = archive.testzip() + if bad_member: + return { + "status": "FAILED", + "method": method, + "error": f"ZIP CRC failed at member: {bad_member}", + "member_count": member_count, + "uncompressed_bytes": total_uncompressed, + } + return { + "status": "OK", + "method": method, + "error": None, + "member_count": member_count, + "uncompressed_bytes": total_uncompressed, + } + + +def _check_tar_archive_integrity(path: str) -> Dict[str, Any]: + method = "tar_stream_list" + member_count = 0 + total_uncompressed = 0 + with tarfile.open(path, "r:*") as archive: + for member in archive: + _safe_archive_member_name(member.name, path) + if member.issym() or member.islnk() or member.isdev(): + raise ValueError(f"Unsupported TAR member type: {member.name}") + if member.isfile(): + member_count += 1 + total_uncompressed += int(member.size or 0) + elif member.isdir(): + continue + else: + raise ValueError(f"Unsupported TAR member type: {member.name}") + return { + "status": "OK", + "method": method, + "error": None, + "member_count": member_count, + "uncompressed_bytes": total_uncompressed, + } + + +def _check_archive_integrity(path: str, source_format: Any = None) -> Dict[str, Any]: + archive = _normalize_path(path) + started = _utcnow() + stat = _stat_path(archive) + if not os.path.isfile(archive): + return { + "status": "FAILED", + "method": None, + "error": f"Archive file is missing: {archive}", + "member_count": None, + "size_bytes": stat.get("size_bytes"), + "mtime_epoch": stat.get("mtime_epoch"), + "duration_seconds": 0.0, + "version": ARCHIVE_INTEGRITY_VERSION, + } + if not _archive_integrity_supported(source_format, archive): + return { + "status": "UNSUPPORTED", + "method": None, + "error": f"Unsupported archive integrity source_format={source_format} ext={_file_ext_for_path(archive)}", + "member_count": None, + "size_bytes": stat.get("size_bytes"), + "mtime_epoch": stat.get("mtime_epoch"), + "duration_seconds": 0.0, + "version": ARCHIVE_INTEGRITY_VERSION, + } + try: + ext = _file_ext_for_path(archive) + if ext == ".zip": + result = _check_zip_archive_integrity(archive) + elif ext in {".tar.gz", ".tgz", ".tar"}: + result = _check_tar_archive_integrity(archive) + else: + result = { + "status": "UNSUPPORTED", + "method": None, + "error": f"Unsupported archive extension: {ext}", + "member_count": None, + } + except Exception as exc: + result = { + "status": "FAILED", + "method": "zip_testzip" if _file_ext_for_path(archive) == ".zip" else "tar_stream_list", + "error": str(exc), + "member_count": None, + } + duration = (_utcnow() - started).total_seconds() + result.update( + { + "size_bytes": stat.get("size_bytes"), + "mtime_epoch": stat.get("mtime_epoch"), + "duration_seconds": round(max(0.0, duration), 3), + "version": ARCHIVE_INTEGRITY_VERSION, + } + ) + result["error"] = _truncate_error_text(result.get("error")) + return result + + def _parse_datetime_token(value: Optional[str]) -> Optional[datetime]: text = str(value or "").strip() if not text: @@ -412,24 +746,45 @@ def _xml_float(value: Optional[str]) -> Optional[float]: return None +def _xml_int(value: Optional[str]) -> Optional[int]: + try: + if value is None or str(value).strip() == "": + return None + return int(float(str(value).strip())) + except (TypeError, ValueError): + return None + + def _parse_radar_xml_metadata_bytes(data: bytes) -> Tuple[Optional[List[Tuple[float, float]]], Dict[str, Any]]: parser = _xml_parser() root = etree.fromstring(data, parser=parser) - corners: List[Tuple[float, float]] = [] + corners: List[Dict[str, Any]] = [] for element in root.iter(): if etree.QName(element).localname.lower() != "scenecornercoord": continue lon = _xml_float(_xml_text_under_local_path(element, "sceneCornerCoord", "lon") or _xml_text_by_local_names(element, ["lon"])) lat = _xml_float(_xml_text_under_local_path(element, "sceneCornerCoord", "lat") or _xml_text_by_local_names(element, ["lat"])) if lon is not None and lat is not None: - corners.append((lon, lat)) + corners.append( + { + "name": element.get("name"), + "lon": lon, + "lat": lat, + "ref_row": _xml_int( + _xml_text_under_local_path(element, "sceneCornerCoord", "refRow") + or _xml_text_by_local_names(element, ["refRow"]) + ), + "ref_col": _xml_int( + _xml_text_under_local_path(element, "sceneCornerCoord", "refColumn") + or _xml_text_by_local_names(element, ["refColumn"]) + ), + } + ) coverage_polygon: Optional[List[Tuple[float, float]]] = None if len(corners) >= 4: - coverage_polygon = corners[:4] - if coverage_polygon[0] != coverage_polygon[-1]: - coverage_polygon.append(coverage_polygon[0]) + coverage_polygon = _ordered_closed_polygon_from_corners(corners[:4]) start_time = ( _xml_text_under_local_path(root, "start", "timeUTC") @@ -461,14 +816,24 @@ def _parse_radar_xml_metadata_bytes(data: bytes) -> Tuple[Optional[List[Tuple[fl "scene_center_lat": center_lat, "acquisition_time_utc": start_time, "acquisition_stop_time_utc": stop_time, - "product_type": _xml_text_under_local_path(root, "imageDataInfo", "imageDataType") - or _xml_text_under_local_path(root, "orderInfo", "productVariant") - or _xml_text_by_local_names(root, ["productType", "imageDataType", "productVariant"]), + "product_type": _xml_text_by_local_names(root, ["productType"]), + "image_data_type": _xml_text_under_local_path(root, "imageDataInfo", "imageDataType") + or _xml_text_by_local_names(root, ["imageDataType"]), + "product_variant": _xml_text_under_local_path(root, "orderInfo", "productVariant") + or _xml_text_by_local_names(root, ["productVariant"]), "image_data_format": _xml_text_under_local_path(root, "imageDataInfo", "imageDataFormat") or _xml_text_by_local_names(root, ["imageDataFormat"]), "product_level": _xml_text_by_local_names(root, ["productLevel", "itemName"]), "product_unique_id": _xml_text_by_local_names(root, ["logicalProductID", "sceneID", "productID"]), "look_direction": (_xml_text_under_local_path(root, "acquisitionInfo", "lookDirection") or "").upper() or None, + "corner_ref_pixels": { + str(item.get("name")): { + "ref_row": item.get("ref_row"), + "ref_col": item.get("ref_col"), + } + for item in corners + if item.get("name") + }, "coverage_polygon": coverage_polygon, } return coverage_polygon, {key: value for key, value in metadata.items() if value not in (None, "", [])} @@ -494,6 +859,92 @@ def _json_safe(value: Any) -> Any: return value +def _metadata_document( + *, + document_type: str, + member_path: str, + content: bytes, + source_format: Optional[str], + satellite_family: Optional[str], + archive_path: str, + archive_mtime: Optional[float], + parse_status: str = "OK", + parse_error: Optional[str] = None, +) -> Dict[str, Any]: + payload = bytes(content or b"") + return { + "document_type": document_type, + "member_path": member_path or document_type, + "content_sha256": hashlib.sha256(payload).hexdigest(), + "content_encoding": "gzip", + "content_bytes": gzip.compress(payload), + "content_size_bytes": len(payload), + "source_format": source_format, + "satellite_family": satellite_family, + "archive_path": archive_path, + "archive_mtime": archive_mtime, + "parser_version": PARSER_VERSION, + "parse_status": parse_status, + "parse_error": parse_error, + "extracted_at": _utcnow(), + } + + +def _extract_s1_annotation_documents(source_path: str, *, limit: int = 16) -> List[Dict[str, Any]]: + docs: List[Dict[str, Any]] = [] + stat = _stat_path(source_path) + if os.path.isdir(source_path): + annotation_root = os.path.join(source_path, "annotation") + if not os.path.isdir(annotation_root): + return docs + candidates: List[str] = [] + for current_root, _, files in os.walk(annotation_root): + for file_name in files: + if file_name.lower().endswith(".xml"): + candidates.append(os.path.join(current_root, file_name)) + for path in sorted(candidates)[: max(0, limit)]: + try: + with open(path, "rb") as stream: + data = stream.read() + docs.append( + _metadata_document( + document_type="S1_ANNOTATION", + member_path=os.path.relpath(path, source_path).replace("\\", "/"), + content=data, + source_format="S1_SAFE_DIR", + satellite_family="S1", + archive_path=source_path, + archive_mtime=stat.get("mtime_epoch"), + ) + ) + except OSError: + continue + return docs + + try: + with zipfile.ZipFile(source_path) as archive: + names = [ + name + for name in archive.namelist() + if "/annotation/" in name.lower() and name.lower().endswith(".xml") + ] + for name in sorted(names)[: max(0, limit)]: + docs.append( + _metadata_document( + document_type="S1_ANNOTATION", + member_path=name, + content=archive.read(name), + source_format="S1_ZIP", + satellite_family="S1", + archive_path=source_path, + archive_mtime=stat.get("mtime_epoch"), + ) + ) + except Exception: + return docs + return docs + + def _xml_parser() -> etree.XMLParser: return etree.XMLParser( resolve_entities=False, @@ -556,26 +1007,24 @@ def _s1_polygon_from_coordinates(text: Optional[str]) -> Optional[List[Tuple[flo lon, lat = second, first points.append((lon, lat)) - if len(points) < 3: - return None - if points[0] != points[-1]: - points.append(points[0]) - return points + return _ordered_closed_polygon(points) def _bbox_from_polygon(points: Optional[List[Tuple[float, float]]]) -> Optional[Tuple[float, float, float, float]]: - if not points or len(points) < 3: + ordered = _ordered_closed_polygon(points or []) + if not ordered or len(ordered) < 4: return None - lons = [float(point[0]) for point in points] - lats = [float(point[1]) for point in points] + lons = [float(point[0]) for point in ordered] + lats = [float(point[1]) for point in ordered] return min(lons), min(lats), max(lons), max(lats) def _centroid_from_polygon(points: Optional[List[Tuple[float, float]]]) -> Tuple[Optional[float], Optional[float]]: - if not points or len(points) < 3: + ordered = _ordered_closed_polygon(points or []) + if not ordered or len(ordered) < 4: return None, None try: - poly = Polygon(points) + poly = Polygon(ordered) if not poly.is_valid: poly = poly.buffer(0) if poly.is_empty: @@ -648,6 +1097,7 @@ def _parse_s1_manifest_bytes(data: bytes) -> Dict[str, Any]: def _parse_s1_zip_manifest(path: str) -> Dict[str, Any]: + stat = _stat_path(path) with zipfile.ZipFile(path) as archive: manifest_name = next( (name for name in archive.namelist() if name.lower().endswith("/manifest.safe") or name.lower() == "manifest.safe"), @@ -655,10 +1105,23 @@ def _parse_s1_zip_manifest(path: str) -> Dict[str, Any]: ) if not manifest_name: return {"manifest_parse_status": "MISSING"} + manifest_bytes = archive.read(manifest_name) return { "manifest_parse_status": "OK", "manifest_path": manifest_name, - **_parse_s1_manifest_bytes(archive.read(manifest_name)), + "metadata_documents": [ + _metadata_document( + document_type="S1_MANIFEST", + member_path=manifest_name, + content=manifest_bytes, + source_format="S1_ZIP", + satellite_family="S1", + archive_path=path, + archive_mtime=stat.get("mtime_epoch"), + ), + *_extract_s1_annotation_documents(path), + ], + **_parse_s1_manifest_bytes(manifest_bytes), } @@ -666,16 +1129,31 @@ def _parse_s1_safe_manifest(path: str) -> Dict[str, Any]: manifest_path = os.path.join(path, "manifest.safe") if not os.path.isfile(manifest_path): return {"manifest_parse_status": "MISSING"} + stat = _stat_path(path) with open(manifest_path, "rb") as stream: + manifest_bytes = stream.read() return { "manifest_parse_status": "OK", "manifest_path": manifest_path, - **_parse_s1_manifest_bytes(stream.read()), + "metadata_documents": [ + _metadata_document( + document_type="S1_MANIFEST", + member_path="manifest.safe", + content=manifest_bytes, + source_format="S1_SAFE_DIR", + satellite_family="S1", + archive_path=path, + archive_mtime=stat.get("mtime_epoch"), + ), + *_extract_s1_annotation_documents(path), + ], + **_parse_s1_manifest_bytes(manifest_bytes), } def _parse_lt1_archive_metadata(path: str) -> Dict[str, Any]: archive_stem = _strip_known_suffix(os.path.basename(path)) + stat = _stat_path(path) xml_member, xml_data, members = _archive_read_first_matching( path, lambda name: _archive_member_base_name(name).lower().endswith(".meta.xml"), @@ -697,6 +1175,17 @@ def _parse_lt1_archive_metadata(path: str) -> Dict[str, Any]: "archive_xml_member": xml_member, "archive_scene_name": _archive_member_scene_name(xml_member, archive_stem), "contained_tiff_members": tiff_members, + "metadata_documents": [ + _metadata_document( + document_type="LT1_META", + member_path=xml_member, + content=xml_data, + source_format="LT1_ARCHIVE", + satellite_family="LT1", + archive_path=path, + archive_mtime=stat.get("mtime_epoch"), + ) + ], "coverage_polygon": coverage_polygon, **xml_meta, } @@ -851,6 +1340,18 @@ def _parse_source_entry(path: str, root: ManagedRootORM) -> Optional[Dict[str, A coverage_polygon, parsed_xml = parse_xml_metadata(xml_path) if parsed_xml: xml_meta = parsed_xml + with open(xml_path, "rb") as stream: + xml_meta["metadata_documents"] = [ + _metadata_document( + document_type="LT1_META", + member_path=os.path.relpath(xml_path, path).replace("\\", "/"), + content=stream.read(), + source_format="LT1_DIR", + satellite_family="LT1", + archive_path=path, + archive_mtime=stat.get("mtime_epoch"), + ) + ] except Exception as exc: xml_meta = {"xml_parse_error": str(exc), "xml_path": xml_path} return _build_lt1_source_asset(path, root, parsed, xml_meta, coverage_polygon, stat, now) @@ -869,7 +1370,7 @@ def _build_s1_source_asset( now: datetime, ) -> Dict[str, Any]: metadata = dict(name_meta.get("metadata") or {}) - metadata.update(manifest_meta) + metadata.update({key: value for key, value in manifest_meta.items() if key != "metadata_documents"}) coverage_polygon = manifest_meta.get("coverage_polygon") centroid_lon, centroid_lat = _centroid_from_polygon(coverage_polygon) bbox = _bbox_from_polygon(coverage_polygon) @@ -886,7 +1387,7 @@ def _build_s1_source_asset( if manifest_pols: metadata["polarization_channels"] = manifest_pols - return { + row = { "asset_uid": _asset_uid("source", path), "logical_product_uid": name_meta.get("logical_product_uid"), "satellite_family": "S1", @@ -909,7 +1410,7 @@ def _build_s1_source_asset( "path_kind": _path_kind(path), "file_name": os.path.basename(path), "file_stem": _strip_known_suffix(os.path.basename(path)), - "file_ext": os.path.splitext(path)[1].lower(), + "file_ext": _file_ext_for_path(path), "size_bytes": stat.get("size_bytes"), "mtime_epoch": stat.get("mtime_epoch"), "checksum_status": "NOT_COMPUTED", @@ -923,6 +1424,8 @@ def _build_s1_source_asset( "missing_since": None, "updated_at": now, } + row["_metadata_documents"] = manifest_meta.get("metadata_documents") or [] + return row def _build_lt1_source_asset( @@ -941,7 +1444,7 @@ def _build_lt1_source_asset( parse_error: Optional[str] = None, ) -> Dict[str, Any]: metadata = dict(parsed) - metadata.update({key: value for key, value in xml_meta.items() if value not in (None, "")}) + metadata.update({key: value for key, value in xml_meta.items() if key != "metadata_documents" and value not in (None, "")}) metadata["coverage_polygon"] = coverage_polygon metadata["coverage_bbox"] = _bbox_from_polygon(coverage_polygon) centroid_lon, centroid_lat = _centroid_from_polygon(coverage_polygon) @@ -950,13 +1453,13 @@ def _build_lt1_source_asset( satellite = parsed.get("satellite") imaging_date = parsed.get("imaging_date") - return { + row = { "asset_uid": _asset_uid("source", path), "logical_product_uid": _strip_known_suffix(os.path.basename(path)), "satellite_family": normalize_satellite_family(satellite), "satellite": satellite, "source_format": source_format, - "product_type": xml_meta.get("product_type") or parsed.get("product_type"), + "product_type": parsed.get("product_type") or xml_meta.get("product_type"), "product_level": xml_meta.get("product_level") or parsed.get("product_level"), "imaging_mode": xml_meta.get("imaging_mode") or parsed.get("imaging_mode"), "polarization": xml_meta.get("polarization") or parsed.get("polarization"), @@ -973,7 +1476,7 @@ def _build_lt1_source_asset( "path_kind": _path_kind(path), "file_name": os.path.basename(path), "file_stem": _strip_known_suffix(os.path.basename(path)), - "file_ext": os.path.splitext(path)[1].lower(), + "file_ext": _file_ext_for_path(path), "size_bytes": stat.get("size_bytes"), "mtime_epoch": stat.get("mtime_epoch"), "checksum_status": "NOT_COMPUTED", @@ -987,6 +1490,8 @@ def _build_lt1_source_asset( "missing_since": None, "updated_at": now, } + row["_metadata_documents"] = xml_meta.get("metadata_documents") or [] + return row def _build_gf3_archive_asset( @@ -1037,7 +1542,7 @@ def _build_gf3_archive_asset( "path_kind": _path_kind(path), "file_name": os.path.basename(path), "file_stem": stem, - "file_ext": os.path.splitext(path)[1].lower(), + "file_ext": _file_ext_for_path(path), "size_bytes": stat.get("size_bytes"), "mtime_epoch": stat.get("mtime_epoch"), "checksum_status": "NOT_COMPUTED", @@ -1167,8 +1672,15 @@ def _iter_source_candidates(root_path: str) -> Iterable[str]: continue stack.append(entry.path) elif entry.is_file(follow_symlinks=False): + stem_upper = _strip_known_suffix(entry.name).upper() if entry.name.upper().startswith("S1") and entry.name.lower().endswith(".zip"): yield _normalize_path(entry.path) + continue + if stem_upper.startswith("LT1") and _has_archive_suffix(entry.name, _LT1_ARCHIVE_EXTS): + yield _normalize_path(entry.path) + continue + if stem_upper.startswith("GF3") and _has_archive_suffix(entry.name, _GF3_ARCHIVE_EXTS): + yield _normalize_path(entry.path) except OSError: continue except OSError: @@ -1187,16 +1699,8 @@ def _iter_s1_zip_candidates(root_path: str) -> Iterable[str]: stack.append(entry.path) elif entry.is_file(follow_symlinks=False): name_upper = entry.name.upper() - stem_upper = _strip_known_suffix(entry.name).upper() if name_upper.startswith("S1") and entry.name.lower().endswith(".zip"): yield _normalize_path(entry.path) - continue - if stem_upper.startswith("LT1") and _has_archive_suffix(entry.name, _LT1_ARCHIVE_EXTS): - yield _normalize_path(entry.path) - continue - if stem_upper.startswith("GF3") and _has_archive_suffix(entry.name, _GF3_ARCHIVE_EXTS): - yield _normalize_path(entry.path) - continue except OSError: continue except OSError: @@ -1256,28 +1760,258 @@ def _collect_source_assets(root: ManagedRootORM) -> Tuple[List[Dict[str, Any]], return rows, issues, entry_count -def _collect_orbit_assets(root: ManagedRootORM) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], int]: +def _same_mtime(left: Any, right: Any) -> bool: + if left is None or right is None: + return left is None and right is None + try: + return abs(float(left) - float(right)) <= 0.001 + except (TypeError, ValueError): + return False + + +def _same_size(left: Any, right: Any) -> bool: + if left is None or right is None: + return left is None and right is None + try: + return int(left) == int(right) + except (TypeError, ValueError): + return False + + +def _cached_source_asset_is_unchanged( + cached: Optional[Dict[str, Any]], + stat: Dict[str, Optional[float]], + root: ManagedRootORM, +) -> bool: + if not cached: + return False + source_format = str(cached.get("source_format") or "").upper() + if source_format not in {"S1_ZIP", "LT1_ARCHIVE", "GF3_ARCHIVE"}: + return False + try: + if int(cached.get("root_ref_id") or 0) != int(root.id or 0): + return False + except (TypeError, ValueError): + return False + if not bool(cached.get("is_active")): + return False + if str(cached.get("parser_version") or "") != PARSER_VERSION: + return False + if str(cached.get("parse_status") or "").upper() != "OK": + return False + return _same_size(cached.get("size_bytes"), stat.get("size_bytes")) and _same_mtime( + cached.get("mtime_epoch"), + stat.get("mtime_epoch"), + ) + + +def _cached_orbit_asset_is_unchanged( + cached: Optional[Dict[str, Any]], + stat: Dict[str, Optional[float]], + root: ManagedRootORM, +) -> bool: + if not cached: + return False + native_format = str(cached.get("native_format") or "").upper() + if native_format not in {"TXT", "EOF"}: + return False + try: + if int(cached.get("root_ref_id") or 0) != int(root.id or 0): + return False + except (TypeError, ValueError): + return False + if not bool(cached.get("is_active")): + return False + if str(cached.get("parser_version") or "") != PARSER_VERSION: + return False + if str(cached.get("parse_status") or "").upper() != "OK": + return False + return _same_size(cached.get("size_bytes"), stat.get("size_bytes")) and _same_mtime( + cached.get("mtime_epoch"), + stat.get("mtime_epoch"), + ) + + +def _collect_source_assets_incremental( + root: ManagedRootORM, + existing_by_path: Dict[str, Dict[str, Any]], + *, + progress_callback: Optional[Callable[[int, str], None]] = None, + log_callback: Optional[Callable[[str, str], None]] = None, + progress_start: int = 0, + progress_end: int = 100, +) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], int, int, List[str]]: rows: List[Dict[str, Any]] = [] issues: List[Dict[str, Any]] = [] + seen_paths: List[str] = [] entry_count = 0 - for path in _iter_orbit_candidates(root.path): + skipped_unchanged = 0 + parse_attempts = 0 + last_progress_count = 0 + + def _log(level: str, message: str) -> None: + if log_callback: + log_callback(level, message) + + def _progress(message: str) -> None: + if progress_callback: + progress_callback(_activity_progress(progress_start, progress_end, entry_count), message) + + _log("INFO", f"Source root discovery started: {root.path}") + for path in _iter_source_candidates(root.path): + entry_count += 1 + normalized_path = _normalize_path(path) + stat = _stat_path(normalized_path) + if _cached_source_asset_is_unchanged(existing_by_path.get(normalized_path), stat, root): + seen_paths.append(normalized_path) + skipped_unchanged += 1 + if skipped_unchanged % ASSET_SCAN_LOG_INTERVAL == 0: + _log( + "INFO", + f"Skipped unchanged source archives: {skipped_unchanged} (candidates={entry_count})", + ) + if entry_count - last_progress_count >= ASSET_SCAN_LOG_INTERVAL: + _progress( + "Scanning source archives: " + f"candidates={entry_count}, skipped={skipped_unchanged}, parsed={len(rows)}, issues={len(issues)}" + ) + last_progress_count = entry_count + continue + parse_attempts += 1 + file_name = os.path.basename(normalized_path) + if parse_attempts <= ASSET_SCAN_DETAILED_PARSE_LOG_LIMIT or parse_attempts % ASSET_SCAN_LOG_INTERVAL == 0: + _log("INFO", f"Extracting source archive metadata {parse_attempts}: {file_name}") + if parse_attempts <= 50 or parse_attempts % 25 == 0: + _progress( + "Extracting source archive metadata: " + f"{file_name} (changed/new={parse_attempts}, skipped={skipped_unchanged})" + ) try: - row = _parse_orbit_entry(path, root) + row = _parse_source_entry(normalized_path, root) except Exception as exc: row = None + _log("WARNING", f"Source archive metadata parse failed: {file_name}: {exc}") + issues.append( + { + "severity": "warning", + "issue_code": "source_parse_failed", + "issue_message": str(exc), + "source_path": normalized_path, + } + ) + if row is None: + continue + rows.append(row) + seen_paths.append(str(row["file_path"])) + if row.get("parse_status") in {"FAILED", "PARTIAL"}: + _log( + "WARNING", + f"Source archive metadata {str(row.get('parse_status')).lower()}: " + f"{os.path.basename(str(row.get('file_path') or normalized_path))}: {row.get('parse_error')}", + ) + issues.append( + { + "severity": "warning", + "issue_code": "source_parse_partial" if row.get("parse_status") == "PARTIAL" else "source_parse_failed", + "issue_message": row.get("parse_error"), + "source_path": row.get("file_path"), + } + ) + if entry_count - last_progress_count >= ASSET_SCAN_LOG_INTERVAL: + _progress( + "Scanning source archives: " + f"candidates={entry_count}, skipped={skipped_unchanged}, parsed={len(rows)}, issues={len(issues)}" + ) + last_progress_count = entry_count + _log( + "INFO", + "Source root discovery finished: " + f"candidates={entry_count}, skipped={skipped_unchanged}, changed_or_new={len(rows)}, issues={len(issues)}", + ) + _progress( + "Source archive discovery finished: " + f"candidates={entry_count}, skipped={skipped_unchanged}, changed_or_new={len(rows)}" + ) + return rows, issues, entry_count, skipped_unchanged, seen_paths + + +def _collect_orbit_assets_incremental( + root: ManagedRootORM, + existing_by_path: Dict[str, Dict[str, Any]], + *, + progress_callback: Optional[Callable[[int, str], None]] = None, + log_callback: Optional[Callable[[str, str], None]] = None, + progress_start: int = 0, + progress_end: int = 100, +) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], int, int, List[str]]: + rows: List[Dict[str, Any]] = [] + issues: List[Dict[str, Any]] = [] + seen_paths: List[str] = [] + entry_count = 0 + skipped_unchanged = 0 + parse_attempts = 0 + last_progress_count = 0 + + def _log(level: str, message: str) -> None: + if log_callback: + log_callback(level, message) + + def _progress(message: str) -> None: + if progress_callback: + progress_callback(_activity_progress(progress_start, progress_end, entry_count), message) + + _log("INFO", f"Orbit root discovery started: {root.path}") + for path in _iter_orbit_candidates(root.path): + entry_count += 1 + normalized_path = _normalize_path(path) + stat = _stat_path(normalized_path) + if _cached_orbit_asset_is_unchanged(existing_by_path.get(normalized_path), stat, root): + seen_paths.append(normalized_path) + skipped_unchanged += 1 + if skipped_unchanged % ASSET_SCAN_LOG_INTERVAL == 0: + _log( + "INFO", + f"Skipped unchanged orbit assets: {skipped_unchanged} (candidates={entry_count})", + ) + if entry_count - last_progress_count >= ASSET_SCAN_LOG_INTERVAL: + _progress( + "Scanning orbit assets: " + f"candidates={entry_count}, skipped={skipped_unchanged}, parsed={len(rows)}, issues={len(issues)}" + ) + last_progress_count = entry_count + continue + parse_attempts += 1 + file_name = os.path.basename(normalized_path) + if parse_attempts <= ASSET_SCAN_DETAILED_PARSE_LOG_LIMIT or parse_attempts % ASSET_SCAN_LOG_INTERVAL == 0: + _log("INFO", f"Extracting orbit asset metadata {parse_attempts}: {file_name}") + if parse_attempts <= 50 or parse_attempts % 25 == 0: + _progress( + "Extracting orbit asset metadata: " + f"{file_name} (changed/new={parse_attempts}, skipped={skipped_unchanged})" + ) + try: + row = _parse_orbit_entry(normalized_path, root) + except Exception as exc: + row = None + _log("WARNING", f"Orbit asset parse failed: {file_name}: {exc}") issues.append( { "severity": "warning", "issue_code": "orbit_parse_failed", "issue_message": str(exc), - "source_path": path, + "source_path": normalized_path, } ) if row is None: continue - entry_count += 1 rows.append(row) + seen_paths.append(str(row["file_path"])) if row.get("parse_status") in {"FAILED", "PARTIAL"}: + _log( + "WARNING", + f"Orbit asset metadata {str(row.get('parse_status')).lower()}: " + f"{os.path.basename(str(row.get('file_path') or normalized_path))}: {row.get('parse_error')}", + ) issues.append( { "severity": "warning", @@ -1286,11 +2020,27 @@ def _collect_orbit_assets(root: ManagedRootORM) -> Tuple[List[Dict[str, Any]], L "source_path": row.get("file_path"), } ) - return rows, issues, entry_count + if entry_count - last_progress_count >= ASSET_SCAN_LOG_INTERVAL: + _progress( + "Scanning orbit assets: " + f"candidates={entry_count}, skipped={skipped_unchanged}, parsed={len(rows)}, issues={len(issues)}" + ) + last_progress_count = entry_count + _log( + "INFO", + "Orbit root discovery finished: " + f"candidates={entry_count}, skipped={skipped_unchanged}, changed_or_new={len(rows)}, issues={len(issues)}", + ) + _progress( + "Orbit asset discovery finished: " + f"candidates={entry_count}, skipped={skipped_unchanged}, changed_or_new={len(rows)}" + ) + return rows, issues, entry_count, skipped_unchanged, seen_paths def _insar_source_ready(row: Dict[str, Any], coverage_polygon: Optional[List[Tuple[float, float]]]) -> Tuple[bool, Optional[str]]: reasons: List[str] = [] + metadata = dict(row.get("metadata_json") or {}) if not coverage_polygon or len(coverage_polygon) < 3: reasons.append("missing_footprint") if not row.get("imaging_date"): @@ -1299,7 +2049,14 @@ def _insar_source_ready(row: Dict[str, Any], coverage_polygon: Optional[List[Tup reasons.append("missing_imaging_mode") if not row.get("polarization"): reasons.append("missing_polarization") - if str(row.get("product_type") or "").upper() not in {"SLC", "SSC"}: + complex_tokens = { + str(row.get("product_type") or "").strip().upper(), + str(metadata.get("image_data_type") or "").strip().upper(), + str(metadata.get("product_variant") or "").strip().upper(), + str(metadata.get("filename_class_token") or "").strip().upper(), + str(metadata.get("source_product_token") or "").strip().upper(), + } + if not complex_tokens.intersection({"COMPLEX", "SLC", "SSC"}): reasons.append("not_complex_source") if reasons: return False, ";".join(reasons) @@ -1319,20 +2076,297 @@ class AssetInventoryService: return await task_service.update_task(task_id, message=message, progress=max(0, min(100, int(progress)))) + def _normalize_scan_families(self, families: Optional[Sequence[str]]) -> List[str]: + normalized: List[str] = [] + for item in families or []: + family = str(normalize_satellite_family(item) or item or "").strip().upper() + if family and family not in normalized: + normalized.append(family) + return normalized + + def _scan_includes_type(self, inventory_types: Optional[Sequence[str]], *names: str) -> bool: + type_set = {str(item or "").strip().lower() for item in (inventory_types or []) if str(item or "").strip()} + if not type_set: + return True + return bool(type_set.intersection({str(name).strip().lower() for name in names})) + + def _normalize_family_filter(self, satellite_family: Optional[str]) -> List[str]: + values: List[str] = [] + for raw in re.split(r"[,;]", str(satellite_family or "")): + family = str(normalize_satellite_family(raw) or raw or "").strip().upper() + if family and family not in values: + values.append(family) + return values + + def _thread_callbacks( + self, + task_id: Optional[str], + loop: asyncio.AbstractEventLoop, + ) -> Tuple[ + Optional[Callable[[int, str], None]], + Optional[Callable[[str, str], None]], + Callable[[], Awaitable[None]], + ]: + async def _noop() -> None: + return None + + if not task_id: + return None, None, _noop + + pending: List[Any] = [] + + def _submit(coro: Any) -> None: + pending.append(asyncio.run_coroutine_threadsafe(coro, loop)) + + def _progress(progress: int, message: str) -> None: + _submit( + task_service.update_task( + task_id, + progress=max(0, min(100, int(progress))), + message=message, + ) + ) + + def _log(level: str, message: str) -> None: + _submit(task_service.add_log(task_id, level, message)) + + async def _drain() -> None: + while pending: + current = pending[:] + pending.clear() + await asyncio.gather( + *(asyncio.wrap_future(item) for item in current), + return_exceptions=True, + ) + + return _progress, _log, _drain + + async def build_archive_preview_caches( + self, + db: Optional[AsyncSession] = None, + *, + families: Optional[Sequence[str]] = None, + limit: int = 0, + force: bool = False, + apply: bool = True, + task_id: Optional[str] = None, + progress_start: int = 84, + progress_end: int = 96, + ) -> Dict[str, Any]: + generated_session = db is None + if generated_session: + db = _new_session() + assert db is not None + + family_map = { + "LT1": {"LT1_ARCHIVE"}, + "S1": {"S1_ZIP"}, + } + requested_families = self._normalize_scan_families(families) or ["LT1", "S1"] + target_families = [item for item in requested_families if item in family_map] + source_formats = sorted( + { + source_format + for family in target_families + for source_format in family_map.get(family, set()) + } + ) + summary: Dict[str, Any] = { + "records_seen": 0, + "candidate_count": 0, + "ready": 0, + "cached": 0, + "skipped_ready": 0, + "failed": 0, + "missing_source": 0, + "raw_cached": 0, + "raw_skipped": 0, + "raw_failed": 0, + "families": target_families, + } + if not target_families or not source_formats: + await self._progress(task_id, "No LT1/S1 archive previews to build for selected families.", progress_end) + return summary + + try: + stmt = ( + select(RadarDataORM) + .where(RadarDataORM.satellite_family.in_(target_families)) + .where(RadarDataORM.source_format.in_(source_formats)) + .order_by(RadarDataORM.satellite_family.asc(), RadarDataORM.id.asc()) + ) + result = await db.execute(stmt) + records = list(result.scalars().all()) + summary["records_seen"] = len(records) + candidates: List[RadarDataORM] = [] + for record in records: + unique_id = record.unique_id or record.file_path + raw_cache_path = DataService.get_radar_raw_cache_path(unique_id, record.file_path) + geo_cache_path = DataService.get_radar_geo_cache_path(unique_id, record.file_path) + ready = ( + (record.preview_cache_status or "NONE") == "READY" + and (record.preview_cache_version or "") == settings.RADAR_GEO_CACHE_VERSION + and bool(record.preview_cache_path or geo_cache_path) + and os.path.exists(record.preview_cache_path or geo_cache_path) + and os.path.exists(raw_cache_path) + ) + if ready and not force: + summary["skipped_ready"] += 1 + continue + candidates.append(record) + + if limit and limit > 0: + candidates = candidates[: int(limit)] + summary["candidate_count"] = len(candidates) + if not apply: + return summary + if not candidates: + await self._progress( + task_id, + f"Archive preview cache already ready: skipped={summary['skipped_ready']}", + progress_end, + ) + return summary + + await self._progress( + task_id, + f"Building archive preview cache: candidates={len(candidates)}, skipped_ready={summary['skipped_ready']}", + progress_start, + ) + thumb_size = (settings.RADAR_THUMBNAIL_MAX_SIZE, settings.RADAR_THUMBNAIL_MAX_SIZE) + total = len(candidates) + for index, record in enumerate(candidates, start=1): + unique_id = record.unique_id or record.file_path + raw_cache_path = DataService.get_radar_raw_cache_path(unique_id, record.file_path) + geo_cache_path = DataService.get_radar_geo_cache_path(unique_id, record.file_path) + product_name = os.path.basename(str(record.file_path or "")) + preview_source = await asyncio.to_thread(DataService.find_radar_preview_source, record.file_path) + + if not preview_source: + record.preview_cache_status = "NONE" + record.preview_cache_path = None + record.preview_cache_version = settings.RADAR_GEO_CACHE_VERSION + record.preview_cache_updated_at = _utcnow() + record.preview_cache_error = "preview_source_not_found" + summary["missing_source"] += 1 + if task_id: + await task_service.add_log(task_id, "WARNING", f"Preview source missing: {product_name}") + db.add(record) + else: + coverage_polygon = DataService._normalize_coverage_polygon(record.coverage_polygon) + try: + bbox = ( + float(record.min_lon), + float(record.min_lat), + float(record.max_lon), + float(record.max_lat), + ) + if bbox[0] >= bbox[2] or bbox[1] >= bbox[3]: + bbox = None + except (TypeError, ValueError): + bbox = None + + if not coverage_polygon or not bbox: + record.preview_cache_status = "FAILED" + record.preview_cache_path = None + record.preview_cache_version = settings.RADAR_GEO_CACHE_VERSION + record.preview_cache_updated_at = _utcnow() + record.preview_cache_error = "invalid_coverage_polygon" if not coverage_polygon else "invalid_bbox" + summary["failed"] += 1 + if task_id: + await task_service.add_log(task_id, "ERROR", f"Preview geometry invalid: {product_name}: {record.preview_cache_error}") + db.add(record) + else: + source_corner_mapping = await asyncio.to_thread( + DataService.get_radar_source_corner_mapping, + record.file_path, + ) + ok_geo, geo_error = await asyncio.to_thread( + image_service.create_geocorrected_radar_cached_image, + preview_source, + geo_cache_path, + coverage_polygon, + bbox, + source_corner_mapping, + thumb_size, + settings.RADAR_GEO_CACHE_QUALITY, + ) + ok_raw = await asyncio.to_thread( + image_service.create_radar_cached_image, + preview_source, + raw_cache_path, + thumb_size, + ) + if ok_raw: + summary["raw_cached"] += 1 + else: + summary["raw_failed"] += 1 + + if ok_geo and os.path.exists(geo_cache_path): + record.preview_cache_status = "READY" + record.preview_cache_path = geo_cache_path + record.preview_cache_error = None + summary["ready"] += 1 + summary["cached"] += 1 + if task_id: + await task_service.add_log(task_id, "INFO", f"Preview cache ready: {index}/{total} {product_name}") + else: + record.preview_cache_status = "FAILED" + record.preview_cache_path = None + record.preview_cache_error = geo_error or ("raw_cache_ready_only" if ok_raw else "preview_cache_build_failed") + summary["failed"] += 1 + if task_id: + await task_service.add_log(task_id, "ERROR", f"Preview cache failed: {product_name}: {record.preview_cache_error}") + record.preview_cache_version = settings.RADAR_GEO_CACHE_VERSION + record.preview_cache_updated_at = _utcnow() + db.add(record) + + progress = progress_start + int(index / max(1, total) * max(1, progress_end - progress_start)) + await self._progress( + task_id, + f"Building archive preview cache ({index}/{total}): ready={summary['ready']}, failed={summary['failed']}, missing={summary['missing_source']}", + min(progress_end, progress), + ) + await db.commit() + + await self._progress( + task_id, + ( + "Archive preview cache completed: " + f"ready={summary['ready']}, skipped={summary['skipped_ready']}, " + f"failed={summary['failed']}, missing={summary['missing_source']}" + ), + progress_end, + ) + return summary + except Exception: + if db is not None: + await db.rollback() + raise + finally: + if generated_session and db is not None: + await db.close() + async def _get_scan_roots( self, db: AsyncSession, *, inventory_types: Optional[Sequence[str]] = None, root_ids: Optional[Sequence[int]] = None, + families: Optional[Sequence[str]] = None, ) -> List[ManagedRootORM]: type_set = {str(item or "").strip().lower() for item in (inventory_types or []) if str(item or "").strip()} + family_set = { + str(normalize_satellite_family(item) or item or "").strip().upper() + for item in (families or []) + if str(item or "").strip() + } + family_set.discard("") roles: List[str] = [] if not type_set or "source_product" in type_set or "source" in type_set: roles.extend( [ "source_product_pool", - "source_pool_gf3_archive", ] ) if not type_set or "orbit_asset" in type_set or "orbit" in type_set: @@ -1346,7 +2380,14 @@ class AssetInventoryService: if root_ids: stmt = stmt.where(ManagedRootORM.id.in_([int(item) for item in root_ids])) result = await db.execute(stmt) - return result.scalars().all() + roots = result.scalars().all() + if family_set: + roots = [ + root + for root in roots + if family_set.intersection(_root_supported_families(root)) + ] + return roots async def scan_configured_roots( self, @@ -1354,7 +2395,9 @@ class AssetInventoryService: *, inventory_types: Optional[Sequence[str]] = None, root_ids: Optional[Sequence[int]] = None, + families: Optional[Sequence[str]] = None, bind_orbits: bool = True, + build_previews: bool = True, task_id: Optional[str] = None, ) -> Dict[str, Any]: generated_session = db is None @@ -1364,7 +2407,12 @@ class AssetInventoryService: try: await self._progress(task_id, "Preparing source/orbit asset scan...", 2) - roots = await self._get_scan_roots(db, inventory_types=inventory_types, root_ids=root_ids) + roots = await self._get_scan_roots( + db, + inventory_types=inventory_types, + root_ids=root_ids, + families=families, + ) results: List[Dict[str, Any]] = [] totals = { "source_roots": 0, @@ -1377,18 +2425,36 @@ class AssetInventoryService: total_roots = len(roots) for index, root in enumerate(roots, start=1): - progress = 5 + int((index - 1) / max(1, total_roots) * 75) + progress = 5 + int((index - 1) / max(1, total_roots) * 78) + next_progress = 5 + int(index / max(1, total_roots) * 78) await self._progress(task_id, f"Scanning {root.display_name}: {root.path}", progress) - if root.root_role in {"source_product_pool", "source_pool_gf3_archive"}: - result = await self.scan_source_root(db, root) + if root.root_role == "source_product_pool": + result = await self.scan_source_root( + db, + root, + task_id=task_id, + progress_start=progress, + progress_end=next_progress, + ) totals["source_roots"] += 1 totals["source_assets"] += int(result.get("asset_count") or 0) elif root.root_role == "orbit_asset_pool": - result = await self.scan_orbit_root(db, root) + result = await self.scan_orbit_root( + db, + root, + task_id=task_id, + progress_start=progress, + progress_end=next_progress, + ) totals["orbit_roots"] += 1 totals["orbit_assets"] += int(result.get("asset_count") or 0) else: continue + await self._progress( + task_id, + f"Finished {root.display_name}: assets={result.get('asset_count', 0)}, issues={result.get('issue_count', 0)}", + max(progress, next_progress - 1), + ) totals["issues"] += int(result.get("issue_count") or 0) if result.get("status") == "INACCESSIBLE": totals["inaccessible_roots"] += 1 @@ -1397,15 +2463,32 @@ class AssetInventoryService: binding_summary: Dict[str, Any] = {} if bind_orbits: - await self._progress(task_id, "Binding scenes to precise orbit assets...", 88) + await self._progress(task_id, "Binding scenes to precise orbit assets...", 84) binding_summary = await self.bind_scene_orbits(db) await db.commit() + preview_summary: Dict[str, Any] = {} + should_build_previews = ( + build_previews + and totals["source_roots"] > 0 + and self._scan_includes_type(inventory_types, "source_product", "source") + ) + if should_build_previews: + preview_summary = await self.build_archive_preview_caches( + db, + families=families, + task_id=task_id, + progress_start=88, + progress_end=98, + ) + await db.commit() + summary = { "message": "Asset inventory scan completed", "root_count": total_roots, **totals, "binding": binding_summary, + "preview_cache": preview_summary, "results": results, } await self._progress(task_id, "Asset inventory scan completed", 100) @@ -1418,7 +2501,206 @@ class AssetInventoryService: if generated_session and db is not None: await db.close() - async def scan_source_root(self, db: AsyncSession, root: ManagedRootORM) -> Dict[str, Any]: + async def audit_source_archive_integrity( + self, + db: Optional[AsyncSession] = None, + *, + families: Optional[Sequence[str]] = None, + source_formats: Optional[Sequence[str]] = None, + asset_ids: Optional[Sequence[int]] = None, + force: bool = False, + limit: Optional[int] = None, + task_id: Optional[str] = None, + ) -> Dict[str, Any]: + generated_session = db is None + if generated_session: + db = _new_session() + assert db is not None + + safe_limit: Optional[int] = None + if limit is not None: + try: + parsed_limit = int(limit) + safe_limit = parsed_limit if parsed_limit > 0 else None + except (TypeError, ValueError): + safe_limit = None + + family_filter = self._normalize_scan_families(families) + format_filter = [ + str(item or "").strip().upper() + for item in (source_formats or []) + if str(item or "").strip() + ] + if not format_filter: + format_filter = ["LT1_ARCHIVE", "S1_ZIP"] + format_filter = [item for item in format_filter if item in {"LT1_ARCHIVE", "S1_ZIP", "GF3_ARCHIVE"}] + if not format_filter: + format_filter = ["LT1_ARCHIVE", "S1_ZIP"] + + try: + await self._progress(task_id, "Preparing source archive integrity audit...", 2) + filters = [ + SourceProductAssetORM.is_active == True, # noqa: E712 + SourceProductAssetORM.source_format.in_(format_filter), + ] + if family_filter: + filters.append(SourceProductAssetORM.satellite_family.in_(family_filter)) + if asset_ids: + filters.append(SourceProductAssetORM.id.in_([int(item) for item in asset_ids])) + stmt = ( + select(SourceProductAssetORM) + .where(*filters) + .order_by( + SourceProductAssetORM.satellite_family.asc().nullslast(), + SourceProductAssetORM.imaging_date.asc().nullslast(), + SourceProductAssetORM.id.asc(), + ) + ) + if safe_limit: + stmt = stmt.limit(safe_limit) + rows = (await db.execute(stmt)).scalars().all() + total = len(rows) + summary: Dict[str, Any] = { + "message": "Source archive integrity audit completed", + "total": total, + "checked": 0, + "skipped": 0, + "ok": 0, + "failed": 0, + "unsupported": 0, + "missing": 0, + "force": bool(force), + "families": family_filter, + "source_formats": format_filter, + "version": ARCHIVE_INTEGRITY_VERSION, + } + if total <= 0: + await self._progress(task_id, "No source archives matched integrity audit filters.", 100) + return summary + if task_id: + await task_service.add_log( + task_id, + "INFO", + ( + "Source archive integrity audit candidates: " + f"total={total}, families={family_filter or 'ALL'}, formats={format_filter}, force={bool(force)}" + ), + ) + + for index, asset in enumerate(rows, start=1): + file_path = _normalize_path(asset.file_path or "") + file_name = os.path.basename(file_path) or str(asset.logical_product_uid or asset.id) + progress = 5 + int((index - 1) / max(1, total) * 90) + stat = _stat_path(file_path) + unchanged = ( + _same_size(asset.size_bytes, stat.get("size_bytes")) + and _same_mtime(asset.mtime_epoch, stat.get("mtime_epoch")) + ) + previous_status = str(asset.archive_integrity_status or "NOT_CHECKED").upper() + previous_version = str(asset.archive_integrity_version or "") + can_skip = ( + not force + and unchanged + and previous_version == ARCHIVE_INTEGRITY_VERSION + and previous_status in {"OK", "FAILED", "UNSUPPORTED"} + ) + if can_skip: + summary["skipped"] += 1 + if previous_status == "OK": + summary["ok"] += 1 + elif previous_status == "FAILED": + summary["failed"] += 1 + elif previous_status == "UNSUPPORTED": + summary["unsupported"] += 1 + if index <= 5 or index % ARCHIVE_INTEGRITY_LOG_INTERVAL == 0 or index == total: + await self._progress( + task_id, + f"Skipping unchanged archive integrity {index}/{total}: {file_name}", + 5 + int(index / max(1, total) * 90), + ) + if task_id and (summary["skipped"] <= 5 or summary["skipped"] % ARCHIVE_INTEGRITY_LOG_INTERVAL == 0): + await task_service.add_log( + task_id, + "INFO", + f"Skipped unchanged archive integrity: {summary['skipped']} skipped, {file_name}, status={previous_status}", + ) + continue + + await self._progress( + task_id, + f"Checking archive integrity {index}/{total}: {file_name}", + progress, + ) + if task_id: + await task_service.add_log( + task_id, + "INFO", + f"Checking archive integrity {index}/{total}: {file_path}", + ) + result = await asyncio.to_thread(_check_archive_integrity, file_path, asset.source_format) + checked_at = _utcnow() + status = str(result.get("status") or "FAILED").upper() + asset.size_bytes = result.get("size_bytes") + asset.mtime_epoch = result.get("mtime_epoch") + asset.archive_integrity_status = status + asset.archive_integrity_method = result.get("method") + asset.archive_integrity_checked_at = checked_at + asset.archive_integrity_error = result.get("error") + asset.archive_integrity_version = ARCHIVE_INTEGRITY_VERSION + asset.archive_integrity_member_count = result.get("member_count") + asset.updated_at = checked_at + db.add(asset) + summary["checked"] += 1 + if status == "OK": + summary["ok"] += 1 + await self._resolve_archive_integrity_issue(db, asset, now=checked_at) + elif status == "UNSUPPORTED": + summary["unsupported"] += 1 + await self._resolve_archive_integrity_issue(db, asset, now=checked_at) + else: + summary["failed"] += 1 + if str(result.get("error") or "").lower().startswith("archive file is missing"): + summary["missing"] += 1 + await self._record_archive_integrity_issue(db, asset, result, now=checked_at) + await db.commit() + if task_id: + level = "INFO" if status == "OK" else "WARNING" if status == "UNSUPPORTED" else "ERROR" + duration = result.get("duration_seconds") + detail = ( + f"Archive integrity {status}: {file_name}, " + f"members={result.get('member_count')}, duration={duration}s" + ) + if result.get("error"): + detail += f", error={result.get('error')}" + await task_service.add_log(task_id, level, detail) + + await self._progress( + task_id, + ( + "Source archive integrity audit completed: " + f"checked={summary['checked']}, skipped={summary['skipped']}, " + f"ok={summary['ok']}, failed={summary['failed']}, unsupported={summary['unsupported']}" + ), + 100, + ) + return summary + except Exception: + if db is not None: + await db.rollback() + raise + finally: + if generated_session and db is not None: + await db.close() + + async def scan_source_root( + self, + db: AsyncSession, + root: ManagedRootORM, + *, + task_id: Optional[str] = None, + progress_start: int = 0, + progress_end: int = 100, + ) -> Dict[str, Any]: started_at = _utcnow() state = await self._ensure_state(db, root, "source_product", started_at) if not os.path.isdir(root.path): @@ -1447,10 +2729,55 @@ class AssetInventoryService: ) return {"root_id": root.id, "inventory_type": "source_product", "status": "INACCESSIBLE", "asset_count": 0, "issue_count": 1} - rows, issues, entry_count = await asyncio.to_thread(_collect_source_assets, root) - seen_paths = [row["file_path"] for row in rows] + existing_result = await db.execute( + select(SourceProductAssetORM).where(SourceProductAssetORM.root_ref_id == root.id) + ) + existing_by_path = { + str(asset.file_path): { + "root_ref_id": asset.root_ref_id, + "source_format": asset.source_format, + "size_bytes": asset.size_bytes, + "mtime_epoch": asset.mtime_epoch, + "parser_version": asset.parser_version, + "parse_status": asset.parse_status, + "is_active": bool(asset.is_active), + } + for asset in existing_result.scalars().all() + if asset.file_path + } + + loop = asyncio.get_running_loop() + progress_callback, log_callback, drain_thread_events = self._thread_callbacks(task_id, loop) + rows, issues, entry_count, skipped_unchanged, seen_paths = await asyncio.to_thread( + _collect_source_assets_incremental, + root, + existing_by_path, + progress_callback=progress_callback, + log_callback=log_callback, + progress_start=progress_start, + progress_end=max(progress_start + 1, progress_end - 8), + ) + await drain_thread_events() + changed_paths = [row["file_path"] for row in rows] now = _utcnow() - for row in rows: + write_start = max(progress_start, progress_end - 7) + write_end = max(write_start, progress_end - 3) + await self._progress( + task_id, + f"Writing source asset index: changed_or_new={len(rows)}, skipped={skipped_unchanged}", + write_start, + ) + db_rows = [ + {key: value for key, value in row.items() if not str(key).startswith("_")} + for row in rows + ] + if task_id: + await task_service.add_log( + task_id, + "INFO", + f"Source asset DB upsert started: changed_or_new={len(rows)}, skipped={skipped_unchanged}, seen={len(seen_paths)}", + ) + for index, row in enumerate(db_rows, start=1): stmt = pg_insert(SourceProductAssetORM).values(row) excluded = stmt.excluded stmt = stmt.on_conflict_do_update( @@ -1481,6 +2808,12 @@ class AssetInventoryService: "size_bytes": excluded.size_bytes, "mtime_epoch": excluded.mtime_epoch, "checksum_status": excluded.checksum_status, + "archive_integrity_status": "NOT_CHECKED", + "archive_integrity_method": None, + "archive_integrity_checked_at": None, + "archive_integrity_error": None, + "archive_integrity_version": None, + "archive_integrity_member_count": None, "parser_name": excluded.parser_name, "parser_version": excluded.parser_version, "parse_status": excluded.parse_status, @@ -1493,16 +2826,36 @@ class AssetInventoryService: }, ) await db.execute(stmt) + if index % ASSET_SCAN_LOG_INTERVAL == 0 or index == len(rows): + progress_value = write_start + if rows and write_end > write_start: + progress_value = write_start + int(index / max(1, len(rows)) * (write_end - write_start)) + await self._progress( + task_id, + f"Writing source asset index: {index}/{len(rows)} changed_or_new, skipped={skipped_unchanged}", + progress_value, + ) await db.flush() asset_ids_by_path: Dict[str, int] = {} - if seen_paths: + if changed_paths: + await self._progress( + task_id, + f"Updating radar scene records for changed source assets: {len(changed_paths)}", + max(write_end, progress_end - 2), + ) result = await db.execute( - select(SourceProductAssetORM.file_path, SourceProductAssetORM.id).where(SourceProductAssetORM.file_path.in_(seen_paths)) + select(SourceProductAssetORM.file_path, SourceProductAssetORM.id).where( + SourceProductAssetORM.file_path.in_(changed_paths) + ) ) asset_ids_by_path = {str(path): int(asset_id) for path, asset_id in result.all()} + await self._upsert_metadata_documents_for_source_assets(db, rows, asset_ids_by_path) await self._upsert_radar_records_for_source_assets(db, rows, asset_ids_by_path) + elif task_id: + await task_service.add_log(task_id, "INFO", "No changed source assets; radar scene record update skipped.") + await self._progress(task_id, "Marking missing source assets and refreshing scan issues...", max(progress_end - 2, progress_start)) await self._mark_missing_source_assets(db, root, seen_paths, now) await self._replace_root_issues(db, root, "source_product", issues) await self._finish_state( @@ -1511,21 +2864,39 @@ class AssetInventoryService: status="OK" if not any(item.get("severity") == "error" for item in issues) else "WARNING", started_at=started_at, entry_count=entry_count, - asset_count=len(rows), + asset_count=len(seen_paths), issue_count=len(issues), error=None, ) + if task_id: + await task_service.add_log( + task_id, + "INFO", + "Source root scan summary: " + f"path={root.path}, candidates={entry_count}, active={len(seen_paths)}, " + f"changed_or_new={len(rows)}, skipped={skipped_unchanged}, issues={len(issues)}", + ) return { "root_id": root.id, "root_path": root.path, "inventory_type": "source_product", "status": state.status, "entry_count": entry_count, - "asset_count": len(rows), + "asset_count": len(seen_paths), + "changed_asset_count": len(rows), + "unchanged_asset_count": skipped_unchanged, "issue_count": len(issues), } - async def scan_orbit_root(self, db: AsyncSession, root: ManagedRootORM) -> Dict[str, Any]: + async def scan_orbit_root( + self, + db: AsyncSession, + root: ManagedRootORM, + *, + task_id: Optional[str] = None, + progress_start: int = 0, + progress_end: int = 100, + ) -> Dict[str, Any]: started_at = _utcnow() state = await self._ensure_state(db, root, "orbit_asset", started_at) if not os.path.isdir(root.path): @@ -1554,10 +2925,50 @@ class AssetInventoryService: ) return {"root_id": root.id, "inventory_type": "orbit_asset", "status": "INACCESSIBLE", "asset_count": 0, "issue_count": 1} - rows, issues, entry_count = await asyncio.to_thread(_collect_orbit_assets, root) - seen_paths = [row["file_path"] for row in rows] + existing_result = await db.execute( + select(OrbitAssetORM).where(OrbitAssetORM.root_ref_id == root.id) + ) + existing_by_path = { + str(asset.file_path): { + "root_ref_id": asset.root_ref_id, + "native_format": asset.native_format, + "size_bytes": asset.size_bytes, + "mtime_epoch": asset.mtime_epoch, + "parser_version": asset.parser_version, + "parse_status": asset.parse_status, + "is_active": bool(asset.is_active), + } + for asset in existing_result.scalars().all() + if asset.file_path + } + + loop = asyncio.get_running_loop() + progress_callback, log_callback, drain_thread_events = self._thread_callbacks(task_id, loop) + rows, issues, entry_count, skipped_unchanged, seen_paths = await asyncio.to_thread( + _collect_orbit_assets_incremental, + root, + existing_by_path, + progress_callback=progress_callback, + log_callback=log_callback, + progress_start=progress_start, + progress_end=max(progress_start + 1, progress_end - 6), + ) + await drain_thread_events() now = _utcnow() - for row in rows: + write_start = max(progress_start, progress_end - 5) + write_end = max(write_start, progress_end - 2) + await self._progress( + task_id, + f"Writing orbit asset index: changed_or_new={len(rows)}, skipped={skipped_unchanged}", + write_start, + ) + if task_id: + await task_service.add_log( + task_id, + "INFO", + f"Orbit asset DB upsert started: changed_or_new={len(rows)}, skipped={skipped_unchanged}, seen={len(seen_paths)}, issues={len(issues)}", + ) + for index, row in enumerate(rows, start=1): stmt = pg_insert(OrbitAssetORM).values(row) excluded = stmt.excluded stmt = stmt.on_conflict_do_update( @@ -1593,7 +3004,70 @@ class AssetInventoryService: }, ) await db.execute(stmt) + if index % ASSET_SCAN_LOG_INTERVAL == 0 or index == len(rows): + progress_value = write_start + if rows and write_end > write_start: + progress_value = write_start + int(index / max(1, len(rows)) * (write_end - write_start)) + await self._progress( + task_id, + f"Writing orbit asset index: {index}/{len(rows)} changed_or_new, skipped={skipped_unchanged}", + progress_value, + ) + await db.flush() + derivative_summary: Dict[str, Any] = {} + has_lt1_seen = any( + str(existing_by_path.get(path, {}).get("native_format") or "").upper() == "TXT" + for path in seen_paths + ) or any(row.get("satellite_family") == "LT1" for row in rows) + if has_lt1_seen and settings.ORBIT_POOL_ENVI: + await self._progress(task_id, "Syncing LT-1 TXT orbit production pool...", max(progress_end - 3, progress_start)) + isce2_pool = settings.ORBIT_POOL_ISCE2 if settings.ISCE2_ENABLED else "" + try: + derivative_summary = await asyncio.to_thread( + sync_orbit_pools, + root.path, + settings.ORBIT_POOL_ENVI, + isce2_pool, + settings.ORBIT_POOL_LANDSAR, + bool(settings.ISCE2_ENABLED), + ) + except Exception as exc: + derivative_summary = {"error": str(exc)} + issues.append( + { + "severity": "warning", + "issue_code": "lt1_orbit_pool_sync_failed", + "issue_message": str(exc), + "source_path": root.path, + } + ) + + derivative_summary["db_derivatives"] = await self._record_lt1_orbit_pool_derivatives_for_paths(db, seen_paths, now=now) + if task_id: + envi_summary = derivative_summary.get("envi") or {} + isce2_summary = derivative_summary.get("isce2") or {} + db_derivatives = derivative_summary.get("db_derivatives") or {} + await task_service.add_log( + task_id, + "INFO" if not derivative_summary.get("error") else "WARN", + ( + "LT-1 orbit production pool sync: " + f"txt copied={len(envi_summary.get('copied', []) or [])}, " + f"updated={len(envi_summary.get('updated', []) or [])}, " + f"skipped={len(envi_summary.get('skipped', []) or [])}; " + + ( + f"isce2 converted={len(isce2_summary.get('converted', []) or [])}, " + f"reconverted={len(isce2_summary.get('reconverted', []) or [])}; " + if settings.ISCE2_ENABLED + else "isce2 disabled; " + ) + + + f"derivatives recorded={int(db_derivatives.get('recorded') or 0)}" + ), + ) + + await self._progress(task_id, "Marking missing orbit assets and refreshing scan issues...", max(progress_end - 2, progress_start)) await self._mark_missing_orbit_assets(db, root, seen_paths, now) await self._replace_root_issues(db, root, "orbit_asset", issues) await self._finish_state( @@ -1602,18 +3076,29 @@ class AssetInventoryService: status="OK" if not any(item.get("severity") == "error" for item in issues) else "WARNING", started_at=started_at, entry_count=entry_count, - asset_count=len(rows), + asset_count=len(seen_paths), issue_count=len(issues), error=None, ) + if task_id: + await task_service.add_log( + task_id, + "INFO", + "Orbit root scan summary: " + f"path={root.path}, candidates={entry_count}, active={len(seen_paths)}, " + f"changed_or_new={len(rows)}, skipped={skipped_unchanged}, issues={len(issues)}", + ) return { "root_id": root.id, "root_path": root.path, "inventory_type": "orbit_asset", "status": state.status, "entry_count": entry_count, - "asset_count": len(rows), + "asset_count": len(seen_paths), + "changed_asset_count": len(rows), + "unchanged_asset_count": skipped_unchanged, "issue_count": len(issues), + "derivative_summary": derivative_summary, } async def _find_source_asset_root( @@ -1639,7 +3124,7 @@ class AssetInventoryService: db: AsyncSession, root_path: str, *, - source_ref: str = "SENTINEL1_STORAGE_DIRS", + source_ref: str = "TASK_POOL_ROOT", ) -> ManagedRootORM: from .root_registry_service import root_registry_service @@ -1653,21 +3138,23 @@ class AssetInventoryService: return root normalized = _normalize_path(root_path) - root_code = f"source_product_pool__sentinel1_storage_{hashlib.sha1(normalized.encode('utf-8')).hexdigest()[:12]}" + source_ref_text = str(source_ref or "TASK_POOL_ROOT").strip() or "TASK_POOL_ROOT" + source_ref_slug = re.sub(r"[^a-z0-9]+", "_", source_ref_text.lower()).strip("_") or "task_pool_root" + root_code = f"source_product_pool__{source_ref_slug}_{hashlib.sha1(normalized.encode('utf-8')).hexdigest()[:12]}" root = ManagedRootORM( root_code=root_code, root_role="source_product_pool", - display_name="Sentinel-1 Storage Pool", + display_name="Task Pool Materialized Source Pool" if source_ref_text == "TASK_POOL_ROOT" else "Source Product Pool", path=normalized, path_kind=_path_kind(normalized), source_kind="env", - source_ref=source_ref, + source_ref=source_ref_text, scan_mode="file_pool", enabled=True, exists_flag=os.path.exists(normalized), metadata_json={ - "env_var": source_ref, - "created_by": "sentinel1_unpack", + "env_var": source_ref_text, + "created_by": "source_materialize", }, ) db.add(root) @@ -1700,7 +3187,8 @@ class AssetInventoryService: } now = _utcnow() - stmt = pg_insert(SourceProductAssetORM).values(row) + db_row = {key: value for key, value in row.items() if not str(key).startswith("_")} + stmt = pg_insert(SourceProductAssetORM).values(db_row) excluded = stmt.excluded stmt = stmt.on_conflict_do_update( index_elements=["file_path"], @@ -1730,6 +3218,12 @@ class AssetInventoryService: "size_bytes": excluded.size_bytes, "mtime_epoch": excluded.mtime_epoch, "checksum_status": excluded.checksum_status, + "archive_integrity_status": "NOT_CHECKED", + "archive_integrity_method": None, + "archive_integrity_checked_at": None, + "archive_integrity_error": None, + "archive_integrity_version": None, + "archive_integrity_member_count": None, "parser_name": excluded.parser_name, "parser_version": excluded.parser_version, "parse_status": excluded.parse_status, @@ -1750,6 +3244,7 @@ class AssetInventoryService: asset_id = result.scalar_one_or_none() radar_data_id = None if asset_id is not None: + await self._upsert_metadata_documents_for_source_assets(db, [row], {path: int(asset_id)}) await self._upsert_radar_records_for_source_assets(db, [row], {path: int(asset_id)}) radar_result = await db.execute( select(RadarDataORM.id).where(RadarDataORM.file_path == path) @@ -1781,23 +3276,21 @@ class AssetInventoryService: log_callback: Optional[Callable[[str, str], None]] = None, ) -> Dict[str, Any]: archive = _normalize_path(archive_path) + _ensure_local_runtime_path(archive, "Sentinel-1 archive source") if not os.path.isfile(archive): raise FileNotFoundError(archive) if not archive.lower().endswith(".zip"): raise ValueError("Only Sentinel-1 ZIP archives can be unpacked.") - target_dir = _target_root_for_s1_archive(archive, target_root) + target_dir = _ensure_local_runtime_path( + _target_root_for_s1_archive(archive, target_root), + "Sentinel-1 unpack target_root", + ) os.makedirs(target_dir, exist_ok=True) tmp_suffix_text = str(tmp_suffix or os.getenv("UNPACK_TMP_SUFFIX") or ".unpack_tmp").strip() or ".unpack_tmp" min_free_gb = min_disk_space_gb if min_free_gb is None: min_free_gb = _parse_float(os.getenv("UNPACK_MIN_DISK_SPACE_GB"), 50.0) - should_delete_archive = ( - bool(delete_archive) - if delete_archive is not None - else _parse_bool(os.getenv("UNPACK_DELETE_ARCHIVE"), False) - ) - def _log(level: str, message: str) -> None: if log_callback: log_callback(level, message) @@ -1806,6 +3299,9 @@ class AssetInventoryService: if progress_callback: progress_callback(progress, message) + if delete_archive: + _log("WARNING", "Ignoring delete_archive=true; Sentinel-1 ZIP archives are the source of record.") + _progress(3, "Reading Sentinel-1 ZIP manifest...") with zipfile.ZipFile(archive) as zip_obj: names = zip_obj.namelist() @@ -1897,10 +3393,6 @@ class AssetInventoryService: except OSError: pass - if should_delete_archive: - os.remove(archive) - _log("INFO", f"Deleted Sentinel-1 ZIP after unpack: {archive}") - _progress(92, "Sentinel-1 SAFE extracted.") return { "status": "EXTRACTED", @@ -1922,8 +3414,11 @@ class AssetInventoryService: source_path = _normalize_path(str(asset.archive_path or asset.file_path or "")) if not source_path: raise ValueError("Source asset path is empty.") + _ensure_local_runtime_path(source_path, "Source asset path") if source_format == "S1_ZIP": - return self.unpack_sentinel1_archive(source_path, target_root=target_root, overwrite=overwrite) + requested_root = _normalize_path(target_root or "") or _task_pool_materialize_root(source_format) + requested_root = _ensure_local_runtime_path(requested_root, "Source materialize target_root") + return self.unpack_sentinel1_archive(source_path, target_root=requested_root, overwrite=overwrite) if source_format not in {"LT1_ARCHIVE", "GF3_ARCHIVE"}: if os.path.isdir(source_path): return { @@ -1937,7 +3432,8 @@ class AssetInventoryService: requested_root = _normalize_path(target_root or "") if not requested_root: - requested_root = _normalize_path(os.path.join(settings.PYINT_WORK_ROOT, "source_materialized", source_format.lower())) + requested_root = _task_pool_materialize_root(source_format) + requested_root = _ensure_local_runtime_path(requested_root, "Source materialize target_root") scene_name = _strip_known_suffix(os.path.basename(source_path)) target_dir = os.path.join(requested_root, scene_name) result = _extract_archive_to_dir(source_path, target_dir, overwrite=overwrite) @@ -1960,7 +3456,10 @@ class AssetInventoryService: raise ValueError("Only Sentinel-1 ZIP assets can be unpacked.") archive_path = asset.file_path - target_root = payload.get("target_root") or None + target_root = _ensure_local_runtime_path( + payload.get("target_root") or _task_pool_materialize_root("S1_ZIP"), + "Sentinel-1 unpack target_root", + ) overwrite = bool(payload.get("overwrite", False)) min_disk_space_gb = payload.get("min_disk_space_gb") delete_archive = payload.get("delete_archive") if "delete_archive" in payload else None @@ -1993,7 +3492,12 @@ class AssetInventoryService: ) if result.get("target_root"): - await self.ensure_source_root_for_path(db, str(result["target_root"])) + target_root_text = str(result["target_root"]) + await self.ensure_source_root_for_path( + db, + target_root_text, + source_ref=_source_ref_for_materialized_root(target_root_text), + ) await db.commit() metadata = dict(asset.metadata_json or {}) @@ -2028,7 +3532,10 @@ class AssetInventoryService: overwrite = bool(payload.get("overwrite", False)) min_disk_space_gb = payload.get("min_disk_space_gb") delete_archive = payload.get("delete_archive") if "delete_archive" in payload else None - target_root = payload.get("target_root") or None + target_root = _ensure_local_runtime_path( + payload.get("target_root") or _task_pool_materialize_root("S1_ZIP"), + "Sentinel-1 batch unpack target_root", + ) scan_before_unpack = bool(payload.get("scan_before_unpack", True)) await task_service.start_task(task_id, message="Sentinel-1 batch unpack started") @@ -2111,7 +3618,12 @@ class AssetInventoryService: log_callback=_log, ) if result.get("target_root"): - await self.ensure_source_root_for_path(db, str(result["target_root"])) + target_root_text = str(result["target_root"]) + await self.ensure_source_root_for_path( + db, + target_root_text, + source_ref=_source_ref_for_materialized_root(target_root_text), + ) await db.commit() scan_summary: Dict[str, Any] = {} @@ -2235,6 +3747,62 @@ class AssetInventoryService: ) ) + async def _resolve_archive_integrity_issue( + self, + db: AsyncSession, + asset: SourceProductAssetORM, + *, + now: datetime, + ) -> None: + if asset.id is None: + return + await db.execute( + update(AssetInventoryIssueORM) + .where( + AssetInventoryIssueORM.asset_ref_id == int(asset.id), + AssetInventoryIssueORM.inventory_type == "source_product", + AssetInventoryIssueORM.issue_code == "source_archive_integrity_failed", + AssetInventoryIssueORM.status == "OPEN", + ) + .values(status="RESOLVED", resolved_at=now, last_seen_at=now) + ) + + async def _record_archive_integrity_issue( + self, + db: AsyncSession, + asset: SourceProductAssetORM, + result: Dict[str, Any], + *, + now: datetime, + ) -> None: + if asset.id is None: + return + await self._resolve_archive_integrity_issue(db, asset, now=now) + db.add( + AssetInventoryIssueORM( + root_ref_id=asset.root_ref_id, + inventory_type="source_product", + asset_ref_id=int(asset.id), + severity="error", + issue_code="source_archive_integrity_failed", + issue_message=result.get("error") or "Source archive integrity check failed.", + source_path=asset.file_path, + status="OPEN", + first_seen_at=now, + last_seen_at=now, + metadata_json={ + "asset_uid": asset.asset_uid, + "logical_product_uid": asset.logical_product_uid, + "satellite_family": asset.satellite_family, + "source_format": asset.source_format, + "method": result.get("method"), + "member_count": result.get("member_count"), + "duration_seconds": result.get("duration_seconds"), + "version": ARCHIVE_INTEGRITY_VERSION, + }, + ) + ) + async def _mark_missing_source_assets( self, db: AsyncSession, @@ -2259,6 +3827,110 @@ class AssetInventoryService: stmt = stmt.where(OrbitAssetORM.file_path.notin_(list(seen_paths))) await db.execute(stmt.values(is_active=False, missing_since=now, updated_at=now)) + async def _record_lt1_orbit_pool_derivatives( + self, + db: AsyncSession, + rows: Sequence[Dict[str, Any]], + *, + now: datetime, + ) -> Dict[str, Any]: + lt1_rows = [row for row in rows if row.get("satellite_family") == "LT1"] + if not lt1_rows or not settings.ORBIT_POOL_ENVI: + return {"recorded": 0, "missing": 0} + + paths = [str(row.get("file_path") or "") for row in lt1_rows if row.get("file_path")] + result = await db.execute(select(OrbitAssetORM).where(OrbitAssetORM.file_path.in_(paths))) + assets_by_path = {str(asset.file_path): asset for asset in result.scalars().all()} + + recorded = 0 + missing = 0 + for row in lt1_rows: + asset = assets_by_path.get(str(row.get("file_path") or "")) + if not asset or not asset.id: + continue + satellite = str(row.get("satellite") or "").upper() + file_name = str(row.get("file_name") or "").strip() + pool_path = _normalize_path(os.path.join(settings.ORBIT_POOL_ENVI, satellite, file_name)) + if not os.path.isfile(pool_path): + missing += 1 + continue + stat = _stat_path(pool_path) + stmt = pg_insert(OrbitAssetDerivativeORM).values( + orbit_asset_id=int(asset.id), + engine_code="lt1_txt_pool", + derivative_format="LT1_TXT", + derivative_role="production_orbit_txt", + pool_path=pool_path, + size_bytes=stat.get("size_bytes"), + mtime_epoch=stat.get("mtime_epoch"), + checksum_sha256=None, + generation_status="READY", + generation_error=None, + generated_at=now, + metadata_json=_json_safe( + { + "pool_root": settings.ORBIT_POOL_ENVI, + "layout": "satellite_split", + "consumers": ["ENVI/SARscape", "Gamma/PyINT D-InSAR", "Gamma SBAS"], + } + ), + created_at=now, + updated_at=now, + ) + excluded = stmt.excluded + stmt = stmt.on_conflict_do_update( + index_elements=["orbit_asset_id", "engine_code", "derivative_format", "pool_path"], + set_={ + "derivative_role": excluded.derivative_role, + "size_bytes": excluded.size_bytes, + "mtime_epoch": excluded.mtime_epoch, + "generation_status": "READY", + "generation_error": None, + "generated_at": now, + "metadata_json": excluded.metadata_json, + "updated_at": now, + }, + ) + await db.execute(stmt) + recorded += 1 + return {"recorded": recorded, "missing": missing, "pool_root": settings.ORBIT_POOL_ENVI} + + async def _record_lt1_orbit_pool_derivatives_for_paths( + self, + db: AsyncSession, + paths: Sequence[str], + *, + now: datetime, + ) -> Dict[str, Any]: + unique_paths = [] + seen = set() + for path in paths: + normalized = _normalize_path(str(path or "")) + if not normalized or normalized in seen: + continue + seen.add(normalized) + unique_paths.append(normalized) + if not unique_paths or not settings.ORBIT_POOL_ENVI: + return {"recorded": 0, "missing": 0} + + result = await db.execute( + select(OrbitAssetORM).where( + OrbitAssetORM.file_path.in_(unique_paths), + OrbitAssetORM.satellite_family == "LT1", + OrbitAssetORM.is_active == True, # noqa: E712 + ) + ) + rows = [ + { + "satellite_family": asset.satellite_family, + "satellite": asset.satellite, + "file_name": asset.file_name, + "file_path": asset.file_path, + } + for asset in result.scalars().all() + ] + return await self._record_lt1_orbit_pool_derivatives(db, rows, now=now) + async def _upsert_radar_records_for_source_assets( self, db: AsyncSession, @@ -2266,11 +3938,14 @@ class AssetInventoryService: asset_ids_by_path: Dict[str, int], ) -> None: dirty_scene_ids: List[int] = [] + profile_inputs: List[Tuple[Dict[str, Any], int, int]] = [] for row in rows: metadata = dict(row.get("metadata_json") or {}) - coverage_polygon = metadata.get("coverage_polygon") + coverage_polygon = _ordered_closed_polygon(metadata.get("coverage_polygon") or []) if not coverage_polygon or len(coverage_polygon) < 3: continue + metadata["coverage_polygon"] = coverage_polygon + metadata["coverage_bbox"] = _bbox_from_polygon(coverage_polygon) family = normalize_satellite_family(row.get("satellite_family") or row.get("satellite")) if family not in {"S1", "LT1"}: continue @@ -2290,8 +3965,6 @@ class AssetInventoryService: asset_id = asset_ids_by_path.get(str(row.get("file_path"))) if not asset_id: continue - if await self._s1_zip_has_unpacked_safe(db, row): - continue archive_asset_id = await self._resolve_archive_asset_id_for_source_row(db, row, asset_id) center_lon, center_lat = _centroid_from_polygon(coverage_polygon) metadata_center_lon = metadata.get("scene_center_lon") @@ -2352,15 +4025,18 @@ class AssetInventoryService: ) existing = result.scalar_one_or_none() if existing is None: - db.add( - RadarDataORM( - unique_id=f"asset:{row.get('asset_uid')}", - has_orbit_data=False, - orbit_binding_status="UNBOUND", - is_envi_processed=False, - **radar_values, - ) + scene = RadarDataORM( + unique_id=f"asset:{row.get('asset_uid')}", + has_orbit_data=False, + orbit_binding_status="UNBOUND", + is_envi_processed=False, + **radar_values, ) + db.add(scene) + await db.flush() + if scene.id is not None: + dirty_scene_ids.append(int(scene.id)) + profile_inputs.append((row, asset_id, int(scene.id))) else: before_orbit_id = existing.selected_orbit_asset_id for key, value in radar_values.items(): @@ -2368,13 +4044,199 @@ class AssetInventoryService: if not existing.orbit_binding_status: existing.orbit_binding_status = "UNBOUND" db.add(existing) + if existing.id is not None: + profile_inputs.append((row, asset_id, int(existing.id))) if existing.id is not None and before_orbit_id != existing.selected_orbit_asset_id: dirty_scene_ids.append(int(existing.id)) await db.flush() + if profile_inputs: + await self._upsert_geometry_profiles(db, profile_inputs) + await self._attach_radar_ids_to_metadata_documents(db, profile_inputs) + for _, _, radar_id in profile_inputs: + if radar_id not in dirty_scene_ids: + dirty_scene_ids.append(radar_id) if dirty_scene_ids: await pairing_state_service.mark_scenes_dirty(db, scene_ids=dirty_scene_ids, reason="asset_inventory_source_update", commit=False) + async def _upsert_metadata_documents_for_source_assets( + self, + db: AsyncSession, + rows: Sequence[Dict[str, Any]], + asset_ids_by_path: Dict[str, int], + ) -> None: + now = _utcnow() + for row in rows: + asset_id = asset_ids_by_path.get(str(row.get("file_path"))) + if not asset_id: + continue + for doc in row.get("_metadata_documents") or []: + values = { + "source_asset_id": int(asset_id), + "satellite_family": doc.get("satellite_family") or row.get("satellite_family"), + "source_format": doc.get("source_format") or row.get("source_format"), + "document_type": doc.get("document_type") or "UNKNOWN", + "member_path": doc.get("member_path") or "", + "content_sha256": doc.get("content_sha256") or "", + "content_encoding": doc.get("content_encoding") or "gzip", + "content_bytes": doc.get("content_bytes") or b"", + "content_size_bytes": doc.get("content_size_bytes"), + "archive_path": doc.get("archive_path") or row.get("archive_path") or row.get("file_path"), + "archive_mtime": doc.get("archive_mtime") if doc.get("archive_mtime") is not None else row.get("mtime_epoch"), + "parser_version": doc.get("parser_version") or PARSER_VERSION, + "parse_status": doc.get("parse_status") or "OK", + "parse_error": doc.get("parse_error"), + "extracted_at": doc.get("extracted_at") or now, + "updated_at": now, + } + stmt = pg_insert(SourceMetadataDocumentORM).values(values) + excluded = stmt.excluded + stmt = stmt.on_conflict_do_update( + constraint="uq_source_metadata_document_member", + set_={ + "radar_data_id": excluded.radar_data_id, + "satellite_family": excluded.satellite_family, + "source_format": excluded.source_format, + "content_sha256": excluded.content_sha256, + "content_encoding": excluded.content_encoding, + "content_bytes": excluded.content_bytes, + "content_size_bytes": excluded.content_size_bytes, + "archive_path": excluded.archive_path, + "archive_mtime": excluded.archive_mtime, + "parser_version": excluded.parser_version, + "parse_status": excluded.parse_status, + "parse_error": excluded.parse_error, + "extracted_at": excluded.extracted_at, + "updated_at": now, + }, + ) + await db.execute(stmt) + await db.flush() + + async def _attach_radar_ids_to_metadata_documents( + self, + db: AsyncSession, + profile_inputs: Sequence[Tuple[Dict[str, Any], int, int]], + ) -> None: + for row, asset_id, radar_id in profile_inputs: + await db.execute( + update(SourceMetadataDocumentORM) + .where(SourceMetadataDocumentORM.source_asset_id == int(asset_id)) + .values( + radar_data_id=int(radar_id), + satellite_family=row.get("satellite_family"), + source_format=row.get("source_format"), + updated_at=_utcnow(), + ) + ) + + async def _upsert_geometry_profiles( + self, + db: AsyncSession, + profile_inputs: Sequence[Tuple[Dict[str, Any], int, int]], + ) -> None: + now = _utcnow() + for row, asset_id, radar_id in profile_inputs: + metadata = dict(row.get("metadata_json") or {}) + footprint = _ordered_closed_polygon(metadata.get("coverage_polygon") or []) + footprint_geom = None + if footprint and len(footprint) >= 4: + try: + poly = Polygon(footprint) + if not poly.is_valid: + poly = poly.buffer(0) + if not poly.is_empty: + footprint_geom = from_shape(poly, srid=4326) + except Exception: + footprint_geom = None + + reasons: List[str] = [] + for key, reason in ( + ("satellite_family", "missing_satellite_family"), + ("imaging_mode", "missing_imaging_mode"), + ("polarization", "missing_polarization"), + ("orbit_direction", "missing_orbit_direction"), + ): + if not row.get(key): + reasons.append(reason) + if not footprint: + reasons.append("missing_footprint") + family = normalize_satellite_family(row.get("satellite_family") or row.get("satellite")) + relative_orbit = row.get("relative_orbit") + if family == "S1" and not relative_orbit: + reasons.append("missing_relative_orbit") + + metadata_quality = "READY" if not reasons else ("PARTIAL" if footprint else "INCOMPLETE") + production_readiness = "READY" if metadata_quality == "READY" else "CANDIDATE" + values = { + "source_asset_id": int(asset_id), + "radar_data_id": int(radar_id), + "satellite_family": family, + "satellite": row.get("satellite"), + "source_format": row.get("source_format"), + "imaging_mode": row.get("imaging_mode"), + "polarization": row.get("polarization"), + "orbit_direction": row.get("orbit_direction"), + "look_direction": metadata.get("look_direction"), + "absolute_orbit": row.get("absolute_orbit"), + "relative_orbit": relative_orbit, + "acquisition_start_time_utc": row.get("acquisition_start_time_utc"), + "acquisition_stop_time_utc": row.get("acquisition_stop_time_utc"), + "scene_center_lon": metadata.get("scene_center_lon"), + "scene_center_lat": metadata.get("scene_center_lat"), + "footprint_geom": footprint_geom, + "footprint_polygon": _json_safe(footprint), + "swath_summary_json": metadata.get("swath_summary"), + "burst_summary_json": metadata.get("burst_summary"), + "incidence_angle_min": metadata.get("incidence_angle_min"), + "incidence_angle_max": metadata.get("incidence_angle_max"), + "doppler_summary_json": metadata.get("doppler_summary"), + "state_vector_summary_json": metadata.get("state_vector_summary"), + "metadata_quality": metadata_quality, + "production_readiness": production_readiness, + "readiness_reasons_json": reasons, + "parser_version": PARSER_VERSION, + "parsed_at": now, + "updated_at": now, + } + stmt = pg_insert(SARSceneGeometryProfileORM).values(values) + excluded = stmt.excluded + stmt = stmt.on_conflict_do_update( + index_elements=["source_asset_id"], + set_={ + "radar_data_id": excluded.radar_data_id, + "satellite_family": excluded.satellite_family, + "satellite": excluded.satellite, + "source_format": excluded.source_format, + "imaging_mode": excluded.imaging_mode, + "polarization": excluded.polarization, + "orbit_direction": excluded.orbit_direction, + "look_direction": excluded.look_direction, + "absolute_orbit": excluded.absolute_orbit, + "relative_orbit": excluded.relative_orbit, + "acquisition_start_time_utc": excluded.acquisition_start_time_utc, + "acquisition_stop_time_utc": excluded.acquisition_stop_time_utc, + "scene_center_lon": excluded.scene_center_lon, + "scene_center_lat": excluded.scene_center_lat, + "footprint_geom": excluded.footprint_geom, + "footprint_polygon": excluded.footprint_polygon, + "swath_summary_json": excluded.swath_summary_json, + "burst_summary_json": excluded.burst_summary_json, + "incidence_angle_min": excluded.incidence_angle_min, + "incidence_angle_max": excluded.incidence_angle_max, + "doppler_summary_json": excluded.doppler_summary_json, + "state_vector_summary_json": excluded.state_vector_summary_json, + "metadata_quality": excluded.metadata_quality, + "production_readiness": excluded.production_readiness, + "readiness_reasons_json": excluded.readiness_reasons_json, + "parser_version": excluded.parser_version, + "parsed_at": excluded.parsed_at, + "updated_at": now, + }, + ) + await db.execute(stmt) + await db.flush() + async def _s1_zip_has_unpacked_safe(self, db: AsyncSession, row: Dict[str, Any]) -> bool: if row.get("source_format") != "S1_ZIP": return False @@ -2421,28 +4283,47 @@ class AssetInventoryService: archive_asset_id: Optional[int], ): logical_uid = str(row.get("logical_product_uid") or "").strip() + file_path_match = RadarDataORM.file_path == row.get("file_path") + unique_id_match = RadarDataORM.unique_id == f"asset:{row.get('asset_uid')}" clauses = [ - RadarDataORM.file_path == row.get("file_path"), - RadarDataORM.unique_id == f"asset:{row.get('asset_uid')}", + file_path_match, + unique_id_match, + ] + priority = [ + (file_path_match, 0), + (unique_id_match, 1), ] if archive_asset_id is not None: - clauses.append(RadarDataORM.source_archive_asset_id == int(archive_asset_id)) + archive_match = RadarDataORM.source_archive_asset_id == int(archive_asset_id) + clauses.append(archive_match) + priority.append((archive_match, 2)) if row.get("source_format") == "S1_SAFE_DIR" and logical_uid: - clauses.append( - and_( - RadarDataORM.satellite_family == "S1", - RadarDataORM.product_unique_id == logical_uid, - ) + logical_match = and_( + RadarDataORM.satellite_family == "S1", + RadarDataORM.product_unique_id == logical_uid, ) + clauses.append(logical_match) + priority.append((logical_match, 3)) elif row.get("source_format") == "S1_ZIP" and logical_uid: - clauses.append( - and_( - RadarDataORM.satellite_family == "S1", - RadarDataORM.product_unique_id == logical_uid, - RadarDataORM.source_archive_asset_id == int(asset_id), - ) + logical_match = and_( + RadarDataORM.satellite_family == "S1", + RadarDataORM.product_unique_id == logical_uid, ) - return select(RadarDataORM).where(or_(*clauses)) + clauses.append(logical_match) + priority.append((logical_match, 3)) + elif row.get("source_format") == "LT1_ARCHIVE" and logical_uid: + logical_match = and_( + RadarDataORM.satellite_family == "LT1", + RadarDataORM.product_unique_id == logical_uid, + ) + clauses.append(logical_match) + priority.append((logical_match, 3)) + return ( + select(RadarDataORM) + .where(or_(*clauses)) + .order_by(case(*priority, else_=9), RadarDataORM.id.asc()) + .limit(1) + ) async def bind_scene_orbits( self, @@ -2628,13 +4509,23 @@ class AssetInventoryService: return [] async def get_status(self, db: AsyncSession) -> Dict[str, Any]: - state_rows = ( + root_rows = ( await db.execute( - select(AssetInventoryStateORM) - .join(ManagedRootORM, AssetInventoryStateORM.root_ref_id == ManagedRootORM.id) - .order_by(AssetInventoryStateORM.inventory_type.asc(), AssetInventoryStateORM.root_path.asc()) + select(ManagedRootORM) + .where(ManagedRootORM.enabled == True) # noqa: E712 + .where(ManagedRootORM.root_role.in_(["source_product_pool", "orbit_asset_pool"])) + .order_by(ManagedRootORM.root_role.asc(), ManagedRootORM.path.asc()) ) ).scalars().all() + state_rows = ( + await db.execute( + select(AssetInventoryStateORM, ManagedRootORM) + .join(ManagedRootORM, AssetInventoryStateORM.root_ref_id == ManagedRootORM.id) + .where(ManagedRootORM.enabled == True) # noqa: E712 + .where(ManagedRootORM.root_role.in_(["source_product_pool", "orbit_asset_pool"])) + .order_by(AssetInventoryStateORM.inventory_type.asc(), AssetInventoryStateORM.root_path.asc()) + ) + ).all() source_count = int( ( await db.execute( @@ -2648,16 +4539,54 @@ class AssetInventoryService: orbit_count = int((await db.execute(select(func.count(OrbitAssetORM.id)).where(OrbitAssetORM.is_active == True))).scalar_one() or 0) # noqa: E712 binding_count = int((await db.execute(select(func.count(SceneOrbitBindingORM.id)).where(SceneOrbitBindingORM.selection_status == "SELECTED"))).scalar_one() or 0) open_issue_count = int((await db.execute(select(func.count(AssetInventoryIssueORM.id)).where(AssetInventoryIssueORM.status == "OPEN"))).scalar_one() or 0) + integrity_rows = ( + await db.execute( + select(SourceProductAssetORM.archive_integrity_status, func.count(SourceProductAssetORM.id)) + .where( + SourceProductAssetORM.is_active == True, # noqa: E712 + SourceProductAssetORM.source_format.in_(["LT1_ARCHIVE", "S1_ZIP"]), + ) + .group_by(SourceProductAssetORM.archive_integrity_status) + ) + ).all() + archive_integrity_counts = { + str(status or "NOT_CHECKED").upper(): int(count or 0) + for status, count in integrity_rows + } return { "source_asset_count": source_count, "orbit_asset_count": orbit_count, "selected_binding_count": binding_count, "open_issue_count": open_issue_count, + "archive_integrity_counts": archive_integrity_counts, + "roots": [ + { + "id": row.id, + "root_code": row.root_code, + "root_role": row.root_role, + "display_name": row.display_name, + "root_path": row.path, + "path_kind": row.path_kind, + "source_ref": row.source_ref, + "scan_mode": row.scan_mode, + "enabled": bool(row.enabled), + "exists_flag": bool(row.exists_flag), + "supported_families": _root_supported_families(row), + } + for row in root_rows + ], "states": [ { "id": row.id, "root_ref_id": row.root_ref_id, + "root_role": root.root_role, + "root_code": root.root_code, + "display_name": root.display_name, + "source_ref": root.source_ref, + "enabled": bool(root.enabled), + "exists_flag": bool(root.exists_flag), + "supported_families": _root_supported_families(root), "inventory_type": row.inventory_type, "root_path": row.root_path, "scan_mode": row.scan_mode, @@ -2671,7 +4600,7 @@ class AssetInventoryService: "needs_rescan": bool(row.needs_rescan), "last_error": row.last_error, } - for row in state_rows + for row, root in state_rows ], } @@ -2692,8 +4621,9 @@ class AssetInventoryService: filters = [] if not include_inactive: filters.append(SourceProductAssetORM.is_active == True) # noqa: E712 - if satellite_family: - filters.append(SourceProductAssetORM.satellite_family == satellite_family.upper()) + family_filter = self._normalize_family_filter(satellite_family) + if family_filter: + filters.append(SourceProductAssetORM.satellite_family.in_(family_filter)) if satellite: filters.append(SourceProductAssetORM.satellite == satellite.upper()) if source_format: @@ -2738,8 +4668,9 @@ class AssetInventoryService: filters = [] if not include_inactive: filters.append(OrbitAssetORM.is_active == True) # noqa: E712 - if satellite_family: - filters.append(OrbitAssetORM.satellite_family == satellite_family.upper()) + family_filter = self._normalize_family_filter(satellite_family) + if family_filter: + filters.append(OrbitAssetORM.satellite_family.in_(family_filter)) if satellite: filters.append(OrbitAssetORM.satellite == satellite.upper()) if orbit_type: @@ -2851,6 +4782,12 @@ class AssetInventoryService: "size_bytes": row.size_bytes, "mtime_epoch": row.mtime_epoch, "checksum_status": row.checksum_status, + "archive_integrity_status": row.archive_integrity_status, + "archive_integrity_method": row.archive_integrity_method, + "archive_integrity_checked_at": row.archive_integrity_checked_at, + "archive_integrity_error": row.archive_integrity_error, + "archive_integrity_version": row.archive_integrity_version, + "archive_integrity_member_count": row.archive_integrity_member_count, "parser_name": row.parser_name, "parser_version": row.parser_version, "parse_status": row.parse_status, diff --git a/backend/app/services/data_service.py b/backend/app/services/data_service.py index 8cc7e6a..7afa8ab 100644 --- a/backend/app/services/data_service.py +++ b/backend/app/services/data_service.py @@ -16,6 +16,8 @@ import json import asyncio import hashlib import re +import tarfile +import zipfile from typing import List, Dict, Optional, Tuple, Any from datetime import datetime from shapely.geometry import Point, Polygon, shape, mapping @@ -240,6 +242,8 @@ def _chunked(items: List[Any], size: int): _RADAR_PREVIEW_EXTENSIONS = (".jpg", ".jpeg", ".png", ".webp", ".bmp", ".tif", ".tiff") _RADAR_PREVIEW_KEYWORDS = ("quicklook", "quick-look", "preview", "browse", "thumbnail", "thumb", "overview") _RADAR_CACHE_NAME_RE = re.compile(r"[^A-Za-z0-9._-]+") +_RADAR_ARCHIVE_SUFFIXES = (".zip", ".tar", ".tgz", ".tar.gz") +_RADAR_ARCHIVE_PREVIEW_MAX_BYTES = 256 * 1024 * 1024 def _sanitize_cache_name(name: str) -> str: @@ -265,6 +269,208 @@ def _build_radar_geo_cache_filename(unique_id: str, file_path: str, cache_versio return f"RGID_{safe_version}_{digest}_{base}.webp" +def _has_radar_archive_suffix(path: str) -> bool: + return str(path or "").lower().endswith(_RADAR_ARCHIVE_SUFFIXES) + + +def _archive_product_stem(path: str) -> str: + base = os.path.basename(str(path or "")) + lower_base = base.lower() + for suffix in (".tar.gz", ".tgz", ".tar", ".zip"): + if lower_base.endswith(suffix): + return base[: -len(suffix)] + return os.path.splitext(base)[0] + + +def _radar_archive_expected_preview_rank(archive_path: str, member_name: str) -> Optional[int]: + product_stem = _archive_product_stem(archive_path) + if not product_stem: + return None + + member = str(member_name or "").replace("\\", "/").strip("/") + member_lower = member.lower() + product_lower = product_stem.lower() + expected_names = [ + f"{product_lower}/{product_lower}.browse.jpg", + f"{product_lower}/{product_lower}.browse.jpeg", + f"{product_lower}/{product_lower}.browse.png", + f"{product_lower}/{product_lower}.quicklook.jpg", + f"{product_lower}/{product_lower}.quicklook.png", + f"{product_lower}/{product_lower}.quick-look.png", + f"{product_lower}/{product_lower}.thumb.jpg", + f"{product_lower}/{product_lower}.thumb.jpeg", + f"{product_lower}/{product_lower}.thumb.png", + f"{product_lower}/preview/quick-look.png", + f"{product_lower}.browse.jpg", + f"{product_lower}.browse.jpeg", + f"{product_lower}.browse.png", + f"{product_lower}.thumb.jpg", + f"{product_lower}.thumb.jpeg", + f"{product_lower}.thumb.png", + ] + try: + return expected_names.index(member_lower) + except ValueError: + return None + + +def _radar_archive_preview_score(member_name: str, size_bytes: int = 0) -> Optional[Tuple[int, int, int, int, str]]: + lower_name = str(member_name or "").replace("\\", "/").lower() + base_name = os.path.basename(lower_name) + if not base_name.endswith(_RADAR_PREVIEW_EXTENSIONS): + return None + if size_bytes and size_bytes > _RADAR_ARCHIVE_PREVIEW_MAX_BYTES: + return None + + has_keyword = any(key in lower_name for key in _RADAR_PREVIEW_KEYWORDS) + if base_name.endswith((".tif", ".tiff")) and not has_keyword: + return None + + keyword_score = 0 if has_keyword else 2 + if base_name == "quick-look.png": + keyword_score = -4 + elif base_name == "quicklook.png": + keyword_score = -3 + elif base_name.startswith("quick-look.") or base_name.startswith("quicklook."): + keyword_score = min(keyword_score, -2) + if "/preview/" in lower_name: + keyword_score -= 1 + if "/icons/" in lower_name: + keyword_score += 2 + if base_name.endswith((".jpg", ".jpeg", ".png", ".webp")): + ext_score = 0 + elif base_name.endswith(".bmp"): + ext_score = 1 + else: + ext_score = 2 + depth = lower_name.count("/") + size_score = -int(size_bytes or 0) + return (keyword_score, depth, ext_score, size_score, member_name) + + +def _radar_archive_preview_cache_path(archive_path: str, member_name: str) -> str: + digest = hashlib.sha1( + f"{archive_path}|{member_name}".encode("utf-8", errors="ignore") + ).hexdigest()[:16] + raw_base = os.path.basename(str(member_name or "preview")) + base = _sanitize_cache_name(raw_base) + _, ext = os.path.splitext(raw_base) + ext = _RADAR_CACHE_NAME_RE.sub("", ext.lower())[:12] + if ext and not base.lower().endswith(ext): + base = f"{base[:max(1, 48 - len(ext))]}{ext}" + return os.path.join(settings.CACHE_DIR, "radar_archive_preview_sources", f"APS_{digest}_{base}") + + +def _write_archive_preview_cache(target_path: str, source_obj: Any, archive_mtime: float) -> Optional[str]: + try: + os.makedirs(os.path.dirname(target_path), exist_ok=True) + if os.path.exists(target_path) and _safe_mtime(target_path) >= archive_mtime: + return target_path + tmp_path = f"{target_path}.tmp" + with open(tmp_path, "wb") as target: + while True: + chunk = source_obj.read(1024 * 1024) + if not chunk: + break + target.write(chunk) + os.replace(tmp_path, target_path) + if archive_mtime: + os.utime(target_path, (archive_mtime, archive_mtime)) + return target_path + except Exception: + try: + if "tmp_path" in locals() and os.path.exists(tmp_path): + os.remove(tmp_path) + except OSError: + pass + return None + + +def _find_radar_archive_preview_source(archive_path: str) -> Optional[str]: + if not archive_path or not os.path.isfile(archive_path) or not _has_radar_archive_suffix(archive_path): + return None + + archive_mtime = _safe_mtime(archive_path) + lower_path = archive_path.lower() + try: + if lower_path.endswith(".zip"): + with zipfile.ZipFile(archive_path) as archive: + candidates = [] + info_by_name = {} + for info in archive.infolist(): + if info.is_dir(): + continue + score = _radar_archive_preview_score(info.filename, int(info.file_size or 0)) + if score is None: + continue + candidates.append(score) + info_by_name[info.filename] = info + if not candidates: + return None + candidates.sort() + member_name = candidates[0][-1] + target_path = _radar_archive_preview_cache_path(archive_path, member_name) + with archive.open(info_by_name[member_name], "r") as source_obj: + return _write_archive_preview_cache(target_path, source_obj, archive_mtime) + + with tarfile.open(archive_path, "r:*") as archive: + candidates = [] + member_by_name = {} + best_rank: Optional[int] = None + best_member: Optional[tarfile.TarInfo] = None + for member in archive: + if not member.isfile(): + continue + expected_rank = _radar_archive_expected_preview_rank(archive_path, member.name) + if expected_rank is not None: + if best_rank is None or expected_rank < best_rank: + best_rank = expected_rank + best_member = member + if expected_rank == 0: + source_obj = archive.extractfile(member) + if source_obj is None: + return None + target_path = _radar_archive_preview_cache_path(archive_path, member.name) + with source_obj: + return _write_archive_preview_cache(target_path, source_obj, archive_mtime) + continue + + score = _radar_archive_preview_score(member.name, int(member.size or 0)) + if score is None: + if best_member is not None and int(member.size or 0) > _RADAR_ARCHIVE_PREVIEW_MAX_BYTES: + break + continue + candidates.append(score) + member_by_name[member.name] = member + if os.path.basename(str(member.name or "").lower()) in {"quick-look.png", "quicklook.png"}: + source_obj = archive.extractfile(member) + if source_obj is None: + return None + target_path = _radar_archive_preview_cache_path(archive_path, member.name) + with source_obj: + return _write_archive_preview_cache(target_path, source_obj, archive_mtime) + if best_member is not None: + source_obj = archive.extractfile(best_member) + if source_obj is None: + return None + target_path = _radar_archive_preview_cache_path(archive_path, best_member.name) + with source_obj: + return _write_archive_preview_cache(target_path, source_obj, archive_mtime) + if not candidates: + return None + candidates.sort() + member_name = candidates[0][-1] + source_obj = archive.extractfile(member_by_name[member_name]) + if source_obj is None: + return None + target_path = _radar_archive_preview_cache_path(archive_path, member_name) + with source_obj: + return _write_archive_preview_cache(target_path, source_obj, archive_mtime) + except Exception: + return None + return None + + def extract_geotiff_bounds(tiff_path: str) -> Optional[List[Tuple[float, float]]]: """Extract coverage polygon from a GeoTIFF file using GDAL. @@ -380,6 +586,8 @@ class DataService: @staticmethod def find_radar_preview_source(scene_dir: str) -> Optional[str]: + if scene_dir and os.path.isfile(scene_dir) and _has_radar_archive_suffix(scene_dir): + return _find_radar_archive_preview_source(scene_dir) if not scene_dir or not os.path.isdir(scene_dir): return None @@ -520,21 +728,26 @@ class DataService: if orbit_dir and orbit_files_map: update_progress("正在同步精轨到本地引擎池...", 8) try: + isce2_pool = settings.ORBIT_POOL_ISCE2 if settings.ISCE2_ENABLED else "" sync_result = await asyncio.to_thread( sync_orbit_pools, orbit_dir, settings.ORBIT_POOL_ENVI, - settings.ORBIT_POOL_ISCE2, + isce2_pool, settings.ORBIT_POOL_LANDSAR, + bool(settings.ISCE2_ENABLED), ) envi_new = len(sync_result.get("envi", {}).get("copied", [])) envi_updated = len(sync_result.get("envi", {}).get("updated", [])) isce2_new = len(sync_result.get("isce2", {}).get("converted", [])) isce2_updated = len(sync_result.get("isce2", {}).get("reconverted", [])) - print( - f" [精轨同步] ENVI 新增 {envi_new}、刷新 {envi_updated}，" - f"ISCE2 转换 {isce2_new}、重转 {isce2_updated}" - ) + if settings.ISCE2_ENABLED: + print( + f" [精轨同步] ENVI/Gamma TXT 新增 {envi_new}、刷新 {envi_updated}，" + f"ISCE2 XML 转换 {isce2_new}、重转 {isce2_updated}" + ) + else: + print(f" [精轨同步] ENVI/Gamma TXT 新增 {envi_new}、刷新 {envi_updated}，ISCE2 已停用") invalid_orbit_stems = { item.get("name") for item in sync_result.get("invalid_sources", []) @@ -550,7 +763,11 @@ class DataService: sync_errors = ( sync_result.get("source", {}).get("errors", []) + [item.get("error", "") for item in sync_result.get("envi", {}).get("errors", [])] - + [item.get("error", "") for item in sync_result.get("isce2", {}).get("errors", [])] + + ( + [item.get("error", "") for item in sync_result.get("isce2", {}).get("errors", [])] + if settings.ISCE2_ENABLED + else [] + ) ) if task_id and sync_errors: await task_service.add_log( @@ -560,15 +777,16 @@ class DataService: "精轨同步存在异常: " f"源目录异常 {len(sync_result.get('source', {}).get('errors', []))} 项, " f"ENVI 复制异常 {len(sync_result.get('envi', {}).get('errors', []))} 项, " - f"ISCE2 转换异常 {len(sync_result.get('isce2', {}).get('errors', []))} 项" + f"ISCE2 转换异常 {len(sync_result.get('isce2', {}).get('errors', [])) if settings.ISCE2_ENABLED else 0} 项" ), ) - for item in sync_result.get("isce2", {}).get("errors", [])[:20]: - await task_service.add_log( - task_id, - "WARN", - f"ISCE2 转换失败: {item.get('file')} -> {item.get('error')}", - ) + if settings.ISCE2_ENABLED: + for item in sync_result.get("isce2", {}).get("errors", [])[:20]: + await task_service.add_log( + task_id, + "WARN", + f"ISCE2 转换失败: {item.get('file')} -> {item.get('error')}", + ) if task_id and invalid_orbit_stems: await task_service.add_log( task_id, diff --git a/backend/app/services/gf3_native_inventory_service.py b/backend/app/services/gf3_native_inventory_service.py index f440739..f5f9f79 100644 --- a/backend/app/services/gf3_native_inventory_service.py +++ b/backend/app/services/gf3_native_inventory_service.py @@ -17,10 +17,13 @@ from ..utils import parse_gf3_l2_dirname NATIVE_MANIFEST_NAME = "gf3_native_manifest.json" NATIVE_MANIFEST_SCHEMA = "gf3_sarscape_native.v1" +FLAT_SCENE_CATALOG_DIR_NAME = ".gf3_flat_scenes" POLARIZATION_PRIORITY = ("HH", "VV", "HV", "VH") SKIP_DIR_NAMES = { ".git", + FLAT_SCENE_CATALOG_DIR_NAME, ".gf3_extract", + ".gf3_runtime", ".sarmap", "__pycache__", "temp", @@ -87,6 +90,16 @@ def _polarization_from_geo_name(name: str) -> str | None: return None +def _scene_name_from_geo_name(name: str) -> str: + text = str(name or "").strip() + match = re.match(r"^(?P.+)_(?:hh|hv|vh|vv)_geo$", text, flags=re.IGNORECASE) + if match: + return match.group("scene") + if text.lower().endswith("_geo"): + return text[:-4] + return Path(text).stem + + def _is_geo_native_data_file(path: Path) -> bool: return path.is_file() and path.name.lower().endswith("_geo") @@ -128,6 +141,62 @@ def _parse_scene_metadata(scene_name: str, assets: list[dict[str, Any]]) -> dict return metadata +def _native_asset_from_base(base: Path) -> dict[str, Any]: + hdr = Path(str(base) + ".hdr") + sml = Path(str(base) + ".sml") + aux_xml = Path(str(base) + ".aux.xml") + ovr = Path(str(base) + ".ovr") + kml = Path(str(base) + ".kml") + quicklook = base.with_name(base.name + "_ql.tif") + ql_kml = base.with_name(base.name + "_ql.kml") + polarization = _polarization_from_geo_name(base.name) or "UNKNOWN" + complete = _is_nonempty_file(base) and _is_nonempty_file(hdr) and _is_nonempty_file(sml) + + return { + "polarization": polarization, + "role": "geo_native", + "path": str(base), + "hdr": str(hdr) if hdr.exists() else None, + "sml": str(sml) if sml.exists() else None, + "aux_xml": str(aux_xml) if aux_xml.exists() else None, + "ovr": str(ovr) if ovr.exists() else None, + "quicklook": str(quicklook) if quicklook.exists() else None, + "kml": str(kml) if kml.exists() else (str(ql_kml) if ql_kml.exists() else None), + "complete": bool(complete), + "source": _safe_stat(base), + "hdr_info": _safe_stat(hdr) if hdr.exists() else None, + "sml_info": _safe_stat(sml) if sml.exists() else None, + } + + +def _asset_fingerprint_part(asset: dict[str, Any]) -> dict[str, Any]: + return { + "polarization": str(asset.get("polarization") or "").upper(), + "path": asset.get("path"), + "complete": bool(asset.get("complete")), + "source": asset.get("source"), + "hdr": asset.get("hdr_info"), + "sml": asset.get("sml_info"), + } + + +def _native_fingerprint(assets: list[dict[str, Any]]) -> str: + payload = [ + _asset_fingerprint_part(asset) + for asset in sorted( + assets, + key=lambda item: ( + str(item.get("polarization") or ""), + str(item.get("path") or ""), + ), + ) + ] + text = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str) + import hashlib + + return hashlib.sha1(text.encode("utf-8", errors="ignore")).hexdigest() + + def _collect_native_assets(scene_dir: Path) -> list[dict[str, Any]]: assets: list[dict[str, Any]] = [] try: @@ -139,43 +208,25 @@ def _collect_native_assets(scene_dir: Path) -> list[dict[str, Any]]: if not _is_geo_native_data_file(path): continue - base = path - hdr = Path(str(base) + ".hdr") - sml = Path(str(base) + ".sml") - aux_xml = Path(str(base) + ".aux.xml") - ovr = Path(str(base) + ".ovr") - kml = Path(str(base) + ".kml") - quicklook = base.with_name(base.name + "_ql.tif") - polarization = _polarization_from_geo_name(base.name) or "UNKNOWN" - complete = _is_nonempty_file(base) and _is_nonempty_file(hdr) and _is_nonempty_file(sml) - - asset: dict[str, Any] = { - "polarization": polarization, - "role": "geo_native", - "path": str(base), - "hdr": str(hdr) if hdr.exists() else None, - "sml": str(sml) if sml.exists() else None, - "aux_xml": str(aux_xml) if aux_xml.exists() else None, - "ovr": str(ovr) if ovr.exists() else None, - "quicklook": str(quicklook) if quicklook.exists() else None, - "kml": str(kml) if kml.exists() else None, - "complete": bool(complete), - "source": _safe_stat(base), - "hdr_info": _safe_stat(hdr) if hdr.exists() else None, - "sml_info": _safe_stat(sml) if sml.exists() else None, - } - assets.append(asset) + assets.append(_native_asset_from_base(path)) return assets -def _collect_scene_manifest(root: Path, scene_dir: Path) -> dict[str, Any] | None: - assets = _collect_native_assets(scene_dir) +def _build_scene_manifest( + root: Path, + *, + scene_name: str, + native_dir: Path, + manifest_dir: Path, + assets: list[dict[str, Any]], + source_dir: Path, + storage_layout: str, +) -> dict[str, Any] | None: if not assets: return None - scene_name = scene_dir.name - batch_name = _scene_batch_name(root, scene_dir, scene_name) + batch_name = _scene_batch_name(root, source_dir, scene_name) complete_assets = [asset for asset in assets if asset.get("complete")] complete_pols = [ pol @@ -200,25 +251,30 @@ def _collect_scene_manifest(root: Path, scene_dir: Path) -> dict[str, Any] | Non logs = [] for name in ("gf3_sarscape_cli.log",): - log_path = scene_dir / name + log_path = source_dir / name if log_path.is_file(): logs.append(str(log_path)) try: - logs.extend(str(path) for path in sorted(scene_dir.glob("*.log"), key=lambda item: item.name.lower()) if str(path) not in logs) + logs.extend(str(path) for path in sorted(source_dir.glob("*.log"), key=lambda item: item.name.lower()) if str(path) not in logs) except OSError: pass metadata = _parse_scene_metadata(scene_name, assets) - manifest_path = scene_dir / NATIVE_MANIFEST_NAME + native_fingerprint = _native_fingerprint(assets) + metadata["native_fingerprint"] = native_fingerprint + manifest_path = manifest_dir / NATIVE_MANIFEST_NAME return { "schema": NATIVE_MANIFEST_SCHEMA, "generated_at": _utc_now(), "scene_name": scene_name, "batch_name": batch_name, "native_root": str(root), - "native_dir": str(scene_dir), + "native_dir": str(native_dir), + "source_dir": str(source_dir), "manifest_path": str(manifest_path), "source_archive": None, + "storage_layout": storage_layout, + "native_fingerprint": native_fingerprint, "status": status, "polarizations": complete_pols, "metadata": metadata, @@ -227,6 +283,49 @@ def _collect_scene_manifest(root: Path, scene_dir: Path) -> dict[str, Any] | Non } +def _collect_scene_manifest(root: Path, scene_dir: Path) -> dict[str, Any] | None: + assets = _collect_native_assets(scene_dir) + return _build_scene_manifest( + root, + scene_name=scene_dir.name, + native_dir=scene_dir, + manifest_dir=scene_dir, + assets=assets, + source_dir=scene_dir, + storage_layout="scene_dir", + ) + + +def _collect_flat_scene_manifests(root: Path, source_dir: Path) -> list[dict[str, Any]]: + try: + entries = sorted(source_dir.iterdir(), key=lambda item: item.name.lower()) + except OSError: + return [] + + grouped_assets: dict[str, list[dict[str, Any]]] = {} + for path in entries: + if not _is_geo_native_data_file(path): + continue + scene_name = _scene_name_from_geo_name(path.name) + grouped_assets.setdefault(scene_name, []).append(_native_asset_from_base(path)) + + manifests: list[dict[str, Any]] = [] + for scene_name in sorted(grouped_assets): + scene_catalog_dir = source_dir / FLAT_SCENE_CATALOG_DIR_NAME / scene_name + manifest = _build_scene_manifest( + root, + scene_name=scene_name, + native_dir=scene_catalog_dir, + manifest_dir=scene_catalog_dir, + assets=grouped_assets[scene_name], + source_dir=source_dir, + storage_layout="flat_files", + ) + if manifest: + manifests.append(manifest) + return manifests + + def _normalize_roots(native_dirs: list[str] | tuple[str, ...] | None) -> tuple[list[Path], list[str]]: roots: list[Path] = [] missing: list[str] = [] @@ -251,6 +350,7 @@ def scan_gf3_sarscape_native_roots( native_dirs: list[str] | tuple[str, ...] | None, *, write_manifest: bool = True, + scene_dirs_only: bool = False, ) -> dict[str, Any]: """Scan configured native roots and return discovered scene manifests.""" roots, missing_roots = _normalize_roots(native_dirs) @@ -258,12 +358,56 @@ def scan_gf3_sarscape_native_roots( seen_scene_dirs: set[str] = set() write_errors: list[dict[str, str]] = [] + def append_manifest(manifest: dict[str, Any], scene_key: str, scene_dir_for_error: Path) -> None: + seen_scene_dirs.add(scene_key) + if write_manifest: + try: + _write_json(Path(manifest["manifest_path"]), manifest) + except OSError as exc: + write_errors.append({"scene_dir": str(scene_dir_for_error), "error": str(exc)}) + scenes.append(manifest) + for root in roots: + if scene_dirs_only: + try: + candidate_dirs = [root] + [ + child + for child in sorted(root.iterdir(), key=lambda item: item.name.lower()) + if child.is_dir() + and child.name not in SKIP_DIR_NAMES + and not child.name.startswith(".SARscape") + and not child.name.startswith(".gf3_") + ] + except OSError: + candidate_dirs = [root] + + for scene_dir in candidate_dirs: + scene_key = str(scene_dir).lower() + if scene_key in seen_scene_dirs: + continue + + if not scene_dir.name.upper().startswith("GF3_"): + flat_manifests = _collect_flat_scene_manifests(root, scene_dir) + if flat_manifests: + for manifest in flat_manifests: + flat_key = str(manifest.get("native_dir") or "").lower() + if flat_key and flat_key not in seen_scene_dirs: + append_manifest(manifest, flat_key, scene_dir) + continue + + manifest = _collect_scene_manifest(root, scene_dir) + if not manifest: + continue + + append_manifest(manifest, scene_key, scene_dir) + continue + for current_dir, dir_names, _file_names in os.walk(root): dir_names[:] = [ name for name in dir_names if name not in SKIP_DIR_NAMES and not name.startswith(".SARscape") + and not name.startswith(".gf3_") ] scene_dir = Path(current_dir) scene_key = str(scene_dir).lower() @@ -271,17 +415,20 @@ def scan_gf3_sarscape_native_roots( dir_names[:] = [] continue + if not scene_dir.name.upper().startswith("GF3_"): + flat_manifests = _collect_flat_scene_manifests(root, scene_dir) + if flat_manifests: + for manifest in flat_manifests: + flat_key = str(manifest.get("native_dir") or "").lower() + if flat_key and flat_key not in seen_scene_dirs: + append_manifest(manifest, flat_key, scene_dir) + continue + manifest = _collect_scene_manifest(root, scene_dir) if not manifest: continue - seen_scene_dirs.add(scene_key) - if write_manifest: - try: - _write_json(Path(manifest["manifest_path"]), manifest) - except OSError as exc: - write_errors.append({"scene_dir": str(scene_dir), "error": str(exc)}) - scenes.append(manifest) + append_manifest(manifest, scene_key, scene_dir) dir_names[:] = [] native_ready = sum(1 for scene in scenes if scene.get("status") == "NATIVE_READY") diff --git a/backend/app/services/gf3_standardize_service.py b/backend/app/services/gf3_standardize_service.py index 2e1dd09..84e92d2 100644 --- a/backend/app/services/gf3_standardize_service.py +++ b/backend/app/services/gf3_standardize_service.py @@ -15,12 +15,13 @@ from shapely.geometry import Polygon from sqlalchemy import func, or_, select from sqlalchemy.ext.asyncio import AsyncSession -from ..config import settings +from ..config import settings, split_env_paths from ..models import ManagedRootORM, RadarDataORM, SARSceneGeoORM, SourceProductAssetORM -from .data_service import extract_geotiff_bounds +from .data_service import DataService, extract_geotiff_bounds from .gf3_native_inventory_service import ( NATIVE_MANIFEST_NAME, POLARIZATION_PRIORITY, + SKIP_DIR_NAMES, scan_gf3_sarscape_native_roots, ) from .image_service import image_service @@ -31,6 +32,9 @@ STANDARD_MANIFEST_SCHEMA = "gf3_standard_geotiff.v1" CONVERTER_NAME = "gf3_sarscape_geo_to_tif" CONVERTER_VERSION = "v1" SOURCE_ASSET_FORMAT = "GF3_SARSCAPE_L2" +SOURCE_ASSET_QUICKLOOK_FORMAT = "GF3_SARSCAPE_QUICKLOOK" +SOURCE_ASSET_NATIVE_PREVIEW_FORMAT = "GF3_SARSCAPE_NATIVE_PREVIEW" +QUICKLOOK_PREVIEW_CACHE_VERSION = "gf3_native_webp.v2" def _utc_now() -> str: @@ -72,6 +76,112 @@ def _path_kind(path: str) -> str: return "relative" +def _path_key(path: str | Path) -> str: + return os.path.normcase(os.path.normpath(str(path))).lower() + + +def _has_native_geo_assets(path: Path) -> bool: + try: + return any(item.is_file() and item.name.lower().endswith("_geo") for item in path.iterdir()) + except OSError: + return False + + +def _discover_gf3_sarscape_scan_roots(native_dirs: list[str] | tuple[str, ...] | None) -> list[str]: + """Expand stable GF3 roots into SARscape ``*_geo`` result pools before scanning. + + Operators usually configure a durable GF3 root rather than a date folder. + Date-specific SARscape output pools live below it as ``YYYYMMDD_geo``. Prefer + direct ``*_geo`` children and fall back to recursive discovery. + """ + roots: list[str] = [] + seen: set[str] = set() + + def add(path: str | Path) -> None: + text = os.path.normpath(str(path)) + key = _path_key(text) + if key and key not in seen: + seen.add(key) + roots.append(text) + + for raw in native_dirs or []: + text = str(raw or "").strip() + if not text: + continue + + if any(ch in text for ch in "*?"): + glob_root = Path(os.path.normpath(text)) + try: + matches = sorted(glob_root.parent.glob(glob_root.name), key=lambda item: item.name.lower()) + except OSError: + matches = [] + if matches: + for match in matches: + if match.is_dir(): + add(match) + continue + add(glob_root) + continue + + root = Path(os.path.normpath(text)) + try: + is_dir = root.is_dir() + except OSError: + is_dir = False + if not is_dir: + add(root) + continue + + if root.name.lower().endswith("_geo"): + add(root) + + try: + immediate_geo_roots = sorted( + ( + child + for child in root.iterdir() + if child.is_dir() and child.name.lower().endswith("_geo") + ), + key=lambda item: item.name.lower(), + ) + except OSError: + immediate_geo_roots = [] + + if immediate_geo_roots: + for child in immediate_geo_roots: + add(child) + continue + + if _has_native_geo_assets(root): + add(root) + continue + + found_recursive = 0 + for current_dir, dir_names, _file_names in os.walk(root): + dir_names[:] = [ + name + for name in dir_names + if name not in SKIP_DIR_NAMES and not name.startswith(".SARscape") + and not name.startswith(".gf3_") + ] + selected: list[str] = [] + remaining: list[str] = [] + for name in dir_names: + if name.lower().endswith("_geo"): + selected.append(name) + else: + remaining.append(name) + for name in selected: + add(Path(current_dir) / name) + found_recursive += len(selected) + dir_names[:] = remaining + + if found_recursive == 0: + add(root) + + return roots + + def _source_asset_uid(path: str) -> str: normalized = os.path.normpath(str(path or "").strip()) digest = hashlib.sha1(normalized.lower().encode("utf-8", errors="ignore")).hexdigest() @@ -439,7 +549,7 @@ def _crs_is_geographic_lonlat(crs: Any) -> bool: return False -def _polygon_from_tif(path: Path) -> list[tuple[float, float]] | None: +def _polygon_from_raster(path: Path) -> list[tuple[float, float]] | None: polygon = extract_geotiff_bounds(str(path)) if polygon and len(polygon) >= 4: return polygon @@ -478,6 +588,22 @@ def _polygon_from_tif(path: Path) -> list[tuple[float, float]] | None: return None +def _polygon_from_native_asset(asset: dict[str, Any] | None) -> list[tuple[float, float]] | None: + if not asset: + return None + for key in ("path", "quicklook"): + text = _path_text(asset.get(key)) + if not text: + continue + path = Path(text) + if not path.is_file(): + continue + polygon = _polygon_from_raster(path) + if polygon and len(polygon) >= 4: + return polygon + return None + + def _scene_center_from_polygon(polygon: list[tuple[float, float]] | None) -> tuple[float | None, float | None]: if not polygon: return None, None @@ -522,6 +648,505 @@ def _select_default_asset(assets: list[dict[str, Any]]) -> dict[str, Any] | None return assets[0] if assets else None +def _is_quicklook_raster_path(path: Any) -> bool: + text = _path_text(path).lower() + return text.endswith("_ql.tif") or text.endswith("_ql.tiff") + + +def _select_default_native_asset(assets: list[dict[str, Any]]) -> dict[str, Any] | None: + candidates = [ + asset + for asset in assets + if _path_text(asset.get("path")) and not _is_quicklook_raster_path(asset.get("path")) + ] + complete = [asset for asset in candidates if asset.get("complete") is not False] + return _select_default_asset(complete or candidates) + + +def _quicklook_assets_from_scene_manifest(scene_manifest: dict[str, Any]) -> list[dict[str, Any]]: + output_assets: list[dict[str, Any]] = [] + for asset in scene_manifest.get("assets") or []: + native_path = _path_text(asset.get("path")) + if not native_path or _is_quicklook_raster_path(native_path) or asset.get("complete") is not True: + continue + output_assets.append( + { + "polarization": str(asset.get("polarization") or "UNKNOWN").upper(), + "role": "native_geo", + "path": native_path, + "hdr": _path_text(asset.get("hdr")) or None, + "sml": _path_text(asset.get("sml")) or None, + "quicklook": _path_text(asset.get("quicklook")) or None, + "complete": bool(asset.get("complete")), + "status": "registered_native_geo", + } + ) + return output_assets + + +def _quicklook_manifest_for_scene(scene_manifest: dict[str, Any], storage_root: str | Path | None) -> dict[str, Any]: + root = Path(storage_root or settings.GF3_STORAGE_DIRS).resolve() + out_dir = _standard_scene_dir(scene_manifest, root) + manifest_path = out_dir / "gf3_native_preview_manifest.json" + assets = _quicklook_assets_from_scene_manifest(scene_manifest) + manifest = { + "schema": "gf3_sarscape_native_preview.v1", + "generated_at": _utc_now(), + "scene_name": scene_manifest.get("scene_name"), + "batch_name": scene_manifest.get("batch_name"), + "native_manifest": scene_manifest.get("manifest_path") or str(Path(scene_manifest.get("native_dir") or "") / NATIVE_MANIFEST_NAME), + "native_dir": scene_manifest.get("native_dir"), + "standard_dir": str(out_dir), + "manifest_path": str(manifest_path), + "status": "NATIVE_READY" if assets else "FAILED", + "assets": assets, + "summary": { + "native_assets": len(assets), + "quicklook_assets": 0, + "full_raster_materialized": True, + "webp_source": "native_geo", + }, + "errors": [] if assets else [{"error": "no native _geo assets found"}], + } + _write_json(manifest_path, manifest) + return manifest + + +def _read_geotiff_quicklook_webp(source_path: str, target_path: str, *, max_size: int | None = None) -> bool: + """Build a local WebP from a registered GF3 raster, including ENVI ``*_geo``.""" + try: + import numpy as np + import rasterio + from PIL import Image + from rasterio.enums import Resampling + except Exception: + return False + + try: + target = Path(target_path) + target.parent.mkdir(parents=True, exist_ok=True) + limit = int(max_size or settings.RADAR_THUMBNAIL_MAX_SIZE or 1600) + with rasterio.open(source_path) as src: + scale = min(limit / max(src.width, 1), limit / max(src.height, 1), 1.0) + out_width = max(1, int(src.width * scale)) + out_height = max(1, int(src.height * scale)) + band = src.read( + 1, + out_shape=(out_height, out_width), + masked=True, + resampling=Resampling.bilinear, + ) + data = band.filled(0) + if data.dtype != np.uint8: + valid = band.compressed() if hasattr(band, "compressed") else data[np.isfinite(data)] + if valid.size: + p2, p98 = np.nanpercentile(valid.astype("float32"), [2, 98]) + scaled = np.clip((data.astype("float32") - p2) / max(float(p98 - p2), 1e-6), 0, 1) + data = (scaled * 255).astype("uint8") + else: + data = np.zeros(data.shape, dtype="uint8") + else: + data = data.astype("uint8", copy=False) + + mask = getattr(band, "mask", None) + if mask is None or np.ndim(mask) == 0: + alpha = np.full(data.shape, 255, dtype="uint8") + else: + alpha = np.where(mask, 0, 255).astype("uint8") + rgba = np.stack([data, data, data, alpha], axis=-1) + + image = Image.fromarray(rgba, "RGBA") + image = image_service.make_edge_dark_transparent(image) + image_service.save_image_as_webp(image, target_path, quality=82) + return target.exists() and target.stat().st_size > 0 + except Exception: + return False + + +def _native_preview_source_from_metadata(metadata: dict[str, Any]) -> str: + candidates: list[Any] = [metadata.get("default_native_path")] + for key in ("native_assets", "quicklook_assets"): + for asset in metadata.get(key) or []: + if isinstance(asset, dict): + candidates.append(asset.get("path")) + for candidate in candidates: + text = _path_text(candidate) + if text and not _is_quicklook_raster_path(text): + return text + return "" + + +def _path_is_under_any(path: str, roots: list[str]) -> bool: + target = _path_key(path) + if not target: + return False + for root in roots: + root_key = _path_key(root) + if root_key and (target == root_key or target.startswith(root_key + os.sep)): + return True + return False + + +def _configured_gf3_native_roots(native_dirs: list[str] | tuple[str, ...] | None = None) -> list[str]: + roots = [str(item) for item in (native_dirs or []) if str(item or "").strip()] + if not roots: + roots = split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS) + return [os.path.normpath(root) for root in roots if str(root or "").strip()] + + +def _scene_native_fingerprint(scene_manifest: dict[str, Any]) -> str: + return str( + scene_manifest.get("native_fingerprint") + or (scene_manifest.get("metadata") or {}).get("native_fingerprint") + or "" + ).strip() + + +def _record_has_bounds(record: RadarDataORM) -> bool: + return ( + record.coverage_polygon is not None + and record.min_lon is not None + and record.min_lat is not None + and record.max_lon is not None + and record.max_lat is not None + ) + + +async def _find_existing_quicklook_radar( + db: AsyncSession, + *, + scene_manifest: dict[str, Any], +) -> RadarDataORM | None: + native_dir = str(scene_manifest.get("native_dir") or "") + scene_name = scene_manifest.get("scene_name") or Path(native_dir).name + unique_id = f"gf3_sarscape_native_preview:{scene_name}" + result = await db.execute( + select(RadarDataORM).where( + or_( + RadarDataORM.unique_id == unique_id, + RadarDataORM.file_path == native_dir, + ) + ) + ) + return result.scalars().first() + + +async def _can_skip_unchanged_quicklook_radar( + db: AsyncSession, + *, + scene_manifest: dict[str, Any], +) -> tuple[bool, int | None, str]: + native_fingerprint = _scene_native_fingerprint(scene_manifest) + if not native_fingerprint: + return False, None, "missing_native_fingerprint" + + record = await _find_existing_quicklook_radar(db, scene_manifest=scene_manifest) + if record is None: + return False, None, "missing_record" + + metadata = record.metadata_json or {} + if str(metadata.get("native_fingerprint") or "") != native_fingerprint: + return False, int(record.id) if record.id is not None else None, "native_fingerprint_changed" + if str(metadata.get("registration_mode") or "") != "native_preview": + return False, int(record.id) if record.id is not None else None, "registration_mode_changed" + if not _record_has_bounds(record): + return False, int(record.id) if record.id is not None else None, "missing_bounds" + + return True, int(record.id) if record.id is not None else None, "unchanged" + + +async def generate_gf3_quicklook_webp_cache( + db: AsyncSession, + *, + force: bool = False, + max_records: int | None = None, + native_dirs: list[str] | tuple[str, ...] | None = None, + progress_callback: Any | None = None, +) -> dict[str, Any]: + """Generate local WebP previews from registered GF3 SARscape native ``*_geo`` rasters.""" + stmt = ( + select(RadarDataORM) + .where(RadarDataORM.source_format == SOURCE_ASSET_NATIVE_PREVIEW_FORMAT) + .order_by(RadarDataORM.imaging_date.desc().nullslast(), RadarDataORM.id.asc()) + ) + + result = await db.execute(stmt) + configured_roots = _configured_gf3_native_roots(native_dirs) + records = [ + record + for record in result.scalars().all() + if _path_is_under_any(_native_preview_source_from_metadata(record.metadata_json or {}), configured_roots) + ] + if max_records and max_records > 0: + records = records[: int(max_records)] + total = len(records) + generated = 0 + skipped = 0 + failed = 0 + record_results: list[dict[str, Any]] = [] + + for idx, record in enumerate(records): + metadata = record.metadata_json or {} + source_path = _native_preview_source_from_metadata(metadata) + cache_path = DataService.get_radar_raw_cache_path(record.unique_id or record.file_path, record.file_path) + if progress_callback: + pct = 5 + int((idx / max(total, 1)) * 90) + progress_callback(pct, f"Generate GF3 native _geo WebP {idx + 1}/{total}: {record.product_unique_id or record.id}") + + if ( + not force + and (record.preview_cache_status or "NONE") == "READY" + and record.preview_cache_path + and os.path.exists(record.preview_cache_path) + ): + skipped += 1 + record_results.append({"id": record.id, "status": "SKIPPED", "cache_path": record.preview_cache_path}) + continue + + if not source_path: + failed += 1 + record.preview_cache_status = "FAILED" + record.preview_cache_path = None + record.preview_cache_error = "default_native_path_missing" + record.preview_cache_version = QUICKLOOK_PREVIEW_CACHE_VERSION + record.preview_cache_updated_at = _db_now() + record_results.append({"id": record.id, "status": "FAILED", "error": record.preview_cache_error}) + await db.commit() + continue + + ok = await asyncio.to_thread(_read_geotiff_quicklook_webp, source_path, cache_path) + record.preview_cache_version = QUICKLOOK_PREVIEW_CACHE_VERSION + record.preview_cache_updated_at = _db_now() + if ok and os.path.exists(cache_path): + generated += 1 + record.preview_cache_status = "READY" + record.preview_cache_path = cache_path + record.preview_cache_error = None + record_results.append({"id": record.id, "status": "READY", "cache_path": cache_path}) + else: + failed += 1 + record.preview_cache_status = "FAILED" + record.preview_cache_path = None + record.preview_cache_error = "native_webp_build_failed" + record_results.append({"id": record.id, "status": "FAILED", "error": record.preview_cache_error}) + await db.commit() + + return { + "ok": failed == 0, + "mode": "gf3_native_webp", + "total": total, + "generated": generated, + "skipped": skipped, + "failed": failed, + "records": record_results, + } + + +async def _upsert_quicklook_source_product_asset( + db: AsyncSession, + scene_manifest: dict[str, Any], + quicklook_manifest: dict[str, Any], +) -> int | None: + native_dir_text = _path_text(scene_manifest.get("native_dir")) + if not native_dir_text: + return None + metadata = scene_manifest.get("metadata") or {} + imaging_date = str(metadata.get("imaging_date") or "").strip() or None + acquisition_start = _date_to_naive_utc(imaging_date) + scene_name = scene_manifest.get("scene_name") or Path(native_dir_text).name + now = _db_now() + root = await _find_managed_root_for_path(db, native_dir_text) + asset_metadata = _json_safe( + { + "source": "GF3 SARscape native _geo", + "native_dir": native_dir_text, + "native_manifest": scene_manifest.get("manifest_path"), + "native_fingerprint": _scene_native_fingerprint(scene_manifest), + "native_preview_manifest": quicklook_manifest.get("manifest_path"), + "native_assets": quicklook_manifest.get("assets") or [], + "full_raster_materialized": True, + "analysis_engine": "gf3_sarscape", + } + ) + data = { + "asset_uid": _source_asset_uid(f"gf3_native_preview:{native_dir_text}"), + "logical_product_uid": scene_name, + "satellite_family": "GF3", + "satellite": "GF3", + "source_format": SOURCE_ASSET_NATIVE_PREVIEW_FORMAT, + "product_type": metadata.get("product_type") or "SARSCAPE_NATIVE", + "product_level": "L2_NATIVE", + "imaging_mode": metadata.get("imaging_mode"), + "polarization": metadata.get("polarization"), + "absolute_orbit": metadata.get("absolute_orbit") or metadata.get("orbit_circle"), + "relative_orbit": metadata.get("relative_orbit"), + "orbit_direction": metadata.get("orbit_direction"), + "acquisition_start_time_utc": acquisition_start, + "acquisition_stop_time_utc": None, + "imaging_date": imaging_date, + "root_ref_id": root.id if root else None, + "root_path": root.path if root else str(Path(native_dir_text).parent), + "file_path": native_dir_text, + "archive_path": scene_manifest.get("source_archive"), + "path_kind": _path_kind(native_dir_text), + "file_name": Path(native_dir_text).name, + "file_stem": Path(native_dir_text).name, + "file_ext": "", + "size_bytes": None, + "mtime_epoch": None, + "checksum_status": "NOT_COMPUTED", + "parser_name": "gf3_sarscape_native_preview_manifest", + "parser_version": CONVERTER_VERSION, + "parse_status": "NATIVE_READY" if quicklook_manifest.get("assets") else "FAILED", + "parse_error": "; ".join(str(item.get("error") or item) for item in (quicklook_manifest.get("errors") or [])) or None, + "parsed_at": now, + "metadata_json": asset_metadata, + "is_active": True, + "missing_since": None, + "updated_at": now, + } + result = await db.execute( + select(SourceProductAssetORM).where( + or_( + SourceProductAssetORM.asset_uid == data["asset_uid"], + SourceProductAssetORM.file_path == native_dir_text, + ) + ) + ) + asset = result.scalars().first() + if asset is None: + asset = SourceProductAssetORM(**data) + db.add(asset) + else: + for key, value in data.items(): + setattr(asset, key, value) + await db.flush() + return int(asset.id) if asset.id is not None else None + + +async def _upsert_quicklook_radar_data( + db: AsyncSession, + scene_manifest: dict[str, Any], + quicklook_manifest: dict[str, Any], + source_product_ref_id: int | None = None, +) -> int | None: + assets = quicklook_manifest.get("assets") or [] + default_asset = _select_default_native_asset(assets) + if not default_asset: + return None + metadata = scene_manifest.get("metadata") or {} + imaging_date = str(metadata.get("imaging_date") or "").strip() or None + acquisition_start = _date_to_naive_utc(imaging_date) + scene_name = scene_manifest.get("scene_name") or Path(str(scene_manifest.get("native_dir") or "")).name + native_dir = str(scene_manifest.get("native_dir") or "") + unique_id = f"gf3_sarscape_native_preview:{scene_name}" + default_native_path = str(default_asset.get("path") or "") + default_quicklook_path = str(default_asset.get("quicklook") or "") + polygon = _polygon_from_native_asset(default_asset) + min_lon = min_lat = max_lon = max_lat = None + geom = None + if polygon: + min_lon, min_lat, max_lon, max_lat = _bounds_from_polygon(polygon) + center_lon, center_lat = _scene_center_from_polygon(polygon) + geom = _geom_from_polygon(polygon) + else: + center_lon = metadata.get("scene_center_lon") + center_lat = metadata.get("scene_center_lat") + preview_cache_path = None + preview_cache_kind = "deferred" + preview_cache_error = "native_webp_not_generated" + radar_metadata = _json_safe( + { + **metadata, + "native_dir": native_dir, + "native_manifest": scene_manifest.get("manifest_path"), + "native_fingerprint": _scene_native_fingerprint(scene_manifest), + "native_preview_manifest": quicklook_manifest.get("manifest_path"), + "native_assets": assets, + "quicklook_assets": [], + "default_native_path": default_native_path, + "default_quicklook_path": default_quicklook_path, + "coverage_source": "native_geo_or_quicklook" if polygon else "scene_name", + "preview_cache_kind": preview_cache_kind, + "analysis_engine": "gf3_sarscape", + "registration_mode": "native_preview", + "full_raster_materialized": True, + } + ) + data_to_upsert = { + "unique_id": unique_id, + "satellite": "GF3", + "satellite_family": "GF3", + "imaging_date": imaging_date, + "imaging_mode": metadata.get("imaging_mode"), + "polarization": ",".join( + pol + for pol in POLARIZATION_PRIORITY + if any(str(asset.get("polarization") or "").upper() == pol for asset in assets) + ) + or metadata.get("polarization"), + "scene_center_lon": center_lon, + "scene_center_lat": center_lat, + "acquisition_time_utc": acquisition_start.isoformat() if acquisition_start else None, + "product_level": "L2_NATIVE", + "product_unique_id": metadata.get("product_unique_id") or scene_name, + "source_product_token": scene_name, + "acquisition_start_time_utc": acquisition_start, + "acquisition_stop_time_utc": None, + "absolute_orbit": metadata.get("absolute_orbit") or metadata.get("orbit_circle"), + "relative_orbit": metadata.get("relative_orbit"), + "source_format": SOURCE_ASSET_NATIVE_PREVIEW_FORMAT, + "source_product_ref_id": source_product_ref_id, + "image_data_format": "ENVI_NATIVE", + "geocoded_flag": True, + "metadata_json": radar_metadata, + "file_path": native_dir, + "has_orbit_data": False, + "orbit_file_path": None, + "is_envi_processed": True, + "coverage_polygon": polygon, + "geom": geom, + "min_lon": min_lon, + "min_lat": min_lat, + "max_lon": max_lon, + "max_lat": max_lat, + "preview_cache_status": "NONE", + "preview_cache_version": QUICKLOOK_PREVIEW_CACHE_VERSION, + "preview_cache_path": preview_cache_path, + "preview_cache_updated_at": _db_now(), + "preview_cache_error": preview_cache_error, + } + result = await db.execute( + select(RadarDataORM).where( + or_( + RadarDataORM.unique_id == unique_id, + RadarDataORM.file_path == native_dir, + ) + ) + ) + radar = result.scalars().first() + if ( + radar is not None + and (radar.preview_cache_status or "").upper() == "READY" + and radar.preview_cache_path + and os.path.exists(radar.preview_cache_path) + and str((radar.metadata_json or {}).get("default_native_path") or "") == default_native_path + ): + data_to_upsert["preview_cache_status"] = radar.preview_cache_status + data_to_upsert["preview_cache_version"] = radar.preview_cache_version or QUICKLOOK_PREVIEW_CACHE_VERSION + data_to_upsert["preview_cache_path"] = radar.preview_cache_path + data_to_upsert["preview_cache_updated_at"] = radar.preview_cache_updated_at + data_to_upsert["preview_cache_error"] = radar.preview_cache_error + if radar is None: + radar = RadarDataORM(**data_to_upsert) + db.add(radar) + else: + for key, value in data_to_upsert.items(): + setattr(radar, key, value) + await db.flush() + return int(radar.id) if radar.id is not None else None + + def _metadata_for_radar(scene_manifest: dict[str, Any], standard_manifest: dict[str, Any]) -> dict[str, Any]: metadata = dict(scene_manifest.get("metadata") or {}) metadata.update( @@ -635,7 +1260,7 @@ async def _upsert_radar_data( if not default_asset: return None - polygon = _polygon_from_tif(Path(_path_text(default_asset.get("path")))) + polygon = _polygon_from_raster(Path(_path_text(default_asset.get("path")))) min_lon, min_lat, max_lon, max_lat = _bounds_from_polygon(polygon) center_lon, center_lat = _scene_center_from_polygon(polygon) geom = _geom_from_polygon(polygon) @@ -851,17 +1476,118 @@ async def standardize_gf3_sarscape_native_roots( storage_root: str | None = None, force: bool = False, register: bool = True, + quicklook_only: bool = False, progress_callback: Any | None = None, ) -> dict[str, Any]: """Scan native roots, convert complete assets, and register standard scenes.""" - inventory = await asyncio.to_thread( - scan_gf3_sarscape_native_roots, - native_dirs, - write_manifest=True, - ) + requested_native_dirs = [str(item) for item in (native_dirs or []) if str(item or "").strip()] + if quicklook_only: + scan_roots = _discover_gf3_sarscape_scan_roots(native_dirs) + inventory = await asyncio.to_thread( + scan_gf3_sarscape_native_roots, + scan_roots, + write_manifest=True, + ) + else: + scan_roots = _discover_gf3_sarscape_scan_roots(native_dirs) + inventory = await asyncio.to_thread( + scan_gf3_sarscape_native_roots, + scan_roots, + write_manifest=True, + ) scenes = inventory.get("scenes") or [] ready_scenes = [scene for scene in scenes if scene.get("status") in {"NATIVE_READY", "PARTIAL"}] + if quicklook_only: + registered = 0 + quicklook_assets = 0 + failed_scenes = 0 + skipped_unchanged = 0 + scene_results: list[dict[str, Any]] = [] + total = len(ready_scenes) + for idx, scene_manifest in enumerate(ready_scenes): + if progress_callback: + pct = 10 + int((idx / max(total, 1)) * 80) + progress_callback(pct, f"Register GF3 native _geo {idx + 1}/{total}: {scene_manifest.get('scene_name')}") + + quicklook_manifest = await asyncio.to_thread( + _quicklook_manifest_for_scene, + scene_manifest, + storage_root, + ) + assets = quicklook_manifest.get("assets") or [] + quicklook_assets += len(assets) + source_asset_id = None + radar_id = None + skipped_reason = None + if register and assets: + can_skip, existing_radar_id, skip_reason = await _can_skip_unchanged_quicklook_radar( + db, + scene_manifest=scene_manifest, + ) + if can_skip: + skipped_unchanged += 1 + radar_id = existing_radar_id + skipped_reason = skip_reason + if progress_callback: + pct = 10 + int(((idx + 1) / max(total, 1)) * 80) + progress_callback(pct, f"Skip unchanged GF3 native _geo {idx + 1}/{total}: {scene_manifest.get('scene_name')}") + else: + skipped_reason = skip_reason + source_asset_id = await _upsert_quicklook_source_product_asset(db, scene_manifest, quicklook_manifest) + radar_id = await _upsert_quicklook_radar_data( + db, + scene_manifest, + quicklook_manifest, + source_product_ref_id=source_asset_id, + ) + if radar_id: + registered += 1 + await db.commit() + if not assets: + failed_scenes += 1 + scene_results.append( + { + "scene_name": scene_manifest.get("scene_name"), + "native_status": scene_manifest.get("status"), + "standard_status": quicklook_manifest.get("status"), + "native_preview_manifest": quicklook_manifest.get("manifest_path"), + "source_asset_id": source_asset_id, + "radar_id": radar_id, + "skipped_unchanged": bool(skipped_reason == "unchanged"), + "skip_reason": skipped_reason, + "native_fingerprint": _scene_native_fingerprint(scene_manifest), + "summary": quicklook_manifest.get("summary") or {}, + "errors": quicklook_manifest.get("errors") or [], + } + ) + + return { + "ok": failed_scenes == 0, + "mode": "native_preview", + "inventory": { + key: value + for key, value in inventory.items() + if key != "scenes" + }, + "requested_native_dirs": requested_native_dirs, + "scan_roots": scan_roots, + "scene_count": len(scenes), + "ready_scene_count": len(ready_scenes), + "converted_scenes": 0, + "partial_scenes": 0, + "failed_scenes": failed_scenes, + "converted_assets": 0, + "skipped_assets": skipped_unchanged, + "skipped_unchanged": skipped_unchanged, + "failed_assets": failed_scenes, + "quicklook_assets": quicklook_assets, + "native_assets": quicklook_assets, + "registered": registered, + "analysis_ready": 0, + "scenes": scene_results, + } + converted_scenes = 0 partial_scenes = 0 failed_scenes = 0 @@ -936,6 +1662,8 @@ async def standardize_gf3_sarscape_native_roots( for key, value in inventory.items() if key != "scenes" }, + "requested_native_dirs": requested_native_dirs, + "scan_roots": scan_roots, "scene_count": len(scenes), "ready_scene_count": len(ready_scenes), "converted_scenes": converted_scenes, diff --git a/backend/app/services/job_handlers.py b/backend/app/services/job_handlers.py index a5f44d8..4e0e6b1 100644 --- a/backend/app/services/job_handlers.py +++ b/backend/app/services/job_handlers.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) from sqlalchemy import select from .. import database -from ..config import settings +from ..config import settings, split_env_paths from ..models import SystemJobORM, DinsarResultORM, HazardPointORM, DinsarTaskItemORM, PsTaskItemORM, RadarDataORM, SARSceneGeoORM, FloodDetectionORM, WaterDetectionORM, WaterExtractionORM, GF3ProcessingORM, AiDiagnosisORM from ..scheduler import scan_data_job from .data_service import data_service @@ -88,6 +88,7 @@ JOB_TYPE_GF3_UNPACK = "GF3_UNPACK" JOB_TYPE_GF3_BATCH_PROCESS = "GF3_BATCH_PROCESS" JOB_TYPE_GF3_SARSCAPE_PRODUCE = "GF3_SARSCAPE_PRODUCE" JOB_TYPE_GF3_SARSCAPE_SYNC = "GF3_SARSCAPE_SYNC" +JOB_TYPE_GF3_QUICKLOOK_WEBP = "GF3_QUICKLOOK_WEBP" JOB_TYPE_GF3_SARSCAPE_CLEAN = "GF3_SARSCAPE_CLEAN" JOB_TYPE_ISCE2_RUN = "ISCE2_RUN" JOB_TYPE_PYINT_RUN = "PYINT_RUN" @@ -97,6 +98,7 @@ JOB_TYPE_REBUILD_DINSAR_CATALOG = "REBUILD_DINSAR_CATALOG" JOB_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG" JOB_TYPE_REBUILD_SBAS_INSAR_CATALOG = "REBUILD_SBAS_INSAR_CATALOG" JOB_TYPE_SCAN_ASSET_INVENTORY = "SCAN_ASSET_INVENTORY" +JOB_TYPE_AUDIT_SOURCE_ARCHIVE_INTEGRITY = "AUDIT_SOURCE_ARCHIVE_INTEGRITY" JOB_TYPE_SBAS_COREGISTRATION = "SBAS_COREGISTRATION" JOB_TYPE_SBAS_RDC_DEM = "SBAS_RDC_DEM" JOB_TYPE_SBAS_INTERFEROGRAMS = "SBAS_INTERFEROGRAMS" @@ -253,9 +255,12 @@ async def _handle_scan_asset_inventory(job: SystemJobORM) -> None: db, inventory_types=payload.get("inventory_types") or None, root_ids=payload.get("root_ids") or None, + families=payload.get("families") or None, bind_orbits=bool(payload.get("bind_orbits", True)), + build_previews=bool(payload.get("build_previews", True)), task_id=job.task_id, ) + preview = result.get("preview_cache") or {} await task_service.update_task( job.task_id, status="COMPLETED", @@ -265,7 +270,42 @@ async def _handle_scan_asset_inventory(job: SystemJobORM) -> None: f"sources={result.get('source_assets', 0)}, " f"orbits={result.get('orbit_assets', 0)}, " f"matched={((result.get('binding') or {}).get('matched_count', 0))}, " - f"missing={((result.get('binding') or {}).get('missing_count', 0))}" + f"missing={((result.get('binding') or {}).get('missing_count', 0))}, " + f"previews_ready={preview.get('ready', 0)}, " + f"previews_skipped={preview.get('skipped_ready', 0)}, " + f"previews_failed={preview.get('failed', 0)}, " + f"previews_missing={preview.get('missing_source', 0)}" + ), + db=db, + ) + + +async def _handle_archive_integrity_audit(job: SystemJobORM) -> None: + if not job.task_id: + raise ValueError("AUDIT_SOURCE_ARCHIVE_INTEGRITY requires task_id for progress tracking.") + payload = job.payload or {} + await task_service.start_task(job.task_id, message="Source archive integrity audit started") + async with AsyncSessionLocal() as db: + result = await asset_inventory_service.audit_source_archive_integrity( + db, + families=payload.get("families") or None, + source_formats=payload.get("source_formats") or None, + asset_ids=payload.get("asset_ids") or None, + force=bool(payload.get("force", False)), + limit=payload.get("limit"), + task_id=job.task_id, + ) + await task_service.update_task( + job.task_id, + status="COMPLETED", + progress=100, + message=( + "Source archive integrity audit completed: " + f"checked={result.get('checked', 0)}, " + f"skipped={result.get('skipped', 0)}, " + f"ok={result.get('ok', 0)}, " + f"failed={result.get('failed', 0)}, " + f"unsupported={result.get('unsupported', 0)}" ), db=db, ) @@ -345,7 +385,7 @@ async def _handle_copy_data(job: SystemJobORM) -> None: copy_statuses = _normalize_copy_statuses(payload.get("copy_statuses")) include_orbit_files = bool(payload.get("include_orbit_files")) export_zip = bool(payload.get("export_zip")) - package_mode = str(payload.get("package_mode") or ("task_zip" if export_zip else "task_folder")).strip().lower() + package_mode = str(payload.get("package_mode") or ("source_bundle" if export_zip else "task_folder")).strip().lower() skip_existing = payload.get("skip_existing") is not False max_items = _normalize_positive_int(payload.get("max_items")) @@ -445,17 +485,27 @@ async def _handle_copy_data(job: SystemJobORM) -> None: skip_existing=skip_existing, max_items=max_items, ) - else: + return + if package_mode in {"task_folder", "task"}: await run_dinsar_copy_items( job.task_id, items, dest_dir, include_orbit_files=include_orbit_files, - export_zip=(package_mode == "task_zip" or export_zip), + export_zip=False, skip_existing=skip_existing, max_items=max_items, ) - return + return + if package_mode == "task_zip": + raise ValueError( + "D-InSAR Task ZIP export is not a production-preparation format. " + "Use package_mode=task_folder for Task_Pool preparation or source_bundle for archive distribution." + ) + raise ValueError( + "Unsupported D-InSAR package_mode. " + "Use task_folder for production preparation or source_bundle for archive distribution." + ) raise ValueError(f"Unknown COPY_DATA file_type: {file_type}") @@ -884,7 +934,9 @@ async def _handle_ai_diagnosis(job: SystemJobORM) -> None: from ..ai_service import analyze_map_with_vlm diagnosis_markdown = await analyze_map_with_vlm( images_base64=[img_base64], - prompt=full_prompt + prompt=full_prompt, + model_name=model_name, + raise_on_error=True, ) # 8. 解析风险等级和置信度（简单正则匹配） @@ -3646,7 +3698,7 @@ async def _handle_gf3_process(job: SystemJobORM) -> None: if not settings.GF3_LEGACY_GDAL_ENABLED: raise ValueError( "Legacy GF3 Python/GDAL preprocessing is disabled. " - "Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly." + "Use GF3 native _geo result registration or set GF3_LEGACY_GDAL_ENABLED=true explicitly." ) payload = job.payload or {} @@ -3725,7 +3777,7 @@ async def _handle_gf3_unpack(job: SystemJobORM) -> None: if not settings.GF3_LEGACY_GDAL_ENABLED: raise ValueError( "Legacy GF3 archive unpack is disabled. " - "Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly." + "Use GF3 native _geo result registration or set GF3_LEGACY_GDAL_ENABLED=true explicitly." ) if not job.task_id: @@ -3792,7 +3844,7 @@ async def _handle_gf3_batch_process(job: SystemJobORM) -> None: if not settings.GF3_LEGACY_GDAL_ENABLED: raise ValueError( "Legacy GF3 Python/GDAL preprocessing is disabled. " - "Use GF3 SARscape production or set GF3_LEGACY_GDAL_ENABLED=true explicitly." + "Use GF3 native _geo result registration or set GF3_LEGACY_GDAL_ENABLED=true explicitly." ) payload = job.payload or {} @@ -3901,7 +3953,7 @@ async def _handle_gf3_batch_process(job: SystemJobORM) -> None: async def _handle_gf3_sarscape_sync(job: SystemJobORM) -> None: - """Scan SARscape native GF3 _geo outputs, convert them to GeoTIFF, and register them.""" + """Scan SARscape native GF3 _geo outputs and register requested assets.""" from .gf3_standardize_service import standardize_gf3_sarscape_native_roots if not job.task_id: @@ -3915,7 +3967,15 @@ async def _handle_gf3_sarscape_sync(job: SystemJobORM) -> None: if not storage_root: raise ValueError("GF3_SARSCAPE_SYNC: storage_root is empty") - await task_service.start_task(job.task_id, message="扫描 GF3 SARscape 原生 _geo 结果池...") + quicklook_only = bool(payload.get("quicklook_only", False)) + await task_service.start_task( + job.task_id, + message=( + "按 GF3 日期/场景命名规则登记本机 SARscape _geo 原生结果..." + if quicklook_only + else "递归扫描 GF3 SARscape 原生 _geo 结果池..." + ), + ) loop = asyncio.get_running_loop() @@ -3943,6 +4003,7 @@ async def _handle_gf3_sarscape_sync(job: SystemJobORM) -> None: storage_root=storage_root, force=bool(payload.get("force", False)), register=bool(payload.get("register", True)), + quicklook_only=quicklook_only, progress_callback=_progress_cb, ) except Exception as exc: @@ -3954,44 +4015,44 @@ async def _handle_gf3_sarscape_sync(job: SystemJobORM) -> None: ) raise - message = ( - "GF3 SARscape 标准化完成: " - f"发现 {int(result.get('scene_count') or 0)} 景, " - f"可转换 {int(result.get('ready_scene_count') or 0)} 景, " - f"转换 {int(result.get('converted_scenes') or 0)} 景, " - f"部分 {int(result.get('partial_scenes') or 0)} 景, " - f"失败 {int(result.get('failed_scenes') or 0)} 景, " - f"新增/更新 GeoTIFF {int(result.get('converted_assets') or 0)} 个, " - f"跳过 {int(result.get('skipped_assets') or 0)} 个, " - f"入库 {int(result.get('registered') or 0)} 景" - ) + if quicklook_only: + message = ( + "GF3 _geo 原生结果登记完成: " + f"规则匹配 {int(result.get('scene_count') or 0)} 景, " + f"可登记 {int(result.get('ready_scene_count') or 0)} 景, " + f"原生资产 {int(result.get('native_assets') or result.get('quicklook_assets') or 0)} 个, " + f"入库 {int(result.get('registered') or 0)} 景, " + f"跳过未变化 {int(result.get('skipped_unchanged') or 0)} 景, " + f"失败 {int(result.get('failed_scenes') or 0)} 景" + ) + else: + message = ( + "GF3 SARscape 标准化完成: " + f"发现 {int(result.get('scene_count') or 0)} 景, " + f"可转换 {int(result.get('ready_scene_count') or 0)} 景, " + f"转换 {int(result.get('converted_scenes') or 0)} 景, " + f"部分 {int(result.get('partial_scenes') or 0)} 景, " + f"失败 {int(result.get('failed_scenes') or 0)} 景, " + f"新增/更新 GeoTIFF {int(result.get('converted_assets') or 0)} 个, " + f"跳过 {int(result.get('skipped_assets') or 0)} 个, " + f"入库 {int(result.get('registered') or 0)} 景" + ) await task_service.update_task(job.task_id, status="COMPLETED", progress=100, message=message) -async def _handle_gf3_sarscape_produce(job: SystemJobORM) -> None: - """Run GF3 raw archive -> SARscape native -> GeoTIFF registration chain.""" - from .gf3_sarscape_production_service import ( - cleanup_gf3_sarscape_native_pool, - run_gf3_sarscape_production, - ) - from .gf3_standardize_service import standardize_gf3_sarscape_native_roots +async def _handle_gf3_quicklook_webp(job: SystemJobORM) -> None: + """Generate local WebP previews from registered GF3 SARscape native _geo records.""" + from .gf3_standardize_service import generate_gf3_quicklook_webp_cache if not job.task_id: - raise ValueError("GF3_SARSCAPE_PRODUCE requires task_id for progress tracking.") + raise ValueError("GF3_QUICKLOOK_WEBP requires task_id for progress tracking.") payload = job.payload or {} - source_dirs = payload.get("source_dirs") or [] - native_dirs = payload.get("native_dirs") or [] - storage_root = payload.get("storage_root") or settings.GF3_STORAGE_DIRS - native_root = payload.get("native_root") or (native_dirs[0] if native_dirs else "") - if not source_dirs: - raise ValueError("GF3_SARSCAPE_PRODUCE: source_dirs is empty") - if not native_root: - raise ValueError("GF3_SARSCAPE_PRODUCE: native_root is empty") - if not storage_root: - raise ValueError("GF3_SARSCAPE_PRODUCE: storage_root is empty") + force = bool(payload.get("force", False)) + max_records = _normalize_positive_int(payload.get("max_records")) + native_dirs = payload.get("native_dirs") or split_env_paths(settings.GF3_SARSCAPE_NATIVE_DIRS) + await task_service.start_task(job.task_id, message="开始从 GF3 _geo 原生数据生成 WebP 缓存...") - await task_service.start_task(job.task_id, message="GF3 SARscape production starting...") loop = asyncio.get_running_loop() def _progress_cb(progress: int, message: str) -> None: @@ -4005,143 +4066,49 @@ async def _handle_gf3_sarscape_produce(job: SystemJobORM) -> None: try: fut.result() except Exception as exc: - logger.warning("[GF3 SARscape Produce] progress callback failed: %s", exc) + logger.warning("[GF3 native WebP] progress callback failed: %s", exc) future.add_done_callback(_swallow_progress_error) except RuntimeError: return - def _log_cb(level: str, message: str) -> None: - try: - future = asyncio.run_coroutine_threadsafe( - task_service.add_log(job.task_id, level, message), - loop, - ) - - def _swallow_log_error(fut): - try: - fut.result() - except Exception as exc: - logger.warning("[GF3 SARscape Produce] log callback failed: %s", exc) - - future.add_done_callback(_swallow_log_error) - except RuntimeError: - return - - async def _production_keepalive() -> None: - progress = 8 - while True: - await asyncio.sleep(60) - progress = min(68, progress + 1) - await task_service.update_task( - job.task_id, - progress=progress, - message="GF3 SARscape production is still running...", - ) - try: - production_task = asyncio.create_task( - asyncio.to_thread( - run_gf3_sarscape_production, - source_dirs=source_dirs, - native_root=native_root, - wrapper_exe=payload.get("wrapper_exe"), - dem_path=payload.get("dem_path"), - idlrt_path=payload.get("idlrt_path"), - polarizations=payload.get("polarizations"), - archive_exts=payload.get("archive_exts") or [], - max_archives_per_run=payload.get("max_archives_per_run"), - selected_dates=payload.get("selected_dates") or [], - task_id=job.task_id, - local_staging_root=payload.get("local_staging_root") or settings.GF3_TASK_POOL_ROOT, - timeout_seconds=payload.get("timeout_seconds"), - keep_extracted=payload.get("keep_extracted"), - log_callback=_log_cb, + async with AsyncSessionLocal() as db: + result = await generate_gf3_quicklook_webp_cache( + db, + force=force, + max_records=max_records, + native_dirs=native_dirs, progress_callback=_progress_cb, ) - ) - keepalive_task = asyncio.create_task(_production_keepalive()) - try: - production_result = await production_task - finally: - keepalive_task.cancel() - try: - await keepalive_task - except asyncio.CancelledError: - pass - - standardize_result: Dict[str, Any] = {} - if bool(payload.get("auto_standardize", True)): - await task_service.update_task( - job.task_id, - progress=72, - message="GF3 SARscape production finished; standardizing native _geo outputs...", - ) - async with AsyncSessionLocal() as db: - standardize_result = await standardize_gf3_sarscape_native_roots( - db, - native_dirs=native_dirs or [native_root], - storage_root=storage_root, - force=bool(payload.get("force_standardize", False)), - register=bool(payload.get("register", True)), - progress_callback=lambda pct, msg: _progress_cb(72 + int(max(0, min(100, pct)) * 0.16), msg), - ) - - cleanup_result: Dict[str, Any] = {} - production_ok = int(production_result.get("failed_count") or 0) == 0 - standardize_ok = ( - not standardize_result - or ( - int(standardize_result.get("failed_assets") or 0) == 0 - and int(standardize_result.get("failed_scenes") or 0) == 0 - ) - ) - if bool(payload.get("clean_after_success", True)) and production_ok and standardize_ok: - await task_service.update_task( - job.task_id, - progress=90, - message="Cleaning GF3 SARscape intermediate files...", - ) - cleanup_result = await asyncio.to_thread( - cleanup_gf3_sarscape_native_pool, - native_dirs=native_dirs or [native_root], - storage_root=storage_root, - require_standardized=bool(payload.get("cleanup_require_standardized", True)), - dry_run=bool(payload.get("cleanup_dry_run", False)), - max_scenes=payload.get("cleanup_max_scenes"), - log_callback=_log_cb, - progress_callback=lambda pct, msg: _progress_cb(90 + int(max(0, min(100, pct)) * 0.09), msg), - ) - elif bool(payload.get("clean_after_success", True)): - _log_cb( - "WARNING", - "GF3 SARscape automatic cleanup skipped because production or standardization had failures.", - ) except Exception as exc: await task_service.update_task( job.task_id, status="FAILED", progress=100, - message=f"GF3 SARscape production failed: {exc}", + message=f"GF3 _geo WebP 生成失败: {exc}", ) raise - failed_count = int(production_result.get("failed_count") or 0) - failed_assets = int(standardize_result.get("failed_assets") or 0) - cleanup_errors = int(cleanup_result.get("error_scene_count") or 0) - final_status = "FAILED" if failed_count or failed_assets or cleanup_errors else "COMPLETED" message = ( - "GF3 SARscape production chain finished: " - f"found={int(production_result.get('found_count') or 0)}, " - f"produced={int(production_result.get('processed_count') or 0)}, " - f"skipped={int(production_result.get('skipped_count') or 0)}, " - f"failed={failed_count}, " - f"converted_assets={int(standardize_result.get('converted_assets') or 0)}, " - f"registered={int(standardize_result.get('registered') or 0)}, " - f"cleaned_scenes={int(cleanup_result.get('cleaned_scene_count') or 0)}, " - f"cleaned_bytes={int(cleanup_result.get('bytes_deleted') or 0)}" + "GF3 _geo WebP 生成完成: " + f"总数 {int(result.get('total') or 0)} 景, " + f"生成 {int(result.get('generated') or 0)} 景, " + f"跳过 {int(result.get('skipped') or 0)} 景, " + f"失败 {int(result.get('failed') or 0)} 景" ) - await task_service.update_task(job.task_id, status=final_status, progress=100, message=message) + await task_service.update_task(job.task_id, status="COMPLETED", progress=100, message=message) + + +async def _handle_gf3_sarscape_produce(job: SystemJobORM) -> None: + """Reject GF3 production jobs on this management machine.""" + message = ( + "GF3 SARscape production is disabled on this management machine. " + "Run production on the SARscape host and register local _geo native results here." + ) + if job.task_id: + await task_service.update_task(job.task_id, status="FAILED", progress=100, message=message) + raise ValueError(message) async def _handle_gf3_sarscape_clean(job: SystemJobORM) -> None: @@ -5263,6 +5230,7 @@ async def _handle_sbas_landsar_workflow(job: SystemJobORM) -> None: _HANDLERS = { JOB_TYPE_SCAN_DATA: _handle_scan_data, JOB_TYPE_SCAN_ASSET_INVENTORY: _handle_scan_asset_inventory, + JOB_TYPE_AUDIT_SOURCE_ARCHIVE_INTEGRITY: _handle_archive_integrity_audit, JOB_TYPE_SCAN_DINSAR: _handle_scan_dinsar, JOB_TYPE_PUBLISH_DINSAR_PRODUCTS: _handle_publish_dinsar_products_clean, JOB_TYPE_REBUILD_DINSAR_CATALOG: _handle_rebuild_dinsar_catalog_clean, @@ -5301,6 +5269,7 @@ _HANDLERS = { JOB_TYPE_GF3_BATCH_PROCESS: _handle_gf3_batch_process, JOB_TYPE_GF3_SARSCAPE_PRODUCE: _handle_gf3_sarscape_produce, JOB_TYPE_GF3_SARSCAPE_SYNC: _handle_gf3_sarscape_sync, + JOB_TYPE_GF3_QUICKLOOK_WEBP: _handle_gf3_quicklook_webp, JOB_TYPE_GF3_SARSCAPE_CLEAN: _handle_gf3_sarscape_clean, JOB_TYPE_SBAS_COREGISTRATION: _handle_sbas_coregistration, JOB_TYPE_SBAS_RDC_DEM: _handle_sbas_rdc_dem, diff --git a/backend/app/services/landsar_sbas_service.py b/backend/app/services/landsar_sbas_service.py index 669951d..6e3cc86 100644 --- a/backend/app/services/landsar_sbas_service.py +++ b/backend/app/services/landsar_sbas_service.py @@ -24,6 +24,8 @@ from ..dinsar_engines.landsar_engine import ( _norm_path, _path_search_dirs, _summarize_landsar_failure, + derive_landsar_dem_bbox_from_xml_paths, + prepare_landsar_dem_crop, ) @@ -633,6 +635,16 @@ class LandsarSbasService: "errors": errors, } + def _derive_task_dem_bbox(self, task: dict[str, Any], input_dir: str) -> tuple[float, float, float, float] | None: + xml_paths = [ + str(scene.get("xml") or "") + for scene in (task.get("scenes") or []) + if str(scene.get("xml") or "").strip() + ] + if not xml_paths and input_dir and os.path.isdir(input_dir): + xml_paths = [str(path) for path in sorted(Path(input_dir).glob("LT1*_SLC.xml"), key=lambda item: item.name.lower())] + return derive_landsar_dem_bbox_from_xml_paths(xml_paths, fallback_paths=xml_paths) + def materialize_stack( self, *, @@ -949,7 +961,7 @@ class LandsarSbasService: params_payload = self.normalize_params(params or {}) normalized_dem = _norm_path(dem_path or self.default_dem_path) if not normalized_dem or not os.path.isfile(normalized_dem): - raise ValueError(f"LandSAR SBAS DEM file is missing: {normalized_dem or ''}") + raise ValueError(f"LandSAR SBAS DEM source file is missing: {normalized_dem or ''}") from .sbas_insar_production_service import sbas_insar_production_service @@ -1035,6 +1047,8 @@ class LandsarSbasService: "input_task_root": import_dest_root, "publish_root": str(publish_root), "dem_path": normalized_dem, + "dem_source_path": normalized_dem, + "dem_role": "global_prepared_source", "params": params_payload, "timeout_seconds": max(60, int(timeout_seconds or settings.LANDSAR_SBAS_TIMEOUT_SECONDS or 172800)), "import_timeout_seconds": max(60, int(import_timeout_seconds or timeout_seconds or settings.LANDSAR_SBAS_TIMEOUT_SECONDS or 172800)), @@ -1541,9 +1555,9 @@ class LandsarSbasService: if not bool(settings.LANDSAR_SBAS_ENABLED): raise ValueError("LandSAR SBAS is disabled.") params = self.normalize_params(extra or {}) - dem_path = _norm_path((extra or {}).get("dem_path") or self.default_dem_path) - if not dem_path or not os.path.isfile(dem_path): - raise ValueError(f"LandSAR SBAS DEM file is missing: {dem_path or ''}") + dem_source_path = _norm_path((extra or {}).get("dem_path") or self.default_dem_path) + if not dem_source_path or not os.path.isfile(dem_source_path): + raise ValueError(f"LandSAR SBAS DEM source file is missing: {dem_source_path or ''}") validation = self.validate_root_dir( root_dir, min_scenes=min_scenes, @@ -1585,7 +1599,9 @@ class LandsarSbasService: "work_root_strategy": "short_landsar_execution_path", "native_root": str(native_root), "publish_root": str(publish_root), - "dem_path": dem_path, + "dem_path": dem_source_path, + "dem_source_path": dem_source_path, + "dem_role": "global_prepared_source", "params": params, "timeout_seconds": max(60, int(timeout_seconds or settings.LANDSAR_SBAS_TIMEOUT_SECONDS or 172800)), "min_scenes": validation.get("min_scenes"), @@ -1716,10 +1732,56 @@ class LandsarSbasService: native_output_dir = native_root / task_alias / "Output_Data" native_output_dir.mkdir(parents=True, exist_ok=True) project_name = str((manifest.get("params") or {}).get("project_name") or task_alias).strip() or task_alias + dem_source_path = _norm_path(str(manifest.get("dem_path") or "")) + effective_dem_path = dem_source_path + dem_crop_info: dict[str, Any] = {} + try: + dem_bbox = self._derive_task_dem_bbox(task, input_dir) + if not dem_bbox: + raise ValueError("cannot derive DEM crop bbox from LandSAR SBAS Input_Data XML corner coordinates") + dem_crop_info = prepare_landsar_dem_crop( + dem_source_path, + str(native_root / task_alias / "dem_crop"), + dem_bbox, + label=task_alias, + ) + effective_dem_path = str(dem_crop_info.get("dem_path") or dem_source_path) + self._emit( + progress_callback, + "INFO", + f"[{index}/{len(tasks)}] LandSAR SBAS DEM crop ready: {effective_dem_path}", + ) + except Exception as exc: + failed_count += 1 + error = f"LandSAR SBAS DEM crop failed: {exc}" + self._emit(progress_callback, "ERROR", f"[{index}/{len(tasks)}] {task_name} failed: {error}") + task_results.append( + { + "task_name": task_name, + "task_alias": task_alias, + "input_data_dir": input_dir, + "native_output_dir": str(native_output_dir), + "param_file": "", + "process_name": SBAS_PROCESS_NAME, + "command": "", + "returncode": -2, + "success": False, + "timed_out": False, + "failure_kind": "dem_crop_failed", + "error": error, + "stdout_tail": "", + "native_logs": {}, + "publish": {}, + "dem_source_path": dem_source_path, + "dem_path": effective_dem_path, + "dem_crop": dem_crop_info, + } + ) + continue param_file = _generate_sbas_param_file( str(native_output_dir / f"{SBAS_PROID}.txt"), slc_folder=input_dir, - dem_path=str(manifest.get("dem_path") or ""), + dem_path=effective_dem_path, output_dir=str(native_output_dir), project_name=project_name, params=dict(manifest.get("params") or {}), @@ -1784,6 +1846,9 @@ class LandsarSbasService: "stdout_tail": _collect_tail(stdout_text, 4000), "native_logs": log_publish_result, "publish": publish_result, + "dem_source_path": dem_source_path, + "dem_path": effective_dem_path, + "dem_crop": dem_crop_info, } ) diff --git a/backend/app/services/orbit_converter.py b/backend/app/services/orbit_converter.py index 1d01ffb..adb67ae 100644 --- a/backend/app/services/orbit_converter.py +++ b/backend/app/services/orbit_converter.py @@ -1,7 +1,7 @@ """轨道文件管理与格式转换服务。 目录约定： - MONITOR_ORBIT_DIR/ 源精轨目录（可为 UNC），平铺 .txt + MONITOR_ORBIT_DIR/ 本机源精轨目录，平铺 .txt ORBIT_POOL_ENVI/ ENVI 本地精轨池 LT1A/ LT-1A 精轨 .txt LT1B/ LT-1B 精轨 .txt @@ -320,16 +320,17 @@ def summarize_source_orbit_gaps( source_files: Dict[str, Dict[str, str]] = source_inventory["files"] envi_files: Dict[str, Dict[str, str]] = pool_inventory["envi"]["files"] - isce2_files: Dict[str, Dict[str, str]] = pool_inventory["isce2"]["files"] + isce2_enabled = bool(str(isce2_pool or "").strip()) + isce2_files: Dict[str, Dict[str, str]] = pool_inventory["isce2"]["files"] if isce2_enabled else {} source_stems = set(source_files) envi_stems = set(envi_files) isce2_stems = set(isce2_files) - source_without_isce2 = sorted(source_stems - isce2_stems) + source_without_isce2 = sorted(source_stems - isce2_stems) if isce2_enabled else [] source_without_envi = sorted(source_stems - envi_stems) envi_without_source = sorted(envi_stems - source_stems) - isce2_without_source = sorted(isce2_stems - source_stems) + isce2_without_source = sorted(isce2_stems - source_stems) if isce2_enabled else [] sample_limit = 20 def _make_samples( @@ -349,7 +350,7 @@ def summarize_source_orbit_gaps( ) return samples - suspect_bad_samples = _make_samples(source_without_isce2, inspect_source=True) + suspect_bad_samples = _make_samples(source_without_isce2, inspect_source=True) if isce2_enabled else [] bad_source_samples = [ item for item in suspect_bad_samples if item.get("has_corruption_signal") @@ -358,6 +359,7 @@ def summarize_source_orbit_gaps( return { "sample_limit": sample_limit, "quarantine_path": _default_quarantine_root(source_dir, quarantine_root), + "isce2_enabled": isce2_enabled, "source_without_isce2_count": len(source_without_isce2), "source_without_envi_count": len(source_without_envi), "envi_without_source_count": len(envi_without_source), @@ -809,6 +811,7 @@ def sync_orbit_pools( envi_pool: str = "", isce2_pool: str = "", landsar_pool: str = "", + validate_without_isce2: bool = True, ) -> Dict: """从 source_dir 扫描平铺 .txt 精轨，同步到各引擎本地池。 @@ -863,11 +866,16 @@ def sync_orbit_pools( xml_needs_refresh = bool(isce2_pool) and ( not os.path.exists(xml_path) or _needs_generated_refresh(fpath, xml_path) ) - requires_validation = xml_needs_refresh or (not isce2_pool and envi_needs_refresh) + requires_validation = xml_needs_refresh or ( + bool(validate_without_isce2) and not isce2_pool and envi_needs_refresh + ) staged_dir = "" staged_xml = "" try: + health = _inspect_orbit_txt_health(fpath) + if _has_orbit_corruption_signal(health): + raise RuntimeError("Source TXT contains NUL bytes or cannot be read as a healthy LT-1 orbit text file.") if requires_validation: staged_dir, staged_xml = _stage_isce2_xml( fpath, @@ -924,8 +932,15 @@ def repair_orbit_pools( isce2_pool: str = "", landsar_pool: str = "", ) -> Dict[str, Any]: + isce2_enabled = bool(str(isce2_pool or "").strip()) before = check_orbit_consistency(envi_pool, isce2_pool) - sync_result = sync_orbit_pools(source_dir, envi_pool, isce2_pool, landsar_pool) + sync_result = sync_orbit_pools( + source_dir, + envi_pool, + isce2_pool, + landsar_pool, + validate_without_isce2=isce2_enabled, + ) repaired_from_envi: List[str] = [] repair_errors: List[Dict[str, str]] = [] @@ -934,7 +949,7 @@ def repair_orbit_pools( envi_files: Dict[str, Dict[str, str]] = pool_inventory["envi"]["files"] isce2_files: Dict[str, Dict[str, str]] = pool_inventory["isce2"]["files"] - if isce2_pool: + if isce2_enabled: os.makedirs(isce2_pool, exist_ok=True) for stem in sorted(set(envi_files) - set(isce2_files)): txt_path = envi_files[stem]["path"] @@ -952,6 +967,7 @@ def repair_orbit_pools( "before": before, "after": after, "sync_result": sync_result, + "isce2_enabled": isce2_enabled, "repaired_from_envi": repaired_from_envi, "repair_error_count": len(repair_errors), "repair_errors": repair_errors, @@ -968,17 +984,19 @@ def quarantine_bad_orbits( isce2_pool: str = "", quarantine_root: str = "", ) -> Dict[str, Any]: + isce2_enabled = bool(str(isce2_pool or "").strip()) source_inventory = get_source_orbit_inventory(source_dir, recursive=True) pool_inventory = get_orbit_pool_inventory(envi_pool, isce2_pool, recursive=True) source_files: Dict[str, Dict[str, str]] = source_inventory["files"] envi_files: Dict[str, Dict[str, str]] = pool_inventory["envi"]["files"] - isce2_files: Dict[str, Dict[str, str]] = pool_inventory["isce2"]["files"] + isce2_files: Dict[str, Dict[str, str]] = pool_inventory["isce2"]["files"] if isce2_enabled else {} quarantine_dir = _default_quarantine_root(source_dir, quarantine_root) - candidate_stems = sorted(set(source_files) - set(isce2_files)) + candidate_stems = sorted(set(source_files) - set(isce2_files)) if isce2_enabled else [] result: Dict[str, Any] = { + "isce2_enabled": isce2_enabled, "quarantine_root": quarantine_dir, "candidate_count": len(candidate_stems), "validated_count": 0, @@ -1083,14 +1101,18 @@ def check_orbit_consistency( "healthy": bool, } """ + isce2_enabled = bool(str(isce2_pool or "").strip()) inventory = get_orbit_pool_inventory(envi_pool, isce2_pool, recursive=True) envi_files: Dict[str, Dict[str, str]] = inventory["envi"]["files"] - isce2_files: Dict[str, Dict[str, str]] = inventory["isce2"]["files"] + isce2_files: Dict[str, Dict[str, str]] = inventory["isce2"]["files"] if isce2_enabled else {} by_satellite: Dict[str, int] = inventory["envi"]["by_satellite"] # 对比 envi_stems = set(envi_files) isce2_stems = set(isce2_files) + if not isce2_enabled: + envi_stems = set() + isce2_stems = set() mismatches = [] for stem in envi_stems - isce2_stems: mismatches.append( @@ -1112,6 +1134,9 @@ def check_orbit_consistency( ) mismatches.sort(key=lambda item: item["name"]) + errors = list(inventory["envi"]["errors"]) + if isce2_enabled: + errors.extend(inventory["isce2"]["errors"]) return { "envi": { @@ -1121,11 +1146,12 @@ def check_orbit_consistency( }, "isce2": { "path": isce2_pool, + "enabled": isce2_enabled, "total": len(isce2_files), }, - "error_count": len(inventory["envi"]["errors"]) + len(inventory["isce2"]["errors"]), - "errors": list(inventory["envi"]["errors"]) + list(inventory["isce2"]["errors"]), + "error_count": len(errors), + "errors": errors, "mismatch_count": len(mismatches), "mismatches": mismatches, - "healthy": len(mismatches) == 0, + "healthy": len(mismatches) == 0 and len(errors) == 0, } diff --git a/backend/app/services/pairing_cache_service.py b/backend/app/services/pairing_cache_service.py index 659fbc2..3eeaaf7 100644 --- a/backend/app/services/pairing_cache_service.py +++ b/backend/app/services/pairing_cache_service.py @@ -52,6 +52,15 @@ def _same_satellite_family_expr(left_alias: str, right_alias: str) -> str: ) +def _supported_dinsar_family_pair_expr(left_alias: str, right_alias: str) -> str: + left_family = _satellite_family_expr(left_alias) + right_family = _satellite_family_expr(right_alias) + return ( + f"(({left_family} = 'LT1' AND {right_family} = 'LT1') " + f"OR ({left_family} = 'S1' AND {right_family} = 'S1'))" + ) + + def _same_look_direction_expr(left_alias: str, right_alias: str) -> str: return ( f"(NULLIF({left_alias}.look_direction, '') IS NULL " @@ -60,6 +69,60 @@ def _same_look_direction_expr(left_alias: str, right_alias: str) -> str: ) +def _relative_orbit_value_expr(alias: str) -> str: + return f"NULLIF(upper(trim(COALESCE({alias}.relative_orbit, ''))), '')" + + +def _same_relative_orbit_expr(left_alias: str, right_alias: str) -> str: + left_rel = _relative_orbit_value_expr(left_alias) + right_rel = _relative_orbit_value_expr(right_alias) + return f"({left_rel} IS NOT NULL AND {right_rel} IS NOT NULL AND {left_rel} = {right_rel})" + + +def _dinsar_quality_tier_expr(left_alias: str, right_alias: str, center_distance_expr: str) -> str: + same_rel = _same_relative_orbit_expr(left_alias, right_alias) + return ( + "CASE " + f"WHEN {same_rel} AND {center_distance_expr} <= 5000 THEN 'A' " + f"WHEN {center_distance_expr} <= 5000 THEN 'B' " + f"WHEN {center_distance_expr} <= 12000 THEN 'C' " + "ELSE 'REJECT' END" + ) + + +def _dinsar_readiness_expr(left_alias: str, right_alias: str, center_distance_expr: str) -> str: + tier = _dinsar_quality_tier_expr(left_alias, right_alias, center_distance_expr) + return ( + "CASE " + f"WHEN {tier} IN ('A', 'B') THEN 'RECOMMENDED' " + f"WHEN {tier} = 'C' THEN 'CANDIDATE' " + "ELSE 'NOT_RECOMMENDED' END" + ) + + +def _dinsar_quality_score_expr(left_alias: str, right_alias: str, center_distance_expr: str) -> str: + same_rel = _same_relative_orbit_expr(left_alias, right_alias) + return ( + "GREATEST(0.0, LEAST(1.0, " + f"0.45 * COALESCE((ST_Area(ST_Intersection({left_alias}.geom, {right_alias}.geom)::geography) / " + f"NULLIF(GREATEST(ST_Area({left_alias}.geom::geography), ST_Area({right_alias}.geom::geography)), 0)), 0) + " + f"0.30 * CASE WHEN {same_rel} THEN 1 ELSE 0 END + " + f"0.25 * GREATEST(0.0, 1.0 - ({center_distance_expr} / 12000.0))" + "))" + ) + + +def _dinsar_reasons_expr(left_alias: str, right_alias: str, center_distance_expr: str) -> str: + same_rel = _same_relative_orbit_expr(left_alias, right_alias) + return ( + "COALESCE(jsonb_path_query_array(jsonb_build_array(" + f"CASE WHEN NOT ({same_rel}) THEN 'relative_orbit_missing_or_mismatch' END, " + f"CASE WHEN {center_distance_expr} > 5000 THEN 'center_distance_over_5km' END, " + f"CASE WHEN {center_distance_expr} > 12000 THEN 'center_distance_over_12km' END" + "), '$[*] ? (@ != null)')::json, '[]'::json)" + ) + + def _orientation_is_left_master_expr(left_alias: str, right_alias: str) -> str: left_uid = _scene_uid_expr(left_alias) right_uid = _scene_uid_expr(right_alias) @@ -82,6 +145,7 @@ def _hard_constraints_expr(left_alias: str, right_alias: str) -> str: f"AND {left_alias}.orbit_direction = {right_alias}.orbit_direction " f"AND COALESCE({left_alias}.insar_source_ready, false) " f"AND COALESCE({right_alias}.insar_source_ready, false) " + f"AND { _supported_dinsar_family_pair_expr(left_alias, right_alias) } " f"AND { _same_look_direction_expr(left_alias, right_alias) } " f"AND ST_Intersects({left_alias}.geom, {right_alias}.geom)" ) @@ -106,7 +170,15 @@ def _full_rebuild_insert_sql() -> str: spatial_baseline_meters, scene_center_distance_meters, scene_overlap_ratio, + pair_aoi_overlap_ratio, orbit_direction, + same_relative_orbit, + master_relative_orbit, + slave_relative_orbit, + dinsar_quality_tier, + dinsar_quality_score, + dinsar_readiness, + dinsar_reasons_json, same_satellite, same_satellite_family, same_look_direction, @@ -144,7 +216,15 @@ def _full_rebuild_insert_sql() -> str: ST_Area(ST_Intersection(m.geom, s.geom)::geography) / NULLIF(GREATEST(ST_Area(m.geom::geography), ST_Area(s.geom::geography)), 0) )::double precision AS scene_overlap_ratio, + NULL::double precision AS pair_aoi_overlap_ratio, m.orbit_direction, + { _same_relative_orbit_expr('m', 's') } AS same_relative_orbit, + {_relative_orbit_value_expr('m')} AS master_relative_orbit, + {_relative_orbit_value_expr('s')} AS slave_relative_orbit, + { _dinsar_quality_tier_expr('m', 's', center_distance) } AS dinsar_quality_tier, + { _dinsar_quality_score_expr('m', 's', center_distance) } AS dinsar_quality_score, + { _dinsar_readiness_expr('m', 's', center_distance) } AS dinsar_readiness, + { _dinsar_reasons_expr('m', 's', center_distance) } AS dinsar_reasons_json, (m.satellite IS NOT NULL AND s.satellite IS NOT NULL AND m.satellite = s.satellite) AS same_satellite, { _same_satellite_family_expr('m', 's') } AS same_satellite_family, { _same_look_direction_expr('m', 's') } AS same_look_direction, @@ -201,7 +281,15 @@ def _incremental_insert_sql() -> str: spatial_baseline_meters, scene_center_distance_meters, scene_overlap_ratio, + pair_aoi_overlap_ratio, orbit_direction, + same_relative_orbit, + master_relative_orbit, + slave_relative_orbit, + dinsar_quality_tier, + dinsar_quality_score, + dinsar_readiness, + dinsar_reasons_json, same_satellite, same_satellite_family, same_look_direction, @@ -244,7 +332,15 @@ def _incremental_insert_sql() -> str: ST_Area(ST_Intersection(d.geom, o.geom)::geography) / NULLIF(GREATEST(ST_Area(d.geom::geography), ST_Area(o.geom::geography)), 0) )::double precision AS scene_overlap_ratio, + NULL::double precision AS pair_aoi_overlap_ratio, d.orbit_direction, + { _same_relative_orbit_expr('d', 'o') } AS same_relative_orbit, + CASE WHEN {dirty_is_master} THEN {_relative_orbit_value_expr('d')} ELSE {_relative_orbit_value_expr('o')} END AS master_relative_orbit, + CASE WHEN {dirty_is_master} THEN {_relative_orbit_value_expr('o')} ELSE {_relative_orbit_value_expr('d')} END AS slave_relative_orbit, + { _dinsar_quality_tier_expr('d', 'o', center_distance) } AS dinsar_quality_tier, + { _dinsar_quality_score_expr('d', 'o', center_distance) } AS dinsar_quality_score, + { _dinsar_readiness_expr('d', 'o', center_distance) } AS dinsar_readiness, + { _dinsar_reasons_expr('d', 'o', center_distance) } AS dinsar_reasons_json, (d.satellite IS NOT NULL AND o.satellite IS NOT NULL AND d.satellite = o.satellite) AS same_satellite, { _same_satellite_family_expr('d', 'o') } AS same_satellite_family, { _same_look_direction_expr('d', 'o') } AS same_look_direction, diff --git a/backend/app/services/pairing_state_service.py b/backend/app/services/pairing_state_service.py index 7b3cafc..9f86189 100644 --- a/backend/app/services/pairing_state_service.py +++ b/backend/app/services/pairing_state_service.py @@ -18,7 +18,7 @@ from ..models import ( PAIRING_CACHE_SCOPE_GLOBAL = "global" -DEFAULT_PAIRING_METRIC_VERSION = "2026.05.raw.v1" +DEFAULT_PAIRING_METRIC_VERSION = "2026.06.dinsar.family.v1" PAIRING_ORIENTATION_RULE_VERSION = "date_then_scene_uid_v1" diff --git a/backend/app/services/root_registry_service.py b/backend/app/services/root_registry_service.py index ee4fe6b..d78bf6a 100644 --- a/backend/app/services/root_registry_service.py +++ b/backend/app/services/root_registry_service.py @@ -235,15 +235,6 @@ def _build_root_specs_from_settings() -> List[RootSpec]: scan_mode="directory_walk", ) ) - specs.extend( - _iter_multi_root_specs( - env_var="GF3_ARCHIVE_SOURCE_DIRS", - paths=split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS), - root_role="source_pool_gf3_archive", - display_prefix="GF3 Archive Pool", - scan_mode="archive_walk", - ) - ) specs.extend( _iter_multi_root_specs( env_var="GF3_SOURCE_DIRS", diff --git a/backend/app/services/spatial_service.py b/backend/app/services/spatial_service.py index d990e98..fb44afe 100644 --- a/backend/app/services/spatial_service.py +++ b/backend/app/services/spatial_service.py @@ -14,7 +14,7 @@ from typing import Any, Dict, List, Optional, Tuple from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select -from sqlalchemy import and_, case, cast, func, or_ +from sqlalchemy import and_, case, cast, func, or_, text from sqlalchemy.orm import aliased from geoalchemy2 import Geography @@ -24,6 +24,8 @@ from shapely.geometry import Polygon from shapely.ops import unary_union from ..models import ( + DinsarProductionRunItemORM, + DinsarProductionRunORM, HazardPoint, HazardPointORM, PairingNetworkEdgeORM, @@ -43,7 +45,7 @@ from .dinsar_naming import build_pair_key, build_task_alias, ensure_unique_task_ from .pairing_state_service import pairing_state_service -PAIRING_POLICY_VERSION = "2026.05.raw-source.v2" +PAIRING_POLICY_VERSION = "2026.06.dinsar-production.v1" PAIRING_WARNING_CANDIDATE_THRESHOLD = 3000 PAIRING_ALL_STRATEGY_HARD_LIMIT = 20000 logger = logging.getLogger(__name__) @@ -145,10 +147,10 @@ class SpatialService: require_orbit_data=require_orbit_data, ) - if effective_params.strategy == "all" and len(candidate_pool) > PAIRING_ALL_STRATEGY_HARD_LIMIT: + if len(candidate_pool) > PAIRING_ALL_STRATEGY_HARD_LIMIT: raise RuntimeError( f"全部配对命中 {len(candidate_pool)} 条候选边，超过系统一次性返回上限 " - f"{PAIRING_ALL_STRATEGY_HARD_LIMIT}。请改用 SBAS/Sequential 策略，或收紧 AOI、日期范围、重叠率。" + f"{PAIRING_ALL_STRATEGY_HARD_LIMIT}。请收紧 AOI、日期范围、重叠率或中心距离阈值。" ) if len(candidate_pool) > PAIRING_WARNING_CANDIDATE_THRESHOLD: @@ -156,15 +158,21 @@ class SpatialService: f"候选配对数超过 {PAIRING_WARNING_CANDIDATE_THRESHOLD}（当前: {len(candidate_pool)}），建议收紧参数或缩小 AOI。" ) - selected_candidates, strategy_warnings = self._apply_strategy( - candidate_pool, - effective_params, - aoi_wkt=aoi_wkt, - ) + selected_candidates, strategy_warnings = self._apply_dinsar_production_strategy(candidate_pool, effective_params) warnings.extend(strategy_warnings) + if not selected_candidates: + warnings.extend( + await self._build_empty_pairing_diagnostics( + db, + effective_params, + aoi_wkt=aoi_wkt, + require_orbit_data=require_orbit_data, + ) + ) for candidate in selected_candidates: - candidate.setdefault("selection_strategy", effective_params.strategy) + candidate["selection_strategy"] = "dinsar_production" + self._ensure_candidate_identity(candidate) network_run_id = await self._persist_network_run( db, @@ -176,6 +184,7 @@ class SpatialService: selected_candidates=selected_candidates, ) + await self._attach_dinsar_production_summaries(db, selected_candidates) result_pairs = self._generate_task_names(self._build_radar_pairs(selected_candidates)) metadata = { "fallback_used": False, @@ -227,13 +236,20 @@ class SpatialService: PairingMetricCacheORM.status == "READY", PairingMetricCacheORM.time_baseline_days >= params.time_baseline_min, PairingMetricCacheORM.time_baseline_days <= params.time_baseline_max, + PairingMetricCacheORM.master_imaging_date < PairingMetricCacheORM.slave_imaging_date, PairingMetricCacheORM.scene_overlap_ratio >= params.overlap_threshold, PairingMetricCacheORM.same_look_direction.is_(True), + PairingMetricCacheORM.dinsar_readiness.in_(["RECOMMENDED", "CANDIDATE"]), ) ) - if params.limit_footprint_center_distance: - stmt = stmt.where(center_distance_expr <= params.spatial_baseline_max_meters) + stmt = stmt.where(center_distance_expr <= params.spatial_baseline_max_meters) + stmt = stmt.where( + or_( + and_(master_family_expr == "LT1", slave_family_expr == "LT1"), + and_(master_family_expr == "S1", slave_family_expr == "S1"), + ) + ) if require_orbit_data: stmt = stmt.where( @@ -241,8 +257,7 @@ class SpatialService: slave_alias.has_orbit_data.is_(True), ) - if not params.cross_satellite_pairing: - stmt = stmt.where(PairingMetricCacheORM.same_satellite_family.is_(True)) + stmt = stmt.where(PairingMetricCacheORM.same_satellite_family.is_(True)) if params.require_same_imaging_mode: stmt = stmt.where(PairingMetricCacheORM.same_imaging_mode.is_(True)) @@ -259,20 +274,19 @@ class SpatialService: ) if params.allowed_satellites: - allowed_satellites = [ - str(item).strip().upper() - for item in params.allowed_satellites - if str(item).strip() - ] + allowed_satellites = [] + for item in params.allowed_satellites: + compact = str(item).strip().upper().replace("-", "").replace("_", "").replace(" ", "") + if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}: + allowed_satellites.append("LT1") + elif compact in {"S1", "S1A", "S1B", "S1C", "SENTINEL1", "SENTINEL1A", "SENTINEL1B", "SENTINEL1C"}: + allowed_satellites.append("S1") + allowed_satellites = list(dict.fromkeys(allowed_satellites)) + if not allowed_satellites: + return [] stmt = stmt.where( - or_( - func.upper(master_alias.satellite).in_(allowed_satellites), - func.upper(master_alias.satellite_family).in_(allowed_satellites), - ), - or_( - func.upper(slave_alias.satellite).in_(allowed_satellites), - func.upper(slave_alias.satellite_family).in_(allowed_satellites), - ), + master_family_expr.in_(allowed_satellites), + slave_family_expr.in_(allowed_satellites), ) if params.master_date_from: @@ -286,23 +300,23 @@ class SpatialService: if aoi_wkt: aoi_geom = func.ST_GeomFromText(aoi_wkt, 4326) + aoi_geog = cast(aoi_geom, Geography) + aoi_area = func.nullif(ST_Area(aoi_geog), 0) + pair_overlap_geom = ST_Intersection(master_alias.geom, slave_alias.geom) + pair_aoi_geom = ST_Intersection(pair_overlap_geom, aoi_geom) + pair_aoi_overlap_expr = (ST_Area(cast(pair_aoi_geom, Geography)) / aoi_area).label("pair_aoi_overlap_ratio") + stmt = stmt.add_columns(pair_aoi_overlap_expr) stmt = stmt.where( - ST_Intersects(master_alias.geom, aoi_geom), - ST_Intersects(slave_alias.geom, aoi_geom), + ST_Intersects(pair_overlap_geom, aoi_geom), ) if params.aoi_overlap_threshold is not None: - aoi_geog = cast(aoi_geom, Geography) - aoi_area = func.nullif(ST_Area(aoi_geog), 0) - master_inter_geog = cast(ST_Intersection(master_alias.geom, aoi_geom), Geography) - slave_inter_geog = cast(ST_Intersection(slave_alias.geom, aoi_geom), Geography) - stmt = stmt.where( - ST_Area(master_inter_geog) / aoi_area >= params.aoi_overlap_threshold, - ST_Area(slave_inter_geog) / aoi_area >= params.aoi_overlap_threshold, - ) + stmt = stmt.where(pair_aoi_overlap_expr >= params.aoi_overlap_threshold) stmt = stmt.order_by( PairingMetricCacheORM.master_imaging_date.asc(), PairingMetricCacheORM.slave_imaging_date.asc(), + PairingMetricCacheORM.dinsar_quality_tier.asc(), + func.coalesce(PairingMetricCacheORM.dinsar_quality_score, 0).desc(), func.coalesce(PairingMetricCacheORM.scene_overlap_ratio, 0).desc(), center_distance_expr.asc(), PairingMetricCacheORM.pair_uid.asc(), @@ -310,7 +324,9 @@ class SpatialService: result = await db.execute(stmt) candidate_pool: List[dict] = [] - for metric_row, master_row, slave_row in result.all(): + for row in result.all(): + metric_row, master_row, slave_row = row[0], row[1], row[2] + pair_aoi_overlap_ratio = row[3] if len(row) > 3 else None center_distance = float( metric_row.scene_center_distance_meters if metric_row.scene_center_distance_meters is not None @@ -328,16 +344,54 @@ class SpatialService: "dist": center_distance, "scene_center_distance_meters": center_distance, "overlap_ratio": float(metric_row.scene_overlap_ratio or 0), + "dinsar_quality_tier": metric_row.dinsar_quality_tier, + "dinsar_quality_score": ( + float(metric_row.dinsar_quality_score) + if metric_row.dinsar_quality_score is not None + else None + ), + "dinsar_readiness": metric_row.dinsar_readiness, + "dinsar_reasons": [ + str(item) + for item in (metric_row.dinsar_reasons_json or []) + if isinstance(item, str) and item.strip() + ], + "same_relative_orbit": bool(metric_row.same_relative_orbit), + "master_relative_orbit": metric_row.master_relative_orbit, + "slave_relative_orbit": metric_row.slave_relative_orbit, + "pair_aoi_overlap_ratio": ( + float(pair_aoi_overlap_ratio) + if pair_aoi_overlap_ratio is not None + else None + ), } ) return candidate_pool + def _ensure_candidate_identity(self, candidate: dict) -> Tuple[str, str]: + master = candidate["master"] + slave = candidate["slave"] + task_alias = str(candidate.get("task_alias") or "").strip() or build_task_alias( + master.imaging_date, + slave.imaging_date, + ) + pair_key = str(candidate.get("pair_key") or "").strip() or build_pair_key( + master.file_path, + slave.file_path, + master.imaging_date, + slave.imaging_date, + master.satellite_family or slave.satellite_family or master.satellite or slave.satellite, + ) + candidate["task_alias"] = task_alias + candidate["pair_key"] = pair_key + return task_alias, pair_key + def _build_radar_pairs(self, selected_candidates: List[dict]) -> List[RadarPair]: result_pairs: List[RadarPair] = [] for candidate in selected_candidates: master = candidate["master"] slave = candidate["slave"] - task_alias = build_task_alias(master.imaging_date, slave.imaging_date) + task_alias, pair_key = self._ensure_candidate_identity(candidate) selection_score = candidate.get("selection_score") result_pairs.append( RadarPair( @@ -345,13 +399,7 @@ class SpatialService: slave=slave, task_name=task_alias, task_alias=task_alias, - pair_key=build_pair_key( - master.file_path, - slave.file_path, - master.imaging_date, - slave.imaging_date, - master.satellite_family or slave.satellite_family or master.satellite or slave.satellite, - ), + pair_key=pair_key, pair_uid=candidate.get("pair_uid"), metric_cache_ref_id=candidate.get("metric_cache_ref_id"), network_run_id=candidate.get("network_run_id"), @@ -367,10 +415,292 @@ class SpatialService: if candidate.get("scene_center_distance_meters") is not None else candidate.get("dist") or 0 ), + scene_overlap_ratio=float(candidate.get("overlap_ratio") or 0.0), + pair_aoi_overlap_ratio=( + float(candidate["pair_aoi_overlap_ratio"]) + if candidate.get("pair_aoi_overlap_ratio") is not None + else None + ), + dinsar_quality_tier=candidate.get("dinsar_quality_tier"), + dinsar_quality_score=( + float(candidate["dinsar_quality_score"]) + if candidate.get("dinsar_quality_score") is not None + else None + ), + dinsar_readiness=candidate.get("dinsar_readiness"), + dinsar_reasons=candidate.get("dinsar_reasons") or [], + same_relative_orbit=bool(candidate.get("same_relative_orbit")), + master_relative_orbit=candidate.get("master_relative_orbit"), + slave_relative_orbit=candidate.get("slave_relative_orbit"), + production_summary=candidate.get("production_summary"), ) ) return result_pairs + async def _attach_dinsar_production_summaries( + self, + db: AsyncSession, + selected_candidates: List[dict], + ) -> None: + if not selected_candidates: + return + + pair_uids: set[str] = set() + pair_keys: set[str] = set() + aliases: set[str] = set() + for candidate in selected_candidates: + task_alias, pair_key = self._ensure_candidate_identity(candidate) + pair_uid = str(candidate.get("pair_uid") or "").strip() + if pair_uid: + pair_uids.add(pair_uid) + if pair_key: + pair_keys.add(pair_key) + if task_alias: + aliases.add(task_alias) + + run_conditions = [] + if pair_uids: + run_conditions.append(DinsarProductionRunItemORM.pair_uid.in_(pair_uids)) + if pair_keys: + run_conditions.append(DinsarProductionRunItemORM.pair_key.in_(pair_keys)) + if aliases: + run_conditions.append(DinsarProductionRunItemORM.task_alias.in_(aliases)) + run_conditions.append(DinsarProductionRunItemORM.task_name.in_(aliases)) + + product_conditions = [] + if pair_uids: + product_conditions.append(ResultProductORM.pair_uid.in_(pair_uids)) + if pair_keys: + product_conditions.append(ResultProductORM.pair_key.in_(pair_keys)) + if aliases: + product_conditions.append(ResultProductORM.task_alias.in_(aliases)) + product_conditions.append(ResultProductORM.task_name.in_(aliases)) + + run_rows = [] + if run_conditions: + result = await db.execute( + select(DinsarProductionRunItemORM, DinsarProductionRunORM) + .join(DinsarProductionRunORM, DinsarProductionRunItemORM.run_id == DinsarProductionRunORM.run_id) + .where(or_(*run_conditions)) + .order_by( + DinsarProductionRunItemORM.updated_at.desc().nullslast(), + DinsarProductionRunItemORM.id.desc(), + ) + ) + run_rows = result.all() + + products = [] + if product_conditions: + result = await db.execute( + select(ResultProductORM) + .where(ResultProductORM.catalog_name == "dinsar") + .where(or_(*product_conditions)) + .order_by( + ResultProductORM.published_at.desc().nullslast(), + ResultProductORM.id.desc(), + ) + ) + products = result.scalars().all() + + run_by_uid: Dict[str, List[Tuple[DinsarProductionRunItemORM, DinsarProductionRunORM]]] = defaultdict(list) + run_by_key: Dict[str, List[Tuple[DinsarProductionRunItemORM, DinsarProductionRunORM]]] = defaultdict(list) + run_by_alias: Dict[str, List[Tuple[DinsarProductionRunItemORM, DinsarProductionRunORM]]] = defaultdict(list) + for item, run in run_rows: + pair_uid = str(item.pair_uid or "").strip() + pair_key = str(item.pair_key or "").strip() + if pair_uid: + run_by_uid[pair_uid].append((item, run)) + if pair_key: + run_by_key[pair_key].append((item, run)) + for alias in {str(item.task_alias or "").strip(), str(item.task_name or "").strip()}: + if alias: + run_by_alias[alias].append((item, run)) + + products_by_uid: Dict[str, List[ResultProductORM]] = defaultdict(list) + products_by_key: Dict[str, List[ResultProductORM]] = defaultdict(list) + products_by_alias: Dict[str, List[ResultProductORM]] = defaultdict(list) + for product in products: + pair_uid = str(product.pair_uid or "").strip() + pair_key = str(product.pair_key or "").strip() + if pair_uid: + products_by_uid[pair_uid].append(product) + if pair_key: + products_by_key[pair_key].append(product) + for alias in {str(product.task_alias or "").strip(), str(product.task_name or "").strip()}: + if alias: + products_by_alias[alias].append(product) + + for candidate in selected_candidates: + task_alias, pair_key = self._ensure_candidate_identity(candidate) + pair_uid = str(candidate.get("pair_uid") or "").strip() + exact_runs = self._dedupe_by_object_id( + [*run_by_uid.get(pair_uid, []), *run_by_key.get(pair_key, [])], + key=lambda row: getattr(row[0], "id", None), + ) + alias_runs = self._dedupe_by_object_id( + run_by_alias.get(task_alias, []), + key=lambda row: getattr(row[0], "id", None), + ) + exact_products = self._dedupe_by_object_id( + [*products_by_uid.get(pair_uid, []), *products_by_key.get(pair_key, [])], + key=lambda product: getattr(product, "id", None), + ) + alias_products = self._dedupe_by_object_id( + products_by_alias.get(task_alias, []), + key=lambda product: getattr(product, "id", None), + ) + matched_runs = exact_runs or alias_runs + matched_products = exact_products or alias_products + candidate["production_summary"] = self._summarize_dinsar_production( + matched_runs, + matched_products, + match_level="identity" if (exact_runs or exact_products) else ("task_alias" if (alias_runs or alias_products) else "none"), + ) + + def _summarize_dinsar_production( + self, + run_rows: List[Tuple[DinsarProductionRunItemORM, DinsarProductionRunORM]], + products: List[ResultProductORM], + *, + match_level: str = "none", + ) -> Dict[str, Any]: + latest_run_row = max( + run_rows, + key=lambda row: self._datetime_sort_key( + row[0].updated_at, + row[0].ended_at, + row[0].started_at, + row[0].created_at, + ), + default=None, + ) + latest_product = max( + products, + key=lambda product: self._datetime_sort_key( + product.published_at, + product.produced_at, + product.updated_at, + product.registered_at, + ), + default=None, + ) + ready_products = [product for product in products if self._is_ready_result_product(product)] + completed_statuses = {"COMPLETED", "READY", "SUCCESS", "PUBLISHED"} + failed_statuses = {"FAILED", "ERROR", "CANCELLED", "CANCELED"} + completed_run_count = sum( + 1 + for item, run in run_rows + if str(item.status or "").strip().upper() in completed_statuses + or str(run.status or "").strip().upper() in completed_statuses + ) + failed_run_count = sum( + 1 + for item, run in run_rows + if str(item.status or "").strip().upper() in failed_statuses + or str(run.status or "").strip().upper() in failed_statuses + ) + + latest_item = latest_run_row[0] if latest_run_row else None + latest_run = latest_run_row[1] if latest_run_row else None + if ready_products and latest_product is not None: + latest_status = str(latest_product.status or "").strip().upper() + elif latest_item is not None or latest_run is not None: + latest_status = str( + (latest_item.status if latest_item is not None else None) + or (latest_run.status if latest_run is not None else None) + or "" + ).strip().upper() + elif latest_product is not None: + latest_status = str(latest_product.status or "").strip().upper() + else: + latest_status = "" + if ready_products: + status = "READY" + elif completed_run_count > 0: + status = "COMPLETED" + elif latest_status: + status = latest_status + else: + status = "MISSING" + + engine_codes = sorted( + { + str(value or "").strip().lower() + for value in [ + *(product.engine_code for product in products), + *(run.engine_code for _, run in run_rows), + ] + if str(value or "").strip() + } + ) + return { + "has_record": bool(run_rows or products), + "is_produced": bool(ready_products or completed_run_count > 0), + "status": status, + "match_level": match_level, + "run_item_count": len(run_rows), + "completed_run_count": completed_run_count, + "failed_run_count": failed_run_count, + "product_count": len(products), + "ready_product_count": len(ready_products), + "engine_codes": engine_codes, + "latest_engine_code": ( + str(latest_product.engine_code or "").strip().lower() + if latest_product is not None and latest_product.engine_code + else ( + str(latest_run.engine_code or "").strip().lower() + if latest_run is not None and latest_run.engine_code + else None + ) + ), + "latest_run_id": latest_run.run_id if latest_run is not None else None, + "latest_run_status": latest_run.status if latest_run is not None else None, + "latest_item_status": latest_item.status if latest_item is not None else None, + "latest_output_dir": latest_item.latest_output_dir if latest_item is not None else None, + "latest_product_id": latest_product.id if latest_product is not None else None, + "latest_product_identifier": latest_product.product_id if latest_product is not None else None, + "latest_product_status": latest_product.status if latest_product is not None else None, + "latest_product_health": latest_product.health_status if latest_product is not None else None, + "latest_product_published_at": latest_product.published_at if latest_product is not None else None, + "updated_at": ( + latest_item.updated_at + if latest_item is not None + else ( + latest_product.updated_at + if latest_product is not None + else None + ) + ), + } + + def _is_ready_result_product(self, product: ResultProductORM) -> bool: + status = str(product.status or "").strip().upper() + health = str(product.health_status or "").strip().upper() + return status in {"READY", "COMPLETED", "SUCCESS"} and health not in {"ERROR", "FAILED"} + + def _datetime_sort_key(self, *values: Any) -> float: + for value in values: + if value is None: + continue + try: + return float(value.timestamp()) + except Exception: + continue + return 0.0 + + def _dedupe_by_object_id(self, items: List[Any], *, key) -> List[Any]: + seen: set[Any] = set() + output: List[Any] = [] + for item in items: + item_key = key(item) + if item_key is None: + item_key = id(item) + if item_key in seen: + continue + seen.add(item_key) + output.append(item) + return output + async def _persist_network_run( self, db: AsyncSession, @@ -451,6 +781,8 @@ class SpatialService: "master_scene_uid": candidate.get("master_scene_uid"), "slave_scene_uid": candidate.get("slave_scene_uid"), "pair_uid": candidate.get("pair_uid"), + "pair_key": candidate.get("pair_key"), + "task_alias": candidate.get("task_alias"), "time_baseline_days": int(candidate.get("days") or 0), "spatial_baseline_meters": float(candidate.get("dist") or 0.0), "scene_center_distance_meters": float( @@ -460,6 +792,11 @@ class SpatialService: ), "legacy_spatial_baseline_field": "scene_center_distance_meters", "scene_overlap_ratio": float(candidate.get("overlap_ratio") or 0.0), + "pair_aoi_overlap_ratio": ( + float(candidate["pair_aoi_overlap_ratio"]) + if candidate.get("pair_aoi_overlap_ratio") is not None + else None + ), } def _stable_sha1(self, value: Any) -> str: @@ -814,6 +1151,226 @@ class SpatialService: "scenes": scene_payloads, } + def _apply_dinsar_production_strategy( + self, + candidate_pool: List[dict], + params: PairingRequest, + ) -> Tuple[List[dict], List[str]]: + if not candidate_pool: + return [], [] + + warnings: List[str] = [] + rejected_count = sum( + 1 + for candidate in candidate_pool + if str(candidate.get("dinsar_readiness") or "").upper() == "NOT_RECOMMENDED" + ) + if rejected_count: + warnings.append(f"{rejected_count}条候选因D-InSAR生产前置条件不足被过滤。") + + selected = [] + for candidate in candidate_pool: + readiness = str(candidate.get("dinsar_readiness") or "CANDIDATE").upper() + if readiness == "NOT_RECOMMENDED": + continue + tier = str(candidate.get("dinsar_quality_tier") or "C").upper() + quality_score = candidate.get("dinsar_quality_score") + selected.append( + { + **candidate, + "selection_reason": f"dinsar_{readiness.lower()}_{tier.lower()}", + "selection_score": ( + float(quality_score) + if quality_score is not None + else self._score_pair_candidate(candidate, params) + ), + } + ) + + selected.sort( + key=lambda item: ( + {"A": 0, "B": 1, "C": 2}.get(str(item.get("dinsar_quality_tier") or "C").upper(), 9), + -float(item.get("selection_score") or 0), + int(item.get("days") or 0), + float(item.get("dist") or 0), + str(getattr(item.get("master"), "imaging_date", "") or ""), + str(getattr(item.get("slave"), "imaging_date", "") or ""), + ) + ) + return selected, warnings + + async def _build_empty_pairing_diagnostics( + self, + db: AsyncSession, + params: PairingRequest, + *, + aoi_wkt: Optional[str], + require_orbit_data: bool, + ) -> List[str]: + allowed_families: List[str] = [] + for item in params.allowed_satellites or []: + compact = str(item).strip().upper().replace("-", "").replace("_", "").replace(" ", "") + if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}: + allowed_families.append("LT1") + elif compact in {"S1", "S1A", "S1B", "S1C", "SENTINEL1", "SENTINEL1A", "SENTINEL1B", "SENTINEL1C"}: + allowed_families.append("S1") + allowed_families = list(dict.fromkeys(allowed_families)) + + sql = text( + """ + WITH base AS ( + SELECT + pmc.*, + m.satellite AS master_satellite_actual, + s.satellite AS slave_satellite_actual, + m.has_orbit_data AS master_has_orbit, + s.has_orbit_data AS slave_has_orbit, + COALESCE(pmc.scene_center_distance_meters, pmc.spatial_baseline_meters) AS center_m + FROM pairing_metric_cache pmc + JOIN radar_data m ON m.id = pmc.master_scene_ref_id + JOIN radar_data s ON s.id = pmc.slave_scene_ref_id + WHERE pmc.metric_version = :metric_version + AND pmc.status = 'READY' + AND pmc.master_imaging_date < pmc.slave_imaging_date + AND pmc.same_look_direction IS TRUE + AND pmc.same_satellite_family IS TRUE + AND pmc.dinsar_readiness IN ('RECOMMENDED', 'CANDIDATE') + AND (:require_orbit_data IS FALSE OR (m.has_orbit_data IS TRUE AND s.has_orbit_data IS TRUE)) + AND (:require_same_imaging_mode IS FALSE OR pmc.same_imaging_mode IS TRUE) + AND (:require_same_polarization IS FALSE OR pmc.same_polarization IS TRUE) + AND ( + :allowed_families_is_empty IS TRUE + OR pmc.master_satellite_family = ANY(:allowed_families) + OR pmc.master_satellite = ANY(:allowed_families) + ) + AND (CAST(:master_date_from AS text) IS NULL OR pmc.master_imaging_date >= CAST(:master_date_from AS text)) + AND (CAST(:master_date_to AS text) IS NULL OR pmc.master_imaging_date <= CAST(:master_date_to AS text)) + AND (CAST(:slave_date_from AS text) IS NULL OR pmc.slave_imaging_date >= CAST(:slave_date_from AS text)) + AND (CAST(:slave_date_to AS text) IS NULL OR pmc.slave_imaging_date <= CAST(:slave_date_to AS text)) + AND ( + CAST(:aoi_wkt AS text) IS NULL + OR ST_Intersects( + ST_Intersection(m.geom, s.geom), + ST_GeomFromText(CAST(:aoi_wkt AS text), 4326) + ) + ) + ), + time_ok AS ( + SELECT * FROM base + WHERE time_baseline_days BETWEEN :time_baseline_min AND :time_baseline_max + ), + overlap_ok AS ( + SELECT * FROM time_ok + WHERE scene_overlap_ratio >= :overlap_threshold + ), + center_ok AS ( + SELECT * FROM overlap_ok + WHERE center_m <= :center_distance_max + ) + SELECT + (SELECT count(*) FROM base) AS base_count, + (SELECT count(*) FROM time_ok) AS time_ok_count, + (SELECT count(*) FROM overlap_ok) AS overlap_ok_count, + (SELECT count(*) FROM center_ok) AS center_ok_count, + ( + SELECT json_build_object( + 'master_date', master_imaging_date, + 'slave_date', slave_imaging_date, + 'master_satellite', master_satellite_actual, + 'slave_satellite', slave_satellite_actual, + 'time_baseline_days', time_baseline_days, + 'center_meters', center_m, + 'overlap_ratio', scene_overlap_ratio, + 'quality_tier', dinsar_quality_tier, + 'readiness', dinsar_readiness + ) + FROM base + ORDER BY + CASE + WHEN time_baseline_days BETWEEN :time_baseline_min AND :time_baseline_max + THEN 0 ELSE 1 + END, + CASE WHEN scene_overlap_ratio >= :overlap_threshold THEN 0 ELSE 1 END, + abs(time_baseline_days - :time_baseline_max), + center_m ASC NULLS LAST + LIMIT 1 + ) AS nearest_candidate; + """ + ) + result = await db.execute( + sql, + { + "metric_version": pairing_state_service.metric_version, + "require_orbit_data": require_orbit_data, + "require_same_imaging_mode": bool(params.require_same_imaging_mode), + "require_same_polarization": bool(params.require_same_polarization), + "allowed_families": allowed_families or ["__NONE__"], + "allowed_families_is_empty": not allowed_families, + "master_date_from": params.master_date_from or None, + "master_date_to": params.master_date_to or None, + "slave_date_from": params.slave_date_from or None, + "slave_date_to": params.slave_date_to or None, + "aoi_wkt": aoi_wkt, + "time_baseline_min": int(params.time_baseline_min), + "time_baseline_max": int(params.time_baseline_max), + "overlap_threshold": float(params.overlap_threshold), + "center_distance_max": float(params.spatial_baseline_max_meters), + }, + ) + row = result.mappings().first() + if not row: + return ["未找到满足条件的 D-InSAR 配对；诊断查询未返回统计结果。"] + + base_count = int(row.get("base_count") or 0) + time_ok_count = int(row.get("time_ok_count") or 0) + overlap_ok_count = int(row.get("overlap_ok_count") or 0) + center_ok_count = int(row.get("center_ok_count") or 0) + if base_count <= 0: + family_text = "、".join(allowed_families) if allowed_families else "LT-1/Sentinel-1" + return [ + f"未找到 {family_text} 的可生产基础候选边。请检查数据体系、主从日期范围、AOI、精轨绑定和配对缓存状态。" + ] + + messages = [ + ( + "当前筛选下基础候选 {base} 条；时间基线 {min_days}-{max_days} 天后剩 {time_ok} 条；" + "重叠率 >= {overlap:.2f} 后剩 {overlap_ok} 条；footprint 中心距离 <= {center:.0f} 米后剩 {center_ok} 条。" + ).format( + base=base_count, + min_days=int(params.time_baseline_min), + max_days=int(params.time_baseline_max), + time_ok=time_ok_count, + overlap=float(params.overlap_threshold), + overlap_ok=overlap_ok_count, + center=float(params.spatial_baseline_max_meters), + center_ok=center_ok_count, + ) + ] + nearest = row.get("nearest_candidate") + if isinstance(nearest, str): + try: + nearest = json.loads(nearest) + except Exception: + nearest = None + if isinstance(nearest, dict): + messages.append( + ( + "最接近的一对是 {master_satellite} {master_date} -> {slave_satellite} {slave_date}，" + "时间基线 {days} 天，中心距离 {center:.1f} 米，重叠率 {overlap:.3f}，质量 {tier}/{readiness}。" + ).format( + master_satellite=nearest.get("master_satellite") or "?", + master_date=nearest.get("master_date") or "?", + slave_satellite=nearest.get("slave_satellite") or "?", + slave_date=nearest.get("slave_date") or "?", + days=int(nearest.get("time_baseline_days") or 0), + center=float(nearest.get("center_meters") or 0.0), + overlap=float(nearest.get("overlap_ratio") or 0.0), + tier=nearest.get("quality_tier") or "?", + readiness=nearest.get("readiness") or "?", + ) + ) + return messages + def _apply_strategy( self, candidate_pool: List[dict], diff --git a/backend/app/services/task_service.py b/backend/app/services/task_service.py index 8fc70f8..c4621b9 100644 --- a/backend/app/services/task_service.py +++ b/backend/app/services/task_service.py @@ -259,12 +259,15 @@ class TaskService: task = result.scalar_one_or_none() if task: + previous_message = task.message if status: task.status = status if progress is not None: task.progress = progress if message: task.message = message + if message != previous_message: + await self._add_log(task_id, "INFO", message, db=db) # 如果任务结束，更新结束时间 if status in ["COMPLETED", "FAILED", "CANCELLED"]: diff --git a/backend/app/services/unpack_service.py b/backend/app/services/unpack_service.py index 5771049..1d04dbe 100644 --- a/backend/app/services/unpack_service.py +++ b/backend/app/services/unpack_service.py @@ -39,7 +39,7 @@ def get_unpack_config() -> Dict[str, Any]: "source_dirs": source_dirs, "insar_storage_dirs": insar_storage_dirs, "min_disk_space_gb": float(env.get("UNPACK_MIN_DISK_SPACE_GB", "50")), - "delete_archive": module.parse_bool(env.get("UNPACK_DELETE_ARCHIVE", "true")), + "delete_archive": False, "tmp_suffix": env.get("UNPACK_TMP_SUFFIX", ".unpack_tmp"), "archive_exts": archive_exts, "scan_workers": module.parse_int( diff --git a/backend/app/utils.py b/backend/app/utils.py index 4905d88..ba7f7ed 100644 --- a/backend/app/utils.py +++ b/backend/app/utils.py @@ -1,5 +1,6 @@ import os import re +import math from datetime import datetime from typing import Optional, Tuple, List, Callable, Dict, Any from lxml import etree @@ -20,6 +21,80 @@ def _create_secure_xml_parser() -> etree.XMLParser: recover=False, ) + +def _ordered_closed_polygon(points: List[Tuple[float, float]]) -> Optional[List[Tuple[float, float]]]: + unique: List[Tuple[float, float]] = [] + for point in points or []: + try: + lon = float(point[0]) + lat = float(point[1]) + except (TypeError, ValueError, IndexError): + continue + current = (lon, lat) + if unique and abs(unique[-1][0] - lon) < 1e-12 and abs(unique[-1][1] - lat) < 1e-12: + continue + if unique and abs(unique[0][0] - lon) < 1e-12 and abs(unique[0][1] - lat) < 1e-12: + continue + if current not in unique: + unique.append(current) + if len(unique) < 3: + return None + if len(unique) == 4: + center_lon = sum(item[0] for item in unique) / len(unique) + center_lat = sum(item[1] for item in unique) / len(unique) + ordered = sorted( + unique, + key=lambda item: math.atan2(item[1] - center_lat, item[0] - center_lon), + ) + else: + ordered = unique + if ordered[0] != ordered[-1]: + ordered.append(ordered[0]) + return ordered + + +def _closed_polygon_if_valid(points: List[Tuple[float, float]]) -> Optional[List[Tuple[float, float]]]: + ring = [(float(point[0]), float(point[1])) for point in points or []] + if len(ring) < 3: + return None + if ring[0] != ring[-1]: + ring.append(ring[0]) + return ring + + +def _ordered_closed_polygon_from_corner_details(corner_details: Dict[str, Dict[str, Any]]) -> Optional[List[Tuple[float, float]]]: + by_name = {str(key or "").strip().lower(): value for key, value in (corner_details or {}).items()} + name_order = ["bottomleft", "bottomright", "topright", "topleft"] + if all(name in by_name for name in name_order): + return _closed_polygon_if_valid([(by_name[name]["lon"], by_name[name]["lat"]) for name in name_order]) + + entries = [ + value + for value in (corner_details or {}).values() + if value.get("lon") is not None + and value.get("lat") is not None + and value.get("ref_row") is not None + and value.get("ref_col") is not None + ] + if len(entries) >= 4: + min_row = min(float(item["ref_row"]) for item in entries) + max_row = max(float(item["ref_row"]) for item in entries) + min_col = min(float(item["ref_col"]) for item in entries) + max_col = max(float(item["ref_col"]) for item in entries) + targets = [(min_row, min_col), (min_row, max_col), (max_row, max_col), (max_row, min_col)] + remaining = list(entries) + ordered_entries: List[Dict[str, Any]] = [] + for target_row, target_col in targets: + chosen = min( + remaining, + key=lambda item: abs(float(item["ref_row"]) - target_row) + abs(float(item["ref_col"]) - target_col), + ) + ordered_entries.append(chosen) + remaining.remove(chosen) + return _closed_polygon_if_valid([(item["lon"], item["lat"]) for item in ordered_entries]) + + return _ordered_closed_polygon([(value["lon"], value["lat"]) for value in (corner_details or {}).values()]) + # --- Sentinel-1 (S1A/S1B/S1C) Parsers --- def _radar_meta_base() -> Dict[str, Any]: @@ -573,7 +648,10 @@ def parse_xml_metadata( return None, None if len(polygon) == 4: - polygon.append(polygon[0]) + ordered_polygon = _ordered_closed_polygon_from_corner_details(corner_details) + if not ordered_polygon: + return None, None + polygon = ordered_polygon corner_pixel_mapping = _build_corner_pixel_mapping(corner_details) meta = { "orbit_direction": orbit_direction, diff --git a/backend/migrations/011_source_metadata_documents.sql b/backend/migrations/011_source_metadata_documents.sql new file mode 100644 index 0000000..121d107 --- /dev/null +++ b/backend/migrations/011_source_metadata_documents.sql @@ -0,0 +1,138 @@ +-- Persist source XML/manifest documents and normalized scene geometry profiles. + +CREATE TABLE IF NOT EXISTS source_metadata_documents ( + id SERIAL PRIMARY KEY, + source_asset_id INTEGER NOT NULL REFERENCES source_product_assets(id) ON DELETE CASCADE, + radar_data_id INTEGER NULL REFERENCES radar_data(id) ON DELETE SET NULL, + satellite_family VARCHAR(32) NULL, + source_format VARCHAR(32) NULL, + document_type VARCHAR(32) NOT NULL, + member_path TEXT NOT NULL, + content_sha256 VARCHAR(64) NOT NULL, + content_encoding VARCHAR(16) NOT NULL DEFAULT 'gzip', + content_bytes BYTEA NOT NULL, + content_size_bytes BIGINT NULL, + archive_path TEXT NULL, + archive_mtime DOUBLE PRECISION NULL, + parser_version VARCHAR(32) NULL, + parse_status VARCHAR(32) NOT NULL DEFAULT 'OK', + parse_error TEXT NULL, + extracted_at TIMESTAMP NULL, + created_at TIMESTAMP NOT NULL DEFAULT NOW(), + updated_at TIMESTAMP NULL DEFAULT NOW(), + CONSTRAINT uq_source_metadata_document_member UNIQUE (source_asset_id, document_type, member_path) +); + +CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_source_asset_id + ON source_metadata_documents (source_asset_id); +CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_radar_data_id + ON source_metadata_documents (radar_data_id); +CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_satellite_family + ON source_metadata_documents (satellite_family); +CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_source_format + ON source_metadata_documents (source_format); +CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_document_type + ON source_metadata_documents (document_type); +CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_content_sha256 + ON source_metadata_documents (content_sha256); +CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_parse_status + ON source_metadata_documents (parse_status); +CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_asset_type + ON source_metadata_documents (source_asset_id, document_type); +CREATE INDEX IF NOT EXISTS idx_source_metadata_documents_radar_type + ON source_metadata_documents (radar_data_id, document_type); + +CREATE TABLE IF NOT EXISTS sar_scene_geometry_profiles ( + id SERIAL PRIMARY KEY, + source_asset_id INTEGER NOT NULL UNIQUE REFERENCES source_product_assets(id) ON DELETE CASCADE, + radar_data_id INTEGER NULL UNIQUE REFERENCES radar_data(id) ON DELETE CASCADE, + satellite_family VARCHAR(32) NULL, + satellite VARCHAR(32) NULL, + source_format VARCHAR(32) NULL, + imaging_mode VARCHAR(64) NULL, + polarization VARCHAR(64) NULL, + orbit_direction VARCHAR(32) NULL, + look_direction VARCHAR(32) NULL, + absolute_orbit VARCHAR(64) NULL, + relative_orbit VARCHAR(64) NULL, + acquisition_start_time_utc TIMESTAMP NULL, + acquisition_stop_time_utc TIMESTAMP NULL, + scene_center_lon DOUBLE PRECISION NULL, + scene_center_lat DOUBLE PRECISION NULL, + footprint_geom GEOMETRY(POLYGON, 4326) NULL, + footprint_polygon JSON NULL, + swath_summary_json JSON NULL, + burst_summary_json JSON NULL, + incidence_angle_min DOUBLE PRECISION NULL, + incidence_angle_max DOUBLE PRECISION NULL, + doppler_summary_json JSON NULL, + state_vector_summary_json JSON NULL, + metadata_quality VARCHAR(32) NOT NULL DEFAULT 'UNKNOWN', + production_readiness VARCHAR(32) NOT NULL DEFAULT 'UNKNOWN', + readiness_reasons_json JSON NULL, + parser_version VARCHAR(32) NULL, + parsed_at TIMESTAMP NULL, + created_at TIMESTAMP NOT NULL DEFAULT NOW(), + updated_at TIMESTAMP NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_source_asset_id + ON sar_scene_geometry_profiles (source_asset_id); +CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_radar_data_id + ON sar_scene_geometry_profiles (radar_data_id); +CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_satellite_family + ON sar_scene_geometry_profiles (satellite_family); +CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_satellite + ON sar_scene_geometry_profiles (satellite); +CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_source_format + ON sar_scene_geometry_profiles (source_format); +CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_imaging_mode + ON sar_scene_geometry_profiles (imaging_mode); +CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_polarization + ON sar_scene_geometry_profiles (polarization); +CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_orbit_direction + ON sar_scene_geometry_profiles (orbit_direction); +CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_look_direction + ON sar_scene_geometry_profiles (look_direction); +CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_absolute_orbit + ON sar_scene_geometry_profiles (absolute_orbit); +CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_relative_orbit + ON sar_scene_geometry_profiles (relative_orbit); +CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_acquisition_start_time_utc + ON sar_scene_geometry_profiles (acquisition_start_time_utc); +CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_footprint_geom + ON sar_scene_geometry_profiles USING GIST (footprint_geom); +CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_metadata_quality + ON sar_scene_geometry_profiles (metadata_quality); +CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_production_readiness + ON sar_scene_geometry_profiles (production_readiness); +CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_family_date + ON sar_scene_geometry_profiles (satellite_family, acquisition_start_time_utc); +CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_track + ON sar_scene_geometry_profiles (satellite_family, relative_orbit, orbit_direction); +CREATE INDEX IF NOT EXISTS idx_sar_scene_geometry_profiles_readiness + ON sar_scene_geometry_profiles (production_readiness, metadata_quality); + +ALTER TABLE IF EXISTS pairing_metric_cache + ADD COLUMN IF NOT EXISTS pair_aoi_overlap_ratio DOUBLE PRECISION NULL; +ALTER TABLE IF EXISTS pairing_metric_cache + ADD COLUMN IF NOT EXISTS same_relative_orbit BOOLEAN NOT NULL DEFAULT FALSE; +ALTER TABLE IF EXISTS pairing_metric_cache + ADD COLUMN IF NOT EXISTS master_relative_orbit VARCHAR(64) NULL; +ALTER TABLE IF EXISTS pairing_metric_cache + ADD COLUMN IF NOT EXISTS slave_relative_orbit VARCHAR(64) NULL; +ALTER TABLE IF EXISTS pairing_metric_cache + ADD COLUMN IF NOT EXISTS dinsar_quality_tier VARCHAR(16) NOT NULL DEFAULT 'C'; +ALTER TABLE IF EXISTS pairing_metric_cache + ADD COLUMN IF NOT EXISTS dinsar_quality_score DOUBLE PRECISION NULL; +ALTER TABLE IF EXISTS pairing_metric_cache + ADD COLUMN IF NOT EXISTS dinsar_readiness VARCHAR(32) NOT NULL DEFAULT 'CANDIDATE'; +ALTER TABLE IF EXISTS pairing_metric_cache + ADD COLUMN IF NOT EXISTS dinsar_reasons_json JSON NULL; + +CREATE INDEX IF NOT EXISTS idx_pairing_metric_cache_same_relative_orbit + ON pairing_metric_cache (same_relative_orbit); +CREATE INDEX IF NOT EXISTS idx_pairing_metric_cache_quality_tier + ON pairing_metric_cache (dinsar_quality_tier); +CREATE INDEX IF NOT EXISTS idx_pairing_metric_cache_readiness + ON pairing_metric_cache (dinsar_readiness); diff --git a/backend/migrations/012_source_archive_integrity.sql b/backend/migrations/012_source_archive_integrity.sql new file mode 100644 index 0000000..691014a --- /dev/null +++ b/backend/migrations/012_source_archive_integrity.sql @@ -0,0 +1,20 @@ +ALTER TABLE source_product_assets + ADD COLUMN IF NOT EXISTS archive_integrity_status VARCHAR(32) NOT NULL DEFAULT 'NOT_CHECKED'; + +ALTER TABLE source_product_assets + ADD COLUMN IF NOT EXISTS archive_integrity_method VARCHAR(64) NULL; + +ALTER TABLE source_product_assets + ADD COLUMN IF NOT EXISTS archive_integrity_checked_at TIMESTAMP NULL; + +ALTER TABLE source_product_assets + ADD COLUMN IF NOT EXISTS archive_integrity_error TEXT NULL; + +ALTER TABLE source_product_assets + ADD COLUMN IF NOT EXISTS archive_integrity_version VARCHAR(32) NULL; + +ALTER TABLE source_product_assets + ADD COLUMN IF NOT EXISTS archive_integrity_member_count INTEGER NULL; + +CREATE INDEX IF NOT EXISTS idx_source_product_assets_archive_integrity_status + ON source_product_assets (archive_integrity_status); diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 2d1c0b9..5bad103 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -97,14 +97,16 @@ NGINX_HEALTH_URL=http://127.0.0.1/ UNPACK_SOURCE_DIRS=D:\Archives INSAR_STORAGE_DIRS=D:\LuTan1_Image_Pool MONITOR_RADAR_DIRS=D:\LuTan1_Image_Pool -MONITOR_DINSAR_DIRS=D:\DInSARResult +MONITOR_DINSAR_DIRS=D:\production_results\dinsar MONITOR_ORBIT_DIR=D:\LT1_data_lsarorbit ORBIT_POOL_ENVI=D:\orbit_pools\envi -ORBIT_POOL_ISCE2=D:\orbit_pools\isce2 +ORBIT_POOL_ISCE2= ORBIT_POOL_LANDSAR= ``` +`ORBIT_SOURCE_DIRS` is the source asset layer. LT-1 orbit scans also synchronize the production TXT pool under `ORBIT_POOL_ENVI\LT1A|LT1B`. `PYINT_ORBIT_POOL_TXT` and `GAMMA_SBAS_ORBIT_ROOTS` should point to that same local TXT pool unless a separate Gamma pool is deliberately maintained. `ORBIT_POOL_ISCE2` is legacy and remains empty while `ISCE2_ENABLED=false`. + ### 3.4 ENVI / SARscape ```env @@ -120,24 +122,25 @@ ENVI_TASK_TIMEOUT_SECONDS=21600 GF3 当前主线是“ENVI/SARscape 生产原生 `_geo` 证据层，平台标准化成 GeoTIFF 后入库和供洪涝/水体算法消费”。旧 Python/GDAL L1A 预处理链路默认关闭。 ```env -GF3_ARCHIVE_SOURCE_DIRS=D:\production_inputs\gf3\archives +GF3_TASK_POOL_ROOT=D:\GaoFen3_Pool\task_pool +GF3_ARCHIVE_SOURCE_DIRS=D:\GaoFen3_Pool\archives GF3_LEGACY_GDAL_ENABLED=false GF3_SOURCE_DIRS= -GF3_SARSCAPE_NATIVE_DIRS=D:\production_results\gf3\sarscape_native -GF3_STORAGE_DIRS=D:\production_results\gf3\standard_l2 -GF3_SARSCAPE_RUNTIME_DIR=D:\production_runtime\gf3\sarscape_runtime +GF3_SARSCAPE_NATIVE_DIRS=D:\GaoFen3_Pool\native_geo +GF3_STORAGE_DIRS=D:\GaoFen3_Pool\catalog +GF3_SARSCAPE_RUNTIME_DIR=D:\GaoFen3_Pool\task_pool\sarscape_runtime GF3_SARSCAPE_WRAPPER_EXE=D:\Code\Insar_management_system_v2\third_party\GF3_L1A_To_L2_pipeline\dist\windows\gf3wrapper.exe GF3_SARSCAPE_IDLRT_PATH=C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlrt.exe GF3_SARSCAPE_DEM_PATH=D:\DEM\COPDEM_GLO30_China_4326_DEM GF3_SARSCAPE_POLARIZATIONS=HH,HV -GF3_SARSCAPE_AUTO_STANDARDIZE=true +GF3_SARSCAPE_AUTO_STANDARDIZE=false GF3_SARSCAPE_CLEAN_AFTER_SUCCESS=true ``` 说明： - `GF3_SARSCAPE_NATIVE_DIRS` 长期保留 `_geo`、`.hdr`、`.sml`、快视、KML、日志和 manifest。 -- `GF3_STORAGE_DIRS` 是标准 L2 GeoTIFF 池，后续入库、预览、洪涝/水体分析优先消费这里和 `SAR_ANALYSIS_READY_ROOT`。 +- `GF3_STORAGE_DIRS` 是标准目录池，后续入库、预览、洪涝/水体分析优先消费这里和 `SAR_ANALYSIS_READY_ROOT`。 - `GF3_SARSCAPE_RUNTIME_DIR` 只放 wrapper 配置和运行时临时文件，不应混入原生结果池。 - 如确需恢复旧 L1A 解包/预处理，需要同时设置 `GF3_LEGACY_GDAL_ENABLED=true` 和 `GF3_SOURCE_DIRS`。 diff --git a/docs/DINSAR_TASK_POOL_THREE_ENGINE_REFACTOR_20260614.md b/docs/DINSAR_TASK_POOL_THREE_ENGINE_REFACTOR_20260614.md index 2bc82cb..1e9b8da 100644 --- a/docs/DINSAR_TASK_POOL_THREE_ENGINE_REFACTOR_20260614.md +++ b/docs/DINSAR_TASK_POOL_THREE_ENGINE_REFACTOR_20260614.md @@ -222,7 +222,7 @@ Current direction: - directory sources are copied into `master/` and `slave/`; - `S1_ZIP`, `LT1_ARCHIVE`, and other supported archives are extracted into `master/` and `slave/`; - staged orbit files go into `orbit/`. -- Engines must consume local Task_Pool paths, not UNC source archive paths. +- Engines must consume local Task_Pool paths. LT-1 and Sentinel-1 source archives are also local; UNC is not an active production source pool. - `.dinsar_pair.json` records `source_materialization` so cleanup can distinguish copied directories, extracted archives, and staged files. Cleanup implication: diff --git a/docs/FLOOD_WATER_ALGORITHM_ENGINEERING_HANDOFF_20260602.md b/docs/FLOOD_WATER_ALGORITHM_ENGINEERING_HANDOFF_20260602.md index bc5cbcc..1fc9a55 100644 --- a/docs/FLOOD_WATER_ALGORITHM_ENGINEERING_HANDOFF_20260602.md +++ b/docs/FLOOD_WATER_ALGORITHM_ENGINEERING_HANDOFF_20260602.md @@ -54,7 +54,7 @@ | `pixel_size_m` | 近似像元大小 | | `status` | `PENDING/RUNNING/DONE/FAILED` | -当前 GF3 SARscape 链路会先产出原生 `_geo` ENVI 二进制，再由平台转换为 `D:\production_results\gf3\standard_l2` 下的 GeoTIFF，并注册到这里。 +当前 GF3 SARscape 链路会先产出原生 `_geo` ENVI 二进制，再由平台转换为 `D:\GaoFen3_Image_Pool\standard_l2` 下的 GeoTIFF，并注册到这里。 ### 3.2 WaterExtractionORM diff --git a/docs/FRONTEND_NAVIGATION_ARCHITECTURE.md b/docs/FRONTEND_NAVIGATION_ARCHITECTURE.md index 1cdd201..327e0bc 100644 --- a/docs/FRONTEND_NAVIGATION_ARCHITECTURE.md +++ b/docs/FRONTEND_NAVIGATION_ARCHITECTURE.md @@ -1,28 +1,14 @@ # Frontend Navigation Architecture -## 1. Purpose +Last updated: 2026-06-20 -This document is the source of truth for the left-side navigation structure and the production workspace view model in the frontend. +This document is the frontend navigation source of truth. It reflects the current product decision: the system manages data for LT-1, Sentinel-1, and GF-3, but production is organized only around D-InSAR and SBAS-InSAR. PS-InSAR and legacy time-series entries are compatibility code, not visible primary workflows. -It explains: - -- the first-level menu groups -- the sectioned groups and their leaf tabs -- the special-case production workspace entry -- the reserved entries and legacy route aliases -- the files that must be updated when navigation changes - -The goal is to keep module boundaries stable as the system expands across D-InSAR, time-series InSAR, AI analysis, and operations workflows. - -## 2. Current First-Level Groups - -The current first-level menu groups are: +## Current First-Level Groups - `data`: 数据管理 -- `production_planning`: 生产规划 - `production_management`: 生产管理 - `insar_analysis`: InSAR形变分析 -- `ai_analysis`: AI分析 - `flood_analysis`: 洪涝灾害分析 - `ops`: 运行维护 @@ -30,179 +16,106 @@ Definition files: - `frontend/src/config/appConstants.js` - `frontend/src/utils/appUiHelpers.js` +- `frontend/src/components/app/AppSidePanel.jsx` +- `frontend/src/ProductionWorkspace.jsx` -## 3. Navigation Model - -The current frontend uses two navigation patterns: - -1. Sectioned navigation: - first-level group -> second-level section -> leaf tab -2. Workspace navigation: - first-level group -> single leaf tab -> internal workspace view switcher - -### 3.1 生产规划 - -This is a sectioned group. +## Data Management ```text -生产规划 -├─ 规划编组 -│ ├─ 配对规划 (`pairing`) -│ ├─ 任务规划 (`pairs`) -│ ├─ 时序候选栈 (`ps_results`) -│ └─ 任务批次 (`batches`) -└─ 数据分发 - └─ 数据分发 (`copier`) +数据管理 +├─ 入库监控 (`ingest`) +├─ 资产库存 (`asset_inventory`) +├─ 数据列表 (`data`) +└─ 灾害点 (`hazard`) ``` -Notes: +Boundary: -- `ps_results` here means planning-stage candidate stacks, not analysis-facing result pages. -- This group no longer hosts D-InSAR production or product pages. +- LT-1 and Sentinel-1 source data are managed as local compressed archives. +- Metadata, footprint, and preview are extracted from archives without full unpacking. +- Full materialization happens only when production preparation creates a task under the local Task_Pool. +- GF-3 registers copied native `_geo` production results and generates local WebP previews. -### 3.2 生产管理 +## Production Management -This is a workspace group, not a multi-tab planning tree. +`production_management` is a workspace group. The left navigation exposes one entry; the workspace owns the internal production views. ```text -生产管理 -└─ 生产管理 (`production_management`) - ├─ D-InSAR运行 (`dinsar_runs`) - ├─ SBAS-InSAR Production (`sbas_insar_production`) - ├─ SBAS-InSAR结果 (`sbas_insar_products`) - └─ D-InSAR产物 (`dinsar_products`) +生产管理 (`production_management`) +├─ D-InSAR配对规划 (`dinsar_pairing`) +├─ D-InSAR任务规划 (`dinsar_pairs`) +├─ D-InSAR任务批次 (`dinsar_batches`) +├─ D-InSAR生产准备/分发 (`dinsar_prepare`) +├─ D-InSAR运行 (`dinsar_runs`) +├─ D-InSAR产物 (`dinsar_products`) +├─ SBAS-InSAR Production (`sbas_insar_production`) +├─ SBAS-InSAR结果 (`sbas_insar_products`) +├─ 陆探生产占位 (`lt1_production`) +├─ 哨兵生产占位 (`sentinel1_production`) +└─ 高分三结果登记 (`gf3_native_registration`) ``` -Notes: +Boundary: -- The left navigation contains only one tab for this group: `production_management`. -- Internal workspace views are controlled by `PRODUCTION_WORKSPACE_VIEWS`. -- Route alias mapping is controlled by `PRODUCTION_WORKSPACE_ENTRY_TO_VIEW`. -- Legacy route tabs such as `dinsar_production`, `ps_production`, and `ps_products` map into this workspace and should not be treated as standalone left-nav entries. -- The old ISCE2/MintPy `timeseries_runs` and `timeseries_products` workspace views are deprecated and hidden; SBAS production is handled by the Gamma `sbas_insar_production` view. +- D-InSAR uses the sequence: pair planning -> selected pairs -> D-InSAR batch -> production preparation -> run -> product catalog. +- D-InSAR production preparation materializes archive sources into `DINSAR_TASK_POOL_ROOT` and must not use UNC paths. +- Data distribution is a separate D-InSAR mode that exports source archive bundles under `DATA_DISTRIBUTION_ROOT`; it is not the production runtime path. +- SBAS-InSAR uses the dedicated Gamma/LandSAR SBAS production page. It does not depend on the old D-InSAR pair list or PS candidate-stack page. +- GF-3 is not produced on this server. The server registers native `_geo` results copied into the configured GF-3 pool and builds WebP from the produced binary raster, not from quicklook TIFFs. -### 3.3 InSAR形变分析 +Compatibility route aliases: -This is a sectioned group. +- `pairing` -> `dinsar_pairing` +- `pairs` -> `dinsar_pairs` +- `ps_results` -> `sbas_insar_production` +- `batches` -> `dinsar_batches` +- `copier` -> `dinsar_prepare` +- `dinsar_production` -> `dinsar_runs` +- `dinsar_products` -> `dinsar_products` +- `ps_production` -> `sbas_insar_production` +- `ps_products` -> `sbas_insar_products` + +These aliases exist so existing code paths can redirect into the workspace. They are not standalone left-navigation entries. + +## InSAR Analysis ```text InSAR形变分析 ├─ D-InSAR │ ├─ D-InSAR结果 (`dinsar_results`) │ └─ D-InSAR分析 (`dinsar_analysis`) -└─ 时序InSAR - ├─ 时序InSAR结果 (`psinsar_results`) - └─ 时序InSAR分析 (`psinsar_analysis`) +│ ├─ AI质量评估 +│ └─ D-InSAR诊断 +└─ SBAS + └─ SBAS-InSAR分析 (`psinsar_analysis`) ``` -Notes: +Boundary: -- This group is for business-facing result browsing and interpretation. -- AI diagnosis does not belong here. -- `dinsar_analysis`, `psinsar_results`, and `psinsar_analysis` are currently reserved placeholders. +- Analysis pages consume registered result catalogs. +- They should not submit production jobs or materialize source archives. +- The standalone `AI分析` first-level page has been removed. D-InSAR quality assessment and D-InSAR diagnosis are owned by `dinsar_analysis`. +- D-InSAR diagnosis uses the `AI_DIAGNOSIS` task type and persists reports in the `ai_diagnosis` table. The older `AI_ANALYZE` endpoint/task is compatibility code only. +- `psinsar_analysis` remains the historical route key, but its user-facing meaning is SBAS-InSAR analysis. -### 3.4 AI分析 +## Deprecated Visible Workflows -This is a sectioned group. +The following workflows must not be shown as primary UI entries: -```text -AI分析 -├─ 形变智能分析 -│ ├─ AI质量评估 (`ai_quality`) -│ └─ D-InSAR诊断 (`ai_diagnosis`) -└─ 遥感视觉分析 - ├─ 滑坡语义分割 (`landslide_segmentation`) - └─ 无人机影像分析 (`uav_image_analysis`) -``` +- PS-InSAR production +- PS candidate-stack distribution +- legacy ISCE2/MintPy time-series production +- source-folder distribution for unpacked LT-1 or Sentinel-1 folders +- standalone AI analysis first-level navigation +- remote-sensing vision AI placeholder pages -Notes: +Backend compatibility code may remain until historical data models and catalog names are migrated. -- `ai_diagnosis` is the actual tab key; its display label is `D-InSAR诊断`. -- `landslide_segmentation` and `uav_image_analysis` remain reserved placeholders. +## Navigation Update Rules -### 3.5 无二级分组的一级入口 - -The following groups do not define second-level sections: - -- `data` - leaf tabs: `ingest`, `data`, `hazard` -- `flood_analysis` - leaf tabs: `flood_analysis` -- `ops` - leaf tabs: `health`, `users`, `audit` - -## 4. Source-Of-Truth Rules - -The navigation follows these rules: - -- `LEFT_GROUP_LABELS` defines the first-level group vocabulary. -- `LEFT_GROUP_SECTIONS` defines second-level sections where they exist. -- `LEFT_GROUP_TABS` defines which leaf tabs belong to each group. -- `LEFT_TAB_GROUP` and `LEFT_TAB_SECTION` are derived maps and should not be edited manually. -- `leftPanelTab` remains the route/state source of truth for the selected leaf entry. -- `production_management` is a special case: one left-nav tab owns multiple internal workspace views. -- New features should be added under an existing group whenever possible. -- A new first-level group should be introduced only for a durable, independent capability area. - -## 5. Naming Rules - -To avoid future ambiguity, use these naming constraints: - -- Use `结果` for browsing, querying, and result-facing visualization pages. -- Use `产物` for extraction, publishing, packaging, and catalog-management pages. -- Use `运行` for task submission, engine selection, execution control, and runtime monitoring views. -- Use `分析` for interpretation, statistics, and analyst-facing thematic workflows. -- Use `诊断` for model-assisted fault analysis or AI-driven reasoning pages. -- Use `时序候选栈` only for planning-stage candidate stacks under `production_planning`. -- Use `时序InSAR结果` for analysis-facing result pages under `insar_analysis`. - -## 6. Files To Update When Navigation Changes - -When adding or moving a tab, update these files together: - -- `frontend/src/config/appConstants.js` - Defines first-level groups, sections, tab ownership, workspace view mappings, and admin-only visibility. -- `frontend/src/utils/appUiHelpers.js` - Defines display labels for leaf tabs. -- `frontend/src/components/app/AppSidePanel.jsx` - Renders the side-panel navigation and group/section switching behavior. -- `frontend/src/App.jsx` - Connects route state with panel rendering. -- `frontend/src/ProductionWorkspace.jsx` - Owns the internal production workspace view switcher. -- `frontend/src/App.css` - Styles the navigation hierarchy and workspace entry state. - -If the new tab is a real page instead of a placeholder, also add or update the corresponding panel component. - -## 7. Reserved Entries And Legacy Route Aliases - -Reserved leaf tabs: - -- `dinsar_analysis` -- `psinsar_results` -- `psinsar_analysis` -- `landslide_segmentation` -- `uav_image_analysis` - -Legacy route aliases mapped into `production_management`: - -- `dinsar_production` -- `dinsar_products` -- `ps_production` -- `ps_products` - -These aliases exist for compatibility, but they are not first-class left-nav entries anymore. - -## 8. Future Extension Guidance - -Recommended future additions: - -- Use `flood_analysis` as the combined first-level group for water extraction, flood detection, overlay analysis, and flood results. The legacy `water` route may remain in code for compatibility, but it is no longer a first-class left-nav entry. -- Put new production execution or product-governance capability under `production_management` as an internal workspace view unless a separate first-level domain is clearly required. -- Put planning, batching, pairing, and dispatch preparation capability under `production_planning`. -- Put result browsing and analyst-facing deformation interpretation under `insar_analysis`. -- Put intelligent interpretation, diagnosis, segmentation, and computer-vision modules under `ai_analysis`. - -If a new feature belongs to intelligent interpretation or computer vision, prefer `AI分析`. -If a new feature belongs to result browsing or deformation business analysis, prefer `InSAR形变分析`. +- Add production execution, preparation, product registration, and product catalog features inside `ProductionWorkspace`. +- Add source ingestion, archive scanning, orbit scanning, and storage inventory under `data`. +- Add result interpretation and map analysis under `insar_analysis`. +- Keep `production_management` as the only production first-level group. +- Do not reintroduce a separate `production_planning` first-level group. +- When changing navigation, update `appConstants.js`, `appUiHelpers.js`, `AppSidePanel.jsx`, `ProductionWorkspace.jsx`, and this document together. diff --git a/docs/GF3_SARSCAPE_NATIVE_TO_GEOTIFF_DESIGN_20260530.md b/docs/GF3_SARSCAPE_NATIVE_TO_GEOTIFF_DESIGN_20260530.md index 5544664..89a430a 100644 --- a/docs/GF3_SARSCAPE_NATIVE_TO_GEOTIFF_DESIGN_20260530.md +++ b/docs/GF3_SARSCAPE_NATIVE_TO_GEOTIFF_DESIGN_20260530.md @@ -22,12 +22,13 @@ GF3 原始压缩包池 推荐把输入、生产结果和运行时目录分开，避免把系统工作文件混入业务结果池。 ```env -GF3_ARCHIVE_SOURCE_DIRS=D:\production_inputs\gf3\archives +GF3_TASK_POOL_ROOT=D:\GaoFen3_Task_Pool +GF3_ARCHIVE_SOURCE_DIRS=D:\GaoFen3_Image_Pool\archives GF3_LEGACY_GDAL_ENABLED=false GF3_SOURCE_DIRS= -GF3_SARSCAPE_NATIVE_DIRS=D:\production_results\gf3\sarscape_native -GF3_STORAGE_DIRS=D:\production_results\gf3\standard_l2 -GF3_SARSCAPE_RUNTIME_DIR=D:\production_runtime\gf3\sarscape_runtime +GF3_SARSCAPE_NATIVE_DIRS=D:\GaoFen3_Image_Pool\sarscape_native +GF3_STORAGE_DIRS=D:\GaoFen3_Image_Pool\standard_l2 +GF3_SARSCAPE_RUNTIME_DIR=D:\GaoFen3_Task_Pool\sarscape_runtime SAR_ANALYSIS_READY_ROOT=D:\production_results\sar_analysis_ready ``` @@ -49,7 +50,7 @@ SAR_ANALYSIS_READY_ROOT=D:\production_results\sar_analysis_ready 原生池以批次日期或人工批次号分组。单景目录名尽量保持 GF3 原始产品名。 ```text -D:\production_results\gf3\sarscape_native +D:\GaoFen3_Image_Pool\sarscape_native 20260514 GF3_MH1_FSII_051377_E132.3_N48.2_20260514_L1A_HHHV_L10007356478 GF3_MH1_FSII_..._hh_geo @@ -115,7 +116,7 @@ SLC 中间产物 系统从原生池转换后写入 `GF3_STORAGE_DIRS`。 ```text -D:\production_results\gf3\standard_l2 +D:\GaoFen3_Image_Pool\standard_l2 20260514 GF3_MH1_FSII_051377_E132.3_N48.2_20260514_L1A_HHHV_L10007356478 HH_L2.tif @@ -456,7 +457,7 @@ GF3_SARSCAPE_IDLRT_PATH=C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlr GF3_SARSCAPE_DEM_PATH=D:\DEM\GMTED2010.jp2 GF3_SARSCAPE_POLARIZATIONS=HH,HV GF3_SARSCAPE_KEEP_EXTRACTED=true -GF3_SARSCAPE_AUTO_STANDARDIZE=true +GF3_SARSCAPE_AUTO_STANDARDIZE=false GF3_SARSCAPE_CLEAN_AFTER_SUCCESS=true GF3_SARSCAPE_PRODUCE_TIMEOUT_SECONDS=0 ``` @@ -465,8 +466,9 @@ GF3_SARSCAPE_PRODUCE_TIMEOUT_SECONDS=0 | 任务 | 接口 | 用途 | | --- | --- | --- | -| `GF3_SARSCAPE_PRODUCE` | `POST /api/monitor/gf3-sarscape-produce` | 从原始 `.tar.gz/.tgz` 触发 SARscape 生产，随后自动标准化、入库、清理 | -| `GF3_SARSCAPE_SYNC` | `POST /api/monitor/gf3-sarscape-sync` | 仅扫描已有 `_geo` 原生结果并转 GeoTIFF 入库 | +| `GF3_SARSCAPE_PRODUCE` | `POST /api/monitor/gf3-sarscape-produce` | 停用；本机不触发 SARscape wrapper 生产 | +| `GF3_SARSCAPE_SYNC` | `POST /api/monitor/gf3-sarscape-sync` | 登记本机已有 `_geo` 原生结果；监控面板使用原生结果登记模式 | +| `GF3_QUICKLOOK_WEBP` | `POST /api/monitor/gf3-quicklook-webp` | 从已登记 `_geo` ENVI 二进制生成本机 WebP 预览缓存 | | `GF3_SARSCAPE_CLEAN` | `POST /api/monitor/gf3-sarscape-clean` | 手动清理原生池中间数据 | 清理策略： @@ -478,34 +480,23 @@ GF3_SARSCAPE_PRODUCE_TIMEOUT_SECONDS=0 - 每景写 `gf3_cleanup_manifest.json`，记录删除条目和释放字节数。 - 不删除 `GF3_ARCHIVE_SOURCE_DIRS` 中的原始压缩包，也不删除 `GF3_STORAGE_DIRS` 中的标准 GeoTIFF。 -这样 `D:\production_results\gf3\sarscape_native` 只长期保存可追溯的最终 `_geo` 原生结果组，中间过程文件在标准化完成后自动释放空间；wrapper 配置和临时运行文件放在 `D:\production_runtime\gf3\sarscape_runtime`。 +这样 `D:\GaoFen3_Image_Pool\sarscape_native` 只长期保存可追溯的最终 `_geo` 原生结果组，中间过程文件在标准化完成后自动释放空间；wrapper 配置和临时运行文件放在 `D:\GaoFen3_Task_Pool\sarscape_runtime`。 -## 2026-06-15 Production Preflight Rule +## 2026-06-15 Local Production Stop Rule -GF3 SARscape production must check existing results before invoking the external wrapper. +GF3 SARscape production on this management machine is disabled. -Skip production when either condition is true: +- `POST /api/monitor/gf3-sarscape-produce` returns 409. +- `GET /api/monitor/gf3-sarscape-dates` returns 409 because date-scoped production selection is no longer used here. +- Existing queued `GF3_SARSCAPE_PRODUCE` jobs fail immediately and do not call `gf3wrapper.exe`. +- The durable result layer is the registered local `_geo` native result plus optional standardized L2 assets. -- SARscape native output is already complete in `GF3_SARSCAPE_NATIVE_DIRS` for every requested polarization. -- Standardized L2 output already exists in `GF3_STORAGE_DIRS//` with `gf3_standard_manifest.json` status `DONE`/`PARTIAL` and every requested polarization has a valid `*_L2.tif`. +## 2026-06-15 Local Native Registration Rule -This prevents reprocessing when native intermediates were cleaned but registered/standardized results still exist. The standardized L2 and registered assets are the durable result layer; source archives on UNC should not be reprocessed unless the operator explicitly removes or invalidates the existing result. +GF3 SARscape production is not run on this management machine. -## 2026-06-15 Date-Scoped Production - -GF3 SARscape production now supports an optional scene-date filter. - -- `GET /api/monitor/gf3-sarscape-dates` scans `GF3_ARCHIVE_SOURCE_DIRS`, groups wrapper-supported raw archives by the `YYYYMMDD` date embedded in the GF3 scene name, and returns scene counts per date. -- `POST /api/monitor/gf3-sarscape-produce` accepts `selected_dates: ["YYYYMMDD"]`. When omitted or empty, production keeps the previous all-date behavior. -- The frontend monitor panel exposes a date selector in the GF3 SARscape production section. Operators can select one image date before starting production, or leave it as all dates. -- The existing duplicate-result preflight still runs after date filtering. A selected date will not reprocess scenes whose standardized L2 result or complete native `_geo` outputs already exist. - -## 2026-06-15 Local Task_Pool Staging - -GF3 SARscape production no longer passes UNC archives directly to `gf3wrapper.exe`. - -- Source archives may remain on UNC storage for source management. -- Before each wrapper run, the selected archive is copied to `GF3_TASK_POOL_ROOT\SARscape\\\source\`. -- The wrapper receives the local staged archive path as `-input`. -- Native SARscape output still goes to `GF3_SARSCAPE_NATIVE_DIRS`, then standardization writes durable L2 GeoTIFFs to `GF3_STORAGE_DIRS`. -- This avoids network extraction stalls and makes source staging part of the local Task_Pool cleanup domain. +- Completed SARscape `_geo` result folders are copied to local `GF3_SARSCAPE_NATIVE_DIRS`. +- The monitor registration action scans real `_geo` ENVI binaries plus `.hdr/.sml` sidecars. +- `*_geo_ql.tif` remains an auxiliary quicklook only and is not used as the WebP source. +- WebP preview cache is generated locally from the `_geo` ENVI binary with rasterio. +- Full GeoTIFF standardization remains a separate explicit path and is not triggered by native-result registration. diff --git a/docs/INDEX.md b/docs/INDEX.md index 2ec865f..ed2f48f 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -1,6 +1,6 @@ # 文档索引 -最后更新：2026-06-14 +最后更新：2026-06-16 本页是当前有效文档入口。没有列在本页的历史设计、实验记录和过程文档不再作为当前系统事实依据。 @@ -23,15 +23,25 @@ - [GLOBAL_TASK_STATUS_LOCK_REDESIGN_20260612.md](GLOBAL_TASK_STATUS_LOCK_REDESIGN_20260612.md) 全局界面锁降级为任务状态中心、功能级任务面板和后端资源锁的重构设计。 +- [OLLAMA_DINSAR_DIAGNOSIS_DEPLOYMENT_20260620.md](OLLAMA_DINSAR_DIAGNOSIS_DEPLOYMENT_20260620.md) + D-InSAR 分析中 Ollama 本机 VLM 诊断的部署配置、模型选择和运行约定。 + ## 生产与结果 +- [THREE_SENSOR_LOCAL_PRODUCTION_CONTRACT_20260616.md](THREE_SENSOR_LOCAL_PRODUCTION_CONTRACT_20260616.md) + 当前陆探一号、Sentinel-1、高分三本机生产、按需解包、GF3 外部生产登记、结果管理和 UNC 退出约定。 + - [PRODUCTION_RESULTS_MULTI_ENGINE_DESIGN_20260423.md](PRODUCTION_RESULTS_MULTI_ENGINE_DESIGN_20260423.md) 统一结果目录、标准产品包、catalog 与多引擎结果共存约定。D-InSAR 当前引擎集合以 2026-06-14 三引擎 Task_Pool 重构设计为准。 - [DINSAR_TASK_POOL_THREE_ENGINE_REFACTOR_20260614.md](DINSAR_TASK_POOL_THREE_ENGINE_REFACTOR_20260614.md) D-InSAR 保留 ENVI/SARscape、LandSAR、Gamma/PyINT 三引擎，退出 ISCE2，统一 Task_Pool、结果聚合和中间文件清理的当前设计。 +- [LANDSAR_DEM_PREPARATION_CONTRACT_20260618.md](LANDSAR_DEM_PREPARATION_CONTRACT_20260618.md) + LandSAR D-InSAR/SBAS 的全球 DEM 一次性 Int16 标准化、区域裁剪 tif、生产配置和 guardrail 约定。 - [UNC_SOURCE_ARCHIVE_AND_MATERIALIZE_DESIGN_20260615.md](UNC_SOURCE_ARCHIVE_AND_MATERIALIZE_DESIGN_20260615.md) - UNC/SMB 源压缩包管理、包内 XML/manifest 资产化、本地 materialize 和 D-InSAR/SBAS 生产边界。 + LT-1/Sentinel-1 本地源压缩包管理、包内 XML/manifest 资产化、本地 Task_Pool materialize，以及 UNC 退出运行链路后的本机部署边界。 +- [SOURCE_ARCHIVE_INTEGRITY_AUDIT_20260620.md](SOURCE_ARCHIVE_INTEGRITY_AUDIT_20260620.md) + LT-1/Sentinel-1 源压缩包完整性审计的独立任务、增量语义、数据库字段和问题登记规则。 - [DINSAR_PRODUCTION_CORES_OVERVIEW.md](DINSAR_PRODUCTION_CORES_OVERVIEW.md) 旧版 ENVI/SARscape、ISCE2、Gamma/PyINT D-InSAR 生产核心说明。ISCE2 相关内容仅作历史背景。 @@ -57,6 +67,8 @@ - [SENTINEL1_SOURCE_ORBIT_ASSET_DESIGN_20260512.md](SENTINEL1_SOURCE_ORBIT_ASSET_DESIGN_20260512.md) Sentinel-1 / LT-1 源数据与精密轨道资产层设计。 +- [PRECISE_ORBIT_PRODUCTION_CONTRACT_20260617.md](PRECISE_ORBIT_PRODUCTION_CONTRACT_20260617.md) + Current LT-1/Sentinel-1 precise-orbit source assets, LT-1 production TXT pools, Gamma/PyINT, LandSAR, and retired ISCE2 orbit boundaries. - [FLOOD_GEOTIFF_GAMMA_PREPROCESS_DESIGN_20260515.md](FLOOD_GEOTIFF_GAMMA_PREPROCESS_DESIGN_20260515.md) 洪涝模块 GeoTIFF 化与 Gamma 前处理方向。 diff --git a/docs/LANDSAR_DEM_PREPARATION_CONTRACT_20260618.md b/docs/LANDSAR_DEM_PREPARATION_CONTRACT_20260618.md new file mode 100644 index 0000000..812eb52 --- /dev/null +++ b/docs/LANDSAR_DEM_PREPARATION_CONTRACT_20260618.md @@ -0,0 +1,63 @@ +# LandSAR DEM Preparation Contract + +Updated: 2026-06-18 + +## Decision + +LandSAR D-InSAR and LandSAR SBAS must not run directly against the full global DEM. + +The maintained flow is: + +1. Convert the global DEM once into an uncompressed Int16 GeoTIFF. +2. Configure LandSAR with that global Int16 GeoTIFF as the DEM source. +3. Before each LandSAR task executes, crop a task-level DEM from the global Int16 source. +4. Write the task-level crop path into `200014.txt` or `280039.txt`. + +The global Int16 file is a reusable source DEM. The actual LandSAR console input is the small crop stored under the run work directory. + +## Paths + +Prepared global DEM source: + +```text +D:\DEM\SRTMDEM_RSP_SARscape_global_int16.tif +``` + +Runtime task crop location: + +```text +\...\dem_crop\__dem.tif +``` + +Production configuration: + +```text +LANDSAR_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape_global_int16.tif +LANDSAR_SBAS_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape_global_int16.tif +``` + +## Commands + +One-time global conversion: + +```powershell +Set-Location D:\Code\Insar_management_system_v2; & C:\ProgramData\anaconda3\envs\InSAR\python.exe scripts\prepare_landsar_dem_int16.py --source D:\DEM\SRTMDEM_RSP_SARscape.wgs84 --target D:\DEM\SRTMDEM_RSP_SARscape_global_int16.tif --block-size 1024 --overwrite +``` + +Optional manual crop test from the prepared global GeoTIFF: + +```powershell +Set-Location D:\Code\Insar_management_system_v2; & C:\ProgramData\anaconda3\envs\InSAR\python.exe scripts\prepare_landsar_dem_int16.py --source D:\DEM\SRTMDEM_RSP_SARscape_global_int16.tif --crop-only --bbox 120,42,136,54 --target D:\DEM\landsar_prepared\SRTMDEM_RSP_SARscape_ne_china_int16.tif --block-size 2048 --overwrite +``` + +The manual crop command is for verification or emergency operation. Normal D-InSAR/SBAS production performs task-level cropping automatically. + +## Guardrails + +- `LANDSAR_DEM_PATH` and `LANDSAR_SBAS_DEM_PATH` point to the global prepared Int16 GeoTIFF source. +- D-InSAR derives the crop bbox from master/slave LT-1 XML corner coordinates. +- LandSAR SBAS derives the crop bbox from all selected `Input_Data` LT-1 XML corner coordinates. +- The crop bbox is expanded by a margin before writing the task DEM. +- The source DEM must be Int16; Float/ENVI sources are rejected at runtime. +- Each crop writes a JSON manifest next to the crop tif. +- Run metadata records both `dem_source_path` and the actual task `dem_path`. diff --git a/docs/OLLAMA_DINSAR_DIAGNOSIS_DEPLOYMENT_20260620.md b/docs/OLLAMA_DINSAR_DIAGNOSIS_DEPLOYMENT_20260620.md new file mode 100644 index 0000000..64346f6 --- /dev/null +++ b/docs/OLLAMA_DINSAR_DIAGNOSIS_DEPLOYMENT_20260620.md @@ -0,0 +1,97 @@ +# Ollama D-InSAR Diagnosis Deployment + +Last updated: 2026-06-20 + +This document is the current deployment contract for local Ollama integration in the D-InSAR analysis workflow. + +## Scope + +- Ollama is used only for D-InSAR diagnosis and map/image interpretation tasks. +- The visible UI entry is `InSAR形变分析 / D-InSAR / D-InSAR分析 / D-InSAR诊断`. +- The standalone `AI分析` first-level page is retired. +- Quality model training and batch quality prediction remain local backend tasks. They do not call Ollama. + +## Configuration + +Set these values in the backend environment: + +```env +OLLAMA_BASE_URL=http://127.0.0.1:11434 +OLLAMA_API_URL=http://127.0.0.1:11434/api/generate +DEFAULT_VLM_MODEL=qwen3-vl:30b +``` + +The backend reads them through `backend/app/config.py`. + +`OLLAMA_BASE_URL` must point to the server-local Ollama service. Do not point it to UNC or a workstation share. `OLLAMA_API_URL` should normally be `${OLLAMA_BASE_URL}/api/generate`. + +## Model Selection + +`GET /ai/status` checks `${OLLAMA_BASE_URL}/api/tags` and returns: + +- `ollama_online` +- `ollama_models` +- `ollama_vlm_models` +- `ollama_base_url` +- `default_vlm_model` + +The D-InSAR diagnosis panel uses `ollama_vlm_models` for its model dropdown. `ollama_models` is the raw installed-model list and may include pure text models. If `DEFAULT_VLM_MODEL` is installed and classified as a vision model, it is selected. Otherwise the first detected vision model is selected. + +The backend still has a fallback detector for compatibility. Preference order is: + +1. User-selected model, if present in Ollama. +2. Model name containing `qwen3-vl`. +3. Model name containing `qwen2-vl`. +4. Model name containing `minicpm-v`. +5. Model name containing `llama3.2-vision` or `llava`. +6. Any model name containing `vl`, `vision`, or `llava`. +7. `DEFAULT_VLM_MODEL`. + +## Runtime Flow + +1. User opens `D-InSAR分析`. +2. User selects `D-InSAR诊断`. +3. Frontend calls `POST /ai/diagnosis`. +4. Backend creates an `AI_DIAGNOSIS` task and job. +5. Job handler reads the registered D-InSAR preview image, injects spatial context and quality context into the prompt, then calls Ollama `/api/generate`. +6. The diagnosis report is written to `ai_diagnosis`. +7. The panel lists diagnosis records from `GET /ai/diagnosis`. + +The older `POST /ai/analyze-result/{result_id}` and `AI_ANALYZE` task remain compatibility code. New UI should use `POST /ai/diagnosis` and `AI_DIAGNOSIS`. + +## Deployment Check + +Run these checks on the server: + +```powershell +ollama list +curl http://127.0.0.1:11434/api/tags +``` + +Then check the system endpoint: + +```powershell +curl http://127.0.0.1:8000/api/ai/status +``` + +Expected result: + +- `ollama_online` is `true`. +- `ollama_vlm_models` contains at least one vision-capable model. + +Recommended model families for this project: + +- `qwen3-vl` +- `qwen2-vl` +- `minicpm-v` +- `llama3.2-vision` +- `llava` + +Avoid pure text models such as `qwen2`, `llama3`, `mistral`, or `gemma` for D-InSAR diagnosis. + +## Failure Handling + +- If the panel shows Ollama offline, verify `ollama serve` is running and the configured port matches `.env`. +- If diagnosis stays queued or fails quickly, inspect the task log for `AI_DIAGNOSIS`. +- If the model dropdown is empty, `/api/tags` is unreachable or Ollama has no models installed. +- If a selected model fails at generation time, confirm the model supports image input. diff --git a/docs/PRECISE_ORBIT_PRODUCTION_CONTRACT_20260617.md b/docs/PRECISE_ORBIT_PRODUCTION_CONTRACT_20260617.md new file mode 100644 index 0000000..e18488b --- /dev/null +++ b/docs/PRECISE_ORBIT_PRODUCTION_CONTRACT_20260617.md @@ -0,0 +1,109 @@ +# Precise Orbit Production Contract 2026-06-17 + +This document is the current contract for LT-1 and Sentinel-1 precise orbit management. UNC paths are not allowed in active source, orbit, task, or production paths. + +## Two Layers + +There are two different orbit layers. + +`ORBIT_SOURCE_DIRS` is the source asset layer. It is scanned into `orbit_assets` and used for scene-orbit binding: + +```env +ORBIT_SOURCE_DIRS=D:\LT1_data_lsarorbit;D:\Sentinel1_EOF_Pool +``` + +`ORBIT_POOL_ENVI` / `PYINT_ORBIT_POOL_TXT` / `GAMMA_SBAS_ORBIT_ROOTS` are LT-1 production orbit pools. They are local TXT pools consumed by ENVI/SARscape and Gamma/PyINT: + +```env +ORBIT_POOL_ENVI=D:\orbit_pools\envi +PYINT_ORBIT_POOL_TXT=D:\orbit_pools\envi +GAMMA_SBAS_ORBIT_ROOTS=D:\orbit_pools\envi +``` + +Expected LT-1 production layout: + +```text +D:\orbit_pools\envi + LT1A\ + LT1A_GpsData_GAS_C_YYYYMMDD.txt + LT1B\ + LT1B_GpsData_GAS_C_YYYYMMDD.txt +``` + +`LT1A` and `LT1B` are satellite names, not product levels. + +## Scan Semantics + +The active scan entry is `/assets/inventory/scan`. + +- `inventory_types=["orbit_asset"], families=["LT1"]` scans LT-1 orbit TXT files. +- `inventory_types=["orbit_asset"], families=["S1"]` scans Sentinel-1 EOF files. +- `inventory_types=["orbit_asset"], families=["LT1","S1"]` scans both. + +LT-1 scanning recognizes `LT1A_GpsData_GAS_C_YYYYMMDD.txt` and `LT1B_GpsData_GAS_C_YYYYMMDD.txt`. + +Sentinel-1 scanning recognizes `S1*_OPER_AUX_*.EOF` and matches EOF validity windows to scene acquisition windows. + +After an LT-1 orbit scan, the scanner also synchronizes the LT-1 production TXT pool under `ORBIT_POOL_ENVI`. This keeps Gamma/PyINT and Gamma SBAS able to find the same TXT files without relying on the old monitor scan. + +Orbit asset scans are incremental. For already indexed TXT/EOF files, the scanner skips metadata parsing and database upsert when the file path, size, mtime, parser version, active flag, and `parse_status=OK` still match the database record. Missing files are still marked inactive by comparing the scanned `seen_paths` set with existing assets under the same managed root. + +## Engine Consumers + +ENVI/SARscape D-InSAR: + +- Uses local LT-1 data prepared for the SARscape workflow. +- The LT-1 TXT production pool is `ORBIT_POOL_ENVI`. +- The pool must be split by satellite because ENVI-side tools expect stable satellite folders. + +Gamma/PyINT D-InSAR: + +- Reads LT-1 TXT orbit files from `PYINT_ORBIT_POOL_TXT`. +- Current default keeps `PYINT_ORBIT_POOL_TXT=ORBIT_POOL_ENVI`. +- For LT-1, input assets may stage TXT orbits into the task input manifest when the precise-orbit bridge is enabled. +- For Sentinel-1, EOF paths come from source/orbit asset binding or task `orbit` staging. + +Gamma SBAS: + +- For LT-1, reads TXT orbit roots from `GAMMA_SBAS_ORBIT_ROOTS`. +- For Sentinel-1, planning uses `ORBIT_SOURCE_DIRS` EOF roots; S1 SBAS execution is not enabled. +- LT-1 Gamma SBAS scripts use the orbit path recorded in scene discovery. + +LandSAR: + +- Current D-InSAR integration does not independently scan an orbit pool. +- It consumes already prepared LT-1 task input. +- `ORBIT_POOL_LANDSAR` is not an active synchronization target in current code. + +ISCE2: + +- ISCE2 is retired from the active D-InSAR production path. +- `ORBIT_POOL_ISCE2` is legacy only and should be empty unless `ISCE2_ENABLED=true`. +- When `ISCE2_ENABLED=false`, health and orbit status must not treat missing ISCE2 XML as a production error. + +## Database State + +`orbit_assets` records original orbit files from `ORBIT_SOURCE_DIRS`. + +`scene_orbit_bindings` records candidate and selected scene-orbit matches. + +`radar_data.selected_orbit_asset_id`, `radar_data.orbit_binding_status`, `radar_data.has_orbit_data`, and `radar_data.orbit_file_path` are compatibility fields for production and search. + +`orbit_asset_derivatives` records production-pool derivatives. For current LT-1 TXT production, derivative records use: + +```text +engine_code=lt1_txt_pool +derivative_format=LT1_TXT +derivative_role=production_orbit_txt +pool_path=D:\orbit_pools\envi\LT1A|LT1B\*.txt +``` + +## Current Defaults + +```env +ISCE2_ENABLED=false +ORBIT_POOL_ISCE2= +ORBIT_POOL_LANDSAR= +``` + +The old `/monitor/run-now?target=orbit` path is legacy. It may still synchronize the LT-1 production pool for compatibility, but new UI should use asset inventory orbit scans. diff --git a/docs/SBAS_INSAR_CURRENT_WORKFLOW.md b/docs/SBAS_INSAR_CURRENT_WORKFLOW.md index 6b1d222..273bdb5 100644 --- a/docs/SBAS_INSAR_CURRENT_WORKFLOW.md +++ b/docs/SBAS_INSAR_CURRENT_WORKFLOW.md @@ -276,4 +276,4 @@ D:\Task_Pool\SBAS\ └─ publish ``` -Source archives remain on UNC. Selected scenes and orbit files are materialized under the SBAS task directory before Gamma/LandSAR execution. Cleanup may remove `sources`, `orbits`, and `work` after result registration, but must preserve manifests, `publish`, previews, and catalog assets. +LT-1 and Sentinel-1 source archives remain local. Selected scenes and orbit files are materialized under the SBAS task directory before Gamma/LandSAR execution. Cleanup may remove `sources`, `orbits`, and `work` after result registration, but must preserve manifests, `publish`, previews, and catalog assets. diff --git a/docs/SENTINEL1_SOURCE_ORBIT_ASSET_DESIGN_20260512.md b/docs/SENTINEL1_SOURCE_ORBIT_ASSET_DESIGN_20260512.md index 73500a9..c8a04e9 100644 --- a/docs/SENTINEL1_SOURCE_ORBIT_ASSET_DESIGN_20260512.md +++ b/docs/SENTINEL1_SOURCE_ORBIT_ASSET_DESIGN_20260512.md @@ -345,7 +345,7 @@ ORBIT_SOURCE_DIRS= 对用户当前样本，推荐后续配置形态是： ```text -SOURCE_PRODUCT_DIRS=D:\LuTan1_Image_Pool;D:\Sentinel1_Image_Pool_ZIP +SOURCE_PRODUCT_DIRS=D:\LuTan1_Image_Pool_Zip;D:\Sentinel1_Image_Pool_ZIP ORBIT_SOURCE_DIRS=D:\LT1_data_lsarorbit;D:\Sentinel1_EOF_Pool ``` diff --git a/docs/SOURCE_ARCHIVE_INTEGRITY_AUDIT_20260620.md b/docs/SOURCE_ARCHIVE_INTEGRITY_AUDIT_20260620.md new file mode 100644 index 0000000..efc144c --- /dev/null +++ b/docs/SOURCE_ARCHIVE_INTEGRITY_AUDIT_20260620.md @@ -0,0 +1,67 @@ +# Source Archive Integrity Audit + +Date: 2026-06-20 + +This document defines the LT-1 / Sentinel-1 source archive integrity audit flow. It is separate from normal source asset inventory scanning. + +## Boundary + +Normal asset inventory scan remains lightweight: + +- Recursively discovers local `SOURCE_PRODUCT_DIRS`. +- Extracts LT-1 XML or Sentinel-1 `manifest.safe` metadata from archives. +- Builds metadata documents, preview caches, radar scene records, and orbit bindings. +- Uses `file_path + size_bytes + mtime_epoch + parser_version + parse_status` to skip unchanged archives. + +Archive integrity audit is an explicit background task: + +- Task type: `AUDIT_SOURCE_ARCHIVE_INTEGRITY`. +- API: `POST /assets/inventory/archive-integrity-audit`. +- Default formats: `LT1_ARCHIVE`, `S1_ZIP`. +- Default families: caller supplied; UI uses LT-1 and Sentinel-1. +- It can be started from the asset inventory panel or data monitor panel. + +This prevents every metadata scan from fully reading multi-GB archives. + +## Validation Method + +ZIP archives use Python `zipfile.ZipFile.testzip()` and safe member path validation. + +TAR, TGZ, and TAR.GZ archives use Python `tarfile.open(..., "r:*")` and full member iteration. For gzip-compressed tar streams, reaching EOF through `tarfile` validates the gzip stream CRC/truncation state. TAR members are also checked for unsafe paths, links, devices, and other unsupported special member types. + +The audit records: + +- `archive_integrity_status`: `NOT_CHECKED`, `OK`, `FAILED`, `UNSUPPORTED` +- `archive_integrity_method` +- `archive_integrity_checked_at` +- `archive_integrity_error` +- `archive_integrity_version` +- `archive_integrity_member_count` + +## Incremental Semantics + +An archive is skipped when all conditions hold: + +- `force=false` +- `size_bytes` and `mtime_epoch` still match the filesystem +- `archive_integrity_version` equals the current audit version +- previous status is `OK`, `FAILED`, or `UNSUPPORTED` + +If the archive changes or the audit version changes, the audit runs again. When normal metadata scanning reparses a changed source archive, it resets the integrity fields to `NOT_CHECKED`. + +## Issue Handling + +Audit failures create an open `asset_inventory_issues` row: + +- `inventory_type=source_product` +- `asset_ref_id=` +- `issue_code=source_archive_integrity_failed` +- `severity=error` + +When a later audit passes or marks the archive unsupported, the previous open integrity issue for that source asset is resolved. + +## Operational Notes + +The audit is intentionally I/O heavy. A 1.22 GB LT-1 `tar.gz` test archive with 8 members completed a full stream audit in about 9.8 seconds on the current server. Full-pool audits should be run manually, not on backend startup. + +The original source archive remains the source of record and must not be deleted after materialization. diff --git a/docs/THREE_SENSOR_LOCAL_PRODUCTION_CONTRACT_20260616.md b/docs/THREE_SENSOR_LOCAL_PRODUCTION_CONTRACT_20260616.md new file mode 100644 index 0000000..6dc1a00 --- /dev/null +++ b/docs/THREE_SENSOR_LOCAL_PRODUCTION_CONTRACT_20260616.md @@ -0,0 +1,121 @@ +# 三数据本机生产与结果管理约定 + +最后更新：2026-06-17 + +本文件是陆探一号、Sentinel-1、高分三在当前系统中的运行边界。2026-06-16 起，UNC 不再作为活动生产链路、源数据池、精轨池或 Task_Pool。网络共享只可作为人工搬运的外部介质，不进入后台生产任务。 + +## 1. 总原则 + +1. 源数据和精轨都放在本机路径，UNC 不进入活动生产链路。 +2. LT-1 和 Sentinel-1 当前管理对象是本机压缩包源池，不管理旧解包目录。 +3. 管理阶段只从压缩包中抽取 XML/manifest、元数据和预览图，用于资产索引、日期/轨道绑定和检索。 +4. 生产任务需要真实文件树时，才按任务解包或复制到本机 `Task_Pool`，不从 UNC 拉取。 +5. D-InSAR 和 SBAS-InSAR 的运行材料、工作目录、结果发布目录必须是本机路径。 +6. 生产结果进入数据管理或结果 catalog，InSAR 形变分析只读取已登记结果，不直接扫描临时工作目录。 +7. 洪水检测模块本轮冻结，不纳入本次审计和改造。 + +## 2. 三类数据边界 + +| 数据 | 源数据位置 | 精轨位置 | 生产模块 | 结果管理 | +| --- | --- | --- | --- | --- | +| 陆探一号 LT-1 | `D:\LuTan1_Image_Pool_Zip`，本机 LT-1 压缩包源池 | `D:\LT1_data_lsarorbit` | 生产管理保留占位；D-InSAR 走 LandSAR、ENVI+SARscape、Gamma/PyINT；SBAS 走 Gamma/PyINT 主线 | D-InSAR/SBAS 结果 catalog | +| Sentinel-1 | `D:\Sentinel1_Image_Pool_ZIP`，本机 Sentinel-1 ZIP/SAFE 压缩包源池 | `D:\Sentinel1_EOF_Pool` | 生产管理保留占位；D-InSAR 当前只走 Gamma/PyINT；SBAS 只做发现和规划，执行未启用 | D-InSAR 结果 catalog；SBAS 暂无执行产物 | +| 高分三 GF3 | 本机只管理已生产的 `_geo` 成品；原始归档仅追踪 | 无精轨链路 | 外部 SARscape 服务器生产 `_geo`；本机不启动 SARscape wrapper | 复制到 `D:\GaoFen3_Pool\native_geo` 后登记，WebP 从 `_geo` 主二进制生成 | + +高分三外部结果命名约定： + +```text +D:\GaoFen3_Pool\native_geo\20260609_geo\ + GF3_MDJ_FSI_051759_E130.2_N43.5_20260609_L1A_HHHV_L10007375467\ + GF3_MDJ_FSI_051759_E130.2_N43.5_20260609_L1A_HHHV_L10007375467_hh_geo + GF3_MDJ_FSI_051759_E130.2_N43.5_20260609_L1A_HHHV_L10007375467_hh_geo.hdr + GF3_MDJ_FSI_051759_E130.2_N43.5_20260609_L1A_HHHV_L10007375467_hh_geo.sml +``` + +`*_geo_ql.tif` 可作为辅助材料，但不作为正式 WebP 预览源。正式预览从 SARscape `_geo` ENVI 二进制读取生成。 + +### GF3 成品池扫描入库流程 + +1. 外部 SARscape 服务器完成生产后，把整景目录复制到本机 `D:\GaoFen3_Pool\native_geo\YYYYMMDD_geo\`。 +2. 每个场景目录至少包含一个完整极化的 `*_geo`、`*_geo.hdr`、`*_geo.sml`；`*_geo_ql.tif` 可以同时提供，作为范围读取兜底和人工核验材料。 +3. 在系统中点击 `登记 _geo 结果`，后端递归扫描所有 `*_geo` 场景目录，写入 `gf3_native_manifest.json` 和 `gf3_native_preview_manifest.json`，并登记 `source_product_assets` 与 `radar_data`。 +4. 登记时优先从 `*_geo` ENVI 主数据读取 CRS、范围和中心点；如果主数据无法读取范围，再尝试对应的 `*_geo_ql.tif`。 +5. 点击 `生成 WebP` 后，系统从已登记的 `*_geo` 主数据生成本机 WebP 缓存，缓存路径仍使用 `backend\image_cache\radar_raw` / `radar_geo` 体系。 +6. `D:\GaoFen3_Pool\catalog` 只保存平台登记 manifest 和后续可选派生物；默认不复制完整影像、不把 `_geo` 转成全量 GeoTIFF。 + +## 3. 本机路径与按需解包 + +当前核心路径： + +```text +UNPACK_SOURCE_DIRS= +SOURCE_PRODUCT_DIRS=D:\LuTan1_Image_Pool_Zip;D:\Sentinel1_Image_Pool_ZIP +SENTINEL1_STORAGE_DIRS= +INSAR_STORAGE_DIRS= +MONITOR_RADAR_DIRS= +ORBIT_SOURCE_DIRS=D:\LT1_data_lsarorbit;D:\Sentinel1_EOF_Pool +ORBIT_POOL_ENVI=D:\orbit_pools\envi +PYINT_ORBIT_POOL_TXT=D:\orbit_pools\envi +GAMMA_SBAS_ORBIT_ROOTS=D:\orbit_pools\envi +ISCE2_ENABLED=false +ORBIT_POOL_ISCE2= +TASK_POOL_ROOT=D:\Task_Pool +DINSAR_TASK_POOL_ROOT=D:\Task_Pool\DInSAR +SBAS_TASK_POOL_ROOT=D:\Task_Pool\SBAS +DATA_DISTRIBUTION_ROOT=D:\Task_Pool\Data_Distribution +GF3_TASK_POOL_ROOT=D:\GaoFen3_Pool\task_pool +GF3_ARCHIVE_SOURCE_DIRS=D:\GaoFen3_Pool\archives +GF3_SARSCAPE_NATIVE_DIRS=D:\GaoFen3_Pool\native_geo +GF3_STORAGE_DIRS=D:\GaoFen3_Pool\catalog +GF3_SARSCAPE_RUNTIME_DIR=D:\GaoFen3_Pool\task_pool\sarscape_runtime +``` + +压缩包资产索引只建立资产索引、预览缓存和轨道绑定，不提供“全量入库/全量解包”按钮。当前 LT-1/Sentinel-1 主流程从本机压缩包源池扫描，只抽取 XML/manifest、元数据和预览图；参与计算时才按任务解包或复制到 `Task_Pool` 或具体任务目录。 + +生产准备和数据分发是两个归口。前端不让用户输入服务器绝对路径，只填写任务名；后端按 `.env` 中固定根目录创建子目录，避免远程浏览器把用户本机路径传给服务器。`生产数据准备` 以批次配对为输入，从 LT-1/Sentinel-1 源压缩包按需 materialize 到 `DINSAR_TASK_POOL_ROOT\<任务名>`，在其下生成 `Task_YYYYMMDD_YYYYMMDD\master`、`slave`、`orbit` 和 `pair_metadata.json`，供 D-InSAR 引擎直接运行；这是生产工作区，不是源池。`数据分发` 只用于外发或跨目录搬运源压缩包，导出到 `DATA_DISTRIBUTION_ROOT\<任务名>`，目录结构为 `data/`、`orbit/`、`pairs.json`、`manifest.json`。其中 `data/` 只能保存 LT-1/Sentinel-1 源压缩包文件，不能保存旧解包目录。旧批次如果仍指向 `D:\LuTan1_Image_Pool` 或 `D:\Sentinel1_Image_Pool` 解包目录，应重新从压缩包资产重建批次后再执行生产准备或分发。 + +源池扫描采用增量解析语义：系统仍递归列出 `SOURCE_PRODUCT_DIRS` 下的候选压缩包文件名，用于发现新增和删除；数据库中已有且 `file_path`、`size_bytes`、`mtime_epoch`、`parser_version`、`parse_status` 均满足未变化条件的 `S1_ZIP` / `LT1_ARCHIVE` 资产会跳过包内 XML/manifest 读取，不再每次全量重读压缩包内容。新增、修改、曾经失效或解析器版本变化的压缩包会重新解析并更新资产索引。 + +LT-1 解析字段必须区分：规范文件名中的 `SLC/SSC` 是 `product_type` 和干涉源类型判断依据；XML 中 `imageDataInfo/imageDataType=COMPLEX` 只能写入 `image_data_type`，不能覆盖 `product_type`。前端“范围/可用性”依赖 `coverage_polygon + imaging_date + imaging_mode + polarization + complex token`，因此解析器变更必须同时验证 LT-1 `coverage_polygon` 和 `insar_source_ready` 计数。 + +LT-1/GF3 XML 的四个 `sceneCornerCoord` 不得直接按 XML 出现顺序连线。解析器必须把角点重排为非自交闭合四边形后再写入 `coverage_polygon` 和 PostGIS `geom`；验收时需检查 Shapely/PostGIS polygon valid，避免前端显示成沙漏形。 + +资产扫描入口支持三种语义：`families=["LT1"]` 只扫陆探源包/精轨，`families=["S1"]` 只扫哨兵源包/精轨，空 `families` 表示合扫。精密轨道扫描同样按 `inventory_types=["orbit_asset"] + families` 区分 LT-1、S1 或全部，前端不得再通过固定路径顺序猜测 root id。 + +精轨生产池按 [PRECISE_ORBIT_PRODUCTION_CONTRACT_20260617.md](PRECISE_ORBIT_PRODUCTION_CONTRACT_20260617.md) 执行：`ORBIT_SOURCE_DIRS` 是原生资产层；LT-1 扫描后同步到 `ORBIT_POOL_ENVI\LT1A|LT1B`，供 ENVI/SARscape、Gamma/PyINT D-InSAR 和 Gamma SBAS 使用；Sentinel-1 使用 EOF 原生资产绑定；ISCE2 精轨 XML 池为 legacy，默认不启用。 + +`UNPACK_SOURCE_DIRS` 为空是当前设计状态；通用全量解包入口不进入前端主流程。压缩包源池由 `SOURCE_PRODUCT_DIRS` 管理。 + +已执行的代码约束： + +- `validate_runtime_config()` 对源数据、精轨、Task_Pool、D-InSAR/SBAS 工作根、结果根、GF3 `_geo` 根执行 UNC 校验。 +- Sentinel-1 单个/批量解包和通用 materialize 拒绝 UNC 源路径与 UNC 目标路径。 +- `/monitor/status` 返回按实际磁盘卷汇总的 `storage_roots`，例如多个 `D:\...` 路径只展示一个 `D:\` 容量项，同时报告配置路径数量和缺失路径数量。 + +## 4. 生产管理界面 + +生产管理工作台现在承担三类数据的生产边界展示： + +- `陆探生产占位`：说明 LT-1 本机压缩包源池、精轨、按需解包、D-InSAR/SBAS 本机 Task_Pool。 +- `哨兵生产占位`：说明 Sentinel-1 本机压缩包源池、EOF 精轨、按需解包、D-InSAR Gamma/PyINT、SBAS 规划态。 +- `高分三结果登记`：说明外部 SARscape 生产、本机 `_geo` 登记和 WebP 生成。 +- `D-InSAR 运行`、`D-InSAR 产物`、`SBAS-InSAR Production`、`SBAS-InSAR 结果` 保留现有生产和结果 catalog 功能。 + +这些占位不是最终生产向导，但先把三类数据放进同一生产管理域，避免继续把数据扫描、生产运行、结果登记混在数据监控按钮里。 + +## 5. InSAR 形变分析审计 + +当前边界： + +- D-InSAR 结果由 D-InSAR product catalog 管理，分析页不应直接读取 Task_Pool 临时目录。 +- SBAS 结果由 SBAS product catalog 管理，速率图、质量指标和监测点曲线从发布包读取。 +- Sentinel-1 SBAS 目前只是规划态，不能在形变分析中伪装成可执行生产链路。 +- GF3 `_geo` 登记到雷达资产和 WebP 缓存后，可作为数据管理资产；它不是 D-InSAR/SBAS 形变分析的生产输入。 + +后续如果要把分析页做成正式工作台，应先统一读取结果 catalog，再补地图叠加、剖面、时间序列和质量过滤，不应回退到扫描任意目录。 + +## 6. 本轮不改内容 + +- 洪水检测、GF3 水体检测和洪水 GeoTIFF 预处理暂不调整。 +- D-InSAR 三引擎内部执行细节不在本轮重写，只强化本机路径边界。 +- Sentinel-1 SBAS 不启用执行，只保留发现和规划。 diff --git a/docs/UNC_SOURCE_ARCHIVE_AND_MATERIALIZE_DESIGN_20260615.md b/docs/UNC_SOURCE_ARCHIVE_AND_MATERIALIZE_DESIGN_20260615.md index 8958d60..96b6c0c 100644 --- a/docs/UNC_SOURCE_ARCHIVE_AND_MATERIALIZE_DESIGN_20260615.md +++ b/docs/UNC_SOURCE_ARCHIVE_AND_MATERIALIZE_DESIGN_20260615.md @@ -1,29 +1,35 @@ -# UNC Source Archive and Local Materialize Design +# Local-Only Source Archive and Task_Pool Materialize Design + +Current canonical operating contract: [THREE_SENSOR_LOCAL_PRODUCTION_CONTRACT_20260616.md](THREE_SENSOR_LOCAL_PRODUCTION_CONTRACT_20260616.md). This file remains the implementation detail for local archive metadata extraction and Task_Pool materialization. ## Decision -UNC/SMB storage is treated as the source archive pool. Production engines should not use UNC paths as their working input. D-InSAR, SBAS, Gamma/PyINT, LandSAR, and SARscape should consume local materialized task inputs. +As of 2026-06-15, UNC/SMB storage is removed from the active production path. -This keeps the 20 TB storage useful for long-term source management while protecting production from SMB disconnects, credential scope, WSL path conversion, and external engine UNC compatibility. +The switch throughput is too low for production data movement. Large source archives, precise orbit files, GF3 native result pools, task staging, and engine inputs must all live on local disks. A network share may exist outside this system for manual backup or manual transfer, but it must not be configured in runtime environment variables used by backend scans, inventory, materialization, or production. + +Production engines consume local Task_Pool inputs. If a selected LT-1 or Sentinel-1 source product is archived, it is extracted from a local source archive into `D:\Task_Pool` or a task-specific subdirectory before engine execution. + +The local source archive remains the source of record. LT-1 and Sentinel-1 materialization must never delete the original archive after extraction. ## Current Implementation -- Source asset inventory now recognizes archive assets: +- Source asset inventory recognizes archive assets: - `S1_ZIP` - `LT1_ARCHIVE` - `GF3_ARCHIVE` -- Sentinel-1 ZIP manifest parsing already reads `manifest.safe` directly from the ZIP. +- Sentinel-1 ZIP manifest parsing reads `manifest.safe` directly from the ZIP. - LT-1 archive parsing reads `*.meta.xml` directly from `.zip`, `.tar`, `.tar.gz`, or `.tgz` and records contained TIFF members. -- GF3 archive parsing reads the first XML member directly from `.zip`, `.tar`, `.tar.gz`, or `.tgz` and records quicklook-like members when present. -- `GF3_ARCHIVE_SOURCE_DIRS` roots are included in asset inventory scans as source pools. -- Source asset listing and inventory counts now include archive assets instead of hiding `S1_ZIP`. +- `SOURCE_PRODUCT_DIRS` is the local LT-1/Sentinel-1 source archive inventory root. +- Source asset listing and inventory counts include archive assets instead of hiding `S1_ZIP`. - A generic source materialize endpoint exists: - `POST /api/assets/sources/{asset_id}/materialize` - `S1_ZIP` uses the existing Sentinel-1 SAFE unpacker. - `LT1_ARCHIVE` and `GF3_ARCHIVE` extract to a local materialized directory. - Directory assets return `DIRECTORY_READY`. +- If no `target_root` is supplied, generic materialize defaults to `TASK_POOL_ROOT\source_materialized\`. -Default source materialization is task-scoped. D-InSAR and SBAS callers should pass a Task_Pool target directory: +Default source materialization is local and task-scoped. D-InSAR and SBAS callers should pass a Task_Pool target directory: ```text D:\Task_Pool\DInSAR\\master @@ -31,84 +37,65 @@ D:\Task_Pool\DInSAR\\slave D:\Task_Pool\SBAS\\sources\ ``` -The generic materialize endpoint still accepts `target_root` for ad hoc checks. Production callers must provide a Task_Pool destination. +The generic materialize endpoint still accepts `target_root` for ad hoc checks. Production callers should provide a task-specific Task_Pool destination. ## Production Boundary -D-InSAR and SBAS should store source asset references in task/run manifests, then materialize selected inputs into the run directory before engine execution. +D-InSAR and SBAS store local source asset references in task/run manifests, then materialize selected inputs into the run directory before engine execution. -Required next integration points: +Required integration points: - D-InSAR Task_Pool publishing: - - store `source_product_asset_id`, `archive_path`, `source_format`; + - store `source_product_asset_id`, `archive_path`, and `source_format`; - materialize master/slave archive assets into the task directory before engine dispatch. - Gamma/PyINT: - - always consume local materialized paths because WSL conversion rejects or cannot reliably map UNC paths. + - consume local materialized paths because WSL conversion rejects or cannot reliably map network paths. - LandSAR and ENVI/SARscape: - - prefer local materialized paths even when Windows can see UNC, to avoid external engine path and credential issues. + - consume local materialized paths. - SBAS: - stack discovery can use archive metadata; - selected scenes must be materialized into the SBAS `RAW`/input structure before Gamma commands such as `par_LT1_SLC`. -## GF3 Management - -GF3 has two asset layers: - -- `GF3_ARCHIVE`: original source archive, suitable for UNC source management and migration tracking. -- GF3 SARscape standardized L2: production result/analysis-ready layer, used for map footprint, preview, radar data management, and water extraction. - -Do not replace standardized L2 management with raw archive management. Archive assets should link migration and production status; previews and water extraction should continue to consume standardized L2/analysis-ready products. - -## Migration Guidance - -1. Register UNC roots first and scan inventory. -2. Verify archive asset counts and parse status. -3. Keep existing local standardized results and D-InSAR/SBAS products in place. -4. Move source archives to UNC and update root configuration. -5. Only after inventory and materialize tests pass, switch D-InSAR/SBAS publishing to archive asset references. - Production safety rule: if a run cannot materialize every selected source asset locally, the run must fail before invoking the engine. -## Recommended UNC Layout +## GF3 Management -The current deployment uses two SMB shares: +GF3 now has a separate operational rule: -```text -\\DESKTOP-N16HJ84\InSAR_Storage_1 -\\DESKTOP-N16HJ84\InSAR_Storage_2 -``` +- GF3 SARscape production is not run on this management machine. +- Already-produced SARscape `_geo` ENVI binary results are stored locally under `GF3_SARSCAPE_NATIVE_DIRS`. +- The system registers those local `_geo` native results and their `.hdr/.sml` sidecars. +- `*_geo_ql.tif` is retained only as an auxiliary quicklook file. +- WebP preview cache is generated locally from the `_geo` ENVI binary, not from `*_geo_ql.tif`. +- Standard GeoTIFF conversion remains a separate explicit path; the monitor button used for GF3 registration is native-result registration only. -Recommended source archive layout: - -```text -\\DESKTOP-N16HJ84\InSAR_Storage_1 - └─ GaoFen-3 - ├─ 20260513 - │ └─ GF3_*.tar.gz - └─ 20260514 - -\\DESKTOP-N16HJ84\InSAR_Storage_2 - ├─ LuTan-1 - │ └─ Archive - │ ├─ 20260513 - │ │ └─ LT1*.tar.gz / LT1*.tgz / LT1*.zip / LT1*.tar - │ └─ 20260514 - ├─ Sentinel-1 - │ └─ Archive - │ ├─ 20260513 - │ │ └─ S1*.zip - │ └─ 20260514 - └─ Orbit - ├─ LuTan-1 - │ ├─ LT1A_GpsData_GAS_C_YYYYMMDD.txt - │ └─ LT1B_GpsData_GAS_C_YYYYMMDD.txt - └─ Sentinel-1 - └─ S1*.EOF -``` - -Recommended local Task_Pool layout: +## Recommended Local Layout ```text +D:\ + ├─ LuTan1_Image_Pool_Zip + │ └─ LT1*.tar.gz / LT1*.tgz / LT1*.zip / LT1*.tar + ├─ Sentinel1_Image_Pool_ZIP + │ └─ S1*.zip + ├─ LuTan1_Image_Pool + │ └─ LT1 unpacked scene directories + ├─ Sentinel1_Image_Pool + │ └─ S1*.SAFE directories + ├─ LT1_data_lsarorbit + │ └─ LT1*_GpsData_*.txt + ├─ Sentinel1_EOF_Pool + │ └─ S1*.EOF + ├─ production_results + │ └─ gf3 + │ ├─ sarscape_native + │ │ └─ YYYYMMDD_geo + │ │ └─ GF3_* + │ │ ├─ *_geo + │ │ ├─ *_geo.hdr + │ │ ├─ *_geo.sml + │ │ └─ *_geo_ql.tif + │ └─ standard_l2 + └─ Task_Pool D:\Task_Pool ├─ DInSAR │ └─ @@ -129,30 +116,35 @@ D:\Task_Pool └─ publish ``` -Date folders are optional for the scanner because source and orbit inventory recurse through configured roots. They are recommended for operator readability and migration checks. +Date folders are optional for the LT-1/Sentinel-1 scanners because inventory recurses through configured roots. GF3 native pools should keep the SARscape `YYYYMMDD_geo/` convention. ## Current Local Configuration Example -The local `.env` should keep legacy local roots and UNC roots side by side during migration: +The local `.env` should keep all runtime roots local: ```text -SOURCE_PRODUCT_DIRS=D:\LuTan1_Image_Pool;D:\Sentinel1_Image_Pool_ZIP;\\DESKTOP-N16HJ84\InSAR_Storage_2\LuTan-1\Archive;\\DESKTOP-N16HJ84\InSAR_Storage_2\Sentinel-1\Archive -ORBIT_SOURCE_DIRS=D:\LT1_data_lsarorbit;D:\Sentinel1_EOF_Pool;\\DESKTOP-N16HJ84\InSAR_Storage_2\Orbit\LuTan-1;\\DESKTOP-N16HJ84\InSAR_Storage_2\Orbit\Sentinel-1 -GF3_ARCHIVE_SOURCE_DIRS=\\DESKTOP-N16HJ84\InSAR_Storage_1\GaoFen-3 +SOURCE_PRODUCT_DIRS=D:\LuTan1_Image_Pool_Zip;D:\Sentinel1_Image_Pool_ZIP +SENTINEL1_STORAGE_DIRS= +ORBIT_SOURCE_DIRS=D:\LT1_data_lsarorbit;D:\Sentinel1_EOF_Pool +MONITOR_ORBIT_DIR=D:\LT1_data_lsarorbit +GF3_TASK_POOL_ROOT=D:\GaoFen3_Task_Pool +GF3_ARCHIVE_SOURCE_DIRS=D:\GaoFen3_Image_Pool\archives +GF3_SARSCAPE_NATIVE_DIRS=D:\GaoFen3_Image_Pool\sarscape_native +GF3_STORAGE_DIRS=D:\GaoFen3_Image_Pool\standard_l2 TASK_POOL_ROOT=D:\Task_Pool DINSAR_TASK_POOL_ROOT=D:\Task_Pool\DInSAR SBAS_TASK_POOL_ROOT=D:\Task_Pool\SBAS GAMMA_SBAS_WORK_ROOT=D:\Task_Pool\SBAS ``` -Do not store SMB credentials in `.env`. Credentials should be stored in Windows Credential Manager for the account that runs the backend/worker service. +Do not configure UNC paths in these variables. ## Orbit Pool Contract There are two different orbit concepts: -- `ORBIT_SOURCE_DIRS`: source inventory roots. These can be UNC and may be date-organized or flat. -- `ORBIT_POOL_ENVI` / `PYINT_ORBIT_POOL_TXT`: local production orbit pools. These should remain local disk paths. +- `ORBIT_SOURCE_DIRS`: local source inventory roots. +- `ORBIT_POOL_ENVI` / `PYINT_ORBIT_POOL_TXT`: local production orbit pools. LT-1 local production orbit pool should support both flat and satellite-split layouts: @@ -166,34 +158,40 @@ D:\orbit_pools\envi └─ envi ``` -The `LT1A` and `LT1B` names are satellite names, not product levels. ENVI/Gamma/PyINT/SBAS should use local orbit files copied or synchronized from `ORBIT_SOURCE_DIRS`; they should not be required to read UNC directly. - -Sentinel-1 EOF files can be indexed from UNC. Gamma/PyINT/SBAS execution should stage required EOF files locally with the selected scenes. +The `LT1A` and `LT1B` names are satellite names, not product levels. ENVI/Gamma/PyINT/SBAS should use local orbit files copied or synchronized from `ORBIT_SOURCE_DIRS`. ## Migration Phases -### Phase 1: Source archive migration +### Phase 1: Local source archive inventory -Move or copy source archives only: +Keep production source archives local: -- LT-1 compressed scenes to `\\DESKTOP-N16HJ84\InSAR_Storage_2\LuTan-1\Archive\\`. -- Sentinel-1 ZIP scenes to `\\DESKTOP-N16HJ84\InSAR_Storage_2\Sentinel-1\Archive\\`. -- GF3 raw archives to `\\DESKTOP-N16HJ84\InSAR_Storage_1\GaoFen-3\\`. +- LT-1 compressed scenes in `D:\LuTan1_Image_Pool_Zip`. +- Sentinel-1 ZIP scenes in `D:\Sentinel1_Image_Pool_ZIP`. +- Existing unpacked local scene directories are not active management pools; they should only be task materialization outputs. -Keep current local unpacked scene directories in place until D-InSAR and SBAS archive materialization have been tested. +### Phase 2: Local orbit deployment -### Phase 2: Orbit source migration +Deploy orbit source files on this machine: -Copy orbit source files to UNC: +- LT-1 TXT files under `D:\LT1_data_lsarorbit`. +- Sentinel-1 EOF files under `D:\Sentinel1_EOF_Pool`. -- LT-1 TXT files to `\\DESKTOP-N16HJ84\InSAR_Storage_2\Orbit\LuTan-1\`. -- Sentinel-1 EOF files to `\\DESKTOP-N16HJ84\InSAR_Storage_2\Orbit\Sentinel-1\`. +Keep `ORBIT_POOL_ENVI` and `PYINT_ORBIT_POOL_TXT` local. -Keep `ORBIT_POOL_ENVI` and `PYINT_ORBIT_POOL_TXT` local. Add a later sync/materialize step to populate local orbit pools from the indexed UNC source assets. +### Phase 3: GF3 native-result registration -### Phase 3: Production cutover +Copy completed SARscape `_geo` result folders to the local GF3 native pool: -After inventory scan verifies UNC assets: +```text +D:\GaoFen3_Image_Pool\sarscape_native\YYYYMMDD_geo\\ +``` + +Then run GF3 native-result registration and GF3 WebP generation. WebP generation reads the `_geo` ENVI binary and requires its `.hdr` sidecar. + +### Phase 4: Task_Pool production + +After inventory scan verifies local assets: 1. D-InSAR Task_Pool stores source asset IDs and archive paths. 2. Task preparation materializes master/slave scenes and orbit files under `D:\Task_Pool\DInSAR\`. @@ -201,26 +199,19 @@ After inventory scan verifies UNC assets: 4. Results register normally. 5. Local materialized inputs and intermediate products are eligible for cleanup after result registration. -### Phase 4: Retire old local source pools - -Only after repeated D-InSAR/SBAS runs succeed from archive materialization: - -- remove old local source roots from `SOURCE_PRODUCT_DIRS`; -- keep local work/result roots; -- keep standardized GF3 L2 products unless explicitly migrated and revalidated. - ## Local Cleanup Design -After source archives are managed on UNC and production results are registered as assets, local disk can be treated as a cache/work area. Cleanup should be explicit and asset-aware. +After production results are registered as assets, Task_Pool materialized inputs and work folders can be treated as cleanup candidates. Local source archive roots are durable production inputs and must not be cleaned as cache. ### Keep Classes Cleanup must never delete: -- configured UNC source archive roots; -- local or UNC orbit source roots; +- configured local source archive roots; +- configured local orbit source roots; - registered D-InSAR result assets; - registered SBAS result assets; +- registered GF3 SARscape native result assets; - registered GF3 standardized L2 assets; - `SAR_ANALYSIS_READY_ROOT` products and water extraction result assets; - current pointers, manifests, previews, and catalog metadata needed to open results. @@ -234,7 +225,7 @@ Cleanup may delete only these local classes after verification: - D-InSAR engine intermediate folders not listed in the result manifest; - Gamma/PyINT temporary project work directories after result registration; - SBAS `RAW`, `SLC`, `RSLC`, `MLI`, `DIFF`, `DIFF1`, script logs, and temporary staging after SBAS product registration; -- GF3 SARscape native intermediates only after standardized L2 registration and optional native-retention policy allows cleanup. +- GF3 SARscape runtime or temporary staging only after local native `_geo` registration and optional native-retention policy allows cleanup. ### Safety Contract @@ -249,59 +240,5 @@ Deletion must require: - path is not inside any configured source archive root; - path is not inside a result publish root unless the exact file is classified as intermediate; - associated result or standardized asset is registered; -- candidate is older than a configurable minimum age; -- no active task references the path. - -### Proposed API - -```text -POST /api/maintenance/cleanup/preview -POST /api/maintenance/cleanup/execute -``` - -Preview request fields: - -```json -{ - "scope": "dinsar|sbas|gf3|materialized|all", - "root_ids": [], - "older_than_hours": 24, - "require_registered_result": true, - "include_task_pool_inputs": false -} -``` - -Preview response should include: - -```json -{ - "preview_id": "...", - "total_bytes": 0, - "candidates": [ - { - "path": "D:\\production_runtime\\...", - "class": "materialized_source", - "owner": "task/run/product id", - "size_bytes": 0, - "eligible": true, - "reason": "registered_result_exists" - } - ], - "blocked": [] -} -``` - -### Recommended Defaults - -- `materialized`: delete after 24 hours if no active task references it. -- `dinsar`: delete engine intermediates after result registration; keep Task_Pool inputs until all selected engines are complete or user opts in. -- `sbas`: delete heavy Gamma working directories after SBAS catalog registration and product assets exist. -- `gf3`: keep standardized L2; clean SARscape native only when `GF3_SARSCAPE_CLEAN_AFTER_SUCCESS=true` and standardized registration is confirmed. - -### Implementation Order - -1. Add read-only cleanup preview service. -2. Add path classification and approved-root checks. -3. Add execute endpoint with preview token. -4. Add frontend maintenance panel. -5. Wire D-InSAR/SBAS/GF3 run pages to show cleanup eligibility after successful registration. +- cleanup policy explicitly allows the class; +- operator confirmation is present. diff --git a/frontend/src/AiAnalysisPanel.jsx b/frontend/src/AiAnalysisPanel.jsx index 7659001..2b926c1 100644 --- a/frontend/src/AiAnalysisPanel.jsx +++ b/frontend/src/AiAnalysisPanel.jsx @@ -23,7 +23,7 @@ const cardStyle = { export default function AiAnalysisPanel({ readOnly = false, onJobQueued }) { const { en } = useI18n(); const aiTaskMonitor = useTaskMonitor({ - taskTypes: ['AI_ANALYZE'], + taskTypes: ['AI_DIAGNOSIS'], showRecent: true, recentLimit: 1, pollRecentMs: 10000, @@ -82,6 +82,15 @@ export default function AiAnalysisPanel({ readOnly = false, onJobQueued }) { loadInitialData(); }, [loadInitialData]); + useEffect(() => { + const models = aiStatus?.ollama_vlm_models || []; + if (models.length > 0 && !models.includes(selectedModel)) { + setSelectedModel(aiStatus?.default_vlm_model && models.includes(aiStatus.default_vlm_model) + ? aiStatus.default_vlm_model + : models[0]); + } + }, [aiStatus, selectedModel]); + // 加载诊断列表 const loadDiagnoses = useCallback(async () => { setLoading(true); @@ -118,6 +127,14 @@ export default function AiAnalysisPanel({ readOnly = false, onJobQueued }) { setMessage(en ? 'Please select a D-InSAR result' : '请选择 D-InSAR 结果'); return; } + if (!aiStatus?.ollama_online) { + setMessage(en ? 'Failed: Ollama is offline' : '失败: Ollama 未在线'); + return; + } + if (!aiStatus?.ollama_vlm_models?.length) { + setMessage(en ? 'Failed: no local Ollama vision model is installed' : '失败: 未检测到本机 Ollama 视觉模型'); + return; + } setLoading(true); setMessage(''); @@ -175,6 +192,11 @@ export default function AiAnalysisPanel({ readOnly = false, onJobQueued }) { critical: '#c53030', }; + const ollamaVlmModels = aiStatus?.ollama_vlm_models || []; + const modelOptions = ollamaVlmModels.length > 0 + ? ollamaVlmModels + : [aiStatus?.default_vlm_model || selectedModel].filter(Boolean); + const canCreateDiagnosis = !!selectedResultId && !!aiStatus?.ollama_online && ollamaVlmModels.length > 0; const totalPages = Math.ceil(totalDiagnoses / pageSize); return ( @@ -182,7 +204,7 @@ export default function AiAnalysisPanel({ readOnly = false, onJobQueued }) { {/* Header */}

- {en ? 'AI Analysis' : 'AI 分析'} + {en ? 'D-InSAR Diagnosis' : 'D-InSAR诊断'}

@@ -218,12 +240,12 @@ export default function AiAnalysisPanel({ readOnly = false, onJobQueued }) { @@ -294,10 +316,15 @@ export default function AiAnalysisPanel({ readOnly = false, onJobQueued }) { fontSize: '13px', }} > - - - + {modelOptions.map((modelName) => ( + + ))} + {aiStatus?.ollama_online && ollamaVlmModels.length === 0 && ( +
+ {en ? 'Ollama is online, but no local vision model is installed.' : 'Ollama 已在线，但未检测到本机视觉模型。'} +
+ )} {/* Prompt Template Selection */} @@ -363,17 +390,17 @@ export default function AiAnalysisPanel({ readOnly = false, onJobQueued }) { {/* Submit Button */} - + + + + + + + @@ -147,6 +198,11 @@ export default function AssetInventoryPanel({ readOnly = false, onTaskStart }) { +
@@ -183,6 +239,7 @@ export default function AssetInventoryPanel({ readOnly = false, onTaskStart }) { 产品 轨道 状态 + 完整性 动作 文件 @@ -196,6 +253,10 @@ export default function AssetInventoryPanel({ readOnly = false, onTaskStart }) { {item.source_format}{item.imaging_mode} / {item.polarization} {item.relative_orbit || '-'}abs {item.absolute_orbit || '-'} + + + {item.archive_integrity_member_count != null ? `${item.archive_integrity_member_count} files` : item.archive_integrity_method || '-'} + - diff --git a/frontend/src/DataCopierPanel.jsx b/frontend/src/DataCopierPanel.jsx index 3980743..36fb030 100644 --- a/frontend/src/DataCopierPanel.jsx +++ b/frontend/src/DataCopierPanel.jsx @@ -10,14 +10,19 @@ const COPY_STATUS_OPTIONS = [ ]; const BATCH_API_PAGE_LIMIT = 500; const BATCH_API_MAX_PAGES = 200; +const DINSAR_PURPOSE_PRODUCTION = 'production_prepare'; +const DINSAR_PURPOSE_DISTRIBUTION = 'source_distribution'; +const FALLBACK_DINSAR_TASK_POOL_ROOT = 'D:\\Task_Pool\\DInSAR'; +const FALLBACK_DATA_DISTRIBUTION_ROOT = 'D:\\Task_Pool\\Data_Distribution'; const DataCopierPanel = ({ apiEndpoint, readOnly = false, onJobQueued }) => { const { t } = useI18n(); - const [activeTab, setActiveTab] = useState('dinsar'); - const [destDir, setDestDir] = useState(''); + const [targetName, setTargetName] = useState(''); + const [dinsarTaskPoolRoot, setDinsarTaskPoolRoot] = useState(''); + const [dataDistributionRoot, setDataDistributionRoot] = useState(''); + const [dinsarPurpose, setDinsarPurpose] = useState(DINSAR_PURPOSE_PRODUCTION); const [copyStatuses, setCopyStatuses] = useState(['COMPLETED']); const [includeDinsarOrbitFiles, setIncludeDinsarOrbitFiles] = useState(true); - const [dinsarPackageMode, setDinsarPackageMode] = useState('task_folder'); const [skipExistingDinsarTasks, setSkipExistingDinsarTasks] = useState(true); const [dinsarMaxItems, setDinsarMaxItems] = useState('200'); const [batches, setBatches] = useState([]); @@ -57,13 +62,28 @@ const DataCopierPanel = ({ apiEndpoint, readOnly = false, onJobQueued }) => { useEffect(() => { fetchBatchesRef.current?.(); - }, [activeTab]); + }, []); + + useEffect(() => { + axios.get(`${apiEndpoint}/monitor/status`, { withCredentials: true }) + .then((response) => { + const taskRoot = (response.data?.dinsar_task_pool_root || '').toString().trim(); + const distributionRoot = (response.data?.data_distribution_root || '').toString().trim(); + if (taskRoot) { + setDinsarTaskPoolRoot(taskRoot); + } + if (distributionRoot) { + setDataDistributionRoot(distributionRoot); + } + }) + .catch((error) => { + console.error('Failed to load monitor status:', error); + }); + }, [apiEndpoint]); const fetchBatches = async () => { try { - const endpoint = activeTab === 'ps' - ? `${apiEndpoint}/task-batches/ps` - : `${apiEndpoint}/task-batches/dinsar`; + const endpoint = `${apiEndpoint}/task-batches/dinsar`; const allBatches = []; for (let page = 0; page < BATCH_API_MAX_PAGES; page += 1) { const offset = page * BATCH_API_PAGE_LIMIT; @@ -101,13 +121,21 @@ const DataCopierPanel = ({ apiEndpoint, readOnly = false, onJobQueued }) => { }; fetchLogsRef.current = fetchLogs; + const handleDinsarPurposeChange = (nextPurpose) => { + setDinsarPurpose(nextPurpose); + setTaskId(null); + setLogs([]); + setStatus('IDLE'); + setTargetName(''); + }; + const handleStartCopy = async () => { if (readOnly) { alert('当前账号为只读模式，无法执行复制任务。'); return; } - if (!selectedBatchId || !destDir) { - alert('请选择批次并设置目标目录。'); + if (!selectedBatchId || !targetName.trim()) { + alert('请选择批次并填写任务名。'); return; } if (!copyStatuses.length) { @@ -119,25 +147,21 @@ const DataCopierPanel = ({ apiEndpoint, readOnly = false, onJobQueued }) => { setLogs([]); setStatus('RUNNING'); - const endpoint = activeTab === 'ps' - ? `${apiEndpoint}/tools/copy-ps-stack` - : `${apiEndpoint}/tools/copy-dinsar-pairs`; + const endpoint = `${apiEndpoint}/tools/copy-dinsar-pairs`; try { const payload = { batch_id: selectedBatchId, - dest_dir: destDir, + target_name: targetName.trim(), copy_statuses: copyStatuses, }; - if (activeTab === 'dinsar') { - payload.include_orbit_files = includeDinsarOrbitFiles; - payload.package_mode = dinsarPackageMode; - payload.export_zip = dinsarPackageMode === 'task_zip'; - payload.skip_existing = skipExistingDinsarTasks; - const parsedMaxItems = Number.parseInt(dinsarMaxItems, 10); - if (Number.isFinite(parsedMaxItems) && parsedMaxItems > 0) { - payload.max_items = parsedMaxItems; - } + payload.include_orbit_files = includeDinsarOrbitFiles; + payload.package_mode = dinsarPurpose === DINSAR_PURPOSE_PRODUCTION ? 'task_folder' : 'source_bundle'; + payload.export_zip = false; + payload.skip_existing = skipExistingDinsarTasks; + const parsedMaxItems = Number.parseInt(dinsarMaxItems, 10); + if (Number.isFinite(parsedMaxItems) && parsedMaxItems > 0) { + payload.max_items = parsedMaxItems; } const response = await axios.post(endpoint, payload, { withCredentials: true }); const taskId = response.data.task_id; @@ -175,110 +199,85 @@ const DataCopierPanel = ({ apiEndpoint, readOnly = false, onJobQueued }) => { return (
-
- - -
-
{readOnly && (
当前账号为只读模式，无法发起复制任务。
)} - {activeTab === 'dinsar' && ( -
- -
- - - -
-
- - -
-
- - setDinsarMaxItems(event.target.value)} - disabled={status === 'RUNNING' || readOnly} - style={{ width: '110px', padding: '5px 7px' }} - /> - 0 或留空表示不限制 -
-
- 去重源数据包只复制唯一影像和精轨，并写出 pairs.json；再次分发到同一目录时会接着追加未导出的配对。 -
+
+ +
+ +
- )} +
+ {dinsarPurpose === DINSAR_PURPOSE_PRODUCTION + ? '生成可直接运行的 Task_Pool 任务目录（Task_YYYYMMDD_YYYYMMDD / master / slave / orbit）' + : '导出源压缩包去重包（data / orbit / pairs.json / manifest.json）'} +
+
+ + +
+
+ + setDinsarMaxItems(event.target.value)} + disabled={status === 'RUNNING' || readOnly} + style={{ width: '110px', padding: '5px 7px' }} + /> + 0 或留空表示不限制 +
+
+ {dinsarPurpose === DINSAR_PURPOSE_PRODUCTION + ? '源池仍管理压缩包；这里按任务解包到本机 Task_Pool，供 D-InSAR 引擎直接使用。' + : '该归口用于跨目录/跨机器下发源压缩包，不作为生产运行入口。'} +
+
@@ -320,25 +319,36 @@ const DataCopierPanel = ({ apiEndpoint, readOnly = false, onJobQueued }) => {
- + setDestDir(e.target.value)} - placeholder="例如：D:/Data/Project_X/PS_Stack" + value={targetName} + onChange={(e) => setTargetName(e.target.value)} + placeholder={ + dinsarPurpose === DINSAR_PURPOSE_PRODUCTION + ? '例如：MDJ_20240422_20240520' + : '例如：Project_X_DInSAR_Source_Bundle' + } disabled={status === 'RUNNING' || readOnly} style={{ width: '100%', padding: '8px' }} /> +
+ {dinsarPurpose === DINSAR_PURPOSE_PRODUCTION + ? `服务器写入目录：${dinsarTaskPoolRoot || FALLBACK_DINSAR_TASK_POOL_ROOT}\\${targetName || '<任务名>'}` + : `服务器写入目录：${dataDistributionRoot || FALLBACK_DATA_DISTRIBUTION_ROOT}\\${targetName || '<任务名>'}`} +
{status !== 'IDLE' && status !== 'RUNNING' && ( - - -
- {unpackMessage} -
-
-
- -
-
Sentinel-1 解包 / 扫描
-
-
S1 源数据{formatList(config.s1_source_dirs)}
-
S1 存储{formatList(config.s1_storage_dirs)}
-
S1 精轨{formatList(config.s1_orbit_dirs)}
-
-
- - - -
- {s1ActiveTask ? (s1ActiveTask.message || 'Sentinel-1 任务运行中...') : s1Message} -
-
-
- -
-
GF3 SARscape 生产
-
-
压缩包来源{formatList(config.gf3_archive_source_dirs)}
-
SARscape 原生{formatList(config.gf3_sarscape_native_dirs)}
-
L2 存储{formatList(config.gf3_storage_dirs)}
-
Runtime{config.gf3_sarscape_runtime_dir || '未配置'}
-
Wrapper{config.gf3_sarscape_wrapper_exe || '未配置'}
-
SARscape DEM{config.gf3_sarscape_dem_path || '未配置'}
-
极化{config.gf3_sarscape_polarizations || 'HH,HV'}
-
Legacy GDAL{gf3LegacyGdalEnabled ? '启用' : '关闭'}
-
- 影像日期 - - - - -
-
-
- {gf3LegacyGdalEnabled && ( - <> - - - +
本机存储感知
+
+ {config.storage_roots.length ? config.storage_roots.map((item, index) => ( +
+ {item.label || item.role || '路径'} + + {storageStatusText(item.status)} · {formatGb(item.free_gb)} 可用 + + + {item.path || '未配置'} + {item.total_gb != null ? `（总量 ${formatGb(item.total_gb)}）` : ''} + {item.message ? ` ${item.message}` : ''} + +
+ )) : ( +
未返回本机存储状态。
)} +
+
+ +
+
LT-1 / Sentinel-1 压缩包资产索引
+
+
压缩包源池{formatList(sourceInventoryDirs)}
+
精轨源资产{formatList(orbitInventoryDirs)}
+
LT-1 生产 TXT 池{config.orbit_production_txt_pool || '未配置'}
+
管理规则压缩包只抽取 XML/manifest 和预览图；LT-1 精轨扫描后同步到生产 TXT 池，S1 精轨只登记 EOF 资产。
+
+
+ + + + + +
+ {archiveAuditActiveTask + ? (archiveAuditActiveTask.message || '完整性审计运行中...') + : sourceInventoryActiveTask + ? (sourceInventoryActiveTask.message || '资产索引运行中...') + : s1Message} +
+
+
+ +
+
GF3 _geo 原生结果登记
+
+
GF3 _geo 结果根目录{formatList(config.gf3_sarscape_native_dirs)}
+
GF3 标准/索引池{formatList(config.gf3_storage_dirs)}
+
登记内容按 YYYYMMDD_geo/场景目录扫描 *_geo ENVI 二进制，WebP 从 _geo 主数据生成
+
+
- - -
+
{gf3ActiveTask ? (gf3ActiveTask.message || 'GF3 任务运行中...') : gf3Message}
+ +
+
+
扫描任务状态
+ +
+ {clearScanHistoryMessage && ( +
+ {clearScanHistoryMessage} +
+ )} + {displayedScanTasks.length ? ( +
+ {displayedScanTasks.map((task) => { + const progress = clampProgress(task.progress); + const isSelected = task.task_id === selectedTask?.task_id; + return ( + + ); + })} +
+ ) : ( +
暂无扫描任务。
+ )} +
+
-
实时日志
+
+ {selectedTask ? `${taskTitle(selectedTask)} 日志` : '实时日志'} +
- {displayLogs.length === 0 ? ( + {selectedTask ? ( + selectedTaskLogs.length ? ( + selectedTaskLogs.map((log) => ( +
+ [{formatTaskTime(log.timestamp)}] [{log.level || 'INFO'}] {log.message} +
+ )) + ) : ( +
暂无任务日志...
+ ) + ) : displayLogs.length === 0 ? (
暂无日志...
) : ( displayLogs.map((log, index) => ( @@ -1100,85 +819,6 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
- {showUnpackDialog && ( -
-
event.stopPropagation()}> -

LT-1 解包任务参数

-
{ - event.preventDefault(); - handleUnpackRun(); - }} - > -
- 本次填写的参数只覆盖当前这一次解包任务，不会修改 `.env` 默认值。输入 `0` 表示不限。 -
- -
-
来源目录{formatList(unpackConfig.source_dirs)}
-
LT-1 存储{formatList(unpackConfig.insar_storage_dirs)}
-
- -
- - handleUnpackOptionChange('max_files_per_run', event.target.value)} - disabled={unpackLoading} - /> -
- -
- - handleUnpackOptionChange('max_runtime_minutes', event.target.value)} - disabled={unpackLoading} - /> -
- - {unpackDialogError && ( -
- {unpackDialogError} -
- )} - -
- - -
-
-
-
- )}
); }; diff --git a/frontend/src/DinsarProductionPanel.jsx b/frontend/src/DinsarProductionPanel.jsx index 7c1ea35..657d866 100644 --- a/frontend/src/DinsarProductionPanel.jsx +++ b/frontend/src/DinsarProductionPanel.jsx @@ -1,6 +1,6 @@ import React, { useCallback, useEffect, useState } from 'react'; -import { deleteRunLog, deleteRunRecord, getRunLog, listEngines, listRuns, previewPyintInputAssets, submitRun } from './api/dinsarProduction'; +import { deleteRunLog, deleteRunRecord, getRunLog, listEngines, listRuns, listTaskRoots, previewPyintInputAssets, submitRun } from './api/dinsarProduction'; import { clearTaskLogs, deleteTaskLog, deleteTaskRecord, getRecentTasks, getTaskLogs } from './api/tasks'; import { formatSatelliteFamilyLabel, inferSatelliteFamilyFromResultLike } from './utils/satelliteFamily'; import useTaskMonitor from './hooks/useTaskMonitor'; @@ -16,8 +16,9 @@ const card = { const EMPTY_ARRAY = []; const EMPTY_OBJECT = {}; const RUN_HISTORY_PAGE_SIZE = 200; -const TASK_HISTORY_PAGE_SIZE = 500; +const TASK_HISTORY_PAGE_SIZE = 200; const TASK_LOG_PAGE_SIZE = 1000; +const INLINE_TASK_LOG_LIMIT = 200; const TERMINAL_STATUS_VALUES = new Set(['COMPLETED', 'FAILED', 'CANCELLED', 'CANCELED', 'success', 'failed', 'cancelled', 'canceled']); const ENGINE_STATUS_COLOR = { @@ -96,6 +97,8 @@ const RERUN_MODE_OPTIONS = [ description: '忽略已有结果，对本次选中的任务全部重新执行。', }, ]; +const MANUAL_TASK_ROOT_VALUE = '__manual__'; +const NO_TASK_ROOT_VALUE = ''; function formatEngineLabel(engineCode, engineLabel = '') { return engineLabel || ENGINE_LABEL[engineCode] || engineCode || '-'; @@ -255,6 +258,15 @@ function formatPathValue(value) { return text || '-'; } +function formatTaskRootUpdatedAt(value) { + if (!value) return ''; + try { + return new Date(value).toLocaleString(); + } catch { + return String(value); + } +} + function RunPathBlock({ run }) { const items = Array.isArray(run?.items) ? run.items : []; const item = items.find(entry => entry?.status === 'RUNNING') || items[0] || null; @@ -560,6 +572,12 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) const [selectedEngine, setSelectedEngine] = useState('sarscape'); const [selectedProfile, setSelectedProfile] = useState('custom6'); const [rootDir, setRootDir] = useState(''); + const [taskRoots, setTaskRoots] = useState([]); + const [taskRootsLoading, setTaskRootsLoading] = useState(false); + const [taskRootsError, setTaskRootsError] = useState(''); + const [taskPoolRoot, setTaskPoolRoot] = useState(''); + const [selectedTaskRootPath, setSelectedTaskRootPath] = useState(''); + const [manualRootDir, setManualRootDir] = useState(''); const [numToProcess, setNumToProcess] = useState(0); const [timeoutSec, setTimeoutSec] = useState(''); const [engineExtraParams, setEngineExtraParams] = useState({}); @@ -588,6 +606,7 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) const [taskLogsLoading, setTaskLogsLoading] = useState(false); const [taskLogActionLoading, setTaskLogActionLoading] = useState(false); const [taskLogDeletingId, setTaskLogDeletingId] = useState(null); + const [monitorLoaded, setMonitorLoaded] = useState(false); const currentEngineObj = engines.find(engine => engine.engine_code === selectedEngine) || null; const currentProfiles = currentEngineObj?.profiles || EMPTY_ARRAY; @@ -601,9 +620,10 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) ? 'LandSAR 当前使用已跑通的稳定参数。GACOS 大气相位改正需要外部大气延迟文件，未配置文件前不可启用；垂直向形变为可选输出，默认关闭。' : '这些参数影响当前引擎的生产模板。建议先使用默认值，只有在结果边界、噪声或几何表现异常时再逐项调整。'; const pyintPreviewBlocksSubmit = selectedEngine === 'pyint' && pyintPreview && pyintPreview.allow_submit === false; + const selectedTaskRoot = taskRoots.find(item => item.path === selectedTaskRootPath) || null; const taskMonitor = useTaskMonitor({ taskTypes: DINSAR_PRODUCTION_TASK_TYPES, - showRecent: true, + showRecent: false, recentLimit: 1, }); const latestRunWithTask = runs.find(run => run?.task_id) || null; @@ -638,40 +658,38 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) } }, []); + const loadTaskRoots = useCallback(async () => { + setTaskRootsLoading(true); + setTaskRootsError(''); + try { + const data = await listTaskRoots(); + const items = Array.isArray(data?.items) ? data.items : []; + setTaskPoolRoot(String(data?.root || '')); + setTaskRoots(items); + setSelectedTaskRootPath(current => { + if (current === MANUAL_TASK_ROOT_VALUE) return current; + if (current && items.some(item => item.path === current)) return current; + return NO_TASK_ROOT_VALUE; + }); + } catch (err) { + setTaskRoots([]); + setTaskRootsError(err?.response?.data?.detail || err.message || '生产任务根目录加载失败'); + } finally { + setTaskRootsLoading(false); + } + }, []); + const loadRuns = useCallback(async (options = {}) => { const silent = !!options.silent; if (!silent) setRunsLoading(true); try { - const loadProductionRuns = async () => { - const allRuns = []; - let offset = 0; - while (true) { - const data = await listRuns(RUN_HISTORY_PAGE_SIZE, offset); - const pageRuns = data?.runs || []; - allRuns.push(...pageRuns); - const total = Number(data?.total || 0); - if (pageRuns.length < RUN_HISTORY_PAGE_SIZE || allRuns.length >= total) break; - offset += pageRuns.length; - } - return allRuns; - }; - const loadRecentTasks = async () => { - const allTasks = []; - let offset = 0; - while (true) { - const data = await getRecentTasks(DINSAR_PRODUCTION_TASK_TYPES, [], TASK_HISTORY_PAGE_SIZE, offset); - const pageTasks = Array.isArray(data) ? data : (data?.tasks || []); - allTasks.push(...pageTasks); - if (pageTasks.length < TASK_HISTORY_PAGE_SIZE) break; - offset += pageTasks.length; - } - return allTasks; - }; - const [productionRuns, recentTasks] = await Promise.all([ - loadProductionRuns(), - loadRecentTasks(), + const [productionRunData, recentTaskData] = await Promise.all([ + listRuns(RUN_HISTORY_PAGE_SIZE, 0), + getRecentTasks(DINSAR_PRODUCTION_TASK_TYPES, [], TASK_HISTORY_PAGE_SIZE, 0), ]); - const nextRuns = mergeRunRows(productionRuns, recentTasks); + const productionRuns = productionRunData?.runs || []; + const recentTasks = Array.isArray(recentTaskData) ? recentTaskData : (recentTaskData?.tasks || []); + const nextRuns = mergeRunRows(productionRuns, recentTasks, RUN_HISTORY_PAGE_SIZE); setRuns(nextRuns); return nextRuns; } catch { @@ -690,7 +708,8 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) } if (!silent) setTaskLogsLoading(true); try { - const logs = await fetchAllTaskLogs(taskId); + const data = await getTaskLogs(taskId, INLINE_TASK_LOG_LIMIT, 0); + const logs = data?.logs || []; setTaskLogs(logs); } catch { setTaskLogs([]); @@ -737,30 +756,27 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) const refreshMonitor = useCallback(async (options = {}) => { const silent = !!options.silent; - const [nextRuns, nextRecentTasks] = await Promise.all([ - loadRuns({ silent }), - taskMonitor.refreshRecentTasks(), - ]); + const nextRuns = await loadRuns({ silent }); + setMonitorLoaded(true); const fallbackTaskId = taskMonitor.activeTasks[0]?.task_id - || nextRecentTasks[0]?.task_id || nextRuns.find(run => run?.task_id)?.task_id || ''; await loadTaskLogs(fallbackTaskId, { silent }); - }, [loadRuns, loadTaskLogs, taskMonitor]); + }, [loadRuns, loadTaskLogs, taskMonitor.activeTasks]); useEffect(() => { loadEngines(); - refreshMonitor(); - }, [loadEngines, refreshMonitor]); + loadTaskRoots(); + }, [loadEngines, loadTaskRoots]); useEffect(() => { - const intervalMs = taskMonitor.isBusy ? 5000 : 15000; - const timer = window.setInterval(() => { - refreshMonitor({ silent: true }); - }, intervalMs); - return () => window.clearInterval(timer); - }, [taskMonitor.isBusy, refreshMonitor]); + if (selectedTaskRootPath === MANUAL_TASK_ROOT_VALUE) { + setRootDir(manualRootDir); + return; + } + setRootDir(selectedTaskRootPath); + }, [manualRootDir, selectedTaskRootPath]); useEffect(() => { if (currentProfiles.length > 0) { @@ -984,7 +1000,7 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) } }, [logModal.open, logModal.runId, logTaskId, readOnly, refreshMonitor, runLogDeletingId]); - const isSubmitDisabled = readOnly || submitting || !currentEngineObj?.available || pyintPreviewBlocksSubmit; + const isSubmitDisabled = readOnly || submitting || !currentEngineObj?.available || !rootDir.trim() || pyintPreviewBlocksSubmit; return (
@@ -1153,6 +1169,17 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued })
模板：{currentProfileObj?.label || selectedProfile}
任务数量：{Number(numToProcess) > 0 ? Number(numToProcess) : '全部'}
执行策略：{RERUN_MODE_LABEL[rerunMode] || rerunMode}
+
+ 根目录：{selectedTaskRoot?.name || (selectedTaskRootPath === MANUAL_TASK_ROOT_VALUE ? '手动路径' : '-')} +
+ {selectedTaskRoot && ( +
+ 可识别 Task：{Number(selectedTaskRoot.task_count || 0)} +
+ )} +
+ 路径：{rootDir || '-'} +
@@ -1252,13 +1279,30 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) )}
-
- - setRootDir(event.target.value)} - placeholder="批处理根目录或单个任务目录" - disabled={readOnly} +
+
+ + +
+ + {selectedTaskRootPath === MANUAL_TASK_ROOT_VALUE && ( + setManualRootDir(event.target.value)} + placeholder="批处理根目录或单个任务目录" + disabled={readOnly} + style={{ + width: '100%', + marginTop: 6, + padding: '5px 8px', + borderRadius: 4, + border: '1px solid #e2e8f0', + fontSize: 13, + boxSizing: 'border-box', + }} + /> + )} +
+ {taskRootsError || ( + rootDir + ? `服务器路径：${rootDir}` + : `扫描目录：${taskPoolRoot || '未配置'}；请选择其中一个一级生产任务目录。` + )} +
+ {selectedTaskRoot && ( +
+ 最近更新：{formatTaskRootUpdatedAt(selectedTaskRoot.updated_at)}；不可识别 Task：{Number(selectedTaskRoot.invalid_child_count || 0)} +
+ )}
@@ -1544,7 +1628,7 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) border: 'none', background: currentEngineObj?.available ? '#3b82f6' : '#94a3b8', color: '#fff', - cursor: currentEngineObj?.available ? 'pointer' : 'not-allowed', + cursor: isSubmitDisabled ? 'not-allowed' : 'pointer', fontSize: 13, fontWeight: 600, }} @@ -1564,20 +1648,21 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) 运行监控
- 监控与日志改为手动刷新，避免界面持续轮询请求。 + 监控与日志不会自动轮询；点击手动刷新时只加载最近记录和最近日志。
{monitoredTask && ( @@ -1714,7 +1799,9 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) )}
运行记录（已加载 {runs.length} 条）
- {runsLoading ? ( + {!monitorLoaded && !runsLoading ? ( +
尚未加载监控记录，请点击手动刷新。
+ ) : runsLoading ? (
加载中...
) : runs.length === 0 ? (
暂无记录。
diff --git a/frontend/src/HealthCheckPanel.jsx b/frontend/src/HealthCheckPanel.jsx index 7966b78..cbdcfaf 100644 --- a/frontend/src/HealthCheckPanel.jsx +++ b/frontend/src/HealthCheckPanel.jsx @@ -374,9 +374,10 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => { const orbitPools = orbitStatus?.pools || {}; const orbitConsistency = orbitStatus?.consistency || {}; const orbitDatabase = orbitStatus?.database || {}; + const orbitIsce2Enabled = Boolean(orbitPools.isce2?.enabled || orbitConsistency.isce2?.enabled || orbitDatabase.isce2_enabled); const orbitMismatchCount = toNumber(orbitConsistency.mismatch_count); const orbitDbMissingEnviCount = toNumber(orbitDatabase.stems_missing_in_envi_count); - const orbitDbMissingIsce2Count = toNumber(orbitDatabase.stems_missing_in_isce2_count); + const orbitDbMissingIsce2Count = orbitIsce2Enabled ? toNumber(orbitDatabase.stems_missing_in_isce2_count) : 0; const orbitDbMissingPathCount = toNumber(orbitDatabase.db_missing_path_count); const orbitDbFlagIssueCount = toNumber(orbitDatabase.has_orbit_but_missing_path_count) + @@ -384,15 +385,15 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => { const orbitScanErrorCount = (orbitSource.errors?.length || 0) + (orbitPools.envi?.errors?.length || 0) + - (orbitPools.isce2?.errors?.length || 0); + (orbitIsce2Enabled ? (orbitPools.isce2?.errors?.length || 0) : 0); const orbitDuplicateCount = toNumber(orbitSource.duplicate_count) + toNumber(orbitPools.envi?.duplicate_count) + - toNumber(orbitPools.isce2?.duplicate_count); + (orbitIsce2Enabled ? toNumber(orbitPools.isce2?.duplicate_count) : 0); const orbitSuspectBadCount = toNumber(orbitSource.suspect_bad_count); const orbitSourceWithoutEnviCount = toNumber(orbitSource.source_without_envi_count); const orbitEnviWithoutSourceCount = toNumber(orbitSource.envi_without_source_count); - const orbitIsce2WithoutSourceCount = toNumber(orbitSource.isce2_without_source_count); + const orbitIsce2WithoutSourceCount = orbitIsce2Enabled ? toNumber(orbitSource.isce2_without_source_count) : 0; const orbitQuarantinePath = orbitSource.quarantine_path || orbitStatus?.source_gaps?.quarantine_path; const orbitBadSourceSamples = asArray(orbitSource.bad_source_samples).filter(hasOrbitCorruptionSignal); const orbitSuspectBadSamples = asArray(orbitSource.suspect_bad_samples); @@ -1118,9 +1119,15 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => { )}
- {en ? 'Source / ENVI / ISCE2' : '源目录 / ENVI / ISCE2'} - {toNumber(orbitSource.total_source)} / {toNumber(orbitPools.envi?.total)} / {toNumber(orbitPools.isce2?.total)} + {orbitIsce2Enabled + ? (en ? 'Source / ENVI-Gamma TXT / ISCE2 XML' : '源目录 / ENVI-Gamma TXT / ISCE2 XML') + : (en ? 'Source / ENVI-Gamma TXT' : '源目录 / ENVI-Gamma TXT')} + + + {orbitIsce2Enabled + ? `${toNumber(orbitSource.total_source)} / ${toNumber(orbitPools.envi?.total)} / ${toNumber(orbitPools.isce2?.total)}` + : `${toNumber(orbitSource.total_source)} / ${toNumber(orbitPools.envi?.total)}`}
@@ -1133,17 +1140,24 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => { {en ? 'Pool mismatches' : '池不一致'} {orbitMismatchCount}
+ {orbitIsce2Enabled ? ( +
+ {en ? 'Suspect bad TXT / source-only' : '疑似坏 TXT / 仅源存在'} + {orbitSuspectBadCount} / {orbitSourceWithoutEnviCount} +
+ ) : ( +
+ {en ? 'Source-only' : '仅源存在'} + {orbitSourceWithoutEnviCount} +
+ )}
- {en ? 'Suspect bad TXT / source-only' : '疑似坏 TXT / 仅源存在'} - {orbitSuspectBadCount} / {orbitSourceWithoutEnviCount} + {orbitIsce2Enabled ? (en ? 'TXT-only / ISCE2-only' : '仅 TXT / 仅 ISCE2') : (en ? 'TXT-only' : '仅 TXT')} + {orbitIsce2Enabled ? `${orbitEnviWithoutSourceCount} / ${orbitIsce2WithoutSourceCount}` : orbitEnviWithoutSourceCount}
- {en ? 'ENVI-only / ISCE2-only' : '仅 ENVI / 仅 ISCE2'} - {orbitEnviWithoutSourceCount} / {orbitIsce2WithoutSourceCount} -
-
- {en ? 'DB missing in ENVI / ISCE2' : '数据库在 ENVI / ISCE2 缺失'} - {orbitDbMissingEnviCount} / {orbitDbMissingIsce2Count} + {orbitIsce2Enabled ? (en ? 'DB missing in TXT / ISCE2' : '数据库在 TXT / ISCE2 缺失') : (en ? 'DB missing in TXT' : '数据库在 TXT 缺失')} + {orbitIsce2Enabled ? `${orbitDbMissingEnviCount} / ${orbitDbMissingIsce2Count}` : orbitDbMissingEnviCount}
{en ? 'Duplicate stems / scan errors' : '重复 stem / 扫描异常'} @@ -1152,14 +1166,21 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => {
{en - ? 'Orbit scan writes ENVI TXT and ISCE2 XML pools automatically. This panel now shows source, pool, and database consistency together.' - : '“扫描精轨”会自动同步 ENVI TXT 和 ISCE2 XML。本卡片同时展示源目录、本地池和数据库三侧的一致性。'} + ? (orbitIsce2Enabled + ? 'LT-1 orbit scans synchronize the production TXT pool and the legacy ISCE2 XML pool. S1 EOF files remain registered as source orbit assets.' + : 'LT-1 orbit scans synchronize the production TXT pool for ENVI/SARscape and Gamma. S1 EOF files remain registered as source orbit assets; ISCE2 XML is disabled.') + : (orbitIsce2Enabled + ? 'LT-1 精轨扫描会同步生产 TXT 池和 legacy ISCE2 XML 池；S1 EOF 只登记为源精轨资产。' + : 'LT-1 精轨扫描会同步 ENVI/SARscape 与 Gamma 共用的生产 TXT 池；S1 EOF 只登记为源精轨资产，ISCE2 XML 已停用。')}
{en ? 'Source path: ' : '源目录路径：'}{formatPathText(orbitSource.path)}
-
{en ? 'ENVI pool: ' : 'ENVI 池：'}{formatPathText(orbitPools.envi?.path)}
-
{en ? 'ISCE2 pool: ' : 'ISCE2 池：'}{formatPathText(orbitPools.isce2?.path)}
-
{en ? 'LANDSAR pool: ' : 'LANDSAR 池：'}{formatPathText(orbitPools.landsar?.path)}
-
{en ? 'Quarantine path: ' : '隔离目录：'}{formatPathText(orbitQuarantinePath)}
+
{en ? 'Production TXT pool: ' : '生产 TXT 池：'}{formatPathText(orbitPools.envi?.path)}
+ {orbitIsce2Enabled && ( + <> +
{en ? 'Legacy ISCE2 pool: ' : 'Legacy ISCE2 池：'}{formatPathText(orbitPools.isce2?.path)}
+
{en ? 'Quarantine path: ' : '隔离目录：'}{formatPathText(orbitQuarantinePath)}
+ + )} {orbitDuplicateCount > 0 && (
@@ -1182,7 +1203,7 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => { : `数据库 orbit_file_path 指向不存在文件：${orbitDbMissingPathCount} / ${toNumber(orbitDatabase.distinct_orbit_path_count)}`}
)} - {orbitSuspectBadCount > 0 && ( + {orbitIsce2Enabled && orbitSuspectBadCount > 0 && (
{en ? `Suspect bad source TXT (source exists but ISCE2 XML missing): ${orbitSuspectBadCount}` @@ -1202,7 +1223,7 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => { {orbitDatabase.sample_missing_in_envi.slice(0, 5).join(', ')}
)} - {orbitDatabase.sample_missing_in_isce2?.length > 0 && ( + {orbitIsce2Enabled && orbitDatabase.sample_missing_in_isce2?.length > 0 && (
{en ? 'DB expected but ISCE2 pool missing: ' : '数据库期望但 ISCE2 池缺失：'} {orbitDatabase.sample_missing_in_isce2.slice(0, 5).join(', ')} @@ -1214,7 +1235,7 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => { {renderOrbitSourceIssueDetails(item, en, formatPathText)}
))} - {orbitSuspectWithoutCorruptionSamples.slice(0, 5).map((item) => ( + {orbitIsce2Enabled && orbitSuspectWithoutCorruptionSamples.slice(0, 5).map((item) => (
{item.name} {renderOrbitSourceIssueDetails(item, en, formatPathText)} @@ -1257,7 +1278,7 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => { {en ? 'ENVI pool scan error: ' : 'ENVI 池扫描异常：'}{item}
))} - {(orbitPools.isce2?.errors || []).slice(0, 3).map((item, index) => ( + {orbitIsce2Enabled && (orbitPools.isce2?.errors || []).slice(0, 3).map((item, index) => (
{en ? 'ISCE2 pool scan error: ' : 'ISCE2 池扫描异常：'}{item}
@@ -1283,44 +1304,48 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => { > {orbitSyncing ? (en ? 'Checking...' : '检查中...') : (en ? 'Check Consistency' : '精轨一致性检查')} - - + {orbitIsce2Enabled && ( + <> + + + + )}
{orbitSyncResult && ( @@ -1387,8 +1412,8 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => {
{en - ? `Source scan ${toNumber(orbitSyncResult.sync_result?.total_source)}, ENVI copied ${(orbitSyncResult.sync_result?.envi?.copied || []).length}, ENVI refreshed ${(orbitSyncResult.sync_result?.envi?.updated || []).length}, ISCE2 converted ${(orbitSyncResult.sync_result?.isce2?.converted || []).length}, ISCE2 refreshed ${(orbitSyncResult.sync_result?.isce2?.reconverted || []).length}` - : `源目录扫描 ${toNumber(orbitSyncResult.sync_result?.total_source)} 项，ENVI 新增 ${(orbitSyncResult.sync_result?.envi?.copied || []).length} 项、刷新 ${(orbitSyncResult.sync_result?.envi?.updated || []).length} 项，ISCE2 新增转换 ${(orbitSyncResult.sync_result?.isce2?.converted || []).length} 项、重转 ${(orbitSyncResult.sync_result?.isce2?.reconverted || []).length} 项`} + ? `Source scan ${toNumber(orbitSyncResult.sync_result?.total_source)}, TXT copied ${(orbitSyncResult.sync_result?.envi?.copied || []).length}, TXT refreshed ${(orbitSyncResult.sync_result?.envi?.updated || []).length}${orbitSyncResult.isce2_enabled ? `, ISCE2 converted ${(orbitSyncResult.sync_result?.isce2?.converted || []).length}, ISCE2 refreshed ${(orbitSyncResult.sync_result?.isce2?.reconverted || []).length}` : ', ISCE2 disabled'}` + : `源目录扫描 ${toNumber(orbitSyncResult.sync_result?.total_source)} 项，TXT 新增 ${(orbitSyncResult.sync_result?.envi?.copied || []).length} 项、刷新 ${(orbitSyncResult.sync_result?.envi?.updated || []).length} 项${orbitSyncResult.isce2_enabled ? `，ISCE2 新增转换 ${(orbitSyncResult.sync_result?.isce2?.converted || []).length} 项、重转 ${(orbitSyncResult.sync_result?.isce2?.reconverted || []).length} 项` : '，ISCE2 已停用'}`}
{(orbitSyncResult.repaired_from_envi || []).slice(0, 5).length > 0 && (
@@ -1428,8 +1453,8 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => {
{en - ? `ENVI ${toNumber(orbitSyncResult.envi?.total)}, ISCE2 ${toNumber(orbitSyncResult.isce2?.total)}, scan errors ${toNumber(orbitSyncResult.error_count)}` - : `ENVI ${toNumber(orbitSyncResult.envi?.total)} 项，ISCE2 ${toNumber(orbitSyncResult.isce2?.total)} 项，扫描异常 ${toNumber(orbitSyncResult.error_count)} 项`} + ? `TXT ${toNumber(orbitSyncResult.envi?.total)}${orbitSyncResult.isce2?.enabled ? `, ISCE2 ${toNumber(orbitSyncResult.isce2?.total)}` : ', ISCE2 disabled'}, scan errors ${toNumber(orbitSyncResult.error_count)}` + : `TXT ${toNumber(orbitSyncResult.envi?.total)} 项${orbitSyncResult.isce2?.enabled ? `，ISCE2 ${toNumber(orbitSyncResult.isce2?.total)} 项` : '，ISCE2 已停用'}，扫描异常 ${toNumber(orbitSyncResult.error_count)} 项`}
{(orbitSyncResult.mismatches || []).slice(0, 5).map((item, index) => (
diff --git a/frontend/src/ProductionWorkspace.jsx b/frontend/src/ProductionWorkspace.jsx index 6cc5c67..4dccc55 100644 --- a/frontend/src/ProductionWorkspace.jsx +++ b/frontend/src/ProductionWorkspace.jsx @@ -4,6 +4,7 @@ import { PRODUCTION_WORKSPACE_ENTRY_TO_VIEW, PRODUCTION_WORKSPACE_TAB, PRODUCTION_WORKSPACE_VIEWS, + PRODUCTION_WORKSPACE_WORKBENCHES, } from './config/appConstants'; import { PanelLoadingBody } from './components/app/AppLoadingFallbacks'; @@ -11,6 +12,10 @@ const LazyDinsarProductionPanel = lazy(() => import('./DinsarProductionPanel')); const LazySbasInsarProductionPanel = lazy(() => import('./SbasInsarProductionPanel')); const LazySbasInsarProductsPanel = lazy(() => import('./SbasInsarProductsPanel')); const LazyDinsarProductsPanel = lazy(() => import('./DinsarProductsPanel')); +const LazyPairPlanningPanel = lazy(() => import('./panels/PairPlanningPanel')); +const LazyPairsListPanel = lazy(() => import('./panels/PairsListPanel')); +const LazyBatchPanel = lazy(() => import('./panels/BatchPanel')); +const LazyDataCopierPanel = lazy(() => import('./DataCopierPanel')); const shellStyle = { minHeight: '100%', @@ -40,25 +45,128 @@ const summaryCardStyle = { background: 'rgba(255, 255, 255, 0.82)', }; +const SENSOR_PRODUCTION_PLACEHOLDERS = { + lt1_production: { + title: '陆探一号生产模块', + subtitle: '当前先占位纳入生产管理，执行链路保留 LandSAR、ENVI+SARscape、Gamma/PyINT。', + rows: [ + ['源压缩包', 'D:\\LuTan1_Image_Pool_Zip，只索引包内 XML/元数据，不做全量解包。'], + ['精密轨道', 'D:\\LT1_data_lsarorbit，本机部署并绑定到源资产。'], + ['按需解包', '生产任务需要时才 materialize 到 D:\\Task_Pool\\DInSAR 或 D:\\Task_Pool\\SBAS。'], + ['生产边界', 'D-InSAR 与 SBAS-InSAR 均使用本机 Task_Pool，不允许 UNC 参与运行。'], + ['结果管理', '生成结果进入 D-InSAR/SBAS 产物目录，由生产管理结果页统一重建 catalog。'], + ], + }, + sentinel1_production: { + title: 'Sentinel-1 生产模块', + subtitle: '当前先占位纳入生产管理，D-InSAR 保留 Gamma/PyINT 路径，SBAS 仍为规划态。', + rows: [ + ['源压缩包', 'D:\\Sentinel1_Image_Pool_ZIP，本机登记 ZIP/SAFE 元数据。'], + ['精密轨道', 'D:\\Sentinel1_EOF_Pool，本机保存 AUX_POEORB/RESORB。'], + ['按需解包', '需要运行时才将 ZIP 解包到本机 Task_Pool，界面不提供全量解包按钮。'], + ['D-InSAR', 'Gamma/PyINT 可作为生产方向，运行材料必须来自本机路径。'], + ['SBAS', '当前仅做堆栈发现和规划，执行链路未启用。'], + ], + }, + gf3_native_registration: { + title: '高分三结果登记', + subtitle: 'GF3 不在本机生产；另一台 SARscape 服务器完成 _geo 后复制到本机登记。', + rows: [ + ['外部生产', '外部机器按 YYYYMMDD_geo/场景目录输出 SARscape 原生 _geo 二进制。'], + ['本机落盘', '复制到 D:\\GaoFen3_Pool\\native_geo 后递归扫描登记。'], + ['预览生成', 'WebP 从 *_geo 主二进制读取生成，不使用 *_geo_ql.tif 作为正式预览源。'], + ['精轨', 'GF3 本链路无精密轨道管理。'], + ['结果管理', '登记后的 GF3 资产进入数据管理，后续需要全影像时再提取/标准化。'], + ], + }, +}; + +function SensorProductionPlaceholder({ viewKey }) { + const data = SENSOR_PRODUCTION_PLACEHOLDERS[viewKey]; + if (!data) { + return null; + } + return ( +
+
当前设计约定
+

{data.title}

+

+ {data.subtitle} +

+
+ {data.rows.map(([label, value]) => ( +
+ {label} + {value} +
+ ))} +
+
+ ); +} + function resolveView(entry) { return PRODUCTION_WORKSPACE_ENTRY_TO_VIEW[entry] || PRODUCTION_WORKSPACE_ENTRY_TO_VIEW[PRODUCTION_WORKSPACE_TAB]; } +function resolveWorkbenchKey(viewKey) { + const workbench = PRODUCTION_WORKSPACE_WORKBENCHES.find(item => ( + item.views.some(view => view.key === viewKey) + )); + return workbench?.key || PRODUCTION_WORKSPACE_WORKBENCHES[0]?.key || 'dinsar_workbench'; +} + export default function ProductionWorkspace({ activeEntry = PRODUCTION_WORKSPACE_TAB, readOnly = false, onTaskStart, + apiEndpoint, + language, + foundPairs = [], + selectedPairsCount = 0, + isLoading = false, + hasEnoughRadarScenesForPlanning = false, + hasRadarSearched = false, + pairingPanel = {}, + radarPanel = {}, + pairsPanel = {}, }) { const [activeView, setActiveView] = useState(() => resolveView(activeEntry)); + const [activeWorkbench, setActiveWorkbench] = useState(() => resolveWorkbenchKey(resolveView(activeEntry))); useEffect(() => { - setActiveView(resolveView(activeEntry)); + const nextView = resolveView(activeEntry); + setActiveView(nextView); + setActiveWorkbench(resolveWorkbenchKey(nextView)); }, [activeEntry]); const activeViewMeta = useMemo( () => PRODUCTION_WORKSPACE_VIEWS.find(view => view.key === activeView) || PRODUCTION_WORKSPACE_VIEWS[0], [activeView] ); + const activeWorkbenchMeta = useMemo( + () => PRODUCTION_WORKSPACE_WORKBENCHES.find(item => item.key === activeWorkbench) || PRODUCTION_WORKSPACE_WORKBENCHES[0], + [activeWorkbench] + ); + const activeSubViews = activeWorkbenchMeta?.views || []; + + const switchWorkbench = (workbench) => { + setActiveWorkbench(workbench.key); + if (!workbench.views.some(view => view.key === activeView)) { + setActiveView(workbench.defaultView); + } + }; const handleDinsarRunQueued = taskId => { onTaskStart?.(taskId, 'D-InSAR 任务已入队，等待处理...'); @@ -68,6 +176,13 @@ export default function ProductionWorkspace({ onTaskStart?.(taskId, 'D-InSAR 产物任务已入队，等待处理...'); }; + const handleDinsarPrepareQueued = taskId => { + onTaskStart?.(taskId, 'D-InSAR生产准备任务已入队，正在处理...', { + taskType: 'COPY_DATA', + nonBlocking: true, + }); + }; + const handleSbasProductQueued = taskId => { onTaskStart?.(taskId, 'SBAS-InSAR result catalog task queued.', { taskType: 'REBUILD_SBAS_INSAR_CATALOG', @@ -84,8 +199,8 @@ export default function ProductionWorkspace({

生产管理

- 这里统一承载 D-InSAR 与 Gamma SBAS-InSAR 生产工作台。旧 ISCE2/MintPy 时序入口已停用， - SBAS 生产、速率图、质量指标与监测点曲线统一进入独立 SBAS-InSAR 页面。 + 这里统一承载 D-InSAR 配对、批次、生产准备、运行和产物管理，以及 Gamma SBAS-InSAR 生产链。 + 陆探与哨兵源数据按压缩包登记，生产时再解包到本机 Task_Pool；高分三只登记外部 SARscape 服务器复制回来的 _geo 结果。

@@ -99,24 +214,24 @@ export default function ProductionWorkspace({ }} >
-
统一入口
-
运行与产物同域编排
+
主生产链
+
D-InSAR / SBAS
- 生产运行、目录重建、产物编目全部收口到同一顶级工作区。 + D-InSAR 使用配对批次驱动；SBAS 使用 Gamma IPTA 工作流驱动。PS/旧时序入口不再作为主流程展示。
-
SBAS 当前实现
-
Gamma 独立入口
+
运行边界
+
本机 Task_Pool
- 生产链路绕开旧时序配对层，由 SBAS 页面管理栈发现、基线审核、配准与产物发布。 + 源压缩包先登记元数据，生产需要时再按需解包；D-InSAR/SBAS 不走 UNC。
-
旧链路状态
-
ISCE2/MintPy 停用
+
结果管理
+
产物 catalog
- 历史代码暂时保留兼容，生产管理不再暴露旧“时序运行/产物”页面。 + 生产结果进入 D-InSAR、SBAS 或 GF3 数据目录，后续分析从结果 catalog 读取。
@@ -130,20 +245,20 @@ export default function ProductionWorkspace({ }} >
- {PRODUCTION_WORKSPACE_VIEWS.map(view => { - const isActive = view.key === activeView; + {PRODUCTION_WORKSPACE_WORKBENCHES.map(workbench => { + const isActive = workbench.key === activeWorkbench; return ( + ); + })} +
+ + +
+
+ {activeSubViews.map(view => { + const isActive = view.key === activeView; + return ( + ); })} @@ -173,18 +324,61 @@ export default function ProductionWorkspace({
-
{activeViewMeta.label}
+
+ {activeWorkbenchMeta?.label} / {activeViewMeta.label} +
}> + {SENSOR_PRODUCTION_PLACEHOLDERS[activeView] && ( + + )} + {activeView === 'dinsar_pairing' && ( + + )} + {activeView === 'dinsar_pairs' && ( +
+ + +
+ )} + {activeView === 'dinsar_prepare' && ( + + )} {activeView === 'dinsar_runs' && ( )} - {activeView === 'sbas_insar_production' && ( + {['sbas_insar_planning', 'sbas_insar_batches', 'sbas_insar_prepare', 'sbas_insar_runs'].includes(activeView) && ( )} {activeView === 'sbas_insar_products' && ( diff --git a/frontend/src/SbasInsarProductionPanel.jsx b/frontend/src/SbasInsarProductionPanel.jsx index 77f1831..50fd6fe 100644 --- a/frontend/src/SbasInsarProductionPanel.jsx +++ b/frontend/src/SbasInsarProductionPanel.jsx @@ -1,4 +1,5 @@ -import React, { useCallback, useEffect, useMemo, useState } from 'react'; +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import MiniCoverageMap from './components/MiniCoverageMap'; import { auditSbasInsarStack, @@ -387,6 +388,40 @@ function LocationSummaryPanel({ coverage }) { ); } +function StackCoverageMiniMap({ stack, coverage, title = 'SBAS序列范围预览' }) { + const source = coverage || stack || {}; + const bbox = source.bbox || source.stack_bbox || stack?.bbox || stack?.bbox_intersection; + const intersection = source.bbox_intersection || stack?.bbox_intersection; + const sceneFootprints = source.scene_footprints_geojson || stack?.scene_footprints_geojson; + const coverageGeojson = source.geojson || stack?.geojson || sceneFootprints; + const bboxes = [ + bbox && { + bbox, + label: 'stack bbox', + color: '#2563eb', + fillOpacity: 0.05, + }, + intersection && { + bbox: intersection, + label: 'common overlap', + color: '#16a34a', + fillOpacity: 0.12, + dashArray: null, + }, + ].filter(Boolean); + const sceneCount = (sceneFootprints?.features || []).length || source.scene_bbox_count || stack?.usable_scene_count || stack?.scene_count || 0; + return ( + + ); +} + /* function UnusedSceneFootprintGeographicCoverageMap({ coverage }) { const mapElementRef = useRef(null); @@ -628,7 +663,15 @@ function UnusedGeographicCoveragePanel({ coverage }) { } */ -export default function SbasInsarProductionPanel({ readOnly = false, onTaskStart }) { +const SBAS_FOCUS_TO_SECTION = { + planning: 'sbas-planning-section', + batches: 'sbas-run-section', + prepare: 'sbas-prepare-section', + runs: 'sbas-run-section', +}; + +export default function SbasInsarProductionPanel({ readOnly = false, onTaskStart, initialFocus = 'planning' }) { + const lastAppliedFocusRef = useRef(''); const [processorMode, setProcessorMode] = useState('landsar'); const [capabilities, setCapabilities] = useState(null); const [runs, setRuns] = useState([]); @@ -686,6 +729,34 @@ export default function SbasInsarProductionPanel({ readOnly = false, onTaskStart const [workflowJob, setWorkflowJob] = useState(null); const [runDeleteLoading, setRunDeleteLoading] = useState(false); + useEffect(() => { + if (typeof window === 'undefined') return undefined; + const sectionId = SBAS_FOCUS_TO_SECTION[initialFocus] || SBAS_FOCUS_TO_SECTION.planning; + const focusToken = [ + processorMode, + initialFocus, + selectedRunId, + selectedLandsarRunId, + stackCandidates.length, + runs.length, + landsarRuns.length, + ].join(':'); + if (lastAppliedFocusRef.current === focusToken) return undefined; + lastAppliedFocusRef.current = focusToken; + const timer = window.setTimeout(() => { + document.getElementById(sectionId)?.scrollIntoView({ block: 'start', behavior: 'smooth' }); + }, 80); + return () => window.clearTimeout(timer); + }, [ + initialFocus, + landsarRuns.length, + processorMode, + runs.length, + selectedLandsarRunId, + selectedRunId, + stackCandidates.length, + ]); + const stackDiscoveryPayload = useMemo(() => { const adminRegion = stackAdminRegionQuery.trim(); const isLandsar = processorMode === 'landsar'; @@ -1387,7 +1458,7 @@ export default function SbasInsarProductionPanel({ readOnly = false, onTaskStart )}
-
+

SBAS 生产区域

@@ -1456,7 +1527,7 @@ export default function SbasInsarProductionPanel({ readOnly = false, onTaskStart
-
+