diff --git a/.env.example b/.env.example index d4c79f4..73c5477 100644 --- a/.env.example +++ b/.env.example @@ -88,7 +88,7 @@ GF3_STORAGE_DIRS=D:\GaoFen3_Pool\catalog GF3_SARSCAPE_RUNTIME_DIR=D:\GaoFen3_Pool\task_pool\sarscape_runtime GF3_SARSCAPE_WRAPPER_EXE=D:\Code\Insar_management_system_v2\third_party\GF3_L1A_To_L2_pipeline\dist\windows\gf3wrapper.exe GF3_SARSCAPE_IDLRT_PATH=C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlrt.exe -GF3_SARSCAPE_DEM_PATH=D:\DEM\COPDEM_GLO30_China_4326_DEM +GF3_SARSCAPE_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape GF3_SARSCAPE_POLARIZATIONS=HH,HV GF3_SARSCAPE_KEEP_EXTRACTED=true GF3_SARSCAPE_AUTO_STANDARDIZE=false @@ -126,9 +126,9 @@ ORBIT_QUARANTINE_DIR= # DEM / 其他数据 # ----------------------------------------------------------------------------- # Raw D-InSAR DEM source for SARscape/ENVI. Keep the base path without the .wgs84 suffix here. -IDL_DINSAR_DEM_BASE_FILE=D:\SRTM30m\SRTMDEM_RSP_SARscape -SRTM_DEM_DIR=D:\SRTM30m -GF3_GEO_DEM_PATH=D:\DEM\gf3_dem.jp2 +IDL_DINSAR_DEM_BASE_FILE=D:\DEM\SRTMDEM_RSP_SARscape +SRTM_DEM_DIR=D:\DEM +GF3_GEO_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape_global_int16.tif WATER_RESULTS_DIR=D:\WaterResult GF3_WATER_DEM_PATH= GF3_WATER_DEFAULT_CARTOGRAPHIC=true @@ -137,6 +137,14 @@ SAR_ANALYSIS_READY_ROOT=D:\production_results\sar_analysis_ready SAR_ANALYSIS_WORK_ROOT=D:\production_runtime\sar_analysis_work SAR_ANALYSIS_NODATA_VALUE=-9999 SAR_ANALYSIS_OUTPUT_COG=true +SAR_ANALYSIS_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84 +SAR_ANALYSIS_TARGET_GRID_SIZE_M=30.0 +SAR_ANALYSIS_DEM_RESOLUTION_M=30.0 +SAR_ANALYSIS_RANGE_LOOKS=6 +SAR_ANALYSIS_AZIMUTH_LOOKS=5 +SAR_ANALYSIS_SPECKLE_FILTER_ENABLED=true +SAR_ANALYSIS_SPECKLE_FILTER_METHOD=lee +SAR_ANALYSIS_SPECKLE_FILTER_SIZE=5 # ----------------------------------------------------------------------------- @@ -188,7 +196,7 @@ ISCE2_PROFILE=lt1_stripmap ISCE2_STRIPMAP_APP=/home/administrator/miniconda3/envs/insar_wsl_v1/lib/python3.11/site-packages/isce/applications/stripmapApp.py ISCE2_PIPELINE_SCRIPT= # ISCE2 should point to the prepared WGS84 DEM after the one-time conversion. -ISCE2_DEM_PATH=D:\SRTM30m\SRTMDEM_RSP_SARscape.wgs84 +ISCE2_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84 ISCE2_WORK_ROOT=D:\production_runtime\isce2_work ISCE2_OUTPUT_ROOT=D:\production_results\dinsar ISCE2_PER_TASK_TIMEOUT_SECONDS=43200 @@ -245,11 +253,11 @@ PYINT_DEM_ROOT=D:\production_runtime\pyint_dem PYINT_DEM_MODE=local_fabdem PYINT_FABDEM_ROOT= # When PYINT_DEM_MODE=prepared_file, point this to the same prepared WGS84 DEM. -PYINT_PREPARED_DEM_PATH= +PYINT_PREPARED_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84 # 0 means do not derive looks from a target output grid. PYINT_DEFAULT_TARGET_GRID_SIZE_M=0 # Source DEM resolution recorded in PyINT/Gamma run metadata. -PYINT_DEM_RESOLUTION_M=30.0 +PYINT_DEM_RESOLUTION_M=90.0 PYINT_OPENTOPO_DEM_TYPE=SRTMGL1 PYINT_OPENTOPO_API_KEY= PYINT_DEM_STRICT=true @@ -293,7 +301,7 @@ GAMMA_SBAS_TRIAL_ROOT=D:\production_runtime\gamma_ipta_trials GAMMA_SBAS_SCRIPT_TEMPLATE_ROOT=D:\Code\Insar_management_system_v2\backend\templates\gamma_sbas GAMMA_SBAS_SOURCE_ROOTS=D:\Task_Pool\source_materialized\lutan1 GAMMA_SBAS_ORBIT_ROOTS=D:\orbit_pools\envi -GAMMA_SBAS_DEM_PATH=D:\DEM\HeiLongJiang10M_DEM.tif +GAMMA_SBAS_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84 GAMMA_SBAS_DEFAULT_RLKS=8 GAMMA_SBAS_DEFAULT_AZLKS=8 GAMMA_SBAS_DEFAULT_MB_MODE=0 @@ -312,7 +320,7 @@ TIMESERIES_WSL_DISTRO=Ubuntu-24.04 TIMESERIES_ENV_NAME=insar_wsl_v1 TIMESERIES_PYTHON=/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python TIMESERIES_WORK_ROOT=D:\production_runtime\timeseries_work -TIMESERIES_DEM_PATH=D:\SRTM30m\SRTMDEM_RSP_SARscape.wgs84 +TIMESERIES_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84 TIMESERIES_ORBIT_POOL_ISCE2= TIMESERIES_EXPERIMENT_ROOT= TIMESERIES_STACK_PREP_SCRIPT= @@ -350,6 +358,8 @@ CLUSTER_TRANSFER_TIMEOUT_SECONDS=3600 CLUSTER_MAIN_SERVER_URL= # Empty means the worker can claim all job types. Remote LandSAR nodes should set: # JOB_WORKER_ALLOWED_TYPES=LANDSAR_CLUSTER_ITEM +JOB_WORKER_POLL_INTERVAL=1.0 +JOB_WORKER_CONCURRENCY=2 JOB_WORKER_ALLOWED_TYPES= @@ -413,7 +423,7 @@ DEFAULT_VLM_MODEL=qwen3-vl:30b VITE_TILE_SERVER_URL= VITE_TILE_SERVER_TOKEN=change_me VITE_APP_ORG_NAME=黑龙江省自然资源卫星应用技术中心 -VITE_APP_SYSTEM_NAME=InSAR 自动化管理系统 +VITE_APP_SYSTEM_NAME=雷达数据生产管理系统 VITE_APP_SYSTEM_TAGLINE=科研工程生产平台 # Optional override for deployed builds. Empty uses frontend/src/logo.jpg. VITE_APP_LOGO_URL= diff --git a/backend/app/config.py b/backend/app/config.py index a69df4f..cd87ea0 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -252,6 +252,14 @@ class Settings(BaseSettings): SAR_ANALYSIS_WORK_ROOT: str = "" SAR_ANALYSIS_NODATA_VALUE: float = -9999.0 SAR_ANALYSIS_OUTPUT_COG: bool = True + SAR_ANALYSIS_DEM_PATH: str = "" + SAR_ANALYSIS_TARGET_GRID_SIZE_M: float = 30.0 + SAR_ANALYSIS_DEM_RESOLUTION_M: float = 30.0 + SAR_ANALYSIS_RANGE_LOOKS: int = 6 + SAR_ANALYSIS_AZIMUTH_LOOKS: int = 5 + SAR_ANALYSIS_SPECKLE_FILTER_ENABLED: bool = True + SAR_ANALYSIS_SPECKLE_FILTER_METHOD: str = "lee" + SAR_ANALYSIS_SPECKLE_FILTER_SIZE: int = 5 SRTM_DEM_DIR: str = "" GF3_GEO_DEM_PATH: str = "" @@ -318,7 +326,7 @@ class Settings(BaseSettings): ISCE2_WSL_DISTRO: str = "Ubuntu-24.04" ISCE2_PYTHON: str = "/home/administrator/miniconda3/envs/isce2/bin/python" ISCE2_PROFILE: str = "lt1_stripmap" - ISCE2_DEM_PATH: str = "D:\\SRTM30m\\SRTMDEM_RSP_SARscape.wgs84" + ISCE2_DEM_PATH: str = "D:\\DEM\\SRTMDEM_RSP_SARscape.wgs84" ISCE2_WORK_ROOT: str = "" ISCE2_OUTPUT_ROOT: str = "" ISCE2_PER_TASK_TIMEOUT_SECONDS: int = 43200 @@ -355,7 +363,7 @@ class Settings(BaseSettings): PYINT_DEM_MODE: str = "local_fabdem" PYINT_FABDEM_ROOT: str = "" PYINT_PREPARED_DEM_PATH: str = "" - PYINT_DEM_RESOLUTION_M: float = 30.0 + PYINT_DEM_RESOLUTION_M: float = 90.0 PYINT_OPENTOPO_DEM_TYPE: str = "SRTMGL1" PYINT_OPENTOPO_API_KEY: str = "" PYINT_DEM_STRICT: bool = True @@ -423,6 +431,7 @@ class Settings(BaseSettings): JOB_WORKER_STALE_RECOVER_INTERVAL: float = 15.0 JOB_WORKER_STALE_RUNNING_SECONDS: int = 300 JOB_WORKER_HEARTBEAT_INTERVAL: float = 5.0 + JOB_WORKER_CONCURRENCY: int = 1 JOB_WORKER_ALLOWED_TYPES: str = "" TIMESERIES_ENABLED: bool = False @@ -500,6 +509,39 @@ class Settings(BaseSettings): "SAR_ANALYSIS_NODATA_VALUE", float(self.SAR_ANALYSIS_NODATA_VALUE if self.SAR_ANALYSIS_NODATA_VALUE is not None else -9999.0), ) + if not self.SAR_ANALYSIS_DEM_PATH: + object.__setattr__( + self, + "SAR_ANALYSIS_DEM_PATH", + self.GAMMA_SBAS_DEM_PATH + or self.PYINT_PREPARED_DEM_PATH + or self.ISCE2_DEM_PATH + or self.IDL_DINSAR_DEM_BASE_FILE, + ) + object.__setattr__( + self, + "SAR_ANALYSIS_TARGET_GRID_SIZE_M", + max(1.0, float(self.SAR_ANALYSIS_TARGET_GRID_SIZE_M or 30.0)), + ) + object.__setattr__( + self, + "SAR_ANALYSIS_DEM_RESOLUTION_M", + max(1.0, float(self.SAR_ANALYSIS_DEM_RESOLUTION_M or self.SAR_ANALYSIS_TARGET_GRID_SIZE_M or 30.0)), + ) + object.__setattr__(self, "SAR_ANALYSIS_RANGE_LOOKS", max(1, int(self.SAR_ANALYSIS_RANGE_LOOKS or 6))) + object.__setattr__(self, "SAR_ANALYSIS_AZIMUTH_LOOKS", max(1, int(self.SAR_ANALYSIS_AZIMUTH_LOOKS or 5))) + filter_method = str(self.SAR_ANALYSIS_SPECKLE_FILTER_METHOD or "lee").strip().lower() + if filter_method in {"", "0", "false", "none", "off", "disabled", "no"}: + filter_method = "none" + elif filter_method not in {"lee"}: + filter_method = "lee" + if not self.SAR_ANALYSIS_SPECKLE_FILTER_ENABLED: + filter_method = "none" + filter_size = max(3, min(99, int(self.SAR_ANALYSIS_SPECKLE_FILTER_SIZE or 5))) + if filter_size % 2 == 0: + filter_size += 1 + object.__setattr__(self, "SAR_ANALYSIS_SPECKLE_FILTER_METHOD", filter_method) + object.__setattr__(self, "SAR_ANALYSIS_SPECKLE_FILTER_SIZE", filter_size) if not self.SRTM_DEM_DIR: object.__setattr__(self, "SRTM_DEM_DIR", os.path.join(backend_dir, "dem_data")) object.__setattr__(self, "ASSET_SCAN_PARSE_WORKERS", max(1, int(self.ASSET_SCAN_PARSE_WORKERS or 1))) @@ -514,6 +556,7 @@ class Settings(BaseSettings): max(60, int(self.ASSET_SCAN_PARSE_TIMEOUT_SECONDS or 600)), ) object.__setattr__(self, "ASSET_SCAN_DB_BATCH_SIZE", max(1, int(self.ASSET_SCAN_DB_BATCH_SIZE or 1))) + object.__setattr__(self, "JOB_WORKER_CONCURRENCY", max(1, int(self.JOB_WORKER_CONCURRENCY or 1))) if not self.GF3_ARCHIVE_SOURCE_DIRS: object.__setattr__( self, @@ -725,7 +768,7 @@ class Settings(BaseSettings): if pyint_dem_mode not in {"local_fabdem", "opentopo", "prepared_file"}: pyint_dem_mode = "local_fabdem" object.__setattr__(self, "PYINT_DEM_MODE", pyint_dem_mode) - object.__setattr__(self, "PYINT_DEM_RESOLUTION_M", max(0.1, float(self.PYINT_DEM_RESOLUTION_M or 30.0))) + object.__setattr__(self, "PYINT_DEM_RESOLUTION_M", max(0.1, float(self.PYINT_DEM_RESOLUTION_M or 90.0))) object.__setattr__( self, "PYINT_UNWRAP_COH_THRESHOLD", @@ -1291,6 +1334,7 @@ def validate_runtime_config() -> dict[str, Any]: _check_path(label="WATER_RESULTS_DIR", value=settings.WATER_RESULTS_DIR, errors=errors, warnings=warnings, expect_file=False) _check_path(label="SAR_ANALYSIS_READY_ROOT", value=settings.SAR_ANALYSIS_READY_ROOT, errors=errors, warnings=warnings, expect_file=False) _check_path(label="SAR_ANALYSIS_WORK_ROOT", value=settings.SAR_ANALYSIS_WORK_ROOT, errors=errors, warnings=warnings, expect_file=False) + _check_path(label="SAR_ANALYSIS_DEM_PATH", value=settings.SAR_ANALYSIS_DEM_PATH, errors=errors, warnings=warnings, expect_file=True) _check_path(label="MONITOR_ORBIT_DIR", value=settings.MONITOR_ORBIT_DIR, errors=errors, warnings=warnings, expect_file=False) for label, value in ( ("SOURCE_PRODUCT_DIRS", settings.SOURCE_PRODUCT_DIRS), diff --git a/backend/app/copier.py b/backend/app/copier.py index 920d7f2..a782d6e 100644 --- a/backend/app/copier.py +++ b/backend/app/copier.py @@ -305,6 +305,10 @@ def _normalize_source_bundle_archive_path(source_path: str) -> str: def _safe_archive_member_name(member_name: str, archive_path: str) -> str: name = str(member_name or "").replace("\\", "/").strip("/") + while name.startswith("./"): + name = name[2:] + if name in {"", "."}: + return "" if not name or name.startswith("../") or "/../" in f"/{name}/": raise ValueError(f"Unsafe archive member path in {archive_path}: {member_name}") if os.path.isabs(name) or os.path.splitdrive(name)[0]: @@ -318,6 +322,11 @@ def _extract_archive_to_dir(archive_path: str, dest_dir: str) -> int: with zipfile.ZipFile(archive_path) as zip_obj: for info in zip_obj.infolist(): rel_name = _safe_archive_member_name(info.filename, archive_path) + if not rel_name: + if info.is_dir(): + os.makedirs(dest_dir, exist_ok=True) + continue + raise ValueError(f"Unsafe ZIP member path: {info.filename}") dest_path = os.path.abspath(os.path.join(dest_dir, rel_name)) if not dest_path.startswith(os.path.abspath(dest_dir) + os.sep): raise ValueError(f"Unsafe ZIP member path: {info.filename}") @@ -335,6 +344,11 @@ def _extract_archive_to_dir(archive_path: str, dest_dir: str) -> int: with tarfile.open(archive_path, "r:*") as tar_obj: for member in tar_obj: rel_name = _safe_archive_member_name(member.name, archive_path) + if not rel_name: + if member.isdir(): + os.makedirs(dest_dir, exist_ok=True) + continue + raise ValueError(f"Unsafe TAR member path: {member.name}") dest_path = os.path.abspath(os.path.join(dest_dir, rel_name)) if not dest_path.startswith(os.path.abspath(dest_dir) + os.sep): raise ValueError(f"Unsafe TAR member path: {member.name}") diff --git a/backend/app/isce2_pipeline/lt1_input_resolver.py b/backend/app/isce2_pipeline/lt1_input_resolver.py index 33f047e..fe71204 100644 --- a/backend/app/isce2_pipeline/lt1_input_resolver.py +++ b/backend/app/isce2_pipeline/lt1_input_resolver.py @@ -31,12 +31,12 @@ PathTransform = Callable[[str | Path], Path] DEFAULT_WINDOWS_DEM_CANDIDATES = ( - r"D:\SRTM30m\SRTMDEM_RSP_SARscape.wgs84", - r"D:\SRTM30m\SRTMDEM_RSP_SARscape", + r"D:\DEM\SRTMDEM_RSP_SARscape.wgs84", + r"D:\DEM\SRTMDEM_RSP_SARscape", ) DEFAULT_WSL_DEM_CANDIDATES = ( - "/mnt/d/SRTM30m/SRTMDEM_RSP_SARscape.wgs84", - "/mnt/d/SRTM30m/SRTMDEM_RSP_SARscape", + "/mnt/d/DEM/SRTMDEM_RSP_SARscape.wgs84", + "/mnt/d/DEM/SRTMDEM_RSP_SARscape", ) DEFAULT_WINDOWS_ORBIT_POOL_CANDIDATES = (r"D:\orbit_pools\isce2",) DEM_SIDECAR_PROPERTY_NAMES = ("file_name", "metadata_location", "extra_file_name") diff --git a/backend/app/models/schemas.py b/backend/app/models/schemas.py index eb8b0c7..e628218 100644 --- a/backend/app/models/schemas.py +++ b/backend/app/models/schemas.py @@ -231,6 +231,10 @@ class RadarData(BaseModel): stack_selection_mode: Optional[str] = None stack_network_edge_count: Optional[int] = None stack_network_warnings: Optional[List[str]] = None + lt1_image_produced: bool = False + lt1_image_product: Optional[Dict[str, Any]] = None + lt1_landsar_produced: bool = False + lt1_landsar_product: Optional[Dict[str, Any]] = None model_config = ConfigDict(from_attributes=True) diff --git a/backend/app/pyint_pipeline/run_gamma_scene_preprocess.py b/backend/app/pyint_pipeline/run_gamma_scene_preprocess.py index d51e682..d0ed7c8 100644 --- a/backend/app/pyint_pipeline/run_gamma_scene_preprocess.py +++ b/backend/app/pyint_pipeline/run_gamma_scene_preprocess.py @@ -2,7 +2,7 @@ """Single-scene Gamma preprocessing to analysis-ready GeoTIFF. The script is intentionally narrower than the full PyINT DInSAR pipeline: -LT source product -> Gamma SLC -> multilook amplitude -> geocode -> GeoTIFF. +LT source product -> Gamma SLC -> multilook amplitude -> geocode -> speckle-filtered dB GeoTIFF. It is executed inside WSL by backend.app.services.lt_gamma_scene_service. """ from __future__ import annotations @@ -27,6 +27,10 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--pyint-home", required=True) parser.add_argument("--dem-root", required=True) parser.add_argument("--prepared-dem-path", default="") + parser.add_argument("--dem-resolution-m", type=float, default=30.0) + parser.add_argument("--target-grid-size-m", type=float, default=30.0) + parser.add_argument("--dem-lat-ovr", type=float, default=0.0) + parser.add_argument("--dem-lon-ovr", type=float, default=0.0) parser.add_argument("--project-name", required=True) parser.add_argument("--date", required=True) parser.add_argument("--satellite-family", default="LT1") @@ -35,9 +39,175 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--geo-interp", default="1") parser.add_argument("--nodata-value", type=float, default=-9999.0) parser.add_argument("--to-db", action="store_true") + parser.add_argument("--speckle-filter-method", default="lee") + parser.add_argument("--speckle-filter-size", type=int, default=5) + parser.add_argument("--speckle-filter-enl", type=float, default=0.0) return parser.parse_args() +def clamp_float(value: float, minimum: float, maximum: float) -> float: + if not math.isfinite(value): + return minimum + return min(maximum, max(minimum, float(value))) + + +def format_gamma_number(value: float) -> str: + text = f"{float(value):.6f}".rstrip("0").rstrip(".") + return text or "0" + + +def calculate_dem_oversampling( + *, + dem_resolution_m: float, + target_grid_size_m: float, + dem_lat_ovr: float, + dem_lon_ovr: float, +) -> dict[str, Any]: + dem_resolution = float(dem_resolution_m or 30.0) + target_grid = float(target_grid_size_m or 30.0) + if dem_resolution <= 0: + dem_resolution = 30.0 + if target_grid <= 0: + target_grid = dem_resolution + + derived = dem_resolution / target_grid + lat_factor = clamp_float(float(dem_lat_ovr or derived), 0.25, 16.0) + lon_factor = clamp_float(float(dem_lon_ovr or derived), 0.25, 16.0) + actual_grid = dem_resolution / ((lat_factor + lon_factor) / 2.0) + return { + "dem_resolution_m": dem_resolution, + "target_grid_size_m": target_grid, + "derived_oversampling": derived, + "dem_lat_ovr": lat_factor, + "dem_lon_ovr": lon_factor, + "actual_grid_size_m": actual_grid, + } + + +def meters_per_degree_lon(latitude_deg: float) -> float: + latitude_rad = math.radians(float(latitude_deg)) + return max(1.0, 111_320.0 * math.cos(latitude_rad)) + + +def inspect_prepared_dem_path(path_text: str) -> dict[str, str]: + text = str(path_text or "").strip() + if not text: + return {"kind": "", "direct_dem_path": "", "source_dem_path": ""} + + path = Path(text) + try: + resolved = path.resolve() + except Exception: + resolved = path + + if resolved.is_file() and Path(str(resolved) + ".par").is_file(): + return {"kind": "gamma_ready", "direct_dem_path": str(resolved), "source_dem_path": ""} + if resolved.is_file(): + return {"kind": "source_dem", "direct_dem_path": "", "source_dem_path": str(resolved)} + return {"kind": "", "direct_dem_path": "", "source_dem_path": str(resolved)} + + +def read_slc_bbox( + pyint_home: Path, + slc_par: Path, + env: dict[str, str], + *, + margin_deg: float = 0.1, +) -> tuple[float, float, float, float]: + result = subprocess.run( + ["SLC_corners", str(slc_par)], + cwd=str(pyint_home), + env=env, + text=True, + capture_output=True, + check=False, + ) + if result.returncode != 0: + detail = (result.stderr or result.stdout or "").strip() + raise RuntimeError(f"SLC_corners failed rc={result.returncode}: {detail}") + lines = result.stdout.splitlines() + if len(lines) < 10: + raise RuntimeError(f"Unexpected SLC_corners output for {slc_par}") + lat_line = lines[8].rstrip() + lon_line = lines[9].rstrip() + min_lat = float(lat_line.split(":")[1].split(" max. ")[0]) + max_lat = float(lat_line.split(":")[2]) + min_lon = float(lon_line.split(":")[1].split(" max. ")[0]) + max_lon = float(lon_line.split(":")[2]) + margin = max(0.0, float(margin_deg or 0.0)) + return min_lon - margin, min_lat - margin, max_lon + margin, max_lat + margin + + +def build_gamma_dem_from_source( + *, + source_dem: Path, + target_base: Path, + slc_par: Path, + pyint_home: Path, + log_dir: Path, + env: dict[str, str], +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + west, south, east, north = read_slc_bbox(pyint_home, slc_par, env) + log_dir.mkdir(parents=True, exist_ok=True) + source_open = Path(str(source_dem) + ".vrt") if Path(str(source_dem) + ".vrt").is_file() else source_dem + clipped_tif = target_base.with_suffix(".prepared_source_clip.tif") + clipped_aux = Path(str(clipped_tif) + ".aux.xml") + commands: list[dict[str, Any]] = [] + commands.append(run_logged( + [ + "gdal_translate", + "-projwin", + str(west), + str(north), + str(east), + str(south), + "-of", + "GTiff", + str(source_open), + str(clipped_tif), + ], + cwd=target_base.parent, + env=env, + log_dir=log_dir, + stage="clip_prepared_dem", + )) + commands.append(run_logged( + [ + "makedem.py", + "-d", + str(clipped_tif), + "-p", + "gamma", + "-o", + str(target_base), + ], + cwd=target_base.parent, + env=env, + log_dir=log_dir, + stage="convert_prepared_dem", + )) + for path in (clipped_tif, clipped_aux): + try: + if path.exists(): + path.unlink() + except OSError: + pass + dem_path = Path(str(target_base) + ".dem") + dem_par_path = Path(str(target_base) + ".dem.par") + if not dem_path.is_file() or not dem_par_path.is_file(): + raise RuntimeError(f"Prepared source DEM conversion did not create Gamma DEM: {dem_path}") + return ( + { + "kind": "source_dem_converted", + "source_dem_path": str(source_dem), + "source_open_path": str(source_open), + "gamma_dem_path": str(dem_path), + "bbox": {"west": west, "south": south, "east": east, "north": north}, + }, + commands, + ) + + def run_logged(command: list[str], *, cwd: Path, env: dict[str, str], log_dir: Path, stage: str) -> dict[str, Any]: log_dir.mkdir(parents=True, exist_ok=True) stdout_path = log_dir / f"{stage}.stdout.log" @@ -74,6 +244,48 @@ def read_gamma_par(path: Path, key: str) -> str: raise KeyError(f"Cannot read {key} from {path}") +def calculate_dem_oversampling_from_gamma_dem( + *, + dem_par_path: Path, + target_grid_size_m: float, + explicit_dem_lat_ovr: float, + explicit_dem_lon_ovr: float, +) -> dict[str, Any]: + target_grid = max(1.0, float(target_grid_size_m or 30.0)) + post_lat_deg = abs(float(read_gamma_par(dem_par_path, "post_lat"))) + post_lon_deg = abs(float(read_gamma_par(dem_par_path, "post_lon"))) + corner_lat = float(read_gamma_par(dem_par_path, "corner_lat")) + nlines = int(float(read_gamma_par(dem_par_path, "nlines"))) + center_lat = corner_lat - (post_lat_deg * max(0, nlines - 1) / 2.0) + + lat_spacing_m = post_lat_deg * 111_320.0 + lon_spacing_m = post_lon_deg * meters_per_degree_lon(center_lat) + derived_lat = lat_spacing_m / target_grid + derived_lon = lon_spacing_m / target_grid + lat_factor = clamp_float(float(explicit_dem_lat_ovr or derived_lat), 0.25, 16.0) + lon_factor = clamp_float(float(explicit_dem_lon_ovr or derived_lon), 0.25, 16.0) + actual_lat_m = lat_spacing_m / lat_factor + actual_lon_m = lon_spacing_m / lon_factor + return { + "dem_resolution_m": (lat_spacing_m + lon_spacing_m) / 2.0, + "target_grid_size_m": target_grid, + "derived_oversampling": (derived_lat + derived_lon) / 2.0, + "derived_dem_lat_ovr": derived_lat, + "derived_dem_lon_ovr": derived_lon, + "dem_lat_ovr": lat_factor, + "dem_lon_ovr": lon_factor, + "actual_grid_size_m": (actual_lat_m + actual_lon_m) / 2.0, + "actual_lat_grid_size_m": actual_lat_m, + "actual_lon_grid_size_m": actual_lon_m, + "source_dem_post_lat_deg": post_lat_deg, + "source_dem_post_lon_deg": post_lon_deg, + "source_dem_lat_spacing_m": lat_spacing_m, + "source_dem_lon_spacing_m": lon_spacing_m, + "source_dem_center_lat": center_lat, + "source_dem_par_path": str(dem_par_path), + } + + def discover_lt_inputs(source_path: Path, date: str) -> list[Path]: patterns = [f"LT1*{date}*.tar.gz", f"LT1*{date}*.tiff", f"LT1*{date}*.tif"] if source_path.is_file(): @@ -119,15 +331,18 @@ def write_template( range_looks: int, azimuth_looks: int, geo_interp: str, - prepared_dem_path: str, + dem_path: str, + prepared_dem_source: str, + dem_oversampling: dict[str, Any], ) -> None: lines = [ "satelite = LT", f"masterDate = {date}", f"range_looks = {range_looks}", f"azimuth_looks = {azimuth_looks}", - "dem_lat_ovr = 0.5", - "dem_lon_ovr = 0.5", + f"target_grid_size_m = {format_gamma_number(float(dem_oversampling.get('target_grid_size_m') or 0.0))}", + f"dem_lat_ovr = {format_gamma_number(float(dem_oversampling.get('dem_lat_ovr') or 1.0))}", + f"dem_lon_ovr = {format_gamma_number(float(dem_oversampling.get('dem_lon_ovr') or 1.0))}", "Simphase_rpos = -", "Simphase_azpos = -", "Simphase_rwin = 256", @@ -135,24 +350,153 @@ def write_template( "Simphase_thresh = -", f"geo_interp = {geo_interp}", ] - dem = str(prepared_dem_path or "").strip() + dem = str(dem_path or "").strip() if dem and Path(dem).is_file() and Path(dem + ".par").is_file(): lines.append(f"DEM = {dem}") + source = str(prepared_dem_source or "").strip() + if source: + lines.append(f"prepared_dem_source = {source}") template_path.parent.mkdir(parents=True, exist_ok=True) template_path.write_text("\n".join(lines) + "\n", encoding="utf-8") -def convert_to_db_geotiff(source_tif: Path, target_tif: Path, nodata_value: float) -> dict[str, Any]: +def normalize_speckle_filter_method(method: str) -> str: + text = str(method or "").strip().lower() + if text in {"", "0", "false", "none", "off", "disabled", "no"}: + return "none" + if text in {"lee", "lee_filter"}: + return "lee" + raise ValueError(f"Unsupported speckle filter method: {method}") + + +def normalize_speckle_filter_size(size: int | float | str) -> int: + try: + value = int(float(size or 5)) + except Exception: + value = 5 + value = max(3, min(99, value)) + if value % 2 == 0: + value += 1 + return value + + +def moving_sum_axis(values: Any, size: int, axis: int) -> Any: + import numpy as np + + radius = size // 2 + pad_width = [(0, 0)] * values.ndim + pad_width[axis] = (radius, size - 1 - radius) + padded = np.pad(values, pad_width, mode="edge") + cumulative = np.cumsum(padded, axis=axis, dtype="float64") + zero_shape = list(cumulative.shape) + zero_shape[axis] = 1 + cumulative = np.concatenate([np.zeros(zero_shape, dtype="float64"), cumulative], axis=axis) + length = values.shape[axis] + start = np.arange(0, length) + end = np.arange(size, size + length) + return np.take(cumulative, end, axis=axis) - np.take(cumulative, start, axis=axis) + + +def box_sum(values: Any, size: int) -> Any: + return moving_sum_axis(moving_sum_axis(values, size, axis=0), size, axis=1) + + +def local_power_stats(data: Any, valid: Any, window_size: int) -> tuple[Any, Any, Any]: + import numpy as np + + values = np.where(valid, data, 0.0).astype("float64", copy=False) + weights = valid.astype("float64", copy=False) + count = box_sum(weights, window_size) + power_sum = box_sum(values, window_size) + power_sq_sum = box_sum(values * values, window_size) + mean = np.divide(power_sum, count, out=np.zeros_like(power_sum), where=count > 0) + mean_sq = np.divide(power_sq_sum, count, out=np.zeros_like(power_sq_sum), where=count > 0) + variance = np.maximum(mean_sq - mean * mean, 0.0) + valid_fraction = count / float(window_size * window_size) + return mean, variance, valid_fraction + + +def apply_speckle_filter_power( + data: Any, + invalid: Any, + *, + method: str, + window_size: int, + equivalent_number_of_looks: float = 0.0, +) -> tuple[Any, dict[str, Any]]: + import numpy as np + + normalized_method = normalize_speckle_filter_method(method) + normalized_size = normalize_speckle_filter_size(window_size) + record: dict[str, Any] = { + "enabled": normalized_method != "none", + "method": normalized_method, + "window_size": normalized_size, + "equivalent_number_of_looks": float(equivalent_number_of_looks or 0.0), + "domain": "linear_power", + } + if normalized_method == "none": + return data, record + + valid = ~invalid + valid_count = int(np.count_nonzero(valid)) + record["valid_pixels"] = valid_count + if valid_count == 0: + record["enabled"] = False + record["warning"] = "no valid positive pixels to filter" + return data, record + + local_mean, local_variance, valid_fraction = local_power_stats(data, valid, normalized_size) + stats_mask = valid & np.isfinite(local_variance) & np.isfinite(local_mean) & (valid_fraction > 0.0) + enl = float(equivalent_number_of_looks or 0.0) + if math.isfinite(enl) and enl > 0: + noise_variance = np.maximum((local_mean * local_mean) / enl, 0.0) + weight = np.divide( + np.maximum(local_variance - noise_variance, 0.0), + local_variance, + out=np.zeros_like(local_variance), + where=local_variance > 0, + ) + record["noise_variance_model"] = "local_mean_squared_over_enl" + else: + noise_samples = local_variance[stats_mask] + global_noise_variance = float(np.nanmedian(noise_samples)) if noise_samples.size else 0.0 + if not math.isfinite(global_noise_variance) or global_noise_variance <= 0: + record["enabled"] = False + record["warning"] = "local variance estimate is zero; kept unfiltered power values" + return data, record + noise_variance = global_noise_variance + weight = np.divide( + local_variance, + local_variance + noise_variance, + out=np.zeros_like(local_variance), + where=(local_variance + noise_variance) > 0, + ) + record["noise_variance"] = global_noise_variance + record["noise_variance_model"] = "global_median_local_variance" + filtered = local_mean + weight * (data.astype("float64", copy=False) - local_mean) + filtered = np.where(np.isfinite(filtered) & (filtered > 0), filtered, data) + output = data.astype("float32", copy=True) + output[stats_mask] = filtered[stats_mask].astype("float32") + return output, record + + +def convert_to_db_geotiff( + source_tif: Path, + target_tif: Path, + nodata_value: float, + *, + speckle_filter_method: str = "none", + speckle_filter_size: int = 5, + speckle_filter_enl: float = 0.0, +) -> dict[str, Any]: + filter_method = normalize_speckle_filter_method(speckle_filter_method) + filter_size = normalize_speckle_filter_size(speckle_filter_size) try: import numpy as np import rasterio except Exception as exc: - shutil.copy2(source_tif, target_tif) - return { - "target": str(target_tif), - "backscatter_unit": "gamma_mli_power", - "warning": f"rasterio/numpy unavailable; kept power values: {exc}", - } + raise RuntimeError(f"rasterio/numpy unavailable; cannot create filtered dB GeoTIFF: {exc}") from exc with rasterio.open(source_tif) as src: data = src.read(1).astype("float32") @@ -163,14 +507,23 @@ def convert_to_db_geotiff(source_tif: Path, target_tif: Path, nodata_value: floa if src_nodata is not None: invalid |= data == src_nodata invalid |= data <= 0 + filtered_data, speckle_filter = apply_speckle_filter_power( + data, + invalid, + method=filter_method, + window_size=filter_size, + equivalent_number_of_looks=float(speckle_filter_enl or 0.0), + ) + invalid |= ~np.isfinite(filtered_data) + invalid |= filtered_data <= 0 db_data = np.full(data.shape, nodata_value, dtype="float32") - db_data[~invalid] = (10.0 * np.log10(data[~invalid])).astype("float32") + db_data[~invalid] = (10.0 * np.log10(filtered_data[~invalid])).astype("float32") profile.update(dtype="float32", count=1, nodata=nodata_value, compress="deflate") target_tif.parent.mkdir(parents=True, exist_ok=True) with rasterio.open(target_tif, "w", **profile) as dst: dst.write(db_data, 1) - return {"target": str(target_tif), "backscatter_unit": "gamma_mli_db"} + return {"target": str(target_tif), "backscatter_unit": "gamma_mli_db", "speckle_filter": speckle_filter} def main() -> int: @@ -201,6 +554,18 @@ def main() -> int: env["PATH"] = f"{pyint_home / 'pyint'}:{env.get('PATH', '')}" staged_inputs = stage_lt_inputs(source_path, download_dir, date) + dem_oversampling = calculate_dem_oversampling( + dem_resolution_m=float(args.dem_resolution_m or 30.0), + target_grid_size_m=float(args.target_grid_size_m or 30.0), + dem_lat_ovr=float(args.dem_lat_ovr or 0.0), + dem_lon_ovr=float(args.dem_lon_ovr or 0.0), + ) + prepared_dem = inspect_prepared_dem_path(args.prepared_dem_path) + if not prepared_dem.get("kind"): + raise RuntimeError(f"A prepared DEM is required for LT analysis GeoTIFF production: {args.prepared_dem_path}") + + dem_path = prepared_dem.get("direct_dem_path") or "" + prepared_dem_conversion: dict[str, Any] | None = None template_path = template_dir / f"{project_name}.template" write_template( template_path=template_path, @@ -208,7 +573,9 @@ def main() -> int: range_looks=max(1, int(args.range_looks)), azimuth_looks=max(1, int(args.azimuth_looks)), geo_interp=str(args.geo_interp or "1"), - prepared_dem_path=args.prepared_dem_path, + dem_path=dem_path, + prepared_dem_source=str(prepared_dem.get("source_dem_path") or ""), + dem_oversampling=dem_oversampling, ) commands: list[dict[str, Any]] = [] @@ -221,6 +588,44 @@ def main() -> int: stage="down2slc_lt1", ) ) + + if prepared_dem.get("kind") == "source_dem": + slc_par = project_dir / "SLC" / date / f"{date}.slc.par" + if not slc_par.is_file(): + raise FileNotFoundError(f"Gamma SLC parameter file missing before DEM conversion: {slc_par}") + dem_target_base = dem_root / project_name / project_name + dem_target_base.parent.mkdir(parents=True, exist_ok=True) + prepared_dem_conversion, dem_commands = build_gamma_dem_from_source( + source_dem=Path(str(prepared_dem.get("source_dem_path"))), + target_base=dem_target_base, + slc_par=slc_par, + pyint_home=pyint_home, + log_dir=log_dir, + env=env, + ) + commands.extend(dem_commands) + dem_path = str(Path(str(dem_target_base) + ".dem")) + + dem_par_path = Path(str(dem_path) + ".par") if dem_path else Path() + if dem_path and dem_par_path.is_file(): + dem_oversampling = calculate_dem_oversampling_from_gamma_dem( + dem_par_path=dem_par_path, + target_grid_size_m=float(args.target_grid_size_m or 30.0), + explicit_dem_lat_ovr=float(args.dem_lat_ovr or 0.0), + explicit_dem_lon_ovr=float(args.dem_lon_ovr or 0.0), + ) + + write_template( + template_path=template_path, + date=date, + range_looks=max(1, int(args.range_looks)), + azimuth_looks=max(1, int(args.azimuth_looks)), + geo_interp=str(args.geo_interp or "1"), + dem_path=dem_path, + prepared_dem_source=str(prepared_dem.get("source_dem_path") or ""), + dem_oversampling=dem_oversampling, + ) + commands.append( run_logged( [sys.executable, str(pyint_home / "pyint" / "generate_rdc_dem.py"), project_name], @@ -286,9 +691,25 @@ def main() -> int: raise RuntimeError(f"data2geotiff did not create output: {power_tif}") final_tif = output_dir / "analysis_ready.tif" - conversion = convert_to_db_geotiff(power_tif, final_tif, float(args.nodata_value)) if args.to_db else { + speckle_filter_config = { + "method": normalize_speckle_filter_method(args.speckle_filter_method), + "window_size": normalize_speckle_filter_size(args.speckle_filter_size), + } + conversion = convert_to_db_geotiff( + power_tif, + final_tif, + float(args.nodata_value), + speckle_filter_method=args.speckle_filter_method, + speckle_filter_size=args.speckle_filter_size, + speckle_filter_enl=float(args.speckle_filter_enl or (range_looks * max(1, int(args.azimuth_looks)))), + ) if args.to_db else { "target": str(final_tif), "backscatter_unit": "gamma_mli_power", + "speckle_filter": { + "enabled": False, + **speckle_filter_config, + "warning": "not applied because --to-db was disabled", + }, } if not args.to_db: shutil.copy2(power_tif, final_tif) @@ -313,6 +734,24 @@ def main() -> int: "geo_amp": str(geo_amp), }, "looks": {"range": range_looks, "azimuth": max(1, int(args.azimuth_looks))}, + "speckle_filter": conversion.get("speckle_filter"), + "processing_steps": { + "multilook": { + "enabled": True, + "range_looks": range_looks, + "azimuth_looks": max(1, int(args.azimuth_looks)), + }, + "geocode": {"enabled": True, "interpolation": str(args.geo_interp or "1")}, + "speckle_filter": conversion.get("speckle_filter"), + "db_conversion": {"enabled": bool(args.to_db), "unit": conversion.get("backscatter_unit")}, + }, + "dem": { + "prepared_dem_path": str(args.prepared_dem_path or "").strip(), + "prepared_dem_kind": prepared_dem.get("kind"), + "gamma_dem_path": dem_path, + "conversion": prepared_dem_conversion, + "oversampling": dem_oversampling, + }, "commands": commands, "conversion": conversion, } diff --git a/backend/app/routers/__init__.py b/backend/app/routers/__init__.py index 11ce453..eae33af 100644 --- a/backend/app/routers/__init__.py +++ b/backend/app/routers/__init__.py @@ -14,6 +14,7 @@ from . import ( hazard, health, idl, + landsar_lt1_production, license, logs, monitor, @@ -55,6 +56,7 @@ def include_all_routers(router: APIRouter) -> None: router.include_router(dinsar.router) router.include_router(dinsar_products.router) router.include_router(dinsar_production.router) + router.include_router(landsar_lt1_production.router) router.include_router(sbas_insar_production.router) router.include_router(sbas_insar_products.router) router.include_router(timeseries_production.router) diff --git a/backend/app/routers/idl.py b/backend/app/routers/idl.py index e5d2f2f..4e991bb 100644 --- a/backend/app/routers/idl.py +++ b/backend/app/routers/idl.py @@ -16,7 +16,6 @@ from .dependencies import _require_admin, _get_current_user, _validate_export_pa from ..models import AuthUserORM from ..services import envi_service from ..services.job_queue_service import job_queue_service -from ..services.result_catalog_service import result_catalog_service from ..services.task_service import task_service router = APIRouter() @@ -61,38 +60,6 @@ class SarscapeSbasInspectRequest(BaseModel): timeout_seconds: Optional[int] = Field(default=120, ge=10, le=600) -def _normalize_existing_dir(path: Optional[str]) -> Optional[str]: - text = str(path or "").strip() - if not text: - return None - normalized = os.path.normpath(os.path.abspath(text)) - if not os.path.isdir(normalized): - return None - return normalized - - -def _dedupe_publish_roots(*paths: Optional[str]) -> list[str]: - ordered: list[str] = [] - for raw_path in paths: - normalized = _normalize_existing_dir(raw_path) - if not normalized: - continue - - if any( - normalized == existing or normalized.startswith(existing + os.sep) - for existing in ordered - ): - continue - - ordered = [ - existing - for existing in ordered - if not existing.startswith(normalized + os.sep) - ] - ordered.append(normalized) - return ordered - - # --------------------------------------------------------------------------- # Job queue helper # --------------------------------------------------------------------------- @@ -292,7 +259,7 @@ async def get_task_overview_endpoint( return result -@router.post("/idl/extract-disp") +@router.post("/idl/extract-disp", status_code=202) async def extract_disp_endpoint( request: ExtractDispRequest, admin_user: AuthUserORM = Depends(_require_admin), @@ -303,52 +270,29 @@ async def extract_disp_endpoint( if request.dest_dir: _validate_export_path(request.dest_dir, "dest_dir") try: - result = await asyncio.to_thread( - envi_service.extract_disp_results, request.root_dir, request.dest_dir + payload = { + "root_dir": request.root_dir, + "dest_dir": request.dest_dir, + } + task_id = await task_service.create_task( + "EXTRACT_DINSAR_PRODUCTS", + "D-InSAR 结果提取与登记", + params=payload, + db=db, ) + job_id = await job_queue_service.create_job( + "EXTRACT_DINSAR_PRODUCTS", + payload=payload, + task_id=task_id, + db=db, + ) + await db.commit() except ValueError as exc: - raise HTTPException(status_code=400, detail=str(exc)) from exc + raise HTTPException(status_code=409, detail=str(exc)) from exc - publish_roots = _dedupe_publish_roots(result.get("target_dir")) - catalog_status: Dict[str, Any] = { - "attempted": False, - "status": "skipped", - "source_directories": publish_roots, - "message": "catalog publish skipped", + return { + "queued": True, + "task_id": task_id, + "job_id": job_id, + "message": "D-InSAR 结果提取与登记任务已入队", } - if publish_roots: - try: - catalog_status["attempted"] = True - publish_result = await result_catalog_service.publish_from_sources( - db, - publish_roots, - ) - rebuild_result = None - if int(publish_result.get("processed", 0) or 0) > 0: - rebuild_result = await result_catalog_service.rebuild_catalog( - db, - full_rebuild=True, - ) - catalog_status = { - "attempted": True, - "status": "ok", - "source_directories": publish_roots, - "publish": publish_result, - "rebuild": rebuild_result, - "message": ( - "catalog published and rebuilt" - if rebuild_result is not None - else "catalog publish finished with no rebuild needed" - ), - } - except Exception as exc: - await db.rollback() - catalog_status = { - "attempted": True, - "status": "error", - "source_directories": publish_roots, - "message": str(exc), - } - - result["catalog"] = catalog_status - return result diff --git a/backend/app/routers/landsar_lt1_production.py b/backend/app/routers/landsar_lt1_production.py new file mode 100644 index 0000000..fb0d321 --- /dev/null +++ b/backend/app/routers/landsar_lt1_production.py @@ -0,0 +1,485 @@ +from __future__ import annotations + +import os +from typing import Any, Dict, List, Optional + +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import FileResponse +from pydantic import BaseModel, Field, field_validator +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from ..database import get_db +from ..models import AuthUserORM, RadarDataORM, SARSceneGeoORM +from ..services.job_handlers import JOB_TYPE_SAR_SCENE_PREPROCESS +from ..services.job_queue_service import job_queue_service +from ..services.landsar_lt1_production_service import landsar_lt1_production_service +from ..services.task_service import task_service +from ..utils import normalize_satellite_family +from .dependencies import _add_operation_audit_log, _get_current_user, _require_admin + + +router = APIRouter() +STATIC_ASSET_CACHE_HEADERS = {"Cache-Control": "public, max-age=31536000, immutable"} + + +class LandsarLt1ImageProductionRequest(BaseModel): + source_asset_ids: List[int] = Field(default_factory=list) + radar_data_ids: List[int] = Field(default_factory=list) + mode: str = "scene" + task_name: Optional[str] = None + + @field_validator("mode") + @classmethod + def _validate_mode(cls, value): + mode = str(value or "scene").strip().lower() + if mode == "stack": + mode = "batch" + if mode not in {"scene", "batch"}: + raise ValueError("mode must be scene or batch") + return mode + + +def _dedupe_positive_ids(values: List[int]) -> List[int]: + result: List[int] = [] + for value in values or []: + try: + parsed = int(value) + except (TypeError, ValueError): + continue + if parsed > 0 and parsed not in result: + result.append(parsed) + return result + + +def _scene_product_marker(scene: SARSceneGeoORM) -> Dict[str, Any]: + return { + "scene_id": scene.id, + "radar_data_id": scene.radar_data_id, + "product_id": f"sar_scene_geo:{scene.id}", + "product_family": "lt1_analysis_ready_geotiff", + "engine_code": scene.analysis_engine, + "profile_code": scene.analysis_profile, + "analysis_tif_path": scene.analysis_tif_path, + "analysis_dir": scene.analysis_dir, + "analysis_preview_path": scene.analysis_preview_path, + "status": scene.status, + "published_at": scene.updated_at.isoformat() if scene.updated_at else None, + } + + +def _scene_asset_items(scene: SARSceneGeoORM) -> List[Dict[str, Any]]: + candidates = [ + (1, "analysis_tif", "analysis_ready.tif", scene.analysis_tif_path, "image/tiff", True), + (2, "preview", "preview.png", scene.analysis_preview_path, "image/png", False), + ] + metadata = scene.analysis_metadata_json if isinstance(scene.analysis_metadata_json, dict) else {} + manifest_path = str(metadata.get("manifest_path") or "").strip() + if manifest_path: + candidates.append((3, "manifest", "manifest.json", manifest_path, "application/json", False)) + if scene.analysis_dir: + quality_path = os.path.join(scene.analysis_dir, "quality.json") + candidates.append((4, "quality", "quality.json", quality_path, "application/json", False)) + assets: List[Dict[str, Any]] = [] + for asset_id, role, name, path, media_type, primary in candidates: + if not path: + continue + assets.append( + { + "id": asset_id, + "role": role, + "name": name, + "relative_path": os.path.basename(path), + "absolute_path": path, + "format": os.path.splitext(path)[1].lower().lstrip(".") or None, + "media_type": media_type, + "is_required": primary, + "is_primary": primary, + "exists": os.path.isfile(path), + "file_size": os.path.getsize(path) if os.path.isfile(path) else None, + } + ) + return assets + + +async def _resolve_lt1_radars_for_request( + db: AsyncSession, + request: LandsarLt1ImageProductionRequest, +) -> List[RadarDataORM]: + source_asset_ids = _dedupe_positive_ids(request.source_asset_ids) + radar_data_ids = _dedupe_positive_ids(request.radar_data_ids) + filters = [] + if radar_data_ids: + filters.append(RadarDataORM.id.in_(radar_data_ids)) + if source_asset_ids: + filters.append(RadarDataORM.source_product_ref_id.in_(source_asset_ids)) + if not filters: + return [] + result = await db.execute(select(RadarDataORM).where(*([filters[0]] if len(filters) == 1 else [filters[0] | filters[1]]))) + radars = list(result.scalars().all()) + unique: Dict[int, RadarDataORM] = {} + for radar in radars: + if not radar.id: + continue + family = normalize_satellite_family(radar.satellite_family or radar.satellite) + if str(family or "").upper() != "LT1": + continue + unique[int(radar.id)] = radar + return [unique[key] for key in sorted(unique.keys())] + + +async def _produced_radars_for_request( + db: AsyncSession, + request: LandsarLt1ImageProductionRequest, +) -> Dict[int, dict]: + radars = await _resolve_lt1_radars_for_request(db, request) + radar_ids = [int(item.id) for item in radars if item.id] + if not radar_ids: + return {} + result = await db.execute( + select(SARSceneGeoORM).where( + SARSceneGeoORM.radar_data_id.in_(radar_ids), + SARSceneGeoORM.status == "DONE", + SARSceneGeoORM.analysis_tif_path.isnot(None), + SARSceneGeoORM.analysis_engine == "lt_gamma", + SARSceneGeoORM.analysis_profile == "lt1_gamma_geocoded_mli", + ) + ) + return {int(scene.radar_data_id): _scene_product_marker(scene) for scene in result.scalars().all()} + + +async def _active_radars_for_request( + db: AsyncSession, + request: LandsarLt1ImageProductionRequest, +) -> Dict[int, dict]: + radars = await _resolve_lt1_radars_for_request(db, request) + radar_ids = [int(item.id) for item in radars if item.id] + if not radar_ids: + return {} + result = await db.execute( + select(SARSceneGeoORM).where( + SARSceneGeoORM.radar_data_id.in_(radar_ids), + SARSceneGeoORM.status.in_(["PENDING", "RUNNING"]), + ) + ) + return {int(scene.radar_data_id): _scene_product_marker(scene) for scene in result.scalars().all()} + + +def _already_produced_blocker(produced: Dict[int, dict]) -> str: + first_id = sorted(produced.keys())[0] + marker = produced[first_id] or {} + product_id = marker.get("product_id") or "unknown" + return f"Radar data {first_id} already has an analysis-ready GeoTIFF: {product_id}" + + +def _active_blocker(active: Dict[int, dict]) -> str: + first_id = sorted(active.keys())[0] + marker = active[first_id] or {} + return f"Radar data {first_id} already has an active GeoTIFF production task (scene_id={marker.get('scene_id')})." + + +@router.get("/landsar-lt1-production/capabilities") +async def get_landsar_lt1_capabilities( + current_user: AuthUserORM = Depends(_get_current_user), +): + _ = current_user + legacy = landsar_lt1_production_service.check_capabilities() + return { + "catalog_name": "sar_scene_geo", + "supported_profiles": ["lt1_gamma_geocoded_mli"], + "engine": "lt_gamma", + "available": True, + "status": "configured", + "message": "LT-1 image production uses the existing Gamma single-scene pipeline: multilook, geocode, and analysis-ready GeoTIFF registration.", + "legacy_landsar_import": legacy, + } + + +@router.post("/landsar-lt1-production/preview") +async def preview_landsar_lt1_production( + request: LandsarLt1ImageProductionRequest, + current_user: AuthUserORM = Depends(_get_current_user), + db: AsyncSession = Depends(get_db), +): + _ = current_user + blockers: List[str] = [] + warnings: List[str] = [] + radars = await _resolve_lt1_radars_for_request(db, request) + if not radars: + blockers.append("No LT-1 radar records were resolved from the selected source assets.") + if request.mode == "scene" and len(radars) != 1: + blockers.append("Scene mode requires exactly one LT-1 source asset.") + if request.mode == "batch" and len(radars) < 1: + blockers.append("Batch mode requires at least one LT-1 source asset.") + produced = await _produced_radars_for_request(db, request) + if produced: + blockers.append(_already_produced_blocker(produced)) + active = await _active_radars_for_request(db, request) + if active: + blockers.append(_active_blocker(active)) + if request.mode == "batch": + warnings.append("Batch mode submits one independent geocoded GeoTIFF task per scene; it does not build a D-InSAR stack.") + preview = { + "allow_submit": not blockers, + "blockers": blockers, + "warnings": warnings, + "mode": request.mode, + "profile_code": "lt1_gamma_geocoded_mli", + "engine": "lt_gamma", + "scene_count": len(radars), + "source_asset_count": len(_dedupe_positive_ids(request.source_asset_ids)), + "radar_data_count": len(radars), + "produced_radars": produced, + "active_radars": active, + "scenes": [ + { + "radar_data_id": radar.id, + "source_asset_id": radar.source_product_ref_id, + "satellite": radar.satellite, + "imaging_date": radar.imaging_date, + "imaging_mode": radar.imaging_mode, + "polarization": radar.polarization, + "file_path": radar.file_path, + } + for radar in radars + ], + } + return preview + + +@router.post("/landsar-lt1-production/run", status_code=202) +async def queue_landsar_lt1_production( + request: LandsarLt1ImageProductionRequest, + http_request: Request, + db: AsyncSession = Depends(get_db), + admin_user: AuthUserORM = Depends(_require_admin), +): + _ = admin_user + preview = await preview_landsar_lt1_production(request, current_user=admin_user, db=db) + if preview.get("blockers"): + raise HTTPException(status_code=400, detail={"blockers": preview.get("blockers")}) + + queued: List[Dict[str, Any]] = [] + radars = await _resolve_lt1_radars_for_request(db, request) + for radar in radars: + result = await db.execute( + select(SARSceneGeoORM) + .where(SARSceneGeoORM.radar_data_id == int(radar.id)) + .with_for_update(skip_locked=True) + ) + scene = result.scalar_one_or_none() + if scene and scene.status in ("PENDING", "RUNNING"): + raise HTTPException(status_code=409, detail=f"Radar data {radar.id} already has an active GeoTIFF production task.") + if scene and scene.status == "DONE" and scene.analysis_tif_path: + raise HTTPException(status_code=409, detail=f"Radar data {radar.id} already has an analysis-ready GeoTIFF.") + if not scene: + scene = SARSceneGeoORM(radar_data_id=int(radar.id), status="PENDING") + db.add(scene) + await db.flush() + else: + scene.status = "PENDING" + scene.error_msg = None + await db.flush() + scene_id = int(scene.id) + await db.commit() + payload = { + "scene_id": scene_id, + "radar_data_id": int(radar.id), + "engine": "lt_gamma", + "source_asset_id": radar.source_product_ref_id, + "requested_from": "landsar_lt1_production", + } + task_label = request.task_name or radar.product_unique_id or radar.unique_id or f"radar_id={radar.id}" + task_type = f"LT1_SCENE_GEOTIFF_{scene_id}" + try: + task_id = await task_service.create_task( + task_type, + f"LT-1 geocoded GeoTIFF: {task_label}", + params=payload, + ) + job_id = await job_queue_service.create_job( + JOB_TYPE_SAR_SCENE_PREPROCESS, + payload=payload, + task_id=task_id, + max_attempts=3, + ) + except Exception as exc: + failed_scene = await db.get(SARSceneGeoORM, scene_id) + if failed_scene and failed_scene.status == "PENDING": + failed_scene.status = "FAILED" + failed_scene.error_msg = "Job queue failed" + await db.commit() + raise HTTPException(status_code=409 if "conflict" in str(exc).lower() else 400, detail=str(exc)) from exc + queued.append( + { + "task_id": task_id, + "job_id": job_id, + "scene_id": scene_id, + "radar_data_id": int(radar.id), + "source_asset_id": radar.source_product_ref_id, + } + ) + await _add_operation_audit_log( + db, + request=http_request, + action="lt1_geotiff_production_queued", + resource="landsar-lt1-production/run", + detail={ + "queued": queued, + "mode": request.mode, + "scene_count": len(queued), + }, + ) + await db.commit() + return { + "message": "LT-1 geocoded GeoTIFF production job queued.", + "task_id": queued[0]["task_id"] if len(queued) == 1 else None, + "job_id": queued[0]["job_id"] if len(queued) == 1 else None, + "queued": queued, + "preview": preview, + } + + +@router.get("/landsar-lt1-production/products") +async def list_landsar_lt1_products( + limit: int = 100, + offset: int = 0, + status: Optional[str] = None, + query: Optional[str] = None, + current_user: AuthUserORM = Depends(_get_current_user), + db: AsyncSession = Depends(get_db), +): + _ = current_user + safe_limit = max(1, min(500, int(limit or 100))) + safe_offset = max(0, int(offset or 0)) + filters = [ + SARSceneGeoORM.analysis_engine == "lt_gamma", + SARSceneGeoORM.analysis_profile == "lt1_gamma_geocoded_mli", + ] + if status: + filters.append(SARSceneGeoORM.status == str(status).strip().upper()) + if query: + like = f"%{str(query).strip()}%" + filters.append(RadarDataORM.product_unique_id.ilike(like) | RadarDataORM.unique_id.ilike(like) | RadarDataORM.file_path.ilike(like)) + total_result = await db.execute( + select(func.count(SARSceneGeoORM.id)) + .join(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id) + .where(*filters) + ) + total = int(total_result.scalar_one() or 0) + result = await db.execute( + select(SARSceneGeoORM, RadarDataORM) + .join(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id) + .where(*filters) + .order_by(SARSceneGeoORM.updated_at.desc().nullslast(), SARSceneGeoORM.id.desc()) + .limit(safe_limit) + .offset(safe_offset) + ) + items = [] + for scene, radar in result.all(): + marker = _scene_product_marker(scene) + items.append( + { + "id": scene.id, + "product_id": marker["product_id"], + "catalog_name": "sar_scene_geo", + "product_family": "lt1_analysis_ready_geotiff", + "product_type": "analysis_ready_geotiff", + "display_name": radar.product_unique_id or radar.unique_id or f"radar_id={radar.id}", + "task_name": "", + "profile_code": scene.analysis_profile, + "engine_code": scene.analysis_engine, + "status": scene.status, + "health_status": "OK" if scene.status == "DONE" and scene.analysis_tif_path else "PENDING", + "publish_dir": scene.analysis_dir, + "manifest_path": (scene.analysis_metadata_json or {}).get("manifest_path") if isinstance(scene.analysis_metadata_json, dict) else None, + "native_output_dir": scene.analysis_dir, + "primary_asset_path": scene.analysis_tif_path, + "summary": { + "scene_count": 1, + "radar_data_id": radar.id, + "source_asset_ids": [radar.source_product_ref_id] if radar.source_product_ref_id else [], + "imaging_date": radar.imaging_date, + "polarization": radar.polarization, + "pixel_size_m": scene.pixel_size_m, + "backscatter_unit": scene.analysis_backscatter_unit, + }, + "tags": {"engine": scene.analysis_engine, "profile": scene.analysis_profile}, + "produced_at": scene.updated_at.isoformat() if scene.updated_at else None, + "published_at": scene.updated_at.isoformat() if scene.updated_at else None, + "registered_at": scene.created_at.isoformat() if scene.created_at else None, + } + ) + return {"total": total, "limit": safe_limit, "offset": safe_offset, "items": items} + + +@router.get("/landsar-lt1-production/products/{product_db_id}") +async def get_landsar_lt1_product_detail( + product_db_id: int, + current_user: AuthUserORM = Depends(_get_current_user), + db: AsyncSession = Depends(get_db), +): + _ = current_user + result = await db.execute( + select(SARSceneGeoORM, RadarDataORM) + .join(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id) + .where( + SARSceneGeoORM.id == product_db_id, + SARSceneGeoORM.analysis_engine == "lt_gamma", + SARSceneGeoORM.analysis_profile == "lt1_gamma_geocoded_mli", + ) + ) + row = result.first() + if row is None: + raise HTTPException(status_code=404, detail="LT-1 geocoded GeoTIFF product not found") + scene, radar = row + marker = _scene_product_marker(scene) + detail = { + "id": scene.id, + "product_id": marker["product_id"], + "catalog_name": "sar_scene_geo", + "product_family": "lt1_analysis_ready_geotiff", + "product_type": "analysis_ready_geotiff", + "display_name": radar.product_unique_id or radar.unique_id or f"radar_id={radar.id}", + "profile_code": scene.analysis_profile, + "engine_code": scene.analysis_engine, + "status": scene.status, + "publish_dir": scene.analysis_dir, + "primary_asset_path": scene.analysis_tif_path, + "summary": { + "scene_count": 1, + "radar_data_id": radar.id, + "source_asset_ids": [radar.source_product_ref_id] if radar.source_product_ref_id else [], + "imaging_date": radar.imaging_date, + "polarization": radar.polarization, + "pixel_size_m": scene.pixel_size_m, + "backscatter_unit": scene.analysis_backscatter_unit, + }, + "assets": _scene_asset_items(scene), + } + return detail + + +@router.get("/landsar-lt1-production/products/{product_db_id}/assets/{asset_id}") +async def get_landsar_lt1_product_asset( + product_db_id: int, + asset_id: int, + current_user: AuthUserORM = Depends(_get_current_user), + db: AsyncSession = Depends(get_db), +): + _ = current_user + scene = await db.get(SARSceneGeoORM, product_db_id) + if scene is None or scene.analysis_engine != "lt_gamma" or scene.analysis_profile != "lt1_gamma_geocoded_mli": + raise HTTPException(status_code=404, detail="LT-1 geocoded GeoTIFF product not found") + asset = next((item for item in _scene_asset_items(scene) if int(item["id"]) == int(asset_id)), None) + if asset is None: + raise HTTPException(status_code=404, detail="LT-1 geocoded GeoTIFF asset not found") + absolute_path = str(asset.get("absolute_path") or "") + if not absolute_path or not os.path.isfile(absolute_path): + raise HTTPException(status_code=404, detail="Asset file not found") + return FileResponse( + absolute_path, + media_type=str(asset.get("media_type") or "application/octet-stream"), + filename=str(asset.get("name") or os.path.basename(absolute_path)), + headers=STATIC_ASSET_CACHE_HEADERS, + ) diff --git a/backend/app/routers/pairing.py b/backend/app/routers/pairing.py index b9d04d6..bbf1739 100644 --- a/backend/app/routers/pairing.py +++ b/backend/app/routers/pairing.py @@ -26,7 +26,10 @@ from ..models import ( TimeseriesStackPlanORM, ) from ..services.pairing_cache_service import pairing_cache_service +from ..services.job_handlers import JOB_TYPE_PAIRING_CACHE_REBUILD +from ..services.job_queue_service import job_queue_service from ..services.spatial_service import spatial_service +from ..services.task_service import task_service from .dependencies import ( _parse_aoi_from_files, _parse_aoi_geojson_form_value, @@ -38,6 +41,49 @@ logger = logging.getLogger(__name__) router = APIRouter() +async def _queue_pairing_cache_job( + *, + db: AsyncSession, + mode: str, + force_full: bool = False, +) -> Dict[str, object]: + full_rebuild = mode in {"full", "full_rebuild"} or force_full + task_name = ( + "D-InSAR pairing cache full rebuild" + if full_rebuild + else "D-InSAR pairing cache dirty reconcile" + ) + payload = { + "mode": "full_rebuild" if full_rebuild else "auto_reconcile", + "force_full": bool(full_rebuild), + } + try: + task_id = await task_service.create_task( + JOB_TYPE_PAIRING_CACHE_REBUILD, + task_name, + params=payload, + db=db, + ) + job_id = await job_queue_service.create_job( + JOB_TYPE_PAIRING_CACHE_REBUILD, + payload=payload, + max_attempts=1, + task_id=task_id, + db=db, + ) + await db.commit() + except ValueError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + return { + "ok": True, + "queued": True, + "mode": payload["mode"], + "task_id": task_id, + "job_id": job_id, + "message": f"{task_name} queued", + } + + def get_pairing_request_from_form( time_baseline_min: int = Form(1), time_baseline_max: int = Form(30), @@ -130,26 +176,26 @@ async def get_pairing_health_endpoint( return await pairing_cache_service.get_admin_summary(db) -@router.post("/pairing/rebuild-cache") +@router.post("/pairing/rebuild-cache", status_code=202) async def rebuild_pairing_cache_endpoint( db: AsyncSession = Depends(get_db), current_user: AuthUserORM = Depends(_require_admin), ): _ = current_user - return await pairing_cache_service.rebuild_metric_cache(db, commit=True) + return await _queue_pairing_cache_job(db=db, mode="full_rebuild", force_full=True) -@router.post("/pairing/reconcile-dirty") +@router.post("/pairing/reconcile-dirty", status_code=202) async def reconcile_dirty_pairing_endpoint( force_full: bool = Query(False), db: AsyncSession = Depends(get_db), current_user: AuthUserORM = Depends(_require_admin), ): _ = current_user - return await pairing_cache_service.reconcile_dirty_scenes( - db, + return await _queue_pairing_cache_job( + db=db, + mode="full_rebuild" if force_full else "auto_reconcile", force_full=force_full, - commit=True, ) diff --git a/backend/app/routers/radar.py b/backend/app/routers/radar.py index 9e4b801..4d35ef0 100644 --- a/backend/app/routers/radar.py +++ b/backend/app/routers/radar.py @@ -23,6 +23,7 @@ from ..models import ( RadarDataORM, RadarDataPage, RadarPreviewStatusInfo, + SARSceneGeoORM, ScanRequest, ) from ..services.data_service import data_service @@ -154,6 +155,51 @@ def _normalize_list_pagination(limit: int, offset: int) -> Tuple[int, int]: return safe_limit, safe_offset +def _lt1_image_marker(scene: SARSceneGeoORM) -> Dict[str, Any]: + return { + "scene_id": scene.id, + "radar_data_id": scene.radar_data_id, + "product_id": f"sar_scene_geo:{scene.id}", + "product_family": "lt1_analysis_ready_geotiff", + "engine_code": scene.analysis_engine, + "profile_code": scene.analysis_profile, + "analysis_tif_path": scene.analysis_tif_path, + "analysis_dir": scene.analysis_dir, + "analysis_preview_path": scene.analysis_preview_path, + "status": scene.status, + "published_at": scene.updated_at.isoformat() if scene.updated_at else None, + } + + +async def _decorate_lt1_landsar_status(db: AsyncSession, items: List[RadarDataORM]) -> List[RadarData]: + payloads = [RadarData.model_validate(item) for item in items] + radar_ids = [ + int(item.id) + for item in items + if item.id and str(item.satellite_family or item.satellite or "").upper().replace("-", "") in {"LT1", "LT"} + ] + if not radar_ids: + return payloads + result = await db.execute( + select(SARSceneGeoORM).where( + SARSceneGeoORM.radar_data_id.in_(radar_ids), + SARSceneGeoORM.status == "DONE", + SARSceneGeoORM.analysis_tif_path.isnot(None), + SARSceneGeoORM.analysis_engine == "lt_gamma", + SARSceneGeoORM.analysis_profile == "lt1_gamma_geocoded_mli", + ) + ) + produced = {int(scene.radar_data_id): _lt1_image_marker(scene) for scene in result.scalars().all()} + for payload in payloads: + marker = produced.get(int(payload.id or 0)) + if marker: + payload.lt1_image_produced = True + payload.lt1_image_product = marker + payload.lt1_landsar_produced = True + payload.lt1_landsar_product = marker + return payloads + + def _normalize_optional_text(value: Optional[str]) -> Optional[str]: if value is None: return None @@ -256,6 +302,24 @@ def _build_radar_preview_status( ) +def _build_cached_radar_preview_status(record: RadarDataORM) -> RadarPreviewStatusInfo: + raw_cache_path, geo_cache_path = _radar_preview_paths(record) + preview_cache_path = str(record.preview_cache_path or "") + has_geo_cache = ( + os.path.exists(geo_cache_path) + or bool(preview_cache_path and os.path.exists(preview_cache_path)) + ) + has_raw_cache = os.path.exists(raw_cache_path) + cached_status = str(record.preview_cache_status or "NONE").upper() + source_found = cached_status in {"READY", "FAILED"} or has_geo_cache or has_raw_cache + return _build_radar_preview_status( + record=record, + source_found=source_found, + has_geo_cache=has_geo_cache, + has_raw_cache=has_raw_cache, + ) + + async def _build_radar_preview_cache( record: RadarDataORM, db: AsyncSession, @@ -510,10 +574,13 @@ async def search_radar_data_endpoint( limit: int = Form(500), offset: int = Form(0), satellite: Optional[str] = Form(None), + satellite_family: Optional[str] = Form(None), + source_format: Optional[str] = Form(None), satellite_mode: Optional[str] = Form(None), receiving_station: Optional[str] = Form(None), imaging_mode: Optional[str] = Form(None), orbit_circle: Optional[str] = Form(None), + relative_orbit: Optional[str] = Form(None), acquisition_time_utc: Optional[str] = Form(None), product_type: Optional[str] = Form(None), polarization: Optional[str] = Form(None), @@ -536,10 +603,13 @@ async def search_radar_data_endpoint( n_satellite_list: Optional[List[str]] = None if n_satellite_raw and "," in n_satellite_raw: n_satellite_list = [s.strip() for s in n_satellite_raw.split(",") if s.strip()] + n_satellite_family = _normalize_optional_text(satellite_family) + n_source_format = _normalize_optional_text(source_format) n_satellite_mode = _normalize_optional_text(satellite_mode) n_receiving_station = _normalize_optional_text(receiving_station) n_imaging_mode = _normalize_optional_text(imaging_mode) n_orbit_circle = _normalize_optional_text(orbit_circle) + n_relative_orbit = _normalize_optional_text(relative_orbit) n_acquisition_time = _normalize_optional_text(acquisition_time_utc) n_product_type = _normalize_optional_text(product_type) n_polarization = _normalize_optional_text(polarization) @@ -584,6 +654,10 @@ async def search_radar_data_endpoint( filters.append(RadarDataORM.satellite.in_(n_satellite_list)) elif n_satellite_raw: filters.append(RadarDataORM.satellite.ilike(f"%{n_satellite_raw}%")) + if n_satellite_family: + filters.append(func.upper(RadarDataORM.satellite_family) == n_satellite_family.upper()) + if n_source_format: + filters.append(func.upper(RadarDataORM.source_format) == n_source_format.upper()) if n_satellite_mode: filters.append(RadarDataORM.satellite_mode.ilike(f"%{n_satellite_mode}%")) if n_receiving_station: @@ -592,6 +666,8 @@ async def search_radar_data_endpoint( filters.append(RadarDataORM.imaging_mode.ilike(f"%{n_imaging_mode}%")) if n_orbit_circle: filters.append(RadarDataORM.orbit_circle.ilike(f"%{n_orbit_circle}%")) + if n_relative_orbit: + filters.append(RadarDataORM.relative_orbit.ilike(f"%{n_relative_orbit}%")) if n_acquisition_time: filters.append(RadarDataORM.acquisition_time_utc.ilike(f"%{n_acquisition_time}%")) if n_product_type: @@ -606,10 +682,11 @@ async def search_radar_data_endpoint( filters.append(RadarDataORM.orbit_direction.ilike(f"%{n_orbit_direction}%")) if has_orbit_data is not None: filters.append(RadarDataORM.has_orbit_data == has_orbit_data) + normalized_imaging_date = func.replace(RadarDataORM.imaging_date, "-", "") if n_date_from: - filters.append(RadarDataORM.imaging_date >= n_date_from) + filters.append(normalized_imaging_date >= n_date_from.replace("-", "")) if n_date_to: - filters.append(RadarDataORM.imaging_date <= n_date_to) + filters.append(normalized_imaging_date <= n_date_to.replace("-", "")) if resolved_aoi_wkt: aoi_geom = func.ST_GeomFromText(resolved_aoi_wkt, 4326) filters.append(ST_Intersects(RadarDataORM.geom, aoi_geom)) @@ -630,8 +707,9 @@ async def search_radar_data_endpoint( result = await db.execute(data_stmt) items = result.scalars().all() + decorated_items = await _decorate_lt1_landsar_status(db, items) return RadarDataSearchPageResponse( - items=items, + items=decorated_items, total=total, limit=limit, offset=offset, @@ -663,8 +741,9 @@ async def get_all_data_endpoint( .limit(limit) ) items = result.scalars().all() + decorated_items = await _decorate_lt1_landsar_status(db, items) return RadarDataPage( - items=items, + items=decorated_items, total=total, limit=limit, offset=offset, @@ -724,16 +803,7 @@ async def get_radar_preview_status_endpoint(data_id: int, db: AsyncSession = Dep if _is_gf3_native_preview_record(record): return _build_gf3_native_preview_status(record) - raw_cache_path, geo_cache_path = _radar_preview_paths(record) - has_geo_cache = os.path.exists(geo_cache_path) - has_raw_cache = os.path.exists(raw_cache_path) - source_found = bool(await asyncio.to_thread(data_service.find_radar_preview_source, record.file_path)) - return _build_radar_preview_status( - record=record, - source_found=source_found, - has_geo_cache=has_geo_cache, - has_raw_cache=has_raw_cache, - ) + return _build_cached_radar_preview_status(record) @router.post("/radar-data/{data_id}/rebuild-preview-cache", response_model=RadarPreviewStatusInfo) diff --git a/backend/app/routers/tasks_runtime.py b/backend/app/routers/tasks_runtime.py index b0551a7..f29b243 100644 --- a/backend/app/routers/tasks_runtime.py +++ b/backend/app/routers/tasks_runtime.py @@ -2,16 +2,19 @@ from __future__ import annotations import asyncio import json +from datetime import datetime, timedelta from typing import List, Optional from fastapi import APIRouter, Depends, HTTPException, Query, Request from fastapi.responses import StreamingResponse from pydantic import BaseModel +from sqlalchemy import case, func, select from .. import database from ..auth_service import SESSION_COOKIE_NAME, get_user_by_session_token from ..auth_utils import verify_password -from ..models import AuthUserORM, TaskInfo +from ..config import settings +from ..models import AuthUserORM, SystemJobORM, SystemTaskORM, SystemWorkerHeartbeatORM, TaskInfo from ..services.dinsar_production_service import dinsar_production_service from ..services.task_service import ( TASK_ACTIVE_DEFAULT_LIMIT, @@ -47,6 +50,73 @@ def _split_csv_param(raw: Optional[str]) -> List[str]: return values +def _dt(value): + return value.isoformat() if value else None + + +def _task_payload(task: SystemTaskORM) -> dict: + return TaskInfo.model_validate(task).model_dump(mode="json") + + +def _worker_note(worker: SystemWorkerHeartbeatORM) -> dict: + try: + parsed = json.loads(str(worker.note or "") or "{}") + return parsed if isinstance(parsed, dict) else {} + except Exception: + return {} + + +def _worker_concurrency(worker: SystemWorkerHeartbeatORM) -> int: + note = _worker_note(worker) + try: + return max(1, int(note.get("concurrency") or 1)) + except (TypeError, ValueError): + return 1 + + +def _job_payload(job: SystemJobORM, task_by_id: dict[str, SystemTaskORM]) -> dict: + task = task_by_id.get(str(job.task_id or "")) + return { + "job_id": job.job_id, + "job_type": job.job_type, + "status": job.status, + "priority": int(job.priority or 0), + "attempts": int(job.attempts or 0), + "max_attempts": int(job.max_attempts or 0), + "task_id": job.task_id, + "task_type": task.task_type if task else None, + "task_name": task.task_name if task else None, + "task_status": task.status if task else None, + "task_progress": int(task.progress or 0) if task else None, + "task_message": task.message if task else None, + "workflow_run_id": job.workflow_run_id, + "workflow_step_id": job.workflow_step_id, + "locked_by": job.locked_by, + "locked_at": _dt(job.locked_at), + "heartbeat_at": _dt(job.heartbeat_at), + "next_run_at": _dt(job.next_run_at), + "created_at": _dt(job.created_at), + "started_at": _dt(job.started_at), + "finished_at": _dt(job.finished_at), + "last_error": job.last_error, + } + + +def _worker_payload(worker: SystemWorkerHeartbeatORM, active_job_count: int, concurrency: int) -> dict: + note = _worker_note(worker) + return { + "worker_id": worker.worker_id, + "hostname": worker.hostname, + "pid": worker.pid, + "note": worker.note, + "concurrency": concurrency, + "allowed_job_types": note.get("allowed_job_types") if isinstance(note.get("allowed_job_types"), list) else [], + "started_at": _dt(worker.started_at), + "last_seen": _dt(worker.last_seen), + "active_job_count": active_job_count, + } + + @router.get("/tasks/active", response_model=List[TaskInfo]) async def get_active_tasks(limit: int = TASK_ACTIVE_DEFAULT_LIMIT, offset: int = 0): safe_limit = min(TASK_ACTIVE_MAX_LIMIT, max(1, int(limit or TASK_ACTIVE_DEFAULT_LIMIT))) @@ -73,6 +143,199 @@ async def get_recent_tasks( return [TaskInfo.model_validate(task) for task in orm_tasks] +@router.get("/tasks/runtime-summary") +async def get_task_runtime_summary(limit: int = TASK_ACTIVE_DEFAULT_LIMIT, offset: int = 0): + safe_limit = min(TASK_ACTIVE_MAX_LIMIT, max(1, int(limit or TASK_ACTIVE_DEFAULT_LIMIT))) + safe_offset = min(TASK_QUERY_MAX_OFFSET, max(0, int(offset or 0))) + active_job_statuses = ["READY", "RETRY", "RUNNING"] + scan_job_types = { + "SCAN_DATA", + "SCAN_DINSAR", + "SCAN_ASSET_INVENTORY", + "AUDIT_SOURCE_ARCHIVE_INTEGRITY", + "GF3_SARSCAPE_SYNC", + "GF3_QUICKLOOK_WEBP", + } + + worker_timeout = max(5, int(getattr(settings, "JOB_WORKER_HEALTH_TIMEOUT", 60) or 60)) + worker_threshold = datetime.utcnow() - timedelta(seconds=worker_timeout) + configured_concurrency = max(1, int(getattr(settings, "JOB_WORKER_CONCURRENCY", 1) or 1)) + + async with _new_session() as db: + active_tasks = await task_service.get_active_tasks(limit=safe_limit, offset=safe_offset, db=db) + + workers_result = await db.execute( + select(SystemWorkerHeartbeatORM) + .where(SystemWorkerHeartbeatORM.last_seen >= worker_threshold) + .order_by(SystemWorkerHeartbeatORM.last_seen.desc()) + ) + active_workers = workers_result.scalars().all() + active_worker_ids = {str(worker.worker_id) for worker in active_workers} + + running_by_worker_result = await db.execute( + select(SystemJobORM.locked_by, func.count(SystemJobORM.id)) + .where(SystemJobORM.status == "RUNNING") + .group_by(SystemJobORM.locked_by) + ) + running_by_worker = { + str(worker_id or ""): int(count or 0) + for worker_id, count in running_by_worker_result.all() + } + + status_counts_result = await db.execute( + select(SystemJobORM.status, func.count(SystemJobORM.id)) + .where(SystemJobORM.status.in_(active_job_statuses)) + .group_by(SystemJobORM.status) + ) + job_status_counts = { + "READY": 0, + "RETRY": 0, + "RUNNING": 0, + } + for status, count in status_counts_result.all(): + job_status_counts[str(status or "").upper()] = int(count or 0) + + status_rank = case( + (SystemJobORM.status == "RUNNING", 0), + (SystemJobORM.status == "RETRY", 1), + else_=2, + ) + jobs_result = await db.execute( + select(SystemJobORM) + .where(SystemJobORM.status.in_(active_job_statuses)) + .order_by(status_rank, SystemJobORM.priority.desc(), SystemJobORM.id.asc()) + .offset(safe_offset) + .limit(safe_limit) + ) + active_jobs = jobs_result.scalars().all() + + task_ids = { + str(task.task_id) + for task in active_tasks + if task.task_id + } + task_ids.update( + str(job.task_id) + for job in active_jobs + if job.task_id + ) + task_by_id: dict[str, SystemTaskORM] = {} + if task_ids: + task_result = await db.execute( + select(SystemTaskORM).where(SystemTaskORM.task_id.in_(sorted(task_ids))) + ) + task_by_id = { + str(task.task_id): task + for task in task_result.scalars().all() + if task.task_id + } + + worker_concurrency_by_id = { + str(worker.worker_id): _worker_concurrency(worker) + for worker in active_workers + } + total_slots = sum(worker_concurrency_by_id.values()) + busy_slots = sum( + count + for worker_id, count in running_by_worker.items() + if worker_id in active_worker_ids + ) + running_count = int(job_status_counts.get("RUNNING") or 0) + stale_running_count = max(0, running_count - busy_slots) + queued_count = int(job_status_counts.get("READY") or 0) + int(job_status_counts.get("RETRY") or 0) + + task_items = [_task_payload(task) for task in active_tasks] + job_items = [_job_payload(job, task_by_id) for job in active_jobs] + scan_jobs = [ + item for item in job_items + if str(item.get("job_type") or "").upper() in scan_job_types + ] + scan_tasks = [ + item for item in task_items + if str(item.get("task_type") or "").upper() in scan_job_types + ] + + return { + "timestamp": datetime.utcnow().isoformat() + "Z", + "worker": { + "ok": len(active_workers) > 0, + "worker_count": len(active_workers), + "configured_concurrency": configured_concurrency, + "total_slots": total_slots, + "busy_slots": busy_slots, + "idle_slots": max(0, total_slots - busy_slots), + "timeout_seconds": worker_timeout, + "stale_running_job_count": stale_running_count, + "workers": [ + _worker_payload( + worker, + running_by_worker.get(str(worker.worker_id), 0), + worker_concurrency_by_id.get(str(worker.worker_id), 1), + ) + for worker in active_workers + ], + }, + "jobs": { + "active_count": running_count + queued_count, + "running_count": running_count, + "queued_count": queued_count, + "ready_count": int(job_status_counts.get("READY") or 0), + "retry_count": int(job_status_counts.get("RETRY") or 0), + "items": job_items, + }, + "tasks": { + "active_count": len(task_items), + "running_count": sum(1 for item in task_items if item.get("status") == "RUNNING"), + "pending_count": sum(1 for item in task_items if item.get("status") == "PENDING"), + "items": task_items, + }, + "scan": { + "active_task_count": len(scan_tasks), + "active_job_count": len(scan_jobs), + "running_job_count": sum(1 for item in scan_jobs if item.get("status") == "RUNNING"), + "queued_job_count": sum(1 for item in scan_jobs if item.get("status") in {"READY", "RETRY"}), + "tasks": scan_tasks, + "jobs": scan_jobs, + }, + } + + +@router.get("/tasks/runtime-summary/stream") +async def stream_task_runtime_summary(request: Request): + token = request.cookies.get(SESSION_COOKIE_NAME) + if not token: + raise HTTPException(status_code=401, detail="Authentication required.") + + async with _new_session() as db: + user = await get_user_by_session_token(db, token) + if not user: + raise HTTPException(status_code=401, detail="Authentication required.") + + async def event_generator(): + while True: + if await request.is_disconnected(): + break + try: + summary = await get_task_runtime_summary( + limit=TASK_ACTIVE_MAX_LIMIT, + offset=0, + ) + yield f"data: {json.dumps(summary)}\n\n" + except Exception: + yield "data: {}\n\n" + await asyncio.sleep(3) + + return StreamingResponse( + event_generator(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "X-Accel-Buffering": "no", + "Connection": "keep-alive", + }, + ) + + @router.get("/tasks/active/stream") async def stream_active_tasks(request: Request): token = request.cookies.get(SESSION_COOKIE_NAME) diff --git a/backend/app/services/asset_inventory_service.py b/backend/app/services/asset_inventory_service.py index 7cf2682..fb59e6e 100644 --- a/backend/app/services/asset_inventory_service.py +++ b/backend/app/services/asset_inventory_service.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio import gzip import hashlib +import logging import math import multiprocessing as mp import os @@ -48,6 +49,8 @@ from ..utils import ( ) from .pairing_state_service import pairing_state_service from .data_service import DataService + +logger = logging.getLogger(__name__) from .image_service import image_service from .orbit_converter import sync_orbit_pools from .task_service import task_service @@ -2599,6 +2602,53 @@ def _image_data_format_for_source(row: Dict[str, Any]) -> str: return "DIRECTORY" +PAIRING_RELEVANT_RADAR_FIELDS = { + "satellite", + "satellite_family", + "imaging_date", + "imaging_mode", + "orbit_direction", + "polarization", + "look_direction", + "relative_orbit", + "insar_source_ready", + "file_path", + "coverage_polygon", + "min_lon", + "min_lat", + "max_lon", + "max_lat", + "scene_center_lon", + "scene_center_lat", +} + + +def _normalize_pairing_compare_value(value: Any) -> Any: + if isinstance(value, str): + text = value.strip() + return text or None + if isinstance(value, bool): + return bool(value) + if isinstance(value, (int, float)): + return round(float(value), 9) + if isinstance(value, list): + return [_normalize_pairing_compare_value(item) for item in value] + if isinstance(value, tuple): + return [_normalize_pairing_compare_value(item) for item in value] + return value + + +def _radar_pairing_fields_changed(existing: RadarDataORM, new_values: Dict[str, Any]) -> bool: + for field in PAIRING_RELEVANT_RADAR_FIELDS: + if field not in new_values: + continue + before = _normalize_pairing_compare_value(getattr(existing, field, None)) + after = _normalize_pairing_compare_value(new_values.get(field)) + if before != after: + return True + return False + + class AssetInventoryService: async def _progress(self, task_id: Optional[str], message: str, progress: int) -> None: if not task_id: @@ -2784,6 +2834,13 @@ class AssetInventoryService: raw_cache_path = DataService.get_radar_raw_cache_path(unique_id, record.file_path) geo_cache_path = DataService.get_radar_geo_cache_path(unique_id, record.file_path) product_name = os.path.basename(str(record.file_path or "")) + if task_id: + progress = progress_start + int(index / max(1, total) * max(1, progress_end - progress_start)) + await task_service.update_task( + task_id, + message=f"Building archive preview cache ({index}/{total}): {product_name}", + progress=min(progress_end, progress), + ) preview_source = await asyncio.to_thread(DataService.find_radar_preview_source, record.file_path) if not preview_source: @@ -4675,6 +4732,7 @@ class AssetInventoryService: profile_inputs.append((row, asset_id, int(scene.id))) else: before_orbit_id = existing.selected_orbit_asset_id + pairing_fields_changed = _radar_pairing_fields_changed(existing, radar_values) for key, value in radar_values.items(): setattr(existing, key, value) if not existing.orbit_binding_status: @@ -4682,16 +4740,15 @@ class AssetInventoryService: db.add(existing) if existing.id is not None: profile_inputs.append((row, asset_id, int(existing.id))) - if existing.id is not None and before_orbit_id != existing.selected_orbit_asset_id: + if existing.id is not None and ( + pairing_fields_changed or before_orbit_id != existing.selected_orbit_asset_id + ): dirty_scene_ids.append(int(existing.id)) await db.flush() if profile_inputs: await self._upsert_geometry_profiles(db, profile_inputs) await self._attach_radar_ids_to_metadata_documents(db, profile_inputs) - for _, _, radar_id in profile_inputs: - if radar_id not in dirty_scene_ids: - dirty_scene_ids.append(radar_id) if dirty_scene_ids: await pairing_state_service.mark_scenes_dirty(db, scene_ids=dirty_scene_ids, reason="asset_inventory_source_update", commit=False) @@ -4988,7 +5045,6 @@ class AssetInventoryService: matched = 0 missing = 0 candidate_count = 0 - dirty_scene_ids: List[int] = [] for scene in scenes: candidates = await self._find_orbit_candidates(db, scene) if not candidates: @@ -5018,8 +5074,6 @@ class AssetInventoryService: }, ) ) - if scene.id is not None: - dirty_scene_ids.append(int(scene.id)) continue candidate_count += len(candidates) @@ -5049,8 +5103,6 @@ class AssetInventoryService: scene.orbit_binding_reason = selected[2] db.add(scene) matched += 1 - if scene.id is not None: - dirty_scene_ids.append(int(scene.id)) if len(candidates) > 1 and abs(float(candidates[0][1]) - float(candidates[1][1])) < 0.001: db.add( @@ -5068,13 +5120,8 @@ class AssetInventoryService: ) ) - if dirty_scene_ids: - await pairing_state_service.mark_scenes_dirty( - db, - scene_ids=sorted(set(dirty_scene_ids)), - reason="asset_inventory_orbit_binding", - commit=False, - ) + # Pairing queries join radar_data and filter has_orbit_data live; orbit rebinding + # does not change the cached geometric/time pairing metrics. return { "scene_count": len(scenes), "matched_count": matched, @@ -5279,8 +5326,15 @@ class AssetInventoryService: .limit(safe_limit) ) ).scalars().all() + items = [self._source_asset_payload(row) for row in rows] + try: + from .landsar_lt1_production_service import landsar_lt1_production_service + + await landsar_lt1_production_service.decorate_source_asset_payloads(db, items) + except Exception: + logger.debug("Failed to decorate LT-1 LandSAR production status", exc_info=True) return { - "items": [self._source_asset_payload(row) for row in rows], + "items": items, "total": total, "limit": safe_limit, "offset": safe_offset, diff --git a/backend/app/services/data_service.py b/backend/app/services/data_service.py index 37eb01a..8b988a4 100644 --- a/backend/app/services/data_service.py +++ b/backend/app/services/data_service.py @@ -288,6 +288,8 @@ def _radar_archive_expected_preview_rank(archive_path: str, member_name: str) -> return None member = str(member_name or "").replace("\\", "/").strip("/") + while member.startswith("./"): + member = member[2:].strip("/") member_lower = member.lower() product_lower = product_stem.lower() expected_names = [ @@ -315,7 +317,10 @@ def _radar_archive_expected_preview_rank(archive_path: str, member_name: str) -> def _radar_archive_preview_score(member_name: str, size_bytes: int = 0) -> Optional[Tuple[int, int, int, int, str]]: - lower_name = str(member_name or "").replace("\\", "/").lower() + normalized_name = str(member_name or "").replace("\\", "/").strip("/") + while normalized_name.startswith("./"): + normalized_name = normalized_name[2:].strip("/") + lower_name = normalized_name.lower() base_name = os.path.basename(lower_name) if not base_name.endswith(_RADAR_PREVIEW_EXTENSIONS): return None @@ -1024,7 +1029,6 @@ class DataService: # 3. 更新缺失精轨的现有记录 update_progress("正在关联精轨数据...", 85) updated_orbits = 0 - updated_orbit_scene_ids = set() if orbit_files_map: stmt_select = select(RadarDataORM).where(RadarDataORM.has_orbit_data == False) result = await db.execute(stmt_select) @@ -1037,8 +1041,6 @@ class DataService: record.orbit_file_path = orbit_files_map[key] db.add(record) updated_orbits += 1 - if record.id is not None: - updated_orbit_scene_ids.add(int(record.id)) for data_type, root_path, mtime in scan_state_updates: await DataService._upsert_scan_state(db, data_type, root_path, mtime) @@ -1046,7 +1048,9 @@ class DataService: await db.commit() pairing_dirty_summary: Dict[str, Any] = {} - dirty_scene_ids = set(updated_orbit_scene_ids) + # Orbit availability is filtered live by pairing queries; it is not part of + # pairing_metric_cache, so orbit-only updates should not dirty the cache. + dirty_scene_ids = set() processed_unique_ids = [key for key in radar_cache_candidates.keys() if key] for chunk in _chunked(processed_unique_ids, 500): id_result = await db.execute( @@ -1061,7 +1065,7 @@ class DataService: reason="radar_scan", commit=True, ) - elif processed_scenes > 0 or updated_orbits > 0: + elif processed_scenes > 0: pairing_dirty_summary = await pairing_state_service.mark_global_dirty( db, reason="radar_scan", diff --git a/backend/app/services/dinsar_production_service.py b/backend/app/services/dinsar_production_service.py index e9759bd..feb05f8 100644 --- a/backend/app/services/dinsar_production_service.py +++ b/backend/app/services/dinsar_production_service.py @@ -1007,6 +1007,7 @@ class DinsarProductionService: "root_dir": run.source_root, "publish_root_dir": run.publish_root_dir, "message": run.latest_message, + "summary_json": run.summary_json if isinstance(run.summary_json, dict) else {}, "total_items": run.total_items, "completed_items": run.completed_items, "failed_items": run.failed_items, diff --git a/backend/app/services/job_handlers.py b/backend/app/services/job_handlers.py index 621aa0c..aacc902 100644 --- a/backend/app/services/job_handlers.py +++ b/backend/app/services/job_handlers.py @@ -41,7 +41,7 @@ from .dinsar_result_layout_service import ( ) from .dinsar_scan_service import dinsar_scan_service from .engine_lock_service import engine_lock_service -from .envi_service import build_envi_runner_command, get_envi_runner_cwd, get_envi_runner_env +from .envi_service import build_envi_runner_command, extract_disp_results, get_envi_runner_cwd, get_envi_runner_env from .psinsar_catalog_service import psinsar_catalog_service from .result_catalog_service import result_catalog_service from .sbas_insar_catalog_service import sbas_insar_catalog_service @@ -101,10 +101,13 @@ JOB_TYPE_ISCE2_RUN = "ISCE2_RUN" JOB_TYPE_PYINT_RUN = "PYINT_RUN" JOB_TYPE_LANDSAR_RUN = "LANDSAR_RUN" JOB_TYPE_LANDSAR_CLUSTER_ITEM = "LANDSAR_CLUSTER_ITEM" +JOB_TYPE_LANDSAR_LT1_IMPORT = "LANDSAR_LT1_IMPORT" +JOB_TYPE_EXTRACT_DINSAR_PRODUCTS = "EXTRACT_DINSAR_PRODUCTS" JOB_TYPE_PUBLISH_DINSAR_PRODUCTS = "PUBLISH_DINSAR_PRODUCTS" JOB_TYPE_REBUILD_DINSAR_CATALOG = "REBUILD_DINSAR_CATALOG" JOB_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG" JOB_TYPE_REBUILD_SBAS_INSAR_CATALOG = "REBUILD_SBAS_INSAR_CATALOG" +JOB_TYPE_PAIRING_CACHE_REBUILD = "PAIRING_CACHE_REBUILD" JOB_TYPE_SCAN_ASSET_INVENTORY = "SCAN_ASSET_INVENTORY" JOB_TYPE_AUDIT_SOURCE_ARCHIVE_INTEGRITY = "AUDIT_SOURCE_ARCHIVE_INTEGRITY" JOB_TYPE_SBAS_COREGISTRATION = "SBAS_COREGISTRATION" @@ -206,6 +209,142 @@ def _dedupe_existing_dirs(paths: Any) -> List[str]: return ordered +def _compact_failure_text(value: Any, *, max_length: int = 700) -> str: + text = re.sub(r"\s+", " ", str(value or "")).strip() + if not text: + return "Unknown error" + if len(text) <= max_length: + return text + return text[: max(0, max_length - 3)].rstrip() + "..." + + +def _classify_dinsar_failure(error_message: Any) -> str: + text = str(error_message or "") + lowered = text.lower() + if "3221225477" in text or "status_access_violation" in lowered: + return "LandSAR access violation during coherence mask/phase unwrapping" + if "not enough gcps" in lowered or "space insar calibration failed" in lowered: + return "Insufficient GCPs for baseline/calibration" + if ( + "no enough points" in lowered + or "not enough points" in lowered + or "geo_extract_gcp" in lowered + or "无满足snr" in lowered + or "离散采样点" in text + ): + return "Insufficient tie/GCP points for DEM/geocoding" + if "dem/sub-terrain" in lowered or "subterrain" in lowered or "sub-terrain" in lowered: + return "DEM/sub-terrain processing failed" + if "相干性掩膜" in text and "相位解缠" in text: + return "Coherence mask/phase unwrapping failed" + if "timeout" in lowered or "timed out" in lowered or "超时" in text: + return "Processing timeout" + if "publish" in lowered: + return "Result catalog publish failed" + compact = _compact_failure_text(text, max_length=120) + return compact if compact != "Unknown error" else "Unclassified D-InSAR failure" + + +async def _build_dinsar_failure_summary(db, run) -> Dict[str, Any]: + result = await db.execute( + select(DinsarProductionRunItemORM) + .where( + DinsarProductionRunItemORM.run_id == run.run_id, + DinsarProductionRunItemORM.status == "FAILED", + ) + .order_by( + DinsarProductionRunItemORM.order_index.asc().nullslast(), + DinsarProductionRunItemORM.id.asc(), + ) + ) + failed_items = result.scalars().all() + details: List[Dict[str, Any]] = [] + grouped: Dict[str, Dict[str, Any]] = {} + + for item in failed_items: + label = str(item.task_alias or item.task_name or f"item-{item.id}").strip() + reason = _classify_dinsar_failure(item.last_error) + compact_error = _compact_failure_text(item.last_error) + detail = { + "id": item.id, + "order_index": item.order_index, + "task_name": item.task_name, + "task_alias": item.task_alias, + "reason": reason, + "error": compact_error, + "source_task_dir": item.source_task_dir, + "latest_output_dir": item.latest_output_dir, + "latest_log_path": item.latest_log_path, + } + details.append(detail) + + group = grouped.setdefault(reason, {"reason": reason, "count": 0, "items": []}) + group["count"] += 1 + if len(group["items"]) < 25: + group["items"].append(label) + + groups = sorted(grouped.values(), key=lambda item: (-int(item["count"]), str(item["reason"]))) + return { + "failed_count": len(details), + "groups": groups, + "items": details, + } + + +def _chunk_log_lines(lines: List[str], *, max_chars: int = 3500) -> List[str]: + chunks: List[str] = [] + current: List[str] = [] + current_len = 0 + for line in lines: + line_len = len(line) + 1 + if current and current_len + line_len > max_chars: + chunks.append("\n".join(current)) + current = [] + current_len = 0 + current.append(line) + current_len += line_len + if current: + chunks.append("\n".join(current)) + return chunks + + +async def _log_dinsar_failure_summary( + *, + task_id: str, + run_id: str, + engine_title: str, + run, + failure_summary: Dict[str, Any], + run_log, +) -> None: + if not failure_summary or int(failure_summary.get("failed_count") or 0) <= 0: + return + + lines = [ + ( + f"{engine_title} D-InSAR failure summary: " + f"completed={run.completed_items} failed={run.failed_items} total={run.total_items}" + ), + "Failure groups:", + ] + for group in failure_summary.get("groups") or []: + items = ", ".join(str(item) for item in (group.get("items") or [])) + omitted = int(group.get("count") or 0) - len(group.get("items") or []) + suffix = f" (+{omitted} more)" if omitted > 0 else "" + lines.append(f"- {group.get('reason')}: {group.get('count')} item(s): {items}{suffix}") + + lines.append("Failed items:") + for item in failure_summary.get("items") or []: + order_index = item.get("order_index") + order_text = f"{order_index}/{run.total_items}" if order_index else f"id={item.get('id')}" + label = item.get("task_alias") or item.get("task_name") or f"item-{item.get('id')}" + lines.append(f"- [{order_text}] {label}: {item.get('reason')} | {item.get('error')}") + + for chunk in _chunk_log_lines(lines): + await task_service.add_log(task_id, "WARNING", chunk) + run_log(run_id, f"[failure-summary]\n{chunk}") + + async def _run_scan_data_custom(task_id: str, payload: Dict[str, Any]) -> None: from ..database import AsyncSessionLocal @@ -2086,6 +2225,7 @@ async def _run_dinsar_production_controller(job: SystemJobORM) -> None: f"failed={run.failed_items} total={run.total_items}" ) + failure_summary = await _build_dinsar_failure_summary(db, run) summary_payload = { "workflow": workflow, "engine_code": run.engine_code, @@ -2098,7 +2238,17 @@ async def _run_dinsar_production_controller(job: SystemJobORM) -> None: "publish": publish_result, "publish_error": publish_error, "published_output_dirs": publish_dirs, + "failure_summary": failure_summary, } + if failure_summary.get("failed_count"): + await _log_dinsar_failure_summary( + task_id=job.task_id, + run_id=run.run_id, + engine_title="ENVI", + run=run, + failure_summary=failure_summary, + run_log=run_log, + ) await dinsar_production_service.finalize_run( run, db=db, @@ -3061,6 +3211,7 @@ async def _run_wsl_dinsar_production_controller( f"failed={run.failed_items} total={run.total_items}" ) + failure_summary = await _build_dinsar_failure_summary(db, run) summary_payload = { "workflow": f"dinsar_{engine_code}", "engine_code": run.engine_code, @@ -3074,7 +3225,17 @@ async def _run_wsl_dinsar_production_controller( "rebuild": rebuild_result, "publish_error": publish_error, "published_output_dirs": publish_dirs, + "failure_summary": failure_summary, } + if failure_summary.get("failed_count"): + await _log_dinsar_failure_summary( + task_id=job.task_id, + run_id=run.run_id, + engine_title=engine_title, + run=run, + failure_summary=failure_summary, + run_log=run_log, + ) await dinsar_production_service.finalize_run( run, db=db, @@ -3266,6 +3427,211 @@ async def _handle_landsar_run(job: SystemJobORM) -> None: ) +async def _handle_landsar_lt1_import(job: SystemJobORM) -> None: + from .landsar_lt1_production_service import landsar_lt1_production_service + from .asset_inventory_service import asset_inventory_service + + payload = dict(job.payload or {}) + task_id = job.task_id + if not task_id: + raise ValueError("LANDSAR_LT1_IMPORT job missing task_id") + + loop = asyncio.get_running_loop() + + def _progress(event: Dict[str, Any]) -> None: + message = str(event.get("message") or event.get("event") or "").strip() + if not message: + return + progress = event.get("progress") + + async def _write() -> None: + try: + await task_service.add_log(task_id, "INFO", message) + if progress is not None: + await task_service.update_task(task_id, progress=int(progress), message=message) + except Exception: + logger.debug("Failed to write LandSAR LT-1 progress", exc_info=True) + + asyncio.run_coroutine_threadsafe(_write(), loop) + + await task_service.start_task(task_id, message="LandSAR LT-1 import started") + await task_service.update_task(task_id, progress=5, message="Checking LandSAR LT-1 runtime") + try: + source_asset_ids = _dedupe_positive_ints(payload.get("source_asset_ids")) + radar_data_ids = _dedupe_positive_ints(payload.get("radar_data_ids")) + if source_asset_ids or radar_data_ids: + await task_service.update_task(task_id, progress=8, message="Preparing LT-1 source assets") + prepared = await _prepare_landsar_lt1_source_assets( + task_id, + payload, + source_asset_ids=source_asset_ids, + radar_data_ids=radar_data_ids, + landsar_lt1_production_service=landsar_lt1_production_service, + asset_inventory_service=asset_inventory_service, + ) + payload = { + **payload, + "source_asset_ids": prepared["source_asset_ids"], + "radar_data_ids": prepared["radar_data_ids"], + "__prepared_scene_dirs": prepared["scene_dirs"], + "__materialized": prepared["materialized"], + "__materialize_task_root": prepared["task_root"], + } + + async with _local_engine_lock("landsar"): + result = await asyncio.to_thread( + landsar_lt1_production_service.run_import, + payload, + progress_callback=_progress, + ) + async with AsyncSessionLocal() as db: + catalog_result = await landsar_lt1_production_service.register_manifest( + db, + result["manifest_path"], + ) + await task_service.add_log( + task_id, + "INFO", + ( + "LandSAR LT-1 product registered: " + f"product_id={catalog_result.get('product_id')}, " + f"assets={catalog_result.get('asset_count')}" + ), + ) + await task_service.update_task( + task_id, + status="COMPLETED", + progress=100, + message=( + "LandSAR LT-1 import completed: " + f"product_id={catalog_result.get('product_id')}, " + f"Input_Data={result.get('input_data_dir')}" + ), + ) + except Exception as exc: + await task_service.add_log(task_id, "ERROR", f"LandSAR LT-1 import failed: {exc}") + await task_service.update_task( + task_id, + status="FAILED", + progress=100, + message=f"LandSAR LT-1 import failed: {exc}", + ) + raise + + +def _dedupe_positive_ints(values: Any) -> List[int]: + result: List[int] = [] + if not isinstance(values, list): + return result + for value in values: + try: + parsed = int(value) + except (TypeError, ValueError): + continue + if parsed > 0 and parsed not in result: + result.append(parsed) + return result + + +async def _prepare_landsar_lt1_source_assets( + task_id: str, + payload: Dict[str, Any], + *, + source_asset_ids: List[int], + radar_data_ids: List[int], + landsar_lt1_production_service: Any, + asset_inventory_service: Any, +) -> Dict[str, Any]: + from ..models import SourceProductAssetORM + + scene_dirs = _dedupe_existing_dirs(payload.get("scene_dirs") or []) + radar_ids: List[int] = [] + async with AsyncSessionLocal() as db: + if radar_data_ids: + rows = ( + await db.execute(select(RadarDataORM).where(RadarDataORM.id.in_(radar_data_ids))) + ).scalars().all() + radar_by_id = {int(row.id): row for row in rows} + for radar_id in radar_data_ids: + radar = radar_by_id.get(int(radar_id)) + if radar is None: + raise ValueError(f"Radar data not found: {radar_id}") + radar_ids.append(int(radar.id)) + if radar.source_product_ref_id and int(radar.source_product_ref_id) not in source_asset_ids: + source_asset_ids.append(int(radar.source_product_ref_id)) + elif radar.file_path and os.path.isdir(radar.file_path): + scene_dirs.append(os.path.normpath(os.path.abspath(radar.file_path))) + else: + raise ValueError(f"Radar data {radar_id} has no source asset or scene directory.") + + produced = await landsar_lt1_production_service.find_produced_source_asset_map(db, source_asset_ids) + if produced: + first_id = sorted(produced.keys())[0] + product_id = (produced[first_id] or {}).get("product_id") + raise ValueError(f"Source asset {first_id} already has a LandSAR LT-1 product: {product_id}") + + assets = [] + if source_asset_ids: + assets = ( + await db.execute(select(SourceProductAssetORM).where(SourceProductAssetORM.id.in_(source_asset_ids))) + ).scalars().all() + asset_by_id = {int(asset.id): asset for asset in assets} + + task_root = os.path.join( + os.path.normpath(os.path.abspath(settings.LANDSAR_WORK_ROOT)), + "lt1_import_tasks", + task_id, + "scenes", + ) + os.makedirs(task_root, exist_ok=True) + materialized: List[Dict[str, Any]] = [] + overwrite = bool(payload.get("materialize_overwrite", False)) + for asset_id in source_asset_ids: + asset = asset_by_id.get(int(asset_id)) + if asset is None: + raise ValueError(f"Source asset not found: {asset_id}") + if str(asset.satellite_family or "").upper() != "LT1": + raise ValueError(f"Source asset {asset_id} is not LT-1.") + result = await asyncio.to_thread( + asset_inventory_service.materialize_source_asset, + asset, + target_root=task_root, + overwrite=overwrite, + ) + target_dir = os.path.normpath(os.path.abspath(str(result.get("safe_dir") or result.get("target_dir") or ""))) + if not target_dir or not os.path.isdir(target_dir): + raise ValueError(f"Source asset {asset_id} materialize did not produce a directory.") + if target_dir not in scene_dirs: + scene_dirs.append(target_dir) + materialized.append( + { + "source_asset_id": int(asset.id), + "asset_uid": asset.asset_uid, + "logical_product_uid": asset.logical_product_uid, + "source_path": asset.file_path, + "scene_dir": target_dir, + "status": result.get("status"), + "member_count": result.get("member_count"), + } + ) + await task_service.add_log( + task_id, + "INFO", + f"Prepared LT-1 source asset {asset.id}: {target_dir} ({result.get('status')})", + ) + + scene_dirs = list(dict.fromkeys(scene_dirs)) + if not scene_dirs: + raise ValueError("No LT-1 scene directories were prepared.") + return { + "scene_dirs": scene_dirs, + "source_asset_ids": list(dict.fromkeys(source_asset_ids)), + "radar_data_ids": radar_ids, + "materialized": materialized, + "task_root": task_root, + } + + async def _handle_landsar_cluster_item(job: SystemJobORM) -> None: payload = job.payload or {} production_run_id = str(payload.get("production_run_id") or "").strip() @@ -4636,6 +5002,85 @@ async def _handle_gf3_sarscape_clean(job: SystemJobORM) -> None: await task_service.update_task(job.task_id, status=status, progress=100, message=message) +async def _handle_extract_dinsar_products(job: SystemJobORM) -> None: + if not job.task_id: + raise ValueError("EXTRACT_DINSAR_PRODUCTS requires task_id for progress tracking.") + + payload = job.payload or {} + root_dir = str(payload.get("root_dir") or "").strip() + dest_dir = payload.get("dest_dir") or None + if not root_dir: + raise ValueError("EXTRACT_DINSAR_PRODUCTS requires root_dir payload.") + + await task_service.start_task(job.task_id, message="正在提取 D-InSAR 位移结果...") + result = await asyncio.to_thread( + extract_disp_results, + root_dir, + dest_dir, + ) + await task_service.update_task( + job.task_id, + progress=45, + message=( + "D-InSAR 位移结果提取完成: " + f"processed={result.get('processed', 0)}, " + f"copied={result.get('copied', 0)}, " + f"overwritten={result.get('overwritten', 0)}, " + f"failed={result.get('failed', 0)}" + ), + ) + + target_dir = result.get("target_dir") + publish_roots = [target_dir] if target_dir and os.path.isdir(str(target_dir)) else [] + publish_result = None + rebuild_result = None + if publish_roots: + async with AsyncSessionLocal() as db: + await task_service.update_task( + job.task_id, + progress=60, + message="正在发布 D-InSAR 标准结果包...", + ) + publish_result = await result_catalog_service.publish_from_sources( + db, + publish_roots, + ) + if int(publish_result.get("processed", 0) or 0) > 0: + await task_service.update_task( + job.task_id, + progress=80, + message="正在重建 D-InSAR 结果目录索引...", + ) + rebuild_result = await result_catalog_service.rebuild_catalog( + db, + full_rebuild=True, + ) + + failed = int(result.get("failed", 0) or 0) + publish_failed = int((publish_result or {}).get("failed", 0) or 0) + rebuild_failed = int((rebuild_result or {}).get("failed", 0) or 0) + status = "FAILED" if (failed or publish_failed or rebuild_failed) else "COMPLETED" + message = ( + "D-InSAR 结果提取与登记完成: " + f"提取 {int(result.get('processed', 0) or 0)} 项, " + f"复制 {int(result.get('copied', 0) or 0)} 个, " + f"覆盖 {int(result.get('overwritten', 0) or 0)} 个" + ) + if publish_result is not None: + message += f", 发布 {int(publish_result.get('processed', 0) or 0)} 项" + if rebuild_result is not None: + message += f", 入库 {int(rebuild_result.get('registered', 0) or 0)} 项" + if status == "FAILED": + message += f", 失败 {failed + publish_failed + rebuild_failed} 项" + + await task_service.update_task( + job.task_id, + status=status, + progress=100, + message=message, + ) + + async def _handle_publish_dinsar_products_clean(job: SystemJobORM) -> None: if not job.task_id: raise ValueError("PUBLISH_DINSAR_PRODUCTS requires task_id for progress tracking.") @@ -5669,13 +6114,99 @@ async def _handle_sbas_landsar_workflow(job: SystemJobORM) -> None: ) +async def _handle_pairing_cache_rebuild(job: SystemJobORM) -> None: + if not job.task_id: + raise ValueError("PAIRING_CACHE_REBUILD requires task_id for progress tracking.") + + from .pairing_cache_service import pairing_cache_service + + payload = job.payload or {} + mode = str(payload.get("mode") or "auto_reconcile").strip().lower() + force_full = bool(payload.get("force_full", False)) + full_rebuild = mode in {"full", "full_rebuild"} or force_full + action_label = "full rebuild" if full_rebuild else "dirty reconcile" + + await task_service.start_task( + job.task_id, + message=f"D-InSAR pairing cache {action_label} started", + ) + await task_service.update_task( + job.task_id, + progress=5, + message=f"D-InSAR pairing cache {action_label} is running", + ) + progress_state: Dict[str, Any] = { + "progress": 5, + "message": f"D-InSAR pairing cache {action_label} is running", + } + + async def _report_progress(message: str, progress: int) -> None: + safe_progress = max(5, min(95, int(progress))) + progress_state["progress"] = safe_progress + progress_state["message"] = message + await task_service.update_task( + job.task_id, + progress=safe_progress, + message=message, + ) + + async def _keepalive() -> None: + while True: + await asyncio.sleep(60) + message = str(progress_state.get("message") or f"D-InSAR pairing cache {action_label} is still running") + progress = int(progress_state.get("progress") or 5) + await task_service.update_task( + job.task_id, + progress=progress, + message=f"{message} (still running)", + ) + + keepalive_task = asyncio.create_task(_keepalive()) + try: + async with AsyncSessionLocal() as db: + if full_rebuild: + result = await pairing_cache_service.rebuild_metric_cache( + db, + commit=True, + progress_callback=_report_progress, + ) + else: + result = await pairing_cache_service.reconcile_dirty_scenes( + db, + force_full=False, + commit=True, + progress_callback=_report_progress, + ) + finally: + keepalive_task.cancel() + try: + await keepalive_task + except asyncio.CancelledError: + pass + + await task_service.update_task( + job.task_id, + status="COMPLETED", + progress=100, + message=( + "D-InSAR pairing cache completed: " + f"mode={result.get('mode')}, " + f"scenes={result.get('scene_count', 0)}, " + f"pairs={result.get('pair_count', 0)}, " + f"dirty={result.get('dirty_scene_count', 0)}" + ), + ) + + _HANDLERS = { JOB_TYPE_SCAN_DATA: _handle_scan_data, JOB_TYPE_SCAN_ASSET_INVENTORY: _handle_scan_asset_inventory, JOB_TYPE_AUDIT_SOURCE_ARCHIVE_INTEGRITY: _handle_archive_integrity_audit, JOB_TYPE_SCAN_DINSAR: _handle_scan_dinsar, + JOB_TYPE_EXTRACT_DINSAR_PRODUCTS: _handle_extract_dinsar_products, JOB_TYPE_PUBLISH_DINSAR_PRODUCTS: _handle_publish_dinsar_products_clean, JOB_TYPE_REBUILD_DINSAR_CATALOG: _handle_rebuild_dinsar_catalog_clean, + JOB_TYPE_PAIRING_CACHE_REBUILD: _handle_pairing_cache_rebuild, JOB_TYPE_TIMESERIES_PREPARE: _handle_timeseries_prepare, JOB_TYPE_TIMESERIES_STACK_PREP: _handle_timeseries_stack_prep, JOB_TYPE_TIMESERIES_MATERIALIZE: _handle_timeseries_materialize, @@ -5701,6 +6232,7 @@ _HANDLERS = { JOB_TYPE_ISCE2_RUN: _handle_isce2_run, JOB_TYPE_PYINT_RUN: _handle_pyint_run, JOB_TYPE_LANDSAR_RUN: _handle_landsar_run, + JOB_TYPE_LANDSAR_LT1_IMPORT: _handle_landsar_lt1_import, JOB_TYPE_LANDSAR_CLUSTER_ITEM: _handle_landsar_cluster_item, JOB_TYPE_WATER_GEOCODE: _handle_water_geocode, JOB_TYPE_SAR_SCENE_PREPROCESS: _handle_sar_scene_preprocess, diff --git a/backend/app/services/job_queue_service.py b/backend/app/services/job_queue_service.py index 86352d8..4c93754 100644 --- a/backend/app/services/job_queue_service.py +++ b/backend/app/services/job_queue_service.py @@ -299,10 +299,11 @@ class JobQueueService: ) stale_jobs = result.scalars().all() if not stale_jobs: - return {"recovered": 0, "failed": 0} + return {"recovered": 0, "failed": 0, "failed_task_ids": []} recovered = 0 failed = 0 + failed_task_ids = [] for job in stale_jobs: attempts = int(job.attempts or 0) + 1 if attempts < int(job.max_attempts or 1): @@ -315,6 +316,8 @@ class JobQueueService: next_run_at = None failed += 1 finished_at = now + if job.task_id: + failed_task_ids.append(str(job.task_id)) await db.execute( update(SystemJobORM) @@ -331,7 +334,7 @@ class JobQueueService: ) ) await db.commit() - return {"recovered": recovered, "failed": failed} + return {"recovered": recovered, "failed": failed, "failed_task_ids": failed_task_ids} finally: if gen_db: await db.close() diff --git a/backend/app/services/job_worker.py b/backend/app/services/job_worker.py index 57ac57e..a0b5eb8 100644 --- a/backend/app/services/job_worker.py +++ b/backend/app/services/job_worker.py @@ -1,4 +1,5 @@ import asyncio +import json import os import socket import uuid @@ -50,20 +51,35 @@ def _new_session(): return database.AsyncSessionLocal() -async def _touch_worker(worker_id: str) -> None: +async def _touch_worker( + worker_id: str, + *, + concurrency: int, + allowed_job_types: Optional[Set[str]], +) -> None: host = socket.gethostname() pid = os.getpid() + note = json.dumps( + { + "concurrency": max(1, int(concurrency or 1)), + "allowed_job_types": sorted(allowed_job_types or []), + }, + ensure_ascii=False, + separators=(",", ":"), + ) async with _new_session() as db: stmt = pg_insert(SystemWorkerHeartbeatORM).values( worker_id=worker_id, hostname=host, pid=pid, + note=note, ) stmt = stmt.on_conflict_do_update( index_elements=["worker_id"], set_={ "hostname": host, "pid": pid, + "note": note, "last_seen": func.now(), }, ) @@ -172,7 +188,11 @@ async def run_worker_loop( now = time.monotonic() if now - last_heartbeat >= heartbeat_interval: try: - await _touch_worker(worker_id) + await _touch_worker( + worker_id, + concurrency=concurrency, + allowed_job_types=allowed_job_types, + ) except Exception as exc: print(f"[WARN] worker cleanup: {exc}") last_heartbeat = now @@ -184,6 +204,12 @@ async def run_worker_loop( f"[*] Recovered stale jobs: retry={recovered.get('recovered', 0)} " f"failed={recovered.get('failed', 0)}" ) + for task_id in recovered.get("failed_task_ids", []) or []: + await task_service.update_task( + task_id, + status="FAILED", + message="后台任务心跳超时，任务已被标记为失败", + ) except Exception as exc: print(f"[WARN] recover_stale: {exc}") last_recover = now diff --git a/backend/app/services/landsar_lt1_production_service.py b/backend/app/services/landsar_lt1_production_service.py new file mode 100644 index 0000000..0c1f8ad --- /dev/null +++ b/backend/app/services/landsar_lt1_production_service.py @@ -0,0 +1,1080 @@ +from __future__ import annotations + +import hashlib +import json +import mimetypes +import os +import re +import shutil +import subprocess +from dataclasses import dataclass, replace +from datetime import datetime, timezone +from typing import Any, Callable, Dict, Iterable, List, Optional + +from sqlalchemy import delete, func, or_, select +from sqlalchemy.ext.asyncio import AsyncSession + +from ..config import settings, split_env_paths +from ..dinsar_engines.landsar_engine import LandsarEngine +from ..models import RadarDataORM, ResultAssetORM, ResultCatalogStateORM, ResultProductORM, SARSceneGeoORM + + +LANDSAR_LT1_CATALOG = "lt1_landsar" +IMPORT_PROID = "100016" +ORBIT_PROID = "100206" +PROFILE_SCENE_IMPORT = "landsar.import.lt1.scene.v1" +PROFILE_STACK_IMPORT = "landsar.import.lt1.stack.v1" +PRODUCT_FAMILY_SCENE = "lt1_scene_import" +PRODUCT_FAMILY_STACK = "lt1_stack_import" + + +ProgressCallback = Callable[[Dict[str, Any]], None] + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc).replace(tzinfo=None) + + +def _utc_text(value: Optional[datetime] = None) -> str: + return (value or _utc_now()).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _norm_path(path: Any) -> str: + text = str(path or "").strip().strip('"') + if not text: + return "" + return os.path.normpath(os.path.abspath(os.path.expandvars(os.path.expanduser(text)))) + + +def _safe_key(value: Any, *, fallback: str = "lt1") -> str: + text = str(value or "").strip() + if not text: + text = fallback + text = re.sub(r"[^A-Za-z0-9_.-]+", "_", text).strip("._-") + return text[:96] or fallback + + +def _short_hash(values: Iterable[Any], *, length: int = 10) -> str: + digest = hashlib.sha256() + for value in values: + digest.update(str(value or "").encode("utf-8", errors="ignore")) + digest.update(b"\0") + return digest.hexdigest()[:length] + + +def _write_json(path: str, payload: Dict[str, Any]) -> None: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8", newline="\n") as fp: + json.dump(payload, fp, ensure_ascii=False, indent=2) + fp.write("\n") + + +def _read_text_tail(path: str, *, max_chars: int = 12000) -> str: + if not path or not os.path.isfile(path): + return "" + try: + with open(path, "rb") as fp: + fp.seek(0, os.SEEK_END) + size = fp.tell() + fp.seek(max(0, size - max_chars)) + return fp.read().decode("utf-8", errors="replace") + except OSError: + return "" + + +def _file_size(path: str) -> Optional[int]: + try: + return os.path.getsize(path) + except OSError: + return None + + +def _media_type(path: str) -> tuple[Optional[str], Optional[str]]: + ext = os.path.splitext(path)[1].lower().lstrip(".") + media, _ = mimetypes.guess_type(path) + if ext == "xml": + media = "application/xml" + elif ext == "json": + media = "application/json" + elif ext in {"tif", "tiff"}: + media = "image/tiff" + elif ext == "txt": + media = "text/plain" + return ext or None, media + + +def _success_in_logs(log_path: str, keywords: Iterable[str], exit_code: int) -> bool: + content = _read_text_tail(log_path, max_chars=80000).lower() + if any(str(keyword).lower() in content for keyword in keywords): + return True + return int(exit_code) == 0 and ("console success" in content or "success" in content) + + +def _run_console( + console_path: str, + param_file: str, + log_path: str, + *, + cwd: str, + timeout_seconds: int, +) -> Dict[str, Any]: + os.makedirs(os.path.dirname(log_path), exist_ok=True) + started_at = _utc_now() + env = dict(os.environ) + configured_runtime_dirs: List[str] = [] + for env_name in ("LANDSAR_RUNTIME_PATHS", "LANDSAR_DLL_DIRS"): + configured_runtime_dirs.extend( + _norm_path(item) + for item in str(os.environ.get(env_name, "") or "").split(os.pathsep) + if _norm_path(item) + ) + system_root = os.environ.get("SystemRoot", r"C:\Windows") + prepend_dirs = [ + os.path.dirname(_norm_path(console_path)), + _norm_path(getattr(settings, "LANDSAR_HOME", "")), + *configured_runtime_dirs, + os.path.join(system_root, "System32"), + os.path.join(system_root, "SysWOW64"), + system_root, + ] + current_path = env.get("PATH", "") + env["PATH"] = os.pathsep.join([path for path in prepend_dirs if path] + ([current_path] if current_path else [])) + command = [console_path, param_file] + with open(log_path, "a", encoding="utf-8", errors="replace") as log_fp: + log_fp.write(f"\n[{_utc_text(started_at)}] command: {' '.join(command)}\n") + log_fp.flush() + try: + proc = subprocess.run( + command, + cwd=cwd, + env=env, + stdout=log_fp, + stderr=subprocess.STDOUT, + text=True, + timeout=max(60, int(timeout_seconds or 3600)), + check=False, + ) + returncode = int(proc.returncode) + except subprocess.TimeoutExpired: + returncode = -9 + log_fp.write(f"\n[{_utc_text()}] timeout after {int(timeout_seconds or 3600)} seconds\n") + log_fp.write(f"\n[{_utc_text()}] returncode: {returncode}\n") + return { + "command": command, + "returncode": returncode, + "started_at": _utc_text(started_at), + "finished_at": _utc_text(), + "log_path": log_path, + "stdout_tail": _read_text_tail(log_path), + } + + +def generate_lt1_import_param_file( + filepath: str, + scene_dirs: List[str], + export_dir: str, + *, + sat_mode: str = "MONO", +) -> str: + if not scene_dirs: + raise ValueError("scene_dirs cannot be empty.") + mode = str(sat_mode or "MONO").strip().upper() + lines = [ + "卫星数据导入LT-1", + f"处理编号 {IMPORT_PROID}", + "设置数据导入形式_0文件夹导入_1数据导入 文件夹导入", + "读取成像参数文件_0否_1是 1", + "读取SLC数据文件_0否_1是 1", + "文件夹导入标识 TRUE", + f"文件夹导入个数 {len(scene_dirs)}", + ] + for idx, scene_dir in enumerate(scene_dirs, 1): + lines.append(f"文件夹{idx}路径 <{scene_dir}>") + lines.extend( + [ + "数据导入 FALSE", + f"输入卫星数据格式 {mode}", + "输入主影像成像参数文件路径 <>", + "输入主影像SLC数据文件路径 <>", + "输入主影像RPB数据文件路径 <>", + "输入辅影像成像参数文件路径 <>", + "输入辅影像SLC数据文件路径 <>", + "输入辅影像RPB数据文件路径 <>", + "设置数据导出目标路径_0原目录_1新目录 1", + f"设置输出文件目录 <{export_dir}>", + ] + ) + os.makedirs(os.path.dirname(filepath), exist_ok=True) + with open(filepath, "w", encoding="utf-8", newline="\n") as fp: + fp.write("\n".join(lines) + "\n") + return filepath + + +def generate_lt1_orbit_param_file( + filepath: str, + xml_paths: List[str], + orbit_dir: str, + output_dir: str, + *, + xml_save_mode: int = 0, + export_to_new: bool = False, +) -> str: + if not xml_paths: + raise ValueError("xml_paths cannot be empty.") + lines = [ + "LT-1精密轨道数据导入", + f"处理编号 {ORBIT_PROID}", + f"输入数据个数 {len(xml_paths)}", + ] + for idx, xml_path in enumerate(xml_paths, 1): + lines.append(f"输入数据{idx}的xml <{xml_path}>") + lines.extend( + [ + f"输入精密轨道数据文件夹 <{orbit_dir}>", + f"选择XML文件保存方式 {int(xml_save_mode)}", + f"设置数据导出目录形式0原目录1新目录 {1 if export_to_new else 0}", + f"输出更新处理后数据目录 <{output_dir}>", + ] + ) + os.makedirs(os.path.dirname(filepath), exist_ok=True) + with open(filepath, "w", encoding="utf-8", newline="\n") as fp: + fp.write("\n".join(lines) + "\n") + return filepath + + +@dataclass(frozen=True) +class Lt1ImportRequest: + scene_dirs: List[str] + source_asset_ids: List[int] + radar_data_ids: List[int] + materialize_assets: bool = True + materialize_overwrite: bool = False + mode: str = "scene" + task_name: Optional[str] = None + sat_mode: str = "MONO" + import_orbit: bool = False + orbit_dir: Optional[str] = None + timeout_seconds: int = 7200 + + +class LandsarLt1ProductionService: + def __init__(self) -> None: + self._engine = LandsarEngine() + + @property + def publish_root(self) -> str: + return _norm_path(os.path.join(settings.RESULT_PUBLISH_ROOT, LANDSAR_LT1_CATALOG)) + + def _console_path(self) -> str: + return _norm_path(getattr(settings, "LANDSAR_CONSOLE_EXE", "")) + + def _landsar_home(self) -> str: + return _norm_path(getattr(settings, "LANDSAR_HOME", "")) or os.path.dirname(self._console_path()) + + def check_capabilities(self) -> Dict[str, Any]: + availability = self._engine.check_available() + orbit_roots = [ + _norm_path(path) + for path in split_env_paths(getattr(settings, "ORBIT_SOURCE_DIRS", "")) + if _norm_path(path) + ] + return { + "catalog_name": LANDSAR_LT1_CATALOG, + "supported_profiles": [PROFILE_SCENE_IMPORT, PROFILE_STACK_IMPORT], + "proids": { + "import": IMPORT_PROID, + "orbit": ORBIT_PROID, + }, + "available": bool(getattr(availability, "available", False)), + "status": getattr(availability, "status", "unknown"), + "message": getattr(availability, "message", ""), + "checks": getattr(availability, "checks", []), + "console_path": self._console_path(), + "landsar_home": self._landsar_home(), + "publish_root": self.publish_root, + "orbit_roots": orbit_roots, + } + + def _normalize_request(self, payload: Dict[str, Any], *, allow_empty_scene_dirs: bool = False) -> Lt1ImportRequest: + raw_dirs = payload.get("scene_dirs") or payload.get("source_dirs") or [] + if not isinstance(raw_dirs, list): + raise ValueError("scene_dirs must be a list.") + scene_dirs: List[str] = [] + for raw in raw_dirs: + path = _norm_path(raw) + if not path: + continue + if path not in scene_dirs: + scene_dirs.append(path) + if not scene_dirs and not allow_empty_scene_dirs: + raise ValueError("scene_dirs cannot be empty.") + for path in scene_dirs: + if not os.path.isdir(path): + raise FileNotFoundError(f"LT-1 scene directory not found: {path}") + + source_asset_ids = self._normalize_id_list(payload.get("source_asset_ids")) + radar_data_ids = self._normalize_id_list(payload.get("radar_data_ids")) + planned_count = len(scene_dirs) + len(source_asset_ids) + len(radar_data_ids) + + mode = str(payload.get("mode") or ("stack" if planned_count > 1 else "scene")).strip().lower() + if mode not in {"scene", "stack"}: + raise ValueError("mode must be scene or stack.") + if mode == "scene" and planned_count != 1: + raise ValueError("scene mode requires exactly one scene directory or source asset.") + + orbit_dir = _norm_path(payload.get("orbit_dir")) + return Lt1ImportRequest( + scene_dirs=scene_dirs, + source_asset_ids=source_asset_ids, + radar_data_ids=radar_data_ids, + materialize_assets=bool(payload.get("materialize_assets", True)), + materialize_overwrite=bool(payload.get("materialize_overwrite", False)), + mode=mode, + task_name=str(payload.get("task_name") or "").strip() or None, + sat_mode=str(payload.get("sat_mode") or "MONO").strip().upper(), + import_orbit=bool(payload.get("import_orbit")), + orbit_dir=orbit_dir or None, + timeout_seconds=max(60, int(payload.get("timeout_seconds") or 7200)), + ) + + def _normalize_id_list(self, value: Any) -> List[int]: + if value in (None, ""): + return [] + if not isinstance(value, list): + raise ValueError("asset id fields must be lists.") + normalized: List[int] = [] + for raw in value: + try: + parsed = int(raw) + except (TypeError, ValueError): + continue + if parsed > 0 and parsed not in normalized: + normalized.append(parsed) + return normalized + + def preview_import(self, payload: Dict[str, Any]) -> Dict[str, Any]: + request = self._normalize_request(payload, allow_empty_scene_dirs=True) + scene_info = [self._inspect_scene_dir(path) for path in request.scene_dirs] + orbit_dir = request.orbit_dir or self._default_orbit_dir() + blockers: List[str] = [] + warnings: List[str] = [] + planned_scene_count = len(request.scene_dirs) + len(request.source_asset_ids) + len(request.radar_data_ids) + if not planned_scene_count: + blockers.append("No LT-1 scene directory or source asset was selected.") + for item in scene_info: + if not item["xml_count"]: + blockers.append(f"Missing LT-1 XML in {item['path']}") + if not item["slc_count"]: + blockers.append(f"Missing LT-1 SLC TIFF in {item['path']}") + if request.import_orbit and not orbit_dir: + blockers.append("import_orbit is enabled but no orbit directory is configured.") + elif request.import_orbit and orbit_dir and not os.path.isdir(orbit_dir): + blockers.append(f"Orbit directory not found: {orbit_dir}") + if request.mode == "stack" and planned_scene_count < 2: + warnings.append("Stack mode normally expects at least two scenes.") + return { + "allow_submit": not blockers, + "blockers": blockers, + "warnings": warnings, + "mode": request.mode, + "profile_code": PROFILE_STACK_IMPORT if request.mode == "stack" else PROFILE_SCENE_IMPORT, + "scene_count": planned_scene_count, + "directory_scene_count": len(request.scene_dirs), + "source_asset_count": len(request.source_asset_ids), + "radar_data_count": len(request.radar_data_ids), + "scenes": scene_info, + "sat_mode": request.sat_mode, + "import_orbit": request.import_orbit, + "orbit_dir": orbit_dir, + "publish_root": self.publish_root, + } + + def _inspect_scene_dir(self, path: str) -> Dict[str, Any]: + xmls: List[str] = [] + slcs: List[str] = [] + try: + with os.scandir(path) as entries: + for entry in entries: + if not entry.is_file(): + continue + lower = entry.name.lower() + if lower.startswith("lt1") and lower.endswith(".xml"): + xmls.append(entry.path) + elif lower.startswith("lt1") and lower.endswith((".tif", ".tiff")): + slcs.append(entry.path) + except OSError: + pass + return { + "path": path, + "name": os.path.basename(path), + "xml_count": len(xmls), + "slc_count": len(slcs), + "sample_xml": xmls[0] if xmls else "", + "sample_slc": slcs[0] if slcs else "", + } + + def _default_orbit_dir(self) -> str: + for raw in split_env_paths(getattr(settings, "ORBIT_SOURCE_DIRS", "")): + path = _norm_path(raw) + if path and os.path.isdir(path): + return path + return "" + + def _product_keys(self, request: Lt1ImportRequest) -> tuple[str, str, str]: + profile = PROFILE_STACK_IMPORT if request.mode == "stack" else PROFILE_SCENE_IMPORT + family = PRODUCT_FAMILY_STACK if request.mode == "stack" else PRODUCT_FAMILY_SCENE + if request.mode == "scene": + identity_key = _safe_key(os.path.basename(request.scene_dirs[0])) + else: + names = [_safe_key(os.path.basename(path)) for path in request.scene_dirs] + identity_key = _safe_key(f"stack_{len(names)}_{names[0]}_{names[-1]}_{_short_hash(request.scene_dirs)}") + return profile, family, identity_key + + def _ensure_runtime_ready(self) -> None: + availability = self._engine.check_available() + if not bool(getattr(availability, "available", False)): + raise RuntimeError(getattr(availability, "message", "") or "LandSAR console is unavailable.") + + def _find_imported_xmls(self, input_data_dir: str) -> List[str]: + matches: List[str] = [] + for root, _, files in os.walk(input_data_dir): + for name in files: + lower = name.lower() + if lower.startswith("lt1") and lower.endswith("_slc.xml"): + matches.append(os.path.join(root, name)) + return sorted(matches) + + def _copy_scene_summary(self, request: Lt1ImportRequest, run_dir: str) -> str: + target = os.path.join(run_dir, "source_scenes.json") + payload = { + "scene_count": len(request.scene_dirs), + "scene_dirs": request.scene_dirs, + "source_asset_ids": request.source_asset_ids, + "radar_data_ids": request.radar_data_ids, + "mode": request.mode, + "sat_mode": request.sat_mode, + "generated_at": _utc_text(), + } + _write_json(target, payload) + return target + + async def find_produced_source_asset_map( + self, + db: AsyncSession, + source_asset_ids: Iterable[int], + ) -> Dict[int, Dict[str, Any]]: + ids: List[int] = [] + for item in source_asset_ids: + try: + parsed = int(item or 0) + except (TypeError, ValueError): + continue + if parsed > 0 and parsed not in ids: + ids.append(parsed) + if not ids: + return {} + result = await db.execute( + select(RadarDataORM.source_product_ref_id, SARSceneGeoORM) + .join(SARSceneGeoORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id) + .where( + RadarDataORM.source_product_ref_id.in_(ids), + RadarDataORM.satellite_family == "LT1", + SARSceneGeoORM.status == "DONE", + SARSceneGeoORM.analysis_tif_path.isnot(None), + SARSceneGeoORM.analysis_engine == "lt_gamma", + SARSceneGeoORM.analysis_profile == "lt1_gamma_geocoded_mli", + ) + .order_by(SARSceneGeoORM.updated_at.desc().nullslast(), SARSceneGeoORM.id.desc()) + ) + wanted = set(ids) + produced: Dict[int, Dict[str, Any]] = {} + for source_id, scene in result.all(): + try: + parsed_source_id = int(source_id or 0) + except (TypeError, ValueError): + continue + if parsed_source_id not in wanted or parsed_source_id in produced: + continue + produced[parsed_source_id] = { + "product_db_id": scene.id, + "scene_id": scene.id, + "radar_data_id": scene.radar_data_id, + "product_id": f"sar_scene_geo:{scene.id}", + "product_family": "lt1_analysis_ready_geotiff", + "engine_code": scene.analysis_engine, + "profile_code": scene.analysis_profile, + "analysis_tif_path": scene.analysis_tif_path, + "analysis_dir": scene.analysis_dir, + "analysis_preview_path": scene.analysis_preview_path, + "published_at": scene.updated_at.isoformat() if scene.updated_at else None, + "native_output_dir": scene.analysis_dir, + } + return produced + + async def decorate_source_asset_payloads(self, db: AsyncSession, items: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + produced = await self.find_produced_source_asset_map( + db, + [int(item.get("id") or 0) for item in items], + ) + for item in items: + marker = produced.get(int(item.get("id") or 0)) + item["lt1_image_produced"] = bool(marker) + item["lt1_image_product"] = marker + item["lt1_landsar_produced"] = bool(marker) + item["lt1_landsar_product"] = marker + return items + + def run_import( + self, + payload: Dict[str, Any], + *, + progress_callback: Optional[ProgressCallback] = None, + ) -> Dict[str, Any]: + original_source_asset_ids = self._normalize_id_list(payload.get("source_asset_ids")) + original_radar_data_ids = self._normalize_id_list(payload.get("radar_data_ids")) + prepared_scene_dirs = [ + _norm_path(path) + for path in (payload.get("__prepared_scene_dirs") or []) + if _norm_path(path) + ] + if prepared_scene_dirs: + payload = { + **payload, + "scene_dirs": [*payload.get("scene_dirs", []), *prepared_scene_dirs], + "source_asset_ids": [], + "radar_data_ids": [], + } + request = self._normalize_request(payload, allow_empty_scene_dirs=True) + if prepared_scene_dirs: + request = replace( + request, + source_asset_ids=original_source_asset_ids, + radar_data_ids=original_radar_data_ids, + ) + if (request.source_asset_ids or request.radar_data_ids) and not request.scene_dirs: + raise ValueError("Source assets must be materialized before LandSAR LT-1 import starts.") + self._ensure_runtime_ready() + + profile, product_family, identity_key = self._product_keys(request) + started_at = _utc_now() + run_hash = _short_hash([started_at.isoformat(), *request.scene_dirs]) + run_key = f"run_{started_at.strftime('%Y%m%dT%H%M%SZ')}_{run_hash}" + product_prefix = "lt1_stack" if request.mode == "stack" else "lt1_scene" + readable_len = max(8, 64 - len(product_prefix) - len(run_hash) - 2) + product_id = _safe_key( + f"{product_prefix}_{identity_key[:readable_len]}_{run_hash}", + fallback="lt1_product", + )[:64] + + publish_dir = os.path.join(self.publish_root, product_family, identity_key, "runs", run_key) + native_dir = os.path.join(publish_dir, "native") + input_data_dir = os.path.join(native_dir, "Input_Data") + param_dir = os.path.join(publish_dir, "params") + log_dir = os.path.join(publish_dir, "logs") + os.makedirs(input_data_dir, exist_ok=True) + os.makedirs(param_dir, exist_ok=True) + os.makedirs(log_dir, exist_ok=True) + self._copy_scene_summary(request, publish_dir) + + console_path = self._console_path() + landsar_home = self._landsar_home() + timeout = int(request.timeout_seconds or 7200) + import_param = os.path.join(param_dir, f"{IMPORT_PROID}.txt") + import_log = os.path.join(log_dir, f"{IMPORT_PROID}_console.log") + generate_lt1_import_param_file( + import_param, + request.scene_dirs, + input_data_dir, + sat_mode=request.sat_mode, + ) + self._emit(progress_callback, "import_started", progress=10, message="LandSAR LT-1 import started") + import_run = _run_console( + console_path, + import_param, + import_log, + cwd=landsar_home, + timeout_seconds=timeout, + ) + import_ok = _success_in_logs( + import_log, + ["module [lt-1数据导入] success", "lt-1数据导入] success", "console success"], + int(import_run["returncode"]), + ) + imported_xmls = self._find_imported_xmls(input_data_dir) + if not import_ok or not imported_xmls: + raise RuntimeError( + "LandSAR LT-1 import failed or produced no Input_Data XML. " + f"returncode={import_run['returncode']}; log={import_log}" + ) + + orbit_result: Optional[Dict[str, Any]] = None + orbit_dir = request.orbit_dir or self._default_orbit_dir() + if request.import_orbit: + if not orbit_dir or not os.path.isdir(orbit_dir): + raise FileNotFoundError(f"Orbit directory not found: {orbit_dir or ''}") + orbit_param = os.path.join(param_dir, f"{ORBIT_PROID}.txt") + orbit_log = os.path.join(log_dir, f"{ORBIT_PROID}_console.log") + generate_lt1_orbit_param_file( + orbit_param, + imported_xmls, + orbit_dir, + input_data_dir, + xml_save_mode=0, + export_to_new=False, + ) + self._emit(progress_callback, "orbit_started", progress=60, message="LandSAR LT-1 orbit import started") + orbit_run = _run_console( + console_path, + orbit_param, + orbit_log, + cwd=landsar_home, + timeout_seconds=timeout, + ) + orbit_ok = _success_in_logs( + orbit_log, + ["module [lt-1精密轨道数据导入] success", "精密轨道数据导入] success", "console success"], + int(orbit_run["returncode"]), + ) + if not orbit_ok: + raise RuntimeError( + "LandSAR LT-1 precise orbit import failed. " + f"returncode={orbit_run['returncode']}; log={orbit_log}" + ) + orbit_result = { + **orbit_run, + "param_file": orbit_param, + "orbit_dir": orbit_dir, + "xml_count": len(imported_xmls), + } + + self._emit(progress_callback, "packaging_started", progress=85, message="Registering LT-1 product") + manifest_path = self._build_manifest( + request=request, + product_id=product_id, + profile=profile, + product_family=product_family, + identity_key=identity_key, + run_key=run_key, + publish_dir=publish_dir, + native_dir=native_dir, + input_data_dir=input_data_dir, + started_at=started_at, + import_result={**import_run, "param_file": import_param}, + orbit_result=orbit_result, + imported_xmls=imported_xmls, + materialized=payload.get("__materialized") or [], + task_root=payload.get("__materialize_task_root"), + ) + self._emit(progress_callback, "completed", progress=95, message="LT-1 LandSAR import completed") + return { + "product_id": product_id, + "manifest_path": manifest_path, + "publish_dir": publish_dir, + "native_output_dir": native_dir, + "input_data_dir": input_data_dir, + "run_key": run_key, + "profile_code": profile, + "product_family": product_family, + "imported_xml_count": len(imported_xmls), + "import_orbit": bool(orbit_result), + } + + def _emit(self, callback: Optional[ProgressCallback], event: str, **payload: Any) -> None: + if not callable(callback): + return + try: + callback({"event": event, **payload}) + except Exception: + return + + def _collect_assets(self, publish_dir: str, native_dir: str, manifest_path: str) -> List[Dict[str, Any]]: + assets: List[Dict[str, Any]] = [] + + def add(path: str, role: str, *, primary: bool = False, required: bool = False) -> None: + absolute = _norm_path(path) + if not absolute: + return + try: + relative = os.path.relpath(absolute, publish_dir) + except ValueError: + relative = absolute + fmt, media = _media_type(absolute) + assets.append( + { + "role": role, + "name": os.path.basename(absolute), + "relative_path": relative, + "absolute_path": absolute, + "format": fmt, + "media_type": media, + "is_required": required, + "is_primary": primary, + "exists": os.path.isfile(absolute), + "file_size": _file_size(absolute), + } + ) + + add(manifest_path, "manifest", required=True) + source_summary = os.path.join(publish_dir, "source_scenes.json") + add(source_summary, "source_summary", required=True) + for folder in (os.path.join(publish_dir, "params"), os.path.join(publish_dir, "logs")): + if not os.path.isdir(folder): + continue + for name in sorted(os.listdir(folder)): + path = os.path.join(folder, name) + if os.path.isfile(path): + add(path, "param" if name.lower().endswith(".txt") else "log", required=True) + + input_data_dir = os.path.join(native_dir, "Input_Data") + primary_set = False + for root, _, files in os.walk(input_data_dir): + for name in sorted(files): + lower = name.lower() + path = os.path.join(root, name) + if lower.endswith(".xml"): + add(path, "input_xml", primary=not primary_set, required=True) + primary_set = True + elif lower.endswith((".tif", ".tiff")): + add(path, "input_tif", primary=not primary_set) + primary_set = True + elif lower.endswith((".jpg", ".jpeg", ".png", ".webp")): + add(path, "preview") + return assets + + def _build_manifest( + self, + *, + request: Lt1ImportRequest, + product_id: str, + profile: str, + product_family: str, + identity_key: str, + run_key: str, + publish_dir: str, + native_dir: str, + input_data_dir: str, + started_at: datetime, + import_result: Dict[str, Any], + orbit_result: Optional[Dict[str, Any]], + imported_xmls: List[str], + materialized: List[Dict[str, Any]], + task_root: Optional[str], + ) -> str: + manifest_path = os.path.join(publish_dir, "manifest.json") + task_name = request.task_name or ( + os.path.basename(request.scene_dirs[0]) if request.mode == "scene" else f"LT-1 stack {len(request.scene_dirs)} scenes" + ) + summary = { + "scene_count": len(request.scene_dirs), + "imported_xml_count": len(imported_xmls), + "source_asset_ids": request.source_asset_ids, + "radar_data_ids": request.radar_data_ids, + "materialized_dirs": [item.get("scene_dir") for item in materialized if item.get("scene_dir")], + "materialize_task_root": task_root, + "input_data_dir": input_data_dir, + "sat_mode": request.sat_mode, + "import_orbit": bool(orbit_result), + "orbit_dir": (orbit_result or {}).get("orbit_dir"), + "import_returncode": import_result.get("returncode"), + "orbit_returncode": (orbit_result or {}).get("returncode"), + } + payload: Dict[str, Any] = { + "schema_version": "lt1_landsar.import.v1", + "catalog_name": LANDSAR_LT1_CATALOG, + "product_family": product_family, + "product_type": "landsar_input_data", + "product_id": product_id, + "display_name": task_name, + "task_name": task_name, + "identity": { + "mode": request.mode, + "scene_key": identity_key if request.mode == "scene" else None, + "stack_key": identity_key if request.mode == "stack" else None, + "run_key": run_key, + }, + "engine": { + "code": "landsar", + "console_path": self._console_path(), + "home": self._landsar_home(), + }, + "processor": { + "code": "landsar.lt1.import", + "profile_code": profile, + "import_proid": IMPORT_PROID, + "orbit_proid": ORBIT_PROID if orbit_result else None, + }, + "run": { + "started_at": _utc_text(started_at), + "finished_at": _utc_text(), + "status": "COMPLETED", + }, + "source": { + "scene_dirs": request.scene_dirs, + "source_asset_ids": request.source_asset_ids, + "radar_data_ids": request.radar_data_ids, + "materialized": materialized, + "native_output_dir": native_dir, + "publish_dir": publish_dir, + }, + "summary": summary, + "execution": { + "import": import_result, + "orbit": orbit_result, + }, + "assets": [], + } + _write_json(manifest_path, payload) + payload["assets"] = self._collect_assets(publish_dir, native_dir, manifest_path) + _write_json(manifest_path, payload) + execution_manifest_path = os.path.join(publish_dir, "execution_manifest.json") + shutil.copy2(manifest_path, execution_manifest_path) + return manifest_path + + async def register_manifest(self, db: AsyncSession, manifest_path: str) -> Dict[str, Any]: + normalized_manifest = _norm_path(manifest_path) + with open(normalized_manifest, "r", encoding="utf-8") as fp: + manifest = json.load(fp) + + product_id = str(manifest.get("product_id") or "").strip() + if not product_id: + raise ValueError("manifest.product_id is required.") + if str(manifest.get("catalog_name") or "") != LANDSAR_LT1_CATALOG: + raise ValueError("manifest is not a LandSAR LT-1 product manifest.") + + await db.execute(delete(ResultProductORM).where(ResultProductORM.product_id == product_id)) + + identity = manifest.get("identity") or {} + processor = manifest.get("processor") or {} + engine = manifest.get("engine") or {} + source = manifest.get("source") or {} + summary = manifest.get("summary") or {} + run = manifest.get("run") or {} + publish_dir = _norm_path(source.get("publish_dir")) or os.path.dirname(normalized_manifest) + native_dir = _norm_path(source.get("native_output_dir")) + assets = list(manifest.get("assets") or []) + primary_asset_path = "" + for asset in assets: + if asset.get("is_primary"): + primary_asset_path = _norm_path(asset.get("absolute_path")) + break + + product = ResultProductORM( + product_id=product_id, + catalog_name=LANDSAR_LT1_CATALOG, + product_family=str(manifest.get("product_family") or PRODUCT_FAMILY_SCENE), + product_type=str(manifest.get("product_type") or "landsar_input_data"), + display_name=str(manifest.get("display_name") or product_id), + task_name=str(manifest.get("task_name") or ""), + pair_key=None, + stack_key=identity.get("stack_key"), + run_key=identity.get("run_key"), + profile_code=processor.get("profile_code"), + engine_code=str(engine.get("code") or "landsar"), + processor_code=str(processor.get("code") or "landsar.lt1.import"), + package_schema=str(manifest.get("schema_version") or "lt1_landsar.import.v1"), + package_layout="lt1_landsar_import", + status="READY", + health_status="OK", + publish_dir=publish_dir, + manifest_path=normalized_manifest, + native_output_dir=native_dir, + primary_asset_path=primary_asset_path or None, + summary_json=summary, + tags_json={ + "catalog_name": LANDSAR_LT1_CATALOG, + "mode": identity.get("mode"), + "import_orbit": bool(summary.get("import_orbit")), + }, + produced_at=self._parse_time(run.get("finished_at")) or _utc_now(), + published_at=_utc_now(), + ) + db.add(product) + await db.flush() + + for asset in assets: + absolute = _norm_path(asset.get("absolute_path")) + fmt = asset.get("format") + media = asset.get("media_type") + if not fmt or not media: + fmt, media = _media_type(absolute) + db.add( + ResultAssetORM( + product_ref_id=product.id, + asset_role=str(asset.get("role") or "asset")[:32], + asset_name=str(asset.get("name") or os.path.basename(absolute) or "asset"), + relative_path=str(asset.get("relative_path") or ""), + absolute_path=absolute, + format=fmt, + media_type=media, + is_required=bool(asset.get("is_required")), + is_primary=bool(asset.get("is_primary")), + exists_flag=os.path.isfile(absolute), + file_size=_file_size(absolute), + ) + ) + + await self._update_catalog_state(db) + await db.commit() + return { + "product_db_id": product.id, + "product_id": product_id, + "asset_count": len(assets), + "manifest_path": normalized_manifest, + } + + def _parse_time(self, value: Any) -> Optional[datetime]: + text = str(value or "").strip() + if not text: + return None + try: + return datetime.strptime(text, "%Y-%m-%dT%H:%M:%SZ") + except ValueError: + return None + + async def _update_catalog_state(self, db: AsyncSession) -> None: + result = await db.execute( + select(ResultCatalogStateORM).where(ResultCatalogStateORM.catalog_name == LANDSAR_LT1_CATALOG) + ) + state = result.scalar_one_or_none() + count_result = await db.execute( + select(func.count(ResultProductORM.id)).where(ResultProductORM.catalog_name == LANDSAR_LT1_CATALOG) + ) + db_count = int(count_result.scalar_one() or 0) + if state is None: + state = ResultCatalogStateORM( + catalog_name=LANDSAR_LT1_CATALOG, + product_family="lt1", + storage_root=self.publish_root, + ) + db.add(state) + state.status = "READY" + state.needs_rebuild = False + state.storage_root = self.publish_root + state.db_count = db_count + state.manifest_count = db_count + state.last_message = f"LandSAR LT-1 catalog ready: {db_count} products" + state.last_incremental_scan_at = _utc_now() + + async def list_products( + self, + db: AsyncSession, + *, + limit: int = 100, + offset: int = 0, + status: Optional[str] = None, + query: Optional[str] = None, + ) -> Dict[str, Any]: + safe_limit = max(1, min(500, int(limit or 100))) + safe_offset = max(0, int(offset or 0)) + filters = [ResultProductORM.catalog_name == LANDSAR_LT1_CATALOG] + if status: + filters.append(ResultProductORM.status == str(status).strip().upper()) + if query: + like = f"%{str(query).strip()}%" + filters.append( + or_( + ResultProductORM.product_id.ilike(like), + ResultProductORM.display_name.ilike(like), + ResultProductORM.task_name.ilike(like), + ResultProductORM.stack_key.ilike(like), + ResultProductORM.run_key.ilike(like), + ) + ) + total_result = await db.execute(select(func.count(ResultProductORM.id)).where(*filters)) + total = int(total_result.scalar_one() or 0) + rows_result = await db.execute( + select(ResultProductORM) + .where(*filters) + .order_by(ResultProductORM.published_at.desc(), ResultProductORM.id.desc()) + .limit(safe_limit) + .offset(safe_offset) + ) + products = rows_result.scalars().all() + return { + "total": total, + "limit": safe_limit, + "offset": safe_offset, + "items": [self._serialize_product(product) for product in products], + } + + async def get_product_detail(self, db: AsyncSession, *, product_id: int) -> Optional[Dict[str, Any]]: + result = await db.execute( + select(ResultProductORM) + .where( + ResultProductORM.id == product_id, + ResultProductORM.catalog_name == LANDSAR_LT1_CATALOG, + ) + ) + product = result.scalar_one_or_none() + if product is None: + return None + assets_result = await db.execute( + select(ResultAssetORM) + .where(ResultAssetORM.product_ref_id == product.id) + .order_by(ResultAssetORM.is_primary.desc(), ResultAssetORM.asset_role.asc(), ResultAssetORM.id.asc()) + ) + detail = self._serialize_product(product) + detail["assets"] = [self._serialize_asset(asset) for asset in assets_result.scalars().all()] + return detail + + async def get_asset(self, db: AsyncSession, *, product_id: int, asset_id: int) -> Optional[ResultAssetORM]: + product_result = await db.execute( + select(ResultProductORM.id).where( + ResultProductORM.id == product_id, + ResultProductORM.catalog_name == LANDSAR_LT1_CATALOG, + ) + ) + product_db_id = product_result.scalar_one_or_none() + if product_db_id is None: + return None + asset_result = await db.execute( + select(ResultAssetORM).where( + ResultAssetORM.id == asset_id, + ResultAssetORM.product_ref_id == product_db_id, + ) + ) + return asset_result.scalar_one_or_none() + + def _serialize_product(self, product: ResultProductORM) -> Dict[str, Any]: + return { + "id": product.id, + "product_id": product.product_id, + "catalog_name": product.catalog_name, + "product_family": product.product_family, + "product_type": product.product_type, + "display_name": product.display_name, + "task_name": product.task_name, + "stack_key": product.stack_key, + "run_key": product.run_key, + "profile_code": product.profile_code, + "engine_code": product.engine_code, + "status": product.status, + "health_status": product.health_status, + "publish_dir": product.publish_dir, + "manifest_path": product.manifest_path, + "native_output_dir": product.native_output_dir, + "primary_asset_path": product.primary_asset_path, + "summary": product.summary_json or {}, + "tags": product.tags_json or {}, + "produced_at": product.produced_at.isoformat() if product.produced_at else None, + "published_at": product.published_at.isoformat() if product.published_at else None, + "registered_at": product.registered_at.isoformat() if product.registered_at else None, + } + + def _serialize_asset(self, asset: ResultAssetORM) -> Dict[str, Any]: + return { + "id": asset.id, + "role": asset.asset_role, + "name": asset.asset_name, + "relative_path": asset.relative_path, + "absolute_path": asset.absolute_path, + "format": asset.format, + "media_type": asset.media_type, + "is_required": asset.is_required, + "is_primary": asset.is_primary, + "exists": asset.exists_flag, + "file_size": asset.file_size, + } + + +landsar_lt1_production_service = LandsarLt1ProductionService() diff --git a/backend/app/services/lt_gamma_scene_service.py b/backend/app/services/lt_gamma_scene_service.py index dca94b1..4dae628 100644 --- a/backend/app/services/lt_gamma_scene_service.py +++ b/backend/app/services/lt_gamma_scene_service.py @@ -88,6 +88,15 @@ def run_lt_gamma_scene_preprocess( if not runner.is_file(): raise FileNotFoundError(f"Gamma scene runner not found: {runner}") + analysis_dem_path = ( + settings.SAR_ANALYSIS_DEM_PATH + or settings.GAMMA_SBAS_DEM_PATH + or settings.PYINT_PREPARED_DEM_PATH + or _prepared_dem_path() + ) + if not analysis_dem_path: + raise RuntimeError("SAR_ANALYSIS_DEM_PATH is not configured for LT analysis GeoTIFF production") + args = [ pyint_python, to_wsl_path(str(runner)), @@ -102,7 +111,11 @@ def run_lt_gamma_scene_preprocess( "--dem-root", to_wsl_path(str(settings.PYINT_DEM_ROOT)), "--prepared-dem-path", - to_wsl_path(_prepared_dem_path()), + to_wsl_path(str(analysis_dem_path)), + "--dem-resolution-m", + str(float(settings.SAR_ANALYSIS_DEM_RESOLUTION_M or 30.0)), + "--target-grid-size-m", + str(float(settings.SAR_ANALYSIS_TARGET_GRID_SIZE_M or 30.0)), "--project-name", run_name, "--date", @@ -110,9 +123,13 @@ def run_lt_gamma_scene_preprocess( "--satellite-family", "LT1", "--range-looks", - str(DEFAULT_RANGE_LOOKS), + str(int(settings.SAR_ANALYSIS_RANGE_LOOKS or DEFAULT_RANGE_LOOKS)), "--azimuth-looks", - str(DEFAULT_AZIMUTH_LOOKS), + str(int(settings.SAR_ANALYSIS_AZIMUTH_LOOKS or DEFAULT_AZIMUTH_LOOKS)), + "--speckle-filter-method", + str(settings.SAR_ANALYSIS_SPECKLE_FILTER_METHOD or "none"), + "--speckle-filter-size", + str(int(settings.SAR_ANALYSIS_SPECKLE_FILTER_SIZE or 5)), "--geo-interp", str(settings.PYINT_GEO_INTERP or "1"), "--nodata-value", diff --git a/backend/app/services/pairing_cache_service.py b/backend/app/services/pairing_cache_service.py index 3eeaaf7..6f34fb1 100644 --- a/backend/app/services/pairing_cache_service.py +++ b/backend/app/services/pairing_cache_service.py @@ -1,7 +1,7 @@ from __future__ import annotations from datetime import datetime -from typing import Any, Dict, List, Optional, Sequence +from typing import Any, Awaitable, Callable, Dict, List, Optional, Sequence from sqlalchemy import delete, func, or_, select, text, update from sqlalchemy.ext.asyncio import AsyncSession @@ -380,6 +380,15 @@ def _incremental_insert_sql() -> str: class PairingCacheService: + async def _notify_progress( + self, + progress_callback: Optional[Callable[[str, int], Awaitable[None]]], + message: str, + progress: int, + ) -> None: + if progress_callback is not None: + await progress_callback(message, progress) + async def _get_state_row(self, db: AsyncSession) -> PairingCacheStateORM: payload = await pairing_state_service.ensure_pairing_cache_state(db, commit=False) result = await db.execute( @@ -501,11 +510,24 @@ class PairingCacheService: db: AsyncSession, *, commit: bool = True, + progress_callback: Optional[Callable[[str, int], Awaitable[None]]] = None, ) -> Dict[str, Any]: await pairing_state_service.ensure_pairing_cache_state(db, commit=False) await self._set_state_rebuilding(db) + if commit: + await db.commit() try: + await self._notify_progress( + progress_callback, + "Pairing cache full rebuild: clearing old metric rows", + 10, + ) delete_result = await db.execute(delete(PairingMetricCacheORM)) + await self._notify_progress( + progress_callback, + "Pairing cache full rebuild: computing spatial/temporal metrics", + 20, + ) await db.execute( text(_full_rebuild_insert_sql()), { @@ -513,7 +535,17 @@ class PairingCacheService: "orientation_rule_version": pairing_state_service.orientation_rule_version, }, ) + await self._notify_progress( + progress_callback, + "Pairing cache full rebuild: resolving dirty scene markers", + 80, + ) resolved_dirty = await self._resolve_dirty_rows(db) + await self._notify_progress( + progress_callback, + "Pairing cache full rebuild: finalizing state", + 90, + ) summary = await self._finalize_state_success(db, full_rebuild=True) if commit: await db.commit() @@ -539,6 +571,7 @@ class PairingCacheService: *, force_full: bool = False, commit: bool = True, + progress_callback: Optional[Callable[[str, int], Awaitable[None]]] = None, ) -> Dict[str, Any]: await pairing_state_service.ensure_pairing_cache_state(db, commit=False) dirty_result = await db.execute( @@ -557,12 +590,21 @@ class PairingCacheService: pair_count=pair_count, force_full=force_full, ): - result = await self.rebuild_metric_cache(db, commit=commit) + result = await self.rebuild_metric_cache( + db, + commit=commit, + progress_callback=progress_callback, + ) result["trigger_dirty_scene_count"] = dirty_scene_count result["forced"] = force_full return result if dirty_scene_count == 0: + await self._notify_progress( + progress_callback, + "Pairing cache reconcile: no dirty scenes, refreshing state", + 80, + ) summary = await self._finalize_state_success(db, full_rebuild=False) if commit: await db.commit() @@ -582,13 +624,24 @@ class PairingCacheService: pair_count=pair_count, force_full=force_full, ): - result = await self.rebuild_metric_cache(db, commit=commit) + result = await self.rebuild_metric_cache( + db, + commit=commit, + progress_callback=progress_callback, + ) result["trigger_dirty_scene_count"] = dirty_scene_count result["forced"] = force_full return result await self._set_state_rebuilding(db) + if commit: + await db.commit() try: + await self._notify_progress( + progress_callback, + f"Pairing cache incremental reconcile: deleting stale rows for {dirty_scene_count} dirty scenes", + 20, + ) delete_result = await db.execute( delete(PairingMetricCacheORM).where( or_( @@ -600,7 +653,7 @@ class PairingCacheService: insert_attempts = 0 insert_sql = text(_incremental_insert_sql()) - for dirty_scene_id in dirty_scene_ids: + for index, dirty_scene_id in enumerate(dirty_scene_ids, start=1): insert_result = await db.execute( insert_sql, { @@ -610,8 +663,28 @@ class PairingCacheService: }, ) insert_attempts += int(insert_result.rowcount or 0) + if index == 1 or index == dirty_scene_count or index % 25 == 0: + progress = 20 + int(index / max(1, dirty_scene_count) * 55) + await self._notify_progress( + progress_callback, + ( + "Pairing cache incremental reconcile: " + f"processed {index}/{dirty_scene_count} dirty scenes" + ), + progress, + ) + await self._notify_progress( + progress_callback, + "Pairing cache incremental reconcile: resolving dirty scene markers", + 80, + ) resolved_dirty = await self._resolve_dirty_rows(db, scene_ids=dirty_scene_ids) + await self._notify_progress( + progress_callback, + "Pairing cache incremental reconcile: finalizing state", + 90, + ) summary = await self._finalize_state_success(db, full_rebuild=False) if commit: await db.commit() diff --git a/backend/app/services/sar_analysis_ready_service.py b/backend/app/services/sar_analysis_ready_service.py index aff70dd..4d98567 100644 --- a/backend/app/services/sar_analysis_ready_service.py +++ b/backend/app/services/sar_analysis_ready_service.py @@ -207,15 +207,20 @@ def _raster_quality(path: Path) -> dict[str, Any]: return quality +def _is_geographic_crs(crs_text: str) -> bool: + text = str(crs_text or "").upper() + return "4326" in text or "GEOGCS" in text or 'UNIT["DEGREE"' in text or "UNIT['DEGREE'" in text + + def _pixel_size_m_from_quality(quality: dict[str, Any]) -> float | None: try: transform = quality.get("transform") or [] xres = abs(float(transform[0])) yres = abs(float(transform[4])) - crs = str(quality.get("crs") or "").upper() + crs = str(quality.get("crs") or "") if not xres or not yres: return None - if crs and "4326" not in crs: + if crs and not _is_geographic_crs(crs): return round((xres + yres) / 2.0, 3) bounds = quality.get("bounds") or {} lat = (float(bounds.get("bottom", 0.0)) + float(bounds.get("top", 0.0))) / 2.0 diff --git a/backend/migrations/009_raw_source_pairing_fields.sql b/backend/migrations/009_raw_source_pairing_fields.sql index f0183e6..d956098 100644 --- a/backend/migrations/009_raw_source_pairing_fields.sql +++ b/backend/migrations/009_raw_source_pairing_fields.sql @@ -174,7 +174,24 @@ SET ) FROM radar_data m, radar_data s WHERE pmc.master_scene_ref_id = m.id - AND pmc.slave_scene_ref_id = s.id; + AND pmc.slave_scene_ref_id = s.id + AND ( + (pmc.scene_center_distance_meters IS NULL AND pmc.spatial_baseline_meters IS NOT NULL) + OR (pmc.master_satellite_family IS NULL AND m.satellite_family IS NOT NULL) + OR (pmc.slave_satellite_family IS NULL AND s.satellite_family IS NOT NULL) + OR (pmc.master_look_direction IS NULL AND m.look_direction IS NOT NULL) + OR (pmc.slave_look_direction IS NULL AND s.look_direction IS NOT NULL) + OR pmc.same_satellite_family IS DISTINCT FROM ( + NULLIF(COALESCE(m.satellite_family, m.satellite), '') IS NOT NULL + AND NULLIF(COALESCE(s.satellite_family, s.satellite), '') IS NOT NULL + AND COALESCE(m.satellite_family, m.satellite) = COALESCE(s.satellite_family, s.satellite) + ) + OR pmc.same_look_direction IS DISTINCT FROM ( + NULLIF(m.look_direction, '') IS NULL + OR NULLIF(s.look_direction, '') IS NULL + OR m.look_direction = s.look_direction + ) + ); UPDATE pairing_cache_state SET diff --git a/backend/tests/test_landsar_lt1_production_service.py b/backend/tests/test_landsar_lt1_production_service.py new file mode 100644 index 0000000..e1ae714 --- /dev/null +++ b/backend/tests/test_landsar_lt1_production_service.py @@ -0,0 +1,307 @@ +from pathlib import Path +import json +import re +import tempfile +import unittest +from datetime import datetime +from types import SimpleNamespace + +import backend.app.services.landsar_lt1_production_service as landsar_lt1_module +from backend.app.services.landsar_lt1_production_service import ( + IMPORT_PROID, + ORBIT_PROID, + generate_lt1_import_param_file, + generate_lt1_orbit_param_file, + landsar_lt1_production_service, +) + + +CN_PROCESS_ID = "\u5904\u7406\u7f16\u53f7" +CN_FOLDER_COUNT = "\u6587\u4ef6\u5939\u5bfc\u5165\u4e2a\u6570" +CN_FOLDER_PATH = "\u6587\u4ef6\u5939{index}\u8def\u5f84" +CN_OUTPUT_DIR = "\u8bbe\u7f6e\u8f93\u51fa\u6587\u4ef6\u76ee\u5f55" +CN_SAT_MODE = "\u8f93\u5165\u536b\u661f\u6570\u636e\u683c\u5f0f" +CN_DATA_COUNT = "\u8f93\u5165\u6570\u636e\u4e2a\u6570" +CN_DATA_XML = "\u8f93\u5165\u6570\u636e{index}\u7684xml" +CN_ORBIT_DIR = "\u8f93\u5165\u7cbe\u5bc6\u8f68\u9053\u6570\u636e\u6587\u4ef6\u5939" + + +def test_generate_lt1_import_param_file_single_scene(tmp_path: Path) -> None: + scene_dir = tmp_path / "LT1A_SCENE" + scene_dir.mkdir() + export_dir = tmp_path / "Input_Data" + param_file = tmp_path / "params" / "100016.txt" + + generate_lt1_import_param_file( + str(param_file), + [str(scene_dir)], + str(export_dir), + sat_mode="MONO", + ) + + content = param_file.read_text(encoding="utf-8") + assert f"{CN_PROCESS_ID} {IMPORT_PROID}" in content + assert f"{CN_FOLDER_COUNT} 1" in content + assert f"{CN_FOLDER_PATH.format(index=1)} <{scene_dir}>" in content + assert f"{CN_OUTPUT_DIR} <{export_dir}>" in content + assert f"{CN_SAT_MODE} MONO" in content + + +def test_generate_lt1_import_param_file_stack(tmp_path: Path) -> None: + scene_dirs = [] + for index in range(3): + scene_dir = tmp_path / f"LT1A_SCENE_{index}" + scene_dir.mkdir() + scene_dirs.append(str(scene_dir)) + param_file = tmp_path / "params" / "100016_stack.txt" + + generate_lt1_import_param_file( + str(param_file), + scene_dirs, + str(tmp_path / "Input_Data"), + sat_mode="BIST", + ) + + content = param_file.read_text(encoding="utf-8") + assert f"{CN_FOLDER_COUNT} 3" in content + assert f"{CN_SAT_MODE} BIST" in content + for index, scene_dir in enumerate(scene_dirs, 1): + assert f"{CN_FOLDER_PATH.format(index=index)} <{scene_dir}>" in content + + +def test_generate_lt1_orbit_param_file(tmp_path: Path) -> None: + xml_paths = [] + for index in range(2): + xml_path = tmp_path / f"LT1A_{index}_SLC.xml" + xml_path.write_text("", encoding="utf-8") + xml_paths.append(str(xml_path)) + orbit_dir = tmp_path / "orbit" + orbit_dir.mkdir() + param_file = tmp_path / "params" / "100206.txt" + + generate_lt1_orbit_param_file( + str(param_file), + xml_paths, + str(orbit_dir), + str(tmp_path / "Input_Data"), + ) + + content = param_file.read_text(encoding="utf-8") + assert f"{CN_PROCESS_ID} {ORBIT_PROID}" in content + assert f"{CN_DATA_COUNT} 2" in content + assert re.search(rf"{CN_ORBIT_DIR}\s+<{re.escape(str(orbit_dir))}>", content) + for index, xml_path in enumerate(xml_paths, 1): + assert f"{CN_DATA_XML.format(index=index)} <{xml_path}>" in content + + +def test_preview_blocks_scene_without_lt1_files(tmp_path: Path) -> None: + scene_dir = tmp_path / "EMPTY_SCENE" + scene_dir.mkdir() + + preview = landsar_lt1_production_service.preview_import( + { + "scene_dirs": [str(scene_dir)], + "mode": "scene", + "sat_mode": "MONO", + "import_orbit": False, + } + ) + + assert preview["allow_submit"] is False + assert any("Missing LT-1 XML" in item for item in preview["blockers"]) + assert any("Missing LT-1 SLC TIFF" in item for item in preview["blockers"]) + + +def test_preview_accepts_source_asset_only_scene() -> None: + preview = landsar_lt1_production_service.preview_import( + { + "source_asset_ids": [101], + "mode": "scene", + "sat_mode": "MONO", + "import_orbit": False, + } + ) + + assert preview["allow_submit"] is True + assert preview["scene_count"] == 1 + assert preview["directory_scene_count"] == 0 + assert preview["source_asset_count"] == 1 + + +def test_stack_preview_counts_source_assets() -> None: + preview = landsar_lt1_production_service.preview_import( + { + "source_asset_ids": [101, 102], + "mode": "stack", + "sat_mode": "MONO", + "import_orbit": False, + } + ) + + assert preview["allow_submit"] is True + assert preview["scene_count"] == 2 + assert not preview["warnings"] + + +def test_scene_mode_rejects_multiple_directories(tmp_path: Path) -> None: + left = tmp_path / "LEFT" + right = tmp_path / "RIGHT" + left.mkdir() + right.mkdir() + + try: + landsar_lt1_production_service.preview_import( + { + "scene_dirs": [str(left), str(right)], + "mode": "scene", + } + ) + except ValueError as exc: + assert "exactly one" in str(exc) + else: + raise AssertionError("scene mode accepted multiple directories") + + +class _ScalarRows: + def __init__(self, rows): + self._rows = rows + + def all(self): + return self._rows + + +class _ExecuteResult: + def __init__(self, rows): + self._rows = rows + + def scalars(self): + return _ScalarRows(self._rows) + + def all(self): + return self._rows + + +class _FakeAsyncDb: + def __init__(self, rows): + self.rows = rows + + async def execute(self, _stmt): + return _ExecuteResult(self.rows) + + +def test_find_produced_source_asset_map_matches_summary_source_ids() -> None: + scene = SimpleNamespace( + id=7, + radar_data_id=77, + analysis_engine="lt_gamma", + analysis_profile="lt1_gamma_geocoded_mli", + analysis_tif_path="D:/ready/analysis_ready.tif", + analysis_dir="D:/ready", + analysis_preview_path="D:/ready/preview.png", + updated_at=datetime(2026, 6, 27, 1, 2, 3), + ) + + async def _run(): + return await landsar_lt1_production_service.find_produced_source_asset_map( + _FakeAsyncDb([(12, scene)]), + [12, 13], + ) + + import asyncio + + produced = asyncio.run(_run()) + assert 12 in produced + assert produced[12]["product_id"] == "sar_scene_geo:7" + assert produced[12]["analysis_tif_path"] == "D:/ready/analysis_ready.tif" + assert 13 not in produced + + +def test_run_import_materialized_asset_does_not_double_count_scene(tmp_path: Path) -> None: + scene_dir = tmp_path / "LT1A_SCENE" + scene_dir.mkdir() + + service = landsar_lt1_production_service + original_ensure = service._ensure_runtime_ready + original_console = service._console_path + original_home = service._landsar_home + + original_publish_root = landsar_lt1_module.settings.RESULT_PUBLISH_ROOT + original_run_console = landsar_lt1_module._run_console + original_find_imported_xmls = service._find_imported_xmls + + def fake_run_console(_console_path, param_file, log_path, *, cwd, timeout_seconds): + Path(log_path).parent.mkdir(parents=True, exist_ok=True) + Path(log_path).write_text("console success\n", encoding="utf-8") + return { + "command": [_console_path, param_file], + "returncode": 0, + "started_at": "2026-06-27T00:00:00Z", + "finished_at": "2026-06-27T00:00:01Z", + "log_path": log_path, + "stdout_tail": "console success", + } + + try: + landsar_lt1_module.settings.RESULT_PUBLISH_ROOT = str(tmp_path / "publish") + service._ensure_runtime_ready = lambda: None + service._console_path = lambda: "LandSARConsole.exe" + service._landsar_home = lambda: str(tmp_path) + service._find_imported_xmls = lambda input_data_dir: [str(Path(input_data_dir) / "LT1A_SCENE_SLC.xml")] + landsar_lt1_module._run_console = fake_run_console + + result = service.run_import( + { + "source_asset_ids": [101], + "__prepared_scene_dirs": [str(scene_dir)], + "__materialized": [{"source_asset_id": 101, "scene_dir": str(scene_dir)}], + "__materialize_task_root": str(tmp_path / "tasks"), + "mode": "scene", + "sat_mode": "MONO", + "import_orbit": False, + } + ) + + manifest = json.loads(Path(result["manifest_path"]).read_text(encoding="utf-8")) + assert manifest["summary"]["scene_count"] == 1 + assert manifest["summary"]["source_asset_ids"] == [101] + assert manifest["source"]["scene_dirs"] == [str(scene_dir)] + finally: + landsar_lt1_module.settings.RESULT_PUBLISH_ROOT = original_publish_root + service._ensure_runtime_ready = original_ensure + service._console_path = original_console + service._landsar_home = original_home + service._find_imported_xmls = original_find_imported_xmls + landsar_lt1_module._run_console = original_run_console + + +class LandsarLt1ProductionServiceTests(unittest.TestCase): + def _with_tmp_path(self, fn) -> None: + with tempfile.TemporaryDirectory() as root: + fn(Path(root)) + + def test_generate_lt1_import_param_file_single_scene_unittest(self) -> None: + self._with_tmp_path(test_generate_lt1_import_param_file_single_scene) + + def test_generate_lt1_import_param_file_stack_unittest(self) -> None: + self._with_tmp_path(test_generate_lt1_import_param_file_stack) + + def test_generate_lt1_orbit_param_file_unittest(self) -> None: + self._with_tmp_path(test_generate_lt1_orbit_param_file) + + def test_preview_blocks_scene_without_lt1_files_unittest(self) -> None: + self._with_tmp_path(test_preview_blocks_scene_without_lt1_files) + + def test_preview_accepts_source_asset_only_scene_unittest(self) -> None: + test_preview_accepts_source_asset_only_scene() + + def test_stack_preview_counts_source_assets_unittest(self) -> None: + test_stack_preview_counts_source_assets() + + def test_scene_mode_rejects_multiple_directories_unittest(self) -> None: + self._with_tmp_path(test_scene_mode_rejects_multiple_directories) + + def test_find_produced_source_asset_map_matches_summary_source_ids_unittest(self) -> None: + test_find_produced_source_asset_map_matches_summary_source_ids() + + def test_run_import_materialized_asset_does_not_double_count_scene_unittest(self) -> None: + self._with_tmp_path(test_run_import_materialized_asset_does_not_double_count_scene) diff --git a/docs/DEM_PRODUCTION_SOURCE_CONTRACT_20260628.md b/docs/DEM_PRODUCTION_SOURCE_CONTRACT_20260628.md new file mode 100644 index 0000000..53f087e --- /dev/null +++ b/docs/DEM_PRODUCTION_SOURCE_CONTRACT_20260628.md @@ -0,0 +1,52 @@ +# DEM Production Source Contract (2026-06-28) + +## Decision + +New production should use `D:\DEM\SRTMDEM_RSP_SARscape` as the common DEM source family unless a task explicitly declares another DEM in its manifest. The system should not mix SRTM, COPDEM, GMTED, and the interpolated Heilongjiang 10 m DEM silently. + +This does not mean every engine reads the same physical file. The same SRTM source is maintained in several engine-compatible forms: + +| Role | Path | Format | Intended consumers | +| --- | --- | --- | --- | +| SARscape/ENVI source | `D:\DEM\SRTMDEM_RSP_SARscape` | ENVI/SARscape float32 binary with `.hdr` | `IDL_DINSAR_DEM_BASE_FILE`, `GF3_SARSCAPE_DEM_PATH` | +| WGS84 prepared source | `D:\DEM\SRTMDEM_RSP_SARscape.wgs84` | ENVI float32 binary/VRT-readable raster | `ISCE2_DEM_PATH`, `PYINT_PREPARED_DEM_PATH`, `SAR_ANALYSIS_DEM_PATH`, `GAMMA_SBAS_DEM_PATH`, `TIMESERIES_DEM_PATH` | +| Int16 GeoTIFF source | `D:\DEM\SRTMDEM_RSP_SARscape_global_int16.tif` | GeoTIFF int16 | `LANDSAR_DEM_PATH`, `LANDSAR_SBAS_DEM_PATH`, GDAL/RPC-style GeoTIFF consumers such as `GF3_GEO_DEM_PATH` | + +The LT-1 single-scene production profile uses a 30 m analysis grid. Product manifests must still record the selected SRTM-derived source file, the cropped/converted DEM path, the actual Gamma DEM spacing, the derived `dem_lat_ovr`/`dem_lon_ovr`, and the configured target grid so reviewers can distinguish the source DEM family from the output grid. + +The LT-1 single-scene profile also performs multilook and speckle filtering before registering the final analysis GeoTIFF. The production manifest must record `SAR_ANALYSIS_RANGE_LOOKS`, `SAR_ANALYSIS_AZIMUTH_LOOKS`, and the speckle filter method/window so reviewers can reproduce the output pixel statistics. + +## Current Configuration + +The current server should use: + +```env +IDL_DINSAR_DEM_BASE_FILE=D:\DEM\SRTMDEM_RSP_SARscape +GF3_SARSCAPE_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape + +ISCE2_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84 +PYINT_PREPARED_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84 +SAR_ANALYSIS_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84 +SAR_ANALYSIS_DEM_RESOLUTION_M=30.0 +SAR_ANALYSIS_TARGET_GRID_SIZE_M=30.0 +SAR_ANALYSIS_RANGE_LOOKS=6 +SAR_ANALYSIS_AZIMUTH_LOOKS=5 +SAR_ANALYSIS_SPECKLE_FILTER_ENABLED=true +SAR_ANALYSIS_SPECKLE_FILTER_METHOD=lee +SAR_ANALYSIS_SPECKLE_FILTER_SIZE=5 +GAMMA_SBAS_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84 +TIMESERIES_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84 + +LANDSAR_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape_global_int16.tif +LANDSAR_SBAS_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape_global_int16.tif +GF3_GEO_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape_global_int16.tif +``` + +## Non-Default DEMs + +- `D:\DEM\HeiLongJiang10M_DEM.tif` is a regional interpolated DEM. It is not the default production DEM. +- `D:\DEM\landsar_prepared\HeiLongJiang10M_DEM_full_4326_int16.tif` is a LandSAR-compatible regional derivative of the interpolated DEM. It should only be used by an explicitly named regional/high-resolution experiment. +- `D:\DEM\COPDEM_GLO30_China_4326_DEM` remains a possible China-coverage fallback, but it is not the default after this contract. +- `D:\DEM\GMTED2010.jp2` is too coarse for production geocoding and should not be used as a production DEM default. + +Any task that intentionally uses a non-default DEM must record the selected source path, derived/cropped path, DEM resolution, target output grid, and coverage decision in its manifest. diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 5bad103..30a3f71 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -112,7 +112,7 @@ ORBIT_POOL_LANDSAR= ```env IDL_EXECUTABLE=C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idl.exe IDL_WORKBENCH_PATH=C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlde.exe -IDL_DINSAR_DEM_BASE_FILE=D:\SRTM30m\SRTMDEM_RSP_SARscape +IDL_DINSAR_DEM_BASE_FILE=D:\DEM\SRTMDEM_RSP_SARscape IDL_WORKER_RUNTIME_DIR=D:\production_runtime\idl_worker ENVI_TASK_TIMEOUT_SECONDS=21600 ``` @@ -131,7 +131,7 @@ GF3_STORAGE_DIRS=D:\GaoFen3_Pool\catalog GF3_SARSCAPE_RUNTIME_DIR=D:\GaoFen3_Pool\task_pool\sarscape_runtime GF3_SARSCAPE_WRAPPER_EXE=D:\Code\Insar_management_system_v2\third_party\GF3_L1A_To_L2_pipeline\dist\windows\gf3wrapper.exe GF3_SARSCAPE_IDLRT_PATH=C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlrt.exe -GF3_SARSCAPE_DEM_PATH=D:\DEM\COPDEM_GLO30_China_4326_DEM +GF3_SARSCAPE_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape GF3_SARSCAPE_POLARIZATIONS=HH,HV GF3_SARSCAPE_AUTO_STANDARDIZE=false GF3_SARSCAPE_CLEAN_AFTER_SUCCESS=true diff --git a/docs/GF3_SARSCAPE_NATIVE_TO_GEOTIFF_DESIGN_20260530.md b/docs/GF3_SARSCAPE_NATIVE_TO_GEOTIFF_DESIGN_20260530.md index 89a430a..615b0a2 100644 --- a/docs/GF3_SARSCAPE_NATIVE_TO_GEOTIFF_DESIGN_20260530.md +++ b/docs/GF3_SARSCAPE_NATIVE_TO_GEOTIFF_DESIGN_20260530.md @@ -454,7 +454,7 @@ GF3_ARCHIVE_SOURCE_DIRS ```env GF3_SARSCAPE_WRAPPER_EXE=D:\Code\Insar_management_system_v2\third_party\GF3_L1A_To_L2_pipeline\dist\windows\gf3wrapper.exe GF3_SARSCAPE_IDLRT_PATH=C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlrt.exe -GF3_SARSCAPE_DEM_PATH=D:\DEM\GMTED2010.jp2 +GF3_SARSCAPE_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape GF3_SARSCAPE_POLARIZATIONS=HH,HV GF3_SARSCAPE_KEEP_EXTRACTED=true GF3_SARSCAPE_AUTO_STANDARDIZE=false diff --git a/docs/INDEX.md b/docs/INDEX.md index cf0d193..e214c05 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -31,14 +31,22 @@ 褰撳墠闄嗘帰涓€鍙枫€丼entinel-1銆侀珮鍒嗕笁鏈満鐢熶骇銆佹寜闇€瑙ｅ寘銆丟F3 澶栭儴鐢熶骇鐧昏銆佺粨鏋滅鐞嗗拰 UNC 閫€鍑虹害瀹氥€? - [PRODUCTION_RESULTS_MULTI_ENGINE_DESIGN_20260423.md](PRODUCTION_RESULTS_MULTI_ENGINE_DESIGN_20260423.md) 缁熶竴缁撴灉鐩綍銆佹爣鍑嗕骇鍝佸寘銆乧atalog 涓庡寮曟搸缁撴灉鍏卞瓨绾﹀畾銆? +- [RESULT_EXTRACTION_ACCESS_CONTROL_AUDIT_20260630.md](RESULT_EXTRACTION_ACCESS_CONTROL_AUDIT_20260630.md) + Result extraction and access-control audit: current D-InSAR export/registration boundaries, placeholder channels, admin/viewer limitations, and recommended exporter/operator/admin permission model. - [DINSAR_TASK_POOL_THREE_ENGINE_REFACTOR_20260614.md](DINSAR_TASK_POOL_THREE_ENGINE_REFACTOR_20260614.md) D-InSAR 淇濈暀 ENVI/SARscape銆丩andSAR銆丟amma/PyINT 涓夊紩鎿庯紝閫€鍑?ISCE2锛岀粺涓€ Task_Pool銆佺粨鏋滆仛鍚堝拰涓棿鏂囦欢娓呯悊鐨勫綋鍓嶈璁°€? - [LANDSAR_DEM_PREPARATION_CONTRACT_20260618.md](LANDSAR_DEM_PREPARATION_CONTRACT_20260618.md) LandSAR D-InSAR/SBAS 鐨勫叏鐞?DEM 涓€娆℃€?Int16 鏍囧噯鍖栥€佸尯鍩熻鍓?tif銆佺敓浜ч厤缃拰 guardrail 绾﹀畾銆? +- [DEM_PRODUCTION_SOURCE_CONTRACT_20260628.md](DEM_PRODUCTION_SOURCE_CONTRACT_20260628.md) + Production DEM source contract: SRTM-derived common source family, engine-specific derived formats, and non-default DEM guardrails. - [LANDSAR_CLUSTER_WORKER_DEPLOYMENT_20260624.md](LANDSAR_CLUSTER_WORKER_DEPLOYMENT_20260624.md) LandSAR D-InSAR 集群 worker 的队列分片设计、主服务器 IP 白名单、远端 Windows 节点 192.168.1.6 部署和运行约束。 - [LANDSAR_CLUSTER_DATA_TRANSPORT_DESIGN_20260625.md](LANDSAR_CLUSTER_DATA_TRANSPORT_DESIGN_20260625.md) LandSAR 集群数据搬运（HTTP Task_Pool 下载 + 结果回传）、Windows 集群运维（Task Scheduler 开机自启 + 心跳监控）。 +- [PRODUCTION_NODE_SUBSYSTEM_DESIGN_20260627.md](PRODUCTION_NODE_SUBSYSTEM_DESIGN_20260627.md) + D-InSAR 与 LT-1/Sentinel-1 单景影像生产的统一生产节点子系统设计，明确 LandSAR 集群 MVP、worker-only 部署、安全边界和后续本机/集群双模式路线。 +- [LANDSAR_LT1_SCENE_STACK_PRODUCTION_DESIGN_20260627.md](LANDSAR_LT1_SCENE_STACK_PRODUCTION_DESIGN_20260627.md) + LandSAR 陆探一号单景/多景生产 proID 与参数链探索，设计 `100016/100206` 导入产品化、结果 catalog、本机/集群双模式和后续影像产品验证路线。 - [UNC_SOURCE_ARCHIVE_AND_MATERIALIZE_DESIGN_20260615.md](UNC_SOURCE_ARCHIVE_AND_MATERIALIZE_DESIGN_20260615.md) LT-1/Sentinel-1 鏈湴婧愬帇缂╁寘绠＄悊銆佸寘鍐?XML/manifest 璧勪骇鍖栥€佹湰鍦?Task_Pool materialize锛屼互鍙?UNC 閫€鍑哄悗鐨勬湰鏈洪儴缃茶竟鐣屻€? - [SOURCE_ARCHIVE_INTEGRITY_AUDIT_20260620.md](SOURCE_ARCHIVE_INTEGRITY_AUDIT_20260620.md) diff --git a/docs/LANDSAR_LT1_SCENE_STACK_PRODUCTION_DESIGN_20260627.md b/docs/LANDSAR_LT1_SCENE_STACK_PRODUCTION_DESIGN_20260627.md new file mode 100644 index 0000000..6934d8f --- /dev/null +++ b/docs/LANDSAR_LT1_SCENE_STACK_PRODUCTION_DESIGN_20260627.md @@ -0,0 +1,642 @@ +# LandSAR 陆探一号单景/多景生产链路探索与系统设计（2026-06-27） + +## 1. 结论 + +LandSAR 不只承担 LT-1 D-InSAR。仓库内 LandSAR 工具和文档显示，LandSAR 已有一条明确的 LT-1 数据生产基础链： + +```text +LT-1 源数据 + -> 100016 LT-1 数据导入 + -> 100206 LT-1 精密轨道导入 + -> Task_*/Input_Data +``` + +这条链可以支撑两类非 D-InSAR 生产能力： + +1. **单景生产基础产品**：把一景 LT-1 源数据导入为 LandSAR 统一 `Input_Data` 格式，形成后续处理可复用的标准输入。 +2. **多景生产基础产品**：把同轨道、同模式、同极化的一组 LT-1 场景导入并注入精密轨道，形成 LandSAR 时序栈 `Task_TS_*/Input_Data`。 + +当前系统的问题不是 LandSAR 没能力，而是这条能力没有被后端产品化： + +- 现有后端只在 `LandsarEngine.run()` 的 D-InSAR 前置阶段调用 `100016`。 +- `100206` 精轨导入和多景导入逻辑主要沉淀在 `third_party/LandSAR/lt1_import_gui.py`，还没有进入后端生产服务。 +- 单景/多景导入结果还没有独立 catalog、运行记录和本机/集群双模式。 +- 地理编码、正射、强度图这类“业务影像产品”还需要进一步验证 LandSAR 的 `180016`、`180044`、`200016`、`200046` 等 proID 参数链，不能直接把 `100016` 导入结果命名为正射影像。 + +因此第一阶段建议先产品化： + +```text +landsar.import.lt1.scene.v1 +landsar.import.lt1.stack.v1 +landsar.orbit.lt1.v1 +``` + +第二阶段再在真实样例参数和运行验证基础上开放： + +```text +landsar.image.lt1.geocode.v1 +landsar.image.lt1.ortho.v1 +``` + +## 1.1 当前实现状态 + +截至 2026-06-27，系统已落地第一阶段的本机生产入口： + +- 后端新增 `LANDSAR_LT1_IMPORT` 任务类型。 +- 后端新增 `/api/landsar-lt1-production/*` API。 +- 已按 LandSAR GUI 中的真实参数格式生成 `100016.txt` 与可选 `100206.txt`。 +- 运行结果以 `catalog_name=lt1_landsar` 写入 `result_products/result_assets`。 +- 前端“生产管理 -> 陆探一工作台 -> 陆探一影像生产”已可提交单景或多景 LT-1 源资产/目录导入任务。 +- 选择 `source_product_assets` 中的 LT-1 源资产时，任务会先 materialize 到 `LANDSAR_WORK_ROOT/lt1_import_tasks//scenes/`，再把解包后的 scene 目录交给 LandSAR。 +- `manifest.summary.source_asset_ids` 会记录源资产 id；资产台账和检索列表根据 `result_products.summary_json.source_asset_ids` 下发 `lt1_landsar_produced` 标识。 +- 已经登记为 READY 的 LT-1 LandSAR 产品会在生产面板中禁止再次选择；后端任务执行前也会再次拒绝已生产源资产。 + +当前实现边界： + +- 输入支持资产台账中的 LT-1 源资产，或人工指定的已 materialize/解包 LT-1 scene 目录；人工目录无法可靠反查源资产去重状态。 +- 输出产品语义是 LandSAR `Input_Data` 标准化生产结果，不是正射影像、地理编码强度图或 D-InSAR 结果。 +- 集群执行尚未接入这一任务类型；当前先采用主服务器本机 LandSAR 执行，并通过同一 manifest/catalog 结构为后续生产节点适配保留边界。 + +## 2. 已确认的 LandSAR proID 与参数链 + +### 2.1 已有可复用参数生成器 + +当前仓库中 `third_party/LandSAR/lt1_import_gui.py` 已有以下参数生成器： + +| 能力 | proID | 现有函数 | 状态 | +| --- | --- | --- | --- | +| LT-1 数据导入 | `100016` | `generate_param_file()` | 已有，偏 master/slave 两目录 | +| LT-1 多景导入 | `100016` | `generate_lt1_multiscene_import_param_file()` | 已有，支持 `文件夹导入个数=N` | +| LT-1 精密轨道导入 | `100206` | `generate_orbit_param_file()` | 已有 | +| D-InSAR | `200014` | `generate_dinsar_param_file()` | 已接入后端 | +| SBAS 一体化 | `280039` | `generate_sbas_param_file()` | 已有参数生成器，运行授权/能力需另行验证 | +| PS-InSAR 分步 | `280000~280032` | `generate_psinsar_step_param_file()` | 已有参数生成器，运行链路需另行验证 | +| Stacking | `300001` | `generate_stacking_param_file()` | 已有参数生成器 | + +这说明 `100016 + 100206` 不是猜测能力，已经有明确参数格式、GUI 调用方式和成功判定逻辑。 + +### 2.2 已确认的第一阶段链路 + +#### `100016`: LT-1 数据导入 + +用途： + +- 从 LT-1 源 scene 目录读取 XML/TIFF。 +- 输出 LandSAR 统一格式 `Input_Data`。 +- 可用于单景，也可用于多景。 + +关键参数形态： + +```text +卫星数据导入LT-1 +处理编号 100016 +设置数据导入形式_0文件夹导入_1数据导入 文件夹导入 +读取成像参数文件_0否_1是 1 +读取SLC数据文件_0否_1是 1 +文件夹导入标识 TRUE +文件夹导入个数 N +文件夹1路径 +... +设置数据导出目标路径_0原目录_1新目录 1 +设置输出文件目录 +``` + +成功判定： + +- 日志包含 `module [LT-1数据导入] success`。 +- 或 `console success`。 +- 输出目录存在 `LT1*_SLC.xml` 和对应 `LT1*_SLC.tif`。 + +注意： + +- 当前后端 `backend/app/dinsar_engines/landsar_engine.py` 的 `_generate_import_param_file()` 写死 `文件夹导入个数 2`，适合 D-InSAR pair 前置导入。 +- 单景/多景产品化应改用 N 景参数生成逻辑，而不是复用 pair-shaped 参数。 + +#### `100206`: LT-1 精密轨道导入 + +用途： + +- 对 `Input_Data` 中的 LT-1 XML 注入或关联精密轨道。 +- 为后续 D-InSAR、SBAS、PS、Stacking 或影像处理提供已校正输入。 + +关键参数形态： + +```text +LT-1精密轨道数据导入 +处理编号 100206 +输入数据个数 N +输入数据1的xml +... +输入精密轨道数据文件夹 +选择XML文件保存方式 0 +设置数据导出目录形式0原目录1新目录 0 +输出更新处理后数据目录 +``` + +成功判定： + +- 日志包含 `module [LT-1精密轨道数据导入] success`。 +- 或日志同时包含 `精密轨道` 和 `success`。 +- 或 `console success`。 + +### 2.3 候选但未验证的影像产品链 + +LandSAR 文档列出了以下与单景影像产品相关的 proID： + +| proID | 功能 | 当前判断 | +| --- | --- | --- | +| `180044` | 多视处理 | 可能用于单景强度/幅度产品前置，但参数格式未在后端沉淀 | +| `180016` | 地理编码 | 可能用于单景地理编码产品，但缺少样例参数 | +| `180070` | SLC 处理 | 可能用于单景 SLC 派生处理，但语义需验证 | +| `200016` | 地理编码（流程） | 可能是一体化地理编码流程，但缺少样例参数 | +| `200046` | SLC 处理（流程） | 可能是一体化 SLC 处理流程，但缺少样例参数 | +| `280032` | SLC 数据多视 | PS/MTInSAR 链路中的多视步骤，不应直接等同于通用单景影像生产 | + +这些 proID 不能直接进入正式 UI。需要先从 LandSAR GUI 生成真实参数文件，拿一景样本跑通，再决定产品定义。 + +## 3. 产品能力分层 + +### 3.1 第一层：导入型生产产品 + +这是近期可落地层。 + +#### `landsar.import.lt1.scene.v1` + +```text +输入： + LT-1 单景源压缩包或 materialized scene 目录 + +处理： + 100016 LT-1 数据导入 + 可选 100206 LT-1 精密轨道导入 + +输出： + Input_Data/ + LT1*_SLC.xml + LT1*_SLC.tif + *.thumb.jpg + params/ + 100016.txt + 100206.txt + logs/ + 100016_console.log + 100206_console.log + import_manifest.json + +catalog: + catalog_name = lt1_landsar + product_family = lt1_scene_import + product_type = landsar_input_data +``` + +产品语义： + +- 这是 LandSAR 输入标准化产品。 +- 不是正射影像。 +- 不是地理编码强度图。 +- 可作为 D-InSAR、SBAS、PS、后续影像产品的上游缓存。 + +#### `landsar.import.lt1.stack.v1` + +```text +输入： + 同一轨道/模式/极化/方向的一组 LT-1 scene + +处理： + 100016 LT-1 多景导入 + 100206 LT-1 精密轨道导入 + +输出： + Task_TS______/ + Input_Data/ + Output_Data/ + stack_import_manifest.json + 时序数据构建报告.txt + +catalog: + catalog_name = lt1_landsar + product_family = lt1_stack_import + product_type = landsar_timeseries_input_data +``` + +产品语义： + +- 这是 LandSAR 多景时序输入产品。 +- 可作为 PS/SBAS/MT-InSAR 的上游准备结果。 +- 不直接表示形变结果。 + +### 3.2 第二层：影像型生产产品 + +这层需要先验证 LandSAR 影像处理 proID。 + +候选 profile： + +```text +landsar.image.lt1.multilook.v1 +landsar.image.lt1.geocode.v1 +landsar.image.lt1.ortho.v1 +``` + +可能链路： + +```text +landsar.import.lt1.scene.v1 + -> 180044 多视处理 + -> 180016 地理编码 +``` + +或： + +```text +landsar.import.lt1.scene.v1 + -> 200046 SLC 处理（流程） + -> 200016 地理编码（流程） +``` + +当前必须标记为待验证： + +- 缺少真实参数文件。 +- 缺少真实输出样例。 +- 缺少对输出单位、坐标系、辐射定标、nodata、分辨率的确认。 +- 缺少成功判定和错误摘要规则。 + +## 4. 系统架构设计 + +### 4.1 后端模块 + +建议新增独立模块，不放进 `dinsar_engines`： + +```text +backend/app/landsar_lt1/ + contracts.py + param_files.py + runtime.py + discovery.py + scene_import_adapter.py + stack_import_adapter.py + result_package.py +``` + +职责： + +- `contracts.py`：定义 scene/stack 输入 manifest、结果 manifest、能力描述。 +- `param_files.py`：沉淀 `100016`、`100206` 参数文件生成器，不再依赖 GUI 代码。 +- `runtime.py`：统一调用 `InSAR_Console.exe`、日志捕获、超时、成功判定、错误摘要。 +- `discovery.py`：从资产表或 materialized 目录解析 LT-1 scene，按轨道/极化/日期分组。 +- `scene_import_adapter.py`：执行 `landsar.import.lt1.scene.v1`。 +- `stack_import_adapter.py`：执行 `landsar.import.lt1.stack.v1`。 +- `result_package.py`：生成标准发布包、manifest、current 指针。 + +LandSAR D-InSAR 当前已有的 runtime 检查、授权服务启动、DLL 校验逻辑可以抽取共用，但不要把单景/多景生产继续塞进 `LandsarEngine.run()`。 + +### 4.2 调度模型 + +建议走统一生产节点协议： + +```text +主服务器创建 production run + -> 每个 scene 或 stack group 生成 production item + -> 本机 worker 或远端 production node 领取 + -> adapter 执行 LandSAR + -> 上传/登记标准产品包 +``` + +执行模式： + +| 模式 | 说明 | +| --- | --- | +| `local` | 主服务器本机执行 | +| `cluster` | 指定远端生产节点执行 | +| `auto` | 按节点能力、负载、数据缓存选择 | + +节点能力： + +```json +{ + "capabilities": [ + "landsar.import.lt1.scene.v1", + "landsar.import.lt1.stack.v1", + "landsar.orbit.lt1.v1" + ] +} +``` + +### 4.3 运行记录模型 + +现有 `dinsar_production_runs` 虽然有 `product_family` 字段，但表名、字段和 item 语义都偏 pair。为了避免技术债，不建议把单景/多景陆探生产继续塞进 D-InSAR run 表。 + +推荐新增通用生产表： + +```text +production_runs + run_id + product_family + processor_code + profile_code + execution_mode + source_scope + status + total_items + completed_items + failed_items + params_json + +production_run_items + item_id + run_id + item_key + item_type # scene / stack_group + source_asset_ids_json + source_paths_json + target_key # scene_uid / stack_key + status + latest_run_key + latest_manifest_path + metrics_json + +production_executions + execution_id + run_id + item_id + node_id + run_key + status + output_dir + manifest_path + log_path + error_message +``` + +也可以短期复用 `system_jobs` 做执行队列，但正式 UI、重试、批次统计、集群调度和结果追溯需要上述通用运行表。 + +### 4.4 结果 catalog + +现有 `result_products` / `result_assets` 表可以承载单景/多景产品，因为它们已经有： + +- `catalog_name` +- `product_family` +- `product_type` +- `engine_code` +- `processor_code` +- `profile_code` +- `stack_key` +- `summary_json` +- `assets` + +但现有 `result_catalog_service._load_manifest()` 明确拒绝非 `dinsar` 的 manifest。需要拆出通用结果登记服务： + +```text +result_package_registry + register_manifest(manifest_path) + validate_manifest(product_family) + upsert_result_product() + upsert_result_assets() +``` + +LandSAR LT-1 生产建议使用： + +```text +catalog_name = lt1_landsar +product_family = lt1_scene_import / lt1_stack_import +product_type = landsar_input_data / landsar_timeseries_input_data +engine_code = landsar +processor_code = landsar.import.lt1.scene / landsar.import.lt1.stack +profile_code = landsar.import.lt1.scene.v1 / landsar.import.lt1.stack.v1 +``` + +## 5. 输入与分组设计 + +### 5.1 单景输入 + +来源： + +- `source_product_assets` 中的 `LT1_ARCHIVE`。 +- 已 materialized 的 LT-1 scene 目录。 +- 人工指定的受控服务器目录。 + +流程： + +```text +source asset + -> materialize scene + -> run 100016 + -> optional run 100206 + -> publish result package +``` + +### 5.2 多景输入 + +分组键建议： + +```text +satellite_family = LT1 +track / orbit number +imaging_mode +polarization +orbit_direction +product_type = SLC +admin/aoi 或用户选择范围 +date range +``` + +多景生产不应简单把用户勾选的所有 LT-1 都丢给 LandSAR。需要先做分组预检： + +- 是否同一轨道或可构成同一时序栈。 +- 是否同一极化。 +- 是否同一成像模式。 +- 日期是否可解析。 +- 是否都有源包。 +- 精轨是否匹配。 +- scene footprint 是否满足业务区域覆盖要求。 + +输出 `stack_key` 示例： + +```text +lt1_stack_______ +``` + +## 6. 发布目录设计 + +```text +D:\production_results\lt1_landsar\ + scene_import\ + \ + current\ + landsar.import.lt1.scene.v1.json + runs\ + \ + manifest.json + execution_manifest.json + native\ + Input_Data\ + params\ + logs\ + assets\ + input_data\ + thumb\ + metadata\ + + stack_import\ + \ + current\ + landsar.import.lt1.stack.v1.json + runs\ + \ + manifest.json + execution_manifest.json + native\ + Task_TS_...\Input_Data\ + Task_TS_...\Output_Data\ + params\ + logs\ + assets\ + input_data_manifest.json + stack_report.txt + scene_index.json +``` + +`native/` 保留 LandSAR 原始结构，`assets/` 放系统标准化索引文件和必要缩略图。不要把完整 `Input_Data` 再复制两份；标准资产层可以用 manifest 指向 native 内的相对路径。 + +## 7. API 与前端入口 + +### 7.1 API + +建议新增： + +```text +GET /api/landsar-lt1-production/capabilities +POST /api/landsar-lt1-production/preview +POST /api/landsar-lt1-production/run +GET /api/landsar-lt1-production/products +GET /api/landsar-lt1-production/products/{product_db_id} +GET /api/landsar-lt1-production/products/{product_db_id}/assets/{asset_id} +``` + +`preview` 必须先返回可执行性： + +- 选中 scene 数。 +- 分组结果。 +- 缺失源包。 +- 缺失精轨。 +- 预计输出目录。 +- 是否可本机执行。 +- 是否有集群节点支持。 +- 源资产是否已经存在 READY 状态的 LT-1 LandSAR 产品。 + +### 7.2 前端 + +生产管理里将 `陆探一生产占位` 改成实际工作台： + +```text +陆探一生产 + ├─ 单景导入 + ├─ 多景时序输入构建 + ├─ 运行记录 + └─ 产品结果 +``` + +第一版按钮只开放： + +- 预检。 +- 提交单景导入。 +- 提交多景导入。 +- 查看日志。 +- 打开结果目录。 +- 查看 catalog 资产。 + +不要第一版就开放： + +- 正射产品。 +- 地理编码强度图。 +- PS/SBAS 自动执行。 +- Stacking 自动执行。 + +这些应建立在第一层 `Input_Data` 产品稳定之后。 + +## 8. 集群化设计 + +单景/多景导入非常适合纳入生产节点： + +- 输入大但结构明确。 +- 输出可通过 manifest 回传。 +- 不需要主服务器承担长时间 LandSAR 进程。 +- `Input_Data` 可作为远端缓存，后续 D-InSAR/SBAS 复用。 + +远端节点能力： + +```text +landsar.import.lt1.scene.v1 +landsar.import.lt1.stack.v1 +landsar.orbit.lt1.v1 +``` + +第一版并发建议： + +- LandSAR 进程并发：每节点 1。 +- 多 scene 导入内部由 LandSAR 控制，不在外层并发拆太碎。 +- 多个单景任务可以排队，但不要同节点同时启动多个 `InSAR_Console.exe`，除非实测证明安全。 + +数据搬运： + +- 主服务器给 input manifest。 +- worker 下载源包或 materialized scene。 +- worker 执行 `100016/100206`。 +- worker 上传 manifest 和必要产物。 +- 大体量 `Input_Data` 是否全量回传可配置：第一版建议回传完整受管产品；后续可做远端缓存引用。 + +## 9. 实施路线 + +### 阶段 1：参数链固化 + +- 从 `third_party/LandSAR/lt1_import_gui.py` 抽取 `100016`、`100206` 参数生成逻辑。 +- 增加参数文件 golden tests。 +- 不接 UI，不跑真实任务。 + +### 阶段 2：本机单景导入 + +- 实现 `landsar.import.lt1.scene.v1` adapter。 +- 输入一个 LT-1 scene。 +- 执行 `100016`。 +- 可选执行 `100206`。 +- 生成标准 manifest。 +- 登记到 `result_products`，catalog 为 `lt1_landsar`。 + +### 阶段 3：本机多景导入 + +- 实现分组预检。 +- 实现 `landsar.import.lt1.stack.v1` adapter。 +- 复用 `generate_lt1_multiscene_import_param_file()` 语义。 +- 执行 `100016 -> 100206`。 +- 发布 `Task_TS_*/Input_Data` 产品。 + +### 阶段 4：生产节点接入 + +- 把 scene/stack import adapter 接入生产节点协议。 +- 远端节点上报能力。 +- 实现输入下载、结果回传、日志上报。 + +### 阶段 5：影像产品验证 + +- 用 LandSAR GUI 对单景生成多视、地理编码或正射产品。 +- 收集真实参数文件。 +- 确认 proID、输出命名、坐标系、单位和成功判定。 +- 再实现 `landsar.image.lt1.*`。 + +## 10. 需要运行验证的问题 + +1. `100016` 单景导入时 `文件夹导入个数=1` 是否被当前 LandSAR runtime 接受。 +2. `100016` 多景导入最大稳定 scene 数是多少。 +3. `100206` 对已导入 XML 是原地改写还是复制输出。 +4. `100206` 精轨注入后 XML 中可验证字段是什么。 +5. `180044 -> 180016` 是否能独立从单景 `Input_Data` 生成地理编码影像。 +6. `200046/200016` 是否比单步 `180xxx` 更适合作为正式影像产品链。 +7. LandSAR 同一台机器是否允许多个导入任务并发。 +8. 导入输出是否可跨机器复用，还是强依赖本机路径。 + +## 11. 近期不建议做的事 + +- 不建议把 `100016` 单景导入继续藏在 D-InSAR 前置步骤里。 +- 不建议把 `100016` 输出命名为正射或地理编码产品。 +- 不建议把单景/多景生产塞进 `dinsar_production_runs`。 +- 不建议在未验证 `180016/200016` 参数前开放“LandSAR 正射生产”按钮。 +- 不建议先做复杂 UI。应先做 adapter、manifest、catalog、真实样例验证。 diff --git a/docs/LT1_GEOCODED_GEOTIFF_PRODUCTION_20260628.md b/docs/LT1_GEOCODED_GEOTIFF_PRODUCTION_20260628.md new file mode 100644 index 0000000..7423f47 --- /dev/null +++ b/docs/LT1_GEOCODED_GEOTIFF_PRODUCTION_20260628.md @@ -0,0 +1,68 @@ +# LT-1 Geocoded GeoTIFF Production Update (2026-06-28) + +## Decision + +The LT-1 production button must produce a usable geocoded GeoTIFF, not only prepare LandSAR `Input_Data` for later D-InSAR work. + +The current implemented production path is: + +```text +LT-1 source asset + -> radar_data record + -> SAR_SCENE_PREPROCESS job + -> lt_gamma single-scene pipeline + -> SARSceneGeoORM.analysis_tif_path = analysis_ready.tif +``` + +This is the platform's real LT-1 single-scene backscatter image product for now. It performs the Gamma single-scene preprocessing chain: LT source product to Gamma SLC, multilook amplitude, geocode, speckle filtering in the linear-power domain, dB conversion, and analysis-ready GeoTIFF registration. + +The chain must not silently fall back to unrelated DEM sources. The default production contract is SRTM-derived on the current server: + +- `SAR_ANALYSIS_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84` +- `SAR_ANALYSIS_TARGET_GRID_SIZE_M=30.0` +- `SAR_ANALYSIS_DEM_RESOLUTION_M=30.0` +- `SAR_ANALYSIS_RANGE_LOOKS=6` +- `SAR_ANALYSIS_AZIMUTH_LOOKS=5` +- `SAR_ANALYSIS_SPECKLE_FILTER_ENABLED=true` +- `SAR_ANALYSIS_SPECKLE_FILTER_METHOD=lee` +- `SAR_ANALYSIS_SPECKLE_FILTER_SIZE=5` +- `PYINT_GEO_INTERP=1` + +The runtime clips the configured SRTM-derived DEM to the scene footprint with a small margin, converts that clip to Gamma DEM format, runs `generate_rdc_dem.py`, geocodes the multilooked amplitude with `geocode_back`, exports a GeoTIFF with `data2geotiff`, applies a Lee speckle filter to the linear power raster, then converts power to dB. The output `pixel_size_m` stored in `sar_scene_geo` is derived from the GeoTIFF transform; WGS84 degree grids are converted to approximate meters before registration. + +Multilook is already part of the Gamma preprocessing path through `generate_rdc_dem.py`; it is controlled by `SAR_ANALYSIS_RANGE_LOOKS` and `SAR_ANALYSIS_AZIMUTH_LOOKS`. Speckle filtering is a separate post-geocode raster operation before dB conversion. The filter manifest records the method, window size, domain, and equivalent number of looks used for the Lee weighting. + +For SRTM-derived DEMs, the configured 30 m analysis grid must be translated into Gamma `dem_lat_ovr` and `dem_lon_ovr` from the actual converted `.dem.par` spacing. Do not assume that `SAR_ANALYSIS_DEM_RESOLUTION_M=30.0` alone changes the output GeoTIFF grid. A 3 arc-second source DEM with `dem_lat_ovr=1` and `dem_lon_ovr=1` will still produce about 90 m latitude spacing. The runner now reads `post_lat`, `post_lon`, and scene latitude from the converted Gamma DEM parameter file, then writes the derived oversampling into the template before `generate_rdc_dem.py`. + +The previous `D:\DEM\HeiLongJiang10M_DEM.tif` default is not the production default. It is an interpolated regional DEM and must not be mixed silently with SRTM-derived products. If a future production profile intentionally uses it, the selected DEM and output grid must be recorded in the product manifest. + +## What Changed + +- `/api/landsar-lt1-production/run` queues `SAR_SCENE_PREPROCESS` jobs with `engine=lt_gamma`. +- Each selected LT-1 source asset resolves to a `radar_data` scene and produces one independent `SARSceneGeoORM` record. +- Batch mode means batch single-scene GeoTIFF production. It does not create a D-InSAR pair/stack product. +- The product list under `/api/landsar-lt1-production/products` reads from `sar_scene_geo`, not from `result_products.catalog_name=lt1_landsar`. +- Asset inventory and radar search mark LT-1 assets as produced only when `sar_scene_geo.status = DONE` and `analysis_tif_path` exists for the LT-1 `lt_gamma` profile. + +## UI And Query Contract + +- The LT-1 GeoTIFF production UI does not accept arbitrary LT-1 scene directories. Operators must select scanned source assets so each output can be linked back to `radar_data` and `sar_scene_geo`. +- The UI does not expose a satellite-mode/BIST selector. Acquisition mode should come from scanned source metadata, and the current Gamma single-scene pipeline does not require the operator to choose LandSAR import mode manually. +- The production candidate list reuses `/radar-data/search` with `satellite_family=LT1` and `source_format=LT1_ARCHIVE`, so operators can plan production by acquisition date, administrative AOI, orbit, polarization, and product name. +- `/assets/sources` remains the asset-led inventory view. It is not the main LT-1 production planning surface because it lacks the full image search/AOI workflow. +- Items that already have a completed LT-1 GeoTIFF product are shown as produced and are not selectable for a new production task. + +## What Is Not A Finished Image Product + +LandSAR `100016` and `100206` are still useful, but they only prepare/import LT-1 data: + +```text +100016 -> LandSAR Input_Data +100206 -> precise-orbit injection for imported XML +``` + +Those outputs must not be displayed as "image production complete" and must not block reprocessing as if they were geocoded images. + +## Open LandSAR Work + +LandSAR single-scene geocoded image production may still be possible through `180044`, `180016`, `200046`, or `200016`, but the repository does not yet contain a verified parameter chain for those modules. Do not expose those proIDs in the formal UI until a real parameter file and sample run are verified. diff --git a/docs/PRODUCTION_NODE_SUBSYSTEM_DESIGN_20260627.md b/docs/PRODUCTION_NODE_SUBSYSTEM_DESIGN_20260627.md new file mode 100644 index 0000000..47312c0 --- /dev/null +++ b/docs/PRODUCTION_NODE_SUBSYSTEM_DESIGN_20260627.md @@ -0,0 +1,462 @@ +# 生产节点子系统设计：D-InSAR 与单景影像生产（2026-06-27） + +## 1. 结论 + +当前 `LANDSAR_CLUSTER_ITEM` 已经证明：LandSAR D-InSAR 可以从主服务器拆分 pair，并在远端 Windows 节点完成输入搬运、LandSAR 执行和结果回传。 + +但这仍是 LandSAR D-InSAR 的集群 MVP，不应直接扩展成长期架构。后续陆探一号和 Sentinel-1 的“只生产影像、不做 D-InSAR”能力也会进入生产管理域。陆探一号这条线应优先承认 LandSAR 已有的 `100016` LT-1 数据导入/统一格式转换能力，再决定是否继续扩展为地理编码或正射影像产品。因此远端节点不能只理解 `LANDSAR_CLUSTER_ITEM`，而应该抽象成一个受控的“生产节点子系统”。 + +建议把后续设计目标调整为： + +1. 主服务器继续负责资产索引、任务编排、调度策略、结果 catalog 和权限边界。 +2. 子服务器只部署生产节点运行包，不部署完整项目仓库、前端、管理后台和无关源码。 +3. 所有生产任务按“产品类型 + 处理器能力”分发，同一任务可以选择本机执行或集群执行。 +4. LandSAR、Gamma/PyINT、未来 Sentinel-1 影像生产处理器都通过 adapter 接入生产节点协议。 +5. 结果以标准产品 manifest 回传，由主服务器统一入库，而不是让子服务器直接写主库或扫描任意目录。 + +## 2. 当前事实 + +### 2.1 已有设计边界 + +- [THREE_SENSOR_LOCAL_PRODUCTION_CONTRACT_20260616.md](THREE_SENSOR_LOCAL_PRODUCTION_CONTRACT_20260616.md) 明确 LT-1、Sentinel-1 当前管理对象是本机压缩包源池，生产时才按任务 materialize 到 `Task_Pool`。 +- [DINSAR_TASK_POOL_THREE_ENGINE_REFACTOR_20260614.md](DINSAR_TASK_POOL_THREE_ENGINE_REFACTOR_20260614.md) 明确 D-InSAR 保留 `sarscape`、`landsar`、`pyint` 三条主线，其中 LandSAR 只处理 LT-1，Gamma/PyINT 同时支持 LT-1 和 Sentinel-1。 +- [LANDSAR_CLUSTER_DATA_TRANSPORT_DESIGN_20260625.md](LANDSAR_CLUSTER_DATA_TRANSPORT_DESIGN_20260625.md) 已经为 LandSAR D-InSAR 定义了输入下载和结果上传链路。 +- 当前 192.168.1.6 节点已能执行 LandSAR D-InSAR 集群 item，并调用与本机一致的 `LandsarEngine.run()`。 +- `third_party/LandSAR/LT-1_数据导入功能说明.md` 明确 `100016` 是 LT-1 数据导入算法 ID；当前 `backend/app/dinsar_engines/landsar_engine.py` 已经能生成 `100016.txt` 并调用 `InSAR_Console.exe`，但这段能力目前只作为 D-InSAR 前置导入阶段存在。 + +### 2.2 仍未完成的能力 + +- 生产管理里的陆探一号已接入第一阶段本机生产链：`100016` LT-1 导入生成 LandSAR `Input_Data`，并支持可选 `100206` 精轨注入。它仍不是正射/地理编码影像产品。 +- 生产管理里的 Sentinel-1“只生产影像”仍是占位，不是已实现链路。 +- 陆探一号单景生产至少应拆成两层：`landsar.import.lt1` 表示 LandSAR `100016` 导入/统一格式转换，`landsar.image.lt1` 表示后续地理编码、正射或业务可用影像产品。前者已在本机任务/API/前端/catalog 中产品化；后者仍需要确认 LandSAR 调用链和输出规格。 +- Sentinel-1 影像生产的处理器尚未最终确认，不能把 Sentinel-1 影像生产硬编码到 LandSAR 集群链路。 +- 当前集群 worker 更接近“把后端生产代码部署到远端执行”，还不是一个最小权限、最小源码暴露的生产节点运行包。 + +## 3. 需要解决的问题 + +### 3.1 不要把集群等同于 LandSAR D-InSAR + +如果继续按 `LANDSAR_CLUSTER_ITEM` 的方式增长，后续很容易出现： + +- `LANDSAR_IMAGE_CLUSTER_ITEM` +- `S1_IMAGE_CLUSTER_ITEM` +- `PYINT_CLUSTER_ITEM` +- `SBAS_CLUSTER_ITEM` + +每新增一种生产能力都复制一套领取、搬运、执行、上传、入库逻辑，技术债会快速扩大。 + +更稳妥的边界是： + +```text +生产任务协议 + ├─ 输入 manifest + ├─ 处理器 adapter + ├─ 执行状态上报 + ├─ 结果 manifest + └─ 结果上传与 catalog + +具体处理器 + ├─ landsar.dinsar.lt1 + ├─ landsar.import.lt1 + ├─ landsar.image.lt1 + ├─ pyint.dinsar.lt1 + ├─ pyint.dinsar.s1 + └─ s1.image.<待定处理器> +``` + +### 3.2 单景影像生产也需要本机/集群双模式 + +LT-1 和 Sentinel-1 的影像生产虽然不是 D-InSAR，但仍可能是重计算、重 IO、长耗时任务。它们不应该只作为本机按钮实现。 + +推荐统一执行模式： + +| 模式 | 含义 | 适用场景 | +| --- | --- | --- | +| `local` | 主服务器本机执行 adapter | 调试、小批量、没有可用节点 | +| `cluster` | 远端生产节点领取执行 | 大批量、长耗时、需要释放主服务器 | +| `auto` | 主服务器按能力、负载、数据位置选择 | 正式生产默认模式 | + +前端可以先只暴露“本机执行 / 集群执行”，内部仍按统一任务协议创建任务。 + +### 3.3 子服务器不应长期部署完整代码仓库 + +当前 MVP 为了快速跑通，子服务器需要较完整的项目运行环境。这对验证是可接受的，但长期有三个问题： + +1. 源码暴露面过大：子服务器不需要前端、后台管理、资产扫描、用户接口等源码。 +2. 配置权限过宽：子服务器不应持有主库高权限连接信息。 +3. 升级不可控：完整仓库部署容易出现主服务器和子服务器代码版本漂移。 + +长期应改为生产节点运行包： + +```text +production-node/ + worker_service.py + config.py + client.py + adapters/ + landsar_dinsar.py + landsar_lt1_image.py + pyint_dinsar.py + s1_image.py + contracts/ + job_manifest.py + product_manifest.py + status_event.py + scripts/ + install_windows_service.ps1 + requirements.lock +``` + +这个运行包只包含： + +- 任务领取和心跳客户端。 +- 输入下载和结果上传客户端。 +- 必要的生产 adapter。 +- 与主服务器共享的 manifest schema。 +- Windows 服务安装脚本。 + +不包含： + +- 前端源码。 +- 管理后台路由。 +- 数据扫描入口。 +- 用户认证管理。 +- 数据库迁移脚本。 +- 与该节点能力无关的处理器源码。 + +## 4. 目标架构 + +```mermaid +flowchart LR + UI["生产管理前端"] --> API["主服务器 API"] + API --> Scheduler["调度器"] + Scheduler --> Queue["生产任务队列"] + API --> Catalog["结果 catalog"] + API --> Assets["源数据/精轨资产库"] + + NodeA["本机生产节点"] --> Queue + NodeB["远端生产节点 1.6"] --> Queue + NodeC["远端生产节点 N"] --> Queue + + NodeB --> Adapter1["landsar.dinsar.lt1"] + NodeB --> Adapter2["landsar.image.lt1"] + NodeC --> Adapter3["pyint.dinsar.s1"] + NodeC --> Adapter4["s1.image.<待定>"] + + Queue --> Manifest["输入 manifest"] + Manifest --> NodeB + NodeB --> Upload["结果上传 API"] + Upload --> Catalog +``` + +### 4.1 主服务器职责 + +- 维护源数据、精轨、DEM、Task_Pool、结果 catalog。 +- 根据资产状态生成生产任务。 +- 决定任务执行模式：本机、指定节点、自动调度。 +- 为 worker 生成输入 manifest，包含文件清单、hash、大小、产品类型、处理器 profile。 +- 接收 worker 状态、日志摘要、进度事件和结果包。 +- 校验结果 manifest 后入库。 +- 维护节点注册、能力、版本、心跳和并发上限。 + +### 4.2 生产节点职责 + +- 启动后向主服务器注册或发送心跳。 +- 上报能力，例如 `landsar.dinsar.lt1`、`landsar.import.lt1`、`landsar.image.lt1`、`pyint.dinsar.s1`。 +- 按能力领取任务。 +- 下载输入文件或复用本地缓存。 +- 调用本机已安装的生产软件或 adapter。 +- 将运行日志、状态、结果 manifest 和产品文件上传回主服务器。 +- 清理本地临时目录，保留可配置缓存。 + +### 4.3 Adapter 职责 + +Adapter 是生产节点中唯一知道具体软件细节的层： + +| Adapter | 输入 | 输出 | 备注 | +| --- | --- | --- | --- | +| `landsar.dinsar.lt1` | LT-1 pair Task_Pool | D-InSAR 标准产品包 | 当前集群 MVP 已覆盖核心执行 | +| `landsar.import.lt1` | LT-1 单景源包、解包 scene，或多景导入目录 | LandSAR `Input_Data` 统一格式、缩略图、导入 manifest | 基于 `100016`，当前代码已有 pair-shaped 前置调用，需要拆成一等单景 adapter | +| `landsar.image.lt1` | `landsar.import.lt1` 输出 + 可选精轨/DEM | LT-1 地理编码、正射或业务影像产品 | 需要确认 LandSAR 后续 proID/参数和产品规格 | +| `pyint.dinsar.lt1` | LT-1 pair Task_Pool | D-InSAR 标准产品包 | 可后续接入 | +| `pyint.dinsar.s1` | S1 pair Task_Pool + EOF | D-InSAR 标准产品包 | 当前只应按 Gamma/PyINT 能力开放 | +| `s1.image.<待定>` | S1 ZIP/SAFE + EOF | S1 单景影像产品 | 处理器未确定前保持占位 | + +主服务器不应该把某个 adapter 的内部目录结构暴露给前端；前端只看到任务类型、执行位置、状态和结果。 + +## 5. 统一任务类型 + +### 5.1 产品族 + +建议把生产任务按产品族建模，而不是按按钮建模： + +| 产品族 | 数据粒度 | 当前状态 | 目标执行模式 | +| --- | --- | --- | --- | +| `dinsar_pair` | 两景 pair | LandSAR LT-1 集群 MVP 已跑通 | 本机 + 集群 | +| `single_scene_import` | 单景或多景导入 | LT-1 LandSAR `100016/100206` 已作为本机 `LANDSAR_LT1_IMPORT` 任务、API、前端入口和 `lt1_landsar` catalog 产品发布；集群执行尚未接入 | 本机 + 集群 | +| `single_scene_image` | 单景 | LT-1 后续影像产品和 S1 均为占位 | 本机 + 集群 | +| `sbas_stack` | 多景 stack | 当前不纳入本轮集群化 | 后续再设计 | +| `gf3_native_register` | 外部结果登记 | 本机登记 `_geo` | 不建议进入生产节点 | + +### 5.2 推荐任务字段 + +```json +{ + "job_id": 123, + "product_family": "single_scene_image", + "sensor": "LT1", + "processor": "landsar", + "profile": "landsar.image.lt1", + "execution_mode": "cluster", + "input_manifest_url": "/api/production-node/jobs/123/input-manifest", + "result_contract": "standard_product_manifest.v1", + "priority": 50, + "retry_policy": { + "max_retries": 2, + "timeout_seconds": 7200 + } +} +``` + +### 5.3 结果 manifest + +D-InSAR 和单景影像生产都应该回传标准 manifest，差异放在 `product_family` 和 `product_type` 中： + +```json +{ + "manifest_version": 1, + "product_family": "single_scene_image", + "sensor": "LT1", + "processor": "landsar", + "profile": "landsar.image.lt1", + "scene_id": "LT1A_MONO_KSC_STRIP1_...", + "run_key": "run_20260627T010203Z_landsar_image_lt1_456", + "products": [ + { + "role": "main_image", + "path": "products/main.tif", + "format": "GeoTIFF", + "crs": "EPSG:4326" + }, + { + "role": "preview", + "path": "preview/main.webp", + "format": "WEBP" + }, + { + "role": "metadata", + "path": "metadata/product.json", + "format": "JSON" + } + ] +} +``` + +## 6. 陆探一号单景生产设计方向 + +陆探一号单景生产如果由 LandSAR 承担，应先把“导入/统一格式转换”和“正式影像产品”分开。 + +### 6.1 `landsar.import.lt1` + +这是当前最清楚、风险最低的第一版能力。 + +LandSAR `100016` 的语义是 LT-1 数据导入：把 LT-1A/LT-1B SLC XML/TIFF 转成 LandSAR 统一内部格式，形成 `Task_*/Input_Data` 可消费的 XML/TIF 组织，并生成缩略图等辅助文件。当前代码已经在 `_ensure_imported_input_data()` 中调用这条链路，但有两个限制： + +- 它被包在 D-InSAR 执行内部，只在缺少 `Input_Data` 时作为前置阶段触发。 +- 当前参数生成器按 `master/slave` 两文件夹导入写死，不是正式的单景产品 adapter。 + +第一版应把它产品化为： + +```text +landsar.import.lt1 + 输入：LT-1 单景源包、解包 scene，或显式 scene 目录 + 执行：InSAR_Console.exe + 100016.txt + 输出：LandSAR Input_Data 统一格式 + import_manifest.json + 缩略图/日志 + 入库：单景预处理/影像生产 catalog + 执行模式：local / cluster / auto +``` + +这个产品不应伪装成正射影像或地理编码强度图。它的价值是把陆探源数据转成 LandSAR 后续 D-InSAR、SBAS、影像处理可复用的标准输入。 + +### 6.2 `landsar.image.lt1` + +如果“只生产影像”指的是业务可用影像，例如地理编码强度图、幅度图、正射 GeoTIFF、洪涝分析输入图，那么还需要确认 LandSAR 是否有对应单景 proID 或可复用处理链。不能把 `100016` 的导入输出直接命名为正射产品。 + +需要确认的产品规格： + +- 输入是源压缩包、解包目录，还是现有 Task_Pool scene 目录。 +- 输出是 SLC/SSC 的标准化影像、地理编码强度图、幅度图、还是系统用于浏览和洪水分析的 GeoTIFF。 +- 是否需要精轨。 +- 是否需要 DEM。 +- 是否需要生成 WebP 预览。 +- 是否进入 `radar_data`、`source_product_assets`、D-InSAR catalog，还是新的影像产品 catalog。 + +如果后续确认 LandSAR 能从 `Input_Data` 继续生成地理编码/正射产品，再实现第二层： + +```text +LT-1 single scene image product + 输入：landsar.import.lt1 输出 + 可选精轨 + 可选 DEM + 执行器：LandSAR + 输出：标准产品目录 + product_manifest.json + preview.webp + 入库：影像产品 catalog + 执行模式：local / cluster / auto +``` + +不要在第一版同时承诺“原始归档标准化、地理编码、洪水分析输入、全部极化派生物、可视化浏览缓存”这些目标。先把一个产品闭环做对，再扩展产品角色。 + +## 7. Sentinel-1 影像生产设计方向 + +Sentinel-1 单景影像生产目前不能直接套用 LandSAR。需要先确定处理器： + +- 如果走 Gamma/PyINT，需要定义单景预处理 profile。 +- 如果走 GDAL/SNAP/其他工具，需要单独 adapter。 +- 如果只是生成浏览预览，应该归入资产扫描/预览缓存，不应叫正式生产任务。 + +建议在处理器未确认前只保留协议占位： + +```text +s1.image. + 状态：设计占位 + 不进入正式调度 + 不在 UI 上展示为可执行生产能力 +``` + +这样可以避免前端提前出现“哨兵影像集群生产”按钮，但后端没有可信处理链。 + +## 8. 调度与效率 + +### 8.1 节点能力上报 + +生产节点心跳应包含： + +```json +{ + "node_id": "production-node-192-168-1-6", + "version": "2026.06.27", + "capabilities": [ + "landsar.dinsar.lt1", + "landsar.import.lt1", + "landsar.image.lt1" + ], + "max_concurrency": 1, + "active_jobs": 0, + "free_disk_gb": 512, + "runtime": { + "os": "windows", + "landsar_available": true, + "python_version": "3.12" + } +} +``` + +LandSAR 类任务的并发不能只看 CPU 核心数。需要同时考虑： + +- LandSAR 是否支持多实例并发。 +- 许可证或硬件锁是否允许并行。 +- 工作目录是否互相隔离。 +- 磁盘 IO 是否成为瓶颈。 +- 单任务内部是否已经使用多线程。 + +因此第一版远端 LandSAR 节点建议 `max_concurrency=1`。等确认 LandSAR 多实例隔离和资源占用后，再按节点开放 2 个或更多并发。 + +### 8.2 缓存策略 + +单景影像生产和 D-InSAR 可以共享部分输入缓存： + +- LT-1 源包下载缓存。 +- LT-1 解包缓存。 +- LandSAR 导入后的中间目录。 +- DEM 裁剪缓存。 +- Sentinel-1 ZIP/SAFE 和 EOF 缓存。 + +缓存键应基于源文件 hash、mtime、size、processor profile 和关键参数，不应只基于文件名。否则源包被替换后容易复用错误缓存。 + +### 8.3 数据搬运策略 + +优先级建议： + +1. 第一阶段：沿用 HTTP manifest + file download + result upload，路径最清楚。 +2. 第二阶段：增加断点续传和文件级 hash 校验。 +3. 第三阶段：支持节点本地缓存命中，避免重复下载同一源包。 +4. 第四阶段：在受控环境下可选共享只读源池，但不作为默认安全模型。 + +不要让 worker 任意访问主服务器磁盘路径。worker 应只根据主服务器签发的 manifest 下载白名单文件。 + +## 9. 安全边界 + +长期目标： + +- worker 不持有主数据库账号。 +- worker 只持有节点 token。 +- token 按节点、能力和有效期管理。 +- 所有输入下载和结果上传都走主服务器 API。 +- 主服务器校验每个上传文件的相对路径，拒绝目录逃逸。 +- 主服务器校验 result manifest，只有白名单产品角色进入 catalog。 +- worker 运行包只包含生产节点必要代码。 +- worker 版本和 adapter 版本必须上报，主服务器可以拒绝过旧节点领取任务。 + +当前 LandSAR 集群 MVP 可以作为过渡，但文档上应明确：完整仓库部署、DB 直接领取队列、共享 token 都不是长期安全边界。 + +## 10. 实施路线 + +### 阶段 0：保持现状可用 + +- 保留当前 LandSAR D-InSAR 集群能力。 +- 不在未设计清楚前扩展新的集群 job type。 +- 继续记录 1.6 节点运行结果、失败原因、传输耗时和 LandSAR 执行耗时。 + +### 阶段 1：抽取生产节点协议 + +- 定义 `job_manifest`、`input_manifest`、`product_manifest`、`status_event`。 +- 把 LandSAR D-InSAR 当前输入下载、执行、上传流程映射到协议。 +- 主服务器保留当前 API，同时新增通用 `/api/production-node/*` 命名空间。 + +### 阶段 2：拆出 worker-only 运行包 + +- 从完整仓库部署改成生产节点运行包部署。 +- Windows 节点用服务方式启动。 +- 节点只配置主服务器 URL、节点 token、工作根、结果根、缓存根和能力列表。 +- 先支持 `landsar.dinsar.lt1`。 + +### 阶段 3：陆探一号单景生产 + +- 先实现 `landsar.import.lt1` adapter，把 LandSAR `100016` 从 D-InSAR 前置阶段拆成一等生产能力。 +- 当前 `_generate_import_param_file()` 按 master/slave 两文件夹导入写死，单景 adapter 需要支持单 scene 输入、单文件夹导入或显式 file import。 +- 本机模式和集群模式同时接入同一任务协议。 +- 结果进入单景预处理/影像产品 catalog，而不是混入 D-InSAR 结果 catalog。 +- 确认 LandSAR 后续单景地理编码或正射处理链后，再实现 `landsar.image.lt1`。 + +### 阶段 4：Sentinel-1 单景影像生产 + +- 先确认处理器和产品规格。 +- 再实现 `s1.image.` adapter。 +- 未确认前不开放 UI 执行入口。 + +### 阶段 5：节点运维和调度完善 + +- 节点版本管理。 +- 能力矩阵管理。 +- 节点禁用/启用。 +- 任务重分配。 +- 节点磁盘清理。 +- 节点运行日志集中查看。 + +## 11. 近期不建议做的事 + +- 不建议继续复制 `LANDSAR_CLUSTER_ITEM` 形成多个专用 cluster item。 +- 不建议在子服务器长期部署完整项目仓库。 +- 不建议让子服务器直接扫描主服务器源数据目录。 +- 不建议让子服务器直接写主数据库结果表。 +- 不建议在 Sentinel-1 处理器未确认前实现“哨兵影像生产”按钮。 +- 不建议把资产扫描阶段的 WebP 预览生成混同为正式影像生产。 + +## 12. 待确认问题 + +1. 陆探一号“只生产影像”的正式产品定义是什么：只做 LandSAR `100016` 导入/统一格式转换，还是继续生成地理编码强度图、幅度图、正射 GeoTIFF？ +2. 陆探一号单景影像生产是否必须使用精轨和 DEM？ +3. Sentinel-1 单景影像生产准备用哪个处理器承担？ +4. 单景影像产品是否需要新建 catalog，还是复用现有 `radar_data` 资产表加产品 manifest？ +5. 远端节点是否允许访问只读共享源池，还是严格走 HTTP 下载？ +6. LandSAR 在同一台 Windows 节点上是否允许多个实例并发？ + +这些问题确认前，可以继续完善 LandSAR D-InSAR 集群，但不宜把新的影像生产能力直接硬接到当前 MVP worker 上。 diff --git a/docs/RESULT_EXTRACTION_ACCESS_CONTROL_AUDIT_20260630.md b/docs/RESULT_EXTRACTION_ACCESS_CONTROL_AUDIT_20260630.md new file mode 100644 index 0000000..c0efcc5 --- /dev/null +++ b/docs/RESULT_EXTRACTION_ACCESS_CONTROL_AUDIT_20260630.md @@ -0,0 +1,219 @@ +# 结果提取与用户权限审计（2026-06-30） + +## 1. 审计范围 + +本次审计聚焦“结果提取”相关入口和用户权限边界，覆盖： + +- 前端结果提取工作台：`frontend/src/ResultExtractionPanel.jsx` +- D-InSAR 结果管理页：`frontend/src/DinsarProductsPanel.jsx` +- D-InSAR 结果导出接口：`POST /api/dinsar-results/export` +- D-InSAR 生产结果提取与登记接口：`POST /api/idl/extract-disp` +- 用户与权限模型：`auth_users.role`、全局认证守卫、用户管理页 + +本次文档只记录审计结论和后续设计约束，不包含代码修复。 + +## 2. 当前实现事实 + +### 2.1 两条“提取”链路 + +当前系统里“结果提取”实际包含两种不同语义： + +1. **生产结果入库** + - 前端入口：`DinsarProductsPanel.jsx` + - 后端入口：`POST /api/idl/extract-disp` + - 后台任务：`EXTRACT_DINSAR_PRODUCTS` + - 作用：从生产目录提取 D-InSAR 位移结果，发布成标准结果包，并重建结果 catalog。 + +2. **成果交付导出** + - 前端入口：`ResultExtractionPanel.jsx` + - 后端入口：`POST /api/dinsar-results/export` + - 作用：从已经登记的 D-InSAR catalog 中选择成果，复制到服务器指定交付目录。 + +这两条链路目前在产品文案上都叫“提取”，容易让用户混淆“入库”和“交付”。 + +### 2.2 当前接入状态 + +`ResultExtractionPanel.jsx` 中的通道状态： + +| 通道 | 当前状态 | 说明 | +| --- | --- | --- | +| D-InSAR 结果 | 已接入 | 支持查询已登记结果并导出到服务器目录 | +| SBAS-InSAR 结果 | 半接入 | 可读取目录样例，但统一提取接口未实现 | +| LT-1 正射结果 | 占位 | 单景/正射结果 catalog 与导出链路未完成 | +| Sentinel-1 正射结果 | 占位 | 生产和导出链路未完成 | +| GF3 SARscape `_geo` | 占位 | 登记/标准化思路存在，统一导出接口未完成 | + +## 3. 当前权限模型 + +系统当前只有两类角色： + +- `admin` +- `viewer` + +定义位置：`backend/app/auth_service.py` + +全局认证守卫位于 `backend/app/routers/dependencies.py`： + +- `GET / HEAD / OPTIONS` 默认视为只读操作，登录用户可访问。 +- 除少数显式安全 POST 外，非只读请求要求 `admin`。 +- 非管理员执行写操作会被拒绝，返回 `403 Read-only account cannot perform this operation.` + +前端在 `App.jsx` 中把非管理员账号映射为 `readOnly`： + +- viewer 可以浏览结果、查看任务、查看 catalog。 +- viewer 不能提交生产、扫描、提取、导出、删除、修改。 +- admin 拥有所有写权限，包括生产、扫描、结果入库、结果导出、用户管理和运维配置。 + +后端没有依赖前端按钮禁用来保护写操作。`/api/idl/extract-disp` 和 `/api/dinsar-results/export` 都显式要求 `admin`，这一点是正确的。 + +## 4. 审计发现 + +### P1：成果交付权限与系统管理员权限耦合过重 + +当前只有 `admin` 能执行成果导出，但 `admin` 同时拥有用户管理、系统配置、生产扫描、删除记录等高权限。 + +从业务职责看，成果交付导出不应天然等同于系统管理员权限。后续应拆出更细的权限，例如： + +- `operator`：可提交生产任务、结果入库、目录重建。 +- `exporter`：可导出已登记成果到受控交付目录。 +- `viewer`：只读浏览、预览、查询。 +- `admin`：用户管理、系统配置、根目录维护、许可证和高风险运维。 + +### P1：结果导出是同步请求，存在 504 风险 + +`POST /api/dinsar-results/export` 在请求线程内执行文件复制，最多允许 500 个结果 ID。成果文件较大或目标目录较慢时，容易再次触发前端或 Nginx 超时。 + +后续应改成后台任务： + +- 接口只创建任务并返回 `task_id`。 +- 文件复制由 worker 执行。 +- 前端通过任务中心/结果提取工作台展示进度、成功数、失败数和目标目录。 + +### P1：生产结果入库缺少显式操作审计 + +`/api/dinsar-results/export` 已写入 `dinsar_results_exported` 审计日志。 + +`/api/idl/extract-disp` 当前会创建系统任务，但缺少独立的操作审计记录。它会改变结果 catalog，应记录： + +- 操作用户 +- 源生产目录 +- 目标发布目录 +- 创建的 `task_id` / `job_id` +- 完成后的 processed/copied/failed/published/registered 数量 + +### P2：页面命名和工作流边界不清 + +当前“D-InSAR 结果提取与登记”和“结果提取工作台”容易混淆。 + +建议命名： + +- “生产结果入库”：从生产目录提取并登记为系统 catalog。 +- “成果交付导出”：从已登记 catalog 选择成果并复制到交付目录。 + +这两个动作应该放在同一结果管理域下，但用不同分区和不同权限提示。 + +### P2：占位通道需要降低可执行暗示 + +SBAS、LT-1 正射、Sentinel-1 正射、GF3 `_geo` 目前不应被呈现成可执行导出能力。 + +建议 UI 明确显示： + +- `已接入` +- `目录可查，导出未接入` +- `规划中` +- `不可执行` + +并隐藏或禁用导出按钮，避免用户误以为功能已经上线。 + +### P2：导出目录策略需要产品化 + +当前后端已有 `_validate_export_path()` 和 `ALLOWED_EXPORT_DIRS` 约束能力，但前端仍允许用户输入服务器绝对路径。 + +后续建议： + +- 普通业务用户不输入任意服务器路径。 +- 管理员在系统配置中维护“交付目录白名单”。 +- 结果导出页只让用户选择白名单目录和子任务名。 +- 审计记录保存最终解析后的服务器路径。 + +### P3：部分前端文案存在历史编码损坏 + +`DinsarProductsPanel.jsx`、`ResultExtractionPanel.jsx`、`UserAdminPanel.jsx` 等文件存在局部中文乱码。功能不一定受影响，但会降低维护性和产品可信度。 + +建议后续单独做一次 UTF-8 文案修复，不与权限重构混在同一次提交中。 + +## 5. 建议目标模型 + +### 5.1 功能分区 + +结果管理应拆成三个清晰分区： + +1. **产品目录** + - 查看已登记成果。 + - 预览、筛选、查看详情。 + - viewer 可访问。 + +2. **生产结果入库** + - 从生产结果根目录扫描、提取、发布、登记。 + - operator/admin 可执行。 + +3. **成果交付导出** + - 从 catalog 选择成果，导出到受控交付目录。 + - exporter/operator/admin 可执行。 + +### 5.2 权限矩阵建议 + +| 操作 | viewer | exporter | operator | admin | +| --- | --- | --- | --- | --- | +| 查看结果 catalog | yes | yes | yes | yes | +| 查看预览与详情 | yes | yes | yes | yes | +| 生产结果入库 | no | no | yes | yes | +| 目录重建/发布 | no | no | yes | yes | +| 成果交付导出 | no | yes | yes | yes | +| 生产任务提交 | no | no | yes | yes | +| 用户管理 | no | no | no | yes | +| 根目录/许可证/运维配置 | no | no | no | yes | + +实现上可以先保留 `role` 字段，扩展角色枚举；长期可引入权限位表，避免角色继续膨胀。 + +## 6. 推荐实施顺序 + +### 阶段 1：修正产品语义和审计 + +- 页面文案区分“生产结果入库”和“成果交付导出”。 +- `/api/idl/extract-disp` 增加操作审计。 +- 结果提取工作台明确标注未接入通道。 +- 修复相关页面乱码文案。 + +### 阶段 2：导出任务化 + +- 新增 `EXPORT_DINSAR_RESULTS` 后台任务类型。 +- `/api/dinsar-results/export` 改为返回 `task_id`。 +- 前端展示导出任务进度和失败明细。 +- 导出结果保留 task log 和 audit log。 + +### 阶段 3：权限细分 + +- 扩展角色：`viewer/exporter/operator/admin`。 +- 用户管理页支持新角色说明。 +- 后端增加能力级依赖，例如 `require_capability("result.export")`。 +- 所有高风险写操作按 capability 而不是只按 admin 判断。 + +### 阶段 4：交付目录白名单产品化 + +- 将 `ALLOWED_EXPORT_DIRS` 从环境变量能力升级为系统配置/受控根目录。 +- 前端从白名单选择交付根目录。 +- 用户只输入子目录名或交付批次名。 + +## 7. 验收标准 + +完成上述改造后，应满足： + +1. viewer 能看结果，不能导出、不能入库。 +2. exporter 能导出已登记成果，但不能提交生产、不能用户管理。 +3. operator 能生产、入库、导出，但不能用户管理和系统配置。 +4. admin 保留全部权限。 +5. 所有入库和导出动作都有 task log 和 audit log。 +6. 大批量导出不再产生 HTTP 504。 +7. 未实现通道在 UI 上不会被误认为可执行功能。 + diff --git a/docs/SOURCE_ORBIT_ASSET_SCAN_OPERATIONS_20260626.md b/docs/SOURCE_ORBIT_ASSET_SCAN_OPERATIONS_20260626.md index 4d9861b..a9d9d2e 100644 --- a/docs/SOURCE_ORBIT_ASSET_SCAN_OPERATIONS_20260626.md +++ b/docs/SOURCE_ORBIT_ASSET_SCAN_OPERATIONS_20260626.md @@ -61,6 +61,15 @@ The binding step is currently a full re-evaluation of active LT-1/Sentinel-1 sou If a scan must be stopped before retrying with new concurrency settings, stop the running worker processes and mark the active `system_jobs`, `system_tasks`, and `asset_inventory_states` rows as terminal failed states. This prevents the job queue from recovering and re-claiming the old job. +The main application worker is expected to support more than one queued job at a time so long-running preview generation does not block independent production jobs. Current main-server baseline: + +```env +JOB_WORKER_CONCURRENCY=2 +JOB_WORKER_ALLOWED_TYPES= +``` + +Do not start ad-hoc one-off workers during formal testing. Change the configured worker concurrency, stop stale processes, and let the operator restart the application normally. + Recommended health checks during a source scan: - Task log should show `workers=16` and `pending=64` / `active_or_queued=64` after the parser pool fills. diff --git a/frontend/index.html b/frontend/index.html index c0843f2..7d23e62 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -1,11 +1,11 @@ - + - + - frontend + 雷达数据生产管理系统
diff --git a/frontend/public/app-icon.png b/frontend/public/app-icon.png new file mode 100644 index 0000000..6ef1d6c Binary files /dev/null and b/frontend/public/app-icon.png differ diff --git a/frontend/src/App.css b/frontend/src/App.css index ea1327a..023b75f 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -2589,6 +2589,96 @@ input[type="checkbox"] { text-align: left; } +.task-runtime-summary { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 8px; + margin-bottom: 12px; +} + +.task-runtime-summary > div { + border: 1px solid #e2e8f0; + border-radius: 8px; + background: #f8fafc; + padding: 8px; + min-width: 0; +} + +.task-runtime-summary span { + display: block; + color: #64748b; + font-size: 11px; + margin-bottom: 3px; +} + +.task-runtime-summary strong { + color: #0f172a; + font-size: 15px; + font-family: var(--font-mono); +} + +.active-jobs-container { + display: flex; + flex-direction: column; + gap: 6px; + margin-bottom: 12px; +} + +.job-runtime-row { + display: grid; + grid-template-columns: auto minmax(0, 1fr) minmax(92px, 0.8fr); + align-items: center; + gap: 8px; + padding: 7px 8px; + border: 1px solid #e2e8f0; + border-radius: 7px; + background: #ffffff; + font-size: 12px; +} + +.job-status-chip { + display: inline-flex; + align-items: center; + justify-content: center; + min-width: 58px; + padding: 2px 6px; + border-radius: 999px; + border: 1px solid #cbd5e1; + color: #475569; + background: #f8fafc; + font-family: var(--font-mono); + font-size: 11px; +} + +.job-status-chip.running { + border-color: rgba(37, 99, 235, 0.22); + color: #1d4ed8; + background: rgba(37, 99, 235, 0.08); +} + +.job-status-chip.retry { + border-color: rgba(217, 119, 6, 0.24); + color: #b45309; + background: rgba(245, 158, 11, 0.1); +} + +.job-runtime-title, +.job-runtime-worker { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.job-runtime-title { + color: #334155; + font-weight: 600; +} + +.job-runtime-worker { + color: #64748b; + text-align: right; +} + .task-progress-item { background: #f8fafc; padding: 10px; @@ -2596,6 +2686,10 @@ input[type="checkbox"] { border: 1px solid #e2e8f0; } +.task-progress-item--muted { + background: #ffffff; +} + .task-info-row { display: flex; justify-content: space-between; @@ -3502,6 +3596,7 @@ input[type="checkbox"] { color: var(--color-text-secondary); padding: 4px 8px; border-radius: 6px; + min-width: 132px; } .status-task.has-active-tasks { @@ -3509,6 +3604,13 @@ input[type="checkbox"] { color: #1890ff; } +.status-task small { + color: var(--color-text-muted); + font-size: 0.86em; + line-height: 1.25; + white-space: nowrap; +} + .status-license { font-size: 0.75em; color: var(--color-text-muted); @@ -5615,6 +5717,17 @@ input[type="checkbox"] { box-sizing: border-box; } +.production-workspace-shell .dinsar-production-shell, +.production-workspace-shell .dinsar-products-page { + max-width: none; + margin-left: 0; + margin-right: 0; +} + +.production-workspace-shell .dinsar-production-shell { + padding: 0; +} + .dinsar-production-header { display: grid; grid-template-columns: minmax(280px, 1fr) minmax(420px, 0.95fr); @@ -6485,6 +6598,18 @@ input[type="checkbox"] { .dinsar-products-catalog-section { display: grid; gap: 10px; + width: 100%; + min-width: 0; +} + +.dinsar-products-catalog-section .dinsar-catalog-shell { + width: 100%; + min-width: 0; + box-sizing: border-box; +} + +.dinsar-products-catalog-section .dinsar-catalog-workspace { + grid-template-columns: minmax(360px, 420px) minmax(0, 1fr); } .sbas-products-page { @@ -7170,6 +7295,7 @@ input[type="checkbox"] { .dinsar-filter-layout, .dinsar-catalog-summary, .dinsar-catalog-workspace, + .dinsar-products-catalog-section .dinsar-catalog-workspace, .dinsar-catalog-manage, .dinsar-catalog-filter-bar, .dinsar-catalog-hero, diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index a7e0636..8f269f0 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -208,12 +208,14 @@ function App() { setHealthError: state.setHealthError, }))); const { - activeTasks, setActiveTasks, + activeTasks, setActiveTasks, runtimeSummary, setRuntimeSummary, isCheckingTasks, setIsCheckingTasks, pendingTaskIds, setPendingTaskIds, } = useTaskStore(useShallow((state) => ({ activeTasks: state.activeTasks, setActiveTasks: state.setActiveTasks, + runtimeSummary: state.runtimeSummary, + setRuntimeSummary: state.setRuntimeSummary, isCheckingTasks: state.isCheckingTasks, setIsCheckingTasks: state.setIsCheckingTasks, pendingTaskIds: state.pendingTaskIds, @@ -918,6 +920,7 @@ function App() { licenseOk: !!licenseStatus?.ok, activeTasks, setActiveTasks, + setRuntimeSummary, pendingTaskIds, setPendingTaskIds, setIsCheckingTasks, @@ -2114,6 +2117,7 @@ function App() { isReadOnlyUser={isReadOnlyUser} activeTasks={activeTasks} avgTaskProgress={avgTaskProgress} + runtimeSummary={runtimeSummary} licenseStatus={licenseStatus} healthStatus={healthStatus} healthLoading={healthLoading} @@ -2197,6 +2201,7 @@ function App() { licenseFileName={licenseFileName} licenseUploadStatus={licenseUploadStatus} activeTasks={activeTasks} + runtimeSummary={runtimeSummary} showCancelTask={showCancelTask} cancelTaskPwd={cancelTaskPwd} onShowCancelTask={() => setShowCancelTask(true)} diff --git a/frontend/src/AssetInventoryPanel.jsx b/frontend/src/AssetInventoryPanel.jsx index f414b24..aec2851 100644 --- a/frontend/src/AssetInventoryPanel.jsx +++ b/frontend/src/AssetInventoryPanel.jsx @@ -28,7 +28,7 @@ const fmtBytes = (value) => { const StatusBadge = ({ value }) => { const text = String(value || '-'); const status = text.toUpperCase(); - const tone = status === 'OK' || status === 'MATCHED' || status === 'SELECTED' + const tone = status === 'OK' || status === 'MATCHED' || status === 'SELECTED' || text.startsWith('已生产') ? 'ok' : status === 'WARNING' || status === 'OPEN' || status === 'MISSING' ? 'warn' @@ -202,6 +202,7 @@ export default function AssetInventoryPanel({ readOnly = false, onTaskStart }) { 产品 轨道 状态 + 生产 完整性 动作 文件 @@ -216,6 +217,7 @@ export default function AssetInventoryPanel({ readOnly = false, onTaskStart }) { {item.source_format}{item.imaging_mode} / {item.polarization} {item.relative_orbit || '-'}abs {item.absolute_orbit || '-'} + {item.archive_integrity_member_count != null ? `${item.archive_integrity_member_count} files` : item.archive_integrity_method || '-'} diff --git a/frontend/src/DinsarProductionPanel.jsx b/frontend/src/DinsarProductionPanel.jsx index 2592869..ea22d08 100644 --- a/frontend/src/DinsarProductionPanel.jsx +++ b/frontend/src/DinsarProductionPanel.jsx @@ -63,6 +63,17 @@ const STATUS_LABEL = { pending: '等待中', }; +const FAILURE_REASON_LABEL = { + 'LandSAR access violation during coherence mask/phase unwrapping': 'LandSAR 访问冲突（相干性掩膜/相位解缠）', + 'Insufficient tie/GCP points for DEM/geocoding': 'DEM / 地理编码控制点不足', + 'DEM/sub-terrain processing failed': 'DEM / 去地形阶段失败', + 'Insufficient GCPs for baseline/calibration': '基线精估计控制点不足', + 'Coherence mask/phase unwrapping failed': '相干性掩膜 / 相位解缠失败', + 'Processing timeout': '处理超时', + 'Result catalog publish failed': '结果目录发布失败', + 'Unclassified D-InSAR failure': '未分类失败', +}; + const PYINT_DEM_MODE_LABEL = { local_fabdem: '本地 FABDEM', opentopo: 'OpenTopography', @@ -267,6 +278,123 @@ function formatTaskRootUpdatedAt(value) { } } +function safeCount(value) { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; +} + +function getRunCounts(run) { + const completed = safeCount(run?.completed_items); + const failed = safeCount(run?.failed_items); + const skipped = safeCount(run?.skipped_items); + const total = safeCount(run?.total_items); + return { completed, failed, skipped, total }; +} + +function hasRunCounts(run) { + const { completed, failed, skipped, total } = getRunCounts(run); + return [completed, failed, skipped, total].some(value => value != null); +} + +function formatRunCounts(run) { + const { completed, failed, skipped, total } = getRunCounts(run); + const parts = []; + if (completed != null) parts.push(`成功 ${completed}`); + if (failed != null) parts.push(`失败 ${failed}`); + if (skipped != null && skipped > 0) parts.push(`跳过 ${skipped}`); + if (total != null) parts.push(`总数 ${total}`); + return parts.join(' / '); +} + +function statusToneClass(status) { + const normalized = String(status || '').toLowerCase(); + if (normalized === 'success' || normalized === 'completed') return 'tone-ready'; + if (normalized === 'failed') return 'tone-error'; + if (normalized === 'running') return 'tone-info'; + if (normalized === 'cancelled' || normalized === 'canceled') return 'tone-neutral'; + return 'tone-warn'; +} + +function formatFailureReason(reason) { + return FAILURE_REASON_LABEL[reason] || reason || '未分类失败'; +} + +function compactText(value, maxLength = 180) { + const text = String(value || '').replace(/\s+/g, ' ').trim(); + if (!text) return ''; + return text.length > maxLength ? `${text.slice(0, maxLength - 1).trim()}…` : text; +} + +function classifyFailureReason(errorMessage) { + const text = String(errorMessage || ''); + const lower = text.toLowerCase(); + if (text.includes('3221225477') || lower.includes('status_access_violation')) { + return 'LandSAR access violation during coherence mask/phase unwrapping'; + } + if (lower.includes('not enough gcps') || lower.includes('space insar calibration failed')) { + return 'Insufficient GCPs for baseline/calibration'; + } + if ( + lower.includes('no enough points') + || lower.includes('not enough points') + || lower.includes('geo_extract_gcp') + || lower.includes('无满足snr') + || text.includes('离散采样点') + ) { + return 'Insufficient tie/GCP points for DEM/geocoding'; + } + if (lower.includes('dem/sub-terrain') || lower.includes('subterrain') || lower.includes('sub-terrain')) { + return 'DEM/sub-terrain processing failed'; + } + if (text.includes('相干性掩膜') && text.includes('相位解缠')) { + return 'Coherence mask/phase unwrapping failed'; + } + if (lower.includes('timeout') || lower.includes('timed out') || text.includes('超时')) { + return 'Processing timeout'; + } + if (lower.includes('publish')) { + return 'Result catalog publish failed'; + } + return 'Unclassified D-InSAR failure'; +} + +function buildFailureSummaryFromItems(run) { + const items = Array.isArray(run?.items) ? run.items : []; + const failedItems = items.filter(item => String(item?.status || '').toUpperCase() === 'FAILED' || item?.last_error); + if (failedItems.length === 0) return null; + + const groupsByReason = new Map(); + const detailItems = failedItems.map(item => { + const reason = classifyFailureReason(item?.last_error); + const label = item?.task_alias || item?.task_name || item?.pair_key || '未命名任务'; + const group = groupsByReason.get(reason) || { reason, count: 0, items: [] }; + group.count += 1; + group.items.push(label); + groupsByReason.set(reason, group); + return { + task_alias: item?.task_alias, + task_name: item?.task_name, + reason, + error: compactText(item?.last_error, 260), + }; + }); + + return { + failed_count: failedItems.length, + partial: true, + groups: Array.from(groupsByReason.values()), + items: detailItems, + }; +} + +function getRunFailureSummary(run) { + const summary = run?.summary_json?.failure_summary; + if (summary && Number(summary.failed_count || 0) > 0) { + return summary; + } + return buildFailureSummaryFromItems(run); +} + function RunPathBlock({ run }) { const items = Array.isArray(run?.items) ? run.items : []; const item = items.find(entry => entry?.status === 'RUNNING') || items[0] || null; @@ -299,6 +427,67 @@ function RunPathBlock({ run }) { ); } +function RunStatusBlock({ run }) { + return ( +
+ + {formatStatus(run?.status)} + + {hasRunCounts(run) && ( + + {formatRunCounts(run)} + + )} +
+ ); +} + +function RunSituationBlock({ run }) { + const failureSummary = getRunFailureSummary(run); + const countsText = hasRunCounts(run) ? formatRunCounts(run) : ''; + const message = compactText(run?.message, 220); + if (!countsText && !failureSummary && !message) return null; + + return ( +
+
+ 运行情况 + {countsText && {countsText}} +
+ {failureSummary ? ( +
+ {(failureSummary.groups || []).slice(0, 4).map((group, index) => ( +
+ {formatFailureReason(group.reason)} + ：{Number(group.count || 0)} 项 + {Array.isArray(group.items) && group.items.length > 0 && ( + （{group.items.slice(0, 3).join('、')}{group.items.length > 3 ? ' 等' : ''}） + )} +
+ ))} + {failureSummary.partial && Number(run?.failed_items || 0) > Number(failureSummary.failed_count || 0) && ( +
+ 当前接口仅返回部分失败明细，请查看日志获取完整失败项。 +
+ )} +
+ ) : message ? ( +
{message}
+ ) : null} +
+ ); +} + function PreviewIssueList({ title, items, tone = 'warning' }) { if (!Array.isArray(items) || items.length === 0) { return null; @@ -1957,19 +2146,15 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) {run.run_id} {formatEngineLabel(run.engine)} {formatSatelliteFamilyLabel(inferSatelliteFamilyFromResultLike(run))} - - {formatStatus(run.status)} + + {run.started_at ? new Date(run.started_at * 1000).toLocaleString() : '-'} + + + + + {message &&
{message}
} + + +
+
+
+

生产候选检索

+
+ 调用影像检索能力，按时间、行政区、轨道、极化等条件规划要生产的 LT-1 场景。 +
+
+
已选 {selectedRadarIdList.length} 景
+
+ +
+ + + + + + +
+ +
+ + {regionMode === 'region' && ( + <> + + + + )} + +
+ +
+
+ {searchLoading ? '正在检索影像...' : `第 ${sceneStart}-${sceneEnd} 景 / 共 ${scenePage.total} 景`} +
+
+ + + + + +
+
+ +
+ + + + + + + + + + + + + + + {scenes.map(scene => { + const produced = sceneProduced(scene); + const selectable = !produced && Boolean(scene.source_product_ref_id); + const selected = selectedRadarIds.has(scene.id); + return ( + + + + + + + + + + + ); + })} + {scenes.length === 0 && ( + + + + )} + +
选择产品日期模式轨道极化状态路径
+ toggleScene(scene)} + /> + + {getSceneTitle(scene)} + {formatYmd(scene.imaging_date)}{scene.imaging_mode || '-'}{scene.relative_orbit || scene.orbit_circle || '-'}{scene.polarization || '-'} + {produced ? '已生产 GeoTIFF' : (scene.source_product_ref_id ? '可生产' : '未关联源资产')} + + {scene.file_path} +
+ 暂无符合条件的 LT-1 影像 +
+
+
+ + {preview && ( +
+

预览结果

+
+
场景数: {preview.scene_count}
+
engine: {preview.engine}
+
profile: {preview.profile_code}
+
+ {Array.isArray(preview.blockers) && preview.blockers.length > 0 && ( +
+ {preview.blockers.map(item => ( +
{item}
+ ))} +
+ )} + {Array.isArray(preview.warnings) && preview.warnings.length > 0 && ( +
+ {preview.warnings.map(item => ( +
{item}
+ ))} +
+ )} +
+ )} + +
+
+

最近 LT-1 GeoTIFF 产品

+
{products.length} 项
+
+
+ + + + + + + + + + + + + {products.map(product => ( + + + + + + + + + ))} + {products.length === 0 && ( + + + + )} + +
产品状态日期单位时间GeoTIFF
+ {product.display_name || product.product_id} + {product.status}{formatYmd(product.summary?.imaging_date)}{product.summary?.backscatter_unit || '-'}{formatTime(product.published_at)} + {product.primary_asset_path || product.publish_dir || '-'} +
+ 暂无产品 +
+
+
+ + ); +} diff --git a/frontend/src/ProductionWorkspace.jsx b/frontend/src/ProductionWorkspace.jsx index a6f6906..68f1186 100644 --- a/frontend/src/ProductionWorkspace.jsx +++ b/frontend/src/ProductionWorkspace.jsx @@ -11,6 +11,7 @@ const LazyDinsarProductionPanel = lazy(() => import('./DinsarProductionPanel')); const LazySbasInsarProductionPanel = lazy(() => import('./SbasInsarProductionPanel')); const LazySbasInsarProductsPanel = lazy(() => import('./SbasInsarProductsPanel')); const LazyDinsarProductsPanel = lazy(() => import('./DinsarProductsPanel')); +const LazyLandsarLt1ProductionPanel = lazy(() => import('./LandsarLt1ProductionPanel')); const LazyPairPlanningPanel = lazy(() => import('./panels/PairPlanningPanel')); const LazyPairsListPanel = lazy(() => import('./panels/PairsListPanel')); const LazyBatchPanel = lazy(() => import('./panels/BatchPanel')); @@ -25,17 +26,6 @@ const WORKFLOW_STEPS = [ ]; const SENSOR_PRODUCTION_PLACEHOLDERS = { - lt1_production: { - title: '陆探一生产占位', - note: '当前保留 LT-1 源压缩包本机登记与按需 materialize 入口。', - rows: [ - ['数据来源', '本机源压缩包 archive'], - ['精轨策略', '按生产任务关联 orbit 资产'], - ['准备方式', '按需 materialize 到 Task_Pool'], - ['生产边界', 'D-InSAR/SBAS 不走 UNC'], - ['结果管理', '进入统一产品 catalog'], - ], - }, sentinel1_production: { title: 'Sentinel-1 生产占位', note: '当前主要沉淀数据与精轨管理约束，SBAS 仅保留规划能力。', @@ -206,6 +196,13 @@ export default function ProductionWorkspace({ }); }; + const handleLt1ImageQueued = taskId => { + onTaskStart?.(taskId, 'LT-1 地理编码 GeoTIFF 生产任务已入队。', { + taskType: 'SAR_SCENE_PREPROCESS', + nonBlocking: true, + }); + }; + const renderContent = () => { if (activeView === 'dinsar_pairing') { return ( @@ -215,11 +212,11 @@ export default function ProductionWorkspace({ isLoading={isLoading} isReadOnlyUser={readOnly} hasEnoughRadarScenesForPlanning={hasEnoughRadarScenesForPlanning} - onOpenPairingModal={pairingPanel?.openModal} + onOpenPairingModal={pairingPanel?.onOpenPairingModal} hasRadarSearched={hasRadarSearched} - onRefreshRadarSearch={radarPanel?.refresh} - onSearchAll={radarPanel?.searchAll} - onRefreshDinsar={pairsPanel?.refreshDinsar} + onRefreshRadarSearch={pairingPanel?.onRefreshRadarSearch} + onSearchAll={radarPanel?.onSearchAll} + onRefreshDinsar={pairingPanel?.onRefreshDinsar} language={language} /> ); @@ -228,8 +225,12 @@ export default function ProductionWorkspace({ if (activeView === 'dinsar_pairs') { return (
- - + +
); } @@ -272,11 +273,15 @@ export default function ProductionWorkspace({ return ; } + if (activeView === 'lt1_production') { + return ; + } + return ; }; return ( -
+
diff --git a/frontend/src/api/index.js b/frontend/src/api/index.js index 86b70af..fe52c78 100644 --- a/frontend/src/api/index.js +++ b/frontend/src/api/index.js @@ -17,3 +17,4 @@ export * as unpackApi from './unpack'; export * as statsApi from './stats'; export * as timeseriesProductionApi from './timeseriesProduction'; export * as psinsarProductsApi from './psinsarProducts'; +export * as landsarLt1ProductionApi from './landsarLt1Production'; diff --git a/frontend/src/api/landsarLt1Production.js b/frontend/src/api/landsarLt1Production.js new file mode 100644 index 0000000..75b7020 --- /dev/null +++ b/frontend/src/api/landsarLt1Production.js @@ -0,0 +1,22 @@ +import apiClient from './client'; + +export const getLandsarLt1Capabilities = () => + apiClient.get('/landsar-lt1-production/capabilities').then(r => r.data); + +export const previewLandsarLt1Production = payload => + apiClient.post('/landsar-lt1-production/preview', payload).then(r => r.data); + +export const submitLandsarLt1Production = payload => + apiClient.post('/landsar-lt1-production/run', payload).then(r => r.data); + +export const previewLandsarLt1Import = previewLandsarLt1Production; +export const submitLandsarLt1Import = submitLandsarLt1Production; + +export const listLandsarLt1Products = (params = {}) => + apiClient.get('/landsar-lt1-production/products', { params }).then(r => r.data); + +export const getLandsarLt1Product = productId => + apiClient.get(`/landsar-lt1-production/products/${encodeURIComponent(productId)}`).then(r => r.data); + +export const getLandsarLt1AssetUrl = (productId, assetId) => + `/api/landsar-lt1-production/products/${encodeURIComponent(productId)}/assets/${encodeURIComponent(assetId)}`; diff --git a/frontend/src/api/tasks.js b/frontend/src/api/tasks.js index bc8c9fe..80a090d 100644 --- a/frontend/src/api/tasks.js +++ b/frontend/src/api/tasks.js @@ -1,6 +1,7 @@ import apiClient from './client'; export const getActiveTasks = () => apiClient.get('/tasks/active').then(r => r.data); +export const getTaskRuntimeSummary = () => apiClient.get('/tasks/runtime-summary').then(r => r.data); export const getRecentTasks = (taskTypes = [], statuses = [], limit = 20, offset = 0) => apiClient.get('/tasks/recent', { params: { diff --git a/frontend/src/components/GlobalTaskCenter.jsx b/frontend/src/components/GlobalTaskCenter.jsx index 77a2d94..16d353d 100644 --- a/frontend/src/components/GlobalTaskCenter.jsx +++ b/frontend/src/components/GlobalTaskCenter.jsx @@ -4,6 +4,7 @@ import { getTaskTypeLabel } from '../config/taskUiPolicies'; export default function GlobalTaskCenter({ isVisible, activeTasks, + runtimeSummary, t, isAdmin, showCancelTask, @@ -14,14 +15,27 @@ export default function GlobalTaskCenter({ onCloseCancelTask, }) { const [expanded, setExpanded] = useState(false); - if (!isVisible || activeTasks.length === 0) { + const jobs = runtimeSummary?.jobs || {}; + const worker = runtimeSummary?.worker || {}; + const scan = runtimeSummary?.scan || {}; + const activeJobs = Array.isArray(jobs.items) ? jobs.items : []; + const activeCount = Math.max( + activeTasks.length, + Number(jobs.active_count) || 0, + ); + const workerCount = Number(worker.worker_count) || 0; + const queuedJobs = Number(jobs.queued_count) || 0; + const runningJobs = Number(jobs.running_count) || 0; + const scanJobCount = Number(scan.active_job_count) || 0; + + if (!isVisible || activeCount === 0) { return null; } - const activeCount = activeTasks.length; const avgProgress = Math.round( activeTasks.reduce((sum, task) => sum + (Number(task.progress) || 0), 0) / Math.max(1, activeCount) ); + const visibleJobs = activeJobs.slice(0, 8); return (
@@ -29,7 +43,7 @@ export default function GlobalTaskCenter({ )} {expanded && ( @@ -37,12 +51,45 @@ export default function GlobalTaskCenter({

后台任务

-

任务正在执行，你可以继续使用其他功能；同类重复提交由系统限制。

+

展示 Worker、执行中 Job、排队 Job 和任务进度；同类重复提交由后端冲突检查处理。

+
+
+ Worker + {workerCount} +
+
+ 执行中 + {runningJobs} +
+
+ 排队 + {queuedJobs} +
+
+ 扫描 + {scanJobCount} +
+
+ {visibleJobs.length > 0 && ( +
+ {visibleJobs.map((job) => ( +
+ {job.status || '-'} + + {getTaskTypeLabel(job.task_type || job.job_type)} + + + {job.locked_by ? `Worker ${job.locked_by}` : (job.status === 'RETRY' ? '等待重试' : '等待领取')} + +
+ ))} +
+ )}
{(() => { const waterTasks = activeTasks.filter(task => @@ -73,11 +120,16 @@ export default function GlobalTaskCenter({
)} + {otherTasks.length === 0 && waterTasks.length === 0 && visibleJobs.length > 0 && ( +
+

当前只有 Job 运行态，任务进度尚未写入 system_tasks。

+
+ )} ); })()}
-

任务中心只展示状态，不再锁定整个界面。需要互斥的操作由功能页按钮和后端任务冲突检查处理。

+

取消按钮只作用于可跟踪的 Task；纯 Job 取消需要在对应功能页或运维接口处理。

{isAdmin && (
{!showCancelTask ? ( diff --git a/frontend/src/components/app/AppOverlays.jsx b/frontend/src/components/app/AppOverlays.jsx index be43216..6141296 100644 --- a/frontend/src/components/app/AppOverlays.jsx +++ b/frontend/src/components/app/AppOverlays.jsx @@ -26,6 +26,7 @@ export default function AppOverlays({ licenseFileName, licenseUploadStatus, activeTasks, + runtimeSummary, showCancelTask, cancelTaskPwd, onShowCancelTask, @@ -97,11 +98,12 @@ export default function AppOverlays({ )} - {activeTasks.length > 0 && ( + {(activeTasks.length > 0 || Number(runtimeSummary?.jobs?.active_count || 0) > 0) && ( }> 0} + isVisible={activeTasks.length > 0 || Number(runtimeSummary?.jobs?.active_count || 0) > 0} activeTasks={activeTasks} + runtimeSummary={runtimeSummary} t={t} isAdmin={isAdmin} showCancelTask={showCancelTask} diff --git a/frontend/src/components/app/AppStatusHeader.jsx b/frontend/src/components/app/AppStatusHeader.jsx index 4e6be4e..51f5158 100644 --- a/frontend/src/components/app/AppStatusHeader.jsx +++ b/frontend/src/components/app/AppStatusHeader.jsx @@ -3,7 +3,7 @@ import defaultLogoUrl from '../../logo.jpg'; import { formatUtc } from '../../utils/appUiHelpers'; const ORGANIZATION_NAME = import.meta.env.VITE_APP_ORG_NAME || '黑龙江省自然资源卫星应用技术中心'; -const SYSTEM_NAME = import.meta.env.VITE_APP_SYSTEM_NAME || 'InSAR 自动化管理系统'; +const SYSTEM_NAME = import.meta.env.VITE_APP_SYSTEM_NAME || '雷达数据生产管理系统'; const SYSTEM_TAGLINE = import.meta.env.VITE_APP_SYSTEM_TAGLINE || '科研工程生产平台'; const LOGO_URL = import.meta.env.VITE_APP_LOGO_URL || defaultLogoUrl; @@ -14,11 +14,40 @@ function AppStatusHeader({ isReadOnlyUser, activeTasks, avgTaskProgress, + runtimeSummary, licenseStatus, onLogout, }) { const licenseOk = !!licenseStatus?.ok; - const hasActiveTasks = activeTasks.length > 0; + const worker = runtimeSummary?.worker || {}; + const jobs = runtimeSummary?.jobs || {}; + const scan = runtimeSummary?.scan || {}; + const workerCount = Number(worker.worker_count) || 0; + const runningJobs = Number(jobs.running_count) || 0; + const queuedJobs = Number(jobs.queued_count) || 0; + const scanJobs = Number(scan.active_job_count) || 0; + const scanRunningJobs = Number(scan.running_job_count) || 0; + const staleJobs = Number(worker.stale_running_job_count) || 0; + const hasRuntimeActivity = activeTasks.length > 0 || runningJobs > 0 || queuedJobs > 0; + const taskProgress = hasRuntimeActivity ? avgTaskProgress : 0; + let runtimeLabel = 'Worker 未连接'; + if (!runtimeSummary && activeTasks.length > 0) { + runtimeLabel = `运行中 ${activeTasks.length}`; + } else if (runningJobs > 0 && staleJobs > 0) { + runtimeLabel = `运行态待恢复 ${staleJobs}`; + } else if (runningJobs > 0) { + runtimeLabel = `执行中 ${runningJobs}`; + } else if (queuedJobs > 0) { + runtimeLabel = `排队 ${queuedJobs}`; + } else if (workerCount > 0) { + runtimeLabel = `Worker ${workerCount} 空闲`; + } + const runtimeDetail = !runtimeSummary && activeTasks.length > 0 + ? '任务状态来自兼容接口' + : workerCount > 0 + ? `Worker ${workerCount}` + : '无在线 worker'; + const staleDetail = staleJobs > 0 ? ` · 待恢复 ${staleJobs}` : ''; return ( <> @@ -50,11 +79,12 @@ function AppStatusHeader({
-
- {hasActiveTasks ? `运行中 ${activeTasks.length}` : '任务空闲'} - {hasActiveTasks && ( +
+ {runtimeLabel} + {runtimeDetail}{staleDetail}{scanJobs > 0 ? ` · 扫描 ${scanRunningJobs}/${scanJobs}` : ''} + {hasRuntimeActivity && (