diff --git a/.env.example b/.env.example index 64acea0..399de61 100644 --- a/.env.example +++ b/.env.example @@ -63,6 +63,9 @@ MONITOR_DINSAR_DIRS=D:\DInSARResult ORBIT_SOURCE_DIRS=D:\LT1_data_lsarorbit;D:\Sentinel1_EOF_Pool MONITOR_ORBIT_DIR=D:\LT1_data_lsarorbit +GF3_ARCHIVE_SOURCE_DIRS=D:\GF3_L1A_Image_Zip +GF3_ARCHIVE_EXTS=.zip,.tar,.tar.gz,.tgz +GF3_UNPACK_DELETE_ARCHIVE=true GF3_SOURCE_DIRS=D:\GF3_L1A_Image GF3_STORAGE_DIRS=D:\GF3_L2_Image @@ -101,6 +104,11 @@ IDL_DINSAR_DEM_BASE_FILE=D:\SRTM30m\SRTMDEM_RSP_SARscape SRTM_DEM_DIR=D:\SRTM30m GF3_GEO_DEM_PATH=D:\DEM\gf3_dem.jp2 WATER_RESULTS_DIR=D:\WaterResult +SAR_ANALYSIS_READY_ROOT=D:\production_results\sar_analysis_ready +SAR_ANALYSIS_WORK_ROOT=D:\production_results\sar_analysis_work +SAR_ANALYSIS_PREVIEW_ROOT=D:\production_results\sar_analysis_preview +SAR_ANALYSIS_NODATA_VALUE=-9999 +SAR_ANALYSIS_OUTPUT_COG=true # ----------------------------------------------------------------------------- @@ -215,16 +223,16 @@ PYINT_SMOKE_TEST_ENABLED=false # ----------------------------------------------------------------------------- -# 时序 InSAR +# 时序 InSAR（旧 ISCE2/MintPy 链路） # ----------------------------------------------------------------------------- -TIMESERIES_ENABLED=true +TIMESERIES_ENABLED=false TIMESERIES_WSL_DISTRO=Ubuntu-24.04 TIMESERIES_ENV_NAME=insar_wsl_v1 TIMESERIES_PYTHON=/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python TIMESERIES_WORK_ROOT=D:\Code\Insar_management_system_v2\backend\runtime\timeseries_work TIMESERIES_DEM_PATH=D:\SRTM30m\SRTMDEM_RSP_SARscape.wgs84 TIMESERIES_ORBIT_POOL_ISCE2=D:\orbit_pools\isce2 -TIMESERIES_EXPERIMENT_ROOT=D:\Code\Insar_management_system_v2\experiments\isce2_sbas_timeseries +TIMESERIES_EXPERIMENT_ROOT= TIMESERIES_STACK_PREP_SCRIPT= TIMESERIES_MATERIALIZE_SCRIPT= TIMESERIES_PREPARE_DEM_SCRIPT= @@ -232,7 +240,7 @@ TIMESERIES_STACK_RUNNER_SCRIPT= TIMESERIES_MINTPY_SBAS_SCRIPT= TIMESERIES_EXPORT_PUBLISH_SCRIPT= TIMESERIES_STACK_WORKFLOW=interferogram -TIMESERIES_DEFAULT_PROCESSOR_CODE=isce2_stack_mintpy +TIMESERIES_DEFAULT_PROCESSOR_CODE=legacy_isce2_stack_mintpy TIMESERIES_WSL_STEP_TIMEOUT_SECONDS=7200 TIMESERIES_ALLOW_SYNTHETIC_WATER_MASK=true SARSCAPE_SBAS_PARAMETER_TEMPLATE_PATH= diff --git a/backend/alembic/versions/0003_flood_pipeline_models.py b/backend/alembic/versions/0003_flood_pipeline_models.py new file mode 100644 index 0000000..818c520 --- /dev/null +++ b/backend/alembic/versions/0003_flood_pipeline_models.py @@ -0,0 +1,152 @@ +"""add flood pipeline models + +Revision ID: 0003 +Revises: 0002 +Create Date: 2026-05-14 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from geoalchemy2 import Geometry + +revision: str = "0003" +down_revision: Union[str, None] = "0002" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _has_table(table_name: str) -> bool: + return table_name in set(sa.inspect(op.get_bind()).get_table_names()) + + +def _has_index(table_name: str, index_name: str) -> bool: + inspector = sa.inspect(op.get_bind()) + return index_name in {idx["name"] for idx in inspector.get_indexes(table_name)} + + +def _create_index_if_missing( + table_name: str, + index_name: str, + columns: list[str], + *, + unique: bool = False, + **kwargs, +) -> None: + if _has_table(table_name) and not _has_index(table_name, index_name): + op.create_index(index_name, table_name, columns, unique=unique, **kwargs) + + +def upgrade() -> None: + op.execute("CREATE EXTENSION IF NOT EXISTS postgis") + + if not _has_table("water_extractions"): + op.create_table( + "water_extractions", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("scene_id", sa.Integer(), nullable=True), + sa.Column("processor", sa.String(length=32), nullable=False, server_default="otsu"), + sa.Column("task_id", sa.String(length=64), nullable=True), + sa.Column("input_path", sa.String(), nullable=True), + sa.Column("output_path", sa.String(), nullable=True), + sa.Column("preview_path", sa.String(), nullable=True), + sa.Column("vector_path", sa.String(), nullable=True), + sa.Column("water_area_km2", sa.Float(), nullable=True), + sa.Column("water_pixel_count", sa.Integer(), nullable=True), + sa.Column("threshold_value", sa.Float(), nullable=True), + sa.Column("metadata_json", sa.JSON(), nullable=True), + sa.Column("status", sa.String(length=16), nullable=False, server_default="PENDING"), + sa.Column("error_msg", sa.Text(), nullable=True), + sa.Column("created_at", sa.DateTime(), server_default=sa.text("now()"), nullable=True), + sa.Column("updated_at", sa.DateTime(), server_default=sa.text("now()"), nullable=True), + sa.ForeignKeyConstraint(["scene_id"], ["sar_scene_geo.id"]), + sa.PrimaryKeyConstraint("id"), + ) + _create_index_if_missing("water_extractions", "ix_water_extractions_scene_id", ["scene_id"]) + _create_index_if_missing("water_extractions", "ix_water_extractions_processor", ["processor"]) + _create_index_if_missing("water_extractions", "ix_water_extractions_task_id", ["task_id"]) + _create_index_if_missing("water_extractions", "ix_water_extractions_status", ["status"]) + + if not _has_table("flood_overlays"): + op.create_table( + "flood_overlays", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("detection_id", sa.Integer(), nullable=False), + sa.Column("flood_vector_path", sa.String(), nullable=True), + sa.Column("hazard_points_hit", sa.Integer(), nullable=False, server_default="0"), + sa.Column("hazard_points_near", sa.Integer(), nullable=False, server_default="0"), + sa.Column("hazard_points_total", sa.Integer(), nullable=False, server_default="0"), + sa.Column("dinsar_products_intersecting", sa.Integer(), nullable=False, server_default="0"), + sa.Column("affected_area_km2", sa.Float(), nullable=True), + sa.Column("summary_json", sa.JSON(), nullable=True), + sa.Column("created_at", sa.DateTime(), server_default=sa.text("now()"), nullable=True), + sa.ForeignKeyConstraint(["detection_id"], ["flood_detections.id"]), + sa.PrimaryKeyConstraint("id"), + ) + _create_index_if_missing("flood_overlays", "ix_flood_overlays_detection_id", ["detection_id"]) + + if not _has_table("flood_products"): + op.create_table( + "flood_products", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("product_id", sa.String(length=64), nullable=False), + sa.Column("detection_id", sa.Integer(), nullable=True), + sa.Column("overlay_id", sa.Integer(), nullable=True), + sa.Column("display_name", sa.String(length=255), nullable=False), + sa.Column("status", sa.String(length=16), nullable=False, server_default="READY"), + sa.Column("publish_dir", sa.String(), nullable=True), + sa.Column("manifest_path", sa.String(), nullable=True), + sa.Column("geom", Geometry("POLYGON", srid=4326), nullable=True), + sa.Column("summary_json", sa.JSON(), nullable=True), + sa.Column("created_at", sa.DateTime(), server_default=sa.text("now()"), nullable=True), + sa.ForeignKeyConstraint(["detection_id"], ["flood_detections.id"]), + sa.ForeignKeyConstraint(["overlay_id"], ["flood_overlays.id"]), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("publish_dir", name="uq_flood_products_publish_dir"), + sa.UniqueConstraint("manifest_path", name="uq_flood_products_manifest_path"), + ) + _create_index_if_missing("flood_products", "ix_flood_products_product_id", ["product_id"], unique=True) + _create_index_if_missing("flood_products", "ix_flood_products_detection_id", ["detection_id"]) + _create_index_if_missing("flood_products", "ix_flood_products_overlay_id", ["overlay_id"]) + _create_index_if_missing("flood_products", "ix_flood_products_status", ["status"]) + + if _has_table("water_detections") and _has_table("water_extractions"): + op.execute( + sa.text( + """ + INSERT INTO water_extractions ( + id, scene_id, processor, input_path, output_path, + water_area_km2, water_pixel_count, threshold_value, + status, error_msg, created_at, updated_at + ) + SELECT + wd.id, wd.scene_id, 'otsu', wd.input_path, wd.output_path, + wd.water_area_km2, wd.water_pixel_count, wd.otsu_threshold_db, + wd.status, wd.error_msg, wd.created_at, wd.updated_at + FROM water_detections wd + WHERE NOT EXISTS ( + SELECT 1 FROM water_extractions we WHERE we.id = wd.id + ) + """ + ) + ) + op.execute( + sa.text( + """ + SELECT setval( + pg_get_serial_sequence('water_extractions', 'id'), + COALESCE((SELECT MAX(id) FROM water_extractions), 1), + (SELECT COUNT(*) FROM water_extractions) > 0 + ) + """ + ) + ) + + +def downgrade() -> None: + if _has_table("flood_products"): + op.drop_table("flood_products") + if _has_table("flood_overlays"): + op.drop_table("flood_overlays") + if _has_table("water_extractions"): + op.drop_table("water_extractions") diff --git a/backend/alembic/versions/0004_sar_analysis_ready_fields.py b/backend/alembic/versions/0004_sar_analysis_ready_fields.py new file mode 100644 index 0000000..8bb34c5 --- /dev/null +++ b/backend/alembic/versions/0004_sar_analysis_ready_fields.py @@ -0,0 +1,86 @@ +"""add SAR analysis-ready scene fields + +Revision ID: 0004 +Revises: 0003 +Create Date: 2026-05-16 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0004" +down_revision: Union[str, None] = "0003" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _has_table(table_name: str) -> bool: + return table_name in set(sa.inspect(op.get_bind()).get_table_names()) + + +def _has_column(table_name: str, column_name: str) -> bool: + inspector = sa.inspect(op.get_bind()) + return column_name in {col["name"] for col in inspector.get_columns(table_name)} + + +def _has_index(table_name: str, index_name: str) -> bool: + inspector = sa.inspect(op.get_bind()) + return index_name in {idx["name"] for idx in inspector.get_indexes(table_name)} + + +def _add_column_if_missing(table_name: str, column: sa.Column) -> None: + if _has_table(table_name) and not _has_column(table_name, column.name): + op.add_column(table_name, column) + + +def _create_index_if_missing(table_name: str, index_name: str, columns: list[str]) -> None: + if _has_table(table_name) and not _has_index(table_name, index_name): + op.create_index(index_name, table_name, columns) + + +def _drop_index_if_exists(table_name: str, index_name: str) -> None: + if _has_table(table_name) and _has_index(table_name, index_name): + op.drop_index(index_name, table_name=table_name) + + +def _drop_column_if_exists(table_name: str, column_name: str) -> None: + if _has_table(table_name) and _has_column(table_name, column_name): + op.drop_column(table_name, column_name) + + +def upgrade() -> None: + table = "sar_scene_geo" + _add_column_if_missing(table, sa.Column("analysis_tif_path", sa.String(), nullable=True)) + _add_column_if_missing(table, sa.Column("analysis_dir", sa.String(), nullable=True)) + _add_column_if_missing(table, sa.Column("analysis_preview_path", sa.String(), nullable=True)) + _add_column_if_missing(table, sa.Column("analysis_engine", sa.String(length=32), nullable=True)) + _add_column_if_missing(table, sa.Column("analysis_profile", sa.String(length=64), nullable=True)) + _add_column_if_missing(table, sa.Column("analysis_backscatter_unit", sa.String(length=32), nullable=True)) + _add_column_if_missing(table, sa.Column("analysis_nodata_value", sa.Float(), nullable=True)) + _add_column_if_missing(table, sa.Column("analysis_metadata_json", sa.JSON(), nullable=True)) + _add_column_if_missing(table, sa.Column("analysis_quality_json", sa.JSON(), nullable=True)) + + _create_index_if_missing(table, "ix_sar_scene_geo_analysis_tif_path", ["analysis_tif_path"]) + _create_index_if_missing(table, "ix_sar_scene_geo_analysis_engine", ["analysis_engine"]) + _create_index_if_missing(table, "ix_sar_scene_geo_analysis_profile", ["analysis_profile"]) + + +def downgrade() -> None: + table = "sar_scene_geo" + _drop_index_if_exists(table, "ix_sar_scene_geo_analysis_profile") + _drop_index_if_exists(table, "ix_sar_scene_geo_analysis_engine") + _drop_index_if_exists(table, "ix_sar_scene_geo_analysis_tif_path") + + for column_name in ( + "analysis_quality_json", + "analysis_metadata_json", + "analysis_nodata_value", + "analysis_backscatter_unit", + "analysis_profile", + "analysis_engine", + "analysis_preview_path", + "analysis_dir", + "analysis_tif_path", + ): + _drop_column_if_exists(table, column_name) diff --git a/backend/app/config.py b/backend/app/config.py index f5a8ab0..5b3e733 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -195,9 +195,17 @@ class Settings(BaseSettings): RADAR_PREVIEW_BUILD_ON_DEMAND: bool = True WATER_RESULTS_DIR: str = "" + SAR_ANALYSIS_READY_ROOT: str = "" + SAR_ANALYSIS_WORK_ROOT: str = "" + SAR_ANALYSIS_PREVIEW_ROOT: str = "" + SAR_ANALYSIS_NODATA_VALUE: float = -9999.0 + SAR_ANALYSIS_OUTPUT_COG: bool = True SRTM_DEM_DIR: str = "" GF3_GEO_DEM_PATH: str = "" + GF3_ARCHIVE_SOURCE_DIRS: str = "" + GF3_ARCHIVE_EXTS: str = ".zip,.tar,.tar.gz,.tgz" + GF3_UNPACK_DELETE_ARCHIVE: bool = True GF3_SOURCE_DIRS: str = "" GF3_STORAGE_DIRS: str = "" @@ -306,7 +314,7 @@ class Settings(BaseSettings): JOB_WORKER_STALE_RUNNING_SECONDS: int = 300 JOB_WORKER_HEARTBEAT_INTERVAL: float = 5.0 - TIMESERIES_ENABLED: bool = True + TIMESERIES_ENABLED: bool = False TIMESERIES_WSL_DISTRO: str = "" TIMESERIES_PYTHON: str = "" TIMESERIES_ENV_NAME: str = "" @@ -354,6 +362,29 @@ class Settings(BaseSettings): "WATER_RESULTS_DIR", os.path.join(backend_dir, "water_results"), ) + if not self.SAR_ANALYSIS_READY_ROOT: + object.__setattr__( + self, + "SAR_ANALYSIS_READY_ROOT", + os.path.join(backend_dir, "runtime", "sar_analysis_ready"), + ) + if not self.SAR_ANALYSIS_WORK_ROOT: + object.__setattr__( + self, + "SAR_ANALYSIS_WORK_ROOT", + os.path.join(backend_dir, "runtime", "sar_analysis_work"), + ) + if not self.SAR_ANALYSIS_PREVIEW_ROOT: + object.__setattr__( + self, + "SAR_ANALYSIS_PREVIEW_ROOT", + os.path.join(backend_dir, "runtime", "sar_analysis_preview"), + ) + object.__setattr__( + self, + "SAR_ANALYSIS_NODATA_VALUE", + float(self.SAR_ANALYSIS_NODATA_VALUE if self.SAR_ANALYSIS_NODATA_VALUE is not None else -9999.0), + ) if not self.SRTM_DEM_DIR: object.__setattr__(self, "SRTM_DEM_DIR", os.path.join(backend_dir, "dem_data")) if not self.GF3_STORAGE_DIRS: @@ -592,104 +623,105 @@ class Settings(BaseSettings): ) if not self.PYINT_ORBIT_POOL_TXT: object.__setattr__(self, "PYINT_ORBIT_POOL_TXT", self.ORBIT_POOL_ENVI) - if not self.TIMESERIES_WSL_DISTRO: - object.__setattr__(self, "TIMESERIES_WSL_DISTRO", self.WSL_DISTRO or self.ISCE2_WSL_DISTRO) - if not self.TIMESERIES_ENV_NAME: - object.__setattr__( - self, - "TIMESERIES_ENV_NAME", - str(self.WSL_SHARED_CONDA_ENV or "insar_wsl_v1").strip() or "insar_wsl_v1", - ) - if not self.TIMESERIES_PYTHON: - shared_python = str(self.WSL_SHARED_PYTHON or "").strip() - if shared_python: - object.__setattr__(self, "TIMESERIES_PYTHON", shared_python) - else: - env_name = ( - str(self.TIMESERIES_ENV_NAME or "insar_wsl_v1").strip() - or "insar_wsl_v1" - ) + if self.TIMESERIES_ENABLED: + if not self.TIMESERIES_WSL_DISTRO: + object.__setattr__(self, "TIMESERIES_WSL_DISTRO", self.WSL_DISTRO or self.ISCE2_WSL_DISTRO) + if not self.TIMESERIES_ENV_NAME: object.__setattr__( self, - "TIMESERIES_PYTHON", - f"/home/administrator/miniconda3/envs/{env_name}/bin/python", + "TIMESERIES_ENV_NAME", + str(self.WSL_SHARED_CONDA_ENV or "insar_wsl_v1").strip() or "insar_wsl_v1", + ) + if not self.TIMESERIES_PYTHON: + shared_python = str(self.WSL_SHARED_PYTHON or "").strip() + if shared_python: + object.__setattr__(self, "TIMESERIES_PYTHON", shared_python) + else: + env_name = ( + str(self.TIMESERIES_ENV_NAME or "insar_wsl_v1").strip() + or "insar_wsl_v1" + ) + object.__setattr__( + self, + "TIMESERIES_PYTHON", + f"/home/administrator/miniconda3/envs/{env_name}/bin/python", + ) + if not self.TIMESERIES_WORK_ROOT: + object.__setattr__( + self, + "TIMESERIES_WORK_ROOT", + os.path.join(backend_dir, "runtime", "timeseries_work"), + ) + if not self.TIMESERIES_DEM_PATH: + object.__setattr__(self, "TIMESERIES_DEM_PATH", self.ISCE2_DEM_PATH) + if not self.TIMESERIES_ORBIT_POOL_ISCE2: + object.__setattr__(self, "TIMESERIES_ORBIT_POOL_ISCE2", self.ORBIT_POOL_ISCE2) + if not self.TIMESERIES_EXPERIMENT_ROOT: + object.__setattr__( + self, + "TIMESERIES_EXPERIMENT_ROOT", + os.path.join(project_root, "experiments", "isce2_sbas_timeseries"), + ) + if not self.TIMESERIES_STACK_PREP_SCRIPT: + object.__setattr__( + self, + "TIMESERIES_STACK_PREP_SCRIPT", + os.path.join( + self.TIMESERIES_EXPERIMENT_ROOT, + "scripts", + "build_lt1_stack_prep.py", + ), + ) + if not self.TIMESERIES_MATERIALIZE_SCRIPT: + object.__setattr__( + self, + "TIMESERIES_MATERIALIZE_SCRIPT", + os.path.join( + self.TIMESERIES_EXPERIMENT_ROOT, + "scripts", + "materialize_lt1_stack_scenes.py", + ), + ) + if not self.TIMESERIES_PREPARE_DEM_SCRIPT: + object.__setattr__( + self, + "TIMESERIES_PREPARE_DEM_SCRIPT", + os.path.join( + self.TIMESERIES_EXPERIMENT_ROOT, + "scripts", + "prepare_lt1_stack_dem.py", + ), + ) + if not self.TIMESERIES_STACK_RUNNER_SCRIPT: + object.__setattr__( + self, + "TIMESERIES_STACK_RUNNER_SCRIPT", + os.path.join( + self.TIMESERIES_EXPERIMENT_ROOT, + "scripts", + "run_generated_stack_runfile_ubuntu2404.sh", + ), + ) + if not self.TIMESERIES_MINTPY_SBAS_SCRIPT: + object.__setattr__( + self, + "TIMESERIES_MINTPY_SBAS_SCRIPT", + os.path.join( + self.TIMESERIES_EXPERIMENT_ROOT, + "scripts", + "run_mintpy_sbas_unified_env_smoketest_ubuntu2404.sh", + ), + ) + if not self.TIMESERIES_EXPORT_PUBLISH_SCRIPT: + object.__setattr__( + self, + "TIMESERIES_EXPORT_PUBLISH_SCRIPT", + os.path.join( + self.TIMESERIES_EXPERIMENT_ROOT, + "scripts", + "export_mintpy_publish_products_unified_env_ubuntu2404.sh", + ), ) - if not self.TIMESERIES_WORK_ROOT: - object.__setattr__( - self, - "TIMESERIES_WORK_ROOT", - os.path.join(backend_dir, "runtime", "timeseries_work"), - ) - if not self.TIMESERIES_DEM_PATH: - object.__setattr__(self, "TIMESERIES_DEM_PATH", self.ISCE2_DEM_PATH) - if not self.TIMESERIES_ORBIT_POOL_ISCE2: - object.__setattr__(self, "TIMESERIES_ORBIT_POOL_ISCE2", self.ORBIT_POOL_ISCE2) - if not self.TIMESERIES_EXPERIMENT_ROOT: - object.__setattr__( - self, - "TIMESERIES_EXPERIMENT_ROOT", - os.path.join(project_root, "experiments", "isce2_sbas_timeseries"), - ) - if not self.TIMESERIES_STACK_PREP_SCRIPT: - object.__setattr__( - self, - "TIMESERIES_STACK_PREP_SCRIPT", - os.path.join( - self.TIMESERIES_EXPERIMENT_ROOT, - "scripts", - "build_lt1_stack_prep.py", - ), - ) - if not self.TIMESERIES_MATERIALIZE_SCRIPT: - object.__setattr__( - self, - "TIMESERIES_MATERIALIZE_SCRIPT", - os.path.join( - self.TIMESERIES_EXPERIMENT_ROOT, - "scripts", - "materialize_lt1_stack_scenes.py", - ), - ) - if not self.TIMESERIES_PREPARE_DEM_SCRIPT: - object.__setattr__( - self, - "TIMESERIES_PREPARE_DEM_SCRIPT", - os.path.join( - self.TIMESERIES_EXPERIMENT_ROOT, - "scripts", - "prepare_lt1_stack_dem.py", - ), - ) - if not self.TIMESERIES_STACK_RUNNER_SCRIPT: - object.__setattr__( - self, - "TIMESERIES_STACK_RUNNER_SCRIPT", - os.path.join( - self.TIMESERIES_EXPERIMENT_ROOT, - "scripts", - "run_generated_stack_runfile_ubuntu2404.sh", - ), - ) - if not self.TIMESERIES_MINTPY_SBAS_SCRIPT: - object.__setattr__( - self, - "TIMESERIES_MINTPY_SBAS_SCRIPT", - os.path.join( - self.TIMESERIES_EXPERIMENT_ROOT, - "scripts", - "run_mintpy_sbas_unified_env_smoketest_ubuntu2404.sh", - ), - ) - if not self.TIMESERIES_EXPORT_PUBLISH_SCRIPT: - object.__setattr__( - self, - "TIMESERIES_EXPORT_PUBLISH_SCRIPT", - os.path.join( - self.TIMESERIES_EXPERIMENT_ROOT, - "scripts", - "export_mintpy_publish_products_unified_env_ubuntu2404.sh", - ), - ) if not self.SARSCAPE_SBAS_PARAMETER_TEMPLATE_PATH: object.__setattr__( self, @@ -715,12 +747,16 @@ class Settings(BaseSettings): os.makedirs(settings.TIMESERIES_PRODUCT_DIR, exist_ok=True) os.makedirs(settings.PSINSAR_PRODUCT_DIR, exist_ok=True) os.makedirs(settings.RESULT_QUARANTINE_ROOT, exist_ok=True) + os.makedirs(settings.SAR_ANALYSIS_READY_ROOT, exist_ok=True) + os.makedirs(settings.SAR_ANALYSIS_WORK_ROOT, exist_ok=True) + os.makedirs(settings.SAR_ANALYSIS_PREVIEW_ROOT, exist_ok=True) os.makedirs(settings.WSL_BROKER_JOB_ROOT, exist_ok=True) os.makedirs(settings.PYINT_TEMPLATE_ROOT, exist_ok=True) os.makedirs(settings.PYINT_WORK_ROOT, exist_ok=True) os.makedirs(settings.PYINT_OUTPUT_ROOT, exist_ok=True) os.makedirs(settings.PYINT_DEM_ROOT, exist_ok=True) - os.makedirs(settings.TIMESERIES_WORK_ROOT, exist_ok=True) + if settings.TIMESERIES_ENABLED and settings.TIMESERIES_WORK_ROOT: + os.makedirs(settings.TIMESERIES_WORK_ROOT, exist_ok=True) settings = Settings() @@ -861,6 +897,9 @@ def validate_runtime_config() -> dict[str, Any]: _check_path(label="GF3_GEO_DEM_PATH", value=settings.GF3_GEO_DEM_PATH, errors=errors, warnings=warnings, expect_file=True) _check_path(label="SRTM_DEM_DIR", value=settings.SRTM_DEM_DIR, errors=errors, warnings=warnings, expect_file=False) _check_path(label="WATER_RESULTS_DIR", value=settings.WATER_RESULTS_DIR, errors=errors, warnings=warnings, expect_file=False) + _check_path(label="SAR_ANALYSIS_READY_ROOT", value=settings.SAR_ANALYSIS_READY_ROOT, errors=errors, warnings=warnings, expect_file=False) + _check_path(label="SAR_ANALYSIS_WORK_ROOT", value=settings.SAR_ANALYSIS_WORK_ROOT, errors=errors, warnings=warnings, expect_file=False) + _check_path(label="SAR_ANALYSIS_PREVIEW_ROOT", value=settings.SAR_ANALYSIS_PREVIEW_ROOT, errors=errors, warnings=warnings, expect_file=False) _check_path(label="MONITOR_ORBIT_DIR", value=settings.MONITOR_ORBIT_DIR, errors=errors, warnings=warnings, expect_file=False) for label, value in ( ("SOURCE_PRODUCT_DIRS", settings.SOURCE_PRODUCT_DIRS), @@ -887,6 +926,7 @@ def validate_runtime_config() -> dict[str, Any]: ("INSAR_STORAGE_DIRS", settings.INSAR_STORAGE_DIRS), ("MONITOR_RADAR_DIRS", settings.MONITOR_RADAR_DIRS), ("MONITOR_DINSAR_DIRS", settings.MONITOR_DINSAR_DIRS), + ("GF3_ARCHIVE_SOURCE_DIRS", settings.GF3_ARCHIVE_SOURCE_DIRS), ("GF3_SOURCE_DIRS", settings.GF3_SOURCE_DIRS), ("GF3_STORAGE_DIRS", settings.GF3_STORAGE_DIRS), ): @@ -1030,6 +1070,11 @@ def validate_runtime_config() -> dict[str, Any]: "Gamma runtime has not been fully migrated to the fixed profile model." ) + if not settings.TIMESERIES_ENABLED: + info.append( + "Legacy ISCE2/MintPy timeseries pipeline is disabled; current SBAS-InSAR production uses the Gamma /sbas-insar-production workflow." + ) + if settings.TIMESERIES_ENABLED: _check_path( label="TIMESERIES_PYTHON", diff --git a/backend/app/db_maintenance.py b/backend/app/db_maintenance.py index db97927..035c34a 100644 --- a/backend/app/db_maintenance.py +++ b/backend/app/db_maintenance.py @@ -6,7 +6,7 @@ from urllib.parse import urlparse from sqlalchemy import create_engine, inspect, text from sqlalchemy.orm import sessionmaker -from sqlalchemy.schema import CreateColumn +from sqlalchemy.schema import CreateColumn, CreateIndex from .config import read_bool_env, settings @@ -23,9 +23,12 @@ POSTGIS_VIEWS = { } ALLOWED_EXTRA_TABLES = { + "alembic_version", "spatial_query_logs", } +ALEMBIC_HEAD_REVISION = "0004" + MIGRATION_FILES = [ "001_st_intersection_agg.sql", "002_spatial_functions.sql", @@ -337,6 +340,50 @@ def _add_missing_columns(conn, inspector, base, dialect) -> List[str]: return added_columns +def _create_missing_indexes(conn, inspector, base, dialect) -> List[str]: + created_indexes: List[str] = [] + existing_tables = set(inspector.get_table_names()) + for table_name, table in base.metadata.tables.items(): + if table_name not in existing_tables: + continue + + current_indexes = {index["name"] for index in inspector.get_indexes(table_name)} + for index in sorted(table.indexes, key=lambda item: item.name or ""): + if not index.name or index.name in current_indexes: + continue + try: + index_sql = str(CreateIndex(index).compile(dialect=dialect)) + conn.exec_driver_sql(index_sql) + created_indexes.append(index.name) + current_indexes.add(index.name) + except Exception as exc: + print(f"[WARN] Failed to create index {table_name}.{index.name}: {exc}") + return created_indexes + + +def _ensure_alembic_version_marker(conn, revision: str = ALEMBIC_HEAD_REVISION) -> Dict[str, Any]: + status: Dict[str, Any] = {"revision": revision, "created": False, "updated": False} + conn.exec_driver_sql( + """ + CREATE TABLE IF NOT EXISTS alembic_version ( + version_num VARCHAR(32) NOT NULL + ) + """ + ) + rows = conn.execute(text("SELECT version_num FROM alembic_version")).fetchall() + previous = [row[0] for row in rows] + status["previous"] = previous + if not rows: + conn.execute(text("INSERT INTO alembic_version (version_num) VALUES (:revision)"), {"revision": revision}) + status["created"] = True + return status + if previous != [revision] or len(rows) != 1: + conn.execute(text("DELETE FROM alembic_version")) + conn.execute(text("INSERT INTO alembic_version (version_num) VALUES (:revision)"), {"revision": revision}) + status["updated"] = True + return status + + def _resolve_hazard_shapefile() -> str: hazard_dir = settings.HAZARD_POINTS_DIR hazard_filename = settings.HAZARD_POINTS_FILENAME or "Point.shp" @@ -513,6 +560,8 @@ def ensure_database_ready( "mismatch_detected": False, "mismatch_reasons": [], "added_columns": [], + "created_indexes": [], + "alembic_version": None, "applied_sql_files": [], "admin": None, "hazard_seed": None, @@ -545,10 +594,14 @@ def ensure_database_ready( base.metadata.create_all(bind=conn) inspector = inspect(conn) result["added_columns"] = _add_missing_columns(conn, inspector, base, engine.dialect) + inspector = inspect(conn) + result["created_indexes"] = _create_missing_indexes(conn, inspector, base, engine.dialect) if bootstrap_required: result["bootstrap_initialized"] = True else: base.metadata.create_all(bind=conn) + inspector = inspect(conn) + result["created_indexes"] = _create_missing_indexes(conn, inspector, base, engine.dialect) migrations_dir = os.path.join(project_root(), "backend", "migrations") for migration_file in MIGRATION_FILES: @@ -556,6 +609,12 @@ def ensure_database_ready( if _apply_sql_file(conn, migration_path): result["applied_sql_files"].append(migration_file) + final_diagnostics = inspect_database_structure(conn) + if not final_diagnostics.get("mismatch"): + result["alembic_version"] = _ensure_alembic_version_marker(conn) + else: + result["post_maintenance_mismatch_reasons"] = final_diagnostics.get("reasons", []) + session = Session() try: if bootstrap_admin: diff --git a/backend/app/main.py b/backend/app/main.py index 46b97f1..63d8367 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -108,6 +108,11 @@ async def lifespan(app: FastAPI): ) if maintenance_result.get("added_columns"): print(f">>> [DB] Added columns: {maintenance_result['added_columns']}") + if maintenance_result.get("created_indexes"): + print(f">>> [DB] Created indexes: {maintenance_result['created_indexes']}") + if maintenance_result.get("alembic_version"): + alembic_status = maintenance_result["alembic_version"] + print(f">>> [DB] Alembic version marker: {alembic_status.get('revision')}") if maintenance_result.get("admin", {}).get("message"): print(f">>> [DB] {maintenance_result['admin']['message']}") if maintenance_result.get("hazard_seed", {}).get("message"): diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index 2ea15ea..1a9d4aa 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -52,6 +52,9 @@ from .orm import ( SARSceneGeoORM, FloodDetectionORM, WaterDetectionORM, + WaterExtractionORM, + FloodOverlayORM, + FloodProductORM, GF3ProcessingORM, AiDiagnosisORM, ) @@ -110,7 +113,8 @@ __all__ = [ "PsTimeseriesRunORM", "AuthUserORM", "AuthSessionORM", "AuthAuditLogORM", "AuthRateLimitORM", "SARSceneGeoORM", "FloodDetectionORM", - "WaterDetectionORM", "GF3ProcessingORM", "AiDiagnosisORM", + "WaterDetectionORM", "WaterExtractionORM", "FloodOverlayORM", "FloodProductORM", + "GF3ProcessingORM", "AiDiagnosisORM", # Schemas "HazardPoint", "DinsarResult", "ScanRequest", "ManagedRootInfo", "ScanCursorInfo", "RadarData", "RadarDataPage", "DinsarResultPage", diff --git a/backend/app/models/orm.py b/backend/app/models/orm.py index 984c508..5167eb4 100644 --- a/backend/app/models/orm.py +++ b/backend/app/models/orm.py @@ -1478,6 +1478,15 @@ class SARSceneGeoORM(Base): radar_data_id = Column(Integer, ForeignKey("radar_data.id"), index=True, nullable=False) geo_path = Column(String, nullable=True) # 地理编码 dB 文件路径（ENVI 格式，无扩展名） pixel_size_m = Column(Float, nullable=True) # 输出像素大小（m） + analysis_tif_path = Column(String, nullable=True, index=True) + analysis_dir = Column(String, nullable=True) + analysis_preview_path = Column(String, nullable=True) + analysis_engine = Column(String(32), nullable=True, index=True) + analysis_profile = Column(String(64), nullable=True, index=True) + analysis_backscatter_unit = Column(String(32), nullable=True) + analysis_nodata_value = Column(Float, nullable=True) + analysis_metadata_json = Column(JSON, nullable=True) + analysis_quality_json = Column(JSON, nullable=True) status = Column(String, nullable=False, default="PENDING", index=True) error_msg = Column(Text, nullable=True) created_at = Column(DateTime, server_default=func.now()) @@ -1514,6 +1523,8 @@ class FloodDetectionORM(Base): pre_scene = relationship("SARSceneGeoORM", foreign_keys=[pre_scene_id], back_populates="pre_flood_detections") post_scene = relationship("SARSceneGeoORM", foreign_keys=[post_scene_id], back_populates="post_flood_detections") + overlays = relationship("FloodOverlayORM", back_populates="detection") + products = relationship("FloodProductORM", back_populates="detection", foreign_keys="FloodProductORM.detection_id") __table_args__ = ( UniqueConstraint("pre_scene_id", "post_scene_id", name="uq_flood_detection_pair"), @@ -1537,6 +1548,69 @@ class WaterDetectionORM(Base): updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) +class WaterExtractionORM(Base): + """Water extraction result used by the flood-analysis pipeline.""" + __tablename__ = "water_extractions" + + id = Column(Integer, primary_key=True, autoincrement=True) + scene_id = Column(Integer, ForeignKey("sar_scene_geo.id"), nullable=True, index=True) + processor = Column(String(32), nullable=False, default="otsu", index=True) + task_id = Column(String(64), nullable=True, index=True) + input_path = Column(String, nullable=True) + output_path = Column(String, nullable=True) + preview_path = Column(String, nullable=True) + vector_path = Column(String, nullable=True) + water_area_km2 = Column(Float, nullable=True) + water_pixel_count = Column(Integer, nullable=True) + threshold_value = Column(Float, nullable=True) + metadata_json = Column(JSON, nullable=True) + status = Column(String(16), nullable=False, default="PENDING", index=True) + error_msg = Column(Text, nullable=True) + created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + + scene = relationship("SARSceneGeoORM", foreign_keys=[scene_id]) + + +class FloodOverlayORM(Base): + """Spatial overlay result for a flood detection.""" + __tablename__ = "flood_overlays" + + id = Column(Integer, primary_key=True, autoincrement=True) + detection_id = Column(Integer, ForeignKey("flood_detections.id"), nullable=False, index=True) + flood_vector_path = Column(String, nullable=True) + hazard_points_hit = Column(Integer, nullable=False, default=0) + hazard_points_near = Column(Integer, nullable=False, default=0) + hazard_points_total = Column(Integer, nullable=False, default=0) + dinsar_products_intersecting = Column(Integer, nullable=False, default=0) + affected_area_km2 = Column(Float, nullable=True) + summary_json = Column(JSON, nullable=True) + created_at = Column(DateTime, server_default=func.now()) + + detection = relationship("FloodDetectionORM", back_populates="overlays") + products = relationship("FloodProductORM", back_populates="overlay", foreign_keys="FloodProductORM.overlay_id") + + +class FloodProductORM(Base): + """Published flood-analysis product package.""" + __tablename__ = "flood_products" + + id = Column(Integer, primary_key=True, autoincrement=True) + product_id = Column(String(64), unique=True, index=True, nullable=False) + detection_id = Column(Integer, ForeignKey("flood_detections.id"), nullable=True, index=True) + overlay_id = Column(Integer, ForeignKey("flood_overlays.id"), nullable=True, index=True) + display_name = Column(String(255), nullable=False) + status = Column(String(16), nullable=False, default="READY", index=True) + publish_dir = Column(String, unique=True, nullable=True) + manifest_path = Column(String, unique=True, nullable=True) + geom = Column(Geometry("POLYGON", srid=4326), nullable=True) + summary_json = Column(JSON, nullable=True) + created_at = Column(DateTime, server_default=func.now()) + + detection = relationship("FloodDetectionORM", back_populates="products", foreign_keys=[detection_id]) + overlay = relationship("FloodOverlayORM", back_populates="products", foreign_keys=[overlay_id]) + + class GF3ProcessingORM(Base): """GF3 L1A→L2 处理记录（辐射定标 + RPC 几何校正）。""" __tablename__ = "gf3_processing" diff --git a/backend/app/pyint_pipeline/run_gamma_scene_preprocess.py b/backend/app/pyint_pipeline/run_gamma_scene_preprocess.py new file mode 100644 index 0000000..d51e682 --- /dev/null +++ b/backend/app/pyint_pipeline/run_gamma_scene_preprocess.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +"""Single-scene Gamma preprocessing to analysis-ready GeoTIFF. + +The script is intentionally narrower than the full PyINT DInSAR pipeline: +LT source product -> Gamma SLC -> multilook amplitude -> geocode -> GeoTIFF. +It is executed inside WSL by backend.app.services.lt_gamma_scene_service. +""" +from __future__ import annotations + +import argparse +import json +import math +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Preprocess a SAR scene with Gamma/PyINT.") + parser.add_argument("--source-path", required=True) + parser.add_argument("--output-dir", required=True) + parser.add_argument("--work-dir", required=True) + parser.add_argument("--pyint-home", required=True) + parser.add_argument("--dem-root", required=True) + parser.add_argument("--prepared-dem-path", default="") + parser.add_argument("--project-name", required=True) + parser.add_argument("--date", required=True) + parser.add_argument("--satellite-family", default="LT1") + parser.add_argument("--range-looks", type=int, default=2) + parser.add_argument("--azimuth-looks", type=int, default=2) + parser.add_argument("--geo-interp", default="1") + parser.add_argument("--nodata-value", type=float, default=-9999.0) + parser.add_argument("--to-db", action="store_true") + return parser.parse_args() + + +def run_logged(command: list[str], *, cwd: Path, env: dict[str, str], log_dir: Path, stage: str) -> dict[str, Any]: + log_dir.mkdir(parents=True, exist_ok=True) + stdout_path = log_dir / f"{stage}.stdout.log" + stderr_path = log_dir / f"{stage}.stderr.log" + result = subprocess.run(command, cwd=str(cwd), env=env, text=True, capture_output=True, check=False) + stdout_path.write_text(result.stdout or "", encoding="utf-8", errors="ignore") + stderr_path.write_text(result.stderr or "", encoding="utf-8", errors="ignore") + if result.returncode != 0: + detail = (result.stderr or result.stdout or "").strip() + raise RuntimeError(f"{stage} failed rc={result.returncode}: {' '.join(command)}\n{detail}") + return { + "stage": stage, + "command": command, + "returncode": result.returncode, + "stdout_path": str(stdout_path), + "stderr_path": str(stderr_path), + } + + +def read_gamma_par(path: Path, key: str) -> str: + wanted = str(key or "").strip().rstrip(":") + with path.open("r", encoding="utf-8", errors="ignore") as stream: + for line in stream: + stripped = line.strip() + if not stripped: + continue + label = stripped.split()[0].rstrip(":") + if label != wanted: + continue + tail = stripped.split(":", 1)[1] if ":" in stripped else " ".join(stripped.split()[1:]) + tokens = tail.strip().split() + if tokens: + return tokens[0] + raise KeyError(f"Cannot read {key} from {path}") + + +def discover_lt_inputs(source_path: Path, date: str) -> list[Path]: + patterns = [f"LT1*{date}*.tar.gz", f"LT1*{date}*.tiff", f"LT1*{date}*.tif"] + if source_path.is_file(): + return [source_path] + if not source_path.is_dir(): + raise FileNotFoundError(f"Source path does not exist: {source_path}") + found: list[Path] = [] + for pattern in patterns: + found.extend(path for path in source_path.rglob(pattern) if path.is_file()) + return sorted(set(found)) + + +def stage_lt_inputs(source_path: Path, download_dir: Path, date: str) -> list[str]: + download_dir.mkdir(parents=True, exist_ok=True) + inputs = discover_lt_inputs(source_path, date) + if not inputs: + raise FileNotFoundError(f"No LT inputs for date {date} under {source_path}") + + staged: list[str] = [] + for source in inputs: + target = download_dir / source.name + shutil.copy2(source, target) + staged.append(str(target)) + + lower_name = source.name.lower() + if lower_name.endswith((".tiff", ".tif")): + base_candidates = [ + source.with_suffix(source.suffix + ".meta.xml"), + source.with_suffix(".meta.xml"), + source.with_name(source.stem + ".meta.xml"), + ] + for meta in base_candidates: + if meta.is_file(): + shutil.copy2(meta, download_dir / meta.name) + break + return staged + + +def write_template( + *, + template_path: Path, + date: str, + range_looks: int, + azimuth_looks: int, + geo_interp: str, + prepared_dem_path: str, +) -> None: + lines = [ + "satelite = LT", + f"masterDate = {date}", + f"range_looks = {range_looks}", + f"azimuth_looks = {azimuth_looks}", + "dem_lat_ovr = 0.5", + "dem_lon_ovr = 0.5", + "Simphase_rpos = -", + "Simphase_azpos = -", + "Simphase_rwin = 256", + "Simphase_azwin = 256", + "Simphase_thresh = -", + f"geo_interp = {geo_interp}", + ] + dem = str(prepared_dem_path or "").strip() + if dem and Path(dem).is_file() and Path(dem + ".par").is_file(): + lines.append(f"DEM = {dem}") + template_path.parent.mkdir(parents=True, exist_ok=True) + template_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def convert_to_db_geotiff(source_tif: Path, target_tif: Path, nodata_value: float) -> dict[str, Any]: + try: + import numpy as np + import rasterio + except Exception as exc: + shutil.copy2(source_tif, target_tif) + return { + "target": str(target_tif), + "backscatter_unit": "gamma_mli_power", + "warning": f"rasterio/numpy unavailable; kept power values: {exc}", + } + + with rasterio.open(source_tif) as src: + data = src.read(1).astype("float32") + profile = src.profile.copy() + src_nodata = src.nodata + + invalid = ~np.isfinite(data) + if src_nodata is not None: + invalid |= data == src_nodata + invalid |= data <= 0 + db_data = np.full(data.shape, nodata_value, dtype="float32") + db_data[~invalid] = (10.0 * np.log10(data[~invalid])).astype("float32") + + profile.update(dtype="float32", count=1, nodata=nodata_value, compress="deflate") + target_tif.parent.mkdir(parents=True, exist_ok=True) + with rasterio.open(target_tif, "w", **profile) as dst: + dst.write(db_data, 1) + return {"target": str(target_tif), "backscatter_unit": "gamma_mli_db"} + + +def main() -> int: + args = parse_args() + source_path = Path(args.source_path).resolve() + output_dir = Path(args.output_dir).resolve() + work_root = Path(args.work_dir).resolve() + pyint_home = Path(args.pyint_home).resolve() + dem_root = Path(args.dem_root).resolve() + project_name = re.sub(r"[^0-9A-Za-z._-]+", "_", args.project_name).strip("._-") or "sar_scene" + date = re.sub(r"\D", "", str(args.date or ""))[:8] + if not re.fullmatch(r"20\d{6}", date): + raise ValueError(f"Invalid scene date: {args.date}") + + scratch_dir = work_root / "scratch" + template_dir = work_root / "templates" + project_dir = scratch_dir / project_name + download_dir = project_dir / "DOWNLOAD" + log_dir = output_dir / "logs" + output_dir.mkdir(parents=True, exist_ok=True) + dem_root.mkdir(parents=True, exist_ok=True) + + env = os.environ.copy() + env["SCRATCHDIR"] = str(scratch_dir) + env["TEMPLATEDIR"] = str(template_dir) + env["DEMDIR"] = str(dem_root) + env["PYTHONPATH"] = f"{pyint_home}:{env.get('PYTHONPATH', '')}" + env["PATH"] = f"{pyint_home / 'pyint'}:{env.get('PATH', '')}" + + staged_inputs = stage_lt_inputs(source_path, download_dir, date) + template_path = template_dir / f"{project_name}.template" + write_template( + template_path=template_path, + date=date, + range_looks=max(1, int(args.range_looks)), + azimuth_looks=max(1, int(args.azimuth_looks)), + geo_interp=str(args.geo_interp or "1"), + prepared_dem_path=args.prepared_dem_path, + ) + + commands: list[dict[str, Any]] = [] + commands.append( + run_logged( + [sys.executable, str(pyint_home / "pyint" / "down2slc_LT1.py"), project_name, date], + cwd=work_root, + env=env, + log_dir=log_dir, + stage="down2slc_lt1", + ) + ) + commands.append( + run_logged( + [sys.executable, str(pyint_home / "pyint" / "generate_rdc_dem.py"), project_name], + cwd=work_root, + env=env, + log_dir=log_dir, + stage="generate_rdc_dem", + ) + ) + + dem_dir = project_dir / "DEM" + range_looks = max(1, int(args.range_looks)) + amp = dem_dir / f"{date}_{range_looks}rlks.amp" + amp_par = dem_dir / f"{date}_{range_looks}rlks.amp.par" + utm_dem_par = dem_dir / f"{date}_{range_looks}rlks.utm.dem.par" + utm_to_rdc = dem_dir / f"{date}_{range_looks}rlks.UTM_TO_RDC" + for required in (amp, amp_par, utm_dem_par, utm_to_rdc): + if not required.is_file(): + raise FileNotFoundError(f"Required Gamma product missing: {required}") + + width = read_gamma_par(amp_par, "range_samples") + geo_width = read_gamma_par(utm_dem_par, "width") + geo_nlines = read_gamma_par(utm_dem_par, "nlines") + geo_amp = output_dir / "gamma_geo_amp" + commands.append( + run_logged( + [ + "geocode_back", + str(amp), + str(width), + str(utm_to_rdc), + str(geo_amp), + str(geo_width), + str(geo_nlines), + str(args.geo_interp or "1"), + "0", + ], + cwd=work_root, + env=env, + log_dir=log_dir, + stage="geocode_amp", + ) + ) + + power_tif = output_dir / "analysis_ready_power.tif" + commands.append( + run_logged( + [ + "data2geotiff", + str(utm_dem_par), + str(geo_amp), + "2", + str(power_tif), + f"{float(args.nodata_value):g}", + ], + cwd=work_root, + env=env, + log_dir=log_dir, + stage="data2geotiff_amp", + ) + ) + if not power_tif.is_file() or power_tif.stat().st_size <= 0: + raise RuntimeError(f"data2geotiff did not create output: {power_tif}") + + final_tif = output_dir / "analysis_ready.tif" + conversion = convert_to_db_geotiff(power_tif, final_tif, float(args.nodata_value)) if args.to_db else { + "target": str(final_tif), + "backscatter_unit": "gamma_mli_power", + } + if not args.to_db: + shutil.copy2(power_tif, final_tif) + + manifest = { + "ok": True, + "satellite_family": args.satellite_family, + "project_name": project_name, + "date": date, + "source_path": str(source_path), + "staged_inputs": staged_inputs, + "work_dir": str(work_root), + "output_dir": str(output_dir), + "analysis_tif_path": str(final_tif), + "power_tif_path": str(power_tif), + "backscatter_unit": conversion.get("backscatter_unit"), + "gamma_products": { + "amp": str(amp), + "amp_par": str(amp_par), + "utm_dem_par": str(utm_dem_par), + "utm_to_rdc": str(utm_to_rdc), + "geo_amp": str(geo_amp), + }, + "looks": {"range": range_looks, "azimuth": max(1, int(args.azimuth_looks))}, + "commands": commands, + "conversion": conversion, + } + manifest_path = output_dir / "manifest.json" + manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8") + print(json.dumps({"ok": True, "manifest_path": str(manifest_path), "analysis_tif_path": str(final_tif)})) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/app/routers/__init__.py b/backend/app/routers/__init__.py index 5874879..a94db0f 100644 --- a/backend/app/routers/__init__.py +++ b/backend/app/routers/__init__.py @@ -22,6 +22,7 @@ from . import ( ps_products, radar, root_registry, + sbas_insar_production, stats, task_batches, tasks_runtime, @@ -53,6 +54,7 @@ def include_all_routers(router: APIRouter) -> None: router.include_router(dinsar.router) router.include_router(dinsar_products.router) router.include_router(dinsar_production.router) + router.include_router(sbas_insar_production.router) router.include_router(timeseries_production.router) router.include_router(ps_products.router) router.include_router(ai.router) diff --git a/backend/app/routers/flood.py b/backend/app/routers/flood.py index 87b5790..97ebdc2 100644 --- a/backend/app/routers/flood.py +++ b/backend/app/routers/flood.py @@ -1,45 +1,65 @@ -"""Flood disaster analysis router. - -This router exposes the flood-analysis business API while reusing the -existing water/flood processing records and jobs during migration. -""" +"""Flood disaster analysis router.""" from __future__ import annotations -from typing import Optional +from typing import Any, Optional -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends from pydantic import BaseModel, Field from sqlalchemy.ext.asyncio import AsyncSession from ..database import get_db from ..models import AuthUserORM -from . import water as water_compat +from ..services import flood_analysis_service +from ..services import flood_overlay_service +from ..services import flood_product_service from .dependencies import _get_current_user, _require_admin router = APIRouter() class FloodPreprocessRequest(BaseModel): - radar_data_id: int = Field(..., description="RadarDataORM 主键") + radar_data_id: int = Field(..., description="RadarDataORM primary key") class FloodWaterExtractionRequest(BaseModel): - scene_id: Optional[int] = Field(default=None, description="SARSceneGeoORM 主键") - input_path: Optional[str] = Field(default=None, description="直接指定 GeoTIFF/ENVI 路径") + scene_id: Optional[int] = Field(default=None, description="SARSceneGeoORM primary key") + input_path: Optional[str] = Field(default=None, description="Direct GeoTIFF/ENVI input path") class FloodPairSearchRequest(BaseModel): - pre_start: Optional[str] = Field(default=None, description="灾前开始日期 YYYYMMDD") - pre_end: Optional[str] = Field(default=None, description="灾前结束日期 YYYYMMDD") - post_start: Optional[str] = Field(default=None, description="灾后开始日期 YYYYMMDD") - post_end: Optional[str] = Field(default=None, description="灾后结束日期 YYYYMMDD") - overlap_threshold: float = Field(default=0.3, ge=0.0, le=1.0, description="最小重叠比例") + pre_start: Optional[str] = Field(default=None, description="Pre-flood start date in YYYYMMDD") + pre_end: Optional[str] = Field(default=None, description="Pre-flood end date in YYYYMMDD") + post_start: Optional[str] = Field(default=None, description="Post-flood start date in YYYYMMDD") + post_end: Optional[str] = Field(default=None, description="Post-flood end date in YYYYMMDD") + overlap_threshold: float = Field(default=0.3, ge=0.0, le=1.0, description="Minimum overlap ratio") + + +class FloodDisasterPairSearchRequest(BaseModel): + disaster_name: Optional[str] = Field(default=None, description="Disaster/event display name") + disaster_date: str = Field(..., description="Disaster date in YYYYMMDD") + region_tree_id: Optional[str] = Field(default=None, description="AOI admin region tree id") + aoi_geojson: Optional[dict[str, Any]] = Field(default=None, description="AOI GeoJSON FeatureCollection") + pre_window_days: int = Field(default=30, ge=1, le=365) + post_window_days: int = Field(default=30, ge=1, le=365) + min_aoi_coverage_ratio: float = Field(default=0.2, ge=0.0, le=1.0) + min_pair_overlap_ratio: float = Field(default=0.3, ge=0.0, le=1.0) + max_pairs: int = Field(default=50, ge=1, le=200) + satellites: Optional[list[str]] = None + polarization: Optional[str] = None + imaging_mode: Optional[str] = None + product_level: Optional[str] = None + require_same_polarization: bool = True + require_same_imaging_mode: bool = False class FloodDetectionRequest(BaseModel): - pre_scene_id: int = Field(..., description="灾前 SARSceneGeoORM 主键") - post_scene_id: int = Field(..., description="灾后 SARSceneGeoORM 主键") - refine: bool = Field(default=False, description="是否启用 MRF 精化") + pre_scene_id: int = Field(..., description="Pre-flood SARSceneGeoORM primary key") + post_scene_id: int = Field(..., description="Post-flood SARSceneGeoORM primary key") + refine: bool = Field(default=False, description="Enable MRF refinement") + + +class FloodOverlayRequest(BaseModel): + near_threshold_m: float = Field(default=500.0, ge=0.0, le=10000.0, description="Near-flood threshold in meters") @router.post("/flood/preprocess", status_code=202) @@ -48,8 +68,7 @@ async def submit_flood_preprocess( db: AsyncSession = Depends(get_db), admin_user: AuthUserORM = Depends(_require_admin), ): - """提交水体提取前置处理任务，当前复用旧 water geocode 链路。""" - return await water_compat.submit_geocode(req, db=db, admin_user=admin_user) + return await flood_analysis_service.submit_geocode_job(req, db=db) @router.get("/flood/scenes") @@ -59,8 +78,7 @@ async def list_flood_scenes( db: AsyncSession = Depends(get_db), current_user: AuthUserORM = Depends(_get_current_user), ): - """列出可作为水体提取输入的地理编码场景。""" - return await water_compat.list_scenes(limit=limit, offset=offset, db=db, current_user=current_user) + return await flood_analysis_service.list_scenes(limit=limit, offset=offset, db=db) @router.get("/flood/scenes/done-radar-ids") @@ -68,7 +86,7 @@ async def list_flood_done_radar_ids( db: AsyncSession = Depends(get_db), current_user: AuthUserORM = Depends(_get_current_user), ): - return await water_compat.list_done_scene_radar_ids(db=db, current_user=current_user) + return await flood_analysis_service.list_done_scene_radar_ids(db=db) @router.get("/flood/scenes/active-radar-ids") @@ -76,7 +94,7 @@ async def list_flood_active_radar_ids( db: AsyncSession = Depends(get_db), current_user: AuthUserORM = Depends(_get_current_user), ): - return await water_compat.list_active_scene_radar_ids(db=db, current_user=current_user) + return await flood_analysis_service.list_active_scene_radar_ids(db=db) @router.post("/flood/scenes/{scene_id}/reset") @@ -85,7 +103,7 @@ async def reset_flood_scene( db: AsyncSession = Depends(get_db), admin_user: AuthUserORM = Depends(_require_admin), ): - return await water_compat.reset_scene_status(scene_id=scene_id, db=db, admin_user=admin_user) + return await flood_analysis_service.reset_scene_status(scene_id=scene_id, db=db) @router.post("/flood/water-extractions", status_code=202) @@ -94,8 +112,7 @@ async def submit_flood_water_extraction( db: AsyncSession = Depends(get_db), admin_user: AuthUserORM = Depends(_require_admin), ): - """提交单景水体提取任务，当前复用 Otsu 快速水体检测实现。""" - return await water_compat.submit_water_detect(req, db=db, admin_user=admin_user) + return await flood_analysis_service.submit_water_extraction(req, db=db) @router.get("/flood/water-extractions") @@ -106,12 +123,11 @@ async def list_flood_water_extractions( db: AsyncSession = Depends(get_db), current_user: AuthUserORM = Depends(_get_current_user), ): - return await water_compat.list_water_detections( + return await flood_analysis_service.list_water_extractions( limit=limit, offset=offset, status=status, db=db, - current_user=current_user, ) @@ -121,11 +137,7 @@ async def get_flood_water_extraction_preview( db: AsyncSession = Depends(get_db), current_user: AuthUserORM = Depends(_get_current_user), ): - return await water_compat.get_water_detection_preview( - detection_id=extraction_id, - db=db, - current_user=current_user, - ) + return await flood_analysis_service.get_water_extraction_preview(extraction_id=extraction_id, db=db) @router.post("/flood/pairs/search") @@ -134,7 +146,16 @@ async def search_flood_pairs( db: AsyncSession = Depends(get_db), current_user: AuthUserORM = Depends(_get_current_user), ): - return await water_compat.find_water_pairs(req, db=db, current_user=current_user) + return await flood_analysis_service.search_pairs(req, db=db) + + +@router.post("/flood/disaster-pairs/search") +async def search_flood_disaster_pairs( + req: FloodDisasterPairSearchRequest, + db: AsyncSession = Depends(get_db), + current_user: AuthUserORM = Depends(_get_current_user), +): + return await flood_analysis_service.search_disaster_pairs(req, db=db) @router.post("/flood/detections", status_code=202) @@ -143,7 +164,7 @@ async def submit_flood_detection( db: AsyncSession = Depends(get_db), admin_user: AuthUserORM = Depends(_require_admin), ): - return await water_compat.submit_flood_detect(req, db=db, admin_user=admin_user) + return await flood_analysis_service.submit_flood_detection(req, db=db) @router.get("/flood/detections") @@ -151,7 +172,7 @@ async def list_flood_detections( db: AsyncSession = Depends(get_db), current_user: AuthUserORM = Depends(_get_current_user), ): - return await water_compat.list_flood_events(db=db, current_user=current_user) + return await flood_analysis_service.list_flood_detections(db=db) @router.get("/flood/detections/{detection_id}/preview/{layer}") @@ -161,23 +182,109 @@ async def get_flood_detection_preview( db: AsyncSession = Depends(get_db), current_user: AuthUserORM = Depends(_get_current_user), ): - normalized = layer.strip().lower() - if normalized == "pre": - return await water_compat.flood_event_pre_preview( - event_id=detection_id, - db=db, - current_user=current_user, - ) - if normalized == "post": - return await water_compat.flood_event_post_preview( - event_id=detection_id, - db=db, - current_user=current_user, - ) - if normalized == "classified": - return await water_compat.flood_event_classified_preview( - event_id=detection_id, - db=db, - current_user=current_user, - ) - raise HTTPException(status_code=404, detail=f"不支持的洪涝预览图层: {layer}") + return await flood_analysis_service.get_flood_detection_preview( + detection_id=detection_id, + layer=layer, + db=db, + ) + + +@router.post("/flood/detections/{detection_id}/overlay", status_code=201) +async def run_flood_overlay( + detection_id: int, + req: FloodOverlayRequest | None = None, + db: AsyncSession = Depends(get_db), + admin_user: AuthUserORM = Depends(_require_admin), +): + threshold = req.near_threshold_m if req else 500.0 + return await flood_overlay_service.run_overlay( + detection_id=detection_id, + db=db, + near_threshold_m=threshold, + ) + + +@router.get("/flood/detections/{detection_id}/impact") +async def get_flood_impact( + detection_id: int, + db: AsyncSession = Depends(get_db), + current_user: AuthUserORM = Depends(_get_current_user), +): + return await flood_overlay_service.get_overlay_result(detection_id=detection_id, db=db) + + +@router.post("/flood/detections/{detection_id}/products", status_code=201) +async def create_flood_product( + detection_id: int, + db: AsyncSession = Depends(get_db), + admin_user: AuthUserORM = Depends(_require_admin), +): + return await flood_product_service.create_flood_product_for_detection(detection_id=detection_id, db=db) + + +@router.get("/flood/products") +async def list_flood_products( + limit: int = 20, + offset: int = 0, + status: Optional[str] = None, + db: AsyncSession = Depends(get_db), + current_user: AuthUserORM = Depends(_get_current_user), +): + return await flood_product_service.list_flood_products( + db=db, + limit=limit, + offset=offset, + status=status, + ) + + +@router.get("/flood/products/{product_id}") +async def get_flood_product( + product_id: str, + db: AsyncSession = Depends(get_db), + current_user: AuthUserORM = Depends(_get_current_user), +): + return await flood_product_service.get_flood_product(product_id_or_pk=product_id, db=db) + + +@router.get("/flood/products/{product_id}/manifest") +async def get_flood_product_manifest( + product_id: str, + db: AsyncSession = Depends(get_db), + current_user: AuthUserORM = Depends(_get_current_user), +): + return await flood_product_service.get_flood_product_manifest(product_id_or_pk=product_id, db=db) + + +@router.get("/flood/results") +async def list_flood_results( + limit: int = 20, + offset: int = 0, + status: Optional[str] = None, + db: AsyncSession = Depends(get_db), + current_user: AuthUserORM = Depends(_get_current_user), +): + return await flood_product_service.list_flood_products( + db=db, + limit=limit, + offset=offset, + status=status, + ) + + +@router.get("/flood/results/{product_id}") +async def get_flood_result( + product_id: str, + db: AsyncSession = Depends(get_db), + current_user: AuthUserORM = Depends(_get_current_user), +): + return await flood_product_service.get_flood_product(product_id_or_pk=product_id, db=db) + + +@router.get("/flood/results/{product_id}/manifest") +async def get_flood_result_manifest( + product_id: str, + db: AsyncSession = Depends(get_db), + current_user: AuthUserORM = Depends(_get_current_user), +): + return await flood_product_service.get_flood_product_manifest(product_id_or_pk=product_id, db=db) diff --git a/backend/app/routers/monitor.py b/backend/app/routers/monitor.py index fa92477..eca5cf3 100644 --- a/backend/app/routers/monitor.py +++ b/backend/app/routers/monitor.py @@ -3,11 +3,11 @@ from __future__ import annotations from typing import List, Optional from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException -from pydantic import BaseModel +from pydantic import BaseModel, Field from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select -from ..config import read_int_env +from ..config import read_int_env, settings, split_env_paths from ..database import get_db from ..models import AuthUserORM, SystemTaskORM, TaskLogORM from ..scheduler import MONITOR_CONFIG @@ -41,11 +41,23 @@ class MonitorConfig(BaseModel): radar_dirs: List[str] = [] orbit_dir: Optional[str] = None dinsar_dirs: List[str] = [] + gf3_archive_source_dirs: List[str] = [] gf3_source_dirs: List[str] = [] gf3_storage_dirs: List[str] = [] # Manual-only: config is read from .env +class GF3UnpackConfig(BaseModel): + source_dirs: List[str] = [] + target_dirs: List[str] = [] + archive_exts: List[str] = [] + delete_archive: bool = False + + +class GF3UnpackRunRequest(BaseModel): + max_files_per_run: Optional[int] = Field(default=None, ge=0) + + @router.post("/monitor/config") async def update_monitor_config(config: MonitorConfig): """ @@ -142,6 +154,56 @@ async def run_gf3_batch_process(admin_user: AuthUserORM = Depends(_require_admin raise HTTPException(status_code=409, detail=str(e)) +@router.get("/monitor/gf3-unpack/config") +async def get_gf3_unpack_config(admin_user: AuthUserORM = Depends(_require_admin)): + return GF3UnpackConfig( + source_dirs=MONITOR_CONFIG.get("gf3_archive_source_dirs") or [], + target_dirs=MONITOR_CONFIG.get("gf3_source_dirs") or [], + archive_exts=split_env_paths(settings.GF3_ARCHIVE_EXTS), + delete_archive=bool(settings.GF3_UNPACK_DELETE_ARCHIVE), + ) + + +@router.post("/monitor/gf3-unpack", status_code=202) +async def run_gf3_unpack( + request_data: GF3UnpackRunRequest | None = None, + admin_user: AuthUserORM = Depends(_require_admin), +): + """ + 将 GF3 压缩包池解包到 GF3_SOURCE_DIRS，作为后续 L1A→L2 预处理输入。 + """ + gf3_archive_source_dirs = MONITOR_CONFIG.get("gf3_archive_source_dirs") or [] + gf3_source_dirs = MONITOR_CONFIG.get("gf3_source_dirs") or [] + if not gf3_archive_source_dirs: + raise HTTPException(status_code=400, detail="GF3_ARCHIVE_SOURCE_DIRS is not configured.") + if not gf3_source_dirs: + raise HTTPException(status_code=400, detail="GF3_SOURCE_DIRS is not configured.") + + max_files = None + if request_data is not None and request_data.max_files_per_run is not None: + max_files = max(0, int(request_data.max_files_per_run)) + + task_type = "GF3_UNPACK" + task_name = "GF3 压缩包解包" + payload = { + "source_dirs": gf3_archive_source_dirs, + "target_dirs": gf3_source_dirs, + "archive_exts": split_env_paths(settings.GF3_ARCHIVE_EXTS), + } + if max_files is not None: + payload["max_files_per_run"] = max_files + + try: + task_id = await task_service.create_task(task_type, task_name, params=payload) + await job_queue_service.create_job(task_type, payload=payload, task_id=task_id) + return { + "message": "GF3 解包任务已提交", + "task_id": task_id, + } + except ValueError as e: + raise HTTPException(status_code=409, detail=str(e)) + + @router.get("/monitor/status") async def get_monitor_status(): """ diff --git a/backend/app/routers/sbas_insar_production.py b/backend/app/routers/sbas_insar_production.py new file mode 100644 index 0000000..824c400 --- /dev/null +++ b/backend/app/routers/sbas_insar_production.py @@ -0,0 +1,323 @@ +from __future__ import annotations + +import asyncio +import mimetypes +import subprocess + +from fastapi import APIRouter, HTTPException +from fastapi.responses import FileResponse +from pydantic import BaseModel, Field, field_validator + +from ..services.job_queue_service import job_queue_service +from ..services.sbas_insar_production_service import sbas_insar_production_service +from ..services.task_service import task_service + + +router = APIRouter(prefix="/sbas-insar-production", tags=["sbas-insar-production"]) + + +class SbasStackDiscoverRequest(BaseModel): + source_roots: list[str] | None = None + orbit_roots: list[str] | None = None + min_scenes: int = Field(default=3, ge=2, le=100) + require_orbits: bool = True + include_scenes: bool = False + limit: int = Field(default=30, ge=0, le=500) + platform: str | None = Field(default=None, max_length=16) + relative_orbit: str | None = Field(default=None, max_length=32) + orbit_direction: str | None = Field(default=None, max_length=32) + + @field_validator("source_roots", "orbit_roots", mode="before") + @classmethod + def _normalize_roots(cls, value): + if value is None: + return None + if isinstance(value, str): + items = [value] + else: + items = list(value) + cleaned = [str(item or "").strip() for item in items if str(item or "").strip()] + return cleaned or None + + @field_validator("platform", "relative_orbit", "orbit_direction", mode="before") + @classmethod + def _normalize_optional_text(cls, value): + if value is None: + return None + text = str(value).strip() + return text or None + + +class SbasMonitorPoint(BaseModel): + point_id: str | None = Field(default=None, max_length=64) + label: str | None = Field(default=None, max_length=120) + lon: float = Field(ge=-180, le=180) + lat: float = Field(ge=-90, le=90) + + +class SbasRunSubmitRequest(SbasStackDiscoverRequest): + run_label: str | None = Field(default=None, max_length=120) + dry_run: bool = True + monitor_point_strategy: str = Field(default="auto_low_sigma_high_rate", max_length=64) + monitor_points: list[SbasMonitorPoint] | None = None + + +class SbasBaselineAuditRequest(BaseModel): + execute: bool = True + rlks: int = Field(default=8, ge=1, le=64) + azlks: int = Field(default=8, ge=1, le=64) + max_delta_n: int = Field(default=1, ge=1, le=100) + timeout_seconds: int = Field(default=21600, ge=60, le=86400) + + +class SbasItabDecisionRequest(BaseModel): + decision: str = Field(pattern="^(approve|reject)$") + reviewer: str | None = Field(default=None, max_length=120) + note: str | None = Field(default=None, max_length=1000) + + +class SbasCoregistrationRequest(BaseModel): + execute: bool = False + rlks: int = Field(default=8, ge=1, le=64) + azlks: int = Field(default=8, ge=1, le=64) + + +class SbasCoregistrationJobRequest(BaseModel): + rlks: int = Field(default=8, ge=1, le=64) + azlks: int = Field(default=8, ge=1, le=64) + timeout_seconds: int = Field(default=43200, ge=60, le=172800) + + +@router.get("/capabilities") +async def get_sbas_insar_capabilities(): + return sbas_insar_production_service.get_capabilities() + + +@router.post("/stacks/discover") +async def discover_sbas_insar_stacks(request: SbasStackDiscoverRequest): + try: + return await asyncio.to_thread( + sbas_insar_production_service.discover_stacks, + source_roots=request.source_roots, + orbit_roots=request.orbit_roots, + min_scenes=request.min_scenes, + require_orbits=request.require_orbits, + include_scenes=request.include_scenes, + limit=request.limit, + platform=request.platform, + relative_orbit=request.relative_orbit, + orbit_direction=request.orbit_direction, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.post("/stacks/{stack_id}/audit") +async def audit_sbas_insar_stack(stack_id: str, request: SbasStackDiscoverRequest): + try: + return await asyncio.to_thread( + sbas_insar_production_service.audit_stack, + stack_id, + source_roots=request.source_roots, + orbit_roots=request.orbit_roots, + min_scenes=request.min_scenes, + require_orbits=request.require_orbits, + ) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.post("/stacks/{stack_id}/runs", status_code=202) +async def submit_sbas_insar_run(stack_id: str, request: SbasRunSubmitRequest): + try: + return await asyncio.to_thread( + sbas_insar_production_service.create_run, + stack_id, + run_label=request.run_label, + source_roots=request.source_roots, + orbit_roots=request.orbit_roots, + min_scenes=request.min_scenes, + require_orbits=request.require_orbits, + monitor_points=[ + point.model_dump(exclude_none=True) + for point in (request.monitor_points or []) + ], + monitor_point_strategy=request.monitor_point_strategy, + dry_run=request.dry_run, + ) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.get("/runs") +async def list_sbas_insar_runs(): + return await asyncio.to_thread(sbas_insar_production_service.list_runs) + + +@router.get("/runs/{run_id}") +async def get_sbas_insar_run(run_id: str): + try: + return await asyncio.to_thread(sbas_insar_production_service.get_run_detail, run_id) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.post("/runs/{run_id}/baseline-audit", status_code=202) +async def run_sbas_insar_baseline_audit(run_id: str, request: SbasBaselineAuditRequest): + try: + return await asyncio.to_thread( + sbas_insar_production_service.run_baseline_audit, + run_id, + execute=request.execute, + rlks=request.rlks, + azlks=request.azlks, + max_delta_n=request.max_delta_n, + timeout_seconds=request.timeout_seconds, + ) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except subprocess.TimeoutExpired as exc: + raise HTTPException(status_code=504, detail=f"baseline audit timed out after {exc.timeout}s") from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.post("/runs/{run_id}/itab-decision") +async def decide_sbas_insar_itab(run_id: str, request: SbasItabDecisionRequest): + try: + return await asyncio.to_thread( + sbas_insar_production_service.decide_itab, + run_id, + decision=request.decision, + reviewer=request.reviewer, + note=request.note, + ) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.post("/runs/{run_id}/coregistration", status_code=202) +async def prepare_sbas_insar_coregistration(run_id: str, request: SbasCoregistrationRequest): + try: + return await asyncio.to_thread( + sbas_insar_production_service.prepare_coregistration, + run_id, + execute=request.execute, + rlks=request.rlks, + azlks=request.azlks, + ) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.post("/runs/{run_id}/coregistration/jobs", status_code=202) +async def submit_sbas_insar_coregistration_job(run_id: str, request: SbasCoregistrationJobRequest): + try: + run_detail = await asyncio.to_thread(sbas_insar_production_service.get_run_detail, run_id) + status = str((run_detail.get("run") or {}).get("status") or "").strip() + if status in {"ITAB_APPROVED", "COREGISTRATION_FAILED"}: + await asyncio.to_thread( + sbas_insar_production_service.prepare_coregistration, + run_id, + execute=False, + rlks=request.rlks, + azlks=request.azlks, + ) + run_detail = await asyncio.to_thread(sbas_insar_production_service.get_run_detail, run_id) + status = str((run_detail.get("run") or {}).get("status") or "").strip() + if status not in {"COREGISTRATION_SCRIPT_READY", "COREGISTRATION_RUNNING"}: + raise ValueError(f"run status does not allow coregistration job submission: {status}") + if status == "COREGISTRATION_RUNNING": + raise ValueError("coregistration is already running for this run") + + from ..services.job_handlers import JOB_TYPE_SBAS_COREGISTRATION + + payload = { + "run_id": run_id, + "rlks": request.rlks, + "azlks": request.azlks, + "timeout_seconds": request.timeout_seconds, + } + task_id = await task_service.create_task( + task_type=JOB_TYPE_SBAS_COREGISTRATION, + task_name=f"SBAS-InSAR 共参考配准: {run_id}", + params=payload, + ) + job_id = await job_queue_service.create_job( + job_type=JOB_TYPE_SBAS_COREGISTRATION, + payload=payload, + task_id=task_id, + max_attempts=1, + ) + return { + "message": "SBAS-InSAR coregistration job queued.", + "run_id": run_id, + "task_id": task_id, + "job_id": job_id, + "job_type": JOB_TYPE_SBAS_COREGISTRATION, + } + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + message = str(exc) + status_code = 409 if "冲突" in message or "conflict" in message.lower() else 400 + raise HTTPException(status_code=status_code, detail=message) from exc + + +@router.get("/runs/{run_id}/artifacts/{relative_path:path}") +async def get_sbas_insar_run_artifact(run_id: str, relative_path: str): + try: + artifact_path = sbas_insar_production_service.resolve_run_artifact_path(run_id, relative_path) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + media_type = mimetypes.guess_type(str(artifact_path))[0] or "application/octet-stream" + return FileResponse( + artifact_path, + media_type=media_type, + filename=artifact_path.name, + ) + + +@router.get("/trial-runs") +async def list_sbas_insar_trial_runs(): + return await asyncio.to_thread(sbas_insar_production_service.list_trial_runs) + + +@router.get("/trial-runs/{trial_id}") +async def get_sbas_insar_trial_run(trial_id: str): + try: + return await asyncio.to_thread(sbas_insar_production_service.get_trial_detail, trial_id) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.get("/trial-runs/{trial_id}/artifacts/{relative_path:path}") +async def get_sbas_insar_artifact(trial_id: str, relative_path: str): + try: + artifact_path = sbas_insar_production_service.resolve_artifact_path(trial_id, relative_path) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + media_type = mimetypes.guess_type(str(artifact_path))[0] or "application/octet-stream" + return FileResponse( + artifact_path, + media_type=media_type, + filename=artifact_path.name, + ) diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index dd0e7ae..f09ba52 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -19,6 +19,7 @@ MONITOR_CONFIG = { "s1_storage_dirs": split_env_paths(settings.SENTINEL1_STORAGE_DIRS), "s1_orbit_dirs": split_env_paths(settings.ORBIT_SOURCE_DIRS), # GF3 链路 + "gf3_archive_source_dirs": split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS), "gf3_source_dirs": split_env_paths(settings.GF3_SOURCE_DIRS), "gf3_storage_dirs": split_env_paths(settings.GF3_STORAGE_DIRS), "mode": "manual", diff --git a/backend/app/services/flood_analysis_service.py b/backend/app/services/flood_analysis_service.py new file mode 100644 index 0000000..99f1291 --- /dev/null +++ b/backend/app/services/flood_analysis_service.py @@ -0,0 +1,967 @@ +"""Flood-analysis service layer. + +This module owns the flood business API implementation used by +``backend.app.routers.flood``. It intentionally does not import the legacy +water router; the old router remains only as a compatibility surface. +""" +from __future__ import annotations + +import asyncio +import base64 +import io +import json +import os +from datetime import datetime, timedelta +from typing import Any + +from fastapi import HTTPException +from fastapi.responses import JSONResponse +from sqlalchemy import select, func +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from ..models import FloodDetectionORM, RadarDataORM, SARSceneGeoORM, WaterDetectionORM, WaterExtractionORM +from ..services.job_handlers import ( + JOB_TYPE_FLOOD_DETECTION, + JOB_TYPE_SAR_SCENE_PREPROCESS, + JOB_TYPE_WATER_DETECT, +) +from ..services.job_queue_service import job_queue_service +from ..services.task_service import task_service +from ..utils import normalize_satellite_family + +_FLOOD_JOB_MAX_ATTEMPTS = 3 + + +async def _queue_flood_job( + *, + job_type: str, + task_type: str, + task_name: str, + payload: dict[str, Any], +) -> dict[str, Any]: + try: + task_id = await task_service.create_task( + task_type=task_type, + task_name=task_name, + params=payload, + ) + job_id = await job_queue_service.create_job( + job_type=job_type, + payload=payload, + task_id=task_id, + max_attempts=_FLOOD_JOB_MAX_ATTEMPTS, + ) + return {"task_id": task_id, "job_id": job_id, "job_type": job_type, "message": "Job queued."} + except ValueError as exc: + message = str(exc) + raise HTTPException(status_code=409 if "conflict" in message.lower() else 400, detail=message) from exc + + +def _overlap_ratio(poly_a: Any, poly_b: Any) -> float: + try: + import json + from shapely.geometry import Polygon, shape + + def _to_geom(poly: Any): + if isinstance(poly, str): + poly = json.loads(poly) + if isinstance(poly, list): + return Polygon(poly) + return shape(poly) + + a = _to_geom(poly_a) + b = _to_geom(poly_b) + if not a.is_valid or not b.is_valid: + return 0.0 + intersection_area = a.intersection(b).area + smaller_area = min(a.area, b.area) + return intersection_area / smaller_area if smaller_area > 0 else 0.0 + except Exception: + return 0.0 + + +def _parse_ymd(value: str | None, *, field: str) -> datetime: + try: + normalized = str(value or "").replace("-", "").strip() + return datetime.strptime(normalized, "%Y%m%d") + except Exception as exc: + raise HTTPException(status_code=400, detail=f"{field} must be YYYYMMDD") from exc + + +def _format_ymd(value: datetime) -> str: + return value.strftime("%Y%m%d") + + +def _to_float(value: Any, default: float = 0.0) -> float: + try: + if value is None: + return default + return float(value) + except Exception: + return default + + +def _same_text(left: Any, right: Any) -> bool: + left_text = str(left or "").strip().lower() + right_text = str(right or "").strip().lower() + if not left_text or not right_text: + return True + return left_text == right_text + + +def _feature_collection_name(feature_collection: dict[str, Any]) -> str | None: + try: + features = feature_collection.get("features") or [] + properties = features[0].get("properties") or {} + return properties.get("name") or properties.get("NAME") or properties.get("treeID") + except Exception: + return None + + +def _preprocess_engine_for_radar(radar: RadarDataORM) -> str | None: + family = str(normalize_satellite_family(radar.satellite_family or radar.satellite) or "").upper() + if family == "GF3": + return "gf3_gdal" + if family == "LT1": + return "lt_gamma" + return None + + +def _scene_analysis_path(scene: SARSceneGeoORM | None) -> str | None: + if not scene: + return None + return scene.analysis_tif_path + + +def _resolve_aoi_wkt_from_request(req: Any) -> tuple[str, dict[str, Any], dict[str, Any]]: + """Resolve region/GeoJSON AOI using the same parser as the management search page.""" + aoi_geojson = getattr(req, "aoi_geojson", None) + region_tree_id = getattr(req, "region_tree_id", None) + + if aoi_geojson: + feature_collection = aoi_geojson + source = "geojson" + elif region_tree_id: + from ..routers.dependencies import _resolve_region_aoi_payload + + payload = _resolve_region_aoi_payload(str(region_tree_id)) + feature_collection = payload.get("aoi_geojson") or payload + source = "region" + else: + raise HTTPException(status_code=400, detail="region_tree_id or aoi_geojson is required") + + from ..routers.dependencies import _parse_aoi_geojson_form_value + + parsed = _parse_aoi_geojson_form_value(json.dumps(feature_collection, ensure_ascii=False)) + if not parsed: + raise HTTPException(status_code=400, detail="AOI geometry is empty") + aoi_wkt, normalized_feature_collection = parsed + meta = { + "source": source, + "region_tree_id": region_tree_id, + "name": _feature_collection_name(normalized_feature_collection), + } + return aoi_wkt, normalized_feature_collection, meta + + +def _radar_scene_item(scene: SARSceneGeoORM, radar: RadarDataORM, *, aoi_coverage_ratio: float | None = None) -> dict[str, Any]: + return { + "id": scene.id, + "scene_id": scene.id, + "radar_data_id": scene.radar_data_id, + "satellite": radar.satellite, + "imaging_date": radar.imaging_date, + "acquisition_time_utc": radar.acquisition_time_utc, + "imaging_mode": radar.imaging_mode, + "product_level": radar.product_level, + "polarization": radar.polarization, + "orbit_direction": radar.orbit_direction, + "geo_path": scene.geo_path, + "analysis_tif_path": scene.analysis_tif_path, + "analysis_dir": scene.analysis_dir, + "analysis_preview_path": scene.analysis_preview_path, + "analysis_engine": scene.analysis_engine, + "analysis_profile": scene.analysis_profile, + "analysis_backscatter_unit": scene.analysis_backscatter_unit, + "analysis_quality_json": scene.analysis_quality_json, + "coverage_polygon": radar.coverage_polygon, + "min_lat": radar.min_lat, + "max_lat": radar.max_lat, + "min_lon": radar.min_lon, + "max_lon": radar.max_lon, + "aoi_coverage_ratio": aoi_coverage_ratio, + } + + +async def _query_disaster_scene_pool( + *, + db: AsyncSession, + req: Any, + aoi_wkt: str, + start_ymd: str, + end_ymd: str, + min_aoi_coverage_ratio: float, + descending: bool, +) -> list[dict[str, Any]]: + aoi_geom = func.ST_GeomFromText(aoi_wkt, 4326) + aoi_area = func.ST_Area(func.Geography(aoi_geom)) + coverage_expr = ( + func.ST_Area(func.Geography(func.ST_Intersection(RadarDataORM.geom, aoi_geom))) + / func.nullif(aoi_area, 0) + ).label("aoi_coverage_ratio") + + filters = [ + SARSceneGeoORM.status == "DONE", + SARSceneGeoORM.analysis_tif_path.isnot(None), + RadarDataORM.geom.isnot(None), + RadarDataORM.imaging_date.isnot(None), + RadarDataORM.imaging_date >= start_ymd, + RadarDataORM.imaging_date <= end_ymd, + func.ST_Intersects(RadarDataORM.geom, aoi_geom), + ] + + satellites = [str(item).strip() for item in (getattr(req, "satellites", None) or []) if str(item).strip()] + if satellites: + filters.append(RadarDataORM.satellite.in_(satellites)) + + polarization = str(getattr(req, "polarization", "") or "").strip() + if polarization: + filters.append(RadarDataORM.polarization.ilike(f"%{polarization}%")) + + imaging_mode = str(getattr(req, "imaging_mode", "") or "").strip() + if imaging_mode: + filters.append(RadarDataORM.imaging_mode == imaging_mode) + + product_level = str(getattr(req, "product_level", "") or "").strip() + if product_level: + filters.append(RadarDataORM.product_level == product_level) + + order_by = RadarDataORM.imaging_date.desc() if descending else RadarDataORM.imaging_date.asc() + result = await db.execute( + select(SARSceneGeoORM, RadarDataORM, coverage_expr) + .join(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id) + .where(*filters) + .order_by(order_by, SARSceneGeoORM.id.desc()) + ) + + pool: list[dict[str, Any]] = [] + for scene, radar, aoi_coverage_ratio in result.all(): + coverage_ratio = max(0.0, min(1.0, _to_float(aoi_coverage_ratio))) + if coverage_ratio < min_aoi_coverage_ratio: + continue + pool.append(_radar_scene_item(scene, radar, aoi_coverage_ratio=round(coverage_ratio, 4))) + return pool + + +async def submit_geocode_job(req: Any, db: AsyncSession) -> dict[str, Any]: + radar = await db.get(RadarDataORM, req.radar_data_id) + if not radar: + raise HTTPException(status_code=404, detail=f"RadarData id={req.radar_data_id} not found") + + result = await db.execute( + select(SARSceneGeoORM) + .where(SARSceneGeoORM.radar_data_id == req.radar_data_id) + .with_for_update(skip_locked=True) + ) + scene = result.scalar_one_or_none() + if scene and scene.status in ("PENDING", "RUNNING"): + raise HTTPException(status_code=409, detail="Scene already has an active geocode job") + if not scene: + scene = SARSceneGeoORM(radar_data_id=req.radar_data_id, status="PENDING") + db.add(scene) + await db.flush() + else: + scene.status = "PENDING" + scene.error_msg = None + await db.flush() + scene_id = scene.id + await db.commit() + + engine = _preprocess_engine_for_radar(radar) + if not engine: + async with db.begin(): + failed_scene = await db.get(SARSceneGeoORM, scene_id) + if failed_scene and failed_scene.status == "PENDING": + failed_scene.status = "FAILED" + failed_scene.error_msg = "No analysis-ready GeoTIFF preprocessor configured for this satellite" + raise HTTPException( + status_code=400, + detail="洪涝模块不再使用 ENVI 兜底预处理；该卫星暂未配置 analysis-ready GeoTIFF 预处理器", + ) + + job_type = JOB_TYPE_SAR_SCENE_PREPROCESS + task_type = f"FLOOD_SCENE_PREPROCESS_{scene_id}" + task_name = f"Flood analysis-ready preprocess radar_id={req.radar_data_id} engine={engine}" + payload = {"scene_id": scene_id, "radar_data_id": req.radar_data_id} + payload["engine"] = engine + + try: + return await _queue_flood_job( + job_type=job_type, + task_type=task_type, + task_name=task_name, + payload=payload, + ) + except HTTPException: + async with db.begin(): + failed_scene = await db.get(SARSceneGeoORM, scene_id) + if failed_scene and failed_scene.status == "PENDING": + failed_scene.status = "FAILED" + failed_scene.error_msg = "Job queue failed" + raise + + +async def reset_scene_status(scene_id: int, db: AsyncSession) -> dict[str, Any]: + scene = await db.get(SARSceneGeoORM, scene_id) + if not scene: + raise HTTPException(status_code=404, detail=f"Scene id={scene_id} not found") + if scene.status not in ("PENDING", "RUNNING"): + raise HTTPException(status_code=400, detail=f"Scene status is {scene.status}; reset is not needed") + scene.status = "FAILED" + scene.error_msg = "Manually reset" + await db.commit() + return {"id": scene_id, "status": "FAILED", "message": "Scene reset"} + + +async def list_done_scene_radar_ids(db: AsyncSession) -> dict[str, list[int]]: + result = await db.execute( + select(SARSceneGeoORM.radar_data_id).where( + SARSceneGeoORM.status == "DONE", + SARSceneGeoORM.analysis_tif_path.isnot(None), + ) + ) + return {"ids": [row for (row,) in result.all()]} + + +async def list_active_scene_radar_ids(db: AsyncSession) -> dict[str, list[int]]: + result = await db.execute( + select(SARSceneGeoORM.radar_data_id).where(SARSceneGeoORM.status.in_(["PENDING", "RUNNING"])) + ) + return {"ids": [row for (row,) in result.all()]} + + +async def list_scenes(limit: int, offset: int, db: AsyncSession) -> dict[str, Any]: + total_result = await db.execute(select(func.count()).select_from(SARSceneGeoORM)) + total = total_result.scalar_one() + + result = await db.execute( + select(SARSceneGeoORM, RadarDataORM) + .join(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id) + .order_by(SARSceneGeoORM.id.desc()) + .limit(limit) + .offset(offset) + ) + + items = [] + for scene, radar in result.all(): + items.append( + { + "id": scene.id, + "radar_data_id": scene.radar_data_id, + "satellite": radar.satellite, + "imaging_date": radar.imaging_date, + "acquisition_time_utc": radar.acquisition_time_utc, + "imaging_mode": radar.imaging_mode, + "product_level": radar.product_level, + "polarization": radar.polarization, + "orbit_direction": radar.orbit_direction, + "geo_path": scene.geo_path, + "analysis_tif_path": scene.analysis_tif_path, + "analysis_dir": scene.analysis_dir, + "analysis_preview_path": scene.analysis_preview_path, + "analysis_engine": scene.analysis_engine, + "analysis_profile": scene.analysis_profile, + "analysis_backscatter_unit": scene.analysis_backscatter_unit, + "analysis_nodata_value": scene.analysis_nodata_value, + "analysis_metadata_json": scene.analysis_metadata_json, + "analysis_quality_json": scene.analysis_quality_json, + "pixel_size_m": scene.pixel_size_m, + "status": scene.status, + "error_msg": scene.error_msg, + "created_at": scene.created_at.isoformat() if scene.created_at else None, + "coverage_polygon": radar.coverage_polygon, + "min_lat": radar.min_lat, + "max_lat": radar.max_lat, + "min_lon": radar.min_lon, + "max_lon": radar.max_lon, + } + ) + return {"items": items, "total": total} + + +async def submit_water_extraction(req: Any, db: AsyncSession) -> dict[str, Any]: + input_path = req.input_path + scene_id = req.scene_id + + if scene_id: + scene = await db.get(SARSceneGeoORM, scene_id) + if not scene: + raise HTTPException(status_code=404, detail=f"SARSceneGeoORM id={scene_id} not found") + input_path = _scene_analysis_path(scene) + if not input_path: + raise HTTPException(status_code=400, detail="Scene has no analysis-ready GeoTIFF") + + if not input_path: + raise HTTPException(status_code=400, detail="scene_id or input_path is required") + + extraction = WaterExtractionORM( + scene_id=scene_id, + processor=getattr(req, "processor", None) or "otsu", + input_path=input_path, + status="PENDING", + ) + db.add(extraction) + await db.flush() + extraction_id = extraction.id + await db.commit() + + try: + queued = await _queue_flood_job( + job_type=JOB_TYPE_WATER_DETECT, + task_type=f"FLOOD_WATER_EXTRACTION_{extraction_id}", + task_name=f"Flood water extraction id={extraction_id}", + payload={"extraction_id": extraction_id, "processor": extraction.processor}, + ) + async with db.begin(): + queued_extraction = await db.get(WaterExtractionORM, extraction_id) + if queued_extraction: + queued_extraction.task_id = queued.get("task_id") + return queued + except HTTPException: + async with db.begin(): + failed_extraction = await db.get(WaterExtractionORM, extraction_id) + if failed_extraction and failed_extraction.status == "PENDING": + failed_extraction.status = "FAILED" + failed_extraction.error_msg = "Job queue failed" + raise + + +async def list_water_extractions( + *, + limit: int, + offset: int, + status: str | None, + db: AsyncSession, +) -> dict[str, Any]: + count_query = select(func.count()).select_from(WaterExtractionORM) + if status: + count_query = count_query.where(WaterExtractionORM.status == status) + total = (await db.execute(count_query)).scalar_one() + + query = ( + select(WaterExtractionORM, SARSceneGeoORM, RadarDataORM) + .outerjoin(SARSceneGeoORM, WaterExtractionORM.scene_id == SARSceneGeoORM.id) + .outerjoin(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id) + .order_by(WaterExtractionORM.id.desc()) + .limit(limit) + .offset(offset) + ) + if status: + query = query.where(WaterExtractionORM.status == status) + rows = (await db.execute(query)).all() + + items = [] + for detection, scene, radar in rows: + items.append( + { + "id": detection.id, + "scene_id": detection.scene_id, + "processor": detection.processor, + "task_id": detection.task_id, + "radar_data_id": scene.radar_data_id if scene else None, + "satellite": radar.satellite if radar else None, + "imaging_date": radar.imaging_date if radar else None, + "acquisition_time_utc": radar.acquisition_time_utc if radar else None, + "imaging_mode": radar.imaging_mode if radar else None, + "product_level": radar.product_level if radar else None, + "polarization": radar.polarization if radar else None, + "orbit_direction": radar.orbit_direction if radar else None, + "coverage_polygon": radar.coverage_polygon if radar else None, + "min_lat": radar.min_lat if radar else None, + "max_lat": radar.max_lat if radar else None, + "min_lon": radar.min_lon if radar else None, + "max_lon": radar.max_lon if radar else None, + "input_path": detection.input_path, + "output_path": detection.output_path, + "preview_path": detection.preview_path, + "vector_path": detection.vector_path, + "water_area_km2": detection.water_area_km2, + "water_pixel_count": detection.water_pixel_count, + "otsu_threshold_db": detection.threshold_value, + "threshold_value": detection.threshold_value, + "metadata_json": detection.metadata_json, + "status": detection.status, + "error_msg": detection.error_msg, + "created_at": detection.created_at.isoformat() if detection.created_at else None, + "updated_at": detection.updated_at.isoformat() if detection.updated_at else None, + } + ) + return {"items": items, "total": total} + + +async def submit_flood_detection(req: Any, db: AsyncSession) -> dict[str, Any]: + pre_scene = await db.get(SARSceneGeoORM, req.pre_scene_id) + post_scene = await db.get(SARSceneGeoORM, req.post_scene_id) + if not pre_scene: + raise HTTPException(status_code=404, detail=f"Pre-scene id={req.pre_scene_id} not found") + if not post_scene: + raise HTTPException(status_code=404, detail=f"Post-scene id={req.post_scene_id} not found") + if pre_scene.status != "DONE": + raise HTTPException(status_code=400, detail=f"Pre-scene is not DONE: {pre_scene.status}") + if post_scene.status != "DONE": + raise HTTPException(status_code=400, detail=f"Post-scene is not DONE: {post_scene.status}") + if not pre_scene.analysis_tif_path: + raise HTTPException(status_code=400, detail="Pre-scene has no analysis-ready GeoTIFF") + if not post_scene.analysis_tif_path: + raise HTTPException(status_code=400, detail="Post-scene has no analysis-ready GeoTIFF") + + result = await db.execute( + select(FloodDetectionORM) + .where( + FloodDetectionORM.pre_scene_id == req.pre_scene_id, + FloodDetectionORM.post_scene_id == req.post_scene_id, + ) + .with_for_update(skip_locked=True) + ) + detection = result.scalar_one_or_none() + if detection and detection.status in ("PENDING", "RUNNING"): + raise HTTPException(status_code=409, detail="Pair already has an active flood-detection job") + if not detection: + detection = FloodDetectionORM( + pre_scene_id=req.pre_scene_id, + post_scene_id=req.post_scene_id, + status="PENDING", + ) + db.add(detection) + await db.flush() + else: + detection.status = "PENDING" + detection.error_msg = None + await db.flush() + detection_id = detection.id + await db.commit() + + try: + return await _queue_flood_job( + job_type=JOB_TYPE_FLOOD_DETECTION, + task_type=f"FLOOD_DETECTION_{detection_id}", + task_name=f"GeoTIFF flood detection pre={req.pre_scene_id} post={req.post_scene_id}", + payload={"detection_id": detection_id, "refine": req.refine}, + ) + except HTTPException: + async with db.begin(): + failed_detection = await db.get(FloodDetectionORM, detection_id) + if failed_detection and failed_detection.status == "PENDING": + failed_detection.status = "FAILED" + failed_detection.error_msg = "Job queue failed" + raise + + +async def list_flood_detections(db: AsyncSession) -> dict[str, Any]: + result = await db.execute( + select(FloodDetectionORM) + .options( + selectinload(FloodDetectionORM.pre_scene).selectinload(SARSceneGeoORM.radar_data), + selectinload(FloodDetectionORM.post_scene).selectinload(SARSceneGeoORM.radar_data), + ) + .order_by(FloodDetectionORM.id.desc()) + ) + detections = result.scalars().all() + + items = [] + for detection in detections: + pre_radar = detection.pre_scene.radar_data if detection.pre_scene else None + post_radar = detection.post_scene.radar_data if detection.post_scene else None + items.append( + { + "id": detection.id, + "pre_scene_id": detection.pre_scene_id, + "post_scene_id": detection.post_scene_id, + "pre_imaging_date": pre_radar.imaging_date if pre_radar else None, + "post_imaging_date": post_radar.imaging_date if post_radar else None, + "pre_satellite": pre_radar.satellite if pre_radar else None, + "post_satellite": post_radar.satellite if post_radar else None, + "pre_geo_path": _scene_analysis_path(detection.pre_scene), + "post_geo_path": _scene_analysis_path(detection.post_scene), + "pre_analysis_tif_path": _scene_analysis_path(detection.pre_scene), + "post_analysis_tif_path": _scene_analysis_path(detection.post_scene), + "classified_path": detection.classified_path, + "flood_area_km2": detection.flood_area_km2, + "stable_water_area_km2": detection.stable_water_area_km2, + "status": detection.status, + "error_msg": detection.error_msg, + "created_at": detection.created_at.isoformat() if detection.created_at else None, + "updated_at": detection.updated_at.isoformat() if detection.updated_at else None, + } + ) + return {"items": items, "total": len(items)} + + +async def search_pairs(req: Any, db: AsyncSession) -> dict[str, Any]: + pre_filters = [SARSceneGeoORM.status == "DONE", SARSceneGeoORM.analysis_tif_path.isnot(None)] + if req.pre_start: + pre_filters.append(RadarDataORM.imaging_date >= req.pre_start) + if req.pre_end: + pre_filters.append(RadarDataORM.imaging_date <= req.pre_end) + pre_result = await db.execute( + select(SARSceneGeoORM, RadarDataORM) + .join(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id) + .where(*pre_filters) + ) + + post_filters = [SARSceneGeoORM.status == "DONE", SARSceneGeoORM.analysis_tif_path.isnot(None)] + if req.post_start: + post_filters.append(RadarDataORM.imaging_date >= req.post_start) + if req.post_end: + post_filters.append(RadarDataORM.imaging_date <= req.post_end) + post_result = await db.execute( + select(SARSceneGeoORM, RadarDataORM) + .join(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id) + .where(*post_filters) + ) + + candidates = [] + for pre_scene, pre_radar in pre_result.all(): + for post_scene, post_radar in post_result.all(): + if pre_scene.id == post_scene.id: + continue + ratio = 0.0 + if pre_radar.coverage_polygon and post_radar.coverage_polygon: + ratio = _overlap_ratio(pre_radar.coverage_polygon, post_radar.coverage_polygon) + if ratio < req.overlap_threshold: + continue + try: + pre_date = datetime.strptime(pre_radar.imaging_date, "%Y%m%d") + post_date = datetime.strptime(post_radar.imaging_date, "%Y%m%d") + time_diff = abs((post_date - pre_date).days) + except Exception: + time_diff = None + candidates.append( + { + "pre": { + "id": pre_scene.id, + "imaging_date": pre_radar.imaging_date, + "satellite": pre_radar.satellite, + "geo_path": _scene_analysis_path(pre_scene), + "analysis_tif_path": _scene_analysis_path(pre_scene), + }, + "post": { + "id": post_scene.id, + "imaging_date": post_radar.imaging_date, + "satellite": post_radar.satellite, + "geo_path": _scene_analysis_path(post_scene), + "analysis_tif_path": _scene_analysis_path(post_scene), + }, + "overlap_ratio": round(ratio, 4), + "time_diff_days": time_diff, + } + ) + + candidates.sort(key=lambda item: item["overlap_ratio"], reverse=True) + used_pre: set[int] = set() + used_post: set[int] = set() + pairs = [] + for candidate in candidates: + pre_id = candidate["pre"]["id"] + post_id = candidate["post"]["id"] + if pre_id in used_pre or post_id in used_post: + continue + used_pre.add(pre_id) + used_post.add(post_id) + pairs.append(candidate) + + pairs.sort(key=lambda item: item["overlap_ratio"], reverse=True) + return {"pairs": pairs, "total": len(pairs)} + + +async def search_disaster_pairs(req: Any, db: AsyncSession) -> dict[str, Any]: + disaster_date = _parse_ymd(req.disaster_date, field="disaster_date") + pre_window_days = max(1, int(getattr(req, "pre_window_days", 30) or 30)) + post_window_days = max(1, int(getattr(req, "post_window_days", 30) or 30)) + min_aoi_coverage_ratio = max(0.0, min(1.0, float(getattr(req, "min_aoi_coverage_ratio", 0.2) or 0.0))) + min_pair_overlap_ratio = max(0.0, min(1.0, float(getattr(req, "min_pair_overlap_ratio", 0.3) or 0.0))) + max_pairs = max(1, min(200, int(getattr(req, "max_pairs", 50) or 50))) + + aoi_wkt, aoi_geojson, aoi_meta = _resolve_aoi_wkt_from_request(req) + pre_start = disaster_date - timedelta(days=pre_window_days) + pre_end = disaster_date - timedelta(days=1) + post_start = disaster_date + post_end = disaster_date + timedelta(days=post_window_days) + + pre_pool = await _query_disaster_scene_pool( + db=db, + req=req, + aoi_wkt=aoi_wkt, + start_ymd=_format_ymd(pre_start), + end_ymd=_format_ymd(pre_end), + min_aoi_coverage_ratio=min_aoi_coverage_ratio, + descending=True, + ) + post_pool = await _query_disaster_scene_pool( + db=db, + req=req, + aoi_wkt=aoi_wkt, + start_ymd=_format_ymd(post_start), + end_ymd=_format_ymd(post_end), + min_aoi_coverage_ratio=min_aoi_coverage_ratio, + descending=False, + ) + + candidates: list[dict[str, Any]] = [] + require_same_polarization = bool(getattr(req, "require_same_polarization", True)) + require_same_imaging_mode = bool(getattr(req, "require_same_imaging_mode", False)) + total_window = max(1, pre_window_days + post_window_days) + + for pre in pre_pool: + pre_date = _parse_ymd(pre.get("imaging_date"), field="pre.imaging_date") + for post in post_pool: + if pre["id"] == post["id"]: + continue + if require_same_polarization and not _same_text(pre.get("polarization"), post.get("polarization")): + continue + if require_same_imaging_mode and not _same_text(pre.get("imaging_mode"), post.get("imaging_mode")): + continue + + post_date = _parse_ymd(post.get("imaging_date"), field="post.imaging_date") + scene_overlap = _overlap_ratio(pre.get("coverage_polygon"), post.get("coverage_polygon")) + if scene_overlap < min_pair_overlap_ratio: + continue + + pre_delta_days = max(0, (disaster_date - pre_date).days) + post_delta_days = max(0, (post_date - disaster_date).days) + time_score = max(0.0, 1.0 - ((pre_delta_days + post_delta_days) / total_window)) + aoi_score = min(_to_float(pre.get("aoi_coverage_ratio")), _to_float(post.get("aoi_coverage_ratio"))) + score = (scene_overlap * 0.45) + (aoi_score * 0.35) + (time_score * 0.20) + candidates.append( + { + "pre": pre, + "post": post, + "overlap_ratio": round(scene_overlap, 4), + "aoi_coverage_ratio": round(aoi_score, 4), + "time_score": round(time_score, 4), + "score": round(score, 4), + "pre_delta_days": pre_delta_days, + "post_delta_days": post_delta_days, + "time_diff_days": max(0, (post_date - pre_date).days), + "same_polarization": _same_text(pre.get("polarization"), post.get("polarization")), + "same_imaging_mode": _same_text(pre.get("imaging_mode"), post.get("imaging_mode")), + } + ) + + candidates.sort( + key=lambda item: ( + item["score"], + item["overlap_ratio"], + item["aoi_coverage_ratio"], + -item["time_diff_days"], + ), + reverse=True, + ) + selected_pairs = candidates[:max_pairs] + + warnings: list[str] = [] + if not pre_pool: + warnings.append("No pre-disaster DONE scenes match the disaster AOI and time window") + if not post_pool: + warnings.append("No post-disaster DONE scenes match the disaster AOI and time window") + if pre_pool and post_pool and not selected_pairs: + warnings.append("Pre/post scene pools exist, but no pair meets the overlap/polarization constraints") + + return { + "disaster": { + "name": getattr(req, "disaster_name", None), + "date": _format_ymd(disaster_date), + "pre_start": _format_ymd(pre_start), + "pre_end": _format_ymd(pre_end), + "post_start": _format_ymd(post_start), + "post_end": _format_ymd(post_end), + }, + "aoi": { + **aoi_meta, + "geojson": aoi_geojson, + }, + "pre_pool": pre_pool, + "post_pool": post_pool, + "candidate_pairs": selected_pairs, + "pairs": selected_pairs, + "total": len(selected_pairs), + "summary": { + "pre_pool_count": len(pre_pool), + "post_pool_count": len(post_pool), + "candidate_count": len(selected_pairs), + "min_aoi_coverage_ratio": min_aoi_coverage_ratio, + "min_pair_overlap_ratio": min_pair_overlap_ratio, + }, + "warnings": warnings, + } + + +def _open_envi_rasterio(path: str): + import rasterio + + normalized_path = path.replace("\\", "/") + try: + return rasterio.open(normalized_path) + except Exception: + pass + for ext in (".bin", ".img", ".tif", ".tiff"): + try: + return rasterio.open(normalized_path + ext) + except Exception: + pass + raise FileNotFoundError(f"Raster file cannot be opened: {path}") + + +def _raster_to_png_bytes(path: str, colormap: dict[int, tuple[int, int, int, int]]) -> tuple[bytes, list[float]]: + import numpy as np + from PIL import Image + + with _open_envi_rasterio(path) as ds: + data = ds.read(1) + bounds = ds.bounds + geo_bounds = [bounds.bottom, bounds.left, bounds.top, bounds.right] + + rgba = np.zeros((data.shape[0], data.shape[1], 4), dtype=np.uint8) + for value, color in colormap.items(): + rgba[data == value] = color + image = Image.fromarray(rgba, "RGBA") + buffer = io.BytesIO() + image.save(buffer, format="PNG") + return buffer.getvalue(), geo_bounds + + +def _geo_raster_to_png_bytes(path: str) -> tuple[bytes, list[float]]: + import numpy as np + from PIL import Image + + with _open_envi_rasterio(path) as ds: + data = ds.read(1).astype("float32") + nodata = ds.nodata + bounds = ds.bounds + geo_bounds = [bounds.bottom, bounds.left, bounds.top, bounds.right] + + if nodata is not None: + nodata_mask = (data == nodata) | ~np.isfinite(data) + else: + nodata_mask = ~np.isfinite(data) + + valid = data[~nodata_mask] + if valid.size == 0: + normalized = np.zeros_like(data, dtype=np.uint8) + else: + p2, p98 = np.percentile(valid, 2), np.percentile(valid, 98) + clipped = np.clip(data, p2, p98) + normalized = ((clipped - p2) / max(p98 - p2, 1e-9) * 255).astype(np.uint8) + + rgba = np.stack([normalized, normalized, normalized, np.full_like(normalized, 200)], axis=-1) + rgba[nodata_mask, 3] = 0 + image = Image.fromarray(rgba, "RGBA") + buffer = io.BytesIO() + image.save(buffer, format="PNG") + return buffer.getvalue(), geo_bounds + + +_FLOOD_COLORMAP = { + 1: (24, 144, 255, 200), + 2: (255, 77, 79, 220), + 3: (250, 173, 20, 180), + 4: (80, 80, 80, 80), +} + + +async def get_flood_detection_preview(detection_id: int, layer: str, db: AsyncSession): + normalized_layer = layer.strip().lower() + if normalized_layer == "classified": + detection = await db.get(FloodDetectionORM, detection_id) + if not detection or not detection.classified_path: + raise HTTPException(status_code=404, detail="Classified result not found") + path = detection.classified_path.replace("\\", "/") + png_bytes, geo_bounds = await _render_classified_preview(path) + return JSONResponse( + { + "image_b64": base64.b64encode(png_bytes).decode(), + "bounds": geo_bounds, + "legend": { + "stable_water": "#1890ff", + "flood": "#ff4d4f", + "high_backscatter": "#faad14", + "non_water": "#505050", + }, + } + ) + if normalized_layer in ("pre", "post"): + return await _get_scene_preview_for_detection(detection_id, normalized_layer, db) + raise HTTPException(status_code=404, detail=f"Unsupported preview layer: {layer}") + + +async def _render_classified_preview(path: str) -> tuple[bytes, list[float]]: + if not os.path.isfile(path): + raise HTTPException(status_code=404, detail="Requested file does not exist") + try: + return await asyncio.to_thread(_raster_to_png_bytes, path, _FLOOD_COLORMAP) + except Exception as exc: + raise HTTPException(status_code=500, detail=f"Render failed: {exc}") from exc + + +async def _get_scene_preview_for_detection(detection_id: int, layer: str, db: AsyncSession): + detection = await db.get(FloodDetectionORM, detection_id) + if not detection: + raise HTTPException(status_code=404, detail="Flood detection not found") + scene_id = detection.pre_scene_id if layer == "pre" else detection.post_scene_id + scene = await db.get(SARSceneGeoORM, scene_id) + scene_path = _scene_analysis_path(scene) + if not scene_path: + raise HTTPException(status_code=404, detail="Scene analysis-ready GeoTIFF not found") + path = scene_path.replace("\\", "/") + if not os.path.isfile(path): + raise HTTPException(status_code=404, detail="Requested file does not exist") + try: + png_bytes, geo_bounds = await asyncio.to_thread(_geo_raster_to_png_bytes, path) + except Exception as exc: + raise HTTPException(status_code=500, detail=f"Render failed: {exc}") from exc + return JSONResponse({"image_b64": base64.b64encode(png_bytes).decode(), "bounds": geo_bounds}) + + +async def get_water_extraction_preview(extraction_id: int, db: AsyncSession) -> dict[str, Any]: + detection = await db.get(WaterExtractionORM, extraction_id) + if not detection: + detection = await db.get(WaterDetectionORM, extraction_id) + if not detection: + raise HTTPException(status_code=404, detail=f"Water extraction id={extraction_id} not found") + if not detection.output_path or not os.path.isfile(detection.output_path): + raise HTTPException(status_code=404, detail="Output file does not exist") + + import numpy as np + import rasterio + from PIL import Image + + with rasterio.open(detection.output_path) as src: + data = src.read(1) + transform = src.transform + height, width = data.shape + min_lon = transform.c + max_lon = transform.c + width * transform.a + max_lat = transform.f + min_lat = transform.f + height * transform.e + + rgba = np.zeros((data.shape[0], data.shape[1], 4), dtype=np.uint8) + rgba[data > 0] = [24, 144, 255, 160] + + image = Image.fromarray(rgba, "RGBA") + max_dim = 1024 + if max(width, height) > max_dim: + ratio = max_dim / max(width, height) + image = image.resize((int(width * ratio), int(height * ratio)), Image.NEAREST) + + buffer = io.BytesIO() + image.save(buffer, format="PNG") + return { + "png_base64": base64.b64encode(buffer.getvalue()).decode(), + "bounds": { + "min_lon": min_lon, + "min_lat": min_lat, + "max_lon": max_lon, + "max_lat": max_lat, + }, + } diff --git a/backend/app/services/flood_detection_service.py b/backend/app/services/flood_detection_service.py new file mode 100644 index 0000000..3a1d5f2 --- /dev/null +++ b/backend/app/services/flood_detection_service.py @@ -0,0 +1,327 @@ +"""Pure GeoTIFF flood detection for the flood-analysis module. + +This service deliberately does not depend on ENVI/SARscape. Satellite-specific +preprocessors are responsible only for producing analysis-ready GeoTIFFs; the +flood classification below operates on those GeoTIFFs with Python/rasterio. +""" +from __future__ import annotations + +import json +import math +import os +from pathlib import Path +from typing import Any + +import numpy as np + + +def _valid_mask(data: np.ndarray, nodata: float | int | None) -> np.ndarray: + valid = np.isfinite(data) + if nodata is not None and np.isfinite(float(nodata)): + valid &= data != float(nodata) + return valid + + +def _sample_valid(values: np.ndarray, max_samples: int = 1_000_000) -> np.ndarray: + flat = values[np.isfinite(values)] + if flat.size <= max_samples: + return flat + step = max(1, int(math.ceil(flat.size / max_samples))) + return flat[::step] + + +def _otsu_threshold(values: np.ndarray) -> float: + sample = _sample_valid(values) + if sample.size < 100: + raise ValueError("Too few valid pixels for thresholding") + manual_threshold = _manual_otsu_threshold(sample) + try: + from skimage.filters import threshold_otsu + + skimage_threshold = float(threshold_otsu(sample)) + if np.isfinite(skimage_threshold): + p05, p95 = np.nanpercentile(sample, [5, 95]) + if p05 < skimage_threshold < p95: + return skimage_threshold + return manual_threshold + except Exception: + return manual_threshold + + +def _manual_otsu_threshold(values: np.ndarray) -> float: + sample = _sample_valid(values) + if sample.size < 100: + raise ValueError("Too few valid pixels for thresholding") + vmin = float(np.nanmin(sample)) + vmax = float(np.nanmax(sample)) + if not np.isfinite(vmin) or not np.isfinite(vmax): + raise ValueError("Input pixels are not finite") + if math.isclose(vmin, vmax): + return vmin + + hist, edges = np.histogram(sample, bins=256, range=(vmin, vmax)) + hist = hist.astype("float64") + centers = (edges[:-1] + edges[1:]) / 2.0 + total = hist.sum() + if total <= 0: + return float(np.nanpercentile(sample, 10)) + + weight_background = np.cumsum(hist) + weight_foreground = total - weight_background + mean_background = np.cumsum(hist * centers) / np.maximum(weight_background, 1e-12) + mean_foreground = ( + np.cumsum((hist * centers)[::-1]) / np.maximum(np.cumsum(hist[::-1]), 1e-12) + )[::-1] + variance = weight_background[:-1] * weight_foreground[:-1] * ( + mean_background[:-1] - mean_foreground[1:] + ) ** 2 + if variance.size == 0 or not np.isfinite(variance).any(): + return float(np.nanpercentile(sample, 10)) + idx = int(np.nanargmax(variance)) + return float(edges[idx + 1]) + + +def _pixel_area_km2(transform: Any, crs: Any, bounds: Any) -> float: + px_w = abs(float(transform.a)) + px_h = abs(float(transform.e)) + if crs and getattr(crs, "is_geographic", False): + lat_center = (float(bounds.top) + float(bounds.bottom)) / 2.0 + px_w_m = px_w * math.cos(math.radians(lat_center)) * 111_320.0 + px_h_m = px_h * 111_320.0 + else: + px_w_m, px_h_m = px_w, px_h + return max(0.0, (px_w_m * px_h_m) / 1_000_000.0) + + +def _clean_mask(mask: np.ndarray, min_pixels: int) -> np.ndarray: + try: + from scipy.ndimage import binary_closing, binary_opening, generate_binary_structure, label + except Exception: + return mask + + structure = generate_binary_structure(2, 2) + cleaned = binary_closing(mask, structure=structure, iterations=1) + cleaned = binary_opening(cleaned, structure=structure, iterations=1) + if min_pixels <= 1: + return cleaned + + labels, count = label(cleaned) + if count <= 0: + return cleaned + component_sizes = np.bincount(labels.ravel()) + keep = component_sizes >= int(min_pixels) + keep[0] = False + return keep[labels] + + +def _read_pre_on_post_grid(pre_path: str, post_profile: dict[str, Any]) -> tuple[np.ndarray, dict[str, Any]]: + import rasterio + from rasterio.enums import Resampling + from rasterio.warp import reproject + + with rasterio.open(pre_path) as pre_ds: + pre_data = pre_ds.read(1).astype("float32") + pre_nodata = pre_ds.nodata + same_grid = ( + pre_ds.width == int(post_profile["width"]) + and pre_ds.height == int(post_profile["height"]) + and pre_ds.transform == post_profile["transform"] + and str(pre_ds.crs or "") == str(post_profile["crs"] or "") + ) + metadata = { + "path": pre_path, + "crs": pre_ds.crs.to_string() if pre_ds.crs else None, + "width": pre_ds.width, + "height": pre_ds.height, + "nodata": pre_nodata, + "reprojected_to_post_grid": not same_grid, + } + if same_grid: + data = pre_data.astype("float32") + if pre_nodata is not None and np.isfinite(float(pre_nodata)): + data[data == float(pre_nodata)] = np.nan + return data, metadata + if not pre_ds.crs or not post_profile["crs"]: + raise ValueError("Pre/post GeoTIFF CRS is required when grids differ") + destination = np.full( + (int(post_profile["height"]), int(post_profile["width"])), + np.nan, + dtype="float32", + ) + reproject( + source=pre_data, + destination=destination, + src_transform=pre_ds.transform, + src_crs=pre_ds.crs, + src_nodata=pre_nodata, + dst_transform=post_profile["transform"], + dst_crs=post_profile["crs"], + dst_nodata=np.nan, + resampling=Resampling.bilinear, + ) + return destination, metadata + + +def run_geotiff_flood_detection( + *, + pre_tif_path: str, + post_tif_path: str, + output_dir: str, + job_id: str | None = None, + refine: bool = False, +) -> dict[str, Any]: + """Classify stable water and new flood extent from two analysis-ready GeoTIFFs.""" + import rasterio + + pre_path = Path(os.path.normpath(str(pre_tif_path or "").strip())) + post_path = Path(os.path.normpath(str(post_tif_path or "").strip())) + out_dir = Path(os.path.normpath(str(output_dir or "").strip())) + if not pre_path.is_file(): + return {"ok": False, "error": f"Pre-event analysis GeoTIFF not found: {pre_path}"} + if not post_path.is_file(): + return {"ok": False, "error": f"Post-event analysis GeoTIFF not found: {post_path}"} + out_dir.mkdir(parents=True, exist_ok=True) + + with rasterio.open(post_path) as post_ds: + post_data = post_ds.read(1).astype("float32") + post_nodata = post_ds.nodata + post_profile = post_ds.profile.copy() + post_grid = { + "height": post_ds.height, + "width": post_ds.width, + "transform": post_ds.transform, + "crs": post_ds.crs, + } + post_metadata = { + "path": str(post_path), + "crs": post_ds.crs.to_string() if post_ds.crs else None, + "width": post_ds.width, + "height": post_ds.height, + "nodata": post_nodata, + } + pixel_area_km2 = _pixel_area_km2(post_ds.transform, post_ds.crs, post_ds.bounds) + + pre_data, pre_metadata = _read_pre_on_post_grid(str(pre_path), post_grid) + valid_pre = _valid_mask(pre_data, None) + valid_post = _valid_mask(post_data, post_nodata) + valid = valid_pre & valid_post + if int(np.count_nonzero(valid)) < 100: + return {"ok": False, "error": "Too few overlapping valid pixels between pre/post GeoTIFFs"} + + pre_valid_values = pre_data[valid] + post_valid_values = post_data[valid] + pre_threshold = _otsu_threshold(pre_valid_values) + post_threshold = _otsu_threshold(post_valid_values) + + pre_water = (pre_data <= pre_threshold) & valid + post_water = (post_data <= post_threshold) & valid + stable_water = pre_water & post_water + flood = post_water & ~pre_water + + if refine: + min_pixels = max(4, int(round(3_000.0 / max(pixel_area_km2 * 1_000_000.0, 1.0)))) + stable_water = _clean_mask(stable_water, min_pixels=min_pixels) + flood = _clean_mask(flood, min_pixels=min_pixels) + + high_threshold = float(np.nanpercentile(post_valid_values, 98)) + high_backscatter = (post_data >= high_threshold) & valid & ~(stable_water | flood) + + classified = np.zeros(post_data.shape, dtype="uint8") + classified[valid] = 4 + classified[high_backscatter] = 3 + classified[stable_water] = 1 + classified[flood] = 2 + + classified_path = out_dir / "classified.tif" + flood_mask_path = out_dir / "flood_mask.tif" + stable_mask_path = out_dir / "stable_water_mask.tif" + + classified_profile = post_profile.copy() + classified_profile.update( + driver="GTiff", + dtype="uint8", + count=1, + nodata=0, + compress="deflate", + ) + with rasterio.open(classified_path, "w", **classified_profile) as dst: + dst.write(classified, 1) + try: + dst.write_colormap( + 1, + { + 0: (0, 0, 0, 0), + 1: (24, 144, 255, 255), + 2: (255, 77, 79, 255), + 3: (250, 173, 20, 255), + 4: (80, 80, 80, 255), + }, + ) + except Exception: + pass + + mask_profile = classified_profile.copy() + mask_profile.update(nodata=0) + with rasterio.open(flood_mask_path, "w", **mask_profile) as dst: + dst.write(np.where(flood, 255, 0).astype("uint8"), 1) + with rasterio.open(stable_mask_path, "w", **mask_profile) as dst: + dst.write(np.where(stable_water, 255, 0).astype("uint8"), 1) + + flood_pixels = int(np.count_nonzero(flood)) + stable_pixels = int(np.count_nonzero(stable_water)) + high_pixels = int(np.count_nonzero(high_backscatter)) + non_water_pixels = int(np.count_nonzero(classified == 4)) + metadata = { + "schema": "flood_detection_geotiff.v1", + "job_id": job_id, + "processor": "python_geotiff_otsu_change", + "refine": bool(refine), + "pre": pre_metadata, + "post": post_metadata, + "thresholds": { + "pre_water_threshold": pre_threshold, + "post_water_threshold": post_threshold, + "post_high_backscatter_threshold": high_threshold, + }, + "pixel_area_km2": pixel_area_km2, + "class_values": { + "0": "nodata", + "1": "stable_water", + "2": "flood", + "3": "high_backscatter", + "4": "non_water", + }, + "counts": { + "valid_pixels": int(np.count_nonzero(valid)), + "stable_water_pixels": stable_pixels, + "flood_pixels": flood_pixels, + "high_backscatter_pixels": high_pixels, + "non_water_pixels": non_water_pixels, + }, + "outputs": { + "classified_path": str(classified_path), + "flood_mask_path": str(flood_mask_path), + "stable_water_mask_path": str(stable_mask_path), + }, + } + metadata_path = out_dir / "metadata.json" + metadata_path.write_text(json.dumps(metadata, ensure_ascii=False, indent=2, default=str), encoding="utf-8") + + return { + "ok": True, + "classified_path": str(classified_path), + "flood_mask_path": str(flood_mask_path), + "stable_water_mask_path": str(stable_mask_path), + "metadata_path": str(metadata_path), + "flood_area_km2": round(flood_pixels * pixel_area_km2, 4), + "stable_water_area_km2": round(stable_pixels * pixel_area_km2, 4), + "flood_pixel_count": flood_pixels, + "stable_water_pixel_count": stable_pixels, + "processor": "python_geotiff_otsu_change", + "log": [ + "pre/post analysis-ready GeoTIFFs loaded", + "pre scene reprojected to post-event grid", + f"thresholds pre={pre_threshold:.4f}, post={post_threshold:.4f}", + f"flood_pixels={flood_pixels}, stable_water_pixels={stable_pixels}", + ], + } diff --git a/backend/app/services/flood_overlay_service.py b/backend/app/services/flood_overlay_service.py new file mode 100644 index 0000000..cb25e01 --- /dev/null +++ b/backend/app/services/flood_overlay_service.py @@ -0,0 +1,389 @@ +"""Flood overlay and impact analysis service.""" +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any + +from fastapi import HTTPException +from geoalchemy2.functions import ST_Intersects +from geoalchemy2.shape import from_shape +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession +from shapely.geometry import mapping, shape +from shapely.ops import unary_union + +from ..config import settings +from ..models import FloodDetectionORM, FloodOverlayORM, HazardPointORM, ResultProductORM + + +def _to_float(value: Any) -> float | None: + try: + if value is None: + return None + return float(value) + except Exception: + return None + + +def _hazard_point_to_dict(point: HazardPointORM, *, distance_m: float | None = None) -> dict[str, Any]: + return { + "id": point.id, + "name": point.hazard_name, + "type": point.hazard_type, + "city": point.city, + "county": point.county, + "township": point.township, + "longitude": _to_float(point.longitude), + "latitude": _to_float(point.latitude), + "distance_m": 0 if distance_m is None else round(float(distance_m), 2), + } + + +def _dinsar_product_to_dict(product: ResultProductORM) -> dict[str, Any]: + summary = product.summary_json if isinstance(product.summary_json, dict) else {} + deformation = ( + summary.get("deformation_mm") + or summary.get("max_deformation_mm") + or summary.get("mean_deformation_mm") + or summary.get("deformation") + ) + return { + "id": product.id, + "product_id": product.product_id, + "display_name": product.display_name, + "engine": product.engine_code, + "status": product.status, + "deformation_mm": deformation, + "ai_score": product.ai_score, + "manifest_path": product.manifest_path, + "preview_path": product.preview_path, + } + + +def _open_raster(path: str): + import rasterio + + normalized_path = path.replace("\\", "/") + try: + return rasterio.open(normalized_path) + except Exception: + pass + for ext in (".bin", ".img", ".tif", ".tiff"): + candidate = normalized_path + ext + try: + return rasterio.open(candidate) + except Exception: + pass + raise FileNotFoundError(f"Raster file cannot be opened: {path}") + + +def _classified_flood_to_geojson(path: str) -> tuple[dict[str, Any], float | None, list[str]]: + import rasterio.features + from pyproj import CRS, Transformer + from shapely.geometry import shape as shape_geojson + from shapely.ops import transform + + warnings: list[str] = [] + with _open_raster(path) as src: + data = src.read(1) + mask = data == 2 + if not mask.any(): + return {"type": "FeatureCollection", "features": []}, 0.0, warnings + + polygons = [] + for geom, value in rasterio.features.shapes(data, mask=mask, transform=src.transform): + if int(value) != 2: + continue + polygon = shape_geojson(geom) + if not polygon.is_empty and polygon.is_valid: + polygons.append(polygon) + + if not polygons: + return {"type": "FeatureCollection", "features": []}, 0.0, warnings + + flood_geom = unary_union(polygons) + source_crs = src.crs + + area_km2 = None + output_geom = flood_geom + if source_crs: + try: + crs = CRS.from_user_input(source_crs) + if not crs.is_geographic: + area_km2 = float(flood_geom.area) / 1_000_000.0 + transformer = Transformer.from_crs(crs, CRS.from_epsg(4326), always_xy=True) + output_geom = transform(transformer.transform, flood_geom) + else: + centroid = flood_geom.centroid + zone = int((centroid.x + 180) // 6) + 1 + epsg = 32600 + zone if centroid.y >= 0 else 32700 + zone + transformer = Transformer.from_crs(crs, CRS.from_epsg(epsg), always_xy=True) + projected = transform(transformer.transform, flood_geom) + area_km2 = float(projected.area) / 1_000_000.0 + except Exception as exc: + warnings.append(f"area calculation failed: {exc}") + else: + warnings.append("classified raster has no CRS; geometry is stored in source coordinates") + + feature_collection = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {"class": 2, "name": "flood"}, + "geometry": mapping(output_geom), + } + ], + } + return feature_collection, area_km2, warnings + + +def _write_geojson(detection_id: int, feature_collection: dict[str, Any]) -> str: + out_dir = Path(settings.WATER_RESULTS_DIR or Path(settings.BACKEND_DIR) / "water_results") / "flood_overlays" + out_dir.mkdir(parents=True, exist_ok=True) + target = out_dir / f"flood_detection_{detection_id}_overlay.geojson" + target.write_text(json.dumps(feature_collection, ensure_ascii=False, indent=2), encoding="utf-8") + return str(target) + + +def _read_geojson(path: str | None) -> dict[str, Any] | None: + if not path or not os.path.isfile(path): + return None + try: + with open(path, "r", encoding="utf-8") as stream: + payload = json.load(stream) + return payload if isinstance(payload, dict) else None + except Exception: + return None + + +def _geometry_area_km2(geom) -> float: + if geom.is_empty: + return 0.0 + try: + from pyproj import CRS, Transformer + from shapely.ops import transform + + centroid = geom.centroid + zone = int((centroid.x + 180) // 6) + 1 + epsg = 32600 + zone if centroid.y >= 0 else 32700 + zone + transformer = Transformer.from_crs(CRS.from_epsg(4326), CRS.from_epsg(epsg), always_xy=True) + projected = transform(transformer.transform, geom) + return float(projected.area) / 1_000_000.0 + except Exception: + return 0.0 + + +def _calculate_affected_aois(flood_geom, warnings: list[str], *, limit: int = 50) -> list[dict[str, Any]]: + try: + from ..routers import dependencies as deps + + deps._load_region_index() + deps._load_region_geometry_index() + region_by_id = deps._REGION_BY_ID_CACHE or {} + geometry_by_id = deps._REGION_GEOMETRY_BY_ID_CACHE or {} + except Exception as exc: + warnings.append(f"AOI overlay unavailable: {exc}") + return [] + + affected: list[dict[str, Any]] = [] + for tree_id, features in geometry_by_id.items(): + node = region_by_id.get(tree_id) or {} + level = node.get("level") + if level in {"country", "province"}: + continue + try: + geometries = [shape(feature["geometry"]) for feature in features if feature.get("geometry")] + if not geometries: + continue + region_geom = unary_union(geometries) + if region_geom.is_empty or not flood_geom.intersects(region_geom): + continue + intersection = flood_geom.intersection(region_geom) + area_km2 = _geometry_area_km2(intersection) + if area_km2 <= 0.0001: + continue + affected.append( + { + "tree_id": tree_id, + "name": node.get("name") or tree_id, + "level": level, + "flood_area_km2": round(area_km2, 4), + } + ) + except Exception: + continue + + affected.sort(key=lambda item: item["flood_area_km2"], reverse=True) + return affected[:limit] + + +def _attach_overlay_payload(overlay: FloodOverlayORM) -> dict[str, Any]: + payload = dict(overlay.summary_json) if isinstance(overlay.summary_json, dict) else {} + payload["overlay_id"] = overlay.id + payload["detection_id"] = overlay.detection_id + payload["flood_vector_path"] = overlay.flood_vector_path + payload["flood_vector_geojson"] = _read_geojson(overlay.flood_vector_path) + return payload + + +async def run_overlay(detection_id: int, db: AsyncSession, *, near_threshold_m: float = 500.0) -> dict[str, Any]: + detection = await db.get(FloodDetectionORM, detection_id) + if not detection: + raise HTTPException(status_code=404, detail=f"Flood detection id={detection_id} not found") + if not detection.classified_path: + raise HTTPException(status_code=400, detail="Flood detection has no classified raster") + path = detection.classified_path.replace("\\", "/") + if not os.path.isfile(path): + raise HTTPException(status_code=404, detail="Classified raster file does not exist") + + feature_collection, flood_area_km2, warnings = _classified_flood_to_geojson(path) + flood_vector_path = _write_geojson(detection_id, feature_collection) + impact = await _query_impact_from_geojson( + detection_id=detection_id, + feature_collection=feature_collection, + db=db, + near_threshold_m=near_threshold_m, + warnings=warnings, + ) + if flood_area_km2 is not None: + impact["flood_area_km2"] = round(flood_area_km2, 4) + impact["flood_vector_path"] = flood_vector_path + + overlay = FloodOverlayORM( + detection_id=detection_id, + flood_vector_path=flood_vector_path, + hazard_points_hit=len(impact["hazard_points"]["inside_flood"]), + hazard_points_near=len(impact["hazard_points"]["near_flood"]), + hazard_points_total=impact["hazard_points"]["total_in_scene"], + dinsar_products_intersecting=len(impact["dinsar_products"]), + affected_area_km2=impact.get("flood_area_km2"), + summary_json=impact, + ) + db.add(overlay) + await db.flush() + impact["overlay_id"] = overlay.id + overlay.summary_json = impact + if flood_area_km2 is not None: + detection.flood_area_km2 = round(flood_area_km2, 4) + await db.commit() + await db.refresh(overlay) + + return { + "id": overlay.id, + "detection_id": detection_id, + "flood_vector_path": overlay.flood_vector_path, + "flood_vector_geojson": feature_collection, + "summary": _attach_overlay_payload(overlay), + } + + +async def get_overlay_result(detection_id: int, db: AsyncSession) -> dict[str, Any]: + overlay = ( + await db.execute( + select(FloodOverlayORM) + .where(FloodOverlayORM.detection_id == detection_id) + .order_by(FloodOverlayORM.id.desc()) + ) + ).scalars().first() + if overlay and isinstance(overlay.summary_json, dict): + return _attach_overlay_payload(overlay) + + detection = await db.get(FloodDetectionORM, detection_id) + if not detection: + raise HTTPException(status_code=404, detail=f"Flood detection id={detection_id} not found") + return { + "detection_id": detection_id, + "flood_area_km2": detection.flood_area_km2, + "hazard_points": {"inside_flood": [], "near_flood": [], "total_in_scene": 0}, + "dinsar_products": [], + "affected_aois": [], + "flood_vector_path": None, + "flood_vector_geojson": None, + "warnings": ["overlay has not been run"], + } + + +async def _query_impact_from_geojson( + *, + detection_id: int, + feature_collection: dict[str, Any], + db: AsyncSession, + near_threshold_m: float, + warnings: list[str], +) -> dict[str, Any]: + features = feature_collection.get("features") or [] + if not features: + return { + "detection_id": detection_id, + "flood_area_km2": 0.0, + "hazard_points": {"inside_flood": [], "near_flood": [], "total_in_scene": 0}, + "dinsar_products": [], + "affected_aois": [], + "warnings": warnings, + } + + flood_geom = unary_union([shape(feature["geometry"]) for feature in features if feature.get("geometry")]) + if flood_geom.is_empty: + warnings.append("flood geometry is empty") + flood_wkt = flood_geom.wkt + area_geom = func.ST_GeomFromText(flood_wkt, 4326) + area_geog = func.Geography(area_geom) + + inside_points: list[dict[str, Any]] = [] + near_points: list[dict[str, Any]] = [] + dinsar_products: list[dict[str, Any]] = [] + affected_aois = _calculate_affected_aois(flood_geom, warnings) + + try: + inside_rows = ( + await db.execute( + select(HazardPointORM).where(ST_Intersects(HazardPointORM.geom, area_geom)) + ) + ).scalars().all() + inside_ids = {point.id for point in inside_rows} + inside_points = [_hazard_point_to_dict(point, distance_m=0) for point in inside_rows] + + near_rows = ( + await db.execute( + select( + HazardPointORM, + func.ST_Distance(func.Geography(HazardPointORM.geom), area_geog).label("distance_m"), + ).where(func.ST_DWithin(func.Geography(HazardPointORM.geom), area_geog, near_threshold_m)) + ) + ).all() + for point, distance_m in near_rows: + if point.id in inside_ids: + continue + near_points.append(_hazard_point_to_dict(point, distance_m=distance_m)) + except Exception as exc: + warnings.append(f"hazard point overlay unavailable: {exc}") + + try: + products = ( + await db.execute( + select(ResultProductORM).where( + ResultProductORM.catalog_name == "dinsar", + ResultProductORM.status == "READY", + ST_Intersects(ResultProductORM.geom, area_geom), + ) + ) + ).scalars().all() + dinsar_products = [_dinsar_product_to_dict(product) for product in products] + except Exception as exc: + warnings.append(f"dinsar product overlay unavailable: {exc}") + + return { + "detection_id": detection_id, + "flood_area_km2": None, + "hazard_points": { + "inside_flood": inside_points, + "near_flood": near_points, + "total_in_scene": len(inside_points) + len(near_points), + }, + "dinsar_products": dinsar_products, + "affected_aois": affected_aois, + "warnings": warnings, + } diff --git a/backend/app/services/flood_product_service.py b/backend/app/services/flood_product_service.py new file mode 100644 index 0000000..c980633 --- /dev/null +++ b/backend/app/services/flood_product_service.py @@ -0,0 +1,199 @@ +"""Flood product listing and manifest helpers.""" +from __future__ import annotations + +import json +import os +from datetime import datetime +from typing import Any + +from fastapi import HTTPException +from sqlalchemy import select, func +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from ..models import FloodDetectionORM, FloodOverlayORM, FloodProductORM, SARSceneGeoORM + + +def _iso(value: Any) -> str | None: + return value.isoformat() if value else None + + +def _product_to_dict(product: FloodProductORM) -> dict[str, Any]: + summary = product.summary_json if isinstance(product.summary_json, dict) else {} + detection = product.detection + overlay = product.overlay + return { + "id": product.id, + "product_id": product.product_id, + "detection_id": product.detection_id, + "overlay_id": product.overlay_id, + "display_name": product.display_name, + "status": product.status, + "publish_dir": product.publish_dir, + "manifest_path": product.manifest_path, + "summary": summary, + "created_at": _iso(product.created_at), + "flood_area_km2": getattr(detection, "flood_area_km2", None), + "affected_area_km2": getattr(overlay, "affected_area_km2", None), + } + + +async def list_flood_products( + *, + db: AsyncSession, + limit: int = 20, + offset: int = 0, + status: str | None = None, +) -> dict[str, Any]: + count_query = select(func.count()).select_from(FloodProductORM) + query = ( + select(FloodProductORM) + .options( + selectinload(FloodProductORM.detection), + selectinload(FloodProductORM.overlay), + ) + .order_by(FloodProductORM.id.desc()) + .limit(limit) + .offset(offset) + ) + if status: + count_query = count_query.where(FloodProductORM.status == status) + query = query.where(FloodProductORM.status == status) + + total = (await db.execute(count_query)).scalar_one() + rows = (await db.execute(query)).scalars().all() + return {"items": [_product_to_dict(row) for row in rows], "total": total} + + +async def get_flood_product(product_id_or_pk: str, db: AsyncSession) -> dict[str, Any]: + product = await _get_product(product_id_or_pk, db) + return _product_to_dict(product) + + +async def get_flood_product_manifest(product_id_or_pk: str, db: AsyncSession) -> dict[str, Any]: + product = await _get_product(product_id_or_pk, db) + if product.manifest_path and os.path.isfile(product.manifest_path): + try: + with open(product.manifest_path, "r", encoding="utf-8") as stream: + payload = json.load(stream) + if isinstance(payload, dict): + return payload + except Exception as exc: + raise HTTPException(status_code=500, detail=f"Failed to read manifest: {exc}") from exc + + return _build_manifest_from_db(product) + + +async def create_flood_product_for_detection(detection_id: int, db: AsyncSession) -> dict[str, Any]: + detection = await db.get(FloodDetectionORM, detection_id) + if not detection: + raise HTTPException(status_code=404, detail=f"Flood detection id={detection_id} not found") + if detection.status != "DONE": + raise HTTPException(status_code=400, detail=f"Flood detection is not DONE: {detection.status}") + + existing = ( + await db.execute( + select(FloodProductORM) + .options( + selectinload(FloodProductORM.detection), + selectinload(FloodProductORM.overlay), + ) + .where(FloodProductORM.detection_id == detection_id) + .order_by(FloodProductORM.id.desc()) + ) + ).scalars().first() + if existing: + return _product_to_dict(existing) + + overlay = ( + await db.execute( + select(FloodOverlayORM) + .where(FloodOverlayORM.detection_id == detection_id) + .order_by(FloodOverlayORM.id.desc()) + ) + ).scalars().first() + product = FloodProductORM( + product_id=f"FLOOD-{detection_id:06d}", + detection_id=detection_id, + overlay_id=overlay.id if overlay else None, + display_name=f"Flood detection #{detection_id}", + status="READY", + publish_dir=detection.output_dir, + manifest_path=None, + summary_json={ + "created_from": "flood_detection", + "flood_area_km2": detection.flood_area_km2, + "stable_water_area_km2": detection.stable_water_area_km2, + "classified_path": detection.classified_path, + "created_at": datetime.utcnow().isoformat(), + }, + ) + db.add(product) + await db.commit() + return await get_flood_product(str(product.id), db) + + +async def _get_product(product_id_or_pk: str, db: AsyncSession) -> FloodProductORM: + value = str(product_id_or_pk).strip() + query = select(FloodProductORM).options( + selectinload(FloodProductORM.detection).selectinload(FloodDetectionORM.pre_scene).selectinload(SARSceneGeoORM.radar_data), + selectinload(FloodProductORM.detection).selectinload(FloodDetectionORM.post_scene).selectinload(SARSceneGeoORM.radar_data), + selectinload(FloodProductORM.overlay), + ) + if value.isdigit(): + query = query.where(FloodProductORM.id == int(value)) + else: + query = query.where(FloodProductORM.product_id == value) + product = (await db.execute(query)).scalars().first() + if not product: + raise HTTPException(status_code=404, detail=f"Flood product {product_id_or_pk} not found") + return product + + +def _build_manifest_from_db(product: FloodProductORM) -> dict[str, Any]: + detection = product.detection + overlay = product.overlay + pre_scene = detection.pre_scene if detection else None + post_scene = detection.post_scene if detection else None + pre_radar = pre_scene.radar_data if pre_scene else None + post_radar = post_scene.radar_data if post_scene else None + return { + "schema": "flood_product_manifest.v1", + "product": _product_to_dict(product), + "detection": { + "id": detection.id if detection else None, + "status": detection.status if detection else None, + "classified_path": detection.classified_path if detection else None, + "flood_area_km2": detection.flood_area_km2 if detection else None, + "stable_water_area_km2": detection.stable_water_area_km2 if detection else None, + "pre_scene": { + "id": pre_scene.id if pre_scene else None, + "radar_data_id": pre_scene.radar_data_id if pre_scene else None, + "satellite": pre_radar.satellite if pre_radar else None, + "imaging_date": pre_radar.imaging_date if pre_radar else None, + "geo_path": pre_scene.geo_path if pre_scene else None, + "analysis_tif_path": pre_scene.analysis_tif_path if pre_scene else None, + "analysis_engine": pre_scene.analysis_engine if pre_scene else None, + "analysis_profile": pre_scene.analysis_profile if pre_scene else None, + }, + "post_scene": { + "id": post_scene.id if post_scene else None, + "radar_data_id": post_scene.radar_data_id if post_scene else None, + "satellite": post_radar.satellite if post_radar else None, + "imaging_date": post_radar.imaging_date if post_radar else None, + "geo_path": post_scene.geo_path if post_scene else None, + "analysis_tif_path": post_scene.analysis_tif_path if post_scene else None, + "analysis_engine": post_scene.analysis_engine if post_scene else None, + "analysis_profile": post_scene.analysis_profile if post_scene else None, + }, + }, + "overlay": { + "id": overlay.id if overlay else None, + "flood_vector_path": overlay.flood_vector_path if overlay else None, + "affected_area_km2": overlay.affected_area_km2 if overlay else None, + "hazard_points_hit": overlay.hazard_points_hit if overlay else 0, + "hazard_points_near": overlay.hazard_points_near if overlay else 0, + "dinsar_products_intersecting": overlay.dinsar_products_intersecting if overlay else 0, + "summary": overlay.summary_json if overlay else None, + }, + } diff --git a/backend/app/services/gf3_unpack_service.py b/backend/app/services/gf3_unpack_service.py new file mode 100644 index 0000000..f43dd82 --- /dev/null +++ b/backend/app/services/gf3_unpack_service.py @@ -0,0 +1,369 @@ +from __future__ import annotations + +import logging +import os +import shutil +import stat +import tarfile +import zipfile +from datetime import datetime +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple + +from ..config import settings, split_env_paths + +logger = logging.getLogger(__name__) + +LogCallback = Callable[[str, str], None] +ProgressCallback = Callable[[int, str], None] + +DEFAULT_GF3_ARCHIVE_EXTS = (".zip", ".tar", ".tar.gz", ".tgz") + + +def _parse_float(value: Any, default: float) -> float: + try: + return float(value) + except (TypeError, ValueError): + return default + + +def _normalize_paths(paths: Optional[Iterable[str]]) -> List[str]: + ordered: List[str] = [] + for raw_path in paths or []: + text = str(raw_path or "").strip().strip('"').strip("'") + if not text: + continue + normalized = os.path.normpath(os.path.abspath(text)) + if normalized not in ordered: + ordered.append(normalized) + return ordered + + +def _normalize_extensions(extensions: Optional[Iterable[str]]) -> List[str]: + ordered: List[str] = [] + for raw_ext in extensions or DEFAULT_GF3_ARCHIVE_EXTS: + ext = str(raw_ext or "").strip().lower() + if not ext: + continue + if not ext.startswith("."): + ext = f".{ext}" + if ext not in ordered: + ordered.append(ext) + return sorted(ordered or list(DEFAULT_GF3_ARCHIVE_EXTS), key=len, reverse=True) + + +def _strip_archive_extension(file_name: str, extensions: Iterable[str]) -> str: + lower_name = file_name.lower() + for ext in _normalize_extensions(extensions): + if lower_name.endswith(ext): + return file_name[: -len(ext)] + return os.path.splitext(file_name)[0] + + +def _resolve_target_root(archive_path: str, source_dirs: List[str], target_dirs: List[str]) -> str: + if not target_dirs: + raise ValueError("GF3_SOURCE_DIRS is not configured.") + if len(target_dirs) == 1: + return target_dirs[0] + if source_dirs and len(source_dirs) == len(target_dirs): + archive_norm = os.path.normcase(os.path.abspath(archive_path)) + matches: List[Tuple[int, int]] = [] + for index, source_dir in enumerate(source_dirs): + source_norm = os.path.normcase(os.path.abspath(source_dir)) + if archive_norm == source_norm or archive_norm.startswith(source_norm + os.sep): + matches.append((len(source_norm), index)) + if matches: + _prefix_len, best_index = max(matches) + return target_dirs[best_index] + return target_dirs[0] + + +def _validate_relative_member_name(member_name: str, archive_path: str) -> str: + name = str(member_name or "").replace("\\", "/") + if not name or name in {".", "./"}: + return "" + if name.startswith("/") or os.path.isabs(name) or os.path.splitdrive(name)[0]: + raise ValueError(f"Unsafe archive member path in {archive_path}: {member_name}") + parts = [part for part in name.split("/") if part not in ("", ".")] + if any(part == ".." for part in parts): + raise ValueError(f"Unsafe archive member path in {archive_path}: {member_name}") + if not parts: + return "" + return os.path.join(*parts) + + +def _safe_destination(root_dir: str, relative_name: str) -> str: + root_abs = os.path.abspath(root_dir) + destination = os.path.abspath(os.path.join(root_abs, relative_name)) + if destination != root_abs and not destination.startswith(root_abs + os.sep): + raise ValueError(f"Unsafe extraction destination: {relative_name}") + return destination + + +def _validate_tar_members(members: Iterable[tarfile.TarInfo], archive_path: str) -> None: + for member in members: + _validate_relative_member_name(member.name, archive_path) + if member.issym() or member.islnk(): + raise ValueError(f"Unsupported link entry in {archive_path}: {member.name}") + if not (member.isdir() or member.isfile()): + raise ValueError(f"Unsupported special entry in {archive_path}: {member.name}") + + +def _validate_zip_members(infos: Iterable[zipfile.ZipInfo], archive_path: str) -> None: + for info in infos: + _validate_relative_member_name(info.filename, archive_path) + mode = (info.external_attr >> 16) & 0o170000 + if stat.S_ISLNK(mode): + raise ValueError(f"Unsupported symlink entry in {archive_path}: {info.filename}") + + +def _estimate_archive_size(archive_path: str) -> int: + if zipfile.is_zipfile(archive_path): + with zipfile.ZipFile(archive_path, "r") as zip_obj: + infos = zip_obj.infolist() + _validate_zip_members(infos, archive_path) + return sum(max(0, int(info.file_size or 0)) for info in infos if not info.is_dir()) + if tarfile.is_tarfile(archive_path): + with tarfile.open(archive_path, "r:*") as tar_obj: + members = tar_obj.getmembers() + _validate_tar_members(members, archive_path) + return sum(max(0, int(member.size or 0)) for member in members if member.isfile()) + raise ValueError(f"Unsupported GF3 archive format: {archive_path}") + + +def _ensure_disk_space(target_root: str, required_bytes: int, min_disk_space_gb: float) -> None: + os.makedirs(target_root, exist_ok=True) + _total, _used, free_bytes = shutil.disk_usage(target_root) + min_free_bytes = int(max(0.0, min_disk_space_gb) * (1024 ** 3)) + if free_bytes - max(0, int(required_bytes or 0)) < min_free_bytes: + raise OSError( + "GF3 L1A storage has insufficient free space: " + f"needed {required_bytes / (1024 ** 3):.2f} GB, " + f"free {free_bytes / (1024 ** 3):.2f} GB, " + f"min free after {min_disk_space_gb:.2f} GB" + ) + + +def _prepare_atomic_output(output_dir: str, tmp_suffix: str) -> Tuple[bool, str, str]: + tmp_dir = output_dir + tmp_suffix + lock_path = output_dir + ".unpacking" + if os.path.exists(output_dir): + return False, tmp_dir, lock_path + if os.path.exists(tmp_dir): + return False, tmp_dir, lock_path + if os.path.exists(lock_path): + return False, tmp_dir, lock_path + os.makedirs(tmp_dir, exist_ok=False) + with open(lock_path, "w", encoding="utf-8") as stream: + stream.write(datetime.now().isoformat()) + return True, tmp_dir, lock_path + + +def _cleanup_atomic_paths(tmp_dir: str, lock_path: str) -> None: + if os.path.exists(lock_path): + try: + os.remove(lock_path) + except OSError: + pass + if os.path.exists(tmp_dir): + try: + shutil.rmtree(tmp_dir) + except OSError: + pass + + +def _extract_tar_archive(archive_path: str, tmp_dir: str) -> int: + extracted_files = 0 + with tarfile.open(archive_path, "r:*") as tar_obj: + members = tar_obj.getmembers() + _validate_tar_members(members, archive_path) + for member in members: + relative_name = _validate_relative_member_name(member.name, archive_path) + if not relative_name: + continue + destination = _safe_destination(tmp_dir, relative_name) + if member.isdir(): + os.makedirs(destination, exist_ok=True) + continue + if not member.isfile(): + continue + os.makedirs(os.path.dirname(destination), exist_ok=True) + source = tar_obj.extractfile(member) + if source is None: + raise OSError(f"Failed to read tar member: {member.name}") + with source, open(destination, "wb") as target: + shutil.copyfileobj(source, target, length=1024 * 1024) + extracted_files += 1 + return extracted_files + + +def _extract_zip_archive(archive_path: str, tmp_dir: str) -> int: + extracted_files = 0 + with zipfile.ZipFile(archive_path, "r") as zip_obj: + infos = zip_obj.infolist() + _validate_zip_members(infos, archive_path) + for info in infos: + relative_name = _validate_relative_member_name(info.filename, archive_path) + if not relative_name: + continue + destination = _safe_destination(tmp_dir, relative_name) + if info.is_dir(): + os.makedirs(destination, exist_ok=True) + continue + os.makedirs(os.path.dirname(destination), exist_ok=True) + with zip_obj.open(info, "r") as source, open(destination, "wb") as target: + shutil.copyfileobj(source, target, length=1024 * 1024) + extracted_files += 1 + return extracted_files + + +def _extract_archive_atomic(archive_path: str, output_dir: str, tmp_suffix: str) -> Tuple[str, int]: + prepared, tmp_dir, lock_path = _prepare_atomic_output(output_dir, tmp_suffix) + if not prepared: + return "EXISTS", 0 + + try: + if zipfile.is_zipfile(archive_path): + extracted_files = _extract_zip_archive(archive_path, tmp_dir) + elif tarfile.is_tarfile(archive_path): + extracted_files = _extract_tar_archive(archive_path, tmp_dir) + else: + raise ValueError(f"Unsupported GF3 archive format: {archive_path}") + + if extracted_files <= 0: + raise OSError("GF3 archive extraction produced no files.") + os.replace(tmp_dir, output_dir) + return "EXTRACTED", extracted_files + finally: + _cleanup_atomic_paths(tmp_dir, lock_path) + + +def _discover_archives(source_dirs: List[str], extensions: List[str], log_callback: Optional[LogCallback]) -> List[str]: + archives: List[str] = [] + normalized_extensions = _normalize_extensions(extensions) + for source_dir in source_dirs: + if not os.path.isdir(source_dir): + message = f"GF3 archive source does not exist or is not a directory: {source_dir}" + logger.warning(message) + if log_callback: + log_callback("WARNING", message) + continue + for root, _dirs, files in os.walk(source_dir): + for file_name in files: + lower_name = file_name.lower() + if any(lower_name.endswith(ext) for ext in normalized_extensions): + archives.append(os.path.join(root, file_name)) + return sorted(archives) + + +def run_gf3_archive_unpack( + *, + source_dirs: Optional[Iterable[str]] = None, + target_dirs: Optional[Iterable[str]] = None, + archive_exts: Optional[Iterable[str]] = None, + max_files_per_run: Optional[int] = None, + delete_archive: Optional[bool] = None, + min_disk_space_gb: Optional[float] = None, + tmp_suffix: Optional[str] = None, + log_callback: Optional[LogCallback] = None, + progress_callback: Optional[ProgressCallback] = None, +) -> Dict[str, Any]: + configured_source_dirs = _normalize_paths( + source_dirs if source_dirs is not None else split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS) + ) + configured_target_dirs = _normalize_paths( + target_dirs if target_dirs is not None else split_env_paths(settings.GF3_SOURCE_DIRS) + ) + extensions = _normalize_extensions( + archive_exts if archive_exts is not None else split_env_paths(settings.GF3_ARCHIVE_EXTS) + ) + should_delete_archive = settings.GF3_UNPACK_DELETE_ARCHIVE if delete_archive is None else bool(delete_archive) + limit = max(0, int(max_files_per_run or 0)) + min_free_gb = ( + _parse_float(os.getenv("UNPACK_MIN_DISK_SPACE_GB"), 50.0) + if min_disk_space_gb is None + else float(min_disk_space_gb) + ) + atomic_tmp_suffix = str(tmp_suffix or os.getenv("UNPACK_TMP_SUFFIX") or ".unpack_tmp").strip() or ".unpack_tmp" + + if not configured_source_dirs: + raise ValueError("GF3_ARCHIVE_SOURCE_DIRS is not configured.") + if not configured_target_dirs: + raise ValueError("GF3_SOURCE_DIRS is not configured.") + + def _log(level: str, message: str) -> None: + logger.log(getattr(logging, level.upper(), logging.INFO), message) + if log_callback: + log_callback(level.upper(), message) + + def _progress(progress: int, message: str) -> None: + if progress_callback: + progress_callback(max(0, min(100, int(progress))), message) + + _progress(2, "Scanning GF3 archive source directories...") + archives = _discover_archives(configured_source_dirs, extensions, log_callback) + if limit > 0: + archives_to_process = archives[:limit] + else: + archives_to_process = archives + + total = len(archives_to_process) + summary: Dict[str, Any] = { + "total": total, + "found": len(archives), + "processed": 0, + "skipped": 0, + "failed": 0, + "remaining": max(0, len(archives) - total), + "source_dirs": configured_source_dirs, + "target_dirs": configured_target_dirs, + "archive_exts": extensions, + "delete_archive": should_delete_archive, + "failures": [], + } + + if not archives_to_process: + _progress(100, "No GF3 archives pending.") + summary["message"] = "No GF3 archives found." + return summary + + _log("INFO", f"Found {len(archives)} GF3 archives; processing {total}.") + + for index, archive_path in enumerate(archives_to_process, start=1): + archive_name = os.path.basename(archive_path) + progress_base = 5 + int(((index - 1) / max(1, total)) * 90) + _progress(progress_base, f"Unpacking GF3 archive {index}/{total}: {archive_name}") + + try: + target_root = _resolve_target_root(archive_path, configured_source_dirs, configured_target_dirs) + os.makedirs(target_root, exist_ok=True) + output_name = _strip_archive_extension(os.path.basename(archive_path), extensions) + output_dir = os.path.join(target_root, output_name) + required_bytes = _estimate_archive_size(archive_path) + _ensure_disk_space(target_root, required_bytes, min_free_gb) + status, extracted_files = _extract_archive_atomic(archive_path, output_dir, atomic_tmp_suffix) + + if status == "EXISTS": + summary["skipped"] += 1 + _log("INFO", f"GF3 archive already unpacked, skipped: {archive_path}") + continue + + if should_delete_archive: + os.remove(archive_path) + _log("INFO", f"GF3 archive deleted after successful unpack: {archive_path}") + summary["processed"] += 1 + _log("INFO", f"GF3 archive unpacked: {archive_path} -> {output_dir} ({extracted_files} files)") + except Exception as exc: + summary["failed"] += 1 + failure = { + "archive_path": archive_path, + "error": str(exc), + } + summary["failures"].append(failure) + _log("ERROR", f"GF3 archive unpack failed: {archive_path}: {exc}") + + _progress(100, "GF3 archive unpack completed.") + summary["message"] = ( + f"GF3 unpack complete: processed {summary['processed']}, " + f"skipped {summary['skipped']}, failed {summary['failed']}" + ) + return summary diff --git a/backend/app/services/health_service.py b/backend/app/services/health_service.py index dc92526..639bfc2 100644 --- a/backend/app/services/health_service.py +++ b/backend/app/services/health_service.py @@ -17,6 +17,7 @@ from ..models import ( OrbitAssetORM, ResultCatalogStateORM, ResultProductORM, + SARSceneGeoORM, SceneOrbitBindingORM, SourceProductAssetORM, SystemWorkerHeartbeatORM, @@ -362,6 +363,19 @@ def _sanitize_source_roots_status(payload: Dict[str, Any]) -> Dict[str, Any]: } +def _sanitize_sar_analysis_ready_status(payload: Dict[str, Any]) -> Dict[str, Any]: + scenes = payload.get("scenes", {}) or {} + roots = payload.get("roots", {}) or {} + return { + "ok": bool(payload.get("ok")), + "configured_root_count": len(roots), + "accessible_root_count": sum(1 for item in roots.values() if item.get("accessible")), + "scene_count": int(scenes.get("scene_count") or 0), + "analysis_scene_count": int(scenes.get("analysis_scene_count") or 0), + "missing_file_count": int(scenes.get("missing_file_count") or 0), + } + + def _sanitize_product_package_status(payload: Dict[str, Any]) -> Dict[str, Any]: return { "ok": bool(payload.get("ok")), @@ -460,6 +474,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]: psinsar_result_catalog = timeseries_result_catalog dinsar_bridge = payload.get("dinsar_bridge", {}) or {} source_roots = payload.get("source_roots", {}) or {} + sar_analysis_ready = payload.get("sar_analysis_ready", {}) or {} product_packages = payload.get("product_packages", {}) or {} asset_inventory = payload.get("asset_inventory", {}) or {} wsl_runtime = payload.get("wsl_runtime", {}) or {} @@ -473,6 +488,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]: sanitized_psinsar_catalog = sanitized_timeseries_catalog sanitized_dinsar_bridge = _sanitize_bridge_status(dinsar_bridge) sanitized_source_roots = _sanitize_source_roots_status(source_roots) + sanitized_sar_analysis_ready = _sanitize_sar_analysis_ready_status(sar_analysis_ready) sanitized_product_packages = _sanitize_product_package_status(product_packages) sanitized_asset_inventory = _sanitize_asset_inventory_status(asset_inventory) sanitized_wsl_runtime = _sanitize_wsl_runtime_status(wsl_runtime) @@ -502,6 +518,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]: }, "dinsar_bridge": sanitized_dinsar_bridge, "source_roots": sanitized_source_roots, + "sar_analysis_ready": sanitized_sar_analysis_ready, "product_packages": sanitized_product_packages, "asset_inventory": sanitized_asset_inventory, "wsl_runtime": sanitized_wsl_runtime, @@ -869,6 +886,21 @@ async def _check_source_roots() -> Dict[str, Any]: status["role"] = "dinsar_source" items.append(status) + for path in split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS): + status = _probe_directory_status(path) + status["role"] = "gf3_archive_source" + items.append(status) + + for path in split_env_paths(settings.GF3_SOURCE_DIRS): + status = _probe_directory_status(path) + status["role"] = "gf3_l1a_source" + items.append(status) + + for path in split_env_paths(settings.GF3_STORAGE_DIRS): + status = _probe_directory_status(path) + status["role"] = "gf3_l2_storage" + items.append(status) + configured_count = len(items) accessible_count = sum(1 for item in items if item.get("accessible")) inaccessible_count = configured_count - accessible_count @@ -882,6 +914,56 @@ async def _check_source_roots() -> Dict[str, Any]: } +async def _check_sar_analysis_ready() -> Dict[str, Any]: + roots = { + "ready": _probe_directory_status(settings.SAR_ANALYSIS_READY_ROOT), + "work": _probe_directory_status(settings.SAR_ANALYSIS_WORK_ROOT), + "preview": _probe_directory_status(settings.SAR_ANALYSIS_PREVIEW_ROOT), + } + for role, payload in roots.items(): + payload["role"] = role + + status: Dict[str, Any] = { + "ok": False, + "roots": roots, + "scenes": { + "scene_count": 0, + "analysis_scene_count": 0, + "missing_file_count": 0, + "missing_files": [], + }, + "error": None, + } + try: + session_factory = _get_session_factory() + async with session_factory() as db: + scene_count_result = await db.execute(select(func.count(SARSceneGeoORM.id))) + status["scenes"]["scene_count"] = int(scene_count_result.scalar_one() or 0) + analysis_rows_result = await db.execute( + select(SARSceneGeoORM.id, SARSceneGeoORM.analysis_tif_path) + .where(SARSceneGeoORM.analysis_tif_path.is_not(None)) + .order_by(SARSceneGeoORM.id.desc()) + ) + analysis_rows = analysis_rows_result.all() + status["scenes"]["analysis_scene_count"] = len(analysis_rows) + + missing_files = [] + for scene_id, tif_path in analysis_rows: + path_text = str(tif_path or "").strip() + if path_text and not os.path.isfile(path_text): + missing_files.append({"scene_id": scene_id, "path": path_text}) + status["scenes"]["missing_file_count"] = len(missing_files) + status["scenes"]["missing_files"] = missing_files[:20] + + status["ok"] = ( + all(item.get("accessible") for item in roots.values()) + and status["scenes"]["missing_file_count"] == 0 + ) + except Exception as exc: + status["error"] = str(exc) + return status + + async def _check_product_packages() -> Dict[str, Any]: status = { "ok": False, @@ -1256,6 +1338,7 @@ async def get_health_status( psinsar_result_catalog_status = timeseries_result_catalog_status dinsar_bridge_status = await _check_dinsar_bridge() source_roots_status = await _check_source_roots() + sar_analysis_ready_status = await _check_sar_analysis_ready() product_packages_status = await _check_product_packages() asset_inventory_status = await _check_asset_inventory() wsl_runtime_status = await _check_wsl_runtime() @@ -1273,6 +1356,7 @@ async def get_health_status( result_catalog_status.get("ok"), dinsar_bridge_status.get("ok"), source_roots_status.get("ok"), + sar_analysis_ready_status.get("ok"), product_packages_status.get("ok"), asset_inventory_status.get("ok"), wsl_runtime_status.get("ok"), @@ -1297,6 +1381,7 @@ async def get_health_status( }, "dinsar_bridge": dinsar_bridge_status, "source_roots": source_roots_status, + "sar_analysis_ready": sar_analysis_ready_status, "product_packages": product_packages_status, "asset_inventory": asset_inventory_status, "wsl_runtime": wsl_runtime_status, diff --git a/backend/app/services/job_handlers.py b/backend/app/services/job_handlers.py index d4822e2..e5da949 100644 --- a/backend/app/services/job_handlers.py +++ b/backend/app/services/job_handlers.py @@ -19,7 +19,7 @@ from sqlalchemy import select from .. import database from ..config import settings -from ..models import SystemJobORM, DinsarResultORM, HazardPointORM, DinsarTaskItemORM, PsTaskItemORM, RadarDataORM, SARSceneGeoORM, FloodDetectionORM, WaterDetectionORM, GF3ProcessingORM, AiDiagnosisORM +from ..models import SystemJobORM, DinsarResultORM, HazardPointORM, DinsarTaskItemORM, PsTaskItemORM, RadarDataORM, SARSceneGeoORM, FloodDetectionORM, WaterDetectionORM, WaterExtractionORM, GF3ProcessingORM, AiDiagnosisORM from ..scheduler import scan_data_job from .data_service import data_service from .asset_inventory_service import asset_inventory_service @@ -80,7 +80,10 @@ JOB_TYPE_IDL_RUN_DINSAR = "IDL_RUN_DINSAR" JOB_TYPE_WATER_GEOCODE = "WATER_GEOCODE" JOB_TYPE_WATER_FLOOD = "WATER_FLOOD" JOB_TYPE_WATER_DETECT = "WATER_DETECT" +JOB_TYPE_SAR_SCENE_PREPROCESS = "SAR_SCENE_PREPROCESS" +JOB_TYPE_FLOOD_DETECTION = "FLOOD_DETECTION" JOB_TYPE_GF3_PROCESS = "GF3_PROCESS" +JOB_TYPE_GF3_UNPACK = "GF3_UNPACK" JOB_TYPE_GF3_BATCH_PROCESS = "GF3_BATCH_PROCESS" JOB_TYPE_ISCE2_RUN = "ISCE2_RUN" JOB_TYPE_PYINT_RUN = "PYINT_RUN" @@ -88,6 +91,7 @@ JOB_TYPE_PUBLISH_DINSAR_PRODUCTS = "PUBLISH_DINSAR_PRODUCTS" JOB_TYPE_REBUILD_DINSAR_CATALOG = "REBUILD_DINSAR_CATALOG" JOB_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG" JOB_TYPE_SCAN_ASSET_INVENTORY = "SCAN_ASSET_INVENTORY" +JOB_TYPE_SBAS_COREGISTRATION = "SBAS_COREGISTRATION" COPY_ALLOWED_STATUSES = {"PENDING", "IN_PROGRESS", "COMPLETED", "FAILED"} @@ -3149,6 +3153,106 @@ async def _handle_water_geocode(job: SystemJobORM) -> None: ) +async def _handle_sar_scene_preprocess(job: SystemJobORM) -> None: + """Build one analysis-ready GeoTIFF for flood/water algorithms.""" + payload = job.payload or {} + scene_id = payload.get("scene_id") + radar_data_id = payload.get("radar_data_id") + engine = str(payload.get("engine") or "").strip().lower() + if not scene_id and not radar_data_id: + raise ValueError("SAR_SCENE_PREPROCESS requires scene_id or radar_data_id") + if engine not in {"gf3_gdal", "lt_gamma"}: + raise ValueError(f"Unsupported SAR scene preprocessing engine: {engine}") + + await task_service.start_task(job.task_id, message="Preparing analysis-ready SAR GeoTIFF...") + + async with AsyncSessionLocal() as db: + scene: SARSceneGeoORM | None = None + if scene_id: + scene = await db.get(SARSceneGeoORM, int(scene_id)) + if not scene and radar_data_id: + result = await db.execute( + select(SARSceneGeoORM).where(SARSceneGeoORM.radar_data_id == int(radar_data_id)) + ) + scene = result.scalar_one_or_none() + if not scene: + scene = SARSceneGeoORM(radar_data_id=int(radar_data_id), status="PENDING") + db.add(scene) + await db.flush() + radar = await db.get(RadarDataORM, int(scene.radar_data_id)) + if not radar: + raise ValueError(f"RadarDataORM id={scene.radar_data_id} does not exist") + scene.status = "RUNNING" + scene.error_msg = None + await db.commit() + scene_id = int(scene.id) + radar_data_id = int(radar.id) + + try: + if engine == "gf3_gdal": + await task_service.update_task(job.task_id, progress=20, message="Standardizing GF3 L2 GeoTIFF...") + from .sar_analysis_ready_service import standardize_gf3_l2_for_radar + + async with AsyncSessionLocal() as db: + manifest = await standardize_gf3_l2_for_radar( + db=db, + radar_id=int(radar_data_id), + l2_path=payload.get("l2_path"), + polarization=payload.get("polarization"), + ) + else: + await task_service.update_task(job.task_id, progress=15, message="Running LT Gamma single-scene preprocessing...") + from .lt_gamma_scene_service import run_lt_gamma_scene_preprocess + from .sar_analysis_ready_service import register_analysis_ready_tif + + async with AsyncSessionLocal() as db: + scene = await db.get(SARSceneGeoORM, int(scene_id)) + radar = await db.get(RadarDataORM, int(radar_data_id)) + if not scene or not radar: + raise ValueError("Scene or radar record disappeared before LT Gamma preprocessing") + + def _run_lt() -> Dict[str, Any]: + return run_lt_gamma_scene_preprocess(radar=radar, scene=scene, job_id=job.job_id) + + lt_manifest = await asyncio.to_thread(_run_lt) + await task_service.update_task(job.task_id, progress=85, message="Registering LT analysis-ready GeoTIFF...") + analysis_tif_path = str(lt_manifest.get("analysis_tif_path") or "").strip() + if not analysis_tif_path: + raise RuntimeError("LT Gamma preprocessing returned no analysis_tif_path") + async with AsyncSessionLocal() as db: + scene = await db.get(SARSceneGeoORM, int(scene_id)) + radar = await db.get(RadarDataORM, int(radar_data_id)) + if not scene or not radar: + raise ValueError("Scene or radar record disappeared before analysis-ready registration") + manifest = await register_analysis_ready_tif( + db=db, + scene=scene, + radar=radar, + source_tif_path=analysis_tif_path, + engine="lt_gamma", + profile="lt1_gamma_geocoded_mli", + backscatter_unit=str(lt_manifest.get("backscatter_unit") or "gamma_mli_db"), + polarization=radar.polarization, + metadata=lt_manifest, + ) + await db.commit() + except Exception as exc: + async with AsyncSessionLocal() as db: + scene = await db.get(SARSceneGeoORM, int(scene_id)) + if scene: + scene.status = "FAILED" + scene.error_msg = str(exc) + await db.commit() + raise + + await task_service.update_task( + job.task_id, + status="COMPLETED", + progress=100, + message=f"Analysis-ready GeoTIFF ready: {manifest.get('analysis_tif_path')}", + ) + + async def _handle_water_flood(job: SystemJobORM) -> None: """洪涝检测 job handler（灾前 + 灾后配对分类）。""" from .water_service import run_flood_detection, WATER_RESULTS_DIR @@ -3171,8 +3275,10 @@ async def _handle_water_flood(job: SystemJobORM) -> None: raise ValueError("灾前或灾后场景记录不存在") if pre_scene.status != "DONE" or post_scene.status != "DONE": raise ValueError("灾前或灾后场景尚未完成地理编码") - pre_geo = pre_scene.geo_path - post_geo = post_scene.geo_path + pre_geo = pre_scene.analysis_tif_path or pre_scene.geo_path + post_geo = post_scene.analysis_tif_path or post_scene.geo_path + if not pre_geo or not post_geo: + raise ValueError("Pre/post scenes must have analysis-ready GeoTIFF paths") output_dir = os.path.join(WATER_RESULTS_DIR, f"flood_{detection_id}") os.makedirs(output_dir, exist_ok=True) @@ -3228,36 +3334,129 @@ async def _handle_water_flood(job: SystemJobORM) -> None: ) -async def _handle_water_detect(job: SystemJobORM) -> None: - """水体检测 job handler（Otsu + DEM + 形态学 + 连通分量）。""" - from .water_detect_service import run_water_detection +async def _handle_flood_detection(job: SystemJobORM) -> None: + """Flood-analysis detection job: pure Python GeoTIFF change classification.""" + from .flood_detection_service import run_geotiff_flood_detection payload = job.payload or {} detection_id = payload.get("detection_id") if not detection_id: - raise ValueError("WATER_DETECT job 缺少 detection_id") + raise ValueError("FLOOD_DETECTION job requires detection_id") + refine = bool(payload.get("refine", False)) + + await task_service.start_task(job.task_id, message="Reading flood-detection scene pair...") + + async with AsyncSessionLocal() as db: + det = await db.get(FloodDetectionORM, int(detection_id)) + if not det: + raise ValueError(f"FloodDetectionORM id={detection_id} does not exist") + pre_scene = await db.get(SARSceneGeoORM, det.pre_scene_id) + post_scene = await db.get(SARSceneGeoORM, det.post_scene_id) + if not pre_scene or not post_scene: + raise ValueError("Pre/post scene records do not exist") + if pre_scene.status != "DONE" or post_scene.status != "DONE": + raise ValueError("Pre/post scenes are not DONE") + pre_tif = pre_scene.analysis_tif_path + post_tif = post_scene.analysis_tif_path + if not pre_tif or not post_tif: + raise ValueError("Flood detection requires analysis-ready GeoTIFF paths for both scenes") + + output_dir = os.path.join(settings.WATER_RESULTS_DIR, f"flood_{detection_id}") + os.makedirs(output_dir, exist_ok=True) + + await task_service.update_task(job.task_id, progress=10, message="Running GeoTIFF flood classification...") + + def _run() -> Dict[str, Any]: + return run_geotiff_flood_detection( + pre_tif_path=pre_tif, + post_tif_path=post_tif, + output_dir=output_dir, + job_id=job.job_id, + refine=refine, + ) + + try: + result = await asyncio.to_thread(_run) + except Exception as exc: + async with AsyncSessionLocal() as db: + det = await db.get(FloodDetectionORM, int(detection_id)) + if det: + det.status = "FAILED" + det.error_msg = str(exc) + await db.commit() + raise + + async with AsyncSessionLocal() as db: + det = await db.get(FloodDetectionORM, int(detection_id)) + if det: + if result.get("ok"): + det.classified_path = result.get("classified_path") + det.flood_area_km2 = result.get("flood_area_km2") + det.stable_water_area_km2 = result.get("stable_water_area_km2") + det.output_dir = output_dir + det.status = "DONE" + det.error_msg = None + else: + det.status = "FAILED" + det.error_msg = result.get("error", "Unknown error") + await db.commit() + + if not result.get("ok"): + raise RuntimeError(f"Flood detection failed: {result.get('error')}") + + await task_service.update_task( + job.task_id, + status="COMPLETED", + progress=100, + message=( + f"GeoTIFF flood detection completed: flood_area={result.get('flood_area_km2')} km2, " + f"stable_water={result.get('stable_water_area_km2')} km2" + ), + ) + + +async def _handle_water_detect(job: SystemJobORM) -> None: + """水体检测 job handler（Otsu + DEM + 形态学 + 连通分量）。""" + from .water_extraction_service import run_otsu_water_extraction + + payload = job.payload or {} + extraction_id = payload.get("extraction_id") + detection_id = payload.get("detection_id") + record_id = extraction_id or detection_id + if not record_id: + raise ValueError("WATER_DETECT job 缺少 extraction_id/detection_id") + use_extraction_table = extraction_id is not None await task_service.start_task(job.task_id, message="读取检测任务信息...") async with AsyncSessionLocal() as db: - det = await db.get(WaterDetectionORM, int(detection_id)) + det = await db.get(WaterExtractionORM if use_extraction_table else WaterDetectionORM, int(record_id)) if not det: - raise ValueError(f"WaterDetectionORM id={detection_id} 不存在") + model_name = "WaterExtractionORM" if use_extraction_table else "WaterDetectionORM" + raise ValueError(f"{model_name} id={record_id} 不存在") input_path = det.input_path det.status = "RUNNING" + if use_extraction_table and hasattr(det, "task_id"): + det.task_id = job.task_id + if not use_extraction_table: + mirror = await db.get(WaterExtractionORM, int(record_id)) + if mirror: + mirror.status = "RUNNING" + mirror.task_id = job.task_id await db.commit() if not input_path: raise ValueError("水体检测缺少输入路径 input_path") - output_dir = os.path.join(os.path.dirname(input_path), f"water_detect_{detection_id}") + output_name = f"water_extraction_{record_id}" if use_extraction_table else f"water_detect_{record_id}" + output_dir = os.path.join(os.path.dirname(input_path), output_name) os.makedirs(output_dir, exist_ok=True) await task_service.update_task(job.task_id, progress=10, message="启动水体检测算法...") def _run() -> Dict[str, Any]: - return run_water_detection( - geo_tiff_path=input_path, + return run_otsu_water_extraction( + input_path=input_path, output_dir=output_dir, job_id=job.job_id, ) @@ -3266,26 +3465,56 @@ async def _handle_water_detect(job: SystemJobORM) -> None: result = await asyncio.to_thread(_run) except Exception as exc: async with AsyncSessionLocal() as db: - det = await db.get(WaterDetectionORM, int(detection_id)) + det = await db.get(WaterExtractionORM if use_extraction_table else WaterDetectionORM, int(record_id)) if det: det.status = "FAILED" det.error_msg = str(exc) - await db.commit() + if not use_extraction_table: + mirror = await db.get(WaterExtractionORM, int(record_id)) + if mirror: + mirror.status = "FAILED" + mirror.error_msg = str(exc) + mirror.task_id = job.task_id + await db.commit() raise async with AsyncSessionLocal() as db: - det = await db.get(WaterDetectionORM, int(detection_id)) + det = await db.get(WaterExtractionORM if use_extraction_table else WaterDetectionORM, int(record_id)) if det: if result.get("ok"): det.output_path = result.get("output_path") det.water_area_km2 = result.get("water_area_km2") det.water_pixel_count = result.get("water_pixel_count") - det.otsu_threshold_db = result.get("otsu_threshold_db") + if use_extraction_table: + det.processor = result.get("processor") or det.processor or "otsu" + det.threshold_value = result.get("threshold_value") + det.metadata_json = { + "legacy_otsu_threshold_db": result.get("otsu_threshold_db"), + "job_id": job.job_id, + } + else: + det.otsu_threshold_db = result.get("otsu_threshold_db") det.status = "DONE" det.error_msg = None else: det.status = "FAILED" det.error_msg = result.get("error", "Unknown error") + if not use_extraction_table: + mirror = await db.get(WaterExtractionORM, int(record_id)) + if mirror: + mirror.output_path = det.output_path + mirror.water_area_km2 = det.water_area_km2 + mirror.water_pixel_count = det.water_pixel_count + mirror.threshold_value = result.get("threshold_value") or result.get("otsu_threshold_db") + mirror.processor = result.get("processor") or mirror.processor or "otsu" + mirror.status = det.status + mirror.error_msg = det.error_msg + mirror.task_id = job.task_id + mirror.metadata_json = { + "legacy_otsu_threshold_db": result.get("otsu_threshold_db"), + "legacy_detection_id": int(record_id), + "job_id": job.job_id, + } await db.commit() if not result.get("ok"): @@ -3376,9 +3605,70 @@ async def _handle_gf3_process(job: SystemJobORM) -> None: ) +async def _handle_gf3_unpack(job: SystemJobORM) -> None: + """GF3 archive inbox -> persistent L1A source pool.""" + from .gf3_unpack_service import run_gf3_archive_unpack + + if not job.task_id: + raise ValueError("GF3_UNPACK requires task_id for progress tracking.") + + payload = job.payload or {} + await task_service.start_task(job.task_id, message="扫描 GF3 压缩包来源目录...") + + loop = asyncio.get_running_loop() + + def _submit(coro): + try: + future = asyncio.run_coroutine_threadsafe(coro, loop) + except RuntimeError: + return + + def _swallow_errors(fut): + try: + fut.result() + except Exception as exc: + logger.warning("[GF3 Unpack] task callback failed: %s", exc) + + future.add_done_callback(_swallow_errors) + + def _log_cb(level: str, message: str) -> None: + _submit(task_service.add_log(job.task_id, level, message)) + + def _progress_cb(progress: int, message: str) -> None: + _submit(task_service.update_task(job.task_id, progress=progress, message=message)) + + try: + result = await asyncio.to_thread( + run_gf3_archive_unpack, + source_dirs=payload.get("source_dirs"), + target_dirs=payload.get("target_dirs"), + archive_exts=payload.get("archive_exts"), + max_files_per_run=payload.get("max_files_per_run"), + delete_archive=payload.get("delete_archive") if "delete_archive" in payload else None, + min_disk_space_gb=payload.get("min_disk_space_gb"), + tmp_suffix=payload.get("tmp_suffix"), + log_callback=_log_cb, + progress_callback=_progress_cb, + ) + except Exception as exc: + await task_service.update_task(job.task_id, status="FAILED", progress=100, message=f"GF3 解包失败: {exc}") + raise + + message = ( + f"GF3 解包完成: 成功 {int(result.get('processed') or 0)}, " + f"跳过 {int(result.get('skipped') or 0)}, " + f"失败 {int(result.get('failed') or 0)}" + ) + remaining = int(result.get("remaining") or 0) + if remaining > 0: + message += f", 剩余 {remaining}" + await task_service.update_task(job.task_id, status="COMPLETED", progress=100, message=message) + + async def _handle_gf3_batch_process(job: SystemJobORM) -> None: """批量 GF3 L1A→L2：扫描来源目录，逐个处理并自动入库到 radar_data。""" from .gf3_service import run_gf3_l1a_to_l2, register_l2_to_radar_data + from .sar_analysis_ready_service import standardize_gf3_l2_for_radar payload = job.payload or {} source_dirs = payload.get("source_dirs") or [] @@ -3458,12 +3748,18 @@ async def _handle_gf3_batch_process(job: SystemJobORM) -> None: # Auto-register to radar_data try: async with AsyncSessionLocal() as db: - await register_l2_to_radar_data( + radar_id = await register_l2_to_radar_data( l2_dir=result.get("output_dir", output_dir), input_dir_name=dir_name, polarizations=result.get("polarizations", []), db=db, ) + if radar_id: + await standardize_gf3_l2_for_radar( + db=db, + radar_id=int(radar_id), + l2_path=result.get("output_dir", output_dir), + ) except Exception as reg_err: logger.warning("[GF3 Batch] Auto-register failed for %s: %s", dir_name, reg_err) else: @@ -3888,6 +4184,85 @@ async def _handle_rebuild_psinsar_catalog(job: SystemJobORM) -> None: ) +async def _handle_sbas_coregistration(job: SystemJobORM) -> None: + if not job.task_id: + raise ValueError("SBAS_COREGISTRATION requires task_id for progress tracking.") + payload = job.payload or {} + run_id = str(payload.get("run_id") or "").strip() + if not run_id: + raise ValueError("SBAS_COREGISTRATION requires run_id payload.") + + rlks = _normalize_positive_int(payload.get("rlks")) or 8 + azlks = _normalize_positive_int(payload.get("azlks")) or 8 + timeout_seconds = _normalize_positive_int(payload.get("timeout_seconds")) or 43200 + + await task_service.start_task(job.task_id, message="正在执行 SBAS-InSAR Gamma 共参考配准...") + await task_service.update_task( + job.task_id, + progress=5, + message=f"准备运行 Gamma SLC_coreg.py: run_id={run_id}", + ) + await task_service.add_log( + job.task_id, + "INFO", + f"SBAS coregistration queued: run_id={run_id}, rlks={rlks}, azlks={azlks}, timeout={timeout_seconds}s", + ) + + from .sbas_insar_production_service import sbas_insar_production_service + + async def _task_keepalive() -> None: + progress = 12 + while True: + await asyncio.sleep(60) + progress = min(88, progress + 2) + await task_service.update_task( + job.task_id, + progress=progress, + message=f"Gamma 共参考配准仍在运行: run_id={run_id}", + ) + + runner_task = asyncio.create_task( + asyncio.to_thread( + sbas_insar_production_service.execute_coregistration, + run_id, + rlks=rlks, + azlks=azlks, + timeout_seconds=timeout_seconds, + ) + ) + keepalive_task = asyncio.create_task(_task_keepalive()) + try: + result = await runner_task + finally: + keepalive_task.cancel() + try: + await keepalive_task + except asyncio.CancelledError: + pass + + manifest = result.get("manifest") or {} + run = result.get("run") or {} + summary = (manifest.get("coregistration") or {}).get("summary") or {} + status = str(run.get("status") or manifest.get("status") or "").strip() + if status != "COREGISTRATION_READY": + raise RuntimeError( + "SBAS coregistration failed: " + f"status={status or 'UNKNOWN'}, " + f"missing_dates={summary.get('missing_dates') or []}, " + f"missing_tabs={summary.get('missing_tabs') or []}" + ) + + await task_service.update_task( + job.task_id, + status="COMPLETED", + progress=100, + message=( + "SBAS-InSAR 共参考配准完成: " + f"{summary.get('ready_secondary_count', 0)}/{summary.get('expected_secondary_count', 0)} secondary scenes ready" + ), + ) + + _HANDLERS = { JOB_TYPE_SCAN_DATA: _handle_scan_data, JOB_TYPE_SCAN_ASSET_INVENTORY: _handle_scan_asset_inventory, @@ -3918,10 +4293,14 @@ _HANDLERS = { JOB_TYPE_ISCE2_RUN: _handle_isce2_run, JOB_TYPE_PYINT_RUN: _handle_pyint_run, JOB_TYPE_WATER_GEOCODE: _handle_water_geocode, + JOB_TYPE_SAR_SCENE_PREPROCESS: _handle_sar_scene_preprocess, JOB_TYPE_WATER_FLOOD: _handle_water_flood, + JOB_TYPE_FLOOD_DETECTION: _handle_flood_detection, JOB_TYPE_WATER_DETECT: _handle_water_detect, JOB_TYPE_GF3_PROCESS: _handle_gf3_process, + JOB_TYPE_GF3_UNPACK: _handle_gf3_unpack, JOB_TYPE_GF3_BATCH_PROCESS: _handle_gf3_batch_process, + JOB_TYPE_SBAS_COREGISTRATION: _handle_sbas_coregistration, } diff --git a/backend/app/services/lt_gamma_scene_service.py b/backend/app/services/lt_gamma_scene_service.py new file mode 100644 index 0000000..dca94b1 --- /dev/null +++ b/backend/app/services/lt_gamma_scene_service.py @@ -0,0 +1,158 @@ +"""LT single-scene Gamma preprocessing service.""" +from __future__ import annotations + +import json +import os +import re +from pathlib import Path +from typing import Any + +from ..config import settings +from ..models import RadarDataORM, SARSceneGeoORM +from .pyint_service import ( + DEFAULT_AZIMUTH_LOOKS, + DEFAULT_RANGE_LOOKS, + quote_shell, + resolve_gamma_env_script, + to_wsl_path, +) +from .wsl_service import run_wsl_exec + + +def _safe_token(value: Any, *, default: str = "scene") -> str: + text = str(value or "").strip() + if not text: + text = default + text = re.sub(r"[^0-9A-Za-z._-]+", "_", text).strip("._-") + return text or default + + +def _scene_date(radar: RadarDataORM) -> str: + for value in (radar.imaging_date, radar.file_path, radar.unique_id): + text = str(value or "") + match = re.search(r"(20\d{6})", re.sub(r"\D", "", text)) + if match: + return match.group(1) + match = re.search(r"(20\d{6})", text) + if match: + return match.group(1) + raise ValueError(f"Cannot infer LT scene date for radar_data id={radar.id}") + + +def _prepared_dem_path() -> str: + if str(settings.PYINT_DEM_MODE or "").strip().lower() == "prepared_file": + return ( + settings.PYINT_PREPARED_DEM_PATH + or settings.ISCE2_DEM_PATH + or settings.IDL_DINSAR_DEM_BASE_FILE + or "" + ) + return "" + + +def _build_shell_command(parts: list[str]) -> str: + return " ".join(quote_shell(part) for part in parts) + + +def _runner_script() -> Path: + return Path(settings.PROJECT_ROOT) / "backend" / "app" / "pyint_pipeline" / "run_gamma_scene_preprocess.py" + + +def run_lt_gamma_scene_preprocess( + *, + radar: RadarDataORM, + scene: SARSceneGeoORM, + job_id: str | None = None, +) -> dict[str, Any]: + if not settings.PYINT_ENABLED: + raise RuntimeError("PYINT_ENABLED=false; LT Gamma scene preprocessing is disabled") + if not radar.file_path: + raise ValueError(f"RadarDataORM id={radar.id} has no file_path") + + date = _scene_date(radar) + token = _safe_token(radar.unique_id or Path(str(radar.file_path)).stem or f"radar_{radar.id}") + run_name = _safe_token(f"lt_{date}_{token}_scene_{scene.id}_{job_id or 'manual'}") + work_dir = Path(settings.SAR_ANALYSIS_WORK_ROOT) / "lt_gamma" / run_name + output_dir = work_dir / "output" + work_dir.mkdir(parents=True, exist_ok=True) + output_dir.mkdir(parents=True, exist_ok=True) + + pyint_home = settings.PYINT_HOME + if not pyint_home: + raise RuntimeError("PYINT_HOME is not configured") + pyint_python = settings.PYINT_WSL_PYTHON or settings.WSL_SHARED_PYTHON + if not pyint_python: + raise RuntimeError("PYINT_WSL_PYTHON is not configured") + + runner = _runner_script() + if not runner.is_file(): + raise FileNotFoundError(f"Gamma scene runner not found: {runner}") + + args = [ + pyint_python, + to_wsl_path(str(runner)), + "--source-path", + to_wsl_path(str(radar.file_path)), + "--output-dir", + to_wsl_path(str(output_dir)), + "--work-dir", + to_wsl_path(str(work_dir)), + "--pyint-home", + to_wsl_path(str(pyint_home)), + "--dem-root", + to_wsl_path(str(settings.PYINT_DEM_ROOT)), + "--prepared-dem-path", + to_wsl_path(_prepared_dem_path()), + "--project-name", + run_name, + "--date", + date, + "--satellite-family", + "LT1", + "--range-looks", + str(DEFAULT_RANGE_LOOKS), + "--azimuth-looks", + str(DEFAULT_AZIMUTH_LOOKS), + "--geo-interp", + str(settings.PYINT_GEO_INTERP or "1"), + "--nodata-value", + str(float(settings.SAR_ANALYSIS_NODATA_VALUE)), + "--to-db", + ] + + gamma_env_script = resolve_gamma_env_script(settings.PYINT_GAMMA_ENV_SCRIPT) + prefix = "" + if gamma_env_script: + prefix = f". {quote_shell(to_wsl_path(gamma_env_script))} >/dev/null 2>&1 || exit 1; " + command = prefix + f"export PYTHONPATH={quote_shell(to_wsl_path(str(pyint_home)))}:$PYTHONPATH; " + _build_shell_command(args) + + rc, stdout, stderr = run_wsl_exec( + ["bash", "-lc", command], + distro=settings.PYINT_WSL_DISTRO or settings.WSL_DISTRO, + timeout=int(settings.PYINT_DEFAULT_TIMEOUT_SECONDS or 43200), + ) + if rc != 0: + detail = (stderr or stdout or "").strip() + raise RuntimeError(f"LT Gamma scene preprocessing failed rc={rc}: {detail}") + + manifest_path = output_dir / "manifest.json" + if not manifest_path.is_file(): + raise RuntimeError(f"LT Gamma scene preprocessing produced no manifest: {manifest_path}") + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except Exception as exc: + raise RuntimeError(f"Cannot read LT Gamma manifest: {manifest_path}: {exc}") from exc + + analysis_tif_path = manifest.get("analysis_tif_path") + if analysis_tif_path and str(analysis_tif_path).startswith("/mnt/"): + # The service registers the Windows-side path below. + manifest["analysis_tif_path_wsl"] = analysis_tif_path + manifest["analysis_tif_path"] = str(output_dir / "analysis_ready.tif") + manifest["service"] = { + "work_dir": str(work_dir), + "output_dir": str(output_dir), + "job_id": job_id, + "stdout": stdout[-4000:] if stdout else "", + "stderr": stderr[-4000:] if stderr else "", + } + return manifest diff --git a/backend/app/services/root_registry_service.py b/backend/app/services/root_registry_service.py index 03d8716..7d65d34 100644 --- a/backend/app/services/root_registry_service.py +++ b/backend/app/services/root_registry_service.py @@ -235,6 +235,15 @@ def _build_root_specs_from_settings() -> List[RootSpec]: scan_mode="directory_walk", ) ) + specs.extend( + _iter_multi_root_specs( + env_var="GF3_ARCHIVE_SOURCE_DIRS", + paths=split_env_paths(settings.GF3_ARCHIVE_SOURCE_DIRS), + root_role="source_pool_gf3_archive", + display_prefix="GF3 Archive Pool", + scan_mode="archive_walk", + ) + ) specs.extend( _iter_multi_root_specs( env_var="GF3_SOURCE_DIRS", @@ -253,6 +262,24 @@ def _build_root_specs_from_settings() -> List[RootSpec]: scan_mode="scene_directory", ) ) + specs.extend( + _iter_single_root_specs( + env_var="SAR_ANALYSIS_READY_ROOT", + path=settings.SAR_ANALYSIS_READY_ROOT, + root_role="sar_analysis_ready", + display_name="SAR Analysis-ready GeoTIFF Root", + scan_mode="scene_directory", + ) + ) + specs.extend( + _iter_single_root_specs( + env_var="SAR_ANALYSIS_WORK_ROOT", + path=settings.SAR_ANALYSIS_WORK_ROOT, + root_role="sar_analysis_work", + display_name="SAR Analysis Work Root", + scan_mode="directory_walk", + ) + ) orbit_source_paths = split_env_paths(settings.ORBIT_SOURCE_DIRS) if not orbit_source_paths: orbit_source_paths = split_env_paths(settings.MONITOR_ORBIT_DIR) diff --git a/backend/app/services/sar_analysis_ready_service.py b/backend/app/services/sar_analysis_ready_service.py new file mode 100644 index 0000000..2889667 --- /dev/null +++ b/backend/app/services/sar_analysis_ready_service.py @@ -0,0 +1,360 @@ +"""Analysis-ready SAR GeoTIFF registration for flood/water algorithms. + +This service owns the common contract between satellite-specific preprocessing +and downstream flood/water algorithms: one geocoded, single-band GeoTIFF plus +sidecar metadata under SAR_ANALYSIS_READY_ROOT. +""" +from __future__ import annotations + +import json +import math +import os +import re +import shutil +from pathlib import Path +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from ..config import settings +from ..models import RadarDataORM, SARSceneGeoORM +from ..utils import normalize_satellite_family + +_SAFE_TEXT_RE = re.compile(r"[^0-9A-Za-z._-]+") +_POLARIZATION_PRIORITY = ("HH", "VV", "HV", "VH") + + +def _safe_slug(value: Any, *, default: str = "unknown") -> str: + text = str(value or "").strip() + if not text: + text = default + text = _SAFE_TEXT_RE.sub("_", text).strip("._-") + return text or default + + +def _scene_family(radar: RadarDataORM | None) -> str: + family = normalize_satellite_family( + getattr(radar, "satellite_family", None) or getattr(radar, "satellite", None) + ) + return _safe_slug(family or "SAR").upper() + + +def _scene_date(radar: RadarDataORM | None) -> str: + text = str(getattr(radar, "imaging_date", None) or "").strip() + match = re.search(r"(20\d{6})", re.sub(r"\D", "", text)) + if match: + return match.group(1) + return "unknown_date" + + +def _scene_token( + *, + radar: RadarDataORM | None, + scene: SARSceneGeoORM, + polarization: str | None = None, +) -> str: + unique = getattr(radar, "unique_id", None) or f"radar_{getattr(radar, 'id', scene.radar_data_id)}" + parts = [_scene_date(radar), _safe_slug(unique), f"scene_{scene.id}"] + if polarization: + parts.append(_safe_slug(polarization).upper()) + return "_".join(parts) + + +def scene_analysis_dir( + *, + radar: RadarDataORM | None, + scene: SARSceneGeoORM, + engine: str, + profile: str, + polarization: str | None = None, +) -> Path: + return ( + Path(settings.SAR_ANALYSIS_READY_ROOT) + / _scene_family(radar) + / _safe_slug(engine) + / _safe_slug(profile) + / _scene_date(radar) + / _scene_token(radar=radar, scene=scene, polarization=polarization) + ) + + +def _write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as stream: + json.dump(payload, stream, ensure_ascii=False, indent=2, default=str) + + +def _link_or_copy(source: Path, target: Path) -> str: + target.parent.mkdir(parents=True, exist_ok=True) + if source.resolve() == target.resolve(): + return "same_path" + if target.exists(): + target.unlink() + try: + os.link(source, target) + return "hardlink" + except OSError: + shutil.copy2(source, target) + return "copy" + + +def _choose_gf3_l2_tif(l2_dir: str, polarization: str | None = None) -> Path: + root = Path(os.path.normpath(str(l2_dir or "").strip())) + if root.is_file(): + return root + if not root.is_dir(): + raise FileNotFoundError(f"GF3 L2 directory does not exist: {l2_dir}") + + candidates = sorted( + path + for path in root.rglob("*") + if path.is_file() + and path.suffix.lower() in {".tif", ".tiff"} + and "L2" in path.name.upper() + ) + if not candidates: + raise FileNotFoundError(f"No GF3 L2 GeoTIFF found in: {l2_dir}") + + requested = str(polarization or "").strip().upper() + if requested: + for path in candidates: + if requested in path.name.upper(): + return path + + for pol in _POLARIZATION_PRIORITY: + for path in candidates: + if pol in path.name.upper(): + return path + return candidates[0] + + +def _infer_polarization_from_path(path: Path) -> str | None: + upper_name = path.name.upper() + for pol in _POLARIZATION_PRIORITY: + if pol in upper_name: + return pol + return None + + +def _raster_quality(path: Path) -> dict[str, Any]: + try: + import numpy as np + import rasterio + except Exception as exc: + return {"ok": False, "warning": f"rasterio unavailable: {exc}"} + + with rasterio.open(path) as src: + if src.height > 2048 or src.width > 2048: + scale = min(1024 / src.width, 1024 / src.height) + out_width = max(1, int(src.width * scale)) + out_height = max(1, int(src.height * scale)) + sampled = src.read(1, out_shape=(out_height, out_width), masked=True) + else: + sampled = src.read(1, masked=True) + + valid = sampled.compressed() if hasattr(sampled, "compressed") else sampled[np.isfinite(sampled)] + bounds = src.bounds + transform = src.transform + quality: dict[str, Any] = { + "ok": True, + "driver": src.driver, + "width": src.width, + "height": src.height, + "count": src.count, + "dtype": str(src.dtypes[0]) if src.dtypes else None, + "crs": src.crs.to_string() if src.crs else None, + "bounds": { + "left": bounds.left, + "bottom": bounds.bottom, + "right": bounds.right, + "top": bounds.top, + }, + "transform": list(transform)[:6], + "nodata": src.nodata, + "valid_sample_count": int(valid.size), + "valid_sample_percent": float(valid.size / sampled.size) if sampled.size else 0.0, + } + if valid.size: + quality.update( + { + "sample_min": float(np.nanmin(valid)), + "sample_max": float(np.nanmax(valid)), + "sample_mean": float(np.nanmean(valid)), + "sample_p02": float(np.nanpercentile(valid, 2)), + "sample_p98": float(np.nanpercentile(valid, 98)), + } + ) + return quality + + +def _pixel_size_m_from_quality(quality: dict[str, Any]) -> float | None: + try: + transform = quality.get("transform") or [] + xres = abs(float(transform[0])) + yres = abs(float(transform[4])) + crs = str(quality.get("crs") or "").upper() + if not xres or not yres: + return None + if crs and "4326" not in crs: + return round((xres + yres) / 2.0, 3) + bounds = quality.get("bounds") or {} + lat = (float(bounds.get("bottom", 0.0)) + float(bounds.get("top", 0.0))) / 2.0 + meters_per_degree_lon = 111320.0 * max(0.01, math.cos(math.radians(lat))) + x_m = xres * meters_per_degree_lon + y_m = yres * 110540.0 + return round((x_m + y_m) / 2.0, 3) + except Exception: + return None + + +def _build_preview_png(source: Path, target: Path) -> str | None: + try: + import numpy as np + import rasterio + from PIL import Image + except Exception: + return None + + target.parent.mkdir(parents=True, exist_ok=True) + with rasterio.open(source) as src: + if src.height > 1600 or src.width > 1600: + scale = min(1600 / src.width, 1600 / src.height) + out_width = max(1, int(src.width * scale)) + out_height = max(1, int(src.height * scale)) + band = src.read(1, out_shape=(out_height, out_width), masked=True) + else: + band = src.read(1, masked=True) + data = band.filled(np.nan).astype("float32") + + valid = data[np.isfinite(data)] + if valid.size: + p2, p98 = np.nanpercentile(valid, [2, 98]) + normalized = np.clip((data - p2) / max(p98 - p2, 1e-6), 0, 1) + gray = (normalized * 255).astype("uint8") + else: + gray = np.zeros(data.shape, dtype="uint8") + alpha = np.where(np.isfinite(data), 255, 0).astype("uint8") + rgba = np.stack([gray, gray, gray, alpha], axis=-1) + Image.fromarray(rgba, "RGBA").save(target) + return str(target) + + +async def _get_or_create_scene(db: AsyncSession, radar_id: int) -> SARSceneGeoORM: + result = await db.execute(select(SARSceneGeoORM).where(SARSceneGeoORM.radar_data_id == radar_id)) + scene = result.scalar_one_or_none() + if scene: + return scene + scene = SARSceneGeoORM(radar_data_id=radar_id, status="PENDING") + db.add(scene) + await db.flush() + return scene + + +async def register_analysis_ready_tif( + *, + db: AsyncSession, + scene: SARSceneGeoORM, + radar: RadarDataORM | None, + source_tif_path: str, + engine: str, + profile: str, + backscatter_unit: str, + polarization: str | None = None, + metadata: dict[str, Any] | None = None, + copy_mode: str = "link_or_copy", +) -> dict[str, Any]: + source = Path(os.path.normpath(str(source_tif_path or "").strip())) + if not source.is_file(): + raise FileNotFoundError(f"Analysis-ready source GeoTIFF does not exist: {source}") + + out_dir = scene_analysis_dir( + radar=radar, + scene=scene, + engine=engine, + profile=profile, + polarization=polarization, + ) + target_tif = out_dir / "analysis_ready.tif" + transfer = "none" + if copy_mode == "reference": + target_tif = source + else: + transfer = _link_or_copy(source, target_tif) + + quality = _raster_quality(target_tif) + preview_path = _build_preview_png(target_tif, out_dir / "preview.png") + manifest = { + "scene_id": scene.id, + "radar_data_id": scene.radar_data_id, + "source_tif_path": str(source), + "analysis_tif_path": str(target_tif), + "analysis_dir": str(out_dir), + "analysis_preview_path": preview_path, + "engine": engine, + "profile": profile, + "backscatter_unit": backscatter_unit, + "polarization": polarization, + "transfer": transfer, + "metadata": metadata or {}, + "quality": quality, + } + _write_json(out_dir / "manifest.json", manifest) + _write_json(out_dir / "quality.json", quality) + + scene.geo_path = str(target_tif) + scene.analysis_tif_path = str(target_tif) + scene.analysis_dir = str(out_dir) + scene.analysis_preview_path = preview_path + scene.analysis_engine = engine + scene.analysis_profile = profile + scene.analysis_backscatter_unit = backscatter_unit + nodata_value = quality.get("nodata") + scene.analysis_nodata_value = ( + float(nodata_value) + if nodata_value is not None + else float(settings.SAR_ANALYSIS_NODATA_VALUE) + ) + scene.analysis_metadata_json = {**(metadata or {}), "manifest_path": str(out_dir / "manifest.json")} + scene.analysis_quality_json = quality + scene.pixel_size_m = _pixel_size_m_from_quality(quality) or scene.pixel_size_m + scene.status = "DONE" + scene.error_msg = None + + return manifest + + +async def standardize_gf3_l2_for_radar( + *, + db: AsyncSession, + radar_id: int, + l2_path: str | None = None, + polarization: str | None = None, +) -> dict[str, Any]: + radar = await db.get(RadarDataORM, int(radar_id)) + if not radar: + raise ValueError(f"RadarDataORM id={radar_id} does not exist") + + scene = await _get_or_create_scene(db, int(radar_id)) + source_root = l2_path or radar.file_path + selected_tif = _choose_gf3_l2_tif(source_root, polarization=polarization or radar.polarization) + selected_pol = polarization or _infer_polarization_from_path(selected_tif) + + manifest = await register_analysis_ready_tif( + db=db, + scene=scene, + radar=radar, + source_tif_path=str(selected_tif), + engine="gf3_gdal", + profile="gf3_l1a_l2_rpc", + backscatter_unit="sigma0_db", + polarization=selected_pol, + metadata={ + "source": "GF3 L2", + "source_l2_path": str(selected_tif), + "source_l2_dir": str(Path(source_root).resolve()) if source_root else None, + "available_polarization": radar.polarization, + }, + ) + await db.commit() + return manifest diff --git a/backend/app/services/sbas_insar_production_service.py b/backend/app/services/sbas_insar_production_service.py new file mode 100644 index 0000000..af99605 --- /dev/null +++ b/backend/app/services/sbas_insar_production_service.py @@ -0,0 +1,2208 @@ +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import subprocess +from datetime import datetime +from pathlib import Path +from typing import Any +from xml.etree import ElementTree as ET + +from ..config import settings + + +PRODUCT_DEFINITIONS = ( + { + "key": "los_rate_toward_mm_per_year_geo_preview_png", + "label": "LOS velocity geocoded preview, toward radar positive", + "role": "primary_geocoded_preview", + "relative_path": "publish/geotiff/los_rate_toward_mm_per_year.geo_preview.png", + }, + { + "key": "los_rate_toward_mm_per_year_bmp", + "label": "LOS velocity RDC processing preview, toward radar positive", + "role": "rdc_processing_preview", + "relative_path": "publish/geotiff/los_rate_toward_mm_per_year.bmp", + }, + { + "key": "los_rate_toward_mm_per_year_tif", + "label": "LOS velocity GeoTIFF, toward radar positive", + "role": "primary_geotiff", + "relative_path": "publish/geotiff/los_rate_toward_mm_per_year.tif", + }, + { + "key": "los_rate_away_mm_per_year_bmp", + "label": "LOS velocity RDC processing preview, away from radar positive", + "role": "rdc_processing_preview", + "relative_path": "publish/geotiff/los_rate_away_mm_per_year.bmp", + }, + { + "key": "los_rate_away_mm_per_year_tif", + "label": "LOS velocity GeoTIFF, away from radar positive", + "role": "alternate_geotiff", + "relative_path": "publish/geotiff/los_rate_away_mm_per_year.tif", + }, + { + "key": "los_sigma_mm_per_year_geo_preview_png", + "label": "LOS velocity sigma geocoded preview", + "role": "quality_geocoded_preview", + "relative_path": "publish/geotiff/los_sigma_mm_per_year.geo_preview.png", + }, + { + "key": "los_sigma_mm_per_year_bmp", + "label": "LOS velocity sigma RDC processing preview", + "role": "rdc_processing_preview", + "relative_path": "publish/geotiff/los_sigma_mm_per_year.bmp", + }, + { + "key": "los_sigma_mm_per_year_tif", + "label": "LOS velocity sigma GeoTIFF", + "role": "quality_geotiff", + "relative_path": "publish/geotiff/los_sigma_mm_per_year.tif", + }, + { + "key": "ts_rate_rad_per_year_tif", + "label": "Gamma ts_rate phase-rate GeoTIFF", + "role": "gamma_phase_rate", + "relative_path": "publish/geotiff/ts_rate_rad_per_year.tif", + }, + { + "key": "sigma_rate_rad_per_year_tif", + "label": "Gamma sigma_rate GeoTIFF", + "role": "gamma_sigma_rate", + "relative_path": "publish/geotiff/sigma_rate_rad_per_year.tif", + }, + { + "key": "trial_summary_json", + "label": "Trial summary JSON", + "role": "summary", + "relative_path": "publish/trial_summary.json", + }, +) + + +MONITOR_ARTIFACT_SUFFIXES = ( + ("timeseries_png", "Monitoring point curve", ".png"), + ("timeseries_csv", "Monitoring point values", ".csv"), + ("metadata_json", "Monitoring point metadata", ".json"), +) + +GAMMA_STAGE_PLAN = ( + { + "stage_id": "prepare_slc", + "label": "Prepare LT1 SLCs", + "gamma_tools": ["par_LT1_SLC", "LT1_precision_orbit.py", "multi_look"], + "status": "PLANNED", + }, + { + "stage_id": "baseline_audit", + "label": "Gamma baseline audit and itab approval", + "gamma_tools": ["base_calc"], + "status": "PENDING_REQUIRED_AUDIT", + }, + { + "stage_id": "coregistration", + "label": "Stack co-registration", + "gamma_tools": ["SLC_coreg.py"], + "status": "PLANNED_AFTER_BASELINE_AUDIT", + }, + { + "stage_id": "rdc_dem", + "label": "RDC DEM and lookup table", + "gamma_tools": ["gc_map1", "geocode", "gc_map_fine"], + "status": "PLANNED_AFTER_BASELINE_AUDIT", + }, + { + "stage_id": "interferograms", + "label": "Differential interferograms", + "gamma_tools": ["phase_sim_orb", "SLC_diff_intf", "adf", "mcf"], + "status": "PLANNED_AFTER_BASELINE_AUDIT", + }, + { + "stage_id": "ipta_timeseries", + "label": "IPTA SBAS time-series inversion", + "gamma_tools": ["mb", "ts_rate"], + "status": "PLANNED_AFTER_BASELINE_AUDIT", + }, + { + "stage_id": "publish_products", + "label": "Geocode and publish products", + "gamma_tools": ["geocode_back", "data2geotiff", "dispmap"], + "status": "PLANNED_AFTER_BASELINE_AUDIT", + }, + { + "stage_id": "monitor_points", + "label": "Monitoring-point time-series extraction", + "gamma_tools": [], + "status": "PLANNED_AFTER_PRODUCTS", + }, +) + +LT1_SCENE_RE = re.compile( + r"^(?PLT1[AB])_" + r"(?P[A-Z0-9]+)_" + r"(?P[A-Z0-9]+)_" + r"(?P[A-Z0-9]+)_" + r"(?P\d+)_" + r"E(?P-?\d+(?:\.\d+)?)_" + r"N(?P-?\d+(?:\.\d+)?)_" + r"(?P\d{8})_" + r"(?P[A-Z0-9]+)_" + r"(?P[A-Z0-9]+)_", + re.IGNORECASE, +) + + +class SbasInsarProductionService: + def __init__(self) -> None: + self.trial_root = Path(settings.BACKEND_DIR) / "runtime" / "gamma_ipta_trials" + self.production_root = Path(settings.BACKEND_DIR) / "runtime" / "sbas_insar_production" + + def get_capabilities(self) -> dict[str, Any]: + return { + "workflow_code": "sbas_insar", + "processor_code": "gamma_ipta_sbas", + "engine_code": "gamma", + "implementation_state": "baseline_audit_and_coregistration_queue", + "trial_root": str(self.trial_root), + "production_root": str(self.production_root), + "supported_sensors": ["LT1"], + "supported_products": [item["key"] for item in PRODUCT_DEFINITIONS], + "run_submission": { + "enabled": True, + "execution_enabled": True, + "status_after_submit": "PLANNED_GAMMA_BASELINE_AUDIT", + "description": "Creates reproducible filesystem manifests and can execute the Gamma SLC preparation plus base_calc baseline-audit stage.", + }, + "baseline_audit": { + "enabled": True, + "default_rlks": 8, + "default_azlks": 8, + "default_max_delta_n": 1, + "stage_status_after_success": "BASELINE_AUDIT_READY", + }, + "coregistration": { + "enabled": True, + "execution_enabled": True, + "execution_mode": "queued_background_task", + "job_type": "SBAS_COREGISTRATION", + "default_strategy": "common_reference_to_stack_reference_date", + "requires_status": "ITAB_APPROVED", + }, + "monitor_point_modes": ["auto_low_sigma_high_rate", "manual_lonlat"], + "default_los_convention": { + "key": "los_rate_toward_mm_per_year", + "description": "toward radar positive; away from radar negative", + "gamma_dispmap_equivalent": "sflg=0", + }, + "sign_conventions": [ + { + "key": "away_positive", + "formula": "phase_rate*wavelength/(4*pi)*1000", + "description": "away from radar positive; same sign as Gamma phase", + }, + { + "key": "toward_positive", + "formula": "-phase_rate*wavelength/(4*pi)*1000", + "description": "toward radar positive; Gamma dispmap default sflg=0", + }, + ], + "next_enabled_operation": "gamma_coregistration_background_job", + } + + def discover_stacks( + self, + *, + source_roots: list[str] | None = None, + orbit_roots: list[str] | None = None, + min_scenes: int = 3, + require_orbits: bool = True, + include_scenes: bool = False, + limit: int = 30, + platform: str | None = None, + relative_orbit: str | None = None, + orbit_direction: str | None = None, + ) -> dict[str, Any]: + source_paths = self._resolve_source_roots(source_roots) + orbit_paths = self._resolve_orbit_roots(orbit_roots) + scenes: list[dict[str, Any]] = [] + errors: list[dict[str, str]] = [] + + platform_filter = str(platform or "").strip().upper() + rel_filter = str(relative_orbit or "").strip() + direction_filter = str(orbit_direction or "").strip().upper() + + for root in source_paths: + try: + for scene_dir in self._iter_lt1_scene_dirs(root): + try: + scene = self._parse_lt1_scene(scene_dir, orbit_paths) + except Exception as exc: + errors.append({"scene_dir": str(scene_dir), "error": str(exc)}) + continue + if platform_filter and scene.get("satellite") != platform_filter: + continue + if rel_filter and str(scene.get("relative_orbit") or "") != rel_filter: + continue + if direction_filter and str(scene.get("orbit_direction") or "").upper() != direction_filter: + continue + scenes.append(scene) + except Exception as exc: + errors.append({"source_root": str(root), "error": str(exc)}) + + grouped: dict[str, list[dict[str, Any]]] = {} + for scene in scenes: + grouped.setdefault(self._stack_group_key(scene), []).append(scene) + + candidates = [ + self._build_stack_candidate(group_scenes, min_scenes=min_scenes, require_orbits=require_orbits) + for group_scenes in grouped.values() + ] + candidates.sort( + key=lambda item: ( + int(item.get("status") != "READY"), + -int(item.get("orbit_ready_scene_count") or 0), + -int(item.get("scene_count") or 0), + str(item.get("date_start") or ""), + ) + ) + if not include_scenes: + for candidate in candidates: + candidate.pop("scenes", None) + if limit > 0: + candidates = candidates[:limit] + + snapshot = { + "schema": "insar.sbas-stack-discovery/v1", + "generated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "source_roots": [str(path) for path in source_paths], + "orbit_roots": [str(path) for path in orbit_paths], + "min_scenes": min_scenes, + "require_orbits": require_orbits, + "scene_count": len(scenes), + "candidate_count": len(candidates), + "errors": errors[:50], + "items": candidates, + } + snapshot_path = self._write_runtime_json( + "discoveries", + f"discovery_{datetime.utcnow().strftime('%Y%m%dT%H%M%SZ')}.json", + snapshot, + ) + snapshot["snapshot_path"] = str(snapshot_path) + return snapshot + + def audit_stack( + self, + stack_id: str, + *, + source_roots: list[str] | None = None, + orbit_roots: list[str] | None = None, + min_scenes: int = 3, + require_orbits: bool = True, + ) -> dict[str, Any]: + discovery = self.discover_stacks( + source_roots=source_roots, + orbit_roots=orbit_roots, + min_scenes=min_scenes, + require_orbits=require_orbits, + include_scenes=True, + limit=0, + ) + candidate = next( + (item for item in discovery.get("items", []) if item.get("stack_id") == stack_id), + None, + ) + if not candidate: + raise FileNotFoundError(f"stack candidate not found: {stack_id}") + + usable_scenes = [ + scene for scene in candidate.get("scenes", []) + if (scene.get("has_orbit") or not require_orbits) + ] + usable_scenes.sort(key=lambda item: str(item.get("date") or "")) + pairs = self._build_adjacent_pairs(usable_scenes) + blockers: list[str] = [] + warnings: list[str] = [] + + if len(usable_scenes) < min_scenes: + blockers.append( + f"Only {len(usable_scenes)} usable scenes; minimum required is {min_scenes}." + ) + if require_orbits and candidate.get("missing_orbit_count"): + warnings.append( + f"{candidate.get('missing_orbit_count')} scenes are excluded because precise orbit TXT is missing." + ) + if len(pairs) < max(0, len(usable_scenes) - 1): + blockers.append("Adjacent pair network is not fully connected.") + for pair in pairs: + if int(pair.get("delta_days") or 0) > 180: + warnings.append( + f"Long temporal gap: {pair.get('master_date')} -> {pair.get('slave_date')} " + f"({pair.get('delta_days')} days)." + ) + + timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%S%fZ") + manifest = { + "schema": "insar.gamma-ipta-sbas-stack-manifest/v1", + "generated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "stack_id": stack_id, + "processor_code": "gamma_ipta_sbas", + "engine_code": "gamma", + "workflow": "Gamma DIFF + IPTA mb/ts_rate", + "status": "READY_FOR_GAMMA_BASELINE_AUDIT" if not blockers else "BLOCKED", + "require_orbits": require_orbits, + "min_scenes": min_scenes, + "stack": { + key: candidate.get(key) + for key in [ + "satellite", + "satellite_mode", + "receiving_station", + "relative_orbit", + "orbit_direction", + "imaging_mode", + "polarization", + "center_bucket", + "reference_date", + ] + }, + "scenes": usable_scenes, + "excluded_scenes": [ + scene for scene in candidate.get("scenes", []) + if scene not in usable_scenes + ], + "pair_network": { + "strategy": "adjacent_temporal_initial", + "gamma_baseline_status": "PENDING", + "pairs": pairs, + }, + "blockers": blockers, + "warnings": sorted(set(warnings)), + "next_stage": "convert selected LT1 scenes with par_LT1_SLC, then run Gamma base_calc before final itab approval", + } + manifest_path = self._write_runtime_json( + Path("stack_manifests") / stack_id, + f"{timestamp}_stack_manifest.json", + manifest, + ) + pair_network_path = self._write_runtime_json( + Path("stack_manifests") / stack_id, + f"{timestamp}_pair_network.json", + manifest["pair_network"], + ) + return { + "stack_id": stack_id, + "status": manifest["status"], + "manifest_path": str(manifest_path), + "pair_network_path": str(pair_network_path), + "manifest": manifest, + } + + def create_run( + self, + stack_id: str, + *, + run_label: str | None = None, + source_roots: list[str] | None = None, + orbit_roots: list[str] | None = None, + min_scenes: int = 3, + require_orbits: bool = True, + monitor_points: list[dict[str, Any]] | None = None, + monitor_point_strategy: str = "auto_low_sigma_high_rate", + dry_run: bool = True, + ) -> dict[str, Any]: + if not dry_run: + raise ValueError("Gamma SBAS execution is not wired yet; submit with dry_run=true.") + + audit = self.audit_stack( + stack_id, + source_roots=source_roots, + orbit_roots=orbit_roots, + min_scenes=min_scenes, + require_orbits=require_orbits, + ) + manifest = audit["manifest"] + if manifest.get("status") != "READY_FOR_GAMMA_BASELINE_AUDIT": + raise ValueError("stack manifest is not ready for run planning") + + timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") + run_id = self._stable_id(f"{stack_id}|{timestamp}|{run_label or ''}") + run_dir = self.production_root / "runs" / run_id + work_dir = run_dir / "work" + publish_dir = run_dir / "publish" + log_dir = run_dir / "logs" + for path in (work_dir, publish_dir, log_dir): + path.mkdir(parents=True, exist_ok=True) + + monitor_config = self._build_monitor_point_config( + monitor_points=monitor_points, + strategy=monitor_point_strategy, + stack_manifest=manifest, + ) + run_manifest = { + "schema": "insar.gamma-ipta-sbas-run/v1", + "run_id": run_id, + "run_label": run_label or None, + "workflow_code": "sbas_insar", + "processor_code": "gamma_ipta_sbas", + "engine_code": "gamma", + "execution_mode": "dry_run_plan", + "status": "PLANNED_GAMMA_BASELINE_AUDIT", + "created_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "stack_id": stack_id, + "stack_manifest_path": audit["manifest_path"], + "pair_network_path": audit["pair_network_path"], + "work_root": str(work_dir), + "publish_root": str(publish_dir), + "log_root": str(log_dir), + "stack": manifest.get("stack") or {}, + "scene_count": len(manifest.get("scenes") or []), + "pair_count": len(((manifest.get("pair_network") or {}).get("pairs")) or []), + "next_stage": "baseline_audit", + "requires_user_action": [ + "Review Gamma base_calc baseline table before approving final itab.", + "Confirm monitoring-point source: manual points, imported layer, or automatic sampler.", + "Confirm geocoded preview products are published from EPSG:4326 GeoTIFFs.", + ], + "monitor_points": monitor_config, + "dry_run": dry_run, + } + command_manifest = self._build_command_manifest(run_manifest, manifest) + + run_manifest_path = self._write_json(run_dir / "run_manifest.json", run_manifest) + command_manifest_path = self._write_json(run_dir / "gamma_command_manifest.json", command_manifest) + monitor_config_path = self._write_json(run_dir / "monitor_points.json", monitor_config) + self._write_json(run_dir / "stack_manifest.json", manifest) + self._write_json(run_dir / "pair_network.json", manifest.get("pair_network") or {}) + + index_item = { + **self._build_run_card(run_dir, run_manifest), + "run_manifest_path": str(run_manifest_path), + "gamma_command_manifest_path": str(command_manifest_path), + "monitor_config_path": str(monitor_config_path), + } + return { + "run": index_item, + "manifest": run_manifest, + "command_manifest": command_manifest, + "monitor_points": monitor_config, + } + + def list_runs(self) -> dict[str, Any]: + run_root = self.production_root / "runs" + items: list[dict[str, Any]] = [] + if not run_root.exists(): + return {"items": items, "count": 0, "run_root": str(run_root)} + + for manifest_path in sorted(run_root.glob("*/run_manifest.json")): + try: + manifest = self._read_json(manifest_path) + items.append(self._build_run_card(manifest_path.parent, manifest)) + except Exception as exc: + items.append( + { + "run_id": manifest_path.parent.name, + "status": "RUN_MANIFEST_UNREADABLE", + "run_dir": str(manifest_path.parent), + "error": str(exc), + } + ) + items.sort(key=lambda item: str(item.get("created_at") or ""), reverse=True) + return {"items": items, "count": len(items), "run_root": str(run_root)} + + def get_run_detail(self, run_id: str) -> dict[str, Any]: + run_dir = self._resolve_run_dir(run_id) + manifest = self._read_json(run_dir / "run_manifest.json") + command_manifest = self._read_optional_json(run_dir / "gamma_command_manifest.json") + monitor_points = self._read_optional_json(run_dir / "monitor_points.json") + return { + "run": self._build_run_card(run_dir, manifest), + "manifest": manifest, + "command_manifest": command_manifest, + "monitor_points": monitor_points, + "artifacts": self._build_run_artifacts(run_dir), + } + + def run_baseline_audit( + self, + run_id: str, + *, + execute: bool = True, + rlks: int = 8, + azlks: int = 8, + max_delta_n: int = 1, + timeout_seconds: int = 21600, + ) -> dict[str, Any]: + run_dir = self._resolve_run_dir(run_id) + manifest_path = run_dir / "run_manifest.json" + manifest = self._read_json(manifest_path) + stack_manifest = self._read_json(run_dir / "stack_manifest.json") + if manifest.get("status") not in { + "PLANNED_GAMMA_BASELINE_AUDIT", + "BASELINE_AUDIT_SCRIPT_READY", + "BASELINE_AUDIT_FAILED", + "BASELINE_AUDIT_READY", + }: + raise ValueError(f"run status does not allow baseline audit: {manifest.get('status')}") + + rlks = self._bounded_int(rlks, default=8, minimum=1, maximum=64) + azlks = self._bounded_int(azlks, default=8, minimum=1, maximum=64) + max_delta_n = self._bounded_int(max_delta_n, default=1, minimum=1, maximum=100) + timeout_seconds = self._bounded_int(timeout_seconds, default=21600, minimum=60, maximum=86400) + + script_path = self._write_baseline_audit_script( + run_dir, + stack_manifest=stack_manifest, + rlks=rlks, + azlks=azlks, + max_delta_n=max_delta_n, + ) + manifest["baseline_audit"] = { + "script_path": str(script_path), + "rlks": rlks, + "azlks": azlks, + "max_delta_n": max_delta_n, + "updated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + } + if not execute: + baseline_summary = self._build_baseline_summary(run_dir) + if baseline_summary.get("adjacent_pair_count"): + manifest["status"] = "BASELINE_AUDIT_READY" + manifest["next_stage"] = "approve_itab" + manifest["baseline_audit"]["summary"] = baseline_summary + manifest["baseline_audit"]["approved_for_next_stage"] = False + self._write_json(run_dir / "baseline_audit_summary.json", baseline_summary) + self._write_json(run_dir / "pair_network_baseline_audit.json", baseline_summary.get("pair_network") or {}) + self._write_json(run_dir / "pair_network.json", baseline_summary.get("pair_network") or {}) + else: + manifest["status"] = "BASELINE_AUDIT_SCRIPT_READY" + manifest["next_stage"] = "execute_baseline_audit" + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_baseline(run_dir, manifest, baseline_summary if baseline_summary.get("adjacent_pair_count") else None) + return self.get_run_detail(run_id) + + started_at = datetime.utcnow().isoformat(timespec="seconds") + "Z" + script_wsl = self._windows_path_to_wsl_mount(str(script_path)) + command = self._baseline_execution_command(str(script_wsl)) + completed = subprocess.run( + command, + cwd=str(run_dir), + text=True, + capture_output=True, + timeout=timeout_seconds, + check=False, + ) + execution = { + "started_at": started_at, + "ended_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "command": command, + "returncode": completed.returncode, + "stdout_tail": completed.stdout[-4000:], + "stderr_tail": completed.stderr[-4000:], + } + + baseline_summary = self._build_baseline_summary(run_dir) + manifest["baseline_audit"] = { + **manifest["baseline_audit"], + "execution": execution, + "summary": baseline_summary, + } + if completed.returncode == 0 and baseline_summary.get("adjacent_pair_count"): + manifest["status"] = "BASELINE_AUDIT_READY" + manifest["next_stage"] = "approve_itab" + manifest["baseline_audit"]["approved_for_next_stage"] = False + self._write_json(run_dir / "baseline_audit_summary.json", baseline_summary) + self._write_json(run_dir / "pair_network_baseline_audit.json", baseline_summary.get("pair_network") or {}) + self._write_json(run_dir / "pair_network.json", baseline_summary.get("pair_network") or {}) + else: + manifest["status"] = "BASELINE_AUDIT_FAILED" + manifest["next_stage"] = "fix_baseline_audit" + + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_baseline(run_dir, manifest, baseline_summary) + return self.get_run_detail(run_id) + + def _baseline_execution_command(self, script_wsl: str) -> list[str]: + return self._script_execution_command(script_wsl) + + def _script_execution_command(self, script_wsl: str) -> list[str]: + if os.name != "nt": + return ["bash", script_wsl] + return [ + "wsl.exe", + "-d", + settings.WSL_DISTRO or settings.PYINT_WSL_DISTRO or "Ubuntu-24.04", + "bash", + script_wsl, + ] + + def decide_itab( + self, + run_id: str, + *, + decision: str, + reviewer: str | None = None, + note: str | None = None, + ) -> dict[str, Any]: + run_dir = self._resolve_run_dir(run_id) + manifest_path = run_dir / "run_manifest.json" + manifest = self._read_json(manifest_path) + normalized_decision = str(decision or "").strip().lower() + if normalized_decision not in {"approve", "reject"}: + raise ValueError("decision must be approve or reject") + if manifest.get("status") in {"COREGISTRATION_SCRIPT_READY", "COREGISTRATION_RUNNING", "COREGISTRATION_READY"}: + existing_decision = ((manifest.get("baseline_audit") or {}).get("itab_decision") or {}).get("decision") + if normalized_decision == "approve" and existing_decision == "approve": + return self.get_run_detail(run_id) + if manifest.get("status") not in {"BASELINE_AUDIT_READY", "ITAB_APPROVED", "ITAB_REJECTED"}: + raise ValueError(f"run status does not allow itab decision: {manifest.get('status')}") + + baseline_summary = self._read_optional_json(run_dir / "baseline_audit_summary.json") + if not baseline_summary or not baseline_summary.get("adjacent_pair_count"): + raise ValueError("baseline audit summary is missing or empty") + + decided_at = datetime.utcnow().isoformat(timespec="seconds") + "Z" + decision_payload = { + "schema": "insar.sbas-itab-decision/v1", + "run_id": run_id, + "decision": normalized_decision, + "reviewer": str(reviewer or "system").strip()[:120], + "note": str(note or "").strip()[:1000], + "decided_at": decided_at, + "baseline_summary": { + "adjacent_pair_count": baseline_summary.get("adjacent_pair_count"), + "max_abs_bperp_m": baseline_summary.get("max_abs_bperp_m"), + "max_delta_days": baseline_summary.get("max_delta_days"), + }, + } + + baseline_state = manifest.setdefault("baseline_audit", {}) + if normalized_decision == "approve": + source_itab = run_dir / "work" / "gamma" / "diff" / "itab_adjacent" + if not source_itab.is_file(): + raise FileNotFoundError(f"Gamma adjacent itab not found: {source_itab}") + approved_itab = run_dir / "work" / "gamma" / "diff" / "itab_approved" + shutil.copyfile(source_itab, approved_itab) + self._write_json(run_dir / "itab_decision.json", decision_payload) + baseline_state["approved_for_next_stage"] = True + baseline_state["itab_decision"] = decision_payload + baseline_state["approved_itab_path"] = str(approved_itab) + manifest["status"] = "ITAB_APPROVED" + manifest["next_stage"] = "coregistration" + else: + self._write_json(run_dir / "itab_decision.json", decision_payload) + baseline_state["approved_for_next_stage"] = False + baseline_state["itab_decision"] = decision_payload + manifest["status"] = "ITAB_REJECTED" + manifest["next_stage"] = "revise_pair_network" + + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_itab_decision(run_dir, manifest) + return self.get_run_detail(run_id) + + def prepare_coregistration( + self, + run_id: str, + *, + execute: bool = False, + rlks: int = 8, + azlks: int = 8, + ) -> dict[str, Any]: + if execute: + raise ValueError("Coregistration execution is not enabled yet; submit with execute=false.") + run_dir = self._resolve_run_dir(run_id) + manifest_path = run_dir / "run_manifest.json" + manifest = self._read_json(manifest_path) + if manifest.get("status") not in {"ITAB_APPROVED", "COREGISTRATION_SCRIPT_READY", "COREGISTRATION_FAILED"}: + raise ValueError(f"run status does not allow coregistration preparation: {manifest.get('status')}") + approved_itab = run_dir / "work" / "gamma" / "diff" / "itab_approved" + if not approved_itab.is_file(): + raise FileNotFoundError(f"approved itab not found: {approved_itab}") + stack_manifest = self._read_json(run_dir / "stack_manifest.json") + scenes = sorted(stack_manifest.get("scenes") or [], key=lambda item: str(item.get("date") or "")) + reference_date = str((stack_manifest.get("stack") or {}).get("reference_date") or "").strip() + if reference_date not in {str(scene.get("date")) for scene in scenes}: + reference_date = str(scenes[len(scenes) // 2].get("date")) + + rlks = self._bounded_int(rlks, default=8, minimum=1, maximum=64) + azlks = self._bounded_int(azlks, default=8, minimum=1, maximum=64) + itab_rows = self._parse_itab(approved_itab) + if not itab_rows: + raise ValueError("approved itab is empty") + + script_path = self._write_coregistration_script( + run_dir, + scenes=scenes, + reference_date=reference_date, + rlks=rlks, + azlks=azlks, + ) + coregistration = { + "schema": "insar.gamma-coregistration-stage/v1", + "strategy": "common_reference_to_stack_reference_date", + "script_path": str(script_path), + "approved_itab_path": str(approved_itab), + "reference_date": reference_date, + "scene_count": len(scenes), + "approved_pair_count": len(itab_rows), + "rlks": rlks, + "azlks": azlks, + "updated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "outputs": { + "common_dir": str(run_dir / "work" / "gamma" / f"common_{reference_date}"), + "slc_tab": str(run_dir / "work" / "gamma" / f"common_{reference_date}" / "SLC_tab"), + "rmli_tab": str(run_dir / "work" / "gamma" / f"common_{reference_date}" / "RMLI_tab"), + }, + } + manifest["coregistration"] = coregistration + manifest["status"] = "COREGISTRATION_SCRIPT_READY" + manifest["next_stage"] = "execute_coregistration" + self._write_json(run_dir / "coregistration_plan.json", coregistration) + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_coregistration(run_dir, manifest) + return self.get_run_detail(run_id) + + def execute_coregistration( + self, + run_id: str, + *, + rlks: int = 8, + azlks: int = 8, + timeout_seconds: int = 43200, + ) -> dict[str, Any]: + run_dir = self._resolve_run_dir(run_id) + manifest_path = run_dir / "run_manifest.json" + manifest = self._read_json(manifest_path) + status = str(manifest.get("status") or "").strip() + if status == "COREGISTRATION_READY": + return self.get_run_detail(run_id) + if status in {"ITAB_APPROVED", "COREGISTRATION_FAILED"}: + self.prepare_coregistration(run_id, execute=False, rlks=rlks, azlks=azlks) + manifest = self._read_json(manifest_path) + status = str(manifest.get("status") or "").strip() + if status not in {"COREGISTRATION_SCRIPT_READY", "COREGISTRATION_RUNNING"}: + raise ValueError(f"run status does not allow coregistration execution: {manifest.get('status')}") + + coregistration = dict(manifest.get("coregistration") or {}) + script_path = Path(self._path_to_windows(str(coregistration.get("script_path") or "")) or "") + if not script_path.is_file(): + raise FileNotFoundError(f"coregistration script not found: {script_path}") + + timeout_seconds = self._bounded_int(timeout_seconds, default=43200, minimum=60, maximum=172800) + started_at = datetime.utcnow().isoformat(timespec="seconds") + "Z" + command = self._script_execution_command(str(self._windows_path_to_wsl_mount(str(script_path)))) + + coregistration["execution"] = { + "started_at": started_at, + "command": command, + "timeout_seconds": timeout_seconds, + "status": "RUNNING", + } + manifest["coregistration"] = coregistration + manifest["status"] = "COREGISTRATION_RUNNING" + manifest["next_stage"] = "coregistration" + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_coregistration(run_dir, manifest) + + try: + completed = subprocess.run( + command, + cwd=str(run_dir), + text=True, + capture_output=True, + timeout=timeout_seconds, + check=False, + ) + except subprocess.TimeoutExpired as exc: + summary = self._build_coregistration_summary( + run_dir, + reference_date=coregistration.get("reference_date"), + ) + execution = { + **coregistration.get("execution", {}), + "ended_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "status": "TIMEOUT", + "timed_out": True, + "stdout_tail": self._tail_text(exc.stdout), + "stderr_tail": self._tail_text(exc.stderr), + } + coregistration = {**coregistration, "execution": execution, "summary": summary} + manifest["coregistration"] = coregistration + manifest["status"] = "COREGISTRATION_FAILED" + manifest["next_stage"] = "fix_coregistration" + self._write_json(run_dir / "coregistration_summary.json", summary) + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_coregistration(run_dir, manifest) + raise + + summary = self._build_coregistration_summary( + run_dir, + reference_date=coregistration.get("reference_date"), + ) + execution = { + **coregistration.get("execution", {}), + "ended_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "status": "COMPLETED" if completed.returncode == 0 else "FAILED", + "returncode": completed.returncode, + "stdout_tail": self._tail_text(completed.stdout), + "stderr_tail": self._tail_text(completed.stderr), + } + coregistration = {**coregistration, "execution": execution, "summary": summary} + manifest["coregistration"] = coregistration + if completed.returncode == 0 and summary.get("ready"): + manifest["status"] = "COREGISTRATION_READY" + manifest["next_stage"] = "rdc_dem" + else: + manifest["status"] = "COREGISTRATION_FAILED" + manifest["next_stage"] = "fix_coregistration" + + self._write_json(run_dir / "coregistration_summary.json", summary) + self._write_json(manifest_path, manifest) + self._refresh_command_manifest_after_coregistration(run_dir, manifest) + return self.get_run_detail(run_id) + + def list_trial_runs(self) -> dict[str, Any]: + items: list[dict[str, Any]] = [] + if not self.trial_root.exists(): + return {"items": items, "count": 0, "trial_root": str(self.trial_root)} + + for summary_path in sorted(self.trial_root.glob("*/publish/trial_summary.json")): + try: + summary = self._read_json(summary_path) + items.append(self._build_trial_card(summary_path.parent.parent, summary)) + except Exception as exc: + items.append( + { + "trial_id": summary_path.parent.parent.name, + "status": "SUMMARY_UNREADABLE", + "summary_path": str(summary_path), + "error": str(exc), + } + ) + + items.sort(key=lambda item: str(item.get("generated_at") or ""), reverse=True) + return {"items": items, "count": len(items), "trial_root": str(self.trial_root)} + + def get_trial_detail(self, trial_id: str) -> dict[str, Any]: + trial_dir = self._resolve_trial_dir(trial_id) + summary_path = trial_dir / "publish" / "trial_summary.json" + if not summary_path.is_file(): + raise FileNotFoundError(f"trial summary not found: {summary_path}") + + summary = self._read_json(summary_path) + artifacts = self._build_artifacts(trial_dir) + return { + "trial": self._build_trial_card(trial_dir, summary), + "summary": summary, + "artifacts": artifacts, + "stage_contract": [ + "par_LT1_SLC", + "LT1_precision_orbit.py", + "multi_look", + "base_calc", + "SLC_coreg.py", + "gc_map1/geocode/gc_map_fine", + "phase_sim_orb", + "SLC_diff_intf", + "adf", + "mcf", + "mb", + "ts_rate", + "geocode_back", + "data2geotiff", + "LOS sign conversion", + "monitoring point time series", + ], + } + + def resolve_artifact_path(self, trial_id: str, relative_path: str) -> Path: + trial_dir = self._resolve_trial_dir(trial_id) + normalized = str(relative_path or "").replace("\\", "/").strip("/") + if not normalized or normalized.startswith("../") or "/../" in normalized: + raise ValueError("invalid artifact path") + if not normalized.startswith("publish/"): + raise ValueError("only published artifacts can be served") + + candidate = (trial_dir / normalized).resolve() + trial_resolved = trial_dir.resolve() + try: + candidate.relative_to(trial_resolved) + except ValueError as exc: + raise ValueError("artifact path escapes trial root") from exc + if not candidate.is_file(): + raise FileNotFoundError(f"artifact not found: {normalized}") + return candidate + + def resolve_run_artifact_path(self, run_id: str, relative_path: str) -> Path: + run_dir = self._resolve_run_dir(run_id) + normalized = str(relative_path or "").replace("\\", "/").strip("/") + if not normalized or normalized.startswith("../") or "/../" in normalized: + raise ValueError("invalid artifact path") + allowed_paths = {item["relative_path"] for item in self._build_run_artifacts(run_dir)} + if normalized not in allowed_paths: + raise ValueError("run artifact is not published") + + candidate = (run_dir / normalized).resolve() + run_resolved = run_dir.resolve() + try: + candidate.relative_to(run_resolved) + except ValueError as exc: + raise ValueError("artifact path escapes run root") from exc + if not candidate.is_file(): + raise FileNotFoundError(f"artifact not found: {normalized}") + return candidate + + def _resolve_source_roots(self, roots: list[str] | None) -> list[Path]: + raw_values = roots or self._split_config_paths( + settings.SOURCE_PRODUCT_DIRS, + settings.MONITOR_RADAR_DIRS, + settings.INSAR_STORAGE_DIRS, + ) + if not raw_values: + raw_values = [r"D:\LuTan1_Image_Pool"] + return self._dedupe_existing_dirs(raw_values) + + def _resolve_orbit_roots(self, roots: list[str] | None) -> list[Path]: + raw_values = roots or self._split_config_paths( + settings.PYINT_ORBIT_POOL_TXT, + settings.ORBIT_POOL_ENVI, + ) + if not raw_values: + raw_values = [r"D:\orbit_pools\envi"] + return self._dedupe_existing_dirs(raw_values) + + @staticmethod + def _split_config_paths(*values: str) -> list[str]: + paths: list[str] = [] + for value in values: + for item in str(value or "").replace(";", ",").split(","): + text = item.strip().strip('"').strip("'") + if text: + paths.append(text) + return paths + + @staticmethod + def _dedupe_existing_dirs(values: list[str]) -> list[Path]: + roots: list[Path] = [] + seen: set[str] = set() + for value in values: + for path in SbasInsarProductionService._existing_path_variants(value): + key = os.path.normcase(str(path.resolve())) + if key in seen: + continue + seen.add(key) + roots.append(path) + return roots + + @staticmethod + def _existing_path_variants(value: str) -> list[Path]: + text = str(value or "").strip().strip('"').strip("'") + if not text: + return [] + candidates = [Path(os.path.normpath(text))] + wsl_path = SbasInsarProductionService._windows_path_to_wsl_mount(text) + if wsl_path and wsl_path != text: + candidates.append(Path(wsl_path)) + windows_path = SbasInsarProductionService._path_to_windows(text) + if windows_path and windows_path != text: + candidates.append(Path(os.path.normpath(windows_path))) + + existing: list[Path] = [] + seen: set[str] = set() + for candidate in candidates: + key = os.path.normcase(str(candidate)) + if key in seen: + continue + seen.add(key) + if candidate.is_dir(): + existing.append(candidate) + return existing + + def _iter_lt1_scene_dirs(self, root: Path): + if root.name.upper().startswith("LT1") and self._looks_like_lt1_scene_dir(root): + yield root + return + + try: + children = list(root.iterdir()) + except OSError: + return + + for child in children: + if child.is_dir() and child.name.upper().startswith("LT1") and self._looks_like_lt1_scene_dir(child): + yield child + + # Some source roots may have one extra grouping level. Keep recursion shallow + # to avoid walking runtime work directories by accident. + for child in children: + if not child.is_dir() or child.name.startswith((".", "_")): + continue + if child.name.upper().startswith(("LT1A", "LT1B")): + continue + try: + for grandchild in child.iterdir(): + if ( + grandchild.is_dir() + and grandchild.name.upper().startswith("LT1") + and self._looks_like_lt1_scene_dir(grandchild) + ): + yield grandchild + except OSError: + continue + + @staticmethod + def _looks_like_lt1_scene_dir(path: Path) -> bool: + try: + return any(path.glob("*.meta.xml")) and any( + list(path.glob("*.tiff")) + list(path.glob("*.tif")) + ) + except OSError: + return False + + def _parse_lt1_scene(self, scene_dir: Path, orbit_roots: list[Path]) -> dict[str, Any]: + scene_name = scene_dir.name + filename_meta = self._parse_lt1_scene_name(scene_name) + meta_path = self._select_meta_file(scene_dir) + tiff_path = self._select_tiff_file(scene_dir) + xml_meta = self._parse_lt1_product_info(meta_path) + meta = {**filename_meta, **{key: value for key, value in xml_meta.items() if value not in (None, "")}} + + date = str(meta.get("date") or "")[:8] + satellite = str(meta.get("satellite") or "").upper() + orbit_path = self._find_lt1_orbit(orbit_roots, satellite, date) + center_lon = self._as_float(meta.get("center_lon")) + center_lat = self._as_float(meta.get("center_lat")) + return { + "scene_name": scene_name, + "scene_dir_windows": self._path_to_windows(str(scene_dir)), + "scene_dir_wsl": self._windows_path_to_wsl_mount(str(scene_dir)), + "tiff_windows": self._path_to_windows(str(tiff_path)), + "tiff_wsl": self._windows_path_to_wsl_mount(str(tiff_path)), + "meta_windows": self._path_to_windows(str(meta_path)), + "meta_wsl": self._windows_path_to_wsl_mount(str(meta_path)), + "orbit_windows": self._path_to_windows(str(orbit_path)) if orbit_path else None, + "orbit_wsl": self._windows_path_to_wsl_mount(str(orbit_path)) if orbit_path else None, + "has_orbit": bool(orbit_path), + "date": date, + "satellite": satellite, + "satellite_mode": str(meta.get("satellite_mode") or "").upper() or None, + "receiving_station": str(meta.get("receiving_station") or "").upper() or None, + "absolute_orbit": str(meta.get("absolute_orbit") or "") or None, + "relative_orbit": str(meta.get("relative_orbit") or "") or None, + "orbit_direction": str(meta.get("orbit_direction") or "").upper() or None, + "imaging_mode": str(meta.get("imaging_mode") or "").upper() or None, + "look_direction": str(meta.get("look_direction") or "").upper() or None, + "polarization": str(meta.get("polarization") or "").upper() or None, + "product_type": str(meta.get("product_type") or "").upper() or None, + "center_lon": center_lon, + "center_lat": center_lat, + "center_bucket": self._center_bucket(center_lon, center_lat), + "bbox": meta.get("bbox"), + "start_time_utc": meta.get("start_time_utc"), + "stop_time_utc": meta.get("stop_time_utc"), + } + + @staticmethod + def _parse_lt1_scene_name(scene_name: str) -> dict[str, Any]: + match = LT1_SCENE_RE.match(scene_name) + if not match: + return {} + data = match.groupdict() + return { + "satellite": data.get("satellite", "").upper(), + "satellite_mode": data.get("satellite_mode", "").upper(), + "receiving_station": data.get("receiving_station", "").upper(), + "imaging_mode": data.get("imaging_mode", "").upper(), + "absolute_orbit": data.get("absolute_orbit"), + "center_lon": data.get("center_lon"), + "center_lat": data.get("center_lat"), + "date": data.get("date"), + "product_type": data.get("product_type", "").upper(), + "polarization": data.get("polarization", "").upper(), + } + + @staticmethod + def _select_meta_file(scene_dir: Path) -> Path: + candidates = sorted(scene_dir.glob("*.meta.xml")) + if not candidates: + raise FileNotFoundError(f"No LT1 meta XML found in {scene_dir}") + return candidates[0] + + @staticmethod + def _select_tiff_file(scene_dir: Path) -> Path: + candidates = sorted(list(scene_dir.glob("*.tiff")) + list(scene_dir.glob("*.tif"))) + if not candidates: + raise FileNotFoundError(f"No LT1 TIFF found in {scene_dir}") + slc_candidates = [path for path in candidates if "_SLC_" in path.name.upper()] + return slc_candidates[0] if slc_candidates else candidates[0] + + def _parse_lt1_product_info(self, meta_path: Path) -> dict[str, Any]: + text = meta_path.read_text(encoding="utf-8", errors="ignore") + match = re.search(r"]*>.*?", text, flags=re.IGNORECASE | re.DOTALL) + if not match: + return {} + root = ET.fromstring(match.group(0)) + corners: list[tuple[float, float]] = [] + for element in root.findall(".//sceneCornerCoord"): + lat = self._as_float(self._child_text(element, "lat")) + lon = self._as_float(self._child_text(element, "lon")) + if lat is not None and lon is not None: + corners.append((lon, lat)) + bbox = None + if corners: + lons = [item[0] for item in corners] + lats = [item[1] for item in corners] + bbox = { + "min_lon": min(lons), + "min_lat": min(lats), + "max_lon": max(lons), + "max_lat": max(lats), + } + center = root.find(".//sceneCenterCoord") + return { + "satellite": self._find_text(root, ".//missionInfo/mission"), + "absolute_orbit": self._find_text(root, ".//missionInfo/absOrbit"), + "relative_orbit": self._find_text(root, ".//missionInfo/relOrbit"), + "orbit_direction": self._find_text(root, ".//missionInfo/orbitDirection"), + "receiving_station": self._find_text(root, ".//generationInfo/receivingStation"), + "imaging_mode": self._find_text(root, ".//acquisitionInfo/imagingMode"), + "look_direction": self._find_text(root, ".//acquisitionInfo/lookDirection"), + "polarization": ( + self._find_text(root, ".//acquisitionInfo/polarisationMode") + or self._find_text(root, ".//acquisitionInfo/polarisationList/polLayer") + ), + "start_time_utc": self._find_text(root, ".//sceneInfo/start/timeUTC"), + "stop_time_utc": self._find_text(root, ".//sceneInfo/stop/timeUTC"), + "date": self._date_from_time(self._find_text(root, ".//sceneInfo/start/timeUTC")), + "center_lon": self._child_text(center, "lon") if center is not None else None, + "center_lat": self._child_text(center, "lat") if center is not None else None, + "bbox": bbox, + } + + @staticmethod + def _find_text(root: ET.Element, path: str) -> str | None: + element = root.find(path) + if element is None or element.text is None: + return None + text = element.text.strip() + return text or None + + @staticmethod + def _child_text(root: ET.Element | None, name: str) -> str | None: + if root is None: + return None + element = root.find(name) + if element is None or element.text is None: + return None + text = element.text.strip() + return text or None + + @staticmethod + def _date_from_time(value: str | None) -> str | None: + text = str(value or "").strip() + if len(text) >= 10: + return text[:10].replace("-", "") + return None + + @staticmethod + def _as_float(value: Any) -> float | None: + try: + return float(value) + except (TypeError, ValueError): + return None + + @staticmethod + def _center_bucket(lon: float | None, lat: float | None) -> str: + if lon is None or lat is None: + return "UNKNOWN_CENTER" + return f"E{lon:.1f}_N{lat:.1f}" + + @staticmethod + def _windows_path_to_wsl_mount(path: str | None) -> str | None: + text = str(path or "").strip() + if not text: + return None + normalized_posix = text.replace("\\", "/") + wsl_match = re.match(r"^/mnt/([a-zA-Z])/(.*)$", normalized_posix) + if wsl_match: + return f"/mnt/{wsl_match.group(1).lower()}/{wsl_match.group(2)}" + drive_match = re.match(r"^([a-zA-Z]):/(.*)$", normalized_posix) + if drive_match: + return f"/mnt/{drive_match.group(1).lower()}/{drive_match.group(2).lstrip('/')}" + drive, tail = os.path.splitdrive(os.path.normpath(text)) + if not drive: + return text.replace("\\", "/") + return f"/mnt/{drive.rstrip(':').lower()}/{tail.replace(os.sep, '/').lstrip('/')}" + + @staticmethod + def _path_to_windows(path: str | None) -> str | None: + text = str(path or "").strip() + if not text: + return None + normalized_posix = text.replace("\\", "/") + wsl_match = re.match(r"^/mnt/([a-zA-Z])/(.*)$", normalized_posix) + if wsl_match: + drive = wsl_match.group(1).upper() + tail = wsl_match.group(2).replace("/", "\\") + return f"{drive}:\\{tail}" + return os.path.normpath(text) + + @staticmethod + def _find_lt1_orbit(orbit_roots: list[Path], satellite: str, date: str) -> Path | None: + if not satellite or not date: + return None + name = f"{satellite}_GpsData_GAS_C_{date}.txt" + for root in orbit_roots: + candidates = [ + root / satellite / name, + root / name, + ] + for candidate in candidates: + if candidate.is_file(): + return candidate + return None + + @staticmethod + def _stack_group_key(scene: dict[str, Any]) -> str: + parts = [ + scene.get("satellite"), + scene.get("satellite_mode"), + scene.get("receiving_station"), + scene.get("relative_orbit"), + scene.get("orbit_direction"), + scene.get("imaging_mode"), + scene.get("polarization"), + scene.get("center_bucket"), + ] + return "|".join(str(part or "") for part in parts) + + def _build_stack_candidate( + self, + scenes: list[dict[str, Any]], + *, + min_scenes: int, + require_orbits: bool, + ) -> dict[str, Any]: + scenes = sorted(scenes, key=lambda item: str(item.get("date") or "")) + first = scenes[0] + orbit_ready = [scene for scene in scenes if scene.get("has_orbit")] + usable = orbit_ready if require_orbits else scenes + dates = [scene.get("date") for scene in scenes if scene.get("date")] + usable_dates = [scene.get("date") for scene in usable if scene.get("date")] + group_key = self._stack_group_key(first) + stack_id = self._stable_id(group_key) + temporal_gaps = self._temporal_gaps(usable_dates) + blockers: list[str] = [] + if len(usable) < min_scenes: + blockers.append(f"usable_scene_count {len(usable)} < min_scenes {min_scenes}") + if require_orbits and len(orbit_ready) < len(scenes): + blockers.append("missing precise orbit for one or more scenes") + return { + "stack_id": stack_id, + "status": "READY" if not blockers else "BLOCKED", + "blockers": blockers, + "group_key": group_key, + "satellite": first.get("satellite"), + "satellite_mode": first.get("satellite_mode"), + "receiving_station": first.get("receiving_station"), + "relative_orbit": first.get("relative_orbit"), + "orbit_direction": first.get("orbit_direction"), + "imaging_mode": first.get("imaging_mode"), + "polarization": first.get("polarization"), + "center_bucket": first.get("center_bucket"), + "scene_count": len(scenes), + "orbit_ready_scene_count": len(orbit_ready), + "usable_scene_count": len(usable), + "missing_orbit_count": len(scenes) - len(orbit_ready), + "date_start": dates[0] if dates else None, + "date_end": dates[-1] if dates else None, + "dates": dates, + "usable_dates": usable_dates, + "reference_date": usable_dates[len(usable_dates) // 2] if usable_dates else None, + "temporal_gaps_days": temporal_gaps, + "max_temporal_gap_days": max(temporal_gaps) if temporal_gaps else 0, + "bbox_intersection": self._bbox_intersection([scene.get("bbox") for scene in usable]), + "scenes": scenes, + } + + @staticmethod + def _stable_id(value: str) -> str: + digest = hashlib.sha1(value.encode("utf-8", errors="ignore")).hexdigest()[:12] + return f"sbas_{digest}" + + @staticmethod + def _temporal_gaps(dates: list[str]) -> list[int]: + parsed: list[datetime] = [] + for date in sorted(set(dates)): + try: + parsed.append(datetime.strptime(date, "%Y%m%d")) + except ValueError: + continue + return [ + int((parsed[index + 1] - parsed[index]).days) + for index in range(len(parsed) - 1) + ] + + @staticmethod + def _bbox_intersection(items: list[dict[str, Any] | None]) -> dict[str, float] | None: + boxes = [item for item in items if item] + if not boxes: + return None + min_lon = max(float(item["min_lon"]) for item in boxes) + min_lat = max(float(item["min_lat"]) for item in boxes) + max_lon = min(float(item["max_lon"]) for item in boxes) + max_lat = min(float(item["max_lat"]) for item in boxes) + if min_lon >= max_lon or min_lat >= max_lat: + return None + return { + "min_lon": min_lon, + "min_lat": min_lat, + "max_lon": max_lon, + "max_lat": max_lat, + } + + @staticmethod + def _build_adjacent_pairs(scenes: list[dict[str, Any]]) -> list[dict[str, Any]]: + pairs: list[dict[str, Any]] = [] + for index in range(len(scenes) - 1): + master = scenes[index] + slave = scenes[index + 1] + delta_days = None + try: + delta_days = int( + ( + datetime.strptime(str(slave.get("date")), "%Y%m%d") + - datetime.strptime(str(master.get("date")), "%Y%m%d") + ).days + ) + except ValueError: + pass + pairs.append( + { + "pair_index": index + 1, + "master_date": master.get("date"), + "slave_date": slave.get("date"), + "delta_days": delta_days, + "master_scene_name": master.get("scene_name"), + "slave_scene_name": slave.get("scene_name"), + "itab_row_initial": [index + 1, index + 2, index + 1, 1], + "gamma_baseline_status": "PENDING", + } + ) + return pairs + + def _write_runtime_json(self, relative_dir: str | Path, filename: str, payload: dict[str, Any]) -> Path: + out_dir = self.production_root / relative_dir + out_dir.mkdir(parents=True, exist_ok=True) + out_path = out_dir / filename + out_path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8") + return out_path + + @staticmethod + def _write_json(path: Path, payload: dict[str, Any]) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8") + return path + + def _resolve_trial_dir(self, trial_id: str) -> Path: + clean_id = str(trial_id or "").strip() + if not clean_id or Path(clean_id).name != clean_id: + raise ValueError("invalid trial id") + trial_dir = (self.trial_root / clean_id).resolve() + root_resolved = self.trial_root.resolve() + try: + trial_dir.relative_to(root_resolved) + except ValueError as exc: + raise ValueError("trial id escapes trial root") from exc + if not trial_dir.is_dir(): + raise FileNotFoundError(f"trial not found: {clean_id}") + return trial_dir + + def _resolve_run_dir(self, run_id: str) -> Path: + clean_id = str(run_id or "").strip() + if not clean_id or Path(clean_id).name != clean_id: + raise ValueError("invalid run id") + run_dir = (self.production_root / "runs" / clean_id).resolve() + root_resolved = (self.production_root / "runs").resolve() + try: + run_dir.relative_to(root_resolved) + except ValueError as exc: + raise ValueError("run id escapes production root") from exc + if not run_dir.is_dir(): + raise FileNotFoundError(f"run not found: {clean_id}") + return run_dir + + @staticmethod + def _read_json(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + def _read_optional_json(self, path: Path) -> dict[str, Any] | None: + if not path.is_file(): + return None + return self._read_json(path) + + def _build_monitor_point_config( + self, + *, + monitor_points: list[dict[str, Any]] | None, + strategy: str, + stack_manifest: dict[str, Any], + ) -> dict[str, Any]: + normalized_points = [self._normalize_monitor_point(item, index) for index, item in enumerate(monitor_points or [])] + if normalized_points: + mode = "manual_lonlat" + note = "Manual monitoring points are stored for extraction after geocoded products are available." + else: + mode = strategy or "auto_low_sigma_high_rate" + note = ( + "Automatic point is only a production placeholder until users provide a point layer " + "or approve a quality-filtered sampler." + ) + return { + "schema": "insar.sbas-monitor-points/v1", + "mode": mode, + "points": normalized_points, + "default_auto_strategy": { + "key": "auto_low_sigma_high_rate", + "selection": "low LOS sigma, high absolute LOS velocity, non-edge valid pixel", + "usage": "debug/sample only; not a business monitoring network", + }, + "reference_date": (stack_manifest.get("stack") or {}).get("reference_date"), + "coordinate_system": "EPSG:4326 for manual lon/lat points; radar coordinates are derived during publishing", + "note": note, + } + + def _normalize_monitor_point(self, item: dict[str, Any], index: int) -> dict[str, Any]: + lon = self._as_float(item.get("lon") if item.get("lon") is not None else item.get("longitude")) + lat = self._as_float(item.get("lat") if item.get("lat") is not None else item.get("latitude")) + if lon is None or lat is None: + raise ValueError(f"monitor point {index + 1} requires lon/lat") + if not (-180 <= lon <= 180 and -90 <= lat <= 90): + raise ValueError(f"monitor point {index + 1} lon/lat out of range") + point_id = str(item.get("point_id") or item.get("id") or f"manual_{index + 1:03d}").strip() + if not re.match(r"^[A-Za-z0-9_.-]{1,64}$", point_id): + raise ValueError(f"monitor point {index + 1} has invalid point_id") + return { + "point_id": point_id, + "lon": lon, + "lat": lat, + "label": str(item.get("label") or point_id).strip()[:120], + "source": "manual_lonlat", + } + + @staticmethod + def _bounded_int(value: Any, *, default: int, minimum: int, maximum: int) -> int: + try: + number = int(value) + except (TypeError, ValueError): + number = default + return max(minimum, min(maximum, number)) + + def _write_baseline_audit_script( + self, + run_dir: Path, + *, + stack_manifest: dict[str, Any], + rlks: int, + azlks: int, + max_delta_n: int, + ) -> Path: + scenes = sorted(stack_manifest.get("scenes") or [], key=lambda item: str(item.get("date") or "")) + if len(scenes) < 2: + raise ValueError("baseline audit requires at least two scenes") + reference_date = str((stack_manifest.get("stack") or {}).get("reference_date") or "").strip() + if reference_date not in {str(scene.get("date")) for scene in scenes}: + reference_date = str(scenes[len(scenes) // 2].get("date")) + + scripts_dir = run_dir / "scripts" + script_path = scripts_dir / "01_baseline_audit.sh" + gamma_root = run_dir / "work" / "gamma" + slc_dir = gamma_root / "slc" + mli_dir = gamma_root / "mli" + diff_dir = gamma_root / "diff" + log_dir = run_dir / "logs" + python_bin = settings.WSL_SHARED_PYTHON or settings.PYINT_WSL_PYTHON or "/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python" + env_script = ( + self._windows_path_to_wsl_mount(settings.PYINT_GAMMA_ENV_SCRIPT) + or f"{self._windows_path_to_wsl_mount(settings.PROJECT_ROOT)}/deploy/wsl/profiles/gamma_env.sh" + ) + + lines = [ + "#!/usr/bin/env bash", + "set -euo pipefail", + "", + f'RUN_ROOT="{self._windows_path_to_wsl_mount(str(run_dir))}"', + f'SLC_DIR="{self._windows_path_to_wsl_mount(str(slc_dir))}"', + f'MLI_DIR="{self._windows_path_to_wsl_mount(str(mli_dir))}"', + f'DIFF_DIR="{self._windows_path_to_wsl_mount(str(diff_dir))}"', + f'LOG_DIR="{self._windows_path_to_wsl_mount(str(log_dir))}"', + f'PYTHON_BIN="{python_bin}"', + f'ORBIT_SCRIPT="${{GAMMA_HOME:-/usr/local/GAMMA_SOFTWARE-20240627}}/ISP/scripts/LT1_precision_orbit.py"', + f'RLKS="{rlks}"', + f'AZLKS="{azlks}"', + f'REF_DATE="{reference_date}"', + f'MAX_DELTA_N="{max_delta_n}"', + "", + f'source "{env_script}" >/dev/null 2>&1', + 'ORBIT_SCRIPT="${GAMMA_HOME}/ISP/scripts/LT1_precision_orbit.py"', + 'mkdir -p "${SLC_DIR}" "${MLI_DIR}" "${DIFF_DIR}" "${LOG_DIR}"', + "", + "run_scene() {", + ' local date="$1"', + ' local tiff="$2"', + ' local meta="$3"', + ' local orbit="$4"', + ' local slc="${SLC_DIR}/${date}.slc"', + ' local par="${SLC_DIR}/${date}.slc.par"', + ' local log="${LOG_DIR}/${date}_slc_prepare.log"', + ' {', + ' echo "== ${date} SLC prepare =="', + ' echo "tiff=${tiff}"', + ' echo "meta=${meta}"', + ' echo "orbit=${orbit}"', + ' test -r "${tiff}"', + ' test -r "${meta}"', + ' test -r "${orbit}"', + ' if [ ! -s "${slc}" ] || [ ! -s "${par}" ]; then', + ' rm -f "${slc}" "${par}"', + ' par_LT1_SLC "${tiff}" "${meta}" "${par}" "${slc}"', + " else", + ' echo "SLC already exists, skipping par_LT1_SLC"', + " fi", + ' if [ ! -s "${par}.before_precision_orbit" ]; then', + ' cp -f "${par}" "${par}.before_precision_orbit"', + ' "${PYTHON_BIN}" "${ORBIT_SCRIPT}" "${par}" "${orbit}"', + " else", + ' echo "Precision-orbit backup exists, assuming orbit correction is already applied"', + " fi", + ' test -s "${slc}"', + ' test -s "${par}"', + ' ls -lh "${slc}" "${par}" "${par}.before_precision_orbit"', + ' } >"${log}" 2>&1', + "}", + "", + "run_multilook() {", + ' local date="$1"', + ' local slc="${SLC_DIR}/${date}.slc"', + ' local slc_par="${SLC_DIR}/${date}.slc.par"', + ' local mli="${MLI_DIR}/${date}.mli"', + ' local mli_par="${MLI_DIR}/${date}.mli.par"', + ' local log="${LOG_DIR}/${date}_multi_look.log"', + ' {', + ' echo "== ${date} multi_look rlks=${RLKS} azlks=${AZLKS} =="', + ' test -s "${slc}"', + ' test -s "${slc_par}"', + ' if [ ! -s "${mli}" ] || [ ! -s "${mli_par}" ]; then', + ' multi_look "${slc}" "${slc_par}" "${mli}" "${mli_par}" "${RLKS}" "${AZLKS}"', + " else", + ' echo "MLI already exists, skipping multi_look"', + " fi", + ' ls -lh "${mli}" "${mli_par}"', + ' } >"${log}" 2>&1', + "}", + "", + ] + for scene in scenes: + date = str(scene.get("date") or "") + lines.append( + "run_scene " + f'"{date}" ' + f'"{scene.get("tiff_wsl")}" ' + f'"{scene.get("meta_wsl")}" ' + f'"{scene.get("orbit_wsl")}"' + ) + lines.extend( + [ + "", + ': >"${SLC_DIR}/SLC_tab"', + ] + ) + for scene in scenes: + date = str(scene.get("date") or "") + lines.append(f'printf "%s %s\\n" "${{SLC_DIR}}/{date}.slc" "${{SLC_DIR}}/{date}.slc.par" >>"${{SLC_DIR}}/SLC_tab"') + lines.append("") + for scene in scenes: + date = str(scene.get("date") or "") + lines.append(f'run_multilook "{date}"') + lines.extend( + [ + "", + ': >"${MLI_DIR}/RMLI_tab"', + ] + ) + for scene in scenes: + date = str(scene.get("date") or "") + lines.append(f'printf "%s %s\\n" "${{MLI_DIR}}/{date}.mli" "${{MLI_DIR}}/{date}.mli.par" >>"${{MLI_DIR}}/RMLI_tab"') + lines.extend( + [ + "", + 'base_calc "${SLC_DIR}/SLC_tab" "${SLC_DIR}/${REF_DATE}.slc.par" "${DIFF_DIR}/bperp_all_pairs.txt" "${DIFF_DIR}/itab_all_pairs" 1 0 - - 1 3650 - >"${LOG_DIR}/base_calc_all_pairs.log" 2>&1', + 'base_calc "${SLC_DIR}/SLC_tab" "${SLC_DIR}/${REF_DATE}.slc.par" "${DIFF_DIR}/bperp_adjacent.txt" "${DIFF_DIR}/itab_adjacent" 1 0 - - 1 3650 "${MAX_DELTA_N}" >"${LOG_DIR}/base_calc_adjacent.log" 2>&1', + 'du -h "${SLC_DIR}"/* "${MLI_DIR}"/* "${DIFF_DIR}"/* | sort -h >"${LOG_DIR}/baseline_audit_inventory.txt"', + 'echo "baseline audit complete: ${DIFF_DIR}/bperp_adjacent.txt"', + "", + ] + ) + scripts_dir.mkdir(parents=True, exist_ok=True) + script_path.write_text("\n".join(lines), encoding="utf-8", newline="\n") + return script_path + + def _write_coregistration_script( + self, + run_dir: Path, + *, + scenes: list[dict[str, Any]], + reference_date: str, + rlks: int, + azlks: int, + ) -> Path: + scripts_dir = run_dir / "scripts" + script_path = scripts_dir / "02_coreg_common_ref.sh" + gamma_root = run_dir / "work" / "gamma" + slc_dir = gamma_root / "slc" + mli_dir = gamma_root / "mli" + diff_dir = gamma_root / "diff" + common_dir = gamma_root / f"common_{reference_date}" + common_rslc_dir = common_dir / "rslc" + common_rmli_dir = common_dir / "rmli" + log_dir = run_dir / "logs" + python_bin = settings.WSL_SHARED_PYTHON or settings.PYINT_WSL_PYTHON or "/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python" + env_script = ( + self._windows_path_to_wsl_mount(settings.PYINT_GAMMA_ENV_SCRIPT) + or f"{self._windows_path_to_wsl_mount(settings.PROJECT_ROOT)}/deploy/wsl/profiles/gamma_env.sh" + ) + dates = [str(scene.get("date") or "") for scene in scenes if scene.get("date")] + lines = [ + "#!/usr/bin/env bash", + "set -euo pipefail", + "", + f'RUN_ROOT="{self._windows_path_to_wsl_mount(str(run_dir))}"', + f'SLC_DIR="{self._windows_path_to_wsl_mount(str(slc_dir))}"', + f'MLI_DIR="{self._windows_path_to_wsl_mount(str(mli_dir))}"', + f'DIFF_DIR="{self._windows_path_to_wsl_mount(str(diff_dir))}"', + f'COMMON_DIR="{self._windows_path_to_wsl_mount(str(common_dir))}"', + f'COMMON_RSLC_DIR="{self._windows_path_to_wsl_mount(str(common_rslc_dir))}"', + f'COMMON_RMLI_DIR="{self._windows_path_to_wsl_mount(str(common_rmli_dir))}"', + f'LOG_DIR="{self._windows_path_to_wsl_mount(str(log_dir))}"', + f'PYTHON_BIN="{python_bin}"', + f'REF_DATE="{reference_date}"', + f'RLKS="{rlks}"', + f'AZLKS="{azlks}"', + "", + f'source "{env_script}" >/dev/null 2>&1', + 'SLC_COREG="${GAMMA_HOME}/DIFF/scripts/SLC_coreg.py"', + 'APPROVED_ITAB="${DIFF_DIR}/itab_approved"', + 'test -s "${APPROVED_ITAB}"', + 'mkdir -p "${COMMON_RSLC_DIR}" "${COMMON_RMLI_DIR}" "${LOG_DIR}"', + "", + "DATES=(", + ] + lines.extend(f' "{date}"' for date in dates) + lines.extend( + [ + ")", + "", + 'REF_SLC="${SLC_DIR}/${REF_DATE}.slc"', + 'REF_PAR="${SLC_DIR}/${REF_DATE}.slc.par"', + "", + "coreg_to_ref() {", + ' local date="$1"', + ' local slc="${SLC_DIR}/${date}.slc"', + ' local par="${SLC_DIR}/${date}.slc.par"', + ' local rslc="${COMMON_RSLC_DIR}/${date}.rslc"', + ' local rslc_par="${COMMON_RSLC_DIR}/${date}.rslc.par"', + ' local rmli="${COMMON_RMLI_DIR}/${date}.mli"', + ' local rmli_par="${COMMON_RMLI_DIR}/${date}.mli.par"', + ' local gamma_off="${SLC_DIR}/${date}.slc.off"', + ' local off="${COMMON_RSLC_DIR}/${date}_to_${REF_DATE}.off"', + ' {', + ' echo "== common-reference coreg ${date} -> ${REF_DATE} =="', + ' test -s "${slc}"', + ' test -s "${par}"', + ' test -s "${REF_SLC}"', + ' test -s "${REF_PAR}"', + ' if [ "${date}" = "${REF_DATE}" ]; then', + ' echo "reference date, no resampling needed"', + ' return', + ' fi', + ' if [ ! -s "${rslc}" ] || [ ! -s "${rslc_par}" ] || [ ! -s "${rmli}" ] || [ ! -s "${rmli_par}" ] || [ ! -s "${off}" ]; then', + ' rm -f "${rslc}" "${rslc_par}" "${rmli}" "${rmli_par}" "${off}"', + ' "${PYTHON_BIN}" "${SLC_COREG}" \\', + ' "${slc}" "${par}" \\', + ' "${rslc}" "${rslc_par}" \\', + ' "${rmli}" "${rmli_par}" \\', + ' "${REF_SLC}" "${REF_PAR}" \\', + ' 0.1 "${RLKS}" "${AZLKS}" \\', + ' --init_offset', + ' test -s "${gamma_off}"', + ' cp -f "${gamma_off}" "${off}"', + " else", + ' echo "common-reference RSLC/coreg outputs already exist, skipping"', + " fi", + ' test -s "${rslc}"', + ' test -s "${rslc_par}"', + ' test -s "${rmli}"', + ' test -s "${rmli_par}"', + ' test -s "${off}"', + ' ls -lh "${rslc}" "${rslc_par}" "${rmli}" "${rmli_par}" "${off}" "${rslc}.coreg_quality"', + ' } >"${LOG_DIR}/${date}_to_${REF_DATE}_common_coreg.log" 2>&1', + "}", + "", + "slc_path() {", + ' local date="$1"', + ' if [ "${date}" = "${REF_DATE}" ]; then', + ' printf "%s %s\\n" "${SLC_DIR}/${date}.slc" "${SLC_DIR}/${date}.slc.par"', + " else", + ' printf "%s %s\\n" "${COMMON_RSLC_DIR}/${date}.rslc" "${COMMON_RSLC_DIR}/${date}.rslc.par"', + " fi", + "}", + "", + "rmli_path() {", + ' local date="$1"', + ' if [ "${date}" = "${REF_DATE}" ]; then', + ' printf "%s %s\\n" "${MLI_DIR}/${date}.mli" "${MLI_DIR}/${date}.mli.par"', + " else", + ' printf "%s %s\\n" "${COMMON_RMLI_DIR}/${date}.mli" "${COMMON_RMLI_DIR}/${date}.mli.par"', + " fi", + "}", + "", + 'for date in "${DATES[@]}"; do', + ' coreg_to_ref "${date}"', + "done", + "", + ': >"${COMMON_DIR}/SLC_tab"', + ': >"${COMMON_DIR}/RMLI_tab"', + 'for date in "${DATES[@]}"; do', + ' slc_path "${date}" >>"${COMMON_DIR}/SLC_tab"', + ' rmli_path "${date}" >>"${COMMON_DIR}/RMLI_tab"', + "done", + "", + 'cp -f "${APPROVED_ITAB}" "${COMMON_DIR}/itab_approved"', + 'du -h "${COMMON_DIR}"/* "${COMMON_RSLC_DIR}"/* "${COMMON_RMLI_DIR}"/* 2>/dev/null | sort -h >"${LOG_DIR}/coregistration_inventory.txt"', + 'echo "coregistration script complete: ${COMMON_DIR}"', + "", + ] + ) + scripts_dir.mkdir(parents=True, exist_ok=True) + script_path.write_text("\n".join(lines), encoding="utf-8", newline="\n") + return script_path + + def _build_baseline_summary(self, run_dir: Path) -> dict[str, Any]: + diff_dir = run_dir / "work" / "gamma" / "diff" + all_pairs = self._parse_bperp_table(diff_dir / "bperp_all_pairs.txt") + adjacent_pairs = self._parse_bperp_table(diff_dir / "bperp_adjacent.txt") + itab_rows = self._parse_itab(diff_dir / "itab_adjacent") + pair_network = { + "strategy": "gamma_base_calc_adjacent", + "gamma_baseline_status": "READY" if adjacent_pairs else "EMPTY", + "pairs": [], + } + for index, pair in enumerate(adjacent_pairs): + itab = itab_rows[index] if index < len(itab_rows) else None + pair_network["pairs"].append( + { + "pair_index": pair.get("pair_index"), + "master_date": pair.get("master_date"), + "slave_date": pair.get("slave_date"), + "delta_days": pair.get("delta_days"), + "bperp_m": pair.get("bperp_m"), + "itab_row": itab, + "gamma_baseline_status": "READY", + } + ) + bperps = [abs(float(item["bperp_m"])) for item in adjacent_pairs if item.get("bperp_m") is not None] + gaps = [float(item["delta_days"]) for item in adjacent_pairs if item.get("delta_days") is not None] + return { + "schema": "insar.gamma-baseline-audit/v1", + "generated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "all_pair_count": len(all_pairs), + "adjacent_pair_count": len(adjacent_pairs), + "max_abs_bperp_m": max(bperps) if bperps else None, + "mean_abs_bperp_m": sum(bperps) / len(bperps) if bperps else None, + "max_delta_days": max(gaps) if gaps else None, + "all_pairs": all_pairs, + "adjacent_pairs": adjacent_pairs, + "itab_adjacent": itab_rows, + "pair_network": pair_network, + "outputs": { + "bperp_all_pairs": str(diff_dir / "bperp_all_pairs.txt"), + "bperp_adjacent": str(diff_dir / "bperp_adjacent.txt"), + "itab_all_pairs": str(diff_dir / "itab_all_pairs"), + "itab_adjacent": str(diff_dir / "itab_adjacent"), + }, + } + + def _build_coregistration_summary( + self, + run_dir: Path, + *, + reference_date: str | None, + ) -> dict[str, Any]: + stack_manifest = self._read_json(run_dir / "stack_manifest.json") + scenes = sorted(stack_manifest.get("scenes") or [], key=lambda item: str(item.get("date") or "")) + dates = [str(scene.get("date") or "") for scene in scenes if scene.get("date")] + reference = str(reference_date or "").strip() + if reference not in dates and dates: + reference = str((stack_manifest.get("stack") or {}).get("reference_date") or "").strip() + if reference not in dates and dates: + reference = dates[len(dates) // 2] + + gamma_root = run_dir / "work" / "gamma" + common_dir = gamma_root / f"common_{reference}" + slc_dir = gamma_root / "slc" + mli_dir = gamma_root / "mli" + rslc_dir = common_dir / "rslc" + rmli_dir = common_dir / "rmli" + + per_date: list[dict[str, Any]] = [] + missing_dates: list[str] = [] + for date in dates: + if date == reference: + required = { + "slc": slc_dir / f"{date}.slc", + "slc_par": slc_dir / f"{date}.slc.par", + "mli": mli_dir / f"{date}.mli", + "mli_par": mli_dir / f"{date}.mli.par", + } + role = "reference" + else: + required = { + "rslc": rslc_dir / f"{date}.rslc", + "rslc_par": rslc_dir / f"{date}.rslc.par", + "rmli": rmli_dir / f"{date}.mli", + "rmli_par": rmli_dir / f"{date}.mli.par", + "offset": rslc_dir / f"{date}_to_{reference}.off", + } + role = "secondary" + missing = [name for name, path in required.items() if not path.is_file() or path.stat().st_size <= 0] + if missing: + missing_dates.append(date) + per_date.append( + { + "date": date, + "role": role, + "ready": not missing, + "missing": missing, + "quality_file": str(rslc_dir / f"{date}.rslc.coreg_quality") if date != reference else None, + } + ) + + expected_secondary_count = max(0, len(dates) - (1 if reference in dates else 0)) + ready_secondary_count = len( + [ + item for item in per_date + if item.get("role") == "secondary" and item.get("ready") + ] + ) + slc_tab = common_dir / "SLC_tab" + rmli_tab = common_dir / "RMLI_tab" + itab_approved = common_dir / "itab_approved" + required_tabs = { + "slc_tab": slc_tab, + "rmli_tab": rmli_tab, + "itab_approved": itab_approved, + } + missing_tabs = [ + name for name, path in required_tabs.items() + if not path.is_file() or path.stat().st_size <= 0 + ] + ready = not missing_dates and not missing_tabs and bool(dates) + return { + "schema": "insar.gamma-coregistration-summary/v1", + "generated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "reference_date": reference, + "scene_count": len(dates), + "expected_secondary_count": expected_secondary_count, + "ready_secondary_count": ready_secondary_count, + "missing_dates": missing_dates, + "missing_tabs": missing_tabs, + "ready": ready, + "per_date": per_date, + "outputs": { + "common_dir": str(common_dir), + "rslc_dir": str(rslc_dir), + "rmli_dir": str(rmli_dir), + "slc_tab": str(slc_tab), + "rmli_tab": str(rmli_tab), + "itab_approved": str(itab_approved), + }, + } + + @staticmethod + def _tail_text(value: Any, length: int = 4000) -> str: + if value is None: + return "" + if isinstance(value, bytes): + text = value.decode("utf-8", errors="replace") + else: + text = str(value) + return text[-length:] + + @staticmethod + def _parse_bperp_table(path: Path) -> list[dict[str, Any]]: + if not path.is_file(): + return [] + rows: list[dict[str, Any]] = [] + for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): + parts = line.split() + if len(parts) < 8: + continue + try: + rows.append( + { + "pair_index": int(parts[0]), + "master_date": parts[1], + "slave_date": parts[2], + "bperp_m": float(parts[3]), + "delta_days": float(parts[4]), + "mjd1": float(parts[5]), + "mjd2": float(parts[6]), + "bperp1_m": float(parts[7]), + "bperp2_m": float(parts[8]) if len(parts) > 8 else None, + } + ) + except ValueError: + continue + return rows + + @staticmethod + def _parse_itab(path: Path) -> list[list[int]]: + if not path.is_file(): + return [] + rows: list[list[int]] = [] + for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): + parts = line.split() + if len(parts) < 4: + continue + try: + rows.append([int(parts[0]), int(parts[1]), int(parts[2]), int(parts[3])]) + except ValueError: + continue + return rows + + def _refresh_command_manifest_after_baseline( + self, + run_dir: Path, + run_manifest: dict[str, Any], + baseline_summary: dict[str, Any] | None, + ) -> None: + path = run_dir / "gamma_command_manifest.json" + command_manifest = self._read_optional_json(path) or {} + stage_plan = command_manifest.get("stage_plan") or [dict(item) for item in GAMMA_STAGE_PLAN] + for stage in stage_plan: + if stage.get("stage_id") == "prepare_slc": + stage["status"] = "COMPLETED" if baseline_summary else "SCRIPT_READY" + if stage.get("stage_id") == "baseline_audit": + if run_manifest.get("status") == "BASELINE_AUDIT_READY": + stage["status"] = "COMPLETED_PENDING_ITAB_APPROVAL" + elif run_manifest.get("status") == "BASELINE_AUDIT_FAILED": + stage["status"] = "FAILED" + else: + stage["status"] = "SCRIPT_READY" + command_manifest["execution_enabled"] = True + command_manifest["reason_execution_disabled"] = None + command_manifest["stage_plan"] = stage_plan + command_manifest["baseline_audit"] = run_manifest.get("baseline_audit") + self._write_json(path, command_manifest) + + def _refresh_command_manifest_after_itab_decision( + self, + run_dir: Path, + run_manifest: dict[str, Any], + ) -> None: + path = run_dir / "gamma_command_manifest.json" + command_manifest = self._read_optional_json(path) or {} + stage_plan = command_manifest.get("stage_plan") or [dict(item) for item in GAMMA_STAGE_PLAN] + status = run_manifest.get("status") + for stage in stage_plan: + if stage.get("stage_id") == "baseline_audit": + if status == "ITAB_APPROVED": + stage["status"] = "COMPLETED_ITAB_APPROVED" + elif status == "ITAB_REJECTED": + stage["status"] = "COMPLETED_ITAB_REJECTED" + if stage.get("stage_id") == "coregistration": + if status == "ITAB_APPROVED": + stage["status"] = "READY" + elif status == "ITAB_REJECTED": + stage["status"] = "BLOCKED_PAIR_NETWORK_REJECTED" + command_manifest["stage_plan"] = stage_plan + command_manifest["baseline_audit"] = run_manifest.get("baseline_audit") + command_manifest["next_stage"] = run_manifest.get("next_stage") + self._write_json(path, command_manifest) + + def _refresh_command_manifest_after_coregistration( + self, + run_dir: Path, + run_manifest: dict[str, Any], + ) -> None: + path = run_dir / "gamma_command_manifest.json" + command_manifest = self._read_optional_json(path) or {} + stage_plan = command_manifest.get("stage_plan") or [dict(item) for item in GAMMA_STAGE_PLAN] + for stage in stage_plan: + if stage.get("stage_id") == "coregistration": + if run_manifest.get("status") == "COREGISTRATION_SCRIPT_READY": + stage["status"] = "SCRIPT_READY" + elif run_manifest.get("status") == "COREGISTRATION_RUNNING": + stage["status"] = "RUNNING" + elif run_manifest.get("status") == "COREGISTRATION_READY": + stage["status"] = "COMPLETED" + elif run_manifest.get("status") == "COREGISTRATION_FAILED": + stage["status"] = "FAILED" + if stage.get("stage_id") == "rdc_dem" and run_manifest.get("status") == "COREGISTRATION_READY": + stage["status"] = "READY" + command_manifest["stage_plan"] = stage_plan + command_manifest["coregistration"] = run_manifest.get("coregistration") + command_manifest["next_stage"] = run_manifest.get("next_stage") + self._write_json(path, command_manifest) + + def _build_command_manifest(self, run_manifest: dict[str, Any], stack_manifest: dict[str, Any]) -> dict[str, Any]: + scenes = stack_manifest.get("scenes") or [] + pair_network = stack_manifest.get("pair_network") or {} + return { + "schema": "insar.gamma-command-manifest/v1", + "run_id": run_manifest["run_id"], + "engine": "gamma", + "processor_code": "gamma_ipta_sbas", + "execution_enabled": False, + "reason_execution_disabled": "The managed Gamma runner is intentionally not attached in this planning slice.", + "stage_plan": [dict(item) for item in GAMMA_STAGE_PLAN], + "inputs": { + "scene_count": len(scenes), + "scenes": [ + { + "date": scene.get("date"), + "scene_name": scene.get("scene_name"), + "tiff_wsl": scene.get("tiff_wsl"), + "meta_wsl": scene.get("meta_wsl"), + "orbit_wsl": scene.get("orbit_wsl"), + } + for scene in scenes + ], + "pair_count": len(pair_network.get("pairs") or []), + "pair_network_strategy": pair_network.get("strategy"), + }, + "expected_outputs": [item["relative_path"] for item in PRODUCT_DEFINITIONS], + "next_manual_review": "Run Gamma base_calc, inspect perpendicular/temporal baselines, then replace initial adjacent itab if needed.", + } + + def _build_run_card(self, run_dir: Path, manifest: dict[str, Any]) -> dict[str, Any]: + stack = manifest.get("stack") or {} + return { + "run_id": manifest.get("run_id") or run_dir.name, + "run_label": manifest.get("run_label"), + "status": manifest.get("status") or "UNKNOWN", + "created_at": manifest.get("created_at"), + "workflow_code": manifest.get("workflow_code"), + "processor_code": manifest.get("processor_code"), + "engine_code": manifest.get("engine_code"), + "stack_id": manifest.get("stack_id"), + "scene_count": manifest.get("scene_count"), + "pair_count": manifest.get("pair_count"), + "next_stage": manifest.get("next_stage"), + "platform": stack.get("satellite"), + "relative_orbit": stack.get("relative_orbit"), + "direction": stack.get("orbit_direction"), + "polarization": stack.get("polarization"), + "center_bucket": stack.get("center_bucket"), + "reference_date": stack.get("reference_date"), + "run_dir": str(run_dir), + } + + def _build_run_artifacts(self, run_dir: Path) -> list[dict[str, Any]]: + artifacts: list[dict[str, Any]] = [] + for relative_path, label, role in [ + ("run_manifest.json", "SBAS run manifest", "run_manifest"), + ("stack_manifest.json", "Stack manifest", "stack_manifest"), + ("pair_network.json", "Initial pair network", "pair_network"), + ("pair_network_baseline_audit.json", "Gamma baseline-audited pair network", "pair_network_baseline_audit"), + ("baseline_audit_summary.json", "Gamma baseline audit summary", "baseline_audit_summary"), + ("itab_decision.json", "Approved/rejected itab decision", "itab_decision"), + ("coregistration_plan.json", "Coregistration stage plan", "coregistration_plan"), + ("coregistration_summary.json", "Coregistration execution summary", "coregistration_summary"), + ("gamma_command_manifest.json", "Gamma command manifest", "command_manifest"), + ("monitor_points.json", "Monitoring-point configuration", "monitor_points"), + ("scripts/01_baseline_audit.sh", "Gamma baseline audit script", "baseline_audit_script"), + ("scripts/02_coreg_common_ref.sh", "Gamma common-reference coregistration script", "coregistration_script"), + ]: + path = run_dir / relative_path + if path.is_file(): + artifacts.append( + { + "key": Path(relative_path).stem, + "label": label, + "role": role, + "relative_path": relative_path, + "size_bytes": path.stat().st_size, + } + ) + return artifacts + + def _build_trial_card(self, trial_dir: Path, summary: dict[str, Any]) -> dict[str, Any]: + stack = summary.get("stack") or {} + quality = summary.get("quality_stats") or {} + exports = summary.get("exports") or {} + monitor_points = summary.get("monitor_points") or [] + primary_rate_stats = quality.get("los_rate_toward_mm_per_year_rdc") or {} + sigma_stats = quality.get("los_sigma_mm_per_year_rdc") or {} + return { + "trial_id": summary.get("trial_id") or trial_dir.name, + "status": "TRIAL_READY", + "generated_at": summary.get("generated_at"), + "engine": summary.get("engine") or {}, + "stack": stack, + "dates": stack.get("dates") or [], + "reference_date": stack.get("reference_date"), + "scene_count": len(stack.get("dates") or []), + "platform": stack.get("platform"), + "direction": stack.get("direction"), + "relative_orbit": stack.get("relative_orbit"), + "polarization": stack.get("polarization"), + "mode": stack.get("mode"), + "default_los_product": "los_rate_toward_mm_per_year", + "los_sign_convention": (summary.get("radar") or {}).get("los_sign_convention"), + "primary_rate_median_mm_year": primary_rate_stats.get("median"), + "primary_rate_p01_mm_year": primary_rate_stats.get("p01"), + "primary_rate_p99_mm_year": primary_rate_stats.get("p99"), + "sigma_median_mm_year": sigma_stats.get("median"), + "monitor_point_count": len(monitor_points), + "export_count": len(exports), + "trial_dir": str(trial_dir), + } + + def _build_artifacts(self, trial_dir: Path) -> list[dict[str, Any]]: + artifacts: list[dict[str, Any]] = [] + for item in PRODUCT_DEFINITIONS: + path = trial_dir / item["relative_path"] + if path.is_file(): + artifacts.append( + { + **item, + "size_bytes": path.stat().st_size, + } + ) + + monitor_dir = trial_dir / "publish" / "monitor_points" + if monitor_dir.is_dir(): + for path in sorted(monitor_dir.iterdir()): + if not path.is_file(): + continue + for suffix_key, label, ext in MONITOR_ARTIFACT_SUFFIXES: + if path.name.endswith(ext): + artifacts.append( + { + "key": f"monitor_{path.stem}_{suffix_key}", + "label": label, + "role": "monitor_point", + "relative_path": str(path.relative_to(trial_dir)).replace("\\", "/"), + "size_bytes": path.stat().st_size, + } + ) + break + return artifacts + + +sbas_insar_production_service = SbasInsarProductionService() diff --git a/backend/app/services/water_extraction_service.py b/backend/app/services/water_extraction_service.py new file mode 100644 index 0000000..be8533b --- /dev/null +++ b/backend/app/services/water_extraction_service.py @@ -0,0 +1,33 @@ +"""Water extraction processors for the flood-analysis pipeline.""" +from __future__ import annotations + +from typing import Any + + +def run_otsu_water_extraction( + *, + input_path: str, + output_dir: str, + job_id: str | None = None, +) -> dict[str, Any]: + """Run the fast Otsu water-extraction processor. + + This wraps the legacy implementation while exposing the terminology used by + the flood pipeline: extraction, processor and threshold_value. + """ + from .water_detect_service import run_water_detection + + result = run_water_detection( + geo_tiff_path=input_path, + output_dir=output_dir, + job_id=job_id, + ) + result["processor"] = "otsu" + if "threshold_value" not in result and "otsu_threshold_db" in result: + result["threshold_value"] = result.get("otsu_threshold_db") + return result + + +def run_envi_water_extraction(*args: Any, **kwargs: Any) -> dict[str, Any]: + """Placeholder for the future ENVI/SARscape precise extractor.""" + raise NotImplementedError("ENVI/SARscape water extraction is not wired yet") diff --git a/docs/FLOOD_GEOTIFF_GAMMA_PREPROCESS_DESIGN_20260515.md b/docs/FLOOD_GEOTIFF_GAMMA_PREPROCESS_DESIGN_20260515.md new file mode 100644 index 0000000..815ad76 --- /dev/null +++ b/docs/FLOOD_GEOTIFF_GAMMA_PREPROCESS_DESIGN_20260515.md @@ -0,0 +1,515 @@ +# 洪涝模块 GeoTIFF 化与 GAMMA 前处理评估 + +更新日期：2026-05-15 + +## 1. 结论 + +洪涝模块后续应把业务算法全部下放到标准 GeoTIFF 层： + +```text +原始 SAR 数据 + -> 可插拔前处理器：GAMMA 优先，其他处理器兜底 + -> 分析级 GeoTIFF + -> Python 水体提取 / 洪涝检测 / 套合分析 / 产品生成 +``` + +ENVI/SARscape 不再作为洪涝算法主线。它可以短期保留为历史兼容处理器，但不应继续决定数据模型、接口命名和算法逻辑。 + +GAMMA 适合作为第一候选前处理底座，原因是当前项目已经具备 WSL、PyINT、GAMMA 环境注入、DEM、`geocode_back`、`data2geotiff` 和 D-InSAR 生产发布经验。但现有 GAMMA/PyINT 接入是 D-InSAR pair 流程，不是洪涝需要的“单景分析级 GeoTIFF 预处理服务”。因此不能直接把 `lt1_gamma_dinsar` 或 `s1_gamma_dinsar` 原样接到洪涝预处理按钮上，必须新增单景前处理适配层。 + +## 2. 当前真实情况 + +### 2.1 洪涝模块当前仍绑定 ENVI/SARscape + +当前 `/flood/preprocess` 最终仍执行旧的 `water_service.run_geocoding_workflow()`，内部流程是： + +```text +SARsImportLuTan1 + -> SARsBasicMultilooking + -> SARsBasicGeocoding +``` + +输出写入 `SARSceneGeoORM.geo_path`，语义是 SARscape 地理编码 dB 影像 base path，不是标准 GeoTIFF 交付物。 + +当前 `/flood/detections` 后台任务仍调用 `water_service.run_flood_detection()`，内部流程是： + +```text +SARsBasicFeFloodingClassification + -> 可选 SARsBasicFeFloodingClassificationRefinement + -> rasterio 统计分类结果面积 +``` + +分类值约定是： + +```text +0 = 无数据 / 背景 +1 = 稳定水体 +2 = 洪涝 / 新增水体 +3 = 高散射点 +4 = 非水体 +``` + +这说明当前“洪涝检测算法”实际是 SARscape 黑盒，不是平台自有算法。 + +### 2.2 已经接近目标形态的部分 + +以下部分已经更接近“标准栅格 + Python 算法”的目标： + +```text +水体提取： + water_extraction_service -> water_detect_service + Otsu + DEM/坡度约束 + 形态学 + 连通分量过滤 + +套合分析： + flood_overlay_service + 读取 classified 栅格，class=2 转矢量，套合灾害点、DInSAR 产品和 AOI + +GF3： + gf3_service + Python/GDAL 完成 L1A -> L2：辐射定标 + RPC 几何校正 +``` + +其中 GF3 路线虽然不走 GAMMA，但输出理念已经是“处理到 GeoTIFF 后进入平台算法”。它应该作为 `external/gdal` 前处理器接入统一 Scene，而不是继续挂在 `/water/gf3-process` 下。 + +### 2.3 当前 GAMMA/PyINT 接入情况 + +当前仓库里的 GAMMA/PyINT 能力主要服务 D-InSAR 生产： + +```text +backend/app/dinsar_engines/pyint_engine.py + - lt1_gamma_dinsar + - s1_gamma_dinsar + +backend/app/pyint_pipeline/run_lt1_pyint_pipeline.py + - LT-1 托管 runner + - geocode_gamma.py + - data2geotiff + - 标准产品发布 + +backend/app/pyint_pipeline/run_s1_pyint_pipeline.py + - Sentinel-1 runner + - 复用 LT-1 runner 的日志、发布和质量检查框架 + +deploy/wsl/profiles/gamma_env.sh + - 注入 GAMMA_HOME + - 注入 GAMMA 各模块 bin/scripts + - 注入 third_party/PyINT/pyint +``` + +当前 `.env` 中 PyINT/GAMMA 是启用状态： + +```text +PYINT_ENABLED=true +PYINT_WSL_DISTRO=Ubuntu-24.04 +PYINT_WSL_PYTHON=/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python +PYINT_DEM_MODE=prepared_file +PYINT_PREPARED_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84 +PYINT_GEO_INTERP=1 +PYINT_GAMMA_NODATA_VALUE=-9999.0 +``` + +需要注意一个不一致点：`pyint_engine.py` 已经暴露 `s1_gamma_dinsar`，但 `wsl_runtime_registry.py` 当前允许的 PyINT operation 仍只有 `lt1_gamma_dinsar` 和 `gamma_refine`。如果后续洪涝或生产任务走 WSL Broker，需要同步补齐允许列表。 + +### 2.4 GAMMA 现有输出与洪涝需求的差距 + +现有 D-InSAR GAMMA 链已经能导出 GeoTIFF，但输出重点是 pair 产品： + +```text +disp.tif +disp_unmasked.tif +coh.tif +wrapped_phase.tif +vertical displacement +look vector +``` + +`hyp3format_gamma.py` 也能把 `geocode_gamma.py` 产生的 geocoded GAMMA 二进制导出为 GeoTIFF，包含： + +```text +amp +corr +dem +unw_phase +wrapped_phase +los_disp +vert_disp +lv_theta +lv_phi +``` + +但洪涝模块真正需要的是单景产品： + +```text +analysis_ready.tif + - 单景后向散射强度 + - 明确单位：sigma0_db / gamma0_db / power_db + - 明确极化：VV / VH / HH / HV + - 明确 CRS、分辨率、nodata、覆盖范围 + - 可与另一景重采样到同一网格 +``` + +当前没有一个独立的 `gamma_single_scene_preprocess` 服务稳定地产出上述文件。现有 D-InSAR 流程里的 `geo__*.amp` 可以证明 GAMMA 有地理编码 MLI/AMP 的能力，但它嵌在 pair 配准和干涉流程中，不能直接作为洪涝单景预处理接口。 + +## 3. 目标数据契约 + +后续洪涝模块不再以 `geo_path` 表示模糊的“地理编码产物”，而应引入明确的分析级 Scene 产物语义。 + +建议模型命名： + +```text +SarAnalysisScene / AnalysisReadyScene +``` + +最小字段： + +```text +id +radar_data_id +engine # gamma / gf3_gdal / snap / envi_legacy / external +engine_profile # lt1_gamma_scene / s1_gamma_scene / gf3_rpc_l2 +analysis_tif_path # 主分析 GeoTIFF +preview_path +coverage_polygon +satellite +satellite_family # LT1 / S1 / GF3 / ... +imaging_date +polarization +orbit_direction +relative_orbit +backscatter_unit # sigma0_db / gamma0_db / power_db / unknown +crs +pixel_size_x +pixel_size_y +nodata_value +grid_id # 同网格配对时使用 +dem_path +incidence_angle_tif_path # 可选 +layover_shadow_mask_path # 可选 +quality_json +lineage_json +status +error_msg +created_at +updated_at +``` + +输出文件要求： + +```text +analysis_ready.tif: + - GeoTIFF，优先 COG + - float32 + - nodata = -9999 + - 推荐值域为 dB + - rasterio/GDAL 可直接读取 + +preview.png: + - 仅用于地图快视 + +metadata.json: + - 记录处理器、命令、输入资产、DEM、参数、软件版本、质量检查 +``` + +## 4. 目标流水线 + +### 4.1 前处理器只负责到 GeoTIFF 为止 + +前处理器接口建议统一为： + +```text +submit_scene_preprocess(radar_data_id, engine="gamma", profile="auto") +``` + +处理器职责： + +```text +1. 识别输入资产和传感器类型。 +2. 导入原始产品到处理器内部格式。 +3. 做必要的辐射定标。 +4. 做多视 / 滤波 / speckle 控制。 +5. 使用 DEM 做地形校正或地理编码。 +6. 导出 analysis_ready.tif。 +7. 生成 coverage、preview、metadata、quality。 +8. 注册 SarAnalysisScene。 +``` + +处理器不负责： + +```text +1. 水体提取。 +2. 洪涝分类。 +3. 洪涝矢量化。 +4. 灾害点套合。 +5. 报告生成。 +``` + +### 4.2 洪涝算法只认 GeoTIFF + +水体提取： + +```text +analysis_ready.tif + -> 读取 dB 栅格 + -> 有效像元 / nodata / DEM / 坡度约束 + -> Otsu 或自适应阈值 + -> 形态学与连通域过滤 + -> water_mask.tif + water_vector.geojson +``` + +洪涝检测： + +```text +pre analysis_ready.tif +post analysis_ready.tif + -> 检查 CRS / 分辨率 / 极化 / 覆盖范围 + -> 重采样到同一网格 + -> 分别生成 pre_water_mask / post_water_mask + -> post_water && !pre_water = 洪涝 / 新增水体 + -> pre_water && post_water = 稳定水体 + -> 结合 dB 差值、DEM 坡度、连通域和面积阈值过滤 + -> classified.tif +``` + +建议分类值继续沿用现有约定，降低前端和套合分析改动量： + +```text +0 = nodata +1 = stable_water +2 = flood +3 = high_scatter_or_uncertain +4 = non_water +``` + +套合分析： + +```text +classified.tif + -> class=2 转矢量 + -> 灾害点命中 + -> 近洪涝风险点 + -> DInSAR 产品 footprint 相交 + -> AOI / 行政区面积统计 +``` + +## 5. GAMMA 适配性判断 + +| 能力项 | 当前情况 | 适配判断 | +| --- | --- | --- | +| WSL/GAMMA 环境 | 已有 `gamma_env.sh`，能注入 GAMMA 与 PyINT 路径 | 可复用 | +| LT-1 导入 | 已有 `down2slc_LT1_*` 与精轨桥接，历史上处理过几何问题 | 可作为第一批验证对象，但需专项 QA | +| Sentinel-1 导入 | PyINT 脚本支持 ZIP、EOF、burst、TOPS 相关流程，项目已有 `s1_gamma_dinsar` runner | 可做第二批，但 runtime operation 白名单需补齐 | +| GF3 | 当前项目已有 Python/GDAL L1A->L2，不依赖 GAMMA | 不建议强行走 GAMMA，先走 `gf3_gdal` 处理器 | +| 地理编码 | 已有 `geocode_gamma.py`、`geocode_back`、DEM 生成链 | 可复用思想和部分脚本 | +| GeoTIFF 导出 | 已有 `data2geotiff`、HyP3/标准产品导出经验 | 可复用 | +| 单景分析级输出 | 当前没有独立服务，D-InSAR 中只顺带产出 master amp | 需要新建 | +| 辐射定标语义 | Sentinel-1 导入使用 calibration XML；LT-1/GF3 各有差异 | 必须在 metadata 中明确单位，不能含糊写 dB | +| 生产稳定性 | 当前 D-InSAR runner 已有日志、repair、quality、manifest | 可复用框架，但不能复用 pair 假设 | + +结论：GAMMA 适合做洪涝模块的优先前处理引擎，但第一阶段目标应是： + +```text +LT-1 单景 -> GAMMA -> analysis_ready.tif +``` + +跑通并验证后再扩展： + +```text +Sentinel-1 ZIP -> GAMMA -> analysis_ready.tif +GF3 L1A -> gf3_gdal -> analysis_ready.tif +``` + +## 6. 不应采用的方案 + +### 6.1 不应继续以 ENVI/SARscape 作为洪涝主线 + +原因： + +```text +1. 数据格式和任务对象绑定 SARscape。 +2. 洪涝分类是黑盒，难以解释和调参。 +3. 服务器部署依赖重。 +4. 后续多源数据扩展会反复补导入适配器。 +5. 输出不天然适合平台统一发布。 +``` + +### 6.2 不应把洪涝算法绑定到 GAMMA + +GAMMA 应只负责把原始 SAR 处理成标准 GeoTIFF。洪涝算法不能依赖 GAMMA 中间二进制，也不能要求输入来自 GAMMA。否则只是把“绑定 ENVI”换成“绑定 GAMMA”。 + +正确边界是： + +```text +GAMMA / GF3_GDAL / SNAP / ISCE / external + -> analysis_ready.tif + -> 同一套 Python 洪涝算法 +``` + +### 6.3 不应直接复用 D-InSAR pair profile 做单景预处理 + +现有 `lt1_gamma_dinsar` / `s1_gamma_dinsar` 是 pair 生产 profile，包含： + +```text +master/slave +coreg +diff +unwrap +geocode pair products +disp/coh export +``` + +洪涝单景预处理只需要： + +```text +single scene import +radiometric calibration +multilook/filter +terrain correction/geocoding +GeoTIFF export +QA/register +``` + +两者不能混成一个 profile。 + +## 7. 建议实施阶段 + +### Phase 1：建立 GeoTIFF 数据契约 + +```text +1. 新建或扩展 Scene 表，明确 analysis_tif_path。 +2. 保留旧 SARSceneGeoORM.geo_path 兼容字段。 +3. /flood/scenes 改为优先返回 analysis_tif_path。 +4. 前端把“可处理”判断改为是否存在 analysis_ready scene。 +``` + +### Phase 2：先把洪涝检测改成 Python GeoTIFF 算法 + +```text +1. 新建 flood_detection_service.py。 +2. 输入 pre_scene.analysis_tif_path 和 post_scene.analysis_tif_path。 +3. 对齐网格。 +4. 复用水体提取算法生成 pre/post water mask。 +5. 输出 classified.tif。 +6. 统计 flood_area_km2 和 stable_water_area_km2。 +``` + +这一步可以先用 ENVI 现有 geocode 结果或 GF3 L2 GeoTIFF 做输入测试，目的是先解除洪涝分类对 SARscape 的依赖。 + +### Phase 3：新增 GAMMA LT-1 单景前处理器 + +建议新增： + +```text +backend/app/services/sar_scene_preprocess_service.py +backend/app/services/gamma_scene_preprocess_service.py +backend/app/pyint_pipeline/run_gamma_scene_preprocess.py +``` + +最小 runner： + +```text +1. stage LT-1 输入资产。 +2. 调用 LT-1 导入脚本，产出 SLC。 +3. multi_look 生成 MLI/AMP。 +4. generate_rdc_dem 或等价 DEM 几何。 +5. geocode_back 把 AMP 映射到地理网格。 +6. data2geotiff 输出 analysis_ready.tif。 +7. 可选转 COG。 +8. rasterio 质量检查。 +``` + +### Phase 4：接 Sentinel-1 和 GF3 + +Sentinel-1： + +```text +1. 复用 Sentinel-1 ZIP/SAFE/EOF 管理设计。 +2. 新增 s1_gamma_scene profile。 +3. 不走 diff/unwrap，只产出单景 analysis_ready.tif。 +4. 修正 WSL runtime operation 白名单。 +``` + +GF3： + +```text +1. 保留 gf3_service.py 的 GDAL/RPC 路线。 +2. 输出注册为 analysis_ready scene。 +3. 不强行改成 GAMMA。 +``` + +### Phase 5：退役 ENVI 洪涝主线 + +```text +1. `/flood/detections` 默认使用 GeoTIFF 算法。 +2. ENVI/SARscape 作为 `envi_legacy` 处理器保留一段时间。 +3. 新任务不再默认创建 SARscape 洪涝分类。 +4. 文档、界面和部署说明移除“ENVI 是洪涝必须项”的表述。 +``` + +## 8. 验收标准 + +### 8.1 Scene 前处理 + +```text +1. 任意一个完成的 analysis_ready.tif 可以被 rasterio 打开。 +2. CRS、bounds、transform、nodata 完整。 +3. 有效像元比例达到阈值。 +4. dB 值域在合理范围内。 +5. metadata.json 能追溯输入资产、处理器、DEM、命令和版本。 +6. preview 和 coverage 能在地图上显示和清除。 +``` + +### 8.2 洪涝检测 + +```text +1. pre/post 两个 GeoTIFF 可自动对齐到同一网格。 +2. classified.tif 只依赖 Python/rasterio/numpy/scipy/skimage，不调用 ENVI/SARscape。 +3. class=1/2 面积统计稳定。 +4. 套合分析可直接读取 classified.tif。 +5. 同一算法能跑 GAMMA TIF、GF3 L2 TIF 和其他外部 TIF。 +``` + +### 8.3 GAMMA 适配 + +```text +1. LT-1 单景 GAMMA 前处理能稳定产出 analysis_ready.tif。 +2. 失败时有明确阶段日志，不出现“任务成功但全 0”的假成功。 +3. data2geotiff 输出通过 GDAL/rasterio 质量检查。 +4. 处理器输出不要求后续算法知道 GAMMA 中间目录结构。 +``` + +## 9. 对部署的影响 + +Git 可以提供平台代码、Python 算法、任务编排、数据库迁移和前端能力，但不能内置 GAMMA 商业软件本体。另一台服务器部署时必须额外满足： + +```text +1. WSL 或 Linux 环境可用。 +2. GAMMA 安装路径可被 gamma_env.sh 发现，或配置 PYINT_GAMMA_HOME/GAMMA_HOME。 +3. PyINT 代码随仓库存在。 +4. WSL Python/conda 环境具备 rasterio、GDAL、numpy、scipy、skimage 等依赖。 +5. DEM、轨道池和源数据池路径在 .env 中配置。 +``` + +如果目标服务器没有 GAMMA，系统仍应允许： + +```text +1. 使用已经存在的 analysis_ready.tif。 +2. 使用 GF3_GDAL 处理器。 +3. 后续接 SNAP/ISCE/external 处理器。 +4. 继续执行水体提取、洪涝检测、套合分析和产品生成。 +``` + +这也是把算法下放到 GeoTIFF 层的核心价值。 +## 2026-05-16 implementation note + +The first code pass now implements LT1 and GF3 as analysis-ready scene preprocessors: + +- New config roots: `SAR_ANALYSIS_READY_ROOT`, `SAR_ANALYSIS_WORK_ROOT`, `SAR_ANALYSIS_PREVIEW_ROOT`. +- `sar_scene_geo` now stores `analysis_tif_path`, `analysis_dir`, preview path, engine/profile, backscatter unit, nodata, metadata and quality JSON. +- `/flood/preprocess` routes GF3 scenes to `gf3_gdal` standardization and LT1 scenes to `lt_gamma` single-scene preprocessing. +- GF3 uses the existing GDAL/RPC L1A->L2 result, then registers the selected L2 GeoTIFF under `SAR_ANALYSIS_READY_ROOT`. +- LT1 uses a new Gamma/PyINT runner that stops at single-scene geocoded MLI GeoTIFF and does not write into D-InSAR product directories. +- Water extraction, flood detection and scene previews now prefer `analysis_tif_path` and fall back to legacy `geo_path`. +- Startup database maintenance now treats Alembic `0004` as the managed head, can auto-add missing ORM columns/indexes, and records the `alembic_version` marker after the schema is healthy. +- `/health?full=true&refresh=true` now includes `sar_analysis_ready` root and scene-path checks. + +This keeps D-InSAR product publication isolated from flood/water analysis inputs. diff --git a/docs/FLOOD_MODULE_REFACTOR_PLAN_20260514.md b/docs/FLOOD_MODULE_REFACTOR_PLAN_20260514.md new file mode 100644 index 0000000..4cc96df --- /dev/null +++ b/docs/FLOOD_MODULE_REFACTOR_PLAN_20260514.md @@ -0,0 +1,693 @@ +# 洪涝监测模块整改实施方案 + +> 日期：2026-05-14 +> 目标：将当前分散的水体监测、洪涝检测、GF3 处理和结果管理，收敛为可部署、可维护、可扩展的洪涝灾害分析流水线。 + +## 1. 总体原则 + +1. 新业务统一走 `/flood/*`。 +2. “水体监测”不再作为独立业务模块出现，只作为“洪涝灾害分析 -> 水体提取”步骤存在。 +3. `/water/*` 暂时保留兼容窗口，但标记 deprecated，不再新增功能。 +4. 数据库优先新增表和回填数据，避免直接破坏旧表。 +5. 每个阶段独立提交，保证任一阶段都能构建、部署和回滚。 + +## 2. 目标流水线 + +```text +场景准备 Scene + -> 水体提取 WaterExtraction + -> 洪涝检测 FloodDetection + -> 套合分析 FloodOverlay + -> 洪涝产品 FloodProduct / Report +``` + +其中水体提取是洪涝分析的前置步骤，不再和洪涝灾害分析并列成两个业务入口。 + +## 3. 阶段 1：收敛 API 契约 + +目标：先消灭“前端调用不存在接口”的问题，避免部署后出现静默 404。 + +修改范围： + +```text +frontend/src/api/flood.js +backend/app/routers/flood.py +docs/FLOOD_MODULE_REFACTOR_PLAN_20260514.md +``` + +处理方式： + +- `frontend/src/api/flood.js` 只暴露后端当前真实支持的接口。 +- 暂时移除或注释未实现接口，例如 `/flood/sources`、`/flood/ready-products`、`/flood/pairs` 保存删除、`/flood/reports`、`/flood/results`。 +- 后端 `/flood` 路由继续保留现有功能，但接口命名统一为 `preprocess`、`scenes`、`water-extractions`、`pairs/search`、`detections`、`detections/{id}/preview/{layer}`。 + +验收标准： + +```text +npm run build 通过 +前端 flood API 文件中没有明显会 404 的已导出函数 +/flood 主流程现有功能不退化 +``` + +## 4. 阶段 2：新增数据模型 + +目标：建立洪涝流水线需要的数据承载，不直接破坏旧表。 + +新增模型： + +```text +WaterExtractionORM -> water_extractions +FloodOverlayORM -> flood_overlays +FloodProductORM -> flood_products +``` + +关键取舍： + +- 保留旧 `WaterDetectionORM -> water_detections`。 +- 新表 `water_extractions` 从旧表回填。 +- 后续新任务写入 `water_extractions`。 +- 旧接口读旧表或兼容映射，等稳定后再清理。 + +建议字段： + +```text +water_extractions: + id + scene_id + processor + task_id + input_path + output_path + preview_path + vector_path + water_area_km2 + water_pixel_count + threshold_value + metadata_json + status + error_msg + created_at + updated_at + +flood_overlays: + id + detection_id + flood_vector_path + hazard_points_hit + hazard_points_near + hazard_points_total + dinsar_products_intersecting + affected_area_km2 + summary_json + created_at + +flood_products: + id + product_id + detection_id + overlay_id + display_name + status + publish_dir + manifest_path + summary_json + created_at +``` + +验收标准： + +```text +alembic upgrade head 成功 +旧 water_detections 数据可迁移到 water_extractions +模型 import 正常 +``` + +## 5. 阶段 3：抽离 Service + +目标：让 `/flood` 成为真正的业务路由，而不是代理 `water.py` 的壳。 + +新增服务： + +```text +backend/app/services/flood_analysis_service.py +backend/app/services/water_extraction_service.py +backend/app/services/flood_product_service.py +backend/app/services/flood_overlay_service.py +``` + +职责划分： + +```text +flood_analysis_service.py + 场景列表、预处理提交、配对搜索、洪涝检测提交、检测列表 + +water_extraction_service.py + Otsu 水体提取、ENVI/SARscape 水体提取 + +flood_product_service.py + 产品列表、manifest、产品包生成 + +flood_overlay_service.py + 分类栅格矢量化、灾害点/DInSAR/AOI 套合 +``` + +完成后： + +- `backend/app/routers/flood.py` 不再 `import water as water_compat`。 +- `backend/app/routers/water.py` 标记 deprecated，后续可反向调用新 service。 +- `water_detect_service.py` 逐步迁移到 `water_extraction_service.py`。 + +验收标准： + +```text +flood.py 不再 import water.py +/water/* 旧接口仍可用 +/flood/* 主接口可用 +``` + +## 6. 阶段 4：补齐产品端点 + +目标：让“结果与任务”视图有真实后端数据。 + +主接口建议使用 `products`，避免和 DInSAR result/product 概念混淆： + +```text +POST /flood/detections/{id}/products +GET /flood/products +GET /flood/products/{id} +GET /flood/products/{id}/manifest +``` + +兼容别名可选： + +```text +GET /flood/results +GET /flood/results/{id} +GET /flood/results/{id}/manifest +``` + +验收标准： + +```text +前端结果视图能展示真实产品 +manifest 能返回 JSON +没有空接口或静默失败 +``` + +## 7. 阶段 5：实现套合分析 + +目标：把洪涝模块从检测工具提升为灾害分析模块。 + +新增接口： + +```text +POST /flood/detections/{id}/overlay +GET /flood/detections/{id}/impact +``` + +处理逻辑： + +```text +1. 读取 flood_detections.classified_path +2. 提取 class=2 洪涝区域 +3. 栅格转矢量 +4. 写 flood_overlays.flood_vector_path +5. 查询灾害点命中 +6. 查询近邻风险点 +7. 查询相交 DInSAR 产品 +8. 查询 AI 诊断摘要 +9. 写 summary_json +``` + +注意： + +- AOI、PostGIS、DInSAR 产品几何不完整时，不让整个任务失败。 +- 返回部分结果，并在 `summary_json.warnings` 中说明缺失项。 + +验收标准： + +```text +POST overlay 能生成 flood_overlays 记录 +GET impact 返回结构化 JSON +没有数据时返回空数组而不是 500 +``` + +## 8. 阶段 6：前端重整 + +目标：只保留一个洪涝灾害分析工作台。 + +新增目录： + +```text +frontend/src/components/flood/ +``` + +建议组件： + +```text +FloodStatusBadge.jsx +FloodButton.jsx +FloodSceneRow.jsx +FloodWaterExtractionRow.jsx +FloodDetectionRow.jsx +FloodOverlayPanel.jsx +FloodProductPanel.jsx +``` + +界面改成四站式： + +```text +1. 场景准备 +2. 水体提取 +3. 洪涝检测 +4. 套合与产品 +``` + +处理方式： + +- `WaterMonitorPanel.jsx` 加弃用提示。 +- 左侧导航只引导用户进入洪涝灾害分析。 +- `api/water.js` 和 `api/gf3.js` 标记 deprecated。 +- 新功能只接入 `api/flood.js`。 + +验收标准： + +```text +左侧导航只引导用户进入洪涝灾害分析 +水体提取不再作为独立业务重复出现 +套合分析和产品视图接真实接口 +npm run build 通过 +``` + +## 9. 阶段 7：报告生成壳 + +目标：先形成 Markdown 报告能力，再扩展 PDF。 + +新增接口： + +```text +POST /flood/reports +GET /flood/reports/{id} +``` + +第一版报告包含： + +```text +灾前/灾后场景信息 +洪涝面积 +灾害点命中 +DInSAR 产品关联 +AI 诊断摘要 +套合统计 +``` + +## 10. 推荐提交顺序 + +```text +1. refactor flood api contract +2. add flood pipeline orm models +3. extract flood analysis services +4. add flood product endpoints +5. implement flood overlay impact analysis +6. consolidate flood frontend workspace +7. scaffold flood report generation +``` + +核心思路：先稳住接口和数据，再拆服务，再补业务能力，最后整理 UI。这样 Git 中的代码、迁移、前端调用和部署环境是闭合的，另一台服务器拉取后不会出现关键页面依赖未实现接口的问题。 + +## 11. 2026-05-15 补充设计：水体提取、灾害配对与套合展示 + +本节根据最新审阅意见补充。结论：洪涝工作台不应该继续以“手动选日期范围 + 查配对”为核心，而应该改成“灾害事件驱动”： + +```text +灾害时间 + 灾害位置 + -> 过滤可用 SAR 场景匹配池 + -> 推荐灾前/灾后配对 + -> 执行洪涝检测 + -> 套合分析 + -> 结果展示与产品导出 +``` + +### 11.1 水体提取视图复用管理页能力 + +当前管理页面已经具备三类能力，洪涝模块应复用，而不是重新做一套弱化版： + +1. 雷达数据查询能力：`/radar-data/search` 已支持成像日期、成像模式、极化方式、产品级别、行政区 AOI、上传 AOI 文件。 +2. 地图能力：`App.jsx` 中已经有源影像 footprint 上图、源影像预览缓存 `radar-data/{id}/thumb`、地图定位和图层开关逻辑。 +3. 行政区 AOI 能力：`/aoi/regions/children` 与 `/aoi/regions/{treeId}/geometry` 已可用于按省/市范围查询和定位。 + +因此水体提取界面调整为： + +```text +左侧：场景匹配池 + - 灾害位置 / AOI + - 成像日期范围 + - 卫星、模式、极化、产品级别 + - 只显示有 footprint 的数据 + - 可显示源影像预览图 + - 显示成像时间、极化方式、成像模式、产品级别 + +右侧：已完成地理编码场景 / 水体提取结果 + - 场景 footprint 上图 + - 水体提取掩膜上图 + - 与源影像预览可叠加对比 +``` + +前端复用建议： + +```text +复用 buildRadarSearchFormData / normalizeRadarSearchCriteria +复用 UnifiedDatePicker +复用 RadarDataRow 的预览状态表达 +复用 App.jsx 中 updateRadarPreviewVisibility 的源影像预览图层逻辑 +复用行政区 AOI 选择和地图定位逻辑 +``` + +需要新增的洪涝专用组件： + +```text +FloodSourceSearchPanel.jsx + 封装灾害位置、日期、极化、模式、产品级别过滤。 + +FloodSourceSceneRow.jsx + 显示源影像：成像时间、极化方式、模式、产品级别、预览状态、上图按钮。 + +FloodWaterExtractionRow.jsx + 显示水体提取结果：面积、状态、输入场景、上图按钮、错误信息。 +``` + +地图图层约定： + +```text +source_preview:{radar_data_id} 源影像预览 +source_footprint:{radar_data_id} 源影像覆盖范围 +scene_footprint:{scene_id} 地理编码场景范围 +water_mask:{water_extraction_id} 水体提取掩膜 +``` + +### 11.2 洪涝检测配对改为灾害事件驱动 + +当前 `/flood/pairs/search` 只接收灾前/灾后日期区间和 overlap 阈值，实际使用体验不够：用户通常知道的是“灾害发生时间”和“灾害位置”，不是一开始就知道灾前灾后影像窗口。 + +新的配对入口应改为： + +```text +灾害名称 disaster_name 可选 +灾害时间 disaster_date 必填 +灾害位置 disaster_aoi 必填，来自行政区 / 地图框选 / 上传 SHP / GeoJSON +灾前窗口 pre_window_days 默认 30 天 +灾后窗口 post_window_days 默认 30 天 +最小 AOI 覆盖率 min_aoi_coverage_ratio 默认 0.3 +最小两景重叠率 min_pair_overlap_ratio 默认 0.5 +是否要求同极化 require_same_polarization 默认 true +是否要求同成像模式 require_same_imaging_mode 默认 false +卫星过滤 satellites 可选 +极化过滤 polarization 可选 +``` + +前端交互： + +```text +1. 用户选择灾害日期 +2. 用户选择灾害位置 + - 行政区：省 / 市 + - 地图框选：后续实现 + - 上传 SHP/GeoJSON：沿用现有 AOI 解析 +3. 系统自动推导： + - 灾前窗口：disaster_date - pre_window_days 至 disaster_date - 1 + - 灾后窗口：disaster_date 至 disaster_date + post_window_days +4. 后端返回： + - pre_pool + - post_pool + - candidate_pairs + - warnings +5. 用户在候选配对中选择一组提交洪涝检测 +``` + +推荐新增接口： + +```text +POST /flood/disaster-pairs/search +``` + +请求格式： + +```json +{ + "disaster_name": "汶川洪涝", + "disaster_date": "20260715", + "aoi_geojson": {}, + "region_tree_id": "510000", + "pre_window_days": 30, + "post_window_days": 30, + "min_aoi_coverage_ratio": 0.3, + "min_pair_overlap_ratio": 0.5, + "require_same_polarization": true, + "require_same_imaging_mode": false, + "satellites": ["LT-1", "GF3"], + "polarization": "VV" +} +``` + +返回格式： + +```json +{ + "disaster": { + "name": "汶川洪涝", + "date": "20260715", + "pre_range": ["20260615", "20260714"], + "post_range": ["20260715", "20260814"] + }, + "aoi": { + "source": "region", + "name": "汶川县", + "aoi_geojson": {} + }, + "pre_pool": [], + "post_pool": [], + "candidate_pairs": [ + { + "pre": { + "scene_id": 1, + "radar_data_id": 10, + "imaging_date": "20260701", + "satellite": "LT-1", + "polarization": "VV", + "imaging_mode": "SM", + "aoi_coverage_ratio": 0.82, + "coverage_polygon": [] + }, + "post": { + "scene_id": 2, + "radar_data_id": 11, + "imaging_date": "20260718", + "satellite": "LT-1", + "polarization": "VV", + "imaging_mode": "SM", + "aoi_coverage_ratio": 0.79, + "coverage_polygon": [] + }, + "pair_overlap_ratio": 0.91, + "pre_delta_days": 14, + "post_delta_days": 3, + "score": 0.86, + "warnings": [] + } + ], + "warnings": [] +} +``` + +排序建议： + +```text +score = + pair_overlap_ratio * 0.35 + + min(pre_aoi_coverage, post_aoi_coverage) * 0.30 + + time_score * 0.20 + + same_polarization_bonus * 0.10 + + same_mode_bonus * 0.05 +``` + +后端实现建议： + +```text +1. 复用 radar search 的 AOI 解析能力，避免重复解析行政区和 SHP。 +2. 查询 sar_scene_geo.status=DONE 的场景，并 join radar_data。 +3. 按 disaster_date 自动构造灾前/灾后窗口。 +4. 用 PostGIS 计算单景 AOI 覆盖率。 +5. 用 footprint 相交面积计算 pair_overlap_ratio。 +6. 返回匹配池和候选配对，而不是只返回最终 pairs。 +``` + +兼容处理： + +```text +旧接口 POST /flood/pairs/search 保留，但只作为手动日期模式。 +新工作台默认使用 POST /flood/disaster-pairs/search。 +``` + +### 11.3 套合分析结果展示逻辑 + +套合分析不能只是一个“运行按钮”。它应展示“为什么这个洪涝结果重要”： + +```text +洪涝范围 +灾害点命中 +近洪涝风险点 +DInSAR 产品关联 +行政区影响统计 +结果图层 +产品/报告入口 +``` + +前端 `FloodOverlayPanel.jsx` 设计： + +```text +顶部摘要： + - 洪涝面积 + - 命中灾害点数量 + - 近洪涝风险点数量 + - 关联 DInSAR 产品数量 + - warnings 数量 + +中部地图控制： + - 洪涝分类图 + - 洪涝矢量范围 + - 命中灾害点 + - 近洪涝风险点 + - DInSAR 产品 footprint + +下部结果表： + - 灾害点列表：名称、类型、行政区、距离、是否命中 + - DInSAR 产品列表：product_id、engine、形变量、AI 风险等级、预览/详情 + - AOI 统计：行政区、洪涝面积、占比 +``` + +后端 `GET /flood/detections/{id}/impact` 应保证即使没有运行套合，也返回稳定结构： + +```json +{ + "detection_id": 1, + "flood_area_km2": 12.5, + "hazard_points": { + "inside_flood": [], + "near_flood": [], + "total_in_scene": 0 + }, + "dinsar_products": [], + "affected_aois": [], + "map_layers": { + "classified_preview": true, + "flood_vector_path": null, + "hazard_points": true, + "dinsar_footprints": true + }, + "warnings": [] +} +``` + +需要新增或完善的地图层约定： + +```text +flood_classified:{detection_id} 洪涝分类栅格预览 +flood_vector:{overlay_id} 洪涝矢量面 +hazard_inside:{overlay_id} 洪涝范围内灾害点 +hazard_near:{overlay_id} 近洪涝风险点 +dinsar_intersect:{overlay_id} 相交 DInSAR 产品 footprint +``` + +### 11.4 前端整改顺序调整 + +基于以上补充，前端整改顺序调整为： + +```text +1. 抽 FloodSourceSearchPanel，复用管理页雷达搜索/AOI 查询。 +2. 水体提取页增加源影像预览图、成像时间、极化方式、模式、产品级别。 +3. 洪涝检测页改为灾害事件输入：灾害时间 + 灾害位置。 +4. 新增 /flood/disaster-pairs/search 后接入候选池和推荐配对。 +5. 套合分析页接入 /flood/detections/{id}/overlay 和 /impact。 +6. 结果与任务页从 flood_detections 过渡到 flood_products。 +``` + +验收标准补充： + +```text +水体提取： + - 能按行政区/上传 AOI 查询源影像 + - 能显示源影像预览图 + - 列表中显示成像日期、极化方式、成像模式、产品级别 + - 水体提取结果可叠加到地图 + +洪涝检测： + - 用户只需输入灾害时间和灾害位置即可获得匹配池 + - 返回 pre_pool/post_pool/candidate_pairs + - 推荐配对可在地图上预览灾前/灾后 footprint + - 配对结果显示时间差、AOI 覆盖率、两景重叠率、极化一致性 + +套合分析： + - 运行后能看到摘要指标 + - 能看到灾害点命中和近邻风险点列表 + - 能看到关联 DInSAR 产品列表 + - warnings 可见，不静默失败 +``` + +## 12. 2026-05-15 第一轮落地记录 + +本轮先闭合三条可直接使用的业务链路，不把报告生成和完整产品包导出放进同一次改动。 + +### 12.1 已落地 + +后端： +```text +1. 新增 POST /flood/disaster-pairs/search。 + 输入 disaster_date + region_tree_id/aoi_geojson + 灾前灾后窗口。 + 输出 pre_pool、post_pool、candidate_pairs、summary、warnings。 + +2. 水体提取列表补充源影像元数据。 + /flood/scenes 与 /flood/water-extractions 现在返回 imaging_date、polarization、 + imaging_mode、product_level、coverage_polygon、min/max lon/lat。 + +3. 套合分析接口保持并接入前端。 + POST /flood/detections/{id}/overlay + GET /flood/detections/{id}/impact +``` + +前端： +```text +1. 洪涝灾害分析 / 水体提取页： + - 复用 AOI 行政区索引。 + - 支持按行政区筛选雷达源影像。 + - 雷达结果行增加“覆盖”和“源影像”上图。 + - 场景/水体结果行显示成像日期、极化方式、模式、产品级别。 + +2. 洪涝检测页： + - 从手工填写灾前/灾后日期，改为输入灾害名称、灾害发生日期、灾害位置。 + - 自动根据窗口期生成灾前池和灾后池。 + - 候选配对展示评分、AOI 覆盖率、两景重叠率、灾前/灾后时间差、极化方式。 + +3. 套合分析页： + - 可运行 overlay。 + - 可刷新并展示 impact。 + - 展示洪涝面积、命中灾害点、近洪涝风险点、关联 DInSAR 产品和 warnings。 +``` + +### 12.2 当前保留的兼容点 + +```text +1. GET/POST /water/* 暂未删除，旧页面仍可过渡使用。 +2. 旧 POST /flood/pairs/search 保留为手工日期兼容接口，但新工作台默认使用 /flood/disaster-pairs/search。 +3. 水体提取任务底层仍写 water_detections；water_extractions 新表和正式迁移已建模，但任务处理器尚未完全切换。 +4. AOI 只先复用行政区查询；上传 SHP/GeoJSON 到洪涝工作台可作为下一步补充。 +``` + +### 12.3 下一轮建议 + +```text +1. 把水体提取 job handler 从 WaterDetectionORM 切到 WaterExtractionORM。 +2. 给候选配对增加地图 footprint 预览按钮，辅助人工确认灾前/灾后覆盖。 +3. 把 overlay 生成的 flood_vector_path 也作为地图矢量层显示。 +4. 套合分析补行政区受影响面积统计。 +5. 结果与任务页开始接 flood_products，而不是只列 flood_detections。 +``` diff --git a/docs/FLOOD_MODULE_REFACTOR_PROGRESS_20260515.md b/docs/FLOOD_MODULE_REFACTOR_PROGRESS_20260515.md new file mode 100644 index 0000000..d047fb8 --- /dev/null +++ b/docs/FLOOD_MODULE_REFACTOR_PROGRESS_20260515.md @@ -0,0 +1,128 @@ +# 洪涝模块整改进展记录 + +## 2026-05-15 第二轮：水体提取表切换 + +本轮目标是把洪涝工作台里的水体提取正式从旧 `water_detections` +迁到新 `water_extractions`，同时保留旧 `/water/*` 兼容窗口。 + +### 已完成 + +```text +1. 新增 backend/app/services/water_extraction_service.py。 + - run_otsu_water_extraction 复用旧 Otsu 实现。 + - 对外使用 extraction/processor/threshold_value 命名。 + - run_envi_water_extraction 暂留占位，后续接 ENVI/SARscape。 + +2. /flood/water-extractions 新提交任务开始写 water_extractions。 + - 任务 payload 使用 extraction_id。 + - WaterExtractionORM.processor 默认 otsu。 + - task_id、threshold_value、metadata_json 随任务更新。 + +3. WATER_DETECT job handler 支持双轨兼容。 + - 新 payload: extraction_id -> 写 water_extractions。 + - 旧 payload: detection_id -> 继续写 water_detections。 + - 如果旧 detection_id 已经被 alembic 0003 回填到 water_extractions， + handler 会同步更新同 ID 的新表记录，避免迁移期状态卡住。 + +4. /flood/water-extractions 列表改读 water_extractions。 + - 旧数据依赖 alembic 0003 从 water_detections 回填。 + - preview 优先读 WaterExtractionORM，找不到时回退 WaterDetectionORM。 +``` + +### 仍保留 + +```text +1. 旧 /water/detect 仍创建 water_detections。 + 这是兼容窗口内的刻意保留，不再作为洪涝工作台主链路。 + +2. ENVI/SARscape 精密水体提取还没有接入任务队列。 + +3. 如果部署环境已有旧 PENDING water_detections 任务， + 需要先执行 alembic 0003，再启动 worker。 +``` + +### 下一步 + +```text +1. 给候选洪涝配对增加灾前/灾后 footprint 地图预览。 +2. 把 flood_vector_path 作为矢量图层上图。 +3. 套合分析补行政区受影响面积统计。 +4. 结果与任务页接 flood_products。 +``` + +## 2026-05-15 第三轮：地图闭环与产品页 + +### 已完成 + +```text +1. 候选洪涝配对支持地图预览。 + - 灾前 footprint 蓝色。 + - 灾后 footprint 绿色。 + - 配对行增加“预览覆盖”。 + +2. 套合结果支持洪涝矢量上图。 + - GET /flood/detections/{id}/impact 返回 overlay_id、flood_vector_path、flood_vector_geojson。 + - 前端套合分析页增加“加载洪涝矢量”。 + +3. 套合分析补行政区影响统计。 + - overlay 运行时读取 AOI 行政区边界索引。 + - 返回 affected_aois，包含 tree_id、name、level、flood_area_km2。 + - 前端展示影响行政区数量和前 5 个行政区。 + +4. 结果与任务页接 flood_products。 + - 洪涝结果行增加“生成产品”。 + - 结果页增加“产品”tab。 + - 产品行展示 product_id、洪涝面积、影响面积、生成时间。 + - Manifest 按钮读取 /flood/products/{id}/manifest。 +``` + +### 当前可集中测试的链路 + +```text +1. 水体提取： + 入库影像查询 -> 行政区筛选 -> 源影像上图 -> 提交预处理 -> 提交水体提取 -> 水体结果上图。 + +2. 洪涝检测： + 输入灾害时间 + 灾害位置 -> 推荐配对 -> 预览覆盖 -> 提交洪涝检测 -> 分类图上图。 + +3. 套合分析： + 选择 DONE 洪涝结果 -> 运行套合分析 -> 加载洪涝矢量 -> 查看灾害点、DInSAR、行政区统计。 + +4. 产品： + DONE 洪涝结果 -> 生成产品 -> 产品 tab 查看 -> 读取 manifest。 +``` + +## 2026-05-15 第四轮：GeoTIFF 化技术决策 + +本轮根据新的产品判断调整洪涝模块技术路线：不再把 ENVI/SARscape +作为洪涝算法主线，后续水体提取、洪涝检测、套合分析全部运行在标准 +GeoTIFF 层。原始 SAR 到 GeoTIFF 的部分做成可插拔前处理器，优先评估 +GAMMA，GAMMA 覆盖不了的传感器使用 GF3_GDAL、SNAP、ISCE 或 external +处理器兜底。 + +### 已完成 + +```text +1. 新增 docs/FLOOD_GEOTIFF_GAMMA_PREPROCESS_DESIGN_20260515.md。 + - 总结当前洪涝模块仍绑定 ENVI/SARscape 的真实状态。 + - 梳理当前 GAMMA/PyINT 在项目中的能力和边界。 + - 明确 GAMMA 适合作为 TIF 前处理候选，但不能把 D-InSAR pair profile + 直接复用为洪涝单景预处理。 + - 定义 analysis_ready.tif / SarAnalysisScene 数据契约。 + - 给出 Python GeoTIFF 洪涝检测和 GAMMA 单景前处理分阶段方案。 + +2. 更新 docs/INDEX.md。 + - 将 GeoTIFF 化设计列入当前执行中的核心设计。 + - 标明 2026-05-14 旧洪涝设计中的 ENVI/SARscape 主线表述已被新设计取代。 +``` + +### 当前判断 + +```text +1. 洪涝算法应先从 SARscape 黑盒迁出，改成 pre/post GeoTIFF 上的 Python 算法。 +2. GAMMA 适合优先做 LT-1/Sentinel-1 的 analysis_ready.tif 生产器。 +3. 当前 GAMMA 接入已有 geocode/data2geotiff 能力，但缺少独立单景预处理服务。 +4. GF3 已有 Python/GDAL L1A->L2 路线，不应强行改走 GAMMA。 +5. 另一台服务器部署时，Git 能提供平台代码和算法；GAMMA 本体、DEM、轨道池、 + conda/WSL 运行时仍是外部部署前提。 +``` diff --git a/docs/FRONTEND_NAVIGATION_ARCHITECTURE.md b/docs/FRONTEND_NAVIGATION_ARCHITECTURE.md index ad77447..6139f97 100644 --- a/docs/FRONTEND_NAVIGATION_ARCHITECTURE.md +++ b/docs/FRONTEND_NAVIGATION_ARCHITECTURE.md @@ -68,9 +68,8 @@ This is a workspace group, not a multi-tab planning tree. 生产管理 └─ 生产管理 (`production_management`) ├─ D-InSAR运行 (`dinsar_runs`) - ├─ 时序InSAR运行 (`timeseries_runs`) - ├─ D-InSAR产物 (`dinsar_products`) - └─ 时序InSAR产物 (`timeseries_products`) + ├─ SBAS-InSAR Production (`sbas_insar_production`) + └─ D-InSAR产物 (`dinsar_products`) ``` Notes: @@ -78,7 +77,8 @@ Notes: - The left navigation contains only one tab for this group: `production_management`. - Internal workspace views are controlled by `PRODUCTION_WORKSPACE_VIEWS`. - Route alias mapping is controlled by `PRODUCTION_WORKSPACE_ENTRY_TO_VIEW`. -- Legacy route tabs such as `dinsar_production` and `ps_products` map into this workspace and should not be treated as standalone left-nav entries. +- Legacy route tabs such as `dinsar_production`, `ps_production`, and `ps_products` map into this workspace and should not be treated as standalone left-nav entries. +- The old ISCE2/MintPy `timeseries_runs` and `timeseries_products` workspace views are deprecated and hidden; SBAS production is handled by the Gamma `sbas_insar_production` view. ### 3.3 InSAR形变分析 diff --git a/docs/GAMMA_IPTA_LT1_SBAS_TRIAL_RUNBOOK_20260518.md b/docs/GAMMA_IPTA_LT1_SBAS_TRIAL_RUNBOOK_20260518.md new file mode 100644 index 0000000..3dca496 --- /dev/null +++ b/docs/GAMMA_IPTA_LT1_SBAS_TRIAL_RUNBOOK_20260518.md @@ -0,0 +1,551 @@ +# Gamma IPTA LT1 SBAS Trial Runbook + +Date: 2026-05-18 + +## Goal + +Validate whether the current local LT1 data pool and WSL Gamma installation can produce one usable SBAS/IPTA trial result with the official Gamma toolchain. + +This is not yet a production integration design. The immediate goal is to run one conservative stack, inspect failures and product quality, then decide what should be productized in the system. + +Actual trial root used in this repository: + +```text +D:\Code\Insar_management_system_v2\backend\runtime\gamma_ipta_trials\lt1b_r114_e1312_n438_20240516_20251002 +``` + +## Current Environment + +- WSL distro configured by the project: `Ubuntu-24.04`. +- Gamma install found in WSL: `/usr/local/GAMMA_SOFTWARE-20240627`. +- Installed package name present on disk: `GAMMA_SOFTWARE-20240627_MSP_ISP_DIFF_IPTA.linux64_ubuntu2404.tar.gz`. +- Installed Gamma modules include `MSP`, `ISP`, `DIFF`, `DISP`, and `IPTA`. +- `GEO` is not a separate directory in this install, but DIFF contains the relevant geocoding tools, including `gc_map`, `geocode`, and `geocode_back`. +- Project Gamma environment script: `deploy/wsl/profiles/gamma_env.sh`. +- Project WSL Python: `/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python`. + +Important command checks already performed: + +- `IPTA/bin/ts_rate` runs and prints usage. No license-denied error observed. +- `IPTA/bin/multi_def_pt` runs and prints usage. No license-denied error observed. +- `ISP/bin/par_LT1_SLC` exists and prints LT1 SLC conversion usage. +- `ISP/scripts/LT1_precision_orbit.py` runs with the project conda Python and prints usage. + +## Data Pool Findings + +Configured LT1 source pool: + +- `D:\LuTan1_Image_Pool` + +Configured LT1 precise orbit pool: + +- `D:\orbit_pools\envi\LT1A` +- `D:\orbit_pools\envi\LT1B` + +High-level inventory: + +- LT1 scene directories found: `1500`. +- LT1A orbit TXT files found: `896`, spanning `20230508` to `20251219`. +- LT1B orbit TXT files found: `858`, spanning `20230510` to `20251219`. + +Metadata caveat: + +- `*.meta.xml` and `*_Check.xml` are not safe to parse as whole XML documents in PowerShell because many files contain malformed Chinese text near the tail, for example bad `usePreciseOrbit` closing text. +- The needed `productInfo` block is structurally valid. For stack discovery, parse only `...`. +- Gamma `par_LT1_SLC` should still use the original `.meta.xml`; do not rewrite source metadata unless a Gamma run proves the malformed tail is a blocker. + +## Candidate Stack + +First trial should use a narrow single-center stack, not the broad system time-series grouping. + +Recommended trial stack: + +- Satellite: `LT1B` +- Relative orbit: `114` +- Direction: `DESCENDING` +- Imaging mode: `STRIP1` +- Polarization: `HH` +- Approximate center: `E131.2 / N43.8` +- Scene count at this center: `7` +- Scenes with precise orbit TXT currently present: `5` + +Use the 5 scenes with available precise orbit first: + +| Date | Orbit TXT | Scene | +| --- | --- | --- | +| 20240516 | yes | `LT1B_MONO_SYC_STRIP1_012047_E131.2_N43.8_20240516_SLC_HH_S2A_0000399289` | +| 20240711 | yes | `LT1B_MONO_SYC_STRIP1_012880_E131.2_N43.8_20240711_SLC_HH_S2A_0000450956` | +| 20240905 | yes | `LT1B_MONO_SYC_STRIP1_013713_E131.2_N43.8_20240905_SLC_HH_S2A_0000501650` | +| 20250417 | yes | `LT1B_MONO_SYC_STRIP1_017045_E131.2_N43.8_20250417_SLC_HH_S2A_0000713375` | +| 20251002 | yes | `LT1B_MONO_SYC_STRIP1_019544_E131.2_N43.8_20251002_SLC_HH_S2A_0000891257` | + +Do not include these two in the first run unless the missing orbits are added: + +| Date | Orbit TXT | Scene | +| --- | --- | --- | +| 20250612 | no | `LT1B_MONO_SYC_STRIP1_017878_E131.2_N43.8_20250612_SLC_HH_S2A_0000772122` | +| 20250807 | no | `LT1B_MONO_SYC_STRIP1_018711_E131.2_N43.8_20250807_SLC_HH_S2A_0000831367` | + +Common bounding box across the broader 13-scene `LT1B relOrbit 114 / E131-N44` candidate: + +- lon: `130.8615 .. 131.1638` +- lat: `43.7127 .. 44.0987` + +For the narrow `E131.2/N43.8` trial stack, overlap is visually/metadata-wise much tighter: + +- Each scene center is around `131.20E, 43.79N`. +- Each scene bbox is roughly `130.81..131.62E`, `43.48..44.10N`. + +Secondary candidate if the first stack fails: + +- `LT1B relOrbit 114 DESCENDING STRIP1 HH`, center `E130.8/N43.9`. +- 5 dates, 4 with orbit: `20250425`, `20250620`, `20250815`, `20251010`. +- One scene is `MONO_MH1` while the others are `MONO_SYC`; keep it as secondary, not first choice. + +## Current System Pairing Limitations + +The current time-series stack selection is useful for broad discovery but is too coarse for Gamma IPTA production. + +Observed code behavior: + +- LT1A/LT1B are normalized into the same satellite family `LT1`. +- The compatibility key only uses direction, satellite family, imaging mode, and polarization. +- Relative orbit, absolute track family, scene center/strip identity, receiving station, and detailed LT1 product variant are not hard grouping keys. +- The stable-stack selector tries to recover by common AOI overlap and pairwise network connectivity. +- The SBAS network selector uses time-baseline, center-distance and overlap thresholds, but does not use Gamma-derived perpendicular baseline at planning time. +- Time-series processors currently accepted by the service are only `isce2_stack_mintpy` and `sarscape_sbas`; there is no `gamma_ipta_sbas` processor code yet. + +For the Gamma IPTA trial, do not rely on the current automatic PS stack plan as the source of truth. Use a manually audited stack manifest first. + +## Trial Run Checklist + +### 1. Create Isolated Work Directory + +Actual root: + +```text +D:\Code\Insar_management_system_v2\backend\runtime\gamma_ipta_trials\lt1b_r114_e1312_n438_20240516_20251002 +``` + +Keep these subdirectories: + +```text +input\scenes +input\orbits +gamma\slc +gamma\mli +gamma\diff +gamma\ipta +logs +publish +``` + +For the first trial, prefer symlinks or a manifest that points to source scenes. Avoid duplicating large TIFF files unless Gamma scripts require local flat layout. + +### 2. Build Scene Manifest + +For each selected scene, record: + +- scene directory +- `.tiff` +- `.meta.xml` +- precise orbit TXT +- date +- relative orbit +- direction +- mode +- polarization +- center lon/lat +- bbox + +This manifest becomes the hand-audited truth for the trial. + +### 3. Convert LT1 Products To Gamma SLC + +For each selected scene: + +```bash +source /mnt/d/Code/Insar_management_system_v2/deploy/wsl/profiles/gamma_env.sh +par_LT1_SLC .slc.par .slc +``` + +Then apply precise orbit: + +```bash +/home/administrator/miniconda3/envs/insar_wsl_v1/bin/python \ + /usr/local/GAMMA_SOFTWARE-20240627/ISP/scripts/LT1_precision_orbit.py \ + .slc.par +``` + +### 4. Build SLC/MLI Tables + +Create Gamma tables for the stack: + +```text +SLC_tab +RMLI_tab +``` + +Use multilook settings conservative enough for a first run. The goal is robustness and fast feedback, not final product resolution. + +### 5. Baseline And Pair Network + +Use Gamma baseline tools first, not the system's center-distance approximation: + +- `base_init` +- `base_perp` +- IPTA baseline tools such as `base_orbit_pt`, `base_par_pt`, `base_ls_pt` as needed by the official IPTA path. + +For the first 5-scene stack, start with a simple connected small-baseline network: + +- adjacent pairs by time +- add one or two skip pairs only if coherence and baseline look acceptable + +Expected adjacent temporal intervals: + +- `20240516 -> 20240711`: 56 days +- `20240711 -> 20240905`: 56 days +- `20240905 -> 20250417`: 224 days +- `20250417 -> 20251002`: 168 days + +The large seasonal gaps are acceptable for a trial only if perpendicular baseline and coherence are reasonable. If they are poor, switch to a denser 2025-only or 2024-only local test, even with fewer dates. + +### 6. Differential Interferograms + +Use official Gamma DIFF commands/scripts for: + +- coregistration +- interferogram generation +- simulated topographic phase +- differential phase +- filtering +- coherence +- unwrapping if needed by the chosen IPTA path + +Do not implement custom SBAS inversion in the management system. + +### 7. IPTA Processing + +Use Gamma IPTA commands for point/stack time-series processing. Confirm exact command sequence against the installed: + +```text +/usr/local/GAMMA_SOFTWARE-20240627/IPTA/html/IPTA_users_guide.pdf +``` + +Commands observed in the local IPTA module include: + +- `multi_def_pt` +- `ts_rate` +- `ts_rate_pt` +- `base_ls_pt` +- `base_par_pt` +- `ph_base_pt` +- `atm_mod_pt` +- `pt2geo` +- `dis_ipta` + +### 8. Review Outputs + +Minimum acceptance checks for the first run: + +- every selected scene converts to SLC +- precise orbit update succeeds for every SLC +- all intended pairs generate interferograms +- coherence is not uniformly poor +- unwrapping or IPTA point solution is not globally unstable +- one geocoded velocity or displacement-rate raster/vector product can be inspected +- logs and command manifests are complete enough to reproduce the run + +## Trial Progress On 2026-05-18 + +### Files Created For This Trial + +- `backend/runtime/gamma_ipta_trials/lt1b_r114_e1312_n438_20240516_20251002/input/scene_manifest.json` +- `backend/runtime/gamma_ipta_trials/lt1b_r114_e1312_n438_20240516_20251002/scripts/01_prepare_slc.sh` +- `backend/runtime/gamma_ipta_trials/lt1b_r114_e1312_n438_20240516_20251002/scripts/02_mli_and_baseline.sh` +- `backend/runtime/gamma_ipta_trials/lt1b_r114_e1312_n438_20240516_20251002/scripts/03_coreg_one_pair.sh` +- `backend/runtime/gamma_ipta_trials/lt1b_r114_e1312_n438_20240516_20251002/scripts/04_cc_stats.py` +- `backend/runtime/gamma_ipta_trials/lt1b_r114_e1312_n438_20240516_20251002/scripts/05_coreg_adjacent_pairs.sh` +- `backend/runtime/gamma_ipta_trials/lt1b_r114_e1312_n438_20240516_20251002/scripts/06_prepare_rdc_dem.sh` +- `backend/runtime/gamma_ipta_trials/lt1b_r114_e1312_n438_20240516_20251002/scripts/07_coreg_common_ref_stack.sh` +- `backend/runtime/gamma_ipta_trials/lt1b_r114_e1312_n438_20240516_20251002/scripts/08_diff_unwrap_common_ref.sh` +- `backend/runtime/gamma_ipta_trials/lt1b_r114_e1312_n438_20240516_20251002/scripts/09_mb_ts_rate.sh` +- `backend/runtime/gamma_ipta_trials/lt1b_r114_e1312_n438_20240516_20251002/scripts/10_float_stats.py` +- `backend/runtime/gamma_ipta_trials/lt1b_r114_e1312_n438_20240516_20251002/scripts/11_make_timeseries_previews.sh` +- `backend/runtime/gamma_ipta_trials/lt1b_r114_e1312_n438_20240516_20251002/scripts/12_geocode_export_timeseries.sh` +- `backend/runtime/gamma_ipta_trials/lt1b_r114_e1312_n438_20240516_20251002/scripts/13_phase_to_los.py` +- `backend/runtime/gamma_ipta_trials/lt1b_r114_e1312_n438_20240516_20251002/scripts/14_build_trial_summary.py` +- `backend/runtime/gamma_ipta_trials/lt1b_r114_e1312_n438_20240516_20251002/scripts/15_make_los_velocity_maps.sh` +- `backend/runtime/gamma_ipta_trials/lt1b_r114_e1312_n438_20240516_20251002/scripts/16_plot_monitor_point_timeseries.py` +- `backend/runtime/gamma_ipta_trials/lt1b_r114_e1312_n438_20240516_20251002/scripts/17_make_geocoded_web_previews.sh` + +### Completed + +- Created a hand-audited 5-scene LT1B manifest for `relOrbit 114 / DESCENDING / STRIP1 / HH / E131.2 N43.8`. +- Converted all 5 LT1 TIFF products with Gamma `par_LT1_SLC`. +- Applied precise orbit updates with Gamma `LT1_precision_orbit.py`. +- Built `gamma/slc/SLC_tab`. +- Built 8x8 multilooked MLI products with Gamma `multi_look`. +- Built `gamma/mli/RMLI_tab`. +- Generated all-pair and adjacent-pair baseline tables with Gamma `base_calc`. +- Ran adjacent-pair coregistration for all 4 selected adjacent pairs using Gamma `SLC_coreg.py`. +- Generated 4 interferograms and coherence products using Gamma `create_offset`, `SLC_intf`, and `cc_wave`. +- Generated browse previews using Gamma `rasmph` and `raspwr`. +- Computed coherence statistics for each adjacent pair. +- Reused the existing Copernicus30 DEM cache covering `E131.2/N43.8`. +- Generated a reference-geometry RDC DEM for `20240905` using Gamma `gc_map1`, `geocode`, and `gc_map_fine`. +- Built a common-reference RSLC/RMLI stack in `20240905` geometry. +- Generated common-reference differential interferograms with Gamma `phase_sim_orb` and `SLC_diff_intf`. +- Filtered common-reference differential interferograms with Gamma `adf`. +- Unwrapped common-reference differential interferograms with Gamma `mcf`. +- Ran Gamma IPTA `mb` to solve the image-based phase time-series. +- Ran Gamma IPTA `ts_rate` to estimate a linear phase-rate map. +- Generated Gamma preview rasters for `ts_rate`, `sigma_rate`, and `hgt_correction`. +- Exported phase-rate, sigma, height-correction, and LOS-rate rasters to EPSG:4326 GeoTIFF. +- Generated explicit LOS velocity maps in `mm/year` for both away-from-radar-positive and toward-radar-positive conventions. +- Generated one example monitoring-point LOS displacement curve as PNG, CSV, and metadata JSON. +- Generated north-up WGS84 web preview PNGs from the geocoded LOS GeoTIFF products. + +### Key Outputs + +- SLC stack: `gamma/slc/SLC_tab` +- MLI stack: `gamma/mli/RMLI_tab` +- All-pair baseline table: `gamma/diff/bperp_all_pairs.txt` +- Adjacent-pair table: `gamma/diff/itab_adjacent` +- Pair-specific RSLC products: `gamma/rslc/*_to_*.rslc` +- Pair-specific quality reports: `gamma/rslc/*_to_*.rslc.coreg_quality` +- Adjacent interferograms: `gamma/int/*.int` +- Adjacent coherence rasters: `gamma/int/*.cc` +- Adjacent interferogram previews: `gamma/int/*.int.bmp` +- Adjacent coherence previews: `gamma/int/*.cc.bmp` +- Coherence statistics: `logs/*_cc_stats.txt` +- Reference RDC DEM: `gamma/dem/20240905_8rlks.rdc.dem` +- Common-reference stack tables: `gamma/common_20240905/SLC_tab`, `gamma/common_20240905/RMLI_tab` +- Common-reference differential stack: `gamma/common_20240905/diff/*/*_8rlks.diff_filt.unw` +- Gamma `mb` time-series list: `gamma/common_20240905/timeseries/diff_ts.tab` +- Gamma `mb` phase time-series: `gamma/common_20240905/timeseries/diff_ts_*.diff` +- Gamma `mb` residual sigma: `gamma/common_20240905/timeseries/sigma_ts` +- Gamma `mb` height correction: `gamma/common_20240905/timeseries/hgt_correction` +- Gamma `ts_rate` output: `gamma/common_20240905/timeseries/ts_rate` +- Gamma `ts_rate` sigma output: `gamma/common_20240905/timeseries/sigma_rate` +- Preview rasters: `gamma/common_20240905/timeseries/*.bmp` +- Geocoded GeoTIFF exports: `publish/geotiff/*.tif` +- Explicit LOS velocity previews: `publish/geotiff/los_rate_toward_mm_per_year.bmp`, `publish/geotiff/los_rate_away_mm_per_year.bmp` +- Geocoded web previews: `publish/geotiff/los_rate_toward_mm_per_year.geo_preview.png`, `publish/geotiff/los_sigma_mm_per_year.geo_preview.png` +- Monitoring-point curve: `publish/monitor_points/auto_low_sigma_high_rate_timeseries.png` +- Monitoring-point values: `publish/monitor_points/auto_low_sigma_high_rate_timeseries.csv` +- Trial summary: `publish/trial_summary.json` + +### Baseline Notes + +`base_calc` generated 10 all-pair entries. The adjacent temporal network is: + +| Pair | Delta days | Bperp from `base_calc` | +| --- | ---: | ---: | +| 20240516 -> 20240711 | 56 | 646.09670 | +| 20240711 -> 20240905 | 56 | -236.76960 | +| 20240905 -> 20250417 | 224 | 249.35880 | +| 20250417 -> 20251002 | 168 | 59.18150 | + +The first validated pair was `20240711 -> 20240905` because it has a 56-day interval and moderate perpendicular baseline in this stack. + +### Adjacent-Pair Quality + +Final `SLC_coreg.py` quality-test summaries: + +| Pair | Accepted offsets | Final std range | Final std azimuth | +| --- | ---: | ---: | ---: | +| 20240516 -> 20240711 | 2476 / 2688 | 0.0861 | 0.1120 | +| 20240711 -> 20240905 | 2483 / 2656 | 0.0370 | 0.1245 | +| 20240905 -> 20250417 | 2322 / 2688 | 0.0487 | 0.0617 | +| 20250417 -> 20251002 | 2274 / 2688 | 0.0304 | 0.0533 | + +Coherence statistics from Gamma big-endian float files: + +| Pair | valid `[0,1]` | p25 | median | p75 | p99 | Comment | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| 20240516 -> 20240711 | 100% | 0.127489 | 0.212285 | 0.323520 | 0.649398 | Weakest pair; high Bperp and low coherence | +| 20240711 -> 20240905 | 100% | 0.629307 | 0.733220 | 0.802884 | 0.909973 | Strong pair | +| 20240905 -> 20250417 | 100% | 0.339094 | 0.485239 | 0.612798 | 0.833928 | Usable trial pair | +| 20250417 -> 20251002 | 100% | 0.394165 | 0.582921 | 0.734878 | 0.932300 | Usable trial pair | + +All adjacent pairs generated expected Gamma products. The full 5-scene chain can continue, but the first pair is a quality risk for unwrapping/IPTA. A more conservative follow-up is to run the 4-scene sub-stack from `20240711` to `20251002`, or keep the 5-scene stack but down-weight or exclude `20240516 -> 20240711` if later unwrapping/IPTA residuals are poor. + +Implementation note: + +- `SLC_coreg.py` writes the refined offset parameter file to a secondary-date path such as `gamma/slc/20240711.slc.off`. +- Trial scripts copy that file to pair-specific paths such as `gamma/rslc/20240711_to_20240905.rslc.off`. +- Future skip-pair or non-adjacent networks must use pair-specific offset files to avoid accidental reuse after the same secondary scene is coregistered to another reference. + +### Common-Reference Time-Series Trial + +The image-based Gamma IPTA path was also run using a common `20240905` reference geometry. This is closer to the official `mb -> ts_rate` time-series chain than the first adjacent-pair wrapped interferogram check. + +Common-reference inputs and products: + +- reference geometry: `20240905` +- DEM source: existing Copernicus30 cache under `backend/runtime/pyint_dem_cache` +- RDC DEM: `gamma/dem/20240905_8rlks.rdc.dem` +- common-reference SLC table: `gamma/common_20240905/SLC_tab` +- common-reference MLI table: `gamma/common_20240905/RMLI_tab` +- common-reference differential ITAB: + +```text +1 3 1 1 +2 3 2 1 +3 4 3 1 +3 5 4 1 +``` + +Filtered differential coherence statistics: + +| Pair | valid `[0,1]` | p25 | median | p75 | p99 | +| --- | ---: | ---: | ---: | ---: | ---: | +| 20240516 -> 20240905 | 100% | 0.497369 | 0.884586 | 0.945308 | 0.980583 | +| 20240711 -> 20240905 | 100% | 0.919149 | 0.968188 | 0.979449 | 0.990185 | +| 20240905 -> 20250417 | 100% | 0.366279 | 0.790519 | 0.909991 | 0.978094 | +| 20240905 -> 20251002 | 100% | 0.193104 | 0.815562 | 0.917209 | 0.972355 | + +Gamma `mb` outputs: + +- `gamma/common_20240905/timeseries/diff_ts_001.diff` through `diff_ts_005.diff` +- `gamma/common_20240905/timeseries/diff_ts.tab` +- `gamma/common_20240905/timeseries/itab_ts` +- `gamma/common_20240905/timeseries/sigma_ts` +- `gamma/common_20240905/timeseries/hgt_correction` + +Gamma `ts_rate` outputs: + +- `gamma/common_20240905/timeseries/ts_rate` +- `gamma/common_20240905/timeseries/ts_const` +- `gamma/common_20240905/timeseries/sigma_rate` +- `gamma/common_20240905/timeseries/ts_rate.bmp` +- `gamma/common_20240905/timeseries/sigma_rate.bmp` +- `gamma/common_20240905/timeseries/hgt_correction.bmp` + +GeoTIFF exports: + +- `publish/geotiff/ts_rate_rad_per_year.tif` +- `publish/geotiff/sigma_rate_rad_per_year.tif` +- `publish/geotiff/sigma_ts_rad.tif` +- `publish/geotiff/hgt_correction_m.tif` +- `publish/geotiff/los_rate_m_per_year.tif` +- `publish/geotiff/los_sigma_m_per_year.tif` +- `publish/geotiff/los_rate_away_mm_per_year.tif` +- `publish/geotiff/los_rate_toward_mm_per_year.tif` +- `publish/geotiff/los_sigma_mm_per_year.tif` +- `publish/geotiff/los_rate_toward_mm_per_year.geo_preview.png` +- `publish/geotiff/los_sigma_mm_per_year.geo_preview.png` +- `publish/trial_summary.json` + +Float output statistics using Gamma big-endian float: + +| File | Non-zero pixels | p25 | median | p75 | p99 | +| --- | ---: | ---: | ---: | ---: | ---: | +| `ts_rate` | 8,868,956 | -1.663257 | -0.271621 | 0.869116 | 2.787572 | +| `sigma_rate` | 8,868,956 | 0.396037 | 0.677578 | 1.010539 | 2.298520 | +| `sigma_ts` | 8,892,237 | 0.000704 | 0.001557 | 0.002535 | 0.518188 | +| `hgt_correction` | 8,892,250 | -16.989384 | 7.596325 | 36.411520 | 98.636933 | + +Explicit LOS velocity output statistics in `mm/year`: + +| File | Non-zero pixels | p01 | p25 | median | p75 | p99 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| `los_rate_toward_mm_per_year.rdc` | 8,868,956 | -52.779640 | -16.455760 | 5.142855 | 31.491957 | 84.590944 | +| `los_sigma_mm_per_year.rdc` | 8,868,956 | 0.530715 | 7.498536 | 12.829198 | 19.133450 | 43.519972 | + +This is a successful first local Gamma official-chain time-series trial. It is still a technical validation run, not a production-grade SBAS product: the stack has only 5 dates, the network is minimal, and reference region and unwrapping masks were conservative defaults. + +The exported GeoTIFFs are EPSG:4326, `2222 x 2237`, Float32, LZW-compressed Cloud Optimized GeoTIFFs with `NoData=0`. + +The `*.bmp` files generated by Gamma `rasdt_pwr` are RDC processing-geometry browse images. They are useful for quick processing QA but are not map products and should not be used as the default UI map preview. The UI/default web preview should use the `*.geo_preview.png` files generated from the EPSG:4326 GeoTIFF products. + +### LOS Sign Convention + +Gamma `ts_rate` is a phase-rate raster in `rad/year`. Converting phase rate to LOS displacement rate requires a sign convention: + +- `los_rate_away_mm_per_year = phase_rate * wavelength / (4*pi) * 1000` +- `los_rate_toward_mm_per_year = -phase_rate * wavelength / (4*pi) * 1000` + +Gamma `dispmap` documents two conventions: + +- `sflg=0`, the default: motion away from radar is negative, so motion toward radar is positive; deformation and unwrapped phase have opposite signs. +- `sflg=1`: motion away from radar is positive; deformation and unwrapped phase have the same sign. + +For system productization, use explicit names and prefer `los_rate_toward_mm_per_year` as the default display product because it matches Gamma `dispmap` default `sflg=0`. Keep the away-positive version available when another downstream convention requires direct phase-sign products. + +### Monitoring Point Curve + +An example monitoring point was selected automatically from low-sigma, high-rate, non-edge pixels: + +- radar pixel: range `336`, azimuth `2290` +- approximate lon/lat: `131.4340324903`, `43.8008322757` +- reference date in the plotted time series: `20240711` +- LOS convention: toward radar positive, away from radar negative +- fitted LOS velocity: `50.1085 mm/year` +- fitted LOS velocity sigma: `0.0063 mm/year` + +Generated outputs: + +- `publish/monitor_points/auto_low_sigma_high_rate_timeseries.png` +- `publish/monitor_points/auto_low_sigma_high_rate_timeseries.csv` +- `publish/monitor_points/auto_low_sigma_high_rate_metadata.json` + +The plotted values are: + +| Date | LOS displacement, toward-positive mm | +| --- | ---: | +| 20240516 | -7.673957 | +| 20240711 | 0.000000 | +| 20240905 | 7.673922 | +| 20250417 | 38.412669 | +| 20251002 | 61.464305 | + +This is a single example point only. It is not a validated monitoring-point network and should not be interpreted as a representative area-wide deformation curve. The automatic selection favors a non-edge pixel with relatively high absolute velocity and low fitted sigma so the curve is visually inspectable. Production monitoring points need one of these inputs: + +- user-clicked map lon/lat +- imported engineering monitoring-point layer +- a configured regular grid or point-of-interest set +- a quality-filtered automatic point sampler with spacing, coherence/sigma thresholds, and manual review + +Until that is implemented, the single curve is a capability demonstration and should be labeled as such in the UI. + +### Current Open Items + +- Review the `ts_rate.bmp`, `sigma_rate.bmp`, `hgt_correction.bmp`, LOS velocity BMPs, monitoring-point PNG, and exported GeoTIFFs visually in GIS. +- Tune reference region, coherence thresholds, and pair network before treating the result as production. +- Decide whether to keep the weak/long 2025 pair, add skip-pairs, or use a denser data sequence when more LT1 precise orbits are available. +- Keep system integration as orchestration around Gamma commands; do not implement custom SBAS inversion in application code. + +## Productization Decisions After Trial + +If the 5-scene trial succeeds, add a new managed processor instead of bending existing ISCE/SARscape flows: + +```text +processor_code = gamma_ipta_sbas +engine_code = gamma +workflow = gamma_ipta_sbas +``` + +Required system changes: + +- Add a Gamma IPTA stack manifest builder. +- Add LT1-specific hard grouping keys: + - satellite platform, not only family, unless cross-satellite LT1A/LT1B is explicitly validated + - relative orbit + - orbit direction + - imaging mode + - polarization + - scene strip/center bucket + - product variant or station/submode where it affects compatibility +- Add a Gamma baseline audit step before final pair network selection. +- Persist selected Gamma pair network separately from the coarse planning graph. +- Keep system code as orchestration only; Gamma remains the processing authority. + +## Current Recommendation + +Use the manual audited `LT1B relOrbit 114 DESCENDING STRIP1 HH / E131.2 N43.8` stack and the trial scripts as the reference path for productization. + +Do not start Gamma IPTA production from the current automatic time-series plan. It can be used for discovery, but production stack selection needs the hard grouping keys and Gamma baseline audit described above. + +For the next engineering step, add a managed `gamma_ipta_sbas` processor that orchestrates the Gamma commands rather than reimplementing SBAS inversion in application code. diff --git a/docs/INDEX.md b/docs/INDEX.md index f09b319..c49659d 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -36,8 +36,11 @@ - [PRODUCTION_RESULTS_MULTI_ENGINE_DESIGN_20260423.md](PRODUCTION_RESULTS_MULTI_ENGINE_DESIGN_20260423.md) 多引擎结果目录、发布结构和 catalog 治理设计。 +- [FLOOD_GEOTIFF_GAMMA_PREPROCESS_DESIGN_20260515.md](FLOOD_GEOTIFF_GAMMA_PREPROCESS_DESIGN_20260515.md) + 洪涝模块 GeoTIFF 化与 GAMMA 前处理评估，定义“原始数据到分析级 TIF 由可插拔处理器完成，水体/洪涝算法只运行在 GeoTIFF 层”的当前技术决策。 + - [FLOOD_DISASTER_ANALYSIS_SYSTEM_DESIGN_20260514.md](FLOOD_DISASTER_ANALYSIS_SYSTEM_DESIGN_20260514.md) - 洪涝灾害分析独立系统设计，定义多源 SAR 数据、ENVI/SARscape 洪涝流程、标准产品包和矢量套合分析边界。 + 洪涝灾害分析独立系统早期设计，保留工作台、产品包和矢量套合分析边界；其中 ENVI/SARscape 作为洪涝主线的表述已被 2026-05-15 GeoTIFF 化设计取代。 - [WSL_RUNTIME_REFACTOR_DESIGN_20260422.md](WSL_RUNTIME_REFACTOR_DESIGN_20260422.md) WSL 共享运行时和 Broker 设计。 @@ -50,15 +53,18 @@ ## 3. 时序 InSAR / SBAS -- [ISCE2_SBAS_TIMESERIES_DESIGN.md](ISCE2_SBAS_TIMESERIES_DESIGN.md) -- [ISCE2_SBAS_PRODUCT_SPEC.md](ISCE2_SBAS_PRODUCT_SPEC.md) -- [ISCE2_SBAS_ENGINEERING_DESIGN_20260428.md](ISCE2_SBAS_ENGINEERING_DESIGN_20260428.md) - Current-phase engineering design for the managed ISCE2 + MintPy SBAS route. +- [SBAS_INSAR_PRODUCTION_PIPELINE_DESIGN_20260519.md](SBAS_INSAR_PRODUCTION_PIPELINE_DESIGN_20260519.md) + Gamma/LT1 SBAS-InSAR 独立生产页面与统一流水线收口设计。 +- [TIMESERIES_LEGACY_DEPRECATION_20260521.md](TIMESERIES_LEGACY_DEPRECATION_20260521.md) + 旧 ISCE2/MintPy 时序生产链路停用记录，定义当前入口隐藏、配置默认关闭和后续物理删除条件。 +- [GAMMA_IPTA_LT1_SBAS_TRIAL_RUNBOOK_20260518.md](GAMMA_IPTA_LT1_SBAS_TRIAL_RUNBOOK_20260518.md) + Gamma 官方 DIFF + IPTA 路径处理 LT1 数据的本地试验记录、产物和符号约定。 说明： -- 当前前端顶级显示名已经统一为“时序 InSAR”。 -- 当前默认接入仍然是 SBAS 路径，因此这两份文档仍然有效。 +- 当前 SBAS-InSAR 生产主线是 Gamma 独立生产页面，不再使用旧 ISCE2/MintPy 时序生产页作为入口。 +- 旧 `timeseries-production` 后端、旧前端面板和 `ps_timeseries_runs` 相关结构暂时保留兼容，但默认关闭。 +- `ISCE2_SBAS_TIMESERIES_DESIGN.md`、`ISCE2_SBAS_PRODUCT_SPEC.md`、`ISCE2_SBAS_ENGINEERING_DESIGN_20260428.md` 仅作为历史参考，不再作为当前生产事实依据。 - `SBAS_*` 命名的一批旧文档已经归档，只保留历史追溯价值。 ## 4. 配对与前端导航 @@ -130,4 +136,5 @@ - 已归档的 `项目汇报.md` - 已归档的各类 `*_EXPERIMENT_*` / `*_PROGRESS_*` / `*_TODO_*` -最后更新：2026-05-12 +最后更新：2026-05-15 +最近修订：2026-05-21 diff --git a/docs/SBAS_INSAR_PRODUCTION_PIPELINE_DESIGN_20260519.md b/docs/SBAS_INSAR_PRODUCTION_PIPELINE_DESIGN_20260519.md new file mode 100644 index 0000000..977613b --- /dev/null +++ b/docs/SBAS_INSAR_PRODUCTION_PIPELINE_DESIGN_20260519.md @@ -0,0 +1,439 @@ +# SBAS-InSAR Production Pipeline Design + +Date: 2026-05-19 + +## Decision + +SBAS-InSAR production becomes an independent production workflow and page. It must not depend on the existing coarse time-series pairing layer as its production authority. + +The old time-series pairing code may remain temporarily for compatibility and candidate discovery, but the new SBAS-InSAR page and backend API should bypass it by default. Deletion should happen only after the new workflow can create, run, publish, and browse Gamma SBAS/IPTA products end to end. + +The legacy ISCE2/MintPy time-series production chain is disabled by default. The `timeseries-production` backend code and old catalog pages may remain as compatibility code, but they are no longer exposed as production-management subpages. The active SBAS production route is `/api/sbas-insar-production` and the active UI view is `sbas_insar_production`. + +## Scope + +Initial production target: + +- Sensor: LT1 SLC +- Engine: Gamma +- Workflow: Gamma DIFF + IPTA `mb` + `ts_rate` +- Processor code: `gamma_ipta_sbas` +- Default display product: LOS velocity toward radar positive + +Out of scope for the first implementation slice: + +- custom SBAS inversion in application code +- direct reuse of the current PS/time-series pair graph as final Gamma `itab` +- full automatic stack approval without a Gamma baseline and quality audit +- cross-satellite LT1A/LT1B stack mixing + +## Product Contract + +Every successful SBAS-InSAR production run should publish: + +- `product_summary.json` +- `stack_manifest.json` +- `gamma_command_manifest.json` +- `pair_network.json` +- `quality_summary.json` +- `los_rate_toward_mm_per_year.tif` +- `los_rate_toward_mm_per_year.geo_preview.png` +- `los_rate_toward_mm_per_year.rdc_preview.bmp` +- `los_rate_away_mm_per_year.tif` +- `los_rate_away_mm_per_year.rdc_preview.bmp` +- `los_sigma_mm_per_year.tif` +- `los_sigma_mm_per_year.geo_preview.png` +- `los_sigma_mm_per_year.rdc_preview.bmp` +- `ts_rate_rad_per_year.tif` +- `sigma_rate_rad_per_year.tif` +- monitoring-point time-series `png/csv/json` +- raw logs for each Gamma stage + +The current trial product remains the reference implementation: + +```text +backend/runtime/gamma_ipta_trials/lt1b_r114_e1312_n438_20240516_20251002 +``` + +Preview rule: + +- UI default map previews must be rendered from geocoded EPSG:4326 GeoTIFFs. +- RDC/RMLI BMP browse images are processing QA artifacts only. +- Product names should make coordinate state explicit: `geo_preview` for map previews and `rdc_preview` for radar-geometry previews. + +## LOS Sign Convention + +Gamma `ts_rate` outputs phase rate in `rad/year`. The system must store both sign conventions explicitly: + +```text +los_rate_away_mm_per_year = phase_rate * wavelength / (4*pi) * 1000 +los_rate_toward_mm_per_year = -phase_rate * wavelength / (4*pi) * 1000 +``` + +Default UI display: + +```text +LOS toward radar positive +``` + +This matches Gamma `dispmap` default `sflg=0`: motion away from radar is negative, motion toward radar is positive. + +## Page Design + +Add a separate production view: + +```text +Production Management + - D-InSAR Runs + - SBAS-InSAR Production + - D-InSAR Products +``` + +The SBAS page is operational, not a marketing landing page. First screen should show: + +- runtime capability: Gamma install, WSL distro, workflow support +- available SBAS stacks or trial runs +- selected run summary +- LOS velocity geocoded preview +- product file list +- monitoring-point curve +- quality metrics +- stage checklist + +The current old "time-series run" and "time-series products" views are hidden from the production workspace. Legacy route aliases such as `ps_production` and `ps_products` should redirect to the SBAS-InSAR production view rather than opening the old ISCE2/MintPy workflow. + +## Backend API + +Initial read-only API: + +```text +GET /api/sbas-insar-production/capabilities +GET /api/sbas-insar-production/trial-runs +GET /api/sbas-insar-production/trial-runs/{trial_id} +GET /api/sbas-insar-production/trial-runs/{trial_id}/artifacts/{relative_path} +``` + +Stack discovery and hard-constraint audit API: + +```text +POST /api/sbas-insar-production/stacks/discover +POST /api/sbas-insar-production/stacks/{stack_id}/audit +``` + +`stacks/discover` scans LT1 source roots directly and groups scenes by: + +- platform, for example `LT1A` or `LT1B` +- satellite mode, for example `MONO` +- receiving station +- relative orbit +- orbit direction +- imaging mode +- polarization +- center bucket, for example `E131.2_N43.8` + +It also checks LT1 precise orbit TXT availability against `PYINT_ORBIT_POOL_TXT` / `ORBIT_POOL_ENVI`. + +`stacks/{stack_id}/audit` writes a reproducible manifest under: + +```text +backend/runtime/sbas_insar_production/stack_manifests/{stack_id}/ +``` + +The manifest status is `READY_FOR_GAMMA_BASELINE_AUDIT` only after hard grouping, minimum scene count, precise orbit availability, and an initial adjacent temporal network are satisfied. Gamma `base_calc` remains the next required audit before final `itab` approval. + +Writable production planning API: + +```text +POST /api/sbas-insar-production/stacks/{stack_id}/runs +GET /api/sbas-insar-production/runs +GET /api/sbas-insar-production/runs/{run_id} +GET /api/sbas-insar-production/runs/{run_id}/artifacts/{relative_path} +``` + +The current `runs` submission is a dry-run planning submission. It writes: + +```text +backend/runtime/sbas_insar_production/runs/{run_id}/run_manifest.json +backend/runtime/sbas_insar_production/runs/{run_id}/stack_manifest.json +backend/runtime/sbas_insar_production/runs/{run_id}/pair_network.json +backend/runtime/sbas_insar_production/runs/{run_id}/gamma_command_manifest.json +backend/runtime/sbas_insar_production/runs/{run_id}/monitor_points.json +``` + +The created run status is: + +```text +PLANNED_GAMMA_BASELINE_AUDIT +``` + +This is intentionally not a Gamma execution trigger yet. The next runnable slice should add: + +```text +POST /api/sbas-insar-production/runs/{run_id}/baseline-audit +POST /api/sbas-insar-production/runs/{run_id}/itab-decision +POST /api/sbas-insar-production/runs/{run_id}/coregistration +POST /api/sbas-insar-production/runs/{run_id}/coregistration/jobs +POST /api/sbas-insar-production/runs/{run_id}/monitor-points +POST /api/sbas-insar-production/runs/{run_id}/retry-stage +``` + +`baseline-audit` currently supports: + +- script-only mode: generate/reparse `scripts/01_baseline_audit.sh` and existing outputs +- execution mode: run Gamma `par_LT1_SLC`, `LT1_precision_orbit.py`, `multi_look`, and `base_calc` +- output parsing: write `baseline_audit_summary.json` and `pair_network_baseline_audit.json` + +`itab-decision` is the current production gate: + +- `approve`: copies Gamma `work/gamma/diff/itab_adjacent` to `work/gamma/diff/itab_approved`, writes `itab_decision.json`, moves the run to `ITAB_APPROVED`, and makes `coregistration` the next stage +- `reject`: writes `itab_decision.json`, moves the run to `ITAB_REJECTED`, and blocks further Gamma stages until the pair network is revised + +`coregistration` supports script generation. It writes: + +```text +backend/runtime/sbas_insar_production/runs/{run_id}/scripts/02_coreg_common_ref.sh +backend/runtime/sbas_insar_production/runs/{run_id}/coregistration_plan.json +``` + +The generated script consumes `work/gamma/diff/itab_approved` as the approval gate, uses the stack reference date as common geometry, and prepares Gamma `SLC_coreg.py` calls for every non-reference date. + +`coregistration/jobs` submits the generated script to the existing `SystemTask` + `SystemJob` background queue as job type `SBAS_COREGISTRATION`. The job runs Gamma `SLC_coreg.py`, writes `coregistration_summary.json`, updates the run manifest to `COREGISTRATION_READY` or `COREGISTRATION_FAILED`, and advances the next stage to `rdc_dem` only when all expected RSLC/RMLI outputs and common tab files exist. + +## Backend Services + +First slice: + +```text +sbas_insar_production_service.py + - discover local Gamma IPTA trial summaries + - normalize products and artifact URLs + - expose sign convention and product metadata + - serve safe artifacts from trial roots +``` + +Second slice: + +```text +gamma_ipta_stack_planner.py + - hard group LT1 scenes by platform, relative orbit, direction, mode, polarization, center bucket + - require precise orbit availability + - emit stack_manifest.json + +gamma_ipta_pair_planner.py + - build initial temporal network + - run Gamma baseline audit + - emit pair_network.json and Gamma itab + +gamma_ipta_job_runner.py + - execute official Gamma commands stage by stage + - write command manifests and logs + +sbas_insar_product_publisher.py + - publish GeoTIFF/BMP/CSV/JSON products + - register products into unified result catalog +``` + +## Future Database Model + +Use unified pipeline tables rather than adding many one-off SBAS tables: + +```text +pipeline_runs +pipeline_stages +pipeline_products +pipeline_quality_metrics +pipeline_logs +``` + +Minimum fields for `pipeline_runs`: + +- `run_id` +- `workflow_code = sbas_insar` +- `processor_code = gamma_ipta_sbas` +- `engine_code = gamma` +- `status` +- `stack_manifest_path` +- `work_root` +- `publish_root` +- `created_by` +- `created_at` +- `started_at` +- `ended_at` +- `summary_json` + +For the first slice, use filesystem discovery only. Do not add migrations until the run submission workflow is ready. + +## Gamma Stage Contract + +The managed runner should preserve the successful trial chain: + +1. `par_LT1_SLC` +2. `LT1_precision_orbit.py` +3. `multi_look` +4. `base_calc` +5. `SLC_coreg.py` +6. `gc_map1` / `geocode` / `gc_map_fine` +7. `phase_sim_orb` +8. `SLC_diff_intf` +9. `adf` +10. `mcf` +11. `mb` +12. `ts_rate` +13. `geocode_back` +14. `data2geotiff` +15. LOS sign conversion and preview generation +16. monitoring-point time-series extraction + +The application is an orchestrator. Gamma remains the processing authority. + +## Migration Plan + +Phase 1: read-only SBAS production page + +- add design document +- add backend API for existing Gamma trial discovery +- add page entry and product preview +- keep old time-series page available as legacy + +Phase 2: managed run submission + +- add stack discovery and audit endpoints +- add planned-run submission endpoint +- write production run manifest, command manifest, and monitor-point config +- add Gamma runner skeleton +- queue job with stage updates + +Phase 3: unified pipeline management + +- add generic pipeline tables +- move Gamma SBAS run records into pipeline tables +- register products through the unified product catalog + +Phase 4: remove old SBAS/time-series pairing dependency + +- hide old SBAS entry completely +- keep any reusable discovery functions as internal utilities +- delete obsolete UI and API routes after dependency audit + +## Acceptance Criteria For Phase 1 + +- SBAS-InSAR production appears as its own production workspace view. +- Existing Gamma IPTA trial can be listed from the backend API. +- The selected trial shows LOS velocity preview, sigma/GeoTIFF products, monitor-point curve, and quality summary. +- Artifact serving is constrained to the trial root. +- No existing D-InSAR, flood, or legacy time-series routes are broken. + +## Implementation Progress On 2026-05-19 + +Implemented: + +- read-only SBAS-InSAR production page +- trial product browser for the local Gamma IPTA validation run +- artifact API constrained to published trial outputs +- LT1 filesystem stack discovery independent of the old time-series pairing layer +- hard grouping by platform, satellite mode, receiving station, relative orbit, orbit direction, imaging mode, polarization, and center bucket +- precise orbit TXT availability check against the configured Gamma/PyINT orbit pool +- stack manifest and initial adjacent pair-network JSON generation +- geocoded web previews generated from `los_rate_toward_mm_per_year.tif` and `los_sigma_mm_per_year.tif` +- planned SBAS production run creation from a READY stack manifest +- filesystem production run browser and artifact download API +- Gamma stage plan manifest with execution disabled until baseline audit runner is attached +- monitoring-point config contract with explicit placeholder status for `auto_low_sigma_high_rate` +- Gamma baseline audit script generation and output parser +- baseline audit result display in the SBAS production page +- itab approval/rejection API and page controls +- common-reference coregistration script generation and page summary +- queued `SBAS_COREGISTRATION` background job submission through the existing task/job queue +- coregistration execution summary parser and manifest status update to `COREGISTRATION_READY` / `COREGISTRATION_FAILED` + +Local verification: + +- scanned `1500` LT1 scene directories from the local data pool +- found READY candidates with all required precise orbit TXT files +- generated one manifest at: + +```text +backend/runtime/sbas_insar_production/stack_manifests/sbas_2e6301f64a10/20260519T122146Z_stack_manifest.json +``` + +- created one dry-run production plan at: + +```text +backend/runtime/sbas_insar_production/runs/sbas_ab96afabead5/run_manifest.json +``` + +The dry-run production plan uses the local LT1B relOrbit `114` stack around `E129.2_N44.1`, with `7` scenes and `6` initial adjacent temporal pairs. + +- baseline audit script: + +```text +backend/runtime/sbas_insar_production/runs/sbas_ab96afabead5/scripts/01_baseline_audit.sh +``` + +- baseline audit summary: + +```text +backend/runtime/sbas_insar_production/runs/sbas_ab96afabead5/baseline_audit_summary.json +``` + +The baseline audit ran Gamma 20240627 `par_LT1_SLC`, `LT1_precision_orbit.py`, `multi_look`, and `base_calc` against all 7 LT1B scenes. It completed with status `BASELINE_AUDIT_READY` after the outer terminal command timed out, because the WSL process continued to completion in the background. + +Gamma `base_calc` adjacent-network result: + +- all-pair count: `21` +- adjacent-pair count: `6` +- max absolute perpendicular baseline: `731.9957 m` +- max temporal gap: `224 days` + +The current adjacent network is connected, but several Bperp values are large enough that a human baseline/quality review is still required before using this `itab` for the full SBAS inversion. + +The current run has been approved for the next controlled trial step: + +```text +status = ITAB_APPROVED +next_stage = coregistration +approved itab = backend/runtime/sbas_insar_production/runs/sbas_ab96afabead5/work/gamma/diff/itab_approved +decision record = backend/runtime/sbas_insar_production/runs/sbas_ab96afabead5/itab_decision.json +``` + +The common-reference co-registration script has been generated: + +```text +backend/runtime/sbas_insar_production/runs/sbas_ab96afabead5/scripts/02_coreg_common_ref.sh +backend/runtime/sbas_insar_production/runs/sbas_ab96afabead5/coregistration_plan.json +``` + +Current status before executing the queued job: + +```text +status = COREGISTRATION_SCRIPT_READY +next_stage = execute_coregistration +common reference date = 20241007 +``` + +The actual `SLC_coreg.py` execution is now wired as a background job endpoint and page action, but has not been production-tested in this pass. It consumes `itab_approved`, not the pre-audit or unapproved pair plan. + +Expected post-job outputs: + +```text +backend/runtime/sbas_insar_production/runs/sbas_ab96afabead5/coregistration_summary.json +backend/runtime/sbas_insar_production/runs/sbas_ab96afabead5/work/gamma/common_20241007/SLC_tab +backend/runtime/sbas_insar_production/runs/sbas_ab96afabead5/work/gamma/common_20241007/RMLI_tab +backend/runtime/sbas_insar_production/runs/sbas_ab96afabead5/work/gamma/common_20241007/rslc/*.rslc +backend/runtime/sbas_insar_production/runs/sbas_ab96afabead5/work/gamma/common_20241007/rmli/*.mli +``` + +After the job succeeds: + +```text +status = COREGISTRATION_READY +next_stage = rdc_dem +``` + +Open product-display decisions: + +- the current Gamma `*.bmp` previews are RDC processing-geometry products; keep them visible only as QA artifacts +- the first UI map preview should use `los_rate_toward_mm_per_year.geo_preview.png` +- the first sigma preview should use `los_sigma_mm_per_year.geo_preview.png` +- the current monitoring-point curve is a single automatic sample point, not a monitoring network +- production monitoring curves need user-selected lon/lat points, imported monitoring points, or a quality-filtered automatic sampler before they can be treated as formal outputs diff --git a/docs/TIMESERIES_LEGACY_DEPRECATION_20260521.md b/docs/TIMESERIES_LEGACY_DEPRECATION_20260521.md new file mode 100644 index 0000000..68a80b7 --- /dev/null +++ b/docs/TIMESERIES_LEGACY_DEPRECATION_20260521.md @@ -0,0 +1,184 @@ +# Legacy Time-Series InSAR Deprecation Record + +Date: 2026-05-21 + +## Background + +The project previously kept an ISCE2 + MintPy time-series/SBAS production path under: + +```text +experiments/isce2_sbas_timeseries +``` + +That experiment directory has been removed. The active SBAS-InSAR direction is now the independent Gamma production workflow documented in: + +```text +docs/SBAS_INSAR_PRODUCTION_PIPELINE_DESIGN_20260519.md +docs/GAMMA_IPTA_LT1_SBAS_TRIAL_RUNBOOK_20260518.md +``` + +Before this cleanup, startup validation still generated warnings for missing legacy paths: + +```text +TIMESERIES_EXPERIMENT_ROOT +TIMESERIES_STACK_PREP_SCRIPT +TIMESERIES_MATERIALIZE_SCRIPT +TIMESERIES_PREPARE_DEM_SCRIPT +TIMESERIES_STACK_RUNNER_SCRIPT +TIMESERIES_MINTPY_SBAS_SCRIPT +TIMESERIES_EXPORT_PUBLISH_SCRIPT +``` + +Those warnings were misleading because they referred to the abandoned ISCE2/MintPy line, not the current Gamma SBAS-InSAR production line. + +## Decision + +The ISCE2/MintPy time-series production chain is deprecated and disabled by default. + +The current SBAS-InSAR production authority is: + +```text +Frontend view: sbas_insar_production +Backend route: /api/sbas-insar-production +Engine: Gamma +Workflow: DIFF + IPTA SBAS +``` + +The legacy code is not physically deleted yet. It remains only as compatibility and historical-reference code until the Gamma SBAS workflow can be tested end to end and historical result access is confirmed. + +## Changes Made + +Backend configuration: + +- `backend/app/config.py` + - `TIMESERIES_ENABLED` default changed from `true` to `false`. + - Legacy `TIMESERIES_*` experiment/script defaults are now populated only when `TIMESERIES_ENABLED=true`. + - `ensure_dirs()` no longer creates `TIMESERIES_WORK_ROOT` unless the legacy chain is explicitly enabled. + - Runtime validation now reports an info line instead of warning about missing legacy experiment scripts when the chain is disabled. + +Environment example: + +- `.env.example` + - `TIMESERIES_ENABLED=false` + - `TIMESERIES_EXPERIMENT_ROOT=` is blank. + - `TIMESERIES_DEFAULT_PROCESSOR_CODE=legacy_isce2_stack_mintpy` + +Frontend production management: + +- `frontend/src/config/appConstants.js` + - Removed production workspace views: + - `timeseries_runs` + - `timeseries_products` + - Legacy route aliases now map to the Gamma SBAS page: + - `ps_production -> sbas_insar_production` + - `ps_products -> sbas_insar_production` + +- `frontend/src/ProductionWorkspace.jsx` + - Removed lazy imports and render branches for: + - `TimeseriesProductionPanel` + - production-management `PsinsarCatalogPanel` + - Updated production workspace text to describe Gamma SBAS as an independent entry. + +- `frontend/src/components/app/AppSidePanel.jsx` + - Updated production-management description to state that the old ISCE2/MintPy time-series entry is disabled. + +Documentation: + +- `docs/SBAS_INSAR_PRODUCTION_PIPELINE_DESIGN_20260519.md` + - Added the legacy-chain deprecation decision. + +- `docs/FRONTEND_NAVIGATION_ARCHITECTURE.md` + - Updated production-management internal views to: + - `dinsar_runs` + - `sbas_insar_production` + - `dinsar_products` + +## Current Behavior + +After backend restart, deployment validation should no longer warn about the removed `experiments/isce2_sbas_timeseries` path. + +Expected validation line: + +```text +[INFO] Legacy ISCE2/MintPy timeseries pipeline is disabled; current SBAS-InSAR production uses the Gamma /sbas-insar-production workflow. +``` + +The production-management page should show: + +```text +D-InSAR 运行 +SBAS-InSAR Production +D-InSAR 产物 +``` + +It should no longer expose: + +```text +时序InSAR 运行 +时序InSAR 产物 +``` + +## Verification + +Completed on 2026-05-21: + +- Frontend build passed: + +```text +npm run build +``` + +- Runtime configuration check passed: + +```text +scripts/check_runtime_config.py +``` + +Observed output included: + +```text +[INFO] Legacy ISCE2/MintPy timeseries pipeline is disabled; current SBAS-InSAR production uses the Gamma /sbas-insar-production workflow. +[OK] Deployment configuration check passed. +``` + +- Backend config syntax was checked with Python AST parsing. + +`python -m py_compile backend/app/config.py` was not used as the final check because Windows denied replacement of an existing `__pycache__` file. This was a local cache-permission issue, not a syntax failure. + +## Retained Compatibility Code + +The following code is intentionally retained for now: + +```text +backend/app/routers/timeseries_production.py +backend/app/services/timeseries_service.py +frontend/src/TimeseriesProductionPanel.jsx +frontend/src/api/timeseriesProduction.js +frontend/src/components/PsinsarCatalogPanel.jsx +``` + +Database tables and historical product catalog structures such as `ps_timeseries_runs` are also retained. + +`PsinsarCatalogPanel` may still be useful outside production management, especially for analysis/result browsing. Do not delete it until those usages are audited. + +## Re-Enabling Legacy Chain + +Re-enabling the legacy ISCE2/MintPy chain is not part of the current production plan. + +If it must be revived for a controlled comparison, the operator must explicitly set: + +```text +TIMESERIES_ENABLED=true +``` + +and provide valid values for all legacy experiment/script paths. The removed `experiments/isce2_sbas_timeseries` directory is no longer assumed to exist. + +## Follow-Up Cleanup Criteria + +Physical deletion of the legacy chain should wait until all of the following are true: + +- Gamma SBAS production has completed an end-to-end run from stack discovery to published LOS velocity/sigma products. +- Historical `ps_timeseries_runs` and old time-series product records have a clear migration or read-only archival plan. +- Frontend navigation, route aliases, and analysis pages have been audited for remaining dependencies. +- Backend callers of `/api/timeseries-production` have either been removed or explicitly marked as legacy-only. +- Test coverage or manual regression notes confirm that D-InSAR production, SBAS production, product browsing, and task monitoring still work. diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index bbab7d2..d0d833e 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -282,6 +282,8 @@ function App() { const aoeLayerRef = useRef(null); const waterSceneLayersRef = useRef({}); const floodEventLayersRef = useRef({}); // { eventId: { pre, post, classified } } + const floodPairPreviewLayerRef = useRef(null); + const floodVectorLayersRef = useRef({}); const mapRegionLayerRef = useRef(null); const prevLicenseOkRef = useRef(false); const initializeAppDataRef = useRef(null); @@ -327,6 +329,7 @@ function App() { pairLayersRef: pairLayersRef.current, aoeLayerRef: aoeLayerRef.current, floodEventLayersRef: floodEventLayersRef.current, + floodVectorLayersRef: floodVectorLayersRef.current, }), []); const mapExport = useMapExport({ mapRef, getVisibleLayerRefs, addLog, language }); @@ -1468,8 +1471,11 @@ function App() { const handleWaterSceneOnMap = (scene) => { if (!mapRef.current || !scene.coverage_polygon) return; - if (waterSceneLayersRef.current[scene.id]) { - waterSceneLayersRef.current[scene.id].remove(); + const layerKey = scene.radar_data_id ? `scene_${scene.id}` : `radar_${scene.id}`; + if (waterSceneLayersRef.current[layerKey]) { + waterSceneLayersRef.current[layerKey].remove(); + delete waterSceneLayersRef.current[layerKey]; + return false; } const latLngs = scene.coverage_polygon.map(p => [p[1], p[0]]); const polygon = L.polygon(latLngs, { @@ -1484,10 +1490,35 @@ function App() { `${scene.satellite || ''} · ${scene.imaging_date || ''}
` + `${(scene.geo_path || '').split(/[\\/]/).pop()}` ); - waterSceneLayersRef.current[scene.id] = polygon; + waterSceneLayersRef.current[layerKey] = polygon; polygon.addTo(mapRef.current); mapRef.current.flyToBounds(L.latLngBounds(latLngs), { padding: [50, 50], maxZoom: 10 }); polygon.openPopup(); + return true; + }; + + const handleFloodSourcePreviewOnMap = (item) => { + if (!mapRef.current || !item) return; + const enriched = { + ...item, + displayName: item.displayName || item.file_path?.split(/[\\/]/).pop() || `${item.satellite || 'SAR'} #${item.id}`, + previewCacheKey: item.previewCacheKey || item.preview_cache_updated_at || `${Date.now()}-${item.id}`, + }; + if (radarPreviewLayersRef.current[enriched.id]) { + updateRadarPreviewVisibility(enriched, false); + return false; + } + updateRadarPreviewVisibility(enriched, true); + if ( + enriched.min_lat != null && enriched.max_lat != null + && enriched.min_lon != null && enriched.max_lon != null + ) { + mapRef.current.flyToBounds( + L.latLngBounds([[enriched.min_lat, enriched.min_lon], [enriched.max_lat, enriched.max_lon]]), + { padding: [50, 50], maxZoom: 10 }, + ); + } + return true; }; // 洪涝事件三图层叠加（灾前/灾后/分类结果），每层可独立开关 @@ -1534,6 +1565,66 @@ function App() { else layers[key].remove(); }; + const handleFloodPairPreviewOnMap = (pair) => { + if (!mapRef.current || !pair?.pre?.coverage_polygon || !pair?.post?.coverage_polygon) return; + if (floodPairPreviewLayerRef.current) { + floodPairPreviewLayerRef.current.remove(); + floodPairPreviewLayerRef.current = null; + } + const makePolygon = (scene, color, label) => { + const latLngs = scene.coverage_polygon.map(p => [p[1], p[0]]); + const polygon = L.polygon(latLngs, { + color, + weight: 2, + fillColor: color, + fillOpacity: 0.14, + }); + polygon.bindPopup( + `${escapeHtml(label)}
` + + `场景 #${escapeHtml(scene.id)} · ${escapeHtml(scene.satellite || '')}
` + + `${escapeHtml(formatYmd(scene.imaging_date, language))} · ${escapeHtml(scene.polarization || '')}` + ); + return polygon; + }; + + const prePolygon = makePolygon(pair.pre, '#2563eb', '灾前覆盖'); + const postPolygon = makePolygon(pair.post, '#16a34a', '灾后覆盖'); + const group = L.featureGroup([prePolygon, postPolygon]); + floodPairPreviewLayerRef.current = group; + group.addTo(mapRef.current); + mapRef.current.flyToBounds(group.getBounds(), { padding: [50, 50], maxZoom: 10 }); + }; + + const handleFloodVectorOnMap = (impact) => { + if (!mapRef.current || !impact?.flood_vector_geojson) return; + const layerId = impact.overlay_id || impact.detection_id || 'current'; + const existing = floodVectorLayersRef.current[layerId]; + if (existing) existing.remove(); + + const layer = L.geoJSON(impact.flood_vector_geojson, { + style: { + color: '#dc2626', + weight: 2, + fillColor: '#ef4444', + fillOpacity: 0.26, + }, + onEachFeature: (feature, featureLayer) => { + const props = feature?.properties || {}; + featureLayer.bindPopup( + `洪涝矢量
` + + `Overlay #${escapeHtml(layerId)}
` + + `${escapeHtml(props.name || props.class || '')}` + ); + }, + }); + floodVectorLayersRef.current[layerId] = layer; + layer.addTo(mapRef.current); + const bounds = layer.getBounds(); + if (bounds.isValid()) { + mapRef.current.flyToBounds(bounds, { padding: [50, 50], maxZoom: 12 }); + } + }; + const visualizePair = useCallback((pair) => { const masterLatLngs = pair.master.coverage_polygon.map(p => [p[1], p[0]]); const slaveLatLngs = pair.slave.coverage_polygon.map(p => [p[1], p[0]]); @@ -1644,11 +1735,14 @@ function App() { }; const floodPanel = { onShowSourceSceneOnMap: handleWaterSceneOnMap, + onShowSourcePreviewOnMap: handleFloodSourcePreviewOnMap, onShowReadyProductOnMap: handleWaterSceneOnMap, onShowOnMap: handleWaterSceneOnMap, onShowFloodOnMap: handleFloodEventOnMap, onShowFloodRunOnMap: handleFloodEventOnMap, onToggleFloodLayer: toggleFloodEventLayer, + onShowFloodPairOnMap: handleFloodPairPreviewOnMap, + onShowFloodVectorOnMap: handleFloodVectorOnMap, }; const dinsarPanel = { dinsarCurrentPage, diff --git a/frontend/src/DataMonitorPanel.jsx b/frontend/src/DataMonitorPanel.jsx index 6adaddd..c1cab23 100644 --- a/frontend/src/DataMonitorPanel.jsx +++ b/frontend/src/DataMonitorPanel.jsx @@ -7,6 +7,7 @@ const DEFAULT_MONITOR_CONFIG = { radar_dirs: [], orbit_dir: '', dinsar_dirs: [], + gf3_archive_source_dirs: [], gf3_source_dirs: [], gf3_storage_dirs: [], s1_source_dirs: [], @@ -65,8 +66,9 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled const [s1Loading, setS1Loading] = useState(false); const [s1ScanLoading, setS1ScanLoading] = useState(false); const [s1Message, setS1Message] = useState(''); - const [gf3Loading, setGf3Loading] = useState(false); + const [gf3UnpackLoading, setGf3UnpackLoading] = useState(false); const [gf3ProcessLoading, setGf3ProcessLoading] = useState(false); + const [gf3ScanLoading, setGf3ScanLoading] = useState(false); const [gf3Message, setGf3Message] = useState(''); const logEndRef = useRef(null); @@ -84,6 +86,9 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled task.task_id === unpackTaskId || task.task_type === 'UNPACK_ARCHIVES' ); const s1ActiveTask = displayActiveTasks.find((task) => task.task_type === 'UNPACK_SENTINEL1'); + const gf3ActiveTask = displayActiveTasks.find((task) => + ['GF3_UNPACK', 'GF3_BATCH_PROCESS'].includes(task.task_type) + ); useEffect(() => { if (!enabled) { @@ -113,6 +118,7 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled s1_source_dirs: toArray(data?.s1_source_dirs), s1_storage_dirs: toArray(data?.s1_storage_dirs), s1_orbit_dirs: toArray(data?.s1_orbit_dirs), + gf3_archive_source_dirs: toArray(data?.gf3_archive_source_dirs), gf3_source_dirs: toArray(data?.gf3_source_dirs), gf3_storage_dirs: toArray(data?.gf3_storage_dirs), }); @@ -281,6 +287,7 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled const hasS1SourceDirs = config.s1_source_dirs.length > 0; const hasS1StorageDirs = config.s1_storage_dirs.length > 0; const hasS1OrbitDirs = config.s1_orbit_dirs.length > 0; + const hasGf3ArchiveSourceDirs = config.gf3_archive_source_dirs.length > 0; const hasGf3SourceDirs = config.gf3_source_dirs.length > 0; const hasGf3StorageDirs = config.gf3_storage_dirs.length > 0; @@ -290,6 +297,7 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled const canRunS1Scan = !readOnly && configLoaded && hasS1SourceDirs; const canRunS1OrbitScan = !readOnly && configLoaded && hasS1OrbitDirs; const canRunGf3Scan = !readOnly && configLoaded && hasGf3StorageDirs; + const canRunGf3Unpack = !readOnly && configLoaded && hasGf3ArchiveSourceDirs && hasGf3SourceDirs; const canRunGf3Process = !readOnly && configLoaded && hasGf3SourceDirs; const canOpenUnpackDialog = !readOnly && unpackConfig.source_dirs.length > 0; @@ -409,6 +417,41 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled } }; + const handleGf3Unpack = async () => { + if (readOnly) { + setGf3Message('当前账户为只读模式，无法触发 GF3 解包。'); + return; + } + setGf3UnpackLoading(true); + setGf3Message('GF3 解包启动中...'); + try { + const res = await fetch(`${apiEndpoint}/monitor/gf3-unpack`, { + method: 'POST', + credentials: 'include', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({}), + }); + const data = await parseJsonSafe(res, {}); + if (res.ok) { + setGf3Message(data.message || 'GF3 解包任务已启动'); + if (onTaskStart) { + onTaskStart(data.task_id, 'GF3 解包任务已启动。', { + nonBlocking: true, + taskType: 'GF3_UNPACK', + }); + } + } else { + setGf3Message(`失败：${data.detail || '未知错误'}`); + } + } catch (err) { + setGf3Message(`失败：${err.message || '未知错误'}`); + } finally { + setGf3UnpackLoading(false); + } + }; + const handleOpenUnpackDialog = () => { if (readOnly) { setUnpackMessage('当前账户为只读模式，无法触发解包任务。'); @@ -546,7 +589,7 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled setGf3Message('当前账户为只读模式，无法触发扫描。'); return; } - setGf3Loading(true); + setGf3ScanLoading(true); setGf3Message('GF3 扫描启动中...'); try { const res = await fetch(`${apiEndpoint}/monitor/run-now?target=gf3`, { @@ -565,7 +608,7 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled } catch (err) { setGf3Message(`失败：${err.message || '未知错误'}`); } finally { - setGf3Loading(false); + setGf3ScanLoading(false); } }; @@ -631,6 +674,7 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
S1 源数据{formatList(config.s1_source_dirs)}
S1 存储{formatList(config.s1_storage_dirs)}
S1 精轨{formatList(config.s1_orbit_dirs)}
+
GF3 压缩包{formatList(config.gf3_archive_source_dirs)}
GF3 来源{formatList(config.gf3_source_dirs)}
GF3 存储{formatList(config.gf3_storage_dirs)}
D-InSAR 结果{formatList(config.dinsar_dirs)}
@@ -723,10 +767,18 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
GF3 归档预处理
+
压缩包来源{formatList(config.gf3_archive_source_dirs)}
L1A 来源{formatList(config.gf3_source_dirs)}
L2 存储{formatList(config.gf3_storage_dirs)}
-
+
+
- {gf3Message} + {gf3ActiveTask ? (gf3ActiveTask.message || 'GF3 任务运行中...') : gf3Message}
diff --git a/frontend/src/FloodAnalysisWorkspace.jsx b/frontend/src/FloodAnalysisWorkspace.jsx index 3b6ccb8..364a1c3 100644 --- a/frontend/src/FloodAnalysisWorkspace.jsx +++ b/frontend/src/FloodAnalysisWorkspace.jsx @@ -3,19 +3,25 @@ import { useCallback, useEffect, useMemo, useState } from 'react'; import UnifiedDatePicker from './components/UnifiedDatePicker'; import { getSearchOptions, searchRadarData } from './api/radar'; import { + createFloodProduct, getFloodActiveRadarIds, getFloodDetectionPreview, getFloodDetections, + getFloodImpact, getFloodDoneRadarIds, + getFloodProductManifest, getFloodScenes, + getFloodProducts, getFloodWaterExtractionPreview, getFloodWaterExtractions, resetFloodScene, - searchFloodPairs, + runFloodOverlay, + searchFloodDisasterPairs, submitFloodDetection, submitFloodPreprocess, submitFloodWaterExtraction, } from './api/flood'; +import { getRegionChildren } from './api/aoi'; import { buildRadarSearchFormData, formatYmd, @@ -121,6 +127,11 @@ function compactDate(value) { return String(value || '').replaceAll('-', '').trim(); } +function formatPercent(value) { + if (value === null || value === undefined || Number.isNaN(Number(value))) return '-'; + return `${(Number(value) * 100).toFixed(1)}%`; +} + function formatArea(value) { if (value === null || value === undefined || Number.isNaN(Number(value))) return '-'; return `${Number(value).toFixed(2)} km²`; @@ -225,6 +236,37 @@ function KeyValue({ label, value, strong = false }) { ); } +function RegionSelector({ options, selection, onProvinceChange, onCityChange, disabled = false }) { + const provinces = options?.provinces || []; + const cities = options?.cities || []; + return ( +
+ + +
+ ); +} + function normalizeMapPreview(preview) { if (!preview) return null; const image = preview.image_b64 || preview.png_base64; @@ -239,6 +281,11 @@ function normalizeMapPreview(preview) { function SceneRow({ scene, readOnly, onShowMap, onExtractWater, onReset }) { const status = asStatus(scene.status); const canExtract = status === 'DONE'; + const [coverageVisible, setCoverageVisible] = useState(false); + const handleToggleCoverage = () => { + const visible = onShowMap(scene); + if (typeof visible === 'boolean') setCoverageVisible(visible); + }; return (
@@ -248,13 +295,16 @@ function SceneRow({ scene, readOnly, onShowMap, onExtractWater, onReset }) { {scene.satellite || '-'} {formatYmd(scene.imaging_date, 'zh')} + {scene.polarization && {scene.polarization}}
- {scene.geo_path ? scene.geo_path.split(/[\\/]/).pop() : scene.error_msg || `Radar ID ${scene.radar_data_id}`} + {[scene.imaging_mode, scene.product_level, scene.geo_path ? scene.geo_path.split(/[\\/]/).pop() : scene.error_msg || `Radar ID ${scene.radar_data_id}`].filter(Boolean).join(' · ')}
- + {isActiveStatus(scene.status) && ( @@ -274,6 +324,9 @@ function WaterResultRow({ item, onShowMap }) { 水体 #{item.id} {item.scene_id && 场景 #{item.scene_id}} + {item.satellite && {item.satellite}} + {item.imaging_date && {formatYmd(item.imaging_date, 'zh')}} + {item.polarization && {item.polarization}}
@@ -288,7 +341,29 @@ function WaterResultRow({ item, onShowMap }) { ); } -function FloodEventRow({ event, mapLayerVis, mapLoading, onShowMap, onToggleLayer, onSelectImpact }) { +function FloodProductRow({ product, onOpenManifest }) { + return ( +
+
+
+
+ {product.product_id || `产品 #${product.id}`} + + {product.detection_id && 洪涝 #{product.detection_id}} +
+
+ + + +
+
+ +
+
+ ); +} + +function FloodEventRow({ event, mapLayerVis, mapLoading, productBusy, onShowMap, onToggleLayer, onSelectImpact, onCreateProduct }) { const done = asStatus(event.status) === 'DONE'; const preDate = formatYmd(event.pre_imaging_date, 'zh'); const postDate = formatYmd(event.post_imaging_date, 'zh'); @@ -312,9 +387,9 @@ function FloodEventRow({ event, mapLayerVis, mapLoading, onShowMap, onToggleLaye
{event.error_msg &&
{event.error_msg}
} - {layers && ( + {(layers || onSelectImpact || onCreateProduct) && (
- {[ + {layers && [ ['pre', '灾前影像'], ['post', '灾后影像'], ['classified', '分类结果'], @@ -333,6 +408,11 @@ function FloodEventRow({ event, mapLayerVis, mapLoading, onShowMap, onToggleLaye 套合分析 )} + {onCreateProduct && ( + + )}
)}
@@ -357,6 +437,8 @@ export default function FloodAnalysisWorkspace({ const [radarLoading, setRadarLoading] = useState(false); const [radarSearched, setRadarSearched] = useState(false); const [selectedRadars, setSelectedRadars] = useState([]); + const [sourceCoverageVisibleIds, setSourceCoverageVisibleIds] = useState(() => new Set()); + const [sourcePreviewVisibleIds, setSourcePreviewVisibleIds] = useState(() => new Set()); const [scenes, setScenes] = useState([]); const [scenesTotal, setScenesTotal] = useState(0); @@ -370,10 +452,21 @@ export default function FloodAnalysisWorkspace({ const [waterPage, setWaterPage] = useState(0); const [waterLoading, setWaterLoading] = useState(false); - const [pairPreStart, setPairPreStart] = useState(''); - const [pairPreEnd, setPairPreEnd] = useState(''); - const [pairPostStart, setPairPostStart] = useState(''); - const [pairPostEnd, setPairPostEnd] = useState(''); + const [sourceAoiMode, setSourceAoiMode] = useState('none'); + const [sourceRegionOptions, setSourceRegionOptions] = useState({ provinces: [], cities: [] }); + const [sourceRegionSelection, setSourceRegionSelection] = useState({ province: '', city: '' }); + const [regionLoading, setRegionLoading] = useState(false); + const [regionError, setRegionError] = useState(''); + + const [disasterName, setDisasterName] = useState(''); + const [disasterDate, setDisasterDate] = useState(''); + const [preWindowDays, setPreWindowDays] = useState(30); + const [postWindowDays, setPostWindowDays] = useState(30); + const [minAoiCoverage, setMinAoiCoverage] = useState(0.2); + const [pairRegionOptions, setPairRegionOptions] = useState({ provinces: [], cities: [] }); + const [pairRegionSelection, setPairRegionSelection] = useState({ province: '', city: '' }); + const [pairSearchSummary, setPairSearchSummary] = useState(null); + const [overlapThreshold, setOverlapThreshold] = useState(0.3); const [refine, setRefine] = useState(false); const [pairSearching, setPairSearching] = useState(false); @@ -382,9 +475,15 @@ export default function FloodAnalysisWorkspace({ const [floodEvents, setFloodEvents] = useState([]); const [floodLoading, setFloodLoading] = useState(false); + const [floodProducts, setFloodProducts] = useState([]); + const [productLoading, setProductLoading] = useState(false); + const [productBusyId, setProductBusyId] = useState(null); const [mapLoadingId, setMapLoadingId] = useState(null); const [mapLayerVis, setMapLayerVis] = useState({}); const [selectedImpactEventId, setSelectedImpactEventId] = useState(''); + const [impactLoading, setImpactLoading] = useState(false); + const [overlayRunning, setOverlayRunning] = useState(false); + const [impactResult, setImpactResult] = useState(null); const [resultMode, setResultMode] = useState('flood'); const readyScenes = useMemo(() => scenes.filter(item => asStatus(item.status) === 'DONE'), [scenes]); @@ -400,11 +499,52 @@ export default function FloodAnalysisWorkspace({ () => doneFloodEvents.find(item => String(item.id) === String(selectedImpactEventId)) || doneFloodEvents[0] || null, [doneFloodEvents, selectedImpactEventId], ); + const selectedSourceRegionTreeId = sourceRegionSelection.city || sourceRegionSelection.province || ''; + const selectedPairRegionTreeId = pairRegionSelection.city || pairRegionSelection.province || ''; const showMessage = useCallback((type, text) => { setMessage({ type, text }); }, []); + const loadRegionProvinces = useCallback(async () => { + setRegionLoading(true); + setRegionError(''); + try { + const data = await getRegionChildren('1'); + const provinces = data.children || []; + setSourceRegionOptions(prev => ({ ...prev, provinces })); + setPairRegionOptions(prev => ({ ...prev, provinces })); + } catch (error) { + setRegionError(getErrorText(error, '行政区加载失败')); + } finally { + setRegionLoading(false); + } + }, []); + + const updateRegionProvince = useCallback(async (target, provinceTreeId) => { + const setSelection = target === 'source' ? setSourceRegionSelection : setPairRegionSelection; + const setOptions = target === 'source' ? setSourceRegionOptions : setPairRegionOptions; + setSelection({ province: provinceTreeId, city: '' }); + setOptions(prev => ({ ...prev, cities: [] })); + if (!provinceTreeId) return; + + setRegionLoading(true); + setRegionError(''); + try { + const data = await getRegionChildren(provinceTreeId); + setOptions(prev => ({ ...prev, cities: data.children || [] })); + } catch (error) { + setRegionError(getErrorText(error, '行政区加载失败')); + } finally { + setRegionLoading(false); + } + }, []); + + const updateRegionCity = useCallback((target, cityTreeId) => { + const setSelection = target === 'source' ? setSourceRegionSelection : setPairRegionSelection; + setSelection(prev => ({ ...prev, city: cityTreeId })); + }, []); + const loadStatusIds = useCallback(async () => { try { const [done, active] = await Promise.all([getFloodDoneRadarIds(), getFloodActiveRadarIds()]); @@ -461,14 +601,28 @@ export default function FloodAnalysisWorkspace({ } }, [showMessage]); + const loadFloodProducts = useCallback(async () => { + setProductLoading(true); + try { + const res = (await getFloodProducts({ limit: LIST_PAGE_SIZE, offset: 0 })).data; + setFloodProducts(res.items || []); + } catch (error) { + setFloodProducts([]); + showMessage('error', `洪涝产品加载失败：${getErrorText(error)}`); + } finally { + setProductLoading(false); + } + }, [showMessage]); + const refreshAll = useCallback(async () => { await Promise.all([ loadStatusIds(), loadScenes(scenesPage), loadWaterResults(waterPage), loadFloodEvents(), + loadFloodProducts(), ]); - }, [loadFloodEvents, loadScenes, loadStatusIds, loadWaterResults, scenesPage, waterPage]); + }, [loadFloodEvents, loadFloodProducts, loadScenes, loadStatusIds, loadWaterResults, scenesPage, waterPage]); useEffect(() => { getSearchOptions() @@ -479,8 +633,9 @@ export default function FloodAnalysisWorkspace({ polarization: data.polarization || [], })) .catch(() => {}); + loadRegionProvinces(); refreshAll(); - }, [refreshAll]); + }, [loadRegionProvinces, refreshAll]); useEffect(() => { if (!runningCount) return undefined; @@ -504,12 +659,16 @@ export default function FloodAnalysisWorkspace({ setRadarLoading(true); setRadarSearched(true); try { + if (sourceAoiMode === 'region' && !selectedSourceRegionTreeId) { + throw new Error('请选择用于筛选的行政区。'); + } const criteria = normalizeRadarSearchCriteria(radarDraft, RADAR_SEARCH_DEFAULTS); const formData = buildRadarSearchFormData({ limit: SEARCH_PAGE_SIZE, offset: page * SEARCH_PAGE_SIZE, criteria, - aoiMode: 'none', + aoiMode: sourceAoiMode, + regionTreeId: sourceAoiMode === 'region' ? selectedSourceRegionTreeId : '', }); const data = await searchRadarData(formData); setRadarResults(data.items || []); @@ -526,6 +685,9 @@ export default function FloodAnalysisWorkspace({ const resetRadarSearch = () => { setRadarDraft({ ...RADAR_SEARCH_DEFAULTS }); + setSourceAoiMode('none'); + setSourceRegionSelection({ province: '', city: '' }); + setSourceRegionOptions(prev => ({ ...prev, cities: [] })); setRadarResults([]); setRadarTotal(0); setRadarPage(0); @@ -541,6 +703,35 @@ export default function FloodAnalysisWorkspace({ )); }; + const updateVisibleIdSet = (setter, itemId, visible) => { + setter(prev => { + const next = new Set(prev); + if (visible) next.add(String(itemId)); + else next.delete(String(itemId)); + return next; + }); + }; + + const handleToggleSourceCoverage = (item) => { + const handler = floodPanel.onShowSourceSceneOnMap || floodPanel.onShowOnMap; + if (!handler) return; + const visible = handler(item); + if (typeof visible === 'boolean') { + updateVisibleIdSet(setSourceCoverageVisibleIds, item.id, visible); + showMessage(visible ? 'success' : 'info', visible ? `范围框 #${item.id} 已显示。` : `范围框 #${item.id} 已清除。`); + } + }; + + const handleToggleSourcePreview = (item) => { + const handler = floodPanel.onShowSourcePreviewOnMap || floodPanel.onShowSourceSceneOnMap || floodPanel.onShowOnMap; + if (!handler) return; + const visible = handler(item); + if (typeof visible === 'boolean') { + updateVisibleIdSet(setSourcePreviewVisibleIds, item.id, visible); + showMessage(visible ? 'success' : 'info', visible ? `源影像 #${item.id} 已显示。` : `源影像 #${item.id} 已清除。`); + } + }; + const handleSubmitPreprocess = async () => { if (readOnly || selectedRadars.length === 0) return; setActionBusy('geocode'); @@ -615,17 +806,36 @@ export default function FloodAnalysisWorkspace({ setPairSearching(true); setCandidatePairs([]); setSelectedPairIdx(null); + setPairSearchSummary(null); try { - const data = await searchFloodPairs({ - pre_start: compactDate(pairPreStart), - pre_end: compactDate(pairPreEnd), - post_start: compactDate(pairPostStart), - post_end: compactDate(pairPostEnd), - overlap_threshold: Number(overlapThreshold) || 0, + if (!compactDate(disasterDate)) { + throw new Error('请选择灾害发生日期。'); + } + if (!selectedPairRegionTreeId) { + throw new Error('请选择灾害影响位置。'); + } + const data = await searchFloodDisasterPairs({ + disaster_name: disasterName || undefined, + disaster_date: compactDate(disasterDate), + region_tree_id: selectedPairRegionTreeId, + pre_window_days: Number(preWindowDays) || 30, + post_window_days: Number(postWindowDays) || 30, + min_aoi_coverage_ratio: Number(minAoiCoverage) || 0, + min_pair_overlap_ratio: Number(overlapThreshold) || 0, + polarization: radarDraft.polarization || undefined, + imaging_mode: radarDraft.imaging_mode || undefined, + product_level: radarDraft.product_level || undefined, + require_same_polarization: true, + require_same_imaging_mode: false, }); - setCandidatePairs(data.pairs || []); - setSelectedPairIdx(data.pairs?.length ? 0 : null); - showMessage(data.pairs?.length ? 'success' : 'warn', data.pairs?.length ? `找到 ${data.pairs.length} 组候选配对。` : '没有找到满足条件的配对。'); + const pairs = data.candidate_pairs || data.pairs || []; + setPairSearchSummary(data.summary || null); + setCandidatePairs(pairs); + setSelectedPairIdx(pairs.length ? 0 : null); + showMessage( + pairs.length ? 'success' : 'warn', + pairs.length ? `找到 ${pairs.length} 组候选配对。` : (data.warnings?.[0] || '没有找到满足条件的配对。'), + ); } catch (error) { showMessage('error', `配对推荐失败：${getErrorText(error)}`); } finally { @@ -685,6 +895,80 @@ export default function FloodAnalysisWorkspace({ } }; + const handleShowPairCoverage = (pair) => { + if (!pair?.pre?.coverage_polygon || !pair?.post?.coverage_polygon) { + showMessage('warn', '该候选配对缺少覆盖范围，无法上图。'); + return; + } + floodPanel.onShowFloodPairOnMap?.(pair); + showMessage('success', '候选配对覆盖范围已加载到地图。'); + }; + + const handleShowFloodVector = () => { + if (!impactResult?.flood_vector_geojson) { + showMessage('warn', '该套合结果没有可用洪涝矢量。'); + return; + } + floodPanel.onShowFloodVectorOnMap?.(impactResult); + showMessage('success', '洪涝矢量已加载到地图。'); + }; + + const loadImpactResult = useCallback(async (detectionId, { silent = false } = {}) => { + if (!detectionId) return; + setImpactLoading(true); + try { + const data = await getFloodImpact(detectionId); + setImpactResult(data); + if (!silent && data?.warnings?.includes?.('overlay has not been run')) { + showMessage('warn', '该洪涝结果尚未运行套合分析。'); + } + } catch (error) { + setImpactResult(null); + if (!silent) showMessage('error', `套合结果加载失败：${getErrorText(error)}`); + } finally { + setImpactLoading(false); + } + }, [showMessage]); + + const handleRunOverlay = async () => { + if (readOnly || !selectedImpactEvent) return; + setOverlayRunning(true); + try { + await runFloodOverlay(selectedImpactEvent.id, { near_threshold_m: 500 }); + await loadImpactResult(selectedImpactEvent.id, { silent: true }); + await loadFloodEvents(); + showMessage('success', `洪涝结果 #${selectedImpactEvent.id} 套合分析已完成。`); + } catch (error) { + showMessage('error', `套合分析失败：${getErrorText(error)}`); + } finally { + setOverlayRunning(false); + } + }; + + const handleCreateFloodProduct = async (event) => { + if (readOnly || !event) return; + setProductBusyId(event.id); + try { + await createFloodProduct(event.id); + await loadFloodProducts(); + showMessage('success', `洪涝结果 #${event.id} 产品已生成。`); + setResultMode('products'); + } catch (error) { + showMessage('error', `产品生成失败：${getErrorText(error)}`); + } finally { + setProductBusyId(null); + } + }; + + const handleOpenProductManifest = async (product) => { + try { + const manifest = await getFloodProductManifest(product.id || product.product_id); + showMessage('success', `Manifest 已读取：${manifest?.schema || product.product_id || product.id}`); + } catch (error) { + showMessage('error', `Manifest 读取失败：${getErrorText(error)}`); + } + }; + const handleToggleFloodLayer = (eventId, key, visible) => { setMapLayerVis(prev => ({ ...prev, [eventId]: { ...prev[eventId], [key]: visible } })); floodPanel.onToggleFloodLayer?.(eventId, key, visible); @@ -692,9 +976,15 @@ export default function FloodAnalysisWorkspace({ const handleSelectImpact = (event) => { setSelectedImpactEventId(String(event.id)); + setImpactResult(null); setActiveView('impact'); }; + useEffect(() => { + if (activeView !== 'impact' || !selectedImpactEvent) return; + loadImpactResult(selectedImpactEvent.id, { silent: true }); + }, [activeView, loadImpactResult, selectedImpactEvent]); + const radarTotalPages = Math.ceil(radarTotal / SEARCH_PAGE_SIZE); const scenesTotalPages = Math.ceil(scenesTotal / LIST_PAGE_SIZE); const waterTotalPages = Math.ceil(waterTotal / LIST_PAGE_SIZE); @@ -799,6 +1089,23 @@ export default function FloodAnalysisWorkspace({ {searchOptions.polarization.map(item => )} +
+
+ 位置过滤 + + +
+ {sourceAoiMode === 'region' && ( + updateRegionProvince('source', value)} + onCityChange={value => updateRegionCity('source', value)} + disabled={regionLoading} + /> + )} + {regionError &&
{regionError}
} +
{!radarSearched && 输入条件后查询入库雷达数据。} @@ -807,6 +1114,8 @@ export default function FloodAnalysisWorkspace({ const selected = selectedRadars.some(row => row.id === item.id); const done = doneRadarIds.includes(item.id); const active = activeRadarIds.includes(item.id); + const coverageVisible = sourceCoverageVisibleIds.has(String(item.id)); + const previewVisible = sourcePreviewVisibleIds.has(String(item.id)); return (
@@ -822,14 +1131,32 @@ export default function FloodAnalysisWorkspace({ {[item.imaging_mode, item.product_level, item.polarization].filter(Boolean).join(' · ') || '-'}
- +
+ + + +
); @@ -911,19 +1238,47 @@ export default function FloodAnalysisWorkspace({ )} /> -
- - - - - +
+
+ setDisasterName(event.target.value)} placeholder="灾害名称（可选）" style={inputStyle} /> + +
+ updateRegionProvince('pair', value)} + onCityChange={value => updateRegionCity('pair', value)} + disabled={regionLoading} + /> +
+ + + + +
+ {pairSearchSummary && ( +
+ + + +
+ )}
{candidatePairs.length === 0 && 暂无候选配对。} @@ -946,12 +1301,37 @@ export default function FloodAnalysisWorkspace({ #{pair.pre.id} {formatYmd(pair.pre.imaging_date, language)} {'->'} #{pair.post.id} {formatYmd(pair.post.imaging_date, language)} + {pair.score != null && 评分 {Number(pair.score).toFixed(2)}} {active ? '已选' : '候选'}
- 重叠 {(Number(pair.overlap_ratio || 0) * 100).toFixed(1)}% + 配对重叠 {formatPercent(pair.overlap_ratio)} + {pair.aoi_coverage_ratio != null ? ` · AOI覆盖 ${formatPercent(pair.aoi_coverage_ratio)}` : ''} {pair.time_diff_days != null ? ` · 间隔 ${pair.time_diff_days} 天` : ''} + {pair.pre_delta_days != null ? ` · 灾前 ${pair.pre_delta_days} 天` : ''} + {pair.post_delta_days != null ? ` · 灾后 ${pair.post_delta_days} 天` : ''} {pair.pre.satellite ? ` · ${pair.pre.satellite}` : ''} + {pair.pre.polarization ? ` · ${pair.pre.polarization}` : ''} +
+
+ { + event.stopPropagation(); + handleShowPairCoverage(pair); + }} + onKeyDown={event => { + if (event.key === 'Enter' || event.key === ' ') { + event.preventDefault(); + event.stopPropagation(); + handleShowPairCoverage(pair); + } + }} + > + 预览覆盖 +
); @@ -973,6 +1353,8 @@ export default function FloodAnalysisWorkspace({ onShowMap={handleShowFloodEvent} onToggleLayer={handleToggleFloodLayer} onSelectImpact={handleSelectImpact} + onCreateProduct={handleCreateFloodProduct} + productBusy={productBusyId === event.id} /> ))}
@@ -983,15 +1365,18 @@ export default function FloodAnalysisWorkspace({ {activeView === 'impact' && (
- + selectedImpactEvent && loadImpactResult(selectedImpactEvent.id)}>刷新结果} + />
{[ ['洪涝分类栅格', selectedImpactEvent ? `结果 #${selectedImpactEvent.id}` : '请选择结果', selectedImpactEvent ? 'ok' : 'muted'], - ['灾害点', '已有底库', 'ok'], - ['洪涝矢量化', '待接入', 'warn'], - ['行政区统计', '待接入', 'muted'], - ['当前 AOI', '待接入', 'muted'], - ['自定义矢量', '待接入', 'muted'], + ['洪涝矢量化', impactResult?.warnings?.includes?.('overlay has not been run') ? '未生成' : (impactResult ? '已生成' : '待运行'), impactResult && !impactResult?.warnings?.includes?.('overlay has not been run') ? 'ok' : 'warn'], + ['灾害点命中', impactResult ? `${impactResult.hazard_points?.inside_flood?.length || 0} 个` : '待运行', impactResult ? 'ok' : 'muted'], + ['近洪涝风险点', impactResult ? `${impactResult.hazard_points?.near_flood?.length || 0} 个` : '待运行', impactResult ? 'info' : 'muted'], + ['DInSAR关联', impactResult ? `${impactResult.dinsar_products?.length || 0} 个` : '待运行', impactResult ? 'info' : 'muted'], + ['行政区统计', impactResult?.affected_aois?.length ? `${impactResult.affected_aois.length} 个` : '暂无数据', impactResult ? 'muted' : 'muted'], ].map(([name, status, tone]) => (
{name} @@ -1035,10 +1420,69 @@ export default function FloodAnalysisWorkspace({
- + +
+ +
+ + {impactLoading && 套合结果加载中...} + {!impactLoading && !impactResult && 选择一个已完成的洪涝结果后运行套合分析。} + {!impactLoading && impactResult && ( +
+
+ + + + + +
+ {impactResult.warnings?.length > 0 && ( +
+ {impactResult.warnings.join('；')} +
+ )} +
+ {(impactResult.hazard_points?.inside_flood || []).slice(0, 5).map(point => ( +
+ {point.name || `灾害点 #${point.id}`} +
+ 洪涝范围内 · {[point.type, point.city, point.county].filter(Boolean).join(' / ') || '-'} +
+
+ ))} + {(impactResult.hazard_points?.near_flood || []).slice(0, 5).map(point => ( +
+ {point.name || `灾害点 #${point.id}`} +
+ 距洪涝范围 {point.distance_m ?? '-'} m · {[point.type, point.city, point.county].filter(Boolean).join(' / ') || '-'} +
+
+ ))} + {(impactResult.dinsar_products || []).slice(0, 5).map(product => ( +
+ {product.display_name || product.product_id} +
+ {product.engine || '-'} · 形变 {product.deformation_mm ?? '-'} mm · AI {product.ai_score ?? '-'} +
+
+ ))} + {(impactResult.affected_aois || []).slice(0, 5).map(aoi => ( +
+ {aoi.name || aoi.tree_id} +
+ {aoi.level || '-'} · 受淹面积 {formatArea(aoi.flood_area_km2)} +
+
+ ))} +
+
+ )} +
)} @@ -1051,6 +1495,7 @@ export default function FloodAnalysisWorkspace({ <> + )} @@ -1067,6 +1512,8 @@ export default function FloodAnalysisWorkspace({ onShowMap={handleShowFloodEvent} onToggleLayer={handleToggleFloodLayer} onSelectImpact={handleSelectImpact} + onCreateProduct={handleCreateFloodProduct} + productBusy={productBusyId === event.id} /> ))} @@ -1079,13 +1526,22 @@ export default function FloodAnalysisWorkspace({ ))} )} + {resultMode === 'products' && ( +
+ {productLoading && 洪涝产品加载中...} + {!productLoading && floodProducts.length === 0 && 暂无洪涝产品。可在洪涝结果行点击“生成产品”。} + {!productLoading && floodProducts.map(product => ( + + ))} +
+ )}
- - + +
diff --git a/frontend/src/ProductionWorkspace.jsx b/frontend/src/ProductionWorkspace.jsx index b98bb4f..0738775 100644 --- a/frontend/src/ProductionWorkspace.jsx +++ b/frontend/src/ProductionWorkspace.jsx @@ -8,9 +8,8 @@ import { import { PanelLoadingBody } from './components/app/AppLoadingFallbacks'; const LazyDinsarProductionPanel = lazy(() => import('./DinsarProductionPanel')); -const LazyTimeseriesProductionPanel = lazy(() => import('./TimeseriesProductionPanel')); +const LazySbasInsarProductionPanel = lazy(() => import('./SbasInsarProductionPanel')); const LazyDinsarProductsPanel = lazy(() => import('./DinsarProductsPanel')); -const LazyPsinsarCatalogPanel = lazy(() => import('./components/PsinsarCatalogPanel')); const shellStyle = { minHeight: '100%', @@ -64,18 +63,10 @@ export default function ProductionWorkspace({ onTaskStart?.(taskId, 'D-InSAR 任务已入队，等待处理...'); }; - const handleTimeseriesRunQueued = taskId => { - onTaskStart?.(taskId, '时序InSAR 运行已入队，当前默认执行 SBAS 流程...'); - }; - const handleDinsarProductQueued = taskId => { onTaskStart?.(taskId, 'D-InSAR 产物任务已入队，等待处理...'); }; - const handleTimeseriesProductQueued = taskId => { - onTaskStart?.(taskId, '时序InSAR 产物目录任务已入队，等待处理...'); - }; - return (
@@ -85,8 +76,8 @@ export default function ProductionWorkspace({

生产管理

- 这里统一承载 D-InSAR 与时序InSAR的运行和产物工作台。当前时序入口默认接入 SBAS 实现， - 后续可在同一界面继续扩展 PS-InSAR、SBAS-InSAR 以及更多 WSL2 引擎实例。 + 这里统一承载 D-InSAR 与 Gamma SBAS-InSAR 生产工作台。旧 ISCE2/MintPy 时序入口已停用， + SBAS 生产、速率图、质量指标与监测点曲线统一进入独立 SBAS-InSAR 页面。

@@ -107,17 +98,17 @@ export default function ProductionWorkspace({
-
时序当前实现
-
SBAS 默认接入
+
SBAS 当前实现
+
Gamma 独立入口
- 现阶段实际运行链路为 SBAS，界面命名已统一为时序InSAR。 + 生产链路绕开旧时序配对层，由 SBAS 页面管理栈发现、基线审核、配准与产物发布。
-
引擎预留
-
ISCE / Gamma 可扩
+
旧链路状态
+
ISCE2/MintPy 停用
- 现有 WSL2 嵌入保持不变，后续增加 Gamma 时可直接挂入当前工作台。 + 历史代码暂时保留兼容，生产管理不再暴露旧“时序运行/产物”页面。
@@ -182,10 +173,9 @@ export default function ProductionWorkspace({ onJobQueued={handleDinsarRunQueued} /> )} - {activeView === 'timeseries_runs' && ( - )} {activeView === 'dinsar_products' && ( @@ -194,13 +184,6 @@ export default function ProductionWorkspace({ onJobQueued={handleDinsarProductQueued} /> )} - {activeView === 'timeseries_products' && ( - - )} diff --git a/frontend/src/SbasInsarProductionPanel.jsx b/frontend/src/SbasInsarProductionPanel.jsx new file mode 100644 index 0000000..5b5d73e --- /dev/null +++ b/frontend/src/SbasInsarProductionPanel.jsx @@ -0,0 +1,1167 @@ +import React, { useCallback, useEffect, useMemo, useState } from 'react'; + +import { + auditSbasInsarStack, + decideSbasInsarItab, + discoverSbasInsarStacks, + getSbasInsarArtifactUrl, + getSbasInsarCapabilities, + getSbasInsarRun, + getSbasInsarRunArtifactUrl, + getSbasInsarTrialRun, + listSbasInsarRuns, + listSbasInsarTrialRuns, + prepareSbasInsarCoregistration, + runSbasInsarBaselineAudit, + submitSbasInsarCoregistrationJob, + submitSbasInsarRun, +} from './api/sbasInsarProduction'; + +const shellStyle = { + display: 'grid', + gap: 12, +}; + +const sectionStyle = { + background: '#ffffff', + border: '1px solid #d8dee8', + borderRadius: 8, + padding: 14, +}; + +const mutedStyle = { + color: '#64748b', + fontSize: 12, + lineHeight: 1.55, +}; + +const labelStyle = { + color: '#475569', + fontSize: 12, +}; + +const valueStyle = { + color: '#0f172a', + fontSize: 14, + fontWeight: 650, +}; + +const gridStyle = { + display: 'grid', + gridTemplateColumns: 'minmax(280px, 360px) minmax(0, 1fr)', + gap: 12, + alignItems: 'start', +}; + +const metricGridStyle = { + display: 'grid', + gridTemplateColumns: 'repeat(auto-fit, minmax(150px, 1fr))', + gap: 10, +}; + +const buttonBaseStyle = { + width: '100%', + textAlign: 'left', + border: '1px solid #d8dee8', + borderRadius: 8, + background: '#ffffff', + padding: '10px 12px', + cursor: 'pointer', +}; + +function formatValue(value, suffix = '') { + const numeric = Number(value); + if (!Number.isFinite(numeric)) return '-'; + return `${numeric.toFixed(Math.abs(numeric) >= 100 ? 1 : 2)}${suffix}`; +} + +function formatBytes(value) { + const size = Number(value || 0); + if (!Number.isFinite(size) || size <= 0) return '-'; + if (size < 1024) return `${size} B`; + const units = ['KB', 'MB', 'GB', 'TB']; + let current = size / 1024; + let index = 0; + while (current >= 1024 && index < units.length - 1) { + current /= 1024; + index += 1; + } + return `${current.toFixed(current >= 100 ? 0 : 1)} ${units[index]}`; +} + +function StatusBadge({ value }) { + const okValues = new Set(['TRIAL_READY', 'READY', 'READY_FOR_GAMMA_BASELINE_AUDIT']); + const isOk = okValues.has(value); + const color = isOk ? '#0f766e' : '#92400e'; + return ( + + + {value || 'UNKNOWN'} + + ); +} + +function Metric({ label, value }) { + return ( +
+
{label}
+
{value}
+
+ ); +} + +function ArtifactLink({ trialId, artifact }) { + const href = getSbasInsarArtifactUrl(trialId, artifact.relative_path); + return ( + + 打开 + + ); +} + +function RunArtifactLink({ runId, artifact }) { + const href = getSbasInsarRunArtifactUrl(runId, artifact.relative_path); + return ( + + 打开 + + ); +} + +export default function SbasInsarProductionPanel({ readOnly = false }) { + const [capabilities, setCapabilities] = useState(null); + const [trials, setTrials] = useState([]); + const [runs, setRuns] = useState([]); + const [selectedTrialId, setSelectedTrialId] = useState(''); + const [selectedRunId, setSelectedRunId] = useState(''); + const [detail, setDetail] = useState(null); + const [runDetail, setRunDetail] = useState(null); + const [loading, setLoading] = useState(false); + const [detailLoading, setDetailLoading] = useState(false); + const [runDetailLoading, setRunDetailLoading] = useState(false); + const [error, setError] = useState(''); + const [discovering, setDiscovering] = useState(false); + const [stackCandidates, setStackCandidates] = useState([]); + const [selectedStackId, setSelectedStackId] = useState(''); + const [auditLoading, setAuditLoading] = useState(false); + const [stackAudit, setStackAudit] = useState(null); + const [submitLoading, setSubmitLoading] = useState(false); + const [baselineAuditLoading, setBaselineAuditLoading] = useState(false); + const [itabDecisionLoading, setItabDecisionLoading] = useState(false); + const [coregistrationLoading, setCoregistrationLoading] = useState(false); + const [coregistrationJobLoading, setCoregistrationJobLoading] = useState(false); + const [coregistrationJob, setCoregistrationJob] = useState(null); + + const loadTrials = useCallback(async () => { + setLoading(true); + setError(''); + try { + const [capabilityData, trialData, runData] = await Promise.all([ + getSbasInsarCapabilities(), + listSbasInsarTrialRuns(), + listSbasInsarRuns(), + ]); + const items = Array.isArray(trialData?.items) ? trialData.items : []; + const runItems = Array.isArray(runData?.items) ? runData.items : []; + setCapabilities(capabilityData); + setTrials(items); + setRuns(runItems); + setSelectedTrialId(current => current || items[0]?.trial_id || ''); + setSelectedRunId(current => current || runItems[0]?.run_id || ''); + } catch (exc) { + setError(exc?.response?.data?.detail || exc.message || 'SBAS-InSAR 列表加载失败'); + setCapabilities(null); + setTrials([]); + setRuns([]); + } finally { + setLoading(false); + } + }, []); + + const loadDetail = useCallback(async trialId => { + if (!trialId) { + setDetail(null); + return; + } + setDetailLoading(true); + setError(''); + try { + const data = await getSbasInsarTrialRun(trialId); + setDetail(data); + } catch (exc) { + setError(exc?.response?.data?.detail || exc.message || 'SBAS-InSAR 详情加载失败'); + setDetail(null); + } finally { + setDetailLoading(false); + } + }, []); + + useEffect(() => { + loadTrials(); + }, [loadTrials]); + + useEffect(() => { + loadDetail(selectedTrialId); + }, [loadDetail, selectedTrialId]); + + const loadRunDetail = useCallback(async runId => { + if (!runId) { + setRunDetail(null); + return; + } + setRunDetailLoading(true); + setError(''); + try { + const data = await getSbasInsarRun(runId); + setRunDetail(data); + } catch (exc) { + setError(exc?.response?.data?.detail || exc.message || 'SBAS-InSAR 生产 Run 详情加载失败'); + setRunDetail(null); + } finally { + setRunDetailLoading(false); + } + }, []); + + useEffect(() => { + loadRunDetail(selectedRunId); + }, [loadRunDetail, selectedRunId]); + + const handleDiscoverStacks = useCallback(async () => { + setDiscovering(true); + setError(''); + setStackAudit(null); + try { + const data = await discoverSbasInsarStacks({ + min_scenes: 3, + require_orbits: true, + include_scenes: false, + limit: 30, + }); + const items = Array.isArray(data?.items) ? data.items : []; + setStackCandidates(items); + setSelectedStackId(current => current || items[0]?.stack_id || ''); + } catch (exc) { + setError(exc?.response?.data?.detail || exc.message || 'SBAS-InSAR 栈发现失败'); + setStackCandidates([]); + } finally { + setDiscovering(false); + } + }, []); + + const handleAuditStack = useCallback(async stackId => { + if (!stackId) return; + setAuditLoading(true); + setError(''); + try { + const data = await auditSbasInsarStack(stackId, { + min_scenes: 3, + require_orbits: true, + }); + setStackAudit(data); + } catch (exc) { + setError(exc?.response?.data?.detail || exc.message || 'SBAS-InSAR 栈审查失败'); + setStackAudit(null); + } finally { + setAuditLoading(false); + } + }, []); + + const handleSubmitRun = useCallback(async () => { + if (!selectedStackId || readOnly) return; + setSubmitLoading(true); + setError(''); + try { + const candidate = stackCandidates.find(item => item.stack_id === selectedStackId); + const data = await submitSbasInsarRun(selectedStackId, { + run_label: candidate + ? `${candidate.satellite || 'LT1'} ${candidate.relative_orbit || ''} ${candidate.center_bucket || ''}`.trim() + : undefined, + min_scenes: 3, + require_orbits: true, + dry_run: true, + monitor_point_strategy: 'auto_low_sigma_high_rate', + }); + const runId = data?.run?.run_id; + const runData = await listSbasInsarRuns(); + const runItems = Array.isArray(runData?.items) ? runData.items : []; + setRuns(runItems); + if (runId) { + setSelectedRunId(runId); + setRunDetail(data); + } + } catch (exc) { + setError(exc?.response?.data?.detail || exc.message || 'SBAS-InSAR 生产 Run 创建失败'); + } finally { + setSubmitLoading(false); + } + }, [readOnly, selectedStackId, stackCandidates]); + + const handleBaselineAudit = useCallback(async (execute = false) => { + if (!selectedRunId || readOnly) return; + setBaselineAuditLoading(true); + setError(''); + try { + const data = await runSbasInsarBaselineAudit(selectedRunId, { + execute, + rlks: 8, + azlks: 8, + max_delta_n: 1, + timeout_seconds: 21600, + }); + setRunDetail(data); + const runData = await listSbasInsarRuns(); + setRuns(Array.isArray(runData?.items) ? runData.items : []); + } catch (exc) { + setError(exc?.response?.data?.detail || exc.message || 'SBAS-InSAR baseline audit 失败'); + } finally { + setBaselineAuditLoading(false); + } + }, [readOnly, selectedRunId]); + + const handleItabDecision = useCallback(async decision => { + if (!selectedRunId || readOnly) return; + setItabDecisionLoading(true); + setError(''); + try { + const data = await decideSbasInsarItab(selectedRunId, { + decision, + reviewer: 'ui', + note: decision === 'approve' + ? 'Baseline-audited adjacent itab accepted from SBAS production page.' + : 'Baseline-audited adjacent itab rejected from SBAS production page.', + }); + setRunDetail(data); + const runData = await listSbasInsarRuns(); + setRuns(Array.isArray(runData?.items) ? runData.items : []); + } catch (exc) { + setError(exc?.response?.data?.detail || exc.message || 'SBAS-InSAR itab 审批失败'); + } finally { + setItabDecisionLoading(false); + } + }, [readOnly, selectedRunId]); + + const handlePrepareCoregistration = useCallback(async () => { + if (!selectedRunId || readOnly) return; + setCoregistrationLoading(true); + setError(''); + try { + const data = await prepareSbasInsarCoregistration(selectedRunId, { + execute: false, + rlks: 8, + azlks: 8, + }); + setRunDetail(data); + const runData = await listSbasInsarRuns(); + setRuns(Array.isArray(runData?.items) ? runData.items : []); + } catch (exc) { + setError(exc?.response?.data?.detail || exc.message || 'SBAS-InSAR 共参考配准计划生成失败'); + } finally { + setCoregistrationLoading(false); + } + }, [readOnly, selectedRunId]); + + const handleSubmitCoregistrationJob = useCallback(async () => { + if (!selectedRunId || readOnly) return; + setCoregistrationJobLoading(true); + setError(''); + try { + const data = await submitSbasInsarCoregistrationJob(selectedRunId, { + rlks: 8, + azlks: 8, + timeout_seconds: 43200, + }); + setCoregistrationJob(data); + const detailData = await getSbasInsarRun(selectedRunId); + setRunDetail(detailData); + const runData = await listSbasInsarRuns(); + setRuns(Array.isArray(runData?.items) ? runData.items : []); + } catch (exc) { + setError(exc?.response?.data?.detail || exc.message || 'SBAS-InSAR 共参考配准任务提交失败'); + setCoregistrationJob(null); + } finally { + setCoregistrationJobLoading(false); + } + }, [readOnly, selectedRunId]); + + const artifacts = useMemo(() => detail?.artifacts || [], [detail]); + const primaryPreview = ( + artifacts.find(item => item.key === 'los_rate_toward_mm_per_year_geo_preview_png') + || artifacts.find(item => item.key === 'los_rate_toward_mm_per_year_bmp') + ); + const sigmaPreview = ( + artifacts.find(item => item.key === 'los_sigma_mm_per_year_geo_preview_png') + || artifacts.find(item => item.key === 'los_sigma_mm_per_year_bmp') + ); + const monitorPreview = artifacts.find(item => item.role === 'monitor_point' && item.relative_path.endsWith('.png')); + const monitorCsv = artifacts.find(item => item.role === 'monitor_point' && item.relative_path.endsWith('.csv')); + const productArtifacts = artifacts.filter(item => item.role !== 'monitor_point'); + const trial = detail?.trial || null; + const stack = trial?.stack || {}; + const summary = detail?.summary || {}; + const radar = summary.radar || {}; + const monitorPoint = Array.isArray(summary.monitor_points) ? summary.monitor_points[0] : null; + const selectedStack = stackCandidates.find(item => item.stack_id === selectedStackId) || null; + const run = runDetail?.run || null; + const runManifest = runDetail?.manifest || {}; + const stagePlan = runDetail?.command_manifest?.stage_plan || []; + const runArtifacts = runDetail?.artifacts || []; + const baselineSummary = runManifest.baseline_audit?.summary || null; + const itabDecision = runManifest.baseline_audit?.itab_decision || null; + const coregistrationPlan = runManifest.coregistration || null; + const itabApproved = itabDecision?.decision === 'approve' || runManifest.baseline_audit?.approved_for_next_stage === true; + const itabRejected = itabDecision?.decision === 'reject'; + + return ( +
+
+
+
+

SBAS-InSAR 生产

+
+ Gamma IPTA SBAS 生产入口。当前阶段接入已验证的 LT1/Gamma 试验成果，作业提交在下一阶段开放。 +
+
+ +
+ {capabilities && ( +
+ + + + +
+ )} + {readOnly && ( +
+ 当前账号只读，生产提交按钮不会显示。 +
+ )} + {error && ( +
+ {error} +
+ )} +
+ +
+
+
+

候选 SBAS 序列发现

+
+ 直接扫描本地 LT1 数据池，按平台、相对轨道、升降轨、模式、极化、接收站和中心桶硬分组，并检查精轨 TXT。 +
+
+
+ + + {!readOnly && ( + + )} +
+
+ + {stackCandidates.length > 0 && ( +
+
+ {stackCandidates.map(item => { + const active = item.stack_id === selectedStackId; + return ( + + ); + })} +
+
+ {selectedStack && ( +
+ + + + +
+ )} + {stackAudit && ( +
+
Manifest 已生成
+
+ {stackAudit.manifest_path} +
+
+ 状态：{stackAudit.status}；pair 数： + {stackAudit.manifest?.pair_network?.pairs?.length || 0} +
+ {(stackAudit.manifest?.warnings || []).length > 0 && ( +
+ 警告：{stackAudit.manifest.warnings.join('；')} +
+ )} + {(stackAudit.manifest?.blockers || []).length > 0 && ( +
+ 阻断：{stackAudit.manifest.blockers.join('；')} +
+ )} +
+ )} + {run && ( +
+
计划 Run 已创建
+
+ {run.run_id}；状态：{run.status}；下一步：{run.next_stage || '-'} +
+
+ )} +
+
+ )} +
+ +
+
+
+

生产 Run 计划

+
+ 当前只创建可复现实验记录、Gamma 命令计划和监测点配置；正式执行在 baseline 审核后接入。 +
+
+ {runs.length} 个 +
+ +
+
+ {runs.map(item => { + const active = item.run_id === selectedRunId; + return ( + + ); + })} + {!loading && runs.length === 0 && ( +
+ 暂无计划 Run。先发现序列，再创建计划 Run。 +
+ )} +
+ +
+ {runDetailLoading &&
正在加载 Run 详情...
} + {!runDetailLoading && run && ( + <> +
+ + + + +
+ + {!readOnly && ( +
+ + + + +
+ )} + +
+
Gamma 阶段计划
+
+ {stagePlan.map(stage => ( +
+
+
{stage.label}
+
{(stage.gamma_tools || []).join(', ') || '应用内产品处理'}
+
+ +
+ ))} +
+
+ + {baselineSummary && ( +
+
Baseline Audit 结果
+
+ + + + +
+
+
+ itab 状态： + {itabDecision + ? `${itabDecision.decision} / ${itabDecision.decided_at || '-'}` + : '等待审批'} +
+ {!readOnly && ( +
+ + +
+ )} +
+
+ + + + + + + + + + + {(baselineSummary.adjacent_pairs || []).map(pair => ( + + + + + + + ))} + +
Pair日期Bperp间隔
{pair.pair_index} + {pair.master_date}{' -> '}{pair.slave_date} + + {formatValue(pair.bperp_m, ' m')} + + {formatValue(pair.delta_days, ' d')} +
+
+
+ )} + + {coregistrationPlan && ( +
+
共参考配准计划
+
+ + + + +
+
+ 脚本：{coregistrationPlan.script_path || '-'} +
+
+ 输出目录：{coregistrationPlan.outputs?.common_dir || '-'} +
+ {coregistrationPlan.execution && ( +
+ 执行状态：{coregistrationPlan.execution.status || '-'}； + {coregistrationPlan.execution.ended_at || coregistrationPlan.execution.started_at || '-'} +
+ )} + {coregistrationPlan.summary && ( +
+ RSLC 就绪： + {coregistrationPlan.summary.ready_secondary_count || 0}/ + {coregistrationPlan.summary.expected_secondary_count || 0}； + 缺失日期：{(coregistrationPlan.summary.missing_dates || []).join(', ') || '无'} +
+ )} + {coregistrationJob && coregistrationJob.run_id === run.run_id && ( +
+ 已提交后台任务：{coregistrationJob.task_id}；Job：{coregistrationJob.job_id} +
+ )} +
+ )} + +
+
监测点配置
+
+ 模式：{runDetail.monitor_points?.mode || '-'}； + 点数：{runDetail.monitor_points?.points?.length || 0} +
+
+ {runDetail.monitor_points?.note || '等待产品生成后提取时序曲线。'} +
+
+ + {runArtifacts.length > 0 && ( +
+ + + + + + + + + + {runArtifacts.map(item => ( + + + + + + ))} + +
文件角色操作
{item.label}{item.role} + +
+
+ )} + + )} + {!runDetailLoading && !run && ( +
请选择一个生产 Run，或从候选序列创建新的计划 Run。
+ )} +
+
+
+ +
+
+
+

试验/生产序列

+ {trials.length} 组 +
+
+ {trials.map(item => { + const active = item.trial_id === selectedTrialId; + return ( + + ); + })} + {!loading && trials.length === 0 && ( +
+ 未发现可读取的 Gamma SBAS/IPTA 试验汇总。 +
+ )} +
+
+ +
+ {detailLoading &&
正在加载详情...
} + {!detailLoading && trial && ( +
+
+
+

{trial.trial_id}

+
+ {stack.platform} / relOrbit {stack.relative_orbit} / {stack.direction} / {stack.mode} / {stack.polarization} +
+
+ +
+ +
+ + + + +
+ +
+ {primaryPreview && ( +
+
LOS 速率图
+
+ {primaryPreview.key.endsWith('_geo_preview_png') ? 'WGS84 地理编码预览' : 'RDC 处理几何浏览图'} +
+ LOS velocity toward radar positive +
+ )} + {monitorPreview && ( +
+
监测点形变曲线
+ Monitoring point LOS displacement time series +
+ )} + {sigmaPreview && ( +
+
LOS sigma 图
+
+ {sigmaPreview.key.endsWith('_geo_preview_png') ? 'WGS84 地理编码预览' : 'RDC 处理几何浏览图'} +
+ LOS velocity sigma +
+ )} +
+ +
+
+
LOS 符号约定
+
+ {radar.los_sign_convention || trial.los_sign_convention || '-'} +
+
+
+
监测点
+
+ {monitorPoint ? ( + <> + {monitorPoint.point_id}，约 {formatValue(monitorPoint.approx_lonlat?.lon)}E / + {formatValue(monitorPoint.approx_lonlat?.lat)}N，速率 + {formatValue(monitorPoint.los_rate_toward_mm_per_year, ' mm/yr')} + {monitorCsv && ( + <> + {' '} + + + )} + + ) : '-'} +
+
+
+
+ 当前曲线是自动选取的单个样例点，用于验证时序曲线能力；正式监测点需要用户点击、导入点位或质量筛选后的点集。 +
+ +
+
产品文件
+
+ + + + + + + + + + + {productArtifacts.map(item => ( + + + + + + + ))} + +
产品角色大小操作
{item.label}{item.role} + {formatBytes(item.size_bytes)} + + +
+
+
+
+ )} + {!detailLoading && !trial && ( +
请选择一个 SBAS-InSAR 试验或生产序列。
+ )} +
+
+
+ ); +} diff --git a/frontend/src/api/flood.js b/frontend/src/api/flood.js index 2d1b502..d923afb 100644 --- a/frontend/src/api/flood.js +++ b/frontend/src/api/flood.js @@ -1,14 +1,5 @@ import apiClient from './client'; -export const getFloodSources = (params = {}) => - apiClient.get('/flood/sources', { params }); - -export const refreshFloodSources = () => - apiClient.post('/flood/sources/refresh'); - -export const getFloodSourceReadiness = (id) => - apiClient.get(`/flood/sources/${id}/readiness`); - export const submitFloodPreprocess = (payload) => apiClient.post('/flood/preprocess', payload); @@ -33,26 +24,11 @@ export const getFloodWaterExtractions = (limit = 20, offset = 0, status = null) export const getFloodWaterExtractionPreview = (id) => apiClient.get(`/flood/water-extractions/${id}/preview`).then(r => r.data); -export const getFloodPreprocessRuns = (params = {}) => - apiClient.get('/flood/preprocess-runs', { params }); - -export const getFloodReadyProducts = (params = {}) => - apiClient.get('/flood/ready-products', { params }); - -export const getFloodReadyProductPreview = (id) => - apiClient.get(`/flood/ready-products/${id}/preview`).then(r => r.data); - export const searchFloodPairs = (payload) => - apiClient.post('/flood/pairs/search', payload); + apiClient.post('/flood/pairs/search', payload).then(r => r.data); -export const saveFloodPair = (payload) => - apiClient.post('/flood/pairs', payload); - -export const getFloodPairs = (params = {}) => - apiClient.get('/flood/pairs', { params }); - -export const deleteFloodPair = (id) => - apiClient.delete(`/flood/pairs/${id}`); +export const searchFloodDisasterPairs = (payload) => + apiClient.post('/flood/disaster-pairs/search', payload).then(r => r.data); export const submitFloodDetection = (payload) => apiClient.post('/flood/detections', payload); @@ -60,21 +36,27 @@ export const submitFloodDetection = (payload) => export const getFloodDetections = (params = {}) => apiClient.get('/flood/detections', { params }); -export const getFloodDetection = (id) => - apiClient.get(`/flood/detections/${id}`); - export const getFloodDetectionPreview = (id, layer) => apiClient.get(`/flood/detections/${id}/preview/${layer}`).then(r => r.data); -export const vectorizeFloodDetection = (id, payload = {}) => - apiClient.post(`/flood/detections/${id}/vectorize`, payload); - export const runFloodOverlay = (id, payload = {}) => apiClient.post(`/flood/detections/${id}/overlay`, payload); export const getFloodImpact = (id) => apiClient.get(`/flood/detections/${id}/impact`).then(r => r.data); +export const createFloodProduct = (detectionId) => + apiClient.post(`/flood/detections/${detectionId}/products`); + +export const getFloodProducts = (params = {}) => + apiClient.get('/flood/products', { params }); + +export const getFloodProduct = (id) => + apiClient.get(`/flood/products/${id}`); + +export const getFloodProductManifest = (id) => + apiClient.get(`/flood/products/${id}/manifest`).then(r => r.data); + export const getFloodResults = (params = {}) => apiClient.get('/flood/results', { params }); @@ -83,9 +65,3 @@ export const getFloodResult = (id) => export const getFloodResultManifest = (id) => apiClient.get(`/flood/results/${id}/manifest`).then(r => r.data); - -export const createFloodReport = (payload) => - apiClient.post('/flood/reports', payload); - -export const getFloodReport = (id) => - apiClient.get(`/flood/reports/${id}`).then(r => r.data); diff --git a/frontend/src/api/sbasInsarProduction.js b/frontend/src/api/sbasInsarProduction.js new file mode 100644 index 0000000..3746de1 --- /dev/null +++ b/frontend/src/api/sbasInsarProduction.js @@ -0,0 +1,49 @@ +import apiClient from './client'; + +const encodeArtifactPath = relativePath => + String(relativePath || '') + .split('/') + .map(segment => encodeURIComponent(segment)) + .join('/'); + +export const getSbasInsarCapabilities = () => + apiClient.get('/sbas-insar-production/capabilities').then(r => r.data); + +export const discoverSbasInsarStacks = (payload = {}) => + apiClient.post('/sbas-insar-production/stacks/discover', payload).then(r => r.data); + +export const auditSbasInsarStack = (stackId, payload = {}) => + apiClient.post(`/sbas-insar-production/stacks/${encodeURIComponent(stackId)}/audit`, payload).then(r => r.data); + +export const submitSbasInsarRun = (stackId, payload = {}) => + apiClient.post(`/sbas-insar-production/stacks/${encodeURIComponent(stackId)}/runs`, payload).then(r => r.data); + +export const listSbasInsarRuns = () => + apiClient.get('/sbas-insar-production/runs').then(r => r.data); + +export const getSbasInsarRun = runId => + apiClient.get(`/sbas-insar-production/runs/${encodeURIComponent(runId)}`).then(r => r.data); + +export const runSbasInsarBaselineAudit = (runId, payload = {}) => + apiClient.post(`/sbas-insar-production/runs/${encodeURIComponent(runId)}/baseline-audit`, payload).then(r => r.data); + +export const decideSbasInsarItab = (runId, payload = {}) => + apiClient.post(`/sbas-insar-production/runs/${encodeURIComponent(runId)}/itab-decision`, payload).then(r => r.data); + +export const prepareSbasInsarCoregistration = (runId, payload = {}) => + apiClient.post(`/sbas-insar-production/runs/${encodeURIComponent(runId)}/coregistration`, payload).then(r => r.data); + +export const submitSbasInsarCoregistrationJob = (runId, payload = {}) => + apiClient.post(`/sbas-insar-production/runs/${encodeURIComponent(runId)}/coregistration/jobs`, payload).then(r => r.data); + +export const getSbasInsarRunArtifactUrl = (runId, relativePath) => + `/api/sbas-insar-production/runs/${encodeURIComponent(runId)}/artifacts/${encodeArtifactPath(relativePath)}`; + +export const listSbasInsarTrialRuns = () => + apiClient.get('/sbas-insar-production/trial-runs').then(r => r.data); + +export const getSbasInsarTrialRun = trialId => + apiClient.get(`/sbas-insar-production/trial-runs/${encodeURIComponent(trialId)}`).then(r => r.data); + +export const getSbasInsarArtifactUrl = (trialId, relativePath) => + `/api/sbas-insar-production/trial-runs/${encodeURIComponent(trialId)}/artifacts/${encodeArtifactPath(relativePath)}`; diff --git a/frontend/src/components/ActiveTasksOverlay.jsx b/frontend/src/components/ActiveTasksOverlay.jsx index 4434aa9..98a4e2a 100644 --- a/frontend/src/components/ActiveTasksOverlay.jsx +++ b/frontend/src/components/ActiveTasksOverlay.jsx @@ -22,6 +22,10 @@ const getTaskTypeLabel = (taskType) => { return 'LT-1 解包'; case 'UNPACK_SENTINEL1': return 'Sentinel-1 解包'; + case 'GF3_UNPACK': + return 'GF3 解包'; + case 'GF3_BATCH_PROCESS': + return 'GF3 预处理'; case 'SCAN_ASSET_INVENTORY': return '资产库存扫描'; case 'IDL_IMPORT': diff --git a/frontend/src/components/app/AppSidePanel.jsx b/frontend/src/components/app/AppSidePanel.jsx index d1b672f..1efaa5e 100644 --- a/frontend/src/components/app/AppSidePanel.jsx +++ b/frontend/src/components/app/AppSidePanel.jsx @@ -110,7 +110,7 @@ export default function AppSidePanel({ ? '生产管理' : getLeftTabLabel(leftPanelTab, leftTabLabelContext); const standaloneDescription = isProductionWorkspace - ? '这里统一承载 D-InSAR 与时序InSAR的运行和产物页面，当前时序流程默认接入 SBAS，后续可继续扩展 PS-InSAR / SBAS-InSAR。' + ? '这里统一承载 D-InSAR 与 Gamma SBAS-InSAR 生产；旧 ISCE2/MintPy 时序入口已停用。' : '当前模块已切换为独立工作区模式。'; return ( diff --git a/frontend/src/config/appConstants.js b/frontend/src/config/appConstants.js index 75353a5..c16ba4c 100644 --- a/frontend/src/config/appConstants.js +++ b/frontend/src/config/appConstants.js @@ -66,28 +66,23 @@ export const PRODUCTION_WORKSPACE_VIEWS = [ description: '运行任务编排、引擎切换与过程监控', }, { - key: 'timeseries_runs', - label: '时序InSAR 运行', - description: '当前默认接入 SBAS 流程，后续可扩展 PS-InSAR / SBAS-InSAR', + key: 'sbas_insar_production', + label: 'SBAS-InSAR Production', + description: 'Gamma IPTA SBAS stack production, velocity maps, quality metrics, and monitor-point curves', }, { key: 'dinsar_products', label: 'D-InSAR 产物', description: '结果提取、标准目录发布与产物编目', }, - { - key: 'timeseries_products', - label: '时序InSAR 产物', - description: '当前统一登记时序产物，后续兼容多类型时序成果', - }, ]; export const PRODUCTION_WORKSPACE_ENTRY_TO_VIEW = Object.freeze({ [PRODUCTION_WORKSPACE_TAB]: 'dinsar_runs', dinsar_production: 'dinsar_runs', dinsar_products: 'dinsar_products', - ps_production: 'timeseries_runs', - ps_products: 'timeseries_products', + ps_production: 'sbas_insar_production', + ps_products: 'sbas_insar_production', }); export const PRODUCTION_WORKSPACE_ROUTE_TABS = new Set([ diff --git a/frontend/src/hooks/useDinsarOperations.js b/frontend/src/hooks/useDinsarOperations.js index c94be9d..a413fa0 100644 --- a/frontend/src/hooks/useDinsarOperations.js +++ b/frontend/src/hooks/useDinsarOperations.js @@ -11,7 +11,7 @@ import { normalizePagePayload } from '../utils/appHelpers'; import { normalizeTaskStatus } from '../utils/appUiHelpers'; import { DEFAULT_LIST_PAGE_SIZE } from '../config/appConstants'; -const NON_BLOCKING_TASK_TYPES = new Set(['UNPACK_ARCHIVES', 'UNPACK_SENTINEL1', 'SCAN_ASSET_INVENTORY', 'COPY_DATA']); +const NON_BLOCKING_TASK_TYPES = new Set(['UNPACK_ARCHIVES', 'UNPACK_SENTINEL1', 'GF3_UNPACK', 'SCAN_ASSET_INVENTORY', 'COPY_DATA']); export default function useDinsarOperations({ onCleanupDinsarLayers, @@ -229,6 +229,12 @@ export default function useDinsarOperations({ } else if (taskStatus === 'FAILED') { addLog('error', `LT-1 解包失败: ${taskInfo.message || '未知错误'}`); } + } else if (taskInfo.task_type === 'GF3_UNPACK') { + if (taskStatus === 'COMPLETED') { + addLog('success', taskInfo.message || 'GF3 解包完成。'); + } else if (taskStatus === 'FAILED') { + addLog('error', `GF3 解包失败: ${taskInfo.message || '未知错误'}`); + } } }; diff --git a/frontend/src/hooks/useGlobalTaskControl.js b/frontend/src/hooks/useGlobalTaskControl.js index 728a86e..d64042a 100644 --- a/frontend/src/hooks/useGlobalTaskControl.js +++ b/frontend/src/hooks/useGlobalTaskControl.js @@ -2,7 +2,7 @@ import { useCallback, useEffect, useRef, useState } from 'react'; import apiClient from '../api/client'; import { normalizeTaskStatus } from '../utils/appUiHelpers'; -const NON_BLOCKING_TASK_TYPES = new Set(['UNPACK_ARCHIVES', 'UNPACK_SENTINEL1', 'SCAN_ASSET_INVENTORY', 'COPY_DATA']); +const NON_BLOCKING_TASK_TYPES = new Set(['UNPACK_ARCHIVES', 'UNPACK_SENTINEL1', 'GF3_UNPACK', 'SCAN_ASSET_INVENTORY', 'COPY_DATA']); const isTaskNonBlocking = (taskId, taskType, nonBlockingTaskIds = []) => ( NON_BLOCKING_TASK_TYPES.has(String(taskType || '').toUpperCase())