Add Sentinel-1 asset management and PyINT pipeline support

This commit is contained in:
2026-05-14 03:02:56 +08:00
parent 508ca5641a
commit 729c5c2a7f
61 changed files with 7800 additions and 885 deletions
+3
View File
@@ -55,9 +55,12 @@ ALLOWED_EXPORT_DIRS=
# 源数据目录 # 源数据目录
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
UNPACK_SOURCE_DIRS=D:\Archives UNPACK_SOURCE_DIRS=D:\Archives
SOURCE_PRODUCT_DIRS=D:\LuTan1_Image_Pool;D:\Sentinel1_Image_Pool_ZIP
SENTINEL1_STORAGE_DIRS=D:\Sentinel1_Image_Pool
INSAR_STORAGE_DIRS=D:\LuTan1_Image_Pool INSAR_STORAGE_DIRS=D:\LuTan1_Image_Pool
MONITOR_RADAR_DIRS=D:\LuTan1_Image_Pool MONITOR_RADAR_DIRS=D:\LuTan1_Image_Pool
MONITOR_DINSAR_DIRS=D:\DInSARResult MONITOR_DINSAR_DIRS=D:\DInSARResult
ORBIT_SOURCE_DIRS=D:\LT1_data_lsarorbit;D:\Sentinel1_EOF_Pool
MONITOR_ORBIT_DIR=D:\LT1_data_lsarorbit MONITOR_ORBIT_DIR=D:\LT1_data_lsarorbit
GF3_SOURCE_DIRS=D:\GF3_L1A_Image GF3_SOURCE_DIRS=D:\GF3_L1A_Image
+10
View File
@@ -167,6 +167,9 @@ class Settings(BaseSettings):
DB_SCHEMA_RESET_CONFIRM: bool = False DB_SCHEMA_RESET_CONFIRM: bool = False
UNPACK_SOURCE_DIRS: str = "" UNPACK_SOURCE_DIRS: str = ""
SOURCE_PRODUCT_DIRS: str = ""
SENTINEL1_STORAGE_DIRS: str = ""
ORBIT_SOURCE_DIRS: str = ""
INSAR_STORAGE_DIRS: str = "" INSAR_STORAGE_DIRS: str = ""
MONITOR_RADAR_DIRS: str = "" MONITOR_RADAR_DIRS: str = ""
MONITOR_DINSAR_DIRS: str = "" MONITOR_DINSAR_DIRS: str = ""
@@ -859,6 +862,13 @@ def validate_runtime_config() -> dict[str, Any]:
_check_path(label="SRTM_DEM_DIR", value=settings.SRTM_DEM_DIR, errors=errors, warnings=warnings, expect_file=False) _check_path(label="SRTM_DEM_DIR", value=settings.SRTM_DEM_DIR, errors=errors, warnings=warnings, expect_file=False)
_check_path(label="WATER_RESULTS_DIR", value=settings.WATER_RESULTS_DIR, errors=errors, warnings=warnings, expect_file=False) _check_path(label="WATER_RESULTS_DIR", value=settings.WATER_RESULTS_DIR, errors=errors, warnings=warnings, expect_file=False)
_check_path(label="MONITOR_ORBIT_DIR", value=settings.MONITOR_ORBIT_DIR, errors=errors, warnings=warnings, expect_file=False) _check_path(label="MONITOR_ORBIT_DIR", value=settings.MONITOR_ORBIT_DIR, errors=errors, warnings=warnings, expect_file=False)
for label, value in (
("SOURCE_PRODUCT_DIRS", settings.SOURCE_PRODUCT_DIRS),
("SENTINEL1_STORAGE_DIRS", settings.SENTINEL1_STORAGE_DIRS),
("ORBIT_SOURCE_DIRS", settings.ORBIT_SOURCE_DIRS),
):
for item in split_env_paths(value):
_check_path(label=label, value=item, errors=errors, warnings=warnings, expect_file=False)
_check_path(label="ORBIT_POOL_ENVI", value=settings.ORBIT_POOL_ENVI, errors=errors, warnings=warnings, expect_file=False) _check_path(label="ORBIT_POOL_ENVI", value=settings.ORBIT_POOL_ENVI, errors=errors, warnings=warnings, expect_file=False)
_check_path(label="ORBIT_POOL_ISCE2", value=settings.ORBIT_POOL_ISCE2, errors=errors, warnings=warnings, expect_file=False) _check_path(label="ORBIT_POOL_ISCE2", value=settings.ORBIT_POOL_ISCE2, errors=errors, warnings=warnings, expect_file=False)
_check_path(label="RESULT_PUBLISH_ROOT", value=settings.RESULT_PUBLISH_ROOT, errors=errors, warnings=warnings, expect_file=False) _check_path(label="RESULT_PUBLISH_ROOT", value=settings.RESULT_PUBLISH_ROOT, errors=errors, warnings=warnings, expect_file=False)
+1
View File
@@ -36,6 +36,7 @@ MIGRATION_FILES = [
"007_timeseries_stack_plan_trace.sql", "007_timeseries_stack_plan_trace.sql",
"008_timeseries_stack_plan_edges.sql", "008_timeseries_stack_plan_edges.sql",
"009_raw_source_pairing_fields.sql", "009_raw_source_pairing_fields.sql",
"010_source_orbit_asset_inventory.sql",
] ]
+17 -2
View File
@@ -10,6 +10,7 @@ from typing import Any, Dict, List
from ..config import get_env_text, read_bool_env, settings from ..config import get_env_text, read_bool_env, settings
from .base import DinsarEngine, EngineAvailability, EngineProfile, RunRequest, RunResult from .base import DinsarEngine, EngineAvailability, EngineProfile, RunRequest, RunResult
from ..utils import normalize_satellite_family
from ..services.dinsar_naming import ( from ..services.dinsar_naming import (
PAIR_META_FILENAME, PAIR_META_FILENAME,
build_fallback_pair_key, build_fallback_pair_key,
@@ -728,7 +729,14 @@ class Isce2Engine(DinsarEngine):
task_name = os.path.basename(os.path.normpath(task_dir)) task_name = os.path.basename(os.path.normpath(task_dir))
pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {} pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {}
task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(task_alias, task_dir) satellite_family = normalize_satellite_family(
pair_meta.get("master_satellite") or pair_meta.get("slave_satellite")
)
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(
task_alias,
task_dir,
satellite_family=satellite_family,
)
pointer_path = os.path.join( pointer_path = os.path.join(
settings.DINSAR_PRODUCT_DIR, settings.DINSAR_PRODUCT_DIR,
pair_key, pair_key,
@@ -1085,7 +1093,14 @@ class Isce2Engine(DinsarEngine):
task_name = os.path.basename(os.path.normpath(task_dir)) task_name = os.path.basename(os.path.normpath(task_dir))
pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {} pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {}
task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(task_alias, task_dir) satellite_family = normalize_satellite_family(
pair_meta.get("master_satellite") or pair_meta.get("slave_satellite")
)
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(
task_alias,
task_dir,
satellite_family=satellite_family,
)
output_root = self._output_root or os.path.join(task_dir, "isce2_output") output_root = self._output_root or os.path.join(task_dir, "isce2_output")
run_dir = managed_run_dir_override or os.path.normpath( run_dir = managed_run_dir_override or os.path.normpath(
os.path.join(output_root, pair_key, "runs", run_key) os.path.join(output_root, pair_key, "runs", run_key)
+31 -11
View File
@@ -43,6 +43,7 @@ from ..services.pyint_service import (
REFLATTEN_MODEL_CHOICES, REFLATTEN_MODEL_CHOICES,
TARGET_GRID_SIZE_MAX_M, TARGET_GRID_SIZE_MAX_M,
TARGET_GRID_SIZE_MIN_M, TARGET_GRID_SIZE_MIN_M,
build_profile_project_name,
build_project_name, build_project_name,
calculate_dem_oversampling, calculate_dem_oversampling,
calculate_looks_from_task_dir, calculate_looks_from_task_dir,
@@ -290,13 +291,13 @@ class PyintEngine(DinsarEngine):
) )
return _windows_path_to_wsl_mount(str(local_script)) return _windows_path_to_wsl_mount(str(local_script))
def _pipeline_script_for_profile(self, profile: str) -> str:
script_name = "run_s1_pyint_pipeline.py" if str(profile or "").strip() == "s1_gamma_dinsar" else "run_lt1_pyint_pipeline.py"
local_script = Path(__file__).resolve().parent.parent / "pyint_pipeline" / script_name
return _windows_path_to_wsl_mount(str(local_script))
def get_profiles(self) -> List[EngineProfile]: def get_profiles(self) -> List[EngineProfile]:
return [ shared_schema = {
EngineProfile(
code="lt1_gamma_dinsar",
label="LT-1 Gamma D-InSAR",
description="Use PyINT + Gamma in WSL for single-pair LT-1 D-InSAR processing.",
params_schema={
"force": { "force": {
"label": "强制重跑", "label": "强制重跑",
"type": "boolean", "type": "boolean",
@@ -459,7 +460,19 @@ class PyintEngine(DinsarEngine):
"section": "Execution", "section": "Execution",
"description": "关闭后不导出地理编码结果。", "description": "关闭后不导出地理编码结果。",
}, },
}, }
return [
EngineProfile(
code="lt1_gamma_dinsar",
label="LT-1 Gamma D-InSAR",
description="Use PyINT + Gamma in WSL for single-pair LT-1 D-InSAR processing.",
params_schema=shared_schema,
),
EngineProfile(
code="s1_gamma_dinsar",
label="Sentinel-1 Gamma D-InSAR",
description="Use PyINT + Gamma in WSL for single-pair Sentinel-1 D-InSAR processing.",
params_schema=shared_schema,
), ),
] ]
@@ -694,7 +707,7 @@ class PyintEngine(DinsarEngine):
error="PyINT is disabled.", error="PyINT is disabled.",
) )
if request.profile != "lt1_gamma_dinsar": if request.profile not in {"lt1_gamma_dinsar", "s1_gamma_dinsar"}:
return RunResult( return RunResult(
success=False, success=False,
engine_code=self.engine_code, engine_code=self.engine_code,
@@ -859,7 +872,11 @@ class PyintEngine(DinsarEngine):
else os.path.join(run_dir, "native") else os.path.join(run_dir, "native")
) )
template_root = os.path.normpath(os.path.join(self._template_root, pair_key, run_key)) template_root = os.path.normpath(os.path.join(self._template_root, pair_key, run_key))
project_name = build_project_name(pair_key, run_key) project_name = build_profile_project_name(
task_identity.get("satellite_family"),
pair_key,
run_key,
)
project_dir = os.path.join(work_run_root, project_name) project_dir = os.path.join(work_run_root, project_name)
# Keep input assets outside the run root because the WSL pipeline may delete run_root on --force. # Keep input assets outside the run root because the WSL pipeline may delete run_root on --force.
input_assets_dir = os.path.join(self._work_root, pair_key, "input_assets", run_key) input_assets_dir = os.path.join(self._work_root, pair_key, "input_assets", run_key)
@@ -1103,7 +1120,7 @@ class PyintEngine(DinsarEngine):
) )
cmd_parts = [ cmd_parts = [
f"{quote_shell(self._python)} {quote_shell(self._pipeline_script)} {quote_shell(wsl_task_dir)}", f"{quote_shell(self._python)} {quote_shell(self._pipeline_script_for_profile(request.profile))} {quote_shell(wsl_task_dir)}",
f"--project-dir {quote_shell(wsl_project_dir)}", f"--project-dir {quote_shell(wsl_project_dir)}",
f"--template-root {quote_shell(wsl_template_root)}", f"--template-root {quote_shell(wsl_template_root)}",
f"--output-dir {quote_shell(wsl_output_dir)}", f"--output-dir {quote_shell(wsl_output_dir)}",
@@ -1486,6 +1503,9 @@ class PyintEngine(DinsarEngine):
@staticmethod @staticmethod
def _discover_archives(task_dir: str) -> Dict[str, List[str]]: def _discover_archives(task_dir: str) -> Dict[str, List[str]]:
from ..services.pyint_service import discover_lt1_archives from ..services.pyint_service import discover_lt1_archives, discover_s1_scene_sources, infer_task_identity
task_identity = infer_task_identity(task_dir)
if str(task_identity.get("satellite_family") or "").strip().upper() == "S1":
return discover_s1_scene_sources(task_dir)
return discover_lt1_archives(task_dir) return discover_lt1_archives(task_dir)
+8
View File
@@ -26,6 +26,12 @@ from .orm import (
ManagedRootORM, ManagedRootORM,
ScanCursorORM, ScanCursorORM,
PathInventoryORM, PathInventoryORM,
SourceProductAssetORM,
OrbitAssetORM,
SceneOrbitBindingORM,
OrbitAssetDerivativeORM,
AssetInventoryStateORM,
AssetInventoryIssueORM,
WorkflowDefORM, WorkflowDefORM,
WorkflowRunORM, WorkflowRunORM,
WorkflowStepORM, WorkflowStepORM,
@@ -94,6 +100,8 @@ __all__ = [
"TimeseriesStackPlanORM", "TimeseriesStackPlanItemORM", "TimeseriesStackPlanEdgeORM", "TimeseriesStackPlanORM", "TimeseriesStackPlanItemORM", "TimeseriesStackPlanEdgeORM",
"SystemTaskORM", "TaskLogORM", "SystemJobORM", "ScanStateORM", "SystemTaskORM", "TaskLogORM", "SystemJobORM", "ScanStateORM",
"ManagedRootORM", "ScanCursorORM", "PathInventoryORM", "ManagedRootORM", "ScanCursorORM", "PathInventoryORM",
"SourceProductAssetORM", "OrbitAssetORM", "SceneOrbitBindingORM",
"OrbitAssetDerivativeORM", "AssetInventoryStateORM", "AssetInventoryIssueORM",
"WorkflowDefORM", "WorkflowRunORM", "WorkflowStepORM", "WorkflowArtifactORM", "WorkflowDefORM", "WorkflowRunORM", "WorkflowStepORM", "WorkflowArtifactORM",
"SystemWorkerHeartbeatORM", "SystemWorkerHeartbeatORM",
"DinsarTaskBatchORM", "DinsarTaskItemORM", "DinsarTaskBatchORM", "DinsarTaskItemORM",
+244
View File
@@ -41,6 +41,32 @@ class RadarDataORM(Base):
product_unique_id = Column(String, nullable=True) product_unique_id = Column(String, nullable=True)
satellite_family = Column(String, index=True, nullable=True) satellite_family = Column(String, index=True, nullable=True)
look_direction = Column(String, index=True, nullable=True) look_direction = Column(String, index=True, nullable=True)
acquisition_start_time_utc = Column(DateTime, nullable=True, index=True)
acquisition_stop_time_utc = Column(DateTime, nullable=True)
absolute_orbit = Column(String, nullable=True, index=True)
relative_orbit = Column(String, nullable=True, index=True)
source_format = Column(String(32), nullable=True, index=True)
source_product_ref_id = Column(
Integer,
ForeignKey("source_product_assets.id", ondelete="SET NULL"),
nullable=True,
index=True,
)
source_archive_asset_id = Column(
Integer,
ForeignKey("source_product_assets.id", ondelete="SET NULL"),
nullable=True,
index=True,
)
selected_orbit_asset_id = Column(
Integer,
ForeignKey("orbit_assets.id", ondelete="SET NULL"),
nullable=True,
index=True,
)
orbit_binding_status = Column(String(32), nullable=False, default="UNBOUND", server_default="UNBOUND", index=True)
orbit_binding_reason = Column(Text, nullable=True)
metadata_json = Column(JSON, nullable=True)
geocoded_flag = Column(Boolean, nullable=True) geocoded_flag = Column(Boolean, nullable=True)
insar_source_ready = Column(Boolean, nullable=False, default=False, server_default="false") insar_source_ready = Column(Boolean, nullable=False, default=False, server_default="false")
insar_source_reason = Column(Text, nullable=True) insar_source_reason = Column(Text, nullable=True)
@@ -63,6 +89,10 @@ class RadarDataORM(Base):
preview_cache_updated_at = Column(DateTime, nullable=True) preview_cache_updated_at = Column(DateTime, nullable=True)
preview_cache_error = Column(Text, nullable=True) preview_cache_error = Column(Text, nullable=True)
source_product_asset = relationship("SourceProductAssetORM", foreign_keys=[source_product_ref_id])
source_archive_asset = relationship("SourceProductAssetORM", foreign_keys=[source_archive_asset_id])
selected_orbit_asset = relationship("OrbitAssetORM", foreign_keys=[selected_orbit_asset_id])
class DinsarResultORM(Base): class DinsarResultORM(Base):
__tablename__ = 'dinsar_results' __tablename__ = 'dinsar_results'
@@ -742,6 +772,11 @@ class ManagedRootORM(Base):
back_populates="root", back_populates="root",
cascade="all, delete-orphan", cascade="all, delete-orphan",
) )
asset_inventory_states = relationship(
"AssetInventoryStateORM",
back_populates="root",
cascade="all, delete-orphan",
)
__table_args__ = ( __table_args__ = (
Index("idx_managed_roots_role_enabled", "root_role", "enabled"), Index("idx_managed_roots_role_enabled", "root_role", "enabled"),
@@ -809,6 +844,215 @@ class PathInventoryORM(Base):
) )
class SourceProductAssetORM(Base):
__tablename__ = "source_product_assets"
id = Column(Integer, primary_key=True, autoincrement=True)
asset_uid = Column(String(128), unique=True, index=True, nullable=False)
logical_product_uid = Column(String(128), index=True, nullable=True)
satellite_family = Column(String(32), index=True, nullable=True)
satellite = Column(String(32), index=True, nullable=True)
source_format = Column(String(32), index=True, nullable=False)
product_type = Column(String(64), nullable=True)
product_level = Column(String(64), nullable=True)
imaging_mode = Column(String(64), nullable=True)
polarization = Column(String(64), nullable=True)
absolute_orbit = Column(String(64), index=True, nullable=True)
relative_orbit = Column(String(64), index=True, nullable=True)
orbit_direction = Column(String(32), index=True, nullable=True)
acquisition_start_time_utc = Column(DateTime, index=True, nullable=True)
acquisition_stop_time_utc = Column(DateTime, nullable=True)
imaging_date = Column(String(8), index=True, nullable=True)
root_ref_id = Column(Integer, ForeignKey("managed_roots.id", ondelete="SET NULL"), index=True, nullable=True)
root_path = Column(String, nullable=True)
file_path = Column(String, unique=True, index=True, nullable=False)
archive_path = Column(String, nullable=True)
path_kind = Column(String(24), nullable=True)
file_name = Column(String(255), nullable=True)
file_stem = Column(String(255), nullable=True)
file_ext = Column(String(32), nullable=True)
size_bytes = Column(BigInteger, nullable=True)
mtime_epoch = Column(Float, nullable=True)
checksum_sha256 = Column(String(64), nullable=True)
checksum_status = Column(String(32), nullable=False, default="NOT_COMPUTED", server_default="NOT_COMPUTED")
parser_name = Column(String(64), nullable=True)
parser_version = Column(String(32), nullable=True)
parse_status = Column(String(32), nullable=False, default="PENDING", server_default="PENDING", index=True)
parse_error = Column(Text, nullable=True)
parsed_at = Column(DateTime, nullable=True)
metadata_json = Column(JSON, nullable=True)
is_active = Column(Boolean, nullable=False, default=True, server_default="true", index=True)
missing_since = Column(DateTime, nullable=True)
created_at = Column(DateTime, server_default=func.now(), nullable=False)
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
root = relationship("ManagedRootORM")
__table_args__ = (
Index("idx_source_product_assets_family_date", "satellite_family", "imaging_date"),
Index("idx_source_product_assets_logical_product", "logical_product_uid"),
Index("idx_source_product_assets_root_active", "root_ref_id", "is_active"),
)
class OrbitAssetORM(Base):
__tablename__ = "orbit_assets"
id = Column(Integer, primary_key=True, autoincrement=True)
orbit_uid = Column(String(128), unique=True, index=True, nullable=False)
satellite_family = Column(String(32), index=True, nullable=True)
satellite = Column(String(32), index=True, nullable=True)
orbit_type = Column(String(64), index=True, nullable=False)
native_format = Column(String(32), index=True, nullable=False)
quality_class = Column(String(32), index=True, nullable=False, default="unknown", server_default="unknown")
root_ref_id = Column(Integer, ForeignKey("managed_roots.id", ondelete="SET NULL"), index=True, nullable=True)
root_path = Column(String, nullable=True)
file_path = Column(String, unique=True, index=True, nullable=False)
file_name = Column(String(255), nullable=True)
file_stem = Column(String(255), nullable=True)
file_ext = Column(String(32), nullable=True)
size_bytes = Column(BigInteger, nullable=True)
mtime_epoch = Column(Float, nullable=True)
checksum_sha256 = Column(String(64), nullable=True)
checksum_status = Column(String(32), nullable=False, default="NOT_COMPUTED", server_default="NOT_COMPUTED")
validity_start_time_utc = Column(DateTime, index=True, nullable=True)
validity_stop_time_utc = Column(DateTime, index=True, nullable=True)
generation_time_utc = Column(DateTime, nullable=True)
published_time_utc = Column(DateTime, nullable=True)
parser_name = Column(String(64), nullable=True)
parser_version = Column(String(32), nullable=True)
parse_status = Column(String(32), nullable=False, default="PENDING", server_default="PENDING", index=True)
parse_error = Column(Text, nullable=True)
parsed_at = Column(DateTime, nullable=True)
metadata_json = Column(JSON, nullable=True)
is_active = Column(Boolean, nullable=False, default=True, server_default="true", index=True)
missing_since = Column(DateTime, nullable=True)
created_at = Column(DateTime, server_default=func.now(), nullable=False)
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
root = relationship("ManagedRootORM")
__table_args__ = (
Index("idx_orbit_assets_family_sat_window", "satellite_family", "satellite", "validity_start_time_utc", "validity_stop_time_utc"),
Index("idx_orbit_assets_root_active", "root_ref_id", "is_active"),
)
class SceneOrbitBindingORM(Base):
__tablename__ = "scene_orbit_bindings"
id = Column(Integer, primary_key=True, autoincrement=True)
radar_data_id = Column(Integer, ForeignKey("radar_data.id", ondelete="CASCADE"), index=True, nullable=False)
orbit_asset_id = Column(Integer, ForeignKey("orbit_assets.id", ondelete="CASCADE"), index=True, nullable=False)
binding_role = Column(String(32), nullable=False, default="primary_orbit", server_default="primary_orbit")
match_status = Column(String(32), nullable=False, default="CANDIDATE", server_default="CANDIDATE", index=True)
selection_status = Column(String(32), nullable=False, default="CANDIDATE", server_default="CANDIDATE", index=True)
selection_rank = Column(Integer, nullable=True)
priority_score = Column(Float, nullable=True)
coverage_margin_before_seconds = Column(Float, nullable=True)
coverage_margin_after_seconds = Column(Float, nullable=True)
match_rule_version = Column(String(64), nullable=True)
match_reason = Column(Text, nullable=True)
selected_at = Column(DateTime, nullable=True)
metadata_json = Column(JSON, nullable=True)
created_at = Column(DateTime, server_default=func.now(), nullable=False)
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
radar_data = relationship("RadarDataORM")
orbit_asset = relationship("OrbitAssetORM")
__table_args__ = (
UniqueConstraint("radar_data_id", "orbit_asset_id", "binding_role", name="uq_scene_orbit_binding_role"),
Index("idx_scene_orbit_bindings_scene_selected", "radar_data_id", "selection_status"),
)
class OrbitAssetDerivativeORM(Base):
__tablename__ = "orbit_asset_derivatives"
id = Column(Integer, primary_key=True, autoincrement=True)
orbit_asset_id = Column(Integer, ForeignKey("orbit_assets.id", ondelete="CASCADE"), index=True, nullable=False)
engine_code = Column(String(32), index=True, nullable=False)
derivative_format = Column(String(32), nullable=False)
derivative_role = Column(String(64), nullable=True)
pool_path = Column(String, index=True, nullable=False)
size_bytes = Column(BigInteger, nullable=True)
mtime_epoch = Column(Float, nullable=True)
checksum_sha256 = Column(String(64), nullable=True)
generation_status = Column(String(32), nullable=False, default="PENDING", server_default="PENDING", index=True)
generation_error = Column(Text, nullable=True)
generated_at = Column(DateTime, nullable=True)
metadata_json = Column(JSON, nullable=True)
created_at = Column(DateTime, server_default=func.now(), nullable=False)
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
orbit_asset = relationship("OrbitAssetORM")
__table_args__ = (
UniqueConstraint("orbit_asset_id", "engine_code", "derivative_format", "pool_path", name="uq_orbit_asset_derivative_pool_path"),
Index("idx_orbit_asset_derivatives_asset_engine", "orbit_asset_id", "engine_code"),
)
class AssetInventoryStateORM(Base):
__tablename__ = "asset_inventory_states"
id = Column(Integer, primary_key=True, autoincrement=True)
root_ref_id = Column(Integer, ForeignKey("managed_roots.id", ondelete="CASCADE"), index=True, nullable=False)
inventory_type = Column(String(32), index=True, nullable=False)
root_path = Column(String, nullable=False)
scan_mode = Column(String(32), nullable=False, default="file_pool", server_default="file_pool")
status = Column(String(32), nullable=False, default="NEVER_SCANNED", server_default="NEVER_SCANNED", index=True)
last_scan_started_at = Column(DateTime, nullable=True)
last_scan_finished_at = Column(DateTime, nullable=True)
last_seen_entry_count = Column(Integer, nullable=True)
last_asset_count = Column(Integer, nullable=True)
last_issue_count = Column(Integer, nullable=True)
parser_version = Column(String(32), nullable=True)
needs_rescan = Column(Boolean, nullable=False, default=True, server_default="true", index=True)
last_error = Column(Text, nullable=True)
metadata_json = Column(JSON, nullable=True)
created_at = Column(DateTime, server_default=func.now(), nullable=False)
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
root = relationship("ManagedRootORM", back_populates="asset_inventory_states")
__table_args__ = (
UniqueConstraint("root_ref_id", "inventory_type", name="uq_asset_inventory_state_root_type"),
Index("idx_asset_inventory_states_type_status", "inventory_type", "status"),
)
class AssetInventoryIssueORM(Base):
__tablename__ = "asset_inventory_issues"
id = Column(Integer, primary_key=True, autoincrement=True)
root_ref_id = Column(Integer, ForeignKey("managed_roots.id", ondelete="SET NULL"), index=True, nullable=True)
inventory_type = Column(String(32), index=True, nullable=False)
asset_ref_id = Column(Integer, ForeignKey("source_product_assets.id", ondelete="SET NULL"), index=True, nullable=True)
radar_data_id = Column(Integer, ForeignKey("radar_data.id", ondelete="SET NULL"), index=True, nullable=True)
orbit_asset_id = Column(Integer, ForeignKey("orbit_assets.id", ondelete="SET NULL"), index=True, nullable=True)
severity = Column(String(16), nullable=False, default="warning", server_default="warning", index=True)
issue_code = Column(String(64), index=True, nullable=False)
issue_message = Column(Text, nullable=True)
source_path = Column(String, nullable=True)
status = Column(String(16), nullable=False, default="OPEN", server_default="OPEN", index=True)
first_seen_at = Column(DateTime, server_default=func.now(), nullable=False)
last_seen_at = Column(DateTime, server_default=func.now(), nullable=False)
resolved_at = Column(DateTime, nullable=True)
metadata_json = Column(JSON, nullable=True)
root = relationship("ManagedRootORM")
source_asset = relationship("SourceProductAssetORM", foreign_keys=[asset_ref_id])
radar_data = relationship("RadarDataORM")
orbit_asset = relationship("OrbitAssetORM")
__table_args__ = (
Index("idx_asset_inventory_issues_open", "status", "severity"),
Index("idx_asset_inventory_issues_root_type", "root_ref_id", "inventory_type"),
)
class WorkflowDefORM(Base): class WorkflowDefORM(Base):
"""Workflow definition (DAG template).""" """Workflow definition (DAG template)."""
__tablename__ = "workflow_defs" __tablename__ = "workflow_defs"
+11
View File
@@ -192,6 +192,17 @@ class RadarData(BaseModel):
product_unique_id: Optional[str] = None product_unique_id: Optional[str] = None
satellite_family: Optional[str] = None satellite_family: Optional[str] = None
look_direction: Optional[str] = None look_direction: Optional[str] = None
acquisition_start_time_utc: Optional[datetime] = None
acquisition_stop_time_utc: Optional[datetime] = None
absolute_orbit: Optional[str] = None
relative_orbit: Optional[str] = None
source_format: Optional[str] = None
source_product_ref_id: Optional[int] = None
source_archive_asset_id: Optional[int] = None
selected_orbit_asset_id: Optional[int] = None
orbit_binding_status: str = "UNBOUND"
orbit_binding_reason: Optional[str] = None
metadata_json: Optional[Dict[str, Any]] = None
geocoded_flag: Optional[bool] = None geocoded_flag: Optional[bool] = None
insar_source_ready: bool = False insar_source_ready: bool = False
insar_source_reason: Optional[str] = None insar_source_reason: Optional[str] = None
@@ -0,0 +1,651 @@
#!/usr/bin/env python3
from __future__ import annotations
import json
import math
import os
import re
import shutil
import sys
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List
try:
from .run_lt1_pyint_pipeline import (
DEFAULT_DERAMP_MODE,
DEFAULT_REFERENCE_MODE,
DEFAULT_REFLATTEN_AZIMUTH_STEP,
DEFAULT_REFLATTEN_COH_THRESHOLD,
DEFAULT_REFLATTEN_FALLBACK_COH_THRESHOLD,
DEFAULT_REFLATTEN_MODEL,
DEFAULT_REFLATTEN_RANGE_STEP,
assert_output_sanity,
assert_required_outputs,
calculate_dem_oversampling,
collect_expected_outputs,
collect_output_sanity_checks,
collect_stage_error_logs,
copy_native_outputs,
ensure_directory,
export_standard_products,
format_gamma_number,
hardlink_or_copy,
inspect_prepared_dem_path,
load_json_file,
load_pair_meta,
load_shell_environment,
normalize_date_text,
parse_args,
require_task_layout,
rerun_pair_product_stages,
run_gamma_reflatten,
run_logged,
safe_rmtree,
validate_unit_interval,
write_ifgram_list,
write_text,
write_wrapper_scripts,
)
except ImportError:
from run_lt1_pyint_pipeline import (
DEFAULT_DERAMP_MODE,
DEFAULT_REFERENCE_MODE,
DEFAULT_REFLATTEN_AZIMUTH_STEP,
DEFAULT_REFLATTEN_COH_THRESHOLD,
DEFAULT_REFLATTEN_FALLBACK_COH_THRESHOLD,
DEFAULT_REFLATTEN_MODEL,
DEFAULT_REFLATTEN_RANGE_STEP,
assert_output_sanity,
assert_required_outputs,
calculate_dem_oversampling,
collect_expected_outputs,
collect_output_sanity_checks,
collect_stage_error_logs,
copy_native_outputs,
ensure_directory,
export_standard_products,
format_gamma_number,
hardlink_or_copy,
inspect_prepared_dem_path,
load_json_file,
load_pair_meta,
load_shell_environment,
normalize_date_text,
parse_args,
require_task_layout,
rerun_pair_product_stages,
run_gamma_reflatten,
run_logged,
safe_rmtree,
validate_unit_interval,
write_ifgram_list,
write_text,
write_wrapper_scripts,
)
_WINDOWS_DRIVE_RE = re.compile(r"^[A-Za-z]:[\\/]")
def _normalize_runtime_path(value: Any) -> Path:
text = str(value or "").strip()
if not text:
return Path()
normalized = text.replace("\\", "/")
if normalized.startswith("/mnt/") or normalized.startswith("/"):
return Path(normalized).resolve()
if _WINDOWS_DRIVE_RE.match(text):
drive = text[0].lower()
tail = text[2:].replace("\\", "/").lstrip("/")
return Path(f"/mnt/{drive}/{tail}").resolve()
return Path(normalized).resolve()
def _build_s1_template_text(
*,
project_name: str,
satellite: str,
master_date: str,
range_looks: int,
azimuth_looks: int,
target_grid_size_m: int,
dem_lat_ovr: float,
dem_lon_ovr: float,
unwrap_coh_threshold: float,
geo_interp: str,
atmcor: bool,
atmcor_use_for_disp: bool,
reflatten: bool,
reflatten_model: str,
reflatten_coh_threshold: float,
parallel_workers: int,
unwrap: bool,
geocode: bool,
dem_mode: str,
fabdem_root: str,
prepared_dem_path: str,
opentopo_dem_type: str,
opentopo_api_key: str,
) -> str:
prepared_dem = inspect_prepared_dem_path(prepared_dem_path) if dem_mode == "prepared_file" else {}
lines = [
f"# Auto-generated for {project_name}",
f"satelite={satellite}",
f"masterDate={master_date}",
f"range_looks={int(range_looks)}",
f"azimuth_looks={int(azimuth_looks)}",
f"target_grid_size_m={int(target_grid_size_m or 0)}",
f"dem_lat_ovr={format_gamma_number(dem_lat_ovr)}",
f"dem_lon_ovr={format_gamma_number(dem_lon_ovr)}",
"download_data=0",
"raw2slc_all=1",
f"raw2slc_all_parallel={int(parallel_workers)}",
"extract_burst_all=1",
f"extract_all_parallel={int(parallel_workers)}",
"coreg_all=1",
f"coreg_all_parallel={int(parallel_workers)}",
"select_pairs=0",
"diff_all=1",
f"diff_all_parallel={int(parallel_workers)}",
"pot_all=0",
f"pot_all_parallel={int(parallel_workers)}",
f"unwrap_all={1 if unwrap else 0}",
f"unwrap_all_parallel={int(parallel_workers)}",
f"unwrapThreshold={format_gamma_number(unwrap_coh_threshold)}",
"make_mask=1",
"auto_unw=1",
"r_refer=-",
"a_refer=-",
f"atmcor_all={1 if atmcor else 0}",
f"atmcor_all_parallel={int(parallel_workers)}",
f"atmcor_use_for_disp={1 if (atmcor and atmcor_use_for_disp) else 0}",
f"reflatten={1 if reflatten else 0}",
f"reflatten_model={str(reflatten_model or DEFAULT_REFLATTEN_MODEL).strip().lower()}",
f"reflatten_coh_threshold={format_gamma_number(reflatten_coh_threshold)}",
f"geocode_all={1 if geocode else 0}",
f"geocode_all_parallel={int(parallel_workers)}",
f"geo_interp={str(geo_interp or '0').strip()}",
"gacos_correction=0",
"load_data=0",
"geocode_products=hyp3,licsbas",
"start_swath=1",
"end_swath=3",
"start_burst=1",
"end_burst=20",
]
if dem_mode == "local_fabdem" and fabdem_root:
lines.append(f"fabdem_dir={fabdem_root}")
else:
lines.append("fabdem_dir=-")
if dem_mode == "prepared_file" and prepared_dem.get("kind") == "gamma_ready":
lines.append(f"DEM={prepared_dem['direct_dem_path']}")
if dem_mode == "prepared_file" and prepared_dem.get("kind") == "source_dem":
lines.append(f"prepared_dem_source={prepared_dem['source_dem_path']}")
else:
lines.append("prepared_dem_source=-")
if dem_mode == "opentopo":
lines.append(f"opentopo_dem_type={opentopo_dem_type or 'SRTMGL1'}")
lines.append(f"opentopo_api_key={opentopo_api_key or '-'}")
else:
lines.append("opentopo_dem_type=-")
lines.append("opentopo_api_key=-")
return "\n".join(lines) + "\n"
def _resolve_s1_inputs(input_assets_payload: Dict[str, Any]) -> Dict[str, Any]:
task_source = input_assets_payload.get("task_source") or {}
prod = task_source.get("production_inputs") or {}
orbits = input_assets_payload.get("orbits") or {}
master_scene = _normalize_runtime_path(
((prod.get("master_scene") or {}).get("staged_path") or (prod.get("master_scene") or {}).get("path") or (prod.get("master_zip") or {}).get("staged_path") or (prod.get("master_zip") or {}).get("path") or "")
)
slave_scene = _normalize_runtime_path(
((prod.get("slave_scene") or {}).get("staged_path") or (prod.get("slave_scene") or {}).get("path") or (prod.get("slave_zip") or {}).get("staged_path") or (prod.get("slave_zip") or {}).get("path") or "")
)
master_eof = _normalize_runtime_path(
((orbits.get("master") or {}).get("staged_path") or (orbits.get("master") or {}).get("path") or "")
)
slave_eof = _normalize_runtime_path(
((orbits.get("slave") or {}).get("staged_path") or (orbits.get("slave") or {}).get("path") or "")
)
return {
"master_scene": master_scene,
"slave_scene": slave_scene,
"master_eof": master_eof,
"slave_eof": slave_eof,
}
def main() -> int:
args = parse_args()
task_dir = Path(args.task_dir).resolve()
project_dir = Path(args.project_dir).resolve()
run_root = project_dir.parent
template_root = Path(args.template_root).resolve()
output_dir = Path(args.output_dir).resolve()
pyint_home = Path(args.pyint_home).resolve()
pyint_app_script = Path(args.pyint_app_script).resolve()
dem_root = Path(args.dem_root).resolve()
input_assets_dir = Path(args.input_assets_dir).resolve() if args.input_assets_dir else None
input_assets_json = Path(args.input_assets_json).resolve() if args.input_assets_json else None
input_assets_payload = load_json_file(input_assets_json)
dem_mode = str(args.dem_mode or "local_fabdem").strip().lower() or "local_fabdem"
prepared_dem_info = inspect_prepared_dem_path(args.prepared_dem_path) if dem_mode == "prepared_file" else {}
dem_oversampling = calculate_dem_oversampling(
dem_resolution_m=float(args.dem_resolution_m),
target_grid_size_m=float(args.target_grid_size_m or 0),
dem_lat_ovr=float(args.dem_lat_ovr or 0.0),
dem_lon_ovr=float(args.dem_lon_ovr or 0.0),
)
unwrap_coh_threshold = validate_unit_interval(args.unwrap_coh_threshold, "--unwrap-coh-threshold")
coherence_mask_threshold = validate_unit_interval(args.coherence_mask_threshold, "--coherence-mask-threshold")
reference_mode = str(args.reference_mode or DEFAULT_REFERENCE_MODE).strip().lower() or DEFAULT_REFERENCE_MODE
deramp_mode = str(args.deramp_mode or DEFAULT_DERAMP_MODE).strip().lower() or DEFAULT_DERAMP_MODE
reference_coh_threshold = validate_unit_interval(args.reference_coh_threshold, "--reference-coh-threshold")
deramp_coh_threshold = validate_unit_interval(args.deramp_coh_threshold, "--deramp-coh-threshold")
geo_interp = str(args.geo_interp or "1").strip()
if geo_interp not in {"0", "1"}:
raise ValueError("--geo-interp must be 0 or 1.")
gamma_nodata_value = float(args.gamma_nodata_value)
if not math.isfinite(gamma_nodata_value):
raise ValueError("--gamma-nodata-value must be a finite number.")
reflatten_model = str(args.reflatten_model or DEFAULT_REFLATTEN_MODEL).strip().lower()
if reflatten_model == "linear":
reflatten_model = "plane"
if reflatten_model not in {"plane", "quadratic"}:
raise ValueError("--reflatten-model must be plane or quadratic.")
reflatten_coh_threshold = validate_unit_interval(args.reflatten_coh_threshold, "--reflatten-coh-threshold")
reflatten_fallback_coh_threshold = validate_unit_interval(
args.reflatten_fallback_coh_threshold,
"--reflatten-fallback-coh-threshold",
)
reflatten_range_step = max(1, int(args.reflatten_range_step or DEFAULT_REFLATTEN_RANGE_STEP))
reflatten_azimuth_step = max(1, int(args.reflatten_azimuth_step or DEFAULT_REFLATTEN_AZIMUTH_STEP))
require_task_layout(task_dir)
if not pyint_app_script.is_file():
raise FileNotFoundError(f"pyintApp.py not found: {pyint_app_script}")
if not input_assets_json or not input_assets_json.is_file():
raise RuntimeError("Sentinel-1 PyINT runner requires --input-assets-json.")
if dem_mode == "prepared_file" and not prepared_dem_info.get("kind"):
raise RuntimeError(
"Prepared DEM mode requires either a Gamma DEM with .par, "
"or a source DEM with .xml/.hdr/.vrt sidecars."
)
if args.force:
safe_rmtree(run_root)
safe_rmtree(template_root)
safe_rmtree(output_dir)
if run_root.exists():
raise RuntimeError(f"PyINT run root already exists, rerun with --force: {run_root}")
pair_meta = load_pair_meta(task_dir)
s1_inputs = _resolve_s1_inputs(input_assets_payload)
master_scene = s1_inputs["master_scene"]
slave_scene = s1_inputs["slave_scene"]
master_eof = s1_inputs["master_eof"]
slave_eof = s1_inputs["slave_eof"]
if not (master_scene.is_file() or master_scene.is_dir()):
raise FileNotFoundError(f"Sentinel-1 master scene not found: {master_scene}")
if not (slave_scene.is_file() or slave_scene.is_dir()):
raise FileNotFoundError(f"Sentinel-1 slave scene not found: {slave_scene}")
if not master_eof.is_file():
raise FileNotFoundError(f"Sentinel-1 master EOF not found: {master_eof}")
if not slave_eof.is_file():
raise FileNotFoundError(f"Sentinel-1 slave EOF not found: {slave_eof}")
master_date = normalize_date_text(args.master_date) or normalize_date_text(pair_meta.get("master_imaging_date"))
slave_date = normalize_date_text(args.slave_date) or normalize_date_text(pair_meta.get("slave_imaging_date"))
if not master_date or not slave_date:
raise RuntimeError("Unable to determine master/slave dates from pair metadata.")
pair_name = f"{master_date}-{slave_date}"
task_alias = str(args.task_alias or pair_meta.get("task_alias") or task_dir.name).strip() or task_dir.name
pair_key = str(args.pair_key or pair_meta.get("pair_key") or "").strip()
time_baseline_days = int(args.time_baseline_days or pair_meta.get("time_baseline_days") or 0)
satellite = str(pair_meta.get("master_satellite") or "S1A").strip().upper() or "S1A"
ensure_directory(run_root)
ensure_directory(template_root)
ensure_directory(output_dir)
ensure_directory(dem_root)
pyint_scripts_dir = pyint_home / "pyint"
wrappers_dir = ensure_directory(run_root / "wrappers")
write_wrapper_scripts(
wrappers_dir=wrappers_dir,
pyint_home=pyint_home,
python_cmd=args.python,
gamma_env_script=args.gamma_env_script,
)
for shim_name in ("python", "python3"):
shim_path = wrappers_dir / shim_name
write_text(
shim_path,
"\n".join(
[
"#!/usr/bin/env bash",
f"exec '{args.python}' \"$@\"",
"",
]
),
)
shim_path.chmod(shim_path.stat().st_mode | 0o111)
template_path = write_text(
template_root / f"{args.project_name}.template",
_build_s1_template_text(
project_name=args.project_name,
satellite=satellite,
master_date=master_date,
range_looks=args.range_looks,
azimuth_looks=args.azimuth_looks,
target_grid_size_m=int(args.target_grid_size_m or 0),
dem_lat_ovr=dem_oversampling["dem_lat_ovr"],
dem_lon_ovr=dem_oversampling["dem_lon_ovr"],
unwrap_coh_threshold=unwrap_coh_threshold,
geo_interp=geo_interp,
atmcor=bool(args.atmcor),
atmcor_use_for_disp=bool(args.atmcor_use_for_disp),
reflatten=bool(args.reflatten),
reflatten_model=reflatten_model,
reflatten_coh_threshold=reflatten_coh_threshold,
parallel_workers=args.parallel_workers,
unwrap=bool(args.unwrap),
geocode=bool(args.geocode),
dem_mode=dem_mode,
fabdem_root=str(args.fabdem_root or "").strip(),
prepared_dem_path=str(args.prepared_dem_path or "").strip(),
opentopo_dem_type=str(args.opentopo_dem_type or "SRTMGL1").strip(),
opentopo_api_key=str(args.opentopo_api_key or "").strip(),
),
)
scratch_root = ensure_directory(project_dir.parent)
archive_materialization: List[Dict[str, str]] = []
env = os.environ.copy()
if args.gamma_env_script:
env.update(load_shell_environment(args.gamma_env_script, env))
opod_dir = project_dir / "OPOD"
env.update(
{
"SCRATCHDIR": str(scratch_root),
"TEMPLATEDIR": str(template_root),
"DEMDIR": str(dem_root),
"OPOD_DIR": str(opod_dir),
"PATH": f"{wrappers_dir}:{pyint_scripts_dir}:{env.get('PATH', '')}",
"PYTHONPATH": f"{pyint_home}:{env.get('PYTHONPATH', '')}",
"PYINT_LT1_PRECISE_ORBIT_ENABLED": "false",
}
)
generate_stdout = run_root / "pyint_generate.stdout.log"
generate_stderr = run_root / "pyint_generate.stderr.log"
generate_result = run_logged(
[str(wrappers_dir / "pyintApp.py"), "-g", args.project_name],
env=env,
cwd=scratch_root,
stdout_path=generate_stdout,
stderr_path=generate_stderr,
)
if generate_result.returncode != 0:
raise RuntimeError(
f"pyintApp.py -g failed with rc={generate_result.returncode}: "
f"{(generate_result.stderr or generate_result.stdout or '').strip()}"
)
pyint_project_dir = project_dir
ensure_directory(pyint_project_dir)
opod_dir = ensure_directory(opod_dir)
download_dir = ensure_directory(pyint_project_dir / "DOWNLOAD")
ifgram_list_path = write_ifgram_list(pyint_project_dir / "ifgram_list.txt", master_date, slave_date, time_baseline_days)
for role, src_path in (("master", master_scene), ("slave", slave_scene)):
target_path = download_dir / src_path.name
if src_path.is_dir():
if target_path.exists():
if target_path.is_dir():
safe_rmtree(target_path)
else:
target_path.unlink()
shutil.copytree(src_path, target_path)
op = "copied_dir"
else:
op = hardlink_or_copy(src_path, target_path)
archive_materialization.append(
{
"role": role,
"source": str(src_path),
"target": str(target_path),
"operation": op,
}
)
for role, src_path in (("master_orbit", master_eof), ("slave_orbit", slave_eof)):
target_path = opod_dir / src_path.name
op = hardlink_or_copy(src_path, target_path)
archive_materialization.append(
{
"role": role,
"source": str(src_path),
"target": str(target_path),
"operation": op,
}
)
run_stdout = run_root / "pyint.stdout.log"
run_stderr = run_root / "pyint.stderr.log"
run_started_at = datetime.utcnow().isoformat(timespec="seconds") + "Z"
run_result = run_logged(
[str(wrappers_dir / "pyintApp.py"), args.project_name],
env=env,
cwd=scratch_root,
stdout_path=run_stdout,
stderr_path=run_stderr,
)
if run_result.returncode != 0:
stage_error_logs = collect_stage_error_logs(pyint_project_dir)
detail_text = (run_result.stderr or run_result.stdout or "").strip()
if stage_error_logs:
log_text = ", ".join(f"{name}={path}" for name, path in stage_error_logs.items())
detail_text = f"{detail_text}\nStage logs: {log_text}" if detail_text else f"Stage logs: {log_text}"
raise RuntimeError(f"pyintApp.py failed with rc={run_result.returncode}: {detail_text}")
repair_summary: Dict[str, Any] = {
"attempted": False,
"attempt_count": 0,
"max_attempts": 1,
}
outputs = collect_expected_outputs(pyint_project_dir, pair_name, args.range_looks)
try:
assert_required_outputs(outputs, unwrap=bool(args.unwrap), geocode=bool(args.geocode))
output_sanity_checks = collect_output_sanity_checks(outputs, unwrap=bool(args.unwrap), geocode=bool(args.geocode))
assert_output_sanity(output_sanity_checks)
except RuntimeError as exc:
repair_summary = rerun_pair_product_stages(
project_name=args.project_name,
project_dir=pyint_project_dir,
run_root=run_root,
scratch_root=scratch_root,
env=env,
pair_name=pair_name,
master_date=master_date,
slave_date=slave_date,
range_looks=args.range_looks,
unwrap=bool(args.unwrap),
atmcor=bool(args.atmcor),
geocode=bool(args.geocode),
)
outputs = collect_expected_outputs(pyint_project_dir, pair_name, args.range_looks)
assert_required_outputs(outputs, unwrap=bool(args.unwrap), geocode=bool(args.geocode))
output_sanity_checks = collect_output_sanity_checks(outputs, unwrap=bool(args.unwrap), geocode=bool(args.geocode))
assert_output_sanity(output_sanity_checks)
stage_error_logs = collect_stage_error_logs(pyint_project_dir)
reflatten_summary: Dict[str, Any] = {
"enabled": bool(args.reflatten),
"applied": False,
"reason": "",
"model": reflatten_model,
"coherence_threshold": reflatten_coh_threshold,
"fallback_coherence_threshold": reflatten_fallback_coh_threshold,
"range_step": reflatten_range_step,
"azimuth_step": reflatten_azimuth_step,
}
if bool(args.reflatten) and bool(args.unwrap):
reflatten_summary = run_gamma_reflatten(
project_dir=pyint_project_dir,
run_root=run_root,
output_dir=output_dir,
outputs=outputs,
pair_name=pair_name,
master_date=master_date,
range_looks=args.range_looks,
env=env,
model=reflatten_model,
coherence_threshold=reflatten_coh_threshold,
fallback_coherence_threshold=reflatten_fallback_coh_threshold,
range_step=reflatten_range_step,
azimuth_step=reflatten_azimuth_step,
geo_interp=geo_interp,
)
outputs = collect_expected_outputs(pyint_project_dir, pair_name, args.range_looks)
output_sanity_checks = collect_output_sanity_checks(outputs, unwrap=bool(args.unwrap), geocode=bool(args.geocode))
assert_output_sanity(output_sanity_checks)
elif bool(args.reflatten):
reflatten_summary["reason"] = "unwrap disabled"
else:
reflatten_summary["reason"] = "disabled"
copied_paths = copy_native_outputs(
project_dir=pyint_project_dir,
output_dir=output_dir,
pair_name=pair_name,
template_path=template_path,
ifgram_list_path=ifgram_list_path,
stdout_path=run_stdout,
stderr_path=run_stderr,
)
standard_products = (
export_standard_products(
project_dir=pyint_project_dir,
output_dir=output_dir,
pair_name=pair_name,
master_date=master_date,
range_looks=args.range_looks,
azimuth_looks=args.azimuth_looks,
target_grid_size_m=int(args.target_grid_size_m or 0),
coherence_mask_threshold=coherence_mask_threshold,
reference_mode=reference_mode,
reference_coh_threshold=reference_coh_threshold,
deramp_mode=deramp_mode,
deramp_coh_threshold=deramp_coh_threshold,
atmcor_enabled=bool(args.atmcor),
atmcor_use_for_disp=bool(args.atmcor_use_for_disp),
reflatten_summary=reflatten_summary,
gamma_nodata_value=gamma_nodata_value,
outputs=outputs,
env=env,
run_root=run_root,
)
if bool(args.geocode)
else {"enabled": False, "reason": "geocode disabled"}
)
summary = {
"ok": True,
"task_dir": str(task_dir),
"task_alias": task_alias,
"pair_key": pair_key,
"project_name": args.project_name,
"project_dir": str(pyint_project_dir),
"run_root": str(run_root),
"template_root": str(template_root),
"output_dir": str(output_dir),
"pyint_home": str(pyint_home),
"pyint_app_script": str(pyint_app_script),
"gamma_env_script": args.gamma_env_script,
"dem": {
"mode": dem_mode,
"dem_root": str(dem_root),
"fabdem_root": str(args.fabdem_root or "").strip(),
"prepared_dem_path": str(args.prepared_dem_path or "").strip(),
"prepared_dem_kind": str(prepared_dem_info.get("kind") or ""),
"configured_resolution_m": float(args.dem_resolution_m),
"oversampling": dem_oversampling,
"opentopo_dem_type": str(args.opentopo_dem_type or "SRTMGL1").strip(),
"opentopo_api_key_configured": bool(str(args.opentopo_api_key or "").strip()),
},
"orbit_policy": "require_eof",
"precise_orbit_bridge": {"enabled": False, "mode": "not_applicable"},
"input_assets_dir": str(input_assets_dir) if input_assets_dir else "",
"input_assets_json": str(input_assets_json) if input_assets_json else "",
"input_assets": input_assets_payload,
"master_date": master_date,
"slave_date": slave_date,
"pair_name": pair_name,
"time_baseline_days": time_baseline_days,
"target_grid_size_m": int(args.target_grid_size_m or 0),
"range_looks": int(args.range_looks),
"azimuth_looks": int(args.azimuth_looks),
"dem_resolution_m": float(args.dem_resolution_m),
"dem_oversampling": dem_oversampling,
"unwrap_coh_threshold": unwrap_coh_threshold,
"coherence_quality_threshold": coherence_mask_threshold,
"reference_mode": reference_mode,
"reference_coh_threshold": reference_coh_threshold,
"deramp_mode": deramp_mode,
"deramp_coh_threshold": deramp_coh_threshold,
"gamma_nodata_value": gamma_nodata_value,
"geo_interp": geo_interp,
"atmcor": bool(args.atmcor),
"atmcor_use_for_disp": bool(args.atmcor_use_for_disp),
"reflatten": bool(args.reflatten),
"reflatten_model": reflatten_model,
"reflatten_coh_threshold": reflatten_coh_threshold,
"reflatten_fallback_coh_threshold": reflatten_fallback_coh_threshold,
"reflatten_range_step": reflatten_range_step,
"reflatten_azimuth_step": reflatten_azimuth_step,
"parallel_workers": int(args.parallel_workers),
"unwrap": bool(args.unwrap),
"geocode": bool(args.geocode),
"archives": {
"master": [str(master_scene)],
"slave": [str(slave_scene)],
},
"orbit_files": {
"master": str(master_eof),
"slave": str(slave_eof),
},
"archive_materialization": archive_materialization,
"workspace_outputs": outputs,
"output_sanity_checks": output_sanity_checks,
"output_repair": repair_summary,
"reflatten_summary": reflatten_summary,
"copied_outputs": copied_paths,
"standard_products": standard_products,
"logs": {
"generate_stdout": str(generate_stdout),
"generate_stderr": str(generate_stderr),
"run_stdout": str(run_stdout),
"run_stderr": str(run_stderr),
"stage_error_logs": stage_error_logs,
},
"started_at": run_started_at,
"finished_at": datetime.utcnow().isoformat(timespec="seconds") + "Z",
}
summary_path = output_dir / "pyint_run_summary.json"
write_text(summary_path, json.dumps(summary, ensure_ascii=True, indent=2) + "\n")
print(json.dumps(summary, ensure_ascii=True, indent=2))
return 0
if __name__ == "__main__":
try:
raise SystemExit(main())
except Exception as exc:
print(str(exc), file=sys.stderr)
raise
+2
View File
@@ -5,6 +5,7 @@ from fastapi import APIRouter
from . import ( from . import (
ai, ai,
aoi, aoi,
assets,
auth, auth,
dinsar, dinsar,
dinsar_products, dinsar_products,
@@ -43,6 +44,7 @@ def include_all_routers(router: APIRouter) -> None:
router.include_router(unpack.router) router.include_router(unpack.router)
router.include_router(monitor.router) router.include_router(monitor.router)
router.include_router(orbit.router) router.include_router(orbit.router)
router.include_router(assets.router)
router.include_router(root_registry.router) router.include_router(root_registry.router)
router.include_router(radar.router) router.include_router(radar.router)
router.include_router(aoi.router) router.include_router(aoi.router)
+222
View File
@@ -0,0 +1,222 @@
from __future__ import annotations
from typing import Any, Dict, List, Optional
from fastapi import APIRouter, Depends, HTTPException, Query
from pydantic import BaseModel, Field
from sqlalchemy.ext.asyncio import AsyncSession
from ..database import get_db
from ..models import AuthUserORM
from ..models.orm import SourceProductAssetORM
from ..services.asset_inventory_service import asset_inventory_service
from ..services.job_queue_service import job_queue_service
from ..services.task_service import task_service
from .dependencies import _get_current_user, _require_admin
router = APIRouter(prefix="/assets", tags=["assets"])
class AssetScanRequest(BaseModel):
inventory_types: List[str] = Field(default_factory=list)
root_ids: List[int] = Field(default_factory=list)
bind_orbits: bool = True
class S1UnpackRequest(BaseModel):
target_root: Optional[str] = None
overwrite: bool = False
min_disk_space_gb: Optional[float] = Field(default=None, ge=0)
delete_archive: Optional[bool] = None
class S1BatchUnpackRequest(BaseModel):
target_root: Optional[str] = None
overwrite: bool = False
min_disk_space_gb: Optional[float] = Field(default=None, ge=0)
delete_archive: Optional[bool] = None
scan_before_unpack: bool = True
@router.get("/inventory/status")
async def get_asset_inventory_status(
current_user: AuthUserORM = Depends(_get_current_user),
db: AsyncSession = Depends(get_db),
):
_ = current_user
return await asset_inventory_service.get_status(db)
@router.post("/inventory/scan", status_code=202)
async def run_asset_inventory_scan(
request: Optional[AssetScanRequest] = None,
admin_user: AuthUserORM = Depends(_require_admin),
):
_ = admin_user
payload = (request or AssetScanRequest()).model_dump()
try:
task_id = await task_service.create_task(
"SCAN_ASSET_INVENTORY",
"Source/orbit asset inventory scan",
params=payload,
)
job_id = await job_queue_service.create_job(
"SCAN_ASSET_INVENTORY",
payload=payload,
task_id=task_id,
)
return {"message": "Asset inventory scan queued", "task_id": task_id, "job_id": job_id}
except ValueError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
@router.post("/inventory/scan-now")
async def run_asset_inventory_scan_now(
request: Optional[AssetScanRequest] = None,
admin_user: AuthUserORM = Depends(_require_admin),
db: AsyncSession = Depends(get_db),
):
_ = admin_user
payload = (request or AssetScanRequest()).model_dump()
return await asset_inventory_service.scan_configured_roots(
db,
inventory_types=payload.get("inventory_types") or None,
root_ids=payload.get("root_ids") or None,
bind_orbits=bool(payload.get("bind_orbits", True)),
)
@router.get("/sources")
async def list_source_product_assets(
satellite_family: Optional[str] = None,
satellite: Optional[str] = None,
source_format: Optional[str] = None,
parse_status: Optional[str] = None,
include_inactive: bool = False,
limit: int = Query(200, ge=1, le=1000),
offset: int = Query(0, ge=0),
current_user: AuthUserORM = Depends(_get_current_user),
db: AsyncSession = Depends(get_db),
):
_ = current_user
return await asset_inventory_service.list_source_products(
db,
satellite_family=satellite_family,
satellite=satellite,
source_format=source_format,
parse_status=parse_status,
include_inactive=include_inactive,
limit=limit,
offset=offset,
)
@router.get("/orbits")
async def list_orbit_assets(
satellite_family: Optional[str] = None,
satellite: Optional[str] = None,
orbit_type: Optional[str] = None,
parse_status: Optional[str] = None,
include_inactive: bool = False,
limit: int = Query(200, ge=1, le=1000),
offset: int = Query(0, ge=0),
current_user: AuthUserORM = Depends(_get_current_user),
db: AsyncSession = Depends(get_db),
):
_ = current_user
return await asset_inventory_service.list_orbits(
db,
satellite_family=satellite_family,
satellite=satellite,
orbit_type=orbit_type,
parse_status=parse_status,
include_inactive=include_inactive,
limit=limit,
offset=offset,
)
@router.get("/issues")
async def list_asset_inventory_issues(
status: str = "OPEN",
severity: Optional[str] = None,
issue_code: Optional[str] = None,
limit: int = Query(200, ge=1, le=1000),
offset: int = Query(0, ge=0),
current_user: AuthUserORM = Depends(_get_current_user),
db: AsyncSession = Depends(get_db),
):
_ = current_user
return await asset_inventory_service.list_issues(
db,
status=status,
severity=severity,
issue_code=issue_code,
limit=limit,
offset=offset,
)
@router.post("/sources/{asset_id}/unpack-sentinel1")
async def unpack_sentinel1_source_asset(
asset_id: int,
request: Optional[S1UnpackRequest] = None,
admin_user: AuthUserORM = Depends(_require_admin),
db: AsyncSession = Depends(get_db),
):
_ = admin_user
asset = await db.get(SourceProductAssetORM, asset_id)
if asset is None:
raise HTTPException(status_code=404, detail="Source product asset not found.")
if asset.source_format != "S1_ZIP":
raise HTTPException(status_code=400, detail="Only Sentinel-1 ZIP assets can be unpacked by this endpoint.")
request_data = request or S1UnpackRequest()
payload = {
"asset_id": asset_id,
"target_root": request_data.target_root,
"overwrite": bool(request_data.overwrite),
}
if request_data.min_disk_space_gb is not None:
payload["min_disk_space_gb"] = request_data.min_disk_space_gb
if request_data.delete_archive is not None:
payload["delete_archive"] = request_data.delete_archive
try:
task_id = await task_service.create_task(
"UNPACK_SENTINEL1",
"Sentinel-1 unpack",
params=payload,
)
job_id = await job_queue_service.create_job(
"UNPACK_SENTINEL1",
payload=payload,
task_id=task_id,
)
except ValueError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
await db.commit()
return {"message": "Sentinel-1 unpack task queued", "task_id": task_id, "job_id": job_id}
@router.post("/inventory/unpack-sentinel1", status_code=202)
async def run_sentinel1_unpack_batch(
request: Optional[S1BatchUnpackRequest] = None,
admin_user: AuthUserORM = Depends(_require_admin),
):
_ = admin_user
payload = (request or S1BatchUnpackRequest()).model_dump()
try:
task_id = await task_service.create_task(
"UNPACK_SENTINEL1",
"Sentinel-1 batch unpack",
params=payload,
)
job_id = await job_queue_service.create_job(
"UNPACK_SENTINEL1",
payload=payload,
task_id=task_id,
)
return {"message": "Sentinel-1 batch unpack task queued", "task_id": task_id, "job_id": job_id}
except ValueError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
+6
View File
@@ -99,9 +99,15 @@ def _normalize_list_pagination(limit: int, offset: int) -> tuple[int, int]:
class DinsarBatchSceneCreate(BaseModel): class DinsarBatchSceneCreate(BaseModel):
file_path: str = Field(min_length=1) file_path: str = Field(min_length=1)
satellite: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH) satellite: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH)
satellite_family: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH)
imaging_date: Optional[str] = Field(default=None, max_length=32) imaging_date: Optional[str] = Field(default=None, max_length=32)
imaging_mode: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH) imaging_mode: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH)
polarization: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH) polarization: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH)
orbit_direction: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH)
relative_orbit: Optional[str] = Field(default=None, max_length=64)
absolute_orbit: Optional[str] = Field(default=None, max_length=64)
has_orbit_data: Optional[bool] = None
orbit_file_path: Optional[str] = Field(default=None, max_length=4096)
model_config = ConfigDict(extra="ignore") model_config = ConfigDict(extra="ignore")
+4
View File
@@ -14,6 +14,10 @@ MONITOR_CONFIG = {
"radar_dirs": split_env_paths(settings.MONITOR_RADAR_DIRS), "radar_dirs": split_env_paths(settings.MONITOR_RADAR_DIRS),
"orbit_dir": settings.MONITOR_ORBIT_DIR, "orbit_dir": settings.MONITOR_ORBIT_DIR,
"dinsar_dirs": split_env_paths(settings.MONITOR_DINSAR_DIRS), "dinsar_dirs": split_env_paths(settings.MONITOR_DINSAR_DIRS),
# Sentinel-1 链路
"s1_source_dirs": split_env_paths(settings.SOURCE_PRODUCT_DIRS),
"s1_storage_dirs": split_env_paths(settings.SENTINEL1_STORAGE_DIRS),
"s1_orbit_dirs": split_env_paths(settings.ORBIT_SOURCE_DIRS),
# GF3 链路 # GF3 链路
"gf3_source_dirs": split_env_paths(settings.GF3_SOURCE_DIRS), "gf3_source_dirs": split_env_paths(settings.GF3_SOURCE_DIRS),
"gf3_storage_dirs": split_env_paths(settings.GF3_STORAGE_DIRS), "gf3_storage_dirs": split_env_paths(settings.GF3_STORAGE_DIRS),
File diff suppressed because it is too large Load Diff
+12 -1
View File
@@ -238,7 +238,7 @@ def _chunked(items: List[Any], size: int):
_RADAR_PREVIEW_EXTENSIONS = (".jpg", ".jpeg", ".png", ".webp", ".bmp", ".tif", ".tiff") _RADAR_PREVIEW_EXTENSIONS = (".jpg", ".jpeg", ".png", ".webp", ".bmp", ".tif", ".tiff")
_RADAR_PREVIEW_KEYWORDS = ("quicklook", "preview", "browse", "thumbnail", "thumb", "overview") _RADAR_PREVIEW_KEYWORDS = ("quicklook", "quick-look", "preview", "browse", "thumbnail", "thumb", "overview")
_RADAR_CACHE_NAME_RE = re.compile(r"[^A-Za-z0-9._-]+") _RADAR_CACHE_NAME_RE = re.compile(r"[^A-Za-z0-9._-]+")
@@ -396,7 +396,18 @@ class DataService:
continue continue
path = os.path.join(root, name) path = os.path.join(root, name)
root_lower = root.lower()
keyword_score = 0 if any(key in lower_name for key in _RADAR_PREVIEW_KEYWORDS) else 1 keyword_score = 0 if any(key in lower_name for key in _RADAR_PREVIEW_KEYWORDS) else 1
if lower_name == "quick-look.png":
keyword_score = -3
elif lower_name == "quicklook.png":
keyword_score = -2
elif lower_name.startswith("quick-look.") or lower_name.startswith("quicklook."):
keyword_score = min(keyword_score, -1)
if f"{os.sep}preview" in root_lower:
keyword_score -= 1
if f"{os.sep}icons" in root_lower:
keyword_score += 2
ext_score = 0 if lower_name.endswith((".jpg", ".jpeg")) else 1 ext_score = 0 if lower_name.endswith((".jpg", ".jpeg")) else 1
try: try:
size_score = -os.path.getsize(path) size_score = -os.path.getsize(path)
+12 -3
View File
@@ -7,6 +7,8 @@ import re
from datetime import datetime from datetime import datetime
from typing import Any, Dict, Optional from typing import Any, Dict, Optional
from ..utils import normalize_satellite_family
PAIR_META_FILENAME = ".dinsar_pair.json" PAIR_META_FILENAME = ".dinsar_pair.json"
RUN_META_FILENAME = ".dinsar_run.json" RUN_META_FILENAME = ".dinsar_run.json"
@@ -48,6 +50,7 @@ def build_pair_key(
slave_path: Any, slave_path: Any,
master_date: Any = None, master_date: Any = None,
slave_date: Any = None, slave_date: Any = None,
satellite_family: Any = None,
) -> str: ) -> str:
master_date_text = _normalize_date(master_date) master_date_text = _normalize_date(master_date)
slave_date_text = _normalize_date(slave_date) slave_date_text = _normalize_date(slave_date)
@@ -60,17 +63,23 @@ def build_pair_key(
] ]
) )
digest = hashlib.sha1(payload.encode("utf-8", errors="ignore")).hexdigest()[:10] digest = hashlib.sha1(payload.encode("utf-8", errors="ignore")).hexdigest()[:10]
return f"lt1_{master_date_text}_{slave_date_text}_{digest}" family = str(normalize_satellite_family(satellite_family) or "").strip().lower()
if not family:
family = "pair"
return f"{family}_{master_date_text}_{slave_date_text}_{digest}"
def build_fallback_pair_key(task_alias: Any, source_hint: Any = None) -> str: def build_fallback_pair_key(task_alias: Any, source_hint: Any = None, satellite_family: Any = None) -> str:
alias = str(task_alias or "").strip() or "Task_unknown_unknown" alias = str(task_alias or "").strip() or "Task_unknown_unknown"
parts = alias.split("_") parts = alias.split("_")
master_date = parts[1] if len(parts) > 2 else "unknown" master_date = parts[1] if len(parts) > 2 else "unknown"
slave_date = parts[2] if len(parts) > 2 else "unknown" slave_date = parts[2] if len(parts) > 2 else "unknown"
payload = "||".join([alias, normalize_path(source_hint)]) payload = "||".join([alias, normalize_path(source_hint)])
digest = hashlib.sha1(payload.encode("utf-8", errors="ignore")).hexdigest()[:10] digest = hashlib.sha1(payload.encode("utf-8", errors="ignore")).hexdigest()[:10]
return f"lt1_{_normalize_date(master_date)}_{_normalize_date(slave_date)}_{digest}" family = str(normalize_satellite_family(satellite_family) or "").strip().lower()
if not family:
family = "pair"
return f"{family}_{_normalize_date(master_date)}_{_normalize_date(slave_date)}_{digest}"
def build_run_key( def build_run_key(
+9 -1
View File
@@ -21,6 +21,7 @@ from typing import Any, Dict, List, Optional
from ..config import get_env_text, settings from ..config import get_env_text, settings
from ..process_utils import is_any_process_running from ..process_utils import is_any_process_running
from ..utils import normalize_satellite_family
from .dinsar_naming import ( from .dinsar_naming import (
PAIR_META_FILENAME, PAIR_META_FILENAME,
build_fallback_pair_key, build_fallback_pair_key,
@@ -480,7 +481,14 @@ def _utc_now_text() -> str:
def _resolve_dinsar_pair_identity(task_dir: str, task_name: str) -> tuple[str, str, Dict[str, Any]]: def _resolve_dinsar_pair_identity(task_dir: str, task_name: str) -> tuple[str, str, Dict[str, Any]]:
pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {} pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {}
task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(task_alias, task_dir) satellite_family = normalize_satellite_family(
pair_meta.get("master_satellite") or pair_meta.get("slave_satellite")
)
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(
task_alias,
task_dir,
satellite_family=satellite_family,
)
return task_alias, pair_key, pair_meta return task_alias, pair_key, pair_meta
+272 -1
View File
@@ -1,7 +1,7 @@
import os import os
import asyncio import asyncio
from datetime import datetime, timedelta from datetime import datetime, timedelta
from typing import Any, Dict, Optional from typing import Any, Dict, List, Optional
import httpx import httpx
from sqlalchemy import and_, func, literal, or_, select, text from sqlalchemy import and_, func, literal, or_, select, text
@@ -11,9 +11,14 @@ from ..config import read_int_env, settings, split_env_paths
from ..db_maintenance import inspect_database_structure from ..db_maintenance import inspect_database_structure
from ..models import ( from ..models import (
AiDiagnosisORM, AiDiagnosisORM,
AssetInventoryIssueORM,
AssetInventoryStateORM,
DinsarResultORM, DinsarResultORM,
OrbitAssetORM,
ResultCatalogStateORM, ResultCatalogStateORM,
ResultProductORM, ResultProductORM,
SceneOrbitBindingORM,
SourceProductAssetORM,
SystemWorkerHeartbeatORM, SystemWorkerHeartbeatORM,
) )
from ..idl_service import get_idl_status from ..idl_service import get_idl_status
@@ -399,6 +404,51 @@ def _sanitize_pairing_system_status(payload: Dict[str, Any]) -> Dict[str, Any]:
} }
def _sanitize_asset_inventory_status(payload: Dict[str, Any]) -> Dict[str, Any]:
source_roots = payload.get("source_roots", {}) or {}
orbit_roots = payload.get("orbit_roots", {}) or {}
source_assets = payload.get("source_assets", {}) or {}
orbit_assets = payload.get("orbit_assets", {}) or {}
bindings = payload.get("bindings", {}) or {}
issues = payload.get("issues", {}) or {}
return {
"ok": bool(payload.get("ok")),
"source_roots": {
"configured_count": int(source_roots.get("configured_count") or 0),
"accessible_count": int(source_roots.get("accessible_count") or 0),
"needs_rescan_count": int(source_roots.get("needs_rescan_count") or 0),
},
"orbit_roots": {
"configured_count": int(orbit_roots.get("configured_count") or 0),
"accessible_count": int(orbit_roots.get("accessible_count") or 0),
"needs_rescan_count": int(orbit_roots.get("needs_rescan_count") or 0),
},
"source_assets": {
"total_count": int(source_assets.get("total_count") or 0),
"lt1_count": int(source_assets.get("lt1_count") or 0),
"s1_count": int(source_assets.get("s1_count") or 0),
"parse_failed_count": int(source_assets.get("parse_failed_count") or 0),
},
"orbit_assets": {
"total_count": int(orbit_assets.get("total_count") or 0),
"lt1_count": int(orbit_assets.get("lt1_count") or 0),
"s1_count": int(orbit_assets.get("s1_count") or 0),
"parse_failed_count": int(orbit_assets.get("parse_failed_count") or 0),
},
"bindings": {
"scene_count": int(bindings.get("scene_count") or 0),
"matched_count": int(bindings.get("matched_count") or 0),
"missing_count": int(bindings.get("missing_count") or 0),
"ambiguous_count": int(bindings.get("ambiguous_count") or 0),
},
"issues": {
"open_count": int(issues.get("open_count") or 0),
"error_count": int(issues.get("error_count") or 0),
"warning_count": int(issues.get("warning_count") or 0),
},
}
def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]: def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
database = payload.get("database", {}) or {} database = payload.get("database", {}) or {}
worker = payload.get("worker", {}) or {} worker = payload.get("worker", {}) or {}
@@ -411,6 +461,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
dinsar_bridge = payload.get("dinsar_bridge", {}) or {} dinsar_bridge = payload.get("dinsar_bridge", {}) or {}
source_roots = payload.get("source_roots", {}) or {} source_roots = payload.get("source_roots", {}) or {}
product_packages = payload.get("product_packages", {}) or {} product_packages = payload.get("product_packages", {}) or {}
asset_inventory = payload.get("asset_inventory", {}) or {}
wsl_runtime = payload.get("wsl_runtime", {}) or {} wsl_runtime = payload.get("wsl_runtime", {}) or {}
pairing_system = payload.get("pairing_system", {}) or {} pairing_system = payload.get("pairing_system", {}) or {}
idl = payload.get("idl", {}) or {} idl = payload.get("idl", {}) or {}
@@ -423,6 +474,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
sanitized_dinsar_bridge = _sanitize_bridge_status(dinsar_bridge) sanitized_dinsar_bridge = _sanitize_bridge_status(dinsar_bridge)
sanitized_source_roots = _sanitize_source_roots_status(source_roots) sanitized_source_roots = _sanitize_source_roots_status(source_roots)
sanitized_product_packages = _sanitize_product_package_status(product_packages) sanitized_product_packages = _sanitize_product_package_status(product_packages)
sanitized_asset_inventory = _sanitize_asset_inventory_status(asset_inventory)
sanitized_wsl_runtime = _sanitize_wsl_runtime_status(wsl_runtime) sanitized_wsl_runtime = _sanitize_wsl_runtime_status(wsl_runtime)
sanitized_pairing_system = _sanitize_pairing_system_status(pairing_system) sanitized_pairing_system = _sanitize_pairing_system_status(pairing_system)
@@ -451,6 +503,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
"dinsar_bridge": sanitized_dinsar_bridge, "dinsar_bridge": sanitized_dinsar_bridge,
"source_roots": sanitized_source_roots, "source_roots": sanitized_source_roots,
"product_packages": sanitized_product_packages, "product_packages": sanitized_product_packages,
"asset_inventory": sanitized_asset_inventory,
"wsl_runtime": sanitized_wsl_runtime, "wsl_runtime": sanitized_wsl_runtime,
"pairing_system": sanitized_pairing_system, "pairing_system": sanitized_pairing_system,
"idl": { "idl": {
@@ -905,6 +958,221 @@ async def _check_product_packages() -> Dict[str, Any]:
return status return status
async def _check_asset_inventory() -> Dict[str, Any]:
status: Dict[str, Any] = {
"ok": False,
"source_roots": {
"configured_count": 0,
"accessible_count": 0,
"inaccessible_count": 0,
"needs_rescan_count": 0,
"items": [],
},
"orbit_roots": {
"configured_count": 0,
"accessible_count": 0,
"inaccessible_count": 0,
"needs_rescan_count": 0,
"items": [],
},
"source_assets": {
"total_count": 0,
"lt1_count": 0,
"s1_count": 0,
"parse_failed_count": 0,
"by_family": {},
},
"orbit_assets": {
"total_count": 0,
"lt1_count": 0,
"s1_count": 0,
"parse_failed_count": 0,
"by_family": {},
},
"bindings": {
"scene_count": 0,
"matched_count": 0,
"missing_count": 0,
"ambiguous_count": 0,
},
"issues": {
"open_count": 0,
"error_count": 0,
"warning_count": 0,
"by_code": {},
},
"error": None,
}
source_paths: List[str] = []
for value in (
settings.SOURCE_PRODUCT_DIRS,
settings.INSAR_STORAGE_DIRS,
settings.MONITOR_RADAR_DIRS,
):
for path in split_env_paths(value):
if path not in source_paths:
source_paths.append(path)
orbit_paths: List[str] = []
for value in (
settings.ORBIT_SOURCE_DIRS,
settings.MONITOR_ORBIT_DIR,
):
for path in split_env_paths(value):
if path not in orbit_paths:
orbit_paths.append(path)
for path in source_paths:
item = _probe_directory_status(path)
item["role"] = "source_product_pool"
status["source_roots"]["items"].append(item)
for path in orbit_paths:
item = _probe_directory_status(path)
item["role"] = "orbit_asset_pool"
status["orbit_roots"]["items"].append(item)
for key in ("source_roots", "orbit_roots"):
root_status = status[key]
root_status["configured_count"] = len(root_status["items"])
root_status["accessible_count"] = sum(1 for item in root_status["items"] if item.get("accessible"))
root_status["inaccessible_count"] = root_status["configured_count"] - root_status["accessible_count"]
try:
session_factory = _get_session_factory()
async with session_factory() as db:
state_rows = (
await db.execute(
select(
AssetInventoryStateORM.inventory_type,
func.count(AssetInventoryStateORM.id),
)
.where(AssetInventoryStateORM.needs_rescan == True) # noqa: E712
.group_by(AssetInventoryStateORM.inventory_type)
)
).all()
for inventory_type, count in state_rows:
key = "orbit_roots" if str(inventory_type or "").lower().startswith("orbit") else "source_roots"
status[key]["needs_rescan_count"] += int(count or 0)
source_family_rows = (
await db.execute(
select(
SourceProductAssetORM.satellite_family,
func.count(SourceProductAssetORM.id),
)
.where(
SourceProductAssetORM.is_active == True, # noqa: E712
SourceProductAssetORM.source_format != "S1_ZIP",
)
.group_by(SourceProductAssetORM.satellite_family)
)
).all()
for family, count in source_family_rows:
family_key = str(family or "unknown").strip().upper() or "unknown"
value = int(count or 0)
status["source_assets"]["by_family"][family_key] = value
status["source_assets"]["total_count"] += value
status["source_assets"]["lt1_count"] = int(status["source_assets"]["by_family"].get("LT1", 0))
status["source_assets"]["s1_count"] = int(status["source_assets"]["by_family"].get("S1", 0))
source_parse_failed = await db.execute(
select(func.count(SourceProductAssetORM.id)).where(
SourceProductAssetORM.parse_status == "FAILED",
SourceProductAssetORM.source_format != "S1_ZIP",
)
)
status["source_assets"]["parse_failed_count"] = int(source_parse_failed.scalar_one() or 0)
orbit_family_rows = (
await db.execute(
select(
OrbitAssetORM.satellite_family,
func.count(OrbitAssetORM.id),
)
.where(OrbitAssetORM.is_active == True) # noqa: E712
.group_by(OrbitAssetORM.satellite_family)
)
).all()
for family, count in orbit_family_rows:
family_key = str(family or "unknown").strip().upper() or "unknown"
value = int(count or 0)
status["orbit_assets"]["by_family"][family_key] = value
status["orbit_assets"]["total_count"] += value
status["orbit_assets"]["lt1_count"] = int(status["orbit_assets"]["by_family"].get("LT1", 0))
status["orbit_assets"]["s1_count"] = int(status["orbit_assets"]["by_family"].get("S1", 0))
orbit_parse_failed = await db.execute(
select(func.count(OrbitAssetORM.id)).where(OrbitAssetORM.parse_status == "FAILED")
)
status["orbit_assets"]["parse_failed_count"] = int(orbit_parse_failed.scalar_one() or 0)
scene_count = await db.execute(select(func.count(SceneOrbitBindingORM.radar_data_id.distinct())))
status["bindings"]["scene_count"] = int(scene_count.scalar_one() or 0)
selected_count = await db.execute(
select(func.count(SceneOrbitBindingORM.id)).where(SceneOrbitBindingORM.selection_status == "SELECTED")
)
status["bindings"]["matched_count"] = int(selected_count.scalar_one() or 0)
missing_count = await db.execute(
select(func.count(AssetInventoryIssueORM.id)).where(
AssetInventoryIssueORM.status == "OPEN",
AssetInventoryIssueORM.issue_code == "scene_missing_orbit",
)
)
status["bindings"]["missing_count"] = int(missing_count.scalar_one() or 0)
ambiguous_count = await db.execute(
select(func.count(AssetInventoryIssueORM.id)).where(
AssetInventoryIssueORM.status == "OPEN",
AssetInventoryIssueORM.issue_code == "scene_ambiguous_orbit",
)
)
status["bindings"]["ambiguous_count"] = int(ambiguous_count.scalar_one() or 0)
issue_rows = (
await db.execute(
select(
AssetInventoryIssueORM.severity,
func.count(AssetInventoryIssueORM.id),
)
.where(AssetInventoryIssueORM.status == "OPEN")
.group_by(AssetInventoryIssueORM.severity)
)
).all()
for severity, count in issue_rows:
severity_key = str(severity or "warning").strip().lower() or "warning"
value = int(count or 0)
status["issues"]["open_count"] += value
if severity_key == "error":
status["issues"]["error_count"] += value
elif severity_key == "warning":
status["issues"]["warning_count"] += value
issue_code_rows = (
await db.execute(
select(
AssetInventoryIssueORM.issue_code,
func.count(AssetInventoryIssueORM.id),
)
.where(AssetInventoryIssueORM.status == "OPEN")
.group_by(AssetInventoryIssueORM.issue_code)
)
).all()
status["issues"]["by_code"] = {
str(code or "unknown"): int(count or 0)
for code, count in issue_code_rows
}
status["ok"] = (
status["source_roots"]["inaccessible_count"] == 0
and status["orbit_roots"]["inaccessible_count"] == 0
and status["source_assets"]["parse_failed_count"] == 0
and status["orbit_assets"]["parse_failed_count"] == 0
and status["issues"]["error_count"] == 0
)
except Exception as exc:
status["error"] = str(exc)
return status
async def _check_wsl_runtime() -> Dict[str, Any]: async def _check_wsl_runtime() -> Dict[str, Any]:
status = { status = {
"ok": False, "ok": False,
@@ -989,6 +1257,7 @@ async def get_health_status(
dinsar_bridge_status = await _check_dinsar_bridge() dinsar_bridge_status = await _check_dinsar_bridge()
source_roots_status = await _check_source_roots() source_roots_status = await _check_source_roots()
product_packages_status = await _check_product_packages() product_packages_status = await _check_product_packages()
asset_inventory_status = await _check_asset_inventory()
wsl_runtime_status = await _check_wsl_runtime() wsl_runtime_status = await _check_wsl_runtime()
pairing_system_status = await pairing_state_service.get_pairing_system_status() pairing_system_status = await pairing_state_service.get_pairing_system_status()
engines_status = {"ok": None, "overall": None, "engines": []} engines_status = {"ok": None, "overall": None, "engines": []}
@@ -1005,6 +1274,7 @@ async def get_health_status(
dinsar_bridge_status.get("ok"), dinsar_bridge_status.get("ok"),
source_roots_status.get("ok"), source_roots_status.get("ok"),
product_packages_status.get("ok"), product_packages_status.get("ok"),
asset_inventory_status.get("ok"),
wsl_runtime_status.get("ok"), wsl_runtime_status.get("ok"),
pairing_system_status.get("ok"), pairing_system_status.get("ok"),
(not settings.TIMESERIES_ENABLED) or timeseries_result_catalog_status.get("ok"), (not settings.TIMESERIES_ENABLED) or timeseries_result_catalog_status.get("ok"),
@@ -1028,6 +1298,7 @@ async def get_health_status(
"dinsar_bridge": dinsar_bridge_status, "dinsar_bridge": dinsar_bridge_status,
"source_roots": source_roots_status, "source_roots": source_roots_status,
"product_packages": product_packages_status, "product_packages": product_packages_status,
"asset_inventory": asset_inventory_status,
"wsl_runtime": wsl_runtime_status, "wsl_runtime": wsl_runtime_status,
"pairing_system": pairing_system_status, "pairing_system": pairing_system_status,
"idl": { "idl": {
+43
View File
@@ -22,6 +22,7 @@ from ..config import settings
from ..models import SystemJobORM, DinsarResultORM, HazardPointORM, DinsarTaskItemORM, PsTaskItemORM, RadarDataORM, SARSceneGeoORM, FloodDetectionORM, WaterDetectionORM, GF3ProcessingORM, AiDiagnosisORM from ..models import SystemJobORM, DinsarResultORM, HazardPointORM, DinsarTaskItemORM, PsTaskItemORM, RadarDataORM, SARSceneGeoORM, FloodDetectionORM, WaterDetectionORM, GF3ProcessingORM, AiDiagnosisORM
from ..scheduler import scan_data_job from ..scheduler import scan_data_job
from .data_service import data_service from .data_service import data_service
from .asset_inventory_service import asset_inventory_service
from .dinsar_compat_service import dinsar_compat_service from .dinsar_compat_service import dinsar_compat_service
from .dinsar_naming import build_run_key from .dinsar_naming import build_run_key
from .dinsar_production_service import dinsar_production_service from .dinsar_production_service import dinsar_production_service
@@ -67,6 +68,7 @@ JOB_TYPE_SCAN_DATA = "SCAN_DATA"
JOB_TYPE_SCAN_DINSAR = "SCAN_DINSAR" JOB_TYPE_SCAN_DINSAR = "SCAN_DINSAR"
JOB_TYPE_COPY_DATA = "COPY_DATA" JOB_TYPE_COPY_DATA = "COPY_DATA"
JOB_TYPE_UNPACK = "UNPACK_ARCHIVES" JOB_TYPE_UNPACK = "UNPACK_ARCHIVES"
JOB_TYPE_UNPACK_SENTINEL1 = "UNPACK_SENTINEL1"
JOB_TYPE_AI_TRAIN = "AI_TRAIN" JOB_TYPE_AI_TRAIN = "AI_TRAIN"
JOB_TYPE_AI_PREDICT = "AI_PREDICT" JOB_TYPE_AI_PREDICT = "AI_PREDICT"
JOB_TYPE_AI_ANALYZE = "AI_ANALYZE" JOB_TYPE_AI_ANALYZE = "AI_ANALYZE"
@@ -85,6 +87,7 @@ JOB_TYPE_PYINT_RUN = "PYINT_RUN"
JOB_TYPE_PUBLISH_DINSAR_PRODUCTS = "PUBLISH_DINSAR_PRODUCTS" JOB_TYPE_PUBLISH_DINSAR_PRODUCTS = "PUBLISH_DINSAR_PRODUCTS"
JOB_TYPE_REBUILD_DINSAR_CATALOG = "REBUILD_DINSAR_CATALOG" JOB_TYPE_REBUILD_DINSAR_CATALOG = "REBUILD_DINSAR_CATALOG"
JOB_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG" JOB_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG"
JOB_TYPE_SCAN_ASSET_INVENTORY = "SCAN_ASSET_INVENTORY"
COPY_ALLOWED_STATUSES = {"PENDING", "IN_PROGRESS", "COMPLETED", "FAILED"} COPY_ALLOWED_STATUSES = {"PENDING", "IN_PROGRESS", "COMPLETED", "FAILED"}
@@ -225,6 +228,34 @@ async def _handle_scan_data(job: SystemJobORM) -> None:
await _run_scan_data_custom(job.task_id, payload) await _run_scan_data_custom(job.task_id, payload)
async def _handle_scan_asset_inventory(job: SystemJobORM) -> None:
if not job.task_id:
raise ValueError("SCAN_ASSET_INVENTORY requires task_id for progress tracking.")
payload = job.payload or {}
await task_service.start_task(job.task_id, message="Source/orbit asset inventory scan started")
async with AsyncSessionLocal() as db:
result = await asset_inventory_service.scan_configured_roots(
db,
inventory_types=payload.get("inventory_types") or None,
root_ids=payload.get("root_ids") or None,
bind_orbits=bool(payload.get("bind_orbits", True)),
task_id=job.task_id,
)
await task_service.update_task(
job.task_id,
status="COMPLETED",
progress=100,
message=(
"Asset inventory scan completed: "
f"sources={result.get('source_assets', 0)}, "
f"orbits={result.get('orbit_assets', 0)}, "
f"matched={((result.get('binding') or {}).get('matched_count', 0))}, "
f"missing={((result.get('binding') or {}).get('missing_count', 0))}"
),
db=db,
)
def _resolve_hazard_shp_path() -> str: def _resolve_hazard_shp_path() -> str:
base_dir = settings.HAZARD_POINTS_DIR base_dir = settings.HAZARD_POINTS_DIR
filename = settings.HAZARD_POINTS_FILENAME filename = settings.HAZARD_POINTS_FILENAME
@@ -418,6 +449,16 @@ async def _handle_unpack_archives(job: SystemJobORM) -> None:
await run_unpack_task(job.task_id) await run_unpack_task(job.task_id)
async def _handle_unpack_sentinel1(job: SystemJobORM) -> None:
if not job.task_id:
raise ValueError("UNPACK_SENTINEL1 requires task_id for progress tracking.")
payload = job.payload or {}
if payload.get("asset_id"):
await asset_inventory_service.run_sentinel1_unpack_task(job.task_id, payload)
return
await asset_inventory_service.run_sentinel1_unpack_batch_task(job.task_id, payload)
async def _handle_ai_train(job: SystemJobORM) -> None: async def _handle_ai_train(job: SystemJobORM) -> None:
if not job.task_id: if not job.task_id:
raise ValueError("AI_TRAIN requires task_id for progress tracking.") raise ValueError("AI_TRAIN requires task_id for progress tracking.")
@@ -3849,6 +3890,7 @@ async def _handle_rebuild_psinsar_catalog(job: SystemJobORM) -> None:
_HANDLERS = { _HANDLERS = {
JOB_TYPE_SCAN_DATA: _handle_scan_data, JOB_TYPE_SCAN_DATA: _handle_scan_data,
JOB_TYPE_SCAN_ASSET_INVENTORY: _handle_scan_asset_inventory,
JOB_TYPE_SCAN_DINSAR: _handle_scan_dinsar, JOB_TYPE_SCAN_DINSAR: _handle_scan_dinsar,
JOB_TYPE_PUBLISH_DINSAR_PRODUCTS: _handle_publish_dinsar_products_clean, JOB_TYPE_PUBLISH_DINSAR_PRODUCTS: _handle_publish_dinsar_products_clean,
JOB_TYPE_REBUILD_DINSAR_CATALOG: _handle_rebuild_dinsar_catalog_clean, JOB_TYPE_REBUILD_DINSAR_CATALOG: _handle_rebuild_dinsar_catalog_clean,
@@ -3864,6 +3906,7 @@ _HANDLERS = {
JOB_TYPE_REBUILD_PSINSAR_CATALOG: _handle_rebuild_psinsar_catalog, JOB_TYPE_REBUILD_PSINSAR_CATALOG: _handle_rebuild_psinsar_catalog,
JOB_TYPE_COPY_DATA: _handle_copy_data, JOB_TYPE_COPY_DATA: _handle_copy_data,
JOB_TYPE_UNPACK: _handle_unpack_archives, JOB_TYPE_UNPACK: _handle_unpack_archives,
JOB_TYPE_UNPACK_SENTINEL1: _handle_unpack_sentinel1,
JOB_TYPE_AI_TRAIN: _handle_ai_train, JOB_TYPE_AI_TRAIN: _handle_ai_train,
JOB_TYPE_AI_PREDICT: _handle_ai_predict, JOB_TYPE_AI_PREDICT: _handle_ai_predict,
JOB_TYPE_AI_ANALYZE: _handle_ai_analyze, JOB_TYPE_AI_ANALYZE: _handle_ai_analyze,
@@ -36,7 +36,7 @@ def _satellite_family_expr(alias: str) -> str:
f"COALESCE(NULLIF({alias}.satellite_family, ''), " f"COALESCE(NULLIF({alias}.satellite_family, ''), "
f"CASE " f"CASE "
f"WHEN {compact} IN ('LT1', 'LT1A', 'LT1B', 'LUTAN1', 'LUTAN1A', 'LUTAN1B') THEN 'LT1' " f"WHEN {compact} IN ('LT1', 'LT1A', 'LT1B', 'LUTAN1', 'LUTAN1A', 'LUTAN1B') THEN 'LT1' "
f"WHEN {compact} IN ('S1', 'S1A', 'S1B', 'SENTINEL1', 'SENTINEL1A', 'SENTINEL1B') THEN 'S1' " f"WHEN {compact} IN ('S1', 'S1A', 'S1B', 'S1C', 'SENTINEL1', 'SENTINEL1A', 'SENTINEL1B', 'SENTINEL1C') THEN 'S1' "
f"WHEN NULLIF({alias}.satellite, '') IS NOT NULL THEN upper({alias}.satellite) " f"WHEN NULLIF({alias}.satellite, '') IS NOT NULL THEN upper({alias}.satellite) "
f"ELSE NULL END)" f"ELSE NULL END)"
) )
+616 -103
View File
@@ -10,11 +10,14 @@ from typing import Any, Dict, List
from ..config import settings from ..config import settings
from .orbit_converter import get_source_orbit_inventory from .orbit_converter import get_source_orbit_inventory
from .pyint_service import ( from .pyint_service import (
discover_s1_scene_sources,
discover_lt1_archives, discover_lt1_archives,
infer_scene_date_from_archives, infer_scene_date_from_archives,
infer_task_identity, infer_task_identity,
validate_pyint_root_dir, validate_pyint_root_dir,
) )
from .asset_inventory_service import _configured_sentinel1_archive_dirs, _parse_s1_source_name
from ..utils import normalize_satellite_family
VALID_DEM_MODES = {"local_fabdem", "opentopo", "prepared_file"} VALID_DEM_MODES = {"local_fabdem", "opentopo", "prepared_file"}
@@ -62,6 +65,198 @@ def _infer_satellite_from_archives(paths: List[str]) -> str:
return "" return ""
def _normalize_s1_satellite(value: Any) -> str:
text = str(value or "").strip().upper().replace("-", "").replace("_", "")
if text in {"S1A", "S1B", "S1C"}:
return text
return ""
def _find_s1_zip_by_logical_uid(logical_uid: str) -> str:
logical = str(logical_uid or "").strip()
if not logical:
return ""
expected_name = logical if logical.lower().endswith(".zip") else f"{logical}.zip"
for root in _configured_sentinel1_archive_dirs():
if not root or not os.path.isdir(root):
continue
direct_candidate = os.path.join(root, expected_name)
if os.path.isfile(direct_candidate):
return _normalize_path(direct_candidate)
for current_root, _, files in os.walk(root):
if expected_name in files:
return _normalize_path(os.path.join(current_root, expected_name))
return ""
def _is_s1_safe_dir(path: str) -> bool:
normalized = _normalize_path(path)
if not normalized or not os.path.isdir(normalized):
return False
return os.path.isfile(os.path.join(normalized, "manifest.safe"))
def _resolve_s1_scene_input(
*,
role: str,
scene_path: str,
pair_meta: Dict[str, Any],
) -> Dict[str, Any]:
normalized_scene = _normalize_path(scene_path)
scene_name = os.path.basename(normalized_scene.rstrip("\\/"))
parsed = _parse_s1_source_name(scene_name) or {}
logical_uid = str(parsed.get("logical_product_uid") or "").strip()
resolved_path = ""
input_kind = ""
resolution_method = ""
if normalized_scene.lower().endswith(".zip") and os.path.isfile(normalized_scene):
resolved_path = normalized_scene
input_kind = "zip"
resolution_method = "task_or_pair_meta"
elif _is_s1_safe_dir(normalized_scene):
resolved_path = normalized_scene
input_kind = "safe_dir"
resolution_method = "task_or_pair_meta"
elif normalized_scene.lower().endswith(".safe") and os.path.isdir(normalized_scene):
resolved_path = normalized_scene
input_kind = "safe_dir"
resolution_method = "task_or_pair_meta"
else:
sibling_zip = ""
if normalized_scene.lower().endswith(".safe"):
sibling_zip = normalized_scene[:-5] + ".zip"
if sibling_zip and os.path.isfile(sibling_zip):
resolved_path = _normalize_path(sibling_zip)
input_kind = "zip"
resolution_method = "safe_sibling_zip"
elif logical_uid:
looked_up_zip = _find_s1_zip_by_logical_uid(logical_uid)
if looked_up_zip:
resolved_path = looked_up_zip
input_kind = "zip"
resolution_method = "source_pool_lookup"
expected_name = scene_name
if input_kind == "zip":
if not expected_name.lower().endswith(".zip"):
expected_name = f"{logical_uid}.zip" if logical_uid else os.path.basename(resolved_path)
elif input_kind == "safe_dir":
if not expected_name.lower().endswith(".safe"):
expected_name = f"{logical_uid}.SAFE" if logical_uid else os.path.basename(resolved_path)
satellite = _normalize_s1_satellite(pair_meta.get(f"{role}_satellite")) or _normalize_s1_satellite(parsed.get("satellite"))
date_text = str(pair_meta.get(f"{role}_imaging_date") or parsed.get("imaging_date") or "").strip()
return {
"role": role,
"scene_path": normalized_scene,
"scene_name": scene_name,
"logical_product_uid": logical_uid,
"satellite": satellite,
"date": date_text,
"resolved": bool(resolved_path),
"path": resolved_path,
"input_kind": input_kind,
"resolution_method": resolution_method,
"expected_name": expected_name,
"staged_path": "",
}
def _resolve_s1_zip_file(
*,
role: str,
scene_path: str,
pair_meta: Dict[str, Any],
) -> Dict[str, Any]:
normalized_scene = _normalize_path(scene_path)
scene_name = os.path.basename(normalized_scene)
parsed = _parse_s1_source_name(scene_name) or {}
logical_uid = str(parsed.get("logical_product_uid") or "").strip()
direct_zip = ""
if normalized_scene.lower().endswith(".zip") and os.path.isfile(normalized_scene):
direct_zip = normalized_scene
elif normalized_scene.lower().endswith(".safe"):
sibling_zip = normalized_scene[:-5] + ".zip"
if os.path.isfile(sibling_zip):
direct_zip = _normalize_path(sibling_zip)
if not direct_zip and logical_uid:
direct_zip = _find_s1_zip_by_logical_uid(logical_uid)
satellite = _normalize_s1_satellite(pair_meta.get(f"{role}_satellite")) or _normalize_s1_satellite(parsed.get("satellite"))
date_text = str(pair_meta.get(f"{role}_imaging_date") or parsed.get("imaging_date") or "").strip()
return {
"role": role,
"scene_path": normalized_scene,
"scene_name": scene_name,
"logical_product_uid": logical_uid,
"satellite": satellite,
"date": date_text,
"resolved": bool(direct_zip),
"path": direct_zip,
"resolution_method": "scene_or_source_pool_lookup" if direct_zip else "",
"expected_name": f"{logical_uid}.zip" if logical_uid else "",
"staged_path": "",
}
def _resolve_s1_orbit_file(
*,
role: str,
pair_meta: Dict[str, Any],
task_dir: str,
) -> Dict[str, Any]:
direct_path = _normalize_path(pair_meta.get(f"{role}_orbit_file_path"))
if direct_path and os.path.isfile(direct_path):
return {
"role": role,
"resolved": True,
"path": direct_path,
"resolution_method": "pair_meta",
"expected_name": os.path.basename(direct_path),
"satellite": _normalize_s1_satellite(pair_meta.get(f"{role}_satellite")),
"date": str(pair_meta.get(f"{role}_imaging_date") or "").strip(),
"staged_path": "",
}
role_orbit_dir = os.path.join(task_dir, "orbit")
satellite = _normalize_s1_satellite(pair_meta.get(f"{role}_satellite"))
date_text = str(pair_meta.get(f"{role}_imaging_date") or "").strip()
candidates: List[str] = []
if os.path.isdir(role_orbit_dir):
for entry in os.scandir(role_orbit_dir):
if not entry.is_file():
continue
if not entry.name.lower().endswith(".eof"):
continue
if satellite and satellite not in entry.name.upper():
continue
candidates.append(_normalize_path(entry.path))
candidates.sort()
if candidates:
return {
"role": role,
"resolved": True,
"path": candidates[0],
"resolution_method": "task_orbit_dir",
"expected_name": os.path.basename(candidates[0]),
"satellite": satellite,
"date": date_text,
"staged_path": "",
}
return {
"role": role,
"resolved": False,
"path": "",
"resolution_method": "",
"expected_name": "",
"satellite": satellite,
"date": date_text,
"staged_path": "",
"error": f"{role} Sentinel-1 EOF 缺失",
}
def _get_dem_mode() -> str: def _get_dem_mode() -> str:
raw_mode = str(getattr(settings, "PYINT_DEM_MODE", "local_fabdem") or "local_fabdem").strip().lower() raw_mode = str(getattr(settings, "PYINT_DEM_MODE", "local_fabdem") or "local_fabdem").strip().lower()
if raw_mode not in VALID_DEM_MODES: if raw_mode not in VALID_DEM_MODES:
@@ -569,6 +764,427 @@ def resolve_pyint_task_input_assets(
} }
def build_pyint_input_preview(root_dir: str, num_to_process: int = 0) -> Dict[str, Any]:
validation = validate_pyint_root_dir(root_dir, num_to_process)
dem_summary = get_pyint_dem_summary()
orbit_context = get_pyint_orbit_context()
warnings: List[str] = list(dem_summary.get("warnings") or [])
blockers: List[str] = list(dem_summary.get("blockers") or [])
task_summaries: List[Dict[str, Any]] = []
resolved_task_count = 0
missing_task_count = 0
effective_orbit_policy = _get_orbit_policy()
for task_dir in validation.get("task_dirs", []) or []:
task_summary = resolve_pyint_task_input_assets(
task_dir,
dem_summary=dem_summary,
orbit_context=orbit_context,
)
task_summaries.append(task_summary)
task_orbit_policy = str(((task_summary.get("input_assets") or {}).get("orbits") or {}).get("policy") or "").strip()
if task_orbit_policy:
effective_orbit_policy = task_orbit_policy
if task_summary.get("warnings"):
warnings.extend(
f"{task_summary['task_alias']}: {item}"
for item in task_summary["warnings"]
)
if task_summary.get("blockers"):
blockers.extend(
f"{task_summary['task_alias']}: {item}"
for item in task_summary["blockers"]
)
if task_summary["input_assets"]["orbits"]["missing_count"] == 0:
resolved_task_count += 1
else:
missing_task_count += 1
allow_submit = not blockers
precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary()
return {
"root_dir": validation["root_dir"],
"mode": validation["mode"],
"task_count": len(task_summaries),
"selected_task_count": len(task_summaries),
"allow_submit": allow_submit,
"warnings": warnings,
"blockers": blockers,
"invalid_candidates": validation.get("invalid_candidates", []),
"dem": dem_summary,
"orbits": {
"policy": effective_orbit_policy,
"pool_root": orbit_context.get("pool_root", ""),
"pool_exists": bool(orbit_context.get("pool_exists")),
"resolved_task_count": resolved_task_count,
"missing_task_count": missing_task_count,
"duplicate_count": int(orbit_context.get("duplicate_count", 0) or 0),
"warnings": list(orbit_context.get("warnings") or []),
},
"precise_orbit_bridge": precise_orbit_bridge,
"tasks": task_summaries,
}
def summarize_preview_blockers(preview: Dict[str, Any], limit: int = 8) -> str:
blockers = [str(item).strip() for item in (preview.get("blockers") or []) if str(item).strip()]
if not blockers:
return ""
if len(blockers) <= limit:
return "; ".join(blockers)
return "; ".join(blockers[:limit]) + f"; 其余 {len(blockers) - limit} 项已省略"
def materialize_pyint_input_assets(
*,
task_summary: Dict[str, Any],
input_assets_dir: str,
project_name: str = "",
) -> Dict[str, Any]:
input_assets_dir = _normalize_path(input_assets_dir)
os.makedirs(input_assets_dir, exist_ok=True)
record_enabled = bool(getattr(settings, "PYINT_RECORD_INPUT_ASSETS", True))
orbits_dir = os.path.join(input_assets_dir, "orbits")
dem_dir = os.path.join(input_assets_dir, "dem")
downloads_dir = os.path.join(input_assets_dir, "downloads")
if record_enabled:
os.makedirs(orbits_dir, exist_ok=True)
os.makedirs(dem_dir, exist_ok=True)
os.makedirs(downloads_dir, exist_ok=True)
manifest = _copy_json_safe(task_summary.get("input_assets") or {})
manifest["generated_at"] = _utc_now_text()
manifest["task_name"] = task_summary.get("task_name")
manifest["task_alias"] = task_summary.get("task_alias")
manifest["pair_key"] = task_summary.get("pair_key")
manifest["task_dir"] = task_summary.get("task_dir")
manifest["allow_submit"] = bool(task_summary.get("allow_submit"))
manifest["warnings"] = list(task_summary.get("warnings") or [])
manifest["blockers"] = list(task_summary.get("blockers") or [])
dem_summary = manifest.get("dem") or {}
if project_name:
dem_summary["resolved_output_dir"] = os.path.join(_normalize_path(settings.PYINT_DEM_ROOT), project_name)
manifest["dem"] = dem_summary
orbits_summary = manifest.get("orbits") or {}
staged_count = 0
precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary()
should_stage_orbits = record_enabled and (
str(orbits_summary.get("policy") or "").strip().lower() == "stage_txt"
or precise_orbit_bridge.get("enabled")
or str((manifest.get("task_source") or {}).get("satellite_family") or "").strip().upper() == "S1"
)
if should_stage_orbits:
for role in ("master", "slave"):
orbit_item = orbits_summary.get(role) or {}
orbit_path = _normalize_path(orbit_item.get("path"))
expected_name = str(orbit_item.get("expected_name") or "").strip()
if not orbit_item.get("resolved") or not orbit_path or not expected_name:
continue
target_path = os.path.join(orbits_dir, expected_name)
if not os.path.exists(target_path):
shutil.copy2(orbit_path, target_path)
orbit_item["staged_path"] = target_path
orbit_item["stage_operation"] = "copied"
orbit_item["stage_reason"] = "precise_orbit_bridge" if precise_orbit_bridge.get("enabled") else "stage_txt_policy"
staged_count += 1
orbits_summary[role] = orbit_item
manifest["orbits"] = orbits_summary
download_staged_count = 0
task_source = manifest.get("task_source") or {}
if record_enabled and str(task_source.get("satellite_family") or "").strip().upper() == "S1":
production_inputs = task_source.get("production_inputs") or {}
for role_key in ("master_scene", "slave_scene", "master_zip", "slave_zip"):
scene_item = production_inputs.get(role_key) or {}
scene_path = _normalize_path(scene_item.get("path"))
expected_name = str(scene_item.get("expected_name") or os.path.basename(scene_path) or "").strip()
if not scene_item.get("resolved") or not scene_path or not expected_name:
continue
input_kind = str(scene_item.get("input_kind") or "").strip().lower()
if input_kind == "safe_dir" or os.path.isdir(scene_path):
scene_item["staged_path"] = scene_path
scene_item["stage_operation"] = "source_reference"
production_inputs[role_key] = scene_item
continue
target_path = os.path.join(downloads_dir, expected_name)
if not os.path.exists(target_path):
shutil.copy2(scene_path, target_path)
scene_item["staged_path"] = target_path
scene_item["stage_operation"] = "copied"
production_inputs[role_key] = scene_item
download_staged_count += 1
task_source["production_inputs"] = production_inputs
manifest["task_source"] = task_source
materialized = {
"input_assets_dir": input_assets_dir,
"record_enabled": record_enabled,
"orbits_dir": orbits_dir if record_enabled else "",
"dem_dir": dem_dir if record_enabled else "",
"downloads_dir": downloads_dir if record_enabled else "",
"orbits_staged_count": staged_count,
"downloads_staged_count": download_staged_count,
"task_manifest_path": "",
"dem_summary_path": "",
"orbit_summary_path": "",
"input_assets": manifest,
}
if not record_enabled:
return materialized
task_manifest_path = os.path.join(input_assets_dir, "task_manifest.json")
dem_summary_path = os.path.join(dem_dir, "dem_summary.json")
orbit_summary_path = os.path.join(orbits_dir, "orbit_summary.json")
with open(task_manifest_path, "w", encoding="utf-8") as fp:
json.dump(manifest, fp, ensure_ascii=False, indent=2)
fp.write("\n")
with open(dem_summary_path, "w", encoding="utf-8") as fp:
json.dump(dem_summary, fp, ensure_ascii=False, indent=2)
fp.write("\n")
with open(orbit_summary_path, "w", encoding="utf-8") as fp:
json.dump(orbits_summary, fp, ensure_ascii=False, indent=2)
fp.write("\n")
materialized.update(
{
"task_manifest_path": task_manifest_path,
"dem_summary_path": dem_summary_path,
"orbit_summary_path": orbit_summary_path,
}
)
return materialized
def resolve_pyint_task_input_assets(
task_dir: str,
*,
dem_summary: Dict[str, Any] | None = None,
orbit_context: Dict[str, Any] | None = None,
) -> Dict[str, Any]:
task_dir = _normalize_path(task_dir)
task_identity = infer_task_identity(task_dir)
pair_meta = task_identity["pair_meta"]
satellite_family = str(task_identity.get("satellite_family") or "").strip().upper()
warnings: List[str] = []
blockers: List[str] = []
if satellite_family == "S1":
scene_sources = discover_s1_scene_sources(task_dir)
master_archives = list(scene_sources.get("master", []) or [])
slave_archives = list(scene_sources.get("slave", []) or [])
master_date = task_identity["master_date"] or infer_scene_date_from_archives(master_archives)
slave_date = task_identity["slave_date"] or infer_scene_date_from_archives(slave_archives)
master_satellite = _normalize_s1_satellite(pair_meta.get("master_satellite")) or _normalize_s1_satellite(task_identity.get("master_satellite"))
slave_satellite = _normalize_s1_satellite(pair_meta.get("slave_satellite")) or _normalize_s1_satellite(task_identity.get("slave_satellite"))
if not master_archives:
blockers.append("master/ 未识别到 Sentinel-1 SAFE 源目录。")
if not slave_archives:
blockers.append("slave/ 未识别到 Sentinel-1 SAFE 源目录。")
if not master_date:
blockers.append("未能识别主影像日期。")
if not slave_date:
blockers.append("未能识别从影像日期。")
master_scene = _resolve_s1_scene_input(
role="master",
scene_path=master_archives[0] if master_archives else "",
pair_meta=pair_meta,
)
slave_scene = _resolve_s1_scene_input(
role="slave",
scene_path=slave_archives[0] if slave_archives else "",
pair_meta=pair_meta,
)
master_orbit = _resolve_s1_orbit_file(
role="master",
pair_meta=pair_meta,
task_dir=task_dir,
)
slave_orbit = _resolve_s1_orbit_file(
role="slave",
pair_meta=pair_meta,
task_dir=task_dir,
)
if not master_scene.get("resolved"):
blockers.append("master Sentinel-1 源场景缺失。")
if not slave_scene.get("resolved"):
blockers.append("slave Sentinel-1 源场景缺失。")
if not master_orbit.get("resolved"):
blockers.append(str(master_orbit.get("error") or "master Sentinel-1 EOF 缺失"))
if not slave_orbit.get("resolved"):
blockers.append(str(slave_orbit.get("error") or "slave Sentinel-1 EOF 缺失"))
task_source = {
"task_dir": task_dir,
"task_name": task_identity["task_name"],
"task_alias": task_identity["task_alias"],
"pair_key": task_identity["pair_key"],
"satellite_family": "S1",
"master_date": master_date,
"slave_date": slave_date,
"master_satellite": master_satellite,
"slave_satellite": slave_satellite,
"archives": {
"master": master_archives,
"slave": slave_archives,
},
"production_inputs": {
"master_scene": master_scene,
"slave_scene": slave_scene,
},
}
orbits_summary = {
"policy": "require_eof",
"pool_root": "",
"pool_exists": True,
"master": master_orbit,
"slave": slave_orbit,
"resolved_count": int(bool(master_orbit.get("resolved"))) + int(bool(slave_orbit.get("resolved"))),
"missing_count": int(not master_orbit.get("resolved")) + int(not slave_orbit.get("resolved")),
"warnings": [],
"stage_mode": "copy",
"precise_orbit_bridge": {
"enabled": False,
"mode": "not_applicable",
"strict": True,
},
}
else:
archives = discover_lt1_archives(task_dir)
master_archives = list(archives.get("master", []) or [])
slave_archives = list(archives.get("slave", []) or [])
master_date = task_identity["master_date"] or infer_scene_date_from_archives(master_archives)
slave_date = task_identity["slave_date"] or infer_scene_date_from_archives(slave_archives)
master_satellite = _normalize_lt1_satellite(pair_meta.get("master_satellite")) or _infer_satellite_from_archives(master_archives)
slave_satellite = _normalize_lt1_satellite(pair_meta.get("slave_satellite")) or _infer_satellite_from_archives(slave_archives)
if not master_archives:
blockers.append("master/ 未发现 LT-1 原始输入(LT1*.tar.gz 或 LT1*.tiff)。")
if not slave_archives:
blockers.append("slave/ 未发现 LT-1 原始输入(LT1*.tar.gz 或 LT1*.tiff)。")
if not master_date:
blockers.append("未能识别主影像日期。")
if not slave_date:
blockers.append("未能识别从影像日期。")
orbit_policy = _get_orbit_policy()
orbit_context = orbit_context or get_pyint_orbit_context()
orbit_pool_root = orbit_context.get("pool_root", "")
orbit_pool_exists = bool(orbit_context.get("pool_exists"))
orbit_files = orbit_context.get("files", {}) or {}
orbit_warnings: List[str] = []
if orbit_context.get("warnings"):
orbit_warnings.extend(str(item) for item in orbit_context["warnings"] if item)
master_orbit = _resolve_orbit_file(
role="master",
satellite=master_satellite,
date_text=master_date,
pool_root=orbit_pool_root,
orbit_files=orbit_files,
)
slave_orbit = _resolve_orbit_file(
role="slave",
satellite=slave_satellite,
date_text=slave_date,
pool_root=orbit_pool_root,
orbit_files=orbit_files,
)
for orbit_item in (master_orbit, slave_orbit):
if orbit_item.get("resolved"):
continue
message = str(orbit_item.get("error") or f"{orbit_item.get('role')} orbit missing").strip()
if orbit_policy == "validate_only":
orbit_warnings.append(message)
else:
blockers.append(message)
if not orbit_pool_root:
if orbit_policy == "validate_only":
orbit_warnings.append("轨道池未配置,当前仅记录警告。")
else:
blockers.append("轨道池未配置。")
elif not orbit_pool_exists:
if orbit_policy == "validate_only":
orbit_warnings.append(f"轨道池目录不可用: {orbit_pool_root}")
else:
blockers.append(f"轨道池目录不可用: {orbit_pool_root}")
warnings.extend(orbit_warnings)
precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary()
task_source = {
"task_dir": task_dir,
"task_name": task_identity["task_name"],
"task_alias": task_identity["task_alias"],
"pair_key": task_identity["pair_key"],
"satellite_family": "LT1",
"master_date": master_date,
"slave_date": slave_date,
"master_satellite": master_satellite,
"slave_satellite": slave_satellite,
"archives": {
"master": master_archives,
"slave": slave_archives,
},
}
orbits_summary = {
"policy": orbit_policy,
"pool_root": orbit_pool_root,
"pool_exists": orbit_pool_exists,
"master": master_orbit,
"slave": slave_orbit,
"resolved_count": int(bool(master_orbit.get("resolved"))) + int(bool(slave_orbit.get("resolved"))),
"missing_count": int(not master_orbit.get("resolved")) + int(not slave_orbit.get("resolved")),
"warnings": orbit_warnings,
"stage_mode": "copy" if orbit_policy == "stage_txt" or precise_orbit_bridge.get("enabled") else "none",
"precise_orbit_bridge": precise_orbit_bridge,
}
dem_payload = _copy_json_safe(dem_summary or get_pyint_dem_summary())
allow_submit = not blockers and bool(dem_payload.get("allow_submit", True))
return {
"task_name": task_identity["task_name"],
"task_alias": task_identity["task_alias"],
"pair_key": task_identity["pair_key"],
"task_dir": task_dir,
"master_date": master_date,
"slave_date": slave_date,
"master_satellite": master_satellite,
"slave_satellite": slave_satellite,
"satellite_family": satellite_family or normalize_satellite_family(master_satellite or slave_satellite),
"archive_counts": {
"master": len(master_archives),
"slave": len(slave_archives),
},
"warnings": warnings,
"blockers": blockers,
"allow_submit": allow_submit,
"task_source": task_source,
"dem": dem_payload,
"orbit_resolution": {
"master": master_orbit,
"slave": slave_orbit,
},
"input_assets": {
"task_source": task_source,
"dem": dem_payload,
"orbits": orbits_summary,
},
}
def build_pyint_input_preview(root_dir: str, num_to_process: int = 0) -> Dict[str, Any]: def build_pyint_input_preview(root_dir: str, num_to_process: int = 0) -> Dict[str, Any]:
validation = validate_pyint_root_dir(root_dir, num_to_process) validation = validate_pyint_root_dir(root_dir, num_to_process)
dem_summary = get_pyint_dem_summary() dem_summary = get_pyint_dem_summary()
@@ -626,106 +1242,3 @@ def build_pyint_input_preview(root_dir: str, num_to_process: int = 0) -> Dict[st
"precise_orbit_bridge": precise_orbit_bridge, "precise_orbit_bridge": precise_orbit_bridge,
"tasks": task_summaries, "tasks": task_summaries,
} }
def summarize_preview_blockers(preview: Dict[str, Any], limit: int = 8) -> str:
blockers = [str(item).strip() for item in (preview.get("blockers") or []) if str(item).strip()]
if not blockers:
return ""
if len(blockers) <= limit:
return "; ".join(blockers)
return "; ".join(blockers[:limit]) + f"; 其余 {len(blockers) - limit} 项已省略"
def materialize_pyint_input_assets(
*,
task_summary: Dict[str, Any],
input_assets_dir: str,
project_name: str = "",
) -> Dict[str, Any]:
input_assets_dir = _normalize_path(input_assets_dir)
os.makedirs(input_assets_dir, exist_ok=True)
record_enabled = bool(getattr(settings, "PYINT_RECORD_INPUT_ASSETS", True))
orbits_dir = os.path.join(input_assets_dir, "orbits")
dem_dir = os.path.join(input_assets_dir, "dem")
if record_enabled:
os.makedirs(orbits_dir, exist_ok=True)
os.makedirs(dem_dir, exist_ok=True)
manifest = _copy_json_safe(task_summary.get("input_assets") or {})
manifest["generated_at"] = _utc_now_text()
manifest["task_name"] = task_summary.get("task_name")
manifest["task_alias"] = task_summary.get("task_alias")
manifest["pair_key"] = task_summary.get("pair_key")
manifest["task_dir"] = task_summary.get("task_dir")
manifest["allow_submit"] = bool(task_summary.get("allow_submit"))
manifest["warnings"] = list(task_summary.get("warnings") or [])
manifest["blockers"] = list(task_summary.get("blockers") or [])
dem_summary = manifest.get("dem") or {}
if project_name:
dem_summary["resolved_output_dir"] = os.path.join(_normalize_path(settings.PYINT_DEM_ROOT), project_name)
manifest["dem"] = dem_summary
orbits_summary = manifest.get("orbits") or {}
staged_count = 0
precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary()
should_stage_orbits = record_enabled and (
str(orbits_summary.get("policy") or "").strip().lower() == "stage_txt"
or precise_orbit_bridge.get("enabled")
)
if should_stage_orbits:
for role in ("master", "slave"):
orbit_item = orbits_summary.get(role) or {}
orbit_path = _normalize_path(orbit_item.get("path"))
expected_name = str(orbit_item.get("expected_name") or "").strip()
if not orbit_item.get("resolved") or not orbit_path or not expected_name:
continue
target_path = os.path.join(orbits_dir, expected_name)
if not os.path.exists(target_path):
shutil.copy2(orbit_path, target_path)
orbit_item["staged_path"] = target_path
orbit_item["stage_operation"] = "copied"
orbit_item["stage_reason"] = "precise_orbit_bridge" if precise_orbit_bridge.get("enabled") else "stage_txt_policy"
staged_count += 1
orbits_summary[role] = orbit_item
manifest["orbits"] = orbits_summary
materialized = {
"input_assets_dir": input_assets_dir,
"record_enabled": record_enabled,
"orbits_dir": orbits_dir if record_enabled else "",
"dem_dir": dem_dir if record_enabled else "",
"orbits_staged_count": staged_count,
"task_manifest_path": "",
"dem_summary_path": "",
"orbit_summary_path": "",
"input_assets": manifest,
}
if not record_enabled:
return materialized
task_manifest_path = os.path.join(input_assets_dir, "task_manifest.json")
dem_summary_path = os.path.join(dem_dir, "dem_summary.json")
orbit_summary_path = os.path.join(orbits_dir, "orbit_summary.json")
with open(task_manifest_path, "w", encoding="utf-8") as fp:
json.dump(manifest, fp, ensure_ascii=False, indent=2)
fp.write("\n")
with open(dem_summary_path, "w", encoding="utf-8") as fp:
json.dump(dem_summary, fp, ensure_ascii=False, indent=2)
fp.write("\n")
with open(orbit_summary_path, "w", encoding="utf-8") as fp:
json.dump(orbits_summary, fp, ensure_ascii=False, indent=2)
fp.write("\n")
materialized.update(
{
"task_manifest_path": task_manifest_path,
"dem_summary_path": dem_summary_path,
"orbit_summary_path": orbit_summary_path,
}
)
return materialized
+38 -1
View File
@@ -13,10 +13,12 @@ from typing import Any, Dict, Iterable, List, Optional
from ..config import get_env_text, read_bool_env, settings from ..config import get_env_text, read_bool_env, settings
from .dinsar_naming import PAIR_META_FILENAME, build_fallback_pair_key, find_json_sidecar from .dinsar_naming import PAIR_META_FILENAME, build_fallback_pair_key, find_json_sidecar
from ..utils import normalize_satellite_family
from .wsl_service import run_wsl_exec from .wsl_service import run_wsl_exec
LT1_INPUT_GLOBS = ("LT1*.tar.gz", "LT1*.tiff") LT1_INPUT_GLOBS = ("LT1*.tar.gz", "LT1*.tiff")
S1_INPUT_GLOBS = ("S1*.zip",)
DEFAULT_RANGE_LOOKS = 2 DEFAULT_RANGE_LOOKS = 2
DEFAULT_AZIMUTH_LOOKS = 2 DEFAULT_AZIMUTH_LOOKS = 2
DEFAULT_DEM_RESOLUTION_M = 30.0 DEFAULT_DEM_RESOLUTION_M = 30.0
@@ -334,6 +336,15 @@ def build_project_name(pair_key: str, run_key: str) -> str:
return slugify_text(f"{pair_key}_{run_key}", default="pyint_project", max_len=120) return slugify_text(f"{pair_key}_{run_key}", default="pyint_project", max_len=120)
def build_profile_project_name(satellite_family: Any, pair_key: str, run_key: str) -> str:
family = str(normalize_satellite_family(satellite_family) or "").strip().upper()
if family == "S1":
return slugify_text(f"s1_{pair_key}_{run_key}", default="s1_pyint_project", max_len=120)
if family == "LT1":
return slugify_text(f"lt1_{pair_key}_{run_key}", default="lt1_pyint_project", max_len=120)
return build_project_name(pair_key, run_key)
def windows_path_to_wsl_mount(path: str) -> str: def windows_path_to_wsl_mount(path: str) -> str:
text = str(path or "").strip().strip('"').strip("'") text = str(path or "").strip().strip('"').strip("'")
if not text: if not text:
@@ -377,6 +388,22 @@ def discover_lt1_archives(task_dir: str) -> Dict[str, List[str]]:
return result return result
def discover_s1_scene_sources(task_dir: str) -> Dict[str, List[str]]:
task_path = Path(os.path.normpath(os.path.abspath(str(task_dir or "").strip())))
pair_meta = find_json_sidecar(str(task_path), PAIR_META_FILENAME, max_levels=0) or {}
result: Dict[str, List[str]] = {"master": [], "slave": []}
for role in ("master", "slave"):
explicit_path = str(pair_meta.get(f"{role}_path") or "").strip()
role_dir = task_path / role
candidates: List[str] = []
if explicit_path:
candidates.append(str(Path(explicit_path).resolve()))
if not candidates and role_dir.is_dir() and (role_dir / "manifest.safe").is_file():
candidates.append(str(role_dir.resolve()))
result[role] = sorted(set(candidates))
return result
def infer_scene_date_from_archives(paths: Iterable[str]) -> str: def infer_scene_date_from_archives(paths: Iterable[str]) -> str:
dates = { dates = {
date_text date_text
@@ -393,7 +420,14 @@ def infer_task_identity(task_dir: str) -> Dict[str, Any]:
task_name = os.path.basename(os.path.normpath(task_dir)) task_name = os.path.basename(os.path.normpath(task_dir))
pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {} pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {}
task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(task_alias, task_dir) master_satellite = str(pair_meta.get("master_satellite") or "").strip().upper()
slave_satellite = str(pair_meta.get("slave_satellite") or "").strip().upper()
satellite_family = normalize_satellite_family(master_satellite or slave_satellite)
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(
task_alias,
task_dir,
satellite_family=satellite_family,
)
master_date = normalize_date_text(pair_meta.get("master_imaging_date")) master_date = normalize_date_text(pair_meta.get("master_imaging_date"))
slave_date = normalize_date_text(pair_meta.get("slave_imaging_date")) slave_date = normalize_date_text(pair_meta.get("slave_imaging_date"))
return { return {
@@ -403,6 +437,9 @@ def infer_task_identity(task_dir: str) -> Dict[str, Any]:
"pair_meta": pair_meta, "pair_meta": pair_meta,
"master_date": master_date, "master_date": master_date,
"slave_date": slave_date, "slave_date": slave_date,
"master_satellite": master_satellite,
"slave_satellite": slave_satellite,
"satellite_family": satellite_family,
} }
@@ -39,6 +39,7 @@ from .dinsar_naming import (
build_fallback_pair_key, build_fallback_pair_key,
find_json_sidecar, find_json_sidecar,
) )
from ..utils import normalize_satellite_family
from .dinsar_result_layout_service import ( from .dinsar_result_layout_service import (
RUN_CURRENT_DIRNAME, RUN_CURRENT_DIRNAME,
RUN_NATIVE_DIRNAME, RUN_NATIVE_DIRNAME,
@@ -203,10 +204,16 @@ def _resolve_candidate_identity(candidate: Dict[str, Any]) -> Dict[str, Any]:
pair_meta.get("task_alias"), pair_meta.get("task_alias"),
candidate.get("task_name"), candidate.get("task_name"),
) or "Task_unknown_unknown" ) or "Task_unknown_unknown"
satellite_family = normalize_satellite_family(
pair_meta.get("master_satellite")
or pair_meta.get("slave_satellite")
or run_meta.get("master_satellite")
or run_meta.get("slave_satellite")
)
pair_key = _first_text( pair_key = _first_text(
run_meta.get("pair_key"), run_meta.get("pair_key"),
pair_meta.get("pair_key"), pair_meta.get("pair_key"),
) or build_fallback_pair_key(task_alias, source_dir) ) or build_fallback_pair_key(task_alias, source_dir, satellite_family=satellite_family)
run_key = _first_text(run_meta.get("run_key")) or ( run_key = _first_text(run_meta.get("run_key")) or (
"legacy_" + _stable_digest(candidate.get("engine_code"), pair_key, source_dir, candidate["primary_file"], length=16) "legacy_" + _stable_digest(candidate.get("engine_code"), pair_key, source_dir, candidate["primary_file"], length=16)
) )
+98 -1
View File
@@ -10,7 +10,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from .. import database from .. import database
from ..config import settings, split_env_paths from ..config import settings, split_env_paths
from ..models import ManagedRootORM, PathInventoryORM, ScanCursorORM from ..models import AssetInventoryStateORM, ManagedRootORM, PathInventoryORM, ScanCursorORM
_SLUG_RE = re.compile(r"[^a-z0-9]+") _SLUG_RE = re.compile(r"[^a-z0-9]+")
@@ -87,6 +87,15 @@ def _cursor_type_for_scan_mode(scan_mode: str) -> str:
return mapping.get(str(scan_mode or "").strip().lower(), "directory_walk") return mapping.get(str(scan_mode or "").strip().lower(), "directory_walk")
def _asset_inventory_type_for_root_role(root_role: str) -> Optional[str]:
role = str(root_role or "").strip().lower()
if role == "source_product_pool":
return "source_product"
if role == "orbit_asset_pool":
return "orbit_asset"
return None
def _iter_multi_root_specs( def _iter_multi_root_specs(
*, *,
env_var: str, env_var: str,
@@ -169,6 +178,36 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
scan_mode="archive_walk", scan_mode="archive_walk",
) )
) )
source_product_paths = split_env_paths(settings.SOURCE_PRODUCT_DIRS)
if not source_product_paths:
source_product_paths = (
split_env_paths(settings.INSAR_STORAGE_DIRS)
+ split_env_paths(settings.MONITOR_RADAR_DIRS)
)
specs.extend(
_iter_multi_root_specs(
env_var="SOURCE_PRODUCT_DIRS",
paths=source_product_paths,
root_role="source_product_pool",
display_prefix="Source Product Pool",
scan_mode="file_pool",
)
)
source_product_path_set = {_normalize_root_path(path) for path in source_product_paths}
sentinel1_storage_paths = [
path
for path in split_env_paths(settings.SENTINEL1_STORAGE_DIRS)
if _normalize_root_path(path) not in source_product_path_set
]
specs.extend(
_iter_multi_root_specs(
env_var="SENTINEL1_STORAGE_DIRS",
paths=sentinel1_storage_paths,
root_role="source_product_pool",
display_prefix="Sentinel-1 Storage Pool",
scan_mode="file_pool",
)
)
specs.extend( specs.extend(
_iter_multi_root_specs( _iter_multi_root_specs(
env_var="INSAR_STORAGE_DIRS", env_var="INSAR_STORAGE_DIRS",
@@ -214,6 +253,18 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
scan_mode="scene_directory", scan_mode="scene_directory",
) )
) )
orbit_source_paths = split_env_paths(settings.ORBIT_SOURCE_DIRS)
if not orbit_source_paths:
orbit_source_paths = split_env_paths(settings.MONITOR_ORBIT_DIR)
specs.extend(
_iter_multi_root_specs(
env_var="ORBIT_SOURCE_DIRS",
paths=orbit_source_paths,
root_role="orbit_asset_pool",
display_prefix="Orbit Asset Pool",
scan_mode="file_pool",
)
)
specs.extend( specs.extend(
_iter_single_root_specs( _iter_single_root_specs(
env_var="MONITOR_ORBIT_DIR", env_var="MONITOR_ORBIT_DIR",
@@ -374,6 +425,43 @@ class RootRegistryService:
changed = True changed = True
return "updated" if changed else None return "updated" if changed else None
async def _ensure_asset_inventory_state(self, db: AsyncSession, root: ManagedRootORM) -> Optional[str]:
inventory_type = _asset_inventory_type_for_root_role(root.root_role)
if not inventory_type:
return None
result = await db.execute(
select(AssetInventoryStateORM).where(
AssetInventoryStateORM.root_ref_id == root.id,
AssetInventoryStateORM.inventory_type == inventory_type,
)
)
state = result.scalar_one_or_none()
if state is None:
state = AssetInventoryStateORM(
root_ref_id=root.id,
inventory_type=inventory_type,
root_path=root.path,
scan_mode=root.scan_mode,
status="NEVER_SCANNED",
needs_rescan=True,
metadata_json={
"root_role": root.root_role,
"created_by": "root_registry_sync",
},
)
db.add(state)
return "created"
changed = False
if state.root_path != root.path:
state.root_path = root.path
changed = True
if state.scan_mode != root.scan_mode:
state.scan_mode = root.scan_mode
changed = True
return "updated" if changed else None
async def sync_from_settings(self, db: Optional[AsyncSession] = None) -> Dict[str, Any]: async def sync_from_settings(self, db: Optional[AsyncSession] = None) -> Dict[str, Any]:
generated_session = db is None generated_session = db is None
if generated_session: if generated_session:
@@ -394,6 +482,8 @@ class RootRegistryService:
disabled = 0 disabled = 0
cursor_created = 0 cursor_created = 0
cursor_updated = 0 cursor_updated = 0
inventory_state_created = 0
inventory_state_updated = 0
synced_codes: set[str] = set() synced_codes: set[str] = set()
for spec in specs: for spec in specs:
@@ -448,6 +538,12 @@ class RootRegistryService:
elif cursor_change == "updated": elif cursor_change == "updated":
cursor_updated += 1 cursor_updated += 1
inventory_state_change = await self._ensure_asset_inventory_state(db, row)
if inventory_state_change == "created":
inventory_state_created += 1
elif inventory_state_change == "updated":
inventory_state_updated += 1
for row in existing_rows: for row in existing_rows:
if row.root_code in synced_codes: if row.root_code in synced_codes:
continue continue
@@ -464,6 +560,7 @@ class RootRegistryService:
"updated": updated, "updated": updated,
"disabled": disabled, "disabled": disabled,
"cursor_created_or_updated": cursor_created + cursor_updated, "cursor_created_or_updated": cursor_created + cursor_updated,
"asset_inventory_state_created_or_updated": inventory_state_created + inventory_state_updated,
"summary": summary, "summary": summary,
} }
except Exception: except Exception:
+63 -3
View File
@@ -14,7 +14,7 @@ from typing import Any, Dict, List, Optional, Tuple
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select from sqlalchemy.future import select
from sqlalchemy import and_, cast, func, or_ from sqlalchemy import and_, case, cast, func, or_
from sqlalchemy.orm import aliased from sqlalchemy.orm import aliased
from geoalchemy2 import Geography from geoalchemy2 import Geography
@@ -43,11 +43,60 @@ from .dinsar_naming import build_pair_key, build_task_alias, ensure_unique_task_
from .pairing_state_service import pairing_state_service from .pairing_state_service import pairing_state_service
PAIRING_POLICY_VERSION = "2026.05.raw-source.v1" PAIRING_POLICY_VERSION = "2026.05.raw-source.v2"
PAIRING_WARNING_CANDIDATE_THRESHOLD = 3000 PAIRING_WARNING_CANDIDATE_THRESHOLD = 3000
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def _normalized_satellite_family_expr(alias):
compact_satellite = func.upper(
func.replace(
func.replace(
func.replace(func.coalesce(alias.satellite, ""), "-", ""),
"_",
"",
),
" ",
"",
)
)
inferred_family = case(
(
compact_satellite.in_(
["LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"]
),
"LT1",
),
(
compact_satellite.in_(
[
"S1",
"S1A",
"S1B",
"S1C",
"SENTINEL1",
"SENTINEL1A",
"SENTINEL1B",
"SENTINEL1C",
]
),
"S1",
),
else_=func.upper(alias.satellite),
)
return func.coalesce(func.nullif(func.upper(alias.satellite_family), ""), inferred_family)
def _same_relative_orbit_expr(left_alias, right_alias):
left_relative_orbit = func.upper(func.trim(func.coalesce(left_alias.relative_orbit, "")))
right_relative_orbit = func.upper(func.trim(func.coalesce(right_alias.relative_orbit, "")))
return and_(
left_relative_orbit != "",
right_relative_orbit != "",
left_relative_orbit == right_relative_orbit,
)
class SpatialService: class SpatialService:
""" """
纯 PostGIS 空间计算服务 纯 PostGIS 空间计算服务
@@ -155,6 +204,8 @@ class SpatialService:
) -> List[dict]: ) -> List[dict]:
master_alias = aliased(RadarDataORM) master_alias = aliased(RadarDataORM)
slave_alias = aliased(RadarDataORM) slave_alias = aliased(RadarDataORM)
master_family_expr = _normalized_satellite_family_expr(master_alias)
slave_family_expr = _normalized_satellite_family_expr(slave_alias)
center_distance_expr = func.coalesce( center_distance_expr = func.coalesce(
PairingMetricCacheORM.scene_center_distance_meters, PairingMetricCacheORM.scene_center_distance_meters,
PairingMetricCacheORM.spatial_baseline_meters, PairingMetricCacheORM.spatial_baseline_meters,
@@ -192,6 +243,14 @@ class SpatialService:
if params.require_same_polarization: if params.require_same_polarization:
stmt = stmt.where(PairingMetricCacheORM.same_polarization.is_(True)) stmt = stmt.where(PairingMetricCacheORM.same_polarization.is_(True))
stmt = stmt.where(
or_(
master_family_expr != "S1",
slave_family_expr != "S1",
_same_relative_orbit_expr(master_alias, slave_alias),
)
)
if params.allowed_satellites: if params.allowed_satellites:
allowed_satellites = [ allowed_satellites = [
str(item).strip().upper() str(item).strip().upper()
@@ -284,6 +343,7 @@ class SpatialService:
slave.file_path, slave.file_path,
master.imaging_date, master.imaging_date,
slave.imaging_date, slave.imaging_date,
master.satellite_family or slave.satellite_family or master.satellite or slave.satellite,
), ),
pair_uid=candidate.get("pair_uid"), pair_uid=candidate.get("pair_uid"),
metric_cache_ref_id=candidate.get("metric_cache_ref_id"), metric_cache_ref_id=candidate.get("metric_cache_ref_id"),
@@ -1281,7 +1341,7 @@ class SpatialService:
compact = raw_satellite.replace("-", "").replace("_", "").replace(" ", "") compact = raw_satellite.replace("-", "").replace("_", "").replace(" ", "")
if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}: if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}:
return "LT1" return "LT1"
if compact in {"S1", "S1A", "S1B", "SENTINEL1", "SENTINEL1A", "SENTINEL1B"}: if compact in {"S1", "S1A", "S1B", "S1C", "SENTINEL1", "SENTINEL1A", "SENTINEL1B", "SENTINEL1C"}:
return "S1" return "S1"
return raw_satellite or "UNKNOWN" return raw_satellite or "UNKNOWN"
+39 -12
View File
@@ -1,4 +1,6 @@
import os import os
import re
from datetime import datetime
from typing import Optional, Tuple, List, Callable, Dict, Any from typing import Optional, Tuple, List, Callable, Dict, Any
from lxml import etree from lxml import etree
@@ -18,7 +20,7 @@ def _create_secure_xml_parser() -> etree.XMLParser:
recover=False, recover=False,
) )
# --- Sentinel-1 (S1A/S1B) Parsers --- # --- Sentinel-1 (S1A/S1B/S1C) Parsers ---
def _radar_meta_base() -> Dict[str, Any]: def _radar_meta_base() -> Dict[str, Any]:
return { return {
@@ -83,7 +85,7 @@ def normalize_satellite_family(value: Optional[str]) -> Optional[str]:
compact = raw.replace("-", "").replace("_", "").replace(" ", "") compact = raw.replace("-", "").replace("_", "").replace(" ", "")
if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}: if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}:
return "LT1" return "LT1"
if compact in {"S1", "S1A", "S1B", "SENTINEL1", "SENTINEL1A", "SENTINEL1B"}: if compact in {"S1", "S1A", "S1B", "S1C", "SENTINEL1", "SENTINEL1A", "SENTINEL1B", "SENTINEL1C"}:
return "S1" return "S1"
if compact in {"GF3", "GAOFEN3"}: if compact in {"GF3", "GAOFEN3"}:
return "GF3" return "GF3"
@@ -96,19 +98,44 @@ def parse_s1_radar_filename(folder_name: str) -> Optional[Dict[str, Any]]:
Example: S1A_IW_SLC__1SDV_20250101T104105_... Example: S1A_IW_SLC__1SDV_20250101T104105_...
Returns a metadata dict. Returns a metadata dict.
""" """
parts = folder_name.split('_') name = os.path.basename(str(folder_name or "").strip())
if len(parts) < 5 or not parts[0].startswith('S1'): if name.lower().endswith(".zip"):
name = name[:-4]
if name.lower().endswith(".safe"):
name = name[:-5]
match = re.match(
r"^(?P<satellite>S1[A-Z])_"
r"(?P<mode>[A-Z0-9]+)_"
r"(?P<product>[A-Z0-9]+)_+"
r"(?P<class>[0-9A-Z]{4})_"
r"(?P<start>\d{8}T\d{6}(?:\.\d+)?)_"
r"(?P<stop>\d{8}T\d{6}(?:\.\d+)?)_"
r"(?P<absolute_orbit>\d+)_"
r"(?P<datatake>[0-9A-F]+)_"
r"(?P<product_uid>[0-9A-F]+)$",
name,
flags=re.IGNORECASE,
)
if not match:
return None return None
meta = _radar_meta_base() meta = _radar_meta_base()
meta["satellite"] = parts[0] meta["satellite"] = match.group("satellite").upper()
meta["satellite_family"] = normalize_satellite_family(parts[0]) meta["satellite_family"] = normalize_satellite_family(meta["satellite"])
meta["imaging_date"] = parts[4].split('T')[0] meta["imaging_date"] = match.group("start")[:8]
meta["imaging_mode"] = parts[1] meta["imaging_mode"] = match.group("mode").upper()
meta["source_product_token"] = parts[2] meta["source_product_token"] = match.group("class").upper()
meta["product_type"] = parts[2] meta["product_type"] = match.group("product").upper()
polarization = parts[3] # e.g., '1SDV' -> 'DV' is dual-pol VV/VH meta["product_level"] = "L1"
meta["polarization"] = polarization[2:] if len(polarization) > 2 else polarization polarization = match.group("class").upper() # e.g. 1SDV -> DV
meta["polarization"] = polarization[-2:] if len(polarization) > 2 else polarization
meta["orbit_circle"] = match.group("absolute_orbit").lstrip("0") or match.group("absolute_orbit")
meta["product_unique_id"] = name
try:
start_time = datetime.strptime(match.group("start").split(".")[0], "%Y%m%dT%H%M%S")
meta["acquisition_time_utc"] = start_time.isoformat()
except ValueError:
meta["acquisition_time_utc"] = match.group("start")
return meta return meta
@@ -81,7 +81,7 @@ SET
('LT1', 'LT1A', 'LT1B', 'LUTAN1', 'LUTAN1A', 'LUTAN1B') ('LT1', 'LT1A', 'LT1B', 'LUTAN1', 'LUTAN1A', 'LUTAN1B')
THEN 'LT1' THEN 'LT1'
WHEN upper(replace(replace(replace(COALESCE(satellite, ''), '-', ''), '_', ''), ' ', '')) IN WHEN upper(replace(replace(replace(COALESCE(satellite, ''), '-', ''), '_', ''), ' ', '')) IN
('S1', 'S1A', 'S1B', 'SENTINEL1', 'SENTINEL1A', 'SENTINEL1B') ('S1', 'S1A', 'S1B', 'S1C', 'SENTINEL1', 'SENTINEL1A', 'SENTINEL1B', 'SENTINEL1C')
THEN 'S1' THEN 'S1'
WHEN NULLIF(satellite, '') IS NOT NULL WHEN NULLIF(satellite, '') IS NOT NULL
THEN upper(satellite) THEN upper(satellite)
@@ -0,0 +1,371 @@
-- Migration: Source product and orbit asset inventory
-- Version: 10.0
-- Date: 2026-05-12
-- Purpose: Add first-class source product assets, orbit assets, scene-orbit bindings,
-- inventory state, and radar_data compatibility fields.
ALTER TABLE IF EXISTS radar_data
ADD COLUMN IF NOT EXISTS source_product_ref_id INTEGER NULL;
ALTER TABLE IF EXISTS radar_data
ADD COLUMN IF NOT EXISTS source_archive_asset_id INTEGER NULL;
ALTER TABLE IF EXISTS radar_data
ADD COLUMN IF NOT EXISTS selected_orbit_asset_id INTEGER NULL;
ALTER TABLE IF EXISTS radar_data
ADD COLUMN IF NOT EXISTS orbit_binding_status VARCHAR(32) NOT NULL DEFAULT 'UNBOUND';
ALTER TABLE IF EXISTS radar_data
ADD COLUMN IF NOT EXISTS orbit_binding_reason TEXT NULL;
ALTER TABLE IF EXISTS radar_data
ADD COLUMN IF NOT EXISTS acquisition_start_time_utc TIMESTAMP NULL;
ALTER TABLE IF EXISTS radar_data
ADD COLUMN IF NOT EXISTS acquisition_stop_time_utc TIMESTAMP NULL;
ALTER TABLE IF EXISTS radar_data
ADD COLUMN IF NOT EXISTS absolute_orbit VARCHAR NULL;
ALTER TABLE IF EXISTS radar_data
ADD COLUMN IF NOT EXISTS relative_orbit VARCHAR NULL;
ALTER TABLE IF EXISTS radar_data
ADD COLUMN IF NOT EXISTS source_format VARCHAR(32) NULL;
ALTER TABLE IF EXISTS radar_data
ADD COLUMN IF NOT EXISTS metadata_json JSON NULL;
CREATE TABLE IF NOT EXISTS source_product_assets (
id SERIAL PRIMARY KEY,
asset_uid VARCHAR(128) NOT NULL UNIQUE,
logical_product_uid VARCHAR(128) NULL,
satellite_family VARCHAR(32) NULL,
satellite VARCHAR(32) NULL,
source_format VARCHAR(32) NOT NULL,
product_type VARCHAR(64) NULL,
product_level VARCHAR(64) NULL,
imaging_mode VARCHAR(64) NULL,
polarization VARCHAR(64) NULL,
absolute_orbit VARCHAR(64) NULL,
relative_orbit VARCHAR(64) NULL,
orbit_direction VARCHAR(32) NULL,
acquisition_start_time_utc TIMESTAMP NULL,
acquisition_stop_time_utc TIMESTAMP NULL,
imaging_date VARCHAR(8) NULL,
root_ref_id INTEGER NULL REFERENCES managed_roots(id) ON DELETE SET NULL,
root_path VARCHAR NULL,
file_path VARCHAR NOT NULL UNIQUE,
archive_path VARCHAR NULL,
path_kind VARCHAR(24) NULL,
file_name VARCHAR(255) NULL,
file_stem VARCHAR(255) NULL,
file_ext VARCHAR(32) NULL,
size_bytes BIGINT NULL,
mtime_epoch DOUBLE PRECISION NULL,
checksum_sha256 VARCHAR(64) NULL,
checksum_status VARCHAR(32) NOT NULL DEFAULT 'NOT_COMPUTED',
parser_name VARCHAR(64) NULL,
parser_version VARCHAR(32) NULL,
parse_status VARCHAR(32) NOT NULL DEFAULT 'PENDING',
parse_error TEXT NULL,
parsed_at TIMESTAMP NULL,
metadata_json JSON NULL,
is_active BOOLEAN NOT NULL DEFAULT TRUE,
missing_since TIMESTAMP NULL,
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
updated_at TIMESTAMP DEFAULT NOW()
);
CREATE TABLE IF NOT EXISTS orbit_assets (
id SERIAL PRIMARY KEY,
orbit_uid VARCHAR(128) NOT NULL UNIQUE,
satellite_family VARCHAR(32) NULL,
satellite VARCHAR(32) NULL,
orbit_type VARCHAR(64) NOT NULL,
native_format VARCHAR(32) NOT NULL,
quality_class VARCHAR(32) NOT NULL DEFAULT 'unknown',
root_ref_id INTEGER NULL REFERENCES managed_roots(id) ON DELETE SET NULL,
root_path VARCHAR NULL,
file_path VARCHAR NOT NULL UNIQUE,
file_name VARCHAR(255) NULL,
file_stem VARCHAR(255) NULL,
file_ext VARCHAR(32) NULL,
size_bytes BIGINT NULL,
mtime_epoch DOUBLE PRECISION NULL,
checksum_sha256 VARCHAR(64) NULL,
checksum_status VARCHAR(32) NOT NULL DEFAULT 'NOT_COMPUTED',
validity_start_time_utc TIMESTAMP NULL,
validity_stop_time_utc TIMESTAMP NULL,
generation_time_utc TIMESTAMP NULL,
published_time_utc TIMESTAMP NULL,
parser_name VARCHAR(64) NULL,
parser_version VARCHAR(32) NULL,
parse_status VARCHAR(32) NOT NULL DEFAULT 'PENDING',
parse_error TEXT NULL,
parsed_at TIMESTAMP NULL,
metadata_json JSON NULL,
is_active BOOLEAN NOT NULL DEFAULT TRUE,
missing_since TIMESTAMP NULL,
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
updated_at TIMESTAMP DEFAULT NOW()
);
CREATE TABLE IF NOT EXISTS scene_orbit_bindings (
id SERIAL PRIMARY KEY,
radar_data_id INTEGER NOT NULL REFERENCES radar_data(id) ON DELETE CASCADE,
orbit_asset_id INTEGER NOT NULL REFERENCES orbit_assets(id) ON DELETE CASCADE,
binding_role VARCHAR(32) NOT NULL DEFAULT 'primary_orbit',
match_status VARCHAR(32) NOT NULL DEFAULT 'CANDIDATE',
selection_status VARCHAR(32) NOT NULL DEFAULT 'CANDIDATE',
selection_rank INTEGER NULL,
priority_score DOUBLE PRECISION NULL,
coverage_margin_before_seconds DOUBLE PRECISION NULL,
coverage_margin_after_seconds DOUBLE PRECISION NULL,
match_rule_version VARCHAR(64) NULL,
match_reason TEXT NULL,
selected_at TIMESTAMP NULL,
metadata_json JSON NULL,
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
updated_at TIMESTAMP DEFAULT NOW(),
CONSTRAINT uq_scene_orbit_binding_role UNIQUE (radar_data_id, orbit_asset_id, binding_role)
);
CREATE TABLE IF NOT EXISTS orbit_asset_derivatives (
id SERIAL PRIMARY KEY,
orbit_asset_id INTEGER NOT NULL REFERENCES orbit_assets(id) ON DELETE CASCADE,
engine_code VARCHAR(32) NOT NULL,
derivative_format VARCHAR(32) NOT NULL,
derivative_role VARCHAR(64) NULL,
pool_path VARCHAR NOT NULL,
size_bytes BIGINT NULL,
mtime_epoch DOUBLE PRECISION NULL,
checksum_sha256 VARCHAR(64) NULL,
generation_status VARCHAR(32) NOT NULL DEFAULT 'PENDING',
generation_error TEXT NULL,
generated_at TIMESTAMP NULL,
metadata_json JSON NULL,
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
updated_at TIMESTAMP DEFAULT NOW(),
CONSTRAINT uq_orbit_asset_derivative_pool_path UNIQUE (orbit_asset_id, engine_code, derivative_format, pool_path)
);
CREATE TABLE IF NOT EXISTS asset_inventory_states (
id SERIAL PRIMARY KEY,
root_ref_id INTEGER NOT NULL REFERENCES managed_roots(id) ON DELETE CASCADE,
inventory_type VARCHAR(32) NOT NULL,
root_path VARCHAR NOT NULL,
scan_mode VARCHAR(32) NOT NULL DEFAULT 'file_pool',
status VARCHAR(32) NOT NULL DEFAULT 'NEVER_SCANNED',
last_scan_started_at TIMESTAMP NULL,
last_scan_finished_at TIMESTAMP NULL,
last_seen_entry_count INTEGER NULL,
last_asset_count INTEGER NULL,
last_issue_count INTEGER NULL,
parser_version VARCHAR(32) NULL,
needs_rescan BOOLEAN NOT NULL DEFAULT TRUE,
last_error TEXT NULL,
metadata_json JSON NULL,
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
updated_at TIMESTAMP DEFAULT NOW(),
CONSTRAINT uq_asset_inventory_state_root_type UNIQUE (root_ref_id, inventory_type)
);
CREATE TABLE IF NOT EXISTS asset_inventory_issues (
id SERIAL PRIMARY KEY,
root_ref_id INTEGER NULL REFERENCES managed_roots(id) ON DELETE SET NULL,
inventory_type VARCHAR(32) NOT NULL,
asset_ref_id INTEGER NULL REFERENCES source_product_assets(id) ON DELETE SET NULL,
radar_data_id INTEGER NULL REFERENCES radar_data(id) ON DELETE SET NULL,
orbit_asset_id INTEGER NULL REFERENCES orbit_assets(id) ON DELETE SET NULL,
severity VARCHAR(16) NOT NULL DEFAULT 'warning',
issue_code VARCHAR(64) NOT NULL,
issue_message TEXT NULL,
source_path VARCHAR NULL,
status VARCHAR(16) NOT NULL DEFAULT 'OPEN',
first_seen_at TIMESTAMP NOT NULL DEFAULT NOW(),
last_seen_at TIMESTAMP NOT NULL DEFAULT NOW(),
resolved_at TIMESTAMP NULL,
metadata_json JSON NULL
);
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM pg_constraint WHERE conname = 'fk_radar_data_source_product_ref_id'
) THEN
ALTER TABLE radar_data
ADD CONSTRAINT fk_radar_data_source_product_ref_id
FOREIGN KEY (source_product_ref_id)
REFERENCES source_product_assets(id)
ON DELETE SET NULL;
END IF;
IF NOT EXISTS (
SELECT 1 FROM pg_constraint WHERE conname = 'fk_radar_data_source_archive_asset_id'
) THEN
ALTER TABLE radar_data
ADD CONSTRAINT fk_radar_data_source_archive_asset_id
FOREIGN KEY (source_archive_asset_id)
REFERENCES source_product_assets(id)
ON DELETE SET NULL;
END IF;
IF NOT EXISTS (
SELECT 1 FROM pg_constraint WHERE conname = 'fk_radar_data_selected_orbit_asset_id'
) THEN
ALTER TABLE radar_data
ADD CONSTRAINT fk_radar_data_selected_orbit_asset_id
FOREIGN KEY (selected_orbit_asset_id)
REFERENCES orbit_assets(id)
ON DELETE SET NULL;
END IF;
END $$;
CREATE INDEX IF NOT EXISTS idx_radar_data_source_product_ref
ON radar_data (source_product_ref_id);
CREATE INDEX IF NOT EXISTS idx_radar_data_source_archive_asset
ON radar_data (source_archive_asset_id);
CREATE INDEX IF NOT EXISTS idx_radar_data_selected_orbit_asset
ON radar_data (selected_orbit_asset_id);
CREATE INDEX IF NOT EXISTS idx_radar_data_orbit_binding_status
ON radar_data (orbit_binding_status);
CREATE INDEX IF NOT EXISTS idx_radar_data_acquisition_start
ON radar_data (acquisition_start_time_utc);
CREATE INDEX IF NOT EXISTS idx_radar_data_absolute_orbit
ON radar_data (absolute_orbit);
CREATE INDEX IF NOT EXISTS idx_radar_data_relative_orbit
ON radar_data (relative_orbit);
CREATE INDEX IF NOT EXISTS idx_radar_data_source_format
ON radar_data (source_format);
CREATE INDEX IF NOT EXISTS idx_source_product_assets_asset_uid
ON source_product_assets (asset_uid);
CREATE INDEX IF NOT EXISTS idx_source_product_assets_family_date
ON source_product_assets (satellite_family, imaging_date);
CREATE INDEX IF NOT EXISTS idx_source_product_assets_satellite
ON source_product_assets (satellite);
CREATE INDEX IF NOT EXISTS idx_source_product_assets_source_format
ON source_product_assets (source_format);
CREATE INDEX IF NOT EXISTS idx_source_product_assets_parse_status
ON source_product_assets (parse_status);
CREATE INDEX IF NOT EXISTS idx_source_product_assets_root_active
ON source_product_assets (root_ref_id, is_active);
CREATE INDEX IF NOT EXISTS idx_source_product_assets_logical_product
ON source_product_assets (logical_product_uid);
CREATE INDEX IF NOT EXISTS idx_source_product_assets_file_path
ON source_product_assets (file_path);
CREATE INDEX IF NOT EXISTS idx_orbit_assets_orbit_uid
ON orbit_assets (orbit_uid);
CREATE INDEX IF NOT EXISTS idx_orbit_assets_family_sat_window
ON orbit_assets (satellite_family, satellite, validity_start_time_utc, validity_stop_time_utc);
CREATE INDEX IF NOT EXISTS idx_orbit_assets_orbit_type
ON orbit_assets (orbit_type);
CREATE INDEX IF NOT EXISTS idx_orbit_assets_native_format
ON orbit_assets (native_format);
CREATE INDEX IF NOT EXISTS idx_orbit_assets_quality_class
ON orbit_assets (quality_class);
CREATE INDEX IF NOT EXISTS idx_orbit_assets_parse_status
ON orbit_assets (parse_status);
CREATE INDEX IF NOT EXISTS idx_orbit_assets_root_active
ON orbit_assets (root_ref_id, is_active);
CREATE INDEX IF NOT EXISTS idx_orbit_assets_file_path
ON orbit_assets (file_path);
CREATE INDEX IF NOT EXISTS idx_scene_orbit_bindings_radar
ON scene_orbit_bindings (radar_data_id);
CREATE INDEX IF NOT EXISTS idx_scene_orbit_bindings_orbit
ON scene_orbit_bindings (orbit_asset_id);
CREATE INDEX IF NOT EXISTS idx_scene_orbit_bindings_match_status
ON scene_orbit_bindings (match_status);
CREATE INDEX IF NOT EXISTS idx_scene_orbit_bindings_selection_status
ON scene_orbit_bindings (selection_status);
CREATE INDEX IF NOT EXISTS idx_scene_orbit_bindings_scene_selected
ON scene_orbit_bindings (radar_data_id, selection_status);
CREATE INDEX IF NOT EXISTS idx_orbit_asset_derivatives_asset_engine
ON orbit_asset_derivatives (orbit_asset_id, engine_code);
CREATE INDEX IF NOT EXISTS idx_orbit_asset_derivatives_pool_path
ON orbit_asset_derivatives (pool_path);
CREATE INDEX IF NOT EXISTS idx_orbit_asset_derivatives_generation_status
ON orbit_asset_derivatives (generation_status);
CREATE INDEX IF NOT EXISTS idx_asset_inventory_states_type_status
ON asset_inventory_states (inventory_type, status);
CREATE INDEX IF NOT EXISTS idx_asset_inventory_states_needs_rescan
ON asset_inventory_states (needs_rescan);
CREATE INDEX IF NOT EXISTS idx_asset_inventory_issues_open
ON asset_inventory_issues (status, severity);
CREATE INDEX IF NOT EXISTS idx_asset_inventory_issues_root_type
ON asset_inventory_issues (root_ref_id, inventory_type);
CREATE INDEX IF NOT EXISTS idx_asset_inventory_issues_asset
ON asset_inventory_issues (asset_ref_id);
CREATE INDEX IF NOT EXISTS idx_asset_inventory_issues_orbit
ON asset_inventory_issues (orbit_asset_id);
CREATE INDEX IF NOT EXISTS idx_asset_inventory_issues_radar
ON asset_inventory_issues (radar_data_id);
UPDATE radar_data
SET
orbit_binding_status = CASE
WHEN has_orbit_data IS TRUE AND NULLIF(orbit_file_path, '') IS NOT NULL THEN 'MATCHED'
WHEN has_orbit_data IS TRUE THEN 'MATCHED'
ELSE COALESCE(NULLIF(orbit_binding_status, ''), 'UNBOUND')
END
WHERE orbit_binding_status IS NULL
OR orbit_binding_status = ''
OR (has_orbit_data IS TRUE AND orbit_binding_status = 'UNBOUND');
UPDATE radar_data
SET absolute_orbit = COALESCE(NULLIF(absolute_orbit, ''), NULLIF(orbit_circle, ''))
WHERE absolute_orbit IS NULL OR absolute_orbit = '';
UPDATE radar_data
SET source_format = COALESCE(
NULLIF(source_format, ''),
CASE
WHEN upper(replace(replace(replace(COALESCE(satellite, ''), '-', ''), '_', ''), ' ', '')) LIKE 'S1%%'
AND lower(COALESCE(file_path, '')) LIKE '%%.zip' THEN 'S1_ZIP'
WHEN upper(replace(replace(replace(COALESCE(satellite, ''), '-', ''), '_', ''), ' ', '')) LIKE 'S1%%'
AND lower(COALESCE(file_path, '')) LIKE '%%.safe' THEN 'S1_SAFE_DIR'
WHEN upper(replace(replace(replace(COALESCE(satellite, ''), '-', ''), '_', ''), ' ', '')) LIKE 'LT1%%' THEN 'LT1_DIR'
WHEN upper(replace(replace(replace(COALESCE(satellite, ''), '-', ''), '_', ''), ' ', '')) LIKE 'GF3%%' THEN 'GF3_DIR'
ELSE NULL
END
)
WHERE source_format IS NULL OR source_format = '';
+3 -1
View File
@@ -60,6 +60,8 @@
## 4. 配对与前端导航 ## 4. 配对与前端导航
- [SENTINEL1_SOURCE_ORBIT_ASSET_DESIGN_20260512.md](SENTINEL1_SOURCE_ORBIT_ASSET_DESIGN_20260512.md)
Sentinel-1 / LT-1 源数据与精密轨道资产层设计,定义统一源产品库存、轨道资产、scene 绑定、启动自维护和健康检查边界。
- [SENTINEL1_SYSTEM_ENHANCEMENT_MASTER_PLAN_20260510.md](SENTINEL1_SYSTEM_ENHANCEMENT_MASTER_PLAN_20260510.md) - [SENTINEL1_SYSTEM_ENHANCEMENT_MASTER_PLAN_20260510.md](SENTINEL1_SYSTEM_ENHANCEMENT_MASTER_PLAN_20260510.md)
Sentinel-1 系统增强主维护文档,汇总数据管理、精轨、配对、任务分发、Gamma/PyINT 生产、结果管理和分阶段实施边界。 Sentinel-1 系统增强主维护文档,汇总数据管理、精轨、配对、任务分发、Gamma/PyINT 生产、结果管理和分阶段实施边界。
- [SENTINEL1_DATA_MANAGEMENT_ADAPTATION_PLAN_20260510.md](SENTINEL1_DATA_MANAGEMENT_ADAPTATION_PLAN_20260510.md) - [SENTINEL1_DATA_MANAGEMENT_ADAPTATION_PLAN_20260510.md](SENTINEL1_DATA_MANAGEMENT_ADAPTATION_PLAN_20260510.md)
@@ -125,4 +127,4 @@
- 已归档的 `项目汇报.md` - 已归档的 `项目汇报.md`
- 已归档的各类 `*_EXPERIMENT_*` / `*_PROGRESS_*` / `*_TODO_*` - 已归档的各类 `*_EXPERIMENT_*` / `*_PROGRESS_*` / `*_TODO_*`
最后更新:2026-04-26 最后更新:2026-05-12
@@ -0,0 +1,579 @@
# Sentinel-1 / LT-1 源数据与精密轨道资产设计
日期:2026-05-12
本文定义源数据管理和精密轨道管理的下一阶段底座设计。目标是把 Sentinel-1 和 LT-1 放到同等位置和能力上,而不是把 Sentinel-1 临时塞进现有 LT-1 扫描和轨道字段里。
本文只覆盖源数据、精密轨道、资产库存、轨道绑定、启动自维护和健康检查。D-InSAR 配对、任务分发和生产 profile 不在本阶段实现范围内,但后续应消费这里建立的资产与绑定结果。
## 1. 目标
本阶段要完成:
1. 统一管理 LT-1、Sentinel-1 的源产品资产。
2. 统一管理 LT-1 原生精轨和 Sentinel-1 EOF 精轨。
3. 让 `radar_data` 成为统一 scene 业务入口,而不是唯一资产库存表。
4. 建立 scene 与精轨的候选、选中和异常状态。
5. 把新 schema 纳入启动自维护、数据库 schema 检查和健康检查。
6. 保持现有 LT-1、GF-3、水体、配对、结果 catalog 的兼容运行。
本阶段暂不做:
1. 不新增 Sentinel-1 D-InSAR 生产链路。
2. 不改造现有 `lt1_gamma_dinsar` 或 `lt1_stripmap` 为多卫星 profile。
3. 不在启动时扫描大目录或解析大型 ZIP。
4. 不把 Sentinel-1 EOF 送入 LT-1 txt 到 XML 转换链路。
## 2. 现有约束
当前系统启动链路中,`backend/app/main.py` 的 lifespan 会先调用 `ensure_database_ready()`,再初始化数据库会话、同步 managed roots、同步 manifest catalog、启动 pairing cache 状态。因此新资产层必须满足:
- ORM 是 schema 事实来源。新增表和字段必须进入 ORM。
- SQL migration 是约束、索引、幂等修补的事实来源。新增 migration 必须加入 `backend/app/db_maintenance.py` 的 `MIGRATION_FILES`。
- migration 必须可重复运行,使用 `CREATE TABLE IF NOT EXISTS`、`ALTER TABLE ... ADD COLUMN IF NOT EXISTS`、`CREATE INDEX IF NOT EXISTS`。
- 启动自维护只保证 schema 和轻量状态存在,不做 ZIP 解包、不扫大目录、不重建全量绑定。
- 健康检查必须能报告资产层状态,不能只看旧的 `MONITOR_RADAR_DIRS` 和 `MONITOR_ORBIT_DIR` 是否存在。
现有兼容字段需要保留:
- `radar_data.has_orbit_data`
- `radar_data.orbit_file_path`
- `radar_data.file_path`
- `radar_data.coverage_polygon`
- `radar_data.geom`
这些字段继续服务旧接口和旧前端,但由新的资产与绑定结果回填。
## 3. 领域模型
新的数据模型分成五类。
### 3.1 源产品资产
`source_product_assets` 记录物理源产品。一个资产是一份实际存在的文件或目录,例如:
- LT-1 scene 目录
- LT-1 原始压缩包或 tiff
- Sentinel-1 `.zip`
- Sentinel-1 `.SAFE` 目录
- GF-3 现有处理输入或输出目录,后续可接入同一库存
关键字段建议:
```text
id
asset_uid
logical_product_uid
satellite_family
satellite
source_format
product_type
product_level
imaging_mode
polarization
absolute_orbit
relative_orbit
orbit_direction
acquisition_start_time_utc
acquisition_stop_time_utc
imaging_date
root_path
file_path
archive_path
path_kind
file_name
file_stem
file_ext
size_bytes
mtime_epoch
checksum_sha256
checksum_status
parser_name
parser_version
parse_status
parse_error
parsed_at
metadata_json
is_active
missing_since
created_at
updated_at
```
字段语义:
- `asset_uid` 是物理资产唯一键,优先由规范化绝对路径和大小/mtime 派生。
- `logical_product_uid` 是同一个遥感产品的逻辑身份,用于把 Sentinel-1 ZIP 和解包后的 SAFE 关联起来。
- `source_format` 使用稳定枚举,例如 `LT1_DIR`、`LT1_ARCHIVE`、`S1_ZIP`、`S1_SAFE_DIR`、`GF3_DIR`。
- `archive_path` 用于记录生产更偏好的原始归档路径,例如 Sentinel-1 ZIP。
- `metadata_json` 保存传感器专有字段,不把所有 Sentinel-1 annotation 字段拆成列。
### 3.2 Scene 业务入口
`radar_data` 仍然是前端检索、地图展示、水体、配对和后续生产的 scene 入口。它需要补充资产引用和 Sentinel-1/LT-1 通用字段:
```text
source_product_ref_id
source_archive_asset_id
selected_orbit_asset_id
orbit_binding_status
orbit_binding_reason
acquisition_start_time_utc
acquisition_stop_time_utc
absolute_orbit
relative_orbit
source_format
metadata_json
```
字段语义:
- `source_product_ref_id` 指向用于生成该 scene 元数据的主资产。
- `source_archive_asset_id` 指向后续分发或生产优先使用的原始资产。Sentinel-1 通常是 ZIP。
- `selected_orbit_asset_id` 指向当前选中的精轨资产。
- `orbit_binding_status` 标识 `UNBOUND`、`MATCHED`、`MISSING`、`AMBIGUOUS`、`ERROR`。
- 旧字段 `has_orbit_data` 和 `orbit_file_path` 从 `selected_orbit_asset_id` 兼容回填。
### 3.3 精密轨道资产
`orbit_assets` 记录原生轨道文件。它不记录派生到 ENVI/ISCE2 池里的文件。
关键字段建议:
```text
id
orbit_uid
satellite_family
satellite
orbit_type
native_format
quality_class
root_path
file_path
file_name
file_stem
file_ext
size_bytes
mtime_epoch
checksum_sha256
checksum_status
validity_start_time_utc
validity_stop_time_utc
generation_time_utc
published_time_utc
parser_name
parser_version
parse_status
parse_error
parsed_at
metadata_json
is_active
missing_since
created_at
updated_at
```
枚举建议:
- `satellite_family`: `LT1`、`S1`、`GF3`
- `native_format`: `LT1_TXT`、`S1_EOF`
- `orbit_type`: `LT1_GPS`、`AUX_POEORB`、`AUX_RESORB`
- `quality_class`: `precise`、`restituted`、`predicted`、`unknown`
Sentinel-1 EOF 的匹配逻辑基于:
```text
orbit.satellite == scene.satellite
and orbit.validity_start_time_utc <= scene.acquisition_time <= orbit.validity_stop_time_utc
```
LT-1 第一阶段可继续复用现有日期匹配逻辑,后续如果轨道文件能提供明确有效期,也应收敛到相同的时间窗模型。
### 3.4 Scene 与轨道绑定
`scene_orbit_bindings` 记录每个 scene 的候选轨道和最终选择。
关键字段建议:
```text
id
radar_data_id
orbit_asset_id
binding_role
match_status
selection_status
selection_rank
priority_score
coverage_margin_before_seconds
coverage_margin_after_seconds
match_rule_version
match_reason
selected_at
metadata_json
created_at
updated_at
```
字段语义:
- `binding_role` 第一阶段使用 `primary_orbit`。
- `match_status`: `CANDIDATE`、`MATCHED`、`REJECTED`、`STALE`、`ERROR`。
- `selection_status`: `SELECTED`、`CANDIDATE`、`NOT_SELECTED`。
- Sentinel-1 选择优先级:POEORB 优先于 RESORB;覆盖余量更大优先;generation time 更新优先。
- LT-1 选择优先级:现有可用轨道优先;坏源精轨不得入选;派生池同步失败时保留资产但标注异常。
### 3.5 轨道派生产物
`orbit_asset_derivatives` 记录从原生轨道资产生成或同步到引擎池的文件。
关键字段建议:
```text
id
orbit_asset_id
engine_code
derivative_format
derivative_role
pool_path
size_bytes
mtime_epoch
checksum_sha256
generation_status
generation_error
generated_at
metadata_json
created_at
updated_at
```
用途:
- LT-1 txt 同步到 ENVI pool。
- LT-1 txt 转换成 ISCE2 XML。
- Sentinel-1 EOF 第一阶段通常不转换,但后续可以记录 staging 到 OPOD 目录的结果。
### 3.6 库存状态与问题
为避免健康检查每次扫大目录,新增轻量状态表:
`asset_inventory_states`
```text
id
root_ref_id
inventory_type
root_path
scan_mode
status
last_scan_started_at
last_scan_finished_at
last_seen_entry_count
last_asset_count
last_issue_count
parser_version
needs_rescan
last_error
metadata_json
created_at
updated_at
```
`asset_inventory_issues`
```text
id
root_ref_id
inventory_type
asset_ref_id
radar_data_id
orbit_asset_id
severity
issue_code
issue_message
source_path
status
first_seen_at
last_seen_at
resolved_at
metadata_json
```
典型 issue:
- `source_path_missing`
- `source_parse_failed`
- `orbit_parse_failed`
- `duplicate_logical_product`
- `scene_missing_orbit`
- `scene_ambiguous_orbit`
- `selected_orbit_missing_file`
- `lt1_derivative_generation_failed`
## 4. Root 与配置
建议新增通用配置,同时保留旧配置作为兼容入口:
```text
SOURCE_PRODUCT_DIRS=
ORBIT_SOURCE_DIRS=
```
兼容关系:
- `INSAR_STORAGE_DIRS` 和 `MONITOR_RADAR_DIRS` 继续生效,并作为 `SOURCE_PRODUCT_DIRS` 的兼容来源。
- `MONITOR_ORBIT_DIR` 继续生效,并作为 `ORBIT_SOURCE_DIRS` 的兼容来源。
- `ORBIT_POOL_ENVI`、`ORBIT_POOL_ISCE2` 仍然是派生轨道池,不作为原生轨道资产源。
`root_registry_service` 需要增加或调整 root role:
- `source_product_pool`
- `legacy_scan_root_radar`
- `orbit_asset_pool`
- `orbit_pool_envi`
- `orbit_pool_isce2`
每个 root 的 `metadata_json` 可以记录:
```json
{
"supported_families": ["LT1", "S1"],
"discovery_patterns": ["LT1*", "S1*.zip", "*.SAFE", "*.EOF"],
"imported_from": "settings"
}
```
对用户当前样本,推荐后续配置形态是:
```text
SOURCE_PRODUCT_DIRS=D:\LuTan1_Image_Pool;D:\Sentinel1_Image_Pool_ZIP
ORBIT_SOURCE_DIRS=D:\LT1_data_lsarorbit;D:\Sentinel1_EOF_Pool
```
## 5. Parser 设计
新增解析器按资产类型组织:
- `LT1SourceParser`
- `Sentinel1ZipParser`
- `Sentinel1SafeParser`
- `GF3SourceParser`
- `LT1OrbitParser`
- `Sentinel1EofParser`
Sentinel-1 ZIP 解析原则:
- 不全量解压。
- 使用 Python `zipfile` 读取 `manifest.safe` 和 annotation XML。
- 只读取必要 XML 文件和 quicklook 候选,不读取 measurement 大文件。
- 文件名提供粗字段,XML 提供 relative orbit、升降轨、footprint、swath、极化细节。
- `S1A`、`S1B`、`S1C` 都归一化为 `satellite_family = S1`。
Sentinel-1 EOF 解析原则:
- 优先从文件名解析 mission、orbit type、generation time、validity start、validity stop。
- 文件名不可靠时再读取 EOF XML 内容。
- `AUX_POEORB` 标为 precise,`AUX_RESORB` 标为 restituted。
LT-1 解析原则:
- 现有文件名和 XML 解析逻辑保留。
- 现有坏源精轨检测和 quarantine 逻辑保留,但结果写入 `orbit_assets` 和 `asset_inventory_issues`。
- 现有 `sync_orbit_pools()` 逐步下沉为 `orbit_asset_derivatives` 的生成步骤。
## 6. 扫描与绑定流程
### 6.1 源产品扫描
流程:
1. 从 managed roots 找到 source product roots。
2. 发现候选文件或目录。
3. upsert `source_product_assets`。
4. 调用对应 parser。
5. 生成或更新 `radar_data`。
6. 更新 `asset_inventory_states`。
7. 记录解析失败或重复产品到 `asset_inventory_issues`。
`radar_data.unique_id` 的生成应优先使用稳定逻辑身份:
```text
satellite_family + satellite + product_unique_id
```
如果缺少产品唯一号,再回退到规范化路径。
### 6.2 轨道扫描
流程:
1. 从 managed roots 找到 orbit asset roots。
2. 发现 LT-1 txt 和 Sentinel-1 EOF。
3. upsert `orbit_assets`。
4. 调用轨道 parser。
5. LT-1 轨道执行坏源检测。
6. 记录 parse status 和 inventory issues。
### 6.3 轨道绑定
流程:
1. 找出新增或变更的 scene 与 orbit asset。
2. 按卫星族调用绑定规则。
3. 写入候选 `scene_orbit_bindings`。
4. 选出 `selection_status = SELECTED` 的绑定。
5. 回填 `radar_data.selected_orbit_asset_id`、`orbit_binding_status`、`has_orbit_data`、`orbit_file_path`。
6. 对无轨道或多候选未决的 scene 写入 issue。
绑定规则版本需要显式记录,例如:
```text
lt1_orbit_binding.v1
s1_eof_window_binding.v1
```
## 7. 启动自维护
新增资产层后,启动自维护必须按以下方式运行:
1. `ensure_database_ready()` 创建 PostGIS 扩展。
2. ORM `create_all()` 创建新增表和新增列。
3. migration `010_source_orbit_asset_inventory.sql` 补齐索引、约束、兼容列、视图或必要函数。
4. `root_registry_service.sync_from_settings()` 同步新旧 root。
5. 启动阶段只初始化 `asset_inventory_states` 的空状态,不做真实扫描。
6. 健康检查报告 `needs_rescan = true`,由用户手动触发扫描任务。
注意事项:
- 不允许启动时遍历 `D:\Sentinel1_Image_Pool_ZIP` 这类大目录并打开 ZIP。
- 不允许启动时自动删除资产记录。文件消失时标记 `is_active = false` 和 `missing_since`。
- schema mismatch 必须在启动日志中可见,健康检查必须能展示缺表、缺列和资产库存异常。
## 8. 健康检查
`health_service` 增加 `asset_inventory` 检查项。
建议返回结构:
```json
{
"ok": true,
"source_roots": {
"configured_count": 2,
"accessible_count": 2,
"needs_rescan_count": 1
},
"orbit_roots": {
"configured_count": 2,
"accessible_count": 2,
"needs_rescan_count": 1
},
"source_assets": {
"total_count": 0,
"lt1_count": 0,
"s1_count": 0,
"parse_failed_count": 0
},
"orbit_assets": {
"total_count": 0,
"lt1_count": 0,
"s1_count": 0,
"parse_failed_count": 0
},
"bindings": {
"scene_count": 0,
"matched_count": 0,
"missing_count": 0,
"ambiguous_count": 0
},
"issues": {
"open_count": 0,
"error_count": 0,
"warning_count": 0
}
}
```
整体健康状态建议:
- schema 不完整:`ok = false`
- root 不可访问:`ok = false`
- 有 error 级 issue:`ok = false`
- 仅 `needs_rescan` 或 warning issue:`ok = true`,但显示 degraded/warning
## 9. API 与前端边界
本阶段后端需要提供:
- 源产品资产列表、详情、重扫。
- 轨道资产列表、详情、重扫。
- scene 的轨道绑定详情。
- 资产库存健康摘要。
- 手动触发源产品扫描、轨道扫描、轨道绑定重建。
前端数据管理页面需要显示:
- satellite family: LT1、S1、GF3
- source format: LT1_DIR、S1_ZIP、S1_SAFE_DIR
- relative orbit、absolute orbit
- acquisition start/stop
- orbit binding status
- selected orbit asset
- parser status 和 issue 摘要
轨道管理页面需要显示:
- 原生轨道资产列表。
- LT-1 与 Sentinel-1 的轨道类型。
- validity window。
- 派生产物状态。
- 绑定到多少 scene。
- 解析失败或缺文件问题。
## 10. 兼容与迁移策略
迁移后,现有业务继续读 `radar_data`。新资产层提供更完整的 provenance。
兼容规则:
- 旧扫描入口可以保留,但应逐步改为调用资产扫描服务。
- `radar_data.file_path` 继续保存主 scene 路径。
- `radar_data.orbit_file_path` 继续保存选中原生轨道路径。
- `radar_data.has_orbit_data` 继续表示是否存在选中轨道。
- 现有 pairing cache 可以暂时继续使用旧字段,后续再改为读取 `relative_orbit` 和资产绑定状态。
数据回填:
1. 对已有 `radar_data` 记录创建 `source_product_assets`。
2. 对已有 `orbit_file_path` 创建 `orbit_assets`。
3. 生成 `scene_orbit_bindings`。
4. 确认兼容字段和新绑定结果一致。
## 11. 验收标准
以当前样本池作为第一轮验收:
- `D:\Sentinel1_Image_Pool_ZIP` 中 29 个 Sentinel-1 ZIP 能登记为 `source_product_assets`。
- S1A 与 S1C 都归入 `satellite_family = S1`。
- 每个 ZIP 能生成或更新一条 `radar_data` scene。
- `D:\Sentinel1_EOF_Pool` 中 24 个 EOF 能登记为 `orbit_assets`。
- 29 个 Sentinel-1 scene 全部能通过 validity window 绑定到 EOF。
- `has_orbit_data = true` 和 `orbit_file_path` 能兼容回填。
- LT-1 现有源数据扫描和精轨同步不退化。
- 启动后数据库健康检查不出现 schema mismatch。
- 健康检查能展示源产品、轨道资产、绑定状态和 issue 统计。
## 12. 实施顺序
建议按以下顺序开工:
1. Schema 与 ORM:新增资产表、绑定表、状态表,扩展 `radar_data`。
2. Migration 与启动自维护:新增 `010_source_orbit_asset_inventory.sql`,加入 `MIGRATION_FILES`。
3. 健康检查:新增 `asset_inventory` 状态,不依赖真实扫描也能返回稳定结构。
4. Root registry:支持通用 `SOURCE_PRODUCT_DIRS`、`ORBIT_SOURCE_DIRS`,兼容旧配置。
5. Parser 与库存扫描:先实现 Sentinel-1 ZIP、Sentinel-1 EOF、LT-1 现有源数据和轨道。
6. 轨道绑定服务:实现 LT-1 日期绑定和 Sentinel-1 EOF 时间窗绑定。
7. 兼容回填:更新 `radar_data.has_orbit_data`、`orbit_file_path`。
8. API 与前端:展示资产、轨道、绑定和 issue。
9. 回归测试:用 Sentinel-1 样本池和现有 LT-1 数据池验证。
这一路径先把底座做稳,再接 D-InSAR 配对和 `s1_gamma_dinsar` 生产 profile。
+793 -501
View File
File diff suppressed because it is too large Load Diff
+3 -6
View File
@@ -11,7 +11,6 @@
}, },
"dependencies": { "dependencies": {
"axios": "^1.13.0", "axios": "^1.13.0",
"zustand": "^5.0.5",
"chart.js": "^4.5.0", "chart.js": "^4.5.0",
"chartjs-adapter-date-fns": "^3.0.0", "chartjs-adapter-date-fns": "^3.0.0",
"date-fns": "^4.1.0", "date-fns": "^4.1.0",
@@ -21,7 +20,8 @@
"react": "^19.1.1", "react": "^19.1.1",
"react-chartjs-2": "^5.3.0", "react-chartjs-2": "^5.3.0",
"react-dom": "^19.1.1", "react-dom": "^19.1.1",
"react-markdown": "^10.1.0" "react-markdown": "^10.1.0",
"zustand": "^5.0.5"
}, },
"devDependencies": { "devDependencies": {
"@eslint/js": "^9.36.0", "@eslint/js": "^9.36.0",
@@ -33,9 +33,6 @@
"eslint-plugin-react-hooks": "^5.2.0", "eslint-plugin-react-hooks": "^5.2.0",
"eslint-plugin-react-refresh": "^0.4.20", "eslint-plugin-react-refresh": "^0.4.20",
"globals": "^16.4.0", "globals": "^16.4.0",
"vite": "npm:rolldown-vite@7.1.12" "vite": "^7.1.12"
},
"overrides": {
"vite": "npm:rolldown-vite@7.1.12"
} }
} }
+231
View File
@@ -2400,6 +2400,237 @@ input[type="checkbox"] {
margin-top: 6px; margin-top: 6px;
} }
.asset-inventory-panel {
padding: 12px;
display: flex;
flex-direction: column;
gap: 12px;
min-width: 0;
}
.asset-toolbar {
display: flex;
justify-content: space-between;
gap: 10px;
align-items: flex-start;
}
.asset-toolbar h3 {
margin: 0;
font-size: 16px;
}
.asset-toolbar p {
margin: 4px 0 0;
color: var(--color-text-muted);
font-size: 12px;
}
.asset-actions {
display: flex;
flex-wrap: wrap;
justify-content: flex-end;
gap: 6px;
}
.asset-actions select,
.asset-actions button {
height: 30px;
font-size: 12px;
}
.asset-message {
border: 1px solid var(--color-border);
background: var(--color-accent-soft);
color: var(--color-accent-strong);
border-radius: 6px;
padding: 8px 10px;
font-size: 12px;
word-break: break-word;
}
.asset-message--error {
background: #fee2e2;
color: #991b1b;
border-color: #fecaca;
}
.asset-metrics {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
gap: 8px;
}
.asset-metric {
border: 1px solid var(--color-border);
border-radius: 8px;
padding: 8px;
background: #f8fafc;
}
.asset-metric span,
.asset-metric small {
display: block;
color: var(--color-text-muted);
font-size: 11px;
}
.asset-metric strong {
display: block;
margin: 2px 0;
font-size: 20px;
color: var(--color-text-primary);
}
.asset-root-strip {
display: flex;
flex-direction: column;
gap: 6px;
}
.asset-root-item {
display: grid;
grid-template-columns: minmax(0, 1fr) auto;
gap: 8px;
align-items: center;
border: 1px solid var(--color-border);
border-radius: 6px;
padding: 7px 8px;
background: #fff;
}
.asset-root-item strong,
.asset-root-item span {
display: block;
min-width: 0;
}
.asset-root-item span {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
color: var(--color-text-muted);
font-size: 11px;
}
.asset-tabbar {
display: flex;
gap: 6px;
}
.asset-tabbar button {
flex: 1;
font-size: 12px;
padding: 7px 6px;
}
.asset-tabbar button.active-tab {
background: var(--color-accent);
color: #fff;
border-color: var(--color-accent-strong);
}
.asset-table-wrap {
overflow-x: auto;
border: 1px solid var(--color-border);
border-radius: 8px;
background: #fff;
}
.asset-table {
width: 100%;
min-width: 760px;
border-collapse: collapse;
font-size: 12px;
}
.asset-table th,
.asset-table td {
border-bottom: 1px solid var(--color-border);
padding: 7px 8px;
text-align: left;
vertical-align: top;
}
.asset-table th {
background: #f8fafc;
color: var(--color-text-secondary);
font-weight: 700;
}
.asset-table td small {
display: block;
margin-top: 2px;
color: var(--color-text-muted);
max-width: 280px;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.asset-inline-action {
min-width: 46px;
height: 26px;
padding: 0 8px;
font-size: 12px;
line-height: 1;
}
.asset-action-placeholder {
color: var(--color-text-muted);
font-size: 12px;
}
.asset-badge {
display: inline-flex;
align-items: center;
max-width: 100%;
border-radius: 999px;
padding: 2px 7px;
font-size: 11px;
font-weight: 700;
border: 1px solid var(--color-border);
white-space: nowrap;
}
.asset-badge--ok {
background: #dcfce7;
color: #166534;
border-color: #bbf7d0;
}
.asset-badge--warn {
background: #fef3c7;
color: #92400e;
border-color: #fde68a;
}
.asset-badge--bad {
background: #fee2e2;
color: #991b1b;
border-color: #fecaca;
}
.asset-badge--neutral {
background: #f1f5f9;
color: #475569;
}
.asset-pager {
display: flex;
align-items: center;
justify-content: flex-end;
gap: 8px;
padding: 8px;
color: var(--color-text-muted);
font-size: 12px;
}
.asset-pager button {
padding: 5px 9px;
font-size: 12px;
}
@media (max-width: 1200px) { @media (max-width: 1200px) {
.top-status-bar { .top-status-bar {
grid-template-columns: 1fr; grid-template-columns: 1fr;
+270
View File
@@ -0,0 +1,270 @@
import { useCallback, useEffect, useMemo, useState } from 'react';
import {
getAssetInventoryStatus,
listAssetIssues,
listOrbitAssets,
listSourceAssets,
scanAssetInventory,
} from './api/assets';
const PAGE_SIZE = 100;
const fmtDateTime = (value) => {
if (!value) return '-';
const date = new Date(value);
if (Number.isNaN(date.getTime())) return String(value);
return date.toLocaleString();
};
const fmtBytes = (value) => {
const n = Number(value);
if (!Number.isFinite(n) || n <= 0) return '-';
if (n >= 1024 ** 3) return `${(n / (1024 ** 3)).toFixed(2)} GB`;
if (n >= 1024 ** 2) return `${(n / (1024 ** 2)).toFixed(1)} MB`;
if (n >= 1024) return `${(n / 1024).toFixed(1)} KB`;
return `${n} B`;
};
const StatusBadge = ({ value }) => {
const text = String(value || '-');
const status = text.toUpperCase();
const tone = status === 'OK' || status === 'MATCHED' || status === 'SELECTED'
? 'ok'
: status === 'WARNING' || status === 'OPEN' || status === 'MISSING'
? 'warn'
: status === 'FAILED' || status === 'INACCESSIBLE' || status === 'ERROR'
? 'bad'
: 'neutral';
return <span className={`asset-badge asset-badge--${tone}`}>{text}</span>;
};
const Metric = ({ label, value, hint }) => (
<div className="asset-metric">
<span>{label}</span>
<strong>{value ?? 0}</strong>
{hint ? <small>{hint}</small> : null}
</div>
);
export default function AssetInventoryPanel({ readOnly = false, onTaskStart }) {
const [status, setStatus] = useState(null);
const [sources, setSources] = useState({ items: [], total: 0, offset: 0, has_more: false });
const [orbits, setOrbits] = useState({ items: [], total: 0, offset: 0, has_more: false });
const [issues, setIssues] = useState({ items: [], total: 0, offset: 0, has_more: false });
const [activeTab, setActiveTab] = useState('sources');
const [family, setFamily] = useState('all');
const [loading, setLoading] = useState(false);
const [scanLoading, setScanLoading] = useState(false);
const [message, setMessage] = useState('');
const [error, setError] = useState('');
const familyParam = useMemo(() => (family === 'all' ? undefined : family), [family]);
const refresh = useCallback(async ({ sourceOffset = 0, orbitOffset = 0, issueOffset = 0 } = {}) => {
setLoading(true);
setError('');
try {
const [nextStatus, nextSources, nextOrbits, nextIssues] = await Promise.all([
getAssetInventoryStatus(),
listSourceAssets({ satellite_family: familyParam, limit: PAGE_SIZE, offset: sourceOffset }),
listOrbitAssets({ satellite_family: familyParam, limit: PAGE_SIZE, offset: orbitOffset }),
listAssetIssues({ status: 'OPEN', limit: PAGE_SIZE, offset: issueOffset }),
]);
setStatus(nextStatus);
setSources(nextSources);
setOrbits(nextOrbits);
setIssues(nextIssues);
} catch (err) {
setError(err?.response?.data?.detail || err.message || '加载资产库存失败');
} finally {
setLoading(false);
}
}, [familyParam]);
useEffect(() => {
refresh();
}, [refresh]);
const handleScan = async () => {
if (readOnly || scanLoading) return;
setScanLoading(true);
setMessage('');
setError('');
try {
const result = await scanAssetInventory({ inventory_types: [], root_ids: [], bind_orbits: true });
setMessage(`资产扫描任务已入队: ${result.task_id}`);
onTaskStart?.(result.task_id, '源数据/精轨资产扫描已入队', {
taskType: 'SCAN_ASSET_INVENTORY',
nonBlocking: true,
});
} catch (err) {
setError(err?.response?.data?.detail || err.message || '启动资产扫描失败');
} finally {
setScanLoading(false);
}
};
const states = status?.states || [];
const sourceRoots = states.filter(item => item.inventory_type === 'source_product');
const orbitRoots = states.filter(item => item.inventory_type === 'orbit_asset');
const renderPager = (data, onPage) => (
<div className="asset-pager">
<button type="button" disabled={data.offset <= 0 || loading} onClick={() => onPage(Math.max(0, data.offset - PAGE_SIZE))}>
上一页
</button>
<span>{data.offset + 1}-{data.offset + data.items.length} / {data.total}</span>
<button type="button" disabled={!data.has_more || loading} onClick={() => onPage(data.offset + PAGE_SIZE)}>
下一页
</button>
</div>
);
return (
<div className="asset-inventory-panel">
<div className="asset-toolbar">
<div>
<h3>源数据与精轨资产</h3>
<p>Sentinel-1 与 LT-1 的源产品、精密轨道和绑定状态</p>
</div>
<div className="asset-actions">
<select value={family} onChange={(e) => setFamily(e.target.value)} disabled={loading}>
<option value="all">全部卫星族</option>
<option value="S1">Sentinel-1</option>
<option value="LT1">LT-1</option>
</select>
<button type="button" onClick={() => refresh()} disabled={loading}>刷新</button>
<button type="button" onClick={handleScan} disabled={readOnly || scanLoading}>扫描资产</button>
</div>
</div>
{error ? <div className="asset-message asset-message--error">{error}</div> : null}
{message ? <div className="asset-message">{message}</div> : null}
<div className="asset-metrics">
<Metric label="源产品" value={status?.source_asset_count} hint={`${sourceRoots.length} 个源数据根`} />
<Metric label="精轨资产" value={status?.orbit_asset_count} hint={`${orbitRoots.length} 个精轨根`} />
<Metric label="已绑定场景" value={status?.selected_binding_count} />
<Metric label="开放问题" value={status?.open_issue_count} />
</div>
<div className="asset-root-strip">
{states.map((item) => (
<div className="asset-root-item" key={`${item.inventory_type}-${item.root_ref_id}`}>
<div>
<strong>{item.inventory_type === 'source_product' ? '源数据池' : '精轨池'}</strong>
<span title={item.root_path}>{item.root_path}</span>
</div>
<StatusBadge value={item.status} />
</div>
))}
</div>
<div className="asset-tabbar">
<button type="button" className={activeTab === 'sources' ? 'active-tab' : ''} onClick={() => setActiveTab('sources')}>
源产品 ({sources.total})
</button>
<button type="button" className={activeTab === 'orbits' ? 'active-tab' : ''} onClick={() => setActiveTab('orbits')}>
精轨 ({orbits.total})
</button>
<button type="button" className={activeTab === 'issues' ? 'active-tab' : ''} onClick={() => setActiveTab('issues')}>
问题 ({issues.total})
</button>
</div>
{activeTab === 'sources' && (
<div className="asset-table-wrap">
<table className="asset-table">
<thead>
<tr>
<th>卫星</th>
<th>日期/时间</th>
<th>产品</th>
<th>轨道</th>
<th>状态</th>
<th>动作</th>
<th>文件</th>
</tr>
</thead>
<tbody>
{sources.items.map(item => {
return (
<tr key={item.id}>
<td><strong>{item.satellite}</strong><small>{item.satellite_family}</small></td>
<td>{item.imaging_date}<small>{fmtDateTime(item.acquisition_start_time_utc)}</small></td>
<td>{item.source_format}<small>{item.imaging_mode} / {item.polarization}</small></td>
<td>{item.relative_orbit || '-'}<small>abs {item.absolute_orbit || '-'}</small></td>
<td><StatusBadge value={item.parse_status} /></td>
<td>
<span className="asset-action-placeholder">-</span>
</td>
<td title={item.file_path}>{item.file_name || item.logical_product_uid}<small>{fmtBytes(item.size_bytes)}</small></td>
</tr>
);
})}
</tbody>
</table>
{renderPager(sources, (offset) => refresh({ sourceOffset: offset, orbitOffset: orbits.offset, issueOffset: issues.offset }))}
</div>
)}
{activeTab === 'orbits' && (
<div className="asset-table-wrap">
<table className="asset-table">
<thead>
<tr>
<th>卫星</th>
<th>类型</th>
<th>有效期</th>
<th>质量</th>
<th>状态</th>
<th>文件</th>
</tr>
</thead>
<tbody>
{orbits.items.map(item => (
<tr key={item.id}>
<td><strong>{item.satellite}</strong><small>{item.satellite_family}</small></td>
<td>{item.orbit_type}<small>{item.native_format}</small></td>
<td>{fmtDateTime(item.validity_start_time_utc)}<small>{fmtDateTime(item.validity_stop_time_utc)}</small></td>
<td>{item.quality_class}</td>
<td><StatusBadge value={item.parse_status} /></td>
<td title={item.file_path}>{item.file_name}<small>{fmtBytes(item.size_bytes)}</small></td>
</tr>
))}
</tbody>
</table>
{renderPager(orbits, (offset) => refresh({ sourceOffset: sources.offset, orbitOffset: offset, issueOffset: issues.offset }))}
</div>
)}
{activeTab === 'issues' && (
<div className="asset-table-wrap">
<table className="asset-table">
<thead>
<tr>
<th>级别</th>
<th>代码</th>
<th>对象</th>
<th>说明</th>
<th>时间</th>
</tr>
</thead>
<tbody>
{issues.items.map(item => (
<tr key={item.id}>
<td><StatusBadge value={item.severity} /></td>
<td>{item.issue_code}</td>
<td>{item.inventory_type}<small>{item.source_path || `radar ${item.radar_data_id || '-'}`}</small></td>
<td>{item.issue_message || '-'}</td>
<td>{fmtDateTime(item.last_seen_at)}</td>
</tr>
))}
</tbody>
</table>
{renderPager(issues, (offset) => refresh({ sourceOffset: sources.offset, orbitOffset: orbits.offset, issueOffset: offset }))}
</div>
)}
</div>
);
}
+267 -101
View File
@@ -1,6 +1,7 @@
import React, { useEffect, useRef, useState } from 'react'; import React, { useEffect, useRef, useState } from 'react';
import './App.css'; import './App.css';
import { useI18n } from './i18n/I18nContext'; import { useI18n } from './i18n/I18nContext';
import { getAssetInventoryStatus, scanAssetInventory, unpackSentinel1Batch } from './api/assets';
const DEFAULT_MONITOR_CONFIG = { const DEFAULT_MONITOR_CONFIG = {
radar_dirs: [], radar_dirs: [],
@@ -8,6 +9,9 @@ const DEFAULT_MONITOR_CONFIG = {
dinsar_dirs: [], dinsar_dirs: [],
gf3_source_dirs: [], gf3_source_dirs: [],
gf3_storage_dirs: [], gf3_storage_dirs: [],
s1_source_dirs: [],
s1_storage_dirs: [],
s1_orbit_dirs: [],
}; };
const DEFAULT_UNPACK_CONFIG = { const DEFAULT_UNPACK_CONFIG = {
@@ -42,6 +46,7 @@ const parseUnpackRunValue = (rawValue, label) => {
}; };
const formatList = (list) => (Array.isArray(list) && list.length ? list.join('; ') : '未配置'); const formatList = (list) => (Array.isArray(list) && list.length ? list.join('; ') : '未配置');
const normalizeComparePath = (value) => String(value || '').trim().replace(/\\/g, '/').replace(/\/+$/, '').toLowerCase();
const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled = true }) => { const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled = true }) => {
const { t } = useI18n(); const { t } = useI18n();
@@ -50,8 +55,6 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
const [logs, setLogs] = useState([]); const [logs, setLogs] = useState([]);
const [activeTasks, setActiveTasks] = useState([]); const [activeTasks, setActiveTasks] = useState([]);
const [unpackConfig, setUnpackConfig] = useState(DEFAULT_UNPACK_CONFIG); const [unpackConfig, setUnpackConfig] = useState(DEFAULT_UNPACK_CONFIG);
const [loading, setLoading] = useState(false);
const [message, setMessage] = useState('');
const [unpackLoading, setUnpackLoading] = useState(false); const [unpackLoading, setUnpackLoading] = useState(false);
const [unpackMessage, setUnpackMessage] = useState(''); const [unpackMessage, setUnpackMessage] = useState('');
const [showUnpackDialog, setShowUnpackDialog] = useState(false); const [showUnpackDialog, setShowUnpackDialog] = useState(false);
@@ -59,7 +62,11 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
const [unpackDialogError, setUnpackDialogError] = useState(''); const [unpackDialogError, setUnpackDialogError] = useState('');
const [unpackTaskId, setUnpackTaskId] = useState(''); const [unpackTaskId, setUnpackTaskId] = useState('');
const [unpackTaskTerminal, setUnpackTaskTerminal] = useState(false); const [unpackTaskTerminal, setUnpackTaskTerminal] = useState(false);
const [s1Loading, setS1Loading] = useState(false);
const [s1ScanLoading, setS1ScanLoading] = useState(false);
const [s1Message, setS1Message] = useState('');
const [gf3Loading, setGf3Loading] = useState(false); const [gf3Loading, setGf3Loading] = useState(false);
const [gf3ProcessLoading, setGf3ProcessLoading] = useState(false);
const [gf3Message, setGf3Message] = useState(''); const [gf3Message, setGf3Message] = useState('');
const logEndRef = useRef(null); const logEndRef = useRef(null);
@@ -76,6 +83,7 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
const unpackActiveTask = displayActiveTasks.find((task) => const unpackActiveTask = displayActiveTasks.find((task) =>
task.task_id === unpackTaskId || task.task_type === 'UNPACK_ARCHIVES' task.task_id === unpackTaskId || task.task_type === 'UNPACK_ARCHIVES'
); );
const s1ActiveTask = displayActiveTasks.find((task) => task.task_type === 'UNPACK_SENTINEL1');
useEffect(() => { useEffect(() => {
if (!enabled) { if (!enabled) {
@@ -102,6 +110,9 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
...data, ...data,
radar_dirs: toArray(data?.radar_dirs), radar_dirs: toArray(data?.radar_dirs),
dinsar_dirs: toArray(data?.dinsar_dirs), dinsar_dirs: toArray(data?.dinsar_dirs),
s1_source_dirs: toArray(data?.s1_source_dirs),
s1_storage_dirs: toArray(data?.s1_storage_dirs),
s1_orbit_dirs: toArray(data?.s1_orbit_dirs),
gf3_source_dirs: toArray(data?.gf3_source_dirs), gf3_source_dirs: toArray(data?.gf3_source_dirs),
gf3_storage_dirs: toArray(data?.gf3_storage_dirs), gf3_storage_dirs: toArray(data?.gf3_storage_dirs),
}); });
@@ -267,51 +278,134 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
const hasRadarDirs = config.radar_dirs.length > 0; const hasRadarDirs = config.radar_dirs.length > 0;
const hasOrbitDir = typeof config.orbit_dir === 'string' && config.orbit_dir.trim() !== ''; const hasOrbitDir = typeof config.orbit_dir === 'string' && config.orbit_dir.trim() !== '';
const hasDinsarDirs = config.dinsar_dirs.length > 0; const hasS1SourceDirs = config.s1_source_dirs.length > 0;
const hasS1StorageDirs = config.s1_storage_dirs.length > 0;
const hasS1OrbitDirs = config.s1_orbit_dirs.length > 0;
const hasGf3SourceDirs = config.gf3_source_dirs.length > 0; const hasGf3SourceDirs = config.gf3_source_dirs.length > 0;
const hasGf3StorageDirs = config.gf3_storage_dirs.length > 0; const hasGf3StorageDirs = config.gf3_storage_dirs.length > 0;
const canRunRadar = !readOnly && configLoaded && hasRadarDirs; const canRunRadar = !readOnly && configLoaded && hasRadarDirs;
const canRunOrbit = !readOnly && configLoaded && hasOrbitDir; const canRunOrbit = !readOnly && configLoaded && hasOrbitDir;
const canRunDinsar = !readOnly && configLoaded && hasDinsarDirs; const canRunS1 = !readOnly && configLoaded && (hasS1SourceDirs || hasS1StorageDirs || hasS1OrbitDirs);
const canRunS1Scan = !readOnly && configLoaded && hasS1SourceDirs;
const canRunS1OrbitScan = !readOnly && configLoaded && hasS1OrbitDirs;
const canRunGf3Scan = !readOnly && configLoaded && hasGf3StorageDirs; const canRunGf3Scan = !readOnly && configLoaded && hasGf3StorageDirs;
const canRunGf3Process = !readOnly && configLoaded && hasGf3SourceDirs; const canRunGf3Process = !readOnly && configLoaded && hasGf3SourceDirs;
const canOpenUnpackDialog = !readOnly && unpackConfig.source_dirs.length > 0; const canOpenUnpackDialog = !readOnly && unpackConfig.source_dirs.length > 0;
const handleRunNow = async (target) => { const handleS1Run = async () => {
if (readOnly) { if (readOnly) {
setMessage('当前账户为只读模式,无法触发扫描。'); setS1Message('当前账户为只读模式,无法触发 Sentinel-1 任务。');
return; return;
} }
setS1Loading(true);
setLoading(true); setS1Message('Sentinel-1 任务启动中...');
const targetMap = {
radar: 'LT-1 数据',
orbit: '精轨数据',
dinsar: 'D-InSAR 结果',
gf3: 'GF3 数据',
};
setMessage(`正在触发${targetMap[target] || '全部'}手动扫描...`);
try { try {
const url = target ? `${apiEndpoint}/monitor/run-now?target=${target}` : `${apiEndpoint}/monitor/run-now`; const res = await unpackSentinel1Batch({
const res = await fetch(url, { scan_before_unpack: true,
overwrite: false,
});
setS1Message(res.message || 'Sentinel-1 任务已启动');
if (onTaskStart) {
onTaskStart(res.task_id, 'Sentinel-1 任务已启动。', {
nonBlocking: true,
taskType: 'UNPACK_SENTINEL1',
});
}
} catch (err) {
setS1Message(`失败:${err?.response?.data?.detail || err.message || '未知错误'}`);
} finally {
setS1Loading(false);
}
};
const handleS1Scan = async () => {
if (readOnly) {
setS1Message('当前账户为只读模式,无法触发 Sentinel-1 扫描。');
return;
}
setS1ScanLoading(true);
setS1Message('Sentinel-1 源数据扫描启动中...');
try {
const inventoryStatus = await getAssetInventoryStatus();
const sourcePathSet = new Set(config.s1_source_dirs.map(normalizeComparePath));
const rootIds = (inventoryStatus?.states || [])
.filter((item) => item?.inventory_type === 'source_product' && sourcePathSet.has(normalizeComparePath(item?.root_path)))
.map((item) => item.root_ref_id)
.filter((value, index, array) => value && array.indexOf(value) === index);
const res = await scanAssetInventory({
inventory_types: ['source_product'],
root_ids: rootIds,
bind_orbits: true,
});
setS1Message(res.message || 'Sentinel-1 源数据扫描任务已启动');
onTaskStart?.(res.task_id, 'Sentinel-1 源数据扫描任务已启动。', {
nonBlocking: true,
taskType: 'SCAN_ASSET_INVENTORY',
});
} catch (err) {
setS1Message(`失败:${err?.response?.data?.detail || err.message || '未知错误'}`);
} finally {
setS1ScanLoading(false);
}
};
const handleS1OrbitScan = async () => {
if (readOnly) {
setS1Message('当前账户为只读模式,无法触发 Sentinel-1 精轨扫描。');
return;
}
setS1ScanLoading(true);
setS1Message('Sentinel-1 精轨扫描启动中...');
try {
const inventoryStatus = await getAssetInventoryStatus();
const orbitPathSet = new Set(config.s1_orbit_dirs.map(normalizeComparePath));
const rootIds = (inventoryStatus?.states || [])
.filter((item) => item?.inventory_type === 'orbit_asset' && orbitPathSet.has(normalizeComparePath(item?.root_path)))
.map((item) => item.root_ref_id)
.filter((value, index, array) => value && array.indexOf(value) === index);
const res = await scanAssetInventory({
inventory_types: ['orbit_asset'],
root_ids: rootIds,
bind_orbits: true,
});
setS1Message(res.message || 'Sentinel-1 精轨扫描任务已启动');
onTaskStart?.(res.task_id, 'Sentinel-1 精轨扫描任务已启动。', {
nonBlocking: true,
taskType: 'SCAN_ASSET_INVENTORY',
});
} catch (err) {
setS1Message(`失败:${err?.response?.data?.detail || err.message || '未知错误'}`);
} finally {
setS1ScanLoading(false);
}
};
const handleGf3BatchProcess = async () => {
if (readOnly) {
setGf3Message('当前账户为只读模式,无法触发 GF3 预处理。');
return;
}
setGf3ProcessLoading(true);
setGf3Message('GF3 预处理启动中...');
try {
const res = await fetch(`${apiEndpoint}/monitor/gf3-process`, {
method: 'POST', method: 'POST',
credentials: 'include', credentials: 'include',
}); });
const data = await parseJsonSafe(res, {}); const data = await parseJsonSafe(res, {});
if (res.ok) { if (res.ok) {
setMessage(data.message || '扫描任务已启动。'); setGf3Message(data.message || 'GF3 批量处理任务已启动');
if (onTaskStart) { if (onTaskStart) {
onTaskStart(data.task_id, `已触发${targetMap[target] || '全部'}手动扫描...`); onTaskStart(data.task_id, 'GF3 L1A→L2 批量处理已启动。');
} }
} else { } else {
setMessage(`触发失败: ${data.detail || '未知错误'}`); setGf3Message(`失败:${data.detail || '未知错误'}`);
} }
} catch (err) { } catch (err) {
setMessage(`触发失败: ${err.message}`); setGf3Message(`失败:${err.message || '未知错误'}`);
} finally { } finally {
setLoading(false); setGf3ProcessLoading(false);
} }
}; };
@@ -397,29 +491,79 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
} }
}; };
const handleGf3BatchProcess = async () => { const handleRadarScan = async () => {
if (readOnly) { if (readOnly) {
setGf3Message('当前账户为只读模式,无法触发 GF3 处理。'); setUnpackMessage('当前账户为只读模式,无法触发扫描。');
return; return;
} }
setGf3Loading(true); setUnpackMessage('LT-1 扫描启动中...');
setGf3Message('GF3 批量处理启动中...');
try { try {
const res = await fetch(`${apiEndpoint}/monitor/gf3-process`, { const res = await fetch(`${apiEndpoint}/monitor/run-now?target=radar`, {
method: 'POST', method: 'POST',
credentials: 'include', credentials: 'include',
}); });
const data = await parseJsonSafe(res, {}); const data = await parseJsonSafe(res, {});
if (res.ok) { if (res.ok) {
setGf3Message(data.message || 'GF3 批量处理任务已启动'); setUnpackMessage(data.message || 'LT-1 扫描任务已启动');
if (onTaskStart) { if (onTaskStart) {
onTaskStart(data.task_id, 'GF3 L1A→L2 批量处理已启动。'); onTaskStart(data.task_id, '已触发 LT-1 手动扫描...');
}
} else {
setUnpackMessage(`失败:${data.detail || '未知错误'}`);
}
} catch (err) {
setUnpackMessage(`失败:${err.message || '未知错误'}`);
}
};
const handleOrbitScan = async () => {
if (readOnly) {
setUnpackMessage('当前账户为只读模式,无法触发扫描。');
return;
}
setUnpackMessage('精轨扫描启动中...');
try {
const res = await fetch(`${apiEndpoint}/monitor/run-now?target=orbit`, {
method: 'POST',
credentials: 'include',
});
const data = await parseJsonSafe(res, {});
if (res.ok) {
setUnpackMessage(data.message || '精轨扫描任务已启动');
if (onTaskStart) {
onTaskStart(data.task_id, '已触发精轨手动扫描...');
}
} else {
setUnpackMessage(`失败:${data.detail || '未知错误'}`);
}
} catch (err) {
setUnpackMessage(`失败:${err.message || '未知错误'}`);
}
};
const handleGf3Scan = async () => {
if (readOnly) {
setGf3Message('当前账户为只读模式,无法触发扫描。');
return;
}
setGf3Loading(true);
setGf3Message('GF3 扫描启动中...');
try {
const res = await fetch(`${apiEndpoint}/monitor/run-now?target=gf3`, {
method: 'POST',
credentials: 'include',
});
const data = await parseJsonSafe(res, {});
if (res.ok) {
setGf3Message(data.message || 'GF3 扫描任务已启动');
if (onTaskStart) {
onTaskStart(data.task_id, '已触发 GF3 手动扫描...');
} }
} else { } else {
setGf3Message(`失败:${data.detail || '未知错误'}`); setGf3Message(`失败:${data.detail || '未知错误'}`);
} }
} catch (err) { } catch (err) {
setGf3Message(`失败:${err.message}`); setGf3Message(`失败:${err.message || '未知错误'}`);
} finally { } finally {
setGf3Loading(false); setGf3Loading(false);
} }
@@ -437,17 +581,6 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
const rowStyle = { display: 'flex', gap: '8px' }; const rowStyle = { display: 'flex', gap: '8px' };
const gridStyle = { display: 'grid', rowGap: '6px', fontSize: '0.9em', color: 'var(--color-text-secondary)' }; const gridStyle = { display: 'grid', rowGap: '6px', fontSize: '0.9em', color: 'var(--color-text-secondary)' };
const scanBtnStyle = (canRun) => ({
flex: 1,
padding: '8px 5px',
backgroundColor: 'var(--color-info)',
color: 'white',
border: 'none',
borderRadius: '4px',
cursor: loading || !canRun ? 'not-allowed' : 'pointer',
fontSize: '0.85em',
});
const actionBtnStyle = (isLoading, isDisabled) => ({ const actionBtnStyle = (isLoading, isDisabled) => ({
padding: '6px 10px', padding: '6px 10px',
backgroundColor: 'var(--color-accent)', backgroundColor: 'var(--color-accent)',
@@ -495,6 +628,9 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
<div style={gridStyle}> <div style={gridStyle}>
<div style={rowStyle}><span style={labelStyle}>LT-1 存储</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.radar_dirs)}</span></div> <div style={rowStyle}><span style={labelStyle}>LT-1 存储</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.radar_dirs)}</span></div>
<div style={rowStyle}><span style={labelStyle}>LT-1 精轨</span><span style={{ wordBreak: 'break-all' }}>{config.orbit_dir || '未配置'}</span></div> <div style={rowStyle}><span style={labelStyle}>LT-1 精轨</span><span style={{ wordBreak: 'break-all' }}>{config.orbit_dir || '未配置'}</span></div>
<div style={rowStyle}><span style={labelStyle}>S1 源数据</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.s1_source_dirs)}</span></div>
<div style={rowStyle}><span style={labelStyle}>S1 存储</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.s1_storage_dirs)}</span></div>
<div style={rowStyle}><span style={labelStyle}>S1 精轨</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.s1_orbit_dirs)}</span></div>
<div style={rowStyle}><span style={labelStyle}>GF3 来源</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.gf3_source_dirs)}</span></div> <div style={rowStyle}><span style={labelStyle}>GF3 来源</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.gf3_source_dirs)}</span></div>
<div style={rowStyle}><span style={labelStyle}>GF3 存储</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.gf3_storage_dirs)}</span></div> <div style={rowStyle}><span style={labelStyle}>GF3 存储</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.gf3_storage_dirs)}</span></div>
<div style={rowStyle}><span style={labelStyle}>D-InSAR 结果</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.dinsar_dirs)}</span></div> <div style={rowStyle}><span style={labelStyle}>D-InSAR 结果</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.dinsar_dirs)}</span></div>
@@ -502,14 +638,14 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
</div> </div>
<div style={sectionStyle}> <div style={sectionStyle}>
<div style={{ fontWeight: 'bold', marginBottom: '8px', color: 'var(--color-text-primary)' }}>LT-1 归档解包</div> <div style={{ fontWeight: 'bold', marginBottom: '8px', color: 'var(--color-text-primary)' }}>LT-1 归档解包 / 扫描</div>
<div style={{ ...gridStyle, marginBottom: '8px' }}> <div style={{ ...gridStyle, marginBottom: '8px' }}>
<div style={rowStyle}><span style={labelStyle}>来源目录</span><span style={{ wordBreak: 'break-all' }}>{formatList(unpackConfig.source_dirs)}</span></div> <div style={rowStyle}><span style={labelStyle}>来源目录</span><span style={{ wordBreak: 'break-all' }}>{formatList(unpackConfig.source_dirs)}</span></div>
<div style={rowStyle}><span style={labelStyle}>LT-1 存储</span><span style={{ wordBreak: 'break-all' }}>{formatList(unpackConfig.insar_storage_dirs)}</span></div> <div style={rowStyle}><span style={labelStyle}>LT-1 存储</span><span style={{ wordBreak: 'break-all' }}>{formatList(unpackConfig.insar_storage_dirs)}</span></div>
<div style={rowStyle}><span style={labelStyle}>单次上限</span><span>{unpackConfig.max_files_per_run > 0 ? `${unpackConfig.max_files_per_run} 个压缩包` : '不限'}</span></div> <div style={rowStyle}><span style={labelStyle}>单次上限</span><span>{unpackConfig.max_files_per_run > 0 ? `${unpackConfig.max_files_per_run} 个压缩包` : '不限'}</span></div>
<div style={rowStyle}><span style={labelStyle}>最长运行</span><span>{unpackConfig.max_runtime_minutes > 0 ? `${unpackConfig.max_runtime_minutes} 分钟` : '不限'}</span></div> <div style={rowStyle}><span style={labelStyle}>最长运行</span><span>{unpackConfig.max_runtime_minutes > 0 ? `${unpackConfig.max_runtime_minutes} 分钟` : '不限'}</span></div>
</div> </div>
<div style={{ display: 'flex', gap: '10px' }}> <div style={{ display: 'flex', gap: '10px', flexWrap: 'wrap' }}>
<button <button
onClick={handleOpenUnpackDialog} onClick={handleOpenUnpackDialog}
disabled={unpackLoading || !canOpenUnpackDialog} disabled={unpackLoading || !canOpenUnpackDialog}
@@ -517,6 +653,20 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
> >
{unpackLoading ? '运行中...' : (readOnly ? '只读模式' : 'LT-1 解包')} {unpackLoading ? '运行中...' : (readOnly ? '只读模式' : 'LT-1 解包')}
</button> </button>
<button
onClick={handleRadarScan}
disabled={readOnly || !canRunRadar}
style={actionBtnStyle(false, !canRunRadar)}
>
{readOnly ? '只读模式' : '扫描 LT-1'}
</button>
<button
onClick={handleOrbitScan}
disabled={readOnly || !canRunOrbit}
style={actionBtnStyle(false, !canRunOrbit)}
>
{readOnly ? '只读模式' : '扫描精轨'}
</button>
<div <div
style={{ style={{
fontSize: '0.85em', fontSize: '0.85em',
@@ -530,7 +680,48 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
</div> </div>
<div style={sectionStyle}> <div style={sectionStyle}>
<div style={{ fontWeight: 'bold', marginBottom: '8px', color: 'var(--color-text-primary)' }}>GF3 L1A → L2 处理</div> <div style={{ fontWeight: 'bold', marginBottom: '8px', color: 'var(--color-text-primary)' }}>Sentinel-1 解包 / 扫描</div>
<div style={{ ...gridStyle, marginBottom: '8px' }}>
<div style={rowStyle}><span style={labelStyle}>S1 源数据</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.s1_source_dirs)}</span></div>
<div style={rowStyle}><span style={labelStyle}>S1 存储</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.s1_storage_dirs)}</span></div>
<div style={rowStyle}><span style={labelStyle}>S1 精轨</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.s1_orbit_dirs)}</span></div>
</div>
<div style={{ display: 'flex', gap: '10px' }}>
<button
onClick={handleS1Run}
disabled={s1Loading || s1ScanLoading || !canRunS1}
style={actionBtnStyle(s1Loading, !canRunS1)}
>
{s1Loading ? '运行中...' : (readOnly ? '只读模式' : 'Sentinel-1 解包')}
</button>
<button
onClick={handleS1Scan}
disabled={s1ScanLoading || s1Loading || !canRunS1Scan}
style={actionBtnStyle(s1ScanLoading, !canRunS1Scan)}
>
{s1ScanLoading ? '运行中...' : (readOnly ? '只读模式' : '扫描 S1 源数据')}
</button>
<button
onClick={handleS1OrbitScan}
disabled={s1ScanLoading || s1Loading || !canRunS1OrbitScan}
style={actionBtnStyle(s1ScanLoading, !canRunS1OrbitScan)}
>
{s1ScanLoading ? '运行中...' : (readOnly ? '只读模式' : '扫描 S1 精轨')}
</button>
<div
style={{
fontSize: '0.85em',
color: s1Message.includes('失败') ? 'var(--color-danger)' : 'var(--color-text-muted)',
alignSelf: 'center',
}}
>
{s1ActiveTask ? (s1ActiveTask.message || 'Sentinel-1 任务运行中...') : s1Message}
</div>
</div>
</div>
<div style={sectionStyle}>
<div style={{ fontWeight: 'bold', marginBottom: '8px', color: 'var(--color-text-primary)' }}>GF3 归档预处理</div>
<div style={{ ...gridStyle, marginBottom: '8px' }}> <div style={{ ...gridStyle, marginBottom: '8px' }}>
<div style={rowStyle}><span style={labelStyle}>L1A 来源</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.gf3_source_dirs)}</span></div> <div style={rowStyle}><span style={labelStyle}>L1A 来源</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.gf3_source_dirs)}</span></div>
<div style={rowStyle}><span style={labelStyle}>L2 存储</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.gf3_storage_dirs)}</span></div> <div style={rowStyle}><span style={labelStyle}>L2 存储</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.gf3_storage_dirs)}</span></div>
@@ -538,10 +729,17 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
<div style={{ display: 'flex', gap: '10px' }}> <div style={{ display: 'flex', gap: '10px' }}>
<button <button
onClick={handleGf3BatchProcess} onClick={handleGf3BatchProcess}
disabled={gf3Loading || readOnly || !canRunGf3Process} disabled={gf3ProcessLoading || readOnly || !canRunGf3Process}
style={actionBtnStyle(gf3Loading, readOnly || !canRunGf3Process)} style={actionBtnStyle(gf3ProcessLoading, readOnly || !canRunGf3Process)}
> >
{gf3Loading ? '运行中...' : (readOnly ? '只读模式' : 'GF3 L1A→L2')} {gf3ProcessLoading ? '运行中...' : (readOnly ? '只读模式' : 'GF3 预处理')}
</button>
<button
onClick={handleGf3Scan}
disabled={gf3Loading || readOnly || !canRunGf3Scan}
style={actionBtnStyle(gf3Loading, readOnly || !canRunGf3Scan)}
>
{gf3Loading ? '运行中...' : (readOnly ? '只读模式' : '扫描 GF3')}
</button> </button>
<div <div
style={{ style={{
@@ -554,63 +752,31 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
</div> </div>
</div> </div>
</div> </div>
<div style={sectionStyle}>
<div style={{ fontWeight: 'bold', marginBottom: '8px', color: 'var(--color-text-primary)' }}>活动任务</div>
{displayActiveTasks.length === 0 ? (
<div style={{ fontSize: '0.85em', color: 'var(--color-text-muted)' }}>当前无活动任务。</div>
) : (
<div style={{ display: 'grid', rowGap: '8px' }}>
{displayActiveTasks.slice(0, 4).map((task) => (
<div key={task.task_id} style={{ fontSize: '0.85em', color: 'var(--color-text-secondary)' }}>
<div style={{ display: 'flex', justifyContent: 'space-between', marginBottom: '4px' }}>
<span>{task.task_type}</span>
<span>{task.progress}%</span>
</div>
<div style={{ height: '6px', background: 'var(--color-panel-muted)', borderRadius: '3px', overflow: 'hidden' }}>
<div style={{ width: `${task.progress}%`, height: '100%', background: 'var(--color-info)' }} />
</div>
<div style={{ color: 'var(--color-text-muted)', marginTop: '4px', wordBreak: 'break-all' }}>{t(task.message || '')}</div>
</div>
))}
</div>
)}
</div>
<div style={{ marginBottom: '4px' }}>
<h4 style={{ margin: '0 0 5px 0', fontSize: '1em' }}>实时日志</h4>
<div
style={{
height: '160px',
overflowY: 'auto',
backgroundColor: '#0f172a',
color: '#22c55e',
padding: '10px',
fontFamily: 'monospace',
fontSize: '0.85em',
borderRadius: '4px',
}}
>
{displayLogs.length === 0 ? (
<div style={{ color: 'var(--color-text-muted)' }}>暂无日志...</div>
) : (
displayLogs.map((log, index) => (
<div key={index} style={{ whiteSpace: 'pre-wrap', wordBreak: 'break-all' }}>{t(log)}</div>
))
)}
<div ref={logEndRef} />
</div>
</div>
</div> </div>
<div style={{ flexShrink: 0, borderTop: '1px solid var(--color-border)', paddingTop: '10px', marginTop: '6px' }}> <div style={{ flexShrink: 0, borderTop: '1px solid var(--color-border)', paddingTop: '10px', marginTop: '6px' }}>
<div style={{ display: 'flex', gap: '8px', marginBottom: '6px' }}> <div style={{ fontWeight: 'bold', marginBottom: '6px', color: 'var(--color-text-primary)' }}>实时日志</div>
<button onClick={() => handleRunNow('radar')} disabled={loading || !canRunRadar} style={scanBtnStyle(canRunRadar)}>扫描 LT-1</button> <div
<button onClick={() => handleRunNow('gf3')} disabled={loading || !canRunGf3Scan} style={scanBtnStyle(canRunGf3Scan)}>扫描 GF3</button> style={{
<button onClick={() => handleRunNow('orbit')} disabled={loading || !canRunOrbit} style={scanBtnStyle(canRunOrbit)}>扫描精轨</button> height: '160px',
<button onClick={() => handleRunNow('dinsar')} disabled={loading || !canRunDinsar} style={scanBtnStyle(canRunDinsar)}>扫描 D-InSAR</button> overflowY: 'auto',
backgroundColor: '#0f172a',
color: '#22c55e',
padding: '10px',
fontFamily: 'monospace',
fontSize: '0.85em',
borderRadius: '4px',
}}
>
{displayLogs.length === 0 ? (
<div style={{ color: 'var(--color-text-muted)' }}>暂无日志...</div>
) : (
displayLogs.map((log, index) => (
<div key={index} style={{ whiteSpace: 'pre-wrap', wordBreak: 'break-all' }}>{t(log)}</div>
))
)}
<div ref={logEndRef} />
</div> </div>
{message && <div style={{ color: message.includes('失败') ? 'red' : 'green', fontSize: '0.9em' }}>{message}</div>}
</div> </div>
{showUnpackDialog && ( {showUnpackDialog && (
+12 -7
View File
@@ -2,6 +2,7 @@
import { deleteRunLog, deleteRunRecord, getRunLog, listEngines, listRuns, previewPyintInputAssets, submitRun } from './api/dinsarProduction'; import { deleteRunLog, deleteRunRecord, getRunLog, listEngines, listRuns, previewPyintInputAssets, submitRun } from './api/dinsarProduction';
import { clearTaskLogs, deleteTaskLog, deleteTaskRecord, getActiveTasks, getRecentTasks, getTaskLogs } from './api/tasks'; import { clearTaskLogs, deleteTaskLog, deleteTaskRecord, getActiveTasks, getRecentTasks, getTaskLogs } from './api/tasks';
import { formatSatelliteFamilyLabel, inferSatelliteFamilyFromResultLike } from './utils/satelliteFamily';
const card = { const card = {
background: '#fff', background: '#fff',
@@ -160,6 +161,9 @@ function taskToRunRow(task) {
completed_items: null, completed_items: null,
failed_items: null, failed_items: null,
skipped_items: null, skipped_items: null,
master_satellite: task?.master_satellite || '',
slave_satellite: task?.slave_satellite || '',
pair_key: task?.pair_key || '',
}; };
} }
@@ -1375,12 +1379,12 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued })
}} }}
> >
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', gap: 10, flexWrap: 'wrap', marginBottom: 8 }}> <div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', gap: 10, flexWrap: 'wrap', marginBottom: 8 }}>
<div> <div>
<div style={{ fontSize: 12, color: '#0f172a', fontWeight: 600 }}>PyINT 输入资产预检</div> <div style={{ fontSize: 12, color: '#0f172a', fontWeight: 600 }}>PyINT 输入资产预检</div>
<div style={{ fontSize: 11, color: '#64748b', marginTop: 4 }}> <div style={{ fontSize: 11, color: '#64748b', marginTop: 4 }}>
提交前检查 Task_* 结构、DEM 策略和 LT-1 轨道是否齐备。即使不手动预检,后端提交时也会做同样校验。 提交前检查 Task_* 结构、DEM 策略以及生产所需源数据和轨道文件是否齐备。即使不手动预检,后端提交时也会做同样校验。
</div>
</div> </div>
</div>
<button <button
onClick={handlePreviewPyint} onClick={handlePreviewPyint}
disabled={readOnly || pyintPreviewLoading || !rootDir.trim()} disabled={readOnly || pyintPreviewLoading || !rootDir.trim()}
@@ -1436,7 +1440,7 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued })
{ label: 'DEM 策略', value: formatPyintDemMode(pyintPreview?.dem?.mode), color: '#1d4ed8' }, { label: 'DEM 策略', value: formatPyintDemMode(pyintPreview?.dem?.mode), color: '#1d4ed8' },
{ label: '轨道策略', value: formatPyintOrbitPolicy(pyintPreview?.orbits?.policy), color: '#7c3aed' }, { label: '轨道策略', value: formatPyintOrbitPolicy(pyintPreview?.orbits?.policy), color: '#7c3aed' },
{ {
label: '精轨桥接', label: '轨道处理',
value: pyintPreview?.precise_orbit_bridge?.enabled value: pyintPreview?.precise_orbit_bridge?.enabled
? formatPyintPreciseOrbitMode(pyintPreview?.precise_orbit_bridge?.mode) ? formatPyintPreciseOrbitMode(pyintPreview?.precise_orbit_bridge?.mode)
: '关闭', : '关闭',
@@ -1732,7 +1736,7 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued })
<table style={{ width: '100%', borderCollapse: 'collapse', fontSize: 12 }}> <table style={{ width: '100%', borderCollapse: 'collapse', fontSize: 12 }}>
<thead> <thead>
<tr> <tr>
{['运行ID', '引擎', '状态', '时间', '路径', '操作'].map(header => ( {['运行ID', '引擎', '数据', '状态', '时间', '路径', '操作'].map(header => (
<th <th
key={header} key={header}
style={{ style={{
@@ -1759,6 +1763,7 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued })
<tr key={`${run.record_type || 'run'}-${run.run_id}`} style={{ borderBottom: '1px solid #f1f5f9' }}> <tr key={`${run.record_type || 'run'}-${run.run_id}`} style={{ borderBottom: '1px solid #f1f5f9' }}>
<td style={{ padding: '6px 8px', fontFamily: 'monospace', fontSize: 11 }}>{run.run_id}</td> <td style={{ padding: '6px 8px', fontFamily: 'monospace', fontSize: 11 }}>{run.run_id}</td>
<td style={{ padding: '6px 8px' }}>{formatEngineLabel(run.engine)}</td> <td style={{ padding: '6px 8px' }}>{formatEngineLabel(run.engine)}</td>
<td style={{ padding: '6px 8px' }}>{formatSatelliteFamilyLabel(inferSatelliteFamilyFromResultLike(run))}</td>
<td <td
style={{ style={{
padding: '6px 8px', padding: '6px 8px',
+22
View File
@@ -0,0 +1,22 @@
import apiClient from './client';
export const getAssetInventoryStatus = () =>
apiClient.get('/assets/inventory/status').then(r => r.data);
export const scanAssetInventory = (payload = {}) =>
apiClient.post('/assets/inventory/scan', payload).then(r => r.data);
export const listSourceAssets = (params = {}) =>
apiClient.get('/assets/sources', { params }).then(r => r.data);
export const listOrbitAssets = (params = {}) =>
apiClient.get('/assets/orbits', { params }).then(r => r.data);
export const listAssetIssues = (params = {}) =>
apiClient.get('/assets/issues', { params }).then(r => r.data);
export const unpackSentinel1Source = (assetId, payload = {}) =>
apiClient.post(`/assets/sources/${assetId}/unpack-sentinel1`, payload).then(r => r.data);
export const unpackSentinel1Batch = (payload = {}) =>
apiClient.post('/assets/inventory/unpack-sentinel1', payload).then(r => r.data);
+3
View File
@@ -3,6 +3,9 @@ import axios from 'axios';
const apiClient = axios.create({ const apiClient = axios.create({
baseURL: '/api', baseURL: '/api',
withCredentials: true, withCredentials: true,
paramsSerializer: {
indexes: null,
},
}); });
export default apiClient; export default apiClient;
@@ -20,6 +20,10 @@ const getTaskTypeLabel = (taskType) => {
return '灾害点同步'; return '灾害点同步';
case 'UNPACK_ARCHIVES': case 'UNPACK_ARCHIVES':
return 'LT-1 解包'; return 'LT-1 解包';
case 'UNPACK_SENTINEL1':
return 'Sentinel-1 解包';
case 'SCAN_ASSET_INVENTORY':
return '资产库存扫描';
case 'IDL_IMPORT': case 'IDL_IMPORT':
return 'ENVI 数据导入'; return 'ENVI 数据导入';
case 'IDL_DINSAR': case 'IDL_DINSAR':
+86 -27
View File
@@ -1,33 +1,92 @@
const createRows = (dataInfo, language, formatYmd) => [ const formatDateTime = (value) => {
{ label: language === 'en' ? 'Satellite:' : '卫星:', value: dataInfo.satellite || '-' }, if (!value) return '-';
{ label: language === 'en' ? 'Satellite Mode:' : '卫星模式:', value: dataInfo.satellite_mode || '-' }, const date = new Date(value);
{ label: language === 'en' ? 'Receiving Station:' : '接收站:', value: dataInfo.receiving_station || '-' }, if (Number.isNaN(date.getTime())) return String(value);
{ label: language === 'en' ? 'Imaging Date:' : '成像日期:', value: formatYmd(dataInfo.imaging_date) }, return date.toLocaleString();
{ label: language === 'en' ? 'Imaging Mode:' : '成像模式:', value: dataInfo.imaging_mode || '-' }, };
{ label: language === 'en' ? 'Orbit Circle:' : '轨道圈号:', value: dataInfo.orbit_circle || '-' },
{ label: language === 'en' ? 'Scene Center Lon:' : '场景中心经度:', value: dataInfo.scene_center_lon ?? '-' }, const joinChannels = (value) => {
{ label: language === 'en' ? 'Scene Center Lat:' : '场景中心纬度:', value: dataInfo.scene_center_lat ?? '-' }, if (Array.isArray(value)) {
{ label: language === 'en' ? 'Acquisition Time:' : '采集时间:', value: dataInfo.acquisition_time_utc || '-' }, const items = value.map((item) => String(item || '').trim()).filter(Boolean);
{ label: language === 'en' ? 'Product Type:' : '产品类型:', value: dataInfo.product_type || '-' }, return items.length ? items.join(' / ') : '-';
{ label: language === 'en' ? 'Polarization:' : '极化方式:', value: dataInfo.polarization || '-' }, }
{ label: language === 'en' ? 'Product Level:' : '产品级别:', value: dataInfo.product_level || '-' }, if (value === null || value === undefined || value === '') return '-';
{ label: language === 'en' ? 'Product Unique ID:' : '产品唯一ID:', value: dataInfo.product_unique_id || '-' }, return String(value);
{ label: language === 'en' ? 'Orbit Direction:' : '轨道方向:', value: dataInfo.orbit_direction || '-' }, };
{
label: language === 'en' ? 'Has Orbit:' : '有精轨:', const yesNo = (flag, language) => {
value: dataInfo.has_orbit_data ? (language === 'en' ? 'Yes' : '是') : (language === 'en' ? 'No' : '否'), if (flag) return language === 'en' ? 'Yes' : '是';
}, return language === 'en' ? 'No' : '否';
{ };
label: language === 'en' ? 'Orbit File:' : '轨道文件:',
value: dataInfo.orbit_file_path || '-', const field = (label, value, extra = {}) => ({
label,
value: value === null || value === undefined || value === '' ? '-' : value,
...extra,
});
const createSentinelRows = (dataInfo, language, formatYmd) => {
const metadata = dataInfo.metadata_json || {};
const polarizationChannels = metadata.polarization_channels || metadata.manifest_polarizations;
return [
field(language === 'en' ? 'Satellite:' : '卫星:', dataInfo.satellite),
field(language === 'en' ? 'Satellite Family:' : '卫星系列:', dataInfo.satellite_family),
field(language === 'en' ? 'Source Format:' : '源格式:', dataInfo.source_format),
field(language === 'en' ? 'Imaging Date:' : '成像日期:', formatYmd(dataInfo.imaging_date)),
field(language === 'en' ? 'Acquisition Start:' : '采集开始:', formatDateTime(dataInfo.acquisition_start_time_utc)),
field(language === 'en' ? 'Acquisition Stop:' : '采集结束:', formatDateTime(dataInfo.acquisition_stop_time_utc)),
field(language === 'en' ? 'Imaging Mode:' : '成像模式:', dataInfo.imaging_mode),
field(language === 'en' ? 'Product Type:' : '产品类型:', dataInfo.product_type),
field(language === 'en' ? 'Product Level:' : '产品级别:', dataInfo.product_level),
field(language === 'en' ? 'Orbit Direction:' : '轨道方向:', dataInfo.orbit_direction),
field(language === 'en' ? 'Relative Orbit:' : '相对轨道:', dataInfo.relative_orbit),
field(language === 'en' ? 'Absolute Orbit:' : '绝对轨道:', dataInfo.absolute_orbit),
field(language === 'en' ? 'Polarization:' : '极化方式:', dataInfo.polarization),
field(language === 'en' ? 'Polarization Channels:' : '极化通道:', joinChannels(polarizationChannels)),
field(language === 'en' ? 'Datatake:' : '数据采集号:', metadata.filename_datatake),
field(language === 'en' ? 'Scene Center Lon:' : '场景中心经度:', dataInfo.scene_center_lon),
field(language === 'en' ? 'Scene Center Lat:' : '场景中心纬度:', dataInfo.scene_center_lat),
field(language === 'en' ? 'Product Unique ID:' : '产品唯一ID:', dataInfo.product_unique_id, {
valueStyle: { wordBreak: 'break-all' },
}),
field(language === 'en' ? 'Has Orbit:' : '有精轨:', yesNo(dataInfo.has_orbit_data, language)),
field(language === 'en' ? 'Orbit File:' : '轨道文件:', dataInfo.orbit_file_path, {
valueStyle: { wordBreak: 'break-all' },
}),
field(language === 'en' ? 'ENVI Processed:' : 'ENVI已处理:', yesNo(dataInfo.is_envi_processed, language)),
];
};
const createDefaultRows = (dataInfo, language, formatYmd) => [
field(language === 'en' ? 'Satellite:' : '卫星:', dataInfo.satellite),
field(language === 'en' ? 'Satellite Mode:' : '卫星模式:', dataInfo.satellite_mode),
field(language === 'en' ? 'Receiving Station:' : '接收站:', dataInfo.receiving_station),
field(language === 'en' ? 'Imaging Date:' : '成像日期:', formatYmd(dataInfo.imaging_date)),
field(language === 'en' ? 'Imaging Mode:' : '成像模式:', dataInfo.imaging_mode),
field(language === 'en' ? 'Orbit Circle:' : '轨道圈号:', dataInfo.orbit_circle),
field(language === 'en' ? 'Scene Center Lon:' : '场景中心经度:', dataInfo.scene_center_lon),
field(language === 'en' ? 'Scene Center Lat:' : '场景中心纬度:', dataInfo.scene_center_lat),
field(language === 'en' ? 'Acquisition Time:' : '采集时间:', dataInfo.acquisition_time_utc),
field(language === 'en' ? 'Product Type:' : '产品类型:', dataInfo.product_type),
field(language === 'en' ? 'Polarization:' : '极化方式:', dataInfo.polarization),
field(language === 'en' ? 'Product Level:' : '产品级别:', dataInfo.product_level),
field(language === 'en' ? 'Product Unique ID:' : '产品唯一ID:', dataInfo.product_unique_id),
field(language === 'en' ? 'Orbit Direction:' : '轨道方向:', dataInfo.orbit_direction),
field(language === 'en' ? 'Has Orbit:' : '有精轨:', yesNo(dataInfo.has_orbit_data, language)),
field(language === 'en' ? 'Orbit File:' : '轨道文件:', dataInfo.orbit_file_path, {
valueStyle: { wordBreak: 'break-all' }, valueStyle: { wordBreak: 'break-all' },
}, }),
{ field(language === 'en' ? 'ENVI Processed:' : 'ENVI已处理:', yesNo(dataInfo.is_envi_processed, language)),
label: language === 'en' ? 'ENVI Processed:' : 'ENVI已处理:',
value: dataInfo.is_envi_processed ? (language === 'en' ? 'Yes' : '是') : (language === 'en' ? 'No' : '否'),
},
]; ];
const createRows = (dataInfo, language, formatYmd) => {
if ((dataInfo.satellite_family || '').toUpperCase() === 'S1') {
return createSentinelRows(dataInfo, language, formatYmd);
}
return createDefaultRows(dataInfo, language, formatYmd);
};
export default function DataInfoModal({ export default function DataInfoModal({
visible, visible,
dataInfo, dataInfo,
@@ -13,6 +13,7 @@ import {
buildDinsarEngineOptions, buildDinsarEngineOptions,
getDinsarEngineMeta, getDinsarEngineMeta,
} from '../utils/dinsarEngines'; } from '../utils/dinsarEngines';
import { formatSatelliteFamilyLabel, inferSatelliteFamilyFromResultLike } from '../utils/satelliteFamily';
const STATUS_TONE_MAP = { const STATUS_TONE_MAP = {
READY: 'ready', READY: 'ready',
@@ -383,6 +384,7 @@ export default function DinsarCatalogPanel({
{products.map((item) => { {products.map((item) => {
const tone = STATUS_TONE_MAP[item.status] || 'neutral'; const tone = STATUS_TONE_MAP[item.status] || 'neutral';
const engineMeta = getDinsarEngineMeta(item.engine_code); const engineMeta = getDinsarEngineMeta(item.engine_code);
const satelliteFamily = inferSatelliteFamilyFromResultLike(item);
return ( return (
<button <button
key={item.id} key={item.id}
@@ -396,6 +398,9 @@ export default function DinsarCatalogPanel({
</div> </div>
<div className="dinsar-catalog-list-item-badges"> <div className="dinsar-catalog-list-item-badges">
<span className={`dinsar-engine-badge tone-${engineMeta.tone}`}>{engineMeta.shortLabel}</span> <span className={`dinsar-engine-badge tone-${engineMeta.tone}`}>{engineMeta.shortLabel}</span>
{satelliteFamily && (
<span className="dinsar-engine-badge tone-unknown">{formatSatelliteFamilyLabel(satelliteFamily)}</span>
)}
<span>{formatDateTime(item.published_at)}</span> <span>{formatDateTime(item.published_at)}</span>
</div> </div>
<div className="dinsar-catalog-list-item-meta"> <div className="dinsar-catalog-list-item-meta">
@@ -434,6 +439,9 @@ export default function DinsarCatalogPanel({
<div className="dinsar-catalog-empty error">{selectedProduct.error}</div> <div className="dinsar-catalog-empty error">{selectedProduct.error}</div>
) : ( ) : (
<div className="dinsar-catalog-detail-body"> <div className="dinsar-catalog-detail-body">
{(() => {
const satelliteFamily = inferSatelliteFamilyFromResultLike(selectedProduct?.profile || selectedProduct);
return (
<div className="dinsar-catalog-hero"> <div className="dinsar-catalog-hero">
<div className="dinsar-catalog-preview-frame"> <div className="dinsar-catalog-preview-frame">
<img <img
@@ -452,6 +460,11 @@ export default function DinsarCatalogPanel({
<span className={`dinsar-engine-badge tone-${selectedProductEngine.tone}`}> <span className={`dinsar-engine-badge tone-${selectedProductEngine.tone}`}>
{selectedProductEngine.shortLabel} {selectedProductEngine.shortLabel}
</span> </span>
{satelliteFamily && (
<span className="dinsar-engine-badge tone-unknown">
{formatSatelliteFamilyLabel(satelliteFamily)}
</span>
)}
<StatusPill label={selectedProduct.status || 'UNKNOWN'} tone={selectedStatusTone} /> <StatusPill label={selectedProduct.status || 'UNKNOWN'} tone={selectedStatusTone} />
</div> </div>
</div> </div>
@@ -469,6 +482,8 @@ export default function DinsarCatalogPanel({
</div> </div>
</div> </div>
</div> </div>
);
})()}
<div className="dinsar-catalog-detail-grid"> <div className="dinsar-catalog-detail-grid">
<div className="dinsar-catalog-section-card"> <div className="dinsar-catalog-section-card">
@@ -14,6 +14,7 @@ import { getLeftTabLabel } from '../../utils/appUiHelpers';
import { PanelLoadingBody, PanelLoadingPanel } from './AppLoadingFallbacks'; import { PanelLoadingBody, PanelLoadingPanel } from './AppLoadingFallbacks';
const LazyDataMonitorPanel = lazy(() => import('../../DataMonitorPanel')); const LazyDataMonitorPanel = lazy(() => import('../../DataMonitorPanel'));
const LazyAssetInventoryPanel = lazy(() => import('../../AssetInventoryPanel'));
const LazyDataCopierPanel = lazy(() => import('../../DataCopierPanel')); const LazyDataCopierPanel = lazy(() => import('../../DataCopierPanel'));
const LazyIDLAutomationPanel = lazy(() => import('../../IDLAutomationPanel')); const LazyIDLAutomationPanel = lazy(() => import('../../IDLAutomationPanel'));
const LazyHazardPointPanel = lazy(() => import('../../HazardPointPanel')); const LazyHazardPointPanel = lazy(() => import('../../HazardPointPanel'));
@@ -237,6 +238,17 @@ export default function AppSidePanel({
</div> </div>
)} )}
{leftPanelTab === 'asset_inventory' && (
<div className="panel-content" style={{ flex: '1 1 auto', padding: 0, overflow: 'auto' }}>
<Suspense fallback={<PanelLoadingBody message="正在加载资产库存..." />}>
<LazyAssetInventoryPanel
readOnly={isReadOnlyUser}
onTaskStart={taskPanel.onTaskStart}
/>
</Suspense>
</div>
)}
{leftPanelTab === 'pairing' && ( {leftPanelTab === 'pairing' && (
<Suspense fallback={<PanelLoadingPanel message="正在加载组网规划面板..." />}> <Suspense fallback={<PanelLoadingPanel message="正在加载组网规划面板..." />}>
<LazyPairingPanel <LazyPairingPanel
@@ -1,6 +1,7 @@
import { memo } from 'react'; import { memo } from 'react';
import { parseDatesFromName, formatYmd } from '../../utils/appUiHelpers'; import { parseDatesFromName, formatYmd } from '../../utils/appUiHelpers';
import { getDinsarEngineMeta } from '../../utils/dinsarEngines'; import { getDinsarEngineMeta } from '../../utils/dinsarEngines';
import { formatSatelliteFamilyLabel, inferSatelliteFamilyFromResultLike } from '../../utils/satelliteFamily';
function truncateMiddle(value, maxLength = 28) { function truncateMiddle(value, maxLength = 28) {
const text = String(value || '').trim(); const text = String(value || '').trim();
@@ -23,6 +24,7 @@ function DinsarResultRow({
}) { }) {
const dates = showDates ? parseDatesFromName(result.name, (value) => formatYmd(value, language)) : null; const dates = showDates ? parseDatesFromName(result.name, (value) => formatYmd(value, language)) : null;
const engineMeta = getDinsarEngineMeta(result.engine_code); const engineMeta = getDinsarEngineMeta(result.engine_code);
const satelliteFamily = inferSatelliteFamilyFromResultLike(result);
const hasTrace = !!( const hasTrace = !!(
result.selection_strategy || result.selection_strategy ||
result.network_run_id || result.network_run_id ||
@@ -45,6 +47,14 @@ function DinsarResultRow({
> >
{engineMeta.shortLabel} {engineMeta.shortLabel}
</span> </span>
{satelliteFamily && (
<span
className="dinsar-engine-badge tone-unknown"
title={language === 'en' ? 'Satellite family' : '卫星系列'}
>
{formatSatelliteFamilyLabel(satelliteFamily)}
</span>
)}
{result.ai_score !== null && ( {result.ai_score !== null && (
<span <span
className={`ai-score ${result.ai_score > 0.7 ? 'good' : (result.ai_score < 0.4 ? 'bad' : 'medium')}`} className={`ai-score ${result.ai_score > 0.7 ? 'good' : (result.ai_score < 0.4 ? 'bad' : 'medium')}`}
+3 -1
View File
@@ -145,7 +145,7 @@ export const LEFT_GROUP_SECTIONS = {
}; };
export const LEFT_GROUP_TABS = { export const LEFT_GROUP_TABS = {
data: ['ingest', 'data', 'hazard'], data: ['ingest', 'asset_inventory', 'data', 'hazard'],
production_planning: LEFT_GROUP_SECTIONS.production_planning.flatMap(section => section.tabs), production_planning: LEFT_GROUP_SECTIONS.production_planning.flatMap(section => section.tabs),
production_management: [PRODUCTION_WORKSPACE_TAB], production_management: [PRODUCTION_WORKSPACE_TAB],
insar_analysis: LEFT_GROUP_SECTIONS.insar_analysis.flatMap(section => section.tabs), insar_analysis: LEFT_GROUP_SECTIONS.insar_analysis.flatMap(section => section.tabs),
@@ -180,6 +180,7 @@ export const FULL_WIDTH_LEFT_TABS = new Set([
export const ADMIN_ONLY_TABS = new Set([ export const ADMIN_ONLY_TABS = new Set([
'ingest', 'ingest',
'asset_inventory',
'pairing', 'pairing',
'pairs', 'pairs',
'ps_results', 'ps_results',
@@ -199,6 +200,7 @@ export const BATCH_API_MAX_PAGES = 200;
export const SATELLITE_GROUPS = [ export const SATELLITE_GROUPS = [
{ key: 'LT-1', label: 'LT-1', prefixes: ['LT1'] }, { key: 'LT-1', label: 'LT-1', prefixes: ['LT1'] },
{ key: 'S1', label: 'Sentinel-1', prefixes: ['S1'] },
{ key: 'GF-3', label: 'GF-3', prefixes: ['GF3'] }, { key: 'GF-3', label: 'GF-3', prefixes: ['GF3'] },
]; ];
+1 -1
View File
@@ -11,7 +11,7 @@ import { normalizePagePayload } from '../utils/appHelpers';
import { normalizeTaskStatus } from '../utils/appUiHelpers'; import { normalizeTaskStatus } from '../utils/appUiHelpers';
import { DEFAULT_LIST_PAGE_SIZE } from '../config/appConstants'; import { DEFAULT_LIST_PAGE_SIZE } from '../config/appConstants';
const NON_BLOCKING_TASK_TYPES = new Set(['UNPACK_ARCHIVES', 'COPY_DATA']); const NON_BLOCKING_TASK_TYPES = new Set(['UNPACK_ARCHIVES', 'UNPACK_SENTINEL1', 'SCAN_ASSET_INVENTORY', 'COPY_DATA']);
export default function useDinsarOperations({ export default function useDinsarOperations({
onCleanupDinsarLayers, onCleanupDinsarLayers,
+1 -1
View File
@@ -2,7 +2,7 @@ import { useCallback, useEffect, useRef, useState } from 'react';
import apiClient from '../api/client'; import apiClient from '../api/client';
import { normalizeTaskStatus } from '../utils/appUiHelpers'; import { normalizeTaskStatus } from '../utils/appUiHelpers';
const NON_BLOCKING_TASK_TYPES = new Set(['UNPACK_ARCHIVES', 'COPY_DATA']); const NON_BLOCKING_TASK_TYPES = new Set(['UNPACK_ARCHIVES', 'UNPACK_SENTINEL1', 'SCAN_ASSET_INVENTORY', 'COPY_DATA']);
const isTaskNonBlocking = (taskId, taskType, nonBlockingTaskIds = []) => ( const isTaskNonBlocking = (taskId, taskType, nonBlockingTaskIds = []) => (
NON_BLOCKING_TASK_TYPES.has(String(taskType || '').toUpperCase()) NON_BLOCKING_TASK_TYPES.has(String(taskType || '').toUpperCase())
+41 -21
View File
@@ -59,6 +59,19 @@ export default function useRadarSearch({
clearRadarSearchResults, clearRadarSearchResults,
clearRadarMapLayers, clearRadarMapLayers,
}) { }) {
const getSatelliteCatalog = () => {
const satellites = useRadarStore.getState().radarSearchOptions?.satellite;
return Array.isArray(satellites) ? satellites.filter(Boolean) : [];
};
const getSatellitesForGroup = (groupKey, satellites = getSatelliteCatalog()) => {
const group = SATELLITE_GROUPS.find((item) => item.key === groupKey);
if (!group) return [];
return satellites.filter((sat) =>
group.prefixes.some((prefix) => String(sat || '').startsWith(prefix))
);
};
const fetchRadarImagingDates = useCallback(async () => { const fetchRadarImagingDates = useCallback(async () => {
try { try {
const response = await apiClient.get('/radar-data/imaging-dates'); const response = await apiClient.get('/radar-data/imaging-dates');
@@ -74,13 +87,31 @@ export default function useRadarSearch({
try { try {
setRadarSearchOptionsLoading(true); setRadarSearchOptionsLoading(true);
const params = {}; const params = {};
if (Array.isArray(satelliteFilter) && satelliteFilter.length > 0) { const storeState = useRadarStore.getState();
params.satellite = satelliteFilter; const satelliteCatalog = getSatelliteCatalog();
const hasExplicitSatelliteFilter = Array.isArray(satelliteFilter);
let resolvedSatelliteFilter = hasExplicitSatelliteFilter
? satelliteFilter.filter(Boolean)
: [];
if (!hasExplicitSatelliteFilter) {
const groupKey = storeState.selectedSatelliteGroup;
if (groupKey && groupKey !== 'all') {
resolvedSatelliteFilter = getSatellitesForGroup(groupKey, satelliteCatalog);
}
}
if (resolvedSatelliteFilter.length > 0) {
params.satellite = resolvedSatelliteFilter;
} }
const response = await apiClient.get('/radar-data/search/options', { params }); const response = await apiClient.get('/radar-data/search/options', { params });
const payload = response?.data && typeof response.data === 'object' ? response.data : {}; const payload = response?.data && typeof response.data === 'object' ? response.data : {};
const payloadSatellites = Array.isArray(payload.satellite) ? payload.satellite.filter(Boolean) : [];
const nextSatelliteCatalog = satelliteCatalog.length > payloadSatellites.length
? satelliteCatalog
: payloadSatellites;
setRadarSearchOptions({ setRadarSearchOptions({
satellite: Array.isArray(payload.satellite) ? payload.satellite : [], satellite: nextSatelliteCatalog,
satellite_mode: Array.isArray(payload.satellite_mode) ? payload.satellite_mode : [], satellite_mode: Array.isArray(payload.satellite_mode) ? payload.satellite_mode : [],
receiving_station: Array.isArray(payload.receiving_station) ? payload.receiving_station : [], receiving_station: Array.isArray(payload.receiving_station) ? payload.receiving_station : [],
imaging_mode: Array.isArray(payload.imaging_mode) ? payload.imaging_mode : [], imaging_mode: Array.isArray(payload.imaging_mode) ? payload.imaging_mode : [],
@@ -119,14 +150,9 @@ export default function useRadarSearch({
orbit_direction: '', orbit_direction: '',
})); }));
if (groupKey === 'all') { if (groupKey === 'all') {
fetchRadarSearchOptions(); fetchRadarSearchOptions([]);
} else { } else {
const group = SATELLITE_GROUPS.find((g) => g.key === groupKey); const matched = getSatellitesForGroup(groupKey);
if (!group) return;
const allSatellites = useRadarStore.getState().radarSearchOptions.satellite;
const matched = allSatellites.filter((sat) =>
group.prefixes.some((prefix) => sat.startsWith(prefix))
);
if (matched.length > 0) { if (matched.length > 0) {
fetchRadarSearchOptions(matched); fetchRadarSearchOptions(matched);
} }
@@ -265,15 +291,9 @@ export default function useRadarSearch({
const applyRadarSearch = useCallback(async () => { const applyRadarSearch = useCallback(async () => {
const draftWithSatelliteGroup = { ...radarSearchDraft }; const draftWithSatelliteGroup = { ...radarSearchDraft };
if (selectedSatelliteGroup && selectedSatelliteGroup !== 'all') { if (selectedSatelliteGroup && selectedSatelliteGroup !== 'all') {
const group = SATELLITE_GROUPS.find((g) => g.key === selectedSatelliteGroup); const matched = getSatellitesForGroup(selectedSatelliteGroup);
if (group) { if (matched.length > 0) {
const allSatellites = useRadarStore.getState().radarSearchOptions.satellite; draftWithSatelliteGroup.satellite = matched.join(',');
const matched = allSatellites.filter((sat) =>
group.prefixes.some((prefix) => sat.startsWith(prefix))
);
if (matched.length > 0) {
draftWithSatelliteGroup.satellite = matched.join(',');
}
} }
} }
const normalizedCriteria = normalizeRadarSearchCriteria(draftWithSatelliteGroup, RADAR_SEARCH_DEFAULTS); const normalizedCriteria = normalizeRadarSearchCriteria(draftWithSatelliteGroup, RADAR_SEARCH_DEFAULTS);
@@ -349,7 +369,7 @@ export default function useRadarSearch({
radarSearchRequestSeqRef.current += 1; radarSearchRequestSeqRef.current += 1;
setHasRadarSearched(false); setHasRadarSearched(false);
clearRadarSearchResults({ limit: radarPagination.limit }); clearRadarSearchResults({ limit: radarPagination.limit });
fetchRadarSearchOptions(); fetchRadarSearchOptions([]);
addLog('info', '已清除检索条件,请点击"搜索"或"搜索全部"获取数据。'); addLog('info', '已清除检索条件,请点击"搜索"或"搜索全部"获取数据。');
}, [ }, [
radarPagination.limit, radarSearchRequestSeqRef, radarPagination.limit, radarSearchRequestSeqRef,
@@ -377,7 +397,7 @@ export default function useRadarSearch({
setRadarSearchRegionError(''); setRadarSearchRegionError('');
setRadarSearchAoiToken(''); setRadarSearchAoiToken('');
setSelectedSatelliteGroup('all'); setSelectedSatelliteGroup('all');
fetchRadarSearchOptions(); fetchRadarSearchOptions([]);
setHasRadarSearched(true); setHasRadarSearched(true);
setIsLoading(true); setIsLoading(true);
+3 -3
View File
@@ -20,8 +20,8 @@ export const usePairingStore = create((set) => ({
spatial_baseline_max_meters: 3000, spatial_baseline_max_meters: 3000,
limit_footprint_center_distance: false, limit_footprint_center_distance: false,
coverage_diversity_penalty: 0.3, coverage_diversity_penalty: 0.3,
require_same_imaging_mode: false, require_same_imaging_mode: true,
require_same_polarization: false, require_same_polarization: true,
aoi_overlap_threshold: 0, aoi_overlap_threshold: 0,
start_date: '', start_date: '',
// === 新增字段 === // === 新增字段 ===
@@ -32,7 +32,7 @@ export const usePairingStore = create((set) => ({
strategy: 'all', strategy: 'all',
num_connections: 1, num_connections: 1,
reference_image_id: null, reference_image_id: null,
allowed_satellites: ['LT1A', 'LT1B'], allowed_satellites: null,
cross_satellite_pairing: false, cross_satellite_pairing: false,
}, },
pairingAlert: { warnings: [], fallbackUsed: false }, pairingAlert: { warnings: [], fallbackUsed: false },
+2
View File
@@ -33,6 +33,8 @@ export const getLeftTabLabel = (tabKey, metrics = {}) => {
switch (tabKey) { switch (tabKey) {
case 'ingest': case 'ingest':
return '入库监控'; return '入库监控';
case 'asset_inventory':
return '资产库存';
case 'data': case 'data':
return '数据列表'; return '数据列表';
case 'hazard': case 'hazard':
+33
View File
@@ -0,0 +1,33 @@
export function normalizeSatelliteFamily(value) {
const raw = String(value || '').trim().toUpperCase();
if (!raw) return '';
const compact = raw.replace(/[-_\s]+/g, '');
if (['LT1', 'LT1A', 'LT1B', 'LUTAN1', 'LUTAN1A', 'LUTAN1B'].includes(compact)) return 'LT1';
if (['S1', 'S1A', 'S1B', 'S1C', 'SENTINEL1', 'SENTINEL1A', 'SENTINEL1B', 'SENTINEL1C'].includes(compact)) return 'S1';
if (['GF3', 'GAOFEN3'].includes(compact)) return 'GF3';
return raw;
}
export function inferSatelliteFamilyFromResultLike(item) {
const direct = normalizeSatelliteFamily(
item?.satellite_family
|| item?.master_satellite
|| item?.slave_satellite
|| item?.satellite
);
if (direct) return direct;
const pairKey = String(item?.pair_key || '').trim().toLowerCase();
if (pairKey.startsWith('s1_')) return 'S1';
if (pairKey.startsWith('lt1_')) return 'LT1';
if (pairKey.startsWith('gf3_')) return 'GF3';
return '';
}
export function formatSatelliteFamilyLabel(value) {
const family = normalizeSatelliteFamily(value);
if (family === 'S1') return 'Sentinel-1';
if (family === 'LT1') return 'LT-1';
if (family === 'GF3') return 'GF3';
return family || '-';
}
+5 -2
View File
@@ -158,8 +158,11 @@ def main(argv):
if not os.path.isdir(rslcDir): os.mkdir(rslcDir) if not os.path.isdir(rslcDir): os.mkdir(rslcDir)
ensure_master_rslc(projectName) ensure_master_rslc(projectName)
if 'S1' in projectName: cmd_command = 'coreg_s1_gamma.py' template_satellite = str(ut.update_template(templateDir + "/" + projectName + ".template").get('satelite', '') or '')
else: cmd_command = 'coreg_gamma.py' if str(template_satellite).startswith('S1') or 'S1' in projectName:
cmd_command = 'coreg_s1_gamma.py'
else:
cmd_command = 'coreg_gamma.py'
err_txt = scratchDir + '/' + projectName + '/coreg_gamma_all.err' err_txt = scratchDir + '/' + projectName + '/coreg_gamma_all.err'
if os.path.isfile(err_txt): os.remove(err_txt) if os.path.isfile(err_txt): os.remove(err_txt)
+2 -2
View File
@@ -231,9 +231,9 @@ def main(argv):
# if os.path.isfile(HGTSIM): os.remove(HGTSIM) # if os.path.isfile(HGTSIM): os.remove(HGTSIM)
else: else:
print('The SLC has already coregister Done') print('The SLC has already coregister Done')
sys.exit(1) sys.exit(0)
print("Coregister TOP SLC image to the reference TOPS image is done !!") print("Coregister TOP SLC image to the reference TOPS image is done !!")
sys.exit(1) sys.exit(0)
if __name__ == '__main__': if __name__ == '__main__':
main(sys.argv[:]) main(sys.argv[:])
+72 -25
View File
@@ -14,6 +14,7 @@ import time
import glob import glob
import argparse import argparse
import csv import csv
from pathlib import Path
from pyint import _utils as ut from pyint import _utils as ut
@@ -29,10 +30,11 @@ def get_s1_time(raw_file):
return times return times
def get_satellite(raw_file): def get_satellite(raw_file):
if 'S1A_IW_SLC_' in raw_file: name = os.path.basename(str(raw_file))
s0 = 'A' if name.startswith('S1') and len(name) >= 3:
s0 = name[2]
else: else:
s0 = 'B' s0 = 'A'
return s0 return s0
@@ -86,8 +88,8 @@ def main(argv):
#if not os.path.isdir(opod_dir): #if not os.path.isdir(opod_dir):
# os.mkdir(opod_dir) # os.mkdir(opod_dir)
call_str = " eof --save-dir " + opod_dir + " -p " + down_dir if opod_dir and not os.path.isdir(opod_dir):
os.system(call_str) os.makedirs(opod_dir, exist_ok=True)
if not os.path.isdir(slc_dir): if not os.path.isdir(slc_dir):
os.mkdir(slc_dir) os.mkdir(slc_dir)
@@ -99,9 +101,6 @@ def main(argv):
t_date = 't_' + date t_date = 't_' + date
call_str = 'ls ' + down_dir + '/S1*' + date + '* > ' + t_date
os.system(call_str)
start_swath = templateDict['start_swath'] start_swath = templateDict['start_swath']
end_swath = templateDict['end_swath'] end_swath = templateDict['end_swath']
@@ -119,13 +118,23 @@ def main(argv):
# k_swath = '-' # k_swath = '-'
k_swath = ut.get_sardata_swath(start_swath,end_swath) k_swath = ut.get_sardata_swath(start_swath,end_swath)
raw_files = ut.read_txt2list(t_date) raw_files = sorted(glob.glob(down_dir + '/S1*' + date + '*.zip') + glob.glob(down_dir + '/S1*' + date + '*.SAFE'))
if not raw_files:
raise FileNotFoundError('No Sentinel-1 scenes found for date ' + date)
with open(t_date, 'w', encoding='utf-8') as f:
for raw_file in raw_files:
f.write(str(raw_file) + '\n')
satellite = get_satellite(str(raw_files[0])) satellite = get_satellite(str(raw_files[0]))
#orbit_file = ut.download_s1_orbit(date,opod_dir,satellite=satellite) #orbit_file = ut.download_s1_orbit(date,opod_dir,satellite=satellite)
raw_file_list = glob.glob(down_dir + '/S1*' + date + '*.zip') raw_file_list = glob.glob(down_dir + '/S1*' + date + '*.zip') + glob.glob(down_dir + '/S1*' + date + '*.SAFE')
file_num=len(raw_files) file_num=len(raw_files)
call_str = 'S1_BURST_tab_from_zipfile.py 3 --zip_ref_list ' + t_date + ' --zip_list ' + t_date if raw_files and os.path.isdir(str(raw_files[0])):
os.system(call_str) call_str = 'S1_BURST_tab_from_zipfile.py 3 --dir_ref_list ' + t_date + ' --dir_list ' + t_date
else:
call_str = 'S1_BURST_tab_from_zipfile.py 3 --zip_ref_list ' + t_date + ' --zip_list ' + t_date
rc = os.system(call_str)
if rc != 0:
raise RuntimeError('S1_BURST_tab_from_zipfile.py failed with rc=' + str(rc))
for kk in range(file_num): for kk in range(file_num):
zipfile_ref=str(raw_files[kk]) zipfile_ref=str(raw_files[kk])
@@ -135,22 +144,39 @@ def main(argv):
#os.system(call_str) #os.system(call_str)
#call_str = 'S1_import_SLC_from_zipfiles ' + t_date + ' ' + burst_number_table_ref + ' vv 0 ' + k_swath + ' ' + opod_dir + ' 1 1 ' #call_str = 'S1_import_SLC_from_zipfiles ' + t_date + ' ' + burst_number_table_ref + ' vv 0 ' + k_swath + ' ' + opod_dir + ' 1 1 '
call_str = 'read_S1_TOPS_SLC.py ' + zipfile_ref + ' --burst_sel ' + burst_number_table_ref + ' --pol vv --root_name ' + date + ' --sw_start ' + start_swath + ' --swn ' + end_swath + ' --OPOD_dir ' + opod_dir call_str = 'read_S1_TOPS_SLC.py ' + zipfile_ref + ' --burst_sel ' + burst_number_table_ref + ' --pol vv --root_name ' + date + ' --sw_start ' + start_swath + ' --swn ' + end_swath + ' --OPOD_dir ' + opod_dir
os.system(call_str) rc = os.system(call_str)
if rc != 0:
raise RuntimeError('read_S1_TOPS_SLC.py failed with rc=' + str(rc))
os.chdir(work_dir) os.chdir(work_dir)
call_str = "rename 's/vv.iw1.slc/iw1_" + str(kk)+".slc/g' *" rename_rules = [
os.system(call_str) ('vv.iw1.slc', 'iw1_' + str(kk) + '.slc'),
call_str = "rename 's/vv.iw2.slc/iw2_" + str(kk)+".slc/g' *" ('vv.iw2.slc', 'iw2_' + str(kk) + '.slc'),
os.system(call_str) ('vv.iw3.slc', 'iw3_' + str(kk) + '.slc'),
call_str = "rename 's/vv.iw3.slc/iw3_" + str(kk)+".slc/g' *" ('vv.SLC_tab', 'SLC_tab'),
os.system(call_str) ('vv.slc', 'slc'),
('tops_par', 'TOPS_par'),
]
for src_token, dst_token in rename_rules:
for candidate in list(Path(work_dir).iterdir()):
if src_token not in candidate.name:
continue
target = candidate.with_name(candidate.name.replace(src_token, dst_token))
if target != candidate and not target.exists():
candidate.rename(target)
SLC_Tab = work_dir + '/' + date + '_SLC_Tab' +str(kk) SLC_Tab = work_dir + '/' + date + '_SLC_Tab' +str(kk)
SLC_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc')) SLC_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc'))
SLC_par_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc.par')) SLC_par_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc.par'))
TOP_par_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc.tops_par')) TOP_par_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc.TOPS_par'))
if not TOP_par_list:
TOP_par_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc.tops_par'))
if (not SLC_list or not SLC_par_list or not TOP_par_list
or len(SLC_list) != len(SLC_par_list)
or len(SLC_list) != len(TOP_par_list)):
raise RuntimeError('Sentinel-1 concatenated SLC generation did not produce expected IW products for ' + date)
cat_str = 'touch slc_list slc_par_list top_par_list' cat_str = 'touch slc_list slc_par_list top_par_list'
os.system(call_str) os.system(cat_str)
list_num=len(SLC_list) list_num=len(SLC_list)
for tt in range(list_num): for tt in range(list_num):
call_str = 'echo ' + SLC_list[tt] + ' >> slc_list' call_str = 'echo ' + SLC_list[tt] + ' >> slc_list'
@@ -160,7 +186,9 @@ def main(argv):
call_str = 'echo ' + TOP_par_list[tt] + ' >> top_par_list' call_str = 'echo ' + TOP_par_list[tt] + ' >> top_par_list'
os.system(call_str) os.system(call_str)
call_str = ' paste slc_list slc_par_list top_par_list > ' + SLC_Tab call_str = ' paste slc_list slc_par_list top_par_list > ' + SLC_Tab
os.system(call_str) rc = os.system(call_str)
if rc != 0:
raise RuntimeError('paste SLC tab failed with rc=' + str(rc))
call_str = 'rm slc_list slc_par_list top_par_list' call_str = 'rm slc_list slc_par_list top_par_list'
os.system(call_str) os.system(call_str)
SLC_list = sorted(glob.glob(work_dir + '/*iw*.slc')) SLC_list = sorted(glob.glob(work_dir + '/*iw*.slc'))
@@ -188,23 +216,42 @@ def main(argv):
SLC_list = sorted(glob.glob(work_dir + '/*IW*.slc')) SLC_list = sorted(glob.glob(work_dir + '/*IW*.slc'))
SLC_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.par')) SLC_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.par'))
TOP_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.TOPS_par')) TOP_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.TOPS_par'))
if not TOP_par_list:
TOP_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.tops_par'))
#call_str = 'rm *iw* *vv.SLC_tab' #call_str = 'rm *iw* *vv.SLC_tab'
#os.system(call_str) #os.system(call_str)
if os.path.isfile(SLC_Tab): if os.path.isfile(SLC_Tab):
os.remove(SLC_Tab) os.remove(SLC_Tab)
if (not SLC_list or not SLC_par_list or not TOP_par_list
or len(SLC_list) != len(SLC_par_list)
or len(SLC_list) != len(TOP_par_list)):
raise RuntimeError('Sentinel-1 concatenated final SLC products are incomplete for ' + date)
for kk in range(len(SLC_list)): for kk in range(len(SLC_list)):
call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' >> ' + SLC_Tab call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' >> ' + SLC_Tab
os.system(call_str) rc = os.system(call_str)
if rc != 0:
raise RuntimeError('failed to append concatenated SLC tab with rc=' + str(rc))
BURST = SLC_par_list[kk].replace('slc.par','burst.par') BURST = SLC_par_list[kk].replace('slc.par','burst.par')
call_str = 'SLC_burst_corners ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' > ' +BURST call_str = 'SLC_burst_corners ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' > ' +BURST
os.system(call_str) rc = os.system(call_str)
if rc != 0:
raise RuntimeError('SLC_burst_corners failed with rc=' + str(rc))
call_str = "echo 'SLC has already down' >down2slc.dat" call_str = "echo 'SLC has already down' >down2slc.dat"
os.system(call_str) os.system(call_str)
TSLC = work_dir + '/' + date + '.slc'
TSLCPar = work_dir + '/' + date + '.slc.par'
if not os.path.isfile(TSLC) or not os.path.isfile(TSLCPar):
call_str = 'SLC_mosaic_ScanSAR ' + SLC_Tab + ' ' + TSLC + ' ' + TSLCPar + ' 10 2'
rc = os.system(call_str)
if rc != 0:
raise RuntimeError('SLC_mosaic_ScanSAR failed with rc=' + str(rc))
print("Down to SLC for %s is done! " % date) print("Down to SLC for %s is done! " % date)
sys.exit(1) sys.exit(0)
if __name__ == '__main__': if __name__ == '__main__':
main(sys.argv[:]) main(sys.argv[:])
+59 -26
View File
@@ -13,6 +13,7 @@ import getopt
import time import time
import glob import glob
import argparse import argparse
from pathlib import Path
from pyint import _utils as ut from pyint import _utils as ut
@@ -23,10 +24,11 @@ def get_s1_date(raw_file):
return date return date
def get_satellite(raw_file): def get_satellite(raw_file):
if 'S1A_IW_SLC_' in raw_file: name = os.path.basename(str(raw_file))
s0 = 'A' if name.startswith('S1') and len(name) >= 3:
s0 = name[2]
else: else:
s0 = 'B' s0 = 'A'
return s0 return s0
@@ -88,16 +90,13 @@ def main(argv):
# os.mkdir(opod_dir) # os.mkdir(opod_dir)
call_str = " eof --save-dir " + opod_dir + " -p " + down_dir if opod_dir and not os.path.isdir(opod_dir):
os.system(call_str) os.makedirs(opod_dir, exist_ok=True)
os.chdir(work_dir) os.chdir(work_dir)
t_date = 't_' + date t_date = 't_' + date
call_str = 'ls ' + down_dir + '/S1*' + date + '*.zip > ' + t_date
os.system(call_str)
start_swath = templateDict['start_swath'] start_swath = templateDict['start_swath']
end_swath = templateDict['end_swath'] end_swath = templateDict['end_swath']
@@ -115,53 +114,87 @@ def main(argv):
# k_swath = '-' # k_swath = '-'
k_swath = ut.get_sardata_swath(start_swath,end_swath) k_swath = ut.get_sardata_swath(start_swath,end_swath)
raw_files = ut.read_txt2list(t_date) raw_files = sorted(glob.glob(down_dir + '/S1*' + date + '*.zip') + glob.glob(down_dir + '/S1*' + date + '*.SAFE'))
if not raw_files:
raise FileNotFoundError('No Sentinel-1 scenes found for date ' + date)
with open(t_date, 'w', encoding='utf-8') as f:
for raw_file in raw_files:
f.write(str(raw_file) + '\n')
satellite = get_satellite(str(raw_files[0])) satellite = get_satellite(str(raw_files[0]))
#orbit_file = ut.download_s1_orbit(date,opod_dir,satellite=satellite) #orbit_file = ut.download_s1_orbit(date,opod_dir,satellite=satellite)
zipfile_ref=str(raw_files[0]) zipfile_ref=str(raw_files[0])
outfile_name=zipfile_ref.split('/')[-1].split('.')[0] outfile_name=zipfile_ref.split('/')[-1].split('.')[0]
burst_number_table_ref=outfile_name + '.BURST_tab' burst_number_table_ref=outfile_name + '.BURST_tab'
call_str = 'S1_BURST_tab_from_zipfile.py 3 --zip_ref_list ' + t_date + ' --zip_list ' + t_date if raw_files and os.path.isdir(str(raw_files[0])):
os.system(call_str) call_str = 'S1_BURST_tab_from_zipfile.py 3 --dir_ref_list ' + t_date + ' --dir_list ' + t_date
else:
call_str = 'S1_BURST_tab_from_zipfile.py 3 --zip_ref_list ' + t_date + ' --zip_list ' + t_date
rc = os.system(call_str)
if rc != 0:
raise RuntimeError('S1_BURST_tab_from_zipfile.py failed with rc=' + str(rc))
# call_str = 'S1_import_SLC_from_zipfiles ' + t_date + ' ' + burst_number_table_ref + ' vv 0 ' + k_swath # call_str = 'S1_import_SLC_from_zipfiles ' + t_date + ' ' + burst_number_table_ref + ' vv 0 ' + k_swath
call_str = 'read_S1_TOPS_SLC.py ' + zipfile_ref + ' --burst_sel ' + burst_number_table_ref + ' --pol vv --root_name ' + date + ' --sw_start ' + start_swath + ' --swn ' + end_swath + ' --OPOD_dir ' + opod_dir call_str = 'read_S1_TOPS_SLC.py ' + zipfile_ref + ' --burst_sel ' + burst_number_table_ref + ' --pol vv --root_name ' + date + ' --sw_start ' + start_swath + ' --swn ' + end_swath + ' --OPOD_dir ' + opod_dir
os.system(call_str) rc = os.system(call_str)
if rc != 0:
raise RuntimeError('read_S1_TOPS_SLC.py failed with rc=' + str(rc))
os.chdir(work_dir) os.chdir(work_dir)
call_str = "rename 's/vv.iw1.slc/IW1.slc/g' *" rename_rules = [
#call_str = "rename vv.slc.iw1 IW1.slc * " ('vv.iw1.slc', 'IW1.slc'),
os.system(call_str) ('vv.iw2.slc', 'IW2.slc'),
call_str = "rename 's/vv.iw2.slc/IW2.slc/g' *" ('vv.iw3.slc', 'IW3.slc'),
#call_str = "rename vv.slc.iw2 IW2.slc * " ('vv.SLC_tab', 'SLC_tab'),
os.system(call_str) ('vv.slc', 'slc'),
call_str = "rename 's/vv.iw3.slc/IW3.slc/g' *" ('tops_par', 'TOPS_par'),
#call_str = "rename vv.slc.iw3 IW3.slc * " ]
os.system(call_str) for src_token, dst_token in rename_rules:
call_str = "rename 's/tops_par/TOPS_par/g' *.tops_par " for candidate in list(Path(work_dir).iterdir()):
os.system(call_str) if src_token not in candidate.name:
continue
target = candidate.with_name(candidate.name.replace(src_token, dst_token))
if target != candidate and not target.exists():
candidate.rename(target)
SLC_Tab = work_dir + '/' + date+'_SLC_Tab' SLC_Tab = work_dir + '/' + date+'_SLC_Tab'
SLC_list = sorted(glob.glob(work_dir + '/*IW*.slc')) SLC_list = sorted(glob.glob(work_dir + '/*IW*.slc'))
SLC_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.par')) SLC_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.par'))
TOP_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.TOPS_par')) TOP_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.TOPS_par'))
if not TOP_par_list:
TOP_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.tops_par'))
if (not SLC_list or not SLC_par_list or not TOP_par_list
or len(SLC_list) != len(SLC_par_list)
or len(SLC_list) != len(TOP_par_list)):
raise RuntimeError('Sentinel-1 SLC generation did not produce expected IW SLC products for ' + date)
if os.path.isfile(SLC_Tab): if os.path.isfile(SLC_Tab):
os.remove(SLC_Tab) os.remove(SLC_Tab)
for kk in range(len(SLC_list)): for kk in range(len(SLC_list)):
call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' >> ' + SLC_Tab call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' >> ' + SLC_Tab
os.system(call_str) rc = os.system(call_str)
if rc != 0:
raise RuntimeError('failed to append SLC tab with rc=' + str(rc))
BURST = SLC_par_list[kk].replace('slc.par','burst.par') BURST = SLC_par_list[kk].replace('slc.par','burst.par')
call_str = 'SLC_burst_corners ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' > ' +BURST call_str = 'SLC_burst_corners ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' > ' +BURST
os.system(call_str) rc = os.system(call_str)
if rc != 0:
raise RuntimeError('SLC_burst_corners failed with rc=' + str(rc))
call_str = "echo 'SLC has already down' >down2slc.dat" call_str = "echo 'SLC has already down' >down2slc.dat"
os.system(call_str) os.system(call_str)
TSLC = work_dir + '/' + date + '.slc'
TSLCPar = work_dir + '/' + date + '.slc.par'
if not os.path.isfile(TSLC) or not os.path.isfile(TSLCPar):
call_str = 'SLC_mosaic_ScanSAR ' + SLC_Tab + ' ' + TSLC + ' ' + TSLCPar + ' 10 2'
rc = os.system(call_str)
if rc != 0:
raise RuntimeError('SLC_mosaic_ScanSAR failed with rc=' + str(rc))
print("Down to SLC for %s is done! " % date) print("Down to SLC for %s is done! " % date)
sys.exit(1) sys.exit(0)
if __name__ == '__main__': if __name__ == '__main__':
main(sys.argv[:]) main(sys.argv[:])
+25 -4
View File
@@ -32,9 +32,27 @@ def work(data0):
stderr = p.stderr stderr = p.stderr
if type(stderr) == bytes: if type(stderr) == bytes:
aa=stderr.decode("utf-8") aa=stderr.decode("utf-8", errors="replace")
else: else:
aa = stderr aa = stderr
if type(stdout) == bytes:
bb=stdout.decode("utf-8", errors="replace")
else:
bb = stdout
if p.returncode != 0:
detail_parts = []
if bb:
detail_parts.append(bb)
if aa:
detail_parts.append(aa)
detail = '\n'.join(part.strip() for part in detail_parts if part and part.strip())
str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n'
with open(err_txt, 'a') as f:
f.write(str0)
if detail:
f.write(detail)
f.write('\n')
raise RuntimeError(str0.strip() + ' failed with rc=' + str(p.returncode) + ('\n' + detail if detail else ''))
if aa: if aa:
str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n'
@@ -83,7 +101,7 @@ def main(argv):
projectDir = scratchDir + '/' + projectName projectDir = scratchDir + '/' + projectName
downDir = scratchDir + '/' + projectName + "/DOWNLOAD" downDir = scratchDir + '/' + projectName + "/DOWNLOAD"
slcDir = scratchDir + '/' + projectName + "/SLC" slcDir = scratchDir + '/' + projectName + "/SLC"
raw_file_list = glob.glob(downDir + '/S1*.zip') raw_file_list = glob.glob(downDir + '/S1*.zip') + glob.glob(downDir + '/S1*.SAFE')
templateDir = os.getenv('TEMPLATEDIR') templateDir = os.getenv('TEMPLATEDIR')
templateFile = templateDir + "/" + projectName + ".template" templateFile = templateDir + "/" + projectName + ".template"
templateDict=ut.update_template(templateFile) templateDict=ut.update_template(templateFile)
@@ -139,12 +157,15 @@ def main(argv):
k00 = 0 k00 = 0
if k00==0: if k00==0:
data_para.append(data0) data_para.append(data0)
ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) results = ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False)
failures = [str(item) for item in results if isinstance(item, Exception)]
if failures:
raise RuntimeError('\n\n'.join(failures))
os.chdir(downDir) os.chdir(downDir)
print("Down to SLC for project %s is done! " % projectName) print("Down to SLC for project %s is done! " % projectName)
ut.print_process_time(start_time, time.time()) ut.print_process_time(start_time, time.time())
sys.exit(1) sys.exit(0)
if __name__ == '__main__': if __name__ == '__main__':
main(sys.argv[:]) main(sys.argv[:])
+1 -1
View File
@@ -291,7 +291,7 @@ def main(argv):
iw3 = Sdate + '.IW3.slc*' iw3 = Sdate + '.IW3.slc*'
call_str = 'rm '+ iw1 + ' ' + iw2 + ' '+ iw3 call_str = 'rm '+ iw1 + ' ' + iw2 + ' '+ iw3
os.system(call_str) os.system(call_str)
sys.exit(1) sys.exit(0)
if __name__ == '__main__': if __name__ == '__main__':
main(sys.argv[:]) main(sys.argv[:])
+30 -3
View File
@@ -26,9 +26,27 @@ def work(data0):
stderr = p.stderr stderr = p.stderr
if type(stderr) == bytes: if type(stderr) == bytes:
aa=stderr.decode("utf-8") aa=stderr.decode("utf-8", errors="replace")
else: else:
aa = stderr aa = stderr
if type(stdout) == bytes:
bb=stdout.decode("utf-8", errors="replace")
else:
bb = stdout
if p.returncode != 0:
detail_parts = []
if bb:
detail_parts.append(bb)
if aa:
detail_parts.append(aa)
detail = '\n'.join(part.strip() for part in detail_parts if part and part.strip())
str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n'
with open(err_file, 'a') as f:
f.write(str0)
if detail:
f.write(detail)
f.write('\n')
raise RuntimeError(str0.strip() + ' failed with rc=' + str(p.returncode) + ('\n' + detail if detail else ''))
if aa: if aa:
str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n'
@@ -75,21 +93,30 @@ def main(argv):
projectDir = scratchDir + '/' + projectName projectDir = scratchDir + '/' + projectName
slcDir = scratchDir + '/' + projectName + '/SLC' slcDir = scratchDir + '/' + projectName + '/SLC'
slc_list = [os.path.basename(fname) for fname in sorted(glob.glob(slcDir + '/*'))] slc_list = [os.path.basename(fname) for fname in sorted(glob.glob(slcDir + '/*'))]
templateDir = os.getenv('TEMPLATEDIR')
templateFile = templateDir + "/" + projectName + ".template"
templateDict=ut.update_template(templateFile)
master_date = templateDict['masterDate']
err_txt = scratchDir + '/' + projectName + '/extract_s1_bursts_all.err' err_txt = scratchDir + '/' + projectName + '/extract_s1_bursts_all.err'
if os.path.isfile(err_txt): os.remove(err_txt) if os.path.isfile(err_txt): os.remove(err_txt)
data_para = [] data_para = []
for i in range(len(slc_list)): for i in range(len(slc_list)):
if slc_list[i] == master_date:
continue
cmd0 = ['extract_s1_bursts.py',projectName,slc_list[i]] cmd0 = ['extract_s1_bursts.py',projectName,slc_list[i]]
data0 = [cmd0,err_txt] data0 = [cmd0,err_txt]
data_para.append(data0) data_para.append(data0)
ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) results = ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False)
failures = [str(item) for item in results if isinstance(item, Exception)]
if failures:
raise RuntimeError('\n\n'.join(failures))
print("Extract TOPS bursts for project %s is done! " % projectName) print("Extract TOPS bursts for project %s is done! " % projectName)
ut.print_process_time(start_time, time.time()) ut.print_process_time(start_time, time.time())
sys.exit(1) sys.exit(0)
if __name__ == '__main__': if __name__ == '__main__':
main(sys.argv[:]) main(sys.argv[:])
+1 -2
View File
@@ -83,7 +83,7 @@ def main(argv):
wavelength = 0.0312283810417 wavelength = 0.0312283810417
elif satelite == 'TSX': elif satelite == 'TSX':
wavelength = 0.03106657823461874 wavelength = 0.03106657823461874
elif satelite == 'S1A': elif str(satelite).startswith('S1'):
wavelength = 0.0554657647 wavelength = 0.0554657647
elif satelite == 'ALOS2': elif satelite == 'ALOS2':
wavelength = 0.2424525 wavelength = 0.2424525
@@ -106,4 +106,3 @@ def main(argv):
if __name__ == '__main__': if __name__ == '__main__':
main(sys.argv[:]) main(sys.argv[:])
+1 -1
View File
@@ -105,7 +105,7 @@ def main(argv):
print("Generate differential interferograms for project %s is done! " % projectName) print("Generate differential interferograms for project %s is done! " % projectName)
ut.print_process_time(start_time, time.time()) ut.print_process_time(start_time, time.time())
sys.exit(1) sys.exit(0)
if __name__ == '__main__': if __name__ == '__main__':
main(sys.argv[:]) main(sys.argv[:])
+1 -1
View File
@@ -210,7 +210,7 @@ def main(argv):
if templateDict['raw2slc_all'] == '1': # only for S1 data now if templateDict['raw2slc_all'] == '1': # only for S1 data now
print('Start to convert downloaded-raw data into SLC ...') print('Start to convert downloaded-raw data into SLC ...')
print('Number of processor: %s' % str(templateDict['raw2slc_all_parallel'])) print('Number of processor: %s' % str(templateDict['raw2slc_all_parallel']))
if satelite=='S1A': if str(satelite).startswith('S1'):
call_str = 'down2slc_sen_all.py ' + projectName + ' --parallel ' + templateDict['raw2slc_all_parallel'] call_str = 'down2slc_sen_all.py ' + projectName + ' --parallel ' + templateDict['raw2slc_all_parallel']
_run_or_raise(call_str, 'raw2slc_s1') _run_or_raise(call_str, 'raw2slc_s1')
elif satelite=='ALOS': elif satelite=='ALOS':
+1 -1
View File
@@ -159,7 +159,7 @@ def main(argv):
call_str = 'SLC_burst_corners ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' > ' +BURST call_str = 'SLC_burst_corners ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' > ' +BURST
os.system(call_str) os.system(call_str)
print("Down to SLC for %s is done! " % date) print("Down to SLC for %s is done! " % date)
sys.exit(1) sys.exit(0)
if __name__ == '__main__': if __name__ == '__main__':
main(sys.argv[:]) main(sys.argv[:])