Add Sentinel-1 asset management and PyINT pipeline support
This commit is contained in:
@@ -55,9 +55,12 @@ ALLOWED_EXPORT_DIRS=
|
||||
# 源数据目录
|
||||
# -----------------------------------------------------------------------------
|
||||
UNPACK_SOURCE_DIRS=D:\Archives
|
||||
SOURCE_PRODUCT_DIRS=D:\LuTan1_Image_Pool;D:\Sentinel1_Image_Pool_ZIP
|
||||
SENTINEL1_STORAGE_DIRS=D:\Sentinel1_Image_Pool
|
||||
INSAR_STORAGE_DIRS=D:\LuTan1_Image_Pool
|
||||
MONITOR_RADAR_DIRS=D:\LuTan1_Image_Pool
|
||||
MONITOR_DINSAR_DIRS=D:\DInSARResult
|
||||
ORBIT_SOURCE_DIRS=D:\LT1_data_lsarorbit;D:\Sentinel1_EOF_Pool
|
||||
MONITOR_ORBIT_DIR=D:\LT1_data_lsarorbit
|
||||
|
||||
GF3_SOURCE_DIRS=D:\GF3_L1A_Image
|
||||
|
||||
@@ -167,6 +167,9 @@ class Settings(BaseSettings):
|
||||
DB_SCHEMA_RESET_CONFIRM: bool = False
|
||||
|
||||
UNPACK_SOURCE_DIRS: str = ""
|
||||
SOURCE_PRODUCT_DIRS: str = ""
|
||||
SENTINEL1_STORAGE_DIRS: str = ""
|
||||
ORBIT_SOURCE_DIRS: str = ""
|
||||
INSAR_STORAGE_DIRS: str = ""
|
||||
MONITOR_RADAR_DIRS: str = ""
|
||||
MONITOR_DINSAR_DIRS: str = ""
|
||||
@@ -859,6 +862,13 @@ def validate_runtime_config() -> dict[str, Any]:
|
||||
_check_path(label="SRTM_DEM_DIR", value=settings.SRTM_DEM_DIR, errors=errors, warnings=warnings, expect_file=False)
|
||||
_check_path(label="WATER_RESULTS_DIR", value=settings.WATER_RESULTS_DIR, errors=errors, warnings=warnings, expect_file=False)
|
||||
_check_path(label="MONITOR_ORBIT_DIR", value=settings.MONITOR_ORBIT_DIR, errors=errors, warnings=warnings, expect_file=False)
|
||||
for label, value in (
|
||||
("SOURCE_PRODUCT_DIRS", settings.SOURCE_PRODUCT_DIRS),
|
||||
("SENTINEL1_STORAGE_DIRS", settings.SENTINEL1_STORAGE_DIRS),
|
||||
("ORBIT_SOURCE_DIRS", settings.ORBIT_SOURCE_DIRS),
|
||||
):
|
||||
for item in split_env_paths(value):
|
||||
_check_path(label=label, value=item, errors=errors, warnings=warnings, expect_file=False)
|
||||
_check_path(label="ORBIT_POOL_ENVI", value=settings.ORBIT_POOL_ENVI, errors=errors, warnings=warnings, expect_file=False)
|
||||
_check_path(label="ORBIT_POOL_ISCE2", value=settings.ORBIT_POOL_ISCE2, errors=errors, warnings=warnings, expect_file=False)
|
||||
_check_path(label="RESULT_PUBLISH_ROOT", value=settings.RESULT_PUBLISH_ROOT, errors=errors, warnings=warnings, expect_file=False)
|
||||
|
||||
@@ -36,6 +36,7 @@ MIGRATION_FILES = [
|
||||
"007_timeseries_stack_plan_trace.sql",
|
||||
"008_timeseries_stack_plan_edges.sql",
|
||||
"009_raw_source_pairing_fields.sql",
|
||||
"010_source_orbit_asset_inventory.sql",
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ from typing import Any, Dict, List
|
||||
|
||||
from ..config import get_env_text, read_bool_env, settings
|
||||
from .base import DinsarEngine, EngineAvailability, EngineProfile, RunRequest, RunResult
|
||||
from ..utils import normalize_satellite_family
|
||||
from ..services.dinsar_naming import (
|
||||
PAIR_META_FILENAME,
|
||||
build_fallback_pair_key,
|
||||
@@ -728,7 +729,14 @@ class Isce2Engine(DinsarEngine):
|
||||
task_name = os.path.basename(os.path.normpath(task_dir))
|
||||
pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {}
|
||||
task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name
|
||||
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(task_alias, task_dir)
|
||||
satellite_family = normalize_satellite_family(
|
||||
pair_meta.get("master_satellite") or pair_meta.get("slave_satellite")
|
||||
)
|
||||
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(
|
||||
task_alias,
|
||||
task_dir,
|
||||
satellite_family=satellite_family,
|
||||
)
|
||||
pointer_path = os.path.join(
|
||||
settings.DINSAR_PRODUCT_DIR,
|
||||
pair_key,
|
||||
@@ -1085,7 +1093,14 @@ class Isce2Engine(DinsarEngine):
|
||||
task_name = os.path.basename(os.path.normpath(task_dir))
|
||||
pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {}
|
||||
task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name
|
||||
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(task_alias, task_dir)
|
||||
satellite_family = normalize_satellite_family(
|
||||
pair_meta.get("master_satellite") or pair_meta.get("slave_satellite")
|
||||
)
|
||||
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(
|
||||
task_alias,
|
||||
task_dir,
|
||||
satellite_family=satellite_family,
|
||||
)
|
||||
output_root = self._output_root or os.path.join(task_dir, "isce2_output")
|
||||
run_dir = managed_run_dir_override or os.path.normpath(
|
||||
os.path.join(output_root, pair_key, "runs", run_key)
|
||||
|
||||
@@ -43,6 +43,7 @@ from ..services.pyint_service import (
|
||||
REFLATTEN_MODEL_CHOICES,
|
||||
TARGET_GRID_SIZE_MAX_M,
|
||||
TARGET_GRID_SIZE_MIN_M,
|
||||
build_profile_project_name,
|
||||
build_project_name,
|
||||
calculate_dem_oversampling,
|
||||
calculate_looks_from_task_dir,
|
||||
@@ -290,13 +291,13 @@ class PyintEngine(DinsarEngine):
|
||||
)
|
||||
return _windows_path_to_wsl_mount(str(local_script))
|
||||
|
||||
def _pipeline_script_for_profile(self, profile: str) -> str:
|
||||
script_name = "run_s1_pyint_pipeline.py" if str(profile or "").strip() == "s1_gamma_dinsar" else "run_lt1_pyint_pipeline.py"
|
||||
local_script = Path(__file__).resolve().parent.parent / "pyint_pipeline" / script_name
|
||||
return _windows_path_to_wsl_mount(str(local_script))
|
||||
|
||||
def get_profiles(self) -> List[EngineProfile]:
|
||||
return [
|
||||
EngineProfile(
|
||||
code="lt1_gamma_dinsar",
|
||||
label="LT-1 Gamma D-InSAR",
|
||||
description="Use PyINT + Gamma in WSL for single-pair LT-1 D-InSAR processing.",
|
||||
params_schema={
|
||||
shared_schema = {
|
||||
"force": {
|
||||
"label": "强制重跑",
|
||||
"type": "boolean",
|
||||
@@ -459,7 +460,19 @@ class PyintEngine(DinsarEngine):
|
||||
"section": "Execution",
|
||||
"description": "关闭后不导出地理编码结果。",
|
||||
},
|
||||
},
|
||||
}
|
||||
return [
|
||||
EngineProfile(
|
||||
code="lt1_gamma_dinsar",
|
||||
label="LT-1 Gamma D-InSAR",
|
||||
description="Use PyINT + Gamma in WSL for single-pair LT-1 D-InSAR processing.",
|
||||
params_schema=shared_schema,
|
||||
),
|
||||
EngineProfile(
|
||||
code="s1_gamma_dinsar",
|
||||
label="Sentinel-1 Gamma D-InSAR",
|
||||
description="Use PyINT + Gamma in WSL for single-pair Sentinel-1 D-InSAR processing.",
|
||||
params_schema=shared_schema,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -694,7 +707,7 @@ class PyintEngine(DinsarEngine):
|
||||
error="PyINT is disabled.",
|
||||
)
|
||||
|
||||
if request.profile != "lt1_gamma_dinsar":
|
||||
if request.profile not in {"lt1_gamma_dinsar", "s1_gamma_dinsar"}:
|
||||
return RunResult(
|
||||
success=False,
|
||||
engine_code=self.engine_code,
|
||||
@@ -859,7 +872,11 @@ class PyintEngine(DinsarEngine):
|
||||
else os.path.join(run_dir, "native")
|
||||
)
|
||||
template_root = os.path.normpath(os.path.join(self._template_root, pair_key, run_key))
|
||||
project_name = build_project_name(pair_key, run_key)
|
||||
project_name = build_profile_project_name(
|
||||
task_identity.get("satellite_family"),
|
||||
pair_key,
|
||||
run_key,
|
||||
)
|
||||
project_dir = os.path.join(work_run_root, project_name)
|
||||
# Keep input assets outside the run root because the WSL pipeline may delete run_root on --force.
|
||||
input_assets_dir = os.path.join(self._work_root, pair_key, "input_assets", run_key)
|
||||
@@ -1103,7 +1120,7 @@ class PyintEngine(DinsarEngine):
|
||||
)
|
||||
|
||||
cmd_parts = [
|
||||
f"{quote_shell(self._python)} {quote_shell(self._pipeline_script)} {quote_shell(wsl_task_dir)}",
|
||||
f"{quote_shell(self._python)} {quote_shell(self._pipeline_script_for_profile(request.profile))} {quote_shell(wsl_task_dir)}",
|
||||
f"--project-dir {quote_shell(wsl_project_dir)}",
|
||||
f"--template-root {quote_shell(wsl_template_root)}",
|
||||
f"--output-dir {quote_shell(wsl_output_dir)}",
|
||||
@@ -1486,6 +1503,9 @@ class PyintEngine(DinsarEngine):
|
||||
|
||||
@staticmethod
|
||||
def _discover_archives(task_dir: str) -> Dict[str, List[str]]:
|
||||
from ..services.pyint_service import discover_lt1_archives
|
||||
from ..services.pyint_service import discover_lt1_archives, discover_s1_scene_sources, infer_task_identity
|
||||
|
||||
task_identity = infer_task_identity(task_dir)
|
||||
if str(task_identity.get("satellite_family") or "").strip().upper() == "S1":
|
||||
return discover_s1_scene_sources(task_dir)
|
||||
return discover_lt1_archives(task_dir)
|
||||
|
||||
@@ -26,6 +26,12 @@ from .orm import (
|
||||
ManagedRootORM,
|
||||
ScanCursorORM,
|
||||
PathInventoryORM,
|
||||
SourceProductAssetORM,
|
||||
OrbitAssetORM,
|
||||
SceneOrbitBindingORM,
|
||||
OrbitAssetDerivativeORM,
|
||||
AssetInventoryStateORM,
|
||||
AssetInventoryIssueORM,
|
||||
WorkflowDefORM,
|
||||
WorkflowRunORM,
|
||||
WorkflowStepORM,
|
||||
@@ -94,6 +100,8 @@ __all__ = [
|
||||
"TimeseriesStackPlanORM", "TimeseriesStackPlanItemORM", "TimeseriesStackPlanEdgeORM",
|
||||
"SystemTaskORM", "TaskLogORM", "SystemJobORM", "ScanStateORM",
|
||||
"ManagedRootORM", "ScanCursorORM", "PathInventoryORM",
|
||||
"SourceProductAssetORM", "OrbitAssetORM", "SceneOrbitBindingORM",
|
||||
"OrbitAssetDerivativeORM", "AssetInventoryStateORM", "AssetInventoryIssueORM",
|
||||
"WorkflowDefORM", "WorkflowRunORM", "WorkflowStepORM", "WorkflowArtifactORM",
|
||||
"SystemWorkerHeartbeatORM",
|
||||
"DinsarTaskBatchORM", "DinsarTaskItemORM",
|
||||
|
||||
@@ -41,6 +41,32 @@ class RadarDataORM(Base):
|
||||
product_unique_id = Column(String, nullable=True)
|
||||
satellite_family = Column(String, index=True, nullable=True)
|
||||
look_direction = Column(String, index=True, nullable=True)
|
||||
acquisition_start_time_utc = Column(DateTime, nullable=True, index=True)
|
||||
acquisition_stop_time_utc = Column(DateTime, nullable=True)
|
||||
absolute_orbit = Column(String, nullable=True, index=True)
|
||||
relative_orbit = Column(String, nullable=True, index=True)
|
||||
source_format = Column(String(32), nullable=True, index=True)
|
||||
source_product_ref_id = Column(
|
||||
Integer,
|
||||
ForeignKey("source_product_assets.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
index=True,
|
||||
)
|
||||
source_archive_asset_id = Column(
|
||||
Integer,
|
||||
ForeignKey("source_product_assets.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
index=True,
|
||||
)
|
||||
selected_orbit_asset_id = Column(
|
||||
Integer,
|
||||
ForeignKey("orbit_assets.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
index=True,
|
||||
)
|
||||
orbit_binding_status = Column(String(32), nullable=False, default="UNBOUND", server_default="UNBOUND", index=True)
|
||||
orbit_binding_reason = Column(Text, nullable=True)
|
||||
metadata_json = Column(JSON, nullable=True)
|
||||
geocoded_flag = Column(Boolean, nullable=True)
|
||||
insar_source_ready = Column(Boolean, nullable=False, default=False, server_default="false")
|
||||
insar_source_reason = Column(Text, nullable=True)
|
||||
@@ -63,6 +89,10 @@ class RadarDataORM(Base):
|
||||
preview_cache_updated_at = Column(DateTime, nullable=True)
|
||||
preview_cache_error = Column(Text, nullable=True)
|
||||
|
||||
source_product_asset = relationship("SourceProductAssetORM", foreign_keys=[source_product_ref_id])
|
||||
source_archive_asset = relationship("SourceProductAssetORM", foreign_keys=[source_archive_asset_id])
|
||||
selected_orbit_asset = relationship("OrbitAssetORM", foreign_keys=[selected_orbit_asset_id])
|
||||
|
||||
|
||||
class DinsarResultORM(Base):
|
||||
__tablename__ = 'dinsar_results'
|
||||
@@ -742,6 +772,11 @@ class ManagedRootORM(Base):
|
||||
back_populates="root",
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
asset_inventory_states = relationship(
|
||||
"AssetInventoryStateORM",
|
||||
back_populates="root",
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_managed_roots_role_enabled", "root_role", "enabled"),
|
||||
@@ -809,6 +844,215 @@ class PathInventoryORM(Base):
|
||||
)
|
||||
|
||||
|
||||
class SourceProductAssetORM(Base):
|
||||
__tablename__ = "source_product_assets"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
asset_uid = Column(String(128), unique=True, index=True, nullable=False)
|
||||
logical_product_uid = Column(String(128), index=True, nullable=True)
|
||||
satellite_family = Column(String(32), index=True, nullable=True)
|
||||
satellite = Column(String(32), index=True, nullable=True)
|
||||
source_format = Column(String(32), index=True, nullable=False)
|
||||
product_type = Column(String(64), nullable=True)
|
||||
product_level = Column(String(64), nullable=True)
|
||||
imaging_mode = Column(String(64), nullable=True)
|
||||
polarization = Column(String(64), nullable=True)
|
||||
absolute_orbit = Column(String(64), index=True, nullable=True)
|
||||
relative_orbit = Column(String(64), index=True, nullable=True)
|
||||
orbit_direction = Column(String(32), index=True, nullable=True)
|
||||
acquisition_start_time_utc = Column(DateTime, index=True, nullable=True)
|
||||
acquisition_stop_time_utc = Column(DateTime, nullable=True)
|
||||
imaging_date = Column(String(8), index=True, nullable=True)
|
||||
root_ref_id = Column(Integer, ForeignKey("managed_roots.id", ondelete="SET NULL"), index=True, nullable=True)
|
||||
root_path = Column(String, nullable=True)
|
||||
file_path = Column(String, unique=True, index=True, nullable=False)
|
||||
archive_path = Column(String, nullable=True)
|
||||
path_kind = Column(String(24), nullable=True)
|
||||
file_name = Column(String(255), nullable=True)
|
||||
file_stem = Column(String(255), nullable=True)
|
||||
file_ext = Column(String(32), nullable=True)
|
||||
size_bytes = Column(BigInteger, nullable=True)
|
||||
mtime_epoch = Column(Float, nullable=True)
|
||||
checksum_sha256 = Column(String(64), nullable=True)
|
||||
checksum_status = Column(String(32), nullable=False, default="NOT_COMPUTED", server_default="NOT_COMPUTED")
|
||||
parser_name = Column(String(64), nullable=True)
|
||||
parser_version = Column(String(32), nullable=True)
|
||||
parse_status = Column(String(32), nullable=False, default="PENDING", server_default="PENDING", index=True)
|
||||
parse_error = Column(Text, nullable=True)
|
||||
parsed_at = Column(DateTime, nullable=True)
|
||||
metadata_json = Column(JSON, nullable=True)
|
||||
is_active = Column(Boolean, nullable=False, default=True, server_default="true", index=True)
|
||||
missing_since = Column(DateTime, nullable=True)
|
||||
created_at = Column(DateTime, server_default=func.now(), nullable=False)
|
||||
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
|
||||
|
||||
root = relationship("ManagedRootORM")
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_source_product_assets_family_date", "satellite_family", "imaging_date"),
|
||||
Index("idx_source_product_assets_logical_product", "logical_product_uid"),
|
||||
Index("idx_source_product_assets_root_active", "root_ref_id", "is_active"),
|
||||
)
|
||||
|
||||
|
||||
class OrbitAssetORM(Base):
|
||||
__tablename__ = "orbit_assets"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
orbit_uid = Column(String(128), unique=True, index=True, nullable=False)
|
||||
satellite_family = Column(String(32), index=True, nullable=True)
|
||||
satellite = Column(String(32), index=True, nullable=True)
|
||||
orbit_type = Column(String(64), index=True, nullable=False)
|
||||
native_format = Column(String(32), index=True, nullable=False)
|
||||
quality_class = Column(String(32), index=True, nullable=False, default="unknown", server_default="unknown")
|
||||
root_ref_id = Column(Integer, ForeignKey("managed_roots.id", ondelete="SET NULL"), index=True, nullable=True)
|
||||
root_path = Column(String, nullable=True)
|
||||
file_path = Column(String, unique=True, index=True, nullable=False)
|
||||
file_name = Column(String(255), nullable=True)
|
||||
file_stem = Column(String(255), nullable=True)
|
||||
file_ext = Column(String(32), nullable=True)
|
||||
size_bytes = Column(BigInteger, nullable=True)
|
||||
mtime_epoch = Column(Float, nullable=True)
|
||||
checksum_sha256 = Column(String(64), nullable=True)
|
||||
checksum_status = Column(String(32), nullable=False, default="NOT_COMPUTED", server_default="NOT_COMPUTED")
|
||||
validity_start_time_utc = Column(DateTime, index=True, nullable=True)
|
||||
validity_stop_time_utc = Column(DateTime, index=True, nullable=True)
|
||||
generation_time_utc = Column(DateTime, nullable=True)
|
||||
published_time_utc = Column(DateTime, nullable=True)
|
||||
parser_name = Column(String(64), nullable=True)
|
||||
parser_version = Column(String(32), nullable=True)
|
||||
parse_status = Column(String(32), nullable=False, default="PENDING", server_default="PENDING", index=True)
|
||||
parse_error = Column(Text, nullable=True)
|
||||
parsed_at = Column(DateTime, nullable=True)
|
||||
metadata_json = Column(JSON, nullable=True)
|
||||
is_active = Column(Boolean, nullable=False, default=True, server_default="true", index=True)
|
||||
missing_since = Column(DateTime, nullable=True)
|
||||
created_at = Column(DateTime, server_default=func.now(), nullable=False)
|
||||
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
|
||||
|
||||
root = relationship("ManagedRootORM")
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_orbit_assets_family_sat_window", "satellite_family", "satellite", "validity_start_time_utc", "validity_stop_time_utc"),
|
||||
Index("idx_orbit_assets_root_active", "root_ref_id", "is_active"),
|
||||
)
|
||||
|
||||
|
||||
class SceneOrbitBindingORM(Base):
|
||||
__tablename__ = "scene_orbit_bindings"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
radar_data_id = Column(Integer, ForeignKey("radar_data.id", ondelete="CASCADE"), index=True, nullable=False)
|
||||
orbit_asset_id = Column(Integer, ForeignKey("orbit_assets.id", ondelete="CASCADE"), index=True, nullable=False)
|
||||
binding_role = Column(String(32), nullable=False, default="primary_orbit", server_default="primary_orbit")
|
||||
match_status = Column(String(32), nullable=False, default="CANDIDATE", server_default="CANDIDATE", index=True)
|
||||
selection_status = Column(String(32), nullable=False, default="CANDIDATE", server_default="CANDIDATE", index=True)
|
||||
selection_rank = Column(Integer, nullable=True)
|
||||
priority_score = Column(Float, nullable=True)
|
||||
coverage_margin_before_seconds = Column(Float, nullable=True)
|
||||
coverage_margin_after_seconds = Column(Float, nullable=True)
|
||||
match_rule_version = Column(String(64), nullable=True)
|
||||
match_reason = Column(Text, nullable=True)
|
||||
selected_at = Column(DateTime, nullable=True)
|
||||
metadata_json = Column(JSON, nullable=True)
|
||||
created_at = Column(DateTime, server_default=func.now(), nullable=False)
|
||||
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
|
||||
|
||||
radar_data = relationship("RadarDataORM")
|
||||
orbit_asset = relationship("OrbitAssetORM")
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("radar_data_id", "orbit_asset_id", "binding_role", name="uq_scene_orbit_binding_role"),
|
||||
Index("idx_scene_orbit_bindings_scene_selected", "radar_data_id", "selection_status"),
|
||||
)
|
||||
|
||||
|
||||
class OrbitAssetDerivativeORM(Base):
|
||||
__tablename__ = "orbit_asset_derivatives"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
orbit_asset_id = Column(Integer, ForeignKey("orbit_assets.id", ondelete="CASCADE"), index=True, nullable=False)
|
||||
engine_code = Column(String(32), index=True, nullable=False)
|
||||
derivative_format = Column(String(32), nullable=False)
|
||||
derivative_role = Column(String(64), nullable=True)
|
||||
pool_path = Column(String, index=True, nullable=False)
|
||||
size_bytes = Column(BigInteger, nullable=True)
|
||||
mtime_epoch = Column(Float, nullable=True)
|
||||
checksum_sha256 = Column(String(64), nullable=True)
|
||||
generation_status = Column(String(32), nullable=False, default="PENDING", server_default="PENDING", index=True)
|
||||
generation_error = Column(Text, nullable=True)
|
||||
generated_at = Column(DateTime, nullable=True)
|
||||
metadata_json = Column(JSON, nullable=True)
|
||||
created_at = Column(DateTime, server_default=func.now(), nullable=False)
|
||||
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
|
||||
|
||||
orbit_asset = relationship("OrbitAssetORM")
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("orbit_asset_id", "engine_code", "derivative_format", "pool_path", name="uq_orbit_asset_derivative_pool_path"),
|
||||
Index("idx_orbit_asset_derivatives_asset_engine", "orbit_asset_id", "engine_code"),
|
||||
)
|
||||
|
||||
|
||||
class AssetInventoryStateORM(Base):
|
||||
__tablename__ = "asset_inventory_states"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
root_ref_id = Column(Integer, ForeignKey("managed_roots.id", ondelete="CASCADE"), index=True, nullable=False)
|
||||
inventory_type = Column(String(32), index=True, nullable=False)
|
||||
root_path = Column(String, nullable=False)
|
||||
scan_mode = Column(String(32), nullable=False, default="file_pool", server_default="file_pool")
|
||||
status = Column(String(32), nullable=False, default="NEVER_SCANNED", server_default="NEVER_SCANNED", index=True)
|
||||
last_scan_started_at = Column(DateTime, nullable=True)
|
||||
last_scan_finished_at = Column(DateTime, nullable=True)
|
||||
last_seen_entry_count = Column(Integer, nullable=True)
|
||||
last_asset_count = Column(Integer, nullable=True)
|
||||
last_issue_count = Column(Integer, nullable=True)
|
||||
parser_version = Column(String(32), nullable=True)
|
||||
needs_rescan = Column(Boolean, nullable=False, default=True, server_default="true", index=True)
|
||||
last_error = Column(Text, nullable=True)
|
||||
metadata_json = Column(JSON, nullable=True)
|
||||
created_at = Column(DateTime, server_default=func.now(), nullable=False)
|
||||
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
|
||||
|
||||
root = relationship("ManagedRootORM", back_populates="asset_inventory_states")
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("root_ref_id", "inventory_type", name="uq_asset_inventory_state_root_type"),
|
||||
Index("idx_asset_inventory_states_type_status", "inventory_type", "status"),
|
||||
)
|
||||
|
||||
|
||||
class AssetInventoryIssueORM(Base):
|
||||
__tablename__ = "asset_inventory_issues"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
root_ref_id = Column(Integer, ForeignKey("managed_roots.id", ondelete="SET NULL"), index=True, nullable=True)
|
||||
inventory_type = Column(String(32), index=True, nullable=False)
|
||||
asset_ref_id = Column(Integer, ForeignKey("source_product_assets.id", ondelete="SET NULL"), index=True, nullable=True)
|
||||
radar_data_id = Column(Integer, ForeignKey("radar_data.id", ondelete="SET NULL"), index=True, nullable=True)
|
||||
orbit_asset_id = Column(Integer, ForeignKey("orbit_assets.id", ondelete="SET NULL"), index=True, nullable=True)
|
||||
severity = Column(String(16), nullable=False, default="warning", server_default="warning", index=True)
|
||||
issue_code = Column(String(64), index=True, nullable=False)
|
||||
issue_message = Column(Text, nullable=True)
|
||||
source_path = Column(String, nullable=True)
|
||||
status = Column(String(16), nullable=False, default="OPEN", server_default="OPEN", index=True)
|
||||
first_seen_at = Column(DateTime, server_default=func.now(), nullable=False)
|
||||
last_seen_at = Column(DateTime, server_default=func.now(), nullable=False)
|
||||
resolved_at = Column(DateTime, nullable=True)
|
||||
metadata_json = Column(JSON, nullable=True)
|
||||
|
||||
root = relationship("ManagedRootORM")
|
||||
source_asset = relationship("SourceProductAssetORM", foreign_keys=[asset_ref_id])
|
||||
radar_data = relationship("RadarDataORM")
|
||||
orbit_asset = relationship("OrbitAssetORM")
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_asset_inventory_issues_open", "status", "severity"),
|
||||
Index("idx_asset_inventory_issues_root_type", "root_ref_id", "inventory_type"),
|
||||
)
|
||||
|
||||
|
||||
class WorkflowDefORM(Base):
|
||||
"""Workflow definition (DAG template)."""
|
||||
__tablename__ = "workflow_defs"
|
||||
|
||||
@@ -192,6 +192,17 @@ class RadarData(BaseModel):
|
||||
product_unique_id: Optional[str] = None
|
||||
satellite_family: Optional[str] = None
|
||||
look_direction: Optional[str] = None
|
||||
acquisition_start_time_utc: Optional[datetime] = None
|
||||
acquisition_stop_time_utc: Optional[datetime] = None
|
||||
absolute_orbit: Optional[str] = None
|
||||
relative_orbit: Optional[str] = None
|
||||
source_format: Optional[str] = None
|
||||
source_product_ref_id: Optional[int] = None
|
||||
source_archive_asset_id: Optional[int] = None
|
||||
selected_orbit_asset_id: Optional[int] = None
|
||||
orbit_binding_status: str = "UNBOUND"
|
||||
orbit_binding_reason: Optional[str] = None
|
||||
metadata_json: Optional[Dict[str, Any]] = None
|
||||
geocoded_flag: Optional[bool] = None
|
||||
insar_source_ready: bool = False
|
||||
insar_source_reason: Optional[str] = None
|
||||
|
||||
@@ -0,0 +1,651 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
try:
|
||||
from .run_lt1_pyint_pipeline import (
|
||||
DEFAULT_DERAMP_MODE,
|
||||
DEFAULT_REFERENCE_MODE,
|
||||
DEFAULT_REFLATTEN_AZIMUTH_STEP,
|
||||
DEFAULT_REFLATTEN_COH_THRESHOLD,
|
||||
DEFAULT_REFLATTEN_FALLBACK_COH_THRESHOLD,
|
||||
DEFAULT_REFLATTEN_MODEL,
|
||||
DEFAULT_REFLATTEN_RANGE_STEP,
|
||||
assert_output_sanity,
|
||||
assert_required_outputs,
|
||||
calculate_dem_oversampling,
|
||||
collect_expected_outputs,
|
||||
collect_output_sanity_checks,
|
||||
collect_stage_error_logs,
|
||||
copy_native_outputs,
|
||||
ensure_directory,
|
||||
export_standard_products,
|
||||
format_gamma_number,
|
||||
hardlink_or_copy,
|
||||
inspect_prepared_dem_path,
|
||||
load_json_file,
|
||||
load_pair_meta,
|
||||
load_shell_environment,
|
||||
normalize_date_text,
|
||||
parse_args,
|
||||
require_task_layout,
|
||||
rerun_pair_product_stages,
|
||||
run_gamma_reflatten,
|
||||
run_logged,
|
||||
safe_rmtree,
|
||||
validate_unit_interval,
|
||||
write_ifgram_list,
|
||||
write_text,
|
||||
write_wrapper_scripts,
|
||||
)
|
||||
except ImportError:
|
||||
from run_lt1_pyint_pipeline import (
|
||||
DEFAULT_DERAMP_MODE,
|
||||
DEFAULT_REFERENCE_MODE,
|
||||
DEFAULT_REFLATTEN_AZIMUTH_STEP,
|
||||
DEFAULT_REFLATTEN_COH_THRESHOLD,
|
||||
DEFAULT_REFLATTEN_FALLBACK_COH_THRESHOLD,
|
||||
DEFAULT_REFLATTEN_MODEL,
|
||||
DEFAULT_REFLATTEN_RANGE_STEP,
|
||||
assert_output_sanity,
|
||||
assert_required_outputs,
|
||||
calculate_dem_oversampling,
|
||||
collect_expected_outputs,
|
||||
collect_output_sanity_checks,
|
||||
collect_stage_error_logs,
|
||||
copy_native_outputs,
|
||||
ensure_directory,
|
||||
export_standard_products,
|
||||
format_gamma_number,
|
||||
hardlink_or_copy,
|
||||
inspect_prepared_dem_path,
|
||||
load_json_file,
|
||||
load_pair_meta,
|
||||
load_shell_environment,
|
||||
normalize_date_text,
|
||||
parse_args,
|
||||
require_task_layout,
|
||||
rerun_pair_product_stages,
|
||||
run_gamma_reflatten,
|
||||
run_logged,
|
||||
safe_rmtree,
|
||||
validate_unit_interval,
|
||||
write_ifgram_list,
|
||||
write_text,
|
||||
write_wrapper_scripts,
|
||||
)
|
||||
|
||||
|
||||
_WINDOWS_DRIVE_RE = re.compile(r"^[A-Za-z]:[\\/]")
|
||||
|
||||
|
||||
def _normalize_runtime_path(value: Any) -> Path:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return Path()
|
||||
normalized = text.replace("\\", "/")
|
||||
if normalized.startswith("/mnt/") or normalized.startswith("/"):
|
||||
return Path(normalized).resolve()
|
||||
if _WINDOWS_DRIVE_RE.match(text):
|
||||
drive = text[0].lower()
|
||||
tail = text[2:].replace("\\", "/").lstrip("/")
|
||||
return Path(f"/mnt/{drive}/{tail}").resolve()
|
||||
return Path(normalized).resolve()
|
||||
|
||||
|
||||
def _build_s1_template_text(
|
||||
*,
|
||||
project_name: str,
|
||||
satellite: str,
|
||||
master_date: str,
|
||||
range_looks: int,
|
||||
azimuth_looks: int,
|
||||
target_grid_size_m: int,
|
||||
dem_lat_ovr: float,
|
||||
dem_lon_ovr: float,
|
||||
unwrap_coh_threshold: float,
|
||||
geo_interp: str,
|
||||
atmcor: bool,
|
||||
atmcor_use_for_disp: bool,
|
||||
reflatten: bool,
|
||||
reflatten_model: str,
|
||||
reflatten_coh_threshold: float,
|
||||
parallel_workers: int,
|
||||
unwrap: bool,
|
||||
geocode: bool,
|
||||
dem_mode: str,
|
||||
fabdem_root: str,
|
||||
prepared_dem_path: str,
|
||||
opentopo_dem_type: str,
|
||||
opentopo_api_key: str,
|
||||
) -> str:
|
||||
prepared_dem = inspect_prepared_dem_path(prepared_dem_path) if dem_mode == "prepared_file" else {}
|
||||
lines = [
|
||||
f"# Auto-generated for {project_name}",
|
||||
f"satelite={satellite}",
|
||||
f"masterDate={master_date}",
|
||||
f"range_looks={int(range_looks)}",
|
||||
f"azimuth_looks={int(azimuth_looks)}",
|
||||
f"target_grid_size_m={int(target_grid_size_m or 0)}",
|
||||
f"dem_lat_ovr={format_gamma_number(dem_lat_ovr)}",
|
||||
f"dem_lon_ovr={format_gamma_number(dem_lon_ovr)}",
|
||||
"download_data=0",
|
||||
"raw2slc_all=1",
|
||||
f"raw2slc_all_parallel={int(parallel_workers)}",
|
||||
"extract_burst_all=1",
|
||||
f"extract_all_parallel={int(parallel_workers)}",
|
||||
"coreg_all=1",
|
||||
f"coreg_all_parallel={int(parallel_workers)}",
|
||||
"select_pairs=0",
|
||||
"diff_all=1",
|
||||
f"diff_all_parallel={int(parallel_workers)}",
|
||||
"pot_all=0",
|
||||
f"pot_all_parallel={int(parallel_workers)}",
|
||||
f"unwrap_all={1 if unwrap else 0}",
|
||||
f"unwrap_all_parallel={int(parallel_workers)}",
|
||||
f"unwrapThreshold={format_gamma_number(unwrap_coh_threshold)}",
|
||||
"make_mask=1",
|
||||
"auto_unw=1",
|
||||
"r_refer=-",
|
||||
"a_refer=-",
|
||||
f"atmcor_all={1 if atmcor else 0}",
|
||||
f"atmcor_all_parallel={int(parallel_workers)}",
|
||||
f"atmcor_use_for_disp={1 if (atmcor and atmcor_use_for_disp) else 0}",
|
||||
f"reflatten={1 if reflatten else 0}",
|
||||
f"reflatten_model={str(reflatten_model or DEFAULT_REFLATTEN_MODEL).strip().lower()}",
|
||||
f"reflatten_coh_threshold={format_gamma_number(reflatten_coh_threshold)}",
|
||||
f"geocode_all={1 if geocode else 0}",
|
||||
f"geocode_all_parallel={int(parallel_workers)}",
|
||||
f"geo_interp={str(geo_interp or '0').strip()}",
|
||||
"gacos_correction=0",
|
||||
"load_data=0",
|
||||
"geocode_products=hyp3,licsbas",
|
||||
"start_swath=1",
|
||||
"end_swath=3",
|
||||
"start_burst=1",
|
||||
"end_burst=20",
|
||||
]
|
||||
if dem_mode == "local_fabdem" and fabdem_root:
|
||||
lines.append(f"fabdem_dir={fabdem_root}")
|
||||
else:
|
||||
lines.append("fabdem_dir=-")
|
||||
if dem_mode == "prepared_file" and prepared_dem.get("kind") == "gamma_ready":
|
||||
lines.append(f"DEM={prepared_dem['direct_dem_path']}")
|
||||
if dem_mode == "prepared_file" and prepared_dem.get("kind") == "source_dem":
|
||||
lines.append(f"prepared_dem_source={prepared_dem['source_dem_path']}")
|
||||
else:
|
||||
lines.append("prepared_dem_source=-")
|
||||
if dem_mode == "opentopo":
|
||||
lines.append(f"opentopo_dem_type={opentopo_dem_type or 'SRTMGL1'}")
|
||||
lines.append(f"opentopo_api_key={opentopo_api_key or '-'}")
|
||||
else:
|
||||
lines.append("opentopo_dem_type=-")
|
||||
lines.append("opentopo_api_key=-")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def _resolve_s1_inputs(input_assets_payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
task_source = input_assets_payload.get("task_source") or {}
|
||||
prod = task_source.get("production_inputs") or {}
|
||||
orbits = input_assets_payload.get("orbits") or {}
|
||||
master_scene = _normalize_runtime_path(
|
||||
((prod.get("master_scene") or {}).get("staged_path") or (prod.get("master_scene") or {}).get("path") or (prod.get("master_zip") or {}).get("staged_path") or (prod.get("master_zip") or {}).get("path") or "")
|
||||
)
|
||||
slave_scene = _normalize_runtime_path(
|
||||
((prod.get("slave_scene") or {}).get("staged_path") or (prod.get("slave_scene") or {}).get("path") or (prod.get("slave_zip") or {}).get("staged_path") or (prod.get("slave_zip") or {}).get("path") or "")
|
||||
)
|
||||
master_eof = _normalize_runtime_path(
|
||||
((orbits.get("master") or {}).get("staged_path") or (orbits.get("master") or {}).get("path") or "")
|
||||
)
|
||||
slave_eof = _normalize_runtime_path(
|
||||
((orbits.get("slave") or {}).get("staged_path") or (orbits.get("slave") or {}).get("path") or "")
|
||||
)
|
||||
return {
|
||||
"master_scene": master_scene,
|
||||
"slave_scene": slave_scene,
|
||||
"master_eof": master_eof,
|
||||
"slave_eof": slave_eof,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
|
||||
task_dir = Path(args.task_dir).resolve()
|
||||
project_dir = Path(args.project_dir).resolve()
|
||||
run_root = project_dir.parent
|
||||
template_root = Path(args.template_root).resolve()
|
||||
output_dir = Path(args.output_dir).resolve()
|
||||
pyint_home = Path(args.pyint_home).resolve()
|
||||
pyint_app_script = Path(args.pyint_app_script).resolve()
|
||||
dem_root = Path(args.dem_root).resolve()
|
||||
input_assets_dir = Path(args.input_assets_dir).resolve() if args.input_assets_dir else None
|
||||
input_assets_json = Path(args.input_assets_json).resolve() if args.input_assets_json else None
|
||||
input_assets_payload = load_json_file(input_assets_json)
|
||||
dem_mode = str(args.dem_mode or "local_fabdem").strip().lower() or "local_fabdem"
|
||||
prepared_dem_info = inspect_prepared_dem_path(args.prepared_dem_path) if dem_mode == "prepared_file" else {}
|
||||
dem_oversampling = calculate_dem_oversampling(
|
||||
dem_resolution_m=float(args.dem_resolution_m),
|
||||
target_grid_size_m=float(args.target_grid_size_m or 0),
|
||||
dem_lat_ovr=float(args.dem_lat_ovr or 0.0),
|
||||
dem_lon_ovr=float(args.dem_lon_ovr or 0.0),
|
||||
)
|
||||
unwrap_coh_threshold = validate_unit_interval(args.unwrap_coh_threshold, "--unwrap-coh-threshold")
|
||||
coherence_mask_threshold = validate_unit_interval(args.coherence_mask_threshold, "--coherence-mask-threshold")
|
||||
reference_mode = str(args.reference_mode or DEFAULT_REFERENCE_MODE).strip().lower() or DEFAULT_REFERENCE_MODE
|
||||
deramp_mode = str(args.deramp_mode or DEFAULT_DERAMP_MODE).strip().lower() or DEFAULT_DERAMP_MODE
|
||||
reference_coh_threshold = validate_unit_interval(args.reference_coh_threshold, "--reference-coh-threshold")
|
||||
deramp_coh_threshold = validate_unit_interval(args.deramp_coh_threshold, "--deramp-coh-threshold")
|
||||
geo_interp = str(args.geo_interp or "1").strip()
|
||||
if geo_interp not in {"0", "1"}:
|
||||
raise ValueError("--geo-interp must be 0 or 1.")
|
||||
gamma_nodata_value = float(args.gamma_nodata_value)
|
||||
if not math.isfinite(gamma_nodata_value):
|
||||
raise ValueError("--gamma-nodata-value must be a finite number.")
|
||||
reflatten_model = str(args.reflatten_model or DEFAULT_REFLATTEN_MODEL).strip().lower()
|
||||
if reflatten_model == "linear":
|
||||
reflatten_model = "plane"
|
||||
if reflatten_model not in {"plane", "quadratic"}:
|
||||
raise ValueError("--reflatten-model must be plane or quadratic.")
|
||||
reflatten_coh_threshold = validate_unit_interval(args.reflatten_coh_threshold, "--reflatten-coh-threshold")
|
||||
reflatten_fallback_coh_threshold = validate_unit_interval(
|
||||
args.reflatten_fallback_coh_threshold,
|
||||
"--reflatten-fallback-coh-threshold",
|
||||
)
|
||||
reflatten_range_step = max(1, int(args.reflatten_range_step or DEFAULT_REFLATTEN_RANGE_STEP))
|
||||
reflatten_azimuth_step = max(1, int(args.reflatten_azimuth_step or DEFAULT_REFLATTEN_AZIMUTH_STEP))
|
||||
|
||||
require_task_layout(task_dir)
|
||||
if not pyint_app_script.is_file():
|
||||
raise FileNotFoundError(f"pyintApp.py not found: {pyint_app_script}")
|
||||
if not input_assets_json or not input_assets_json.is_file():
|
||||
raise RuntimeError("Sentinel-1 PyINT runner requires --input-assets-json.")
|
||||
if dem_mode == "prepared_file" and not prepared_dem_info.get("kind"):
|
||||
raise RuntimeError(
|
||||
"Prepared DEM mode requires either a Gamma DEM with .par, "
|
||||
"or a source DEM with .xml/.hdr/.vrt sidecars."
|
||||
)
|
||||
|
||||
if args.force:
|
||||
safe_rmtree(run_root)
|
||||
safe_rmtree(template_root)
|
||||
safe_rmtree(output_dir)
|
||||
|
||||
if run_root.exists():
|
||||
raise RuntimeError(f"PyINT run root already exists, rerun with --force: {run_root}")
|
||||
|
||||
pair_meta = load_pair_meta(task_dir)
|
||||
s1_inputs = _resolve_s1_inputs(input_assets_payload)
|
||||
master_scene = s1_inputs["master_scene"]
|
||||
slave_scene = s1_inputs["slave_scene"]
|
||||
master_eof = s1_inputs["master_eof"]
|
||||
slave_eof = s1_inputs["slave_eof"]
|
||||
if not (master_scene.is_file() or master_scene.is_dir()):
|
||||
raise FileNotFoundError(f"Sentinel-1 master scene not found: {master_scene}")
|
||||
if not (slave_scene.is_file() or slave_scene.is_dir()):
|
||||
raise FileNotFoundError(f"Sentinel-1 slave scene not found: {slave_scene}")
|
||||
if not master_eof.is_file():
|
||||
raise FileNotFoundError(f"Sentinel-1 master EOF not found: {master_eof}")
|
||||
if not slave_eof.is_file():
|
||||
raise FileNotFoundError(f"Sentinel-1 slave EOF not found: {slave_eof}")
|
||||
|
||||
master_date = normalize_date_text(args.master_date) or normalize_date_text(pair_meta.get("master_imaging_date"))
|
||||
slave_date = normalize_date_text(args.slave_date) or normalize_date_text(pair_meta.get("slave_imaging_date"))
|
||||
if not master_date or not slave_date:
|
||||
raise RuntimeError("Unable to determine master/slave dates from pair metadata.")
|
||||
|
||||
pair_name = f"{master_date}-{slave_date}"
|
||||
task_alias = str(args.task_alias or pair_meta.get("task_alias") or task_dir.name).strip() or task_dir.name
|
||||
pair_key = str(args.pair_key or pair_meta.get("pair_key") or "").strip()
|
||||
time_baseline_days = int(args.time_baseline_days or pair_meta.get("time_baseline_days") or 0)
|
||||
satellite = str(pair_meta.get("master_satellite") or "S1A").strip().upper() or "S1A"
|
||||
|
||||
ensure_directory(run_root)
|
||||
ensure_directory(template_root)
|
||||
ensure_directory(output_dir)
|
||||
ensure_directory(dem_root)
|
||||
|
||||
pyint_scripts_dir = pyint_home / "pyint"
|
||||
wrappers_dir = ensure_directory(run_root / "wrappers")
|
||||
write_wrapper_scripts(
|
||||
wrappers_dir=wrappers_dir,
|
||||
pyint_home=pyint_home,
|
||||
python_cmd=args.python,
|
||||
gamma_env_script=args.gamma_env_script,
|
||||
)
|
||||
for shim_name in ("python", "python3"):
|
||||
shim_path = wrappers_dir / shim_name
|
||||
write_text(
|
||||
shim_path,
|
||||
"\n".join(
|
||||
[
|
||||
"#!/usr/bin/env bash",
|
||||
f"exec '{args.python}' \"$@\"",
|
||||
"",
|
||||
]
|
||||
),
|
||||
)
|
||||
shim_path.chmod(shim_path.stat().st_mode | 0o111)
|
||||
|
||||
template_path = write_text(
|
||||
template_root / f"{args.project_name}.template",
|
||||
_build_s1_template_text(
|
||||
project_name=args.project_name,
|
||||
satellite=satellite,
|
||||
master_date=master_date,
|
||||
range_looks=args.range_looks,
|
||||
azimuth_looks=args.azimuth_looks,
|
||||
target_grid_size_m=int(args.target_grid_size_m or 0),
|
||||
dem_lat_ovr=dem_oversampling["dem_lat_ovr"],
|
||||
dem_lon_ovr=dem_oversampling["dem_lon_ovr"],
|
||||
unwrap_coh_threshold=unwrap_coh_threshold,
|
||||
geo_interp=geo_interp,
|
||||
atmcor=bool(args.atmcor),
|
||||
atmcor_use_for_disp=bool(args.atmcor_use_for_disp),
|
||||
reflatten=bool(args.reflatten),
|
||||
reflatten_model=reflatten_model,
|
||||
reflatten_coh_threshold=reflatten_coh_threshold,
|
||||
parallel_workers=args.parallel_workers,
|
||||
unwrap=bool(args.unwrap),
|
||||
geocode=bool(args.geocode),
|
||||
dem_mode=dem_mode,
|
||||
fabdem_root=str(args.fabdem_root or "").strip(),
|
||||
prepared_dem_path=str(args.prepared_dem_path or "").strip(),
|
||||
opentopo_dem_type=str(args.opentopo_dem_type or "SRTMGL1").strip(),
|
||||
opentopo_api_key=str(args.opentopo_api_key or "").strip(),
|
||||
),
|
||||
)
|
||||
|
||||
scratch_root = ensure_directory(project_dir.parent)
|
||||
archive_materialization: List[Dict[str, str]] = []
|
||||
env = os.environ.copy()
|
||||
if args.gamma_env_script:
|
||||
env.update(load_shell_environment(args.gamma_env_script, env))
|
||||
opod_dir = project_dir / "OPOD"
|
||||
env.update(
|
||||
{
|
||||
"SCRATCHDIR": str(scratch_root),
|
||||
"TEMPLATEDIR": str(template_root),
|
||||
"DEMDIR": str(dem_root),
|
||||
"OPOD_DIR": str(opod_dir),
|
||||
"PATH": f"{wrappers_dir}:{pyint_scripts_dir}:{env.get('PATH', '')}",
|
||||
"PYTHONPATH": f"{pyint_home}:{env.get('PYTHONPATH', '')}",
|
||||
"PYINT_LT1_PRECISE_ORBIT_ENABLED": "false",
|
||||
}
|
||||
)
|
||||
|
||||
generate_stdout = run_root / "pyint_generate.stdout.log"
|
||||
generate_stderr = run_root / "pyint_generate.stderr.log"
|
||||
generate_result = run_logged(
|
||||
[str(wrappers_dir / "pyintApp.py"), "-g", args.project_name],
|
||||
env=env,
|
||||
cwd=scratch_root,
|
||||
stdout_path=generate_stdout,
|
||||
stderr_path=generate_stderr,
|
||||
)
|
||||
if generate_result.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"pyintApp.py -g failed with rc={generate_result.returncode}: "
|
||||
f"{(generate_result.stderr or generate_result.stdout or '').strip()}"
|
||||
)
|
||||
|
||||
pyint_project_dir = project_dir
|
||||
ensure_directory(pyint_project_dir)
|
||||
opod_dir = ensure_directory(opod_dir)
|
||||
download_dir = ensure_directory(pyint_project_dir / "DOWNLOAD")
|
||||
ifgram_list_path = write_ifgram_list(pyint_project_dir / "ifgram_list.txt", master_date, slave_date, time_baseline_days)
|
||||
for role, src_path in (("master", master_scene), ("slave", slave_scene)):
|
||||
target_path = download_dir / src_path.name
|
||||
if src_path.is_dir():
|
||||
if target_path.exists():
|
||||
if target_path.is_dir():
|
||||
safe_rmtree(target_path)
|
||||
else:
|
||||
target_path.unlink()
|
||||
shutil.copytree(src_path, target_path)
|
||||
op = "copied_dir"
|
||||
else:
|
||||
op = hardlink_or_copy(src_path, target_path)
|
||||
archive_materialization.append(
|
||||
{
|
||||
"role": role,
|
||||
"source": str(src_path),
|
||||
"target": str(target_path),
|
||||
"operation": op,
|
||||
}
|
||||
)
|
||||
for role, src_path in (("master_orbit", master_eof), ("slave_orbit", slave_eof)):
|
||||
target_path = opod_dir / src_path.name
|
||||
op = hardlink_or_copy(src_path, target_path)
|
||||
archive_materialization.append(
|
||||
{
|
||||
"role": role,
|
||||
"source": str(src_path),
|
||||
"target": str(target_path),
|
||||
"operation": op,
|
||||
}
|
||||
)
|
||||
|
||||
run_stdout = run_root / "pyint.stdout.log"
|
||||
run_stderr = run_root / "pyint.stderr.log"
|
||||
run_started_at = datetime.utcnow().isoformat(timespec="seconds") + "Z"
|
||||
run_result = run_logged(
|
||||
[str(wrappers_dir / "pyintApp.py"), args.project_name],
|
||||
env=env,
|
||||
cwd=scratch_root,
|
||||
stdout_path=run_stdout,
|
||||
stderr_path=run_stderr,
|
||||
)
|
||||
if run_result.returncode != 0:
|
||||
stage_error_logs = collect_stage_error_logs(pyint_project_dir)
|
||||
detail_text = (run_result.stderr or run_result.stdout or "").strip()
|
||||
if stage_error_logs:
|
||||
log_text = ", ".join(f"{name}={path}" for name, path in stage_error_logs.items())
|
||||
detail_text = f"{detail_text}\nStage logs: {log_text}" if detail_text else f"Stage logs: {log_text}"
|
||||
raise RuntimeError(f"pyintApp.py failed with rc={run_result.returncode}: {detail_text}")
|
||||
|
||||
repair_summary: Dict[str, Any] = {
|
||||
"attempted": False,
|
||||
"attempt_count": 0,
|
||||
"max_attempts": 1,
|
||||
}
|
||||
outputs = collect_expected_outputs(pyint_project_dir, pair_name, args.range_looks)
|
||||
try:
|
||||
assert_required_outputs(outputs, unwrap=bool(args.unwrap), geocode=bool(args.geocode))
|
||||
output_sanity_checks = collect_output_sanity_checks(outputs, unwrap=bool(args.unwrap), geocode=bool(args.geocode))
|
||||
assert_output_sanity(output_sanity_checks)
|
||||
except RuntimeError as exc:
|
||||
repair_summary = rerun_pair_product_stages(
|
||||
project_name=args.project_name,
|
||||
project_dir=pyint_project_dir,
|
||||
run_root=run_root,
|
||||
scratch_root=scratch_root,
|
||||
env=env,
|
||||
pair_name=pair_name,
|
||||
master_date=master_date,
|
||||
slave_date=slave_date,
|
||||
range_looks=args.range_looks,
|
||||
unwrap=bool(args.unwrap),
|
||||
atmcor=bool(args.atmcor),
|
||||
geocode=bool(args.geocode),
|
||||
)
|
||||
outputs = collect_expected_outputs(pyint_project_dir, pair_name, args.range_looks)
|
||||
assert_required_outputs(outputs, unwrap=bool(args.unwrap), geocode=bool(args.geocode))
|
||||
output_sanity_checks = collect_output_sanity_checks(outputs, unwrap=bool(args.unwrap), geocode=bool(args.geocode))
|
||||
assert_output_sanity(output_sanity_checks)
|
||||
|
||||
stage_error_logs = collect_stage_error_logs(pyint_project_dir)
|
||||
reflatten_summary: Dict[str, Any] = {
|
||||
"enabled": bool(args.reflatten),
|
||||
"applied": False,
|
||||
"reason": "",
|
||||
"model": reflatten_model,
|
||||
"coherence_threshold": reflatten_coh_threshold,
|
||||
"fallback_coherence_threshold": reflatten_fallback_coh_threshold,
|
||||
"range_step": reflatten_range_step,
|
||||
"azimuth_step": reflatten_azimuth_step,
|
||||
}
|
||||
if bool(args.reflatten) and bool(args.unwrap):
|
||||
reflatten_summary = run_gamma_reflatten(
|
||||
project_dir=pyint_project_dir,
|
||||
run_root=run_root,
|
||||
output_dir=output_dir,
|
||||
outputs=outputs,
|
||||
pair_name=pair_name,
|
||||
master_date=master_date,
|
||||
range_looks=args.range_looks,
|
||||
env=env,
|
||||
model=reflatten_model,
|
||||
coherence_threshold=reflatten_coh_threshold,
|
||||
fallback_coherence_threshold=reflatten_fallback_coh_threshold,
|
||||
range_step=reflatten_range_step,
|
||||
azimuth_step=reflatten_azimuth_step,
|
||||
geo_interp=geo_interp,
|
||||
)
|
||||
outputs = collect_expected_outputs(pyint_project_dir, pair_name, args.range_looks)
|
||||
output_sanity_checks = collect_output_sanity_checks(outputs, unwrap=bool(args.unwrap), geocode=bool(args.geocode))
|
||||
assert_output_sanity(output_sanity_checks)
|
||||
elif bool(args.reflatten):
|
||||
reflatten_summary["reason"] = "unwrap disabled"
|
||||
else:
|
||||
reflatten_summary["reason"] = "disabled"
|
||||
|
||||
copied_paths = copy_native_outputs(
|
||||
project_dir=pyint_project_dir,
|
||||
output_dir=output_dir,
|
||||
pair_name=pair_name,
|
||||
template_path=template_path,
|
||||
ifgram_list_path=ifgram_list_path,
|
||||
stdout_path=run_stdout,
|
||||
stderr_path=run_stderr,
|
||||
)
|
||||
standard_products = (
|
||||
export_standard_products(
|
||||
project_dir=pyint_project_dir,
|
||||
output_dir=output_dir,
|
||||
pair_name=pair_name,
|
||||
master_date=master_date,
|
||||
range_looks=args.range_looks,
|
||||
azimuth_looks=args.azimuth_looks,
|
||||
target_grid_size_m=int(args.target_grid_size_m or 0),
|
||||
coherence_mask_threshold=coherence_mask_threshold,
|
||||
reference_mode=reference_mode,
|
||||
reference_coh_threshold=reference_coh_threshold,
|
||||
deramp_mode=deramp_mode,
|
||||
deramp_coh_threshold=deramp_coh_threshold,
|
||||
atmcor_enabled=bool(args.atmcor),
|
||||
atmcor_use_for_disp=bool(args.atmcor_use_for_disp),
|
||||
reflatten_summary=reflatten_summary,
|
||||
gamma_nodata_value=gamma_nodata_value,
|
||||
outputs=outputs,
|
||||
env=env,
|
||||
run_root=run_root,
|
||||
)
|
||||
if bool(args.geocode)
|
||||
else {"enabled": False, "reason": "geocode disabled"}
|
||||
)
|
||||
|
||||
summary = {
|
||||
"ok": True,
|
||||
"task_dir": str(task_dir),
|
||||
"task_alias": task_alias,
|
||||
"pair_key": pair_key,
|
||||
"project_name": args.project_name,
|
||||
"project_dir": str(pyint_project_dir),
|
||||
"run_root": str(run_root),
|
||||
"template_root": str(template_root),
|
||||
"output_dir": str(output_dir),
|
||||
"pyint_home": str(pyint_home),
|
||||
"pyint_app_script": str(pyint_app_script),
|
||||
"gamma_env_script": args.gamma_env_script,
|
||||
"dem": {
|
||||
"mode": dem_mode,
|
||||
"dem_root": str(dem_root),
|
||||
"fabdem_root": str(args.fabdem_root or "").strip(),
|
||||
"prepared_dem_path": str(args.prepared_dem_path or "").strip(),
|
||||
"prepared_dem_kind": str(prepared_dem_info.get("kind") or ""),
|
||||
"configured_resolution_m": float(args.dem_resolution_m),
|
||||
"oversampling": dem_oversampling,
|
||||
"opentopo_dem_type": str(args.opentopo_dem_type or "SRTMGL1").strip(),
|
||||
"opentopo_api_key_configured": bool(str(args.opentopo_api_key or "").strip()),
|
||||
},
|
||||
"orbit_policy": "require_eof",
|
||||
"precise_orbit_bridge": {"enabled": False, "mode": "not_applicable"},
|
||||
"input_assets_dir": str(input_assets_dir) if input_assets_dir else "",
|
||||
"input_assets_json": str(input_assets_json) if input_assets_json else "",
|
||||
"input_assets": input_assets_payload,
|
||||
"master_date": master_date,
|
||||
"slave_date": slave_date,
|
||||
"pair_name": pair_name,
|
||||
"time_baseline_days": time_baseline_days,
|
||||
"target_grid_size_m": int(args.target_grid_size_m or 0),
|
||||
"range_looks": int(args.range_looks),
|
||||
"azimuth_looks": int(args.azimuth_looks),
|
||||
"dem_resolution_m": float(args.dem_resolution_m),
|
||||
"dem_oversampling": dem_oversampling,
|
||||
"unwrap_coh_threshold": unwrap_coh_threshold,
|
||||
"coherence_quality_threshold": coherence_mask_threshold,
|
||||
"reference_mode": reference_mode,
|
||||
"reference_coh_threshold": reference_coh_threshold,
|
||||
"deramp_mode": deramp_mode,
|
||||
"deramp_coh_threshold": deramp_coh_threshold,
|
||||
"gamma_nodata_value": gamma_nodata_value,
|
||||
"geo_interp": geo_interp,
|
||||
"atmcor": bool(args.atmcor),
|
||||
"atmcor_use_for_disp": bool(args.atmcor_use_for_disp),
|
||||
"reflatten": bool(args.reflatten),
|
||||
"reflatten_model": reflatten_model,
|
||||
"reflatten_coh_threshold": reflatten_coh_threshold,
|
||||
"reflatten_fallback_coh_threshold": reflatten_fallback_coh_threshold,
|
||||
"reflatten_range_step": reflatten_range_step,
|
||||
"reflatten_azimuth_step": reflatten_azimuth_step,
|
||||
"parallel_workers": int(args.parallel_workers),
|
||||
"unwrap": bool(args.unwrap),
|
||||
"geocode": bool(args.geocode),
|
||||
"archives": {
|
||||
"master": [str(master_scene)],
|
||||
"slave": [str(slave_scene)],
|
||||
},
|
||||
"orbit_files": {
|
||||
"master": str(master_eof),
|
||||
"slave": str(slave_eof),
|
||||
},
|
||||
"archive_materialization": archive_materialization,
|
||||
"workspace_outputs": outputs,
|
||||
"output_sanity_checks": output_sanity_checks,
|
||||
"output_repair": repair_summary,
|
||||
"reflatten_summary": reflatten_summary,
|
||||
"copied_outputs": copied_paths,
|
||||
"standard_products": standard_products,
|
||||
"logs": {
|
||||
"generate_stdout": str(generate_stdout),
|
||||
"generate_stderr": str(generate_stderr),
|
||||
"run_stdout": str(run_stdout),
|
||||
"run_stderr": str(run_stderr),
|
||||
"stage_error_logs": stage_error_logs,
|
||||
},
|
||||
"started_at": run_started_at,
|
||||
"finished_at": datetime.utcnow().isoformat(timespec="seconds") + "Z",
|
||||
}
|
||||
|
||||
summary_path = output_dir / "pyint_run_summary.json"
|
||||
write_text(summary_path, json.dumps(summary, ensure_ascii=True, indent=2) + "\n")
|
||||
print(json.dumps(summary, ensure_ascii=True, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except Exception as exc:
|
||||
print(str(exc), file=sys.stderr)
|
||||
raise
|
||||
@@ -5,6 +5,7 @@ from fastapi import APIRouter
|
||||
from . import (
|
||||
ai,
|
||||
aoi,
|
||||
assets,
|
||||
auth,
|
||||
dinsar,
|
||||
dinsar_products,
|
||||
@@ -43,6 +44,7 @@ def include_all_routers(router: APIRouter) -> None:
|
||||
router.include_router(unpack.router)
|
||||
router.include_router(monitor.router)
|
||||
router.include_router(orbit.router)
|
||||
router.include_router(assets.router)
|
||||
router.include_router(root_registry.router)
|
||||
router.include_router(radar.router)
|
||||
router.include_router(aoi.router)
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ..database import get_db
|
||||
from ..models import AuthUserORM
|
||||
from ..models.orm import SourceProductAssetORM
|
||||
from ..services.asset_inventory_service import asset_inventory_service
|
||||
from ..services.job_queue_service import job_queue_service
|
||||
from ..services.task_service import task_service
|
||||
from .dependencies import _get_current_user, _require_admin
|
||||
|
||||
|
||||
router = APIRouter(prefix="/assets", tags=["assets"])
|
||||
|
||||
|
||||
class AssetScanRequest(BaseModel):
|
||||
inventory_types: List[str] = Field(default_factory=list)
|
||||
root_ids: List[int] = Field(default_factory=list)
|
||||
bind_orbits: bool = True
|
||||
|
||||
|
||||
class S1UnpackRequest(BaseModel):
|
||||
target_root: Optional[str] = None
|
||||
overwrite: bool = False
|
||||
min_disk_space_gb: Optional[float] = Field(default=None, ge=0)
|
||||
delete_archive: Optional[bool] = None
|
||||
|
||||
|
||||
class S1BatchUnpackRequest(BaseModel):
|
||||
target_root: Optional[str] = None
|
||||
overwrite: bool = False
|
||||
min_disk_space_gb: Optional[float] = Field(default=None, ge=0)
|
||||
delete_archive: Optional[bool] = None
|
||||
scan_before_unpack: bool = True
|
||||
|
||||
|
||||
@router.get("/inventory/status")
|
||||
async def get_asset_inventory_status(
|
||||
current_user: AuthUserORM = Depends(_get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
_ = current_user
|
||||
return await asset_inventory_service.get_status(db)
|
||||
|
||||
|
||||
@router.post("/inventory/scan", status_code=202)
|
||||
async def run_asset_inventory_scan(
|
||||
request: Optional[AssetScanRequest] = None,
|
||||
admin_user: AuthUserORM = Depends(_require_admin),
|
||||
):
|
||||
_ = admin_user
|
||||
payload = (request or AssetScanRequest()).model_dump()
|
||||
try:
|
||||
task_id = await task_service.create_task(
|
||||
"SCAN_ASSET_INVENTORY",
|
||||
"Source/orbit asset inventory scan",
|
||||
params=payload,
|
||||
)
|
||||
job_id = await job_queue_service.create_job(
|
||||
"SCAN_ASSET_INVENTORY",
|
||||
payload=payload,
|
||||
task_id=task_id,
|
||||
)
|
||||
return {"message": "Asset inventory scan queued", "task_id": task_id, "job_id": job_id}
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@router.post("/inventory/scan-now")
|
||||
async def run_asset_inventory_scan_now(
|
||||
request: Optional[AssetScanRequest] = None,
|
||||
admin_user: AuthUserORM = Depends(_require_admin),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
_ = admin_user
|
||||
payload = (request or AssetScanRequest()).model_dump()
|
||||
return await asset_inventory_service.scan_configured_roots(
|
||||
db,
|
||||
inventory_types=payload.get("inventory_types") or None,
|
||||
root_ids=payload.get("root_ids") or None,
|
||||
bind_orbits=bool(payload.get("bind_orbits", True)),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/sources")
|
||||
async def list_source_product_assets(
|
||||
satellite_family: Optional[str] = None,
|
||||
satellite: Optional[str] = None,
|
||||
source_format: Optional[str] = None,
|
||||
parse_status: Optional[str] = None,
|
||||
include_inactive: bool = False,
|
||||
limit: int = Query(200, ge=1, le=1000),
|
||||
offset: int = Query(0, ge=0),
|
||||
current_user: AuthUserORM = Depends(_get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
_ = current_user
|
||||
return await asset_inventory_service.list_source_products(
|
||||
db,
|
||||
satellite_family=satellite_family,
|
||||
satellite=satellite,
|
||||
source_format=source_format,
|
||||
parse_status=parse_status,
|
||||
include_inactive=include_inactive,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/orbits")
|
||||
async def list_orbit_assets(
|
||||
satellite_family: Optional[str] = None,
|
||||
satellite: Optional[str] = None,
|
||||
orbit_type: Optional[str] = None,
|
||||
parse_status: Optional[str] = None,
|
||||
include_inactive: bool = False,
|
||||
limit: int = Query(200, ge=1, le=1000),
|
||||
offset: int = Query(0, ge=0),
|
||||
current_user: AuthUserORM = Depends(_get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
_ = current_user
|
||||
return await asset_inventory_service.list_orbits(
|
||||
db,
|
||||
satellite_family=satellite_family,
|
||||
satellite=satellite,
|
||||
orbit_type=orbit_type,
|
||||
parse_status=parse_status,
|
||||
include_inactive=include_inactive,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/issues")
|
||||
async def list_asset_inventory_issues(
|
||||
status: str = "OPEN",
|
||||
severity: Optional[str] = None,
|
||||
issue_code: Optional[str] = None,
|
||||
limit: int = Query(200, ge=1, le=1000),
|
||||
offset: int = Query(0, ge=0),
|
||||
current_user: AuthUserORM = Depends(_get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
_ = current_user
|
||||
return await asset_inventory_service.list_issues(
|
||||
db,
|
||||
status=status,
|
||||
severity=severity,
|
||||
issue_code=issue_code,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/sources/{asset_id}/unpack-sentinel1")
|
||||
async def unpack_sentinel1_source_asset(
|
||||
asset_id: int,
|
||||
request: Optional[S1UnpackRequest] = None,
|
||||
admin_user: AuthUserORM = Depends(_require_admin),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
_ = admin_user
|
||||
asset = await db.get(SourceProductAssetORM, asset_id)
|
||||
if asset is None:
|
||||
raise HTTPException(status_code=404, detail="Source product asset not found.")
|
||||
if asset.source_format != "S1_ZIP":
|
||||
raise HTTPException(status_code=400, detail="Only Sentinel-1 ZIP assets can be unpacked by this endpoint.")
|
||||
|
||||
request_data = request or S1UnpackRequest()
|
||||
payload = {
|
||||
"asset_id": asset_id,
|
||||
"target_root": request_data.target_root,
|
||||
"overwrite": bool(request_data.overwrite),
|
||||
}
|
||||
if request_data.min_disk_space_gb is not None:
|
||||
payload["min_disk_space_gb"] = request_data.min_disk_space_gb
|
||||
if request_data.delete_archive is not None:
|
||||
payload["delete_archive"] = request_data.delete_archive
|
||||
try:
|
||||
task_id = await task_service.create_task(
|
||||
"UNPACK_SENTINEL1",
|
||||
"Sentinel-1 unpack",
|
||||
params=payload,
|
||||
)
|
||||
job_id = await job_queue_service.create_job(
|
||||
"UNPACK_SENTINEL1",
|
||||
payload=payload,
|
||||
task_id=task_id,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
await db.commit()
|
||||
return {"message": "Sentinel-1 unpack task queued", "task_id": task_id, "job_id": job_id}
|
||||
|
||||
|
||||
@router.post("/inventory/unpack-sentinel1", status_code=202)
|
||||
async def run_sentinel1_unpack_batch(
|
||||
request: Optional[S1BatchUnpackRequest] = None,
|
||||
admin_user: AuthUserORM = Depends(_require_admin),
|
||||
):
|
||||
_ = admin_user
|
||||
payload = (request or S1BatchUnpackRequest()).model_dump()
|
||||
try:
|
||||
task_id = await task_service.create_task(
|
||||
"UNPACK_SENTINEL1",
|
||||
"Sentinel-1 batch unpack",
|
||||
params=payload,
|
||||
)
|
||||
job_id = await job_queue_service.create_job(
|
||||
"UNPACK_SENTINEL1",
|
||||
payload=payload,
|
||||
task_id=task_id,
|
||||
)
|
||||
return {"message": "Sentinel-1 batch unpack task queued", "task_id": task_id, "job_id": job_id}
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
@@ -99,9 +99,15 @@ def _normalize_list_pagination(limit: int, offset: int) -> tuple[int, int]:
|
||||
class DinsarBatchSceneCreate(BaseModel):
|
||||
file_path: str = Field(min_length=1)
|
||||
satellite: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH)
|
||||
satellite_family: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH)
|
||||
imaging_date: Optional[str] = Field(default=None, max_length=32)
|
||||
imaging_mode: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH)
|
||||
polarization: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH)
|
||||
orbit_direction: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH)
|
||||
relative_orbit: Optional[str] = Field(default=None, max_length=64)
|
||||
absolute_orbit: Optional[str] = Field(default=None, max_length=64)
|
||||
has_orbit_data: Optional[bool] = None
|
||||
orbit_file_path: Optional[str] = Field(default=None, max_length=4096)
|
||||
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
@@ -14,6 +14,10 @@ MONITOR_CONFIG = {
|
||||
"radar_dirs": split_env_paths(settings.MONITOR_RADAR_DIRS),
|
||||
"orbit_dir": settings.MONITOR_ORBIT_DIR,
|
||||
"dinsar_dirs": split_env_paths(settings.MONITOR_DINSAR_DIRS),
|
||||
# Sentinel-1 链路
|
||||
"s1_source_dirs": split_env_paths(settings.SOURCE_PRODUCT_DIRS),
|
||||
"s1_storage_dirs": split_env_paths(settings.SENTINEL1_STORAGE_DIRS),
|
||||
"s1_orbit_dirs": split_env_paths(settings.ORBIT_SOURCE_DIRS),
|
||||
# GF3 链路
|
||||
"gf3_source_dirs": split_env_paths(settings.GF3_SOURCE_DIRS),
|
||||
"gf3_storage_dirs": split_env_paths(settings.GF3_STORAGE_DIRS),
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -238,7 +238,7 @@ def _chunked(items: List[Any], size: int):
|
||||
|
||||
|
||||
_RADAR_PREVIEW_EXTENSIONS = (".jpg", ".jpeg", ".png", ".webp", ".bmp", ".tif", ".tiff")
|
||||
_RADAR_PREVIEW_KEYWORDS = ("quicklook", "preview", "browse", "thumbnail", "thumb", "overview")
|
||||
_RADAR_PREVIEW_KEYWORDS = ("quicklook", "quick-look", "preview", "browse", "thumbnail", "thumb", "overview")
|
||||
_RADAR_CACHE_NAME_RE = re.compile(r"[^A-Za-z0-9._-]+")
|
||||
|
||||
|
||||
@@ -396,7 +396,18 @@ class DataService:
|
||||
continue
|
||||
|
||||
path = os.path.join(root, name)
|
||||
root_lower = root.lower()
|
||||
keyword_score = 0 if any(key in lower_name for key in _RADAR_PREVIEW_KEYWORDS) else 1
|
||||
if lower_name == "quick-look.png":
|
||||
keyword_score = -3
|
||||
elif lower_name == "quicklook.png":
|
||||
keyword_score = -2
|
||||
elif lower_name.startswith("quick-look.") or lower_name.startswith("quicklook."):
|
||||
keyword_score = min(keyword_score, -1)
|
||||
if f"{os.sep}preview" in root_lower:
|
||||
keyword_score -= 1
|
||||
if f"{os.sep}icons" in root_lower:
|
||||
keyword_score += 2
|
||||
ext_score = 0 if lower_name.endswith((".jpg", ".jpeg")) else 1
|
||||
try:
|
||||
size_score = -os.path.getsize(path)
|
||||
|
||||
@@ -7,6 +7,8 @@ import re
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from ..utils import normalize_satellite_family
|
||||
|
||||
|
||||
PAIR_META_FILENAME = ".dinsar_pair.json"
|
||||
RUN_META_FILENAME = ".dinsar_run.json"
|
||||
@@ -48,6 +50,7 @@ def build_pair_key(
|
||||
slave_path: Any,
|
||||
master_date: Any = None,
|
||||
slave_date: Any = None,
|
||||
satellite_family: Any = None,
|
||||
) -> str:
|
||||
master_date_text = _normalize_date(master_date)
|
||||
slave_date_text = _normalize_date(slave_date)
|
||||
@@ -60,17 +63,23 @@ def build_pair_key(
|
||||
]
|
||||
)
|
||||
digest = hashlib.sha1(payload.encode("utf-8", errors="ignore")).hexdigest()[:10]
|
||||
return f"lt1_{master_date_text}_{slave_date_text}_{digest}"
|
||||
family = str(normalize_satellite_family(satellite_family) or "").strip().lower()
|
||||
if not family:
|
||||
family = "pair"
|
||||
return f"{family}_{master_date_text}_{slave_date_text}_{digest}"
|
||||
|
||||
|
||||
def build_fallback_pair_key(task_alias: Any, source_hint: Any = None) -> str:
|
||||
def build_fallback_pair_key(task_alias: Any, source_hint: Any = None, satellite_family: Any = None) -> str:
|
||||
alias = str(task_alias or "").strip() or "Task_unknown_unknown"
|
||||
parts = alias.split("_")
|
||||
master_date = parts[1] if len(parts) > 2 else "unknown"
|
||||
slave_date = parts[2] if len(parts) > 2 else "unknown"
|
||||
payload = "||".join([alias, normalize_path(source_hint)])
|
||||
digest = hashlib.sha1(payload.encode("utf-8", errors="ignore")).hexdigest()[:10]
|
||||
return f"lt1_{_normalize_date(master_date)}_{_normalize_date(slave_date)}_{digest}"
|
||||
family = str(normalize_satellite_family(satellite_family) or "").strip().lower()
|
||||
if not family:
|
||||
family = "pair"
|
||||
return f"{family}_{_normalize_date(master_date)}_{_normalize_date(slave_date)}_{digest}"
|
||||
|
||||
|
||||
def build_run_key(
|
||||
|
||||
@@ -21,6 +21,7 @@ from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..config import get_env_text, settings
|
||||
from ..process_utils import is_any_process_running
|
||||
from ..utils import normalize_satellite_family
|
||||
from .dinsar_naming import (
|
||||
PAIR_META_FILENAME,
|
||||
build_fallback_pair_key,
|
||||
@@ -480,7 +481,14 @@ def _utc_now_text() -> str:
|
||||
def _resolve_dinsar_pair_identity(task_dir: str, task_name: str) -> tuple[str, str, Dict[str, Any]]:
|
||||
pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {}
|
||||
task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name
|
||||
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(task_alias, task_dir)
|
||||
satellite_family = normalize_satellite_family(
|
||||
pair_meta.get("master_satellite") or pair_meta.get("slave_satellite")
|
||||
)
|
||||
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(
|
||||
task_alias,
|
||||
task_dir,
|
||||
satellite_family=satellite_family,
|
||||
)
|
||||
return task_alias, pair_key, pair_meta
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import os
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Dict, Optional
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import and_, func, literal, or_, select, text
|
||||
@@ -11,9 +11,14 @@ from ..config import read_int_env, settings, split_env_paths
|
||||
from ..db_maintenance import inspect_database_structure
|
||||
from ..models import (
|
||||
AiDiagnosisORM,
|
||||
AssetInventoryIssueORM,
|
||||
AssetInventoryStateORM,
|
||||
DinsarResultORM,
|
||||
OrbitAssetORM,
|
||||
ResultCatalogStateORM,
|
||||
ResultProductORM,
|
||||
SceneOrbitBindingORM,
|
||||
SourceProductAssetORM,
|
||||
SystemWorkerHeartbeatORM,
|
||||
)
|
||||
from ..idl_service import get_idl_status
|
||||
@@ -399,6 +404,51 @@ def _sanitize_pairing_system_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _sanitize_asset_inventory_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
source_roots = payload.get("source_roots", {}) or {}
|
||||
orbit_roots = payload.get("orbit_roots", {}) or {}
|
||||
source_assets = payload.get("source_assets", {}) or {}
|
||||
orbit_assets = payload.get("orbit_assets", {}) or {}
|
||||
bindings = payload.get("bindings", {}) or {}
|
||||
issues = payload.get("issues", {}) or {}
|
||||
return {
|
||||
"ok": bool(payload.get("ok")),
|
||||
"source_roots": {
|
||||
"configured_count": int(source_roots.get("configured_count") or 0),
|
||||
"accessible_count": int(source_roots.get("accessible_count") or 0),
|
||||
"needs_rescan_count": int(source_roots.get("needs_rescan_count") or 0),
|
||||
},
|
||||
"orbit_roots": {
|
||||
"configured_count": int(orbit_roots.get("configured_count") or 0),
|
||||
"accessible_count": int(orbit_roots.get("accessible_count") or 0),
|
||||
"needs_rescan_count": int(orbit_roots.get("needs_rescan_count") or 0),
|
||||
},
|
||||
"source_assets": {
|
||||
"total_count": int(source_assets.get("total_count") or 0),
|
||||
"lt1_count": int(source_assets.get("lt1_count") or 0),
|
||||
"s1_count": int(source_assets.get("s1_count") or 0),
|
||||
"parse_failed_count": int(source_assets.get("parse_failed_count") or 0),
|
||||
},
|
||||
"orbit_assets": {
|
||||
"total_count": int(orbit_assets.get("total_count") or 0),
|
||||
"lt1_count": int(orbit_assets.get("lt1_count") or 0),
|
||||
"s1_count": int(orbit_assets.get("s1_count") or 0),
|
||||
"parse_failed_count": int(orbit_assets.get("parse_failed_count") or 0),
|
||||
},
|
||||
"bindings": {
|
||||
"scene_count": int(bindings.get("scene_count") or 0),
|
||||
"matched_count": int(bindings.get("matched_count") or 0),
|
||||
"missing_count": int(bindings.get("missing_count") or 0),
|
||||
"ambiguous_count": int(bindings.get("ambiguous_count") or 0),
|
||||
},
|
||||
"issues": {
|
||||
"open_count": int(issues.get("open_count") or 0),
|
||||
"error_count": int(issues.get("error_count") or 0),
|
||||
"warning_count": int(issues.get("warning_count") or 0),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
database = payload.get("database", {}) or {}
|
||||
worker = payload.get("worker", {}) or {}
|
||||
@@ -411,6 +461,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
dinsar_bridge = payload.get("dinsar_bridge", {}) or {}
|
||||
source_roots = payload.get("source_roots", {}) or {}
|
||||
product_packages = payload.get("product_packages", {}) or {}
|
||||
asset_inventory = payload.get("asset_inventory", {}) or {}
|
||||
wsl_runtime = payload.get("wsl_runtime", {}) or {}
|
||||
pairing_system = payload.get("pairing_system", {}) or {}
|
||||
idl = payload.get("idl", {}) or {}
|
||||
@@ -423,6 +474,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
sanitized_dinsar_bridge = _sanitize_bridge_status(dinsar_bridge)
|
||||
sanitized_source_roots = _sanitize_source_roots_status(source_roots)
|
||||
sanitized_product_packages = _sanitize_product_package_status(product_packages)
|
||||
sanitized_asset_inventory = _sanitize_asset_inventory_status(asset_inventory)
|
||||
sanitized_wsl_runtime = _sanitize_wsl_runtime_status(wsl_runtime)
|
||||
sanitized_pairing_system = _sanitize_pairing_system_status(pairing_system)
|
||||
|
||||
@@ -451,6 +503,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"dinsar_bridge": sanitized_dinsar_bridge,
|
||||
"source_roots": sanitized_source_roots,
|
||||
"product_packages": sanitized_product_packages,
|
||||
"asset_inventory": sanitized_asset_inventory,
|
||||
"wsl_runtime": sanitized_wsl_runtime,
|
||||
"pairing_system": sanitized_pairing_system,
|
||||
"idl": {
|
||||
@@ -905,6 +958,221 @@ async def _check_product_packages() -> Dict[str, Any]:
|
||||
return status
|
||||
|
||||
|
||||
async def _check_asset_inventory() -> Dict[str, Any]:
|
||||
status: Dict[str, Any] = {
|
||||
"ok": False,
|
||||
"source_roots": {
|
||||
"configured_count": 0,
|
||||
"accessible_count": 0,
|
||||
"inaccessible_count": 0,
|
||||
"needs_rescan_count": 0,
|
||||
"items": [],
|
||||
},
|
||||
"orbit_roots": {
|
||||
"configured_count": 0,
|
||||
"accessible_count": 0,
|
||||
"inaccessible_count": 0,
|
||||
"needs_rescan_count": 0,
|
||||
"items": [],
|
||||
},
|
||||
"source_assets": {
|
||||
"total_count": 0,
|
||||
"lt1_count": 0,
|
||||
"s1_count": 0,
|
||||
"parse_failed_count": 0,
|
||||
"by_family": {},
|
||||
},
|
||||
"orbit_assets": {
|
||||
"total_count": 0,
|
||||
"lt1_count": 0,
|
||||
"s1_count": 0,
|
||||
"parse_failed_count": 0,
|
||||
"by_family": {},
|
||||
},
|
||||
"bindings": {
|
||||
"scene_count": 0,
|
||||
"matched_count": 0,
|
||||
"missing_count": 0,
|
||||
"ambiguous_count": 0,
|
||||
},
|
||||
"issues": {
|
||||
"open_count": 0,
|
||||
"error_count": 0,
|
||||
"warning_count": 0,
|
||||
"by_code": {},
|
||||
},
|
||||
"error": None,
|
||||
}
|
||||
|
||||
source_paths: List[str] = []
|
||||
for value in (
|
||||
settings.SOURCE_PRODUCT_DIRS,
|
||||
settings.INSAR_STORAGE_DIRS,
|
||||
settings.MONITOR_RADAR_DIRS,
|
||||
):
|
||||
for path in split_env_paths(value):
|
||||
if path not in source_paths:
|
||||
source_paths.append(path)
|
||||
|
||||
orbit_paths: List[str] = []
|
||||
for value in (
|
||||
settings.ORBIT_SOURCE_DIRS,
|
||||
settings.MONITOR_ORBIT_DIR,
|
||||
):
|
||||
for path in split_env_paths(value):
|
||||
if path not in orbit_paths:
|
||||
orbit_paths.append(path)
|
||||
|
||||
for path in source_paths:
|
||||
item = _probe_directory_status(path)
|
||||
item["role"] = "source_product_pool"
|
||||
status["source_roots"]["items"].append(item)
|
||||
for path in orbit_paths:
|
||||
item = _probe_directory_status(path)
|
||||
item["role"] = "orbit_asset_pool"
|
||||
status["orbit_roots"]["items"].append(item)
|
||||
|
||||
for key in ("source_roots", "orbit_roots"):
|
||||
root_status = status[key]
|
||||
root_status["configured_count"] = len(root_status["items"])
|
||||
root_status["accessible_count"] = sum(1 for item in root_status["items"] if item.get("accessible"))
|
||||
root_status["inaccessible_count"] = root_status["configured_count"] - root_status["accessible_count"]
|
||||
|
||||
try:
|
||||
session_factory = _get_session_factory()
|
||||
async with session_factory() as db:
|
||||
state_rows = (
|
||||
await db.execute(
|
||||
select(
|
||||
AssetInventoryStateORM.inventory_type,
|
||||
func.count(AssetInventoryStateORM.id),
|
||||
)
|
||||
.where(AssetInventoryStateORM.needs_rescan == True) # noqa: E712
|
||||
.group_by(AssetInventoryStateORM.inventory_type)
|
||||
)
|
||||
).all()
|
||||
for inventory_type, count in state_rows:
|
||||
key = "orbit_roots" if str(inventory_type or "").lower().startswith("orbit") else "source_roots"
|
||||
status[key]["needs_rescan_count"] += int(count or 0)
|
||||
|
||||
source_family_rows = (
|
||||
await db.execute(
|
||||
select(
|
||||
SourceProductAssetORM.satellite_family,
|
||||
func.count(SourceProductAssetORM.id),
|
||||
)
|
||||
.where(
|
||||
SourceProductAssetORM.is_active == True, # noqa: E712
|
||||
SourceProductAssetORM.source_format != "S1_ZIP",
|
||||
)
|
||||
.group_by(SourceProductAssetORM.satellite_family)
|
||||
)
|
||||
).all()
|
||||
for family, count in source_family_rows:
|
||||
family_key = str(family or "unknown").strip().upper() or "unknown"
|
||||
value = int(count or 0)
|
||||
status["source_assets"]["by_family"][family_key] = value
|
||||
status["source_assets"]["total_count"] += value
|
||||
status["source_assets"]["lt1_count"] = int(status["source_assets"]["by_family"].get("LT1", 0))
|
||||
status["source_assets"]["s1_count"] = int(status["source_assets"]["by_family"].get("S1", 0))
|
||||
source_parse_failed = await db.execute(
|
||||
select(func.count(SourceProductAssetORM.id)).where(
|
||||
SourceProductAssetORM.parse_status == "FAILED",
|
||||
SourceProductAssetORM.source_format != "S1_ZIP",
|
||||
)
|
||||
)
|
||||
status["source_assets"]["parse_failed_count"] = int(source_parse_failed.scalar_one() or 0)
|
||||
|
||||
orbit_family_rows = (
|
||||
await db.execute(
|
||||
select(
|
||||
OrbitAssetORM.satellite_family,
|
||||
func.count(OrbitAssetORM.id),
|
||||
)
|
||||
.where(OrbitAssetORM.is_active == True) # noqa: E712
|
||||
.group_by(OrbitAssetORM.satellite_family)
|
||||
)
|
||||
).all()
|
||||
for family, count in orbit_family_rows:
|
||||
family_key = str(family or "unknown").strip().upper() or "unknown"
|
||||
value = int(count or 0)
|
||||
status["orbit_assets"]["by_family"][family_key] = value
|
||||
status["orbit_assets"]["total_count"] += value
|
||||
status["orbit_assets"]["lt1_count"] = int(status["orbit_assets"]["by_family"].get("LT1", 0))
|
||||
status["orbit_assets"]["s1_count"] = int(status["orbit_assets"]["by_family"].get("S1", 0))
|
||||
orbit_parse_failed = await db.execute(
|
||||
select(func.count(OrbitAssetORM.id)).where(OrbitAssetORM.parse_status == "FAILED")
|
||||
)
|
||||
status["orbit_assets"]["parse_failed_count"] = int(orbit_parse_failed.scalar_one() or 0)
|
||||
|
||||
scene_count = await db.execute(select(func.count(SceneOrbitBindingORM.radar_data_id.distinct())))
|
||||
status["bindings"]["scene_count"] = int(scene_count.scalar_one() or 0)
|
||||
selected_count = await db.execute(
|
||||
select(func.count(SceneOrbitBindingORM.id)).where(SceneOrbitBindingORM.selection_status == "SELECTED")
|
||||
)
|
||||
status["bindings"]["matched_count"] = int(selected_count.scalar_one() or 0)
|
||||
missing_count = await db.execute(
|
||||
select(func.count(AssetInventoryIssueORM.id)).where(
|
||||
AssetInventoryIssueORM.status == "OPEN",
|
||||
AssetInventoryIssueORM.issue_code == "scene_missing_orbit",
|
||||
)
|
||||
)
|
||||
status["bindings"]["missing_count"] = int(missing_count.scalar_one() or 0)
|
||||
ambiguous_count = await db.execute(
|
||||
select(func.count(AssetInventoryIssueORM.id)).where(
|
||||
AssetInventoryIssueORM.status == "OPEN",
|
||||
AssetInventoryIssueORM.issue_code == "scene_ambiguous_orbit",
|
||||
)
|
||||
)
|
||||
status["bindings"]["ambiguous_count"] = int(ambiguous_count.scalar_one() or 0)
|
||||
|
||||
issue_rows = (
|
||||
await db.execute(
|
||||
select(
|
||||
AssetInventoryIssueORM.severity,
|
||||
func.count(AssetInventoryIssueORM.id),
|
||||
)
|
||||
.where(AssetInventoryIssueORM.status == "OPEN")
|
||||
.group_by(AssetInventoryIssueORM.severity)
|
||||
)
|
||||
).all()
|
||||
for severity, count in issue_rows:
|
||||
severity_key = str(severity or "warning").strip().lower() or "warning"
|
||||
value = int(count or 0)
|
||||
status["issues"]["open_count"] += value
|
||||
if severity_key == "error":
|
||||
status["issues"]["error_count"] += value
|
||||
elif severity_key == "warning":
|
||||
status["issues"]["warning_count"] += value
|
||||
|
||||
issue_code_rows = (
|
||||
await db.execute(
|
||||
select(
|
||||
AssetInventoryIssueORM.issue_code,
|
||||
func.count(AssetInventoryIssueORM.id),
|
||||
)
|
||||
.where(AssetInventoryIssueORM.status == "OPEN")
|
||||
.group_by(AssetInventoryIssueORM.issue_code)
|
||||
)
|
||||
).all()
|
||||
status["issues"]["by_code"] = {
|
||||
str(code or "unknown"): int(count or 0)
|
||||
for code, count in issue_code_rows
|
||||
}
|
||||
|
||||
status["ok"] = (
|
||||
status["source_roots"]["inaccessible_count"] == 0
|
||||
and status["orbit_roots"]["inaccessible_count"] == 0
|
||||
and status["source_assets"]["parse_failed_count"] == 0
|
||||
and status["orbit_assets"]["parse_failed_count"] == 0
|
||||
and status["issues"]["error_count"] == 0
|
||||
)
|
||||
except Exception as exc:
|
||||
status["error"] = str(exc)
|
||||
|
||||
return status
|
||||
|
||||
|
||||
async def _check_wsl_runtime() -> Dict[str, Any]:
|
||||
status = {
|
||||
"ok": False,
|
||||
@@ -989,6 +1257,7 @@ async def get_health_status(
|
||||
dinsar_bridge_status = await _check_dinsar_bridge()
|
||||
source_roots_status = await _check_source_roots()
|
||||
product_packages_status = await _check_product_packages()
|
||||
asset_inventory_status = await _check_asset_inventory()
|
||||
wsl_runtime_status = await _check_wsl_runtime()
|
||||
pairing_system_status = await pairing_state_service.get_pairing_system_status()
|
||||
engines_status = {"ok": None, "overall": None, "engines": []}
|
||||
@@ -1005,6 +1274,7 @@ async def get_health_status(
|
||||
dinsar_bridge_status.get("ok"),
|
||||
source_roots_status.get("ok"),
|
||||
product_packages_status.get("ok"),
|
||||
asset_inventory_status.get("ok"),
|
||||
wsl_runtime_status.get("ok"),
|
||||
pairing_system_status.get("ok"),
|
||||
(not settings.TIMESERIES_ENABLED) or timeseries_result_catalog_status.get("ok"),
|
||||
@@ -1028,6 +1298,7 @@ async def get_health_status(
|
||||
"dinsar_bridge": dinsar_bridge_status,
|
||||
"source_roots": source_roots_status,
|
||||
"product_packages": product_packages_status,
|
||||
"asset_inventory": asset_inventory_status,
|
||||
"wsl_runtime": wsl_runtime_status,
|
||||
"pairing_system": pairing_system_status,
|
||||
"idl": {
|
||||
|
||||
@@ -22,6 +22,7 @@ from ..config import settings
|
||||
from ..models import SystemJobORM, DinsarResultORM, HazardPointORM, DinsarTaskItemORM, PsTaskItemORM, RadarDataORM, SARSceneGeoORM, FloodDetectionORM, WaterDetectionORM, GF3ProcessingORM, AiDiagnosisORM
|
||||
from ..scheduler import scan_data_job
|
||||
from .data_service import data_service
|
||||
from .asset_inventory_service import asset_inventory_service
|
||||
from .dinsar_compat_service import dinsar_compat_service
|
||||
from .dinsar_naming import build_run_key
|
||||
from .dinsar_production_service import dinsar_production_service
|
||||
@@ -67,6 +68,7 @@ JOB_TYPE_SCAN_DATA = "SCAN_DATA"
|
||||
JOB_TYPE_SCAN_DINSAR = "SCAN_DINSAR"
|
||||
JOB_TYPE_COPY_DATA = "COPY_DATA"
|
||||
JOB_TYPE_UNPACK = "UNPACK_ARCHIVES"
|
||||
JOB_TYPE_UNPACK_SENTINEL1 = "UNPACK_SENTINEL1"
|
||||
JOB_TYPE_AI_TRAIN = "AI_TRAIN"
|
||||
JOB_TYPE_AI_PREDICT = "AI_PREDICT"
|
||||
JOB_TYPE_AI_ANALYZE = "AI_ANALYZE"
|
||||
@@ -85,6 +87,7 @@ JOB_TYPE_PYINT_RUN = "PYINT_RUN"
|
||||
JOB_TYPE_PUBLISH_DINSAR_PRODUCTS = "PUBLISH_DINSAR_PRODUCTS"
|
||||
JOB_TYPE_REBUILD_DINSAR_CATALOG = "REBUILD_DINSAR_CATALOG"
|
||||
JOB_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG"
|
||||
JOB_TYPE_SCAN_ASSET_INVENTORY = "SCAN_ASSET_INVENTORY"
|
||||
|
||||
COPY_ALLOWED_STATUSES = {"PENDING", "IN_PROGRESS", "COMPLETED", "FAILED"}
|
||||
|
||||
@@ -225,6 +228,34 @@ async def _handle_scan_data(job: SystemJobORM) -> None:
|
||||
await _run_scan_data_custom(job.task_id, payload)
|
||||
|
||||
|
||||
async def _handle_scan_asset_inventory(job: SystemJobORM) -> None:
|
||||
if not job.task_id:
|
||||
raise ValueError("SCAN_ASSET_INVENTORY requires task_id for progress tracking.")
|
||||
payload = job.payload or {}
|
||||
await task_service.start_task(job.task_id, message="Source/orbit asset inventory scan started")
|
||||
async with AsyncSessionLocal() as db:
|
||||
result = await asset_inventory_service.scan_configured_roots(
|
||||
db,
|
||||
inventory_types=payload.get("inventory_types") or None,
|
||||
root_ids=payload.get("root_ids") or None,
|
||||
bind_orbits=bool(payload.get("bind_orbits", True)),
|
||||
task_id=job.task_id,
|
||||
)
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
status="COMPLETED",
|
||||
progress=100,
|
||||
message=(
|
||||
"Asset inventory scan completed: "
|
||||
f"sources={result.get('source_assets', 0)}, "
|
||||
f"orbits={result.get('orbit_assets', 0)}, "
|
||||
f"matched={((result.get('binding') or {}).get('matched_count', 0))}, "
|
||||
f"missing={((result.get('binding') or {}).get('missing_count', 0))}"
|
||||
),
|
||||
db=db,
|
||||
)
|
||||
|
||||
|
||||
def _resolve_hazard_shp_path() -> str:
|
||||
base_dir = settings.HAZARD_POINTS_DIR
|
||||
filename = settings.HAZARD_POINTS_FILENAME
|
||||
@@ -418,6 +449,16 @@ async def _handle_unpack_archives(job: SystemJobORM) -> None:
|
||||
await run_unpack_task(job.task_id)
|
||||
|
||||
|
||||
async def _handle_unpack_sentinel1(job: SystemJobORM) -> None:
|
||||
if not job.task_id:
|
||||
raise ValueError("UNPACK_SENTINEL1 requires task_id for progress tracking.")
|
||||
payload = job.payload or {}
|
||||
if payload.get("asset_id"):
|
||||
await asset_inventory_service.run_sentinel1_unpack_task(job.task_id, payload)
|
||||
return
|
||||
await asset_inventory_service.run_sentinel1_unpack_batch_task(job.task_id, payload)
|
||||
|
||||
|
||||
async def _handle_ai_train(job: SystemJobORM) -> None:
|
||||
if not job.task_id:
|
||||
raise ValueError("AI_TRAIN requires task_id for progress tracking.")
|
||||
@@ -3849,6 +3890,7 @@ async def _handle_rebuild_psinsar_catalog(job: SystemJobORM) -> None:
|
||||
|
||||
_HANDLERS = {
|
||||
JOB_TYPE_SCAN_DATA: _handle_scan_data,
|
||||
JOB_TYPE_SCAN_ASSET_INVENTORY: _handle_scan_asset_inventory,
|
||||
JOB_TYPE_SCAN_DINSAR: _handle_scan_dinsar,
|
||||
JOB_TYPE_PUBLISH_DINSAR_PRODUCTS: _handle_publish_dinsar_products_clean,
|
||||
JOB_TYPE_REBUILD_DINSAR_CATALOG: _handle_rebuild_dinsar_catalog_clean,
|
||||
@@ -3864,6 +3906,7 @@ _HANDLERS = {
|
||||
JOB_TYPE_REBUILD_PSINSAR_CATALOG: _handle_rebuild_psinsar_catalog,
|
||||
JOB_TYPE_COPY_DATA: _handle_copy_data,
|
||||
JOB_TYPE_UNPACK: _handle_unpack_archives,
|
||||
JOB_TYPE_UNPACK_SENTINEL1: _handle_unpack_sentinel1,
|
||||
JOB_TYPE_AI_TRAIN: _handle_ai_train,
|
||||
JOB_TYPE_AI_PREDICT: _handle_ai_predict,
|
||||
JOB_TYPE_AI_ANALYZE: _handle_ai_analyze,
|
||||
|
||||
@@ -36,7 +36,7 @@ def _satellite_family_expr(alias: str) -> str:
|
||||
f"COALESCE(NULLIF({alias}.satellite_family, ''), "
|
||||
f"CASE "
|
||||
f"WHEN {compact} IN ('LT1', 'LT1A', 'LT1B', 'LUTAN1', 'LUTAN1A', 'LUTAN1B') THEN 'LT1' "
|
||||
f"WHEN {compact} IN ('S1', 'S1A', 'S1B', 'SENTINEL1', 'SENTINEL1A', 'SENTINEL1B') THEN 'S1' "
|
||||
f"WHEN {compact} IN ('S1', 'S1A', 'S1B', 'S1C', 'SENTINEL1', 'SENTINEL1A', 'SENTINEL1B', 'SENTINEL1C') THEN 'S1' "
|
||||
f"WHEN NULLIF({alias}.satellite, '') IS NOT NULL THEN upper({alias}.satellite) "
|
||||
f"ELSE NULL END)"
|
||||
)
|
||||
|
||||
@@ -10,11 +10,14 @@ from typing import Any, Dict, List
|
||||
from ..config import settings
|
||||
from .orbit_converter import get_source_orbit_inventory
|
||||
from .pyint_service import (
|
||||
discover_s1_scene_sources,
|
||||
discover_lt1_archives,
|
||||
infer_scene_date_from_archives,
|
||||
infer_task_identity,
|
||||
validate_pyint_root_dir,
|
||||
)
|
||||
from .asset_inventory_service import _configured_sentinel1_archive_dirs, _parse_s1_source_name
|
||||
from ..utils import normalize_satellite_family
|
||||
|
||||
|
||||
VALID_DEM_MODES = {"local_fabdem", "opentopo", "prepared_file"}
|
||||
@@ -62,6 +65,198 @@ def _infer_satellite_from_archives(paths: List[str]) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
def _normalize_s1_satellite(value: Any) -> str:
|
||||
text = str(value or "").strip().upper().replace("-", "").replace("_", "")
|
||||
if text in {"S1A", "S1B", "S1C"}:
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
def _find_s1_zip_by_logical_uid(logical_uid: str) -> str:
|
||||
logical = str(logical_uid or "").strip()
|
||||
if not logical:
|
||||
return ""
|
||||
expected_name = logical if logical.lower().endswith(".zip") else f"{logical}.zip"
|
||||
for root in _configured_sentinel1_archive_dirs():
|
||||
if not root or not os.path.isdir(root):
|
||||
continue
|
||||
direct_candidate = os.path.join(root, expected_name)
|
||||
if os.path.isfile(direct_candidate):
|
||||
return _normalize_path(direct_candidate)
|
||||
for current_root, _, files in os.walk(root):
|
||||
if expected_name in files:
|
||||
return _normalize_path(os.path.join(current_root, expected_name))
|
||||
return ""
|
||||
|
||||
|
||||
def _is_s1_safe_dir(path: str) -> bool:
|
||||
normalized = _normalize_path(path)
|
||||
if not normalized or not os.path.isdir(normalized):
|
||||
return False
|
||||
return os.path.isfile(os.path.join(normalized, "manifest.safe"))
|
||||
|
||||
|
||||
def _resolve_s1_scene_input(
|
||||
*,
|
||||
role: str,
|
||||
scene_path: str,
|
||||
pair_meta: Dict[str, Any],
|
||||
) -> Dict[str, Any]:
|
||||
normalized_scene = _normalize_path(scene_path)
|
||||
scene_name = os.path.basename(normalized_scene.rstrip("\\/"))
|
||||
parsed = _parse_s1_source_name(scene_name) or {}
|
||||
logical_uid = str(parsed.get("logical_product_uid") or "").strip()
|
||||
resolved_path = ""
|
||||
input_kind = ""
|
||||
resolution_method = ""
|
||||
|
||||
if normalized_scene.lower().endswith(".zip") and os.path.isfile(normalized_scene):
|
||||
resolved_path = normalized_scene
|
||||
input_kind = "zip"
|
||||
resolution_method = "task_or_pair_meta"
|
||||
elif _is_s1_safe_dir(normalized_scene):
|
||||
resolved_path = normalized_scene
|
||||
input_kind = "safe_dir"
|
||||
resolution_method = "task_or_pair_meta"
|
||||
elif normalized_scene.lower().endswith(".safe") and os.path.isdir(normalized_scene):
|
||||
resolved_path = normalized_scene
|
||||
input_kind = "safe_dir"
|
||||
resolution_method = "task_or_pair_meta"
|
||||
else:
|
||||
sibling_zip = ""
|
||||
if normalized_scene.lower().endswith(".safe"):
|
||||
sibling_zip = normalized_scene[:-5] + ".zip"
|
||||
if sibling_zip and os.path.isfile(sibling_zip):
|
||||
resolved_path = _normalize_path(sibling_zip)
|
||||
input_kind = "zip"
|
||||
resolution_method = "safe_sibling_zip"
|
||||
elif logical_uid:
|
||||
looked_up_zip = _find_s1_zip_by_logical_uid(logical_uid)
|
||||
if looked_up_zip:
|
||||
resolved_path = looked_up_zip
|
||||
input_kind = "zip"
|
||||
resolution_method = "source_pool_lookup"
|
||||
|
||||
expected_name = scene_name
|
||||
if input_kind == "zip":
|
||||
if not expected_name.lower().endswith(".zip"):
|
||||
expected_name = f"{logical_uid}.zip" if logical_uid else os.path.basename(resolved_path)
|
||||
elif input_kind == "safe_dir":
|
||||
if not expected_name.lower().endswith(".safe"):
|
||||
expected_name = f"{logical_uid}.SAFE" if logical_uid else os.path.basename(resolved_path)
|
||||
|
||||
satellite = _normalize_s1_satellite(pair_meta.get(f"{role}_satellite")) or _normalize_s1_satellite(parsed.get("satellite"))
|
||||
date_text = str(pair_meta.get(f"{role}_imaging_date") or parsed.get("imaging_date") or "").strip()
|
||||
return {
|
||||
"role": role,
|
||||
"scene_path": normalized_scene,
|
||||
"scene_name": scene_name,
|
||||
"logical_product_uid": logical_uid,
|
||||
"satellite": satellite,
|
||||
"date": date_text,
|
||||
"resolved": bool(resolved_path),
|
||||
"path": resolved_path,
|
||||
"input_kind": input_kind,
|
||||
"resolution_method": resolution_method,
|
||||
"expected_name": expected_name,
|
||||
"staged_path": "",
|
||||
}
|
||||
|
||||
|
||||
def _resolve_s1_zip_file(
|
||||
*,
|
||||
role: str,
|
||||
scene_path: str,
|
||||
pair_meta: Dict[str, Any],
|
||||
) -> Dict[str, Any]:
|
||||
normalized_scene = _normalize_path(scene_path)
|
||||
scene_name = os.path.basename(normalized_scene)
|
||||
parsed = _parse_s1_source_name(scene_name) or {}
|
||||
logical_uid = str(parsed.get("logical_product_uid") or "").strip()
|
||||
direct_zip = ""
|
||||
if normalized_scene.lower().endswith(".zip") and os.path.isfile(normalized_scene):
|
||||
direct_zip = normalized_scene
|
||||
elif normalized_scene.lower().endswith(".safe"):
|
||||
sibling_zip = normalized_scene[:-5] + ".zip"
|
||||
if os.path.isfile(sibling_zip):
|
||||
direct_zip = _normalize_path(sibling_zip)
|
||||
if not direct_zip and logical_uid:
|
||||
direct_zip = _find_s1_zip_by_logical_uid(logical_uid)
|
||||
|
||||
satellite = _normalize_s1_satellite(pair_meta.get(f"{role}_satellite")) or _normalize_s1_satellite(parsed.get("satellite"))
|
||||
date_text = str(pair_meta.get(f"{role}_imaging_date") or parsed.get("imaging_date") or "").strip()
|
||||
return {
|
||||
"role": role,
|
||||
"scene_path": normalized_scene,
|
||||
"scene_name": scene_name,
|
||||
"logical_product_uid": logical_uid,
|
||||
"satellite": satellite,
|
||||
"date": date_text,
|
||||
"resolved": bool(direct_zip),
|
||||
"path": direct_zip,
|
||||
"resolution_method": "scene_or_source_pool_lookup" if direct_zip else "",
|
||||
"expected_name": f"{logical_uid}.zip" if logical_uid else "",
|
||||
"staged_path": "",
|
||||
}
|
||||
|
||||
|
||||
def _resolve_s1_orbit_file(
|
||||
*,
|
||||
role: str,
|
||||
pair_meta: Dict[str, Any],
|
||||
task_dir: str,
|
||||
) -> Dict[str, Any]:
|
||||
direct_path = _normalize_path(pair_meta.get(f"{role}_orbit_file_path"))
|
||||
if direct_path and os.path.isfile(direct_path):
|
||||
return {
|
||||
"role": role,
|
||||
"resolved": True,
|
||||
"path": direct_path,
|
||||
"resolution_method": "pair_meta",
|
||||
"expected_name": os.path.basename(direct_path),
|
||||
"satellite": _normalize_s1_satellite(pair_meta.get(f"{role}_satellite")),
|
||||
"date": str(pair_meta.get(f"{role}_imaging_date") or "").strip(),
|
||||
"staged_path": "",
|
||||
}
|
||||
|
||||
role_orbit_dir = os.path.join(task_dir, "orbit")
|
||||
satellite = _normalize_s1_satellite(pair_meta.get(f"{role}_satellite"))
|
||||
date_text = str(pair_meta.get(f"{role}_imaging_date") or "").strip()
|
||||
candidates: List[str] = []
|
||||
if os.path.isdir(role_orbit_dir):
|
||||
for entry in os.scandir(role_orbit_dir):
|
||||
if not entry.is_file():
|
||||
continue
|
||||
if not entry.name.lower().endswith(".eof"):
|
||||
continue
|
||||
if satellite and satellite not in entry.name.upper():
|
||||
continue
|
||||
candidates.append(_normalize_path(entry.path))
|
||||
candidates.sort()
|
||||
if candidates:
|
||||
return {
|
||||
"role": role,
|
||||
"resolved": True,
|
||||
"path": candidates[0],
|
||||
"resolution_method": "task_orbit_dir",
|
||||
"expected_name": os.path.basename(candidates[0]),
|
||||
"satellite": satellite,
|
||||
"date": date_text,
|
||||
"staged_path": "",
|
||||
}
|
||||
return {
|
||||
"role": role,
|
||||
"resolved": False,
|
||||
"path": "",
|
||||
"resolution_method": "",
|
||||
"expected_name": "",
|
||||
"satellite": satellite,
|
||||
"date": date_text,
|
||||
"staged_path": "",
|
||||
"error": f"{role} Sentinel-1 EOF 缺失",
|
||||
}
|
||||
|
||||
|
||||
def _get_dem_mode() -> str:
|
||||
raw_mode = str(getattr(settings, "PYINT_DEM_MODE", "local_fabdem") or "local_fabdem").strip().lower()
|
||||
if raw_mode not in VALID_DEM_MODES:
|
||||
@@ -569,6 +764,427 @@ def resolve_pyint_task_input_assets(
|
||||
}
|
||||
|
||||
|
||||
def build_pyint_input_preview(root_dir: str, num_to_process: int = 0) -> Dict[str, Any]:
|
||||
validation = validate_pyint_root_dir(root_dir, num_to_process)
|
||||
dem_summary = get_pyint_dem_summary()
|
||||
orbit_context = get_pyint_orbit_context()
|
||||
|
||||
warnings: List[str] = list(dem_summary.get("warnings") or [])
|
||||
blockers: List[str] = list(dem_summary.get("blockers") or [])
|
||||
task_summaries: List[Dict[str, Any]] = []
|
||||
resolved_task_count = 0
|
||||
missing_task_count = 0
|
||||
effective_orbit_policy = _get_orbit_policy()
|
||||
|
||||
for task_dir in validation.get("task_dirs", []) or []:
|
||||
task_summary = resolve_pyint_task_input_assets(
|
||||
task_dir,
|
||||
dem_summary=dem_summary,
|
||||
orbit_context=orbit_context,
|
||||
)
|
||||
task_summaries.append(task_summary)
|
||||
task_orbit_policy = str(((task_summary.get("input_assets") or {}).get("orbits") or {}).get("policy") or "").strip()
|
||||
if task_orbit_policy:
|
||||
effective_orbit_policy = task_orbit_policy
|
||||
if task_summary.get("warnings"):
|
||||
warnings.extend(
|
||||
f"{task_summary['task_alias']}: {item}"
|
||||
for item in task_summary["warnings"]
|
||||
)
|
||||
if task_summary.get("blockers"):
|
||||
blockers.extend(
|
||||
f"{task_summary['task_alias']}: {item}"
|
||||
for item in task_summary["blockers"]
|
||||
)
|
||||
if task_summary["input_assets"]["orbits"]["missing_count"] == 0:
|
||||
resolved_task_count += 1
|
||||
else:
|
||||
missing_task_count += 1
|
||||
|
||||
allow_submit = not blockers
|
||||
precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary()
|
||||
return {
|
||||
"root_dir": validation["root_dir"],
|
||||
"mode": validation["mode"],
|
||||
"task_count": len(task_summaries),
|
||||
"selected_task_count": len(task_summaries),
|
||||
"allow_submit": allow_submit,
|
||||
"warnings": warnings,
|
||||
"blockers": blockers,
|
||||
"invalid_candidates": validation.get("invalid_candidates", []),
|
||||
"dem": dem_summary,
|
||||
"orbits": {
|
||||
"policy": effective_orbit_policy,
|
||||
"pool_root": orbit_context.get("pool_root", ""),
|
||||
"pool_exists": bool(orbit_context.get("pool_exists")),
|
||||
"resolved_task_count": resolved_task_count,
|
||||
"missing_task_count": missing_task_count,
|
||||
"duplicate_count": int(orbit_context.get("duplicate_count", 0) or 0),
|
||||
"warnings": list(orbit_context.get("warnings") or []),
|
||||
},
|
||||
"precise_orbit_bridge": precise_orbit_bridge,
|
||||
"tasks": task_summaries,
|
||||
}
|
||||
|
||||
|
||||
def summarize_preview_blockers(preview: Dict[str, Any], limit: int = 8) -> str:
|
||||
blockers = [str(item).strip() for item in (preview.get("blockers") or []) if str(item).strip()]
|
||||
if not blockers:
|
||||
return ""
|
||||
if len(blockers) <= limit:
|
||||
return "; ".join(blockers)
|
||||
return "; ".join(blockers[:limit]) + f"; 其余 {len(blockers) - limit} 项已省略"
|
||||
|
||||
|
||||
def materialize_pyint_input_assets(
|
||||
*,
|
||||
task_summary: Dict[str, Any],
|
||||
input_assets_dir: str,
|
||||
project_name: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
input_assets_dir = _normalize_path(input_assets_dir)
|
||||
os.makedirs(input_assets_dir, exist_ok=True)
|
||||
|
||||
record_enabled = bool(getattr(settings, "PYINT_RECORD_INPUT_ASSETS", True))
|
||||
orbits_dir = os.path.join(input_assets_dir, "orbits")
|
||||
dem_dir = os.path.join(input_assets_dir, "dem")
|
||||
downloads_dir = os.path.join(input_assets_dir, "downloads")
|
||||
if record_enabled:
|
||||
os.makedirs(orbits_dir, exist_ok=True)
|
||||
os.makedirs(dem_dir, exist_ok=True)
|
||||
os.makedirs(downloads_dir, exist_ok=True)
|
||||
|
||||
manifest = _copy_json_safe(task_summary.get("input_assets") or {})
|
||||
manifest["generated_at"] = _utc_now_text()
|
||||
manifest["task_name"] = task_summary.get("task_name")
|
||||
manifest["task_alias"] = task_summary.get("task_alias")
|
||||
manifest["pair_key"] = task_summary.get("pair_key")
|
||||
manifest["task_dir"] = task_summary.get("task_dir")
|
||||
manifest["allow_submit"] = bool(task_summary.get("allow_submit"))
|
||||
manifest["warnings"] = list(task_summary.get("warnings") or [])
|
||||
manifest["blockers"] = list(task_summary.get("blockers") or [])
|
||||
|
||||
dem_summary = manifest.get("dem") or {}
|
||||
if project_name:
|
||||
dem_summary["resolved_output_dir"] = os.path.join(_normalize_path(settings.PYINT_DEM_ROOT), project_name)
|
||||
manifest["dem"] = dem_summary
|
||||
|
||||
orbits_summary = manifest.get("orbits") or {}
|
||||
staged_count = 0
|
||||
precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary()
|
||||
should_stage_orbits = record_enabled and (
|
||||
str(orbits_summary.get("policy") or "").strip().lower() == "stage_txt"
|
||||
or precise_orbit_bridge.get("enabled")
|
||||
or str((manifest.get("task_source") or {}).get("satellite_family") or "").strip().upper() == "S1"
|
||||
)
|
||||
if should_stage_orbits:
|
||||
for role in ("master", "slave"):
|
||||
orbit_item = orbits_summary.get(role) or {}
|
||||
orbit_path = _normalize_path(orbit_item.get("path"))
|
||||
expected_name = str(orbit_item.get("expected_name") or "").strip()
|
||||
if not orbit_item.get("resolved") or not orbit_path or not expected_name:
|
||||
continue
|
||||
target_path = os.path.join(orbits_dir, expected_name)
|
||||
if not os.path.exists(target_path):
|
||||
shutil.copy2(orbit_path, target_path)
|
||||
orbit_item["staged_path"] = target_path
|
||||
orbit_item["stage_operation"] = "copied"
|
||||
orbit_item["stage_reason"] = "precise_orbit_bridge" if precise_orbit_bridge.get("enabled") else "stage_txt_policy"
|
||||
staged_count += 1
|
||||
orbits_summary[role] = orbit_item
|
||||
manifest["orbits"] = orbits_summary
|
||||
|
||||
download_staged_count = 0
|
||||
task_source = manifest.get("task_source") or {}
|
||||
if record_enabled and str(task_source.get("satellite_family") or "").strip().upper() == "S1":
|
||||
production_inputs = task_source.get("production_inputs") or {}
|
||||
for role_key in ("master_scene", "slave_scene", "master_zip", "slave_zip"):
|
||||
scene_item = production_inputs.get(role_key) or {}
|
||||
scene_path = _normalize_path(scene_item.get("path"))
|
||||
expected_name = str(scene_item.get("expected_name") or os.path.basename(scene_path) or "").strip()
|
||||
if not scene_item.get("resolved") or not scene_path or not expected_name:
|
||||
continue
|
||||
input_kind = str(scene_item.get("input_kind") or "").strip().lower()
|
||||
if input_kind == "safe_dir" or os.path.isdir(scene_path):
|
||||
scene_item["staged_path"] = scene_path
|
||||
scene_item["stage_operation"] = "source_reference"
|
||||
production_inputs[role_key] = scene_item
|
||||
continue
|
||||
target_path = os.path.join(downloads_dir, expected_name)
|
||||
if not os.path.exists(target_path):
|
||||
shutil.copy2(scene_path, target_path)
|
||||
scene_item["staged_path"] = target_path
|
||||
scene_item["stage_operation"] = "copied"
|
||||
production_inputs[role_key] = scene_item
|
||||
download_staged_count += 1
|
||||
task_source["production_inputs"] = production_inputs
|
||||
manifest["task_source"] = task_source
|
||||
|
||||
materialized = {
|
||||
"input_assets_dir": input_assets_dir,
|
||||
"record_enabled": record_enabled,
|
||||
"orbits_dir": orbits_dir if record_enabled else "",
|
||||
"dem_dir": dem_dir if record_enabled else "",
|
||||
"downloads_dir": downloads_dir if record_enabled else "",
|
||||
"orbits_staged_count": staged_count,
|
||||
"downloads_staged_count": download_staged_count,
|
||||
"task_manifest_path": "",
|
||||
"dem_summary_path": "",
|
||||
"orbit_summary_path": "",
|
||||
"input_assets": manifest,
|
||||
}
|
||||
|
||||
if not record_enabled:
|
||||
return materialized
|
||||
|
||||
task_manifest_path = os.path.join(input_assets_dir, "task_manifest.json")
|
||||
dem_summary_path = os.path.join(dem_dir, "dem_summary.json")
|
||||
orbit_summary_path = os.path.join(orbits_dir, "orbit_summary.json")
|
||||
|
||||
with open(task_manifest_path, "w", encoding="utf-8") as fp:
|
||||
json.dump(manifest, fp, ensure_ascii=False, indent=2)
|
||||
fp.write("\n")
|
||||
with open(dem_summary_path, "w", encoding="utf-8") as fp:
|
||||
json.dump(dem_summary, fp, ensure_ascii=False, indent=2)
|
||||
fp.write("\n")
|
||||
with open(orbit_summary_path, "w", encoding="utf-8") as fp:
|
||||
json.dump(orbits_summary, fp, ensure_ascii=False, indent=2)
|
||||
fp.write("\n")
|
||||
|
||||
materialized.update(
|
||||
{
|
||||
"task_manifest_path": task_manifest_path,
|
||||
"dem_summary_path": dem_summary_path,
|
||||
"orbit_summary_path": orbit_summary_path,
|
||||
}
|
||||
)
|
||||
return materialized
|
||||
|
||||
|
||||
def resolve_pyint_task_input_assets(
|
||||
task_dir: str,
|
||||
*,
|
||||
dem_summary: Dict[str, Any] | None = None,
|
||||
orbit_context: Dict[str, Any] | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
task_dir = _normalize_path(task_dir)
|
||||
task_identity = infer_task_identity(task_dir)
|
||||
pair_meta = task_identity["pair_meta"]
|
||||
satellite_family = str(task_identity.get("satellite_family") or "").strip().upper()
|
||||
|
||||
warnings: List[str] = []
|
||||
blockers: List[str] = []
|
||||
|
||||
if satellite_family == "S1":
|
||||
scene_sources = discover_s1_scene_sources(task_dir)
|
||||
master_archives = list(scene_sources.get("master", []) or [])
|
||||
slave_archives = list(scene_sources.get("slave", []) or [])
|
||||
master_date = task_identity["master_date"] or infer_scene_date_from_archives(master_archives)
|
||||
slave_date = task_identity["slave_date"] or infer_scene_date_from_archives(slave_archives)
|
||||
master_satellite = _normalize_s1_satellite(pair_meta.get("master_satellite")) or _normalize_s1_satellite(task_identity.get("master_satellite"))
|
||||
slave_satellite = _normalize_s1_satellite(pair_meta.get("slave_satellite")) or _normalize_s1_satellite(task_identity.get("slave_satellite"))
|
||||
|
||||
if not master_archives:
|
||||
blockers.append("master/ 未识别到 Sentinel-1 SAFE 源目录。")
|
||||
if not slave_archives:
|
||||
blockers.append("slave/ 未识别到 Sentinel-1 SAFE 源目录。")
|
||||
if not master_date:
|
||||
blockers.append("未能识别主影像日期。")
|
||||
if not slave_date:
|
||||
blockers.append("未能识别从影像日期。")
|
||||
|
||||
master_scene = _resolve_s1_scene_input(
|
||||
role="master",
|
||||
scene_path=master_archives[0] if master_archives else "",
|
||||
pair_meta=pair_meta,
|
||||
)
|
||||
slave_scene = _resolve_s1_scene_input(
|
||||
role="slave",
|
||||
scene_path=slave_archives[0] if slave_archives else "",
|
||||
pair_meta=pair_meta,
|
||||
)
|
||||
master_orbit = _resolve_s1_orbit_file(
|
||||
role="master",
|
||||
pair_meta=pair_meta,
|
||||
task_dir=task_dir,
|
||||
)
|
||||
slave_orbit = _resolve_s1_orbit_file(
|
||||
role="slave",
|
||||
pair_meta=pair_meta,
|
||||
task_dir=task_dir,
|
||||
)
|
||||
|
||||
if not master_scene.get("resolved"):
|
||||
blockers.append("master Sentinel-1 源场景缺失。")
|
||||
if not slave_scene.get("resolved"):
|
||||
blockers.append("slave Sentinel-1 源场景缺失。")
|
||||
if not master_orbit.get("resolved"):
|
||||
blockers.append(str(master_orbit.get("error") or "master Sentinel-1 EOF 缺失"))
|
||||
if not slave_orbit.get("resolved"):
|
||||
blockers.append(str(slave_orbit.get("error") or "slave Sentinel-1 EOF 缺失"))
|
||||
|
||||
task_source = {
|
||||
"task_dir": task_dir,
|
||||
"task_name": task_identity["task_name"],
|
||||
"task_alias": task_identity["task_alias"],
|
||||
"pair_key": task_identity["pair_key"],
|
||||
"satellite_family": "S1",
|
||||
"master_date": master_date,
|
||||
"slave_date": slave_date,
|
||||
"master_satellite": master_satellite,
|
||||
"slave_satellite": slave_satellite,
|
||||
"archives": {
|
||||
"master": master_archives,
|
||||
"slave": slave_archives,
|
||||
},
|
||||
"production_inputs": {
|
||||
"master_scene": master_scene,
|
||||
"slave_scene": slave_scene,
|
||||
},
|
||||
}
|
||||
orbits_summary = {
|
||||
"policy": "require_eof",
|
||||
"pool_root": "",
|
||||
"pool_exists": True,
|
||||
"master": master_orbit,
|
||||
"slave": slave_orbit,
|
||||
"resolved_count": int(bool(master_orbit.get("resolved"))) + int(bool(slave_orbit.get("resolved"))),
|
||||
"missing_count": int(not master_orbit.get("resolved")) + int(not slave_orbit.get("resolved")),
|
||||
"warnings": [],
|
||||
"stage_mode": "copy",
|
||||
"precise_orbit_bridge": {
|
||||
"enabled": False,
|
||||
"mode": "not_applicable",
|
||||
"strict": True,
|
||||
},
|
||||
}
|
||||
else:
|
||||
archives = discover_lt1_archives(task_dir)
|
||||
master_archives = list(archives.get("master", []) or [])
|
||||
slave_archives = list(archives.get("slave", []) or [])
|
||||
master_date = task_identity["master_date"] or infer_scene_date_from_archives(master_archives)
|
||||
slave_date = task_identity["slave_date"] or infer_scene_date_from_archives(slave_archives)
|
||||
master_satellite = _normalize_lt1_satellite(pair_meta.get("master_satellite")) or _infer_satellite_from_archives(master_archives)
|
||||
slave_satellite = _normalize_lt1_satellite(pair_meta.get("slave_satellite")) or _infer_satellite_from_archives(slave_archives)
|
||||
|
||||
if not master_archives:
|
||||
blockers.append("master/ 未发现 LT-1 原始输入(LT1*.tar.gz 或 LT1*.tiff)。")
|
||||
if not slave_archives:
|
||||
blockers.append("slave/ 未发现 LT-1 原始输入(LT1*.tar.gz 或 LT1*.tiff)。")
|
||||
if not master_date:
|
||||
blockers.append("未能识别主影像日期。")
|
||||
if not slave_date:
|
||||
blockers.append("未能识别从影像日期。")
|
||||
|
||||
orbit_policy = _get_orbit_policy()
|
||||
orbit_context = orbit_context or get_pyint_orbit_context()
|
||||
orbit_pool_root = orbit_context.get("pool_root", "")
|
||||
orbit_pool_exists = bool(orbit_context.get("pool_exists"))
|
||||
orbit_files = orbit_context.get("files", {}) or {}
|
||||
|
||||
orbit_warnings: List[str] = []
|
||||
if orbit_context.get("warnings"):
|
||||
orbit_warnings.extend(str(item) for item in orbit_context["warnings"] if item)
|
||||
|
||||
master_orbit = _resolve_orbit_file(
|
||||
role="master",
|
||||
satellite=master_satellite,
|
||||
date_text=master_date,
|
||||
pool_root=orbit_pool_root,
|
||||
orbit_files=orbit_files,
|
||||
)
|
||||
slave_orbit = _resolve_orbit_file(
|
||||
role="slave",
|
||||
satellite=slave_satellite,
|
||||
date_text=slave_date,
|
||||
pool_root=orbit_pool_root,
|
||||
orbit_files=orbit_files,
|
||||
)
|
||||
|
||||
for orbit_item in (master_orbit, slave_orbit):
|
||||
if orbit_item.get("resolved"):
|
||||
continue
|
||||
message = str(orbit_item.get("error") or f"{orbit_item.get('role')} orbit missing").strip()
|
||||
if orbit_policy == "validate_only":
|
||||
orbit_warnings.append(message)
|
||||
else:
|
||||
blockers.append(message)
|
||||
|
||||
if not orbit_pool_root:
|
||||
if orbit_policy == "validate_only":
|
||||
orbit_warnings.append("轨道池未配置,当前仅记录警告。")
|
||||
else:
|
||||
blockers.append("轨道池未配置。")
|
||||
elif not orbit_pool_exists:
|
||||
if orbit_policy == "validate_only":
|
||||
orbit_warnings.append(f"轨道池目录不可用: {orbit_pool_root}")
|
||||
else:
|
||||
blockers.append(f"轨道池目录不可用: {orbit_pool_root}")
|
||||
|
||||
warnings.extend(orbit_warnings)
|
||||
|
||||
precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary()
|
||||
task_source = {
|
||||
"task_dir": task_dir,
|
||||
"task_name": task_identity["task_name"],
|
||||
"task_alias": task_identity["task_alias"],
|
||||
"pair_key": task_identity["pair_key"],
|
||||
"satellite_family": "LT1",
|
||||
"master_date": master_date,
|
||||
"slave_date": slave_date,
|
||||
"master_satellite": master_satellite,
|
||||
"slave_satellite": slave_satellite,
|
||||
"archives": {
|
||||
"master": master_archives,
|
||||
"slave": slave_archives,
|
||||
},
|
||||
}
|
||||
orbits_summary = {
|
||||
"policy": orbit_policy,
|
||||
"pool_root": orbit_pool_root,
|
||||
"pool_exists": orbit_pool_exists,
|
||||
"master": master_orbit,
|
||||
"slave": slave_orbit,
|
||||
"resolved_count": int(bool(master_orbit.get("resolved"))) + int(bool(slave_orbit.get("resolved"))),
|
||||
"missing_count": int(not master_orbit.get("resolved")) + int(not slave_orbit.get("resolved")),
|
||||
"warnings": orbit_warnings,
|
||||
"stage_mode": "copy" if orbit_policy == "stage_txt" or precise_orbit_bridge.get("enabled") else "none",
|
||||
"precise_orbit_bridge": precise_orbit_bridge,
|
||||
}
|
||||
|
||||
dem_payload = _copy_json_safe(dem_summary or get_pyint_dem_summary())
|
||||
allow_submit = not blockers and bool(dem_payload.get("allow_submit", True))
|
||||
return {
|
||||
"task_name": task_identity["task_name"],
|
||||
"task_alias": task_identity["task_alias"],
|
||||
"pair_key": task_identity["pair_key"],
|
||||
"task_dir": task_dir,
|
||||
"master_date": master_date,
|
||||
"slave_date": slave_date,
|
||||
"master_satellite": master_satellite,
|
||||
"slave_satellite": slave_satellite,
|
||||
"satellite_family": satellite_family or normalize_satellite_family(master_satellite or slave_satellite),
|
||||
"archive_counts": {
|
||||
"master": len(master_archives),
|
||||
"slave": len(slave_archives),
|
||||
},
|
||||
"warnings": warnings,
|
||||
"blockers": blockers,
|
||||
"allow_submit": allow_submit,
|
||||
"task_source": task_source,
|
||||
"dem": dem_payload,
|
||||
"orbit_resolution": {
|
||||
"master": master_orbit,
|
||||
"slave": slave_orbit,
|
||||
},
|
||||
"input_assets": {
|
||||
"task_source": task_source,
|
||||
"dem": dem_payload,
|
||||
"orbits": orbits_summary,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def build_pyint_input_preview(root_dir: str, num_to_process: int = 0) -> Dict[str, Any]:
|
||||
validation = validate_pyint_root_dir(root_dir, num_to_process)
|
||||
dem_summary = get_pyint_dem_summary()
|
||||
@@ -626,106 +1242,3 @@ def build_pyint_input_preview(root_dir: str, num_to_process: int = 0) -> Dict[st
|
||||
"precise_orbit_bridge": precise_orbit_bridge,
|
||||
"tasks": task_summaries,
|
||||
}
|
||||
|
||||
|
||||
def summarize_preview_blockers(preview: Dict[str, Any], limit: int = 8) -> str:
|
||||
blockers = [str(item).strip() for item in (preview.get("blockers") or []) if str(item).strip()]
|
||||
if not blockers:
|
||||
return ""
|
||||
if len(blockers) <= limit:
|
||||
return "; ".join(blockers)
|
||||
return "; ".join(blockers[:limit]) + f"; 其余 {len(blockers) - limit} 项已省略"
|
||||
|
||||
|
||||
def materialize_pyint_input_assets(
|
||||
*,
|
||||
task_summary: Dict[str, Any],
|
||||
input_assets_dir: str,
|
||||
project_name: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
input_assets_dir = _normalize_path(input_assets_dir)
|
||||
os.makedirs(input_assets_dir, exist_ok=True)
|
||||
|
||||
record_enabled = bool(getattr(settings, "PYINT_RECORD_INPUT_ASSETS", True))
|
||||
orbits_dir = os.path.join(input_assets_dir, "orbits")
|
||||
dem_dir = os.path.join(input_assets_dir, "dem")
|
||||
if record_enabled:
|
||||
os.makedirs(orbits_dir, exist_ok=True)
|
||||
os.makedirs(dem_dir, exist_ok=True)
|
||||
|
||||
manifest = _copy_json_safe(task_summary.get("input_assets") or {})
|
||||
manifest["generated_at"] = _utc_now_text()
|
||||
manifest["task_name"] = task_summary.get("task_name")
|
||||
manifest["task_alias"] = task_summary.get("task_alias")
|
||||
manifest["pair_key"] = task_summary.get("pair_key")
|
||||
manifest["task_dir"] = task_summary.get("task_dir")
|
||||
manifest["allow_submit"] = bool(task_summary.get("allow_submit"))
|
||||
manifest["warnings"] = list(task_summary.get("warnings") or [])
|
||||
manifest["blockers"] = list(task_summary.get("blockers") or [])
|
||||
|
||||
dem_summary = manifest.get("dem") or {}
|
||||
if project_name:
|
||||
dem_summary["resolved_output_dir"] = os.path.join(_normalize_path(settings.PYINT_DEM_ROOT), project_name)
|
||||
manifest["dem"] = dem_summary
|
||||
|
||||
orbits_summary = manifest.get("orbits") or {}
|
||||
staged_count = 0
|
||||
precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary()
|
||||
should_stage_orbits = record_enabled and (
|
||||
str(orbits_summary.get("policy") or "").strip().lower() == "stage_txt"
|
||||
or precise_orbit_bridge.get("enabled")
|
||||
)
|
||||
if should_stage_orbits:
|
||||
for role in ("master", "slave"):
|
||||
orbit_item = orbits_summary.get(role) or {}
|
||||
orbit_path = _normalize_path(orbit_item.get("path"))
|
||||
expected_name = str(orbit_item.get("expected_name") or "").strip()
|
||||
if not orbit_item.get("resolved") or not orbit_path or not expected_name:
|
||||
continue
|
||||
target_path = os.path.join(orbits_dir, expected_name)
|
||||
if not os.path.exists(target_path):
|
||||
shutil.copy2(orbit_path, target_path)
|
||||
orbit_item["staged_path"] = target_path
|
||||
orbit_item["stage_operation"] = "copied"
|
||||
orbit_item["stage_reason"] = "precise_orbit_bridge" if precise_orbit_bridge.get("enabled") else "stage_txt_policy"
|
||||
staged_count += 1
|
||||
orbits_summary[role] = orbit_item
|
||||
manifest["orbits"] = orbits_summary
|
||||
|
||||
materialized = {
|
||||
"input_assets_dir": input_assets_dir,
|
||||
"record_enabled": record_enabled,
|
||||
"orbits_dir": orbits_dir if record_enabled else "",
|
||||
"dem_dir": dem_dir if record_enabled else "",
|
||||
"orbits_staged_count": staged_count,
|
||||
"task_manifest_path": "",
|
||||
"dem_summary_path": "",
|
||||
"orbit_summary_path": "",
|
||||
"input_assets": manifest,
|
||||
}
|
||||
|
||||
if not record_enabled:
|
||||
return materialized
|
||||
|
||||
task_manifest_path = os.path.join(input_assets_dir, "task_manifest.json")
|
||||
dem_summary_path = os.path.join(dem_dir, "dem_summary.json")
|
||||
orbit_summary_path = os.path.join(orbits_dir, "orbit_summary.json")
|
||||
|
||||
with open(task_manifest_path, "w", encoding="utf-8") as fp:
|
||||
json.dump(manifest, fp, ensure_ascii=False, indent=2)
|
||||
fp.write("\n")
|
||||
with open(dem_summary_path, "w", encoding="utf-8") as fp:
|
||||
json.dump(dem_summary, fp, ensure_ascii=False, indent=2)
|
||||
fp.write("\n")
|
||||
with open(orbit_summary_path, "w", encoding="utf-8") as fp:
|
||||
json.dump(orbits_summary, fp, ensure_ascii=False, indent=2)
|
||||
fp.write("\n")
|
||||
|
||||
materialized.update(
|
||||
{
|
||||
"task_manifest_path": task_manifest_path,
|
||||
"dem_summary_path": dem_summary_path,
|
||||
"orbit_summary_path": orbit_summary_path,
|
||||
}
|
||||
)
|
||||
return materialized
|
||||
|
||||
@@ -13,10 +13,12 @@ from typing import Any, Dict, Iterable, List, Optional
|
||||
|
||||
from ..config import get_env_text, read_bool_env, settings
|
||||
from .dinsar_naming import PAIR_META_FILENAME, build_fallback_pair_key, find_json_sidecar
|
||||
from ..utils import normalize_satellite_family
|
||||
from .wsl_service import run_wsl_exec
|
||||
|
||||
|
||||
LT1_INPUT_GLOBS = ("LT1*.tar.gz", "LT1*.tiff")
|
||||
S1_INPUT_GLOBS = ("S1*.zip",)
|
||||
DEFAULT_RANGE_LOOKS = 2
|
||||
DEFAULT_AZIMUTH_LOOKS = 2
|
||||
DEFAULT_DEM_RESOLUTION_M = 30.0
|
||||
@@ -334,6 +336,15 @@ def build_project_name(pair_key: str, run_key: str) -> str:
|
||||
return slugify_text(f"{pair_key}_{run_key}", default="pyint_project", max_len=120)
|
||||
|
||||
|
||||
def build_profile_project_name(satellite_family: Any, pair_key: str, run_key: str) -> str:
|
||||
family = str(normalize_satellite_family(satellite_family) or "").strip().upper()
|
||||
if family == "S1":
|
||||
return slugify_text(f"s1_{pair_key}_{run_key}", default="s1_pyint_project", max_len=120)
|
||||
if family == "LT1":
|
||||
return slugify_text(f"lt1_{pair_key}_{run_key}", default="lt1_pyint_project", max_len=120)
|
||||
return build_project_name(pair_key, run_key)
|
||||
|
||||
|
||||
def windows_path_to_wsl_mount(path: str) -> str:
|
||||
text = str(path or "").strip().strip('"').strip("'")
|
||||
if not text:
|
||||
@@ -377,6 +388,22 @@ def discover_lt1_archives(task_dir: str) -> Dict[str, List[str]]:
|
||||
return result
|
||||
|
||||
|
||||
def discover_s1_scene_sources(task_dir: str) -> Dict[str, List[str]]:
|
||||
task_path = Path(os.path.normpath(os.path.abspath(str(task_dir or "").strip())))
|
||||
pair_meta = find_json_sidecar(str(task_path), PAIR_META_FILENAME, max_levels=0) or {}
|
||||
result: Dict[str, List[str]] = {"master": [], "slave": []}
|
||||
for role in ("master", "slave"):
|
||||
explicit_path = str(pair_meta.get(f"{role}_path") or "").strip()
|
||||
role_dir = task_path / role
|
||||
candidates: List[str] = []
|
||||
if explicit_path:
|
||||
candidates.append(str(Path(explicit_path).resolve()))
|
||||
if not candidates and role_dir.is_dir() and (role_dir / "manifest.safe").is_file():
|
||||
candidates.append(str(role_dir.resolve()))
|
||||
result[role] = sorted(set(candidates))
|
||||
return result
|
||||
|
||||
|
||||
def infer_scene_date_from_archives(paths: Iterable[str]) -> str:
|
||||
dates = {
|
||||
date_text
|
||||
@@ -393,7 +420,14 @@ def infer_task_identity(task_dir: str) -> Dict[str, Any]:
|
||||
task_name = os.path.basename(os.path.normpath(task_dir))
|
||||
pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {}
|
||||
task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name
|
||||
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(task_alias, task_dir)
|
||||
master_satellite = str(pair_meta.get("master_satellite") or "").strip().upper()
|
||||
slave_satellite = str(pair_meta.get("slave_satellite") or "").strip().upper()
|
||||
satellite_family = normalize_satellite_family(master_satellite or slave_satellite)
|
||||
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(
|
||||
task_alias,
|
||||
task_dir,
|
||||
satellite_family=satellite_family,
|
||||
)
|
||||
master_date = normalize_date_text(pair_meta.get("master_imaging_date"))
|
||||
slave_date = normalize_date_text(pair_meta.get("slave_imaging_date"))
|
||||
return {
|
||||
@@ -403,6 +437,9 @@ def infer_task_identity(task_dir: str) -> Dict[str, Any]:
|
||||
"pair_meta": pair_meta,
|
||||
"master_date": master_date,
|
||||
"slave_date": slave_date,
|
||||
"master_satellite": master_satellite,
|
||||
"slave_satellite": slave_satellite,
|
||||
"satellite_family": satellite_family,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -39,6 +39,7 @@ from .dinsar_naming import (
|
||||
build_fallback_pair_key,
|
||||
find_json_sidecar,
|
||||
)
|
||||
from ..utils import normalize_satellite_family
|
||||
from .dinsar_result_layout_service import (
|
||||
RUN_CURRENT_DIRNAME,
|
||||
RUN_NATIVE_DIRNAME,
|
||||
@@ -203,10 +204,16 @@ def _resolve_candidate_identity(candidate: Dict[str, Any]) -> Dict[str, Any]:
|
||||
pair_meta.get("task_alias"),
|
||||
candidate.get("task_name"),
|
||||
) or "Task_unknown_unknown"
|
||||
satellite_family = normalize_satellite_family(
|
||||
pair_meta.get("master_satellite")
|
||||
or pair_meta.get("slave_satellite")
|
||||
or run_meta.get("master_satellite")
|
||||
or run_meta.get("slave_satellite")
|
||||
)
|
||||
pair_key = _first_text(
|
||||
run_meta.get("pair_key"),
|
||||
pair_meta.get("pair_key"),
|
||||
) or build_fallback_pair_key(task_alias, source_dir)
|
||||
) or build_fallback_pair_key(task_alias, source_dir, satellite_family=satellite_family)
|
||||
run_key = _first_text(run_meta.get("run_key")) or (
|
||||
"legacy_" + _stable_digest(candidate.get("engine_code"), pair_key, source_dir, candidate["primary_file"], length=16)
|
||||
)
|
||||
|
||||
@@ -10,7 +10,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from .. import database
|
||||
from ..config import settings, split_env_paths
|
||||
from ..models import ManagedRootORM, PathInventoryORM, ScanCursorORM
|
||||
from ..models import AssetInventoryStateORM, ManagedRootORM, PathInventoryORM, ScanCursorORM
|
||||
|
||||
|
||||
_SLUG_RE = re.compile(r"[^a-z0-9]+")
|
||||
@@ -87,6 +87,15 @@ def _cursor_type_for_scan_mode(scan_mode: str) -> str:
|
||||
return mapping.get(str(scan_mode or "").strip().lower(), "directory_walk")
|
||||
|
||||
|
||||
def _asset_inventory_type_for_root_role(root_role: str) -> Optional[str]:
|
||||
role = str(root_role or "").strip().lower()
|
||||
if role == "source_product_pool":
|
||||
return "source_product"
|
||||
if role == "orbit_asset_pool":
|
||||
return "orbit_asset"
|
||||
return None
|
||||
|
||||
|
||||
def _iter_multi_root_specs(
|
||||
*,
|
||||
env_var: str,
|
||||
@@ -169,6 +178,36 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
|
||||
scan_mode="archive_walk",
|
||||
)
|
||||
)
|
||||
source_product_paths = split_env_paths(settings.SOURCE_PRODUCT_DIRS)
|
||||
if not source_product_paths:
|
||||
source_product_paths = (
|
||||
split_env_paths(settings.INSAR_STORAGE_DIRS)
|
||||
+ split_env_paths(settings.MONITOR_RADAR_DIRS)
|
||||
)
|
||||
specs.extend(
|
||||
_iter_multi_root_specs(
|
||||
env_var="SOURCE_PRODUCT_DIRS",
|
||||
paths=source_product_paths,
|
||||
root_role="source_product_pool",
|
||||
display_prefix="Source Product Pool",
|
||||
scan_mode="file_pool",
|
||||
)
|
||||
)
|
||||
source_product_path_set = {_normalize_root_path(path) for path in source_product_paths}
|
||||
sentinel1_storage_paths = [
|
||||
path
|
||||
for path in split_env_paths(settings.SENTINEL1_STORAGE_DIRS)
|
||||
if _normalize_root_path(path) not in source_product_path_set
|
||||
]
|
||||
specs.extend(
|
||||
_iter_multi_root_specs(
|
||||
env_var="SENTINEL1_STORAGE_DIRS",
|
||||
paths=sentinel1_storage_paths,
|
||||
root_role="source_product_pool",
|
||||
display_prefix="Sentinel-1 Storage Pool",
|
||||
scan_mode="file_pool",
|
||||
)
|
||||
)
|
||||
specs.extend(
|
||||
_iter_multi_root_specs(
|
||||
env_var="INSAR_STORAGE_DIRS",
|
||||
@@ -214,6 +253,18 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
|
||||
scan_mode="scene_directory",
|
||||
)
|
||||
)
|
||||
orbit_source_paths = split_env_paths(settings.ORBIT_SOURCE_DIRS)
|
||||
if not orbit_source_paths:
|
||||
orbit_source_paths = split_env_paths(settings.MONITOR_ORBIT_DIR)
|
||||
specs.extend(
|
||||
_iter_multi_root_specs(
|
||||
env_var="ORBIT_SOURCE_DIRS",
|
||||
paths=orbit_source_paths,
|
||||
root_role="orbit_asset_pool",
|
||||
display_prefix="Orbit Asset Pool",
|
||||
scan_mode="file_pool",
|
||||
)
|
||||
)
|
||||
specs.extend(
|
||||
_iter_single_root_specs(
|
||||
env_var="MONITOR_ORBIT_DIR",
|
||||
@@ -374,6 +425,43 @@ class RootRegistryService:
|
||||
changed = True
|
||||
return "updated" if changed else None
|
||||
|
||||
async def _ensure_asset_inventory_state(self, db: AsyncSession, root: ManagedRootORM) -> Optional[str]:
|
||||
inventory_type = _asset_inventory_type_for_root_role(root.root_role)
|
||||
if not inventory_type:
|
||||
return None
|
||||
|
||||
result = await db.execute(
|
||||
select(AssetInventoryStateORM).where(
|
||||
AssetInventoryStateORM.root_ref_id == root.id,
|
||||
AssetInventoryStateORM.inventory_type == inventory_type,
|
||||
)
|
||||
)
|
||||
state = result.scalar_one_or_none()
|
||||
if state is None:
|
||||
state = AssetInventoryStateORM(
|
||||
root_ref_id=root.id,
|
||||
inventory_type=inventory_type,
|
||||
root_path=root.path,
|
||||
scan_mode=root.scan_mode,
|
||||
status="NEVER_SCANNED",
|
||||
needs_rescan=True,
|
||||
metadata_json={
|
||||
"root_role": root.root_role,
|
||||
"created_by": "root_registry_sync",
|
||||
},
|
||||
)
|
||||
db.add(state)
|
||||
return "created"
|
||||
|
||||
changed = False
|
||||
if state.root_path != root.path:
|
||||
state.root_path = root.path
|
||||
changed = True
|
||||
if state.scan_mode != root.scan_mode:
|
||||
state.scan_mode = root.scan_mode
|
||||
changed = True
|
||||
return "updated" if changed else None
|
||||
|
||||
async def sync_from_settings(self, db: Optional[AsyncSession] = None) -> Dict[str, Any]:
|
||||
generated_session = db is None
|
||||
if generated_session:
|
||||
@@ -394,6 +482,8 @@ class RootRegistryService:
|
||||
disabled = 0
|
||||
cursor_created = 0
|
||||
cursor_updated = 0
|
||||
inventory_state_created = 0
|
||||
inventory_state_updated = 0
|
||||
synced_codes: set[str] = set()
|
||||
|
||||
for spec in specs:
|
||||
@@ -448,6 +538,12 @@ class RootRegistryService:
|
||||
elif cursor_change == "updated":
|
||||
cursor_updated += 1
|
||||
|
||||
inventory_state_change = await self._ensure_asset_inventory_state(db, row)
|
||||
if inventory_state_change == "created":
|
||||
inventory_state_created += 1
|
||||
elif inventory_state_change == "updated":
|
||||
inventory_state_updated += 1
|
||||
|
||||
for row in existing_rows:
|
||||
if row.root_code in synced_codes:
|
||||
continue
|
||||
@@ -464,6 +560,7 @@ class RootRegistryService:
|
||||
"updated": updated,
|
||||
"disabled": disabled,
|
||||
"cursor_created_or_updated": cursor_created + cursor_updated,
|
||||
"asset_inventory_state_created_or_updated": inventory_state_created + inventory_state_updated,
|
||||
"summary": summary,
|
||||
}
|
||||
except Exception:
|
||||
|
||||
@@ -14,7 +14,7 @@ from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.future import select
|
||||
from sqlalchemy import and_, cast, func, or_
|
||||
from sqlalchemy import and_, case, cast, func, or_
|
||||
from sqlalchemy.orm import aliased
|
||||
|
||||
from geoalchemy2 import Geography
|
||||
@@ -43,11 +43,60 @@ from .dinsar_naming import build_pair_key, build_task_alias, ensure_unique_task_
|
||||
from .pairing_state_service import pairing_state_service
|
||||
|
||||
|
||||
PAIRING_POLICY_VERSION = "2026.05.raw-source.v1"
|
||||
PAIRING_POLICY_VERSION = "2026.05.raw-source.v2"
|
||||
PAIRING_WARNING_CANDIDATE_THRESHOLD = 3000
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _normalized_satellite_family_expr(alias):
|
||||
compact_satellite = func.upper(
|
||||
func.replace(
|
||||
func.replace(
|
||||
func.replace(func.coalesce(alias.satellite, ""), "-", ""),
|
||||
"_",
|
||||
"",
|
||||
),
|
||||
" ",
|
||||
"",
|
||||
)
|
||||
)
|
||||
inferred_family = case(
|
||||
(
|
||||
compact_satellite.in_(
|
||||
["LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"]
|
||||
),
|
||||
"LT1",
|
||||
),
|
||||
(
|
||||
compact_satellite.in_(
|
||||
[
|
||||
"S1",
|
||||
"S1A",
|
||||
"S1B",
|
||||
"S1C",
|
||||
"SENTINEL1",
|
||||
"SENTINEL1A",
|
||||
"SENTINEL1B",
|
||||
"SENTINEL1C",
|
||||
]
|
||||
),
|
||||
"S1",
|
||||
),
|
||||
else_=func.upper(alias.satellite),
|
||||
)
|
||||
return func.coalesce(func.nullif(func.upper(alias.satellite_family), ""), inferred_family)
|
||||
|
||||
|
||||
def _same_relative_orbit_expr(left_alias, right_alias):
|
||||
left_relative_orbit = func.upper(func.trim(func.coalesce(left_alias.relative_orbit, "")))
|
||||
right_relative_orbit = func.upper(func.trim(func.coalesce(right_alias.relative_orbit, "")))
|
||||
return and_(
|
||||
left_relative_orbit != "",
|
||||
right_relative_orbit != "",
|
||||
left_relative_orbit == right_relative_orbit,
|
||||
)
|
||||
|
||||
|
||||
class SpatialService:
|
||||
"""
|
||||
纯 PostGIS 空间计算服务
|
||||
@@ -155,6 +204,8 @@ class SpatialService:
|
||||
) -> List[dict]:
|
||||
master_alias = aliased(RadarDataORM)
|
||||
slave_alias = aliased(RadarDataORM)
|
||||
master_family_expr = _normalized_satellite_family_expr(master_alias)
|
||||
slave_family_expr = _normalized_satellite_family_expr(slave_alias)
|
||||
center_distance_expr = func.coalesce(
|
||||
PairingMetricCacheORM.scene_center_distance_meters,
|
||||
PairingMetricCacheORM.spatial_baseline_meters,
|
||||
@@ -192,6 +243,14 @@ class SpatialService:
|
||||
if params.require_same_polarization:
|
||||
stmt = stmt.where(PairingMetricCacheORM.same_polarization.is_(True))
|
||||
|
||||
stmt = stmt.where(
|
||||
or_(
|
||||
master_family_expr != "S1",
|
||||
slave_family_expr != "S1",
|
||||
_same_relative_orbit_expr(master_alias, slave_alias),
|
||||
)
|
||||
)
|
||||
|
||||
if params.allowed_satellites:
|
||||
allowed_satellites = [
|
||||
str(item).strip().upper()
|
||||
@@ -284,6 +343,7 @@ class SpatialService:
|
||||
slave.file_path,
|
||||
master.imaging_date,
|
||||
slave.imaging_date,
|
||||
master.satellite_family or slave.satellite_family or master.satellite or slave.satellite,
|
||||
),
|
||||
pair_uid=candidate.get("pair_uid"),
|
||||
metric_cache_ref_id=candidate.get("metric_cache_ref_id"),
|
||||
@@ -1281,7 +1341,7 @@ class SpatialService:
|
||||
compact = raw_satellite.replace("-", "").replace("_", "").replace(" ", "")
|
||||
if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}:
|
||||
return "LT1"
|
||||
if compact in {"S1", "S1A", "S1B", "SENTINEL1", "SENTINEL1A", "SENTINEL1B"}:
|
||||
if compact in {"S1", "S1A", "S1B", "S1C", "SENTINEL1", "SENTINEL1A", "SENTINEL1B", "SENTINEL1C"}:
|
||||
return "S1"
|
||||
return raw_satellite or "UNKNOWN"
|
||||
|
||||
|
||||
+39
-12
@@ -1,4 +1,6 @@
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Optional, Tuple, List, Callable, Dict, Any
|
||||
from lxml import etree
|
||||
|
||||
@@ -18,7 +20,7 @@ def _create_secure_xml_parser() -> etree.XMLParser:
|
||||
recover=False,
|
||||
)
|
||||
|
||||
# --- Sentinel-1 (S1A/S1B) Parsers ---
|
||||
# --- Sentinel-1 (S1A/S1B/S1C) Parsers ---
|
||||
|
||||
def _radar_meta_base() -> Dict[str, Any]:
|
||||
return {
|
||||
@@ -83,7 +85,7 @@ def normalize_satellite_family(value: Optional[str]) -> Optional[str]:
|
||||
compact = raw.replace("-", "").replace("_", "").replace(" ", "")
|
||||
if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}:
|
||||
return "LT1"
|
||||
if compact in {"S1", "S1A", "S1B", "SENTINEL1", "SENTINEL1A", "SENTINEL1B"}:
|
||||
if compact in {"S1", "S1A", "S1B", "S1C", "SENTINEL1", "SENTINEL1A", "SENTINEL1B", "SENTINEL1C"}:
|
||||
return "S1"
|
||||
if compact in {"GF3", "GAOFEN3"}:
|
||||
return "GF3"
|
||||
@@ -96,19 +98,44 @@ def parse_s1_radar_filename(folder_name: str) -> Optional[Dict[str, Any]]:
|
||||
Example: S1A_IW_SLC__1SDV_20250101T104105_...
|
||||
Returns a metadata dict.
|
||||
"""
|
||||
parts = folder_name.split('_')
|
||||
if len(parts) < 5 or not parts[0].startswith('S1'):
|
||||
name = os.path.basename(str(folder_name or "").strip())
|
||||
if name.lower().endswith(".zip"):
|
||||
name = name[:-4]
|
||||
if name.lower().endswith(".safe"):
|
||||
name = name[:-5]
|
||||
match = re.match(
|
||||
r"^(?P<satellite>S1[A-Z])_"
|
||||
r"(?P<mode>[A-Z0-9]+)_"
|
||||
r"(?P<product>[A-Z0-9]+)_+"
|
||||
r"(?P<class>[0-9A-Z]{4})_"
|
||||
r"(?P<start>\d{8}T\d{6}(?:\.\d+)?)_"
|
||||
r"(?P<stop>\d{8}T\d{6}(?:\.\d+)?)_"
|
||||
r"(?P<absolute_orbit>\d+)_"
|
||||
r"(?P<datatake>[0-9A-F]+)_"
|
||||
r"(?P<product_uid>[0-9A-F]+)$",
|
||||
name,
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
if not match:
|
||||
return None
|
||||
|
||||
meta = _radar_meta_base()
|
||||
meta["satellite"] = parts[0]
|
||||
meta["satellite_family"] = normalize_satellite_family(parts[0])
|
||||
meta["imaging_date"] = parts[4].split('T')[0]
|
||||
meta["imaging_mode"] = parts[1]
|
||||
meta["source_product_token"] = parts[2]
|
||||
meta["product_type"] = parts[2]
|
||||
polarization = parts[3] # e.g., '1SDV' -> 'DV' is dual-pol VV/VH
|
||||
meta["polarization"] = polarization[2:] if len(polarization) > 2 else polarization
|
||||
meta["satellite"] = match.group("satellite").upper()
|
||||
meta["satellite_family"] = normalize_satellite_family(meta["satellite"])
|
||||
meta["imaging_date"] = match.group("start")[:8]
|
||||
meta["imaging_mode"] = match.group("mode").upper()
|
||||
meta["source_product_token"] = match.group("class").upper()
|
||||
meta["product_type"] = match.group("product").upper()
|
||||
meta["product_level"] = "L1"
|
||||
polarization = match.group("class").upper() # e.g. 1SDV -> DV
|
||||
meta["polarization"] = polarization[-2:] if len(polarization) > 2 else polarization
|
||||
meta["orbit_circle"] = match.group("absolute_orbit").lstrip("0") or match.group("absolute_orbit")
|
||||
meta["product_unique_id"] = name
|
||||
try:
|
||||
start_time = datetime.strptime(match.group("start").split(".")[0], "%Y%m%dT%H%M%S")
|
||||
meta["acquisition_time_utc"] = start_time.isoformat()
|
||||
except ValueError:
|
||||
meta["acquisition_time_utc"] = match.group("start")
|
||||
return meta
|
||||
|
||||
|
||||
|
||||
@@ -81,7 +81,7 @@ SET
|
||||
('LT1', 'LT1A', 'LT1B', 'LUTAN1', 'LUTAN1A', 'LUTAN1B')
|
||||
THEN 'LT1'
|
||||
WHEN upper(replace(replace(replace(COALESCE(satellite, ''), '-', ''), '_', ''), ' ', '')) IN
|
||||
('S1', 'S1A', 'S1B', 'SENTINEL1', 'SENTINEL1A', 'SENTINEL1B')
|
||||
('S1', 'S1A', 'S1B', 'S1C', 'SENTINEL1', 'SENTINEL1A', 'SENTINEL1B', 'SENTINEL1C')
|
||||
THEN 'S1'
|
||||
WHEN NULLIF(satellite, '') IS NOT NULL
|
||||
THEN upper(satellite)
|
||||
|
||||
@@ -0,0 +1,371 @@
|
||||
-- Migration: Source product and orbit asset inventory
|
||||
-- Version: 10.0
|
||||
-- Date: 2026-05-12
|
||||
-- Purpose: Add first-class source product assets, orbit assets, scene-orbit bindings,
|
||||
-- inventory state, and radar_data compatibility fields.
|
||||
|
||||
ALTER TABLE IF EXISTS radar_data
|
||||
ADD COLUMN IF NOT EXISTS source_product_ref_id INTEGER NULL;
|
||||
|
||||
ALTER TABLE IF EXISTS radar_data
|
||||
ADD COLUMN IF NOT EXISTS source_archive_asset_id INTEGER NULL;
|
||||
|
||||
ALTER TABLE IF EXISTS radar_data
|
||||
ADD COLUMN IF NOT EXISTS selected_orbit_asset_id INTEGER NULL;
|
||||
|
||||
ALTER TABLE IF EXISTS radar_data
|
||||
ADD COLUMN IF NOT EXISTS orbit_binding_status VARCHAR(32) NOT NULL DEFAULT 'UNBOUND';
|
||||
|
||||
ALTER TABLE IF EXISTS radar_data
|
||||
ADD COLUMN IF NOT EXISTS orbit_binding_reason TEXT NULL;
|
||||
|
||||
ALTER TABLE IF EXISTS radar_data
|
||||
ADD COLUMN IF NOT EXISTS acquisition_start_time_utc TIMESTAMP NULL;
|
||||
|
||||
ALTER TABLE IF EXISTS radar_data
|
||||
ADD COLUMN IF NOT EXISTS acquisition_stop_time_utc TIMESTAMP NULL;
|
||||
|
||||
ALTER TABLE IF EXISTS radar_data
|
||||
ADD COLUMN IF NOT EXISTS absolute_orbit VARCHAR NULL;
|
||||
|
||||
ALTER TABLE IF EXISTS radar_data
|
||||
ADD COLUMN IF NOT EXISTS relative_orbit VARCHAR NULL;
|
||||
|
||||
ALTER TABLE IF EXISTS radar_data
|
||||
ADD COLUMN IF NOT EXISTS source_format VARCHAR(32) NULL;
|
||||
|
||||
ALTER TABLE IF EXISTS radar_data
|
||||
ADD COLUMN IF NOT EXISTS metadata_json JSON NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS source_product_assets (
|
||||
id SERIAL PRIMARY KEY,
|
||||
asset_uid VARCHAR(128) NOT NULL UNIQUE,
|
||||
logical_product_uid VARCHAR(128) NULL,
|
||||
satellite_family VARCHAR(32) NULL,
|
||||
satellite VARCHAR(32) NULL,
|
||||
source_format VARCHAR(32) NOT NULL,
|
||||
product_type VARCHAR(64) NULL,
|
||||
product_level VARCHAR(64) NULL,
|
||||
imaging_mode VARCHAR(64) NULL,
|
||||
polarization VARCHAR(64) NULL,
|
||||
absolute_orbit VARCHAR(64) NULL,
|
||||
relative_orbit VARCHAR(64) NULL,
|
||||
orbit_direction VARCHAR(32) NULL,
|
||||
acquisition_start_time_utc TIMESTAMP NULL,
|
||||
acquisition_stop_time_utc TIMESTAMP NULL,
|
||||
imaging_date VARCHAR(8) NULL,
|
||||
root_ref_id INTEGER NULL REFERENCES managed_roots(id) ON DELETE SET NULL,
|
||||
root_path VARCHAR NULL,
|
||||
file_path VARCHAR NOT NULL UNIQUE,
|
||||
archive_path VARCHAR NULL,
|
||||
path_kind VARCHAR(24) NULL,
|
||||
file_name VARCHAR(255) NULL,
|
||||
file_stem VARCHAR(255) NULL,
|
||||
file_ext VARCHAR(32) NULL,
|
||||
size_bytes BIGINT NULL,
|
||||
mtime_epoch DOUBLE PRECISION NULL,
|
||||
checksum_sha256 VARCHAR(64) NULL,
|
||||
checksum_status VARCHAR(32) NOT NULL DEFAULT 'NOT_COMPUTED',
|
||||
parser_name VARCHAR(64) NULL,
|
||||
parser_version VARCHAR(32) NULL,
|
||||
parse_status VARCHAR(32) NOT NULL DEFAULT 'PENDING',
|
||||
parse_error TEXT NULL,
|
||||
parsed_at TIMESTAMP NULL,
|
||||
metadata_json JSON NULL,
|
||||
is_active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
missing_since TIMESTAMP NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMP DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS orbit_assets (
|
||||
id SERIAL PRIMARY KEY,
|
||||
orbit_uid VARCHAR(128) NOT NULL UNIQUE,
|
||||
satellite_family VARCHAR(32) NULL,
|
||||
satellite VARCHAR(32) NULL,
|
||||
orbit_type VARCHAR(64) NOT NULL,
|
||||
native_format VARCHAR(32) NOT NULL,
|
||||
quality_class VARCHAR(32) NOT NULL DEFAULT 'unknown',
|
||||
root_ref_id INTEGER NULL REFERENCES managed_roots(id) ON DELETE SET NULL,
|
||||
root_path VARCHAR NULL,
|
||||
file_path VARCHAR NOT NULL UNIQUE,
|
||||
file_name VARCHAR(255) NULL,
|
||||
file_stem VARCHAR(255) NULL,
|
||||
file_ext VARCHAR(32) NULL,
|
||||
size_bytes BIGINT NULL,
|
||||
mtime_epoch DOUBLE PRECISION NULL,
|
||||
checksum_sha256 VARCHAR(64) NULL,
|
||||
checksum_status VARCHAR(32) NOT NULL DEFAULT 'NOT_COMPUTED',
|
||||
validity_start_time_utc TIMESTAMP NULL,
|
||||
validity_stop_time_utc TIMESTAMP NULL,
|
||||
generation_time_utc TIMESTAMP NULL,
|
||||
published_time_utc TIMESTAMP NULL,
|
||||
parser_name VARCHAR(64) NULL,
|
||||
parser_version VARCHAR(32) NULL,
|
||||
parse_status VARCHAR(32) NOT NULL DEFAULT 'PENDING',
|
||||
parse_error TEXT NULL,
|
||||
parsed_at TIMESTAMP NULL,
|
||||
metadata_json JSON NULL,
|
||||
is_active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
missing_since TIMESTAMP NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMP DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS scene_orbit_bindings (
|
||||
id SERIAL PRIMARY KEY,
|
||||
radar_data_id INTEGER NOT NULL REFERENCES radar_data(id) ON DELETE CASCADE,
|
||||
orbit_asset_id INTEGER NOT NULL REFERENCES orbit_assets(id) ON DELETE CASCADE,
|
||||
binding_role VARCHAR(32) NOT NULL DEFAULT 'primary_orbit',
|
||||
match_status VARCHAR(32) NOT NULL DEFAULT 'CANDIDATE',
|
||||
selection_status VARCHAR(32) NOT NULL DEFAULT 'CANDIDATE',
|
||||
selection_rank INTEGER NULL,
|
||||
priority_score DOUBLE PRECISION NULL,
|
||||
coverage_margin_before_seconds DOUBLE PRECISION NULL,
|
||||
coverage_margin_after_seconds DOUBLE PRECISION NULL,
|
||||
match_rule_version VARCHAR(64) NULL,
|
||||
match_reason TEXT NULL,
|
||||
selected_at TIMESTAMP NULL,
|
||||
metadata_json JSON NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMP DEFAULT NOW(),
|
||||
CONSTRAINT uq_scene_orbit_binding_role UNIQUE (radar_data_id, orbit_asset_id, binding_role)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS orbit_asset_derivatives (
|
||||
id SERIAL PRIMARY KEY,
|
||||
orbit_asset_id INTEGER NOT NULL REFERENCES orbit_assets(id) ON DELETE CASCADE,
|
||||
engine_code VARCHAR(32) NOT NULL,
|
||||
derivative_format VARCHAR(32) NOT NULL,
|
||||
derivative_role VARCHAR(64) NULL,
|
||||
pool_path VARCHAR NOT NULL,
|
||||
size_bytes BIGINT NULL,
|
||||
mtime_epoch DOUBLE PRECISION NULL,
|
||||
checksum_sha256 VARCHAR(64) NULL,
|
||||
generation_status VARCHAR(32) NOT NULL DEFAULT 'PENDING',
|
||||
generation_error TEXT NULL,
|
||||
generated_at TIMESTAMP NULL,
|
||||
metadata_json JSON NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMP DEFAULT NOW(),
|
||||
CONSTRAINT uq_orbit_asset_derivative_pool_path UNIQUE (orbit_asset_id, engine_code, derivative_format, pool_path)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS asset_inventory_states (
|
||||
id SERIAL PRIMARY KEY,
|
||||
root_ref_id INTEGER NOT NULL REFERENCES managed_roots(id) ON DELETE CASCADE,
|
||||
inventory_type VARCHAR(32) NOT NULL,
|
||||
root_path VARCHAR NOT NULL,
|
||||
scan_mode VARCHAR(32) NOT NULL DEFAULT 'file_pool',
|
||||
status VARCHAR(32) NOT NULL DEFAULT 'NEVER_SCANNED',
|
||||
last_scan_started_at TIMESTAMP NULL,
|
||||
last_scan_finished_at TIMESTAMP NULL,
|
||||
last_seen_entry_count INTEGER NULL,
|
||||
last_asset_count INTEGER NULL,
|
||||
last_issue_count INTEGER NULL,
|
||||
parser_version VARCHAR(32) NULL,
|
||||
needs_rescan BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
last_error TEXT NULL,
|
||||
metadata_json JSON NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMP DEFAULT NOW(),
|
||||
CONSTRAINT uq_asset_inventory_state_root_type UNIQUE (root_ref_id, inventory_type)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS asset_inventory_issues (
|
||||
id SERIAL PRIMARY KEY,
|
||||
root_ref_id INTEGER NULL REFERENCES managed_roots(id) ON DELETE SET NULL,
|
||||
inventory_type VARCHAR(32) NOT NULL,
|
||||
asset_ref_id INTEGER NULL REFERENCES source_product_assets(id) ON DELETE SET NULL,
|
||||
radar_data_id INTEGER NULL REFERENCES radar_data(id) ON DELETE SET NULL,
|
||||
orbit_asset_id INTEGER NULL REFERENCES orbit_assets(id) ON DELETE SET NULL,
|
||||
severity VARCHAR(16) NOT NULL DEFAULT 'warning',
|
||||
issue_code VARCHAR(64) NOT NULL,
|
||||
issue_message TEXT NULL,
|
||||
source_path VARCHAR NULL,
|
||||
status VARCHAR(16) NOT NULL DEFAULT 'OPEN',
|
||||
first_seen_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
last_seen_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
resolved_at TIMESTAMP NULL,
|
||||
metadata_json JSON NULL
|
||||
);
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_constraint WHERE conname = 'fk_radar_data_source_product_ref_id'
|
||||
) THEN
|
||||
ALTER TABLE radar_data
|
||||
ADD CONSTRAINT fk_radar_data_source_product_ref_id
|
||||
FOREIGN KEY (source_product_ref_id)
|
||||
REFERENCES source_product_assets(id)
|
||||
ON DELETE SET NULL;
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_constraint WHERE conname = 'fk_radar_data_source_archive_asset_id'
|
||||
) THEN
|
||||
ALTER TABLE radar_data
|
||||
ADD CONSTRAINT fk_radar_data_source_archive_asset_id
|
||||
FOREIGN KEY (source_archive_asset_id)
|
||||
REFERENCES source_product_assets(id)
|
||||
ON DELETE SET NULL;
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_constraint WHERE conname = 'fk_radar_data_selected_orbit_asset_id'
|
||||
) THEN
|
||||
ALTER TABLE radar_data
|
||||
ADD CONSTRAINT fk_radar_data_selected_orbit_asset_id
|
||||
FOREIGN KEY (selected_orbit_asset_id)
|
||||
REFERENCES orbit_assets(id)
|
||||
ON DELETE SET NULL;
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_radar_data_source_product_ref
|
||||
ON radar_data (source_product_ref_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_radar_data_source_archive_asset
|
||||
ON radar_data (source_archive_asset_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_radar_data_selected_orbit_asset
|
||||
ON radar_data (selected_orbit_asset_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_radar_data_orbit_binding_status
|
||||
ON radar_data (orbit_binding_status);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_radar_data_acquisition_start
|
||||
ON radar_data (acquisition_start_time_utc);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_radar_data_absolute_orbit
|
||||
ON radar_data (absolute_orbit);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_radar_data_relative_orbit
|
||||
ON radar_data (relative_orbit);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_radar_data_source_format
|
||||
ON radar_data (source_format);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_product_assets_asset_uid
|
||||
ON source_product_assets (asset_uid);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_product_assets_family_date
|
||||
ON source_product_assets (satellite_family, imaging_date);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_product_assets_satellite
|
||||
ON source_product_assets (satellite);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_product_assets_source_format
|
||||
ON source_product_assets (source_format);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_product_assets_parse_status
|
||||
ON source_product_assets (parse_status);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_product_assets_root_active
|
||||
ON source_product_assets (root_ref_id, is_active);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_product_assets_logical_product
|
||||
ON source_product_assets (logical_product_uid);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_product_assets_file_path
|
||||
ON source_product_assets (file_path);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_orbit_assets_orbit_uid
|
||||
ON orbit_assets (orbit_uid);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_orbit_assets_family_sat_window
|
||||
ON orbit_assets (satellite_family, satellite, validity_start_time_utc, validity_stop_time_utc);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_orbit_assets_orbit_type
|
||||
ON orbit_assets (orbit_type);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_orbit_assets_native_format
|
||||
ON orbit_assets (native_format);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_orbit_assets_quality_class
|
||||
ON orbit_assets (quality_class);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_orbit_assets_parse_status
|
||||
ON orbit_assets (parse_status);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_orbit_assets_root_active
|
||||
ON orbit_assets (root_ref_id, is_active);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_orbit_assets_file_path
|
||||
ON orbit_assets (file_path);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_scene_orbit_bindings_radar
|
||||
ON scene_orbit_bindings (radar_data_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_scene_orbit_bindings_orbit
|
||||
ON scene_orbit_bindings (orbit_asset_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_scene_orbit_bindings_match_status
|
||||
ON scene_orbit_bindings (match_status);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_scene_orbit_bindings_selection_status
|
||||
ON scene_orbit_bindings (selection_status);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_scene_orbit_bindings_scene_selected
|
||||
ON scene_orbit_bindings (radar_data_id, selection_status);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_orbit_asset_derivatives_asset_engine
|
||||
ON orbit_asset_derivatives (orbit_asset_id, engine_code);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_orbit_asset_derivatives_pool_path
|
||||
ON orbit_asset_derivatives (pool_path);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_orbit_asset_derivatives_generation_status
|
||||
ON orbit_asset_derivatives (generation_status);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_asset_inventory_states_type_status
|
||||
ON asset_inventory_states (inventory_type, status);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_asset_inventory_states_needs_rescan
|
||||
ON asset_inventory_states (needs_rescan);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_asset_inventory_issues_open
|
||||
ON asset_inventory_issues (status, severity);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_asset_inventory_issues_root_type
|
||||
ON asset_inventory_issues (root_ref_id, inventory_type);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_asset_inventory_issues_asset
|
||||
ON asset_inventory_issues (asset_ref_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_asset_inventory_issues_orbit
|
||||
ON asset_inventory_issues (orbit_asset_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_asset_inventory_issues_radar
|
||||
ON asset_inventory_issues (radar_data_id);
|
||||
|
||||
UPDATE radar_data
|
||||
SET
|
||||
orbit_binding_status = CASE
|
||||
WHEN has_orbit_data IS TRUE AND NULLIF(orbit_file_path, '') IS NOT NULL THEN 'MATCHED'
|
||||
WHEN has_orbit_data IS TRUE THEN 'MATCHED'
|
||||
ELSE COALESCE(NULLIF(orbit_binding_status, ''), 'UNBOUND')
|
||||
END
|
||||
WHERE orbit_binding_status IS NULL
|
||||
OR orbit_binding_status = ''
|
||||
OR (has_orbit_data IS TRUE AND orbit_binding_status = 'UNBOUND');
|
||||
|
||||
UPDATE radar_data
|
||||
SET absolute_orbit = COALESCE(NULLIF(absolute_orbit, ''), NULLIF(orbit_circle, ''))
|
||||
WHERE absolute_orbit IS NULL OR absolute_orbit = '';
|
||||
|
||||
UPDATE radar_data
|
||||
SET source_format = COALESCE(
|
||||
NULLIF(source_format, ''),
|
||||
CASE
|
||||
WHEN upper(replace(replace(replace(COALESCE(satellite, ''), '-', ''), '_', ''), ' ', '')) LIKE 'S1%%'
|
||||
AND lower(COALESCE(file_path, '')) LIKE '%%.zip' THEN 'S1_ZIP'
|
||||
WHEN upper(replace(replace(replace(COALESCE(satellite, ''), '-', ''), '_', ''), ' ', '')) LIKE 'S1%%'
|
||||
AND lower(COALESCE(file_path, '')) LIKE '%%.safe' THEN 'S1_SAFE_DIR'
|
||||
WHEN upper(replace(replace(replace(COALESCE(satellite, ''), '-', ''), '_', ''), ' ', '')) LIKE 'LT1%%' THEN 'LT1_DIR'
|
||||
WHEN upper(replace(replace(replace(COALESCE(satellite, ''), '-', ''), '_', ''), ' ', '')) LIKE 'GF3%%' THEN 'GF3_DIR'
|
||||
ELSE NULL
|
||||
END
|
||||
)
|
||||
WHERE source_format IS NULL OR source_format = '';
|
||||
+3
-1
@@ -60,6 +60,8 @@
|
||||
|
||||
## 4. 配对与前端导航
|
||||
|
||||
- [SENTINEL1_SOURCE_ORBIT_ASSET_DESIGN_20260512.md](SENTINEL1_SOURCE_ORBIT_ASSET_DESIGN_20260512.md)
|
||||
Sentinel-1 / LT-1 源数据与精密轨道资产层设计,定义统一源产品库存、轨道资产、scene 绑定、启动自维护和健康检查边界。
|
||||
- [SENTINEL1_SYSTEM_ENHANCEMENT_MASTER_PLAN_20260510.md](SENTINEL1_SYSTEM_ENHANCEMENT_MASTER_PLAN_20260510.md)
|
||||
Sentinel-1 系统增强主维护文档,汇总数据管理、精轨、配对、任务分发、Gamma/PyINT 生产、结果管理和分阶段实施边界。
|
||||
- [SENTINEL1_DATA_MANAGEMENT_ADAPTATION_PLAN_20260510.md](SENTINEL1_DATA_MANAGEMENT_ADAPTATION_PLAN_20260510.md)
|
||||
@@ -125,4 +127,4 @@
|
||||
- 已归档的 `项目汇报.md`
|
||||
- 已归档的各类 `*_EXPERIMENT_*` / `*_PROGRESS_*` / `*_TODO_*`
|
||||
|
||||
最后更新:2026-04-26
|
||||
最后更新:2026-05-12
|
||||
|
||||
@@ -0,0 +1,579 @@
|
||||
# Sentinel-1 / LT-1 源数据与精密轨道资产设计
|
||||
|
||||
日期:2026-05-12
|
||||
|
||||
本文定义源数据管理和精密轨道管理的下一阶段底座设计。目标是把 Sentinel-1 和 LT-1 放到同等位置和能力上,而不是把 Sentinel-1 临时塞进现有 LT-1 扫描和轨道字段里。
|
||||
|
||||
本文只覆盖源数据、精密轨道、资产库存、轨道绑定、启动自维护和健康检查。D-InSAR 配对、任务分发和生产 profile 不在本阶段实现范围内,但后续应消费这里建立的资产与绑定结果。
|
||||
|
||||
## 1. 目标
|
||||
|
||||
本阶段要完成:
|
||||
|
||||
1. 统一管理 LT-1、Sentinel-1 的源产品资产。
|
||||
2. 统一管理 LT-1 原生精轨和 Sentinel-1 EOF 精轨。
|
||||
3. 让 `radar_data` 成为统一 scene 业务入口,而不是唯一资产库存表。
|
||||
4. 建立 scene 与精轨的候选、选中和异常状态。
|
||||
5. 把新 schema 纳入启动自维护、数据库 schema 检查和健康检查。
|
||||
6. 保持现有 LT-1、GF-3、水体、配对、结果 catalog 的兼容运行。
|
||||
|
||||
本阶段暂不做:
|
||||
|
||||
1. 不新增 Sentinel-1 D-InSAR 生产链路。
|
||||
2. 不改造现有 `lt1_gamma_dinsar` 或 `lt1_stripmap` 为多卫星 profile。
|
||||
3. 不在启动时扫描大目录或解析大型 ZIP。
|
||||
4. 不把 Sentinel-1 EOF 送入 LT-1 txt 到 XML 转换链路。
|
||||
|
||||
## 2. 现有约束
|
||||
|
||||
当前系统启动链路中,`backend/app/main.py` 的 lifespan 会先调用 `ensure_database_ready()`,再初始化数据库会话、同步 managed roots、同步 manifest catalog、启动 pairing cache 状态。因此新资产层必须满足:
|
||||
|
||||
- ORM 是 schema 事实来源。新增表和字段必须进入 ORM。
|
||||
- SQL migration 是约束、索引、幂等修补的事实来源。新增 migration 必须加入 `backend/app/db_maintenance.py` 的 `MIGRATION_FILES`。
|
||||
- migration 必须可重复运行,使用 `CREATE TABLE IF NOT EXISTS`、`ALTER TABLE ... ADD COLUMN IF NOT EXISTS`、`CREATE INDEX IF NOT EXISTS`。
|
||||
- 启动自维护只保证 schema 和轻量状态存在,不做 ZIP 解包、不扫大目录、不重建全量绑定。
|
||||
- 健康检查必须能报告资产层状态,不能只看旧的 `MONITOR_RADAR_DIRS` 和 `MONITOR_ORBIT_DIR` 是否存在。
|
||||
|
||||
现有兼容字段需要保留:
|
||||
|
||||
- `radar_data.has_orbit_data`
|
||||
- `radar_data.orbit_file_path`
|
||||
- `radar_data.file_path`
|
||||
- `radar_data.coverage_polygon`
|
||||
- `radar_data.geom`
|
||||
|
||||
这些字段继续服务旧接口和旧前端,但由新的资产与绑定结果回填。
|
||||
|
||||
## 3. 领域模型
|
||||
|
||||
新的数据模型分成五类。
|
||||
|
||||
### 3.1 源产品资产
|
||||
|
||||
`source_product_assets` 记录物理源产品。一个资产是一份实际存在的文件或目录,例如:
|
||||
|
||||
- LT-1 scene 目录
|
||||
- LT-1 原始压缩包或 tiff
|
||||
- Sentinel-1 `.zip`
|
||||
- Sentinel-1 `.SAFE` 目录
|
||||
- GF-3 现有处理输入或输出目录,后续可接入同一库存
|
||||
|
||||
关键字段建议:
|
||||
|
||||
```text
|
||||
id
|
||||
asset_uid
|
||||
logical_product_uid
|
||||
satellite_family
|
||||
satellite
|
||||
source_format
|
||||
product_type
|
||||
product_level
|
||||
imaging_mode
|
||||
polarization
|
||||
absolute_orbit
|
||||
relative_orbit
|
||||
orbit_direction
|
||||
acquisition_start_time_utc
|
||||
acquisition_stop_time_utc
|
||||
imaging_date
|
||||
root_path
|
||||
file_path
|
||||
archive_path
|
||||
path_kind
|
||||
file_name
|
||||
file_stem
|
||||
file_ext
|
||||
size_bytes
|
||||
mtime_epoch
|
||||
checksum_sha256
|
||||
checksum_status
|
||||
parser_name
|
||||
parser_version
|
||||
parse_status
|
||||
parse_error
|
||||
parsed_at
|
||||
metadata_json
|
||||
is_active
|
||||
missing_since
|
||||
created_at
|
||||
updated_at
|
||||
```
|
||||
|
||||
字段语义:
|
||||
|
||||
- `asset_uid` 是物理资产唯一键,优先由规范化绝对路径和大小/mtime 派生。
|
||||
- `logical_product_uid` 是同一个遥感产品的逻辑身份,用于把 Sentinel-1 ZIP 和解包后的 SAFE 关联起来。
|
||||
- `source_format` 使用稳定枚举,例如 `LT1_DIR`、`LT1_ARCHIVE`、`S1_ZIP`、`S1_SAFE_DIR`、`GF3_DIR`。
|
||||
- `archive_path` 用于记录生产更偏好的原始归档路径,例如 Sentinel-1 ZIP。
|
||||
- `metadata_json` 保存传感器专有字段,不把所有 Sentinel-1 annotation 字段拆成列。
|
||||
|
||||
### 3.2 Scene 业务入口
|
||||
|
||||
`radar_data` 仍然是前端检索、地图展示、水体、配对和后续生产的 scene 入口。它需要补充资产引用和 Sentinel-1/LT-1 通用字段:
|
||||
|
||||
```text
|
||||
source_product_ref_id
|
||||
source_archive_asset_id
|
||||
selected_orbit_asset_id
|
||||
orbit_binding_status
|
||||
orbit_binding_reason
|
||||
acquisition_start_time_utc
|
||||
acquisition_stop_time_utc
|
||||
absolute_orbit
|
||||
relative_orbit
|
||||
source_format
|
||||
metadata_json
|
||||
```
|
||||
|
||||
字段语义:
|
||||
|
||||
- `source_product_ref_id` 指向用于生成该 scene 元数据的主资产。
|
||||
- `source_archive_asset_id` 指向后续分发或生产优先使用的原始资产。Sentinel-1 通常是 ZIP。
|
||||
- `selected_orbit_asset_id` 指向当前选中的精轨资产。
|
||||
- `orbit_binding_status` 标识 `UNBOUND`、`MATCHED`、`MISSING`、`AMBIGUOUS`、`ERROR`。
|
||||
- 旧字段 `has_orbit_data` 和 `orbit_file_path` 从 `selected_orbit_asset_id` 兼容回填。
|
||||
|
||||
### 3.3 精密轨道资产
|
||||
|
||||
`orbit_assets` 记录原生轨道文件。它不记录派生到 ENVI/ISCE2 池里的文件。
|
||||
|
||||
关键字段建议:
|
||||
|
||||
```text
|
||||
id
|
||||
orbit_uid
|
||||
satellite_family
|
||||
satellite
|
||||
orbit_type
|
||||
native_format
|
||||
quality_class
|
||||
root_path
|
||||
file_path
|
||||
file_name
|
||||
file_stem
|
||||
file_ext
|
||||
size_bytes
|
||||
mtime_epoch
|
||||
checksum_sha256
|
||||
checksum_status
|
||||
validity_start_time_utc
|
||||
validity_stop_time_utc
|
||||
generation_time_utc
|
||||
published_time_utc
|
||||
parser_name
|
||||
parser_version
|
||||
parse_status
|
||||
parse_error
|
||||
parsed_at
|
||||
metadata_json
|
||||
is_active
|
||||
missing_since
|
||||
created_at
|
||||
updated_at
|
||||
```
|
||||
|
||||
枚举建议:
|
||||
|
||||
- `satellite_family`: `LT1`、`S1`、`GF3`
|
||||
- `native_format`: `LT1_TXT`、`S1_EOF`
|
||||
- `orbit_type`: `LT1_GPS`、`AUX_POEORB`、`AUX_RESORB`
|
||||
- `quality_class`: `precise`、`restituted`、`predicted`、`unknown`
|
||||
|
||||
Sentinel-1 EOF 的匹配逻辑基于:
|
||||
|
||||
```text
|
||||
orbit.satellite == scene.satellite
|
||||
and orbit.validity_start_time_utc <= scene.acquisition_time <= orbit.validity_stop_time_utc
|
||||
```
|
||||
|
||||
LT-1 第一阶段可继续复用现有日期匹配逻辑,后续如果轨道文件能提供明确有效期,也应收敛到相同的时间窗模型。
|
||||
|
||||
### 3.4 Scene 与轨道绑定
|
||||
|
||||
`scene_orbit_bindings` 记录每个 scene 的候选轨道和最终选择。
|
||||
|
||||
关键字段建议:
|
||||
|
||||
```text
|
||||
id
|
||||
radar_data_id
|
||||
orbit_asset_id
|
||||
binding_role
|
||||
match_status
|
||||
selection_status
|
||||
selection_rank
|
||||
priority_score
|
||||
coverage_margin_before_seconds
|
||||
coverage_margin_after_seconds
|
||||
match_rule_version
|
||||
match_reason
|
||||
selected_at
|
||||
metadata_json
|
||||
created_at
|
||||
updated_at
|
||||
```
|
||||
|
||||
字段语义:
|
||||
|
||||
- `binding_role` 第一阶段使用 `primary_orbit`。
|
||||
- `match_status`: `CANDIDATE`、`MATCHED`、`REJECTED`、`STALE`、`ERROR`。
|
||||
- `selection_status`: `SELECTED`、`CANDIDATE`、`NOT_SELECTED`。
|
||||
- Sentinel-1 选择优先级:POEORB 优先于 RESORB;覆盖余量更大优先;generation time 更新优先。
|
||||
- LT-1 选择优先级:现有可用轨道优先;坏源精轨不得入选;派生池同步失败时保留资产但标注异常。
|
||||
|
||||
### 3.5 轨道派生产物
|
||||
|
||||
`orbit_asset_derivatives` 记录从原生轨道资产生成或同步到引擎池的文件。
|
||||
|
||||
关键字段建议:
|
||||
|
||||
```text
|
||||
id
|
||||
orbit_asset_id
|
||||
engine_code
|
||||
derivative_format
|
||||
derivative_role
|
||||
pool_path
|
||||
size_bytes
|
||||
mtime_epoch
|
||||
checksum_sha256
|
||||
generation_status
|
||||
generation_error
|
||||
generated_at
|
||||
metadata_json
|
||||
created_at
|
||||
updated_at
|
||||
```
|
||||
|
||||
用途:
|
||||
|
||||
- LT-1 txt 同步到 ENVI pool。
|
||||
- LT-1 txt 转换成 ISCE2 XML。
|
||||
- Sentinel-1 EOF 第一阶段通常不转换,但后续可以记录 staging 到 OPOD 目录的结果。
|
||||
|
||||
### 3.6 库存状态与问题
|
||||
|
||||
为避免健康检查每次扫大目录,新增轻量状态表:
|
||||
|
||||
`asset_inventory_states`
|
||||
|
||||
```text
|
||||
id
|
||||
root_ref_id
|
||||
inventory_type
|
||||
root_path
|
||||
scan_mode
|
||||
status
|
||||
last_scan_started_at
|
||||
last_scan_finished_at
|
||||
last_seen_entry_count
|
||||
last_asset_count
|
||||
last_issue_count
|
||||
parser_version
|
||||
needs_rescan
|
||||
last_error
|
||||
metadata_json
|
||||
created_at
|
||||
updated_at
|
||||
```
|
||||
|
||||
`asset_inventory_issues`
|
||||
|
||||
```text
|
||||
id
|
||||
root_ref_id
|
||||
inventory_type
|
||||
asset_ref_id
|
||||
radar_data_id
|
||||
orbit_asset_id
|
||||
severity
|
||||
issue_code
|
||||
issue_message
|
||||
source_path
|
||||
status
|
||||
first_seen_at
|
||||
last_seen_at
|
||||
resolved_at
|
||||
metadata_json
|
||||
```
|
||||
|
||||
典型 issue:
|
||||
|
||||
- `source_path_missing`
|
||||
- `source_parse_failed`
|
||||
- `orbit_parse_failed`
|
||||
- `duplicate_logical_product`
|
||||
- `scene_missing_orbit`
|
||||
- `scene_ambiguous_orbit`
|
||||
- `selected_orbit_missing_file`
|
||||
- `lt1_derivative_generation_failed`
|
||||
|
||||
## 4. Root 与配置
|
||||
|
||||
建议新增通用配置,同时保留旧配置作为兼容入口:
|
||||
|
||||
```text
|
||||
SOURCE_PRODUCT_DIRS=
|
||||
ORBIT_SOURCE_DIRS=
|
||||
```
|
||||
|
||||
兼容关系:
|
||||
|
||||
- `INSAR_STORAGE_DIRS` 和 `MONITOR_RADAR_DIRS` 继续生效,并作为 `SOURCE_PRODUCT_DIRS` 的兼容来源。
|
||||
- `MONITOR_ORBIT_DIR` 继续生效,并作为 `ORBIT_SOURCE_DIRS` 的兼容来源。
|
||||
- `ORBIT_POOL_ENVI`、`ORBIT_POOL_ISCE2` 仍然是派生轨道池,不作为原生轨道资产源。
|
||||
|
||||
`root_registry_service` 需要增加或调整 root role:
|
||||
|
||||
- `source_product_pool`
|
||||
- `legacy_scan_root_radar`
|
||||
- `orbit_asset_pool`
|
||||
- `orbit_pool_envi`
|
||||
- `orbit_pool_isce2`
|
||||
|
||||
每个 root 的 `metadata_json` 可以记录:
|
||||
|
||||
```json
|
||||
{
|
||||
"supported_families": ["LT1", "S1"],
|
||||
"discovery_patterns": ["LT1*", "S1*.zip", "*.SAFE", "*.EOF"],
|
||||
"imported_from": "settings"
|
||||
}
|
||||
```
|
||||
|
||||
对用户当前样本,推荐后续配置形态是:
|
||||
|
||||
```text
|
||||
SOURCE_PRODUCT_DIRS=D:\LuTan1_Image_Pool;D:\Sentinel1_Image_Pool_ZIP
|
||||
ORBIT_SOURCE_DIRS=D:\LT1_data_lsarorbit;D:\Sentinel1_EOF_Pool
|
||||
```
|
||||
|
||||
## 5. Parser 设计
|
||||
|
||||
新增解析器按资产类型组织:
|
||||
|
||||
- `LT1SourceParser`
|
||||
- `Sentinel1ZipParser`
|
||||
- `Sentinel1SafeParser`
|
||||
- `GF3SourceParser`
|
||||
- `LT1OrbitParser`
|
||||
- `Sentinel1EofParser`
|
||||
|
||||
Sentinel-1 ZIP 解析原则:
|
||||
|
||||
- 不全量解压。
|
||||
- 使用 Python `zipfile` 读取 `manifest.safe` 和 annotation XML。
|
||||
- 只读取必要 XML 文件和 quicklook 候选,不读取 measurement 大文件。
|
||||
- 文件名提供粗字段,XML 提供 relative orbit、升降轨、footprint、swath、极化细节。
|
||||
- `S1A`、`S1B`、`S1C` 都归一化为 `satellite_family = S1`。
|
||||
|
||||
Sentinel-1 EOF 解析原则:
|
||||
|
||||
- 优先从文件名解析 mission、orbit type、generation time、validity start、validity stop。
|
||||
- 文件名不可靠时再读取 EOF XML 内容。
|
||||
- `AUX_POEORB` 标为 precise,`AUX_RESORB` 标为 restituted。
|
||||
|
||||
LT-1 解析原则:
|
||||
|
||||
- 现有文件名和 XML 解析逻辑保留。
|
||||
- 现有坏源精轨检测和 quarantine 逻辑保留,但结果写入 `orbit_assets` 和 `asset_inventory_issues`。
|
||||
- 现有 `sync_orbit_pools()` 逐步下沉为 `orbit_asset_derivatives` 的生成步骤。
|
||||
|
||||
## 6. 扫描与绑定流程
|
||||
|
||||
### 6.1 源产品扫描
|
||||
|
||||
流程:
|
||||
|
||||
1. 从 managed roots 找到 source product roots。
|
||||
2. 发现候选文件或目录。
|
||||
3. upsert `source_product_assets`。
|
||||
4. 调用对应 parser。
|
||||
5. 生成或更新 `radar_data`。
|
||||
6. 更新 `asset_inventory_states`。
|
||||
7. 记录解析失败或重复产品到 `asset_inventory_issues`。
|
||||
|
||||
`radar_data.unique_id` 的生成应优先使用稳定逻辑身份:
|
||||
|
||||
```text
|
||||
satellite_family + satellite + product_unique_id
|
||||
```
|
||||
|
||||
如果缺少产品唯一号,再回退到规范化路径。
|
||||
|
||||
### 6.2 轨道扫描
|
||||
|
||||
流程:
|
||||
|
||||
1. 从 managed roots 找到 orbit asset roots。
|
||||
2. 发现 LT-1 txt 和 Sentinel-1 EOF。
|
||||
3. upsert `orbit_assets`。
|
||||
4. 调用轨道 parser。
|
||||
5. LT-1 轨道执行坏源检测。
|
||||
6. 记录 parse status 和 inventory issues。
|
||||
|
||||
### 6.3 轨道绑定
|
||||
|
||||
流程:
|
||||
|
||||
1. 找出新增或变更的 scene 与 orbit asset。
|
||||
2. 按卫星族调用绑定规则。
|
||||
3. 写入候选 `scene_orbit_bindings`。
|
||||
4. 选出 `selection_status = SELECTED` 的绑定。
|
||||
5. 回填 `radar_data.selected_orbit_asset_id`、`orbit_binding_status`、`has_orbit_data`、`orbit_file_path`。
|
||||
6. 对无轨道或多候选未决的 scene 写入 issue。
|
||||
|
||||
绑定规则版本需要显式记录,例如:
|
||||
|
||||
```text
|
||||
lt1_orbit_binding.v1
|
||||
s1_eof_window_binding.v1
|
||||
```
|
||||
|
||||
## 7. 启动自维护
|
||||
|
||||
新增资产层后,启动自维护必须按以下方式运行:
|
||||
|
||||
1. `ensure_database_ready()` 创建 PostGIS 扩展。
|
||||
2. ORM `create_all()` 创建新增表和新增列。
|
||||
3. migration `010_source_orbit_asset_inventory.sql` 补齐索引、约束、兼容列、视图或必要函数。
|
||||
4. `root_registry_service.sync_from_settings()` 同步新旧 root。
|
||||
5. 启动阶段只初始化 `asset_inventory_states` 的空状态,不做真实扫描。
|
||||
6. 健康检查报告 `needs_rescan = true`,由用户手动触发扫描任务。
|
||||
|
||||
注意事项:
|
||||
|
||||
- 不允许启动时遍历 `D:\Sentinel1_Image_Pool_ZIP` 这类大目录并打开 ZIP。
|
||||
- 不允许启动时自动删除资产记录。文件消失时标记 `is_active = false` 和 `missing_since`。
|
||||
- schema mismatch 必须在启动日志中可见,健康检查必须能展示缺表、缺列和资产库存异常。
|
||||
|
||||
## 8. 健康检查
|
||||
|
||||
`health_service` 增加 `asset_inventory` 检查项。
|
||||
|
||||
建议返回结构:
|
||||
|
||||
```json
|
||||
{
|
||||
"ok": true,
|
||||
"source_roots": {
|
||||
"configured_count": 2,
|
||||
"accessible_count": 2,
|
||||
"needs_rescan_count": 1
|
||||
},
|
||||
"orbit_roots": {
|
||||
"configured_count": 2,
|
||||
"accessible_count": 2,
|
||||
"needs_rescan_count": 1
|
||||
},
|
||||
"source_assets": {
|
||||
"total_count": 0,
|
||||
"lt1_count": 0,
|
||||
"s1_count": 0,
|
||||
"parse_failed_count": 0
|
||||
},
|
||||
"orbit_assets": {
|
||||
"total_count": 0,
|
||||
"lt1_count": 0,
|
||||
"s1_count": 0,
|
||||
"parse_failed_count": 0
|
||||
},
|
||||
"bindings": {
|
||||
"scene_count": 0,
|
||||
"matched_count": 0,
|
||||
"missing_count": 0,
|
||||
"ambiguous_count": 0
|
||||
},
|
||||
"issues": {
|
||||
"open_count": 0,
|
||||
"error_count": 0,
|
||||
"warning_count": 0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
整体健康状态建议:
|
||||
|
||||
- schema 不完整:`ok = false`
|
||||
- root 不可访问:`ok = false`
|
||||
- 有 error 级 issue:`ok = false`
|
||||
- 仅 `needs_rescan` 或 warning issue:`ok = true`,但显示 degraded/warning
|
||||
|
||||
## 9. API 与前端边界
|
||||
|
||||
本阶段后端需要提供:
|
||||
|
||||
- 源产品资产列表、详情、重扫。
|
||||
- 轨道资产列表、详情、重扫。
|
||||
- scene 的轨道绑定详情。
|
||||
- 资产库存健康摘要。
|
||||
- 手动触发源产品扫描、轨道扫描、轨道绑定重建。
|
||||
|
||||
前端数据管理页面需要显示:
|
||||
|
||||
- satellite family: LT1、S1、GF3
|
||||
- source format: LT1_DIR、S1_ZIP、S1_SAFE_DIR
|
||||
- relative orbit、absolute orbit
|
||||
- acquisition start/stop
|
||||
- orbit binding status
|
||||
- selected orbit asset
|
||||
- parser status 和 issue 摘要
|
||||
|
||||
轨道管理页面需要显示:
|
||||
|
||||
- 原生轨道资产列表。
|
||||
- LT-1 与 Sentinel-1 的轨道类型。
|
||||
- validity window。
|
||||
- 派生产物状态。
|
||||
- 绑定到多少 scene。
|
||||
- 解析失败或缺文件问题。
|
||||
|
||||
## 10. 兼容与迁移策略
|
||||
|
||||
迁移后,现有业务继续读 `radar_data`。新资产层提供更完整的 provenance。
|
||||
|
||||
兼容规则:
|
||||
|
||||
- 旧扫描入口可以保留,但应逐步改为调用资产扫描服务。
|
||||
- `radar_data.file_path` 继续保存主 scene 路径。
|
||||
- `radar_data.orbit_file_path` 继续保存选中原生轨道路径。
|
||||
- `radar_data.has_orbit_data` 继续表示是否存在选中轨道。
|
||||
- 现有 pairing cache 可以暂时继续使用旧字段,后续再改为读取 `relative_orbit` 和资产绑定状态。
|
||||
|
||||
数据回填:
|
||||
|
||||
1. 对已有 `radar_data` 记录创建 `source_product_assets`。
|
||||
2. 对已有 `orbit_file_path` 创建 `orbit_assets`。
|
||||
3. 生成 `scene_orbit_bindings`。
|
||||
4. 确认兼容字段和新绑定结果一致。
|
||||
|
||||
## 11. 验收标准
|
||||
|
||||
以当前样本池作为第一轮验收:
|
||||
|
||||
- `D:\Sentinel1_Image_Pool_ZIP` 中 29 个 Sentinel-1 ZIP 能登记为 `source_product_assets`。
|
||||
- S1A 与 S1C 都归入 `satellite_family = S1`。
|
||||
- 每个 ZIP 能生成或更新一条 `radar_data` scene。
|
||||
- `D:\Sentinel1_EOF_Pool` 中 24 个 EOF 能登记为 `orbit_assets`。
|
||||
- 29 个 Sentinel-1 scene 全部能通过 validity window 绑定到 EOF。
|
||||
- `has_orbit_data = true` 和 `orbit_file_path` 能兼容回填。
|
||||
- LT-1 现有源数据扫描和精轨同步不退化。
|
||||
- 启动后数据库健康检查不出现 schema mismatch。
|
||||
- 健康检查能展示源产品、轨道资产、绑定状态和 issue 统计。
|
||||
|
||||
## 12. 实施顺序
|
||||
|
||||
建议按以下顺序开工:
|
||||
|
||||
1. Schema 与 ORM:新增资产表、绑定表、状态表,扩展 `radar_data`。
|
||||
2. Migration 与启动自维护:新增 `010_source_orbit_asset_inventory.sql`,加入 `MIGRATION_FILES`。
|
||||
3. 健康检查:新增 `asset_inventory` 状态,不依赖真实扫描也能返回稳定结构。
|
||||
4. Root registry:支持通用 `SOURCE_PRODUCT_DIRS`、`ORBIT_SOURCE_DIRS`,兼容旧配置。
|
||||
5. Parser 与库存扫描:先实现 Sentinel-1 ZIP、Sentinel-1 EOF、LT-1 现有源数据和轨道。
|
||||
6. 轨道绑定服务:实现 LT-1 日期绑定和 Sentinel-1 EOF 时间窗绑定。
|
||||
7. 兼容回填:更新 `radar_data.has_orbit_data`、`orbit_file_path`。
|
||||
8. API 与前端:展示资产、轨道、绑定和 issue。
|
||||
9. 回归测试:用 Sentinel-1 样本池和现有 LT-1 数据池验证。
|
||||
|
||||
这一路径先把底座做稳,再接 D-InSAR 配对和 `s1_gamma_dinsar` 生产 profile。
|
||||
Generated
+793
-501
File diff suppressed because it is too large
Load Diff
@@ -11,7 +11,6 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"axios": "^1.13.0",
|
||||
"zustand": "^5.0.5",
|
||||
"chart.js": "^4.5.0",
|
||||
"chartjs-adapter-date-fns": "^3.0.0",
|
||||
"date-fns": "^4.1.0",
|
||||
@@ -21,7 +20,8 @@
|
||||
"react": "^19.1.1",
|
||||
"react-chartjs-2": "^5.3.0",
|
||||
"react-dom": "^19.1.1",
|
||||
"react-markdown": "^10.1.0"
|
||||
"react-markdown": "^10.1.0",
|
||||
"zustand": "^5.0.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.36.0",
|
||||
@@ -33,9 +33,6 @@
|
||||
"eslint-plugin-react-hooks": "^5.2.0",
|
||||
"eslint-plugin-react-refresh": "^0.4.20",
|
||||
"globals": "^16.4.0",
|
||||
"vite": "npm:rolldown-vite@7.1.12"
|
||||
},
|
||||
"overrides": {
|
||||
"vite": "npm:rolldown-vite@7.1.12"
|
||||
"vite": "^7.1.12"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2400,6 +2400,237 @@ input[type="checkbox"] {
|
||||
margin-top: 6px;
|
||||
}
|
||||
|
||||
.asset-inventory-panel {
|
||||
padding: 12px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 12px;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.asset-toolbar {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
gap: 10px;
|
||||
align-items: flex-start;
|
||||
}
|
||||
|
||||
.asset-toolbar h3 {
|
||||
margin: 0;
|
||||
font-size: 16px;
|
||||
}
|
||||
|
||||
.asset-toolbar p {
|
||||
margin: 4px 0 0;
|
||||
color: var(--color-text-muted);
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.asset-actions {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
justify-content: flex-end;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.asset-actions select,
|
||||
.asset-actions button {
|
||||
height: 30px;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.asset-message {
|
||||
border: 1px solid var(--color-border);
|
||||
background: var(--color-accent-soft);
|
||||
color: var(--color-accent-strong);
|
||||
border-radius: 6px;
|
||||
padding: 8px 10px;
|
||||
font-size: 12px;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.asset-message--error {
|
||||
background: #fee2e2;
|
||||
color: #991b1b;
|
||||
border-color: #fecaca;
|
||||
}
|
||||
|
||||
.asset-metrics {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.asset-metric {
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: 8px;
|
||||
padding: 8px;
|
||||
background: #f8fafc;
|
||||
}
|
||||
|
||||
.asset-metric span,
|
||||
.asset-metric small {
|
||||
display: block;
|
||||
color: var(--color-text-muted);
|
||||
font-size: 11px;
|
||||
}
|
||||
|
||||
.asset-metric strong {
|
||||
display: block;
|
||||
margin: 2px 0;
|
||||
font-size: 20px;
|
||||
color: var(--color-text-primary);
|
||||
}
|
||||
|
||||
.asset-root-strip {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.asset-root-item {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) auto;
|
||||
gap: 8px;
|
||||
align-items: center;
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: 6px;
|
||||
padding: 7px 8px;
|
||||
background: #fff;
|
||||
}
|
||||
|
||||
.asset-root-item strong,
|
||||
.asset-root-item span {
|
||||
display: block;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.asset-root-item span {
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
color: var(--color-text-muted);
|
||||
font-size: 11px;
|
||||
}
|
||||
|
||||
.asset-tabbar {
|
||||
display: flex;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.asset-tabbar button {
|
||||
flex: 1;
|
||||
font-size: 12px;
|
||||
padding: 7px 6px;
|
||||
}
|
||||
|
||||
.asset-tabbar button.active-tab {
|
||||
background: var(--color-accent);
|
||||
color: #fff;
|
||||
border-color: var(--color-accent-strong);
|
||||
}
|
||||
|
||||
.asset-table-wrap {
|
||||
overflow-x: auto;
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: 8px;
|
||||
background: #fff;
|
||||
}
|
||||
|
||||
.asset-table {
|
||||
width: 100%;
|
||||
min-width: 760px;
|
||||
border-collapse: collapse;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.asset-table th,
|
||||
.asset-table td {
|
||||
border-bottom: 1px solid var(--color-border);
|
||||
padding: 7px 8px;
|
||||
text-align: left;
|
||||
vertical-align: top;
|
||||
}
|
||||
|
||||
.asset-table th {
|
||||
background: #f8fafc;
|
||||
color: var(--color-text-secondary);
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.asset-table td small {
|
||||
display: block;
|
||||
margin-top: 2px;
|
||||
color: var(--color-text-muted);
|
||||
max-width: 280px;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.asset-inline-action {
|
||||
min-width: 46px;
|
||||
height: 26px;
|
||||
padding: 0 8px;
|
||||
font-size: 12px;
|
||||
line-height: 1;
|
||||
}
|
||||
|
||||
.asset-action-placeholder {
|
||||
color: var(--color-text-muted);
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.asset-badge {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
max-width: 100%;
|
||||
border-radius: 999px;
|
||||
padding: 2px 7px;
|
||||
font-size: 11px;
|
||||
font-weight: 700;
|
||||
border: 1px solid var(--color-border);
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.asset-badge--ok {
|
||||
background: #dcfce7;
|
||||
color: #166534;
|
||||
border-color: #bbf7d0;
|
||||
}
|
||||
|
||||
.asset-badge--warn {
|
||||
background: #fef3c7;
|
||||
color: #92400e;
|
||||
border-color: #fde68a;
|
||||
}
|
||||
|
||||
.asset-badge--bad {
|
||||
background: #fee2e2;
|
||||
color: #991b1b;
|
||||
border-color: #fecaca;
|
||||
}
|
||||
|
||||
.asset-badge--neutral {
|
||||
background: #f1f5f9;
|
||||
color: #475569;
|
||||
}
|
||||
|
||||
.asset-pager {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: flex-end;
|
||||
gap: 8px;
|
||||
padding: 8px;
|
||||
color: var(--color-text-muted);
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.asset-pager button {
|
||||
padding: 5px 9px;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
@media (max-width: 1200px) {
|
||||
.top-status-bar {
|
||||
grid-template-columns: 1fr;
|
||||
|
||||
@@ -0,0 +1,270 @@
|
||||
import { useCallback, useEffect, useMemo, useState } from 'react';
|
||||
import {
|
||||
getAssetInventoryStatus,
|
||||
listAssetIssues,
|
||||
listOrbitAssets,
|
||||
listSourceAssets,
|
||||
scanAssetInventory,
|
||||
} from './api/assets';
|
||||
|
||||
const PAGE_SIZE = 100;
|
||||
|
||||
const fmtDateTime = (value) => {
|
||||
if (!value) return '-';
|
||||
const date = new Date(value);
|
||||
if (Number.isNaN(date.getTime())) return String(value);
|
||||
return date.toLocaleString();
|
||||
};
|
||||
|
||||
const fmtBytes = (value) => {
|
||||
const n = Number(value);
|
||||
if (!Number.isFinite(n) || n <= 0) return '-';
|
||||
if (n >= 1024 ** 3) return `${(n / (1024 ** 3)).toFixed(2)} GB`;
|
||||
if (n >= 1024 ** 2) return `${(n / (1024 ** 2)).toFixed(1)} MB`;
|
||||
if (n >= 1024) return `${(n / 1024).toFixed(1)} KB`;
|
||||
return `${n} B`;
|
||||
};
|
||||
|
||||
const StatusBadge = ({ value }) => {
|
||||
const text = String(value || '-');
|
||||
const status = text.toUpperCase();
|
||||
const tone = status === 'OK' || status === 'MATCHED' || status === 'SELECTED'
|
||||
? 'ok'
|
||||
: status === 'WARNING' || status === 'OPEN' || status === 'MISSING'
|
||||
? 'warn'
|
||||
: status === 'FAILED' || status === 'INACCESSIBLE' || status === 'ERROR'
|
||||
? 'bad'
|
||||
: 'neutral';
|
||||
return <span className={`asset-badge asset-badge--${tone}`}>{text}</span>;
|
||||
};
|
||||
|
||||
const Metric = ({ label, value, hint }) => (
|
||||
<div className="asset-metric">
|
||||
<span>{label}</span>
|
||||
<strong>{value ?? 0}</strong>
|
||||
{hint ? <small>{hint}</small> : null}
|
||||
</div>
|
||||
);
|
||||
|
||||
export default function AssetInventoryPanel({ readOnly = false, onTaskStart }) {
|
||||
const [status, setStatus] = useState(null);
|
||||
const [sources, setSources] = useState({ items: [], total: 0, offset: 0, has_more: false });
|
||||
const [orbits, setOrbits] = useState({ items: [], total: 0, offset: 0, has_more: false });
|
||||
const [issues, setIssues] = useState({ items: [], total: 0, offset: 0, has_more: false });
|
||||
const [activeTab, setActiveTab] = useState('sources');
|
||||
const [family, setFamily] = useState('all');
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [scanLoading, setScanLoading] = useState(false);
|
||||
const [message, setMessage] = useState('');
|
||||
const [error, setError] = useState('');
|
||||
|
||||
const familyParam = useMemo(() => (family === 'all' ? undefined : family), [family]);
|
||||
|
||||
const refresh = useCallback(async ({ sourceOffset = 0, orbitOffset = 0, issueOffset = 0 } = {}) => {
|
||||
setLoading(true);
|
||||
setError('');
|
||||
try {
|
||||
const [nextStatus, nextSources, nextOrbits, nextIssues] = await Promise.all([
|
||||
getAssetInventoryStatus(),
|
||||
listSourceAssets({ satellite_family: familyParam, limit: PAGE_SIZE, offset: sourceOffset }),
|
||||
listOrbitAssets({ satellite_family: familyParam, limit: PAGE_SIZE, offset: orbitOffset }),
|
||||
listAssetIssues({ status: 'OPEN', limit: PAGE_SIZE, offset: issueOffset }),
|
||||
]);
|
||||
setStatus(nextStatus);
|
||||
setSources(nextSources);
|
||||
setOrbits(nextOrbits);
|
||||
setIssues(nextIssues);
|
||||
} catch (err) {
|
||||
setError(err?.response?.data?.detail || err.message || '加载资产库存失败');
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, [familyParam]);
|
||||
|
||||
useEffect(() => {
|
||||
refresh();
|
||||
}, [refresh]);
|
||||
|
||||
const handleScan = async () => {
|
||||
if (readOnly || scanLoading) return;
|
||||
setScanLoading(true);
|
||||
setMessage('');
|
||||
setError('');
|
||||
try {
|
||||
const result = await scanAssetInventory({ inventory_types: [], root_ids: [], bind_orbits: true });
|
||||
setMessage(`资产扫描任务已入队: ${result.task_id}`);
|
||||
onTaskStart?.(result.task_id, '源数据/精轨资产扫描已入队', {
|
||||
taskType: 'SCAN_ASSET_INVENTORY',
|
||||
nonBlocking: true,
|
||||
});
|
||||
} catch (err) {
|
||||
setError(err?.response?.data?.detail || err.message || '启动资产扫描失败');
|
||||
} finally {
|
||||
setScanLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const states = status?.states || [];
|
||||
const sourceRoots = states.filter(item => item.inventory_type === 'source_product');
|
||||
const orbitRoots = states.filter(item => item.inventory_type === 'orbit_asset');
|
||||
|
||||
const renderPager = (data, onPage) => (
|
||||
<div className="asset-pager">
|
||||
<button type="button" disabled={data.offset <= 0 || loading} onClick={() => onPage(Math.max(0, data.offset - PAGE_SIZE))}>
|
||||
上一页
|
||||
</button>
|
||||
<span>{data.offset + 1}-{data.offset + data.items.length} / {data.total}</span>
|
||||
<button type="button" disabled={!data.has_more || loading} onClick={() => onPage(data.offset + PAGE_SIZE)}>
|
||||
下一页
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="asset-inventory-panel">
|
||||
<div className="asset-toolbar">
|
||||
<div>
|
||||
<h3>源数据与精轨资产</h3>
|
||||
<p>Sentinel-1 与 LT-1 的源产品、精密轨道和绑定状态</p>
|
||||
</div>
|
||||
<div className="asset-actions">
|
||||
<select value={family} onChange={(e) => setFamily(e.target.value)} disabled={loading}>
|
||||
<option value="all">全部卫星族</option>
|
||||
<option value="S1">Sentinel-1</option>
|
||||
<option value="LT1">LT-1</option>
|
||||
</select>
|
||||
<button type="button" onClick={() => refresh()} disabled={loading}>刷新</button>
|
||||
<button type="button" onClick={handleScan} disabled={readOnly || scanLoading}>扫描资产</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{error ? <div className="asset-message asset-message--error">{error}</div> : null}
|
||||
{message ? <div className="asset-message">{message}</div> : null}
|
||||
|
||||
<div className="asset-metrics">
|
||||
<Metric label="源产品" value={status?.source_asset_count} hint={`${sourceRoots.length} 个源数据根`} />
|
||||
<Metric label="精轨资产" value={status?.orbit_asset_count} hint={`${orbitRoots.length} 个精轨根`} />
|
||||
<Metric label="已绑定场景" value={status?.selected_binding_count} />
|
||||
<Metric label="开放问题" value={status?.open_issue_count} />
|
||||
</div>
|
||||
|
||||
<div className="asset-root-strip">
|
||||
{states.map((item) => (
|
||||
<div className="asset-root-item" key={`${item.inventory_type}-${item.root_ref_id}`}>
|
||||
<div>
|
||||
<strong>{item.inventory_type === 'source_product' ? '源数据池' : '精轨池'}</strong>
|
||||
<span title={item.root_path}>{item.root_path}</span>
|
||||
</div>
|
||||
<StatusBadge value={item.status} />
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<div className="asset-tabbar">
|
||||
<button type="button" className={activeTab === 'sources' ? 'active-tab' : ''} onClick={() => setActiveTab('sources')}>
|
||||
源产品 ({sources.total})
|
||||
</button>
|
||||
<button type="button" className={activeTab === 'orbits' ? 'active-tab' : ''} onClick={() => setActiveTab('orbits')}>
|
||||
精轨 ({orbits.total})
|
||||
</button>
|
||||
<button type="button" className={activeTab === 'issues' ? 'active-tab' : ''} onClick={() => setActiveTab('issues')}>
|
||||
问题 ({issues.total})
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{activeTab === 'sources' && (
|
||||
<div className="asset-table-wrap">
|
||||
<table className="asset-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>卫星</th>
|
||||
<th>日期/时间</th>
|
||||
<th>产品</th>
|
||||
<th>轨道</th>
|
||||
<th>状态</th>
|
||||
<th>动作</th>
|
||||
<th>文件</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{sources.items.map(item => {
|
||||
return (
|
||||
<tr key={item.id}>
|
||||
<td><strong>{item.satellite}</strong><small>{item.satellite_family}</small></td>
|
||||
<td>{item.imaging_date}<small>{fmtDateTime(item.acquisition_start_time_utc)}</small></td>
|
||||
<td>{item.source_format}<small>{item.imaging_mode} / {item.polarization}</small></td>
|
||||
<td>{item.relative_orbit || '-'}<small>abs {item.absolute_orbit || '-'}</small></td>
|
||||
<td><StatusBadge value={item.parse_status} /></td>
|
||||
<td>
|
||||
<span className="asset-action-placeholder">-</span>
|
||||
</td>
|
||||
<td title={item.file_path}>{item.file_name || item.logical_product_uid}<small>{fmtBytes(item.size_bytes)}</small></td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
{renderPager(sources, (offset) => refresh({ sourceOffset: offset, orbitOffset: orbits.offset, issueOffset: issues.offset }))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{activeTab === 'orbits' && (
|
||||
<div className="asset-table-wrap">
|
||||
<table className="asset-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>卫星</th>
|
||||
<th>类型</th>
|
||||
<th>有效期</th>
|
||||
<th>质量</th>
|
||||
<th>状态</th>
|
||||
<th>文件</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{orbits.items.map(item => (
|
||||
<tr key={item.id}>
|
||||
<td><strong>{item.satellite}</strong><small>{item.satellite_family}</small></td>
|
||||
<td>{item.orbit_type}<small>{item.native_format}</small></td>
|
||||
<td>{fmtDateTime(item.validity_start_time_utc)}<small>{fmtDateTime(item.validity_stop_time_utc)}</small></td>
|
||||
<td>{item.quality_class}</td>
|
||||
<td><StatusBadge value={item.parse_status} /></td>
|
||||
<td title={item.file_path}>{item.file_name}<small>{fmtBytes(item.size_bytes)}</small></td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
{renderPager(orbits, (offset) => refresh({ sourceOffset: sources.offset, orbitOffset: offset, issueOffset: issues.offset }))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{activeTab === 'issues' && (
|
||||
<div className="asset-table-wrap">
|
||||
<table className="asset-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>级别</th>
|
||||
<th>代码</th>
|
||||
<th>对象</th>
|
||||
<th>说明</th>
|
||||
<th>时间</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{issues.items.map(item => (
|
||||
<tr key={item.id}>
|
||||
<td><StatusBadge value={item.severity} /></td>
|
||||
<td>{item.issue_code}</td>
|
||||
<td>{item.inventory_type}<small>{item.source_path || `radar ${item.radar_data_id || '-'}`}</small></td>
|
||||
<td>{item.issue_message || '-'}</td>
|
||||
<td>{fmtDateTime(item.last_seen_at)}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
{renderPager(issues, (offset) => refresh({ sourceOffset: sources.offset, orbitOffset: orbits.offset, issueOffset: offset }))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import React, { useEffect, useRef, useState } from 'react';
|
||||
import './App.css';
|
||||
import { useI18n } from './i18n/I18nContext';
|
||||
import { getAssetInventoryStatus, scanAssetInventory, unpackSentinel1Batch } from './api/assets';
|
||||
|
||||
const DEFAULT_MONITOR_CONFIG = {
|
||||
radar_dirs: [],
|
||||
@@ -8,6 +9,9 @@ const DEFAULT_MONITOR_CONFIG = {
|
||||
dinsar_dirs: [],
|
||||
gf3_source_dirs: [],
|
||||
gf3_storage_dirs: [],
|
||||
s1_source_dirs: [],
|
||||
s1_storage_dirs: [],
|
||||
s1_orbit_dirs: [],
|
||||
};
|
||||
|
||||
const DEFAULT_UNPACK_CONFIG = {
|
||||
@@ -42,6 +46,7 @@ const parseUnpackRunValue = (rawValue, label) => {
|
||||
};
|
||||
|
||||
const formatList = (list) => (Array.isArray(list) && list.length ? list.join('; ') : '未配置');
|
||||
const normalizeComparePath = (value) => String(value || '').trim().replace(/\\/g, '/').replace(/\/+$/, '').toLowerCase();
|
||||
|
||||
const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled = true }) => {
|
||||
const { t } = useI18n();
|
||||
@@ -50,8 +55,6 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
|
||||
const [logs, setLogs] = useState([]);
|
||||
const [activeTasks, setActiveTasks] = useState([]);
|
||||
const [unpackConfig, setUnpackConfig] = useState(DEFAULT_UNPACK_CONFIG);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [message, setMessage] = useState('');
|
||||
const [unpackLoading, setUnpackLoading] = useState(false);
|
||||
const [unpackMessage, setUnpackMessage] = useState('');
|
||||
const [showUnpackDialog, setShowUnpackDialog] = useState(false);
|
||||
@@ -59,7 +62,11 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
|
||||
const [unpackDialogError, setUnpackDialogError] = useState('');
|
||||
const [unpackTaskId, setUnpackTaskId] = useState('');
|
||||
const [unpackTaskTerminal, setUnpackTaskTerminal] = useState(false);
|
||||
const [s1Loading, setS1Loading] = useState(false);
|
||||
const [s1ScanLoading, setS1ScanLoading] = useState(false);
|
||||
const [s1Message, setS1Message] = useState('');
|
||||
const [gf3Loading, setGf3Loading] = useState(false);
|
||||
const [gf3ProcessLoading, setGf3ProcessLoading] = useState(false);
|
||||
const [gf3Message, setGf3Message] = useState('');
|
||||
const logEndRef = useRef(null);
|
||||
|
||||
@@ -76,6 +83,7 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
|
||||
const unpackActiveTask = displayActiveTasks.find((task) =>
|
||||
task.task_id === unpackTaskId || task.task_type === 'UNPACK_ARCHIVES'
|
||||
);
|
||||
const s1ActiveTask = displayActiveTasks.find((task) => task.task_type === 'UNPACK_SENTINEL1');
|
||||
|
||||
useEffect(() => {
|
||||
if (!enabled) {
|
||||
@@ -102,6 +110,9 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
|
||||
...data,
|
||||
radar_dirs: toArray(data?.radar_dirs),
|
||||
dinsar_dirs: toArray(data?.dinsar_dirs),
|
||||
s1_source_dirs: toArray(data?.s1_source_dirs),
|
||||
s1_storage_dirs: toArray(data?.s1_storage_dirs),
|
||||
s1_orbit_dirs: toArray(data?.s1_orbit_dirs),
|
||||
gf3_source_dirs: toArray(data?.gf3_source_dirs),
|
||||
gf3_storage_dirs: toArray(data?.gf3_storage_dirs),
|
||||
});
|
||||
@@ -267,51 +278,134 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
|
||||
|
||||
const hasRadarDirs = config.radar_dirs.length > 0;
|
||||
const hasOrbitDir = typeof config.orbit_dir === 'string' && config.orbit_dir.trim() !== '';
|
||||
const hasDinsarDirs = config.dinsar_dirs.length > 0;
|
||||
const hasS1SourceDirs = config.s1_source_dirs.length > 0;
|
||||
const hasS1StorageDirs = config.s1_storage_dirs.length > 0;
|
||||
const hasS1OrbitDirs = config.s1_orbit_dirs.length > 0;
|
||||
const hasGf3SourceDirs = config.gf3_source_dirs.length > 0;
|
||||
const hasGf3StorageDirs = config.gf3_storage_dirs.length > 0;
|
||||
|
||||
const canRunRadar = !readOnly && configLoaded && hasRadarDirs;
|
||||
const canRunOrbit = !readOnly && configLoaded && hasOrbitDir;
|
||||
const canRunDinsar = !readOnly && configLoaded && hasDinsarDirs;
|
||||
const canRunS1 = !readOnly && configLoaded && (hasS1SourceDirs || hasS1StorageDirs || hasS1OrbitDirs);
|
||||
const canRunS1Scan = !readOnly && configLoaded && hasS1SourceDirs;
|
||||
const canRunS1OrbitScan = !readOnly && configLoaded && hasS1OrbitDirs;
|
||||
const canRunGf3Scan = !readOnly && configLoaded && hasGf3StorageDirs;
|
||||
const canRunGf3Process = !readOnly && configLoaded && hasGf3SourceDirs;
|
||||
const canOpenUnpackDialog = !readOnly && unpackConfig.source_dirs.length > 0;
|
||||
|
||||
const handleRunNow = async (target) => {
|
||||
const handleS1Run = async () => {
|
||||
if (readOnly) {
|
||||
setMessage('当前账户为只读模式,无法触发扫描。');
|
||||
setS1Message('当前账户为只读模式,无法触发 Sentinel-1 任务。');
|
||||
return;
|
||||
}
|
||||
|
||||
setLoading(true);
|
||||
const targetMap = {
|
||||
radar: 'LT-1 数据',
|
||||
orbit: '精轨数据',
|
||||
dinsar: 'D-InSAR 结果',
|
||||
gf3: 'GF3 数据',
|
||||
};
|
||||
setMessage(`正在触发${targetMap[target] || '全部'}手动扫描...`);
|
||||
|
||||
setS1Loading(true);
|
||||
setS1Message('Sentinel-1 任务启动中...');
|
||||
try {
|
||||
const url = target ? `${apiEndpoint}/monitor/run-now?target=${target}` : `${apiEndpoint}/monitor/run-now`;
|
||||
const res = await fetch(url, {
|
||||
const res = await unpackSentinel1Batch({
|
||||
scan_before_unpack: true,
|
||||
overwrite: false,
|
||||
});
|
||||
setS1Message(res.message || 'Sentinel-1 任务已启动');
|
||||
if (onTaskStart) {
|
||||
onTaskStart(res.task_id, 'Sentinel-1 任务已启动。', {
|
||||
nonBlocking: true,
|
||||
taskType: 'UNPACK_SENTINEL1',
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
setS1Message(`失败:${err?.response?.data?.detail || err.message || '未知错误'}`);
|
||||
} finally {
|
||||
setS1Loading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleS1Scan = async () => {
|
||||
if (readOnly) {
|
||||
setS1Message('当前账户为只读模式,无法触发 Sentinel-1 扫描。');
|
||||
return;
|
||||
}
|
||||
setS1ScanLoading(true);
|
||||
setS1Message('Sentinel-1 源数据扫描启动中...');
|
||||
try {
|
||||
const inventoryStatus = await getAssetInventoryStatus();
|
||||
const sourcePathSet = new Set(config.s1_source_dirs.map(normalizeComparePath));
|
||||
const rootIds = (inventoryStatus?.states || [])
|
||||
.filter((item) => item?.inventory_type === 'source_product' && sourcePathSet.has(normalizeComparePath(item?.root_path)))
|
||||
.map((item) => item.root_ref_id)
|
||||
.filter((value, index, array) => value && array.indexOf(value) === index);
|
||||
const res = await scanAssetInventory({
|
||||
inventory_types: ['source_product'],
|
||||
root_ids: rootIds,
|
||||
bind_orbits: true,
|
||||
});
|
||||
setS1Message(res.message || 'Sentinel-1 源数据扫描任务已启动');
|
||||
onTaskStart?.(res.task_id, 'Sentinel-1 源数据扫描任务已启动。', {
|
||||
nonBlocking: true,
|
||||
taskType: 'SCAN_ASSET_INVENTORY',
|
||||
});
|
||||
} catch (err) {
|
||||
setS1Message(`失败:${err?.response?.data?.detail || err.message || '未知错误'}`);
|
||||
} finally {
|
||||
setS1ScanLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleS1OrbitScan = async () => {
|
||||
if (readOnly) {
|
||||
setS1Message('当前账户为只读模式,无法触发 Sentinel-1 精轨扫描。');
|
||||
return;
|
||||
}
|
||||
setS1ScanLoading(true);
|
||||
setS1Message('Sentinel-1 精轨扫描启动中...');
|
||||
try {
|
||||
const inventoryStatus = await getAssetInventoryStatus();
|
||||
const orbitPathSet = new Set(config.s1_orbit_dirs.map(normalizeComparePath));
|
||||
const rootIds = (inventoryStatus?.states || [])
|
||||
.filter((item) => item?.inventory_type === 'orbit_asset' && orbitPathSet.has(normalizeComparePath(item?.root_path)))
|
||||
.map((item) => item.root_ref_id)
|
||||
.filter((value, index, array) => value && array.indexOf(value) === index);
|
||||
const res = await scanAssetInventory({
|
||||
inventory_types: ['orbit_asset'],
|
||||
root_ids: rootIds,
|
||||
bind_orbits: true,
|
||||
});
|
||||
setS1Message(res.message || 'Sentinel-1 精轨扫描任务已启动');
|
||||
onTaskStart?.(res.task_id, 'Sentinel-1 精轨扫描任务已启动。', {
|
||||
nonBlocking: true,
|
||||
taskType: 'SCAN_ASSET_INVENTORY',
|
||||
});
|
||||
} catch (err) {
|
||||
setS1Message(`失败:${err?.response?.data?.detail || err.message || '未知错误'}`);
|
||||
} finally {
|
||||
setS1ScanLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleGf3BatchProcess = async () => {
|
||||
if (readOnly) {
|
||||
setGf3Message('当前账户为只读模式,无法触发 GF3 预处理。');
|
||||
return;
|
||||
}
|
||||
setGf3ProcessLoading(true);
|
||||
setGf3Message('GF3 预处理启动中...');
|
||||
try {
|
||||
const res = await fetch(`${apiEndpoint}/monitor/gf3-process`, {
|
||||
method: 'POST',
|
||||
credentials: 'include',
|
||||
});
|
||||
const data = await parseJsonSafe(res, {});
|
||||
if (res.ok) {
|
||||
setMessage(data.message || '扫描任务已启动。');
|
||||
setGf3Message(data.message || 'GF3 批量处理任务已启动');
|
||||
if (onTaskStart) {
|
||||
onTaskStart(data.task_id, `已触发${targetMap[target] || '全部'}手动扫描...`);
|
||||
onTaskStart(data.task_id, 'GF3 L1A→L2 批量处理已启动。');
|
||||
}
|
||||
} else {
|
||||
setMessage(`触发失败: ${data.detail || '未知错误'}`);
|
||||
setGf3Message(`失败:${data.detail || '未知错误'}`);
|
||||
}
|
||||
} catch (err) {
|
||||
setMessage(`触发失败: ${err.message}`);
|
||||
setGf3Message(`失败:${err.message || '未知错误'}`);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
setGf3ProcessLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -397,29 +491,79 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
|
||||
}
|
||||
};
|
||||
|
||||
const handleGf3BatchProcess = async () => {
|
||||
const handleRadarScan = async () => {
|
||||
if (readOnly) {
|
||||
setGf3Message('当前账户为只读模式,无法触发 GF3 处理。');
|
||||
setUnpackMessage('当前账户为只读模式,无法触发扫描。');
|
||||
return;
|
||||
}
|
||||
setGf3Loading(true);
|
||||
setGf3Message('GF3 批量处理启动中...');
|
||||
setUnpackMessage('LT-1 扫描启动中...');
|
||||
try {
|
||||
const res = await fetch(`${apiEndpoint}/monitor/gf3-process`, {
|
||||
const res = await fetch(`${apiEndpoint}/monitor/run-now?target=radar`, {
|
||||
method: 'POST',
|
||||
credentials: 'include',
|
||||
});
|
||||
const data = await parseJsonSafe(res, {});
|
||||
if (res.ok) {
|
||||
setGf3Message(data.message || 'GF3 批量处理任务已启动');
|
||||
setUnpackMessage(data.message || 'LT-1 扫描任务已启动');
|
||||
if (onTaskStart) {
|
||||
onTaskStart(data.task_id, 'GF3 L1A→L2 批量处理已启动。');
|
||||
onTaskStart(data.task_id, '已触发 LT-1 手动扫描...');
|
||||
}
|
||||
} else {
|
||||
setUnpackMessage(`失败:${data.detail || '未知错误'}`);
|
||||
}
|
||||
} catch (err) {
|
||||
setUnpackMessage(`失败:${err.message || '未知错误'}`);
|
||||
}
|
||||
};
|
||||
|
||||
const handleOrbitScan = async () => {
|
||||
if (readOnly) {
|
||||
setUnpackMessage('当前账户为只读模式,无法触发扫描。');
|
||||
return;
|
||||
}
|
||||
setUnpackMessage('精轨扫描启动中...');
|
||||
try {
|
||||
const res = await fetch(`${apiEndpoint}/monitor/run-now?target=orbit`, {
|
||||
method: 'POST',
|
||||
credentials: 'include',
|
||||
});
|
||||
const data = await parseJsonSafe(res, {});
|
||||
if (res.ok) {
|
||||
setUnpackMessage(data.message || '精轨扫描任务已启动');
|
||||
if (onTaskStart) {
|
||||
onTaskStart(data.task_id, '已触发精轨手动扫描...');
|
||||
}
|
||||
} else {
|
||||
setUnpackMessage(`失败:${data.detail || '未知错误'}`);
|
||||
}
|
||||
} catch (err) {
|
||||
setUnpackMessage(`失败:${err.message || '未知错误'}`);
|
||||
}
|
||||
};
|
||||
|
||||
const handleGf3Scan = async () => {
|
||||
if (readOnly) {
|
||||
setGf3Message('当前账户为只读模式,无法触发扫描。');
|
||||
return;
|
||||
}
|
||||
setGf3Loading(true);
|
||||
setGf3Message('GF3 扫描启动中...');
|
||||
try {
|
||||
const res = await fetch(`${apiEndpoint}/monitor/run-now?target=gf3`, {
|
||||
method: 'POST',
|
||||
credentials: 'include',
|
||||
});
|
||||
const data = await parseJsonSafe(res, {});
|
||||
if (res.ok) {
|
||||
setGf3Message(data.message || 'GF3 扫描任务已启动');
|
||||
if (onTaskStart) {
|
||||
onTaskStart(data.task_id, '已触发 GF3 手动扫描...');
|
||||
}
|
||||
} else {
|
||||
setGf3Message(`失败:${data.detail || '未知错误'}`);
|
||||
}
|
||||
} catch (err) {
|
||||
setGf3Message(`失败:${err.message}`);
|
||||
setGf3Message(`失败:${err.message || '未知错误'}`);
|
||||
} finally {
|
||||
setGf3Loading(false);
|
||||
}
|
||||
@@ -437,17 +581,6 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
|
||||
const rowStyle = { display: 'flex', gap: '8px' };
|
||||
const gridStyle = { display: 'grid', rowGap: '6px', fontSize: '0.9em', color: 'var(--color-text-secondary)' };
|
||||
|
||||
const scanBtnStyle = (canRun) => ({
|
||||
flex: 1,
|
||||
padding: '8px 5px',
|
||||
backgroundColor: 'var(--color-info)',
|
||||
color: 'white',
|
||||
border: 'none',
|
||||
borderRadius: '4px',
|
||||
cursor: loading || !canRun ? 'not-allowed' : 'pointer',
|
||||
fontSize: '0.85em',
|
||||
});
|
||||
|
||||
const actionBtnStyle = (isLoading, isDisabled) => ({
|
||||
padding: '6px 10px',
|
||||
backgroundColor: 'var(--color-accent)',
|
||||
@@ -495,6 +628,9 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
|
||||
<div style={gridStyle}>
|
||||
<div style={rowStyle}><span style={labelStyle}>LT-1 存储</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.radar_dirs)}</span></div>
|
||||
<div style={rowStyle}><span style={labelStyle}>LT-1 精轨</span><span style={{ wordBreak: 'break-all' }}>{config.orbit_dir || '未配置'}</span></div>
|
||||
<div style={rowStyle}><span style={labelStyle}>S1 源数据</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.s1_source_dirs)}</span></div>
|
||||
<div style={rowStyle}><span style={labelStyle}>S1 存储</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.s1_storage_dirs)}</span></div>
|
||||
<div style={rowStyle}><span style={labelStyle}>S1 精轨</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.s1_orbit_dirs)}</span></div>
|
||||
<div style={rowStyle}><span style={labelStyle}>GF3 来源</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.gf3_source_dirs)}</span></div>
|
||||
<div style={rowStyle}><span style={labelStyle}>GF3 存储</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.gf3_storage_dirs)}</span></div>
|
||||
<div style={rowStyle}><span style={labelStyle}>D-InSAR 结果</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.dinsar_dirs)}</span></div>
|
||||
@@ -502,14 +638,14 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
|
||||
</div>
|
||||
|
||||
<div style={sectionStyle}>
|
||||
<div style={{ fontWeight: 'bold', marginBottom: '8px', color: 'var(--color-text-primary)' }}>LT-1 归档解包</div>
|
||||
<div style={{ fontWeight: 'bold', marginBottom: '8px', color: 'var(--color-text-primary)' }}>LT-1 归档解包 / 扫描</div>
|
||||
<div style={{ ...gridStyle, marginBottom: '8px' }}>
|
||||
<div style={rowStyle}><span style={labelStyle}>来源目录</span><span style={{ wordBreak: 'break-all' }}>{formatList(unpackConfig.source_dirs)}</span></div>
|
||||
<div style={rowStyle}><span style={labelStyle}>LT-1 存储</span><span style={{ wordBreak: 'break-all' }}>{formatList(unpackConfig.insar_storage_dirs)}</span></div>
|
||||
<div style={rowStyle}><span style={labelStyle}>单次上限</span><span>{unpackConfig.max_files_per_run > 0 ? `${unpackConfig.max_files_per_run} 个压缩包` : '不限'}</span></div>
|
||||
<div style={rowStyle}><span style={labelStyle}>最长运行</span><span>{unpackConfig.max_runtime_minutes > 0 ? `${unpackConfig.max_runtime_minutes} 分钟` : '不限'}</span></div>
|
||||
</div>
|
||||
<div style={{ display: 'flex', gap: '10px' }}>
|
||||
<div style={{ display: 'flex', gap: '10px', flexWrap: 'wrap' }}>
|
||||
<button
|
||||
onClick={handleOpenUnpackDialog}
|
||||
disabled={unpackLoading || !canOpenUnpackDialog}
|
||||
@@ -517,6 +653,20 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
|
||||
>
|
||||
{unpackLoading ? '运行中...' : (readOnly ? '只读模式' : 'LT-1 解包')}
|
||||
</button>
|
||||
<button
|
||||
onClick={handleRadarScan}
|
||||
disabled={readOnly || !canRunRadar}
|
||||
style={actionBtnStyle(false, !canRunRadar)}
|
||||
>
|
||||
{readOnly ? '只读模式' : '扫描 LT-1'}
|
||||
</button>
|
||||
<button
|
||||
onClick={handleOrbitScan}
|
||||
disabled={readOnly || !canRunOrbit}
|
||||
style={actionBtnStyle(false, !canRunOrbit)}
|
||||
>
|
||||
{readOnly ? '只读模式' : '扫描精轨'}
|
||||
</button>
|
||||
<div
|
||||
style={{
|
||||
fontSize: '0.85em',
|
||||
@@ -530,7 +680,48 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
|
||||
</div>
|
||||
|
||||
<div style={sectionStyle}>
|
||||
<div style={{ fontWeight: 'bold', marginBottom: '8px', color: 'var(--color-text-primary)' }}>GF3 L1A → L2 处理</div>
|
||||
<div style={{ fontWeight: 'bold', marginBottom: '8px', color: 'var(--color-text-primary)' }}>Sentinel-1 解包 / 扫描</div>
|
||||
<div style={{ ...gridStyle, marginBottom: '8px' }}>
|
||||
<div style={rowStyle}><span style={labelStyle}>S1 源数据</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.s1_source_dirs)}</span></div>
|
||||
<div style={rowStyle}><span style={labelStyle}>S1 存储</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.s1_storage_dirs)}</span></div>
|
||||
<div style={rowStyle}><span style={labelStyle}>S1 精轨</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.s1_orbit_dirs)}</span></div>
|
||||
</div>
|
||||
<div style={{ display: 'flex', gap: '10px' }}>
|
||||
<button
|
||||
onClick={handleS1Run}
|
||||
disabled={s1Loading || s1ScanLoading || !canRunS1}
|
||||
style={actionBtnStyle(s1Loading, !canRunS1)}
|
||||
>
|
||||
{s1Loading ? '运行中...' : (readOnly ? '只读模式' : 'Sentinel-1 解包')}
|
||||
</button>
|
||||
<button
|
||||
onClick={handleS1Scan}
|
||||
disabled={s1ScanLoading || s1Loading || !canRunS1Scan}
|
||||
style={actionBtnStyle(s1ScanLoading, !canRunS1Scan)}
|
||||
>
|
||||
{s1ScanLoading ? '运行中...' : (readOnly ? '只读模式' : '扫描 S1 源数据')}
|
||||
</button>
|
||||
<button
|
||||
onClick={handleS1OrbitScan}
|
||||
disabled={s1ScanLoading || s1Loading || !canRunS1OrbitScan}
|
||||
style={actionBtnStyle(s1ScanLoading, !canRunS1OrbitScan)}
|
||||
>
|
||||
{s1ScanLoading ? '运行中...' : (readOnly ? '只读模式' : '扫描 S1 精轨')}
|
||||
</button>
|
||||
<div
|
||||
style={{
|
||||
fontSize: '0.85em',
|
||||
color: s1Message.includes('失败') ? 'var(--color-danger)' : 'var(--color-text-muted)',
|
||||
alignSelf: 'center',
|
||||
}}
|
||||
>
|
||||
{s1ActiveTask ? (s1ActiveTask.message || 'Sentinel-1 任务运行中...') : s1Message}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div style={sectionStyle}>
|
||||
<div style={{ fontWeight: 'bold', marginBottom: '8px', color: 'var(--color-text-primary)' }}>GF3 归档预处理</div>
|
||||
<div style={{ ...gridStyle, marginBottom: '8px' }}>
|
||||
<div style={rowStyle}><span style={labelStyle}>L1A 来源</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.gf3_source_dirs)}</span></div>
|
||||
<div style={rowStyle}><span style={labelStyle}>L2 存储</span><span style={{ wordBreak: 'break-all' }}>{formatList(config.gf3_storage_dirs)}</span></div>
|
||||
@@ -538,10 +729,17 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
|
||||
<div style={{ display: 'flex', gap: '10px' }}>
|
||||
<button
|
||||
onClick={handleGf3BatchProcess}
|
||||
disabled={gf3Loading || readOnly || !canRunGf3Process}
|
||||
style={actionBtnStyle(gf3Loading, readOnly || !canRunGf3Process)}
|
||||
disabled={gf3ProcessLoading || readOnly || !canRunGf3Process}
|
||||
style={actionBtnStyle(gf3ProcessLoading, readOnly || !canRunGf3Process)}
|
||||
>
|
||||
{gf3Loading ? '运行中...' : (readOnly ? '只读模式' : 'GF3 L1A→L2')}
|
||||
{gf3ProcessLoading ? '运行中...' : (readOnly ? '只读模式' : 'GF3 预处理')}
|
||||
</button>
|
||||
<button
|
||||
onClick={handleGf3Scan}
|
||||
disabled={gf3Loading || readOnly || !canRunGf3Scan}
|
||||
style={actionBtnStyle(gf3Loading, readOnly || !canRunGf3Scan)}
|
||||
>
|
||||
{gf3Loading ? '运行中...' : (readOnly ? '只读模式' : '扫描 GF3')}
|
||||
</button>
|
||||
<div
|
||||
style={{
|
||||
@@ -554,31 +752,10 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div style={sectionStyle}>
|
||||
<div style={{ fontWeight: 'bold', marginBottom: '8px', color: 'var(--color-text-primary)' }}>活动任务</div>
|
||||
{displayActiveTasks.length === 0 ? (
|
||||
<div style={{ fontSize: '0.85em', color: 'var(--color-text-muted)' }}>当前无活动任务。</div>
|
||||
) : (
|
||||
<div style={{ display: 'grid', rowGap: '8px' }}>
|
||||
{displayActiveTasks.slice(0, 4).map((task) => (
|
||||
<div key={task.task_id} style={{ fontSize: '0.85em', color: 'var(--color-text-secondary)' }}>
|
||||
<div style={{ display: 'flex', justifyContent: 'space-between', marginBottom: '4px' }}>
|
||||
<span>{task.task_type}</span>
|
||||
<span>{task.progress}%</span>
|
||||
</div>
|
||||
<div style={{ height: '6px', background: 'var(--color-panel-muted)', borderRadius: '3px', overflow: 'hidden' }}>
|
||||
<div style={{ width: `${task.progress}%`, height: '100%', background: 'var(--color-info)' }} />
|
||||
</div>
|
||||
<div style={{ color: 'var(--color-text-muted)', marginTop: '4px', wordBreak: 'break-all' }}>{t(task.message || '')}</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div style={{ marginBottom: '4px' }}>
|
||||
<h4 style={{ margin: '0 0 5px 0', fontSize: '1em' }}>实时日志</h4>
|
||||
<div style={{ flexShrink: 0, borderTop: '1px solid var(--color-border)', paddingTop: '10px', marginTop: '6px' }}>
|
||||
<div style={{ fontWeight: 'bold', marginBottom: '6px', color: 'var(--color-text-primary)' }}>实时日志</div>
|
||||
<div
|
||||
style={{
|
||||
height: '160px',
|
||||
@@ -601,17 +778,6 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
|
||||
<div ref={logEndRef} />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div style={{ flexShrink: 0, borderTop: '1px solid var(--color-border)', paddingTop: '10px', marginTop: '6px' }}>
|
||||
<div style={{ display: 'flex', gap: '8px', marginBottom: '6px' }}>
|
||||
<button onClick={() => handleRunNow('radar')} disabled={loading || !canRunRadar} style={scanBtnStyle(canRunRadar)}>扫描 LT-1</button>
|
||||
<button onClick={() => handleRunNow('gf3')} disabled={loading || !canRunGf3Scan} style={scanBtnStyle(canRunGf3Scan)}>扫描 GF3</button>
|
||||
<button onClick={() => handleRunNow('orbit')} disabled={loading || !canRunOrbit} style={scanBtnStyle(canRunOrbit)}>扫描精轨</button>
|
||||
<button onClick={() => handleRunNow('dinsar')} disabled={loading || !canRunDinsar} style={scanBtnStyle(canRunDinsar)}>扫描 D-InSAR</button>
|
||||
</div>
|
||||
{message && <div style={{ color: message.includes('失败') ? 'red' : 'green', fontSize: '0.9em' }}>{message}</div>}
|
||||
</div>
|
||||
|
||||
{showUnpackDialog && (
|
||||
<div className="modal-overlay visible" onClick={handleCloseUnpackDialog}>
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import { deleteRunLog, deleteRunRecord, getRunLog, listEngines, listRuns, previewPyintInputAssets, submitRun } from './api/dinsarProduction';
|
||||
import { clearTaskLogs, deleteTaskLog, deleteTaskRecord, getActiveTasks, getRecentTasks, getTaskLogs } from './api/tasks';
|
||||
import { formatSatelliteFamilyLabel, inferSatelliteFamilyFromResultLike } from './utils/satelliteFamily';
|
||||
|
||||
const card = {
|
||||
background: '#fff',
|
||||
@@ -160,6 +161,9 @@ function taskToRunRow(task) {
|
||||
completed_items: null,
|
||||
failed_items: null,
|
||||
skipped_items: null,
|
||||
master_satellite: task?.master_satellite || '',
|
||||
slave_satellite: task?.slave_satellite || '',
|
||||
pair_key: task?.pair_key || '',
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1378,7 +1382,7 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued })
|
||||
<div>
|
||||
<div style={{ fontSize: 12, color: '#0f172a', fontWeight: 600 }}>PyINT 输入资产预检</div>
|
||||
<div style={{ fontSize: 11, color: '#64748b', marginTop: 4 }}>
|
||||
提交前检查 Task_* 结构、DEM 策略和 LT-1 轨道是否齐备。即使不手动预检,后端提交时也会做同样校验。
|
||||
提交前检查 Task_* 结构、DEM 策略以及生产所需源数据和轨道文件是否齐备。即使不手动预检,后端提交时也会做同样校验。
|
||||
</div>
|
||||
</div>
|
||||
<button
|
||||
@@ -1436,7 +1440,7 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued })
|
||||
{ label: 'DEM 策略', value: formatPyintDemMode(pyintPreview?.dem?.mode), color: '#1d4ed8' },
|
||||
{ label: '轨道策略', value: formatPyintOrbitPolicy(pyintPreview?.orbits?.policy), color: '#7c3aed' },
|
||||
{
|
||||
label: '精轨桥接',
|
||||
label: '轨道处理',
|
||||
value: pyintPreview?.precise_orbit_bridge?.enabled
|
||||
? formatPyintPreciseOrbitMode(pyintPreview?.precise_orbit_bridge?.mode)
|
||||
: '关闭',
|
||||
@@ -1732,7 +1736,7 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued })
|
||||
<table style={{ width: '100%', borderCollapse: 'collapse', fontSize: 12 }}>
|
||||
<thead>
|
||||
<tr>
|
||||
{['运行ID', '引擎', '状态', '时间', '路径', '操作'].map(header => (
|
||||
{['运行ID', '引擎', '数据', '状态', '时间', '路径', '操作'].map(header => (
|
||||
<th
|
||||
key={header}
|
||||
style={{
|
||||
@@ -1759,6 +1763,7 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued })
|
||||
<tr key={`${run.record_type || 'run'}-${run.run_id}`} style={{ borderBottom: '1px solid #f1f5f9' }}>
|
||||
<td style={{ padding: '6px 8px', fontFamily: 'monospace', fontSize: 11 }}>{run.run_id}</td>
|
||||
<td style={{ padding: '6px 8px' }}>{formatEngineLabel(run.engine)}</td>
|
||||
<td style={{ padding: '6px 8px' }}>{formatSatelliteFamilyLabel(inferSatelliteFamilyFromResultLike(run))}</td>
|
||||
<td
|
||||
style={{
|
||||
padding: '6px 8px',
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import apiClient from './client';
|
||||
|
||||
export const getAssetInventoryStatus = () =>
|
||||
apiClient.get('/assets/inventory/status').then(r => r.data);
|
||||
|
||||
export const scanAssetInventory = (payload = {}) =>
|
||||
apiClient.post('/assets/inventory/scan', payload).then(r => r.data);
|
||||
|
||||
export const listSourceAssets = (params = {}) =>
|
||||
apiClient.get('/assets/sources', { params }).then(r => r.data);
|
||||
|
||||
export const listOrbitAssets = (params = {}) =>
|
||||
apiClient.get('/assets/orbits', { params }).then(r => r.data);
|
||||
|
||||
export const listAssetIssues = (params = {}) =>
|
||||
apiClient.get('/assets/issues', { params }).then(r => r.data);
|
||||
|
||||
export const unpackSentinel1Source = (assetId, payload = {}) =>
|
||||
apiClient.post(`/assets/sources/${assetId}/unpack-sentinel1`, payload).then(r => r.data);
|
||||
|
||||
export const unpackSentinel1Batch = (payload = {}) =>
|
||||
apiClient.post('/assets/inventory/unpack-sentinel1', payload).then(r => r.data);
|
||||
@@ -3,6 +3,9 @@ import axios from 'axios';
|
||||
const apiClient = axios.create({
|
||||
baseURL: '/api',
|
||||
withCredentials: true,
|
||||
paramsSerializer: {
|
||||
indexes: null,
|
||||
},
|
||||
});
|
||||
|
||||
export default apiClient;
|
||||
|
||||
@@ -20,6 +20,10 @@ const getTaskTypeLabel = (taskType) => {
|
||||
return '灾害点同步';
|
||||
case 'UNPACK_ARCHIVES':
|
||||
return 'LT-1 解包';
|
||||
case 'UNPACK_SENTINEL1':
|
||||
return 'Sentinel-1 解包';
|
||||
case 'SCAN_ASSET_INVENTORY':
|
||||
return '资产库存扫描';
|
||||
case 'IDL_IMPORT':
|
||||
return 'ENVI 数据导入';
|
||||
case 'IDL_DINSAR':
|
||||
|
||||
@@ -1,33 +1,92 @@
|
||||
const createRows = (dataInfo, language, formatYmd) => [
|
||||
{ label: language === 'en' ? 'Satellite:' : '卫星:', value: dataInfo.satellite || '-' },
|
||||
{ label: language === 'en' ? 'Satellite Mode:' : '卫星模式:', value: dataInfo.satellite_mode || '-' },
|
||||
{ label: language === 'en' ? 'Receiving Station:' : '接收站:', value: dataInfo.receiving_station || '-' },
|
||||
{ label: language === 'en' ? 'Imaging Date:' : '成像日期:', value: formatYmd(dataInfo.imaging_date) },
|
||||
{ label: language === 'en' ? 'Imaging Mode:' : '成像模式:', value: dataInfo.imaging_mode || '-' },
|
||||
{ label: language === 'en' ? 'Orbit Circle:' : '轨道圈号:', value: dataInfo.orbit_circle || '-' },
|
||||
{ label: language === 'en' ? 'Scene Center Lon:' : '场景中心经度:', value: dataInfo.scene_center_lon ?? '-' },
|
||||
{ label: language === 'en' ? 'Scene Center Lat:' : '场景中心纬度:', value: dataInfo.scene_center_lat ?? '-' },
|
||||
{ label: language === 'en' ? 'Acquisition Time:' : '采集时间:', value: dataInfo.acquisition_time_utc || '-' },
|
||||
{ label: language === 'en' ? 'Product Type:' : '产品类型:', value: dataInfo.product_type || '-' },
|
||||
{ label: language === 'en' ? 'Polarization:' : '极化方式:', value: dataInfo.polarization || '-' },
|
||||
{ label: language === 'en' ? 'Product Level:' : '产品级别:', value: dataInfo.product_level || '-' },
|
||||
{ label: language === 'en' ? 'Product Unique ID:' : '产品唯一ID:', value: dataInfo.product_unique_id || '-' },
|
||||
{ label: language === 'en' ? 'Orbit Direction:' : '轨道方向:', value: dataInfo.orbit_direction || '-' },
|
||||
{
|
||||
label: language === 'en' ? 'Has Orbit:' : '有精轨:',
|
||||
value: dataInfo.has_orbit_data ? (language === 'en' ? 'Yes' : '是') : (language === 'en' ? 'No' : '否'),
|
||||
},
|
||||
{
|
||||
label: language === 'en' ? 'Orbit File:' : '轨道文件:',
|
||||
value: dataInfo.orbit_file_path || '-',
|
||||
const formatDateTime = (value) => {
|
||||
if (!value) return '-';
|
||||
const date = new Date(value);
|
||||
if (Number.isNaN(date.getTime())) return String(value);
|
||||
return date.toLocaleString();
|
||||
};
|
||||
|
||||
const joinChannels = (value) => {
|
||||
if (Array.isArray(value)) {
|
||||
const items = value.map((item) => String(item || '').trim()).filter(Boolean);
|
||||
return items.length ? items.join(' / ') : '-';
|
||||
}
|
||||
if (value === null || value === undefined || value === '') return '-';
|
||||
return String(value);
|
||||
};
|
||||
|
||||
const yesNo = (flag, language) => {
|
||||
if (flag) return language === 'en' ? 'Yes' : '是';
|
||||
return language === 'en' ? 'No' : '否';
|
||||
};
|
||||
|
||||
const field = (label, value, extra = {}) => ({
|
||||
label,
|
||||
value: value === null || value === undefined || value === '' ? '-' : value,
|
||||
...extra,
|
||||
});
|
||||
|
||||
const createSentinelRows = (dataInfo, language, formatYmd) => {
|
||||
const metadata = dataInfo.metadata_json || {};
|
||||
const polarizationChannels = metadata.polarization_channels || metadata.manifest_polarizations;
|
||||
|
||||
return [
|
||||
field(language === 'en' ? 'Satellite:' : '卫星:', dataInfo.satellite),
|
||||
field(language === 'en' ? 'Satellite Family:' : '卫星系列:', dataInfo.satellite_family),
|
||||
field(language === 'en' ? 'Source Format:' : '源格式:', dataInfo.source_format),
|
||||
field(language === 'en' ? 'Imaging Date:' : '成像日期:', formatYmd(dataInfo.imaging_date)),
|
||||
field(language === 'en' ? 'Acquisition Start:' : '采集开始:', formatDateTime(dataInfo.acquisition_start_time_utc)),
|
||||
field(language === 'en' ? 'Acquisition Stop:' : '采集结束:', formatDateTime(dataInfo.acquisition_stop_time_utc)),
|
||||
field(language === 'en' ? 'Imaging Mode:' : '成像模式:', dataInfo.imaging_mode),
|
||||
field(language === 'en' ? 'Product Type:' : '产品类型:', dataInfo.product_type),
|
||||
field(language === 'en' ? 'Product Level:' : '产品级别:', dataInfo.product_level),
|
||||
field(language === 'en' ? 'Orbit Direction:' : '轨道方向:', dataInfo.orbit_direction),
|
||||
field(language === 'en' ? 'Relative Orbit:' : '相对轨道:', dataInfo.relative_orbit),
|
||||
field(language === 'en' ? 'Absolute Orbit:' : '绝对轨道:', dataInfo.absolute_orbit),
|
||||
field(language === 'en' ? 'Polarization:' : '极化方式:', dataInfo.polarization),
|
||||
field(language === 'en' ? 'Polarization Channels:' : '极化通道:', joinChannels(polarizationChannels)),
|
||||
field(language === 'en' ? 'Datatake:' : '数据采集号:', metadata.filename_datatake),
|
||||
field(language === 'en' ? 'Scene Center Lon:' : '场景中心经度:', dataInfo.scene_center_lon),
|
||||
field(language === 'en' ? 'Scene Center Lat:' : '场景中心纬度:', dataInfo.scene_center_lat),
|
||||
field(language === 'en' ? 'Product Unique ID:' : '产品唯一ID:', dataInfo.product_unique_id, {
|
||||
valueStyle: { wordBreak: 'break-all' },
|
||||
},
|
||||
{
|
||||
label: language === 'en' ? 'ENVI Processed:' : 'ENVI已处理:',
|
||||
value: dataInfo.is_envi_processed ? (language === 'en' ? 'Yes' : '是') : (language === 'en' ? 'No' : '否'),
|
||||
},
|
||||
}),
|
||||
field(language === 'en' ? 'Has Orbit:' : '有精轨:', yesNo(dataInfo.has_orbit_data, language)),
|
||||
field(language === 'en' ? 'Orbit File:' : '轨道文件:', dataInfo.orbit_file_path, {
|
||||
valueStyle: { wordBreak: 'break-all' },
|
||||
}),
|
||||
field(language === 'en' ? 'ENVI Processed:' : 'ENVI已处理:', yesNo(dataInfo.is_envi_processed, language)),
|
||||
];
|
||||
};
|
||||
|
||||
const createDefaultRows = (dataInfo, language, formatYmd) => [
|
||||
field(language === 'en' ? 'Satellite:' : '卫星:', dataInfo.satellite),
|
||||
field(language === 'en' ? 'Satellite Mode:' : '卫星模式:', dataInfo.satellite_mode),
|
||||
field(language === 'en' ? 'Receiving Station:' : '接收站:', dataInfo.receiving_station),
|
||||
field(language === 'en' ? 'Imaging Date:' : '成像日期:', formatYmd(dataInfo.imaging_date)),
|
||||
field(language === 'en' ? 'Imaging Mode:' : '成像模式:', dataInfo.imaging_mode),
|
||||
field(language === 'en' ? 'Orbit Circle:' : '轨道圈号:', dataInfo.orbit_circle),
|
||||
field(language === 'en' ? 'Scene Center Lon:' : '场景中心经度:', dataInfo.scene_center_lon),
|
||||
field(language === 'en' ? 'Scene Center Lat:' : '场景中心纬度:', dataInfo.scene_center_lat),
|
||||
field(language === 'en' ? 'Acquisition Time:' : '采集时间:', dataInfo.acquisition_time_utc),
|
||||
field(language === 'en' ? 'Product Type:' : '产品类型:', dataInfo.product_type),
|
||||
field(language === 'en' ? 'Polarization:' : '极化方式:', dataInfo.polarization),
|
||||
field(language === 'en' ? 'Product Level:' : '产品级别:', dataInfo.product_level),
|
||||
field(language === 'en' ? 'Product Unique ID:' : '产品唯一ID:', dataInfo.product_unique_id),
|
||||
field(language === 'en' ? 'Orbit Direction:' : '轨道方向:', dataInfo.orbit_direction),
|
||||
field(language === 'en' ? 'Has Orbit:' : '有精轨:', yesNo(dataInfo.has_orbit_data, language)),
|
||||
field(language === 'en' ? 'Orbit File:' : '轨道文件:', dataInfo.orbit_file_path, {
|
||||
valueStyle: { wordBreak: 'break-all' },
|
||||
}),
|
||||
field(language === 'en' ? 'ENVI Processed:' : 'ENVI已处理:', yesNo(dataInfo.is_envi_processed, language)),
|
||||
];
|
||||
|
||||
const createRows = (dataInfo, language, formatYmd) => {
|
||||
if ((dataInfo.satellite_family || '').toUpperCase() === 'S1') {
|
||||
return createSentinelRows(dataInfo, language, formatYmd);
|
||||
}
|
||||
return createDefaultRows(dataInfo, language, formatYmd);
|
||||
};
|
||||
|
||||
export default function DataInfoModal({
|
||||
visible,
|
||||
dataInfo,
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
buildDinsarEngineOptions,
|
||||
getDinsarEngineMeta,
|
||||
} from '../utils/dinsarEngines';
|
||||
import { formatSatelliteFamilyLabel, inferSatelliteFamilyFromResultLike } from '../utils/satelliteFamily';
|
||||
|
||||
const STATUS_TONE_MAP = {
|
||||
READY: 'ready',
|
||||
@@ -383,6 +384,7 @@ export default function DinsarCatalogPanel({
|
||||
{products.map((item) => {
|
||||
const tone = STATUS_TONE_MAP[item.status] || 'neutral';
|
||||
const engineMeta = getDinsarEngineMeta(item.engine_code);
|
||||
const satelliteFamily = inferSatelliteFamilyFromResultLike(item);
|
||||
return (
|
||||
<button
|
||||
key={item.id}
|
||||
@@ -396,6 +398,9 @@ export default function DinsarCatalogPanel({
|
||||
</div>
|
||||
<div className="dinsar-catalog-list-item-badges">
|
||||
<span className={`dinsar-engine-badge tone-${engineMeta.tone}`}>{engineMeta.shortLabel}</span>
|
||||
{satelliteFamily && (
|
||||
<span className="dinsar-engine-badge tone-unknown">{formatSatelliteFamilyLabel(satelliteFamily)}</span>
|
||||
)}
|
||||
<span>{formatDateTime(item.published_at)}</span>
|
||||
</div>
|
||||
<div className="dinsar-catalog-list-item-meta">
|
||||
@@ -434,6 +439,9 @@ export default function DinsarCatalogPanel({
|
||||
<div className="dinsar-catalog-empty error">{selectedProduct.error}</div>
|
||||
) : (
|
||||
<div className="dinsar-catalog-detail-body">
|
||||
{(() => {
|
||||
const satelliteFamily = inferSatelliteFamilyFromResultLike(selectedProduct?.profile || selectedProduct);
|
||||
return (
|
||||
<div className="dinsar-catalog-hero">
|
||||
<div className="dinsar-catalog-preview-frame">
|
||||
<img
|
||||
@@ -452,6 +460,11 @@ export default function DinsarCatalogPanel({
|
||||
<span className={`dinsar-engine-badge tone-${selectedProductEngine.tone}`}>
|
||||
{selectedProductEngine.shortLabel}
|
||||
</span>
|
||||
{satelliteFamily && (
|
||||
<span className="dinsar-engine-badge tone-unknown">
|
||||
{formatSatelliteFamilyLabel(satelliteFamily)}
|
||||
</span>
|
||||
)}
|
||||
<StatusPill label={selectedProduct.status || 'UNKNOWN'} tone={selectedStatusTone} />
|
||||
</div>
|
||||
</div>
|
||||
@@ -469,6 +482,8 @@ export default function DinsarCatalogPanel({
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})()}
|
||||
|
||||
<div className="dinsar-catalog-detail-grid">
|
||||
<div className="dinsar-catalog-section-card">
|
||||
|
||||
@@ -14,6 +14,7 @@ import { getLeftTabLabel } from '../../utils/appUiHelpers';
|
||||
import { PanelLoadingBody, PanelLoadingPanel } from './AppLoadingFallbacks';
|
||||
|
||||
const LazyDataMonitorPanel = lazy(() => import('../../DataMonitorPanel'));
|
||||
const LazyAssetInventoryPanel = lazy(() => import('../../AssetInventoryPanel'));
|
||||
const LazyDataCopierPanel = lazy(() => import('../../DataCopierPanel'));
|
||||
const LazyIDLAutomationPanel = lazy(() => import('../../IDLAutomationPanel'));
|
||||
const LazyHazardPointPanel = lazy(() => import('../../HazardPointPanel'));
|
||||
@@ -237,6 +238,17 @@ export default function AppSidePanel({
|
||||
</div>
|
||||
)}
|
||||
|
||||
{leftPanelTab === 'asset_inventory' && (
|
||||
<div className="panel-content" style={{ flex: '1 1 auto', padding: 0, overflow: 'auto' }}>
|
||||
<Suspense fallback={<PanelLoadingBody message="正在加载资产库存..." />}>
|
||||
<LazyAssetInventoryPanel
|
||||
readOnly={isReadOnlyUser}
|
||||
onTaskStart={taskPanel.onTaskStart}
|
||||
/>
|
||||
</Suspense>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{leftPanelTab === 'pairing' && (
|
||||
<Suspense fallback={<PanelLoadingPanel message="正在加载组网规划面板..." />}>
|
||||
<LazyPairingPanel
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { memo } from 'react';
|
||||
import { parseDatesFromName, formatYmd } from '../../utils/appUiHelpers';
|
||||
import { getDinsarEngineMeta } from '../../utils/dinsarEngines';
|
||||
import { formatSatelliteFamilyLabel, inferSatelliteFamilyFromResultLike } from '../../utils/satelliteFamily';
|
||||
|
||||
function truncateMiddle(value, maxLength = 28) {
|
||||
const text = String(value || '').trim();
|
||||
@@ -23,6 +24,7 @@ function DinsarResultRow({
|
||||
}) {
|
||||
const dates = showDates ? parseDatesFromName(result.name, (value) => formatYmd(value, language)) : null;
|
||||
const engineMeta = getDinsarEngineMeta(result.engine_code);
|
||||
const satelliteFamily = inferSatelliteFamilyFromResultLike(result);
|
||||
const hasTrace = !!(
|
||||
result.selection_strategy ||
|
||||
result.network_run_id ||
|
||||
@@ -45,6 +47,14 @@ function DinsarResultRow({
|
||||
>
|
||||
{engineMeta.shortLabel}
|
||||
</span>
|
||||
{satelliteFamily && (
|
||||
<span
|
||||
className="dinsar-engine-badge tone-unknown"
|
||||
title={language === 'en' ? 'Satellite family' : '卫星系列'}
|
||||
>
|
||||
{formatSatelliteFamilyLabel(satelliteFamily)}
|
||||
</span>
|
||||
)}
|
||||
{result.ai_score !== null && (
|
||||
<span
|
||||
className={`ai-score ${result.ai_score > 0.7 ? 'good' : (result.ai_score < 0.4 ? 'bad' : 'medium')}`}
|
||||
|
||||
@@ -145,7 +145,7 @@ export const LEFT_GROUP_SECTIONS = {
|
||||
};
|
||||
|
||||
export const LEFT_GROUP_TABS = {
|
||||
data: ['ingest', 'data', 'hazard'],
|
||||
data: ['ingest', 'asset_inventory', 'data', 'hazard'],
|
||||
production_planning: LEFT_GROUP_SECTIONS.production_planning.flatMap(section => section.tabs),
|
||||
production_management: [PRODUCTION_WORKSPACE_TAB],
|
||||
insar_analysis: LEFT_GROUP_SECTIONS.insar_analysis.flatMap(section => section.tabs),
|
||||
@@ -180,6 +180,7 @@ export const FULL_WIDTH_LEFT_TABS = new Set([
|
||||
|
||||
export const ADMIN_ONLY_TABS = new Set([
|
||||
'ingest',
|
||||
'asset_inventory',
|
||||
'pairing',
|
||||
'pairs',
|
||||
'ps_results',
|
||||
@@ -199,6 +200,7 @@ export const BATCH_API_MAX_PAGES = 200;
|
||||
|
||||
export const SATELLITE_GROUPS = [
|
||||
{ key: 'LT-1', label: 'LT-1', prefixes: ['LT1'] },
|
||||
{ key: 'S1', label: 'Sentinel-1', prefixes: ['S1'] },
|
||||
{ key: 'GF-3', label: 'GF-3', prefixes: ['GF3'] },
|
||||
];
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ import { normalizePagePayload } from '../utils/appHelpers';
|
||||
import { normalizeTaskStatus } from '../utils/appUiHelpers';
|
||||
import { DEFAULT_LIST_PAGE_SIZE } from '../config/appConstants';
|
||||
|
||||
const NON_BLOCKING_TASK_TYPES = new Set(['UNPACK_ARCHIVES', 'COPY_DATA']);
|
||||
const NON_BLOCKING_TASK_TYPES = new Set(['UNPACK_ARCHIVES', 'UNPACK_SENTINEL1', 'SCAN_ASSET_INVENTORY', 'COPY_DATA']);
|
||||
|
||||
export default function useDinsarOperations({
|
||||
onCleanupDinsarLayers,
|
||||
|
||||
@@ -2,7 +2,7 @@ import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import apiClient from '../api/client';
|
||||
import { normalizeTaskStatus } from '../utils/appUiHelpers';
|
||||
|
||||
const NON_BLOCKING_TASK_TYPES = new Set(['UNPACK_ARCHIVES', 'COPY_DATA']);
|
||||
const NON_BLOCKING_TASK_TYPES = new Set(['UNPACK_ARCHIVES', 'UNPACK_SENTINEL1', 'SCAN_ASSET_INVENTORY', 'COPY_DATA']);
|
||||
|
||||
const isTaskNonBlocking = (taskId, taskType, nonBlockingTaskIds = []) => (
|
||||
NON_BLOCKING_TASK_TYPES.has(String(taskType || '').toUpperCase())
|
||||
|
||||
@@ -59,6 +59,19 @@ export default function useRadarSearch({
|
||||
clearRadarSearchResults,
|
||||
clearRadarMapLayers,
|
||||
}) {
|
||||
const getSatelliteCatalog = () => {
|
||||
const satellites = useRadarStore.getState().radarSearchOptions?.satellite;
|
||||
return Array.isArray(satellites) ? satellites.filter(Boolean) : [];
|
||||
};
|
||||
|
||||
const getSatellitesForGroup = (groupKey, satellites = getSatelliteCatalog()) => {
|
||||
const group = SATELLITE_GROUPS.find((item) => item.key === groupKey);
|
||||
if (!group) return [];
|
||||
return satellites.filter((sat) =>
|
||||
group.prefixes.some((prefix) => String(sat || '').startsWith(prefix))
|
||||
);
|
||||
};
|
||||
|
||||
const fetchRadarImagingDates = useCallback(async () => {
|
||||
try {
|
||||
const response = await apiClient.get('/radar-data/imaging-dates');
|
||||
@@ -74,13 +87,31 @@ export default function useRadarSearch({
|
||||
try {
|
||||
setRadarSearchOptionsLoading(true);
|
||||
const params = {};
|
||||
if (Array.isArray(satelliteFilter) && satelliteFilter.length > 0) {
|
||||
params.satellite = satelliteFilter;
|
||||
const storeState = useRadarStore.getState();
|
||||
const satelliteCatalog = getSatelliteCatalog();
|
||||
const hasExplicitSatelliteFilter = Array.isArray(satelliteFilter);
|
||||
let resolvedSatelliteFilter = hasExplicitSatelliteFilter
|
||||
? satelliteFilter.filter(Boolean)
|
||||
: [];
|
||||
|
||||
if (!hasExplicitSatelliteFilter) {
|
||||
const groupKey = storeState.selectedSatelliteGroup;
|
||||
if (groupKey && groupKey !== 'all') {
|
||||
resolvedSatelliteFilter = getSatellitesForGroup(groupKey, satelliteCatalog);
|
||||
}
|
||||
}
|
||||
|
||||
if (resolvedSatelliteFilter.length > 0) {
|
||||
params.satellite = resolvedSatelliteFilter;
|
||||
}
|
||||
const response = await apiClient.get('/radar-data/search/options', { params });
|
||||
const payload = response?.data && typeof response.data === 'object' ? response.data : {};
|
||||
const payloadSatellites = Array.isArray(payload.satellite) ? payload.satellite.filter(Boolean) : [];
|
||||
const nextSatelliteCatalog = satelliteCatalog.length > payloadSatellites.length
|
||||
? satelliteCatalog
|
||||
: payloadSatellites;
|
||||
setRadarSearchOptions({
|
||||
satellite: Array.isArray(payload.satellite) ? payload.satellite : [],
|
||||
satellite: nextSatelliteCatalog,
|
||||
satellite_mode: Array.isArray(payload.satellite_mode) ? payload.satellite_mode : [],
|
||||
receiving_station: Array.isArray(payload.receiving_station) ? payload.receiving_station : [],
|
||||
imaging_mode: Array.isArray(payload.imaging_mode) ? payload.imaging_mode : [],
|
||||
@@ -119,14 +150,9 @@ export default function useRadarSearch({
|
||||
orbit_direction: '',
|
||||
}));
|
||||
if (groupKey === 'all') {
|
||||
fetchRadarSearchOptions();
|
||||
fetchRadarSearchOptions([]);
|
||||
} else {
|
||||
const group = SATELLITE_GROUPS.find((g) => g.key === groupKey);
|
||||
if (!group) return;
|
||||
const allSatellites = useRadarStore.getState().radarSearchOptions.satellite;
|
||||
const matched = allSatellites.filter((sat) =>
|
||||
group.prefixes.some((prefix) => sat.startsWith(prefix))
|
||||
);
|
||||
const matched = getSatellitesForGroup(groupKey);
|
||||
if (matched.length > 0) {
|
||||
fetchRadarSearchOptions(matched);
|
||||
}
|
||||
@@ -265,17 +291,11 @@ export default function useRadarSearch({
|
||||
const applyRadarSearch = useCallback(async () => {
|
||||
const draftWithSatelliteGroup = { ...radarSearchDraft };
|
||||
if (selectedSatelliteGroup && selectedSatelliteGroup !== 'all') {
|
||||
const group = SATELLITE_GROUPS.find((g) => g.key === selectedSatelliteGroup);
|
||||
if (group) {
|
||||
const allSatellites = useRadarStore.getState().radarSearchOptions.satellite;
|
||||
const matched = allSatellites.filter((sat) =>
|
||||
group.prefixes.some((prefix) => sat.startsWith(prefix))
|
||||
);
|
||||
const matched = getSatellitesForGroup(selectedSatelliteGroup);
|
||||
if (matched.length > 0) {
|
||||
draftWithSatelliteGroup.satellite = matched.join(',');
|
||||
}
|
||||
}
|
||||
}
|
||||
const normalizedCriteria = normalizeRadarSearchCriteria(draftWithSatelliteGroup, RADAR_SEARCH_DEFAULTS);
|
||||
const selectedRegionTreeId = getSelectedRegionTreeId(radarSearchRegionSelection);
|
||||
const hasUploadedFiles = !!(radarSearchFiles && radarSearchFiles.length > 0);
|
||||
@@ -349,7 +369,7 @@ export default function useRadarSearch({
|
||||
radarSearchRequestSeqRef.current += 1;
|
||||
setHasRadarSearched(false);
|
||||
clearRadarSearchResults({ limit: radarPagination.limit });
|
||||
fetchRadarSearchOptions();
|
||||
fetchRadarSearchOptions([]);
|
||||
addLog('info', '已清除检索条件,请点击"搜索"或"搜索全部"获取数据。');
|
||||
}, [
|
||||
radarPagination.limit, radarSearchRequestSeqRef,
|
||||
@@ -377,7 +397,7 @@ export default function useRadarSearch({
|
||||
setRadarSearchRegionError('');
|
||||
setRadarSearchAoiToken('');
|
||||
setSelectedSatelliteGroup('all');
|
||||
fetchRadarSearchOptions();
|
||||
fetchRadarSearchOptions([]);
|
||||
|
||||
setHasRadarSearched(true);
|
||||
setIsLoading(true);
|
||||
|
||||
@@ -20,8 +20,8 @@ export const usePairingStore = create((set) => ({
|
||||
spatial_baseline_max_meters: 3000,
|
||||
limit_footprint_center_distance: false,
|
||||
coverage_diversity_penalty: 0.3,
|
||||
require_same_imaging_mode: false,
|
||||
require_same_polarization: false,
|
||||
require_same_imaging_mode: true,
|
||||
require_same_polarization: true,
|
||||
aoi_overlap_threshold: 0,
|
||||
start_date: '',
|
||||
// === 新增字段 ===
|
||||
@@ -32,7 +32,7 @@ export const usePairingStore = create((set) => ({
|
||||
strategy: 'all',
|
||||
num_connections: 1,
|
||||
reference_image_id: null,
|
||||
allowed_satellites: ['LT1A', 'LT1B'],
|
||||
allowed_satellites: null,
|
||||
cross_satellite_pairing: false,
|
||||
},
|
||||
pairingAlert: { warnings: [], fallbackUsed: false },
|
||||
|
||||
@@ -33,6 +33,8 @@ export const getLeftTabLabel = (tabKey, metrics = {}) => {
|
||||
switch (tabKey) {
|
||||
case 'ingest':
|
||||
return '入库监控';
|
||||
case 'asset_inventory':
|
||||
return '资产库存';
|
||||
case 'data':
|
||||
return '数据列表';
|
||||
case 'hazard':
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
export function normalizeSatelliteFamily(value) {
|
||||
const raw = String(value || '').trim().toUpperCase();
|
||||
if (!raw) return '';
|
||||
const compact = raw.replace(/[-_\s]+/g, '');
|
||||
if (['LT1', 'LT1A', 'LT1B', 'LUTAN1', 'LUTAN1A', 'LUTAN1B'].includes(compact)) return 'LT1';
|
||||
if (['S1', 'S1A', 'S1B', 'S1C', 'SENTINEL1', 'SENTINEL1A', 'SENTINEL1B', 'SENTINEL1C'].includes(compact)) return 'S1';
|
||||
if (['GF3', 'GAOFEN3'].includes(compact)) return 'GF3';
|
||||
return raw;
|
||||
}
|
||||
|
||||
export function inferSatelliteFamilyFromResultLike(item) {
|
||||
const direct = normalizeSatelliteFamily(
|
||||
item?.satellite_family
|
||||
|| item?.master_satellite
|
||||
|| item?.slave_satellite
|
||||
|| item?.satellite
|
||||
);
|
||||
if (direct) return direct;
|
||||
|
||||
const pairKey = String(item?.pair_key || '').trim().toLowerCase();
|
||||
if (pairKey.startsWith('s1_')) return 'S1';
|
||||
if (pairKey.startsWith('lt1_')) return 'LT1';
|
||||
if (pairKey.startsWith('gf3_')) return 'GF3';
|
||||
return '';
|
||||
}
|
||||
|
||||
export function formatSatelliteFamilyLabel(value) {
|
||||
const family = normalizeSatelliteFamily(value);
|
||||
if (family === 'S1') return 'Sentinel-1';
|
||||
if (family === 'LT1') return 'LT-1';
|
||||
if (family === 'GF3') return 'GF3';
|
||||
return family || '-';
|
||||
}
|
||||
+5
-2
@@ -158,8 +158,11 @@ def main(argv):
|
||||
if not os.path.isdir(rslcDir): os.mkdir(rslcDir)
|
||||
ensure_master_rslc(projectName)
|
||||
|
||||
if 'S1' in projectName: cmd_command = 'coreg_s1_gamma.py'
|
||||
else: cmd_command = 'coreg_gamma.py'
|
||||
template_satellite = str(ut.update_template(templateDir + "/" + projectName + ".template").get('satelite', '') or '')
|
||||
if str(template_satellite).startswith('S1') or 'S1' in projectName:
|
||||
cmd_command = 'coreg_s1_gamma.py'
|
||||
else:
|
||||
cmd_command = 'coreg_gamma.py'
|
||||
|
||||
err_txt = scratchDir + '/' + projectName + '/coreg_gamma_all.err'
|
||||
if os.path.isfile(err_txt): os.remove(err_txt)
|
||||
|
||||
+2
-2
@@ -231,9 +231,9 @@ def main(argv):
|
||||
# if os.path.isfile(HGTSIM): os.remove(HGTSIM)
|
||||
else:
|
||||
print('The SLC has already coregister Done')
|
||||
sys.exit(1)
|
||||
sys.exit(0)
|
||||
print("Coregister TOP SLC image to the reference TOPS image is done !!")
|
||||
sys.exit(1)
|
||||
sys.exit(0)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(sys.argv[:])
|
||||
|
||||
+70
-23
@@ -14,6 +14,7 @@ import time
|
||||
import glob
|
||||
import argparse
|
||||
import csv
|
||||
from pathlib import Path
|
||||
|
||||
from pyint import _utils as ut
|
||||
|
||||
@@ -29,10 +30,11 @@ def get_s1_time(raw_file):
|
||||
return times
|
||||
|
||||
def get_satellite(raw_file):
|
||||
if 'S1A_IW_SLC_' in raw_file:
|
||||
s0 = 'A'
|
||||
name = os.path.basename(str(raw_file))
|
||||
if name.startswith('S1') and len(name) >= 3:
|
||||
s0 = name[2]
|
||||
else:
|
||||
s0 = 'B'
|
||||
s0 = 'A'
|
||||
|
||||
return s0
|
||||
|
||||
@@ -86,8 +88,8 @@ def main(argv):
|
||||
#if not os.path.isdir(opod_dir):
|
||||
# os.mkdir(opod_dir)
|
||||
|
||||
call_str = " eof --save-dir " + opod_dir + " -p " + down_dir
|
||||
os.system(call_str)
|
||||
if opod_dir and not os.path.isdir(opod_dir):
|
||||
os.makedirs(opod_dir, exist_ok=True)
|
||||
if not os.path.isdir(slc_dir):
|
||||
os.mkdir(slc_dir)
|
||||
|
||||
@@ -99,9 +101,6 @@ def main(argv):
|
||||
|
||||
t_date = 't_' + date
|
||||
|
||||
call_str = 'ls ' + down_dir + '/S1*' + date + '* > ' + t_date
|
||||
os.system(call_str)
|
||||
|
||||
start_swath = templateDict['start_swath']
|
||||
end_swath = templateDict['end_swath']
|
||||
|
||||
@@ -119,13 +118,23 @@ def main(argv):
|
||||
# k_swath = '-'
|
||||
k_swath = ut.get_sardata_swath(start_swath,end_swath)
|
||||
|
||||
raw_files = ut.read_txt2list(t_date)
|
||||
raw_files = sorted(glob.glob(down_dir + '/S1*' + date + '*.zip') + glob.glob(down_dir + '/S1*' + date + '*.SAFE'))
|
||||
if not raw_files:
|
||||
raise FileNotFoundError('No Sentinel-1 scenes found for date ' + date)
|
||||
with open(t_date, 'w', encoding='utf-8') as f:
|
||||
for raw_file in raw_files:
|
||||
f.write(str(raw_file) + '\n')
|
||||
satellite = get_satellite(str(raw_files[0]))
|
||||
#orbit_file = ut.download_s1_orbit(date,opod_dir,satellite=satellite)
|
||||
raw_file_list = glob.glob(down_dir + '/S1*' + date + '*.zip')
|
||||
raw_file_list = glob.glob(down_dir + '/S1*' + date + '*.zip') + glob.glob(down_dir + '/S1*' + date + '*.SAFE')
|
||||
file_num=len(raw_files)
|
||||
if raw_files and os.path.isdir(str(raw_files[0])):
|
||||
call_str = 'S1_BURST_tab_from_zipfile.py 3 --dir_ref_list ' + t_date + ' --dir_list ' + t_date
|
||||
else:
|
||||
call_str = 'S1_BURST_tab_from_zipfile.py 3 --zip_ref_list ' + t_date + ' --zip_list ' + t_date
|
||||
os.system(call_str)
|
||||
rc = os.system(call_str)
|
||||
if rc != 0:
|
||||
raise RuntimeError('S1_BURST_tab_from_zipfile.py failed with rc=' + str(rc))
|
||||
|
||||
for kk in range(file_num):
|
||||
zipfile_ref=str(raw_files[kk])
|
||||
@@ -135,22 +144,39 @@ def main(argv):
|
||||
#os.system(call_str)
|
||||
#call_str = 'S1_import_SLC_from_zipfiles ' + t_date + ' ' + burst_number_table_ref + ' vv 0 ' + k_swath + ' ' + opod_dir + ' 1 1 '
|
||||
call_str = 'read_S1_TOPS_SLC.py ' + zipfile_ref + ' --burst_sel ' + burst_number_table_ref + ' --pol vv --root_name ' + date + ' --sw_start ' + start_swath + ' --swn ' + end_swath + ' --OPOD_dir ' + opod_dir
|
||||
os.system(call_str)
|
||||
rc = os.system(call_str)
|
||||
if rc != 0:
|
||||
raise RuntimeError('read_S1_TOPS_SLC.py failed with rc=' + str(rc))
|
||||
os.chdir(work_dir)
|
||||
call_str = "rename 's/vv.iw1.slc/iw1_" + str(kk)+".slc/g' *"
|
||||
os.system(call_str)
|
||||
call_str = "rename 's/vv.iw2.slc/iw2_" + str(kk)+".slc/g' *"
|
||||
os.system(call_str)
|
||||
call_str = "rename 's/vv.iw3.slc/iw3_" + str(kk)+".slc/g' *"
|
||||
os.system(call_str)
|
||||
rename_rules = [
|
||||
('vv.iw1.slc', 'iw1_' + str(kk) + '.slc'),
|
||||
('vv.iw2.slc', 'iw2_' + str(kk) + '.slc'),
|
||||
('vv.iw3.slc', 'iw3_' + str(kk) + '.slc'),
|
||||
('vv.SLC_tab', 'SLC_tab'),
|
||||
('vv.slc', 'slc'),
|
||||
('tops_par', 'TOPS_par'),
|
||||
]
|
||||
for src_token, dst_token in rename_rules:
|
||||
for candidate in list(Path(work_dir).iterdir()):
|
||||
if src_token not in candidate.name:
|
||||
continue
|
||||
target = candidate.with_name(candidate.name.replace(src_token, dst_token))
|
||||
if target != candidate and not target.exists():
|
||||
candidate.rename(target)
|
||||
SLC_Tab = work_dir + '/' + date + '_SLC_Tab' +str(kk)
|
||||
SLC_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc'))
|
||||
SLC_par_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc.par'))
|
||||
TOP_par_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc.TOPS_par'))
|
||||
if not TOP_par_list:
|
||||
TOP_par_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc.tops_par'))
|
||||
if (not SLC_list or not SLC_par_list or not TOP_par_list
|
||||
or len(SLC_list) != len(SLC_par_list)
|
||||
or len(SLC_list) != len(TOP_par_list)):
|
||||
raise RuntimeError('Sentinel-1 concatenated SLC generation did not produce expected IW products for ' + date)
|
||||
|
||||
|
||||
cat_str = 'touch slc_list slc_par_list top_par_list'
|
||||
os.system(call_str)
|
||||
os.system(cat_str)
|
||||
list_num=len(SLC_list)
|
||||
for tt in range(list_num):
|
||||
call_str = 'echo ' + SLC_list[tt] + ' >> slc_list'
|
||||
@@ -160,7 +186,9 @@ def main(argv):
|
||||
call_str = 'echo ' + TOP_par_list[tt] + ' >> top_par_list'
|
||||
os.system(call_str)
|
||||
call_str = ' paste slc_list slc_par_list top_par_list > ' + SLC_Tab
|
||||
os.system(call_str)
|
||||
rc = os.system(call_str)
|
||||
if rc != 0:
|
||||
raise RuntimeError('paste SLC tab failed with rc=' + str(rc))
|
||||
call_str = 'rm slc_list slc_par_list top_par_list'
|
||||
os.system(call_str)
|
||||
SLC_list = sorted(glob.glob(work_dir + '/*iw*.slc'))
|
||||
@@ -188,23 +216,42 @@ def main(argv):
|
||||
SLC_list = sorted(glob.glob(work_dir + '/*IW*.slc'))
|
||||
SLC_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.par'))
|
||||
TOP_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.TOPS_par'))
|
||||
if not TOP_par_list:
|
||||
TOP_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.tops_par'))
|
||||
|
||||
#call_str = 'rm *iw* *vv.SLC_tab'
|
||||
#os.system(call_str)
|
||||
if os.path.isfile(SLC_Tab):
|
||||
os.remove(SLC_Tab)
|
||||
|
||||
if (not SLC_list or not SLC_par_list or not TOP_par_list
|
||||
or len(SLC_list) != len(SLC_par_list)
|
||||
or len(SLC_list) != len(TOP_par_list)):
|
||||
raise RuntimeError('Sentinel-1 concatenated final SLC products are incomplete for ' + date)
|
||||
|
||||
for kk in range(len(SLC_list)):
|
||||
call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' >> ' + SLC_Tab
|
||||
os.system(call_str)
|
||||
rc = os.system(call_str)
|
||||
if rc != 0:
|
||||
raise RuntimeError('failed to append concatenated SLC tab with rc=' + str(rc))
|
||||
|
||||
BURST = SLC_par_list[kk].replace('slc.par','burst.par')
|
||||
call_str = 'SLC_burst_corners ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' > ' +BURST
|
||||
os.system(call_str)
|
||||
rc = os.system(call_str)
|
||||
if rc != 0:
|
||||
raise RuntimeError('SLC_burst_corners failed with rc=' + str(rc))
|
||||
call_str = "echo 'SLC has already down' >down2slc.dat"
|
||||
os.system(call_str)
|
||||
|
||||
TSLC = work_dir + '/' + date + '.slc'
|
||||
TSLCPar = work_dir + '/' + date + '.slc.par'
|
||||
if not os.path.isfile(TSLC) or not os.path.isfile(TSLCPar):
|
||||
call_str = 'SLC_mosaic_ScanSAR ' + SLC_Tab + ' ' + TSLC + ' ' + TSLCPar + ' 10 2'
|
||||
rc = os.system(call_str)
|
||||
if rc != 0:
|
||||
raise RuntimeError('SLC_mosaic_ScanSAR failed with rc=' + str(rc))
|
||||
print("Down to SLC for %s is done! " % date)
|
||||
sys.exit(1)
|
||||
sys.exit(0)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(sys.argv[:])
|
||||
|
||||
+58
-25
@@ -13,6 +13,7 @@ import getopt
|
||||
import time
|
||||
import glob
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
from pyint import _utils as ut
|
||||
|
||||
@@ -23,10 +24,11 @@ def get_s1_date(raw_file):
|
||||
return date
|
||||
|
||||
def get_satellite(raw_file):
|
||||
if 'S1A_IW_SLC_' in raw_file:
|
||||
s0 = 'A'
|
||||
name = os.path.basename(str(raw_file))
|
||||
if name.startswith('S1') and len(name) >= 3:
|
||||
s0 = name[2]
|
||||
else:
|
||||
s0 = 'B'
|
||||
s0 = 'A'
|
||||
|
||||
return s0
|
||||
|
||||
@@ -88,16 +90,13 @@ def main(argv):
|
||||
# os.mkdir(opod_dir)
|
||||
|
||||
|
||||
call_str = " eof --save-dir " + opod_dir + " -p " + down_dir
|
||||
os.system(call_str)
|
||||
if opod_dir and not os.path.isdir(opod_dir):
|
||||
os.makedirs(opod_dir, exist_ok=True)
|
||||
|
||||
os.chdir(work_dir)
|
||||
|
||||
t_date = 't_' + date
|
||||
|
||||
call_str = 'ls ' + down_dir + '/S1*' + date + '*.zip > ' + t_date
|
||||
os.system(call_str)
|
||||
|
||||
start_swath = templateDict['start_swath']
|
||||
end_swath = templateDict['end_swath']
|
||||
|
||||
@@ -115,53 +114,87 @@ def main(argv):
|
||||
# k_swath = '-'
|
||||
k_swath = ut.get_sardata_swath(start_swath,end_swath)
|
||||
|
||||
raw_files = ut.read_txt2list(t_date)
|
||||
raw_files = sorted(glob.glob(down_dir + '/S1*' + date + '*.zip') + glob.glob(down_dir + '/S1*' + date + '*.SAFE'))
|
||||
if not raw_files:
|
||||
raise FileNotFoundError('No Sentinel-1 scenes found for date ' + date)
|
||||
with open(t_date, 'w', encoding='utf-8') as f:
|
||||
for raw_file in raw_files:
|
||||
f.write(str(raw_file) + '\n')
|
||||
satellite = get_satellite(str(raw_files[0]))
|
||||
#orbit_file = ut.download_s1_orbit(date,opod_dir,satellite=satellite)
|
||||
zipfile_ref=str(raw_files[0])
|
||||
outfile_name=zipfile_ref.split('/')[-1].split('.')[0]
|
||||
burst_number_table_ref=outfile_name + '.BURST_tab'
|
||||
|
||||
if raw_files and os.path.isdir(str(raw_files[0])):
|
||||
call_str = 'S1_BURST_tab_from_zipfile.py 3 --dir_ref_list ' + t_date + ' --dir_list ' + t_date
|
||||
else:
|
||||
call_str = 'S1_BURST_tab_from_zipfile.py 3 --zip_ref_list ' + t_date + ' --zip_list ' + t_date
|
||||
os.system(call_str)
|
||||
rc = os.system(call_str)
|
||||
if rc != 0:
|
||||
raise RuntimeError('S1_BURST_tab_from_zipfile.py failed with rc=' + str(rc))
|
||||
|
||||
# call_str = 'S1_import_SLC_from_zipfiles ' + t_date + ' ' + burst_number_table_ref + ' vv 0 ' + k_swath
|
||||
call_str = 'read_S1_TOPS_SLC.py ' + zipfile_ref + ' --burst_sel ' + burst_number_table_ref + ' --pol vv --root_name ' + date + ' --sw_start ' + start_swath + ' --swn ' + end_swath + ' --OPOD_dir ' + opod_dir
|
||||
os.system(call_str)
|
||||
rc = os.system(call_str)
|
||||
if rc != 0:
|
||||
raise RuntimeError('read_S1_TOPS_SLC.py failed with rc=' + str(rc))
|
||||
|
||||
os.chdir(work_dir)
|
||||
|
||||
call_str = "rename 's/vv.iw1.slc/IW1.slc/g' *"
|
||||
#call_str = "rename vv.slc.iw1 IW1.slc * "
|
||||
os.system(call_str)
|
||||
call_str = "rename 's/vv.iw2.slc/IW2.slc/g' *"
|
||||
#call_str = "rename vv.slc.iw2 IW2.slc * "
|
||||
os.system(call_str)
|
||||
call_str = "rename 's/vv.iw3.slc/IW3.slc/g' *"
|
||||
#call_str = "rename vv.slc.iw3 IW3.slc * "
|
||||
os.system(call_str)
|
||||
call_str = "rename 's/tops_par/TOPS_par/g' *.tops_par "
|
||||
os.system(call_str)
|
||||
rename_rules = [
|
||||
('vv.iw1.slc', 'IW1.slc'),
|
||||
('vv.iw2.slc', 'IW2.slc'),
|
||||
('vv.iw3.slc', 'IW3.slc'),
|
||||
('vv.SLC_tab', 'SLC_tab'),
|
||||
('vv.slc', 'slc'),
|
||||
('tops_par', 'TOPS_par'),
|
||||
]
|
||||
for src_token, dst_token in rename_rules:
|
||||
for candidate in list(Path(work_dir).iterdir()):
|
||||
if src_token not in candidate.name:
|
||||
continue
|
||||
target = candidate.with_name(candidate.name.replace(src_token, dst_token))
|
||||
if target != candidate and not target.exists():
|
||||
candidate.rename(target)
|
||||
|
||||
SLC_Tab = work_dir + '/' + date+'_SLC_Tab'
|
||||
SLC_list = sorted(glob.glob(work_dir + '/*IW*.slc'))
|
||||
SLC_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.par'))
|
||||
TOP_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.TOPS_par'))
|
||||
if not TOP_par_list:
|
||||
TOP_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.tops_par'))
|
||||
if (not SLC_list or not SLC_par_list or not TOP_par_list
|
||||
or len(SLC_list) != len(SLC_par_list)
|
||||
or len(SLC_list) != len(TOP_par_list)):
|
||||
raise RuntimeError('Sentinel-1 SLC generation did not produce expected IW SLC products for ' + date)
|
||||
|
||||
if os.path.isfile(SLC_Tab):
|
||||
os.remove(SLC_Tab)
|
||||
|
||||
for kk in range(len(SLC_list)):
|
||||
call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' >> ' + SLC_Tab
|
||||
os.system(call_str)
|
||||
rc = os.system(call_str)
|
||||
if rc != 0:
|
||||
raise RuntimeError('failed to append SLC tab with rc=' + str(rc))
|
||||
|
||||
BURST = SLC_par_list[kk].replace('slc.par','burst.par')
|
||||
call_str = 'SLC_burst_corners ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' > ' +BURST
|
||||
os.system(call_str)
|
||||
rc = os.system(call_str)
|
||||
if rc != 0:
|
||||
raise RuntimeError('SLC_burst_corners failed with rc=' + str(rc))
|
||||
call_str = "echo 'SLC has already down' >down2slc.dat"
|
||||
os.system(call_str)
|
||||
|
||||
TSLC = work_dir + '/' + date + '.slc'
|
||||
TSLCPar = work_dir + '/' + date + '.slc.par'
|
||||
if not os.path.isfile(TSLC) or not os.path.isfile(TSLCPar):
|
||||
call_str = 'SLC_mosaic_ScanSAR ' + SLC_Tab + ' ' + TSLC + ' ' + TSLCPar + ' 10 2'
|
||||
rc = os.system(call_str)
|
||||
if rc != 0:
|
||||
raise RuntimeError('SLC_mosaic_ScanSAR failed with rc=' + str(rc))
|
||||
print("Down to SLC for %s is done! " % date)
|
||||
sys.exit(1)
|
||||
sys.exit(0)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(sys.argv[:])
|
||||
|
||||
+25
-4
@@ -32,9 +32,27 @@ def work(data0):
|
||||
stderr = p.stderr
|
||||
|
||||
if type(stderr) == bytes:
|
||||
aa=stderr.decode("utf-8")
|
||||
aa=stderr.decode("utf-8", errors="replace")
|
||||
else:
|
||||
aa = stderr
|
||||
if type(stdout) == bytes:
|
||||
bb=stdout.decode("utf-8", errors="replace")
|
||||
else:
|
||||
bb = stdout
|
||||
if p.returncode != 0:
|
||||
detail_parts = []
|
||||
if bb:
|
||||
detail_parts.append(bb)
|
||||
if aa:
|
||||
detail_parts.append(aa)
|
||||
detail = '\n'.join(part.strip() for part in detail_parts if part and part.strip())
|
||||
str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n'
|
||||
with open(err_txt, 'a') as f:
|
||||
f.write(str0)
|
||||
if detail:
|
||||
f.write(detail)
|
||||
f.write('\n')
|
||||
raise RuntimeError(str0.strip() + ' failed with rc=' + str(p.returncode) + ('\n' + detail if detail else ''))
|
||||
|
||||
if aa:
|
||||
str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n'
|
||||
@@ -83,7 +101,7 @@ def main(argv):
|
||||
projectDir = scratchDir + '/' + projectName
|
||||
downDir = scratchDir + '/' + projectName + "/DOWNLOAD"
|
||||
slcDir = scratchDir + '/' + projectName + "/SLC"
|
||||
raw_file_list = glob.glob(downDir + '/S1*.zip')
|
||||
raw_file_list = glob.glob(downDir + '/S1*.zip') + glob.glob(downDir + '/S1*.SAFE')
|
||||
templateDir = os.getenv('TEMPLATEDIR')
|
||||
templateFile = templateDir + "/" + projectName + ".template"
|
||||
templateDict=ut.update_template(templateFile)
|
||||
@@ -139,12 +157,15 @@ def main(argv):
|
||||
k00 = 0
|
||||
if k00==0:
|
||||
data_para.append(data0)
|
||||
ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False)
|
||||
results = ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False)
|
||||
failures = [str(item) for item in results if isinstance(item, Exception)]
|
||||
if failures:
|
||||
raise RuntimeError('\n\n'.join(failures))
|
||||
os.chdir(downDir)
|
||||
print("Down to SLC for project %s is done! " % projectName)
|
||||
ut.print_process_time(start_time, time.time())
|
||||
|
||||
sys.exit(1)
|
||||
sys.exit(0)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(sys.argv[:])
|
||||
|
||||
+1
-1
@@ -291,7 +291,7 @@ def main(argv):
|
||||
iw3 = Sdate + '.IW3.slc*'
|
||||
call_str = 'rm '+ iw1 + ' ' + iw2 + ' '+ iw3
|
||||
os.system(call_str)
|
||||
sys.exit(1)
|
||||
sys.exit(0)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(sys.argv[:])
|
||||
|
||||
+30
-3
@@ -26,9 +26,27 @@ def work(data0):
|
||||
stderr = p.stderr
|
||||
|
||||
if type(stderr) == bytes:
|
||||
aa=stderr.decode("utf-8")
|
||||
aa=stderr.decode("utf-8", errors="replace")
|
||||
else:
|
||||
aa = stderr
|
||||
if type(stdout) == bytes:
|
||||
bb=stdout.decode("utf-8", errors="replace")
|
||||
else:
|
||||
bb = stdout
|
||||
if p.returncode != 0:
|
||||
detail_parts = []
|
||||
if bb:
|
||||
detail_parts.append(bb)
|
||||
if aa:
|
||||
detail_parts.append(aa)
|
||||
detail = '\n'.join(part.strip() for part in detail_parts if part and part.strip())
|
||||
str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n'
|
||||
with open(err_file, 'a') as f:
|
||||
f.write(str0)
|
||||
if detail:
|
||||
f.write(detail)
|
||||
f.write('\n')
|
||||
raise RuntimeError(str0.strip() + ' failed with rc=' + str(p.returncode) + ('\n' + detail if detail else ''))
|
||||
|
||||
if aa:
|
||||
str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n'
|
||||
@@ -75,21 +93,30 @@ def main(argv):
|
||||
projectDir = scratchDir + '/' + projectName
|
||||
slcDir = scratchDir + '/' + projectName + '/SLC'
|
||||
slc_list = [os.path.basename(fname) for fname in sorted(glob.glob(slcDir + '/*'))]
|
||||
templateDir = os.getenv('TEMPLATEDIR')
|
||||
templateFile = templateDir + "/" + projectName + ".template"
|
||||
templateDict=ut.update_template(templateFile)
|
||||
master_date = templateDict['masterDate']
|
||||
|
||||
err_txt = scratchDir + '/' + projectName + '/extract_s1_bursts_all.err'
|
||||
if os.path.isfile(err_txt): os.remove(err_txt)
|
||||
|
||||
data_para = []
|
||||
for i in range(len(slc_list)):
|
||||
if slc_list[i] == master_date:
|
||||
continue
|
||||
cmd0 = ['extract_s1_bursts.py',projectName,slc_list[i]]
|
||||
data0 = [cmd0,err_txt]
|
||||
data_para.append(data0)
|
||||
|
||||
ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False)
|
||||
results = ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False)
|
||||
failures = [str(item) for item in results if isinstance(item, Exception)]
|
||||
if failures:
|
||||
raise RuntimeError('\n\n'.join(failures))
|
||||
print("Extract TOPS bursts for project %s is done! " % projectName)
|
||||
ut.print_process_time(start_time, time.time())
|
||||
|
||||
sys.exit(1)
|
||||
sys.exit(0)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(sys.argv[:])
|
||||
|
||||
Vendored
+1
-2
@@ -83,7 +83,7 @@ def main(argv):
|
||||
wavelength = 0.0312283810417
|
||||
elif satelite == 'TSX':
|
||||
wavelength = 0.03106657823461874
|
||||
elif satelite == 'S1A':
|
||||
elif str(satelite).startswith('S1'):
|
||||
wavelength = 0.0554657647
|
||||
elif satelite == 'ALOS2':
|
||||
wavelength = 0.2424525
|
||||
@@ -106,4 +106,3 @@ def main(argv):
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(sys.argv[:])
|
||||
|
||||
|
||||
+1
-1
@@ -105,7 +105,7 @@ def main(argv):
|
||||
print("Generate differential interferograms for project %s is done! " % projectName)
|
||||
ut.print_process_time(start_time, time.time())
|
||||
|
||||
sys.exit(1)
|
||||
sys.exit(0)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(sys.argv[:])
|
||||
|
||||
Vendored
+1
-1
@@ -210,7 +210,7 @@ def main(argv):
|
||||
if templateDict['raw2slc_all'] == '1': # only for S1 data now
|
||||
print('Start to convert downloaded-raw data into SLC ...')
|
||||
print('Number of processor: %s' % str(templateDict['raw2slc_all_parallel']))
|
||||
if satelite=='S1A':
|
||||
if str(satelite).startswith('S1'):
|
||||
call_str = 'down2slc_sen_all.py ' + projectName + ' --parallel ' + templateDict['raw2slc_all_parallel']
|
||||
_run_or_raise(call_str, 'raw2slc_s1')
|
||||
elif satelite=='ALOS':
|
||||
|
||||
+1
-1
@@ -159,7 +159,7 @@ def main(argv):
|
||||
call_str = 'SLC_burst_corners ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' > ' +BURST
|
||||
os.system(call_str)
|
||||
print("Down to SLC for %s is done! " % date)
|
||||
sys.exit(1)
|
||||
sys.exit(0)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(sys.argv[:])
|
||||
|
||||
Reference in New Issue
Block a user