Add Sentinel-1 asset management and PyINT pipeline support
This commit is contained in:
@@ -167,6 +167,9 @@ class Settings(BaseSettings):
|
||||
DB_SCHEMA_RESET_CONFIRM: bool = False
|
||||
|
||||
UNPACK_SOURCE_DIRS: str = ""
|
||||
SOURCE_PRODUCT_DIRS: str = ""
|
||||
SENTINEL1_STORAGE_DIRS: str = ""
|
||||
ORBIT_SOURCE_DIRS: str = ""
|
||||
INSAR_STORAGE_DIRS: str = ""
|
||||
MONITOR_RADAR_DIRS: str = ""
|
||||
MONITOR_DINSAR_DIRS: str = ""
|
||||
@@ -859,6 +862,13 @@ def validate_runtime_config() -> dict[str, Any]:
|
||||
_check_path(label="SRTM_DEM_DIR", value=settings.SRTM_DEM_DIR, errors=errors, warnings=warnings, expect_file=False)
|
||||
_check_path(label="WATER_RESULTS_DIR", value=settings.WATER_RESULTS_DIR, errors=errors, warnings=warnings, expect_file=False)
|
||||
_check_path(label="MONITOR_ORBIT_DIR", value=settings.MONITOR_ORBIT_DIR, errors=errors, warnings=warnings, expect_file=False)
|
||||
for label, value in (
|
||||
("SOURCE_PRODUCT_DIRS", settings.SOURCE_PRODUCT_DIRS),
|
||||
("SENTINEL1_STORAGE_DIRS", settings.SENTINEL1_STORAGE_DIRS),
|
||||
("ORBIT_SOURCE_DIRS", settings.ORBIT_SOURCE_DIRS),
|
||||
):
|
||||
for item in split_env_paths(value):
|
||||
_check_path(label=label, value=item, errors=errors, warnings=warnings, expect_file=False)
|
||||
_check_path(label="ORBIT_POOL_ENVI", value=settings.ORBIT_POOL_ENVI, errors=errors, warnings=warnings, expect_file=False)
|
||||
_check_path(label="ORBIT_POOL_ISCE2", value=settings.ORBIT_POOL_ISCE2, errors=errors, warnings=warnings, expect_file=False)
|
||||
_check_path(label="RESULT_PUBLISH_ROOT", value=settings.RESULT_PUBLISH_ROOT, errors=errors, warnings=warnings, expect_file=False)
|
||||
|
||||
@@ -36,6 +36,7 @@ MIGRATION_FILES = [
|
||||
"007_timeseries_stack_plan_trace.sql",
|
||||
"008_timeseries_stack_plan_edges.sql",
|
||||
"009_raw_source_pairing_fields.sql",
|
||||
"010_source_orbit_asset_inventory.sql",
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ from typing import Any, Dict, List
|
||||
|
||||
from ..config import get_env_text, read_bool_env, settings
|
||||
from .base import DinsarEngine, EngineAvailability, EngineProfile, RunRequest, RunResult
|
||||
from ..utils import normalize_satellite_family
|
||||
from ..services.dinsar_naming import (
|
||||
PAIR_META_FILENAME,
|
||||
build_fallback_pair_key,
|
||||
@@ -728,7 +729,14 @@ class Isce2Engine(DinsarEngine):
|
||||
task_name = os.path.basename(os.path.normpath(task_dir))
|
||||
pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {}
|
||||
task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name
|
||||
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(task_alias, task_dir)
|
||||
satellite_family = normalize_satellite_family(
|
||||
pair_meta.get("master_satellite") or pair_meta.get("slave_satellite")
|
||||
)
|
||||
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(
|
||||
task_alias,
|
||||
task_dir,
|
||||
satellite_family=satellite_family,
|
||||
)
|
||||
pointer_path = os.path.join(
|
||||
settings.DINSAR_PRODUCT_DIR,
|
||||
pair_key,
|
||||
@@ -1085,7 +1093,14 @@ class Isce2Engine(DinsarEngine):
|
||||
task_name = os.path.basename(os.path.normpath(task_dir))
|
||||
pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {}
|
||||
task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name
|
||||
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(task_alias, task_dir)
|
||||
satellite_family = normalize_satellite_family(
|
||||
pair_meta.get("master_satellite") or pair_meta.get("slave_satellite")
|
||||
)
|
||||
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(
|
||||
task_alias,
|
||||
task_dir,
|
||||
satellite_family=satellite_family,
|
||||
)
|
||||
output_root = self._output_root or os.path.join(task_dir, "isce2_output")
|
||||
run_dir = managed_run_dir_override or os.path.normpath(
|
||||
os.path.join(output_root, pair_key, "runs", run_key)
|
||||
|
||||
@@ -43,6 +43,7 @@ from ..services.pyint_service import (
|
||||
REFLATTEN_MODEL_CHOICES,
|
||||
TARGET_GRID_SIZE_MAX_M,
|
||||
TARGET_GRID_SIZE_MIN_M,
|
||||
build_profile_project_name,
|
||||
build_project_name,
|
||||
calculate_dem_oversampling,
|
||||
calculate_looks_from_task_dir,
|
||||
@@ -290,13 +291,13 @@ class PyintEngine(DinsarEngine):
|
||||
)
|
||||
return _windows_path_to_wsl_mount(str(local_script))
|
||||
|
||||
def _pipeline_script_for_profile(self, profile: str) -> str:
|
||||
script_name = "run_s1_pyint_pipeline.py" if str(profile or "").strip() == "s1_gamma_dinsar" else "run_lt1_pyint_pipeline.py"
|
||||
local_script = Path(__file__).resolve().parent.parent / "pyint_pipeline" / script_name
|
||||
return _windows_path_to_wsl_mount(str(local_script))
|
||||
|
||||
def get_profiles(self) -> List[EngineProfile]:
|
||||
return [
|
||||
EngineProfile(
|
||||
code="lt1_gamma_dinsar",
|
||||
label="LT-1 Gamma D-InSAR",
|
||||
description="Use PyINT + Gamma in WSL for single-pair LT-1 D-InSAR processing.",
|
||||
params_schema={
|
||||
shared_schema = {
|
||||
"force": {
|
||||
"label": "强制重跑",
|
||||
"type": "boolean",
|
||||
@@ -459,7 +460,19 @@ class PyintEngine(DinsarEngine):
|
||||
"section": "Execution",
|
||||
"description": "关闭后不导出地理编码结果。",
|
||||
},
|
||||
},
|
||||
}
|
||||
return [
|
||||
EngineProfile(
|
||||
code="lt1_gamma_dinsar",
|
||||
label="LT-1 Gamma D-InSAR",
|
||||
description="Use PyINT + Gamma in WSL for single-pair LT-1 D-InSAR processing.",
|
||||
params_schema=shared_schema,
|
||||
),
|
||||
EngineProfile(
|
||||
code="s1_gamma_dinsar",
|
||||
label="Sentinel-1 Gamma D-InSAR",
|
||||
description="Use PyINT + Gamma in WSL for single-pair Sentinel-1 D-InSAR processing.",
|
||||
params_schema=shared_schema,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -694,7 +707,7 @@ class PyintEngine(DinsarEngine):
|
||||
error="PyINT is disabled.",
|
||||
)
|
||||
|
||||
if request.profile != "lt1_gamma_dinsar":
|
||||
if request.profile not in {"lt1_gamma_dinsar", "s1_gamma_dinsar"}:
|
||||
return RunResult(
|
||||
success=False,
|
||||
engine_code=self.engine_code,
|
||||
@@ -859,7 +872,11 @@ class PyintEngine(DinsarEngine):
|
||||
else os.path.join(run_dir, "native")
|
||||
)
|
||||
template_root = os.path.normpath(os.path.join(self._template_root, pair_key, run_key))
|
||||
project_name = build_project_name(pair_key, run_key)
|
||||
project_name = build_profile_project_name(
|
||||
task_identity.get("satellite_family"),
|
||||
pair_key,
|
||||
run_key,
|
||||
)
|
||||
project_dir = os.path.join(work_run_root, project_name)
|
||||
# Keep input assets outside the run root because the WSL pipeline may delete run_root on --force.
|
||||
input_assets_dir = os.path.join(self._work_root, pair_key, "input_assets", run_key)
|
||||
@@ -1103,7 +1120,7 @@ class PyintEngine(DinsarEngine):
|
||||
)
|
||||
|
||||
cmd_parts = [
|
||||
f"{quote_shell(self._python)} {quote_shell(self._pipeline_script)} {quote_shell(wsl_task_dir)}",
|
||||
f"{quote_shell(self._python)} {quote_shell(self._pipeline_script_for_profile(request.profile))} {quote_shell(wsl_task_dir)}",
|
||||
f"--project-dir {quote_shell(wsl_project_dir)}",
|
||||
f"--template-root {quote_shell(wsl_template_root)}",
|
||||
f"--output-dir {quote_shell(wsl_output_dir)}",
|
||||
@@ -1486,6 +1503,9 @@ class PyintEngine(DinsarEngine):
|
||||
|
||||
@staticmethod
|
||||
def _discover_archives(task_dir: str) -> Dict[str, List[str]]:
|
||||
from ..services.pyint_service import discover_lt1_archives
|
||||
from ..services.pyint_service import discover_lt1_archives, discover_s1_scene_sources, infer_task_identity
|
||||
|
||||
task_identity = infer_task_identity(task_dir)
|
||||
if str(task_identity.get("satellite_family") or "").strip().upper() == "S1":
|
||||
return discover_s1_scene_sources(task_dir)
|
||||
return discover_lt1_archives(task_dir)
|
||||
|
||||
@@ -26,6 +26,12 @@ from .orm import (
|
||||
ManagedRootORM,
|
||||
ScanCursorORM,
|
||||
PathInventoryORM,
|
||||
SourceProductAssetORM,
|
||||
OrbitAssetORM,
|
||||
SceneOrbitBindingORM,
|
||||
OrbitAssetDerivativeORM,
|
||||
AssetInventoryStateORM,
|
||||
AssetInventoryIssueORM,
|
||||
WorkflowDefORM,
|
||||
WorkflowRunORM,
|
||||
WorkflowStepORM,
|
||||
@@ -94,6 +100,8 @@ __all__ = [
|
||||
"TimeseriesStackPlanORM", "TimeseriesStackPlanItemORM", "TimeseriesStackPlanEdgeORM",
|
||||
"SystemTaskORM", "TaskLogORM", "SystemJobORM", "ScanStateORM",
|
||||
"ManagedRootORM", "ScanCursorORM", "PathInventoryORM",
|
||||
"SourceProductAssetORM", "OrbitAssetORM", "SceneOrbitBindingORM",
|
||||
"OrbitAssetDerivativeORM", "AssetInventoryStateORM", "AssetInventoryIssueORM",
|
||||
"WorkflowDefORM", "WorkflowRunORM", "WorkflowStepORM", "WorkflowArtifactORM",
|
||||
"SystemWorkerHeartbeatORM",
|
||||
"DinsarTaskBatchORM", "DinsarTaskItemORM",
|
||||
|
||||
@@ -41,6 +41,32 @@ class RadarDataORM(Base):
|
||||
product_unique_id = Column(String, nullable=True)
|
||||
satellite_family = Column(String, index=True, nullable=True)
|
||||
look_direction = Column(String, index=True, nullable=True)
|
||||
acquisition_start_time_utc = Column(DateTime, nullable=True, index=True)
|
||||
acquisition_stop_time_utc = Column(DateTime, nullable=True)
|
||||
absolute_orbit = Column(String, nullable=True, index=True)
|
||||
relative_orbit = Column(String, nullable=True, index=True)
|
||||
source_format = Column(String(32), nullable=True, index=True)
|
||||
source_product_ref_id = Column(
|
||||
Integer,
|
||||
ForeignKey("source_product_assets.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
index=True,
|
||||
)
|
||||
source_archive_asset_id = Column(
|
||||
Integer,
|
||||
ForeignKey("source_product_assets.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
index=True,
|
||||
)
|
||||
selected_orbit_asset_id = Column(
|
||||
Integer,
|
||||
ForeignKey("orbit_assets.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
index=True,
|
||||
)
|
||||
orbit_binding_status = Column(String(32), nullable=False, default="UNBOUND", server_default="UNBOUND", index=True)
|
||||
orbit_binding_reason = Column(Text, nullable=True)
|
||||
metadata_json = Column(JSON, nullable=True)
|
||||
geocoded_flag = Column(Boolean, nullable=True)
|
||||
insar_source_ready = Column(Boolean, nullable=False, default=False, server_default="false")
|
||||
insar_source_reason = Column(Text, nullable=True)
|
||||
@@ -63,6 +89,10 @@ class RadarDataORM(Base):
|
||||
preview_cache_updated_at = Column(DateTime, nullable=True)
|
||||
preview_cache_error = Column(Text, nullable=True)
|
||||
|
||||
source_product_asset = relationship("SourceProductAssetORM", foreign_keys=[source_product_ref_id])
|
||||
source_archive_asset = relationship("SourceProductAssetORM", foreign_keys=[source_archive_asset_id])
|
||||
selected_orbit_asset = relationship("OrbitAssetORM", foreign_keys=[selected_orbit_asset_id])
|
||||
|
||||
|
||||
class DinsarResultORM(Base):
|
||||
__tablename__ = 'dinsar_results'
|
||||
@@ -742,6 +772,11 @@ class ManagedRootORM(Base):
|
||||
back_populates="root",
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
asset_inventory_states = relationship(
|
||||
"AssetInventoryStateORM",
|
||||
back_populates="root",
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_managed_roots_role_enabled", "root_role", "enabled"),
|
||||
@@ -809,6 +844,215 @@ class PathInventoryORM(Base):
|
||||
)
|
||||
|
||||
|
||||
class SourceProductAssetORM(Base):
|
||||
__tablename__ = "source_product_assets"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
asset_uid = Column(String(128), unique=True, index=True, nullable=False)
|
||||
logical_product_uid = Column(String(128), index=True, nullable=True)
|
||||
satellite_family = Column(String(32), index=True, nullable=True)
|
||||
satellite = Column(String(32), index=True, nullable=True)
|
||||
source_format = Column(String(32), index=True, nullable=False)
|
||||
product_type = Column(String(64), nullable=True)
|
||||
product_level = Column(String(64), nullable=True)
|
||||
imaging_mode = Column(String(64), nullable=True)
|
||||
polarization = Column(String(64), nullable=True)
|
||||
absolute_orbit = Column(String(64), index=True, nullable=True)
|
||||
relative_orbit = Column(String(64), index=True, nullable=True)
|
||||
orbit_direction = Column(String(32), index=True, nullable=True)
|
||||
acquisition_start_time_utc = Column(DateTime, index=True, nullable=True)
|
||||
acquisition_stop_time_utc = Column(DateTime, nullable=True)
|
||||
imaging_date = Column(String(8), index=True, nullable=True)
|
||||
root_ref_id = Column(Integer, ForeignKey("managed_roots.id", ondelete="SET NULL"), index=True, nullable=True)
|
||||
root_path = Column(String, nullable=True)
|
||||
file_path = Column(String, unique=True, index=True, nullable=False)
|
||||
archive_path = Column(String, nullable=True)
|
||||
path_kind = Column(String(24), nullable=True)
|
||||
file_name = Column(String(255), nullable=True)
|
||||
file_stem = Column(String(255), nullable=True)
|
||||
file_ext = Column(String(32), nullable=True)
|
||||
size_bytes = Column(BigInteger, nullable=True)
|
||||
mtime_epoch = Column(Float, nullable=True)
|
||||
checksum_sha256 = Column(String(64), nullable=True)
|
||||
checksum_status = Column(String(32), nullable=False, default="NOT_COMPUTED", server_default="NOT_COMPUTED")
|
||||
parser_name = Column(String(64), nullable=True)
|
||||
parser_version = Column(String(32), nullable=True)
|
||||
parse_status = Column(String(32), nullable=False, default="PENDING", server_default="PENDING", index=True)
|
||||
parse_error = Column(Text, nullable=True)
|
||||
parsed_at = Column(DateTime, nullable=True)
|
||||
metadata_json = Column(JSON, nullable=True)
|
||||
is_active = Column(Boolean, nullable=False, default=True, server_default="true", index=True)
|
||||
missing_since = Column(DateTime, nullable=True)
|
||||
created_at = Column(DateTime, server_default=func.now(), nullable=False)
|
||||
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
|
||||
|
||||
root = relationship("ManagedRootORM")
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_source_product_assets_family_date", "satellite_family", "imaging_date"),
|
||||
Index("idx_source_product_assets_logical_product", "logical_product_uid"),
|
||||
Index("idx_source_product_assets_root_active", "root_ref_id", "is_active"),
|
||||
)
|
||||
|
||||
|
||||
class OrbitAssetORM(Base):
|
||||
__tablename__ = "orbit_assets"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
orbit_uid = Column(String(128), unique=True, index=True, nullable=False)
|
||||
satellite_family = Column(String(32), index=True, nullable=True)
|
||||
satellite = Column(String(32), index=True, nullable=True)
|
||||
orbit_type = Column(String(64), index=True, nullable=False)
|
||||
native_format = Column(String(32), index=True, nullable=False)
|
||||
quality_class = Column(String(32), index=True, nullable=False, default="unknown", server_default="unknown")
|
||||
root_ref_id = Column(Integer, ForeignKey("managed_roots.id", ondelete="SET NULL"), index=True, nullable=True)
|
||||
root_path = Column(String, nullable=True)
|
||||
file_path = Column(String, unique=True, index=True, nullable=False)
|
||||
file_name = Column(String(255), nullable=True)
|
||||
file_stem = Column(String(255), nullable=True)
|
||||
file_ext = Column(String(32), nullable=True)
|
||||
size_bytes = Column(BigInteger, nullable=True)
|
||||
mtime_epoch = Column(Float, nullable=True)
|
||||
checksum_sha256 = Column(String(64), nullable=True)
|
||||
checksum_status = Column(String(32), nullable=False, default="NOT_COMPUTED", server_default="NOT_COMPUTED")
|
||||
validity_start_time_utc = Column(DateTime, index=True, nullable=True)
|
||||
validity_stop_time_utc = Column(DateTime, index=True, nullable=True)
|
||||
generation_time_utc = Column(DateTime, nullable=True)
|
||||
published_time_utc = Column(DateTime, nullable=True)
|
||||
parser_name = Column(String(64), nullable=True)
|
||||
parser_version = Column(String(32), nullable=True)
|
||||
parse_status = Column(String(32), nullable=False, default="PENDING", server_default="PENDING", index=True)
|
||||
parse_error = Column(Text, nullable=True)
|
||||
parsed_at = Column(DateTime, nullable=True)
|
||||
metadata_json = Column(JSON, nullable=True)
|
||||
is_active = Column(Boolean, nullable=False, default=True, server_default="true", index=True)
|
||||
missing_since = Column(DateTime, nullable=True)
|
||||
created_at = Column(DateTime, server_default=func.now(), nullable=False)
|
||||
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
|
||||
|
||||
root = relationship("ManagedRootORM")
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_orbit_assets_family_sat_window", "satellite_family", "satellite", "validity_start_time_utc", "validity_stop_time_utc"),
|
||||
Index("idx_orbit_assets_root_active", "root_ref_id", "is_active"),
|
||||
)
|
||||
|
||||
|
||||
class SceneOrbitBindingORM(Base):
|
||||
__tablename__ = "scene_orbit_bindings"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
radar_data_id = Column(Integer, ForeignKey("radar_data.id", ondelete="CASCADE"), index=True, nullable=False)
|
||||
orbit_asset_id = Column(Integer, ForeignKey("orbit_assets.id", ondelete="CASCADE"), index=True, nullable=False)
|
||||
binding_role = Column(String(32), nullable=False, default="primary_orbit", server_default="primary_orbit")
|
||||
match_status = Column(String(32), nullable=False, default="CANDIDATE", server_default="CANDIDATE", index=True)
|
||||
selection_status = Column(String(32), nullable=False, default="CANDIDATE", server_default="CANDIDATE", index=True)
|
||||
selection_rank = Column(Integer, nullable=True)
|
||||
priority_score = Column(Float, nullable=True)
|
||||
coverage_margin_before_seconds = Column(Float, nullable=True)
|
||||
coverage_margin_after_seconds = Column(Float, nullable=True)
|
||||
match_rule_version = Column(String(64), nullable=True)
|
||||
match_reason = Column(Text, nullable=True)
|
||||
selected_at = Column(DateTime, nullable=True)
|
||||
metadata_json = Column(JSON, nullable=True)
|
||||
created_at = Column(DateTime, server_default=func.now(), nullable=False)
|
||||
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
|
||||
|
||||
radar_data = relationship("RadarDataORM")
|
||||
orbit_asset = relationship("OrbitAssetORM")
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("radar_data_id", "orbit_asset_id", "binding_role", name="uq_scene_orbit_binding_role"),
|
||||
Index("idx_scene_orbit_bindings_scene_selected", "radar_data_id", "selection_status"),
|
||||
)
|
||||
|
||||
|
||||
class OrbitAssetDerivativeORM(Base):
|
||||
__tablename__ = "orbit_asset_derivatives"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
orbit_asset_id = Column(Integer, ForeignKey("orbit_assets.id", ondelete="CASCADE"), index=True, nullable=False)
|
||||
engine_code = Column(String(32), index=True, nullable=False)
|
||||
derivative_format = Column(String(32), nullable=False)
|
||||
derivative_role = Column(String(64), nullable=True)
|
||||
pool_path = Column(String, index=True, nullable=False)
|
||||
size_bytes = Column(BigInteger, nullable=True)
|
||||
mtime_epoch = Column(Float, nullable=True)
|
||||
checksum_sha256 = Column(String(64), nullable=True)
|
||||
generation_status = Column(String(32), nullable=False, default="PENDING", server_default="PENDING", index=True)
|
||||
generation_error = Column(Text, nullable=True)
|
||||
generated_at = Column(DateTime, nullable=True)
|
||||
metadata_json = Column(JSON, nullable=True)
|
||||
created_at = Column(DateTime, server_default=func.now(), nullable=False)
|
||||
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
|
||||
|
||||
orbit_asset = relationship("OrbitAssetORM")
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("orbit_asset_id", "engine_code", "derivative_format", "pool_path", name="uq_orbit_asset_derivative_pool_path"),
|
||||
Index("idx_orbit_asset_derivatives_asset_engine", "orbit_asset_id", "engine_code"),
|
||||
)
|
||||
|
||||
|
||||
class AssetInventoryStateORM(Base):
|
||||
__tablename__ = "asset_inventory_states"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
root_ref_id = Column(Integer, ForeignKey("managed_roots.id", ondelete="CASCADE"), index=True, nullable=False)
|
||||
inventory_type = Column(String(32), index=True, nullable=False)
|
||||
root_path = Column(String, nullable=False)
|
||||
scan_mode = Column(String(32), nullable=False, default="file_pool", server_default="file_pool")
|
||||
status = Column(String(32), nullable=False, default="NEVER_SCANNED", server_default="NEVER_SCANNED", index=True)
|
||||
last_scan_started_at = Column(DateTime, nullable=True)
|
||||
last_scan_finished_at = Column(DateTime, nullable=True)
|
||||
last_seen_entry_count = Column(Integer, nullable=True)
|
||||
last_asset_count = Column(Integer, nullable=True)
|
||||
last_issue_count = Column(Integer, nullable=True)
|
||||
parser_version = Column(String(32), nullable=True)
|
||||
needs_rescan = Column(Boolean, nullable=False, default=True, server_default="true", index=True)
|
||||
last_error = Column(Text, nullable=True)
|
||||
metadata_json = Column(JSON, nullable=True)
|
||||
created_at = Column(DateTime, server_default=func.now(), nullable=False)
|
||||
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
|
||||
|
||||
root = relationship("ManagedRootORM", back_populates="asset_inventory_states")
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("root_ref_id", "inventory_type", name="uq_asset_inventory_state_root_type"),
|
||||
Index("idx_asset_inventory_states_type_status", "inventory_type", "status"),
|
||||
)
|
||||
|
||||
|
||||
class AssetInventoryIssueORM(Base):
|
||||
__tablename__ = "asset_inventory_issues"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
root_ref_id = Column(Integer, ForeignKey("managed_roots.id", ondelete="SET NULL"), index=True, nullable=True)
|
||||
inventory_type = Column(String(32), index=True, nullable=False)
|
||||
asset_ref_id = Column(Integer, ForeignKey("source_product_assets.id", ondelete="SET NULL"), index=True, nullable=True)
|
||||
radar_data_id = Column(Integer, ForeignKey("radar_data.id", ondelete="SET NULL"), index=True, nullable=True)
|
||||
orbit_asset_id = Column(Integer, ForeignKey("orbit_assets.id", ondelete="SET NULL"), index=True, nullable=True)
|
||||
severity = Column(String(16), nullable=False, default="warning", server_default="warning", index=True)
|
||||
issue_code = Column(String(64), index=True, nullable=False)
|
||||
issue_message = Column(Text, nullable=True)
|
||||
source_path = Column(String, nullable=True)
|
||||
status = Column(String(16), nullable=False, default="OPEN", server_default="OPEN", index=True)
|
||||
first_seen_at = Column(DateTime, server_default=func.now(), nullable=False)
|
||||
last_seen_at = Column(DateTime, server_default=func.now(), nullable=False)
|
||||
resolved_at = Column(DateTime, nullable=True)
|
||||
metadata_json = Column(JSON, nullable=True)
|
||||
|
||||
root = relationship("ManagedRootORM")
|
||||
source_asset = relationship("SourceProductAssetORM", foreign_keys=[asset_ref_id])
|
||||
radar_data = relationship("RadarDataORM")
|
||||
orbit_asset = relationship("OrbitAssetORM")
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_asset_inventory_issues_open", "status", "severity"),
|
||||
Index("idx_asset_inventory_issues_root_type", "root_ref_id", "inventory_type"),
|
||||
)
|
||||
|
||||
|
||||
class WorkflowDefORM(Base):
|
||||
"""Workflow definition (DAG template)."""
|
||||
__tablename__ = "workflow_defs"
|
||||
|
||||
@@ -192,6 +192,17 @@ class RadarData(BaseModel):
|
||||
product_unique_id: Optional[str] = None
|
||||
satellite_family: Optional[str] = None
|
||||
look_direction: Optional[str] = None
|
||||
acquisition_start_time_utc: Optional[datetime] = None
|
||||
acquisition_stop_time_utc: Optional[datetime] = None
|
||||
absolute_orbit: Optional[str] = None
|
||||
relative_orbit: Optional[str] = None
|
||||
source_format: Optional[str] = None
|
||||
source_product_ref_id: Optional[int] = None
|
||||
source_archive_asset_id: Optional[int] = None
|
||||
selected_orbit_asset_id: Optional[int] = None
|
||||
orbit_binding_status: str = "UNBOUND"
|
||||
orbit_binding_reason: Optional[str] = None
|
||||
metadata_json: Optional[Dict[str, Any]] = None
|
||||
geocoded_flag: Optional[bool] = None
|
||||
insar_source_ready: bool = False
|
||||
insar_source_reason: Optional[str] = None
|
||||
|
||||
@@ -0,0 +1,651 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
try:
|
||||
from .run_lt1_pyint_pipeline import (
|
||||
DEFAULT_DERAMP_MODE,
|
||||
DEFAULT_REFERENCE_MODE,
|
||||
DEFAULT_REFLATTEN_AZIMUTH_STEP,
|
||||
DEFAULT_REFLATTEN_COH_THRESHOLD,
|
||||
DEFAULT_REFLATTEN_FALLBACK_COH_THRESHOLD,
|
||||
DEFAULT_REFLATTEN_MODEL,
|
||||
DEFAULT_REFLATTEN_RANGE_STEP,
|
||||
assert_output_sanity,
|
||||
assert_required_outputs,
|
||||
calculate_dem_oversampling,
|
||||
collect_expected_outputs,
|
||||
collect_output_sanity_checks,
|
||||
collect_stage_error_logs,
|
||||
copy_native_outputs,
|
||||
ensure_directory,
|
||||
export_standard_products,
|
||||
format_gamma_number,
|
||||
hardlink_or_copy,
|
||||
inspect_prepared_dem_path,
|
||||
load_json_file,
|
||||
load_pair_meta,
|
||||
load_shell_environment,
|
||||
normalize_date_text,
|
||||
parse_args,
|
||||
require_task_layout,
|
||||
rerun_pair_product_stages,
|
||||
run_gamma_reflatten,
|
||||
run_logged,
|
||||
safe_rmtree,
|
||||
validate_unit_interval,
|
||||
write_ifgram_list,
|
||||
write_text,
|
||||
write_wrapper_scripts,
|
||||
)
|
||||
except ImportError:
|
||||
from run_lt1_pyint_pipeline import (
|
||||
DEFAULT_DERAMP_MODE,
|
||||
DEFAULT_REFERENCE_MODE,
|
||||
DEFAULT_REFLATTEN_AZIMUTH_STEP,
|
||||
DEFAULT_REFLATTEN_COH_THRESHOLD,
|
||||
DEFAULT_REFLATTEN_FALLBACK_COH_THRESHOLD,
|
||||
DEFAULT_REFLATTEN_MODEL,
|
||||
DEFAULT_REFLATTEN_RANGE_STEP,
|
||||
assert_output_sanity,
|
||||
assert_required_outputs,
|
||||
calculate_dem_oversampling,
|
||||
collect_expected_outputs,
|
||||
collect_output_sanity_checks,
|
||||
collect_stage_error_logs,
|
||||
copy_native_outputs,
|
||||
ensure_directory,
|
||||
export_standard_products,
|
||||
format_gamma_number,
|
||||
hardlink_or_copy,
|
||||
inspect_prepared_dem_path,
|
||||
load_json_file,
|
||||
load_pair_meta,
|
||||
load_shell_environment,
|
||||
normalize_date_text,
|
||||
parse_args,
|
||||
require_task_layout,
|
||||
rerun_pair_product_stages,
|
||||
run_gamma_reflatten,
|
||||
run_logged,
|
||||
safe_rmtree,
|
||||
validate_unit_interval,
|
||||
write_ifgram_list,
|
||||
write_text,
|
||||
write_wrapper_scripts,
|
||||
)
|
||||
|
||||
|
||||
_WINDOWS_DRIVE_RE = re.compile(r"^[A-Za-z]:[\\/]")
|
||||
|
||||
|
||||
def _normalize_runtime_path(value: Any) -> Path:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return Path()
|
||||
normalized = text.replace("\\", "/")
|
||||
if normalized.startswith("/mnt/") or normalized.startswith("/"):
|
||||
return Path(normalized).resolve()
|
||||
if _WINDOWS_DRIVE_RE.match(text):
|
||||
drive = text[0].lower()
|
||||
tail = text[2:].replace("\\", "/").lstrip("/")
|
||||
return Path(f"/mnt/{drive}/{tail}").resolve()
|
||||
return Path(normalized).resolve()
|
||||
|
||||
|
||||
def _build_s1_template_text(
|
||||
*,
|
||||
project_name: str,
|
||||
satellite: str,
|
||||
master_date: str,
|
||||
range_looks: int,
|
||||
azimuth_looks: int,
|
||||
target_grid_size_m: int,
|
||||
dem_lat_ovr: float,
|
||||
dem_lon_ovr: float,
|
||||
unwrap_coh_threshold: float,
|
||||
geo_interp: str,
|
||||
atmcor: bool,
|
||||
atmcor_use_for_disp: bool,
|
||||
reflatten: bool,
|
||||
reflatten_model: str,
|
||||
reflatten_coh_threshold: float,
|
||||
parallel_workers: int,
|
||||
unwrap: bool,
|
||||
geocode: bool,
|
||||
dem_mode: str,
|
||||
fabdem_root: str,
|
||||
prepared_dem_path: str,
|
||||
opentopo_dem_type: str,
|
||||
opentopo_api_key: str,
|
||||
) -> str:
|
||||
prepared_dem = inspect_prepared_dem_path(prepared_dem_path) if dem_mode == "prepared_file" else {}
|
||||
lines = [
|
||||
f"# Auto-generated for {project_name}",
|
||||
f"satelite={satellite}",
|
||||
f"masterDate={master_date}",
|
||||
f"range_looks={int(range_looks)}",
|
||||
f"azimuth_looks={int(azimuth_looks)}",
|
||||
f"target_grid_size_m={int(target_grid_size_m or 0)}",
|
||||
f"dem_lat_ovr={format_gamma_number(dem_lat_ovr)}",
|
||||
f"dem_lon_ovr={format_gamma_number(dem_lon_ovr)}",
|
||||
"download_data=0",
|
||||
"raw2slc_all=1",
|
||||
f"raw2slc_all_parallel={int(parallel_workers)}",
|
||||
"extract_burst_all=1",
|
||||
f"extract_all_parallel={int(parallel_workers)}",
|
||||
"coreg_all=1",
|
||||
f"coreg_all_parallel={int(parallel_workers)}",
|
||||
"select_pairs=0",
|
||||
"diff_all=1",
|
||||
f"diff_all_parallel={int(parallel_workers)}",
|
||||
"pot_all=0",
|
||||
f"pot_all_parallel={int(parallel_workers)}",
|
||||
f"unwrap_all={1 if unwrap else 0}",
|
||||
f"unwrap_all_parallel={int(parallel_workers)}",
|
||||
f"unwrapThreshold={format_gamma_number(unwrap_coh_threshold)}",
|
||||
"make_mask=1",
|
||||
"auto_unw=1",
|
||||
"r_refer=-",
|
||||
"a_refer=-",
|
||||
f"atmcor_all={1 if atmcor else 0}",
|
||||
f"atmcor_all_parallel={int(parallel_workers)}",
|
||||
f"atmcor_use_for_disp={1 if (atmcor and atmcor_use_for_disp) else 0}",
|
||||
f"reflatten={1 if reflatten else 0}",
|
||||
f"reflatten_model={str(reflatten_model or DEFAULT_REFLATTEN_MODEL).strip().lower()}",
|
||||
f"reflatten_coh_threshold={format_gamma_number(reflatten_coh_threshold)}",
|
||||
f"geocode_all={1 if geocode else 0}",
|
||||
f"geocode_all_parallel={int(parallel_workers)}",
|
||||
f"geo_interp={str(geo_interp or '0').strip()}",
|
||||
"gacos_correction=0",
|
||||
"load_data=0",
|
||||
"geocode_products=hyp3,licsbas",
|
||||
"start_swath=1",
|
||||
"end_swath=3",
|
||||
"start_burst=1",
|
||||
"end_burst=20",
|
||||
]
|
||||
if dem_mode == "local_fabdem" and fabdem_root:
|
||||
lines.append(f"fabdem_dir={fabdem_root}")
|
||||
else:
|
||||
lines.append("fabdem_dir=-")
|
||||
if dem_mode == "prepared_file" and prepared_dem.get("kind") == "gamma_ready":
|
||||
lines.append(f"DEM={prepared_dem['direct_dem_path']}")
|
||||
if dem_mode == "prepared_file" and prepared_dem.get("kind") == "source_dem":
|
||||
lines.append(f"prepared_dem_source={prepared_dem['source_dem_path']}")
|
||||
else:
|
||||
lines.append("prepared_dem_source=-")
|
||||
if dem_mode == "opentopo":
|
||||
lines.append(f"opentopo_dem_type={opentopo_dem_type or 'SRTMGL1'}")
|
||||
lines.append(f"opentopo_api_key={opentopo_api_key or '-'}")
|
||||
else:
|
||||
lines.append("opentopo_dem_type=-")
|
||||
lines.append("opentopo_api_key=-")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def _resolve_s1_inputs(input_assets_payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
task_source = input_assets_payload.get("task_source") or {}
|
||||
prod = task_source.get("production_inputs") or {}
|
||||
orbits = input_assets_payload.get("orbits") or {}
|
||||
master_scene = _normalize_runtime_path(
|
||||
((prod.get("master_scene") or {}).get("staged_path") or (prod.get("master_scene") or {}).get("path") or (prod.get("master_zip") or {}).get("staged_path") or (prod.get("master_zip") or {}).get("path") or "")
|
||||
)
|
||||
slave_scene = _normalize_runtime_path(
|
||||
((prod.get("slave_scene") or {}).get("staged_path") or (prod.get("slave_scene") or {}).get("path") or (prod.get("slave_zip") or {}).get("staged_path") or (prod.get("slave_zip") or {}).get("path") or "")
|
||||
)
|
||||
master_eof = _normalize_runtime_path(
|
||||
((orbits.get("master") or {}).get("staged_path") or (orbits.get("master") or {}).get("path") or "")
|
||||
)
|
||||
slave_eof = _normalize_runtime_path(
|
||||
((orbits.get("slave") or {}).get("staged_path") or (orbits.get("slave") or {}).get("path") or "")
|
||||
)
|
||||
return {
|
||||
"master_scene": master_scene,
|
||||
"slave_scene": slave_scene,
|
||||
"master_eof": master_eof,
|
||||
"slave_eof": slave_eof,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
|
||||
task_dir = Path(args.task_dir).resolve()
|
||||
project_dir = Path(args.project_dir).resolve()
|
||||
run_root = project_dir.parent
|
||||
template_root = Path(args.template_root).resolve()
|
||||
output_dir = Path(args.output_dir).resolve()
|
||||
pyint_home = Path(args.pyint_home).resolve()
|
||||
pyint_app_script = Path(args.pyint_app_script).resolve()
|
||||
dem_root = Path(args.dem_root).resolve()
|
||||
input_assets_dir = Path(args.input_assets_dir).resolve() if args.input_assets_dir else None
|
||||
input_assets_json = Path(args.input_assets_json).resolve() if args.input_assets_json else None
|
||||
input_assets_payload = load_json_file(input_assets_json)
|
||||
dem_mode = str(args.dem_mode or "local_fabdem").strip().lower() or "local_fabdem"
|
||||
prepared_dem_info = inspect_prepared_dem_path(args.prepared_dem_path) if dem_mode == "prepared_file" else {}
|
||||
dem_oversampling = calculate_dem_oversampling(
|
||||
dem_resolution_m=float(args.dem_resolution_m),
|
||||
target_grid_size_m=float(args.target_grid_size_m or 0),
|
||||
dem_lat_ovr=float(args.dem_lat_ovr or 0.0),
|
||||
dem_lon_ovr=float(args.dem_lon_ovr or 0.0),
|
||||
)
|
||||
unwrap_coh_threshold = validate_unit_interval(args.unwrap_coh_threshold, "--unwrap-coh-threshold")
|
||||
coherence_mask_threshold = validate_unit_interval(args.coherence_mask_threshold, "--coherence-mask-threshold")
|
||||
reference_mode = str(args.reference_mode or DEFAULT_REFERENCE_MODE).strip().lower() or DEFAULT_REFERENCE_MODE
|
||||
deramp_mode = str(args.deramp_mode or DEFAULT_DERAMP_MODE).strip().lower() or DEFAULT_DERAMP_MODE
|
||||
reference_coh_threshold = validate_unit_interval(args.reference_coh_threshold, "--reference-coh-threshold")
|
||||
deramp_coh_threshold = validate_unit_interval(args.deramp_coh_threshold, "--deramp-coh-threshold")
|
||||
geo_interp = str(args.geo_interp or "1").strip()
|
||||
if geo_interp not in {"0", "1"}:
|
||||
raise ValueError("--geo-interp must be 0 or 1.")
|
||||
gamma_nodata_value = float(args.gamma_nodata_value)
|
||||
if not math.isfinite(gamma_nodata_value):
|
||||
raise ValueError("--gamma-nodata-value must be a finite number.")
|
||||
reflatten_model = str(args.reflatten_model or DEFAULT_REFLATTEN_MODEL).strip().lower()
|
||||
if reflatten_model == "linear":
|
||||
reflatten_model = "plane"
|
||||
if reflatten_model not in {"plane", "quadratic"}:
|
||||
raise ValueError("--reflatten-model must be plane or quadratic.")
|
||||
reflatten_coh_threshold = validate_unit_interval(args.reflatten_coh_threshold, "--reflatten-coh-threshold")
|
||||
reflatten_fallback_coh_threshold = validate_unit_interval(
|
||||
args.reflatten_fallback_coh_threshold,
|
||||
"--reflatten-fallback-coh-threshold",
|
||||
)
|
||||
reflatten_range_step = max(1, int(args.reflatten_range_step or DEFAULT_REFLATTEN_RANGE_STEP))
|
||||
reflatten_azimuth_step = max(1, int(args.reflatten_azimuth_step or DEFAULT_REFLATTEN_AZIMUTH_STEP))
|
||||
|
||||
require_task_layout(task_dir)
|
||||
if not pyint_app_script.is_file():
|
||||
raise FileNotFoundError(f"pyintApp.py not found: {pyint_app_script}")
|
||||
if not input_assets_json or not input_assets_json.is_file():
|
||||
raise RuntimeError("Sentinel-1 PyINT runner requires --input-assets-json.")
|
||||
if dem_mode == "prepared_file" and not prepared_dem_info.get("kind"):
|
||||
raise RuntimeError(
|
||||
"Prepared DEM mode requires either a Gamma DEM with .par, "
|
||||
"or a source DEM with .xml/.hdr/.vrt sidecars."
|
||||
)
|
||||
|
||||
if args.force:
|
||||
safe_rmtree(run_root)
|
||||
safe_rmtree(template_root)
|
||||
safe_rmtree(output_dir)
|
||||
|
||||
if run_root.exists():
|
||||
raise RuntimeError(f"PyINT run root already exists, rerun with --force: {run_root}")
|
||||
|
||||
pair_meta = load_pair_meta(task_dir)
|
||||
s1_inputs = _resolve_s1_inputs(input_assets_payload)
|
||||
master_scene = s1_inputs["master_scene"]
|
||||
slave_scene = s1_inputs["slave_scene"]
|
||||
master_eof = s1_inputs["master_eof"]
|
||||
slave_eof = s1_inputs["slave_eof"]
|
||||
if not (master_scene.is_file() or master_scene.is_dir()):
|
||||
raise FileNotFoundError(f"Sentinel-1 master scene not found: {master_scene}")
|
||||
if not (slave_scene.is_file() or slave_scene.is_dir()):
|
||||
raise FileNotFoundError(f"Sentinel-1 slave scene not found: {slave_scene}")
|
||||
if not master_eof.is_file():
|
||||
raise FileNotFoundError(f"Sentinel-1 master EOF not found: {master_eof}")
|
||||
if not slave_eof.is_file():
|
||||
raise FileNotFoundError(f"Sentinel-1 slave EOF not found: {slave_eof}")
|
||||
|
||||
master_date = normalize_date_text(args.master_date) or normalize_date_text(pair_meta.get("master_imaging_date"))
|
||||
slave_date = normalize_date_text(args.slave_date) or normalize_date_text(pair_meta.get("slave_imaging_date"))
|
||||
if not master_date or not slave_date:
|
||||
raise RuntimeError("Unable to determine master/slave dates from pair metadata.")
|
||||
|
||||
pair_name = f"{master_date}-{slave_date}"
|
||||
task_alias = str(args.task_alias or pair_meta.get("task_alias") or task_dir.name).strip() or task_dir.name
|
||||
pair_key = str(args.pair_key or pair_meta.get("pair_key") or "").strip()
|
||||
time_baseline_days = int(args.time_baseline_days or pair_meta.get("time_baseline_days") or 0)
|
||||
satellite = str(pair_meta.get("master_satellite") or "S1A").strip().upper() or "S1A"
|
||||
|
||||
ensure_directory(run_root)
|
||||
ensure_directory(template_root)
|
||||
ensure_directory(output_dir)
|
||||
ensure_directory(dem_root)
|
||||
|
||||
pyint_scripts_dir = pyint_home / "pyint"
|
||||
wrappers_dir = ensure_directory(run_root / "wrappers")
|
||||
write_wrapper_scripts(
|
||||
wrappers_dir=wrappers_dir,
|
||||
pyint_home=pyint_home,
|
||||
python_cmd=args.python,
|
||||
gamma_env_script=args.gamma_env_script,
|
||||
)
|
||||
for shim_name in ("python", "python3"):
|
||||
shim_path = wrappers_dir / shim_name
|
||||
write_text(
|
||||
shim_path,
|
||||
"\n".join(
|
||||
[
|
||||
"#!/usr/bin/env bash",
|
||||
f"exec '{args.python}' \"$@\"",
|
||||
"",
|
||||
]
|
||||
),
|
||||
)
|
||||
shim_path.chmod(shim_path.stat().st_mode | 0o111)
|
||||
|
||||
template_path = write_text(
|
||||
template_root / f"{args.project_name}.template",
|
||||
_build_s1_template_text(
|
||||
project_name=args.project_name,
|
||||
satellite=satellite,
|
||||
master_date=master_date,
|
||||
range_looks=args.range_looks,
|
||||
azimuth_looks=args.azimuth_looks,
|
||||
target_grid_size_m=int(args.target_grid_size_m or 0),
|
||||
dem_lat_ovr=dem_oversampling["dem_lat_ovr"],
|
||||
dem_lon_ovr=dem_oversampling["dem_lon_ovr"],
|
||||
unwrap_coh_threshold=unwrap_coh_threshold,
|
||||
geo_interp=geo_interp,
|
||||
atmcor=bool(args.atmcor),
|
||||
atmcor_use_for_disp=bool(args.atmcor_use_for_disp),
|
||||
reflatten=bool(args.reflatten),
|
||||
reflatten_model=reflatten_model,
|
||||
reflatten_coh_threshold=reflatten_coh_threshold,
|
||||
parallel_workers=args.parallel_workers,
|
||||
unwrap=bool(args.unwrap),
|
||||
geocode=bool(args.geocode),
|
||||
dem_mode=dem_mode,
|
||||
fabdem_root=str(args.fabdem_root or "").strip(),
|
||||
prepared_dem_path=str(args.prepared_dem_path or "").strip(),
|
||||
opentopo_dem_type=str(args.opentopo_dem_type or "SRTMGL1").strip(),
|
||||
opentopo_api_key=str(args.opentopo_api_key or "").strip(),
|
||||
),
|
||||
)
|
||||
|
||||
scratch_root = ensure_directory(project_dir.parent)
|
||||
archive_materialization: List[Dict[str, str]] = []
|
||||
env = os.environ.copy()
|
||||
if args.gamma_env_script:
|
||||
env.update(load_shell_environment(args.gamma_env_script, env))
|
||||
opod_dir = project_dir / "OPOD"
|
||||
env.update(
|
||||
{
|
||||
"SCRATCHDIR": str(scratch_root),
|
||||
"TEMPLATEDIR": str(template_root),
|
||||
"DEMDIR": str(dem_root),
|
||||
"OPOD_DIR": str(opod_dir),
|
||||
"PATH": f"{wrappers_dir}:{pyint_scripts_dir}:{env.get('PATH', '')}",
|
||||
"PYTHONPATH": f"{pyint_home}:{env.get('PYTHONPATH', '')}",
|
||||
"PYINT_LT1_PRECISE_ORBIT_ENABLED": "false",
|
||||
}
|
||||
)
|
||||
|
||||
generate_stdout = run_root / "pyint_generate.stdout.log"
|
||||
generate_stderr = run_root / "pyint_generate.stderr.log"
|
||||
generate_result = run_logged(
|
||||
[str(wrappers_dir / "pyintApp.py"), "-g", args.project_name],
|
||||
env=env,
|
||||
cwd=scratch_root,
|
||||
stdout_path=generate_stdout,
|
||||
stderr_path=generate_stderr,
|
||||
)
|
||||
if generate_result.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"pyintApp.py -g failed with rc={generate_result.returncode}: "
|
||||
f"{(generate_result.stderr or generate_result.stdout or '').strip()}"
|
||||
)
|
||||
|
||||
pyint_project_dir = project_dir
|
||||
ensure_directory(pyint_project_dir)
|
||||
opod_dir = ensure_directory(opod_dir)
|
||||
download_dir = ensure_directory(pyint_project_dir / "DOWNLOAD")
|
||||
ifgram_list_path = write_ifgram_list(pyint_project_dir / "ifgram_list.txt", master_date, slave_date, time_baseline_days)
|
||||
for role, src_path in (("master", master_scene), ("slave", slave_scene)):
|
||||
target_path = download_dir / src_path.name
|
||||
if src_path.is_dir():
|
||||
if target_path.exists():
|
||||
if target_path.is_dir():
|
||||
safe_rmtree(target_path)
|
||||
else:
|
||||
target_path.unlink()
|
||||
shutil.copytree(src_path, target_path)
|
||||
op = "copied_dir"
|
||||
else:
|
||||
op = hardlink_or_copy(src_path, target_path)
|
||||
archive_materialization.append(
|
||||
{
|
||||
"role": role,
|
||||
"source": str(src_path),
|
||||
"target": str(target_path),
|
||||
"operation": op,
|
||||
}
|
||||
)
|
||||
for role, src_path in (("master_orbit", master_eof), ("slave_orbit", slave_eof)):
|
||||
target_path = opod_dir / src_path.name
|
||||
op = hardlink_or_copy(src_path, target_path)
|
||||
archive_materialization.append(
|
||||
{
|
||||
"role": role,
|
||||
"source": str(src_path),
|
||||
"target": str(target_path),
|
||||
"operation": op,
|
||||
}
|
||||
)
|
||||
|
||||
run_stdout = run_root / "pyint.stdout.log"
|
||||
run_stderr = run_root / "pyint.stderr.log"
|
||||
run_started_at = datetime.utcnow().isoformat(timespec="seconds") + "Z"
|
||||
run_result = run_logged(
|
||||
[str(wrappers_dir / "pyintApp.py"), args.project_name],
|
||||
env=env,
|
||||
cwd=scratch_root,
|
||||
stdout_path=run_stdout,
|
||||
stderr_path=run_stderr,
|
||||
)
|
||||
if run_result.returncode != 0:
|
||||
stage_error_logs = collect_stage_error_logs(pyint_project_dir)
|
||||
detail_text = (run_result.stderr or run_result.stdout or "").strip()
|
||||
if stage_error_logs:
|
||||
log_text = ", ".join(f"{name}={path}" for name, path in stage_error_logs.items())
|
||||
detail_text = f"{detail_text}\nStage logs: {log_text}" if detail_text else f"Stage logs: {log_text}"
|
||||
raise RuntimeError(f"pyintApp.py failed with rc={run_result.returncode}: {detail_text}")
|
||||
|
||||
repair_summary: Dict[str, Any] = {
|
||||
"attempted": False,
|
||||
"attempt_count": 0,
|
||||
"max_attempts": 1,
|
||||
}
|
||||
outputs = collect_expected_outputs(pyint_project_dir, pair_name, args.range_looks)
|
||||
try:
|
||||
assert_required_outputs(outputs, unwrap=bool(args.unwrap), geocode=bool(args.geocode))
|
||||
output_sanity_checks = collect_output_sanity_checks(outputs, unwrap=bool(args.unwrap), geocode=bool(args.geocode))
|
||||
assert_output_sanity(output_sanity_checks)
|
||||
except RuntimeError as exc:
|
||||
repair_summary = rerun_pair_product_stages(
|
||||
project_name=args.project_name,
|
||||
project_dir=pyint_project_dir,
|
||||
run_root=run_root,
|
||||
scratch_root=scratch_root,
|
||||
env=env,
|
||||
pair_name=pair_name,
|
||||
master_date=master_date,
|
||||
slave_date=slave_date,
|
||||
range_looks=args.range_looks,
|
||||
unwrap=bool(args.unwrap),
|
||||
atmcor=bool(args.atmcor),
|
||||
geocode=bool(args.geocode),
|
||||
)
|
||||
outputs = collect_expected_outputs(pyint_project_dir, pair_name, args.range_looks)
|
||||
assert_required_outputs(outputs, unwrap=bool(args.unwrap), geocode=bool(args.geocode))
|
||||
output_sanity_checks = collect_output_sanity_checks(outputs, unwrap=bool(args.unwrap), geocode=bool(args.geocode))
|
||||
assert_output_sanity(output_sanity_checks)
|
||||
|
||||
stage_error_logs = collect_stage_error_logs(pyint_project_dir)
|
||||
reflatten_summary: Dict[str, Any] = {
|
||||
"enabled": bool(args.reflatten),
|
||||
"applied": False,
|
||||
"reason": "",
|
||||
"model": reflatten_model,
|
||||
"coherence_threshold": reflatten_coh_threshold,
|
||||
"fallback_coherence_threshold": reflatten_fallback_coh_threshold,
|
||||
"range_step": reflatten_range_step,
|
||||
"azimuth_step": reflatten_azimuth_step,
|
||||
}
|
||||
if bool(args.reflatten) and bool(args.unwrap):
|
||||
reflatten_summary = run_gamma_reflatten(
|
||||
project_dir=pyint_project_dir,
|
||||
run_root=run_root,
|
||||
output_dir=output_dir,
|
||||
outputs=outputs,
|
||||
pair_name=pair_name,
|
||||
master_date=master_date,
|
||||
range_looks=args.range_looks,
|
||||
env=env,
|
||||
model=reflatten_model,
|
||||
coherence_threshold=reflatten_coh_threshold,
|
||||
fallback_coherence_threshold=reflatten_fallback_coh_threshold,
|
||||
range_step=reflatten_range_step,
|
||||
azimuth_step=reflatten_azimuth_step,
|
||||
geo_interp=geo_interp,
|
||||
)
|
||||
outputs = collect_expected_outputs(pyint_project_dir, pair_name, args.range_looks)
|
||||
output_sanity_checks = collect_output_sanity_checks(outputs, unwrap=bool(args.unwrap), geocode=bool(args.geocode))
|
||||
assert_output_sanity(output_sanity_checks)
|
||||
elif bool(args.reflatten):
|
||||
reflatten_summary["reason"] = "unwrap disabled"
|
||||
else:
|
||||
reflatten_summary["reason"] = "disabled"
|
||||
|
||||
copied_paths = copy_native_outputs(
|
||||
project_dir=pyint_project_dir,
|
||||
output_dir=output_dir,
|
||||
pair_name=pair_name,
|
||||
template_path=template_path,
|
||||
ifgram_list_path=ifgram_list_path,
|
||||
stdout_path=run_stdout,
|
||||
stderr_path=run_stderr,
|
||||
)
|
||||
standard_products = (
|
||||
export_standard_products(
|
||||
project_dir=pyint_project_dir,
|
||||
output_dir=output_dir,
|
||||
pair_name=pair_name,
|
||||
master_date=master_date,
|
||||
range_looks=args.range_looks,
|
||||
azimuth_looks=args.azimuth_looks,
|
||||
target_grid_size_m=int(args.target_grid_size_m or 0),
|
||||
coherence_mask_threshold=coherence_mask_threshold,
|
||||
reference_mode=reference_mode,
|
||||
reference_coh_threshold=reference_coh_threshold,
|
||||
deramp_mode=deramp_mode,
|
||||
deramp_coh_threshold=deramp_coh_threshold,
|
||||
atmcor_enabled=bool(args.atmcor),
|
||||
atmcor_use_for_disp=bool(args.atmcor_use_for_disp),
|
||||
reflatten_summary=reflatten_summary,
|
||||
gamma_nodata_value=gamma_nodata_value,
|
||||
outputs=outputs,
|
||||
env=env,
|
||||
run_root=run_root,
|
||||
)
|
||||
if bool(args.geocode)
|
||||
else {"enabled": False, "reason": "geocode disabled"}
|
||||
)
|
||||
|
||||
summary = {
|
||||
"ok": True,
|
||||
"task_dir": str(task_dir),
|
||||
"task_alias": task_alias,
|
||||
"pair_key": pair_key,
|
||||
"project_name": args.project_name,
|
||||
"project_dir": str(pyint_project_dir),
|
||||
"run_root": str(run_root),
|
||||
"template_root": str(template_root),
|
||||
"output_dir": str(output_dir),
|
||||
"pyint_home": str(pyint_home),
|
||||
"pyint_app_script": str(pyint_app_script),
|
||||
"gamma_env_script": args.gamma_env_script,
|
||||
"dem": {
|
||||
"mode": dem_mode,
|
||||
"dem_root": str(dem_root),
|
||||
"fabdem_root": str(args.fabdem_root or "").strip(),
|
||||
"prepared_dem_path": str(args.prepared_dem_path or "").strip(),
|
||||
"prepared_dem_kind": str(prepared_dem_info.get("kind") or ""),
|
||||
"configured_resolution_m": float(args.dem_resolution_m),
|
||||
"oversampling": dem_oversampling,
|
||||
"opentopo_dem_type": str(args.opentopo_dem_type or "SRTMGL1").strip(),
|
||||
"opentopo_api_key_configured": bool(str(args.opentopo_api_key or "").strip()),
|
||||
},
|
||||
"orbit_policy": "require_eof",
|
||||
"precise_orbit_bridge": {"enabled": False, "mode": "not_applicable"},
|
||||
"input_assets_dir": str(input_assets_dir) if input_assets_dir else "",
|
||||
"input_assets_json": str(input_assets_json) if input_assets_json else "",
|
||||
"input_assets": input_assets_payload,
|
||||
"master_date": master_date,
|
||||
"slave_date": slave_date,
|
||||
"pair_name": pair_name,
|
||||
"time_baseline_days": time_baseline_days,
|
||||
"target_grid_size_m": int(args.target_grid_size_m or 0),
|
||||
"range_looks": int(args.range_looks),
|
||||
"azimuth_looks": int(args.azimuth_looks),
|
||||
"dem_resolution_m": float(args.dem_resolution_m),
|
||||
"dem_oversampling": dem_oversampling,
|
||||
"unwrap_coh_threshold": unwrap_coh_threshold,
|
||||
"coherence_quality_threshold": coherence_mask_threshold,
|
||||
"reference_mode": reference_mode,
|
||||
"reference_coh_threshold": reference_coh_threshold,
|
||||
"deramp_mode": deramp_mode,
|
||||
"deramp_coh_threshold": deramp_coh_threshold,
|
||||
"gamma_nodata_value": gamma_nodata_value,
|
||||
"geo_interp": geo_interp,
|
||||
"atmcor": bool(args.atmcor),
|
||||
"atmcor_use_for_disp": bool(args.atmcor_use_for_disp),
|
||||
"reflatten": bool(args.reflatten),
|
||||
"reflatten_model": reflatten_model,
|
||||
"reflatten_coh_threshold": reflatten_coh_threshold,
|
||||
"reflatten_fallback_coh_threshold": reflatten_fallback_coh_threshold,
|
||||
"reflatten_range_step": reflatten_range_step,
|
||||
"reflatten_azimuth_step": reflatten_azimuth_step,
|
||||
"parallel_workers": int(args.parallel_workers),
|
||||
"unwrap": bool(args.unwrap),
|
||||
"geocode": bool(args.geocode),
|
||||
"archives": {
|
||||
"master": [str(master_scene)],
|
||||
"slave": [str(slave_scene)],
|
||||
},
|
||||
"orbit_files": {
|
||||
"master": str(master_eof),
|
||||
"slave": str(slave_eof),
|
||||
},
|
||||
"archive_materialization": archive_materialization,
|
||||
"workspace_outputs": outputs,
|
||||
"output_sanity_checks": output_sanity_checks,
|
||||
"output_repair": repair_summary,
|
||||
"reflatten_summary": reflatten_summary,
|
||||
"copied_outputs": copied_paths,
|
||||
"standard_products": standard_products,
|
||||
"logs": {
|
||||
"generate_stdout": str(generate_stdout),
|
||||
"generate_stderr": str(generate_stderr),
|
||||
"run_stdout": str(run_stdout),
|
||||
"run_stderr": str(run_stderr),
|
||||
"stage_error_logs": stage_error_logs,
|
||||
},
|
||||
"started_at": run_started_at,
|
||||
"finished_at": datetime.utcnow().isoformat(timespec="seconds") + "Z",
|
||||
}
|
||||
|
||||
summary_path = output_dir / "pyint_run_summary.json"
|
||||
write_text(summary_path, json.dumps(summary, ensure_ascii=True, indent=2) + "\n")
|
||||
print(json.dumps(summary, ensure_ascii=True, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except Exception as exc:
|
||||
print(str(exc), file=sys.stderr)
|
||||
raise
|
||||
@@ -5,6 +5,7 @@ from fastapi import APIRouter
|
||||
from . import (
|
||||
ai,
|
||||
aoi,
|
||||
assets,
|
||||
auth,
|
||||
dinsar,
|
||||
dinsar_products,
|
||||
@@ -43,6 +44,7 @@ def include_all_routers(router: APIRouter) -> None:
|
||||
router.include_router(unpack.router)
|
||||
router.include_router(monitor.router)
|
||||
router.include_router(orbit.router)
|
||||
router.include_router(assets.router)
|
||||
router.include_router(root_registry.router)
|
||||
router.include_router(radar.router)
|
||||
router.include_router(aoi.router)
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ..database import get_db
|
||||
from ..models import AuthUserORM
|
||||
from ..models.orm import SourceProductAssetORM
|
||||
from ..services.asset_inventory_service import asset_inventory_service
|
||||
from ..services.job_queue_service import job_queue_service
|
||||
from ..services.task_service import task_service
|
||||
from .dependencies import _get_current_user, _require_admin
|
||||
|
||||
|
||||
router = APIRouter(prefix="/assets", tags=["assets"])
|
||||
|
||||
|
||||
class AssetScanRequest(BaseModel):
|
||||
inventory_types: List[str] = Field(default_factory=list)
|
||||
root_ids: List[int] = Field(default_factory=list)
|
||||
bind_orbits: bool = True
|
||||
|
||||
|
||||
class S1UnpackRequest(BaseModel):
|
||||
target_root: Optional[str] = None
|
||||
overwrite: bool = False
|
||||
min_disk_space_gb: Optional[float] = Field(default=None, ge=0)
|
||||
delete_archive: Optional[bool] = None
|
||||
|
||||
|
||||
class S1BatchUnpackRequest(BaseModel):
|
||||
target_root: Optional[str] = None
|
||||
overwrite: bool = False
|
||||
min_disk_space_gb: Optional[float] = Field(default=None, ge=0)
|
||||
delete_archive: Optional[bool] = None
|
||||
scan_before_unpack: bool = True
|
||||
|
||||
|
||||
@router.get("/inventory/status")
|
||||
async def get_asset_inventory_status(
|
||||
current_user: AuthUserORM = Depends(_get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
_ = current_user
|
||||
return await asset_inventory_service.get_status(db)
|
||||
|
||||
|
||||
@router.post("/inventory/scan", status_code=202)
|
||||
async def run_asset_inventory_scan(
|
||||
request: Optional[AssetScanRequest] = None,
|
||||
admin_user: AuthUserORM = Depends(_require_admin),
|
||||
):
|
||||
_ = admin_user
|
||||
payload = (request or AssetScanRequest()).model_dump()
|
||||
try:
|
||||
task_id = await task_service.create_task(
|
||||
"SCAN_ASSET_INVENTORY",
|
||||
"Source/orbit asset inventory scan",
|
||||
params=payload,
|
||||
)
|
||||
job_id = await job_queue_service.create_job(
|
||||
"SCAN_ASSET_INVENTORY",
|
||||
payload=payload,
|
||||
task_id=task_id,
|
||||
)
|
||||
return {"message": "Asset inventory scan queued", "task_id": task_id, "job_id": job_id}
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@router.post("/inventory/scan-now")
|
||||
async def run_asset_inventory_scan_now(
|
||||
request: Optional[AssetScanRequest] = None,
|
||||
admin_user: AuthUserORM = Depends(_require_admin),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
_ = admin_user
|
||||
payload = (request or AssetScanRequest()).model_dump()
|
||||
return await asset_inventory_service.scan_configured_roots(
|
||||
db,
|
||||
inventory_types=payload.get("inventory_types") or None,
|
||||
root_ids=payload.get("root_ids") or None,
|
||||
bind_orbits=bool(payload.get("bind_orbits", True)),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/sources")
|
||||
async def list_source_product_assets(
|
||||
satellite_family: Optional[str] = None,
|
||||
satellite: Optional[str] = None,
|
||||
source_format: Optional[str] = None,
|
||||
parse_status: Optional[str] = None,
|
||||
include_inactive: bool = False,
|
||||
limit: int = Query(200, ge=1, le=1000),
|
||||
offset: int = Query(0, ge=0),
|
||||
current_user: AuthUserORM = Depends(_get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
_ = current_user
|
||||
return await asset_inventory_service.list_source_products(
|
||||
db,
|
||||
satellite_family=satellite_family,
|
||||
satellite=satellite,
|
||||
source_format=source_format,
|
||||
parse_status=parse_status,
|
||||
include_inactive=include_inactive,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/orbits")
|
||||
async def list_orbit_assets(
|
||||
satellite_family: Optional[str] = None,
|
||||
satellite: Optional[str] = None,
|
||||
orbit_type: Optional[str] = None,
|
||||
parse_status: Optional[str] = None,
|
||||
include_inactive: bool = False,
|
||||
limit: int = Query(200, ge=1, le=1000),
|
||||
offset: int = Query(0, ge=0),
|
||||
current_user: AuthUserORM = Depends(_get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
_ = current_user
|
||||
return await asset_inventory_service.list_orbits(
|
||||
db,
|
||||
satellite_family=satellite_family,
|
||||
satellite=satellite,
|
||||
orbit_type=orbit_type,
|
||||
parse_status=parse_status,
|
||||
include_inactive=include_inactive,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/issues")
|
||||
async def list_asset_inventory_issues(
|
||||
status: str = "OPEN",
|
||||
severity: Optional[str] = None,
|
||||
issue_code: Optional[str] = None,
|
||||
limit: int = Query(200, ge=1, le=1000),
|
||||
offset: int = Query(0, ge=0),
|
||||
current_user: AuthUserORM = Depends(_get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
_ = current_user
|
||||
return await asset_inventory_service.list_issues(
|
||||
db,
|
||||
status=status,
|
||||
severity=severity,
|
||||
issue_code=issue_code,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/sources/{asset_id}/unpack-sentinel1")
|
||||
async def unpack_sentinel1_source_asset(
|
||||
asset_id: int,
|
||||
request: Optional[S1UnpackRequest] = None,
|
||||
admin_user: AuthUserORM = Depends(_require_admin),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
_ = admin_user
|
||||
asset = await db.get(SourceProductAssetORM, asset_id)
|
||||
if asset is None:
|
||||
raise HTTPException(status_code=404, detail="Source product asset not found.")
|
||||
if asset.source_format != "S1_ZIP":
|
||||
raise HTTPException(status_code=400, detail="Only Sentinel-1 ZIP assets can be unpacked by this endpoint.")
|
||||
|
||||
request_data = request or S1UnpackRequest()
|
||||
payload = {
|
||||
"asset_id": asset_id,
|
||||
"target_root": request_data.target_root,
|
||||
"overwrite": bool(request_data.overwrite),
|
||||
}
|
||||
if request_data.min_disk_space_gb is not None:
|
||||
payload["min_disk_space_gb"] = request_data.min_disk_space_gb
|
||||
if request_data.delete_archive is not None:
|
||||
payload["delete_archive"] = request_data.delete_archive
|
||||
try:
|
||||
task_id = await task_service.create_task(
|
||||
"UNPACK_SENTINEL1",
|
||||
"Sentinel-1 unpack",
|
||||
params=payload,
|
||||
)
|
||||
job_id = await job_queue_service.create_job(
|
||||
"UNPACK_SENTINEL1",
|
||||
payload=payload,
|
||||
task_id=task_id,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
await db.commit()
|
||||
return {"message": "Sentinel-1 unpack task queued", "task_id": task_id, "job_id": job_id}
|
||||
|
||||
|
||||
@router.post("/inventory/unpack-sentinel1", status_code=202)
|
||||
async def run_sentinel1_unpack_batch(
|
||||
request: Optional[S1BatchUnpackRequest] = None,
|
||||
admin_user: AuthUserORM = Depends(_require_admin),
|
||||
):
|
||||
_ = admin_user
|
||||
payload = (request or S1BatchUnpackRequest()).model_dump()
|
||||
try:
|
||||
task_id = await task_service.create_task(
|
||||
"UNPACK_SENTINEL1",
|
||||
"Sentinel-1 batch unpack",
|
||||
params=payload,
|
||||
)
|
||||
job_id = await job_queue_service.create_job(
|
||||
"UNPACK_SENTINEL1",
|
||||
payload=payload,
|
||||
task_id=task_id,
|
||||
)
|
||||
return {"message": "Sentinel-1 batch unpack task queued", "task_id": task_id, "job_id": job_id}
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
@@ -99,9 +99,15 @@ def _normalize_list_pagination(limit: int, offset: int) -> tuple[int, int]:
|
||||
class DinsarBatchSceneCreate(BaseModel):
|
||||
file_path: str = Field(min_length=1)
|
||||
satellite: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH)
|
||||
satellite_family: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH)
|
||||
imaging_date: Optional[str] = Field(default=None, max_length=32)
|
||||
imaging_mode: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH)
|
||||
polarization: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH)
|
||||
orbit_direction: Optional[str] = Field(default=None, max_length=BATCH_TEXT_MAX_LENGTH)
|
||||
relative_orbit: Optional[str] = Field(default=None, max_length=64)
|
||||
absolute_orbit: Optional[str] = Field(default=None, max_length=64)
|
||||
has_orbit_data: Optional[bool] = None
|
||||
orbit_file_path: Optional[str] = Field(default=None, max_length=4096)
|
||||
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
@@ -14,6 +14,10 @@ MONITOR_CONFIG = {
|
||||
"radar_dirs": split_env_paths(settings.MONITOR_RADAR_DIRS),
|
||||
"orbit_dir": settings.MONITOR_ORBIT_DIR,
|
||||
"dinsar_dirs": split_env_paths(settings.MONITOR_DINSAR_DIRS),
|
||||
# Sentinel-1 链路
|
||||
"s1_source_dirs": split_env_paths(settings.SOURCE_PRODUCT_DIRS),
|
||||
"s1_storage_dirs": split_env_paths(settings.SENTINEL1_STORAGE_DIRS),
|
||||
"s1_orbit_dirs": split_env_paths(settings.ORBIT_SOURCE_DIRS),
|
||||
# GF3 链路
|
||||
"gf3_source_dirs": split_env_paths(settings.GF3_SOURCE_DIRS),
|
||||
"gf3_storage_dirs": split_env_paths(settings.GF3_STORAGE_DIRS),
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -238,7 +238,7 @@ def _chunked(items: List[Any], size: int):
|
||||
|
||||
|
||||
_RADAR_PREVIEW_EXTENSIONS = (".jpg", ".jpeg", ".png", ".webp", ".bmp", ".tif", ".tiff")
|
||||
_RADAR_PREVIEW_KEYWORDS = ("quicklook", "preview", "browse", "thumbnail", "thumb", "overview")
|
||||
_RADAR_PREVIEW_KEYWORDS = ("quicklook", "quick-look", "preview", "browse", "thumbnail", "thumb", "overview")
|
||||
_RADAR_CACHE_NAME_RE = re.compile(r"[^A-Za-z0-9._-]+")
|
||||
|
||||
|
||||
@@ -396,7 +396,18 @@ class DataService:
|
||||
continue
|
||||
|
||||
path = os.path.join(root, name)
|
||||
root_lower = root.lower()
|
||||
keyword_score = 0 if any(key in lower_name for key in _RADAR_PREVIEW_KEYWORDS) else 1
|
||||
if lower_name == "quick-look.png":
|
||||
keyword_score = -3
|
||||
elif lower_name == "quicklook.png":
|
||||
keyword_score = -2
|
||||
elif lower_name.startswith("quick-look.") or lower_name.startswith("quicklook."):
|
||||
keyword_score = min(keyword_score, -1)
|
||||
if f"{os.sep}preview" in root_lower:
|
||||
keyword_score -= 1
|
||||
if f"{os.sep}icons" in root_lower:
|
||||
keyword_score += 2
|
||||
ext_score = 0 if lower_name.endswith((".jpg", ".jpeg")) else 1
|
||||
try:
|
||||
size_score = -os.path.getsize(path)
|
||||
|
||||
@@ -7,6 +7,8 @@ import re
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from ..utils import normalize_satellite_family
|
||||
|
||||
|
||||
PAIR_META_FILENAME = ".dinsar_pair.json"
|
||||
RUN_META_FILENAME = ".dinsar_run.json"
|
||||
@@ -48,6 +50,7 @@ def build_pair_key(
|
||||
slave_path: Any,
|
||||
master_date: Any = None,
|
||||
slave_date: Any = None,
|
||||
satellite_family: Any = None,
|
||||
) -> str:
|
||||
master_date_text = _normalize_date(master_date)
|
||||
slave_date_text = _normalize_date(slave_date)
|
||||
@@ -60,17 +63,23 @@ def build_pair_key(
|
||||
]
|
||||
)
|
||||
digest = hashlib.sha1(payload.encode("utf-8", errors="ignore")).hexdigest()[:10]
|
||||
return f"lt1_{master_date_text}_{slave_date_text}_{digest}"
|
||||
family = str(normalize_satellite_family(satellite_family) or "").strip().lower()
|
||||
if not family:
|
||||
family = "pair"
|
||||
return f"{family}_{master_date_text}_{slave_date_text}_{digest}"
|
||||
|
||||
|
||||
def build_fallback_pair_key(task_alias: Any, source_hint: Any = None) -> str:
|
||||
def build_fallback_pair_key(task_alias: Any, source_hint: Any = None, satellite_family: Any = None) -> str:
|
||||
alias = str(task_alias or "").strip() or "Task_unknown_unknown"
|
||||
parts = alias.split("_")
|
||||
master_date = parts[1] if len(parts) > 2 else "unknown"
|
||||
slave_date = parts[2] if len(parts) > 2 else "unknown"
|
||||
payload = "||".join([alias, normalize_path(source_hint)])
|
||||
digest = hashlib.sha1(payload.encode("utf-8", errors="ignore")).hexdigest()[:10]
|
||||
return f"lt1_{_normalize_date(master_date)}_{_normalize_date(slave_date)}_{digest}"
|
||||
family = str(normalize_satellite_family(satellite_family) or "").strip().lower()
|
||||
if not family:
|
||||
family = "pair"
|
||||
return f"{family}_{_normalize_date(master_date)}_{_normalize_date(slave_date)}_{digest}"
|
||||
|
||||
|
||||
def build_run_key(
|
||||
|
||||
@@ -21,6 +21,7 @@ from typing import Any, Dict, List, Optional
|
||||
|
||||
from ..config import get_env_text, settings
|
||||
from ..process_utils import is_any_process_running
|
||||
from ..utils import normalize_satellite_family
|
||||
from .dinsar_naming import (
|
||||
PAIR_META_FILENAME,
|
||||
build_fallback_pair_key,
|
||||
@@ -480,7 +481,14 @@ def _utc_now_text() -> str:
|
||||
def _resolve_dinsar_pair_identity(task_dir: str, task_name: str) -> tuple[str, str, Dict[str, Any]]:
|
||||
pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {}
|
||||
task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name
|
||||
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(task_alias, task_dir)
|
||||
satellite_family = normalize_satellite_family(
|
||||
pair_meta.get("master_satellite") or pair_meta.get("slave_satellite")
|
||||
)
|
||||
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(
|
||||
task_alias,
|
||||
task_dir,
|
||||
satellite_family=satellite_family,
|
||||
)
|
||||
return task_alias, pair_key, pair_meta
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import os
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Dict, Optional
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import and_, func, literal, or_, select, text
|
||||
@@ -11,9 +11,14 @@ from ..config import read_int_env, settings, split_env_paths
|
||||
from ..db_maintenance import inspect_database_structure
|
||||
from ..models import (
|
||||
AiDiagnosisORM,
|
||||
AssetInventoryIssueORM,
|
||||
AssetInventoryStateORM,
|
||||
DinsarResultORM,
|
||||
OrbitAssetORM,
|
||||
ResultCatalogStateORM,
|
||||
ResultProductORM,
|
||||
SceneOrbitBindingORM,
|
||||
SourceProductAssetORM,
|
||||
SystemWorkerHeartbeatORM,
|
||||
)
|
||||
from ..idl_service import get_idl_status
|
||||
@@ -399,6 +404,51 @@ def _sanitize_pairing_system_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _sanitize_asset_inventory_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
source_roots = payload.get("source_roots", {}) or {}
|
||||
orbit_roots = payload.get("orbit_roots", {}) or {}
|
||||
source_assets = payload.get("source_assets", {}) or {}
|
||||
orbit_assets = payload.get("orbit_assets", {}) or {}
|
||||
bindings = payload.get("bindings", {}) or {}
|
||||
issues = payload.get("issues", {}) or {}
|
||||
return {
|
||||
"ok": bool(payload.get("ok")),
|
||||
"source_roots": {
|
||||
"configured_count": int(source_roots.get("configured_count") or 0),
|
||||
"accessible_count": int(source_roots.get("accessible_count") or 0),
|
||||
"needs_rescan_count": int(source_roots.get("needs_rescan_count") or 0),
|
||||
},
|
||||
"orbit_roots": {
|
||||
"configured_count": int(orbit_roots.get("configured_count") or 0),
|
||||
"accessible_count": int(orbit_roots.get("accessible_count") or 0),
|
||||
"needs_rescan_count": int(orbit_roots.get("needs_rescan_count") or 0),
|
||||
},
|
||||
"source_assets": {
|
||||
"total_count": int(source_assets.get("total_count") or 0),
|
||||
"lt1_count": int(source_assets.get("lt1_count") or 0),
|
||||
"s1_count": int(source_assets.get("s1_count") or 0),
|
||||
"parse_failed_count": int(source_assets.get("parse_failed_count") or 0),
|
||||
},
|
||||
"orbit_assets": {
|
||||
"total_count": int(orbit_assets.get("total_count") or 0),
|
||||
"lt1_count": int(orbit_assets.get("lt1_count") or 0),
|
||||
"s1_count": int(orbit_assets.get("s1_count") or 0),
|
||||
"parse_failed_count": int(orbit_assets.get("parse_failed_count") or 0),
|
||||
},
|
||||
"bindings": {
|
||||
"scene_count": int(bindings.get("scene_count") or 0),
|
||||
"matched_count": int(bindings.get("matched_count") or 0),
|
||||
"missing_count": int(bindings.get("missing_count") or 0),
|
||||
"ambiguous_count": int(bindings.get("ambiguous_count") or 0),
|
||||
},
|
||||
"issues": {
|
||||
"open_count": int(issues.get("open_count") or 0),
|
||||
"error_count": int(issues.get("error_count") or 0),
|
||||
"warning_count": int(issues.get("warning_count") or 0),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
database = payload.get("database", {}) or {}
|
||||
worker = payload.get("worker", {}) or {}
|
||||
@@ -411,6 +461,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
dinsar_bridge = payload.get("dinsar_bridge", {}) or {}
|
||||
source_roots = payload.get("source_roots", {}) or {}
|
||||
product_packages = payload.get("product_packages", {}) or {}
|
||||
asset_inventory = payload.get("asset_inventory", {}) or {}
|
||||
wsl_runtime = payload.get("wsl_runtime", {}) or {}
|
||||
pairing_system = payload.get("pairing_system", {}) or {}
|
||||
idl = payload.get("idl", {}) or {}
|
||||
@@ -423,6 +474,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
sanitized_dinsar_bridge = _sanitize_bridge_status(dinsar_bridge)
|
||||
sanitized_source_roots = _sanitize_source_roots_status(source_roots)
|
||||
sanitized_product_packages = _sanitize_product_package_status(product_packages)
|
||||
sanitized_asset_inventory = _sanitize_asset_inventory_status(asset_inventory)
|
||||
sanitized_wsl_runtime = _sanitize_wsl_runtime_status(wsl_runtime)
|
||||
sanitized_pairing_system = _sanitize_pairing_system_status(pairing_system)
|
||||
|
||||
@@ -451,6 +503,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"dinsar_bridge": sanitized_dinsar_bridge,
|
||||
"source_roots": sanitized_source_roots,
|
||||
"product_packages": sanitized_product_packages,
|
||||
"asset_inventory": sanitized_asset_inventory,
|
||||
"wsl_runtime": sanitized_wsl_runtime,
|
||||
"pairing_system": sanitized_pairing_system,
|
||||
"idl": {
|
||||
@@ -905,6 +958,221 @@ async def _check_product_packages() -> Dict[str, Any]:
|
||||
return status
|
||||
|
||||
|
||||
async def _check_asset_inventory() -> Dict[str, Any]:
|
||||
status: Dict[str, Any] = {
|
||||
"ok": False,
|
||||
"source_roots": {
|
||||
"configured_count": 0,
|
||||
"accessible_count": 0,
|
||||
"inaccessible_count": 0,
|
||||
"needs_rescan_count": 0,
|
||||
"items": [],
|
||||
},
|
||||
"orbit_roots": {
|
||||
"configured_count": 0,
|
||||
"accessible_count": 0,
|
||||
"inaccessible_count": 0,
|
||||
"needs_rescan_count": 0,
|
||||
"items": [],
|
||||
},
|
||||
"source_assets": {
|
||||
"total_count": 0,
|
||||
"lt1_count": 0,
|
||||
"s1_count": 0,
|
||||
"parse_failed_count": 0,
|
||||
"by_family": {},
|
||||
},
|
||||
"orbit_assets": {
|
||||
"total_count": 0,
|
||||
"lt1_count": 0,
|
||||
"s1_count": 0,
|
||||
"parse_failed_count": 0,
|
||||
"by_family": {},
|
||||
},
|
||||
"bindings": {
|
||||
"scene_count": 0,
|
||||
"matched_count": 0,
|
||||
"missing_count": 0,
|
||||
"ambiguous_count": 0,
|
||||
},
|
||||
"issues": {
|
||||
"open_count": 0,
|
||||
"error_count": 0,
|
||||
"warning_count": 0,
|
||||
"by_code": {},
|
||||
},
|
||||
"error": None,
|
||||
}
|
||||
|
||||
source_paths: List[str] = []
|
||||
for value in (
|
||||
settings.SOURCE_PRODUCT_DIRS,
|
||||
settings.INSAR_STORAGE_DIRS,
|
||||
settings.MONITOR_RADAR_DIRS,
|
||||
):
|
||||
for path in split_env_paths(value):
|
||||
if path not in source_paths:
|
||||
source_paths.append(path)
|
||||
|
||||
orbit_paths: List[str] = []
|
||||
for value in (
|
||||
settings.ORBIT_SOURCE_DIRS,
|
||||
settings.MONITOR_ORBIT_DIR,
|
||||
):
|
||||
for path in split_env_paths(value):
|
||||
if path not in orbit_paths:
|
||||
orbit_paths.append(path)
|
||||
|
||||
for path in source_paths:
|
||||
item = _probe_directory_status(path)
|
||||
item["role"] = "source_product_pool"
|
||||
status["source_roots"]["items"].append(item)
|
||||
for path in orbit_paths:
|
||||
item = _probe_directory_status(path)
|
||||
item["role"] = "orbit_asset_pool"
|
||||
status["orbit_roots"]["items"].append(item)
|
||||
|
||||
for key in ("source_roots", "orbit_roots"):
|
||||
root_status = status[key]
|
||||
root_status["configured_count"] = len(root_status["items"])
|
||||
root_status["accessible_count"] = sum(1 for item in root_status["items"] if item.get("accessible"))
|
||||
root_status["inaccessible_count"] = root_status["configured_count"] - root_status["accessible_count"]
|
||||
|
||||
try:
|
||||
session_factory = _get_session_factory()
|
||||
async with session_factory() as db:
|
||||
state_rows = (
|
||||
await db.execute(
|
||||
select(
|
||||
AssetInventoryStateORM.inventory_type,
|
||||
func.count(AssetInventoryStateORM.id),
|
||||
)
|
||||
.where(AssetInventoryStateORM.needs_rescan == True) # noqa: E712
|
||||
.group_by(AssetInventoryStateORM.inventory_type)
|
||||
)
|
||||
).all()
|
||||
for inventory_type, count in state_rows:
|
||||
key = "orbit_roots" if str(inventory_type or "").lower().startswith("orbit") else "source_roots"
|
||||
status[key]["needs_rescan_count"] += int(count or 0)
|
||||
|
||||
source_family_rows = (
|
||||
await db.execute(
|
||||
select(
|
||||
SourceProductAssetORM.satellite_family,
|
||||
func.count(SourceProductAssetORM.id),
|
||||
)
|
||||
.where(
|
||||
SourceProductAssetORM.is_active == True, # noqa: E712
|
||||
SourceProductAssetORM.source_format != "S1_ZIP",
|
||||
)
|
||||
.group_by(SourceProductAssetORM.satellite_family)
|
||||
)
|
||||
).all()
|
||||
for family, count in source_family_rows:
|
||||
family_key = str(family or "unknown").strip().upper() or "unknown"
|
||||
value = int(count or 0)
|
||||
status["source_assets"]["by_family"][family_key] = value
|
||||
status["source_assets"]["total_count"] += value
|
||||
status["source_assets"]["lt1_count"] = int(status["source_assets"]["by_family"].get("LT1", 0))
|
||||
status["source_assets"]["s1_count"] = int(status["source_assets"]["by_family"].get("S1", 0))
|
||||
source_parse_failed = await db.execute(
|
||||
select(func.count(SourceProductAssetORM.id)).where(
|
||||
SourceProductAssetORM.parse_status == "FAILED",
|
||||
SourceProductAssetORM.source_format != "S1_ZIP",
|
||||
)
|
||||
)
|
||||
status["source_assets"]["parse_failed_count"] = int(source_parse_failed.scalar_one() or 0)
|
||||
|
||||
orbit_family_rows = (
|
||||
await db.execute(
|
||||
select(
|
||||
OrbitAssetORM.satellite_family,
|
||||
func.count(OrbitAssetORM.id),
|
||||
)
|
||||
.where(OrbitAssetORM.is_active == True) # noqa: E712
|
||||
.group_by(OrbitAssetORM.satellite_family)
|
||||
)
|
||||
).all()
|
||||
for family, count in orbit_family_rows:
|
||||
family_key = str(family or "unknown").strip().upper() or "unknown"
|
||||
value = int(count or 0)
|
||||
status["orbit_assets"]["by_family"][family_key] = value
|
||||
status["orbit_assets"]["total_count"] += value
|
||||
status["orbit_assets"]["lt1_count"] = int(status["orbit_assets"]["by_family"].get("LT1", 0))
|
||||
status["orbit_assets"]["s1_count"] = int(status["orbit_assets"]["by_family"].get("S1", 0))
|
||||
orbit_parse_failed = await db.execute(
|
||||
select(func.count(OrbitAssetORM.id)).where(OrbitAssetORM.parse_status == "FAILED")
|
||||
)
|
||||
status["orbit_assets"]["parse_failed_count"] = int(orbit_parse_failed.scalar_one() or 0)
|
||||
|
||||
scene_count = await db.execute(select(func.count(SceneOrbitBindingORM.radar_data_id.distinct())))
|
||||
status["bindings"]["scene_count"] = int(scene_count.scalar_one() or 0)
|
||||
selected_count = await db.execute(
|
||||
select(func.count(SceneOrbitBindingORM.id)).where(SceneOrbitBindingORM.selection_status == "SELECTED")
|
||||
)
|
||||
status["bindings"]["matched_count"] = int(selected_count.scalar_one() or 0)
|
||||
missing_count = await db.execute(
|
||||
select(func.count(AssetInventoryIssueORM.id)).where(
|
||||
AssetInventoryIssueORM.status == "OPEN",
|
||||
AssetInventoryIssueORM.issue_code == "scene_missing_orbit",
|
||||
)
|
||||
)
|
||||
status["bindings"]["missing_count"] = int(missing_count.scalar_one() or 0)
|
||||
ambiguous_count = await db.execute(
|
||||
select(func.count(AssetInventoryIssueORM.id)).where(
|
||||
AssetInventoryIssueORM.status == "OPEN",
|
||||
AssetInventoryIssueORM.issue_code == "scene_ambiguous_orbit",
|
||||
)
|
||||
)
|
||||
status["bindings"]["ambiguous_count"] = int(ambiguous_count.scalar_one() or 0)
|
||||
|
||||
issue_rows = (
|
||||
await db.execute(
|
||||
select(
|
||||
AssetInventoryIssueORM.severity,
|
||||
func.count(AssetInventoryIssueORM.id),
|
||||
)
|
||||
.where(AssetInventoryIssueORM.status == "OPEN")
|
||||
.group_by(AssetInventoryIssueORM.severity)
|
||||
)
|
||||
).all()
|
||||
for severity, count in issue_rows:
|
||||
severity_key = str(severity or "warning").strip().lower() or "warning"
|
||||
value = int(count or 0)
|
||||
status["issues"]["open_count"] += value
|
||||
if severity_key == "error":
|
||||
status["issues"]["error_count"] += value
|
||||
elif severity_key == "warning":
|
||||
status["issues"]["warning_count"] += value
|
||||
|
||||
issue_code_rows = (
|
||||
await db.execute(
|
||||
select(
|
||||
AssetInventoryIssueORM.issue_code,
|
||||
func.count(AssetInventoryIssueORM.id),
|
||||
)
|
||||
.where(AssetInventoryIssueORM.status == "OPEN")
|
||||
.group_by(AssetInventoryIssueORM.issue_code)
|
||||
)
|
||||
).all()
|
||||
status["issues"]["by_code"] = {
|
||||
str(code or "unknown"): int(count or 0)
|
||||
for code, count in issue_code_rows
|
||||
}
|
||||
|
||||
status["ok"] = (
|
||||
status["source_roots"]["inaccessible_count"] == 0
|
||||
and status["orbit_roots"]["inaccessible_count"] == 0
|
||||
and status["source_assets"]["parse_failed_count"] == 0
|
||||
and status["orbit_assets"]["parse_failed_count"] == 0
|
||||
and status["issues"]["error_count"] == 0
|
||||
)
|
||||
except Exception as exc:
|
||||
status["error"] = str(exc)
|
||||
|
||||
return status
|
||||
|
||||
|
||||
async def _check_wsl_runtime() -> Dict[str, Any]:
|
||||
status = {
|
||||
"ok": False,
|
||||
@@ -989,6 +1257,7 @@ async def get_health_status(
|
||||
dinsar_bridge_status = await _check_dinsar_bridge()
|
||||
source_roots_status = await _check_source_roots()
|
||||
product_packages_status = await _check_product_packages()
|
||||
asset_inventory_status = await _check_asset_inventory()
|
||||
wsl_runtime_status = await _check_wsl_runtime()
|
||||
pairing_system_status = await pairing_state_service.get_pairing_system_status()
|
||||
engines_status = {"ok": None, "overall": None, "engines": []}
|
||||
@@ -1005,6 +1274,7 @@ async def get_health_status(
|
||||
dinsar_bridge_status.get("ok"),
|
||||
source_roots_status.get("ok"),
|
||||
product_packages_status.get("ok"),
|
||||
asset_inventory_status.get("ok"),
|
||||
wsl_runtime_status.get("ok"),
|
||||
pairing_system_status.get("ok"),
|
||||
(not settings.TIMESERIES_ENABLED) or timeseries_result_catalog_status.get("ok"),
|
||||
@@ -1028,6 +1298,7 @@ async def get_health_status(
|
||||
"dinsar_bridge": dinsar_bridge_status,
|
||||
"source_roots": source_roots_status,
|
||||
"product_packages": product_packages_status,
|
||||
"asset_inventory": asset_inventory_status,
|
||||
"wsl_runtime": wsl_runtime_status,
|
||||
"pairing_system": pairing_system_status,
|
||||
"idl": {
|
||||
|
||||
@@ -22,6 +22,7 @@ from ..config import settings
|
||||
from ..models import SystemJobORM, DinsarResultORM, HazardPointORM, DinsarTaskItemORM, PsTaskItemORM, RadarDataORM, SARSceneGeoORM, FloodDetectionORM, WaterDetectionORM, GF3ProcessingORM, AiDiagnosisORM
|
||||
from ..scheduler import scan_data_job
|
||||
from .data_service import data_service
|
||||
from .asset_inventory_service import asset_inventory_service
|
||||
from .dinsar_compat_service import dinsar_compat_service
|
||||
from .dinsar_naming import build_run_key
|
||||
from .dinsar_production_service import dinsar_production_service
|
||||
@@ -67,6 +68,7 @@ JOB_TYPE_SCAN_DATA = "SCAN_DATA"
|
||||
JOB_TYPE_SCAN_DINSAR = "SCAN_DINSAR"
|
||||
JOB_TYPE_COPY_DATA = "COPY_DATA"
|
||||
JOB_TYPE_UNPACK = "UNPACK_ARCHIVES"
|
||||
JOB_TYPE_UNPACK_SENTINEL1 = "UNPACK_SENTINEL1"
|
||||
JOB_TYPE_AI_TRAIN = "AI_TRAIN"
|
||||
JOB_TYPE_AI_PREDICT = "AI_PREDICT"
|
||||
JOB_TYPE_AI_ANALYZE = "AI_ANALYZE"
|
||||
@@ -85,6 +87,7 @@ JOB_TYPE_PYINT_RUN = "PYINT_RUN"
|
||||
JOB_TYPE_PUBLISH_DINSAR_PRODUCTS = "PUBLISH_DINSAR_PRODUCTS"
|
||||
JOB_TYPE_REBUILD_DINSAR_CATALOG = "REBUILD_DINSAR_CATALOG"
|
||||
JOB_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG"
|
||||
JOB_TYPE_SCAN_ASSET_INVENTORY = "SCAN_ASSET_INVENTORY"
|
||||
|
||||
COPY_ALLOWED_STATUSES = {"PENDING", "IN_PROGRESS", "COMPLETED", "FAILED"}
|
||||
|
||||
@@ -225,6 +228,34 @@ async def _handle_scan_data(job: SystemJobORM) -> None:
|
||||
await _run_scan_data_custom(job.task_id, payload)
|
||||
|
||||
|
||||
async def _handle_scan_asset_inventory(job: SystemJobORM) -> None:
|
||||
if not job.task_id:
|
||||
raise ValueError("SCAN_ASSET_INVENTORY requires task_id for progress tracking.")
|
||||
payload = job.payload or {}
|
||||
await task_service.start_task(job.task_id, message="Source/orbit asset inventory scan started")
|
||||
async with AsyncSessionLocal() as db:
|
||||
result = await asset_inventory_service.scan_configured_roots(
|
||||
db,
|
||||
inventory_types=payload.get("inventory_types") or None,
|
||||
root_ids=payload.get("root_ids") or None,
|
||||
bind_orbits=bool(payload.get("bind_orbits", True)),
|
||||
task_id=job.task_id,
|
||||
)
|
||||
await task_service.update_task(
|
||||
job.task_id,
|
||||
status="COMPLETED",
|
||||
progress=100,
|
||||
message=(
|
||||
"Asset inventory scan completed: "
|
||||
f"sources={result.get('source_assets', 0)}, "
|
||||
f"orbits={result.get('orbit_assets', 0)}, "
|
||||
f"matched={((result.get('binding') or {}).get('matched_count', 0))}, "
|
||||
f"missing={((result.get('binding') or {}).get('missing_count', 0))}"
|
||||
),
|
||||
db=db,
|
||||
)
|
||||
|
||||
|
||||
def _resolve_hazard_shp_path() -> str:
|
||||
base_dir = settings.HAZARD_POINTS_DIR
|
||||
filename = settings.HAZARD_POINTS_FILENAME
|
||||
@@ -418,6 +449,16 @@ async def _handle_unpack_archives(job: SystemJobORM) -> None:
|
||||
await run_unpack_task(job.task_id)
|
||||
|
||||
|
||||
async def _handle_unpack_sentinel1(job: SystemJobORM) -> None:
|
||||
if not job.task_id:
|
||||
raise ValueError("UNPACK_SENTINEL1 requires task_id for progress tracking.")
|
||||
payload = job.payload or {}
|
||||
if payload.get("asset_id"):
|
||||
await asset_inventory_service.run_sentinel1_unpack_task(job.task_id, payload)
|
||||
return
|
||||
await asset_inventory_service.run_sentinel1_unpack_batch_task(job.task_id, payload)
|
||||
|
||||
|
||||
async def _handle_ai_train(job: SystemJobORM) -> None:
|
||||
if not job.task_id:
|
||||
raise ValueError("AI_TRAIN requires task_id for progress tracking.")
|
||||
@@ -3849,6 +3890,7 @@ async def _handle_rebuild_psinsar_catalog(job: SystemJobORM) -> None:
|
||||
|
||||
_HANDLERS = {
|
||||
JOB_TYPE_SCAN_DATA: _handle_scan_data,
|
||||
JOB_TYPE_SCAN_ASSET_INVENTORY: _handle_scan_asset_inventory,
|
||||
JOB_TYPE_SCAN_DINSAR: _handle_scan_dinsar,
|
||||
JOB_TYPE_PUBLISH_DINSAR_PRODUCTS: _handle_publish_dinsar_products_clean,
|
||||
JOB_TYPE_REBUILD_DINSAR_CATALOG: _handle_rebuild_dinsar_catalog_clean,
|
||||
@@ -3864,6 +3906,7 @@ _HANDLERS = {
|
||||
JOB_TYPE_REBUILD_PSINSAR_CATALOG: _handle_rebuild_psinsar_catalog,
|
||||
JOB_TYPE_COPY_DATA: _handle_copy_data,
|
||||
JOB_TYPE_UNPACK: _handle_unpack_archives,
|
||||
JOB_TYPE_UNPACK_SENTINEL1: _handle_unpack_sentinel1,
|
||||
JOB_TYPE_AI_TRAIN: _handle_ai_train,
|
||||
JOB_TYPE_AI_PREDICT: _handle_ai_predict,
|
||||
JOB_TYPE_AI_ANALYZE: _handle_ai_analyze,
|
||||
|
||||
@@ -36,7 +36,7 @@ def _satellite_family_expr(alias: str) -> str:
|
||||
f"COALESCE(NULLIF({alias}.satellite_family, ''), "
|
||||
f"CASE "
|
||||
f"WHEN {compact} IN ('LT1', 'LT1A', 'LT1B', 'LUTAN1', 'LUTAN1A', 'LUTAN1B') THEN 'LT1' "
|
||||
f"WHEN {compact} IN ('S1', 'S1A', 'S1B', 'SENTINEL1', 'SENTINEL1A', 'SENTINEL1B') THEN 'S1' "
|
||||
f"WHEN {compact} IN ('S1', 'S1A', 'S1B', 'S1C', 'SENTINEL1', 'SENTINEL1A', 'SENTINEL1B', 'SENTINEL1C') THEN 'S1' "
|
||||
f"WHEN NULLIF({alias}.satellite, '') IS NOT NULL THEN upper({alias}.satellite) "
|
||||
f"ELSE NULL END)"
|
||||
)
|
||||
|
||||
@@ -10,11 +10,14 @@ from typing import Any, Dict, List
|
||||
from ..config import settings
|
||||
from .orbit_converter import get_source_orbit_inventory
|
||||
from .pyint_service import (
|
||||
discover_s1_scene_sources,
|
||||
discover_lt1_archives,
|
||||
infer_scene_date_from_archives,
|
||||
infer_task_identity,
|
||||
validate_pyint_root_dir,
|
||||
)
|
||||
from .asset_inventory_service import _configured_sentinel1_archive_dirs, _parse_s1_source_name
|
||||
from ..utils import normalize_satellite_family
|
||||
|
||||
|
||||
VALID_DEM_MODES = {"local_fabdem", "opentopo", "prepared_file"}
|
||||
@@ -62,6 +65,198 @@ def _infer_satellite_from_archives(paths: List[str]) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
def _normalize_s1_satellite(value: Any) -> str:
|
||||
text = str(value or "").strip().upper().replace("-", "").replace("_", "")
|
||||
if text in {"S1A", "S1B", "S1C"}:
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
def _find_s1_zip_by_logical_uid(logical_uid: str) -> str:
|
||||
logical = str(logical_uid or "").strip()
|
||||
if not logical:
|
||||
return ""
|
||||
expected_name = logical if logical.lower().endswith(".zip") else f"{logical}.zip"
|
||||
for root in _configured_sentinel1_archive_dirs():
|
||||
if not root or not os.path.isdir(root):
|
||||
continue
|
||||
direct_candidate = os.path.join(root, expected_name)
|
||||
if os.path.isfile(direct_candidate):
|
||||
return _normalize_path(direct_candidate)
|
||||
for current_root, _, files in os.walk(root):
|
||||
if expected_name in files:
|
||||
return _normalize_path(os.path.join(current_root, expected_name))
|
||||
return ""
|
||||
|
||||
|
||||
def _is_s1_safe_dir(path: str) -> bool:
|
||||
normalized = _normalize_path(path)
|
||||
if not normalized or not os.path.isdir(normalized):
|
||||
return False
|
||||
return os.path.isfile(os.path.join(normalized, "manifest.safe"))
|
||||
|
||||
|
||||
def _resolve_s1_scene_input(
|
||||
*,
|
||||
role: str,
|
||||
scene_path: str,
|
||||
pair_meta: Dict[str, Any],
|
||||
) -> Dict[str, Any]:
|
||||
normalized_scene = _normalize_path(scene_path)
|
||||
scene_name = os.path.basename(normalized_scene.rstrip("\\/"))
|
||||
parsed = _parse_s1_source_name(scene_name) or {}
|
||||
logical_uid = str(parsed.get("logical_product_uid") or "").strip()
|
||||
resolved_path = ""
|
||||
input_kind = ""
|
||||
resolution_method = ""
|
||||
|
||||
if normalized_scene.lower().endswith(".zip") and os.path.isfile(normalized_scene):
|
||||
resolved_path = normalized_scene
|
||||
input_kind = "zip"
|
||||
resolution_method = "task_or_pair_meta"
|
||||
elif _is_s1_safe_dir(normalized_scene):
|
||||
resolved_path = normalized_scene
|
||||
input_kind = "safe_dir"
|
||||
resolution_method = "task_or_pair_meta"
|
||||
elif normalized_scene.lower().endswith(".safe") and os.path.isdir(normalized_scene):
|
||||
resolved_path = normalized_scene
|
||||
input_kind = "safe_dir"
|
||||
resolution_method = "task_or_pair_meta"
|
||||
else:
|
||||
sibling_zip = ""
|
||||
if normalized_scene.lower().endswith(".safe"):
|
||||
sibling_zip = normalized_scene[:-5] + ".zip"
|
||||
if sibling_zip and os.path.isfile(sibling_zip):
|
||||
resolved_path = _normalize_path(sibling_zip)
|
||||
input_kind = "zip"
|
||||
resolution_method = "safe_sibling_zip"
|
||||
elif logical_uid:
|
||||
looked_up_zip = _find_s1_zip_by_logical_uid(logical_uid)
|
||||
if looked_up_zip:
|
||||
resolved_path = looked_up_zip
|
||||
input_kind = "zip"
|
||||
resolution_method = "source_pool_lookup"
|
||||
|
||||
expected_name = scene_name
|
||||
if input_kind == "zip":
|
||||
if not expected_name.lower().endswith(".zip"):
|
||||
expected_name = f"{logical_uid}.zip" if logical_uid else os.path.basename(resolved_path)
|
||||
elif input_kind == "safe_dir":
|
||||
if not expected_name.lower().endswith(".safe"):
|
||||
expected_name = f"{logical_uid}.SAFE" if logical_uid else os.path.basename(resolved_path)
|
||||
|
||||
satellite = _normalize_s1_satellite(pair_meta.get(f"{role}_satellite")) or _normalize_s1_satellite(parsed.get("satellite"))
|
||||
date_text = str(pair_meta.get(f"{role}_imaging_date") or parsed.get("imaging_date") or "").strip()
|
||||
return {
|
||||
"role": role,
|
||||
"scene_path": normalized_scene,
|
||||
"scene_name": scene_name,
|
||||
"logical_product_uid": logical_uid,
|
||||
"satellite": satellite,
|
||||
"date": date_text,
|
||||
"resolved": bool(resolved_path),
|
||||
"path": resolved_path,
|
||||
"input_kind": input_kind,
|
||||
"resolution_method": resolution_method,
|
||||
"expected_name": expected_name,
|
||||
"staged_path": "",
|
||||
}
|
||||
|
||||
|
||||
def _resolve_s1_zip_file(
|
||||
*,
|
||||
role: str,
|
||||
scene_path: str,
|
||||
pair_meta: Dict[str, Any],
|
||||
) -> Dict[str, Any]:
|
||||
normalized_scene = _normalize_path(scene_path)
|
||||
scene_name = os.path.basename(normalized_scene)
|
||||
parsed = _parse_s1_source_name(scene_name) or {}
|
||||
logical_uid = str(parsed.get("logical_product_uid") or "").strip()
|
||||
direct_zip = ""
|
||||
if normalized_scene.lower().endswith(".zip") and os.path.isfile(normalized_scene):
|
||||
direct_zip = normalized_scene
|
||||
elif normalized_scene.lower().endswith(".safe"):
|
||||
sibling_zip = normalized_scene[:-5] + ".zip"
|
||||
if os.path.isfile(sibling_zip):
|
||||
direct_zip = _normalize_path(sibling_zip)
|
||||
if not direct_zip and logical_uid:
|
||||
direct_zip = _find_s1_zip_by_logical_uid(logical_uid)
|
||||
|
||||
satellite = _normalize_s1_satellite(pair_meta.get(f"{role}_satellite")) or _normalize_s1_satellite(parsed.get("satellite"))
|
||||
date_text = str(pair_meta.get(f"{role}_imaging_date") or parsed.get("imaging_date") or "").strip()
|
||||
return {
|
||||
"role": role,
|
||||
"scene_path": normalized_scene,
|
||||
"scene_name": scene_name,
|
||||
"logical_product_uid": logical_uid,
|
||||
"satellite": satellite,
|
||||
"date": date_text,
|
||||
"resolved": bool(direct_zip),
|
||||
"path": direct_zip,
|
||||
"resolution_method": "scene_or_source_pool_lookup" if direct_zip else "",
|
||||
"expected_name": f"{logical_uid}.zip" if logical_uid else "",
|
||||
"staged_path": "",
|
||||
}
|
||||
|
||||
|
||||
def _resolve_s1_orbit_file(
|
||||
*,
|
||||
role: str,
|
||||
pair_meta: Dict[str, Any],
|
||||
task_dir: str,
|
||||
) -> Dict[str, Any]:
|
||||
direct_path = _normalize_path(pair_meta.get(f"{role}_orbit_file_path"))
|
||||
if direct_path and os.path.isfile(direct_path):
|
||||
return {
|
||||
"role": role,
|
||||
"resolved": True,
|
||||
"path": direct_path,
|
||||
"resolution_method": "pair_meta",
|
||||
"expected_name": os.path.basename(direct_path),
|
||||
"satellite": _normalize_s1_satellite(pair_meta.get(f"{role}_satellite")),
|
||||
"date": str(pair_meta.get(f"{role}_imaging_date") or "").strip(),
|
||||
"staged_path": "",
|
||||
}
|
||||
|
||||
role_orbit_dir = os.path.join(task_dir, "orbit")
|
||||
satellite = _normalize_s1_satellite(pair_meta.get(f"{role}_satellite"))
|
||||
date_text = str(pair_meta.get(f"{role}_imaging_date") or "").strip()
|
||||
candidates: List[str] = []
|
||||
if os.path.isdir(role_orbit_dir):
|
||||
for entry in os.scandir(role_orbit_dir):
|
||||
if not entry.is_file():
|
||||
continue
|
||||
if not entry.name.lower().endswith(".eof"):
|
||||
continue
|
||||
if satellite and satellite not in entry.name.upper():
|
||||
continue
|
||||
candidates.append(_normalize_path(entry.path))
|
||||
candidates.sort()
|
||||
if candidates:
|
||||
return {
|
||||
"role": role,
|
||||
"resolved": True,
|
||||
"path": candidates[0],
|
||||
"resolution_method": "task_orbit_dir",
|
||||
"expected_name": os.path.basename(candidates[0]),
|
||||
"satellite": satellite,
|
||||
"date": date_text,
|
||||
"staged_path": "",
|
||||
}
|
||||
return {
|
||||
"role": role,
|
||||
"resolved": False,
|
||||
"path": "",
|
||||
"resolution_method": "",
|
||||
"expected_name": "",
|
||||
"satellite": satellite,
|
||||
"date": date_text,
|
||||
"staged_path": "",
|
||||
"error": f"{role} Sentinel-1 EOF 缺失",
|
||||
}
|
||||
|
||||
|
||||
def _get_dem_mode() -> str:
|
||||
raw_mode = str(getattr(settings, "PYINT_DEM_MODE", "local_fabdem") or "local_fabdem").strip().lower()
|
||||
if raw_mode not in VALID_DEM_MODES:
|
||||
@@ -569,6 +764,427 @@ def resolve_pyint_task_input_assets(
|
||||
}
|
||||
|
||||
|
||||
def build_pyint_input_preview(root_dir: str, num_to_process: int = 0) -> Dict[str, Any]:
|
||||
validation = validate_pyint_root_dir(root_dir, num_to_process)
|
||||
dem_summary = get_pyint_dem_summary()
|
||||
orbit_context = get_pyint_orbit_context()
|
||||
|
||||
warnings: List[str] = list(dem_summary.get("warnings") or [])
|
||||
blockers: List[str] = list(dem_summary.get("blockers") or [])
|
||||
task_summaries: List[Dict[str, Any]] = []
|
||||
resolved_task_count = 0
|
||||
missing_task_count = 0
|
||||
effective_orbit_policy = _get_orbit_policy()
|
||||
|
||||
for task_dir in validation.get("task_dirs", []) or []:
|
||||
task_summary = resolve_pyint_task_input_assets(
|
||||
task_dir,
|
||||
dem_summary=dem_summary,
|
||||
orbit_context=orbit_context,
|
||||
)
|
||||
task_summaries.append(task_summary)
|
||||
task_orbit_policy = str(((task_summary.get("input_assets") or {}).get("orbits") or {}).get("policy") or "").strip()
|
||||
if task_orbit_policy:
|
||||
effective_orbit_policy = task_orbit_policy
|
||||
if task_summary.get("warnings"):
|
||||
warnings.extend(
|
||||
f"{task_summary['task_alias']}: {item}"
|
||||
for item in task_summary["warnings"]
|
||||
)
|
||||
if task_summary.get("blockers"):
|
||||
blockers.extend(
|
||||
f"{task_summary['task_alias']}: {item}"
|
||||
for item in task_summary["blockers"]
|
||||
)
|
||||
if task_summary["input_assets"]["orbits"]["missing_count"] == 0:
|
||||
resolved_task_count += 1
|
||||
else:
|
||||
missing_task_count += 1
|
||||
|
||||
allow_submit = not blockers
|
||||
precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary()
|
||||
return {
|
||||
"root_dir": validation["root_dir"],
|
||||
"mode": validation["mode"],
|
||||
"task_count": len(task_summaries),
|
||||
"selected_task_count": len(task_summaries),
|
||||
"allow_submit": allow_submit,
|
||||
"warnings": warnings,
|
||||
"blockers": blockers,
|
||||
"invalid_candidates": validation.get("invalid_candidates", []),
|
||||
"dem": dem_summary,
|
||||
"orbits": {
|
||||
"policy": effective_orbit_policy,
|
||||
"pool_root": orbit_context.get("pool_root", ""),
|
||||
"pool_exists": bool(orbit_context.get("pool_exists")),
|
||||
"resolved_task_count": resolved_task_count,
|
||||
"missing_task_count": missing_task_count,
|
||||
"duplicate_count": int(orbit_context.get("duplicate_count", 0) or 0),
|
||||
"warnings": list(orbit_context.get("warnings") or []),
|
||||
},
|
||||
"precise_orbit_bridge": precise_orbit_bridge,
|
||||
"tasks": task_summaries,
|
||||
}
|
||||
|
||||
|
||||
def summarize_preview_blockers(preview: Dict[str, Any], limit: int = 8) -> str:
|
||||
blockers = [str(item).strip() for item in (preview.get("blockers") or []) if str(item).strip()]
|
||||
if not blockers:
|
||||
return ""
|
||||
if len(blockers) <= limit:
|
||||
return "; ".join(blockers)
|
||||
return "; ".join(blockers[:limit]) + f"; 其余 {len(blockers) - limit} 项已省略"
|
||||
|
||||
|
||||
def materialize_pyint_input_assets(
|
||||
*,
|
||||
task_summary: Dict[str, Any],
|
||||
input_assets_dir: str,
|
||||
project_name: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
input_assets_dir = _normalize_path(input_assets_dir)
|
||||
os.makedirs(input_assets_dir, exist_ok=True)
|
||||
|
||||
record_enabled = bool(getattr(settings, "PYINT_RECORD_INPUT_ASSETS", True))
|
||||
orbits_dir = os.path.join(input_assets_dir, "orbits")
|
||||
dem_dir = os.path.join(input_assets_dir, "dem")
|
||||
downloads_dir = os.path.join(input_assets_dir, "downloads")
|
||||
if record_enabled:
|
||||
os.makedirs(orbits_dir, exist_ok=True)
|
||||
os.makedirs(dem_dir, exist_ok=True)
|
||||
os.makedirs(downloads_dir, exist_ok=True)
|
||||
|
||||
manifest = _copy_json_safe(task_summary.get("input_assets") or {})
|
||||
manifest["generated_at"] = _utc_now_text()
|
||||
manifest["task_name"] = task_summary.get("task_name")
|
||||
manifest["task_alias"] = task_summary.get("task_alias")
|
||||
manifest["pair_key"] = task_summary.get("pair_key")
|
||||
manifest["task_dir"] = task_summary.get("task_dir")
|
||||
manifest["allow_submit"] = bool(task_summary.get("allow_submit"))
|
||||
manifest["warnings"] = list(task_summary.get("warnings") or [])
|
||||
manifest["blockers"] = list(task_summary.get("blockers") or [])
|
||||
|
||||
dem_summary = manifest.get("dem") or {}
|
||||
if project_name:
|
||||
dem_summary["resolved_output_dir"] = os.path.join(_normalize_path(settings.PYINT_DEM_ROOT), project_name)
|
||||
manifest["dem"] = dem_summary
|
||||
|
||||
orbits_summary = manifest.get("orbits") or {}
|
||||
staged_count = 0
|
||||
precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary()
|
||||
should_stage_orbits = record_enabled and (
|
||||
str(orbits_summary.get("policy") or "").strip().lower() == "stage_txt"
|
||||
or precise_orbit_bridge.get("enabled")
|
||||
or str((manifest.get("task_source") or {}).get("satellite_family") or "").strip().upper() == "S1"
|
||||
)
|
||||
if should_stage_orbits:
|
||||
for role in ("master", "slave"):
|
||||
orbit_item = orbits_summary.get(role) or {}
|
||||
orbit_path = _normalize_path(orbit_item.get("path"))
|
||||
expected_name = str(orbit_item.get("expected_name") or "").strip()
|
||||
if not orbit_item.get("resolved") or not orbit_path or not expected_name:
|
||||
continue
|
||||
target_path = os.path.join(orbits_dir, expected_name)
|
||||
if not os.path.exists(target_path):
|
||||
shutil.copy2(orbit_path, target_path)
|
||||
orbit_item["staged_path"] = target_path
|
||||
orbit_item["stage_operation"] = "copied"
|
||||
orbit_item["stage_reason"] = "precise_orbit_bridge" if precise_orbit_bridge.get("enabled") else "stage_txt_policy"
|
||||
staged_count += 1
|
||||
orbits_summary[role] = orbit_item
|
||||
manifest["orbits"] = orbits_summary
|
||||
|
||||
download_staged_count = 0
|
||||
task_source = manifest.get("task_source") or {}
|
||||
if record_enabled and str(task_source.get("satellite_family") or "").strip().upper() == "S1":
|
||||
production_inputs = task_source.get("production_inputs") or {}
|
||||
for role_key in ("master_scene", "slave_scene", "master_zip", "slave_zip"):
|
||||
scene_item = production_inputs.get(role_key) or {}
|
||||
scene_path = _normalize_path(scene_item.get("path"))
|
||||
expected_name = str(scene_item.get("expected_name") or os.path.basename(scene_path) or "").strip()
|
||||
if not scene_item.get("resolved") or not scene_path or not expected_name:
|
||||
continue
|
||||
input_kind = str(scene_item.get("input_kind") or "").strip().lower()
|
||||
if input_kind == "safe_dir" or os.path.isdir(scene_path):
|
||||
scene_item["staged_path"] = scene_path
|
||||
scene_item["stage_operation"] = "source_reference"
|
||||
production_inputs[role_key] = scene_item
|
||||
continue
|
||||
target_path = os.path.join(downloads_dir, expected_name)
|
||||
if not os.path.exists(target_path):
|
||||
shutil.copy2(scene_path, target_path)
|
||||
scene_item["staged_path"] = target_path
|
||||
scene_item["stage_operation"] = "copied"
|
||||
production_inputs[role_key] = scene_item
|
||||
download_staged_count += 1
|
||||
task_source["production_inputs"] = production_inputs
|
||||
manifest["task_source"] = task_source
|
||||
|
||||
materialized = {
|
||||
"input_assets_dir": input_assets_dir,
|
||||
"record_enabled": record_enabled,
|
||||
"orbits_dir": orbits_dir if record_enabled else "",
|
||||
"dem_dir": dem_dir if record_enabled else "",
|
||||
"downloads_dir": downloads_dir if record_enabled else "",
|
||||
"orbits_staged_count": staged_count,
|
||||
"downloads_staged_count": download_staged_count,
|
||||
"task_manifest_path": "",
|
||||
"dem_summary_path": "",
|
||||
"orbit_summary_path": "",
|
||||
"input_assets": manifest,
|
||||
}
|
||||
|
||||
if not record_enabled:
|
||||
return materialized
|
||||
|
||||
task_manifest_path = os.path.join(input_assets_dir, "task_manifest.json")
|
||||
dem_summary_path = os.path.join(dem_dir, "dem_summary.json")
|
||||
orbit_summary_path = os.path.join(orbits_dir, "orbit_summary.json")
|
||||
|
||||
with open(task_manifest_path, "w", encoding="utf-8") as fp:
|
||||
json.dump(manifest, fp, ensure_ascii=False, indent=2)
|
||||
fp.write("\n")
|
||||
with open(dem_summary_path, "w", encoding="utf-8") as fp:
|
||||
json.dump(dem_summary, fp, ensure_ascii=False, indent=2)
|
||||
fp.write("\n")
|
||||
with open(orbit_summary_path, "w", encoding="utf-8") as fp:
|
||||
json.dump(orbits_summary, fp, ensure_ascii=False, indent=2)
|
||||
fp.write("\n")
|
||||
|
||||
materialized.update(
|
||||
{
|
||||
"task_manifest_path": task_manifest_path,
|
||||
"dem_summary_path": dem_summary_path,
|
||||
"orbit_summary_path": orbit_summary_path,
|
||||
}
|
||||
)
|
||||
return materialized
|
||||
|
||||
|
||||
def resolve_pyint_task_input_assets(
|
||||
task_dir: str,
|
||||
*,
|
||||
dem_summary: Dict[str, Any] | None = None,
|
||||
orbit_context: Dict[str, Any] | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
task_dir = _normalize_path(task_dir)
|
||||
task_identity = infer_task_identity(task_dir)
|
||||
pair_meta = task_identity["pair_meta"]
|
||||
satellite_family = str(task_identity.get("satellite_family") or "").strip().upper()
|
||||
|
||||
warnings: List[str] = []
|
||||
blockers: List[str] = []
|
||||
|
||||
if satellite_family == "S1":
|
||||
scene_sources = discover_s1_scene_sources(task_dir)
|
||||
master_archives = list(scene_sources.get("master", []) or [])
|
||||
slave_archives = list(scene_sources.get("slave", []) or [])
|
||||
master_date = task_identity["master_date"] or infer_scene_date_from_archives(master_archives)
|
||||
slave_date = task_identity["slave_date"] or infer_scene_date_from_archives(slave_archives)
|
||||
master_satellite = _normalize_s1_satellite(pair_meta.get("master_satellite")) or _normalize_s1_satellite(task_identity.get("master_satellite"))
|
||||
slave_satellite = _normalize_s1_satellite(pair_meta.get("slave_satellite")) or _normalize_s1_satellite(task_identity.get("slave_satellite"))
|
||||
|
||||
if not master_archives:
|
||||
blockers.append("master/ 未识别到 Sentinel-1 SAFE 源目录。")
|
||||
if not slave_archives:
|
||||
blockers.append("slave/ 未识别到 Sentinel-1 SAFE 源目录。")
|
||||
if not master_date:
|
||||
blockers.append("未能识别主影像日期。")
|
||||
if not slave_date:
|
||||
blockers.append("未能识别从影像日期。")
|
||||
|
||||
master_scene = _resolve_s1_scene_input(
|
||||
role="master",
|
||||
scene_path=master_archives[0] if master_archives else "",
|
||||
pair_meta=pair_meta,
|
||||
)
|
||||
slave_scene = _resolve_s1_scene_input(
|
||||
role="slave",
|
||||
scene_path=slave_archives[0] if slave_archives else "",
|
||||
pair_meta=pair_meta,
|
||||
)
|
||||
master_orbit = _resolve_s1_orbit_file(
|
||||
role="master",
|
||||
pair_meta=pair_meta,
|
||||
task_dir=task_dir,
|
||||
)
|
||||
slave_orbit = _resolve_s1_orbit_file(
|
||||
role="slave",
|
||||
pair_meta=pair_meta,
|
||||
task_dir=task_dir,
|
||||
)
|
||||
|
||||
if not master_scene.get("resolved"):
|
||||
blockers.append("master Sentinel-1 源场景缺失。")
|
||||
if not slave_scene.get("resolved"):
|
||||
blockers.append("slave Sentinel-1 源场景缺失。")
|
||||
if not master_orbit.get("resolved"):
|
||||
blockers.append(str(master_orbit.get("error") or "master Sentinel-1 EOF 缺失"))
|
||||
if not slave_orbit.get("resolved"):
|
||||
blockers.append(str(slave_orbit.get("error") or "slave Sentinel-1 EOF 缺失"))
|
||||
|
||||
task_source = {
|
||||
"task_dir": task_dir,
|
||||
"task_name": task_identity["task_name"],
|
||||
"task_alias": task_identity["task_alias"],
|
||||
"pair_key": task_identity["pair_key"],
|
||||
"satellite_family": "S1",
|
||||
"master_date": master_date,
|
||||
"slave_date": slave_date,
|
||||
"master_satellite": master_satellite,
|
||||
"slave_satellite": slave_satellite,
|
||||
"archives": {
|
||||
"master": master_archives,
|
||||
"slave": slave_archives,
|
||||
},
|
||||
"production_inputs": {
|
||||
"master_scene": master_scene,
|
||||
"slave_scene": slave_scene,
|
||||
},
|
||||
}
|
||||
orbits_summary = {
|
||||
"policy": "require_eof",
|
||||
"pool_root": "",
|
||||
"pool_exists": True,
|
||||
"master": master_orbit,
|
||||
"slave": slave_orbit,
|
||||
"resolved_count": int(bool(master_orbit.get("resolved"))) + int(bool(slave_orbit.get("resolved"))),
|
||||
"missing_count": int(not master_orbit.get("resolved")) + int(not slave_orbit.get("resolved")),
|
||||
"warnings": [],
|
||||
"stage_mode": "copy",
|
||||
"precise_orbit_bridge": {
|
||||
"enabled": False,
|
||||
"mode": "not_applicable",
|
||||
"strict": True,
|
||||
},
|
||||
}
|
||||
else:
|
||||
archives = discover_lt1_archives(task_dir)
|
||||
master_archives = list(archives.get("master", []) or [])
|
||||
slave_archives = list(archives.get("slave", []) or [])
|
||||
master_date = task_identity["master_date"] or infer_scene_date_from_archives(master_archives)
|
||||
slave_date = task_identity["slave_date"] or infer_scene_date_from_archives(slave_archives)
|
||||
master_satellite = _normalize_lt1_satellite(pair_meta.get("master_satellite")) or _infer_satellite_from_archives(master_archives)
|
||||
slave_satellite = _normalize_lt1_satellite(pair_meta.get("slave_satellite")) or _infer_satellite_from_archives(slave_archives)
|
||||
|
||||
if not master_archives:
|
||||
blockers.append("master/ 未发现 LT-1 原始输入（LT1*.tar.gz 或 LT1*.tiff）。")
|
||||
if not slave_archives:
|
||||
blockers.append("slave/ 未发现 LT-1 原始输入（LT1*.tar.gz 或 LT1*.tiff）。")
|
||||
if not master_date:
|
||||
blockers.append("未能识别主影像日期。")
|
||||
if not slave_date:
|
||||
blockers.append("未能识别从影像日期。")
|
||||
|
||||
orbit_policy = _get_orbit_policy()
|
||||
orbit_context = orbit_context or get_pyint_orbit_context()
|
||||
orbit_pool_root = orbit_context.get("pool_root", "")
|
||||
orbit_pool_exists = bool(orbit_context.get("pool_exists"))
|
||||
orbit_files = orbit_context.get("files", {}) or {}
|
||||
|
||||
orbit_warnings: List[str] = []
|
||||
if orbit_context.get("warnings"):
|
||||
orbit_warnings.extend(str(item) for item in orbit_context["warnings"] if item)
|
||||
|
||||
master_orbit = _resolve_orbit_file(
|
||||
role="master",
|
||||
satellite=master_satellite,
|
||||
date_text=master_date,
|
||||
pool_root=orbit_pool_root,
|
||||
orbit_files=orbit_files,
|
||||
)
|
||||
slave_orbit = _resolve_orbit_file(
|
||||
role="slave",
|
||||
satellite=slave_satellite,
|
||||
date_text=slave_date,
|
||||
pool_root=orbit_pool_root,
|
||||
orbit_files=orbit_files,
|
||||
)
|
||||
|
||||
for orbit_item in (master_orbit, slave_orbit):
|
||||
if orbit_item.get("resolved"):
|
||||
continue
|
||||
message = str(orbit_item.get("error") or f"{orbit_item.get('role')} orbit missing").strip()
|
||||
if orbit_policy == "validate_only":
|
||||
orbit_warnings.append(message)
|
||||
else:
|
||||
blockers.append(message)
|
||||
|
||||
if not orbit_pool_root:
|
||||
if orbit_policy == "validate_only":
|
||||
orbit_warnings.append("轨道池未配置，当前仅记录警告。")
|
||||
else:
|
||||
blockers.append("轨道池未配置。")
|
||||
elif not orbit_pool_exists:
|
||||
if orbit_policy == "validate_only":
|
||||
orbit_warnings.append(f"轨道池目录不可用: {orbit_pool_root}")
|
||||
else:
|
||||
blockers.append(f"轨道池目录不可用: {orbit_pool_root}")
|
||||
|
||||
warnings.extend(orbit_warnings)
|
||||
|
||||
precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary()
|
||||
task_source = {
|
||||
"task_dir": task_dir,
|
||||
"task_name": task_identity["task_name"],
|
||||
"task_alias": task_identity["task_alias"],
|
||||
"pair_key": task_identity["pair_key"],
|
||||
"satellite_family": "LT1",
|
||||
"master_date": master_date,
|
||||
"slave_date": slave_date,
|
||||
"master_satellite": master_satellite,
|
||||
"slave_satellite": slave_satellite,
|
||||
"archives": {
|
||||
"master": master_archives,
|
||||
"slave": slave_archives,
|
||||
},
|
||||
}
|
||||
orbits_summary = {
|
||||
"policy": orbit_policy,
|
||||
"pool_root": orbit_pool_root,
|
||||
"pool_exists": orbit_pool_exists,
|
||||
"master": master_orbit,
|
||||
"slave": slave_orbit,
|
||||
"resolved_count": int(bool(master_orbit.get("resolved"))) + int(bool(slave_orbit.get("resolved"))),
|
||||
"missing_count": int(not master_orbit.get("resolved")) + int(not slave_orbit.get("resolved")),
|
||||
"warnings": orbit_warnings,
|
||||
"stage_mode": "copy" if orbit_policy == "stage_txt" or precise_orbit_bridge.get("enabled") else "none",
|
||||
"precise_orbit_bridge": precise_orbit_bridge,
|
||||
}
|
||||
|
||||
dem_payload = _copy_json_safe(dem_summary or get_pyint_dem_summary())
|
||||
allow_submit = not blockers and bool(dem_payload.get("allow_submit", True))
|
||||
return {
|
||||
"task_name": task_identity["task_name"],
|
||||
"task_alias": task_identity["task_alias"],
|
||||
"pair_key": task_identity["pair_key"],
|
||||
"task_dir": task_dir,
|
||||
"master_date": master_date,
|
||||
"slave_date": slave_date,
|
||||
"master_satellite": master_satellite,
|
||||
"slave_satellite": slave_satellite,
|
||||
"satellite_family": satellite_family or normalize_satellite_family(master_satellite or slave_satellite),
|
||||
"archive_counts": {
|
||||
"master": len(master_archives),
|
||||
"slave": len(slave_archives),
|
||||
},
|
||||
"warnings": warnings,
|
||||
"blockers": blockers,
|
||||
"allow_submit": allow_submit,
|
||||
"task_source": task_source,
|
||||
"dem": dem_payload,
|
||||
"orbit_resolution": {
|
||||
"master": master_orbit,
|
||||
"slave": slave_orbit,
|
||||
},
|
||||
"input_assets": {
|
||||
"task_source": task_source,
|
||||
"dem": dem_payload,
|
||||
"orbits": orbits_summary,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def build_pyint_input_preview(root_dir: str, num_to_process: int = 0) -> Dict[str, Any]:
|
||||
validation = validate_pyint_root_dir(root_dir, num_to_process)
|
||||
dem_summary = get_pyint_dem_summary()
|
||||
@@ -626,106 +1242,3 @@ def build_pyint_input_preview(root_dir: str, num_to_process: int = 0) -> Dict[st
|
||||
"precise_orbit_bridge": precise_orbit_bridge,
|
||||
"tasks": task_summaries,
|
||||
}
|
||||
|
||||
|
||||
def summarize_preview_blockers(preview: Dict[str, Any], limit: int = 8) -> str:
|
||||
blockers = [str(item).strip() for item in (preview.get("blockers") or []) if str(item).strip()]
|
||||
if not blockers:
|
||||
return ""
|
||||
if len(blockers) <= limit:
|
||||
return "; ".join(blockers)
|
||||
return "; ".join(blockers[:limit]) + f"; 其余 {len(blockers) - limit} 项已省略"
|
||||
|
||||
|
||||
def materialize_pyint_input_assets(
|
||||
*,
|
||||
task_summary: Dict[str, Any],
|
||||
input_assets_dir: str,
|
||||
project_name: str = "",
|
||||
) -> Dict[str, Any]:
|
||||
input_assets_dir = _normalize_path(input_assets_dir)
|
||||
os.makedirs(input_assets_dir, exist_ok=True)
|
||||
|
||||
record_enabled = bool(getattr(settings, "PYINT_RECORD_INPUT_ASSETS", True))
|
||||
orbits_dir = os.path.join(input_assets_dir, "orbits")
|
||||
dem_dir = os.path.join(input_assets_dir, "dem")
|
||||
if record_enabled:
|
||||
os.makedirs(orbits_dir, exist_ok=True)
|
||||
os.makedirs(dem_dir, exist_ok=True)
|
||||
|
||||
manifest = _copy_json_safe(task_summary.get("input_assets") or {})
|
||||
manifest["generated_at"] = _utc_now_text()
|
||||
manifest["task_name"] = task_summary.get("task_name")
|
||||
manifest["task_alias"] = task_summary.get("task_alias")
|
||||
manifest["pair_key"] = task_summary.get("pair_key")
|
||||
manifest["task_dir"] = task_summary.get("task_dir")
|
||||
manifest["allow_submit"] = bool(task_summary.get("allow_submit"))
|
||||
manifest["warnings"] = list(task_summary.get("warnings") or [])
|
||||
manifest["blockers"] = list(task_summary.get("blockers") or [])
|
||||
|
||||
dem_summary = manifest.get("dem") or {}
|
||||
if project_name:
|
||||
dem_summary["resolved_output_dir"] = os.path.join(_normalize_path(settings.PYINT_DEM_ROOT), project_name)
|
||||
manifest["dem"] = dem_summary
|
||||
|
||||
orbits_summary = manifest.get("orbits") or {}
|
||||
staged_count = 0
|
||||
precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary()
|
||||
should_stage_orbits = record_enabled and (
|
||||
str(orbits_summary.get("policy") or "").strip().lower() == "stage_txt"
|
||||
or precise_orbit_bridge.get("enabled")
|
||||
)
|
||||
if should_stage_orbits:
|
||||
for role in ("master", "slave"):
|
||||
orbit_item = orbits_summary.get(role) or {}
|
||||
orbit_path = _normalize_path(orbit_item.get("path"))
|
||||
expected_name = str(orbit_item.get("expected_name") or "").strip()
|
||||
if not orbit_item.get("resolved") or not orbit_path or not expected_name:
|
||||
continue
|
||||
target_path = os.path.join(orbits_dir, expected_name)
|
||||
if not os.path.exists(target_path):
|
||||
shutil.copy2(orbit_path, target_path)
|
||||
orbit_item["staged_path"] = target_path
|
||||
orbit_item["stage_operation"] = "copied"
|
||||
orbit_item["stage_reason"] = "precise_orbit_bridge" if precise_orbit_bridge.get("enabled") else "stage_txt_policy"
|
||||
staged_count += 1
|
||||
orbits_summary[role] = orbit_item
|
||||
manifest["orbits"] = orbits_summary
|
||||
|
||||
materialized = {
|
||||
"input_assets_dir": input_assets_dir,
|
||||
"record_enabled": record_enabled,
|
||||
"orbits_dir": orbits_dir if record_enabled else "",
|
||||
"dem_dir": dem_dir if record_enabled else "",
|
||||
"orbits_staged_count": staged_count,
|
||||
"task_manifest_path": "",
|
||||
"dem_summary_path": "",
|
||||
"orbit_summary_path": "",
|
||||
"input_assets": manifest,
|
||||
}
|
||||
|
||||
if not record_enabled:
|
||||
return materialized
|
||||
|
||||
task_manifest_path = os.path.join(input_assets_dir, "task_manifest.json")
|
||||
dem_summary_path = os.path.join(dem_dir, "dem_summary.json")
|
||||
orbit_summary_path = os.path.join(orbits_dir, "orbit_summary.json")
|
||||
|
||||
with open(task_manifest_path, "w", encoding="utf-8") as fp:
|
||||
json.dump(manifest, fp, ensure_ascii=False, indent=2)
|
||||
fp.write("\n")
|
||||
with open(dem_summary_path, "w", encoding="utf-8") as fp:
|
||||
json.dump(dem_summary, fp, ensure_ascii=False, indent=2)
|
||||
fp.write("\n")
|
||||
with open(orbit_summary_path, "w", encoding="utf-8") as fp:
|
||||
json.dump(orbits_summary, fp, ensure_ascii=False, indent=2)
|
||||
fp.write("\n")
|
||||
|
||||
materialized.update(
|
||||
{
|
||||
"task_manifest_path": task_manifest_path,
|
||||
"dem_summary_path": dem_summary_path,
|
||||
"orbit_summary_path": orbit_summary_path,
|
||||
}
|
||||
)
|
||||
return materialized
|
||||
|
||||
@@ -13,10 +13,12 @@ from typing import Any, Dict, Iterable, List, Optional
|
||||
|
||||
from ..config import get_env_text, read_bool_env, settings
|
||||
from .dinsar_naming import PAIR_META_FILENAME, build_fallback_pair_key, find_json_sidecar
|
||||
from ..utils import normalize_satellite_family
|
||||
from .wsl_service import run_wsl_exec
|
||||
|
||||
|
||||
LT1_INPUT_GLOBS = ("LT1*.tar.gz", "LT1*.tiff")
|
||||
S1_INPUT_GLOBS = ("S1*.zip",)
|
||||
DEFAULT_RANGE_LOOKS = 2
|
||||
DEFAULT_AZIMUTH_LOOKS = 2
|
||||
DEFAULT_DEM_RESOLUTION_M = 30.0
|
||||
@@ -334,6 +336,15 @@ def build_project_name(pair_key: str, run_key: str) -> str:
|
||||
return slugify_text(f"{pair_key}_{run_key}", default="pyint_project", max_len=120)
|
||||
|
||||
|
||||
def build_profile_project_name(satellite_family: Any, pair_key: str, run_key: str) -> str:
|
||||
family = str(normalize_satellite_family(satellite_family) or "").strip().upper()
|
||||
if family == "S1":
|
||||
return slugify_text(f"s1_{pair_key}_{run_key}", default="s1_pyint_project", max_len=120)
|
||||
if family == "LT1":
|
||||
return slugify_text(f"lt1_{pair_key}_{run_key}", default="lt1_pyint_project", max_len=120)
|
||||
return build_project_name(pair_key, run_key)
|
||||
|
||||
|
||||
def windows_path_to_wsl_mount(path: str) -> str:
|
||||
text = str(path or "").strip().strip('"').strip("'")
|
||||
if not text:
|
||||
@@ -377,6 +388,22 @@ def discover_lt1_archives(task_dir: str) -> Dict[str, List[str]]:
|
||||
return result
|
||||
|
||||
|
||||
def discover_s1_scene_sources(task_dir: str) -> Dict[str, List[str]]:
|
||||
task_path = Path(os.path.normpath(os.path.abspath(str(task_dir or "").strip())))
|
||||
pair_meta = find_json_sidecar(str(task_path), PAIR_META_FILENAME, max_levels=0) or {}
|
||||
result: Dict[str, List[str]] = {"master": [], "slave": []}
|
||||
for role in ("master", "slave"):
|
||||
explicit_path = str(pair_meta.get(f"{role}_path") or "").strip()
|
||||
role_dir = task_path / role
|
||||
candidates: List[str] = []
|
||||
if explicit_path:
|
||||
candidates.append(str(Path(explicit_path).resolve()))
|
||||
if not candidates and role_dir.is_dir() and (role_dir / "manifest.safe").is_file():
|
||||
candidates.append(str(role_dir.resolve()))
|
||||
result[role] = sorted(set(candidates))
|
||||
return result
|
||||
|
||||
|
||||
def infer_scene_date_from_archives(paths: Iterable[str]) -> str:
|
||||
dates = {
|
||||
date_text
|
||||
@@ -393,7 +420,14 @@ def infer_task_identity(task_dir: str) -> Dict[str, Any]:
|
||||
task_name = os.path.basename(os.path.normpath(task_dir))
|
||||
pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {}
|
||||
task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name
|
||||
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(task_alias, task_dir)
|
||||
master_satellite = str(pair_meta.get("master_satellite") or "").strip().upper()
|
||||
slave_satellite = str(pair_meta.get("slave_satellite") or "").strip().upper()
|
||||
satellite_family = normalize_satellite_family(master_satellite or slave_satellite)
|
||||
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(
|
||||
task_alias,
|
||||
task_dir,
|
||||
satellite_family=satellite_family,
|
||||
)
|
||||
master_date = normalize_date_text(pair_meta.get("master_imaging_date"))
|
||||
slave_date = normalize_date_text(pair_meta.get("slave_imaging_date"))
|
||||
return {
|
||||
@@ -403,6 +437,9 @@ def infer_task_identity(task_dir: str) -> Dict[str, Any]:
|
||||
"pair_meta": pair_meta,
|
||||
"master_date": master_date,
|
||||
"slave_date": slave_date,
|
||||
"master_satellite": master_satellite,
|
||||
"slave_satellite": slave_satellite,
|
||||
"satellite_family": satellite_family,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -39,6 +39,7 @@ from .dinsar_naming import (
|
||||
build_fallback_pair_key,
|
||||
find_json_sidecar,
|
||||
)
|
||||
from ..utils import normalize_satellite_family
|
||||
from .dinsar_result_layout_service import (
|
||||
RUN_CURRENT_DIRNAME,
|
||||
RUN_NATIVE_DIRNAME,
|
||||
@@ -203,10 +204,16 @@ def _resolve_candidate_identity(candidate: Dict[str, Any]) -> Dict[str, Any]:
|
||||
pair_meta.get("task_alias"),
|
||||
candidate.get("task_name"),
|
||||
) or "Task_unknown_unknown"
|
||||
satellite_family = normalize_satellite_family(
|
||||
pair_meta.get("master_satellite")
|
||||
or pair_meta.get("slave_satellite")
|
||||
or run_meta.get("master_satellite")
|
||||
or run_meta.get("slave_satellite")
|
||||
)
|
||||
pair_key = _first_text(
|
||||
run_meta.get("pair_key"),
|
||||
pair_meta.get("pair_key"),
|
||||
) or build_fallback_pair_key(task_alias, source_dir)
|
||||
) or build_fallback_pair_key(task_alias, source_dir, satellite_family=satellite_family)
|
||||
run_key = _first_text(run_meta.get("run_key")) or (
|
||||
"legacy_" + _stable_digest(candidate.get("engine_code"), pair_key, source_dir, candidate["primary_file"], length=16)
|
||||
)
|
||||
|
||||
@@ -10,7 +10,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from .. import database
|
||||
from ..config import settings, split_env_paths
|
||||
from ..models import ManagedRootORM, PathInventoryORM, ScanCursorORM
|
||||
from ..models import AssetInventoryStateORM, ManagedRootORM, PathInventoryORM, ScanCursorORM
|
||||
|
||||
|
||||
_SLUG_RE = re.compile(r"[^a-z0-9]+")
|
||||
@@ -87,6 +87,15 @@ def _cursor_type_for_scan_mode(scan_mode: str) -> str:
|
||||
return mapping.get(str(scan_mode or "").strip().lower(), "directory_walk")
|
||||
|
||||
|
||||
def _asset_inventory_type_for_root_role(root_role: str) -> Optional[str]:
|
||||
role = str(root_role or "").strip().lower()
|
||||
if role == "source_product_pool":
|
||||
return "source_product"
|
||||
if role == "orbit_asset_pool":
|
||||
return "orbit_asset"
|
||||
return None
|
||||
|
||||
|
||||
def _iter_multi_root_specs(
|
||||
*,
|
||||
env_var: str,
|
||||
@@ -169,6 +178,36 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
|
||||
scan_mode="archive_walk",
|
||||
)
|
||||
)
|
||||
source_product_paths = split_env_paths(settings.SOURCE_PRODUCT_DIRS)
|
||||
if not source_product_paths:
|
||||
source_product_paths = (
|
||||
split_env_paths(settings.INSAR_STORAGE_DIRS)
|
||||
+ split_env_paths(settings.MONITOR_RADAR_DIRS)
|
||||
)
|
||||
specs.extend(
|
||||
_iter_multi_root_specs(
|
||||
env_var="SOURCE_PRODUCT_DIRS",
|
||||
paths=source_product_paths,
|
||||
root_role="source_product_pool",
|
||||
display_prefix="Source Product Pool",
|
||||
scan_mode="file_pool",
|
||||
)
|
||||
)
|
||||
source_product_path_set = {_normalize_root_path(path) for path in source_product_paths}
|
||||
sentinel1_storage_paths = [
|
||||
path
|
||||
for path in split_env_paths(settings.SENTINEL1_STORAGE_DIRS)
|
||||
if _normalize_root_path(path) not in source_product_path_set
|
||||
]
|
||||
specs.extend(
|
||||
_iter_multi_root_specs(
|
||||
env_var="SENTINEL1_STORAGE_DIRS",
|
||||
paths=sentinel1_storage_paths,
|
||||
root_role="source_product_pool",
|
||||
display_prefix="Sentinel-1 Storage Pool",
|
||||
scan_mode="file_pool",
|
||||
)
|
||||
)
|
||||
specs.extend(
|
||||
_iter_multi_root_specs(
|
||||
env_var="INSAR_STORAGE_DIRS",
|
||||
@@ -214,6 +253,18 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
|
||||
scan_mode="scene_directory",
|
||||
)
|
||||
)
|
||||
orbit_source_paths = split_env_paths(settings.ORBIT_SOURCE_DIRS)
|
||||
if not orbit_source_paths:
|
||||
orbit_source_paths = split_env_paths(settings.MONITOR_ORBIT_DIR)
|
||||
specs.extend(
|
||||
_iter_multi_root_specs(
|
||||
env_var="ORBIT_SOURCE_DIRS",
|
||||
paths=orbit_source_paths,
|
||||
root_role="orbit_asset_pool",
|
||||
display_prefix="Orbit Asset Pool",
|
||||
scan_mode="file_pool",
|
||||
)
|
||||
)
|
||||
specs.extend(
|
||||
_iter_single_root_specs(
|
||||
env_var="MONITOR_ORBIT_DIR",
|
||||
@@ -374,6 +425,43 @@ class RootRegistryService:
|
||||
changed = True
|
||||
return "updated" if changed else None
|
||||
|
||||
async def _ensure_asset_inventory_state(self, db: AsyncSession, root: ManagedRootORM) -> Optional[str]:
|
||||
inventory_type = _asset_inventory_type_for_root_role(root.root_role)
|
||||
if not inventory_type:
|
||||
return None
|
||||
|
||||
result = await db.execute(
|
||||
select(AssetInventoryStateORM).where(
|
||||
AssetInventoryStateORM.root_ref_id == root.id,
|
||||
AssetInventoryStateORM.inventory_type == inventory_type,
|
||||
)
|
||||
)
|
||||
state = result.scalar_one_or_none()
|
||||
if state is None:
|
||||
state = AssetInventoryStateORM(
|
||||
root_ref_id=root.id,
|
||||
inventory_type=inventory_type,
|
||||
root_path=root.path,
|
||||
scan_mode=root.scan_mode,
|
||||
status="NEVER_SCANNED",
|
||||
needs_rescan=True,
|
||||
metadata_json={
|
||||
"root_role": root.root_role,
|
||||
"created_by": "root_registry_sync",
|
||||
},
|
||||
)
|
||||
db.add(state)
|
||||
return "created"
|
||||
|
||||
changed = False
|
||||
if state.root_path != root.path:
|
||||
state.root_path = root.path
|
||||
changed = True
|
||||
if state.scan_mode != root.scan_mode:
|
||||
state.scan_mode = root.scan_mode
|
||||
changed = True
|
||||
return "updated" if changed else None
|
||||
|
||||
async def sync_from_settings(self, db: Optional[AsyncSession] = None) -> Dict[str, Any]:
|
||||
generated_session = db is None
|
||||
if generated_session:
|
||||
@@ -394,6 +482,8 @@ class RootRegistryService:
|
||||
disabled = 0
|
||||
cursor_created = 0
|
||||
cursor_updated = 0
|
||||
inventory_state_created = 0
|
||||
inventory_state_updated = 0
|
||||
synced_codes: set[str] = set()
|
||||
|
||||
for spec in specs:
|
||||
@@ -448,6 +538,12 @@ class RootRegistryService:
|
||||
elif cursor_change == "updated":
|
||||
cursor_updated += 1
|
||||
|
||||
inventory_state_change = await self._ensure_asset_inventory_state(db, row)
|
||||
if inventory_state_change == "created":
|
||||
inventory_state_created += 1
|
||||
elif inventory_state_change == "updated":
|
||||
inventory_state_updated += 1
|
||||
|
||||
for row in existing_rows:
|
||||
if row.root_code in synced_codes:
|
||||
continue
|
||||
@@ -464,6 +560,7 @@ class RootRegistryService:
|
||||
"updated": updated,
|
||||
"disabled": disabled,
|
||||
"cursor_created_or_updated": cursor_created + cursor_updated,
|
||||
"asset_inventory_state_created_or_updated": inventory_state_created + inventory_state_updated,
|
||||
"summary": summary,
|
||||
}
|
||||
except Exception:
|
||||
|
||||
@@ -14,7 +14,7 @@ from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.future import select
|
||||
from sqlalchemy import and_, cast, func, or_
|
||||
from sqlalchemy import and_, case, cast, func, or_
|
||||
from sqlalchemy.orm import aliased
|
||||
|
||||
from geoalchemy2 import Geography
|
||||
@@ -43,11 +43,60 @@ from .dinsar_naming import build_pair_key, build_task_alias, ensure_unique_task_
|
||||
from .pairing_state_service import pairing_state_service
|
||||
|
||||
|
||||
PAIRING_POLICY_VERSION = "2026.05.raw-source.v1"
|
||||
PAIRING_POLICY_VERSION = "2026.05.raw-source.v2"
|
||||
PAIRING_WARNING_CANDIDATE_THRESHOLD = 3000
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _normalized_satellite_family_expr(alias):
|
||||
compact_satellite = func.upper(
|
||||
func.replace(
|
||||
func.replace(
|
||||
func.replace(func.coalesce(alias.satellite, ""), "-", ""),
|
||||
"_",
|
||||
"",
|
||||
),
|
||||
" ",
|
||||
"",
|
||||
)
|
||||
)
|
||||
inferred_family = case(
|
||||
(
|
||||
compact_satellite.in_(
|
||||
["LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"]
|
||||
),
|
||||
"LT1",
|
||||
),
|
||||
(
|
||||
compact_satellite.in_(
|
||||
[
|
||||
"S1",
|
||||
"S1A",
|
||||
"S1B",
|
||||
"S1C",
|
||||
"SENTINEL1",
|
||||
"SENTINEL1A",
|
||||
"SENTINEL1B",
|
||||
"SENTINEL1C",
|
||||
]
|
||||
),
|
||||
"S1",
|
||||
),
|
||||
else_=func.upper(alias.satellite),
|
||||
)
|
||||
return func.coalesce(func.nullif(func.upper(alias.satellite_family), ""), inferred_family)
|
||||
|
||||
|
||||
def _same_relative_orbit_expr(left_alias, right_alias):
|
||||
left_relative_orbit = func.upper(func.trim(func.coalesce(left_alias.relative_orbit, "")))
|
||||
right_relative_orbit = func.upper(func.trim(func.coalesce(right_alias.relative_orbit, "")))
|
||||
return and_(
|
||||
left_relative_orbit != "",
|
||||
right_relative_orbit != "",
|
||||
left_relative_orbit == right_relative_orbit,
|
||||
)
|
||||
|
||||
|
||||
class SpatialService:
|
||||
"""
|
||||
纯 PostGIS 空间计算服务
|
||||
@@ -155,6 +204,8 @@ class SpatialService:
|
||||
) -> List[dict]:
|
||||
master_alias = aliased(RadarDataORM)
|
||||
slave_alias = aliased(RadarDataORM)
|
||||
master_family_expr = _normalized_satellite_family_expr(master_alias)
|
||||
slave_family_expr = _normalized_satellite_family_expr(slave_alias)
|
||||
center_distance_expr = func.coalesce(
|
||||
PairingMetricCacheORM.scene_center_distance_meters,
|
||||
PairingMetricCacheORM.spatial_baseline_meters,
|
||||
@@ -192,6 +243,14 @@ class SpatialService:
|
||||
if params.require_same_polarization:
|
||||
stmt = stmt.where(PairingMetricCacheORM.same_polarization.is_(True))
|
||||
|
||||
stmt = stmt.where(
|
||||
or_(
|
||||
master_family_expr != "S1",
|
||||
slave_family_expr != "S1",
|
||||
_same_relative_orbit_expr(master_alias, slave_alias),
|
||||
)
|
||||
)
|
||||
|
||||
if params.allowed_satellites:
|
||||
allowed_satellites = [
|
||||
str(item).strip().upper()
|
||||
@@ -284,6 +343,7 @@ class SpatialService:
|
||||
slave.file_path,
|
||||
master.imaging_date,
|
||||
slave.imaging_date,
|
||||
master.satellite_family or slave.satellite_family or master.satellite or slave.satellite,
|
||||
),
|
||||
pair_uid=candidate.get("pair_uid"),
|
||||
metric_cache_ref_id=candidate.get("metric_cache_ref_id"),
|
||||
@@ -1281,7 +1341,7 @@ class SpatialService:
|
||||
compact = raw_satellite.replace("-", "").replace("_", "").replace(" ", "")
|
||||
if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}:
|
||||
return "LT1"
|
||||
if compact in {"S1", "S1A", "S1B", "SENTINEL1", "SENTINEL1A", "SENTINEL1B"}:
|
||||
if compact in {"S1", "S1A", "S1B", "S1C", "SENTINEL1", "SENTINEL1A", "SENTINEL1B", "SENTINEL1C"}:
|
||||
return "S1"
|
||||
return raw_satellite or "UNKNOWN"
|
||||
|
||||
|
||||
+40
-13
@@ -1,4 +1,6 @@
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Optional, Tuple, List, Callable, Dict, Any
|
||||
from lxml import etree
|
||||
|
||||
@@ -18,7 +20,7 @@ def _create_secure_xml_parser() -> etree.XMLParser:
|
||||
recover=False,
|
||||
)
|
||||
|
||||
# --- Sentinel-1 (S1A/S1B) Parsers ---
|
||||
# --- Sentinel-1 (S1A/S1B/S1C) Parsers ---
|
||||
|
||||
def _radar_meta_base() -> Dict[str, Any]:
|
||||
return {
|
||||
@@ -83,7 +85,7 @@ def normalize_satellite_family(value: Optional[str]) -> Optional[str]:
|
||||
compact = raw.replace("-", "").replace("_", "").replace(" ", "")
|
||||
if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}:
|
||||
return "LT1"
|
||||
if compact in {"S1", "S1A", "S1B", "SENTINEL1", "SENTINEL1A", "SENTINEL1B"}:
|
||||
if compact in {"S1", "S1A", "S1B", "S1C", "SENTINEL1", "SENTINEL1A", "SENTINEL1B", "SENTINEL1C"}:
|
||||
return "S1"
|
||||
if compact in {"GF3", "GAOFEN3"}:
|
||||
return "GF3"
|
||||
@@ -96,19 +98,44 @@ def parse_s1_radar_filename(folder_name: str) -> Optional[Dict[str, Any]]:
|
||||
Example: S1A_IW_SLC__1SDV_20250101T104105_...
|
||||
Returns a metadata dict.
|
||||
"""
|
||||
parts = folder_name.split('_')
|
||||
if len(parts) < 5 or not parts[0].startswith('S1'):
|
||||
name = os.path.basename(str(folder_name or "").strip())
|
||||
if name.lower().endswith(".zip"):
|
||||
name = name[:-4]
|
||||
if name.lower().endswith(".safe"):
|
||||
name = name[:-5]
|
||||
match = re.match(
|
||||
r"^(?P<satellite>S1[A-Z])_"
|
||||
r"(?P<mode>[A-Z0-9]+)_"
|
||||
r"(?P<product>[A-Z0-9]+)_+"
|
||||
r"(?P<class>[0-9A-Z]{4})_"
|
||||
r"(?P<start>\d{8}T\d{6}(?:\.\d+)?)_"
|
||||
r"(?P<stop>\d{8}T\d{6}(?:\.\d+)?)_"
|
||||
r"(?P<absolute_orbit>\d+)_"
|
||||
r"(?P<datatake>[0-9A-F]+)_"
|
||||
r"(?P<product_uid>[0-9A-F]+)$",
|
||||
name,
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
if not match:
|
||||
return None
|
||||
|
||||
|
||||
meta = _radar_meta_base()
|
||||
meta["satellite"] = parts[0]
|
||||
meta["satellite_family"] = normalize_satellite_family(parts[0])
|
||||
meta["imaging_date"] = parts[4].split('T')[0]
|
||||
meta["imaging_mode"] = parts[1]
|
||||
meta["source_product_token"] = parts[2]
|
||||
meta["product_type"] = parts[2]
|
||||
polarization = parts[3] # e.g., '1SDV' -> 'DV' is dual-pol VV/VH
|
||||
meta["polarization"] = polarization[2:] if len(polarization) > 2 else polarization
|
||||
meta["satellite"] = match.group("satellite").upper()
|
||||
meta["satellite_family"] = normalize_satellite_family(meta["satellite"])
|
||||
meta["imaging_date"] = match.group("start")[:8]
|
||||
meta["imaging_mode"] = match.group("mode").upper()
|
||||
meta["source_product_token"] = match.group("class").upper()
|
||||
meta["product_type"] = match.group("product").upper()
|
||||
meta["product_level"] = "L1"
|
||||
polarization = match.group("class").upper() # e.g. 1SDV -> DV
|
||||
meta["polarization"] = polarization[-2:] if len(polarization) > 2 else polarization
|
||||
meta["orbit_circle"] = match.group("absolute_orbit").lstrip("0") or match.group("absolute_orbit")
|
||||
meta["product_unique_id"] = name
|
||||
try:
|
||||
start_time = datetime.strptime(match.group("start").split(".")[0], "%Y%m%dT%H%M%S")
|
||||
meta["acquisition_time_utc"] = start_time.isoformat()
|
||||
except ValueError:
|
||||
meta["acquisition_time_utc"] = match.group("start")
|
||||
return meta
|
||||
|
||||
|
||||
|
||||
@@ -81,7 +81,7 @@ SET
|
||||
('LT1', 'LT1A', 'LT1B', 'LUTAN1', 'LUTAN1A', 'LUTAN1B')
|
||||
THEN 'LT1'
|
||||
WHEN upper(replace(replace(replace(COALESCE(satellite, ''), '-', ''), '_', ''), ' ', '')) IN
|
||||
('S1', 'S1A', 'S1B', 'SENTINEL1', 'SENTINEL1A', 'SENTINEL1B')
|
||||
('S1', 'S1A', 'S1B', 'S1C', 'SENTINEL1', 'SENTINEL1A', 'SENTINEL1B', 'SENTINEL1C')
|
||||
THEN 'S1'
|
||||
WHEN NULLIF(satellite, '') IS NOT NULL
|
||||
THEN upper(satellite)
|
||||
|
||||
@@ -0,0 +1,371 @@
|
||||
-- Migration: Source product and orbit asset inventory
|
||||
-- Version: 10.0
|
||||
-- Date: 2026-05-12
|
||||
-- Purpose: Add first-class source product assets, orbit assets, scene-orbit bindings,
|
||||
-- inventory state, and radar_data compatibility fields.
|
||||
|
||||
ALTER TABLE IF EXISTS radar_data
|
||||
ADD COLUMN IF NOT EXISTS source_product_ref_id INTEGER NULL;
|
||||
|
||||
ALTER TABLE IF EXISTS radar_data
|
||||
ADD COLUMN IF NOT EXISTS source_archive_asset_id INTEGER NULL;
|
||||
|
||||
ALTER TABLE IF EXISTS radar_data
|
||||
ADD COLUMN IF NOT EXISTS selected_orbit_asset_id INTEGER NULL;
|
||||
|
||||
ALTER TABLE IF EXISTS radar_data
|
||||
ADD COLUMN IF NOT EXISTS orbit_binding_status VARCHAR(32) NOT NULL DEFAULT 'UNBOUND';
|
||||
|
||||
ALTER TABLE IF EXISTS radar_data
|
||||
ADD COLUMN IF NOT EXISTS orbit_binding_reason TEXT NULL;
|
||||
|
||||
ALTER TABLE IF EXISTS radar_data
|
||||
ADD COLUMN IF NOT EXISTS acquisition_start_time_utc TIMESTAMP NULL;
|
||||
|
||||
ALTER TABLE IF EXISTS radar_data
|
||||
ADD COLUMN IF NOT EXISTS acquisition_stop_time_utc TIMESTAMP NULL;
|
||||
|
||||
ALTER TABLE IF EXISTS radar_data
|
||||
ADD COLUMN IF NOT EXISTS absolute_orbit VARCHAR NULL;
|
||||
|
||||
ALTER TABLE IF EXISTS radar_data
|
||||
ADD COLUMN IF NOT EXISTS relative_orbit VARCHAR NULL;
|
||||
|
||||
ALTER TABLE IF EXISTS radar_data
|
||||
ADD COLUMN IF NOT EXISTS source_format VARCHAR(32) NULL;
|
||||
|
||||
ALTER TABLE IF EXISTS radar_data
|
||||
ADD COLUMN IF NOT EXISTS metadata_json JSON NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS source_product_assets (
|
||||
id SERIAL PRIMARY KEY,
|
||||
asset_uid VARCHAR(128) NOT NULL UNIQUE,
|
||||
logical_product_uid VARCHAR(128) NULL,
|
||||
satellite_family VARCHAR(32) NULL,
|
||||
satellite VARCHAR(32) NULL,
|
||||
source_format VARCHAR(32) NOT NULL,
|
||||
product_type VARCHAR(64) NULL,
|
||||
product_level VARCHAR(64) NULL,
|
||||
imaging_mode VARCHAR(64) NULL,
|
||||
polarization VARCHAR(64) NULL,
|
||||
absolute_orbit VARCHAR(64) NULL,
|
||||
relative_orbit VARCHAR(64) NULL,
|
||||
orbit_direction VARCHAR(32) NULL,
|
||||
acquisition_start_time_utc TIMESTAMP NULL,
|
||||
acquisition_stop_time_utc TIMESTAMP NULL,
|
||||
imaging_date VARCHAR(8) NULL,
|
||||
root_ref_id INTEGER NULL REFERENCES managed_roots(id) ON DELETE SET NULL,
|
||||
root_path VARCHAR NULL,
|
||||
file_path VARCHAR NOT NULL UNIQUE,
|
||||
archive_path VARCHAR NULL,
|
||||
path_kind VARCHAR(24) NULL,
|
||||
file_name VARCHAR(255) NULL,
|
||||
file_stem VARCHAR(255) NULL,
|
||||
file_ext VARCHAR(32) NULL,
|
||||
size_bytes BIGINT NULL,
|
||||
mtime_epoch DOUBLE PRECISION NULL,
|
||||
checksum_sha256 VARCHAR(64) NULL,
|
||||
checksum_status VARCHAR(32) NOT NULL DEFAULT 'NOT_COMPUTED',
|
||||
parser_name VARCHAR(64) NULL,
|
||||
parser_version VARCHAR(32) NULL,
|
||||
parse_status VARCHAR(32) NOT NULL DEFAULT 'PENDING',
|
||||
parse_error TEXT NULL,
|
||||
parsed_at TIMESTAMP NULL,
|
||||
metadata_json JSON NULL,
|
||||
is_active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
missing_since TIMESTAMP NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMP DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS orbit_assets (
|
||||
id SERIAL PRIMARY KEY,
|
||||
orbit_uid VARCHAR(128) NOT NULL UNIQUE,
|
||||
satellite_family VARCHAR(32) NULL,
|
||||
satellite VARCHAR(32) NULL,
|
||||
orbit_type VARCHAR(64) NOT NULL,
|
||||
native_format VARCHAR(32) NOT NULL,
|
||||
quality_class VARCHAR(32) NOT NULL DEFAULT 'unknown',
|
||||
root_ref_id INTEGER NULL REFERENCES managed_roots(id) ON DELETE SET NULL,
|
||||
root_path VARCHAR NULL,
|
||||
file_path VARCHAR NOT NULL UNIQUE,
|
||||
file_name VARCHAR(255) NULL,
|
||||
file_stem VARCHAR(255) NULL,
|
||||
file_ext VARCHAR(32) NULL,
|
||||
size_bytes BIGINT NULL,
|
||||
mtime_epoch DOUBLE PRECISION NULL,
|
||||
checksum_sha256 VARCHAR(64) NULL,
|
||||
checksum_status VARCHAR(32) NOT NULL DEFAULT 'NOT_COMPUTED',
|
||||
validity_start_time_utc TIMESTAMP NULL,
|
||||
validity_stop_time_utc TIMESTAMP NULL,
|
||||
generation_time_utc TIMESTAMP NULL,
|
||||
published_time_utc TIMESTAMP NULL,
|
||||
parser_name VARCHAR(64) NULL,
|
||||
parser_version VARCHAR(32) NULL,
|
||||
parse_status VARCHAR(32) NOT NULL DEFAULT 'PENDING',
|
||||
parse_error TEXT NULL,
|
||||
parsed_at TIMESTAMP NULL,
|
||||
metadata_json JSON NULL,
|
||||
is_active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
missing_since TIMESTAMP NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMP DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS scene_orbit_bindings (
|
||||
id SERIAL PRIMARY KEY,
|
||||
radar_data_id INTEGER NOT NULL REFERENCES radar_data(id) ON DELETE CASCADE,
|
||||
orbit_asset_id INTEGER NOT NULL REFERENCES orbit_assets(id) ON DELETE CASCADE,
|
||||
binding_role VARCHAR(32) NOT NULL DEFAULT 'primary_orbit',
|
||||
match_status VARCHAR(32) NOT NULL DEFAULT 'CANDIDATE',
|
||||
selection_status VARCHAR(32) NOT NULL DEFAULT 'CANDIDATE',
|
||||
selection_rank INTEGER NULL,
|
||||
priority_score DOUBLE PRECISION NULL,
|
||||
coverage_margin_before_seconds DOUBLE PRECISION NULL,
|
||||
coverage_margin_after_seconds DOUBLE PRECISION NULL,
|
||||
match_rule_version VARCHAR(64) NULL,
|
||||
match_reason TEXT NULL,
|
||||
selected_at TIMESTAMP NULL,
|
||||
metadata_json JSON NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMP DEFAULT NOW(),
|
||||
CONSTRAINT uq_scene_orbit_binding_role UNIQUE (radar_data_id, orbit_asset_id, binding_role)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS orbit_asset_derivatives (
|
||||
id SERIAL PRIMARY KEY,
|
||||
orbit_asset_id INTEGER NOT NULL REFERENCES orbit_assets(id) ON DELETE CASCADE,
|
||||
engine_code VARCHAR(32) NOT NULL,
|
||||
derivative_format VARCHAR(32) NOT NULL,
|
||||
derivative_role VARCHAR(64) NULL,
|
||||
pool_path VARCHAR NOT NULL,
|
||||
size_bytes BIGINT NULL,
|
||||
mtime_epoch DOUBLE PRECISION NULL,
|
||||
checksum_sha256 VARCHAR(64) NULL,
|
||||
generation_status VARCHAR(32) NOT NULL DEFAULT 'PENDING',
|
||||
generation_error TEXT NULL,
|
||||
generated_at TIMESTAMP NULL,
|
||||
metadata_json JSON NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMP DEFAULT NOW(),
|
||||
CONSTRAINT uq_orbit_asset_derivative_pool_path UNIQUE (orbit_asset_id, engine_code, derivative_format, pool_path)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS asset_inventory_states (
|
||||
id SERIAL PRIMARY KEY,
|
||||
root_ref_id INTEGER NOT NULL REFERENCES managed_roots(id) ON DELETE CASCADE,
|
||||
inventory_type VARCHAR(32) NOT NULL,
|
||||
root_path VARCHAR NOT NULL,
|
||||
scan_mode VARCHAR(32) NOT NULL DEFAULT 'file_pool',
|
||||
status VARCHAR(32) NOT NULL DEFAULT 'NEVER_SCANNED',
|
||||
last_scan_started_at TIMESTAMP NULL,
|
||||
last_scan_finished_at TIMESTAMP NULL,
|
||||
last_seen_entry_count INTEGER NULL,
|
||||
last_asset_count INTEGER NULL,
|
||||
last_issue_count INTEGER NULL,
|
||||
parser_version VARCHAR(32) NULL,
|
||||
needs_rescan BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
last_error TEXT NULL,
|
||||
metadata_json JSON NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMP DEFAULT NOW(),
|
||||
CONSTRAINT uq_asset_inventory_state_root_type UNIQUE (root_ref_id, inventory_type)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS asset_inventory_issues (
|
||||
id SERIAL PRIMARY KEY,
|
||||
root_ref_id INTEGER NULL REFERENCES managed_roots(id) ON DELETE SET NULL,
|
||||
inventory_type VARCHAR(32) NOT NULL,
|
||||
asset_ref_id INTEGER NULL REFERENCES source_product_assets(id) ON DELETE SET NULL,
|
||||
radar_data_id INTEGER NULL REFERENCES radar_data(id) ON DELETE SET NULL,
|
||||
orbit_asset_id INTEGER NULL REFERENCES orbit_assets(id) ON DELETE SET NULL,
|
||||
severity VARCHAR(16) NOT NULL DEFAULT 'warning',
|
||||
issue_code VARCHAR(64) NOT NULL,
|
||||
issue_message TEXT NULL,
|
||||
source_path VARCHAR NULL,
|
||||
status VARCHAR(16) NOT NULL DEFAULT 'OPEN',
|
||||
first_seen_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
last_seen_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
resolved_at TIMESTAMP NULL,
|
||||
metadata_json JSON NULL
|
||||
);
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_constraint WHERE conname = 'fk_radar_data_source_product_ref_id'
|
||||
) THEN
|
||||
ALTER TABLE radar_data
|
||||
ADD CONSTRAINT fk_radar_data_source_product_ref_id
|
||||
FOREIGN KEY (source_product_ref_id)
|
||||
REFERENCES source_product_assets(id)
|
||||
ON DELETE SET NULL;
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_constraint WHERE conname = 'fk_radar_data_source_archive_asset_id'
|
||||
) THEN
|
||||
ALTER TABLE radar_data
|
||||
ADD CONSTRAINT fk_radar_data_source_archive_asset_id
|
||||
FOREIGN KEY (source_archive_asset_id)
|
||||
REFERENCES source_product_assets(id)
|
||||
ON DELETE SET NULL;
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_constraint WHERE conname = 'fk_radar_data_selected_orbit_asset_id'
|
||||
) THEN
|
||||
ALTER TABLE radar_data
|
||||
ADD CONSTRAINT fk_radar_data_selected_orbit_asset_id
|
||||
FOREIGN KEY (selected_orbit_asset_id)
|
||||
REFERENCES orbit_assets(id)
|
||||
ON DELETE SET NULL;
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_radar_data_source_product_ref
|
||||
ON radar_data (source_product_ref_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_radar_data_source_archive_asset
|
||||
ON radar_data (source_archive_asset_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_radar_data_selected_orbit_asset
|
||||
ON radar_data (selected_orbit_asset_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_radar_data_orbit_binding_status
|
||||
ON radar_data (orbit_binding_status);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_radar_data_acquisition_start
|
||||
ON radar_data (acquisition_start_time_utc);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_radar_data_absolute_orbit
|
||||
ON radar_data (absolute_orbit);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_radar_data_relative_orbit
|
||||
ON radar_data (relative_orbit);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_radar_data_source_format
|
||||
ON radar_data (source_format);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_product_assets_asset_uid
|
||||
ON source_product_assets (asset_uid);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_product_assets_family_date
|
||||
ON source_product_assets (satellite_family, imaging_date);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_product_assets_satellite
|
||||
ON source_product_assets (satellite);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_product_assets_source_format
|
||||
ON source_product_assets (source_format);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_product_assets_parse_status
|
||||
ON source_product_assets (parse_status);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_product_assets_root_active
|
||||
ON source_product_assets (root_ref_id, is_active);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_product_assets_logical_product
|
||||
ON source_product_assets (logical_product_uid);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_source_product_assets_file_path
|
||||
ON source_product_assets (file_path);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_orbit_assets_orbit_uid
|
||||
ON orbit_assets (orbit_uid);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_orbit_assets_family_sat_window
|
||||
ON orbit_assets (satellite_family, satellite, validity_start_time_utc, validity_stop_time_utc);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_orbit_assets_orbit_type
|
||||
ON orbit_assets (orbit_type);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_orbit_assets_native_format
|
||||
ON orbit_assets (native_format);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_orbit_assets_quality_class
|
||||
ON orbit_assets (quality_class);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_orbit_assets_parse_status
|
||||
ON orbit_assets (parse_status);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_orbit_assets_root_active
|
||||
ON orbit_assets (root_ref_id, is_active);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_orbit_assets_file_path
|
||||
ON orbit_assets (file_path);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_scene_orbit_bindings_radar
|
||||
ON scene_orbit_bindings (radar_data_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_scene_orbit_bindings_orbit
|
||||
ON scene_orbit_bindings (orbit_asset_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_scene_orbit_bindings_match_status
|
||||
ON scene_orbit_bindings (match_status);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_scene_orbit_bindings_selection_status
|
||||
ON scene_orbit_bindings (selection_status);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_scene_orbit_bindings_scene_selected
|
||||
ON scene_orbit_bindings (radar_data_id, selection_status);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_orbit_asset_derivatives_asset_engine
|
||||
ON orbit_asset_derivatives (orbit_asset_id, engine_code);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_orbit_asset_derivatives_pool_path
|
||||
ON orbit_asset_derivatives (pool_path);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_orbit_asset_derivatives_generation_status
|
||||
ON orbit_asset_derivatives (generation_status);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_asset_inventory_states_type_status
|
||||
ON asset_inventory_states (inventory_type, status);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_asset_inventory_states_needs_rescan
|
||||
ON asset_inventory_states (needs_rescan);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_asset_inventory_issues_open
|
||||
ON asset_inventory_issues (status, severity);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_asset_inventory_issues_root_type
|
||||
ON asset_inventory_issues (root_ref_id, inventory_type);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_asset_inventory_issues_asset
|
||||
ON asset_inventory_issues (asset_ref_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_asset_inventory_issues_orbit
|
||||
ON asset_inventory_issues (orbit_asset_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_asset_inventory_issues_radar
|
||||
ON asset_inventory_issues (radar_data_id);
|
||||
|
||||
UPDATE radar_data
|
||||
SET
|
||||
orbit_binding_status = CASE
|
||||
WHEN has_orbit_data IS TRUE AND NULLIF(orbit_file_path, '') IS NOT NULL THEN 'MATCHED'
|
||||
WHEN has_orbit_data IS TRUE THEN 'MATCHED'
|
||||
ELSE COALESCE(NULLIF(orbit_binding_status, ''), 'UNBOUND')
|
||||
END
|
||||
WHERE orbit_binding_status IS NULL
|
||||
OR orbit_binding_status = ''
|
||||
OR (has_orbit_data IS TRUE AND orbit_binding_status = 'UNBOUND');
|
||||
|
||||
UPDATE radar_data
|
||||
SET absolute_orbit = COALESCE(NULLIF(absolute_orbit, ''), NULLIF(orbit_circle, ''))
|
||||
WHERE absolute_orbit IS NULL OR absolute_orbit = '';
|
||||
|
||||
UPDATE radar_data
|
||||
SET source_format = COALESCE(
|
||||
NULLIF(source_format, ''),
|
||||
CASE
|
||||
WHEN upper(replace(replace(replace(COALESCE(satellite, ''), '-', ''), '_', ''), ' ', '')) LIKE 'S1%%'
|
||||
AND lower(COALESCE(file_path, '')) LIKE '%%.zip' THEN 'S1_ZIP'
|
||||
WHEN upper(replace(replace(replace(COALESCE(satellite, ''), '-', ''), '_', ''), ' ', '')) LIKE 'S1%%'
|
||||
AND lower(COALESCE(file_path, '')) LIKE '%%.safe' THEN 'S1_SAFE_DIR'
|
||||
WHEN upper(replace(replace(replace(COALESCE(satellite, ''), '-', ''), '_', ''), ' ', '')) LIKE 'LT1%%' THEN 'LT1_DIR'
|
||||
WHEN upper(replace(replace(replace(COALESCE(satellite, ''), '-', ''), '_', ''), ' ', '')) LIKE 'GF3%%' THEN 'GF3_DIR'
|
||||
ELSE NULL
|
||||
END
|
||||
)
|
||||
WHERE source_format IS NULL OR source_format = '';
|
||||
Reference in New Issue
Block a user