Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Sentinel-1 asset management and PyINT pipeline support

Browse Source
This commit is contained in:
Harmon
2026-05-14 03:02:56 +08:00
parent 508ca5641a
commit 729c5c2a7f
61 changed files with 7800 additions and 885 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/asset_inventory_service.py
+2410
View File
File diff suppressed because it is too large Load Diff
backend/app/services/data_service.py
+12 -1
View File
@@ -238,7 +238,7 @@ def _chunked(items: List[Any], size: int):
_RADAR_PREVIEW_EXTENSIONS = (".jpg", ".jpeg", ".png", ".webp", ".bmp", ".tif", ".tiff")
_RADAR_PREVIEW_KEYWORDS = ("quicklook", "preview", "browse", "thumbnail", "thumb", "overview")
_RADAR_PREVIEW_KEYWORDS = ("quicklook", "quick-look", "preview", "browse", "thumbnail", "thumb", "overview")
_RADAR_CACHE_NAME_RE = re.compile(r"[^A-Za-z0-9._-]+")
@@ -396,7 +396,18 @@ class DataService:
continue
path = os.path.join(root, name)
root_lower = root.lower()
keyword_score = 0 if any(key in lower_name for key in _RADAR_PREVIEW_KEYWORDS) else 1
if lower_name == "quick-look.png":
keyword_score = -3
elif lower_name == "quicklook.png":
keyword_score = -2
elif lower_name.startswith("quick-look.") or lower_name.startswith("quicklook."):
keyword_score = min(keyword_score, -1)
if f"{os.sep}preview" in root_lower:
keyword_score -= 1
if f"{os.sep}icons" in root_lower:
keyword_score += 2
ext_score = 0 if lower_name.endswith((".jpg", ".jpeg")) else 1
try:
size_score = -os.path.getsize(path)
backend/app/services/dinsar_naming.py
+12 -3
View File
@@ -7,6 +7,8 @@ import re
from datetime import datetime
from typing import Any, Dict, Optional
from ..utils import normalize_satellite_family
PAIR_META_FILENAME = ".dinsar_pair.json"
RUN_META_FILENAME = ".dinsar_run.json"
@@ -48,6 +50,7 @@ def build_pair_key(
slave_path: Any,
master_date: Any = None,
slave_date: Any = None,
satellite_family: Any = None,
) -> str:
master_date_text = _normalize_date(master_date)
slave_date_text = _normalize_date(slave_date)
@@ -60,17 +63,23 @@ def build_pair_key(
]
)
digest = hashlib.sha1(payload.encode("utf-8", errors="ignore")).hexdigest()[:10]
return f"lt1_{master_date_text}_{slave_date_text}_{digest}"
family = str(normalize_satellite_family(satellite_family) or "").strip().lower()
if not family:
family = "pair"
return f"{family}_{master_date_text}_{slave_date_text}_{digest}"
def build_fallback_pair_key(task_alias: Any, source_hint: Any = None) -> str:
def build_fallback_pair_key(task_alias: Any, source_hint: Any = None, satellite_family: Any = None) -> str:
alias = str(task_alias or "").strip() or "Task_unknown_unknown"
parts = alias.split("_")
master_date = parts[1] if len(parts) > 2 else "unknown"
slave_date = parts[2] if len(parts) > 2 else "unknown"
payload = "||".join([alias, normalize_path(source_hint)])
digest = hashlib.sha1(payload.encode("utf-8", errors="ignore")).hexdigest()[:10]
return f"lt1_{_normalize_date(master_date)}_{_normalize_date(slave_date)}_{digest}"
family = str(normalize_satellite_family(satellite_family) or "").strip().lower()
if not family:
family = "pair"
return f"{family}_{_normalize_date(master_date)}_{_normalize_date(slave_date)}_{digest}"
def build_run_key(
backend/app/services/envi_service.py
+9 -1
View File
@@ -21,6 +21,7 @@ from typing import Any, Dict, List, Optional
from ..config import get_env_text, settings
from ..process_utils import is_any_process_running
from ..utils import normalize_satellite_family
from .dinsar_naming import (
PAIR_META_FILENAME,
build_fallback_pair_key,
@@ -480,7 +481,14 @@ def _utc_now_text() -> str:
def _resolve_dinsar_pair_identity(task_dir: str, task_name: str) -> tuple[str, str, Dict[str, Any]]:
pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {}
task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(task_alias, task_dir)
satellite_family = normalize_satellite_family(
pair_meta.get("master_satellite") or pair_meta.get("slave_satellite")
)
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(
task_alias,
task_dir,
satellite_family=satellite_family,
)
return task_alias, pair_key, pair_meta
backend/app/services/health_service.py
+272 -1
View File
@@ -1,7 +1,7 @@
import os
import asyncio
from datetime import datetime, timedelta
from typing import Any, Dict, Optional
from typing import Any, Dict, List, Optional
import httpx
from sqlalchemy import and_, func, literal, or_, select, text
@@ -11,9 +11,14 @@ from ..config import read_int_env, settings, split_env_paths
from ..db_maintenance import inspect_database_structure
from ..models import (
AiDiagnosisORM,
AssetInventoryIssueORM,
AssetInventoryStateORM,
DinsarResultORM,
OrbitAssetORM,
ResultCatalogStateORM,
ResultProductORM,
SceneOrbitBindingORM,
SourceProductAssetORM,
SystemWorkerHeartbeatORM,
)
from ..idl_service import get_idl_status
@@ -399,6 +404,51 @@ def _sanitize_pairing_system_status(payload: Dict[str, Any]) -> Dict[str, Any]:
}
def _sanitize_asset_inventory_status(payload: Dict[str, Any]) -> Dict[str, Any]:
source_roots = payload.get("source_roots", {}) or {}
orbit_roots = payload.get("orbit_roots", {}) or {}
source_assets = payload.get("source_assets", {}) or {}
orbit_assets = payload.get("orbit_assets", {}) or {}
bindings = payload.get("bindings", {}) or {}
issues = payload.get("issues", {}) or {}
return {
"ok": bool(payload.get("ok")),
"source_roots": {
"configured_count": int(source_roots.get("configured_count") or 0),
"accessible_count": int(source_roots.get("accessible_count") or 0),
"needs_rescan_count": int(source_roots.get("needs_rescan_count") or 0),
},
"orbit_roots": {
"configured_count": int(orbit_roots.get("configured_count") or 0),
"accessible_count": int(orbit_roots.get("accessible_count") or 0),
"needs_rescan_count": int(orbit_roots.get("needs_rescan_count") or 0),
},
"source_assets": {
"total_count": int(source_assets.get("total_count") or 0),
"lt1_count": int(source_assets.get("lt1_count") or 0),
"s1_count": int(source_assets.get("s1_count") or 0),
"parse_failed_count": int(source_assets.get("parse_failed_count") or 0),
},
"orbit_assets": {
"total_count": int(orbit_assets.get("total_count") or 0),
"lt1_count": int(orbit_assets.get("lt1_count") or 0),
"s1_count": int(orbit_assets.get("s1_count") or 0),
"parse_failed_count": int(orbit_assets.get("parse_failed_count") or 0),
},
"bindings": {
"scene_count": int(bindings.get("scene_count") or 0),
"matched_count": int(bindings.get("matched_count") or 0),
"missing_count": int(bindings.get("missing_count") or 0),
"ambiguous_count": int(bindings.get("ambiguous_count") or 0),
},
"issues": {
"open_count": int(issues.get("open_count") or 0),
"error_count": int(issues.get("error_count") or 0),
"warning_count": int(issues.get("warning_count") or 0),
},
}
def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
database = payload.get("database", {}) or {}
worker = payload.get("worker", {}) or {}
@@ -411,6 +461,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
dinsar_bridge = payload.get("dinsar_bridge", {}) or {}
source_roots = payload.get("source_roots", {}) or {}
product_packages = payload.get("product_packages", {}) or {}
asset_inventory = payload.get("asset_inventory", {}) or {}
wsl_runtime = payload.get("wsl_runtime", {}) or {}
pairing_system = payload.get("pairing_system", {}) or {}
idl = payload.get("idl", {}) or {}
@@ -423,6 +474,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
sanitized_dinsar_bridge = _sanitize_bridge_status(dinsar_bridge)
sanitized_source_roots = _sanitize_source_roots_status(source_roots)
sanitized_product_packages = _sanitize_product_package_status(product_packages)
sanitized_asset_inventory = _sanitize_asset_inventory_status(asset_inventory)
sanitized_wsl_runtime = _sanitize_wsl_runtime_status(wsl_runtime)
sanitized_pairing_system = _sanitize_pairing_system_status(pairing_system)
@@ -451,6 +503,7 @@ def _sanitize_health_status(payload: Dict[str, Any]) -> Dict[str, Any]:
"dinsar_bridge": sanitized_dinsar_bridge,
"source_roots": sanitized_source_roots,
"product_packages": sanitized_product_packages,
"asset_inventory": sanitized_asset_inventory,
"wsl_runtime": sanitized_wsl_runtime,
"pairing_system": sanitized_pairing_system,
"idl": {
@@ -905,6 +958,221 @@ async def _check_product_packages() -> Dict[str, Any]:
return status
async def _check_asset_inventory() -> Dict[str, Any]:
status: Dict[str, Any] = {
"ok": False,
"source_roots": {
"configured_count": 0,
"accessible_count": 0,
"inaccessible_count": 0,
"needs_rescan_count": 0,
"items": [],
},
"orbit_roots": {
"configured_count": 0,
"accessible_count": 0,
"inaccessible_count": 0,
"needs_rescan_count": 0,
"items": [],
},
"source_assets": {
"total_count": 0,
"lt1_count": 0,
"s1_count": 0,
"parse_failed_count": 0,
"by_family": {},
},
"orbit_assets": {
"total_count": 0,
"lt1_count": 0,
"s1_count": 0,
"parse_failed_count": 0,
"by_family": {},
},
"bindings": {
"scene_count": 0,
"matched_count": 0,
"missing_count": 0,
"ambiguous_count": 0,
},
"issues": {
"open_count": 0,
"error_count": 0,
"warning_count": 0,
"by_code": {},
},
"error": None,
}
source_paths: List[str] = []
for value in (
settings.SOURCE_PRODUCT_DIRS,
settings.INSAR_STORAGE_DIRS,
settings.MONITOR_RADAR_DIRS,
):
for path in split_env_paths(value):
if path not in source_paths:
source_paths.append(path)
orbit_paths: List[str] = []
for value in (
settings.ORBIT_SOURCE_DIRS,
settings.MONITOR_ORBIT_DIR,
):
for path in split_env_paths(value):
if path not in orbit_paths:
orbit_paths.append(path)
for path in source_paths:
item = _probe_directory_status(path)
item["role"] = "source_product_pool"
status["source_roots"]["items"].append(item)
for path in orbit_paths:
item = _probe_directory_status(path)
item["role"] = "orbit_asset_pool"
status["orbit_roots"]["items"].append(item)
for key in ("source_roots", "orbit_roots"):
root_status = status[key]
root_status["configured_count"] = len(root_status["items"])
root_status["accessible_count"] = sum(1 for item in root_status["items"] if item.get("accessible"))
root_status["inaccessible_count"] = root_status["configured_count"] - root_status["accessible_count"]
try:
session_factory = _get_session_factory()
async with session_factory() as db:
state_rows = (
await db.execute(
select(
AssetInventoryStateORM.inventory_type,
func.count(AssetInventoryStateORM.id),
)
.where(AssetInventoryStateORM.needs_rescan == True) # noqa: E712
.group_by(AssetInventoryStateORM.inventory_type)
)
).all()
for inventory_type, count in state_rows:
key = "orbit_roots" if str(inventory_type or "").lower().startswith("orbit") else "source_roots"
status[key]["needs_rescan_count"] += int(count or 0)
source_family_rows = (
await db.execute(
select(
SourceProductAssetORM.satellite_family,
func.count(SourceProductAssetORM.id),
)
.where(
SourceProductAssetORM.is_active == True, # noqa: E712
SourceProductAssetORM.source_format != "S1_ZIP",
)
.group_by(SourceProductAssetORM.satellite_family)
)
).all()
for family, count in source_family_rows:
family_key = str(family or "unknown").strip().upper() or "unknown"
value = int(count or 0)
status["source_assets"]["by_family"][family_key] = value
status["source_assets"]["total_count"] += value
status["source_assets"]["lt1_count"] = int(status["source_assets"]["by_family"].get("LT1", 0))
status["source_assets"]["s1_count"] = int(status["source_assets"]["by_family"].get("S1", 0))
source_parse_failed = await db.execute(
select(func.count(SourceProductAssetORM.id)).where(
SourceProductAssetORM.parse_status == "FAILED",
SourceProductAssetORM.source_format != "S1_ZIP",
)
)
status["source_assets"]["parse_failed_count"] = int(source_parse_failed.scalar_one() or 0)
orbit_family_rows = (
await db.execute(
select(
OrbitAssetORM.satellite_family,
func.count(OrbitAssetORM.id),
)
.where(OrbitAssetORM.is_active == True) # noqa: E712
.group_by(OrbitAssetORM.satellite_family)
)
).all()
for family, count in orbit_family_rows:
family_key = str(family or "unknown").strip().upper() or "unknown"
value = int(count or 0)
status["orbit_assets"]["by_family"][family_key] = value
status["orbit_assets"]["total_count"] += value
status["orbit_assets"]["lt1_count"] = int(status["orbit_assets"]["by_family"].get("LT1", 0))
status["orbit_assets"]["s1_count"] = int(status["orbit_assets"]["by_family"].get("S1", 0))
orbit_parse_failed = await db.execute(
select(func.count(OrbitAssetORM.id)).where(OrbitAssetORM.parse_status == "FAILED")
)
status["orbit_assets"]["parse_failed_count"] = int(orbit_parse_failed.scalar_one() or 0)
scene_count = await db.execute(select(func.count(SceneOrbitBindingORM.radar_data_id.distinct())))
status["bindings"]["scene_count"] = int(scene_count.scalar_one() or 0)
selected_count = await db.execute(
select(func.count(SceneOrbitBindingORM.id)).where(SceneOrbitBindingORM.selection_status == "SELECTED")
)
status["bindings"]["matched_count"] = int(selected_count.scalar_one() or 0)
missing_count = await db.execute(
select(func.count(AssetInventoryIssueORM.id)).where(
AssetInventoryIssueORM.status == "OPEN",
AssetInventoryIssueORM.issue_code == "scene_missing_orbit",
)
)
status["bindings"]["missing_count"] = int(missing_count.scalar_one() or 0)
ambiguous_count = await db.execute(
select(func.count(AssetInventoryIssueORM.id)).where(
AssetInventoryIssueORM.status == "OPEN",
AssetInventoryIssueORM.issue_code == "scene_ambiguous_orbit",
)
)
status["bindings"]["ambiguous_count"] = int(ambiguous_count.scalar_one() or 0)
issue_rows = (
await db.execute(
select(
AssetInventoryIssueORM.severity,
func.count(AssetInventoryIssueORM.id),
)
.where(AssetInventoryIssueORM.status == "OPEN")
.group_by(AssetInventoryIssueORM.severity)
)
).all()
for severity, count in issue_rows:
severity_key = str(severity or "warning").strip().lower() or "warning"
value = int(count or 0)
status["issues"]["open_count"] += value
if severity_key == "error":
status["issues"]["error_count"] += value
elif severity_key == "warning":
status["issues"]["warning_count"] += value
issue_code_rows = (
await db.execute(
select(
AssetInventoryIssueORM.issue_code,
func.count(AssetInventoryIssueORM.id),
)
.where(AssetInventoryIssueORM.status == "OPEN")
.group_by(AssetInventoryIssueORM.issue_code)
)
).all()
status["issues"]["by_code"] = {
str(code or "unknown"): int(count or 0)
for code, count in issue_code_rows
}
status["ok"] = (
status["source_roots"]["inaccessible_count"] == 0
and status["orbit_roots"]["inaccessible_count"] == 0
and status["source_assets"]["parse_failed_count"] == 0
and status["orbit_assets"]["parse_failed_count"] == 0
and status["issues"]["error_count"] == 0
)
except Exception as exc:
status["error"] = str(exc)
return status
async def _check_wsl_runtime() -> Dict[str, Any]:
status = {
"ok": False,
@@ -989,6 +1257,7 @@ async def get_health_status(
dinsar_bridge_status = await _check_dinsar_bridge()
source_roots_status = await _check_source_roots()
product_packages_status = await _check_product_packages()
asset_inventory_status = await _check_asset_inventory()
wsl_runtime_status = await _check_wsl_runtime()
pairing_system_status = await pairing_state_service.get_pairing_system_status()
engines_status = {"ok": None, "overall": None, "engines": []}
@@ -1005,6 +1274,7 @@ async def get_health_status(
dinsar_bridge_status.get("ok"),
source_roots_status.get("ok"),
product_packages_status.get("ok"),
asset_inventory_status.get("ok"),
wsl_runtime_status.get("ok"),
pairing_system_status.get("ok"),
(not settings.TIMESERIES_ENABLED) or timeseries_result_catalog_status.get("ok"),
@@ -1028,6 +1298,7 @@ async def get_health_status(
"dinsar_bridge": dinsar_bridge_status,
"source_roots": source_roots_status,
"product_packages": product_packages_status,
"asset_inventory": asset_inventory_status,
"wsl_runtime": wsl_runtime_status,
"pairing_system": pairing_system_status,
"idl": {
backend/app/services/job_handlers.py
+43
View File
@@ -22,6 +22,7 @@ from ..config import settings
from ..models import SystemJobORM, DinsarResultORM, HazardPointORM, DinsarTaskItemORM, PsTaskItemORM, RadarDataORM, SARSceneGeoORM, FloodDetectionORM, WaterDetectionORM, GF3ProcessingORM, AiDiagnosisORM
from ..scheduler import scan_data_job
from .data_service import data_service
from .asset_inventory_service import asset_inventory_service
from .dinsar_compat_service import dinsar_compat_service
from .dinsar_naming import build_run_key
from .dinsar_production_service import dinsar_production_service
@@ -67,6 +68,7 @@ JOB_TYPE_SCAN_DATA = "SCAN_DATA"
JOB_TYPE_SCAN_DINSAR = "SCAN_DINSAR"
JOB_TYPE_COPY_DATA = "COPY_DATA"
JOB_TYPE_UNPACK = "UNPACK_ARCHIVES"
JOB_TYPE_UNPACK_SENTINEL1 = "UNPACK_SENTINEL1"
JOB_TYPE_AI_TRAIN = "AI_TRAIN"
JOB_TYPE_AI_PREDICT = "AI_PREDICT"
JOB_TYPE_AI_ANALYZE = "AI_ANALYZE"
@@ -85,6 +87,7 @@ JOB_TYPE_PYINT_RUN = "PYINT_RUN"
JOB_TYPE_PUBLISH_DINSAR_PRODUCTS = "PUBLISH_DINSAR_PRODUCTS"
JOB_TYPE_REBUILD_DINSAR_CATALOG = "REBUILD_DINSAR_CATALOG"
JOB_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG"
JOB_TYPE_SCAN_ASSET_INVENTORY = "SCAN_ASSET_INVENTORY"
COPY_ALLOWED_STATUSES = {"PENDING", "IN_PROGRESS", "COMPLETED", "FAILED"}
@@ -225,6 +228,34 @@ async def _handle_scan_data(job: SystemJobORM) -> None:
await _run_scan_data_custom(job.task_id, payload)
async def _handle_scan_asset_inventory(job: SystemJobORM) -> None:
if not job.task_id:
raise ValueError("SCAN_ASSET_INVENTORY requires task_id for progress tracking.")
payload = job.payload or {}
await task_service.start_task(job.task_id, message="Source/orbit asset inventory scan started")
async with AsyncSessionLocal() as db:
result = await asset_inventory_service.scan_configured_roots(
db,
inventory_types=payload.get("inventory_types") or None,
root_ids=payload.get("root_ids") or None,
bind_orbits=bool(payload.get("bind_orbits", True)),
task_id=job.task_id,
)
await task_service.update_task(
job.task_id,
status="COMPLETED",
progress=100,
message=(
"Asset inventory scan completed: "
f"sources={result.get('source_assets', 0)}, "
f"orbits={result.get('orbit_assets', 0)}, "
f"matched={((result.get('binding') or {}).get('matched_count', 0))}, "
f"missing={((result.get('binding') or {}).get('missing_count', 0))}"
),
db=db,
)
def _resolve_hazard_shp_path() -> str:
base_dir = settings.HAZARD_POINTS_DIR
filename = settings.HAZARD_POINTS_FILENAME
@@ -418,6 +449,16 @@ async def _handle_unpack_archives(job: SystemJobORM) -> None:
await run_unpack_task(job.task_id)
async def _handle_unpack_sentinel1(job: SystemJobORM) -> None:
if not job.task_id:
raise ValueError("UNPACK_SENTINEL1 requires task_id for progress tracking.")
payload = job.payload or {}
if payload.get("asset_id"):
await asset_inventory_service.run_sentinel1_unpack_task(job.task_id, payload)
return
await asset_inventory_service.run_sentinel1_unpack_batch_task(job.task_id, payload)
async def _handle_ai_train(job: SystemJobORM) -> None:
if not job.task_id:
raise ValueError("AI_TRAIN requires task_id for progress tracking.")
@@ -3849,6 +3890,7 @@ async def _handle_rebuild_psinsar_catalog(job: SystemJobORM) -> None:
_HANDLERS = {
JOB_TYPE_SCAN_DATA: _handle_scan_data,
JOB_TYPE_SCAN_ASSET_INVENTORY: _handle_scan_asset_inventory,
JOB_TYPE_SCAN_DINSAR: _handle_scan_dinsar,
JOB_TYPE_PUBLISH_DINSAR_PRODUCTS: _handle_publish_dinsar_products_clean,
JOB_TYPE_REBUILD_DINSAR_CATALOG: _handle_rebuild_dinsar_catalog_clean,
@@ -3864,6 +3906,7 @@ _HANDLERS = {
JOB_TYPE_REBUILD_PSINSAR_CATALOG: _handle_rebuild_psinsar_catalog,
JOB_TYPE_COPY_DATA: _handle_copy_data,
JOB_TYPE_UNPACK: _handle_unpack_archives,
JOB_TYPE_UNPACK_SENTINEL1: _handle_unpack_sentinel1,
JOB_TYPE_AI_TRAIN: _handle_ai_train,
JOB_TYPE_AI_PREDICT: _handle_ai_predict,
JOB_TYPE_AI_ANALYZE: _handle_ai_analyze,
backend/app/services/pairing_cache_service.py
+1 -1
View File
@@ -36,7 +36,7 @@ def _satellite_family_expr(alias: str) -> str:
f"COALESCE(NULLIF({alias}.satellite_family, ''), "
f"CASE "
f"WHEN {compact} IN ('LT1', 'LT1A', 'LT1B', 'LUTAN1', 'LUTAN1A', 'LUTAN1B') THEN 'LT1' "
f"WHEN {compact} IN ('S1', 'S1A', 'S1B', 'SENTINEL1', 'SENTINEL1A', 'SENTINEL1B') THEN 'S1' "
f"WHEN {compact} IN ('S1', 'S1A', 'S1B', 'S1C', 'SENTINEL1', 'SENTINEL1A', 'SENTINEL1B', 'SENTINEL1C') THEN 'S1' "
f"WHEN NULLIF({alias}.satellite, '') IS NOT NULL THEN upper({alias}.satellite) "
f"ELSE NULL END)"
)
backend/app/services/pyint_input_assets_service.py
+616 -103
View File
@@ -10,11 +10,14 @@ from typing import Any, Dict, List
from ..config import settings
from .orbit_converter import get_source_orbit_inventory
from .pyint_service import (
discover_s1_scene_sources,
discover_lt1_archives,
infer_scene_date_from_archives,
infer_task_identity,
validate_pyint_root_dir,
)
from .asset_inventory_service import _configured_sentinel1_archive_dirs, _parse_s1_source_name
from ..utils import normalize_satellite_family
VALID_DEM_MODES = {"local_fabdem", "opentopo", "prepared_file"}
@@ -62,6 +65,198 @@ def _infer_satellite_from_archives(paths: List[str]) -> str:
return ""
def _normalize_s1_satellite(value: Any) -> str:
text = str(value or "").strip().upper().replace("-", "").replace("_", "")
if text in {"S1A", "S1B", "S1C"}:
return text
return ""
def _find_s1_zip_by_logical_uid(logical_uid: str) -> str:
logical = str(logical_uid or "").strip()
if not logical:
return ""
expected_name = logical if logical.lower().endswith(".zip") else f"{logical}.zip"
for root in _configured_sentinel1_archive_dirs():
if not root or not os.path.isdir(root):
continue
direct_candidate = os.path.join(root, expected_name)
if os.path.isfile(direct_candidate):
return _normalize_path(direct_candidate)
for current_root, _, files in os.walk(root):
if expected_name in files:
return _normalize_path(os.path.join(current_root, expected_name))
return ""
def _is_s1_safe_dir(path: str) -> bool:
normalized = _normalize_path(path)
if not normalized or not os.path.isdir(normalized):
return False
return os.path.isfile(os.path.join(normalized, "manifest.safe"))
def _resolve_s1_scene_input(
*,
role: str,
scene_path: str,
pair_meta: Dict[str, Any],
) -> Dict[str, Any]:
normalized_scene = _normalize_path(scene_path)
scene_name = os.path.basename(normalized_scene.rstrip("\\/"))
parsed = _parse_s1_source_name(scene_name) or {}
logical_uid = str(parsed.get("logical_product_uid") or "").strip()
resolved_path = ""
input_kind = ""
resolution_method = ""
if normalized_scene.lower().endswith(".zip") and os.path.isfile(normalized_scene):
resolved_path = normalized_scene
input_kind = "zip"
resolution_method = "task_or_pair_meta"
elif _is_s1_safe_dir(normalized_scene):
resolved_path = normalized_scene
input_kind = "safe_dir"
resolution_method = "task_or_pair_meta"
elif normalized_scene.lower().endswith(".safe") and os.path.isdir(normalized_scene):
resolved_path = normalized_scene
input_kind = "safe_dir"
resolution_method = "task_or_pair_meta"
else:
sibling_zip = ""
if normalized_scene.lower().endswith(".safe"):
sibling_zip = normalized_scene[:-5] + ".zip"
if sibling_zip and os.path.isfile(sibling_zip):
resolved_path = _normalize_path(sibling_zip)
input_kind = "zip"
resolution_method = "safe_sibling_zip"
elif logical_uid:
looked_up_zip = _find_s1_zip_by_logical_uid(logical_uid)
if looked_up_zip:
resolved_path = looked_up_zip
input_kind = "zip"
resolution_method = "source_pool_lookup"
expected_name = scene_name
if input_kind == "zip":
if not expected_name.lower().endswith(".zip"):
expected_name = f"{logical_uid}.zip" if logical_uid else os.path.basename(resolved_path)
elif input_kind == "safe_dir":
if not expected_name.lower().endswith(".safe"):
expected_name = f"{logical_uid}.SAFE" if logical_uid else os.path.basename(resolved_path)
satellite = _normalize_s1_satellite(pair_meta.get(f"{role}_satellite")) or _normalize_s1_satellite(parsed.get("satellite"))
date_text = str(pair_meta.get(f"{role}_imaging_date") or parsed.get("imaging_date") or "").strip()
return {
"role": role,
"scene_path": normalized_scene,
"scene_name": scene_name,
"logical_product_uid": logical_uid,
"satellite": satellite,
"date": date_text,
"resolved": bool(resolved_path),
"path": resolved_path,
"input_kind": input_kind,
"resolution_method": resolution_method,
"expected_name": expected_name,
"staged_path": "",
}
def _resolve_s1_zip_file(
*,
role: str,
scene_path: str,
pair_meta: Dict[str, Any],
) -> Dict[str, Any]:
normalized_scene = _normalize_path(scene_path)
scene_name = os.path.basename(normalized_scene)
parsed = _parse_s1_source_name(scene_name) or {}
logical_uid = str(parsed.get("logical_product_uid") or "").strip()
direct_zip = ""
if normalized_scene.lower().endswith(".zip") and os.path.isfile(normalized_scene):
direct_zip = normalized_scene
elif normalized_scene.lower().endswith(".safe"):
sibling_zip = normalized_scene[:-5] + ".zip"
if os.path.isfile(sibling_zip):
direct_zip = _normalize_path(sibling_zip)
if not direct_zip and logical_uid:
direct_zip = _find_s1_zip_by_logical_uid(logical_uid)
satellite = _normalize_s1_satellite(pair_meta.get(f"{role}_satellite")) or _normalize_s1_satellite(parsed.get("satellite"))
date_text = str(pair_meta.get(f"{role}_imaging_date") or parsed.get("imaging_date") or "").strip()
return {
"role": role,
"scene_path": normalized_scene,
"scene_name": scene_name,
"logical_product_uid": logical_uid,
"satellite": satellite,
"date": date_text,
"resolved": bool(direct_zip),
"path": direct_zip,
"resolution_method": "scene_or_source_pool_lookup" if direct_zip else "",
"expected_name": f"{logical_uid}.zip" if logical_uid else "",
"staged_path": "",
}
def _resolve_s1_orbit_file(
*,
role: str,
pair_meta: Dict[str, Any],
task_dir: str,
) -> Dict[str, Any]:
direct_path = _normalize_path(pair_meta.get(f"{role}_orbit_file_path"))
if direct_path and os.path.isfile(direct_path):
return {
"role": role,
"resolved": True,
"path": direct_path,
"resolution_method": "pair_meta",
"expected_name": os.path.basename(direct_path),
"satellite": _normalize_s1_satellite(pair_meta.get(f"{role}_satellite")),
"date": str(pair_meta.get(f"{role}_imaging_date") or "").strip(),
"staged_path": "",
}
role_orbit_dir = os.path.join(task_dir, "orbit")
satellite = _normalize_s1_satellite(pair_meta.get(f"{role}_satellite"))
date_text = str(pair_meta.get(f"{role}_imaging_date") or "").strip()
candidates: List[str] = []
if os.path.isdir(role_orbit_dir):
for entry in os.scandir(role_orbit_dir):
if not entry.is_file():
continue
if not entry.name.lower().endswith(".eof"):
continue
if satellite and satellite not in entry.name.upper():
continue
candidates.append(_normalize_path(entry.path))
candidates.sort()
if candidates:
return {
"role": role,
"resolved": True,
"path": candidates[0],
"resolution_method": "task_orbit_dir",
"expected_name": os.path.basename(candidates[0]),
"satellite": satellite,
"date": date_text,
"staged_path": "",
}
return {
"role": role,
"resolved": False,
"path": "",
"resolution_method": "",
"expected_name": "",
"satellite": satellite,
"date": date_text,
"staged_path": "",
"error": f"{role} Sentinel-1 EOF 缺失",
}
def _get_dem_mode() -> str:
raw_mode = str(getattr(settings, "PYINT_DEM_MODE", "local_fabdem") or "local_fabdem").strip().lower()
if raw_mode not in VALID_DEM_MODES:
@@ -569,6 +764,427 @@ def resolve_pyint_task_input_assets(
}
def build_pyint_input_preview(root_dir: str, num_to_process: int = 0) -> Dict[str, Any]:
validation = validate_pyint_root_dir(root_dir, num_to_process)
dem_summary = get_pyint_dem_summary()
orbit_context = get_pyint_orbit_context()
warnings: List[str] = list(dem_summary.get("warnings") or [])
blockers: List[str] = list(dem_summary.get("blockers") or [])
task_summaries: List[Dict[str, Any]] = []
resolved_task_count = 0
missing_task_count = 0
effective_orbit_policy = _get_orbit_policy()
for task_dir in validation.get("task_dirs", []) or []:
task_summary = resolve_pyint_task_input_assets(
task_dir,
dem_summary=dem_summary,
orbit_context=orbit_context,
)
task_summaries.append(task_summary)
task_orbit_policy = str(((task_summary.get("input_assets") or {}).get("orbits") or {}).get("policy") or "").strip()
if task_orbit_policy:
effective_orbit_policy = task_orbit_policy
if task_summary.get("warnings"):
warnings.extend(
f"{task_summary['task_alias']}: {item}"
for item in task_summary["warnings"]
)
if task_summary.get("blockers"):
blockers.extend(
f"{task_summary['task_alias']}: {item}"
for item in task_summary["blockers"]
)
if task_summary["input_assets"]["orbits"]["missing_count"] == 0:
resolved_task_count += 1
else:
missing_task_count += 1
allow_submit = not blockers
precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary()
return {
"root_dir": validation["root_dir"],
"mode": validation["mode"],
"task_count": len(task_summaries),
"selected_task_count": len(task_summaries),
"allow_submit": allow_submit,
"warnings": warnings,
"blockers": blockers,
"invalid_candidates": validation.get("invalid_candidates", []),
"dem": dem_summary,
"orbits": {
"policy": effective_orbit_policy,
"pool_root": orbit_context.get("pool_root", ""),
"pool_exists": bool(orbit_context.get("pool_exists")),
"resolved_task_count": resolved_task_count,
"missing_task_count": missing_task_count,
"duplicate_count": int(orbit_context.get("duplicate_count", 0) or 0),
"warnings": list(orbit_context.get("warnings") or []),
},
"precise_orbit_bridge": precise_orbit_bridge,
"tasks": task_summaries,
}
def summarize_preview_blockers(preview: Dict[str, Any], limit: int = 8) -> str:
blockers = [str(item).strip() for item in (preview.get("blockers") or []) if str(item).strip()]
if not blockers:
return ""
if len(blockers) <= limit:
return "; ".join(blockers)
return "; ".join(blockers[:limit]) + f"; 其余 {len(blockers) - limit} 项已省略"
def materialize_pyint_input_assets(
*,
task_summary: Dict[str, Any],
input_assets_dir: str,
project_name: str = "",
) -> Dict[str, Any]:
input_assets_dir = _normalize_path(input_assets_dir)
os.makedirs(input_assets_dir, exist_ok=True)
record_enabled = bool(getattr(settings, "PYINT_RECORD_INPUT_ASSETS", True))
orbits_dir = os.path.join(input_assets_dir, "orbits")
dem_dir = os.path.join(input_assets_dir, "dem")
downloads_dir = os.path.join(input_assets_dir, "downloads")
if record_enabled:
os.makedirs(orbits_dir, exist_ok=True)
os.makedirs(dem_dir, exist_ok=True)
os.makedirs(downloads_dir, exist_ok=True)
manifest = _copy_json_safe(task_summary.get("input_assets") or {})
manifest["generated_at"] = _utc_now_text()
manifest["task_name"] = task_summary.get("task_name")
manifest["task_alias"] = task_summary.get("task_alias")
manifest["pair_key"] = task_summary.get("pair_key")
manifest["task_dir"] = task_summary.get("task_dir")
manifest["allow_submit"] = bool(task_summary.get("allow_submit"))
manifest["warnings"] = list(task_summary.get("warnings") or [])
manifest["blockers"] = list(task_summary.get("blockers") or [])
dem_summary = manifest.get("dem") or {}
if project_name:
dem_summary["resolved_output_dir"] = os.path.join(_normalize_path(settings.PYINT_DEM_ROOT), project_name)
manifest["dem"] = dem_summary
orbits_summary = manifest.get("orbits") or {}
staged_count = 0
precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary()
should_stage_orbits = record_enabled and (
str(orbits_summary.get("policy") or "").strip().lower() == "stage_txt"
or precise_orbit_bridge.get("enabled")
or str((manifest.get("task_source") or {}).get("satellite_family") or "").strip().upper() == "S1"
)
if should_stage_orbits:
for role in ("master", "slave"):
orbit_item = orbits_summary.get(role) or {}
orbit_path = _normalize_path(orbit_item.get("path"))
expected_name = str(orbit_item.get("expected_name") or "").strip()
if not orbit_item.get("resolved") or not orbit_path or not expected_name:
continue
target_path = os.path.join(orbits_dir, expected_name)
if not os.path.exists(target_path):
shutil.copy2(orbit_path, target_path)
orbit_item["staged_path"] = target_path
orbit_item["stage_operation"] = "copied"
orbit_item["stage_reason"] = "precise_orbit_bridge" if precise_orbit_bridge.get("enabled") else "stage_txt_policy"
staged_count += 1
orbits_summary[role] = orbit_item
manifest["orbits"] = orbits_summary
download_staged_count = 0
task_source = manifest.get("task_source") or {}
if record_enabled and str(task_source.get("satellite_family") or "").strip().upper() == "S1":
production_inputs = task_source.get("production_inputs") or {}
for role_key in ("master_scene", "slave_scene", "master_zip", "slave_zip"):
scene_item = production_inputs.get(role_key) or {}
scene_path = _normalize_path(scene_item.get("path"))
expected_name = str(scene_item.get("expected_name") or os.path.basename(scene_path) or "").strip()
if not scene_item.get("resolved") or not scene_path or not expected_name:
continue
input_kind = str(scene_item.get("input_kind") or "").strip().lower()
if input_kind == "safe_dir" or os.path.isdir(scene_path):
scene_item["staged_path"] = scene_path
scene_item["stage_operation"] = "source_reference"
production_inputs[role_key] = scene_item
continue
target_path = os.path.join(downloads_dir, expected_name)
if not os.path.exists(target_path):
shutil.copy2(scene_path, target_path)
scene_item["staged_path"] = target_path
scene_item["stage_operation"] = "copied"
production_inputs[role_key] = scene_item
download_staged_count += 1
task_source["production_inputs"] = production_inputs
manifest["task_source"] = task_source
materialized = {
"input_assets_dir": input_assets_dir,
"record_enabled": record_enabled,
"orbits_dir": orbits_dir if record_enabled else "",
"dem_dir": dem_dir if record_enabled else "",
"downloads_dir": downloads_dir if record_enabled else "",
"orbits_staged_count": staged_count,
"downloads_staged_count": download_staged_count,
"task_manifest_path": "",
"dem_summary_path": "",
"orbit_summary_path": "",
"input_assets": manifest,
}
if not record_enabled:
return materialized
task_manifest_path = os.path.join(input_assets_dir, "task_manifest.json")
dem_summary_path = os.path.join(dem_dir, "dem_summary.json")
orbit_summary_path = os.path.join(orbits_dir, "orbit_summary.json")
with open(task_manifest_path, "w", encoding="utf-8") as fp:
json.dump(manifest, fp, ensure_ascii=False, indent=2)
fp.write("\n")
with open(dem_summary_path, "w", encoding="utf-8") as fp:
json.dump(dem_summary, fp, ensure_ascii=False, indent=2)
fp.write("\n")
with open(orbit_summary_path, "w", encoding="utf-8") as fp:
json.dump(orbits_summary, fp, ensure_ascii=False, indent=2)
fp.write("\n")
materialized.update(
{
"task_manifest_path": task_manifest_path,
"dem_summary_path": dem_summary_path,
"orbit_summary_path": orbit_summary_path,
}
)
return materialized
def resolve_pyint_task_input_assets(
task_dir: str,
*,
dem_summary: Dict[str, Any] | None = None,
orbit_context: Dict[str, Any] | None = None,
) -> Dict[str, Any]:
task_dir = _normalize_path(task_dir)
task_identity = infer_task_identity(task_dir)
pair_meta = task_identity["pair_meta"]
satellite_family = str(task_identity.get("satellite_family") or "").strip().upper()
warnings: List[str] = []
blockers: List[str] = []
if satellite_family == "S1":
scene_sources = discover_s1_scene_sources(task_dir)
master_archives = list(scene_sources.get("master", []) or [])
slave_archives = list(scene_sources.get("slave", []) or [])
master_date = task_identity["master_date"] or infer_scene_date_from_archives(master_archives)
slave_date = task_identity["slave_date"] or infer_scene_date_from_archives(slave_archives)
master_satellite = _normalize_s1_satellite(pair_meta.get("master_satellite")) or _normalize_s1_satellite(task_identity.get("master_satellite"))
slave_satellite = _normalize_s1_satellite(pair_meta.get("slave_satellite")) or _normalize_s1_satellite(task_identity.get("slave_satellite"))
if not master_archives:
blockers.append("master/ 未识别到 Sentinel-1 SAFE 源目录。")
if not slave_archives:
blockers.append("slave/ 未识别到 Sentinel-1 SAFE 源目录。")
if not master_date:
blockers.append("未能识别主影像日期。")
if not slave_date:
blockers.append("未能识别从影像日期。")
master_scene = _resolve_s1_scene_input(
role="master",
scene_path=master_archives[0] if master_archives else "",
pair_meta=pair_meta,
)
slave_scene = _resolve_s1_scene_input(
role="slave",
scene_path=slave_archives[0] if slave_archives else "",
pair_meta=pair_meta,
)
master_orbit = _resolve_s1_orbit_file(
role="master",
pair_meta=pair_meta,
task_dir=task_dir,
)
slave_orbit = _resolve_s1_orbit_file(
role="slave",
pair_meta=pair_meta,
task_dir=task_dir,
)
if not master_scene.get("resolved"):
blockers.append("master Sentinel-1 源场景缺失。")
if not slave_scene.get("resolved"):
blockers.append("slave Sentinel-1 源场景缺失。")
if not master_orbit.get("resolved"):
blockers.append(str(master_orbit.get("error") or "master Sentinel-1 EOF 缺失"))
if not slave_orbit.get("resolved"):
blockers.append(str(slave_orbit.get("error") or "slave Sentinel-1 EOF 缺失"))
task_source = {
"task_dir": task_dir,
"task_name": task_identity["task_name"],
"task_alias": task_identity["task_alias"],
"pair_key": task_identity["pair_key"],
"satellite_family": "S1",
"master_date": master_date,
"slave_date": slave_date,
"master_satellite": master_satellite,
"slave_satellite": slave_satellite,
"archives": {
"master": master_archives,
"slave": slave_archives,
},
"production_inputs": {
"master_scene": master_scene,
"slave_scene": slave_scene,
},
}
orbits_summary = {
"policy": "require_eof",
"pool_root": "",
"pool_exists": True,
"master": master_orbit,
"slave": slave_orbit,
"resolved_count": int(bool(master_orbit.get("resolved"))) + int(bool(slave_orbit.get("resolved"))),
"missing_count": int(not master_orbit.get("resolved")) + int(not slave_orbit.get("resolved")),
"warnings": [],
"stage_mode": "copy",
"precise_orbit_bridge": {
"enabled": False,
"mode": "not_applicable",
"strict": True,
},
}
else:
archives = discover_lt1_archives(task_dir)
master_archives = list(archives.get("master", []) or [])
slave_archives = list(archives.get("slave", []) or [])
master_date = task_identity["master_date"] or infer_scene_date_from_archives(master_archives)
slave_date = task_identity["slave_date"] or infer_scene_date_from_archives(slave_archives)
master_satellite = _normalize_lt1_satellite(pair_meta.get("master_satellite")) or _infer_satellite_from_archives(master_archives)
slave_satellite = _normalize_lt1_satellite(pair_meta.get("slave_satellite")) or _infer_satellite_from_archives(slave_archives)
if not master_archives:
blockers.append("master/ 未发现 LT-1 原始输入（LT1*.tar.gz 或 LT1*.tiff）。")
if not slave_archives:
blockers.append("slave/ 未发现 LT-1 原始输入（LT1*.tar.gz 或 LT1*.tiff）。")
if not master_date:
blockers.append("未能识别主影像日期。")
if not slave_date:
blockers.append("未能识别从影像日期。")
orbit_policy = _get_orbit_policy()
orbit_context = orbit_context or get_pyint_orbit_context()
orbit_pool_root = orbit_context.get("pool_root", "")
orbit_pool_exists = bool(orbit_context.get("pool_exists"))
orbit_files = orbit_context.get("files", {}) or {}
orbit_warnings: List[str] = []
if orbit_context.get("warnings"):
orbit_warnings.extend(str(item) for item in orbit_context["warnings"] if item)
master_orbit = _resolve_orbit_file(
role="master",
satellite=master_satellite,
date_text=master_date,
pool_root=orbit_pool_root,
orbit_files=orbit_files,
)
slave_orbit = _resolve_orbit_file(
role="slave",
satellite=slave_satellite,
date_text=slave_date,
pool_root=orbit_pool_root,
orbit_files=orbit_files,
)
for orbit_item in (master_orbit, slave_orbit):
if orbit_item.get("resolved"):
continue
message = str(orbit_item.get("error") or f"{orbit_item.get('role')} orbit missing").strip()
if orbit_policy == "validate_only":
orbit_warnings.append(message)
else:
blockers.append(message)
if not orbit_pool_root:
if orbit_policy == "validate_only":
orbit_warnings.append("轨道池未配置，当前仅记录警告。")
else:
blockers.append("轨道池未配置。")
elif not orbit_pool_exists:
if orbit_policy == "validate_only":
orbit_warnings.append(f"轨道池目录不可用: {orbit_pool_root}")
else:
blockers.append(f"轨道池目录不可用: {orbit_pool_root}")
warnings.extend(orbit_warnings)
precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary()
task_source = {
"task_dir": task_dir,
"task_name": task_identity["task_name"],
"task_alias": task_identity["task_alias"],
"pair_key": task_identity["pair_key"],
"satellite_family": "LT1",
"master_date": master_date,
"slave_date": slave_date,
"master_satellite": master_satellite,
"slave_satellite": slave_satellite,
"archives": {
"master": master_archives,
"slave": slave_archives,
},
}
orbits_summary = {
"policy": orbit_policy,
"pool_root": orbit_pool_root,
"pool_exists": orbit_pool_exists,
"master": master_orbit,
"slave": slave_orbit,
"resolved_count": int(bool(master_orbit.get("resolved"))) + int(bool(slave_orbit.get("resolved"))),
"missing_count": int(not master_orbit.get("resolved")) + int(not slave_orbit.get("resolved")),
"warnings": orbit_warnings,
"stage_mode": "copy" if orbit_policy == "stage_txt" or precise_orbit_bridge.get("enabled") else "none",
"precise_orbit_bridge": precise_orbit_bridge,
}
dem_payload = _copy_json_safe(dem_summary or get_pyint_dem_summary())
allow_submit = not blockers and bool(dem_payload.get("allow_submit", True))
return {
"task_name": task_identity["task_name"],
"task_alias": task_identity["task_alias"],
"pair_key": task_identity["pair_key"],
"task_dir": task_dir,
"master_date": master_date,
"slave_date": slave_date,
"master_satellite": master_satellite,
"slave_satellite": slave_satellite,
"satellite_family": satellite_family or normalize_satellite_family(master_satellite or slave_satellite),
"archive_counts": {
"master": len(master_archives),
"slave": len(slave_archives),
},
"warnings": warnings,
"blockers": blockers,
"allow_submit": allow_submit,
"task_source": task_source,
"dem": dem_payload,
"orbit_resolution": {
"master": master_orbit,
"slave": slave_orbit,
},
"input_assets": {
"task_source": task_source,
"dem": dem_payload,
"orbits": orbits_summary,
},
}
def build_pyint_input_preview(root_dir: str, num_to_process: int = 0) -> Dict[str, Any]:
validation = validate_pyint_root_dir(root_dir, num_to_process)
dem_summary = get_pyint_dem_summary()
@@ -626,106 +1242,3 @@ def build_pyint_input_preview(root_dir: str, num_to_process: int = 0) -> Dict[st
"precise_orbit_bridge": precise_orbit_bridge,
"tasks": task_summaries,
}
def summarize_preview_blockers(preview: Dict[str, Any], limit: int = 8) -> str:
blockers = [str(item).strip() for item in (preview.get("blockers") or []) if str(item).strip()]
if not blockers:
return ""
if len(blockers) <= limit:
return "; ".join(blockers)
return "; ".join(blockers[:limit]) + f"; 其余 {len(blockers) - limit} 项已省略"
def materialize_pyint_input_assets(
*,
task_summary: Dict[str, Any],
input_assets_dir: str,
project_name: str = "",
) -> Dict[str, Any]:
input_assets_dir = _normalize_path(input_assets_dir)
os.makedirs(input_assets_dir, exist_ok=True)
record_enabled = bool(getattr(settings, "PYINT_RECORD_INPUT_ASSETS", True))
orbits_dir = os.path.join(input_assets_dir, "orbits")
dem_dir = os.path.join(input_assets_dir, "dem")
if record_enabled:
os.makedirs(orbits_dir, exist_ok=True)
os.makedirs(dem_dir, exist_ok=True)
manifest = _copy_json_safe(task_summary.get("input_assets") or {})
manifest["generated_at"] = _utc_now_text()
manifest["task_name"] = task_summary.get("task_name")
manifest["task_alias"] = task_summary.get("task_alias")
manifest["pair_key"] = task_summary.get("pair_key")
manifest["task_dir"] = task_summary.get("task_dir")
manifest["allow_submit"] = bool(task_summary.get("allow_submit"))
manifest["warnings"] = list(task_summary.get("warnings") or [])
manifest["blockers"] = list(task_summary.get("blockers") or [])
dem_summary = manifest.get("dem") or {}
if project_name:
dem_summary["resolved_output_dir"] = os.path.join(_normalize_path(settings.PYINT_DEM_ROOT), project_name)
manifest["dem"] = dem_summary
orbits_summary = manifest.get("orbits") or {}
staged_count = 0
precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary()
should_stage_orbits = record_enabled and (
str(orbits_summary.get("policy") or "").strip().lower() == "stage_txt"
or precise_orbit_bridge.get("enabled")
)
if should_stage_orbits:
for role in ("master", "slave"):
orbit_item = orbits_summary.get(role) or {}
orbit_path = _normalize_path(orbit_item.get("path"))
expected_name = str(orbit_item.get("expected_name") or "").strip()
if not orbit_item.get("resolved") or not orbit_path or not expected_name:
continue
target_path = os.path.join(orbits_dir, expected_name)
if not os.path.exists(target_path):
shutil.copy2(orbit_path, target_path)
orbit_item["staged_path"] = target_path
orbit_item["stage_operation"] = "copied"
orbit_item["stage_reason"] = "precise_orbit_bridge" if precise_orbit_bridge.get("enabled") else "stage_txt_policy"
staged_count += 1
orbits_summary[role] = orbit_item
manifest["orbits"] = orbits_summary
materialized = {
"input_assets_dir": input_assets_dir,
"record_enabled": record_enabled,
"orbits_dir": orbits_dir if record_enabled else "",
"dem_dir": dem_dir if record_enabled else "",
"orbits_staged_count": staged_count,
"task_manifest_path": "",
"dem_summary_path": "",
"orbit_summary_path": "",
"input_assets": manifest,
}
if not record_enabled:
return materialized
task_manifest_path = os.path.join(input_assets_dir, "task_manifest.json")
dem_summary_path = os.path.join(dem_dir, "dem_summary.json")
orbit_summary_path = os.path.join(orbits_dir, "orbit_summary.json")
with open(task_manifest_path, "w", encoding="utf-8") as fp:
json.dump(manifest, fp, ensure_ascii=False, indent=2)
fp.write("\n")
with open(dem_summary_path, "w", encoding="utf-8") as fp:
json.dump(dem_summary, fp, ensure_ascii=False, indent=2)
fp.write("\n")
with open(orbit_summary_path, "w", encoding="utf-8") as fp:
json.dump(orbits_summary, fp, ensure_ascii=False, indent=2)
fp.write("\n")
materialized.update(
{
"task_manifest_path": task_manifest_path,
"dem_summary_path": dem_summary_path,
"orbit_summary_path": orbit_summary_path,
}
)
return materialized
backend/app/services/pyint_service.py
+38 -1
View File
@@ -13,10 +13,12 @@ from typing import Any, Dict, Iterable, List, Optional
from ..config import get_env_text, read_bool_env, settings
from .dinsar_naming import PAIR_META_FILENAME, build_fallback_pair_key, find_json_sidecar
from ..utils import normalize_satellite_family
from .wsl_service import run_wsl_exec
LT1_INPUT_GLOBS = ("LT1*.tar.gz", "LT1*.tiff")
S1_INPUT_GLOBS = ("S1*.zip",)
DEFAULT_RANGE_LOOKS = 2
DEFAULT_AZIMUTH_LOOKS = 2
DEFAULT_DEM_RESOLUTION_M = 30.0
@@ -334,6 +336,15 @@ def build_project_name(pair_key: str, run_key: str) -> str:
return slugify_text(f"{pair_key}_{run_key}", default="pyint_project", max_len=120)
def build_profile_project_name(satellite_family: Any, pair_key: str, run_key: str) -> str:
family = str(normalize_satellite_family(satellite_family) or "").strip().upper()
if family == "S1":
return slugify_text(f"s1_{pair_key}_{run_key}", default="s1_pyint_project", max_len=120)
if family == "LT1":
return slugify_text(f"lt1_{pair_key}_{run_key}", default="lt1_pyint_project", max_len=120)
return build_project_name(pair_key, run_key)
def windows_path_to_wsl_mount(path: str) -> str:
text = str(path or "").strip().strip('"').strip("'")
if not text:
@@ -377,6 +388,22 @@ def discover_lt1_archives(task_dir: str) -> Dict[str, List[str]]:
return result
def discover_s1_scene_sources(task_dir: str) -> Dict[str, List[str]]:
task_path = Path(os.path.normpath(os.path.abspath(str(task_dir or "").strip())))
pair_meta = find_json_sidecar(str(task_path), PAIR_META_FILENAME, max_levels=0) or {}
result: Dict[str, List[str]] = {"master": [], "slave": []}
for role in ("master", "slave"):
explicit_path = str(pair_meta.get(f"{role}_path") or "").strip()
role_dir = task_path / role
candidates: List[str] = []
if explicit_path:
candidates.append(str(Path(explicit_path).resolve()))
if not candidates and role_dir.is_dir() and (role_dir / "manifest.safe").is_file():
candidates.append(str(role_dir.resolve()))
result[role] = sorted(set(candidates))
return result
def infer_scene_date_from_archives(paths: Iterable[str]) -> str:
dates = {
date_text
@@ -393,7 +420,14 @@ def infer_task_identity(task_dir: str) -> Dict[str, Any]:
task_name = os.path.basename(os.path.normpath(task_dir))
pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {}
task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(task_alias, task_dir)
master_satellite = str(pair_meta.get("master_satellite") or "").strip().upper()
slave_satellite = str(pair_meta.get("slave_satellite") or "").strip().upper()
satellite_family = normalize_satellite_family(master_satellite or slave_satellite)
pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(
task_alias,
task_dir,
satellite_family=satellite_family,
)
master_date = normalize_date_text(pair_meta.get("master_imaging_date"))
slave_date = normalize_date_text(pair_meta.get("slave_imaging_date"))
return {
@@ -403,6 +437,9 @@ def infer_task_identity(task_dir: str) -> Dict[str, Any]:
"pair_meta": pair_meta,
"master_date": master_date,
"slave_date": slave_date,
"master_satellite": master_satellite,
"slave_satellite": slave_satellite,
"satellite_family": satellite_family,
}
backend/app/services/result_catalog_service.py
+8 -1
View File
@@ -39,6 +39,7 @@ from .dinsar_naming import (
build_fallback_pair_key,
find_json_sidecar,
)
from ..utils import normalize_satellite_family
from .dinsar_result_layout_service import (
RUN_CURRENT_DIRNAME,
RUN_NATIVE_DIRNAME,
@@ -203,10 +204,16 @@ def _resolve_candidate_identity(candidate: Dict[str, Any]) -> Dict[str, Any]:
pair_meta.get("task_alias"),
candidate.get("task_name"),
) or "Task_unknown_unknown"
satellite_family = normalize_satellite_family(
pair_meta.get("master_satellite")
or pair_meta.get("slave_satellite")
or run_meta.get("master_satellite")
or run_meta.get("slave_satellite")
)
pair_key = _first_text(
run_meta.get("pair_key"),
pair_meta.get("pair_key"),
) or build_fallback_pair_key(task_alias, source_dir)
) or build_fallback_pair_key(task_alias, source_dir, satellite_family=satellite_family)
run_key = _first_text(run_meta.get("run_key")) or (
"legacy_" + _stable_digest(candidate.get("engine_code"), pair_key, source_dir, candidate["primary_file"], length=16)
)
backend/app/services/root_registry_service.py
+98 -1
View File
@@ -10,7 +10,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from .. import database
from ..config import settings, split_env_paths
from ..models import ManagedRootORM, PathInventoryORM, ScanCursorORM
from ..models import AssetInventoryStateORM, ManagedRootORM, PathInventoryORM, ScanCursorORM
_SLUG_RE = re.compile(r"[^a-z0-9]+")
@@ -87,6 +87,15 @@ def _cursor_type_for_scan_mode(scan_mode: str) -> str:
return mapping.get(str(scan_mode or "").strip().lower(), "directory_walk")
def _asset_inventory_type_for_root_role(root_role: str) -> Optional[str]:
role = str(root_role or "").strip().lower()
if role == "source_product_pool":
return "source_product"
if role == "orbit_asset_pool":
return "orbit_asset"
return None
def _iter_multi_root_specs(
*,
env_var: str,
@@ -169,6 +178,36 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
scan_mode="archive_walk",
)
)
source_product_paths = split_env_paths(settings.SOURCE_PRODUCT_DIRS)
if not source_product_paths:
source_product_paths = (
split_env_paths(settings.INSAR_STORAGE_DIRS)
+ split_env_paths(settings.MONITOR_RADAR_DIRS)
)
specs.extend(
_iter_multi_root_specs(
env_var="SOURCE_PRODUCT_DIRS",
paths=source_product_paths,
root_role="source_product_pool",
display_prefix="Source Product Pool",
scan_mode="file_pool",
)
)
source_product_path_set = {_normalize_root_path(path) for path in source_product_paths}
sentinel1_storage_paths = [
path
for path in split_env_paths(settings.SENTINEL1_STORAGE_DIRS)
if _normalize_root_path(path) not in source_product_path_set
]
specs.extend(
_iter_multi_root_specs(
env_var="SENTINEL1_STORAGE_DIRS",
paths=sentinel1_storage_paths,
root_role="source_product_pool",
display_prefix="Sentinel-1 Storage Pool",
scan_mode="file_pool",
)
)
specs.extend(
_iter_multi_root_specs(
env_var="INSAR_STORAGE_DIRS",
@@ -214,6 +253,18 @@ def _build_root_specs_from_settings() -> List[RootSpec]:
scan_mode="scene_directory",
)
)
orbit_source_paths = split_env_paths(settings.ORBIT_SOURCE_DIRS)
if not orbit_source_paths:
orbit_source_paths = split_env_paths(settings.MONITOR_ORBIT_DIR)
specs.extend(
_iter_multi_root_specs(
env_var="ORBIT_SOURCE_DIRS",
paths=orbit_source_paths,
root_role="orbit_asset_pool",
display_prefix="Orbit Asset Pool",
scan_mode="file_pool",
)
)
specs.extend(
_iter_single_root_specs(
env_var="MONITOR_ORBIT_DIR",
@@ -374,6 +425,43 @@ class RootRegistryService:
changed = True
return "updated" if changed else None
async def _ensure_asset_inventory_state(self, db: AsyncSession, root: ManagedRootORM) -> Optional[str]:
inventory_type = _asset_inventory_type_for_root_role(root.root_role)
if not inventory_type:
return None
result = await db.execute(
select(AssetInventoryStateORM).where(
AssetInventoryStateORM.root_ref_id == root.id,
AssetInventoryStateORM.inventory_type == inventory_type,
)
)
state = result.scalar_one_or_none()
if state is None:
state = AssetInventoryStateORM(
root_ref_id=root.id,
inventory_type=inventory_type,
root_path=root.path,
scan_mode=root.scan_mode,
status="NEVER_SCANNED",
needs_rescan=True,
metadata_json={
"root_role": root.root_role,
"created_by": "root_registry_sync",
},
)
db.add(state)
return "created"
changed = False
if state.root_path != root.path:
state.root_path = root.path
changed = True
if state.scan_mode != root.scan_mode:
state.scan_mode = root.scan_mode
changed = True
return "updated" if changed else None
async def sync_from_settings(self, db: Optional[AsyncSession] = None) -> Dict[str, Any]:
generated_session = db is None
if generated_session:
@@ -394,6 +482,8 @@ class RootRegistryService:
disabled = 0
cursor_created = 0
cursor_updated = 0
inventory_state_created = 0
inventory_state_updated = 0
synced_codes: set[str] = set()
for spec in specs:
@@ -448,6 +538,12 @@ class RootRegistryService:
elif cursor_change == "updated":
cursor_updated += 1
inventory_state_change = await self._ensure_asset_inventory_state(db, row)
if inventory_state_change == "created":
inventory_state_created += 1
elif inventory_state_change == "updated":
inventory_state_updated += 1
for row in existing_rows:
if row.root_code in synced_codes:
continue
@@ -464,6 +560,7 @@ class RootRegistryService:
"updated": updated,
"disabled": disabled,
"cursor_created_or_updated": cursor_created + cursor_updated,
"asset_inventory_state_created_or_updated": inventory_state_created + inventory_state_updated,
"summary": summary,
}
except Exception:
backend/app/services/spatial_service.py
+63 -3
View File
@@ -14,7 +14,7 @@ from typing import Any, Dict, List, Optional, Tuple
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from sqlalchemy import and_, cast, func, or_
from sqlalchemy import and_, case, cast, func, or_
from sqlalchemy.orm import aliased
from geoalchemy2 import Geography
@@ -43,11 +43,60 @@ from .dinsar_naming import build_pair_key, build_task_alias, ensure_unique_task_
from .pairing_state_service import pairing_state_service
PAIRING_POLICY_VERSION = "2026.05.raw-source.v1"
PAIRING_POLICY_VERSION = "2026.05.raw-source.v2"
PAIRING_WARNING_CANDIDATE_THRESHOLD = 3000
logger = logging.getLogger(__name__)
def _normalized_satellite_family_expr(alias):
compact_satellite = func.upper(
func.replace(
func.replace(
func.replace(func.coalesce(alias.satellite, ""), "-", ""),
"_",
"",
),
" ",
"",
)
)
inferred_family = case(
(
compact_satellite.in_(
["LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"]
),
"LT1",
),
(
compact_satellite.in_(
[
"S1",
"S1A",
"S1B",
"S1C",
"SENTINEL1",
"SENTINEL1A",
"SENTINEL1B",
"SENTINEL1C",
]
),
"S1",
),
else_=func.upper(alias.satellite),
)
return func.coalesce(func.nullif(func.upper(alias.satellite_family), ""), inferred_family)
def _same_relative_orbit_expr(left_alias, right_alias):
left_relative_orbit = func.upper(func.trim(func.coalesce(left_alias.relative_orbit, "")))
right_relative_orbit = func.upper(func.trim(func.coalesce(right_alias.relative_orbit, "")))
return and_(
left_relative_orbit != "",
right_relative_orbit != "",
left_relative_orbit == right_relative_orbit,
)
class SpatialService:
"""
纯 PostGIS 空间计算服务
@@ -155,6 +204,8 @@ class SpatialService:
) -> List[dict]:
master_alias = aliased(RadarDataORM)
slave_alias = aliased(RadarDataORM)
master_family_expr = _normalized_satellite_family_expr(master_alias)
slave_family_expr = _normalized_satellite_family_expr(slave_alias)
center_distance_expr = func.coalesce(
PairingMetricCacheORM.scene_center_distance_meters,
PairingMetricCacheORM.spatial_baseline_meters,
@@ -192,6 +243,14 @@ class SpatialService:
if params.require_same_polarization:
stmt = stmt.where(PairingMetricCacheORM.same_polarization.is_(True))
stmt = stmt.where(
or_(
master_family_expr != "S1",
slave_family_expr != "S1",
_same_relative_orbit_expr(master_alias, slave_alias),
)
)
if params.allowed_satellites:
allowed_satellites = [
str(item).strip().upper()
@@ -284,6 +343,7 @@ class SpatialService:
slave.file_path,
master.imaging_date,
slave.imaging_date,
master.satellite_family or slave.satellite_family or master.satellite or slave.satellite,
),
pair_uid=candidate.get("pair_uid"),
metric_cache_ref_id=candidate.get("metric_cache_ref_id"),
@@ -1281,7 +1341,7 @@ class SpatialService:
compact = raw_satellite.replace("-", "").replace("_", "").replace(" ", "")
if compact in {"LT1", "LT1A", "LT1B", "LUTAN1", "LUTAN1A", "LUTAN1B"}:
return "LT1"
if compact in {"S1", "S1A", "S1B", "SENTINEL1", "SENTINEL1A", "SENTINEL1B"}:
if compact in {"S1", "S1A", "S1B", "S1C", "SENTINEL1", "SENTINEL1A", "SENTINEL1B", "SENTINEL1C"}:
return "S1"
return raw_satellite or "UNKNOWN"