Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: sync production runtime and docs

Browse Source
This commit is contained in:
Harmon
2026-04-25 15:02:40 +08:00
parent cb518b431f
commit 1e44101eb2
50 changed files with 8575 additions and 2177 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/result_catalog_service.py
+201 -66
View File
@@ -38,6 +38,16 @@ from .dinsar_naming import (
build_fallback_pair_key,
find_json_sidecar,
)
from .dinsar_result_layout_service import (
RUN_CURRENT_DIRNAME,
RUN_NATIVE_DIRNAME,
RUN_PREVIEW_DIRNAME,
is_path_within_native_dir,
is_standard_envi_disp_file,
is_standard_isce2_disp_file,
)
from .product_package_schema import build_canonical_descriptor, normalize_package_manifest
from .product_packaging import build_dinsar_package_manifest
DINSAR_CATALOG_NAME = "dinsar"
@@ -120,6 +130,15 @@ def _coerce_optional_int(value: Any) -> Optional[int]:
return None
def _runtime_id_for_engine(engine_code: Optional[str]) -> Optional[str]:
normalized = str(engine_code or "").strip().lower()
if normalized == "isce2":
return settings.ISCE2_RUNTIME_ID or None
if normalized in {"pyint", "gamma"}:
return settings.PYINT_RUNTIME_ID or None
return None
def _build_pairing_trace_payload(
candidate_meta: Dict[str, Any],
task_item: Optional[DinsarTaskItemORM],
@@ -194,11 +213,18 @@ def _resolve_candidate_identity(candidate: Dict[str, Any]) -> Dict[str, Any]:
"task_dir": _first_text(run_meta.get("task_dir")),
"work_dir": _first_text(run_meta.get("work_dir")),
"output_dir": _first_text(run_meta.get("output_dir"), source_dir),
"native_output_dir": _first_text(run_meta.get("native_output_dir")),
"started_at": _first_text(run_meta.get("started_at")),
"finished_at": _first_text(run_meta.get("finished_at")),
"params": run_meta.get("params") if isinstance(run_meta.get("params"), dict) else {},
"metrics": run_meta.get("metrics") if isinstance(run_meta.get("metrics"), dict) else {},
}
if not resolved["native_output_dir"]:
native_dir = os.path.join(str(resolved["output_dir"] or source_dir), RUN_NATIVE_DIRNAME)
if os.path.isdir(native_dir):
resolved["native_output_dir"] = native_dir
else:
resolved["native_output_dir"] = resolved["output_dir"]
for field in (
"master_path",
@@ -247,7 +273,8 @@ def _resolve_candidate_identity(candidate: Dict[str, Any]) -> Dict[str, Any]:
def _iter_flat_result_candidates(root_dir: str) -> Iterable[Dict[str, Any]]:
stack = [_normalize_path(root_dir)]
normalized_root = _normalize_path(root_dir)
stack = [normalized_root]
while stack:
current = stack.pop()
try:
@@ -255,12 +282,64 @@ def _iter_flat_result_candidates(root_dir: str) -> Iterable[Dict[str, Any]]:
for entry in entries:
try:
if entry.is_dir(follow_symlinks=False):
rel_name = os.path.relpath(entry.path, normalized_root)
rel_parts = [part.lower() for part in rel_name.split(os.sep) if part]
if any(
part in {
RUN_NATIVE_DIRNAME,
RUN_CURRENT_DIRNAME,
RUN_PREVIEW_DIRNAME,
}
for part in rel_parts
):
continue
stack.append(entry.path)
continue
if not entry.is_file(follow_symlinks=False):
continue
if is_path_within_native_dir(normalized_root, entry.path):
continue
lower_name = entry.name.lower()
if is_standard_envi_disp_file(normalized_root, entry.path):
primary_file = os.path.join(os.path.dirname(entry.path), "disp")
source_dir = os.path.dirname(os.path.dirname(os.path.dirname(primary_file)))
sidecars = []
if os.path.isfile(primary_file + ".hdr"):
sidecars.append(primary_file + ".hdr")
if os.path.isfile(primary_file + ".sml"):
sidecars.append(primary_file + ".sml")
yield {
"engine_code": "envi",
"name": "disp",
"task_name": "",
"source_dir": source_dir,
"primary_file": primary_file,
"source_files": [primary_file] + sidecars,
}
continue
if is_standard_isce2_disp_file(normalized_root, entry.path):
source_dir = os.path.dirname(os.path.dirname(os.path.dirname(entry.path)))
source_files = [entry.path]
coh_candidates = (
os.path.join(source_dir, "assets", "coh", "coh.tif"),
os.path.join(source_dir, "assets", "coh", "coh.tiff"),
)
for coh_path in coh_candidates:
if os.path.isfile(coh_path):
source_files.append(coh_path)
break
yield {
"engine_code": "isce2",
"name": os.path.splitext(entry.name)[0],
"task_name": "",
"source_dir": source_dir,
"primary_file": entry.path,
"source_files": source_files,
}
continue
if lower_name.endswith(".hdr"):
base_name, _ = os.path.splitext(entry.name)
if not base_name.lower().endswith("_disp"):
@@ -344,6 +423,15 @@ def _resolve_relative_path(base_dir: str, relative_path: str) -> str:
return target
def _is_path_within(base_dir: str, candidate_path: str) -> bool:
base = _normalize_path(base_dir)
candidate = _normalize_path(candidate_path)
try:
return os.path.commonpath([base, candidate]) == base
except ValueError:
return False
def _build_bbox_polygon(
min_lon: Optional[float],
min_lat: Optional[float],
@@ -383,6 +471,7 @@ class ResultCatalogService:
if state is None:
state = ResultCatalogStateORM(
catalog_name=DINSAR_CATALOG_NAME,
product_family="dinsar",
storage_root=root,
status="READY",
needs_rebuild=False,
@@ -391,6 +480,8 @@ class ResultCatalogService:
await db.flush()
elif state.storage_root != root:
state.storage_root = root
if state.product_family != "dinsar":
state.product_family = "dinsar"
return state
async def _lookup_task_item(
@@ -565,57 +656,57 @@ class ResultCatalogService:
}
if pairing_trace:
summary_payload["pairing_trace"] = pairing_trace
return {
"schema_version": "dinsar-product/v1",
"catalog_name": DINSAR_CATALOG_NAME,
"product_id": product_id,
"product_type": "dinsar",
"display_name": display_name,
"task_name": candidate_meta.get("task_alias") or display_name,
"identity": {
return build_dinsar_package_manifest(
product_id=product_id,
display_name=display_name,
task_name=candidate_meta.get("task_alias") or display_name,
engine_code=engine_code,
engine_version=candidate_meta.get("engine_version") or "",
processor_code=candidate_meta.get("profile_code") or engine_code,
profile_code=candidate_meta.get("profile_code"),
runtime_id=_runtime_id_for_engine(engine_code),
source_primary_path=source_primary_path,
source_dir=source_dir,
publish_dir=package_dir,
identity={
"pair_key": candidate_meta.get("pair_key"),
"task_alias": candidate_meta.get("task_alias") or display_name,
"run_key": candidate_meta.get("run_key"),
},
"engine": {
"code": engine_code,
"version": candidate_meta.get("engine_version") or "",
},
"source": {
"primary_path": source_primary_path,
"source_dir": source_dir,
source={
"source_root": candidate_meta.get("source_root"),
"task_dir": candidate_meta.get("task_dir"),
"work_dir": candidate_meta.get("work_dir"),
"output_dir": candidate_meta.get("output_dir"),
"publish_dir": package_dir,
"native_output_dir": candidate_meta.get("native_output_dir") or candidate_meta.get("output_dir"),
},
"run": {
run={
"engine_code": engine_code,
"profile_code": candidate_meta.get("profile_code"),
"source_root": candidate_meta.get("source_root"),
"task_dir": candidate_meta.get("task_dir"),
"work_dir": candidate_meta.get("work_dir"),
"output_dir": candidate_meta.get("output_dir"),
"native_output_dir": candidate_meta.get("native_output_dir") or candidate_meta.get("output_dir"),
"started_at": candidate_meta.get("started_at"),
"finished_at": candidate_meta.get("finished_at"),
"params": profile_params,
"metrics": profile_metrics,
},
"temporal": {
temporal={
"master_imaging_date": master_date,
"slave_imaging_date": slave_date,
"produced_at": candidate_meta.get("finished_at") or candidate_meta.get("started_at"),
"published_at": published_at,
},
"spatial": {
spatial={
"min_lon": meta.get("min_lon"),
"min_lat": meta.get("min_lat"),
"max_lon": meta.get("max_lon"),
"max_lat": meta.get("max_lat"),
"coverage_polygon": meta.get("coverage_polygon"),
},
"dinsar_profile": {
dinsar_profile={
"master_path": getattr(task_item, "master_path", None) or candidate_meta.get("master_path"),
"slave_path": getattr(task_item, "slave_path", None) or candidate_meta.get("slave_path"),
"master_satellite": getattr(task_item, "master_satellite", None) or candidate_meta.get("master_satellite"),
@@ -637,15 +728,15 @@ class ResultCatalogService:
"params": profile_params,
"metrics": profile_metrics,
},
"labels": {
pairing_trace=pairing_trace,
labels={
"ai_score": None,
"user_label": None,
},
"pairing_trace": pairing_trace,
"summary": summary_payload,
"assets": asset_rows,
"issues": [],
}
summary=summary_payload,
assets=asset_rows,
issues=[],
)
async def publish_from_sources(
self,
@@ -691,7 +782,9 @@ class ResultCatalogService:
run_key,
primary_file,
)
package_dir = _ensure_directory(os.path.join(target_root, pair_key, run_key))
package_dir = _ensure_directory(os.path.join(target_root, pair_key, "runs", run_key))
source_dir = _normalize_path(candidate["source_dir"])
in_place_source = _is_path_within(package_dir, source_dir)
task_item = await self._lookup_task_item(
db,
pair_key=pair_key,
@@ -713,24 +806,28 @@ class ResultCatalogService:
)
continue
disp_dir = _ensure_directory(os.path.join(package_dir, "assets", "disp"))
disp_dir = os.path.join(package_dir, "assets", "disp")
preview_dir = _ensure_directory(os.path.join(package_dir, "preview"))
asset_rows: List[Dict[str, Any]] = []
if candidate["engine_code"] == "envi":
target_base = os.path.join(disp_dir, "disp")
target_primary = target_base
source_primary = candidate["source_files"][0]
for src_path in candidate["source_files"]:
suffix = src_path[len(source_primary):]
dst_path = target_base + suffix
op = _copy_file_if_needed(src_path, dst_path)
if op == "copied":
copied += 1
elif op == "overwritten":
overwritten += 1
else:
skipped += 1
source_primary = _normalize_path(candidate["source_files"][0])
if in_place_source:
target_primary = source_primary
else:
_ensure_directory(disp_dir)
target_base = os.path.join(disp_dir, "disp")
target_primary = target_base
for src_path in candidate["source_files"]:
suffix = src_path[len(source_primary):]
dst_path = target_base + suffix
op = _copy_file_if_needed(src_path, dst_path)
if op == "copied":
copied += 1
elif op == "overwritten":
overwritten += 1
else:
skipped += 1
asset_rows.append(
{
"role": "disp",
@@ -769,14 +866,18 @@ class ResultCatalogService:
}
)
else:
target_primary = os.path.join(disp_dir, "disp.tif")
op = _copy_file_if_needed(primary_file, target_primary)
if op == "copied":
copied += 1
elif op == "overwritten":
overwritten += 1
if in_place_source:
target_primary = _normalize_path(primary_file)
else:
skipped += 1
_ensure_directory(disp_dir)
target_primary = os.path.join(disp_dir, "disp.tif")
op = _copy_file_if_needed(primary_file, target_primary)
if op == "copied":
copied += 1
elif op == "overwritten":
overwritten += 1
else:
skipped += 1
asset_rows.append(
{
"role": "disp",
@@ -789,17 +890,20 @@ class ResultCatalogService:
}
)
if len(candidate["source_files"]) > 1:
coh_dir = _ensure_directory(os.path.join(package_dir, "assets", "coh"))
source_coh = candidate["source_files"][1]
coh_ext = os.path.splitext(source_coh)[1] or ".tif"
target_coh = os.path.join(coh_dir, f"coh{coh_ext}")
op = _copy_file_if_needed(source_coh, target_coh)
if op == "copied":
copied += 1
elif op == "overwritten":
overwritten += 1
if in_place_source:
target_coh = _normalize_path(source_coh)
else:
skipped += 1
coh_dir = _ensure_directory(os.path.join(package_dir, "assets", "coh"))
coh_ext = os.path.splitext(source_coh)[1] or ".tif"
target_coh = os.path.join(coh_dir, f"coh{coh_ext}")
op = _copy_file_if_needed(source_coh, target_coh)
if op == "copied":
copied += 1
elif op == "overwritten":
overwritten += 1
else:
skipped += 1
asset_rows.append(
{
"role": "coh",
@@ -859,6 +963,7 @@ class ResultCatalogService:
"run_key": run_key,
"engine_code": candidate_meta["engine_code"],
"package_dir": package_dir,
"in_place": in_place_source,
"thumb_created": thumb_ok,
"status": "ok",
}
@@ -881,11 +986,12 @@ class ResultCatalogService:
def _load_manifest(self, manifest_path: str) -> Dict[str, Any]:
with open(manifest_path, "r", encoding="utf-8") as fp:
payload = json.load(fp)
if str(payload.get("product_type") or "").strip().lower() != "dinsar":
raise ValueError("manifest product_type is not dinsar")
if not str(payload.get("product_id") or "").strip():
normalized = normalize_package_manifest(payload)
if str(normalized.get("product_family") or "").strip().lower() != "dinsar":
raise ValueError("manifest product_family is not dinsar")
if not str(normalized.get("product_id") or "").strip():
raise ValueError("manifest product_id is empty")
return payload
return normalized
def _build_rows_from_manifest(
self,
@@ -921,39 +1027,58 @@ class ResultCatalogService:
profile_payload = manifest.get("dinsar_profile") or {}
labels = manifest.get("labels") or {}
pairing_trace = manifest.get("pairing_trace") or {}
processor_payload = manifest.get("processor") or {}
runtime_payload = manifest.get("runtime") or {}
canonical_payload = manifest.get("canonical") or build_canonical_descriptor(
assets_payload,
product_family="dinsar",
)
summary_json: Optional[Dict[str, Any]] = None
if summary or identity or run_payload or pairing_trace:
if summary or identity or run_payload or pairing_trace or canonical_payload:
summary_json = {
**summary,
"identity": identity,
"run": run_payload,
}
if processor_payload:
summary_json["processor"] = processor_payload
if runtime_payload:
summary_json["runtime"] = runtime_payload
if canonical_payload:
summary_json["canonical"] = canonical_payload
if pairing_trace:
summary_json["pairing_trace"] = pairing_trace
product = ResultProductORM(
product_id=str(manifest.get("product_id")).strip(),
catalog_name=str(manifest.get("catalog_name") or DINSAR_CATALOG_NAME).strip() or DINSAR_CATALOG_NAME,
product_type="dinsar",
product_family=str(manifest.get("product_family") or "dinsar").strip() or "dinsar",
product_type=str(manifest.get("product_type") or "dinsar_interferogram").strip() or "dinsar_interferogram",
display_name=str(manifest.get("display_name") or manifest.get("task_name") or manifest.get("product_id")),
task_name=str(manifest.get("task_name") or manifest.get("display_name") or "").strip() or None,
task_alias=str(identity.get("task_alias") or manifest.get("task_name") or "").strip() or None,
pair_key=str(identity.get("pair_key") or "").strip() or None,
stack_key=str(identity.get("stack_key") or "").strip() or None,
pair_uid=str(pairing_trace.get("pair_uid") or "").strip() or None,
run_key=str(identity.get("run_key") or "").strip() or None,
network_run_id=str(pairing_trace.get("network_run_id") or "").strip() or None,
network_edge_id=_coerce_optional_int(pairing_trace.get("network_edge_id")),
policy_version=str(pairing_trace.get("policy_version") or "").strip() or None,
selection_strategy=str(pairing_trace.get("selection_strategy") or "").strip() or None,
profile_code=str(run_payload.get("profile_code") or "").strip() or None,
profile_code=str(processor_payload.get("profile_code") or run_payload.get("profile_code") or "").strip() or None,
engine_code=str(((manifest.get("engine") or {}).get("code")) or "unknown"),
engine_version=str(((manifest.get("engine") or {}).get("version")) or "") or None,
package_schema=str(manifest.get("schema_version") or "").strip() or None,
package_layout=str(manifest.get("package_layout") or "").strip() or None,
processor_code=str(processor_payload.get("code") or manifest.get("processor_code") or "").strip() or None,
runtime_id=str(runtime_payload.get("runtime_id") or manifest.get("runtime_id") or "").strip() or None,
status="READY",
health_status="OK",
publish_dir=package_dir,
manifest_path=_normalize_path(manifest_path),
source_primary_path=source.get("primary_path"),
native_output_dir=source.get("native_output_dir"),
preview_path=None,
primary_asset_path=None,
summary_json=summary_json,
@@ -1000,6 +1125,7 @@ class ResultCatalogService:
has_warn = False
has_error = False
preview_role = str(canonical_payload.get("preview_asset_role") or "").strip() or "thumb"
for asset_payload in assets_payload:
relative_path = str(asset_payload.get("relative_path") or "").strip()
if not relative_path:
@@ -1031,7 +1157,7 @@ class ResultCatalogService:
product.assets.append(asset)
if asset.is_primary:
product.primary_asset_path = absolute_path
if asset.asset_role == "thumb":
if asset.asset_role == preview_role:
product.preview_path = absolute_path
if asset.is_required and not exists_flag:
has_error = True
@@ -1256,6 +1382,9 @@ class ResultCatalogService:
"selection_strategy": item.selection_strategy,
"profile_code": item.profile_code,
"engine_code": item.engine_code,
"package_schema": item.package_schema,
"processor_code": item.processor_code,
"runtime_id": item.runtime_id,
"status": item.status,
"health_status": item.health_status,
"preview_path": item.preview_path,
@@ -1325,11 +1454,16 @@ class ResultCatalogService:
"profile_code": product.profile_code,
"engine_code": product.engine_code,
"engine_version": product.engine_version,
"package_schema": product.package_schema,
"package_layout": product.package_layout,
"processor_code": product.processor_code,
"runtime_id": product.runtime_id,
"status": product.status,
"health_status": product.health_status,
"publish_dir": product.publish_dir,
"manifest_path": product.manifest_path,
"source_primary_path": product.source_primary_path,
"native_output_dir": product.native_output_dir,
"preview_path": product.preview_path,
"primary_asset_path": product.primary_asset_path,
"summary_json": product.summary_json,
@@ -1440,6 +1574,7 @@ class ResultCatalogService:
db_count = int(db_count_result.scalar_one() or 0)
payload = {
"catalog_name": state.catalog_name,
"product_family": state.product_family,
"storage_root": state.storage_root,
"status": state.status,
"needs_rebuild": state.needs_rebuild,