Apply current workspace changes
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""PyINT pipeline helpers."""
|
||||
@@ -0,0 +1,575 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import bisect
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Optional
|
||||
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
ISCE2_PIPELINE_DIR = SCRIPT_DIR.parent / "isce2_pipeline"
|
||||
if str(ISCE2_PIPELINE_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(ISCE2_PIPELINE_DIR))
|
||||
|
||||
from convert_lt1_orbit_to_isce_xml import StateVector, parse_orbit_file # type: ignore
|
||||
|
||||
|
||||
TRUE_VALUES = {"1", "true", "yes", "on"}
|
||||
VECTOR_POS_RE = re.compile(r"^state_vector_position_(\d+):")
|
||||
VECTOR_VEL_RE = re.compile(r"^state_vector_velocity_(\d+):")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParsedSlcPar:
|
||||
path: Path
|
||||
lines: List[str]
|
||||
trailing_newline: bool
|
||||
acquisition_date: date
|
||||
number_of_state_vectors: int
|
||||
time_of_first_state_vector: float
|
||||
state_vector_interval: float
|
||||
position_line_indexes: Dict[int, int]
|
||||
velocity_line_indexes: Dict[int, int]
|
||||
|
||||
|
||||
def utc_now_text() -> str:
|
||||
return datetime.utcnow().isoformat(timespec="seconds") + "Z"
|
||||
|
||||
|
||||
def read_bool(value: Any, default: bool = False) -> bool:
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, (int, float)):
|
||||
return bool(value)
|
||||
return str(value).strip().lower() in TRUE_VALUES
|
||||
|
||||
|
||||
def windows_path_to_wsl_mount(path: str) -> str:
|
||||
text = str(path or "").strip().strip('"').strip("'")
|
||||
if not text:
|
||||
return ""
|
||||
normalized = text.replace("\\", "/")
|
||||
if normalized.startswith("/"):
|
||||
return normalized
|
||||
if normalized.startswith("//"):
|
||||
return ""
|
||||
match = re.match(r"^([A-Za-z]):/(.*)$", normalized)
|
||||
if not match:
|
||||
return normalized
|
||||
drive_letter = match.group(1).lower()
|
||||
tail = match.group(2).lstrip("/")
|
||||
return f"/mnt/{drive_letter}/{tail}"
|
||||
|
||||
|
||||
def resolve_existing_path(path: str) -> Optional[Path]:
|
||||
text = str(path or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
direct = Path(text)
|
||||
if direct.exists():
|
||||
return direct.resolve()
|
||||
converted = windows_path_to_wsl_mount(text)
|
||||
if converted:
|
||||
candidate = Path(converted)
|
||||
if candidate.exists():
|
||||
return candidate.resolve()
|
||||
return None
|
||||
|
||||
|
||||
def load_json_file(path: Path | None) -> Dict[str, Any]:
|
||||
if path is None or not path.is_file():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
except Exception:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Apply LT-1 precise orbit TXT to Gamma .slc.par state vectors.")
|
||||
parser.add_argument("--date", required=True, help="Scene date in YYYYMMDD format.")
|
||||
parser.add_argument("--manifest-json", default=os.getenv("PYINT_LT1_PRECISE_ORBIT_MANIFEST", ""), help="task_manifest.json path.")
|
||||
parser.add_argument("--summary-json", default="", help="Summary JSON path. Defaults to <slc_dir>/orbit_bridge_summary.json.")
|
||||
parser.add_argument("--role", choices=("auto", "master", "slave"), default="auto")
|
||||
parser.add_argument("--operation-tag", default="raw2slc")
|
||||
parser.add_argument("--mode", default=os.getenv("PYINT_LT1_PRECISE_ORBIT_MODE", "replace"))
|
||||
parser.add_argument("--slc-par", dest="slc_par_files", action="append", default=[], help="Target .slc.par or .slc.update.par file.")
|
||||
parser.add_argument("--backup", dest="backup", action="store_true")
|
||||
parser.add_argument("--no-backup", dest="backup", action="store_false")
|
||||
parser.add_argument("--strict", dest="strict", action="store_true")
|
||||
parser.add_argument("--no-strict", dest="strict", action="store_false")
|
||||
parser.add_argument("--validate-with-orb-filt", dest="validate_with_orb_filt", action="store_true")
|
||||
parser.add_argument("--no-validate-with-orb-filt", dest="validate_with_orb_filt", action="store_false")
|
||||
parser.add_argument(
|
||||
"--orb-filt-degree",
|
||||
type=int,
|
||||
default=int(str(os.getenv("PYINT_LT1_PRECISE_ORBIT_ORB_FILT_DEGREE", "5")).strip() or "5"),
|
||||
)
|
||||
parser.set_defaults(
|
||||
backup=read_bool(os.getenv("PYINT_LT1_PRECISE_ORBIT_BACKUP"), True),
|
||||
strict=read_bool(os.getenv("PYINT_LT1_PRECISE_ORBIT_STRICT"), True),
|
||||
validate_with_orb_filt=read_bool(os.getenv("PYINT_LT1_PRECISE_ORBIT_VALIDATE_WITH_ORB_FILT"), False),
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def get_orbits_payload(manifest: Dict[str, Any]) -> Dict[str, Any]:
|
||||
if isinstance(manifest.get("orbits"), dict):
|
||||
return manifest["orbits"]
|
||||
input_assets = manifest.get("input_assets")
|
||||
if isinstance(input_assets, dict) and isinstance(input_assets.get("orbits"), dict):
|
||||
return input_assets["orbits"]
|
||||
return {}
|
||||
|
||||
|
||||
def resolve_orbit_entry(manifest: Dict[str, Any], date_text: str, role: str) -> Dict[str, Any]:
|
||||
orbits = get_orbits_payload(manifest)
|
||||
candidates: List[tuple[str, Dict[str, Any]]] = []
|
||||
for role_name in ("master", "slave"):
|
||||
item = orbits.get(role_name)
|
||||
if isinstance(item, dict):
|
||||
candidates.append((role_name, item))
|
||||
|
||||
if role in {"master", "slave"}:
|
||||
item = dict(orbits.get(role) or {})
|
||||
if not item:
|
||||
raise RuntimeError(f"Missing orbit entry for role={role}")
|
||||
item["role"] = role
|
||||
return item
|
||||
|
||||
matched: List[Dict[str, Any]] = []
|
||||
for role_name, item in candidates:
|
||||
item_date = str(item.get("date") or "").strip()
|
||||
expected_name = str(item.get("expected_name") or "").strip()
|
||||
if item_date == date_text or date_text in expected_name:
|
||||
candidate = dict(item)
|
||||
candidate["role"] = role_name
|
||||
matched.append(candidate)
|
||||
|
||||
if len(matched) == 1:
|
||||
return matched[0]
|
||||
if not matched:
|
||||
raise RuntimeError(f"Unable to match precise orbit entry for date={date_text}")
|
||||
raise RuntimeError(f"Ambiguous precise orbit entries for date={date_text}")
|
||||
|
||||
|
||||
def resolve_orbit_txt_path(entry: Dict[str, Any]) -> Path:
|
||||
for key in ("staged_path", "path"):
|
||||
candidate = resolve_existing_path(str(entry.get(key) or ""))
|
||||
if candidate is not None and candidate.is_file():
|
||||
return candidate
|
||||
raise FileNotFoundError(
|
||||
f"Precise orbit TXT does not exist: expected {entry.get('expected_name') or '<unknown>'}"
|
||||
)
|
||||
|
||||
|
||||
def parse_float_field(lines: Iterable[str], prefix: str) -> float:
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if not stripped.startswith(prefix):
|
||||
continue
|
||||
_, _, value = stripped.partition(":")
|
||||
first_token = value.strip().split()[0]
|
||||
return float(first_token)
|
||||
raise ValueError(f"Missing field: {prefix}")
|
||||
|
||||
|
||||
def parse_int_field(lines: Iterable[str], prefix: str) -> int:
|
||||
return int(round(parse_float_field(lines, prefix)))
|
||||
|
||||
|
||||
def parse_slc_date(lines: Iterable[str]) -> date:
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if not stripped.startswith("date:"):
|
||||
continue
|
||||
match = re.match(r"^date:\s+(\d+)\s+(\d+)\s+(\d+)", stripped)
|
||||
if not match:
|
||||
raise ValueError(f"Unable to parse date line: {stripped}")
|
||||
return date(int(match.group(1)), int(match.group(2)), int(match.group(3)))
|
||||
raise ValueError("Missing date: field in .slc.par")
|
||||
|
||||
|
||||
def parse_slc_par(path: Path) -> ParsedSlcPar:
|
||||
raw_text = path.read_text(encoding="utf-8", errors="ignore")
|
||||
trailing_newline = raw_text.endswith("\n")
|
||||
lines = raw_text.splitlines()
|
||||
acquisition_date = parse_slc_date(lines)
|
||||
number_of_state_vectors = parse_int_field(lines, "number_of_state_vectors")
|
||||
time_of_first_state_vector = parse_float_field(lines, "time_of_first_state_vector")
|
||||
state_vector_interval = parse_float_field(lines, "state_vector_interval")
|
||||
|
||||
position_line_indexes: Dict[int, int] = {}
|
||||
velocity_line_indexes: Dict[int, int] = {}
|
||||
for idx, line in enumerate(lines):
|
||||
stripped = line.strip()
|
||||
pos_match = VECTOR_POS_RE.match(stripped)
|
||||
if pos_match:
|
||||
position_line_indexes[int(pos_match.group(1))] = idx
|
||||
continue
|
||||
vel_match = VECTOR_VEL_RE.match(stripped)
|
||||
if vel_match:
|
||||
velocity_line_indexes[int(vel_match.group(1))] = idx
|
||||
|
||||
missing_positions = [index for index in range(1, number_of_state_vectors + 1) if index not in position_line_indexes]
|
||||
missing_velocities = [index for index in range(1, number_of_state_vectors + 1) if index not in velocity_line_indexes]
|
||||
if missing_positions or missing_velocities:
|
||||
raise ValueError(
|
||||
"Incomplete state vector block in .slc.par: "
|
||||
f"missing positions={missing_positions[:5]}, missing velocities={missing_velocities[:5]}"
|
||||
)
|
||||
|
||||
return ParsedSlcPar(
|
||||
path=path,
|
||||
lines=lines,
|
||||
trailing_newline=trailing_newline,
|
||||
acquisition_date=acquisition_date,
|
||||
number_of_state_vectors=number_of_state_vectors,
|
||||
time_of_first_state_vector=time_of_first_state_vector,
|
||||
state_vector_interval=state_vector_interval,
|
||||
position_line_indexes=position_line_indexes,
|
||||
velocity_line_indexes=velocity_line_indexes,
|
||||
)
|
||||
|
||||
|
||||
def build_target_times(parsed: ParsedSlcPar) -> List[datetime]:
|
||||
start_time = datetime(parsed.acquisition_date.year, parsed.acquisition_date.month, parsed.acquisition_date.day)
|
||||
return [
|
||||
start_time + timedelta(seconds=parsed.time_of_first_state_vector + parsed.state_vector_interval * index)
|
||||
for index in range(parsed.number_of_state_vectors)
|
||||
]
|
||||
|
||||
|
||||
def norm3(values: Iterable[float]) -> float:
|
||||
items = [float(item) for item in values]
|
||||
return math.sqrt(sum(item * item for item in items))
|
||||
|
||||
|
||||
def interpolate_state_vector(target_time: datetime, vectors: List[StateVector]) -> StateVector:
|
||||
if not vectors:
|
||||
raise ValueError("No precise orbit vectors available for interpolation")
|
||||
|
||||
times = [vector.time for vector in vectors]
|
||||
if target_time < times[0] or target_time > times[-1]:
|
||||
raise ValueError(
|
||||
f"Target time {target_time.isoformat()} is outside orbit range {times[0].isoformat()} - {times[-1].isoformat()}"
|
||||
)
|
||||
|
||||
right_index = bisect.bisect_left(times, target_time)
|
||||
if right_index < len(vectors) and times[right_index] == target_time:
|
||||
return vectors[right_index]
|
||||
if right_index == 0:
|
||||
return vectors[0]
|
||||
if right_index >= len(vectors):
|
||||
return vectors[-1]
|
||||
|
||||
left = vectors[right_index - 1]
|
||||
right = vectors[right_index]
|
||||
interval_seconds = (right.time - left.time).total_seconds()
|
||||
if interval_seconds <= 0:
|
||||
raise ValueError("Orbit vectors are not strictly increasing in time")
|
||||
|
||||
offset_seconds = (target_time - left.time).total_seconds()
|
||||
u = offset_seconds / interval_seconds
|
||||
|
||||
h00 = 2 * u * u * u - 3 * u * u + 1
|
||||
h10 = u * u * u - 2 * u * u + u
|
||||
h01 = -2 * u * u * u + 3 * u * u
|
||||
h11 = u * u * u - u * u
|
||||
|
||||
dh00 = 6 * u * u - 6 * u
|
||||
dh10 = 3 * u * u - 4 * u + 1
|
||||
dh01 = -6 * u * u + 6 * u
|
||||
dh11 = 3 * u * u - 2 * u
|
||||
|
||||
p0 = (left.x, left.y, left.z)
|
||||
p1 = (right.x, right.y, right.z)
|
||||
v0 = (left.vx, left.vy, left.vz)
|
||||
v1 = (right.vx, right.vy, right.vz)
|
||||
|
||||
position = []
|
||||
velocity = []
|
||||
for axis in range(3):
|
||||
pos = (
|
||||
h00 * p0[axis]
|
||||
+ h10 * interval_seconds * v0[axis]
|
||||
+ h01 * p1[axis]
|
||||
+ h11 * interval_seconds * v1[axis]
|
||||
)
|
||||
vel = (
|
||||
dh00 * p0[axis]
|
||||
+ dh10 * interval_seconds * v0[axis]
|
||||
+ dh01 * p1[axis]
|
||||
+ dh11 * interval_seconds * v1[axis]
|
||||
) / interval_seconds
|
||||
position.append(pos)
|
||||
velocity.append(vel)
|
||||
|
||||
return StateVector(
|
||||
time=target_time,
|
||||
x=position[0],
|
||||
y=position[1],
|
||||
z=position[2],
|
||||
vx=velocity[0],
|
||||
vy=velocity[1],
|
||||
vz=velocity[2],
|
||||
)
|
||||
|
||||
|
||||
def format_position_line(index: int, vector: StateVector) -> str:
|
||||
return (
|
||||
f"state_vector_position_{index}:"
|
||||
f" {vector.x:14.4f} {vector.y:14.4f} {vector.z:14.4f} m m m"
|
||||
)
|
||||
|
||||
|
||||
def format_velocity_line(index: int, vector: StateVector) -> str:
|
||||
return (
|
||||
f"state_vector_velocity_{index}:"
|
||||
f" {vector.vx:13.5f} {vector.vy:13.5f} {vector.vz:13.5f} m/s m/s m/s"
|
||||
)
|
||||
|
||||
|
||||
def backup_slc_par(path: Path) -> str:
|
||||
backup_path = path.with_name(path.name + ".orbit_bridge.bak")
|
||||
if not backup_path.exists():
|
||||
shutil.copy2(path, backup_path)
|
||||
return str(backup_path)
|
||||
|
||||
|
||||
def write_bridged_slc_par(
|
||||
parsed: ParsedSlcPar,
|
||||
vectors: List[StateVector],
|
||||
*,
|
||||
backup_enabled: bool,
|
||||
) -> Dict[str, Any]:
|
||||
if len(vectors) != parsed.number_of_state_vectors:
|
||||
raise ValueError("Interpolated vector count does not match .slc.par state vector count")
|
||||
|
||||
backup_path = ""
|
||||
if backup_enabled:
|
||||
backup_path = backup_slc_par(parsed.path)
|
||||
|
||||
updated_lines = list(parsed.lines)
|
||||
for index, vector in enumerate(vectors, start=1):
|
||||
updated_lines[parsed.position_line_indexes[index]] = format_position_line(index, vector)
|
||||
updated_lines[parsed.velocity_line_indexes[index]] = format_velocity_line(index, vector)
|
||||
|
||||
text = "\n".join(updated_lines)
|
||||
if parsed.trailing_newline:
|
||||
text += "\n"
|
||||
parsed.path.write_text(text, encoding="utf-8")
|
||||
|
||||
return {
|
||||
"backup_path": backup_path,
|
||||
}
|
||||
|
||||
|
||||
def run_orb_filt_validation(path: Path, degree: int) -> Dict[str, Any]:
|
||||
command = shutil.which("ORB_filt_spline.py")
|
||||
if not command:
|
||||
return {
|
||||
"requested": True,
|
||||
"ok": False,
|
||||
"status": "missing_command",
|
||||
"command": "ORB_filt_spline.py",
|
||||
}
|
||||
|
||||
validate_path = path.with_name(path.name + ".orb_filt_validate.par")
|
||||
result = subprocess.run(
|
||||
[command, str(path), str(validate_path), "--degree", str(int(degree))],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0 or not validate_path.exists():
|
||||
return {
|
||||
"requested": True,
|
||||
"ok": False,
|
||||
"status": "command_failed",
|
||||
"command": " ".join(result.args),
|
||||
"returncode": int(result.returncode),
|
||||
"stdout": (result.stdout or "")[-2000:],
|
||||
"stderr": (result.stderr or "")[-2000:],
|
||||
"output_par": str(validate_path),
|
||||
}
|
||||
|
||||
current_parsed = parse_slc_par(path)
|
||||
validated_parsed = parse_slc_par(validate_path)
|
||||
position_corrections: List[float] = []
|
||||
velocity_corrections: List[float] = []
|
||||
for index in range(1, current_parsed.number_of_state_vectors + 1):
|
||||
cur_position = parse_vector_values(current_parsed.lines[current_parsed.position_line_indexes[index]])
|
||||
val_position = parse_vector_values(validated_parsed.lines[validated_parsed.position_line_indexes[index]])
|
||||
cur_velocity = parse_vector_values(current_parsed.lines[current_parsed.velocity_line_indexes[index]])
|
||||
val_velocity = parse_vector_values(validated_parsed.lines[validated_parsed.velocity_line_indexes[index]])
|
||||
position_corrections.append(norm3([val_position[i] - cur_position[i] for i in range(3)]))
|
||||
velocity_corrections.append(norm3([val_velocity[i] - cur_velocity[i] for i in range(3)]))
|
||||
|
||||
return {
|
||||
"requested": True,
|
||||
"ok": True,
|
||||
"status": "ok",
|
||||
"command": " ".join(result.args),
|
||||
"returncode": int(result.returncode),
|
||||
"output_par": str(validate_path),
|
||||
"max_position_correction_m": max(position_corrections) if position_corrections else 0.0,
|
||||
"max_velocity_correction_mps": max(velocity_corrections) if velocity_corrections else 0.0,
|
||||
}
|
||||
|
||||
|
||||
def parse_vector_values(line: str) -> List[float]:
|
||||
_, _, payload = line.partition(":")
|
||||
values: List[float] = []
|
||||
for token in payload.split():
|
||||
try:
|
||||
values.append(float(token))
|
||||
except ValueError:
|
||||
break
|
||||
if len(values) == 3:
|
||||
break
|
||||
if len(values) != 3:
|
||||
raise ValueError(f"Unable to parse state vector values from line: {line}")
|
||||
return values
|
||||
|
||||
|
||||
def build_operation_record(args: argparse.Namespace, summary_path: Path, manifest_path: Path | None) -> Dict[str, Any]:
|
||||
return {
|
||||
"generated_at": utc_now_text(),
|
||||
"date": str(args.date or "").strip(),
|
||||
"role": args.role,
|
||||
"operation_tag": str(args.operation_tag or "").strip(),
|
||||
"mode": str(args.mode or "").strip(),
|
||||
"strict": bool(args.strict),
|
||||
"backup": bool(args.backup),
|
||||
"validate_with_orb_filt": bool(args.validate_with_orb_filt),
|
||||
"orb_filt_degree": int(args.orb_filt_degree),
|
||||
"manifest_json": str(manifest_path) if manifest_path else "",
|
||||
"summary_json": str(summary_path),
|
||||
"slc_par_files": [str(path) for path in args.slc_par_files],
|
||||
"ok": False,
|
||||
"error": "",
|
||||
"orbit_source": {},
|
||||
"results": [],
|
||||
}
|
||||
|
||||
|
||||
def append_operation_summary(summary_path: Path, operation: Dict[str, Any]) -> None:
|
||||
existing = load_json_file(summary_path)
|
||||
operations = existing.get("operations")
|
||||
if not isinstance(operations, list):
|
||||
operations = []
|
||||
operations.append(operation)
|
||||
payload = {
|
||||
"generated_at": existing.get("generated_at") or utc_now_text(),
|
||||
"last_updated_at": utc_now_text(),
|
||||
"ok": all(bool(item.get("ok")) for item in operations),
|
||||
"operation_count": len(operations),
|
||||
"operations": operations,
|
||||
}
|
||||
summary_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
summary_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def default_summary_path(slc_par_files: List[str]) -> Path:
|
||||
first_path = Path(slc_par_files[0]).resolve()
|
||||
return first_path.parent / "orbit_bridge_summary.json"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
if not args.slc_par_files:
|
||||
raise SystemExit("--slc-par must be specified at least once")
|
||||
|
||||
manifest_path = resolve_existing_path(args.manifest_json) if args.manifest_json else None
|
||||
summary_path = Path(args.summary_json).resolve() if args.summary_json else default_summary_path(args.slc_par_files)
|
||||
operation = build_operation_record(args, summary_path, manifest_path)
|
||||
|
||||
exit_code = 0
|
||||
try:
|
||||
if manifest_path is None:
|
||||
raise FileNotFoundError("Precise orbit manifest JSON is not available")
|
||||
manifest = load_json_file(manifest_path)
|
||||
orbit_entry = resolve_orbit_entry(manifest, str(args.date or "").strip(), args.role)
|
||||
orbit_txt_path = resolve_orbit_txt_path(orbit_entry)
|
||||
orbit_vectors = sorted(parse_orbit_file(orbit_txt_path), key=lambda item: item.time)
|
||||
operation["orbit_source"] = {
|
||||
"role": orbit_entry.get("role"),
|
||||
"satellite": orbit_entry.get("satellite"),
|
||||
"date": orbit_entry.get("date"),
|
||||
"expected_name": orbit_entry.get("expected_name"),
|
||||
"source_txt": str(orbit_txt_path),
|
||||
"vector_count": len(orbit_vectors),
|
||||
"time_start": orbit_vectors[0].time.isoformat() if orbit_vectors else "",
|
||||
"time_stop": orbit_vectors[-1].time.isoformat() if orbit_vectors else "",
|
||||
}
|
||||
|
||||
results: List[Dict[str, Any]] = []
|
||||
for slc_par_text in args.slc_par_files:
|
||||
slc_par_path = resolve_existing_path(slc_par_text)
|
||||
if slc_par_path is None or not slc_par_path.is_file():
|
||||
raise FileNotFoundError(f"Target .slc.par does not exist: {slc_par_text}")
|
||||
|
||||
parsed = parse_slc_par(slc_par_path)
|
||||
target_times = build_target_times(parsed)
|
||||
bridged_vectors = [interpolate_state_vector(target_time, orbit_vectors) for target_time in target_times]
|
||||
write_info = write_bridged_slc_par(parsed, bridged_vectors, backup_enabled=bool(args.backup))
|
||||
validation = (
|
||||
run_orb_filt_validation(slc_par_path, args.orb_filt_degree)
|
||||
if args.validate_with_orb_filt
|
||||
else {"requested": False, "ok": True, "status": "skipped"}
|
||||
)
|
||||
result_item = {
|
||||
"path": str(slc_par_path),
|
||||
"status": "applied",
|
||||
"ok": bool(validation.get("ok", False)),
|
||||
"backup_path": write_info.get("backup_path", ""),
|
||||
"vector_count": parsed.number_of_state_vectors,
|
||||
"time_of_first_state_vector": parsed.time_of_first_state_vector,
|
||||
"state_vector_interval": parsed.state_vector_interval,
|
||||
"validation": validation,
|
||||
"first_target_time": target_times[0].isoformat() if target_times else "",
|
||||
"last_target_time": target_times[-1].isoformat() if target_times else "",
|
||||
"max_position_norm_m": max(norm3((vector.x, vector.y, vector.z)) for vector in bridged_vectors) if bridged_vectors else 0.0,
|
||||
"max_velocity_norm_mps": max(norm3((vector.vx, vector.vy, vector.vz)) for vector in bridged_vectors) if bridged_vectors else 0.0,
|
||||
}
|
||||
results.append(result_item)
|
||||
|
||||
operation["results"] = results
|
||||
operation["ok"] = all(bool(item.get("ok")) for item in results)
|
||||
if not operation["ok"]:
|
||||
operation["error"] = "One or more target .slc.par files failed validation"
|
||||
if args.strict:
|
||||
exit_code = 1
|
||||
except Exception as exc:
|
||||
operation["error"] = str(exc)
|
||||
operation["ok"] = False
|
||||
exit_code = 1 if args.strict else 0
|
||||
|
||||
append_operation_summary(summary_path, operation)
|
||||
if operation.get("error"):
|
||||
print(operation["error"], file=sys.stderr)
|
||||
else:
|
||||
applied_count = len(operation.get("results") or [])
|
||||
print(
|
||||
f"Applied LT-1 precise orbit bridge to {applied_count} file(s) for {operation.get('date')} "
|
||||
f"[{operation.get('operation_tag')}]"
|
||||
)
|
||||
return exit_code
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
_pyint_gamma_die() {
|
||||
echo "$1" >&2
|
||||
return 1 2>/dev/null || exit 1
|
||||
}
|
||||
|
||||
_pyint_gamma_home=""
|
||||
if [ -n "${PYINT_GAMMA_HOME:-}" ] && [ -d "${PYINT_GAMMA_HOME}" ]; then
|
||||
_pyint_gamma_home="${PYINT_GAMMA_HOME}"
|
||||
elif [ -n "${GAMMA_HOME:-}" ] && [ -d "${GAMMA_HOME}" ]; then
|
||||
_pyint_gamma_home="${GAMMA_HOME}"
|
||||
else
|
||||
for _candidate in \
|
||||
/usr/local/GAMMA_SOFTWARE-20240627 \
|
||||
/usr/local/GAMMA_SOFTWARE-* \
|
||||
/opt/GAMMA_SOFTWARE-*; do
|
||||
[ -d "${_candidate}" ] || continue
|
||||
_pyint_gamma_home="${_candidate}"
|
||||
break
|
||||
done
|
||||
fi
|
||||
|
||||
[ -n "${_pyint_gamma_home}" ] || _pyint_gamma_die "Gamma home not found."
|
||||
|
||||
export GAMMA_HOME="${_pyint_gamma_home}"
|
||||
export MSP_HOME="${GAMMA_HOME}/MSP"
|
||||
export ISP_HOME="${GAMMA_HOME}/ISP"
|
||||
export DIFF_HOME="${GAMMA_HOME}/DIFF"
|
||||
export DISP_HOME="${GAMMA_HOME}/DISP"
|
||||
export LAT_HOME="${GAMMA_HOME}/LAT"
|
||||
export IPTA_HOME="${GAMMA_HOME}/IPTA"
|
||||
export GEO_HOME="${GAMMA_HOME}/GEO"
|
||||
|
||||
_pyint_gamma_prepend_path() {
|
||||
local _dir="$1"
|
||||
[ -d "${_dir}" ] || return 0
|
||||
case ":${PATH}:" in
|
||||
*":${_dir}:"*) ;;
|
||||
*) PATH="${_dir}:${PATH}" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
for _gamma_dir in \
|
||||
"${MSP_HOME}/bin" \
|
||||
"${ISP_HOME}/bin" \
|
||||
"${DIFF_HOME}/bin" \
|
||||
"${DISP_HOME}/bin" \
|
||||
"${LAT_HOME}/bin" \
|
||||
"${IPTA_HOME}/bin" \
|
||||
"${GEO_HOME}/bin" \
|
||||
"${MSP_HOME}/scripts" \
|
||||
"${ISP_HOME}/scripts" \
|
||||
"${DIFF_HOME}/scripts" \
|
||||
"${DISP_HOME}/scripts" \
|
||||
"${LAT_HOME}/scripts" \
|
||||
"${IPTA_HOME}/scripts" \
|
||||
"${GEO_HOME}/scripts"; do
|
||||
_pyint_gamma_prepend_path "${_gamma_dir}"
|
||||
done
|
||||
|
||||
export PATH
|
||||
export OS="linux64"
|
||||
export HDF5_DISABLE_VERSION_CHECK="1"
|
||||
export GNUTERM="${GNUTERM:-qt}"
|
||||
export GAMMA_RASTER="${GAMMA_RASTER:-BMP}"
|
||||
export PYTHONPATH=".:${GAMMA_HOME}${PYTHONPATH:+:${PYTHONPATH}}"
|
||||
|
||||
_pyint_repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
|
||||
_pyint_script_dir="${_pyint_repo_root}/third_party/PyINT/pyint"
|
||||
_pyint_gamma_prepend_path "${_pyint_script_dir}"
|
||||
|
||||
unset _pyint_gamma_home
|
||||
unset _gamma_dir
|
||||
unset _pyint_repo_root
|
||||
unset _pyint_script_dir
|
||||
unset -f _pyint_gamma_prepend_path
|
||||
unset -f _pyint_gamma_die
|
||||
@@ -0,0 +1,847 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List
|
||||
|
||||
|
||||
LT1_INPUT_GLOBS = ("LT1*.tar.gz", "LT1*.tiff")
|
||||
PAIR_META_FILENAME = ".dinsar_pair.json"
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Materialize a PyINT LT-1 workspace from an existing Task_xxx pair directory."
|
||||
)
|
||||
parser.add_argument("task_dir", help="Task directory containing master/ and slave/ subdirectories.")
|
||||
parser.add_argument("--project-dir", required=True, help="Workspace directory for the generated PyINT project.")
|
||||
parser.add_argument("--template-root", required=True, help="Directory where the generated template will be written.")
|
||||
parser.add_argument("--output-dir", required=True, help="Directory where normalized native outputs will be copied.")
|
||||
parser.add_argument("--pyint-home", required=True, help="PyINT repository root inside WSL.")
|
||||
parser.add_argument("--pyint-app-script", required=True, help="pyintApp.py path inside WSL.")
|
||||
parser.add_argument("--python", required=True, help="Python interpreter used to run PyINT inside WSL.")
|
||||
parser.add_argument("--dem-root", required=True, help="DEMDIR root used by PyINT.")
|
||||
parser.add_argument("--dem-mode", default="local_fabdem", help="DEM strategy used for this run.")
|
||||
parser.add_argument("--fabdem-root", default="", help="Optional FABDEM tile root inside WSL.")
|
||||
parser.add_argument("--prepared-dem-path", default="", help="Optional existing DEM path inside WSL.")
|
||||
parser.add_argument("--opentopo-dem-type", default="SRTMGL1", help="DEM type when using OpenTopography.")
|
||||
parser.add_argument("--opentopo-api-key", default="", help="Optional OpenTopography API key.")
|
||||
parser.add_argument("--project-name", required=True, help="Unique PyINT project name for this run.")
|
||||
parser.add_argument("--gamma-env-script", default="", help="Optional shell script used to expose GAMMA commands.")
|
||||
parser.add_argument("--pair-key", default="", help="Pair key recorded into the run summary.")
|
||||
parser.add_argument("--task-alias", default="", help="Task alias recorded into the run summary.")
|
||||
parser.add_argument("--orbit-policy", default="require_txt", help="Orbit governance policy recorded into the run summary.")
|
||||
parser.add_argument("--input-assets-dir", default="", help="Optional input_assets directory for this run.")
|
||||
parser.add_argument("--input-assets-json", default="", help="Optional task_manifest.json path for this run.")
|
||||
parser.add_argument("--master-date", default="", help="Master date in YYYYMMDD format.")
|
||||
parser.add_argument("--slave-date", default="", help="Slave date in YYYYMMDD format.")
|
||||
parser.add_argument("--time-baseline-days", type=int, default=0, help="Time baseline to record in ifgram_list.txt.")
|
||||
parser.add_argument("--range-looks", type=int, default=2)
|
||||
parser.add_argument("--azimuth-looks", type=int, default=2)
|
||||
parser.add_argument("--parallel-workers", type=int, default=1)
|
||||
parser.add_argument("--lt1-precise-orbit-enabled", default="true", help="Enable LT-1 precise orbit bridge.")
|
||||
parser.add_argument("--lt1-precise-orbit-mode", default="replace", help="LT-1 precise orbit bridge mode.")
|
||||
parser.add_argument("--lt1-precise-orbit-strict", default="true", help="Fail the run if precise orbit bridge fails.")
|
||||
parser.add_argument(
|
||||
"--lt1-precise-orbit-validate-with-orb-filt",
|
||||
default="false",
|
||||
help="Run ORB_filt_spline.py on a validation copy after rewriting state vectors.",
|
||||
)
|
||||
parser.add_argument("--lt1-precise-orbit-backup", default="true", help="Backup original .slc.par before rewrite.")
|
||||
parser.add_argument("--lt1-precise-orbit-orb-filt-degree", type=int, default=5)
|
||||
parser.add_argument("--unwrap", dest="unwrap", action="store_true")
|
||||
parser.add_argument("--no-unwrap", dest="unwrap", action="store_false")
|
||||
parser.add_argument("--geocode", dest="geocode", action="store_true")
|
||||
parser.add_argument("--no-geocode", dest="geocode", action="store_false")
|
||||
parser.add_argument("--force", action="store_true", help="Delete an existing run root before rebuilding it.")
|
||||
parser.set_defaults(unwrap=True, geocode=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def normalize_date_text(value: Any) -> str:
|
||||
text = str(value or "").strip()
|
||||
digits = "".join(ch for ch in text if ch.isdigit())
|
||||
if len(digits) >= 8 and digits.startswith("20"):
|
||||
return digits[:8]
|
||||
return ""
|
||||
|
||||
|
||||
def normalize_bool_text(value: Any, default: bool = False) -> bool:
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, (int, float)):
|
||||
return bool(value)
|
||||
return str(value).strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def ensure_directory(path: Path) -> Path:
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
|
||||
def safe_rmtree(path: Path) -> None:
|
||||
if not path.exists():
|
||||
return
|
||||
resolved = path.resolve()
|
||||
if len(resolved.parts) < 4:
|
||||
raise RuntimeError(f"Refusing to remove an unsafe path: {resolved}")
|
||||
shutil.rmtree(resolved)
|
||||
|
||||
|
||||
def load_pair_meta(task_dir: Path) -> Dict[str, Any]:
|
||||
path = task_dir / PAIR_META_FILENAME
|
||||
if not path.is_file():
|
||||
return {}
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def load_json_file(path: Path | None) -> Dict[str, Any]:
|
||||
if path is None or not path.is_file():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
except Exception:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def discover_lt1_archives(scene_dir: Path) -> List[Path]:
|
||||
if not scene_dir.is_dir():
|
||||
return []
|
||||
items: List[Path] = []
|
||||
for pattern in LT1_INPUT_GLOBS:
|
||||
items.extend(path.resolve() for path in scene_dir.rglob(pattern) if path.is_file())
|
||||
return sorted(set(items))
|
||||
|
||||
|
||||
def infer_scene_date(paths: Iterable[Path]) -> str:
|
||||
dates = {
|
||||
normalize_date_text(path.name)
|
||||
for path in paths
|
||||
if normalize_date_text(path.name)
|
||||
}
|
||||
if len(dates) == 1:
|
||||
return next(iter(dates))
|
||||
return ""
|
||||
|
||||
|
||||
def hardlink_or_copy(src: Path, dst: Path) -> str:
|
||||
ensure_directory(dst.parent)
|
||||
if dst.exists():
|
||||
return "skipped"
|
||||
try:
|
||||
os.link(src, dst)
|
||||
return "hardlinked"
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
dst.symlink_to(src)
|
||||
return "symlinked"
|
||||
except OSError:
|
||||
pass
|
||||
shutil.copy2(src, dst)
|
||||
return "copied"
|
||||
|
||||
|
||||
def collect_related_lt1_input_files(path: Path) -> List[Path]:
|
||||
resolved = path.resolve()
|
||||
if resolved.suffix.lower() != ".tiff":
|
||||
return [resolved]
|
||||
|
||||
stem = resolved.stem
|
||||
files = [
|
||||
candidate.resolve()
|
||||
for candidate in resolved.parent.iterdir()
|
||||
if candidate.is_file() and (candidate.name == resolved.name or candidate.name.startswith(stem))
|
||||
]
|
||||
return sorted(set(files))
|
||||
|
||||
|
||||
def write_text(path: Path, content: str) -> Path:
|
||||
ensure_directory(path.parent)
|
||||
path.write_text(content, encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def inspect_prepared_dem_path(path_text: str) -> Dict[str, str]:
|
||||
text = str(path_text or "").strip()
|
||||
if not text:
|
||||
return {
|
||||
"path": "",
|
||||
"kind": "",
|
||||
"direct_dem_path": "",
|
||||
"source_dem_path": "",
|
||||
"source_dem_open_path": "",
|
||||
}
|
||||
|
||||
path = Path(text)
|
||||
try:
|
||||
resolved_path = path.resolve()
|
||||
except Exception:
|
||||
resolved_path = path
|
||||
|
||||
gamma_par_path = Path(str(resolved_path) + ".par")
|
||||
vrt_path = Path(str(resolved_path) + ".vrt")
|
||||
xml_path = Path(str(resolved_path) + ".xml")
|
||||
hdr_path = Path(str(resolved_path) + ".hdr")
|
||||
|
||||
if resolved_path.is_file() and gamma_par_path.is_file():
|
||||
return {
|
||||
"path": str(resolved_path),
|
||||
"kind": "gamma_ready",
|
||||
"direct_dem_path": str(resolved_path),
|
||||
"source_dem_path": "",
|
||||
"source_dem_open_path": "",
|
||||
}
|
||||
|
||||
if resolved_path.is_file() and (vrt_path.is_file() or xml_path.is_file() or hdr_path.is_file()):
|
||||
return {
|
||||
"path": str(resolved_path),
|
||||
"kind": "source_dem",
|
||||
"direct_dem_path": "",
|
||||
"source_dem_path": str(resolved_path),
|
||||
"source_dem_open_path": str(vrt_path if vrt_path.is_file() else resolved_path),
|
||||
}
|
||||
|
||||
if resolved_path.suffix.lower() == ".vrt" and resolved_path.is_file():
|
||||
return {
|
||||
"path": str(resolved_path),
|
||||
"kind": "source_dem",
|
||||
"direct_dem_path": "",
|
||||
"source_dem_path": str(resolved_path),
|
||||
"source_dem_open_path": str(resolved_path),
|
||||
}
|
||||
|
||||
return {
|
||||
"path": str(resolved_path),
|
||||
"kind": "",
|
||||
"direct_dem_path": "",
|
||||
"source_dem_path": "",
|
||||
"source_dem_open_path": "",
|
||||
}
|
||||
|
||||
|
||||
def build_template_text(
|
||||
*,
|
||||
project_name: str,
|
||||
master_date: str,
|
||||
range_looks: int,
|
||||
azimuth_looks: int,
|
||||
parallel_workers: int,
|
||||
unwrap: bool,
|
||||
geocode: bool,
|
||||
dem_mode: str,
|
||||
fabdem_root: str,
|
||||
prepared_dem_path: str,
|
||||
opentopo_dem_type: str,
|
||||
opentopo_api_key: str,
|
||||
) -> str:
|
||||
prepared_dem = inspect_prepared_dem_path(prepared_dem_path) if dem_mode == "prepared_file" else {}
|
||||
lines = [
|
||||
f"# Auto-generated for {project_name}",
|
||||
"satelite=LT",
|
||||
f"masterDate={master_date}",
|
||||
f"range_looks={int(range_looks)}",
|
||||
f"azimuth_looks={int(azimuth_looks)}",
|
||||
"download_data=0",
|
||||
"raw2slc_all=1",
|
||||
f"raw2slc_all_parallel={int(parallel_workers)}",
|
||||
"extract_burst_all=0",
|
||||
f"extract_all_parallel={int(parallel_workers)}",
|
||||
"coreg_all=1",
|
||||
f"coreg_all_parallel={int(parallel_workers)}",
|
||||
"select_pairs=0",
|
||||
"diff_all=1",
|
||||
f"diff_all_parallel={int(parallel_workers)}",
|
||||
"pot_all=0",
|
||||
f"pot_all_parallel={int(parallel_workers)}",
|
||||
f"unwrap_all={1 if unwrap else 0}",
|
||||
f"unwrap_all_parallel={int(parallel_workers)}",
|
||||
"atmcor_all=0",
|
||||
f"atmcor_all_parallel={int(parallel_workers)}",
|
||||
f"geocode_all={1 if geocode else 0}",
|
||||
f"geocode_all_parallel={int(parallel_workers)}",
|
||||
"gacos_correction=0",
|
||||
"load_data=0",
|
||||
"geocode_products=hyp3,licsbas",
|
||||
]
|
||||
if dem_mode == "local_fabdem" and fabdem_root:
|
||||
lines.append(f"fabdem_dir={fabdem_root}")
|
||||
else:
|
||||
lines.append("fabdem_dir=-")
|
||||
if dem_mode == "prepared_file" and prepared_dem.get("kind") == "gamma_ready":
|
||||
lines.append(f"DEM={prepared_dem['direct_dem_path']}")
|
||||
if dem_mode == "prepared_file" and prepared_dem.get("kind") == "source_dem":
|
||||
lines.append(f"prepared_dem_source={prepared_dem['source_dem_path']}")
|
||||
else:
|
||||
lines.append("prepared_dem_source=-")
|
||||
if dem_mode == "opentopo":
|
||||
lines.append(f"opentopo_dem_type={opentopo_dem_type or 'SRTMGL1'}")
|
||||
lines.append(f"opentopo_api_key={opentopo_api_key or '-'}")
|
||||
else:
|
||||
lines.append("opentopo_dem_type=-")
|
||||
lines.append("opentopo_api_key=-")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def write_ifgram_list(path: Path, master_date: str, slave_date: str, time_baseline_days: int) -> Path:
|
||||
content = f"{master_date}-{slave_date} {int(time_baseline_days)} 0.0\n"
|
||||
return write_text(path, content)
|
||||
|
||||
|
||||
def write_wrapper_scripts(
|
||||
*,
|
||||
wrappers_dir: Path,
|
||||
pyint_home: Path,
|
||||
python_cmd: str,
|
||||
gamma_env_script: str,
|
||||
) -> List[Path]:
|
||||
scripts_dir = pyint_home / "pyint"
|
||||
if not scripts_dir.is_dir():
|
||||
raise FileNotFoundError(f"PyINT scripts directory not found: {scripts_dir}")
|
||||
|
||||
ensure_directory(wrappers_dir)
|
||||
created: List[Path] = []
|
||||
pyint_scripts = sorted(path for path in scripts_dir.glob("*.py") if path.is_file())
|
||||
for script_path in pyint_scripts:
|
||||
wrapper_path = wrappers_dir / script_path.name
|
||||
lines = [
|
||||
"#!/usr/bin/env bash",
|
||||
"set -e",
|
||||
]
|
||||
if gamma_env_script:
|
||||
lines.append(f". '{gamma_env_script}' >/dev/null 2>&1")
|
||||
lines.extend(
|
||||
[
|
||||
f"export PATH='{wrappers_dir}':'{scripts_dir}':\"$PATH\"",
|
||||
f"export PYTHONPATH='{pyint_home}':\"${{PYTHONPATH:-}}\"",
|
||||
f"exec '{python_cmd}' '{script_path}' \"$@\"",
|
||||
"",
|
||||
]
|
||||
)
|
||||
write_text(wrapper_path, "\n".join(lines))
|
||||
wrapper_path.chmod(wrapper_path.stat().st_mode | stat.S_IEXEC)
|
||||
created.append(wrapper_path)
|
||||
return created
|
||||
|
||||
|
||||
def run_logged(command: List[str], *, env: Dict[str, str], cwd: Path, stdout_path: Path, stderr_path: Path) -> subprocess.CompletedProcess[str]:
|
||||
result = subprocess.run(
|
||||
command,
|
||||
cwd=str(cwd),
|
||||
env=env,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
write_text(stdout_path, result.stdout or "")
|
||||
write_text(stderr_path, result.stderr or "")
|
||||
return result
|
||||
|
||||
|
||||
def require_task_layout(task_dir: Path) -> None:
|
||||
missing = [name for name in ("master", "slave") if not (task_dir / name).is_dir()]
|
||||
if missing:
|
||||
raise FileNotFoundError(f"Task directory is missing required subdirectories: {', '.join(missing)}")
|
||||
|
||||
|
||||
def collect_expected_outputs(project_dir: Path, pair_name: str, range_looks: int) -> Dict[str, str]:
|
||||
pair_dir = project_dir / "ifgrams" / pair_name
|
||||
look_text = f"{int(range_looks)}rlks"
|
||||
return {
|
||||
"pair_dir": str(pair_dir),
|
||||
"diff_filt": str(pair_dir / f"{pair_name}_{look_text}.diff_filt"),
|
||||
"coh": str(pair_dir / f"{pair_name}_{look_text}.diff_filt.cor"),
|
||||
"unw": str(pair_dir / f"{pair_name}_{look_text}.diff_filt.unw"),
|
||||
"geo_unw": str(pair_dir / f"geo_{pair_name}_{look_text}.diff_filt.unw"),
|
||||
"geo_los": str(pair_dir / f"geo_{pair_name}_{look_text}.los_disp"),
|
||||
}
|
||||
|
||||
|
||||
def assert_required_outputs(outputs: Dict[str, str], *, unwrap: bool, geocode: bool) -> None:
|
||||
required = ["pair_dir", "diff_filt", "coh"]
|
||||
if unwrap:
|
||||
required.append("unw")
|
||||
if geocode:
|
||||
required.append("geo_unw")
|
||||
missing = [name for name in required if not Path(outputs[name]).exists()]
|
||||
if missing:
|
||||
raise RuntimeError(f"PyINT run finished but required outputs are missing: {', '.join(missing)}")
|
||||
|
||||
|
||||
def is_binary_all_zero(path: Path, *, chunk_size: int = 1024 * 1024) -> bool:
|
||||
if not path.is_file():
|
||||
return False
|
||||
with path.open("rb") as handle:
|
||||
while True:
|
||||
chunk = handle.read(chunk_size)
|
||||
if not chunk:
|
||||
return True
|
||||
if any(chunk):
|
||||
return False
|
||||
|
||||
|
||||
def collect_output_sanity_checks(
|
||||
outputs: Dict[str, str],
|
||||
*,
|
||||
unwrap: bool,
|
||||
geocode: bool,
|
||||
) -> List[Dict[str, Any]]:
|
||||
targets = [
|
||||
("diff_filt", "wrapped differential interferogram"),
|
||||
("coh", "coherence"),
|
||||
]
|
||||
if unwrap:
|
||||
targets.append(("unw", "unwrapped interferogram"))
|
||||
if geocode:
|
||||
targets.extend(
|
||||
[
|
||||
("geo_unw", "geocoded unwrapped interferogram"),
|
||||
("geo_los", "geocoded LOS displacement"),
|
||||
]
|
||||
)
|
||||
|
||||
checks: List[Dict[str, Any]] = []
|
||||
for name, label in targets:
|
||||
path = Path(outputs[name])
|
||||
exists = path.exists()
|
||||
size_bytes = path.stat().st_size if exists else 0
|
||||
all_zero = exists and is_binary_all_zero(path)
|
||||
checks.append(
|
||||
{
|
||||
"name": name,
|
||||
"label": label,
|
||||
"path": str(path),
|
||||
"exists": exists,
|
||||
"size_bytes": int(size_bytes),
|
||||
"all_zero": bool(all_zero),
|
||||
"ok": bool(exists and size_bytes > 0 and not all_zero),
|
||||
}
|
||||
)
|
||||
return checks
|
||||
|
||||
|
||||
def assert_output_sanity(checks: List[Dict[str, Any]]) -> None:
|
||||
failed = [item for item in checks if not item.get("ok")]
|
||||
if not failed:
|
||||
return
|
||||
detail = ", ".join(f"{item['name']}={item['path']}" for item in failed)
|
||||
raise RuntimeError(f"PyINT run produced invalid all-zero binary outputs: {detail}")
|
||||
|
||||
|
||||
def collect_stage_error_logs(project_dir: Path) -> Dict[str, str]:
|
||||
logs: Dict[str, str] = {}
|
||||
for filename in (
|
||||
"coreg_gamma_all.err",
|
||||
"diff_gamma_all.err",
|
||||
"unwrap_gamma_all.err",
|
||||
"geocode_gamma_all.err",
|
||||
):
|
||||
path = project_dir / filename
|
||||
if path.is_file():
|
||||
logs[filename] = str(path)
|
||||
return logs
|
||||
|
||||
|
||||
def copy_native_outputs(
|
||||
*,
|
||||
project_dir: Path,
|
||||
output_dir: Path,
|
||||
pair_name: str,
|
||||
template_path: Path,
|
||||
ifgram_list_path: Path,
|
||||
stdout_path: Path,
|
||||
stderr_path: Path,
|
||||
) -> Dict[str, str]:
|
||||
ensure_directory(output_dir)
|
||||
native_pair_dir = project_dir / "ifgrams" / pair_name
|
||||
target_pair_dir = output_dir / "ifgrams" / pair_name
|
||||
if native_pair_dir.is_dir():
|
||||
shutil.copytree(native_pair_dir, target_pair_dir, dirs_exist_ok=True)
|
||||
|
||||
target_template = output_dir / template_path.name
|
||||
shutil.copy2(template_path, target_template)
|
||||
target_ifgram_list = output_dir / ifgram_list_path.name
|
||||
shutil.copy2(ifgram_list_path, target_ifgram_list)
|
||||
target_stdout = output_dir / stdout_path.name
|
||||
target_stderr = output_dir / stderr_path.name
|
||||
shutil.copy2(stdout_path, target_stdout)
|
||||
shutil.copy2(stderr_path, target_stderr)
|
||||
|
||||
return {
|
||||
"pair_dir": str(target_pair_dir),
|
||||
"template_path": str(target_template),
|
||||
"ifgram_list_path": str(target_ifgram_list),
|
||||
"stdout_path": str(target_stdout),
|
||||
"stderr_path": str(target_stderr),
|
||||
}
|
||||
|
||||
|
||||
def collect_orbit_bridge_summaries(project_dir: Path) -> List[Dict[str, Any]]:
|
||||
summaries: List[Dict[str, Any]] = []
|
||||
slc_root = project_dir / "SLC"
|
||||
if not slc_root.is_dir():
|
||||
return summaries
|
||||
|
||||
for summary_path in sorted(slc_root.glob("*/orbit_bridge_summary.json")):
|
||||
payload = load_json_file(summary_path)
|
||||
operations = payload.get("operations") if isinstance(payload.get("operations"), list) else []
|
||||
failed_operations = [item for item in operations if not item.get("ok")]
|
||||
summaries.append(
|
||||
{
|
||||
"path": str(summary_path),
|
||||
"date_dir": summary_path.parent.name,
|
||||
"ok": bool(payload.get("ok", not failed_operations)),
|
||||
"operation_count": len(operations),
|
||||
"failed_operation_count": len(failed_operations),
|
||||
"operations": operations,
|
||||
"payload": payload,
|
||||
}
|
||||
)
|
||||
return summaries
|
||||
|
||||
|
||||
def copy_orbit_bridge_summaries(summaries: List[Dict[str, Any]], output_dir: Path) -> Dict[str, str]:
|
||||
if not summaries:
|
||||
return {}
|
||||
|
||||
target_dir = ensure_directory(output_dir / "orbit_bridge")
|
||||
copied: Dict[str, str] = {}
|
||||
for item in summaries:
|
||||
source_path = Path(item["path"])
|
||||
target_path = target_dir / f"{item['date_dir']}_orbit_bridge_summary.json"
|
||||
shutil.copy2(source_path, target_path)
|
||||
copied[item["date_dir"]] = str(target_path)
|
||||
return copied
|
||||
|
||||
|
||||
def assert_orbit_bridge_ok(
|
||||
*,
|
||||
enabled: bool,
|
||||
strict: bool,
|
||||
summaries: List[Dict[str, Any]],
|
||||
expected_dates: Iterable[str],
|
||||
) -> None:
|
||||
if not enabled:
|
||||
return
|
||||
if not strict:
|
||||
return
|
||||
|
||||
expected = {str(item).strip() for item in expected_dates if str(item).strip()}
|
||||
found = {str(item.get("date_dir") or "").strip() for item in summaries if str(item.get("date_dir") or "").strip()}
|
||||
missing = sorted(expected - found)
|
||||
if missing:
|
||||
raise RuntimeError(f"LT-1 precise orbit bridge summary is missing for: {', '.join(missing)}")
|
||||
|
||||
failed = [item for item in summaries if not item.get("ok")]
|
||||
if failed:
|
||||
failed_dates = ", ".join(sorted(str(item.get("date_dir") or "") for item in failed))
|
||||
raise RuntimeError(f"LT-1 precise orbit bridge reported failures for: {failed_dates}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
|
||||
task_dir = Path(args.task_dir).resolve()
|
||||
project_dir = Path(args.project_dir).resolve()
|
||||
run_root = project_dir.parent
|
||||
template_root = Path(args.template_root).resolve()
|
||||
output_dir = Path(args.output_dir).resolve()
|
||||
pyint_home = Path(args.pyint_home).resolve()
|
||||
pyint_app_script = Path(args.pyint_app_script).resolve()
|
||||
dem_root = Path(args.dem_root).resolve()
|
||||
input_assets_dir = Path(args.input_assets_dir).resolve() if args.input_assets_dir else None
|
||||
input_assets_json = Path(args.input_assets_json).resolve() if args.input_assets_json else None
|
||||
input_assets_payload = load_json_file(input_assets_json)
|
||||
precise_orbit_enabled = normalize_bool_text(args.lt1_precise_orbit_enabled, True)
|
||||
precise_orbit_strict = normalize_bool_text(args.lt1_precise_orbit_strict, True)
|
||||
precise_orbit_validate_with_orb_filt = normalize_bool_text(
|
||||
args.lt1_precise_orbit_validate_with_orb_filt,
|
||||
False,
|
||||
)
|
||||
precise_orbit_backup = normalize_bool_text(args.lt1_precise_orbit_backup, True)
|
||||
precise_orbit_mode = str(args.lt1_precise_orbit_mode or "replace").strip().lower() or "replace"
|
||||
precise_orbit_helper = (Path(__file__).resolve().parent / "apply_lt1_precise_orbit.py").resolve()
|
||||
dem_mode = str(args.dem_mode or "local_fabdem").strip().lower() or "local_fabdem"
|
||||
prepared_dem_info = inspect_prepared_dem_path(args.prepared_dem_path) if dem_mode == "prepared_file" else {}
|
||||
|
||||
require_task_layout(task_dir)
|
||||
if not pyint_app_script.is_file():
|
||||
raise FileNotFoundError(f"pyintApp.py not found: {pyint_app_script}")
|
||||
if precise_orbit_enabled and not precise_orbit_helper.is_file():
|
||||
raise FileNotFoundError(f"Precise orbit bridge helper not found: {precise_orbit_helper}")
|
||||
if precise_orbit_enabled and input_assets_json is None:
|
||||
raise RuntimeError("LT-1 precise orbit bridge requires --input-assets-json.")
|
||||
if dem_mode == "prepared_file" and not prepared_dem_info.get("kind"):
|
||||
raise RuntimeError(
|
||||
"Prepared DEM mode requires either a Gamma DEM with .par, "
|
||||
"or a source DEM with .xml/.hdr/.vrt sidecars."
|
||||
)
|
||||
|
||||
if args.force:
|
||||
safe_rmtree(run_root)
|
||||
safe_rmtree(template_root)
|
||||
safe_rmtree(output_dir)
|
||||
|
||||
if run_root.exists():
|
||||
raise RuntimeError(f"PyINT run root already exists, rerun with --force: {run_root}")
|
||||
|
||||
pair_meta = load_pair_meta(task_dir)
|
||||
master_archives = discover_lt1_archives(task_dir / "master")
|
||||
slave_archives = discover_lt1_archives(task_dir / "slave")
|
||||
if not master_archives:
|
||||
raise FileNotFoundError(f"No LT1 archives found under: {task_dir / 'master'}")
|
||||
if not slave_archives:
|
||||
raise FileNotFoundError(f"No LT1 archives found under: {task_dir / 'slave'}")
|
||||
|
||||
master_date = normalize_date_text(args.master_date) or normalize_date_text(pair_meta.get("master_imaging_date")) or infer_scene_date(master_archives)
|
||||
slave_date = normalize_date_text(args.slave_date) or normalize_date_text(pair_meta.get("slave_imaging_date")) or infer_scene_date(slave_archives)
|
||||
if not master_date or not slave_date:
|
||||
raise RuntimeError("Unable to determine master/slave dates from pair metadata or archive names.")
|
||||
|
||||
pair_name = f"{master_date}-{slave_date}"
|
||||
task_alias = str(args.task_alias or pair_meta.get("task_alias") or task_dir.name).strip() or task_dir.name
|
||||
pair_key = str(args.pair_key or pair_meta.get("pair_key") or "").strip()
|
||||
time_baseline_days = int(args.time_baseline_days or pair_meta.get("time_baseline_days") or 0)
|
||||
|
||||
ensure_directory(run_root)
|
||||
ensure_directory(template_root)
|
||||
ensure_directory(output_dir)
|
||||
ensure_directory(dem_root)
|
||||
|
||||
pyint_scripts_dir = pyint_home / "pyint"
|
||||
wrappers_dir = ensure_directory(run_root / "wrappers")
|
||||
write_wrapper_scripts(
|
||||
wrappers_dir=wrappers_dir,
|
||||
pyint_home=pyint_home,
|
||||
python_cmd=args.python,
|
||||
gamma_env_script=args.gamma_env_script,
|
||||
)
|
||||
|
||||
template_path = write_text(
|
||||
template_root / f"{args.project_name}.template",
|
||||
build_template_text(
|
||||
project_name=args.project_name,
|
||||
master_date=master_date,
|
||||
range_looks=args.range_looks,
|
||||
azimuth_looks=args.azimuth_looks,
|
||||
parallel_workers=args.parallel_workers,
|
||||
unwrap=bool(args.unwrap),
|
||||
geocode=bool(args.geocode),
|
||||
dem_mode=dem_mode,
|
||||
fabdem_root=str(args.fabdem_root or "").strip(),
|
||||
prepared_dem_path=str(args.prepared_dem_path or "").strip(),
|
||||
opentopo_dem_type=str(args.opentopo_dem_type or "SRTMGL1").strip(),
|
||||
opentopo_api_key=str(args.opentopo_api_key or "").strip(),
|
||||
),
|
||||
)
|
||||
|
||||
scratch_root = ensure_directory(project_dir.parent)
|
||||
archive_materialization: List[Dict[str, str]] = []
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"SCRATCHDIR": str(scratch_root),
|
||||
"TEMPLATEDIR": str(template_root),
|
||||
"DEMDIR": str(dem_root),
|
||||
"PATH": f"{wrappers_dir}:{pyint_scripts_dir}:{env.get('PATH', '')}",
|
||||
"PYTHONPATH": f"{pyint_home}:{env.get('PYTHONPATH', '')}",
|
||||
"PYINT_LT1_PRECISE_ORBIT_ENABLED": "true" if precise_orbit_enabled else "false",
|
||||
"PYINT_LT1_PRECISE_ORBIT_MODE": precise_orbit_mode,
|
||||
"PYINT_LT1_PRECISE_ORBIT_STRICT": "true" if precise_orbit_strict else "false",
|
||||
"PYINT_LT1_PRECISE_ORBIT_VALIDATE_WITH_ORB_FILT": "true" if precise_orbit_validate_with_orb_filt else "false",
|
||||
"PYINT_LT1_PRECISE_ORBIT_BACKUP": "true" if precise_orbit_backup else "false",
|
||||
"PYINT_LT1_PRECISE_ORBIT_ORB_FILT_DEGREE": str(int(args.lt1_precise_orbit_orb_filt_degree)),
|
||||
"PYINT_LT1_PRECISE_ORBIT_HELPER": str(precise_orbit_helper),
|
||||
"PYINT_LT1_PRECISE_ORBIT_MANIFEST": str(input_assets_json) if input_assets_json else "",
|
||||
}
|
||||
)
|
||||
|
||||
generate_stdout = run_root / "pyint_generate.stdout.log"
|
||||
generate_stderr = run_root / "pyint_generate.stderr.log"
|
||||
generate_result = run_logged(
|
||||
[str(wrappers_dir / "pyintApp.py"), "-g", args.project_name],
|
||||
env=env,
|
||||
cwd=scratch_root,
|
||||
stdout_path=generate_stdout,
|
||||
stderr_path=generate_stderr,
|
||||
)
|
||||
if generate_result.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"pyintApp.py -g failed with rc={generate_result.returncode}: "
|
||||
f"{(generate_result.stderr or generate_result.stdout or '').strip()}"
|
||||
)
|
||||
|
||||
pyint_project_dir = project_dir
|
||||
download_dir = ensure_directory(pyint_project_dir / "DOWNLOAD")
|
||||
ifgram_list_path = write_ifgram_list(pyint_project_dir / "ifgram_list.txt", master_date, slave_date, time_baseline_days)
|
||||
for role, archives in (("master", master_archives), ("slave", slave_archives)):
|
||||
for src_path in archives:
|
||||
related_files = collect_related_lt1_input_files(src_path)
|
||||
for related_path in related_files:
|
||||
target_path = download_dir / related_path.name
|
||||
op = hardlink_or_copy(related_path, target_path)
|
||||
archive_materialization.append(
|
||||
{
|
||||
"role": role,
|
||||
"source": str(related_path),
|
||||
"group_source": str(src_path),
|
||||
"target": str(target_path),
|
||||
"operation": op,
|
||||
}
|
||||
)
|
||||
|
||||
run_stdout = run_root / "pyint.stdout.log"
|
||||
run_stderr = run_root / "pyint.stderr.log"
|
||||
run_started_at = datetime.utcnow().isoformat(timespec="seconds") + "Z"
|
||||
run_result = run_logged(
|
||||
[str(wrappers_dir / "pyintApp.py"), args.project_name],
|
||||
env=env,
|
||||
cwd=scratch_root,
|
||||
stdout_path=run_stdout,
|
||||
stderr_path=run_stderr,
|
||||
)
|
||||
if run_result.returncode != 0:
|
||||
stage_error_logs = collect_stage_error_logs(pyint_project_dir)
|
||||
detail_text = (run_result.stderr or run_result.stdout or "").strip()
|
||||
if stage_error_logs:
|
||||
log_text = ", ".join(f"{name}={path}" for name, path in stage_error_logs.items())
|
||||
detail_text = f"{detail_text}\nStage logs: {log_text}" if detail_text else f"Stage logs: {log_text}"
|
||||
raise RuntimeError(
|
||||
f"pyintApp.py failed with rc={run_result.returncode}: "
|
||||
f"{detail_text}"
|
||||
)
|
||||
|
||||
orbit_bridge_summaries = collect_orbit_bridge_summaries(pyint_project_dir)
|
||||
assert_orbit_bridge_ok(
|
||||
enabled=precise_orbit_enabled,
|
||||
strict=precise_orbit_strict,
|
||||
summaries=orbit_bridge_summaries,
|
||||
expected_dates=(master_date, slave_date),
|
||||
)
|
||||
|
||||
outputs = collect_expected_outputs(pyint_project_dir, pair_name, args.range_looks)
|
||||
assert_required_outputs(outputs, unwrap=bool(args.unwrap), geocode=bool(args.geocode))
|
||||
output_sanity_checks = collect_output_sanity_checks(
|
||||
outputs,
|
||||
unwrap=bool(args.unwrap),
|
||||
geocode=bool(args.geocode),
|
||||
)
|
||||
assert_output_sanity(output_sanity_checks)
|
||||
stage_error_logs = collect_stage_error_logs(pyint_project_dir)
|
||||
|
||||
copied_paths = copy_native_outputs(
|
||||
project_dir=pyint_project_dir,
|
||||
output_dir=output_dir,
|
||||
pair_name=pair_name,
|
||||
template_path=template_path,
|
||||
ifgram_list_path=ifgram_list_path,
|
||||
stdout_path=run_stdout,
|
||||
stderr_path=run_stderr,
|
||||
)
|
||||
copied_orbit_bridge_paths = copy_orbit_bridge_summaries(orbit_bridge_summaries, output_dir)
|
||||
|
||||
summary = {
|
||||
"ok": True,
|
||||
"task_dir": str(task_dir),
|
||||
"task_alias": task_alias,
|
||||
"pair_key": pair_key,
|
||||
"project_name": args.project_name,
|
||||
"project_dir": str(pyint_project_dir),
|
||||
"run_root": str(run_root),
|
||||
"template_root": str(template_root),
|
||||
"output_dir": str(output_dir),
|
||||
"pyint_home": str(pyint_home),
|
||||
"pyint_app_script": str(pyint_app_script),
|
||||
"gamma_env_script": args.gamma_env_script,
|
||||
"dem": {
|
||||
"mode": dem_mode,
|
||||
"dem_root": str(dem_root),
|
||||
"fabdem_root": str(args.fabdem_root or "").strip(),
|
||||
"prepared_dem_path": str(args.prepared_dem_path or "").strip(),
|
||||
"prepared_dem_kind": str(prepared_dem_info.get("kind") or ""),
|
||||
"prepared_dem_direct_path": str(prepared_dem_info.get("direct_dem_path") or ""),
|
||||
"prepared_dem_source_path": str(prepared_dem_info.get("source_dem_path") or ""),
|
||||
"prepared_dem_open_path": str(prepared_dem_info.get("source_dem_open_path") or ""),
|
||||
"opentopo_dem_type": str(args.opentopo_dem_type or "SRTMGL1").strip(),
|
||||
"opentopo_api_key_configured": bool(str(args.opentopo_api_key or "").strip()),
|
||||
},
|
||||
"orbit_policy": str(args.orbit_policy or "require_txt").strip().lower(),
|
||||
"precise_orbit_bridge": {
|
||||
"enabled": precise_orbit_enabled,
|
||||
"mode": precise_orbit_mode,
|
||||
"strict": precise_orbit_strict,
|
||||
"validate_with_orb_filt": precise_orbit_validate_with_orb_filt,
|
||||
"backup": precise_orbit_backup,
|
||||
"orb_filt_degree": int(args.lt1_precise_orbit_orb_filt_degree),
|
||||
"helper_path": str(precise_orbit_helper),
|
||||
"manifest_json": str(input_assets_json) if input_assets_json else "",
|
||||
"summaries": [
|
||||
{
|
||||
"path": item["path"],
|
||||
"date_dir": item["date_dir"],
|
||||
"ok": item["ok"],
|
||||
"operation_count": item["operation_count"],
|
||||
"failed_operation_count": item["failed_operation_count"],
|
||||
"copied_summary_path": copied_orbit_bridge_paths.get(item["date_dir"], ""),
|
||||
}
|
||||
for item in orbit_bridge_summaries
|
||||
],
|
||||
},
|
||||
"input_assets_dir": str(input_assets_dir) if input_assets_dir else "",
|
||||
"input_assets_json": str(input_assets_json) if input_assets_json else "",
|
||||
"input_assets": input_assets_payload,
|
||||
"master_date": master_date,
|
||||
"slave_date": slave_date,
|
||||
"pair_name": pair_name,
|
||||
"time_baseline_days": time_baseline_days,
|
||||
"range_looks": int(args.range_looks),
|
||||
"azimuth_looks": int(args.azimuth_looks),
|
||||
"parallel_workers": int(args.parallel_workers),
|
||||
"unwrap": bool(args.unwrap),
|
||||
"geocode": bool(args.geocode),
|
||||
"archives": {
|
||||
"master": [str(path) for path in master_archives],
|
||||
"slave": [str(path) for path in slave_archives],
|
||||
},
|
||||
"archive_materialization": archive_materialization,
|
||||
"workspace_outputs": outputs,
|
||||
"output_sanity_checks": output_sanity_checks,
|
||||
"copied_outputs": copied_paths,
|
||||
"copied_orbit_bridge_paths": copied_orbit_bridge_paths,
|
||||
"logs": {
|
||||
"generate_stdout": str(generate_stdout),
|
||||
"generate_stderr": str(generate_stderr),
|
||||
"run_stdout": str(run_stdout),
|
||||
"run_stderr": str(run_stderr),
|
||||
"stage_error_logs": stage_error_logs,
|
||||
},
|
||||
"started_at": run_started_at,
|
||||
"finished_at": datetime.utcnow().isoformat(timespec="seconds") + "Z",
|
||||
}
|
||||
|
||||
summary_path = output_dir / "pyint_run_summary.json"
|
||||
write_text(summary_path, json.dumps(summary, ensure_ascii=False, indent=2) + "\n")
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except Exception as exc:
|
||||
print(str(exc), file=sys.stderr)
|
||||
raise
|
||||
Reference in New Issue
Block a user