diff --git a/.codex_tmp/audit_lt1_dem_geometry_chain.py b/.codex_tmp/audit_lt1_dem_geometry_chain.py deleted file mode 100644 index c12fe9a..0000000 --- a/.codex_tmp/audit_lt1_dem_geometry_chain.py +++ /dev/null @@ -1,403 +0,0 @@ -from __future__ import annotations - -import json -import math -import sys -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -import numpy as np - - -MASTER_DATE = "20230726" -SLAVE_DATES = ("20230624", "20230920") -PATCH_SIZE = 512 -GAMMA_FLOAT32 = np.dtype(">f4") - - -@dataclass -class GammaImageShape: - width: int - lines: int - - -def parse_gamma_par_value(path: Path, key: str) -> str: - prefix = key.strip() + ":" - for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): - stripped = line.strip() - if stripped.startswith(prefix): - _, _, tail = stripped.partition(":") - return tail.strip().split()[0] - raise ValueError(f"Missing key '{key}' in {path}") - - -def parse_gamma_shape(path: Path) -> GammaImageShape: - width = int(float(parse_gamma_par_value(path, "range_samples"))) - lines = int(float(parse_gamma_par_value(path, "azimuth_lines"))) - return GammaImageShape(width=width, lines=lines) - - -def read_float32_image(path: Path, shape: GammaImageShape) -> np.ndarray: - arr = np.fromfile(path, dtype=GAMMA_FLOAT32) - expected = shape.width * shape.lines - if arr.size != expected: - raise ValueError(f"Unexpected size for {path}: expected {expected}, got {arr.size}") - return arr.reshape(shape.lines, shape.width) - - -def read_lt0_lookup(path: Path, shape: GammaImageShape) -> np.ndarray: - arr = np.fromfile(path, dtype=GAMMA_FLOAT32) - expected = shape.width * shape.lines * 2 - if arr.size != expected: - raise ValueError(f"Unexpected lt0 size for {path}: expected {expected}, got {arr.size}") - return arr.reshape(shape.lines, shape.width, 2) - - -def center_slice(size: int, patch: int) -> slice: - patch = min(size, patch) - start = max(0, (size - patch) // 2) - stop = start + patch - return slice(start, stop) - - -def safe_float(value: Any) -> float | None: - if value is None: - return None - try: - result = float(value) - except Exception: - return None - if math.isnan(result) or math.isinf(result): - return None - return result - - -def summarize_array(arr: np.ndarray, *, patch_size: int = PATCH_SIZE) -> dict[str, Any]: - finite = np.isfinite(arr) - zeros = finite & (arr == 0) - nonzero = finite & (arr != 0) - - ys = center_slice(arr.shape[0], patch_size) - xs = center_slice(arr.shape[1], patch_size) - patch = arr[ys, xs] - patch_finite = np.isfinite(patch) - patch_zeros = patch_finite & (patch == 0) - patch_nonzero = patch_finite & (patch != 0) - - nz = arr[nonzero] - patch_nz = patch[patch_nonzero] - stats: dict[str, Any] = { - "shape": [int(arr.shape[0]), int(arr.shape[1])], - "count": int(arr.size), - "finite_count": int(finite.sum()), - "zero_count": int(zeros.sum()), - "zero_ratio": safe_float(zeros.sum() / arr.size if arr.size else None), - "nonzero_count": int(nonzero.sum()), - "center_patch_shape": [int(patch.shape[0]), int(patch.shape[1])], - "center_patch_count": int(patch.size), - "center_patch_zero_count": int(patch_zeros.sum()), - "center_patch_zero_ratio": safe_float(patch_zeros.sum() / patch.size if patch.size else None), - "center_patch_nonzero_count": int(patch_nonzero.sum()), - } - if nz.size: - stats.update( - { - "min_nonzero": safe_float(nz.min()), - "max_nonzero": safe_float(nz.max()), - "mean_nonzero": safe_float(nz.mean()), - "std_nonzero": safe_float(nz.std()), - } - ) - if patch_nz.size: - stats.update( - { - "center_patch_min_nonzero": safe_float(patch_nz.min()), - "center_patch_max_nonzero": safe_float(patch_nz.max()), - "center_patch_mean_nonzero": safe_float(patch_nz.mean()), - "center_patch_std_nonzero": safe_float(patch_nz.std()), - } - ) - return stats - - -def summarize_lt0(arr: np.ndarray, *, patch_size: int = PATCH_SIZE) -> dict[str, Any]: - rng = arr[:, :, 0] - az = arr[:, :, 1] - finite = np.isfinite(rng) & np.isfinite(az) - zero_pair = finite & (rng == 0) & (az == 0) - valid_pair = finite & (~zero_pair) - magnitude = np.sqrt(np.square(rng, dtype=np.float64) + np.square(az, dtype=np.float64)) - - ys = center_slice(arr.shape[0], patch_size) - xs = center_slice(arr.shape[1], patch_size) - patch_valid = valid_pair[ys, xs] - patch_zero = zero_pair[ys, xs] - patch_mag = magnitude[ys, xs][patch_valid] - all_mag = magnitude[valid_pair] - - stats: dict[str, Any] = { - "shape": [int(arr.shape[0]), int(arr.shape[1]), 2], - "count": int(arr.shape[0] * arr.shape[1]), - "valid_pair_count": int(valid_pair.sum()), - "valid_pair_ratio": safe_float(valid_pair.sum() / valid_pair.size if valid_pair.size else None), - "zero_pair_count": int(zero_pair.sum()), - "zero_pair_ratio": safe_float(zero_pair.sum() / zero_pair.size if zero_pair.size else None), - "center_patch_shape": [int(patch_valid.shape[0]), int(patch_valid.shape[1])], - "center_patch_valid_pair_count": int(patch_valid.sum()), - "center_patch_valid_pair_ratio": safe_float(patch_valid.sum() / patch_valid.size if patch_valid.size else None), - "center_patch_zero_pair_count": int(patch_zero.sum()), - "center_patch_zero_pair_ratio": safe_float(patch_zero.sum() / patch_zero.size if patch_zero.size else None), - } - if all_mag.size: - stats.update( - { - "magnitude_min": safe_float(all_mag.min()), - "magnitude_max": safe_float(all_mag.max()), - "magnitude_mean": safe_float(all_mag.mean()), - "magnitude_std": safe_float(all_mag.std()), - } - ) - if patch_mag.size: - stats.update( - { - "center_patch_magnitude_min": safe_float(patch_mag.min()), - "center_patch_magnitude_max": safe_float(patch_mag.max()), - "center_patch_magnitude_mean": safe_float(patch_mag.mean()), - "center_patch_magnitude_std": safe_float(patch_mag.std()), - } - ) - return stats - - -def summarize_overlap(a: np.ndarray, b: np.ndarray, *, patch_size: int = PATCH_SIZE) -> dict[str, Any]: - if a.shape != b.shape: - raise ValueError(f"Shape mismatch for overlap: {a.shape} vs {b.shape}") - finite_a = np.isfinite(a) - finite_b = np.isfinite(b) - nz_a = finite_a & (a != 0) - nz_b = finite_b & (b != 0) - overlap = nz_a & nz_b - - ys = center_slice(a.shape[0], patch_size) - xs = center_slice(a.shape[1], patch_size) - patch_overlap = overlap[ys, xs] - patch_nz_a = nz_a[ys, xs] - patch_nz_b = nz_b[ys, xs] - - return { - "shape": [int(a.shape[0]), int(a.shape[1])], - "overlap_nonzero_count": int(overlap.sum()), - "overlap_nonzero_ratio": safe_float(overlap.sum() / overlap.size if overlap.size else None), - "a_nonzero_count": int(nz_a.sum()), - "b_nonzero_count": int(nz_b.sum()), - "center_patch_overlap_nonzero_count": int(patch_overlap.sum()), - "center_patch_overlap_nonzero_ratio": safe_float(patch_overlap.sum() / patch_overlap.size if patch_overlap.size else None), - "center_patch_a_nonzero_count": int(patch_nz_a.sum()), - "center_patch_b_nonzero_count": int(patch_nz_b.sum()), - } - - -def scale_to_u8(arr: np.ndarray) -> np.ndarray: - finite = np.isfinite(arr) - valid = arr[finite & (arr != 0)] - if valid.size == 0: - return np.zeros(arr.shape, dtype=np.uint8) - lo = np.percentile(valid, 1) - hi = np.percentile(valid, 99) - if not np.isfinite(lo) or not np.isfinite(hi) or hi <= lo: - lo = float(valid.min()) - hi = float(valid.max()) if valid.size else lo + 1.0 - if hi <= lo: - hi = lo + 1.0 - scaled = np.clip((arr - lo) / (hi - lo), 0, 1) - scaled[~finite] = 0 - scaled[arr == 0] = 0 - return np.round(scaled * 255.0).astype(np.uint8) - - -def write_pgm(path: Path, arr_u8: np.ndarray) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - header = f"P5\n{arr_u8.shape[1]} {arr_u8.shape[0]}\n255\n".encode("ascii") - with path.open("wb") as fp: - fp.write(header) - fp.write(arr_u8.tobytes()) - - -def write_quicklook(path: Path, arr: np.ndarray) -> None: - write_pgm(path, scale_to_u8(arr)) - - -def case_root(run_root: Path, case_name: str) -> Path: - return run_root / case_name / "pyint_stage" - - -def audit_case(run_root: Path, case_name: str, out_root: Path) -> dict[str, Any]: - case_dir = case_root(run_root, case_name) - dem_dir = case_dir / "DEM" - out_case_root = out_root / case_name - out_case_root.mkdir(parents=True, exist_ok=True) - - master_shape = parse_gamma_shape(dem_dir / f"{MASTER_DATE}_2rlks.amp.par") - hgtsim = read_float32_image(dem_dir / f"{MASTER_DATE}_2rlks.rdc.dem", master_shape) - lt0_quicklook_written = False - - result: dict[str, Any] = { - "case": case_name, - "master_date": MASTER_DATE, - "master_shape": {"width": master_shape.width, "lines": master_shape.lines}, - "dem": { - "hgtsim": summarize_array(hgtsim), - }, - "slaves": {}, - } - - write_quicklook(out_case_root / "hgtsim.pgm", hgtsim) - - for slave_date in SLAVE_DATES: - slc_dir = case_dir / "SLC" / slave_date - rslc_dir = case_dir / "RSLC" / slave_date - slave_shape = parse_gamma_shape(slc_dir / f"{slave_date}_2rlks.amp.par") - samp = read_float32_image(slc_dir / f"{slave_date}_2rlks.amp", slave_shape) - mli0 = read_float32_image(rslc_dir / "mli0", slave_shape) - lt0 = read_lt0_lookup(rslc_dir / "lt0", master_shape) - - lt0_mag = np.sqrt(np.square(lt0[:, :, 0], dtype=np.float64) + np.square(lt0[:, :, 1], dtype=np.float64)) - - slave_out = out_case_root / slave_date - slave_out.mkdir(parents=True, exist_ok=True) - write_quicklook(slave_out / "samp.pgm", samp) - write_quicklook(slave_out / "mli0.pgm", mli0) - if not lt0_quicklook_written: - write_quicklook(out_case_root / "lt0_magnitude.pgm", lt0_mag.astype(np.float32)) - lt0_quicklook_written = True - - slave_summary = { - "shape": {"width": slave_shape.width, "lines": slave_shape.lines}, - "samp": summarize_array(samp), - "mli0": summarize_array(mli0), - "lt0": summarize_lt0(lt0), - "mli0_samp_overlap": summarize_overlap(mli0, samp), - } - result["slaves"][slave_date] = slave_summary - - (slave_out / "summary.json").write_text( - json.dumps(slave_summary, ensure_ascii=False, indent=2) + "\n", - encoding="utf-8", - ) - - (out_case_root / "summary.json").write_text( - json.dumps(result, ensure_ascii=False, indent=2) + "\n", - encoding="utf-8", - ) - return result - - -def build_summary_rows(result: dict[str, Any]) -> list[dict[str, Any]]: - rows: list[dict[str, Any]] = [] - rows.append( - { - "case": result["case"], - "date": result["master_date"], - "artifact": "hgtsim", - "zero_ratio": result["dem"]["hgtsim"].get("zero_ratio"), - "center_patch_zero_ratio": result["dem"]["hgtsim"].get("center_patch_zero_ratio"), - "overlap_ratio": None, - "center_patch_overlap_ratio": None, - } - ) - for slave_date, payload in result["slaves"].items(): - for artifact in ("samp", "mli0"): - stats = payload[artifact] - rows.append( - { - "case": result["case"], - "date": slave_date, - "artifact": artifact, - "zero_ratio": stats.get("zero_ratio"), - "center_patch_zero_ratio": stats.get("center_patch_zero_ratio"), - "overlap_ratio": None, - "center_patch_overlap_ratio": None, - } - ) - lt0 = payload["lt0"] - rows.append( - { - "case": result["case"], - "date": slave_date, - "artifact": "lt0", - "zero_ratio": lt0.get("zero_pair_ratio"), - "center_patch_zero_ratio": lt0.get("center_patch_zero_pair_ratio"), - "overlap_ratio": lt0.get("valid_pair_ratio"), - "center_patch_overlap_ratio": lt0.get("center_patch_valid_pair_ratio"), - } - ) - overlap = payload["mli0_samp_overlap"] - rows.append( - { - "case": result["case"], - "date": slave_date, - "artifact": "mli0_samp_overlap", - "zero_ratio": None, - "center_patch_zero_ratio": None, - "overlap_ratio": overlap.get("overlap_nonzero_ratio"), - "center_patch_overlap_ratio": overlap.get("center_patch_overlap_nonzero_ratio"), - } - ) - return rows - - -def write_summary_tsv(path: Path, rows: list[dict[str, Any]]) -> None: - header = [ - "case", - "date", - "artifact", - "zero_ratio", - "center_patch_zero_ratio", - "overlap_ratio", - "center_patch_overlap_ratio", - ] - lines = ["\t".join(header)] - for row in rows: - values = [] - for key in header: - value = row.get(key) - if isinstance(value, float): - values.append(f"{value:.6f}") - elif value is None: - values.append("") - else: - values.append(str(value)) - lines.append("\t".join(values)) - path.write_text("\n".join(lines) + "\n", encoding="utf-8") - - -def main() -> int: - if len(sys.argv) < 2: - print("usage: audit_lt1_dem_geometry_chain.py [case ...]", file=sys.stderr) - return 2 - - run_root = Path(sys.argv[1]).resolve() - case_names = tuple(sys.argv[2:]) if len(sys.argv) > 2 else ("case_A_baseline", "case_C_precise_orbit_rewrite") - out_root = run_root / "audit_dem_geometry" - out_root.mkdir(parents=True, exist_ok=True) - - all_results = [] - all_rows: list[dict[str, Any]] = [] - for case_name in case_names: - result = audit_case(run_root, case_name, out_root) - all_results.append(result) - all_rows.extend(build_summary_rows(result)) - - (out_root / "audit_summary.json").write_text( - json.dumps({"run_root": str(run_root), "results": all_results}, ensure_ascii=False, indent=2) + "\n", - encoding="utf-8", - ) - write_summary_tsv(out_root / "audit_summary.tsv", all_rows) - - print(json.dumps({"run_root": str(run_root), "output_dir": str(out_root), "case_count": len(case_names)}, ensure_ascii=False)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/.codex_tmp/audit_lt1_pool_multiscene_root.py b/.codex_tmp/audit_lt1_pool_multiscene_root.py deleted file mode 100644 index 741e637..0000000 --- a/.codex_tmp/audit_lt1_pool_multiscene_root.py +++ /dev/null @@ -1,204 +0,0 @@ -from __future__ import annotations - -import importlib.util -import json -import sys -from pathlib import Path -from typing import Any - - -def load_base_module(script_path: Path): - spec = importlib.util.spec_from_file_location("audit_lt1_dem_geometry_chain_base", script_path) - if spec is None or spec.loader is None: - raise RuntimeError(f"Unable to load base audit script: {script_path}") - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -def build_summary_rows(result: dict[str, Any]) -> list[dict[str, Any]]: - rows: list[dict[str, Any]] = [] - rows.append( - { - "root": result["root"], - "date": result["master_date"], - "artifact": "hgtsim", - "zero_ratio": result["dem"]["hgtsim"].get("zero_ratio"), - "center_patch_zero_ratio": result["dem"]["hgtsim"].get("center_patch_zero_ratio"), - "overlap_ratio": None, - "center_patch_overlap_ratio": None, - } - ) - for slave_date, payload in result["slaves"].items(): - for artifact in ("samp", "mli0"): - stats = payload[artifact] - rows.append( - { - "root": result["root"], - "date": slave_date, - "artifact": artifact, - "zero_ratio": stats.get("zero_ratio"), - "center_patch_zero_ratio": stats.get("center_patch_zero_ratio"), - "overlap_ratio": None, - "center_patch_overlap_ratio": None, - } - ) - lt0 = payload["lt0"] - rows.append( - { - "root": result["root"], - "date": slave_date, - "artifact": "lt0", - "zero_ratio": lt0.get("zero_pair_ratio"), - "center_patch_zero_ratio": lt0.get("center_patch_zero_pair_ratio"), - "overlap_ratio": lt0.get("valid_pair_ratio"), - "center_patch_overlap_ratio": lt0.get("center_patch_valid_pair_ratio"), - } - ) - overlap = payload["mli0_samp_overlap"] - rows.append( - { - "root": result["root"], - "date": slave_date, - "artifact": "mli0_samp_overlap", - "zero_ratio": None, - "center_patch_zero_ratio": None, - "overlap_ratio": overlap.get("overlap_nonzero_ratio"), - "center_patch_overlap_ratio": overlap.get("center_patch_overlap_nonzero_ratio"), - } - ) - return rows - - -def write_summary_tsv(path: Path, rows: list[dict[str, Any]]) -> None: - header = [ - "root", - "date", - "artifact", - "zero_ratio", - "center_patch_zero_ratio", - "overlap_ratio", - "center_patch_overlap_ratio", - ] - lines = ["\t".join(header)] - for row in rows: - values = [] - for key in header: - value = row.get(key) - if isinstance(value, float): - values.append(f"{value:.6f}") - elif value is None: - values.append("") - else: - values.append(str(value)) - lines.append("\t".join(values)) - path.write_text("\n".join(lines) + "\n", encoding="utf-8") - - -def audit_root(run_root: Path, master_date: str, slave_dates: tuple[str, ...], project: str) -> dict[str, Any]: - base_script = Path(__file__).with_name("audit_lt1_dem_geometry_chain.py") - base = load_base_module(base_script) - - project_root = run_root / project - dem_dir = project_root / "DEM" - out_root = run_root / "audit_dem_geometry" - out_root.mkdir(parents=True, exist_ok=True) - - master_shape = base.parse_gamma_shape(dem_dir / f"{master_date}_2rlks.amp.par") - hgtsim = base.read_float32_image(dem_dir / f"{master_date}_2rlks.rdc.dem", master_shape) - - result: dict[str, Any] = { - "root": str(run_root), - "project": project, - "master_date": master_date, - "master_shape": {"width": master_shape.width, "lines": master_shape.lines}, - "dem": { - "hgtsim": base.summarize_array(hgtsim), - }, - "slaves": {}, - } - - base.write_quicklook(out_root / "hgtsim.pgm", hgtsim) - lt0_quicklook_written = False - - for slave_date in slave_dates: - slc_dir = project_root / "SLC" / slave_date - rslc_dir = project_root / "RSLC" / slave_date - slave_shape = base.parse_gamma_shape(slc_dir / f"{slave_date}_2rlks.amp.par") - samp = base.read_float32_image(slc_dir / f"{slave_date}_2rlks.amp", slave_shape) - mli0 = base.read_float32_image(rslc_dir / "mli0", slave_shape) - lt0 = base.read_lt0_lookup(rslc_dir / "lt0", master_shape) - - lt0_mag = (lt0[:, :, 0].astype("float64") ** 2 + lt0[:, :, 1].astype("float64") ** 2) ** 0.5 - - slave_out = out_root / slave_date - slave_out.mkdir(parents=True, exist_ok=True) - base.write_quicklook(slave_out / "samp.pgm", samp) - base.write_quicklook(slave_out / "mli0.pgm", mli0) - if not lt0_quicklook_written: - base.write_quicklook(out_root / "lt0_magnitude.pgm", lt0_mag.astype("float32")) - lt0_quicklook_written = True - - slave_summary = { - "shape": {"width": slave_shape.width, "lines": slave_shape.lines}, - "samp": base.summarize_array(samp), - "mli0": base.summarize_array(mli0), - "lt0": base.summarize_lt0(lt0), - "mli0_samp_overlap": base.summarize_overlap(mli0, samp), - } - result["slaves"][slave_date] = slave_summary - (slave_out / "summary.json").write_text( - json.dumps(slave_summary, ensure_ascii=False, indent=2) + "\n", - encoding="utf-8", - ) - - (out_root / "summary.json").write_text( - json.dumps(result, ensure_ascii=False, indent=2) + "\n", - encoding="utf-8", - ) - rows = build_summary_rows(result) - write_summary_tsv(out_root / "audit_summary.tsv", rows) - return result - - -def main() -> int: - if len(sys.argv) < 4: - print( - "usage: audit_lt1_pool_multiscene_root.py [slave_date ...] [--project name]", - file=sys.stderr, - ) - return 2 - - args = list(sys.argv[1:]) - project = "pyint_stage" - if "--project" in args: - idx = args.index("--project") - try: - project = args[idx + 1] - except IndexError as exc: - raise SystemExit("--project requires a value") from exc - del args[idx : idx + 2] - - run_root = Path(args[0]).resolve() - master_date = args[1] - slave_dates = tuple(args[2:]) - - result = audit_root(run_root, master_date, slave_dates, project) - print( - json.dumps( - { - "run_root": str(run_root), - "output_dir": str(run_root / "audit_dem_geometry"), - "master_date": master_date, - "slave_count": len(slave_dates), - "project": project, - }, - ensure_ascii=False, - ) - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/.codex_tmp/compare_lt1_import_paths.sh b/.codex_tmp/compare_lt1_import_paths.sh deleted file mode 100644 index 00ece1f..0000000 --- a/.codex_tmp/compare_lt1_import_paths.sh +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -if [ $# -lt 1 ]; then - echo "usage: $0 YYYYMMDD" >&2 - exit 1 -fi - -DATE_TEXT="$1" -ROOT='/mnt/d/PyINT_POOL_TEST/LT1A_MONO_SYC_STRIP1_E129.6_N45.0_3scene' -PROJECT_DIR="$ROOT/pyint_stage" -DOWNLOAD_DIR="$PROJECT_DIR/DOWNLOAD" -CURRENT_SLC_DIR="$PROJECT_DIR/SLC/$DATE_TEXT" -COMPARE_ROOT="$ROOT/import_compare/$DATE_TEXT" -LIST_FILE="$COMPARE_ROOT/t_$DATE_TEXT" -CURRENT_PAR="$CURRENT_SLC_DIR/$DATE_TEXT.slc.par" -GAMMA_ENV='/mnt/d/Code/Insar_management_system_v2/backend/app/pyint_pipeline/pyint_gamma_env.sh' - -die() { - echo "$1" >&2 - exit 1 -} - -[ -f "$CURRENT_PAR" ] || die "Current .slc.par not found: $CURRENT_PAR" -[ -f "$GAMMA_ENV" ] || die "Gamma env script not found: $GAMMA_ENV" - -SCENE_TIFF="$(find "$DOWNLOAD_DIR" -maxdepth 1 -type f -name "LT1*${DATE_TEXT}*.tiff" | sort | head -n 1)" -[ -n "$SCENE_TIFF" ] || die "Scene TIFF not found for date: $DATE_TEXT" - -. "$GAMMA_ENV" >/dev/null 2>&1 -mkdir -p "$COMPARE_ROOT" -printf '%s\n' "$SCENE_TIFF" >"$LIST_FILE" - -pushd "$COMPARE_ROOT" >/dev/null -SCENE_XML="${SCENE_TIFF%.tiff}.meta.xml" -[ -f "$SCENE_XML" ] || die "Scene meta xml not found: $SCENE_XML" -par_LT1_SLC "$SCENE_TIFF" "$SCENE_XML" "./$DATE_TEXT.slc.par" "./$DATE_TEXT.slc" > import.stdout.log 2> import.stderr.log -par_LT1_SLC_YSLi "$SCENE_TIFF" "$SCENE_XML" "./$DATE_TEXT.slc.update" "./$DATE_TEXT.slc.update.par" 0 >> import.stdout.log 2>> import.stderr.log -popd >/dev/null - -ALT_PAR="$COMPARE_ROOT/$DATE_TEXT.slc.par" -[ -f "$ALT_PAR" ] || die "Alternate .slc.par not found: $ALT_PAR" - -python3 - "$ALT_PAR" "$COMPARE_ROOT/$DATE_TEXT.slc.update.par" <<'PY' -import sys -from pathlib import Path - -main_path = Path(sys.argv[1]) -update_path = Path(sys.argv[2]) - -main_lines = main_path.read_text(encoding="utf-8", errors="ignore").splitlines() -update_lines = update_path.read_text(encoding="utf-8", errors="ignore").splitlines() - -prefixes = ("number_of_state_vectors:", "time_of_first_state_vector:", "state_vector_interval:", "state_vector_position_", "state_vector_velocity_") -main_prefix = ("number_of_state_vectors:", "time_of_first_state_vector:", "state_vector_interval:", "state_vector_position_", "state_vector_velocity_") - -filtered_main = [line for line in main_lines if not line.startswith(main_prefix)] -replacement = [line for line in update_lines if line.startswith(prefixes)] - -main_path.write_text("\n".join(filtered_main + replacement) + "\n", encoding="utf-8") -PY - -python3 - "$CURRENT_PAR" "$ALT_PAR" <<'PY' -import re -import sys -from pathlib import Path - -keys = [ - "center_latitude", - "center_longitude", - "start_time", - "center_time", - "end_time", - "azimuth_line_time", - "range_samples", - "azimuth_lines", - "range_pixel_spacing", - "azimuth_pixel_spacing", - "near_range_slc", - "center_range_slc", - "far_range_slc", - "incidence_angle", - "heading", - "prf", - "azimuth_proc_bandwidth", - "doppler_polynomial", - "number_of_state_vectors", - "time_of_first_state_vector", - "state_vector_interval", -] - -vector_patterns = [ - "state_vector_position_1", - "state_vector_velocity_1", - "state_vector_position_2", - "state_vector_velocity_2", - "state_vector_position_3", - "state_vector_velocity_3", -] - -def read_map(path: Path): - data = {} - for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): - for key in keys + vector_patterns: - if line.startswith(key + ":"): - data[key] = line.split(":", 1)[1].strip() - break - return data - -cur = read_map(Path(sys.argv[1])) -alt = read_map(Path(sys.argv[2])) -all_keys = keys + vector_patterns -for key in all_keys: - cur_v = cur.get(key, "") - alt_v = alt.get(key, "") - if cur_v != alt_v: - print(f"{key}\n current: {cur_v}\n alt: {alt_v}") -PY diff --git a/.codex_tmp/orbit_smoke/source/LT1A_GpsData_GAS_C_20250622.txt b/.codex_tmp/orbit_smoke/source/LT1A_GpsData_GAS_C_20250622.txt deleted file mode 100644 index d2c1965..0000000 Binary files a/.codex_tmp/orbit_smoke/source/LT1A_GpsData_GAS_C_20250622.txt and /dev/null differ diff --git a/.codex_tmp/orbit_smoke/source/LT1B_GpsData_GAS_C_20250623.txt b/.codex_tmp/orbit_smoke/source/LT1B_GpsData_GAS_C_20250623.txt deleted file mode 100644 index a0d0e95..0000000 --- a/.codex_tmp/orbit_smoke/source/LT1B_GpsData_GAS_C_20250623.txt +++ /dev/null @@ -1 +0,0 @@ -2025 6 23 13 34 55.000 -1307508.7503 -651238.6465 -6835090.2734 2779.6665514 6982.8147436 -12.1250000 diff --git a/.codex_tmp/orbit_smoke_41817fae19fd4434b5e63edb31a01910/source/LT1A_GpsData_GAS_C_20250622.txt b/.codex_tmp/orbit_smoke_41817fae19fd4434b5e63edb31a01910/source/LT1A_GpsData_GAS_C_20250622.txt deleted file mode 100644 index d2c1965..0000000 Binary files a/.codex_tmp/orbit_smoke_41817fae19fd4434b5e63edb31a01910/source/LT1A_GpsData_GAS_C_20250622.txt and /dev/null differ diff --git a/.codex_tmp/orbit_smoke_41817fae19fd4434b5e63edb31a01910/source/LT1B_GpsData_GAS_C_20250623.txt b/.codex_tmp/orbit_smoke_41817fae19fd4434b5e63edb31a01910/source/LT1B_GpsData_GAS_C_20250623.txt deleted file mode 100644 index a0d0e95..0000000 --- a/.codex_tmp/orbit_smoke_41817fae19fd4434b5e63edb31a01910/source/LT1B_GpsData_GAS_C_20250623.txt +++ /dev/null @@ -1 +0,0 @@ -2025 6 23 13 34 55.000 -1307508.7503 -651238.6465 -6835090.2734 2779.6665514 6982.8147436 -12.1250000 diff --git a/.codex_tmp/orbit_smoke_58525e754384446ca1e016affd4cd340/source/LT1A_GpsData_GAS_C_20250622.txt b/.codex_tmp/orbit_smoke_58525e754384446ca1e016affd4cd340/source/LT1A_GpsData_GAS_C_20250622.txt deleted file mode 100644 index d2c1965..0000000 Binary files a/.codex_tmp/orbit_smoke_58525e754384446ca1e016affd4cd340/source/LT1A_GpsData_GAS_C_20250622.txt and /dev/null differ diff --git a/.codex_tmp/orbit_smoke_58525e754384446ca1e016affd4cd340/source/LT1B_GpsData_GAS_C_20250623.txt b/.codex_tmp/orbit_smoke_58525e754384446ca1e016affd4cd340/source/LT1B_GpsData_GAS_C_20250623.txt deleted file mode 100644 index a0d0e95..0000000 --- a/.codex_tmp/orbit_smoke_58525e754384446ca1e016affd4cd340/source/LT1B_GpsData_GAS_C_20250623.txt +++ /dev/null @@ -1 +0,0 @@ -2025 6 23 13 34 55.000 -1307508.7503 -651238.6465 -6835090.2734 2779.6665514 6982.8147436 -12.1250000 diff --git a/.codex_tmp/pyint_variants/no_rescue/LICENSE b/.codex_tmp/pyint_variants/no_rescue/LICENSE deleted file mode 100644 index f288702..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/LICENSE +++ /dev/null @@ -1,674 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. diff --git a/.codex_tmp/pyint_variants/no_rescue/README.md b/.codex_tmp/pyint_variants/no_rescue/README.md deleted file mode 100644 index 440ac43..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/README.md +++ /dev/null @@ -1,99 +0,0 @@ -# PyINT -[![Language](https://img.shields.io/badge/python-3.5%2B-blue.svg)](https://www.python.org/) -[![License](https://img.shields.io/badge/license-GPL-yellow.svg)](https://github.com/ymcmrs/PyINT/blob/master/LICENSE) - -### Single or time-series of interferograms processing based on python and GAMMA for all of the present SAR datasets. - -PYthon-based INterferometry Toolbox (PyINT) is an open-source package for single or time-series of interferograms processing from downloading data (or SLC) to generating differential-unwrapped interferograms by using GAMMA software. You can process in a routine way (e.g., raw2ifg.py, slc2ifg.py or pyintApp.py) or process step by step. There are many GAMMA-independent tools of PyINT could be useful for you no matter you use GAMMA or other interferometry softwares (e.g., ISCE, SNAP). Advantages include (but not limited to) download and update precise-orbit data automatically (support S1, ERS, ASAR), download and process 30m-SRTM dem automatically, cat multi-frames automatically, select swaths and bursts flexibly (for S1), extract the related S1 butsts for Coregistration automatically, etc. Welcome to contribute/improve PyINT. - - -### 1 Download - -Download the development version using git: - - cd ~/python - git clone https://github.com/ymcmrs/PyINT - - -### 2 Installation - - 1） To make pyint importable in python, by adding the path PyINT directory to your $PYTHONPATH - For csh/tcsh user, add to your **_~/.cshrc_** file for example: - - ############################ Python ############################### - if ( ! $?PYTHONPATH ) then - setenv PYTHONPATH "" - endif - - ##--------- Anaconda ---------------## - setenv PYTHON3DIR ~/python/anaconda3 - setenv PATH ${PATH}:${PYTHON3DIR}/bin - - ##--------- PyINT ------------------## - setenv PYINT_HOME ~/python/PyINT - setenv PYTHONPATH ${PYTHONPATH}:${PYINT_HOME} - setenv PATH ${PATH}:${PYINT_HOME}/pyint - - 2） Install gdal, elevation module using pip or conda for DEM processing. - - 3） Install [SSARA](https://github.com/bakerunavco/SSARA) and set account info for downloading data. [option] - - -### 3 Running PyINT - -1). $SCRATCHDIR and $TEMPLATEDIR should be available in your system environment. $SCRATCHDIR for processing, $TEMPLATEDIR for template files to set the related processing parameters, $DEMDIR for saving DEMs: - - setenv SCRATCHDIR /Users/Yunmeng/Documents/SCRATCH - setenv TEMPLATEDIR /Users/Yunmeng/Documents/development/TEMPLATEDIR - setenv DEMDIR /Users/Yunmeng/Documents/SCRATCH/DEM - -2). Preparing your template file, which should be saved in $TEMPLATEDIR, for setting some basic parameters (see the template file above).The template file should be named with a prefix of your project name: - - e.g., MexicoCityT143F529S1D.template [Region + Track + Frame + Satellite + Orbit] - - -3). Single interferogram processing: - - slc2ifg.py projectName Mdate Sdate # start from SLC to unwrapped-differential Ifg - raw2ifg.py projectName Mdate Sdate # start from raw data to unwrapped-differential Ifg - - e.g. : - slc2ifg.py HawaiiT87F526S1D 20150101 20160201 - raw2ifg.py HawaiiT87F526S1D 20150101 20160201 - -4). Time-series of interferograms processing. - - pyintApp.py projectName - - e.g. : - pyintApp.py MexicoCityT143F529S1D # template file MexicoCityT143F529S1D.template should be availabe in TEMPLATEDIR - - General work-flow: - - 1) download data : download SLCs using SSARA (please check https://github.com/bakerunavco/SSARA) - [You should provide Sensor, Track, Frame, or Time information in template] - 2) generate SLC : raw 2 slc (multi-frame processing is also supported) - [include orbit correction for S1,ASAR,ERS and burst-extraction for S1] - 3) generate DEM : reference image related geo-dem, rdc-dem, lookup table will be generated. - [SRTM-1 will be downloaded and processed automatically if not provided] - 4) coregister SLC : coregister SLCs to the reference SLC iamge. - [with assistant of DEM] - 5) select pairs : select interferometric pairs for time-series processing. - [networks of sbas, sequential, delaunay, and stars are supported] - 6) interferometry : generate unwrapped differential interferograms. - [include differential, unwrapping, and geocoding] - 7) load data : loading data for time-series analysis, mintPy is supported presently. - - Note: - - i ) Single interferogram processing please use slc2ifg.py or raw2ifg.py - ii ) Multi-processor parallel processing is supported, but keep in mind GAMMA calls multi-threads already. - iii) You can using pyintApp.py from downloading data to generate time-series of unwrapped-differential Ifgs, - or you also can process step by step. - - -Note: All of the above codes are based on the hypothesis that you have installed [GAMMA](https://www.gamma-rs.ch/). - -### 4 Citing this work - - Y.M., Cao, "PyINT: Python&GAMMA based interferometry toolbox", Remote Sensing Code Library, doi:10.21982/vd48-7p51, April, 2019. diff --git a/.codex_tmp/pyint_variants/no_rescue/VENDORED_FROM.md b/.codex_tmp/pyint_variants/no_rescue/VENDORED_FROM.md deleted file mode 100644 index a8c3232..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/VENDORED_FROM.md +++ /dev/null @@ -1,12 +0,0 @@ -# Vendored PyINT - -- Source copied from local path: `D:\Code\PyINT` -- Vendored into this repository on: `2026-04-19` -- Scope kept in repo: `pyint/`, `template/`, `README.md`, `LICENSE` -- Excluded from vendored copy: `pyint_bk/`, `pyint.zip`, IDE metadata, `__pycache__` - -Notes: - -- This vendored copy is used by the local D-InSAR `PyINT / Gamma` integration. -- Runtime defaults now resolve `PYINT_HOME` to this directory when `.env` does not override it. -- If upstream `PyINT` is updated, sync this directory intentionally and record the source revision or date here. diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/20210110.slc.par b/.codex_tmp/pyint_variants/no_rescue/pyint/20210110.slc.par deleted file mode 100644 index d571806..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/20210110.slc.par +++ /dev/null @@ -1,81 +0,0 @@ -Gamma Interferometric SAR Processor (ISP) - Image Parameter File - -title: s1a-iw1-slc-vv-20210110t110734-20210110t110759-036075-043a78-004.tiff S1A-IW-IW1-VV-36075 (software: Sentinel-1 IPF 003.31) -sensor: S1A IW IW1 VV -date: 2021 01 10 11 07 33.60437 -start_time: 40053.604372 s -center_time: 40067.080599 s -end_time: 40080.556826 s -azimuth_line_time: 2.05555630e-03 s -line_header_size: 0 -range_samples: 68467 -azimuth_lines: 13113 -range_looks: 1 -azimuth_looks: 1 -image_format: FCOMPLEX -image_geometry: SLANT_RANGE -polarization: VV -range_scale_factor: 1.0000000e+00 -azimuth_scale_factor: 1.0000000e+00 -center_latitude: 25.7907364 degrees -center_longitude: 104.4737803 degrees -heading: -12.5014428 degrees -range_pixel_spacing: 2.3295620 m -azimuth_pixel_spacing: 13.9946500 m -near_range_slc: 798980.1369 m -center_range_slc: 878728.0328 m -far_range_slc: 958475.9288 m -first_slant_range_polynomial: 0.00000 0.00000 0.00000e+00 0.00000e+00 0.00000e+00 0.00000e+00 s m 1 m^-1 m^-2 m^-3 -center_slant_range_polynomial: 0.00000 0.00000 0.00000e+00 0.00000e+00 0.00000e+00 0.00000e+00 s m 1 m^-1 m^-2 m^-3 -last_slant_range_polynomial: 0.00000 0.00000 0.00000e+00 0.00000e+00 0.00000e+00 0.00000e+00 s m 1 m^-1 m^-2 m^-3 -incidence_angle: 39.6640 degrees -azimuth_deskew: ON -azimuth_angle: 90.0000 degrees -radar_frequency: 5.4050005e+09 Hz -adc_sampling_rate: 6.4345241e+07 Hz -chirp_bandwidth: 5.6500000e+07 Hz -prf: 486.4863103 Hz -azimuth_proc_bandwidth: 327.00000 Hz -doppler_polynomial: 17.39054 -2.62817e-05 -8.00473e-11 0.00000e+00 Hz Hz/m Hz/m^2 Hz/m^3 -doppler_poly_dot: 0.00000e+00 0.00000e+00 0.00000e+00 0.00000e+00 Hz/s Hz/s/m Hz/s/m^2 Hz/s/m^3 -doppler_poly_ddot: 0.00000e+00 0.00000e+00 0.00000e+00 0.00000e+00 Hz/s^2 Hz/s^2/m Hz/s^2/m^2 Hz/s^2/m^3 -receiver_gain: 0.0000 dB -calibration_gain: 0.0000 dB -sar_to_earth_center: 7072935.2710 m -earth_radius_below_sensor: 6374216.3707 m -earth_semi_major_axis: 6378137.0000 m -earth_semi_minor_axis: 6356752.3141 m -number_of_state_vectors: 15 -time_of_first_state_vector: 40002.000000 s -state_vector_interval: 10.000000 s -state_vector_position_1: -1202502.7493 6500810.8061 2515842.3001 m m m -state_vector_velocity_1: 2079.47309 -2309.53890 6932.46052 m/s m/s m/s -state_vector_position_2: -1181658.0158 6477336.1615 2585023.4928 m m m -state_vector_velocity_2: 2089.41624 -2385.34851 6903.64775 m/s m/s m/s -state_vector_position_3: -1160715.5736 6453104.6806 2653912.6562 m m m -state_vector_velocity_3: 2099.01470 -2460.90464 6874.05521 m/s m/s m/s -state_vector_position_4: -1139678.8732 6428118.9436 2722502.0093 m m m -state_vector_velocity_4: 2108.26777 -2536.19825 6843.68628 m/s m/s m/s -state_vector_position_5: -1118551.3718 6402381.6207 2790783.8056 m m m -state_vector_velocity_5: 2117.17479 -2611.22032 6812.54442 m/s m/s m/s -state_vector_position_6: -1097336.5330 6375895.4722 2858750.3336 m m m -state_vector_velocity_6: 2125.73515 -2685.96187 6780.63319 m/s m/s m/s -state_vector_position_7: -1076037.8261 6348663.3481 2926393.9175 m m m -state_vector_velocity_7: 2133.94831 -2760.41395 6747.95623 m/s m/s m/s -state_vector_position_8: -1054658.7259 6320688.1875 2993706.9187 m m m -state_vector_velocity_8: 2141.81376 -2834.56766 6714.51728 m/s m/s m/s -state_vector_position_9: -1033202.7115 6291973.0188 3060681.7361 m m m -state_vector_velocity_9: 2149.33106 -2908.41412 6680.32013 m/s m/s m/s -state_vector_position_10: -1011673.2665 6262520.9586 3127310.8074 m m m -state_vector_velocity_10: 2156.49982 -2981.94447 6645.36870 m/s m/s m/s -state_vector_position_11: -990073.8782 6232335.2122 3193586.6096 m m m -state_vector_velocity_11: 2163.31969 -3055.14991 6609.66698 m/s m/s m/s -state_vector_position_12: -968408.0367 6201419.0725 3259501.6599 m m m -state_vector_velocity_12: 2169.79039 -3128.02167 6573.21903 m/s m/s m/s -state_vector_position_13: -946679.2350 6169775.9199 3325048.5169 m m m -state_vector_velocity_13: 2175.91169 -3200.55102 6536.02901 m/s m/s m/s -state_vector_position_14: -924890.9683 6137409.2222 3390219.7811 m m m -state_vector_velocity_14: 2181.68339 -3272.72926 6498.10116 m/s m/s m/s -state_vector_position_15: -903046.7331 6104322.5337 3455008.0956 m m m -state_vector_velocity_15: 2187.10536 -3344.54774 6459.43982 m/s m/s m/s - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/API_download_S1_SLC.py b/.codex_tmp/pyint_variants/no_rescue/pyint/API_download_S1_SLC.py deleted file mode 100644 index 9eb046a..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/API_download_S1_SLC.py +++ /dev/null @@ -1,494 +0,0 @@ -#! /usr/bin/env python3 -# -*- coding: utf-8 -*- - -########################################################################### -# Header information -########################################################################### - -"""API_download_S1_SLC.py: Script to download Sentinel-1 SLC images from the ASF mirror - Supports both bounding box and Shapefile for spatial filtering""" - -__author__ = "Alexis Hrysiewicz" -__copyright__ = "Copyright 2022" -__credits__ = ["Alexis Hrysiewicz"] -__license__ = "GPL" -__version__ = "2.0.0" -__maintainer__ = "Alexis Hrysiewicz" -__status__ = "Production" -__date__ = "Jan. 2022" - -########################################################################### -# Python packages -########################################################################### - -import os -import sys -import pandas as pd -import datetime -import os.path -import optparse - -# 尝试导入 geopandas 和 shapely 用于 Shapefile 支持 -try: - import geopandas as gpd - from shapely.geometry import box, Polygon, MultiPolygon - HAS_SHAPEFILE_SUPPORT = True -except ImportError: - HAS_SHAPEFILE_SUPPORT = False - print("Warning: geopandas/shapely not installed. Shapefile support disabled.") - print("Install with: pip install geopandas shapely") - -########################################################################### -########################################################################### - -class OptionParser (optparse.OptionParser): - - def check_required(self, opt): - option = self.get_option(opt) - if getattr(self.values, option.dest) is None: - self.error("%s option not supplied" % option) - -def bbox_from_shapefile(shapefile_path): - """ - 从 Shapefile 文件提取边界框 - - Args: - shapefile_path: Shapefile 文件路径（.shp） - - Returns: - 边界框字符串格式: "min_lon,min_lat,max_lon,max_lat" - """ - try: - # 读取 Shapefile - gdf = gpd.read_file(shapefile_path) - - # 计算总边界框 - total_bounds = gdf.total_bounds - # total_bounds = (minx, miny, maxx, maxy) - - # 转换为字符串格式 - bbox_str = f"{total_bounds[0]},{total_bounds[1]},{total_bounds[2]},{total_bounds[3]}" - - print(f"Shapefile bounding box: {bbox_str}") - return bbox_str - except Exception as e: - print(f"Error reading shapefile: {str(e)}") - return None - -def check_slc_exists(slc_name, path_SLC, path_RSLC, acquisition_date): - """ - 检查 SLC 文件是否已存在 - - Args: - slc_name: SLC 文件名 - path_SLC: SLC 存储路径 - path_RSLC: RSLC 存储路径 - acquisition_date: 采集日期字符串 - - Returns: - bool: 文件是否已存在 - """ - # 检查 .zip 文件 - if os.path.exists(os.path.join(path_SLC, slc_name + '.zip')): - return True - - # 检查 .rslc 文件 - try: - datei = datetime.datetime.strptime(acquisition_date.split('.')[0], '%Y-%m-%dT%H:%M:%S').strftime("%Y%m%d") - if os.path.exists(os.path.join(path_RSLC, datei + '.vv.rslc')): - return True - except: - pass - - return False - -def read_netrc(machine_name): - """ - 从 ~/.netrc 文件读取登录凭证 - - Args: - machine_name: 机器名称 (例如: urs.earthdata.nasa.gov) - - Returns: - tuple: (username, password) 或 (None, None) 如果未找到 - """ - netrc_path = os.path.expanduser('~/.netrc') - - if not os.path.exists(netrc_path): - return None, None - - try: - with open(netrc_path, 'r') as f: - lines = f.readlines() - i = 0 - while i < len(lines): - line = lines[i].strip() - - # 查找 machine 行 - if line.startswith('machine ' + machine_name): - username = None - password = None - - # 查找接下来的几行中的 login 和 password - j = i + 1 - while j < len(lines) and j < i + 5: # 最多查找5行 - next_line = lines[j].strip() - tokens = next_line.split() - - if len(tokens) >= 2: - if tokens[0] == 'login': - username = tokens[1] - elif tokens[0] == 'password': - password = tokens[1] - - # 如果找到下一个 machine，停止查找 - if next_line.startswith('machine '): - break - - j += 1 - - if username and password: - return username, password - - i += 1 - except Exception as e: - print(f"Warning: Error reading .netrc file: {str(e)}") - - return None, None - -def download_slc(url, username, password, path_SLC): - """ - 下载单个 SLC 文件 - - Args: - url: 下载 URL - username: ASF 用户名 - password: ASF 密码 - path_SLC: 存储路径 - - Returns: - bool: 下载是否成功 - """ - # 如果提供了用户名和密码，使用它们 - if username and password: - cmd = f'wget -c --http-user={username} --http-password={password} "{url}" -P {path_SLC}' - else: - # 否则使用 .netrc 文件 - # --auth-no-challenge: 立即发送认证信息，不等待服务器挑战 - # --netrc: 从 ~/.netrc 读取认证信息 - cmd = f'wget -c --auth-no-challenge --netrc "{url}" -P {path_SLC}' - - return os.system(cmd) == 0 - -def convert_yyyymmdd_to_iso(date_str): - """ - 将 YYYYMMDD 格式转换为 ISO 格式 (YYYY-MM-DDTHH:MM:SS) - - Args: - date_str: 日期字符串 (YYYYMMDD) - - Returns: - str: ISO 格式日期字符串 (YYYY-MM-DDT00:00:00) - """ - if len(date_str) == 8: - year = date_str[0:4] - month = date_str[4:6] - day = date_str[6:8] - return f"{year}-{month}-{day}T00:00:00" - else: - # 如果已经是其他格式，直接返回 - return date_str - -########################################################################### -########################################################################### - -if len(sys.argv) < 3: - prog = os.path.basename(sys.argv[0]) - print("="*80) - print("Sentinel-1 SLC Downloader from ASF") - print("="*80) - print("\nUsage examples:") - print("\n1. Using bounding box:") - print(" python3 " + prog + " -u username -p password -s . -r . \\") - print(" -b -10.78,51.27,-5.03,55.70 \\") - print(" -i 20170101 -j 20231231 \\") - print(f" -o 1 -f a -q IW -m csv -w n") - print("\n2. Using Shapefile (new feature):") - print(" python3 " + prog + " -u username -p password -s . -r . \\") - print(" --shp study_area.shp \\") - print(" -i 20170101 -j 20231231 \\") - print(f" -o 1 -f a -q IW -m csv -w n") - print("\nOptions:") - print(" -u, --username ASF username (optional, can use .netrc)") - print(" -p, --password ASF password (optional, can use .netrc)") - print(" -n, --netrc Machine name in ~/.netrc (default: urs.earthdata.nasa.gov)") - print(" -s, --path_SLC Output directory for SLC files") - print(" -r, --path_RSLC Directory to check for processed RSLC files") - print(" -b, --bbox Bounding box (min_lon,min_lat,max_lon,max_lat)") - print(" --shp Shapefile path (.shp) for spatial filtering") - print(" -i, --date_start Start date (YYYYMMDD)") - print(" -j, --date_end End date (YYYYMMDD)") - print(" -o, --orbit_relative Relative orbit number (optional)") - print(" --frame Frame number (optional)") - print(" -f, --flight_direction Flight direction (a=ascending, d=descending, or all)") - print(" -q, --q_acqui_mode Acquisition mode (IW, EW, SM, or all)") - print(" -m, --mode Output format (csv or kml)") - print(" -w, --write Download files? (Y/N)") - print(" --parallel Number of parallel downloads (default: 1, serial)") - print("\nNote: Shapefile support requires geopandas and shapely") - print("="*80) - sys.exit(-1) -else: - usage = "usage: %prog [options] " - parser = OptionParser(usage=usage) - parser.add_option("-u", "--username", action="store", type="string", default=None, - help="ASF username (can be read from ~/.netrc)") - parser.add_option("-p", "--password", action="store", type="string", default=None, - help="ASF password (can be read from ~/.netrc)") - parser.add_option("-n", "--netrc", action="store", type="string", default='urs.earthdata.nasa.gov', - help="Machine name in ~/.netrc file (default: urs.earthdata.nasa.gov)") - parser.add_option("-s", "--path_SLC", action="store", type="string", default='.') - parser.add_option("-r", "--path_RSLC", action="store", type="string", default='.') # Only available for GAMMA stack - parser.add_option("-b", "--bbox", action="store", type="string", default='-10.78,51.27,-5.03,55.70') - parser.add_option("--shp", "--shapefile", action="store", type="string", default=None, - help="Shapefile path for spatial filtering") - parser.add_option("-i", "--date_start", action="store", type="string", default='20170101') - parser.add_option("-j", "--date_end", action="store", type="string", default='20240101') - parser.add_option("-o", "--orbit_relative", action="store", type="float", default=None, - help="Relative orbit number (optional, if not specified will search all orbits)") - parser.add_option("--frame", action="store", type="int", default=None, - help="Frame number (optional, if not specified will search all frames)") - parser.add_option("-f", "--flight_direction", action="store", type="string", default='a', - help="Flight direction (a=ascending, d=descending, or 'all' for both)") - parser.add_option("-q", "--q_acqui_mode", action="store", type="string", default='IW') - parser.add_option("-m", "--mode", action="store", type="string", default='csv') - parser.add_option("-w", "--write", action="store", type="string", default='n') - parser.add_option("--parallel", action="store", type="int", default=1, - help="Number of parallel downloads (default: 1, serial)") - (options, args) = parser.parse_args() - -########################################################################### -# Main -########################################################################### - -date_format = "%Y-%m-%d" - -# 转换日期格式 -date_start_iso = convert_yyyymmdd_to_iso(options.date_start) -date_end_iso = convert_yyyymmdd_to_iso(options.date_end) - -# 处理用户名和密码：如果未提供，尝试从 .netrc 读取 -if options.username is None or options.password is None: - netrc_username, netrc_password = read_netrc(options.netrc) - if netrc_username and netrc_password: - if options.username is None: - options.username = netrc_username - print(f"Username read from ~/.netrc ({options.netrc})") - if options.password is None: - options.password = netrc_password - print(f"Password read from ~/.netrc ({options.netrc})") - else: - if options.username is None: - print("Error: Username not provided and not found in ~/.netrc") - print("Please provide username with -u option or configure ~/.netrc") - sys.exit(1) - if options.password is None: - print("Error: Password not provided and not found in ~/.netrc") - print("Please provide password with -p option or configure ~/.netrc") - sys.exit(1) - -# 处理空间范围：优先使用 Shapefile，否则使用 bounding box -if options.shp: - if not HAS_SHAPEFILE_SUPPORT: - print("Error: Shapefile support requires geopandas and shapely") - print("Install with: pip install geopandas shapely") - sys.exit(1) - - print(f"Using Shapefile: {options.shp}") - bbox = bbox_from_shapefile(options.shp) - if bbox is None: - print("Error: Failed to extract bounding box from shapefile") - sys.exit(1) -else: - bbox = options.bbox - print(f"Using bounding box: {bbox}") - -# 构建 API 查询命令 -# 轨道号、frame 和飞行方向都是可选的 -api_params = f"platform=s1&bbox={bbox}&start={date_start_iso}-UTC&end={date_end_iso}-UTC&processingLevel=SLC&maxResults=10000" - -# 如果指定了轨道号，添加到查询参数 -if options.orbit_relative is not None: - api_params += f"&relativeOrbit={int(options.orbit_relative)}" - -# 如果指定了 frame 号，添加到查询参数 -if hasattr(options, 'frame') and options.frame is not None: - api_params += f"&frame={options.frame}" - -# 如果指定了飞行方向且不是 'all'，添加到查询参数 -if options.flight_direction and options.flight_direction.upper() != 'ALL': - api_params += f"&flightDirection={options.flight_direction.upper()}" - -# 添加输出格式 -api_params += f"&output={options.mode.upper()}" - -cmd1 = f'curl "https://api.daac.asf.alaska.edu/services/search/param?{api_params}" > SLC_list.{options.mode.lower()}' - -print(f"Querying ASF API for SLC data...") -print(f" Bounding Box: {bbox}") -print(f" Date Range: {options.date_start} to {options.date_end}") -if options.orbit_relative is not None: - print(f" Orbit: {options.orbit_relative}") -else: - print(f" Orbit: All") -if hasattr(options, 'frame') and options.frame is not None: - print(f" Frame: {options.frame}") -else: - print(f" Frame: All") -if options.flight_direction and options.flight_direction.upper() != 'ALL': - print(f" Flight Direction: {options.flight_direction.upper()}") -else: - print(f" Flight Direction: All") -print(f" Mode: {options.q_acqui_mode.upper()}") -print() - -# 执行查询 -if options.mode.upper() == 'KML': - os.system(cmd1) -elif options.mode.upper() == 'CSV': - os.system(cmd1) -else: - print('Error: Please select a correct mode (csv or kml)...') - sys.exit(1) - -# 根据波束模式过滤列表 -print(f"Filtering results by acquisition mode: {options.q_acqui_mode.upper()}") -if os.path.exists("SLC_list.csv"): - os.rename("SLC_list.csv","SLC_list_orig.csv") - h = 0 - fout = open("SLC_list.csv",'w') - total_count = 0 - matched_count = 0 - - with open("SLC_list_orig.csv") as fi: - for li in fi: - total_count += 1 - if h > 0: - if options.q_acqui_mode.upper() == 'all': - fout.write(li) - matched_count += 1 - elif options.q_acqui_mode.upper() in li: - fout.write(li) - matched_count += 1 - else: - fout.write(li) - h = h + 1 - - fout.close() - os.remove("SLC_list_orig.csv") - - print(f" Total results: {total_count - 1}") - print(f" Filtered results: {matched_count}") -else: - print("Warning: SLC_list.csv not found") - sys.exit(1) - -# 下载文件 -if options.mode.upper() == 'CSV' and options.write.upper() == 'Y': - try: - listSLC = pd.read_csv("SLC_list.csv") - except Exception as e: - print(f"Error reading SLC_list.csv: {str(e)}") - sys.exit(1) - - print(f"\nFound {len(listSLC)} SLC files to process") - print("="*80) - - # 创建输出目录 - if not os.path.exists(options.path_SLC): - os.makedirs(options.path_SLC) - print(f"Created output directory: {options.path_SLC}") - - # 检查是否有并行参数 - parallel_numb = getattr(options, 'parallel', 1) - if parallel_numb is None: - parallel_numb = 1 - - # 准备下载数据列表 - download_tasks = [] - skipped_tasks = [] - - for h, slci in enumerate(listSLC['Granule Name']): - url = listSLC['URL'][h] - acquisition_date = listSLC['Acquisition Date'][h] - - # 检查文件是否已存在 - if check_slc_exists(slci, options.path_SLC, options.path_RSLC, acquisition_date): - skipped_tasks.append((h+1, slci, "Already exists")) - else: - download_tasks.append((h+1, slci, url, acquisition_date)) - - print(f"Files to download: {len(download_tasks)}") - print(f"Files skipped (already exists): {len(skipped_tasks)}") - - # 并行下载函数 - def download_worker(task): - idx, slci, url, acquisition_date = task - print(f"\n[{idx}/{len(listSLC)}] Downloading: {slci}") - - if download_slc(url, options.username, options.password, options.path_SLC): - print(f" Status: Download SUCCESS") - return (idx, slci, "SUCCESS") - else: - print(f" Status: Download FAILED") - return (idx, slci, "FAILED") - - # 执行下载 - downloaded = 0 - failed = 0 - - if parallel_numb > 1 and len(download_tasks) > 0: - # 并行下载 - from multiprocessing.pool import ThreadPool - print(f"\nStarting parallel download with {parallel_numb} workers...") - - pool = ThreadPool(parallel_numb) - results = pool.map(download_worker, download_tasks) - pool.close() - pool.join() - - # 统计结果 - for result in results: - if result[2] == "SUCCESS": - downloaded += 1 - else: - failed += 1 - elif len(download_tasks) > 0: - # 串行下载 - print("\nStarting serial download...") - for task in download_tasks: - result = download_worker(task) - if result[2] == "SUCCESS": - downloaded += 1 - else: - failed += 1 - - # 打印摘要 - print("\n" + "="*80) - print("DOWNLOAD SUMMARY") - print("="*80) - print(f"Total SLC files: {len(listSLC)}") - print(f"Downloaded: {downloaded}") - print(f"Skipped (already exists): {len(skipped_tasks)}") - print(f"Failed: {failed}") - print("="*80) - - if failed > 0: - print(f"\nWarning: {failed} downloads failed. Check the output above for details.") - sys.exit(1) - -elif options.mode.upper() == 'KML' and options.write.upper() == 'Y': - print('Please, select CSV mode to download..') -elif options.write.upper() != 'Y': - print(f"\nSLC list saved to: SLC_list.csv") - print("To download files, re-run with -w Y option") \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/ASAR_orb_cor.py b/.codex_tmp/pyint_variants/no_rescue/pyint/ASAR_orb_cor.py deleted file mode 100644 index 87eb3c5..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/ASAR_orb_cor.py +++ /dev/null @@ -1,139 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v1.0 ### -### Copy Right (c): 2017, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Email : ymcmrs@gmail.com ### -### Univ. : Central South University & University of Miami ### -################################################################# - -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob -import argparse -import linecache -import datetime - -def StrNum(S): - S = str(S) - if len(S)==1: - S='0' +S - return S - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close() - -######################################################################### - -INTRODUCTION = ''' -############################################################################# - Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 - - Correct ASAR orbit using precise DORIS data. - -''' - -EXAMPLE = ''' - Usage: - ASAR_orb_cor.py projectName Date - - Examples: - ASAR_orb_cor.py AqabaERSA 960101 - -############################################################################## -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Download (and correct) precise ERS orbit data.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='project name') - parser.add_argument('date',help='date of the ERS data') - - inps = parser.parse_args() - return inps - -################################################################################ - - -def main(argv): - - total = time.time() - inps = cmdLineParse() - - projectName = inps.projectName - DATE = inps.date - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - ASAROrbDir = os.getenv('ASARORBDIR') - - - slcDir = scratchDir + '/' + projectName + '/SLC' - projDir = scratchDir + '/' + projectName - - - if len(DATE)==8: - DATE8 = DATE - DATE6 = DATE[2:8] - else: - DATE8 = '20' + DATE - DATE6 = DATE - - - slcDir1 = slcDir + '/' + DATE6 - slcpar = slcDir1 + '/' + DATE6 + '.slc.par' - - - Year = DATE8[0:4] - Month = DATE8[4:6] - Day = DATE8[6:8] - - DATE1 = (datetime.date(int(Year),int(Month),int(Day)) + datetime.timedelta(days=-1)).strftime("%Y%m%d") - DATE2 = (datetime.date(int(Year),int(Month),int(Day)) + datetime.timedelta(days=1)).strftime("%Y%m%d") - - os.chdir(ASAROrbDir) - call_str = 'ls > t0' - os.system(call_str) - - call_str="grep " + DATE1 + " t0 | grep " + DATE2 + " >t01" - os.system(call_str) - - AA= np.loadtxt('t01',dtype=np.str) - AA_file = ASAROrbDir + '/' + str(AA) - - call_str = 'DORIS_vec ' + slcpar + ' ' + AA_file - os.system(call_str) - - - print("Using DORIS orbital data for %s is done." % DATE) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/ASAR_orb_cor_all .py b/.codex_tmp/pyint_variants/no_rescue/pyint/ASAR_orb_cor_all .py deleted file mode 100644 index 0676e4c..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/ASAR_orb_cor_all .py +++ /dev/null @@ -1,125 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v1.0 ### -### Copy Right (c): 2017, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Email : ymcmrs@gmail.com ### -### Univ. : Central South University & University of Miami ### -################################################################# - -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob -import argparse -import linecache - -def StrNum(S): - S = str(S) - if len(S)==1: - S='0' +S - return S - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close() - -def is_number(s): - try: - int(s) - return True - except ValueError: - return False - -######################################################################### - -INTRODUCTION = ''' -############################################################################# - Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 - - Correct ASAR orbit using precise DORIS data. - -''' - -EXAMPLE = ''' - Usage: - ASAR_orb_cor_all.py projectName - - Examples: - ASAR_orb_cor_all.py AqabaERSA - -############################################################################## -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Download (and correct) precise ERS orbit data.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='project name') - - inps = parser.parse_args() - return inps - -################################################################################ - - -def main(argv): - - total = time.time() - inps = cmdLineParse() - - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - slcDir = scratchDir + '/' + projectName + '/SLC' - - ListSLC = os.listdir(slcDir) - Datelist = [] - SLCfile = [] - SLCParfile = [] - - - for kk in range(len(ListSLC)): - if ( is_number(ListSLC[kk]) and len(ListSLC[kk])==6 ): # if SAR date number is 8, 6 should change to 8. - DD=ListSLC[kk] - Year=int(DD[0:2]) - Month = int(DD[2:4]) - Day = int(DD[4:6]) - Datelist.append(ListSLC[kk]) - N = len(Datelist) - - for i in range(N): - call_str = 'ERS_orb_cor.py ' + projectName + ' ' + Datelist[i] - print(call_str) - os.system(call_str) - - print("Using DEFT orbital data for project %s is done." % projectName) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/ASAR_orb_cor_par.py b/.codex_tmp/pyint_variants/no_rescue/pyint/ASAR_orb_cor_par.py deleted file mode 100644 index 78f2aea..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/ASAR_orb_cor_par.py +++ /dev/null @@ -1,122 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v1.0 ### -### Copy Right (c): 2017, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Email : ymcmrs@gmail.com ### -### Univ. : Central South University & University of Miami ### -################################################################# - -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob -import argparse -import linecache -import datetime - -def StrNum(S): - S = str(S) - if len(S)==1: - S='0' +S - return S - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close() - -######################################################################### - -INTRODUCTION = ''' -############################################################################# - Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 - - Precise ENVISAT orbit data from Delft (http://www.deos.tudelft.nl/). - Correct the orbit parameters using DORIS_vec - -''' - -EXAMPLE = ''' - Usage: - ASAR_orb_cor_par.py slc_par - - Examples: - ASAR_orb_cor_par.py 960101.slc.par - -############################################################################## -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Download (and correct) precise ERS orbit data.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('slc_par',help='slc_par file.') - - inps = parser.parse_args() - return inps - -################################################################################ - - -def main(argv): - - total = time.time() - inps = cmdLineParse() - - slcpar = inps.slc_par - ASAROrbDir = os.getenv('ASARORBDIR') - PATH =os.getcwd() - - DATE = UseGamma(slcpar,'read','date:') - Year = str(int(DATE[0:4])) - Month = str(int(DATE[4:8])) - Day = str(int(DATE[8:12])) - - DATE1 = (datetime.date(int(Year),int(Month),int(Day)) + datetime.timedelta(days=-1)).strftime("%Y%m%d") - DATE2 = (datetime.date(int(Year),int(Month),int(Day)) + datetime.timedelta(days=1)).strftime("%Y%m%d") - - os.chdir(ASAROrbDir) - call_str = 'ls > t0' - os.system(call_str) - - call_str="grep " + DATE1 + " t0 | grep " + DATE2 + " > t01" - os.system(call_str) - - AA= np.loadtxt('t01',dtype=np.str) - AA_file = ASAROrbDir + '/' + str(AA) - - os.chdir(PATH) - call_str = 'DORIS_vec ' + slcpar + ' ' + AA_file + ' 20' - os.system(call_str) - - - print("Using DORIS orbital data for %s is done." % slcpar) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/.gitignore b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/.gitignore deleted file mode 100644 index 946f285..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/.gitignore +++ /dev/null @@ -1,167 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# mac -.DS_Store - - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ -tests - - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/#use-with-ide -.pdm.toml - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ - \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/LICENSE b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/LICENSE deleted file mode 100644 index d470d14..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2023 Fanchengyan - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/README.md b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/README.md deleted file mode 100644 index 433a5ed..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# AutoGACOS -A Python library for automatically submitting and downloading GACOS data diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/Makefile b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/Makefile deleted file mode 100644 index d0c3cbf..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = source -BUILDDIR = build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/requirements.txt b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/requirements.txt deleted file mode 100644 index 2a9aa68..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -recommonmark -sphinx>=7 -myst-parser -myst_nb -sphinx_rtd_theme -Jinja2 diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/api/datasets/datasets.rst b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/api/datasets/datasets.rst deleted file mode 100644 index f8fa7f7..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/api/datasets/datasets.rst +++ /dev/null @@ -1,32 +0,0 @@ -InSAR Datasets -============== - -To automatically submit requests to the GACOS service, the user needs to provide -the coordinates of the area of interest and the acquisition dates and times of -the SAR images. We provide a set of classes to automatically retrieve the -those information from well known InSAR datasets. - -Currently, the following InSAR Datasets are supported: - * :class:`.HyP3Dataset` - * :class:`.LiCSARDataset` - -HyP3Dataset ------------ - -.. autoclass:: gacos.HyP3Dataset - :members: - :undoc-members: - :member-order: bysource - :show-inheritance: - -LiCSARDataset -------------- - -.. autoclass:: gacos.LiCSARDataset - :members: - :undoc-members: - :member-order: bysource - :show-inheritance: - - - \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/api/index.rst b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/api/index.rst deleted file mode 100644 index 32d5ef4..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/api/index.rst +++ /dev/null @@ -1,11 +0,0 @@ -Python API Reference -==================== - - -.. toctree:: - - datasets/datasets - submit/submit - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/api/submit/submit.rst b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/api/submit/submit.rst deleted file mode 100644 index a07379b..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/api/submit/submit.rst +++ /dev/null @@ -1,15 +0,0 @@ -Requests Submitter -================== - -The GACOS limits the number of acquisition requests that can be submitted at a time. -This is to prevent the system from being overloaded. But, if you have a large number -of acquisition requests to submit, you can submit them in batches. We designed a -:class:`gacos.Submitter` class to help you submit the requests in batches. The -acquisitions will be split into batches which not exceed the 20 acquisitions per -request. - -.. autoclass:: gacos.Submitter - :members: - :undoc-members: - :member-order: bysource - :show-inheritance: diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/conf.py b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/conf.py deleted file mode 100644 index 073055f..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/conf.py +++ /dev/null @@ -1,58 +0,0 @@ -# Configuration file for the Sphinx documentation builder. -# -# For the full list of built-in configuration values, see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Project information ----------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information - -project = "FanSAR" -copyright = "2023, Fancy" -author = "Fancy" -release = "v1.0" - -# -- General configuration --------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration - -extensions = [ - "myst_nb", - "sphinx.ext.autodoc", - "sphinx.ext.doctest", - "sphinx.ext.intersphinx", - "sphinx.ext.napoleon", - "sphinx.ext.todo", -] -source_suffix = { - ".rst": "restructuredtext", - ".md": "markdown", - ".ipynb": "myst-nb", -} - -# templates_path = ['_templates'] -exclude_patterns = [] - -# to disable execution of notebooks -nb_execution_mode = "off" -nb_execution_excludepatterns = ["quickstart.ipynb"] - - -# -- Options for HTML output ------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output - -html_theme = "sphinx_rtd_theme" -# html_static_path = ['_static'] - -# General information about the project. -project = "faninsar" -copyright = "2023, Fancy" -author = "Chengyan Fan" - -video_enforce_extra_source = True - -autodoc_default_options = { - "members": True, - "undoc-members": True, - "member-order": "bysource", - "special-members": "__init__", - ":show-inheritance:": True, -} diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/index.rst b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/index.rst deleted file mode 100644 index 91f1318..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/index.rst +++ /dev/null @@ -1,19 +0,0 @@ -===================================== -Welcome to AutoGACOS's documentation! -===================================== - - -Introduction ------------- - -.. toctree:: - :maxdepth: 4 - :caption: Contents: - - intro - user_guide/quickstart - AutoGACOS API Reference - terminology - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/intro.rst b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/intro.rst deleted file mode 100644 index e8b5eb2..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/intro.rst +++ /dev/null @@ -1,3 +0,0 @@ -Introduction -============ - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/terminology.rst b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/terminology.rst deleted file mode 100644 index 844e30d..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/terminology.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. _terminology: - -Terminology -=========== - -.. glossary:: - - Acquisition - A single SAR acquisition, and is expressed as ``datetime.datetime`` object. - - Pair - A pair is a combination of two SAR acquisitions. - - Pairs - A collection of pairs. - - Loop - A loop - - Loops - A collection of loops. - - SBASNetwork - A collection of loops and pairs. - - CRS - A coordinate reference system (CRS) is a coordinate-based local, regional or global system used to locate geographical entities. In FanInSAR, the CRS is handled by the ``rasterio`` package. \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/user_guide/quickstart.ipynb b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/user_guide/quickstart.ipynb deleted file mode 100644 index beb1bcd..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/user_guide/quickstart.ipynb +++ /dev/null @@ -1,635 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Quickstart for AutoGACOS\n", - "\n", - "**AutoGACOS** is a Python library for automatically submitting and downloading GACOS data" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "import gacos" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "path = \"/Volumes/Data/Hyp3/descending_roi\"\n", - "ds = gacos.HyP3Dataset(path)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "check all acquisition dates in Hyp3 dateset" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "DatetimeIndex(['2015-11-12 23:26:31', '2015-12-06 23:26:31',\n", - " '2015-12-30 23:26:30', '2016-01-23 23:26:29',\n", - " '2016-02-16 23:26:28', '2016-03-11 23:26:28',\n", - " '2016-04-04 23:26:29', '2016-04-28 23:26:30',\n", - " '2016-05-22 23:26:34', '2016-06-15 23:26:36',\n", - " ...\n", - " '2023-01-10 23:27:15', '2023-01-22 23:27:15',\n", - " '2023-02-03 23:27:15', '2023-02-15 23:27:14',\n", - " '2023-02-27 23:27:15', '2023-03-11 23:27:14',\n", - " '2023-03-23 23:27:14', '2023-04-04 23:27:15',\n", - " '2023-08-14 23:27:22', '2023-09-07 23:27:23'],\n", - " dtype='datetime64[ns]', length=204, freq=None)\n", - "Index(['20151112', '20151206', '20151230', '20160123', '20160216', '20160311',\n", - " '20160404', '20160428', '20160522', '20160615',\n", - " ...\n", - " '20230110', '20230122', '20230203', '20230215', '20230227', '20230311',\n", - " '20230323', '20230404', '20230814', '20230907'],\n", - " dtype='object', length=204)\n", - "['23:25' '23:26' '23:27']\n" - ] - } - ], - "source": [ - "print(ds.date_times)\n", - "print(ds.dates)\n", - "print(ds.times)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Submit GACOS requests automatically" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "bf9575b9deea497089cc8167b8c8a438", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/3 [00:00>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 25, 'date': '20161007\\n20161031\\n20161124\\n20161218\\n20170111\\n20170204', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", - " sleeping for 546 seconds...\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "8fd9eb9543774f3b87058eef331792df", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/6 [00:00>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 26, 'date': '20151112\\n20151206\\n20151230\\n20160123\\n20160216\\n20160311\\n20160404\\n20160428\\n20160522\\n20160615\\n20160709\\n20160802\\n20160826\\n20160919\\n20170222\\n20170318\\n20170330', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", - ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 26, 'date': '20170411\\n20170423\\n20170505\\n20170517\\n20170529\\n20170610\\n20170622\\n20170704\\n20170716\\n20170728\\n20170809\\n20170821\\n20170902\\n20170914\\n20171008\\n20171020\\n20171101', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", - ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 26, 'date': '20171113\\n20171125\\n20171207\\n20171219\\n20171231\\n20180112\\n20180124\\n20180205\\n20180217\\n20180301\\n20180313\\n20180325\\n20180406\\n20180418\\n20180430\\n20180512\\n20180524', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", - ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 26, 'date': '20180605\\n20180617\\n20180629\\n20180711\\n20180723\\n20180804\\n20180816\\n20180921\\n20181003\\n20181015\\n20181027\\n20181108\\n20181120\\n20181202\\n20181214\\n20181226\\n20190107', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", - ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 26, 'date': '20190119\\n20190131\\n20190212\\n20190224\\n20190308\\n20190320\\n20190401\\n20190413\\n20190425\\n20190507\\n20190519\\n20190531\\n20190612\\n20190624\\n20190706\\n20190718\\n20190730', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", - ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 26, 'date': '20190811\\n20190823\\n20190904\\n20191221\\n20200102\\n20200114\\n20200126\\n20200207\\n20200219\\n20200302\\n20200314\\n20200326\\n20200407\\n20200419\\n20200501\\n20200513', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", - " sleeping for 288 seconds...\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "4aa82286c5414723adcddcc8d864c133", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/5 [00:00>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 27, 'date': '20190916\\n20190928\\n20191010\\n20191022\\n20191103\\n20191115\\n20191127\\n20191209\\n20200525\\n20200606\\n20200618\\n20200630\\n20200712\\n20200724\\n20200805\\n20200817\\n20200829\\n20200910\\n20200922\\n20201004', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", - ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 27, 'date': '20201016\\n20201028\\n20201109\\n20201121\\n20201203\\n20201215\\n20201227\\n20210108\\n20210120\\n20210201\\n20210213\\n20210225\\n20210309\\n20210321\\n20210402\\n20210414\\n20210426\\n20210508\\n20210520\\n20210601', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", - ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 27, 'date': '20210613\\n20210625\\n20210707\\n20210719\\n20210731\\n20210812\\n20210824\\n20210905\\n20210917\\n20210929\\n20211011\\n20211023\\n20211104\\n20211116\\n20211128\\n20211210\\n20211222\\n20220103\\n20220115', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", - ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 27, 'date': '20220127\\n20220208\\n20220220\\n20220304\\n20220316\\n20220328\\n20220409\\n20220421\\n20220503\\n20220515\\n20220527\\n20220620\\n20220702\\n20220714\\n20220726\\n20220807\\n20220819\\n20220831\\n20220912', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", - ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 27, 'date': '20220924\\n20221006\\n20221018\\n20221030\\n20221111\\n20221123\\n20221205\\n20221217\\n20221229\\n20230110\\n20230122\\n20230203\\n20230215\\n20230227\\n20230311\\n20230323\\n20230404\\n20230814\\n20230907', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", - " sleeping for 743 seconds...\n" - ] - } - ], - "source": [ - "# initialize a submitter with the dataset and your email\n", - "submitter = gacos.Submitter(ds, email=\"your_email@xxx.com\")\n", - "\n", - "# submit all gacos requests for acquisitions in the Hyp3 dataset\n", - "submitter.post_requests()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# check the status of all requests\n", - "print(f\"Successfully submitted requests: \\n{submitter.succeed}\")\n", - "print(f\"Failed to submit requests: \\n{submitter.failed}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Retrieve GACOS results urls from your email automatically" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "667dccc73a434102a87226ebcb453389", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Retrieving GACOS urls: 0%| | 0/240 [00:00 None: - """Initialize SarDataset class - - Parameters - ---------- - bounds : tuple[float, float, float, float] - The bounding box of the dataset. - date_times : pd.DatetimeIndex - The datetime index of the dataset. - gacos_dir : Optional[Union[Path, str]], optional - The directory used to save gacos data. Used to check if the data is - already downloaded and avoid resubmitting. Default is None. - """ - self.bounds = bounds - self._date_times = date_times - - self._dates = date_times.strftime("%Y%m%d") - - if gacos_dir is not None: - self._dates_remain = self._get_dates_remain(gacos_dir) - else: - self._dates_remain = self.dates - - hour = date_times.hour - minute = np.round((date_times.second / 60) + date_times.minute).astype(int) - times = pd.Series([f"{h:02d}:{m:02d}" for h, m in zip(hour, minute)]) - self._times = times - - self._times_remain = self._get_times_remain() - - def __str__(self) -> str: - return ( - f"{self.__class__.__name__}(\n" - f" bounds={self.bounds}, \n" - f" times={len(self.times)}, \n" - f" dates={len(self.dates)}\n" - ")" - ) - - def __repr__(self) -> str: - return self.__str__() - - def _get_dates_remain(self, gacos_dir: Union[Path, str]): - """Get the dates that are not downloaded yet. - Parameters - ---------- - gacos_dir : Union[Path, str] - The directory used to save gacos data. Used to check if the data is - already downloaded and avoid resubmitting. - - Returns - ------- - dates_remain : np.ndarray - The dates that are not downloaded yet. - """ - gacos_files = list(Path(gacos_dir).rglob("*.ztd.tif")) - gacos_dates = [] - for i in gacos_files: - stem = i.stem.split(".")[0] - if len(stem) == 8: - gacos_dates.append(stem) - dates_remain = np.setdiff1d(self.dates, gacos_dates) - return dates_remain - - def _get_times_remain(self): - """Get the times corresponding to the dates that are not downloaded yet.""" - idx = np.where(np.isin(self.dates, self.dates_remain))[0] - times_remain = self._times[idx] - return times_remain - - @property - def dates(self): - """The dates (YYYYMMDD) of the acquisitions parsed from dataset.""" - return self._dates - - @property - def times(self): - """The times (HH:MM) of the acquisitions parsed from dataset.""" - return self._times.unique() - - @property - def date_times(self): - """The datetime of the acquisitions parsed from dataset.""" - return self._date_times - - @property - def dates_remain(self): - """The dates that are not downloaded yet. If gacos_dir is None, then - dates_remain is the same as dates.""" - return self._dates_remain - - @property - def times_remain(self): - """The times corresponding to the dates that are not downloaded yet.""" - return self._times_remain - - def gen_datetime_patches( - self, - mode: Literal["all", "remain"] = "remain", - ) -> dict: - """Generate datetime patches. - - Parameters - ---------- - mode : Literal["all", "remain"], optional - The mode to generate datetime patches. If "all", then generate all - the datetime patches. If "remain", then generate the datetime - patches of the dates that are not downloaded yet. Default is - "remain". - - Returns - ------- - datetime_patches : dict - The datetime patches. The key is the time (HH:MM) and the value is - the datetime patches. - """ - nums = 20 - datetime_patches = {} - - if mode == "all": - for _time in self.times: - _dts = self.dates[self._times == _time] - n_patch = np.ceil(len(_dts) / nums) - dates_patch = np.array_split(_dts, n_patch) - datetime_patches[_time] = dates_patch - elif mode == "remain": - for _time in self.times_remain: - _dts = self.dates_remain[self._times_remain == _time] - n_patch = np.ceil(len(_dts) / nums) - dates_patch = np.array_split(_dts, n_patch) - datetime_patches[_time] = dates_patch - - return datetime_patches - - def gen_post_data( - self, - dates: Union[list, np.ndarray], - times: Union[tuple[int, int], tuple[str, str]], - email: str, - ): - """Generate post data for gacos website. - - Parameters - ---------- - dates : list or np.ndarray - The list of dates. - times : tuple[int, int] - The time of the acquisition (hour, minute). - email : str - The email address to receive the gacos data. - - Returns - ------- - post_data : dict - The post data. - """ - if isinstance(dates, np.ndarray): - dates = dates.tolist() - times = [int(t) for t in times] - - post_data = { - "N": self.bounds[3], - "W": self.bounds[0], - "S": self.bounds[1], - "E": self.bounds[2], - "H": times[0], - "M": times[1], - "date": "\n".join(dates), - "type": "2", - "email": email, - } - return post_data - - -class LiCSARDataset(SarDataset): - def __init__( - self, - home_dir: Union[Path, str], - gacos_dir: Optional[Union[Path, str]] = None, - ) -> None: - """Initialize LiCSARDataset class - - Parameters - ---------- - home_dir : Union[Path, str] - The home directory of LiCSAR dataset. - gacos_dir : Optional[Union[Path, str]], optional - The directory used to save gacos data. Used to check if the data is - already downloaded and avoid resubmitting. Default is None. - """ - self.home_dir = Path(home_dir) - self.dataset = LiCSAR(home_dir) - bounds = self.dataset.bounds - time = self._get_time() - dates = self.dataset.pairs.dates - date_times = pd.to_datetime([f"{d} {time[0]}:{time[1]}:00" for d in dates]) - super().__init__(bounds, date_times, gacos_dir) - - def _get_time(self): - """Get the acquisition time of acquisitions. - - Returns - ------- - time: tuple[int, int] - A tuple of hour and minute representing the acquisition time. - - Raises - ------ - ValueError - If no center_time found in metadata.txt. - """ - meta_file = sorted(self.home_dir.rglob("metadata.txt"))[0] - - with open(meta_file) as f: - lines = f.readlines() - time = None - for line in lines: - line_split = line.split("=") - key, value = (line_split[0].strip(), line_split[1]) - if "center_time" == key: - center_time = value.strip() - hour, minute, second = center_time.split(":") - hour, minute, second = int(hour), int(minute), float(second) - minute = minute + int(np.round(second / 60, 0)) - return hour, minute - else: - continue - if time is None: - raise ValueError(f"No center_time found in {meta_file}") - - -class HyP3Dataset(SarDataset): - def __init__( - self, - home_dir: Union[Path, str], - gacos_dir: Optional[Union[Path, str]] = None, - ) -> None: - """Initialize HyP3Dataset class - - Parameters - ---------- - home_dir : Union[Path, str] - The home directory of HyP3 dataset. - gacos_dir : Optional[Union[Path, str]], optional - The directory used to save gacos data. Used to check if the data is - already downloaded and avoid resubmitting. Default is None. - """ - self.dataset = HyP3(home_dir) - bounds = self.dataset.bounds.to_crs("epsg:4326") - date_times = self.dataset.datetime - - super().__init__(bounds, date_times, gacos_dir) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/gacos/download.py b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/gacos/download.py deleted file mode 100644 index 5afcf60..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/gacos/download.py +++ /dev/null @@ -1,158 +0,0 @@ -import tarfile -from pathlib import Path -from typing import Optional, Union - -import numpy as np -import pandas as pd -from data_downloader import downloader -from faninsar.query import BoundingBox -from tqdm.auto import tqdm - -from .parse_email import GACOSEmail - - -class Downloader: - def __init__( - self, - url_file: Union[Path, str], - output_dir: Union[Path, str], - tar_gz_dir: Optional[Union[Path, str]] = None, - keep_original: bool = False, - times: Optional[Union[float, list[float]]] = None, - bounds: Optional[tuple[float, float, float, float]] = None, - ) -> None: - """Initialize Downloader class - - Parameters - ---------- - url_file : Union[Path, str] - Path to file containing URLs that created by :meth:`GACOSEmail.retrieve_gacos_urls` - output_dir : Union[Path, str] - directory to output gacos files - tar_gz_dir : Optional[Union[Path, str]], optional - directory to store downloaded *.tar.gz files. If None, then - `output_dir` is used. Default is None. - keep_original : bool, optional - Whether to keep original files (*.tar.gz). Default is False. - times : Optional[float], optional - times of acquisition, used to filter out files that are not needed. - this can be a single time or a list of times. times differ by less - than 10 minutes are considered the same. Default is None. - bounds : Optional[tuple[float, float, float, float]], optional - bounds of area of interest with order (W, S, E, N), used to filter - out files that are not needed. Default is None. - """ - self.url_file = Path(url_file) - self.output_dir = Path(output_dir) - if tar_gz_dir is None: - self.tar_gz_dir = self.output_dir - self.keep_original = keep_original - - if not self.url_file.exists(): - raise FileNotFoundError(f"{self.url_file} does not exist") - if not self.output_dir.exists(): - self.output_dir.mkdir(parents=True) - if not self.tar_gz_dir.exists(): - self.tar_gz_dir.mkdir(parents=True) - - self.df_urls = pd.read_csv(self.url_file, header=0) - - # only keep urls that intersect with bounds - if bounds is not None: - mask_bbox = self._bbox_mask(BoundingBox(*bounds)) - - # only keep urls that acquisition time is within 10 minutes of `time` - if times is not None: - if isinstance(times, float): - times = [times] - mask_time = self._time_mask(times) - - if bounds is not None and times is not None: - self.mask = mask_bbox & mask_time - elif bounds is not None: - self.mask = mask_bbox - elif times is not None: - self.mask = mask_time - else: - self.mask = np.ones(self.df_urls.shape[0], dtype=bool) - - self.mask = self.mask & self.date_mask - - def _bbox_mask(self, bounds) -> np.ndarray: - intersection_bbox = np.array( - [ - BoundingBox(*b).intersects(bounds) - for b in zip( - self.df_urls["south"].astype(float), - self.df_urls["west"].astype(float), - self.df_urls["north"].astype(float), - self.df_urls["east"].astype(float), - ) - ] - ) - return intersection_bbox - - def _time_mask(self, times) -> np.ndarray: - """Only keep urls that acquisition time is within 10 minutes of `time`""" - intersection_times = [] - for time in times: - intersection_times.append( - np.array( - np.abs(self.df_urls["time"].astype(float) - time) - <= 1 / 60 * 10 # 10 minutes - ) - ) - intersection_time = np.any(intersection_times, axis=0) - return intersection_time - - @property - def date_mask(self) -> np.ndarray: - """Remove urls that all acquisition dates have been downloaded""" - dates_urls = self.df_urls["date"].map(lambda x: eval(x)) - intersection_dates = [] - for dt_url in dates_urls: - intersection_dates.append(~np.all(np.isin(dt_url, self.dates_downloaded))) - return np.array(intersection_dates) - - @property - def dates_downloaded(self) -> np.ndarray: - """Return dates that have been downloaded""" - gacos_files = list(self.output_dir.rglob("*.ztd.tif")) - dates = [] - for i in gacos_files: - stem = i.stem.split(".")[0] - if len(stem) == 8: - dates.append(stem) - return np.array(dates) - - def download(self) -> None: - """Download GACOS files from URLs in file created by :meth:`GACOSEmail.retrieve_gacos_urls`""" - urls_used = self.df_urls[self.mask]["url"].values - - for url in tqdm(urls_used, unit="file", desc="Downloading GACOS files"): - gz_file = self.tar_gz_dir / Path(url).name - downloader.download_data(url, file_name=gz_file) - self._extract_tar_gz(gz_file) - if not self.keep_original: - self._delete_file(gz_file) - - def _extract_tar_gz(self, gz_file) -> None: - """Unzip/extract downloaded GACOS files - - Parameters - ---------- - gz_file : Path - path to downloaded GACOS file (*.tar.gz) - """ - with tarfile.open(gz_file, "r:gz") as tar: - tar.extractall(path=self.output_dir) - - def _delete_file(self, gz_file) -> None: - """Delete original GACOS files - - Parameters - ---------- - gz_file : Path - path to downloaded GACOS file (*.tar.gz) - """ - gz_file.unlink() diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/gacos/parse_email.py b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/gacos/parse_email.py deleted file mode 100644 index f074cdf..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/gacos/parse_email.py +++ /dev/null @@ -1,391 +0,0 @@ -import email -import email.message -import getpass -import imaplib -import poplib -import re -from email.parser import Parser -from email.utils import parseaddr -from pathlib import Path -from typing import Literal, Optional, Union - -import pandas as pd -from tqdm.auto import tqdm - - -class GACOSEmail: - """a class to retrieve gacos urls from email. - - .. note:: - The IMAP server is used to retrieve content from email. Some email - service providers may need to enable the IMAP service in the settings. - **You are recommended to use a new email account to receive gacos urls to - avoid polluting your own email account**. - """ - - def __init__( - self, - username: str, - password: str, - host: str, - prompt: bool = False, - email_protocol: Literal["imap", "pop3"] = "imap", - port: Optional[int] = None, - gacos_email: str = "gacos2017@foxmail.com", - gacos_suffix: str = "tar.gz", - start_date: Optional[str] = None, - end_date: Optional[str] = None, - date_args: Optional[dict] = None, - ssl: bool = False, - ) -> None: - """Retrieve gacos urls from email. - - Parameters - ---------- - username : str - The username of the email address. - password : str - The password. - host : str - The host of the email address. For example, the host of gmail for - imap is "imap.gmail.com". You can find the host of your email - settings or search it on the Internet. - prompt: bool, optional - Prompt for username and/or password interactively when they are not - provided as keyword parameters. Default is False. - email_protocol : str, one of ["imap", "pop3"], optional - The protocol of the email. Default is "imap". - port : int, optional - The port of the host of your email. If None, the default port will be - used. Default is None. - gacos_email : str, optional - The email address of gacos. Default is "gacos2017@foxmail.com". - gacos_suffix : str, optional - The suffix of the gacos file url. Default is "tar.gz". The suffix is used - to filter urls in the email. This parameter is used to avoid the - situation that the email contains other urls. - start_date / end_date: str, optional - The start/end date of email. Used to filter the email. Default is None. Can be any format that can be parsed by pandas.to_datetime. - date_args : dict, optional - The arguments are passed to pandas.to_datetime. Default is None. - ssl : bool, optional - Whether to use SSL connection. Default is False. - """ - if prompt: - self.username = None - self.password = None - else: - self.username = username - self.password = password - self.host = host - self.email_protocol = email_protocol - self.port = port - self.gacos_email = gacos_email - self.gacos_suffix = gacos_suffix - self.ssl = ssl - - # date part - self.start_date = start_date - self.end_date = end_date - if date_args is None: - date_args = {} - self.date_args = date_args - - def _retrieve_gacos_urls_pop3(self): - server = login_in_email_pop3( - self.username, self.password, self.host, self.port, ssl=self.ssl - ) - print(server.getwelcome()) - - nums = server.stat()[0] - - gacos = [] - for i in tqdm(range(1, nums + 1), unit=" emails", desc="Retrieving GACOS Urls"): - response, msgLines, octets = server.retr(i) - msgLinesToStr = b"\r\n".join(msgLines).decode("utf8", "ignore") - messageObject = Parser().parsestr(msgLinesToStr) - - senderContent = messageObject["From"] - senderRealName, senderAdr = parseaddr(senderContent) - if senderAdr == self.gacos_email: - if not in_date_range( - pd.to_datetime(messageObject["Date"]).tz_localize(None), - self.start_date, - self.end_date, - self.date_args, - ): - continue - - msgBodyContents = get_content(messageObject) - info = parse_gacos_info( - msgBodyContents, - gacos_suffix=self.gacos_suffix, - ) - if info is not None: - gacos.append(info) - - server.quit() - - return gacos - - def _retrieve_gacos_urls_imap(self): - server = login_in_email_imap( - self.username, self.password, self.host, self.port, ssl=self.ssl - ) - if server is None: - print("IMAP server connection failed, skipping email check.") - return [] - server.select("inbox") - status, data = server.search(None, "ALL") - - gacos = [] - for i in tqdm(data[0].split(), unit=" emails", desc="Retrieving GACOS urls"): - res, msg = server.fetch(i, "(RFC822)") - for response_part in msg: - if isinstance(response_part, tuple): - msgLines = response_part[1].decode("utf8", "ignore") - break - - messageObject = Parser().parsestr(msgLines) - - senderContent = messageObject["From"] - senderRealName, senderAdr = parseaddr(senderContent) - if senderAdr == self.gacos_email: - if not in_date_range( - pd.to_datetime(messageObject["Date"]).tz_localize(None), - self.start_date, - self.end_date, - self.date_args, - ): - continue - - msgBodyContents = get_content(messageObject) - info = parse_gacos_info( - msgBodyContents, - gacos_suffix=self.gacos_suffix, - ) - if info is not None: - gacos.append(info) - - server.close() - - return gacos - - def retrieve_gacos_urls( - self, - output_file: Union[str, Path], - ): - """Retrieve gacos urls from username. - - Parameters - ---------- - output_file : str or Path - The output file used to save the gacos urls. - """ - if self.email_protocol == "pop3": - gacos = self._retrieve_gacos_urls_pop3() - elif self.email_protocol == "imap": - gacos = self._retrieve_gacos_urls_imap() - else: - raise ValueError("email_protocol must be 'pop3' or 'imap'.") - - cols = ["url", "south", "north", "west", "east", "time", "date"] - df_gacos = pd.DataFrame(gacos, columns=cols).drop_duplicates(subset="url") - - # save to file - try: - df_gacos.to_csv(output_file) - print(f"Save gacos urls to {output_file}") - except Exception as e: - self.df_gacos = df_gacos - print(e) - print("Save gacos urls failed") - print("You can access the gacos urls by `df_gacos` attribute.") - - -def in_date_range(date, start_date, end_date, date_args={}): - start_date = pd.to_datetime(start_date, **date_args) - end_date = pd.to_datetime(end_date, **date_args) - start_none = start_date is None or pd.isna(start_date) - end_none = end_date is None or pd.isna(end_date) - if start_none and end_none: - return True - elif start_none: - return date <= end_date - elif end_none: - return date >= start_date - else: - return (date >= start_date) and (date <= end_date) - - -def decodeBody(msgPart: email.message.Message): - """decode email body - - Parameters - ---------- - msgPart : email.message.Message - The email message object. - """ - contentType = msgPart.get_content_type() - textContent = "" - if contentType == "text/plain" or contentType == "text/html": - content = msgPart.get_payload(decode=True) - charset = msgPart.get_charset() - if charset is None: - contentType = msgPart.get("Content-Type", "").lower() - position = contentType.find("charset=") - if position >= 0: - charset = contentType[position + 8 :].strip() - if charset: - textContent = content.decode(charset) - return textContent - - -def get_content(messageObject): - msgBodyContents = [] - if messageObject.is_multipart(): # parse multipart email - messageParts = messageObject.get_payload() - for messagePart in messageParts: - bodyContent = decodeBody(messagePart) - if bodyContent: - msgBodyContents.append(bodyContent) - else: - bodyContent = decodeBody(messageObject) - if bodyContent: - msgBodyContents.append(bodyContent) - return msgBodyContents - - -def login_in_email_pop3(username, password, host, port, ssl=False): - try: - if username is None: - username = input("username: ") - if password is None: - password = getpass.getpass("password: ") - - if ssl: - if port is None: - port = 995 - server = poplib.POP3_SSL(host, port) - else: - if port is None: - port = 110 - server = poplib.POP3(host, port) - - server.user(username) - server.pass_(password) - return server - except Exception as e: - print(e) - print("login failed") - - -def login_in_email_imap(username, password, host, port, ssl=False): - try: - if username is None: - username = input("username: ") - if password is None: - password = getpass.getpass("password: ") - - if ssl: - if port is None: - port = 993 - else: - if port is None: - port = 143 - - # 强制 IPv4: monkey-patch getaddrinfo，避免 IPv6 不可达 - import socket as _socket - _orig_getaddrinfo = _socket.getaddrinfo - def _ipv4_only_getaddrinfo(*args, **kwargs): - return _orig_getaddrinfo(args[0], args[1], _socket.AF_INET, - *args[3:], **kwargs) - _socket.getaddrinfo = _ipv4_only_getaddrinfo - try: - if ssl: - server = imaplib.IMAP4_SSL(host, port) - else: - server = imaplib.IMAP4(host, port) - finally: - _socket.getaddrinfo = _orig_getaddrinfo - - server.login(username, password) - - # 163/126 等网易邮箱要求登录后发送 ID 命令才能执行 SELECT - if '163.com' in host or '126.com' in host or 'yeah.net' in host: - try: - tag = server._new_tag() - server.send(tag + b' ID ("name" "pyint" "version" "1.0" ' - b'"vendor" "pyint")\r\n') - while True: - resp = server.readline() - if resp.startswith(tag): - break - except Exception: - pass - - return server - except Exception as e: - print(f"IMAP login failed: {e}") - return None - - -def parse_gacos_info(msgBodyContents, gacos_suffix="tar.gz"): - """Parse gacos info from email body. - - Parameters - ---------- - msgBodyContents : list - The email body contents. - gacos_suffix : str, optional - The suffix of the gacos file url. Default is "tar.gz". The suffix is used - to filter urls in the email. This parameter is used to avoid the - situation that the email contains other urls. - """ - - url, south, north, west, east, _time, date_list = ( - None, - None, - None, - None, - None, - None, - None, - ) - date_list = [] - for contents in msgBodyContents: - lines = [i.strip() for i in contents.split("\n") if i] - for line in lines: - line = line.strip() - loc = line.split("=") - if len(loc) == 2: - parameter, value = loc - parameter, value = (parameter.strip(), value.strip()) - if "MinLat" == parameter: - south = float(value) - if "MaxLat" == parameter: - north = float(value) - if "MinLon" == parameter: - west = float(value) - if "MaxLon" == parameter: - east = float(value) - loc = line.split(":") - if len(loc) == 2: - parameter, value = loc - parameter, value = (parameter.strip(), value.strip()) - if "Time" == parameter: - _time = float(value) - - if len(line) == 8 and line.isdigit(): - date_list.append(line) - - for i in ["http", "ftp", "https"]: - result = re.search(f"\({i}.*{gacos_suffix}\)", line) - if result: - url = result.group()[1:-1] - break - - if url == south == north == west == east == _time: - return None - else: - return url, south, north, west, east, _time, date_list diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/gacos/submit.py b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/gacos/submit.py deleted file mode 100644 index ce1d235..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/gacos/submit.py +++ /dev/null @@ -1,83 +0,0 @@ -import time -from pathlib import Path -from typing import Optional, Union - -import numpy as np -import requests -from tqdm.auto import tqdm - -from .datasets import SarDataset - - -class Submitter: - def __init__( - self, - dataset: SarDataset, - email: str, - sleep_time_range: tuple[int, int] = (60 / 2, 60 * 5), - gacos_url="http://www.gacos.net/M/action_page.php", - ) -> None: - """Initialize Submitter class - - Parameters - ---------- - dataset : SarDataset - The SarDataset object. - email : str - The email address to submit to gacos. - sleep_time_range : tuple[int, int], optional - The range of sleep time in seconds. Default is (60, 60 * 5). - gacos_url : str, optional - The url of gacos website. Default is "http://www.gacos.net/M/action_page.php". - """ - self.dataset = dataset - self.email = email - self.sleep_time_range = sleep_time_range - self.gacos_url = gacos_url - - self._failed = [] - self._succeed = [] - - def _post_data(self, data): - """Post data to gacos website.""" - r = requests.post(self.gacos_url, data=data) - return "Thanks for using GACOS!" in r.text - - def post_requests(self): - # post gacos info to website - datetime_patches = self.dataset.gen_datetime_patches() - for _key, _dates in tqdm( - datetime_patches.items(), - desc="submitting times", - unit="times", - ): - try: - for _dt in tqdm(_dates, desc="submitting dates", unit="dates"): - post_data = self.dataset.gen_post_data( - _dt, _key.split(":"), self.email - ) - status_ok = self._post_data(post_data) - if status_ok: - self._succeed.append(post_data) - tqdm.write(f">>> succeed post: {post_data}") - else: - self._failed.append(post_data) - tqdm.write(f">>> failed post: {post_data}") - - # wait to avoid be rejected - sleep_time = np.random.randint(*self.sleep_time_range) - tqdm.write(f" sleeping for {sleep_time} seconds...") - time.sleep(sleep_time) - except: - self._failed.append(post_data) - tqdm.write(f">>> failed post: {post_data}") - - @property - def failed(self): - """A list of failed post data.""" - return self._failed - - @property - def succeed(self): - """A list of succeed post data.""" - return self._succeed diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/pyproject.toml b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/pyproject.toml deleted file mode 100644 index c1a0f62..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/pyproject.toml +++ /dev/null @@ -1,23 +0,0 @@ -[project] -name = "AutoGACOS" -authors = [{name = "Fancy", email = "fanchengyan@outlook.com"}] -description = "A Python library for automatically submitting and downloading GACOS data" -version = "0.1.0" -requires-python = ">=3.6" -dependencies = [ - "data_downloader", - "faninsar", -] -readme = "README.md" -license = {file = "LICENSE"} -keywords = ["GACOS", "data", "downloader"] - - -[project.optional-dependencies] -dev = [ - "pytest" -] - -[project.urls] -Homepage = "https://github.com/Fanchengyan/AutoGACOS" -Repository = "https://github.com/Fanchengyan/AutoGACOS" diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/DEM_DOWNLOAD_GUIDE.md b/.codex_tmp/pyint_variants/no_rescue/pyint/DEM_DOWNLOAD_GUIDE.md deleted file mode 100644 index ecabdcf..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/DEM_DOWNLOAD_GUIDE.md +++ /dev/null @@ -1,265 +0,0 @@ -# DEM 数据下载指南 - -本文档介绍如何使用 `makedem.py` 下载和处理不同来源的 DEM 数据。 - -## 支持的 DEM 数据源 - -### 1. Copernicus DEM (默认推荐) - -**特点:** -- 分辨率: 30米 -- 覆盖: 全球 (60°S - 80°N) -- 数据质量: 高精度,全球一致性 -- 下载方式: 自动从 AWS/OpenTopography 下载 -- 费用: 免费 - -**使用方法:** -```bash -# 基本使用 -python makedem.py -r 116/117/39/40 --dem-source copernicus - -# 指定并行下载数量(加速下载) -python makedem.py -r 116/117/39/40 --dem-source copernicus --num-workers 8 - -# 指定输出目录 -python makedem.py -r 116/117/39/40 --dem-source copernicus --dir /path/to/output -``` - -**优点:** -- ✓ 全自动下载 -- ✓ 高精度数据 -- ✓ 全球覆盖 -- ✓ 支持并行下载加速 - -**缺点:** -- ✗ 需要网络连接 -- ✗ 下载时间取决于网络速度 - ---- - -### 2. NASADEM - -**特点:** -- 分辨率: 30米 -- 覆盖: 全球 (60°S - 60°N) -- 数据质量: SRTM + ICESat 数据融合 -- 下载方式: 通过 NASADEM Python 库自动下载 -- 费用: 免费 - -**依赖安装:** -```bash -pip install nasadem rasters -``` - -**使用方法:** -```bash -# 基本使用 -python makedem.py -r 116/117/39/40 --dem-source nasadem - -# 指定输出目录 -python makedem.py -r 116/117/39/40 --dem-source nasadem --dir /path/to/output -``` - -**优点:** -- ✓ 全自动下载 -- ✓ NASA官方数据 -- ✓ 融合多源数据,精度提升 - -**缺点:** -- ✗ 需要网络连接 -- ✗ 首次下载可能较慢 -- ✗ 需要额外安装库 -- ✗ 覆盖范围小于 Copernicus (南北纬60°限制) - ---- - -### 3. SRTM (需要预下载数据) - -**特点:** -- 分辨率: 30米 (SRTM 1 arc-second) 或 90米 (SRTM 3 arc-second) -- 覆盖: 全球 (60°S - 60°N) -- 数据质量: 良好 -- 下载方式: 需要手动预下载 .hgt 文件 -- 费用: 免费 - -**依赖安装:** -```bash -# 注意: srtm库需要 Python >= 3.12 -pip install srtm - -# 检查Python版本 -python --version # 应该显示 3.12 或更高版本 -``` - -**Python版本要求:** -- srtm 库要求 Python >= 3.12 -- 如果您的Python版本 < 3.12，建议使用 Copernicus 或 NASADEM - -**数据准备步骤:** - -1. **下载 SRTM .hgt 文件** - - 从以下网站之一下载所需区域的 .hgt 或 .hgt.zip 文件: - - **CSI-CGIAR SRTM**: https://srtm.csi.cgiar.org/ (推荐,无需注册) - - **USGS EarthExplorer**: https://earthexplorer.usgs.gov/ (需要注册) - - **NASA EarthData**: https://urs.earthdata.nasa.gov/ (需要注册) - -2. **文件命名规则** - - SRTM 文件按 1°×1° 分块命名: - - `N39E116.hgt` → 北纬39-40°, 东经116-117° - - `N39W110.hgt` → 北纬39-40°, 西经110-109° - - `S10E120.hgt` → 南纬10-9°, 东经120-121° - -3. **数据组织** - - 将所有 .hgt 文件放在同一目录下,例如: - ``` - /path/to/srtm_data/ - ├── N39E116.hgt - ├── N39E117.hgt - ├── N40E116.hgt - └── N40E117.hgt - ``` - -**使用方法:** -```bash -# 指定SRTM数据目录 -python makedem.py -r 116/117/39/40 --dem-source srtm --srtm-data-dir /path/to/srtm_data - -# 注意:如果不指定 --srtm-data-dir,程序会报错并提示 -python makedem.py -r 116/117/39/40 --dem-source srtm -# 错误: 使用 SRTM 数据源需要指定 --srtm-data-dir 参数 -``` - -**优点:** -- ✓ 离线使用(数据预下载后) -- ✓ 不依赖实时网络下载 -- ✓ 数据源多样化 - -**缺点:** -- ✗ 需要手动下载数据文件 -- ✗ 需要自行管理数据文件 -- ✗ 需要确保覆盖目标区域的所有分块 -- ✗ srtm 库对 .hgt.zip 压缩文件支持不稳定,建议解压后使用 - ---- - -## 使用建议 - -### 推荐使用顺序: - -1. **首选: Copernicus DEM** - - 全自动,高精度,全球覆盖 - - 使用 `--num-workers 8` 加速下载 - -2. **备选: NASADEM** - - 全自动,NASA官方数据 - - 适合北纬60°以南,南纬60°以北区域 - -3. **离线环境: SRTM** - - 仅在无网络环境或已有SRTM数据时使用 - - 需要提前准备数据文件 - -### 性能对比: - -| DEM源 | 下载速度 | 数据精度 | 全球覆盖 | 离线使用 | 推荐指数 | -|-------|---------|---------|---------|---------|---------| -| Copernicus | ★★★★☆ | ★★★★★ | ★★★★★ | ✗ | ★★★★★ | -| NASADEM | ★★★☆☆ | ★★★★☆ | ★★★☆☆ | ✗ | ★★★★☆ | -| SRTM | N/A (离线) | ★★★★☆ | ★★★☆☆ | ✓ | ★★★☆☆ | - -### 常见问题: - -**Q: 如何确定需要下载哪些 SRTM 文件?** - -A: 根据目标区域范围计算所需文件: -```python -# 示例: 区域 116/117/39/40 (东经116-117°, 北纬39-40°) -west, east, south, north = 116, 117, 39, 40 - -# 需要下载的文件: -# N39E116.hgt (39-40°N, 116-117°E) -# 如果区域更大,需要下载多个文件 -``` - -**Q: SRTM .hgt.zip 文件可以直接使用吗?** - -A: srtm 库对压缩文件支持不稳定,建议先解压: -```bash -cd /path/to/srtm_data/ -unzip "*.hgt.zip" -``` - -**Q: 如何加速 Copernicus DEM 下载?** - -A: 增加并行下载数量: -```bash -# 默认使用 4 个线程 -python makedem.py -r 116/117/39/40 --dem-source copernicus --num-workers 4 - -# 加速到 8-16 个线程(根据网络情况调整) -python makedem.py -r 116/117/39/40 --dem-source copernicus --num-workers 16 -``` - -**Q: 下载的数据格式是什么?** - -A: 所有DEM源最终都会转换为 GAMMA/ROI_PAC 格式: -- `.dem` 文件: 二进制高程数据 (big-endian for GAMMA) -- `.dem.par` 文件: 参数文件 - ---- - -## 示例工作流程 - -### 场景1: 处理Sentinel-1数据(推荐Copernicus) - -```bash -# 1. 确定研究区域范围 -# 使用 SLC 参数文件自动确定 -python makedem.py -s /path/to/slc.par --dem-source copernicus --num-workers 8 - -# 或手动指定区域 -python makedem.py -r 116/117/39/40 --dem-source copernicus --num-workers 8 -``` - -### 场景2: 离线处理(使用SRTM) - -```bash -# 1. 提前下载SRTM数据 -# 从 https://srtm.csi.cgiar.org/ 下载 N39E116.hgt, N39E117.hgt 等 - -# 2. 组织数据 -mkdir -p ~/data/SRTM -mv N*.hgt ~/data/SRTM/ - -# 3. 使用预下载数据生成DEM -python makedem.py -r 116/117/39/40 --dem-source srtm --srtm-data-dir ~/data/SRTM -``` - -### 场景3: NASADEM对比测试 - -```bash -# 下载Copernicus DEM -python makedem.py -r 116/117/39/40 --dem-source copernicus --dir ./copernicus - -# 下载NASADEM -python makedem.py -r 116/117/39/40 --dem-source nasadem --dir ./nasadem - -# 对比两个DEM文件 -# 使用 GMT 或其他工具进行可视化和分析 -``` - ---- - -## 参考资料 - -- **Copernicus DEM**: https://copernicus-dem-90m.s3.amazonaws.com/readme.html -- **NASADEM**: https://github.com/DFS-iData/NASADEM -- **SRTM CSI-CGIAR**: https://srtm.csi.cgiar.org/ -- **GAMMA Software**: https://gamma-rs.ch/ - ---- - -**最后更新**: 2026-03-13 -**作者**: iFlow CLI diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ALOS.py b/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ALOS.py deleted file mode 100644 index f90aadf..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ALOS.py +++ /dev/null @@ -1,193 +0,0 @@ -#! /usr/bin/env python -#''' -################################################################################## -# # -# Author: Yun-Meng Cao # -# Email : ymcmrs@gmail.com # -# Date : June, 2019 # -# # -# Generate SLC from SAR_IMS_P1 data # -# # -################################################################################## -#''' -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob -from pyint import _utils as ut - -def check_variable_name(path): - s=path.split("/")[0] - if len(s)>0 and s[0]=="$": - p0=os.getenv(s[1:]) - path=path.replace(path.split("/")[0],p0) - return path - -def read_template(File, delimiter='='): - '''Reads the template file into a python dictionary structure. - Input : string, full path to the template file - Output: dictionary, pysar template content - Example: - tmpl = read_template(KyushuT424F610_640AlosA.template) - tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') - ''' - template_dict = {} - for line in open(File): - line = line.strip() - c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter - if len(c) < 2 or line.startswith('%') or line.startswith('#'): - next #ignore commented lines or those without variables - else: - atrName = c[0] - atrValue = str.replace(c[1],'\n','').split("#")[0].strip() - atrValue = check_variable_name(atrValue) - template_dict[atrName] = atrValue - return template_dict - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close - -def rm(TXT): - call_str = 'rm ' + TXT - os.system(call_str) - -def usage(): - print(''' -****************************************************************************************************** - - Generate SLC and SLC_par file for ERS/ENVISAT (SAR_IMS_1P format) - - usage: - - Down2SLC_ERS.py ProjectName DownName - - e.g. Down2SLC_ERS.py CotopaxiT120ERSA 910101 - -******************************************************************************************************* - ''') - -def main(argv): - - if len(sys.argv)==3: - projectName = sys.argv[1] - Date = sys.argv[2] - else: - usage();sys.exit(1) - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - - projectDir = scratchDir + '/' + projectName - downDir = scratchDir + '/' + projectName + "/DOWNLOAD" - slcDir = scratchDir + '/' + projectName + '/SLC' - workflow = downDir + '/' + Date - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - if not os.path.isdir(slcDir): - call_str= 'mkdir ' +slcDir - os.system(call_str) - os.chdir(workflow) - tempfile=workflow + '/' + 'workreport' - if os.path.isfile(tempfile): - tempDict=ut.update_template(tempfile) - CEOS_SAR_leader = tempDict['Pdi_L10ProductFileName02'].split('"')[1] - CEOS_raw_data = tempDict['Pdi_L10ProductFileName03'].split('"')[1] - Date0 = Date -# if len(Date)==6: -# Date0 = Date -# elif len(Date)==8: -# Date0 = Date[2:8] - #Date0=Date -# else: -# print('The input Date is invalid.') -# sys.exit(1) - - #FileDir = downDir + '/' + downName - SAR_par='palsar_' +Date0 + '.par' - PROC_par='p' + Date0 + '.slc.par' - raw_out=Date0 + '.raw' - plot_data=Date0 +'.mlcc' - rspec_data=Date0 +'.rspec' - doppler_data=Date0 + '.dop' - rc_data=Date0 + '.rc' - autof_data=Date0 +'.autof' - slc_data=Date0 +'.slc' - slc_par=Date0 +'.slc.par' - call_str = 'PALSAR_proc ' + CEOS_SAR_leader + ' ' + SAR_par +' '+ PROC_par + ' '+ CEOS_raw_data + ' ' + raw_out + ' 0 0' - os.system(call_str) - palsar_ant_data='/home/chen/Software/InSAR/GAMMA/GAMMA_SOFTWARE-20180704/MSP/sensors/palsar_ant_20061024.dat' - call_str = 'cp ' + palsar_ant_data + ' .' - os.system(call_str) - call_str = 'PALSAR_antpat' + ' ' + SAR_par +' '+ PROC_par + ' ' + palsar_ant_data + ' palsar_antpat_msp.dat' - os.system(call_str) - call_str = 'doppler ' + SAR_par +' '+ PROC_par + ' ' + raw_out + ' ' + plot_data - os.system(call_str) - call_str = 'doppler ' + SAR_par +' '+ PROC_par + ' ' + raw_out + ' ' + doppler_data - os.system(call_str) - call_str = 'rspec_IQ ' + SAR_par +' '+ PROC_par + ' ' + raw_out + ' ' + rspec_data - os.system(call_str) - #call_str = 'rspec_JERS ' + SAR_par +' '+ PROC_par + ' ' + CEOS_raw_data + ' ' + rspec_data + ' - - - - - -' - #os.system(call_str) - call_str = 'pre_rc ' + SAR_par +' '+ PROC_par + ' ' + raw_out + ' ' + rc_data - os.system(call_str) - call_str = 'autof ' + SAR_par +' '+ PROC_par + ' ' + rc_data + ' ' + autof_data +' 5.0' - os.system(call_str) - call_str = 'autof ' + SAR_par +' '+ PROC_par + ' ' + rc_data + ' ' + autof_data +' 5.0' - os.system(call_str) - call_str = 'az_proc ' + SAR_par +' '+ PROC_par + ' ' + rc_data + ' ' + slc_data +' 16384' - os.system(call_str) - call_str = 'par_MSP ' + SAR_par +' '+ PROC_par + ' ' + slc_par - os.system(call_str) - else: - print('No data is found for date:' + Date) - sys.exit(1) - - call_str ="rename 's/VV.SLC/slc/g' *" - os.system(call_str) - - Date0 = Date - - dataDir = slcDir + '/' + Date0 - if not os.path.isdir(dataDir): - call_str = 'mkdir ' + dataDir - print('Generate SLC dir for date: ' + Date0) - os.system(call_str) - call_str = 'mv ' + Date0 + '.slc* ' + dataDir - os.system(call_str) - print("Down to SLC for %s is done! " % Date) - SslcImg = dataDir + '/'+ Date0 + '.slc' - SslcPar = dataDir + '/'+ Date0 + '.slc.par' - - SamprlksImg = dataDir + '/'+ Date0 + '_' + rlks + 'rlks' + '.amp' - SamprlksPar = dataDir + '/'+ Date0 + '_' + rlks + 'rlks' + '.amp.par' - call_str = 'multi_look ' + SslcImg + ' ' + SslcPar + ' ' + SamprlksImg + ' ' + SamprlksPar + ' ' + rlks + ' ' + azlks - os.system(call_str) - - nWidth = ut.read_gamma_par(SamprlksPar, 'read', 'range_samples') - call_str = 'raspwr ' + SamprlksImg + ' ' + str(nWidth) - os.system(call_str) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ASAR_Cat.py b/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ASAR_Cat.py deleted file mode 100644 index 5594a52..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ASAR_Cat.py +++ /dev/null @@ -1,307 +0,0 @@ -#! /usr/bin/env python -#''' -################################################################################## -# # -# Author: Yun-Meng Cao # -# Email : ymcmrs@gmail.com # -# Date : June, 2019 # -# # -# Generate SLC from SAR_IMS_P1 data # -# # -################################################################################## -#''' -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob - -def check_variable_name(path): - s=path.split("/")[0] - if len(s)>0 and s[0]=="$": - p0=os.getenv(s[1:]) - path=path.replace(path.split("/")[0],p0) - return path - -def read_template(File, delimiter='='): - '''Reads the template file into a python dictionary structure. - Input : string, full path to the template file - Output: dictionary, pysar template content - Example: - tmpl = read_template(KyushuT424F610_640AlosA.template) - tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') - ''' - template_dict = {} - for line in open(File): - line = line.strip() - c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter - if len(c) < 2 or line.startswith('%') or line.startswith('#'): - next #ignore commented lines or those without variables - else: - atrName = c[0] - atrValue = str.replace(c[1],'\n','').split("#")[0].strip() - atrValue = check_variable_name(atrValue) - template_dict[atrName] = atrValue - return template_dict - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close - -def rm(TXT): - call_str = 'rm ' + TXT - os.system(call_str) - -def usage(): - print(''' -****************************************************************************************************** - - Generate SLC and SLC_par file for ERS/ENVISAT (SAR_IMS_1P format) - - usage: - - Down2SLC_ASAR_Cat.py ProjectName DownName - - e.g. Down2SLC_ASAR_Cat.py CotopaxiT120ERSA 910101 - -******************************************************************************************************* - ''') - -def main(argv): - - if len(sys.argv)==3: - projectName = sys.argv[1] - Date = sys.argv[2] - else: - usage();sys.exit(1) - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + '/' + projectName + '.template' - templateContents=read_template(templateFile) - - if 'rlks4cor' in templateContents: rlks4cor = templateContents['rlks4cor'] - else: rlks4cor = '4' - if 'azlks4cor' in templateContents: azlks4cor = templateContents['azlks4cor'] - else: azlks4cor = '4' - - if 'rwin4cor' in templateContents: rwin4cor = templateContents['rwin4cor'] - else: rwin4cor = '128' - if 'azwin4cor' in templateContents: azwin4cor = templateContents['azwin4cor'] - else: azwin4cor = '128' - if 'rsample4cor' in templateContents: rsample4cor = templateContents['rsample4cor'] - else: rsample4cor = '64' - if 'azsample4cor' in templateContents: azsample4cor = templateContents['azsample4cor'] - else: azsample4cor = '64' - - if ' rpos4cor' in templateContents: rpos4cor = templateContents[' rpos4cor'] - else: rpos4cor = ' - ' - if 'azpos4cor' in templateContents: azpos4cor = templateContents['azpos4cor'] - else: azpos4cor = ' - ' - - - if 'rfwin4cor' in templateContents: rfwin4cor = templateContents['rfwin4cor'] - else: rfwin4cor = str(int(int(rwin4cor)/2)) - if 'azfwin4cor' in templateContents: azfwin4cor = templateContents['azfwin4cor'] - else: azfwin4cor = str(int(int(azwin4cor)/2)) - if 'rfsample4cor' in templateContents: rfsample4cor = templateContents['rfsample4cor'] - else: rfsample4cor = str(2*int(rsample4cor)) - if 'azfsample4cor' in templateContents: azfsample4cor = templateContents['azfsample4cor'] - else: azfsample4cor = str(2*int(azsample4cor)) - - if 'thresh4cor' in templateContents: thresh4cor = templateContents['thresh4cor'] - else: thresh4cor = ' 0.2 ' - - rlks = templateContents['Range_Looks'] - azlks = templateContents['Azimuth_Looks'] - - - projectDir = scratchDir + '/' + projectName - downDir = scratchDir + '/' + projectName + "/DOWNLOAD" - slcDir = scratchDir + '/' + projectName + '/SLC' - - if not os.path.isdir(slcDir): - call_str= 'mkdir ' +slcDir - os.system(call_str) - os.chdir(downDir) - - t0 = 't0_' + Date - call_str = 'ls >' + t0 - os.system(call_str) - - tt = 'tt_' + Date - call_str = "grep " + Date + ' ' + t0 + '> ' + tt - os.system(call_str) - - te = 'te_' + Date - call_str = "grep ASA_IMS_1P " + tt + " > " + te - os.system(call_str) - - AA= np.loadtxt(te,dtype=np.str) - Na = AA.size - AA=AA.reshape(Na,) - - for i in range(Na): - Date0 = Date+'_' + str(i) - downName = str(AA[i]) - FileDir = downDir + '/' + downName - call_str = 'par_ASAR '+ FileDir + ' ' + Date0 - os.system(call_str) - - call_str ="rename 's/VV.SLC/slc/g' *" - os.system(call_str) - - slcpar = Date0 + '.slc.par' - call_str = 'ASAR_orb_cor_par.py ' + slcpar - os.system(call_str) - - Date0 = Date - if len(Date)==6: - Date0 = Date - Date6 = Date - elif len(Date)==8: - Date0 = Date[2:8] - Date6 = Date[2:8] - else: - print('The input Date is invalid.') - sys.exit(1) - - dataDir = slcDir + '/' + Date0 - if not os.path.isdir(dataDir): - call_str = 'mkdir ' + dataDir - print('Generate SLC dir for date: ' + Date0) - os.system(call_str) - call_str = 'mv ' + Date + '*.slc* ' + dataDir - os.system(call_str) - - os.chdir(dataDir) - for i in range(Na): - Date0 = Date+'_' + str(i) - downName = str(AA[i]) - - SLCm = Date0 + '.slc' - SLCm_par = Date0 + '.slc.par' - MamprlksImg = Date0 + '.amp' - MamprlksPar = Date0 + '.amp.par' - - call_str = 'multi_look ' + SLCm + ' ' + SLCm_par + ' ' + MamprlksImg + ' ' + MamprlksPar + ' ' + rlks + ' ' + azlks - os.system(call_str) - nWidth = UseGamma(MamprlksPar, 'read', 'range_samples') - call_str = 'raspwr ' + MamprlksImg + ' ' + nWidth - os.system(call_str) - - SLC = Date6+'.slc' - SLCPAR = Date6 + '.slc.par' - AMP = Date6+'.amp' - AMPPAR = Date6+'.amp.par' - if Na==1: - os.rename(SLCm,SLC) - os.rename(SLCm_par,SLCPAR) - os.rename(MamprlksImg,AMP) - os.rename(MamprlksPar,AMPPAR) - - - for i in range(Na-1): - if i==0: - DateA = Date + '_0' - else: - DateA = Date + '_' + str(i-1) + str(i) - - SLCA = DateA + '.slc' - SLCA_par = DateA + '.slc.par' - - - DateB = Date + '_' +str(i+1) - SLCB = DateB + '.slc' - SLCB_par = DateB + '.slc.par' - - DateC = Date + '_' + str(i) + str(i+1) - SLCC = DateC + '.slc' - SLCC_par = DateC + '.slc.par' - - ##################################################################################### - MamprlksImg = Date + '.amp' - MamprlksPar = Date + '.amp.par' - - off = DateC + '.off' - offs = DateC + '.offs' - offsets = DateC + '.offsets' - coffs = DateC + '.coffs' - coffsets = DateC + '.coffsets' - snr = DateC + '.snr' - off_std = DateC + '.off_std' - - ########################## Generate off file ############################# - - call_str = "create_offset " + SLCA_par + " " + SLCB_par + " " + off + " 1 - - 0" - os.system(call_str) - call_str = 'init_offset_orbit '+ SLCA_par + " " + SLCB_par + ' ' + off - os.system(call_str) - - - call_str = 'init_offset '+ SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' ' + rlks4cor + ' ' + azlks4cor + ' ' + rpos4cor + ' ' + azpos4cor - os.system(call_str) - - call_str = 'init_offset '+ SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' 1 1 - - ' - os.system(call_str) - - call_str = "offset_pwr " + SLCA + " " + SLCB + " " + SLCA_par + " " + SLCB_par + " " + off + " " + offs + " " + snr + " " + rwin4cor + " " + azwin4cor + " " + offsets + " 2 "+ rsample4cor + " " + azsample4cor - os.system(call_str) - - call_str = "offset_fit " + offs + " " + snr + " " + off + " " + coffs + " " + coffsets + " " + thresh4cor +" 3" - os.system(call_str) - - call_str = "offset_pwr " +SLCA + " " + SLCB + " " + SLCA_par + " " + SLCB_par + " " + off + " " + offs + " " + snr + " " + rfwin4cor + " " + azfwin4cor + " " + offsets + " 2 " + rfsample4cor + " " + azfsample4cor - os.system(call_str) - - call_str = "offset_fit " + offs + " " + snr + " " + off + " " + coffs + " " + coffsets + " " + thresh4cor + " 4 >" + off_std - os.system(call_str) - - ######################################################################################## - - call_str = 'SLC_cat ' + SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' ' + SLCC + ' ' + SLCC_par - os.system(call_str) - - if i==(Na-2): - if len(Date)==6: - DD = Date - else: - DD = Date[2:8] - SLCm = DD + '.slc' - SLCm_par = DD + '.slc.par' - - - call_str = 'cp ' + SLCC + ' ' + SLCm - os.system(call_str) - - call_str = 'cp ' + SLCC_par + ' ' + SLCm_par - os.system(call_str) - - call_str = 'multi_look ' + SLCm + ' ' + SLCm_par + ' ' + MamprlksImg + ' ' + MamprlksPar + ' ' + rlks + ' ' + azlks - os.system(call_str) - nWidth = UseGamma(MamprlksPar, 'read', 'range_samples') - call_str = 'raspwr ' + MamprlksImg + ' ' + nWidth - os.system(call_str) - - call_str = 'rm *.amp' - os.system(call_str) - - call_str = 'rm *_*.slc' - os.system(call_str) - - print("Down to SLC for %s is done! " % Date) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ASAR_Cat_All.py b/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ASAR_Cat_All.py deleted file mode 100644 index cd7c04a..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ASAR_Cat_All.py +++ /dev/null @@ -1,136 +0,0 @@ -#! /usr/bin/env python -#''' -################################################################################## -# # -# Author: Yun-Meng Cao # -# Email : ymcmrs@gmail.com # -# Date : March, 2017 # -# # -# Generate Sentinel SLC from the downloaded data # -# # -################################################################################## -#''' -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob - -def check_variable_name(path): - s=path.split("/")[0] - if len(s)>0 and s[0]=="$": - p0=os.getenv(s[1:]) - path=path.replace(path.split("/")[0],p0) - return path - -def read_template(File, delimiter='='): - '''Reads the template file into a python dictionary structure. - Input : string, full path to the template file - Output: dictionary, pysar template content - Example: - tmpl = read_template(KyushuT424F610_640AlosA.template) - tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') - ''' - template_dict = {} - for line in open(File): - line = line.strip() - c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter - if len(c) < 2 or line.startswith('%') or line.startswith('#'): - next #ignore commented lines or those without variables - else: - atrName = c[0] - atrValue = str.replace(c[1],'\n','').split("#")[0].strip() - atrValue = check_variable_name(atrValue) - template_dict[atrName] = atrValue - return template_dict - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close - -def rm(TXT): - call_str = 'rm ' + TXT - os.system(call_str) - -def usage(): - print(''' -****************************************************************************************************** - - Downloading Sentinel-1A/B data based on ssara - - usage: - - Down2SLC_ERS_Cat_All.py ProjectName - - e.g. Down2SLC_ERS_Cat_All.py CotopaxiT120ERSA - -******************************************************************************************************* - ''') - -def main(argv): - - if len(sys.argv)==2: - projectName = sys.argv[1] - else: - usage();sys.exit(1) - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - - projectDir = scratchDir + '/' + projectName - downDir = scratchDir + '/' + projectName + "/DOWNLOAD" - slcDir = scratchDir + '/' + projectName + '/SLC' - - if not os.path.isdir(slcDir): - call_str= 'mkdir ' +slcDir - os.system(call_str) - - os.chdir(downDir) - - call_str = 'ls > tt0' - os.system(call_str) - - call_str = 'grep ASA_IMS_1P tt0 >tt1' - os.system(call_str) - - call_str = "awk -F_ '{print $3}' tt1 > tt2 " - os.system(call_str) - - call_str = "awk -FSA '{print $2}' tt2 > ttt" - os.system(call_str) - - call_str = 'sort ttt | uniq > ttm' - os.system(call_str) - - AA= np.loadtxt('ttm',dtype=np.str) - Na = AA.size - - for i in range(Na): - call_str = 'Down2SLC_ASAR_Cat.py ' + projectName + ' ' + AA[i] - print(call_str) - call_str = 'Down2SLC_ASAR_Cat.py ' + projectName + ' ' + AA[i] + ' >/dev/null' - os.system(call_str) - - - print("Down to SLC for %s is done! " % projectName) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ERS.py b/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ERS.py deleted file mode 100644 index 868a153..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ERS.py +++ /dev/null @@ -1,155 +0,0 @@ -#! /usr/bin/env python -#''' -################################################################################## -# # -# Author: Yun-Meng Cao # -# Email : ymcmrs@gmail.com # -# Date : June, 2019 # -# # -# Generate SLC from SAR_IMS_P1 data # -# # -################################################################################## -#''' -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob - -def check_variable_name(path): - s=path.split("/")[0] - if len(s)>0 and s[0]=="$": - p0=os.getenv(s[1:]) - path=path.replace(path.split("/")[0],p0) - return path - -def read_template(File, delimiter='='): - '''Reads the template file into a python dictionary structure. - Input : string, full path to the template file - Output: dictionary, pysar template content - Example: - tmpl = read_template(KyushuT424F610_640AlosA.template) - tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') - ''' - template_dict = {} - for line in open(File): - line = line.strip() - c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter - if len(c) < 2 or line.startswith('%') or line.startswith('#'): - next #ignore commented lines or those without variables - else: - atrName = c[0] - atrValue = str.replace(c[1],'\n','').split("#")[0].strip() - atrValue = check_variable_name(atrValue) - template_dict[atrName] = atrValue - return template_dict - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close - -def rm(TXT): - call_str = 'rm ' + TXT - os.system(call_str) - -def usage(): - print(''' -****************************************************************************************************** - - Generate SLC and SLC_par file for ERS/ENVISAT (SAR_IMS_1P format) - - usage: - - Down2SLC_ERS.py ProjectName DownName - - e.g. Down2SLC_ERS.py CotopaxiT120ERSA 910101 - -******************************************************************************************************* - ''') - -def main(argv): - - if len(sys.argv)==3: - projectName = sys.argv[1] - Date = sys.argv[2] - else: - usage();sys.exit(1) - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - - projectDir = scratchDir + '/' + projectName - downDir = scratchDir + '/' + projectName + "/DOWNLOAD" - slcDir = scratchDir + '/' + projectName + '/SLC' - - if not os.path.isdir(slcDir): - call_str= 'mkdir ' +slcDir - os.system(call_str) - os.chdir(downDir) - - t0 = 't0_' + Date - call_str = 'ls >' + t0 - os.system(call_str) - - tt = 'tt_' + Date - call_str = "grep " + Date + ' ' + t0 + '> ' + tt - os.system(call_str) - - te = 'te_' + Date - call_str = "grep SAR_IMS_1P " + tt + " > " + te - os.system(call_str) - - AA= np.loadtxt(te,dtype=np.str) - Na = AA.size - - if Na > 0: - downName = str(AA[0]) - FileDir = downDir + '/' + downName - call_str = 'par_ASAR '+ FileDir + ' ' + Date - os.system(call_str) - else: - print('No data is found for date:' + Date) - sys.exit(1) - - call_str ="rename 's/VV.SLC/slc/g' *" - os.system(call_str) - - Date0 = Date - if len(Date)==6: - Date0 = Date - elif len(Date)==8: - Date0 = Date[2:8] - else: - print('The input Date is invalid.') - sys.exit(1) - - dataDir = slcDir + '/' + Date0 - if not os.path.isdir(dataDir): - call_str = 'mkdir ' + dataDir - print('Generate SLC dir for date: ' + Date0) - os.system(call_str) - call_str = 'mv ' + Date + '.slc* ' + dataDir - os.system(call_str) - - print("Down to SLC for %s is done! " % Date) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ERS_All.py b/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ERS_All.py deleted file mode 100644 index 2eefb4a..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ERS_All.py +++ /dev/null @@ -1,229 +0,0 @@ -#! /usr/bin/env python -#''' -################################################################################## -# # -# Author: Yun-Meng Cao # -# Email : ymcmrs@gmail.com # -# Date : March, 2017 # -# # -# Generate Sentinel SLC from the downloaded data # -# # -################################################################################## -#''' -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob - -def check_variable_name(path): - s=path.split("/")[0] - if len(s)>0 and s[0]=="$": - p0=os.getenv(s[1:]) - path=path.replace(path.split("/")[0],p0) - return path - -def read_template(File, delimiter='='): - '''Reads the template file into a python dictionary structure. - Input : string, full path to the template file - Output: dictionary, pysar template content - Example: - tmpl = read_template(KyushuT424F610_640AlosA.template) - tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') - ''' - template_dict = {} - for line in open(File): - line = line.strip() - c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter - if len(c) < 2 or line.startswith('%') or line.startswith('#'): - next #ignore commented lines or those without variables - else: - atrName = c[0] - atrValue = str.replace(c[1],'\n','').split("#")[0].strip() - atrValue = check_variable_name(atrValue) - template_dict[atrName] = atrValue - return template_dict - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close - -def rm(TXT): - call_str = 'rm ' + TXT - os.system(call_str) - -def usage(): - print(''' -****************************************************************************************************** - - Downloading Sentinel-1A/B data based on ssara - - usage: - - Down2SLC_Sen_Gamma.py ProjectName DownName - - e.g. Down2SLC_Sen_Gamma.py CotopaxiT120SenVVA 170118 - -******************************************************************************************************* - ''') - -def main(argv): - - if len(sys.argv)==3: - projectName = sys.argv[1] - Date = sys.argv[2] - else: - usage();sys.exit(1) - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - - projectDir = scratchDir + '/' + projectName - downDir = scratchDir + '/' + projectName + "/DOWNLOAD" - slcDir = scratchDir + '/' + projectName + '/SLC' - - if not os.path.isdir(slcDir): - call_str= 'mkdir ' +slcDir - os.system(call_str) - - os.chdir(downDir) - - - t0 = 't0_' + Date - call_str = 'ls >' + t0 - os.system(call_str) - - tt = 'tt_' + Date - call_str = "grep " + Date + ' ' + t0 + '> ' + tt - os.system(call_str) - - ts = 'ts_' + Date - call_str = "grep SAFE " + tt + ' >' + ts - os.system(call_str) - - tz = 'tz_' + Date - call_str = "grep zip " + tt + " > " + tz - os.system(call_str) - - A1= np.loadtxt(ts,dtype=np.str) - Na1 = A1.size - - A2= np.loadtxt(tz,dtype=np.str) - Na2 = A2.size - - rm(t0);rm(tt);rm(ts);rm(tz) - - if Na1 == 0: - if Na2 > 0: - if Na2 == 1: - downName = str(A2) - else: - downName = str(A2[0]) - FileDir = downDir + '/' + downName - RAWNAME = downName.split('.')[0]+'.SAFE' - call_str = 'unzip '+ FileDir - os.system(call_str) - - else: - if Na1 == 1: - RAWNAME = str(A1) - else: - RAWNAME = str(A1[0]) - - print(RAWNAME) - RAWFILEDir = downDir + '/'+str(RAWNAME) - - - Date = RAWNAME[19:25] - DateDir = slcDir + '/'+Date - - if not os.path.isdir(DateDir): - call_str='mkdir '+DateDir - os.system(call_str) - - measureDir = RAWFILEDir + '/measurement' - annotatDir = RAWFILEDir + '/annotation' - calibraDir = RAWFILEDir + '/annotation/calibration' - - MM = glob.glob(measureDir + '/*vv*tiff') -# MEASURE = glob.glob(measureDir + '/*vv*tiff') -# ANNOTAT = glob.glob(annotatDir + '/*vv*xml' ) -# CALIBRA = glob.glob(calibraDir+'/calibration*vv*') -# NOISE = glob.glob(calibraDir+'/noise*vv*') - - SLC_Tab = DateDir + '/' + Date+'_SLC_Tab' - TEST = DateDir + '/' + Date + '.IW1.slc' - - if not os.path.isfile(TEST): - if os.path.isfile(SLC_Tab): - os.remove(SLC_Tab) - for kk in range(len(MM)): - SLC = DateDir + '/' + Date + '.IW' + str(kk+1)+'.slc' - SLCPar = DateDir + '/' + Date + '.IW' + str(kk+1)+'.slc.par' - TOPPar = DateDir + '/' + Date + '.IW' + str(kk+1)+'.slc.TOPS_par' - BURST = DateDir + '/' + Date + '.IW' + str(kk+1)+'.burst.par' - - if os.path.isfile(BURST): - os.remove(BURST) - call_str = 'echo ' + SLC + ' ' + SLCPar + ' ' + TOPPar + ' >> ' + SLC_Tab - os.system(call_str) - - MEASURE = glob.glob(measureDir + '/*iw' + str(kk+1) + '*vv*tiff') - ANNOTAT = glob.glob(annotatDir + '/*iw' + str(kk+1) + '*vv*xml' ) - CALIBRA = glob.glob(calibraDir+'/calibration*'+ 'iw' + str(kk+1) + '*vv*') - NOISE = glob.glob(calibraDir+'/noise*' + 'iw' + str(kk+1) + '*vv*') - - #call_str = 'S1_burstloc ' + ANNOTAT[0] + '> ' +BURST - #os.system(call_str) - - if int(Date) > 180311: - call_str = 'par_S1_SLC ' + MEASURE[0] + ' ' + ANNOTAT[0] + ' ' + CALIBRA[0] + ' - ' + SLCPar + ' ' + SLC + ' ' + TOPPar - else: - call_str = 'par_S1_SLC ' + MEASURE[0] + ' ' + ANNOTAT[0] + ' ' + CALIBRA[0] + ' ' + NOISE[0] + ' ' + SLCPar + ' ' + SLC + ' ' + TOPPar - - os.system(call_str) - - call_str = 'SLC_burst_corners ' + SLCPar + ' ' + TOPPar + ' > ' +BURST - os.system(call_str) - - TSLC = DateDir + '/' + Date + '.slc' - TSLCPar = DateDir + '/' + Date + '.slc.par' - - TMLI = DateDir + '/' + Date + '_20rlks.amp' - TMLIPar = DateDir + '/' + Date + '_20rlks.amp.par' - - call_str = 'SLC_mosaic_S1_TOPS ' + SLC_Tab + ' ' + TSLC + ' ' + TSLCPar + ' 10 2' - os.system(call_str) - - call_str = 'multi_look ' + TSLC + ' ' + TSLCPar + ' ' + TMLI + ' ' + TMLIPar + ' 20 4' - os.system(call_str) - - nWidth = UseGamma(TMLIPar, 'read','range_samples:') - call_str = 'raspwr ' + TMLI + ' ' + nWidth + ' - - - - - - - ' - os.system(call_str) - - call_str = 'rm -rf ' + RAWFILEDir - os.system(call_str) - - print("Down to SLC for %s is done! " % Date) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ERS_Cat.py b/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ERS_Cat.py deleted file mode 100644 index 1b32f36..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ERS_Cat.py +++ /dev/null @@ -1,306 +0,0 @@ -#! /usr/bin/env python -#''' -################################################################################## -# # -# Author: Yun-Meng Cao # -# Email : ymcmrs@gmail.com # -# Date : June, 2019 # -# # -# Generate SLC from SAR_IMS_P1 data # -# # -################################################################################## -#''' -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob - -def check_variable_name(path): - s=path.split("/")[0] - if len(s)>0 and s[0]=="$": - p0=os.getenv(s[1:]) - path=path.replace(path.split("/")[0],p0) - return path - -def read_template(File, delimiter='='): - '''Reads the template file into a python dictionary structure. - Input : string, full path to the template file - Output: dictionary, pysar template content - Example: - tmpl = read_template(KyushuT424F610_640AlosA.template) - tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') - ''' - template_dict = {} - for line in open(File): - line = line.strip() - c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter - if len(c) < 2 or line.startswith('%') or line.startswith('#'): - next #ignore commented lines or those without variables - else: - atrName = c[0] - atrValue = str.replace(c[1],'\n','').split("#")[0].strip() - atrValue = check_variable_name(atrValue) - template_dict[atrName] = atrValue - return template_dict - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close - -def rm(TXT): - call_str = 'rm ' + TXT - os.system(call_str) - -def usage(): - print(''' -****************************************************************************************************** - - Generate SLC and SLC_par file for ERS/ENVISAT (SAR_IMS_1P format) - - usage: - - Down2SLC_ERS.py ProjectName DownName - - e.g. Down2SLC_ERS.py CotopaxiT120ERSA 910101 - -******************************************************************************************************* - ''') - -def main(argv): - - if len(sys.argv)==3: - projectName = sys.argv[1] - Date = sys.argv[2] - else: - usage();sys.exit(1) - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + '/' + projectName + '.template' - templateContents=read_template(templateFile) - - if 'rlks4cor' in templateContents: rlks4cor = templateContents['rlks4cor'] - else: rlks4cor = '4' - if 'azlks4cor' in templateContents: azlks4cor = templateContents['azlks4cor'] - else: azlks4cor = '4' - - if 'rwin4cor' in templateContents: rwin4cor = templateContents['rwin4cor'] - else: rwin4cor = '128' - if 'azwin4cor' in templateContents: azwin4cor = templateContents['azwin4cor'] - else: azwin4cor = '128' - if 'rsample4cor' in templateContents: rsample4cor = templateContents['rsample4cor'] - else: rsample4cor = '64' - if 'azsample4cor' in templateContents: azsample4cor = templateContents['azsample4cor'] - else: azsample4cor = '64' - - if ' rpos4cor' in templateContents: rpos4cor = templateContents[' rpos4cor'] - else: rpos4cor = ' - ' - if 'azpos4cor' in templateContents: azpos4cor = templateContents['azpos4cor'] - else: azpos4cor = ' - ' - - - if 'rfwin4cor' in templateContents: rfwin4cor = templateContents['rfwin4cor'] - else: rfwin4cor = str(int(int(rwin4cor)/2)) - if 'azfwin4cor' in templateContents: azfwin4cor = templateContents['azfwin4cor'] - else: azfwin4cor = str(int(int(azwin4cor)/2)) - if 'rfsample4cor' in templateContents: rfsample4cor = templateContents['rfsample4cor'] - else: rfsample4cor = str(2*int(rsample4cor)) - if 'azfsample4cor' in templateContents: azfsample4cor = templateContents['azfsample4cor'] - else: azfsample4cor = str(2*int(azsample4cor)) - - if 'thresh4cor' in templateContents: thresh4cor = templateContents['thresh4cor'] - else: thresh4cor = ' 0.2 ' - - rlks = templateContents['Range_Looks'] - azlks = templateContents['Azimuth_Looks'] - - - projectDir = scratchDir + '/' + projectName - downDir = scratchDir + '/' + projectName + "/DOWNLOAD" - slcDir = scratchDir + '/' + projectName + '/SLC' - - if not os.path.isdir(slcDir): - call_str= 'mkdir ' +slcDir - os.system(call_str) - os.chdir(downDir) - - t0 = 't0_' + Date - call_str = 'ls >' + t0 - os.system(call_str) - - tt = 'tt_' + Date - call_str = "grep " + Date + ' ' + t0 + '> ' + tt - os.system(call_str) - - te = 'te_' + Date - call_str = "grep SAR_IMS_1P " + tt + " > " + te - os.system(call_str) - - AA= np.loadtxt(te,dtype=np.str) - Na = AA.size - AA=AA.reshape(Na,) - - for i in range(Na): - Date0 = Date+'_' + str(i) - downName = str(AA[i]) - FileDir = downDir + '/' + downName - call_str = 'par_ASAR '+ FileDir + ' ' + Date0 - os.system(call_str) - - call_str ="rename 's/VV.SLC/slc/g' *" - os.system(call_str) - - slcpar = Date0 + '.slc.par' - call_str = 'ERS_orb_cor_par.py ' + slcpar - os.system(call_str) - - Date0 = Date - if len(Date)==6: - Date6 = Date - Date0 = Date - elif len(Date)==8: - Date0 = Date[2:8] - Date6 = Date[2:8] - else: - print('The input Date is invalid.') - sys.exit(1) - - dataDir = slcDir + '/' + Date0 - if not os.path.isdir(dataDir): - call_str = 'mkdir ' + dataDir - print('Generate SLC dir for date: ' + Date0) - os.system(call_str) - call_str = 'mv ' + Date + '*.slc* ' + dataDir - os.system(call_str) - - os.chdir(dataDir) - for i in range(Na): - Date0 = Date+'_' + str(i) - downName = str(AA[i]) - - SLCm = Date0 + '.slc' - SLCm_par = Date0 + '.slc.par' - MamprlksImg = Date0 + '.amp' - MamprlksPar = Date0 + '.amp.par' - - call_str = 'multi_look ' + SLCm + ' ' + SLCm_par + ' ' + MamprlksImg + ' ' + MamprlksPar + ' ' + rlks + ' ' + azlks - os.system(call_str) - nWidth = UseGamma(MamprlksPar, 'read', 'range_samples') - call_str = 'raspwr ' + MamprlksImg + ' ' + nWidth - os.system(call_str) - - SLC = Date6+'.slc' - SLCPAR = Date6 + '.slc.par' - AMP = Date6+'.amp' - AMPPAR = Date6+'.amp.par' - if Na==1: - os.rename(SLCm,SLC) - os.rename(SLCm_par,SLCPAR) - os.rename(MamprlksImg,AMP) - os.rename(MamprlksPar,AMPPAR) - - for i in range(Na-1): - if i==0: - DateA = Date + '_0' - else: - DateA = Date + '_' + str(i-1) + str(i) - - SLCA = DateA + '.slc' - SLCA_par = DateA + '.slc.par' - - - DateB = Date + '_' +str(i+1) - SLCB = DateB + '.slc' - SLCB_par = DateB + '.slc.par' - - DateC = Date + '_' + str(i) + str(i+1) - SLCC = DateC + '.slc' - SLCC_par = DateC + '.slc.par' - - ##################################################################################### - MamprlksImg = Date + '.amp' - MamprlksPar = Date + '.amp.par' - - off = DateC + '.off' - offs = DateC + '.offs' - offsets = DateC + '.offsets' - coffs = DateC + '.coffs' - coffsets = DateC + '.coffsets' - snr = DateC + '.snr' - off_std = DateC + '.off_std' - - ########################## Generate off file ############################# - - call_str = "create_offset " + SLCA_par + " " + SLCB_par + " " + off + " 1 - - 0" - os.system(call_str) - call_str = 'init_offset_orbit '+ SLCA_par + " " + SLCB_par + ' ' + off - os.system(call_str) - - - call_str = 'init_offset '+ SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' ' + rlks4cor + ' ' + azlks4cor + ' ' + rpos4cor + ' ' + azpos4cor - os.system(call_str) - - call_str = 'init_offset '+ SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' 1 1 - - ' - os.system(call_str) - - call_str = "offset_pwr " + SLCA + " " + SLCB + " " + SLCA_par + " " + SLCB_par + " " + off + " " + offs + " " + snr + " " + rwin4cor + " " + azwin4cor + " " + offsets + " 2 "+ rsample4cor + " " + azsample4cor - os.system(call_str) - - call_str = "offset_fit " + offs + " " + snr + " " + off + " " + coffs + " " + coffsets + " " + thresh4cor +" 3" - os.system(call_str) - - call_str = "offset_pwr " +SLCA + " " + SLCB + " " + SLCA_par + " " + SLCB_par + " " + off + " " + offs + " " + snr + " " + rfwin4cor + " " + azfwin4cor + " " + offsets + " 2 " + rfsample4cor + " " + azfsample4cor - os.system(call_str) - - call_str = "offset_fit " + offs + " " + snr + " " + off + " " + coffs + " " + coffsets + " " + thresh4cor + " 4 >" + off_std - os.system(call_str) - - ######################################################################################## - - call_str = 'SLC_cat ' + SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' ' + SLCC + ' ' + SLCC_par - os.system(call_str) - - if i==(Na-2): - if len(Date)==6: - DD = Date - else: - DD = Date[2:8] - SLCm = DD + '.slc' - SLCm_par = DD + '.slc.par' - - - call_str = 'cp ' + SLCC + ' ' + SLCm - os.system(call_str) - - call_str = 'cp ' + SLCC_par + ' ' + SLCm_par - os.system(call_str) - - call_str = 'multi_look ' + SLCm + ' ' + SLCm_par + ' ' + MamprlksImg + ' ' + MamprlksPar + ' ' + rlks + ' ' + azlks - os.system(call_str) - nWidth = UseGamma(MamprlksPar, 'read', 'range_samples') - call_str = 'raspwr ' + MamprlksImg + ' ' + nWidth - os.system(call_str) - - call_str = 'rm *.amp' - os.system(call_str) - - call_str = 'rm *_*.slc' - os.system(call_str) - - print("Down to SLC for %s is done! " % Date) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ERS_Cat_All.py b/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ERS_Cat_All.py deleted file mode 100644 index d256db3..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ERS_Cat_All.py +++ /dev/null @@ -1,136 +0,0 @@ -#! /usr/bin/env python -#''' -################################################################################## -# # -# Author: Yun-Meng Cao # -# Email : ymcmrs@gmail.com # -# Date : March, 2017 # -# # -# Generate Sentinel SLC from the downloaded data # -# # -################################################################################## -#''' -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob - -def check_variable_name(path): - s=path.split("/")[0] - if len(s)>0 and s[0]=="$": - p0=os.getenv(s[1:]) - path=path.replace(path.split("/")[0],p0) - return path - -def read_template(File, delimiter='='): - '''Reads the template file into a python dictionary structure. - Input : string, full path to the template file - Output: dictionary, pysar template content - Example: - tmpl = read_template(KyushuT424F610_640AlosA.template) - tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') - ''' - template_dict = {} - for line in open(File): - line = line.strip() - c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter - if len(c) < 2 or line.startswith('%') or line.startswith('#'): - next #ignore commented lines or those without variables - else: - atrName = c[0] - atrValue = str.replace(c[1],'\n','').split("#")[0].strip() - atrValue = check_variable_name(atrValue) - template_dict[atrName] = atrValue - return template_dict - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close - -def rm(TXT): - call_str = 'rm ' + TXT - os.system(call_str) - -def usage(): - print(''' -****************************************************************************************************** - - Downloading Sentinel-1A/B data based on ssara - - usage: - - Down2SLC_ERS_Cat_All.py ProjectName - - e.g. Down2SLC_ERS_Cat_All.py CotopaxiT120ERSA - -******************************************************************************************************* - ''') - -def main(argv): - - if len(sys.argv)==2: - projectName = sys.argv[1] - else: - usage();sys.exit(1) - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - - projectDir = scratchDir + '/' + projectName - downDir = scratchDir + '/' + projectName + "/DOWNLOAD" - slcDir = scratchDir + '/' + projectName + '/SLC' - - if not os.path.isdir(slcDir): - call_str= 'mkdir ' +slcDir - os.system(call_str) - - os.chdir(downDir) - - call_str = 'ls > tt0' - os.system(call_str) - - call_str = 'grep SAR_IMS_1P tt0 >tt1' - os.system(call_str) - - call_str = "awk -F_ '{print $3}' tt1 > tt2 " - os.system(call_str) - - call_str = "awk -FSA '{print $2}' tt2 > ttt" - os.system(call_str) - - call_str = 'sort ttt | uniq > ttm' - os.system(call_str) - - AA= np.loadtxt('ttm',dtype=np.str) - Na = AA.size - - for i in range(Na): - call_str = 'Down2SLC_ERS_Cat.py ' + projectName + ' ' + AA[i] - print(call_str) - call_str = 'Down2SLC_ERS_Cat.py ' + projectName + ' ' + AA[i] + ' >/dev/null' - os.system(call_str) - - - print("Down to SLC for %s is done! " % projectName) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/ERS_DEOS.py b/.codex_tmp/pyint_variants/no_rescue/pyint/ERS_DEOS.py deleted file mode 100644 index f97c88e..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/ERS_DEOS.py +++ /dev/null @@ -1,206 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v1.0 ### -### Copy Right (c): 2017, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Email : ymcmrs@gmail.com ### -### Univ. : Central South University & University of Miami ### -################################################################# - -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob -import argparse -import linecache - -def StrNum(S): - S = str(S) - if len(S)==1: - S='0' +S - return S - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close() - -######################################################################### - -INTRODUCTION = ''' -############################################################################# - Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 - - Download precise ERS orbit data from Delft (http://www.deos.tudelft.nl/). - Correct the orbit parameters using DELFT_vec2 - - ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-2/dgm-e04/arclist - - -''' - -EXAMPLE = ''' - Usage: - ERS_DEOS.py projectName Date - - Examples: - ERS_DEOS.py AqabaERSA 960101 - -############################################################################## -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Download (and correct) precise ERS orbit data.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='project name') - parser.add_argument('date',help='date of the ERS data') - - inps = parser.parse_args() - return inps - -################################################################################ - - -def main(argv): - - total = time.time() - inps = cmdLineParse() - - projectName = inps.projectName - DATE = inps.date - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - slcDir = scratchDir + '/' + projectName + '/SLC' - projDir = scratchDir + '/' + projectName - - ERS1_Orb_Dir = scratchDir + '/' + projectName + '/ERS1_Orb' - ERS2_Orb_Dir = scratchDir + '/' + projectName + '/ERS2_Orb' - - if not os.path.isdir(ERS1_Orb_Dir): - call_str = 'mkdir ' + ERS1_Orb_Dir - os.system(call_str) - - if not os.path.isdir(ERS2_Orb_Dir): - call_str = 'mkdir ' + ERS2_Orb_Dir - os.system(call_str) - - ERS1_Orb_list = ERS1_Orb_Dir + '/ERS1_Orb_list' - ERS2_Orb_list = ERS2_Orb_Dir + '/ERS2_Orb_list' - - os.chdir(ERS1_Orb_Dir) - if not os.path.isfile(ERS1_Orb_list): - Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-1/dgm-e04/arclist' - call_str = 'wget ' + Url + ' -O ERS1_Orb_list' - os.system(call_str) - - call_str = 'cp ERS1_Orb_list arclist' - os.system(call_str) - - os.chdir(ERS2_Orb_Dir) - if not os.path.isfile(ERS2_Orb_list): - Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-2/dgm-e04/arclist' - call_str = 'wget ' + Url + ' -O ERS2_Orb_list' - os.system(call_str) - - call_str = 'cp ERS2_Orb_list arclist' - os.system(call_str) - - if len(DATE)==8: - DATE6 = DATE[2:8] - else: - DATE6 = DATE - - slcDir1 = slcDir + '/' + DATE6 - slcpar = slcDir1 + '/' + DATE6 + '.slc.par' - Title = UseGamma(slcpar, 'read', 'title:') - if 'E1' in Title: - Flag = 'ERS1' - Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-1/dgm-e04/' - List = ERS1_Orb_list - Dir = ERS1_Orb_Dir - elif 'E2' in Title: - Flag = 'ERS2' - Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-2/dgm-e04/' - List = ERS2_Orb_list - Dir = ERS2_Orb_Dir - else: - print('The SAR data is invalid.') - sys.exit(1) - - if len(DATE)==6: - A0 = int(DATE[0:2]) - if A0 < 50: - DATE = '20' + DATE - else: - DATA = '19' + DATE - - YYMM = DATE6[0:2] + DATE6[2:4] - - call_str = 'grep ' + YYMM + ' ' + List + ' > list0' - os.system(call_str) - - call_str = "awk '{print $12}' list0 > start_end" - os.system(call_str) - - call_str = "awk '{print $1}' list0 > vec_num" - os.system(call_str) - - AA= np.loadtxt('start_end',dtype=np.str) - Na = AA.size - - print(AA) - - A_num= np.loadtxt('vec_num',dtype=np.str) - Na_num = A_num.size - - for i in range(Na-1): - A0 = AA[i] - A1 = AA[i+1] - print(DATE6) - print(A0) - print(A1) - if int(DATE6)==int(A0): - m = i - elif int(DATE6)==int(A1): - m = i+1 - elif int(A0) < int(DATE6) < int(A1): - m = i - - FF = A_num[int(m)] - SS = Url + 'ODR.' + FF - print(SS) - - call_str = 'wget -q --no-check-certificate ' + SS + ' -P ' + Dir - os.system(call_str) - - - print("Download precise DEFT orbital data for %s is done." % DATE) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/ERS_orb_cor.py b/.codex_tmp/pyint_variants/no_rescue/pyint/ERS_orb_cor.py deleted file mode 100644 index 696cd99..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/ERS_orb_cor.py +++ /dev/null @@ -1,132 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v1.0 ### -### Copy Right (c): 2017, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Email : ymcmrs@gmail.com ### -### Univ. : Central South University & University of Miami ### -################################################################# - -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob -import argparse -import linecache - -def StrNum(S): - S = str(S) - if len(S)==1: - S='0' +S - return S - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close() - -######################################################################### - -INTRODUCTION = ''' -############################################################################# - Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 - - Precise ERS orbit data from Delft (http://www.deos.tudelft.nl/). - Correct the orbit parameters using DELFT_vec2 - -''' - -EXAMPLE = ''' - Usage: - ERS_orb_cor.py projectName Date - - Examples: - ERS_orb_cor.py AqabaERSA 960101 - -############################################################################## -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Download (and correct) precise ERS orbit data.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='project name') - parser.add_argument('date',help='date of the ERS data') - - inps = parser.parse_args() - return inps - -################################################################################ - - -def main(argv): - - total = time.time() - inps = cmdLineParse() - - projectName = inps.projectName - DATE = inps.date - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - Ers1OrbDir = os.getenv('ERS1ORBDIR') - Ers2OrbDir = os.getenv('ERS2ORBDIR') - - slcDir = scratchDir + '/' + projectName + '/SLC' - projDir = scratchDir + '/' + projectName - - - if len(DATE)==8: - DATE6 = DATE[2:8] - else: - DATE6 = DATE - - slcDir1 = slcDir + '/' + DATE6 - slcpar = slcDir1 + '/' + DATE6 + '.slc.par' - Title = UseGamma(slcpar, 'read', 'title:') - if 'E1' in Title: - Flag = 'ERS1' - #Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-1/dgm-e04/' - #List = ERS1_Orb_list - Dir = Ers1OrbDir - elif 'E2' in Title: - Flag = 'ERS2' - #Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-2/dgm-e04/' - #List = ERS2_Orb_list - Dir = Ers2OrbDir - else: - print('The SAR data is invalid.') - sys.exit(1) - - call_str = 'DELFT_vec2 ' + slcpar + ' ' + Dir + ' 30' - os.system(call_str) - - - print("Using DEFT orbital data for %s is done." % DATE) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/ERS_orb_cor_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/ERS_orb_cor_all.py deleted file mode 100644 index ae81bb8..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/ERS_orb_cor_all.py +++ /dev/null @@ -1,126 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v1.0 ### -### Copy Right (c): 2017, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Email : ymcmrs@gmail.com ### -### Univ. : Central South University & University of Miami ### -################################################################# - -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob -import argparse -import linecache - -def StrNum(S): - S = str(S) - if len(S)==1: - S='0' +S - return S - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close() - -def is_number(s): - try: - int(s) - return True - except ValueError: - return False - -######################################################################### - -INTRODUCTION = ''' -############################################################################# - Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 - - Precise ERS orbit data from Delft (http://www.deos.tudelft.nl/). - Correct the orbit parameters using DELFT_vec2 - -''' - -EXAMPLE = ''' - Usage: - ERS_orb_cor_all.py projectName - - Examples: - ERS_orb_cor_all.py AqabaERSA - -############################################################################## -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Download (and correct) precise ERS orbit data.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='project name') - - inps = parser.parse_args() - return inps - -################################################################################ - - -def main(argv): - - total = time.time() - inps = cmdLineParse() - - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - slcDir = scratchDir + '/' + projectName + '/SLC' - - ListSLC = os.listdir(slcDir) - Datelist = [] - SLCfile = [] - SLCParfile = [] - - - for kk in range(len(ListSLC)): - if ( is_number(ListSLC[kk]) and len(ListSLC[kk])==6 ): # if SAR date number is 8, 6 should change to 8. - DD=ListSLC[kk] - Year=int(DD[0:2]) - Month = int(DD[2:4]) - Day = int(DD[4:6]) - Datelist.append(ListSLC[kk]) - N = len(Datelist) - - for i in range(N): - call_str = 'ERS_orb_cor.py ' + projectName + ' ' + Datelist[i] - print(call_str) - os.system(call_str) - - print("Using DEFT orbital data for project %s is done." % projectName) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/ERS_orb_cor_par.py b/.codex_tmp/pyint_variants/no_rescue/pyint/ERS_orb_cor_par.py deleted file mode 100644 index 2514d7c..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/ERS_orb_cor_par.py +++ /dev/null @@ -1,118 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v1.0 ### -### Copy Right (c): 2017, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Email : ymcmrs@gmail.com ### -### Univ. : Central South University & University of Miami ### -################################################################# - -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob -import argparse -import linecache - -def StrNum(S): - S = str(S) - if len(S)==1: - S='0' +S - return S - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close() - -######################################################################### - -INTRODUCTION = ''' -############################################################################# - Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 - - Precise ERS orbit data from Delft (http://www.deos.tudelft.nl/). - Correct the orbit parameters using DELFT_vec2 - -''' - -EXAMPLE = ''' - Usage: - ERS_orb_cor_par.py SLC_par - - Examples: - ERS_orb_cor_par.py 960101.slc.par - -############################################################################## -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Download (and correct) precise ERS orbit data.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('slc_par',help='slc par file') - - inps = parser.parse_args() - return inps - -################################################################################ - - -def main(argv): - - total = time.time() - inps = cmdLineParse() - - slcpar = inps.slc_par - - Ers1OrbDir = os.getenv('ERS1ORBDIR') - Ers2OrbDir = os.getenv('ERS2ORBDIR') - - Title = UseGamma(slcpar, 'read', 'title:') - if 'E1' in Title: - Flag = 'ERS1' - #Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-1/dgm-e04/' - #List = ERS1_Orb_list - Dir = Ers1OrbDir - elif 'E2' in Title: - Flag = 'ERS2' - #Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-2/dgm-e04/' - #List = ERS2_Orb_list - Dir = Ers2OrbDir - else: - print('The SAR data is invalid.') - sys.exit(1) - - call_str = 'DELFT_vec2 ' + slcpar + ' ' + Dir - os.system(call_str) - - - print("Using DEFT orbital data for %s is done." % slcpar) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/GACOS_correction.csh b/.codex_tmp/pyint_variants/no_rescue/pyint/GACOS_correction.csh deleted file mode 100644 index 882b07d..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/GACOS_correction.csh +++ /dev/null @@ -1,131 +0,0 @@ -#!/bin/csh -f -# $Id$ -#GACOS Correction -#Erik Rivas, Oct 18 2021 -# - -if ($#argv != 5) then - echo "" - echo "Usage: GACOS.csh list_interferograms full_path_to_GACOS_data full_path_topo reference_point incidence_angle" - echo "Script needs to be run inside the intf_all folder" - echo "Performs gacos correction" - echo "" - echo "list_interferograms: list of folders inside the intf_all folder containing the phases and coherence grids" - echo "" - echo "full_path_to_GACOS_data" - echo "Example: /home/erikr/gacos/" - echo "" - echo "full_path_topo" - echo "Example: /home/erikr/project/topo/" - echo "" - echo "the topo folder needs to have: dem.grd, master.PRM, and the correspondant .LED file" - echo "" - echo "list_interferograms: list of folders with interferograms created" - echo "" - echo "Reference point in lon lat coordinates (text file)" - echo "" - echo "Indicence angle in degrees from SAT_look (float/integer)" - echo "" - echo "Outputs: phasefilt.grd files corrected and added as additional products in each interferogram folder. These outputs should be used for the unwrap processing" - exit 1 -endif - -set list = $1 -set GACOS_dir = $2 -set topo_dir = $3 -set reference_point = $4 -set incidence = $5 - -#Checking inputs -if !(-e $list) then - echo "$list seems not to exist" - exit 1 -endif -if !(-d $GACOS_dir) then - echo "$GACOS_dir seems not to exist" - exit 1 -endif -if !(-d $topo_dir) then - echo "$topo_dir seems not to exist" - exit 1 -endif -if !(-f $reference_point) then - echo "Reference point file: $reference_point seems not to exist. Provide a text file with lon lat values" - exit 1 -endif - - -#PROJECT POINT FROM LON-LAT TO RADAR COORDINATES -set ref_llh = $topo_dir"ref.llh" -set out_ratll = $topo_dir"out.ratll" -set reference_point_ra = $topo_dir"ref_point.ra" -if (-f $ref_llh) then - echo "Removing old $ref_llh" - rm $ref_llh -endif -if (-f $out_ratll) then - echo "Removing old $out_ratll" - rm $out_ratll -endif -if (-f $reference_point_ra) then - echo "Removing old $reference_point_ra" - rm $reference_point_ra -endif - -gmt grdtrack $reference_point -G$topo_dir"dem.grd" >> $ref_llh -ln -s $topo_dir*.LED . -SAT_llt2rat $topo_dir"master.PRM" 0 < $topo_dir"ref.llh" > $out_ratll -rm *.LED -cat $out_ratll |awk '{print $1, $2}' > $reference_point_ra -set dem_grd = $topo_dir"dem.grd" -#----------------------------------------------# - -#FOR LOOP OVER LIST OF INTERFEROGRAMS -foreach dir (`awk '{print $1}' $list`) - if !(-d $dir) then - echo "$dir directory seems not to exist" - exit 1 - endif - - cd $dir - - #Check if there are only two SLC files (not sure if this step is neccessary) - if (`ls *.SLC|wc -l` != "2") then - echo "the number of SLC files is inconsistent" - endif - - #ls sorts content alphanumeric, therefore it is assume the first as the master on the list - set fst_date = `ls *.SLC|sed -n '1p'|awk '{print substr($1,4,8)}'` - set scd_date = `ls *.SLC|sed -n '2p'|awk '{print substr($1,4,8)}'` - - #Save directory of current intf - set intf_dir = `pwd` - - cd $GACOS_dir - - - #Check if the GACOS files are in the folder - if (-f $fst_date".ztd" && -f $fst_date".ztd.rsc" && -f $scd_date".ztd" && -f $scd_date".ztd.rsc") then - set first_ztd = $GACOS_dir$fst_date".ztd" - set first_rsc = $GACOS_dir$fst_date".ztd.rsc" - set second_ztd = $GACOS_dir$scd_date".ztd" - set second_rsc = $GACOS_dir$scd_date".ztd.rsc" - - #GACOS Correction - cd $intf_dir - #Link trans.dat to each folder. Neccesary to project ztd grids to radar coordinates - ln -s $topo_dir"trans.dat" - operation.csh $first_ztd $first_rsc $second_ztd $second_rsc $reference_point_ra $incidence $dem_grd - rm trans.dat - - else - - echo "GACOS files do not exist / Wrong directory" - exit 1 - - endif - - cd .. -end - -echo "GACOS correction done" diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/Get_off_std.py b/.codex_tmp/pyint_variants/no_rescue/pyint/Get_off_std.py deleted file mode 100644 index 18775a2..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/Get_off_std.py +++ /dev/null @@ -1,145 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v1.0 ### -### Copy Right (c): 2017, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Email : ymcmrs@gmail.com ### -### Univ. : Central South University & University of Miami ### -################################################################# - - -import os -import sys -import glob -import time -import argparse - -import h5py -import numpy as np - -def check_variable_name(path): - s=path.split("/")[0] - if len(s)>0 and s[0]=="$": - p0=os.getenv(s[1:]) - path=path.replace(path.split("/")[0],p0) - return path - -def read_template(File, delimiter='='): - '''Reads the template file into a python dictionary structure. - Input : string, full path to the template file - Output: dictionary, pysar template content - Example: - tmpl = read_template(KyushuT424F610_640AlosA.template) - tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') - ''' - template_dict = {} - for line in open(File): - line = line.strip() - c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter - if len(c) < 2 or line.startswith('%') or line.startswith('#'): - next #ignore commented lines or those without variables - else: - atrName = c[0] - atrValue = str.replace(c[1],'\n','').split("#")[0].strip() - atrValue = check_variable_name(atrValue) - template_dict[atrName] = atrValue - return template_dict - - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close() - -def UseGamma2(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[2].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close() - -def usage(): - print(''' -****************************************************************************************************** - - Get co-registration standard deviation of SLCs for one project - - usage: - - GenerateRSC_Gamma.py projectName - - e.g. Get_off_std.py GalapagosT061EnvA - -******************************************************************************************************* - ''') - -def main(argv): - - if len(sys.argv)==2: - if argv[0] in ['-h','--help']: usage(); sys.exit(1) - else: projectName=sys.argv[1] - else: - usage();sys.exit(1) - - scratchDir = os.getenv('SCRATCHDIR') - OFFDir = scratchDir + '/' + projectName + '/RSLC' - OFFSTR = OFFDir + '/*.off_std' - OFFFile = glob.glob(OFFSTR) - STD_TXT ='COREG_STD_ALL' - os.chdir(OFFDir) - if os.path.isfile(STD_TXT): - os.remove(STD_TXT) - - for ff in OFFFile: - OFF_STD = ff - NM = os.path.basename(ff).split('.')[0] - RR = UseGamma(OFF_STD,'read','final range offset poly. coeff.:') - cor_rg = RR.split(' ')[0] - - AA = UseGamma(OFF_STD,'read','final azimuth offset poly. coeff.:') - cor_az = AA.split(' ')[0] - - STDRR = UseGamma(OFF_STD,'read','final model fit std. dev. (samples) range:') - std_rg=STDRR.split(' ')[0] - - std_az = UseGamma2(OFF_STD,'read','final model fit std. dev. (samples) range:') - - STR = NM + ' ' + cor_rg + ' ' + cor_az + ' ' + std_rg + ' ' + std_az - call_str ='echo ' + STR + ' >> ' + STD_TXT - os.system(call_str) - - - sys.exit(1) - - -if __name__ == '__main__': - main(sys.argv[1:]) - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/.gitignore b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/.gitignore deleted file mode 100644 index 764a6ac..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -*.swp -bin/ -*.tif -*.pyc -__pycache__/ diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/LICENSE b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/LICENSE deleted file mode 100644 index 261eeb9..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_01_Read_Data.py b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_01_Read_Data.py deleted file mode 100644 index a9815c1..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_01_Read_Data.py +++ /dev/null @@ -1,582 +0,0 @@ -#!/usr/bin/env python3 - -import os -import re -from datetime import datetime, timedelta -import numpy as np -import rasterio -from bin.multilook_w import multilook_w -from bin.multilook import multilook -import configparser -import sys -import warnings -import pickle -import glob - -warnings.filterwarnings("ignore", category=DeprecationWarning) - -####################################################################### - -HELP_TEXT = """ -PhaseBias_01_Read_Data.py - -This script automates the process of downloading GeoTIFF files for wrapped interferograms and coherence images. -The files can be retrieved either from the COMET-LiCS web portal or from a root path specified in the configuration file. - -Assumptions: -- Interferograms are organized in folders named as `yyyymmdd_yyyymmdd` (e.g., `20230101_20230107`). - -Files Downloaded: -1. `yyyymmdd_yyyymmdd.geo.diff_pha.tif`: - - Contains the wrapped phase image in radians. - - Values range from -3.14 to 3.14. - -2. `yyyymmdd_yyyymmdd.geo.cc.tif`: - - Contains the coherence image of the interferometric pair. - - Values range from 0 to 255, where: - - 0 represents the lowest coherence. - - 255 represents the highest coherence. - -Outputs: -1. `All_ifgs_start_end` (Pickle file, saved as `.pkl`): - - A dictionary containing all interferograms available between the specified start and end dates. -2. `All_coh_start_end` (Pickle file, saved as `.pkl`): - - A dictionary containing all coherence data available between the specified start and end dates. - -Storage Information: -- All output files are saved in the directory `Data/` under the `output_path` defined in the configuration file. - -Additional Output: -- A summary report is generated that includes: - - The number of available interferograms for each temporal baseline. - - The number of missing interferograms. - -""" - -if "--help" in sys.argv: - print(HELP_TEXT) - sys.exit(0) - -################### - -# Config file path -config_file = "config.txt" - -# Check if config file exists -if not os.path.exists(config_file): - raise FileNotFoundError( - f"Error: The configuration file '{config_file}' was not found in the script's directory." - ) - -# Initialize the configparser -config = configparser.ConfigParser() - -# Read the config.txt file -config.read(config_file) - -# Define required parameters -required_parameters = [ - "root_path", - "frame", - "start", - "end", - "interval", - "nlook", - "LiCSAR_data", -] - -# Initialize a dictionary to store parameters -parameters = {} - -# Attempt to retrieve each parameter and catch errors -try: - parameters["root_path"] = config.get("DEFAULT", "root_path") - parameters["output_path"] = config.get("DEFAULT", "output_path") - parameters["frame"] = config.get("DEFAULT", "frame") - parameters["start"] = config.get("DEFAULT", "start") - parameters["end"] = config.get("DEFAULT", "end") - parameters["interval"] = config.getint("DEFAULT", "interval") - parameters["nlook"] = config.getint( - "DEFAULT", "nlook") # Assuming 'nlook' should be an integer - parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") - -except configparser.NoOptionError as e: - # Error message if a required parameter is missing - missing_param = str(e).split(": ")[1] - raise ValueError( - f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." - ) -except configparser.Error as e: - # General error for any configparser-related issues - raise ValueError(f"Error while reading '{config_file}': {e}") - -# Assign to individual variables for easy access -root_path = parameters["root_path"] -output_path = parameters["output_path"] -frame = parameters["frame"] -start = parameters["start"] -end = parameters["end"] -interval = parameters["interval"] -nlook = parameters["nlook"] -LiCSAR_data = parameters["LiCSAR_data"] - -# hardcoded parameters, not included in the config file -landmask = 1 -filtered_ifgs = "yes" -min_baseline = 5 -max_baseline = 366 - -# Print the values to confirm they're being read correctly -print("Root Path:", root_path) -print("output Path:", output_path) -print("Frame:", frame) -print("Start Date:", start) -print("End Date:", end) -print("Interval:", interval) -print("nlook:", nlook) -print("LiCSAR_data", LiCSAR_data) - -################ -# all_ifgs = read_ifgs(frame, start, end, min_baseline, max_baseline, landmask, nlook, interval, LiCSAR_data, filtered_ifgs) - -def read_ifgs( - start, - end, - min_baseline, - max_baseline, - landmask, - nlook, - interval, - LiCSAR_data, - filtered_ifgs, -): - - start_date = datetime.strptime(start, "%Y%m%d") - end_date = datetime.strptime(end, "%Y%m%d") - # Check if the file ends with "geo.diff_pha.tif" - # if filename.endswith("geo.diff_unfiltered_pha.tif"): # if unfiltered data are used - if filtered_ifgs == "yes": # yes for the filtered ifgs, no for the unfiltered ifgs - req_file_name = "geo.diff_pha.tif" - else: - req_file_name = "geo.diff_unfiltered_pha.tif" - - ############ reading - - if LiCSAR_data == "yes": - track = frame[0:3] # extracting track number from frame id - if track[0] == "0": - track = track[1:3] - if track[0] == "0": - track = track[1:2] - - root_directory = f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}/interferograms/" - else: - root_directory = os.path.join(root_path, "interferograms") # Use os.path.join for consistent path construction - #root_directory = os.path.join(root_path, "") - - ##### Count total files matching the criteria for accurate progress calculation - print(f"Reading data from the path: {root_directory}") - - total_files = sum(1 for dirpath, _, filenames in os.walk(root_directory) - for filename in filenames - if filename.endswith(req_file_name)) - processed_files = 0 - - print("Reading wrapped interferograms: [", end="", flush=True) - ##### - - # Initialize an empty dictionary to store the arrays for each category - category_arrays = {} - last_date = {} - n = 0 - - if landmask is not None: - if LiCSAR_data == "yes": - landmask_path = f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}/metadata/{frame}.geo.landmask.tif" - else: - # Use glob to find the file ending with ".geo.landmask.tif" in the metadata directory - metadata_dir = os.path.join(root_path, "metadata") - landmask_files = glob.glob( - os.path.join(metadata_dir, "*.geo.landmask.tif")) - if landmask_files: - landmask_path = landmask_files[0] # Take the first match - else: - raise FileNotFoundError( - "No file ending with '.geo.landmask.tif' found in the metadata directory." - ) - - # Load the landmask file - with rasterio.open(landmask_path) as src: - array_landmask = src.read(1) - - n = 1 - # Traverse the root directory and its subdirectories - for dirpath, dirnames, filenames in os.walk(root_directory): - dirnames.sort( - ) # This ensures dirnames are processed in alphabetical order - # Iterate through the files in the current directory - for filename in filenames: - # Check if the file ends with "geo.diff_pha.tif" - # if filename.endswith("geo.diff_unfiltered_pha.tif"): # if unfiltered data are used - if (filtered_ifgs == "yes" - ): # yes for the filtered ifgs, no for the unfiltered ifgs - req_file_name = "geo.diff_pha.tif" - else: - req_file_name = "geo.diff_unfiltered_pha.tif" - - if filename.endswith(req_file_name): - file_path = os.path.join(dirpath, filename) - - #### for printing into the console - processed_files += 1 - percentage = (processed_files / total_files) * 100 - - # Clear previous percentage and print updated one - sys.stdout.write( - "\rReading wrapped phases: [{:<50}] {}%".format( - "=" * int(percentage // 2), int(percentage))) - sys.stdout.flush() - - ##### - - # Extract the dates from the file name - match = re.search(r"(\d+)_(\d+)", filename) - if match: - date1 = match.group(1) - date2 = match.group(2) - - # Convert the dates to datetime objects - date1_obj = datetime.strptime(date1, "%Y%m%d") - date2_obj = datetime.strptime(date2, "%Y%m%d") - - # Check if the dates fall within the specified range - category = (date2_obj - date1_obj).days - - if (start_date <= date1_obj <= end_date - and start_date <= date2_obj <= end_date - and category < max_baseline - and category > min_baseline): - if ( - n == 1 - ): # finding the first and last acquisiton in the time-series - first_acq = date1_obj - else: - last_acq = date2_obj - n = n + 1 - - # Calculate the difference in days between the two dates - - # Open the TIFF file - try: - with rasterio.open(file_path) as src: - array_data = src.read(1).astype(np.float32) - except Exception as e: - print(f"\u26a0\ufe0f Error reading {file_path}: {e}. Appending None instead.") - array_data = None - - #if array_data is not None: - array_data[array_data == 0] = np.nan ## converting the zerso values in wrapped phases into nan. This is helpful when calculating loop closures - - #array_data[np.abs(array_data) < 1e-8] = np.nan # as after filter i observed low vlaues outside the frame. I also wanted to nulify them. so used this insteadof above - - - if landmask != None: #and array_data is not None: - array_data[array_landmask != 1] = np.nan - - if ( - nlook != None and nlook != 1 #and array_data is not None - ): # incase of unwrap data because they are already multilooked to 10 we don't multilook them here - array_data = multilook_w(array_data, nlook) - - if category not in last_date: - last_date[category] = date1_obj - # if (date1_obj - date2_obj) > interval - - # appending None to the category_arrays where there are missing ifgs in the middle of time-series - diff_ifgs = ( - date1_obj - last_date[category] - ).days # /category #difference between the first epochs of two consecutive ifg - if diff_ifgs > interval: - n_no_acq = int(diff_ifgs / interval - - 1) ## number of missing ifgs - for i in range( - n_no_acq - ): # the number of none depends on the number of missing ifgs - # Append the none to the existing array - category_arrays[category].append(None) - last_date[category] = date1_obj - - # Check if the category is already in the dictionary - if category in category_arrays: - # If the category already exists, append the array to the existing array - category_arrays[category].append(array_data) - else: - # If the category doesn't exist, create a new list with the array - category_arrays[category] = [array_data] - - # appending None to the category_arrays where there are missing ifgs in the end of time-series - for category in sorted(category_arrays): - while (last_date[category] + timedelta(category)) < last_acq: - category_arrays[category].append(None) - last_date[category] = last_date[category] + timedelta(interval) - - # appending None to the category_arrays where there are missing ifgs in the begining of time-series - # max number of expected 6-day(i.e. interval) interferograms in the full time-series - max_ifg_number = abs(int((first_acq - last_acq).days / interval)) - - # all the categories in the data e.g. 6, 12, 18 etc - cat = [] - for category in category_arrays: - cat.append(category) - - for i in range(interval, max(cat)+1, interval): - if i in cat: - while len(category_arrays[i]) < (max_ifg_number + 1 - i/interval): - category_arrays[i].insert( 0, None) - - print("\nFinished reading all interferograms.") - - return category_arrays - - -########################################################################################################################### -########################################################################################################################## - - -def read_coh(start, end, min_baseline, max_baseline, landmask, nlook, interval, - LiCSAR_data): - - start_date = datetime.strptime(start, "%Y%m%d") - end_date = datetime.strptime(end, "%Y%m%d") - ############ reading - - if LiCSAR_data == "yes": - track = frame[0:3] - if track[0] == "0": - track = track[1:3] - if track[0] == "0": - track = track[1:2] - - root_directory = f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}/interferograms/" - else: - root_directory = os.path.join( - root_path, "interferograms" - ) # Use os.path.join for consistent path construction - - req_file_name = "geo.cc.tif" - - ##### Count total files matching the criteria for accurate progress calculation - - total_files = sum(1 for dirpath, _, filenames in os.walk(root_directory) - for filename in filenames - if filename.endswith(req_file_name)) - processed_files = 0 - - print("Reading coherence data: [", end="", flush=True) - ##### - - # Initialize an empty dictionary to store the arrays for each category - category_arrays = {} - last_date = {} - n = 0 - - if landmask is not None: - if LiCSAR_data == "yes": - landmask_path = f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}/metadata/{frame}.geo.landmask.tif" - else: - # Use glob to find the file ending with ".geo.landmask.tif" in the metadata directory - metadata_dir = os.path.join(root_path, "metadata") - landmask_files = glob.glob( - os.path.join(metadata_dir, "*.geo.landmask.tif")) - if landmask_files: - landmask_path = landmask_files[0] # Take the first match - else: - raise FileNotFoundError( - "No file ending with '.geo.landmask.tif' found in the metadata directory." - ) - - # Load the landmask file - with rasterio.open(landmask_path) as src: - array_landmask = src.read(1) - - n = 1 - # Traverse the root directory and its subdirectories - for dirpath, dirnames, filenames in os.walk(root_directory): - # Iterate through the files in the current directory - for filename in filenames: - # Check if the file ends with "geo.cc.tif" - - if filename.endswith(req_file_name): - file_path = os.path.join(dirpath, filename) - - #### for printing into the console - processed_files += 1 - percentage = (processed_files / total_files) * 100 - - # Clear previous percentage and print updated one - sys.stdout.write( - "\rReading coherence data: [{:<50}] {}%".format( - "=" * int(percentage // 2), int(percentage))) - sys.stdout.flush() - - ##### - - # Extract the dates from the file name - match = re.search(r"(\d+)_(\d+)", filename) - if match: - date1 = match.group(1) - date2 = match.group(2) - - # Convert the dates to datetime objects - date1_obj = datetime.strptime(date1, "%Y%m%d") - date2_obj = datetime.strptime(date2, "%Y%m%d") - - # Check if the dates fall within the specified range - category = (date2_obj - date1_obj).days - - if (start_date <= date1_obj <= end_date - and start_date <= date2_obj <= end_date - and category < max_baseline - and category > min_baseline): - if ( - n == 1 - ): # finding the first and last acquisiton in the time-series - first_acq = date1_obj - else: - last_acq = date2_obj - n = n + 1 - - # Calculate the difference in days between the two dates - - # Open the TIFF file - try: - with rasterio.open(file_path) as src: - array_data = src.read(1).astype(np.float32) - except Exception as e: - print(f"\u26a0\ufe0f Error reading {file_path}: {e}. Appending None instead.") - array_data = None - - if landmask != 0: - array_data[array_landmask != 1] = np.nan - - if ( - nlook != None and nlook != 1 - ): # incase of unwrap data because they are already multilooked to 10 we don't multilook them here - array_data = multilook(array_data, nlook) - - if category not in last_date: - last_date[category] = date1_obj - # if (date1_obj - date2_obj) > interval - - # appending None to the category_arrays where there are missing ifgs in the middle of time-series - diff_ifgs = ( - date1_obj - last_date[category] - ).days # /category #difference between the first epochs of two consecutive ifg - if diff_ifgs > interval: - n_no_acq = int(diff_ifgs / interval - - 1) ## number of missing ifgs - for i in range( - n_no_acq - ): # the number of none depends on the number of missing ifgs - # Append the none to the existing array - category_arrays[category].append(None) - last_date[category] = date1_obj - - # Check if the category is already in the dictionary - if category in category_arrays: - # If the category already exists, append the array to the existing array - category_arrays[category].append(array_data) - else: - # If the category doesn't exist, create a new list with the array - category_arrays[category] = [array_data] - - # appending None to the category_arrays where there are missing ifgs in the end of time-series - for category in sorted(category_arrays): - while (last_date[category] + timedelta(category)) < last_acq: - category_arrays[category].append(None) - last_date[category] = last_date[category] + timedelta(interval) - - # appending None to the category_arrays where there are missing ifgs in the begining of time-series - # max number of expected 6-day(i.e. interval) interferograms in the full time-series - max_ifg_number = abs(int((first_acq - last_acq).days / interval)) - - # all the categories in the data e.g. 6, 12, 18 etc - cat = [] - for category in category_arrays: - cat.append(category) - - for i in range(interval, max(cat)+1, interval): - if i in cat: - while len(category_arrays[i]) < (max_ifg_number + 1 - i/interval): - category_arrays[i].insert(0, None) - - print("\nFinished reading all coherence data.") - - return category_arrays - - -########################################################################################################################## -########################################################################################################################## - -all_ifgs = read_ifgs( - start, - end, - min_baseline, - max_baseline, - landmask, - nlook, - interval, - LiCSAR_data, - filtered_ifgs, -) -# Define the directory path -directory_path = os.path.join(output_path, "Data") - -# Create the directory if it doesn't exist -os.makedirs(directory_path, exist_ok=True) - -# Define the file path -#file_path = os.path.join(directory_path, f"_all_ifgs_{start}_{end}") - -# Remove all files inside the directory -if os.path.exists(directory_path): - files = glob.glob(os.path.join(directory_path, "*")) - for file in files: - os.remove(file) - - -# for writing the all_loops -file_path = output_path + "/Data/" + "All_ifgs_" + start + "_" + end + ".pkl" -with open(file_path, "wb") as file: # for writting a dictionary to a file - pickle.dump(all_ifgs, file) - - -################################################ - -# After processing all interferograms, generate the report -print("\nReport on the number of IFGs and missing IFGs per category:") -for category in sorted(all_ifgs): - num_ifgs = sum(1 for item in all_ifgs[category] - if item is not None) # Count valid IFGs - num_missing_ifgs = sum(1 for item in all_ifgs[category] - if item is None) # Count missing IFGs - print( - f"{category}-days: Number of IFGs: {num_ifgs} Number of missing IFGs: {num_missing_ifgs}" - ) - -################################################################ - -import gc -del all_ifgs -gc.collect() - - - -all_coh = read_coh(start, end, min_baseline, max_baseline, landmask, nlook, - interval, LiCSAR_data) - -## for writting the all_coh -file_path = output_path + "/Data/" + "All_coh_" + start + "_" + end + ".pkl" -with open(file_path, "wb") as file: # for writting a dictionary to a file - pickle.dump(all_coh, file) - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_02_Loop_Closures.py b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_02_Loop_Closures.py deleted file mode 100644 index c7a14d8..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_02_Loop_Closures.py +++ /dev/null @@ -1,323 +0,0 @@ -#!/usr/bin/env python3 - -import numpy as np -import pickle -import glob -import configparser -import os -import sys -import warnings -warnings.filterwarnings("ignore", category=DeprecationWarning) - -################################################################################################ - -HELP_TEXT = """ -PhaseBias_02_Loop_Closures.py - -This script calculates loop closures (Δφ) using the interferograms imported in the first step. - -Definition of Loop Closures: -Loop closures, Δφ, are calculated for the epochs between `i` and `k`, and are defined as: - Δφ_(i,k) = |φ_(i,k) - ∑_(t=i)^k φ_(t,t+1)|_2π - -Where: -- φ_(i,j): Represents the phase difference for a pixel in the interferogram formed between epochs `i` and `j`. -- |.|_2π: Indicates that the result is wrapped modulo 2π (i.e., values range from -π to π). - -Key Information: -- Nonzero closure phase is a by-product of spatial filtering/multilooking and is primarily associated with changes in the scattering and electrical properties of the ground surface. -- The calculated loop closures are based on the minimum temporal-baseline interferograms defined in the configuration file (parameter: `interval`) and are referred to as base interferograms. These may represent: - - 12- and 6-day closures (or 24- and 12-day closures), Δφ_(i,i+2) - - 18- and 6-day closures (or 36- and 12-day closures), Δφ_(i,i+3) - The distinction depends on whether the base interferograms are 6-day or 12-day intervals. - -Outputs: -- The calculated loop closures are stored as a dictionary in a file named: - `All_loops_start_end.pkl` -- The output is saved in the directory `Data/` under the `output_path` defined in the configuration file. - -Loop Closure Report: -At the end of the script, a loop closure report is displayed. This report includes: -- The number of generated and missing loop closures for short-interval closures, such as 12-6 and 18-6. -- The number of generated and missing long-interval closures, obtained using long-term interferograms, such as 204-6 and 204-12. - -Usage: -1. Ensure the interferograms from the first script are available. -2. Run the script to generate loop closures for the specified temporal baselines. - -Example: -```bash -python PhaseBias_02_Loop_Closures.py - - -""" -if "--help" in sys.argv: - print(HELP_TEXT) - sys.exit(0) - -################################################################################################ -# Config file path -config_file = "config.txt" - -# Check if config file exists -if not os.path.exists(config_file): - raise FileNotFoundError( - f"Error: The configuration file '{config_file}' was not found in the script's directory." - ) - -# Initialize the configparser -config = configparser.ConfigParser() - -# Read the config.txt file -config.read(config_file) - -# Define required parameters -required_parameters = [ - "root_path", - "frame", - "start", - "end", - "interval", - "nlook", - "LiCSAR_data", -] - -# Initialize a dictionary to store parameters -parameters = {} - -# Attempt to retrieve each parameter and catch errors -try: - parameters["root_path"] = config.get("DEFAULT", "root_path") - parameters["output_path"] = config.get("DEFAULT", "output_path") - parameters["frame"] = config.get("DEFAULT", "frame") - parameters["start"] = config.get("DEFAULT", "start") - parameters["end"] = config.get("DEFAULT", "end") - parameters["interval"] = config.getint("DEFAULT", "interval") - parameters["nlook"] = config.getint( - "DEFAULT", "nlook") # Assuming 'nlook' should be an integer - parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") - -except configparser.NoOptionError as e: - # Error message if a required parameter is missing - missing_param = str(e).split(": ")[1] - raise ValueError( - f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." - ) -except configparser.Error as e: - # General error for any configparser-related issues - raise ValueError(f"Error while reading '{config_file}': {e}") - -# Assign to individual variables for easy access -root_path = parameters["root_path"] -output_path = parameters["output_path"] -frame = parameters["frame"] -start = parameters["start"] -end = parameters["end"] -interval = parameters["interval"] -nlook = parameters["nlook"] -LiCSAR_data = parameters["LiCSAR_data"] - -# hardcoded parameters, not included in the config file -landmask = 1 -filtered_ifgs = "yes" -max_loop = 5 # in case of 3 all loops with 6,12,18-day will be calculated (e.g. (60,6), (60,12) and (60,18)) - - -############################################################################################# -################################################################################################# -def loop_calc(max_loop): - - ## Read ifgs: - - # Define the path components - sub_dir = "Data" - filename_pattern = "All_ifgs_*" # Pattern to match files starting with "All_ifgs" - - # Construct the full file path with the refined pattern - file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) - # Use glob to find the file and ensure it exists - file_list = glob.glob(file_path_pattern) - if file_list: - file_path = file_list[0] # Get the single file path directly - - # Load the data from the file - with open(file_path, "rb") as file: - all_ifgs = pickle.load(file) - else: - raise FileNotFoundError( - f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." - ) - - loop = {} - missing = {} - - cat = sorted(all_ifgs.keys()) # Get sorted categories - # print('cat = ', cat) - - # Initialize counters for reporting - loop_counts = { - category: { - cat[l]: 0 - for l in range(max_loop) - } - for category in sorted(all_ifgs.keys()) - } - missing_counts = { - category: { - cat[l]: 0 - for l in range(max_loop) - } - for category in sorted(all_ifgs.keys()) - } - - for category in cat: - loop[category] = { - } # Initialize the key in the loop dictionary with an inner dictionary - missing[category] = [] # second ver - for l in range( - max_loop - ): # it was up to 3 in the first version. but this allows to go for longer loops e.g. 288-72 (cat[11] is 72) - loop[category][cat[l]] = [ - ] # Initialize the key in the inner dictionary - # missing[cat[l]] = [] # first ver - # Calculate total iterations for progress percentage - total_iterations = sum( - len(all_ifgs[cat[i]]) for l in range(max_loop) - for i in range(1, len(cat)) - if cat[i] % cat[l] == 0 and cat[i] != cat[l]) - completed_iterations = 0 - - # Display the message before starting the progress bar - print( - f"Calculating all {interval}, {2 * interval}, and {3 * interval} loop closures." - ) - - for l in range( - 0, max_loop - ): # l could be 6, 12 and 18 to calculate e.g. 36-6 or 36-12 or 36-18 - for i in range(1, len(cat)): # index for the category e.g. 6, 12, 18 - if cat[i] % cat[l] == 0 and cat[i] != cat[ - l]: # such as 12/6 or 24/12 - for t in range(len(all_ifgs[ - cat[i]])): # index for the epochs in each category - - ####################progress bar - completed_iterations += 1 - percentage = (completed_iterations / - total_iterations) * 100 - - # Display progress percentage - sys.stdout.write("\rProgress: [{:<50}] {}%".format( - "=" * int(percentage // 2), int(percentage))) - sys.stdout.flush() - ################################## - - if cat[l] == interval: - end_index = t + int(cat[i] / cat[l]) - # print('end_index = ', end_index) - else: - end_index = t + int(cat[i] / cat[l]) * int( - cat[l] / interval) - - # recording the missing ifgs in each cat e.g. 6,12,18 - #if np.any(np.array(all_ifgs[cat[i]])[t] == None): - if np.any(np.array(all_ifgs[cat[i]], dtype=object)[t] == None): - missing[cat[i]].append( - t - ) # to record the missing ifgs index for each cateogory e.g. 18 - - - - # --- Check for overshoot(not enough data at the tail). this allows the calculate the missing in the next for loop - if end_index > len(all_ifgs[cat[l]]): - # Not enough short IFGs to form a valid closure → skip this loop - loop[cat[i]][cat[l]].append(None) - missing_counts[cat[i]][cat[l]] += 1 - continue - - - - for e in range(t, end_index, int(cat[l]/interval)): # to record the missing ifgs index for the period of each loop - #print(f" Long baseline (cat[i]): {cat[i]}") - #print(f" Short baseline (cat[l]): {cat[l]}") - #print(f" Index t: {t}, e: {e}, end_index: {end_index}") - #print(f" len(all_ifgs[cat[l]]): {len(all_ifgs[cat[l]])}") - - elem = all_ifgs[cat[l]][e] - - if elem is None: - missing[cat[l]].append(e) - - - - if all_ifgs[cat[i]][t] is None or any(all_ifgs[cat[l]][elem] is None for elem in range(t, end_index, int(cat[l] / interval))): - - loop[cat[i]][cat[l]].append(None) - missing_counts[cat[i]][cat[l]] += 1 # Increment missing loop counter - - else: - closure = np.angle( np.exp( 1j * ( np.array(all_ifgs[cat[i]][t], dtype=np.float32)- np.sum( np.array(all_ifgs[cat[l]][t:end_index:int(cat[l] / interval)], dtype=np.float32), axis=0)))) - - if cat[i] in loop: - loop[cat[i]][cat[l]].append(closure) - loop_counts[cat[i]][cat[l]] += 1 # Increment successful loop counter - - else: - loop[cat[i]][cat[l]] = [closure] - print("\n Finished calculating the loop closures.") - - # Generate report - print("\n=== Loop Closure Report ===") - for category in [ - 2 * interval, - 3 * interval, - ]: # Restrict to relevant categories e.g. 18-6 and 12-6 as the main observations - for l in [interval]: # Restrict to the loop levels of interest - if category in loop_counts and l in loop_counts[category]: - generated = loop_counts[category].get(l, 0) - missing = missing_counts[category].get(l, 0) - print( - f"Loop Closure {category} - {l}: Generated = {generated}, Missing = {missing}" - ) - - print("\n=== Loop Closure Report for Long-interval Loop Closures ===") - - # Iterate over categories greater than 200 - for category in cat: - if category > 200: # Check for long categories - for l in range( - 1, - 5 * interval): # Check loop levels less than 5 * interval - if category in loop_counts and l in loop_counts[category]: - generated = loop_counts[category].get(l, 0) - if generated >= 1: # Only report if at least one loop is generated - print( - f"Loop Closure {category} - {l}: Generated = {generated}" - ) - - return loop, missing - - -################################################################################################ - -all_loops, missing = loop_calc(max_loop) - -# Define the directory path -directory_path = os.path.join(output_path, "Data") - -# Create the directory if it doesn't exist -os.makedirs(directory_path, exist_ok=True) - -# Define the file path -file_path = os.path.join(directory_path, f"All_loops_{start}_{end}.pkl") - -with open(file_path, "wb") as file: # for writting a dictionary to a file - pickle.dump(all_loops, file) - -### - -# Define the file path -file_path = os.path.join(directory_path, f"missing_loops") - -with open(file_path, "wb") as file: # for writting a dictionary to a file - pickle.dump(missing, file) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_03_calibration_pars.py b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_03_calibration_pars.py deleted file mode 100644 index 63c5c34..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_03_calibration_pars.py +++ /dev/null @@ -1,424 +0,0 @@ -#!/usr/bin/env python3 - -import numpy as np -import matplotlib.pyplot as plt -from matplotlib.gridspec import GridSpec -import pickle -import os -import sys -import glob -import configparser -import warnings - -with warnings.catch_warnings(): - warnings.simplefilter("ignore", category=DeprecationWarning) - -################################################################################ - -HELP_TEXT = """ -PhaseBias_03_calibration_pars.py - -This script estimates the calibration parameters a_n using the loop closures calculated in the previous step. - -The calibration parameters a_n are estimated by calculating the ratio of the long-term loop closure for the interferogram of interest to the long-term loop closure for the base interferogram. Assuming negligible bias in long-term interferograms, a_n is calculated as follows: - - a_n = |φ(i,i+l) - Σ(t=i to i+l)φ(t,t+1+n)|_2π / |φ(i,i+l) - Σ(t=i to i+l)φ(t,t+1)|_2π - -where: -- φ(i,i+l) is a long-term interferogram connecting epoch i to i+l. -- φ(t,t+1) represents the base interferogram. -- φ(t,t+1+n) is the short interferogram to be corrected using a_n. - -### Input: -1. Loop closures from the previous step (e.g., 216-6, 216-12, and 216-18 for 6-day acquisition intervals). -2. The long-term interferogram length (e.g., 216 days) must be specified at the beginning. -3. The number of a_n parameters to estimate, specified using `num_a`. For example: - - Setting `num_a=2` indicates that a1 and a2 will be estimated (used to correct 12- and 18-day interferograms). - - For correcting longer interferograms, `num_a` should be set accordingly. - -### Output: -1. All possible a_n values are calculated. -2. Visualizations: - - Example plots of a_n arrays and their corresponding histograms, illustrating spatial variations. - - Time-series plots showing how a_n values evolve over time. -3. Mean a_n values over time are output for further use. - -### File Outputs: -- Results are stored in the output directory defined in the `config.txt` file under `/Data/` as `Mean_an_values.pkl`. - -### Additional Notes: -- Users should ensure that `num_a` aligns with the desired temporal baselines for correction. - -""" -if "--help" in sys.argv: - print(HELP_TEXT) - sys.exit(0) - - -############################################################################### -# Config file path -config_file = "config.txt" - -# Check if config file exists -if not os.path.exists(config_file): - raise FileNotFoundError( - f"Error: The configuration file '{config_file}' was not found in the script's directory." - ) - -# Initialize the configparser -config = configparser.ConfigParser() - - -# Read the config.txt file -config.read(config_file) - -# Define required parameters -required_parameters = [ - "root_path", - "frame", - "start", - "end", - "interval", - "nlook", - "LiCSAR_data", -] - -# Initialize a dictionary to store parameters -parameters = {} - -# Attempt to retrieve each parameter and catch errors -try: - parameters["root_path"] = config.get("DEFAULT", "root_path") - parameters["output_path"] = config.get("DEFAULT", "output_path") - parameters["frame"] = config.get("DEFAULT", "frame") - parameters["start"] = config.get("DEFAULT", "start") - parameters["end"] = config.get("DEFAULT", "end") - parameters["interval"] = config.getint("DEFAULT", "interval") - parameters["nlook"] = config.getint( - "DEFAULT", "nlook" - ) # Assuming 'nlook' should be an integer - parameters["num_a"] = config.getint("DEFAULT", "num_a") - parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") - -except configparser.NoOptionError as e: - # Error message if a required parameter is missing - missing_param = str(e).split(": ")[1] - raise ValueError( - f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." - ) -except configparser.Error as e: - # General error for any configparser-related issues - raise ValueError(f"Error while reading '{config_file}': {e}") - -# Assign to individual variables for easy access -root_path = parameters["root_path"] -output_path = parameters["output_path"] -frame = parameters["frame"] -start = parameters["start"] -end = parameters["end"] -interval = parameters["interval"] -nlook = parameters["nlook"] -num_a = parameters["num_a"] -LiCSAR_data = parameters["LiCSAR_data"] - -# hardcoded parameters, not included in the config file -landmask = 1 -filtered_ifgs = "yes" -max_loop = 5 # in case of 3 all loops with 6,12,18-day will be calculated (e.g. (60,6), (60,12) and (60,18)) - - -##################################################### Loop Closure calculation ##################################################### -#################################################################################################################################### -track = frame[0:3] -if track[0] == "0": - track = track[1:3] -if track[0] == "0": - track = track[1:2] - - -###################### Read coh: -print("\nReading all coherence data...") -# Define the path components -sub_dir = "Data" -filename_pattern = "All_coh_*" # Pattern to match files starting with "All_ifgs" - -# Construct the full file path with the refined pattern -file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) -# Use glob to find the file and ensure it exists -file_list = glob.glob(file_path_pattern) -if file_list: - file_path = file_list[0] # Get the single file path directly - - # Load the data from the file - with open(file_path, "rb") as file: - coh = pickle.load(file) -else: - raise FileNotFoundError( - f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." - ) -print("Reading coherence data completed.") - - -##################### Read Loops - -print("\nReading all loop closures...") -filename_pattern = "All_loops_*" # Pattern to match files starting with "All_ifgs" - -# Construct the full file path with the refined pattern -file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) -# Use glob to find the file and ensure it exists -file_list = glob.glob(file_path_pattern) -if file_list: - file_path = file_list[0] # Get the single file path directly - - # Load the data from the file - with open(file_path, "rb") as file: - all_loops = pickle.load(file) -else: - raise FileNotFoundError( - f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." - ) - -print("Reading loop closures completed.") - - -##################################################### Loop Closure calculation ##################################################### -#################################################################################################################################### - - -long_baseline = 216 # 36,72,108,144,180,216,252,288,324 -coh_thresh = 25 - - -def calc_an(loop_360_n, loop_360_6): - """ - Calculates the a_n values from loop closures. - - Parameters: - loop_360_n (ndarray): Array of loop closures for n-day intervals. - loop_360_6 (ndarray): Array of loop closures for 6-day intervals. - - Returns: - ndarray: Mean a_n values for all loop closures. - list: List of all a_n arrays for further processing or plotting. - """ - ratios = [] - - # Loop through indices - for i in range(len(loop_360_6)): - if loop_360_6[i] is not None and loop_360_n[i] is not None: - # Calculate the ratio for non-NaN elements - ratio_i = np.divide( - loop_360_n[i], - loop_360_6[i], - out=np.full_like(loop_360_n[i], np.nan), - where=~np.isnan(loop_360_6[i]), - ) - ratios.append(ratio_i) - -# Check if `ratios` is empty before proceeding - if not ratios: - print("Warning: No valid a_n values could be calculated. Returning NaN values.") - return np.array([np.nan]), [] - - - # Convert the list of ratios to a numpy array - an = np.array(ratios, dtype=object) - mean_an_long_baseline = [] - - # Calculate mean values of each a_n - for arr in an: - mask = (arr > 0) & (arr < 1) # Apply mask to filter invalid values - arr[~mask] = np.nan - if not np.isnan(arr).all(): # Check if all elements are NaN - mean_an = np.nanmean(arr) - else: - mean_an = np.nan # Assign NaN for completely empty slices - - mean_an_long_baseline.append(mean_an) - - return np.array(mean_an_long_baseline), an - - -################################################################################## -############################ -def plot_an(an_arrays, mean_a_long_baseline, k): - """ - Plots the a_n arrays with the largest coverage and their histograms in a two-row layout, - as well as the mean values of calibration parameters a_n over time. - - Parameters: - an_arrays (list): List of a_n arrays. - mean_a_long_baseline (ndarray): Array of mean calibration parameter values for a_1, a_2, ..., a_n. - k (int): The current value of k (e.g., 2 for a1, 3 for a2). - """ - - # Calculate the coverage (number of non-NaN values) for each array - coverage = [] - valid_an_arrays = [] # Store only valid arrays for plotting - - for arr in an_arrays: - if ( - arr is not None and arr.ndim == 2 and not np.all(np.isnan(arr)) - ): # Check for valid 2D arrays - coverage.append(np.sum(~np.isnan(arr))) # Count non-NaN values - valid_an_arrays.append(arr) - - # Sort arrays by coverage in descending order - sorted_indices = np.argsort(coverage)[::-1] - selected_arrays = [ - valid_an_arrays[idx] for idx in sorted_indices[:6] - ] # Select top 6 arrays - - # Plot the a_n arrays and their histograms in a two-row layout - if selected_arrays: - num_arrays = len(selected_arrays) - fig = plt.figure(figsize=(4 * num_arrays, 8)) - gs = GridSpec( - 2, num_arrays + 1, width_ratios=[1] * num_arrays + [0.1] - ) # Add space for colorbar - - # Plot selected arrays - axes_img = [] - for i, arr in enumerate(selected_arrays): - ax_img = fig.add_subplot(gs[0, i]) - im = ax_img.imshow(arr, cmap="RdYlBu", vmin=-0.8, vmax=0.8) - ax_img.set_title(f"{i+1}th $a_{{{k-1}}}$ Array") # Dynamically set title - ax_img.axis("off") - axes_img.append(ax_img) - - # Add a single colorbar for all array plots - cbar_ax = fig.add_subplot(gs[0, -1]) - fig.colorbar(im, cax=cbar_ax, orientation="vertical", label="Color Scale") - - # Plot histograms for the selected arrays - for i, arr in enumerate(selected_arrays): - ax_hist = fig.add_subplot(gs[1, i]) - ax_hist.hist(arr.flatten(), bins=100, color="blue", alpha=0.7, density=True) - ax_hist.set_title( - f"{i+1}th $a_{{{k-1}}}$ Histogram" - ) # Dynamically set title - ax_hist.set_xlim([-3, 3]) # Adjust histogram range as needed - - plt.tight_layout() - plt.show() - else: - print("No valid a_n arrays available for plotting.") - - # Plot the mean values of a_n over time - x_values = np.arange(len(mean_a_long_baseline)) # Sequential indices for x-axis - fig, ax = plt.subplots(figsize=(6, 4)) - - - # Filter out NaN values to avoid gaps in the plot - valid_indices = ~np.isnan(mean_a_long_baseline) # Mask for valid (non-NaN) entries - x_valid = x_values[valid_indices] # Only use valid time steps - y_valid = np.array(mean_a_long_baseline)[valid_indices] # Only use valid a_n values - - - # Remove NaN values for trendline fitting - valid_indices = ~np.isnan(mean_a_long_baseline) # Mask for valid (non-NaN) entries - if ( - valid_indices.sum() > 1 - ): # Ensure there are at least two valid points for fitting -# x_valid = x_values[valid_indices] - y_valid = np.array(mean_a_long_baseline)[valid_indices] - x_valid = np.arange(len(y_valid)) - - # Fit and plot a trendline - coefficients = np.polyfit(x_valid, y_valid, 1) - trendline = np.polyval(coefficients, x_valid) - ax.plot(x_valid, trendline, color="red", label="Trendline") - - # Scatter plot for all points (including NaN if any) - ax.scatter( - #x_values, - #mean_a_long_baseline, - x_valid, # Only use valid time steps - y_valid, # Only use valid a_n values - - color="blue", - marker="o", - label="Mean Values", - s=9, - ) - - # Set labels and limits - ax.set_ylim(-3, 3) # Adjust limits as needed - ax.set_xlabel("Time Step") # Sequential index as x-axis - ax.set_ylabel("Mean Values", fontsize=12) - ax.legend() - - # Adjust layout and show the plot - plt.tight_layout() - plt.show() - - -## -######### forming 360-6 - -loop_360_6_all = np.array(all_loops[long_baseline][interval], dtype=object) -coh_360 = np.array(coh[long_baseline][:], dtype=object) - -loop_360_6 = np.full_like( - loop_360_6_all, np.nan -) # create a new array as the same size of loop_360_6_all with nan values - - -for i, arr in enumerate(loop_360_6_all): - if arr is not None: - (frame_row, frame_col) = np.shape(arr) - loop_360_6[i] = np.where((coh_360[i] > coh_thresh), arr, np.nan) - -###### forming 360-n (n=12,18,24 etc) -# mean_a_long_baseline = [] -final_mean_values = [] -an_labels = [] - -# Loop through the values of k corresponding to the number of calibration parameters -for k in range(2, 2 + num_a): # k=2 for a1, k=3 for a2, etc. - mean_an_long_baseline = [] - - # Forming 360-n (n=12, 18, 24, etc.) - loop_360_n_all = np.array(all_loops[long_baseline][k * interval], dtype=object) - loop_360_n = np.full_like(loop_360_n_all, np.nan) - - for i, arr in enumerate(loop_360_n_all): - if arr is not None: - # Apply coherence threshold filtering - loop_360_n[i] = np.where((coh_360[i] > coh_thresh), arr, np.nan) - - # Calculate a_n values - mean_an_long_baseline, an_arrays = calc_an(loop_360_n, loop_360_6) - - # If all values in mean_an_long_baseline are nan (empty ratios), stop execution - if np.isnan(mean_an_long_baseline).all(): - print("No valid a_n values could be calculated. Stopping execution.") - sys.exit(1) # Exit the script with an error code (1 means failure) - - # Calculate the mean of all mean values for this k and store it - overall_mean = np.nanmean(mean_an_long_baseline) - final_mean_values.append(overall_mean) - an_labels.append(f"a{k-1}") - - for idx, mean_a in enumerate(mean_an_long_baseline, start=1): - if not np.isnan(mean_a): # Check if the value is not NaN - print(f"Mean of the {idx}th a({k-1}) is {mean_a:.6f}") - - plot_an(an_arrays, mean_an_long_baseline, k) - - -# Print the final mean values across all k -for k_idx, final_mean in enumerate(final_mean_values, start=2): - print(f"Final mean value for a({k_idx-1}) is {final_mean:.6f}") - -# Write final mean values to a text file -output_path = os.path.join(parameters["output_path"], "Data") -os.makedirs(output_path, exist_ok=True) # Create directory if it doesn't exist -output_file = os.path.join(output_path, "an.txt") - -with open(output_file, "w") as file: - for label, mean_value in zip(an_labels, final_mean_values): - file.write(f"{label}={mean_value:.6f}\n") - -print(f"Final mean values written to {output_file}.") diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_04_Inversion.py b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_04_Inversion.py deleted file mode 100644 index 2913c3e..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_04_Inversion.py +++ /dev/null @@ -1,683 +0,0 @@ -#!/usr/bin/env python3 - -import numpy as np -import pickle -import os -from scipy.sparse import csc_array -from scipy.sparse.linalg import lsqr -import sys -import math -from bin.read_orig_ifgs_coh import read_orig_ifgs_coh -from bin.circular_mean_var import circular_mean_and_variance_over_epochs -import glob -import configparser -import warnings - -warnings.filterwarnings("ignore", category=DeprecationWarning) - -################################################################################ - -HELP_TEXT = """ -PhaseBias_04_Inversion.py - -This script estimates the phase bias terms for the base interferograms using short-term closure phases as input. -The bias terms derived in this step serve as the foundation for correcting other short-term interferograms. - -### Workflow: -1. **Masking Noisy Loop Closures:** - - A moving average is applied to the time-series of loop closures to account for seasonal fluctuations. - - A circular moving average (mean of complex values over time) is computed for each pixel. - - The distance of each loop closure from the moving average is calculated. If the distance exceeds a threshold (2σ), the point is masked as noisy. - - The circular standard deviation (σ) is computed using the von Mises distribution (circular normal distribution). - -2. **Refined Loop Closures:** - - Masked loop closures, denoted as 〖Δφ〗_(i,i+2)^r and 〖Δφ〗_(i,i+3)^r, are used as primary observations for phase bias estimation. - -3. **First Inversion (Without Temporal Smoothing Constraints):** - - Estimates the bias terms that can be corrected directly from the observed loop closures. - -4. **Second Inversion (With Temporal Smoothing Constraints):** - - Incorporates temporal smoothing constraints to estimate additional bias terms that cannot be directly derived from loop closures. - - The temporal smoothing minimizes differences between bias terms over time, ensuring consistency. - -5. **Combining Results:** - - Bias terms from the first inversion are combined with the results of the second inversion to include terms not estimated in the first round. - -### Outputs: -- The phase bias terms for the base interferograms are stored as a NumPy array file: - - `X_base_ifgs_biases.npy`, saved in the `Data` directory under the `output_path` defined in `config.txt`. - - -### Input Requirements: -- Short-term closure phases (from `PhaseBias_02_Loop_Closures.py`). -- Configuration parameters, including thresholds and temporal settings, defined in `config.txt`. - -### Output Files: -- `X_base_ifgs_biases.npy`: Stores the estimated bias terms for the base interferograms. -""" - -if "--help" in sys.argv: - print(HELP_TEXT) - sys.exit(0) - -############################################################################### -# Config file path -config_file = "config.txt" - -# Check if config file exists -if not os.path.exists(config_file): - raise FileNotFoundError( - f"Error: The configuration file '{config_file}' was not found in the script's directory." - ) - -# Initialize the configparser -config = configparser.ConfigParser() - -# Read the config.txt file -config.read(config_file) - -# Define required parameters -required_parameters = [ - "root_path", - "frame", - "start", - "end", - "interval", - "nlook", - "LiCSAR_data", - "a1_6_day", - "a2_6_day", - "a1_12_day", - "a2_12_day", - "estimate_an_values", -] - -# Initialize a dictionary to store parameters -parameters = {} - -# Attempt to retrieve each parameter and catch errors -try: - parameters["root_path"] = config.get("DEFAULT", "root_path") - parameters["output_path"] = config.get("DEFAULT", "output_path") - parameters["frame"] = config.get("DEFAULT", "frame") - parameters["start"] = config.get("DEFAULT", "start") - parameters["end"] = config.get("DEFAULT", "end") - parameters["interval"] = config.getint("DEFAULT", "interval") - parameters["nlook"] = config.getint( - "DEFAULT", "nlook" - ) # Assuming 'nlook' should be an integer - parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") - parameters["a1_6_day"] = config.getfloat("DEFAULT", "a1_6_day") - parameters["a2_6_day"] = config.getfloat("DEFAULT", "a2_6_day") - parameters["a1_12_day"] = config.getfloat("DEFAULT", "a1_12_day") - parameters["a2_12_day"] = config.getfloat("DEFAULT", "a2_12_day") - parameters["estimate_an_values"] = config.get("DEFAULT", "estimate_an_values") - -except configparser.NoOptionError as e: - # Error message if a required parameter is missing - missing_param = str(e).split(": ")[1] - raise ValueError( - f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." - ) -except configparser.Error as e: - # General error for any configparser-related issues - raise ValueError(f"Error while reading '{config_file}': {e}") - -# Assign to individual variables for easy access -root_path = parameters["root_path"] -output_path = parameters["output_path"] -frame = parameters["frame"] -start = parameters["start"] -end = parameters["end"] -interval = parameters["interval"] -nlook = parameters["nlook"] -LiCSAR_data = parameters["LiCSAR_data"] -estimate_an_values = parameters["estimate_an_values"] - - -# Initialize a1 and a2 with None for safety -a1 = None -a2 = None - - -print("estimate_an_values is ", estimate_an_values) -# Check if estimate_an_values is 'yes' -if estimate_an_values == "yes": - an_file_path = os.path.join(output_path, "Data", "an.txt") - - # Try reading the file - try: - if os.path.exists(an_file_path): - print(f"Reading 'a' values from {an_file_path}...") - - # Read the file and load the values - with open(an_file_path, "r") as file: - for line in file: - line = line.strip() - if line.startswith("a1="): - a1 = float(line.split("=")[1]) - elif line.startswith("a2="): - a2 = float(line.split("=")[1]) - - # Check if both a1 and a2 were found - if a1 is None or a2 is None: - print( - "Warning: Missing a1 or a2 in an.txt. Switching to default values from config file." - ) - raise ValueError("Incomplete a1 or a2 values.") - else: - print( - f"Warning: {an_file_path} does not exist. Switching to default values from config file." - ) - raise FileNotFoundError - - except (FileNotFoundError, ValueError): - # Fallback to default values from config file - if interval == 6: - a1 = parameters["a1_6_day"] - a2 = parameters["a2_6_day"] - else: - a1 = parameters["a1_12_day"] - a2 = parameters["a2_12_day"] -else: - # Use default values from the config file - print("Using default 'an' values from the config file.") - if interval == 6: - a1 = parameters["a1_6_day"] - a2 = parameters["a2_6_day"] - else: - a1 = parameters["a1_12_day"] - a2 = parameters["a2_12_day"] - -# Print the chosen values -print(f"Final values - a1: {a1}, a2: {a2}") - - -##################################################### Loop Closure calculation ##################################################### -#################################################################################################################################### -#################################################################################################################################### - -track = frame[0:3] -if track[0] == "0": - track = track[1:3] -if track[0] == "0": - track = track[1:2] - -with_temporals = "yes" # it should be yes, if you want to incorporate the temporal constraints -apply_to_all = "no" # in case of with_temporal='yes', we can decide if you want to use the temporal to all ifgs (i.e. apply_to_all='yes'), or just to those that can not be corrected (i.e. apply_to_all='no') -w = 0.1 # the weigth of the temporal smoothing constraints -####### -coh_thresh = 11 # threhsold on the coherence values. In this version it is off -min_num_eq = 10 # I used 10 in all my experiments. min number of equations in the least square inversion. - -############################### Reading input data ############################################# -#################### Read ifgs: -print("Reading all ifgs...") -# Define the path components -sub_dir = "Data" -filename_pattern = "All_ifgs_*" # Pattern to match files starting with "All_ifgs" - -# Construct the full file path with the refined pattern -file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) -# Use glob to find the file and ensure it exists -file_list = glob.glob(file_path_pattern) -if file_list: - file_path = file_list[0] # Get the single file path directly - - # Load the data from the file - with open(file_path, "rb") as file: - all_ifgs = pickle.load(file) -else: - raise FileNotFoundError( - f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." - ) -print("Reading ifgs completed.") - -###################### Read coh: -print("\nReading all coherence data...") -# Define the path components -sub_dir = "Data" -filename_pattern = "All_coh_*" # Pattern to match files starting with "All_ifgs" - -# Construct the full file path with the refined pattern -file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) -# Use glob to find the file and ensure it exists -file_list = glob.glob(file_path_pattern) -if file_list: - file_path = file_list[0] # Get the single file path directly - - # Load the data from the file - with open(file_path, "rb") as file: - all_coh = pickle.load(file) -else: - raise FileNotFoundError( - f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." - ) -print("Reading coherence data completed.") - -##################### Read Loops - -print("\nReading all loop closures...") -filename_pattern = "All_loops_*" # Pattern to match files starting with "All_ifgs" - -# Construct the full file path with the refined pattern -file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) -# Use glob to find the file and ensure it exists -file_list = glob.glob(file_path_pattern) -if file_list: - file_path = file_list[0] # Get the single file path directly - - # Load the data from the file - with open(file_path, "rb") as file: - all_loops = pickle.load(file) -else: - raise FileNotFoundError( - f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." - ) - -print("Reading loop closures completed.") - -all_cat = [] -for cat in all_loops: - all_cat.append(cat) - -### stacking all the ifgs 6/12/18 in a numpy var all_ifgs -print( - "Extracting the required interferograms(up to 3 epochs), and the corresponding loop closures for inversion..." -) -desired_categories = [interval, 2 * interval, 3 * interval] -all_ifgs, _, existing_ifgs_index = read_orig_ifgs_coh( - all_ifgs, all_coh, desired_categories -) - -######### forming 12-6 and 18-6 loop closures from the imported data -################################################################# -loop_12 = np.array(all_loops[2 * interval][interval], dtype=object) -#loop_12 = np.array(all_loops[2 * interval][interval]) -#coh_12 = np.array(all_coh[2 * interval][:]) -coh_12 = np.array(all_coh[2 * interval][:], dtype=object) - -#loop_18 = np.array(all_loops[3 * interval][interval]) -#coh_18 = np.array(all_coh[3 * interval][:]) -loop_18 = np.array(all_loops[3 * interval][interval], dtype=object) -coh_18 = np.array(all_coh[3 * interval][:], dtype=object) - -############## finding dynamic pixel-based thresholds to remove the noisy loops -loop_12_orig = [] -loop_18_orig = [] -none_indices12 = [] -none_indices18 = [] - -# refine loop_12 to exclude the none indices and create loop_12_orig -for i, arr in enumerate(loop_12): - if arr is not None: - (frame_row, frame_col) = np.shape(arr) - loop_12_orig.append(arr) - else: - none_indices12.append(i) - -# refine loop_18 to exclude the none indices and create loop_18_orig -for i, arr in enumerate(loop_18): - if arr is not None: - loop_18_orig.append(arr) - else: - none_indices18.append(i) - -loop_12_orig = np.array(loop_12_orig) -loop_18_orig = np.array(loop_18_orig) - - -### removing noisy loops -print( - "Masking noisy loop closures using circular moving average and standard deviaiton of the loop closures in time:" -) - -# using circular_mean to calculate the mean phase value using complex numbers, and von_mises_variance to calculate the second moment of the Von Mises distribution -_, thresh_loop12 = circular_mean_and_variance_over_epochs(loop_12_orig, axis=0) -_, thresh_loop18 = circular_mean_and_variance_over_epochs(loop_18_orig, axis=0) - - -# Add a new axis at the beginning (axis=0) of these arrays to have shape (1, ...) -# These are the threshold vlaues obtained for each pixel that will be used for masking later -thresh_loop12 = np.expand_dims(thresh_loop12, axis=0) -thresh_loop18 = np.expand_dims(thresh_loop18, axis=0) - - -# calculate a temporal moving average for wrapped data -def moving_average(arr, window_size): - # Initialize an empty array to store the temporal averages - moving_averages = np.zeros_like(arr) - half_window = window_size // 2 - ### miroring data at both ends for window_size/2 at each end - mirrored_shape = (arr.shape[0] + window_size, arr.shape[1], arr.shape[2]) - - # Create an empty array to hold the mirrored data - mirrored_arr = np.empty(mirrored_shape, dtype=arr.dtype) - - # Fill the mirrored array with the original data - mirrored_arr[window_size // 2 : -window_size // 2] = arr - - # Mirror the data at the beginning - for i in range(window_size // 2): -# mirrored_arr[i] = arr[window_size // 2 - i] - mirrored_arr[i] = arr[min(half_window - i, arr.shape[0] - 1)] - - # Mirror the data at the end - for i in range(window_size // 2): -# mirrored_arr[-(i + 1)] = arr[-(i + 1) - window_size // 2] - mirrored_arr[-(i + 1)] = arr[max(-1 - (half_window - i), 0)] - #### - - # Calculate temporal averages within each window - for t in range(half_window, mirrored_arr.shape[0] - half_window): - moving_averages[t - half_window, :, :] = np.angle( - np.nanmean( - np.exp(1j * (mirrored_arr[t - half_window : t + half_window, :, :])), - axis=0, - ) - ) - return moving_averages - - -# for loop 12 -# Calculate distance between each epoch and the moving average for each pixel -print( - f"Calculating the moving average for {2 * interval} loop closures, followed by masking noisy loop closures..." -) -print( - "This may take a while, as it depends on the number of loops and the size of your dataset.\n" -) - -window_size = 30 - -moving_avg = moving_average(loop_12_orig, window_size) -distances = np.abs(np.angle(np.exp(1j * (loop_12_orig - moving_avg)))) -mask = distances > 2 * thresh_loop12 # it is 4*sigma - -# Replace values with NaN where mask is True -loop_12_bk = loop_12_orig -loop_12_orig = np.where(mask, np.nan, loop_12_orig) -loop_12 = loop_12_orig.tolist() - -for idx in none_indices12: - loop_12.insert(idx, None) -loop_12 = np.array(loop_12, dtype=object) - -# for loop 18 -# Calculate distance between each epoch and the moving average for each pixel -print( - f"Calculating the moving average for {3 * interval} loop closures, followed by masking noisy loop closures..." -) -print( - "This may take a while, as it depends on the number of loops and the size of your dataset.\n" -) - -moving_avg = moving_average(loop_18_orig, window_size) -distances = np.abs(np.angle(np.exp(1j * (loop_18_orig - moving_avg)))) -mask = distances > 2 * thresh_loop18 # it is 4*sigma - -# Replace values with NaN where mask is True -loop_18_bk = loop_18_orig -loop_18_orig = np.where(mask, np.nan, loop_18_orig) -loop_18 = loop_18_orig.tolist() - -for idx in none_indices18: - loop_18.insert(idx, None) -loop_18 = np.array(loop_18, dtype=object) - -print("Masking of noisy loop closures is completed.") - -# ################ Apply coherece thresholding -# print('shape loop_12 = ', np.shape(loop_12)) -# print('shape loop_12_orig = ', np.shape(loop_12_orig)) -# print('shape coh_12 = ', np.shape(coh_12)) -# for i, arr in enumerate(loop_12): -# if arr is not None: -# (frame_row, frame_col) = np.shape(arr) -# loop_12[i] = np.where((coh_12[i] > coh_thresh), arr, np.nan) -# -# -# for i, arr in enumerate(loop_18): -# if arr is not None: -# loop_18[i] = np.where((coh_18[i] > coh_thresh), arr, np.nan) -# -# -# - -len12 = len(loop_12) # number of 12 day loops including None -len18 = len(loop_18) # number of 18 day loops including None - -######################## Forming the design matrix A ############################## -#################################################################################### - -print("Preparing the design matrix and observation vector for the inversion step...") - -b = [] -A = [] -n_unk = ( - len12 + 1 -) # this in an initial value(i.e. the num of 6 biases). will be changed by the actual number of unk after considering the None values/missing -# print('n_unk = ', n_unk) - -row = np.zeros( - n_unk, dtype=np.float32 -) # each row of the design matrix A for the 12-day -for i in range(len12): - b.append(loop_12[i]) - # print('np.shape(loop_12[i] = ', np.shape(loop_12[i])) - row[i : i + 2] = a1 - 1 - A.append(row) - row = np.zeros(n_unk, dtype=np.float32) - -row = np.zeros( - n_unk, dtype=np.float32 -) # each row of the design matrix A for the 18-day -for i in range(len18): - b.append(loop_18[i]) - row[i : i + 3] = a2 - 1 - A.append(row) - row = np.zeros(n_unk, dtype=np.float32) - -A = np.array(A) -b = np.array(b, dtype=object) - - -####################### removing the rows from b and A where b is None -mask = [] -mask = np.array([arr is not None for arr in b]) -# Convert the boolean mask to an integer mask -mask = np.nonzero(mask) - -A = A[mask] # Filter rows of A based on the mask -b = b[mask] - -b = b.tolist() - -############ removing the columns of A where are values are zero (these are the unkonws which doesn't fall in any equations nor 12 neither 18 and thus cannot be corrected) - -# Find the column indices where all values are zero -zero_columns = np.all(A == 0, axis=0) - -## Get the indices of the zero columns -zero_column_indices = np.where(zero_columns)[0] - -# Get the indices of the non-zero columns -non_zero_column_indices = np.where(~zero_columns)[ - 0 -] # the indices of the unknowns that can be corrected - -# Save the file -directory_path = os.path.join(output_path, "Data") -file_path = os.path.join(directory_path, f"indices_unknown_tobe_corrected.npy") -np.save(file_path, non_zero_column_indices) - - -### Adding all the existing 6-day ifgs indices to non_zero_column_indices. We want to use estimate their biases in the second step. - - -A_bk = A # to keep a copy of A in case we want to esimate all 6-day biases with smoothing constraints -b_bk = b -all_column_indices = np.array( - range(A_bk.shape[1]) -) # generating all indices in case we want to esimate all 6-day biases with smoothing constraints - -A = A[:, non_zero_column_indices] - -# finding the last column with value -1. this gives the number of 6-day biases that can be corrected -last_column_with_minus_1 = None - -# Iterate through the columns from right to left -for col in range(len(A[0]) - 1, -1, -1): - if -1 in [row[col] for row in A]: - last_column_with_minus_1 = col - break - -num_rows_before_temporals = A.shape[0] - -################################# Least Square inversion ############################### -######################################################################################## -# First Inversion: Without Temporal Smoothing Constraints -print("Starting the first inversion (without temporal smoothing constraints).") -print( - "This step estimates the bias terms that can be corrected based on observed loop closures." -) - -damp_factor = 0 - -if apply_to_all == "no": # this is without using any temporal constraints - - X1 = np.zeros((np.shape(A_bk)[1], frame_row, frame_col), dtype=np.float32) - # X1[:,:,:] = np.nan # after checking noticed this doesn't have any effect on the final vel - b_bk = np.array(b_bk) - - total_pixels = frame_row * frame_col # Total number of pixels for progress bar - processed_pixels = 0 # Counter for processed pixels - print("Progress: [", end="", flush=True) - - for row in range(frame_row): - for col in range(frame_col): - processed_pixels += 1 - # Update progress bar - percentage = (processed_pixels / total_pixels) * 100 - if processed_pixels % (total_pixels // 100) == 0: # Update every 1% - print(f"{int(percentage)}%", end="", flush=True) - sys.stdout.write( - "\rProgress: [" - + "=" * (int(percentage) // 2) - + " " * (50 - int(percentage) // 2) - + "]" - ) - - non_nan_mask = ~np.isnan(b_bk[:, row, col]) - bb = b_bk[non_nan_mask, row, col] # Remove NaN values - AA = A_bk[non_nan_mask, :] # Remove corresponding rows - if len(bb) > min_num_eq: # what is the min number of equation? - - x, istop, itn, normr, normr2 = lsqr( - csc_array(AA), bb, damp=damp_factor - )[ - :5 - ] # Using sparse matrix representation scipy.sparse.linalg - X1[:, row, col] = x - - -# if with_temporals == "no": # this solution will be the final answer to be saved as X -# # Fill X with values from X1 according to non_zero_column_indices -# for i, idx in enumerate(non_zero_column_indices): -# X[idx, :, :] = X1[i, :, :] - - - - -print("\nFirst inversion (without temporal smoothing constraints) completed.") - -########################### -# Second Inversion: With Temporal Smoothing Constraints -print("Starting the second inversion (with temporal smoothing constraints).") -print( - "This step estimates all bias terms, including those that cannot be corrected, using temporal smoothing constraints." -) - -if with_temporals == "yes": # This is using the temporal constranints on all unknowns - X2 = np.zeros((np.shape(A_bk)[1], frame_row, frame_col), dtype=np.float32) - # X2[:,:,:] = np.nan # after checking noticed this doesn't have any effect on the final vel - b_bk = np.array(b_bk) - - #### Adding temporal constraints - row = np.zeros( - A_bk.shape[1], dtype=np.float32 - ) # additional rows the design matrix A for temporal constraints, setting that to zero again - for i in range(len(all_column_indices) - 2): # for all unknowns - row[i] = -1 * w - row[i + 1] = 2 * w - row[i + 2] = -1 * w - A_bk = np.vstack((A_bk, row)) - row = np.zeros( - A_bk.shape[1], dtype=np.float32 - ) # additional rows the design matrix A for temporal constraints, setting that to zero again - num_temporals = len(A_bk) - len(b_bk) - # print('num_temporals is ', num_temporals) - - ######### Appending zeroes to b according to the added number of temporal constraints obtained by np.shape(A)[0] - np.shape(b)[0] - b_0 = np.zeros( - (np.shape(A_bk)[0] - np.shape(b)[0], frame_row, frame_col), dtype=np.float32 - ) - b_bk = np.vstack((b_bk, b_0)) - - thresh_n_eq = num_temporals # in case of using temporals, we don't want the nan pixels with zero equations with only using temporals - - processed_pixels = 0 # Reset counter for second inversion - print("Progress: [", end="", flush=True) - - for row in range(frame_row): - for col in range(frame_col): - processed_pixels += 1 - # Update progress bar - percentage = (processed_pixels / total_pixels) * 100 - if processed_pixels % (total_pixels // 100) == 0: # Update every 1% - print(f"{int(percentage)}%", end="", flush=True) - sys.stdout.write( - "\rProgress: [" - + "=" * (int(percentage) // 2) - + " " * (50 - int(percentage) // 2) - + "]" - ) - - non_nan_mask = ~np.isnan(b_bk[:, row, col]) - bb = b_bk[non_nan_mask, row, col] # Remove NaN values - AA = A_bk[non_nan_mask, :] # Remove corresponding rows - if len(bb) > thresh_n_eq: - - x, istop, itn, normr, normr2 = lsqr( - csc_array(AA), bb, damp=damp_factor - )[ - :5 - ] # Using sparse matrix representation scipy.sparse.linalg - X2[:, row, col] = x - - print("\nSecond inversion (with temporal smoothing constraints) completed.") - - ####### combining X1 and X2 into X - - X = np.zeros((np.shape(A_bk)[1], frame_row, frame_col), dtype=np.float32) - X[:, :, :] = np.nan - - # Fill X with values from X1 according to non_zero_column_indices - for i, idx in enumerate(non_zero_column_indices): - X[idx, :, :] = X1[i, :, :] - - # Fill in the missing values from X2 - for i, idx in enumerate(all_column_indices): - if idx not in non_zero_column_indices: - X[idx, :, :] = X2[i, :, :] - - # Ensure the dtype is appropriate for your data - X = X.astype(np.float32) - -directory_path = os.path.join(output_path, "Data") -file_path = os.path.join(directory_path, f"X_base_ifgs_biases.npy") -np.save(file_path, X) - -print("Inversion process completed. Results saved to:", file_path) - -########################################################################################################## diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_05_Correction.py b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_05_Correction.py deleted file mode 100644 index 14cbca0..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_05_Correction.py +++ /dev/null @@ -1,535 +0,0 @@ -#!/usr/bin/env python3 - -import numpy as np -from datetime import datetime -import pickle -import time -import os -from bin.generate_ifgs_from_epochs import generate_ifg_pairs -import configparser -import glob -from bin.resample import resample_geotiff -import sys - -# sys.path.append(os.path.dirname(os.path.abspath(__file__))) -import bin -from bin.read_orig_ifgs_coh import read_orig_ifgs_coh - -from bin.ifg_cc_totif import ifg_cc_2tif -import gc - -############################################################################### - -HELP_TEXT = """ -PhaseBias_05_Correction.py - -This script applies phase bias correction to desired interferograms using the estimated bias terms for the base interferograms. The corrected interferograms are saved in GeoTIFF format for further analysis. - -### Workflow: -1. **Inputs:** - - **Original Wrapped Interferograms, and Coherence Data:** Obtained from step 1 (`PhaseBias_01_Read_Data.py`). - - **Estimated Bias Terms for Base Intierferograms:** Obtained from step 4 (`PhaseBias_04_Inversion.py`). - - **Calibration Parameters (*a_n*):** Estimated from step 3 (`PhaseBias_03_calibration_pars.py`). - - **max_con:** Defines the maximum number of connections to be corrected. For instance: - - `max_con=5` corrects interferograms up to 30-day intervals for a 6-day acquisition interval. - - Adjust `max_con` as needed. - -2. **Bias Term Estimation for Desired Interferograms:** - - Bias terms for desired interferograms, δ_(i,i+n+1), are estimated using the relationship: - δ_(i,i+n+1) = a_n * (∑_(t=i)^(i+n) δ_(t,t+1) ) - -3. **Correction of Interferograms:** - - Using the estimated bias terms, the interferograms are corrected as follows: - φ_(i,i+n+1)^c = φ_(i,i+n+1) - δ_(i,i+n+1) - - - Here, φ_(i,i+n+1) is the original interferogram, and φ_(i,i+n+1)^c is the corrected interferogram. - -4. **Outputs:** - - Corrected interferograms are saved in GeoTIFF format in the `GEOC` directory under the `output_path` specified in `config.txt`. - -### Output File Format: -- **Corrected Interferograms:** Saved in GeoTIFF format, organized by temporal baseline. - -### Input Requirements: -- Original wrapped interferograms from step 1. -- Bias terms for base interferograms from step 4. -- Configuration parameters (e.g., `output_path`, `interval`) defined in `config.txt`. - -### Output Directory: -- `GEOC` directory under `output_path`: Contains the corrected interferograms in GeoTIFF format, organized by temporal baseline. - -""" -if "--help" in sys.argv: - print(HELP_TEXT) - sys.exit(0) - - -################################################################################ - - -# Config file path -config_file = "config.txt" - -# Check if config file exists -if not os.path.exists(config_file): - raise FileNotFoundError( - f"Error: The configuration file '{config_file}' was not found in the script's directory." - ) - -# Initialize the configparser -config = configparser.ConfigParser() - - -# Read the config.txt file -config.read(config_file) - -print("Config keys:", list(config.defaults().keys())) - - -# Define required parameters -required_parameters = [ - "root_path", - "frame", - "start", - "end", - "interval", - "nlook", - "LiCSAR_data", - "a1_6_day", - "a2_6_day", - "a3_6_day", - "a4_6_day", - "a1_12_day", - "a2_12_day", - "a3_12_day", - "a4_12_day", - "estimate_an_values", -] - -# Initialize a dictionary to store parameters -parameters = {} - -# Attempt to retrieve each parameter and catch errors -try: - parameters["root_path"] = config.get("DEFAULT", "root_path") - parameters["output_path"] = config.get("DEFAULT", "output_path") - parameters["frame"] = config.get("DEFAULT", "frame") - parameters["start"] = config.get("DEFAULT", "start") - parameters["end"] = config.get("DEFAULT", "end") - parameters["interval"] = config.getint("DEFAULT", "interval") - parameters["nlook"] = config.getint( - "DEFAULT", "nlook" - ) # Assuming 'nlook' should be an integer - parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") - parameters["a1_6_day"] = config.getfloat("DEFAULT", "a1_6_day") - parameters["a2_6_day"] = config.getfloat("DEFAULT", "a2_6_day") - parameters["a3_6_day"] = config.getfloat("DEFAULT", "a3_6_day") - parameters["a4_6_day"] = config.getfloat("DEFAULT", "a4_6_day") - parameters["a1_12_day"] = config.getfloat("DEFAULT", "a1_12_day") - parameters["a2_12_day"] = config.getfloat("DEFAULT", "a2_12_day") - parameters["a3_12_day"] = config.getfloat("DEFAULT", "a3_12_day") - parameters["a4_12_day"] = config.getfloat("DEFAULT", "a4_12_day") - parameters["estimate_an_values"] = config.get("DEFAULT", "estimate_an_values") - -except configparser.NoOptionError as e: - # Error message if a required parameter is missing - missing_param = str(e).split(": ")[1] - raise ValueError( - f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." - ) -except configparser.Error as e: - # General error for any configparser-related issues - raise ValueError(f"Error while reading '{config_file}': {e}") - -# Assign to individual variables for easy access -root_path = parameters["root_path"] -output_path = parameters["output_path"] -frame = parameters["frame"] -start = parameters["start"] -end = parameters["end"] -interval = parameters["interval"] -nlook = parameters["nlook"] -LiCSAR_data = parameters["LiCSAR_data"] -estimate_an_values = parameters["estimate_an_values"] - - -start_date = datetime.strptime(start, "%Y%m%d") -end_date = datetime.strptime(end, "%Y%m%d") - -uncor_only = 1 # in case of 1 only uses 6-12-18 uncorrected ifgs. it is based on the desired_lengths ifgs -max_con = 5 # in case of 5, it will correct up to 6*5=30-day ifgs. - - -##################################################### -# Root directory where the category directories are located - -if LiCSAR_data == "yes": - track = frame[0:3] # extracting track number from frame id - if track[0] == "0": - track = track[1:3] - if track[0] == "0": - track = track[1:2] - - root_directory = ( - f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}" - ) -else: - root_directory = root_path - - -############################################ - -# Initialize the an values with None for safety -an_values = [None] * ( - max_con - 1 -) # For max_con=5, this will create [None, None, None, None] - -print("estimate_an_values is ", estimate_an_values) - -# Check if estimate_an_values is 'yes' -if estimate_an_values == "yes": - an_file_path = os.path.join(output_path, "Data", "an.txt") - - # Try reading the file - try: - if os.path.exists(an_file_path): - print(f"Reading 'a' values from {an_file_path}...") - - # Read the file and load the an values dynamically - with open(an_file_path, "r") as file: - for line in file: - line = line.strip() - for i in range( - 1, max_con - ): # Loop through a1, a2, ..., a(max_con-1) - param_name = f"a{i}" - if line.startswith(f"{param_name}="): - an_values[i - 1] = float(line.split("=")[1]) - - # Check if all required an values were found - if any(value is None for value in an_values): - print( - "Warning: Missing one or more an values in an.txt. Switching to default values from config file." - ) - raise ValueError("Incomplete an values.") - else: - print( - f"Warning: {an_file_path} does not exist. Switching to default values from config file." - ) - raise FileNotFoundError - - except (FileNotFoundError, ValueError): - # Fallback to default values from config file - print("Loading default 'an' values from config file...") - for i in range(1, max_con): - if interval == 6: - an_values[i - 1] = parameters[f"a{i}_6_day"] - else: - an_values[i - 1] = parameters[f"a{i}_12_day"] -else: - # Use default values from the config file - print("Using default 'an' values from the config file.") - for i in range(1, max_con): - if interval == 6: - an_values[i - 1] = parameters[f"a{i}_6_day"] - else: - an_values[i - 1] = parameters[f"a{i}_12_day"] - -# Print the chosen an values -for i, value in enumerate(an_values, start=1): - print(f"Final value - a{i}: {value}") - - -############## generating ifg pairs between two epochs - -# desired_lengths = [interval, 2*interval, 3*interval, 4*interval, 5*interval] -desired_lengths = [interval * i for i in range(1, max_con + 1)] # replacing the above - -all_ifgs_string = generate_ifg_pairs(start_date, end_date, interval, desired_lengths) - - -##### reading original ifgs/coh ############################# -#################################################################################### - -############################### Reading input data ############################################# -#################### Read ifgs: -print("Reading all ifgs...") -# Define the path components -sub_dir = "Data" -filename_pattern = "All_ifgs_*" # Pattern to match files starting with "All_ifgs" - -# Construct the full file path with the refined pattern -file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) -# Use glob to find the file and ensure it exists -file_list = glob.glob(file_path_pattern) -if file_list: - file_path = file_list[0] # Get the single file path directly - - # Load the data from the file - with open(file_path, "rb") as file: - ifgs = pickle.load(file) -else: - raise FileNotFoundError( - f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." - ) - - -###################### Read coh: -print("\nReading all coherence data...") -# Define the path components -sub_dir = "Data" -filename_pattern = "All_coh_*" # Pattern to match files starting with "All_ifgs" - -# Construct the full file path with the refined pattern -file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) -# Use glob to find the file and ensure it exists -file_list = glob.glob(file_path_pattern) -if file_list: - file_path = file_list[0] # Get the single file path directly - - # Load the data from the file - with open(file_path, "rb") as file: - coh = pickle.load(file) -else: - raise FileNotFoundError( - f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." - ) -print("\nReading data completed.") - - -######## stacking all the ifgs 6/12/18 in a numpy var all_ifgs -desired_categories = desired_lengths -print('stacking all the ifgs 6/12/18 in a numpy var all_ifgs') -all_ifgs, all_coh, existing_ifgs_index = read_orig_ifgs_coh( - ifgs, coh, desired_categories -) -print('stacking completed') - -del ifgs, coh -gc.collect() - -print('ifgs coh deleted') -##### Reading the estimated bias terms as well as the indices of unknowns that can be corrected obtained in step 03 ############## -############################################################################################## -print("Correcting the interferograms...") -# Define the path components -sub_dir = "Data" -filename_pattern_X = "X_base_ifgs_biases.npy" # Pattern to match files starting with "X_base" -filename_pattern_indices = "indices*" - -# Construct the full file path with the refined pattern -file_path_pattern_X = os.path.join(output_path, sub_dir, filename_pattern_X) -file_path_pattern_indices = os.path.join(output_path, sub_dir, filename_pattern_indices) - -# Use glob to find the file and ensure it exists -file_list = glob.glob(file_path_pattern_X) - -if file_list: - file_path = file_list[0] # string path to .npy file - X = np.load(file_path, mmap_mode="r") # ✅ correct way -else: - raise FileNotFoundError(f"No file matching '{filename_pattern_X}' was found in '{os.path.join(output_path, sub_dir)}'.") - - - -# Use glob to find the file and ensure it exists -file_list = glob.glob(file_path_pattern_indices) -if file_list: - file_path = file_list[0] # Get the single file path directly - - # Load the data from the file - with open(file_path, "rb") as file: - indices_unknown_tobe_corrected = np.load(file) -else: - raise FileNotFoundError( - f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." - ) - - -#### loading indices of unknown that can be corrected and the estimated unknowns - -indices_unknown_tobe_corrected_orig = indices_unknown_tobe_corrected.copy() - -len_6_biases = indices_unknown_tobe_corrected[-1] + 1 # number of 6-day biases - -all_column_indices = np.array( - range(len_6_biases) -) # generating all indices in case we want to esimate all 6-day biases with smoothing constraints. -indices_unknown_tobe_corrected = all_column_indices # in this case we had provided the corrections for all 6-day biases - - -# extending the indices_unknown_tobe_corrected in the original approach to have the indices of 12/18 correctable ifgs: -extended_indices = ( - indices_unknown_tobe_corrected.copy() -) # because in this case indices_unknown_tobe_corrected referes to all_column_indices -X_extended = ( - X.copy() -) # because in this case we have estimated all 6-day biases but we don't need them for the correction. we just need them to correct the existing 12/18 days based on the 6 days - -extended_indices = extended_indices.tolist() -X_extended = X_extended.tolist() -print('indices loaded') - -# Generalized loop to handle any max_con -for n in range(2, max_con + 1): # Start from 2*interval and go up to (max_con)*interval - print('n is ', n) - current_length = n * interval # Calculate the length (e.g., 12, 18, 24, ...) - - if current_length in desired_lengths: # Check if this length is required - for i in range(len(indices_unknown_tobe_corrected) - (n - 1)): - last_item = extended_indices[-1] # Get the last index - extended_indices.append(last_item + 1) # Append the new index - - # Sum up X values for the current length (n) -# X_sum = sum( -# X[i + j, :, :] for j in range(n) -# ) # Sum over n consecutive indices - - - X_sum = X[i : i + n].sum(axis=0) # more efficient - - - # Append the result to X_extended using the corresponding an_values - X_extended.append( - an_values[n - 2] # an_values is 0-based, so use n-2 - * np.angle(np.exp(complex(0 + 1j) * X_sum)) - ) - - -X = np.array(X_extended) -indices_unknown_tobe_corrected = extended_indices -del X_extended -print('X extended') - - -#### correcting the ifgs (6/12/18) ######################## -################################################################# - - -####### v3 -def wrap_to_pi_inplace(arr): - np.remainder(arr + np.pi, 2*np.pi, out=arr) - arr -= np.pi - return arr - -print(f"all_ifgs.shape: {all_ifgs.shape}") -print(f"len(indices_unknown_tobe_corrected): {len(indices_unknown_tobe_corrected)}") -print(f"len(X): {len(X)}") - -all_ifgs_cor = np.full(all_ifgs.shape, np.nan, dtype=np.float32) -if uncor_only == 0: # if we want to use the corrected ifgs (using wrapped phases) - batch_size = 1 # adjust based on memory - for i in range(0, len(indices_unknown_tobe_corrected), batch_size): - print('batch ', i) - idx = indices_unknown_tobe_corrected[i:i+batch_size] - arr = all_ifgs[idx] - X[i:i+batch_size] - wrap_to_pi_inplace(arr) - all_ifgs_cor[idx] = arr -else: - print("uncor_only = 1 → skipping correction, copying originals") - all_ifgs_cor[indices_unknown_tobe_corrected] = all_ifgs[indices_unknown_tobe_corrected] - -################## v2 -#def wrap_to_pi(x): -# """Wrap values to [-pi, pi).""" -# return (x + np.pi) % (2 * np.pi) - np.pi -# -## Preallocate directly as float array filled with NaN -#all_ifgs_cor = np.full(all_ifgs.shape, np.nan, dtype=np.float32) -# -#if uncor_only == 0: # use corrected ifgs (wrapped phases) -# print('uncor_only 0') -# all_ifgs_cor[indices_unknown_tobe_corrected] = wrap_to_pi(all_ifgs[indices_unknown_tobe_corrected] - X) -#else: # no correction, just copy -# all_ifgs_cor[indices_unknown_tobe_corrected] = all_ifgs[indices_unknown_tobe_corrected] - -############### # v1 -# this was less efficient and so I replaced it with above -#all_ifgs_cor = np.full_like(all_ifgs, None) -#all_ifgs_cor[:, :, :] = np.nan -# -#if uncor_only == 0: # if we want to use the corrected ifgs (using wrapped phases) -# all_ifgs_cor[indices_unknown_tobe_corrected, :, :] = np.angle( -# np.exp( -# complex(0 + 1j) -# * (all_ifgs[indices_unknown_tobe_corrected, :, :] - X[:, :, :]) -# ) -# ) -#else: # no need for correction -# all_ifgs_cor[indices_unknown_tobe_corrected, :, :] = all_ifgs[ -# indices_unknown_tobe_corrected, :, : -# ] -# -print ('before outputting') - -### Outputting to disk ################ -############################################################################# - - -# Create the directory (and parent directories) if they don't exist -GEOC_dir = "GEOC" -metadata_dir = "metadata" - -output_wrap_dir = os.path.join(output_path, GEOC_dir) -os.makedirs( - output_wrap_dir, exist_ok=True -) # create the GEOC directory if it doesn't exist - -template_tif_path = os.path.join(root_directory, metadata_dir, frame + ".geo.hgt.tif") -print('template_tif_path', template_tif_path) -if nlook != 1: - resampled_tif_path = os.path.join(output_wrap_dir, frame + ".geo.hgt.tif") - print('template_tif_path', template_tif_path) - print('resampled_tif_path', resampled_tif_path) - print('nlook ', nlook) - - resample_geotiff(template_tif_path, resampled_tif_path, nlook) - template_tif_path = resampled_tif_path - -print('resample complete') - -total_files = len(all_ifgs) # Total number of interferograms -processed_files = 0 # Counter for processed files - -print(f"Writing the corrected interferograms to {output_wrap_dir}") -print("Progress: [", end="", flush=True) -time.sleep(3) - - -for i in range(len(all_ifgs)): - if i in indices_unknown_tobe_corrected: - - # Writing progress bar - processed_files += 1 - percentage = (processed_files / total_files) * 100 - if processed_files % (total_files // 100) == 0: # Update progress every 1% - sys.stdout.write( - f"\rProgress: [" - + "=" * (int(percentage) // 2) - + " " * (50 - int(percentage) // 2) - + f"] {int(percentage)}%" - ) - sys.stdout.flush() - - inpha = all_ifgs_cor[i, :, :] # your phase input, float32 - incoh = all_coh[i, :, :] # your coh input. It can be either 0-255 or 0-1 - - output_phase_path = os.path.join(output_wrap_dir, all_ifgs_string[i]) - output_cc_path = os.path.join(output_wrap_dir, all_ifgs_string[i]) - - # output_phase_path = output_wrap_dir + all_ifgs_string[i] - # output_cc_path = output_wrap_dir + all_ifgs_string[i] - - os.makedirs(output_phase_path, exist_ok=True) - os.makedirs(output_cc_path, exist_ok=True) - - output_phase_file = ( - output_phase_path + "/" + all_ifgs_string[i] + ".geo.diff_pha.tif" - ) - - output_cc_file = output_cc_path + "/" + all_ifgs_string[i] + ".geo.cc.tif" - - ifg_cc_2tif(inpha, output_phase_file, template_tif_path) - ifg_cc_2tif(incoh, output_cc_file, template_tif_path) - -print("\nAll corrected interferograms have been written to disk.") diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/README.md b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/README.md deleted file mode 100644 index aafee2e..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# InSAR Phase Bias Correction -This repository contains scripts for mitigating phase bias in InSAR data. It includes tools for reading data, calculating loop closures, estimating calibration parameters, performing inversion, and correcting interferograms. - -## Directory Structure -- `PhaseBias_01_Read_Data.py`: Reads input wrapped ifgs and coherence data. -- `PhaseBias_02_Loop_Closures.py`: Calculates loop closures. -- `PhaseBias_03_calibration_pars.py`: Estimates the calibration parameters $a_n$. -- `PhaseBias_04_Inversion.py`: Performs inversion for phase bias estimation. -- `PhaseBias_05_Correction.py`: Corrects interferograms using estimated biases. -- `bin/`: Contains auxiliary functions used by the scripts. -- `config.txt`: Configuration file for setting parameters. - -## Usage -1. Clone the repository: - ```bash - git clone https://github.com/YourUsername/InSAR_PhaseBias_Correction.git - cd InSAR_PhaseBias_Correction - -## Test Dataset - -The test dataset used in this repository originates from one of the islands in the Azores. The full dataset can be accessed through the COMET LiCSAR frame **082D_05125_020000**, available at the following link: - -[https://gws-access.jasmin.ac.uk/public/nceo_geohazards/LiCSAR_products/82/082D_05125_020000/](https://gws-access.jasmin.ac.uk/public/nceo_geohazards/LiCSAR_products/82/082D_05125_020000/) - -Additionally, we provide a **sample dataset**, including example interferograms, which can be downloaded from Zenodo: - -[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.14525360.svg)](https://doi.org/10.5281/zenodo.14525360) - - -# Citation -If you use this repository for your research, please cite the following paper: - -Maghsoudi, Y., Hooper, A.J., Wright, T.J., Lazecky, M., & Ansari, H. (2022). Characterizing and correcting phase biases in short-term, multilooked interferograms. Remote Sensing of Environment, 275, 113022. -https://doi.org/10.1016/j.rse.2022.113022 - - -# Acknowledgment -This work was funded by the European Space Agency (ESA) as part of the Phase Bias Correction Project. We gratefully acknowledge their support. - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_01_Read_Data.py b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_01_Read_Data.py deleted file mode 100644 index 5b72365..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_01_Read_Data.py +++ /dev/null @@ -1,583 +0,0 @@ -#!/usr/bin/env python3 - -import os -import re -from datetime import datetime, timedelta -import numpy as np -from osgeo import gdal -from bin.multilook_w import multilook_w -import configparser -import sys -import warnings -import pickle -import glob - -warnings.filterwarnings("ignore", category=np.VisibleDeprecationWarning) - -####################################################################### - -HELP_TEXT = """ -PhaseBias_01_Read_Data.py - -This script automates the process of downloading GeoTIFF files for wrapped interferograms and coherence images. -The files can be retrieved either from the COMET-LiCS web portal or from a root path specified in the configuration file. - -Assumptions: -- Interferograms are organized in folders named as `yyyymmdd_yyyymmdd` (e.g., `20230101_20230107`). - -Files Downloaded: -1. `yyyymmdd_yyyymmdd.geo.diff_pha.tif`: - - Contains the wrapped phase image in radians. - - Values range from -3.14 to 3.14. - -2. `yyyymmdd_yyyymmdd.geo.cc.tif`: - - Contains the coherence image of the interferometric pair. - - Values range from 0 to 255, where: - - 0 represents the lowest coherence. - - 255 represents the highest coherence. - -Outputs: -1. `All_ifgs_start_end` (Pickle file, saved as `.pkl`): - - A dictionary containing all interferograms available between the specified start and end dates. -2. `All_coh_start_end` (Pickle file, saved as `.pkl`): - - A dictionary containing all coherence data available between the specified start and end dates. - -Storage Information: -- All output files are saved in the directory `Data/` under the `output_path` defined in the configuration file. - -Additional Output: -- A summary report is generated that includes: - - The number of available interferograms for each temporal baseline. - - The number of missing interferograms. - -""" - -if "--help" in sys.argv: - print(HELP_TEXT) - sys.exit(0) - -################### - -# Config file path -config_file = "config.txt" - -# Check if config file exists -if not os.path.exists(config_file): - raise FileNotFoundError( - f"Error: The configuration file '{config_file}' was not found in the script's directory." - ) - -# Initialize the configparser -config = configparser.ConfigParser() - -# Read the config.txt file -config.read(config_file) - -# Define required parameters -required_parameters = [ - "root_path", - "frame", - "start", - "end", - "interval", - "nlook", - "LiCSAR_data", -] - -# Initialize a dictionary to store parameters -parameters = {} - -# Attempt to retrieve each parameter and catch errors -try: - parameters["root_path"] = config.get("DEFAULT", "root_path") - parameters["output_path"] = config.get("DEFAULT", "output_path") - parameters["frame"] = config.get("DEFAULT", "frame") - parameters["start"] = config.get("DEFAULT", "start") - parameters["end"] = config.get("DEFAULT", "end") - parameters["interval"] = config.getint("DEFAULT", "interval") - parameters["nlook"] = config.getint( - "DEFAULT", "nlook") # Assuming 'nlook' should be an integer - parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") - -except configparser.NoOptionError as e: - # Error message if a required parameter is missing - missing_param = str(e).split(": ")[1] - raise ValueError( - f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." - ) -except configparser.Error as e: - # General error for any configparser-related issues - raise ValueError(f"Error while reading '{config_file}': {e}") - -# Assign to individual variables for easy access -root_path = parameters["root_path"] -output_path = parameters["output_path"] -frame = parameters["frame"] -start = parameters["start"] -end = parameters["end"] -interval = parameters["interval"] -nlook = parameters["nlook"] -LiCSAR_data = parameters["LiCSAR_data"] - -# hardcoded parameters, not included in the config file -landmask = 1 -filtered_ifgs = "yes" -min_baseline = 5 -max_baseline = 366 - -# Print the values to confirm they're being read correctly -print("Root Path:", root_path) -print("output Path:", output_path) -print("Frame:", frame) -print("Start Date:", start) -print("End Date:", end) -print("Interval:", interval) -print("nlook:", nlook) -print("LiCSAR_data", LiCSAR_data) - -################ -# all_ifgs = read_ifgs(frame, start, end, min_baseline, max_baseline, landmask, nlook, interval, LiCSAR_data, filtered_ifgs) - -def read_ifgs( - start, - end, - min_baseline, - max_baseline, - landmask, - nlook, - interval, - LiCSAR_data, - filtered_ifgs, -): - - start_date = datetime.strptime(start, "%Y%m%d") - end_date = datetime.strptime(end, "%Y%m%d") - - # Check if the file ends with "geo.diff_pha.tif" - # if filename.endswith("geo.diff_unfiltered_pha.tif"): # if unfiltered data are used - if filtered_ifgs == "yes": # yes for the filtered ifgs, no for the unfiltered ifgs - req_file_name = "geo.diff_pha.tif" - else: - req_file_name = "geo.diff_unfiltered_pha.tif" - - ############ reading - - if LiCSAR_data == "yes": - track = frame[0:3] # extracting track number from frame id - if track[0] == "0": - track = track[1:3] - if track[0] == "0": - track = track[1:2] - - root_directory = f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}/interferograms/" - else: - root_directory = os.path.join( - root_path, "interferograms" - ) # Use os.path.join for consistent path construction - - ##### Count total files matching the criteria for accurate progress calculation - print(f"Reading data from the path: {root_directory}") - - total_files = sum(1 for dirpath, _, filenames in os.walk(root_directory) - for filename in filenames - if filename.endswith(req_file_name)) - processed_files = 0 - - print("Reading wrapped interferograms: [", end="", flush=True) - ##### - - # Initialize an empty dictionary to store the arrays for each category - category_arrays = {} - last_date = {} - n = 0 - - if landmask is not None: - if LiCSAR_data == "yes": - landmask_path = f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}/metadata/{frame}.geo.landmask.tif" - else: - # Use glob to find the file ending with ".geo.landmask.tif" in the metadata directory - metadata_dir = os.path.join(root_path, "metadata") - landmask_files = glob.glob( - os.path.join(metadata_dir, "*.geo.landmask.tif")) - if landmask_files: - landmask_path = landmask_files[0] # Take the first match - else: - raise FileNotFoundError( - "No file ending with '.geo.landmask.tif' found in the metadata directory." - ) - - # Load the landmask file - landmask_dataset = gdal.Open(landmask_path) - landmask_image = landmask_dataset.ReadAsArray() - array_landmask = np.array(landmask_image) - - n = 1 - # Traverse the root directory and its subdirectories - for dirpath, dirnames, filenames in os.walk(root_directory): - dirnames.sort( - ) # This ensures dirnames are processed in alphabetical order - # Iterate through the files in the current directory - for filename in filenames: - # Check if the file ends with "geo.diff_pha.tif" - # if filename.endswith("geo.diff_unfiltered_pha.tif"): # if unfiltered data are used - if (filtered_ifgs == "yes" - ): # yes for the filtered ifgs, no for the unfiltered ifgs - req_file_name = "geo.diff_pha.tif" - else: - req_file_name = "geo.diff_unfiltered_pha.tif" - - if filename.endswith(req_file_name): - file_path = os.path.join(dirpath, filename) - - #### for printing into the console - processed_files += 1 - percentage = (processed_files / total_files) * 100 - - # Clear previous percentage and print updated one - sys.stdout.write( - "\rReading wrapped phases: [{:<50}] {}%".format( - "=" * int(percentage // 2), int(percentage))) - sys.stdout.flush() - - ##### - - # Extract the dates from the file name - match = re.search(r"(\d+)_(\d+)", filename) - if match: - date1 = match.group(1) - date2 = match.group(2) - - # Convert the dates to datetime objects - date1_obj = datetime.strptime(date1, "%Y%m%d") - date2_obj = datetime.strptime(date2, "%Y%m%d") - - # Check if the dates fall within the specified range - category = (date2_obj - date1_obj).days - - if (start_date <= date1_obj <= end_date - and start_date <= date2_obj <= end_date - and category < max_baseline - and category > min_baseline): - if ( - n == 1 - ): # finding the first and last acquisiton in the time-series - first_acq = date1_obj - else: - last_acq = date2_obj - n = n + 1 - - # Calculate the difference in days between the two dates - - # Open the TIFF file - tiff_dataset = gdal.Open(file_path) - # Read the image data as a NumPy array - tiff_image = tiff_dataset.ReadAsArray() - - # Convert the image to a numpy array - array_data = np.array(tiff_image) - array_data[array_data == 0] = ( - np.nan - ) ## converting the zerso values in wrapped phases into nan. This is helpful when calculating loop closures - - if landmask != None: - array_data[array_landmask != 1] = np.nan - - if ( - nlook != None and nlook != 1 - ): # incase of unwrap data because they are already multilooked to 10 we don't multilook them here - array_data = multilook_w(array_data, nlook) - - if category not in last_date: - last_date[category] = date1_obj - # if (date1_obj - date2_obj) > interval - - # appending None to the category_arrays where there are missing ifgs in the middle of time-series - diff_ifgs = ( - date1_obj - last_date[category] - ).days # /category #difference between the first epochs of two consecutive ifg - if diff_ifgs > interval: - n_no_acq = int(diff_ifgs / interval - - 1) ## number of missing ifgs - for i in range( - n_no_acq - ): # the number of none depends on the number of missing ifgs - # Append the none to the existing array - category_arrays[category].append(None) - last_date[category] = date1_obj - - # Check if the category is already in the dictionary - if category in category_arrays: - # If the category already exists, append the array to the existing array - category_arrays[category].append(array_data) - else: - # If the category doesn't exist, create a new list with the array - category_arrays[category] = [array_data] - - # appending None to the category_arrays where there are missing ifgs in the end of time-series - for category in sorted(category_arrays): - while (last_date[category] + timedelta(category)) < last_acq: - category_arrays[category].append(None) - last_date[category] = last_date[category] + timedelta(interval) - - # appending None to the category_arrays where there are missing ifgs in the begining of time-series - # max number of expected 6-day(i.e. interval) interferograms in the full time-series - max_ifg_number = abs(int((first_acq - last_acq).days / interval)) - - # all the categories in the data e.g. 6, 12, 18 etc - cat = [] - for category in category_arrays: - cat.append(category) - - for i in range(interval, max(cat)+1, interval): - if i in cat: - while len(category_arrays[i]) < (max_ifg_number + 1 - i/interval): - category_arrays[i].insert( 0, None) - - print("\nFinished reading all interferograms.") - - return category_arrays - - -########################################################################################################################### -########################################################################################################################## - - -def read_coh(start, end, min_baseline, max_baseline, nlook, landmask, interval, - LiCSAR_data): - - start_date = datetime.strptime(start, "%Y%m%d") - end_date = datetime.strptime(end, "%Y%m%d") - - ############ reading - - if LiCSAR_data == "yes": - track = frame[0:3] - if track[0] == "0": - track = track[1:3] - if track[0] == "0": - track = track[1:2] - - root_directory = f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}/interferograms/" - else: - root_directory = os.path.join( - root_path, "interferograms" - ) # Use os.path.join for consistent path construction - - req_file_name = "geo.cc.tif" - - ##### Count total files matching the criteria for accurate progress calculation - - total_files = sum(1 for dirpath, _, filenames in os.walk(root_directory) - for filename in filenames - if filename.endswith(req_file_name)) - processed_files = 0 - - print("Reading coherence data: [", end="", flush=True) - ##### - - # Initialize an empty dictionary to store the arrays for each category - category_arrays = {} - last_date = {} - n = 0 - - if landmask is not None: - if LiCSAR_data == "yes": - landmask_path = f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}/metadata/{frame}.geo.landmask.tif" - else: - # Use glob to find the file ending with ".geo.landmask.tif" in the metadata directory - metadata_dir = os.path.join(root_path, "metadata") - landmask_files = glob.glob( - os.path.join(metadata_dir, "*.geo.landmask.tif")) - if landmask_files: - landmask_path = landmask_files[0] # Take the first match - else: - raise FileNotFoundError( - "No file ending with '.geo.landmask.tif' found in the metadata directory." - ) - - # Load the landmask file - landmask_dataset = gdal.Open(landmask_path) - landmask_image = landmask_dataset.ReadAsArray() - array_landmask = np.array(landmask_image) - - n = 1 - # Traverse the root directory and its subdirectories - for dirpath, dirnames, filenames in os.walk(root_directory): - # Iterate through the files in the current directory - for filename in filenames: - # Check if the file ends with "geo.diff_pha.tif" - - if filename.endswith(req_file_name): - file_path = os.path.join(dirpath, filename) - - #### for printing into the console - processed_files += 1 - percentage = (processed_files / total_files) * 100 - - # Clear previous percentage and print updated one - sys.stdout.write( - "\rReading coherence data: [{:<50}] {}%".format( - "=" * int(percentage // 2), int(percentage))) - sys.stdout.flush() - - ##### - - # Extract the dates from the file name - match = re.search(r"(\d+)_(\d+)", filename) - if match: - date1 = match.group(1) - date2 = match.group(2) - - # Convert the dates to datetime objects - date1_obj = datetime.strptime(date1, "%Y%m%d") - date2_obj = datetime.strptime(date2, "%Y%m%d") - - # Check if the dates fall within the specified range - category = (date2_obj - date1_obj).days - - if (start_date <= date1_obj <= end_date - and start_date <= date2_obj <= end_date - and category < max_baseline - and category > min_baseline): - if ( - n == 1 - ): # finding the first and last acquisiton in the time-series - first_acq = date1_obj - else: - last_acq = date2_obj - n = n + 1 - - # Calculate the difference in days between the two dates - - # Open the TIFF file - tiff_dataset = gdal.Open(file_path) - - # Read the image data as a NumPy array - tiff_image = tiff_dataset.ReadAsArray() - - # Convert the image to a numpy array - array_data = np.array(tiff_image) - # Convert array_data to a float type that can accommodate np.nan - array_data = array_data.astype(np.float32) - - if landmask != 0: - array_data[array_landmask != 1] = np.nan - - if ( - nlook != None and nlook != 1 - ): # incase of unwrap data because they are already multilooked to 10 we don't multilook them here - array_data = multilook(array_data, nlook) - - if category not in last_date: - last_date[category] = date1_obj - # if (date1_obj - date2_obj) > interval - - # appending None to the category_arrays where there are missing ifgs in the middle of time-series - diff_ifgs = ( - date1_obj - last_date[category] - ).days # /category #difference between the first epochs of two consecutive ifg - if diff_ifgs > interval: - n_no_acq = int(diff_ifgs / interval - - 1) ## number of missing ifgs - for i in range( - n_no_acq - ): # the number of none depends on the number of missing ifgs - # Append the none to the existing array - category_arrays[category].append(None) - last_date[category] = date1_obj - - # Check if the category is already in the dictionary - if category in category_arrays: - # If the category already exists, append the array to the existing array - category_arrays[category].append(array_data) - else: - # If the category doesn't exist, create a new list with the array - category_arrays[category] = [array_data] - - # appending None to the category_arrays where there are missing ifgs in the end of time-series - for category in sorted(category_arrays): - while (last_date[category] + timedelta(category)) < last_acq: - category_arrays[category].append(None) - last_date[category] = last_date[category] + timedelta(interval) - - # appending None to the category_arrays where there are missing ifgs in the begining of time-series - # max number of expected 6-day(i.e. interval) interferograms in the full time-series - max_ifg_number = abs(int((first_acq - last_acq).days / interval)) - - # all the categories in the data e.g. 6, 12, 18 etc - cat = [] - for category in category_arrays: - cat.append(category) - - for i in range(interval, max(cat)+1, interval): - if i in cat: - while len(category_arrays[i]) < (max_ifg_number + 1 - i/interval): - category_arrays[i].insert(0, None) - - print("\nFinished reading all coherence data.") - - return category_arrays - - -########################################################################################################################## -########################################################################################################################## - -all_ifgs = read_ifgs( - start, - end, - min_baseline, - max_baseline, - landmask, - nlook, - interval, - LiCSAR_data, - filtered_ifgs, -) -# Define the directory path -directory_path = os.path.join(output_path, "Data") - -# Create the directory if it doesn't exist -os.makedirs(directory_path, exist_ok=True) - -# Define the file path -#file_path = os.path.join(directory_path, f"_all_ifgs_{start}_{end}") - -# Remove all files inside the directory -if os.path.exists(directory_path): - files = glob.glob(os.path.join(directory_path, "*")) - for file in files: - os.remove(file) - - -# for writing the all_loops -file_path = output_path + "/Data/" + "All_ifgs_" + start + "_" + end + ".pkl" -with open(file_path, "wb") as file: # for writting a dictionary to a file - pickle.dump(all_ifgs, file) - -# Free memory before starting coherence -import gc -del all_ifgs -gc.collect() - -################################################################ - -all_coh = read_coh(start, end, min_baseline, max_baseline, landmask, nlook, - interval, LiCSAR_data) - -## for writting the all_coh -file_path = output_path + "/Data/" + "All_coh_" + start + "_" + end + ".pkl" -with open(file_path, "wb") as file: # for writting a dictionary to a file - pickle.dump(all_coh, file) - -################################################ - -# After processing all interferograms, generate the report -print("\nReport on the number of IFGs and missing IFGs per category:") -for category in sorted(all_ifgs): - num_ifgs = sum(1 for item in all_ifgs[category] - if item is not None) # Count valid IFGs - num_missing_ifgs = sum(1 for item in all_ifgs[category] - if item is None) # Count missing IFGs - print( - f"{category}-days: Number of IFGs: {num_ifgs} Number of missing IFGs: {num_missing_ifgs}" - ) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_02_Loop_Closures.py b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_02_Loop_Closures.py deleted file mode 100644 index 9c6165d..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_02_Loop_Closures.py +++ /dev/null @@ -1,310 +0,0 @@ -#!/usr/bin/env python3 - -import numpy as np -import pickle -import glob -import configparser -import os -import sys -import warnings -warnings.filterwarnings("ignore", category=np.VisibleDeprecationWarning) - -################################################################################################ - -HELP_TEXT = """ -PhaseBias_02_Loop_Closures.py - -This script calculates loop closures (Δφ) using the interferograms imported in the first step. - -Definition of Loop Closures: -Loop closures, Δφ, are calculated for the epochs between `i` and `k`, and are defined as: - Δφ_(i,k) = |φ_(i,k) - ∑_(t=i)^k φ_(t,t+1)|_2π - -Where: -- φ_(i,j): Represents the phase difference for a pixel in the interferogram formed between epochs `i` and `j`. -- |.|_2π: Indicates that the result is wrapped modulo 2π (i.e., values range from -π to π). - -Key Information: -- Nonzero closure phase is a by-product of spatial filtering/multilooking and is primarily associated with changes in the scattering and electrical properties of the ground surface. -- The calculated loop closures are based on the minimum temporal-baseline interferograms defined in the configuration file (parameter: `interval`) and are referred to as base interferograms. These may represent: - - 12- and 6-day closures (or 24- and 12-day closures), Δφ_(i,i+2) - - 18- and 6-day closures (or 36- and 12-day closures), Δφ_(i,i+3) - The distinction depends on whether the base interferograms are 6-day or 12-day intervals. - -Outputs: -- The calculated loop closures are stored as a dictionary in a file named: - `All_loops_start_end.pkl` -- The output is saved in the directory `Data/` under the `output_path` defined in the configuration file. - -Loop Closure Report: -At the end of the script, a loop closure report is displayed. This report includes: -- The number of generated and missing loop closures for short-interval closures, such as 12-6 and 18-6. -- The number of generated and missing long-interval closures, obtained using long-term interferograms, such as 204-6 and 204-12. - -Usage: -1. Ensure the interferograms from the first script are available. -2. Run the script to generate loop closures for the specified temporal baselines. - -Example: -```bash -python PhaseBias_02_Loop_Closures.py - - -""" -if "--help" in sys.argv: - print(HELP_TEXT) - sys.exit(0) - -################################################################################################ -# Config file path -config_file = "config.txt" - -# Check if config file exists -if not os.path.exists(config_file): - raise FileNotFoundError( - f"Error: The configuration file '{config_file}' was not found in the script's directory." - ) - -# Initialize the configparser -config = configparser.ConfigParser() - -# Read the config.txt file -config.read(config_file) - -# Define required parameters -required_parameters = [ - "root_path", - "frame", - "start", - "end", - "interval", - "nlook", - "LiCSAR_data", -] - -# Initialize a dictionary to store parameters -parameters = {} - -# Attempt to retrieve each parameter and catch errors -try: - parameters["root_path"] = config.get("DEFAULT", "root_path") - parameters["output_path"] = config.get("DEFAULT", "output_path") - parameters["frame"] = config.get("DEFAULT", "frame") - parameters["start"] = config.get("DEFAULT", "start") - parameters["end"] = config.get("DEFAULT", "end") - parameters["interval"] = config.getint("DEFAULT", "interval") - parameters["nlook"] = config.getint( - "DEFAULT", "nlook") # Assuming 'nlook' should be an integer - parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") - -except configparser.NoOptionError as e: - # Error message if a required parameter is missing - missing_param = str(e).split(": ")[1] - raise ValueError( - f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." - ) -except configparser.Error as e: - # General error for any configparser-related issues - raise ValueError(f"Error while reading '{config_file}': {e}") - -# Assign to individual variables for easy access -root_path = parameters["root_path"] -output_path = parameters["output_path"] -frame = parameters["frame"] -start = parameters["start"] -end = parameters["end"] -interval = parameters["interval"] -nlook = parameters["nlook"] -LiCSAR_data = parameters["LiCSAR_data"] - -# hardcoded parameters, not included in the config file -landmask = 1 -filtered_ifgs = "yes" -max_loop = 5 # in case of 3 all loops with 6,12,18-day will be calculated (e.g. (60,6), (60,12) and (60,18)) - - -############################################################################################# -################################################################################################# -def loop_calc(max_loop): - - ## Read ifgs: - - # Define the path components - sub_dir = "Data" - filename_pattern = "All_ifgs_*" # Pattern to match files starting with "All_ifgs" - - # Construct the full file path with the refined pattern - file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) - # Use glob to find the file and ensure it exists - file_list = glob.glob(file_path_pattern) - if file_list: - file_path = file_list[0] # Get the single file path directly - - # Load the data from the file - with open(file_path, "rb") as file: - all_ifgs = pickle.load(file) - else: - raise FileNotFoundError( - f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." - ) - - loop = {} - missing = {} - - cat = sorted(all_ifgs.keys()) # Get sorted categories - # print('cat = ', cat) - - # Initialize counters for reporting - loop_counts = { - category: { - cat[l]: 0 - for l in range(max_loop) - } - for category in sorted(all_ifgs.keys()) - } - missing_counts = { - category: { - cat[l]: 0 - for l in range(max_loop) - } - for category in sorted(all_ifgs.keys()) - } - - for category in cat: - loop[category] = { - } # Initialize the key in the loop dictionary with an inner dictionary - missing[category] = [] # second ver - for l in range( - max_loop - ): # it was up to 3 in the first version. but this allows to go for longer loops e.g. 288-72 (cat[11] is 72) - loop[category][cat[l]] = [ - ] # Initialize the key in the inner dictionary - # missing[cat[l]] = [] # first ver - # Calculate total iterations for progress percentage - total_iterations = sum( - len(all_ifgs[cat[i]]) for l in range(max_loop) - for i in range(1, len(cat)) - if cat[i] % cat[l] == 0 and cat[i] != cat[l]) - completed_iterations = 0 - - # Display the message before starting the progress bar - print( - f"Calculating all {interval}, {2 * interval}, and {3 * interval} loop closures." - ) - - for l in range( - 0, max_loop - ): # l could be 6, 12 and 18 to calculate e.g. 36-6 or 36-12 or 36-18 - for i in range(1, len(cat)): # index for the category e.g. 6, 12, 18 - if cat[i] % cat[l] == 0 and cat[i] != cat[ - l]: # such as 12/6 or 24/12 - for t in range(len(all_ifgs[ - cat[i]])): # index for the epochs in each category - - ####################progress bar - completed_iterations += 1 - percentage = (completed_iterations / - total_iterations) * 100 - - # Display progress percentage - sys.stdout.write("\rProgress: [{:<50}] {}%".format( - "=" * int(percentage // 2), int(percentage))) - sys.stdout.flush() - ################################## - - if cat[l] == interval: - end_index = t + int(cat[i] / cat[l]) - # print('end_index = ', end_index) - else: - end_index = t + int(cat[i] / cat[l]) * int( - cat[l] / interval) - - # recording the missing ifgs in each cat e.g. 6,12,18 - #if np.any(np.array(all_ifgs[cat[i]])[t] == None): - if np.any(np.array(all_ifgs[cat[i]], dtype=object)[t] == None): - missing[cat[i]].append( - t - ) # to record the missing ifgs index for each cateogory e.g. 18 - - - for e in range(t, end_index, int(cat[l]/interval)): # to record the missing ifgs index for the period of each loop - elem = all_ifgs[cat[l]][e] - - if elem is None: - missing[cat[l]].append(e) - - - - if all_ifgs[cat[i]][t] is None or any(all_ifgs[cat[l]][elem] is None for elem in range(t, end_index, int(cat[l] / interval))): - - loop[cat[i]][cat[l]].append(None) - missing_counts[cat[i]][cat[l]] += 1 # Increment missing loop counter - - else: - closure = np.angle( np.exp( 1j * ( np.array(all_ifgs[cat[i]][t], dtype=np.float32)- np.sum( np.array(all_ifgs[cat[l]][t:end_index:int(cat[l] / interval)], dtype=np.float32), axis=0)))) - - - if cat[i] in loop: - loop[cat[i]][cat[l]].append(closure) - loop_counts[cat[i]][cat[ - l]] += 1 # Increment successful loop counter - - else: - loop[cat[i]][cat[l]] = [closure] - print("\n Finished calculating the loop closures.") - - # Generate report - print("\n=== Loop Closure Report ===") - for category in [ - 2 * interval, - 3 * interval, - ]: # Restrict to relevant categories e.g. 18-6 and 12-6 as the main observations - for l in [interval]: # Restrict to the loop levels of interest - if category in loop_counts and l in loop_counts[category]: - generated = loop_counts[category].get(l, 0) - missing = missing_counts[category].get(l, 0) - print( - f"Loop Closure {category} - {l}: Generated = {generated}, Missing = {missing}" - ) - - print("\n=== Loop Closure Report for Long-interval Loop Closures ===") - - # Iterate over categories greater than 200 - for category in cat: - if category > 200: # Check for long categories - for l in range( - 1, - 5 * interval): # Check loop levels less than 5 * interval - if category in loop_counts and l in loop_counts[category]: - generated = loop_counts[category].get(l, 0) - if generated >= 1: # Only report if at least one loop is generated - print( - f"Loop Closure {category} - {l}: Generated = {generated}" - ) - - return loop, missing - - -################################################################################################ - -all_loops, missing = loop_calc(max_loop) - -# Define the directory path -directory_path = os.path.join(output_path, "Data") - -# Create the directory if it doesn't exist -os.makedirs(directory_path, exist_ok=True) - -# Define the file path -file_path = os.path.join(directory_path, f"All_loops_{start}_{end}.pkl") - -with open(file_path, "wb") as file: # for writting a dictionary to a file - pickle.dump(all_loops, file) - -### - -# Define the file path -file_path = os.path.join(directory_path, f"missing_loops") - -with open(file_path, "wb") as file: # for writting a dictionary to a file - pickle.dump(missing, file) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_03_calibration_pars.py b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_03_calibration_pars.py deleted file mode 100644 index ce3aba9..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_03_calibration_pars.py +++ /dev/null @@ -1,424 +0,0 @@ -#!/usr/bin/env python3 - -import numpy as np -import matplotlib.pyplot as plt -from matplotlib.gridspec import GridSpec -import pickle -import os -import sys -import glob -import configparser -import warnings - -with warnings.catch_warnings(): - warnings.simplefilter("ignore", category=np.VisibleDeprecationWarning) - -################################################################################ - -HELP_TEXT = """ -PhaseBias_03_calibration_pars.py - -This script estimates the calibration parameters a_n using the loop closures calculated in the previous step. - -The calibration parameters a_n are estimated by calculating the ratio of the long-term loop closure for the interferogram of interest to the long-term loop closure for the base interferogram. Assuming negligible bias in long-term interferograms, a_n is calculated as follows: - - a_n = |φ(i,i+l) - Σ(t=i to i+l)φ(t,t+1+n)|_2π / |φ(i,i+l) - Σ(t=i to i+l)φ(t,t+1)|_2π - -where: -- φ(i,i+l) is a long-term interferogram connecting epoch i to i+l. -- φ(t,t+1) represents the base interferogram. -- φ(t,t+1+n) is the short interferogram to be corrected using a_n. - -### Input: -1. Loop closures from the previous step (e.g., 216-6, 216-12, and 216-18 for 6-day acquisition intervals). -2. The long-term interferogram length (e.g., 216 days) must be specified at the beginning. -3. The number of a_n parameters to estimate, specified using `num_a`. For example: - - Setting `num_a=2` indicates that a1 and a2 will be estimated (used to correct 12- and 18-day interferograms). - - For correcting longer interferograms, `num_a` should be set accordingly. - -### Output: -1. All possible a_n values are calculated. -2. Visualizations: - - Example plots of a_n arrays and their corresponding histograms, illustrating spatial variations. - - Time-series plots showing how a_n values evolve over time. -3. Mean a_n values over time are output for further use. - -### File Outputs: -- Results are stored in the output directory defined in the `config.txt` file under `/Data/` as `Mean_an_values.pkl`. - -### Additional Notes: -- Users should ensure that `num_a` aligns with the desired temporal baselines for correction. - -""" -if "--help" in sys.argv: - print(HELP_TEXT) - sys.exit(0) - - -############################################################################### -# Config file path -config_file = "config.txt" - -# Check if config file exists -if not os.path.exists(config_file): - raise FileNotFoundError( - f"Error: The configuration file '{config_file}' was not found in the script's directory." - ) - -# Initialize the configparser -config = configparser.ConfigParser() - - -# Read the config.txt file -config.read(config_file) - -# Define required parameters -required_parameters = [ - "root_path", - "frame", - "start", - "end", - "interval", - "nlook", - "LiCSAR_data", -] - -# Initialize a dictionary to store parameters -parameters = {} - -# Attempt to retrieve each parameter and catch errors -try: - parameters["root_path"] = config.get("DEFAULT", "root_path") - parameters["output_path"] = config.get("DEFAULT", "output_path") - parameters["frame"] = config.get("DEFAULT", "frame") - parameters["start"] = config.get("DEFAULT", "start") - parameters["end"] = config.get("DEFAULT", "end") - parameters["interval"] = config.getint("DEFAULT", "interval") - parameters["nlook"] = config.getint( - "DEFAULT", "nlook" - ) # Assuming 'nlook' should be an integer - parameters["num_a"] = config.getint("DEFAULT", "num_a") - parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") - -except configparser.NoOptionError as e: - # Error message if a required parameter is missing - missing_param = str(e).split(": ")[1] - raise ValueError( - f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." - ) -except configparser.Error as e: - # General error for any configparser-related issues - raise ValueError(f"Error while reading '{config_file}': {e}") - -# Assign to individual variables for easy access -root_path = parameters["root_path"] -output_path = parameters["output_path"] -frame = parameters["frame"] -start = parameters["start"] -end = parameters["end"] -interval = parameters["interval"] -nlook = parameters["nlook"] -num_a = parameters["num_a"] -LiCSAR_data = parameters["LiCSAR_data"] - -# hardcoded parameters, not included in the config file -landmask = 1 -filtered_ifgs = "yes" -max_loop = 5 # in case of 3 all loops with 6,12,18-day will be calculated (e.g. (60,6), (60,12) and (60,18)) - - -##################################################### Loop Closure calculation ##################################################### -#################################################################################################################################### -track = frame[0:3] -if track[0] == "0": - track = track[1:3] -if track[0] == "0": - track = track[1:2] - - -###################### Read coh: -print("\nReading all coherence data...") -# Define the path components -sub_dir = "Data" -filename_pattern = "All_coh_*" # Pattern to match files starting with "All_ifgs" - -# Construct the full file path with the refined pattern -file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) -# Use glob to find the file and ensure it exists -file_list = glob.glob(file_path_pattern) -if file_list: - file_path = file_list[0] # Get the single file path directly - - # Load the data from the file - with open(file_path, "rb") as file: - coh = pickle.load(file) -else: - raise FileNotFoundError( - f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." - ) -print("Reading coherence data completed.") - - -##################### Read Loops - -print("\nReading all loop closures...") -filename_pattern = "All_loops_*" # Pattern to match files starting with "All_ifgs" - -# Construct the full file path with the refined pattern -file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) -# Use glob to find the file and ensure it exists -file_list = glob.glob(file_path_pattern) -if file_list: - file_path = file_list[0] # Get the single file path directly - - # Load the data from the file - with open(file_path, "rb") as file: - all_loops = pickle.load(file) -else: - raise FileNotFoundError( - f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." - ) - -print("Reading loop closures completed.") - - -##################################################### Loop Closure calculation ##################################################### -#################################################################################################################################### - - -long_baseline = 216 # 36,72,108,144,180,216,252,288,324 -coh_thresh = 25 - - -def calc_an(loop_360_n, loop_360_6): - """ - Calculates the a_n values from loop closures. - - Parameters: - loop_360_n (ndarray): Array of loop closures for n-day intervals. - loop_360_6 (ndarray): Array of loop closures for 6-day intervals. - - Returns: - ndarray: Mean a_n values for all loop closures. - list: List of all a_n arrays for further processing or plotting. - """ - ratios = [] - - # Loop through indices - for i in range(len(loop_360_6)): - if loop_360_6[i] is not None and loop_360_n[i] is not None: - # Calculate the ratio for non-NaN elements - ratio_i = np.divide( - loop_360_n[i], - loop_360_6[i], - out=np.full_like(loop_360_n[i], np.nan), - where=~np.isnan(loop_360_6[i]), - ) - ratios.append(ratio_i) - -# Check if `ratios` is empty before proceeding - if not ratios: - print("Warning: No valid a_n values could be calculated. Returning NaN values.") - return np.array([np.nan]), [] - - - # Convert the list of ratios to a numpy array - an = np.array(ratios, dtype=object) - mean_an_long_baseline = [] - - # Calculate mean values of each a_n - for arr in an: - mask = (arr > 0) & (arr < 1) # Apply mask to filter invalid values - arr[~mask] = np.nan - if not np.isnan(arr).all(): # Check if all elements are NaN - mean_an = np.nanmean(arr) - else: - mean_an = np.nan # Assign NaN for completely empty slices - - mean_an_long_baseline.append(mean_an) - - return np.array(mean_an_long_baseline), an - - -################################################################################## -############################ -def plot_an(an_arrays, mean_a_long_baseline, k): - """ - Plots the a_n arrays with the largest coverage and their histograms in a two-row layout, - as well as the mean values of calibration parameters a_n over time. - - Parameters: - an_arrays (list): List of a_n arrays. - mean_a_long_baseline (ndarray): Array of mean calibration parameter values for a_1, a_2, ..., a_n. - k (int): The current value of k (e.g., 2 for a1, 3 for a2). - """ - - # Calculate the coverage (number of non-NaN values) for each array - coverage = [] - valid_an_arrays = [] # Store only valid arrays for plotting - - for arr in an_arrays: - if ( - arr is not None and arr.ndim == 2 and not np.all(np.isnan(arr)) - ): # Check for valid 2D arrays - coverage.append(np.sum(~np.isnan(arr))) # Count non-NaN values - valid_an_arrays.append(arr) - - # Sort arrays by coverage in descending order - sorted_indices = np.argsort(coverage)[::-1] - selected_arrays = [ - valid_an_arrays[idx] for idx in sorted_indices[:6] - ] # Select top 6 arrays - - # Plot the a_n arrays and their histograms in a two-row layout - if selected_arrays: - num_arrays = len(selected_arrays) - fig = plt.figure(figsize=(4 * num_arrays, 8)) - gs = GridSpec( - 2, num_arrays + 1, width_ratios=[1] * num_arrays + [0.1] - ) # Add space for colorbar - - # Plot selected arrays - axes_img = [] - for i, arr in enumerate(selected_arrays): - ax_img = fig.add_subplot(gs[0, i]) - im = ax_img.imshow(arr, cmap="RdYlBu", vmin=-0.8, vmax=0.8) - ax_img.set_title(f"{i+1}th $a_{{{k-1}}}$ Array") # Dynamically set title - ax_img.axis("off") - axes_img.append(ax_img) - - # Add a single colorbar for all array plots - cbar_ax = fig.add_subplot(gs[0, -1]) - fig.colorbar(im, cax=cbar_ax, orientation="vertical", label="Color Scale") - - # Plot histograms for the selected arrays - for i, arr in enumerate(selected_arrays): - ax_hist = fig.add_subplot(gs[1, i]) - ax_hist.hist(arr.flatten(), bins=100, color="blue", alpha=0.7, density=True) - ax_hist.set_title( - f"{i+1}th $a_{{{k-1}}}$ Histogram" - ) # Dynamically set title - ax_hist.set_xlim([-10, 10]) # Adjust histogram range as needed - - plt.tight_layout() - plt.show() - else: - print("No valid a_n arrays available for plotting.") - - # Plot the mean values of a_n over time - x_values = np.arange(len(mean_a_long_baseline)) # Sequential indices for x-axis - fig, ax = plt.subplots(figsize=(6, 4)) - - - # Filter out NaN values to avoid gaps in the plot - valid_indices = ~np.isnan(mean_a_long_baseline) # Mask for valid (non-NaN) entries - x_valid = x_values[valid_indices] # Only use valid time steps - y_valid = np.array(mean_a_long_baseline)[valid_indices] # Only use valid a_n values - - - # Remove NaN values for trendline fitting - valid_indices = ~np.isnan(mean_a_long_baseline) # Mask for valid (non-NaN) entries - if ( - valid_indices.sum() > 1 - ): # Ensure there are at least two valid points for fitting -# x_valid = x_values[valid_indices] - y_valid = np.array(mean_a_long_baseline)[valid_indices] - x_valid = np.arange(len(y_valid)) - - # Fit and plot a trendline - coefficients = np.polyfit(x_valid, y_valid, 1) - trendline = np.polyval(coefficients, x_valid) - ax.plot(x_valid, trendline, color="red", label="Trendline") - - # Scatter plot for all points (including NaN if any) - ax.scatter( - #x_values, - #mean_a_long_baseline, - x_valid, # Only use valid time steps - y_valid, # Only use valid a_n values - - color="blue", - marker="o", - label="Mean Values", - s=9, - ) - - # Set labels and limits - ax.set_ylim(-3, 3) # Adjust limits as needed - ax.set_xlabel("Time Step") # Sequential index as x-axis - ax.set_ylabel("Mean Values", fontsize=12) - ax.legend() - - # Adjust layout and show the plot - plt.tight_layout() - plt.show() - - -## -######### forming 360-6 - -loop_360_6_all = np.array(all_loops[long_baseline][interval], dtype=object) -coh_360 = np.array(coh[long_baseline][:], dtype=object) - -loop_360_6 = np.full_like( - loop_360_6_all, np.nan -) # create a new array as the same size of loop_360_6_all with nan values - - -for i, arr in enumerate(loop_360_6_all): - if arr is not None: - (frame_row, frame_col) = np.shape(arr) - loop_360_6[i] = np.where((coh_360[i] > coh_thresh), arr, np.nan) - -###### forming 360-n (n=12,18,24 etc) -# mean_a_long_baseline = [] -final_mean_values = [] -an_labels = [] - -# Loop through the values of k corresponding to the number of calibration parameters -for k in range(2, 2 + num_a): # k=2 for a1, k=3 for a2, etc. - mean_an_long_baseline = [] - - # Forming 360-n (n=12, 18, 24, etc.) - loop_360_n_all = np.array(all_loops[long_baseline][k * interval], dtype=object) - loop_360_n = np.full_like(loop_360_n_all, np.nan) - - for i, arr in enumerate(loop_360_n_all): - if arr is not None: - # Apply coherence threshold filtering - loop_360_n[i] = np.where((coh_360[i] > coh_thresh), arr, np.nan) - - # Calculate a_n values - mean_an_long_baseline, an_arrays = calc_an(loop_360_n, loop_360_6) - - # If all values in mean_an_long_baseline are nan (empty ratios), stop execution - if np.isnan(mean_an_long_baseline).all(): - print("No valid a_n values could be calculated. Stopping execution.") - sys.exit(1) # Exit the script with an error code (1 means failure) - - # Calculate the mean of all mean values for this k and store it - overall_mean = np.nanmean(mean_an_long_baseline) - final_mean_values.append(overall_mean) - an_labels.append(f"a{k-1}") - - for idx, mean_a in enumerate(mean_an_long_baseline, start=1): - if not np.isnan(mean_a): # Check if the value is not NaN - print(f"Mean of the {idx}th a({k-1}) is {mean_a:.6f}") - - plot_an(an_arrays, mean_an_long_baseline, k) - - -# Print the final mean values across all k -for k_idx, final_mean in enumerate(final_mean_values, start=2): - print(f"Final mean value for a({k_idx-1}) is {final_mean:.6f}") - -# Write final mean values to a text file -output_path = os.path.join(parameters["output_path"], "Data") -os.makedirs(output_path, exist_ok=True) # Create directory if it doesn't exist -output_file = os.path.join(output_path, "an.txt") - -with open(output_file, "w") as file: - for label, mean_value in zip(an_labels, final_mean_values): - file.write(f"{label}={mean_value:.6f}\n") - -print(f"Final mean values written to {output_file}.") diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_04_Inversion.py b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_04_Inversion.py deleted file mode 100644 index 822a56e..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_04_Inversion.py +++ /dev/null @@ -1,673 +0,0 @@ -#!/usr/bin/env python3 - -import numpy as np -import pickle -import os -from scipy.sparse import csc_array -from scipy.sparse.linalg import lsqr -import sys -import math -from bin.read_orig_ifgs_coh import read_orig_ifgs_coh -from bin.circular_mean_var import circular_mean_and_variance_over_epochs -import glob -import configparser -import warnings - -warnings.filterwarnings("ignore", category=np.VisibleDeprecationWarning) - -################################################################################ - -HELP_TEXT = """ -PhaseBias_04_Inversion.py - -This script estimates the phase bias terms for the base interferograms using short-term closure phases as input. -The bias terms derived in this step serve as the foundation for correcting other short-term interferograms. - -### Workflow: -1. **Masking Noisy Loop Closures:** - - A moving average is applied to the time-series of loop closures to account for seasonal fluctuations. - - A circular moving average (mean of complex values over time) is computed for each pixel. - - The distance of each loop closure from the moving average is calculated. If the distance exceeds a threshold (2σ), the point is masked as noisy. - - The circular standard deviation (σ) is computed using the von Mises distribution (circular normal distribution). - -2. **Refined Loop Closures:** - - Masked loop closures, denoted as 〖Δφ〗_(i,i+2)^r and 〖Δφ〗_(i,i+3)^r, are used as primary observations for phase bias estimation. - -3. **First Inversion (Without Temporal Smoothing Constraints):** - - Estimates the bias terms that can be corrected directly from the observed loop closures. - -4. **Second Inversion (With Temporal Smoothing Constraints):** - - Incorporates temporal smoothing constraints to estimate additional bias terms that cannot be directly derived from loop closures. - - The temporal smoothing minimizes differences between bias terms over time, ensuring consistency. - -5. **Combining Results:** - - Bias terms from the first inversion are combined with the results of the second inversion to include terms not estimated in the first round. - -### Outputs: -- The phase bias terms for the base interferograms are stored as a NumPy array file: - - `X_base_ifgs_biases.npy`, saved in the `Data` directory under the `output_path` defined in `config.txt`. - - -### Input Requirements: -- Short-term closure phases (from `PhaseBias_02_Loop_Closures.py`). -- Configuration parameters, including thresholds and temporal settings, defined in `config.txt`. - -### Output Files: -- `X_base_ifgs_biases.npy`: Stores the estimated bias terms for the base interferograms. -""" - -if "--help" in sys.argv: - print(HELP_TEXT) - sys.exit(0) - -############################################################################### -# Config file path -config_file = "config.txt" - -# Check if config file exists -if not os.path.exists(config_file): - raise FileNotFoundError( - f"Error: The configuration file '{config_file}' was not found in the script's directory." - ) - -# Initialize the configparser -config = configparser.ConfigParser() - -# Read the config.txt file -config.read(config_file) - -# Define required parameters -required_parameters = [ - "root_path", - "frame", - "start", - "end", - "interval", - "nlook", - "LiCSAR_data", - "a1_6_day", - "a2_6_day", - "a1_12_day", - "a2_12_day", - "estimate_an_values", -] - -# Initialize a dictionary to store parameters -parameters = {} - -# Attempt to retrieve each parameter and catch errors -try: - parameters["root_path"] = config.get("DEFAULT", "root_path") - parameters["output_path"] = config.get("DEFAULT", "output_path") - parameters["frame"] = config.get("DEFAULT", "frame") - parameters["start"] = config.get("DEFAULT", "start") - parameters["end"] = config.get("DEFAULT", "end") - parameters["interval"] = config.getint("DEFAULT", "interval") - parameters["nlook"] = config.getint( - "DEFAULT", "nlook" - ) # Assuming 'nlook' should be an integer - parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") - parameters["a1_6_day"] = config.getfloat("DEFAULT", "a1_6_day") - parameters["a2_6_day"] = config.getfloat("DEFAULT", "a2_6_day") - parameters["a1_12_day"] = config.getfloat("DEFAULT", "a1_12_day") - parameters["a2_12_day"] = config.getfloat("DEFAULT", "a2_12_day") - parameters["estimate_an_values"] = config.get("DEFAULT", "estimate_an_values") - -except configparser.NoOptionError as e: - # Error message if a required parameter is missing - missing_param = str(e).split(": ")[1] - raise ValueError( - f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." - ) -except configparser.Error as e: - # General error for any configparser-related issues - raise ValueError(f"Error while reading '{config_file}': {e}") - -# Assign to individual variables for easy access -root_path = parameters["root_path"] -output_path = parameters["output_path"] -frame = parameters["frame"] -start = parameters["start"] -end = parameters["end"] -interval = parameters["interval"] -nlook = parameters["nlook"] -LiCSAR_data = parameters["LiCSAR_data"] -estimate_an_values = parameters["estimate_an_values"] - - -# Initialize a1 and a2 with None for safety -a1 = None -a2 = None - - -print("estimate_an_values is ", estimate_an_values) -# Check if estimate_an_values is 'yes' -if estimate_an_values == "yes": - an_file_path = os.path.join(output_path, "Data", "an.txt") - - # Try reading the file - try: - if os.path.exists(an_file_path): - print(f"Reading 'a' values from {an_file_path}...") - - # Read the file and load the values - with open(an_file_path, "r") as file: - for line in file: - line = line.strip() - if line.startswith("a1="): - a1 = float(line.split("=")[1]) - elif line.startswith("a2="): - a2 = float(line.split("=")[1]) - - # Check if both a1 and a2 were found - if a1 is None or a2 is None: - print( - "Warning: Missing a1 or a2 in an.txt. Switching to default values from config file." - ) - raise ValueError("Incomplete a1 or a2 values.") - else: - print( - f"Warning: {an_file_path} does not exist. Switching to default values from config file." - ) - raise FileNotFoundError - - except (FileNotFoundError, ValueError): - # Fallback to default values from config file - if interval == 6: - a1 = parameters["a1_6_day"] - a2 = parameters["a2_6_day"] - else: - a1 = parameters["a1_12_day"] - a2 = parameters["a2_12_day"] -else: - # Use default values from the config file - print("Using default 'an' values from the config file.") - if interval == 6: - a1 = parameters["a1_6_day"] - a2 = parameters["a2_6_day"] - else: - a1 = parameters["a1_12_day"] - a2 = parameters["a2_12_day"] - -# Print the chosen values -print(f"Final values - a1: {a1}, a2: {a2}") - - -##################################################### Loop Closure calculation ##################################################### -#################################################################################################################################### -#################################################################################################################################### - -track = frame[0:3] -if track[0] == "0": - track = track[1:3] -if track[0] == "0": - track = track[1:2] - -with_temporals = "yes" -apply_to_all = "no" # in case of with_temporal='yes', we can decide if you want to use the temporal to all ifgs (i.e. apply_to_all='yes'), or just to those that can not be corrected (i.e. apply_to_all='no') -w = 0.1 # the weigth of the temporal smoothing constraints -####### -coh_thresh = 11 # threhsold on the coherence values. In this version it is off -min_num_eq = 10 # I used 10 in all my experiments. min number of equations in the least square inversion. - -############################### Reading input data ############################################# - -##################### Read ifgs: -#print("Reading all ifgs...") -## Define the path components -#sub_dir = "Data" -#filename_pattern = "All_ifgs_*" # Pattern to match files starting with "All_ifgs" -# -## Construct the full file path with the refined pattern -#file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) -## Use glob to find the file and ensure it exists -#file_list = glob.glob(file_path_pattern) -#if file_list: -# file_path = file_list[0] # Get the single file path directly -# -# # Load the data from the file -# with open(file_path, "rb") as file: -# all_ifgs = pickle.load(file) -#else: -# raise FileNotFoundError( -# f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." -# ) -#print("Reading ifgs completed.") - - -####################### Read coh: -#print("\nReading all coherence data...") -## Define the path components -#sub_dir = "Data" -#filename_pattern = "All_coh_*" # Pattern to match files starting with "All_ifgs" -# -## Construct the full file path with the refined pattern -#file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) -## Use glob to find the file and ensure it exists -#file_list = glob.glob(file_path_pattern) -#if file_list: -# file_path = file_list[0] # Get the single file path directly -# -# # Load the data from the file -# with open(file_path, "rb") as file: -# all_coh = pickle.load(file) -#else: -# raise FileNotFoundError( -# f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." -# ) -#print("Reading coherence data completed.") - -##################### Read Loops - -print("\nReading all loop closures...") -sub_dir = "Data" -filename_pattern = "All_loops_*" # Pattern to match files starting with "All_ifgs" - -# Construct the full file path with the refined pattern -file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) -# Use glob to find the file and ensure it exists -file_list = glob.glob(file_path_pattern) -if file_list: - file_path = file_list[0] # Get the single file path directly - - # Load the data from the file - with open(file_path, "rb") as file: - all_loops = pickle.load(file) -else: - raise FileNotFoundError( - f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." - ) - -print("Reading loop closures completed.") - -all_cat = [] -for cat in all_loops: - all_cat.append(cat) - -### stacking all the ifgs 6/12/18 in a numpy var all_ifgs -print( - "Extracting the required interferograms(up to 3 epochs), and the corresponding loop closures for inversion..." -) -#desired_categories = [interval, 2 * interval, 3 * interval] -#all_ifgs, _, existing_ifgs_index = read_orig_ifgs_coh( -# all_ifgs, all_coh, desired_categories -#) - -######### forming 12-6 and 18-6 loop closures from the imported data -################################################################# -loop_12 = np.array(all_loops[2 * interval][interval], dtype=object) -#coh_12 = np.array(all_coh[2 * interval][:], dtype=object) - -loop_18 = np.array(all_loops[3 * interval][interval], dtype=object) -#coh_18 = np.array(all_coh[3 * interval][:], dtype=object) - -############## finding dynamic pixel-based thresholds to remove the noisy loops -loop_12_orig = [] -loop_18_orig = [] -none_indices12 = [] -none_indices18 = [] - -# refine loop_12 to exclude the none indices and create loop_12_orig -for i, arr in enumerate(loop_12): - if arr is not None: - (frame_row, frame_col) = np.shape(arr) - loop_12_orig.append(arr) - else: - none_indices12.append(i) - -# refine loop_18 to exclude the none indices and create loop_18_orig -for i, arr in enumerate(loop_18): - if arr is not None: - loop_18_orig.append(arr) - else: - none_indices18.append(i) - -loop_12_orig = np.array(loop_12_orig) -loop_18_orig = np.array(loop_18_orig) - - -### removing noisy loops -print( - "Masking noisy loop closures using circular moving average and standard deviaiton of the loop closures in time:" -) - -# using circular_mean to calculate the mean phase value using complex numbers, and von_mises_variance to calculate the second moment of the Von Mises distribution -_, thresh_loop12 = circular_mean_and_variance_over_epochs(loop_12_orig, axis=0) -_, thresh_loop18 = circular_mean_and_variance_over_epochs(loop_18_orig, axis=0) - - -# Add a new axis at the beginning (axis=0) of these arrays to have shape (1, ...) -# These are the threshold vlaues obtained for each pixel that will be used for masking later -thresh_loop12 = np.expand_dims(thresh_loop12, axis=0) -thresh_loop18 = np.expand_dims(thresh_loop18, axis=0) - - -# calculate a temporal moving average for wrapped data -def moving_average(arr, window_size): - # Initialize an empty array to store the temporal averages - moving_averages = np.zeros_like(arr) - half_window = window_size // 2 - ### miroring data at both ends for window_size/2 at each end - mirrored_shape = (arr.shape[0] + window_size, arr.shape[1], arr.shape[2]) - - # Create an empty array to hold the mirrored data - mirrored_arr = np.empty(mirrored_shape, dtype=arr.dtype) - - # Fill the mirrored array with the original data - mirrored_arr[window_size // 2 : -window_size // 2] = arr - - # Mirror the data at the beginning - for i in range(window_size // 2): -# mirrored_arr[i] = arr[window_size // 2 - i] - mirrored_arr[i] = arr[min(half_window - i, arr.shape[0] - 1)] - - # Mirror the data at the end - for i in range(window_size // 2): -# mirrored_arr[-(i + 1)] = arr[-(i + 1) - window_size // 2] - mirrored_arr[-(i + 1)] = arr[max(-1 - (half_window - i), 0)] - #### - - # Calculate temporal averages within each window - for t in range(half_window, mirrored_arr.shape[0] - half_window): - moving_averages[t - half_window, :, :] = np.angle( - np.nanmean( - np.exp(1j * (mirrored_arr[t - half_window : t + half_window, :, :])), - axis=0, - ) - ) - return moving_averages - - -# for loop 12 -# Calculate distance between each epoch and the moving average for each pixel -print( - f"Calculating the moving average for {2 * interval} loop closures, followed by masking noisy loop closures..." -) -print( - "This may take a while, as it depends on the number of loops and the size of your dataset.\n" -) - -window_size = 30 - -moving_avg = moving_average(loop_12_orig, window_size) -distances = np.abs(np.angle(np.exp(1j * (loop_12_orig - moving_avg)))) -mask = distances > 4 * thresh_loop12 # it is 4*sigma - -# Replace values with NaN where mask is True -loop_12_bk = loop_12_orig -loop_12_orig = np.where(mask, np.nan, loop_12_orig) -loop_12 = loop_12_orig.tolist() - -for idx in none_indices12: - loop_12.insert(idx, None) -loop_12 = np.array(loop_12, dtype=object) - -# for loop 18 -# Calculate distance between each epoch and the moving average for each pixel -print( - f"Calculating the moving average for {3 * interval} loop closures, followed by masking noisy loop closures..." -) -print( - "This may take a while, as it depends on the number of loops and the size of your dataset.\n" -) - -moving_avg = moving_average(loop_18_orig, window_size) -distances = np.abs(np.angle(np.exp(1j * (loop_18_orig - moving_avg)))) -mask = distances > 4 * thresh_loop18 # it is 4*sigma - -# Replace values with NaN where mask is True -loop_18_bk = loop_18_orig -loop_18_orig = np.where(mask, np.nan, loop_18_orig) -loop_18 = loop_18_orig.tolist() - -for idx in none_indices18: - loop_18.insert(idx, None) -loop_18 = np.array(loop_18, dtype=object) - -print("Masking of noisy loop closures is completed.") - -# ################ Apply coherece thresholding -# print('shape loop_12 = ', np.shape(loop_12)) -# print('shape loop_12_orig = ', np.shape(loop_12_orig)) -# print('shape coh_12 = ', np.shape(coh_12)) -# for i, arr in enumerate(loop_12): -# if arr is not None: -# (frame_row, frame_col) = np.shape(arr) -# loop_12[i] = np.where((coh_12[i] > coh_thresh), arr, np.nan) -# -# -# for i, arr in enumerate(loop_18): -# if arr is not None: -# loop_18[i] = np.where((coh_18[i] > coh_thresh), arr, np.nan) -# -# -# - -len12 = len(loop_12) # number of 12 day loops including None -len18 = len(loop_18) # number of 18 day loops including None - -######################## Forming the design matrix A ############################## -#################################################################################### - -print("Preparing the design matrix and observation vector for the inversion step...") - -b = [] -A = [] -n_unk = ( - len12 + 1 -) # this in an initial value(i.e. the num of 6 biases). will be changed by the actual number of unk after considering the None values/missing -# print('n_unk = ', n_unk) - -row = np.zeros( - n_unk, dtype=np.float32 -) # each row of the design matrix A for the 12-day -for i in range(len12): - b.append(loop_12[i]) - # print('np.shape(loop_12[i] = ', np.shape(loop_12[i])) - row[i : i + 2] = a1 - 1 - A.append(row) - row = np.zeros(n_unk, dtype=np.float32) - -row = np.zeros( - n_unk, dtype=np.float32 -) # each row of the design matrix A for the 18-day -for i in range(len18): - b.append(loop_18[i]) - row[i : i + 3] = a2 - 1 - A.append(row) - row = np.zeros(n_unk, dtype=np.float32) - -A = np.array(A) -b = np.array(b, dtype=object) - - -####################### removing the rows from b and A where b is None -mask = [] -mask = np.array([arr is not None for arr in b]) -# Convert the boolean mask to an integer mask -mask = np.nonzero(mask) - -A = A[mask] # Filter rows of A based on the mask -b = b[mask] - -b = b.tolist() - -############ removing the columns of A where are values are zero (these are the unkonws which doesn't fall in any equations nor 12 neither 18 and thus cannot be corrected) - -# Find the column indices where all values are zero -zero_columns = np.all(A == 0, axis=0) - -## Get the indices of the zero columns -zero_column_indices = np.where(zero_columns)[0] - -# Get the indices of the non-zero columns -non_zero_column_indices = np.where(~zero_columns)[ - 0 -] # the indices of the unknowns that can be corrected - -# Save the file -directory_path = os.path.join(output_path, "Data") -file_path = os.path.join(directory_path, f"indices_unknown_tobe_corrected.npy") -np.save(file_path, non_zero_column_indices) - - -### Adding all the existing 6-day ifgs indices to non_zero_column_indices. We want to use estimate their biases in the second step. - - -A_bk = A # to keep a copy of A in case we want to esimate all 6-day biases with smoothing constraints -b_bk = b -all_column_indices = np.array( - range(A_bk.shape[1]) -) # generating all indices in case we want to esimate all 6-day biases with smoothing constraints - -A = A[:, non_zero_column_indices] - -# finding the last column with value -1. this gives the number of 6-day biases that can be corrected -last_column_with_minus_1 = None - -# Iterate through the columns from right to left -for col in range(len(A[0]) - 1, -1, -1): - if -1 in [row[col] for row in A]: - last_column_with_minus_1 = col - break - -num_rows_before_temporals = A.shape[0] - -################################# Least Square inversion ############################### -######################################################################################## -# First Inversion: Without Temporal Smoothing Constraints -print("Starting the first inversion (without temporal smoothing constraints).") -print( - "This step estimates the bias terms that can be corrected based on observed loop closures." -) - -damp_factor = 0 - -if apply_to_all == "no": # this is without using any temporal constraints - - X1 = np.zeros((np.shape(A_bk)[1], frame_row, frame_col), dtype=np.float32) - # X1[:,:,:] = np.nan # after checking noticed this doesn't have any effect on the final vel - b_bk = np.array(b_bk) - - total_pixels = frame_row * frame_col # Total number of pixels for progress bar - processed_pixels = 0 # Counter for processed pixels - print("Progress: [", end="", flush=True) - - for row in range(frame_row): - for col in range(frame_col): - processed_pixels += 1 - # Update progress bar - percentage = (processed_pixels / total_pixels) * 100 - if processed_pixels % (total_pixels // 100) == 0: # Update every 1% - print(f"{int(percentage)}%", end="", flush=True) - sys.stdout.write( - "\rProgress: [" - + "=" * (int(percentage) // 2) - + " " * (50 - int(percentage) // 2) - + "]" - ) - - non_nan_mask = ~np.isnan(b_bk[:, row, col]) - bb = b_bk[non_nan_mask, row, col] # Remove NaN values - AA = A_bk[non_nan_mask, :] # Remove corresponding rows - if len(bb) > min_num_eq: # what is the min number of equation? - - x, istop, itn, normr, normr2 = lsqr( - csc_array(AA), bb, damp=damp_factor - )[ - :5 - ] # Using sparse matrix representation scipy.sparse.linalg - X1[:, row, col] = x - -print("\nFirst inversion (without temporal smoothing constraints) completed.") - -########################### -# Second Inversion: With Temporal Smoothing Constraints -print("Starting the second inversion (with temporal smoothing constraints).") -print( - "This step estimates all bias terms, including those that cannot be corrected, using temporal smoothing constraints." -) - -if with_temporals == "yes": # This is using the temporal constranints on all unknowns - X2 = np.zeros((np.shape(A_bk)[1], frame_row, frame_col), dtype=np.float32) - # X2[:,:,:] = np.nan # after checking noticed this doesn't have any effect on the final vel - b_bk = np.array(b_bk) - - #### Adding temporal constraints - row = np.zeros( - A_bk.shape[1], dtype=np.float32 - ) # additional rows the design matrix A for temporal constraints, setting that to zero again - for i in range(len(all_column_indices) - 2): # for all unknowns - row[i] = -1 * w - row[i + 1] = 2 * w - row[i + 2] = -1 * w - A_bk = np.vstack((A_bk, row)) - row = np.zeros( - A_bk.shape[1], dtype=np.float32 - ) # additional rows the design matrix A for temporal constraints, setting that to zero again - num_temporals = len(A_bk) - len(b_bk) - # print('num_temporals is ', num_temporals) - - ######### Appending zeroes to b according to the added number of temporal constraints obtained by np.shape(A)[0] - np.shape(b)[0] - b_0 = np.zeros( - (np.shape(A_bk)[0] - np.shape(b)[0], frame_row, frame_col), dtype=np.float32 - ) - b_bk = np.vstack((b_bk, b_0)) - - thresh_n_eq = num_temporals # in case of using temporals, we don't want the nan pixels with zero equations with only using temporals - - processed_pixels = 0 # Reset counter for second inversion - print("Progress: [", end="", flush=True) - - for row in range(frame_row): - for col in range(frame_col): - processed_pixels += 1 - # Update progress bar - percentage = (processed_pixels / total_pixels) * 100 - if processed_pixels % (total_pixels // 100) == 0: # Update every 1% - print(f"{int(percentage)}%", end="", flush=True) - sys.stdout.write( - "\rProgress: [" - + "=" * (int(percentage) // 2) - + " " * (50 - int(percentage) // 2) - + "]" - ) - - non_nan_mask = ~np.isnan(b_bk[:, row, col]) - bb = b_bk[non_nan_mask, row, col] # Remove NaN values - AA = A_bk[non_nan_mask, :] # Remove corresponding rows - if len(bb) > thresh_n_eq: - - x, istop, itn, normr, normr2 = lsqr( - csc_array(AA), bb, damp=damp_factor - )[ - :5 - ] # Using sparse matrix representation scipy.sparse.linalg - X2[:, row, col] = x - - print("\nSecond inversion (with temporal smoothing constraints) completed.") - - ####### combining X1 and X2 into X - - X = np.zeros((np.shape(A_bk)[1], frame_row, frame_col), dtype=np.float32) - X[:, :, :] = np.nan - - # Fill X with values from X1 according to non_zero_column_indices - for i, idx in enumerate(non_zero_column_indices): - X[idx, :, :] = X1[i, :, :] - - # Fill in the missing values from X2 - for i, idx in enumerate(all_column_indices): - if idx not in non_zero_column_indices: - X[idx, :, :] = X2[i, :, :] - - # Ensure the dtype is appropriate for your data - X = X.astype(np.float32) - -directory_path = os.path.join(output_path, "Data") -file_path = os.path.join(directory_path, f"X_base_ifgs_biases.npy") -np.save(file_path, X) - -print("Inversion process completed. Results saved to:", file_path) - -########################################################################################################## diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_05_Correction.py b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_05_Correction.py deleted file mode 100644 index a83200d..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_05_Correction.py +++ /dev/null @@ -1,481 +0,0 @@ -#!/usr/bin/env python3 - -import numpy as np -from datetime import datetime -import pickle -import time -import os -from bin.generate_ifgs_from_epochs import generate_ifg_pairs -import configparser -import glob -from bin.resample import resample_geotiff -import sys - -# sys.path.append(os.path.dirname(os.path.abspath(__file__))) -import bin -from bin.read_orig_ifgs_coh import read_orig_ifgs_coh - -from bin.ifg_cc_totif import ifg_cc_2tif -import gc - -############################################################################### - -HELP_TEXT = """ -PhaseBias_05_Correction.py - -This script applies phase bias correction to desired interferograms using the estimated bias terms for the base interferograms. The corrected interferograms are saved in GeoTIFF format for further analysis. - -### Workflow: -1. **Inputs:** - - **Original Wrapped Interferograms, and Coherence Data:** Obtained from step 1 (`PhaseBias_01_Read_Data.py`). - - **Estimated Bias Terms for Base Intierferograms:** Obtained from step 4 (`PhaseBias_04_Inversion.py`). - - **Calibration Parameters (*a_n*):** Estimated from step 3 (`PhaseBias_03_calibration_pars.py`). - - **max_con:** Defines the maximum number of connections to be corrected. For instance: - - `max_con=5` corrects interferograms up to 30-day intervals for a 6-day acquisition interval. - - Adjust `max_con` as needed. - -2. **Bias Term Estimation for Desired Interferograms:** - - Bias terms for desired interferograms, δ_(i,i+n+1), are estimated using the relationship: - δ_(i,i+n+1) = a_n * (∑_(t=i)^(i+n) δ_(t,t+1) ) - -3. **Correction of Interferograms:** - - Using the estimated bias terms, the interferograms are corrected as follows: - φ_(i,i+n+1)^c = φ_(i,i+n+1) - δ_(i,i+n+1) - - - Here, φ_(i,i+n+1) is the original interferogram, and φ_(i,i+n+1)^c is the corrected interferogram. - -4. **Outputs:** - - Corrected interferograms are saved in GeoTIFF format in the `GEOC` directory under the `output_path` specified in `config.txt`. - -### Output File Format: -- **Corrected Interferograms:** Saved in GeoTIFF format, organized by temporal baseline. - -### Input Requirements: -- Original wrapped interferograms from step 1. -- Bias terms for base interferograms from step 4. -- Configuration parameters (e.g., `output_path`, `interval`) defined in `config.txt`. - -### Output Directory: -- `GEOC` directory under `output_path`: Contains the corrected interferograms in GeoTIFF format, organized by temporal baseline. - -""" -if "--help" in sys.argv: - print(HELP_TEXT) - sys.exit(0) - - -################################################################################ - - -# Config file path -config_file = "config.txt" - -# Check if config file exists -if not os.path.exists(config_file): - raise FileNotFoundError( - f"Error: The configuration file '{config_file}' was not found in the script's directory." - ) - -# Initialize the configparser -config = configparser.ConfigParser() - - -# Read the config.txt file -config.read(config_file) - - -# Define required parameters -required_parameters = [ - "root_path", - "frame", - "start", - "end", - "interval", - "nlook", - "LiCSAR_data", - "a1_6_day", - "a2_6_day", - "a3_6_day", - "a4_6_day", - "a1_12_day", - "a2_12_day", - "a3_12_day", - "a4_12_day", - "estimate_an_values", -] - -# Initialize a dictionary to store parameters -parameters = {} - -# Attempt to retrieve each parameter and catch errors -try: - parameters["root_path"] = config.get("DEFAULT", "root_path") - parameters["output_path"] = config.get("DEFAULT", "output_path") - parameters["frame"] = config.get("DEFAULT", "frame") - parameters["start"] = config.get("DEFAULT", "start") - parameters["end"] = config.get("DEFAULT", "end") - parameters["interval"] = config.getint("DEFAULT", "interval") - parameters["nlook"] = config.getint( - "DEFAULT", "nlook" - ) # Assuming 'nlook' should be an integer - parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") - parameters["a1_6_day"] = config.getfloat("DEFAULT", "a1_6_day") - parameters["a2_6_day"] = config.getfloat("DEFAULT", "a2_6_day") - parameters["a3_6_day"] = config.getfloat("DEFAULT", "a3_6_day") - parameters["a4_6_day"] = config.getfloat("DEFAULT", "a4_6_day") - parameters["a1_12_day"] = config.getfloat("DEFAULT", "a1_12_day") - parameters["a2_12_day"] = config.getfloat("DEFAULT", "a2_12_day") - parameters["a3_12_day"] = config.getfloat("DEFAULT", "a3_12_day") - parameters["estimate_an_values"] = config.get("DEFAULT", "estimate_an_values") - -except configparser.NoOptionError as e: - # Error message if a required parameter is missing - missing_param = str(e).split(": ")[1] - raise ValueError( - f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." - ) -except configparser.Error as e: - # General error for any configparser-related issues - raise ValueError(f"Error while reading '{config_file}': {e}") - -# Assign to individual variables for easy access -root_path = parameters["root_path"] -output_path = parameters["output_path"] -frame = parameters["frame"] -start = parameters["start"] -end = parameters["end"] -interval = parameters["interval"] -nlook = parameters["nlook"] -LiCSAR_data = parameters["LiCSAR_data"] -estimate_an_values = parameters["estimate_an_values"] - - -start_date = datetime.strptime(start, "%Y%m%d") -end_date = datetime.strptime(end, "%Y%m%d") - -uncor_only = 0 # in case of 1 only uses 6-12-18 uncorrected ifgs. it is based on the desired_lengths ifgs -max_con = 5 # in case of 5, it will correct up to 6*5=30-day ifgs. - - -##################################################### -# Root directory where the category directories are located - -if LiCSAR_data == "yes": - track = frame[0:3] # extracting track number from frame id - if track[0] == "0": - track = track[1:3] - if track[0] == "0": - track = track[1:2] - - root_directory = ( - f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}" - ) -else: - root_directory = root_path - - -############################################ - -# Initialize the an values with None for safety -an_values = [None] * ( - max_con - 1 -) # For max_con=5, this will create [None, None, None, None] - -print("estimate_an_values is ", estimate_an_values) - -# Check if estimate_an_values is 'yes' -if estimate_an_values == "yes": - an_file_path = os.path.join(output_path, "Data", "an.txt") - - # Try reading the file - try: - if os.path.exists(an_file_path): - print(f"Reading 'a' values from {an_file_path}...") - - # Read the file and load the an values dynamically - with open(an_file_path, "r") as file: - for line in file: - line = line.strip() - for i in range( - 1, max_con - ): # Loop through a1, a2, ..., a(max_con-1) - param_name = f"a{i}" - if line.startswith(f"{param_name}="): - an_values[i - 1] = float(line.split("=")[1]) - - # Check if all required an values were found - if any(value is None for value in an_values): - print( - "Warning: Missing one or more an values in an.txt. Switching to default values from config file." - ) - raise ValueError("Incomplete an values.") - else: - print( - f"Warning: {an_file_path} does not exist. Switching to default values from config file." - ) - raise FileNotFoundError - - except (FileNotFoundError, ValueError): - # Fallback to default values from config file - print("Loading default 'an' values from config file...") - for i in range(1, max_con): - if interval == 6: - an_values[i - 1] = parameters[f"a{i}_6_day"] - else: - an_values[i - 1] = parameters[f"a{i}_12_day"] -else: - # Use default values from the config file - print("Using default 'an' values from the config file.") - for i in range(1, max_con): - if interval == 6: - an_values[i - 1] = parameters[f"a{i}_6_day"] - else: - an_values[i - 1] = parameters[f"a{i}_12_day"] - -# Print the chosen an values -for i, value in enumerate(an_values, start=1): - print(f"Final value - a{i}: {value}") - - -############## generating ifg pairs between two epochs - -# desired_lengths = [interval, 2*interval, 3*interval, 4*interval, 5*interval] -desired_lengths = [interval * i for i in range(1, max_con + 1)] # replacing the above - -all_ifgs_string = generate_ifg_pairs(start_date, end_date, interval, desired_lengths) - - -##### reading original ifgs/coh ############################# -#################################################################################### - -############################### Reading input data ############################################# -#################### Read ifgs: -print("Reading all ifgs...") -# Define the path components -sub_dir = "Data" -filename_pattern = "All_ifgs_*" # Pattern to match files starting with "All_ifgs" - -# Construct the full file path with the refined pattern -file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) -# Use glob to find the file and ensure it exists -file_list = glob.glob(file_path_pattern) -if file_list: - file_path = file_list[0] # Get the single file path directly - - # Load the data from the file - with open(file_path, "rb") as file: - ifgs = pickle.load(file) -else: - raise FileNotFoundError( - f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." - ) - - -###################### Read coh: -print("\nReading all coherence data...") -# Define the path components -sub_dir = "Data" -filename_pattern = "All_coh_*" # Pattern to match files starting with "All_ifgs" - -# Construct the full file path with the refined pattern -file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) -# Use glob to find the file and ensure it exists -file_list = glob.glob(file_path_pattern) -if file_list: - file_path = file_list[0] # Get the single file path directly - - # Load the data from the file - with open(file_path, "rb") as file: - coh = pickle.load(file) -else: - raise FileNotFoundError( - f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." - ) -print("\nReading data completed.") - - -######## stacking all the ifgs 6/12/18 in a numpy var all_ifgs -desired_categories = desired_lengths - -all_ifgs, all_coh, existing_ifgs_index = read_orig_ifgs_coh( - ifgs, coh, desired_categories -) - - -del ifgs, coh -gc.collect() - -##### Reading the estimated bias terms as well as the indices of unknowns that can be corrected obtained in step 03 ############## -############################################################################################## -print("Correcting the interferograms...") -# Define the path components -sub_dir = "Data" -filename_pattern_X = "X_base*" # Pattern to match files starting with "X_base" -filename_pattern_indices = "indices*" - -# Construct the full file path with the refined pattern -file_path_pattern_X = os.path.join(output_path, sub_dir, filename_pattern_X) -file_path_pattern_indices = os.path.join(output_path, sub_dir, filename_pattern_indices) - -# Use glob to find the file and ensure it exists -file_list = glob.glob(file_path_pattern_X) -if file_list: - file_path = file_list[0] # Get the single file path directly - - # Load the data from the file - with open(file_path, "rb") as file: - X = np.load(file) -else: - raise FileNotFoundError( - f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." - ) -# print('Reading ifgs completed.') - -# Use glob to find the file and ensure it exists -file_list = glob.glob(file_path_pattern_indices) -if file_list: - file_path = file_list[0] # Get the single file path directly - - # Load the data from the file - with open(file_path, "rb") as file: - indices_unknown_tobe_corrected = np.load(file) -else: - raise FileNotFoundError( - f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." - ) - - -#### loading indices of unknown that can be corrected and the estimated unknowns - -indices_unknown_tobe_corrected_orig = indices_unknown_tobe_corrected.copy() - -len_6_biases = indices_unknown_tobe_corrected[-1] + 1 # number of 6-day biases - -all_column_indices = np.array( - range(len_6_biases) -) # generating all indices in case we want to esimate all 6-day biases with smoothing constraints. -indices_unknown_tobe_corrected = all_column_indices # in this case we had provided the corrections for all 6-day biases - - -# extending the indices_unknown_tobe_corrected in the original approach to have the indices of 12/18 correctable ifgs: -extended_indices = ( - indices_unknown_tobe_corrected.copy() -) # because in this case indices_unknown_tobe_corrected referes to all_column_indices -X_extended = ( - X.copy() -) # because in this case we have estimated all 6-day biases but we don't need them for the correction. we just need them to correct the existing 12/18 days based on the 6 days - -extended_indices = extended_indices.tolist() -X_extended = X_extended.tolist() - - -# Generalized loop to handle any max_con -for n in range(2, max_con + 1): # Start from 2*interval and go up to (max_con)*interval - current_length = n * interval # Calculate the length (e.g., 12, 18, 24, ...) - - if current_length in desired_lengths: # Check if this length is required - for i in range(len(indices_unknown_tobe_corrected) - (n - 1)): - last_item = extended_indices[-1] # Get the last index - extended_indices.append(last_item + 1) # Append the new index - - # Sum up X values for the current length (n) - X_sum = sum( - X[i + j, :, :] for j in range(n) - ) # Sum over n consecutive indices - - # Append the result to X_extended using the corresponding an_values - X_extended.append( - an_values[n - 2] # an_values is 0-based, so use n-2 - * np.angle(np.exp(complex(0 + 1j) * X_sum)) - ) - - -X = np.array(X_extended) -indices_unknown_tobe_corrected = extended_indices -del X_extended - - -#### correcting the ifgs (6/12/18) ######################## -################################################################# -all_ifgs_cor = np.full_like(all_ifgs, None) -all_ifgs_cor[:, :, :] = np.nan - -if uncor_only == 0: # if we want to use the corrected ifgs (using wrapped phases) - all_ifgs_cor[indices_unknown_tobe_corrected, :, :] = np.angle( - np.exp( - complex(0 + 1j) - * (all_ifgs[indices_unknown_tobe_corrected, :, :] - X[:, :, :]) - ) - ) -else: # no need for correction - all_ifgs_cor[indices_unknown_tobe_corrected, :, :] = all_ifgs[ - indices_unknown_tobe_corrected, :, : - ] - - -### Outputting to disk ################ -############################################################################# - - -# Create the directory (and parent directories) if they don't exist -GEOC_dir = "GEOC" -metadata_dir = "metadata" - -output_wrap_dir = os.path.join(output_path, GEOC_dir) -os.makedirs( - output_wrap_dir, exist_ok=True -) # create the GEOC directory if it doesn't exist - -template_tif_path = os.path.join(root_directory, metadata_dir, frame + ".geo.hgt.tif") - -if nlook != 1: - resampled_tif_path = os.path.join(output_wrap_dir, frame + ".geo.hgt.tif") - resample_geotiff(template_tif_path, resampled_tif_path, nlook) - template_tif_path = resampled_tif_path - - -total_files = len(all_ifgs) # Total number of interferograms -processed_files = 0 # Counter for processed files - -print(f"Writing the corrected interferograms to {output_wrap_dir}") -print("Progress: [", end="", flush=True) -time.sleep(3) - - -for i in range(len(all_ifgs)): - if i in indices_unknown_tobe_corrected: - - # Writing progress bar - processed_files += 1 - percentage = (processed_files / total_files) * 100 - if processed_files % (total_files // 100) == 0: # Update progress every 1% - sys.stdout.write( - f"\rProgress: [" - + "=" * (int(percentage) // 2) - + " " * (50 - int(percentage) // 2) - + f"] {int(percentage)}%" - ) - sys.stdout.flush() - - inpha = all_ifgs_cor[i, :, :] # your phase input, float32 - incoh = all_coh[i, :, :] # your coh input. It can be either 0-255 or 0-1 - - output_phase_path = os.path.join(output_wrap_dir, all_ifgs_string[i]) - output_cc_path = os.path.join(output_wrap_dir, all_ifgs_string[i]) - - # output_phase_path = output_wrap_dir + all_ifgs_string[i] - # output_cc_path = output_wrap_dir + all_ifgs_string[i] - - os.makedirs(output_phase_path, exist_ok=True) - os.makedirs(output_cc_path, exist_ok=True) - - output_phase_file = ( - output_phase_path + "/" + all_ifgs_string[i] + ".geo.diff_pha.tif" - ) - - output_cc_file = output_cc_path + "/" + all_ifgs_string[i] + ".geo.cc.tif" - - ifg_cc_2tif(inpha, output_phase_file, template_tif_path) - ifg_cc_2tif(incoh, output_cc_file, template_tif_path) - -print("\nAll corrected interferograms have been written to disk.") diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/config.txt b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/config.txt deleted file mode 100644 index 15f08de..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/config.txt +++ /dev/null @@ -1,43 +0,0 @@ -[DEFAULT] -# That path to interferograms directory. In case of LiCSAR data set LiCSAR_public='yes', and provide the frame id -root_path = /gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products.public/44/044A_04728_131314 - -output_path = /work/scratch-pw4/earyma/ESA_project/044A_04728_131314 - -LiCSAR_data=no -frame=044A_04728_131314 - -# Provide the start and end dates of the time-series that you want to calculate the phase bias correction -start=20170513 -end=20210504 - -#Provide the data acquisition interval: 6-day or 12-day -interval=6 - - -# If you want to apply further multilooking to interferograms (expand the explanation) -nlook=10 - - -# The calibration parameters 'an' can be estimated from the data by specifying the number of parameters to be determined. For instance, setting num_a=2 indicates that -#a1 and a2 will be estimated. - -num_a=2 - - -### an parameters -# in case of yes the an values will be estimated from the data, otherwise the default values given below will be used -estimate_an_values=no - -# Using 6-days interval -a1_6_day=0.50 -a2_6_day=0.36 -a3_6_day=0.299 -a4_6_day=0.2476 - - -# using 12-day interval -a1_12_day=0.494 -a2_12_day=0.297 -a3_12_day=0.24 -a4_12_day=0.22 diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/config_12day.txt b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/config_12day.txt deleted file mode 100644 index 3c92abc..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/config_12day.txt +++ /dev/null @@ -1,45 +0,0 @@ -[DEFAULT] -# That path to interferograms directory. In case of LiCSAR data set LiCSAR_public='yes', and provide the frame id -root_path = /gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products.public/44/044A_04728_131314 - -output_path = /work/scratch-pw4/earyma/ESA_project/044A_04728_131314 - -LiCSAR_data=no -frame=044A_04728_131314 - -# Provide the start and end dates of the time-series that you want to calculate the phase bias correction -start=20170513 -end=20210504 - -#Provide the data acquisition interval: 6-day or 12-day -# Modified for 12-day and 24-day data -interval=12 - - -# If you want to apply further multilooking to interferograms (expand the explanation) -nlook=10 - -# The calibration parameters 'an' can be estimated from the data by specifying the number of parameters to be determined. For instance, setting num_a=2 indicates that -#a1 and a2 will be estimated. - -num_a=2 - - -### an parameters -# in case of yes the an values will be estimated from the data, otherwise the default values given below will be used -# Modified: estimate from data for 12-day intervals -estimate_an_values=yes - -# Using 6-days interval (original) -a1_6_day=0.50 -a2_6_day=0.36 -a3_6_day=0.299 -a4_6_day=0.2476 - - -# using 12-day interval (for 12-day base interferograms) -# These values will be used if estimate_an_values=no -a1_12_day=0.494 -a2_12_day=0.297 -a3_12_day=0.24 -a4_12_day=0.22 diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/.gitkeep b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/.gitkeep deleted file mode 100644 index 8b13789..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/loop_closure_time_series.png b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/loop_closure_time_series.png deleted file mode 100644 index d5252b1..0000000 Binary files a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/loop_closure_time_series.png and /dev/null differ diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/spatial_distribution_a1.png b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/spatial_distribution_a1.png deleted file mode 100644 index 9f6fd52..0000000 Binary files a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/spatial_distribution_a1.png and /dev/null differ diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/spatial_distribution_a2.png b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/spatial_distribution_a2.png deleted file mode 100644 index 492ff32..0000000 Binary files a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/spatial_distribution_a2.png and /dev/null differ diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/timeseries_a1.png b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/timeseries_a1.png deleted file mode 100644 index 26cac51..0000000 Binary files a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/timeseries_a1.png and /dev/null differ diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/timeseries_a2.png b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/timeseries_a2.png deleted file mode 100644 index 7a8cbdf..0000000 Binary files a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/timeseries_a2.png and /dev/null differ diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/LT1_import_SLC_from_zipfiles b/.codex_tmp/pyint_variants/no_rescue/pyint/LT1_import_SLC_from_zipfiles deleted file mode 100644 index f953e16..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/LT1_import_SLC_from_zipfiles +++ /dev/null @@ -1,121 +0,0 @@ -#! /bin/csh -f -echo "*** S1_import_SLC_from_zipfiles: Script to read in and concatenate S1 TOPS SLC from zip files ***" -echo "*** Copyright 2023 Gamma Remote Sensing, v2.3 19-Apr-2023 uw/cm ***" -echo "" - -if ($#argv < 1)then - echo "usage: S1_import_SLC_from_zipfiles [burst_number_table_ref] [pol] [dtype] [swath_flag] [OPOD_dir] [cleaning] [noise_mode]" - echo " zipfile_list (input) ASCII file containing S1 zip filename(s) of one data take (one per line, in correct sequence)" - echo " (other values are vv, vh, hh, hv)" - echo " dtype output data type (enter - for default)" - echo " 0: FCOMPLEX (default)" - echo " 1: SCOMPLEX" - echo " " - echo "resulting files: burst SLC files (per polarization, with SLC_tab, SLC, SLC_par, TOPS_par and optionally SLC.noise)" - echo "(concatenated, empty bursts added where necessary) at selected polarizations" - echo " " - exit -endif - - - -# 6-Jan-2023: logfile name updated to be the same on every platform - -set zipfile_list = $1 - # in this case it generates temp.burst_number_table_ref -set dtype = "0" # 0: FCOMPLEX - - -if ($#argv >= 2) then - if ( "$2" != "-" ) then - set dtype = $2 - endif -endif - - - -set zipfile=`date "+%Y%m%d_%H%M%S"` -set logfile = "LT1_import_SLC_from_zipfiles.$zipfile.logfile" -echo "the processing is documented in the logfile: $logfile" - -echo "COMMAND_used: $0 $1 $2 " > $logfile -date >> $logfile -echo "" >> $logfile - -############################################################################## -if (-e "$1" == 0) then - echo "ERROR: zipfile_list ($1) does not exist"; exit(-1) -else - #determine number of rows of $zipfile_list - set zipfile=`awk '(NR>=1){print NF}' $zipfile_list` - set nrows=`echo "$zipfile" | awk '(NR==1){print NF}'` -endif - -set i="1" -while ( "$i" <= "$nrows" ) - # interpret names, check consistency of names - # read first zipfile name and reduce it to filename without path - set zipfile = `awk '(NR=='"$i"'){print $1}' $zipfile_list` - if ( -e $zipfile == "0" ) then - echo "ERROR: zipfile number $1 ($zipfile) does not exist"; exit(-1) - endif - - set zipfile = `awk '(NR=='"$i"'){print $1}' $zipfile_list` - # set l = `echo "$zipfile" | awk '{print length($1)}'` - set m = `echo "$zipfile tar.gz" | awk '{print match($1,$2)-71}'` - set filename_without_path = `echo "$zipfile $m" | awk '{print substr($1,$2,70)}'` - set path_only = `echo "$zipfile $m" | awk '{print substr($1,1,$2-1)}'` - # echo "$path_only $filename_without_path" - echo "$filename_without_path" - echo "$path_only" - - set sensor = `echo "$filename_without_path" | awk '{print substr($1,1,4)}'` - set mode = `echo "$filename_without_path" | awk '{print substr($1,15,6)}'` - set polar = `echo "$filename_without_path" | awk '{print substr($1,54,2)}'` - set level = `echo "$filename_without_path" | awk '{print substr($1,50,3)}'` - set date = `echo "$filename_without_path" | awk '{print substr($1,41,8)}'` - set orbit = `echo "$filename_without_path" | awk '{print substr($1,22,6)}'` - set product = `echo "$filename_without_path" | awk '{print substr($1,61,10)}'` - echo "sensor: $sensor mode: $mode polar: $polar level: $level date: $date orbit: $orbit product: $product" - - if ( "$level" != "SLC" ) then - echo "ERROR: type indicated in zipfile ($type) is not SLC"; exit(-1) - endif - - set GeoTIFF = `tar -tzf $zipfile | grep "tiff" | awk '{print $1}'` - set xml_name = `tar -tzf $zipfile | grep "meta.xml" | awk '{print $1}'` - set num = `grep "$date" $zipfile_list | wc -l ` - if ( ! -d $date ) then - mkdir $date - endif - if ( ! -d tmp_data_dir ) then - mkdir tmp_data_dir - endif - if ( $num>1 ) then - set SLC = ${date}_$product.slc - set SLC_par = ${date}_$product.slc.par - set SLC1 = ${date}_$product.slc.update - set SLC1_par = ${date}_$product.slc.update.par - else - set SLC = $date.slc - set SLC_par = $date.slc.par - set SLC1 = $date.slc.update - set SLC1_par = $date.slc.update.par - endif - tar -xzv -C tmp_data_dir -f $zipfile $GeoTIFF - tar -xzv -C tmp_data_dir -f $zipfile $xml_name - echo "COMMAND: par_LT1_SLC ./tmp_data_dir/$GeoTIFF ./tmp_data_dir/$xml_name ./$date/$SLC_par ./$date/$SLC " - par_LT1_SLC ./tmp_data_dir/$GeoTIFF ./tmp_data_dir/$xml_name ./$date/$SLC_par ./$date/$SLC - echo "COMMAND: par_LT1_SLC ./tmp_data_dir/$GeoTIFF ./tmp_data_dir/$xml_name ./$date/$SLC1 ./$date/$SLC1_par " - par_LT1_SLC_YSLi ./tmp_data_dir/$GeoTIFF ./tmp_data_dir/$xml_name ./$date/$SLC1 ./$date/$SLC1_par 0 - grep "state_vector" ./$date/$SLC1_par | awk -F ' ' '{printf " %s %.5f %.5f %.5f %s %s\n", $1, $2, $3, $4, $5,$6}' | awk '{if (NR%2==1){print $0,"m/s"}else{print $0,"m"}}' | awk 'NR<=3 {print $1,$2}' >tmp1 - grep "state_vector" ./$date/$SLC1_par | awk -F ' ' '{printf " %s %.5f %.5f %.5f %s %s\n", $1, $2, $3, $4, $5,$6}' | awk '{if (NR%2==1){print $0,"m/s"}else{print $0,"m"}}' | awk 'NR>3 {print $0}' >tmp2 - set histchars="" - awk '!/state_vector/ ' ./$date/$SLC_par >tmp - cat tmp tmp1 tmp2 > ./$date/$SLC_par - rm tmp1 tmp tmp2 - set i = `echo "$i" | awk '{printf "%d", $1+1}'` -end -rm -rf *.logfile tmp_data_dir - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/LT1_import_SLC_from_zipfiles1 b/.codex_tmp/pyint_variants/no_rescue/pyint/LT1_import_SLC_from_zipfiles1 deleted file mode 100644 index 7141360..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/LT1_import_SLC_from_zipfiles1 +++ /dev/null @@ -1,126 +0,0 @@ -#! /bin/csh -f -echo "*** S1_import_SLC_from_zipfiles: Script to read in and concatenate S1 TOPS SLC from zip files ***" -echo "*** Copyright 2023 Gamma Remote Sensing, v2.3 19-Apr-2023 uw/cm ***" -echo "" - -if ($#argv < 1)then - echo "usage: S1_import_SLC_from_zipfiles [burst_number_table_ref] [pol] [dtype] [swath_flag] [OPOD_dir] [cleaning] [noise_mode]" - echo " zipfile_list (input) ASCII file containing S1 zip filename(s) of one data take (one per line, in correct sequence)" - echo " (other values are vv, vh, hh, hv)" - echo " dtype output data type (enter - for default)" - echo " 0: FCOMPLEX (default)" - echo " 1: SCOMPLEX" - echo " " - echo "resulting files: burst SLC files (per polarization, with SLC_tab, SLC, SLC_par, TOPS_par and optionally SLC.noise)" - echo "(concatenated, empty bursts added where necessary) at selected polarizations" - echo " " - exit -endif - - - -# 6-Jan-2023: logfile name updated to be the same on every platform - -set zipfile_list = $1 - # in this case it generates temp.burst_number_table_ref -set dtype = "0" # 0: FCOMPLEX - - -if ($#argv >= 2) then - if ( "$2" != "-" ) then - set dtype = $2 - endif -endif - - - -set zipfile=`date "+%Y%m%d_%H%M%S"` -set logfile = "LT1_import_SLC_from_zipfiles.$zipfile.logfile" -echo "the processing is documented in the logfile: $logfile" - -echo "COMMAND_used: $0 $1 $2 " > $logfile -date >> $logfile -echo "" >> $logfile - -############################################################################## -if (-e "$1" == 0) then - echo "ERROR: zipfile_list ($1) does not exist"; exit(-1) -else - #determine number of rows of $zipfile_list - set zipfile=`awk '(NR>=1){print NF}' $zipfile_list` - set nrows=`echo "$zipfile" | awk '(NR==1){print NF}'` -endif - - # interpret names, check consistency of names - # read first zipfile name and reduce it to filename without path - set zipfile = `awk '(NR=='"1"'){print $1}' $zipfile_list` - if ( -e $zipfile == "0" ) then - echo "ERROR: zipfile number $1 ($zipfile) does not exist"; exit(-1) - endif - - set zipfile = `awk '(NR=='"1"'){print $1}' $zipfile_list` - set input_basename = `basename "$zipfile"` - set input_mode = "archive" - if ( `echo "$input_basename" | grep -c ".tar.gz$"` > 0 ) then - set filename_without_path = `echo "$input_basename" | sed 's/\.tar\.gz$//'` - set GeoTIFF = `tar -tzf $zipfile | grep "tiff" | awk '{print $1}'` - set xml_name = `tar -tzf $zipfile | grep "meta.xml" | awk '{print $1}'` - if ( ! -d tmp_data_dir ) then - mkdir tmp_data_dir - endif - tar -xzv -C tmp_data_dir -f $zipfile $GeoTIFF - tar -xzv -C tmp_data_dir -f $zipfile $xml_name - set input_tiff = ./tmp_data_dir/$GeoTIFF - set input_xml = ./tmp_data_dir/$xml_name - else if ( `echo "$input_basename" | grep -c ".tiff$"` > 0 ) then - set input_mode = "scene" - set filename_without_path = `echo "$input_basename" | sed 's/\.tiff$//'` - set input_tiff = $zipfile - set input_xml = `echo "$zipfile" | sed 's/\.tiff$/.meta.xml/'` - if ( -e "$input_xml" == 0 ) then - echo "ERROR: meta xml for scene input does not exist ($input_xml)"; exit(-1) - endif - else - echo "ERROR: unsupported LT-1 input ($zipfile), only .tar.gz or .tiff are supported"; exit(-1) - endif - # echo "$filename_without_path" - - set sensor = `echo "$filename_without_path" | awk '{print substr($1,1,3)}'` - set mode = `echo "$filename_without_path" | awk '{print substr($1,15,5)}'` - set polar = `echo "$filename_without_path" | awk '{print substr($1,54,2)}'` - set level = `echo "$filename_without_path" | awk '{print substr($1,50,3)}'` - set date = `echo "$filename_without_path" | awk '{print substr($1,41,8)}'` - set orbit = `echo "$filename_without_path" | awk '{print substr($1,22,5)}'` - set product = `echo "$filename_without_path" | awk '{print substr($1,61,10)}'` - echo "sensor: $sensor mode: $mode polar: $polar level: $level date: $date orbit: $orbit product: $product" - - if ( "$level" != "SLC" ) then - echo "ERROR: type indicated in zipfile ($type) is not SLC"; exit(-1) - endif - - set num = `grep "$date" t_$date | wc -l ` - - if ( $num>1 ) then - set SLC = ${date}_${product}.slc - set SLC_par = ${date}_${product}.slc.par - set SLC1 = ${date}_${product}.slc.update - set SLC1_par = ${date}_${product}.slc.update.par - else - set SLC = $date.slc - set SLC_par = $date.slc.par - set SLC1 = $date.slc.update - set SLC1_par = $date.slc.update.par - endif - - echo "COMMAND: par_LT1_SLC $input_tiff $input_xml ./$SLC_par ./$SLC " - par_LT1_SLC $input_tiff $input_xml ./$SLC_par ./$SLC - echo "COMMAND: par_LT1_SLC $input_tiff $input_xml ./$SLC1 ./$SLC1_par " - par_LT1_SLC_YSLi $input_tiff $input_xml ./$SLC1 ./$SLC1_par 0 - grep "state_vector" ./$SLC1_par | awk -F ' ' '{printf " %s %.5f %.5f %.5f %s %s\n", $1, $2, $3, $4, $5,$6}' | awk '{if (NR%2==1){print $0,"m/s"}else{print $0,"m"}}' | awk 'NR<=3 {print $1,$2}' >tmp1 - grep "state_vector" ./$SLC1_par | awk -F ' ' '{printf " %s %.5f %.5f %.5f %s %s\n", $1, $2, $3, $4, $5,$6}' | awk '{if (NR%2==1){print $0,"m/s"}else{print $0,"m"}}' | awk 'NR>3 {print $0}' >tmp2 - set histchars="" - awk '!/state_vector/ ' ./$SLC_par >tmp - cat tmp tmp1 tmp2 > ./$SLC_par - rm tmp1 tmp tmp2 -rm -rf *.logfile tmp_data_dir - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/MAI_SLC_Gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/MAI_SLC_Gamma.py deleted file mode 100644 index 340d6e6..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/MAI_SLC_Gamma.py +++ /dev/null @@ -1,202 +0,0 @@ -#! /usr/bin/env python -#''' -################################################################################## -# # -# Author: Yun-Meng Cao # -# Email : ymcmrs@gmail.com # -# Date : February, 2017 # -# # -# Split beam of SLC: backward and forward SLC image generation # -# # -################################################################################## -#''' -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob -from pyint import _utils as ut -import argparse - - - -INTRODUCTION = ''' ------------------------------------------------------------------------------------------- - Split beam of original SLC to generate sub-aperture SLC: backward- and forward-SLCs. - -''' - -EXAMPLE = ''' - Usage: - MAI_SLC_gamma.py projectName Mdate Sdate - MAI_SLC_gamma.py PacayaT163TsxHhA 20150102 20150601 ------------------------------------------------------------------------------------------- -''' - - - - -def check_variable_name(path): - s=path.split("/")[0] - if len(s)>0 and s[0]=="$": - p0=os.getenv(s[1:]) - path=path.replace(path.split("/")[0],p0) - return path - -def read_template(File, delimiter='='): - '''Reads the template file into a python dictionary structure. - Input : string, full path to the template file - Output: dictionary, pysar template content - Example: - tmpl = read_template(KyushuT424F610_640AlosA.template) - tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') - ''' - template_dict = {} - for line in open(File): - line = line.strip() - c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter - if len(c) < 2 or line.startswith('%') or line.startswith('#'): - next #ignore commented lines or those without variables - else: - atrName = c[0] - atrValue = str.replace(c[1],'\n','').split("#")[0].strip() - atrValue = check_variable_name(atrValue) - template_dict[atrName] = atrValue - return template_dict - -def ras2jpg(input, strTitle): - call_str = "convert " + input + ".ras " + input + ".jpg" - os.system(call_str) - call_str = "convert " + input + ".jpg -resize 250 " + input + ".thumb.jpg" - os.system(call_str) - call_str = "convert " + input + ".jpg -resize 500 " + input + ".bthumb.jpg" - os.system(call_str) - call_str = "$INT_SCR/addtitle2jpg.pl " + input + ".thumb.jpg 14 " + strTitle - os.system(call_str) - call_str = "$INT_SCR/addtitle2jpg.pl " + input + ".bthumb.jpg 24 " + strTitle - os.system(call_str) - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close() - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Split beam of original SLC to generate sub-aperture SLC: backward- and forward-SLCs',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('Mdate',help='Master date.') - parser.add_argument('Sdate',help='Slave date.') - - inps = parser.parse_args() - return inps - - - - -def main(argv): - start_time = time.time() - inps = cmdLineParse() - Mdate = inps.Mdate - Sdate = inps.Sdate - - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - masterDate = templateDict['masterDate'] - coregCoarse = templateContents['Coreg_Coarse'] - if 'Squint_MAI' in templateContents: Squint = templateContents['Squint_MAI'] - else: Squint = '0.5' - - - - projectDir = scratchDir + '/' + projectName - demDir = scratchDir + '/' + projectName + '/DEM' - - slcDir = scratchDir + '/' + projectName + '/SLC' - rslcDir = scratchDir + '/' + projectName + '/RSLC' - ifgDir = projectDir + '/MAIifg' - if not os.path.isdir(ifgDir): os.mkdir(ifgDir) - - Pair = Mdate + '-' + Sdate - workDir = ifgDir + '/' + Pair - if not os.path.isdir(workDir): os.mkdir(workDir) - - -# input slcs - ####################################################################### - Mamp = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' - MampPar = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' - Samp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' - SampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' - - Mrslc = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc' - MrslcPar = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc.par' - Srslc = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc' - SrslcPar = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc.par' - - HGT = demDir + '/' + masterDate + '_' + rlks + 'rlks.rdc.dem' - - MasterPar = rslcDir + '/' + masterDate + '/' + masterDate + '.rslc.par' - - - - -# split slcs - - MFslcImg = workDir + "/" + Mdate + ".F.slc" - MFslcPar = workDir + "/" + Mdate + ".F.slc.par" - SFslcImg = workDir + "/" + Sdate + ".F.slc" - SFslcPar = workDir + "/" + Sdate + ".F.slc.par" - - MBslcImg = workDir + "/" + Mdate + ".B.slc" - MBslcPar = workDir + "/" + Mdate + ".B.slc.par" - SBslcImg = workDir + "/" + Sdate + ".B.slc" - SBslcPar = workDir + "/" + Sdate + ".B.slc.par" - - print(MFslcImg + " "+ MslcImg) - -# Multi-aperture processing - - call_str = '$GAMMA_BIN/sbi_filt '+ MslcImg + ' ' + MrslcPar + ' '+SrslcPar + ' ' + MFslcImg + ' '+ MFslcPar + ' ' + MBslcImg + ' ' + MBslcPar + ' ' + Squint - os.system(call_str) - print(call_str) - - call_str = '$GAMMA_BIN/sbi_filt '+ SslcImg + ' ' + SrslcPar + ' '+MrslcPar + ' ' + SFslcImg + ' '+ SFslcPar + ' ' + SBslcImg + ' ' + SBslcPar + ' ' + Squint - os.system(call_str) - - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - - - - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/MAI_SLC_Gamma1.py b/.codex_tmp/pyint_variants/no_rescue/pyint/MAI_SLC_Gamma1.py deleted file mode 100644 index bd35061..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/MAI_SLC_Gamma1.py +++ /dev/null @@ -1,134 +0,0 @@ -#! /usr/bin/env python -#''' -################################################################################## -# # -# Author: Wei Chen # -# Email : chenweicug@gmail.com # -# Date : May,25th 2021 # -# # -# Split beam of SLC: backward and forward SLC image generation # -# # -################################################################################## -#''' - -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob -import argparse - -from pyint import _utils as ut - - -def get_s1_date(raw_file): - file0 = os.path.basename(raw_file) - date = file0[17:25] - return date - -def get_satellite(raw_file): - if 'S1A_IW_SLC_' in raw_file: - s0 = 'A' - else: - s0 = 'B' - - return s0 - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Generate SLC from Sentinel-1 raw data with orbit correction using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName', help='project name. e.g., ChangningT55S1A') - parser.add_argument('Mdate',help='date to be processed. e.g., 20180101') - parser.add_argument('Sdate',help='date to be processed. e.g., 20180113') - - inps = parser.parse_args() - - return inps - - -INTRODUCTION = ''' -------------------------------------------------------------------- - - Split beam of original SLC to generate sub-aperture SLC: backward- and forward-SLCs -''' - -EXAMPLE = """Usage: - - MAI_SLC_Gamma.py projectName Mdate Sdate - - MAI_SLC_Gamma.py ChangningT55S1A 20180517 20180529 - -------------------------------------------------------------------- -""" - -def main(argv): - inps = cmdLineParse() - projectName = inps.projectName - Mdate = inps.Mdate - Sdate = inps.Sdate - scratchDir = os.getenv('SCRATCHDIR') - projectDir = scratchDir + '/' + projectName - - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - - slcDir = projectDir + '/SLC' - down_dir = projectDir + '/DOWNLOAD' - workDir = projectDir + '/MAI' - - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - Mdate = templateDict['masterDate'] - coregCoarse = templateDict['Coreg_Coarse'] - - if not os.path.isdir(workDir): - call_str="mkdir " + workDir - os.system(call_str) - if 'Squint_MAI' in templateDict: Squint = templateContents['Squint_MAI'] - else: Squint = '0.5' - - SslcDir = slcDir + "/" + Sdate - MslcDir = slcDir + "/" + Mdate - - MslcImg = MslcDir + "/" + Mdate + ".slc" - MslcPar = MslcDir + "/" + Mdate + ".slc.par" - SslcImg = SslcDir + "/" + Sdate + ".slc" - SslcPar = SslcDir + "/" + Sdate + ".slc.par" - -# split slcs - - MFslcImg = workDir + "/" + Mdate + ".F.slc" - MFslcPar = workDir + "/" + Mdate + ".F.slc.par" - SFslcImg = workDir + "/" + Sdate + ".F.slc" - SFslcPar = workDir + "/" + Sdate + ".F.slc.par" - - MBslcImg = workDir + "/" + Mdate + ".B.slc" - MBslcPar = workDir + "/" + Mdate + ".B.slc.par" - SBslcImg = workDir + "/" + Sdate + ".B.slc" - SBslcPar = workDir + "/" + Sdate + ".B.slc.par" - print(MFslcImg + " "+ MslcImg) - -# Multi-aperture processing - - call_str = 'sbi_filt '+ MslcImg + ' ' + MslcPar + ' '+SslcPar + ' ' + MFslcImg + ' '+ MFslcPar + ' ' + MBslcImg + ' ' + MBslcPar + ' ' + Squint - os.system(call_str) - print(call_str) - - call_str = 'sbi_filt '+ SslcImg + ' ' + SslcPar + ' '+MslcPar + ' ' + SFslcImg + ' '+ SFslcPar + ' ' + SBslcImg + ' ' + SBslcPar + ' ' + Squint - os.system(call_str) - - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/NameChange.py b/.codex_tmp/pyint_variants/no_rescue/pyint/NameChange.py deleted file mode 100644 index 6889a2d..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/NameChange.py +++ /dev/null @@ -1,180 +0,0 @@ -#! /usr/bin/env python -#''' -################################################################################### -# # -# Author: Yun-Meng Cao # -# Email : ymcmrs@gmail.com # -# Date : Mar, 2017 # -# # -# Select Interferometry-Pairs from time series SAR images # -# # -################################################################################### -#''' -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob - -def check_variable_name(path): - s=path.split("/")[0] - if len(s)>0 and s[0]=="$": - p0=os.getenv(s[1:]) - path=path.replace(path.split("/")[0],p0) - return path - -def read_template(File, delimiter='='): - '''Reads the template file into a python dictionary structure. - Input : string, full path to the template file - Output: dictionary, pysar template content - Example: - tmpl = read_template(KyushuT424F610_640AlosA.template) - tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') - ''' - template_dict = {} - for line in open(File): - line = line.strip() - c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter - if len(c) < 2 or line.startswith('%') or line.startswith('#'): - next #ignore commented lines or those without variables - else: - atrName = c[0] - atrValue = str.replace(c[1],'\n','').split("#")[0].strip() - atrValue = check_variable_name(atrValue) - template_dict[atrName] = atrValue - return template_dict - - -def is_number(s): - try: - int(s) - return True - except ValueError: - return False - -def add_zero(s): - if len(s)==1: - s="000"+s - elif len(s)==2: - s="00"+s - elif len(s)==3: - s="0"+s - return s - - -def usage(): - print(''' -****************************************************************************************************** - - Select interferometry pairs from time series SAR images - - usage: - - SelectPairs_Gamma.py ProjectName - - e.g. SelectPairs_Gamma.py PacayaT163TsxHhA - - -******************************************************************************************************* - ''') - -def main(argv): - - if len(sys.argv)==2: - if argv[0] in ['-h','--help']: usage(); sys.exit(1) - else: projectName=sys.argv[1] - else: - usage();sys.exit(1) - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateContents=read_template(templateFile) - processDir = scratchDir + '/' + projectName + "/PROCESS" - slcDir = scratchDir + '/' + projectName + "/SLC" - - if not os.path.isdir(processDir): - call_str = 'mkdir ' + processDir - os.system(call_str) - - - if 'JOB' in templateContents: JOB = templateContents['JOB'] - else: JOB = 'IFG' - - INF=JOB - if INF=='IFG': - Suffix=[''] - print("Time series interferograms will be processed!") - elif INF=='MAI': - Suffix=['.F','.B'] - print("Time series multi-aperture interferograms will be processed!") - elif INF=='RSI': - Suffix=['.HF','.LF'] - print("Time series range split-spectrum interferograms will be processed!") - else: - print("The folder name %s cannot be identified !" % igramDir) - usage();sys.exit(1) - -# define files - - SLC_Tab = processDir + "/SLC_Tab" - TS_Berp = processDir + "/TS_Berp" - TS_Itab = processDir + "/TS_Itab" - itab_type = '1' - pltflg = '0' - - if 'Max_Spacial_Baseline' in templateContents: MaxSB=templateContents['Max_Spacial_Baseline'] - else: - print("Max_Spacial_Baseline is not found in template!! ") - print("500m is chosen as the threshold for spatial baseline!") - MaxSB = '500' - - if 'Max_Temporal_Baseline' in templateContents: MaxTB=templateContents['Max_Temporal_Baseline'] - else: - print("Max_Temporal_Baseline is not found in template!! ") - print("500 days is chosen as the threshold for temporal baseline!") - MaxTB = '500' - - -# extract available SAR images slc and slc_par - ListSLC = os.listdir(slcDir) - Datelist = [] - SLCfile = [] - SLCParfile = [] - - print("All of the available SAR acquisition datelist is :") - for kk in range(len(ListSLC)): - if ( is_number(ListSLC[kk]) and len(ListSLC[kk])==6 ): # if SAR date number is 8, 6 should change to 8. - DD=ListSLC[kk] - Year=int(DD[0:2]) - Month = int(DD[2:4]) - Day = int(DD[4:6]) - if ( 0 < Year < 20 and 0 < Month < 13 and 0 < Day < 32 ): - Datelist.append(ListSLC[kk]) - print(ListSLC[kk]) - DateDir = slcDir+'/'+ListSLC[kk] - SLC0 = glob.glob(DateDir+'/*slc')[0] - SLCPar0 = glob.glob(DateDir+'/*slc.par')[0] - - str_slc = slcDir + "/" + ListSLC[kk] +"/" + ListSLC[kk] + ".slc" - str_slc_par = slcDir + "/" + ListSLC[kk] +"/" + ListSLC[kk] + ".slc.par" - - call_str = 'mv ' + SLC0 + ' ' + str_slc - os.system(call_str) - - call_str = 'mv ' + SLCPar0 + ' ' + str_slc_par - os.system(call_str) - - SLCfile.append(str_slc) - SLCParfile.append(str_slc_par) - - - print("Change name of SLC file is done! ") - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/OPENTOPO_USAGE.md b/.codex_tmp/pyint_variants/no_rescue/pyint/OPENTOPO_USAGE.md deleted file mode 100644 index bbf8be5..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/OPENTOPO_USAGE.md +++ /dev/null @@ -1,83 +0,0 @@ -# OpenTopography DEM 下载使用示例 - -## 快速开始 - -### 1. 获取免费API Key - -访问: https://opentopography.org/myOpenTopo -- 注册账户 (免费) -- 在 'My Account' 页面请求 API key -- API key 立即可用 - -### 2. 下载不同分辨率的 DEM - -#### SRTM 30m (推荐) -```bash -python makedem.py -r 116/117/39/40 --dem-source srtm \ - --opentopo-api-key YOUR_KEY \ - --opentopo-dem-type SRTMGL1 -``` - -#### SRTM 90m -```bash -python makedem.py -r 116/117/39/40 --dem-source srtm \ - --opentopo-api-key YOUR_KEY \ - --opentopo-dem-type SRTMGL3 -``` - -#### NASADEM 30m -```bash -python makedem.py -r 116/117/39/40 --dem-source srtm \ - --opentopo-api-key YOUR_KEY \ - --opentopo-dem-type NASADEM -``` - -#### Copernicus 30m -```bash -python makedem.py -r 116/117/39/40 --dem-source srtm \ - --opentopo-api-key YOUR_KEY \ - --opentopo-dem-type COP30 -``` - -#### Copernicus 90m -```bash -python makedem.py -r 116/117/39/40 --dem-source srtm \ - --opentopo-api-key YOUR_KEY \ - --opentopo-dem-type COP90 -``` - -## 数据对比 - -| DEM类型 | 分辨率 | 覆盖范围 | 推荐用途 | -|---------|--------|----------|----------| -| **SRTMGL1** | **30m** | 60°S-60°N | **SRTM研究首选** | -| SRTMGL3 | 90m | 60°S-60°N | 快速预览 | -| **NASADEM** | **30m** | 60°S-60°N | **高精度研究** | -| **COP30** | **30m** | 全球 | **全球覆盖首选** | -| COP90 | 90m | 全球 | 快速预览 | - -## 推荐选择 - -1. **SRTM研究**: SRTMGL1 (30m) -2. **高精度需求**: NASADEM 或 COP30 (30m) -3. **全球覆盖**: COP30 (30m) -4. **快速预览**: SRTMGL3 或 COP90 (90m) - -## 注意事项 - -- **SRTMGL1/COP30/NASADEM** (30m): 最大支持 450,000 km² -- **SRTMGL3/COP90** (90m): 最大支持 4,050,000 km² -- 所有数据集需要 OpenTopography API key -- 免费配额足够大部分科研用途 - -## 输出文件命名 - -- SRTMGL1_116_117_39_40.tif -- NASADEM_116_117_39_40.tif -- COP30_116_117_39_40.tif - -文件名格式: `{DEM类型}_{西经}_{东经}_{南纬}_{北纬}.tif` - ---- - -**更新日期**: 2026-03-13 diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/POT_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/POT_gamma.py deleted file mode 100644 index 4e87a87..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/POT_gamma.py +++ /dev/null @@ -1,394 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Pixel Offset Tracking (POT) for a single pair ### -### Based on GAMMA offset_pwr_tracking / offset_pwr_tracking2### -### Author: ZYD / Cascade AI ### -################################################################# - -import os -import sys -import time -import argparse -import numpy as np - -from pyint import _utils as ut - - -INTRODUCTION = ''' -------------------------------------------------------------------- - Pixel Offset Tracking (POT) for a single interferometric pair - using GAMMA software. - - Two-round estimation approach (Greenland tracking demo): - Round 1: Large search window for initial offset field - Post-processing: Quality check, outlier removal, gap filling - Round 2: Smaller window with conditioned Round 1 as initial - Final: Convert pixel offsets to displacement in meters + Geocode -''' - -EXAMPLE = ''' - Usage: - POT_gamma.py projectName Mdate Sdate - POT_gamma.py shanghaiT171F128S1A 20241105 20241117 -------------------------------------------------------------------- -''' - - -def run_cmd(cmd_str): - """执行 GAMMA 命令并打印""" - print(f' >> {cmd_str}') - return os.system(cmd_str) - - -def sanitize_gamma_float(filepath, valid_max=1e6): - """ - 清理 GAMMA 浮点数据文件中的无效值。 - GAMMA 对无效像素写入特殊标记值（~3.4e38 / NaN / Inf）， - 这些值无法被 single_class_mapping 的范围过滤器正确捕获。 - 将 NaN / Inf / |val| > valid_max 的像素替换为 0.0。 - """ - data = np.fromfile(filepath, dtype=np.float32) - bad_mask = ~np.isfinite(data) | (np.abs(data) > valid_max) - n_bad = int(np.sum(bad_mask)) - if n_bad > 0: - data[bad_mask] = 0.0 - data.tofile(filepath) - print(f' [sanitize] {os.path.basename(filepath)}: ' - f'清理 {n_bad} 个无效像素 (NaN/Inf/|val|>{valid_max})') - - -def postprocess_offsets(offs_cpx, ccp, mli, width, - ccp_thresh, roff_min, roff_max, - azoff_min, azoff_max, - drange_thresh, dazimuth_thresh, - median_win, median_nmin, - prefix): - """ - 偏移量场后处理流程（参考 GAMMA Greenland tracking demo）: - 1) 提取距离向/方位向分量 + 清理 GAMMA no-data 标记值 - 2) 初始掩膜: 互相关阈值 + 偏移量范围限制 - 3) 中值滤波 + 偏差计算 - 4) 偏差阈值精细掩膜 - 5) 空洞填充 - 6) 空间滤波 → 组合为 conditioned 复数偏移量 - - 返回: (condi, real_interp, imag_interp) - - condi: fspf 平滑后的复数偏移量（供 Round 2 初始值） - - real_interp / imag_interp: 填充后的偏移量（供最终米制转换） - """ - real_file = prefix + '.real' - imag_file = prefix + '.imag' - - # --- 1) 提取距离向 (real) 和方位向 (imag) --- - run_cmd(f'cpx_to_real {offs_cpx} {real_file} {width} 0') - run_cmd(f'cpx_to_real {offs_cpx} {imag_file} {width} 1') - - # 清理 GAMMA no-data 标记值（~3.4e38），避免污染后续掩膜和插值 - valid_max = max(abs(float(roff_max)), abs(float(roff_min)), - abs(float(azoff_max)), abs(float(azoff_min))) * 10 - sanitize_gamma_float(real_file, valid_max) - sanitize_gamma_float(imag_file, valid_max) - sanitize_gamma_float(ccp, 1.0) - - # --- 2) 初始掩膜: 互相关 + 偏移量范围 --- - mask1 = prefix + '.mask1.bmp' - real_m1 = prefix + '.real.masked1' - imag_m1 = prefix + '.imag.masked1' - - run_cmd(f'single_class_mapping 3 ' - f'{ccp} {ccp_thresh} 1.0 ' - f'{real_file} {roff_min} {roff_max} ' - f'{imag_file} {azoff_min} {azoff_max} ' - f'{mask1} {width} 1 0 1 1') - run_cmd(f'mask_class {mask1} {real_file} {real_m1} 0 1 1 1 0 0.0') - run_cmd(f'mask_class {mask1} {imag_file} {imag_m1} 0 1 1 1 0 0.0') - - # 初始掩膜后 BMP - run_cmd(f'rasdt_pwr {real_m1} {mli} {width} - - - - ' - f'{roff_min} {roff_max} 0 rmg.cm {real_m1}.bmp - - 24') - run_cmd(f'rasdt_pwr {imag_m1} {mli} {width} - - - - ' - f'{azoff_min} {azoff_max} 0 rmg.cm {imag_m1}.bmp - - 24') - - # --- 3) 中值滤波 + 偏差 --- - real_med = prefix + '.real.median' - imag_med = prefix + '.imag.median' - dreal = prefix + '.dreal' - dimag = prefix + '.dimag' - - run_cmd(f'median_filter {real_m1} {real_med} {width} ' - f'{median_win} {median_win} {median_nmin}') - run_cmd(f'lin_comb 2 {real_m1} {real_med} 0. 1. -1. ' - f'{dreal} {width} 1 0 1 1') - run_cmd(f'median_filter {imag_m1} {imag_med} {width} ' - f'{median_win} {median_win} {median_nmin}') - run_cmd(f'lin_comb 2 {imag_m1} {imag_med} 0. 1. -1. ' - f'{dimag} {width} 1 0 1 1') - - # --- 4) 偏差阈值精细掩膜 --- - mask2 = prefix + '.mask2.bmp' - real_masked = prefix + '.real.masked' - imag_masked = prefix + '.imag.masked' - - run_cmd(f'single_class_mapping 5 ' - f'{dreal} -{drange_thresh} {drange_thresh} ' - f'{dimag} -{dazimuth_thresh} {dazimuth_thresh} ' - f'{ccp} {ccp_thresh} 1.0 ' - f'{real_file} {roff_min} {roff_max} ' - f'{imag_file} {azoff_min} {azoff_max} ' - f'{mask2} {width} 1 0 1 1 5') - run_cmd(f'mask_class {mask2} {real_file} {real_masked} 0 1 1 1 0 0.0') - run_cmd(f'mask_class {mask2} {imag_file} {imag_masked} 0 1 1 1 0 0.0') - - # 精细掩膜后 BMP - run_cmd(f'rasdt_pwr {real_masked} {mli} {width} - - - - ' - f'{roff_min} {roff_max} 0 rmg.cm {real_masked}.bmp - - 24') - run_cmd(f'rasdt_pwr {imag_masked} {mli} {width} - - - - ' - f'{azoff_min} {azoff_max} 0 rmg.cm {imag_masked}.bmp - - 24') - - # --- 5) 空洞填充 --- - real_interp = prefix + '.real.interp' - imag_interp = prefix + '.imag.interp' - run_cmd(f'fill_gaps {real_masked} {width} {real_interp} 0 4 - 1') - run_cmd(f'fill_gaps {imag_masked} {width} {imag_interp} 0 4 - 1') - - # 清理 fill_gaps 插值可能引入的 NaN/Inf/极端值 - sanitize_gamma_float(real_interp, valid_max) - sanitize_gamma_float(imag_interp, valid_max) - - # 填充后 BMP - run_cmd(f'rasdt_pwr {real_interp} {mli} {width} - - - - ' - f'{roff_min} {roff_max} 0 rmg.cm {real_interp}.bmp - - 24') - run_cmd(f'rasdt_pwr {imag_interp} {mli} {width} - - - - ' - f'{azoff_min} {azoff_max} 0 rmg.cm {imag_interp}.bmp - - 24') - - # --- 6) 空间滤波 + 组合 conditioned --- - real_fspf = prefix + '.real.fspf' - imag_fspf = prefix + '.imag.fspf' - run_cmd(f'fspf {real_interp} {real_fspf} {width} 2 2 2') - run_cmd(f'fspf {imag_interp} {imag_fspf} {width} 2 2 2') - - condi = prefix + '.condi' - run_cmd(f'real_to_cpx {real_fspf} {imag_fspf} {condi} {width} 0') - - return condi, real_interp, imag_interp - - -def cmdLineParse(): - parser = argparse.ArgumentParser( - description='Pixel Offset Tracking for a single pair using GAMMA.', - formatter_class=argparse.RawTextHelpFormatter, - epilog=INTRODUCTION + '\n' + EXAMPLE) - - parser.add_argument('projectName', help='projectName for processing.') - parser.add_argument('Mdate', help='Master date.') - parser.add_argument('Sdate', help='Slave date.') - - inps = parser.parse_args() - return inps - - -def main(argv): - - start_time = time.time() - inps = cmdLineParse() - Mdate = inps.Mdate - Sdate = inps.Sdate - projectName = inps.projectName - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + '/' + projectName + '.template' - templateDict = ut.update_template(templateFile) - - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - masterDate = templateDict['masterDate'] - - # ========== POT 参数 ========== - pot_rstep = templateDict['pot_rstep'] - pot_azstep = templateDict['pot_azstep'] - pot_rwin = templateDict['pot_rwin'] - pot_azwin = templateDict['pot_azwin'] - pot_rwin2 = templateDict['pot_rwin2'] - pot_azwin2 = templateDict['pot_azwin2'] - pot_ovr = templateDict['pot_ovr'] - pot_snr_thresh = templateDict['pot_snr_thresh'] - pot_ccp_thresh = templateDict['pot_ccp_thresh'] - pot_roff_min = templateDict['pot_roff_min'] - pot_roff_max = templateDict['pot_roff_max'] - pot_azoff_min = templateDict['pot_azoff_min'] - pot_azoff_max = templateDict['pot_azoff_max'] - pot_drange_thresh = templateDict['pot_drange_thresh'] - pot_dazimuth_thresh = templateDict['pot_dazimuth_thresh'] - pot_median_win = templateDict['pot_median_win'] - pot_median_nmin = templateDict['pot_median_nmin'] - pot_two_rounds = templateDict['pot_two_rounds'] - pot_geocode = templateDict['pot_geocode'] - pot_disp_max = templateDict['pot_disp_max'] - - # ========== 目录 ========== - projectDir = scratchDir + '/' + projectName - rslcDir = projectDir + '/RSLC' - demDir = projectDir + '/DEM' - potDir = projectDir + '/offsets' - if not os.path.isdir(potDir): - os.mkdir(potDir) - - Pair = Mdate + '-' + Sdate - workDir = potDir + '/' + Pair - if not os.path.isdir(workDir): - os.mkdir(workDir) - - # ========== 输入文件 ========== - Mrslc = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc' - MrslcPar = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc.par' - Srslc = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc' - SrslcPar = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc.par' - - slc_width = ut.read_gamma_par(MrslcPar, 'read', 'range_samples') - off_width = str(int(int(slc_width) // int(pot_rstep))) - - print('=' * 60) - print(f'Pixel Offset Tracking (POT): {Pair}') - print(f' SLC width : {slc_width}') - print(f' Offset width : {off_width}') - print(f' Step (r x az) : {pot_rstep} x {pot_azstep}') - print(f' R1 window : {pot_rwin} x {pot_azwin}') - if pot_two_rounds == '1': - print(f' R2 window : {pot_rwin2} x {pot_azwin2}') - print(f' Offset range : [{pot_roff_min}, {pot_roff_max}] r ' - f'[{pot_azoff_min}, {pot_azoff_max}] az') - print('=' * 60) - - ####################################################################### - # Step 1: 生成偏移量几何下的 MLI（背景图 + 尺寸参考） - ####################################################################### - print('\n[Step 1] 生成偏移量几何 MLI ...') - MLI_pot = workDir + '/' + Mdate + '.mli_pot' - MLI_pot_par = workDir + '/' + Mdate + '.mli_pot.par' - - run_cmd(f'multi_look {Mrslc} {MrslcPar} {MLI_pot} {MLI_pot_par} ' - f'{pot_rstep} {pot_azstep}') - run_cmd(f'raspwr {MLI_pot} {off_width} - - - - 1. .2 - {MLI_pot}.bmp') - - ####################################################################### - # Step 2: 创建偏移量参数文件 - ####################################################################### - print('\n[Step 2] 创建偏移量参数文件 ...') - OFF = workDir + '/' + Pair + '.off' - run_cmd(f'create_offset {MrslcPar} {SrslcPar} {OFF} 1 {rlks} {azlks} 0') - - ####################################################################### - # Step 3: Round 1 — 初始偏移量估计（大窗口） - ####################################################################### - print(f'\n[Step 3] Round 1 偏移量估计 ({pot_rwin}x{pot_azwin}) ...') - r1_tag = f'{pot_rwin}x{pot_azwin}' - r1_prefix = workDir + '/' + Pair + '.offs' + r1_tag - offs_r1 = r1_prefix - ccp_r1 = workDir + '/' + Pair + '.ccp' + r1_tag - - run_cmd(f'offset_pwr_tracking {Mrslc} {Srslc} {MrslcPar} {SrslcPar} ' - f'{OFF} {offs_r1} {ccp_r1} ' - f'{pot_rwin} {pot_azwin} - {pot_ovr} {pot_snr_thresh} ' - f'{pot_rstep} {pot_azstep}') - - ####################################################################### - # Step 4: Round 1 后处理 - ####################################################################### - print(f'\n[Step 4] Round 1 后处理 ...') - r1_condi, _, _ = postprocess_offsets( - offs_cpx=offs_r1, ccp=ccp_r1, mli=MLI_pot, width=off_width, - ccp_thresh=pot_ccp_thresh, - roff_min=pot_roff_min, roff_max=pot_roff_max, - azoff_min=pot_azoff_min, azoff_max=pot_azoff_max, - drange_thresh=pot_drange_thresh, dazimuth_thresh=pot_dazimuth_thresh, - median_win=pot_median_win, median_nmin=pot_median_nmin, - prefix=r1_prefix) - - # 默认使用 Round 1 结果 - final_prefix = r1_prefix - final_ccp = ccp_r1 - final_off = OFF - - ####################################################################### - # Step 5-6: Round 2 — 精细偏移量估计（小窗口，可选） - ####################################################################### - if pot_two_rounds == '1': - print(f'\n[Step 5] Round 2 偏移量估计 ({pot_rwin2}x{pot_azwin2}) ...') - OFF2 = workDir + '/' + Pair + '.off2' - run_cmd(f'create_offset {MrslcPar} {SrslcPar} {OFF2} 1 {rlks} {azlks} 0') - - r2_tag = f'{pot_rwin2}x{pot_azwin2}' - r2_prefix = workDir + '/' + Pair + '.offs' + r2_tag - offs_r2 = r2_prefix - ccp_r2 = workDir + '/' + Pair + '.ccp' + r2_tag - - run_cmd(f'offset_pwr_tracking2 {Mrslc} {Srslc} {MrslcPar} {SrslcPar} ' - f'{OFF2} {offs_r2} {ccp_r2} {OFF} {r1_condi} ' - f'{pot_rwin2} {pot_azwin2} - {pot_ovr} {pot_snr_thresh} ' - f'{pot_rstep} {pot_azstep}') - - print(f'\n[Step 6] Round 2 后处理 ...') - _, _, _ = postprocess_offsets( - offs_cpx=offs_r2, ccp=ccp_r2, mli=MLI_pot, width=off_width, - ccp_thresh=pot_ccp_thresh, - roff_min=pot_roff_min, roff_max=pot_roff_max, - azoff_min=pot_azoff_min, azoff_max=pot_azoff_max, - drange_thresh=pot_drange_thresh, dazimuth_thresh=pot_dazimuth_thresh, - median_win=pot_median_win, median_nmin=pot_median_nmin, - prefix=r2_prefix) - - final_prefix = r2_prefix - final_ccp = ccp_r2 - final_off = OFF2 - - ####################################################################### - # Step 7: 像素偏移量 → 米制位移量 - ####################################################################### - print('\n[Step 7] 像素偏移量转换为地面位移 (米) ...') - - # 用填充后（非 fspf 平滑）的偏移量组合复数，供 offset_tracking 使用 - final_real_interp = final_prefix + '.real.interp' - final_imag_interp = final_prefix + '.imag.interp' - final_offs_combined = final_prefix + '.offs_combined' - run_cmd(f'real_to_cpx {final_real_interp} {final_imag_interp} ' - f'{final_offs_combined} {off_width} 0') - - disp_map = workDir + '/' + Pair + '.disp_map' - run_cmd(f'offset_tracking {final_offs_combined} {final_ccp} ' - f'{MrslcPar} {final_off} {disp_map} - 2 {pot_ccp_thresh} 0') - - # 提取位移分量 - disp_real = disp_map + '.real' # 地距向位移 (米) - disp_imag = disp_map + '.imag' # 方位向位移 (米) - disp_mag = disp_map + '.mag' # 位移幅值 (米) - - run_cmd(f'cpx_to_real {disp_map} {disp_real} {off_width} 0') - run_cmd(f'cpx_to_real {disp_map} {disp_imag} {off_width} 1') - run_cmd(f'cpx_to_real {disp_map} {disp_mag} {off_width} 3') - - # 清理 offset_tracking 输出中的 NaN/Inf/极端值 - disp_max_m = float(pot_disp_max) * 10 - sanitize_gamma_float(disp_real, disp_max_m) - sanitize_gamma_float(disp_imag, disp_max_m) - sanitize_gamma_float(disp_mag, disp_max_m) - - # BMP 可视化 - run_cmd(f'rasdt_pwr {disp_real} {MLI_pot} {off_width} - - - - ' - f'-{pot_disp_max} {pot_disp_max} 1 rmg.cm {disp_real}.bmp - - 24') - run_cmd(f'rasdt_pwr {disp_imag} {MLI_pot} {off_width} - - - - ' - f'-{pot_disp_max} {pot_disp_max} 1 rmg.cm {disp_imag}.bmp - - 24') - run_cmd(f'rasdt_pwr {disp_mag} {MLI_pot} {off_width} - - - - ' - f'-{pot_disp_max} {pot_disp_max} 1 rmg.cm {disp_mag}.bmp - - 24') - - # 注意: 地理编码已移至 geocode_gamma.py --type pot - # 用法: geocode_gamma.py projectName Pair --type pot - - print(f"\nPixel Offset Tracking for {Pair} is done!") - ut.print_process_time(start_time, time.time()) - sys.exit(0) - - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/POT_gamma_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/POT_gamma_all.py deleted file mode 100644 index 07c365c..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/POT_gamma_all.py +++ /dev/null @@ -1,148 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Batch Pixel Offset Tracking (POT) using GAMMA ### -### Author: ZYD / Cascade AI ### -################################################################# - -import numpy as np -import os -import sys -import getopt -import time -import glob -import argparse - -import subprocess -from pyint import _utils as ut - - -def work(data0): - cmd = data0[0] - err_file = data0[1] - p = subprocess.run(cmd, shell=False, stderr=subprocess.PIPE, stdout=subprocess.PIPE) - stdout = p.stdout - stderr = p.stderr - - if type(stderr) == bytes: - aa = stderr.decode("utf-8") - else: - aa = stderr - - if aa: - str0 = ' '.join(cmd) + '\n' - with open(err_file, 'a') as f: - f.write(str0) - f.write(aa) - f.write('\n') - - return - -######################################################################### - -INTRODUCTION = ''' -------------------------------------------------------------------- - Batch Pixel Offset Tracking for one project using GAMMA. - - Runs POT_gamma.py for each interferometric pair in parallel. - Skips pairs that already have completed results (disp_map.mag.bmp). -''' - -EXAMPLE = ''' - Usage: - POT_gamma_all.py projectName - POT_gamma_all.py projectName --parallel 4 - POT_gamma_all.py projectName --parallel 4 --ifgarmList-txt /test/ifgram_list.txt -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser( - description='Batch Pixel Offset Tracking for one project using GAMMA.', - formatter_class=argparse.RawTextHelpFormatter, - epilog=INTRODUCTION + '\n' + EXAMPLE) - - parser.add_argument('projectName', help='projectName for processing.') - parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, - help='Enable parallel processing and specify the number of processors.') - parser.add_argument('--ifgarmList-txt', dest='ifgarmListTxt', - help='Provided ifgram_list_txt. Default: using ifgram_list.txt under projectName folder.') - - inps = parser.parse_args() - return inps - - -def main(argv): - start_time = time.time() - inps = cmdLineParse() - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - projectDir = scratchDir + '/' + projectName - potDir = projectDir + '/offsets' - if not os.path.isdir(potDir): - os.mkdir(potDir) - - templateDict = ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - - if inps.ifgarmListTxt: - ifgramList_txt = inps.ifgarmListTxt - else: - ifgramList_txt = projectDir + '/ifgram_list.txt' - - ifgList0 = ut.read_txt2array(ifgramList_txt) - - if len(ifgList0) == 3: - ifgList = ifgList0[0] - ifgList = [ifgList] - else: - ifgList = ifgList0[:, 0] - - err_txt = projectDir + '/POT_gamma_all.err' - if os.path.isfile(err_txt): - os.remove(err_txt) - - data_para = [] - skip_count = 0 - for i in range(len(ifgList)): - m0 = ut.yyyymmdd(ifgList[i].split('-')[0]) - s0 = ut.yyyymmdd(ifgList[i].split('-')[1]) - cmd0 = ['POT_gamma.py', projectName, m0, s0] - - # 检查完成标志: disp_map.mag.bmp - Pair = ifgList[i] - disp_bmp = potDir + '/' + Pair + '/' + Pair + '.disp_map.mag.bmp' - - k00 = 0 - if os.path.isfile(disp_bmp): - if os.path.getsize(disp_bmp) > 0: - k00 = 1 - skip_count += 1 - - if k00 == 0: - data0 = [cmd0, err_txt] - data_para.append(data0) - - total = len(ifgList) - todo = len(data_para) - print(f'Pixel Offset Tracking: {total} pairs total, ' - f'{skip_count} already done, {todo} to process') - print(f'Parallel processors: {inps.parallelNumb}') - print(f'Output directory: {potDir}') - print('=' * 60) - - if todo > 0: - ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) - - print("Pixel Offset Tracking for project %s is done! " % projectName) - ut.print_process_time(start_time, time.time()) - - sys.exit(0) - - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/README.md b/.codex_tmp/pyint_variants/no_rescue/pyint/README.md deleted file mode 100644 index 1995412..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/README.md +++ /dev/null @@ -1,26 +0,0 @@ -## GACOS CORRECTION FOR GMTSAR -These software is intended to correct unwrapped phases using GACOS grids for GMTSAR. - -They have been tested with: GMT v6.1.0 GMTSAR v6.0 - -On "Test_against_matlab" folder, I perform comparisons with matlab with consistent results. -To reproduce the output just use: source command.txt in each folder. -There is a small README file in each folder - -### SINGLE INTERFEROGRAM -single_GACOS_correction.csh is intented to correct a single interferogram at a time with certain parameters - -### STACK OF INTERFEROGRAMS -GACOS_correction.csh and operation.csh are used to correct a stack of interferograms in a time series process - -Corrections are applied to the unwrap.grd files - -Feel free to use and edit the code if needed - -References: - -*Yu, C., Li, Z., Penna, N. T., & Crippa, P. (2018). Generic atmospheric correction model for Interferometric Synthetic Aperture Radar observations. Journal of Geophysical Research: Solid Earth, 123(10), 9202-9222.* - -*Yu, C., Li, Z., & Penna, N. T. (2018). Interferometric synthetic aperture radar atmospheric correction using a GPS-based iterative tropospheric decomposition model. Remote Sensing of Environment, 204, 109-121.* - -*Yu, C., Penna, N. T., & Li, Z. (2017). Generation of real‐time mode high‐resolution water vapor fields from GPS observations. Journal of Geophysical Research: Atmospheres, 122(3), 2008-2025.* diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/RSI_SLC_Gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/RSI_SLC_Gamma.py deleted file mode 100644 index 8dfb52f..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/RSI_SLC_Gamma.py +++ /dev/null @@ -1,216 +0,0 @@ -#! /usr/bin/env python -#''' -################################################################################### -# # -# Author: Yun-Meng Cao # -# Email : ymcmrs@gmail.com # -# Date : March , 2017 # -# # -# Range Split Spectrum for SAR complex data based on GAMMA # -# # -################################################################################### -#''' -import numpy as np -import os -import pysar._readfile as readfile -import sys -import subprocess -import getopt -import time -import glob - -def check_variable_name(path): - s=path.split("/")[0] - if len(s)>0 and s[0]=="$": - p0=os.getenv(s[1:]) - path=path.replace(path.split("/")[0],p0) - return path - -def read_template(File, delimiter='='): - '''Reads the template file into a python dictionary structure. - Input : string, full path to the template file - Output: dictionary, pysar template content - Example: - tmpl = read_template(KyushuT424F610_640AlosA.template) - tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') - ''' - template_dict = {} - for line in open(File): - line = line.strip() - c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter - if len(c) < 2 or line.startswith('%') or line.startswith('#'): - next #ignore commented lines or those without variables - else: - atrName = c[0] - atrValue = str.replace(c[1],'\n','').split("#")[0].strip() - atrValue = check_variable_name(atrValue) - template_dict[atrName] = atrValue - return template_dict - - -def ras2jpg(input, strTitle): - call_str = "convert " + input + ".ras " + input + ".jpg" - os.system(call_str) - call_str = "convert " + input + ".jpg -resize 250 " + input + ".thumb.jpg" - os.system(call_str) - call_str = "convert " + input + ".jpg -resize 500 " + input + ".bthumb.jpg" - os.system(call_str) - call_str = "$INT_SCR/addtitle2jpg.pl " + input + ".thumb.jpg 14 " + strTitle - os.system(call_str) - call_str = "$INT_SCR/addtitle2jpg.pl " + input + ".bthumb.jpg 24 " + strTitle - os.system(call_str) - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close() - -def geocode(inFile, outFile, UTMTORDC, nWidth, nWidthUTMDEM, nLineUTMDEM): - if inFile.rsplit('.')[1] == 'int': - call_str = '$GAMMA_BIN/geocode_back ' + inFile + ' ' + nWidth + ' ' + UTMTORDC + ' ' + outFile + ' ' + nWidthUTMDEM + ' ' + nLineUTMDEM + ' 0 1' - else: - call_str = '$GAMMA_BIN/geocode_back ' + inFile + ' ' + nWidth + ' ' + UTMTORDC + ' ' + outFile + ' ' + nWidthUTMDEM + ' ' + nLineUTMDEM + ' 0 0' - os.system(call_str) - - -def createBlankFile(strFile): - f = open(strFile,'w') - for i in range (10): - f.write('\n') - f.close() - - -def usage(): - print(''' -****************************************************************************************************** - - Split Spectrum on range direction to generate high-frequency- and low-frequency- SLCs - - usage: - - RSI_SLC_Gamma.py igramDir - - e.g. RSI_SLC_Gamma.py RSI_PacayaT163TsxHhA_131021-131101_0011_-0007 - - -******************************************************************************************************* - ''') - -def main(argv): - - if len(sys.argv)==2: - if argv[0] in ['-h','--help']: usage(); sys.exit(1) - else: igramDir=sys.argv[1] - else: - usage();sys.exit(1) - - INF = igramDir.split('_')[0] - projectName = igramDir.split('_')[1] - IFGPair = igramDir.split(projectName+'_')[1].split('_')[0] - Mdate = IFGPair.split('-')[0] - Sdate = IFGPair.split('-')[1] - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - - processDir = scratchDir + '/' + projectName + "/PROCESS" - slcDir = scratchDir + '/' + projectName + "/SLC" - workdir = processDir + '/' + igramDir - - templateContents=read_template(templateFile) - rlks = templateContents['Range_Looks'] - azlks = templateContents['Azimuth_Looks'] - - if INF!='RSI': - usage();sys.exit(1) - - if not os.path.isdir(workdir): - call_str = 'mkdir ' + workdir - os.system(call_str) - - SslcDir = slcDir + "/" + Sdate - MslcDir = slcDir + "/" + Mdate - -# input slcs - - SslcDir = slcDir + "/" + Sdate - MslcDir = slcDir + "/" + Mdate - - MslcImg = MslcDir + "/" + Mdate + ".slc" - MslcPar = MslcDir + "/" + Mdate + ".slc.par" - SslcImg = SslcDir + "/" + Sdate + ".slc" - SslcPar = SslcDir + "/" + Sdate + ".slc.par" - - - MHslcImg = workdir + '/'+Mdate + '.HF.slc' - MHslcPar = workdir + '/'+Mdate + '.HF.slc.par' - call_str = 'cp ' + MslcPar + ' '+ MHslcPar - os.system(call_str) - - SHslcImg = workdir + '/'+Sdate + '.HF.slc' - SHslcPar = workdir + '/'+Sdate + '.HF.slc.par' - call_str = 'cp ' + SslcPar + ' '+ SHslcPar - os.system(call_str) - - MLslcImg = workdir + '/'+Mdate + '.LF.slc' - MLslcPar = workdir + '/'+Mdate + '.LF.slc.par' - call_str = 'cp ' + MslcPar + ' '+ MLslcPar - os.system(call_str) - - SLslcImg = workdir + '/'+Sdate + '.LF.slc' - SLslcPar = workdir + '/'+Sdate + '.LF.slc.par' - call_str = 'cp ' + SslcPar + ' '+ SLslcPar - os.system(call_str) - - nWidth = UseGamma(MslcPar, 'read','range_samples:') - - call_str= 'bpf ' + MslcImg + ' ' + MHslcImg + ' ' + nWidth + ' 0.25 0.5 0 1 0 0 - - 1' - os.system(call_str) - - call_str= 'bpf ' + MslcImg + ' ' + MLslcImg + ' ' + nWidth + ' -0.25 0.5 0 1 0 0 - - 1' - os.system(call_str) - - nWidth = UseGamma(SslcPar, 'read','range_samples:') - - call_str= 'bpf ' + SslcImg + ' ' + SHslcImg + ' ' + nWidth + ' 0.25 0.5 0 1 0 0 - - 1' - os.system(call_str) - - call_str= 'bpf ' + SslcImg + ' ' + SLslcImg + ' ' + nWidth + ' -0.25 0.5 0 1 0 0 - - 1' - os.system(call_str) - - - print("Split spectrum for both slave and master date is done!") - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - - - - - - - - - - - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/Raw2SLC_ERS_Cat.py b/.codex_tmp/pyint_variants/no_rescue/pyint/Raw2SLC_ERS_Cat.py deleted file mode 100644 index 3bd572c..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/Raw2SLC_ERS_Cat.py +++ /dev/null @@ -1,306 +0,0 @@ -#! /usr/bin/env python -#''' -################################################################################## -# # -# Author: Yun-Meng Cao # -# Email : ymcmrs@gmail.com # -# Date : June, 2019 # -# # -# Generate SLC from SAR_IMS_P1 data # -# # -################################################################################## -#''' -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob - -def check_variable_name(path): - s=path.split("/")[0] - if len(s)>0 and s[0]=="$": - p0=os.getenv(s[1:]) - path=path.replace(path.split("/")[0],p0) - return path - -def read_template(File, delimiter='='): - '''Reads the template file into a python dictionary structure. - Input : string, full path to the template file - Output: dictionary, pysar template content - Example: - tmpl = read_template(KyushuT424F610_640AlosA.template) - tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') - ''' - template_dict = {} - for line in open(File): - line = line.strip() - c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter - if len(c) < 2 or line.startswith('%') or line.startswith('#'): - next #ignore commented lines or those without variables - else: - atrName = c[0] - atrValue = str.replace(c[1],'\n','').split("#")[0].strip() - atrValue = check_variable_name(atrValue) - template_dict[atrName] = atrValue - return template_dict - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close - -def rm(TXT): - call_str = 'rm ' + TXT - os.system(call_str) - -def usage(): - print(''' -****************************************************************************************************** - - Generate SLC and SLC_par file for ERS/ENVISAT (SAR_IMS_1P format) - - usage: - - Down2SLC_ERS.py ProjectName DownName - - e.g. Down2SLC_ERS.py CotopaxiT120ERSA 910101 - -******************************************************************************************************* - ''') - -def main(argv): - - if len(sys.argv)==3: - projectName = sys.argv[1] - Date = sys.argv[2] - else: - usage();sys.exit(1) - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + '/' + projectName + '.template' - templateContents=read_template(templateFile) - - if 'rlks4cor' in templateContents: rlks4cor = templateContents['rlks4cor'] - else: rlks4cor = '4' - if 'azlks4cor' in templateContents: azlks4cor = templateContents['azlks4cor'] - else: azlks4cor = '4' - - if 'rwin4cor' in templateContents: rwin4cor = templateContents['rwin4cor'] - else: rwin4cor = '128' - if 'azwin4cor' in templateContents: azwin4cor = templateContents['azwin4cor'] - else: azwin4cor = '128' - if 'rsample4cor' in templateContents: rsample4cor = templateContents['rsample4cor'] - else: rsample4cor = '64' - if 'azsample4cor' in templateContents: azsample4cor = templateContents['azsample4cor'] - else: azsample4cor = '64' - - if ' rpos4cor' in templateContents: rpos4cor = templateContents[' rpos4cor'] - else: rpos4cor = ' - ' - if 'azpos4cor' in templateContents: azpos4cor = templateContents['azpos4cor'] - else: azpos4cor = ' - ' - - - if 'rfwin4cor' in templateContents: rfwin4cor = templateContents['rfwin4cor'] - else: rfwin4cor = str(int(int(rwin4cor)/2)) - if 'azfwin4cor' in templateContents: azfwin4cor = templateContents['azfwin4cor'] - else: azfwin4cor = str(int(int(azwin4cor)/2)) - if 'rfsample4cor' in templateContents: rfsample4cor = templateContents['rfsample4cor'] - else: rfsample4cor = str(2*int(rsample4cor)) - if 'azfsample4cor' in templateContents: azfsample4cor = templateContents['azfsample4cor'] - else: azfsample4cor = str(2*int(azsample4cor)) - - if 'thresh4cor' in templateContents: thresh4cor = templateContents['thresh4cor'] - else: thresh4cor = ' 0.2 ' - - rlks = templateContents['Range_Looks'] - azlks = templateContents['Azimuth_Looks'] - - - projectDir = scratchDir + '/' + projectName - downDir = scratchDir + '/' + projectName + "/DOWNLOAD" - slcDir = scratchDir + '/' + projectName + '/SLC' - - if not os.path.isdir(slcDir): - call_str= 'mkdir ' +slcDir - os.system(call_str) - os.chdir(downDir) - - t0 = 't0_' + Date - call_str = 'ls >' + t0 - os.system(call_str) - - tt = 'tt_' + Date - call_str = "grep " + Date + ' ' + t0 + '> ' + tt - os.system(call_str) - - te = 'te_' + Date - call_str = "grep SAR_IM " + tt + " > " + te - os.system(call_str) - - AA= np.loadtxt(te,dtype=np.str) - Na = AA.size - AA=AA.reshape(Na,) - - for i in range(Na): - Date0 = Date+'_' + str(i) - downName = str(AA[i]) - FileDir = downDir + '/' + downName - call_str = 'raw2slc_ers_envisat.py '+ FileDir + ' -o ' + Date0 - os.system(call_str) - - #call_str ="rename 's/VV.SLC/slc/g' *" - #os.system(call_str) - - #slcpar = Date0 + '.slc.par' - #call_str = 'ERS_orb_cor_par.py ' + slcpar - #os.system(call_str) - - Date0 = Date - if len(Date)==6: - Date6 = Date - Date0 = Date - elif len(Date)==8: - Date0 = Date[2:8] - Date6 = Date[2:8] - else: - print('The input Date is invalid.') - sys.exit(1) - - dataDir = slcDir + '/' + Date0 - if not os.path.isdir(dataDir): - call_str = 'mkdir ' + dataDir - print('Generate SLC dir for date: ' + Date0) - os.system(call_str) - call_str = 'mv ' + Date + '*.slc* ' + dataDir - os.system(call_str) - - os.chdir(dataDir) - for i in range(Na): - Date0 = Date+'_' + str(i) - downName = str(AA[i]) - - SLCm = Date0 + '.slc' - SLCm_par = Date0 + '.slc.par' - MamprlksImg = Date0 + '.amp' - MamprlksPar = Date0 + '.amp.par' - - call_str = 'multi_look ' + SLCm + ' ' + SLCm_par + ' ' + MamprlksImg + ' ' + MamprlksPar + ' ' + rlks + ' ' + azlks - os.system(call_str) - nWidth = UseGamma(MamprlksPar, 'read', 'range_samples') - call_str = 'raspwr ' + MamprlksImg + ' ' + nWidth - os.system(call_str) - - SLC = Date6+'.slc' - SLCPAR = Date6 + '.slc.par' - AMP = Date6+'.amp' - AMPPAR = Date6+'.amp.par' - if Na==1: - os.rename(SLCm,SLC) - os.rename(SLCm_par,SLCPAR) - os.rename(MamprlksImg,AMP) - os.rename(MamprlksPar,AMPPAR) - - for i in range(Na-1): - if i==0: - DateA = Date + '_0' - else: - DateA = Date + '_' + str(i-1) + str(i) - - SLCA = DateA + '.slc' - SLCA_par = DateA + '.slc.par' - - - DateB = Date + '_' +str(i+1) - SLCB = DateB + '.slc' - SLCB_par = DateB + '.slc.par' - - DateC = Date + '_' + str(i) + str(i+1) - SLCC = DateC + '.slc' - SLCC_par = DateC + '.slc.par' - - ##################################################################################### - MamprlksImg = Date + '.amp' - MamprlksPar = Date + '.amp.par' - - off = DateC + '.off' - offs = DateC + '.offs' - offsets = DateC + '.offsets' - coffs = DateC + '.coffs' - coffsets = DateC + '.coffsets' - snr = DateC + '.snr' - off_std = DateC + '.off_std' - - ########################## Generate off file ############################# - - call_str = "create_offset " + SLCA_par + " " + SLCB_par + " " + off + " 1 - - 0" - os.system(call_str) - call_str = 'init_offset_orbit '+ SLCA_par + " " + SLCB_par + ' ' + off - os.system(call_str) - - - call_str = 'init_offset '+ SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' ' + rlks4cor + ' ' + azlks4cor + ' ' + rpos4cor + ' ' + azpos4cor - os.system(call_str) - - call_str = 'init_offset '+ SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' 1 1 - - ' - os.system(call_str) - - call_str = "offset_pwr " + SLCA + " " + SLCB + " " + SLCA_par + " " + SLCB_par + " " + off + " " + offs + " " + snr + " " + rwin4cor + " " + azwin4cor + " " + offsets + " 2 "+ rsample4cor + " " + azsample4cor - os.system(call_str) - - call_str = "offset_fit " + offs + " " + snr + " " + off + " " + coffs + " " + coffsets + " " + thresh4cor +" 3" - os.system(call_str) - - call_str = "offset_pwr " +SLCA + " " + SLCB + " " + SLCA_par + " " + SLCB_par + " " + off + " " + offs + " " + snr + " " + rfwin4cor + " " + azfwin4cor + " " + offsets + " 2 " + rfsample4cor + " " + azfsample4cor - os.system(call_str) - - call_str = "offset_fit " + offs + " " + snr + " " + off + " " + coffs + " " + coffsets + " " + thresh4cor + " 4 >" + off_std - os.system(call_str) - - ######################################################################################## - - call_str = 'SLC_cat ' + SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' ' + SLCC + ' ' + SLCC_par - os.system(call_str) - - if i==(Na-2): - if len(Date)==6: - DD = Date - else: - DD = Date[2:8] - SLCm = DD + '.slc' - SLCm_par = DD + '.slc.par' - - - call_str = 'cp ' + SLCC + ' ' + SLCm - os.system(call_str) - - call_str = 'cp ' + SLCC_par + ' ' + SLCm_par - os.system(call_str) - - call_str = 'multi_look ' + SLCm + ' ' + SLCm_par + ' ' + MamprlksImg + ' ' + MamprlksPar + ' ' + rlks + ' ' + azlks - os.system(call_str) - nWidth = UseGamma(MamprlksPar, 'read', 'range_samples') - call_str = 'raspwr ' + MamprlksImg + ' ' + nWidth - os.system(call_str) - - call_str = 'rm *.amp' - os.system(call_str) - - call_str = 'rm *_*.slc' - os.system(call_str) - - print("Down to SLC for %s is done! " % Date) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/Raw2SLC_ERS_Cat_All.py b/.codex_tmp/pyint_variants/no_rescue/pyint/Raw2SLC_ERS_Cat_All.py deleted file mode 100644 index 4586dc9..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/Raw2SLC_ERS_Cat_All.py +++ /dev/null @@ -1,136 +0,0 @@ -#! /usr/bin/env python -#''' -################################################################################## -# # -# Author: Yun-Meng Cao # -# Email : ymcmrs@gmail.com # -# Date : March, 2017 # -# # -# Generate Sentinel SLC from the downloaded data # -# # -################################################################################## -#''' -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob - -def check_variable_name(path): - s=path.split("/")[0] - if len(s)>0 and s[0]=="$": - p0=os.getenv(s[1:]) - path=path.replace(path.split("/")[0],p0) - return path - -def read_template(File, delimiter='='): - '''Reads the template file into a python dictionary structure. - Input : string, full path to the template file - Output: dictionary, pysar template content - Example: - tmpl = read_template(KyushuT424F610_640AlosA.template) - tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') - ''' - template_dict = {} - for line in open(File): - line = line.strip() - c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter - if len(c) < 2 or line.startswith('%') or line.startswith('#'): - next #ignore commented lines or those without variables - else: - atrName = c[0] - atrValue = str.replace(c[1],'\n','').split("#")[0].strip() - atrValue = check_variable_name(atrValue) - template_dict[atrName] = atrValue - return template_dict - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close - -def rm(TXT): - call_str = 'rm ' + TXT - os.system(call_str) - -def usage(): - print(''' -****************************************************************************************************** - - Cat ERS SLC from ERS raw data with ENVISAT format. - - usage: - - Raw2SLC_ERS_Cat_All.py ProjectName - - e.g. Raw2SLC_ERS_Cat_All.py CotopaxiT120ERSA - -******************************************************************************************************* - ''') - -def main(argv): - - if len(sys.argv)==2: - projectName = sys.argv[1] - else: - usage();sys.exit(1) - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - - projectDir = scratchDir + '/' + projectName - downDir = scratchDir + '/' + projectName + "/DOWNLOAD" - slcDir = scratchDir + '/' + projectName + '/SLC' - - if not os.path.isdir(slcDir): - call_str= 'mkdir ' +slcDir - os.system(call_str) - - os.chdir(downDir) - - call_str = 'ls > tt0' - os.system(call_str) - - call_str = 'grep SAR_IM tt0 >tt1' - os.system(call_str) - - call_str = "awk -F0PWDSI '{print $2}' tt1 > tt2 " - os.system(call_str) - - call_str = "awk -F_ '{print $1}' tt2 > ttt" - os.system(call_str) - - call_str = 'sort ttt | uniq > ttm' - os.system(call_str) - - AA= np.loadtxt('ttm',dtype=np.str) - Na = AA.size - - for i in range(Na): - call_str = 'Raw2SLC_ERS_Cat.py ' + projectName + ' ' + AA[i] - print(call_str) - call_str = 'Raw2SLC_ERS_Cat.py ' + projectName + ' ' + AA[i] + ' >/dev/null' - os.system(call_str) - - - print("Down to SLC for %s is done! " % projectName) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/SAR2LATLON.py b/.codex_tmp/pyint_variants/no_rescue/pyint/SAR2LATLON.py deleted file mode 100644 index 35a6c3b..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/SAR2LATLON.py +++ /dev/null @@ -1,144 +0,0 @@ -#! /usr/bin/env python - -#''' -################################################################################## -### Author: Yun-Meng Cao ### -### Date : March, 2017 ### -### Email : ymcmrs@gmail.com ### -### Transform SAR coordinates into LAT/LON coordinates based on lookup table ### -################################################################################## -#''' - -import os -import sys -import glob -import time -import argparse - -import h5py -import numpy as np - -def check_variable_name(path): - s=path.split("/")[0] - if len(s)>0 and s[0]=="$": - p0=os.getenv(s[1:]) - path=path.replace(path.split("/")[0],p0) - return path - -def read_template(File, delimiter='='): - '''Reads the template file into a python dictionary structure. - Input : string, full path to the template file - Output: dictionary, pysar template content - Example: - tmpl = read_template(KyushuT424F610_640AlosA.template) - tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') - ''' - template_dict = {} - for line in open(File): - line = line.strip() - c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter - if len(c) < 2 or line.startswith('%') or line.startswith('#'): - next #ignore commented lines or those without variables - else: - atrName = c[0] - atrValue = str.replace(c[1],'\n','').split("#")[0].strip() - atrValue = check_variable_name(atrValue) - template_dict[atrName] = atrValue - return template_dict - -def read_data(inFile, dtype, nWidth, nLength): - data = np.fromfile(inFile, dtype, int(nLength)*int(nWidth)).reshape(int(nLength),int(nWidth)) - - return data - -def is_number(s): - try: - int(s) - return True - except ValueError: - return False - -def add_zero(s): - if len(s)==1: - s="000"+s - elif len(s)==2: - s="00"+s - elif len(s)==3: - s="0"+s - return s - - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close() - - -def usage(): - print(''' -****************************************************************************************************** - - Transform SAR coordinates into LAT/LON coordinates based on lookup table - - usage: - - SAR2LATLON.py Range Azimuth LookupTableFile UTMDEMpar - - e.g. SAR2LATLON.py 1500 1000 /Yunmeng/20201230.utm_to_rdc /Yunmeng/20201230.dem.utm.par - - -******************************************************************************************************* - ''') - -def main(argv): - - if len(sys.argv)==5: - Range = sys.argv[1] - Azimuth = sys.argv[2] - LtFile = sys.argv[3] - UTMPAR = sys.argv[4] - else: - usage();sys.exit(1) - - nWidthUTM = UseGamma(UTMPAR, 'read', 'width:') - nLineUTM = UseGamma(UTMPAR, 'read', 'nlines:') - - Corner_LAT = UseGamma(UTMPAR, 'read', 'corner_lat:') - Corner_LON = UseGamma(UTMPAR, 'read', 'corner_lon:') - - Corner_LAT =Corner_LAT.split(' ')[0] - Corner_LON =Corner_LON.split(' ')[0] - - post_Lat = UseGamma(UTMPAR, 'read', 'post_lat:') - post_Lon = UseGamma(UTMPAR, 'read', 'post_lon:') - - post_Lat =post_Lat.split(' ')[0] - post_Lon =post_Lon.split(' ')[0] - data = read_data(LtFile,'>c8',nWidthUTM,nLineUTM) # real: range imaginary: azimuth - - Range_LT = data.real - Azimuth_LT = data.imag - - CPX_input =complex(Range + '+' + Azimuth+'j') - - DD = abs(CPX_input - data) - - LL= abs(DD) - IDX= np.where(LL == LL.min()) - Lat_out = float(Corner_LAT) + IDX[0]*float(post_Lat) - Lon_out = float(Corner_LON) + IDX[1]*float(post_Lon) - - print(' Range: ' + Range + ' ' + 'Azimuth: ' + Azimuth) - print(' Latitude: ' + str(Lat_out[0]) + ' ' + 'Longitude: ' + str(Lon_out[0])) - -############################################################################## -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/SRTM_AUTO_DOWNLOAD.md b/.codex_tmp/pyint_variants/no_rescue/pyint/SRTM_AUTO_DOWNLOAD.md deleted file mode 100644 index 439e3b4..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/SRTM_AUTO_DOWNLOAD.md +++ /dev/null @@ -1,150 +0,0 @@ -# SRTM 自动下载功能说明 - -## 功能概述 - -`makedem.py` 现在支持自动下载SRTM数据,无需手动准备数据文件! - -## 使用方法 - -### 方法1: 自动下载 (推荐) - -**步骤1: 获取OpenTopography API key** (免费) - -访问: https://opentopography.org/myOpenTopo -- 注册免费账户 -- 在 'My Account' 页面请求API key -- API key立即可用,免费配额足够大部分用途 - -**步骤2: 运行命令** - -```bash -# 使用API key自动下载SRTM数据 -python makedem.py -r 116/117/39/40 --dem-source srtm --opentopo-api-key YOUR_API_KEY - -# 或设置环境变量 -export OPENTOPO_API_KEY="your_api_key_here" -python makedem.py -r 116/117/39/40 --dem-source srtm -``` - -### 方法2: 使用预下载的.hgt文件 - -如果已有SRTM .hgt文件: - -```bash -python makedem.py -r 116/117/39/40 --dem-source srtm --srtm-data-dir /path/to/srtm_data -``` - -## SRTM数据规格 - -- **分辨率**: 90米 (SRTM GL3) -- **覆盖**: 全球 (60°S - 60°N) -- **数据源**: OpenTopography API -- **格式**: GeoTIFF -- **费用**: 免费 (需要免费API key) - -## 对比 - -| 方法 | 优点 | 缺点 | -|------|------|------| -| 自动下载 | ✓ 全自动
✓ 无需准备数据
✓ 快速便捷 | ✗ 需要API key
✗ 需要网络 | -| 预下载文件 | ✓ 离线使用
✓ 不依赖API | ✗ 需要手动下载
✗ 需要管理文件 | - -## 示例 - -### 北京地区 (116-117°E, 39-40°N) - -```bash -# 自动下载 -python makedem.py -r 116/117/39/40 --dem-source srtm --opentopo-api-key YOUR_KEY - -# 输出文件: SRTM_116_117_39_40.tif -``` - -### 使用SLC参数文件 - -```bash -# 自动确定区域范围 -python makedem.py -s /path/to/slc.par --dem-source srtm --opentopo-api-key YOUR_KEY -``` - -## 常见问题 - -**Q: 为什么需要API key?** - -A: OpenTopography要求API key来管理数据访问配额。免费API key配额足够大部分科研和教育用途。 - -**Q: 如何获取API key?** - -A: -1. 访问 https://opentopography.org/myOpenTopo -2. 注册账户 (免费) -3. 在账户页面点击 "Request API Key" -4. API key会立即显示,可以直接使用 - -**Q: 免费API key有什么限制?** - -A: -- SRTM GL3: 最大4,050,000 km² per request -- 对于大部分研究区域完全够用 -- 如果需要更大范围,可以分块下载 - -**Q: SRTM vs Copernicus vs NASADEM?** - -A: -- **Copernicus (推荐)**: 30m分辨率,全球覆盖,全自动下载 -- **NASADEM**: 30m分辨率,NASA官方,60°S-60°N覆盖 -- **SRTM**: 90m分辨率,历史数据,适合对比研究 - -**Q: 没有.hgt文件怎么办?** - -A: 直接使用方法1自动下载,无需准备任何数据文件! - -## 故障排除 - -### 错误: HTTP 401 - -**原因**: API key无效或已过期 - -**解决**: -- 检查API key是否正确 -- 确认账户状态正常 -- 如需要,重新生成API key - -### 错误: HTTP 403 - -**原因**: 超出配额或访问限制 - -**解决**: -- 检查请求区域大小 -- 分块下载大区域 -- 确认API key配额 - -### 错误: 文件下载失败 - -**原因**: 网络问题 - -**解决**: -- 检查网络连接 -- 重试命令 -- 使用预下载方法 - -## Python版本兼容性 - -- **方法1 (自动下载)**: 支持所有Python版本 -- **方法2 (使用.hgt文件)**: - - Python >= 3.12: 使用srtm库自动查询 - - Python < 3.12: 使用GDAL手动处理 - -两种方法结果相同,只是处理方式不同。 - -## 相关资源 - -- OpenTopography: https://opentopography.org/ -- API文档: https://portal.opentopography.org/apidocs/ -- 获取API key: https://opentopography.org/myOpenTopo -- SRTM数据介绍: https://www2.jpl.nasa.gov/srtm/ - ---- - -**更新日期**: 2026-03-13 -**功能版本**: v1.0 diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/UTM2SARPIX.py b/.codex_tmp/pyint_variants/no_rescue/pyint/UTM2SARPIX.py deleted file mode 100644 index 46fc6b8..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/UTM2SARPIX.py +++ /dev/null @@ -1,128 +0,0 @@ -#! /usr/bin/env python -#''' -################################################################################### -# # -# Author: Yun-Meng Cao # -# Email : ymcmrs@gmail.com # -# Date : March, 2017 # -# # -# Transform lat and lon to SAR coordinates based on SLCPar and DEM # -# # -################################################################################### -#''' - -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close() - - -def usage(): - print(''' -****************************************************************************************************** - - Transform lat and lon to SAR coordinates based on SLCPar and DEM - - usage: - - UTM2SARPIX.py latitude longitude SLCPar DEM - - e.g. UTM2SARPIX.py 31.1 -108.2 /Yunmeng/2010.slc.par /Yunmeng/2010.dem - -******************************************************************************************************* - ''') - - -def main(argv): - - if len(sys.argv)==5: - LAT = sys.argv[1] - LON = sys.argv[2] - PAR = sys.argv[3] - DEM = sys.argv[4] - else: - usage();sys.exit(1) - - DEMpar = DEM + '.par' - - DateFormat = UseGamma(DEMpar, 'read', 'data_format:') - - nWidth = UseGamma(DEMpar, 'read', 'width:') - nLength = UseGamma(DEMpar, 'read', 'nlines:') - - Corner_LAT = UseGamma(DEMpar, 'read', 'corner_lat:') - Corner_LON = UseGamma(DEMpar, 'read', 'corner_lon:') - - Corner_LAT =Corner_LAT.split(' ')[0] - Corner_LON =Corner_LON.split(' ')[0] - - post_Lat = UseGamma(DEMpar, 'read', 'post_lat:') - post_Lon = UseGamma(DEMpar, 'read', 'post_lon:') - - post_Lat =post_Lat.split(' ')[0] - post_Lon =post_Lon.split(' ')[0] - - if DateFormat =='INTEGER*2': - STR = '>i2' - else: - STR = '>f4' - - TXT = 'SARCOORD' - - DEMdate = np.fromfile(DEM,STR,int(nLength)*int(nWidth)).reshape(int(nLength),int(nWidth)) - - LAT = float(LAT); LON =float(LON) - nWidth=int(nWidth);nLength=int(nLength) - Corner_LAT = float(Corner_LAT); Corner_LON=float(Corner_LON) - post_Lat = float(post_Lat); post_Lon=float(post_Lon) - - XX = int (( LAT - Corner_LAT ) / post_Lat) # latitude width range - YY = int (( LON - Corner_LON ) / post_Lon) # longitude nline azimuth - - - ELEV = DEMdate[XX][YY] - - call_str = 'coord_to_sarpix ' + PAR + ' - - ' + str(LAT) + ' ' + str(LON) + ' ' + str(ELEV) + ' >' +TXT - os.system(call_str) - - call_str = 'cat ' + TXT - os.system(call_str) - - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - - - - - - - - - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/_network.py b/.codex_tmp/pyint_variants/no_rescue/pyint/_network.py deleted file mode 100644 index 4a3bf94..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/_network.py +++ /dev/null @@ -1,483 +0,0 @@ -############################################################ -# Program is part of PyINT # -# Copyright 2019 Yunmeng Cao # -# Contact: ymcmrs@gmail.com # -############################################################ -# This program is modified from MintPy/select_network.py -# Copyright: 2017-2019, Yunjun Zhang -# Contact: yzhang@rsmas.miami.edu - -import os -import sys -import itertools -import h5py -import numpy as np -from scipy import sparse -from matplotlib import (colors, - dates as mdates, - pyplot as plt) -from matplotlib.tri import Triangulation -from mpl_toolkits.axes_grid1 import make_axes_locatable - - - - -################################################################## -def read_pairs_list(date12ListFile, dateList=[]): - """Read Pairs List file like below: - 070311-070426 - 070311-070611 - ... - """ - # Read date12 list file - date12List = sorted(list(np.loadtxt(date12ListFile, dtype=bytes).astype(str))) - - # Get dateList from date12List - if not dateList: - dateList = [] - for date12 in date12List: - dates = date12.split('-') - if not dates[0] in dateList: - dateList.append(dates[0]) - if not dates[1] in dateList: - dateList.append(dates[1]) - dateList.sort() - date6List = yymmdd(dateList) - - # Get pair index - pairs_idx = [] - for date12 in date12List: - dates = date12.split('-') - pair_idx = [date6List.index(dates[0]), date6List.index(dates[1])] - pairs_idx.append(pair_idx) - - return pairs_idx - -def get_date12_list(fname, dropIfgram=False): - """Read Date12 info from input file: Pairs.list or multi-group hdf5 file - Inputs: - fname - string, path/name of input multi-group hdf5 file or text file - dropIfgram - bool, check the "DROP_IFGRAM" attribute or not for multi-group hdf5 file - Output: - date12_list - list of string in YYMMDD-YYMMDD format - Example: - date12List = get_date12_list('ifgramStack.h5') - date12List = get_date12_list('ifgramStack.h5', dropIfgram=True) - date12List = get_date12_list('Pairs.list') - """ - date12_list = [] - ext = os.path.splitext(fname)[1].lower() - if ext == '.h5': - k = readfile.read_attribute(fname)['FILE_TYPE'] - if k == 'ifgramStack': - date12_list = ifgramStack(fname).get_date12_list(dropIfgram=dropIfgram) - else: - return None - else: - txtContent = np.loadtxt(fname, dtype=bytes).astype(str) - if len(txtContent.shape) == 1: - txtContent = txtContent.reshape(-1, 1) - date12_list = [i for i in txtContent[:, 0]] - date12_list = sorted(date12_list) - return date12_list - - -def igram_perp_baseline_list(fname): - """Get perpendicular baseline list from input multi_group hdf5 file""" - print(('read perp baseline info from '+fname)) - k = readfile.read_attribute(fname)['FILE_TYPE'] - h5 = h5py.File(fname, 'r') - epochList = sorted(h5[k].keys()) - p_baseline_list = [] - for epoch in epochList: - p_baseline = (float(h5[k][epoch].attrs['P_BASELINE_BOTTOM_HDR']) + - float(h5[k][epoch].attrs['P_BASELINE_TOP_HDR']))/2 - p_baseline_list.append(p_baseline) - h5.close() - return p_baseline_list - - -def select_pairs_sbas(date_list, date12_format='YYMMDD-YYMMDD'): - """Select All Possible Pairs/Interferograms - Input : date_list - list of date in YYMMDD/YYYYMMDD format - Output: date12_list - list date12 in YYMMDD-YYMMDD format - Reference: - Berardino, P., G. Fornaro, R. Lanari, and E. Sansosti (2002), A new algorithm for surface deformation monitoring - based on small baseline differential SAR interferograms, IEEE TGRS, 40(11), 2375-2383. - """ - date8_list = sorted(yyyymmdd(date_list)) - date6_list = yymmdd(date8_list) - date12_list = list(itertools.combinations(date6_list, 2)) - date12_list = [date12[0]+'-'+date12[1] for date12 in date12_list] - if date12_format == 'YYYYMMDD_YYYYMMDD': - date12_list = yyyymmdd_date12(date12_list) - return date12_list - - -def select_pairs_sequential(date_list, num_connection=2, date12_format='YYMMDD-YYMMDD'): - """Select Pairs in a Sequential way: - For each acquisition, find its num_connection nearest acquisitions in the past time. - Inputs: - date_list : list of date in YYMMDD/YYYYMMDD format - Reference: - Fattahi, H., and F. Amelung (2013), DEM Error Correction in InSAR Time Series, IEEE TGRS, 51(7), 4249-4259. - """ - date8_list = sorted(yyyymmdd(date_list)) - date6_list = yymmdd(date8_list) - date_idx_list = list(range(len(date6_list))) - - # Get pairs index list - date12_idx_list = [] - for date_idx in date_idx_list: - for i in range(num_connection): - if date_idx-i-1 >= 0: - date12_idx_list.append([date_idx-i-1, date_idx]) - date12_idx_list = [sorted(idx) for idx in sorted(date12_idx_list)] - - # Convert index into date12 - date12_list = [date6_list[idx[0]]+'-'+date6_list[idx[1]] - for idx in date12_idx_list] - if date12_format == 'YYYYMMDD_YYYYMMDD': - date12_list = yyyymmdd_date12(date12_list) - return date12_list - - -def select_pairs_hierarchical(date_list, pbase_list, temp_perp_list, date12_format='YYMMDD-YYMMDD'): - """Select Pairs in a hierarchical way using list of temporal and perpendicular baseline thresholds - For each temporal/perpendicular combination, select all possible pairs; and then merge all combination results - together for the final output (Zhao, 2015). - Inputs: - date_list : list of date in YYMMDD/YYYYMMDD format - pbase_list : list of float, perpendicular spatial baseline - temp_perp_list : list of list of 2 floats, for list of temporal/perp baseline, e.g. - [[32.0, 800.0], [48.0, 600.0], [64.0, 200.0]] - Examples: - pairs = select_pairs_hierarchical(date_list, pbase_list, [[32.0, 800.0], [48.0, 600.0], [64.0, 200.0]]) - Reference: - Zhao, W., (2015), Small deformation detected from InSAR time-series and their applications in geophysics, Doctoral - dissertation, Univ. of Miami, Section 6.3. - """ - # Get all date12 - date12_list_all = select_pairs_all(date_list) - - # Loop of Threshold - print('List of temporal and perpendicular spatial baseline thresholds:') - print(temp_perp_list) - date12_list = [] - for temp_perp in temp_perp_list: - tbase_max = temp_perp[0] - pbase_max = temp_perp[1] - date12_list_tmp = threshold_temporal_baseline(date12_list_all, - tbase_max, - keep_seasonal=False) - date12_list_tmp = threshold_perp_baseline(date12_list_tmp, - date_list, - pbase_list, - pbase_max) - date12_list += date12_list_tmp - date12_list = sorted(list(set(date12_list))) - if date12_format == 'YYYYMMDD_YYYYMMDD': - date12_list = yyyymmdd_date12(date12_list) - return date12_list - - -def select_pairs_delaunay(date_list, tbase_list, pbase_list, norm=True, date12_format='YYMMDD-YYMMDD'): - """Select Pairs using Delaunay Triangulation based on temporal/perpendicular baselines - Inputs: - date_list : list of date in YYMMDD/YYYYMMDD format - pbase_list : list of float, perpendicular spatial baseline - norm : normalize temporal baseline to perpendicular baseline - Key points - 1. Define a ratio between perpendicular and temporal baseline axis units (Pepe and Lanari, 2006, TGRS). - 2. Pairs with too large perpendicular / temporal baseline or Doppler centroid difference should be removed - after this, using a threshold, to avoid strong decorrelations (Zebker and Villasenor, 1992, TGRS). - Reference: - Pepe, A., and R. Lanari (2006), On the extension of the minimum cost flow algorithm for phase unwrapping - of multitemporal differential SAR interferograms, IEEE TGRS, 44(9), 2374-2383. - Zebker, H. A., and J. Villasenor (1992), Decorrelation in interferometric radar echoes, IEEE TGRS, 30(5), 950-959. - """ - # Get temporal baseline in days - date6_list = yymmdd(date_list) - date8_list = yyyymmdd(date_list) - #tbase_list = date_list2tbase(date8_list)[0] - - # Normalization (Pepe and Lanari, 2006, TGRS) - if norm: - temp2perp_scale = (max(pbase_list)-min(pbase_list)) / (max(tbase_list)-min(tbase_list)) - tbase_list = [tbase*temp2perp_scale for tbase in tbase_list] - - # Generate Delaunay Triangulation - date12_idx_list = Triangulation(tbase_list, pbase_list).edges.tolist() - date12_idx_list = [sorted(idx) for idx in sorted(date12_idx_list)] - - # Convert index into date12 - date12_list = [date6_list[idx[0]]+'-'+date6_list[idx[1]] - for idx in date12_idx_list] - if date12_format == 'YYYYMMDD_YYYYMMDD': - date12_list = yyyymmdd_date12(date12_list) - return date12_list - - -def select_pairs_mst(date_list, pbase_list, date12_format='YYMMDD-YYMMDD'): - """Select Pairs using Minimum Spanning Tree technique - Connection Cost is calculated using the baseline distance in perp and scaled temporal baseline (Pepe and Lanari, - 2006, TGRS) plane. - Inputs: - date_list : list of date in YYMMDD/YYYYMMDD format - pbase_list : list of float, perpendicular spatial baseline - References: - Pepe, A., and R. Lanari (2006), On the extension of the minimum cost flow algorithm for phase unwrapping - of multitemporal differential SAR interferograms, IEEE TGRS, 44(9), 2374-2383. - Perissin D., Wang T. (2012), Repeat-pass SAR interferometry with partially coherent targets. IEEE TGRS. 271-280 - """ - # Get temporal baseline in days - date6_list = yymmdd(date_list) - date8_list = yyyymmdd(date_list) - tbase_list = date_list2tbase(date8_list)[0] - # Normalization (Pepe and Lanari, 2006, TGRS) - temp2perp_scale = (max(pbase_list)-min(pbase_list)) / (max(tbase_list)-min(tbase_list)) - tbase_list = [tbase*temp2perp_scale for tbase in tbase_list] - - # Get weight matrix - ttMat1, ttMat2 = np.meshgrid(np.array(tbase_list), np.array(tbase_list)) - ppMat1, ppMat2 = np.meshgrid(np.array(pbase_list), np.array(pbase_list)) - ttMat = np.abs(ttMat1 - ttMat2) # temporal distance matrix - ppMat = np.abs(ppMat1 - ppMat2) # spatial distance matrix - - # 2D distance matrix in temp/perp domain - weightMat = np.sqrt(np.square(ttMat) + np.square(ppMat)) - weightMat = sparse.csr_matrix(weightMat) # compress sparse row matrix - - # MST path based on weight matrix - mstMat = sparse.csgraph.minimum_spanning_tree(weightMat) - - # Convert MST index matrix into date12 list - [s_idx_list, m_idx_list] = [date_idx_array.tolist() - for date_idx_array in sparse.find(mstMat)[0:2]] - date12_list = [] - for i in range(len(m_idx_list)): - idx = sorted([m_idx_list[i], s_idx_list[i]]) - date12 = date6_list[idx[0]]+'-'+date6_list[idx[1]] - date12_list.append(date12) - if date12_format == 'YYYYMMDD_YYYYMMDD': - date12_list = yyyymmdd_date12(date12_list) - return date12_list - - -def select_pairs_star(date_list, m_date=None, pbase_list=[], date12_format='YYMMDD-YYMMDD'): - """Select Star-like network/interferograms/pairs, it's a single master network, similar to PS approach. - Usage: - m_date : master date, choose it based on the following cretiria: - 1) near the center in temporal and spatial baseline - 2) prefer winter season than summer season for less temporal decorrelation - Reference: - Ferretti, A., C. Prati, and F. Rocca (2001), Permanent scatterers in SAR interferometry, IEEE TGRS, 39(1), 8-20. - """ - date8_list = sorted(yyyymmdd(date_list)) - date6_list = yymmdd(date8_list) - - # Select master date if not existed - if not m_date: - m_date = select_master_date(date8_list, pbase_list) - print(('auto select master date: '+m_date)) - - # Check input master date - m_date8 = yyyymmdd(m_date) - if m_date8 not in date8_list: - print('Input master date is not existed in date list!') - print(('Input master date: '+str(m_date8))) - print(('Input date list: '+str(date8_list))) - m_date8 = None - - # Generate star/ps network - m_idx = date8_list.index(m_date8) - date12_idx_list = [sorted([m_idx, s_idx]) for s_idx in range(len(date8_list)) - if s_idx is not m_idx] - date12_list = [date6_list[idx[0]]+'-'+date6_list[idx[1]] - for idx in date12_idx_list] - if date12_format == 'YYYYMMDD_YYYYMMDD': - date12_list = yyyymmdd_date12(date12_list) - return date12_list - - -def select_master_date(date_list, pbase_list=[]): - """Select super master date based on input temporal and/or perpendicular baseline info. - Return master date in YYYYMMDD format. - """ - date8_list = yyyymmdd(date_list) - if not pbase_list: - # Choose date in the middle - m_date8 = date8_list[int(len(date8_list)/2)] - else: - # Get temporal baseline list - tbase_list = date_list2tbase(date8_list)[0] - # Normalization (Pepe and Lanari, 2006, TGRS) - temp2perp_scale = (max(pbase_list)-min(pbase_list)) / (max(tbase_list)-min(tbase_list)) - tbase_list = [tbase*temp2perp_scale for tbase in tbase_list] - # Get distance matrix - ttMat1, ttMat2 = np.meshgrid(np.array(tbase_list), - np.array(tbase_list)) - ppMat1, ppMat2 = np.meshgrid(np.array(pbase_list), - np.array(pbase_list)) - ttMat = np.abs(ttMat1 - ttMat2) # temporal distance matrix - ppMat = np.abs(ppMat1 - ppMat2) # spatial distance matrix - # 2D distance matrix in temp/perp domain - disMat = np.sqrt(np.square(ttMat) + np.square(ppMat)) - - # Choose date minimize the total distance of temp/perp baseline - disMean = np.mean(disMat, 0) - m_idx = np.argmin(disMean) - m_date8 = date8_list[m_idx] - return m_date8 - - -def select_master_interferogram(date12_list, date_list, pbase_list, m_date=None): - """Select reference interferogram based on input temp/perp baseline info - If master_date is specified, select its closest slave_date, which is newer than master_date; - otherwise, choose the closest pair among all pairs as master interferogram. - Example: - master_date12 = pnet.select_master_ifgram(date12_list, date_list, pbase_list) - '080211-080326' = pnet.select_master_ifgram(date12_list, date_list, pbase_list, m_date='080211') - """ - pbase_array = np.array(pbase_list, dtype='float64') - # Get temporal baseline - date8_list = yyyymmdd(date_list) - date6_list = yymmdd(date8_list) - tbase_array = np.array(date_list2tbase(date8_list)[0], dtype='float64') - # Normalization (Pepe and Lanari, 2006, TGRS) - temp2perp_scale = (max(pbase_array)-min(pbase_array)) / (max(tbase_array)-min(tbase_array)) - tbase_array *= temp2perp_scale - - # Calculate sqrt of temp/perp baseline for input pairs - idx1 = np.array([date6_list.index(date12.split('-')[0]) for date12 in date12_list]) - idx2 = np.array([date6_list.index(date12.split('-')[1]) for date12 in date12_list]) - base_distance = np.sqrt((tbase_array[idx2] - tbase_array[idx1])**2 + - (pbase_array[idx2] - pbase_array[idx1])**2) - - # Get master interferogram index - if not m_date: - # Choose pair with shortest temp/perp baseline - m_date12_idx = np.argmin(base_distance) - else: - m_date = yymmdd(m_date) - # Choose pair contains m_date with shortest temp/perp baseline - m_date12_idx_array = np.array([date12_list.index(date12) for date12 in date12_list - if m_date+'-' in date12]) - min_base_distance = np.min(base_distance[m_date12_idx_array]) - m_date12_idx = np.where(base_distance == min_base_distance)[0][0] - - m_date12 = date12_list[m_date12_idx] - return m_date12 - - -########################################################## -def datenum2datetime(datenum): - """Convert Matlab datenum into Python datetime. - Parameters: datenum : Date in datenum format, i.e. 731763.5 - Returns: datetime: Date in datetime.datetime format, datetime.datetime(2003, 7, 1, 12, 0) - """ - return dt.fromordinal(int(datenum)) \ - + timedelta(days=datenum % 1) \ - - timedelta(days=366) - - -def decimal_year2datetime(years): - """read date in 2002.40657084 to datetime format - Parameters: years : (list of) float or str for years - Returns: years_dt : (list of) datetime.datetime objects - """ - def decimal_year2datetime1(x): - x = float(x) - year = np.floor(x).astype(int) - yday = np.floor((x - year) * 365.25).astype(int) + 1 - x2 = '{:d}-{:d}'.format(year, yday) - try: - xt = dt(*time.strptime(x2, "%Y-%j")[0:5]) - except: - raise ValueError('wrong format: ',x) - return xt - - if isinstance(years, (float, str)): - years_dt = decimal_year2datetime1(years) - - elif isinstance(years, list): - years_dt = [] - for year in years: - years_dt.append(decimal_year2datetime1(year)) - - else: - raise ValueError('unrecognized input format: {}. Only float/str/list are supported.'.format(type(years))) - return years_dt - - -def yyyymmdd2years(dates): - if isinstance(dates, str): - d = dt(*time.strptime(dates, "%Y%m%d")[0:5]) - yy = float(d.year)+float(d.timetuple().tm_yday-1)/365.25 - elif isinstance(dates, list): - yy = [] - for date in dates: - d = dt(*time.strptime(date, "%Y%m%d")[0:5]) - yy.append(float(d.year)+float(d.timetuple().tm_yday-1)/365.25) - else: - raise ValueError('Unrecognized date format. Only string and list supported.') - return yy - - -def yymmdd2yyyymmdd(date): - if date[0] == '9': - date = '19'+date - else: - date = '20'+date - return date - - -def yyyymmdd(dates): - if isinstance(dates, str): - if len(dates) == 6: - datesOut = yymmdd2yyyymmdd(dates) - else: - datesOut = dates - elif isinstance(dates, list): - datesOut = [] - for date in dates: - if len(date) == 6: - date = yymmdd2yyyymmdd(date) - datesOut.append(date) - else: - # print 'Un-recognized date input!' - return None - return datesOut - - -def yymmdd(dates): - if isinstance(dates, str): - if len(dates) == 8: - datesOut = dates[2:8] - else: - datesOut = dates - elif isinstance(dates, list): - datesOut = [] - for date in dates: - if len(date) == 8: - date = date[2:8] - datesOut.append(date) - else: - # print 'Un-recognized date input!' - return None - return datesOut - - -def yyyymmdd_date12(date12_list): - """Convert date12 into YYYYMMDD_YYYYMMDD format""" - m_dates = yyyymmdd([i.replace('-', '_').split('_')[0] for i in date12_list]) - s_dates = yyyymmdd([i.replace('-', '_').split('_')[1] for i in date12_list]) - date12_list = ['{}-{}'.format(m, s) for m, s in zip(m_dates, s_dates)] - return date12_list - -def yymmdd_date12(date12_list): - """Convert date12 into YYMMDD-YYMMDD format""" - m_dates = yymmdd([i.replace('-', '_').split('_')[0] for i in date12_list]) - s_dates = yymmdd([i.replace('-', '_').split('_')[1] for i in date12_list]) - date12_list = ['{}-{}'.format(m, s) for m, s in zip(m_dates, s_dates)] - return date12_list diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/_orbit_bridge.py b/.codex_tmp/pyint_variants/no_rescue/pyint/_orbit_bridge.py deleted file mode 100644 index 1a70887..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/_orbit_bridge.py +++ /dev/null @@ -1,93 +0,0 @@ -import os -import subprocess -import sys - - -TRUE_VALUES = {"1", "true", "yes", "on"} - - -def _read_bool_env(name, default=False): - value = os.getenv(name) - if value is None: - return default - return str(value).strip().lower() in TRUE_VALUES - - -def apply_precise_orbit(date_text, slc_par_paths, work_dir=None, operation_tag="raw2slc"): - helper_path = str(os.getenv("PYINT_LT1_PRECISE_ORBIT_HELPER") or "").strip() - manifest_json = str(os.getenv("PYINT_LT1_PRECISE_ORBIT_MANIFEST") or "").strip() - enabled = _read_bool_env("PYINT_LT1_PRECISE_ORBIT_ENABLED", False) - strict = _read_bool_env("PYINT_LT1_PRECISE_ORBIT_STRICT", True) - - result = { - "enabled": enabled, - "helper_path": helper_path, - "manifest_json": manifest_json, - "summary_json": "", - "returncode": 0, - "stdout": "", - "stderr": "", - "command": [], - "applied_files": [], - "status": "disabled", - } - if not enabled: - return result - if not helper_path or not os.path.isfile(helper_path): - message = "PYINT_LT1_PRECISE_ORBIT_HELPER is missing or does not exist." - result.update({"status": "missing_helper", "stderr": message}) - if strict: - raise RuntimeError(message) - return result - if not manifest_json: - message = "PYINT_LT1_PRECISE_ORBIT_MANIFEST is empty." - result.update({"status": "missing_manifest", "stderr": message}) - if strict: - raise RuntimeError(message) - return result - - files = [] - for path in slc_par_paths or []: - text = str(path or "").strip() - if text and os.path.isfile(text): - files.append(text) - result["applied_files"] = files - if not files: - result["status"] = "skipped" - return result - - summary_json = os.path.join(str(work_dir or os.getcwd()), "orbit_bridge_summary.json") - command = [ - sys.executable, - helper_path, - "--date", - str(date_text or "").strip(), - "--manifest-json", - manifest_json, - "--summary-json", - summary_json, - "--operation-tag", - str(operation_tag or "raw2slc"), - ] - for path in files: - command.extend(["--slc-par", path]) - result["summary_json"] = summary_json - result["command"] = command - - proc = subprocess.run( - command, - text=True, - capture_output=True, - check=False, - ) - result.update( - { - "returncode": int(proc.returncode), - "stdout": proc.stdout or "", - "stderr": proc.stderr or "", - "status": "applied" if proc.returncode == 0 else "failed", - } - ) - if proc.returncode != 0 and strict: - raise RuntimeError(result["stderr"] or result["stdout"] or "Precise orbit bridge failed.") - return result diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/_utils.py b/.codex_tmp/pyint_variants/no_rescue/pyint/_utils.py deleted file mode 100644 index 58f607d..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/_utils.py +++ /dev/null @@ -1,689 +0,0 @@ -############################################################ -# Program is part of PyINT V2.1 # -# Copyright 2017-2019 Yunmeng Cao # -# Contact: ymcmrs@gmail.com # -############################################################ -#from datetime import datetime -import datetime -import urllib.request -import os -import numpy as np -import random -import h5py -from pathlib import Path -import linecache -import time -import glob - -from tqdm import tqdm -from concurrent.futures import ProcessPoolExecutor, as_completed - -# 无条件清理 OMP_PLACES / OMP_PROC_BIND，避免 libgomp 报错 -for _k in ('OMP_PLACES', 'OMP_PROC_BIND'): - os.environ.pop(_k, None) - -####################### update template ############################# -def update_template(template_file): - - templateDict = {} - - ######### download using SSARA ###### - #templateDict['sensor'] = 'sentinel-1' - #templateDict['track'] = '14' - #templateDict['frame'] = '75' - templateDict['start_time'] = '1989-01-01' - templateDict['end_time'] = '2189-01-01' - - ######### download using ASF API ###### - templateDict['asf_username'] = '' # ASF username (can be read from ~/.netrc) - templateDict['asf_password'] = '' # ASF password (can be read from ~/.netrc) - templateDict['bbox'] = '' # bounding box: min_lon,min_lat,max_lon,max_lat - templateDict['shapefile'] = '' # shapefile path for spatial filtering - templateDict['relative_orbit'] = '' # relative orbit number (optional) - templateDict['frame'] = '' # frame number (optional) - templateDict['flight_direction'] = 'a' # flight direction: a=ascending, d=descending, all - templateDict['acquisition_mode'] = 'IW' # acquisition mode: IW, EW, SM, all - - ####### basic parameters for interferometry ###### - templateDict['start_swath'] = '1' - templateDict['end_swath'] = '3' - - templateDict['start_burst'] = '1' - templateDict['end_burst'] = '20' - - templateDict['dem_lat_ovr'] = '0.5' # get 30m resolution of lookup table, 0.5 to 60m, 2 to 15m - templateDict['dem_lon_ovr'] = '0.5' # get 30m resolution of lookup table - templateDict['dem_num_workers'] = '4' # number of parallel workers for DEM download - templateDict['dem_source'] = 'copernicus' # DEM data source: copernicus or nasadem - - templateDict['Igram_Spsflg'] = '1' # Range spectral filtering - templateDict['Igram_Azfflg'] = '1' # Azimuth common band filtering - - templateDict['rwin4cor'] = '256' # range window length for coregistration - templateDict['azwin4cor'] = '256' # azimuth window length for coregistration - - templateDict['rsample4cor'] = '32' # range samples used for fitting the coregistration parameters - templateDict['azsample4cor'] = '32' # azimuth samples used for fitting the coregistration parameters - - templateDict['thresh4cor'] = '0.15' # 2016 GAMMA or higher version, for 2015 GAMMA or lower version should be SNR - - templateDict['coreCoarse'] = 'both' # initial coregistration method, [options: orbit, ampcor, both] - templateDict['coreMethod'] = 'DEM' # coregistration method [option: DEM, init] DEM means with DEM assistant - - templateDict['Igram_Cor_rwin'] = '5' # used for cc_wave - templateDict['Igram_Cor_awin'] = '5' # used for cc_wave - - templateDict['Igram_Cor_Win'] = '5' # used for adf - templateDict['adf_alpha'] = '0.4' # used for adf - ######## sim phase ################## - templateDict['Igram_Flag_TDM'] = 'N' - templateDict['Simphase_rpos'] = '-' - templateDict['Simphase_azpos'] = '-' - templateDict['Simphase_rwin'] = '256' - templateDict['Simphase_azwin'] = '256' - templateDict['Simphase_thresh'] = '-' - - #### unwrap phase ########### - templateDict['auto_unw'] = '1' # auto reference-point selection - templateDict['make_mask'] = '1' # coherence-mask switch for unwrap - templateDict['init_flag'] = '1' # set reference-point phase to 0.0 - templateDict['r_refer'] = '-' # range reference pixel for manual unwrap - templateDict['a_refer'] = '-' # azimuth reference pixel for manual unwrap - templateDict['mcf_triangular'] = '0' # triangular type of mcf [0: regular; 1: delaunay;] - templateDict['unwrap_patr'] = '4' - templateDict['unwrap_pataz'] = '4' - templateDict['unwrapThreshold'] = '0.1' # minimum coherence used for unwrap - - #### geocode ######### - templateDict['geo_interp'] = '0' # [0: nearest; 1: bicubic spline] - templateDict['geocode_products'] = 'hyp3,licsbas' # 产品类型: hyp3,licsbas,pot (逗号分隔多选) - - ############## interferometry ################ - templateDict['int_flag'] = '1' # 1 means do interferometry - templateDict['diff_flag'] = '1' # differential process, i.e., remove DEM phase - templateDict['unw_flag'] = '1' # unwrap process - templateDict['geo_flag'] = '0' # geocode process - - ############## select network ############# - templateDict['endDate'] = '21000101' - templateDict['startDate'] = '19000101' - templateDict['network_method'] = 'sbas' # sbas, sequential, delaunay, stars - templateDict['conNumb'] = '2' # connect number for sequential - templateDict['max_tb'] = '50000' - templateDict['max_sb'] = '50000' - - ############## time-series ################ - templateDict['download_data'] = '0' # if 1, track, frame, or time informations should be provided - templateDict['down_parallel'] = '1' # multi-processor number used for downloading - - templateDict['raw2slc_all'] = '0' # i.e., download 2 slc - templateDict['raw2slc_all_parallel'] = '1' # multi-processor number used - - templateDict['extract_burst_all'] = '0' # for TOPS SLC only - templateDict['extract_all_parallel'] = '1' # multi-processor number used - - templateDict['coreg_all'] = '1' - templateDict['coreg_all_parallel'] = '1' # multi-processor number used [4 or 8] - - templateDict['select_pairs'] = '1' - - templateDict['diff_all'] = '1' - templateDict['diff_all_parallel'] = '1' # multi-processor number used [4 or 8] - - templateDict['unwrap_all'] = '1' - templateDict['unwrap_all_parallel'] = '1' # multi-processor number used [8 or 10] - - templateDict['atmcor_all'] = '0' # if 1, run atmospheric correction - templateDict['atmcor_all_parallel'] = '1' # multi-processor number used - - templateDict['geocode_all'] = '0' - templateDict['geocode_all_parallel'] = '1' # multi-processor number used - templateDict['gamma2licsbas_all'] = '0' # legacy conversion stage switch - templateDict['gamma2licsbas_all_parallel'] = '1' - templateDict['hyp3format_all'] = '0' # legacy conversion stage switch - templateDict['hyp3format_all_parallel'] = '1' - - ############## GACOS atmospheric correction ############# - templateDict['gacos_correction'] = '0' # if 1, apply GACOS atmospheric correction - templateDict['gacos_all_parallel'] = '1' # multi-processor number used for GACOS correction - templateDict['gacos_dir'] = '' # directory for GACOS ZTD files (default: projectName/GACOS) - templateDict['gacos_email'] = '' # email for GACOS auto-download - templateDict['gacos_email_user'] = '' # IMAP username - templateDict['gacos_email_pass'] = '' # IMAP authorization code - templateDict['gacos_email_host'] = 'imap.163.com' # IMAP server host - templateDict['gacos_email_port'] = '993' # IMAP server port - templateDict['gacos_email_ssl'] = '1' # Use SSL - templateDict['gacos_check_interval'] = '60' # Email check interval (seconds) - - ############## Pixel Offset Tracking (POT) ############# - templateDict['pot_all'] = '0' # if 1, run offset tracking - templateDict['pot_all_parallel'] = '1' # multi-processor number used - templateDict['pot_rstep'] = '50' # range step in SLC pixels - templateDict['pot_azstep'] = '10' # azimuth step in SLC pixels - templateDict['pot_rwin'] = '256' # range window size (Round 1) - templateDict['pot_azwin'] = '128' # azimuth window size (Round 1) - templateDict['pot_rwin2'] = '128' # range window size (Round 2) - templateDict['pot_azwin2'] = '64' # azimuth window size (Round 2) - templateDict['pot_ovr'] = '1' # SLC oversampling factor (1=no deramp needed) - templateDict['pot_snr_thresh'] = '0.01' # SNR threshold for offset_pwr_tracking - templateDict['pot_ccp_thresh'] = '0.05' # cross-correlation threshold for masking - templateDict['pot_roff_min'] = '-50' # min range offset (pixels) for masking - templateDict['pot_roff_max'] = '50' # max range offset (pixels) for masking - templateDict['pot_azoff_min'] = '-20' # min azimuth offset (pixels) for masking - templateDict['pot_azoff_max'] = '20' # max azimuth offset (pixels) for masking - templateDict['pot_drange_thresh'] = '2.0' # deviation threshold range (pixels) - templateDict['pot_dazimuth_thresh'] = '1.0' # deviation threshold azimuth (pixels) - templateDict['pot_median_win'] = '7' # median filter window size - templateDict['pot_median_nmin'] = '15' # min valid samples for median filter - templateDict['pot_two_rounds'] = '1' # 1=two-round estimation, 0=single round - templateDict['pot_geocode'] = '1' # 1=geocode displacement results - templateDict['pot_disp_max'] = '100' # max displacement for BMP display (meters) - - templateDict['load_data'] = '0' # loading data for mintpy processing - - templateDict0 = read_template(template_file, delimiter='=') - for key, value in templateDict0.items(): - templateDict[key] = str(value) - - return templateDict - -####################### time-related function ################ - -def is_number(s): - try: - int(s) - return True - except ValueError: - return False - -def print_process_time(start, end): - hours, rem = divmod(end-start, 3600) - minutes, seconds = divmod(rem, 60) - print("Total process time: "+"{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) - return - -def date_add1d(date0): - format_str = '%Y%m%d' # The format - date_obj = datetime.datetime.strptime(date0,format_str) - date1 = (date_obj + datetime.timedelta(days=1)).date().strftime('%Y%m%d') - return date1 - -def date_minus1d(date0): - format_str = '%Y%m%d' # The format - date_obj = datetime.datetime.strptime(date0,format_str) - date1 = (date_obj - datetime.timedelta(days=1)).date().strftime('%Y%m%d') - return date1 - - -def generate_random_name(sufix): - - nowTime = datetime.datetime.now().strftime("%Y%m%d%H%M%S") #generate the present time - randomNum = random.randint(0,100) - randomNum1 = random.randint(0,100) - Nm = nowTime + str(randomNum)+ str(randomNum1) + sufix - - return Nm - -def yyyymmdd2yyyyddd(date): - dt = datetime.datetime.strptime(date, "%Y%m%d") # get datetime object - day_of_year = (dt - datetime.datetime(dt.year, 1, 1)) # Jan the 1st is day 1 - doy = day_of_year.days + 1 - year = dt.year - - doy = str(doy) - if len(doy) ==1: - doy = '00' + doy - elif len(doy) ==2: - doy ='0'+ doy - year = str(year) - - return year, doy - -def get_txt_lines(txt): - count=0 - myfile=open(txt,"r") - for line in myfile: - count=count+1 - - return count - -def read_txt2list(txt): - A = np.loadtxt(txt,dtype=str) - if np.array(A).size ==1: - A = [A] - elif isinstance(A[0],bytes): - A = A.astype(str) - A = list(A) - return A - -def read_txt2array(txt): - A = np.loadtxt(txt,dtype=str) - if np.array(A).size ==1: - A = [A] - elif isinstance(A[0],bytes): - A = A.astype(str) - #A = list(A) - return A -def get_sardata_swath(start_swath,end_swath): - k_swath = '0' - if (start_swath == '1') and (end_swath == '1'): - k_swath = '1' - elif (start_swath == '2') and (end_swath == '2'): - k_swath = '2' - elif (start_swath == '3') and (end_swath == '3'): - k_swath = '3' - elif (start_swath == '1') and (end_swath == '2'): - k_swath = '4' - elif (start_swath == '2') and (end_swath == '3'): - k_swath = '5' - elif (start_swath == '2') and (end_swath == '3'): - k_swath = '-' - - return k_swath - -def get_filelist_filesize(url0): - ttt = generate_random_name('.txt') - call_str = 'curl -s ' + url0 + ' > ' + ttt - os.system(call_str) - A = read_txt2array(ttt) - - A_size = A[:,4] - A_size = A_size.astype(int) - A_size = A_size/1024/1024 #bytes to Mb - A_name = A[:,8] - if os.path.isfile(ttt): - os.remove(ttt) - - return A_name, A_size - -def yyyymmdd(date0): - if len(date0) ==6: - if float(date0[0:2]) > 90: - date1 = '19' + date0 - else: - date1 = '20' + date0 - elif len(date0) ==8: - date1 = date0 - else: - print('The input date is invalid!!') - date1 = '' - return date1 - -def yymmdd(date0): - if len(date0) ==6: - date1 = date0 - elif len(date0) ==8: - date1 = date0[2:8] - else: - print('The input date is invalid!!') - date1 = '' - return date1 - -def parallel_process(array, function, n_jobs=16, use_kwargs=False): - """ - A parallel version of the map function with a progress bar. - - Args: - array (array-like): An array to iterate over. - function (function): A python function to apply to the elements of array - n_jobs (int, default=16): The number of cores to use - use_kwargs (boolean, default=False): Whether to consider the elements of array as dictionaries of - keyword arguments to function - Returns: - [function(array[0]), function(array[1]), ...] - """ - #We run the first few iterations serially to catch bugs - #If we set n_jobs to 1, just run a list comprehension. This is useful for benchmarking and debugging. - if n_jobs==1: - return [function(**a) if use_kwargs else function(a) for a in tqdm(array[:])] - #Assemble the workers - with ProcessPoolExecutor(max_workers=n_jobs) as pool: - #Pass the elements of array into function - if use_kwargs: - futures = [pool.submit(function, **a) for a in array[:]] - else: - futures = [pool.submit(function, a) for a in array[:]] - kwargs = { - 'total': len(futures), - 'unit': 'it', - 'unit_scale': True, - 'leave': True - } - #Print out the progress as tasks complete - for f in tqdm(as_completed(futures), **kwargs): - pass - out = [] - #Get the results from the futures. - for i, future in tqdm(enumerate(futures)): - try: - out.append(future.result()) - except Exception as e: - out.append(e) - return out - -def sort_unique_list(numb_list): - list_out = sorted(set(numb_list)) - return list_out - - -############################# download data ################################### -def StrNum(S): - S = str(S) - if len(S)==1: - S='0' +S - return S - -def download_s1_orbit(date,save_path,satellite='A'): - - DATE = date - if len(DATE)==6: - DATE = '20' + DATE - ST = satellite - YEAR = int(DATE[0:4]) - MON = int(DATE[4:6]) - DAY = int(DATE[6:8]) - - MON_DAY = [31,28,31,30,31,30,31,31,30,31,30,31] - - if YEAR%4==0: - MON_DAY[1]=29 - - if MON ==1 and DAY ==1: - DAY0 = 31 - MON0 = 12 - YEAR0 =YEAR -1 - elif MON!=1 and DAY ==1: - DAY0 = MON_DAY[MON-2] - MON0 = MON-1 - YEAR0 = YEAR - else: - DAY0 = DAY -1 - MON0 = MON - YEAR0 = YEAR - - MONDAY0 = MON_DAY[MON0-1] - - TT = [1,4,7,10,13,16,19,22,25,28,MONDAY0] - - T0 = [] - for k in range(len(TT)): - T0.append(TT[k]) - TT[k] = TT[k]-DAY0 - - for k in range(len(TT)): - if k == len(TT)-1: - if TT[k]==0: - ff = k-1 - else: - if TT[k]<=0 and TT[k+1]>0: - ff = k - - - DAY1 = T0[ff] - DAY2 = T0[ff+1] - - S1 = StrNum(YEAR0) + '-' + StrNum(MON0) - S2 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY1) - S3 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY2) - S4 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY0) - - #SS = 'https://qc.sentinel1.eo.esa.int/aux_poeorb/?mission=S1' + ST + '&validity_start_time=' + StrNum(YEAR0) + '&validity_start_time=' + S1 + '&validity_start_time=' + S2 + '..' + S3 + '&validity_start_time=' + S4 - SS = 'https://qc.sentinel1.eo.esa.int/aux_poeorb/?validity_start=' + StrNum(YEAR0) + '&validity_start=' + S1 + '&validity_start=' + S2 + '..' + S3 + '&validity_start=' +S4 + '&sentinel1__mission=S1'+ST+ '&sentinel1_mission=S1'+ST+ '&sentinel1_mission=S1'+ST+ '&sentinel1_mission=S1'+ST - - tt ='tt_orb_' + date - tt0 ='tt0_orb_' + date - tt00 ='tt00_orb_' + date - tt000 ='tt000_orb_' + date - SS = "'" + SS + "'" - #print(SS) - call_str = 'curl -s -l ' + SS + ' > ' + tt - os.system(call_str) - - call_str = "grep 'EOF' -C 0 " + tt + " >" + tt0 - os.system(call_str) - - call_str="awk -F'href=' '{print $2}' " + tt0 +' >' + tt00 - os.system(call_str) - - call_str= "awk -F'>' '{print $1}' " + tt00 + '> ' + tt000 - os.system(call_str) - - SS=linecache.getline(tt000, 1) - SS = SS.split('"')[1] - filename = os.path.basename(SS) - if not os.path.isfile(filename): - call_str = 'wget -q --no-check-certificate ' + SS + ' -O ' + save_path + '/' +filename - os.system(call_str) - call_str = "downlaod the precise Orbit of" + date - print(call_str) - - filename = os.path.basename(SS) - os.remove(tt) - os.remove(tt0) - os.remove(tt00) - os.remove(tt000) - - return filename - - -############################# write & read ##################################### -def copy_file(file0,file1): - call_str = 'cp ' + file0 + ' ' + file1 - os.system(call_str) - - return - -def get_project_slcList(projectName): - scratchDir = os.getenv('SCRATCHDIR') - slcDir = scratchDir + '/' + projectName + "/SLC" - #slcDir = scratchDir + '/SLC' - fn = os.listdir(slcDir) - slc_list0 = [os.path.basename(fname) for fname in sorted(fn)] - #slc_list0 = [os.path.basename(fname) for fname in sorted(glob.glob(slcDir + '/*'))] - - slc_list = [] - for k0 in slc_list0: - if is_number(k0): - slc_list.append(k0) - slc_list = sorted(slc_list) - - return slc_list - -def createBlankFile(strFile): - f = open(strFile,'w') - for i in range (10): - f.write('\n') - f.close() - -def read_attr(fname): - # read hdf5 - with h5py.File(fname, 'r') as f: - atr = dict(f.attrs) - - return atr - -def read_hdf5(fname, datasetName=None, box=None): - # read hdf5 - with h5py.File(fname, 'r') as f: - data = f[datasetName][:] - atr = dict(f.attrs) - - return data, atr - -def get_dataNames(FILE): - with h5py.File(FILE, 'r') as f: - dataNames = [] - for k0 in f.keys(): - dataNames.append(k0) - return dataNames - -def check_variable_name(path): - s=path.split("/")[0] - if len(s)>0 and s[0]=="$": - p0=os.getenv(s[1:]) - path=path.replace(path.split("/")[0],p0) - return path - -def read_template(File, delimiter='='): - '''Reads the template file into a python dictionary structure. - Input : string, full path to the template file - Output: dictionary, pysar template content - Example: - tmpl = read_template(KyushuT424F610_640AlosA.template) - tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') - ''' - template_dict = {} - for line in open(File): - line = line.strip() - c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter - if len(c) < 2 or line.startswith('%') or line.startswith('#'): - next #ignore commented lines or those without variables - else: - atrName = c[0] - atrValue = str.replace(c[1],'\n','').split("#")[0].strip() - atrValue = check_variable_name(atrValue) - template_dict[atrName] = atrValue - return template_dict - -def read_gamma_par(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close - - -def write_h5(datasetDict, out_file, metadata=None, ref_file=None, compression=None): - - if os.path.isfile(out_file): - print('delete exsited file: {}'.format(out_file)) - os.remove(out_file) - - print('create HDF5 file: {} with w mode'.format(out_file)) - dt = h5py.special_dtype(vlen=np.dtype('float64')) - - - with h5py.File(out_file, 'w') as f: - for dsName in datasetDict.keys(): - data = datasetDict[dsName] - ds = f.create_dataset(dsName, - data=data, - compression=compression) - - for key, value in metadata.items(): - f.attrs[key] = str(value) - #print(key + ': ' + value) - print('finished writing to {}'.format(out_file)) - - return out_file - -###################################################################### -class progressBar: - """Creates a text-based progress bar. Call the object with - the simple print command to see the progress bar, which looks - something like this: - [=======> 22% ] - You may specify the progress bar's min and max values on init. - - note: - modified from mintPy (https://github.com/insarlab/MintPy/wiki) - Code originally from http://code.activestate.com/recipes/168639/ - - example: - from mintpy.utils import ptime - date12_list = ptime.list_ifgram2date12(ifgram_list) - prog_bar = ptime.progressBar(maxValue=1000, prefix='calculating:') - for i in range(1000): - prog_bar.update(i+1, suffix=date) - prog_bar.update(i+1, suffix=date12_list[i]) - prog_bar.close() - """ - - def __init__(self, maxValue=100, prefix='', minValue=0, totalWidth=70, print_msg=True): - self.prog_bar = "[]" # This holds the progress bar string - self.min = minValue - self.max = maxValue - self.span = maxValue - minValue - self.suffix = '' - self.prefix = prefix - - self.print_msg = print_msg - ## calculate total width based on console width - #rows, columns = os.popen('stty size', 'r').read().split() - #self.width = round(int(columns) * 0.7 / 10) * 10 - self.width = totalWidth - self.reset() - - def reset(self): - self.start_time = time.time() - self.amount = 0 # When amount == max, we are 100% done - self.update_amount(0) # Build progress bar string - - def update_amount(self, newAmount=0, suffix=''): - """ Update the progress bar with the new amount (with min and max - values set at initialization; if it is over or under, it takes the - min or max value as a default. """ - if newAmount < self.min: - newAmount = self.min - if newAmount > self.max: - newAmount = self.max - self.amount = newAmount - - # Figure out the new percent done, round to an integer - diffFromMin = np.float(self.amount - self.min) - percentDone = (diffFromMin / np.float(self.span)) * 100.0 - percentDone = np.int(np.round(percentDone)) - - # Figure out how many hash bars the percentage should be - allFull = self.width - 2 - 18 - numHashes = (percentDone / 100.0) * allFull - numHashes = np.int(np.round(numHashes)) - - # Build a progress bar with an arrow of equal signs; special cases for - # empty and full - if numHashes == 0: - self.prog_bar = '%s[>%s]' % (self.prefix, ' '*(allFull-1)) - elif numHashes == allFull: - self.prog_bar = '%s[%s]' % (self.prefix, '='*allFull) - if suffix: - self.prog_bar += ' %s' % (suffix) - else: - self.prog_bar = '[%s>%s]' % ('='*(numHashes-1), ' '*(allFull-numHashes)) - # figure out where to put the percentage, roughly centered - percentPlace = int(len(self.prog_bar)/2 - len(str(percentDone))) - percentString = ' ' + str(percentDone) + '% ' - # slice the percentage into the bar - self.prog_bar = ''.join([self.prog_bar[0:percentPlace], - percentString, - self.prog_bar[percentPlace+len(percentString):]]) - # prefix and suffix - self.prog_bar = self.prefix + self.prog_bar - if suffix: - self.prog_bar += ' %s' % (suffix) - # time info - elapsed time and estimated remaining time - if percentDone > 0: - elapsed_time = time.time() - self.start_time - self.prog_bar += '%5ds / %5ds' % (int(elapsed_time), - int(elapsed_time * (100./percentDone-1))) - - def update(self, value, every=1, suffix=''): - """ Updates the amount, and writes to stdout. Prints a - carriage return first, so it will overwrite the current - line in stdout.""" - if value % every == 0 or value >= self.max: - self.update_amount(newAmount=value, suffix=suffix) - if self.print_msg: - sys.stdout.write('\r' + self.prog_bar) - sys.stdout.flush() - - def close(self): - """Prints a blank space at the end to ensure proper printing - of future statements.""" - if self.print_msg: - print(' ') diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/_utils_chen.py b/.codex_tmp/pyint_variants/no_rescue/pyint/_utils_chen.py deleted file mode 100644 index dcb69f5..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/_utils_chen.py +++ /dev/null @@ -1,641 +0,0 @@ -############################################################ -# Program is part of PyINT V2.1 # -# Copyright 2017-2019 Yunmeng Cao # -# Contact: ymcmrs@gmail.com # -############################################################ -#from datetime import datetime -import datetime -import urllib.request -import os -import numpy as np -import random -import h5py -from pathlib import Path -import linecache -import time -import glob - -from tqdm import tqdm -from concurrent.futures import ProcessPoolExecutor, as_completed - -####################### update template ############################# -def update_template(template_file): - - templateDict = {} - - ######### download using SSARA ###### - #templateDict['sensor'] = 'sentinel-1' - #templateDict['track'] = '14' - #templateDict['frame'] = '75' - templateDict['start_time'] = '1989-01-01' - templateDict['end_time'] = '2189-01-01' - - ####### basic parameters for interferometry ###### - templateDict['start_swath'] = '1' - templateDict['end_swath'] = '3' - - templateDict['start_burst'] = '1' - templateDict['end_burst'] = '20' - - templateDict['dem_lat_ovr'] = '0.5' # get 30m resolution of lookup table, 0.5 to 60m, 2 to 15m - templateDict['dem_lon_ovr'] = '0.5' # get 30m resolution of lookup table - - templateDict['Igram_Spsflg'] = '1' # Range spectral filtering - templateDict['Igram_Azfflg'] = '1' # Azimuth common band filtering - - templateDict['rwin4cor'] = '256' # range window length for coregistration - templateDict['azwin4cor'] = '256' # azimuth window length for coregistration - - templateDict['rsample4cor'] = '32' # range samples used for fitting the coregistration parameters - templateDict['azsample4cor'] = '32' # azimuth samples used for fitting the coregistration parameters - - templateDict['thresh4cor'] = '0.15' # 2016 GAMMA or higher version, for 2015 GAMMA or lower version should be SNR - - templateDict['coreCoarse'] = 'both' # initial coregistration method, [options: orbit, ampcor, both] - templateDict['coreMethod'] = 'DEM' # coregistration method [option: DEM, init] DEM means with DEM assistant - - templateDict['Igram_Cor_rwin'] = '5' # used for cc_wave - templateDict['Igram_Cor_awin'] = '5' # used for cc_wave - - templateDict['Igram_Cor_Win'] = '5' # used for adf - templateDict['adf_alpha'] = '0.4' # used for adf - ######## sim phase ################## - templateDict['Igram_Flag_TDM'] = 'N' - templateDict['Simphase_rpos'] = '-' - templateDict['Simphase_azpos'] = '-' - templateDict['Simphase_rwin'] = '256' - templateDict['Simphase_azwin'] = '256' - templateDict['Simphase_thresh'] = '-' - - #### unwrap phase ########### - templateDict['mcf_triangular'] = '0' # triangular type of mcf [0: regular; 1: delaunay;] - templateDict['unwrap_patr'] = '1' - templateDict['unwrap_pataz'] = '1' - templateDict['unwrapThreshold'] = '0.1' # minimum coherence used for unwrap - - #### geocode ######### - templateDict['geo_interp'] = '0' # [0: nearest; 1: bicubic spline] - - ############## interferometry ################ - templateDict['int_flag'] = '1' # 1 means do interferometry - templateDict['diff_flag'] = '1' # differential process, i.e., remove DEM phase - templateDict['unw_flag'] = '1' # unwrap process - templateDict['geo_flag'] = '0' # geocode process - - ############## select network ############# - templateDict['endDate'] = '21000101' - templateDict['startDate'] = '19000101' - templateDict['network_method'] = 'sbas' # sbas, sequential, delaunay, stars - templateDict['conNumb'] = '2' # connect number for sequential - templateDict['max_tb'] = '50000' - templateDict['max_sb'] = '50000' - - ############## time-series ################ - templateDict['download_data'] = '0' # if 1, track, frame, or time informations should be provided - templateDict['down_parallel'] = '1' # multi-processor number used for downloading - - templateDict['raw2slc_all'] = '0' # i.e., download 2 slc - templateDict['raw2slc_all_parallel'] = '1' # multi-processor number used - - templateDict['extract_burst_all'] = '0' # for TOPS SLC only - templateDict['extract_all_parallel'] = '1' # multi-processor number used - - templateDict['coreg_all'] = '1' - templateDict['coreg_all_parallel'] = '1' # multi-processor number used [4 or 8] - - templateDict['select_pairs'] = '1' - - templateDict['diff_all'] = '1' - templateDict['diff_all_parallel'] = '1' # multi-processor number used [4 or 8] - - templateDict['unwrap_all'] = '1' - templateDict['unwrap_all_parallel'] = '1' # multi-processor number used [8 or 10] - - templateDict['geocode_all'] = '0' - templateDict['geocode_all_parallel'] = '1' # multi-processor number used - - templateDict['load_data'] = '0' # loading data for mintpy processing - - templateDict0 = read_template(template_file, delimiter='=') - for key, value in templateDict0.items(): - templateDict[key] = str(value) - - return templateDict - -####################### time-related function ################ - -def is_number(s): - try: - int(s) - return True - except ValueError: - return False - -def print_process_time(start, end): - hours, rem = divmod(end-start, 3600) - minutes, seconds = divmod(rem, 60) - print("Total process time: "+"{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) - return - -def date_add1d(date0): - format_str = '%Y%m%d' # The format - date_obj = datetime.datetime.strptime(date0,format_str) - date1 = (date_obj + datetime.timedelta(days=1)).date().strftime('%Y%m%d') - return date1 - -def date_minus1d(date0): - format_str = '%Y%m%d' # The format - date_obj = datetime.datetime.strptime(date0,format_str) - date1 = (date_obj - datetime.timedelta(days=1)).date().strftime('%Y%m%d') - return date1 - - -def generate_random_name(sufix): - - nowTime = datetime.datetime.now().strftime("%Y%m%d%H%M%S") #generate the present time - randomNum = random.randint(0,100) - randomNum1 = random.randint(0,100) - Nm = nowTime + str(randomNum)+ str(randomNum1) + sufix - - return Nm - -def yyyymmdd2yyyyddd(date): - dt = datetime.datetime.strptime(date, "%Y%m%d") # get datetime object - day_of_year = (dt - datetime.datetime(dt.year, 1, 1)) # Jan the 1st is day 1 - doy = day_of_year.days + 1 - year = dt.year - - doy = str(doy) - if len(doy) ==1: - doy = '00' + doy - elif len(doy) ==2: - doy ='0'+ doy - year = str(year) - - return year, doy - -def get_txt_lines(txt): - count=0 - myfile=open(txt,"r") - for line in myfile: - count=count+1 - - return count - -def read_txt2list(txt): - A = np.loadtxt(txt,dtype=np.str) - if np.array(A).size ==1: - A = [A] - elif isinstance(A[0],bytes): - A = A.astype(str) - A = list(A) - return A - -def read_txt2array(txt): - A = np.loadtxt(txt,dtype=np.str) - if np.array(A).size ==1: - A = [A] - elif isinstance(A[0],bytes): - A = A.astype(str) - #A = list(A) - return A -def get_sardata_swath(start_swath,end_swath): - k_swath = '0' - if (start_swath == '1') and (end_swath == '1'): - k_swath = '1' - elif (start_swath == '2') and (end_swath == '2'): - k_swath = '2' - elif (start_swath == '3') and (end_swath == '3'): - k_swath = '3' - elif (start_swath == '1') and (end_swath == '2'): - k_swath = '4' - elif (start_swath == '2') and (end_swath == '3'): - k_swath = '5' - elif (start_swath == '1') and (end_swath == '3'): - k_swath = '0' - - return k_swath - -def get_filelist_filesize(url0): - ttt = generate_random_name('.txt') - call_str = 'curl -s ' + url0 + ' > ' + ttt - os.system(call_str) - A = read_txt2array(ttt) - - A_size = A[:,4] - A_size = A_size.astype(int) - A_size = A_size/1024/1024 #bytes to Mb - A_name = A[:,8] - if os.path.isfile(ttt): - os.remove(ttt) - - return A_name, A_size - -def yyyymmdd(date0): - if len(date0) ==6: - if float(date0[0:2]) > 90: - date1 = '19' + date0 - else: - date1 = '20' + date0 - elif len(date0) ==8: - date1 = date0 - else: - print('The input date is invalid!!') - date1 = '' - return date1 - -def yymmdd(date0): - if len(date0) ==6: - date1 = date0 - elif len(date0) ==8: - date1 = date0[2:8] - else: - print('The input date is invalid!!') - date1 = '' - return date1 - -def parallel_process(array, function, n_jobs=16, use_kwargs=False): - """ - A parallel version of the map function with a progress bar. - - Args: - array (array-like): An array to iterate over. - function (function): A python function to apply to the elements of array - n_jobs (int, default=16): The number of cores to use - use_kwargs (boolean, default=False): Whether to consider the elements of array as dictionaries of - keyword arguments to function - Returns: - [function(array[0]), function(array[1]), ...] - """ - #We run the first few iterations serially to catch bugs - #If we set n_jobs to 1, just run a list comprehension. This is useful for benchmarking and debugging. - if n_jobs==1: - return [function(**a) if use_kwargs else function(a) for a in tqdm(array[:])] - #Assemble the workers - with ProcessPoolExecutor(max_workers=n_jobs) as pool: - #Pass the elements of array into function - if use_kwargs: - futures = [pool.submit(function, **a) for a in array[:]] - else: - futures = [pool.submit(function, a) for a in array[:]] - kwargs = { - 'total': len(futures), - 'unit': 'it', - 'unit_scale': True, - 'leave': True - } - #Print out the progress as tasks complete - for f in tqdm(as_completed(futures), **kwargs): - pass - out = [] - #Get the results from the futures. - for i, future in tqdm(enumerate(futures)): - try: - out.append(future.result()) - except Exception as e: - out.append(e) - return out - -def sort_unique_list(numb_list): - list_out = sorted(set(numb_list)) - return list_out - - -############################# download data ################################### -def StrNum(S): - S = str(S) - if len(S)==1: - S='0' +S - return S - -def download_s1_orbit(date,save_path,satellite='A'): - - DATE = date - if len(DATE)==6: - DATE = '20' + DATE - ST = satellite - YEAR = int(DATE[0:4]) - MON = int(DATE[4:6]) - DAY = int(DATE[6:8]) - - MON_DAY = [31,28,31,30,31,30,31,31,30,31,30,31] - - if YEAR%4==0: - MON_DAY[1]=29 - - if MON ==1 and DAY ==1: - DAY0 = 31 - MON0 = 12 - YEAR0 =YEAR -1 - elif MON!=1 and DAY ==1: - DAY0 = MON_DAY[MON-2] - MON0 = MON-1 - YEAR0 = YEAR - else: - DAY0 = DAY -1 - MON0 = MON - YEAR0 = YEAR - - MONDAY0 = MON_DAY[MON0-1] - - TT = [1,4,7,10,13,16,19,22,25,28,MONDAY0] - - T0 = [] - for k in range(len(TT)): - T0.append(TT[k]) - TT[k] = TT[k]-DAY0 - - for k in range(len(TT)): - if k == len(TT)-1: - if TT[k]==0: - ff = k-1 - else: - if TT[k]<=0 and TT[k+1]>0: - ff = k - - - DAY1 = T0[ff] - DAY2 = T0[ff+1] - - S1 = StrNum(YEAR0) + '-' + StrNum(MON0) - S2 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY1) - S3 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY2) - S4 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY0) - - #SS = 'https://qc.sentinel1.eo.esa.int/aux_poeorb/?mission=S1' + ST + '&validity_start_time=' + StrNum(YEAR0) + '&validity_start_time=' + S1 + '&validity_start_time=' + S2 + '..' + S3 + '&validity_start_time=' + S4 - SS = 'https://qc.sentinel1.eo.esa.int/aux_poeorb/?validity_start=' + StrNum(YEAR0) + '&validity_start=' + S1 + '&validity_start=' + S2 + '..' + S3 + '&validity_start=' +S4 + '&sentinel1__mission=S1'+ST+ '&sentinel1_mission=S1'+ST+ '&sentinel1_mission=S1'+ST+ '&sentinel1_mission=S1'+ST - - tt ='tt_orb_' + date - tt0 ='tt0_orb_' + date - tt00 ='tt00_orb_' + date - tt000 ='tt000_orb_' + date - SS = "'" + SS + "'" - #print(SS) - call_str = 'curl -s -l ' + SS + ' > ' + tt - os.system(call_str) - - call_str = "grep 'EOF' -C 0 " + tt + " >" + tt0 - os.system(call_str) - - call_str="awk -F'href=' '{print $2}' " + tt0 +' >' + tt00 - os.system(call_str) - - call_str= "awk -F'>' '{print $1}' " + tt00 + '> ' + tt000 - os.system(call_str) - - SS=linecache.getline(tt000, 1) - SS = SS.split('"')[-1] - - filename = os.path.basename(SS) -# filename = os.path.basename(SS) -# SS1 = save_path + '/' + filename -# if os.path.isfile(SS1): -# if os.path.getsize(SS1) > 4409000: -# call_str = "downlaod the precise Orbit of" + date -# print(call_str) -# else: -# call_str = 'wget -q --no-check-certificate ' + SS + ' -O ' + save_path + '/' +filename -# os.system(call_str) -# call_str = "downlaod the precise Orbit of" + date -# print(call_str) -# else: -# call_str = 'wget -q --no-check-certificate ' + SS + ' -O ' + save_path + '/' +filename -# os.system(call_str) -# call_str = "downlaod the precise Orbit of" + date -# print(call_str) -# if not os.path.isfile(filename): - call_str = 'wget -q --no-check-certificate ' + SS + ' -O ' + save_path + '/' +filename - os.system(call_str) - call_str = "downlaod the precise Orbit of" + date - print(call_str) - - filename = os.path.basename(SS) - os.remove(tt) - os.remove(tt0) - os.remove(tt00) - os.remove(tt000) - - return filename - - -############################# write & read ##################################### -def copy_file(file0,file1): - call_str = 'cp ' + file0 + ' ' + file1 - os.system(call_str) - - return - -def get_project_slcList(projectName): - scratchDir = os.getenv('SCRATCHDIR') - slcDir = scratchDir + '/' + projectName + "/SLC" - #slcDir = scratchDir + '/SLC' - fn = os.listdir(slcDir) - slc_list0 = [os.path.basename(fname) for fname in sorted(fn)] - #slc_list0 = [os.path.basename(fname) for fname in sorted(glob.glob(slcDir + '/*'))] - - slc_list = [] - for k0 in slc_list0: - if is_number(k0): - slc_list.append(k0) - slc_list = sorted(slc_list) - - return slc_list - -def createBlankFile(strFile): - f = open(strFile,'w') - for i in range (10): - f.write('\n') - f.close() - -def read_attr(fname): - # read hdf5 - with h5py.File(fname, 'r') as f: - atr = dict(f.attrs) - - return atr - -def read_hdf5(fname, datasetName=None, box=None): - # read hdf5 - with h5py.File(fname, 'r') as f: - data = f[datasetName][:] - atr = dict(f.attrs) - - return data, atr - -def get_dataNames(FILE): - with h5py.File(FILE, 'r') as f: - dataNames = [] - for k0 in f.keys(): - dataNames.append(k0) - return dataNames - -def check_variable_name(path): - s=path.split("/")[0] - if len(s)>0 and s[0]=="$": - p0=os.getenv(s[1:]) - path=path.replace(path.split("/")[0],p0) - return path - -def read_template(File, delimiter='='): - '''Reads the template file into a python dictionary structure. - Input : string, full path to the template file - Output: dictionary, pysar template content - Example: - tmpl = read_template(KyushuT424F610_640AlosA.template) - tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') - ''' - template_dict = {} - for line in open(File): - line = line.strip() - c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter - if len(c) < 2 or line.startswith('%') or line.startswith('#'): - next #ignore commented lines or those without variables - else: - atrName = c[0] - atrValue = str.replace(c[1],'\n','').split("#")[0].strip() - atrValue = check_variable_name(atrValue) - template_dict[atrName] = atrValue - return template_dict - -def read_gamma_par(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close - - -def write_h5(datasetDict, out_file, metadata=None, ref_file=None, compression=None): - - if os.path.isfile(out_file): - print('delete exsited file: {}'.format(out_file)) - os.remove(out_file) - - print('create HDF5 file: {} with w mode'.format(out_file)) - dt = h5py.special_dtype(vlen=np.dtype('float64')) - - - with h5py.File(out_file, 'w') as f: - for dsName in datasetDict.keys(): - data = datasetDict[dsName] - ds = f.create_dataset(dsName, - data=data, - compression=compression) - - for key, value in metadata.items(): - f.attrs[key] = str(value) - #print(key + ': ' + value) - print('finished writing to {}'.format(out_file)) - - return out_file - -###################################################################### -class progressBar: - """Creates a text-based progress bar. Call the object with - the simple print command to see the progress bar, which looks - something like this: - [=======> 22% ] - You may specify the progress bar's min and max values on init. - - note: - modified from mintPy (https://github.com/insarlab/MintPy/wiki) - Code originally from http://code.activestate.com/recipes/168639/ - - example: - from mintpy.utils import ptime - date12_list = ptime.list_ifgram2date12(ifgram_list) - prog_bar = ptime.progressBar(maxValue=1000, prefix='calculating:') - for i in range(1000): - prog_bar.update(i+1, suffix=date) - prog_bar.update(i+1, suffix=date12_list[i]) - prog_bar.close() - """ - - def __init__(self, maxValue=100, prefix='', minValue=0, totalWidth=70, print_msg=True): - self.prog_bar = "[]" # This holds the progress bar string - self.min = minValue - self.max = maxValue - self.span = maxValue - minValue - self.suffix = '' - self.prefix = prefix - - self.print_msg = print_msg - ## calculate total width based on console width - #rows, columns = os.popen('stty size', 'r').read().split() - #self.width = round(int(columns) * 0.7 / 10) * 10 - self.width = totalWidth - self.reset() - - def reset(self): - self.start_time = time.time() - self.amount = 0 # When amount == max, we are 100% done - self.update_amount(0) # Build progress bar string - - def update_amount(self, newAmount=0, suffix=''): - """ Update the progress bar with the new amount (with min and max - values set at initialization; if it is over or under, it takes the - min or max value as a default. """ - if newAmount < self.min: - newAmount = self.min - if newAmount > self.max: - newAmount = self.max - self.amount = newAmount - - # Figure out the new percent done, round to an integer - diffFromMin = np.float(self.amount - self.min) - percentDone = (diffFromMin / np.float(self.span)) * 100.0 - percentDone = np.int(np.round(percentDone)) - - # Figure out how many hash bars the percentage should be - allFull = self.width - 2 - 18 - numHashes = (percentDone / 100.0) * allFull - numHashes = np.int(np.round(numHashes)) - - # Build a progress bar with an arrow of equal signs; special cases for - # empty and full - if numHashes == 0: - self.prog_bar = '%s[>%s]' % (self.prefix, ' '*(allFull-1)) - elif numHashes == allFull: - self.prog_bar = '%s[%s]' % (self.prefix, '='*allFull) - if suffix: - self.prog_bar += ' %s' % (suffix) - else: - self.prog_bar = '[%s>%s]' % ('='*(numHashes-1), ' '*(allFull-numHashes)) - # figure out where to put the percentage, roughly centered - percentPlace = int(len(self.prog_bar)/2 - len(str(percentDone))) - percentString = ' ' + str(percentDone) + '% ' - # slice the percentage into the bar - self.prog_bar = ''.join([self.prog_bar[0:percentPlace], - percentString, - self.prog_bar[percentPlace+len(percentString):]]) - # prefix and suffix - self.prog_bar = self.prefix + self.prog_bar - if suffix: - self.prog_bar += ' %s' % (suffix) - # time info - elapsed time and estimated remaining time - if percentDone > 0: - elapsed_time = time.time() - self.start_time - self.prog_bar += '%5ds / %5ds' % (int(elapsed_time), - int(elapsed_time * (100./percentDone-1))) - - def update(self, value, every=1, suffix=''): - """ Updates the amount, and writes to stdout. Prints a - carriage return first, so it will overwrite the current - line in stdout.""" - if value % every == 0 or value >= self.max: - self.update_amount(newAmount=value, suffix=suffix) - if self.print_msg: - sys.stdout.write('\r' + self.prog_bar) - sys.stdout.flush() - - def close(self): - """Prints a blank space at the end to ensure proper printing - of future statements.""" - if self.print_msg: - print(' ') diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/_utils_old.py b/.codex_tmp/pyint_variants/no_rescue/pyint/_utils_old.py deleted file mode 100644 index 2b2cfa5..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/_utils_old.py +++ /dev/null @@ -1,623 +0,0 @@ -############################################################ -# Program is part of PyINT V2.1 # -# Copyright 2017-2019 Yunmeng Cao # -# Contact: ymcmrs@gmail.com # -############################################################ -#from datetime import datetime -import datetime -import urllib.request -import os -import numpy as np -import random -import h5py -from pathlib import Path -import linecache -import time -import glob - -from tqdm import tqdm -from concurrent.futures import ProcessPoolExecutor, as_completed - -####################### update template ############################# -def update_template(template_file): - - templateDict = {} - - ######### download using SSARA ###### - #templateDict['sensor'] = 'sentinel-1' - #templateDict['track'] = '14' - #templateDict['frame'] = '75' - templateDict['start_time'] = '1989-01-01' - templateDict['end_time'] = '2189-01-01' - - ####### basic parameters for interferometry ###### - templateDict['start_swath'] = '1' - templateDict['end_swath'] = '3' - - templateDict['start_burst'] = '1' - templateDict['end_burst'] = '20' - - templateDict['dem_lat_ovr'] = '0.5' # get 30m resolution of lookup table, 0.5 to 60m, 2 to 15m - templateDict['dem_lon_ovr'] = '0.5' # get 30m resolution of lookup table - - templateDict['Igram_Spsflg'] = '1' # Range spectral filtering - templateDict['Igram_Azfflg'] = '1' # Azimuth common band filtering - - templateDict['rwin4cor'] = '256' # range window length for coregistration - templateDict['azwin4cor'] = '256' # azimuth window length for coregistration - - templateDict['rsample4cor'] = '32' # range samples used for fitting the coregistration parameters - templateDict['azsample4cor'] = '32' # azimuth samples used for fitting the coregistration parameters - - templateDict['thresh4cor'] = '0.15' # 2016 GAMMA or higher version, for 2015 GAMMA or lower version should be SNR - - templateDict['coreCoarse'] = 'both' # initial coregistration method, [options: orbit, ampcor, both] - templateDict['coreMethod'] = 'DEM' # coregistration method [option: DEM, init] DEM means with DEM assistant - - templateDict['Igram_Cor_rwin'] = '5' # used for cc_wave - templateDict['Igram_Cor_awin'] = '5' # used for cc_wave - - templateDict['Igram_Cor_Win'] = '5' # used for adf - templateDict['adf_alpha'] = '0.4' # used for adf - ######## sim phase ################## - templateDict['Igram_Flag_TDM'] = 'N' - templateDict['Simphase_rpos'] = '-' - templateDict['Simphase_azpos'] = '-' - templateDict['Simphase_rwin'] = '256' - templateDict['Simphase_azwin'] = '256' - templateDict['Simphase_thresh'] = '-' - - #### unwrap phase ########### - templateDict['mcf_triangular'] = '0' # triangular type of mcf [0: regular; 1: delaunay;] - templateDict['unwrap_patr'] = '1' - templateDict['unwrap_pataz'] = '1' - templateDict['unwrapThreshold'] = '0.1' # minimum coherence used for unwrap - - #### geocode ######### - templateDict['geo_interp'] = '0' # [0: nearest; 1: bicubic spline] - - ############## interferometry ################ - templateDict['int_flag'] = '1' # 1 means do interferometry - templateDict['diff_flag'] = '1' # differential process, i.e., remove DEM phase - templateDict['unw_flag'] = '1' # unwrap process - templateDict['geo_flag'] = '0' # geocode process - - ############## select network ############# - templateDict['endDate'] = '21000101' - templateDict['startDate'] = '19000101' - templateDict['network_method'] = 'sbas' # sbas, sequential, delaunay, stars - templateDict['conNumb'] = '2' # connect number for sequential - templateDict['max_tb'] = '50000' - templateDict['max_sb'] = '50000' - - ############## time-series ################ - templateDict['download_data'] = '0' # if 1, track, frame, or time informations should be provided - templateDict['down_parallel'] = '1' # multi-processor number used for downloading - - templateDict['raw2slc_all'] = '0' # i.e., download 2 slc - templateDict['raw2slc_all_parallel'] = '1' # multi-processor number used - - templateDict['extract_burst_all'] = '0' # for TOPS SLC only - templateDict['extract_all_parallel'] = '1' # multi-processor number used - - templateDict['coreg_all'] = '1' - templateDict['coreg_all_parallel'] = '1' # multi-processor number used [4 or 8] - - templateDict['select_pairs'] = '1' - - templateDict['diff_all'] = '1' - templateDict['diff_all_parallel'] = '1' # multi-processor number used [4 or 8] - - templateDict['unwrap_all'] = '1' - templateDict['unwrap_all_parallel'] = '1' # multi-processor number used [8 or 10] - - templateDict['geocode_all'] = '0' - templateDict['geocode_all_parallel'] = '1' # multi-processor number used - - templateDict['load_data'] = '0' # loading data for mintpy processing - - templateDict0 = read_template(template_file, delimiter='=') - for key, value in templateDict0.items(): - templateDict[key] = str(value) - - return templateDict - -####################### time-related function ################ - -def is_number(s): - try: - int(s) - return True - except ValueError: - return False - -def print_process_time(start, end): - hours, rem = divmod(end-start, 3600) - minutes, seconds = divmod(rem, 60) - print("Total process time: "+"{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) - return - -def date_add1d(date0): - format_str = '%Y%m%d' # The format - date_obj = datetime.datetime.strptime(date0,format_str) - date1 = (date_obj + datetime.timedelta(days=1)).date().strftime('%Y%m%d') - return date1 - -def date_minus1d(date0): - format_str = '%Y%m%d' # The format - date_obj = datetime.datetime.strptime(date0,format_str) - date1 = (date_obj - datetime.timedelta(days=1)).date().strftime('%Y%m%d') - return date1 - - -def generate_random_name(sufix): - - nowTime = datetime.datetime.now().strftime("%Y%m%d%H%M%S") #generate the present time - randomNum = random.randint(0,100) - randomNum1 = random.randint(0,100) - Nm = nowTime + str(randomNum)+ str(randomNum1) + sufix - - return Nm - -def yyyymmdd2yyyyddd(date): - dt = datetime.datetime.strptime(date, "%Y%m%d") # get datetime object - day_of_year = (dt - datetime.datetime(dt.year, 1, 1)) # Jan the 1st is day 1 - doy = day_of_year.days + 1 - year = dt.year - - doy = str(doy) - if len(doy) ==1: - doy = '00' + doy - elif len(doy) ==2: - doy ='0'+ doy - year = str(year) - - return year, doy - -def get_txt_lines(txt): - count=0 - myfile=open(txt,"r") - for line in myfile: - count=count+1 - - return count - -def read_txt2list(txt): - A = np.loadtxt(txt,dtype=np.str) - if np.array(A).size ==1: - A = [A] - elif isinstance(A[0],bytes): - A = A.astype(str) - A = list(A) - return A - -def read_txt2array(txt): - A = np.loadtxt(txt,dtype=np.str) - if np.array(A).size ==1: - A = [A] - elif isinstance(A[0],bytes): - A = A.astype(str) - #A = list(A) - return A -def get_sardata_swath(start_swath,end_swath): - k_swath = '0' - if (start_swath == '1') and (end_swath == '1'): - k_swath = '1' - elif (start_swath == '2') and (end_swath == '2'): - k_swath = '2' - elif (start_swath == '3') and (end_swath == '3'): - k_swath = '3' - elif (start_swath == '1') and (end_swath == '2'): - k_swath = '4' - elif (start_swath == '2') and (end_swath == '3'): - k_swath = '5' - elif (start_swath == '2') and (end_swath == '3'): - k_swath = '-' - - return k_swath - -def get_filelist_filesize(url0): - ttt = generate_random_name('.txt') - call_str = 'curl -s ' + url0 + ' > ' + ttt - os.system(call_str) - A = read_txt2array(ttt) - - A_size = A[:,4] - A_size = A_size.astype(int) - A_size = A_size/1024/1024 #bytes to Mb - A_name = A[:,8] - if os.path.isfile(ttt): - os.remove(ttt) - - return A_name, A_size - -def yyyymmdd(date0): - if len(date0) ==6: - if float(date0[0:2]) > 90: - date1 = '19' + date0 - else: - date1 = '20' + date0 - elif len(date0) ==8: - date1 = date0 - else: - print('The input date is invalid!!') - date1 = '' - return date1 - -def yymmdd(date0): - if len(date0) ==6: - date1 = date0 - elif len(date0) ==8: - date1 = date0[2:8] - else: - print('The input date is invalid!!') - date1 = '' - return date1 - -def parallel_process(array, function, n_jobs=16, use_kwargs=False): - """ - A parallel version of the map function with a progress bar. - - Args: - array (array-like): An array to iterate over. - function (function): A python function to apply to the elements of array - n_jobs (int, default=16): The number of cores to use - use_kwargs (boolean, default=False): Whether to consider the elements of array as dictionaries of - keyword arguments to function - Returns: - [function(array[0]), function(array[1]), ...] - """ - #We run the first few iterations serially to catch bugs - #If we set n_jobs to 1, just run a list comprehension. This is useful for benchmarking and debugging. - if n_jobs==1: - return [function(**a) if use_kwargs else function(a) for a in tqdm(array[:])] - #Assemble the workers - with ProcessPoolExecutor(max_workers=n_jobs) as pool: - #Pass the elements of array into function - if use_kwargs: - futures = [pool.submit(function, **a) for a in array[:]] - else: - futures = [pool.submit(function, a) for a in array[:]] - kwargs = { - 'total': len(futures), - 'unit': 'it', - 'unit_scale': True, - 'leave': True - } - #Print out the progress as tasks complete - for f in tqdm(as_completed(futures), **kwargs): - pass - out = [] - #Get the results from the futures. - for i, future in tqdm(enumerate(futures)): - try: - out.append(future.result()) - except Exception as e: - out.append(e) - return out - -def sort_unique_list(numb_list): - list_out = sorted(set(numb_list)) - return list_out - - -############################# download data ################################### -def StrNum(S): - S = str(S) - if len(S)==1: - S='0' +S - return S - -def download_s1_orbit(date,save_path,satellite='A'): - - DATE = date - if len(DATE)==6: - DATE = '20' + DATE - ST = satellite - YEAR = int(DATE[0:4]) - MON = int(DATE[4:6]) - DAY = int(DATE[6:8]) - - MON_DAY = [31,28,31,30,31,30,31,31,30,31,30,31] - - if YEAR%4==0: - MON_DAY[1]=29 - - if MON ==1 and DAY ==1: - DAY0 = 31 - MON0 = 12 - YEAR0 =YEAR -1 - elif MON!=1 and DAY ==1: - DAY0 = MON_DAY[MON-2] - MON0 = MON-1 - YEAR0 = YEAR - else: - DAY0 = DAY -1 - MON0 = MON - YEAR0 = YEAR - - MONDAY0 = MON_DAY[MON0-1] - - TT = [1,4,7,10,13,16,19,22,25,28,MONDAY0] - - T0 = [] - for k in range(len(TT)): - T0.append(TT[k]) - TT[k] = TT[k]-DAY0 - - for k in range(len(TT)): - if k == len(TT)-1: - if TT[k]==0: - ff = k-1 - else: - if TT[k]<=0 and TT[k+1]>0: - ff = k - - - DAY1 = T0[ff] - DAY2 = T0[ff+1] - - S1 = StrNum(YEAR0) + '-' + StrNum(MON0) - S2 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY1) - S3 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY2) - S4 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY0) - - #SS = 'https://qc.sentinel1.eo.esa.int/aux_poeorb/?mission=S1' + ST + '&validity_start_time=' + StrNum(YEAR0) + '&validity_start_time=' + S1 + '&validity_start_time=' + S2 + '..' + S3 + '&validity_start_time=' + S4 - SS = 'https://qc.sentinel1.eo.esa.int/aux_poeorb/?validity_start=' + StrNum(YEAR0) + '&validity_start=' + S1 + '&validity_start=' + S2 + '..' + S3 + '&validity_start=' +S4 + '&sentinel1__mission=S1'+ST+ '&sentinel1_mission=S1'+ST+ '&sentinel1_mission=S1'+ST+ '&sentinel1_mission=S1'+ST - - tt ='tt_orb_' + date - tt0 ='tt0_orb_' + date - tt00 ='tt00_orb_' + date - tt000 ='tt000_orb_' + date - SS = "'" + SS + "'" - #print(SS) - call_str = 'curl -s -l ' + SS + ' > ' + tt - os.system(call_str) - - call_str = "grep 'EOF' -C 0 " + tt + " >" + tt0 - os.system(call_str) - - call_str="awk -F'href=' '{print $2}' " + tt0 +' >' + tt00 - os.system(call_str) - - call_str= "awk -F'>' '{print $1}' " + tt00 + '> ' + tt000 - os.system(call_str) - - SS=linecache.getline(tt000, 1) - SS = SS.split('"')[-1] - filename = os.path.basename(SS) - call_str = 'wget -q --no-check-certificate ' + SS + ' -O ' + save_path + '/' +filename - os.system(call_str) - call_str = "downlaod the precise Orbit of" + date - print(call_str) - - filename = os.path.basename(SS) - os.remove(tt) - os.remove(tt0) - os.remove(tt00) - os.remove(tt000) - - return filename - - -############################# write & read ##################################### -def copy_file(file0,file1): - call_str = 'cp ' + file0 + ' ' + file1 - os.system(call_str) - - return - -def get_project_slcList(projectName): - scratchDir = os.getenv('SCRATCHDIR') - slcDir = scratchDir + '/' + projectName + "/SLC" - #slcDir = scratchDir + '/SLC' - fn = os.listdir(slcDir) - slc_list0 = [os.path.basename(fname) for fname in sorted(fn)] - #slc_list0 = [os.path.basename(fname) for fname in sorted(glob.glob(slcDir + '/*'))] - - slc_list = [] - for k0 in slc_list0: - if is_number(k0): - slc_list.append(k0) - slc_list = sorted(slc_list) - - return slc_list - -def createBlankFile(strFile): - f = open(strFile,'w') - for i in range (10): - f.write('\n') - f.close() - -def read_attr(fname): - # read hdf5 - with h5py.File(fname, 'r') as f: - atr = dict(f.attrs) - - return atr - -def read_hdf5(fname, datasetName=None, box=None): - # read hdf5 - with h5py.File(fname, 'r') as f: - data = f[datasetName][:] - atr = dict(f.attrs) - - return data, atr - -def get_dataNames(FILE): - with h5py.File(FILE, 'r') as f: - dataNames = [] - for k0 in f.keys(): - dataNames.append(k0) - return dataNames - -def check_variable_name(path): - s=path.split("/")[0] - if len(s)>0 and s[0]=="$": - p0=os.getenv(s[1:]) - path=path.replace(path.split("/")[0],p0) - return path - -def read_template(File, delimiter='='): - '''Reads the template file into a python dictionary structure. - Input : string, full path to the template file - Output: dictionary, pysar template content - Example: - tmpl = read_template(KyushuT424F610_640AlosA.template) - tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') - ''' - template_dict = {} - for line in open(File): - line = line.strip() - c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter - if len(c) < 2 or line.startswith('%') or line.startswith('#'): - next #ignore commented lines or those without variables - else: - atrName = c[0] - atrValue = str.replace(c[1],'\n','').split("#")[0].strip() - atrValue = check_variable_name(atrValue) - template_dict[atrName] = atrValue - return template_dict - -def read_gamma_par(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close - - -def write_h5(datasetDict, out_file, metadata=None, ref_file=None, compression=None): - - if os.path.isfile(out_file): - print('delete exsited file: {}'.format(out_file)) - os.remove(out_file) - - print('create HDF5 file: {} with w mode'.format(out_file)) - dt = h5py.special_dtype(vlen=np.dtype('float64')) - - - with h5py.File(out_file, 'w') as f: - for dsName in datasetDict.keys(): - data = datasetDict[dsName] - ds = f.create_dataset(dsName, - data=data, - compression=compression) - - for key, value in metadata.items(): - f.attrs[key] = str(value) - #print(key + ': ' + value) - print('finished writing to {}'.format(out_file)) - - return out_file - -###################################################################### -class progressBar: - """Creates a text-based progress bar. Call the object with - the simple print command to see the progress bar, which looks - something like this: - [=======> 22% ] - You may specify the progress bar's min and max values on init. - - note: - modified from mintPy (https://github.com/insarlab/MintPy/wiki) - Code originally from http://code.activestate.com/recipes/168639/ - - example: - from mintpy.utils import ptime - date12_list = ptime.list_ifgram2date12(ifgram_list) - prog_bar = ptime.progressBar(maxValue=1000, prefix='calculating:') - for i in range(1000): - prog_bar.update(i+1, suffix=date) - prog_bar.update(i+1, suffix=date12_list[i]) - prog_bar.close() - """ - - def __init__(self, maxValue=100, prefix='', minValue=0, totalWidth=70, print_msg=True): - self.prog_bar = "[]" # This holds the progress bar string - self.min = minValue - self.max = maxValue - self.span = maxValue - minValue - self.suffix = '' - self.prefix = prefix - - self.print_msg = print_msg - ## calculate total width based on console width - #rows, columns = os.popen('stty size', 'r').read().split() - #self.width = round(int(columns) * 0.7 / 10) * 10 - self.width = totalWidth - self.reset() - - def reset(self): - self.start_time = time.time() - self.amount = 0 # When amount == max, we are 100% done - self.update_amount(0) # Build progress bar string - - def update_amount(self, newAmount=0, suffix=''): - """ Update the progress bar with the new amount (with min and max - values set at initialization; if it is over or under, it takes the - min or max value as a default. """ - if newAmount < self.min: - newAmount = self.min - if newAmount > self.max: - newAmount = self.max - self.amount = newAmount - - # Figure out the new percent done, round to an integer - diffFromMin = np.float(self.amount - self.min) - percentDone = (diffFromMin / np.float(self.span)) * 100.0 - percentDone = np.int(np.round(percentDone)) - - # Figure out how many hash bars the percentage should be - allFull = self.width - 2 - 18 - numHashes = (percentDone / 100.0) * allFull - numHashes = np.int(np.round(numHashes)) - - # Build a progress bar with an arrow of equal signs; special cases for - # empty and full - if numHashes == 0: - self.prog_bar = '%s[>%s]' % (self.prefix, ' '*(allFull-1)) - elif numHashes == allFull: - self.prog_bar = '%s[%s]' % (self.prefix, '='*allFull) - if suffix: - self.prog_bar += ' %s' % (suffix) - else: - self.prog_bar = '[%s>%s]' % ('='*(numHashes-1), ' '*(allFull-numHashes)) - # figure out where to put the percentage, roughly centered - percentPlace = int(len(self.prog_bar)/2 - len(str(percentDone))) - percentString = ' ' + str(percentDone) + '% ' - # slice the percentage into the bar - self.prog_bar = ''.join([self.prog_bar[0:percentPlace], - percentString, - self.prog_bar[percentPlace+len(percentString):]]) - # prefix and suffix - self.prog_bar = self.prefix + self.prog_bar - if suffix: - self.prog_bar += ' %s' % (suffix) - # time info - elapsed time and estimated remaining time - if percentDone > 0: - elapsed_time = time.time() - self.start_time - self.prog_bar += '%5ds / %5ds' % (int(elapsed_time), - int(elapsed_time * (100./percentDone-1))) - - def update(self, value, every=1, suffix=''): - """ Updates the amount, and writes to stdout. Prints a - carriage return first, so it will overwrite the current - line in stdout.""" - if value % every == 0 or value >= self.max: - self.update_amount(newAmount=value, suffix=suffix) - if self.print_msg: - sys.stdout.write('\r' + self.prog_bar) - sys.stdout.flush() - - def close(self): - """Prints a blank space at the end to ensure proper printing - of future statements.""" - if self.print_msg: - print(' ') diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/applygacos.py b/.codex_tmp/pyint_variants/no_rescue/pyint/applygacos.py deleted file mode 100644 index e760869..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/applygacos.py +++ /dev/null @@ -1,305 +0,0 @@ -#!/usr/bin/env python3 -import os,sys,struct,math -import numpy as np -import argparse -from pyint import _utils as ut -import subprocess - - -#if len(sys.argv) != 5: -# print("Usage:") -# print("./applygacos.py inpfilename ztd1filename ztd2filename elevfilename") -# print(" inpfilenmae : input interferogram, a inpfilename.rsc is needed!") -# print(" ztd1filename: input GACOS ztd day1 filename, a ztd1filename.rsc is needed!") -# print(" ztd2filename: input GACOS ztd day2 filename, a ztd2filename.rsc is needed!") -# print(" elevfilename: elevation angle file, must be the same size with interferogram") -# print(" elevation=90-incidence ") -# print(" can be generated by look_vector in gamma") -# print("outputfile will be saved as inpfilename.gacos") -# print("!!Note, python3 is needed!!") -# exit() - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Coregister SM mode SLC to a reference SLC image using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName', help='Name of project.') - parser.add_argument('mdate', help='date of the slave SLC image') - parser.add_argument('sdate', help='date of the slave SLC image') - parser.add_argument('ztd1', help='input GACOS ztd day1 filename, a ztd2filename.rsc is needed!') - parser.add_argument('ztd2', help='input GACOS ztd day2 filename, a ztd2filename.rsc is needed!' ) - inps = parser.parse_args() - return inps - -INTRODUCTION = ''' -------------------------------------------------------------------- - create_files for psokinv software running file by generated by Gamma -''' - -EXAMPLE = """Usage: - - coreg_gamma.py projectName - - coreg_gamma.py PacayaT163TsxHhA -------------------------------------------------------------------- -""" - - - - -#filename=sys.argv[1] -#ztd1filename=sys.argv[2] -#ztd2filename=sys.argv[3] -#elevfilename=sys.argv[4] -#print("start processing "+ filename) - -class HEADER: - width = 0 - length = 0 - xfirst = 0.0 - yfirst = 0.0 - xstep = 0.0 - ystep = 0.0 - - - - - -def read_header(filename): - if not os.path.isfile(filename): - print(filename+" file not exit") - - header = HEADER() - with open(filename) as f: - for line in f: - data=line.split() - if data[0] == "WIDTH": - header.width=int(data[1]) - if data[0] == "FILE_LENGTH": - header.length=int(data[1]) - if data[0] == "X_FIRST": - header.xfirst=float(data[1]) - if data[0] == "Y_FIRST": - header.yfirst=float(data[1]) - if data[0] == "X_STEP": - header.xstep=float(data[1]) - if data[0] == "Y_STEP": - header.ystep=float(data[1]) - - return header - - -def cut_image2(filename,headername,yfirst_new,length_new,xfirst_new,width_new): - header=read_header(headername) - - #print(yfirst_new, header.yfirst) - #print(length_new, header.length) - #print(xfirst_new, header.xfirst) - #print(width_new, header.width) - - with open(filename, 'rb') as f: - data0 = np.fromfile(f, dtype=np.float32) - data = np.reshape(data0, (header.length, header.width)) - - - out=np.zeros((length_new,width_new),dtype=np.float32) - #out=np.zeros(width_new*length_new,dtype=np.float32) - for i in range(header.length): - lat=header.yfirst+header.ystep*i - row=int(round((lat-yfirst_new)/header.ystep)) - if(row<0 or row>=length_new): - continue - for j in range(header.width): - lon=header.xfirst+header.xstep*j - col=int(round((lon-xfirst_new)/header.xstep)) - if(col<0 or col>=width_new): - continue - #out[width_new*row+col]=data[header.width*i+j] - out[row,col]=data[i,j] - - out=np.where(out==0,np.nan,out) - #print(np.nanstd(out)) - out.tofile(filename+".cut") - f = open(filename+".cut.rsc", 'w') - f.write("WIDTH "+str(width_new)+"\n") - f.write("FILE_LENGTH "+str(length_new)+"\n") - f.write("X_FIRST "+ str(xfirst_new)+"\n") - f.write("Y_FIRST "+ str(yfirst_new)+"\n") - f.write("X_STEP "+ str(header.xstep)+"\n") - f.write("Y_STEP "+ str(header.ystep)+"\n") - f.close() - - - - -def make_correction(phsfilename,ztd1filename,ztd2filename,elevfilename): - header=read_header(phsfilename+".rsc") - if not os.path.isfile(ztd1filename+".cut"): - cut_image2(ztd1filename,ztd1filename+".rsc",header.yfirst,header.length,header.xfirst,header.width) - if not os.path.isfile(ztd2filename+".cut"): - cut_image2(ztd2filename,ztd2filename+".rsc",header.yfirst,header.length,header.xfirst,header.width) - - with open(phsfilename, 'rb') as f: - data = np.fromfile(f, dtype=np.float32) - phase = np.reshape(data, [header.length, header.width]) - - with open(ztd1filename+".cut", 'rb') as f: - data = np.fromfile(f, dtype=np.float32) - ztd1 = np.reshape(data, [header.length, header.width]) - - with open(ztd2filename+".cut", 'rb') as f: - data = np.fromfile(f, dtype=np.float32) - ztd2 = np.reshape(data, [header.length, header.width]) - - with open(elevfilename, 'rb') as f: - data = np.fromfile(f, dtype=np.float32) - elev = np.reshape(data, [header.length, header.width]) - - #os.remove(ztd1filename+".cut") - #os.remove(ztd2filename+".cut") - dztd=ztd2-ztd1 - dztd=dztd/0.0044138251819503 - dztd=dztd/np.sin(elev) - index=np.where(phase==0) - phase[index]=np.nan - phasemean=np.nanmean(phase) - print("before "+str(np.nanstd(phase))) - phase=phase-phasemean - #phase.tofile(phsfilename+".raw") - - - phase=phase-dztd - phase[index]=np.nan - phasemean=np.nanmean(phase) - print("after "+str(np.nanstd(phase))) - phase=phase-phasemean - phase[index]=0 - phase.tofile(phsfilename+".gacos") - -def main(argv): - inps = cmdLineParse() - projectName = inps.projectName - Sdate = inps.sdate - Mdate = inps.mdate - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - Smdate = templateDict['masterDate'] - - demDir = scratchDir + '/' + projectName + '/DEM' - slcDir = scratchDir + '/' + projectName + "/SLC" - rslcDir = scratchDir + '/' + projectName + "/RSLC" - ifgramDir = scratchDir + '/' + projectName + "/ifgrams" - workDir = ifgramDir + '/' + Mdate + '-' + Sdate - MslcDir = rslcDir + '/' + Mdate - SslcDir = rslcDir + '/' + Sdate - offpar = SslcDir + '/' + Smdate + '_' + Sdate + '.off' - slcpar = MslcDir + '/' + Smdate + '.rslc.par' - dempar = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem.par' - dem = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem' - unwpar = demDir + '/' + Mdate + '_' + rlks + 'rlks.utm.dem.par' - unw = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.diff_filt.unw" - - os.chdir(workDir) - call_str = "look_vector " + slcpar+ " " + offpar + " " + dempar + " " + dem + ' lv_theta lv_phi' - os.system(call_str) - call_str = "grep 'corner_lat:' " + dempar + " | awk '{print $2}' " - North=subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'corner_lon:' " + dempar + " | awk '{print $2}' " - West=subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'post_lat:' " + dempar + " | awk '{print $2}' " - posty=subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'post_lon:' " + dempar + " | awk '{print $2}' " - postx=subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'width:' " + dempar + " | awk '{print $2}' " - width=subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'nlines:' " + dempar + " | awk '{print $2}' " - length=subprocess.getstatusoutput(call_str)[1] - South=str(round(float(North)+(float(length)-1)*float(posty),7)) - East=str(round(float(West)+(float(width)-1)*float(postx),7)) - - call_str = "swap_bytes lv_theta lv_theta.phase_swap 4 >dinsar.log " - os.system(call_str) - call_str = "gmt xyz2grd lv_theta.phase_swap -Glv_theta.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -N0" - os.system(call_str) - call_str = "gmt grdmath 90 lv_theta.grd 3.1415926 DIV 180 MUL SUB = lv_theta_final.grd" - os.system(call_str) - call_str = "gmt grdmath 90 lv_theta_final.grd SUB = lv_elev.grd" - os.system(call_str) - call_str = "gmt grdsample lv_elev.grd -Glv_elev_3c.grd -I3c" - os.system(call_str) - call_str = "gmt grd2xyz lv_elev_3c.grd -ZTLf -N0 > " + Mdate + '-' + Sdate + '.gacos.elev' - os.system(call_str) - - - call_str = "swap_bytes " + unw+ " unw.phase_swap 4 >dinsar.log " - os.system(call_str) - call_str = "gmt xyz2grd unw.phase_swap -Gunw_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx +" -ZTLf -N0" - os.system(call_str) - call_str = "gmt grdsample unw_f.grd -Gunw.grd -I3c" - os.system(call_str) - call_str = "gmt grd2xyz unw.grd -ZTLf > " + Mdate + '-' + Sdate + '.gacos.unw' - os.system(call_str) - call_str = "gmt grdinfo unw.grd -C | awk '{print $10}' " - Width=subprocess.getstatusoutput(call_str)[1] - call_str = "gmt grdinfo unw.grd -C | awk '{print $11}' " - line=subprocess.getstatusoutput(call_str)[1] - call_str = "gmt grdinfo unw.grd -C | awk '{print $2}' " - West=subprocess.getstatusoutput(call_str)[1] - call_str = "gmt grdinfo unw.grd -C | awk '{print $5}' " - North=subprocess.getstatusoutput(call_str)[1] - ymax=str(int(round(float(line) ,7))) - xmax=str(int(round(float(Width) ,7))) - output=workDir + '/' + Mdate + '-' + Sdate + '.gacos.unw.rsc' - if os.path.exists(output) is True: - os.remove(output) - - fopen=open(output,'a+') - fopen.write('WIDTH ' + Width + '\n') - fopen.write('FILE_LENGTH ' + line + '\n') - fopen.write('XMIN 1' + '\n') - fopen.write('XMAX ' + xmax + '\n') - fopen.write('YMIN 1' + '\n') - fopen.write('YMAX ' + ymax + '\n') - fopen.write('X_FIRST ' + West + '\n') - fopen.write('Y_FIRST ' + North + '\n') - fopen.write('X_STEP 8.33333333E-04' + '\n') - fopen.write('Y_STEP -8.33333333E-04' + '\n') - fopen.write('X_UNIT degrees' + '\n') - fopen.write('Y_UNIT degrees' + '\n') - fopen.write('Z_OFFSET 0' + '\n') - fopen.write('Z_SCALE 1' + '\n') - fopen.write('PROJECTION LATLON' + '\n') - fopen.write('DATUM WGS84' + '\n') - print("convert gamma products to GACOS is done!") - - print("start make gacos correction!") - filename=workDir + '/' + Mdate + '-' + Sdate + '.gacos.unw' - ztd1=inps.ztd1 - ztd2=inps.ztd2 - ztd1filename=workDir + '/' + ztd1 - ztd2filename=workDir + '/' + ztd2 - elevfilename=workDir + '/' + Mdate + '-' + Sdate + '.gacos.elev' - make_correction(filename,ztd1filename,ztd2filename,elevfilename) - - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - -#make_correction(filename,ztd1filename,ztd2filename,elevfilename) - - - - -#exit() - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/applygacos1.py b/.codex_tmp/pyint_variants/no_rescue/pyint/applygacos1.py deleted file mode 100644 index e8e48e0..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/applygacos1.py +++ /dev/null @@ -1,310 +0,0 @@ -#!/usr/bin/env python3 -import os,sys,struct,math -import numpy as np -import argparse -from pyint import _utils as ut -import subprocess - - -#if len(sys.argv) != 5: -# print("Usage:") -# print("./applygacos.py inpfilename ztd1filename ztd2filename elevfilename") -# print(" inpfilenmae : input interferogram, a inpfilename.rsc is needed!") -# print(" ztd1filename: input GACOS ztd day1 filename, a ztd1filename.rsc is needed!") -# print(" ztd2filename: input GACOS ztd day2 filename, a ztd2filename.rsc is needed!") -# print(" elevfilename: elevation angle file, must be the same size with interferogram") -# print(" elevation=90-incidence ") -# print(" can be generated by look_vector in gamma") -# print("outputfile will be saved as inpfilename.gacos") -# print("!!Note, python3 is needed!!") -# exit() - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Coregister SM mode SLC to a reference SLC image using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName', help='Name of project.') - parser.add_argument('mdate', help='date of the slave SLC image') - parser.add_argument('sdate', help='date of the slave SLC image') - parser.add_argument('ztd1', help='input GACOS ztd day1 filename, a ztd2filename.rsc is needed!') - parser.add_argument('ztd2', help='input GACOS ztd day2 filename, a ztd2filename.rsc is needed!' ) - inps = parser.parse_args() - return inps - -INTRODUCTION = ''' -------------------------------------------------------------------- - create_files for psokinv software running file by generated by Gamma -''' - -EXAMPLE = """Usage: - - coreg_gamma.py projectName - - coreg_gamma.py PacayaT163TsxHhA -------------------------------------------------------------------- -""" - - - - -#filename=sys.argv[1] -#ztd1filename=sys.argv[2] -#ztd2filename=sys.argv[3] -#elevfilename=sys.argv[4] -#print("start processing "+ filename) - -class HEADER: - width = 0 - length = 0 - xfirst = 0.0 - yfirst = 0.0 - xstep = 0.0 - ystep = 0.0 - - - - - -def read_header(filename): - if not os.path.isfile(filename): - print(filename+" file not exit") - - header = HEADER() - with open(filename) as f: - for line in f: - data=line.split() - if data[0] == "WIDTH": - header.width=int(data[1]) - if data[0] == "FILE_LENGTH": - header.length=int(data[1]) - if data[0] == "X_FIRST": - header.xfirst=float(data[1]) - if data[0] == "Y_FIRST": - header.yfirst=float(data[1]) - if data[0] == "X_STEP": - header.xstep=float(data[1]) - if data[0] == "Y_STEP": - header.ystep=float(data[1]) - - return header - - -def cut_image2(filename,headername,yfirst_new,length_new,xfirst_new,width_new): - header=read_header(headername) - - #print(yfirst_new, header.yfirst) - #print(length_new, header.length) - #print(xfirst_new, header.xfirst) - #print(width_new, header.width) - - with open(filename, 'rb') as f: - data0 = np.fromfile(f, dtype=np.float32) - data = np.reshape(data0, (header.length, header.width)) - - - out=np.zeros((length_new,width_new),dtype=np.float32) - #out=np.zeros(width_new*length_new,dtype=np.float32) - for i in range(header.length): - lat=header.yfirst+header.ystep*i - row=int(round((lat-yfirst_new)/header.ystep)) - if(row<0 or row>=length_new): - continue - for j in range(header.width): - lon=header.xfirst+header.xstep*j - col=int(round((lon-xfirst_new)/header.xstep)) - if(col<0 or col>=width_new): - continue - #out[width_new*row+col]=data[header.width*i+j] - out[row,col]=data[i,j] - - out=np.where(out==0,np.nan,out) - #print(np.nanstd(out)) - out.tofile(filename+".cut") - f = open(filename+".cut.rsc", 'w') - f.write("WIDTH "+str(width_new)+"\n") - f.write("FILE_LENGTH "+str(length_new)+"\n") - f.write("X_FIRST "+ str(xfirst_new)+"\n") - f.write("Y_FIRST "+ str(yfirst_new)+"\n") - f.write("X_STEP "+ str(header.xstep)+"\n") - f.write("Y_STEP "+ str(header.ystep)+"\n") - f.write("X_UNIT degree"+"\n") - f.write("Y_UNIT degree"+"\n") - f.write("Z_OFFSET 0"+"\n") - f.write("Z_SCALE 1"+"\n") - f.write("PROJECTION LATLON"+"\n") - f.write("DATUM WGS84"+"\n") - f.close() - -def make_correction(phsfilename,ztd1filename,ztd2filename,elevfilename): - header=read_header(phsfilename+".rsc") - #if not os.path.isfile(ztd1filename+".cut"): - cut_image2(ztd1filename,ztd1filename+".rsc",header.yfirst,header.length,header.xfirst,header.width) - #if not os.path.isfile(ztd2filename+".cut"): - cut_image2(ztd2filename,ztd2filename+".rsc",header.yfirst,header.length,header.xfirst,header.width) - - with open(phsfilename, 'rb') as f: - data = np.fromfile(f, dtype=np.float32) - phase = np.reshape(data, [header.length, header.width]) - - with open(ztd1filename+".cut", 'rb') as f: - data = np.fromfile(f, dtype=np.float32) - ztd1 = np.reshape(data, [header.length, header.width]) - - with open(ztd2filename+".cut", 'rb') as f: - data = np.fromfile(f, dtype=np.float32) - ztd2 = np.reshape(data, [header.length, header.width]) - - with open(elevfilename, 'rb') as f: - data = np.fromfile(f, dtype=np.float32) - elev = np.reshape(data, [header.length, header.width]) - - #os.remove(ztd1filename+".cut") - #os.remove(ztd2filename+".cut") - dztd=ztd2-ztd1 - #dztd=dztd/0.0044138251819503 - dztd=dztd/np.sin(elev) - index=np.where(phase==0) - phase[index]=np.nan - phasemean=np.nanmean(phase) - print("before "+str(np.nanstd(phase))) - phase=phase-phasemean - #phase.tofile(phsfilename+".raw") - - - phase=phase-dztd - phase[index]=np.nan - phasemean=np.nanmean(phase) - print("after "+str(np.nanstd(phase))) - phase=phase-phasemean - phase[index]=0 - phase.tofile(phsfilename+".gacos") - -def main(argv): - inps = cmdLineParse() - projectName = inps.projectName - Sdate = inps.sdate - Mdate = inps.mdate - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - Smdate = templateDict['masterDate'] - - demDir = scratchDir + '/' + projectName + '/DEM' - slcDir = scratchDir + '/' + projectName + "/SLC" - rslcDir = scratchDir + '/' + projectName + "/RSLC" - ifgramDir = scratchDir + '/' + projectName + "/ifgrams" - workDir = ifgramDir + '/' + Mdate + '-' + Sdate - MslcDir = rslcDir + '/' + Mdate - SslcDir = rslcDir + '/' + Sdate - offpar = SslcDir + '/' + Smdate + '_' + Sdate + '.off' - slcpar = MslcDir + '/' + Smdate + '.rslc.par' - dempar = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem.par' - dem = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem' - unwpar = demDir + '/' + Mdate + '_' + rlks + 'rlks.utm.dem.par' - unw = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.diff_filt.unw" - - os.chdir(workDir) - call_str = "look_vector " + slcpar+ " " + offpar + " " + dempar + " " + dem + ' lv_theta lv_phi' - os.system(call_str) - call_str = "grep 'corner_lat:' " + dempar + " | awk '{print $2}' " - North=subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'corner_lon:' " + dempar + " | awk '{print $2}' " - West=subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'post_lat:' " + dempar + " | awk '{print $2}' " - posty=subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'post_lon:' " + dempar + " | awk '{print $2}' " - postx=subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'width:' " + dempar + " | awk '{print $2}' " - width=subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'nlines:' " + dempar + " | awk '{print $2}' " - length=subprocess.getstatusoutput(call_str)[1] - South=str(round(float(North)+(float(length)-1)*float(posty),7)) - East=str(round(float(West)+(float(width)-1)*float(postx),7)) - - call_str = "swap_bytes lv_theta lv_theta.phase_swap 4 >dinsar.log " - os.system(call_str) - call_str = "xyz2grd lv_theta.phase_swap -Glv_theta.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -N0" - os.system(call_str) - call_str = "grdmath 90 lv_theta.grd 3.1415926 DIV 180 MUL SUB = lv_theta_final.grd" - os.system(call_str) - call_str = "grdmath 90 lv_theta_final.grd SUB = lv_elev.grd" - os.system(call_str) - call_str = "grdsample lv_elev.grd -Glv_elev_3c.grd -I3c" - # call_str = "grdsample lv_theta_final.grd -Glv_elev_3c.grd -I3c" - os.system(call_str) - call_str = "grd2xyz lv_elev_3c.grd -ZTLf -N0 > " + Mdate + '-' + Sdate + '.gacos.elev' - os.system(call_str) - - - call_str = "swap_bytes " + unw+ " unw.phase_swap 4 >dinsar.log " - os.system(call_str) - call_str = "xyz2grd unw.phase_swap -Gunw_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx +" -ZTLf -N0" - os.system(call_str) - call_str = "grdsample unw_f.grd -Gunw.grd -I3c" - os.system(call_str) - call_str = "grd2xyz unw.grd -ZTLf -N0 > " + Mdate + '-' + Sdate + '.gacos.unw' - os.system(call_str) - call_str = "grdinfo unw.grd -C | awk '{print $10}' " - Width=subprocess.getstatusoutput(call_str)[1] - call_str = "grdinfo unw.grd -C | awk '{print $11}' " - line=subprocess.getstatusoutput(call_str)[1] - call_str = "grdinfo unw.grd -C | awk '{print $2}' " - West=subprocess.getstatusoutput(call_str)[1] - call_str = "grdinfo unw.grd -C | awk '{print $5}' " - North=subprocess.getstatusoutput(call_str)[1] - ymax=str(int(round(float(line) ,7))) - xmax=str(int(round(float(Width) ,7))) - output=workDir + '/' + Mdate + '-' + Sdate + '.gacos.unw.rsc' - if os.path.exists(output) is True: - os.remove(output) - - fopen=open(output,'a+') - fopen.write('WIDTH ' + Width + '\n') - fopen.write('FILE_LENGTH ' + line + '\n') - fopen.write('XMIN 1' + '\n') - fopen.write('XMAX ' + xmax + '\n') - fopen.write('YMIN 1' + '\n') - fopen.write('YMAX ' + ymax + '\n') - fopen.write('X_FIRST ' + West + '\n') - fopen.write('Y_FIRST ' + North + '\n') - fopen.write('X_STEP 8.33333333E-04' + '\n') - fopen.write('Y_STEP -8.33333333E-04' + '\n') - fopen.write('X_UNIT degrees' + '\n') - fopen.write('Y_UNIT degrees' + '\n') - fopen.write('Z_OFFSET 0' + '\n') - fopen.write('Z_SCALE 1' + '\n') - fopen.write('PROJECTION LATLON' + '\n') - fopen.write('DATUM WGS84' + '\n') - fopen.close() - print("convert gamma products to GACOS is done!") - - print("start make gacos correction!") - filename=workDir + '/' + Mdate + '-' + Sdate + '.gacos.unw' - ztd1=inps.ztd1 - ztd2=inps.ztd2 - ztd1filename=workDir + '/' + ztd1 - ztd2filename=workDir + '/' + ztd2 - elevfilename=workDir + '/' + Mdate + '-' + Sdate + '.gacos.elev' - make_correction(filename,ztd1filename,ztd2filename,elevfilename) - - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - -#make_correction(filename,ztd1filename,ztd2filename,elevfilename) - - - - -#exit() - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/atm_correction_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/atm_correction_gamma.py deleted file mode 100644 index 169bad0..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/atm_correction_gamma.py +++ /dev/null @@ -1,105 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -from PIL import Image -from pylab import * -import argparse - -from pyint import _utils as ut - - -INTRODUCTION = ''' -------------------------------------------------------------------- - Unwrap differential interferogram using GAMMA. - [Only support mcf, not implement branch_cut yet] - -''' - -EXAMPLE = ''' - Usage: - unwrap_gamma.py projectName Mdate Sdate - unwrap_gamma.py PacayaT163TsxHhA 20150102 20150601 -------------------------------------------------------------------- -''' - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Unwrap differential interferogram using GAMMA-mcf method.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('Mdate',help='Master date.') - parser.add_argument('Sdate',help='Slave date.') - - inps = parser.parse_args() - return inps - - -def main(argv): - - inps = cmdLineParse() - Mdate = inps.Mdate - Sdate = inps.Sdate - - projectName = inps.projectName - Sdate = inps.Sdate - Mdate = inps.Mdate - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - auto_unw = templateDict['auto_unw'] - init_flag = templateDict['init_flag'] - r_refer = templateDict['r_refer'] - a_refer = templateDict['a_refer'] - make_mask = templateDict['make_mask'] - processDir = scratchDir + '/' + projectName + "/PROCESS" - slcDir = scratchDir + '/' + projectName + "/SLC" - rslcDir = scratchDir + '/' + projectName + '/RSLC' - ifgDir = scratchDir + '/' + projectName + '/ifgrams' - masterDate = templateDict['masterDate'] - demDir = scratchDir + '/' + projectName + '/DEM' - Pair = Mdate + '-' + Sdate - workDir = ifgDir + '/' + Pair - - ################ prepare file for parallel processing ############### - HGTSIM = demDir + '/' + masterDate + '_' + rlks + 'rlks.rdc.dem' - Mamp = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp' - MampPar = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp.par' - Samp = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp' - SampPar = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp.par' - diff_par = workDir + '/' + Mdate + '_' + Sdate + '_diff_par' - UNWlks = workDir + '/' + Pair + '_' +rlks + 'rlks.diff_filt.unw' - CORMASK = workDir + '/' + Pair + '_' +rlks + 'rlks.diff_filt.cor' - ATM_PHASE = workDir + '/' + Pair + '_' +rlks + 'rlks.atm_phae' - ATMCOR_UNW = workDir + '/' + Pair + '_' +rlks + 'rlks.diff_filt.atmcor.unw' - - nWidth = ut.read_gamma_par(MampPar, 'read', 'range_samples') - nLine = ut.read_gamma_par(MampPar, 'read', 'azimuth_lines') - ############################################################### - call_str = "create_diff_par " + MampPar + " " + SampPar + " " + diff_par + " 1 0 " - os.system(call_str) - call_str = "atm_mod_2d " + UNWlks + " " + HGTSIM + " " + CORMASK + " " + diff_par + " " + " - 0 a0 a1 sigma sigma_h s1" - os.system(call_str) - call_str = "atm_sim_2d " + diff_par + " " + HGTSIM + " a0 a1 " + ATM_PHASE - os.system(call_str) - call_str = "sub_phase " + UNWlks + " " + ATM_PHASE + " " + diff_par + " " + ATMCOR_UNW + " 0 0 0 " - os.system(call_str) - call_str = 'rasrmg ' + ATMCOR_UNW + ' ' + Mamp + ' ' + nWidth + ' - - - - - - - - - - ' - os.system(call_str) - - print("Correct atmospheric phase is done!") - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/atm_correction_gamma_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/atm_correction_gamma_all.py deleted file mode 100644 index 5ad8a59..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/atm_correction_gamma_all.py +++ /dev/null @@ -1,121 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import getopt -import time -import glob -import argparse - -import subprocess -from pyint import _utils as ut - - -def work(data0): - cmd = data0[0] - err_file = data0[1] - p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) - stdout = p.stdout - stderr = p.stderr - - if type(stderr) == bytes: - aa=stderr.decode("utf-8") - else: - aa = stderr - - if aa: - str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + cmd[3] + '\n' - #print(aa) - with open(err_file, 'a') as f: - f.write(str0) - f.write(aa) - f.write('\n') - - return -######################################################################### - -INTRODUCTION = ''' -------------------------------------------------------------------- - Correct atmospheric phase for one project using GAMMA. - -''' - -EXAMPLE = ''' - Usage: - atm_correction_gamma_all.py projectName - atm_correction_gamma_all.py projectName --parallel 4 - atm_correction_gamma_all.py projectName --parallel 4 --ifgramList-txt /test/ifgram_list.txt -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Unwrap differential interferograms for one project using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') - parser.add_argument('--ifgarmList-txt', dest='ifgarmListTxt', help='provided ifgram_list_txt. default: using ifgram_list.txt under projectName folder.') - - inps = parser.parse_args() - return inps - - -def main(argv): - start_time = time.time() - inps = cmdLineParse() - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - projectDir = scratchDir + '/' + projectName - ifgDir = scratchDir + '/' + projectName + '/ifgrams' - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - - if inps.ifgarmListTxt: ifgramList_txt = inps.ifgarmListTxt - else: ifgramList_txt = scratchDir + '/' + projectName + '/ifgram_list.txt' - ifgList0 = ut.read_txt2array(ifgramList_txt) -# ifgList = ifgList0[:,0] - if len(ifgList0)==3: - ifgList=ifgList0[0] - ifgList=[ifgList] - else: - ifgList=ifgList0[:,0] - - err_txt = scratchDir + '/' + projectName + '/unwrap_gamma_all.err' - if os.path.isfile(err_txt): os.remove(err_txt) - - data_para = [] - for i in range(len(ifgList)): - m0 = ut.yyyymmdd(ifgList[i].split('-')[0]) - s0 = ut.yyyymmdd(ifgList[i].split('-')[1]) - cmd0 = ['atm_correction_gamma.py',projectName, m0, s0] - unw_file0 = ifgDir + '/' + ifgList[i] + '/' + ifgList[i] + '_' + rlks + 'rlks.diff_filt.atmcor.unw.bmp' - data0 = [cmd0,err_txt] - - k00 = 0 - if os.path.isfile(unw_file0): - if os.path.getsize(unw_file0) > 0: - k00 = 1 - if k00==0: - data_para.append(data0) - - ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) - print("Correct atmospheric phase for project %s is done! " % projectName) - ut.print_process_time(start_time, time.time()) - - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/change_Name_for_mintpy.py b/.codex_tmp/pyint_variants/no_rescue/pyint/change_Name_for_mintpy.py deleted file mode 100644 index 271157f..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/change_Name_for_mintpy.py +++ /dev/null @@ -1,135 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v1.0 ### -### Copy Right (c): 2017, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Email : ymcmrs@gmail.com ### -### Univ. : Central South University & University of Miami ### -################################################################# - -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob - -def check_variable_name(path): - s=path.split("/")[0] - if len(s)>0 and s[0]=="$": - p0=os.getenv(s[1:]) - path=path.replace(path.split("/")[0],p0) - return path - -def read_template(File, delimiter='='): - '''Reads the template file into a python dictionary structure. - Input : string, full path to the template file - Output: dictionary, pysar template content - Example: - tmpl = read_template(KyushuT424F610_640AlosA.template) - tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') - ''' - template_dict = {} - for line in open(File): - line = line.strip() - c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter - if len(c) < 2 or line.startswith('%') or line.startswith('#'): - next #ignore commented lines or those without variables - else: - atrName = c[0] - atrValue = str.replace(c[1],'\n','').split("#")[0].strip() - atrValue = check_variable_name(atrValue) - template_dict[atrName] = atrValue - return template_dict - - -def is_number(s): - try: - int(s) - return True - except ValueError: - return False - -def add_zero(s): - if len(s)==1: - s="000"+s - elif len(s)==2: - s="00"+s - elif len(s)==3: - s="0"+s - return s - -def usage(): - print(''' -****************************************************************************************************** - - Select interferometry pairs from time series SAR images - - usage: - - Generate_IfgDir.py ProjectName IFG_List - - e.g. Generate_IfgDir.py PacayaT163TsxHhA ifg_list - - -******************************************************************************************************* - ''') - -def main(argv): - - if len(sys.argv)==2: - if argv[0] in ['-h','--help']: usage(); sys.exit(1) - else: - projectName=sys.argv[1] - projectName=sys.argv[1] - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - processDir = scratchDir + '/' + projectName + "/ifgrams" - root_dir=os.getcwd() - os.chdir(processDir) - call_str="find . -name '[0-9]*.unw' -print >unw_file" - os.system(call_str) - file_obj=open('unw_file') - all_lines=file_obj.readlines() - for line in all_lines: - unwdate1=line.split("/")[2] - unwdate2=line.split("/")[1] - unwdate3=line.split("/")[0] - str1=unwdate1.split(".")[0] - str2=unwdate1.split(".")[1] - str3=unwdate1.split(".")[2] - unwdate4=str2 + '_' + str1 + '.' + str3 - line1=processDir + '/' + unwdate2+ '/' + unwdate4 - line2=processDir + '/'+ unwdate2 + '/' + unwdate1 - line2=line2.split('\n')[0] - line1=line1.split('\n')[0] - call_str='mv'+ ' ' + line2 + ' ' + line1 - os.system(call_str) - call_str="find . -name '[0-9]*.cor' -print >cor_file" - os.system(call_str) - - file_obj=open('cor_file') - all_lines=file_obj.readlines() - for line in all_lines: - cordate1=line.split("/")[2] - cordate2=line.split("/")[1] - cordate3=line.split("/")[0] - str1=cordate1.split(".")[0] - str2=cordate1.split(".")[1] - str3=cordate1.split(".")[2] - sub1=str2.split('_')[1] - cordate4=sub1 + '_' + str1 + '.' + str3 - line1=processDir + '/' + cordate2 + '/' + cordate4 - line1=line1.split('\n')[0] - line2=processDir + '/'+ cordate2 + '/' + cordate1 - line2=line2.split('\n')[0] - call_str='mv' + ' ' + line2 + ' ' + line1 - os.system(call_str) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/convert_phs_to_grd.csh b/.codex_tmp/pyint_variants/no_rescue/pyint/convert_phs_to_grd.csh deleted file mode 100644 index ec8da05..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/convert_phs_to_grd.csh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/csh -f - -if ($#argv != 2) then - echo "" - echo "Usage: read_phs_file.csh phase_file header.rsc" - echo "" - echo "Convert binary files to .grd file" - echo "" - echo "binary file and header in .rsc is needed" - exit 1 - endif - -#######File to grid######### -set x_first_d1 = `cat $2|grep X_FIRST|awk '{print $2}'` -set y_first_d1 = `cat $2|grep Y_FIRST|awk '{print $2}'` -set width_d1 = `cat $2|grep WIDTH|awk '{print $2}'` -set length_d1 = `cat $2|grep FILE_LENGTH|awk '{print $2}'` -set x_step_d1 = `cat $2|grep X_STEP|awk '{print $2}'` -set y_step_d1 = `cat $2|grep X_STEP|awk '{print $2}'` -set name = `echo $1| awk -F/ '{print $NF}'` -gmt xyz2grd $1 -G"$name.grd" -RLT$x_first_d1/$y_first_d1/$width_d1/$length_d1 -I$x_step_d1/$y_step_d1 -ZTLf -di0 -r - -echo "From $1, $name.grd has been produced" diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_gamma.py deleted file mode 100644 index 49adcad..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_gamma.py +++ /dev/null @@ -1,242 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# -import numpy as np -import os -import sys -import argparse - -from pyint import _utils as ut - - -def _run_or_raise(call_str, stage): - rc = os.system(call_str) - if rc != 0: - raise RuntimeError('%s failed with rc=%s: %s' % (stage, rc, call_str)) - return rc - - -def _safe_remove(path): - if path and os.path.exists(path): - os.remove(path) - - -def _copy_file(src, dst): - ut.copy_file(src, dst) - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Coregister SM mode SLC to a reference SLC image using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName', help='Name of project.') - parser.add_argument('sdate', help='date of the slave SLC image. [mater date is read from template]') - inps = parser.parse_args() - return inps - - -INTRODUCTION = ''' -------------------------------------------------------------------- - Coregister SM mode SLC to a reference SLC image using GAMMA. - [The reference date or master date will be read from the template file.] -''' - -EXAMPLE = """Usage: - - coreg_gamma.py projectName Sdate - - coreg_gamma.py PacayaT163TsxHhA 20150102 -------------------------------------------------------------------- -""" - - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - Sdate = inps.sdate - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - - slcDir = scratchDir + '/' + projectName + "/SLC" - rslcDir = scratchDir + '/' + projectName + "/RSLC" - if not os.path.isdir(rslcDir): os.mkdir(rslcDir) - #workDir = processDir + '/' + igramDir - workDir = rslcDir + '/' + Sdate - if not os.path.isdir(workDir): os.mkdir(workDir) - - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - Mdate = templateDict['masterDate'] - IFGPair = Mdate + '-' + Sdate - - demDir = scratchDir + '/' + projectName + '/DEM' - - SslcDir = slcDir + '/' + Sdate - Samp = slcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' - SampPar = slcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' - Sramp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' - SrampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' - - Sslc = slcDir + '/' + Sdate + '/' + Sdate + '.slc' - SslcPar = slcDir + '/' + Sdate + '/' + Sdate + '.slc.par' - - SrslcDir = rslcDir + "/" + Sdate - - Srslc = SrslcDir + "/" + Sdate + ".rslc" - SrslcPar = SrslcDir + "/" + Sdate + ".rslc.par" - - Srslc0 = SrslcDir + "/" + Sdate + ".rslc0" - SrslcPar0 = SrslcDir + "/" + Sdate + ".rslc0.par" - -##################################################### -## copy all of the master files into slave folder for parallel processing - remove_file = [] - Mslc0 = slcDir + '/' + Mdate + '/' + Mdate + '.slc' - MslcPar0 = slcDir + '/' + Mdate + '/' + Mdate + '.slc.par' - Mamp0 = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' - MampPar0 = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' - Mamp0_dem = demDir + '/' + Mdate + '_' + rlks + 'rlks.amp' - MampPar0_dem = demDir + '/' + Mdate + '_' + rlks + 'rlks.amp.par' - HGTSIM0 = demDir + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' - if not os.path.isfile(HGTSIM0): - call_str = 'generate_rdc_dem.py ' + projectName - _run_or_raise(call_str, 'generate_rdc_dem') - if not os.path.isfile(Mamp0) and os.path.isfile(Mamp0_dem): - ut.copy_file(Mamp0_dem, Mamp0) - if not os.path.isfile(MampPar0) and os.path.isfile(MampPar0_dem): - ut.copy_file(MampPar0_dem, MampPar0) - if not os.path.isfile(Mamp0) or not os.path.isfile(MampPar0): - call_str = 'multi_look ' + Mslc0 + ' ' + MslcPar0 + ' ' + Mamp0 + ' ' + MampPar0 + ' ' + rlks + ' ' + azlks - _run_or_raise(call_str, 'master_multi_look') - if not os.path.isfile(Samp) or not os.path.isfile(SampPar): - call_str = 'multi_look ' + Sslc + ' ' + SslcPar + ' ' + Samp + ' ' + SampPar + ' ' + rlks + ' ' + azlks - _run_or_raise(call_str, 'slave_multi_look') - - Mslc = slcDir + '/' + Sdate + '/' + Mdate + '.slc' - MslcPar = slcDir + '/' + Sdate + '/' + Mdate + '.slc.par' - Mamp = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp' - MampPar = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' - HGTSIM = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' - - ut.copy_file(HGTSIM0,HGTSIM) - ut.copy_file(Mslc0,Mslc) - ut.copy_file(MslcPar0,MslcPar) - ut.copy_file(Mamp0,Mamp) - ut.copy_file(MampPar0,MampPar) - -####################################################### -# define process files # - lt0 = workDir + "/lt0" - lt1 = workDir + "/lt1" - mli0 = workDir + "/mli0" - diff0 = workDir + "/diff0" - offs0 = workDir + "/offs0" - snr0 = workDir + "/snr0" - offsets0 = workDir + "/offsets0" - coffs0 = workDir + "/coffs0" - coffsets0 = workDir + "/coffsets0" - off = workDir + "/" + IFGPair + ".off" - offs = workDir + "/offs" - snr = workDir + "/snr" - offsets = workDir + "/offsets" - coffs = workDir + "/coffs" - coffsets = workDir + "/coffsets" - OFFSTD = workDir + "/" + IFGPair + ".off_std" -############################################## - - call_str = "rdc_trans " + MampPar + " " + HGTSIM + " " + SampPar + " " + lt0 - _run_or_raise(call_str, 'rdc_trans') - - width_Mamp = ut.read_gamma_par(MampPar, 'read', 'range_samples') - width_Samp = ut.read_gamma_par(SampPar, 'read', 'range_samples') - line_Samp = ut.read_gamma_par(SampPar, 'read', 'azimuth_lines') - - call_str = "geocode " + lt0 + " " + Mamp + " " + width_Mamp + " " + mli0 + " " + width_Samp + " " + line_Samp + " 2 0" - _run_or_raise(call_str, 'geocode_lt0') - - call_str = "create_diff_par " + SampPar + " - " + diff0 + " 1 0" - _run_or_raise(call_str, 'create_diff_par') - - try: - call_str = "init_offsetm " + mli0 + " " + Samp + " " + diff0 + " 1 1" - _run_or_raise(call_str, 'init_offsetm') - - call_str = "offset_pwrm " + mli0 + " " + Samp + " " + diff0 + " " + offs0 + " " + snr0 + " 256 256 " + offsets0 + " 2 32 32" - _run_or_raise(call_str, 'offset_pwrm') - - call_str = "offset_fitm " + offs0 + " " + snr0 + " " + diff0 + " " + coffs0 + " " + coffsets0 + " - 4" - _run_or_raise(call_str, 'offset_fitm') - - call_str = "gc_map_fine " + lt0 + " " + width_Mamp + " " + diff0 + " " + lt1 - _run_or_raise(call_str, 'gc_map_fine') - except RuntimeError as exc: - print('ERROR: initial DEM-assisted offset refinement failed in strict no-rescue mode') - print(str(exc)) - raise - - - call_str = "SLC_interp_lt " + Sslc + " " + MslcPar + " " + SslcPar + " " + lt1 + " " + MampPar + " " + SampPar + " - " + Srslc0 + " " + SrslcPar0 - _run_or_raise(call_str, 'SLC_interp_lt_initial') - - -############################################ Refinement ############################################ - try: - call_str = "create_offset " + MslcPar + " " + SrslcPar0 + " " + off + " 1 - - 0" - _run_or_raise(call_str, 'create_offset') - - call_str = "offset_pwr " + Mslc + " " + Srslc0 + " " + MslcPar + " " + SrslcPar0 + " " + off + " " + offs + " " + snr + " " + templateDict['rwin4cor'] + " " + templateDict['azwin4cor'] + " " + offsets + " 2 " + templateDict['rsample4cor'] + " " + templateDict['azsample4cor'] - _run_or_raise(call_str, 'offset_pwr_coarse') - - call_str = "offset_fit " + offs + " " + snr + " " + off + " " + coffs + " " + coffsets + " - 3" - _run_or_raise(call_str, 'offset_fit_coarse') - - rfwin4cor = str(int(1/2*int(templateDict['rwin4cor']))) - azfwin4cor = str(int(1/2*int(templateDict['azwin4cor']))) - - rfsample4cor = str(2*int(templateDict['rsample4cor'])) - azfsample4cor = str(2*int(templateDict['azsample4cor'])) - - call_str = "offset_pwr " + Mslc + " " + Srslc0 + " " + MslcPar + " " + SrslcPar0 + " " + off + " " + offs + " " + snr + " " + rfwin4cor + " " + azfwin4cor + " " + offsets + " 2 " + rfsample4cor + " " + azfsample4cor - _run_or_raise(call_str, 'offset_pwr_fine') - - call_str = "offset_fit " + offs + " " + snr + " " + off + " " + coffs + " " + coffsets + " - 3 >" + OFFSTD - _run_or_raise(call_str, 'offset_fit_fine') - -############################################ Resampling ############################################ - - call_str = "SLC_interp_lt " + Sslc + " " + MslcPar + " " + SslcPar + " " + lt1 + " " + MampPar + " " + SampPar + " " + off + " " + Srslc + " " + SrslcPar - _run_or_raise(call_str, 'SLC_interp_lt_final') - except RuntimeError as exc: - print('ERROR: offset refinement failed in strict no-rescue mode') - print(str(exc)) - raise - - call_str = 'multi_look ' + Srslc + ' ' + SrslcPar + ' ' + Sramp + ' ' + SrampPar + ' ' + rlks + ' ' + azlks - _run_or_raise(call_str, 'multi_look_rslc') - - nWidth = ut.read_gamma_par(SrampPar, 'read', 'range_samples') - call_str = 'raspwr ' + Sramp + ' ' + nWidth - _run_or_raise(call_str, 'raspwr') - - for path in ( - lt0, lt1, mli0, diff0, offs0, snr0, offsets0, coffs0, coffsets0, - off, offs, snr, offsets, coffs, coffsets, Srslc0, SrslcPar0, - Mslc, Mamp, HGTSIM, - ): - _safe_remove(path) - - print("Coregistration with DEM is done!") - - sys.exit(0) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_gamma_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_gamma_all.py deleted file mode 100644 index cf7dd4d..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_gamma_all.py +++ /dev/null @@ -1,187 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import getopt -import time -import glob -import argparse -import shutil - -import subprocess -from pyint import _utils as ut - -def work(data0): - cmd = data0[0] - err_txt = data0[1] - - p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) - stdout = p.stdout - stderr = p.stderr - - if type(stderr) == bytes: - aa=stderr.decode("utf-8") - else: - aa = stderr - - if type(stdout) == bytes: - bb=stdout.decode("utf-8", errors="replace") - else: - bb = stdout - - if p.returncode != 0: - detail_parts = [] - if bb: - detail_parts.append(bb) - if aa: - detail_parts.append(aa) - detail = '\n'.join(part.strip() for part in detail_parts if part and part.strip()) - str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' - with open(err_txt, 'a') as f: - f.write(str0) - if detail: - f.write(detail) - f.write('\n') - raise RuntimeError(str0.strip() + ' failed with rc=' + str(p.returncode) + ('\n' + detail if detail else '')) - - if aa: - str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' - with open(err_txt, 'a') as f: - f.write(str0) - f.write(aa) - f.write('\n') - - return -######################################################################### - - -def ensure_master_rslc(projectName): - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict = ut.update_template(templateFile) - master_date = templateDict['masterDate'] - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - - slc_root = scratchDir + '/' + projectName + '/SLC/' + master_date - rslc_root = scratchDir + '/' + projectName + '/RSLC/' + master_date - if not os.path.isdir(rslc_root): - os.makedirs(rslc_root, exist_ok=True) - - src_slc = slc_root + '/' + master_date + '.slc' - src_slc_par = slc_root + '/' + master_date + '.slc.par' - src_amp = slc_root + '/' + master_date + '_' + rlks + 'rlks.amp' - src_amp_par = slc_root + '/' + master_date + '_' + rlks + 'rlks.amp.par' - - dst_rslc = rslc_root + '/' + master_date + '.rslc' - dst_rslc_par = rslc_root + '/' + master_date + '.rslc.par' - dst_amp = rslc_root + '/' + master_date + '_' + rlks + 'rlks.amp' - dst_amp_par = rslc_root + '/' + master_date + '_' + rlks + 'rlks.amp.par' - - if not os.path.isfile(src_slc) or not os.path.isfile(src_slc_par): - raise FileNotFoundError('Master SLC is missing under SLC/' + master_date) - - if not os.path.isfile(dst_rslc): - shutil.copyfile(src_slc, dst_rslc) - if not os.path.isfile(dst_rslc_par): - shutil.copyfile(src_slc_par, dst_rslc_par) - - if os.path.isfile(src_amp) and not os.path.isfile(dst_amp): - shutil.copyfile(src_amp, dst_amp) - if os.path.isfile(src_amp_par) and not os.path.isfile(dst_amp_par): - shutil.copyfile(src_amp_par, dst_amp_par) - - if not os.path.isfile(dst_amp) or not os.path.isfile(dst_amp_par): - cmd = [ - 'multi_look', - dst_rslc, - dst_rslc_par, - dst_amp, - dst_amp_par, - rlks, - azlks, - ] - p = subprocess.run(cmd, shell=False, stderr=subprocess.PIPE, stdout=subprocess.PIPE) - if p.returncode != 0: - detail_parts = [] - if p.stdout: - detail_parts.append(p.stdout.decode("utf-8", errors="replace")) - if p.stderr: - detail_parts.append(p.stderr.decode("utf-8", errors="replace")) - detail = '\n'.join(part.strip() for part in detail_parts if part and part.strip()) - raise RuntimeError('multi_look master RSLC failed' + ('\n' + detail if detail else '')) - -INTRODUCTION = ''' -------------------------------------------------------------------- - Coregister all of the SLCs to the reference SAR image using GAMMA. - [with assistant of DEM] - -''' - -EXAMPLE = ''' - Usage: - coreg_gamma_all.py projectName - coreg_gamma_all.py projectName --parallel 4 -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Coregister all of the SLCs to the reference SAR image using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') - - inps = parser.parse_args() - return inps - - -def main(argv): - start_time = time.time() - inps = cmdLineParse() - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - projectDir = scratchDir + '/' + projectName - slcDir = scratchDir + '/' + projectName + '/SLC' - rslcDir = scratchDir + '/' + projectName + '/RSLC' - if not os.path.isdir(rslcDir): os.mkdir(rslcDir) - ensure_master_rslc(projectName) - - if 'S1' in projectName: cmd_command = 'coreg_s1_gamma.py' - else: cmd_command = 'coreg_gamma.py' - - err_txt = scratchDir + '/' + projectName + '/coreg_gamma_all.err' - if os.path.isfile(err_txt): os.remove(err_txt) - - data_para = [] - #slc_list = [os.path.basename(fname) for fname in sorted(glob.glob(slcDir + '/*'))] - master_date = ut.update_template(templateDir + "/" + projectName + ".template")['masterDate'] - slc_list = [item for item in ut.get_project_slcList(projectName) if item != master_date] - for i in range(len(slc_list)): - cmd0 = [cmd_command,projectName,slc_list[i]] - data0 = [cmd0,err_txt] - data_para.append(data0) - - results = ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) - failures = [str(item) for item in results if isinstance(item, Exception)] - if failures: - raise RuntimeError('\n\n'.join(failures)) - print("Coregister all of the SLCs %s is done! " % projectName) - ut.print_process_time(start_time, time.time()) - - sys.exit(0) - -if __name__ == '__main__': - main(sys.argv[:]) - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_s1_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_s1_gamma.py deleted file mode 100644 index 97ebd10..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_s1_gamma.py +++ /dev/null @@ -1,239 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# -import numpy as np -import os -import sys -import argparse - -from pyint import _utils as ut - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Coregister TOPS S1-SLC to a reference S1-SLC using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName', help='Name of project.') - parser.add_argument('sdate', help='date of the slave S1 image. [mater date is read from template]') - inps = parser.parse_args() - return inps - - -INTRODUCTION = ''' -------------------------------------------------------------------- - Coregister TOPS S1-SLC to a reference S1-SLC using GAMMA. - [The reference date or master date will be read from the template file.] -''' - -EXAMPLE = """Usage: - - coreg_s1_gamma.py projectName Sdate - - coreg_s1_gamma.py PacayaT163TsxHhA 20150102 -------------------------------------------------------------------- -""" - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - Sdate = inps.sdate - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - - slcDir = scratchDir + '/' + projectName + "/SLC" - rslcDir = scratchDir + '/' + projectName + "/RSLC" - if not os.path.isdir(rslcDir): os.mkdir(rslcDir) - #workDir = processDir + '/' + igramDir - workDir = rslcDir + '/' + Sdate - if not os.path.isdir(workDir): os.mkdir(workDir) - - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - Mdate = templateDict['masterDate'] - - demDir = scratchDir + '/' + projectName + '/DEM' - -# Definition of file - MslcDir = slcDir + '/' + Mdate - SslcDir = slcDir + '/' + Sdate - Samp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' - SampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' - Sampbmp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.bmp' - - Mslc = slcDir + '/' + Mdate + '/' + Mdate + '.slc' - Mslcpar = slcDir + '/' + Mdate + '/' + Mdate + '.slc.par' - Mamp = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' - MampPar = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' - Mampbmp = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.bmp' - - SLC1_INF_tab0 = MslcDir + '/' + Mdate + '_SLC_Tab' - SLC2_INF_tab = SslcDir + '/' + Sdate + '_SLC_Tab' - RSLC_tab = SslcDir + '/' + Sdate + '_RSLC_Tab' - SLC1_INF_tab = SslcDir + '/' + Mdate + '_SLC_Tab_coreg' - - HGTSIM = demDir + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' - if not os.path.isfile(HGTSIM): - call_str = 'generate_rdc_dem.py ' + projectName - os.system(call_str) - - ############## copy master files into slave folder for parallel process ########### - - #if not templateDict['coreg_all_parallel'] == '1': - # Mslc = slcDir + '/' + Sdate + '/' + Mdate + '.slc' - # Mslcpar = slcDir + '/' + Sdate + '/' + Mdate + '.slc.par' - # Mamp = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp' - # MampPar = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' - # HGTSIM = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' - - #else: - - # Mslc = Mslc0 - # Mslcpar = Mslcpar0 - # Mamp = Mamp0 - # MampPar = MampPar0 - # HGTSIM = HGTSIM0 - - - #Mslc = slcDir + '/' + Sdate + '/' + Mdate + '.slc' - #Mslcpar = slcDir + '/' + Sdate + '/' + Mdate + '.slc.par' - #Mamp = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp' - #MampPar = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' - #HGTSIM = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' - SLC1_INF_tab1 = SslcDir + '/' + Mdate + '_SLC_Tab' - if SslcDir==MslcDir: - SLC1_INF_tab1=SLC1_INF_tab0 - else: - ut.copy_file(SLC1_INF_tab0,SLC1_INF_tab1) - ############################################################################## - #with open(SLC1_INF_tab1, "r") as f: - # lines = f.readlines() - - #with open(SLC1_INF_tab, "w") as fw: - # lines_coreg = [] - # for k0 in lines: - # k00 = k0.replace(MslcDir,SslcDir) - # lines_coreg.append(k00) - # fw.write(k00) - - S_IW = ut.read_txt2array(SLC1_INF_tab1) - S_IW = S_IW.flatten() - #M_IW = ut.read_txt2array(SLC1_INF_tab1) - #M_IW = M_IW.flatten() - #S_IW = ut.read_txt2array(SLC1_INF_tab) - #S_IW = S_IW.flatten() - - #RSLC_tab = workDir + '/' + Sdate + '_RSLC_tab' - #if os.path.isfile(RSLC_tab): - # os.remove(RSLC_tab) - - #BURST = SslcDir + '/' + Sdate + '_Burst_Tab' - #AA = np.loadtxt(BURST) - #if EW==SW: - # AA = AA.reshape([1,2]) - # - #for kk in range(int(EW)-int(SW)+1): - # ii = int(int(kk) + 1) - # SB2=AA[ii-1,0] - # EB2=AA[ii-1,1] - # call_str = 'echo ' + workDir + '/'+ Sdate+ '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+str(int(SW)+kk)+ '.rslc' + ' ' + workDir + '/' + Sdate + '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+ str(int(SW)+kk)+ '.rslc.par' + ' ' + workDir + '/'+ Sdate+'_'+ str(int(SB2)) + str(int(EB2)) + '.IW'+str(int(SW)+kk)+ '.rslc.TOPS_par >>' + RSLC_tab - # os.system(call_str) - - os.chdir(workDir) - #TEST = workDir + '/' + Sdate +'_' + rlks + 'rlks.amp.par' - TEST = workDir + '/' + Mdate + '_' + Sdate +'.diff' - - k0 = 0 - if os.path.isfile(TEST): - if os.path.getsize(TEST) > 0: - k0 = 1 - - if k0 == 0: - if not Mdate==Sdate: -# call_str = 'S1_coreg_TOPS ' + SLC1_INF_tab1 + ' ' + Mdate + ' ' + SLC2_INF_tab + ' ' + Sdate + ' ' + RSLC_tab + ' ' + HGTSIM + ' ' + rlks + ' ' + azlks + ' - - 0.8 0.01 1.2 1' - call_str = 'ScanSAR_coreg.py ' + SLC1_INF_tab1 + ' ' + Mdate + ' ' + SLC2_INF_tab + ' ' + Sdate + ' ' + RSLC_tab + ' ' + HGTSIM + ' ' + rlks + ' ' + azlks + ' --cc 0.8 --fraction 0.01 --ph_stdev 0.8 --num_ovr 0 --no_check ' - print(call_str) - ret = os.system(call_str) - - #### clean large file #### - mslc = workDir + '/' + Mdate + '.slc' - mrslc = workDir + '/' + Mdate + '.rslc' - sslc = workDir + '/' + Sdate + '.slc' - srslc = workDir + '/' + Sdate + '.rslc' - srslcPar = workDir + '/' + Sdate + '.rslc.par' - - # 检查 ScanSAR_coreg.py 是否成功生成 .rslc（不检查返回码，spectral diversity 步骤可能报错但 RSLC 已生成） - if not os.path.isfile(srslc) or os.path.getsize(srslc) == 0: - print('ERROR: ScanSAR_coreg.py did not produce a valid ' + srslc) - sys.exit(1) - - call_str = 'multi_look ' + srslc + ' ' + srslcPar + ' ' + Samp + ' ' + SampPar + ' ' + rlks + ' ' + azlks - os.system(call_str) - nWIDTH = ut.read_gamma_par(SampPar,'read', 'range_samples') - if os.path.isfile(mslc): os.remove(mslc) - if os.path.isfile(mrslc): os.remove(mrslc) - if os.path.isfile(sslc): os.remove(sslc) - - call_str = 'raspwr ' + Samp + ' ' + nWIDTH - os.system(call_str) - - #call_str = 'rm *mli*' - #os.system(call_str) - - #call_str = 'rm *IW*' - #os.system(call_str) - - #call_str = 'rm *off*' - #os.system(call_str) - - #call_str = 'rm *diff' - #os.system(call_str) - - #call_str = 'rm *diff_par*' - #os.system(call_str) - - #call_str = 'rm ' + Mdate + '.*' - #os.system(call_str) - - else: - call_str = 'cp ' + Mslc + ' ' + workDir + '/' + Mdate + '.rslc' - os.system(call_str) - - call_str = 'cp ' + Mslcpar + ' ' + workDir+ '/' + Mdate + '.rslc.par' - os.system(call_str) - - call_str = 'cp ' + Mamp + ' ' + workDir+ '/' + Mdate + '_' + rlks + 'rlks.amp' - os.system(call_str) - - call_str = 'cp ' + MampPar + ' ' + workDir+ '/' + Mdate + '_' + rlks + 'rlks.amp.par' - os.system(call_str) - - ut.copy_file(Mampbmp,Sampbmp) - - ################ clean redundant files ############# - - #if not Mdate ==Sdate: - - # if not templateDict['diff_all_parallel'] == '1': - # for i in range(len(S_IW)): - # if os.path.isfile(S_IW[i]): - # os.remove(S_IW[i]) - # if os.path.isfile(Mslc): os.remove(Mslc) - # if os.path.isfile(Mslcpar): os.remove(Mslcpar) - # if os.path.isfile(Mamp): os.remove(Mamp) - # if os.path.isfile(HGTSIM): os.remove(HGTSIM) - else: - print('The SLC has already coregister Done') - sys.exit(1) - print("Coregister TOP SLC image to the reference TOPS image is done !!") - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_s1_gamma_old.py b/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_s1_gamma_old.py deleted file mode 100644 index 15f9e08..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_s1_gamma_old.py +++ /dev/null @@ -1,232 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# -import numpy as np -import os -import sys -import argparse - -from pyint import _utils as ut - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Coregister TOPS S1-SLC to a reference S1-SLC using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName', help='Name of project.') - parser.add_argument('sdate', help='date of the slave S1 image. [mater date is read from template]') - inps = parser.parse_args() - return inps - - -INTRODUCTION = ''' -------------------------------------------------------------------- - Coregister TOPS S1-SLC to a reference S1-SLC using GAMMA. - [The reference date or master date will be read from the template file.] -''' - -EXAMPLE = """Usage: - - coreg_s1_gamma.py projectName Sdate - - coreg_s1_gamma.py PacayaT163TsxHhA 20150102 -------------------------------------------------------------------- -""" - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - Sdate = inps.sdate - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - - slcDir = scratchDir + '/' + projectName + "/SLC" - rslcDir = scratchDir + '/' + projectName + "/RSLC" - if not os.path.isdir(rslcDir): os.mkdir(rslcDir) - #workDir = processDir + '/' + igramDir - workDir = rslcDir + '/' + Sdate - if not os.path.isdir(workDir): os.mkdir(workDir) - - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - Mdate = templateDict['masterDate'] - - demDir = scratchDir + '/' + projectName + '/DEM' - -# Definition of file - MslcDir = slcDir + '/' + Mdate - SslcDir = slcDir + '/' + Sdate - Samp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' - SampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' - Sampbmp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.bmp' - - Mslc = slcDir + '/' + Mdate + '/' + Mdate + '.slc' - Mslcpar = slcDir + '/' + Mdate + '/' + Mdate + '.slc.par' - Mamp = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' - MampPar = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' - Mampbmp = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.bmp' - - #SLC1_INF_tab0 = MslcDir + '/' + Mdate + '_SLC_Tab0' - SLC1_INF_tab0 = MslcDir + '/' + Mdate + '_SLC_Tab' - SLC2_INF_tab = SslcDir + '/' + Sdate + '_SLC_Tab' - RSLC_tab = SslcDir + '/' + Sdate + '_RSLC_Tab' - #SLC1_INF_tab = SslcDir + '/' + Mdate + '_SLC_Tab_coreg' - SLC1_INF_tab = SslcDir + '/' + Mdate + '_SLC_Tab' - - HGTSIM = demDir + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' - if not os.path.isfile(HGTSIM): - call_str = 'generate_rdc_dem.py ' + projectName - os.system(call_str) - - ############## copy master files into slave folder for parallel process ########### - - #if not templateDict['coreg_all_parallel'] == '1': - # Mslc = slcDir + '/' + Sdate + '/' + Mdate + '.slc' - # Mslcpar = slcDir + '/' + Sdate + '/' + Mdate + '.slc.par' - # Mamp = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp' - # MampPar = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' - # HGTSIM = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' - - #else: - - # Mslc = Mslc0 - # Mslcpar = Mslcpar0 - # Mamp = Mamp0 - # MampPar = MampPar0 - # HGTSIM = HGTSIM0 - - - #Mslc = slcDir + '/' + Sdate + '/' + Mdate + '.slc' - #Mslcpar = slcDir + '/' + Sdate + '/' + Mdate + '.slc.par' - #Mamp = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp' - #MampPar = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' - #HGTSIM = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' - SLC1_INF_tab = SslcDir + '/' + Mdate + '_SLC_Tab' - if SslcDir==MslcDir: - SLC1_INF_tab1=SLC1_INF_tab0 - else: - ut.copy_file(SLC1_INF_tab0,SLC1_INF_tab) - ############################################################################## - #with open(SLC1_INF_tab, "r") as f: - # lines = f.readlines() - - #with open(SLC1_INF_tab, "w") as fw: - # lines_coreg = [] - # for k0 in lines: - # k00 = k0.replace(MslcDir,SslcDir) - # lines_coreg.append(k00) - # fw.write(k00) - - S_IW = ut.read_txt2array(SLC1_INF_tab) - S_IW = S_IW.flatten() - #M_IW = ut.read_txt2array(SLC1_INF_tab1) - #M_IW = M_IW.flatten() - #S_IW = ut.read_txt2array(SLC1_INF_tab) - #S_IW = S_IW.flatten() - - #RSLC_tab = workDir + '/' + Sdate + '_RSLC_tab' - #if os.path.isfile(RSLC_tab): - # os.remove(RSLC_tab) - - #BURST = SslcDir + '/' + Sdate + '_Burst_Tab' - #AA = np.loadtxt(BURST) - #if EW==SW: - # AA = AA.reshape([1,2]) - # - #for kk in range(int(EW)-int(SW)+1): - # ii = int(int(kk) + 1) - # SB2=AA[ii-1,0] - # EB2=AA[ii-1,1] - # call_str = 'echo ' + workDir + '/'+ Sdate+ '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+str(int(SW)+kk)+ '.rslc' + ' ' + workDir + '/' + Sdate + '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+ str(int(SW)+kk)+ '.rslc.par' + ' ' + workDir + '/'+ Sdate+'_'+ str(int(SB2)) + str(int(EB2)) + '.IW'+str(int(SW)+kk)+ '.rslc.TOPS_par >>' + RSLC_tab - # os.system(call_str) - - os.chdir(workDir) - #TEST = workDir + '/' + Sdate +'_' + rlks + 'rlks.amp.par' - TEST = workDir + '/' + Sdate +'.rslc.par' - - k0 = 0 - if os.path.isfile(TEST): - if os.path.getsize(TEST) > 0: - k0 = 1 - - if k0==0: - if not Mdate ==Sdate: - call_str = 'S1_coreg_TOPS ' + SLC1_INF_tab + ' ' + Mdate + ' ' + SLC2_INF_tab + ' ' + Sdate + ' ' + RSLC_tab + ' ' + HGTSIM + ' ' + rlks + ' ' + azlks + ' - - 0.8 0.01 1.2 1' - os.system(call_str) - - #### clean large file #### - mslc = workDir + '/' + Mdate + '.slc' - mrslc = workDir + '/' + Mdate + '.rslc' - sslc = workDir + '/' + Sdate + '.slc' - srslc = workDir + '/' + Sdate + '.rslc' - srslcPar = workDir + '/' + Sdate + '.rslc.par' - - call_str = 'multi_look ' + srslc + ' ' + srslcPar + ' ' + Samp + ' ' + SampPar + ' ' + rlks + ' ' + azlks - os.system(call_str) - nWIDTH = ut.read_gamma_par(SampPar,'read', 'range_samples') - if os.path.isfile(mslc): os.remove(mslc) - if os.path.isfile(mrslc): os.remove(mrslc) - if os.path.isfile(sslc): os.remove(sslc) - - call_str = 'raspwr ' + Samp + ' ' + nWIDTH - os.system(call_str) - - #call_str = 'rm *mli*' - #os.system(call_str) - - #call_str = 'rm *IW*' - #os.system(call_str) - - #call_str = 'rm *off*' - #os.system(call_str) - - #call_str = 'rm *diff' - #os.system(call_str) - - #call_str = 'rm *diff_par*' - #os.system(call_str) - - #call_str = 'rm ' + Mdate + '.*' - #os.system(call_str) - - else: - call_str = 'cp ' + Mslc + ' ' + workDir + '/' + Mdate + '.rslc' - os.system(call_str) - - call_str = 'cp ' + Mslcpar + ' ' + workDir+ '/' + Mdate + '.rslc.par' - os.system(call_str) - - call_str = 'cp ' + Mamp + ' ' + workDir+ '/' + Mdate + '_' + rlks + 'rlks.amp' - os.system(call_str) - - call_str = 'cp ' + MampPar + ' ' + workDir+ '/' + Mdate + '_' + rlks + 'rlks.amp.par' - os.system(call_str) - - ut.copy_file(Mampbmp,Sampbmp) - - ################ clean redundant files ############# - - #if not Mdate ==Sdate: - - # if not templateDict['diff_all_parallel'] == '1': - # for i in range(len(S_IW)): - # if os.path.isfile(S_IW[i]): - # os.remove(S_IW[i]) - # if os.path.isfile(Mslc): os.remove(Mslc) - # if os.path.isfile(Mslcpar): os.remove(Mslcpar) - # if os.path.isfile(Mamp): os.remove(Mamp) - # if os.path.isfile(HGTSIM): os.remove(HGTSIM) - - print("Coregister TOP SLC image to the reference TOPS image is done !!") - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_s1_gamma_pot.py b/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_s1_gamma_pot.py deleted file mode 100644 index 712cf57..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_s1_gamma_pot.py +++ /dev/null @@ -1,370 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# -import numpy as np -import os -import sys -import argparse -import glob - -from pyint import _utils as ut - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Coregister TOPS S1-SLC to a reference S1-SLC using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName', help='Name of project.') - parser.add_argument('sdate', help='date of the slave S1 image. [mater date is read from template]') - inps = parser.parse_args() - return inps - - -INTRODUCTION = ''' -------------------------------------------------------------------- - Coregister TOPS S1-SLC to a reference S1-SLC using GAMMA. - [The reference date or master date will be read from the template file.] -''' - -EXAMPLE = """Usage: - - coreg_s1_gamma.py projectName Sdate - - coreg_s1_gamma.py PacayaT163TsxHhA 20150102 -------------------------------------------------------------------- -""" - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - Sdate = inps.sdate - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - - slcDir = scratchDir + '/' + projectName + "/SLC" - rslcDir = scratchDir + '/' + projectName + "/RSLC" - if not os.path.isdir(rslcDir): os.mkdir(rslcDir) - #workDir = processDir + '/' + igramDir - workDir = rslcDir + '/' + Sdate - if not os.path.isdir(workDir): os.mkdir(workDir) - - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - Mdate = templateDict['masterDate'] - deramp = templateDict['deramp'] - - if 'boi' in templateDict: - boi = templateDict['boi'] - else: - boi = '0' - - demDir = scratchDir + '/' + projectName + '/DEM' - -# Definition of file - MslcDir = slcDir + '/' + Mdate - - SslcDir = slcDir + '/' + Sdate - Samp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' - SampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' - Sampbmp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.bmp' - - Mslc = slcDir + '/' + Mdate + '/' + Mdate + '.slc' - Mslcpar = slcDir + '/' + Mdate + '/' + Mdate + '.slc.par' - Mamp = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' - MampPar = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' - Mampbmp = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.bmp' - - #SLC1_INF_tab = MslcDir + '/' + Mdate + '_SLC_Tab' - SLC2_INF_tab = SslcDir + '/' + Sdate + '_SLC_Tab' - RSLC_tab = SslcDir + '/' + Sdate + '_RSLC_Tab' - RSLC_tab2 = workDir + '/' + Sdate + '_RSLC_Tab' - call_str = 'cp ' + RSLC_tab + ' ' + RSLC_tab2 - os.system(call_str) - - #SLC1_INF_tab = SslcDir + '/' + Mdate + '_SLC_Tab_coreg' - - HGTSIM = demDir + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' - if not os.path.isfile(HGTSIM): - call_str = 'generate_rdc_dem.py ' + projectName - os.system(call_str) - - srslc_fboi0 = workDir + '/' + Sdate + '_overlap.fwd.slc'; srslc_fboi = srslc_fboi0.replace('.slc','.rslc') - srslc_fboi_par0 = workDir + '/' + Sdate + '_overlap.fwd.slc.par'; srslc_fboi_par = srslc_fboi_par0.replace('.slc','.rslc') - - srslc_bboi0 = workDir + '/' + Sdate + '_overlap.bwd.slc'; srslc_bboi = srslc_bboi0.replace('.slc','.rslc') - srslc_bboi_par0 = workDir + '/' + Sdate + '_overlap.bwd.slc.par'; srslc_bboi_par = srslc_bboi_par0.replace('.slc','.rslc') - - samp_fboi = workDir + '/' + Sdate + '_overlap.fwd_' + rlks + 'rlks.amp' - samp_fboi_par = workDir + '/' + Sdate + '_overlap.fwd_' + rlks + 'rlks.amp.par' - - samp_bboi = workDir + '/' + Sdate + '_overlap.bwd_' + rlks + 'rlks.amp' - samp_bboi_par = workDir + '/' + Sdate + '_overlap.bwd_' + rlks + 'rlks.amp.par' - ############## copy master files into slave folder for parallel process ########### - - #if not templateDict['coreg_all_parallel'] == '1': - # Mslc = slcDir + '/' + Sdate + '/' + Mdate + '.slc' - # Mslcpar = slcDir + '/' + Sdate + '/' + Mdate + '.slc.par' - # Mamp = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp' - # MampPar = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' - # HGTSIM = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' - - #else: - - # Mslc = Mslc0 - # Mslcpar = Mslcpar0 - # Mamp = Mamp0 - # MampPar = MampPar0 - # HGTSIM = HGTSIM0 - - - #Mslc = slcDir + '/' + Sdate + '/' + Mdate + '.slc' - #Mslcpar = slcDir + '/' + Sdate + '/' + Mdate + '.slc.par' - #Mamp = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp' - #MampPar = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' - #HGTSIM = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' - #SLC1_INF_tab1 = SslcDir + '/' + Mdate + '_SLC_Tab' - #ut.copy_file(SLC1_INF_tab0,SLC1_INF_tab1) - ############################################################################## - #with open(SLC1_INF_tab1, "r") as f: - # lines = f.readlines() - - #with open(SLC1_INF_tab, "w") as fw: - # lines_coreg = [] - # for k0 in lines: - # k00 = k0.replace(MslcDir,SslcDir) - # lines_coreg.append(k00) - # fw.write(k00) - - #S_IW = ut.read_txt2array(SLC1_INF_tab1) - #S_IW = S_IW.flatten() - #M_IW = ut.read_txt2array(SLC1_INF_tab1) - #M_IW = M_IW.flatten()TSLC = slc_dir + '/' + date + '.slc' - #TSLCPar = slc_dir + '/' + date + '.slc.par' - - # - #S_IW = ut.read_txt2array(SLC1_INF_tab) - #S_IW = S_IW.flatten() - - #RSLC_tab = workDir + '/' + Sdate + '_RSLC_tab' - #if os.path.isfile(RSLC_tab): - # os.remove(RSLC_tab) - - #BURST = SslcDir + '/' + Sdate + '_Burst_Tab' - #AA = np.loadtxt(BURST) - #if EW==SW: - # AA = AA.reshape([1,2]) - # - #for kk in range(int(EW)-int(SW)+1): - # ii = int(int(kk) + 1) - # SB2=AA[ii-1,0] - # EB2=AA[ii-1,1] - # call_str = 'echo ' + workDir + '/'+ Sdate+ '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+str(int(SW)+kk)+ '.rslc' + ' ' + workDir + '/' + Sdate + '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+ str(int(SW)+kk)+ '.rslc.par' + ' ' + workDir + '/'+ Sdate+'_'+ str(int(SB2)) + str(int(EB2)) + '.IW'+str(int(SW)+kk)+ '.rslc.TOPS_par >>' + RSLC_tab - # os.system(call_str) - - os.chdir(workDir) - #TEST = workDir + '/' + Sdate +'_' + rlks + 'rlks.amp.par' - TEST = workDir + '/' + Mdate + '_' + Sdate +'.diff.bmp' - - k0 = 0 - if os.path.isfile(TEST): - if os.path.getsize(TEST) > 0: - k0 = 1 - - if k0==0: - if not Mdate ==Sdate: - - call_str = 'cp -rf ' + MslcDir + ' ' + workDir - os.system(call_str) - - MslcDir2 = workDir + '/' + Mdate - - SLC_list = glob.glob(MslcDir2 + '/*IW*.slc') - SLC_par_list = glob.glob(MslcDir2 + '/*IW*.slc.par') - TOP_par_list = glob.glob(MslcDir2 + '/*IW*.slc.TOPS_par') - - SLC1_INF_tab = MslcDir2 + '/' + Mdate + '_SLC_Tab' - - if os.path.isfile(SLC1_INF_tab): - os.remove(SLC1_INF_tab) - - SLC_list = sorted(SLC_list) - SLC_par_list = sorted(SLC_par_list) - TOP_par_list = sorted(TOP_par_list) - - for kk in range(len(SLC_list)): - call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' >> ' + SLC1_INF_tab - os.system(call_str) - - - #call_str = 'S1_coreg_TOPS ' + SLC1_INF_tab + ' ' + Mdate + ' ' + SLC2_INF_tab + ' ' + Sdate + ' ' + RSLC_tab + ' ' + HGTSIM + ' ' + rlks + ' ' + azlks + ' - - 0.8 0.01 1.2 1' - - call_str = 'ScanSAR_coreg.py ' + SLC1_INF_tab + ' ' + Mdate + ' ' + SLC2_INF_tab + ' ' + Sdate + ' ' + RSLC_tab + ' ' + HGTSIM + ' ' + rlks + ' ' + azlks + ' --cc 0.8 --fraction 0.01 --ph_stdev 0.8 --num_ovr 0 --no_check ' - print(call_str) - os.system(call_str) - - #### clean large file #### - mslc = workDir + '/' + Mdate + '.slc' - mrslc = workDir + '/' + Mdate + '.rslc' - sslc = workDir + '/' + Sdate + '.slc' - srslc = workDir + '/' + Sdate + '.rslc' - - srslcPar = workDir + '/' + Sdate + '.rslc.par' - - if deramp=='1': - call_str = 'S1_deramp_TOPS_slave ' + RSLC_tab2 + ' ' + Sdate + ' ' + SLC1_INF_tab + ' 10 2 1' - os.system(call_str) - - call_str = 'mv ' + srslc + '.deramp' + ' ' + srslc - os.system(call_str) - - call_str = 'mv ' + srslc + '.deramp.par' + ' ' + srslcPar - os.system(call_str) - - call_str = 'multi_look ' + srslc + ' ' + srslcPar + ' ' + Samp + ' ' + SampPar + ' ' + rlks + ' ' + azlks - os.system(call_str) - - nWIDTH = ut.read_gamma_par(SampPar,'read', 'range_samples') - - if boi =='1': - call_str = 'ScanSAR_burst_overlap ' + RSLC_tab2 + ' ' + Sdate+'_overlap' + ' ' + rlks + ' ' + azlks + ' - - ' + SLC1_INF_tab + ' - -' - os.system(call_str) - - call_str = 'mv ' + srslc_fboi0 + ' ' + srslc_fboi; os.system(call_str) - call_str = 'mv ' + srslc_fboi_par0 + ' ' + srslc_fboi_par; os.system(call_str) - call_str = 'mv ' + srslc_bboi0 + ' ' + srslc_bboi; os.system(call_str) - call_str = 'mv ' + srslc_bboi_par0 + ' ' + srslc_bboi_par; os.system(call_str) - - call_str = 'multi_look ' + srslc_fboi + ' ' + srslc_fboi_par + ' ' + samp_fboi + ' ' + samp_fboi_par + ' ' + rlks + ' ' + azlks - os.system(call_str) - - call_str = 'multi_look ' + srslc_bboi + ' ' + srslc_bboi_par + ' ' + samp_bboi + ' ' + samp_bboi_par + ' ' + rlks + ' ' + azlks - os.system(call_str) - - call_str = 'raspwr ' + samp_fboi + ' ' + nWIDTH - os.system(call_str) - - call_str = 'raspwr ' + samp_bboi + ' ' + nWIDTH - os.system(call_str) - - if os.path.isfile(mslc): os.remove(mslc) - if os.path.isfile(mrslc): os.remove(mrslc) - if os.path.isfile(sslc): os.remove(sslc) - - call_str = 'raspwr ' + Samp + ' ' + nWIDTH - os.system(call_str) - - call_str = 'rm -rf ' + MslcDir2 - os.system(call_str) - - #call_str = 'rm *mli*' - #os.system(call_str) - - #call_str = 'rm *IW*' - #os.system(call_str) - - #call_str = 'rm *off*' - #os.system(call_str) - - #call_str = 'rm *diff' - #os.system(call_str) - - #call_str = 'rm *diff_par*' - #os.system(call_str) - - #call_str = 'rm ' + Mdate + '.*' - - #os.system(call_str) - - else: - - #generate amp file for check image quality - TSLC = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc' - TSLCPar = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc.par' - - TMLI = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' - TMLIPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' - - if not os.path.isfile(TMLIPar): - A1 = np.loadtxt(SLC2_INF_tab,dtype='str'); A1 = A1.flatten() - A2 = np.loadtxt(RSLC_tab,dtype='str'); A2 = A2.flatten() - - nn = len(A1) - for i in range(nn): - call_str = 'cp ' + A1[i] + ' ' + A2[i] - os.system(call_str) - - if deramp=='1': - call_str = 'S1_deramp_TOPS_reference ' + RSLC_tab2 - os.system(call_str) - - SLC2_INF_tab = RSLC_tab2 + '.deramp' - else: - SLC2_INF_tab = RSLC_tab2 - - call_str = 'SLC_mosaic_S1_TOPS ' + SLC2_INF_tab + ' ' + TSLC + ' ' + TSLCPar + ' ' + rlks + ' ' + azlks - os.system(call_str) - - call_str = 'multi_look ' + TSLC + ' ' + TSLCPar + ' ' + TMLI + ' ' + TMLIPar + ' ' + rlks + ' ' + azlks - os.system(call_str) - - nWidth = ut.read_gamma_par(TMLIPar, 'read','range_samples:') - - if boi =='1': - call_str = 'ScanSAR_burst_overlap ' + RSLC_tab2 + ' ' + Sdate+'_overlap' + ' ' + rlks + ' ' + azlks + ' - - ' + SLC2_INF_tab + ' - -' - os.system(call_str) - - call_str = 'mv ' + srslc_fboi0 + ' ' + srslc_fboi; os.system(call_str) - call_str = 'mv ' + srslc_fboi_par0 + ' ' + srslc_fboi_par; os.system(call_str) - call_str = 'mv ' + srslc_bboi0 + ' ' + srslc_bboi; os.system(call_str) - call_str = 'mv ' + srslc_bboi_par0 + ' ' + srslc_bboi_par; os.system(call_str) - - call_str = 'multi_look ' + srslc_fboi + ' ' + srslc_fboi_par + ' ' + samp_fboi + ' ' + samp_fboi_par + ' ' + rlks + ' ' + azlks - os.system(call_str) - - call_str = 'multi_look ' + srslc_bboi + ' ' + srslc_bboi_par + ' ' + samp_bboi + ' ' + samp_bboi_par + ' ' + rlks + ' ' + azlks - os.system(call_str) - - call_str = 'raspwr ' + samp_fboi + ' ' + nWidth - os.system(call_str) - - call_str = 'raspwr ' + samp_bboi + ' ' + nWidth - os.system(call_str) - - - call_str = 'raspwr ' + TMLI + ' ' + nWidth + ' - - - - - - - ' - os.system(call_str) - rr = glob.glob(workDir + '/*IW*.slc') - if len(rr) > 0: - call_str = 'rm ' + workDir + '/*IW*.slc' - os.system(call_str) - uu = glob.glob(workDir + '/*IW*.rslc') - - if len(uu) > 0: - call_str = 'rm ' + workDir + '/*IW*.rslc' - os.system(call_str) - ################ clean redundant files ############# - - #if not Mdate ==Sdate: - - # if not templateDict['diff_all_parallel'] == '1': - # for i in range(len(S_IW)): - # if os.path.isfile(S_IW[i]): - # os.remove(S_IW[i]) - # if os.path.isfile(Mslc): os.remove(Mslc) - # if os.path.isfile(Mslcpar): os.remove(Mslcpar) - # if os.path.isfile(Mamp): os.remove(Mamp) - # if os.path.isfile(HGTSIM): os.remove(HGTSIM) - - print("Coregister TOP SLC image to the reference TOPS image is done !!") - #sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/corners.txt b/.codex_tmp/pyint_variants/no_rescue/pyint/corners.txt deleted file mode 100644 index cb93de4..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/corners.txt +++ /dev/null @@ -1,22 +0,0 @@ -*** Calculate SLC/MLI image corners in geodetic latitude and longitude (deg.) *** -*** Copyright 2022, Gamma Remote Sensing, v2.2 8-Nov-2022 clw/awi/cm *** -latitude (deg.): 24.74699954 longitude (deg.): 103.30166172 -latitude (deg.): 25.16531434 longitude (deg.): 105.78287221 -latitude (deg.): 26.37363426 longitude (deg.): 102.95054159 -latitude (deg.): 26.78897220 longitude (deg.): 105.46762233 - -center latitude (deg.): 25.79112685 center longitude (deg.): 104.47614292 -min. latitude (deg.): 24.74699954 max. latitude (deg.): 26.78897220 -min. longitude (deg.): 102.95054159 max. longitude (deg.): 105.78287221 -delta latitude (deg.): 2.04197267 delta longitude (deg.): 2.83233062 - -upper left corner latitude, longitude (deg.): 26.83 102.91 -lower right corner latitude, longitude (deg.): 24.71 105.82 - -lower left corner longitude, latitude (deg.): 102.91 24.71 -upper right corner longitude, latitude (deg.): 105.82 26.83 - -user time (s): 0.000 -system time (s): 0.000 -elapsed time (s): 0.000 - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/correct_ifg_for_hpy3_from_murp.py b/.codex_tmp/pyint_variants/no_rescue/pyint/correct_ifg_for_hpy3_from_murp.py deleted file mode 100644 index 2ff3b68..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/correct_ifg_for_hpy3_from_murp.py +++ /dev/null @@ -1,769 +0,0 @@ -#!/usr/bin/env python3 -""" -修复版MuRP算法 - 包含Linux系统中文显示支持 -""" - -import xarray as xr -import rioxarray -import os -import numpy as np -import matplotlib.pyplot as plt -from glob import glob -import argparse -import sys -from datetime import datetime -import rasterio -import warnings -import matplotlib.font_manager as fm -import platform - -# 设置中文字体支持 -def setup_chinese_font(): - """配置中文字体支持""" - system = platform.system() - - if system == "Linux": - # Linux系统字体配置 - chinese_fonts = [ - 'WenQuanYi Micro Hei', # 文泉驿微米黑 - 'Noto Sans CJK SC', # Google思源黑体 - 'DejaVu Sans', # 备选字体 - 'Arial' # 最后备选 - ] - - # 查找可用的字体 - available_fonts = [] - for font in chinese_fonts: - if any(f.name == font for f in fm.fontManager.ttflist): - available_fonts.append(font) - - if available_fonts: - plt.rcParams['font.sans-serif'] = available_fonts - print(f"已设置中文字体: {available_fonts[0]}") - else: - plt.rcParams['font.sans-serif'] = ['DejaVu Sans'] - print("警告: 未找到中文字体，使用默认字体") - else: - # Windows或macOS - plt.rcParams['font.sans-serif'] = ['SimHei', 'Microsoft YaHei', 'Arial'] - - # 解决负号显示问题 - plt.rcParams['axes.unicode_minus'] = False - -# 调用字体设置函数 -setup_chinese_font() -warnings.filterwarnings('ignore') - -class FixedMuRP: - """ - 修复版MuRP算法 - 解决相干性数据形状不一致问题 - """ - - def __init__(self, random_seed=42): - """初始化MuRP处理器""" - self.random_seed = random_seed - np.random.seed(random_seed) - self.ds = None - self.ds_corrected = None - self.refs = [] - self.fits = [] - self.crs = None - self.transform = None - self.bounds = None - self.width = None - self.height = None - - def load_hyp3_data_robust(self, data_path): - """ - 健壮的数据加载函数，处理形状不一致问题 - """ - print(f"正在加载Hyp3数据从: {data_path}") - - # 查找数据文件 - unw_files = glob(f'{data_path}/*/*unw_phase*.tif') - corr_files = glob(f'{data_path}/*/*corr*.tif') - dem_files = glob(f'{data_path}/*/*dem*.tif') - - if not unw_files: - raise FileNotFoundError(f"在 {data_path} 中未找到解缠相位文件") - if not dem_files: - raise FileNotFoundError(f"在 {data_path} 中未找到DEM文件") - - print(f"找到解缠相位文件: {len(unw_files)} 个") - print(f"找到相干性文件: {len(corr_files)} 个") - print(f"找到DEM文件: {len(dem_files)} 个") - - # 使用第一个解缠相位文件作为参考 - reference_file = unw_files[0] - with rasterio.open(reference_file) as src: - self.crs = src.crs - self.transform = src.transform - self.width = src.width - self.height = src.height - self.bounds = src.bounds - - print(f"坐标参考系统: {self.crs}") - print(f"图像尺寸: {self.width} x {self.height}") - - # 读取DEM数据 - 使用第一个DEM文件 - print("正在加载DEM数据...") - with rasterio.open(dem_files[0]) as src: - # 确保DEM尺寸与参考文件一致 - if src.width == self.width and src.height == self.height: - elevation_data = src.read(1) - else: - print("警告: DEM尺寸不匹配，将使用零数组") - elevation_data = np.zeros((self.height, self.width)) - - # 读取解缠相位数据 - print("正在加载解缠相位数据...") - unw_phase_data = [] - granule_names = [] - - for unw_file in unw_files: - with rasterio.open(unw_file) as src: - # 检查尺寸是否匹配 - if src.width == self.width and src.height == self.height: - data = src.read(1) - else: - print(f"警告: {unw_file} 尺寸不匹配，跳过") - continue - - unw_phase_data.append(data) - granule_name = os.path.basename(unw_file).replace('.tif', '') - granule_names.append(granule_name) - print(f" 已加载: {granule_name}") - - if not unw_phase_data: - raise ValueError("没有成功加载任何解缠相位数据") - - # 转换为numpy数组 - unw_phase_array = np.array(unw_phase_data) - print(f"解缠相位数据形状: {unw_phase_array.shape}") - - # 处理相干性数据 - 确保与解缠相位数据对应 - print("正在处理相干性数据...") - coherence_data = [] - used_corr_files = [] - - # 为每个解缠相位文件寻找匹配的相干性文件 - for i, granule_name in enumerate(granule_names): - # 从granule名称提取基本名称 - base_name = granule_name.replace('_unw_phase', '') - - # 查找匹配的相干性文件 - matching_corr_files = [f for f in corr_files if base_name in f and f not in used_corr_files] - - if matching_corr_files: - # 使用第一个匹配的文件 - corr_file = matching_corr_files[0] - try: - with rasterio.open(corr_file) as src: - # 检查尺寸是否匹配 - if src.width == self.width and src.height == self.height: - corr_data = src.read(1) - coherence_data.append(corr_data) - used_corr_files.append(corr_file) - print(f" 已加载相干性: {os.path.basename(corr_file)}") - else: - print(f" 警告: 相干性文件尺寸不匹配，使用默认值") - coherence_data.append(np.ones((self.height, self.width))) - except Exception as e: - print(f" 加载相干性文件失败: {e}") - coherence_data.append(np.ones((self.height, self.width))) - else: - # 没有找到匹配的相干性文件，使用默认值 - print(f" 未找到匹配的相干性文件，使用默认值") - coherence_data.append(np.ones((self.height, self.width))) - - # 如果相干性数据数量与解缠相位不匹配，补充默认值 - while len(coherence_data) < len(unw_phase_data): - coherence_data.append(np.ones((self.height, self.width))) - - # 转换为numpy数组 - coherence_array = np.array(coherence_data) - print(f"相干性数据形状: {coherence_array.shape}") - - # 创建xarray数据集 - ds = xr.Dataset( - { - 'unw_phase': (['granule', 'y', 'x'], unw_phase_array), - 'coherence': (['granule', 'y', 'x'], coherence_array), - 'elevation': (['y', 'x'], elevation_data) - }, - coords={ - 'granule': granule_names, - 'y': np.arange(self.height), - 'x': np.arange(self.width) - } - ) - - # 设置地理参考属性 - try: - ds.rio.set_crs(self.crs) - ds.rio.write_transform(self.transform, inplace=True) - except: - print("警告: 无法设置地理参考信息") - - print(f"成功加载数据: {len(granule_names)} 个干涉图") - self.ds = ds - return ds - - def select_reference_points(self, corr_thresh=0.6, n_refs=1000): - """ - 选择参考点 - """ - if self.ds is None: - raise ValueError("请先加载数据") - - print(f"步骤 1: 选择参考点 (相干性阈值: {corr_thresh}, 数量: {n_refs})") - - # 计算平均相干性 - coh_mean = self.ds.coherence.mean(dim='granule').values - - # 获取高相干性像素 - valid_mask = coh_mean >= corr_thresh - valid_coords = np.argwhere(valid_mask) - - if len(valid_coords) == 0: - print(f"警告: 没有找到相干性大于{corr_thresh}的像素，降低阈值到0.3") - valid_mask = coh_mean >= 0.3 - valid_coords = np.argwhere(valid_mask) - - if len(valid_coords) == 0: - print("警告: 仍然没有找到高相干性像素，使用所有像素") - valid_coords = np.array([[i, j] for i in range(self.height) for j in range(self.width)]) - - print(f" 找到 {len(valid_coords)} 个候选像素点") - - # 调整参考点数量 - if len(valid_coords) < n_refs: - n_refs = len(valid_coords) - print(f" 可用点不足，调整为选择 {n_refs} 个参考点") - - # 使用网格策略选择参考点 - grid_size = int(np.sqrt(n_refs / 4)) - grid_size = max(5, min(grid_size, 50)) - - x_bins = np.linspace(0, self.width, grid_size + 1, dtype=int) - y_bins = np.linspace(0, self.height, grid_size + 1, dtype=int) - - ref_list = [] - points_per_cell = max(1, n_refs // (grid_size * grid_size)) - - for i in range(grid_size): - for j in range(grid_size): - x_min, x_max = x_bins[i], x_bins[i+1] - y_min, y_max = y_bins[j], y_bins[j+1] - - # 在当前网格内选择点 - cell_mask = ((valid_coords[:, 1] >= x_min) & (valid_coords[:, 1] < x_max) & - (valid_coords[:, 0] >= y_min) & (valid_coords[:, 0] < y_max)) - cell_points = valid_coords[cell_mask] - - if len(cell_points) > 0: - n_select = min(points_per_cell, len(cell_points)) - selected_indices = np.random.choice(len(cell_points), n_select, replace=False) - - for idx in selected_indices: - y, x = cell_points[idx] - ref_list.append([int(x), int(y)]) - - # 如果点数不足，随机补充 - if len(ref_list) < n_refs: - remaining = n_refs - len(ref_list) - additional_indices = np.random.choice(len(valid_coords), remaining, replace=False) - for idx in additional_indices: - y, x = valid_coords[idx] - if [int(x), int(y)] not in ref_list: - ref_list.append([int(x), int(y)]) - - self.refs = ref_list[:n_refs] - print(f" 成功选择 {len(self.refs)} 个参考点") - return self.refs - - def sample_reference_data(self): - """采样参考点数据""" - if not self.refs: - raise ValueError("请先选择参考点") - - print("步骤 2: 采样参考点数据") - - # 验证参考点坐标 - valid_refs = [] - for ref in self.refs: - x, y = ref[0], ref[1] - if 0 <= x < self.width and 0 <= y < self.height: - valid_refs.append(ref) - else: - print(f"警告: 参考点 ({x}, {y}) 超出图像范围") - - if not valid_refs: - raise ValueError("没有有效的参考点") - - # 提取坐标 - x_coords = [ref[0] for ref in valid_refs] - y_coords = [ref[1] for ref in valid_refs] - - # 采样高程数据 - ref_elevation = [] - for x, y in valid_refs: - elev_val = self.ds.elevation.isel(x=x, y=y).values - ref_elevation.append(float(elev_val)) - ref_elevation = np.array(ref_elevation) - - # 采样相位数据 - ref_values = [] - for i in range(len(self.ds.granule)): - granule_phases = [] - for x, y in valid_refs: - phase_val = self.ds.unw_phase.isel(granule=i, x=x, y=y).values - granule_phases.append(float(phase_val)) - ref_values.append(granule_phases) - ref_values = np.array(ref_values) - - print(f" 成功采样 {len(valid_refs)} 个参考点，{len(ref_values)} 个干涉图") - return ref_values, ref_elevation - - def numpy_linear_regression(self, x, y): - """ - 使用NumPy实现线性回归 - """ - # 移除NaN值 - valid_mask = ~(np.isnan(x) | np.isnan(y)) - x_valid = x[valid_mask] - y_valid = y[valid_mask] - - if len(x_valid) < 2: - return np.nan, np.nan, 0, 0 - - # 计算线性回归参数 - x_mean = np.mean(x_valid) - y_mean = np.mean(y_valid) - - # 计算协方差和方差 - cov_xy = np.mean((x_valid - x_mean) * (y_valid - y_mean)) - var_x = np.mean((x_valid - x_mean) ** 2) - - if var_x == 0: - return np.nan, np.nan, 0, 0 - - beta = cov_xy / var_x - alpha = y_mean - beta * x_mean - - # 计算R² - y_pred = alpha + beta * x_valid - ss_res = np.sum((y_valid - y_pred) ** 2) - ss_tot = np.sum((y_valid - y_mean) ** 2) - - if ss_tot == 0: - r_squared = 0 - else: - r_squared = 1 - (ss_res / ss_tot) - - n_points = len(x_valid) - - return beta, alpha, r_squared, n_points - - def perform_linear_fits(self, ref_values, ref_elevation): - """ - 执行线性拟合 - """ - print("步骤 3: 线性拟合") - - fits = [] - fit_metrics = [] - elevations = np.array(ref_elevation) - - for i in range(len(ref_values)): - phases = ref_values[i] - - # 执行线性回归 - slope, intercept, r2, n_points = self.numpy_linear_regression(elevations, phases) - - fits.append([slope, intercept]) - fit_metrics.append({ - 'r_squared': r2, - 'n_points': n_points - }) - - if (i + 1) % 10 == 0 or (i + 1) == len(ref_values): - print(f" 已完成 {i+1}/{len(ref_values)} 个干涉图") - - # 统计结果 - valid_fits = sum(1 for fit in fits if not np.isnan(fit[0])) - valid_r2 = [m['r_squared'] for m in fit_metrics if not np.isnan(m['r_squared'])] - avg_r2 = np.mean(valid_r2) if valid_r2 else 0 - - self.fits = fits - self.fit_metrics = fit_metrics - - print(f" 线性拟合完成: {valid_fits}/{len(fits)} 个成功, 平均R²: {avg_r2:.3f}") - return fits, fit_metrics - - def apply_correction(self, min_r2=0.0): - """应用相位校正""" - print("步骤 4: 应用相位校正") - - if not self.fits: - raise ValueError("请先进行线性拟合") - - # 过滤低质量拟合 - valid_fits = [] - valid_indices = [] - - for i, fit in enumerate(self.fits): - if np.isnan(fit[0]) or np.isnan(fit[1]): - continue - - if self.fit_metrics[i]['r_squared'] < min_r2: - continue - - valid_fits.append(fit) - valid_indices.append(i) - - if not valid_fits: - print("警告: 没有通过质量控制的拟合，使用所有有效拟合") - valid_fits = [f for f in self.fits if not np.isnan(f[0]) and not np.isnan(f[1])] - valid_indices = [i for i, f in enumerate(self.fits) - if not np.isnan(f[0]) and not np.isnan(f[1])] - - if not valid_fits: - raise ValueError("没有有效的拟合可用于校正") - - print(f" 使用 {len(valid_fits)}/{len(self.fits)} 个拟合进行校正") - - # 创建校正后的数据集 - self.ds_corrected = self.ds.copy() - - # 创建校正后的相位数组 - corrected_phase = np.zeros_like(self.ds.unw_phase.values) - - # 对每个干涉图应用校正 - for idx, granule_idx in enumerate(valid_indices): - slope, intercept = valid_fits[idx] - - # 计算校正量: phase = slope * elevation + intercept - elevation_data = self.ds.elevation.values - correction = elevation_data * slope + intercept - - # 应用校正: corrected_phase = original_phase - correction - original_phase = self.ds.unw_phase[granule_idx].values - corrected_phase[granule_idx] = original_phase - correction - - # 添加校正后的变量 - self.ds_corrected['unw_phase_corrected'] = (('granule', 'y', 'x'), corrected_phase) - - return self.ds_corrected - - def save_results_as_geotiff(self, output_dir='.'): - """保存结果为GeoTIFF格式""" - if self.ds_corrected is None: - raise ValueError("请先进行校正") - - print("步骤 5: 保存GeoTIFF格式结果") - os.makedirs(output_dir, exist_ok=True) - - # 保存校正后的每个干涉图 - corrected_dir = os.path.join(output_dir, "corrected_phase") - os.makedirs(corrected_dir, exist_ok=True) - - for i, granule in enumerate(self.ds_corrected.granule.values): - # 获取校正后的相位数据 - phase_corrected = self.ds_corrected.unw_phase_corrected[i].values - - # 创建输出文件名 - output_file = os.path.join(corrected_dir, f"{granule}_corrected.tif") - - # 使用rasterio保存为GeoTIFF - try: - with rasterio.open( - output_file, 'w', - driver='GTiff', - height=self.height, - width=self.width, - count=1, - dtype=phase_corrected.dtype, - crs=self.crs, - transform=self.transform - ) as dst: - dst.write(phase_corrected, 1) - - print(f" 已保存: {output_file}") - except Exception as e: - print(f" 保存失败 {output_file}: {e}") - - # 保存平均校正后相位 - try: - mean_corrected = self.ds_corrected.unw_phase_corrected.mean(dim='granule').values - mean_file = os.path.join(output_dir, "mean_corrected_phase.tif") - - with rasterio.open( - mean_file, 'w', - driver='GTiff', - height=self.height, - width=self.width, - count=1, - dtype=mean_corrected.dtype, - crs=self.crs, - transform=self.transform - ) as dst: - dst.write(mean_corrected, 1) - - print(f" 已保存平均校正相位: {mean_file}") - except Exception as e: - print(f" 保存平均校正相位失败: {e}") - - # 保存原始平均相位用于对比 - try: - mean_original = self.ds.unw_phase.mean(dim='granule').values - mean_orig_file = os.path.join(output_dir, "mean_original_phase.tif") - - with rasterio.open( - mean_orig_file, 'w', - driver='GTiff', - height=self.height, - width=self.width, - count=1, - dtype=mean_original.dtype, - crs=self.crs, - transform=self.transform - ) as dst: - dst.write(mean_original, 1) - - print(f" 已保存原始平均相位: {mean_orig_file}") - except Exception as e: - print(f" 保存原始平均相位失败: {e}") - - # 保存高程数据 - try: - elev_file = os.path.join(output_dir, "elevation.tif") - elevation_data = self.ds.elevation.values - - with rasterio.open( - elev_file, 'w', - driver='GTiff', - height=self.height, - width=self.width, - count=1, - dtype=elevation_data.dtype, - crs=self.crs, - transform=self.transform - ) as dst: - dst.write(elevation_data, 1) - - print(f" 已保存高程数据: {elev_file}") - except Exception as e: - print(f" 保存高程数据失败: {e}") - - print(f"所有GeoTIFF文件已保存至: {output_dir}") - - def create_diagnostic_plots(self, ref_values, ref_elevation, output_dir='.'): - """创建诊断图表""" - print("步骤 6: 生成诊断图表") - os.makedirs(output_dir, exist_ok=True) - - # 创建综合诊断图 - fig, axes = plt.subplots(2, 3, figsize=(15, 10)) - - # 1. 参考点分布 - ax1 = axes[0, 0] - elevation_data = self.ds.elevation.values - im1 = ax1.imshow(elevation_data, cmap='terrain') - if self.refs: - x_coords, y_coords = zip(*self.refs) - ax1.scatter(x_coords, y_coords, c='red', s=5, alpha=0.7, label='Reference Points') - ax1.legend() - ax1.set_title('Reference Points Distribution') - plt.colorbar(im1, ax=ax1, label='Elevation (m)') - - # 2. 第一个干涉图的相位-高程关系 - ax2 = axes[0, 1] - if len(ref_values) > 0: - phases = ref_values[0] - valid_mask = ~(np.isnan(ref_elevation) | np.isnan(phases)) - - if np.sum(valid_mask) > 0: - ax2.scatter(ref_elevation[valid_mask], phases[valid_mask], - alpha=0.5, s=10) - - # 绘制拟合线 - if not np.isnan(self.fits[0][0]): - elev_min, elev_max = np.min(ref_elevation[valid_mask]), np.max(ref_elevation[valid_mask]) - elev_range = np.linspace(elev_min, elev_max, 100) - phase_fit = self.fits[0][0] * elev_range + self.fits[0][1] - ax2.plot(elev_range, phase_fit, 'r-', linewidth=2, - label=f'Slope: {self.fits[0][0]:.4f}') - ax2.legend() - - ax2.set_xlabel('Elevation (m)') - ax2.set_ylabel('Phase (rad)') - ax2.set_title('Phase vs Elevation') - ax2.grid(True, alpha=0.3) - - # 3. 拟合斜率分布 - ax3 = axes[0, 2] - slopes = [f[0] for f in self.fits if not np.isnan(f[0])] - if slopes: - ax3.hist(slopes, bins=20, alpha=0.7, density=True) - ax3.axvline(np.mean(slopes), color='r', linestyle='--', - label=f'Mean: {np.mean(slopes):.4f}') - ax3.legend() - ax3.set_xlabel('Slope') - ax3.set_ylabel('Density') - ax3.set_title('Slope Distribution') - ax3.grid(True, alpha=0.3) - - # 4. 校正前后对比 - ax4 = axes[1, 0] - phase_before = self.ds.unw_phase.mean(dim='granule').values - vmin, vmax = -np.pi, np.pi - im4 = ax4.imshow(phase_before, cmap='RdBu', vmin=vmin, vmax=vmax) - ax4.set_title('Mean Phase (Before Correction)') - plt.colorbar(im4, ax=ax4, label='Phase (rad)') - - ax5 = axes[1, 1] - phase_after = self.ds_corrected.unw_phase_corrected.mean(dim='granule').values - im5 = ax5.imshow(phase_after, cmap='RdBu', vmin=vmin, vmax=vmax) - ax5.set_title('Mean Phase (After Correction)') - plt.colorbar(im5, ax=ax5, label='Phase (rad)') - - # 5. 校正量 - ax6 = axes[1, 2] - correction = phase_before - phase_after - im6 = ax6.imshow(correction, cmap='viridis') - ax6.set_title('Phase Correction') - plt.colorbar(im6, ax=ax6, label='Correction (rad)') - - plt.tight_layout() - plt.savefig(f'{output_dir}/MuRP_diagnostics.png', dpi=300, bbox_inches='tight') - plt.close() - - print(f" 诊断图已保存至: {output_dir}/MuRP_diagnostics.png") - - def calculate_improvement(self): - """计算改善统计""" - if self.ds_corrected is None: - return 0, 0, 0 - - # 计算时间序列标准差 - std_before = float(self.ds.unw_phase.std(dim='granule').mean().values) - std_after = float(self.ds_corrected.unw_phase_corrected.std(dim='granule').mean().values) - - improvement = (std_before - std_after) / std_before * 100 if std_before != 0 else 0 - - return improvement, std_before, std_after - - def run_murp_correction(self, data_path, corr_thresh=0.6, n_refs=1000, - output_dir='.', create_plots=True, save_geotiff=True): - """ - 运行完整的MuRP校正流程 - """ - start_time = datetime.now() - print("="*60) - print("修复版MuRP算法开始执行") - print(f"开始时间: {start_time.strftime('%Y-%m-%d %H:%M:%S')}") - print("="*60) - - try: - # 1. 加载数据 - self.load_hyp3_data_robust(data_path) - - # 2. 选择参考点 - self.select_reference_points(corr_thresh, n_refs) - - # 3. 采样参考点数据 - ref_values, ref_elevation = self.sample_reference_data() - - # 4. 线性拟合 - self.perform_linear_fits(ref_values, ref_elevation) - - # 5. 应用校正 - self.apply_correction() - - # 6. 生成诊断图表 - if create_plots: - self.create_diagnostic_plots(ref_values, ref_elevation, output_dir) - - # 7. 保存GeoTIFF格式结果 - if save_geotiff: - self.save_results_as_geotiff(output_dir) - - # 计算改善统计 - improvement, std_before, std_after = self.calculate_improvement() - - end_time = datetime.now() - duration = (end_time - start_time).total_seconds() - - print("="*60) - print("修复版MuRP算法执行完成") - print(f"结束时间: {end_time.strftime('%Y-%m-%d %H:%M:%S')}") - print(f"总耗时: {duration:.1f} 秒") - print(f"时间序列标准差改善: {improvement:.1f}%") - print(f" 校正前: {std_before:.4f} rad") - print(f" 校正后: {std_after:.4f} rad") - print("="*60) - - return self.ds_corrected - - except Exception as e: - print(f"算法执行失败: {e}") - import traceback - traceback.print_exc() - return None - -def main(): - """命令行接口主函数""" - parser = argparse.ArgumentParser( - description='修复版MuRP算法 - 解决相干性数据形状不一致问题', - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=''' -使用示例: - python fixed_murp.py /path/to/hyp3/data - python fixed_murp.py /path/to/hyp3/data --corr_thresh 0.6 --n_refs 500 - python fixed_murp.py /path/to/hyp3/data --output_dir ./results - ''' - ) - - parser.add_argument('data_path', help='Hyp3数据目录路径') - parser.add_argument('--corr_thresh', type=float, default=0.6, - help='相干性阈值 (默认: 0.6)') - parser.add_argument('--n_refs', type=int, default=1000, - help='参考点数量 (默认: 1000)') - parser.add_argument('--output_dir', default='.', - help='输出目录 (默认: 当前目录)') - parser.add_argument('--no_plots', action='store_true', - help='不生成诊断图表') - parser.add_argument('--no_geotiff', action='store_true', - help='不保存GeoTIFF格式') - - args = parser.parse_args() - - try: - # 创建MuRP处理器 - murp = FixedMuRP(random_seed=42) - - # 运行算法 - ds_corrected = murp.run_murp_correction( - data_path=args.data_path, - corr_thresh=args.corr_thresh, - n_refs=args.n_refs, - output_dir=args.output_dir, - create_plots=not args.no_plots, - save_geotiff=not args.no_geotiff - ) - - if ds_corrected is not None: - print("算法执行成功!") - else: - print("算法执行失败!") - sys.exit(1) - - except Exception as e: - print(f"错误: {e}") - import traceback - traceback.print_exc() - sys.exit(1) - -if __name__ == "__main__": - main() diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/create_gacos.py b/.codex_tmp/pyint_variants/no_rescue/pyint/create_gacos.py deleted file mode 100644 index 1d28de3..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/create_gacos.py +++ /dev/null @@ -1,136 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Wei Chen ### -### Contact : chenweicug@gmail.com ### -################################################################# -import numpy as np -import os -import sys -import argparse -import subprocess -from pyint import _utils as ut - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Coregister SM mode SLC to a reference SLC image using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName', help='Name of project.') - parser.add_argument('mdate', help='date of the slave SLC image') - parser.add_argument('sdate', help='date of the slave SLC image') - inps = parser.parse_args() - return inps - - -INTRODUCTION = ''' -------------------------------------------------------------------- - create_files for psokinv software running file by generated by Gamma -''' - -EXAMPLE = """Usage: - - coreg_gamma.py projectName - - coreg_gamma.py PacayaT163TsxHhA -------------------------------------------------------------------- -""" - - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - Sdate = inps.sdate - Mdate = inps.mdate - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - Smdate = templateDict['masterDate'] - - demDir = scratchDir + '/' + projectName + '/DEM' - slcDir = scratchDir + '/' + projectName + "/SLC" - rslcDir = scratchDir + '/' + projectName + "/RSLC" - ifgramDir = scratchDir + '/' + projectName + "/ifgrams" - workDir = ifgramDir + '/' + Mdate + '-' + Sdate - MslcDir = rslcDir + '/' + Mdate - SslcDir = rslcDir + '/' + Sdate - - offpar = SslcDir + '/' + Smdate + '_' + Sdate + '.off' - slcpar = MslcDir + '/' + Smdate + '.rslc.par' - dempar = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem.par' - dem = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem' - unwpar = demDir + '/' + Mdate + '_' + rlks + 'rlks.utm.dem.par' - unw = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.diff_filt.unw" - - os.chdir(workDir) - call_str = "look_vector " + slcpar+ " " + offpar + " " + dempar + " " + dem + ' lv_theta lv_phi' - os.system(call_str) - call_str = "grep 'corner_lat:' " + dempar + " | awk '{print $2}' " - North=subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'corner_lon:' " + dempar + " | awk '{print $2}' " - West=subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'post_lat:' " + dempar + " | awk '{print $2}' " - posty=subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'post_lon:' " + dempar + " | awk '{print $2}' " - postx=subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'width:' " + dempar + " | awk '{print $2}' " - width=subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'nlines:' " + dempar + " | awk '{print $2}' " - length=subprocess.getstatusoutput(call_str)[1] - South=str(round(float(North)+(float(length)-1)*float(posty),7)) - East=str(round(float(West)+(float(width)-1)*float(postx),7)) - - call_str = "swap_bytes " + unw+ " unw.phase_swap 4 >dinsar.log " - os.system(call_str) - call_str = "gmt xyz2grd unw.phase_swap -Gunw_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx +" -ZTLf -N0" - os.system(call_str) - call_str = "gmt grdsample unw_f.grd -Gunw.grd -I3c" - os.system(call_str) - call_str = "gmt grd2xyz unw.grd -ZTLf -N0 > " + Mdate + '-' + Sdate + '.unw' - os.system(call_str) - call_str = "gmt grdinfo unw.grd -C | awk '{print $10}' " - Width=subprocess.getstatusoutput(call_str)[1] - call_str = "gmt grdinfo unw.grd -C | awk '{print $11}' " - line=subprocess.getstatusoutput(call_str)[1] - call_str = "gmt grdinfo unw.grd -C | awk '{print $2}' " - West=subprocess.getstatusoutput(call_str)[1] - call_str = "gmt grdinfo unw.grd -C | awk '{print $5}' " - North=subprocess.getstatusoutput(call_str)[1] - ymax=str(int(round(float(line) ,7))) - xmax=str(int(round(float(Width) ,7))) - output=workDir + '/' + Mdate + '-' + Sdate + '.gacos.unw.rsc' - if os.path.exists(output) is True: - os.remove(output) - - fopen=open(output,'a+') - fopen.write('WIDTH ' + Width + '\n') - fopen.write('FILE_LENGTH ' + line + '\n') - fopen.write('XMIN 1' + '\n') - fopen.write('XMAX ' + xmax + '\n') - fopen.write('YMIN 1' + '\n') - fopen.write('YMAX ' + ymax + '\n') - fopen.write('X_FIRST ' + West + '\n') - fopen.write('Y_FIRST ' + North + '\n') - fopen.write('X_STEP 8.33333333E-04' + '\n') - fopen.write('Y_STEP -8.33333333E-04' + '\n') - fopen.write('X_UNIT degrees' + '\n') - fopen.write('Y_UNIT degrees' + '\n') - fopen.write('Z_OFFSET 0' + '\n') - fopen.write('Z_SCALE 1' + '\n') - fopen.write('PROJECTION LATLON' + '\n') - fopen.write('DATUM WGS84' + '\n') - - print("convert gamma products to GACOS is done!") - - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/create_psokinv.py b/.codex_tmp/pyint_variants/no_rescue/pyint/create_psokinv.py deleted file mode 100644 index fc812a8..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/create_psokinv.py +++ /dev/null @@ -1,180 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# -import numpy as np -import os -import sys -import argparse -import subprocess -from pyint import _utils as ut - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Coregister SM mode SLC to a reference SLC image using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName', help='Name of project.') - parser.add_argument('mdate', help='date of the slave SLC image') - parser.add_argument('sdate', help='date of the slave SLC image') - inps = parser.parse_args() - return inps - - -INTRODUCTION = ''' -------------------------------------------------------------------- - create_files for psokinv software running file by generated by Gamma -''' - -EXAMPLE = """Usage: - - create_psokinv.py projectName mdata sdata lon_min lon_max lat_min lat_max flag - - create_psokinv.py PacayaT163TsxHhA 20241203 20241215 80 81 39 30 1 -------------------------------------------------------------------- -""" - - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - Sdate = inps.sdate - Mdate = inps.mdate - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - - - - - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - Smdate = templateDict['masterDate'] - - demDir = scratchDir + '/' + projectName + '/DEM' - slcDir = scratchDir + '/' + projectName + "/SLC" - rslcDir = scratchDir + '/' + projectName + "/RSLC" - ifgramDir = scratchDir + '/' + projectName + "/ifgrams" - workDir = ifgramDir + '/' + Mdate + '-' + Sdate - MslcDir = rslcDir + '/' + Mdate - SslcDir = rslcDir + '/' + Sdate - - offpar = SslcDir + '/' + Smdate + '_' + Sdate + '.off' - slcpar = MslcDir + '/' + Smdate + '.rslc.par' - dempar = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem.par' - dem = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem' - unwpar = demDir + '/' + Mdate + '_' + rlks + 'rlks.utm.dem.par' - unw = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.diff_filt.unw" - los = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.dispmap" - cor = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.diff_filt.cor" - - os.chdir(workDir) - call_str = "look_vector " + slcpar+ " " + offpar + " " + dempar + " " + dem + ' lv_theta lv_phi' - os.system(call_str) - call_str = "grep 'corner_lat:' " + dempar + " | awk '{print $2}' " - North=subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'corner_lon:' " + dempar + " | awk '{print $2}' " - West=subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'post_lat:' " + dempar + " | awk '{print $2}' " - posty=subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'post_lon:' " + dempar + " | awk '{print $2}' " - postx=subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'width:' " + dempar + " | awk '{print $2}' " - width=subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'nlines:' " + dempar + " | awk '{print $2}' " - length=subprocess.getstatusoutput(call_str)[1] - South=str(float(North)+float(posty)*(float(length)-1)) - East=str(float(West)+float(postx)*(float(width)-1)) - #East=str(round(float(West)+(float(width)-1)*float(postx),7)) - - - call_str = "swap_bytes lv_theta lv_theta.phase_swap 4 >dinsar.log " - os.system(call_str) - call_str = "gmt xyz2grd lv_theta.phase_swap -Glv_theta.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -di0" - os.system(call_str) - call_str = "gmt grdmath 90 lv_theta.grd 3.1415926 DIV 180 MUL SUB = lv_theta_final.grd" - os.system(call_str) - call_str = "gmt grd2xyz lv_theta_final.grd -ZTLf -N0 >" +Mdate + '-' + Sdate + '.inc' - os.system(call_str) - call_str = "swap_bytes lv_phi lv_phi.phase_swap 4 >dinsar.log " - os.system(call_str) - call_str = "gmt xyz2grd lv_phi.phase_swap -Glv_phi.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -di0" - os.system(call_str) - call_str = "gmt grdmath -90 lv_phi.grd 3.1415926 DIV 180 MUL SUB = lv_phi_final.grd" - os.system(call_str) - call_str = "gmt grd2xyz lv_phi_final.grd -ZTLf -N0 >" +Mdate + '-' + Sdate + '.azi' - os.system(call_str) - call_str = "swap_bytes " + unw+ " unw.phase_swap 4 >dinsar.log " - os.system(call_str) - call_str = "gmt xyz2grd unw.phase_swap -Gunw.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -di0" - os.system(call_str) - call_str = "gmt grdmath 0.0556 unw.grd 3.1415926 DIV -4 DIV MUL = los.grd" - os.system(call_str) - call_str = "gmt grd2xyz unw.grd -ZTLf -N0 > " + Mdate + '-' + Sdate + '.unw' - os.system(call_str) - call_str = "swap_bytes " + dem + " dem.phase_swap 4 >dinsar.log " - os.system(call_str) - #call_str = "xyz2grd dem.phase_swap -Gdem_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I0.001" + " -ZTLf -N0" - call_str = "gmt xyz2grd dem.phase_swap -Gdem_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -di0" - os.system(call_str) - call_str = "gmt grd2xyz dem_f.grd -ZTLf -N0 >" + Mdate + '-' + Sdate + '.utm.dem' - os.system(call_str) - call_str = "gmt grd2xyz los.grd -ZTLf -N0 >" + Mdate + '-' + Sdate + '.utm.los' - os.system(call_str) - call_str = "gmt grdinfo lv_theta_final.grd -C | awk '{print $10}' " - Width=subprocess.getstatusoutput(call_str)[1] - call_str = "gmt grdinfo lv_theta_final.grd -C | awk '{print $11}' " - line=subprocess.getstatusoutput(call_str)[1] - call_str = "gmt grdinfo lv_theta_final.grd -C | awk '{print $2}' " - West=subprocess.getstatusoutput(call_str)[1] - call_str = "gmt grdinfo lv_theta_final.grd -C | awk '{print $5}' " - North=subprocess.getstatusoutput(call_str)[1] - ymax=str(int(round(float(line)-1 ,7))) - xmax=str(int(round(float(Width)-1 ,7))) - output=workDir + '/' + Mdate + '-' + Sdate + '.unw.rsc' - if os.path.exists(output) is True: - os.remove(output) - fopen=open(output,'a+') - fopen.write('WIDTH ' + Width + '\n') - fopen.write('FILE_LENGTH ' + line + '\n') - fopen.write('XMIN 0' + '\n') - fopen.write('XMAX ' + xmax + '\n') - fopen.write('YMIN 0' + '\n') - fopen.write('YMAX ' + ymax + '\n') - fopen.write('RLOOKS 1' + '\n') - fopen.write('ALOOKS 1' + '\n') - fopen.write('X_FIRST ' + West + '\n') - fopen.write('X_STEP 8.33333333E-04' + '\n') - fopen.write('X_UNIT degrees' + '\n') - fopen.write('Y_FIRST ' + North + '\n') - fopen.write('Y_STEP -8.33333333E-04' + '\n') - fopen.write('Y_UNIT degrees' + '\n') - fopen.write('WAVELENGTH 0.0555041577' + width + '\n') - #fopen.write('WAVELENGTH 0.236' + width + '\n') - fopen.write('MOLDE insar') - fopen.close() - call_str = "swap_bytes " + Mdate + '-' + Sdate + '.unw' + " geo_" + Mdate + '-' + Sdate + "_dem_flat.unw 4 >dinsar.log " - os.system(call_str) - call_str = "swap_bytes " + Mdate + '-' + Sdate + '.utm.dem' + " geo_" + Mdate + '-' + Sdate + "_dem_flat.utm.dem 4 >dinsar.log " - os.system(call_str) - call_str = "cp -f " + Mdate + '-' + Sdate + ".unw.rsc " + Mdate + '-' + Sdate +".inc.rsc" - os.system(call_str) - call_str = "cp -f " + Mdate + '-' + Sdate + ".unw.rsc " + Mdate + '-' + Sdate +".azi.rsc" - os.system(call_str) - call_str = "cp -f " + Mdate + '-' + Sdate + ".unw.rsc " + " geo_" + Mdate + '-' + Sdate + "_dem_flat.unw.rsc" - os.system(call_str) - call_str = "cp -f " + Mdate + '-' + Sdate + ".unw.rsc " + " geo_" + Mdate + '-' + Sdate + "_dem_flat.utm.dem.rsc" - os.system(call_str) - call_str = "rm -rf swap*grd *phase_swap dinsar.log" - os.system(call_str) - print("convert gamma products to psokinv is done!") - - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/create_psokinv_cut.py b/.codex_tmp/pyint_variants/no_rescue/pyint/create_psokinv_cut.py deleted file mode 100644 index 6affcfb..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/create_psokinv_cut.py +++ /dev/null @@ -1,235 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# -import numpy as np -import os -import sys -import argparse -import subprocess -from pyint import _utils as ut - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Coregister SM mode SLC to a reference SLC image using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName', help='Name of project.') - parser.add_argument('mdate', help='date of the slave SLC image') - parser.add_argument('sdate', help='date of the slave SLC image') - parser.add_argument('lon_min', help='the min lon of cut image') - parser.add_argument('lon_max', help='the max lon of cut image') - parser.add_argument('lat_min', help='the min lat of cut image') - parser.add_argument('lat_max', help='the min lat of cut image') - parser.add_argument('flag', help="the cut image flag (1: do; 0:don't do)") - inps = parser.parse_args() - return inps - - -INTRODUCTION = ''' -------------------------------------------------------------------- - create_files for psokinv software running file by generated by Gamma -''' - -EXAMPLE = """Usage: - - create_psokinv.py projectName mdata sdata lon_min lon_max lat_min lat_max flag - - create_psokinv.py PacayaT163TsxHhA 20241203 20241215 80 81 39 30 1 -------------------------------------------------------------------- -""" - - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - Sdate = inps.sdate - Mdate = inps.mdate - lon_max = inps.lon_max - lon_min = inps.lon_min - lat_min = inps.lat_min - lat_max = inps.lat_max - flag = inps.flag - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - - - - - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - Smdate = templateDict['masterDate'] - - demDir = scratchDir + '/' + projectName + '/DEM' - slcDir = scratchDir + '/' + projectName + "/SLC" - rslcDir = scratchDir + '/' + projectName + "/RSLC" - ifgramDir = scratchDir + '/' + projectName + "/ifgrams" - workDir = ifgramDir + '/' + Mdate + '-' + Sdate - MslcDir = rslcDir + '/' + Mdate - SslcDir = rslcDir + '/' + Sdate - - offpar = SslcDir + '/' + Smdate + '_' + Sdate + '.off' - slcpar = MslcDir + '/' + Smdate + '.rslc.par' - dempar = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem.par' - dem = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem' - unwpar = demDir + '/' + Mdate + '_' + rlks + 'rlks.utm.dem.par' - unw = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.diff_filt.unw" - los = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.dispmap" - cor = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.diff_filt.cor" - - os.chdir(workDir) - call_str = "look_vector " + slcpar+ " " + offpar + " " + dempar + " " + dem + ' lv_theta lv_phi' - os.system(call_str) - call_str = "grep 'corner_lat:' " + dempar + " | awk '{print $2}' " - North=subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'corner_lon:' " + dempar + " | awk '{print $2}' " - West=subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'post_lat:' " + dempar + " | awk '{print $2}' " - posty=subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'post_lon:' " + dempar + " | awk '{print $2}' " - postx=subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'width:' " + dempar + " | awk '{print $2}' " - width=subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'nlines:' " + dempar + " | awk '{print $2}' " - length=subprocess.getstatusoutput(call_str)[1] - South=str(round(float(North)+(float(length)-1)*float(posty),7)) - East=str(round(float(West)+(float(width)-1)*float(postx),7)) - - - call_str = "swap_bytes lv_theta lv_theta.phase_swap 4 >dinsar.log " - os.system(call_str) - call_str = "gmt xyz2grd lv_theta.phase_swap -Glv_theta.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -N0" - os.system(call_str) - call_str = "gmt grdmath 90 lv_theta.grd 3.1415926 DIV 180 MUL SUB = lv_theta_final.grd" - os.system(call_str) - if flag == '0': - call_str = "gmt grdsample lv_theta_final.grd -Glv_theta.grd -I3c" - else: - call_str = "gmt grdsample lv_theta_final.grd -R" + lon_min + "/" + lon_max + "/" + lat_min + "/" + lat_max + " -Glv_theta.grd -I3c" - os.system(call_str) - call_str = "gmt grd2xyz lv_theta.grd -ZTLf -N0 >" +Mdate + '-' + Sdate + '.inc' - os.system(call_str) - call_str = "swap_bytes lv_phi lv_phi.phase_swap 4 >dinsar.log " - os.system(call_str) - call_str = "gmt xyz2grd lv_phi.phase_swap -Glv_phi.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -N0" - os.system(call_str) - call_str = "gmt grdmath -90 lv_phi.grd 3.1415926 DIV 180 MUL SUB = lv_phi_final.grd" - os.system(call_str) - if flag == '0': - call_str = "gmt grdsample lv_phi_final.grd -Glv_phi.grd -I3c" - else: - call_str = "gmt grdsample lv_phi_final.grd -R" + lon_min + "/" + lon_max + "/" + lat_min + "/" + lat_max + " -Glv_phi.grd -I3c" - os.system(call_str) - call_str = "gmt grd2xyz lv_phi.grd -ZTLf -N0 >" +Mdate + '-' + Sdate + '.azi' - os.system(call_str) - call_str = "swap_bytes " + unw+ " unw.phase_swap 4 >dinsar.log " - os.system(call_str) - call_str = "gmt xyz2grd unw.phase_swap -Gunw_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx +" -ZTLf -N0" - os.system(call_str) - call_str = "swap_bytes " + los + " los.swap 4 >dinsar.log " - os.system(call_str) - call_str = "gmt xyz2grd los.swap -Glos_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx +" -ZTLf -N0" - os.system(call_str) - call_str = "swap_bytes " + cor + " cor.swap 4 >dinsar.log " - os.system(call_str) - call_str = "gmt xyz2grd cor.swap -Gcor_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx +" -ZTLf -N0" - os.system(call_str) - - if flag == '0': - call_str = "gmt grdsample unw_f.grd -Gunw.grd -I3c" - else: - call_str = "gmt grdsample unw_f.grd -R" + lon_min + "/" + lon_max + "/" + lat_min + "/" + lat_max + " -Gunw.grd -I3c" - os.system(call_str) - call_str = "gmt grdmath 0.0556 unw_f.grd 3.1415926 DIV -4 DIV MUL = los.grd" - os.system(call_str) - call_str = "gmt grd2xyz unw.grd -ZTLf -N0 > " + Mdate + '-' + Sdate + '.unw' - os.system(call_str) - call_str = "swap_bytes " + dem + " dem.phase_swap 4 >dinsar.log " - os.system(call_str) - #call_str = "xyz2grd dem.phase_swap -Gdem_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I0.001" + " -ZTLf -N0" - call_str = "gmt xyz2grd dem.phase_swap -Gdem_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -N0" - os.system(call_str) - if flag == '0': - call_str = "gmt grdsample dem_f.grd -Gdem.grd -I3c" - else: - call_str = "gmt grdsample dem_f.grd -R" + lon_min + "/" + lon_max + "/" + lat_min + "/" + lat_max + " -Gdem.grd -I3c" - os.system(call_str) - - call_str = "gmt grd2xyz dem.grd -ZTLf -N0 >" + Mdate + '-' + Sdate + '.utm.dem' - os.system(call_str) - - if flag == '0': - call_str = "gmt grdsample los_f.grd -Glos1.grd -I3c" - else: - call_str = "gmt grdsample los_f.grd -R" + lon_min + "/" + lon_max + "/" + lat_min + "/" + lat_max + " -Glos1.grd -I3c" - os.system(call_str) - - call_str = "gmt grd2xyz los1.grd -ZTLf -N0 >" + Mdate + '-' + Sdate + '.utm.los' - os.system(call_str) - if flag == '0': - call_str = "gmt grdsample cor_f.grd -Gcor.grd -I3c" - else: - call_str = "gmt grdsample cor_f.grd -R" + lon_min + "/" + lon_max + "/" + lat_min + "/" + lat_max + " -Gcor.grd -I3c" - os.system(call_str) - - call_str = "gmt grd2xyz cor.grd -ZTLf -N0 >" + Mdate + '-' + Sdate + '.utm.cor' - os.system(call_str) - - call_str = "gmt grdinfo lv_theta.grd -C | awk '{print $10}' " - Width=subprocess.getstatusoutput(call_str)[1] - call_str = "gmt grdinfo lv_theta.grd -C | awk '{print $11}' " - line=subprocess.getstatusoutput(call_str)[1] - call_str = "gmt grdinfo lv_theta.grd -C | awk '{print $2}' " - West=subprocess.getstatusoutput(call_str)[1] - call_str = "gmt grdinfo lv_theta.grd -C | awk '{print $5}' " - North=subprocess.getstatusoutput(call_str)[1] - ymax=str(int(round(float(line)-1 ,7))) - xmax=str(int(round(float(Width)-1 ,7))) - output=workDir + '/' + Mdate + '-' + Sdate + '.unw.rsc' - if os.path.exists(output) is True: - os.remove(output) - fopen=open(output,'a+') - fopen.write('WIDTH ' + Width + '\n') - fopen.write('FILE_LENGTH ' + line + '\n') - fopen.write('XMIN 0' + '\n') - fopen.write('XMAX ' + xmax + '\n') - fopen.write('YMIN 0' + '\n') - fopen.write('YMAX ' + ymax + '\n') - fopen.write('RLOOKS 1' + '\n') - fopen.write('ALOOKS 1' + '\n') - fopen.write('X_FIRST ' + West + '\n') - fopen.write('X_STEP 8.33333333E-04' + '\n') - fopen.write('X_UNIT degrees' + '\n') - fopen.write('Y_FIRST ' + North + '\n') - fopen.write('Y_STEP -8.33333333E-04' + '\n') - fopen.write('Y_UNIT degrees' + '\n') - fopen.write('WAVELENGTH 0.0555041577' + width + '\n') - #fopen.write('WAVELENGTH 0.236' + width + '\n') - fopen.write('MOLDE insar') - fopen.close() - call_str = "swap_bytes " + Mdate + '-' + Sdate + '.unw' + " geo_" + Mdate + '-' + Sdate + "_dem_flat.unw 4 >dinsar.log " - os.system(call_str) - call_str = "swap_bytes " + Mdate + '-' + Sdate + '.utm.dem' + " geo_" + Mdate + '-' + Sdate + "_dem_flat.utm.dem 4 >dinsar.log " - os.system(call_str) - call_str = "cp -f " + Mdate + '-' + Sdate + ".unw.rsc " + Mdate + '-' + Sdate +".inc.rsc" - os.system(call_str) - call_str = "cp -f " + Mdate + '-' + Sdate + ".unw.rsc " + Mdate + '-' + Sdate +".azi.rsc" - os.system(call_str) - call_str = "cp -f " + Mdate + '-' + Sdate + ".unw.rsc " + " geo_" + Mdate + '-' + Sdate + "_dem_flat.unw.rsc" - os.system(call_str) - call_str = "cp -f " + Mdate + '-' + Sdate + ".unw.rsc " + " geo_" + Mdate + '-' + Sdate + "_dem_flat.utm.dem.rsc" - os.system(call_str) - call_str = "rm -rf swap*grd *phase_swap dinsar.log" - os.system(call_str) - print("convert gamma products to psokinv is done!") - - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/diff_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/diff_gamma.py deleted file mode 100644 index 7396087..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/diff_gamma.py +++ /dev/null @@ -1,183 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT ### -### Author: chen ### -### Contact : chenweicug@126.com ### -################################################################# - -import numpy as np -import os -import sys -import getopt -import time -import glob -import argparse - -from pyint import _utils as ut - - -def _run_or_raise(call_str, stage): - rc = os.system(call_str) - if rc != 0: - raise RuntimeError('%s failed with rc=%s: %s' % (stage, rc, call_str)) - return rc - - -INTRODUCTION = ''' -------------------------------------------------------------------- - Generate differential interferogram image from SLC using GAMMA. - -''' - -EXAMPLE = ''' - Usage: - diff_gamma.py projectName Mdate Sdate - diff_gamma.py PacayaT163TsxHhA 20150102 20150601 -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Coregister all of the SLCs to the reference SAR image using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('Mdate',help='Master date.') - parser.add_argument('Sdate',help='Slave date.') - - inps = parser.parse_args() - return inps - - -def main(argv): - - start_time = time.time() - inps = cmdLineParse() - Mdate = inps.Mdate - Sdate = inps.Sdate - - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - masterDate = templateDict['masterDate'] - - projectDir = scratchDir + '/' + projectName - demDir = scratchDir + '/' + projectName + '/DEM' - - slcDir = scratchDir + '/' + projectName + '/SLC' - rslcDir = scratchDir + '/' + projectName + '/RSLC' - ifgDir = projectDir + '/ifgrams' - if not os.path.isdir(ifgDir): os.mkdir(ifgDir) - - Pair = Mdate + '-' + Sdate - workDir = ifgDir + '/' + Pair - if not os.path.isdir(workDir): os.mkdir(workDir) - - ####################################################################### - Mamp = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' - MampPar = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' - Samp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' - SampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' - - Mrslc = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc' - MrslcPar = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc.par' - Srslc = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc' - SrslcPar = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc.par' - - HGT = demDir + '/' + masterDate + '_' + rlks + 'rlks.rdc.dem' - - MasterPar = rslcDir + '/' + masterDate + '/' + masterDate + '.rslc.par' - - ################# copy file for parallel processing ########################## - #Mamp = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp' - #MampPar = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp.par' - #Samp = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp' - #SampPar = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp.par' - - #if not templateDict['diff_all_parallel'] == '1': - - # Mrslc = workDir + '/' + Mdate + '.rslc' - # MrslcPar = workDir + '/' + Mdate + '.rslc.par' - # Srslc = workDir + '/' + Sdate + '.rslc' - # SrslcPar = workDir + '/' + Sdate + '.rslc.par' - # ut.copy_file(Mrslc0,Mrslc) - # ut.copy_file(MrslcPar0,MrslcPar) - # ut.copy_file(Srslc0,Srslc) - # ut.copy_file(SrslcPar0,SrslcPar) - - #else: - - # Mrslc = Mrslc0 - # MrslcPar = MrslcPar0 - # Srslc = Srslc0 - # SrslcPar = SrslcPar0 - # HGT = HGT0 - # MasterPar = MasterPar0 - - #ut.copy_file(Mamp0,Mamp) - #ut.copy_file(MampPar0,MampPar) - #ut.copy_file(Samp0,Samp) - #ut.copy_file(SampPar0,SampPar) - - #ut.copy_file(HGT0,HGT) - #ut.copy_file(MasterPar0,MasterPar) - - ############################################################################ - - OFF = workDir + '/' + Pair +'_' + rlks + 'rlks.off' - call_str = 'create_offset '+ MrslcPar + ' ' + SrslcPar + ' ' + OFF + ' 1 ' + rlks + ' ' + azlks + ' 0' - _run_or_raise(call_str, 'create_offset') - - SIM_UNW = workDir + '/' + Pair + '.sim_unw' - call_str = 'phase_sim_orb ' + MrslcPar + ' ' + SrslcPar + ' ' + OFF + ' ' + HGT + ' ' + SIM_UNW + ' ' + MasterPar + ' - - 1 1' - _run_or_raise(call_str, 'phase_sim_orb') - - DIFF_IFG = workDir + '/' + Pair + '_' + rlks + 'rlks.diff' - call_str = 'SLC_diff_intf ' + Mrslc + ' ' + Srslc + ' ' + MrslcPar + ' ' + SrslcPar + ' ' + OFF + ' ' + SIM_UNW + ' ' + DIFF_IFG + ' ' + rlks + ' ' + azlks + ' ' + templateDict['Igram_Spsflg'] + ' ' + templateDict['Igram_Azfflg'] + ' - 1 1' - _run_or_raise(call_str, 'SLC_diff_intf') - - ##### filtering process & coherence estimation ########### - DIFFFILT = workDir + '/' + Pair + '_' + rlks + 'rlks.diff_filt' - COHFILT = workDir + '/' + Pair + '_' + rlks + 'rlks.diff_filt.cor' - - nWIDTH = ut.read_gamma_par(OFF, 'read', 'interferogram_width') - call_str = 'adf ' + DIFF_IFG + ' ' + DIFFFILT + ' ' + COHFILT + ' ' + nWIDTH + ' ' + templateDict['adf_alpha'] + ' - ' + templateDict['Igram_Cor_Win'] - _run_or_raise(call_str, 'adf') - - ################# coherence estimation ##################### - call_str = 'cc_wave ' + DIFFFILT + ' ' + Mamp + ' ' + Samp + ' ' + COHFILT + ' ' + nWIDTH + ' ' + templateDict['Igram_Cor_rwin'] + ' ' + templateDict['Igram_Cor_awin'] - _run_or_raise(call_str, 'cc_wave') - - - ################ save images ##################### - call_str = 'rasmph_pwr ' + DIFFFILT + ' ' + Mamp + ' ' + nWIDTH + ' - - - - - - - - - ' + COHFILT + ' - 0.1' - _run_or_raise(call_str, 'rasmph_pwr_diff_filt') - - call_str = 'rasmph_pwr ' + DIFF_IFG + ' ' + Mamp + ' ' + nWIDTH + ' - - - - - - - - - ' + COHFILT + ' - 0.1' - _run_or_raise(call_str, 'rasmph_pwr_diff') - - call_str = 'rasdt_pwr ' + COHFILT + ' ' + Mamp + ' ' + nWIDTH + ' 1 0 1 1 0.1 1.0 1 ' - _run_or_raise(call_str, 'rasdt_pwr_coh') - - #os.remove(Mamp) - #os.remove(MampPar) - #os.remove(Samp) - #os.remove(SampPar) - - #if not templateDict['diff_all_parallel'] == '1': - # if os.path.isfile(Mrslc): os.remove(Mrslc) - # if os.path.isfile(Srslc):os.remove(Srslc) - - # if os.path.isfile(HGT):os.remove(HGT) - - print("Subtraction of topography and flattening phase is done!") - ut.print_process_time(start_time, time.time()) - sys.exit(0) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/diff_gamma_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/diff_gamma_all.py deleted file mode 100644 index def2813..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/diff_gamma_all.py +++ /dev/null @@ -1,144 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import getopt -import time -import glob -import argparse - -import subprocess -from pyint import _utils as ut - - -def work(data0): - cmd = data0[0] - err_file = data0[1] - p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) - stdout = p.stdout - stderr = p.stderr - - if type(stderr) == bytes: - aa=stderr.decode("utf-8") - else: - aa = stderr - - if type(stdout) == bytes: - bb=stdout.decode("utf-8", errors="replace") - else: - bb = stdout - - if p.returncode != 0: - detail_parts = [] - if bb: - detail_parts.append(bb) - if aa: - detail_parts.append(aa) - detail = '\n'.join(part.strip() for part in detail_parts if part and part.strip()) - str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + cmd[3] + '\n' - with open(err_file, 'a') as f: - f.write(str0) - if detail: - f.write(detail) - f.write('\n') - raise RuntimeError(str0.strip() + ' failed with rc=' + str(p.returncode) + ('\n' + detail if detail else '')) - - if aa: - str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + cmd[3] + '\n' - with open(err_file, 'a') as f: - f.write(str0) - f.write(aa) - f.write('\n') - - return -######################################################################### - -INTRODUCTION = ''' -------------------------------------------------------------------- - Geneate differential interferograms for one project using GAMMA. - -''' - -EXAMPLE = ''' - Usage: - diff_gamma_all.py projectName - diff_gamma_all.py projectName --parallel 4 - diff_gamma_all.py projectName --parallel 4 --ifgramList-txt /test/ifgram_list.txt -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Geneate differential interferograms for one project using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') - parser.add_argument('--ifgarmList-txt', dest='ifgarmListTxt', help='provided ifgram_list_txt. default: using ifgram_list.txt under projectName folder.') - - inps = parser.parse_args() - return inps - - -def main(argv): - start_time = time.time() - inps = cmdLineParse() - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - projectDir = scratchDir + '/' + projectName - ifgDir = scratchDir + '/' + projectName + '/ifgrams' - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - - if inps.ifgarmListTxt: ifgramList_txt = inps.ifgarmListTxt - else: ifgramList_txt = scratchDir + '/' + projectName + '/ifgram_list.txt' - ifgList0 = ut.read_txt2array(ifgramList_txt) -# ifgList = ifgList0[:,0] - if len(ifgList0)==3: - ifgList=ifgList0[0] - ifgList=[ifgList] - - else: - ifgList=ifgList0[:,0] - - err_txt = scratchDir + '/' + projectName + '/diff_gamma_all.err' - if os.path.isfile(err_txt): os.remove(err_txt) - - data_para = [] - for i in range(len(ifgList)): - m0 = ut.yyyymmdd(ifgList[i].split('-')[0]) - s0 = ut.yyyymmdd(ifgList[i].split('-')[1]) - cmd0 = ['diff_gamma.py',projectName, m0, s0] - diff_file0 = ifgDir + '/' + ifgList[i] + '/' + ifgList[i] + '_' + rlks + 'rlks.diff_filt.bmp' - data0 = [cmd0,err_txt] - - k00 = 0 - if os.path.isfile(diff_file0): - if os.path.getsize(diff_file0) > 0: - k00 = 1 - if k00==0: - data_para.append(data0) - - results = ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) - failures = [str(item) for item in results if isinstance(item, Exception)] - if failures: - raise RuntimeError('\n\n'.join(failures)) - print("Generate differential interferograms for project %s is done! " % projectName) - ut.print_process_time(start_time, time.time()) - - sys.exit(0) - -if __name__ == '__main__': - main(sys.argv[:]) - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_LT1.py b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_LT1.py deleted file mode 100644 index 3c1f69f..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_LT1.py +++ /dev/null @@ -1,228 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.2 ### -### Copy Right (c): 2017-2019, Chen Wei ### -### Author: chenwei ### -### Contact : ymcmrs@gmail.com ### -################################################################# -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob -import argparse -import re -import shutil -import tarfile - -from pyint import _orbit_bridge as orbit_bridge -from pyint import _utils as ut - - -def get_LT1_date(raw_file): - file0 = os.path.basename(raw_file) - date = file0[41:48] - return date - -def get_satellite(raw_file): - if 'LT1A_MONO_' in raw_file: - s0 = 'A' - else: - s0 = 'B' - return s0 - - -def discover_lt1_inputs(down_dir, date): - candidates = [] - for pattern in ( - down_dir + '/LT1*' + date + '*.tar.gz', - down_dir + '/LT1*' + date + '*.tiff', - ): - candidates.extend(glob.glob(pattern)) - return sorted(set(candidates)) - - -def write_input_list(list_path, paths): - with open(list_path, 'w') as f: - for path in paths: - f.write(path + '\n') - - -def _run_checked(command, cwd=None): - result = subprocess.run( - command, - cwd=cwd, - text=True, - capture_output=True, - check=False, - ) - if result.returncode != 0: - detail = (result.stderr or result.stdout or "").strip() - raise RuntimeError( - 'Command failed (%s): %s%s' % ( - result.returncode, - ' '.join(command), - ('\n' + detail) if detail else '', - ) - ) - return result - - -def _cleanup_paths(paths): - for path in paths: - if not path: - continue - try: - if os.path.isdir(path): - shutil.rmtree(path) - elif os.path.isfile(path): - os.remove(path) - except OSError: - continue - - -def _resolve_lt1_input_scene(raw_path, work_dir): - cleanup_paths = [] - raw_lower = raw_path.lower() - if raw_lower.endswith('.tiff'): - input_xml = re.sub(r'\.tiff$', '.meta.xml', raw_path, flags=re.IGNORECASE) - if not os.path.isfile(input_xml): - raise FileNotFoundError('LT-1 meta xml does not exist: ' + input_xml) - return raw_path, input_xml, cleanup_paths - - if raw_lower.endswith('.tar.gz'): - temp_dir = os.path.join(work_dir, 'tmp_data_dir') - if os.path.isdir(temp_dir): - shutil.rmtree(temp_dir) - os.makedirs(temp_dir, exist_ok=True) - cleanup_paths.append(temp_dir) - with tarfile.open(raw_path, 'r:gz') as archive: - member_names = archive.getnames() - tiff_members = [name for name in member_names if name.lower().endswith('.tiff')] - xml_members = [name for name in member_names if name.lower().endswith('meta.xml')] - if not tiff_members or not xml_members: - raise RuntimeError('LT-1 archive is missing .tiff or meta.xml: ' + raw_path) - tiff_member = tiff_members[0] - xml_member = xml_members[0] - archive.extract(tiff_member, path=temp_dir) - archive.extract(xml_member, path=temp_dir) - return os.path.join(temp_dir, tiff_member), os.path.join(temp_dir, xml_member), cleanup_paths - - raise RuntimeError('Unsupported LT-1 input, only .tiff or .tar.gz are supported: ' + raw_path) - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Generate SLC from LT1 raw data with orbit correction using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName', help='project name. e.g., ChangningT55S1A') - parser.add_argument('date',help='date to be processed. e.g., 20180101') - inps = parser.parse_args() - - return inps - - -INTRODUCTION = ''' -------------------------------------------------------------------- - - Generate SLC from Sentinel-1 raw data using S1_import_SLC_from_zipfiles with orbit correction. - [Precise orbit data will be downloaded automatically] -''' - -EXAMPLE = """Usage: - - down2slc_sen.py projectName date - - down2slc_sen.py ChangningT55S1A 20180517 - -------------------------------------------------------------------- -""" - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - date = inps.date - scratchDir = os.getenv('SCRATCHDIR') - projectDir = scratchDir + '/' + projectName - - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - - slc_dir = projectDir + '/SLC' - down_dir = projectDir + '/DOWNLOAD' - - if not os.path.isdir(slc_dir): - os.mkdir(slc_dir) - - work_dir = slc_dir + '/' + date - if not os.path.isdir(work_dir): - os.mkdir(work_dir) - - os.chdir(work_dir) - - t_date = 't_' + date - - input_files = discover_lt1_inputs(down_dir, date) - write_input_list(t_date, input_files) - raw_files = ut.read_txt2list(t_date) - if len(raw_files) == 0: - raise RuntimeError('No LT-1 inputs found for date: ' + date) - satellite = get_satellite(str(raw_files[0])) - - zipfile_ref=str(raw_files[0]) - outfile_name=zipfile_ref.split('/')[-1].split('.')[0] - outfile_name=zipfile_ref.split('/')[-1] - print(outfile_name) - - cleanup_paths = [] - try: - input_tiff, input_xml, cleanup_paths = _resolve_lt1_input_scene(zipfile_ref, work_dir) - slc_path = work_dir + '/' + date + '.slc' - slc_par_path = work_dir + '/' + date + '.slc.par' - _run_checked( - ['par_LT1_SLC', input_tiff, input_xml, slc_par_path, slc_path], - cwd=work_dir, - ) - if not os.path.isfile(slc_path) or not os.path.isfile(slc_par_path): - raise RuntimeError('LT-1 import produced no SLC outputs for date: ' + date) - bridge_result = orbit_bridge.apply_precise_orbit( - date, - [slc_par_path], - work_dir=work_dir, - operation_tag='lt1_import', - ) - if bridge_result.get('stdout'): - print(bridge_result['stdout']) - if bridge_result.get('stderr'): - print(bridge_result['stderr']) - finally: - _cleanup_paths(cleanup_paths) - - - SLC_Tab = work_dir + '/' + date+'_SLC_Tab' - SLC_list = sorted(glob.glob(work_dir + '/*.slc')) - SLC_par_list = sorted(glob.glob(work_dir + '/*.slc.par')) - if len(SLC_list) == 0 or len(SLC_par_list) == 0: - raise RuntimeError('No LT-1 SLC outputs were generated for date: ' + date) - if len(SLC_list) != len(SLC_par_list): - raise RuntimeError('LT-1 SLC and SLC parameter counts do not match for date: ' + date) - - if os.path.isfile(SLC_Tab): - os.remove(SLC_Tab) - - for kk in range(len(SLC_list)): - call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' >> ' + SLC_Tab - if os.system(call_str) != 0: - raise RuntimeError('Failed to write SLC tab for date: ' + date) - with open(work_dir + '/down2slc.dat', 'w') as f: - f.write('ok\n') - print("Down to SLC for %s is done! " % date) - sys.exit(0) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_LT1_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_LT1_all.py deleted file mode 100644 index 09e167f..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_LT1_all.py +++ /dev/null @@ -1,185 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.2 ### -### Copy Right (c): 2017-2019, Chen Wei ### -### Author: chenwei ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import getopt -import time -import glob -import argparse -import re - -import subprocess -from pyint import _utils as ut - -def get_LT1_date(raw_file): - file0 = os.path.basename(raw_file) - match = re.search(r'(20\d{6})', file0) - if match: - return match.group(1) - return '' - - -def discover_lt1_inputs(down_dir): - candidates = [] - for pattern in ('/LT1*.tar.gz', '/LT1*.tiff'): - candidates.extend(glob.glob(down_dir + pattern)) - return sorted(set(candidates)) - -def work(data0): - cmd = data0[0] - err_txt = data0[1] - p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) - stdout = p.stdout - stderr = p.stderr - - if type(stderr) == bytes: - aa=stderr.decode("utf-8") - else: - aa = stderr - - if type(stdout) == bytes: - bb=stdout.decode("utf-8", errors="replace") - else: - bb = stdout - - if p.returncode != 0: - detail_parts = [] - if bb: - detail_parts.append(bb) - if aa: - detail_parts.append(aa) - detail = '\n'.join(part.strip() for part in detail_parts if part and part.strip()) - str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' - with open(err_txt, 'a') as f: - f.write(str0) - if detail: - f.write(detail) - f.write('\n') - raise RuntimeError(str0.strip() + ' failed with rc=' + str(p.returncode) + ('\n' + detail if detail else '')) - - if aa: - str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' - with open(err_txt, 'a') as f: - f.write(str0) - f.write(aa) - f.write('\n') - - return -######################################################################### - -INTRODUCTION = ''' -------------------------------------------------------------------- - - Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA. - -''' - -EXAMPLE = ''' - Usage: - down2slc_LT1_all.py projectName - down2slc_LT1_all.py projectName --parallel 4 - -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') - - inps = parser.parse_args() - return inps - - -def main(argv): - start_time = time.time() - inps = cmdLineParse() - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - projectDir = scratchDir + '/' + projectName - downDir = scratchDir + '/' + projectName + "/DOWNLOAD" - slcDir = scratchDir + '/' + projectName + "/SLC" - raw_file_list = discover_lt1_inputs(downDir) - if len(raw_file_list) == 0: - raise RuntimeError('No LT-1 inputs found under: ' + downDir) - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - - - date_list = [] - cat_list = [] - for kk in range(len(raw_file_list)): - date0 = get_LT1_date(os.path.basename(raw_file_list[kk])) - if not date0: - continue - if date0 not in date_list: - date_list.append(date0) - cat_list.append('0') - else: - cat_list[date_list.index(date0)]='1' -# date_list = set(date_list) -# date_list = sorted(date_list) - - print('Date to be processed:') - for k0 in date_list: - print(k0) - - err_txt = scratchDir + '/' + projectName + '/down2slc_sen_all.err' - if os.path.isfile(err_txt): os.remove(err_txt) - - data_para = [] - for i in range(len(date_list)): - if cat_list[i]=='0': - cmd0 = ['down2slc_LT1.py',projectName,date_list[i]] - work_dir = slcDir + '/' + date_list[i] - slc_file0 = work_dir + '/down2slc.dat' - data0 = [cmd0,err_txt] -# data_para.append(data0) - k00 = 0 - if os.path.isfile(slc_file0): - if os.path.getsize(slc_file0) > 0: - k00 = 1 - else: - k00 = 0 - if k00==0: - data_para.append(data0) - else: - cmd0 = ['down2slc_cat_LT1.py',projectName,date_list[i]] - work_dir = slcDir + '/' + date_list[i] - slc_file0 = work_dir + '/down2slc.dat' - data0 = [cmd0,err_txt] -# data_para.append(data0) - #data_para.append(data0) - k00 = 0 - if os.path.isfile(slc_file0): - if os.path.getsize(slc_file0) > 0: - k00 = 1 - else: - k00 = 0 - if k00==0: - data_para.append(data0) - results = ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) - failures = [str(item) for item in results if isinstance(item, Exception)] - if failures: - raise RuntimeError('\n\n'.join(failures)) - os.chdir(downDir) - print("Down to SLC for project %s is done! " % projectName) - ut.print_process_time(start_time, time.time()) - - sys.exit(0) - -if __name__ == '__main__': - main(sys.argv[:]) - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_alos_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_alos_all.py deleted file mode 100644 index 3f08918..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_alos_all.py +++ /dev/null @@ -1,138 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import getopt -import time -import glob -import argparse - -import subprocess -from pyint import _utils as ut - - -def get_s1_date(raw_file): - file0 = os.path.basename(raw_file) - date = file0[17:25] - return date - - -def work(data0): - cmd = data0[0] - err_txt = data0[1] - p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) - stdout = p.stdout - stderr = p.stderr - - if type(stderr) == bytes: - aa=stderr.decode("utf-8") - else: - aa = stderr - - if aa: - str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' - #print(aa) - with open(err_txt, 'a') as f: - f.write(str0) - f.write(aa) - f.write('\n') - - return -######################################################################### - -INTRODUCTION = ''' -------------------------------------------------------------------- - - Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA. - -''' - -EXAMPLE = ''' - Usage: - down2slc_sen_all.py projectName - down2slc_sen_all.py projectName --parallel 4 - -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') - - inps = parser.parse_args() - return inps - - -def main(argv): - start_time = time.time() - inps = cmdLineParse() - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - projectDir = scratchDir + '/' + projectName - downDir = scratchDir + '/' + projectName + "/DOWNLOAD" - - raw_file_list = glob.glob(downDir + '/ALPSRP*.zip') - slc_dir = scratchDir + '/' + projectName + '/SLC' - if not os.path.isdir(slc_dir): - os.mkdir(slc_dir) - - err_txt = scratchDir + '/' + projectName + '/unzip_alos_all.err' - if os.path.isfile(err_txt): os.remove(err_txt) -########get the ALOS date name ############### - data_para = [] - date_list = [] - for i in range(len(raw_file_list)): - cmd0 = ['unzip',raw_file_list[i],'-d',downDir] - call_str='unzip ' + raw_file_list[i] + ' -d ' + downDir - os.system(call_str) - workflow_file=raw_file_list[i].split(".")[0] + '.' +raw_file_list[i].split(".")[1] - os.chdir(workflow_file) - tempfile=workflow_file + '/' + 'workreport' - tempDict=ut.update_template(tempfile) - SeneceData = tempDict['Img_SceneCenterDateTime'].split('"')[1].split(' ')[0] - SceneID = tempDict['Scs_SceneID'] - os.chdir(downDir) - call_str= 'mv ' + workflow_file + ' ' + SeneceData - os.system(call_str) - data0=[SeneceData, err_txt] - data_para.append(data0) - date0=SeneceData - date_list.append(date0) - date_list = set(date_list) - date_list = sorted(date_list) - - print('Date to be processed:') - for k0 in date_list: - print(k0) - - err_txt = scratchDir + '/' + projectName + '/down2slc_alos_all.err' - if os.path.isfile(err_txt): os.remove(err_txt) - - data_para = [] - for i in range(len(date_list)): - cmd0 = ['Down2SLC_ALOS.py',projectName,date_list[i]] - data0 = [cmd0,err_txt] - data_para.append(data0) - - ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) - os.chdir(downDir) - print("Down to SLC for project %s is done! " % projectName) - ut.print_process_time(start_time, time.time()) - - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_cat_LT1.py b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_cat_LT1.py deleted file mode 100644 index 04e02fd..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_cat_LT1.py +++ /dev/null @@ -1,202 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.2 ### -### Copy Right (c): 2017-2019, Chen Wei ### -### Author: chenwei ### -### Contact : ymcmrs@gmail.com ### -################################################################# -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob -import argparse -import re - -from pyint import _orbit_bridge as orbit_bridge -from pyint import _utils as ut - - -def get_LT1_date(raw_file): - file0 = os.path.basename(raw_file) - date = file0[41:48] - return date - -def get_satellite(raw_file): - if 'LT1A_MONO_' in raw_file: - s0 = 'A' - else: - s0 = 'B' - return s0 - - -def discover_lt1_inputs(down_dir, date): - candidates = [] - for pattern in ( - down_dir + '/LT1*' + date + '*.tar.gz', - down_dir + '/LT1*' + date + '*.tiff', - ): - candidates.extend(glob.glob(pattern)) - return sorted(set(candidates)) - - -def write_input_list(list_path, paths): - with open(list_path, 'w') as f: - for path in paths: - f.write(path + '\n') - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Generate SLC from LT1 raw data with orbit correction using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName', help='project name. e.g., ChangningT55S1A') - parser.add_argument('date',help='date to be processed. e.g., 20180101') - inps = parser.parse_args() - - return inps - - -INTRODUCTION = ''' -------------------------------------------------------------------- - - Generate SLC from Sentinel-1 raw data using S1_import_SLC_from_zipfiles with orbit correction. - [Precise orbit data will be downloaded automatically] -''' - -EXAMPLE = """Usage: - - down2slc_LT1.py projectName date - - down2slc_LT1.py ChangningT55S1A 20180517 - -------------------------------------------------------------------- -""" - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - date = inps.date - scratchDir = os.getenv('SCRATCHDIR') - projectDir = scratchDir + '/' + projectName - - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - - slc_dir = projectDir + '/SLC' - down_dir = projectDir + '/DOWNLOAD' - - if not os.path.isdir(slc_dir): - os.mkdir(slc_dir) - - work_dir = slc_dir + '/' + date - if not os.path.isdir(work_dir): - os.mkdir(work_dir) - - os.chdir(work_dir) - - t_date = 't_' + date - - input_files = discover_lt1_inputs(down_dir, date) - write_input_list(t_date, input_files) - raw_files = ut.read_txt2list(t_date) - if len(raw_files) == 0: - raise RuntimeError('No LT-1 inputs found for date: ' + date) - satellite = get_satellite(str(raw_files[0])) - raw_file_list = list(raw_files) - - - - file_num=len(raw_files) - for kk in range(file_num): - - zipfile_ref=str(raw_files[kk]) - outfile_name=zipfile_ref.split('/')[-1] - print(outfile_name) - before_slc = set(glob.glob(work_dir + '/*.slc')) - before_slc_par = set(glob.glob(work_dir + '/*.slc.par')) - before_update = set(glob.glob(work_dir + '/*.slc.update')) - before_update_par = set(glob.glob(work_dir + '/*.slc.update.par')) - call_str = "echo " + zipfile_ref + " >date" - if os.system(call_str) != 0: - raise RuntimeError('Failed to materialize LT-1 input list for date: ' + date) - call_str = 'LT1_import_SLC_from_zipfiles1 date 0 ' - rc = os.system(call_str) - if rc != 0: - raise RuntimeError('LT1_import_SLC_from_zipfiles1 failed for date %s with rc=%s' % (date, rc)) - after_slc = set(glob.glob(work_dir + '/*.slc')) - after_slc_par = set(glob.glob(work_dir + '/*.slc.par')) - after_update = set(glob.glob(work_dir + '/*.slc.update')) - after_update_par = set(glob.glob(work_dir + '/*.slc.update.par')) - new_slc = sorted(after_slc - before_slc) - new_slc_par = sorted(after_slc_par - before_slc_par) - new_update = sorted(after_update - before_update) - new_update_par = sorted(after_update_par - before_update_par) - if not new_slc or not new_slc_par: - raise RuntimeError('LT-1 import produced no SLC outputs for date: ' + date) - if len(new_update) != len(new_update_par): - raise RuntimeError('LT-1 import produced mismatched update SLC outputs for date: ' + date) - bridge_targets = sorted((after_slc_par - before_slc_par) | (after_update_par - before_update_par)) - if bridge_targets: - bridge_result = orbit_bridge.apply_precise_orbit( - date, - bridge_targets, - work_dir=work_dir, - operation_tag='lt1_import', - ) - if bridge_result.get('stdout'): - print(bridge_result['stdout']) - if bridge_result.get('stderr'): - print(bridge_result['stderr']) - - - SLC_Tab = work_dir + '/' + date+'_SLC_Tab' - SLC_Tab_update = work_dir + '/' + date+'_update_SLC_Tab' - SLC_update_list = sorted(glob.glob(work_dir + '/*.slc.update')) - SLC_update_par_list = sorted(glob.glob(work_dir + '/*.slc.update.par')) - SLC_list = sorted(glob.glob(work_dir + '/*.slc')) - SLC_par_list = sorted(glob.glob(work_dir + '/*.slc.par')) - if len(SLC_list) == 0 or len(SLC_par_list) == 0: - raise RuntimeError('No LT-1 SLC outputs were generated for date: ' + date) - if len(SLC_list) != len(SLC_par_list): - raise RuntimeError('LT-1 SLC and SLC parameter counts do not match for date: ' + date) - if len(SLC_update_list) != len(SLC_update_par_list): - raise RuntimeError('LT-1 update SLC and parameter counts do not match for date: ' + date) - - if os.path.isfile(SLC_Tab): - os.remove(SLC_Tab) - - for kk in range(len(SLC_list)): - call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' >> ' + SLC_Tab - if os.system(call_str) != 0: - raise RuntimeError('Failed to write SLC tab for date: ' + date) - call_str = 'echo ' + SLC_update_list[kk] + ' ' + SLC_update_par_list[kk] + ' >> ' + SLC_Tab_update - if os.system(call_str) != 0: - raise RuntimeError('Failed to write update SLC tab for date: ' + date) - call_str = 'SLC_cat_list.py ' + SLC_Tab_update + ' ' + date + '.slc ' + date + '.slc.par ' - if os.system(call_str) != 0: - raise RuntimeError('SLC_cat_list.py failed for date: ' + date) - if not os.path.isfile(work_dir + '/' + date + '.slc.par'): - raise RuntimeError('Final LT-1 concatenated SLC parameter file is missing for date: ' + date) - final_bridge_result = orbit_bridge.apply_precise_orbit( - date, - [work_dir + '/' + date + '.slc.par'], - work_dir=work_dir, - operation_tag='slc_cat_final', - ) - if final_bridge_result.get('stdout'): - print(final_bridge_result['stdout']) - if final_bridge_result.get('stderr'): - print(final_bridge_result['stderr']) - with open(work_dir + '/down2slc.dat', 'w') as f: - f.write('ok\n') - print("Down to SLC for %s is done! " % date) - sys.exit(0) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_cat_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_cat_all.py deleted file mode 100644 index 2c9fc68..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_cat_all.py +++ /dev/null @@ -1,148 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v1.0 ### -### Copy Right (c): 2017, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Email : ymcmrs@gmail.com ### -### Univ. : Central South University & University of Miami ### -################################################################# - -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob - -def check_variable_name(path): - s=path.split("/")[0] - if len(s)>0 and s[0]=="$": - p0=os.getenv(s[1:]) - path=path.replace(path.split("/")[0],p0) - return path - -def read_template(File, delimiter='='): - '''Reads the template file into a python dictionary structure. - Input : string, full path to the template file - Output: dictionary, pysar template content - Example: - tmpl = read_template(KyushuT424F610_640AlosA.template) - tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') - ''' - template_dict = {} - for line in open(File): - line = line.strip() - c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter - if len(c) < 2 or line.startswith('%') or line.startswith('#'): - next #ignore commented lines or those without variables - else: - atrName = c[0] - atrValue = str.replace(c[1],'\n','').split("#")[0].strip() - atrValue = check_variable_name(atrValue) - template_dict[atrName] = atrValue - return template_dict - -def is_number(s): - try: - int(s) - return True - except ValueError: - return False - - -def ras2jpg(input, strTitle): - call_str = "convert " + input + ".ras " + input + ".jpg" - os.system(call_str) - call_str = "convert " + input + ".jpg -resize 250 " + input + ".thumb.jpg" - os.system(call_str) - call_str = "convert " + input + ".jpg -resize 500 " + input + ".bthumb.jpg" - os.system(call_str) - call_str = "$INT_SCR/addtitle2jpg.pl " + input + ".thumb.jpg 14 " + strTitle - os.system(call_str) - call_str = "$INT_SCR/addtitle2jpg.pl " + input + ".bthumb.jpg 24 " + strTitle - os.system(call_str) - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close() - -def write_template(File, Str): - f = open(File,'a') - f.write(Str) - f.close() - -def write_run_coreg_all(projectName,master,slavelist,workdir): - scratchDir = os.getenv('SCRATCHDIR') - projectDir = scratchDir + '/' + projectName - run_coreg_all = projectDir + "/run_coreg_all" - f_coreg = open(run_coreg_all,'w') - - for kk in range(len(slavelist)): - str_coreg = "GenOff_Gamma.py " + projectName + ' ' + master + ' ' + slavelist[kk] + ' ' + workdir + '\n' - f_coreg.write(str_coreg) - f_coreg.close() - - -def usage(): - print(''' -****************************************************************************************************** - - SLC cat for time-series of SAR datasets. - - usage: - - down2slc_cat_all.py projectName - - e.g. down2slc_cat_all.py PacayaERST163A - down2slc_cat_all.py PacayaEnvT163A - -******************************************************************************************************* - ''') - -def main(argv): - - if len(sys.argv)==2: - if argv[0] in ['-h','--help']: usage(); sys.exit(1) - else: projectName=sys.argv[1] - else: - usage();sys.exit(1) - - if 'ERS' in projectName: - call_str='Down2SLC_ERS_Cat_All.py ' + projectName - else: - call_str='Down2SLC_ASAR_Cat_All.py ' + projectName - - os.system(call_str) - - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - - - - - - - - - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_cat_sen.py b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_cat_sen.py deleted file mode 100644 index 20e331c..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_cat_sen.py +++ /dev/null @@ -1,216 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob -import argparse -import csv - -from pyint import _utils as ut - - -def get_s1_date(raw_file): - file0 = os.path.basename(raw_file) - date = file0[17:25] - return date - -def get_s1_time(raw_file): - file0 = os.path.basename(raw_file) - times= file0[26:32] - return times - -def get_satellite(raw_file): - if 'S1A_IW_SLC_' in raw_file: - s0 = 'A' - else: - s0 = 'B' - - return s0 - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Generate SLC from Sentinel-1 raw data with orbit correction using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName', help='project name. e.g., ChangningT55S1A') - parser.add_argument('date',help='date to be processed. e.g., 20180101') - - inps = parser.parse_args() - - return inps - - -INTRODUCTION = ''' -------------------------------------------------------------------- - - Cat two SLC from Sentinel-1 raw data using S1_import_SLC_from_zipfiles and SLC_cat_S1_TOPS with orbit correction. - [Precise orbit data will be downloaded automatically] -''' - -EXAMPLE = """Usage: - - down2slc_sen.py projectName date - - down2slc_sen.py ChangningT55S1A 20180517 - -------------------------------------------------------------------- -""" - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - date = inps.date - scratchDir = os.getenv('SCRATCHDIR') - projectDir = scratchDir + '/' + projectName - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - slc_dir = projectDir + '/SLC' - down_dir = projectDir + '/DOWNLOAD' - #opod_dir = projectDir + '/OPOD' - opod_dir = os.getenv('OPOD_DIR') - - - #if not os.path.isdir(opod_dir): - # os.mkdir(opod_dir) - - call_str = " eof --save-dir " + opod_dir + " -p " + down_dir - os.system(call_str) - if not os.path.isdir(slc_dir): - os.mkdir(slc_dir) - - work_dir = slc_dir + '/' + date - if not os.path.isdir(work_dir): - os.mkdir(work_dir) - - os.chdir(work_dir) - - t_date = 't_' + date - - call_str = 'ls ' + down_dir + '/S1*' + date + '* > ' + t_date - os.system(call_str) - - start_swath = templateDict['start_swath'] - end_swath = templateDict['end_swath'] - -# if (start_swath == '1') and (end_swath == '1'): -# k_swath = '1' -# elif (start_swath == '2') and (end_swath == '2'): -# k_swath = '2' -# elif (start_swath == '3') and (end_swath == '3'): -# k_swath = '3' -# elif (start_swath == '1') and (end_swath == '2'): -# k_swath = '4' -# elif (start_swath == '2') and (end_swath == '3'): -# k_swath = '5' -# elif (start_swath == '2') and (end_swath == '3'): -# k_swath = '-' - k_swath = ut.get_sardata_swath(start_swath,end_swath) - - raw_files = ut.read_txt2list(t_date) - satellite = get_satellite(str(raw_files[0])) - #orbit_file = ut.download_s1_orbit(date,opod_dir,satellite=satellite) - raw_file_list = glob.glob(down_dir + '/S1*' + date + '*.zip') - file_num=len(raw_files) - call_str = 'S1_BURST_tab_from_zipfile.py 3 --zip_ref_list ' + t_date + ' --zip_list ' + t_date - os.system(call_str) - - for kk in range(file_num): - zipfile_ref=str(raw_files[kk]) - outfile_name=zipfile_ref.split('/')[-1].split('.')[0] - burst_number_table_ref=outfile_name + '.BURST_tab' - #call_str = 'S1_BURST_tab_from_zipfile ' + t_date + ' ' + str(raw_files[kk]) + ' - 1' - #os.system(call_str) - #call_str = 'S1_import_SLC_from_zipfiles ' + t_date + ' ' + burst_number_table_ref + ' vv 0 ' + k_swath + ' ' + opod_dir + ' 1 1 ' - call_str = 'read_S1_TOPS_SLC.py ' + zipfile_ref + ' --burst_sel ' + burst_number_table_ref + ' --pol vv --root_name ' + date + ' --sw_start ' + start_swath + ' --swn ' + end_swath + ' --OPOD_dir ' + opod_dir - os.system(call_str) - os.chdir(work_dir) - call_str = "rename 's/vv.iw1.slc/iw1_" + str(kk)+".slc/g' *" - os.system(call_str) - call_str = "rename 's/vv.iw2.slc/iw2_" + str(kk)+".slc/g' *" - os.system(call_str) - call_str = "rename 's/vv.iw3.slc/iw3_" + str(kk)+".slc/g' *" - os.system(call_str) - SLC_Tab = work_dir + '/' + date + '_SLC_Tab' +str(kk) - SLC_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc')) - SLC_par_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc.par')) - TOP_par_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc.tops_par')) - - - cat_str = 'touch slc_list slc_par_list top_par_list' - os.system(call_str) - list_num=len(SLC_list) - for tt in range(list_num): - call_str = 'echo ' + SLC_list[tt] + ' >> slc_list' - os.system(call_str) - call_str = 'echo ' + SLC_par_list[tt] + ' >> slc_par_list' - os.system(call_str) - call_str = 'echo ' + TOP_par_list[tt] + ' >> top_par_list' - os.system(call_str) - call_str = ' paste slc_list slc_par_list top_par_list > ' + SLC_Tab - os.system(call_str) - call_str = 'rm slc_list slc_par_list top_par_list' - os.system(call_str) - SLC_list = sorted(glob.glob(work_dir + '/*iw*.slc')) - SLC_Tab = work_dir + '/' + date+'_SLC_Tab' - subswath=len(SLC_list)/2 - for kk in range(int(subswath)): - swath=kk+1 - call_str = 'echo ' + date + '.IW' + str(swath) + '.slc ' + date + '.IW' + str(swath) + '.slc.par ' + date + '.IW' + str(swath) + '.slc.TOPS_par ' + '>> ' + SLC_Tab - os.system(call_str) - call_str = 'SLC_cat_S1_TOPS ' +SLC_Tab+'0 ' + SLC_Tab +'1 ' + SLC_Tab - os.system(call_str) - - #call_str = "rename vv.slc.iw1 IW1.slc * " - #os.system(call_str) - #call_str = "rename 's/vv.slc.iw2/IW2.slc/g' *" - #call_str = "rename vv.slc.iw2 IW2.slc * " - #os.system(call_str) - #call_str = "rename 's/vv.slc.iw3/IW3.slc/g' *" - #call_str = "rename vv.slc.iw3 IW3.slc * " - #os.system(call_str) - #call_str = 'rm *.iw* *SLC_Tab*' - #os.system(call_str) - - SLC_Tab = work_dir + '/' + date+'_SLC_Tab' - SLC_list = sorted(glob.glob(work_dir + '/*IW*.slc')) - SLC_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.par')) - TOP_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.TOPS_par')) - - #call_str = 'rm *iw* *vv.SLC_tab' - #os.system(call_str) - if os.path.isfile(SLC_Tab): - os.remove(SLC_Tab) - - for kk in range(len(SLC_list)): - call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' >> ' + SLC_Tab - os.system(call_str) - - BURST = SLC_par_list[kk].replace('slc.par','burst.par') - call_str = 'SLC_burst_corners ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' > ' +BURST - os.system(call_str) - call_str = "echo 'SLC has already down' >down2slc.dat" - os.system(call_str) - print("Down to SLC for %s is done! " % date) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_csk.py b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_csk.py deleted file mode 100644 index a9219cf..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_csk.py +++ /dev/null @@ -1,199 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.0 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Email : ymcmrs@gmail.com ### -### Univ. : Now at KAUST ### -################################################################# - -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob -import argparse -import linecache -import datetime - -def StrNum(S): - S = str(S) - if len(S)==1: - S='0' +S - return S - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close() - - -def check_ERS_par(SAR_IM_0P): - Name = os.path.basename(SAR_IM_0P) - ff = Name.split('.')[1] - date = (Name.split('SAR_IM__0PWDSI')[1]).split('_')[0] - - if ff == 'E1': - par = 'ERS1_ESA.par' - antenna = 'ERS1_antenna.gain' - orbdir = os.getenv('ERS1ORBDIR') - elif ff == 'E2': - par = 'ERS2_ESA.par' - antenna = 'ERS2_antenna.gain' - orbdir = os.getenv('ERS2ORBDIR') - else: - print('Invalid input SAR_IM_0P file.') - sys.exit(1) - - return par, antenna, orbdir, date - -######################################################################### - -INTRODUCTION = ''' -############################################################################# - Copy Right(c): 2017-2020, Yunmeng Cao @PyINT v2.0 - - Generate SLC for CSK raw data (with **.tar.gz format). - -''' - -EXAMPLE = ''' - Usage: - down2slc_csk.py projectName date - - Examples: - down2slc_csk.py KilaueaT10CskA 20180419 - -############################################################################## -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Generate SLC for ERS raw data with ENVISAT format.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='project name.') - parser.add_argument('date',help='to be processed date.') - - inps = parser.parse_args() - return inps - -################################################################################ - - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - date = inps.date - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - - downDir = scratchDir + '/' + projectName + "/DOWNLOAD" - slcDir = scratchDir + '/' + projectName + "/SLC" - rslcDir = scratchDir + '/' + projectName + "/RSLC" - - if not os.path.isdir(slcDir): - call_str = 'mkdir ' + slcDir - os.system(call_str) - - slcDir1 = slcDir + '/' + date - if not os.path.isdir(slcDir1): - call_str = 'mkdir ' + slcDir1 - os.system(call_str) - - tar0 = glob.glob(downDir + '/*' + date + '*.tar.gz')[0] - rawDir = downDir + '/raw_' + date - - if not os.path.isdir(rawDir): - call_str = 'mkdir ' + downDir + '/raw_' + date - os.system(call_str) - - call_str = 'tar -xzf ' + tar0 + ' -C ' + rawDir - os.system(call_str) - - par = rawDir + '/' + date + '.sar_par' - raw = rawDir + '/' + date + '.raw' - azsp = rawDir + '/' + date + '.azsp' - dop = rawDir + '/' + date + '.dop' - rspec = rawDir + '/' + date + '.rspec' - rc = rawDir + '/' + date + '.rc' - autof = rawDir + '/' + date + '.autof' - dop_ambig = rawDir + '/' + date + '.dop_ambig' - - h5file = glob.glob(rawDir + '/*.h5')[0] - - pslc_par = rawDir + '/' + 'p' + date + '.slc.par' - slc_par = slcDir1 + '/' + date + '.slc.par' - slc = slcDir1 + '/' +date + '.slc' - mli_par = slcDir1 + '/' +date + '.mli.par' - mli = slcDir1 + '/' + date + '.mli' - - call_str = 'CS_proc ' + h5file + ' ' + par + ' ' + pslc_par + ' ' + raw + ' - - ' - os.system(call_str) - - cal_str = 'dop_ambig ' + par + ' ' + pslc_par + ' ' + raw + ' 2 - ' + dop_ambig - os.system(call_str) - - call_str = 'azsp_IQ ' + par + ' ' + pslc_par + ' ' + raw + ' ' + azsp - os.system(call_str) - - call_str = 'doppler ' + par + ' ' + pslc_par + ' ' + raw + ' ' + dop - os.system(call_str) - - call_str = 'rspec_IQ ' + par + ' ' + pslc_par + ' ' + raw + ' ' + rspec - os.system(call_str) - - call_str = 'pre_rc ' + par + ' ' + pslc_par + ' ' + raw + ' ' + rc - os.system(call_str) - - call_str = 'autof ' + par + ' ' + pslc_par + ' ' + rc + ' ' + autof + ' 2.0 ' - os.system(call_str) - - call_str = 'autof ' + par + ' ' + pslc_par + ' ' + rc + ' ' + autof + ' 2.0 ' - os.system(call_str) - - call_str = 'az_proc ' + par + ' ' + pslc_par + ' ' + rc + ' ' + slc + ' - 1 ' + ' - 0 2.120 ' - os.system(call_str) - - call_str = 'par_MSP ' + par + ' ' + pslc_par + ' ' + slc_par + ' 1' - os.system(call_str) - - call_str = 'multi_look ' + slc + ' ' + slc_par + ' ' + mli + ' ' + mli_par + ' 20 16' - os.system(call_str) - - - Width = UseGamma(mli_par, 'read', 'range_samples: ') - call_str = 'raspwr ' + mli + ' ' + Width - os.system(call_str) - - - print("Generate SLC for %s is done." % date) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_csk_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_csk_all.py deleted file mode 100644 index 7bd2d8e..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_csk_all.py +++ /dev/null @@ -1,119 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import getopt -import time -import glob -import argparse - -import subprocess -from pyint import _utils as ut - - -def get_csk_date(raw_file): - file0 = os.path.basename(raw_file) - date = file0[27:35] - return date - - -def work(data0): - cmd = data0[0] - err_txt = data0[1] - p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) - stdout = p.stdout - stderr = p.stderr - - if type(stderr) == bytes: - aa=stderr.decode("utf-8") - else: - aa = stderr - - if aa: - str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' - #print(aa) - with open(err_txt, 'a') as f: - f.write(str0) - f.write(aa) - f.write('\n') - - return -######################################################################### - -INTRODUCTION = ''' -------------------------------------------------------------------- - - Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA. - -''' - -EXAMPLE = ''' - Usage: - down2slc_sen_all.py projectName - down2slc_sen_all.py projectName --parallel 4 - -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') - - inps = parser.parse_args() - return inps - - -def main(argv): - start_time = time.time() - inps = cmdLineParse() - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - projectDir = scratchDir + '/' + projectName - downDir = scratchDir + '/' + projectName + "/DOWNLOAD" - raw_file_list = glob.glob(downDir + '/CSK*.tar.gz') - - # get the burst number table of the mater date - - date_list = [] - for kk in range(len(raw_file_list)): - date0 = get_csk_date(os.path.basename(raw_file_list[kk])) - date_list.append(date0) - - date_list = set(date_list) - date_list = sorted(date_list) - - print('Date to be processed:') - for k0 in date_list: - print(k0) - - err_txt = scratchDir + '/' + projectName + '/down2slc_csk_all.err' - if os.path.isfile(err_txt): os.remove(err_txt) - - data_para = [] - for i in range(len(date_list)): - cmd0 = ['down2slc_csk.py',projectName,date_list[i]] - data0 = [cmd0,err_txt] - data_para.append(data0) - - ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) - os.chdir(downDir) - print("Down to SLC for project %s is done! " % projectName) - ut.print_process_time(start_time, time.time()) - - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_sen.py b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_sen.py deleted file mode 100644 index 953fe87..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_sen.py +++ /dev/null @@ -1,167 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob -import argparse - -from pyint import _utils as ut - - -def get_s1_date(raw_file): - file0 = os.path.basename(raw_file) - date = file0[17:25] - return date - -def get_satellite(raw_file): - if 'S1A_IW_SLC_' in raw_file: - s0 = 'A' - else: - s0 = 'B' - - return s0 - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Generate SLC from Sentinel-1 raw data with orbit correction using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName', help='project name. e.g., ChangningT55S1A') - parser.add_argument('date',help='date to be processed. e.g., 20180101') - - inps = parser.parse_args() - - return inps - - -INTRODUCTION = ''' -------------------------------------------------------------------- - - Generate SLC from Sentinel-1 raw data using S1_import_SLC_from_zipfiles with orbit correction. - [Precise orbit data will be downloaded automatically] -''' - -EXAMPLE = """Usage: - - down2slc_sen.py projectName date - - down2slc_sen.py ChangningT55S1A 20180517 - -------------------------------------------------------------------- -""" - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - date = inps.date - scratchDir = os.getenv('SCRATCHDIR') - projectDir = scratchDir + '/' + projectName - - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - - slc_dir = projectDir + '/SLC' - down_dir = projectDir + '/DOWNLOAD' - #opod_dir = projectDir + '/OPOD' - opod_dir = os.getenv('OPOD_DIR') - if not os.path.isdir(slc_dir): - os.mkdir(slc_dir) - # if not os.path.isdir(opod_dir): - # os.mkdir(opod_dir) - - work_dir = slc_dir + '/' + date - if not os.path.isdir(work_dir): - os.mkdir(work_dir) - #if not os.path.isdir(opod_dir): - # os.mkdir(opod_dir) - - - call_str = " eof --save-dir " + opod_dir + " -p " + down_dir - os.system(call_str) - - os.chdir(work_dir) - - t_date = 't_' + date - - call_str = 'ls ' + down_dir + '/S1*' + date + '*.zip > ' + t_date - os.system(call_str) - - start_swath = templateDict['start_swath'] - end_swath = templateDict['end_swath'] - -# if (start_swath == '1') and (end_swath == '1'): -# k_swath = '1' -# elif (start_swath == '2') and (end_swath == '2'): -# k_swath = '2' -# elif (start_swath == '3') and (end_swath == '3'): -# k_swath = '3' -# elif (start_swath == '1') and (end_swath == '2'): -# k_swath = '4' -# elif (start_swath == '2') and (end_swath == '3'): -# k_swath = '5' -# elif (start_swath == '2') and (end_swath == '3'): -# k_swath = '-' - k_swath = ut.get_sardata_swath(start_swath,end_swath) - - raw_files = ut.read_txt2list(t_date) - satellite = get_satellite(str(raw_files[0])) - #orbit_file = ut.download_s1_orbit(date,opod_dir,satellite=satellite) - zipfile_ref=str(raw_files[0]) - outfile_name=zipfile_ref.split('/')[-1].split('.')[0] - burst_number_table_ref=outfile_name + '.BURST_tab' - - call_str = 'S1_BURST_tab_from_zipfile.py 3 --zip_ref_list ' + t_date + ' --zip_list ' + t_date - os.system(call_str) - - # call_str = 'S1_import_SLC_from_zipfiles ' + t_date + ' ' + burst_number_table_ref + ' vv 0 ' + k_swath - call_str = 'read_S1_TOPS_SLC.py ' + zipfile_ref + ' --burst_sel ' + burst_number_table_ref + ' --pol vv --root_name ' + date + ' --sw_start ' + start_swath + ' --swn ' + end_swath + ' --OPOD_dir ' + opod_dir - os.system(call_str) - - os.chdir(work_dir) - - call_str = "rename 's/vv.iw1.slc/IW1.slc/g' *" - #call_str = "rename vv.slc.iw1 IW1.slc * " - os.system(call_str) - call_str = "rename 's/vv.iw2.slc/IW2.slc/g' *" - #call_str = "rename vv.slc.iw2 IW2.slc * " - os.system(call_str) - call_str = "rename 's/vv.iw3.slc/IW3.slc/g' *" - #call_str = "rename vv.slc.iw3 IW3.slc * " - os.system(call_str) - call_str = "rename 's/tops_par/TOPS_par/g' *.tops_par " - os.system(call_str) - - SLC_Tab = work_dir + '/' + date+'_SLC_Tab' - SLC_list = sorted(glob.glob(work_dir + '/*IW*.slc')) - SLC_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.par')) - TOP_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.TOPS_par')) - - if os.path.isfile(SLC_Tab): - os.remove(SLC_Tab) - - for kk in range(len(SLC_list)): - call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' >> ' + SLC_Tab - os.system(call_str) - - BURST = SLC_par_list[kk].replace('slc.par','burst.par') - call_str = 'SLC_burst_corners ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' > ' +BURST - os.system(call_str) - call_str = "echo 'SLC has already down' >down2slc.dat" - os.system(call_str) - print("Down to SLC for %s is done! " % date) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_sen_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_sen_all.py deleted file mode 100644 index 15c7800..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_sen_all.py +++ /dev/null @@ -1,151 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import getopt -import time -import glob -import argparse - -import subprocess -from pyint import _utils as ut - - -def get_s1_date(raw_file): - file0 = os.path.basename(raw_file) - date = file0[17:25] - return date - - -def work(data0): - cmd = data0[0] - err_txt = data0[1] - p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) - stdout = p.stdout - stderr = p.stderr - - if type(stderr) == bytes: - aa=stderr.decode("utf-8") - else: - aa = stderr - - if aa: - str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' - #print(aa) - with open(err_txt, 'a') as f: - f.write(str0) - f.write(aa) - f.write('\n') - - return -######################################################################### - -INTRODUCTION = ''' -------------------------------------------------------------------- - - Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA. - -''' - -EXAMPLE = ''' - Usage: - down2slc_sen_all.py projectName - down2slc_sen_all.py projectName --parallel 4 - -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') - - inps = parser.parse_args() - return inps - - -def main(argv): - start_time = time.time() - inps = cmdLineParse() - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - projectDir = scratchDir + '/' + projectName - downDir = scratchDir + '/' + projectName + "/DOWNLOAD" - slcDir = scratchDir + '/' + projectName + "/SLC" - raw_file_list = glob.glob(downDir + '/S1*.zip') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - - - date_list = [] - cat_list = [] - for kk in range(len(raw_file_list)): - date0 = get_s1_date(os.path.basename(raw_file_list[kk])) - if date0 not in date_list: - date_list.append(date0) - cat_list.append('0') - else: - cat_list[date_list.index(date0)]='1' -# date_list = set(date_list) -# date_list = sorted(date_list) - - print('Date to be processed:') - for k0 in date_list: - print(k0) - - err_txt = scratchDir + '/' + projectName + '/down2slc_sen_all.err' - if os.path.isfile(err_txt): os.remove(err_txt) - - data_para = [] - for i in range(len(date_list)): - if cat_list[i]=='0': - cmd0 = ['down2slc_sen.py',projectName,date_list[i]] - work_dir = slcDir + '/' + date_list[i] - slc_file0 = work_dir + '/down2slc.dat' - data0 = [cmd0,err_txt] -# data_para.append(data0) - k00 = 0 - if os.path.isfile(slc_file0): - if os.path.getsize(slc_file0) > 0: - k00 = 1 - else: - k00 = 0 - if k00==0: - data_para.append(data0) - else: - cmd0 = ['down2slc_cat_sen.py',projectName,date_list[i]] - work_dir = slcDir + '/' + date_list[i] - slc_file0 = work_dir + '/down2slc.dat' - data0 = [cmd0,err_txt] -# data_para.append(data0) - #data_para.append(data0) - k00 = 0 - if os.path.isfile(slc_file0): - if os.path.getsize(slc_file0) > 0: - k00 = 1 - else: - k00 = 0 - if k00==0: - data_para.append(data0) - ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) - os.chdir(downDir) - print("Down to SLC for project %s is done! " % projectName) - ut.print_process_time(start_time, time.time()) - - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_sen_all_old.py b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_sen_all_old.py deleted file mode 100644 index 755cce1..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_sen_all_old.py +++ /dev/null @@ -1,103 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import getopt -import time -import glob -import argparse - -import subprocess -from pyint import _utils as ut - - -def work(data0): - cmd = data0[0] - err_txt = data0[1] - p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) - stdout = p.stdout - stderr = p.stderr - - if type(stderr) == bytes: - aa=stderr.decode("utf-8") - else: - aa = stderr - - if aa: - str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' - #print(aa) - with open(err_txt, 'a') as f: - f.write(str0) - f.write(aa) - f.write('\n') - - return -######################################################################### - -INTRODUCTION = ''' -------------------------------------------------------------------- - - Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA. - -''' - -EXAMPLE = ''' - Usage: - down2slc_sen_all.py projectName - down2slc_sen_all.py projectName --parallel 4 - -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') - - inps = parser.parse_args() - return inps - - -def main(argv): - start_time = time.time() - inps = cmdLineParse() - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - projectDir = scratchDir + '/' + projectName - downDir = scratchDir + '/' + projectName + "/DOWNLOAD" - raw_file_list = glob.glob(downDir + '/S1*.zip') - - slc_dir = scratchDir + '/' + projectName + '/SLC' - if not os.path.isdir(slc_dir): - os.mkdir(slc_dir) - - err_txt = scratchDir + '/' + projectName + '/down2slc_sen_all.err' - if os.path.isfile(err_txt): os.remove(err_txt) - - data_para = [] - for i in range(len(raw_file_list)): - cmd0 = ['down2slc_sen.py',raw_file_list[i],slc_dir] - data0 = [cmd0,err_txt] - data_para.append(data0) - - ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) - os.chdir(downDir) - print("Down to SLC for project %s is done! " % projectName) - ut.print_process_time(start_time, time.time()) - - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_sen_old.py b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_sen_old.py deleted file mode 100644 index af9fb03..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_sen_old.py +++ /dev/null @@ -1,180 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob -import argparse - -from pyint import _utils as ut - - -def get_s1_date(raw_file): - file0 = os.path.basename(raw_file) - date = file0[17:25] - return date - -def get_satellite(raw_file): - if 'S1A_IW_SLC_' in raw_file: - s0 = 'A' - else: - s0 = 'B' - - return s0 - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Generate SLC from Sentinel-1 raw data with orbit correction using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('s1_raw', help='raw Sentinel-1 file. e.g., S1A*.zip') - parser.add_argument('root_path',help='root path for saving the SLC files') - - inps = parser.parse_args() - - return inps - - -INTRODUCTION = ''' -------------------------------------------------------------------- - - Generate SLC from Sentinel-1 raw data with orbit correction using GAMMA. - [Precise orbit data will be downloaded automatically] -''' - -EXAMPLE = """Usage: - - down2slc_sen.py S1A_IW_SLC_XXXX.zip /Yunmeng/S1_test - -------------------------------------------------------------------- -""" - -def main(argv): - - inps = cmdLineParse() - raw_file = inps.s1_raw - root_dir = inps.root_path - satellite = get_satellite(raw_file) - date = get_s1_date(raw_file) - if not os.path.isdir(root_dir): - os.mkdir(root_dir) - - slc_dir = root_dir + '/' + date - if not os.path.isdir(slc_dir): - os.mkdir(slc_dir) - - if len(os.path.dirname(raw_file))==0: - raw_file_dir = os.getcwd() - else: - raw_file_dir = os.path.dirname(raw_file) - - raw_dir = raw_file.replace('.zip','.SAFE') - -# MEASURE = glob.glob(measureDir + '/*vv*tiff') -# ANNOTAT = glob.glob(annotatDir + '/*vv*xml' ) -# CALIBRA = glob.glob(calibraDir+'/calibration*vv*') -# NOISE = glob.glob(calibraDir+'/noise*vv*') - - SLC_Tab = slc_dir + '/' + date+'_SLC_Tab' - TEST = slc_dir + '/' + date + '.IW3.slc.par' - k0 = 0 - if os.path.isfile(TEST): - if os.path.getsize(TEST) > 0: - k0 = 1 - - if k0==0: - if not os.path.isdir(raw_dir): - call_str = 'unzip ' + raw_file + ' -d ' + raw_file_dir - os.system(call_str) - - measureDir = raw_dir + '/measurement' - annotatDir = raw_dir + '/annotation' - calibraDir = raw_dir + '/annotation/calibration' - MM = glob.glob(measureDir + '/*vv*tiff') - - if os.path.isfile(SLC_Tab): - os.remove(SLC_Tab) - for kk in range(len(MM)): - SLC = slc_dir + '/' + date + '.IW' + str(kk+1)+'.slc' - SLCPar = slc_dir + '/' + date + '.IW' + str(kk+1)+'.slc.par' - TOPPar = slc_dir + '/' + date + '.IW' + str(kk+1)+'.slc.TOPS_par' - BURST = slc_dir + '/' + date + '.IW' + str(kk+1)+'.burst.par' - - if os.path.isfile(BURST): - os.remove(BURST) - call_str = 'echo ' + SLC + ' ' + SLCPar + ' ' + TOPPar + ' >> ' + SLC_Tab - os.system(call_str) - - MEASURE = glob.glob(measureDir + '/*iw' + str(kk+1) + '*vv*tiff') - ANNOTAT = glob.glob(annotatDir + '/*iw' + str(kk+1) + '*vv*xml' ) - CALIBRA = glob.glob(calibraDir+'/calibration*'+ 'iw' + str(kk+1) + '*vv*') - NOISE = glob.glob(calibraDir+'/noise*' + 'iw' + str(kk+1) + '*vv*') - - #call_str = 'S1_burstloc ' + ANNOTAT[0] + '> ' +BURST - #os.system(call_str) - - if not os.path.isfile(NOISE[0]): - call_str = 'par_S1_SLC ' + MEASURE[0] + ' ' + ANNOTAT[0] + ' ' + CALIBRA[0] + ' - ' + SLCPar + ' ' + SLC + ' ' + TOPPar - else: - call_str = 'par_S1_SLC ' + MEASURE[0] + ' ' + ANNOTAT[0] + ' ' + CALIBRA[0] + ' ' + NOISE[0] + ' ' + SLCPar + ' ' + SLC + ' ' + TOPPar - - #if int(date) > 180311: - # call_str = 'par_S1_SLC ' + MEASURE[0] + ' ' + ANNOTAT[0] + ' ' + CALIBRA[0] + ' - ' + SLCPar + ' ' + SLC + ' ' + TOPPar - #else: - # call_str = 'par_S1_SLC ' + MEASURE[0] + ' ' + ANNOTAT[0] + ' ' + CALIBRA[0] + ' ' + NOISE[0] + ' ' + SLCPar + ' ' + SLC + ' ' + TOPPar - - os.system(call_str) - - call_str = 'SLC_burst_corners ' + SLCPar + ' ' + TOPPar + ' > ' +BURST - os.system(call_str) - - # orbit correction - slc_pars = glob.glob(slc_dir + '/*.IW*.slc.par') - orbit_file0 = ut.download_s1_orbit(date,slc_dir,satellite=satellite) - orbit_file = slc_dir + '/' + orbit_file0 - - for i in range(len(slc_pars)): - call_str = 'S1_OPOD_vec ' + slc_pars[i] + ' ' + orbit_file - os.system(call_str) - - # generate amp file for check image quality - #TSLC = slc_dir + '/' + date + '.slc' - #TSLCPar = slc_dir + '/' + date + '.slc.par' - - #TMLI = slc_dir + '/' + date + '_40rlks.amp' - #TMLIPar = slc_dir + '/' + date + '_40rlks.amp.par' - - #call_str = 'SLC_mosaic_S1_TOPS ' + SLC_Tab + ' ' + TSLC + ' ' + TSLCPar + ' 10 2' - #os.system(call_str) - - #call_str = 'multi_look ' + TSLC + ' ' + TSLCPar + ' ' + TMLI + ' ' + TMLIPar + ' 40 8' - #os.system(call_str) - - #nWidth = ut.read_gamma_par(TMLIPar, 'read','range_samples:') - #call_str = 'raspwr ' + TMLI + ' ' + nWidth + ' - - - - - - - ' - #os.system(call_str) - - if os.path.isdir(raw_dir): - call_str = 'rm -rf ' + raw_dir - os.system(call_str) - - print("Down to SLC for %s is done! " % date) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/download_ers_deos.py b/.codex_tmp/pyint_variants/no_rescue/pyint/download_ers_deos.py deleted file mode 100644 index 980d752..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/download_ers_deos.py +++ /dev/null @@ -1,171 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v1.0 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Email : ymcmrs@gmail.com ### -### Univ. : Now at KAUST ### -################################################################# - -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob -import argparse -import linecache - -def StrNum(S): - S = str(S) - if len(S)==1: - S='0' +S - return S - -def StrNum1(S): - S=str(S) - if len(S)==1: - S='00' +S - elif len(S)==2: - S = '0' +S - else: - S = S - return S - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close() - -def print_progress(iteration, total, prefix='calculating:', suffix='complete', decimals=1, barLength=50, elapsed_time=None): - """Print iterations progress - Greenstick from Stack Overflow - Call in a loop to create terminal progress bar - @params: - iteration - Required : current iteration (Int) - total - Required : total iterations (Int) - prefix - Optional : prefix string (Str) - suffix - Optional : suffix string (Str) - decimals - Optional : number of decimals in percent complete (Int) - barLength - Optional : character length of bar (Int) - elapsed_time- Optional : elapsed time in seconds (Int/Float) - - Reference: http://stackoverflow.com/questions/3173320/text-progress-bar-in-the-console - """ - filledLength = int(round(barLength * iteration / float(total))) - percents = round(100.00 * (iteration / float(total)), decimals) - bar = '#' * filledLength + '-' * (barLength - filledLength) - if elapsed_time: - sys.stdout.write('%s [%s] %s%s %s %s secs\r' % (prefix, bar, percents, '%', suffix, int(elapsed_time))) - else: - sys.stdout.write('%s [%s] %s%s %s\r' % (prefix, bar, percents, '%', suffix)) - sys.stdout.flush() - if iteration == total: - print("\n") - - ''' - Sample Useage: - for i in range(len(dateList)): - print_progress(i+1,len(dateList)) - ''' - return - - -######################################################################### - -INTRODUCTION = ''' -############################################################################# - Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 - - Download precise ERS orbit data from Delft Institute for Earth-Oriented Space Research (http://www.deos.tudelft.nl/). - -''' - -EXAMPLE = ''' - Usage: - download_ers_deos.py ERS1 - download_ers_deos.py ERS2 - -############################################################################## -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Download (and correct) precise ERS orbit data.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('sar',help='project name') - - inps = parser.parse_args() - return inps - -################################################################################ - - -def main(argv): - - total = time.time() - inps = cmdLineParse() - sar = inps.sar - PATH = os.getcwd() - listfile = 'arclist' - - if sar == 'ERS1': - Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-1/dgm-e04/' - Url_list = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-1/dgm-e04/arclist' - num_min = 6 - num_max = 511 - N_num = 506 - if not os.path.isfile(listfile): - call_str = 'wget -q ' + Url_list - os.system(call_str) - - elif sar =='ERS2': - Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-2/dgm-e04/' - Url_list = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-2/dgm-e04/arclist' - num_min = 3 - num_max = 867 - N_num = 865 - if not os.path.isfile(listfile): - call_str = 'wget -q ' + Url_list - os.system(call_str) - - else: - print('SAR name is invalid!') - sys.exit(1) - - for i in range(N_num): - kk = num_min + i - kk=int(kk) - SS = Url + 'ODR.' + StrNum1(kk) - print_progress(i+1, N_num, prefix='DEFT_vec: ', suffix=StrNum1(kk)) - - call_str = 'wget -q --no-check-certificate ' + SS + ' -P ' + PATH - os.system(call_str) - - - print("Download precise DEFT orbital data for %s is done." % sar) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/downloader_gmtchina.py b/.codex_tmp/pyint_variants/no_rescue/pyint/downloader_gmtchina.py deleted file mode 100644 index f8015fa..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/downloader_gmtchina.py +++ /dev/null @@ -1,388 +0,0 @@ -# -*- coding: utf-8 -* -#''' -#This code is used to download image from google and amap - -#@date : 2020-3-13 -#@author: Zheng Jie -#@E-mail: zhengjie9510@qq.com - -#Modified by GMT-china.org -#''' - -import sys -import io -import math -import multiprocessing -import time -import urllib.request as ur -from math import floor, pi, log, tan, atan, exp -from threading import Thread - -import PIL.Image as pil -import cv2 -import numpy as np -from osgeo import gdal, osr - - -# ------------------Interchange between WGS-84 and Web Mercator------------------------- -# WGS-84 to Web Mercator -def wgs_to_mercator(x, y): - y = 85.0511287798 if y > 85.0511287798 else y - y = -85.0511287798 if y < -85.0511287798 else y - - x2 = x * 20037508.34 / 180 - y2 = log(tan((90 + y) * pi / 360)) / (pi / 180) - y2 = y2 * 20037508.34 / 180 - return x2, y2 - - -# Web Mercator to WGS-84 -def mercator_to_wgs(x, y): - x2 = x / 20037508.34 * 180 - y2 = y / 20037508.34 * 180 - y2 = 180 / pi * (2 * atan(exp(y2 * pi / 180)) - pi / 2) - return x2, y2 - - -# -------------------------------------------------------------------------------------- - -# -----------------Interchange between GCJ-02 to WGS-84--------------------------- -# All public geographic data in mainland China need to be encrypted with GCJ-02, introducing random bias -# This part of the code is used to remove the bias -def transformLat(x, y): - ret = -100.0 + 2.0 * x + 3.0 * y + 0.2 * y * y + 0.1 * x * y + 0.2 * math.sqrt(abs(x)) - ret += (20.0 * math.sin(6.0 * x * math.pi) + 20.0 * math.sin(2.0 * x * math.pi)) * 2.0 / 3.0 - ret += (20.0 * math.sin(y * math.pi) + 40.0 * math.sin(y / 3.0 * math.pi)) * 2.0 / 3.0 - ret += (160.0 * math.sin(y / 12.0 * math.pi) + 320 * math.sin(y * math.pi / 30.0)) * 2.0 / 3.0 - return ret - - -def transformLon(x, y): - ret = 300.0 + x + 2.0 * y + 0.1 * x * x + 0.1 * x * y + 0.1 * math.sqrt(abs(x)) - ret += (20.0 * math.sin(6.0 * x * math.pi) + 20.0 * math.sin(2.0 * x * math.pi)) * 2.0 / 3.0 - ret += (20.0 * math.sin(x * math.pi) + 40.0 * math.sin(x / 3.0 * math.pi)) * 2.0 / 3.0 - ret += (150.0 * math.sin(x / 12.0 * math.pi) + 300.0 * math.sin(x / 30.0 * math.pi)) * 2.0 / 3.0 - return ret - - -def delta(lat, lon): - ''' - Krasovsky 1940 - // - // a = 6378245.0, 1/f = 298.3 - // b = a * (1 - f) - // ee = (a^2 - b^2) / a^2; - ''' - a = 6378245.0 # a: Projection factor of satellite ellipsoidal coordinates projected onto a flat map coordinate system - ee = 0.00669342162296594323 # ee: Eccentricity of ellipsoid - dLat = transformLat(lon - 105.0, lat - 35.0) - dLon = transformLon(lon - 105.0, lat - 35.0) - radLat = lat / 180.0 * math.pi - magic = math.sin(radLat) - magic = 1 - ee * magic * magic - sqrtMagic = math.sqrt(magic) - dLat = (dLat * 180.0) / ((a * (1 - ee)) / (magic * sqrtMagic) * math.pi) - dLon = (dLon * 180.0) / (a / sqrtMagic * math.cos(radLat) * math.pi) - return {'lat': dLat, 'lon': dLon} - - -def outOfChina(lat, lon): - if (lon < 72.004 or lon > 137.8347): - return True - if (lat < 0.8293 or lat > 55.8271): - return True - return False - - -def gcj_to_wgs(gcjLon, gcjLat): - if outOfChina(gcjLat, gcjLon): - return (gcjLon, gcjLat) - d = delta(gcjLat, gcjLon) - return (gcjLon - d["lon"], gcjLat - d["lat"]) - - -def wgs_to_gcj(wgsLon, wgsLat): - if outOfChina(wgsLat, wgsLon): - return wgsLon, wgsLat - d = delta(wgsLat, wgsLon) - return wgsLon + d["lon"], wgsLat + d["lat"] - - -# -------------------------------------------------------------- - -# --------------------------------------------------------- -# Get tile coordinates in Google Maps based on latitude and longitude of WGS-84 -def wgs_to_tile(j, w, z): - ''' - Get google-style tile cooridinate from geographical coordinate - j : Longittude - w : Latitude - z : zoom - ''' - isnum = lambda x: isinstance(x, int) or isinstance(x, float) - if not (isnum(j) and isnum(w)): - raise TypeError("j and w must be int or float!") - - if not isinstance(z, int) or z < 0 or z > 22: - raise TypeError("z must be int and between 0 to 22.") - - if j < 0: - j = 180 + j - else: - j += 180 - j /= 360 # make j to (0,1) - - w = 85.0511287798 if w > 85.0511287798 else w - w = -85.0511287798 if w < -85.0511287798 else w - w = log(tan((90 + w) * pi / 360)) / (pi / 180) - w /= 180 # make w to (-1,1) - w = 1 - (w + 1) / 2 # make w to (0,1) and left top is 0-point - - num = 2 ** z - x = floor(j * num) - y = floor(w * num) - return x, y - - -def pixls_to_mercator(zb): - # Get the web Mercator projection coordinates of the four corners of the area according to the four corner coordinates of the tile - inx, iny = zb["LT"] # left top - inx2, iny2 = zb["RB"] # right bottom - length = 20037508.3427892 - sum = 2 ** zb["z"] - LTx = inx / sum * length * 2 - length - LTy = -(iny / sum * length * 2) + length - - RBx = (inx2 + 1) / sum * length * 2 - length - RBy = -((iny2 + 1) / sum * length * 2) + length - - # LT=left top,RB=right buttom - # Returns the projected coordinates of the four corners - res = {'LT': (LTx, LTy), 'RB': (RBx, RBy), - 'LB': (LTx, RBy), 'RT': (RBx, LTy)} - return res - - -def tile_to_pixls(zb): - # Tile coordinates are converted to pixel coordinates of the four corners - out = {} - width = (zb["RT"][0] - zb["LT"][0] + 1) * 256 - height = (zb["LB"][1] - zb["LT"][1] + 1) * 256 - out["LT"] = (0, 0) - out["RT"] = (width, 0) - out["LB"] = (0, -height) - out["RB"] = (width, -height) - return out - - -# ----------------------------------------------------------- - -# --------------------------------------------------------- -class Downloader(Thread): - # multiple threads downloader - def __init__(self, index, count, urls, datas): - # index represents the number of threads - # count represents the total number of threads - # urls represents the list of URLs nedd to be downloaded - # datas represents the list of data need to be returned. - super().__init__() - self.urls = urls - self.datas = datas - self.index = index - self.count = count - - def download(self, url): - HEADERS = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36 Edg/88.0.705.68'} - header = ur.Request(url, headers=HEADERS) - err = 0 - while (err < 3): - try: - data = ur.urlopen(header).read() - except: - err += 1 - else: - return data - raise Exception("Bad network link.") - - def run(self): - for i, url in enumerate(self.urls): - if i % self.count != self.index: - continue - self.datas[i] = self.download(url) - - -# --------------------------------------------------------- - -# --------------------------------------------------------- -def getExtent(x1, y1, x2, y2, z, source="amap"): - pos1x, pos1y = wgs_to_tile(x1, y1, z) - pos2x, pos2y = wgs_to_tile(x2, y2, z) - Xframe = pixls_to_mercator( - {"LT": (pos1x, pos1y), "RT": (pos2x, pos1y), "LB": (pos1x, pos2y), "RB": (pos2x, pos2y), "z": z}) - for i in ["LT", "LB", "RT", "RB"]: - Xframe[i] = mercator_to_wgs(*Xframe[i]) - if source == "google_sat": - pass - elif source == "amap" or source == "amap_sat" or source == "google": - for i in ["LT", "LB", "RT", "RB"]: - Xframe[i] = gcj_to_wgs(*Xframe[i]) - else: - raise Exception("Invalid argument: source.") - return Xframe - - -def saveTiff(r, g, b, gt, filePath): - fname_out = filePath - driver = gdal.GetDriverByName('GTiff') - # Create a 3-band dataset - dset_output = driver.Create(fname_out, r.shape[1], r.shape[0], 3, gdal.GDT_Byte) - dset_output.SetGeoTransform(gt) - try: - proj = osr.SpatialReference() - proj.ImportFromEPSG(4326) - dset_output.SetSpatialRef(proj) - except: - print("Error: Coordinate system setting failed") - dset_output.GetRasterBand(1).WriteArray(r) - dset_output.GetRasterBand(2).WriteArray(g) - dset_output.GetRasterBand(3).WriteArray(b) - dset_output.FlushCache() - dset_output = None - print("Image Saved") - - -# --------------------------------------------------------- - -# --------------------------------------------------------- -MAP_URLS = { - "google": "https://mt1.google.com/vt/lyrs=r&x={x}&y={y}&z={z}", - "google_sat": "https://mt1.google.com/vt/lyrs=s&x={x}&y={y}&z={z}", - "amap": "https://webrd02.is.autonavi.com/appmaptile?lang=zh_cn&size=1&scale=1&style=8&x={x}&y={y}&z={z}", - "amap_sat": "https://webst01.is.autonavi.com/appmaptile?style=6&x={x}&y={y}&z={z}" - } - - -def get_url(source, x, y, z, style): # - if source == 'amap': - url = MAP_URLS["amap"].format(x=x, y=y, z=z) - elif source == 'amap_sat': - url = MAP_URLS["amap_sat"].format(x=x, y=y, z=z) - elif source == 'google': - url = MAP_URLS["google"].format(x=x, y=y, z=z) - elif source == 'google_sat': - url = MAP_URLS["google_sat"].format(x=x, y=y, z=z) - else: - raise Exception("Unknown Map Source ! ") - return url - - -def get_urls(x1, y1, x2, y2, z, source, style): - pos1x, pos1y = wgs_to_tile(x1, y1, z) - pos2x, pos2y = wgs_to_tile(x2, y2, z) - lenx = pos2x - pos1x + 1 - leny = pos2y - pos1y + 1 - print("Total tiles number：{x} X {y}".format(x=lenx, y=leny)) - urls = [get_url(source, i, j, z, style) for j in range(pos1y, pos1y + leny) for i in range(pos1x, pos1x + lenx)] - return urls - - -# --------------------------------------------------------- - -# --------------------------------------------------------- -def merge_tiles(datas, x1, y1, x2, y2, z): - pos1x, pos1y = wgs_to_tile(x1, y1, z) - pos2x, pos2y = wgs_to_tile(x2, y2, z) - lenx = pos2x - pos1x + 1 - leny = pos2y - pos1y + 1 - outpic = pil.new('RGBA', (lenx * 256, leny * 256)) - for i, data in enumerate(datas): - picio = io.BytesIO(data) - small_pic = pil.open(picio) - y, x = i // lenx, i % lenx - outpic.paste(small_pic, (x * 256, y * 256)) - print('Tiles merge completed') - return outpic - - -def download_tiles(urls, multi=10): - url_len = len(urls) - datas = [None] * url_len - if multi < 1 or multi > 20 or not isinstance(multi, int): - raise Exception("multi of Downloader shuold be int and between 1 to 20.") - tasks = [Downloader(i, multi, urls, datas) for i in range(multi)] - for i in tasks: - i.start() - for i in tasks: - i.join() - return datas - - -# --------------------------------------------------------- - -# --------------------------------------------------------- -def main(left, top, right, bottom, zoom, filePath, style='s', server="amap"): - """ - Download images based on spatial extent. - - East longitude is positive and west longitude is negative. - North latitude is positive, south latitude is negative. - - Parameters - ---------- - left, top : left-top coordinate, for example (100.361,38.866) - - right, bottom : right-bottom coordinate - - z : zoom - - filePath : File path for storing results, TIFF format - - style : - m for map; - s for satellite; - y for satellite with label; - t for terrain; - p for terrain with label; - h for label; - - source : Google or amap - """ - # --------------------------------------------------------- - # Get the urls of all tiles in the extent - urls = get_urls(left, top, right, bottom, zoom, server, style) - - # Group URLs based on the number of CPU cores to achieve roughly equal amounts of tasks - urls_group = [urls[i:i + math.ceil(len(urls) / multiprocessing.cpu_count())] for i in - range(0, len(urls), math.ceil(len(urls) / multiprocessing.cpu_count()))] - - # Each set of URLs corresponds to a process for downloading tile maps - print('Tiles downloading......') - pool = multiprocessing.Pool(multiprocessing.cpu_count()) - results = pool.map(download_tiles, urls_group) - pool.close() - pool.join() - result = [x for j in results for x in j] - print('Tiles download complete') - - # Combine downloaded tile maps into one map - outpic = merge_tiles(result, left, top, right, bottom, zoom) - outpic = outpic.convert('RGB') - r, g, b = cv2.split(np.array(outpic)) - - # Get the spatial information of the four corners of the merged map and use it for outputting - extent = getExtent(left, top, right, bottom, zoom, server) - gt = (extent['LT'][0], (extent['RB'][0] - extent['LT'][0]) / r.shape[1], 0, extent['LT'][1], 0, - (extent['RB'][1] - extent['LT'][1]) / r.shape[0]) - saveTiff(r, g, b, gt, filePath) - - -# --------------------------------------------------------- -if __name__ == '__main__': - start_time = time.time() - - #main(118.055917, 24.559724, 118.244753, 24.399450, 16, r'google_sat.tif', server="google_sat") - main(float(sys.argv[1]), float(sys.argv[4]), float(sys.argv[2]), float(sys.argv[3]), int(sys.argv[5]), sys.argv[6], server=sys.argv[7]) - - end_time = time.time() - print('lasted a total of {:.2f} seconds'.format(end_time - start_time)) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/extract_s1_bursts.py b/.codex_tmp/pyint_variants/no_rescue/pyint/extract_s1_bursts.py deleted file mode 100644 index 2f03583..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/extract_s1_bursts.py +++ /dev/null @@ -1,297 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import glob -import argparse - -from pyint import _utils as ut - -def common_burst(La_M,La_S): - La_M = [float(i) for i in La_M] - La_S = [float(i) for i in La_S] - Min = max(min(La_M),min(La_S)) - Max = min(max(La_M),max(La_S)) - - M_min = [] - M_max = [] - for xm in La_M: - k0_min = float(xm) - float(Min) - k0_max = float(xm) - float(Max) - M_min.append(abs(k0_min)) - M_max.append(abs(k0_max)) - - M_Index_min = M_min.index(min(M_min)) + 1 - M_Index_max = M_max.index(min(M_max)) + 1 - Mindex =[M_Index_min,M_Index_max] - - S_min = [] - S_max = [] - for xs in La_S: - k0_min = float(xs) - float(Min) - k0_max = float(xs) - float(Max) - S_min.append(abs(k0_min)) - S_max.append(abs(k0_max)) - - S_Index_min = S_min.index(min(S_min)) + 1 - S_Index_max = S_max.index(min(S_max)) + 1 - Sindex =[S_Index_min,S_Index_max] - - return min(Mindex) , max(Mindex), min(Sindex),max(Sindex) - -def get_common_burst(Mslc_dir,Sslc_dir,common_burst_txt): - Mpar_list = sorted(glob.glob(Mslc_dir+'/*.IW*.burst.par')) - Spar_list = sorted(glob.glob(Sslc_dir+'/*.IW*.burst.par')) - if os.path.isfile(common_burst_txt): - os.remove(common_burst_txt) - for kk in range(len(Mpar_list)): - MBURST = Mpar_list[kk] - SBURST = Spar_list[kk] - - Mtt = Sslc_dir + '/' + os.path.basename(MBURST.replace('burst.par','tt0')) - Stt = SBURST.replace('burst.par','tt0') - - call_str = "grep 'Burst:' " + MBURST + ' >' + Mtt - os.system(call_str) - call_str = "grep 'Burst:' " + SBURST + ' >' + Stt - os.system(call_str) - - MM = ut.read_txt2array(Mtt) - SM = ut.read_txt2array(Stt) - La_M = MM[:,2] - La_S = SM[:,2] - - PP = common_burst_Ref(La_M,La_S) - - print('Common bursts of swath' + str(kk+1) + ' : (master) ' + str(PP[0]) + ' ' + str(PP[1]) + ' (slave) ' + str(PP[2]) + ' ' + str(PP[3])) - call_str = 'echo ' + str(PP[0]) + ' ' + str(PP[1]) + ' ' + str(PP[2]) + ' ' + str(PP[3]) + ' >>' + common_burst_txt - os.system(call_str) - - return - -def common_burst_Ref(La_M,La_S): - Min = max(min(La_M),min(La_S)) - Max = min(max(La_M),max(La_S)) - - M_min = [] - M_max = [] - for xm in La_M: - k0_min = float(xm) - float(Min) - k0_max = float(xm) - float(Max) - M_min.append(abs(k0_min)) - M_max.append(abs(k0_max)) - - M_Index_min = M_min.index(min(M_min)) + 1 - M_Index_max = M_max.index(min(M_max)) + 1 - Mindex =[M_Index_min,M_Index_max] - - S_min = [] - S_max = [] - for xs in La_S: - k0_min = float(xs) - float(Min) - k0_max = float(xs) - float(Max) - S_min.append(abs(k0_min)) - S_max.append(abs(k0_max)) - - S_Index_min = S_min.index(min(S_min)) + 1 - S_Index_max = S_max.index(min(S_max)) + 1 - Sindex =[S_Index_min,S_Index_max] - #print La_M - #print La_S - #print min(M_min),min(M_max),min(S_min),min(S_max) - M1 = min(Mindex) - M2 = max(Mindex) - - S1 = min(Sindex) - S2 = max(Sindex) - - if M1==1: S1 = S1 - else: - S1=1-M1+1 - M1=1 - - - if M2 ==len(La_M): S2 = S2 - else: - S2 = S2 + len(La_M) - M2 - M2 = len(La_M) - - - return M1 , M2, S1, S2 -######################################################################### - -INTRODUCTION = ''' -############################################################################# - - Extract the common bursts for S1 TOPs based on one master image. -''' - -EXAMPLE = ''' - Usage: - extract_s1_bursts.py projectName 170115 - - Examples: - extract_s1_bursts.py PacayaT163TsxHhA 170115 -############################################################################## -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Check common busrts for TOPS data.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='Name of project.') - parser.add_argument('Sdate',help='Slave date.') - - inps = parser.parse_args() - - return inps - -################################################################################ - - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - Sdate = inps.Sdate - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - Mdate = templateDict['masterDate'] - - processDir = scratchDir + '/' + projectName + "/PROCESS" - slcDir = scratchDir + '/' + projectName + "/SLC" - rslcDir = scratchDir + '/' + projectName + "/RSLC" - - workDir0 = rslcDir + '/' + Sdate - #if not os.path.isdir(rslcDir): - # os.mkdir(rslcDir) - - MslcDir = slcDir + '/' + Mdate - SslcDir = slcDir + '/' + Sdate - #MBurst_Par = slcDir + '/' + Mdate + '/' + - workDir = slcDir + '/' + Sdate - - BURST = workDir + '/' + Mdate + '_' + Sdate + '.common_burst_ref' - - MslcImg = workDir + '/'+Mdate + '.slc' - MslcPar = workDir + '/'+Mdate + '.slc.par' - - MamprlksImg = workDir + '/'+Mdate + '_' + rlks + 'rlks' + '.amp' - MamprlksPar = workDir + '/'+Mdate + '_' + rlks +'rlks' + '.amp.par' - - SslcImg = workDir + '/'+ Sdate + '.slc' - SslcPar = workDir + '/'+ Sdate + '.slc.par' - - SamprlksImg = workDir + '/'+ Sdate + '_' + rlks + 'rlks' + '.amp' - SamprlksPar = workDir + '/'+ Sdate + '_' + rlks + 'rlks' + '.amp.par' - - if not os.path.isdir(workDir): - os.mkdir(workDir) - - get_common_burst(MslcDir,SslcDir,BURST) - AA = ut.read_txt2array(BURST) - - SW = templateDict['start_swath'] - EW = templateDict['end_swath'] - - SB = templateDict['start_burst'] - EB = templateDict['end_burst'] - - - SLC2_tab = workDir + '/' + Sdate + '_SLC_Tab0' - - - SLC2_INF_tab = workDir + '/' + Sdate + '_SLC_Tab' - SLC2_RSLC_tab = workDir + '/' + Sdate + '_RSLC_Tab' - - BURST2_tab = workDir + '/' + Sdate + '_Burst_Tab' - - if os.path.isfile(SLC2_tab): - os.remove(SLC2_tab) - - if os.path.isfile(SLC2_INF_tab): - os.remove(SLC2_INF_tab) - - if os.path.isfile(SLC2_RSLC_tab): - os.remove(SLC2_RSLC_tab) - - if os.path.isfile(BURST2_tab): - os.remove(BURST2_tab) - - for kk in range(int(EW)-int(SW)+1): - - call_str = 'echo ' + SslcDir + '/' + Sdate+'.IW'+str(int(SW)+kk) + '.slc' + ' ' + SslcDir + '/'+ Sdate + '.IW'+str(int(SW)+kk) +'.slc.par' + ' ' + SslcDir + '/'+ Sdate+'.IW'+str(int(SW)+kk) + '.slc.TOPS_par >>' + SLC2_tab - os.system(call_str) - - ii = kk + 1 - if (int(SW)==int(EW)): - SB1=int(AA[0]) - EB1=int(AA[1]) - SB2=int(AA[2]) - EB2=int(AA[3]) - else: - SB1=int(AA[ii-1,0]) - EB1=int(AA[ii-1,1]) - - SB2=int(AA[ii-1,2]) - EB2=int(AA[ii-1,3]) - - if not int(SB)==1: - SB2 = SB2 + int(SB)-int(SB1) - if not int(EB)==20: - EB2 = SB2 + int(EB) - int(SB) - - call_str = 'echo ' + workDir + '/'+ Sdate+ '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+str(int(SW)+kk)+ '.slc' + ' ' + workDir + '/' + Sdate + '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+ str(int(SW)+kk)+ '.slc.par' + ' ' + workDir + '/'+ Sdate+'_'+ str(int(SB2)) + str(int(EB2)) + '.IW'+str(int(SW)+kk)+ '.slc.TOPS_par >>' + SLC2_INF_tab - os.system(call_str) - - call_str = 'echo ' + workDir0 + '/'+ Sdate+ '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+str(int(SW)+kk)+ '.rslc' + ' ' + workDir0 + '/' + Sdate + '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+ str(int(SW)+kk)+ '.rslc.par' + ' ' + workDir0 + '/'+ Sdate+'_'+ str(int(SB2)) + str(int(EB2)) + '.IW'+str(int(SW)+kk)+ '.rslc.TOPS_par >>' + SLC2_RSLC_tab - os.system(call_str) - - call_str = 'echo ' + str(int(SB2)) + ' ' + str(int(EB2)) + ' >>' + BURST2_tab - os.system(call_str) - - - TEST = SamprlksPar - k0 = 0 - if os.path.isfile(TEST): - if os.path.getsize(TEST) > 0: - k0 = 1 - - if k0==0: - - call_str = 'SLC_copy_ScanSAR ' + SLC2_tab + ' ' + SLC2_INF_tab + ' ' + BURST2_tab - os.system(call_str) - - call_str = 'SLC_mosaic_ScanSAR ' + SLC2_INF_tab + ' ' + SslcImg + ' ' + SslcPar + ' ' + rlks + ' ' +azlks - os.system(call_str) - - call_str = 'multi_look ' + SslcImg + ' ' + SslcPar + ' ' + SamprlksImg + ' ' + SamprlksPar + ' ' + rlks + ' ' + azlks - os.system(call_str) - - nWidth = ut.read_gamma_par(SamprlksPar, 'read', 'range_samples') - call_str = 'raspwr ' + SamprlksImg + ' ' + nWidth - os.system(call_str) - iw1 = Sdate + '.IW1.slc*' - iw2 = Sdate + '.IW2.slc*' - iw3 = Sdate + '.IW3.slc*' - call_str = 'rm '+ iw1 + ' ' + iw2 + ' '+ iw3 - os.system(call_str) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/extract_s1_bursts_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/extract_s1_bursts_all.py deleted file mode 100644 index 2a48bf8..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/extract_s1_bursts_all.py +++ /dev/null @@ -1,96 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import getopt -import time -import glob -import argparse - -import subprocess -from pyint import _utils as ut - - -def work(data0): - cmd = data0[0] - err_file = data0[1] - p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) - stdout = p.stdout - stderr = p.stderr - - if type(stderr) == bytes: - aa=stderr.decode("utf-8") - else: - aa = stderr - - if aa: - str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' - #print(aa) - with open(err_file, 'a') as f: - f.write(str0) - f.write(aa) - f.write('\n') - - return -######################################################################### - -INTRODUCTION = ''' -------------------------------------------------------------------- - Extract reference TOPS related bursts for coregistration using GAMMA. - -''' - -EXAMPLE = ''' - Usage: - extract_s1_bursts_all.py projectName - extract_s1_bursts_all.py projectName --parallel 4 -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Extract reference TOPS related bursts for coregistration using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') - - inps = parser.parse_args() - return inps - - -def main(argv): - start_time = time.time() - inps = cmdLineParse() - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - projectDir = scratchDir + '/' + projectName - slcDir = scratchDir + '/' + projectName + '/SLC' - slc_list = [os.path.basename(fname) for fname in sorted(glob.glob(slcDir + '/*'))] - - err_txt = scratchDir + '/' + projectName + '/extract_s1_bursts_all.err' - if os.path.isfile(err_txt): os.remove(err_txt) - - data_para = [] - for i in range(len(slc_list)): - cmd0 = ['extract_s1_bursts.py',projectName,slc_list[i]] - data0 = [cmd0,err_txt] - data_para.append(data0) - - ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) - print("Extract TOPS bursts for project %s is done! " % projectName) - ut.print_process_time(start_time, time.time()) - - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/extract_s1_bursts_old.py b/.codex_tmp/pyint_variants/no_rescue/pyint/extract_s1_bursts_old.py deleted file mode 100644 index 062469c..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/extract_s1_bursts_old.py +++ /dev/null @@ -1,292 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import glob -import argparse - -from pyint import _utils as ut - -def common_burst(La_M,La_S): - La_M = [float(i) for i in La_M] - La_S = [float(i) for i in La_S] - Min = max(min(La_M),min(La_S)) - Max = min(max(La_M),max(La_S)) - - M_min = [] - M_max = [] - for xm in La_M: - k0_min = float(xm) - float(Min) - k0_max = float(xm) - float(Max) - M_min.append(abs(k0_min)) - M_max.append(abs(k0_max)) - - M_Index_min = M_min.index(min(M_min)) + 1 - M_Index_max = M_max.index(min(M_max)) + 1 - Mindex =[M_Index_min,M_Index_max] - - S_min = [] - S_max = [] - for xs in La_S: - k0_min = float(xs) - float(Min) - k0_max = float(xs) - float(Max) - S_min.append(abs(k0_min)) - S_max.append(abs(k0_max)) - - S_Index_min = S_min.index(min(S_min)) + 1 - S_Index_max = S_max.index(min(S_max)) + 1 - Sindex =[S_Index_min,S_Index_max] - return min(Mindex) , max(Mindex), min(Sindex),max(Sindex) - -def get_common_burst(Mslc_dir,Sslc_dir,common_burst_txt): - Mpar_list = sorted(glob.glob(Mslc_dir+'/*.IW*.burst.par')) - Spar_list = sorted(glob.glob(Sslc_dir+'/*.IW*.burst.par')) - if os.path.isfile(common_burst_txt): - os.remove(common_burst_txt) - for kk in range(len(Mpar_list)): - MBURST = Mpar_list[kk] - SBURST = Spar_list[kk] - - Mtt = Sslc_dir + '/' + os.path.basename(MBURST.replace('burst.par','tt0')) - Stt = SBURST.replace('burst.par','tt0') - call_str = "grep 'Burst:' " + MBURST + ' >' + Mtt - os.system(call_str) - - call_str = "grep 'Burst:' " + SBURST + ' >' + Stt - os.system(call_str) - - MM = ut.read_txt2array(Mtt) - SM = ut.read_txt2array(Stt) - La_M = MM[:,2] - La_S = SM[:,2] - - PP = common_burst_Ref(La_M,La_S) - - print('Common bursts of swath' + str(kk+1) + ' : (master) ' + str(PP[0]) + ' ' + str(PP[1]) + ' (slave) ' + str(PP[2]) + ' ' + str(PP[3])) - call_str = 'echo ' + str(PP[0]) + ' ' + str(PP[1]) + ' ' + str(PP[2]) + ' ' + str(PP[3]) + ' >>' + common_burst_txt - os.system(call_str) - - return - -def common_burst_Ref(La_M,La_S): - Min = max(min(La_M),min(La_S)) - Max = min(max(La_M),max(La_S)) - - M_min = [] - M_max = [] - for xm in La_M: - k0_min = float(xm) - float(Min) - k0_max = float(xm) - float(Max) - M_min.append(abs(k0_min)) - M_max.append(abs(k0_max)) - - M_Index_min = M_min.index(min(M_min)) + 1 - M_Index_max = M_max.index(min(M_max)) + 1 - Mindex =[M_Index_min,M_Index_max] - - S_min = [] - S_max = [] - for xs in La_S: - k0_min = float(xs) - float(Min) - k0_max = float(xs) - float(Max) - S_min.append(abs(k0_min)) - S_max.append(abs(k0_max)) - - S_Index_min = S_min.index(min(S_min)) + 1 - S_Index_max = S_max.index(min(S_max)) + 1 - Sindex =[S_Index_min,S_Index_max] - #print La_M - #print La_S - #print min(M_min),min(M_max),min(S_min),min(S_max) - M1 = min(Mindex) - M2 = max(Mindex) - - S1 = min(Sindex) - S2 = max(Sindex) - - if M1==1: S1 = S1 - else: - S1=1-M1+1 - M1=1 - - - if M2 ==len(La_M): S2 = S2 - else: - S2 = S2 + len(La_M) - M2 - M2 = len(La_M) - - - return M1 , M2, S1, S2 -######################################################################### - -INTRODUCTION = ''' -############################################################################# - - Extract the common bursts for S1 TOPs based on one master image. -''' - -EXAMPLE = ''' - Usage: - extract_s1_bursts.py projectName 170115 - - Examples: - extract_s1_bursts.py PacayaT163TsxHhA 170115 -############################################################################## -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Check common busrts for TOPS data.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='Name of project.') - parser.add_argument('Sdate',help='Slave date.') - - inps = parser.parse_args() - - return inps - -################################################################################ - - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - Sdate = inps.Sdate - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - Mdate = templateDict['masterDate'] - - processDir = scratchDir + '/' + projectName + "/PROCESS" - slcDir = scratchDir + '/' + projectName + "/SLC" - rslcDir = scratchDir + '/' + projectName + "/RSLC" - - workDir0 = rslcDir + '/' + Sdate - #if not os.path.isdir(rslcDir): - # os.mkdir(rslcDir) - - MslcDir = slcDir + '/' + Mdate - SslcDir = slcDir + '/' + Sdate - #MBurst_Par = slcDir + '/' + Mdate + '/' + - workDir = slcDir + '/' + Sdate - - BURST = workDir + '/' + Mdate + '_' + Sdate + '.common_burst_ref' - - MslcImg = workDir + '/'+Mdate + '.slc' - MslcPar = workDir + '/'+Mdate + '.slc.par' - - MamprlksImg = workDir + '/'+Mdate + '_' + rlks + 'rlks' + '.amp' - MamprlksPar = workDir + '/'+Mdate + '_' + rlks +'rlks' + '.amp.par' - - SslcImg = workDir + '/'+ Sdate + '.slc' - SslcPar = workDir + '/'+ Sdate + '.slc.par' - - SamprlksImg = workDir + '/'+ Sdate + '_' + rlks + 'rlks' + '.amp' - SamprlksPar = workDir + '/'+ Sdate + '_' + rlks + 'rlks' + '.amp.par' - - if not os.path.isdir(workDir): - os.mkdir(workDir) - - get_common_burst(MslcDir,SslcDir,BURST) - AA = ut.read_txt2array(BURST) - NL = ut.get_txt_lines(BURST) - - if NL ==1: - AA = AA.reshape(1,len(AA)) - - - SW = templateDict['start_swath'] - EW = templateDict['end_swath'] - - SB = templateDict['start_burst'] - EB = templateDict['end_burst'] - - - SLC2_tab = workDir + '/' + Sdate + '_SLC_Tab0' - - - SLC2_INF_tab = workDir + '/' + Sdate + '_SLC_Tab' - SLC2_RSLC_tab = workDir + '/' + Sdate + '_RSLC_Tab' - - BURST2_tab = workDir + '/' + Sdate + '_Burst_Tab' - - if os.path.isfile(SLC2_tab): - os.remove(SLC2_tab) - - if os.path.isfile(SLC2_INF_tab): - os.remove(SLC2_INF_tab) - - if os.path.isfile(SLC2_RSLC_tab): - os.remove(SLC2_RSLC_tab) - - if os.path.isfile(BURST2_tab): - os.remove(BURST2_tab) - - for kk in range(int(EW)-int(SW)+1): - - call_str = 'echo ' + SslcDir + '/' + Sdate+'.IW'+str(int(SW)+kk) + '.slc' + ' ' + SslcDir + '/'+ Sdate + '.IW'+str(int(SW)+kk) +'.slc.par' + ' ' + SslcDir + '/'+ Sdate+'.IW'+str(int(SW)+kk) + '.slc.TOPS_par >>' + SLC2_tab - os.system(call_str) - - ii = int(SW) + kk - SB1=int(AA[ii-1,0]) - EB1=int(AA[ii-1,1]) - - SB2=int(AA[ii-1,2]) - EB2=int(AA[ii-1,3]) - - if not int(SB)==1: - SB2 = SB2 + int(SB) - int(SB1) - if not int(EB)==100: - EB2 = SB2 + int(EB) - int(SB) - - call_str = 'echo ' + workDir + '/'+ Sdate+ '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+str(int(SW)+kk)+ '.slc' + ' ' + workDir + '/' + Sdate + '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+ str(int(SW)+kk)+ '.slc.par' + ' ' + workDir + '/'+ Sdate+'_'+ str(int(SB2)) + str(int(EB2)) + '.IW'+str(int(SW)+kk)+ '.slc.TOPS_par >>' + SLC2_INF_tab - os.system(call_str) - - call_str = 'echo ' + workDir0 + '/'+ Sdate+ '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+str(int(SW)+kk)+ '.rslc' + ' ' + workDir + '/' + Sdate + '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+ str(int(SW)+kk)+ '.rslc.par' + ' ' + workDir + '/'+ Sdate+'_'+ str(int(SB2)) + str(int(EB2)) + '.IW'+str(int(SW)+kk)+ '.rslc.TOPS_par >>' + SLC2_RSLC_tab - os.system(call_str) - - call_str = 'echo ' + str(int(SB2)) + ' ' + str(int(EB2)) + ' >>' + BURST2_tab - os.system(call_str) - - - TEST = SamprlksPar - k0 = 0 - if os.path.isfile(TEST): - if os.path.getsize(TEST) > 0: - k0 = 1 - - if k0==0: - -# call_str = 'SLC_copy_ScanSAR ' + SLC2_tab + ' ' + SLC2_INF_tab + ' ' + BURST2_tab -# os.system(call_str) - -# call_str = 'SLC_mosaic_S1_TOPS ' + SLC2_INF_tab + ' ' + SslcImg + ' ' + SslcPar + ' ' + rlks + ' ' +azlks - call_str = 'SLC_mosaic_S1_TOPS ' + SLC2_tab + ' ' + SslcImg + ' ' + SslcPar + ' ' + rlks + ' ' +azlks - os.system(call_str) - - call_str = 'multi_look ' + SslcImg + ' ' + SslcPar + ' ' + SamprlksImg + ' ' + SamprlksPar + ' ' + rlks + ' ' + azlks - os.system(call_str) - - nWidth = ut.read_gamma_par(SamprlksPar, 'read', 'range_samples') - call_str = 'raspwr ' + SamprlksImg + ' ' + str(nWidth) - os.system(call_str) - - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/gacos_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/gacos_gamma.py deleted file mode 100644 index 75370f5..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/gacos_gamma.py +++ /dev/null @@ -1,1186 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# -""" -GACOS atmospheric correction for GAMMA interferograms. -This script applies GACOS-derived tropospheric corrections to unwrapped interferograms. - -GACOS (Generic Atmospheric Correction Online Service) provides zenith total delay (ZTD) -maps that can be used to correct for atmospheric phase delays in InSAR data. - -Usage: - gacos_gamma.py projectName Mdate-Sdate - gacos_gamma.py PacayaT163TsxHhA 20150102-20150601 -""" - -import numpy as np -import os -import sys -import argparse -import subprocess -import glob -import warnings -from pathlib import Path -from datetime import datetime -from scipy.interpolate import RegularGridInterpolator - -from pyint import _utils as ut - -warnings.filterwarnings('ignore', category=RuntimeWarning) - -# Try to import AutoGACOS modules -try: - from gacos import Downloader, Submitter, SarDataset - AUTOGACOS_AVAILABLE = True -except Exception as _e: - AUTOGACOS_AVAILABLE = False - print(f"[DEBUG] AutoGACOS import failed: {type(_e).__name__}: {_e}") - - -INTRODUCTION = ''' -------------------------------------------------------------------- - Apply GACOS atmospheric correction to interferograms. - - This script: - 1. Generates elevation angle file from GAMMA products - 2. Downloads GACOS ZTD data for master and slave dates - 3. Applies tropospheric correction to the interferogram - - Prerequisites: - - GACOS data for the acquisition dates (or valid email for auto-download) - - Geocoded interferogram from geocode_gamma.py -------------------------------------------------------------------- -''' - -EXAMPLE = ''' - Usage: - gacos_gamma.py projectName Mdate-Sdate - gacos_gamma.py projectName Mdate-Sdate --ztd-dir /path/to/gacos/data - gacos_gamma.py PacayaT163TsxHhA 20150102-20150601 -------------------------------------------------------------------- -''' - - -class HEADER: - """Header class for GACOS file format""" - width = 0 - length = 0 - xfirst = 0.0 - yfirst = 0.0 - xstep = 0.0 - ystep = 0.0 - - -def read_header(filename): - """Read header information from GACOS .rsc file""" - if not os.path.isfile(filename): - print(filename + " file not exist") - return None - - header = HEADER() - with open(filename) as f: - for line in f: - data = line.split() - if len(data) >= 2: - if data[0] == "WIDTH": - header.width = int(data[1]) - if data[0] == "FILE_LENGTH": - header.length = int(data[1]) - if data[0] == "X_FIRST": - header.xfirst = float(data[1]) - if data[0] == "Y_FIRST": - header.yfirst = float(data[1]) - if data[0] == "X_STEP": - header.xstep = float(data[1]) - if data[0] == "Y_STEP": - header.ystep = float(data[1]) - return header - - -def cut_image2(filename, headername, yfirst_new, length_new, xfirst_new, width_new): - """Cut GACOS ZTD image to match interferogram extent""" - header = read_header(headername) - if header is None: - return None - - with open(filename, 'rb') as f: - data0 = np.fromfile(f, dtype=np.float32) - data = np.reshape(data0, (header.length, header.width)) - - out = np.zeros((length_new, width_new), dtype=np.float32) - - for i in range(header.length): - lat = header.yfirst + header.ystep * i - row = int(round((lat - yfirst_new) / header.ystep)) - if row < 0 or row >= length_new: - continue - for j in range(header.width): - lon = header.xfirst + header.xstep * j - col = int(round((lon - xfirst_new) / header.xstep)) - if col < 0 or col >= width_new: - continue - out[row, col] = data[i, j] - - out = np.where(out == 0, np.nan, out) - out.tofile(filename + ".cut") - - f = open(filename + ".cut.rsc", 'w') - f.write("WIDTH " + str(width_new) + "\n") - f.write("FILE_LENGTH " + str(length_new) + "\n") - f.write("X_FIRST " + str(xfirst_new) + "\n") - f.write("Y_FIRST " + str(yfirst_new) + "\n") - f.write("X_STEP " + str(header.xstep) + "\n") - f.write("Y_STEP " + str(header.ystep) + "\n") - f.close() - - return filename + ".cut" - - -def make_correction(phsfilename, ztd1filename, ztd2filename, elevfilename, wavelength=None): - """ - Apply GACOS tropospheric correction to interferogram. - - Parameters: - ----------- - phsfilename : str - Path to unwrapped phase file - ztd1filename : str - Path to GACOS ZTD file for master date - ztd2filename : str - Path to GACOS ZTD file for slave date - elevfilename : str - Path to elevation angle file - wavelength : float, optional - Radar wavelength in meters. If None, read from template. - """ - header = read_header(phsfilename + ".rsc") - if header is None: - print("Error reading phase header file") - return None - - # Cut ZTD files to match interferogram extent - if not os.path.isfile(ztd1filename + ".cut"): - cut_image2(ztd1filename, ztd1filename + ".rsc", header.yfirst, header.length, header.xfirst, header.width) - if not os.path.isfile(ztd2filename + ".cut"): - cut_image2(ztd2filename, ztd2filename + ".rsc", header.yfirst, header.length, header.xfirst, header.width) - - # Read phase data - with open(phsfilename, 'rb') as f: - data = np.fromfile(f, dtype=np.float32) - phase = np.reshape(data, [header.length, header.width]) - - # Read ZTD data - with open(ztd1filename + ".cut", 'rb') as f: - data = np.fromfile(f, dtype=np.float32) - ztd1 = np.reshape(data, [header.length, header.width]) - - with open(ztd2filename + ".cut", 'rb') as f: - data = np.fromfile(f, dtype=np.float32) - ztd2 = np.reshape(data, [header.length, header.width]) - - # Read elevation angle data - with open(elevfilename, 'rb') as f: - data = np.fromfile(f, dtype=np.float32) - elev = np.reshape(data, [header.length, header.width]) - - # Calculate tropospheric phase delay - # ZTD difference to phase: phase = 4*pi*dZTD / (wavelength * sin(elevation)) - # Default wavelength factor for Sentinel-1: 0.0044138251819503 = 4*pi/lambda - if wavelength: - ztd_factor = 4 * np.pi / wavelength - else: - ztd_factor = 1 / 0.0044138251819503 # Default factor - - dztd = ztd2 - ztd1 - dztd = dztd / 0.0044138251819503 # Convert ZTD to phase - dztd = dztd / np.sin(elev) # Incidence angle correction - - # Apply correction - index = np.where(phase == 0) - phase[index] = np.nan - phasemean = np.nanmean(phase) - print("Phase std before correction: " + str(np.nanstd(phase))) - - phase = phase - phasemean - phase = phase - dztd # Apply correction - phase[index] = np.nan - phasemean = np.nanmean(phase) - print("Phase std after correction: " + str(np.nanstd(phase))) - - phase = phase - phasemean - phase[index] = 0 - phase.tofile(phsfilename + ".gacos") - - return phsfilename + ".gacos" - - -def generate_elevation_angle(workDir, Mdate, Sdate, MampPar, offpar, dempar, dem, rlks): - """ - Generate elevation angle file from GAMMA products. - - The elevation angle (90 - incidence angle) is needed for - projecting ZTD to line-of-sight phase delay. - """ - os.chdir(workDir) - - # Generate look vector (incidence angle) - call_str = "look_vector " + MampPar + " " + offpar + " " + dempar + " " + dem + " lv_theta lv_phi" - os.system(call_str) - - # Get DEM parameters - call_str = "grep 'corner_lat:' " + dempar + " | awk '{print $2}' " - North = subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'corner_lon:' " + dempar + " | awk '{print $2}' " - West = subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'post_lat:' " + dempar + " | awk '{print $2}' " - posty = subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'post_lon:' " + dempar + " | awk '{print $2}' " - postx = subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'width:' " + dempar + " | awk '{print $2}' " - width = subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'nlines:' " + dempar + " | awk '{print $2}' " - length = subprocess.getstatusoutput(call_str)[1] - - South = str(round(float(North) + (float(length) - 1) * float(posty), 7)) - East = str(round(float(West) + (float(width) - 1) * float(postx), 7)) - - # Convert to elevation angle - call_str = "swap_bytes lv_theta lv_theta.phase_swap 4 > dinsar.log " - os.system(call_str) - - call_str = "gmt xyz2grd lv_theta.phase_swap -Glv_theta.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" + South + "/" + North + " -I" + postx + " -ZTLf -N0" - os.system(call_str) - - # Elevation angle = 90 - incidence angle - call_str = "gmt grdmath 90 lv_theta.grd 3.1415926 DIV 180 MUL SUB = lv_theta_final.grd" - os.system(call_str) - call_str = "gmt grdmath 90 lv_theta_final.grd SUB = lv_elev.grd" - os.system(call_str) - - # Resample to 3 arc-seconds - call_str = "gmt grdsample lv_elev.grd -Glv_elev_3c.grd -I3c" - os.system(call_str) - - # Convert to binary - call_str = "gmt grd2xyz lv_elev_3c.grd -ZTLf -N0 > " + Mdate + '-' + Sdate + '.gacos.elev' - os.system(call_str) - - return workDir + '/' + Mdate + '-' + Sdate + '.gacos.elev' - - -def prepare_unw_for_gacos(workDir, unw, Mdate, Sdate, dempar, rlks): - """ - Prepare unwrapped interferogram for GACOS correction. - - Converts GAMMA format to GACOS-compatible format with .rsc file. - """ - os.chdir(workDir) - - # Get DEM parameters - call_str = "grep 'corner_lat:' " + dempar + " | awk '{print $2}' " - North = subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'corner_lon:' " + dempar + " | awk '{print $2}' " - West = subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'post_lat:' " + dempar + " | awk '{print $2}' " - posty = subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'post_lon:' " + dempar + " | awk '{print $2}' " - postx = subprocess.getstatusoutput(call_str)[1] - - # Convert unw to grd - call_str = "swap_bytes " + unw + " unw.phase_swap 4 > dinsar.log " - os.system(call_str) - - # Get dimensions - call_str = "gmt grdinfo unw_f.grd -C" - result = subprocess.getstatusoutput(call_str)[1].split() - if len(result) >= 12: - Width = result[9] - line = result[10] - West = result[1] - North = result[4] - else: - # Fallback to DEM parameters - call_str = "grep 'width:' " + dempar + " | awk '{print $2}' " - Width = subprocess.getstatusoutput(call_str)[1] - call_str = "grep 'nlines:' " + dempar + " | awk '{print $2}' " - line = subprocess.getstatusoutput(call_str)[1] - - ymax = str(int(round(float(line), 7))) - xmax = str(int(round(float(Width), 7))) - - # Create .rsc file - output = workDir + '/' + Mdate + '-' + Sdate + '.gacos.unw.rsc' - if os.path.exists(output): - os.remove(output) - - with open(output, 'a+') as f: - f.write('WIDTH ' + str(Width) + '\n') - f.write('FILE_LENGTH ' + str(line) + '\n') - f.write('XMIN 1' + '\n') - f.write('XMAX ' + xmax + '\n') - f.write('YMIN 1' + '\n') - f.write('YMAX ' + ymax + '\n') - f.write('X_FIRST ' + str(West) + '\n') - f.write('Y_FIRST ' + str(North) + '\n') - f.write('X_STEP 8.33333333E-04' + '\n') - f.write('Y_STEP -8.33333333E-04' + '\n') - f.write('X_UNIT degrees' + '\n') - f.write('Y_UNIT degrees' + '\n') - f.write('Z_OFFSET 0' + '\n') - f.write('Z_SCALE 1' + '\n') - f.write('PROJECTION LATLON' + '\n') - f.write('DATUM WGS84' + '\n') - - return workDir + '/' + Mdate + '-' + Sdate + '.gacos.unw' - - -def parse_dem_par(dempar): - """ - 从 GAMMA DEM par 文件读取网格参数（纯 Python，无需 grep/awk） - """ - info = {} - with open(dempar) as f: - for line in f: - parts = line.split(':') - if len(parts) >= 2: - key = parts[0].strip() - val = parts[1].strip().split()[0] if parts[1].strip() else '' - info[key] = val - return { - 'width': int(info['width']), - 'nlines': int(info['nlines']), - 'corner_lat': float(info['corner_lat']), - 'corner_lon': float(info['corner_lon']), - 'post_lat': float(info['post_lat']), - 'post_lon': float(info['post_lon']), - } - - -def resample_ztd_to_dem(ztd_file, rsc_file, dem_info): - """ - 读取 ZTD 数据并双线性插值重采样到 DEM 网格 - 修复: RegularGridInterpolator 要求坐标严格递增， - ZTD 纬度为递减（北→南），需翻转 - """ - # 读取 RSC 头文件 - rsc = {} - with open(rsc_file) as f: - for line in f: - parts = line.split() - if len(parts) >= 2: - rsc[parts[0]] = parts[1] - - ztd_w = int(rsc['WIDTH']) - ztd_h = int(rsc['FILE_LENGTH']) - ztd_x0 = float(rsc['X_FIRST']) - ztd_y0 = float(rsc['Y_FIRST']) - ztd_dx = float(rsc['X_STEP']) - ztd_dy = float(rsc['Y_STEP']) - - raw = np.fromfile(ztd_file, dtype=np.float32).reshape(ztd_h, ztd_w) - - # 构建 ZTD 网格坐标 - ztd_lats = ztd_y0 + np.arange(ztd_h) * ztd_dy - ztd_lons = ztd_x0 + np.arange(ztd_w) * ztd_dx - - # 翻转使坐标严格递增（RegularGridInterpolator 要求） - if ztd_lats[0] > ztd_lats[-1]: - ztd_lats = ztd_lats[::-1] - raw = raw[::-1, :] - if ztd_lons[0] > ztd_lons[-1]: - ztd_lons = ztd_lons[::-1] - raw = raw[:, ::-1] - - interp = RegularGridInterpolator( - (ztd_lats, ztd_lons), raw, - method='linear', bounds_error=False, fill_value=np.nan - ) - - # 构建 DEM 网格查询点 - dem_w = dem_info['width'] - dem_h = dem_info['nlines'] - dem_lats = dem_info['corner_lat'] + np.arange(dem_h) * dem_info['post_lat'] - dem_lons = dem_info['corner_lon'] + np.arange(dem_w) * dem_info['post_lon'] - - # 逐行插值（节省内存） - result = np.empty((dem_h, dem_w), dtype=np.float32) - for i in range(dem_h): - pts = np.column_stack([np.full(dem_w, dem_lats[i]), dem_lons]) - result[i] = interp(pts).astype(np.float32) - - return result - - -def convert_ztd_tif_to_binary(tif_file): - """ - 将 .ztd.tif (GeoTIFF) 转换为 .ztd (raw binary) + .ztd.rsc (header) - 返回 .ztd 文件路径 - """ - try: - import rasterio - except ImportError: - from osgeo import gdal - ds = gdal.Open(tif_file) - gt = ds.GetGeoTransform() - w, h = ds.RasterXSize, ds.RasterYSize - data = ds.GetRasterBand(1).ReadAsArray().astype(np.float32) - x0, y0, dx, dy = gt[0], gt[3], gt[1], gt[5] - ds = None - else: - with rasterio.open(tif_file) as src: - data = src.read(1).astype(np.float32) - w, h = src.width, src.height - x0 = src.transform[2] - y0 = src.transform[5] - dx = src.transform[0] - dy = src.transform[4] - - ztd_file = tif_file.replace('.ztd.tif', '.ztd') - rsc_file = tif_file.replace('.ztd.tif', '.ztd.rsc') - - data.tofile(ztd_file) - with open(rsc_file, 'w') as f: - f.write(f"WIDTH {w}\n") - f.write(f"FILE_LENGTH {h}\n") - f.write(f"X_FIRST {x0}\n") - f.write(f"Y_FIRST {y0}\n") - f.write(f"X_STEP {dx}\n") - f.write(f"Y_STEP {dy}\n") - f.write("X_UNIT degrees\n") - f.write("Y_UNIT degrees\n") - f.write("PROJECTION LATLON\n") - f.write("DATUM WGS84\n") - - return ztd_file - - -def apply_gacos_correction_python(geo_unw_file, ztd1_file, ztd1_rsc, ztd2_file, ztd2_rsc, - dempar, slcpar=None, wavelength=None, out_file=None, - figdir=None, pair_str=None): - """ - 纯 Python GACOS 大气校正（不依赖 GMT） - 修复原 make_correction 的所有 Bug: - - 极端值掩码（GAMMA 无效标记） - - sin/cos 弧度转换 - - cut_image2 慢速循环 → scipy 向量化重采样 - - 增加校正前后对比图 - - 参数: - ----- - geo_unw_file : str 地理编码后的解缠相位文件 - ztd1_file : str Master 日期 ZTD 二进制文件 - ztd1_rsc : str Master 日期 ZTD RSC 头文件 - ztd2_file : str Slave 日期 ZTD 二进制文件 - ztd2_rsc : str Slave 日期 ZTD RSC 头文件 - dempar : str DEM 参数文件 - slcpar : str SLC 参数文件（读取入射角，可选） - wavelength : float 雷达波长（m），None则从slcpar读取 - out_file : str 输出文件路径（默认 geo_unw_file + '.gacos'） - figdir : str 对比图保存目录（None则不绘图） - pair_str : str 干涉对名称如 '20241105-20241117' - - 返回: 输出文件路径 - """ - # 1. 读取 DEM 参数 - dem = parse_dem_par(dempar) - dem_w, dem_h = dem['width'], dem['nlines'] - - # 2. 读取入射角和波长 - inc_angle_deg = 39.0 # 默认值 - if slcpar and os.path.isfile(slcpar): - with open(slcpar) as f: - for line in f: - if line.startswith('incidence_angle:'): - inc_angle_deg = float(line.split(':')[1].strip().split()[0]) - if line.startswith('radar_frequency:') and wavelength is None: - freq = float(line.split(':')[1].strip().split()[0]) - wavelength = 299792458.0 / freq - - if wavelength is None: - wavelength = 0.0554657595 # Sentinel-1 C-band 默认值 - - cos_inc = np.cos(np.radians(inc_angle_deg)) - ztd_to_phase = 4.0 * np.pi / wavelength - - print(f" 入射角: {inc_angle_deg:.4f}°, cos={cos_inc:.6f}") - print(f" 波长: {wavelength:.10f} m") - - # 3. 读取地理编码后的解缠相位 - unw_data = np.fromfile(geo_unw_file, dtype=np.float32).reshape(dem_h, dem_w) - - # 4. 重采样 ZTD 到 DEM 网格 - print(" 重采样 ZTD Master...") - ztd_m = resample_ztd_to_dem(ztd1_file, ztd1_rsc, dem) - print(" 重采样 ZTD Slave...") - ztd_s = resample_ztd_to_dem(ztd2_file, ztd2_rsc, dem) - - # 5. 计算 LOS 相位校正量 - dztd = ztd_s - ztd_m - phase_correction = dztd * ztd_to_phase / cos_inc - - # 6. 构建有效像素掩码 - # 排除: 零值 | NaN | GAMMA 伪影/解缠错误（|x| >= 1000 rad ≈ 4.4m LOS） - PHASE_THRESH = 1000.0 - valid = ((unw_data != 0) - & np.isfinite(unw_data) - & (np.abs(unw_data) < PHASE_THRESH) - & np.isfinite(phase_correction)) - n_valid = np.sum(valid) - n_total = dem_w * dem_h - print(f" 有效像素: {n_valid}/{n_total} ({n_valid/n_total*100:.1f}%)") - - if n_valid == 0: - print(" 错误: 无有效像素，跳过校正") - return None - - # 7. 统计校正前 - std_before = float(np.std(unw_data[valid])) - - # 8. 应用校正（保留原始数据结构，仅修改有效像素） - corrected = unw_data.copy() - corrected[valid] = unw_data[valid] - phase_correction[valid] - - # 去均值（仅对有效像素） - mean_val = np.mean(corrected[valid]) - corrected[valid] -= mean_val - - std_after = float(np.std(corrected[valid])) - reduction = (1 - std_after / std_before) * 100 if std_before > 0 else 0 - - print(f" 校正前 std: {std_before:.3f} rad") - print(f" 校正后 std: {std_after:.3f} rad") - print(f" std 降低: {reduction:.1f}%") - - # 9. 保存 - if out_file is None: - out_file = geo_unw_file + '.gacos' - corrected.astype(np.float32).tofile(out_file) - print(f" 输出: {out_file}") - - # 10. 绘制对比图 - if figdir: - plot_gacos_comparison(unw_data, phase_correction, corrected, valid, - dem, std_before, std_after, pair_str, figdir) - - return out_file - - -def plot_gacos_comparison(unw_before, correction, unw_after, valid_mask, - dem_info, std_before, std_after, pair_str, figdir): - """ - 绘制 GACOS 校正前后相位对比图（三子图） - 版本控制: 自动追加 _v1, _v2... - """ - import matplotlib - matplotlib.use('Agg') - import matplotlib.pyplot as plt - import matplotlib.colors as mcolors - - # 中文字体配置 - plt.rcParams['font.sans-serif'] = ['Noto Serif CJK SC', 'Noto Sans CJK SC', - 'AR PL UMing CN', 'SimHei', 'DejaVu Sans'] - plt.rcParams['axes.unicode_minus'] = False - - figdir = Path(figdir) - figdir.mkdir(parents=True, exist_ok=True) - - # 版本控制 - base_name = f"GACOS_comparison_{pair_str}" if pair_str else "GACOS_comparison" - existing = sorted(figdir.glob(f"{base_name}_v*.png")) - ver = int(existing[-1].stem.split('_v')[-1]) + 1 if existing else 1 - out_path = figdir / f"{base_name}_v{ver}.png" - - # 准备显示数据（仅显示有效像素） - before_disp = np.full_like(unw_before, np.nan, dtype=np.float64) - before_disp[valid_mask] = unw_before[valid_mask] - corr_disp = np.full_like(correction, np.nan, dtype=np.float64) - corr_disp[valid_mask] = correction[valid_mask] - after_disp = np.full_like(unw_after, np.nan, dtype=np.float64) - after_disp[valid_mask] = unw_after[valid_mask] - - # 色标范围：使用有效数据的 P2/P98 - vals_b = unw_before[valid_mask] - vals_c = correction[valid_mask] - vals_a = unw_after[valid_mask] - vlim = max(abs(np.percentile(vals_b, 2)), abs(np.percentile(vals_b, 98)), 0.5) - vlim_c = max(abs(np.percentile(vals_c, 2)), abs(np.percentile(vals_c, 98)), 0.5) - vlim_a = max(abs(np.percentile(vals_a, 2)), abs(np.percentile(vals_a, 98)), 0.5) - - # 地理坐标范围 - w = dem_info['width'] - h = dem_info['nlines'] - extent = [dem_info['corner_lon'], - dem_info['corner_lon'] + w * dem_info['post_lon'], - dem_info['corner_lat'] + h * dem_info['post_lat'], - dem_info['corner_lat']] - - # 降采样显示 - step = max(1, h // 2000) - b_s = before_disp[::step, ::step] - c_s = corr_disp[::step, ::step] - a_s = after_disp[::step, ::step] - - # 使用带 NaN 灰色背景的 colormap - cmap_phase = plt.cm.RdBu_r.copy() - cmap_phase.set_bad(color='#E0E0E0') - cmap_corr = plt.cm.coolwarm.copy() - cmap_corr.set_bad(color='#E0E0E0') - - fig, axes = plt.subplots(1, 3, figsize=(18, 6), dpi=150) - for ax in axes: - ax.set_facecolor('#E0E0E0') - - im1 = axes[0].imshow(b_s, cmap=cmap_phase, vmin=-vlim, vmax=vlim, - extent=extent, aspect='auto', interpolation='nearest') - axes[0].set_title(f'Unwrapped Phase (std={std_before:.2f} rad)', fontsize=11, fontweight='bold') - plt.colorbar(im1, ax=axes[0], label='rad', shrink=0.8) - - im2 = axes[1].imshow(c_s, cmap=cmap_corr, vmin=-vlim_c, vmax=vlim_c, - extent=extent, aspect='auto', interpolation='nearest') - axes[1].set_title('GACOS Correction', fontsize=11, fontweight='bold') - plt.colorbar(im2, ax=axes[1], label='rad', shrink=0.8) - - im3 = axes[2].imshow(a_s, cmap=cmap_phase, vmin=-vlim_a, vmax=vlim_a, - extent=extent, aspect='auto', interpolation='nearest') - axes[2].set_title(f'Corrected (std={std_after:.2f} rad)', fontsize=11, fontweight='bold') - plt.colorbar(im3, ax=axes[2], label='rad', shrink=0.8) - - for ax in axes: - ax.set_xlabel('Longitude') - ax.set_ylabel('Latitude') - ax.tick_params(labelsize=9) - - reduction = (1 - std_after / std_before) * 100 if std_before > 0 else 0 - n_valid = np.sum(valid_mask) - if pair_str: - m, s = pair_str.split('-') - fig.suptitle(f'GACOS: {m}-{s} | valid={n_valid:,} px | std {reduction:+.1f}%', - fontsize=13, fontweight='bold', y=1.02) - plt.tight_layout() - fig.savefig(str(out_path), dpi=150, bbox_inches='tight', - facecolor='white', edgecolor='none') - plt.close(fig) - print(f" 对比图: {out_path}") - - -def find_existing_ztd(date, gacos_dir): - """ - Find existing GACOS ZTD file for a specific date. - - Returns: - -------- - ztd_file : str or None - Path to ZTD file (without extension), or None if not found - """ - # Check for .ztd + .ztd.rsc format (preferred) - ztd_pattern = os.path.join(gacos_dir, "**", date + "*.ztd") - ztd_files = glob.glob(ztd_pattern, recursive=True) - if ztd_files: - for ztd_file in ztd_files: - if ztd_file.endswith('.ztd.tif'): - continue - rsc_file = ztd_file + '.rsc' - if os.path.exists(rsc_file): - return ztd_file - - # Check for .ztd.tif format, auto-convert if needed - ztd_pattern = os.path.join(gacos_dir, "**", date + "*.ztd.tif") - tif_files = glob.glob(ztd_pattern, recursive=True) - if tif_files: - tif_file = tif_files[0] - ztd_bin = tif_file.replace('.ztd.tif', '.ztd') - rsc_file = tif_file.replace('.ztd.tif', '.ztd.rsc') - if not os.path.exists(ztd_bin) or not os.path.exists(rsc_file): - print(f" Auto-converting {os.path.basename(tif_file)} -> .ztd + .rsc") - convert_ztd_tif_to_binary(tif_file) - if os.path.exists(ztd_bin) and os.path.exists(rsc_file): - return ztd_bin - - return None - - -def submit_gacos_request(dates, bounds, email, gacos_dir, acquisition_time=None): - """ - Submit GACOS data request for multiple dates. - - Parameters: - ----------- - dates : list - List of acquisition dates in YYYYMMDD format - bounds : tuple - Bounding box (West, South, East, North) - email : str - Email address for GACOS submission - gacos_dir : str - Directory to save GACOS data - acquisition_time : str, optional - Acquisition time in HH:MM format - - Returns: - -------- - success : bool - True if submission successful - """ - if not AUTOGACOS_AVAILABLE: - print("AutoGACOS module not available. Please download GACOS data manually.") - return False - - try: - from gacos import Submitter, SarDataset - import pandas as pd - - # Create datetime index - if acquisition_time: - hour, minute = map(int, acquisition_time.split(':')) - else: - hour, minute = 10, 0 # Sentinel-1 升轨默认采集时间 ~10:00 UTC - - date_strings = [f"{d} {hour:02d}:{minute:02d}:00" for d in dates] - date_times = pd.to_datetime(date_strings) - - # Create dataset - dataset = SarDataset(bounds, date_times, gacos_dir) - - print(f"\nSubmitting GACOS request for {len(dates)} dates...") - print(f"Bounds: W={bounds[0]:.4f}, S={bounds[1]:.4f}, E={bounds[2]:.4f}, N={bounds[3]:.4f}") - print(f"Acquisition time: {hour:02d}:{minute:02d} UTC") - print(f"Email: {email}") - - # Submit request - submitter = Submitter(dataset, email) - submitter.post_requests() - - if submitter.succeed: - print(f"\nSuccessfully submitted {len(submitter.succeed)} requests.") - if submitter.failed: - print(f"\nFailed to submit {len(submitter.failed)} requests.") - - return len(submitter.succeed) > 0 - - except Exception as e: - print(f"Error in GACOS submission: {e}") - import traceback - traceback.print_exc() - return False - - -def download_gacos_from_email(email_config, gacos_dir, bounds=None, times=None, - submit_time=None): - """ - 从邮箱检索 gacos2017@foxmail.com 发来的 GACOS 下载链接并下载数据。 - - Parameters: - ----------- - email_config : dict - 邮箱配置: username, password, host, port, ssl - gacos_dir : str - GACOS 数据保存目录 - bounds : tuple, optional - 边界框过滤 (West, South, East, North) - times : list, optional - 采集时间过滤 - submit_time : str, optional - 提交时间（YYYY-MM-DD HH:MM:SS），仅检索此时间之后的邮件 - - Returns: - -------- - downloaded_files : list - 已下载的 ZTD 文件路径列表 - """ - if not AUTOGACOS_AVAILABLE: - print("AutoGACOS module not available.") - return [] - - try: - from gacos import Downloader, GACOSEmail - - # Step 1: 从邮箱检索 GACOS 下载链接 - print(" 检索 gacos2017@foxmail.com 邮件中的下载链接...") - - email_retriever = GACOSEmail( - username=email_config.get('username'), - password=email_config.get('password'), - host=email_config.get('host', 'imap.163.com'), - port=email_config.get('port'), - email_protocol=email_config.get('protocol', 'imap'), - ssl=email_config.get('ssl', True), - gacos_email='gacos2017@foxmail.com', - start_date=submit_time - ) - - url_file = os.path.join(gacos_dir, 'gacos_urls.csv') - email_retriever.retrieve_gacos_urls(url_file) - - if not os.path.exists(url_file): - print(" 未找到 GACOS 下载链接") - return [] - - # 检查 CSV 是否有内容 - import pandas as pd - try: - df = pd.read_csv(url_file) - if len(df) == 0: - print(" 邮件中未发现新的 GACOS 数据链接") - return [] - print(f" 找到 {len(df)} 个下载链接") - except Exception: - print(" URL 文件解析失败") - return [] - - # Step 2: 下载文件 - print(" 下载 GACOS 数据...") - - dl = Downloader( - url_file=url_file, - output_dir=gacos_dir, - bounds=bounds, - times=times, - keep_original=False - ) - - dl.download() - - # Step 3: 将下载的 .ztd.tif 转换为 .ztd + .rsc - downloaded = [] - for tif_file in glob.glob(os.path.join(gacos_dir, "*.ztd.tif")): - rsc_file = tif_file.replace('.ztd.tif', '.ztd.rsc') - if not os.path.exists(rsc_file): - ztd_file = convert_ztd_tif_to_binary(tif_file) - if ztd_file: - downloaded.append(ztd_file) - print(f" 转换: {os.path.basename(tif_file)} → .ztd + .rsc") - else: - downloaded.append(tif_file.replace('.ztd.tif', '.ztd')) - - return downloaded - - except Exception as e: - print(f" 邮箱检索/下载出错: {e}") - import traceback - traceback.print_exc() - return [] - - -def auto_gacos_workflow(dates, bounds, gacos_dir, email, email_config=None, - acquisition_time=None, wait_for_email=False, max_wait_hours=24, - check_interval=60): - """ - Complete automatic GACOS workflow: submit request, wait, and download. - - Parameters: - ----------- - dates : list - List of acquisition dates in YYYYMMDD format - bounds : tuple - Bounding box (West, South, East, North) - gacos_dir : str - Directory to save GACOS data - email : str - Email address for GACOS submission - email_config : dict, optional - Email configuration for downloading (username, password, host, etc.) - acquisition_time : str, optional - Acquisition time in HH:MM format - wait_for_email : bool, optional - Whether to wait for email notification before downloading - max_wait_hours : int, optional - Maximum hours to wait for email - - Returns: - -------- - ztd_files : dict - Dictionary mapping dates to ZTD file paths - """ - import time as time_module - from datetime import datetime, timedelta - - ztd_files = {} - missing_dates = [] - - # Step 1: Check existing files - print("\n" + "="*60) - print("Step 1: Checking existing GACOS data...") - print("="*60) - - for date in dates: - ztd_file = find_existing_ztd(date, gacos_dir) - if ztd_file: - ztd_files[date] = ztd_file - else: - missing_dates.append(date) - - if not missing_dates: - print("\nAll required GACOS data already exists!") - return ztd_files - - print(f"\nMissing GACOS data for {len(missing_dates)} dates:") - for d in missing_dates: - print(f" - {d}") - - # Step 2: Submit request for missing dates - print("\n" + "="*60) - print("Step 2: Submitting GACOS requests...") - print("="*60) - - submit_success = submit_gacos_request( - dates=missing_dates, - bounds=bounds, - email=email, - gacos_dir=gacos_dir, - acquisition_time=acquisition_time - ) - - if not submit_success: - print("\n提交失败（可能是重复提交或服务器繁忙），尝试从邮箱获取已有数据...") - else: - print("\nGACOS request submitted successfully!") - print("You will receive an email with download links when data is ready.") - - # Step 3: Wait and download from email (if configured) - if email_config and wait_for_email: - print("\n" + "="*60) - print("Step 3: 检查 gacos2017@foxmail.com 邮件并下载...") - print("="*60) - - # 记录时间基准，仅检索此时间之后的 GACOS 邮件，过滤掉旧邮件 - if submit_success: - submit_time_str = datetime.now().strftime('%Y-%m-%d %H:%M:%S') - else: - # 提交失败（重复提交），向前回溯 1 小时查找已有邮件 - submit_time_str = (datetime.now() - timedelta(hours=1)).strftime('%Y-%m-%d %H:%M:%S') - print(f"邮件过滤起始时间: {submit_time_str}") - - start_time = datetime.now() - # 提交成功则等 3 分钟再检查；提交失败则立即检查邮箱 - first_wait = 180 if submit_success else 5 - retry_wait = 120 # 之后每 2 分钟 - attempt = 0 - - # 首次等待 - print(f"\n提交完成，等待 {first_wait//60} 分钟后首次检查邮箱...") - time_module.sleep(first_wait) - - while (datetime.now() - start_time) < timedelta(hours=max_wait_hours): - attempt += 1 - elapsed = datetime.now() - start_time - elapsed_min = int(elapsed.total_seconds() // 60) - elapsed_sec = int(elapsed.total_seconds() % 60) - print(f"\n[第 {attempt} 次检查] 已等待 {elapsed_min}m{elapsed_sec}s") - - downloaded = download_gacos_from_email( - email_config=email_config, - gacos_dir=gacos_dir, - bounds=bounds, - submit_time=submit_time_str - ) - - # 检查缺失日期是否已下载 - for date in missing_dates[:]: - ztd_file = find_existing_ztd(date, gacos_dir) - if ztd_file: - ztd_files[date] = ztd_file - missing_dates.remove(date) - print(f" ✓ 已获取: {date}") - - if not missing_dates: - print(f"\n所有 {len(dates)} 个日期的 GACOS 数据下载完成！") - break - - print(f" 仍缺 {len(missing_dates)} 个日期，{retry_wait//60} 分钟后重试...") - time_module.sleep(retry_wait) - - if missing_dates: - print(f"\n超时！仍缺 {len(missing_dates)} 个日期的 GACOS 数据。") - - return ztd_files - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Apply GACOS atmospheric correction to GAMMA interferograms.', - formatter_class=argparse.RawTextHelpFormatter, - epilog=INTRODUCTION + '\n' + EXAMPLE) - - parser.add_argument('projectName', help='projectName for processing.') - parser.add_argument('pair', help='Master-Slave, e.g., 20150101-20150106.') - parser.add_argument('--ztd-dir', dest='ztdDir', help='Directory containing GACOS ZTD files.') - parser.add_argument('--ztd1', dest='ztd1', help='GACOS ZTD file for master date.') - parser.add_argument('--ztd2', dest='ztd2', help='GACOS ZTD file for slave date.') - parser.add_argument('--email', dest='email', help='Email for GACOS auto-download.') - parser.add_argument('--wavelength', dest='wavelength', type=float, help='Radar wavelength in meters.') - - # Email configuration for auto-download - parser.add_argument('--email-user', dest='emailUser', help='Email username for downloading.') - parser.add_argument('--email-pass', dest='emailPass', help='Email password for downloading.') - parser.add_argument('--email-host', dest='emailHost', default='imap.gmail.com', help='Email IMAP host.') - parser.add_argument('--email-port', dest='emailPort', type=int, default=993, help='Email IMAP port.') - parser.add_argument('--email-ssl', dest='emailSsl', action='store_true', default=True, help='Use SSL for email.') - - # Auto-download options - parser.add_argument('--auto-download', dest='autoDownload', action='store_true', - help='Automatically submit request and download.') - parser.add_argument('--wait-hours', dest='waitHours', type=int, default=24, - help='Maximum hours to wait for email notification.') - - inps = parser.parse_args() - return inps - - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - Pair = inps.pair - - # Parse dates - Mdate = ut.yyyymmdd(Pair.split('-')[0]) - Sdate = ut.yyyymmdd(Pair.split('-')[1]) - - # Setup directories - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict = ut.update_template(templateFile) - - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - masterDate = templateDict['masterDate'] - - ifgDir = scratchDir + '/' + projectName + "/ifgrams" - demDir = scratchDir + '/' + projectName + "/DEM" - slcDir = scratchDir + '/' + projectName + "/SLC" - workDir = ifgDir + '/' + Pair - - # GACOS data directory - if inps.ztdDir: - gacosDir = inps.ztdDir - else: - gacosDir = scratchDir + '/' + projectName + "/GACOS" - - if not os.path.exists(gacosDir): - os.makedirs(gacosDir) - - # Define input files - dempar = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem.par' - slcpar = slcDir + '/' + masterDate + '/' + masterDate + '.slc.par' - geo_unw = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.unw' - out_file = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.unw.gacos' - figdir = scratchDir + '/' + projectName + "/figure" - - # Check if geocoded interferogram exists - if not os.path.isfile(geo_unw): - print(f"Error: Geocoded interferogram not found: {geo_unw}") - print("Please run geocode_gamma.py first.") - sys.exit(1) - - print("="*60) - print("GACOS Atmospheric Correction (Pure Python)") - print(f"Project: {projectName}") - print(f"Pair: {Pair}") - print("="*60) - - # Step 1: Find GACOS ZTD files - print("\nStep 1: Checking GACOS ZTD files...") - - ztd1 = None - ztd2 = None - - if inps.ztd1 and inps.ztd2: - ztd1 = inps.ztd1 - ztd2 = inps.ztd2 - else: - ztd1 = find_existing_ztd(Mdate, gacosDir) - ztd2 = find_existing_ztd(Sdate, gacosDir) - - if ztd1 and ztd2: - print(f" Master ZTD: {ztd1}") - print(f" Slave ZTD: {ztd2}") - elif inps.email: - # Auto-download workflow (existing logic preserved) - dem_info = parse_dem_par(dempar) - North = dem_info['corner_lat'] - West = dem_info['corner_lon'] - South = North + (dem_info['nlines'] - 1) * dem_info['post_lat'] - East = West + (dem_info['width'] - 1) * dem_info['post_lon'] - bounds = (West, South, East, North) - - missing_dates = [] - if not ztd1: missing_dates.append(Mdate) - if not ztd2: missing_dates.append(Sdate) - - if inps.autoDownload: - email_config = None - if inps.emailUser and inps.emailPass: - email_config = { - 'username': inps.emailUser, 'password': inps.emailPass, - 'host': inps.emailHost, 'port': inps.emailPort, 'ssl': inps.emailSsl - } - ztd_files = auto_gacos_workflow( - dates=missing_dates, bounds=bounds, gacos_dir=gacosDir, - email=inps.email, email_config=email_config, - wait_for_email=(email_config is not None), max_wait_hours=inps.waitHours - ) - ztd1 = ztd1 or ztd_files.get(Mdate) - ztd2 = ztd2 or ztd_files.get(Sdate) - else: - submit_gacos_request(dates=missing_dates, bounds=bounds, - email=inps.email, gacos_dir=gacosDir) - print(f"\nGACOS request submitted for: {missing_dates}") - sys.exit(0) - - if ztd1 is None or ztd2 is None: - print("\nError: GACOS ZTD files not found.") - print(f" Missing: {Mdate if not ztd1 else ''} {Sdate if not ztd2 else ''}") - print(f" Search dir: {gacosDir}") - sys.exit(1) - - # Step 2: Apply GACOS correction (Pure Python, no GMT dependency) - print("\nStep 2: Applying GACOS correction...") - - ztd1_rsc = ztd1 + '.rsc' - ztd2_rsc = ztd2 + '.rsc' - - corrected_file = apply_gacos_correction_python( - geo_unw_file=geo_unw, - ztd1_file=ztd1, ztd1_rsc=ztd1_rsc, - ztd2_file=ztd2, ztd2_rsc=ztd2_rsc, - dempar=dempar, - slcpar=slcpar, - wavelength=inps.wavelength, - out_file=out_file, - figdir=figdir, - pair_str=Pair - ) - - if corrected_file: - # Step 3: Generate BMP preview using rasdt_pwr - print("\nStep 3: Generating BMP preview...") - geo_amp = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.amp' - nWidthUTMDEM = str(parse_dem_par(dempar)['width']) - - if os.path.isfile(geo_amp): - call_str = ('rasdt_pwr ' + corrected_file + ' ' + geo_amp + ' ' - + nWidthUTMDEM + ' - - - - -3.14 3.14 1 rmg.cm') - print(f" {call_str}") - os.system(call_str) - bmp_file = corrected_file + '.bmp' - if os.path.isfile(bmp_file): - print(f" BMP: {bmp_file}") - else: - print(" Warning: BMP file not generated") - else: - print(f" Warning: geo_amp not found: {geo_amp}") - - print(f"\nGACOS atmospheric correction completed successfully!") - else: - print("\nGACOS correction failed!") - sys.exit(1) - - sys.exit(0) - - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/gacos_gamma_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/gacos_gamma_all.py deleted file mode 100644 index f1ff747..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/gacos_gamma_all.py +++ /dev/null @@ -1,494 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import time -import glob -import argparse -import subprocess - -from pyint import _utils as ut - - -def work(data0): - """Worker function for parallel processing""" - cmd = data0[0] - err_file = data0[1] - p = subprocess.run(cmd, shell=False, stderr=subprocess.PIPE, stdout=subprocess.PIPE) - stdout = p.stdout - stderr = p.stderr - - if type(stderr) == bytes: - aa = stderr.decode("utf-8") - else: - aa = stderr - - if aa: - print(aa) - str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' - with open(err_file, 'a') as f: - f.write(str0) - f.write(aa) - f.write('\n') - - return - - -INTRODUCTION = ''' -------------------------------------------------------------------- - Apply GACOS atmospheric correction to all interferograms - for one project using GAMMA. - - This script processes all interferogram pairs listed in - ifgram_list.txt and applies GACOS tropospheric correction. -------------------------------------------------------------------- -''' - -EXAMPLE = ''' - Usage: - gacos_gamma_all.py projectName - gacos_gamma_all.py projectName --parallel 4 - gacos_gamma_all.py projectName --parallel 4 --ztd-dir /path/to/gacos - gacos_gamma_all.py projectName --parallel 4 --ifgramList-txt /test/ifgram_list.txt -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Apply GACOS atmospheric correction to all interferograms for one project.', - formatter_class=argparse.RawTextHelpFormatter, - epilog=INTRODUCTION + '\n' + EXAMPLE) - - parser.add_argument('projectName', help='projectName for processing.') - parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, - help='Enable parallel processing and Specify the number of processors.') - parser.add_argument('--ifgarmList-txt', dest='ifgarmListTxt', - help='provided ifgram_list_txt. default: using ifgram_list.txt under projectName folder.') - parser.add_argument('--ztd-dir', dest='ztdDir', - help='Directory containing GACOS ZTD files. Default: projectName/GACOS') - parser.add_argument('--email', dest='email', - help='Email for GACOS auto-download.') - parser.add_argument('--auto-download', dest='autoDownload', action='store_true', - help='Automatically submit and download missing GACOS data.') - parser.add_argument('--email-user', dest='emailUser', help='Email username for IMAP download.') - parser.add_argument('--email-pass', dest='emailPass', help='Email password for IMAP download.') - parser.add_argument('--email-host', dest='emailHost', default='imap.gmail.com', help='IMAP host.') - parser.add_argument('--email-port', dest='emailPort', type=int, default=None, help='IMAP port.') - parser.add_argument('--email-ssl', dest='emailSsl', action='store_true', default=True, help='Use SSL.') - parser.add_argument('--wait-hours', dest='waitHours', type=int, default=24, - help='Max hours to wait for GACOS email.') - parser.add_argument('--skip-existing', dest='skipExisting', action='store_true', - help='Skip pairs that already have corrected output.') - - inps = parser.parse_args() - return inps - - -def check_gacos_data_availability(dates, gacos_dir, bounds): - """ - Check if GACOS ZTD data is available for all dates. - - Parameters: - ----------- - dates : list - List of acquisition dates (YYYYMMDD format) - gacos_dir : str - Directory containing GACOS data - bounds : tuple - Bounding box (West, South, East, North) - - Returns: - -------- - missing_dates : list - List of dates that don't have GACOS data - """ - missing_dates = [] - - for date in dates: - # Check for ZTD files - ztd_pattern = os.path.join(gacos_dir, date + "*.ztd.tif") - ztd_files = glob.glob(ztd_pattern) - - if not ztd_files: - # Also check for .ztd format - ztd_pattern = os.path.join(gacos_dir, date + "*.ztd") - ztd_files = glob.glob(ztd_pattern) - - if not ztd_files: - missing_dates.append(date) - - return missing_dates - - -def get_project_dates(projectName, scratchDir): - """ - Get all acquisition dates from the project. - - Returns: - -------- - dates : list - List of unique acquisition dates - """ - slcDir = scratchDir + '/' + projectName + "/SLC" - rslcDir = scratchDir + '/' + projectName + "/RSLC" - - dates = set() - - # Check SLC directory - if os.path.exists(slcDir): - for item in os.listdir(slcDir): - if os.path.isdir(os.path.join(slcDir, item)): - if len(item) == 8 and item.isdigit(): - dates.add(item) - - # Check RSLC directory - if os.path.exists(rslcDir): - for item in os.listdir(rslcDir): - if os.path.isdir(os.path.join(rslcDir, item)): - if len(item) == 8 and item.isdigit(): - dates.add(item) - - return sorted(list(dates)) - - -def main(argv): - start_time = time.time() - inps = cmdLineParse() - projectName = inps.projectName - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - projectDir = scratchDir + '/' + projectName - ifgDir = scratchDir + '/' + projectName + '/ifgrams' - - templateDict = ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - masterDate = templateDict['masterDate'] - - # GACOS data directory (CLI > template > default) - if inps.ztdDir: - gacosDir = inps.ztdDir - elif templateDict.get('gacos_dir', '').strip(): - gacosDir = templateDict['gacos_dir'].strip() - else: - gacosDir = projectDir + '/GACOS' - - if not os.path.exists(gacosDir): - os.makedirs(gacosDir) - print(f"Created GACOS directory: {gacosDir}") - - # 从模板读取邮箱配置（CLI 优先于模板） - gacos_email = (inps.email or templateDict.get('gacos_email', '').strip()) or None - gacos_email_user = (getattr(inps, 'emailUser', None) or templateDict.get('gacos_email_user', '').strip()) or None - gacos_email_pass = (getattr(inps, 'emailPass', None) or templateDict.get('gacos_email_pass', '').strip()) or None - gacos_email_host = (getattr(inps, 'emailHost', None) or templateDict.get('gacos_email_host', 'imap.163.com').strip()) - gacos_email_port = getattr(inps, 'emailPort', None) or int(templateDict.get('gacos_email_port', '993').strip()) - gacos_email_ssl = templateDict.get('gacos_email_ssl', '1').strip() == '1' - gacos_check_interval = int(templateDict.get('gacos_check_interval', '60').strip()) - - # 如果有用户名和密码，自动启用 auto-download - auto_download = getattr(inps, 'autoDownload', False) or (gacos_email_user and gacos_email_pass) - - # Key files - demDir = projectDir + '/DEM' - slcDir = projectDir + '/SLC' - dempar = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem.par' - slcpar = slcDir + '/' + masterDate + '/' + masterDate + '.slc.par' - figdir = projectDir + '/figure' - - # Import pure Python correction functions from gacos_gamma - from pyint.gacos_gamma import (parse_dem_par, resample_ztd_to_dem, - find_existing_ztd, apply_gacos_correction_python, - submit_gacos_request, auto_gacos_workflow) - - # Get interferogram list - if inps.ifgarmListTxt: - ifgramList_txt = inps.ifgarmListTxt - else: - ifgramList_txt = projectDir + '/ifgram_list.txt' - - if not os.path.isfile(ifgramList_txt): - print(f"Error: Interferogram list not found: {ifgramList_txt}") - sys.exit(1) - - ifgList0 = ut.read_txt2array(ifgramList_txt) - - import numpy as np - ifgList0 = np.atleast_2d(np.array(ifgList0)) - ifgList = list(ifgList0[:, 0]) - - # Collect all unique dates - all_dates = set() - for ifg in ifgList: - mdate = ut.yyyymmdd(ifg.split('-')[0]) - sdate = ut.yyyymmdd(ifg.split('-')[1]) - all_dates.add(mdate) - all_dates.add(sdate) - all_dates = sorted(list(all_dates)) - - # Read DEM parameters - dem_info = parse_dem_par(dempar) - - print("="*60) - print("GACOS Atmospheric Correction (Pure Python, ZTD Pre-cache)") - print(f"Project: {projectName}") - print(f"Interferograms: {len(ifgList)}, Dates: {len(all_dates)}") - print(f"DEM: {dem_info['width']}×{dem_info['nlines']}") - print(f"GACOS dir: {gacosDir}") - print("="*60) - - # Check GACOS data availability using updated find_existing_ztd - print("\nChecking GACOS data availability...") - missing_dates = [] - ztd_paths = {} # {date: ztd_file_path} - for d in all_dates: - ztd = find_existing_ztd(d, gacosDir) - if ztd: - ztd_paths[d] = ztd - else: - missing_dates.append(d) - - if missing_dates: - print(f"\n⚠ GACOS data missing for {len(missing_dates)} dates: {missing_dates[:5]}...") - print(f"ZTD available: {len(ztd_paths)}/{len(all_dates)} dates") - - # 自动提交/下载缺失的 GACOS 数据 - if gacos_email: - # 从 DEM 参数计算研究区边界框 - North = dem_info['corner_lat'] - West = dem_info['corner_lon'] - South = North + (dem_info['nlines'] - 1) * dem_info['post_lat'] - East = West + (dem_info['width'] - 1) * dem_info['post_lon'] - bounds = (West, South, East, North) - - # 获取 SAR 采集时间（GACOS 要求非零） - acq_time = templateDict.get('gacos_acq_time', '').strip() - if not acq_time and os.path.isfile(slcpar): - try: - with open(slcpar) as f: - for line in f: - if line.startswith('center_time:'): - secs = float(line.split()[1]) - h = int(secs // 3600) - m = int((secs % 3600) // 60) - acq_time = f"{h:02d}:{m:02d}" - print(f"从 SLC par 读取采集时间: {acq_time} UTC") - break - except Exception: - pass - if not acq_time: - acq_time = '10:00' - print(f"使用默认采集时间: {acq_time} UTC") - - if auto_download and gacos_email_user and gacos_email_pass: - # 完整自动流程：提交 → 等待邮件 → 下载 - email_config = { - 'username': gacos_email_user, 'password': gacos_email_pass, - 'host': gacos_email_host, 'port': gacos_email_port, 'ssl': gacos_email_ssl - } - wait_hours = getattr(inps, 'waitHours', 24) - ztd_files = auto_gacos_workflow( - dates=missing_dates, bounds=bounds, gacos_dir=gacosDir, - email=gacos_email, email_config=email_config, - acquisition_time=acq_time, - wait_for_email=True, max_wait_hours=wait_hours, - check_interval=gacos_check_interval - ) - # 重新检查已下载的日期 - for d in missing_dates[:]: - ztd = find_existing_ztd(d, gacosDir) - if ztd: - ztd_paths[d] = ztd - missing_dates.remove(d) - else: - # 仅提交请求，不等待下载 - submit_gacos_request( - dates=missing_dates, bounds=bounds, - email=gacos_email, gacos_dir=gacosDir, - acquisition_time=acq_time - ) - print(f"\nGACOS request submitted for {len(missing_dates)} dates.") - print("Please wait for email notification, then re-run this script.") - if not ztd_paths: - print("No ZTD data available yet. Exiting.") - sys.exit(0) - else: - print("\nTip: 在模板中配置 gacos_email 可自动提交 GACOS 请求") - print(" 配置 gacos_email_user + gacos_email_pass 可自动轮询下载") - - if missing_dates: - print(f"\n仍有 {len(missing_dates)} 个日期缺失 ZTD，相关干涉对将被跳过") - - print(f"\nZTD available: {len(ztd_paths)}/{len(all_dates)} dates") - - # Pre-cache all ZTD resampled to DEM grid (each date only once) - print(f"\nPre-loading {len(ztd_paths)} ZTD files to DEM grid...") - t0 = time.time() - ztd_cache = {} - for i, (d, ztd_file) in enumerate(sorted(ztd_paths.items())): - ztd_rsc = ztd_file + '.rsc' - ztd_cache[d] = resample_ztd_to_dem(ztd_file, ztd_rsc, dem_info) - if (i + 1) % 10 == 0 or i == 0: - print(f" [{i+1}/{len(ztd_paths)}] {d}") - t_cache = time.time() - t0 - mem_gb = sum(v.nbytes for v in ztd_cache.values()) / 1024**3 - print(f" Pre-load done: {t_cache:.1f}s, ~{mem_gb:.2f} GB") - - # Read incidence angle and wavelength from SLC par - import numpy as np - inc_angle_deg = 39.0 - wavelength = 0.0554657595 - if os.path.isfile(slcpar): - with open(slcpar) as f: - for line in f: - if line.startswith('incidence_angle:'): - inc_angle_deg = float(line.split(':')[1].strip().split()[0]) - if line.startswith('radar_frequency:'): - freq = float(line.split(':')[1].strip().split()[0]) - wavelength = 299792458.0 / freq - cos_inc = np.cos(np.radians(inc_angle_deg)) - ztd_to_phase = 4.0 * np.pi / wavelength - - print(f"\n入射角: {inc_angle_deg:.4f}°, 波长: {wavelength:.10f} m") - - # Process each pair - print(f"\nProcessing {len(ifgList)} interferograms...\n") - - ok_count = 0 - skip_count = 0 - err_count = 0 - errors = [] - plot_count = 0 - MAX_PLOTS = 5 # only plot first 5 pairs - - for i, ifg in enumerate(ifgList): - mdate = ut.yyyymmdd(ifg.split('-')[0]) - sdate = ut.yyyymmdd(ifg.split('-')[1]) - pair = mdate + '-' + sdate - - workDir = ifgDir + '/' + pair - unw_file = workDir + '/geo_' + pair + '_' + rlks + 'rlks.diff_filt.unw' - out_file = workDir + '/geo_' + pair + '_' + rlks + 'rlks.diff_filt.unw.gacos' - - # Skip if already exists - if inps.skipExisting and os.path.isfile(out_file) and os.path.getsize(out_file) > 0: - skip_count += 1 - continue - - # Skip if input missing - if not os.path.isfile(unw_file): - err_count += 1 - errors.append(f"{pair}: geo_unw not found") - continue - - # Skip if ZTD missing - if mdate not in ztd_cache or sdate not in ztd_cache: - err_count += 1 - errors.append(f"{pair}: ZTD missing for {mdate if mdate not in ztd_cache else sdate}") - continue - - try: - # Read unwrapped phase - unw_data = np.fromfile(unw_file, dtype=np.float32).reshape( - dem_info['nlines'], dem_info['width']) - - # Compute phase correction from cached ZTD - dztd = ztd_cache[sdate] - ztd_cache[mdate] - phase_correction = dztd * ztd_to_phase / cos_inc - - # Valid pixel mask (exclude zeros, NaN, GAMMA artifacts/unwrap errors) - PHASE_THRESH = 1000.0 - valid = ((unw_data != 0) - & np.isfinite(unw_data) - & (np.abs(unw_data) < PHASE_THRESH) - & np.isfinite(phase_correction)) - n_valid = np.sum(valid) - - if n_valid == 0: - err_count += 1 - errors.append(f"{pair}: no valid pixels") - continue - - std_before = float(np.std(unw_data[valid])) - - # Apply correction (preserve original data, only modify valid pixels) - corrected = unw_data.copy() - corrected[valid] = unw_data[valid] - phase_correction[valid] - - # Demean (only valid pixels) - mean_val = np.mean(corrected[valid]) - corrected[valid] -= mean_val - - std_after = float(np.std(corrected[valid])) - - # Save - corrected.astype(np.float32).tofile(out_file) - - # Generate BMP preview using rasdt_pwr - geo_amp = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.amp' - if os.path.isfile(geo_amp): - call_str = ('rasdt_pwr ' + out_file + ' ' + geo_amp + ' ' - + str(dem_info['width']) - + ' - - - - -3.14 3.14 1 rmg.cm') - os.system(call_str) - - reduction = (1 - std_after / std_before) * 100 if std_before > 0 else 0 - ok_count += 1 - - # Plot first N pairs - do_plot = (plot_count < MAX_PLOTS) - if do_plot: - from pyint.gacos_gamma import plot_gacos_comparison - plot_gacos_comparison(unw_data, phase_correction, corrected, - valid, dem_info, std_before, std_after, - pair, figdir) - plot_count += 1 - - if ok_count <= 5 or ok_count % 50 == 0: - print(f" [{i+1}/{len(ifgList)}] {pair}: ✅ std {std_before:.2f}->{std_after:.2f} ({reduction:.1f}%)") - - except Exception as e: - err_count += 1 - errors.append(f"{pair}: {str(e)}") - if err_count <= 3: - print(f" [{i+1}/{len(ifgList)}] {pair}: ❌ {e}") - - elapsed = time.time() - start_time - - # Summary - print("\n" + "="*60) - print("GACOS Atmospheric Correction Summary") - print("="*60) - print(f" Total: {len(ifgList)}") - print(f" Success: {ok_count}") - print(f" Skipped: {skip_count}") - print(f" Failed: {err_count}") - - if errors: - err_txt = projectDir + '/gacos_gamma_all.err' - with open(err_txt, 'w') as f: - for e in errors: - f.write(e + '\n') - print(f"\n Error log: {err_txt}") - for e in errors[:5]: - print(f" {e}") - - print(f"\n Time: {elapsed/60:.1f} min") - if plot_count > 0: - print(f" Comparison plots: {figdir}/GACOS_comparison_*.png") - print("="*60) - - sys.exit(0) - - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/gamma2licsbas_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/gamma2licsbas_gamma.py deleted file mode 100644 index 695f336..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/gamma2licsbas_gamma.py +++ /dev/null @@ -1,319 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# -import os -import sys -import argparse - -from pyint import _utils as ut - - -INTRODUCTION = ''' -------------------------------------------------------------------- - 将 geocode_gamma.py 已地理编码的 GAMMA 二进制产品转换为 - LiCSBAS 所需的 GeoTIFF 格式, 并组织到 GEOC 目录结构。 - - 本脚本 **不做任何地理编码或计算**, 仅执行格式转换: - 1. data2geotiff : GAMMA 二进制 (EQA) → GeoTIFF - 2. GMT grdmath : 从 look_vector 计算 E/N/U 分量 - 3. 文件复制 : 组织到 GEOC/{Pair}/ 目录 - - 所有地理编码产品由 geocode_gamma.py 提供: - amp, corr, unw, diff_filt, diff, dem, lv_theta, lv_phi -''' - -EXAMPLE = ''' - Usage: - gamma2licsbas_gamma.py projectName Mdate-Sdate - gamma2licsbas_gamma.py PacayaT163TsxHhA 20150102-20150601 -------------------------------------------------------------------- -''' - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Convert geocoded GAMMA binaries to LiCSBAS GeoTIFF format.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('pair',help='Master-Slave, e.g., 20150101-20150106.') - - inps = parser.parse_args() - return inps - - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - Pair = inps.pair - - scratchDir = os.getenv('SCRATCHDIR') - ifgDir = scratchDir + '/' + projectName + "/ifgrams" - demDir = scratchDir + '/' + projectName + "/DEM" - workDir = ifgDir + '/' + Pair - - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - masterDate = templateDict['masterDate'] - - ################################################################# - # DEM 参数 (直接引用, 不复制不删除) - ################################################################# - DEM_par = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem.par' - DEM = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem' - DEM_tif = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem.tif' - - # look_vector 文件 (由 geocode_gamma.py 生成) - lv_theta = workDir + '/' + 'lv_theta' - lv_phi = workDir + '/' + 'lv_phi' - lv_theta_tif = workDir + '/' + 'lv_theta.geo.tif' - lv_phi_tif = workDir + '/' + 'lv_phi.geo.tif' - lv_theta_grd = workDir + '/' + 'lv_theta.grd' - lv_theta_grd_final = workDir + '/' + 'inc_deg.nc' - lv_phi_grd = workDir + '/' + 'lv_phi.grd' - lv_phi_grd_final = workDir + '/' + 'azi_deg.nc' - UE_grd = workDir + '/' + 'ue.grd' - UN_grd = workDir + '/' + 'un.grd' - UU_grd = workDir + '/' + 'uu.grd' - UE_tif = workDir + '/' + 'ue.geo.E.tif' - UN_tif = workDir + '/' + 'un.geo.N.tif' - UU_tif = workDir + '/' + 'uu.geo.U.tif' - - if not os.path.exists(lv_theta) or not os.path.exists(lv_phi): - print(f"WARNING: look_vector 文件不存在, 请先运行 geocode_gamma.py {projectName} {Pair}") - print(f" 缺少: lv_theta={os.path.exists(lv_theta)}, lv_phi={os.path.exists(lv_phi)}") - - GEOC = scratchDir + '/' + projectName + '/GEOC' - if not os.path.isdir(GEOC): - os.mkdir(GEOC) - Store_dir = GEOC + '/' + Pair - if not os.path.isdir(Store_dir): - os.mkdir(Store_dir) - - # 判断DEM_tif是否存在 - if not os.path.exists(DEM_tif): - call_str = 'data2geotiff ' + DEM_par + ' ' + DEM + ' 2 ' + DEM_tif - os.system(call_str) - else: - print(f"File {DEM_tif} already exists, skipping generation.") - - # 判断lv_theta_tif是否存在 - if not os.path.exists(lv_theta_tif): - call_str = 'data2geotiff ' + DEM_par + ' ' + lv_theta + ' 2 ' + lv_theta_tif - os.system(call_str) - else: - print(f"File {lv_theta_tif} already exists, skipping generation.") - - # 判断lv_phi_tif是否存在 - if not os.path.exists(lv_phi_tif): - call_str = 'data2geotiff ' + DEM_par + ' ' + lv_phi + ' 2 ' + lv_phi_tif - os.system(call_str) - else: - print(f"File {lv_phi_tif} already exists, skipping generation.") - - # 判断lv_theta_grd是否存在 - if not os.path.exists(lv_theta_grd): - call_str = 'gdal_translate -of GSBG ' + lv_theta_tif + ' ' + lv_theta_grd - os.system(call_str) - else: - print(f"File {lv_theta_grd} already exists, skipping conversion.") - - # 判断lv_phi_grd是否存在 - if not os.path.exists(lv_phi_grd): - call_str = 'gdal_translate -of GSBG ' + lv_phi_tif + ' ' + lv_phi_grd - os.system(call_str) - else: - print(f"File {lv_phi_grd} already exists, skipping conversion.") - - # 判断lv_theta_grd_final是否存在 - if not os.path.exists(lv_theta_grd_final): - call_str = 'gmt grdmath 90 ' + lv_theta_grd + ' 3.1415926 DIV 180 MUL SUB = ' + lv_theta_grd_final - os.system(call_str) - else: - print(f"File {lv_theta_grd_final} already exists, skipping calculation.") - - # 判断lv_phi_grd_final是否存在 - if not os.path.exists(lv_phi_grd_final): - call_str = 'gmt grdmath -180 ' + lv_phi_grd + ' 3.1415926 DIV 180 MUL SUB = ' + lv_phi_grd_final - os.system(call_str) - else: - print(f"File {lv_phi_grd_final} already exists, skipping calculation.") - - # 判断UE_grd是否存在 - if not os.path.exists(UE_grd): - call_str = 'gmt grdmath ' + lv_phi_grd_final + ' COSD ' + lv_theta_grd_final + ' SIND MUL NEG = ' + UE_grd - os.system(call_str) - else: - print(f"File {UE_grd} already exists, skipping calculation.") - - # 判断UN_grd是否存在 - if not os.path.exists(UN_grd): - call_str = 'gmt grdmath ' + lv_phi_grd_final + ' SIND ' + lv_theta_grd_final + ' SIND MUL = ' + UN_grd - os.system(call_str) - else: - print(f"File {UN_grd} already exists, skipping calculation.") - - # 判断UU_grd是否存在 - if not os.path.exists(UU_grd): - call_str = 'gmt grdmath ' + lv_theta_grd_final + ' COSD = ' + UU_grd - os.system(call_str) - else: - print(f"File {UU_grd} already exists, skipping calculation.") - - # 判断UE_tif是否存在 - if not os.path.exists(UE_tif): - call_str = 'gdal_translate -of GTiff ' + UE_grd + ' ' + UE_tif - os.system(call_str) - else: - print(f"File {UE_tif} already exists, skipping conversion.") - - # 判断UN_tif是否存在 - if not os.path.exists(UN_tif): - call_str = 'gdal_translate -of GTiff ' + UN_grd + ' ' + UN_tif - os.system(call_str) - else: - print(f"File {UN_tif} already exists, skipping conversion.") - - # 判断UU_tif是否存在 - if not os.path.exists(UU_tif): - call_str = 'gdal_translate -of GTiff ' + UU_grd + ' ' + UU_tif - os.system(call_str) - else: - print(f"File {UU_tif} already exists, skipping conversion.") - -##### get unw.tif - GeoMamp = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.amp' - GeoMamp_tif = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.amp.tif' - GeoCOR = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.diff_filt.cor' - GeoCOR_tif = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.cor.tif' - GeoUNW_tif = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.unw.tif' - GeoUNW = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.unw' - GeoDIFF = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt' - GeoDIFF_tif = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.tif' - Geodiff = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff' - Geodiff_tif = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff.tif' - GeoATMCOR_UNW = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.unw.gacos' - GeoATMCOR_UNW_tif = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.unw.gacos.tif' - Geodem = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.hgt' - Geodem_tif = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.hgt.tif' - - # 判断GeoUNW_tif是否存在 - if not os.path.exists(GeoUNW_tif): - call_str = 'data2geotiff ' + DEM_par + ' ' + GeoUNW + ' ' + ' 2 ' + GeoUNW_tif - os.system(call_str) - else: - print(f"File {GeoUNW_tif} already exists, skipping conversion.") - - # 判断Geodem_tif是否存在 - if not os.path.exists(Geodem_tif): - call_str = 'data2geotiff ' + DEM_par + ' ' + Geodem + ' ' + ' 2 ' + Geodem_tif - os.system(call_str) - else: - print(f"File {Geodem_tif} already exists, skipping conversion.") - - # 判断GeoCOR_tif是否存在 - if not os.path.exists(GeoCOR_tif): - call_str = 'data2geotiff ' + DEM_par + ' ' + GeoCOR + ' ' + ' 2 ' + GeoCOR_tif - os.system(call_str) - else: - print(f"File {GeoCOR_tif} already exists, skipping conversion.") - - # 判断GeoMamp_tif是否存在 - if not os.path.exists(GeoMamp_tif): - call_str = 'data2geotiff ' + DEM_par + ' ' + GeoMamp + ' ' + ' 2 ' + GeoMamp_tif - os.system(call_str) - else: - print(f"File {GeoMamp_tif} already exists, skipping conversion.") - - # 判断GeoDIFF_tif是否存在 - if not os.path.exists(GeoDIFF_tif): - call_str = 'data2geotiff ' + DEM_par + ' ' + GeoDIFF + ' ' + ' 2 ' + GeoDIFF_tif - os.system(call_str) - else: - print(f"File {GeoDIFF_tif} already exists, skipping conversion.") - - # 判断Geodiff_tif是否存在 - if not os.path.exists(Geodiff_tif): - call_str = 'data2geotiff ' + DEM_par + ' ' +Geodiff + ' ' + ' 2 ' + Geodiff_tif - os.system(call_str) - else: - print(f"File {Geodiff_tif} already exists, skipping conversion.") - - # 判断GeoATMCOR_UNW_tif是否存在（GACOS校正后的解缠结果） - if os.path.exists(GeoATMCOR_UNW) and not os.path.exists(GeoATMCOR_UNW_tif): - call_str = 'data2geotiff ' + DEM_par + ' ' + GeoATMCOR_UNW + ' ' + ' 2 ' + GeoATMCOR_UNW_tif - os.system(call_str) - elif os.path.exists(GeoATMCOR_UNW_tif): - print(f"File {GeoATMCOR_UNW_tif} already exists, skipping conversion.") - - GeoCOR_tif0 = Store_dir + '/' + Pair + '.geo.cc.tif' - GeoUNW_tif0 = Store_dir + '/' + Pair + '.geo.unw.tif' - UU_tif0 = GEOC + '/' + 'uu.geo.U.tif' - UE_tif0 = GEOC + '/' + 'ue.geo.E.tif' - UN_tif0 = GEOC + '/' + 'un.geo.N.tif' - dem_tif0 = GEOC + '/' + masterDate + '.geo.hgt.tif' - amp_tif0 = GEOC + '/' + masterDate + '.geo.mli.tif' - GeoDIFF_tif0 = Store_dir + '/' + Pair + '.geo.diff_pha.tif' - Geodiff_tif0 = Store_dir + '/' + Pair + '.geo.diff_unfiltered_pha.tif' - GeoATMCOR_UNW_tif0 = Store_dir + '/' + Pair + '.geo.unw.gacos.tif' - # 判断并复制文件，如果目标文件已存在则不复制 - if not os.path.exists(GeoUNW_tif0): - ut.copy_file(GeoUNW_tif, GeoUNW_tif0) - else: - print(f"File {GeoUNW_tif0} already exists, skipping copy.") - - if not os.path.exists(GeoCOR_tif0): - ut.copy_file(GeoCOR_tif, GeoCOR_tif0) - else: - print(f"File {GeoCOR_tif0} already exists, skipping copy.") - - if not os.path.exists(UU_tif0): - ut.copy_file(UU_tif, UU_tif0) - else: - print(f"File {UU_tif0} already exists, skipping copy.") - - if not os.path.exists(UE_tif0): - ut.copy_file(UE_tif, UE_tif0) - else: - print(f"File {UE_tif0} already exists, skipping copy.") - - if not os.path.exists(UN_tif0): - ut.copy_file(UN_tif, UN_tif0) - else: - print(f"File {UN_tif0} already exists, skipping copy.") - if not os.path.exists(GeoDIFF_tif0): - ut.copy_file(GeoDIFF_tif, GeoDIFF_tif0) - else: - print(f"File {GeoDIFF_tif0} already exists, skipping copy.") - if not os.path.exists(Geodiff_tif0): - ut.copy_file(Geodiff_tif, Geodiff_tif0) - else: - print(f"File {Geodiff_tif0} already exists, skipping copy.") - - if os.path.exists(GeoATMCOR_UNW_tif) and not os.path.exists(GeoATMCOR_UNW_tif0): - ut.copy_file(GeoATMCOR_UNW_tif, GeoATMCOR_UNW_tif0) - elif os.path.exists(GeoATMCOR_UNW_tif0): - print(f"File {GeoATMCOR_UNW_tif0} already exists, skipping copy.") - - if not os.path.exists(dem_tif0): - ut.copy_file(Geodem_tif, dem_tif0) - else: - print(f"File {dem_tif0} already exists, skipping copy.") - - if not os.path.exists(amp_tif0): - ut.copy_file(GeoMamp_tif, amp_tif0) - else: - print(f"File {amp_tif0} already exists, skipping copy.") - - print("Convert to LiCSBAS format is done!") - sys.exit(0) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/gamma2licsbas_gamma_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/gamma2licsbas_gamma_all.py deleted file mode 100644 index 26f8916..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/gamma2licsbas_gamma_all.py +++ /dev/null @@ -1,122 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import time -import glob -import argparse - -import subprocess -from pyint import _utils as ut - -def work(data0): - cmd = data0[0] - err_file = data0[1] - p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) - stdout = p.stdout - stderr = p.stderr - - if type(stderr) == bytes: - aa=stderr.decode("utf-8") - else: - aa = stderr - - if aa: - print(aa) - str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + '\n' - with open(err_file, 'a') as f: - f.write(str0) - f.write(aa) - f.write('\n') - - return -######################################################################### - -INTRODUCTION = ''' -------------------------------------------------------------------- - Geocode interferograms for one project using GAMMA. - -''' - -EXAMPLE = ''' - Usage: - geocode_gamma_all.py projectName - geocode_gamma_all.py projectName --parallel 4 - geocode_gamma_all.py projectName --parallel 4 --ifgramList-txt /test/ifgram_list.txt -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Geocode interferograms for one project using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') - parser.add_argument('--ifgarmList-txt', dest='ifgarmListTxt', help='provided ifgram_list_txt. default: using ifgram_list.txt under projectName folder.') - - inps = parser.parse_args() - return inps - - -def main(argv): - start_time = time.time() - inps = cmdLineParse() - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - projectDir = scratchDir + '/' + projectName - ifgDir = scratchDir + '/' + projectName + '/ifgrams' - GEOC = scratchDir + '/' + projectName + '/GEOC' - if not os.path.isdir(GEOC): - os.makedirs(GEOC, exist_ok=True) - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - - if inps.ifgarmListTxt: ifgramList_txt = inps.ifgarmListTxt - else: ifgramList_txt = scratchDir + '/' + projectName + '/ifgram_list.txt' - ifgList0 = ut.read_txt2array(ifgramList_txt) -# ifgList = ifgList0[:,0] - if len(ifgList0)==3: - ifgList=ifgList0[0] - ifgList=[ifgList] - else: - ifgList=ifgList0[:,0] - - err_txt = scratchDir + '/' + projectName + '/gamma2licsbas_gamma_all.err' - if os.path.isfile(err_txt): os.remove(err_txt) - - data_para = [] - for i in range(len(ifgList)): - #m0 = ut.yyyymmdd(ifgList[i].split('-')[0]) - #s0 = ut.yyyymmdd(ifgList[i].split('-')[1]) - cmd0 = ['gamma2licsbas_gamma.py',projectName, ifgList[i]] - data0 = [cmd0,err_txt] - geo_file0 = GEOC + '/' + ifgList[i] + '/'+ ifgList[i] + '.geo.unw.tif' - - k00 = 0 - if os.path.isfile(geo_file0): - if os.path.getsize(geo_file0) > 0: - k00 = 1 - if k00==0: - data_para.append(data0) - - ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) - print("Geocode interferograms for project %s is done! " % projectName) - ut.print_process_time(start_time, time.time()) - - sys.exit(0) - -if __name__ == '__main__': - main(sys.argv[:]) - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/generate_amp_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/generate_amp_all.py deleted file mode 100644 index 17d26bf..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/generate_amp_all.py +++ /dev/null @@ -1,102 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import getopt -import time -import glob -import argparse - -import subprocess -from pyint import _utils as ut - - -def work(data0): - cmd = data0[0] - err_txt = data0[1] - - p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) - stdout = p.stdout - stderr = p.stderr - - if type(stderr) == bytes: - aa=stderr.decode("utf-8") - else: - aa = stderr - - if aa: - str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' - #print(aa) - with open(err_txt, 'a') as f: - f.write(str0) - f.write(aa) - f.write('\n') - - return -######################################################################### - -INTRODUCTION = ''' -------------------------------------------------------------------- - Generate multilooked amp for one project. - -''' - -EXAMPLE = ''' - Usage: - generate_amp_all.py projectName - generate_amp_all.py projectName --parallel 4 -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Generate multilooked amp for one project.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') - - inps = parser.parse_args() - return inps - - -def main(argv): - start_time = time.time() - inps = cmdLineParse() - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - projectDir = scratchDir + '/' + projectName - slcDir = scratchDir + '/' + projectName + '/SLC' - rslcDir = scratchDir + '/' + projectName + '/RSLC' - if not os.path.isdir(rslcDir): os.mkdir(rslcDir) - - cmd_command = 'generate_multilook_amp.py' - - err_txt = scratchDir + '/' + projectName + '/generate_amp_all.err' - if os.path.isfile(err_txt): os.remove(err_txt) - - data_para = [] - slc_list = [os.path.basename(fname) for fname in sorted(glob.glob(rslcDir + '/*'))] - #slc_list = ut.get_project_slcList(projectName) - for i in range(len(slc_list)): - cmd0 = [cmd_command,projectName,slc_list[i]] - data0 = [cmd0,err_txt] - data_para.append(data0) - - ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) - print("Generate multilooked apm for all rslcs is done! ") - ut.print_process_time(start_time, time.time()) - - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/generate_bursts_par.py b/.codex_tmp/pyint_variants/no_rescue/pyint/generate_bursts_par.py deleted file mode 100644 index a5fcdde..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/generate_bursts_par.py +++ /dev/null @@ -1,81 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import glob -import argparse - -from pyint import _utils as ut - - - -INTRODUCTION = ''' -------------------------------------------------------------------- - - Generate burst par files for a project. -''' - -EXAMPLE = """Usage: - - generate_bursts_par.py projectName - -------------------------------------------------------------------- -""" - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Check common busrts for TOPS data.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='Name of project.') - - inps = parser.parse_args() - - return inps - -################################################################################ - - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - Mdate = templateDict['masterDate'] - - processDir = scratchDir + '/' + projectName + "/PROCESS" - slcDir = scratchDir + '/' + projectName + "/SLC" - rslcDir = scratchDir + '/' + projectName + "/RSLC" - dateList = ut.get_project_slcList(projectName) - - for i in range(len(dateList)): - slc_dir = slcDir + '/' + dateList[i] - print(slc_dir) - for j in range(3): - kk = j - SLC = slc_dir + '/' + dateList[i] + '.IW' + str(kk+1)+'.slc' - SLCPar = slc_dir + '/' + dateList[i] + '.IW' + str(kk+1)+'.slc.par' - TOPPar = slc_dir + '/' + dateList[i] + '.IW' + str(kk+1)+'.slc.TOPS_par' - BURST = slc_dir + '/' + dateList[i] + '.IW' + str(kk+1)+'.burst.par' - - call_str = 'SLC_burst_corners ' + SLCPar + ' ' + TOPPar + ' > ' +BURST - os.system(call_str) - - - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/generate_multilook_amp.py b/.codex_tmp/pyint_variants/no_rescue/pyint/generate_multilook_amp.py deleted file mode 100644 index ceb392c..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/generate_multilook_amp.py +++ /dev/null @@ -1,87 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# -import numpy as np -import os -import sys -import argparse - -from pyint import _utils as ut - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Coregister TOPS S1-SLC to a reference S1-SLC using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName', help='Name of project.') - parser.add_argument('date', help='date of the slave S1 image. [mater date is read from template]') - inps = parser.parse_args() - return inps - - -INTRODUCTION = ''' -------------------------------------------------------------------- - Generate multilook amplitude images for coregistered SLCs. - -''' - -EXAMPLE = """Usage: - - generate_multilook_amp.py projectName Date - - generate_multilook_amp.py PacayaT163TsxHhA 20150102 -------------------------------------------------------------------- -""" - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - Date = inps.date - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - - slcDir = scratchDir + '/' + projectName + "/SLC" - rslcDir = scratchDir + '/' + projectName + "/RSLC" - if not os.path.isdir(rslcDir): os.mkdir(rslcDir) - #workDir = processDir + '/' + igramDir - workDir = rslcDir + '/' + Date - if not os.path.isdir(workDir): os.mkdir(workDir) - - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - Mdate = templateDict['masterDate'] - - - rslc = workDir + '/' + Date + '.rslc' - rslcPar = workDir + '/' + Date + '.rslc.par' - - amp = workDir + '/' + Date + '_' + rlks + 'rlks.amp' - ampPar = workDir + '/' + Date + '_' + rlks + 'rlks.amp.par' - - k0 = 0 - if os.path.isfile(ampPar): - if os.path.getsize(ampPar) > 0: - k0 =1 - - if k0==0: - call_str = 'multi_look ' + rslc + ' ' + rslcPar + ' ' + amp + ' ' + ampPar + ' ' + rlks + ' ' + azlks - os.system(call_str) - - nWIDTH = ut.read_gamma_par(ampPar,'read', 'range_samples') - - call_str = 'raspwr ' + amp + ' ' + nWIDTH - os.system(call_str) - - print("Generate amplitude image for RSLC %s is done !!" % Date) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/generate_rdc_dem.py b/.codex_tmp/pyint_variants/no_rescue/pyint/generate_rdc_dem.py deleted file mode 100644 index 12f16bd..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/generate_rdc_dem.py +++ /dev/null @@ -1,204 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# -import numpy as np -import os -import sys -import argparse - -from pyint import _utils as ut - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Generate radar-coordinates based DEM.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName', help='Name of project.') - inps = parser.parse_args() - return inps - - -INTRODUCTION = ''' -------------------------------------------------------------------- - - Generate radar-coordinates based DEM. - [Geo-coordinates DEM can be downloaded automatically if not provided.] -''' - -EXAMPLE = """Usage: - - generate_rdc_dem.py projectName - -------------------------------------------------------------------- -""" - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - - Mdate = templateDict['masterDate'] - - DEMDir = os.getenv('DEMDIR') - - processDir = scratchDir + '/' + projectName + "/ifgrams" - slcDir = scratchDir + '/' + projectName + "/SLC" - - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - - if not os.path.isdir(processDir): - call_str = 'mkdir ' + processDir - os.system(call_str) - - simDir = scratchDir + '/' + projectName + "/DEM" - if not os.path.isdir(simDir): - call_str='mkdir ' + simDir - - - workDir = simDir - - if 'DEM' in templateDict: - dem = templateDict['DEM'] - if not os.path.isfile(dem): - dem = DEMDir + '/' + projectName + '/' + projectName + '.dem' - call_str = 'echo DEM= ' + dem + ' >> ' + templateFile - os.system(call_str) - templateDict['DEM'] = dem - else: - dem = DEMDir + '/' + projectName + '/' + projectName + '.dem' - call_str = 'echo DEM = ' + dem + ' >> ' + templateFile - os.system(call_str) - - demPar = dem + ".par" - - if not os.path.isfile(dem): - call_str = 'makedem_pyint.py ' + projectName - os.system(call_str) - -# Parameter setting for simPhase - latovrSimphase = templateDict['dem_lat_ovr'] - lonovrSimphase = templateDict['dem_lon_ovr'] - - rposSimphase = templateDict['Simphase_rpos'] - azposSimphase = templateDict['Simphase_azpos'] - rwinSimphase = templateDict['Simphase_rwin'] - azwinSimphase = templateDict['Simphase_azwin'] - #rwinSimphase = '128' - #azwinSimphase = '128' - - threshSimphase = templateDict['Simphase_thresh'] - -# Definition of file - MslcDir = slcDir + '/' + Mdate - MslcImg = MslcDir + '/' + Mdate + '.slc' - MslcPar = MslcDir + '/' + Mdate + '.slc.par' - OFFSTD = workDir + '/' + Mdate + '_dem.off_std' - - - BLANK = workDir + '/' + Mdate + '.blk' - MamprlksImg = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp' - MamprlksPar = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp.par' - - - UTMDEMpar = simDir + '/'+ Mdate + '_'+ rlks + 'rlks.utm.dem.par' - UTMDEM = simDir + '/' + Mdate + '_'+ rlks + 'rlks.utm.dem' - UTM2RDC = simDir + '/' + Mdate + '_'+ rlks + 'rlks.utm_to_rdc0' - SIMSARUTM = simDir + '/' + Mdate + '_'+ rlks + 'rlks.sim_sar_utm' - PIX = simDir + '/' + Mdate + '_'+ rlks + 'rlks.pix' - LSMAP = simDir + '/' + Mdate + '_'+ rlks + 'rlks.ls_map' - SIMSARRDC = simDir + '/' + Mdate + '_'+ rlks + 'rlks.sim_sar_rdc' - SIMDIFFpar = simDir + '/' + Mdate + '_'+ rlks + 'rlks.diff_par' - SIMOFFS = simDir + '/' + Mdate + '_'+ rlks + 'rlks.offs' - SIMSNR = simDir + '/' + Mdate + '_'+ rlks + 'rlks.snr' - SIMOFFSET = simDir + '/' + Mdate + '_'+ rlks + 'rlks.offset' - SIMCOFF = simDir + '/' + Mdate + '_'+ rlks + 'rlks.coff' - SIMCOFFSETS = simDir + '/' + Mdate + '_'+ rlks + 'rlks.coffsets' - UTMTORDC = simDir + '/' + Mdate + '_'+ rlks + 'rlks.UTM_TO_RDC' - HGTSIM = simDir + '/' + Mdate + '_'+ rlks + 'rlks.rdc.dem' - - if not (os.path.isdir(simDir)): - os.makedirs(simDir) - - ut.createBlankFile(BLANK) - -### remove DEM look up table if it existed for considering gamma overlapping - - if os.path.isfile(UTMDEM): - os.remove(UTMDEM) - if os.path.isfile(UTMDEMpar): - os.remove(UTMDEMpar) - if os.path.isfile(UTM2RDC): - os.remove(UTM2RDC) - - nWidthUTMDEM0 = ut.read_gamma_par(demPar, 'read', 'width') - DateFormat = ut.read_gamma_par(demPar, 'read', 'data_format:') - - if DateFormat == 'INTEGER*2': - DF_type = '4' - else: - DF_type = '2' - - - tmp_dem = dem + '_tmp' - - if not os.path.isfile(tmp_dem): - call_str = 'replace_values ' + dem + ' -32767 0 ' + tmp_dem + ' ' + nWidthUTMDEM0 + ' 2 ' + DF_type - os.system(call_str) - call_str = 'cp ' + tmp_dem + ' ' + dem - os.system(call_str) - - call_str = "multi_look " + MslcImg + " " + MslcPar + " " + MamprlksImg + " " + MamprlksPar + " " + rlks + " " + azlks - os.system(call_str) - - call_str = 'gc_map1 ' + MamprlksPar + ' ' + '-' + ' ' + demPar + ' ' + dem + ' ' + UTMDEMpar + ' ' + UTMDEM + ' ' + UTM2RDC + ' ' + latovrSimphase + ' ' + lonovrSimphase + ' ' + SIMSARUTM + ' - - - - ' + PIX + ' ' + LSMAP + ' - 3 128' - #call_str = 'gc_map2 ' + MamprlksPar + ' ' + demPar + ' ' + dem + ' ' + UTMDEMpar + ' ' + UTMDEM + ' ' + UTM2RDC + ' ' + latovrSimphase + ' ' + lonovrSimphase + ' ' + ' ' + LSMAP + ' - - - - ' + SIMSARUTM + ' - - - ' + PIX - os.system(call_str) - - nWidthUTMDEM = ut.read_gamma_par(UTMDEMpar, 'read', 'width') - nLinePWR1 = ut.read_gamma_par(MamprlksPar, 'read', 'azimuth_lines') - nWidth = ut.read_gamma_par(MamprlksPar, 'read', 'range_samples') - - call_str = 'geocode ' + UTM2RDC + ' ' + SIMSARUTM + ' ' + nWidthUTMDEM + ' ' + SIMSARRDC + ' ' + nWidth + ' ' + nLinePWR1 + ' 0 0' - os.system(call_str) - - call_str = 'create_diff_par ' + MamprlksPar + ' ' + MamprlksPar + ' ' + SIMDIFFpar + ' 1 < ' + BLANK - os.system(call_str) - - call_str = 'init_offsetm ' + SIMSARRDC + ' ' + MamprlksImg + ' ' + SIMDIFFpar + ' 2 2 ' + rposSimphase + ' ' + azposSimphase #+ ' - - - 512' - os.system(call_str) - - call_str = 'offset_pwrm ' + SIMSARRDC + ' ' + MamprlksImg + ' ' + SIMDIFFpar + ' ' + SIMOFFS + ' ' + SIMSNR + ' ' + rwinSimphase + ' ' + azwinSimphase + ' ' + SIMOFFSET #+ ' - 128 128 ' + threshSimphase - os.system(call_str) - - call_str = 'offset_fitm ' + SIMOFFS + ' ' + SIMSNR + ' ' + SIMDIFFpar + ' ' + SIMCOFF + ' ' + SIMCOFFSETS + ' - > ' + OFFSTD - os.system(call_str) - - call_str = 'gc_map_fine ' + UTM2RDC + ' ' + nWidthUTMDEM + ' ' + SIMDIFFpar + ' ' + UTMTORDC + ' 1' - #print(call_str) - os.system(call_str) - - call_str = 'geocode ' + UTMTORDC + ' ' + UTMDEM + ' ' + nWidthUTMDEM + ' ' + HGTSIM + ' ' + nWidth + ' ' + nLinePWR1 + ' 0 0 - - 1 1 1' - os.system(call_str) - - - required_outputs = [UTMDEMpar, UTMDEM, UTMTORDC, HGTSIM] - missing = [path for path in required_outputs if not os.path.isfile(path)] - if missing: - raise RuntimeError('generate_rdc_dem is missing required outputs: ' + ', '.join(missing)) - - print("Create DEM in Radar Coordinates is done!") - - sys.exit(0) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/geocode_dolphin.py b/.codex_tmp/pyint_variants/no_rescue/pyint/geocode_dolphin.py deleted file mode 100644 index ef5edc8..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/geocode_dolphin.py +++ /dev/null @@ -1,500 +0,0 @@ -#!/usr/bin/env python -""" -Geocode dolphin unwrapped phase results with gdal_translate multi-looking and optional visualization -Output file will have the same pixel dimensions as the input phase file -""" - -import os -import argparse -import numpy as np -import rasterio -from osgeo import gdal, osr -import tempfile -import shutil -import subprocess -import matplotlib.pyplot as plt -from matplotlib.colors import LinearSegmentedColormap - -def gdal_translate_multilook(input_file, output_file, target_width, target_height, resample_alg='average'): - """ - Apply multi-looking using gdal_translate with -outsize option - """ - - cmd = [ - 'gdal_translate', - '-outsize', str(target_width), str(target_height), - '-r', resample_alg, - '-co', 'COMPRESS=LZW', - input_file, - output_file - ] - - print(f"Running: {' '.join(cmd)}") - result = subprocess.run(cmd, capture_output=True, text=True) - - if result.returncode != 0: - print(f"Error in gdal_translate: {result.stderr}") - raise RuntimeError(f"gdal_translate failed with return code {result.returncode}") - - print(f"Multi-looking completed: {input_file} -> {output_file}") - -def geocode_phase_file_same_size(phase_file, lon_file, lat_file, out_file, - output_srs=4326): - """ - Geocode a phase file using corresponding lat, lon files while maintaining same pixel dimensions - - Parameters: - ----------- - phase_file : str - Input unwrapped phase file from dolphin - lon_file : str - Longitude file (should match phase file dimensions) - lat_file : str - Latitude file (should match phase file dimensions) - out_file : str - Output geocoded file - output_srs : int - Output spatial reference system EPSG code - """ - - # Template for VRT source description - sourcexmltmpl = ''' - {0} - {1} - ''' - - # Create a temporary VRT file - temp_dir = tempfile.mkdtemp() - tempvrtname = os.path.join(temp_dir, 'geocode_temp.vrt') - - try: - # Open input file to get dimensions - with rasterio.open(phase_file) as src: - x_size, y_size = src.width, src.height - phase_transform = src.transform - phase_crs = src.crs - - # Create VRT driver - driver = gdal.GetDriverByName('VRT') - tempds = driver.Create(tempvrtname, x_size, y_size, 0) - - # Add band to VRT - tempds.AddBand(gdal.GDT_Float32) - tempds.GetRasterBand(1).SetMetadata( - {'source_0': sourcexmltmpl.format(phase_file, 1)}, - 'vrt_sources' - ) - - # Set spatial reference (WGS84) - sref = osr.SpatialReference() - sref.ImportFromEPSG(4326) - srswkt = sref.ExportToWkt() - - # Set geolocation metadata - tempds.SetMetadata({ - 'SRS': srswkt, - 'X_DATASET': lon_file, - 'X_BAND': '1', - 'Y_DATASET': lat_file, - 'Y_BAND': '1', - 'PIXEL_OFFSET': '0', - 'LINE_OFFSET': '0', - 'PIXEL_STEP': '1', - 'LINE_STEP': '1' - }, 'GEOLOCATION') - - # Clean up - tempds = None - - # Set output SRS - out_sref = osr.SpatialReference() - out_sref.ImportFromEPSG(output_srs) - - # Calculate output bounds from lon/lat files to maintain same pixel dimensions - with rasterio.open(lon_file) as lon_src: - lon_data = lon_src.read(1) - lon_transform = lon_src.transform - - with rasterio.open(lat_file) as lat_src: - lat_data = lat_src.read(1) - lat_transform = lat_src.transform - - # Calculate approximate bounds - valid_mask = ~(np.isnan(lon_data) | np.isnan(lat_data)) - if np.any(valid_mask): - min_lon = np.min(lon_data[valid_mask]) - max_lon = np.max(lon_data[valid_mask]) - min_lat = np.min(lat_data[valid_mask]) - max_lat = np.max(lat_data[valid_mask]) - - # Calculate approximate resolution to maintain same dimensions - approx_res_x = (max_lon - min_lon) / x_size - approx_res_y = (max_lat - min_lat) / y_size - - # Use the larger resolution to ensure we don't oversample - target_res = max(approx_res_x, approx_res_y) - - # Set warp options with calculated resolution to maintain size - warp_options = gdal.WarpOptions( - format='GTiff', - width=x_size, - height=y_size, - dstSRS=out_sref, - resampleAlg='near', - geoloc=True, - creationOptions=['COMPRESS=LZW', 'TILED=YES'] - ) - else: - # Fallback: use default warp options - warp_options = gdal.WarpOptions( - format='GTiff', - dstSRS=out_sref, - resampleAlg='near', - geoloc=True, - creationOptions=['COMPRESS=LZW', 'TILED=YES'] - ) - - # Perform geocoding - print(f"Geocoding {phase_file} to {out_file}") - print(f"Target output dimensions: {x_size} x {y_size}") - gdal.Warp(out_file, tempvrtname, options=warp_options) - - # Verify output dimensions match input - with rasterio.open(out_file) as out_src: - out_width, out_height = out_src.width, out_src.height - if out_width == x_size and out_height == y_size: - print(f"Success: Output dimensions match input ({x_size} x {y_size})") - else: - print(f"Warning: Output dimensions ({out_width} x {out_height}) differ from input ({x_size} x {y_size})") - - except Exception as e: - print(f"Error during geocoding: {e}") - raise - finally: - # Clean up temporary directory - shutil.rmtree(temp_dir) - -def get_file_dimensions(file_path): - """ - Get dimensions of a raster file - """ - with rasterio.open(file_path) as src: - return src.width, src.height - -def check_existing_multilook_files(phase_file, multilook_dir): - """ - Check if downsampled lon/lat files already exist and match phase file dimensions - """ - if not os.path.exists(multilook_dir): - return None, None - - # Get phase file dimensions - phase_width, phase_height = get_file_dimensions(phase_file) - - # Look for common downsampled file patterns - possible_lon_files = [ - os.path.join(multilook_dir, "lon.multilook.tif"), - os.path.join(multilook_dir, "lon.multilook.vrt"), - os.path.join(multilook_dir, "lon.strided.tif"), - os.path.join(multilook_dir, "lon.strided.vrt"), - os.path.join(multilook_dir, "lon_downsampled.tif"), - os.path.join(multilook_dir, "lon_downsampled.vrt"), - ] - - possible_lat_files = [ - os.path.join(multilook_dir, "lat.multilook.tif"), - os.path.join(multilook_dir, "lat.multilook.vrt"), - os.path.join(multilook_dir, "lat.strided.tif"), - os.path.join(multilook_dir, "lat.strided.vrt"), - os.path.join(multilook_dir, "lat_downsampled.tif"), - os.path.join(multilook_dir, "lat_downsampled.vrt"), - ] - - # Check for existing files that match dimensions - for lon_file in possible_lon_files: - if os.path.exists(lon_file): - try: - lon_width, lon_height = get_file_dimensions(lon_file) - if lon_width == phase_width and lon_height == phase_height: - # Found matching lon file, now check for matching lat file - for lat_file in possible_lat_files: - if os.path.exists(lat_file): - lat_width, lat_height = get_file_dimensions(lat_file) - if lat_width == phase_width and lat_height == phase_height: - print(f"Found existing downsampled files:") - print(f" Longitude: {lon_file}") - print(f" Latitude: {lat_file}") - return lon_file, lat_file - except: - # Skip files that can't be read - continue - - return None, None - -def create_phase_colormap(): - """ - Create a colormap suitable for phase data (cyclic colormap) - """ - # Create a cyclic colormap for phase data - colors = [ - (0, 0, 0.5), # Dark blue - (0, 0, 1), # Blue - (0, 1, 1), # Cyan - (0.5, 1, 0.5), # Light green - (1, 1, 0), # Yellow - (1, 0.5, 0), # Orange - (1, 0, 0), # Red - (0.5, 0, 0.5), # Purple - (0, 0, 0.5) # Dark blue (back to start) - ] - - return LinearSegmentedColormap.from_list('phase_cmap', colors, N=256) - -def create_simple_visualization(phase_file, output_file, viz_dir): - """ - Create a simple visualization of the geocoded output - """ - os.makedirs(viz_dir, exist_ok=True) - - # Create phase colormap - phase_cmap = create_phase_colormap() - - # Read the input and output files - with rasterio.open(phase_file) as src: - phase_data = src.read(1) - phase_width, phase_height = src.width, src.height - - with rasterio.open(output_file) as src: - output_data = src.read(1) - output_width, output_height = src.width, src.height - - # Calculate statistics for display - phase_stats = { - 'min': np.nanmin(phase_data), - 'max': np.nanmax(phase_data), - 'mean': np.nanmean(phase_data) - } - - output_stats = { - 'min': np.nanmin(output_data), - 'max': np.nanmax(output_data), - 'mean': np.nanmean(output_data) - } - - # Create figure with subplots - fig, axes = plt.subplots(1, 2, figsize=(16, 6)) - fig.suptitle('Input vs Output Comparison', fontsize=16, fontweight='bold') - - # Original phase - im1 = axes[0].imshow(phase_data, cmap=phase_cmap, - vmin=phase_stats['min'], vmax=phase_stats['max']) - axes[0].set_title(f'Original Phase\nDimensions: {phase_width} x {phase_height}') - axes[0].set_xlabel('Range') - axes[0].set_ylabel('Azimuth') - plt.colorbar(im1, ax=axes[0], label='Phase (rad)') - - # Geocoded output - im2 = axes[1].imshow(output_data, cmap=phase_cmap, - vmin=output_stats['min'], vmax=output_stats['max']) - axes[1].set_title(f'Geocoded Phase\nDimensions: {output_width} x {output_height}') - axes[1].set_xlabel('Longitude') - axes[1].set_ylabel('Latitude') - plt.colorbar(im2, ax=axes[1], label='Phase (rad)') - - # Add statistics text - stats_text1 = f"Min: {phase_stats['min']:.3f}\nMax: {phase_stats['max']:.3f}\nMean: {phase_stats['mean']:.3f}" - axes[0].text(0.02, 0.98, stats_text1, transform=axes[0].transAxes, - verticalalignment='top', fontsize=10, - bbox=dict(boxstyle='round', facecolor='white', alpha=0.8)) - - stats_text2 = f"Min: {output_stats['min']:.3f}\nMax: {output_stats['max']:.3f}\nMean: {output_stats['mean']:.3f}" - axes[1].text(0.02, 0.98, stats_text2, transform=axes[1].transAxes, - verticalalignment='top', fontsize=10, - bbox=dict(boxstyle='round', facecolor='white', alpha=0.8)) - - plt.tight_layout() - plt.savefig(os.path.join(viz_dir, 'input_output_comparison.png'), dpi=300, bbox_inches='tight') - plt.close(fig) - - print(f"Visualization saved to: {os.path.join(viz_dir, 'input_output_comparison.png')}") - -def main(): - parser = argparse.ArgumentParser( - description='Geocode dolphin unwrapped phase results with gdal_translate multi-looking and optional visualization. Output will have same pixel dimensions as input.' - ) - parser.add_argument('-i', '--input', required=True, - help='Input dolphin unwrapped phase file (GeoTIFF)') - parser.add_argument('--lon', required=True, - help='Input longitude file (e.g., lon.rdr.full.vrt)') - parser.add_argument('--lat', required=True, - help='Input latitude file (e.g., lat.rdr.full.vrt)') - parser.add_argument('-o', '--output', default='unwrapped_phase_geocoded.tif', - help='Output geocoded file path') - parser.add_argument('--multilook-dir', default='multilook_geoms', - help='Directory for multi-looked geometry files') - parser.add_argument('--viz-dir', default='visualizations', - help='Directory for visualization images') - parser.add_argument('--output-srs', type=int, default=4326, - help='Output SRS EPSG code (default: 4326 for WGS84)') - parser.add_argument('--resample-alg', default='average', - choices=['near', 'average', 'bilinear', 'cubic', 'cubicspline', 'lanczos', 'mode'], - help='Resampling algorithm for multi-looking (default: average)') - parser.add_argument('--no-cleanup', action='store_true', - help='Keep temporary multi-looked files') - parser.add_argument('--force-multilook', action='store_true', - help='Force multi-looking even if downsampled files exist') - - # Visualization options - viz_group = parser.add_mutually_exclusive_group() - viz_group.add_argument('--no-viz', action='store_true', - help='Skip visualization') - viz_group.add_argument('--viz', action='store_true', - help='Create visualization (default)') - - args = parser.parse_args() - - # Default to visualization if no option is specified - if not (args.no_viz or args.viz): - args.viz = True - - # Check if input files exist - for f in [args.input, args.lon, args.lat]: - if not os.path.exists(f): - print(f"Error: Input file does not exist: {f}") - return 1 - - # Get dimensions from phase file - print("Getting input file dimensions...") - phase_width, phase_height = get_file_dimensions(args.input) - lon_width, lon_height = get_file_dimensions(args.lon) - lat_width, lat_height = get_file_dimensions(args.lat) - - print(f"Phase file dimensions: {phase_width} x {phase_height}") - print(f"Longitude file dimensions: {lon_width} x {lon_height}") - print(f"Latitude file dimensions: {lat_width} x {lat_height}") - - # Check if multi-looking is needed or if files already exist - if (phase_width, phase_height) == (lon_width, lon_height) == (lat_width, lat_height): - print("Files already have matching dimensions. Skipping multi-looking.") - lon_multilook = args.lon - lat_multilook = args.lat - elif not args.force_multilook: - # Check if downsampled files already exist - existing_lon, existing_lat = check_existing_multilook_files(args.input, args.multilook_dir) - if existing_lon and existing_lat: - print("Using existing downsampled files.") - lon_multilook = existing_lon - lat_multilook = existing_lat - else: - # Need to create downsampled files - print("Downsampled files not found or don't match. Creating new ones...") - # Step 1: Apply multi-looking to lon/lat files using gdal_translate - print("Step 1: Applying multi-looking to lon/lat files using gdal_translate...") - - # Create output directory - os.makedirs(args.multilook_dir, exist_ok=True) - - # Generate output file paths - lon_basename = os.path.splitext(os.path.basename(args.lon))[0] - lat_basename = os.path.splitext(os.path.basename(args.lat))[0] - - lon_multilook = os.path.join(args.multilook_dir, f"{lon_basename}_multilook.tif") - lat_multilook = os.path.join(args.multilook_dir, f"{lat_basename}_multilook.tif") - - # Apply multi-looking using gdal_translate - try: - gdal_translate_multilook( - args.lon, lon_multilook, phase_width, phase_height, args.resample_alg - ) - gdal_translate_multilook( - args.lat, lat_multilook, phase_width, phase_height, args.resample_alg - ) - except Exception as e: - print(f"Multi-looking failed: {e}") - return 1 - else: - # Force multi-looking even if files exist - print("Forcing multi-looking...") - # Step 1: Apply multi-looking to lon/lat files using gdal_translate - print("Step 1: Applying multi-looking to lon/lat files using gdal_translate...") - - # Create output directory - os.makedirs(args.multilook_dir, exist_ok=True) - - # Generate output file paths - lon_basename = os.path.splitext(os.path.basename(args.lon))[0] - lat_basename = os.path.splitext(os.path.basename(args.lat))[0] - - lon_multilook = os.path.join(args.multilook_dir, f"{lon_basename}_multilook.tif") - lat_multilook = os.path.join(args.multilook_dir, f"{lat_basename}_multilook.tif") - - # Apply multi-looking using gdal_translate - try: - gdal_translate_multilook( - args.lon, lon_multilook, phase_width, phase_height, args.resample_alg - ) - gdal_translate_multilook( - args.lat, lat_multilook, phase_width, phase_height, args.resample_alg - ) - except Exception as e: - print(f"Multi-looking failed: {e}") - return 1 - - # Step 2: Geocode the phase file using (multi-looked) lon/lat - print("Step 2: Geocoding phase file (maintaining same dimensions)...") - - # Create output directory if needed - output_dir = os.path.dirname(args.output) - if output_dir and not os.path.exists(output_dir): - os.makedirs(output_dir, exist_ok=True) - - try: - geocode_phase_file_same_size( - phase_file=args.input, - lon_file=lon_multilook, - lat_file=lat_multilook, - out_file=args.output, - output_srs=args.output_srs - ) - - print(f"\nGeocoding completed successfully!") - print(f"Geocoded phase file saved to: {args.output}") - - # Display some info about the output file - with rasterio.open(args.output) as src: - print(f"Output file info:") - print(f" Dimensions: {src.width} x {src.height}") - print(f" CRS: {src.crs}") - if src.res: - print(f" Resolution: {src.res[0]:.6f} x {src.res[1]:.6f} degrees") - - except Exception as e: - print(f"Geocoding failed: {e}") - return 1 - - # Step 3: Create visualization if requested - if args.viz: - print("Step 3: Creating visualization...") - try: - create_simple_visualization(args.input, args.output, args.viz_dir) - except Exception as e: - print(f"Visualization failed: {e}") - else: - print("Skipping visualization as requested.") - - # Clean up temporary files if requested - if not args.no_cleanup and (lon_multilook != args.lon or lat_multilook != args.lat): - # Only clean up if we created new files and they're not the original ones - if (lon_multilook != args.lon and lat_multilook != args.lat and - os.path.exists(args.multilook_dir) and - (lon_multilook.startswith(args.multilook_dir) or - lat_multilook.startswith(args.multilook_dir))): - print("Cleaning up temporary files...") - shutil.rmtree(args.multilook_dir) - else: - print(f"Multi-looked files kept in: {args.multilook_dir}") - - return 0 - -if __name__ == "__main__": - exit(main()) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/geocode_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/geocode_gamma.py deleted file mode 100644 index 1ea3cc7..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/geocode_gamma.py +++ /dev/null @@ -1,371 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# -import os -import sys -import argparse -import numpy as np - -from pyint import _utils as ut - - -def sanitize_gamma_float(filepath, valid_max=1e6): - """清理 GAMMA 浮点数据文件中的无效值 (NaN/Inf/极端值 → 0.0)""" - data = np.fromfile(filepath, dtype=np.float32) - bad_mask = ~np.isfinite(data) | (np.abs(data) > valid_max) - n_bad = int(np.sum(bad_mask)) - if n_bad > 0: - data[bad_mask] = 0.0 - data.tofile(filepath) - print(f' [sanitize] {os.path.basename(filepath)}: ' - f'清理 {n_bad} 个无效像素') - -def geocode(inFile, outFile, UTMTORDC, nWidth, nWidthUTMDEM, nLineUTMDEM, geo_interp='0'): - - if '.unw' in os.path.basename(inFile): - call_str = 'geocode_back ' + inFile + ' ' + nWidth + ' ' + UTMTORDC + ' ' + outFile + ' ' + nWidthUTMDEM + ' ' + nLineUTMDEM + ' ' + geo_interp + ' 0' - elif '.amp' in os.path.basename(inFile): - call_str = 'geocode_back ' + inFile + ' ' + nWidth + ' ' + UTMTORDC + ' ' + outFile + ' ' + nWidthUTMDEM + ' ' + nLineUTMDEM + ' ' + geo_interp + ' 0' - elif '.cor' in os.path.basename(inFile): - call_str = 'geocode_back ' + inFile + ' ' + nWidth + ' ' + UTMTORDC + ' ' + outFile + ' ' + nWidthUTMDEM + ' ' + nLineUTMDEM + ' ' + geo_interp + ' 0' - elif '.dem' in os.path.basename(inFile): - call_str = 'geocode_back ' + inFile + ' ' + nWidth + ' ' + UTMTORDC + ' ' + outFile + ' ' + nWidthUTMDEM + ' ' + nLineUTMDEM + ' ' + geo_interp + ' 0' - else: - call_str = 'geocode_back ' + inFile + ' ' + nWidth + ' ' + UTMTORDC + ' ' + outFile + ' ' + nWidthUTMDEM + ' ' + nLineUTMDEM+ ' ' + geo_interp + ' 1' - - os.system(call_str) - - return - -INTRODUCTION = ''' -------------------------------------------------------------------- - Convert radar-coordinates products into geo-coordinates using GAMMA. - - 由模板参数 geocode_products 控制产品类型 (逗号分隔多选): - hyp3 : 基础产品 + dispmap(LOS/vert) + wrapped_phase + look_vector - licsbas : 基础产品 + look_vector - pot : Pixel Offset Tracking 位移图 (两步地理编码) - - 基础产品 (hyp3/licsbas 均需): amp, corr, unw, diff_filt, diff, dem - 一次调用自动处理所有选中的产品类型。 -''' - -EXAMPLE = ''' - Usage: - geocode_gamma.py projectName Mdate-Sdate - geocode_gamma.py PacayaT163TsxHhA 20150102-20150601 - - Template parameter: - geocode_products = hyp3,licsbas (default) - geocode_products = pot - geocode_products = hyp3,licsbas,pot -------------------------------------------------------------------- -''' - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Geocode radar-coordinate products using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('pair',help='Master-Slave, e.g., 20150101-20150106.') - - inps = parser.parse_args() - return inps - - -def geocode_pot(projectName, Pair, templateDict): - """POT 位移图地理编码: 偏移量几何 → MLI 几何 → EQA 几何""" - scratchDir = os.getenv('SCRATCHDIR') - projectDir = scratchDir + '/' + projectName - potDir = projectDir + '/offsets' - workDir = potDir + '/' + Pair - demDir = projectDir + '/DEM' - rslcDir = projectDir + '/RSLC' - slcDir = projectDir + '/SLC' - - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - masterDate = templateDict['masterDate'] - pot_disp_max = float(templateDict.get('pot_disp_max', '10')) - - Mdate = Pair.split('-')[0] - - if not os.path.isdir(workDir): - print(f'ERROR: POT 目录不存在: {workDir}') - return - - # ===== 公共文件 ===== - MampPar = rslcDir + '/' + masterDate + '/' + masterDate + '_' + rlks + 'rlks.amp.par' - Mamp = rslcDir + '/' + masterDate + '/' + masterDate + '_' + rlks + 'rlks.amp' - UTMTORDC = demDir + '/' + masterDate + '_' + rlks + 'rlks.UTM_TO_RDC' - UTMDEMpar = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem.par' - UTMDEM = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem' - SLCpar = slcDir + '/' + Mdate + '/' + Mdate + '.slc.par' - - if not os.path.isfile(UTMTORDC) or not os.path.isfile(UTMDEMpar): - print(' WARNING: POT 地理编码跳过 — 缺少 UTM_TO_RDC 或 utm.dem.par') - return - if not os.path.isfile(MampPar): - print(' WARNING: POT 地理编码跳过 — 缺少 master amp.par') - return - - mli_width = ut.read_gamma_par(MampPar, 'read', 'range_samples') - mli_nlines = ut.read_gamma_par(MampPar, 'read', 'azimuth_lines') - eqa_width = ut.read_gamma_par(UTMDEMpar, 'read', 'width') - eqa_nlines = ut.read_gamma_par(UTMDEMpar, 'read', 'nlines') - - # ===== POT 特有文件 ===== - MLI_pot_par = workDir + '/' + Mdate + '.mli_pot.par' - MLI_pot = workDir + '/' + Mdate + '.mli_pot' - disp_map = workDir + '/' + Pair + '.disp_map' - - if not os.path.isfile(disp_map): - print(f' WARNING: disp_map 不存在: {disp_map}') - return - - # 偏移量几何宽度 - off_width = ut.read_gamma_par(MLI_pot_par, 'read', 'range_samples') - disp_max_m = pot_disp_max * 10 - - # ===== Step A: 偏移量几何 → MLI 几何 (rdc_trans) ===== - print(' [POT] Step A: 偏移量几何 → MLI 几何') - mli_to_pot_lt = workDir + '/mli_to_pot.lt' - disp_map_mli = workDir + '/' + Pair + '.disp_map_mli' - - if not os.path.isfile(mli_to_pot_lt): - os.system(f'rdc_trans {MampPar} 0.1 {MLI_pot_par} {mli_to_pot_lt}') - - if not os.path.isfile(disp_map_mli): - os.system(f'geocode_back {disp_map} {off_width} {mli_to_pot_lt} ' - f'{disp_map_mli} {mli_width} {mli_nlines} 1 1') - - # ===== Step B: MLI 几何 → EQA 几何 ===== - print(' [POT] Step B: MLI 几何 → EQA 几何') - geo_disp = workDir + '/geo_' + Pair + '.disp_map' - - if not os.path.isfile(geo_disp): - os.system(f'geocode_back {disp_map_mli} {mli_width} {UTMTORDC} ' - f'{geo_disp} {eqa_width} {eqa_nlines} 1 1') - - # ===== Step C: 提取地理编码位移分量 ===== - print(' [POT] Step C: 提取位移分量 (real/imag/mag)') - geo_real = geo_disp + '.real' - geo_imag = geo_disp + '.imag' - geo_mag = geo_disp + '.mag' - - if not os.path.isfile(geo_real): - os.system(f'cpx_to_real {geo_disp} {geo_real} {eqa_width} 0') - sanitize_gamma_float(geo_real, disp_max_m) - if not os.path.isfile(geo_imag): - os.system(f'cpx_to_real {geo_disp} {geo_imag} {eqa_width} 1') - sanitize_gamma_float(geo_imag, disp_max_m) - if not os.path.isfile(geo_mag): - os.system(f'cpx_to_real {geo_disp} {geo_mag} {eqa_width} 3') - sanitize_gamma_float(geo_mag, disp_max_m) - - # ===== Step D: 地理编码 MLI 背景 ===== - geo_mli = workDir + '/geo_' + masterDate + '.mli' - if not os.path.isfile(geo_mli): - os.system(f'geocode_back {Mamp} {mli_width} {UTMTORDC} ' - f'{geo_mli} {eqa_width} {eqa_nlines} 5 0') - - # ===== Step E: 地理编码 CCP (互相关系数) ===== - # 自动查找最终 ccp 文件 - ccp_files = sorted([f for f in os.listdir(workDir) if f.startswith(Pair + '.ccp')]) - if ccp_files: - ccp_final = workDir + '/' + ccp_files[-1] - geo_ccp = workDir + '/geo_' + Pair + '.ccp' - if not os.path.isfile(geo_ccp): - # ccp 在偏移量几何, 需要两步 - ccp_mli = workDir + '/' + Pair + '.ccp_mli' - os.system(f'geocode_back {ccp_final} {off_width} {mli_to_pot_lt} ' - f'{ccp_mli} {mli_width} {mli_nlines} 1 0') - os.system(f'geocode_back {ccp_mli} {mli_width} {UTMTORDC} ' - f'{geo_ccp} {eqa_width} {eqa_nlines} 1 0') - - # ===== Step F: 视角矢量 (look_vector) ===== - print(' [POT] Step F: 视角矢量') - lv_theta = workDir + '/lv_theta' - lv_phi = workDir + '/lv_phi' - OFFpar = workDir + '/' + Pair + '.off' - - if not os.path.isfile(lv_theta) or not os.path.isfile(lv_phi): - if os.path.isfile(SLCpar) and os.path.isfile(OFFpar): - os.system(f'look_vector {SLCpar} {OFFpar} {UTMDEMpar} {UTMDEM} {lv_theta} {lv_phi}') - - # ===== BMP 可视化 ===== - print(' [POT] 生成 BMP') - disp_max_str = str(pot_disp_max) - os.system(f'rasdt_pwr {geo_mag} {geo_mli} {eqa_width} - - - - ' - f'-{disp_max_str} {disp_max_str} 1 rmg.cm {geo_mag}.bmp - - 24') - os.system(f'rasdt_pwr {geo_real} {geo_mli} {eqa_width} - - - - ' - f'-{disp_max_str} {disp_max_str} 1 rmg.cm {geo_real}.bmp - - 24') - os.system(f'rasdt_pwr {geo_imag} {geo_mli} {eqa_width} - - - - ' - f'-{disp_max_str} {disp_max_str} 1 rmg.cm {geo_imag}.bmp - - 24') - - print(' [POT] 地理编码完成!') - - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - Pair = inps.pair - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + '/' + projectName + '.template' - templateDict = ut.update_template(templateFile) - - # ===== 解析产品类型 ===== - products_str = templateDict.get('geocode_products', 'hyp3,licsbas') - products = set(p.strip().lower() for p in products_str.split(',')) - need_ifg = ('hyp3' in products) or ('licsbas' in products) - need_hyp3 = 'hyp3' in products - need_licsbas = 'licsbas' in products - need_pot = 'pot' in products - - print(f'[geocode_gamma] {projectName} / {Pair}') - print(f' geocode_products: {products_str} → ifg={need_ifg}, hyp3={need_hyp3}, licsbas={need_licsbas}, pot={need_pot}') - - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - masterDate = templateDict['masterDate'] - - ifgDir = scratchDir + '/' + projectName + '/ifgrams' - potDir = scratchDir + '/' + projectName + '/offsets' - demDir = scratchDir + '/' + projectName + '/DEM' - slcDir = scratchDir + '/' + projectName + '/SLC' - rslcDir = scratchDir + '/' + projectName + '/RSLC' - - # ========================================================= - # IFG 地理编码 (hyp3 和/或 licsbas 需要) - # ========================================================= - ifgWorkDir = ifgDir + '/' + Pair - if need_ifg and os.path.isdir(ifgWorkDir): - print('\n--- IFG 地理编码 ---') - workDir = ifgWorkDir - - ######### copy common file for parallel processing ############# - Mamp0 = rslcDir + '/' + masterDate + '/' + masterDate + '_' + rlks + 'rlks.amp' - MampPar0 = rslcDir + '/' + masterDate + '/' + masterDate + '_' + rlks + 'rlks.amp.par' - Mamp = workDir + '/' + masterDate + '_' + rlks + 'rlks.amp' - MampPar = workDir + '/' + masterDate + '_' + rlks + 'rlks.amp.par' - ut.copy_file(Mamp0, Mamp) - ut.copy_file(MampPar0, MampPar) - - ################################################################# - UNWIFG = workDir + '/' + Pair + '_' + rlks + 'rlks.diff_filt.unw' - ATMCOR_UNW = workDir + '/' + Pair + '_' + rlks + 'rlks.diff_filt.atmcor.unw' - DIFFIFG = workDir + '/' + Pair + '_' + rlks + 'rlks.diff_filt' - diffifg = workDir + '/' + Pair + '_' + rlks + 'rlks.diff' - CORIFG = workDir + '/' + Pair + '_' + rlks + 'rlks.diff_filt.cor' - rdcdem = demDir + '/' + masterDate + '_' + rlks + 'rlks.rdc.dem' - - GeoMamp = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.amp' - GeoCOR = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.diff_filt.cor' - GeoUNW = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.unw' - GeoDIFF = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt' - geodiff = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff' - GeoATMCOR_UNW = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.atmcor.unw' - Geodem = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.hgt' - - UTMTORDC0 = demDir + '/' + masterDate + '_' + rlks + 'rlks.UTM_TO_RDC' - UTMDEMpar0 = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem.par' - UTMTORDC = workDir + '/' + masterDate + '_' + rlks + 'rlks.UTM_TO_RDC' - UTMDEMpar = workDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem.par' - ut.copy_file(UTMTORDC0, UTMTORDC) - ut.copy_file(UTMDEMpar0, UTMDEMpar) - - nWidth = ut.read_gamma_par(MampPar, 'read', 'range_samples') - nWidthUTMDEM = ut.read_gamma_par(UTMDEMpar, 'read', 'width') - nLineUTMDEM = ut.read_gamma_par(UTMDEMpar, 'read', 'nlines') - - # --- 基础产品地理编码 (hyp3/licsbas 均需) --- - geo_interp = templateDict['geo_interp'] - geocode(Mamp, GeoMamp, UTMTORDC, nWidth, nWidthUTMDEM, nLineUTMDEM, geo_interp=geo_interp) - geocode(CORIFG, GeoCOR, UTMTORDC, nWidth, nWidthUTMDEM, nLineUTMDEM, geo_interp=geo_interp) - geocode(DIFFIFG, GeoDIFF, UTMTORDC, nWidth, nWidthUTMDEM, nLineUTMDEM, geo_interp=geo_interp) - geocode(UNWIFG, GeoUNW, UTMTORDC, nWidth, nWidthUTMDEM, nLineUTMDEM, geo_interp=geo_interp) - geocode(diffifg, geodiff, UTMTORDC, nWidth, nWidthUTMDEM, nLineUTMDEM, geo_interp=geo_interp) - geocode(rdcdem, Geodem, UTMTORDC, nWidth, nWidthUTMDEM, nLineUTMDEM, geo_interp=geo_interp) - - Mdate = Pair.split('-')[0] - SLCpar = slcDir + '/' + Mdate + '/' + Mdate + '.slc.par' - OFFpar = workDir + '/' + Pair + '_' + rlks + 'rlks.off' - UTMDEM = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem' - - # --- hyp3 专属: dispmap (LOS/vert 位移场) --- - if need_hyp3: - print(' [hyp3] dispmap + geocode 位移场') - los_disp_rdc = workDir + '/' + Pair + '_' + rlks + 'rlks.los_disp' - vert_disp_rdc = workDir + '/' + Pair + '_' + rlks + 'rlks.vert_disp' - - if os.path.isfile(UNWIFG) and os.path.isfile(SLCpar) and os.path.isfile(OFFpar): - hgt_arg = rdcdem if os.path.isfile(rdcdem) else '-' - if not os.path.isfile(los_disp_rdc): - os.system(f'dispmap {UNWIFG} {hgt_arg} {SLCpar} {OFFpar} {los_disp_rdc} 0') - if not os.path.isfile(vert_disp_rdc): - os.system(f'dispmap {UNWIFG} {hgt_arg} {SLCpar} {OFFpar} {vert_disp_rdc} 1') - - geo_los = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.los_disp' - geo_vert = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.vert_disp' - if os.path.isfile(los_disp_rdc) and not os.path.isfile(geo_los): - os.system(f'geocode_back {los_disp_rdc} {nWidth} {UTMTORDC} {geo_los} {nWidthUTMDEM} {nLineUTMDEM} 1 0') - if os.path.isfile(vert_disp_rdc) and not os.path.isfile(geo_vert): - os.system(f'geocode_back {vert_disp_rdc} {nWidth} {UTMTORDC} {geo_vert} {nWidthUTMDEM} {nLineUTMDEM} 1 0') - - # --- hyp3 专属: 缠绕相位 --- - if need_hyp3: - print(' [hyp3] 提取缠绕相位') - geo_wrapped_pha = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.pha' - if os.path.isfile(GeoDIFF) and not os.path.isfile(geo_wrapped_pha): - os.system(f'cpx_to_real {GeoDIFF} {geo_wrapped_pha} {nWidthUTMDEM} 4') - - # --- hyp3/licsbas 共需: look_vector --- - if need_hyp3 or need_licsbas: - print(' [hyp3/licsbas] look_vector') - lv_theta = workDir + '/lv_theta' - lv_phi = workDir + '/lv_phi' - if not os.path.isfile(lv_theta) or not os.path.isfile(lv_phi): - if os.path.isfile(SLCpar) and os.path.isfile(OFFpar) and os.path.isfile(UTMDEMpar0) and os.path.isfile(UTMDEM): - os.system(f'look_vector {SLCpar} {OFFpar} {UTMDEMpar0} {UTMDEM} {lv_theta} {lv_phi}') - - # --- BMP 可视化 --- - os.system('rasmph_pwr ' + GeoDIFF + ' ' + GeoMamp + ' ' + nWidthUTMDEM + ' - - - - ') - os.system('raspwr ' + GeoMamp + ' ' + nWidthUTMDEM + ' - - - - - - - - - - ') - os.system('rasdt_pwr ' + GeoUNW + ' ' + GeoMamp + ' ' + nWidthUTMDEM + ' - - - - -3.14 3.14 1 rmg.cm') - - if os.path.isfile(GeoATMCOR_UNW): - os.system('rasdt_pwr ' + GeoATMCOR_UNW + ' ' + GeoMamp + ' ' + nWidthUTMDEM + ' - - - - -3.14 3.14 1 rmg.cm') - - if os.path.isfile(UTMTORDC): - os.remove(UTMTORDC) - if os.path.isfile(UTMDEMpar): - os.remove(UTMDEMpar) - - print(' IFG 地理编码完成!') - - elif need_ifg: - print(f' WARNING: ifgrams/{Pair} 不存在, 跳过 IFG 地理编码') - - # ========================================================= - # POT 地理编码 (pot 需要) - # ========================================================= - potWorkDir = potDir + '/' + Pair - if need_pot and os.path.isdir(potWorkDir): - print('\n--- POT 地理编码 ---') - geocode_pot(projectName, Pair, templateDict) - elif need_pot: - print(f' WARNING: offsets/{Pair} 不存在, 跳过 POT 地理编码') - - print("\nGeocoding is done!") - sys.exit(0) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/geocode_gamma_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/geocode_gamma_all.py deleted file mode 100644 index 696a7d9..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/geocode_gamma_all.py +++ /dev/null @@ -1,169 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import time -import glob -import argparse - -import subprocess -from pyint import _utils as ut - -def work(data0): - cmd = data0[0] - err_file = data0[1] - p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) - stdout = p.stdout - stderr = p.stderr - - if type(stderr) == bytes: - aa=stderr.decode("utf-8") - else: - aa = stderr - - if aa: - print(aa) - str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + '\n' - with open(err_file, 'a') as f: - f.write(str0) - f.write(aa) - f.write('\n') - - return -######################################################################### - -INTRODUCTION = ''' -------------------------------------------------------------------- - Geocode products for one project using GAMMA. - 由模板参数 geocode_products 控制产品类型 (hyp3,licsbas,pot). - 一次调用自动处理所有选中的产品类型. -''' - -EXAMPLE = ''' - Usage: - geocode_gamma_all.py projectName - geocode_gamma_all.py projectName --parallel 4 - geocode_gamma_all.py projectName --parallel 4 --ifgramList-txt /test/ifgram_list.txt -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Geocode interferograms for one project using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') - parser.add_argument('--ifgarmList-txt', dest='ifgarmListTxt', help='provided ifgram_list_txt. default: using ifgram_list.txt under projectName folder.') - - inps = parser.parse_args() - return inps - - -def main(argv): - start_time = time.time() - inps = cmdLineParse() - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - projectDir = scratchDir + '/' + projectName - ifgDir = scratchDir + '/' + projectName + '/ifgrams' - potDir = scratchDir + '/' + projectName + '/offsets' - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - - # ===== 解析产品类型 ===== - products_str = templateDict.get('geocode_products', 'hyp3,licsbas') - products = set(p.strip().lower() for p in products_str.split(',')) - need_ifg = ('hyp3' in products) or ('licsbas' in products) - need_hyp3 = 'hyp3' in products - need_pot = 'pot' in products - - print(f'geocode_products: {products_str}') - - if inps.ifgarmListTxt: ifgramList_txt = inps.ifgarmListTxt - else: ifgramList_txt = scratchDir + '/' + projectName + '/ifgram_list.txt' - ifgList0 = ut.read_txt2array(ifgramList_txt) -# ifgList = ifgList0[:,0] - if len(ifgList0)==3: - ifgList=ifgList0[0] - ifgList=[ifgList] - else: - ifgList=ifgList0[:,0] - - err_txt = scratchDir + '/' + projectName + '/geocode_gamma_all.err' - if os.path.isfile(err_txt): os.remove(err_txt) - - data_para = [] - skip_count = 0 - for i in range(len(ifgList)): - pair_i = ifgList[i] - cmd0 = ['geocode_gamma.py', projectName, pair_i] - data0 = [cmd0, err_txt] - - # 根据 geocode_products 组合检查是否已完成 - ifg_done = True - pot_done = True - - if need_ifg: - ifg_dir_i = ifgDir + '/' + pair_i - if os.path.isdir(ifg_dir_i): - # 基础产品检查: unw.bmp - unw_bmp = ifg_dir_i + '/geo_' + pair_i + '_' + rlks + 'rlks.diff_filt.unw.bmp' - if not (os.path.isfile(unw_bmp) and os.path.getsize(unw_bmp) > 0): - ifg_done = False - # hyp3 额外检查: los_disp - if need_hyp3 and ifg_done: - disp_f = ifg_dir_i + '/geo_' + pair_i + '_' + rlks + 'rlks.los_disp' - lv_f = ifg_dir_i + '/lv_theta' - if not (os.path.isfile(disp_f) and os.path.getsize(disp_f) > 0): - ifg_done = False - if not (os.path.isfile(lv_f) and os.path.getsize(lv_f) > 0): - ifg_done = False - else: - ifg_done = True # 目录不存在时不需要处理, geocode_gamma.py 会打 WARNING - - if need_pot: - pot_dir_i = potDir + '/' + pair_i - if os.path.isdir(pot_dir_i): - geo_mag_bmp = pot_dir_i + '/geo_' + pair_i + '.disp_map.mag.bmp' - lv_f = pot_dir_i + '/lv_theta' - if not (os.path.isfile(geo_mag_bmp) and os.path.getsize(geo_mag_bmp) > 0): - pot_done = False - if not (os.path.isfile(lv_f) and os.path.getsize(lv_f) > 0): - pot_done = False - else: - pot_done = True # 目录不存在时不需要处理 - - if ifg_done and pot_done: - skip_count += 1 - else: - data_para.append(data0) - - total = len(ifgList) - todo = len(data_para) - print(f'Geocode: {total} pairs total, {skip_count} done, {todo} to process') - print(f'Parallel processors: {inps.parallelNumb}') - print('=' * 60) - - if todo > 0: - ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) - - print("Geocode products for project %s is done! " % projectName) - ut.print_process_time(start_time, time.time()) - - sys.exit(0) - -if __name__ == '__main__': - main(sys.argv[:]) - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/geotiff2grd.sh b/.codex_tmp/pyint_variants/no_rescue/pyint/geotiff2grd.sh deleted file mode 100644 index a67fcef..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/geotiff2grd.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -if [#agrv < 3]; then - - echo" - Usage: geotiff2grd.sh a.tif b.grd - a.tif is a geotif file include nan data - b.grd is a grd file translate by gdal without nan data in it - - Example: geotiff2grd.sh 20240114-20240126_los.tif 20240114-20240126_los.grd - - - Chen Wei 2024/2/17 -" - -exit - -fi - -export input=$1 -export out_grdfile=$2 -gdal_calc.py -A $input --outfile=tmp.tif --calc='(A==0)*(-9999)+(A!=0)*A' -gdal_translate -a_nodata -9999 -of GSBG tmp.tif $out_grdfile -rm tmp.tif *.xml diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/geotiff_utm2geo.py b/.codex_tmp/pyint_variants/no_rescue/pyint/geotiff_utm2geo.py deleted file mode 100644 index d575f6b..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/geotiff_utm2geo.py +++ /dev/null @@ -1,76 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# -import numpy as np -import os -import sys -import argparse -import os -import warnings - -from osgeo import gdal, osr - -from mintpy.utils import plot as pp, readfile, utils0 as ut - -# link: https://gdal.org/drivers/raster/index.html -GDAL_DRIVER2EXT = { - 'GTiff' : '.tif', - 'ENVI' : '', - 'GMT' : '.grd', - 'GRIB' : '.grb', - 'JPEG' : '.jpg', - 'PNG' : '.png', -} - - -############################################################################## -def cmdLineParse(): - parser = argparse.ArgumentParser(description='convert the UTM geotiff to geography geotiff.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('origin', help='Name of origin hdf5 file .') - parser.add_argument('target', help='Name of target geotiff.') - #parser.add_argument('subdataset', help='the master date and slave date of the dataset') - inps = parser.parse_args() - return inps - - -INTRODUCTION = ''' -------------------------------------------------------------------- - create_files for psokinv software running file by generated by Gamma -''' - -EXAMPLE = """Usage: - - geotiff_utm2geo.py A.h5 result.tif 20210101_20240302 -------------------------------------------------------------------- -""" - - -def main(argv): - - inps = cmdLineParse() - origin_tif = inps.origin - target_tif = inps.target - #dateset = inps.subdataset - ftype = readfile.read_attribute(origin_tif)['FILE_TYPE'] - if ftype == 'timeseries' or ftype == 'ifgramStack': - call_str = 'save_gdal.py ' + origin_tif + ' -d ' + target_tif +' --of GTiff -o tmp.tif ' - os.system(call_str) - else: - call_str = 'save_gdal.py ' + origin_tif +' --of GTiff -o tmp.tif ' - os.system(call_str) - call_str = ' gdalwarp tmp.tif ' + target_tif + '.tif' + ' -t_srs "EPSG:4326" ' - os.system(call_str) - call_str = 'gdal_translate -of GSBG ' + target_tif + '.tif ' + target_tif + '.grd' - os.system(call_str) - - - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/get_master_burst_numb.py b/.codex_tmp/pyint_variants/no_rescue/pyint/get_master_burst_numb.py deleted file mode 100644 index 708f802..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/get_master_burst_numb.py +++ /dev/null @@ -1,217 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob -import argparse - -from pyint import _utils as ut - - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Get burst number table of the master date for TOPS SLC.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName', help='project name. e.g., ChangningT55S1A') - - inps = parser.parse_args() - - return inps - - -INTRODUCTION = ''' -------------------------------------------------------------------- - - Get burst number table of the master date for TOPS SLC. -''' - -EXAMPLE = """Usage: - - get_master_burst_numb.py projectName - - get_master_burst_numb.py ChangningT55S1A - -------------------------------------------------------------------- -""" - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - projectDir = scratchDir + '/' + projectName - - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - - masterDate = templateDict['masterDate'] - date = masterDate - - slc_dir = projectDir + '/SLC' - down_dir = projectDir + '/DOWNLOAD' - opod_dir = projectDir + '/OPOD' - - if not os.path.isdir(slc_dir): - os.mkdir(slc_dir) - if not os.path.isdir(opod_dir): - os.mkdir(opod_dir) - - work_dir = slc_dir + '/' + date - if not os.path.isdir(work_dir): - os.mkdir(work_dir) - - work_dir = down_dir - os.chdir(work_dir) - - t_date = down_dir + '/t_' + date - t_burst_numb = down_dir + '/t_burst_numb_' + date - - call_str = 'ls ' + down_dir + '/S1*' + date + '*.zip > ' + t_date - os.system(call_str) - - zip_files = ut.read_txt2list(t_date) - print(zip_files) - #for i in range(len(zip_files)): - # call_str = 'S1_BURST_tab_from_zipfile - ' + zip_files[i] + ' - ' - # os.system(call_str) - AA=''.join(str(zip_files)).split("'")[1] - call_str = 'S1_BURST_tab_from_zipfile - ' + AA + ' - ' - os.system(call_str) - - call_str = 'ls ' + down_dir + '/S1*' + date + '*.burst_number_table > ' + t_burst_numb - os.system(call_str) - burst_files = ut.read_txt2list(t_burst_numb) - - - iw1_burst_node = [] - iw2_burst_node = [] - iw3_burst_node = [] - - print(burst_files) - for i in range(len(burst_files)): - print(burst_files[i]) - iw1_first = ut.read_gamma_par(str(burst_files[i]),'read','iw1_first_burst') - iw1_last = ut.read_gamma_par(str(burst_files[i]),'read','iw1_last_burst') - iw1_burst_node.append(float(iw1_first)) - iw1_burst_node.append(float(iw1_last)) - - iw2_first = ut.read_gamma_par(str(burst_files[i]),'read','iw2_first_burst') - iw2_last = ut.read_gamma_par(str(burst_files[i]),'read','iw2_last_burst') - iw2_burst_node.append(float(iw2_first)) - iw2_burst_node.append(float(iw2_last)) - - iw3_first = ut.read_gamma_par(str(burst_files[i]),'read','iw3_first_burst') - iw3_last = ut.read_gamma_par(str(burst_files[i]),'read','iw3_last_burst') - iw3_burst_node.append(float(iw3_first)) - iw3_burst_node.append(float(iw3_last)) - - - if 'iw1_first_burst' in templateDict: IW1_0 = templateDict['iw1_first_burst'] - else: IW1_0 = '1' - if 'iw1_last_burst' in templateDict: IW1_1 = templateDict['iw1_last_burst'] - else: IW1_1 = str(round(max(iw1_burst_node) - min(iw1_burst_node) + 1)) - - if 'iw2_first_burst' in templateDict: IW2_0 = templateDict['iw2_first_burst'] - else: IW2_0 = '1' - if 'iw2_last_burst' in templateDict: IW2_1 = templateDict['iw2_last_burst'] - else: IW2_1 = str(round(max(iw2_burst_node) - min(iw2_burst_node) + 1)) - - if 'iw3_first_burst' in templateDict: IW3_0 = templateDict['iw3_first_burst'] - else: IW3_0 = '1' - if 'iw3_last_burst' in templateDict: IW3_1 = templateDict['iw3_last_burst'] - else: IW3_1 = str(round(max(iw3_burst_node) - min(iw3_burst_node) + 1)) - - if int(IW1_1) > round(max(iw1_burst_node) - min(iw1_burst_node) + 1): - IW1_1 = str(round(max(iw1_burst_node) - min(iw1_burst_node) + 1)) - - if int(IW2_1) > round(max(iw2_burst_node) - min(iw2_burst_node) + 1): - IW2_1 = str(round(max(iw2_burst_node) - min(iw2_burst_node) + 1)) - - if int(IW3_1) > round(max(iw3_burst_node) - min(iw3_burst_node) + 1): - IW3_1 = str(round(max(iw3_burst_node) - min(iw3_burst_node) + 1)) - - - D_IW1_first = int(IW1_0) - 1 - DD_IW1_numb = int(IW1_1) - int(IW1_0) + 1 - - D_IW2_first = int(IW2_0) - 1 - DD_IW2_numb = int(IW2_1) - int(IW2_0) + 1 - - D_IW3_first = int(IW3_0) - 1 - DD_IW3_numb = int(IW3_1) - int(IW3_0) + 1 - - master_busrt = down_dir + '/master.burst_numb_table' - if os.path.isfile(master_busrt): - os.remove(master_busrt) - - with open(master_busrt, 'a') as f: - STR0 = 'iw1_number_of_bursts: ' + str(DD_IW1_numb) + '\n' - STR00 = 'iw1_number_of_bursts: ' + str(DD_IW1_numb) - print(STR00) - f.write(STR0) - - STR0 = 'iw1_first_burst: ' + str(min(iw1_burst_node) + D_IW1_first) + '\n' - STR00 = 'iw1_first_burst: ' + str(min(iw1_burst_node) + D_IW1_first) - print(STR00) - f.write(STR0) - - STR0 = 'iw1_last_burst: ' + str(min(iw1_burst_node) + D_IW1_first + DD_IW1_numb - 1) + '\n' - STR00 = 'iw1_last_burst: ' + str(min(iw1_burst_node) + D_IW1_first + DD_IW1_numb - 1) - print(STR00) - f.write(STR0) - - STR0 = 'iw2_number_of_bursts: ' + str(DD_IW2_numb) + '\n' - STR00 = 'iw2_number_of_bursts: ' + str(DD_IW2_numb) - print(STR00) - f.write(STR0) - - STR0 = 'iw2_first_burst: ' + str(min(iw2_burst_node) + D_IW2_first) + '\n' - STR00 = 'iw2_first_burst: ' + str(min(iw2_burst_node) + D_IW2_first) - print(STR00) - f.write(STR0) - - STR0 = 'iw2_last_burst: ' + str(min(iw2_burst_node) + D_IW2_first + DD_IW2_numb - 1) + '\n' - STR00 = 'iw2_last_burst: ' + str(min(iw2_burst_node) + D_IW2_first + DD_IW2_numb - 1) - print(STR00) - f.write(STR0) - - - STR0 = 'iw3_number_of_bursts: ' + str(DD_IW3_numb) + '\n' - STR00 = 'iw3_number_of_bursts: ' + str(DD_IW3_numb) - print(STR00) - f.write(STR0) - - STR0 = 'iw3_first_burst: ' + str(min(iw3_burst_node) + D_IW3_first) + '\n' - STR00 = 'iw3_first_burst: ' + str(min(iw3_burst_node) + D_IW3_first) - print(STR00) - f.write(STR0) - - STR0 = 'iw3_last_burst: ' + str(min(iw3_burst_node) + D_IW3_first + DD_IW3_numb - 1) + '\n' - STR00 = 'iw3_last_burst: ' + str(min(iw3_burst_node) + D_IW3_first + DD_IW3_numb - 1) - print(STR00) - f.write(STR0) - - - print("Get burst number table for master date is done! ") - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/gmt_grdview.sh b/.codex_tmp/pyint_variants/no_rescue/pyint/gmt_grdview.sh deleted file mode 100644 index 39f32e8..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/gmt_grdview.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash -# GMT modern mode bash template -# Date: 2024-10-01 -#Purpose: quickly generate basic GMT plot -#Author: -#Dependencies: GNUPlot, GMT v6 -#Written for: GNU/Linux - - -if [ $# -lt 1 ]; then -more < 0 " | bc` == 1 ] && [ ` echo "$vmax > 1 " | bc` == 1 ]; then - echo "1" - export cmin=`echo | gmt grdinfo $grdfile | grep 'v_min' | awk '{print $5/$3 } '` - export max_num=`echo "scale=6; 1/$vmax" | bc ` - export min_num=`echo "$max_num*$vmin" | bc ` - gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -G$cmin/1 -Z >1.cpt -elif [ ` echo "$flag < 0 " | bc` == 1 ] && [ ` echo "$vmin < -1 " | bc` == 1 ]; then - echo "2" - export cmax=`echo | gmt grdinfo $grdfile | grep 'v_min' | awk '{print $3/$5*-1} '` - export min_num=`echo "scale=6; 1/$vmin" | bc` - export max_num=`echo "$min_num*$vmax" | bc` - gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -G-1/$cmax -Z >1.cpt -elif [ `echo " $flag > 0 " | bc` == 1 ] && [ ` echo "$vmax < 1 " | bc` == 1 ]; then - echo "3" - if [ ` echo "$vmax < 0.1 " | bc` == 1 ]; then - export vmin1=$(echo "scale=2; $vmin*10" | bc) - export vmax1=$(echo "scale=2; $vmax*10" | bc) - gmt makecpt -C$color_file -G$vmin1/$vmax1 -T$vmin/$vmax/0.01 -Z >1.cpt - else - gmt makecpt -C$color_file -G$vmin/$vmax -T$vmin/$vmax/0.01 -Z >1.cpt - fi -elif [ ` echo "$flag < 0 " | bc` == 1 ] && [ ` echo "$vmin > -1 " | bc` == 1 ]; then - if [ ` echo "$vmin >-0.1 " | bc` == 1 ]; then - echo "4" - export vmin1=$(awk "BEGIN{print($vmin*10)}") - export vmax1=$(awk "BEGIN{print($vmax*10)}") - gmt makecpt -C$color_file -G$vmin1/$vmax1 -T$vmin/$vmax/0.01 -Z >1.cpt - else - gmt makecpt -C$color_file -G$vmin/$vmax -T$vmin/$vmax/0.01 -Z >1.cpt - fi -else - echo "5" - gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -Z >1.cpt -fi -gmt begin ${gmt_basename} png - - gmt grdimage $grdfile -C1.cpt -JM7.3c -Bxa1 -Bya1 -BWSen -R$region - gmt colorbar -Dx3.5/-1c+w6c/0.25c+jBC+h -Bxa0.2f0.1 -C1.cpt -gmt end show diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/gmt_makecpt.sh b/.codex_tmp/pyint_variants/no_rescue/pyint/gmt_makecpt.sh deleted file mode 100644 index 48cb96e..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/gmt_makecpt.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash -# GMT modern mode bash template -# Date: 2024-10-01 -#Purpose: quickly generate basic GMT plot -#Author: -#Dependencies: GNUPlot, GMT v6 -#Written for: GNU/Linux - - -if [ $# -lt 2 ]; then -more <0 " | bc ` -eq 1 ] && [ ` echo "$vmax > 1 " | bc ` -eq 1 ]; then - echo "1" - export cmin=$(awk "BEGIN{print($vmin/$vmax)}") - gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -G$cmin/1 -Z >disp.cpt - elif [ ` echo "$flag < 0 " | bc ` -eq 1 ] && [ ` echo "$vmin < -1 " | bc ` -eq 1 ]; then - echo "2" - export cmax=$(awk "BEGIN{print(-$vmax/$vmin)}") - gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -G-1/$cmax -Z >disp.cpt - elif [ `echo "$flag > 0 " | bc ` -eq 1 ] && [ ` echo "$vmax < 1 " | bc ` -eq 1 ]; then - echo "3" - if [ ` echo "$vmax < 0.1 " | bc ` -eq 1 ]; then - export vmin1=$(awk "BEGIN{print($vmin*10)}") - export vmax1=$(awk "BEGIN{print($vmax*10)}") - gmt makecpt -C$color_file -G$vmin1/$vmax1 -T$vmin/$vmax/0.01 -Z >disp.cpt - else - gmt makecpt -C$color_file -G$vmin/$vmax -T$vmin/$vmax/0.01 -Z >disp.cpt - fi - elif [ ` echo "$flag < 0 " | bc ` -eq 1 ] && [ ` echo "$vmin > -1 " | bc ` -eq 1 ]; then - if [ ` echo "$vmin >-0.1 " | bc ` -eq 1 ]; then - echo "4" - export vmin1=$(awk "BEGIN{print($vmin*10)}") - export vmax1=$(awk "BEGIN{print($vmax*10)}") - gmt makecpt -C$color_file -G$vmin1/$vmax1 -T$vmin/$vmax/0.01 -Z >disp.cpt - else - gmt makecpt -C$color_file -G$vmin/$vmax -T$vmin/$vmax/0.01 -Z >disp.cpt - fi - else - echo "5" - gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -Z >disp.cpt - fi - -elif [ "$type" == "phase" ]; then - export inc=$(awk "BEGIN{print($vmax/10)}") - gmt makecpt -Crainbow -T$vmin/$vmax/$inc -Ww -Z >phase.cpt -else - echo "error in the type" - exit -fi -export inc=$(awk "BEGIN{print(($vmax-$vmin)/4)}") -export inc_c=$(awk "BEGIN{print(($vmax-$vmin)/4)}") - -# Draw GMT map diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/gmt_plot_interf.sh b/.codex_tmp/pyint_variants/no_rescue/pyint/gmt_plot_interf.sh deleted file mode 100644 index a5befea..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/gmt_plot_interf.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env bash -# GMT modern mode bash template -# Date: 2024-10-01 -#Purpose: quickly generate basic GMT plot -#Author: -#Dependencies: GNUPlot, GMT v6 -#Written for: GNU/Linux - - -if [ $# -lt 2 ]; then -more <0 " | bc ` -eq 1 ] && [ ` echo "$vmax > 1 " | bc ` -eq 1 ]; then - echo "1" - export cmin=`echo | gmt grdinfo los.grd | grep 'v_min' | awk '{print $5/$3 } '` - export max_num=`echo "scale=6; 1/$vmax" | bc ` - export min_num=`echo "$max_num*$vmin" | bc ` - gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -G$cmin/1 -Z >1.cpt -elif [ ` echo "$flag < 0 " | bc ` -eq 1 ] && [ ` echo "$vmin < -1 " | bc ` -eq 1 ]; then - echo "2" - export cmax=`echo | gmt grdinfo los.grd | grep 'v_min' | awk '{print $3/$5*-1} '` - export min_num=`echo "scale=6; 1/$vmin" | bc` - export max_num=`echo "$min_num*$vmax" | bc` - gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -G-1/$cmax -Z >1.cpt -elif [ `echo "$flag > 0 " | bc ` -eq 1 ] && [ ` echo "$vmax < 1 " | bc ` -eq 1 ]; then - echo "3" - if [ ` echo "$vmax < 0.1 " | bc ` -eq 1 ]; then - export vmin1=$(echo "scale=2; $vmin*10" | bc) - export vmax1=$(echo "scale=2; $vmax*10" | bc) - gmt makecpt -C$color_file -G$vmin1/$vmax1 -T$vmin/$vmax/0.01 -Z >1.cpt - else - gmt makecpt -C$color_file -G$vmin/$vmax -T$vmin/$vmax/0.01 -Z >1.cpt - fi -elif [ ` echo "$flag < 0 " | bc ` -eq 1 ] && [ ` echo "$vmin > -1 " | bc ` -eq 1 ]; then - if [ ` echo "$vmin >-0.1 " | bc ` -eq 1 ]; then - echo "4" - export vmin1=$(awk "BEGIN{print($vmin*10)}") - export vmax1=$(awk "BEGIN{print($vmax*10)}") - echo $vmin1 - echo $vmax1 - gmt makecpt -C$color_file -G$vmin1/$vmax1 -T$vmin/$vmax/0.01 -Z >1.cpt - else - gmt makecpt -C$color_file -G$vmin/$vmax -T$vmin/$vmax/0.01 -Z >1.cpt - fi -else - echo "5" - gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -Z >1.cpt -fi - -#gmt makecpt -C$color_file -G$vmin/$vmax -T$vmin/$vmax/0.01 >1.cpt - gmt makecpt -Crainbow -T-3.14159265/3.14159265/0.31415926 -Z >2.cpt - - -# Draw GMT map -gmt begin insar_los_unw_${gmt_basename} png - gmt subplot begin 1x2 -Fs7.0c -A+jTR+gwhite+p1p+o0.05c/0.2c -M0.6c/0.25c -JM7.3c -Bxa1 -Bya1 -BWSen -Y10c -R$region - gmt subplot set 0 -A"Unw" - gmt grdimage unw_plot.grd -C2.cpt -Q - gmt colorbar -Dx3.5c/-1c+w6c/0.25c+jBC+h -Bxa3.14 -C2.cpt - gmt subplot set 1 -A"Los" - gmt grdimage los.grd -C1.cpt - gmt colorbar -Dx3.5/-1c+w6c/0.25c+jBC+h -Bxa0.06f0.02 -C1.cpt - gmt subplot end -gmt end show \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/hyp3_timeseries_utm2wgs84.py b/.codex_tmp/pyint_variants/no_rescue/pyint/hyp3_timeseries_utm2wgs84.py deleted file mode 100644 index a1bbcb5..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/hyp3_timeseries_utm2wgs84.py +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/env python3 -""" -将 HDF5 文件的坐标系统从 UTM 转换为 WGS84 地理坐标 -同时保持与 MintPy tsview.py 的兼容性 -""" - -import h5py -import numpy as np -import copy -import argparse -import os - -def utm_to_wgs84(easting, northing, zone, northern=True): - """ - 将 UTM 坐标转换为 WGS84 大地坐标 - """ - # UTM 参数 - a = 6378137.0 # WGS84 长半轴 - e = 0.081819190842622 # WGS84 第一偏心率 - k0 = 0.9996 # 比例因子 - - # 中央经线 - lon0 = (zone - 1) * 6 - 180 + 3 - - # 调整东伪偏移 - x = easting - 500000.0 - - # 如果是南半球，调整北伪偏移 - if not northern: - y = northing - 10000000.0 - else: - y = northing - - # M = y / k0 - mu = y / (a * (1 - e**2 / 4 - 3 * e**4 / 64 - 5 * e**6 / 256) * k0) - - e1 = (1 - np.sqrt(1 - e**2)) / (1 + np.sqrt(1 - e**2)) - - # 纬度计算 - N1 = a / np.sqrt(1 - e**2 * np.sin(mu)**2) - T1 = np.tan(mu)**2 - C1 = e1**2 * np.cos(mu)**2 - R1 = a * (1 - e**2) / (1 - e**2 * np.sin(mu)**2)**(3/2) - D = x / (N1 * k0) - - lat = mu - N1 * np.tan(mu) / R1 * (D**2 / 2 + (5 + 3 * T1 + 10 * C1 - 4 * C1**2 - 9 * e1**2) * D**4 / 24 + (61 + 90 * T1 + 298 * C1 + 45 * T1**2 - 252 * e1**2 - 3 * C1**2) * D**6 / 720) - - # 经度计算 - lon = lon0 * np.pi / 180 + D * (1 + (1 + 3 * e1**2 + 2 * e1**4) * D**2 / 6 + (2 - e1**2) * D**4 / 120) / np.cos(mu) - - # 转换为角度 - lat_deg = np.degrees(lat) - lon_deg = np.degrees(lon) - - return lon_deg, lat_deg - - -def main(): - parser = argparse.ArgumentParser( - description='将 HDF5 文件转换为 WGS84 地理坐标（与 MintPy 兼容）', - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=''' -示例: - python convert_to_wgs84.py input.h5 -o output.h5 - -说明: - 脚本会将文件完全转换为 WGS84 地理坐标系统， - 同时设置正确的属性以保持与 MintPy 的兼容性。 - ''' - ) - parser.add_argument('input', help='输入 HDF5 文件路径') - parser.add_argument('-o', '--output', help='输出 HDF5 文件路径（默认：输入文件名_WGS84.h5）') - - args = parser.parse_args() - - # 输入和输出文件 - input_file = args.input - - if args.output: - output_file = args.output - else: - # 自动生成输出文件名 - base, ext = os.path.splitext(input_file) - output_file = f"{base}_WGS84{ext}" - - print(f"正在读取文件: {input_file}") - f_in = h5py.File(input_file, 'r') - - # 读取所有数据集 - datasets = {} - for key in f_in.keys(): - datasets[key] = f_in[key][:] - print(f" 读取数据集: {key}, 形状: {datasets[key].shape}") - - # 读取所有属性 - attrs = dict(f_in.attrs) - print(f"\n共有 {len(attrs)} 个属性") - - # 解析 UTM 坐标信息 - x_first = float(attrs['X_FIRST']) - y_first = float(attrs['Y_FIRST']) - x_step = float(attrs['X_STEP']) - y_step = float(attrs['Y_STEP']) - width = int(attrs['WIDTH']) - length = int(attrs['LENGTH']) - utm_zone = attrs['UTM_ZONE'] - - # 获取参考点位置（像素坐标） - ref_x = int(attrs.get('REF_X', 0)) - ref_y = int(attrs.get('REF_Y', 0)) - - # 转换 UTM Zone - zone_num = int(''.join(filter(str.isdigit, utm_zone))) - is_northern = 'N' in utm_zone.upper() - - print(f"\n原始坐标系统:") - print(f" UTM Zone: {utm_zone} (EPSG: {attrs['EPSG']})") - print(f" 左上角 UTM: ({x_first}, {y_first})") - print(f" 步长: X={x_step}m, Y={y_step}m") - print(f" 参考点像素: ({ref_x}, {ref_y})") - - # 生成网格坐标 - x_coords = x_first + np.arange(width) * x_step - y_coords = y_first + np.arange(length) * y_step - - # 转换为 WGS84 - print(f"\n正在转换为 WGS84...") - lon_grid, lat_grid = np.meshgrid(x_coords, y_coords) - wgs84_lon, wgs84_lat = utm_to_wgs84(lon_grid, lat_grid, zone_num, is_northern) - - # 计算新的坐标参数（注意：lon 是经度，lat 是纬度） - # X 对应经度，Y 对应纬度 - lon_first = wgs84_lon[0, 0] - lat_first = wgs84_lat[0, 0] - lon_step = abs(wgs84_lon[0, 1] - wgs84_lon[0, 0]) if width > 1 else 0 - lat_step = abs(wgs84_lat[1, 0] - wgs84_lat[0, 0]) if length > 1 else 0 - - # 计算参考点的经纬度 - # REF_X 对应 X 轴（经度），REF_Y 对应 Y 轴（纬度） - ref_lon = wgs84_lon[ref_y, ref_x] # 经度 - ref_lat = wgs84_lat[ref_y, ref_x] # 纬度 - - # 更新属性 - 完全转换为 WGS84 - attrs_new = copy.deepcopy(attrs) - - # 更新坐标系统相关属性 - attrs_new['EPSG'] = 4326 # WGS84 - attrs_new['X_FIRST'] = lon_first # 经度 - attrs_new['Y_FIRST'] = lat_first # 纬度 - attrs_new['X_STEP'] = lon_step - attrs_new['Y_STEP'] = -lat_step # 保持Y方向向下为负 - attrs_new['X_UNIT'] = 'degrees' - attrs_new['Y_UNIT'] = 'degrees' - - # 更新参考坐标 - 现在是真正的经纬度 - attrs_new['REF_LON'] = ref_lon - attrs_new['REF_LAT'] = ref_lat - - # 保留原始 UTM 信息作为备份（带 _ORIG 后缀） - attrs_new['UTM_ZONE_ORIG'] = attrs_new.get('UTM_ZONE', '') - attrs_new['EPSG_ORIG'] = attrs_new.get('EPSG', '') - - # 关键：移除或清空 UTM_ZONE 属性，让 MintPy 识别为地理坐标系统 - if 'UTM_ZONE' in attrs_new: - del attrs_new['UTM_ZONE'] - - # 更新角点坐标 - attrs_new['LAT_REF1'] = lat_first - attrs_new['LAT_REF2'] = wgs84_lat[0, -1] - attrs_new['LAT_REF3'] = wgs84_lat[-1, 0] - attrs_new['LAT_REF4'] = wgs84_lat[-1, -1] - attrs_new['LON_REF1'] = lon_first - attrs_new['LON_REF2'] = wgs84_lon[0, -1] - attrs_new['LON_REF3'] = wgs84_lon[-1, 0] - attrs_new['LON_REF4'] = wgs84_lon[-1, -1] - - # 设置其他相关属性 - attrs_new['COORD_SYSTEM'] = 'GEO' # 地理坐标系统 - - print(f"\n新坐标系统 (WGS84):") - print(f" EPSG: {attrs_new['EPSG']}") - print(f" 左上角: ({attrs_new['X_FIRST']:.6f}, {attrs_new['Y_FIRST']:.6f})") - print(f" 步长: X={attrs_new['X_STEP']:.6f}°, Y={attrs_new['Y_STEP']:.6f}°") - print(f" 参考点经纬度: ({attrs_new['REF_LON']:.6f}, {attrs_new['REF_LAT']:.6f})") - print(f" 参考点像素: ({ref_x}, {ref_y})") - - # 创建新的 HDF5 文件 - print(f"\n正在写入文件: {output_file}") - f_out = h5py.File(output_file, 'w') - - # 写入所有数据集 - for key, data in datasets.items(): - f_out.create_dataset(key, data=data) - print(f" 写入数据集: {key}") - - # 写入所有属性 - for key, value in attrs_new.items(): - f_out.attrs[key] = value - - f_in.close() - f_out.close() - - print(f"\n完成！转换后的文件已保存为: {output_file}") - print("\n说明:") - print(" - 文件已完全转换为 WGS84 地理坐标系统") - print(" - 所有坐标属性已更新为经纬度") - print(" - 原始 UTM 信息已保存为 _ORIG 后缀的属性") - print(" - 应该与 MintPy tsview.py 兼容") - - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/hyp3format_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/hyp3format_gamma.py deleted file mode 100644 index dbffcb1..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/hyp3format_gamma.py +++ /dev/null @@ -1,275 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Modified: 2026, Z. Zou - HyP3 UTM GeoTIFF output ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import argparse -import time -from datetime import datetime - -from pyint import _utils as ut - - -INTRODUCTION = ''' -------------------------------------------------------------------- - 将 geocode_gamma.py 已地理编码的 GAMMA 二进制产品转换为 - ASF HyP3 兼容的 UTM GeoTIFF 格式, 用于 MintPy 时间序列分析。 - - 本脚本 **不做任何地理编码或计算**, 仅执行: - 1. data2geotiff : GAMMA 二进制 (EQA) → GeoTIFF (EQA) - 2. gdalwarp : GeoTIFF (EQA) → GeoTIFF (UTM) - 3. rasterio : 生成水体掩膜 - - 所有地理编码产品由 geocode_gamma.py 提供: - amp, corr, dem, unw_phase, wrapped_phase, - los_disp, vert_disp, lv_theta, lv_phi -''' - -EXAMPLE = ''' - Usage: - hyp3format_gamma.py projectName ifgPair - hyp3format_gamma.py shanghaiT171F128S1A 20241105-20241117 - hyp3format_gamma.py shanghaiT171F128S1A 20241105-20241117 --output-dir /path/to/output -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser( - description='Convert geocoded GAMMA binaries to HyP3 UTM GeoTIFF.', - formatter_class=argparse.RawTextHelpFormatter, - epilog=INTRODUCTION + '\n' + EXAMPLE) - parser.add_argument('projectName', help='项目名称') - parser.add_argument('ifgPair', help='干涉图对 (如 20241105-20241117)') - parser.add_argument('--output-dir', dest='output_dir', default=None, - help='输出目录. 默认: projectDir/Hyp3Products/') - inps = parser.parse_args() - return inps - - -def get_utm_epsg(lon, lat): - """根据中心经纬度自动计算 UTM 投影的 EPSG 代码""" - zone = int((lon + 180) / 6) + 1 - return 32600 + zone if lat >= 0 else 32700 + zone - - -def run_cmd(cmd, desc=''): - """执行 shell 命令并检查返回值""" - if desc: - print(f" [{desc}] {cmd}") - else: - print(f" {cmd}") - ret = os.system(cmd) - if ret != 0: - print(f" WARNING: 返回非零退出码 {ret}") - return ret - - -def main(argv): - - start_time = time.time() - inps = cmdLineParse() - projectName = inps.projectName - ifgPair = inps.ifgPair - - Mdate = ifgPair.split('-')[0] - Sdate = ifgPair.split('-')[1] - - # 验证日期并计算时间基线 - d1 = datetime.strptime(Mdate, '%Y%m%d') - d2 = datetime.strptime(Sdate, '%Y%m%d') - interval = abs((d2 - d1).days) - - # 读取模板参数 - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + '/' + projectName + '.template' - templateDict = ut.update_template(templateFile) - - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - masterDate = templateDict['masterDate'] - - # 项目目录 - projectDir = scratchDir + '/' + projectName - ifgDir = projectDir + '/ifgrams' - demDir = projectDir + '/DEM' - - workDir = ifgDir + '/' + ifgPair - if not os.path.isdir(workDir): - print(f"ERROR: 干涉图目录不存在: {workDir}") - sys.exit(1) - - # 输出目录 - outputDir = inps.output_dir or (projectDir + '/Hyp3Products') - productDir = os.path.join(outputDir, ifgPair) - os.makedirs(productDir, exist_ok=True) - - # ============================================================ - # DEM 参数 (直接引用, 不复制不删除) - # ============================================================ - DEMpar = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem.par' - - nWidthDEM = ut.read_gamma_par(DEMpar, 'read', 'width') - nLineDEM = ut.read_gamma_par(DEMpar, 'read', 'nlines') - - # 自动检测 UTM 投影 - corner_lat = float(ut.read_gamma_par(DEMpar, 'read', 'corner_lat').split()[0]) - corner_lon = float(ut.read_gamma_par(DEMpar, 'read', 'corner_lon').split()[0]) - post_lat = float(ut.read_gamma_par(DEMpar, 'read', 'post_lat').split()[0]) - post_lon = float(ut.read_gamma_par(DEMpar, 'read', 'post_lon').split()[0]) - center_lat = corner_lat + float(nLineDEM) / 2 * post_lat - center_lon = corner_lon + float(nWidthDEM) / 2 * post_lon - utm_epsg = get_utm_epsg(center_lon, center_lat) - - # HyP3 文件名前缀 (含 YYYYMMDDTHHMMSS, 兼容 MintPy 日期解析) - hyp3_prefix = f"{projectName}_{Mdate}T000000_{Sdate}T000000" - - print("\n" + "=" * 70) - print(f"HyP3 格式转换: {projectName} / {ifgPair}") - print(f"时间基线: {interval} 天, DEM: {nWidthDEM}×{nLineDEM}") - print(f"中心: {center_lat:.4f}°N, {center_lon:.4f}°E → EPSG:{utm_epsg}") - print(f"输出: {productDir}") - print("=" * 70) - - # ============================================================ - # 已地理编码 (EQA) GAMMA 二进制文件 (由 geocode_gamma.py 生成) - # ============================================================ - geo_unw = workDir + '/geo_' + ifgPair + '_' + rlks + 'rlks.diff_filt.unw' - geo_wrapped_pha = workDir + '/geo_' + ifgPair + '_' + rlks + 'rlks.diff_filt.pha' - geo_los_disp = workDir + '/geo_' + ifgPair + '_' + rlks + 'rlks.los_disp' - geo_vert_disp = workDir + '/geo_' + ifgPair + '_' + rlks + 'rlks.vert_disp' - geo_amp = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.amp' - geo_hgt = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.hgt' - lv_theta = workDir + '/lv_theta' - lv_phi = workDir + '/lv_phi' - - # 相干性可能用 Pair 或 masterDate 命名 - geo_cor = workDir + '/geo_' + ifgPair + '_' + rlks + 'rlks.diff_filt.cor' - if not os.path.exists(geo_cor): - geo_cor = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.diff_filt.cor' - - # ============================================================ - # data2geotiff (EQA binary → EQA GeoTIFF) → gdalwarp (→ UTM) - # ============================================================ - print("\n--- GAMMA 二进制 → EQA GeoTIFF → UTM GeoTIFF ---") - - product_map = [ - ('amp', geo_amp, 'bilinear'), - ('corr', geo_cor, 'bilinear'), - ('dem', geo_hgt, 'bilinear'), - ('unw_phase', geo_unw, 'bilinear'), - ('wrapped_phase', geo_wrapped_pha, 'bilinear'), - ('los_disp', geo_los_disp, 'bilinear'), - ('vert_disp', geo_vert_disp, 'bilinear'), - ('lv_theta', lv_theta, 'bilinear'), - ('lv_phi', lv_phi, 'bilinear'), - ] - - generated = {} - - for name, geo_bin, interp in product_map: - utm_tif = os.path.join(productDir, f'{hyp3_prefix}_{name}.tif') - - # 跳过已存在且非空的输出 - if os.path.exists(utm_tif) and os.path.getsize(utm_tif) > 0: - print(f" {name}: 已存在，跳过") - generated[name] = utm_tif - continue - - if not os.path.exists(geo_bin): - print(f" {name}: 源文件不存在 ({os.path.basename(geo_bin)})，跳过") - continue - - eqa_tif = workDir + '/' + name + '_hyp3_eqa.tif' - - # GAMMA 二进制 → EQA GeoTIFF - run_cmd(f'data2geotiff {DEMpar} {geo_bin} 2 {eqa_tif}', f'{name} → EQA') - - if not os.path.exists(eqa_tif): - print(f" WARNING: {name} EQA GeoTIFF 生成失败") - continue - - # EQA → UTM (gdalwarp) - run_cmd(f'gdalwarp -t_srs EPSG:{utm_epsg} -r {interp} ' - f'-co COMPRESS=LZW -overwrite {eqa_tif} {utm_tif}', - f'{name} → UTM') - - # 清理 EQA 中间文件 - if os.path.exists(eqa_tif): - os.remove(eqa_tif) - - if os.path.exists(utm_tif) and os.path.getsize(utm_tif) > 0: - generated[name] = utm_tif - - # ============================================================ - # 生成水体掩膜 (相干性阈值) - # ============================================================ - print("\n--- 水体掩膜 ---") - water_mask_tif = os.path.join(productDir, f'{hyp3_prefix}_water_mask.tif') - corr_tif = generated.get('corr') - - if corr_tif and os.path.exists(corr_tif) and not os.path.exists(water_mask_tif): - try: - import rasterio - with rasterio.open(corr_tif) as src: - corr_data = src.read(1) - mask = np.ones_like(corr_data, dtype=np.uint8) - mask[corr_data < 0.05] = 0 - mask[np.isnan(corr_data)] = 0 - - profile = src.profile.copy() - profile.update(dtype=rasterio.uint8, count=1, compress='lzw', nodata=None) - with rasterio.open(water_mask_tif, 'w', **profile) as dst: - dst.write(mask, 1) - generated['water_mask'] = water_mask_tif - print(f" 水体掩膜已生成") - except Exception as e: - print(f" WARNING: 水体掩膜生成失败: {e}") - elif os.path.exists(water_mask_tif): - generated['water_mask'] = water_mask_tif - print(f" 水体掩膜已存在，跳过") - - # ============================================================ - # 生成入射角图 (inc_map / inc_map_ell, 复制自 lv_theta) - # ============================================================ - print("\n--- 入射角图 ---") - inc_map_tif = os.path.join(productDir, f'{hyp3_prefix}_inc_map.tif') - inc_map_ell_tif = os.path.join(productDir, f'{hyp3_prefix}_inc_map_ell.tif') - lv_theta_tif = generated.get('lv_theta') - - if lv_theta_tif and os.path.exists(lv_theta_tif): - if not os.path.exists(inc_map_tif): - os.system(f'cp {lv_theta_tif} {inc_map_tif}') - generated['inc_map'] = inc_map_tif - print(f" inc_map 已生成 (= lv_theta)") - if not os.path.exists(inc_map_ell_tif): - os.system(f'cp {lv_theta_tif} {inc_map_ell_tif}') - generated['inc_map_ell'] = inc_map_ell_tif - print(f" inc_map_ell 已生成 (近似 lv_theta)") - - # ============================================================ - # 输出摘要 - # ============================================================ - print("\n" + "=" * 70) - print(f"HyP3 格式转换完成: {ifgPair}") - print(f"共 {len(generated)} 个产品:") - for name, path in sorted(generated.items()): - if os.path.exists(path): - size_mb = os.path.getsize(path) / 1024 / 1024 - print(f" {name:20s} ({size_mb:.1f} MB)") - print("=" * 70) - - ut.print_process_time(start_time, time.time()) - sys.exit(0) - - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/hyp3format_gamma_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/hyp3format_gamma_all.py deleted file mode 100644 index bce90ec..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/hyp3format_gamma_all.py +++ /dev/null @@ -1,147 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Modified: 2026, Z. Zou - HyP3 UTM batch conversion ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import time -import glob -import argparse -import subprocess - -from pyint import _utils as ut - - -def work(data0): - """并行 worker: 执行单对 hyp3format_gamma.py""" - cmd = data0[0] - err_file = data0[1] - p = subprocess.run(cmd, shell=False, stderr=subprocess.PIPE, stdout=subprocess.PIPE) - stderr = p.stderr - - if type(stderr) == bytes: - aa = stderr.decode("utf-8") - else: - aa = stderr - - if aa: - str0 = ' '.join(cmd) + '\n' - with open(err_file, 'a') as f: - f.write(str0) - f.write(aa) - f.write('\n') - - return - -######################################################################### - -INTRODUCTION = ''' -------------------------------------------------------------------- - 批量将 GAMMA 处理结果转换为 HyP3 UTM GeoTIFF 格式, - 支持并行处理。 -''' - -EXAMPLE = ''' - Usage: - hyp3format_gamma_all.py projectName - hyp3format_gamma_all.py projectName --parallel 4 - hyp3format_gamma_all.py projectName --output-dir /path/to/output - hyp3format_gamma_all.py projectName --parallel 4 --ifgramList-txt /test/ifgram_list.txt -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser( - description='批量转换 GAMMA 输出为 HyP3 UTM GeoTIFF 格式.', - formatter_class=argparse.RawTextHelpFormatter, - epilog=INTRODUCTION + '\n' + EXAMPLE) - - parser.add_argument('projectName', help='项目名称') - parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, - help='并行处理器数量') - parser.add_argument('--ifgramList-txt', dest='ifgarmListTxt', - help='干涉图列表文件. 默认: projectName/ifgram_list.txt') - parser.add_argument('--output-dir', dest='output_dir', default=None, - help='输出目录. 默认: projectDir/Hyp3Products/') - - inps = parser.parse_args() - return inps - - -def main(argv): - start_time = time.time() - inps = cmdLineParse() - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - projectDir = scratchDir + '/' + projectName - ifgDir = projectDir + '/ifgrams' - templateDict = ut.update_template(templateFile) - masterDate = templateDict['masterDate'] - - # 输出目录 - output_dir = inps.output_dir or (projectDir + '/Hyp3Products') - os.makedirs(output_dir, exist_ok=True) - - # 读取干涉图列表 - if inps.ifgarmListTxt: - ifgramList_txt = inps.ifgarmListTxt - else: - ifgramList_txt = projectDir + '/ifgram_list.txt' - - ifgList0 = ut.read_txt2array(ifgramList_txt) - - if len(ifgList0) == 3: - ifgList = ifgList0[0] - ifgList = [ifgList] - else: - ifgList = ifgList0[:,0] - - # 错误日志 - err_txt = projectDir + '/hyp3format_gamma_all.err' - if os.path.isfile(err_txt): - os.remove(err_txt) - - # 构建并行命令 - data_para = [] - skip_count = 0 - for i in range(len(ifgList)): - Pair = ifgList[i] - - # 检查输出是否已存在 (用 corr.tif 作为完成标记) - pair_dir = output_dir + '/' + Pair - # 匹配 HyP3 命名: {prefix}_corr.tif - geo_file0 = pair_dir + '/' + projectName + '_' + Pair.replace('-', 'T000000_') + 'T000000_corr.tif' - - if os.path.isfile(geo_file0) and os.path.getsize(geo_file0) > 0: - skip_count += 1 - continue - - cmd0 = ['hyp3format_gamma.py', projectName, Pair] - if inps.output_dir: - cmd0.extend(['--output-dir', inps.output_dir]) - - data0 = [cmd0, err_txt] - data_para.append(data0) - - print(f"HyP3 格式转换: 共 {len(ifgList)} 对, 跳过 {skip_count} 对, 待处理 {len(data_para)} 对") - print(f"并行数: {inps.parallelNumb}, 输出目录: {output_dir}") - - # 并行执行 - ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) - - print("HyP3 格式转换完成: project %s" % projectName) - ut.print_process_time(start_time, time.time()) - - sys.exit(0) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/ionosphere_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/ionosphere_gamma.py deleted file mode 100644 index 61ffab6..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/ionosphere_gamma.py +++ /dev/null @@ -1,411 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import getopt -import time -import glob -import argparse - -from pyint import _utils as ut - - -INTRODUCTION = ''' -------------------------------------------------------------------- - Calculate the ionospheric phases in an interferogram based on RSI - -''' - -EXAMPLE = ''' - Usage: - ionosphere_gamma.py projectName Mdate Sdate - ionosphere_gamma.py PacayaT163TsxHhA 20150102 20150601 -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Calculate the ionospheric phases in an interferogram based on RSI.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('Mdate',help='Master date.') - parser.add_argument('Sdate',help='Slave date.') - - inps = parser.parse_args() - return inps - - -def main(argv): - - start_time = time.time() - inps = cmdLineParse() - Mdate = inps.Mdate - Sdate = inps.Sdate - - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - masterDate = templateDict['masterDate'] - - projectDir = scratchDir + '/' + projectName - demDir = scratchDir + '/' + projectName + '/DEM' - - slcDir = scratchDir + '/' + projectName + '/SLC' - rslcDir = scratchDir + '/' + projectName + '/RSLC' - ifgDir = projectDir + '/ifgrams' - if not os.path.isdir(ifgDir): os.mkdir(ifgDir) - - Pair = Mdate + '-' + Sdate - workDir = ifgDir + '/' + Pair - if not os.path.isdir(workDir): os.mkdir(workDir) - - ####################################################################### - Mamp0 = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' - MampPar0 = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' - Samp0 = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' - SampPar0 = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' - - Mrslc = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc' - MrslcPar = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc.par' - Srslc = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc' - SrslcPar = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc.par' - - HGT0 = demDir + '/' + masterDate + '_' + rlks + 'rlks.rdc.dem' - - MasterPar0 = rslcDir + '/' + masterDate + '/' + masterDate + '.rslc.par' - - ################# copy file for parallel processing ########################## - Mamp = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp' - MampPar = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp.par' - Samp = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp' - SampPar = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp.par' - - #Mrslc = workDir + '/' + Mdate + '.rslc' - #MrslcPar = workDir + '/' + Mdate + '.rslc.par' - #Srslc = workDir + '/' + Sdate + '.rslc' - #SrslcPar = workDir + '/' + Sdate + '.rslc.par' - - HGT = workDir + '/' + masterDate + '_' + rlks + 'rlks.rdc.dem' - MasterPar = workDir + '/' + masterDate + '.rslc.par' - - if not os.path.isfile(Mamp):ut.copy_file(Mamp0,Mamp) - if not os.path.isfile(MampPar):ut.copy_file(MampPar0,MampPar) - if not os.path.isfile(Samp):ut.copy_file(Samp0,Samp) - if not os.path.isfile(SampPar):ut.copy_file(SampPar0,SampPar) - - #if not os.path.isfile(Mrslc): ut.copy_file(Mrslc0,Mrslc) - #if not os.path.isfile(MrslcPar): ut.copy_file(MrslcPar0,MrslcPar) - #if not os.path.isfile(Srslc):ut.copy_file(Srslc0,Srslc) - #if not os.path.isfile(SrslcPar):ut.copy_file(SrslcPar0,SrslcPar) - - if not os.path.isfile(HGT):ut.copy_file(HGT0,HGT) - if not os.path.isfile(MasterPar):ut.copy_file(MasterPar0,MasterPar) - - - nWidth = ut.read_gamma_par(MampPar, 'read', 'range_samples') - nLine = ut.read_gamma_par(MampPar, 'read', 'azimuth_lines') - - ###################### Output files ################################# - ionoDir = workDir + '/ionosphere' - if not os.path.isdir(ionoDir): - os.mkdir(ionoDir) - - workDir = ionoDir - Mrslc_low = ionoDir + '/' + Mdate + '.low.rslc' - MrslcPar_low = ionoDir + '/' + Mdate + '.low.rslc.par' - Mrslc_high = ionoDir + '/' + Mdate + '.high.rslc' - MrslcPar_high = ionoDir + '/' + Mdate + '.high.rslc.par' - - Srslc_low = ionoDir + '/' + Sdate + '.low.rslc' - SrslcPar_low = ionoDir + '/' + Sdate + '.low.rslc.par' - Srslc_high = ionoDir + '/' + Sdate + '.high.rslc' - SrslcPar_high = ionoDir + '/' + Sdate + '.high.rslc.par' -######################################################### - TEST0 = MrslcPar_high - k0 = 0 - if os.path.isfile(TEST0): - if os.path.getsize(TEST0) > 0: - k0 =1 - - TEST1 = SrslcPar_high - k1 = 0 - if os.path.isfile(TEST1): - if os.path.getsize(TEST1) > 0: - k1 =1 - - if k0 ==0: - call_str = 'bpf_ssi ' + Mrslc + ' ' + MrslcPar + ' ' + Mrslc_low + ' ' + MrslcPar_low + ' ' + Mrslc_high + ' ' + MrslcPar_high - os.system(call_str) - - if k1==0: - call_str = 'bpf_ssi ' + Srslc + ' ' + SrslcPar + ' ' + Srslc_low + ' ' + SrslcPar_low + ' ' + Srslc_high + ' ' + SrslcPar_high - os.system(call_str) - - ################## interferometry ##################### - - off_low = workDir + '/' + Pair +'_' + rlks + 'rlks.low.off' - call_str = 'create_offset '+ MrslcPar_low + ' ' + SrslcPar_low + ' ' + off_low + ' 1 ' + rlks + ' ' + azlks + ' 0' - os.system(call_str) - - sim_low_unw = workDir + '/' + Pair + '.low.sim_unw' - call_str = 'phase_sim_orb ' + MrslcPar_low + ' ' + SrslcPar_low + ' ' + off_low + ' ' + HGT + ' ' + sim_low_unw + ' ' + MasterPar + ' - - 1 1' - os.system(call_str) - - diff_low = workDir + '/' + Pair + '_' + rlks + 'rlks.low.diff' - call_str = 'SLC_diff_intf ' + Mrslc_low + ' ' + Srslc_low + ' ' + MrslcPar_low + ' ' + SrslcPar_low + ' ' + off_low + ' ' + sim_low_unw + ' ' + diff_low + ' ' + rlks + ' ' + azlks + ' ' + ' 1 0 0.25' - os.system(call_str) - - - off_high = workDir + '/' + Pair +'_' + rlks + 'rlks.high.off' - call_str = 'create_offset '+ MrslcPar_high + ' ' + SrslcPar_high + ' ' + off_high + ' 1 ' + rlks + ' ' + azlks + ' 0' - os.system(call_str) - - sim_high_unw = workDir + '/' + Pair + '.high.sim_unw' - call_str = 'phase_sim_orb ' + MrslcPar_high + ' ' + SrslcPar_high + ' ' + off_high + ' ' + HGT + ' ' + sim_high_unw + ' ' + MasterPar + ' - - 1 1' - os.system(call_str) - - diff_high = workDir + '/' + Pair + '_' + rlks + 'rlks.high.diff' - call_str = 'SLC_diff_intf ' + Mrslc_high + ' ' + Srslc_high + ' ' + MrslcPar_high + ' ' + SrslcPar_high + ' ' + off_high + ' ' + sim_high_unw + ' ' + diff_high + ' ' + rlks + ' ' + azlks + ' ' + ' 1 0 0.25' - os.system(call_str) - - ###################################################### - hl_diff = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff' - hl_diff_cc = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff.cc' - hl_diff_mask = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff.cc_mask.bmp' - - hl_diff1 = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff1' - hl_diff2 = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff2' - hl_diff3 = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff3' - hl_diff4 = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff4' - - hl_diff4_phase = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff4_phase' - - hl_diff_jpg = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff.jpg' - - diff_low_phase = workDir + '/' + Pair + '_' + rlks + 'rlks.low.diff_phase' - call_str = 'cpx_to_real ' + diff_low + ' ' + diff_low_phase + ' ' + nWidth + ' 4' - os.system(call_str) - - call_str = 'subtract_phase ' + diff_high + ' ' + diff_low_phase + ' ' + hl_diff + ' ' + nWidth + ' 1' - os.system(call_str) - - call_str = 'rasmph_pwr ' + Mamp + ' ' + hl_diff + ' ' + nWidth - #call_str = 'vismph_pwr.py ' + hl_diff + ' ' + Mamp + ' ' + nWidth + ' -f 1.0 ' + ' -z 1000 ' + ' -p ' + hl_diff_jpg - os.system(call_str) - - #### filter - - call_str = 'adf ' + hl_diff + ' ' + hl_diff1 + ' ' + hl_diff_cc + ' ' + nWidth + ' 0.2 512 7 128' - os.system(call_str) - - call_str = 'adf ' + hl_diff1 + ' ' + hl_diff2 + ' ' + hl_diff_cc + ' ' + nWidth + ' 0.3 256 7 64' - os.system(call_str) - - call_str = 'adf ' + hl_diff2 + ' ' + hl_diff3 + ' ' + hl_diff_cc + ' ' + nWidth + ' 0.3 128 7 64' - os.system(call_str) - - call_str = 'adf ' + hl_diff3 + ' ' + hl_diff4 + ' ' + hl_diff_cc + ' ' + nWidth + ' 0.3 128 7 16' - os.system(call_str) - - # unwrawp - call_str = 'cpx_to_real ' + hl_diff4 + ' ' + hl_diff4_phase + ' ' + nWidth + ' 4' - os.system(call_str) - - # shift to zero - # mask very low coherence - - call_str = 'cc_ad ' + hl_diff + ' ' + Mamp + ' ' + Samp + ' - - ' + hl_diff_cc + ' ' + nWidth + ' 3 9' - os.system(call_str) - - call_str = 'rascc_mask ' + hl_diff_cc + ' ' + Mamp + ' ' + nWidth + ' 1 1 0 1 1 0.15 0.0 0.1 0.9 1.0 0.35 1 ' + hl_diff_mask - os.system(call_str) - - nWidth_half = str(float(nWidth)/2) - image_report = workDir + '/' + Pair + '.image_stat.report' - - #call_str = 'image_stat ' + hl_diff4_phase + ' ' + nWidth + ' ' + nWidth_half + ' 200 200 ' + image_report - call_str = 'image_stat ' + hl_diff4_phase + ' ' + nWidth + ' 0 0 ' + ' 200 200 ' + image_report - os.system(call_str) - - mean_phase = ut.read_gamma_par(image_report, 'read', 'mean') - std_phase = ut.read_gamma_par(image_report, 'read', 'stdev') - - - hl_diff_phase_shift = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff.ph_shift' - hl_diff_shifted = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff4.shifted' - hl_diff_shifted_phase = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff4.shifted.phase' - call_str = 'lin_comb 1 ' + hl_diff4_phase + ' ' + mean_phase + ' 0.0 ' + hl_diff_phase_shift + ' ' + nWidth + ' 1 ' + nLine - os.system(call_str) - - call_str = 'subtract_phase ' + hl_diff4_phase + ' ' + hl_diff_phase_shift + ' ' + hl_diff_shifted + ' ' + nWidth + ' 1' - os.system(call_str) - - call_str = 'cpx_to_real ' + hl_diff_shifted + ' ' + hl_diff_shifted_phase + ' ' + nWidth + ' 4' - os.system(call_str) - - call_str = 'lin_comb 1 ' + hl_diff_shifted_phase + ' ' + mean_phase + ' 1.0 ' + hl_diff4_phase + ' ' + nWidth + ' 1 ' + nLine - os.system(call_str) - - ############# determine trend considering cc_mask using unwrapping with multi_cpx before unwrapping ### - - diff_par = workDir + '/' + Pair + '.ddiff.par' - call_str = 'create_diff_par ' + MampPar + ' - ' + diff_par + ' 1 0' - os.system(call_str) - - ddiff_phase_trend = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff4_phase.trend' - call_str = 'quad_fit ' + hl_diff4_phase + ' ' + diff_par + ' 5 5 ' + hl_diff_mask + ' - 3 ' + ddiff_phase_trend - os.system(call_str) - - #call_str = 'vistd_pwr.py ' + ddiff_phase_trend + ' ' + Mamp + ' ' + nWidth + ' -c 12.6 -z 1000 -f 1.0 -m rmg -p ' + ddiff_phase_trend + '.jpg' - #os.system(call_str) - - hl_diff_detrend = hl_diff+ '.detrend' - call_str = 'subtract_phase ' + hl_diff + ' ' + ddiff_phase_trend + ' ' + hl_diff_detrend + ' ' + nWidth + ' 1.' - os.system(call_str) - - #call_str = 'vismph_pwr.py ' + hl_diff_detrend + ' ' + Mamp + ' ' + nWidth + ' -f 1.0 -z 1000 -p ' + hl_diff_detrend+'.jpg' - #os.system(call_str) - - - call_str = 'adf ' + hl_diff_detrend + ' ' + hl_diff1 + ' ' + hl_diff+'.smcc' + ' ' + nWidth + ' 0.2 512 7 128' - os.system(call_str) - - call_str = 'adf ' + hl_diff1 + ' ' + hl_diff2 + ' ' + hl_diff+'.smcc' + ' ' + nWidth + ' 0.3 256 7 64' - os.system(call_str) - - call_str = 'adf ' + hl_diff2 + ' ' + hl_diff3 + ' ' + hl_diff+'.smcc' + ' ' + nWidth + ' 0.3 128 7 64' - os.system(call_str) - - hl_diff4_detrend = hl_diff4+ '.detrend' - hl_diff_cc = hl_diff+'.sm4.cc' - call_str = 'adf ' + hl_diff3 + ' ' + hl_diff4_detrend + ' ' + hl_diff_cc + ' ' + nWidth + ' 0.3 128 7 16' - os.system(call_str) - - off1 = workDir + '/' + Pair + '.off1' - call_str = 'create_offset ' + MampPar + ' ' + MampPar + ' ' + off1 + ' 1 1 1 0' - os.system(call_str) - - hl_diff4_mask = hl_diff+'.sm4.cc_mask.bmp' - call_str = 'rascc_mask ' + hl_diff_cc + ' ' + Mamp + ' ' + nWidth + ' 1 1 0 1 1 0.95 0.0 0.1 0.1 0.9 1.0 0.35 1 ' + hl_diff4_mask - os.system(call_str) - - call_str = 'mask_class ' + hl_diff_mask + ' ' + hl_diff4_detrend + ' ' + hl_diff4_detrend + '.masked.tmp ' + ' 1 1 1 1 0 0.0 0.0' - os.system(call_str) - - call_str = 'mask_class ' + hl_diff4_mask + ' ' + hl_diff4_detrend + '.masked.tmp ' + ' ' + hl_diff4_detrend + '.masked' + ' 1 1 1 1 0 0.0 0.0' - os.system(call_str) - - - ddiff5 = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff5' - off5 = workDir + '/' + Pair + '.off5' - call_str = 'multi_cpx ' + hl_diff4_detrend + '.masked' + ' ' + off1 + ddiff5 + ' ' + off5 + ' 5 5' - os.system(call_str) - - call_str = 'multi_real ' + Mamp + ' ' + off1 + ' ' + Mamp + '.5' + off5 + ' 5 5' - os.system(call_str) - - call_str = 'multi_real ' + hl_diff_cc + ' ' + off1 + ' ' + hl_diff_cc + '.5' + off5 + ' 5 5' - os.system(call_str) - - nWidth5 = ut.read_gamma_par(off5,'read','interferogram_width') - - ddiff5_phase_tmp = ddiff5 + '.phase.tmp' - ddiff5_phase_interp = ddiff5 + '.phase.interp' - call_str = 'cpx_to_real ' + ddiff5 + ' ' + ddiff5_phase_tmp + ' ' + nWidth5 + ' 4' - os.system(call_str) - - call_str = 'fill_gaps ' + ddiff5_phase_tmp + ' ' + nWidth5 + ' ' + ddiff5_phase_interp + ' 0 4 - 1 100 4 400' - os.system(call_str) - - - #### remove outliers - ddiff5_fspf = ddiff5_phase_interp + '.fspf' - call_str = 'fspf ' + ddiff5_phase_interp + ' ' + ddiff5_fspf + ' ' + nWidth5 + ' 2 64 3' - os.system(call_str) - - ddiff5_phase_interp_outliers = ddiff5_phase_interp + '.outliers' - call_str = 'lin_comb 2 ' + ddiff5_phase_interp + ' ' + ddiff5_fspf + ' 100 1.0 -1.0 ' + ddiff5_phase_interp_outliers + ' ' + nWidth5 - os.system(call_str) - - hl_diff_cc5 = hl_diff_cc + '.5' - hl_diff5_mask = ddiff5 + '.mask.bmp' - call_str = 'single_class_mapping 2 ' + ddiff5_phase_interp_outliers + ' 99.95 100.05 ' + hl_diff_cc5 + ' 0.15 1.0 ' + hl_diff5_mask + ' ' + nWidth5 - os.system(call_str) - - ddiff5_phase_tmp1 = ddiff5_phase_tmp + '1' - call_str = 'mask_class ' + hl_diff5_mask + ' ' + ddiff5_phase_tmp + ' ' + ddiff5_phase_tmp1 + ' 0 1 1 1 0 0.0 0.0' - os.system(call_str) - - call_str = 'fill_gaps ' + ddiff5_phase_tmp1 + ' ' + ddiff5_phase_interp + ' 0 4 - 1 100 4 400' - os.system(call_str) - - ddiff_detrend_interp_phase = hl_diff + '.detrend' + '.interp.phase' - call_str = 'multi_real ' + ddiff5_phase_interp + ' ' + off5 + ' ' + ddiff_detrend_interp_phase + ' ' + off1 + ' -5 -5' - os.system(call_str) - - call_str = 'fspf ' + ddiff_detrend_interp_phase + ' ' + ddiff_detrend_interp_phase + '.fspf' + ' ' + nWidth + ' 2 8 3' - os.system(call_str) - - #call_str = ' visdt_pwr.py ' + ddiff_detrend_interp_phase + '.fspf' + ' ' + Mamp + ' ' + nWidth + ' -c 1.6 -z 1000 -f 1.0 -m rmg -p ' + ddiff_detrend_interp_phase + '.fspf.jpg' - #os.system(call_str) - - - ###### add the solutions - - ddiff_phase_fspf = hl_diff + ' .phase.fspf' - call_str = 'lin_comb 2 ' + ddiff_phase_trend + ' ' + ddiff_detrend_interp_phase + '.fspf' + ' 0.0 1.0 1.0 ' + ddiff_phase_fspf + ' ' + nWidth + ' 1 ' + nLine - os.system(call_str) - - #call_str ='visdt_pwr.py ' + ddiff_phase_fspf + ' ' + Mamp + ' ' + nWidth + ' -c 1.6 -z 1000 -f 1.0 -m rmg -p ' + ddiff_phase_fspf + '.jpg' - #os.system(call_str) - - - ###################### determin scaling factor using bpf_ssi - - bpf_ssi_out = workDir + '/bpf_ssi.out' - call_str = 'bpf_ssi ' + Mrslc + ' ' + MrslcPar + ' - - - - 0.6666 > ' + bpf_ssi_out - os.system(call_str) - - a0 = ut.read_gamma_par(bpf_ssi_out,'read','a') - b0 = ut.read_gamma_par(bpf_ssi_out,'read','b') - x0 = ut.read_gamma_par(bpf_ssi_out,'read','x') - y0 = ut.read_gamma_par(bpf_ssi_out,'read','y') - z0 = ut.read_gamma_par(bpf_ssi_out,'read','z') - zz0 = str(float(z0)*2) - - ddiff_phase_fspf_scaled = ddiff_phase_fspf + '.scaled' - call_str = 'lin_comb 1 ' + ddiff_phase_fspf + ' 0.0 ' + zz0 + ' ' + ddiff_phase_fspf_scaled + ' ' + nWidth - os.system(call_str) - - call_str = 'rasrmg ' + ddiff_phase_fspf_scaled + ' ' + Mamp + ' ' + nWidth - #call_str = 'vismph_pwr.py ' + hl_diff + ' ' + Mamp + ' ' + nWidth + ' -f 1.0 ' + ' -z 1000 ' + ' -p ' + hl_diff_jpg - os.system(call_str) - - #call_str ='visdt_pwr.py ' + ddiff_phase_fspf_scaled + ' ' + Mamp + ' ' + nWidth + ' -c 1.6 -z 1000 -f 1.0 -m rmg -p ' + ddiff_phase_fspf_scaled + '.jpg' - #os.system(call_str) - - # 2_phi_iono = phi0 + zz * ddiff_phase_fspf - # 2_phi_non-dispersive = phi0 - zz * ddiff_phase_fspf - - #call_str = ' subtract_phase ' + - - - print("Estimating the scaled ionospheric phases is done!") - ut.print_process_time(start_time, time.time()) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/load_data_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/load_data_gamma.py deleted file mode 100644 index d3a567d..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/load_data_gamma.py +++ /dev/null @@ -1,643 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v1.0 ### -### Copy Right (c): 2017, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Email : ymcmrs@gmail.com ### -### Univ. : Central South University & University of Miami ### -################################################################# - -import os -import sys -import glob -import time -import argparse -from mintpy.utils import readfile, writefile, ptime, utils as ut -import h5py -import numpy as np - -def print_progress(iteration, total, prefix='calculating:', suffix='complete', decimals=1, barLength=50, elapsed_time=None): - """Print iterations progress - Greenstick from Stack Overflow - Call in a loop to create terminal progress bar - @params: - iteration - Required : current iteration (Int) - total - Required : total iterations (Int) - prefix - Optional : prefix string (Str) - suffix - Optional : suffix string (Str) - decimals - Optional : number of decimals in percent complete (Int) - barLength - Optional : character length of bar (Int) - elapsed_time- Optional : elapsed time in seconds (Int/Float) - - Reference: http://stackoverflow.com/questions/3173320/text-progress-bar-in-the-console - """ - filledLength = int(round(barLength * iteration / float(total))) - percents = round(100.00 * (iteration / float(total)), decimals) - bar = '#' * filledLength + '-' * (barLength - filledLength) - if elapsed_time: - sys.stdout.write('%s [%s] %s%s %s %s secs\r' % (prefix, bar, percents, '%', suffix, int(elapsed_time))) - else: - sys.stdout.write('%s [%s] %s%s %s\r' % (prefix, bar, percents, '%', suffix)) - sys.stdout.flush() - if iteration == total: - print("\n") - - ''' - Sample Useage: - for i in range(len(dateList)): - print_progress(i+1,len(dateList)) - ''' - return - - -def check_variable_name(path): - s=path.split("/")[0] - if len(s)>0 and s[0]=="$": - p0=os.getenv(s[1:]) - path=path.replace(path.split("/")[0],p0) - return path - -def read_template(File, delimiter='='): - '''Reads the template file into a python dictionary structure. - Input : string, full path to the template file - Output: dictionary, pysar template content - Example: - tmpl = read_template(KyushuT424F610_640AlosA.template) - tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') - ''' - template_dict = {} - for line in open(File): - line = line.strip() - c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter - if len(c) < 2 or line.startswith('%') or line.startswith('#'): - next #ignore commented lines or those without variables - else: - atrName = c[0] - atrValue = str.replace(c[1],'\n','').split("#")[0].strip() - atrValue = check_variable_name(atrValue) - template_dict[atrName] = atrValue - return template_dict - -def read_data(inFile, dtype, nWidth, nLength): - data = np.fromfile(inFile, dtype, int(nLength)*int(nWidth)).reshape(int(nLength),int(nWidth)) - return data - -def is_number(s): - try: - int(s) - return True - except ValueError: - return False - -def add_zero(s): - if len(s)==1: - s="000"+s - elif len(s)==2: - s="00"+s - elif len(s)==3: - s="0"+s - return s - -def read_list(ListName): - List=glob.glob(ListName) - return List - -def read_list_except(ListName,ExString): - List=glob.glob(ListName) - List_New = [] - for S in List: - if ExString not in os.path.basename(S): - List_New.append(S) - return List_New - - -def load_gamma2multi_group_h5(fileType, fileName, fileList, RSCList, datatype): - - RSC =RSCList[0] - rsc_dic = read_roipac_rsc(RSC) - nWidth = rsc_dic['WIDTH'] - nLine = rsc_dic['FILE_LENGTH'] - - H5FILE = fileName + '.h5' - fileNum = len(fileList) - print('Start to load ' + fileName + ' >>> %s %s files will be loaded for further process' % ( str(fileNum), fileType)) - f = h5py.File(H5FILE,'w') - gg=f.create_group(fileType) - - for i in range(len(fileList)): - S=fileList[i] - RSC =RSCList[i] - rsc_dic = read_roipac_rsc(RSC) - print_progress(i+1, fileNum, prefix='loading', suffix=os.path.basename(S)) - data = read_data(S,datatype,nWidth,nLine) - File = os.path.basename(S) - group = gg.create_group(File) - dset = group.create_dataset(File, data = data, compression='gzip') - for key,value in rsc_dic.items(): - group.attrs[key] = value - group.attrs['X_MIN'] = '0' - group.attrs['X_MAX'] = str(int(nWidth)-1) - group.attrs['Y_MIN'] = '0' - group.attrs['Y_MAX'] = str(int(nLine)-1) - group.attrs['DATE12']=File.split('_')[2] - group.attrs['DATATYPE'] = datatype - group.attrs['PROCCESSOR'] = 'gamma' - - f.close() - - -def Get_Datelist(projectName): - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - rslcDir = scratchDir + '/' + projectName + "/RSLC" - slcDir = scratchDir + '/' + projectName + "/SLC" - - ListSLC = os.listdir(slcDir) - Datelist = [] - - for kk in range(len(ListSLC)): - if ( is_number(ListSLC[kk]) and len(ListSLC[kk])==6 ): # if SAR date number is 8, 6 should change to 8. - DD=ListSLC[kk] - Year=int(DD[0:2]) - Month = int(DD[2:4]) - Day = int(DD[4:6]) - if ( 0 < Year < 20 and 0 < Month < 13 and 0 < Day < 32 ): - Datelist.append(ListSLC[kk]) - Datelist = list(map(int,Datelist)) - Datelist.sort() - Datelist = list(map(str,Datelist)) - return Datelist - -def Get_Inflist(projectName): - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - IFGRAM = scratchDir + '/' + projectName + "/PROCESS/IFG*" - IFGList = glob.glob(IFGRAM) - - ListSLC = os.listdir(slcDir) - Datelist = [] - - -def Get_PairName(projectName): - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - IFGRAM = scratchDir + '/' + projectName + "/PROCESS/IFG*" - IFGList = glob.glob(IFGRAM) - - PAIR_Name = [] - for kk in range(len(IFGList)): - SS = IFGList[kk] - IFGName = os.path.basename(SS) - PAIR=IFGName.split('_')[2] - PAIR_Name.append(PAIR) - - return PAIR_Name - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close() - -def UseGamma2(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[2].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close() - -def Remove_Inf_List(RemoveNumberStr): - AD=RemoveNumberStr - Addlist = [] - if len(AD)>0: - AD= AD.split('[')[1].split(']')[0] - if ',' in AD: - LL=AD.split(',') - for kk in range(len(LL)): - XX=LL[kk] - if is_number(XX): - Addlist.append(XX) - - else: - D1=XX.split(':')[0] - D2=XX.split(':')[1] - for jj in range(int(D1),int(D2)+1): - Addlist.append(str(jj)) - else: - LL = AD - if is_number(LL): - Addlist.append(LL) - else: - D1=LL.split(':')[0] - D2=LL.split(':')[1] - for jj in range(int(D1),int(D2)+1): - Addlist.append(str(jj)) - for ii in range(len(Addlist)): - Addlist[ii]=int(Addlist[ii])-1 - Addlist = list(map(str,Addlist)) - return Addlist - - -def read_roipac_rsc(File): - '''Read ROI_PAC .rsc file into a python dictionary structure.''' - rsc_dict = dict(np.loadtxt(File, dtype=str, usecols=(0,1))) - return rsc_dict - - -######################################################################### - -INTRODUCTION = ''' -############################################################################# - Copy Right(c): 2017, Yunmeng Cao @PyINT v1.0 - - Loading data for PyMIS processing. - -''' - -EXAMPLE= ''' - Usage: - load_data_pymis.py projectName - load_data_pymis.py projectName --demRdc - load_data_pymis.py projectName --wrapIfgram - - Examples: - load_data_pymis.py PacayaT163TsxHhA - load_data_pymis.py PacayaT163TsxHhA --demRdc - load_data_pymis.py PacayaT163TsxHhA --coherence - -############################################################################## -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Loading data for PyMIS processing.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='project name of PyMIS.') - parser.add_argument('--coherence',action="store_true", default=False, help='Loading coherence file.') - parser.add_argument('--wrapIfgram',action="store_true", default=False, help='Loading wrapped interferograms.') - parser.add_argument('--unwrapIfgram',action="store_true", default=False, help='Loading unwrapped interferograms.') - parser.add_argument('--demRdc',action="store_true", default=False, help='Loading radar coordinates DEM.') - parser.add_argument('--demGeo',action="store_true", default=False, help='Loading GEO coordinates DEM.') - parser.add_argument('--geo2rdc',action="store_true", default=False, help='Loading geocoding lookup table.') - - inps = parser.parse_args() - - - - return inps - -################################################################################ - -def main(argv): - - - total = time.time() - inps = cmdLineParse() - projectName = inps.projectName - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateContents=read_template(templateFile) - - processDir = scratchDir + '/' + projectName + "/ifgrams" - #processDir = scratchDir + '/' + projectName - slcDir = scratchDir + '/' + projectName + "/SLC" - rslcDir = scratchDir + '/' + projectName + "/RSLC" - #simDir = scratchDir + '/' + projectName + "/PROCESS/DEM" - simDir = scratchDir + '/' + projectName + "/DEM" - - - masterDate = templateContents['masterDate'] - rlks = templateContents['range_looks'] - azlks = templateContents['azimuth_looks'] - - workDir = scratchDir + '/' + projectName + '/TSSAR' - if not os.path.isdir(workDir): - call_str='mkdir ' + workDir - os.system(call_str) - - if not os.path.isdir(workDir): - call_str='mkdir ' + workDir - os.system(call_str) - - print('Project : ' + projectName) - print('Loading data for timeseries processing >>>') - print('Change process Dir to :' + workDir) - os.chdir(workDir) - - -################# DEM Define ############ - - - - - UTM2RDC = simDir+'/' + masterDate + '_' + rlks + 'rlks.UTM_TO_RDC' - RDCDEM = simDir+'/' + masterDate + '_' + rlks + 'rlks.rdc.dem' - UTMDEM = simDir+'/' + masterDate + '_' + rlks + 'rlks.utm.dem' - UTMPAR = simDir+'/' + masterDate + '_' + rlks + 'rlks.utm.dem.par' - - - -# if not os.path.isfile(UTM2RDC): -# print 'Start to generate subset look-up table and subset-DEM >>>' -# call_str = 'CreateRdcDem_Sub_Gammapy.py ' + igramDir -# print call_str -# os.system(call_str) -# else: -# print 'Subet lookup table %s is existed!' % UTM2RDC - - Dem_Format = UseGamma(UTMPAR, 'read', 'data_format:') - - if Dem_Format=='REAL*4': - dtype_utmdem='f4' - else: - dtype_utmdem='i2' - - nWidthUTM = UseGamma(UTMPAR, 'read', 'width:') - nLineUTM = UseGamma(UTMPAR, 'read', 'nlines:') - - Corner_LAT = UseGamma(UTMPAR, 'read', 'corner_lat:') - Corner_LON = UseGamma(UTMPAR, 'read', 'corner_lon:') - - Corner_LAT =Corner_LAT.split(' ')[0] - Corner_LON =Corner_LON.split(' ')[0] - - post_Lat = UseGamma(UTMPAR, 'read', 'post_lat:') - post_Lon = UseGamma(UTMPAR, 'read', 'post_lon:') - - post_Lat =post_Lat.split(' ')[0] - post_Lon =post_Lon.split(' ')[0] - -############### timeseries file define ################## - - if 'IFG_list' in templateContents: IFG_list = templateContents['IFG_list'] - else: IFG_list = processDir + '/*/diff_filt*rlks.int' - if 'UNW_list' in templateContents: UNW_list = templateContents['UNW_list'] - else: UNW_list = processDir + '/*/diff_filt*rlks.unw' - if 'COR_list' in templateContents: COR_list = templateContents['COR_list'] - else: COR_list = processDir + '/*/filt*rlks.cor' - if 'MASK_list' in templateContents: MASK_list = templateContents['MASK_list'] - # else: MASK_list = processDir + '/*/diff_filt*rlks.cor_mask' - else: MASK_list = processDir + '/*/*.cor_mask.bmp' - if 'RSC_list' in templateContents: RSC_list = templateContents['RSC_list'] - else: RSC_list = processDir + '/*/diff_filt_*unw.rsc' - - MM=MASK_list[0] - if MM.split('.')[len(MM.split('.'))-1]=='bmp': - dtype_mask ='u1' - else: - dtype_mask ='f4' - - if 'Byte_order' in templateContents: byteoder = templateContents['Byte_order'] - else: byteorder = 'big' - - if byteorder =='big': - sign = '>' - else: - sign ='<' - - dtype_inf = sign + 'c8' - dtype_unw = sign + 'f4' - dtype_utmdem = sign + dtype_utmdem - dtype_rdcdem = sign + 'f4' - dtype_lt = sign + 'c8' - dtype_mask = sign + dtype_mask - dtype_cor = sign + 'f4' - -#################### start to load data ######################## - # InfList = read_list(IFG_list) - InfList = ut.get_file_list(IFG_list, abspath=True) - NInf = len(InfList) - - print('RSC_Name is : ' + RSC_list) - # RSCList = read_list(RSC_list) - RSCList = ut.get_file_list(RSC_list, abspath=True) - RSC = RSCList[0] - rsc_dic = read_roipac_rsc(RSC) - nWidth = rsc_dic['WIDTH'] - nLine = rsc_dic['FILE_LENGTH'] - UNWList = ut.get_file_list(UNW_list, abspath=True) - CORList = ut.get_file_list(COR_list, abspath=True) - # UNWList = read_list(UNW_list) - # CORList = read_list(COR_list) - Flag_K = 0 - if inps.coherence: - Flag_K = 1 - print('') - print('COR_Name is : ' + COR_list) - load_gamma2multi_group_h5('coherence','coherence', CORList, RSCList, dtype_cor) - elif inps.wrapIfgram: - Flag_K = 1 - print('') - print('IFG_Name is : ' + IFG_list) - load_gamma2multi_group_h5('wrapped','wrapIfgram', InfList, RSCList, dtype_inf) - elif inps.unwrapIfgram: - Flag_K = 1 - print('') - print('UNW_Name is : ' + UNW_list) - load_gamma2multi_group_h5('interferograms','unwrapIfgram', UNWList, RSCList, dtype_unw) - elif inps.demGeo: - Flag_K = 1 - print('') - print('Start to write GEO-DEM into h5 file >>> ' + UTMDEM) - data = read_data(UTMDEM, dtype_utmdem, nWidthUTM, nLineUTM) - H5FILE = 'demGeo.h5' - f =h5py.File(H5FILE,'w') - group=f.create_group('dem') - dset = group.create_dataset('dem', data=data, compression='gzip') - group.attrs['WIDTH'] = nWidthUTM - group.attrs['FILE_LENGTH'] = nLineUTM - group.attrs['X_FIRST'] = Corner_LON # X: latitude Y: longitude - group.attrs['Y_FIRST'] = Corner_LAT - group.attrs['X_STEP'] = post_Lon - group.attrs['Y_STEP'] = post_Lat - group.attrs['X_UNIT'] = 'degrees' - group.attrs['Y_UNIT'] = 'degrees' - group.attrs['X_MIN'] = '0' - group.attrs['X_MAX'] = str(int(nWidthUTM)-1) - group.attrs['Y_MIN'] = '0' - group.attrs['Y_MAX'] = str(int(nLineUTM)-1) - group.attrs['DATATYPE'] = dtype_utmdem - group.attrs['COORD'] = 'geo' - group.attrs['PROCCESSOR'] = 'gamma' - f.close() - elif inps.demRdc: - Flag_K = 1 - print('') - print('Start to write RDC-DEM into h5 file >>> ' + RDCDEM) - data = read_data(RDCDEM, dtype_rdcdem, nWidth, nLine) - H5FILE = 'demRdc.h5' - f =h5py.File(H5FILE,'w') - group=f.create_group('dem') - dset = group.create_dataset('dem', data=data, compression='gzip') - group.attrs['PROCCESSOR'] = 'gamma' - group.attrs['WIDTH'] = nWidth - group.attrs['FILE_LENGTH'] = nLine - group.attrs['X_MIN'] = '0' - group.attrs['X_MAX'] = str(int(nWidth)-1) - group.attrs['Y_MIN'] = '0' - group.attrs['Y_MAX'] = str(int(nLine)-1) - group.attrs['DATATYPE'] = dtype_rdcdem - group.attrs['COORD'] ='radar' - f.close() - elif inps.geo2rdc: - Flag_K = 1 - print('') - print('Start to write GEO2RDC into h5 file >>> ' + UTM2RDC) - data = read_data(UTM2RDC,dtype_lt,nWidthUTM,nLineUTM) # real: range imaginary: azimuth - H5FILE = 'geo2rdc.h5' - f =h5py.File(H5FILE,'w') - group=f.create_group('lt') - dset = group.create_dataset('lt', data=data, compression='gzip') - group.attrs['WIDTH'] = nWidthUTM - group.attrs['FILE_LENGTH'] = nLineUTM - group.attrs['X_FIRST'] = Corner_LON # X: latitude Y: longitude - group.attrs['Y_FIRST'] = Corner_LAT - group.attrs['X_STEP'] = post_Lon - group.attrs['Y_STEP'] = post_Lat - group.attrs['X_UNIT'] = 'degrees' - group.attrs['Y_UNIT'] = 'degrees' - group.attrs['X_MIN'] = '0' - group.attrs['X_MAX'] = str(int(nWidthUTM)-1) - group.attrs['Y_MIN'] = '0' - group.attrs['Y_MAX'] = str(int(nLineUTM)-1) - group.attrs['DATATYPE'] = dtype_utmdem - group.attrs['COORD'] = 'geo' - group.attrs['PROCCESSOR'] = 'gamma' - f.close() - - - - if not os.path.isfile('wrapIfgram.h5') and Flag_K==0: - print('') - print('IFG_Name is : ' + IFG_list) - load_gamma2multi_group_h5('wrapped','wrapIfgram', InfList, RSCList, dtype_inf) - else: - print('') - print('wrapIfgram.h5 has existed, loading wrapped interferograms is skipped.') - - - - if not os.path.isfile('unwrapIfgram.h5') and Flag_K==0: - print('') - print('UNW_Name is : ' + UNW_list) - load_gamma2multi_group_h5('interferograms','unwrapIfgram', UNWList, RSCList, dtype_unw) - else: - print('') - print('unwrapIfgram.h5 has existed, loading unwrapped interferograms is skipped.') - - - if not os.path.isfile('coherence.h5') and Flag_K==0: - print('') - print('COR_Name is : ' + COR_list) - load_gamma2multi_group_h5('coherence','coherence', CORList, RSCList, dtype_cor) - else: - print('') - print('coherence.h5 has existed, loading coherence is skipped.') - - -######### load DEM and lookup table ########## - - if not os.path.isfile('demGeo.h5') and Flag_K==0: - print('') - print('Start to write GEO-DEM into h5 file >>> ' + UTMDEM) - data = read_data(UTMDEM, dtype_utmdem, nWidthUTM, nLineUTM) - H5FILE = 'demGeo.h5' - f =h5py.File(H5FILE,'w') - group=f.create_group('dem') - dset = group.create_dataset('dem', data=data, compression='gzip') - group.attrs['WIDTH'] = nWidthUTM - group.attrs['FILE_LENGTH'] = nLineUTM - group.attrs['X_FIRST'] = Corner_LON # X: latitude Y: longitude - group.attrs['Y_FIRST'] = Corner_LAT - group.attrs['X_STEP'] = post_Lon - group.attrs['Y_STEP'] = post_Lat - group.attrs['X_UNIT'] = 'degrees' - group.attrs['Y_UNIT'] = 'degrees' - group.attrs['X_MIN'] = '0' - group.attrs['X_MAX'] = str(int(nWidthUTM)-1) - group.attrs['Y_MIN'] = '0' - group.attrs['Y_MAX'] = str(int(nLineUTM)-1) - group.attrs['DATATYPE'] = dtype_utmdem - group.attrs['COORD'] = 'geo' - group.attrs['PROCCESSOR'] = 'gamma' - f.close() - else: - print('') - print('demGeo.h5 has existed, loading GEO-DEM is skipped.') - - - if not os.path.isfile('demRdc.h5') and Flag_K==0: - print('') - print('Start to write RDC-DEM into h5 file >>> ' + RDCDEM) - data = read_data(RDCDEM, dtype_rdcdem, nWidth, nLine) - H5FILE = 'demRdc.h5' - f =h5py.File(H5FILE,'w') - group=f.create_group('dem') - dset = group.create_dataset('dem', data=data, compression='gzip') - group.attrs['PROCCESSOR'] = 'gamma' - group.attrs['WIDTH'] = nWidth - group.attrs['FILE_LENGTH'] = nLine - group.attrs['X_MIN'] = '0' - group.attrs['X_MAX'] = str(int(nWidth)-1) - group.attrs['Y_MIN'] = '0' - group.attrs['Y_MAX'] = str(int(nLine)-1) - group.attrs['DATATYPE'] = dtype_rdcdem - group.attrs['COORD'] ='radar' - f.close() - else: - print('') - print('demRdc.h5 has existed, loading RDC-DEM is skipped.') - - - if not os.path.isfile('geo2rdc.h5') and Flag_K==0: - print('') - print('Start to write GEO2RDC into h5 file >>> ' + UTM2RDC) - data = read_data(UTM2RDC,dtype_lt,nWidthUTM,nLineUTM) # real: range imaginary: azimuth - H5FILE = 'geo2rdc.h5' - f =h5py.File(H5FILE,'w') - group=f.create_group('lt') - dset = group.create_dataset('lt', data=data, compression='gzip') - group.attrs['WIDTH'] = nWidthUTM - group.attrs['FILE_LENGTH'] = nLineUTM - group.attrs['X_FIRST'] = Corner_LON # X: latitude Y: longitude - group.attrs['Y_FIRST'] = Corner_LAT - group.attrs['X_STEP'] = post_Lon - group.attrs['Y_STEP'] = post_Lat - group.attrs['X_UNIT'] = 'degrees' - group.attrs['Y_UNIT'] = 'degrees' - group.attrs['X_MIN'] = '0' - group.attrs['X_MAX'] = str(int(nWidthUTM)-1) - group.attrs['Y_MIN'] = '0' - group.attrs['Y_MAX'] = str(int(nLineUTM)-1) - group.attrs['DATATYPE'] = dtype_utmdem - group.attrs['COORD'] = 'geo' - group.attrs['PROCCESSOR'] = 'gamma' - f.close() - else: - print('') - print('geo2rdc.h5 has existed, loading lookup table is skipped.') - - print('') - print('Done.\nLoading data spend ' + str(time.time()-total) +' secs') - sys.exit(1) - - -############################################################################## -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/load_mintpy.py b/.codex_tmp/pyint_variants/no_rescue/pyint/load_mintpy.py deleted file mode 100644 index 5be813a..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/load_mintpy.py +++ /dev/null @@ -1,159 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Purpose: loading PyINT products for mintPy ### -### Copy Right (c): 2019, Yunmeng Cao ### -### Author : Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import subprocess -import time -import argparse -import glob - -from pyint import _utils as ut - -def write_template(str0,templateFile): - with open(templateFile, 'a') as f: - f.write(str0 + '\n') - return - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Loading pyint products for mintPy time-series analysis.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName', help='name of the project.') - #parser.add_argument('--data-type', dest='dataType', type=str, default='big_endian',choices={'big_endian', 'little_endian'},help='data type, big endian or little endian. [default: big_endian]') - inps = parser.parse_args() - - return inps - - -INTRODUCTION = ''' ------------------------------------------------------------------ - Loading pyint products for mintPy time-series analysis. - -''' - -EXAMPLE = """Usage: - - load_mintpy.py projectName ------------------------------------------------------------------ -""" - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - scratchDir0 = os.getenv('SCRATCHDIR0') - projectDir = scratchDir + '/' + projectName - projectDir0 = scratchDir0 + '/' + projectName - demDir = scratchDir + '/' + projectName + '/DEM' - rslcDir = scratchDir + '/' + projectName + '/RSLC' - - ifgDir = projectDir + '/ifgrams' - #unwFile = projectDir + '/ifgrams/*/2*rlks.diff_filt.unw' # considering geocode unw file - unwFile = projectDir + '/ifgrams/*/diff_filt_*.unw' # considering geocode unw file - corFile = projectDir + '/ifgrams/*/diff_filt_*rlks.cor' - #corFile = projectDir + '/ifgrams/*/filt_*.cor' - print(unwFile) - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - masterDate = templateDict['masterDate'] - MampPar = rslcDir + '/' + masterDate + '/' + masterDate + '_' + rlks + 'rlks.amp.par' - date_list = ut.get_project_slcList(projectName) - date_list = sorted(date_list) - slc_par_list = [rslcDir + '/' + date0 + '/' + date0 + '.rlsc.par' for date0 in date_list ] - ifgdir_list = glob.glob(ifgDir + '/*') - for k0 in ifgdir_list: - pair0 = os.path.basename(k0) - #print(pair0) - m0 = pair0.split('-')[0] - s0 = pair0.split('-')[1] - - workDir = k0 - mamppar0 = rslcDir + '/' + m0 + '/' + m0 + '_' + rlks + 'rlks.amp.par' - samppar0 = rslcDir + '/' + s0 + '/' + s0 + '_' + rlks + 'rlks.amp.par' - mrslcpar0 = rslcDir + '/' + m0 + '/' + m0 + '.rslc.par' - srslcpar0 = rslcDir + '/' + s0 + '/' + s0 + '.rslc.par' - - mamppar = workDir + '/' + m0 + '_' + rlks + 'rlks.amp.par' - samppar = workDir + '/' + s0 + '_' + rlks + 'rlks.amp.par' - mrslcpar = workDir + '/' + m0 + '.rslc.par' - srslcpar = workDir + '/' + s0 + '.rslc.par' - - ut.copy_file(mamppar0, mamppar) - ut.copy_file(samppar0, samppar) - ut.copy_file(mrslcpar0,mrslcpar) - ut.copy_file(srslcpar0,srslcpar) - - #unw_list = glob.glob(ifgDir + '/*/*rlks.diff_filt.unw') - #cor_list = glob.glob(ifgDir + '/*/*rlks.diff_filt.cor') - - dem_geo = glob.glob(demDir + '/*rlks.utm.dem')[0] - geo_par = glob.glob(demDir + '/*rlks.utm.dem.par')[0] - - dem_rdc = glob.glob(demDir + '/*_' + rlks + 'rlks.rdc.dem')[0] - rdc_par = glob.glob(demDir + '/*_' + rlks + 'rlks.diff_par')[0] # diff_par - lt = glob.glob(demDir + '/*_' + rlks + 'rlks.UTM_TO_RDC')[0] - - strPro = 'mintpy.load.processor = gamma' - strUNW = "mintpy.load.unwFile = " + unwFile - strCOR = 'mintpy.load.corFile = ' + corFile - strCon = 'mintpy.load.connCompFile = auto' - strInt = 'mintpy.load.intFile = auto' - strIon = 'mintpy.load.ionoFile = auto' - - strDem = 'mintpy.load.demFile = ' + dem_rdc - strDemGeo = 'mintpy.load.demFile = ' + dem_geo - strLtY = 'mintpy.load.lookupYFile = ' + lt - strLtX = 'mintpy.load.lookupXFile = ' + lt - strInc = 'mintpy.load.incAngleFile = auto' - strAza = 'mintpy.load.azAngleFile = auto' - strSha = 'mintpy.load.shadowMaskFile = auto' - strWat = 'mintpy.load.waterMaskFile = auto' - strBrp = 'mintpy.load.bperpFile = auto' - - templateFile0 = projectDir + '/mintpy.template' - - if 'mintpy.load.processor' not in templateDict: write_template(strPro,templateFile0) - if 'mintpy.load.unwFile' not in templateDict: write_template(strUNW,templateFile0) - if 'mintpy.load.corFile' not in templateDict: write_template(strCOR,templateFile0) - if 'mintpy.load.demFile' not in templateDict: write_template(strDem,templateFile0) - if 'mintpy.load.lookupYFile' not in templateDict: write_template(strLtY,templateFile0) - if 'mintpy.load.lookupXFile' not in templateDict: write_template(strLtX,templateFile0) - - if 'mintpy.load.connCompFile' not in templateDict: write_template(strCon,templateFile0) - if 'mintpy.load.intFile' not in templateDict: write_template(strInt,templateFile0) - if 'mintpy.load.ionoFile' not in templateDict: write_template(strIon,templateFile0) - - if 'mintpy.load.incAngleFile' not in templateDict: write_template(strInc,templateFile0) - if 'mintpy.load.azAngleFile' not in templateDict: write_template(strAza,templateFile0) - if 'mintpy.load.shadowMaskFile' not in templateDict: write_template(strSha,templateFile0) - if 'mintpy.load.waterMaskFile' not in templateDict: write_template(strWat,templateFile0) - if 'mintpy.load.bperpFile' not in templateDict: write_template(strBrp,templateFile0) - - os.chdir(projectDir0) - call_str = 'load_data.py -t ' + templateFile0 - os.system(call_str) - - os.chdir(projectDir0) - write_template(strDemGeo,templateFile0) - call_str = 'load_data.py -t ' + templateFile0 - os.system(call_str) - - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/make_local_dem.py b/.codex_tmp/pyint_variants/no_rescue/pyint/make_local_dem.py deleted file mode 100644 index 0772460..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/make_local_dem.py +++ /dev/null @@ -1,565 +0,0 @@ -#! /usr/bin/env python -################################################################# -### 从本地 FABDEM 瓦片库生成 GAMMA 格式 DEM ### -### 输入: 经纬度范围 + 本地 FABDEM ZIP 目录 ### -### 输出: .dem + .dem.par (与 makedem.py 输出完全一致) ### -### Author: Cascade AI + zouyuandong ### -### Date : 2026-03-15 ### -################################################################# - -import os -import sys -import re -import math -import glob -import zipfile -import tempfile -import shutil -import argparse -import subprocess -import numpy as np -from pathlib import Path - - -# ========================= GAMMA 参数文件写入 ========================= - -def write_dempar_file(filepath, corner_lon, corner_lat, post_lon, post_lat, - width, nlines, data_format='REAL*4'): - """写入 GAMMA DEM 参数文件 (.dem.par)，与 makedem.py 格式完全一致""" - with open(filepath, 'w') as f: - f.write("Gamma DIFF&GEO DEM/MAP parameter file\n") - f.write("title:\tIMPORTED DEM FROM FABDEM V1-2\n") - f.write("DEM_projection: EQA\n") - f.write("data_format: %s\n" % data_format) - f.write("DEM_hgt_offset: 0.00000\n") - f.write("DEM_scale: 1.00000\n") - f.write("width: %s\n" % str(int(width))) - f.write("nlines: %s\n" % str(int(nlines))) - f.write("corner_lat: %s decimal degrees\n" % str(float(corner_lat))) - f.write("corner_lon: %s decimal degrees\n" % str(float(corner_lon))) - f.write("post_lat: %s decimal degrees\n" % str(float(post_lat))) - f.write("post_lon: %s decimal degrees\n" % str(float(post_lon))) - f.write("\n") - f.write("ellipsoid_name: WGS 84\n") - f.write("ellipsoid_ra: 6378137.000 m\n") - f.write("ellipsoid_reciprocal_flattening: 298.2572236\n") - f.write("\n") - f.write("datum_name: WGS 1984\n") - f.write("datum_shift_dx: 0.000 m\n") - f.write("datum_shift_dy: 0.000 m\n") - f.write("datum_shift_dz: 0.000 m\n") - f.write("datum_scale_m: 0.00000e+00\n") - f.write("datum_rotation_alpha: 0.00000e+00 arc-sec\n") - f.write("datum_rotation_beta: 0.00000e+00 arc-sec\n") - f.write("datum_rotation_gamma: 0.00000e+00 arc-sec\n") - f.write("datum_country_list Global Definition, WGS84, World\n") - f.write("\n") - - -# ========================= FABDEM 瓦片索引逻辑 ========================= - -def parse_fabdem_zip_name(zip_name): - """解析 FABDEM ZIP 文件名，提取覆盖的经纬度范围 - - 示例: N30E120-N40E130_FABDEM_V1-2.zip → (lat_min=30, lon_min=120, lat_max=40, lon_max=130) - S10W080-S00W070_FABDEM_V1-2.zip → (lat_min=-10, lon_min=-80, lat_max=0, lon_max=-70) - """ - basename = Path(zip_name).stem # N30E120-N40E130_FABDEM_V1-2 - match = re.match( - r'([NS])(\d+)([EW])(\d+)-([NS])(\d+)([EW])(\d+)_FABDEM', - basename - ) - if not match: - return None - - lat1 = int(match.group(2)) * (1 if match.group(1) == 'N' else -1) - lon1 = int(match.group(4)) * (1 if match.group(3) == 'E' else -1) - lat2 = int(match.group(6)) * (1 if match.group(5) == 'N' else -1) - lon2 = int(match.group(8)) * (1 if match.group(7) == 'E' else -1) - - return { - 'lat_min': min(lat1, lat2), - 'lat_max': max(lat1, lat2), - 'lon_min': min(lon1, lon2), - 'lon_max': max(lon1, lon2), - } - - -def tile_name_for_coord(lat, lon): - """根据经纬度生成 FABDEM 1°×1° 瓦片文件名 - - 参数 lat, lon 为瓦片左下角整数坐标 - 示例: (31, 121) → 'N31E121_FABDEM_V1-2.tif' - (-1, -70) → 'S01W070_FABDEM_V1-2.tif' - """ - lat_prefix = 'N' if lat >= 0 else 'S' - lon_prefix = 'E' if lon >= 0 else 'W' - return f"{lat_prefix}{abs(lat):02d}{lon_prefix}{abs(lon):03d}_FABDEM_V1-2.tif" - - -def find_needed_tiles(west, south, east, north): - """计算覆盖目标区域所需的全部 1°×1° 瓦片坐标列表 - - 返回: [(lat, lon), ...] 瓦片左下角坐标 - """ - # 向下取整到最近的整数度 - lat_start = math.floor(south) - lat_end = math.ceil(north) # 不含 - lon_start = math.floor(west) - lon_end = math.ceil(east) - - tiles = [] - for lat in range(lat_start, lat_end): - for lon in range(lon_start, lon_end): - tiles.append((lat, lon)) - return tiles - - -def find_zip_for_tile(lat, lon, fabdem_dir): - """查找包含指定 1°×1° 瓦片的 ZIP 文件 - - FABDEM ZIP 按 10°×10° 分块，文件名编码了覆盖范围 - """ - fabdem_path = Path(fabdem_dir) - zip_files = sorted(fabdem_path.glob('*_FABDEM_V1-2.zip')) - - for zf in zip_files: - bounds = parse_fabdem_zip_name(zf.name) - if bounds is None: - continue - if (bounds['lat_min'] <= lat < bounds['lat_max'] and - bounds['lon_min'] <= lon < bounds['lon_max']): - return zf - return None - - -def extract_tiles_from_zips(tile_coords, fabdem_dir, extract_dir): - """从 FABDEM ZIP 文件中提取所需的 1°×1° GeoTIFF 瓦片 - - 参数: - tile_coords: [(lat, lon), ...] 需要提取的瓦片坐标 - fabdem_dir: FABDEM ZIP 文件所在目录 - extract_dir: 解压目标目录 - - 返回: - 提取成功的 GeoTIFF 文件路径列表 - """ - extract_path = Path(extract_dir) - extract_path.mkdir(parents=True, exist_ok=True) - - # 按 ZIP 文件分组，避免重复打开同一个 ZIP - zip_to_tiles = {} - missing_tiles = [] - - for lat, lon in tile_coords: - zip_file = find_zip_for_tile(lat, lon, fabdem_dir) - if zip_file is None: - tile_name = tile_name_for_coord(lat, lon) - print(f" ⚠ 未找到包含 {tile_name} 的 ZIP 文件 (lat={lat}, lon={lon})") - missing_tiles.append((lat, lon)) - continue - - zip_key = str(zip_file) - if zip_key not in zip_to_tiles: - zip_to_tiles[zip_key] = [] - zip_to_tiles[zip_key].append((lat, lon)) - - # 逐个 ZIP 文件提取 - extracted_files = [] - for zip_path, coords in zip_to_tiles.items(): - zip_name = Path(zip_path).name - print(f" 📦 从 {zip_name} 提取 {len(coords)} 个瓦片...") - - try: - with zipfile.ZipFile(zip_path, 'r') as zf: - zip_contents = zf.namelist() - - for lat, lon in coords: - tile_name = tile_name_for_coord(lat, lon) - - if tile_name in zip_contents: - zf.extract(tile_name, extract_dir) - out_file = extract_path / tile_name - if out_file.exists() and out_file.stat().st_size > 0: - extracted_files.append(str(out_file)) - print(f" ✓ {tile_name}") - else: - print(f" ✗ {tile_name} 提取后为空") - else: - # 海洋区域可能没有对应瓦片（正常） - print(f" - {tile_name} 不在 ZIP 中（可能是海洋区域）") - except Exception as e: - print(f" ✗ 打开 {zip_name} 失败: {e}") - - if missing_tiles: - print(f"\n ⚠ 共 {len(missing_tiles)} 个瓦片未找到对应的 ZIP 文件") - - return extracted_files - - -# ========================= 瓦片拼接 + GAMMA 格式转换（一步完成） ========================= - -def _parse_gdalinfo(gdalinfo_text): - """从 gdalinfo 输出解析元数据""" - width = nlines = None - corner_lon = corner_lat = post_lon = post_lat = None - for line in gdalinfo_text.splitlines(): - if 'Size is' in line: - parts = line.split('Size is')[1].strip().split(',') - width = int(parts[0].strip()) - nlines = int(parts[1].strip()) - elif 'Origin =' in line: - parts = line.split('(')[1].split(')')[0].split(',') - corner_lon = float(parts[0].strip()) - corner_lat = float(parts[1].strip()) - elif 'Pixel Size =' in line: - parts = line.split('(')[1].split(')')[0].split(',') - post_lon = float(parts[0].strip()) - post_lat = float(parts[1].strip()) - return width, nlines, corner_lon, corner_lat, post_lon, post_lat - - -def tiles_to_gamma_dem(tif_files, output_name, west, south, east, north, byteorder='big'): - """从多个 GeoTIFF 瓦片生成 GAMMA 格式 DEM (.dem + .dem.par) - - 流程: srtm2dem 逐个转换 → mosaic 合并（全部使用 GAMMA 原生工具） - - 参数: - tif_files: GeoTIFF 瓦片文件路径列表 - output_name: 输出文件名前缀（不含扩展名） - west, south, east, north: 裁剪范围（度） - byteorder: 字节序（未使用，srtm2dem 自动处理大端） - 返回: - (dem_file, dem_par_file) 或 (None, None) - """ - if not tif_files: - print(" ✗ 没有可用的瓦片文件") - return None, None - - Path(output_name).parent.mkdir(parents=True, exist_ok=True) - dem_file = output_name + '.dem' - dem_par_file = output_name + '.dem.par' - - # 临时目录放 /tmp/（本地盘），srtm2dem 单瓦片很快 - import tempfile as _tf - _tmp_dir = _tf.mkdtemp(prefix='gamma_dem_conv_') - - # ---- 步骤1: srtm2dem 逐个将 GeoTIFF 转为 GAMMA 格式 ---- - n = len(tif_files) - print(f" [1/2] srtm2dem 转换 {n} 个瓦片...") - gamma_tiles = [] # [(dem_path, dem_par_path), ...] - - for i, tif in enumerate(tif_files): - tile_name = Path(tif).stem - tile_dem = os.path.join(_tmp_dir, f'{tile_name}.dem') - tile_par = os.path.join(_tmp_dir, f'{tile_name}.dem.par') - - # gflg=3: 不做大地水准面校正, NODATA 替换为 0.0 - cmd = f'srtm2dem {tif} {tile_dem} {tile_par} 3' - ret = os.system(cmd + ' > /dev/null 2>&1') - - if ret == 0 and os.path.exists(tile_dem) and os.path.getsize(tile_dem) > 0: - gamma_tiles.append((tile_dem, tile_par)) - print(f" ✓ [{i+1}/{n}] {tile_name}") - else: - print(f" ✗ [{i+1}/{n}] {tile_name} srtm2dem 失败") - - if not gamma_tiles: - print(" ✗ 所有瓦片转换失败") - shutil.rmtree(_tmp_dir, ignore_errors=True) - return None, None - - print(f" 成功转换 {len(gamma_tiles)}/{n} 个瓦片") - - # ---- 步骤2: mosaic 合并所有 GAMMA DEM 瓦片 ---- - if len(gamma_tiles) == 1: - # 只有一个瓦片，直接复制 - shutil.copy2(gamma_tiles[0][0], dem_file) - shutil.copy2(gamma_tiles[0][1], dem_par_file) - print(f" [2/2] 单瓦片，直接输出") - else: - print(f" [2/2] mosaic 合并 {len(gamma_tiles)} 个 GAMMA DEM...") - # 构建 mosaic 命令: mosaic nfiles dem1 par1 dem2 par2 ... dem_out par_out mode format - cmd_parts = ['mosaic', str(len(gamma_tiles))] - for tile_dem, tile_par in gamma_tiles: - cmd_parts.append(tile_dem) - cmd_parts.append(tile_par) - cmd_parts.extend([dem_file, dem_par_file, '1', '0']) - # mode=1: 重叠区取平均, format=0: FLOAT - - cmd_str = ' '.join(cmd_parts) - ret = os.system(cmd_str) - - if ret != 0 or not os.path.exists(dem_file) or os.path.getsize(dem_file) == 0: - print(f" ✗ mosaic 失败 (exit={ret})") - shutil.rmtree(_tmp_dir, ignore_errors=True) - return None, None - - # 清理临时目录 - shutil.rmtree(_tmp_dir, ignore_errors=True) - - size_mb = os.path.getsize(dem_file) / (1024 * 1024) - print(f" ✓ {dem_file} ({size_mb:.1f} MB)") - print(f" ✓ {dem_par_file}") - - return dem_file, dem_par_file - - -# ========================= 主流程 ========================= - -INTRODUCTION = ''' -================================================================================ - make_local_dem.py — 从本地 FABDEM 瓦片库生成 GAMMA 格式 DEM - - 功能: - 1. 根据经纬度范围自动查找所需的 FABDEM 1°×1° 瓦片 - 2. 从 10°×10° ZIP 包中提取瓦片 - 3. 使用 GDAL 拼接并裁剪到目标范围 - 4. 转换为 GAMMA 格式 (.dem + .dem.par) - - 输出与 makedem.py 完全一致，可直接用于 generate_rdc_dem.py -================================================================================ -''' - -EXAMPLE = """ -用法: - # 方式1: 指定经纬度范围 - make_local_dem.py -r 120/123/30/32 -f /mnt/ZYD/全球FABDEM -o output_dem - - # 方式2: 从 SLC 参数文件自动确定范围 - make_local_dem.py -s master.slc.par -f /mnt/ZYD/全球FABDEM -o output_dem - - # 方式3: 从 PyINT 模板文件读取（集成到工作流） - make_local_dem.py --template shanghaiT171F128S1A -f /mnt/ZYD/全球FABDEM -""" - - -def cmdLineParse(): - parser = argparse.ArgumentParser( - description='从本地 FABDEM 瓦片库生成 GAMMA 格式 DEM', - formatter_class=argparse.RawTextHelpFormatter, - epilog=INTRODUCTION + '\n' + EXAMPLE - ) - - parser.add_argument('-r', dest='region', - help='研究区域范围: west/east/south/north (如: 120/123/30/32)') - parser.add_argument('-s', dest='slc_par', - help='SLC 参数文件路径（自动从中提取研究区域范围）') - parser.add_argument('-f', '--fabdem-dir', dest='fabdem_dir', required=True, - help='本地 FABDEM ZIP 文件目录 (如: /mnt/ZYD/全球FABDEM)') - parser.add_argument('-o', '--output', dest='output_name', default=None, - help='输出文件名前缀（不含扩展名）[默认: 工作目录/out]') - parser.add_argument('--dir', dest='work_dir', default=None, - help='工作目录 [默认: 当前目录]') - parser.add_argument('--template', dest='template_name', default=None, - help='PyINT 项目名（从模板文件读取 DEM 路径和 SLC 位置）') - parser.add_argument('--byteorder', dest='byteorder', choices=['big', 'little'], - default='big', - help='输出 DEM 字节序 [默认: big (GAMMA 标准)]') - parser.add_argument('--margin', dest='margin', type=float, default=1.0, - help='在 SLC 覆盖范围外扩展的余量（度）[默认: 1.0]') - - return parser.parse_args() - - -def get_region_from_slc_par(slc_par_file): - """从 GAMMA SLC 参数文件提取研究区域范围""" - corners_txt = 'corners_tmp.txt' - call_str = f"SLC_corners {slc_par_file} > {corners_txt}" - os.system(call_str) - - if not os.path.isfile(corners_txt): - print(f"✗ SLC_corners 执行失败") - return None - - with open(corners_txt, 'r') as f: - lines = f.readlines() - - os.remove(corners_txt) - - # 解析 SLC_corners 输出（第 9-10 行包含 lat/lon 范围） - try: - lat_line = lines[8] - lon_line = lines[9] - min_lat = float(lat_line.split(':')[1].split('max.')[0].strip()) - max_lat = float(lat_line.split(':')[2].strip()) - min_lon = float(lon_line.split(':')[1].split('max.')[0].strip()) - max_lon = float(lon_line.split(':')[2].strip()) - return min_lon, min_lat, max_lon, max_lat - except (IndexError, ValueError) as e: - print(f"✗ 解析 SLC_corners 输出失败: {e}") - return None - - -def get_region_from_template(project_name, margin=1.0): - """从 PyINT 模板文件获取区域范围（通过 master SLC 参数文件）""" - try: - from pyint import _utils as ut - except ImportError: - print("✗ 无法导入 pyint._utils，请确保 PyINT 在 PYTHONPATH 中") - return None, None - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - if not scratchDir or not templateDir: - print("✗ 环境变量 SCRATCHDIR 或 TEMPLATEDIR 未设置") - return None, None - - template_file = Path(templateDir) / f"{project_name}.template" - if not template_file.exists(): - print(f"✗ 模板文件不存在: {template_file}") - return None, None - - template_dict = ut.update_template(str(template_file)) - master_date = template_dict['masterDate'] - slc_dir = Path(scratchDir) / project_name / 'SLC' - master_slc_par = slc_dir / master_date / f"{master_date}.slc.par" - - if not master_slc_par.exists(): - print(f"✗ Master SLC 参数文件不存在: {master_slc_par}") - return None, None - - region = get_region_from_slc_par(str(master_slc_par)) - if region is None: - return None, None - - west, south, east, north = region - # 外扩 margin 度并取整 - west = math.floor(west - margin) - south = math.floor(south - margin) - east = math.ceil(east + margin) - north = math.ceil(north + margin) - - # 确定输出路径（使用模板中 DEM 路径或默认路径） - dem_dir = os.getenv('DEMDIR', '') - if 'DEM' in template_dict and template_dict['DEM'].strip(): - output_name = template_dict['DEM'].replace('.dem', '') - else: - dem_out_dir = Path(dem_dir) / project_name - dem_out_dir.mkdir(parents=True, exist_ok=True) - output_name = str(dem_out_dir / project_name) - - return (west, south, east, north), output_name - - -def main(): - args = cmdLineParse() - - # 确定工作目录 - if args.work_dir: - work_dir = Path(args.work_dir) - else: - work_dir = Path.cwd() - work_dir.mkdir(parents=True, exist_ok=True) - - # 确定 FABDEM 目录 - fabdem_dir = Path(args.fabdem_dir) - if not fabdem_dir.is_dir(): - print(f"✗ FABDEM 目录不存在: {fabdem_dir}") - sys.exit(1) - - zip_count = len(list(fabdem_dir.glob('*_FABDEM_V1-2.zip'))) - print(f"✓ FABDEM 目录: {fabdem_dir} ({zip_count} 个 ZIP 文件)") - - # 确定研究区域 - output_name = args.output_name - region = None - - if args.template_name: - # 从 PyINT 模板获取 - result = get_region_from_template(args.template_name, args.margin) - if result[0] is not None: - region = result[0] - if output_name is None: - output_name = result[1] - else: - print("✗ 无法从模板文件获取区域范围") - sys.exit(1) - - elif args.slc_par: - # 从 SLC 参数文件获取 - slc_region = get_region_from_slc_par(args.slc_par) - if slc_region is None: - sys.exit(1) - west, south, east, north = slc_region - west = math.floor(west - args.margin) - south = math.floor(south - args.margin) - east = math.ceil(east + args.margin) - north = math.ceil(north + args.margin) - region = (west, south, east, north) - - elif args.region: - # 从命令行参数解析 - parts = args.region.split('/') - if len(parts) != 4: - print("✗ 区域格式错误，应为: west/east/south/north") - sys.exit(1) - west, east, south, north = [float(x) for x in parts] - region = (west, south, east, north) - - else: - print("✗ 必须指定研究区域: -r, -s 或 --template") - sys.exit(1) - - if output_name is None: - output_name = str(work_dir / 'out') - - # 确保输出目录存在 - Path(output_name).parent.mkdir(parents=True, exist_ok=True) - - west, south, east, north = region - - print(f"\n{'='*70}") - print(f" 从本地 FABDEM 生成 GAMMA DEM") - print(f"{'='*70}") - print(f" 区域范围: {west}°E ~ {east}°E, {south}°N ~ {north}°N") - print(f" FABDEM 目录: {fabdem_dir}") - print(f" 输出文件: {output_name}.dem / {output_name}.dem.par") - print(f" 字节序: {args.byteorder}") - print(f"{'='*70}\n") - - # 1. 计算所需瓦片 - tiles = find_needed_tiles(west, south, east, north) - print(f"[1/4] 需要 {len(tiles)} 个 1°×1° 瓦片\n") - - # 2. 从 ZIP 中提取瓦片 - print(f"[2/4] 从 FABDEM ZIP 文件中提取瓦片...") - temp_dir = tempfile.mkdtemp(prefix='fabdem_tiles_') - try: - tif_files = extract_tiles_from_zips(tiles, str(fabdem_dir), temp_dir) - - if not tif_files: - print("\n✗ 没有提取到任何瓦片文件，请检查 FABDEM 目录和区域范围") - shutil.rmtree(temp_dir, ignore_errors=True) - sys.exit(1) - - print(f"\n 共提取 {len(tif_files)}/{len(tiles)} 个瓦片\n") - - # 3-4. 一步完成: VRT → ENVI 二进制 → GAMMA .dem - print(f"[3/4] 拼接裁剪 + 转换 GAMMA 格式...") - dem_file, dem_par_file = tiles_to_gamma_dem( - tif_files, output_name, west, south, east, north, args.byteorder - ) - - if dem_file is None: - print("\n✗ DEM 生成失败") - shutil.rmtree(temp_dir, ignore_errors=True) - sys.exit(1) - - finally: - # 清理临时目录 - shutil.rmtree(temp_dir, ignore_errors=True) - - # 完成 - print(f"\n{'='*70}") - print(f" ✓ FABDEM → GAMMA DEM 转换完成!") - print(f"{'='*70}") - print(f" DEM 文件: {dem_file}") - print(f" 参数文件: {dem_par_file}") - print(f" 字节序: {args.byteorder} endian") - print(f" 可直接用于: generate_rdc_dem.py") - print(f"{'='*70}\n") - - -if __name__ == '__main__': - main() diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/makedem.py b/.codex_tmp/pyint_variants/no_rescue/pyint/makedem.py deleted file mode 100644 index 38a211f..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/makedem.py +++ /dev/null @@ -1,1486 +0,0 @@ -#! /usr/bin/env python -import os -import sys -import numpy as np -import argparse -import subprocess -import glob -from skimage import io - -# 尝试导入 NASADEM 库 -try: - from NASADEM import NASADEM - from rasters import RasterGrid - HAS_NASADEM = True -except ImportError: - HAS_NASADEM = False - print("Warning: NASADEM library not installed. NASADEM download will not be available.") - print("Install with: pip install nasadem rasters") - -# 尝试导入 srtm 库 -try: - import srtm - HAS_SRTM = True -except ImportError: - HAS_SRTM = False - print("Warning: srtm library not installed. SRTM download will not be available.") - print("Install with: pip install srtm (requires Python >= 3.12)") - print(f"Current Python version: {sys.version_info.major}.{sys.version_info.minor}") - -resolutions = 30 # 90 - -def write_demrsc_file(FILE, Corner_LON, Corner_LAT, X_STEP, Y_STEP, WIDTH, LENGTH): - """Write ROI_PAC DEM resource file (.dem.rsc) - - Args: - FILE (str): Output resource file path - Corner_LON (str): Corner longitude - Corner_LAT (str): Corner latitude - X_STEP (str): Longitude step size - Y_STEP (str): Latitude step size - WIDTH (str): Number of columns - LENGTH (str): Number of lines - """ - f = open(FILE, 'w') - f.write('DATE12 111111-222222\n') - f.write('FILE_LENGTH ' + str(int(LENGTH)) + '\n') - f.write('FILE_TYPE .dem\n') - f.write('PROCESSOR roipac\n') - f.write('PROJECTION LATLON\n') - f.write('RLOOKS 1\n') - f.write('WIDTH ' + str(int(WIDTH)) + '\n') - f.write('XMAX ' + str(int(int(WIDTH)-1)) + '\n') - f.write('XMIN 0\n') - f.write('X_FIRST ' + str(float(Corner_LON)) + '\n') - f.write('X_STEP ' + str(float(X_STEP)) + '\n') - f.write('X_UNIT degrees\n') - f.write('YMAX ' + str(int(int(LENGTH)-1)) + '\n') - f.write('YMIN 0\n') - f.write('Y_FIRST ' + str(float(Corner_LAT)) + '\n') - f.write('Y_STEP ' + str(float(Y_STEP)) + '\n') - f.write('Y_UNIT degrees\n') - f.write('Z_OFFSET 0\n') - f.write('Z_SCALE 1\n') - f.close() - -def write_dempar_file(FILE, Corner_LON, Corner_LAT, X_STEP, Y_STEP, WIDTH, LENGTH, DATA_FORMAT): - """Write Gamma DEM parameter file (.dem.par) - - Args: - FILE (str): Output parameter file path - Corner_LON (str): Corner longitude - Corner_LAT (str): Corner latitude - X_STEP (str): Longitude step size - Y_STEP (str): Latitude step size - WIDTH (str): Number of columns - LENGTH (str): Number of lines - DATA_FORMAT (str): Data format (INTEGER*2 or REAL*4) - """ - DEM_TYPE = 'Copernicus30' - Proj = 'EQA' - f = open(FILE, 'w') - f.write("Gamma DIFF&GEO DEM/MAP parameter file\n") - f.write("title:\tIMPORTED DEM FROM %s\n" % DEM_TYPE) - f.write("DEM_projection: %s\n" % Proj) - f.write("data_format: %s\n" % DATA_FORMAT) - f.write("DEM_hgt_offset: 0.00000\n") - f.write("DEM_scale: 1.00000\n") - f.write("width: %s\n" % WIDTH) - f.write("nlines: %s\n" % LENGTH) - f.write("corner_lat: %s decimal degrees\n" % Corner_LAT) - f.write("corner_lon: %s decimal degrees\n" % Corner_LON) - f.write("post_lat: %s decimal degrees\n" % Y_STEP) - f.write("post_lon: %s decimal degrees\n" % X_STEP) - f.write("\n") - f.write("ellipsoid_name: WGS 84\n") - f.write("ellipsoid_ra: 6378137.000 m\n") - f.write("ellipsoid_reciprocal_flattening: 298.2572236\n") - f.write("\n") - f.write("datum_name: WGS 1984\n") - f.write("datum_shift_dx: 0.000 m\n") - f.write("datum_shift_dy: 0.000 m\n") - f.write("datum_shift_dz: 0.000 m\n") - f.write("datum_scale_m: 0.00000e+00\n") - f.write("datum_rotation_alpha: 0.00000e+00 arc-sec\n") - f.write("datum_rotation_beta: 0.00000e+00 arc-sec\n") - f.write("datum_rotation_gamma: 0.00000e+00 arc-sec\n") - f.write("datum_country_list Global Definition, WGS84, World\n") - f.write("\n") - f.close() - -def convert_to_gamma(input_file, output_name, byteorder='big', processor='gamma'): - """Convert TIF file to Gamma or ROI_PAC format (.dem and .dem.par/.dem.rsc) - - Args: - input_file (str): Input TIF file path - output_name (str): Output name (without extension) - byteorder (str): Byte order ('big' or 'little') - processor (str): Processor type ('gamma' or 'roi_pac') - """ - processor_name = 'Gamma' if processor == 'gamma' else 'ROI_PAC' - print(f"\n开始转换到 {processor_name} 格式: {input_file}") - - # 读取 DEM 数据 - 优先使用 GDAL（更稳定），然后是 rasterio - dem_data = None - - # 方法1: 使用 GDAL（最稳定） - try: - from osgeo import gdal - print(" 尝试使用 GDAL 读取数据...") - ds = gdal.Open(input_file) - if ds is not None: - band = ds.GetRasterBand(1) - dem_data = band.ReadAsArray() - print(f" ✓ 使用 GDAL 读取成功") - ds = None # 关闭文件 - except Exception as e: - print(f" ✗ GDAL 读取失败: {e}") - - # 方法2: 如果GDAL失败，尝试rasterio - if dem_data is None: - try: - import rasterio - print(" 尝试使用 rasterio 读取数据...") - with rasterio.open(input_file) as src: - dem_data = src.read(1) - print(f" ✓ 使用 rasterio 读取成功") - except Exception as e: - print(f" ✗ rasterio 读取失败: {e}") - - # 方法3: 最后尝试skimage - if dem_data is None: - try: - print(" 尝试使用 skimage 读取数据...") - dem_data = io.imread(input_file) - print(f" ✓ 使用 skimage 读取成功") - except Exception as e: - print(f" ✗ skimage 读取失败: {e}") - - # 如果所有方法都失败 - if dem_data is None: - raise ValueError(f"无法读取DEM文件: {input_file}，所有读取方法都失败了") - - # 确定数据格式 - if dem_data.dtype == 'float32': - DATA_FORMAT = 'REAL*4' - else: - DATA_FORMAT = 'INTEGER*2' - - # 字节序转换 - if not sys.byteorder == byteorder: - dem_data.byteswap(True) - - # 输出文件路径 - dem_file = output_name + '.dem' - if processor == 'gamma': - dem_par_file = output_name + '.dem.par' - else: - dem_par_file = output_name + '.dem.rsc' - - # 写入二进制 DEM 数据 - dem_data.tofile(dem_file) - - # 使用 gdalinfo 获取地理信息 - info_file = 'temp_gdalinfo.txt' - cmd = f"gdalinfo {input_file} > {info_file}" - os.system(cmd) - - # 解析地理信息 - Corner_LON = None - Corner_LAT = None - Post_LON = None - Post_LAT = None - WIDTH = None - FILE_LENGTH = None - - with open(info_file, 'r') as f: - for line in f: - if 'Origin =' in line: - parts = line.split('Origin =')[1].strip().split('(')[1].split(')')[0].split(',') - Corner_LON = parts[0] - Corner_LAT = parts[1] - elif 'Pixel Size ' in line: - parts = line.split('Pixel Size =')[1].strip().split('(')[1].split(')')[0].split(',') - Post_LON = parts[0] - Post_LAT = parts[1] - elif 'Size is' in line: - parts = line.split('Size is')[1].strip().split(',') - WIDTH = parts[0] - FILE_LENGTH = parts[1] - - # 删除临时文件 - if os.path.exists(info_file): - os.remove(info_file) - - # 写入参数文件 - if processor == 'gamma': - write_dempar_file(dem_par_file, Corner_LON, Corner_LAT, Post_LON, Post_LAT, WIDTH, FILE_LENGTH, DATA_FORMAT) - else: - write_demrsc_file(dem_par_file, Corner_LON, Corner_LAT, Post_LON, Post_LAT, WIDTH, FILE_LENGTH) - - print(f"{byteorder} endian {dem_file} and {dem_par_file} are generated.") - print(f"{processor_name} 格式转换完成!") - -def get_sufix(STR): - """Get file extension""" - n = len(STR.split('.')) - SUFIX = STR.split('.')[n-1] - return SUFIX - -def read_region(STR): - """Parse region string 'west/east/south/north'""" - WEST = STR.split('/')[0] - EAST = STR.split('/')[1].split('/')[0] - SOUTH = STR.split(EAST+'/')[1].split('/')[0] - NORTH = STR.split(EAST+'/')[1].split('/')[1] - WEST = float(WEST) - SOUTH = float(SOUTH) - EAST = float(EAST) - NORTH = float(NORTH) - return WEST, SOUTH, EAST, NORTH - -def cmd_init(lon, lat, save_path): - s_lon = str(abs(lon)) - s_lat = str(abs(lat)) - if abs(lon) < 10: - s_lon = "00" + str(abs(lon)) - elif abs(lon) < 100: - s_lon = "0" + str(abs(lon)) - if abs(lat) < 10: - s_lat = "0" + str(abs(lat)) - if lon < 0: - c_lon = "W" + str(s_lon) - else: - c_lon = "E" + str(s_lon) - if lat < 0: - c_lat = "S" + str(s_lat) - else: - c_lat = "N" + str(s_lat) - cmd = "aws s3 cp --no-sign-request" + " s3://copernicus-dem-{0}m/Copernicus_DSM_COG_{1}_{2}_00_{3}_00_DEM/ {4} --recursive".format( - str(resolutions), str(int(resolutions / 3)), str(c_lat), str(c_lon), save_path) - return cmd - -def get_remote_file(lon, lat, save_path, max_retries=3): - """Get Copernicus Dem by lon and lat with retry mechanism - - Args: - lon (number): lontitude - lat (number): latitude - save_path (str): directory to save data - max_retries (int): maximum retry attempts - """ - lon = int(lon) - lat = int(lat) - cmd = cmd_init(lon, lat, save_path) - - for attempt in range(max_retries): - try: - result = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=300) - if result.returncode == 0: - print(f"✓ Successfully downloaded: lon={lon}, lat={lat}") - return True - else: - if attempt < max_retries - 1: - print(f"✗ Failed (attempt {attempt+1}/{max_retries}): lon={lon}, lat={lat}, retrying...") - import time - time.sleep(2) - else: - print(f"✗ Failed after {max_retries} attempts: lon={lon}, lat={lat}") - return False - except subprocess.TimeoutExpired: - if attempt < max_retries - 1: - print(f"⏱ Timeout (attempt {attempt+1}/{max_retries}): lon={lon}, lat={lat}, retrying...") - import time - time.sleep(2) - else: - print(f"✗ Timeout after {max_retries} attempts: lon={lon}, lat={lat}") - return False - except Exception as e: - print(f"✗ Exception occurred for lon={lon}, lat={lat}: {e}") - return False - - return False - -def get_remote_file_batch(west, east, south, north, save_path, num_workers=4): - """Get Copernicus Dems by WESN extent with parallel download - - Args: - west (number): western longitude boundary - east (number): eastern longitude boundary - south (number): southern latitude boundary - north (number): northern latitude boundary - save_path (str): directory to save data - num_workers (int): number of parallel download workers (default: 4) - """ - from concurrent.futures import ThreadPoolExecutor, as_completed - from tqdm import tqdm - - west = int(west) - east = int(east) - south = int(south) - north = int(north) - - # 生成所有需要下载的坐标对 - coords = [(lon, lat) for lon in range(west, east + 1) for lat in range(south, north + 1)] - total_tiles = len(coords) - - print(f"\n{'='*80}") - print(f"并行下载 Copernicus DEM 数据") - print(f"{'='*80}") - print(f"区域范围: {west}°E - {east}°E, {south}°N - {north}°N") - print(f"总瓦片数: {total_tiles}") - print(f"并行线程: {num_workers}") - print(f"{'='*80}\n") - - success_count = 0 - failed_count = 0 - - # 使用线程池并行下载 - with ThreadPoolExecutor(max_workers=num_workers) as executor: - # 提交所有下载任务 - future_to_coord = {executor.submit(get_remote_file, lon, lat, save_path): (lon, lat) - for lon, lat in coords} - - # 使用 tqdm 显示进度条 - with tqdm(total=total_tiles, desc="下载进度", unit="瓦片") as pbar: - for future in as_completed(future_to_coord): - coord = future_to_coord[future] - try: - result = future.result() - if result: - success_count += 1 - else: - failed_count += 1 - except Exception as e: - print(f"\n✗ 下载失败 {coord}: {e}") - failed_count += 1 - - pbar.update(1) - - # 打印统计信息 - print(f"\n{'='*80}") - print(f"下载完成统计") - print(f"{'='*80}") - print(f"成功: {success_count}/{total_tiles} 瓦片") - print(f"失败: {failed_count}/{total_tiles} 瓦片") - if failed_count > 0: - print(f"\n⚠️ 有 {failed_count} 个瓦片下载失败,可能会影响 DEM 完整性") - print(f"{'='*80}\n") - -def download_nasadem(west, south, east, north, save_path, cell_size_deg=0.000277778): - """Download NASADEM data using NASADEM library - - Args: - west (float): western longitude boundary - south (float): southern latitude boundary - east (float): eastern longitude boundary - north (float): northern latitude boundary - save_path (str): directory to save data - cell_size_deg (float): cell size in degrees (default: 0.000277778 ≈ 30m at equator) - - Returns: - str: path to the downloaded DEM file, or None if failed - """ - if not HAS_NASADEM: - print("✗ NASADEM library not available. Cannot download NASADEM.") - print(" Install with: pip install nasadem rasters") - return None - - print(f"\n{'='*80}") - print(f"下载 NASADEM 数据") - print(f"{'='*80}") - print(f"区域范围: {west}°E - {east}°E, {south}°N - {north}°N") - print(f"分辨率: {cell_size_deg}° (约 {cell_size_deg * 111000:.0f} 米)") - print(f"{'='*80}\n") - - try: - from NASADEM.NASADEM import NASADEMConnection - from rasters import RasterGrid - import time - - start_time = time.time() - - # 定义目标区域 - geometry = RasterGrid.from_bbox( - xmin=west, ymin=south, xmax=east, ymax=north, - cell_size=cell_size_deg, - crs="EPSG:4326" - ) - - print(f"RasterGrid 维度: {geometry.shape}") - print(f"正在从NASA服务器下载高程数据...") - print(f"提示: 首次下载需要从NASA服务器获取数据,可能需要几分钟时间...") - - # 创建NASA DEM连接并获取高程数据 - conn = NASADEMConnection() - elevation = conn.elevation_m(geometry) - - elapsed_time = time.time() - start_time - - print(f"\n✓ 下载完成!") - print(f" 高程范围: {float(elevation.min()):.1f} 到 {float(elevation.max()):.1f} 米") - print(f" 数据维度: {elevation.shape}") - print(f" 用时: {elapsed_time:.1f} 秒") - - # 保存为 GeoTIFF - output_file = os.path.join(save_path, f"NASADEM_{west}_{east}_{south}_{north}.tif") - print(f"\n保存到: {output_file}") - - # 检查elevation对象是否有save方法 - if hasattr(elevation, 'save'): - elevation.save(output_file) - print(f"✓ 文件保存成功: {output_file}") - if os.path.exists(output_file): - file_size = os.path.getsize(output_file) / (1024 * 1024) - print(f" 文件大小: {file_size:.2f} MB") - return output_file - - # 否则使用 rasterio 保存 - try: - import rasterio - from rasterio.crs import CRS - from rasterio.transform import from_bounds - - # 获取数据数组 - if hasattr(elevation, 'read'): - elev_array = elevation.read() - else: - elev_array = np.array(elevation) - - # 计算变换矩阵 - height, width = elev_array.shape - transform = from_bounds(west, south, east, north, width, height) - - # 写入文件 - with rasterio.open( - output_file, - 'w', - driver='GTiff', - height=height, - width=width, - count=1, - dtype=elev_array.dtype, - crs=CRS.from_epsg(4326), - transform=transform, - nodata=-32767, - compress='LZW' - ) as dst: - dst.write(elev_array, 1) - - print(f"✓ 文件保存成功: {output_file}") - if os.path.exists(output_file): - file_size = os.path.getsize(output_file) / (1024 * 1024) - print(f" 文件大小: {file_size:.2f} MB") - return output_file - - except ImportError: - print("⚠️ rasterio 未安装,尝试使用 GDAL...") - - # 使用 GDAL 创建 GeoTIFF - width, height = geometry.width, geometry.height - gdal_cmd = f"gdal_create -outsize {width} {height} -of GTiff -co COMPRESS=LZW -a_srs EPSG:4326 -a_ullr {west} {north} {east} {south} {output_file}" - result = os.system(gdal_cmd) - - if result == 0 and os.path.exists(output_file): - print(f"✓ 文件创建成功: {output_file}") - return output_file - else: - print(f"✗ 文件创建失败") - return None - - except Exception as e: - print(f"\n✗ 下载失败: {e}") - import traceback - traceback.print_exc() - return None - -def download_srtm(west, south, east, north, save_path, srtm_data_dir=None, cell_size=0.000277778): - """Download SRTM data using srtm library (requires pre-downloaded .hgt files) - - Args: - west (float): western longitude boundary - south (float): southern latitude boundary - east (float): eastern longitude boundary - north (float): northern latitude boundary - save_path (str): directory to save output DEM - srtm_data_dir (str): directory containing SRTM .hgt files (required) - cell_size (float): cell size in degrees (default: 0.000277778 ≈ 30m) - - Returns: - str: path to the generated DEM file, or None if failed - - Note: - This function requires SRTM .hgt files to be pre-downloaded. - You can download SRTM data from: - - https://srtm.csi.cgiar.org/ - - https://earthexplorer.usgs.gov/ - - Python version requirement: - - srtm library requires Python >= 3.12 - - For Python < 3.12, use process_srtm_hgt_files() instead - """ - if not HAS_SRTM: - print("✗ srtm library not available. Cannot use SRTM.") - print(" Install with: pip install srtm (requires Python >= 3.12)") - print(f" Current Python version: {sys.version_info.major}.{sys.version_info.minor}") - print(" Alternative: Use process_srtm_hgt_files() to manually process .hgt files") - return None - - if srtm_data_dir is None or not os.path.isdir(srtm_data_dir): - print("✗ SRTM data directory not provided or does not exist.") - print(" Please download SRTM .hgt files and specify the directory.") - print(" Download sources:") - print(" - https://srtm.csi.cgiar.org/") - print(" - https://earthexplorer.usgs.gov/") - return None - - print(f"\n{'='*80}") - print(f"使用 SRTM 数据生成 DEM") - print(f"{'='*80}") - print(f"区域范围: {west}°E - {east}°E, {south}°N - {north}°N") - print(f"SRTM数据目录: {srtm_data_dir}") - print(f"分辨率: {cell_size}° (约 {cell_size * 111000:.0f} 米)") - print(f"{'='*80}\n") - - try: - import time - from tqdm import tqdm - - start_time = time.time() - - # 创建 SrtmService - service = srtm.SrtmService(srtm_data_dir, cache_size=100) - print(f"✓ SrtmService 创建成功") - - # 生成坐标网格 - lats = np.arange(south, north + cell_size, cell_size) - lons = np.arange(west, east + cell_size, cell_size) - - print(f"生成网格: {len(lats)} × {len(lons)} = {len(lats) * len(lons)} 个点") - - # 批量查询高程 - coords = [(lat, lon) for lat in lats for lon in lons] - print(f"正在查询高程数据...") - - elevations = service.get_elevations_batch(coords, default=0) - - # 重塑为数组 - elevation_array = np.array(elevations, dtype=np.float32).reshape(len(lats), len(lons)) - - # 替换0值为NaN(可能是缺失数据) - elevation_array[elevation_array == 0] = np.nan - - elapsed_time = time.time() - start_time - - print(f"\n✓ 查询完成!") - print(f" 有效数据: {np.sum(~np.isnan(elevation_array))} / {elevation_array.size} 点") - if np.sum(~np.isnan(elevation_array)) > 0: - print(f" 高程范围: {np.nanmin(elevation_array):.1f} 到 {np.nanmax(elevation_array):.1f} 米") - print(f" 用时: {elapsed_time:.1f} 秒") - - # 保存为 GeoTIFF - output_file = os.path.join(save_path, f"SRTM_{west}_{east}_{south}_{north}.tif") - print(f"\n保存到: {output_file}") - - try: - import rasterio - from rasterio.crs import CRS - from rasterio.transform import from_bounds - - height, width = elevation_array.shape - transform = from_bounds(west, south, east, north, width, height) - - # 替换NaN为nodata值 - elevation_array[np.isnan(elevation_array)] = -32767 - - with rasterio.open( - output_file, - 'w', - driver='GTiff', - height=height, - width=width, - count=1, - dtype=np.float32, - crs=CRS.from_epsg(4326), - transform=transform, - nodata=-32767, - compress='LZW' - ) as dst: - dst.write(elevation_array.astype(np.float32), 1) - - print(f"✓ 文件保存成功: {output_file}") - if os.path.exists(output_file): - file_size = os.path.getsize(output_file) / (1024 * 1024) - print(f" 文件大小: {file_size:.2f} MB") - return output_file - - except ImportError: - print("⚠️ rasterio 未安装,无法保存文件") - return None - - except Exception as e: - print(f"\n✗ 生成失败: {e}") - import traceback - traceback.print_exc() - return None - -def split_large_region(west, south, east, north, tile_size=1.0): - """将大区域分割成小块 - - Args: - west, south, east, north: 区域边界 - tile_size: 分块大小（度），默认1°×1° - - Returns: - list: 分块列表 [(west, south, east, north), ...] - """ - tiles = [] - - w = west - while w < east: - e = min(w + tile_size, east) - - s = south - while s < north: - n = min(s + tile_size, north) - tiles.append((w, s, e, n)) - s += tile_size - - w += tile_size - - return tiles - - -def download_srtm_cgiar(west, south, east, north, save_path, api_key=None, num_workers=4, dem_type='SRTMGL1', auto_tile=True): - """Download DEM data from OpenTopography automatically - - Args: - west (float): western longitude boundary - south (float): southern latitude boundary - east (float): eastern longitude boundary - north (float): northern latitude boundary - save_path (str): directory to save output DEM - api_key (str): OpenTopography API key (optional, will prompt if not provided) - num_workers (int): number of parallel downloads (not used for API, kept for compatibility) - dem_type (str): DEM type (default: 'SRTMGL1') - - SRTMGL1: SRTM GL1 30m (recommended for SRTM) - - SRTMGL3: SRTM GL3 90m - - NASADEM: NASADEM 30m - - COP30: Copernicus 30m - - COP90: Copernicus 90m - auto_tile (bool): 自动分块下载大区域 - - Returns: - str: path to the generated DEM file, or None if failed - - Note: - OpenTopography API provides: - - SRTMGL1: SRTM GL1 30m (recommended) - - SRTMGL3: SRTM GL3 90m - - NASADEM: NASADEM Global DEM 30m - - COP30: Copernicus 30m - - COP90: Copernicus 90m - - API Documentation: https://portal.opentopography.org/apidocs/ - Get API key: https://opentopography.org/myOpenTopo - """ - # DEM类型信息 - dem_info = { - 'SRTMGL1': {'resolution': '30m', 'name': 'SRTM GL1'}, - 'SRTMGL3': {'resolution': '90m', 'name': 'SRTM GL3'}, - 'NASADEM': {'resolution': '30m', 'name': 'NASADEM'}, - 'COP30': {'resolution': '30m', 'name': 'Copernicus 30m'}, - 'COP90': {'resolution': '90m', 'name': 'Copernicus 90m'} - } - - info = dem_info.get(dem_type, {'resolution': 'unknown', 'name': dem_type}) - - print(f"\n{'='*80}") - print(f"从 OpenTopography 自动下载 DEM 数据") - print(f"{'='*80}") - print(f"区域范围: {west}°E - {east}°E, {south}°N - {north}°N") - print(f"数据源: {info['name']} ({info['resolution']} resolution)") - print(f"{'='*80}\n") - - # 计算区域面积 - region_area = (east - west) * (north - south) - - # 如果区域较大，自动分块 - if auto_tile and region_area > 4: - print(f"⚠️ 区域较大 ({region_area:.1f} 平方度)，将自动分块下载以提高稳定性") - - # 计算分块 - tiles = split_large_region(west, south, east, north, tile_size=1.0) - print(f" 将分成 {len(tiles)} 个 1°×1° 的区块下载\n") - - # 创建临时目录存放分块文件 - temp_dir = os.path.join(save_path, f"temp_tiles_{dem_type}_{west}_{east}_{south}_{north}") - if not os.path.exists(temp_dir): - os.makedirs(temp_dir) - - # 下载每个分块 - downloaded_files = [] - failed_tiles = [] - - for i, (w, s, e, n) in enumerate(tiles, 1): - print(f"\n{'='*80}") - print(f"下载分块 {i}/{len(tiles)}: {w}°E-{e}°E, {s}°N-{n}°N") - print(f"{'='*80}") - - tile_file = download_single_tile(w, s, e, n, temp_dir, api_key, dem_type, info) - - if tile_file: - downloaded_files.append(tile_file) - print(f"✓ 分块 {i}/{len(tiles)} 下载成功") - else: - failed_tiles.append((w, s, e, n)) - print(f"✗ 分块 {i}/{len(tiles)} 下载失败") - - # 如果有失败的分块 - if failed_tiles: - print(f"\n⚠️ 有 {len(failed_tiles)} 个分块下载失败") - print("失败的分块:") - for w, s, e, n in failed_tiles: - print(f" {w}°E-{e}°E, {s}°N-{n}°N") - - # 合并所有分块 - if downloaded_files: - print(f"\n{'='*80}") - print(f"合并 {len(downloaded_files)} 个分块...") - print(f"{'='*80}\n") - - merged_file = merge_dem_tiles(downloaded_files, save_path, dem_type, west, east, south, north) - - # 清理临时文件 - import shutil - try: - shutil.rmtree(temp_dir) - print("✓ 临时文件已清理") - except: - pass - - if merged_file: - print(f"\n✓ 最终输出文件: {merged_file}") - return merged_file - else: - print("✗ 合并失败") - return None - else: - print("✗ 所有分块下载失败") - return None - - # 小区域直接下载 - else: - return download_single_tile(west, south, east, north, save_path, api_key, dem_type, info) - - -def download_single_tile(west, south, east, north, save_path, api_key, dem_type, dem_info): - """下载单个DEM块 - - Args: - west, south, east, north: 区域边界 - save_path: 保存路径 - api_key: OpenTopography API key - dem_type: DEM类型 - dem_info: DEM信息字典 - - Returns: - str: 下载的文件路径，失败返回None - """ - try: - import time - import urllib.request - import urllib.parse - - start_time = time.time() - - # OpenTopography API endpoint - base_url = "https://portal.opentopography.org/API/globaldem" - - # 如果没有提供API key，提示用户获取 - if api_key is None: - print("⚠️ 需要OpenTopography API key才能下载SRTM数据") - print("\n获取免费API key的步骤:") - print(" 1. 访问: https://opentopography.org/myOpenTopo") - print(" 2. 注册免费账户") - print(" 3. 在 'My Account' 页面请求API key") - print(" 4. 使用 --opentopo-api-key 参数提供API key") - print("\n或者使用其他DEM数据源:") - print(" - Copernicus (推荐): --dem-source copernicus") - print(" - NASADEM: --dem-source nasadem") - return None - - # 构建请求参数 - params = { - 'demtype': dem_type, - 'south': south, - 'north': north, - 'west': west, - 'east': east, - 'outputFormat': 'GTiff', - 'API_Key': api_key - } - - # 构建完整URL - url = base_url + '?' + urllib.parse.urlencode(params) - - # 输出文件名 - output_file = os.path.join(save_path, f"{dem_type}_{west}_{east}_{south}_{north}.tif") - - # 下载文件 - 使用curl以获得更好的稳定性 - max_retries = 3 - retry_count = 0 - download_success = False - - while retry_count < max_retries and not download_success: - try: - retry_count += 1 - print(f"正在从OpenTopography下载数据... (尝试 {retry_count}/{max_retries})") - - # 尝试使用 curl 下载（更稳定） - try: - import subprocess - curl_cmd = ['curl', '-L', '-o', output_file, '-s', '--show-error', url] - result = subprocess.run(curl_cmd, capture_output=True, text=True, timeout=1800) # 30分钟超时 - - if result.returncode != 0: - raise Exception(f"curl下载失败: {result.stderr}") - - print(" ✓ curl下载完成") - download_success = True - - except (subprocess.TimeoutExpired, FileNotFoundError): - # 如果curl不可用或超时，使用urllib - print(" 使用urllib下载...") - urllib.request.urlretrieve(url, output_file) - download_success = True - - except urllib.error.HTTPError as e: - print(f"\n✗ HTTP错误: {e.code} {e.reason}") - if e.code == 401: - print(" API key无效或已过期") - print(" 请获取新的API key: https://opentopography.org/myOpenTopo") - elif e.code == 403: - print(" 访问被拒绝，可能是:") - print(" - 请求区域过大") - print(" - API key配额已用完") - print(" - 需要注册获取免费API key") - if retry_count < max_retries: - print(f" 将在3秒后重试...") - time.sleep(3) - continue - return None - except Exception as e: - print(f"\n✗ 下载失败: {e}") - if retry_count < max_retries: - print(f" 将在3秒后重试...") - time.sleep(3) - continue - return None - - # 检查文件是否有效 - if os.path.exists(output_file): - file_size = os.path.getsize(output_file) - - # 如果文件很小，可能是错误消息 - if file_size < 1000: - with open(output_file, 'r') as f: - error_msg = f.read() - print(f"\n✗ 下载失败: {error_msg}") - os.remove(output_file) - return None - - elapsed_time = time.time() - start_time - - print(f"\n✓ 下载完成!") - print(f" 输出文件: {output_file}") - print(f" 文件大小: {file_size / (1024 * 1024):.2f} MB") - print(f" 用时: {elapsed_time:.1f} 秒") - - # 验证文件完整性 - print("\n验证文件完整性...") - - try: - import rasterio - with rasterio.open(output_file) as src: - # 尝试读取多个位置的数据验证 - print(f" 文件大小: {src.width} x {src.height} 像素") - - # 读取多个测试点 - test_points = [ - (0, 0, min(100, src.height), min(100, src.width)), # 左上角 - (max(0, src.height-100), max(0, src.width-100), src.height, src.width), # 右下角 - (src.height//2-50, src.width//2-50, src.height//2+50, src.width//2+50), # 中间 - ] - - for i, (row_start, col_start, row_end, col_end) in enumerate(test_points): - try: - window = ((row_start, row_end), (col_start, col_end)) - test_data = src.read(1, window=window) - print(f" ✓ 测试点 {i+1}/3 验证成功") - except Exception as e: - print(f" ✗ 测试点 {i+1}/3 验证失败: {e}") - raise - - print(f" ✓ 所有验证点通过，文件完整") - return output_file - - except Exception as e: - print(f" ✗ 文件验证失败: {e}") - print(f" 文件可能已损坏，正在删除...") - try: - os.remove(output_file) - except: - pass - return None - else: - print("✗ 文件下载失败") - return None - - except Exception as e: - print(f"\n✗ 下载失败: {e}") - import traceback - traceback.print_exc() - return None - -def merge_dem_tiles(tile_files, save_path, dem_type, west, east, south, north): - """合并多个DEM分块文件 - - Args: - tile_files: 分块文件列表 - save_path: 保存路径 - dem_type: DEM类型 - west, east, south, north: 最终区域边界 - - Returns: - str: 合并后的文件路径，失败返回None - """ - if not tile_files: - print("✗ 没有文件需要合并") - return None - - try: - import rasterio - from rasterio.merge import merge - import time - - print(f"开始合并 {len(tile_files)} 个分块文件...") - - # 读取所有分块 - src_files_to_mosaic = [] - for tile_file in tile_files: - src = rasterio.open(tile_file) - src_files_to_mosaic.append(src) - - # 合并 - start_time = time.time() - mosaic, out_trans = merge(src_files_to_mosaic) - - # 获取输出元数据 - out_meta = src_files_to_mosaic[0].meta.copy() - out_meta.update({ - "driver": "GTiff", - "height": mosaic.shape[1], - "width": mosaic.shape[2], - "transform": out_trans, - "compress": "lzw" - }) - - # 输出文件名 - output_file = os.path.join(save_path, f"{dem_type}_{west}_{east}_{south}_{north}.tif") - - # 写入合并后的文件 - with rasterio.open(output_file, "w", **out_meta) as dest: - dest.write(mosaic) - - # 关闭所有源文件 - for src in src_files_to_mosaic: - src.close() - - elapsed_time = time.time() - start_time - file_size = os.path.getsize(output_file) / (1024 * 1024) - - print(f"✓ 合并完成!") - print(f" 输出文件: {output_file}") - print(f" 文件大小: {file_size:.2f} MB") - print(f" 用时: {elapsed_time:.1f} 秒") - - return output_file - - except ImportError: - print("✗ rasterio 未安装，无法合并文件") - print(" 安装方法: pip install rasterio") - return None - except Exception as e: - print(f"✗ 合并失败: {e}") - import traceback - traceback.print_exc() - return None - - -def download_srtm_cgiar_old(west, south, east, north, save_path, num_workers=4): - """Download SRTM data from CSI-CGIAR (DEPRECATED - tiles not accessible) - - This function is kept as backup but CSI-CGIAR tile downloads are not reliable. - Use download_srtm_cgiar() with OpenTopography API instead. - """ - print("警告: CSI-CGIAR瓦片下载不可靠，建议使用OpenTopography API") - return None - -def process_srtm_hgt_files(west, south, east, north, srtm_data_dir, save_path, output_name=None): - """Process SRTM .hgt files manually (alternative for Python < 3.12) - - Args: - west (float): western longitude boundary - south (float): southern latitude boundary - east (float): eastern longitude boundary - north (float): northern latitude boundary - srtm_data_dir (str): directory containing SRTM .hgt files - save_path (str): directory to save output DEM - output_name (str): output filename (without extension) - - Returns: - str: path to the generated DEM file, or None if failed - - Note: - SRTM .hgt files are 1°×1° tiles with 1201×1201 pixels (3 arc-second) or - 3601×3601 pixels (1 arc-second). This function reads and merges them. - """ - print(f"\n{'='*80}") - print(f"手动处理 SRTM .hgt 文件") - print(f"{'='*80}") - print(f"区域范围: {west}°E - {east}°E, {south}°N - {north}°N") - print(f"SRTM数据目录: {srtm_data_dir}") - print(f"{'='*80}\n") - - try: - import rasterio - from rasterio.merge import merge - from rasterio.crs import CRS - import time - - start_time = time.time() - - # 查找需要的SRTM文件 - hgt_files = [] - for lat in range(int(south), int(north) + 1): - for lon in range(int(west), int(east) + 1): - # 构造文件名 - lat_prefix = 'N' if lat >= 0 else 'S' - lon_prefix = 'E' if lon >= 0 else 'W' - - # SRTM文件名格式: N39E116.hgt - filename = f"{lat_prefix}{abs(lat):02d}{lon_prefix}{abs(lon):03d}.hgt" - filepath = os.path.join(srtm_data_dir, filename) - - if os.path.exists(filepath): - hgt_files.append(filepath) - print(f" ✓ 找到文件: {filename}") - else: - print(f" ⚠ 文件缺失: {filename}") - - if not hgt_files: - print("\n✗ 没有找到任何SRTM文件!") - return None - - print(f"\n找到 {len(hgt_files)} 个SRTM文件") - - # 使用GDAL合并.hgt文件 - merged_file = os.path.join(save_path, f"SRTM_merged_{west}_{east}_{south}_{north}.tif") - - # 构建gdal_merge命令 - gdal_merge_cmd = ['gdal_merge.py', '-o', merged_file, '-of', 'GTiff', '-co', 'COMPRESS=LZW'] - gdal_merge_cmd.extend(hgt_files) - - print(f"\n使用GDAL合并文件...") - result = subprocess.run(gdal_merge_cmd, capture_output=True, text=True) - - if result.returncode != 0 or not os.path.exists(merged_file): - print(f"✗ GDAL合并失败: {result.stderr}") - return None - - print(f"✓ 合并完成: {merged_file}") - - # 裁剪到目标范围 - if output_name is None: - output_name = os.path.join(save_path, f"SRTM_{west}_{east}_{south}_{north}") - - output_file = output_name + '.tif' - - gdal_warp_cmd = [ - 'gdalwarp', - '-te', str(west), str(south), str(east), str(north), - '-of', 'GTiff', - '-co', 'COMPRESS=LZW', - merged_file, - output_file - ] - - print(f"\n裁剪到目标范围...") - result = subprocess.run(gdal_warp_cmd, capture_output=True, text=True) - - if result.returncode != 0 or not os.path.exists(output_file): - print(f"✗ GDAL裁剪失败: {result.stderr}") - return None - - elapsed_time = time.time() - start_time - - print(f"\n✓ SRTM DEM 生成完成!") - print(f" 输出文件: {output_file}") - if os.path.exists(output_file): - file_size = os.path.getsize(output_file) / (1024 * 1024) - print(f" 文件大小: {file_size:.2f} MB") - print(f" 用时: {elapsed_time:.1f} 秒") - - # 清理临时文件 - if os.path.exists(merged_file) and merged_file != output_file: - os.remove(merged_file) - print(f" 清理临时文件: {merged_file}") - - return output_file - - except ImportError: - print("✗ rasterio 未安装,无法处理文件") - return None - except Exception as e: - print(f"\n✗ 处理失败: {e}") - import traceback - traceback.print_exc() - return None - -def merge_tif_files(input_pattern, output_file): - """合并多个 TIF 文件为一个文件 - - Args: - input_pattern (str): 输入 TIF 文件的匹配模式 - output_file (str): 输出合并后的 TIF 文件路径 - """ - # 查找所有匹配的 TIF 文件 - tif_files = sorted(glob.glob(input_pattern)) - - if not tif_files: - print(f"未找到匹配的 TIF 文件: {input_pattern}") - return False - - print(f"\n{'='*80}") - print(f"合并 DEM 瓦片") - print(f"{'='*80}") - print(f"找到 {len(tif_files)} 个 TIF 文件待合并") - print(f"输出文件: {output_file}") - print(f"{'='*80}\n") - - # 使用 gdal_merge.py 合并文件 - try: - start_time = __import__('time').time() - cmd = ["gdal_merge.py", "-o", output_file, "-co", "COMPRESS=LZW", "-co", "BIGTIFF=YES"] + tif_files - print(f"执行合并命令...") - - # 使用 tqdm 显示进度 - result = subprocess.run(cmd, capture_output=True, text=True) - - elapsed_time = __import__('time').time() - start_time - - if result.returncode == 0: - # 检查输出文件大小 - if os.path.exists(output_file): - file_size = os.path.getsize(output_file) / (1024 * 1024) # MB - print(f"\n✓ 成功合并文件!") - print(f" 输出文件: {output_file}") - print(f" 文件大小: {file_size:.2f} MB") - print(f" 用时: {elapsed_time:.1f} 秒") - print(f"{'='*80}\n") - return True - else: - print(f"\n✗ 合并失败: {result.stderr}") - print(f"{'='*80}\n") - return False - except Exception as e: - print(f"\n✗ 合并过程中发生错误: {e}") - print(f"{'='*80}\n") - return False - -def convert_format(input_file, output_file, format_type="GTiff", options=None): - """转换 TIF 文件格式 - - Args: - input_file (str): 输入文件路径 - output_file (str): 输出文件路径 - format_type (str): 输出格式类型 (默认: GTiff) - options (list): gdal_translate 的额外选项 - """ - if options is None: - options = [] - - try: - cmd = ["gdal_translate", "-of", format_type] + options + [input_file, output_file] - print(f"执行转换命令: {' '.join(cmd)}") - result = subprocess.run(cmd, capture_output=True, text=True) - - if result.returncode == 0: - print(f"成功转换文件到: {output_file}") - return True - else: - print(f"转换失败: {result.stderr}") - return False - except Exception as e: - print(f"转换过程中发生错误: {e}") - return False - -def process_dem_files(save_path, west, east, south, north, merge=True, convert=True, - output_format="GTiff", convert_options=None, gamma=False, gamma_byteorder='big', - processor='gamma', output_name=None, num_workers=4, - srtm_data_dir=None, opentopo_api_key=None, opentopo_dem_type='SRTMGL1', - fabdem_dir=None): - """处理下载的 DEM 文件：下载、合并和转换 - - Args: - save_path (str): 数据保存目录 - west (float): 西经边界 - east (float): 东经边界 - south (float): 南纬边界 - north (float): 北纬边界 - merge (bool): 是否合并文件 - convert (bool): 是否转换格式 - output_format (str): 输出格式 - convert_options (list): 转换选项 - gamma (bool): 是否转换为 Gamma/ROI_PAC 格式 - gamma_byteorder (str): 字节序 ('big' 或 'little') - processor (str): 处理器类型 ('gamma' 或 'roi_pac') - output_name (str): 输出文件名（不含扩展名） - num_workers (int): 并行下载线程数 (default: 4) - dem_source (str): DEM数据源 ('copernicus', 'nasadem' 或 'srtm', default: 'copernicus') - srtm_data_dir (str): SRTM数据目录 (仅srtm源需要) - """ - if convert_options is None: - convert_options = ["-co", "COMPRESS=LZW", "-co", "TILED=YES"] - - # ============ 优先尝试本地 FABDEM 瓦片库 ============ - if fabdem_dir and os.path.isdir(fabdem_dir): - print(f"\n{'='*80}") - print(f"优先使用本地 FABDEM 瓦片库: {fabdem_dir}") - print(f"{'='*80}") - try: - from make_local_dem import (find_needed_tiles, extract_tiles_from_zips, - tiles_to_gamma_dem) - import tempfile, shutil - - tiles = find_needed_tiles(west, south, east, north) - print(f"需要 {len(tiles)} 个 1°×1° FABDEM 瓦片") - - temp_dir = tempfile.mkdtemp(prefix='fabdem_tiles_') - try: - tif_files = extract_tiles_from_zips(tiles, fabdem_dir, temp_dir) - if tif_files: - # 一步完成: VRT → ENVI 二进制 → .dem（仅一次大文件写入） - out_name = output_name if output_name else os.path.join(save_path, 'out') - dem_file, dem_par_file = tiles_to_gamma_dem( - tif_files, out_name, west, south, east, north, gamma_byteorder) - if dem_file and dem_par_file: - print(f"\n✓ 本地 FABDEM 生成 GAMMA DEM 成功!") - print(f" DEM: {dem_file}") - print(f" PAR: {dem_par_file}") - shutil.rmtree(temp_dir, ignore_errors=True) - return - else: - print("⚠ 本地 FABDEM 转换失败，回退到网络下载") - else: - print("⚠ 未从本地 FABDEM 提取到瓦片，回退到网络下载") - finally: - shutil.rmtree(temp_dir, ignore_errors=True) - except ImportError as e: - print(f"⚠ 无法导入 make_local_dem 模块 ({e})，回退到网络下载") - except Exception as e: - print(f"⚠ 本地 FABDEM 处理异常 ({e})，回退到网络下载") - - # ============ 回退: 网络下载 ============ - # 检查是否使用预下载的SRTM数据 - if srtm_data_dir and os.path.isdir(srtm_data_dir): - print(f"\n{'='*80}") - print(f"使用预下载的 SRTM 数据") - print(f"{'='*80}") - - if HAS_SRTM: - # Python >= 3.12, 使用srtm库 - dem_file = download_srtm(west, south, east, north, save_path, srtm_data_dir) - else: - # Python < 3.12, 使用手动处理方法 - print("注意: srtm库需要Python >= 3.12,使用手动处理方法") - dem_file = process_srtm_hgt_files(west, south, east, north, srtm_data_dir, - save_path, output_name) - else: - # 自动从OpenTopography下载 - print(f"\n{'='*80}") - print(f"开始下载 DEM 数据...") - print(f"{'='*80}") - print("自动从OpenTopography下载DEM数据") - dem_file = download_srtm_cgiar(west, south, east, north, save_path, - api_key=opentopo_api_key, num_workers=num_workers, - dem_type=opentopo_dem_type) - - if dem_file is None: - print("✗ DEM 生成失败,退出") - return - - current_file = dem_file - - # 生成输出文件名 - if output_name is None: - output_name = os.path.join(save_path, f"DEM_W{int(west)}_E{int(east)}_S{int(south)}_N{int(north)}") - - # 转换到 Gamma/ROI_PAC 格式 - if gamma: - print("\n开始转换到 {} 格式...".format(processor.upper())) - convert_to_gamma(current_file, output_name, gamma_byteorder, processor) - - # 转换到其他格式（非 Gamma/ROI_PAC） - if not gamma and convert: - print("\n开始转换文件格式...") - converted_file = os.path.join(save_path, f"final_DEM_W{int(west)}_E{int(east)}_S{int(south)}_N{int(north)}.tif") - convert_format(current_file, converted_file, output_format, convert_options) - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description='Download DEM data from OpenTopography and convert to Gamma/ROI_PAC format.', - formatter_class=argparse.RawTextHelpFormatter - ) - - # 与 makedem.py 相同的参数 - parser.add_argument('-r', dest='region', help='Research region, west/east/south/north (e.g., 106/110/36/40)') - parser.add_argument('-d', dest='dem', help='Raw dem file that used for further processing') - parser.add_argument('-s', dest='par', help='SLC parameter file of SAR image used for determining research region') - parser.add_argument('-p', dest='processor', help='Interferometry processor. [ gamma or roi_pac ] [default: gamma]') - parser.add_argument('-o', dest='out', help='Output name of the generated DEM') - parser.add_argument('--byteorder', dest='byteorder', choices=['big', 'little'], - help='Byteorder of the generated DEM: big or little. [default: big for gamma and little for roi_pac]') - parser.add_argument('--dir', dest='PATH', help='Processing directory for generating DEM. [default: Current directory]') - parser.add_argument('--num-workers', dest='num_workers', type=int, default=4, - help='Number of parallel download workers (used for batch processing). [default: 4]') - parser.add_argument('--srtm-data-dir', dest='srtm_data_dir', default=None, - help='Directory containing SRTM .hgt files (optional).\n' - 'If not provided, DEM data will be automatically downloaded from OpenTopography.\n' - 'Download SRTM data from: https://srtm.csi.cgiar.org/') - parser.add_argument('--opentopo-api-key', dest='opentopo_api_key', - default='09ad77d34545607fdf5cb182b64ac64e', - help='OpenTopography API key for downloading DEM data.\n' - 'Get a free API key at: https://opentopography.org/myOpenTopo\n' - 'Default key is provided, but you can use your own key.') - parser.add_argument('--opentopo-dem-type', dest='opentopo_dem_type', - choices=['SRTMGL1', 'SRTMGL3', 'NASADEM', 'COP30', 'COP90'], - default='SRTMGL1', - help='DEM type for OpenTopography download. [default: SRTMGL1]\n' - ' - SRTMGL1: SRTM GL1 30m (recommended)\n' - ' - SRTMGL3: SRTM GL3 90m\n' - ' - NASADEM: NASADEM 30m\n' - ' - COP30: Copernicus 30m\n' - ' - COP90: Copernicus 90m') - parser.add_argument('--fabdem-dir', dest='fabdem_dir', default=None, - help='Directory containing local FABDEM ZIP tiles (e.g., /mnt/ZYD/全球FABDEM).\n' - 'If provided, local FABDEM tiles will be used FIRST before network download.\n' - 'Falls back to OpenTopography download if local tiles are unavailable.') - - args = parser.parse_args() - - # 确定工作目录 - if args.PATH: - workdir = args.PATH - else: - workdir = os.getcwd() - - os.chdir(workdir) - - # 确定处理器类型 - if args.processor: - processor = args.processor - else: - processor = 'gamma' - - # 确定字节序 - if args.byteorder: - Byteorder = args.byteorder - else: - if processor == 'gamma': - Byteorder = 'big' - else: - Byteorder = 'little' - - # 确定输出名称 - if args.out: - Name = args.out - else: - Name = "out" - - # 处理已有 DEM 文件的情况 - if args.dem: - dem = args.dem - print('Raw dem file is provided: %s.' % dem) - - # 转换为 TIF 格式（如果不是 TIF） - SUFIX = get_sufix(dem) - if SUFIX != 'tif': - DTIF = dem.replace('.' + SUFIX, '.tif') - call_str = f'gdal_translate {dem} -of GTiff {DTIF}' - os.system(call_str) - DEM = DTIF - else: - DEM = dem - - # 转换到 Gamma/ROI_PAC 格式 - convert_to_gamma(DEM, Name, Byteorder, processor) - - BB = Byteorder + ' endian' - print('') - print('%s %s and %s are generated.' % (BB, Name + '.dem', Name + ('.dem.par' if processor == 'gamma' else '.dem.rsc'))) - print('Congratulations! Done!') - sys.exit(0) - - # 处理从 SLC 参数文件确定区域的情况 - if args.par: - Par = args.par - print("SLC_par file is provided: %s" % Par) - print("DEM over research region will be downloaded automatically based on %s" % Par) - print("DEM data will be downloaded from OpenTopography") - call_str = "SLC_corners " + Par + " > corners.txt" - os.system(call_str) - - File = open("corners.txt", "r") - InfoLine = File.readlines()[8:10] - File.close() - - MinLat = float(InfoLine[0].split(':')[1].split(' max. ')[0]) - MaxLat = float(InfoLine[0].split(':')[2]) - MinLon = float(InfoLine[1].split(':')[1].split(' max. ')[0]) - MaxLon = float(InfoLine[1].split(':')[2]) - - north = int(MaxLat) + 2 - south = int(MinLat) - east = int(MaxLon) + 2 - west = int(MinLon) - elif args.region: - region = args.region - west, south, east, north = read_region(region) - else: - parser.print_usage() - sys.exit(os.path.basename(sys.argv[0]) + ': error: research region, raw_demfile and SLC parameter file, at least one is needed.') - - print('Research region: %s(west) %s(south) %s(east) %s(north)' % (west, south, east, north)) - print('>>> Ready to download DEM over research region.') - - # 处理数据（下载、合并和转换到 Gamma/ROI_PAC 格式） - process_dem_files( - save_path=workdir, - west=west, - east=east, - south=south, - north=north, - merge=True, - convert=False, - gamma=True, - gamma_byteorder=Byteorder, - processor=processor, - output_name=os.path.join(workdir, Name), - num_workers=args.num_workers, - srtm_data_dir=args.srtm_data_dir, - opentopo_api_key=args.opentopo_api_key, - opentopo_dem_type=args.opentopo_dem_type, - fabdem_dir=args.fabdem_dir - ) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/makedem_bk.py b/.codex_tmp/pyint_variants/no_rescue/pyint/makedem_bk.py deleted file mode 100644 index df12a49..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/makedem_bk.py +++ /dev/null @@ -1,302 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# -import numpy as np -import os -import sys -import getopt -import array -import argparse -from skimage import io - -from pyint import _utils as ut - -def get_sufix(STR): - n = len(STR.split('.')) - SUFIX = STR.split('.')[n-1] - - return SUFIX - -def read_region(STR): - WEST = STR.split('/')[0] - EAST = STR.split('/')[1].split('/')[0] - - SOUTH = STR.split(EAST+'/')[1].split('/')[0] - NORTH = STR.split(EAST+'/')[1].split('/')[1] - - WEST =float(WEST) - SOUTH=float(SOUTH) - EAST=float(EAST) - NORTH=float(NORTH) - return WEST,SOUTH,EAST,NORTH - - -def write_demrsc_file(FILE,Corner_LON,Corner_LAT,X_STEP,Y_STEP,WIDTH,LENGTH): - f = open(FILE,'w') - f.write('DATE12 111111-222222\n') - f.write('FILE_LENGTH ' + str(int(LENGTH)) + '\n') - f.write('FILE_TYPE .dem\n') - f.write('PROCESSOR roipac\n') - f.write('PROJECTION LATLON\n') - f.write('RLOOKS 1\n') - f.write('WIDTH ' + str(int(WIDTH)) + '\n') - f.write('XMAX ' + str(int(int(WIDTH)-1)) + '\n') - f.write('XMIN 0\n') - f.write('X_FIRST ' + str(float(Corner_LON)) + '\n') - f.write('X_STEP ' + str(float(X_STEP)) + '\n') - f.write('X_UNIT degrees\n') - f.write('YMAX ' + str(int(int(LENGTH)-1)) + '\n') - f.write('YMIN 0\n') - f.write('Y_FIRST ' + str(float(Corner_LAT)) + '\n') - f.write('Y_STEP ' + str(float(Y_STEP)) + '\n') - f.write('Y_UNIT degrees\n') - f.write('Z_OFFSET 0\n') - f.write('Z_SCALE 1\n') - f.close - - -def write_dempar_file(FILE,Corner_LON,Corner_LAT,X_STEP,Y_STEP,WIDTH,LENGTH,DATA_FORMAT): - DEM_TYPE = 'SRTM1' - Proj = 'EQA' - f=open(FILE,'w') - f.write("Gamma DIFF&GEO DEM/MAP parameter file\n") - f.write("title:\tIMPORTED DEM FROM %s\n" % DEM_TYPE) # SRTM1 (30m) or SRTM3 (90m) - f.write("DEM_projection: %s\n" % Proj) # Projection should be checked. - f.write("data_format: %s\n" % DATA_FORMAT) # INTEGER*2 OR REAL*4 should be modified - f.write("DEM_hgt_offset: 0.00000\n") - f.write("DEM_scale: 1.00000\n") - f.write("width: %s\n" % WIDTH) - f.write("nlines: %s\n" % LENGTH) - f.write("corner_lat: %s decimal degrees\n" % Corner_LAT) - f.write("corner_lon: %s decimal degrees\n" % Corner_LON) - f.write("post_lat: %s decimal degrees\n" % Y_STEP) - f.write("post_lon: %s decimal degrees\n" % X_STEP) - f.write("\n") - f.write("ellipsoid_name: WGS 84\n") - f.write("ellipsoid_ra: 6378137.000 m\n") - f.write("ellipsoid_reciprocal_flattening: 298.2572236\n") - f.write("\n") - f.write("datum_name: WGS 1984\n") - f.write("datum_shift_dx: 0.000 m\n") - f.write("datum_shift_dy: 0.000 m\n") - f.write("datum_shift_dz: 0.000 m\n") - f.write("datum_scale_m: 0.00000e+00\n") - f.write("datum_rotation_alpha: 0.00000e+00 arc-sec\n") - f.write("datum_rotation_beta: 0.00000e+00 arc-sec\n") - f.write("datum_rotation_gamma: 0.00000e+00 arc-sec\n") - f.write("datum_country_list Global Definition, WGS84, World\n") - f.write("\n") - - f.close() - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close() - -######################################################################### - -INTRODUCTION = ''' -############################################################################# - Copy Right(c): 2017-2019, Yunmeng Cao [ymcmrs@gmail.com] - - Generating DEM used in interferometry both for GAMMA and ROI_PAC processor. - - 1) Available raw DEM files can be used, e.g., dem.tif, dem.grd; - 2) If no raw DEM is provided, SRTM-1 (30m) can be downloaded automatically. - - Requirement: - Python 2.7 or higher version. GDAL should be installed in your PC. -''' - -EXAMPLE = ''' - Usage: - makedem.py -r west/east/south/north -d raw_demfile -p processor -o output - makedem.py -r west/east/south/north -p processor - makedem.py -d raw_demfile --byteorder - - Examples: - makedem.py -r " -118/-116/33/34 " -p gamma -o SouthCalifornia - makedem.py -r " -118/-116/33/34 " --byteorder little - makedem.py -d dem.tif -p roi_pac --byteorder big - makedem.py -s 20101108.slc.par -p roi_pac -############################################################################## -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Generate DEM for interferometry processing.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - #parser.add_argument('-r','--lalo', dest='region', nargs=4, type=float, help='lalo limit') - parser.add_argument('-r',dest = 'region',help='Research region, west/east/south/north.') - parser.add_argument('-d', dest='dem', help='Raw dem file that used for further processing.') - parser.add_argument('-s', dest='par', help='SLC parameter file of SAR image used for determining research region.') - parser.add_argument('-p', dest='processor', help='Interferometry processor. [ gamma or roi_pac ] [default: gamma]') - parser.add_argument('-o', dest='out', help='Output name of the generated DEM.') - parser.add_argument('--byteorder', dest='byteorder', help='Byteorder of the generated DEM: big or little. [default: big for gamma and little for roi_pac]') - parser.add_argument('--dir', dest='PATH', help='Processing directory for generating DEM. [default: Current directory]') - - inps = parser.parse_args() - - if not inps.region and not inps.dem and not inps.par: - parser.print_usage() - sys.exit(os.path.basename(sys.argv[0])+': error: research region, raw_demfile and SLC parameter file, at least one is needed.') - - return inps - -################################################################################ - -def main(argv): - - inps = cmdLineParse() - - if inps.par: - Par = inps.par - print("SLC_par file is provided: %s" % Par) - print("SRTM1 over research region will be downloaded automatically based on %s" % Par) - call_str = "SLC_corners "+ Par + " > corners.txt" - os.system(call_str) - - File = open("corners.txt","r") - InfoLine = File.readlines()[8:10] - File.close() - - MinLat = float(InfoLine[0].split(':')[1].split(' max. ')[0]) - MaxLat = float(InfoLine[0].split(':')[2]) - MinLon = float(InfoLine[1].split(':')[1].split(' max. ')[0]) - MaxLon = float(InfoLine[1].split(':')[2]) - - north = int(MaxLat) + 2 - south = int(MinLat) - east = int(MaxLon) + 2 - west = int(MinLon) - - - if inps.region: - region = inps.region - west,south,east,north = read_region(region) - - #print 'Research region: %s/%s/%s/%s' % west,south,east,north - - if inps.dem: - dem = inps.dem - print('Raw dem file is provided: %s .' % dem) - - if inps.out: Name = inps.out - else: Name = "out" - - if inps.processor: processor = inps.processor - else: processor ='gamma' - - if inps.byteorder: Byteorder = inps.byteorder - else: - if processor == 'gamma': Byteorder = 'big' - else: Byteorder ='little' - - if inps.PATH: workdir = inps.PATH - else: workdir = os.getcwd() - os.chdir(workdir) - - if not inps.dem: - - print('Research region: %s(west) %s(south) %s(east) %s(north)' % (west,south,east,north)) - print('>>> Ready to download SRTM1 dem over research region.') - #call_str='wget -q -O dem.tif "http://ot-data1.sdsc.edu:9090/otr/getdem?north=%f&south=%f&east=%f&west=%f&demtype=SRTMGL1"' % (north,south,east,west) - #os.system(call_str) - #call_str = 'eio --product SRTM1 clip -o dem.tif --bounds ' + str(west) + ' ' + str(south) + ' ' + str(east) + ' ' + str(north) - call_str = 'sardem --bbox ' + str(west) + ' ' + str(south) + ' ' + str(east) + ' ' + str(north) + ' -o ' + Name + '.dem' - os.system(call_str) - print('>>> DEM download finished.') - - DEM_TYPE='SRTM1' - Proj = 'EQA' - - DEM = 'dem.tif' - call_str = 'gdal_translate ' + Name + '.dem' + ' -of GTiff ' + DEM - os.system(call_str) - else: - DEM = inps.dem - - - - SUFIX = get_sufix(DEM) - SS ='.' + SUFIX - DTIF = DEM.replace(SS,'.tif') - - if not SUFIX == 'tif': - call_str = 'gdal_translate ' + DEM + ' -of GTiff ' + DTIF - os.system(call_str) - - - DEM = DTIF - call_str = 'gdalinfo ' + DEM + ' >ttt' - os.system(call_str) - - f = open('ttt') - for line in f: - if 'Origin =' in line: - STR1 = line - AA = STR1.split('Origin =')[1] - Corner_LON = AA.split('(')[1].split(',')[0] - Corner_LAT = AA.split('(')[1].split(',')[1].split(')')[0] - elif 'Pixel Size ' in line: - STR2 = line - AA = STR2.split('Pixel Size =')[1] - Post_LON = AA.split('(')[1].split(',')[0] - Post_LAT = AA.split('(')[1].split(',')[1].split(')')[0] - - elif 'Size is' in line: - STR3 = line - AA =STR3.split('Size is')[1] - WIDTH = AA.split(',')[0] - FILE_LENGTH = AA.split(',')[1] - f.close() - - dem_data = io.imread(DEM) - if dem_data.dtype=='float32': - DATA_FORMAT='REAL*4' - else: - DATA_FORMAT='INTEGER*2' - - - if not sys.byteorder == Byteorder: - dem_data.byteswap(True) - - if processor =='gamma': - DEMDATA = Name + '.dem' - DEMPAR = Name + '.dem.par' - elif processor =='roi_pac': - DEMDATA = Name + '.dem' - DEMPAR = Name + '.dem.rsc' - - dem_data.tofile(DEMDATA) - - if processor =='gamma': - write_dempar_file(DEMPAR,Corner_LON,Corner_LAT,Post_LON,Post_LAT,WIDTH,FILE_LENGTH,DATA_FORMAT) - elif processor =='roi_pac': - write_demrsc_file(DEMPAR,Corner_LON,Corner_LAT,Post_LON,Post_LAT,WIDTH,FILE_LENGTH) - - BB =Byteorder + ' endian' - print('') - print('%s %s and %s are generated.' % (BB,DEMDATA,DEMPAR)) - print('Congratulations! Done!') - #print "Generating %s processor %s and %s is done!" % (processor,DEMDATA,DEMPAR) - sys.exit(1) - - -if __name__ == '__main__': - main(sys.argv[1:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/makedem_pyint.py b/.codex_tmp/pyint_variants/no_rescue/pyint/makedem_pyint.py deleted file mode 100644 index 1705b07..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/makedem_pyint.py +++ /dev/null @@ -1,203 +0,0 @@ -#! /usr/bin/env python -from __future__ import annotations - -import argparse -import os -import subprocess -import sys -from pathlib import Path - -from pyint import _utils as ut - - -INTRODUCTION = """ -------------------------------------------------------------------- - - Generate radar-coordinates based DEM. - [Geo-coordinates DEM can be downloaded automatically if not provided.] -""" - -EXAMPLE = """Usage: - - makedem_pyint.py projectName --processor gamma -""" - - -def cmdLineParse() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Generate radar-coordinates based DEM.", - formatter_class=argparse.RawTextHelpFormatter, - epilog=INTRODUCTION + "\n" + EXAMPLE, - ) - parser.add_argument("projectName", help="Name of project.") - parser.add_argument( - "-p", - "--processor", - dest="processor", - choices={"gamma", "roi_pac"}, - default="gamma", - help="Interferometry processor. [default: gamma]", - ) - return parser.parse_args() - - -def _run_checked(command: list[str], *, cwd: str | None = None) -> subprocess.CompletedProcess[str]: - result = subprocess.run( - command, - cwd=cwd, - text=True, - capture_output=True, - check=False, - ) - if result.returncode != 0: - detail = (result.stderr or result.stdout or "").strip() - raise RuntimeError(f"Command failed ({result.returncode}): {' '.join(command)}\n{detail}") - return result - - -def _resolve_existing_dem_open_path(source_dem: str) -> Path: - source_path = Path(source_dem) - if source_path.is_file() and source_path.suffix.lower() == ".vrt": - return source_path - - vrt_path = Path(str(source_path) + ".vrt") - if vrt_path.is_file(): - return vrt_path - - if source_path.is_file(): - return source_path - - raise FileNotFoundError(f"Prepared DEM source does not exist: {source_dem}") - - -def _resolve_research_bbox_from_slc_par(slc_par: str) -> tuple[int, int, int, int]: - result = _run_checked(["SLC_corners", slc_par]) - lines = result.stdout.splitlines() - if len(lines) < 10: - raise RuntimeError(f"Unexpected SLC_corners output for {slc_par}") - - lat_line = lines[8].rstrip() - lon_line = lines[9].rstrip() - min_lat = float(lat_line.split(":")[1].split(" max. ")[0]) - max_lat = float(lat_line.split(":")[2]) - min_lon = float(lon_line.split(":")[1].split(" max. ")[0]) - max_lon = float(lon_line.split(":")[2]) - - north = int(max_lat) + 2 - south = int(min_lat) - east = int(max_lon) + 2 - west = int(min_lon) - return west, south, east, north - - -def _cleanup_temp_outputs(paths: list[Path]) -> None: - for path in paths: - try: - if path.exists(): - path.unlink() - except OSError: - continue - - -def _build_dem_from_existing_source( - *, - project_name: str, - processor: str, - slc_par: str, - source_dem: str, - work_dir: Path, -) -> None: - west, south, east, north = _resolve_research_bbox_from_slc_par(slc_par) - source_open_path = _resolve_existing_dem_open_path(source_dem) - clipped_tif = work_dir / f"{project_name}.prepared_source_clip.tif" - clipped_aux = Path(str(clipped_tif) + ".aux.xml") - - print(f"Using prepared DEM source: {source_dem}") - print(f"Clipping prepared DEM window: west={west}, south={south}, east={east}, north={north}") - - _run_checked( - [ - "gdal_translate", - "-projwin", - str(west), - str(north), - str(east), - str(south), - "-of", - "GTiff", - str(source_open_path), - str(clipped_tif), - ], - cwd=str(work_dir), - ) - _run_checked( - [ - "makedem.py", - "-d", - str(clipped_tif), - "-p", - processor, - "-o", - project_name, - ], - cwd=str(work_dir), - ) - _cleanup_temp_outputs([clipped_tif, clipped_aux]) - - -def main(argv: list[str]) -> None: - inps = cmdLineParse() - projectName = inps.projectName - processor = inps.processor - - scratchDir = os.getenv("SCRATCHDIR") - slcDir = scratchDir + "/" + projectName + "/SLC" - templateDir = os.getenv("TEMPLATEDIR") - templateFile = templateDir + "/" + projectName + ".template" - templateDict = ut.update_template(templateFile) - - masterDate = templateDict["masterDate"] - SLC_PAR = slcDir + "/" + masterDate + "/" + masterDate + ".slc.par" - - demDir = os.getenv("DEMDIR") - demDir1 = demDir + "/" + projectName - if not os.path.isdir(demDir1): - os.mkdir(demDir1) - - os.chdir(demDir1) - work_dir = Path(demDir1) - - prepared_dem_source = str(templateDict.get("prepared_dem_source", "") or "").strip() - if prepared_dem_source not in {"", "-"}: - _build_dem_from_existing_source( - project_name=projectName, - processor=processor, - slc_par=SLC_PAR, - source_dem=prepared_dem_source, - work_dir=work_dir, - ) - print(f"Generate DEM for project {projectName} is done.") - sys.exit(0) - - call_str = "makedem.py " + "-s " + SLC_PAR + " -p gamma " + " -o " + projectName - - if "fabdem_dir" in templateDict and templateDict["fabdem_dir"].strip() not in ["", "-"]: - fabdem_dir = templateDict["fabdem_dir"].strip() - if os.path.isdir(fabdem_dir): - call_str += " --fabdem-dir " + fabdem_dir - print("Using local FABDEM directory: %s" % fabdem_dir) - - if "opentopo_api_key" in templateDict and templateDict["opentopo_api_key"].strip() not in ["", "-"]: - call_str += " --opentopo-api-key " + templateDict["opentopo_api_key"].strip() - if "opentopo_dem_type" in templateDict and templateDict["opentopo_dem_type"].strip() not in ["", "-"]: - call_str += " --opentopo-dem-type " + templateDict["opentopo_dem_type"].strip() - - print("Running: %s" % call_str) - os.system(call_str) - - print("Generate DEM for project %s is done." % projectName) - sys.exit(0) - - -if __name__ == "__main__": - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/mintpy_extract_timeseries_to_geptiff.py b/.codex_tmp/pyint_variants/no_rescue/pyint/mintpy_extract_timeseries_to_geptiff.py deleted file mode 100644 index 688b713..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/mintpy_extract_timeseries_to_geptiff.py +++ /dev/null @@ -1,167 +0,0 @@ -#!/usr/bin/env python3 -""" -SBAS-InSAR时间序列数据提取脚本 -从H5文件中提取时间序列数据、日期和基线信息 -""" - -import h5py -import numpy as np -import os -import argparse -from osgeo import gdal - - -def save_as_geotiff(data, output_file, nodata=-9999): - """ - 将数据保存为GeoTIFF格式 - - Parameters: - ----------- - data : numpy.ndarray - 要保存的数据（2D数组） - output_file : str - 输出文件路径 - nodata : float - 无效值 - """ - # 设置无效值 - data_masked = np.where(np.isnan(data), nodata, data) - - # 获取数据形状 - rows, cols = data_masked.shape - - # 创建GeoTIFF驱动 - driver = gdal.GetDriverByName('GTiff') - - # 创建数据集 - dataset = driver.Create(output_file, cols, rows, 1, gdal.GDT_Float32) - - # 设置波段 - band = dataset.GetRasterBand(1) - band.WriteArray(data_masked) - band.SetNoDataValue(nodata) - - # 设置无数据值 - band.FlushCache() - - # 关闭数据集 - dataset = None - - print(f" 已保存: {output_file}") - - -def extract_timeseries(h5_file, output_dir='./output'): - """ - 提取SBAS-InSAR时间序列数据 - - Parameters: - ----------- - h5_file : str - H5文件路径 - output_dir : str - 输出目录 - """ - - # 创建输出目录 - os.makedirs(output_dir, exist_ok=True) - - # 读取H5文件 - print(f"正在读取文件: {h5_file}") - with h5py.File(h5_file, 'r') as f: - # 提取日期信息 - dates = f['date'][:] - # 将bytes转为字符串 - dates_str = [d.decode('utf-8') if isinstance(d, bytes) else d for d in dates] - print(f"时间序列长度: {len(dates_str)}") - print(f"日期范围: {dates_str[0]} 到 {dates_str[-1]}") - - # 提取基线信息 - bperp = f['bperp'][:] - print(f"基线形状: {bperp.shape}") - - # 提取时间序列数据 - timeseries = f['timeseries'][:] - print(f"时间序列数据形状: {timeseries.shape} (时间, 行, 列)") - - # 保存日期信息 - date_file = os.path.join(output_dir, 'dates.txt') - with open(date_file, 'w') as f_out: - for i, date in enumerate(dates_str): - f_out.write(f"{i+1} {date}\n") - print(f"日期信息已保存到: {date_file}") - - # 保存基线信息 - bperp_file = os.path.join(output_dir, 'bperp.npy') - np.save(bperp_file, bperp) - print(f"基线信息已保存到: {bperp_file}") - - # 保存时间序列数据（numpy格式） - timeseries_file = os.path.join(output_dir, 'timeseries.npy') - np.save(timeseries_file, timeseries) - print(f"时间序列数据已保存到: {timeseries_file}") - - # 保存每个时间点为单独的GeoTIFF文件 - print("\n正在保存每个时间点的GeoTIFF文件...") - geotiff_dir = os.path.join(output_dir, 'geotiff') - os.makedirs(geotiff_dir, exist_ok=True) - - for i, date in enumerate(dates_str): - # 保存为GeoTIFF - geotiff_file = os.path.join(geotiff_dir, f'timeseries_{date}.tif') - save_as_geotiff(timeseries[i], geotiff_file) - print(f"已保存 {len(dates_str)} 个时间点的GeoTIFF文件") - - # 计算并保存累积形变 - print("\n计算累积形变...") - cumulative = np.cumsum(timeseries, axis=0) - cumulative_file = os.path.join(output_dir, 'cumulative_deformation.npy') - np.save(cumulative_file, cumulative) - print(f"累积形变已保存到: {cumulative_file}") - - # 保存累积形变为GeoTIFF - print("\n正在保存累积形变GeoTIFF文件...") - for i, date in enumerate(dates_str): - cumulative_file = os.path.join(geotiff_dir, f'cumulative_{date}.tif') - save_as_geotiff(cumulative[i], cumulative_file) - print(f"已保存 {len(dates_str)} 个累积形变GeoTIFF文件") - - # 保存统计信息 - stats_file = os.path.join(output_dir, 'statistics.txt') - with open(stats_file, 'w') as f_out: - f_out.write("SBAS-InSAR时间序列统计信息\n") - f_out.write("=" * 50 + "\n\n") - f_out.write(f"总时间点数: {len(dates_str)}\n") - f_out.write(f"影像尺寸: {timeseries.shape[1]} 行 x {timeseries.shape[2]} 列\n") - f_out.write(f"日期范围: {dates_str[0]} 到 {dates_str[-1]}\n\n") - f_out.write("形变统计 (单位: mm):\n") - f_out.write(f" 最小值: {np.nanmin(timeseries):.4f}\n") - f_out.write(f" 最大值: {np.nanmax(timeseries):.4f}\n") - f_out.write(f" 平均值: {np.nanmean(timeseries):.4f}\n") - f_out.write(f" 标准差: {np.nanstd(timeseries):.4f}\n") - print(f"统计信息已保存到: {stats_file}") - - print("\n数据提取完成！") - print(f"所有数据已保存到目录: {output_dir}") - - -if __name__ == '__main__': - # 解析命令行参数 - parser = argparse.ArgumentParser( - description='从SBAS-InSAR的H5文件中提取时间序列数据并保存为GeoTIFF格式' - ) - parser.add_argument( - 'h5_file', - type=str, - help='输入的H5文件路径' - ) - parser.add_argument( - '-o', '--output', - type=str, - default='./extracted_data', - help='输出目录路径 (默认: ./extracted_data)' - ) - - args = parser.parse_args() - - # 执行提取 - extract_timeseries(args.h5_file, args.output) \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/mintpy_h5_form_utm_to_wgs84.py b/.codex_tmp/pyint_variants/no_rescue/pyint/mintpy_h5_form_utm_to_wgs84.py deleted file mode 100644 index a1bbcb5..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/mintpy_h5_form_utm_to_wgs84.py +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/env python3 -""" -将 HDF5 文件的坐标系统从 UTM 转换为 WGS84 地理坐标 -同时保持与 MintPy tsview.py 的兼容性 -""" - -import h5py -import numpy as np -import copy -import argparse -import os - -def utm_to_wgs84(easting, northing, zone, northern=True): - """ - 将 UTM 坐标转换为 WGS84 大地坐标 - """ - # UTM 参数 - a = 6378137.0 # WGS84 长半轴 - e = 0.081819190842622 # WGS84 第一偏心率 - k0 = 0.9996 # 比例因子 - - # 中央经线 - lon0 = (zone - 1) * 6 - 180 + 3 - - # 调整东伪偏移 - x = easting - 500000.0 - - # 如果是南半球，调整北伪偏移 - if not northern: - y = northing - 10000000.0 - else: - y = northing - - # M = y / k0 - mu = y / (a * (1 - e**2 / 4 - 3 * e**4 / 64 - 5 * e**6 / 256) * k0) - - e1 = (1 - np.sqrt(1 - e**2)) / (1 + np.sqrt(1 - e**2)) - - # 纬度计算 - N1 = a / np.sqrt(1 - e**2 * np.sin(mu)**2) - T1 = np.tan(mu)**2 - C1 = e1**2 * np.cos(mu)**2 - R1 = a * (1 - e**2) / (1 - e**2 * np.sin(mu)**2)**(3/2) - D = x / (N1 * k0) - - lat = mu - N1 * np.tan(mu) / R1 * (D**2 / 2 + (5 + 3 * T1 + 10 * C1 - 4 * C1**2 - 9 * e1**2) * D**4 / 24 + (61 + 90 * T1 + 298 * C1 + 45 * T1**2 - 252 * e1**2 - 3 * C1**2) * D**6 / 720) - - # 经度计算 - lon = lon0 * np.pi / 180 + D * (1 + (1 + 3 * e1**2 + 2 * e1**4) * D**2 / 6 + (2 - e1**2) * D**4 / 120) / np.cos(mu) - - # 转换为角度 - lat_deg = np.degrees(lat) - lon_deg = np.degrees(lon) - - return lon_deg, lat_deg - - -def main(): - parser = argparse.ArgumentParser( - description='将 HDF5 文件转换为 WGS84 地理坐标（与 MintPy 兼容）', - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=''' -示例: - python convert_to_wgs84.py input.h5 -o output.h5 - -说明: - 脚本会将文件完全转换为 WGS84 地理坐标系统， - 同时设置正确的属性以保持与 MintPy 的兼容性。 - ''' - ) - parser.add_argument('input', help='输入 HDF5 文件路径') - parser.add_argument('-o', '--output', help='输出 HDF5 文件路径（默认：输入文件名_WGS84.h5）') - - args = parser.parse_args() - - # 输入和输出文件 - input_file = args.input - - if args.output: - output_file = args.output - else: - # 自动生成输出文件名 - base, ext = os.path.splitext(input_file) - output_file = f"{base}_WGS84{ext}" - - print(f"正在读取文件: {input_file}") - f_in = h5py.File(input_file, 'r') - - # 读取所有数据集 - datasets = {} - for key in f_in.keys(): - datasets[key] = f_in[key][:] - print(f" 读取数据集: {key}, 形状: {datasets[key].shape}") - - # 读取所有属性 - attrs = dict(f_in.attrs) - print(f"\n共有 {len(attrs)} 个属性") - - # 解析 UTM 坐标信息 - x_first = float(attrs['X_FIRST']) - y_first = float(attrs['Y_FIRST']) - x_step = float(attrs['X_STEP']) - y_step = float(attrs['Y_STEP']) - width = int(attrs['WIDTH']) - length = int(attrs['LENGTH']) - utm_zone = attrs['UTM_ZONE'] - - # 获取参考点位置（像素坐标） - ref_x = int(attrs.get('REF_X', 0)) - ref_y = int(attrs.get('REF_Y', 0)) - - # 转换 UTM Zone - zone_num = int(''.join(filter(str.isdigit, utm_zone))) - is_northern = 'N' in utm_zone.upper() - - print(f"\n原始坐标系统:") - print(f" UTM Zone: {utm_zone} (EPSG: {attrs['EPSG']})") - print(f" 左上角 UTM: ({x_first}, {y_first})") - print(f" 步长: X={x_step}m, Y={y_step}m") - print(f" 参考点像素: ({ref_x}, {ref_y})") - - # 生成网格坐标 - x_coords = x_first + np.arange(width) * x_step - y_coords = y_first + np.arange(length) * y_step - - # 转换为 WGS84 - print(f"\n正在转换为 WGS84...") - lon_grid, lat_grid = np.meshgrid(x_coords, y_coords) - wgs84_lon, wgs84_lat = utm_to_wgs84(lon_grid, lat_grid, zone_num, is_northern) - - # 计算新的坐标参数（注意：lon 是经度，lat 是纬度） - # X 对应经度，Y 对应纬度 - lon_first = wgs84_lon[0, 0] - lat_first = wgs84_lat[0, 0] - lon_step = abs(wgs84_lon[0, 1] - wgs84_lon[0, 0]) if width > 1 else 0 - lat_step = abs(wgs84_lat[1, 0] - wgs84_lat[0, 0]) if length > 1 else 0 - - # 计算参考点的经纬度 - # REF_X 对应 X 轴（经度），REF_Y 对应 Y 轴（纬度） - ref_lon = wgs84_lon[ref_y, ref_x] # 经度 - ref_lat = wgs84_lat[ref_y, ref_x] # 纬度 - - # 更新属性 - 完全转换为 WGS84 - attrs_new = copy.deepcopy(attrs) - - # 更新坐标系统相关属性 - attrs_new['EPSG'] = 4326 # WGS84 - attrs_new['X_FIRST'] = lon_first # 经度 - attrs_new['Y_FIRST'] = lat_first # 纬度 - attrs_new['X_STEP'] = lon_step - attrs_new['Y_STEP'] = -lat_step # 保持Y方向向下为负 - attrs_new['X_UNIT'] = 'degrees' - attrs_new['Y_UNIT'] = 'degrees' - - # 更新参考坐标 - 现在是真正的经纬度 - attrs_new['REF_LON'] = ref_lon - attrs_new['REF_LAT'] = ref_lat - - # 保留原始 UTM 信息作为备份（带 _ORIG 后缀） - attrs_new['UTM_ZONE_ORIG'] = attrs_new.get('UTM_ZONE', '') - attrs_new['EPSG_ORIG'] = attrs_new.get('EPSG', '') - - # 关键：移除或清空 UTM_ZONE 属性，让 MintPy 识别为地理坐标系统 - if 'UTM_ZONE' in attrs_new: - del attrs_new['UTM_ZONE'] - - # 更新角点坐标 - attrs_new['LAT_REF1'] = lat_first - attrs_new['LAT_REF2'] = wgs84_lat[0, -1] - attrs_new['LAT_REF3'] = wgs84_lat[-1, 0] - attrs_new['LAT_REF4'] = wgs84_lat[-1, -1] - attrs_new['LON_REF1'] = lon_first - attrs_new['LON_REF2'] = wgs84_lon[0, -1] - attrs_new['LON_REF3'] = wgs84_lon[-1, 0] - attrs_new['LON_REF4'] = wgs84_lon[-1, -1] - - # 设置其他相关属性 - attrs_new['COORD_SYSTEM'] = 'GEO' # 地理坐标系统 - - print(f"\n新坐标系统 (WGS84):") - print(f" EPSG: {attrs_new['EPSG']}") - print(f" 左上角: ({attrs_new['X_FIRST']:.6f}, {attrs_new['Y_FIRST']:.6f})") - print(f" 步长: X={attrs_new['X_STEP']:.6f}°, Y={attrs_new['Y_STEP']:.6f}°") - print(f" 参考点经纬度: ({attrs_new['REF_LON']:.6f}, {attrs_new['REF_LAT']:.6f})") - print(f" 参考点像素: ({ref_x}, {ref_y})") - - # 创建新的 HDF5 文件 - print(f"\n正在写入文件: {output_file}") - f_out = h5py.File(output_file, 'w') - - # 写入所有数据集 - for key, data in datasets.items(): - f_out.create_dataset(key, data=data) - print(f" 写入数据集: {key}") - - # 写入所有属性 - for key, value in attrs_new.items(): - f_out.attrs[key] = value - - f_in.close() - f_out.close() - - print(f"\n完成！转换后的文件已保存为: {output_file}") - print("\n说明:") - print(" - 文件已完全转换为 WGS84 地理坐标系统") - print(" - 所有坐标属性已更新为经纬度") - print(" - 原始 UTM 信息已保存为 _ORIG 后缀的属性") - print(" - 应该与 MintPy tsview.py 兼容") - - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/mintpy_ssa.py b/.codex_tmp/pyint_variants/no_rescue/pyint/mintpy_ssa.py deleted file mode 100644 index 6f92b0a..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/mintpy_ssa.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python -import argparse -import sys -import h5py -import numpy as np -from ssa import SSA -import time, math -from mintpy.utils import readfile, writefile, ptime, utils as ut -from mintpy.objects import timeseries - -def create_parser(): - parser=argparse.ArgumentParser() - parser.add_argument('timeseries_file', - help='time-series file to do SSA \n' - 'i.e.:geo_timeseries_GACOS_ramp_demErr.h5 (MintPy)\n') - parser.add_argument("-w", "--window", - type=int, - help='Widow length (default is 7) ') - parser.add_argument('-o', '--output', dest='outfile', help='output file name (default ssa.h5)') - # args=parser.parse_args() - # computing - # parser = arg_group.add_memory_argument(parser) - return parser -def cmd_line_parse(iargs=None): - parser = create_parser() - inps = parser.parse_args(args=iargs) - inps.key = readfile.read_attribute(inps.timeseries_file)['FILE_TYPE'] - if inps.key not in ['timeseries', 'giantTimeseries', 'HDFEOS']: - raise Exception('input file is {}, NOT timeseries!'.format(inps.key)) - if not inps.window: - inps.window = int(7) - if not inps.outfile: - outname = 'ssa' - outname += '.h5' - inps.outfile = outname - tsobj = timeseries(inps.timeseries_file) - return inps -def is_number(s): - if s==0: - return False - elif math.isnan(s): - return False - else: - return True -def main(iargs=None): - # if cmd: - # iargs = cmd.split()[1:] - inps = cmd_line_parse(iargs) - date_list = timeseries(inps.timeseries_file).get_date_list() - data, atr = readfile.read(inps.timeseries_file) - length, width = int(atr['LENGTH']), int(atr['WIDTH']) - N_dates=len(date_list) - # with h5py.File(inps.timeseries_file,'r') as f: - # data = f['timeseries'][:] - # date_list = f['date'][:] - data_matrix = np.empty((len(date_list),length,width)) - data_matrix[:] = np.nan - start_time = time.time() - for i in range(length): - for j in range(width): - if is_number(sum(data[:,i,j])): - a=SSA(data[:,i,j], inps.window) - data_matrix[:,i,j]=a.reconstruct([0,1]) # Grouping recostructed components, by defaults it takes F1 and F2 as trend - m, s = divmod(time.time()-start_time, 60) - block= [0,N_dates,0,length,0,width] - writefile.layout_hdf5(inps.outfile, metadata=atr, ref_file=inps.timeseries_file) - writefile.write_hdf5_block(inps.outfile, - data=data_matrix, - datasetName='timeseries', - block=block, - print_msg=False) - - - print(inps.timeseries_file) - print(inps.window) - # print(date_list) - print(length,width) - print('time used: {:02.0f} mins {:02.1f} secs.'.format(m, s)) -if __name__ == '__main__': - main(sys.argv[1:]) - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/operation.csh b/.codex_tmp/pyint_variants/no_rescue/pyint/operation.csh deleted file mode 100644 index 691259d..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/operation.csh +++ /dev/null @@ -1,126 +0,0 @@ -#!/bin/csh -f - -if ($#argv != 7) then - echo "" - echo "Usage: operation.csh master_ztd master_ztd.rsc slave_ztd slave_ztd.rsc reference_point incidence_angle dem_grid" - echo " Performs gacos correction over the phasefilt.grd files for Sentinel 1" - echo "" - echo "It works in each directory where the phase grids are located. This script works jointly with GACOS_correction.csh" - echo "" - echo "Reference point in radar coordinates (text file)" - echo "" - echo "Indicence angle in degrees from SAT_look (float/integer)" - echo "" - exit 1 - endif - -#reference point/stable point in radar coordinates -set reference_point = $5 - -#incidence angle obtained with the "SAT_look" script in degrees -set incidence = $6 - -#wavelength for Sentinel 1 (m) -set wavelength = 0.0554658 -set pi = 3.141592653589793238462 - -#######FIRST ZTD to grid######### -set x_first_d1 = `cat $2|grep X_FIRST|awk '{print $2}'` -set y_first_d1 = `cat $2|grep Y_FIRST|awk '{print $2}'` -set width_d1 = `cat $2|grep WIDTH|awk '{print $2}'` -set length_d1 = `cat $2|grep FILE_LENGTH|awk '{print $2}'` -set x_step_d1 = `cat $2|grep X_STEP|awk '{print $2}'` -set y_step_d1 = `cat $2|grep X_STEP|awk '{print $2}'` -set date_ztd_d1 = `echo $1|awk -F/ '{print $NF}'|cut -c1-8` -gmt xyz2grd $1 -G"date1_ztd.grd" -RLT$x_first_d1/$y_first_d1/$width_d1/$length_d1 -I$x_step_d1/$y_step_d1 -ZTLf -di0 -r - -#######SECOND ZTD to grid######### -set x_first_d2 = `cat $4|grep X_FIRST|awk '{print $2}'` -set y_first_d2 = `cat $4|grep Y_FIRST|awk '{print $2}'` -set width_d2 = `cat $4|grep WIDTH|awk '{print $2}'` -set length_d2 = `cat $4|grep FILE_LENGTH|awk '{print $2}'` -set x_step_d2 = `cat $4|grep X_STEP|awk '{print $2}'` -set y_step_d2 = `cat $4|grep X_STEP|awk '{print $2}'` -set date_ztd_d2 = `echo $3|awk -F/ '{print $NF}'|cut -c1-8` -gmt xyz2grd $3 -G"date2_ztd.grd" -RLT$x_first_d2/$y_first_d2/$width_d2/$length_d2 -I$x_step_d2/$y_step_d2 -ZTLf -di0 -r - -#TIME DIFFERENCE -gmt grdmath date2_ztd.grd date1_ztd.grd SUB = zpddm.grd - -#Checking GACOS grids within the DEM boundaries -set xmin_dem = `gmt grdinfo -C $7|awk '{print $2}'` -set xmax_dem = `gmt grdinfo -C $7|awk '{print $3}'` -set ymin_dem = `gmt grdinfo -C $7|awk '{print $4}'` -set ymax_dem = `gmt grdinfo -C $7|awk '{print $5}'` - -set xmin_gacos = `gmt grdinfo -C zpddm.grd|awk '{print $2}'` -set xmax_gacos = `gmt grdinfo -C zpddm.grd|awk '{print $3}'` -set ymin_gacos = `gmt grdinfo -C zpddm.grd|awk '{print $4}'` -set ymax_gacos = `gmt grdinfo -C zpddm.grd|awk '{print $5}'` - -set check = `echo "$xmin_dem < $xmin_gacos && $ymin_dem < $ymin_gacos && $xmax_dem > $xmax_gacos && $ymax_dem > $ymax_gacos"| bc` - -if ($check == 0) then - echo "Seems like your DEM is not large enough. GACOS grids dimensions must be within the DEM." - echo "DEM dimensions: xmin: $xmin_dem xmax: $xmax_dem ymin: $ymin_dem ymax: $ymax_dem" - echo "GACOS dimensions: xmin: $xmin_gacos xmax: $xmax_gacos ymin: $ymin_gacos ymax: $ymax_gacos" - exit 1 -else - echo "GACOS grid dimensions within DEM boundaries... continue..." -endif - -#PROJECT TO RADAR COORDINATES -proj_ll2ra.csh trans.dat zpddm.grd zpddm_ra.grd - -#RESAMPLE ZTD FILES WITH UNWRAP GRID PARAMETERS -set xmin = `gmt grdinfo -C phasefilt.grd|awk '{print $2}'` -set xmax = `gmt grdinfo -C phasefilt.grd|awk '{print $3}'` -set ymin = `gmt grdinfo -C phasefilt.grd|awk '{print $4}'` -set ymax = `gmt grdinfo -C phasefilt.grd|awk '{print $5}'` -set xinc = `gmt grdinfo -C phasefilt.grd|awk '{print $8}'` -set yinc = `gmt grdinfo -C phasefilt.grd|awk '{print $9}'` -gmt grdsample zpddm_ra.grd -Gresample_zpddm.grd -R$xmin/$xmax/$ymin/$ymax -I$xinc/$yinc -r - - -#REFERENCE POINT -set ref_value = `gmt grdtrack $reference_point -Gresample_zpddm.grd -Z` -set ref_value_phase = `gmt grdtrack $reference_point -Gphasefilt.grd -Z` -gmt grdmath resample_zpddm.grd $ref_value SUB = szpddm.grd -gmt grdmath phasefilt.grd $ref_value_phase SUB = phasefilt_ref.grd - -#Checking reference point values -if ($ref_value == "" || $ref_value_phase == "") then - echo "Problems with the reference point. Is the reference point within the AOI?" - exit 1 -endif - -#FROM METER TO PHASE -#gmt grdmath szpddm.grd 4 MUL $pi MUL $wavelength DIV = szpddm_phase.grd -#multiplying by -4 instead of 4 (edition: 13/4/2023) -gmt grdmath szpddm.grd -4 MUL $pi MUL $wavelength DIV = szpddm_phase.grd - -#PROJECTION FROM ZENITH VIEW TO LOS -gmt grdmath szpddm_phase.grd $incidence COSD DIV = szpddm_phase_LOS.grd - -#CORRECTION WITH GACOS DATA -#attention: Here the phasefilt.grd is corrected, therefore for the unwrap process this output or the detrended output -#should be used. -gmt grdmath phasefilt_ref.grd szpddm_phase_LOS.grd SUB = phasefilt_GACOS_corrected.grd - -#DETRENDING -gmt grdtrend phasefilt_GACOS_corrected.grd -N3r -Dphasefilt_GACOS_corrected_detrended.grd - -#checking existence of final outputs -if !(-f phasefilt_GACOS_corrected.grd || -f phasefilt_GACOS_corrected_detrended.grd) then - echo "Seems like there was an issue correcting with GACOS $1 and $3" - exit 1 -endif - -#clean up -# -rm date1_ztd.grd date2_ztd.grd -rm zpddm.grd zpddm_ra.grd resample_zpddm.grd -rm szpddm_phase.grd -rm phasefilt_ref.grd -rm szpddm_phase_LOS.grd -echo "corrections done with $date_ztd_d1 and $date_ztd_d2 over phasefilt.grd" diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/out.dem b/.codex_tmp/pyint_variants/no_rescue/pyint/out.dem deleted file mode 100644 index bd9b847..0000000 Binary files a/.codex_tmp/pyint_variants/no_rescue/pyint/out.dem and /dev/null differ diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/out.dem.par b/.codex_tmp/pyint_variants/no_rescue/pyint/out.dem.par deleted file mode 100644 index dda0c34..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/out.dem.par +++ /dev/null @@ -1,27 +0,0 @@ -Gamma DIFF&GEO DEM/MAP parameter file -title: IMPORTED DEM FROM Copernicus30 -DEM_projection: EQA -data_format: INTEGER*2 -DEM_hgt_offset: 0.00000 -DEM_scale: 1.00000 -width: 4800 -nlines: 3600 -corner_lat: 26.000416666674390 decimal degrees -corner_lon: 99.999583333269726 decimal degrees -post_lat: -0.000833333333333 decimal degrees -post_lon: 0.000833333333333 decimal degrees - -ellipsoid_name: WGS 84 -ellipsoid_ra: 6378137.000 m -ellipsoid_reciprocal_flattening: 298.2572236 - -datum_name: WGS 1984 -datum_shift_dx: 0.000 m -datum_shift_dy: 0.000 m -datum_shift_dz: 0.000 m -datum_scale_m: 0.00000e+00 -datum_rotation_alpha: 0.00000e+00 arc-sec -datum_rotation_beta: 0.00000e+00 arc-sec -datum_rotation_gamma: 0.00000e+00 arc-sec -datum_country_list Global Definition, WGS84, World - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/phase2los.py b/.codex_tmp/pyint_variants/no_rescue/pyint/phase2los.py deleted file mode 100644 index 3cff4db..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/phase2los.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python -#! /usr/bin/env python -################################################################# -### This program is part of PyINT ### -### Copy Right (c): 2019, Wei Chen ### -### Author: Wei Chen ### -### Contact : chenweicug@126.com ### -################################################################# -import numpy as np -import os -import sys -import time -import glob -import argparse - -import subprocess -from pyint import _utils as ut - -INTRODUCTION = ''' -------------------------------------------------------------------- - convert interferogram to los displacement and get the inc angle and azi angle. - -''' - -EXAMPLE = ''' - Usage: - phase2los.py projectName Mdate Sdate - phase2los.py PacayaT163TsxHhA 20150102 20150601 -------------------------------------------------------------------- -''' - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='convert interferogram to los displacement and get the inc angle and azi angle.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('Mdate',help='Master date.') - parser.add_argument('Sdate',help='Slave date.') - - inps = parser.parse_args() - return inps - - - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - Mdate = inps.Mdate - Sdate= inps.Sdate - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - satelite = templateDict['satelite'] - rlks = templateDict['range_looks'] - - projectDir = scratchDir + '/' + projectName - ifgDir = projectDir + '/ifgrams' - simDir = scratchDir + '/' + projectName + "/DEM" - losDir = projectDir + '/LosResult' - slcDir = projectDir + '/SLC' - - - if not os.path.isdir(losDir): os.mkdir(losDir) - - Pair = Mdate + '-' + Sdate - workDir = losDir + '/' + Pair - if not os.path.isdir(workDir): os.mkdir(workDir) - workdir = ifgDir + '/' + Pair - UTMDEMpar = simDir + '/'+ Mdate + '_'+ rlks + 'rlks.utm.dem.par' - GeoUNW = workdir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.unw' - GeoATMCOR_UNW = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.atmcor.unw' - call_str = 'data2geotiff ' + UTMDEMpar + ' ' + GeoATMCOR_UNW + ' 2 ' + GeoATMCOR_UNW +'.tif' - os.system(call_str) - call_str = 'data2geotiff ' + UTMDEMpar + ' ' + GeoUNW + ' 2 ' + GeoUNW +'.tif' - os.system(call_str) - call_str = 'gdal_translate -of GSBG ' + ' ' + GeoUNW +'.tif' + ' ' + workDir + '/' + GeoUNW + '.grd' - os.system(call_str) - call_str = 'gdal_translate -of GSBG ' + ' ' + GeoATMCOR_UNW +'.tif' + ' ' + workDir + '/' + GeoATMCOR_UNW +'.grd' - os.system(call_str) - if satelite == 'CSK': - wavelength = 0.0312283810417 - elif satelite == 'TSX': - wavelength = 0.03106657823461874 - elif satelite == 'S1A': - wavelength = 0.0554657647 - elif satelite == 'ALOS2': - wavelength = 0.2424525 - elif satelite == 'ALOS': - wavelength = 0.2360571 - elif satelite == 'ENVISAT': - wavelength = 0.056 - call_str = 'grdmath ' + workDir + '/' + GeoUNW + '.grd' + str(wavelength) + ' 1 3.141592653589 MUL DIV MUL = ' + workDir + '/los.grd' - os.system(call_str) - call_str = 'grdmath ' + workDir + '/' + GeoATMCOR_UNW +'.grd' + str(wavelength) + ' 1 3.141592653589 MUL DIV MUL = ' + workDir + '/los_atmcor.grd' - os.system(call_str) - print('the los displacement of ' + Pair + " has successfully done") - print ("#####################run data2inc_azi###################") - Mslcpar = slcDir + '/' + Mdate + '/' + Mdate + '.slc.par' - UTMDEM = simDir + '/'+ Mdate + '_'+ rlks + 'rlks.utm.dem' - OFF = workdir + '/' + Pair +'_' + rlks + 'rlks.off' - os.chdir(workDir) - call_str = 'data2inc_azi ' + Mslcpar + ' ' + OFF + ' ' + UTMDEMpar + ' ' + UTMDEM - os.system(call_str) - -if __name__ == '__main__': - main(sys.argv[:]) - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/phase2los_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/phase2los_all.py deleted file mode 100644 index 38f9ad2..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/phase2los_all.py +++ /dev/null @@ -1,111 +0,0 @@ -import getopt -import time -import glob -import argparse - -import subprocess -from pyint import _utils as ut - - -def work(data0): - cmd = data0[0] - err_file = data0[1] - p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) - stdout = p.stdout - stderr = p.stderr - - if type(stderr) == bytes: - aa=stderr.decode("utf-8") - else: - aa = stderr - - if aa: - str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + cmd[3] + '\n' - #print(aa) - with open(err_file, 'a') as f: - f.write(str0) - f.write(aa) - f.write('\n') - - return -######################################################################### - -INTRODUCTION = ''' -------------------------------------------------------------------- - convert interferograms phase to los deformation for one project using GAMMA. - -''' - -EXAMPLE = ''' - Usage: - phase2los_all.py projectName - phase2los_all.py projectName --parallel 4 - diff_gamma_all.py projectName --parallel 4 --ifgramList-txt /test/ifgram_list.txt -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Geneate differential interferograms for one project using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') - parser.add_argument('--ifgarmList-txt', dest='ifgarmListTxt', help='provided ifgram_list_txt. default: using ifgram_list.txt under projectName folder.') - - inps = parser.parse_args() - return inps - - -def main(argv): - start_time = time.time() - inps = cmdLineParse() - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - projectDir = scratchDir + '/' + projectName - ifgDir = scratchDir + '/' + projectName + '/ifgrams' - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - - if inps.ifgarmListTxt: ifgramList_txt = inps.ifgarmListTxt - else: ifgramList_txt = scratchDir + '/' + projectName + '/ifgram_list.txt' - ifgList0 = ut.read_txt2array(ifgramList_txt) -# ifgList = ifgList0[:,0] - if len(ifgList0)==3: - ifgList=ifgList0[0] - ifgList=[ifgList] - - else: - ifgList=ifgList0[:,0] - - err_txt = scratchDir + '/' + projectName + '/diff_gamma_all.err' - if os.path.isfile(err_txt): os.remove(err_txt) - - data_para = [] - for i in range(len(ifgList)): - m0 = ut.yyyymmdd(ifgList[i].split('-')[0]) - print(m0) - s0 = ut.yyyymmdd(ifgList[i].split('-')[1]) - cmd0 = ['diff_gamma.py',projectName, m0, s0] - diff_file0 = ifgDir + '/' + ifgList[i] + '/' + ifgList[i] + '_' + rlks + 'rlks.diff_filt.bmp' - data0 = [cmd0,err_txt] - - k00 = 0 - if os.path.isfile(diff_file0): - if os.path.getsize(diff_file0) > 0: - k00 = 1 - if k00==0: - data_para.append(data0) - - ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) - print("Generate differential interferograms for project %s is done! " % projectName) - ut.print_process_time(start_time, time.time()) - - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/phasebias_correction_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/phasebias_correction_gamma.py deleted file mode 100644 index 51e824f..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/phasebias_correction_gamma.py +++ /dev/null @@ -1,545 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Phase Bias Correction for GAMMA format interferograms ### -### Author: ZYD / Cascade AI ### -################################################################# - -import numpy as np -import os -import sys -import time -import argparse -from pathlib import Path -from datetime import datetime -from concurrent.futures import ProcessPoolExecutor, as_completed - -import rasterio -from rasterio.transform import Affine -from rasterio.crs import CRS - -from pyint import _utils as ut - - -INTRODUCTION = ''' -------------------------------------------------------------------- - Apply phase bias correction to interferograms. - This script integrates the phase bias correction algorithm from: - InSAR_PhaseBias_Correction/ directory - - Based on the paper: - "Correcting InSAR phase bias caused by atmospheric delays - and other error sources using multiple aperture interferometry" - (Reference: 10.1016/j.remote.2022.100013) - - Algorithm Steps: - 1. Read interferogram data (PhaseBias_01_Read_Data.py) - 2. Calculate loop closures (PhaseBias_02_Loop_Closures.py) - 3. Estimate calibration parameters an (PhaseBias_03_calibration_pars.py) - 4. Inversion for phase bias terms (PhaseBias_04_Inversion.py) - 5. Apply correction (PhaseBias_05_Correction.py) - -''' - -EXAMPLE = ''' - Usage: - phasebias_correction_gamma.py projectName [options] - phasebias_correction_gamma.py PacayaT163TsxHhA --interval 12 - phasebias_correction_gamma.py PacayaT163TsxHhA --interval 12 --nlook 10 - phasebias_correction_gamma.py PacayaT163TsxHhA --interval 24 --num-a 2 -------------------------------------------------------------------- -''' - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Apply phase bias correction to interferograms.', - formatter_class=argparse.RawTextHelpFormatter, - epilog=INTRODUCTION+'\n'+EXAMPLE) - parser.add_argument('projectName', help='projectName for processing.') - parser.add_argument('--interval', type=int, default=12, - help='Data acquisition interval in days (6, 12, or 24). [default: 12]') - parser.add_argument('--nlook', type=int, default=10, - help='Number of looks for multilooking. [default: 10]') - parser.add_argument('--num-a', type=int, default=2, - help='Number of calibration parameters to estimate. [default: 2]') - parser.add_argument('--start', type=str, default=None, - help='Start date (YYYYMMDD). If not specified, will use all available data.') - parser.add_argument('--end', type=str, default=None, - help='End date (YYYYMMDD). If not specified, will use all available data.') - parser.add_argument('--estimate-an', dest='estimate_an', action='store_true', - help='Estimate calibration parameters from data instead of using defaults.') - parser.add_argument('--max-con', type=int, default=5, - help='Maximum number of connections to correct. [default: 5]') - parser.add_argument('--parallel', type=int, default=8, - help='Number of parallel workers for data conversion. [default: 8]') - parser.add_argument('--skip-convert', action='store_true', - help='Skip data conversion if GeoTIFF already exists.') - - inps = parser.parse_args() - return inps - - -def find_date_range(ifgDir, interval=12): - """Find the start and end dates from interferogram directory - - Args: - ifgDir: interferogram directory - interval: temporal baseline in days - - Returns: - tuple: (start_date, end_date) in YYYYMMDD format - """ - dates = [] - - all_dirs = sorted([d for d in os.listdir(ifgDir) if os.path.isdir(os.path.join(ifgDir, d))]) - - for dirname in all_dirs: - if '-' in dirname: - date_pair = dirname.split('-') - elif '_' in dirname: - date_pair = dirname.split('_') - else: - continue - - if len(date_pair) == 2: - try: - date1 = date_pair[0] - date2 = date_pair[1] - - d1 = datetime.strptime(date1, '%Y%m%d') - d2 = datetime.strptime(date2, '%Y%m%d') - days = abs((d2 - d1).days) - - if days == interval: - dates.extend([date1, date2]) - except: - continue - - if dates: - dates = sorted(set(dates)) - return dates[0], dates[-1] - - return None, None - - -# ===================================================================== -# GAMMA 二进制 → GeoTIFF 转换函数 -# ===================================================================== - -def parse_utm_dem_par(par_file): - """解析 GAMMA utm.dem.par 文件，提取地理编码参数""" - params = {} - with open(par_file) as f: - for line in f: - if ':' in line: - key, val = line.split(':', 1) - key = key.strip() - val = val.strip().split()[0] # 取第一个字段 - params[key] = val - return { - 'width': int(params['width']), - 'nlines': int(params['nlines']), - 'corner_lat': float(params['corner_lat']), - 'corner_lon': float(params['corner_lon']), - 'post_lat': float(params['post_lat']), - 'post_lon': float(params['post_lon']), - } - - -def write_geotiff(data, out_path, geo_info, dtype='float32', nodata=None): - """将 numpy 数组写入带地理信息的 GeoTIFF (rasterio)""" - height, width = data.shape - transform = Affine( - geo_info['post_lon'], 0, geo_info['corner_lon'], - 0, geo_info['post_lat'], geo_info['corner_lat'] - ) - profile = { - 'driver': 'GTiff', - 'dtype': dtype, - 'width': width, - 'height': height, - 'count': 1, - 'crs': CRS.from_epsg(4326), - 'transform': transform, - 'compress': 'lzw', - 'tiled': True, - } - if nodata is not None: - profile['nodata'] = nodata - with rasterio.open(str(out_path), 'w', **profile) as dst: - dst.write(data, 1) - - -def convert_one_pair(args): - """转换单个干涉对: GAMMA 二进制 → GeoTIFF - - 返回: (pair_name, True/False, message) - """ - pair_dir, out_ifg_dir, geo_info, master_date, rlks = args - pair_dir = Path(pair_dir) - pair_name = pair_dir.name # e.g. 20241105-20241117 - d1, d2 = pair_name.split('-') - pair_us = f'{d1}_{d2}' # 下划线分隔 - width = geo_info['width'] - nlines = geo_info['nlines'] - - out_dir = Path(out_ifg_dir) / pair_us - out_dir.mkdir(parents=True, exist_ok=True) - - pha_tif = out_dir / f'{pair_us}.geo.diff_pha.tif' - cc_tif = out_dir / f'{pair_us}.geo.cc.tif' - - # 如果两个文件已存在则跳过 - if pha_tif.exists() and cc_tif.exists(): - return (pair_name, True, '已存在,跳过') - - try: - # --- 缩缩相位: FCOMPLEX → phase --- - diff_filt = pair_dir / f'geo_{pair_name}_{rlks}rlks.diff_filt' - if not diff_filt.exists(): - return (pair_name, False, f'缺少 {diff_filt.name}') - - cpx = np.fromfile(str(diff_filt), dtype=np.complex64).reshape(nlines, width) - phase = np.angle(cpx).astype(np.float32) - phase[cpx == 0] = 0.0 # GAMMA 无数据区域设为 0 - del cpx - write_geotiff(phase, pha_tif, geo_info, dtype='float32', nodata=0) - del phase - - # --- 相干性: FLOAT → 0-255 uint8 --- - cor_file = pair_dir / f'geo_{master_date}_{rlks}rlks.diff_filt.cor' - if not cor_file.exists(): - # 尝试备选命名 - cor_file = pair_dir / f'{pair_name}_{rlks}rlks.diff_filt.cor' - if not cor_file.exists(): - return (pair_name, False, f'缺少 cor 文件') - - cor = np.fromfile(str(cor_file), dtype=np.float32).reshape(nlines, width) - # 清理无效值 - cor[~np.isfinite(cor)] = 0 - cor = np.clip(cor, 0, 1) - cc_uint8 = (cor * 255).astype(np.uint8) - del cor - write_geotiff(cc_uint8, cc_tif, geo_info, dtype='uint8', nodata=0) - del cc_uint8 - - return (pair_name, True, 'OK') - - except Exception as e: - return (pair_name, False, str(e)) - - -def create_landmask(amp_file, geo_info, out_path): - """从地理编码幅度图生成陆地掩膜 GeoTIFF""" - width = geo_info['width'] - nlines = geo_info['nlines'] - amp = np.fromfile(str(amp_file), dtype=np.float32).reshape(nlines, width) - mask = np.zeros_like(amp, dtype=np.uint8) - mask[amp > 0] = 1 - write_geotiff(mask, out_path, geo_info, dtype='uint8', nodata=0) - print(f' 陆地掩膜已生成: {out_path}') - print(f' 有效像素: {int(np.sum(mask))}/{mask.size} ({100*np.sum(mask)/mask.size:.1f}%)') - - -def prepare_gamma_data_for_phasebias(ifgDir, demDir, masterDate, rlks, - staging_dir, n_parallel=8, - skip_existing=True): - """将 GAMMA 格式地理编码干涉图转换为 PhaseBias 脚本期望的 GeoTIFF 格式 - - 目录结构: - staging_dir/ - interferograms/ - YYYYMMDD_YYYYMMDD/ - YYYYMMDD_YYYYMMDD.geo.diff_pha.tif - YYYYMMDD_YYYYMMDD.geo.cc.tif - metadata/ - frame.geo.landmask.tif - """ - staging = Path(staging_dir) - ifg_out = staging / 'interferograms' - meta_out = staging / 'metadata' - ifg_out.mkdir(parents=True, exist_ok=True) - meta_out.mkdir(parents=True, exist_ok=True) - - # 解析地理编码参数 - utm_par = Path(demDir) / f'{masterDate}_{rlks}rlks.utm.dem.par' - if not utm_par.exists(): - raise FileNotFoundError(f'utm.dem.par 不存在: {utm_par}') - geo_info = parse_utm_dem_par(str(utm_par)) - print(f' 地理编码参数: {geo_info["width"]}x{geo_info["nlines"]}, ' - f'corner=({geo_info["corner_lat"]:.4f}, {geo_info["corner_lon"]:.4f})') - - # 生成陆地掩膜 - landmask_tif = meta_out / 'frame.geo.landmask.tif' - if not landmask_tif.exists(): - # 从任意干涉对目录中取地理编码幅度图 - first_pair = sorted(Path(ifgDir).iterdir())[0] - amp_file = first_pair / f'geo_{masterDate}_{rlks}rlks.amp' - if not amp_file.exists(): - print(f' 警告: 未找到幅度图 {amp_file}，将创建全 1 掩膜') - mask = np.ones((geo_info['nlines'], geo_info['width']), dtype=np.uint8) - write_geotiff(mask, landmask_tif, geo_info, dtype='uint8') - print(f' 全 1 掩膜已生成: {landmask_tif}') - else: - create_landmask(amp_file, geo_info, landmask_tif) - else: - print(f' 陆地掩膜已存在: {landmask_tif}') - - # 收集需要转换的干涉对 - pair_dirs = sorted([d for d in Path(ifgDir).iterdir() - if d.is_dir() and '-' in d.name]) - print(f' 发现 {len(pair_dirs)} 个干涉对目录') - - # 并行转换 - tasks = [(str(pd), str(ifg_out), geo_info, masterDate, rlks) for pd in pair_dirs] - - done_ok = 0 - done_skip = 0 - done_fail = 0 - t0 = time.time() - - with ProcessPoolExecutor(max_workers=n_parallel) as pool: - futures = {pool.submit(convert_one_pair, t): t[0] for t in tasks} - for i, fut in enumerate(as_completed(futures), 1): - pair_name, success, msg = fut.result() - if success: - if '跳过' in msg: - done_skip += 1 - else: - done_ok += 1 - else: - done_fail += 1 - print(f' ✗ {pair_name}: {msg}') - - if i % 50 == 0 or i == len(tasks): - elapsed = time.time() - t0 - print(f' 进度: {i}/{len(tasks)} ' - f'(新转换={done_ok}, 跳过={done_skip}, ' - f'失败={done_fail}, 耗时={elapsed:.0f}s)') - - print(f'\n 转换完成: 新转换={done_ok}, 跳过={done_skip}, 失败={done_fail}') - if done_fail > 0: - print(f' 警告: {done_fail} 个干涉对转换失败，将被 PhaseBias 脚本跳过') - - return str(staging) - - -# ===================================================================== -# 配置文件生成 -# ===================================================================== - -def create_config_file(output_path, root_path, interval, nlook, num_a, - start_date, end_date, estimate_an): - """创建 PhaseBias 脚本的配置文件 - - Args: - output_path: 输出目录 - root_path: 包含 interferograms/ 和 metadata/ 子目录的根路径 - """ - config_content = f"""[DEFAULT] -# 包含 interferograms/ 和 metadata/ 子目录的根路径 -root_path = {root_path} - -output_path = {output_path} - -LiCSAR_data=no -frame=NA - -# Start and end dates -start={start_date} -end={end_date} - -# Data acquisition interval: 6-day or 12-day or 24-day -interval={interval} - -# Multilooking factor -nlook={nlook} - -# Number of calibration parameters to estimate -num_a={num_a} - -### an parameters -# Estimate from data or use default values -estimate_an_values={'yes' if estimate_an else 'no'} - -# Using 6-days interval -a1_6_day=0.50 -a2_6_day=0.36 -a3_6_day=0.299 -a4_6_day=0.2476 - -# Using 12-day interval -a1_12_day=0.494 -a2_12_day=0.297 -a3_12_day=0.24 -a4_12_day=0.22 - -# Using 24-day interval (estimated) -a1_24_day=0.48 -a2_24_day=0.28 -a3_24_day=0.22 -a4_24_day=0.20 -""" - config_file = os.path.join(output_path, 'config.txt') - with open(config_file, 'w') as f: - f.write(config_content) - return config_file - - -def run_phasebias_scripts(phasebias_dir, output_path, script_name): - """Run a PhaseBias script - - Args: - phasebias_dir: directory containing PhaseBias scripts - output_path: working directory - script_name: name of the script to run - """ - script_path = os.path.join(phasebias_dir, script_name) - - if not os.path.exists(script_path): - print(f"ERROR: Script not found: {script_path}") - return False - - print(f"\n{'='*80}") - print(f"Running {script_name}...") - print(f"{'='*80}") - - # Run the script - original_dir = os.getcwd() - os.chdir(output_path) - - try: - import subprocess - result = subprocess.run([sys.executable, script_path], - capture_output=True, text=True, check=True) - print(result.stdout) - if result.stderr: - print("STDERR:", result.stderr) - return True - except subprocess.CalledProcessError as e: - print(f"ERROR running {script_name}:") - print(e.stdout) - print(e.stderr) - return False - finally: - os.chdir(original_dir) - - -def main(argv): - - start_time = time.time() - inps = cmdLineParse() - projectName = inps.projectName - interval = inps.interval - nlook = inps.nlook - num_a = inps.num_a - estimate_an = inps.estimate_an - max_con = inps.max_con - n_parallel = inps.parallel - skip_convert = inps.skip_convert - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - - # PyINT 路径 - templateFile = templateDir + "/" + projectName + ".template" - templateDict = ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - masterDate = templateDict['masterDate'] - - ifgDir = scratchDir + '/' + projectName + "/ifgrams" - demDir = scratchDir + '/' + projectName + "/DEM" - - # PhaseBias 脚本目录 - pyintDir = os.path.dirname(os.path.abspath(__file__)) - phasebiasDir = os.path.join(pyintDir, 'InSAR_PhaseBias_Correction') - - # 工作目录 - workDir = scratchDir + '/' + projectName + "/PhaseBiasCorrection" - os.makedirs(workDir, exist_ok=True) - os.makedirs(os.path.join(workDir, 'Data'), exist_ok=True) - - print("\n" + "="*80) - print(f"Phase Bias Correction for {projectName}") - print(f"Temporal interval: {interval} days") - print(f"Multilooking factor: {nlook}") - print(f"Number of calibration parameters: {num_a}") - print(f"Estimate an from data: {estimate_an}") - print(f"Max connections: {max_con}") - print(f"Parallel workers: {n_parallel}") - print("="*80) - - # 确定日期范围 - if inps.start and inps.end: - start_date = inps.start - end_date = inps.end - else: - print("\n自动检测日期范围...") - start_date, end_date = find_date_range(ifgDir, interval) - if not start_date or not end_date: - print("ERROR: 无法从干涉图目录确定日期范围!") - print("请指定 --start 和 --end 参数.") - sys.exit(1) - - print(f"\n处理日期范围: {start_date} ~ {end_date}") - - # ====== Step 0: GAMMA 二进制 → GeoTIFF 转换 ====== - print("\n" + "="*80) - print("Step 0: GAMMA 二进制 → GeoTIFF 数据转换") - print("="*80) - - staging_dir = os.path.join(workDir, 'GEOC_staging') - staging_root = prepare_gamma_data_for_phasebias( - ifgDir, demDir, masterDate, str(rlks), - staging_dir, n_parallel=n_parallel, - skip_existing=skip_convert - ) - - # ====== 创建配置文件 ====== - print("\n创建配置文件...") - # root_path 指向 staging 目录（包含 interferograms/ 和 metadata/） - config_file = create_config_file(workDir, staging_root, interval, nlook, - num_a, start_date, end_date, estimate_an) - print(f"配置文件: {config_file}") - - # ====== 运行 PhaseBias 流水线 ====== - print("\n" + "="*80) - print("运行 Phase Bias Correction 流水线") - print("="*80) - - # Step 1: 读取数据 - if not run_phasebias_scripts(phasebiasDir, workDir, 'PhaseBias_01_Read_Data.py'): - print("ERROR in Step 1: Read Data") - sys.exit(1) - - # Step 2: 计算闭合环 - if not run_phasebias_scripts(phasebiasDir, workDir, 'PhaseBias_02_Loop_Closures.py'): - print("ERROR in Step 2: Loop Closures") - sys.exit(1) - - # Step 3: 估计标定参数（可选） - if estimate_an: - if not run_phasebias_scripts(phasebiasDir, workDir, 'PhaseBias_03_calibration_pars.py'): - print("ERROR in Step 3: Calibration Parameters") - sys.exit(1) - - # Step 4: 反演 - if not run_phasebias_scripts(phasebiasDir, workDir, 'PhaseBias_04_Inversion.py'): - print("ERROR in Step 4: Inversion") - sys.exit(1) - - # Step 5: 应用校正 - if not run_phasebias_scripts(phasebiasDir, workDir, 'PhaseBias_05_Correction.py'): - print("ERROR in Step 5: Correction") - sys.exit(1) - - print("\n" + "="*80) - print("相位偏差校正完成!") - print(f"输出目录: {workDir}") - print("="*80) - - ut.print_process_time(start_time, time.time()) - - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/phasebias_correction_gamma_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/phasebias_correction_gamma_all.py deleted file mode 100644 index 4d5d2c8..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/phasebias_correction_gamma_all.py +++ /dev/null @@ -1,232 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import time -import glob -import argparse -import subprocess -from datetime import datetime - -from pyint import _utils as ut - - -def find_ifgs_by_interval(ifgDir, interval=12): - """Find all interferograms with specified temporal interval - - Args: - ifgDir: interferogram directory - interval: temporal baseline in days (6, 12, or 24) - - Returns: - list: list of date pairs - """ - ifgs = [] - - all_dirs = sorted([d for d in os.listdir(ifgDir) if os.path.isdir(os.path.join(ifgDir, d))]) - - for dirname in all_dirs: - if '-' in dirname: - dates = dirname.split('-') - elif '_' in dirname: - dates = dirname.split('_') - else: - continue - - if len(dates) == 2: - try: - date1 = dates[0] - date2 = dates[1] - - d1 = datetime.strptime(date1, '%Y%m%d') - d2 = datetime.strptime(date2, '%Y%m%d') - days = abs((d2 - d1).days) - - if days == interval: - ifgs.append(dirname) - except ValueError: - continue - - return ifgs - - -INTRODUCTION = ''' -------------------------------------------------------------------- - Apply phase bias correction to all interferograms with - specified temporal interval in a project. - - This script runs the full PhaseBias correction pipeline: - 1. Read interferogram data - 2. Calculate loop closures - 3. Estimate calibration parameters (optional) - 4. Inversion for phase bias terms - 5. Apply correction - -''' - -EXAMPLE = ''' - Usage: - phasebias_correction_gamma_all.py projectName - phasebias_correction_gamma_all.py projectName --interval 12 - phasebias_correction_gamma_all.py projectName --interval 12 --nlook 10 - phasebias_correction_gamma_all.py projectName --interval 24 --estimate-an -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Apply phase bias correction to interferograms.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName', help='projectName for processing.') - parser.add_argument('--interval', type=int, default=12, - help='Data acquisition interval in days (6, 12, or 24). [default: 12]') - parser.add_argument('--nlook', type=int, default=10, - help='Number of looks for multilooking. [default: 10]') - parser.add_argument('--num-a', type=int, default=2, - help='Number of calibration parameters to estimate. [default: 2]') - parser.add_argument('--start', type=str, default=None, - help='Start date (YYYYMMDD). If not specified, will use all available data.') - parser.add_argument('--end', type=str, default=None, - help='End date (YYYYMMDD). If not specified, will use all available data.') - parser.add_argument('--estimate-an', dest='estimate_an', action='store_true', - help='Estimate calibration parameters from data instead of using defaults.') - parser.add_argument('--max-con', type=int, default=5, - help='Maximum number of connections to correct. [default: 5]') - - inps = parser.parse_args() - return inps - - -def main(argv): - start_time = time.time() - inps = cmdLineParse() - projectName = inps.projectName - interval = inps.interval - nlook = inps.nlook - num_a = inps.num_a - estimate_an = inps.estimate_an - max_con = inps.max_con - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - projectDir = scratchDir + '/' + projectName - ifgDir = scratchDir + '/' + projectName + '/ifgrams' - - templateDict = ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - - # Check if interferograms with specified interval exist - ifgList = find_ifgs_by_interval(ifgDir, interval) - - print("\n" + "="*80) - print(f"Phase Bias Correction for Project: {projectName}") - print(f"Temporal interval: {interval} days") - print(f"Number of interferograms found: {len(ifgList)}") - print("="*80) - - if len(ifgList) == 0: - print(f"\nERROR: No interferograms found with {interval}-day interval!") - print("Available interferograms:") - all_dirs = sorted([d for d in os.listdir(ifgDir) if os.path.isdir(os.path.join(ifgDir, d))]) - for dirname in all_dirs[:10]: # Show first 10 - if '-' in dirname: - dates = dirname.split('-') - if len(dates) == 2: - try: - d1 = datetime.strptime(dates[0], '%Y%m%d') - d2 = datetime.strptime(dates[1], '%Y%m%d') - days = abs((d2 - d1).days) - print(f" {dirname}: {days} days") - except: - pass - sys.exit(1) - - # Determine date range - if inps.start and inps.end: - start_date = inps.start - end_date = inps.end - else: - # Extract dates from interferogram list - dates = [] - for pair in ifgList: - if '-' in pair: - date_pair = pair.split('-') - dates.extend(date_pair) - elif '_' in pair: - date_pair = pair.split('_') - dates.extend(date_pair) - - dates = sorted(set(dates)) - start_date = dates[0] - end_date = dates[-1] - - print(f"\nDate range: {start_date} to {end_date}") - print(f"Multilooking factor: {nlook}") - print(f"Number of calibration parameters: {num_a}") - print(f"Estimate an from data: {estimate_an}") - print(f"Max connections: {max_con}") - - # Check if correction has already been done - outputDir = projectDir + '/PhaseBiasCorrection' - configFile = outputDir + '/config.txt' - - if os.path.exists(configFile): - print("\n" + "="*80) - print("Found existing PhaseBiasCorrection directory") - print("This script will process ALL interferograms together using the") - print("InSAR_PhaseBias_Correction pipeline.") - print("="*80) - - # Build command - cmd_list = [ - 'phasebias_correction_gamma.py', - projectName, - '--interval', str(interval), - '--nlook', str(nlook), - '--num-a', str(num_a), - '--max-con', str(max_con) - ] - - if start_date: - cmd_list.extend(['--start', start_date]) - if end_date: - cmd_list.extend(['--end', end_date]) - if estimate_an: - cmd_list.append('--estimate-an') - - # Run the phase bias correction script - print("\n" + "="*80) - print("Running phase bias correction...") - print("="*80) - print(f"Command: {' '.join(cmd_list)}") - - try: - result = subprocess.run(cmd_list, check=True, capture_output=False) - print("\n" + "="*80) - print("Phase bias correction completed successfully!") - print(f"Output directory: {outputDir}") - print(f"Corrected interferograms: {outputDir}/GEOC/") - print("="*80) - except subprocess.CalledProcessError as e: - print("\n" + "="*80) - print("ERROR: Phase bias correction failed!") - print("="*80) - sys.exit(1) - - ut.print_process_time(start_time, time.time()) - sys.exit(1) - - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/plot_auto_grd.sh b/.codex_tmp/pyint_variants/no_rescue/pyint/plot_auto_grd.sh deleted file mode 100644 index ac1e8ae..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/plot_auto_grd.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -# 描述：自动根据GRD文件生成地图（依赖GMT 6+） -# 用法：./auto_plot_grd.sh <输入.grd> [输出文件名] [选项] - -# 默认参数 -input_grd=$1 -output="map.png" # 默认输出文件名 -projection="M15c" # 默认投影（Mercator，宽度15厘米） -colormap="viridis" # 默认色标 -title="Topography" # 默认标题 -annotate="a" # 默认标注间隔（自动） - -# 解析命令行参数 -while [[ $# -gt 0 ]]; do - case $1 in - -o|--output) output="$2"; shift ;; - -P|--projection) projection="$2"; shift ;; - -C|--colormap) colormap="$2"; shift ;; - -T|--title) title="$2"; shift ;; - -A|--annotate) annotate="$2"; shift ;; - *) ;; - esac - shift -done - -# 检查输入文件是否存在 -if [[ ! -f $input_grd ]]; then - echo "错误：输入文件 $input_grd 不存在！" - exit 1 -fi - -# 从GRD文件获取地理范围 -region=$(gmt grdinfo $input_grd -I- | awk -F'R' '{print $2}') - -# 生成临时CPT色标文件 -gmt makecpt -Cturbo -T$(gmt grdinfo $input_grd -T | awk -F'T' '{print $2}') > tmp.cpt - -# 开始绘图 -gmt begin ${output%.*} pdf,png # 去除扩展名 - # 绘制GRD数据 - gmt grdimage $input_grd -R$region -J$projection -Ctmp.cpt -B$annotate -BWSne+t"$title" - # 添加海岸线 - gmt coast -R$region -J$projection -W0.5p,black -Df - # 添加色标 - gmt colorbar -DJBC+w10c/0.5c+o0/1c -Bxaf -By+l"Elevation (m)" -gmt end show - -# 清理临时文件 -#rm -f tmp.cpt - -echo "地图已生成：$output" diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/plot_geotiff.py b/.codex_tmp/pyint_variants/no_rescue/pyint/plot_geotiff.py deleted file mode 100644 index 81fa575..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/plot_geotiff.py +++ /dev/null @@ -1,229 +0,0 @@ -#! /usr/bin/env python -import argparse -import os -import numpy as np -import matplotlib.pyplot as plt -import rasterio -from rasterio.plot import show -import matplotlib.colors as colors -from matplotlib import cm -import warnings -warnings.filterwarnings('ignore') - -def parse_arguments(): - """解析命令行参数""" - parser = argparse.ArgumentParser(description='绘制GeoTIFF文件') - - # 必需参数 - parser.add_argument('-i', '--input', required=True, - help='输入GeoTIFF文件路径') - - # 输出参数 - parser.add_argument('-o', '--output', - help='输出图片文件路径 (默认: 输入文件名.png)') - - # 绘图参数 - parser.add_argument('--dpi', type=int, default=300, - help='输出图片分辨率 (默认: 300)') - parser.add_argument('--cmap', default='viridis', - help='色彩映射 (默认: viridis)') - parser.add_argument('--title', - help='图片标题 (默认: 使用文件名)') - parser.add_argument('--band', type=int, default=1, - help='要绘制的波段 (默认: 1)') - - # 显示参数 - parser.add_argument('--vmin', type=float, - help='颜色范围最小值') - parser.add_argument('--vmax', type=float, - help='颜色范围最大值') - parser.add_argument('--log', action='store_true', - help='使用对数颜色标尺') - parser.add_argument('--percentile', type=float, nargs=2, - help='使用百分位数设置颜色范围，例如 --percentile 2 98') - - # 图片尺寸 - parser.add_argument('--width', type=float, default=10, - help='图片宽度 (英寸, 默认: 10)') - parser.add_argument('--height', type=float, default=8, - help='图片高度 (英寸, 默认: 8)') - - # 其他选项 - parser.add_argument('--no-colorbar', action='store_true', - help='不显示颜色条') - parser.add_argument('--show', action='store_true', - help='显示图片 (默认只保存)') - - return parser.parse_args() - -def get_default_output_filename(input_file): - """根据输入文件名生成默认输出文件名""" - dir_name = os.path.dirname(input_file) - base_name = os.path.basename(input_file) - name, ext = os.path.splitext(base_name) - return os.path.join(dir_name, f"{name}.png") - -def read_geotiff(file_path, band=1): - """读取GeoTIFF文件""" - if not os.path.exists(file_path): - raise FileNotFoundError(f"文件不存在: {file_path}") - - with rasterio.open(file_path) as src: - # 检查波段数量 - if band > src.count: - raise ValueError(f"文件只有 {src.count} 个波段，无法读取波段 {band}") - - data = src.read(band) - profile = src.profile - bounds = src.bounds - crs = src.crs - nodata = src.nodata - - # 如果有无效值，创建掩码 - if nodata is not None: - data = np.ma.masked_where(data == nodata, data) - - return data, profile, bounds, crs, nodata - -def plot_geotiff(args): - """主绘图函数""" - # 设置默认输出文件名 - if args.output is None: - args.output = get_default_output_filename(args.input) - - # 设置默认标题 - if args.title is None: - args.title = os.path.basename(args.input) - - print(f"读取文件: {args.input}") - print(f"输出文件: {args.output}") - print(f"使用波段: {args.band}") - - # 读取数据 - data, profile, bounds, crs, nodata = read_geotiff(args.input, args.band) - - print(f"数据形状: {data.shape}") - print(f"数据类型: {data.dtype}") - print(f"坐标参考系统: {crs}") - print(f"数据范围: {bounds}") - - if nodata is not None: - print(f"无效值: {nodata}") - valid_data = data[~data.mask] if hasattr(data, 'mask') else data[data != nodata] - else: - valid_data = data.compressed() if hasattr(data, 'mask') else data.flatten() - - print(f"有效数据点数: {len(valid_data)}") - print(f"数据范围: {valid_data.min():.6f} - {valid_data.max():.6f}") - - # 设置颜色范围 - if args.percentile: - vmin = np.percentile(valid_data, args.percentile[0]) - vmax = np.percentile(valid_data, args.percentile[1]) - print(f"使用百分位数范围: {args.percentile[0]}% - {args.percentile[1]}%") - print(f"对应数值范围: {vmin:.6f} - {vmax:.6f}") - elif args.vmin is not None and args.vmax is not None: - vmin, vmax = args.vmin, args.vmax - print(f"使用指定范围: {vmin} - {vmax}") - else: - vmin, vmax = valid_data.min(), valid_data.max() - print(f"使用数据范围: {vmin:.6f} - {vmax:.6f}") - - # 创建图形 - fig, ax = plt.subplots(1, 1, figsize=(args.width, args.height)) - - # 选择颜色映射 - try: - cmap = plt.get_cmap(args.cmap) - except: - print(f"警告: 色彩映射 '{args.cmap}' 不存在，使用默认的 'viridis'") - cmap = plt.get_cmap('viridis') - - # 选择颜色标准化方式 - if args.log: - norm = colors.LogNorm(vmin=vmin, vmax=vmax) - print("使用对数颜色标尺") - # 对于对数标准化，不能同时传递vmin/vmax参数 - show_kwargs = {'norm': norm, 'cmap': cmap} - else: - # 对于线性标准化，可以直接传递vmin/vmax参数 - show_kwargs = {'vmin': vmin, 'vmax': vmax, 'cmap': cmap} - - # 使用rasterio的show函数显示地理参考数据 - if crs is not None: - with rasterio.open(args.input) as src: - # 使用rasterio的show函数，它能够正确处理地理参考 - # 注意：show()返回的是Axes对象，不是mappable对象 - image_axes = show( - (src, args.band), - ax=ax, - **show_kwargs - ) - - # 从axes对象中获取图像对象 - if hasattr(image_axes, 'images') and len(image_axes.images) > 0: - im = image_axes.images[0] - else: - # 如果无法获取图像对象，创建一个ScalarMappable用于颜色条 - im = cm.ScalarMappable(norm=norm if args.log else colors.Normalize(vmin=vmin, vmax=vmax), - cmap=cmap) - im.set_array([]) # 设置一个空数组 - else: - # 如果没有地理参考，使用普通的imshow - if args.log: - im = ax.imshow(data, norm=norm, cmap=cmap) - else: - im = ax.imshow(data, vmin=vmin, vmax=vmax, cmap=cmap) - - # 设置标题和标签 - ax.set_title(args.title, fontsize=14, fontweight='bold') - - # 添加颜色条 - if not args.no_colorbar: - # 确保im是一个mappable对象 - if not hasattr(im, 'set_array'): - # 如果不是mappable对象，创建一个 - im = cm.ScalarMappable(norm=norm if args.log else colors.Normalize(vmin=vmin, vmax=vmax), - cmap=cmap) - im.set_array([]) # 设置一个空数组 - - cbar = plt.colorbar(im, ax=ax, shrink=0.8) - cbar.set_label('值', rotation=270, labelpad=15) - - # 添加网格 - ax.grid(True, alpha=0.3) - - # 设置坐标轴标签 - if crs is not None: - ax.set_xlabel('经度') - ax.set_ylabel('纬度') - else: - ax.set_xlabel('X') - ax.set_ylabel('Y') - - # 保存图片 - plt.tight_layout() - plt.savefig(args.output, dpi=args.dpi, bbox_inches='tight') - print(f"图片已保存: {args.output}") - - # 显示图片 - if args.show: - plt.show() - - plt.close() - - return args.output - -def main(): - """主函数""" - args = parse_arguments() - - try: - output_file = plot_geotiff(args) - print(f"成功生成图片: {output_file}") - except Exception as e: - print(f"处理过程中发生错误: {e}") - raise - -if __name__ == "__main__": - main() diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/pot_gamma_subset.py b/.codex_tmp/pyint_variants/no_rescue/pyint/pot_gamma_subset.py deleted file mode 100644 index 100c3f7..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/pot_gamma_subset.py +++ /dev/null @@ -1,191 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2022, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import getopt -import time -import glob -import argparse - -from pyint import _utils as ut - - -INTRODUCTION = ''' -------------------------------------------------------------------- - Pixel offset tracking for co-registered Images. TOPs should be deramped in advance. - -''' - -EXAMPLE = ''' - Usage: - pot_gamma_subset.py projectName Mdate Sdate subset - pot_gamma_subset.py PacayaT163TsxHhA 20150102 20150601 0102 -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Pixel offset tracking based Azimuth/Range dispalcement estimation.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('Mdate',help='Master date.') - parser.add_argument('Sdate',help='Slave date.') - parser.add_argument('Subset',help='Subset name, e.g., 0102') - - - inps = parser.parse_args() - return inps - - -def main(argv): - - start_time = time.time() - inps = cmdLineParse() - Mdate = inps.Mdate - Sdate = inps.Sdate - subset = inps.Subset - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - masterDate = templateDict['masterDate'] - - if 'POT_rwin' in templateDict: - POT_rwin = templateDict['POT_rwin'] - else: - POT_rwin = '160' - print('POT range window sise: ' + POT_rwin) - - if 'POT_awin' in templateDict: - POT_awin = templateDict['POT_awin'] - else: - POT_awin = '160' - print('POT azimuth window sise: ' + POT_awin) - - if 'POT_astep' in templateDict: - POT_astep = templateDict['POT_astep'] - else: - POT_astep = azlks - - if 'POT_rstep' in templateDict: - POT_rstep = templateDict['POT_rstep'] - else: - POT_rstep = rlks - - print('POT range steps: ' + POT_rstep) - print('POT azimuth steps: ' + POT_astep) - - projectDir = scratchDir + '/' + projectName - demDir = scratchDir + '/' + projectName + '/DEM' - - slcDir = scratchDir + '/' + projectName + '/SLC' - rslcDir = scratchDir + '/' + projectName + '/RSLC' - ifgDir = projectDir + '/ifgrams' - if not os.path.isdir(ifgDir): os.mkdir(ifgDir) - - Pair = Mdate + '-' + Sdate - workDir = ifgDir + '/' + Pair - if not os.path.isdir(workDir): os.mkdir(workDir) - - ####################################################################### - Mamp = rslcDir + '/' + Mdate + '/' + Mdate + '_' + subset + '_' + rlks + 'rlks.amp' - MampPar = rslcDir + '/' + Mdate + '/' + Mdate + '_' + subset + '_' + rlks + 'rlks.amp.par' - Samp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + subset + '_' + rlks + 'rlks.amp' - SampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + subset + '_' + rlks + 'rlks.amp.par' - - Mrslc = rslcDir + '/' + Mdate + '/' + Mdate + '_' + subset + '.rslc' - MrslcPar = rslcDir + '/' + Mdate + '/' + Mdate + '_' + subset + '.rslc.par' - Srslc = rslcDir + '/' + Sdate + '/' + Sdate + '_' + subset + '.rslc' - SrslcPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + subset + '.rslc.par' - - #HGT = demDir + '/' + masterDate + '_' + rlks + 'rlks.rdc.dem' - #MasterPar = rslcDir + '/' + masterDate + '/' + masterDate + '.rslc.par' - - ################# copy file for parallel processing ########################## - #Mamp = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp' - #MampPar = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp.par' - #Samp = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp' - #SampPar = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp.par' - - #if not templateDict['diff_all_parallel'] == '1': - - # Mrslc = workDir + '/' + Mdate + '.rslc' - # MrslcPar = workDir + '/' + Mdate + '.rslc.par' - # Srslc = workDir + '/' + Sdate + '.rslc' - # SrslcPar = workDir + '/' + Sdate + '.rslc.par' - # ut.copy_file(Mrslc0,Mrslc) - # ut.copy_file(MrslcPar0,MrslcPar) - # ut.copy_file(Srslc0,Srslc) - # ut.copy_file(SrslcPar0,SrslcPar) - - #else: - - # Mrslc = Mrslc0 - # MrslcPar = MrslcPar0 - # Srslc = Srslc0 - # SrslcPar = SrslcPar0 - # HGT = HGT0 - # MasterPar = MasterPar0 - - #ut.copy_file(Mamp0,Mamp) - #ut.copy_file(MampPar0,MampPar) - #ut.copy_file(Samp0,Samp) - #ut.copy_file(SampPar0,SampPar) - - #ut.copy_file(HGT0,HGT) - #ut.copy_file(MasterPar0,MasterPar) - - ############################################################################ - - OFF = workDir + '/' + Pair + '_' + subset + '_' + rlks + 'rlks_pot.off' - OFFS = workDir + '/' + Pair + '_' + subset + '_' + rlks + 'rlks_pot.offs' - CCP = workDir + '/' + Pair + '_' + subset + '_' + rlks + 'rlks_pot.ccp' - COFFS = workDir + '/' + Pair + '_' + subset + '_' + rlks + 'rlks_pot.coffs' - COFFS_FILT = workDir + '/' + Pair + '_' + subset + '_' + rlks + 'rlks_pot_filt.coffs' - POTA = workDir + '/' + Pair + '_' + subset + '_' + rlks + 'rlks_pot.az' - POTR = workDir + '/' + Pair + '_' + subset + '_' + rlks + 'rlks_pot.rg' - - call_str = 'create_offset '+ MrslcPar + ' ' + SrslcPar + ' ' + OFF + ' 1 ' + rlks + ' ' + azlks + ' 0' - os.system(call_str) - - call_str = 'offset_pwr_tracking ' + Mrslc + ' ' + Srslc + ' ' + MrslcPar + ' ' + SrslcPar + ' ' + OFF + ' ' + OFFS + ' ' + CCP + ' ' + POT_rwin + ' ' + POT_awin + ' - 2 0.05 ' + POT_rstep + ' ' + POT_astep + ' - - - - - - ' - os.system(call_str) - - call_str = 'offset_tracking ' + OFFS + ' ' + CCP + ' ' + MrslcPar + ' ' + OFF + ' ' + COFFS + ' - 1 - 1' - os.system(call_str) - - nWidth = ut.read_gamma_par(OFF, 'read', 'interferogram_width') - - #call_str = 'adf ' + COFFS + ' ' + COFFS_FILT + ' ' + CCP + ' ' + nWidth - #os.system(call_str) - - call_str = 'cpx_to_real ' + COFFS + ' ' + POTR + ' ' + nWidth + ' 0 ' - os.system(call_str) - - call_str = 'cpx_to_real ' + COFFS + ' ' + POTA + ' ' + nWidth + ' 1 ' - os.system(call_str) - - call_str = 'rasdt_pwr ' + POTR + ' ' + Mamp + ' ' + nWidth + ' - - - - - - - BuYlRd.cm' - os.system(call_str) - - call_str = 'rasdt_pwr ' + POTA + ' ' + Mamp + ' ' + nWidth + ' - - - - - - - BuYlRd.cm' - os.system(call_str) - - print("Pixel offset tracking is Done!") - ut.print_process_time(start_time, time.time()) - #sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/pot_gamma_subset_combine.py b/.codex_tmp/pyint_variants/no_rescue/pyint/pot_gamma_subset_combine.py deleted file mode 100644 index 026b510..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/pot_gamma_subset_combine.py +++ /dev/null @@ -1,372 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# -import time -import numpy as np -import os -import sys -import argparse -import h5py - -from pyint import _utils as ut -from scipy.interpolate import griddata -from scipy.interpolate import NearestNDInterpolator - -#def resamp2d_near(row0,col0,z0,row1,col1): - -# xx0, yy0 = np.meshgrid((np.arange(col0) + 1), (np.arange(row0) + 1)) -# xx1, yy1 = np.meshgrid((np.arange(col1) + 1), (np.arange(row1) + 1)) - -# xx00 = xx0/col0*col1; yy00 = yy0/row0*row1 -# xx = xx00.flatten(); yy = yy00.flatten(); zz = z0.flatten() -# interp = NearestNDInterpolator(list(zip(xx, yy)), zz) -# z1 = interp(xx1, yy1) - -# return z1 - -def generate_name(i,j): - - if len(str(int(i)))==1: - i0 = '0' + str(int(i)) - else: - i0 = str(int(i)) - - if len(str(int(j)))==1: - j0 = '0' + str(int(j)) - else: - j0 = str(int(j)) - - name0 = i0+j0 - return name0 - -def reduce_samp(xx,yy,zz,xg,yg,extend): - - max_x = np.max(xg)+extend; min_x = np.min(xg)-extend # e.g. extend 100 - max_y = np.max(yg)+extend; min_y = np.min(yg)-extend # e.g., extend 100 - - xx1 = xx[((min_x (2/3*int(awidth)): - Ap_samp1 = Ap_samp - else: - Ap_samp1 = Ap_samp[0:(Na0-1)] # last batch with '-' means to the end - - if LR_end > (2/3*int(rwidth)): - Rp_samp1 = Rp_samp - else: - Rp_samp1 = Rp_samp[0:(Nr0-1)] # last batch with '-' means to the end - return Ap_samp1, Rp_samp1 - -def subset2coord(subset, astep, rstep, nLine, nWidth, awidth, rwidth, extend): - # default awidth = 5000 rwidth = 5000 extend = 200 - # astep: azimuth multilook numbers - # rstep: range multilook numbers - extend = int(extend) - Ap_samp1, Rp_samp1 = get_startSamp(nLine, nWidth, awidth, rwidth) - Na = len(Ap_samp1); Nr = len(Rp_samp1) - ii = int(subset[0:2]); jj = int(subset[2:4]) - #print(ii); print(jj) - rstart = str(Rp_samp1[jj]); astart = str(Ap_samp1[ii]); - #print(nLine);print(Ap_samp1);print(Rp_samp1) - - if not ii==0: - astart0 = str(int(int(astart) - extend)) # extend 200 to avoid edge effect - else: - astart0 = astart - - if not jj==0: - rstart0 = str(int(int(rstart) - extend)) # extend 200 to avoid edge effect - else: - rstart0 = rstart - - #print('astart') - #print(astart);print(Ap_samp1[Na-1]) - if astart == str(Ap_samp1[Na-1]): - awidth0 = '-' - aend0 = str(nLine) - else: - awidth0 = str(int(int(awidth) + extend)) # extend 200 to avoid edge effect - aend0 = str(int(int(astart0) + int(awidth) + extend - 1)) - - if rstart == str(Rp_samp1[Nr-1]): - rwidth0 = '-' - rend0 = str(nWidth) - else: - rwidth0 = str(int(int(rwidth) + extend)) # extend 200 to avoid edge effect - rend0 = str(int(int(rstart0) + int(rwidth) + extend - 1)) - - xx0 = np.arange(int(rstart0), int(rend0), int(rstep)); Nx0 = len(xx0); xx1 = xx0[0:Nx0-1] - yy0 = np.arange(int(astart0), int(aend0), int(astep)); Ny0 = len(yy0); yy1 = yy0[1:Ny0] - #print(rstart0); print(rend0); print(astart0); print(aend0) - - rwidth_total = int(int(rend0) - int(rstart0) + 1) - awidth_total = int(int(aend0) - int(astart0) + 1) - if np.mod(rwidth_total,int(rstep))==0: - xx1 = xx0 - else: - xx1 = xx0[0:Nx0-1] - - if np.mod(awidth_total,int(astep))==0: - yy1 = yy0 - else: - yy1 =yy0[0:Ny0-1] - - return xx1,yy1 - -def read_gammadata(file0,nWidth0,nLength0): - data0 = np.fromfile(file0,dtype='>f4',count=int(nLength0)*int(nWidth0)).reshape(int(nLength0), int(nWidth0)) - return data0 - -def write_h5(datasetDict, out_file, metadata=None, ref_file=None, compression=None): - - if os.path.isfile(out_file): - print('delete exsited file: {}'.format(out_file)) - os.remove(out_file) - - print('create HDF5 file: {} with w mode'.format(out_file)) - dt = h5py.special_dtype(vlen=np.dtype('float64')) - - - with h5py.File(out_file, 'w') as f: - for dsName in datasetDict.keys(): - data = datasetDict[dsName] - ds = f.create_dataset(dsName, - data=data, - compression=compression) - - for key, value in metadata.items(): - f.attrs[key] = str(value) - #print(key + ': ' + value) - print('finished writing to {}'.format(out_file)) - - return out_file - -INTRODUCTION = ''' -------------------------------------------------------------------- - Combine subset of POT results from pot_gamma_subset_jobs.py -''' - -EXAMPLE = ''' - Usage: - pot_gamma_subset_combine.py projectName Mdate Sdate - pot_gamma_subset_combine.py PacayaT163TsxHhA 20150601 20150613 -------------------------------------------------------------------- -''' - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Combine subset of POT results from pot_gamma_subset_jobs.py',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('Mdate',help='First date.') - parser.add_argument('Sdate',help='Second date.') - #parser.add_argument('--out',dest ='out', help='Output file name.') - parser.add_argument('--rwidth',dest ='rwidth', default = '5000', help='Patch range size.') - parser.add_argument('--awidth',dest ='awidth', default = '5000', help='Patch azimuth size.') - parser.add_argument('--extend',dest ='extend', default = '200', help='Patch extend to ensure overlap regions.') - - inps = parser.parse_args() - return inps - - -def main(argv): - - inps = cmdLineParse() - start_time = time.time() - projectName = inps.projectName - rwidth = inps.rwidth; awidth = inps.awidth; extend = inps.extend - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - - processDir = scratchDir + '/' + projectName + "/PROCESS" - slcDir = scratchDir + '/' + projectName + "/SLC" - rslcDir = scratchDir + '/' + projectName + '/RSLC' - ifgDir = scratchDir + '/' + projectName + '/ifgrams' - Mdate = inps.Mdate; Sdate = inps.Sdate - Pair = Mdate + '-' + Sdate - workDir = ifgDir + '/' + Pair - MslcPar = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc.par' - - Samp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' - SampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' - SWidth = ut.read_gamma_par(SampPar, 'read', 'range_samples') - SLine = ut.read_gamma_par(SampPar, 'read', 'azimuth_lines') - SampData = read_gammadata(Samp,SWidth,SLine) - - nWidth = ut.read_gamma_par(MslcPar, 'read', 'range_samples'); Rp_samp_large0 = np.arange(1,int(nWidth),int(rlks)) - nLine = ut.read_gamma_par(MslcPar, 'read', 'azimuth_lines'); Ap_samp_large0 = np.arange(1,int(nLine),int(azlks)) - Nr0 = len(Rp_samp_large0); Na0 = len(Ap_samp_large0) - - if np.mod(int(nWidth),int(rlks))==0: - Rp_samp_large = Rp_samp_large0 - else: - Rp_samp_large = Rp_samp_large0[0:Nr0-1] - - if np.mod(int(nLine),int(azlks))==0: - Ap_samp_large= Ap_samp_large0 - else: - Ap_samp_large =Ap_samp_large0[0:Na0-1] - - rr,aa = np.meshgrid(Rp_samp_large,Ap_samp_large) - - Nr = len(Rp_samp_large); Na = len(Ap_samp_large) - print('Total samples along range and azimuth: ' + str(Nr) + ' ' + str(Na)) - - Ap_samp1,Rp_samp1 = get_startSamp(nLine, nWidth, awidth, rwidth) - Na = len(Ap_samp1); Nr = len(Rp_samp1) - - xx_total = []; yy_total = []; az_total = []; rg_total = []; cc_total = []; - for i in range(Na): - for j in range(Nr): - subset0 = generate_name(i,j) - xx0,yy0 = subset2coord(subset0, azlks, rlks, nLine, nWidth, awidth, rwidth, extend) - #print(len(xx0)); print(len(yy0)) - [xx0,yy0] = np.meshgrid(xx0,yy0) - az0 = workDir + '/' + Pair + '_' + subset0 + '_' + rlks + 'rlks_pot.az' - rg0 = workDir + '/' + Pair + '_' + subset0 + '_' + rlks + 'rlks_pot.rg' - cc0 = workDir + '/' + Pair + '_' + subset0 + '_' + rlks + 'rlks_pot.ccp' - off0 = workDir + '/' + Pair + '_' + subset0 + '_' + rlks + 'rlks_pot.off' - nLine0 = ut.read_gamma_par(off0, 'read', 'interferogram_azimuth_lines') - nWidth0 = ut.read_gamma_par(off0, 'read', 'interferogram_width') - #print(nWidth0);print(nLine0) - if os.path.isfile(az0): - if os.path.getsize(az0)>0: - az_data0 = read_gammadata(az0,nWidth0,nLine0) - rg_data0 = read_gammadata(rg0,nWidth0,nLine0) - cc_data0 = read_gammadata(cc0,nWidth0,nLine0) - - xx_total.extend(list(xx0.flatten()));yy_total.extend(list(yy0.flatten())) - az_total.extend(list(az_data0.flatten())) - rg_total.extend(list(rg_data0.flatten())) - cc_total.extend(list(cc_data0.flatten())) - else: - data0 = np.zeros((int(nWidth0),int(nLine0)),dtype='float32') - xx_total.extend(list(data0.flatten()));yy_total.extend(list(data0.flatten())) - az_total.extend(list(data0.flatten())) - rg_total.extend(list(data0.flatten())) - cc_total.extend(list(data0.flatten())) - - xx_total = np.asarray(xx_total); #print(xx_total.shape) - yy_total = np.asarray(yy_total); #print(yy_total.shape) - az_total = np.asarray(az_total); #print(az_total.shape) - rg_total = np.asarray(rg_total); #print(rg_total.shape) - cc_total = np.asarray(cc_total); #print(cc_total.shape) - - xxa = xx_total[az_total!=0]; yya = yy_total[az_total!=0]; az_total1 = az_total[az_total!=0] - xxr = xx_total[rg_total!=0]; yyr = yy_total[rg_total!=0]; rg_total1 = rg_total[rg_total!=0] - xxc = xx_total[cc_total!=0]; yyc = yy_total[cc_total!=0]; cc_total1 = cc_total[cc_total!=0] - - #start_time = time.time() - #az_grid_large = interp_split(xxa,yya,az_total1,Rp_samp_large,Ap_samp_large,2); print('1 finish') - #rg_grid_large = interp_split(xx_total,yy_total,rg_total1,Rp_samp_large,Ap_samp_large,2) - #cc_grid_large = interp_split(xx_total,yy_total,cc_total1,Rp_samp_large,Ap_samp_large,2) - - #start_time = time.time() - #interp_az = NearestNDInterpolator(list(zip(xxa, yya)), az_total1) - #az_grid_large = interp_az(rr, aa) - #interp_rg = NearestNDInterpolator(list(zip(xxr, yyr)), rg_total1) - #rg_grid_large = interp_rg(rr, aa) - #interp_cc = NearestNDInterpolator(list(zip(xxc, yyc)), cc_total1) - #cc_grid_large = interp_cc(rr, aa) - #ut.print_process_time(start_time, time.time()) - - method0 = 'nearest' # keep 0 values to make good mask - print('Start to combine all of the sub-patches ...') - start_time = time.time() - az_grid_large = griddata((xxa,yya),az_total1,(rr,aa),method=method0); print('Azimuth interpolate finish') - #ut.print_process_time(start_time, time.time()) - #start_time = time.time() - rg_grid_large = griddata((xxr,yyr),rg_total1,(rr,aa),method=method0); print('Range interpolate finish') - #ut.print_process_time(start_time, time.time()) - #start_time = time.time() - cc_grid_large = griddata((xxc,yyc),cc_total1,(rr,aa),method=method0); print('CCP interpolate finish') - ut.print_process_time(start_time, time.time()) - - az_grid_large[SampData==0]=0 - rg_grid_large[SampData==0]=0 - cc_grid_large[SampData==0]=0 - - row1,col1 = rr.shape - meta = dict() - meta['WIDTH'] = str(col1); meta['LENGTH'] = str(row1); meta['UNIT'] = 'm' - meta['FILE_TYPE'] ='offset_tracking'; meta['DATE12'] = Pair - - datasetDict = dict() - datasetDict['azimuth'] = az_grid_large - #datasetDict['range'] = rg_grid_large - #datasetDict['ccp'] = cc_grid_large - #if inps.out: - # out_file = inps.out - #else: - out_file = Pair + '_pot_az.h5' - write_h5(datasetDict, out_file, metadata=meta, ref_file=None, compression=None) - - datasetDict = dict() - #datasetDict['azimuth'] = az_grid_large - datasetDict['range'] = rg_grid_large - #datasetDict['ccp'] = cc_grid_large - out_file = Pair + '_pot_rg.h5' - write_h5(datasetDict, out_file, metadata=meta, ref_file=None, compression=None) - - - datasetDict = dict() - #datasetDict['azimuth'] = az_grid_large - #datasetDict['range'] = rg_grid_large - datasetDict['coherence'] = cc_grid_large - #meta['FILE_TYPE'] = 'coherence' - out_file = Pair + '_pot_cc.h5' - write_h5(datasetDict, out_file, metadata=meta, ref_file=None, compression=None) - print("POT calculation done: " + Pair) - #sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/pot_gamma_subset_jobs.py b/.codex_tmp/pyint_variants/no_rescue/pyint/pot_gamma_subset_jobs.py deleted file mode 100644 index f6bd3c7..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/pot_gamma_subset_jobs.py +++ /dev/null @@ -1,95 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2022, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import getopt -import time -import glob -import argparse - -from pyint import _utils as ut - - -INTRODUCTION = ''' -------------------------------------------------------------------- - Pixel offset tracking for co-registered Images. TOPs should be deramped in advance. - -''' - -EXAMPLE = ''' - Usage: - pot_gamma_subset_jobs.py projectName Mdate Sdate --memory memory_single_job --walltime walltime_single_job - pot_gamma_subset_jobs.py PacayaT163TsxHhA 20150102 20150601 --memory 5000 --walltime 00:30:00 -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Pixel offset tracking based Azimuth/Range dispalcement estimation.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('Mdate',help='Master date.') - parser.add_argument('Sdate',help='Slave date.') - parser.add_argument('--memory',dest = 'memory',default = '5000', help='memory to be allocated for one job, default: 5GB') - parser.add_argument('--walltime',dest = 'walltime',default = '00:30:00', help='walltime to be allocated for one single job') - inps = parser.parse_args() - inps = parser.parse_args() - return inps - - -def main(argv): - - start_time = time.time() - inps = cmdLineParse() - Mdate = inps.Mdate - Sdate = inps.Sdate - - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - - - slcDir = scratchDir + '/' + projectName + '/SLC' - rslcDir = scratchDir + '/' + projectName + '/RSLC' - - MrslcDir = rslcDir + '/' + Mdate - SrslcDir = rslcDir + '/' + Sdate - Pair = Mdate + '-' + Sdate - batch_txt = scratchDir + '/' + projectName + '/ifgrams/' + Pair + '/Batch_POT' - workDir = scratchDir + '/' + projectName + '/ifgrams/' + Pair - if not os.path.isdir(workDir): - os.mkdir(workDir) - os.chdir(workDir) - if os.path.isfile(batch_txt): - os.remove(batch_txt) - - mrslc_list = glob.glob(MrslcDir + '/*_0*.rslc');mrslc_list.sort(); nn = len(mrslc_list) - print('Total patch numbers: ' + str(nn)) - - for i in range(nn): - base0 = os.path.basename(mrslc_list[i]); subset1 = base0.split('_')[1]; subset0 = subset1.split('.rslc')[0] - str0 = 'pot_gamma_subset.py ' + projectName + ' ' + Mdate + ' ' + Sdate + ' ' + subset0 - call_str = 'echo ' + str0 + ' >>' + batch_txt - os.system(call_str) - - call_str = 'sbatch_jobs.py ' + batch_txt + ' --memory ' + inps.memory + ' --walltime ' + inps.walltime + ' --job-name ' + Pair + '_POT' - os.system(call_str) - - - print("Pixel offset tracking is Done!") - ut.print_process_time(start_time, time.time()) - #sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/process_tsifg.py b/.codex_tmp/pyint_variants/no_rescue/pyint/process_tsifg.py deleted file mode 100644 index 7f3d95d..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/process_tsifg.py +++ /dev/null @@ -1,148 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v1.0 ### -### Copy Right (c): 2017, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Email : ymcmrs@gmail.com ### -### Univ. : Central South University & University of Miami ### -################################################################# - -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob - -def check_variable_name(path): - s=path.split("/")[0] - if len(s)>0 and s[0]=="$": - p0=os.getenv(s[1:]) - path=path.replace(path.split("/")[0],p0) - return path - -def read_template(File, delimiter='='): - '''Reads the template file into a python dictionary structure. - Input : string, full path to the template file - Output: dictionary, pysar template content - Example: - tmpl = read_template(KyushuT424F610_640AlosA.template) - tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') - ''' - template_dict = {} - for line in open(File): - line = line.strip() - c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter - if len(c) < 2 or line.startswith('%') or line.startswith('#'): - next #ignore commented lines or those without variables - else: - atrName = c[0] - atrValue = str.replace(c[1],'\n','').split("#")[0].strip() - atrValue = check_variable_name(atrValue) - template_dict[atrName] = atrValue - return template_dict - -def is_number(s): - try: - int(s) - return True - except ValueError: - return False - - -def ras2jpg(input, strTitle): - call_str = "convert " + input + ".ras " + input + ".jpg" - os.system(call_str) - call_str = "convert " + input + ".jpg -resize 250 " + input + ".thumb.jpg" - os.system(call_str) - call_str = "convert " + input + ".jpg -resize 500 " + input + ".bthumb.jpg" - os.system(call_str) - call_str = "$INT_SCR/addtitle2jpg.pl " + input + ".thumb.jpg 14 " + strTitle - os.system(call_str) - call_str = "$INT_SCR/addtitle2jpg.pl " + input + ".bthumb.jpg 24 " + strTitle - os.system(call_str) - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close() - -def write_template(File, Str): - f = open(File,'a') - f.write(Str) - f.close() - -def write_run_coreg_all(projectName,master,slavelist,workdir): - scratchDir = os.getenv('SCRATCHDIR') - projectDir = scratchDir + '/' + projectName - run_coreg_all = projectDir + "/run_coreg_all" - f_coreg = open(run_coreg_all,'w') - - for kk in range(len(slavelist)): - str_coreg = "GenOff_Gamma.py " + projectName + ' ' + master + ' ' + slavelist[kk] + ' ' + workdir + '\n' - f_coreg.write(str_coreg) - f_coreg.close() - - -def usage(): - print(''' -****************************************************************************************************** - - Process time series of interferograms from downloading data or SLC images. - - usage: - - process_tsifg.py projectName - - e.g. process_tsifg.py PacayaT163TsxHhA - process_tsifg.py PacayaT163S1A - -******************************************************************************************************* - ''') - -def main(argv): - - if len(sys.argv)==2: - if argv[0] in ['-h','--help']: usage(); sys.exit(1) - else: projectName=sys.argv[1] - else: - usage();sys.exit(1) - - if 'S1' in projectName: - call_str='process_tsifg_sen.py ' + projectName - else: - call_str='process_tsifg_gamma.py ' + projectName - - os.system(call_str) - - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - - - - - - - - - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/psokinv2sdm.py b/.codex_tmp/pyint_variants/no_rescue/pyint/psokinv2sdm.py deleted file mode 100644 index 2da39ea..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/psokinv2sdm.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python - -import numpy as np -import os -import sys -import argparse -import subprocess -from pyint import _utils as ut - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='convert psokinv quadtree result to SDM.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('inputfile', help='psokinv quadtree file name ') - parser.add_argument('outfile', help=' file name for SDM ') - inps = parser.parse_args() - return inps -INTRODUCTION = ''' -------------------------------------------------------------------- - create_files for SDM software running file by generated by psokinv -''' - -EXAMPLE = """Usage: - - PSOKINV2SDM.py inputfile outfile - - create_psokinv.py 20241015_20250107.inp 20241015_20250107_los_for_sdm -------------------------------------------------------------------- -""" - -def phitheta2neu(phi,theta): - # - import numpy as np - # - deg2rad=np.pi/180. - # - theta=theta*deg2rad; phi=phi*deg2rad; - # - vn= np.sin(theta)*np.sin(phi) - ve=-np.sin(theta)*np.cos(phi) - vu= np.cos(theta) - # - return vn,ve,vu -# -def neu2phitheta(vn,ve,vu): - # - import numpy as np - # - rad2deg=180./np.pi; - # - theta=np.arccos(vu)*rad2deg; phi=np.arctan2(vn,-ve)*rad2deg; - # - return theta,phi -# -def psokinv2sdm(file,file_out): - # - import numpy as np - # - data=np.genfromtxt(file); - length=len(data) - inc=[] - azi=[] - out_data={} - for i in range(length): - (inc,azi)=neu2phitheta(data[i,4],data[i,3],data[i,5]); - out_data[i]=(data[i,0],data[i,1],data[i,2],inc,azi); - data1=np.array(list(out_data.values())) - txt_data = np.savetxt(file_out,data1,fmt='%.8f') - - -def main(argv): - inps = cmdLineParse() - input_file = inps.inputfile - out_file = inps.outfile - psokinv2sdm(input_file, out_file) - print("convert psokinv products to sdm is done!") - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/pyint.template b/.codex_tmp/pyint_variants/no_rescue/pyint/pyint.template deleted file mode 100644 index fe85f8b..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/pyint.template +++ /dev/null @@ -1,206 +0,0 @@ -# *********************************** PyINT Template *******************************************# -# Template for PyINT v2.1 -# Please fill in the parameters according to your project requirements - -# ========================= Basic parameters ======================== - -# DEM file path (if not provided, SRTM will be downloaded automatically from OpenTopography) -#DEM = /path/to/your/dem.dem - -# Master date for interferometric processing -masterDate = YYYYMMDD # Master date in YYYYMMDD format - -# Swath and burst parameters (for Sentinel-1 TOPS mode only) -start_swath = 1 # Starting swath number -end_swath = 3 # Ending swath number -start_burst = 1 # Starting burst position (relative to master image) -end_burst = 9 # Ending burst position (relative to master image) - -# Multilooking parameters -range_looks = 20 # Multi-look numbers in range direction -azimuth_looks = 4 # Multi-look numbers in azimuth direction - -# ========================= Download Data ======================== - -# Data source and search parameters -Data_Hub = ASF # Data download source: SSARA, Scihub, ASF -sensor = Sentinel-1A # Envisat, ERS-1/2, Sentinel-1A/B, etc. -track = 128 # Track number of the SAR image -frame = 89 # Frame number [defined according to ASF] -start_time = YYYY-MM-DD # Start date for data search -end_time = YYYY-MM-DD # End date for data search - -# ASF download specific parameters -asf_platform = Sentinel-1A # Platform: Sentinel-1A, Sentinel-1B -asf_beam_mode = IW # Beam mode: IW (Interferometric Wide) -asf_polarization = VV # Polarization: VV, VH, HH, HV -asf_flight_direction = ascending # Flight direction: ascending, descending -asf_bbox = W,S,E,N # Bounding box: West,South,East,North - -# ========================= DEM Generation ======================== - -# DEM download parameters (for OpenTopography) -opentopo_api_key = YOUR_API_KEY # OpenTopography API key -opentopo_dem_type = SRTMGL1 # DEM type: SRTMGL1(30m), SRTMGL3(90m), NASADEM(30m), COP30, COP90 -opentopo_auto_tile = 1 # Auto-tile large regions (recommended: 1) - -# DEM processing parameters -dem_lat_ovr = 1 # Latitude oversampling (as to SRTM-1: 0.5 for 60m, 2 for 15m) -dem_lon_ovr = 1 # Longitude oversampling - -# ========================= Interferometry Parameters ======================== - -# Spectral filtering -Igram_Spsflg = 1 # Implement range spectral filtering -rbw_min = 1 # Minimum range bandwidth fraction (0.1 --> 1.0) - -# Coregistration parameters -rwin4cor = 256 # Range window length for coregistration -azwin4cor = 256 # Azimuth window length for coregistration -rsample4cor = 32 # Range samples for fitting coregistration parameters -azsample4cor = 32 # Azimuth samples for fitting coregistration parameters -thresh4cor = 0.3 # Coherence threshold (GAMMA 2016+, SNR for older versions) - -# Coherence estimation -Igram_Cor_rwin = 5 # Range window length for cc_wave coherence estimation -Igram_Cor_awin = 5 # Azimuth window length for cc_wave coherence estimation -Igram_Cor_Win = 5 # Window for cc estimation in adf -adf_alpha = 0.4 # Alpha for Gold-Stein filtering - -# ========================= Simulation Phase ======================== - -Igram_Flag_TDM = N # Y for Tandem-X -Simphase_rpos = - -Simphase_azpos = - -Simphase_rwin = 256 -Simphase_azwin = 256 -Simphase_thresh = - - -# ========================= Unwrapping ======================== - -mcf_triangular = 1 # Triangular type of mcf [0: regular; 1: delaunay] -unwrap_patr = 1 # Unwrap patches in range direction -unwrap_pataz = 1 # Unwrap patches in azimuth direction -unwrapThreshold = 0.05 # Coherence threshold for unwrapping -auto_unw = 1 # Auto or manual reference point [1: auto; 0: manual] -make_mask = 0 # Make mask based on coherence [0: don't mask; 1: make mask] -init_flag = 1 # Flag to set phase at reference point [0: use initial; 1: set to 0.0] -r_refer = - # Phase reference point range offset -a_refer = - # Phase reference point azimuth offset - -# ========================= Geocoding ======================== - -geo_interp = 0 # Interpolation method [0: nearest; 1: bicubic spline] - -# ========================= Satellite Configuration ======================== - -satelite = S1A # Satellite: S1A, S1B, CSK, TSX, ALOS2, ALOS, ENVISAT - -# ========================= Pair Selection ======================== - -network_method = sbas # Network method: sbas, sequential, delaunay, stars -endDate = YYYYMMDD # Exclude dates after this date -startDate = YYYYMMDD # Exclude dates before this date -conNumb = 15 # Connect number for sequential network -max_tb = 365 # Maximum temporal baseline (days) -max_sb = 100 # Maximum spatial baseline (meters) -min_tb = 1 # Minimum temporal baseline (days) -min_sb = 0 # Minimum spatial baseline (meters) - -# ========================= Phase Gradient (Optional) ======================== - -dataformat = cpxfloat32 # Data format of interferogram -scale = 10000 # Original phase/scale -step_windows = 1 # Step length for spatial phase gradient -filter_wins = [7, 7] # Windows of median filter -directions = ['grad_east', 'grad_north', 'grad_northeast', 'grad_southeast'] - -# Mapping parameters for phase gradient -in_low_high = [0.001, 1] # Input range for mapping -out_low_high = [0.001, 1] # Output range for mapping -gamma = 0.8 # Gamma for curve shape -choose_linear = 'n' # Linear or nonlinear mapping ['y': linear; 'n': nonlinear] - -# ========================= Hyp3 Format Conversion (NEW) ======================== - -# Convert GAMMA outputs to Hyp3-compatible GeoTIFF format -hyp3format = 0 # Enable Hyp3 format conversion [0: skip; 1: process] -hyp3format_parallel = 4 # Number of parallel processors for Hyp3 conversion -hyp3_output_dir = - # Output directory for Hyp3 products [default: projectName/Hyp3Products/] - -# ========================= Phase Bias Correction (NEW) ======================== - -# Apply phase bias correction based on loop closures algorithm -# Reference: 10.1016/j.remote.2022.100013 - -phasebias = 0 # Enable phase bias correction [0: skip; 1: process] -phasebias_interval = 12 # Temporal baseline in days [6, 12, or 24] -phasebias_nlook = 10 # Number of looks for multilooking -phasebias_num_a = 2 # Number of calibration parameters to estimate -phasebias_estimate_an = 0 # Estimate an from data [0: use defaults; 1: estimate from loop closures] -phasebias_max_con = 5 # Maximum number of connections - -# Default an values (used if phasebias_estimate_an = 0) -a1_6_day = 0.50 -a2_6_day = 0.36 -a3_6_day = 0.299 -a4_6_day = 0.2476 - -a1_12_day = 0.494 -a2_12_day = 0.297 -a3_12_day = 0.24 -a4_12_day = 0.22 - -a1_24_day = 0.48 -a2_24_day = 0.28 -a3_24_day = 0.22 -a4_24_day = 0.20 - -phasebias_start = - # Start date YYYYMMDD [default: auto-detect] -phasebias_end = - # End date YYYYMMDD [default: auto-detect] - -# ========================= Atmospheric Correction ======================== - -# GACOS tropospheric correction -gacos_correction = 0 # Enable GACOS correction [0: skip; 1: process] -gacos_all_parallel = 1 # Number of parallel processors (1=sequential with ZTD pre-cache, recommended) -gacos_dir = # Directory for GACOS ZTD files [default: projectName/GACOS] -gacos_email = # Email for GACOS auto-download (optional) - -# Ionospheric correction -ionosphere_correction = 0 # Enable ionospheric correction [0: skip; 1: process] -ionosphere_parallel = 4 # Number of parallel processors - -# ========================= Processing Workflow Control ======================== - -download_data = 0 # Enable data download [0: skip; 1: process] -down_parallel = 4 # Number of parallel processors for download - -raw2slc_all = 1 # Convert raw to SLC [0: skip; 1: process] -raw2slc_all_parallel = 4 # Number of parallel processors - -extract_burst_all = 1 # Extract bursts [0: skip; 1: process] -extract_all_parallel = 4 # Number of parallel processors - -coreg_all = 1 # Coregister SLCs [0: skip; 1: process] -coreg_all_parallel = 4 # Number of parallel processors - -select_pairs = 1 # Select interferometric pairs [0: skip; 1: process] - -diff_all = 1 # Generate differential interferograms [0: skip; 1: process] -diff_all_parallel = 4 # Number of parallel processors - -unwrap_all = 1 # Unwrap interferograms [0: skip; 1: process] -unwrap_all_parallel = 4 # Number of parallel processors - -geocode_all = 1 # Geocode products [0: skip; 1: process] -geocode_all_parallel = 4 # Number of parallel processors - -hyp3format_all = 0 # Convert to Hyp3 format [0: skip; 1: process] -hyp3format_all_parallel = 4 # Number of parallel processors - -phasebias_all = 0 # Apply phase bias correction [0: skip; 1: process] - -load_data = 0 # Load data for MintPy processing [0: skip; 1: process] - -# ========================= Good Luck ======================================= diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/pyintApp.py b/.codex_tmp/pyint_variants/no_rescue/pyint/pyintApp.py deleted file mode 100644 index 1ad8c15..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/pyintApp.py +++ /dev/null @@ -1,333 +0,0 @@ -#! /usr/bin/env python -########################################################### -# Project: PyINT # -# Purpose: Interferograms process using python/GAMMA # -# Author: Yunmeng Cao # -# Created: Feb. 2017 # -# Contact : ymcmrs@gmail.com # -# Copy Right (c): 2017-2019, Yunmeng Cao # -########################################################### - -import numpy as np -import os -import sys -import subprocess -import time -import argparse - -from pyint import _utils as ut - - -def _run_or_raise(call_str, stage): - rc = os.system(call_str) - if rc != 0: - raise RuntimeError('%s failed with rc=%s: %s' % (stage, rc, call_str)) - return rc - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Interferograms processing using PyINT.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName', help='name of the project.') - parser.add_argument('-g', '--generate', action='store_true', dest='generate_structure', \ - help='Generate project directory structure if not exists.') - - inps = parser.parse_args() - return inps - - -INTRODUCTION = ''' ------------------------------------------------------------------------------------ - - Single or time-series of interferometry processing for satellite based - Synthetic Aperture Radar (SAR) images start from downloading data to - generate unwrapped-differential interferograms. - - Details please check: https://github.com/ymcmrs/PyINT - - General work-flow: - - 1) download data : download SLCs using SSARA/Scihub/ASF - SSARA: https://github.com/bakerunavco/SSARA - Scihub: Copernicus Open Access Hub - ASF: Alaska Satellite Facility (for Sentinel-1) - [You should provide Sensor, Track, Frame, or Time information in template] - 2) generate SLC : raw 2 slc (multi-frame processing is also supported) - [include orbit correction for S1,ASAR,ERS and burst-extraction for S1] - 3) generate DEM : reference image related geo-dem, rdc-dem, lookup table will be generated. - [SRTM-1 will be downloaded and processed automatically if not provided] - 4) coregister SLC : coregister SLCs to the reference SLC iamge. - [with assistant of DEM] - 5) select pairs : select interferometric pairs for time-series processing. - [networks of sbas, sequential, delaunay, and stars are supported] - 6) interferometry : generate unwrapped differential interferograms. - [include differential, unwrapping, and geocoding] - 7) offset tracking: pixel offset tracking (POT) for displacement measurement. - [two-round estimation with GAMMA offset_pwr_tracking] - 8) GACOS correction: apply GACOS tropospheric atmospheric correction. - [requires GACOS ZTD data for master and slave dates] - - Note: - - i) Single interferogram processing please use slc2ifg.py or raw2ifg.py - ii) Multi-processor parallel processing is supported, but keep in mind GAMMA calls multi-threads already. - iii) ASF download supports both bounding box and Shapefile for spatial filtering. - iv) GACOS correction requires geocoded interferograms (set geocode_all=1 first). - -''' - -EXAMPLE = """Usage: - - pyintApp.py -h - pyintApp.py projectName #[projectName.template should be available in TEMPLATEDIR] - pyintApp.py -g projectName #Generate project directory structure if not exists - ------------------------------------------------------------------------------------- -""" - -def main(argv): - - start_time = time.time() - inps = cmdLineParse() - projectName = inps.projectName - templateDir = os.getenv('TEMPLATEDIR') - scratchDir = os.getenv('SCRATCHDIR') - workDir = scratchDir + '/' + projectName - - # Generate project directory structure if -g flag is set and directory doesn't exist - if inps.generate_structure: - if not os.path.isdir(workDir): - print('Generating project directory structure for: %s' % projectName) - os.makedirs(workDir, exist_ok=True) - os.makedirs(workDir + '/DOWNLOAD', exist_ok=True) - os.makedirs(workDir + '/SLC', exist_ok=True) - os.makedirs(workDir + '/RSLC', exist_ok=True) - os.makedirs(workDir + '/DEM', exist_ok=True) - os.makedirs(workDir + '/ifgrams', exist_ok=True) - print('Project directory structure created successfully.') - print('Directory structure:') - print(' %s/' % projectName) - print(' ├── DOWNLOAD/') - print(' ├── SLC/') - print(' ├── RSLC/') - print(' ├── DEM/') - print(' └── ifgrams/') - return - else: - print('Project directory already exists: %s' % workDir) - print('Skipping directory structure generation.') - return - - templateFile = templateDir + "/" + projectName + ".template" - templateDict = ut.update_template(templateFile) - masterDate = templateDict['masterDate'] - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - satelite = templateDict['satelite'] - HGTSIM = scratchDir + '/' + projectName + '/DEM/' + ut.yyyymmdd(masterDate) + '_' + rlks + 'rlks.rdc.dem' - DataDir = workDir + '/' + 'DOWNLOAD' - DEMDir = workDir + '/' + 'DEM' - os.chdir(scratchDir) - if not os.path.isdir(workDir): - os.mkdir(workDir) - os.chdir(workDir) - os.mkdir(DataDir) - os.mkdir(DEMDir) - os.chdir(DataDir) - else: - os.chdir(DataDir) - ### download data - if templateDict['download_data'] == '1': - if templateDict['Data_Hub'] == 'SSARA': - print('Start to download SAR data using SSARA...') - call_str = 'ssara_federated_query.py -p ' + templateDict['sensor'] + ' -r ' + templateDict['track'] + ' -f ' + templateDict['frame'] + ' -s ' + templateDict['start_time'] + ' -e ' + templateDict['end_time'] + ' --print --download --parallel ' + templateDict['down_parallel'] - print(call_str) - _run_or_raise(call_str, 'download_data_ssara') - elif templateDict['Data_Hub'] == 'Scihub': - print('Start to downlad SAR data from Scihub') - call_str = 'scihub_search_s1_data.py ' + projectName + ' -s ' + templateDict['start_time'] + ' -e ' + templateDict['end_time'] + ' -p ' + templateDict['producttype'] +' -r ' + templateDict['region_box'] + ' -n ' + templateDict['Orbit_number'] + ' -d ' + templateDict['Direction'] + ' -sm ' + templateDict['Sence_model'] + ' -o ' + templateDict['output_file'] - print(call_str) - _run_or_raise(call_str, 'download_data_scihub') - elif templateDict['Data_Hub'] == 'ASF': - print('Start to download Sentinel-1 SLC data from ASF...') - # 获取必要的参数 - path_SLC = DataDir - path_RSLC = workDir + '/RSLC' - date_start = templateDict.get('start_time', '20170101') - date_end = templateDict.get('end_time', '20240101') - - # 构建基本命令 - call_str = 'API_download_S1_SLC.py -s ' + path_SLC + ' -r ' + path_RSLC + ' -i ' + date_start + ' -j ' + date_end - - # 添加用户名和密码（如果提供） - if 'asf_username' in templateDict and templateDict['asf_username'].strip(): - call_str += ' -u ' + templateDict['asf_username'] - if 'asf_password' in templateDict and templateDict['asf_password'].strip(): - call_str += ' -p ' + templateDict['asf_password'] - - # 添加边界框或Shapefile - if 'shapefile' in templateDict and templateDict['shapefile'].strip(): - call_str += ' --shp ' + templateDict['shapefile'] - elif 'bbox' in templateDict and templateDict['bbox'].strip(): - call_str += ' -b ' + templateDict['bbox'] - elif 'asf_bbox' in templateDict and templateDict['asf_bbox'].strip(): - call_str += ' -b ' + templateDict['asf_bbox'] - - # 添加轨道号（可选） - if 'relative_orbit' in templateDict and templateDict['relative_orbit'].strip(): - call_str += ' -o ' + str(templateDict['relative_orbit']) - elif 'track' in templateDict and templateDict['track'].strip(): - call_str += ' -o ' + str(templateDict['track']) - - # 添加帧号（可选） - if 'frame' in templateDict and templateDict['frame'].strip(): - call_str += ' --frame ' + str(templateDict['frame']) - - # 添加飞行方向（可选，默认为升轨） - if 'flight_direction' in templateDict: - call_str += ' -f ' + templateDict['flight_direction'] - else: - call_str += ' -f a' # 默认升轨 - - # 添加波束模式（默认IW） - if 'acquisition_mode' in templateDict: - call_str += ' -q ' + templateDict['acquisition_mode'] - else: - call_str += ' -q IW' # 默认IW模式 - - # 输出格式和下载选项 - call_str += ' -m csv -w Y' - - # 添加并行下载参数 - if 'down_parallel' in templateDict and templateDict['down_parallel'].strip(): - call_str += ' --parallel ' + str(templateDict['down_parallel']) - - print(call_str) - _run_or_raise(call_str, 'download_data_asf') - ### raw 2 slc - if templateDict['raw2slc_all'] == '1': # only for S1 data now - print('Start to convert downloaded-raw data into SLC ...') - print('Number of processor: %s' % str(templateDict['raw2slc_all_parallel'])) - if satelite=='S1A': - call_str = 'down2slc_sen_all.py ' + projectName + ' --parallel ' + templateDict['raw2slc_all_parallel'] - _run_or_raise(call_str, 'raw2slc_s1') - elif satelite=='ALOS': - call_str = 'down2slc_alos_all.py ' + projectName + ' --parallel ' + templateDict['raw2slc_all_parallel'] - _run_or_raise(call_str, 'raw2slc_alos') - elif satelite=='LT': - call_str = 'down2slc_LT1_all.py ' + projectName + ' --parallel ' + templateDict['raw2slc_all_parallel'] - _run_or_raise(call_str, 'raw2slc_lt1') - ### extract bursts - if templateDict['extract_burst_all'] == '1': - print('Start to extract common bursts ...') - print('Number of processor: %s' % str(templateDict['extract_all_parallel'])) - call_str = 'extract_s1_bursts_all.py ' + projectName + ' --parallel ' + templateDict['extract_all_parallel'] - _run_or_raise(call_str, 'extract_burst_all') - - ### generate rdc_dem - if not os.path.isfile(HGTSIM): - print('Start to generate geometry file ...') - call_str = 'makedem_pyint.py ' + projectName - _run_or_raise(call_str, 'makedem_pyint') - call_str = 'generate_rdc_dem.py ' + projectName - _run_or_raise(call_str, 'generate_rdc_dem') - - ### coreg SLC - if templateDict['coreg_all'] == '1': - print('Start to coregister SLCs ...') - print('Number of processor: %s' % str(templateDict['coreg_all_parallel'])) - call_str = 'coreg_gamma_all.py ' + projectName + ' --parallel ' + templateDict['coreg_all_parallel'] - _run_or_raise(call_str, 'coreg_all') - - ### select interferometric pairs - if templateDict['select_pairs'] == '1': - print('Start to select interferometric pairs ...') - print('Network selection method: %s' % templateDict['network_method']) - #print('Meximum temporal baseline threshold: %s' % templateDict['max_tb']) - #print('Meximum spatial baseline threshold: %s' % templateDict['max_sb']) - call_str = 'select_pairs.py ' + projectName - _run_or_raise(call_str, 'select_pairs') - - ### diff ifg - if templateDict['diff_all'] == '1': - print('Start to generate differential interferograms ...') - print('Number of processor: %s' % str(templateDict['diff_all_parallel'])) - call_str = 'diff_gamma_all.py ' + projectName + ' --parallel ' + templateDict['diff_all_parallel'] - _run_or_raise(call_str, 'diff_all') - - ### Pixel Offset Tracking (POT) - if templateDict['pot_all'] == '1': - print('Start to run Pixel Offset Tracking (POT) ...') - print('Number of processor: %s' % str(templateDict['pot_all_parallel'])) - call_str = 'POT_gamma_all.py ' + projectName + ' --parallel ' + templateDict['pot_all_parallel'] - _run_or_raise(call_str, 'pot_all') - - ### unw ifg - if templateDict['unwrap_all'] == '1': - print('Start to unwrap interferometric phases ...') - print('Number of processor: %s' % str(templateDict['unwrap_all_parallel'])) - call_str = 'unwrap_gamma_all.py ' + projectName + ' --parallel ' + templateDict['unwrap_all_parallel'] - _run_or_raise(call_str, 'unwrap_all') - - ### atmcor ifg - if templateDict['atmcor_all'] == '1': - print('Start to correct atmospheric phase ...') - print('Number of processor: %s' % str(templateDict['atmcor_all_parallel'])) - call_str = 'atm_correction_gamma_all.py ' + projectName + ' --parallel ' + templateDict['atmcor_all_parallel'] - _run_or_raise(call_str, 'atmcor_all') - - ### geocode ifg - if templateDict['geocode_all'] == '1': - print('Start to geocode Ifgs ...') - print('Number of processor: %s' % str(templateDict['geocode_all_parallel'])) - call_str = 'geocode_gamma_all.py ' + projectName + ' --parallel ' + templateDict['geocode_all_parallel'] - _run_or_raise(call_str, 'geocode_all') - - ### Convert GAMMA outputs to LiCSBAS format - if templateDict.get('gamma2licsbas_all', '0') == '1': - print('Start to convert GAMMA outputs to LiCSBAS format ...') - print('Number of processor: %s' % str(templateDict.get('gamma2licsbas_all_parallel', '1'))) - call_str = 'gamma2licsbas_gamma_all.py ' + projectName + ' --parallel ' + templateDict.get('gamma2licsbas_all_parallel', '1') - _run_or_raise(call_str, 'gamma2licsbas_all') - - ### GACOS atmospheric correction (after geocoding) - if templateDict['gacos_correction'] == '1': - print('Start to apply GACOS atmospheric correction ...') - gacos_dir = templateDict.get('gacos_dir', '').strip() - gacos_email = templateDict.get('gacos_email', '').strip() - - call_str = 'gacos_gamma_all.py ' + projectName + ' --skip-existing' - if gacos_dir: - call_str += ' --ztd-dir ' + gacos_dir - if gacos_email: - call_str += ' --email ' + gacos_email - print(call_str) - _run_or_raise(call_str, 'gacos_correction') - - ### Convert GAMMA outputs to HyP3 UTM format - if templateDict.get('hyp3format_all', '0') == '1': - print('Start to convert GAMMA outputs to HyP3 UTM format ...') - print('Number of processor: %s' % str(templateDict.get('hyp3format_all_parallel', '1'))) - call_str = 'hyp3format_gamma_all.py ' + projectName + ' --parallel ' + templateDict.get('hyp3format_all_parallel', '1') - _run_or_raise(call_str, 'hyp3format_all') - - ### load data - if templateDict['load_data'] == '1': - print('Start to load data for mintPy time-series analysis ...') - call_str = 'load_mintpy.py ' + projectName - _run_or_raise(call_str, 'load_data') - - print("PyINT processing for project %s is done." % projectName) - ut.print_process_time(start_time, time.time()) - sys.exit(0) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/raw2ifg.py b/.codex_tmp/pyint_variants/no_rescue/pyint/raw2ifg.py deleted file mode 100644 index 0835ffe..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/raw2ifg.py +++ /dev/null @@ -1,70 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import subprocess -import time -import argparse - -from pyint import _utils as ut - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Generate unwrapped differential Ifg from SLC-raw data using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName', help='name of the project.') - parser.add_argument('Mdate',help='master date for interferometry.') - parser.add_argument('Sdate',help='slave date for interferometry.') - - inps = parser.parse_args() - - return inps - - -INTRODUCTION = ''' --------------------------------------------------------------------------------------- - Generate unwrapped differential Ifg from SLC-raw data using GAMMA. - - Note: 1) Precise orbit data will be downloaded and processed automatically - 2) SRTM-1 will be downloaded and processed automatically if not provided - in the template file. - -''' - -EXAMPLE = """Usage: - - raw2ifg.py projectName Mdate Sdate ---------------------------------------------------------------------------------------- -""" - -def main(argv): - - inps = cmdLineParse() - Mdate = inps.Mdate - Sdate = inps.Sdate - - projectName = inps.projectName - - if 'S1' in projectName: - call_str = 'raw2ifg_s1.py ' + projectName + ' ' + ut.yyyymmdd(Mdate) + ' ' + ut.yyyymmdd(Sdate) - os.system(call_str) - - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/raw2ifg_s1.py b/.codex_tmp/pyint_variants/no_rescue/pyint/raw2ifg_s1.py deleted file mode 100644 index f192afd..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/raw2ifg_s1.py +++ /dev/null @@ -1,132 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import subprocess -import time -import glob -import argparse - -from pyint import _utils as ut - - -def get_s1_date(raw_file): - file0 = os.path.basename(raw_file) - date = file0[17:25] - return date - -def get_satellite(raw_file): - if 'S1A_IW_SLC_' in raw_file: - s0 = 'A' - else: - s0 = 'B' - - return s0 - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Generate Ifg from Sentinel-1 raw data with orbit correction using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName', help='name of the project.') - parser.add_argument('Mdate',help='master date for interferometry.') - parser.add_argument('Sdate',help='slave date for interferometry.') - - inps = parser.parse_args() - - return inps - - -INTRODUCTION = ''' ---------------------------------------------------------------------------------------------------- - Generate unwrapped differential Ifg from Sentinel-1 raw data with orbit correction using GAMMA. - - Note: 1) Precise orbit data will be downloaded and processed automatically - 2) SRTM-1 will be downloaded and processed automatically if not provided - in the template file. - -''' - -EXAMPLE = """Usage: - - raw2ifg_s1.py projectName Mdate Sdate ----------------------------------------------------------------------------------------------------- -""" - -def main(argv): - - start_time = time.time() - inps = cmdLineParse() - Mdate = inps.Mdate - Sdate = inps.Sdate - - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - masterDate = templateDict['masterDate'] - downDir = scratchDir + '/' + projectName + '/DOWNLOAD' - M_raw = glob.glob(downDir + '/S1*_' + ut.yyyymmdd(Mdate)+'*')[0] - S_raw = glob.glob(downDir + '/S1*_' + ut.yyyymmdd(Sdate)+'*')[0] - slcDir = scratchDir + '/' + projectName + '/SLC' - - - ######### down 2 slc ############# - call_str = 'down2slc_sen.py ' + M_raw + ' ' + slcDir - os.system(call_str) - - call_str = 'down2slc_sen.py ' + S_raw + ' ' + slcDir - os.system(call_str) - - ########## extract common bursts ## - call_str = 'extract_s1_bursts.py ' + projectName + ' ' + Mdate - os.system(call_str) - - call_str = 'extract_s1_bursts.py ' + projectName + ' ' + Sdate - os.system(call_str) - - ######### generate rdc_dem ########## - call_str = 'generate_rdc_dem.py ' + projectName - os.system(call_str) - - ########## coregister SLC ######## - - call_str = 'coreg_s1_gamma.py ' + projectName + ' ' + Mdate - os.system(call_str) - - call_str = 'coreg_s1_gamma.py ' + projectName + ' ' + Sdate - os.system(call_str) - - ######## Interferometry process ########### - call_str = 'diff_gamma.py ' + projectName + ' ' + Mdate + ' ' + Sdate - os.system(call_str) - - call_str = 'unwrap_gamma.py ' + projectName + ' ' + Mdate + ' ' + Sdate - os.system(call_str) - - call_str = 'geocode_gamma.py ' + projectName + ' ' + Mdate + '-' + Sdate - os.system(call_str) - - print("Generate Ifg from raw-TOPs data is done! ") - ut.print_process_time(start_time, time.time()) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/raw2slc_ers_envisat.py b/.codex_tmp/pyint_variants/no_rescue/pyint/raw2slc_ers_envisat.py deleted file mode 100644 index 6384f3f..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/raw2slc_ers_envisat.py +++ /dev/null @@ -1,186 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.0 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Email : ymcmrs@gmail.com ### -### Univ. : Now at KAUST ### -################################################################# - -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob -import argparse -import linecache -import datetime - -def StrNum(S): - S = str(S) - if len(S)==1: - S='0' +S - return S - -def UseGamma(inFile, task, keyword): - if task == "read": - f = open(inFile, "r") - while 1: - line = f.readline() - if not line: break - if line.count(keyword) == 1: - strtemp = line.split(":") - value = strtemp[1].strip() - return value - print("Keyword " + keyword + " doesn't exist in " + inFile) - f.close() - - -def check_ERS_par(SAR_IM_0P): - Name = os.path.basename(SAR_IM_0P) - ff = Name.split('.')[1] - date = (Name.split('SAR_IM__0PWDSI')[1]).split('_')[0] - - if ff == 'E1': - par = 'ERS1_ESA.par' - antenna = 'ERS1_antenna.gain' - orbdir = os.getenv('ERS1ORBDIR') - elif ff == 'E2': - par = 'ERS2_ESA.par' - antenna = 'ERS2_antenna.gain' - orbdir = os.getenv('ERS2ORBDIR') - else: - print('Invalid input SAR_IM_0P file.') - sys.exit(1) - - return par, antenna, orbdir, date - -######################################################################### - -INTRODUCTION = ''' -############################################################################# - Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 - - Generate SLC for ERS raw data with ENVISAT format - -''' - -EXAMPLE = ''' - Usage: - raw2slc_ers_envisat.py SAR_IM_OP -o output_prefix - - Examples: - raw2slc_ers_envisat.py SAR_IM_OP -o 19950102 - -############################################################################## -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Generate SLC for ERS raw data with ENVISAT format.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('ers_envisat_raw',help='ERS raw data with ENVISAT format, e.g., SAR_IM_0PXXXX.E1') - parser.add_argument('-o',dest='out_put',help='prefix of the output file, e.g., output.slc, output.slc.par') - - inps = parser.parse_args() - return inps - -################################################################################ - - -def main(argv): - - inps = cmdLineParse() - SAR_IM_0P = inps.ers_envisat_raw - par,antenna,orbdir,date = check_ERS_par(SAR_IM_0P) - - raw = date + '.raw' - azsp = date + '.azsp' - dop = date + '.dop' - rspec = date + '.rspec' - rc = date + '.rc' - autof = date + '.autof' - dop_ambig = date + '.dop_ambig' - - if not os.path.isfile(par): - call_str = 'cp $GAMMA_HOME/MSP/sensors/' + par + ' .' - os.system(call_str) - if not os.path.isfile(antenna): - call_str = 'cp $GAMMA_HOME/MSP/sensors/' + antenna + ' .' - os.system(call_str) - - if inps.out_put: - pslc_par = 'p' + inps.out_put + '.slc.par' - slc_par = inps.out_put + '.slc.par' - slc = inps.out_put + '.slc' - mli_par = inps.out_put + '.mli.par' - mli = inps.out_put + '.mli' - else: - pslc_par = 'p' + date + '.slc.par' - slc_par = date + '.slc.par' - slc = date + '.slc' - mli_par = date + '.mli.par' - mli = date + '.mli' - - call_str = 'ERS_ENVISAT_proc ' + SAR_IM_0P + ' ' + par + ' ' + pslc_par + ' ' + raw - os.system(call_str) - - call_str = 'DELFT_proc2 ' + pslc_par + ' ' + orbdir + ' 20' - os.system(call_str) - - cal_str = 'dop_ambig ' + par + ' ' + pslc_par + ' ' + raw + ' 2 - ' + dop_ambig - os.system(call_str) - - call_str = 'azsp_IQ ' + par + ' ' + pslc_par + ' ' + raw + ' ' + azsp - os.system(call_str) - - call_str = 'doppler ' + par + ' ' + pslc_par + ' ' + raw + ' ' + dop - os.system(call_str) - - call_str = 'rspec_IQ ' + par + ' ' + pslc_par + ' ' + raw + ' ' + rspec - os.system(call_str) - - call_str = 'pre_rc ' + par + ' ' + pslc_par + ' ' + raw + ' ' + rc - os.system(call_str) - - call_str = 'autof ' + par + ' ' + pslc_par + ' ' + rc + ' ' + autof + ' 2.0 ' - os.system(call_str) - - call_str = 'autof ' + par + ' ' + pslc_par + ' ' + rc + ' ' + autof + ' 2.0 ' - os.system(call_str) - - call_str = 'az_proc ' + par + ' ' + pslc_par + ' ' + rc + ' ' + slc + ' 4096 1 ' + ' 57.2 0 2.120 ' - os.system(call_str) - - call_str = 'par_MSP ' + par + ' ' + pslc_par + ' ' + slc_par + ' 1' - os.system(call_str) - - call_str = 'multi_look ' + slc + ' ' + slc_par + ' ' + mli + ' ' + mli_par + ' 2 10' - os.system(call_str) - - - Width = UseGamma(mli_par, 'read', 'range_samples: ') - call_str = 'raspwr ' + mli + ' ' + Width - os.system(call_str) - - - print("Generate SLC from %s is done." % SAR_IM_0P) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/rslcCopy_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/rslcCopy_gamma.py deleted file mode 100644 index 455686f..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/rslcCopy_gamma.py +++ /dev/null @@ -1,89 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import argparse - -from pyint import _utils as ut - - -INTRODUCTION = ''' -------------------------------------------------------------------- - copy SLC/RSLC from large SLC/RSLC - -''' - -EXAMPLE = ''' - Usage: - slcCopy_gamma.py projectName Sdate rstart rwidth astart awidth outname - slcCopy_gamma.py PacayaT163TsxHhA 20150601 100 345 200 500 20150601_subset -------------------------------------------------------------------- -''' - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Copy RSLC subset from a large SLC.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('Date',help='Master date.') - parser.add_argument('rstart',help='start point of range') - parser.add_argument('rwidth',help='width of range direction') - parser.add_argument('astart',help='start point of azimuth') - parser.add_argument('awidth',help='width of azimith direction') - parser.add_argument('name',help='output name. e.g., 20150101_subset, then .slc and .slc.par will be generated') - - inps = parser.parse_args() - return inps - - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - Date = inps.Date - rstart = inps.rstart; rwidth = inps.rwidth; astart = inps.astart; awidth = inps.awidth; name = inps.name - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - - processDir = scratchDir + '/' + projectName + "/PROCESS" - slcDir = scratchDir + '/' + projectName + "/SLC" - rslcDir = scratchDir + '/' + projectName + '/RSLC' - ifgDir = scratchDir + '/' + projectName + '/ifgrams' - - Dslc = rslcDir + '/' + Date + '/' + Date + '.rslc' - DslcPar = rslcDir + '/' + Date + '/' + Date + '.rslc.par' - - Oslc = rslcDir + '/' + Date + '/' + name + '.rslc' - OslcPar = rslcDir + '/' + Date + '/' + name + '.rslc.par' - - Oamp = rslcDir + '/' + Date + '/' + name + '_' + rlks + 'rlks.amp' - OampPar = rslcDir + '/' + Date + '/' + name + '_' + rlks + 'rlks.amp.par' - - - - call_str = 'SLC_copy ' + Dslc + ' ' + DslcPar + ' ' + Oslc + ' ' + OslcPar + ' - - ' + rstart + ' ' + rwidth + ' ' + astart + ' ' + awidth + ' - -' - os.system(call_str) - - call_str = 'multi_look ' + Oslc + ' ' + OslcPar + ' ' + Oamp + ' ' + OampPar + ' ' + rlks + ' ' + azlks - os.system(call_str) - #os.remove(MampPar) - #os.remove(SampPar) - print(call_str) - print("RSLC copy done.") - #sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/rslcCopy_gamma_jobs.py b/.codex_tmp/pyint_variants/no_rescue/pyint/rslcCopy_gamma_jobs.py deleted file mode 100644 index c4d728a..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/rslcCopy_gamma_jobs.py +++ /dev/null @@ -1,135 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import argparse - -from pyint import _utils as ut - - -INTRODUCTION = ''' -------------------------------------------------------------------- - copy SLC/RSLC from large SLC/RSLC - -''' - -EXAMPLE = ''' - Usage: - slcCopy_gamma_jobs.py projectName Sdate rwidth awidth --memory memory_single_job --walltime walltime_single_job - slcCopy_gamma_jobs.py PacayaT163TsxHhA 20150601 5000 5000 --memory 5000 --walltime 00:30:00 -------------------------------------------------------------------- -''' - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Copy SLC subset from a large SLC.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('Date',help='Master date.') - parser.add_argument('rwidth',help='width of range direction') - parser.add_argument('awidth',help='width of azimith direction') - parser.add_argument('--memory',dest = 'memory',default = '2000', help='memory to be allocated for one job, default: 5GB') - parser.add_argument('--walltime',dest = 'walltime',default = '00:05:00', help='walltime to be allocated for one single job') - inps = parser.parse_args() - return inps - - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - Date = inps.Date; rwidth = inps.rwidth; awidth = inps.awidth; memory = inps.memory; walltime = inps.walltime - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - - processDir = scratchDir + '/' + projectName + "/PROCESS" - slcDir = scratchDir + '/' + projectName + "/SLC" - rslcDir = scratchDir + '/' + projectName + '/RSLC' - ifgDir = scratchDir + '/' + projectName + '/ifgrams' - - Dslc = rslcDir + '/' + Date + '/' + Date + '.rslc' - DslcPar = rslcDir + '/' + Date + '/' + Date + '.rslc.par' - - nWidth = ut.read_gamma_par(DslcPar, 'read', 'range_samples') - nLine = ut.read_gamma_par(DslcPar, 'read', 'azimuth_lines') - - Ap_samp = np.arange(1,int(nLine),int(awidth)); Na0 = len(Ap_samp); LA_end = int(nLine) - Ap_samp[Na0-1] - Rp_samp = np.arange(1,int(nWidth),int(rwidth)); Nr0 = len(Rp_samp); LR_end = int(nWidth) - Rp_samp[Nr0-1] - - if LA_end > (2/3*int(awidth)): - Ap_samp1 = Ap_samp - else: - Ap_samp1 = Ap_samp[0:(Na0-1)] # last batch with '-' means to the end - - if LR_end > (2/3*int(rwidth)): - Rp_samp1 = Rp_samp - else: - Rp_samp1 = Rp_samp[0:(Nr0-1)] # last batch with '-' means to the end - - Na = len(Ap_samp1); Nr = len(Rp_samp1); Ntotal = Na*Nr - print('Azimuth samples: ' + str(Na)) - print('Range samples: ' + str(Nr)) - print('Total batch numbers: ' + str(Ntotal)) - - batch_txt = scratchDir + '/' + projectName + '/RSLC/' + Date + '/Batch_copySLC' - workDir = scratchDir + '/' + projectName + '/RSLC/' + Date - if os.path.isfile(batch_txt): - os.remove(batch_txt) - for i in range(Na): - for j in range(Nr): - if len(str(i)) == 1: - i0 = '0' + str(i) - else: - i0 = str(i) - if len(str(j)) == 1: - j0 = '0' + str(j) - else: - j0 = str(j) - name0 = Date + '_' +i0 + j0 - rstart = str(Rp_samp1[j]); astart = str(Ap_samp1[i]) - if rstart == str(Rp_samp1[Nr-1]): - rwidth0 = '-' - else: - rwidth0 = str(int(int(rwidth) + 200)) # extend 200 to avoid edge effect - - if astart == str(Ap_samp1[Na-1]): - awidth0 = '-' - else: - awidth0 = str(int(int(awidth) + 200)) # extend 200 to avoid edge effect - - if not i==0: - astart0 = str(int(int(astart) - 200)) # extend 200 to avoid edge effect - else: - astart0 = astart - - if not j==0: - rstart0 = str(int(int(rstart) - 200)) # extend 200 to avoid edge effect - else: - rstart0 = rstart - - #str0 = 'SLC_copy ' + Dslc + ' ' + DslcPar + ' ' + workDir + '/' + name0 + '.rslc' + ' ' + workDir + '/' + name0 + '.rslc.par - - ' + rstart0 + ' ' + rwidth0 + ' ' + astart0 + ' ' + awidth0 + ' - -' - str0 = 'rslcCopy_gamma.py ' + projectName + ' ' + Date + ' ' + rstart0 + ' ' + rwidth0 + ' ' + astart0 + ' ' + awidth0 + ' ' + name0 - call_str = 'echo ' + str0 + ' >>' + batch_txt - os.system(call_str) - - call_str = 'sbatch_jobs.py ' + batch_txt + ' --memory ' + inps.memory + ' --walltime ' + inps.walltime + ' --job-name ' + Date + '_copySLC' - os.system(call_str) - - print("RSLC copy done: " + Date) - #sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/s1_orb_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/s1_orb_all.py deleted file mode 100644 index fa225b8..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/s1_orb_all.py +++ /dev/null @@ -1,114 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import getopt -import time -import glob -import argparse - -import subprocess -from pyint import _utils as ut - - -def work(data0): - cmd = data0[0] - err_txt = data0[1] - - p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) - stdout = p.stdout - stderr = p.stderr - - if type(stderr) == bytes: - aa=stderr.decode("utf-8") - else: - aa = stderr - - if aa: - str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' - #print(aa) - with open(err_txt, 'a') as f: - f.write(str0) - f.write(aa) - f.write('\n') - - return -######################################################################### - -INTRODUCTION = ''' -------------------------------------------------------------------- - Using precise orbit data for all rslcs. - -''' - -EXAMPLE = ''' - Usage: - generate_amp_all.py projectName - generate_amp_all.py projectName --parallel 4 -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Using precise orbit data for all rslcs.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') - - inps = parser.parse_args() - return inps - - -def main(argv): - start_time = time.time() - inps = cmdLineParse() - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - projectDir = scratchDir + '/' + projectName - slcDir = scratchDir + '/' + projectName + '/SLC' - rslcDir = scratchDir + '/' + projectName + '/RSLC' - if not os.path.isdir(rslcDir): os.mkdir(rslcDir) - - cmd_command = 'generate_multilook_amp.py' - - err_txt = scratchDir + '/' + projectName + '/generate_amp_all.err' - if os.path.isfile(err_txt): os.remove(err_txt) - - data_para = [] - slc_list = [os.path.basename(fname) for fname in sorted(glob.glob(rslcDir + '/*'))] - #slc_list = ut.get_project_slcList(projectName) - for i in range(len(slc_list)): - rslcPar = rslcDir + '/' + slc_list[i] + '/' + slc_list[i] + '.rslc.par' - #print(rslcPar) - workDir0 = rslcDir + '/' + slc_list[i] - BB = glob.glob(workDir0 + '/*.EOF') - if len(BB)==0: - Sensor = ut.read_gamma_par(rslcPar,'read','sensor') - if 'A' in Sensor: satellite = 'A' - else: satellite = 'B' - ut.download_s1_orbit(slc_list[i],workDir0,satellite=satellite) - - BB = glob.glob(workDir0 + '/*.EOF') - if len(BB) > 0: - orb_file = BB[0] - - call_str = 'S1_OPOD_vec ' + rslcPar + ' ' + orb_file + ' 31' - os.system(call_str) - - print("Using precise orbit data for all rslcs is done! ") - ut.print_process_time(start_time, time.time()) - - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/scihub_search_s1_data.py b/.codex_tmp/pyint_variants/no_rescue/pyint/scihub_search_s1_data.py deleted file mode 100644 index 6a36c96..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/scihub_search_s1_data.py +++ /dev/null @@ -1,220 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT ### -### Author: chen ### -### Contact : chenweicug@126.com ### -################################################################# - -import getopt -import time -import glob -import numpy as np -import os -import sys -import subprocess -import time -import argparse -from pyint import _utils as ut - - -INTRODUCTION = ''' -------------------------------------------------------------------- - Generate differential interferogram image from SLC using GAMMA. - -''' - -EXAMPLE = ''' - Usage: - scihub_search_s1_data.sh -options - scihub_search_s1_data.sh -s 2015-08-01 -e NOW -r "123.0/-123.3/40.0/40.2" -d Descending - -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Coregister all of the SLCs to the reference SAR image using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('-s', dest='Start_time', help='start_time [yyyy-mm-dd or NOW]') - parser.add_argument('-e', dest='End_time', help='end_time [yyyy-mm-dd or NOW]') - parser.add_argument('-r', dest='Region_box',help='region_box "lonW/lonE/latS/latN"') - parser.add_argument('-p', dest='producttype', help='producttype "[SLC GRD OCN]"') - parser.add_argument('-n', dest='Orbit_number',help='orbit_number [0-175]') - parser.add_argument('-d', dest='Direction', help='direction [Ascending/Descending]') - parser.add_argument('-sm',dest='Sence_model', help='sence model [IW/EW/SM]') - parser.add_argument('-o', dest='output_file', help='output_file name') - inps = parser.parse_args() - return inps -def main(argv): - - start_time = time.time() - inps = cmdLineParse() - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - projectDir = scratchDir + '/' + projectName - dataDir = scratchDir + '/' + projectName + '/' +'DOWNLOAD' - #dataDir="DOWNLOAD" - if not os.path.isdir(dataDir): - os.mkdir(dataDir) - os.chdir(projectDir) -# The basic stuff that doesn't change^^ - search_query="https://scihub.copernicus.eu/dhus/search?q=platformname:Sentinel-1" - - if inps.output_file: - Output_file=inps.output_file - else: - Output_file='search_result.txt' - -# If we are searching based on starttime and endtime - if inps.Start_time: - starttime=inps.Start_time - else: - starttime='1990-01-01' - if inps.End_time: - endtime=inps.End_time - else: - endtime='NOW' - search_query=search_query+" AND beginposition:[" + starttime + "T00:00:00.000Z TO "+ endtime + "T00:00:00.000Z]" - #print(search_query) - - if inps.Region_box: - Region=inps.Region_box - compments=Region.split('/') - lonW=compments[0] - lonE=compments[1] - latS=compments[2] - latN=compments[3] - lonw=float(lonW) - lone=float(lonE) - latn=float(latN) - lats=float(latS) - if lonW >=lonE: - print("Error! Longitudes not in increasing order!") - sys.exit(1) - elif lats >= latn: - print("Error! Latitudes not in increasing order!") - sys.exit(1) - elif lats >= 90 or latn >= 90: - print("Error! latS or latN not in bounds!") - sys.exit(1) - else: - parser.print_usage() - sys.exit(1) -# add the SAR covered area - search_query+=" AND footprint:\"intersects(POLYGON((" + lonW + " " + latN + "," + lonE + " " + latN + "," + lonE + " " + latS + "," + lonW + " " + latS + "," + lonW + " " + latN +")))\"" - - if inps.Direction: - direction=inps.Direction - else: - direction="Ascending" -# add the orbit direction - search_query+=" AND orbitdirection:" + direction - - if inps.Sence_model: - model=inps.Sence_model - else: - model='IW' -# add the sense operational mode - search_query+=" AND sensoroperationalmode:"+ model - - if inps.Orbit_number: - if int(inps.Orbit_number) >= 175: - print("Error: orbit must be between 0 and 175") - sys.exit(1) - else: - orbit=inps.Orbit_number - # add the relative orbit number - search_query+=" AND relativeorbitnumber:" + orbit - if inps.producttype: - producttype=inps.producttype - else: - producttype='SLC' -# add the orbit direction - search_query+=" AND producttype:" + producttype - - - - - - -# how many rows to display and where to start? -# Max rows = 100 (slightly annoying rule from the Copernicus server) - if os.path.isfile(Output_file): os.remove(Output_file) - search_query0=search_query+"&start=0&rows=100" - print("the current search_query is: ",search_query0) - #call_str="echo \"Input options:\" $@ > " +Output_file - #os.system(call_str) - call_str="echo \"wget --no-check-certificate --user=chenwei --password=cw1425 \""+ search_query0 + " >> " + Output_file - os.system(call_str) -# Execute the search using wget - search_query0="'"+search_query0+ "'" - call_str="wget --no-check-certificate --user=chenwei --password=cw1425 " + search_query0 + " -O ->> " + Output_file - os.system(call_str) - call_str="`grep \'title>S1\' " + Output_file +" | wc -l`" - num_results=os.system(call_str) - print(num_results) - if num_results >= 100 : - print("We have 100 results... automatically searching for results #100-200") - search_query1=search_query + "&start=100&rows=100" - search_query1="'"+search_query1+ "'" - call_str="wget --no-check-certificate --user=chenwei --password=cw1425 " + search_query1 + " -O ->> " + Output_file - os.system(call_str) - call_str="`grep \'title>S1\' " + Output_file +" | wc -l`" - num_results=os.system(call_str) - elif num_results >= 200 : - print("We have 200 results... automatically searching for results #100-200") - search_query2=search_query + "&start=100&rows=100" - search_query2="'"+search_query2+ "'" - call_str="wget --no-check-certificate --user=chenwei --password=cw1425 " + search_query2 + " -O ->> " + Output_file - os.system(call_str) - call_str="`grep \'title>S1\' " + Output_file +" | wc -l`" - num_results=os.system(call_str) - -# Displaying a summary of the results - call_str="grep \'title>S1\' " + Output_file # displaying the results - os.system(call_str) - print("number of total results is: ") - call_str="grep \'title>S1\' " + Output_file + " | wc -l" - os.system(call_str) - print("####################################################") - print("the search has done and start to download") - print("####################################################") - id_results='uuid_file.txt' - if os.path.isfile(id_results): - os.remove(id_results) - call_str="grep -E 'uuid|S1' " + Output_file + " >> " + id_results - os.system(call_str) - call_str="sed -i 's/<str name=\"uuid\">//g' " + id_results - os.system(call_str) - call_str="sed -i 's/<title>//g' " + id_results - os.system(call_str) - call_str="sed -i 's/<\/title>//g' " + id_results - os.system(call_str) - call_str="sed -i 's/<\/str>//g' "+ id_results - os.system(call_str) - infopen = open(id_results,'r',encoding='utf-8') - lines = infopen.readlines() - count=int(len(lines)/2-1) - for i in range(0,count,2): - j=i+1 - title=lines[i].strip("\n") - uuid=lines[j].strip("\n") - url_address="\"https://scihub.copernicus.eu/dhus/odata/v1/Products('" + uuid + "')/\$value\"" - call_str= "wget -c --no-check-certificate --user=chenwei --password=cw1425 -O " +dataDir + "/" + title + ".zip " + url_address - print(call_str) - os.system(call_str) - call_str=title+".zip download has done successfully!" - print(call_str) - print("#########################################################") - print("#########all the data download successfully!#############") - - - - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/select_pairs.py b/.codex_tmp/pyint_variants/no_rescue/pyint/select_pairs.py deleted file mode 100644 index 0db9cb2..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/select_pairs.py +++ /dev/null @@ -1,297 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# -# This program is modified from MintPy/select_network.py -# Copyright: 2017-2019, Yunjun Zhang -# Contact: yzhang@rsmas.miami.edu - -import os -import sys -import glob -import argparse -import datetime -import inspect -import numpy as np - -from pyint import _utils as ut -from pyint import _network as nt - -def get_datelist_bperplist(TS_Net): - - IFG_Flag=np.asarray(TS_Net[:,0]) - MDatelist=np.asarray(TS_Net[:,1]) - SDatelist=np.asarray(TS_Net[:,2]) - Berplist=np.asarray(TS_Net[:,3]) - TBaselist=np.asarray(TS_Net[:,4]) - - date_list = [] - date_list.append(MDatelist[0]) - for k0 in SDatelist: - date_list.append(k0) - bperp_list = [] - bperp_list.append(0) - for k0 in Berplist: - bperp_list.append(float(k0)) - - tbase_list = [] - tbase_list.append(0) - for k0 in TBaselist: - tbase_list.append(float(k0)) - return date_list, tbase_list, bperp_list - - -def prune_datelist(date_list, tbase_list, pbase_list, templateDict): - nn = len(date_list) - - date_list_out = [] - for k0 in date_list: - if (k0 not in templateDict['exclude_list']) and (float(templateDict['startDate']) < float(k0)) and (float(templateDict['endDate']) > float(k0)): - date_list_out.append(k0) - date_list_out = sorted(date_list_out) - tbase_list_out = [] - pbase_list_out = [] - for k0 in date_list_out: - tbase0 = tbase_list[date_list.index(k0)] - pbase0 = pbase_list[date_list.index(k0)] - tbase_list_out.append(float(tbase0)) - pbase_list_out.append(float(pbase0)) - - tbase_list_out0 = np.asarray(tbase_list_out) - pbase_list_out0 = np.asarray(pbase_list_out) - - tbase_list_out1 = tbase_list_out0 - tbase_list_out0[0] - pbase_list_out1 = pbase_list_out0 - pbase_list_out0[0] - - return date_list_out, list(tbase_list_out1), list(pbase_list_out1) - -def select_network_candidate(date_list,tbase_list,pbase_list,templateDict): - - method = templateDict['network_method'] - - # Pais selection from method - if method == 'sbas': - date12_list = nt.select_pairs_sbas(date_list) - elif method == 'delaunay': - date12_list = nt.select_pairs_delaunay(date_list, tbase_list, pbase_list, norm = True) - elif method == 'star': - date12_list = nt.select_pairs_star(date_list) - elif method == 'sequential': - date12_list = nt.select_pairs_sequential(date_list, int(templateDict['conNumb'])) - #elif method == 'hierarchical': - # date12_list = nt.select_pairs_hierarchical(date_list, pbase_list, inps.tempPerpList) - #elif method == 'mst': - # date12_list = nt.select_pairs_mst(date_list, pbase_list) - else: - raise Exception('Unrecoganized select method: '+ method) - - date12_list =nt.yyyymmdd_date12(date12_list) - nn = len(date12_list) - - ifgram_tbase_list = [] - ifgram_pbase_list = [] - for i in range(nn): - m0 = date12_list[i].split('-')[0] - s0 = date12_list[i].split('-')[1] - - tbase0 = float(tbase_list[date_list.index(s0)]) - float(tbase_list[date_list.index(m0)]) - pbase0 = float(pbase_list[date_list.index(s0)]) - float(pbase_list[date_list.index(m0)]) - - ifgram_tbase_list.append(tbase0) - ifgram_pbase_list.append(pbase0) - - return date12_list, ifgram_tbase_list, ifgram_pbase_list - - -def prune_network(date12_list, ifgram_tbase_list, ifgram_pbase_list, templateDict): - """Pruning network candidates based on temp/perp baseline""" - date12_list0 = date12_list.copy() - ifgram_tbase_list = [float(i) for i in ifgram_tbase_list] - ifgram_pbase_list = [float(i) for i in ifgram_pbase_list] - - ifgram_tbase_list_abs = [abs(i) for i in ifgram_tbase_list] - ifgram_pbase_list_abs = [abs(i) for i in ifgram_pbase_list] - - ifgram_tbase_list_abs = np.asarray(ifgram_tbase_list_abs) - ifgram_pbase_list_abs = np.asarray(ifgram_pbase_list_abs) - - date12_list0 = np.asarray(date12_list0) - ifgram_tbase_list = np.asarray(ifgram_tbase_list) - ifgram_pbase_list = np.asarray(ifgram_pbase_list) - -# date12_list_out = date12_list0[(ifgram_tbase_list_abs < float(templateDict['max_tb'])) & (ifgram_pbase_list_abs < float(templateDict['max_sb']))] - date12_list_out = date12_list0[(ifgram_tbase_list_abs < float(templateDict['max_tb'])) & (ifgram_tbase_list_abs > float(templateDict['min_tb'])) & (ifgram_pbase_list_abs < float(templateDict['max_sb']))] -# ifgram_tbase_list_out = ifgram_tbase_list[(ifgram_tbase_list_abs < float(templateDict['max_tb'])) & (ifgram_pbase_list_abs < float(templateDict['max_sb']))] -# ifgram_pbase_list_out = ifgram_pbase_list[(ifgram_tbase_list_abs < float(templateDict['max_tb'])) & (ifgram_pbase_list_abs < float(templateDict['max_sb']))] - ifgram_tbase_list_out = ifgram_tbase_list[(ifgram_tbase_list_abs < float(templateDict['max_tb'])) & (ifgram_tbase_list_abs > float(templateDict['min_tb'])) & (ifgram_pbase_list_abs < float(templateDict['max_sb']))] - ifgram_pbase_list_out = ifgram_pbase_list[(ifgram_tbase_list_abs < float(templateDict['max_tb'])) & (ifgram_tbase_list_abs > float(templateDict['min_tb'])) & (ifgram_pbase_list_abs < float(templateDict['max_sb']))] - - return list(date12_list_out), list(ifgram_tbase_list_out), list(ifgram_pbase_list_out) - - -def write_ifgram_list(date12_list,ifgram_tbase_list,ifgram_pbase_list, out_file): - # Output directory/filename - # Write txt file - f = open(out_file, 'w') - #f.write('#Interferograms configuration generated by select_network.py\n') - #f.write('# Date12 Btemp(days) Bperp(m) sim_coherence\n') - for i in range(len(date12_list)): - line = '{} {:6.0f} {:6.1f}'.format(date12_list[i], - ifgram_tbase_list[i], - ifgram_pbase_list[i]) - f.write(line+'\n') - f.close() - return out_file - -#---------------------------------------------------------------------------------------------------# -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Select interferometric pairs for time-series InSAR process.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('-o', '--outfile',dest = 'outfile',help='Output list file for network, ifgram_list.txt by default.') - parser.add_argument('--exclude',dest='excludeDate', nargs='*', default=[], - help='date(s) excluded for network selection, e.g. --exclude 060713 070831') - parser.add_argument('--start-date', dest='startDate', type=str, help='start/min date of network') - parser.add_argument('--end-date', dest='endDate', type=str, help='end/max date of network') - parser.add_argument('--max-sb', dest='maxSB', type=float, help='maximum spatial baseline') - parser.add_argument('--max-tb', dest='maxTB', type=str, help='maximum temporal baseline') - parser.add_argument('--min-tb', dest='minTB', type=str, help='minmum temporal baseline') - parser.add_argument('--method', dest='method', default='sbas', choices={'sbas', 'sequential', 'delaunay', 'stars'}, - help='network selection method:\n' + - 'sbas - select based on the threshold values of the spatio-temporal baselines\n' + - 'sequential - select based on the sequential of the SAR acquisitions\n' + - 'delaunay - select based on delaunay triangulars\n' + - 'stars - one master image network, like PS.') - parser.add_argument('--conNumb', dest='conNumb', type=int, default=2, - help='Number of the neibour-connected SAR images at one side for sequential method.') - parser.add_argument('--coreg', dest='coreg', action='store_true', help='Using rslc/rslcPar to check orbit history.') - - inps = parser.parse_args() - return inps - - -INTRODUCTION = ''' -------------------------------------------------------------------- - Select interferometric pairs based on SLC_TAB. - [sbas, sequential, delaunay, stars are supported.] - -''' - -EXAMPLE = ''' - Examples: - select_pairs.py PacayaT163TsxHhA - select_pairs.py PacayaT163TsxHhA --max-tb 300 --min-tb 200 --max-sb 100 - select_pairs.py PacayaT163TsxHhA --method sequential --conNumb 3 - select_pairs.py PacayaT163TsxHhA --method denaulay --max-tb 100 --max-sb 100 - select_pairs.py PacayaT163TsxHhA --method stars - select_pairs.py PacayaT163TsxHhA --coreg -------------------------------------------------------------------- -''' - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - templateDict['network_method'] = inps.method - templateDict['conNumb'] = inps.conNumb - - processDir = scratchDir + '/' + projectName - slcDir = scratchDir + '/' + projectName + "/SLC" - rslcDir = scratchDir + '/' + projectName + '/RSLC' - - #slc_list = ut.get_project_slcList(projectName) - masterDate = templateDict['masterDate'] - - if inps.coreg: - slcDir = rslcDir - slc0 = '.rslc' - slcpar0 = '.rslc.par' - else: - slc0 = '.rslc' - slcpar0 = '.rslc.par' - - slc_list0 = [os.path.basename(fname) for fname in sorted(glob.glob(rslcDir + '/*'))] - slc_list = [] - for k0 in slc_list0: - if ut.is_number(k0): - slc_list.append(k0) - slc_list = sorted(slc_list) - - SLCfile = [] - SLCParfile = [] - for kk in range(len(slc_list)): - #str_slc = slcDir + "/" + slc_list[kk] +"/" + slc_list[kk] + slc0 - str_slc = rslcDir + "/" + slc_list[kk] +"/" + slc_list[kk] + slc0 - str_slc_par = rslcDir + "/" + slc_list[kk] +"/" + slc_list[kk] + slcpar0 - SLCfile.append(str_slc) - SLCParfile.append(str_slc_par) - - RefPar = rslcDir + "/" + masterDate +"/" + masterDate + slcpar0 - SLC_Tab = scratchDir + '/' + projectName + "/SLC_Tab" - - SLC_Tab_all = processDir + "/SLC_Tab_all" - TS_Berp_all = processDir + "/TS_Berp_all" - TS_Itab_all = processDir + "/TS_Itab_all" - itab_type = '1' - pltflg = '0' - - if os.path.isfile(SLC_Tab): os.remove(SLC_Tab) - with open(SLC_Tab, 'a') as f: - for kk in range(len(SLCfile)): - f.write(str(SLCfile[kk])+ ' '+str(SLCParfile[kk])+'\n') - - ## Get all of the pairs using GAMMA - call_str = "base_calc " + SLC_Tab + " " + RefPar + " " + TS_Berp_all + " " + TS_Itab_all + " " + '1 0 ' + '- - - - >/dev/null' - os.system(call_str) - - with open(TS_Berp_all, 'r') as f: - lines = f.readlines() - - TS_Net_all = ut.read_txt2array(TS_Berp_all) - if len(lines) ==1: - TS_Net_all = TS_Net_all.reshape(1,9) - - TS_Net = TS_Net_all[0:(len(SLCfile)-1),:] - date_list, tbase_list, pbase_list = get_datelist_bperplist(TS_Net) - bl_list_txt = scratchDir + '/' + projectName + '/bl_list.txt' - with open(bl_list_txt, 'w') as f: - for kk in range(len(SLCfile)): - f.write(str(date_list[kk])+ ' '+str(pbase_list[kk])+'\n') - - exclude_list = [] - if 'exclude_date' in templateDict: - exclude_list = templateDict['exclude_date'].split(',')[:] - if inps.excludeDate: - exclude_list0 = inps.excludeDate - for k0 in exclude_list0: - exclude_list.append(k0) - templateDict['exclude_list'] = exclude_list - - if inps.startDate: templateDict['startDate'] = inps.startDate - if inps.endDate: templateDict['endDate'] = inps.endDate - if inps.maxTB: templateDict['max_tb'] = inps.maxTB - if inps.maxSB: templateDict['max_sb'] = inps.maxSB - - date_list, tbase_list, pbase_list = prune_datelist(date_list, tbase_list, pbase_list, templateDict) - date12_list, ifgram_tbase_list, ifgram_pbase_list = select_network_candidate(date_list,tbase_list,pbase_list,templateDict) - date12_list_final, ifgram_tbase_list_final, ifgram_pbase_list_final = prune_network(date12_list, ifgram_tbase_list, ifgram_pbase_list, templateDict) - - if inps.outfile: out_file = inps.outfile - else: out_file = scratchDir + '/' + projectName + "/ifgram_list.txt" - - write_ifgram_list(date12_list_final,ifgram_tbase_list_final,ifgram_pbase_list_final, out_file) - sys.exit(1) - -#################################################################### -if __name__ == '__main__': - main(sys.argv[:]) - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/select_paris_by_cor.py b/.codex_tmp/pyint_variants/no_rescue/pyint/select_paris_by_cor.py deleted file mode 100644 index ae3b0fc..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/select_paris_by_cor.py +++ /dev/null @@ -1,139 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT ### -### Author: chen ### -### Contact : chenweicug@126.com ### -################################################################# - -import numpy as np -import os -import sys -import getopt -import time -import glob -import argparse -from pyint import _utils as ut -import cv2 - - -INTRODUCTION = ''' -------------------------------------------------------------------- - compare two corherence for SAR images for delete the bad images . - -''' - -EXAMPLE = ''' - Usage: - cor_correlation.py projectName Mdate Sdate - cor_correlation.py PacayaT163TsxHhA 20150102 20150601 ifgrams_list.txt -------------------------------------------------------------------- -''' -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Coregister all of the SLCs to the reference SAR image using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('Mdate',help='Master date.') - parser.add_argument('Sdate',help='Slave date.') - parser.add_argument('ifgs', help='provided ifgram_list_txt. default: using ifgram_list.txt under projectName folder.') - parser.add_argument('correlation_index', help='provided the accept correlation for the file . default: 0.55.') - - inps = parser.parse_args() - return inps - - -def main(argv): - - start_time = time.time() - inps = cmdLineParse() - Mdate = inps.Mdate - Sdate = inps.Sdate - - if inps.correlation_index: Score = inps.correlation_index - else: Score = 0.55 - - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - masterDate = templateDict['masterDate'] - - projectDir = scratchDir + '/' + projectName - - slcDir = scratchDir + '/' + projectName + '/SLC' - rslcDir = scratchDir + '/' + projectName + '/RSLC' - ifgDir = projectDir + '/ifgrams' - badifgDir = projectDir + '/bad_ifgrams' - if not os.path.isdir(ifgDir): os.mkdir(ifgDir) - if not os.path.isdir(badifgDir): os.mkdir(badifgDir) - Pair = Mdate + '-' + Sdate - workDir = ifgDir + '/' + Pair - - ####################################################################### - Mamp = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' - MampPar = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' - Samp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' - SampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' - - Mrslc = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc' - MrslcPar = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc.par' - Srslc = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc' - SrslcPar = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc.par' - - if inps.ifgs: ifgramList_txt = inps.ifgs - else: ifgramList_txt = scratchDir + '/' + projectName + '/ifgram_list.txt' - MasterPar = rslcDir + '/' + masterDate + '/' + masterDate + '.rslc.par' - first_image= workDir + '/' + Pair + '_' + rlks + 'rlks.diff_filt.cor.bmp' - ifgList0 = ut.read_txt2array(ifgramList_txt) -# 加载第一张图 - image1 = cv2.imread(first_image) - ifgList0 = ut.read_txt2array(ifgramList_txt) -# ifgList = ifgList0[:,0] - if len(ifgList0)==3: - ifgList=ifgList0[0] - ifgList=[ifgList] - - else: - # ifgList=ifgList0[:,0] - ifgList=ifgList0 - err_txt = scratchDir + '/' + projectName + '/cor_correlation_all.err' - if os.path.isfile(err_txt): os.remove(err_txt) - - out_file='cor_correltion.txt' - if os.path.isfile(out_file): os.remove(out_file) - for i in range(len(ifgList)): - m0 = ut.yyyymmdd(ifgList[i].split('-')[0]) - s0 = ut.yyyymmdd(ifgList[i].split('-')[1]) - Pair2 = m0 + '-' + s0 - workdir = ifgDir + '/' + Pair2 - # 加载第二张图片 - second_image = workdir + '/' + Pair2 + '_' + rlks + 'rlks.diff_filt.cor.bmp' - image2=cv2.imread(second_image) - # 将图片转换为灰度图像 - gray_image1 = cv2.cvtColor(image1, cv2.COLOR_BGR2GRAY) - gray_image2 = cv2.cvtColor(image2, cv2.COLOR_BGR2GRAY) - # 计算两张图片之间的结构相似性指数（SSIM） - - ssim_score = cv2.matchTemplate(gray_image1, gray_image2, cv2.TM_CCOEFF_NORMED) - score = ssim_score[0][0] - - if str(score) < Score: - print(m0, "-", s0, "相关性得分：", ssim_score[0][0]) - call_str=' mv ' + workdir + ' ' + badifgDir + '/' - os.system(call_str) - else: - print("This pair", m0,'-', s0,"is accept") - call_str = 'echo ' + m0 + '-' + s0 + ' ' + ' ' + str(score) + '>>' + out_file - os.system(call_str) - - print("Delete the bad interferograms for project %s is done! " % projectName) - ut.print_process_time(start_time, time.time()) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/single_GACOS_correction.csh b/.codex_tmp/pyint_variants/no_rescue/pyint/single_GACOS_correction.csh deleted file mode 100644 index 09bea26..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/single_GACOS_correction.csh +++ /dev/null @@ -1,126 +0,0 @@ -#!/bin/csh -f - -if ($#argv != 7) then - echo "" - echo "Usage: single_GACOS_correction.csh master_ztd master_ztd.rsc slave_ztd slave_ztd.rsc los.grd reference_point incidence_angle" - echo "" - echo "Performs gacos correction for Sentinel 1 for a single interferogram in grd format" - echo "" - echo "Output: corrected unwrapped phase" - echo "" - echo "Reference point text file in lon lat format" - echo "" - echo "" - echo "Indicence angle in degrees (float or integer)" - echo "" - exit 1 - endif - -#Checking input files -if !(-e $1) then - echo "master_ztd binary file: $1 seems not to exist" - exit 1 -endif -if !(-e $2) then - echo "master_ztd.rsc file: $2 seems not to exist" - exit 1 -endif -if !(-e $3) then - echo "slave_ztd binary file: $3 seems not to exist" - exit 1 -endif -if !(-e $4) then - echo "slave_ztd.rsc file: $4 seems not to exist" - exit 1 -endif -if !(-e $5) then - echo "Interferogram to correct: $5 seems not to exist" - exit 1 -endif -if !(-e $6) then - echo "Reference point text file: $6 seems not to exist" - exit 1 -endif - - -#reference point/stable point in radar coordinates -set reference_point = $6 - -#incidence angle obtained with the "SAT_look" script in degrees -set incidence = $7 - -#wavelength for Sentinel 1 (m) -set wavelength = 0.055165 -set pi = 3.141592653589793238462 - -#######FIRST ZTD to grid######### -set x_first_d1 = `cat $2|grep X_FIRST|awk '{print $2}'` -set y_first_d1 = `cat $2|grep Y_FIRST|awk '{print $2}'` -set width_d1 = `cat $2|grep WIDTH|awk '{print $2}'` -set length_d1 = `cat $2|grep FILE_LENGTH|awk '{print $2}'` -set x_step_d1 = `cat $2|grep X_STEP|awk '{print $2}'` -set y_step_d1 = `cat $2|grep X_STEP|awk '{print $2}'` -gmt xyz2grd $1 -G"date1_ztd.grd" -RLT$x_first_d1/$y_first_d1/$width_d1/$length_d1 -I$x_step_d1/$y_step_d1 -ZTLf -di0 -r - - -#######SECOND ZTD to grid######### -set x_first_d2 = `cat $4|grep X_FIRST|awk '{print $2}'` -set y_first_d2 = `cat $4|grep Y_FIRST|awk '{print $2}'` -set width_d2 = `cat $4|grep WIDTH|awk '{print $2}'` -set length_d2 = `cat $4|grep FILE_LENGTH|awk '{print $2}'` -set x_step_d2 = `cat $4|grep X_STEP|awk '{print $2}'` -set y_step_d2 = `cat $4|grep X_STEP|awk '{print $2}'` -gmt xyz2grd $3 -G"date2_ztd.grd" -RLT$x_first_d2/$y_first_d2/$width_d2/$length_d2 -I$x_step_d2/$y_step_d2 -ZTLf -di0 -r -gmt grdsample -R"date1_ztd.grd" date2_ztd.grd -I3s -Gtmp.grd -mv tmp.grd date2_ztd.grd -#TIME DIFFERENCE -gmt grdmath date2_ztd.grd date1_ztd.grd SUB = zpddm.grd -#######RESAMPLE DIFFERENCE WITH THE INTERFEROGRAM PARAMETERS#### -set xmin = `gmt grdinfo -C $5|awk '{print $2}'` -set xmax = `gmt grdinfo -C $5|awk '{print $3}'` -set ymin = `gmt grdinfo -C $5|awk '{print $4}'` -set ymax = `gmt grdinfo -C $5|awk '{print $5}'` -set xinc = `gmt grdinfo -C $5|awk '{print $8}'` -set yinc = `gmt grdinfo -C $5|awk '{print $9}'` -#Resample -#gmt grdsample zpddm.grd -Gresample_zpddm.grd -R$xmin/$xmax/$ymin/$ymax -I$xinc/$yinc -r -#gmt grd2xyz zpddm.grd >tmp -#gmt xyz2grd -R$xmin/$xmax/$ymin/$ymax -I$xinc/$yinc tmp -Gresample_zpddm.grd -gmt grdsample zpddm.grd -Gresample_zpddm.grd -R$5 -I$xinc/$yinc - -#FROM METERS TO PHASE -#gmt grdmath resample_zpddm.grd $wavelength DIV 4 MUL $pi MUL = resample_zpddm_phs.grd -# multiplying by -4 (edition 13/4/2023) -gmt grdmath resample_zpddm.grd $wavelength DIV -4 MUL $pi MUL = resample_zpddm_phs.grd -#cp resample_zpddm.grd resample_zpddm_phs.grd -#REFERENCE POINT -set ref_value_zpddm = `gmt grdtrack $reference_point -Gresample_zpddm_phs.grd -Z` -set ref_value_phase = `gmt grdtrack $reference_point -G$5 -Z` -gmt grdmath resample_zpddm_phs.grd $ref_value_zpddm SUB = szpddm.grd -gmt grdmath $5 $ref_value_phase SUB = phase_ref.grd - -#FROM METER TO PHASE -#gmt grdmath szpddm.grd 4 MUL $pi MUL $wavelength DIV = szpddm_phase.grd -#gmt grdinfo szpddm.grd -#PROJECTION FROM ZENITH VIEW TO LOS -#gmt grdmath szpddm_phase.grd $incidence COSD DIV = szpddm_phase_LOS.grd -gmt grdmath szpddm.grd $incidence COSD DIV = szpddm_LOS.grd -#CORRECTION WITH GACOS DATA -#UNITS: Phase -gmt grdmath phase_ref.grd szpddm_LOS.grd SUB = phase_GACOS_corrected_phs.grd - -#DETRENDING -#UNITS: phase -gmt grdtrend phase_GACOS_corrected_phs.grd -N3r -Dphase_GACOS_corrected_phs_detrended.grd -gmt grdmath phase_GACOS_corrected_phs.grd $wavelength MUL -4 DIV $pi DIV = phase_GACOS_corrected_los.grd -gmt grdmath phase_GACOS_corrected_phs_detrended.grd $wavelength MUL -4 DIV $pi DIV = phase_GACOS_corrected_los_detrended.grd - -#clean up -# -rm date1_ztd.grd date2_ztd.grd -rm zpddm.grd -rm resample_zpddm.grd resample_zpddm_phs.grd -rm phase_ref.grd -rm szpddm.grd -rm szpddm_LOS.grd -echo "corrections done with GACOS files $1 and $3 over interferogram: $5" diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/slc2ifg.py b/.codex_tmp/pyint_variants/no_rescue/pyint/slc2ifg.py deleted file mode 100644 index 43d1d26..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/slc2ifg.py +++ /dev/null @@ -1,138 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import subprocess -import time -import glob -import argparse - -from pyint import _utils as ut - - -def get_s1_date(raw_file): - file0 = os.path.basename(raw_file) - date = file0[17:25] - return date - -def get_satellite(raw_file): - if 'S1A_IW_SLC_' in raw_file: - s0 = 'A' - else: - s0 = 'B' - - return s0 - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Generate Ifg from SLC using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName', help='name of the project.') - parser.add_argument('Mdate',help='master date for interferometry.') - parser.add_argument('Sdate',help='slave date for interferometry.') - - inps = parser.parse_args() - - return inps - - -INTRODUCTION = ''' --------------------------------------------------------------- - Generate unwrapped differential Ifg from SLC using GAMMA. - - Note: SRTM-1 will be downloaded and processed automatically - if not provided in the template file. - -''' - -EXAMPLE = """Usage: - - slc2ifg.py projectName Mdate Sdate --------------------------------------------------------------- -""" - -def main(argv): - - start_time = time.time() - inps = cmdLineParse() - Mdate = inps.Mdate - Sdate = inps.Sdate - - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - masterDate = templateDict['masterDate'] - #downDir = scratchDir + '/' + projectName + '/DOWNLOAD' - #M_raw = glob.glob(downDir + '/S1*_' + ut.yyyymmdd(Mdate)+'*')[0] - #S_raw = glob.glob(downDir + '/S1*_' + ut.yyyymmdd(Sdate)+'*')[0] - slcDir = scratchDir + '/' + projectName + '/SLC' - - - ######### down 2 slc ############# - #call_str = 'down2slc_sen.py ' + M_raw + ' ' + slcDir - #os.system(call_str) - - #call_str = 'down2slc_sen.py ' + S_raw + ' ' + slcDir - #os.system(call_str) - - ########## extract common bursts ## - if 'S1' in projectName: - call_str = 'extract_s1_bursts.py ' + projectName + ' ' + Mdate - os.system(call_str) - - call_str = 'extract_s1_bursts.py ' + projectName + ' ' + Sdate - os.system(call_str) - - ######### generate rdc_dem ########## - call_str = 'generate_rdc_dem.py ' + projectName - os.system(call_str) - - ########## coregister SLC ######## - if 'S1' in projectName: - call_str = 'coreg_s1_gamma.py ' + projectName + ' ' + Mdate - os.system(call_str) - - call_str = 'coreg_s1_gamma.py ' + projectName + ' ' + Sdate - os.system(call_str) - else: - call_str = 'coreg_gamma.py ' + projectName + ' ' + Mdate - os.system(call_str) - - call_str = 'coreg_gamma.py ' + projectName + ' ' + Sdate - os.system(call_str) - - ######## Interferometry process ########### - call_str = 'diff_gamma.py ' + projectName + ' ' + Mdate + ' ' + Sdate - os.system(call_str) - - call_str = 'unwrap_gamma.py ' + projectName + ' ' + Mdate + ' ' + Sdate - os.system(call_str) - - call_str = 'geocode_gamma.py ' + projectName + ' ' + Mdate + '-' + Sdate - os.system(call_str) - - print("Generate Ifg from SLC data is done! ") - ut.print_process_time(start_time, time.time()) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/slc_sen_cat.py b/.codex_tmp/pyint_variants/no_rescue/pyint/slc_sen_cat.py deleted file mode 100644 index e1ff019..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/slc_sen_cat.py +++ /dev/null @@ -1,182 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# -import numpy as np -import os -import sys -import subprocess -import getopt -import time -import glob -import argparse -from pyint import _utils as ut -def get_s1_date(raw_file): - file0 = os.path.basename(raw_file) - date = file0[17:25] - return date - -def get_satellite(raw_file): - if 'S1A_IW_SLC_' in raw_file: - s0 = 'A' - else: - s0 = 'B' - - return s0 - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Generate SLC from Sentinel-1 raw data with orbit correction using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName', help='project name. e.g., ChangningT55S1A') - parser.add_argument('date',help='date to be processed. e.g., 20180101') - - inps = parser.parse_args() - - return inps - - -INTRODUCTION = ''' -------------------------------------------------------------------- - - Generate SLC from Sentinel-1 raw data using S1_import_SLC_from_zipfiles with orbit correction. - [Precise orbit data will be downloaded automatically] -''' - -EXAMPLE = """Usage: - - slc_sen_cat.py projectName date - - slc_sen_cat.py ChangningT55S1A 20180517 - -------------------------------------------------------------------- -""" - - - - - - -def main(argv): - - inps = cmdLineParse() - projectName = inps.projectName - date = inps.date - scratchDir = os.getenv('SCRATCHDIR') - projectDir = scratchDir + '/' + projectName - - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - slc_dir = projectDir + '/SLC' - down_dir = scratchDir + '/' + projectName + "/DOWNLOAD" - opod_dir = projectDir + '/OPOD' - - if not os.path.isdir(slc_dir): - os.mkdir(slc_dir) - if not os.path.isdir(opod_dir): - os.mkdir(opod_dir) - - work_dir = slc_dir + '/' + date - if not os.path.isdir(work_dir): - os.mkdir(work_dir) - - os.chdir(work_dir) - - t_date = 't_' + date - - call_str = 'ls ' + down_dir + '/S1*' + date + '* > ' + t_date - os.system(call_str) - start_swath = templateDict['start_swath'] - end_swath = templateDict['end_swath'] - k_swath = ut.get_sardata_swath(start_swath,end_swath) - - call_str = 'grep ' + date + ' ' + t_date + ' > t1_' + date - os.system(call_str) - t1_date = 't1_' + date - raw_files = ut.read_txt2list(t1_date) - raw_files = sorted(raw_files) - satellite = get_satellite(str(raw_files[0])) - orbit_file = ut.download_s1_orbit(date,opod_dir,satellite=satellite) - - for i in range(len(raw_files)): - zipfile_ref=str(raw_files[i]) - outfile_name=zipfile_ref.split('/')[-1].split('.')[0] - burst_number_table_ref=outfile_name + '.burst_number_table' - call_str = 'S1_BURST_tab_from_zipfile ' + t1_date + ' ' + str(raw_files[0]) + ' - 1' - os.system(call_str) - - call_str = call_str = 'grep ' + outfile_name + ' ' + 't1_' + date + ' > t2_'+date - os.system(call_str) - - t2_date = 't2_' + date - #call_str = 'S1_import_SLC_from_zipfiles ' + t2_date + ' ' + burst_number_table_ref + ' vv 0 ' + k_swath - call_str = 'S1_import_SLC_from_zipfiles ' + t2_date + ' ' + burst_number_table_ref + ' vv 0 ' + k_swath + ' ' + opod_dir + ' 1 1 ' - os.system(call_str) - - os.chdir(work_dir) - #call_str = "rename vv.slc.iw1 IW1_" + str(i+1) + ".slc * " - call_str = "rename 's/vv.slc.iw1/IW1_" + str(i+1) + ".slc/g' *" - os.system(call_str) - call_str = "rename 's/vv.slc.iw2/IW2_" + str(i+1) + ".slc/g' *" - #call_str = "rename vv.slc.iw2 IW2_" + str(i+1) + ".slc * " - os.system(call_str) - call_str = "rename 's/vv.slc.iw3/IW3_" + str(i+1) + ".slc/g' *" - #call_str = "rename vv.slc.iw3 IW3_" + str(i+1) + ".slc * " - os.system(call_str) - SLC_Tab = work_dir + '/' + date + '_' + str(i+1) + '_SLC_Tab' - SLC_list = sorted(glob.glob(work_dir + '/*IW?_' + str(i+1) + '.slc')) - SLC_par_list = sorted(glob.glob(work_dir + '/*IW?_' + str(i+1) + '.slc.par')) - TOP_par_list = sorted(glob.glob(work_dir + '/*IW?_' + str(i+1) + '.slc.TOPS_par')) - - for kk in range(len(SLC_list)): - call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' >> ' + SLC_Tab - os.system(call_str) - - - SLC_Tab = work_dir + '/' + date + '_SLC_Tab' - SLC_list = len(raw_files) - SLC_tab1 = date + '_' + '1_SLC_Tab' - SLC_tab2 = date + '_' + '2_SLC_Tab' - #call_str = 'grep "IW*" ' + date +'_1_SLC_Tab | wc -l' - n_swath = len(open(SLC_tab1, 'r').readlines()) - for kk in range(n_swath): - call_str = 'echo ' + date + '.IW' + str(kk+1) + '.slc ' + date + '.IW' + str(kk+1) + '.slc.par ' + date + '.IW' + str(kk+1) + '.slc.TOPS_par ' + ' >> ' + SLC_Tab - os.system(call_str) - - call_str = 'SLC_cat_S1_TOPS ' + SLC_tab1 + ' ' + SLC_tab2 + ' ' + SLC_Tab - os.system(call_str) - SLC_list = sorted(glob.glob(work_dir + '/*IW?.slc')) - SLC_par_list = sorted(glob.glob(work_dir + '/*IW?.slc.par')) - TOP_par_list = sorted(glob.glob(work_dir + '/*IW?.slc.TOPS_par')) - for kk in range(len(SLC_list)): - BURST = SLC_par_list[kk].replace('slc.par','burst.par') - call_str = 'SLC_burst_corners ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' > ' +BURST - os.system(call_str) - print("Down to SLC for %s is done! " % date) - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - - - - - - - - - - - - - - - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/test_srtm_manual.py b/.codex_tmp/pyint_variants/no_rescue/pyint/test_srtm_manual.py deleted file mode 100644 index 91bc344..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/test_srtm_manual.py +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env python -""" -测试手动处理 SRTM .hgt 文件的脚本 - -这个脚本演示如何在 Python < 3.12 环境下手动处理 SRTM 数据 -""" - -import os -import sys - -print("=" * 80) -print("SRTM 手动处理功能测试") -print("=" * 80) - -# 检查Python版本 -print(f"\nPython版本: {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}") - -# 导入处理函数 -from makedem import process_srtm_hgt_files, HAS_SRTM - -print(f"\nsrtm库状态: {'已安装' if HAS_SRTM else '未安装 (需要Python >= 3.12)'}") - -print("\n" + "=" * 80) -print("使用说明") -print("=" * 80) - -print(""" -步骤1: 下载SRTM .hgt文件 ---------------------------- -从以下网站下载所需区域的 .hgt 文件: - -1. CSI-CGIAR SRTM (推荐,无需注册) - https://srtm.csi.cgiar.org/ - -2. USGS EarthExplorer (需要注册) - https://earthexplorer.usgs.gov/ - -文件命名规则: -- N39E116.hgt → 北纬39-40°, 东经116-117° -- N39W110.hgt → 北纬39-40°, 西经110-109° -- S10E120.hgt → 南纬10-9°, 东经120-121° - -步骤2: 组织文件 ---------------------------- -将所有 .hgt 文件放在同一目录下: - -mkdir -p ~/data/SRTM -mv N*.hgt S*.hgt ~/data/SRTM/ - -步骤3: 运行命令 ---------------------------- -使用以下命令生成DEM: - -makedem.py -r 116/117/39/40 --dem-source srtm --srtm-data-dir ~/data/SRTM - -或者直接调用Python函数: - -from makedem import process_srtm_hgt_files - -dem_file = process_srtm_hgt_files( - west=116.0, - south=39.0, - east=117.0, - north=40.0, - srtm_data_dir='/path/to/srtm/data', - save_path='/path/to/output' -) - -""") - -print("=" * 80) -print("示例: 为北京地区(116-117°E, 39-40°N)下载SRTM数据") -print("=" * 80) - -print(""" -需要下载的文件: -1. 访问 https://srtm.csi.cgiar.org/ -2. 选择区域: 39-40°N, 116-117°E -3. 下载文件: N39E116.hgt - -下载命令示例: --------------""") -print(f""" -# 创建数据目录 -mkdir -p ~/data/SRTM - -# 假设已下载 N39E116.hgt 到 ~/Downloads -mv ~/Downloads/N39E116.hgt ~/data/SRTM/ - -# 生成DEM -makedem.py -r 116/117/39/40 --dem-source srtm --srtm-data-dir ~/data/SRTM - -# 输出文件将保存为: SRTM_116_117_39_40.tif -""") - -print("=" * 80) -print("注意事项") -print("=" * 80) - -print(""" -1. SRTM文件必须覆盖整个目标区域 - - 对于区域 116-117°E, 39-40°N - - 需要 N39E116.hgt 文件 - -2. 文件格式 - - 支持 .hgt 文件 (未压缩) - - 如果下载的是 .hgt.zip, 请先解压 - -3. 空数据区域 - - SRTM数据在海洋区域可能为空值 - - 程序会自动处理空值区域 - -4. Python版本兼容性 - - Python >= 3.12: 可使用 srtm 库(自动查询) - - Python < 3.12: 使用手动处理方法(GDAL合并) - - 两种方法结果相同,只是处理方式不同 -""") - -print("=" * 80) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/test_tiled.dem b/.codex_tmp/pyint_variants/no_rescue/pyint/test_tiled.dem deleted file mode 100644 index 266abaa..0000000 Binary files a/.codex_tmp/pyint_variants/no_rescue/pyint/test_tiled.dem and /dev/null differ diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/test_tiled.dem.par b/.codex_tmp/pyint_variants/no_rescue/pyint/test_tiled.dem.par deleted file mode 100644 index 3123a30..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/test_tiled.dem.par +++ /dev/null @@ -1,27 +0,0 @@ -Gamma DIFF&GEO DEM/MAP parameter file -title: IMPORTED DEM FROM Copernicus30 -DEM_projection: EQA -data_format: INTEGER*2 -DEM_hgt_offset: 0.00000 -DEM_scale: 1.00000 -width: 6000 -nlines: 4800 -corner_lat: 28.000416666673939 decimal degrees -corner_lon: 101.999583333269271 decimal degrees -post_lat: -0.000833333333333 decimal degrees -post_lon: 0.000833333333333 decimal degrees - -ellipsoid_name: WGS 84 -ellipsoid_ra: 6378137.000 m -ellipsoid_reciprocal_flattening: 298.2572236 - -datum_name: WGS 1984 -datum_shift_dx: 0.000 m -datum_shift_dy: 0.000 m -datum_shift_dz: 0.000 m -datum_scale_m: 0.00000e+00 -datum_rotation_alpha: 0.00000e+00 arc-sec -datum_rotation_beta: 0.00000e+00 arc-sec -datum_rotation_gamma: 0.00000e+00 arc-sec -datum_country_list Global Definition, WGS84, World - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/tsview_mintpy_ssa.py b/.codex_tmp/pyint_variants/no_rescue/pyint/tsview_mintpy_ssa.py deleted file mode 100644 index 4e24c1f..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/tsview_mintpy_ssa.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python -# Load the usual suspects: -import os -import sys -import argparse -import pandas as pd -import numpy as np -import math -import matplotlib.pyplot as plt -from mintpy.utils import utils as ut -from ssa import SSA -from sklearn.metrics import mean_squared_error - - -def create_parser(): - parser = argparse.ArgumentParser(description='SSA time series analysis for a specific location') - parser.add_argument('ts_dir', help='Time series directory (e.g., ./SBAS_atm_gacos)') - parser.add_argument('ts_file', help='Time series file name (e.g., geo/geo_timeseries_SET_GACOS_ramp_demErr.h5)') - parser.add_argument('lat', type=float, help='Latitude of the point') - parser.add_argument('lon', type=float, help='Longitude of the point') - parser.add_argument('-L', '--window', type=int, default=8, help='Window length for SSA (default: 8)') - parser.add_argument('-o', '--output', default='ssa_ts.txt', help='Output file name (default: ssa_ts.txt)') - return parser - - -def parse_args(): - parser = create_parser() - return parser.parse_args() - - -def DecYr(x): - return x.dt.to_period('D').dt.to_timestamp().dt.year + x.dt.to_period('D').dt.to_timestamp().dt.dayofyear / 365.25 - - -def sigma_sum(Sigma): - sigma_sumsq = (Sigma**2).sum() - return Sigma**2 / sigma_sumsq * 100 - - -def main(): - args = parse_args() - - proj_dir = os.path.expanduser(args.ts_dir) - ts_file = os.path.join(proj_dir, args.ts_file) - geom_file = None - - print(f"Reading time series from: {ts_file}") - print(f"Location: lat={args.lat}, lon={args.lon}") - print(f"Window length: {args.window}") - - dates, dis, std = ut.read_timeseries_lalo(lat=args.lat, lon=args.lon, ts_file=ts_file, lookup_file=geom_file) - - # Convert from meter to mm and save to panda df - df = pd.DataFrame({'date': dates, 'dis': dis*1000}) - - # Decomposition - F_ssa = SSA(dis*1000, args.window) - contri = sigma_sum(F_ssa.Sigma) - df_ssa = pd.concat([F_ssa.components_to_df()], axis=1) - np.savetxt('sigma_ssa.txt', np.c_[np.arange(1, args.window+1), contri], fmt="%.2f") - df_comp = pd.concat([df['date'], F_ssa.components_to_df()], axis=1) - print('\n \n', 'Writing SSA all components to: ssa_all_compo.txt ', '\n') - df_comp.to_csv('ssa_all_compo.txt', index=False, header=True, sep='\t') - - plt.rcParams.update({'font.size': 10}) - fig, ax = plt.subplots(nrows=args.window, ncols=1, sharex=True, figsize=(5, 11*args.window)) - for i, column in enumerate(df_ssa.columns): - ax[i].plot(df_ssa.index, df_ssa[column], label=column) - ax[i].set_title(f'{column} ({contri[i]:.1f}%)') - plt.subplots_adjust(hspace=.5) - - # Reconstruction - trend_indices = list(range(min(2, args.window))) - noise_indices = list(range(2, args.window)) - - df['ssa_trend'] = pd.DataFrame(F_ssa.reconstruct(trend_indices)) - df['noise'] = pd.DataFrame(F_ssa.reconstruct(noise_indices)) if noise_indices else 0 - print('\n \n', 'Writing to: ', args.output, '\n') - df.to_csv(args.output, index=False, header=True, sep='\t') - - fig, ax = plt.subplots(nrows=3, ncols=1, sharex=True, figsize=(10, 10)) - - ax[0].plot(df['date'], df['dis'], label='InSAR') - ax[0].set_title('InSAR') - ax[1].plot(df['date'], df['noise'], label='Noise') - ax[1].set_title('Periodic+Noise') - ax[2].plot(df['date'], df['ssa_trend'], label='Trend') - ax[2].set_title('Trend') - plt.show() - - -if __name__ == '__main__': - main() diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/unwrap_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/unwrap_gamma.py deleted file mode 100644 index 0995fc0..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/unwrap_gamma.py +++ /dev/null @@ -1,156 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -from PIL import Image -from pylab import * -import argparse - -from pyint import _utils as ut - - -def _run_or_raise(call_str, stage): - rc = os.system(call_str) - if rc != 0: - raise RuntimeError('%s failed with rc=%s: %s' % (stage, rc, call_str)) - return rc - - -INTRODUCTION = ''' -------------------------------------------------------------------- - Unwrap differential interferogram using GAMMA. - [Only support mcf, not implement branch_cut yet] - -''' - -EXAMPLE = ''' - Usage: - unwrap_gamma.py projectName Mdate Sdate - unwrap_gamma.py PacayaT163TsxHhA 20150102 20150601 -------------------------------------------------------------------- -''' - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Unwrap differential interferogram using GAMMA-mcf method.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('Mdate',help='Master date.') - parser.add_argument('Sdate',help='Slave date.') - - inps = parser.parse_args() - return inps - - -def main(argv): - - inps = cmdLineParse() - Mdate = inps.Mdate - Sdate = inps.Sdate - - projectName = inps.projectName - Sdate = inps.Sdate - Mdate = inps.Mdate - - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - auto_unw = templateDict['auto_unw'] - init_flag = templateDict['init_flag'] - r_refer = templateDict['r_refer'] - a_refer = templateDict['a_refer'] - make_mask = templateDict['make_mask'] - processDir = scratchDir + '/' + projectName + "/PROCESS" - slcDir = scratchDir + '/' + projectName + "/SLC" - rslcDir = scratchDir + '/' + projectName + '/RSLC' - ifgDir = scratchDir + '/' + projectName + '/ifgrams' - - Pair = Mdate + '-' + Sdate - workDir = ifgDir + '/' + Pair - - ################ copy file for parallel processing ############### - Mamp0 = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' - MampPar0 = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' - Samp0 = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' - SampPar0 = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' - - - Mamp = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp' - MampPar = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp.par' - Samp = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp' - SampPar = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp.par' - - ut.copy_file(Mamp0,Mamp) - ut.copy_file(Samp0,Samp) - ut.copy_file(MampPar0,MampPar) - ut.copy_file(SampPar0,SampPar) - ############################################################### - - nWidth = ut.read_gamma_par(MampPar, 'read', 'range_samples') - nLine = ut.read_gamma_par(MampPar, 'read', 'azimuth_lines') - if auto_unw == '1': - if str(r_refer).strip() in ('', '-'): - r_refer = str(int(int(nWidth) / 2)) - if str(a_refer).strip() in ('', '-'): - a_refer = str(int(int(nLine) / 2)) - unwrap_window = '0 0 ' + nWidth + ' ' + nLine - - CORMASK = workDir + '/' + Pair + '_' +rlks + 'rlks.diff_filt.cor' - WRAPlks = workDir + '/' + Pair + '_' +rlks + 'rlks.diff_filt' - UNWlks = workDir + '/' + Pair + '_' +rlks + 'rlks.diff_filt.unw' - - CORMASKbmp = CORMASK.replace('.diff_filt.cor','.diff_filt.cor_mask.bmp') - - if os.path.isfile(CORMASKbmp): - os.remove(CORMASKbmp) - - call_str = 'rascc_mask ' + CORMASK + ' ' + Mamp + ' ' + nWidth + ' 1 1 0 1 1 ' + templateDict['unwrapThreshold'] + ' 0.0 0.1 0.9 1. .35 1 ' + CORMASKbmp # based on int coherence - _run_or_raise(call_str, 'rascc_mask') - - if auto_unw == '1': - if make_mask == "1": - call_str = 'mcf ' + WRAPlks + ' ' + CORMASK + ' ' + CORMASKbmp + ' ' + UNWlks + ' ' + nWidth + ' ' + templateDict['mcf_triangular'] + ' ' + unwrap_window + ' ' + templateDict['unwrap_patr'] + ' ' + templateDict['unwrap_pataz'] +' - '+ r_refer + ' ' + a_refer + ' ' + init_flag - print(call_str) - _run_or_raise(call_str, 'mcf_masked') - else: - call_str = 'mcf ' + WRAPlks + ' ' + CORMASK + ' - ' + ' ' + UNWlks + ' ' + nWidth + ' ' + templateDict['mcf_triangular'] + ' ' + unwrap_window + ' ' + templateDict['unwrap_patr'] + ' ' + templateDict['unwrap_pataz'] + ' - ' + r_refer + ' ' + a_refer + ' ' + init_flag - _run_or_raise(call_str, 'mcf_unmasked') - else: - im = array(Image.open(CORMASKbmp)) - imshow(im) - print('Please select the reference points,max click 10 times') - x =ginput(10) - print ('you clicked:',x) - ref_point=list(x[-1]) - r_init=int(ref_point[0]) - a_init=int(ref_point[1]) - if make_mask == '1': - call_str = 'mcf ' + WRAPlks + ' ' + CORMASK + ' ' + CORMASKbmp + ' ' + UNWlks + ' ' + nWidth + ' ' + templateDict['mcf_triangular'] + ' ' + unwrap_window + ' ' + templateDict['unwrap_patr'] + ' ' + templateDict['unwrap_pataz'] + ' - ' + str(r_init) + ' ' + str(a_init) + ' ' + init_flag - _run_or_raise(call_str, 'mcf_masked_manual') - else: - call_str = 'mcf ' + WRAPlks + ' - ' + ' - ' + ' ' + UNWlks + ' ' + nWidth + ' ' + templateDict['mcf_triangular'] + ' ' + unwrap_window + ' ' + templateDict['unwrap_patr'] + ' ' + templateDict['unwrap_pataz'] + ' - ' + str(r_init) + ' ' + str(a_init) + ' ' + init_flag - _run_or_raise(call_str, 'mcf_unmasked_manual') - call_str = 'rasdt_pwr ' + UNWlks + ' ' + Mamp + ' ' + nWidth + ' 1 0 1 1 -3.14 3.14 1' - _run_or_raise(call_str, 'rasdt_pwr_unw') - - if os.path.isfile(Mamp): - os.remove(Mamp) - if os.path.isfile(Samp): - os.remove(Samp) - #os.remove(MampPar) - #os.remove(SampPar) - print("Uwrapping interferometric phase is done!") - sys.exit(0) - -if __name__ == '__main__': - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/unwrap_gamma_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/unwrap_gamma_all.py deleted file mode 100644 index 0cf1749..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/unwrap_gamma_all.py +++ /dev/null @@ -1,143 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import getopt -import time -import glob -import argparse - -import subprocess -from pyint import _utils as ut - - -def work(data0): - cmd = data0[0] - err_file = data0[1] - p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) - stdout = p.stdout - stderr = p.stderr - - if type(stderr) == bytes: - aa=stderr.decode("utf-8") - else: - aa = stderr - - if type(stdout) == bytes: - bb=stdout.decode("utf-8", errors="replace") - else: - bb = stdout - - if p.returncode != 0: - detail_parts = [] - if bb: - detail_parts.append(bb) - if aa: - detail_parts.append(aa) - detail = '\n'.join(part.strip() for part in detail_parts if part and part.strip()) - str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + cmd[3] + '\n' - with open(err_file, 'a') as f: - f.write(str0) - if detail: - f.write(detail) - f.write('\n') - raise RuntimeError(str0.strip() + ' failed with rc=' + str(p.returncode) + ('\n' + detail if detail else '')) - - if aa: - str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + cmd[3] + '\n' - with open(err_file, 'a') as f: - f.write(str0) - f.write(aa) - f.write('\n') - - return -######################################################################### - -INTRODUCTION = ''' -------------------------------------------------------------------- - Unwrap differential interferograms for one project using GAMMA. - -''' - -EXAMPLE = ''' - Usage: - unwrap_gamma_all.py projectName - unwrap_gamma_all.py projectName --parallel 4 - unwrap_gamma_all.py projectName --parallel 4 --ifgramList-txt /test/ifgram_list.txt -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Unwrap differential interferograms for one project using GAMMA.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') - parser.add_argument('--ifgarmList-txt', dest='ifgarmListTxt', help='provided ifgram_list_txt. default: using ifgram_list.txt under projectName folder.') - - inps = parser.parse_args() - return inps - - -def main(argv): - start_time = time.time() - inps = cmdLineParse() - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - templateDir = os.getenv('TEMPLATEDIR') - templateFile = templateDir + "/" + projectName + ".template" - projectDir = scratchDir + '/' + projectName - ifgDir = scratchDir + '/' + projectName + '/ifgrams' - templateDict=ut.update_template(templateFile) - rlks = templateDict['range_looks'] - azlks = templateDict['azimuth_looks'] - - if inps.ifgarmListTxt: ifgramList_txt = inps.ifgarmListTxt - else: ifgramList_txt = scratchDir + '/' + projectName + '/ifgram_list.txt' - ifgList0 = ut.read_txt2array(ifgramList_txt) -# ifgList = ifgList0[:,0] - if len(ifgList0)==3: - ifgList=ifgList0[0] - ifgList=[ifgList] - else: - ifgList=ifgList0[:,0] - - err_txt = scratchDir + '/' + projectName + '/unwrap_gamma_all.err' - if os.path.isfile(err_txt): os.remove(err_txt) - - data_para = [] - for i in range(len(ifgList)): - m0 = ut.yyyymmdd(ifgList[i].split('-')[0]) - s0 = ut.yyyymmdd(ifgList[i].split('-')[1]) - cmd0 = ['unwrap_gamma.py',projectName, m0, s0] - unw_file0 = ifgDir + '/' + ifgList[i] + '/' + ifgList[i] + '_' + rlks + 'rlks.diff_filt.unw.bmp' - data0 = [cmd0,err_txt] - - k00 = 0 - if os.path.isfile(unw_file0): - if os.path.getsize(unw_file0) > 0: - k00 = 1 - if k00==0: - data_para.append(data0) - - results = ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) - failures = [str(item) for item in results if isinstance(item, Exception)] - if failures: - raise RuntimeError('\n\n'.join(failures)) - print("Unwrap differential interferograms for project %s is done! " % projectName) - ut.print_process_time(start_time, time.time()) - - sys.exit(0) - -if __name__ == '__main__': - main(sys.argv[:]) - diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/unwrap_snaphu_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/unwrap_snaphu_gamma.py deleted file mode 100644 index 9c51057..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/unwrap_snaphu_gamma.py +++ /dev/null @@ -1,447 +0,0 @@ -#! /usr/bin/env python - -################################################################# - -### This program is part of PyINT v2.1 ### - -### Copy Right (c): 2017-2019, Yunmeng Cao ### - -### Author: Yunmeng Cao ### - -### Contact : ymcmrs@gmail.com ### - -################################################################# - - - -import numpy as np - -import os - -import sys - -import argparse - -import subprocess - -import re - - - -from pyint import _utils as ut - - - -INTRODUCTION = ''' - -------------------------------------------------------------------- - - Unwrap differential interferogram using SNAPHU. - - - -''' - - - -EXAMPLE = ''' - - Usage: - - unwrap_snaphu.py projectName Mdate Sdate - - unwrap_snaphu.py PacayaT163TsxHhA 20150102 20150601 - -------------------------------------------------------------------- - -''' - - - -def cmdLineParse(): - - parser = argparse.ArgumentParser(description='Unwrap differential interferogram using SNAPHU.',\ - - formatter_class=argparse.RawTextHelpFormatter,\ - - epilog=INTRODUCTION+'\n'+EXAMPLE) - - - - parser.add_argument('projectName', help='projectName for processing.') - - parser.add_argument('Mdate', help='Master date.') - - parser.add_argument('Sdate', help='Slave date.') - - - - inps = parser.parse_args() - - return inps - - - -def run_command(cmd): - - print(f"Running: {cmd}") - - status = subprocess.call(cmd, shell=True) - - if status != 0: - - print(f"Error running command: {cmd}") - - sys.exit(1) - - - -def extract_value(s): - - """Extract numeric value from string that may contain units""" - - # Remove any non-numeric characters except decimal point and minus sign - - return re.sub(r'[^\d\.\-]', '', s) - - - -def main(argv): - - inps = cmdLineParse() - - projectName = inps.projectName - - Mdate = inps.Mdate - - Sdate = inps.Sdate - - - - scratchDir = os.getenv('SCRATCHDIR') - - templateDir = os.getenv('TEMPLATEDIR') - - templateFile = templateDir + "/" + projectName + ".template" - - templateDict = ut.update_template(templateFile) - - rlks = templateDict['range_looks'] - - azlks = templateDict['azimuth_looks'] - - - - processDir = scratchDir + '/' + projectName + "/PROCESS" - - slcDir = scratchDir + '/' + projectName + "/SLC" - - rslcDir = scratchDir + '/' + projectName + '/RSLC' - - ifgDir = scratchDir + '/' + projectName + '/ifgrams' - - - - Pair = Mdate + '-' + Sdate - - workDir = ifgDir + '/' + Pair - - - - # Create working directory if not exists - - os.makedirs(workDir, exist_ok=True) - - - - # Copy required files for processing - - Mamp0 = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' - - MampPar0 = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' - - Samp0 = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' - - SampPar0 = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' - - - - Mamp = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp' - - MampPar = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp.par' - - Samp = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp' - - SampPar = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp.par' - - off_par = workDir + '/' + Pair + '_' + rlks + 'rlks.off' - - ut.copy_file(Mamp0, Mamp) - - ut.copy_file(Samp0, Samp) - - ut.copy_file(MampPar0, MampPar) - - ut.copy_file(SampPar0, SampPar) - - - - # Define input/output files - - int_dir = workDir - - rootname = Pair + '_' + rlks + 'rlks' - - int_file = int_dir + '/' + rootname + '.diff_filt' - - cor_file = int_dir + '/' + rootname + '.diff_filt.cor' - - - - # Read parameters from par files - - width = ut.read_gamma_par(off_par, 'read', 'interferogram_azimuth_lines:') - length = ut.read_gamma_par(off_par, 'read', 'interferogram_width:') - #length = ut.read_gamma_par(MampPar, 'read', 'azimuth_lines') - row = ut.read_gamma_par(off_par, 'read', 'interferogram_azimuth_lines:') - col = ut.read_gamma_par(off_par, 'read', 'interferogram_width:') - pos = ut.read_gamma_par(MampPar, 'read', 'sar_to_earth_center') - earth = ut.read_gamma_par(MampPar, 'read', 'earth_radius_below_sensor') - near = ut.read_gamma_par(MampPar, 'read', 'near_range_slc') - dr = ut.read_gamma_par(off_par, 'read', 'interferogram_range_pixel_spacing:') - da_flight = ut.read_gamma_par(off_par, 'read', 'interferogram_azimuth_pixel_spacing:') - rangeres = ut.read_gamma_par(MampPar, 'read', 'range_pixel_spacing') - azres = ut.read_gamma_par(MampPar, 'read', 'azimuth_pixel_spacing') - nrange = ut.read_gamma_par(off_par, 'read', 'interferogram_range_looks:') - nazi = ut.read_gamma_par(off_par, 'read', 'interferogram_azimuth_looks:') - - - # Extract numeric values - earth = extract_value(earth) - pos = extract_value(pos) - near = extract_value(near) - dr = extract_value(dr) - da_flight = extract_value(da_flight) - nrange = extract_value(nrange) - nazi = extract_value(nazi) - rangeres = extract_value(rangeres) - azres = extract_value(azres) - - alt = str(float(pos) - float(earth)) - da = str(float(da_flight) * float(earth) / (float(earth) + float(alt))) - ncor = str(float(dr) * float(da) / (float(rangeres) * float(azres))) - - - - - - - - - - - - # Create SNAPHU configuration file - - conf_file = workDir + '/' + rootname + '.snaphuconf' - - with open(conf_file, 'w') as f: - - f.write(f"STATCOSTMODE SMOOTH\n") - - f.write(f"INFILE {rootname}.int\n") - - f.write(f"LINELENGTH {width}\n") # 使用原始宽度 - - f.write(f"OUTFILE {rootname}.unw\n") - - f.write(f"CORRFILE {rootname}.cor\n") - - f.write(f"LOGFILE {rootname}.snaphulog\n") - - f.write(f"\n") - - f.write(f"PIECEFIRSTROW 1\n") - - f.write(f"PIECEFIRSTCOL 1\n") - - f.write(f"PIECENROW {length}\n") # 使用原始长度 - - f.write(f"PIECENCOL {width}\n") # 使用原始宽度 - - f.write(f"ALTITUDE {alt}\n") - - f.write(f"EARTHRADIUS {earth}\n") - - f.write(f"NEARRANGE {near}\n") - - f.write(f"BASELINE 0.000000\n") - - f.write(f"BASELINEANGLE_DEG 0.000000\n") - - f.write(f"TRANSMITMODE REPEATPASS\n") - - f.write(f"DR {dr}\n") - - f.write(f"DA {da}\n") - - f.write(f"RANGERES {rangeres}\n") - - f.write(f"AZRES {azres}\n") - - f.write(f"LAMBDA 0.0556\n") - - f.write(f"NLOOKSRANGE {nrange}\n") - - f.write(f"NLOOKSAZ {nazi}\n") - - f.write(f"NLOOKSOTHER 1\n") - - f.write(f"NCORRLOOKS {ncor}\n") - - f.write(f"\n") - - f.write(f"CONNCOMPFILE {rootname}.byt\n") - - f.write(f"MAXNCOMPS 32\n") - - f.write(f"\n") - - f.write(f"INFILEFORMAT COMPLEX_DATA\n") - - f.write(f"OUTFILEFORMAT ALT_LINE_DATA\n") - - f.write(f"CORRFILEFORMAT FLOAT_DATA\n") - - f.write(f"VERBOSE FALSE\n") - - - - # Swap bytes for input files - - run_command(f"swap_bytes {int_file} {rootname}.int 4") - - run_command(f"swap_bytes {cor_file} {rootname}.cor 4") - - - - # Run SNAPHU - - run_command(f"snaphu -f {conf_file}") - - - - # Post-processing steps - - xmin = "0" - - ymin = "0" - - xmax = width - - ymax = length - - - - # 获取SNAPHU输出文件的尺寸 - - unwrapped_file = f"{rootname}.unw" - new_width = str(int(xmax)-int(xmin)) - - # 计算填充量 - - npad_bottom = str(int(length) - int(ymax)) - - npad_right = str(int(width) - int(xmax)) - - - - print(f"Original dimensions: {width}x{length}") - - print(f"SNAPHU output dimensions: {new_width}x{new_width}") - - print(f"Padding: bottom={npad_bottom}, right={npad_right}") - - - - # Zero padding - - unw_zero_file = rootname + "_zeropad.unw" - - mask_zero_file = rootname + "_mask_zeropad.unw" - - mask_zero_file_msk = rootname + "_mask_zeropad.msk" - - tobegeocodedm = rootname + "_masked.unw" - - unw_file = rootname + "_msk.unw" - - - - run_command(f"zeropad_msk {rootname}.unw {width} {ymin} {xmin} {npad_right} {npad_bottom} {unw_zero_file} rmg") - - - - # Phase and magnitude processing - - run_command(f"rmg2mag_phs {unw_zero_file} /dev/null phs {width}") - - run_command(f"swap_bytes phs phs4 4") - - run_command(f"cpx2mag_phs {rootname}.int pwr /dev/null {width}") - - run_command(f"mag_phs2rmg pwr phs {unw_zero_file} {width}") - - - - # Mask processing - - run_command(f"zeropad_msk {rootname}.byt {width} {ymin} {xmin} {npad_right} {npad_bottom} {mask_zero_file_msk} msk") - - run_command(f"cpx2mag_phs {rootname}.int pwr phs {width}") - - run_command(f"mag_phs2rmg pwr {mask_zero_file_msk} {mask_zero_file} {width}") - - - - # Combine results - - run_command(f"rmg2mag_phs {unw_zero_file} pwr phs1 {width}") - - run_command(f"rmg2mag_phs {mask_zero_file} /dev/null phs2 {width}") - - run_command(f"add_phs phs1 phs2 phs3 {width} {length} 0 1") - - run_command(f"add_phs pwr phs2 pwr3 {width} {length} 0 1") - - run_command(f"mag_phs2rmg pwr3 phs3 {tobegeocodedm} {width}") - - run_command(f"swap_bytes phs3 {unw_file} 4") - - - - # Clean up temporary files - - run_command("rm -rf pwr phs phs1 phs2 phs3 pwr3") - - run_command(f"mv phs4 {unw_zero_file}") - - - - # Remove copied files - - os.remove(Mamp) - - os.remove(Samp) - - - - print("Unwrapping with SNAPHU is done!") - - sys.exit(0) - - - -if __name__ == '__main__': - - main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/unzip_s1_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/unzip_s1_all.py deleted file mode 100644 index bb8c162..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/unzip_s1_all.py +++ /dev/null @@ -1,101 +0,0 @@ -#! /usr/bin/env python -################################################################# -### This program is part of PyINT v2.1 ### -### Copy Right (c): 2017-2019, Yunmeng Cao ### -### Author: Yunmeng Cao ### -### Contact : ymcmrs@gmail.com ### -################################################################# - -import numpy as np -import os -import sys -import getopt -import time -import glob -import argparse - -import subprocess -from pyint import _utils as ut - -def work(data0): - cmd = data0[0] - err_txt = data0[1] - p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) - stdout = p.stdout - stderr = p.stderr - - if type(stderr) == bytes: - aa=stderr.decode("utf-8") - else: - aa = stderr - - if aa: - str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + cmd[3] + '\n' - #print(aa) - with open(err_txt, 'a') as f: - f.write(str0) - f.write(aa) - f.write('\n') - - return -######################################################################### - -INTRODUCTION = ''' -------------------------------------------------------------------- - Unzip Sentinel-1 raw dataset for one project. - [unzip does not need much CPU, so we can use parallel processing] - -''' - -EXAMPLE = ''' - Usage: - unzip_s1_all.py projectName - unzip_s1_all.py projectName --parallel 4 -------------------------------------------------------------------- -''' - - -def cmdLineParse(): - parser = argparse.ArgumentParser(description='Unzip Sentinel-1 raw dataset for one project.',\ - formatter_class=argparse.RawTextHelpFormatter,\ - epilog=INTRODUCTION+'\n'+EXAMPLE) - - parser.add_argument('projectName',help='projectName for processing.') - parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') - - inps = parser.parse_args() - return inps - - -def main(argv): - start_time = time.time() - inps = cmdLineParse() - projectName = inps.projectName - scratchDir = os.getenv('SCRATCHDIR') - projectDir = scratchDir + '/' + projectName - downDir = scratchDir + '/' + projectName + "/DOWNLOAD" - raw_file_list = glob.glob(downDir + '/S1*.zip') - - slc_dir = scratchDir + '/' + projectName + '/SLC' - if not os.path.isdir(slc_dir): - os.mkdir(slc_dir) - - err_txt = scratchDir + '/' + projectName + '/unzip_s1_all.err' - if os.path.isfile(err_txt): os.remove(err_txt) - - data_para = [] - for i in range(len(raw_file_list)): - cmd0 = ['unzip',raw_file_list[i],'-d',downDir] - data0 = [cmd0,err_txt] - data_para.append(data0) - - ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) - os.chdir(downDir) - print("Unzip Sentinel-1 raw dataset for project %s is done! " % projectName) - ut.print_process_time(start_time, time.time()) - - sys.exit(1) - -if __name__ == '__main__': - main(sys.argv[:]) - \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/utm2ll b/.codex_tmp/pyint_variants/no_rescue/pyint/utm2ll deleted file mode 100644 index 722d337..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/pyint/utm2ll +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/csh -f -# -#convet utm coord. to lon and lat -# -if ( $#argv < 5) then -more <<EOF - -utm2ll is a C-shell script which convets utm coord. to lon and lat - -Usage: utm2ll zone x0 y0 dx dy -Where zone is the conventional zone number=int(lon/6)+31 - x0,y0 is the origin of UTM projection at easting and northing(meter) - dx,dy is the offset from origin at meter unit - -Example: utm2ll 45 311072.36 3542183.56 27000 62000 - origin 85 32(lon,lat) - - Qiao Xuejun 2008/2/20 -EOF -exit -endif - -set zone = $argv[1] -set x0 = $argv[2] -set y0 = $argv[3] -set dx = $argv[4] -set dy = $argv[5] -set pi = 3.141592653589793 -set a = 6378137.00 -set b = 6356752.3142 -#echo $a $b -set k0 = 0.9996 -set x = `echo $x0 $dx |awk '{print 500000-($1+$2)}'` -set y = `echo $y0 $dy |awk '{print $1+$2}'` -set e = `echo $a $b |awk '{print sqrt(1-($2*$2)/($1*$1))}'` -set ep2 = `echo $e $a $b |awk '{print ($1*$2/$3)*($1*$2/$3)}'` -set sin1 = `echo $pi |awk '{print $1/(180*60*60)}'` -set m = `echo $y $k0|awk '{print $1/$2}'` -set mu = `echo $m $a $e|awk '{print $1/($2*(1-$3*$3/4-3*$3*$3*$3*$3/64-5*$3*$3*$3*$3*$3*$3/256))}'` -set e1 = `echo $e |awk '{print (1-sqrt(1-$1*$1))/ (1+sqrt(1-$1*$1))}'` -set j1 = `echo $e1 |awk '{print 3*$1/2-27*$1*$1*$1/32}'` -set j2 = `echo $e1 |awk '{print 21*$1*$1/16-55*$1*$1*$1*$1/32}'` -set j3 = `echo $e1 |awk '{print 151*$1*$1*$1/96}'` -set j4 = `echo $e1 |awk '{print 1097*$1*$1*$1*$1/512}'` -set fp = `echo $mu $j1 $j2 $j3 $j4|awk '{print $1+$2*sin(2*$1)+$3*sin(4*$1)+$4*sin(6*$1)+$5*sin(8*$1)}'` -set c1 = `echo $ep2 $fp|awk '{print $1*cos($2)*cos($2)}'` -set t1 = `echo $fp |awk '{print sin($1)*sin($1)/(cos($1)*cos($1))}'` -set r1 = `echo $a $e $fp |awk '{print $1*(1-$2*$2)/((1-$2*$2*sin($3)*sin($3))*sqrt(1-$2*$2*sin($3)*sin($3)))}'` -set n1 = `echo $a $e $fp |awk '{print $1/sqrt(1-$2*$2*sin($3)*sin($3))}'` -set d = `echo $x $n1 $k0 |awk '{print $1/($2*$3)}'` -set q1 = `echo $n1 $fp $r1|awk '{print $1*(sin($2)/cos($2))/$3}'` -set q2 = `echo $d |awk '{print $1*$1/2}'` -set q3 = `echo $t1 $c1 $ep2 $d |awk '{print (5+3*$1+10*$2-4*$2*$2-9*$3)*$4*$4*$4*$4/24 }'` -set q4 = `echo $t1 $c1 $ep2 $d |awk '{print (61+90*$1+298*$2+45*$1*$1-3*$2*$2-252*$3)*$4*$4*$4*$4*$4*$4/720 }'` -set q5 = $d -set q6 = `echo $t1 $c1 $d |awk '{print (1+2*$1+$2)*$3*$3*$3/6}'` -set q7 = `echo $c1 $t1 $ep2 $d |awk '{print (5-2*$1+28*$2-3*$1*$1+8*$3+24*$2*$2)*$4*$4*$4*$4*$4/120}'` -#echo $x $e $ep2 $sin1 $m $mu $e1 $j1 $j2 $j3 $j4 $fp $c1 $t1 $r1 $n1 $d $q1 $q2 $q3 $q4 $q5 $q6 $q7 -set dlong = `echo $q5 $q6 $q7 $fp|awk '{print ($1-$2+$3)/cos($4)}'` -set lat = `echo $fp $q1 $q2 $q3 $q4 |awk '{print $1-$2*($3-$4+$5)}'` -set lon0 = `echo $zone |awk '{print 6*$1-183}'` -set lon = `echo $lon0 $dlong $pi |awk '{print $1-$2*180/$3 }'` -set lat = `echo $lat $pi |awk '{print $1*180/$2 }'` -echo $lon $lat - - - diff --git a/.codex_tmp/pyint_variants/no_rescue/template/ensishenlongxiT134F058S1A.template b/.codex_tmp/pyint_variants/no_rescue/template/ensishenlongxiT134F058S1A.template deleted file mode 100644 index 24930b5..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/template/ensishenlongxiT134F058S1A.template +++ /dev/null @@ -1,117 +0,0 @@ -# *********************************** PyINT Template *******************************************# -# Modifed Sep. , 2019 - YM.Cao - -# ------------------- Basic parameters ----------------------- - -DEM = /data/insar/chenwei/SCRATCHDIR/ensishenlongxiT134F058S1A/DEM/srtm.dem -# if not provide, SRTM-1 will be downloaded and processed automatically. - -masterDate = 20180123 # [If not provided, the first date in the date_list will be selected as the master date] - -start_swath = 2 # [option (for Sentinel-1 only)] -end_swath = 3 # [option (for Sentinel-1 only)] -start_burst = 3 # [option: bursts position is based on the master image (for Sentinel-1 only)] -end_burst = 6 # [option: bursts position is based on the master image (for Sentinel-1 only)] - -range_looks = 20 # multi-look numbers in range direction -azimuth_looks = 4 # multi-look numbers in azimuth direction - -# ------------------- download data ----------------------- - -sensor = Sentinel-1A # Envisat, ERS-1/2, .. please check SSARA for details -track = 128 # track number of the SAR image -frame = 89 # frame number of the SAR image [ defined according to ASF ] -start_time = 2017-11-01 # -end_time = 2109-01-01 # - -# ------------------- basic parameters for interferometry ----------------------- - -dem_lat_ovr = 1 # as to SRTM-1, 0.5 for 60 m, 2 for 15 m -dem_lon_ovr = 1 - -Igram_Spsflg = 1 # Implement range spectral filtering -rbw_min = 1 # minimum range bandwidth fraction (0.1 --> 1.0)(default: - 0.250) - -rwin4cor = 256 # range window length for coregistration -azwin4cor = 256 # azimuth window length for coregistration -rsample4cor = 32 # range samples used for fitting the coregistration parameters -azsample4cor = 32 # azimuth samples used for fitting the coregistration parameters - -thresh4cor = 0.3 # 2016 GAMMA or higher version, for 2015 GAMMA or lower version should be SNR - -Igram_Cor_rwin = 5 # range window length for cc_wave coherence estimation -Igram_Cor_awin = 5 # azimuth window length for cc_wave coherence estimation - -Igram_Cor_Win = 5 # cc estimation in adf -adf_alpha = 0.4 # adf alpha for Gold-Stein filtering - -# ------------------- sim phase ----------------------- - -Igram_Flag_TDM = N # Y for Tandem-X -Simphase_rpos = - -Simphase_azpos = - -Simphase_rwin = 256 -Simphase_azwin = 256 -Simphase_thresh = - - -# ------------------- unwrap phase ----------------------- - -mcf_triangular = 1 # triangular type of mcf [0: regular; 1: delaunay;] -unwrap_patr = 1 # unwrap patches in range direction -unwrap_pataz = 1 # unwrap patches in azimuth direction -unwrapThreshold = 0.25 # coherence threshold used for unwrap -auto_unw = 1 # auto or manual to select the reference point. True:auto False: manual -make_mask = 1 # make a mask based on the cor file to mask the lower field 0: don't mask 1: make a mask -init_flag = 1 # flag to set phase at reference point. 0: use initial point phase value (default) 1: set phase to 0.0 at initial point -r_refer = 396 #phase reference point range offset (default(-): roff) -a_refer = 537 # phase reference point azimuth offset (default(-): loff) -# ------------------- geocode ----------------------- - -geo_interp = 0 # [0: nearest; 1: bicubic spline] - -#-------------------- trans phase to los displacement -------------- - -satelite = S1A # the value of satelite must be one in the list['S1A', 'CSK','TSX','ALSO2','ALOS','ENVISAT'] - -# ------------------- select interferometric pairs -------------- - -network_method = sbas # sbas, sequential, delaunay, stars -endDate = 20210809 # date before this date will be excluded -startDate = 20170320 # date after this date will be excluded -conNumb = 2 # connect number for sequential -max_tb = 100 # temporal baseline threshold in days -max_sb = 100 # spatial baseline threshold in meters -#exclude_date = # exclude the date which will not be used - -# ------------------- time-series process -------------- - -download_data = 0 # [0: skip; 1: process] -down_parallel = 0 # multi-processor number used for downloading - -raw2slc_all = 0 # i.e., download2slc [0: skip; 1: process] -raw2slc_all_parallel = 4 # multi-processor number used - -extract_burst_all = 0 # for TOPS SLC only [0: skip; 1: process] -extract_all_parallel = 1 # multi-processor number used - -coreg_all = 0 # do coregister [0: skip; 1: process] -coreg_all_parallel = 4 # multi-processor number used [4 or 8] - -select_pairs = 0 # if 0: skip, and use the ifgram_list.txt under PROJECTNAME folder - -diff_all = 0 # generate differential Ifg [0: skip; 1: process] -diff_all_parallel = 4 # multi-processor number used [4 or 8] - -unwrap_all = 1 # unwrap Ifg [0: skip; 1: process] -unwrap_all_parallel = 8 # multi-processor number used [8 or 10] - -geocode_all = 1 # geocode diff,unw products [0: skip; 1: process] -geocode_all_parallel = 4 # multi-processor number used - -load_data = 0 # loading data for mintpy processing [0: skip; 1: process] - -# ------------------- Good Luck -------------- -DEM= /data/insar/chenwei/SCRATCHDIR/ensishenlongxiT134F058S1A/DEM/ensishenlongxiT134F058S1A.dem -DEM= /data/insar/chenwei/SCRATCHDIR/DEM/ensishenlongxiT134F058S1A/ensishenlongxiT134F058S1A.dem -DEM= /data/insar/chenwei/SCRATCHDIR/DEM/ensishenlongxiT134F058S1A/ensishenlongxiT134F058S1A.dem -DEM= /data/insar/chenwei/SCRATCHDIR/DEM/ensishenlongxiT134F058S1A/ensishenlongxiT134F058S1A.dem diff --git a/.codex_tmp/pyint_variants/no_rescue/template/pyint.template b/.codex_tmp/pyint_variants/no_rescue/template/pyint.template deleted file mode 100644 index 6459886..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/template/pyint.template +++ /dev/null @@ -1,204 +0,0 @@ -# *********************************** PyINT Template *******************************************# -# Template for PyINT v2.1 -# Please fill in the parameters according to your project requirements - -# ========================= Basic parameters ======================== - -# DEM file path (if not provided, SRTM will be downloaded automatically from OpenTopography) -#DEM = /path/to/your/dem.dem - -# Master date for interferometric processing -masterDate = YYYYMMDD # Master date in YYYYMMDD format - -# Swath and burst parameters (for Sentinel-1 TOPS mode only) -start_swath = 1 # Starting swath number -end_swath = 3 # Ending swath number -start_burst = 1 # Starting burst position (relative to master image) -end_burst = 9 # Ending burst position (relative to master image) - -# Multilooking parameters -range_looks = 20 # Multi-look numbers in range direction -azimuth_looks = 4 # Multi-look numbers in azimuth direction - -# ========================= Download Data ======================== - -# Data source and search parameters -Data_Hub = ASF # Data download source: SSARA, Scihub, ASF -sensor = Sentinel-1A # Envisat, ERS-1/2, Sentinel-1A/B, etc. -track = 128 # Track number of the SAR image -frame = 89 # Frame number [defined according to ASF] -start_time = YYYY-MM-DD # Start date for data search -end_time = YYYY-MM-DD # End date for data search - -# ASF download specific parameters -asf_platform = Sentinel-1A # Platform: Sentinel-1A, Sentinel-1B -asf_beam_mode = IW # Beam mode: IW (Interferometric Wide) -asf_polarization = VV # Polarization: VV, VH, HH, HV -asf_flight_direction = ascending # Flight direction: ascending, descending -asf_bbox = W,S,E,N # Bounding box: West,South,East,North - -# ========================= DEM Generation ======================== - -# DEM download parameters (for OpenTopography) -opentopo_api_key = YOUR_API_KEY # OpenTopography API key -opentopo_dem_type = SRTMGL1 # DEM type: SRTMGL1(30m), SRTMGL3(90m), NASADEM(30m), COP30, COP90 -opentopo_auto_tile = 1 # Auto-tile large regions (recommended: 1) - -# DEM processing parameters -dem_lat_ovr = 1 # Latitude oversampling (as to SRTM-1: 0.5 for 60m, 2 for 15m) -dem_lon_ovr = 1 # Longitude oversampling - -# ========================= Interferometry Parameters ======================== - -# Spectral filtering -Igram_Spsflg = 1 # Implement range spectral filtering -rbw_min = 1 # Minimum range bandwidth fraction (0.1 --> 1.0) - -# Coregistration parameters -rwin4cor = 256 # Range window length for coregistration -azwin4cor = 256 # Azimuth window length for coregistration -rsample4cor = 32 # Range samples for fitting coregistration parameters -azsample4cor = 32 # Azimuth samples for fitting coregistration parameters -thresh4cor = 0.3 # Coherence threshold (GAMMA 2016+, SNR for older versions) - -# Coherence estimation -Igram_Cor_rwin = 5 # Range window length for cc_wave coherence estimation -Igram_Cor_awin = 5 # Azimuth window length for cc_wave coherence estimation -Igram_Cor_Win = 5 # Window for cc estimation in adf -adf_alpha = 0.4 # Alpha for Gold-Stein filtering - -# ========================= Simulation Phase ======================== - -Igram_Flag_TDM = N # Y for Tandem-X -Simphase_rpos = - -Simphase_azpos = - -Simphase_rwin = 256 -Simphase_azwin = 256 -Simphase_thresh = - - -# ========================= Unwrapping ======================== - -mcf_triangular = 1 # Triangular type of mcf [0: regular; 1: delaunay] -unwrap_patr = 1 # Unwrap patches in range direction -unwrap_pataz = 1 # Unwrap patches in azimuth direction -unwrapThreshold = 0.05 # Coherence threshold for unwrapping -auto_unw = 1 # Auto or manual reference point [1: auto; 0: manual] -make_mask = 0 # Make mask based on coherence [0: don't mask; 1: make mask] -init_flag = 1 # Flag to set phase at reference point [0: use initial; 1: set to 0.0] -r_refer = - # Phase reference point range offset -a_refer = - # Phase reference point azimuth offset - -# ========================= Geocoding ======================== - -geo_interp = 0 # Interpolation method [0: nearest; 1: bicubic spline] - -# ========================= Satellite Configuration ======================== - -satelite = S1A # Satellite: S1A, S1B, CSK, TSX, ALOS2, ALOS, ENVISAT - -# ========================= Pair Selection ======================== - -network_method = sbas # Network method: sbas, sequential, delaunay, stars -endDate = YYYYMMDD # Exclude dates after this date -startDate = YYYYMMDD # Exclude dates before this date -conNumb = 15 # Connect number for sequential network -max_tb = 365 # Maximum temporal baseline (days) -max_sb = 100 # Maximum spatial baseline (meters) -min_tb = 1 # Minimum temporal baseline (days) -min_sb = 0 # Minimum spatial baseline (meters) - -# ========================= Phase Gradient (Optional) ======================== - -dataformat = cpxfloat32 # Data format of interferogram -scale = 10000 # Original phase/scale -step_windows = 1 # Step length for spatial phase gradient -filter_wins = [7, 7] # Windows of median filter -directions = ['grad_east', 'grad_north', 'grad_northeast', 'grad_southeast'] - -# Mapping parameters for phase gradient -in_low_high = [0.001, 1] # Input range for mapping -out_low_high = [0.001, 1] # Output range for mapping -gamma = 0.8 # Gamma for curve shape -choose_linear = 'n' # Linear or nonlinear mapping ['y': linear; 'n': nonlinear] - -# ========================= Hyp3 Format Conversion (NEW) ======================== - -# Convert GAMMA outputs to Hyp3-compatible GeoTIFF format -hyp3format = 0 # Enable Hyp3 format conversion [0: skip; 1: process] -hyp3format_parallel = 4 # Number of parallel processors for Hyp3 conversion -hyp3_output_dir = - # Output directory for Hyp3 products [default: projectName/Hyp3Products/] - -# ========================= Phase Bias Correction (NEW) ======================== - -# Apply phase bias correction based on loop closures algorithm -# Reference: 10.1016/j.remote.2022.100013 - -phasebias = 0 # Enable phase bias correction [0: skip; 1: process] -phasebias_interval = 12 # Temporal baseline in days [6, 12, or 24] -phasebias_nlook = 10 # Number of looks for multilooking -phasebias_num_a = 2 # Number of calibration parameters to estimate -phasebias_estimate_an = 0 # Estimate an from data [0: use defaults; 1: estimate from loop closures] -phasebias_max_con = 5 # Maximum number of connections - -# Default an values (used if phasebias_estimate_an = 0) -a1_6_day = 0.50 -a2_6_day = 0.36 -a3_6_day = 0.299 -a4_6_day = 0.2476 - -a1_12_day = 0.494 -a2_12_day = 0.297 -a3_12_day = 0.24 -a4_12_day = 0.22 - -a1_24_day = 0.48 -a2_24_day = 0.28 -a3_24_day = 0.22 -a4_24_day = 0.20 - -phasebias_start = - # Start date YYYYMMDD [default: auto-detect] -phasebias_end = - # End date YYYYMMDD [default: auto-detect] - -# ========================= Atmospheric Correction ======================== - -# GACOS tropospheric correction -gacos_correction = 0 # Enable GACOS correction [0: skip; 1: process] -gacos_parallel = 4 # Number of parallel processors - -# Ionospheric correction -ionosphere_correction = 0 # Enable ionospheric correction [0: skip; 1: process] -ionosphere_parallel = 4 # Number of parallel processors - -# ========================= Processing Workflow Control ======================== - -download_data = 0 # Enable data download [0: skip; 1: process] -down_parallel = 4 # Number of parallel processors for download - -raw2slc_all = 1 # Convert raw to SLC [0: skip; 1: process] -raw2slc_all_parallel = 4 # Number of parallel processors - -extract_burst_all = 1 # Extract bursts [0: skip; 1: process] -extract_all_parallel = 4 # Number of parallel processors - -coreg_all = 1 # Coregister SLCs [0: skip; 1: process] -coreg_all_parallel = 4 # Number of parallel processors - -select_pairs = 1 # Select interferometric pairs [0: skip; 1: process] - -diff_all = 1 # Generate differential interferograms [0: skip; 1: process] -diff_all_parallel = 4 # Number of parallel processors - -unwrap_all = 1 # Unwrap interferograms [0: skip; 1: process] -unwrap_all_parallel = 4 # Number of parallel processors - -geocode_all = 1 # Geocode products [0: skip; 1: process] -geocode_all_parallel = 4 # Number of parallel processors - -hyp3format_all = 0 # Convert to Hyp3 format [0: skip; 1: process] -hyp3format_all_parallel = 4 # Number of parallel processors - -phasebias_all = 0 # Apply phase bias correction [0: skip; 1: process] - -load_data = 0 # Load data for MintPy processing [0: skip; 1: process] - -# ========================= Good Luck ======================================= diff --git a/.codex_tmp/pyint_variants/no_rescue/template/shanghaiT171F128S1A.template b/.codex_tmp/pyint_variants/no_rescue/template/shanghaiT171F128S1A.template deleted file mode 100644 index d350dd9..0000000 --- a/.codex_tmp/pyint_variants/no_rescue/template/shanghaiT171F128S1A.template +++ /dev/null @@ -1,204 +0,0 @@ -# *********************************** PyINT Template *******************************************# -# Template for PyINT v2.1 -# Please fill in the parameters according to your project requirements - -# ========================= Basic parameters ======================== - -# DEM file path (if not provided, SRTM will be downloaded automatically from OpenTopography) -#DEM = /path/to/your/dem.dem - -# Master date for interferometric processing -masterDate = 20250711 # Master date in YYYYMMDD format - -# Swath and burst parameters (for Sentinel-1 TOPS mode only) -start_swath = 1 # Starting swath number -end_swath = 3 # Ending swath number -start_burst = 1 # Starting burst position (relative to master image) -end_burst = 9 # Ending burst position (relative to master image) - -# Multilooking parameters -range_looks = 10 # Multi-look numbers in range direction -azimuth_looks = 2 # Multi-look numbers in azimuth direction - -# ========================= Download Data ======================== - -# Data source and search parameters -Data_Hub = ASF # Data download source: SSARA, Scihub, ASF -sensor = Sentinel-1A # Envisat, ERS-1/2, Sentinel-1A/B, etc. -track = 171 # Track number of the SAR image -frame = 96 # Frame number [defined according to ASF] -start_time = 2024-11-01 # Start date for data search -end_time = 2026-02-04 # End date for data search - -# ASF download specific parameters -asf_platform = Sentinel-1A # Platform: Sentinel-1A, Sentinel-1B -asf_beam_mode = IW # Beam mode: IW (Interferometric Wide) -asf_polarization = VV # Polarization: VV, VH, HH, HV -asf_flight_direction = ascending # Flight direction: ascending, descending -asf_bbox = 120.1,30.7,120.2,30.9 # Bounding box: West,South,East,North - -# ========================= DEM Generation ======================== - -# DEM download parameters (for OpenTopography) -opentopo_api_key = 09ad77d34545607fdf5cb182b64ac64e # OpenTopography API key -opentopo_dem_type = SRTMGL1 # DEM type: SRTMGL1(30m), SRTMGL3(90m), NASADEM(30m), COP30, COP90 -opentopo_auto_tile = 1 # Auto-tile large regions (recommended: 1) - -# DEM processing parameters -dem_lat_ovr = 1 # Latitude oversampling (as to SRTM-1: 0.5 for 60m, 2 for 15m) -dem_lon_ovr = 1 # Longitude oversampling - -# ========================= Interferometry Parameters ======================== - -# Spectral filtering -Igram_Spsflg = 1 # Implement range spectral filtering -rbw_min = 1 # Minimum range bandwidth fraction (0.1 --> 1.0) - -# Coregistration parameters -rwin4cor = 256 # Range window length for coregistration -azwin4cor = 256 # Azimuth window length for coregistration -rsample4cor = 32 # Range samples for fitting coregistration parameters -azsample4cor = 32 # Azimuth samples for fitting coregistration parameters -thresh4cor = 0.3 # Coherence threshold (GAMMA 2016+, SNR for older versions) - -# Coherence estimation -Igram_Cor_rwin = 5 # Range window length for cc_wave coherence estimation -Igram_Cor_awin = 5 # Azimuth window length for cc_wave coherence estimation -Igram_Cor_Win = 5 # Window for cc estimation in adf -adf_alpha = 0.4 # Alpha for Gold-Stein filtering - -# ========================= Simulation Phase ======================== - -Igram_Flag_TDM = N # Y for Tandem-X -Simphase_rpos = - -Simphase_azpos = - -Simphase_rwin = 256 -Simphase_azwin = 256 -Simphase_thresh = - - -# ========================= Unwrapping ======================== - -mcf_triangular = 1 # Triangular type of mcf [0: regular; 1: delaunay] -unwrap_patr = 1 # Unwrap patches in range direction -unwrap_pataz = 1 # Unwrap patches in azimuth direction -unwrapThreshold = 0.05 # Coherence threshold for unwrapping -auto_unw = 1 # Auto or manual reference point [1: auto; 0: manual] -make_mask = 0 # Make mask based on coherence [0: don't mask; 1: make mask] -init_flag = 1 # Flag to set phase at reference point [0: use initial; 1: set to 0.0] -r_refer = - # Phase reference point range offset -a_refer = - # Phase reference point azimuth offset - -# ========================= Geocoding ======================== - -geo_interp = 0 # Interpolation method [0: nearest; 1: bicubic spline] - -# ========================= Satellite Configuration ======================== - -satelite = S1A # Satellite: S1A, S1B, CSK, TSX, ALOS2, ALOS, ENVISAT - -# ========================= Pair Selection ======================== - -network_method = sbas # Network method: sbas, sequential, delaunay, stars -endDate = YYYYMMDD # Exclude dates after this date -startDate = YYYYMMDD # Exclude dates before this date -conNumb = 15 # Connect number for sequential network -max_tb = 365 # Maximum temporal baseline (days) -max_sb = 100 # Maximum spatial baseline (meters) -min_tb = 1 # Minimum temporal baseline (days) -min_sb = 0 # Minimum spatial baseline (meters) - -# ========================= Phase Gradient (Optional) ======================== - -dataformat = cpxfloat32 # Data format of interferogram -scale = 10000 # Original phase/scale -step_windows = 1 # Step length for spatial phase gradient -filter_wins = [7, 7] # Windows of median filter -directions = ['grad_east', 'grad_north', 'grad_northeast', 'grad_southeast'] - -# Mapping parameters for phase gradient -in_low_high = [0.001, 1] # Input range for mapping -out_low_high = [0.001, 1] # Output range for mapping -gamma = 0.8 # Gamma for curve shape -choose_linear = 'n' # Linear or nonlinear mapping ['y': linear; 'n': nonlinear] - -# ========================= Hyp3 Format Conversion (NEW) ======================== - -# Convert GAMMA outputs to Hyp3-compatible GeoTIFF format -hyp3format = 0 # Enable Hyp3 format conversion [0: skip; 1: process] -hyp3format_parallel = 4 # Number of parallel processors for Hyp3 conversion -hyp3_output_dir = - # Output directory for Hyp3 products [default: projectName/Hyp3Products/] - -# ========================= Phase Bias Correction (NEW) ======================== - -# Apply phase bias correction based on loop closures algorithm -# Reference: 10.1016/j.remote.2022.100013 - -phasebias = 0 # Enable phase bias correction [0: skip; 1: process] -phasebias_interval = 12 # Temporal baseline in days [6, 12, or 24] -phasebias_nlook = 10 # Number of looks for multilooking -phasebias_num_a = 2 # Number of calibration parameters to estimate -phasebias_estimate_an = 0 # Estimate an from data [0: use defaults; 1: estimate from loop closures] -phasebias_max_con = 5 # Maximum number of connections - -# Default an values (used if phasebias_estimate_an = 0) -a1_6_day = 0.50 -a2_6_day = 0.36 -a3_6_day = 0.299 -a4_6_day = 0.2476 - -a1_12_day = 0.494 -a2_12_day = 0.297 -a3_12_day = 0.24 -a4_12_day = 0.22 - -a1_24_day = 0.48 -a2_24_day = 0.28 -a3_24_day = 0.22 -a4_24_day = 0.20 - -phasebias_start = - # Start date YYYYMMDD [default: auto-detect] -phasebias_end = - # End date YYYYMMDD [default: auto-detect] - -# ========================= Atmospheric Correction ======================== - -# GACOS tropospheric correction -gacos_correction = 0 # Enable GACOS correction [0: skip; 1: process] -gacos_parallel = 4 # Number of parallel processors - -# Ionospheric correction -ionosphere_correction = 0 # Enable ionospheric correction [0: skip; 1: process] -ionosphere_parallel = 4 # Number of parallel processors - -# ========================= Processing Workflow Control ======================== - -download_data = 1 # Enable data download [0: skip; 1: process] -down_parallel = 4 # Number of parallel processors for download - -raw2slc_all = 0 # Convert raw to SLC [0: skip; 1: process] -raw2slc_all_parallel = 4 # Number of parallel processors - -extract_burst_all = 0 # Extract bursts [0: skip; 1: process] -extract_all_parallel = 4 # Number of parallel processors - -coreg_all = 0 # Coregister SLCs [0: skip; 1: process] -coreg_all_parallel = 4 # Number of parallel processors - -select_pairs = 0 # Select interferometric pairs [0: skip; 1: process] - -diff_all = 0 # Generate differential interferograms [0: skip; 1: process] -diff_all_parallel = 4 # Number of parallel processors - -unwrap_all = 0 # Unwrap interferograms [0: skip; 1: process] -unwrap_all_parallel = 4 # Number of parallel processors - -geocode_all = 0 # Geocode products [0: skip; 1: process] -geocode_all_parallel = 4 # Number of parallel processors - -hyp3format_all = 0 # Convert to Hyp3 format [0: skip; 1: process] -hyp3format_all_parallel = 4 # Number of parallel processors - -phasebias_all = 0 # Apply phase bias correction [0: skip; 1: process] - -load_data = 0 # Load data for MintPy processing [0: skip; 1: process] - -# ========================= Good Luck ======================================= diff --git a/.codex_tmp/run_lt1_coreg_orbit_experiment.sh b/.codex_tmp/run_lt1_coreg_orbit_experiment.sh deleted file mode 100644 index 67809aa..0000000 --- a/.codex_tmp/run_lt1_coreg_orbit_experiment.sh +++ /dev/null @@ -1,291 +0,0 @@ -#!/usr/bin/env bash -set -u -set -o pipefail - -BASE_ROOT='/mnt/d/PyINT_POOL_TEST/LT1A_MONO_SYC_STRIP1_E129.6_N45.0_3scene' -CASES_ROOT='/mnt/d/PyINT_POOL_TEST/LT1A_MONO_SYC_STRIP1_E129.6_N45.0_3scene_cases' -PROJECT='pyint_stage' -MASTER_DATE='20230726' -SLAVE_DATES=(20230624 20230920) -SATELLITE='LT1A' -ORBIT_DIR='/mnt/d/orbit_pools/envi/LT1A' - -REPO='/mnt/d/Code/Insar_management_system_v2' -PYINT_HOME="$REPO/third_party/PyINT" -PYINT_SCRIPT_DIR="$PYINT_HOME/pyint" -PYTHON_BIN='/home/administrator/miniconda3/envs/isce2/bin/python' -GAMMA_ENV="$REPO/backend/app/pyint_pipeline/pyint_gamma_env.sh" -ORBIT_HELPER="$REPO/backend/app/pyint_pipeline/apply_lt1_precise_orbit.py" -GAMMA_SCRIPT_DIR='/usr/local/GAMMA_SOFTWARE-20240627/ISP/scripts' - -RUN_STAMP="${1:-$(date -u +%Y%m%dT%H%M%SZ)}" -RUN_ROOT="$CASES_ROOT/run_$RUN_STAMP" -STATUS_FILE="$RUN_ROOT/stage_status.tsv" - -fail() { - echo "$1" >&2 - exit 1 -} - -ensure_file() { - local path="$1" - [ -f "$path" ] || fail "Required file not found: $path" -} - -ensure_dir() { - local path="$1" - [ -d "$path" ] || fail "Required directory not found: $path" -} - -write_status() { - local case_name="$1" - local stage="$2" - local rc="$3" - local stdout_log="$4" - local stderr_log="$5" - printf '%s\t%s\t%s\t%s\t%s\n' "$case_name" "$stage" "$rc" "$stdout_log" "$stderr_log" >>"$STATUS_FILE" -} - -run_stage() { - local case_name="$1" - local stage="$2" - shift 2 - local stdout_log="$RUN_ROOT/$case_name/logs/${stage}.stdout.log" - local stderr_log="$RUN_ROOT/$case_name/logs/${stage}.stderr.log" - local rc=0 - - echo "[stage] $case_name :: $stage" - "$@" >"$stdout_log" 2>"$stderr_log" || rc=$? - write_status "$case_name" "$stage" "$rc" "$stdout_log" "$stderr_log" - - if [ "$rc" -eq 0 ]; then - echo "[ok] $case_name :: $stage" - else - echo "[fail] $case_name :: $stage (rc=$rc)" >&2 - fi - return "$rc" -} - -link_or_copy_static() { - local src="$1" - local dst="$2" - if [ -e "$src" ]; then - ln -s "$src" "$dst" - fi -} - -build_scene_dir() { - local case_root="$1" - local scene_date="$2" - local base_dir="$BASE_ROOT/$PROJECT/SLC/$scene_date" - local case_dir="$case_root/$PROJECT/SLC/$scene_date" - - ensure_dir "$base_dir" - mkdir -p "$case_dir" - - ensure_file "$base_dir/$scene_date.slc" - ensure_file "$base_dir/$scene_date.slc.par" - - ln -s "$base_dir/$scene_date.slc" "$case_dir/$scene_date.slc" - cp "$base_dir/$scene_date.slc.par" "$case_dir/$scene_date.slc.par" - - link_or_copy_static "$base_dir/${scene_date}_2rlks.amp" "$case_dir/${scene_date}_2rlks.amp" - link_or_copy_static "$base_dir/${scene_date}_2rlks.amp.par" "$case_dir/${scene_date}_2rlks.amp.par" - link_or_copy_static "$base_dir/${scene_date}_SLC_Tab" "$case_dir/${scene_date}_SLC_Tab" - link_or_copy_static "$base_dir/down2slc.dat" "$case_dir/down2slc.dat" - link_or_copy_static "$base_dir/t_${scene_date}" "$case_dir/t_${scene_date}" -} - -build_case() { - local case_name="$1" - local case_root="$RUN_ROOT/$case_name" - - mkdir -p "$case_root/templates" "$case_root/logs" "$case_root/manifests" - mkdir -p "$case_root/$PROJECT/SLC" "$case_root/$PROJECT/DEM" "$case_root/$PROJECT/RSLC" "$case_root/$PROJECT/ifgrams" - mkdir -p "$case_root/dem_store" - - cp "$BASE_ROOT/templates/$PROJECT.template" "$case_root/templates/$PROJECT.template" - cp -a "$BASE_ROOT/dem_store/$PROJECT" "$case_root/dem_store/" - - build_scene_dir "$case_root" "$MASTER_DATE" - for slave_date in "${SLAVE_DATES[@]}"; do - build_scene_dir "$case_root" "$slave_date" - done - - printf '%s\n' "$case_root" -} - -write_manifest() { - local role="$1" - local date_text="$2" - local output_path="$3" - local orbit_path="$ORBIT_DIR/${SATELLITE}_GpsData_GAS_C_${date_text}.txt" - - ensure_file "$orbit_path" - cat >"$output_path" <<EOF -{ - "orbits": { - "$role": { - "satellite": "$SATELLITE", - "date": "$date_text", - "expected_name": "${SATELLITE}_GpsData_GAS_C_${date_text}.txt", - "path": "$orbit_path" - } - } -} -EOF -} - -apply_bridge_once() { - local case_name="$1" - local case_root="$2" - local stage_name="$3" - local date_text="$4" - local role="$5" - local validate_flag="$6" - local strict_flag="$7" - shift 7 - local manifest_path="$case_root/manifests/${role}_${date_text}.json" - local summary_path="$case_root/logs/${stage_name}.summary.json" - local -a cmd - - write_manifest "$role" "$date_text" "$manifest_path" - - cmd=( - "$PYTHON_BIN" "$ORBIT_HELPER" - "--date" "$date_text" - "--role" "$role" - "--manifest-json" "$manifest_path" - "--summary-json" "$summary_path" - "--backup" - ) - - if [ "$validate_flag" = "true" ]; then - cmd+=("--validate-with-orb-filt") - else - cmd+=("--no-validate-with-orb-filt") - fi - - if [ "$strict_flag" = "true" ]; then - cmd+=("--strict") - else - cmd+=("--no-strict") - fi - - while [ "$#" -gt 0 ]; do - cmd+=("--slc-par" "$1") - shift - done - - run_stage "$case_name" "$stage_name" "${cmd[@]}" - return $? -} - -run_generate_and_coreg() { - local case_name="$1" - local case_root="$2" - local rc=0 - - export SCRATCHDIR="$case_root" - export TEMPLATEDIR="$case_root/templates" - export DEMDIR="$case_root/dem_store" - - run_stage "$case_name" "generate_rdc_dem" "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/generate_rdc_dem.py" "$PROJECT" || return $? - - for slave_date in "${SLAVE_DATES[@]}"; do - run_stage "$case_name" "coreg_${slave_date}" "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/coreg_gamma.py" "$PROJECT" "$slave_date" || rc=1 - done - - return "$rc" -} - -run_case_a() { - local case_name='case_A_baseline' - local case_root - case_root="$(build_case "$case_name")" - run_generate_and_coreg "$case_name" "$case_root" || true -} - -run_case_c() { - local case_name='case_C_precise_orbit_rewrite' - local case_root - case_root="$(build_case "$case_name")" - - apply_bridge_once \ - "$case_name" "$case_root" "orbit_bridge_master" "$MASTER_DATE" "master" "false" "true" \ - "$case_root/$PROJECT/SLC/$MASTER_DATE/$MASTER_DATE.slc.par" || return 1 - - for slave_date in "${SLAVE_DATES[@]}"; do - apply_bridge_once \ - "$case_name" "$case_root" "orbit_bridge_${slave_date}" "$slave_date" "slave" "false" "true" \ - "$case_root/$PROJECT/SLC/$slave_date/$slave_date.slc.par" || return 1 - done - - run_generate_and_coreg "$case_name" "$case_root" || true -} - -run_case_d() { - local case_name='case_D_precise_orbit_validate' - local case_root - case_root="$(build_case "$case_name")" - - apply_bridge_once \ - "$case_name" "$case_root" "orbit_bridge_master" "$MASTER_DATE" "master" "false" "true" \ - "$case_root/$PROJECT/SLC/$MASTER_DATE/$MASTER_DATE.slc.par" || return 1 - - for slave_date in "${SLAVE_DATES[@]}"; do - apply_bridge_once \ - "$case_name" "$case_root" "orbit_bridge_${slave_date}" "$slave_date" "slave" "false" "true" \ - "$case_root/$PROJECT/SLC/$slave_date/$slave_date.slc.par" || return 1 - done - - apply_bridge_once \ - "$case_name" "$case_root" "orbit_validate_master" "$MASTER_DATE" "master" "true" "false" \ - "$case_root/$PROJECT/SLC/$MASTER_DATE/$MASTER_DATE.slc.par" || true - - for slave_date in "${SLAVE_DATES[@]}"; do - apply_bridge_once \ - "$case_name" "$case_root" "orbit_validate_${slave_date}" "$slave_date" "slave" "true" "false" \ - "$case_root/$PROJECT/SLC/$slave_date/$slave_date.slc.par" || true - done - - run_generate_and_coreg "$case_name" "$case_root" || true -} - -main() { - ensure_dir "$BASE_ROOT" - ensure_dir "$BASE_ROOT/$PROJECT/SLC" - ensure_dir "$BASE_ROOT/dem_store/$PROJECT" - ensure_file "$BASE_ROOT/templates/$PROJECT.template" - ensure_file "$GAMMA_ENV" - ensure_file "$ORBIT_HELPER" - ensure_file "$PYTHON_BIN" - - mkdir -p "$RUN_ROOT" - printf 'case\tstage\trc\tstdout_log\tstderr_log\n' >"$STATUS_FILE" - - . "$GAMMA_ENV" >/dev/null 2>&1 - export PATH="/home/administrator/miniconda3/envs/isce2/bin:$GAMMA_SCRIPT_DIR:$PYINT_SCRIPT_DIR:$PATH" - export PYTHONPATH="$PYINT_HOME${PYTHONPATH:+:$PYTHONPATH}" - - cat >"$RUN_ROOT/run_info.txt" <<EOF -run_root=$RUN_ROOT -base_root=$BASE_ROOT -project=$PROJECT -master=$MASTER_DATE -slaves=${SLAVE_DATES[*]} -satellite=$SATELLITE -orbit_dir=$ORBIT_DIR -python=$PYTHON_BIN -pyint_home=$PYINT_HOME -EOF - - run_case_a - run_case_c - run_case_d - - echo "[done] run_root=$RUN_ROOT" - echo "[done] status_file=$STATUS_FILE" -} - -main "$@" diff --git a/.codex_tmp/run_lt1_dem_source_experiment.sh b/.codex_tmp/run_lt1_dem_source_experiment.sh deleted file mode 100644 index 3669082..0000000 --- a/.codex_tmp/run_lt1_dem_source_experiment.sh +++ /dev/null @@ -1,221 +0,0 @@ -#!/usr/bin/env bash -set -u -set -o pipefail - -BASE_ROOT='/mnt/d/PyINT_POOL_TEST/LT1A_MONO_SYC_STRIP1_E129.6_N45.0_3scene' -CASES_ROOT='/mnt/d/PyINT_POOL_TEST/LT1A_MONO_SYC_STRIP1_E129.6_N45.0_3scene_dem_cases' -PROJECT='pyint_stage' -MASTER_DATE='20230726' -SLAVE_DATES=(20230624 20230920) - -REPO='/mnt/d/Code/Insar_management_system_v2' -PYINT_HOME="$REPO/third_party/PyINT" -PYINT_SCRIPT_DIR="$PYINT_HOME/pyint" -PYTHON_BIN='/home/administrator/miniconda3/envs/isce2/bin/python' -GAMMA_ENV="$REPO/backend/app/pyint_pipeline/pyint_gamma_env.sh" -GAMMA_SCRIPT_DIR='/usr/local/GAMMA_SOFTWARE-20240627/ISP/scripts' - -RUN_STAMP="${1:-$(date -u +%Y%m%dT%H%M%SZ)}" -RUN_ROOT="$CASES_ROOT/run_$RUN_STAMP" -STATUS_FILE="$RUN_ROOT/stage_status.tsv" -CASE_FILTER="${CASE_FILTER:-}" - -fail() { - echo "$1" >&2 - exit 1 -} - -ensure_file() { - local path="$1" - [ -f "$path" ] || fail "Required file not found: $path" -} - -ensure_dir() { - local path="$1" - [ -d "$path" ] || fail "Required directory not found: $path" -} - -write_status() { - local case_name="$1" - local stage="$2" - local rc="$3" - local stdout_log="$4" - local stderr_log="$5" - printf '%s\t%s\t%s\t%s\t%s\n' "$case_name" "$stage" "$rc" "$stdout_log" "$stderr_log" >>"$STATUS_FILE" -} - -run_stage() { - local case_name="$1" - local stage="$2" - shift 2 - local stdout_log="$RUN_ROOT/$case_name/logs/${stage}.stdout.log" - local stderr_log="$RUN_ROOT/$case_name/logs/${stage}.stderr.log" - local rc=0 - - echo "[stage] $case_name :: $stage" - "$@" >"$stdout_log" 2>"$stderr_log" || rc=$? - write_status "$case_name" "$stage" "$rc" "$stdout_log" "$stderr_log" - - if [ "$rc" -eq 0 ]; then - echo "[ok] $case_name :: $stage" - else - echo "[fail] $case_name :: $stage (rc=$rc)" >&2 - fi - return "$rc" -} - -link_or_copy_static() { - local src="$1" - local dst="$2" - if [ -e "$src" ]; then - ln -s "$src" "$dst" - fi -} - -build_scene_dir() { - local case_root="$1" - local scene_date="$2" - local base_dir="$BASE_ROOT/$PROJECT/SLC/$scene_date" - local case_dir="$case_root/$PROJECT/SLC/$scene_date" - - ensure_dir "$base_dir" - mkdir -p "$case_dir" - - ensure_file "$base_dir/$scene_date.slc" - ensure_file "$base_dir/$scene_date.slc.par" - - ln -s "$base_dir/$scene_date.slc" "$case_dir/$scene_date.slc" - cp "$base_dir/$scene_date.slc.par" "$case_dir/$scene_date.slc.par" - - link_or_copy_static "$base_dir/${scene_date}_2rlks.amp" "$case_dir/${scene_date}_2rlks.amp" - link_or_copy_static "$base_dir/${scene_date}_2rlks.amp.par" "$case_dir/${scene_date}_2rlks.amp.par" - link_or_copy_static "$base_dir/${scene_date}_SLC_Tab" "$case_dir/${scene_date}_SLC_Tab" - link_or_copy_static "$base_dir/down2slc.dat" "$case_dir/down2slc.dat" - link_or_copy_static "$base_dir/t_${scene_date}" "$case_dir/t_${scene_date}" -} - -set_template_key() { - local template_path="$1" - local key="$2" - local value="$3" - - "$PYTHON_BIN" -c ' -from pathlib import Path -import sys - -path = Path(sys.argv[1]) -key = sys.argv[2] -value = sys.argv[3] -lines = path.read_text(encoding="utf-8").splitlines() -prefix = key + "=" -updated = False -for idx, line in enumerate(lines): - if line.startswith(prefix): - lines[idx] = prefix + value - updated = True - break -if not updated: - lines.append(prefix + value) -path.write_text("\n".join(lines) + "\n", encoding="utf-8") -' "$template_path" "$key" "$value" -} - -build_case() { - local case_name="$1" - local dem_source="$2" - local opentopo_dem_type="$3" - local case_root="$RUN_ROOT/$case_name" - local template_path - - mkdir -p "$case_root/templates" "$case_root/logs" - mkdir -p "$case_root/$PROJECT/SLC" "$case_root/$PROJECT/DEM" "$case_root/$PROJECT/RSLC" "$case_root/$PROJECT/ifgrams" - mkdir -p "$case_root/dem_store/$PROJECT" - - cp "$BASE_ROOT/templates/$PROJECT.template" "$case_root/templates/$PROJECT.template" - template_path="$case_root/templates/$PROJECT.template" - set_template_key "$template_path" "prepared_dem_source" "$dem_source" - set_template_key "$template_path" "fabdem_dir" "-" - set_template_key "$template_path" "opentopo_dem_type" "$opentopo_dem_type" - set_template_key "$template_path" "opentopo_api_key" "-" - - build_scene_dir "$case_root" "$MASTER_DATE" - for slave_date in "${SLAVE_DATES[@]}"; do - build_scene_dir "$case_root" "$slave_date" - done - - printf '%s\n' "$case_root" -} - -run_case() { - local case_name="$1" - local dem_source="$2" - local opentopo_dem_type="$3" - local case_root - local rc=0 - - case_root="$(build_case "$case_name" "$dem_source" "$opentopo_dem_type")" - - export SCRATCHDIR="$case_root" - export TEMPLATEDIR="$case_root/templates" - export DEMDIR="$case_root/dem_store" - - run_stage "$case_name" "makedem_pyint" "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/makedem_pyint.py" "$PROJECT" || return $? - run_stage "$case_name" "generate_rdc_dem" "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/generate_rdc_dem.py" "$PROJECT" || return $? - - for slave_date in "${SLAVE_DATES[@]}"; do - run_stage "$case_name" "coreg_${slave_date}" "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/coreg_gamma.py" "$PROJECT" "$slave_date" || rc=1 - done - - return "$rc" -} - -should_run_case() { - local case_name="$1" - if [ -z "$CASE_FILTER" ]; then - return 0 - fi - case ",$CASE_FILTER," in - *",$case_name,"*) return 0 ;; - *) return 1 ;; - esac -} - -main() { - ensure_dir "$BASE_ROOT" - ensure_dir "$BASE_ROOT/$PROJECT/SLC" - ensure_file "$BASE_ROOT/templates/$PROJECT.template" - ensure_file "$GAMMA_ENV" - ensure_file "$PYTHON_BIN" - - mkdir -p "$RUN_ROOT" - printf 'case\tstage\trc\tstdout_log\tstderr_log\n' >"$STATUS_FILE" - - . "$GAMMA_ENV" >/dev/null 2>&1 - export PATH="/home/administrator/miniconda3/envs/isce2/bin:$GAMMA_SCRIPT_DIR:$PYINT_SCRIPT_DIR:$PATH" - export PYTHONPATH="$PYINT_HOME${PYTHONPATH:+:$PYTHONPATH}" - - cat >"$RUN_ROOT/run_info.txt" <<EOF -run_root=$RUN_ROOT -base_root=$BASE_ROOT -project=$PROJECT -master=$MASTER_DATE -slaves=${SLAVE_DATES[*]} -python=$PYTHON_BIN -pyint_home=$PYINT_HOME -EOF - - if should_run_case 'case_A_copdem_baseline'; then - run_case 'case_A_copdem_baseline' '/mnt/d/DEM/COPDEM_GLO30_China_4326_DEM' '-' - fi - if should_run_case 'case_B_gmted2010_jp2'; then - run_case 'case_B_gmted2010_jp2' '/mnt/d/DEM/GMTED2010.jp2' '-' - fi - if should_run_case 'case_C_opentopo_srtmgl1'; then - run_case 'case_C_opentopo_srtmgl1' '-' 'SRTMGL1' - fi - - echo "[done] run_root=$RUN_ROOT" - echo "[done] status_file=$STATUS_FILE" -} - -main "$@" diff --git a/.codex_tmp/run_lt1_pool_multiscene_generic.sh b/.codex_tmp/run_lt1_pool_multiscene_generic.sh deleted file mode 100644 index d1c601b..0000000 --- a/.codex_tmp/run_lt1_pool_multiscene_generic.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="${ROOT:-${1:-}}" -PROJECT="${PROJECT:-pyint_stage}" -MASTER_DATE="${MASTER_DATE:-}" -START_DATE="${START_DATE:-}" -END_DATE="${END_DATE:-}" -PREPARED_DEM="${PREPARED_DEM:-/mnt/d/DEM/COPDEM_GLO30_China_4326_DEM}" - -SCRATCHDIR="$ROOT" -TEMPLATEDIR="$ROOT/templates" -DEMDIR="$ROOT/dem_store" -PROJECT_DIR="$SCRATCHDIR/$PROJECT" -LOG_DIR="$ROOT/logs" -PYINT_HOME='/mnt/d/Code/Insar_management_system_v2/third_party/PyINT' -PYINT_SCRIPT_DIR="$PYINT_HOME/pyint" -PYTHON_BIN='/home/administrator/miniconda3/envs/isce2/bin/python' -GAMMA_ENV='/mnt/d/Code/Insar_management_system_v2/backend/app/pyint_pipeline/pyint_gamma_env.sh' -TEMPLATE_PATH="$TEMPLATEDIR/$PROJECT.template" - -die() { - echo "$1" >&2 - exit 1 -} - -run_stage() { - local stage="$1" - shift - local stdout_log="$LOG_DIR/${stage}.stdout.log" - local stderr_log="$LOG_DIR/${stage}.stderr.log" - echo "[stage] $stage" - if "$@" >"$stdout_log" 2>"$stderr_log"; then - echo "[ok] $stage" - else - local rc=$? - echo "[fail] $stage (rc=$rc)" >&2 - echo "stdout: $stdout_log" >&2 - echo "stderr: $stderr_log" >&2 - exit "$rc" - fi -} - -run_single_coreg() { - local slave_date="$1" - mkdir -p "$PROJECT_DIR/SLC" "$PROJECT_DIR/RSLC" "$PROJECT_DIR/DEM" "$PROJECT_DIR/ifgrams" - run_stage "coreg_${slave_date}" "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/coreg_gamma.py" "$PROJECT" "$slave_date" -} - -[ -n "$ROOT" ] || die "ROOT is required" -[ -n "$MASTER_DATE" ] || die "MASTER_DATE is required" -[ -n "$START_DATE" ] || die "START_DATE is required" -[ -n "$END_DATE" ] || die "END_DATE is required" -[ -d "$ROOT" ] || die "Experiment root not found: $ROOT" -[ -d "$PROJECT_DIR/DOWNLOAD" ] || die "DOWNLOAD directory not found: $PROJECT_DIR/DOWNLOAD" -[ -f "$PREPARED_DEM" ] || die "Prepared DEM not found: $PREPARED_DEM" -[ -f "$GAMMA_ENV" ] || die "Gamma env script not found: $GAMMA_ENV" -[ -x "$PYTHON_BIN" ] || die "Python not found: $PYTHON_BIN" - -. "$GAMMA_ENV" >/dev/null 2>&1 -export SCRATCHDIR -export TEMPLATEDIR -export DEMDIR -export PATH="/home/administrator/miniconda3/envs/isce2/bin:$PYINT_SCRIPT_DIR:$PATH" -export PYTHONPATH="$PYINT_HOME${PYTHONPATH:+:$PYTHONPATH}" -export PYINT_LT1_PRECISE_ORBIT_ENABLED='false' -export PYINT_LT1_PRECISE_ORBIT_MODE='bridge' -export PYINT_LT1_PRECISE_ORBIT_STRICT='false' -export PYINT_LT1_PRECISE_ORBIT_VALIDATE_WITH_ORB_FILT='false' -export PYINT_LT1_PRECISE_ORBIT_BACKUP='false' -unset PYINT_LT1_PRECISE_ORBIT_HELPER -unset PYINT_LT1_PRECISE_ORBIT_MANIFEST - -mkdir -p "$TEMPLATEDIR" "$DEMDIR" "$LOG_DIR" -mkdir -p "$PROJECT_DIR/SLC" "$PROJECT_DIR/RSLC" "$PROJECT_DIR/DEM" "$PROJECT_DIR/ifgrams" - -cat >"$TEMPLATE_PATH" <<EOF -# Auto-generated LT-1 pool multiscene test -satelite=LT -masterDate=$MASTER_DATE -range_looks=2 -azimuth_looks=2 -download_data=0 -raw2slc_all=1 -raw2slc_all_parallel=1 -coreg_all=1 -coreg_all_parallel=1 -select_pairs=1 -network_method=sbas -startDate=$START_DATE -endDate=$END_DATE -max_tb=50000 -max_sb=50000 -min_tb=1 -diff_all=0 -unwrap_all=0 -geocode_all=0 -atmcor_all=0 -load_data=0 -prepared_dem_source=$PREPARED_DEM -fabdem_dir=- -opentopo_dem_type=- -opentopo_api_key=- -EOF - -if [ "${2:-}" = "--single-coreg" ] || [ "${1:-}" = "--single-coreg" ]; then - local_date="${3:-${2:-}}" - [ -n "$local_date" ] || die "Usage: ROOT=... MASTER_DATE=... START_DATE=... END_DATE=... $0 --single-coreg YYYYMMDD" - run_single_coreg "$local_date" - echo "[done] single_coreg=$local_date" - exit 0 -fi - -run_stage down2slc_all "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/down2slc_LT1_all.py" "$PROJECT" --parallel 1 -run_stage makedem_pyint "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/makedem_pyint.py" "$PROJECT" -run_stage generate_rdc_dem "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/generate_rdc_dem.py" "$PROJECT" -run_stage coreg_gamma_all "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/coreg_gamma_all.py" "$PROJECT" --parallel 1 -run_stage select_pairs "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/select_pairs.py" "$PROJECT" - -echo "[done] project=$PROJECT" -echo "[done] root=$ROOT" diff --git a/.codex_tmp/run_lt1_pool_multiscene_test.sh b/.codex_tmp/run_lt1_pool_multiscene_test.sh deleted file mode 100644 index bdb9986..0000000 --- a/.codex_tmp/run_lt1_pool_multiscene_test.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT='/mnt/d/PyINT_POOL_TEST/LT1A_MONO_SYC_STRIP1_E129.6_N45.0_3scene' -PROJECT='pyint_stage' -SCRATCHDIR="$ROOT" -TEMPLATEDIR="$ROOT/templates" -DEMDIR="$ROOT/dem_store" -PROJECT_DIR="$SCRATCHDIR/$PROJECT" -LOG_DIR="$ROOT/logs" -PYINT_HOME='/mnt/d/Code/Insar_management_system_v2/third_party/PyINT' -PYINT_SCRIPT_DIR="$PYINT_HOME/pyint" -PYTHON_BIN='/home/administrator/miniconda3/envs/isce2/bin/python' -GAMMA_ENV='/mnt/d/Code/Insar_management_system_v2/backend/app/pyint_pipeline/pyint_gamma_env.sh' -TEMPLATE_PATH="$TEMPLATEDIR/$PROJECT.template" -PREPARED_DEM='/mnt/d/DEM/COPDEM_GLO30_China_4326_DEM' - -die() { - echo "$1" >&2 - exit 1 -} - -run_stage() { - local stage="$1" - shift - local stdout_log="$LOG_DIR/${stage}.stdout.log" - local stderr_log="$LOG_DIR/${stage}.stderr.log" - echo "[stage] $stage" - if "$@" >"$stdout_log" 2>"$stderr_log"; then - echo "[ok] $stage" - else - local rc=$? - echo "[fail] $stage (rc=$rc)" >&2 - echo "stdout: $stdout_log" >&2 - echo "stderr: $stderr_log" >&2 - exit "$rc" - fi -} - -run_single_coreg() { - local slave_date="$1" - mkdir -p "$PROJECT_DIR/SLC" "$PROJECT_DIR/RSLC" "$PROJECT_DIR/DEM" "$PROJECT_DIR/ifgrams" - run_stage "coreg_${slave_date}" "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/coreg_gamma.py" "$PROJECT" "$slave_date" -} - -[ -d "$ROOT" ] || die "Experiment root not found: $ROOT" -[ -d "$PROJECT_DIR/DOWNLOAD" ] || die "DOWNLOAD directory not found: $PROJECT_DIR/DOWNLOAD" -[ -f "$PREPARED_DEM" ] || die "Prepared DEM not found: $PREPARED_DEM" -[ -f "$GAMMA_ENV" ] || die "Gamma env script not found: $GAMMA_ENV" -[ -x "$PYTHON_BIN" ] || die "Python not found: $PYTHON_BIN" - -. "$GAMMA_ENV" >/dev/null 2>&1 -export SCRATCHDIR -export TEMPLATEDIR -export DEMDIR -export PATH="/home/administrator/miniconda3/envs/isce2/bin:$PYINT_SCRIPT_DIR:$PATH" -export PYTHONPATH="$PYINT_HOME${PYTHONPATH:+:$PYTHONPATH}" -export PYINT_LT1_PRECISE_ORBIT_ENABLED='false' -export PYINT_LT1_PRECISE_ORBIT_MODE='bridge' -export PYINT_LT1_PRECISE_ORBIT_STRICT='false' -export PYINT_LT1_PRECISE_ORBIT_VALIDATE_WITH_ORB_FILT='false' -export PYINT_LT1_PRECISE_ORBIT_BACKUP='false' -unset PYINT_LT1_PRECISE_ORBIT_HELPER -unset PYINT_LT1_PRECISE_ORBIT_MANIFEST - -mkdir -p "$TEMPLATEDIR" "$DEMDIR" "$LOG_DIR" -mkdir -p "$PROJECT_DIR/SLC" "$PROJECT_DIR/RSLC" "$PROJECT_DIR/DEM" "$PROJECT_DIR/ifgrams" - -cat >"$TEMPLATE_PATH" <<'EOF' -# Auto-generated LT-1 pool multiscene test -satelite=LT -masterDate=20230726 -range_looks=2 -azimuth_looks=2 -download_data=0 -raw2slc_all=1 -raw2slc_all_parallel=1 -coreg_all=1 -coreg_all_parallel=1 -select_pairs=1 -network_method=sbas -startDate=20230601 -endDate=20231001 -max_tb=50000 -max_sb=50000 -min_tb=1 -diff_all=0 -unwrap_all=0 -geocode_all=0 -atmcor_all=0 -load_data=0 -prepared_dem_source=/mnt/d/DEM/COPDEM_GLO30_China_4326_DEM -fabdem_dir=- -opentopo_dem_type=- -opentopo_api_key=- -EOF - -if [ "${1:-}" = "--single-coreg" ]; then - [ -n "${2:-}" ] || die "Usage: $0 --single-coreg YYYYMMDD" - run_single_coreg "$2" - echo "[done] single_coreg=$2" - exit 0 -fi - -run_stage down2slc_all "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/down2slc_LT1_all.py" "$PROJECT" --parallel 1 -run_stage makedem_pyint "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/makedem_pyint.py" "$PROJECT" -run_stage generate_rdc_dem "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/generate_rdc_dem.py" "$PROJECT" -run_stage coreg_gamma_all "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/coreg_gamma_all.py" "$PROJECT" --parallel 1 -run_stage select_pairs "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/select_pairs.py" "$PROJECT" - -echo "[done] project=$PROJECT" -echo "[done] root=$ROOT" diff --git a/.codex_tmp/run_pyint_ab_case.sh b/.codex_tmp/run_pyint_ab_case.sh deleted file mode 100644 index 00d3f9d..0000000 --- a/.codex_tmp/run_pyint_ab_case.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -if [[ $# -lt 3 ]]; then - echo "usage: run_pyint_ab_case.sh <case_name> <pyint_home> <orbit_enabled:true|false> [unwrap:true|false] [geocode:true|false]" >&2 - exit 2 -fi - -case_name=$1 -pyint_home=$2 -orbit_enabled=$3 -unwrap_enabled=${4:-false} -geocode_enabled=${5:-false} - -repo=/mnt/d/Code/Insar_management_system_v2 -task_name=Task_20230602_20230720 -pair_key=lt1_20230602_20230720_ef1cd36538 -task_dir=/mnt/d/Task_Pool/DInSAR/Task_260416_Gamma_PyINT/Task_20230602_20230720 -manifest=/mnt/d/Code/Insar_management_system_v2/backend/runtime/pyint_input_assets/$pair_key/run_20260419T065041Z_pyint_lt1_gamma_dinsar/task_manifest.json -input_assets_dir=$(dirname "$manifest") -python_bin=/home/administrator/miniconda3/envs/isce2/bin/python -script=$repo/backend/app/pyint_pipeline/run_lt1_pyint_pipeline.py -pyint_app=$pyint_home/pyint/pyintApp.py -gamma_env=$repo/backend/app/pyint_pipeline/pyint_gamma_env.sh -prepared_dem=/mnt/d/DEM/COPDEM_GLO30_China_4326_DEM - -case_root=/mnt/d/PyINT_AB/$case_name -run_root=$case_root/work -project_name=${pair_key}_${case_name} -project_dir=$run_root/$project_name -template_root=$run_root/templates -output_dir=$case_root/output -dem_root=$case_root/dem_cache - -rm -rf "$case_root" -mkdir -p "$case_root" - -cmd=( - "$python_bin" "$script" "$task_dir" - --project-dir "$project_dir" - --template-root "$template_root" - --output-dir "$output_dir" - --pyint-home "$pyint_home" - --pyint-app-script "$pyint_app" - --python "$python_bin" - --dem-root "$dem_root" - --dem-mode prepared_file - --prepared-dem-path "$prepared_dem" - --project-name "$project_name" - --gamma-env-script "$gamma_env" - --pair-key "$pair_key" - --task-alias "$task_name" - --orbit-policy require_txt - --input-assets-dir "$input_assets_dir" - --input-assets-json "$manifest" - --master-date 20230602 - --slave-date 20230720 - --time-baseline-days 48 - --range-looks 2 - --azimuth-looks 2 - --parallel-workers 1 - --lt1-precise-orbit-enabled "$orbit_enabled" -) - -if [[ "$unwrap_enabled" == "true" ]]; then - cmd+=(--unwrap) -else - cmd+=(--no-unwrap) -fi - -if [[ "$geocode_enabled" == "true" ]]; then - cmd+=(--geocode) -else - cmd+=(--no-geocode) -fi - -printf 'CASE=%s\nPYINT_HOME=%s\nORBIT=%s\nUNWRAP=%s\nGEOCODE=%s\nCASE_ROOT=%s\n' \ - "$case_name" "$pyint_home" "$orbit_enabled" "$unwrap_enabled" "$geocode_enabled" "$case_root" - -"${cmd[@]}" diff --git a/.codex_tmp/run_scan_init_offsetm_patch.sh b/.codex_tmp/run_scan_init_offsetm_patch.sh deleted file mode 100644 index 4c9c23b..0000000 --- a/.codex_tmp/run_scan_init_offsetm_patch.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -REPO='/mnt/d/Code/Insar_management_system_v2' -RUN_ROOT='/mnt/d/PyINT_POOL_TEST/LT1A_MONO_SYC_STRIP1_E129.6_N45.0_3scene_cases/run_20260420T093322Z' -PYTHON_BIN='/home/administrator/miniconda3/envs/isce2/bin/python' -SCAN_SCRIPT="$REPO/.codex_tmp/scan_init_offsetm_patch.py" -GAMMA_ENV="$REPO/backend/app/pyint_pipeline/pyint_gamma_env.sh" - -. "$GAMMA_ENV" >/dev/null 2>&1 -export PATH="/usr/local/GAMMA_SOFTWARE-20240627/ISP/bin:/usr/local/GAMMA_SOFTWARE-20240627/ISP/scripts:$PATH" - -"$PYTHON_BIN" "$SCAN_SCRIPT" "$RUN_ROOT" "$@" diff --git a/.codex_tmp/scan_init_offsetm_patch.py b/.codex_tmp/scan_init_offsetm_patch.py deleted file mode 100644 index 9df317c..0000000 --- a/.codex_tmp/scan_init_offsetm_patch.py +++ /dev/null @@ -1,317 +0,0 @@ -from __future__ import annotations - -import argparse -import json -import os -import re -import shutil -import subprocess -import sys -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -import numpy as np - - -GAMMA_FLOAT32 = np.dtype(">f4") -ZERO_ERROR_RE = re.compile(r"number of zero values\s+(\d+)\s+in MLI1 image patch exceeds threshold:\s+(\d+)", re.IGNORECASE) - - -@dataclass -class GammaShape: - width: int - lines: int - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Scan init_offsetm patch positions for LT-1 DEM-assisted coreg.") - parser.add_argument("run_root", help="Experiment run root, e.g. /mnt/d/.../run_20260420T093322Z") - parser.add_argument("--case", dest="cases", action="append", default=[], help="Case name to scan. Can be repeated.") - parser.add_argument("--date", dest="dates", action="append", default=[], help="Slave date to scan. Can be repeated.") - parser.add_argument("--project", default="pyint_stage") - parser.add_argument("--patch-size", type=int, default=512) - parser.add_argument("--stride", type=int, default=256) - parser.add_argument("--max-candidates", type=int, default=20) - return parser.parse_args() - - -def parse_gamma_par_value(path: Path, key: str) -> str: - prefix = key.strip() + ":" - for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): - stripped = line.strip() - if stripped.startswith(prefix): - _, _, tail = stripped.partition(":") - return tail.strip().split()[0] - raise ValueError(f"Missing key '{key}' in {path}") - - -def parse_gamma_shape(path: Path) -> GammaShape: - width = int(float(parse_gamma_par_value(path, "range_samples"))) - lines = int(float(parse_gamma_par_value(path, "azimuth_lines"))) - return GammaShape(width=width, lines=lines) - - -def build_valid_mask(path: Path, shape: GammaShape) -> np.ndarray: - arr = np.memmap(path, dtype=GAMMA_FLOAT32, mode="r", shape=(shape.lines, shape.width)) - valid = np.isfinite(arr) & (arr != 0) - return np.asarray(valid, dtype=np.uint8) - - -def patch_sum(integral: np.ndarray, top: int, left: int, bottom: int, right: int) -> int: - br = int(integral[bottom, right]) - tr = int(integral[top, right]) - bl = int(integral[bottom, left]) - tl = int(integral[top, left]) - return br - tr - bl + tl - - -def candidate_positions(mask: np.ndarray, *, patch_size: int, stride: int, max_candidates: int) -> list[dict[str, Any]]: - lines, width = mask.shape - patch_size = max(1, min(patch_size, lines, width)) - half = patch_size // 2 - valid_y = range(half, lines - (patch_size - half) + 1, max(1, stride)) - valid_x = range(half, width - (patch_size - half) + 1, max(1, stride)) - - integral = np.pad(mask.astype(np.int64), ((1, 0), (1, 0)), mode="constant") - integral = integral.cumsum(axis=0).cumsum(axis=1) - patch_area = patch_size * patch_size - - center_y = lines // 2 - center_x = width // 2 - center_key = (center_x, center_y) - - candidates: list[dict[str, Any]] = [] - seen: set[tuple[int, int]] = set() - - for y in valid_y: - for x in valid_x: - top = y - half - left = x - half - bottom = top + patch_size - right = left + patch_size - nonzero = patch_sum(integral, top, left, bottom, right) - ratio = float(nonzero / patch_area) - entry = { - "rpos": int(x), - "azpos": int(y), - "patch_nonzero_count": int(nonzero), - "patch_nonzero_ratio": ratio, - "is_center": bool((x, y) == center_key), - } - candidates.append(entry) - - candidates.sort(key=lambda item: item["patch_nonzero_ratio"], reverse=True) - - selected: list[dict[str, Any]] = [] - for entry in candidates: - key = (entry["rpos"], entry["azpos"]) - if key in seen: - continue - selected.append(entry) - seen.add(key) - if len(selected) >= max_candidates: - break - - if center_key not in seen: - top = center_y - half - left = center_x - half - top = max(0, min(lines - patch_size, top)) - left = max(0, min(width - patch_size, left)) - bottom = top + patch_size - right = left + patch_size - nonzero = patch_sum(integral, top, left, bottom, right) - selected.append( - { - "rpos": int(left + half), - "azpos": int(top + half), - "patch_nonzero_count": int(nonzero), - "patch_nonzero_ratio": float(nonzero / patch_area), - "is_center": True, - } - ) - - for index, entry in enumerate(selected, start=1): - entry["rank"] = index - return selected - - -def parse_zero_error(text: str) -> dict[str, Any]: - match = ZERO_ERROR_RE.search(text or "") - if not match: - return {} - return { - "zero_count": int(match.group(1)), - "zero_threshold": int(match.group(2)), - } - - -def run_init_offsetm( - *, - mli0: Path, - samp: Path, - diff0: Path, - output_dir: Path, - patch_size: int, - rpos: int, - azpos: int, -) -> dict[str, Any]: - output_dir.mkdir(parents=True, exist_ok=True) - diff_copy = output_dir / f"r{rpos}_a{azpos}.diff_par" - shutil.copy2(diff0, diff_copy) - - cmd = [ - "init_offsetm", - str(mli0), - str(samp), - str(diff_copy), - "1", - "1", - str(int(rpos)), - str(int(azpos)), - "-", - "-", - "-", - str(int(patch_size)), - "0", - ] - result = subprocess.run(cmd, text=True, capture_output=True, check=False) - combined = ((result.stdout or "") + "\n" + (result.stderr or "")).strip() - payload = { - "command": " ".join(cmd), - "returncode": int(result.returncode), - "ok": result.returncode == 0, - "stdout_tail": (result.stdout or "")[-4000:], - "stderr_tail": (result.stderr or "")[-4000:], - "diff_par_copy": str(diff_copy), - } - payload.update(parse_zero_error(combined)) - return payload - - -def scan_case_date( - *, - run_root: Path, - project: str, - case_name: str, - slave_date: str, - patch_size: int, - stride: int, - max_candidates: int, - output_root: Path, -) -> dict[str, Any]: - case_root = run_root / case_name / project - slc_dir = case_root / "SLC" / slave_date - rslc_dir = case_root / "RSLC" / slave_date - amp_par = slc_dir / f"{slave_date}_2rlks.amp.par" - shape = parse_gamma_shape(amp_par) - mask = build_valid_mask(rslc_dir / "mli0", shape) - - candidates = candidate_positions(mask, patch_size=patch_size, stride=stride, max_candidates=max_candidates) - output_dir = output_root / case_name / slave_date - results = [] - for candidate in candidates: - command_result = run_init_offsetm( - mli0=rslc_dir / "mli0", - samp=slc_dir / f"{slave_date}_2rlks.amp", - diff0=rslc_dir / "diff0", - output_dir=output_dir / "diff_par", - patch_size=patch_size, - rpos=int(candidate["rpos"]), - azpos=int(candidate["azpos"]), - ) - row = dict(candidate) - row.update(command_result) - results.append(row) - - success_count = sum(1 for item in results if item["ok"]) - best_ratio = max((float(item["patch_nonzero_ratio"]) for item in results), default=0.0) - best_zero = min((int(item.get("zero_count", 10**18)) for item in results if "zero_count" in item), default=None) - payload = { - "case": case_name, - "slave_date": slave_date, - "shape": {"width": shape.width, "lines": shape.lines}, - "patch_size": int(patch_size), - "stride": int(stride), - "candidate_count": len(results), - "success_count": int(success_count), - "best_patch_nonzero_ratio": best_ratio, - "best_zero_count": best_zero, - "results": results, - } - output_dir.mkdir(parents=True, exist_ok=True) - (output_dir / "scan_summary.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - return payload - - -def write_summary_tsv(path: Path, summaries: list[dict[str, Any]]) -> None: - header = [ - "case", - "slave_date", - "rank", - "is_center", - "rpos", - "azpos", - "patch_nonzero_ratio", - "returncode", - "ok", - "zero_count", - "zero_threshold", - ] - lines = ["\t".join(header)] - for summary in summaries: - for row in summary["results"]: - values: list[str] = [] - for key in header: - if key in {"case", "slave_date"}: - value = summary[key] - else: - value = row.get(key) - if isinstance(value, float): - values.append(f"{value:.6f}") - elif value is None: - values.append("") - else: - values.append(str(value)) - lines.append("\t".join(values)) - path.write_text("\n".join(lines) + "\n", encoding="utf-8") - - -def main() -> int: - args = parse_args() - run_root = Path(args.run_root).resolve() - case_names = tuple(args.cases) if args.cases else ("case_A_baseline", "case_C_precise_orbit_rewrite") - slave_dates = tuple(args.dates) if args.dates else ("20230624", "20230920") - output_root = run_root / "scan_init_offsetm_patch" - output_root.mkdir(parents=True, exist_ok=True) - - if not shutil.which("init_offsetm"): - raise RuntimeError("init_offsetm is not available in PATH") - - summaries = [] - for case_name in case_names: - for slave_date in slave_dates: - summaries.append( - scan_case_date( - run_root=run_root, - project=args.project, - case_name=case_name, - slave_date=slave_date, - patch_size=int(args.patch_size), - stride=int(args.stride), - max_candidates=int(args.max_candidates), - output_root=output_root, - ) - ) - - (output_root / "scan_summary.json").write_text( - json.dumps({"run_root": str(run_root), "summaries": summaries}, ensure_ascii=False, indent=2) + "\n", - encoding="utf-8", - ) - write_summary_tsv(output_root / "scan_summary.tsv", summaries) - print(json.dumps({"run_root": str(run_root), "output_root": str(output_root), "scan_count": len(summaries)}, ensure_ascii=False)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/.codex_tmp/setup_lt1_pool_multiscene_experiment.ps1 b/.codex_tmp/setup_lt1_pool_multiscene_experiment.ps1 deleted file mode 100644 index 65fb0cd..0000000 --- a/.codex_tmp/setup_lt1_pool_multiscene_experiment.ps1 +++ /dev/null @@ -1,87 +0,0 @@ -param( - [Parameter(Mandatory = $true)] - [string]$Root, - - [Parameter(Mandatory = $true)] - [string]$SourcePool, - - [Parameter(Mandatory = $true)] - [string[]]$Scenes -) - -$ErrorActionPreference = "Stop" - -function Fail([string]$Message) { - throw $Message -} - -$rootPath = [System.IO.Path]::GetFullPath($Root) -$sourcePoolPath = [System.IO.Path]::GetFullPath($SourcePool) - -if (-not (Test-Path -LiteralPath $sourcePoolPath -PathType Container)) { - Fail "Source pool not found: $sourcePoolPath" -} - -if (Test-Path -LiteralPath $rootPath) { - Fail "Experiment root already exists: $rootPath" -} - -$downloadDir = Join-Path $rootPath "pyint_stage\DOWNLOAD" -$inputDir = Join-Path $rootPath "input" -$templatesDir = Join-Path $rootPath "templates" -$logsDir = Join-Path $rootPath "logs" -$demStoreDir = Join-Path $rootPath "dem_store" - -New-Item -ItemType Directory -Path $downloadDir -Force | Out-Null -New-Item -ItemType Directory -Path $inputDir -Force | Out-Null -New-Item -ItemType Directory -Path $templatesDir -Force | Out-Null -New-Item -ItemType Directory -Path $logsDir -Force | Out-Null -New-Item -ItemType Directory -Path $demStoreDir -Force | Out-Null - -$manifestLines = @( - "Experiment Root: $rootPath" - "Created: $(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')" - "Source Pool: $sourcePoolPath" - "Scenes:" -) - -$copiedScenes = @() - -foreach ($scene in $Scenes) { - $sceneSourceDir = Join-Path $sourcePoolPath $scene - if (-not (Test-Path -LiteralPath $sceneSourceDir -PathType Container)) { - Fail "Scene not found: $sceneSourceDir" - } - - $sceneInputDir = Join-Path $inputDir $scene - New-Item -ItemType Directory -Path $sceneInputDir -Force | Out-Null - - $files = Get-ChildItem -LiteralPath $sceneSourceDir -File - if ($files.Count -eq 0) { - Fail "No files found in scene: $sceneSourceDir" - } - - foreach ($file in $files) { - Copy-Item -LiteralPath $file.FullName -Destination (Join-Path $sceneInputDir $file.Name) - Copy-Item -LiteralPath $file.FullName -Destination (Join-Path $downloadDir $file.Name) - } - - $manifestLines += "$scene | files=$($files.Count)" - $copiedScenes += [pscustomobject]@{ - scene = $scene - file_count = $files.Count - } -} - -$manifestPath = Join-Path $rootPath "MANIFEST.txt" -Set-Content -LiteralPath $manifestPath -Value $manifestLines -Encoding UTF8 - -$result = [pscustomobject]@{ - root = $rootPath - download_dir = $downloadDir - input_dir = $inputDir - manifest = $manifestPath - scenes = $copiedScenes -} - -$result | ConvertTo-Json -Depth 4 diff --git a/.codex_tmp/summarize_pyint_case.py b/.codex_tmp/summarize_pyint_case.py deleted file mode 100644 index b8b67a4..0000000 --- a/.codex_tmp/summarize_pyint_case.py +++ /dev/null @@ -1,82 +0,0 @@ -from __future__ import annotations - -import json -import math -import sys -from pathlib import Path - -import numpy as np - - -def summarize_float_raster(path: Path) -> dict[str, object]: - if not path.is_file(): - return {"exists": False} - - arr = np.fromfile(path, dtype=np.float32) - if arr.size == 0: - return {"exists": True, "count": 0} - - finite = arr[np.isfinite(arr)] - zeros = int((finite == 0).sum()) - nz = finite[finite != 0] - result: dict[str, object] = { - "exists": True, - "count": int(finite.size), - "zero_ratio": float(zeros / finite.size) if finite.size else None, - "nonzero_count": int(nz.size), - } - if nz.size: - result.update( - { - "min_nonzero": float(nz.min()), - "max_nonzero": float(nz.max()), - "mean_nonzero": float(nz.mean()), - "std_nonzero": float(nz.std()), - } - ) - return result - - -def main() -> int: - if len(sys.argv) != 2: - print("usage: summarize_pyint_case.py <case_root>", file=sys.stderr) - return 2 - - case_root = Path(sys.argv[1]).resolve() - output_root = case_root / "output" - summary_path = output_root / "pyint_run_summary.json" - if not summary_path.is_file(): - print(json.dumps({"case_root": str(case_root), "summary_exists": False}, ensure_ascii=False, indent=2)) - return 1 - - payload = json.loads(summary_path.read_text(encoding="utf-8")) - pair_dir = Path(payload["copied_outputs"]["pair_dir"]) - - pair_name = str(payload["pair_name"]) - rlks = int(payload["range_looks"]) - look_text = f"{rlks}rlks" - - files = { - "diff_filt": pair_dir / f"{pair_name}_{look_text}.diff_filt", - "cor": pair_dir / f"{pair_name}_{look_text}.diff_filt.cor", - "unw": pair_dir / f"{pair_name}_{look_text}.diff_filt.unw", - "los": pair_dir / f"{pair_name}_{look_text}.los_disp", - "vert": pair_dir / f"{pair_name}_{look_text}.vert_disp", - "geo_unw": pair_dir / f"geo_{pair_name}_{look_text}.diff_filt.unw", - "geo_los": pair_dir / f"geo_{pair_name}_{look_text}.los_disp", - "geo_vert": pair_dir / f"geo_{pair_name}_{look_text}.vert_disp", - } - - result = { - "case_root": str(case_root), - "ok": payload.get("ok"), - "project_name": payload.get("project_name"), - "pair_dir": str(pair_dir), - "files": {name: summarize_float_raster(path) for name, path in files.items()}, - } - print(json.dumps(result, ensure_ascii=False, indent=2)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/.env.example b/.env.example index 25f26d5..857e2c4 100644 --- a/.env.example +++ b/.env.example @@ -181,6 +181,35 @@ ISCE2_SMOKE_TEST_ENABLED=false # The WSL runtime must include cv2 + scipy for ionosphere correction and astropy for rubbersheeting. +# ----------------------------------------------------------------------------- +# LandSAR D-InSAR +# ----------------------------------------------------------------------------- +LANDSAR_ENABLED=true +LANDSAR_HOME=D:\LandSAR +LANDSAR_CONSOLE_EXE=D:\LandSAR\InSAR_Console.exe +LANDSAR_WORK_ROOT=D:\LandSAR_Work +# Optional extra DLL directories separated by semicolons. +LANDSAR_RUNTIME_PATHS=D:\LandSAR +LANDSAR_LICENSE_MODE=netVersion +LANDSAR_LICENSE_HOST=127.0.0.1 +LANDSAR_LICENSE_PORT=6666 +LANDSAR_CONFIG_ROW=netVersion,zh,127.0.0.1,6666 +LANDSAR_CONFIG_AUTO_WRITE=true +LANDSAR_AUTH_SERVER_EXE=D:\Code\Insar_management_system_v2\third_party\LandSAR\tools\_portable_release\LandSAR_auth_tools_win64\landsar_net_auth_server.exe +LANDSAR_AUTH_SERVER_AUTO_START=true +LANDSAR_AUTH_SERVER_HOST=127.0.0.1 +LANDSAR_AUTH_SERVER_PORT=6666 +LANDSAR_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84 +LANDSAR_DINSAR_TIMEOUT_SECONDS=43200 +LANDSAR_SBAS_ENABLED=true +LANDSAR_SBAS_WORK_ROOT=D:\LandSAR_Work\sbas +LANDSAR_SBAS_PRODUCT_ROOT=D:\production_results\timeseries\sbas_landsar +LANDSAR_SBAS_DEM_PATH=D:\DEM\SRTMDEM_RSP_SARscape.wgs84 +LANDSAR_SBAS_SOURCE_ROOTS=D:\LandSAR_Work +LANDSAR_SBAS_TIMEOUT_SECONDS=172800 +LANDSAR_SBAS_MIN_SCENES=3 + + # ----------------------------------------------------------------------------- # Gamma / PyINT # ----------------------------------------------------------------------------- @@ -243,11 +272,13 @@ GAMMA_SBAS_PRODUCT_ROOT=D:\production_results\timeseries\sbas GAMMA_SBAS_SCRIPT_TEMPLATE_ROOT=D:\Code\Insar_management_system_v2\backend\templates\gamma_sbas GAMMA_SBAS_SOURCE_ROOTS=D:\LuTan1_Image_Pool GAMMA_SBAS_ORBIT_ROOTS=D:\orbit_pools\envi +GAMMA_SBAS_DEM_PATH=D:\DEM\HeiLongJiang10M_DEM.tif GAMMA_SBAS_DEFAULT_RLKS=8 GAMMA_SBAS_DEFAULT_AZLKS=8 GAMMA_SBAS_DEFAULT_MB_MODE=0 GAMMA_SBAS_DEFAULT_REFERENCE_WINDOW=16 GAMMA_SBAS_AUTO_APPROVE_ITAB=true +GAMMA_SBAS_MIN_COMMON_OVERLAP_RATIO=0.30 GAMMA_SBAS_STEP_TIMEOUT_SECONDS=43200 GAMMA_SBAS_WORKFLOW_TIMEOUT_SECONDS=172800 diff --git a/.gitignore b/.gitignore index 6425da7..a0e2157 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,12 @@ backend/quality_model.pkl /IDL*.tmp /env_*.xyz .codex_tmp/ +.codex_tmp_* +base.out +tmp_dir_coreg_*/ + +# Local third-party runtime bundles +third_party/LandSAR/ # Large local datasets / installers Data/ diff --git a/backend/app/config.py b/backend/app/config.py index be4d655..cd13710 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -273,6 +273,21 @@ class Settings(BaseSettings): "site-packages/isce/applications/stripmapApp.py" ) ISCE2_PIPELINE_SCRIPT: str = "" + LANDSAR_ENABLED: bool = True + LANDSAR_HOME: str = "" + LANDSAR_CONSOLE_EXE: str = "" + LANDSAR_WORK_ROOT: str = "" + LANDSAR_LICENSE_MODE: str = "netVersion" + LANDSAR_LICENSE_HOST: str = "127.0.0.1" + LANDSAR_LICENSE_PORT: int = 6666 + LANDSAR_CONFIG_ROW: str = "" + LANDSAR_CONFIG_AUTO_WRITE: bool = True + LANDSAR_AUTH_SERVER_EXE: str = "" + LANDSAR_AUTH_SERVER_AUTO_START: bool = True + LANDSAR_AUTH_SERVER_HOST: str = "127.0.0.1" + LANDSAR_AUTH_SERVER_PORT: int = 6666 + LANDSAR_DEM_PATH: str = "" + LANDSAR_DINSAR_TIMEOUT_SECONDS: int = 43200 PYINT_ENABLED: bool = False PYINT_WSL_DISTRO: str = "" PYINT_WSL_PYTHON: str = "" @@ -328,13 +343,24 @@ class Settings(BaseSettings): GAMMA_SBAS_SCRIPT_TEMPLATE_ROOT: str = "" GAMMA_SBAS_SOURCE_ROOTS: str = "" GAMMA_SBAS_ORBIT_ROOTS: str = "" + GAMMA_SBAS_DEM_PATH: str = "" GAMMA_SBAS_DEFAULT_RLKS: int = 8 GAMMA_SBAS_DEFAULT_AZLKS: int = 8 GAMMA_SBAS_DEFAULT_MB_MODE: int = 0 GAMMA_SBAS_DEFAULT_REFERENCE_WINDOW: int = 16 GAMMA_SBAS_AUTO_APPROVE_ITAB: bool = True + GAMMA_SBAS_MIN_COMMON_OVERLAP_RATIO: float = 0.30 GAMMA_SBAS_STEP_TIMEOUT_SECONDS: int = 43200 GAMMA_SBAS_WORKFLOW_TIMEOUT_SECONDS: int = 172800 + LANDSAR_SBAS_ENABLED: bool = True + LANDSAR_SBAS_WORK_ROOT: str = "" + LANDSAR_SBAS_PRODUCT_ROOT: str = "" + LANDSAR_SBAS_DEM_PATH: str = "" + LANDSAR_SBAS_SOURCE_ROOTS: str = "" + LANDSAR_SBAS_TIMEOUT_SECONDS: int = 172800 + LANDSAR_SBAS_MIN_SCENES: int = 3 + LANDSAR_SBAS_PROID: str = "280039" + LANDSAR_SBAS_PROCESS_NAME: str = "SBAS Stream" JOB_WORKER_HEALTH_TIMEOUT: int = 60 JOB_WORKER_JOB_HEARTBEAT_INTERVAL: float = 5.0 @@ -491,6 +517,42 @@ class Settings(BaseSettings): "ISCE2_PIPELINE_SCRIPT", _windows_path_to_wsl_mount(local_pipeline), ) + if not self.LANDSAR_HOME: + object.__setattr__( + self, + "LANDSAR_HOME", + os.path.join(project_root, "third_party", "LandSAR", "dist", "LandSAR_Portable"), + ) + if not self.LANDSAR_CONSOLE_EXE: + object.__setattr__( + self, + "LANDSAR_CONSOLE_EXE", + os.path.join(self.LANDSAR_HOME, "InSAR_Console.exe"), + ) + if not self.LANDSAR_WORK_ROOT: + object.__setattr__( + self, + "LANDSAR_WORK_ROOT", + os.path.join(self.RESULT_PUBLISH_ROOT, "landsar_work"), + ) + if not self.LANDSAR_AUTH_SERVER_EXE: + auth_server = os.path.join( + project_root, + "third_party", + "LandSAR", + "tools", + "_portable_release", + "LandSAR_auth_tools_win64", + "landsar_net_auth_server.exe", + ) + object.__setattr__(self, "LANDSAR_AUTH_SERVER_EXE", auth_server) + if not self.LANDSAR_DEM_PATH: + landsar_dem = ( + _clean_path_text(self.PYINT_PREPARED_DEM_PATH) + or _clean_path_text(self.ISCE2_DEM_PATH) + or _clean_path_text(self.IDL_DINSAR_DEM_BASE_FILE) + ) + object.__setattr__(self, "LANDSAR_DEM_PATH", landsar_dem) if not self.WSL_DISTRO: fallback_distro = str( self.ISCE2_WSL_DISTRO @@ -667,6 +729,16 @@ class Settings(BaseSettings): ) if not self.GAMMA_SBAS_ENV_SCRIPT: object.__setattr__(self, "GAMMA_SBAS_ENV_SCRIPT", self.PYINT_GAMMA_ENV_SCRIPT) + if not self.GAMMA_SBAS_DEM_PATH: + object.__setattr__( + self, + "GAMMA_SBAS_DEM_PATH", + self.LANDSAR_SBAS_DEM_PATH + or self.LANDSAR_DEM_PATH + or self.IDL_DINSAR_DEM_BASE_FILE + or self.ISCE2_DEM_PATH + or self.PYINT_PREPARED_DEM_PATH, + ) if not self.GAMMA_SBAS_WORK_ROOT: object.__setattr__( self, @@ -714,6 +786,11 @@ class Settings(BaseSettings): "GAMMA_SBAS_DEFAULT_REFERENCE_WINDOW", max(1, int(self.GAMMA_SBAS_DEFAULT_REFERENCE_WINDOW or 16)), ) + object.__setattr__( + self, + "GAMMA_SBAS_MIN_COMMON_OVERLAP_RATIO", + min(1.0, max(0.0, float(self.GAMMA_SBAS_MIN_COMMON_OVERLAP_RATIO or 0.30))), + ) object.__setattr__( self, "GAMMA_SBAS_STEP_TIMEOUT_SECONDS", @@ -724,6 +801,26 @@ class Settings(BaseSettings): "GAMMA_SBAS_WORKFLOW_TIMEOUT_SECONDS", max(self.GAMMA_SBAS_STEP_TIMEOUT_SECONDS, int(self.GAMMA_SBAS_WORKFLOW_TIMEOUT_SECONDS or 172800)), ) + if not self.LANDSAR_SBAS_WORK_ROOT: + object.__setattr__( + self, + "LANDSAR_SBAS_WORK_ROOT", + os.path.join(self.LANDSAR_WORK_ROOT or os.path.join(self.RESULT_PUBLISH_ROOT, "landsar_work"), "sbas"), + ) + if not self.LANDSAR_SBAS_PRODUCT_ROOT: + object.__setattr__( + self, + "LANDSAR_SBAS_PRODUCT_ROOT", + os.path.join(self.TIMESERIES_PRODUCT_DIR, "sbas_landsar"), + ) + if not self.LANDSAR_SBAS_DEM_PATH: + object.__setattr__(self, "LANDSAR_SBAS_DEM_PATH", self.LANDSAR_DEM_PATH) + if not self.LANDSAR_SBAS_SOURCE_ROOTS: + object.__setattr__(self, "LANDSAR_SBAS_SOURCE_ROOTS", self.LANDSAR_WORK_ROOT or r"D:\LandSAR_Work") + object.__setattr__(self, "LANDSAR_SBAS_TIMEOUT_SECONDS", max(60, int(self.LANDSAR_SBAS_TIMEOUT_SECONDS or 172800))) + object.__setattr__(self, "LANDSAR_SBAS_MIN_SCENES", max(3, int(self.LANDSAR_SBAS_MIN_SCENES or 3))) + object.__setattr__(self, "LANDSAR_SBAS_PROID", _clean_path_text(self.LANDSAR_SBAS_PROID) or "280039") + object.__setattr__(self, "LANDSAR_SBAS_PROCESS_NAME", str(self.LANDSAR_SBAS_PROCESS_NAME or "").strip() or "SBAS Stream") if self.TIMESERIES_ENABLED: if not self.TIMESERIES_WSL_DISTRO: object.__setattr__(self, "TIMESERIES_WSL_DISTRO", self.WSL_DISTRO or self.ISCE2_WSL_DISTRO) @@ -856,10 +953,15 @@ class Settings(BaseSettings): os.makedirs(settings.PYINT_WORK_ROOT, exist_ok=True) os.makedirs(settings.PYINT_OUTPUT_ROOT, exist_ok=True) os.makedirs(settings.PYINT_DEM_ROOT, exist_ok=True) + if settings.LANDSAR_WORK_ROOT: + os.makedirs(settings.LANDSAR_WORK_ROOT, exist_ok=True) if settings.GAMMA_SBAS_ENABLED: os.makedirs(settings.GAMMA_SBAS_WORK_ROOT, exist_ok=True) os.makedirs(settings.GAMMA_SBAS_PRODUCT_ROOT, exist_ok=True) os.makedirs(settings.GAMMA_SBAS_SCRIPT_TEMPLATE_ROOT, exist_ok=True) + if settings.LANDSAR_SBAS_ENABLED: + os.makedirs(settings.LANDSAR_SBAS_WORK_ROOT, exist_ok=True) + os.makedirs(settings.LANDSAR_SBAS_PRODUCT_ROOT, exist_ok=True) if settings.TIMESERIES_ENABLED and settings.TIMESERIES_WORK_ROOT: os.makedirs(settings.TIMESERIES_WORK_ROOT, exist_ok=True) @@ -1026,6 +1128,7 @@ def validate_runtime_config() -> dict[str, Any]: _check_path(label="ORBIT_POOL_ISCE2", value=settings.ORBIT_POOL_ISCE2, errors=errors, warnings=warnings, expect_file=False) _check_path(label="RESULT_PUBLISH_ROOT", value=settings.RESULT_PUBLISH_ROOT, errors=errors, warnings=warnings, expect_file=False) _check_path(label="DINSAR_PRODUCT_DIR", value=settings.DINSAR_PRODUCT_DIR, errors=errors, warnings=warnings, expect_file=False) + _check_path(label="LANDSAR_WORK_ROOT", value=settings.LANDSAR_WORK_ROOT, errors=errors, warnings=warnings, expect_file=False) _check_path( label="TIMESERIES_PRODUCT_DIR", value=settings.TIMESERIES_PRODUCT_DIR, @@ -1183,6 +1286,13 @@ def validate_runtime_config() -> dict[str, Any]: warnings=warnings, expect_file=False, ) + _check_path( + label="GAMMA_SBAS_DEM_PATH", + value=settings.GAMMA_SBAS_DEM_PATH, + errors=errors, + warnings=warnings, + expect_file=True, + ) if settings.ISCE2_ENABLED or settings.PYINT_ENABLED: info.append( diff --git a/backend/app/dinsar_engines/isce2_engine.py b/backend/app/dinsar_engines/isce2_engine.py index 3337091..fe39b29 100644 --- a/backend/app/dinsar_engines/isce2_engine.py +++ b/backend/app/dinsar_engines/isce2_engine.py @@ -710,7 +710,7 @@ class Isce2Engine(DinsarEngine): available = True else: critical_failed = [check for check in report.checks if not check.ok and not check.skipped] - status = "degraded" if critical_failed else "unavailable" + status = "unavailable" if critical_failed else "degraded" available = False return EngineAvailability( diff --git a/backend/app/dinsar_engines/landsar_engine.py b/backend/app/dinsar_engines/landsar_engine.py index aaae498..0b35acd 100644 --- a/backend/app/dinsar_engines/landsar_engine.py +++ b/backend/app/dinsar_engines/landsar_engine.py @@ -1,49 +1,1963 @@ -"""LANDSAR 引擎占位实现。 +"""LandSAR D-InSAR engine integration. -本轮不实现算法，仅保留接口和前端占位。 -check_available() 永远返回 not_implemented。 -run() 直接抛出 NotImplementedError。 +This engine wraps LandSAR's console workflow: + InSAR_Console.exe <Output_Data/200014.txt> + +It intentionally does not import the bundled PyQt GUI helper. Only the +documented proID 200014 parameter-file format is reproduced here so the backend +can run in the service process without desktop dependencies. """ from __future__ import annotations -from typing import List +import json +import os +import queue +import re +import shutil +import socket +import subprocess +import threading +import time +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional +from ..config import get_env_text, read_bool_env, settings +from ..services.dinsar_naming import ( + PAIR_META_FILENAME, + build_fallback_pair_key, + write_pair_metadata, + write_run_metadata, +) +from ..services.dinsar_result_layout_service import normalize_isce2_run_layout +from ..services.isce2_result_validator import validate_isce2_result_files +from ..utils import normalize_satellite_family from .base import DinsarEngine, EngineAvailability, EngineProfile, RunRequest, RunResult -class LandsarEngine(DinsarEngine): +DINSAR_PROID = "200014" +IMPORT_PROID = "100016" +RERUN_MODE_UNFINISHED_ONLY = "unfinished_only" +SUPPORTED_PROFILES = {"lt1_dinsar", "standard"} +_PROJECT_ROOT = Path(__file__).resolve().parents[3] +_DATE_RE = re.compile(r"(?:^|[_-])((?:19|20)\d{6})(?:[_-]|$)") +_SAFE_NAME_RE = re.compile(r"[^0-9A-Za-z._-]+") +_SUCCESS_RE = re.compile(r"(success|成功)", re.IGNORECASE) + +_DEFAULT_PARAM_VALUES: Dict[str, Any] = { + "dem_file_type": 0, + "dem_product_type": 0, + "gcp_file": "", + "gacos_file": "", + "do_registration": 1, + "reg_method": 0, + "reg_grid_points": 64, + "reg_window": 128, + "reg_snr": 7, + "do_resample": 1, + "crop_invalid": 1, + "crop_gcps": 0, + "do_interferogram": 1, + "az_looks": 3, + "rg_looks": 3, + "gen_intensity": 1, + "gcp_geometric": 0, + "gcp_multilook": 0, + "gen_8bit": 0, + "do_deflatten": 1, + "deflat_method": 0, + "deflat_window": 128, + "deflat_oversample": 3, + "do_filter": 1, + "filter_alpha": 0.6, + "filter_iterations": 1, + "do_coherence_mask": 1, + "coh_mask_threshold": 0.3, + "do_unwrap": 1, + "unwrap_method": 0, + "unwrap_az_blocks": 1, + "unwrap_rg_blocks": 1, + "unwrap_coh_threshold": 0.3, + "do_gcp_extract": 1, + "gcp_grid_points": 25, + "gcp_flat_area": 10, + "gcp_height_diff": 10, + "gcp_coh_diff": 0.1, + "do_baseline_refine": 1, + "baseline_method": 0, + "phase_correction": 0, + "baseline_coh_threshold": 0.9, + "optimize_gcps": 0, + "gcp_error_threshold": 10, + "do_diff_fitting": 1, + "diff_coh_threshold": 0.91, + "diff_az_samples": 64, + "diff_rg_samples": 64, + "do_los_displacement": 1, + "displacement_format": 0, + "do_vertical_displacement": 0, + "do_atmosphere": 0, + "do_displacement_correction": 0, + "do_geocoding": 1, + "geo_wrapped": 1, + "geo_unwrapped": 1, + "geo_coherence": 1, + "geo_los": 1, + "geo_vertical": 1, +} + +_INT_PARAM_KEYS = { + "dem_file_type", + "dem_product_type", + "do_registration", + "reg_method", + "reg_grid_points", + "reg_window", + "reg_snr", + "do_resample", + "crop_invalid", + "crop_gcps", + "do_interferogram", + "az_looks", + "rg_looks", + "gen_intensity", + "gcp_geometric", + "gcp_multilook", + "gen_8bit", + "do_deflatten", + "deflat_method", + "deflat_window", + "deflat_oversample", + "do_filter", + "filter_iterations", + "do_coherence_mask", + "do_unwrap", + "unwrap_method", + "unwrap_az_blocks", + "unwrap_rg_blocks", + "do_gcp_extract", + "gcp_grid_points", + "gcp_flat_area", + "gcp_height_diff", + "do_baseline_refine", + "baseline_method", + "phase_correction", + "optimize_gcps", + "gcp_error_threshold", + "do_diff_fitting", + "diff_az_samples", + "diff_rg_samples", + "do_los_displacement", + "displacement_format", + "do_vertical_displacement", + "do_atmosphere", + "do_displacement_correction", + "do_geocoding", + "geo_wrapped", + "geo_unwrapped", + "geo_coherence", + "geo_los", + "geo_vertical", +} + +_FLOAT_PARAM_KEYS = { + "filter_alpha", + "coh_mask_threshold", + "unwrap_coh_threshold", + "gcp_coh_diff", + "baseline_coh_threshold", + "diff_coh_threshold", +} + +_PATH_PARAM_KEYS = {"dem_path", "gcp_file", "gacos_file"} +_LANDSAR_REQUIRED_DLLS = [ + "Qt5Core.dll", + "Qt5Network.dll", + "Qt5Widgets.dll", + "libgcc_s_seh-1.dll", + "libwinpthread-1.dll", + "libstdc++-6.dll", + "SAR_ImagePrinterModel.dll", + "SAR_InSAR_DInSARModel.dll", + "SAR_InSAR_GeneralModel.dll", + "SAR_InSAR_GeoInterferometricModel.dll", + "SAR_InSAR_GeometricModel.dll", + "SAR_InSAR_InSARModel.dll", + "SAR_InSAR_IOModel.dll", + "SAR_InSAR_Model.dll", + "SAR_InSAR_MTInSARModel.dll", + "SAR_InSAR_Sequential.dll", + "SAR_SwapIO.dll", +] +_SYSTEM_EXTRA_KEYS = { + "__managed_run_dir", + "__managed_native_output_dir", + "__managed_work_dir", + "__managed_export_dir", + "__managed_orbit_output_dir", + "__managed_run_key", + "__source_root_override", + "__rerun_mode", + "__validated_task_count", + "__validated_mode", + "__discovered_task_count", + "__skipped_completed_count", +} + + +def _read_env(name: str, default: str = "") -> str: + value = os.getenv(name) + if value is None: + value = get_env_text(name, default) + return str(value or default).strip().strip('"').strip("'") + + +def _norm_path(path: Any) -> str: + text = str(path or "").strip().strip('"').strip("'") + if not text: + return "" + return os.path.normpath(os.path.abspath(text)) + + +def _utc_text(value: Optional[datetime] = None) -> str: + return (value or datetime.utcnow()).isoformat(timespec="seconds") + "Z" + + +def _coerce_bool(value: Any) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, (int, float)): + return bool(value) + text = str(value or "").strip().lower() + if text in {"1", "true", "yes", "on"}: + return True + if text in {"0", "false", "no", "off", ""}: + return False + return bool(value) + + +def _read_int_env(name: str, default: int) -> int: + try: + return int(_read_env(name, str(default)) or default) + except (TypeError, ValueError): + return int(default) + + +def _tcp_connect(host: str, port: int, timeout: float = 1.0) -> bool: + if not host or int(port or 0) <= 0: + return False + try: + with socket.create_connection((host, int(port)), timeout=timeout): + return True + except OSError: + return False + + +def _decode_line(raw: bytes) -> str: + for encoding in ("utf-8", "gbk", "mbcs"): + try: + return raw.decode(encoding, errors="strict").rstrip("\r\n") + except Exception: + continue + return raw.decode("utf-8", errors="replace").rstrip("\r\n") + + +def _extract_date(name: str) -> str: + match = _DATE_RE.search(name or "") + if match: + return match.group(1) + fallback = re.search(r"((?:19|20)\d{6})", name or "") + return fallback.group(1) if fallback else "" + + +def _collect_tail(text: str, max_chars: int = 4000) -> str: + content = str(text or "") + if len(content) <= max_chars: + return content + return content[-max_chars:] + + +def _summarize_landsar_failure(stdout_text: str, stage: str, return_code: int) -> str: + content = str(stdout_text or "") + lowered = content.lower() + prefix = f"{stage} returned {return_code}" + if "connect server failed" in lowered or "load_server_memory_2_dongle failed" in lowered: + return f"{prefix}: LandSAR network license server is unreachable or rejected the request." + license_failure_markers = [ + "hasp_login failed", + "dongle_read failed", + "read_memory(from server) failed", + "buff_parsing failed", + ] + if any(marker in lowered for marker in license_failure_markers): + match = re.search(r"HASP_STATUS\s*==\s*([0-9]+)", content, re.IGNORECASE) + status_text = f" (HASP_STATUS == {match.group(1)})" if match else "" + return f"{prefix}: LandSAR license dongle login failed{status_text}." + if "cann't find 'config.csv'" in lowered or "can't find 'config.csv'" in lowered: + return f"{prefix}: LandSAR config.csv is missing. Run versionControl.exe once from LANDSAR_HOME." + lines = [line.strip() for line in content.splitlines() if line.strip()] + error_lines = [ + line + for line in lines + if ( + re.search(r"\berror\b", line, re.IGNORECASE) + or re.search(r"\bfailed\b", line, re.IGNORECASE) + or "CurProModule:" in line + ) + and "dongle.info:" not in line + and line.lower() != "console failed." + ] + if error_lines: + return f"{prefix}: {' | '.join(error_lines[-5:])}" + if lines: + return f"{prefix}: {' | '.join(lines[-4:])}" + return f"{prefix}." + + +def _find_first_existing(*paths: Path) -> str: + for path in paths: + try: + if path.is_file(): + return str(path) + except OSError: + continue + return "" + + +def _split_path_env(value: str) -> List[str]: + return [part.strip() for part in str(value or "").split(os.pathsep) if part.strip()] + + +def _path_search_dirs(*leading_dirs: str) -> List[str]: + seen = set() + configured_dirs = [ + *_split_path_env(_read_env("LANDSAR_RUNTIME_PATHS", "")), + *_split_path_env(_read_env("LANDSAR_DLL_DIRS", "")), + ] + result: List[str] = [] + for directory in list(leading_dirs) + configured_dirs + [ + os.path.join(os.environ.get("SystemRoot", r"C:\Windows"), "System32"), + os.path.join(os.environ.get("SystemRoot", r"C:\Windows"), "SysWOW64"), + os.environ.get("SystemRoot", r"C:\Windows"), + *os.environ.get("PATH", "").split(os.pathsep), + ]: + normalized = _norm_path(directory) + if not normalized or normalized.lower() in seen: + continue + seen.add(normalized.lower()) + result.append(normalized) + return result + + +def _find_dll(name: str, search_dirs: List[str]) -> str: + for directory in search_dirs: + candidate = os.path.join(directory, name) + try: + if os.path.isfile(candidate): + return _norm_path(candidate) + except OSError: + continue + return "" + + +def _check_required_dlls(console_path: str, home: str) -> Dict[str, Any]: + console_dir = os.path.dirname(_norm_path(console_path)) + search_dirs = _path_search_dirs(console_dir, home) + missing: List[str] = [] + found: Dict[str, str] = {} + for name in _LANDSAR_REQUIRED_DLLS: + resolved = _find_dll(name, search_dirs) + if resolved: + found[name] = resolved + else: + missing.append(name) + return { + "ok": not missing, + "missing": missing, + "found": found, + "search_dirs": search_dirs, + } + + +def _landsar_process_env(console_path: str, home: str) -> Dict[str, str]: + env = dict(os.environ) + leading_dirs = _path_search_dirs(os.path.dirname(_norm_path(console_path)), home) + existing_path = env.get("PATH", "") + env["PATH"] = os.pathsep.join(leading_dirs + ([existing_path] if existing_path else [])) + return env + + +def _copy_file(src: str, dst: str) -> None: + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copy2(src, dst) + + +def _has_lt1_source_data(directory: str) -> bool: + source_dir = _norm_path(directory) + if not os.path.isdir(source_dir): + return False + for entry in os.scandir(source_dir): + if not entry.is_file(): + continue + lower_name = entry.name.lower() + if lower_name.startswith("lt1") and lower_name.endswith((".xml", ".tif", ".tiff")): + return True + return False + + +def _looks_like_raw_task_dir(task_dir: str) -> bool: + normalized = _norm_path(task_dir) + master_dir = os.path.join(normalized, "master") + slave_dir = os.path.join(normalized, "slave") + return ( + os.path.isdir(master_dir) + and os.path.isdir(slave_dir) + and _has_lt1_source_data(master_dir) + and _has_lt1_source_data(slave_dir) + ) + + +def parse_lt1_slc_pair(input_data_dir: str) -> Optional[Dict[str, Any]]: + """Return the first chronological LT-1 SLC pair from Input_Data.""" + + input_dir = _norm_path(input_data_dir) + if not os.path.isdir(input_dir): + return None + + by_date: Dict[str, Dict[str, str]] = {} + for entry in os.scandir(input_dir): + if not entry.is_file(): + continue + lower_name = entry.name.lower() + if not lower_name.endswith((".xml", ".tif", ".tiff")): + continue + date_text = _extract_date(entry.name) + if not date_text: + continue + payload = by_date.setdefault(date_text, {}) + if lower_name.endswith(".xml") and "xml" not in payload: + payload["xml"] = entry.path + elif lower_name.endswith((".tif", ".tiff")) and "tif" not in payload: + payload["tif"] = entry.path + + scenes = [ + {"date": date_text, **paths} + for date_text, paths in sorted(by_date.items()) + if paths.get("xml") and paths.get("tif") + ] + if len(scenes) < 2: + return None + + master = scenes[0] + slave = scenes[1] + return { + "master_date": master["date"], + "slave_date": slave["date"], + "master_xml": _norm_path(master["xml"]), + "master_tif": _norm_path(master["tif"]), + "slave_xml": _norm_path(slave["xml"]), + "slave_tif": _norm_path(slave["tif"]), + } + + +def _generate_import_param_file( + filepath: str, + *, + master_dir: str, + slave_dir: str, + export_dir: str, + import_method: str = "dir", + sat_mode: str = "BIST", + read_xml: bool = True, + read_slc: bool = True, + export_to_new: bool = True, + master_xml: str = "", + master_slc: str = "", + master_rpb: str = "", + slave_xml: str = "", + slave_slc: str = "", + slave_rpb: str = "", +) -> str: + is_dir_import = import_method == "dir" + lines = [ + "卫星数据导入LT-1", + f"处理 {IMPORT_PROID}", + f"设置数据导入形式_0文件夹导入_1数据导入 {'文件夹导入' if is_dir_import else '数据导入'}", + f"读取成像参数文件_0否_1是 {'1' if read_xml else '0'}", + f"读取SLC数据文件_0否_1是 {'1' if read_slc else '0'}", + f"文件夹导入标识 {'TRUE' if is_dir_import else 'FALSE'}", + "文件夹导入个数 2", + f"文件夹1路径 <{master_dir}>", + f"文件夹2路径 <{slave_dir}>", + f"数据导入 {'FALSE' if is_dir_import else 'TRUE'}", + f"输入卫星数据格式 {sat_mode}", + f"输入主影像成像参数文件路径 <{master_xml}>", + f"输入主影像SLC数据文件路径 <{master_slc}>", + f"输入主影像RPB数据文件路径 <{master_rpb}>", + f"输入辅影像成像参数文件路径 <{slave_xml}>", + f"输入辅影像SLC数据文件路径 <{slave_slc}>", + f"输入辅影像RPB数据文件路径 <{slave_rpb}>", + f"设置数据导出目标路径_0原目录_1新目录 {'1' if export_to_new else '0'}", + f"设置输出文件目录 <{export_dir}>", + ] + target = _norm_path(filepath) + os.makedirs(os.path.dirname(target), exist_ok=True) + with open(target, "w", encoding="utf-8") as fp: + fp.write("\n".join(lines) + "\n") + return target + + +def _generate_dinsar_param_file( + filepath: str, + *, + master_xml: str, + master_tif: str, + slave_xml: str, + slave_tif: str, + dem_path: str, + output_dir: str, + params: Dict[str, Any], +) -> str: + def p(key: str, default: Any) -> Any: + return params.get(key, default) + + lines = [ + "DInSARProcess流程化处理", + f"ID\t {DINSAR_PROID}", + "", + "输入输出数据设置", + f"输入主影像XML文件路径\t <{master_xml}>", + f"输入主影像SLC数据路径\t <{master_tif}>", + f"输入辅影像XML文件路径\t <{slave_xml}>", + f"输入辅影像SLC数据路径\t <{slave_tif}>", + f"输入外部参考DEM文件类型_0文件_1目录\t {p('dem_file_type', 0)}", + f"输入外部参考DEM产品类型_0STRM_1TanDEM\t {p('dem_product_type', 0)}", + f"输入外部参考DEM路径或目录\t <{dem_path}>", + f"输入控制点文件 <{p('gcp_file', '')}>", + f"输入外部GACOS大气相位数据 <{p('gacos_file', '')}>", + f"输出差分干涉处理结果目录 <{output_dir}>", + "", + f"是否处理干涉对配准模块 {p('do_registration', 1)}", + f"配准方法\t {p('reg_method', 0)}", + f"规则格网点数\t {p('reg_grid_points', 64)}", + f"配准窗口\t {p('reg_window', 128)}", + f"信噪比\t {p('reg_snr', 7)}", + "", + f"是否处理辅影像重采样模块 {p('do_resample', 1)}", + f"裁剪无效区域标识\t {p('crop_invalid', 1)}", + f"裁剪主影像对应的GCPs文件\t {p('crop_gcps', 0)}", + "", + f"是否处理干涉条纹图模块 {p('do_interferogram', 1)}", + f"方位向多视\t {p('az_looks', 3)}", + f"距离向多视\t {p('rg_looks', 3)}", + f"是否生成多视强度影像(0表示不生成_1表示生成) {p('gen_intensity', 1)}", + f"是否利用控制点进行几何纠正(0表示不纠正_1表示纠正) {p('gcp_geometric', 0)}", + f"是否对控制点数据进行多视(0表示不多视_1表示多视) {p('gcp_multilook', 0)}", + f"是否生成生成8bit灰度影像(0表示不生成_1表示生成) {p('gen_8bit', 0)}", + "", + f"是否处理去除平地地形相位模块 {p('do_deflatten', 1)}", + f"去除相位方法标识 {p('deflat_method', 0)}", + f"精配准窗口大小 {p('deflat_window', 128)}", + f"采样倍数 {p('deflat_oversample', 3)}", + "", + f"是否处理滤波干涉条纹图模块 {p('do_filter', 1)}", + f"滤波因子 {p('filter_alpha', 0.6)}", + f"滤波次数 {p('filter_iterations', 1)}", + "", + f"是否处理相干性掩膜模块 {p('do_coherence_mask', 1)}", + f"相干性掩膜阈值 {p('coh_mask_threshold', 0.3)}", + "", + f"是否处理相位解缠模块 {p('do_unwrap', 1)}", + f"解缠方法 {p('unwrap_method', 0)}", + f"方位向分块 {p('unwrap_az_blocks', 1)}", + f"距离向分块 {p('unwrap_rg_blocks', 1)}", + f"相干性阈值 {p('unwrap_coh_threshold', 0.3)}", + "", + f"是否处理GCP提取模块 {p('do_gcp_extract', 1)}", + f"规则格网点数 {p('gcp_grid_points', 25)}", + f"在窗口内筛选相对平坦区域 {p('gcp_flat_area', 10)}", + f"窗口内高差设置 {p('gcp_height_diff', 10)}", + f"窗口内相干性差异设置 {p('gcp_coh_diff', 0.1)}", + "", + f"是否处理基线精估计模块 {p('do_baseline_refine', 1)}", + f"基线精估计方法 {p('baseline_method', 0)}", + f"相位校正标识 {p('phase_correction', 0)}", + f"基线精估计相干性阈值 {p('baseline_coh_threshold', 0.9)}", + f"优选控制点标识 {p('optimize_gcps', 0)}", + f"优选控制点误差阈值 {p('gcp_error_threshold', 10)}", + "", + f"是否处理差分干涉相位拟合模块 {p('do_diff_fitting', 1)}", + f"相干阈值 \t\t\t\t{p('diff_coh_threshold', 0.91)}", + f"方位向采样点数 {p('diff_az_samples', 64)}", + f"距离向采样点数\t {p('diff_rg_samples', 64)}", + "", + f"是否处理相位转LOS向形变模块 {p('do_los_displacement', 1)}", + f"形变图产品形式 {p('displacement_format', 0)}", + "", + f"是否处理LOS向形变转垂直向形变模块 {p('do_vertical_displacement', 0)}", + "", + f"是否处理大气相位改正模块 {p('do_atmosphere', 0)}", + "", + f"是否处理形变结果校正模块 {p('do_displacement_correction', 0)}", + "", + f"是否处理地理编码模块 {p('do_geocoding', 1)}", + f"是否编码滤波后缠绕数据(差分干涉相位拟合) {p('geo_wrapped', 1)}", + f"是否编码滤波后解缠数据(差分干涉相位拟合) {p('geo_unwrapped', 1)}", + f"是否编码滤波后相干系数数据 {p('geo_coherence', 1)}", + f"是否编码视线向形变场数据 {p('geo_los', 1)}", + f"是否编码垂直向形变场数据 {p('geo_vertical', 1)}", + ] + + target = _norm_path(filepath) + os.makedirs(os.path.dirname(target), exist_ok=True) + with open(target, "w", encoding="utf-8") as fp: + fp.write("\n".join(lines) + "\n") + return target + + +class LandsarEngine(DinsarEngine): @property def engine_code(self) -> str: return "landsar" @property def engine_label(self) -> str: - return "LANDSAR（预留）" + return "LandSAR" + + @property + def default_timeout_seconds(self) -> int: + return max(60, int(getattr(settings, "LANDSAR_DINSAR_TIMEOUT_SECONDS", 0) or _read_env("LANDSAR_DINSAR_TIMEOUT_SECONDS", "43200") or 43200)) + + @property + def _enabled(self) -> bool: + return read_bool_env("LANDSAR_ENABLED", True) + + @property + def _default_home(self) -> str: + return str(_PROJECT_ROOT / "third_party" / "LandSAR" / "dist" / "LandSAR_Portable") + + @property + def _home(self) -> str: + return _norm_path(_read_env("LANDSAR_HOME", self._default_home)) + + @property + def _console_exe(self) -> str: + explicit = _read_env("LANDSAR_CONSOLE_EXE", "") + if explicit: + return _norm_path(explicit) + home = self._home + candidates = [ + Path(home) / "InSAR_Console.exe", + _PROJECT_ROOT / "third_party" / "LandSAR" / "dist" / "LandSAR_Portable" / "InSAR_Console.exe", + _PROJECT_ROOT / "third_party" / "LandSAR" / "dist" / "InSAR_Console.exe", + ] + found = _find_first_existing(*candidates) + return _norm_path(found or str(candidates[0])) + + @property + def _config_csv(self) -> str: + return _norm_path(os.path.join(self._home, "config", "config.csv")) + + @property + def _version_control_exe(self) -> str: + return _norm_path(os.path.join(self._home, "versionControl.exe")) + + @property + def _license_mode(self) -> str: + return _read_env("LANDSAR_LICENSE_MODE", str(getattr(settings, "LANDSAR_LICENSE_MODE", "netVersion") or "netVersion")) + + @property + def _license_host(self) -> str: + return _read_env("LANDSAR_LICENSE_HOST", str(getattr(settings, "LANDSAR_LICENSE_HOST", "127.0.0.1") or "127.0.0.1")) + + @property + def _license_port(self) -> int: + return _read_int_env("LANDSAR_LICENSE_PORT", int(getattr(settings, "LANDSAR_LICENSE_PORT", 6666) or 6666)) + + @property + def _config_auto_write(self) -> bool: + return read_bool_env("LANDSAR_CONFIG_AUTO_WRITE", bool(getattr(settings, "LANDSAR_CONFIG_AUTO_WRITE", True))) + + @property + def _auth_server_exe(self) -> str: + return _norm_path(_read_env("LANDSAR_AUTH_SERVER_EXE", str(getattr(settings, "LANDSAR_AUTH_SERVER_EXE", "") or ""))) + + @property + def _auth_server_auto_start(self) -> bool: + return read_bool_env("LANDSAR_AUTH_SERVER_AUTO_START", bool(getattr(settings, "LANDSAR_AUTH_SERVER_AUTO_START", True))) + + @property + def _auth_server_host(self) -> str: + return _read_env("LANDSAR_AUTH_SERVER_HOST", str(getattr(settings, "LANDSAR_AUTH_SERVER_HOST", "127.0.0.1") or "127.0.0.1")) + + @property + def _auth_server_port(self) -> int: + return _read_int_env("LANDSAR_AUTH_SERVER_PORT", int(getattr(settings, "LANDSAR_AUTH_SERVER_PORT", 6666) or 6666)) + + @property + def _expected_config_row(self) -> str: + explicit = _read_env("LANDSAR_CONFIG_ROW", str(getattr(settings, "LANDSAR_CONFIG_ROW", "") or "")) + if explicit: + return explicit + mode = self._license_mode or "netVersion" + host = self._license_host or "127.0.0.1" + port = self._license_port + return f"{mode},zh,{host},{port}" + + def _expected_config_parts(self) -> List[str]: + return [part.strip() for part in self._expected_config_row.split(",")] + + def _expected_config_mode(self) -> str: + parts = self._expected_config_parts() + return parts[0] if parts else self._license_mode + + def _expected_config_host(self) -> str: + parts = self._expected_config_parts() + if len(parts) >= 3 and parts[2]: + return parts[2] + return self._license_host or "127.0.0.1" + + def _expected_config_port(self) -> int: + parts = self._expected_config_parts() + if len(parts) >= 4: + try: + return int(parts[3]) + except (TypeError, ValueError): + pass + return int(self._license_port or 6666) + + @property + def _default_dem_path(self) -> str: + landsar_dem = _read_env("LANDSAR_DEM_PATH", "") + if landsar_dem: + return _norm_path(landsar_dem) + pyint_dem = _read_env("PYINT_PREPARED_DEM_PATH", "") + if pyint_dem: + return _norm_path(pyint_dem) + isce2_dem = _read_env("ISCE2_DEM_PATH", "") + if isce2_dem: + return _norm_path(isce2_dem) + return _norm_path(getattr(settings, "IDL_DINSAR_DEM_BASE_FILE", "") or "") + + def _read_config_rows(self) -> List[str]: + path = self._config_csv + try: + with open(path, "r", encoding="utf-8-sig", errors="ignore") as fp: + return [line.strip() for line in fp.read().splitlines() if line.strip()] + except OSError: + return [] + + def _config_matches_expected(self) -> bool: + rows = self._read_config_rows() + if len(rows) < 2: + return False + return rows[1].strip().lower() == self._expected_config_row.lower() + + def _ensure_config_csv(self) -> tuple[bool, str]: + path = self._config_csv + if self._config_matches_expected(): + return True, f"ok: {path}" + if not self._config_auto_write: + if os.path.isfile(path): + return False, f"LandSAR config.csv is not set to expected license row: {path}" + return False, f"LandSAR config.csv is missing: {path}" + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8", newline="\n") as fp: + fp.write("version,language,Address,port\n") + fp.write(self._expected_config_row + "\n") + return True, f"wrote: {path}" + except OSError as exc: + return False, f"failed to write {path}: {exc}" + + def _is_network_license_mode(self) -> bool: + return self._expected_config_mode().strip().lower() == "netversion" + + def _start_auth_server_if_needed(self) -> tuple[bool, str]: + if not self._is_network_license_mode(): + return True, "not required for standalone license mode" + + client_host = self._expected_config_host() + client_port = int(self._expected_config_port()) + bind_host = self._auth_server_host or client_host + bind_port = int(self._auth_server_port or client_port) + if _tcp_connect(client_host, client_port): + return True, f"listening on {client_host}:{client_port}" + + exe = self._auth_server_exe + if not self._auth_server_auto_start: + return False, f"not listening on {client_host}:{client_port}; auto-start disabled" + if not exe or not os.path.isfile(exe): + return False, f"auth server executable missing: {exe or '<empty>'}" + + server_dir = os.path.dirname(exe) + memory_bin = os.path.join(server_dir, "dongle_0xa0.bin") + if not os.path.isfile(memory_bin): + fallback_bin = str(_PROJECT_ROOT / "third_party" / "LandSAR" / "tools" / "dongle_0xa0.bin") + if os.path.isfile(fallback_bin): + try: + shutil.copy2(fallback_bin, memory_bin) + except OSError as exc: + return False, f"failed to copy LandSAR authorization block to auth server dir: {exc}" + else: + return False, f"auth server memory image missing: {memory_bin}" + + command = [exe, "--host", bind_host, "--port", str(bind_port)] + try: + subprocess.Popen( + command, + cwd=server_dir, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + stdin=subprocess.DEVNULL, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + except OSError as exc: + return False, f"failed to start auth server: {exc}" + + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + if _tcp_connect(client_host, client_port): + return True, f"started: {exe} on {bind_host}:{bind_port}; client={client_host}:{client_port}" + time.sleep(0.25) + return False, f"started auth server but client port is not reachable: {client_host}:{client_port}" def get_profiles(self) -> List[EngineProfile]: + params_schema = { + "dem_path": { + "label": "DEM 文件", + "type": "string", + "default": self._default_dem_path, + "section": "输入数据", + "description": "传给 LandSAR 200014 D-InSAR 模块的外部参考 DEM，建议使用已验证可用的 GeoTIFF。", + }, + "az_looks": { + "label": "方位向多视数", + "type": "number", + "default": _DEFAULT_PARAM_VALUES["az_looks"], + "step": 1, + "min": 1, + "section": "核心参数", + }, + "rg_looks": { + "label": "距离向多视数", + "type": "number", + "default": _DEFAULT_PARAM_VALUES["rg_looks"], + "step": 1, + "min": 1, + "section": "核心参数", + }, + "coh_mask_threshold": { + "label": "相干掩膜阈值", + "type": "number", + "default": _DEFAULT_PARAM_VALUES["coh_mask_threshold"], + "step": 0.01, + "min": 0, + "max": 1, + "section": "核心参数", + }, + "unwrap_coh_threshold": { + "label": "解缠相干阈值", + "type": "number", + "default": _DEFAULT_PARAM_VALUES["unwrap_coh_threshold"], + "step": 0.01, + "min": 0, + "max": 1, + "section": "核心参数", + }, + "filter_alpha": { + "label": "Goldstein 滤波因子", + "type": "number", + "default": _DEFAULT_PARAM_VALUES["filter_alpha"], + "step": 0.05, + "min": 0, + "max": 1, + "section": "核心参数", + }, + "do_vertical_displacement": { + "label": "生成垂直向形变", + "type": "boolean", + "default": False, + "section": "输出与改正", + "description": "可选模块，将 LOS 向形变换算为垂直向形变；LOS 形变仍会保留。", + }, + "do_atmosphere": { + "label": "GACOS 大气相位改正", + "type": "boolean", + "default": False, + "section": "输出与改正", + "readonly": True, + "readonly_label": "暂未开放", + "include_in_payload": False, + "description": "暂未开放。该模块需要外部 GACOS 大气延迟文件，LandSAR 不会自动生成该文件。", + }, + "do_geocoding": { + "label": "地理编码输出", + "type": "boolean", + "default": True, + "section": "输出与改正", + "readonly": True, + "readonly_label": "固定开启", + "include_in_payload": False, + "description": "系统入库和地图展示依赖地理编码 GeoTIFF，因此固定开启。", + }, + } return [ EngineProfile( - code="standard", - label="标准链路（未实现）", - description="LANDSAR 处理链路，本版本暂未实现", + code="lt1_dinsar", + label="LT-1 LandSAR D-InSAR", + description="Run LandSAR import 100016 when needed, then proID 200014 D-InSAR.", + params_schema=params_schema, ), ] + def normalize_extra(self, extra: Optional[Dict[str, Any]]) -> Dict[str, Any]: + normalized: Dict[str, Any] = {} + for key, value in dict(extra or {}).items(): + if key in _SYSTEM_EXTRA_KEYS: + normalized[key] = value + continue + if value is None: + continue + if isinstance(value, str) and value.strip() == "": + continue + normalized[key] = value + + for key in set(_DEFAULT_PARAM_VALUES) | {"dem_path"}: + if key not in normalized: + continue + value = normalized[key] + if key in _PATH_PARAM_KEYS: + path = _norm_path(value) + if key in {"gcp_file", "gacos_file"} and not path: + normalized.pop(key, None) + continue + normalized[key] = path + continue + if key in _INT_PARAM_KEYS: + try: + parsed = int(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{key} must be an integer.") from exc + normalized[key] = parsed + continue + if key in _FLOAT_PARAM_KEYS: + try: + parsed_f = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{key} must be a number.") from exc + if key.endswith("threshold") and (parsed_f < 0 or parsed_f > 1): + raise ValueError(f"{key} must be between 0 and 1.") + normalized[key] = parsed_f + + for bool_key in ("do_vertical_displacement", "do_atmosphere", "do_geocoding"): + if bool_key in normalized: + normalized[bool_key] = 1 if _coerce_bool(normalized[bool_key]) else 0 + + return normalized + + def _iter_candidate_task_dirs(self, root_dir: str) -> Iterable[str]: + root = _norm_path(root_dir) + if os.path.isdir(os.path.join(root, "Input_Data")) or _looks_like_raw_task_dir(root): + yield root + return + if not os.path.isdir(root): + return + for entry in sorted(os.scandir(root), key=lambda item: item.name.lower()): + if entry.is_dir() and entry.name.lower().startswith("task_"): + yield _norm_path(entry.path) + + def _has_completed_task_result(self, task_dir: str) -> bool: + output_dir = os.path.join(_norm_path(task_dir), "Output_Data") + return self._is_completed_output(output_dir) + + def _is_completed_output(self, output_dir: str) -> bool: + if not output_dir or not os.path.isdir(output_dir): + return False + has_geo = bool(self._select_primary_file(output_dir)) + if not has_geo: + return False + + log_candidates = [ + os.path.join(output_dir, f"{DINSAR_PROID}.log"), + os.path.join(output_dir, f"{DINSAR_PROID}_console.log"), + ] + log_candidates.extend(str(path) for path in Path(output_dir).glob(f"*{DINSAR_PROID}*.log")) + for log_path in log_candidates: + if not os.path.isfile(log_path): + continue + try: + with open(log_path, "r", encoding="utf-8", errors="ignore") as fp: + content = fp.read() + if "console success" in content.lower(): + return True + if _SUCCESS_RE.search(content) and ("DInSAR" in content or "差分" in content or "module" in content.lower()): + return True + except OSError: + continue + return False + + def validate_root_dir( + self, + root_dir: str, + num_to_process: int = 0, + rerun_mode: str = "rerun_all", + ) -> Dict[str, Any]: + normalized_root = _norm_path(root_dir) + if not normalized_root or not os.path.isdir(normalized_root): + raise ValueError(f"LandSAR root_dir does not exist or is not a directory: {root_dir}") + + candidates = list(self._iter_candidate_task_dirs(normalized_root)) + invalid_candidates: List[Dict[str, Any]] = [] + valid_task_dirs: List[str] = [] + for task_dir in candidates: + input_dir = os.path.join(task_dir, "Input_Data") + raw_ready = _looks_like_raw_task_dir(task_dir) + if not os.path.isdir(input_dir): + if raw_ready: + valid_task_dirs.append(task_dir) + continue + invalid_candidates.append( + { + "name": os.path.basename(task_dir), + "path": task_dir, + "reason": "missing Input_Data and no valid master/slave raw LT-1 data", + } + ) + continue + pair = parse_lt1_slc_pair(input_dir) + if pair is None: + if raw_ready: + valid_task_dirs.append(task_dir) + continue + invalid_candidates.append( + { + "name": os.path.basename(task_dir), + "path": task_dir, + "reason": "less than two valid Input_Data SLC xml/tif pairs and no valid master/slave raw LT-1 data", + } + ) + continue + valid_task_dirs.append(task_dir) + + if not valid_task_dirs: + detail = "" + if invalid_candidates: + formatted = ", ".join(f"{item['name']} {item['reason']}" for item in invalid_candidates[:5]) + detail = f" Invalid candidates: {formatted}." + raise ValueError( + "LandSAR root_dir must be either one Task_* directory containing Input_Data, " + "one Task_* directory containing master/slave raw LT-1 folders, " + "or a parent directory containing either layout." + f"{detail}" + ) + + discovered_task_count = len(valid_task_dirs) + selected_dirs: List[str] = [] + skipped_completed_count = 0 + if str(rerun_mode or "").strip().lower() == RERUN_MODE_UNFINISHED_ONLY: + for task_dir in valid_task_dirs: + if self._has_completed_task_result(task_dir): + skipped_completed_count += 1 + continue + selected_dirs.append(task_dir) + else: + selected_dirs = list(valid_task_dirs) + + limit = int(num_to_process or 0) + if limit > 0: + selected_dirs = selected_dirs[:limit] + + mode = ( + "single_task_dir" + if os.path.isdir(os.path.join(normalized_root, "Input_Data")) or _looks_like_raw_task_dir(normalized_root) + else "task_root_dir" + ) + return { + "root_dir": normalized_root, + "mode": mode, + "task_dirs": selected_dirs, + "task_count": len(selected_dirs), + "selected_task_count": len(selected_dirs), + "discovered_task_count": discovered_task_count, + "skipped_completed_count": skipped_completed_count, + "invalid_candidates": invalid_candidates, + } + def check_available(self) -> EngineAvailability: + console_path = self._console_exe + home = self._home + config_ok, config_detail = self._ensure_config_csv() if os.path.isdir(home) else (False, f"LANDSAR_HOME missing: {home}") + auth_ok, auth_detail = self._start_auth_server_if_needed() if config_ok else (False, "skipped until config.csv is available") + version_control_exe = self._version_control_exe + enabled = self._enabled + dll_check = _check_required_dlls(console_path, home) if os.path.isfile(console_path) else { + "ok": False, + "missing": list(_LANDSAR_REQUIRED_DLLS), + "found": {}, + "search_dirs": _path_search_dirs(home), + } + checks = [ + {"name": "LANDSAR_ENABLED", "ok": enabled, "detail": str(enabled).lower()}, + {"name": "LANDSAR_HOME", "ok": os.path.isdir(home), "detail": home}, + {"name": "InSAR_Console.exe", "ok": os.path.isfile(console_path), "detail": console_path}, + { + "name": "LandSAR runtime DLLs", + "ok": bool(dll_check["ok"]), + "detail": "ok" if dll_check["ok"] else f"missing: {', '.join(dll_check['missing'])}", + }, + { + "name": "LandSAR config.csv", + "ok": config_ok, + "detail": config_detail, + }, + { + "name": "LandSAR license mode", + "ok": True, + "detail": self._expected_config_row, + "optional": True, + }, + { + "name": "LandSAR auth server", + "ok": auth_ok, + "detail": auth_detail, + "optional": not self._is_network_license_mode(), + }, + { + "name": "versionControl.exe", + "ok": os.path.isfile(version_control_exe), + "detail": version_control_exe, + "optional": True, + }, + { + "name": "LANDSAR_DEM_PATH", + "ok": True, + "detail": self._default_dem_path or "set per run", + "optional": True, + }, + ] + available = enabled and os.path.isfile(console_path) and bool(dll_check["ok"]) and config_ok and auth_ok + status = "ok" if available else "unavailable" + if available and not os.path.isdir(home): + status = "degraded" + if available: + message = "LandSAR console is available." + elif os.path.isfile(console_path) and not dll_check["ok"]: + message = f"LandSAR console dependencies are missing: {', '.join(dll_check['missing'])}" + elif not config_ok: + message = config_detail + elif not auth_ok: + message = auth_detail + else: + message = "LandSAR console is not configured." return EngineAvailability( engine_code=self.engine_code, - status="not_implemented", - available=False, - checks=[], - message="LANDSAR 引擎尚未实现，仅作接口预留", + status=status, + available=available, + checks=checks, + message=message, ) def run(self, request: RunRequest) -> RunResult: + if not self._enabled: + return RunResult( + success=False, + engine_code=self.engine_code, + profile=request.profile, + job_id=request.job_id, + error="LandSAR is disabled.", + ) + if request.profile not in SUPPORTED_PROFILES: + return RunResult( + success=False, + engine_code=self.engine_code, + profile=request.profile, + job_id=request.job_id, + error=f"Unknown LandSAR profile: {request.profile}", + ) + + extra = self.normalize_extra(request.extra) + dem_path = _norm_path(extra.get("dem_path") or self._default_dem_path) + if not dem_path or not os.path.isfile(dem_path): + return RunResult( + success=False, + engine_code=self.engine_code, + profile=request.profile, + job_id=request.job_id, + error=f"LandSAR DEM file is missing: {dem_path or '<empty>'}", + ) + if _coerce_bool(extra.get("do_atmosphere")): + gacos_path = _norm_path(extra.get("gacos_file")) + if not gacos_path or not os.path.isfile(gacos_path): + return RunResult( + success=False, + engine_code=self.engine_code, + profile=request.profile, + job_id=request.job_id, + error="LandSAR GACOS atmospheric correction requires a valid external GACOS file.", + ) + + console_path = self._console_exe + if not os.path.isfile(console_path): + return RunResult( + success=False, + engine_code=self.engine_code, + profile=request.profile, + job_id=request.job_id, + error=f"InSAR_Console.exe not found: {console_path}", + ) + config_ok, config_detail = self._ensure_config_csv() + if not config_ok: + return RunResult( + success=False, + engine_code=self.engine_code, + profile=request.profile, + job_id=request.job_id, + error=f"LandSAR config.csv is not ready: {config_detail}", + ) + auth_ok, auth_detail = self._start_auth_server_if_needed() + if not auth_ok: + return RunResult( + success=False, + engine_code=self.engine_code, + profile=request.profile, + job_id=request.job_id, + error=f"LandSAR network license server is not ready: {auth_detail}", + ) + + validation = self.validate_root_dir( + request.root_dir, + request.num_to_process, + str(extra.get("__rerun_mode") or "rerun_all"), + ) + task_dirs: List[str] = list(validation["task_dirs"]) + run_started_at = datetime.utcnow() + run_started_at_text = _utc_text(run_started_at) + run_key_override = str(extra.get("__managed_run_key") or "").strip() + run_key = run_key_override or f"run_{run_started_at.strftime('%Y%m%dT%H%M%SZ')}_{self.engine_code}_{request.profile}" + timeout = max(60, int(request.timeout_seconds or self.default_timeout_seconds)) + output_dirs: List[str] = [] + task_results: List[Dict[str, Any]] = [] + pairs_processed = 0 + pairs_failed = 0 + + progress_callback = request.progress_callback + + def emit_progress(event_type: str, **payload: Any) -> None: + if not callable(progress_callback): + return + try: + progress_callback({"event": event_type, **payload}) + except Exception: + return + + for pair_index, task_dir in enumerate(task_dirs, start=1): + task_name = os.path.basename(task_dir) + initial_input_data_dir = os.path.join(task_dir, "Input_Data") + parsed_pair = parse_lt1_slc_pair(initial_input_data_dir) + task_alias, pair_key, pair_meta = self._resolve_task_identity(task_dir, task_name, parsed_pair) + + managed_run_dir = _norm_path(extra.get("__managed_run_dir")) + if managed_run_dir: + run_dir = managed_run_dir + else: + pair_root = os.path.join(settings.DINSAR_PRODUCT_DIR, pair_key, "runs") + run_dir = os.path.join(pair_root, run_key) + native_output_dir = _norm_path(extra.get("__managed_native_output_dir")) or os.path.join(run_dir, "native") + landsar_input_dir = os.path.join(native_output_dir, "landsar_input") + landsar_output_dir = os.path.join(native_output_dir, "landsar_output") + os.makedirs(landsar_output_dir, exist_ok=True) + + command = [console_path, os.path.join(landsar_output_dir, f"{DINSAR_PROID}.txt")] + emit_progress( + "pair_started", + pair_index=pair_index, + pair_total=len(task_dirs), + task_name=task_name, + task_alias=task_alias, + pair_key=pair_key, + ) + + if not parsed_pair: + import_result = self._ensure_imported_input_data( + task_dir=task_dir, + task_name=task_name, + task_alias=task_alias, + pair_key=pair_key, + export_dir=landsar_input_dir, + console_path=console_path, + timeout=timeout, + pair_index=pair_index, + pair_total=len(task_dirs), + emit_progress=emit_progress, + ) + if not import_result.get("success"): + pairs_failed += 1 + error_text = str(import_result.get("error") or "LandSAR import failed.") + emit_progress("pair_finished", pair_index=pair_index, pair_total=len(task_dirs), success=False, error=error_text) + task_results.append( + self._build_task_result( + task_name=task_name, + task_alias=task_alias, + pair_key=pair_key, + run_key=run_key, + task_dir=task_dir, + run_dir=run_dir, + native_output_dir=native_output_dir, + landsar_output_dir=landsar_output_dir, + command=import_result.get("command") or command, + success=False, + returncode=int(import_result.get("returncode") or -2), + error=error_text, + stdout_tail=str(import_result.get("stdout_tail") or ""), + param_file=str(import_result.get("param_file") or ""), + ) + ) + continue + parsed_pair = parse_lt1_slc_pair(str(import_result.get("input_data_dir") or landsar_input_dir)) + if not parsed_pair: + pairs_failed += 1 + error_text = "LandSAR import completed but no valid Input_Data xml/tif pair was produced." + emit_progress("pair_finished", pair_index=pair_index, pair_total=len(task_dirs), success=False, error=error_text) + task_results.append( + self._build_task_result( + task_name=task_name, + task_alias=task_alias, + pair_key=pair_key, + run_key=run_key, + task_dir=task_dir, + run_dir=run_dir, + native_output_dir=native_output_dir, + landsar_output_dir=landsar_output_dir, + command=import_result.get("command") or command, + success=False, + returncode=-2, + error=error_text, + stdout_tail=str(import_result.get("stdout_tail") or ""), + param_file=str(import_result.get("param_file") or ""), + ) + ) + continue + task_alias, pair_key, pair_meta = self._resolve_task_identity(task_dir, task_name, parsed_pair) + + param_values = {**_DEFAULT_PARAM_VALUES} + param_values.update({key: value for key, value in extra.items() if key in _DEFAULT_PARAM_VALUES}) + param_file = _generate_dinsar_param_file( + command[1], + master_xml=parsed_pair["master_xml"], + master_tif=parsed_pair["master_tif"], + slave_xml=parsed_pair["slave_xml"], + slave_tif=parsed_pair["slave_tif"], + dem_path=dem_path, + output_dir=landsar_output_dir, + params=param_values, + ) + command = [console_path, param_file] + + emit_progress( + "log", + pair_index=pair_index, + pair_total=len(task_dirs), + task_name=task_name, + task_alias=task_alias, + pair_key=pair_key, + level="INFO", + source="input", + message=f"master={os.path.basename(parsed_pair['master_xml'])}, slave={os.path.basename(parsed_pair['slave_xml'])}", + ) + emit_progress( + "log", + pair_index=pair_index, + pair_total=len(task_dirs), + task_name=task_name, + task_alias=task_alias, + pair_key=pair_key, + level="INFO", + source="dem", + message=dem_path, + ) + + rc, stdout_text, timed_out = self._run_console( + command, + cwd=self._home if os.path.isdir(self._home) else os.path.dirname(console_path), + log_path=os.path.join(landsar_output_dir, f"{DINSAR_PROID}_console.log"), + timeout=timeout, + emit_log=lambda level, message: emit_progress( + "log", + pair_index=pair_index, + pair_total=len(task_dirs), + task_name=task_name, + task_alias=task_alias, + pair_key=pair_key, + level=level, + source="console", + message=message, + ), + ) + + success_marker = self._is_completed_output(landsar_output_dir) + primary_raw = self._select_primary_file(landsar_output_dir) + coherence_raw = self._select_coherence_file(landsar_output_dir) + success = rc == 0 and success_marker and bool(primary_raw) + error_text = "" + primary_file = "" + source_files: List[str] = [] + validation_result: Dict[str, Any] = {} + layout_result: Dict[str, Any] = {} + if timed_out: + error_text = f"LandSAR console timed out after {timeout}s." + success = False + elif rc != 0: + error_text = _summarize_landsar_failure(stdout_text, "LandSAR console", rc) + elif not success_marker: + error_text = "LandSAR success marker or geocoded output is missing." + elif not primary_raw: + error_text = "LandSAR primary displacement GeoTIFF is missing." + + if success: + try: + raw_sources = [primary_raw] + if coherence_raw: + raw_sources.append(coherence_raw) + validation_result = validate_isce2_result_files(primary_raw, raw_sources) + if not bool(validation_result.get("accepted")): + issues = validation_result.get("issues") or [] + raise RuntimeError("; ".join(str(item) for item in issues[:3]) or "GeoTIFF validation failed.") + + os.makedirs(run_dir, exist_ok=True) + standard_disp = os.path.join(run_dir, "assets", "disp", "disp.tif") + standard_coh = os.path.join(run_dir, "assets", "coh", "coh.tif") + _copy_file(primary_raw, standard_disp) + if coherence_raw: + _copy_file(coherence_raw, standard_coh) + source_files = [standard_disp] + if os.path.isfile(standard_coh): + source_files.append(standard_coh) + layout_result = normalize_isce2_run_layout( + run_dir, + primary_file=standard_disp, + source_files=source_files, + rewrite_metadata=False, + ) + primary_file = layout_result["primary_file"] + source_files = list(layout_result["source_files"]) + + pair_meta_payload = self._build_pair_meta_payload( + pair_key=pair_key, + task_alias=task_alias, + pair_meta=pair_meta, + parsed_pair=parsed_pair, + ) + write_pair_metadata(task_dir, pair_meta_payload) + self._write_run_metadata( + run_dir=run_dir, + native_output_dir=native_output_dir, + task_dir=task_dir, + task_name=task_name, + task_alias=task_alias, + pair_key=pair_key, + run_key=run_key, + request=request, + pair_meta=pair_meta_payload, + params=param_values, + dem_path=dem_path, + landsar_output_dir=landsar_output_dir, + primary_file=primary_file, + source_files=source_files, + returncode=rc, + started_at=run_started_at_text, + ) + output_dirs.append(run_dir) + pairs_processed += 1 + except Exception as exc: + success = False + error_text = f"LandSAR output packaging failed: {exc}" + + if not success: + pairs_failed += 1 + + emit_progress( + "pair_finished", + pair_index=pair_index, + pair_total=len(task_dirs), + task_name=task_name, + task_alias=task_alias, + pair_key=pair_key, + success=success, + returncode=rc, + error=error_text, + ) + task_results.append( + self._build_task_result( + task_name=task_name, + task_alias=task_alias, + pair_key=pair_key, + run_key=run_key, + task_dir=task_dir, + run_dir=run_dir, + native_output_dir=native_output_dir, + landsar_output_dir=landsar_output_dir, + command=command, + success=success, + returncode=rc, + error=error_text, + stdout_tail=_collect_tail(stdout_text, 3000), + primary_file=primary_file, + source_files=source_files, + validation=validation_result, + layout=layout_result, + param_file=param_file, + raw_primary_file=primary_raw, + raw_coherence_file=coherence_raw, + ) + ) + + invalid_candidates = validation.get("invalid_candidates", []) + pairs_failed += len(invalid_candidates) + overall_success = pairs_processed > 0 and pairs_failed == 0 + if pairs_processed > 0 and pairs_failed > 0: + overall_success = False + error = None + if not overall_success: + failed_names = [item.get("task_alias") or item.get("task_name") for item in task_results if not item.get("success")] + error = f"LandSAR run failed: {', '.join(failed_names[:10])}" if failed_names else "LandSAR run failed." + return RunResult( - success=False, + success=overall_success, engine_code=self.engine_code, profile=request.profile, job_id=request.job_id, - error="LANDSAR 引擎尚未实现", + pairs_processed=pairs_processed, + pairs_failed=pairs_failed, + output_dirs=output_dirs, + error=error, + detail={ + "mode": validation["mode"], + "task_count": len(task_dirs), + "selected_tasks": [item.get("task_alias") or item.get("task_name") for item in task_results], + "invalid_candidates": invalid_candidates, + "task_results": task_results, + "run_key": run_key, + "started_at": run_started_at_text, + "timeout_seconds": timeout, + "dem_path": dem_path, + "console_path": console_path, + }, ) + + def _run_console( + self, + command: Any, + *, + cwd: str, + log_path: str, + timeout: int, + emit_log, + ) -> tuple[int, str, bool]: + os.makedirs(os.path.dirname(log_path), exist_ok=True) + with open(log_path, "a", encoding="utf-8", errors="replace") as log_fp: + log_fp.write(f"\n[{_utc_text()}] command: {' '.join(command)}\n") + process = subprocess.Popen( + command, + cwd=cwd, + env=_landsar_process_env(command[0], self._home), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + stdin=subprocess.DEVNULL, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + output_lines: List[str] = [] + line_queue: queue.Queue[Any] = queue.Queue() + sentinel = object() + + def _reader() -> None: + try: + if process.stdout is None: + return + for raw_line in iter(process.stdout.readline, b""): + line_queue.put(raw_line) + finally: + line_queue.put(sentinel) + + reader = threading.Thread(target=_reader, name="landsar-console-reader", daemon=True) + reader.start() + started = time.monotonic() + timed_out = False + stdout_closed = False + while True: + try: + raw_item = line_queue.get(timeout=1) + except queue.Empty: + raw_item = None + + if raw_item is sentinel: + stdout_closed = True + elif raw_item: + line = _decode_line(raw_item) + output_lines.append(line) + log_fp.write(line + "\n") + log_fp.flush() + if line.strip(): + emit_log("INFO", line.strip()) + + if process.poll() is not None and stdout_closed: + break + if time.monotonic() - started > timeout: + timed_out = True + process.kill() + break + + if timed_out: + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + pass + reader.join(timeout=5) + while True: + try: + raw_item = line_queue.get_nowait() + except queue.Empty: + break + if raw_item is sentinel or not raw_item: + continue + line = _decode_line(raw_item) + output_lines.append(line) + log_fp.write(line + "\n") + if line.strip(): + emit_log("INFO", line.strip()) + if process.poll() is None: + timed_out = True + process.kill() + return_code = int(process.poll() if process.poll() is not None else -9) + if timed_out: + log_fp.write(f"[{_utc_text()}] timeout after {timeout}s\n") + log_fp.write(f"[{_utc_text()}] returncode={return_code}\n") + return return_code, "\n".join(output_lines), timed_out + + def _ensure_imported_input_data( + self, + *, + task_dir: str, + task_name: str, + task_alias: str, + pair_key: str, + export_dir: str, + console_path: str, + timeout: int, + pair_index: int, + pair_total: int, + emit_progress, + ) -> Dict[str, Any]: + master_dir = os.path.join(_norm_path(task_dir), "master") + slave_dir = os.path.join(_norm_path(task_dir), "slave") + export_dir = _norm_path(export_dir) + os.makedirs(export_dir, exist_ok=True) + + existing_pair = parse_lt1_slc_pair(export_dir) + if existing_pair: + return { + "success": True, + "input_data_dir": export_dir, + "skipped": True, + "returncode": 0, + "command": "", + "param_file": "", + "stdout_tail": "", + } + + if not _looks_like_raw_task_dir(task_dir): + return { + "success": False, + "input_data_dir": export_dir, + "returncode": -2, + "error": "Task directory has neither valid Input_Data nor valid master/slave raw LT-1 folders.", + } + + param_file = _generate_import_param_file( + os.path.join(export_dir, f"{IMPORT_PROID}.txt"), + master_dir=master_dir, + slave_dir=slave_dir, + export_dir=export_dir, + import_method="dir", + sat_mode="BIST", + read_xml=True, + read_slc=True, + export_to_new=True, + ) + command = [console_path, param_file] + emit_progress( + "log", + pair_index=pair_index, + pair_total=pair_total, + task_name=task_name, + task_alias=task_alias, + pair_key=pair_key, + level="INFO", + source="import", + message=f"LandSAR import 100016 -> {export_dir}", + ) + rc, stdout_text, timed_out = self._run_console( + command, + cwd=self._home if os.path.isdir(self._home) else os.path.dirname(console_path), + log_path=os.path.join(export_dir, f"{IMPORT_PROID}_console.log"), + timeout=timeout, + emit_log=lambda level, message: emit_progress( + "log", + pair_index=pair_index, + pair_total=pair_total, + task_name=task_name, + task_alias=task_alias, + pair_key=pair_key, + level=level, + source="import", + message=message, + ), + ) + + success = rc == 0 and self._is_completed_import(export_dir) and bool(parse_lt1_slc_pair(export_dir)) + error = "" + if timed_out: + error = f"LandSAR import timed out after {timeout}s." + success = False + elif rc != 0: + error = _summarize_landsar_failure(stdout_text, "LandSAR import", rc) + elif not self._is_completed_import(export_dir): + error = "LandSAR import success marker is missing." + elif not parse_lt1_slc_pair(export_dir): + error = "LandSAR import did not produce at least two LT1*_SLC xml/tif pairs." + + return { + "success": success, + "input_data_dir": export_dir, + "returncode": rc, + "error": error, + "command": " ".join(command), + "param_file": param_file, + "stdout_tail": _collect_tail(stdout_text, 3000), + } + + def _is_completed_import(self, output_dir: str) -> bool: + if not output_dir or not os.path.isdir(output_dir): + return False + log_candidates = [ + os.path.join(output_dir, f"{IMPORT_PROID}.log"), + os.path.join(output_dir, f"{IMPORT_PROID}_console.log"), + ] + log_candidates.extend(str(path) for path in Path(output_dir).glob(f"*{IMPORT_PROID}*.log")) + for log_path in log_candidates: + if not os.path.isfile(log_path): + continue + try: + with open(log_path, "r", encoding="utf-8", errors="ignore") as fp: + content = fp.read() + lowered = content.lower() + if "console success" in lowered: + return True + if "module [LT-1数据导入] success" in content: + return True + if "lt-1" in lowered and "success" in lowered: + return True + except OSError: + continue + return False + + def _resolve_task_identity( + self, + task_dir: str, + task_name: str, + parsed_pair: Optional[Dict[str, Any]], + ) -> tuple[str, str, Dict[str, Any]]: + sidecar_path = os.path.join(task_dir, PAIR_META_FILENAME) + pair_meta: Dict[str, Any] = {} + if os.path.isfile(sidecar_path): + try: + with open(sidecar_path, "r", encoding="utf-8") as fp: + payload = json.load(fp) + if isinstance(payload, dict): + pair_meta = payload + except Exception: + pair_meta = {} + task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name + satellite_family = normalize_satellite_family( + pair_meta.get("master_satellite") or pair_meta.get("slave_satellite") or "lt1" + ) + pair_key = str(pair_meta.get("pair_key") or "").strip() + if not pair_key and parsed_pair: + pair_key = build_fallback_pair_key( + task_alias, + "||".join([parsed_pair["master_xml"], parsed_pair["slave_xml"]]), + satellite_family=satellite_family, + ) + if not pair_key: + pair_key = build_fallback_pair_key(task_alias, task_dir, satellite_family=satellite_family) + return task_alias, pair_key, pair_meta + + def _build_pair_meta_payload( + self, + *, + pair_key: str, + task_alias: str, + pair_meta: Dict[str, Any], + parsed_pair: Dict[str, Any], + ) -> Dict[str, Any]: + return { + **dict(pair_meta or {}), + "pair_key": pair_key, + "task_alias": task_alias, + "master_path": pair_meta.get("master_path") or parsed_pair["master_tif"], + "slave_path": pair_meta.get("slave_path") or parsed_pair["slave_tif"], + "master_satellite": pair_meta.get("master_satellite") or "LT-1", + "slave_satellite": pair_meta.get("slave_satellite") or "LT-1", + "master_imaging_date": pair_meta.get("master_imaging_date") or parsed_pair.get("master_date"), + "slave_imaging_date": pair_meta.get("slave_imaging_date") or parsed_pair.get("slave_date"), + } + + def _write_run_metadata( + self, + *, + run_dir: str, + native_output_dir: str, + task_dir: str, + task_name: str, + task_alias: str, + pair_key: str, + run_key: str, + request: RunRequest, + pair_meta: Dict[str, Any], + params: Dict[str, Any], + dem_path: str, + landsar_output_dir: str, + primary_file: str, + source_files: List[str], + returncode: int, + started_at: str, + ) -> None: + payload = { + "run_key": run_key, + "pair_key": pair_key, + "task_name": task_name, + "task_alias": task_alias, + "engine_code": self.engine_code, + "profile_code": request.profile, + "source_root": _norm_path(request.extra.get("__source_root_override") or request.root_dir), + "task_dir": _norm_path(task_dir), + "work_dir": landsar_output_dir, + "output_dir": _norm_path(run_dir), + "native_output_dir": _norm_path(native_output_dir), + "started_at": started_at, + "finished_at": _utc_text(), + "params": { + **dict(params or {}), + "dem_path": dem_path, + }, + "metrics": { + "returncode": returncode, + "primary_file": primary_file, + "source_files": source_files, + }, + "master_path": pair_meta.get("master_path"), + "slave_path": pair_meta.get("slave_path"), + "master_satellite": pair_meta.get("master_satellite"), + "slave_satellite": pair_meta.get("slave_satellite"), + "master_imaging_date": pair_meta.get("master_imaging_date"), + "slave_imaging_date": pair_meta.get("slave_imaging_date"), + "master_imaging_mode": pair_meta.get("master_imaging_mode"), + "slave_imaging_mode": pair_meta.get("slave_imaging_mode"), + "master_polarization": pair_meta.get("master_polarization"), + "slave_polarization": pair_meta.get("slave_polarization"), + "time_baseline_days": pair_meta.get("time_baseline_days"), + "spatial_baseline_meters": pair_meta.get("spatial_baseline_meters"), + "scene_center_distance_meters": pair_meta.get("scene_center_distance_meters"), + "scene_pair_uid": pair_meta.get("scene_pair_uid") or pair_meta.get("pair_uid"), + "pair_uid": pair_meta.get("pair_uid") or pair_meta.get("scene_pair_uid"), + "network_run_id": pair_meta.get("network_run_id"), + "network_edge_id": pair_meta.get("network_edge_id"), + "policy_version": pair_meta.get("policy_version"), + "selection_strategy": pair_meta.get("selection_strategy"), + } + write_run_metadata(run_dir, payload) + write_run_metadata(native_output_dir, payload) + + def _build_task_result( + self, + *, + task_name: str, + task_alias: str, + pair_key: str, + run_key: str, + task_dir: str, + run_dir: str, + native_output_dir: str, + landsar_output_dir: str, + command: List[str], + success: bool, + returncode: int, + error: str = "", + stdout_tail: str = "", + primary_file: str = "", + source_files: Optional[List[str]] = None, + validation: Optional[Dict[str, Any]] = None, + layout: Optional[Dict[str, Any]] = None, + param_file: str = "", + raw_primary_file: str = "", + raw_coherence_file: str = "", + ) -> Dict[str, Any]: + command_text = command if isinstance(command, str) else " ".join(str(part) for part in command) + return { + "task_name": task_name, + "task_alias": task_alias, + "pair_key": pair_key, + "run_key": run_key, + "task_dir": _norm_path(task_dir), + "run_dir": _norm_path(run_dir), + "native_output_dir": _norm_path(native_output_dir), + "output_dir": _norm_path(run_dir), + "landsar_output_dir": _norm_path(landsar_output_dir), + "primary_file": _norm_path(primary_file) if primary_file else "", + "source_files": [_norm_path(path) for path in (source_files or [])], + "raw_primary_file": _norm_path(raw_primary_file) if raw_primary_file else "", + "raw_coherence_file": _norm_path(raw_coherence_file) if raw_coherence_file else "", + "param_file": _norm_path(param_file) if param_file else "", + "validation": validation or {}, + "layout": layout or {}, + "command": command_text, + "success": success, + "returncode": returncode, + "stdout_tail": stdout_tail, + "stderr_tail": "", + "error": error, + } + + def _select_primary_file(self, output_dir: str) -> str: + patterns = [ + "*displ.geo.tif", + "*displ.geo.tiff", + "*dispv.geo.tif", + "*dispv.geo.tiff", + "*diff.unw.geo.tif", + "*diff.unw.geo.tiff", + "*unw.geo.tif", + "*unw.geo.tiff", + "*.geo.tif", + "*.geo.tiff", + ] + return self._first_matching_file(output_dir, patterns, exclude=("coh", "filcc", "wrap")) + + def _select_coherence_file(self, output_dir: str) -> str: + return self._first_matching_file(output_dir, ["*coh.geo.tif", "*coh.geo.tiff", "*filcc.geo.tif", "*filcc.geo.tiff"]) + + def _first_matching_file(self, output_dir: str, patterns: List[str], exclude: tuple[str, ...] = ()) -> str: + root = Path(output_dir) + if not root.is_dir(): + return "" + for pattern in patterns: + matches = sorted(root.rglob(pattern), key=lambda path: str(path).lower()) + for match in matches: + if not match.is_file(): + continue + lower_name = match.name.lower() + if any(token in lower_name for token in exclude): + continue + return _norm_path(match) + return "" diff --git a/backend/app/dinsar_engines/pyint_engine.py b/backend/app/dinsar_engines/pyint_engine.py index e3f8ae2..a0ebd79 100644 --- a/backend/app/dinsar_engines/pyint_engine.py +++ b/backend/app/dinsar_engines/pyint_engine.py @@ -687,7 +687,7 @@ class PyintEngine(DinsarEngine): available = True else: critical_failed = [check for check in report.checks if not check.ok and not check.skipped] - status = "degraded" if critical_failed else "unavailable" + status = "unavailable" if critical_failed else "degraded" available = False return EngineAvailability( engine_code=self.engine_code, diff --git a/backend/app/main.py b/backend/app/main.py index 841954a..d732f97 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -192,7 +192,7 @@ async def lifespan(app: FastAPI): if ps_catalog_bootstrap.get("error"): print(f">>> [Timeseries Catalog] Startup bootstrap failed: {ps_catalog_bootstrap['error']}") print( - ">>> [Gamma SBAS Catalog] root={0} runs={1} db={2} rebuild={3} rebuilt={4}".format( + ">>> [SBAS Catalog] root={0} runs={1} db={2} rebuild={3} rebuilt={4}".format( sbas_catalog_bootstrap.get("storage_root") or "?", sbas_catalog_bootstrap.get("manifest_count", 0), sbas_catalog_bootstrap.get("db_count", 0), @@ -201,7 +201,7 @@ async def lifespan(app: FastAPI): ) ) if sbas_catalog_bootstrap.get("error"): - print(f">>> [Gamma SBAS Catalog] Startup bootstrap failed: {sbas_catalog_bootstrap['error']}") + print(f">>> [SBAS Catalog] Startup bootstrap failed: {sbas_catalog_bootstrap['error']}") print( ">>> [Pairing] status={0} scenes={1} pairs={2} dirty={3} metric={4} rebuild={5}".format( pairing_bootstrap.get("status") or "?", diff --git a/backend/app/routers/dinsar_production.py b/backend/app/routers/dinsar_production.py index 8767d2c..0d99dfb 100644 --- a/backend/app/routers/dinsar_production.py +++ b/backend/app/routers/dinsar_production.py @@ -40,6 +40,12 @@ PYINT_PRODUCTION_JOB_MAX_ATTEMPTS = read_int_env( minimum=1, maximum=10, ) +LANDSAR_PRODUCTION_JOB_MAX_ATTEMPTS = read_int_env( + "LANDSAR_PRODUCTION_JOB_MAX_ATTEMPTS", + 1, + minimum=1, + maximum=10, +) class RunJobRequest(BaseModel): @@ -243,7 +249,12 @@ async def submit_run( "extra": dict(req.extra or {}), } - from ..services.job_handlers import JOB_TYPE_IDL_RUN_DINSAR, JOB_TYPE_ISCE2_RUN, JOB_TYPE_PYINT_RUN + from ..services.job_handlers import ( + JOB_TYPE_IDL_RUN_DINSAR, + JOB_TYPE_ISCE2_RUN, + JOB_TYPE_LANDSAR_RUN, + JOB_TYPE_PYINT_RUN, + ) create_managed_run = False normalized_extra = dict(payload["extra"]) @@ -252,7 +263,7 @@ async def submit_run( job_type = JOB_TYPE_IDL_RUN_DINSAR max_attempts = DINSAR_PRODUCTION_JOB_MAX_ATTEMPTS create_managed_run = True - elif req.engine_code in {"isce2", "pyint"}: + elif req.engine_code in {"isce2", "pyint", "landsar"}: if hasattr(engine, "normalize_extra"): try: payload["extra"] = engine.normalize_extra(payload["extra"]) @@ -263,10 +274,14 @@ async def submit_run( job_type = JOB_TYPE_ISCE2_RUN max_attempts = ISCE2_PRODUCTION_JOB_MAX_ATTEMPTS create_managed_run = True - else: + elif req.engine_code == "pyint": job_type = JOB_TYPE_PYINT_RUN max_attempts = PYINT_PRODUCTION_JOB_MAX_ATTEMPTS create_managed_run = True + else: + job_type = JOB_TYPE_LANDSAR_RUN + max_attempts = LANDSAR_PRODUCTION_JOB_MAX_ATTEMPTS + create_managed_run = True if validation_summary is not None: validated_task_count = validation_summary.get("task_count", 0) payload["extra"].update( diff --git a/backend/app/routers/sbas_insar_production.py b/backend/app/routers/sbas_insar_production.py index 43f74c5..5dbfde8 100644 --- a/backend/app/routers/sbas_insar_production.py +++ b/backend/app/routers/sbas_insar_production.py @@ -4,18 +4,204 @@ import asyncio import mimetypes import subprocess -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import FileResponse from pydantic import BaseModel, Field, field_validator, model_validator +from sqlalchemy import String, cast, or_, select +from .. import database +from ..config import settings +from ..models import AuthUserORM, SystemJobORM, SystemTaskORM, TaskLogORM from ..services.job_queue_service import job_queue_service +from ..services.landsar_sbas_service import landsar_sbas_service from ..services.sbas_insar_production_service import sbas_insar_production_service from ..services.task_service import task_service +from .dependencies import _require_admin router = APIRouter(prefix="/sbas-insar-production", tags=["sbas-insar-production"]) +def _new_session(): + if database.AsyncSessionLocal is None: + database.init_db() + if database.AsyncSessionLocal is None: + raise RuntimeError("Database session factory is not initialized.") + return database.AsyncSessionLocal() + + +def _dt(value): + return value.isoformat() if value else None + + +def _task_payload_matches(run_id: str): + return or_( + cast(SystemTaskORM.params, String).ilike(f"%{run_id}%"), + SystemTaskORM.task_name.ilike(f"%{run_id}%"), + ) + + +def _job_payload_matches(run_id: str, task_ids: list[str]): + conditions = [ + cast(SystemJobORM.payload, String).ilike(f"%{run_id}%"), + SystemJobORM.workflow_run_id == run_id, + ] + if task_ids: + conditions.append(SystemJobORM.task_id.in_(task_ids)) + return or_(*conditions) + + +async def _load_run_background_activity(run_id: str) -> dict: + try: + async with _new_session() as db: + task_result = await db.execute( + select(SystemTaskORM) + .where(_task_payload_matches(run_id)) + .order_by(SystemTaskORM.created_at.desc(), SystemTaskORM.id.desc()) + .limit(10) + ) + tasks = list(task_result.scalars().all()) + task_ids = [ + str(task.task_id or "").strip() + for task in tasks + if str(task.task_id or "").strip() + ] + job_result = await db.execute( + select(SystemJobORM) + .where(_job_payload_matches(run_id, task_ids)) + .order_by(SystemJobORM.created_at.desc(), SystemJobORM.id.desc()) + .limit(10) + ) + jobs = list(job_result.scalars().all()) + if not task_ids: + task_ids = sorted({ + str(job.task_id or "").strip() + for job in jobs + if str(job.task_id or "").strip() + }) + logs = [] + if task_ids: + log_result = await db.execute( + select(TaskLogORM) + .where(TaskLogORM.task_id.in_(task_ids)) + .order_by(TaskLogORM.timestamp.desc(), TaskLogORM.id.desc()) + .limit(20) + ) + logs = list(log_result.scalars().all()) + except Exception as exc: + return { + "schema": "insar.sbas-background-activity/v1", + "error": str(exc), + "tasks": [], + "jobs": [], + "task_logs": [], + "active": False, + } + + active_task_statuses = {"PENDING", "RUNNING"} + active_job_statuses = {"READY", "PENDING", "RUNNING", "RETRY"} + return { + "schema": "insar.sbas-background-activity/v1", + "tasks": [ + { + "task_id": task.task_id, + "task_type": task.task_type, + "task_name": task.task_name, + "status": task.status, + "progress": task.progress, + "message": task.message, + "created_at": _dt(task.created_at), + "updated_at": _dt(task.updated_at), + "started_at": _dt(task.started_at), + "ended_at": _dt(task.ended_at), + } + for task in tasks + ], + "jobs": [ + { + "job_id": job.job_id, + "job_type": job.job_type, + "status": job.status, + "attempts": job.attempts, + "max_attempts": job.max_attempts, + "locked_by": job.locked_by, + "locked_at": _dt(job.locked_at), + "heartbeat_at": _dt(job.heartbeat_at), + "created_at": _dt(job.created_at), + "updated_at": _dt(job.updated_at), + "started_at": _dt(job.started_at), + "finished_at": _dt(job.finished_at), + "last_error": job.last_error, + "task_id": job.task_id, + } + for job in jobs + ], + "task_logs": [ + { + "task_id": log.task_id, + "level": log.log_level, + "message": log.message, + "timestamp": _dt(log.timestamp), + } + for log in logs + ], + "active": any(str(task.status or "").upper() in active_task_statuses for task in tasks) + or any(str(job.status or "").upper() in active_job_statuses for job in jobs), + } + + +def _load_wsl_process_summary(run_id: str) -> dict: + distro = str(settings.GAMMA_SBAS_WSL_DISTRO or settings.WSL_DISTRO or "").strip() + command = [ + "wsl.exe", + *([] if not distro else ["-d", distro]), + "--", + "bash", + "-lc", + ( + "ps -eo pid,etime,stat,args --no-headers | " + "grep -E 'gamma|SLC_interp|base_calc|mk_diff|mk_unw|ts_rate| mb |python|bash' | " + "grep -v grep | head -20" + ), + ] + try: + completed = subprocess.run( + command, + capture_output=True, + text=True, + timeout=5, + check=False, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + except FileNotFoundError: + return {"available": False, "error": "wsl.exe not found", "processes": []} + except Exception as exc: + return {"available": False, "error": str(exc), "processes": []} + + processes = [] + for line in (completed.stdout or "").splitlines(): + text = line.strip() + if not text: + continue + parts = text.split(None, 3) + processes.append( + { + "pid": parts[0] if len(parts) > 0 else "", + "etime": parts[1] if len(parts) > 1 else "", + "stat": parts[2] if len(parts) > 2 else "", + "command": parts[3] if len(parts) > 3 else text, + "matches_run": run_id in text, + } + ) + return { + "available": completed.returncode in {0, 1}, + "returncode": completed.returncode, + "distro": distro or None, + "processes": processes, + "stderr_tail": (completed.stderr or "")[-1200:], + } + + class SbasAoiBbox(BaseModel): min_lon: float = Field(ge=-180, le=180) min_lat: float = Field(ge=-90, le=90) @@ -30,6 +216,7 @@ class SbasAoiBbox(BaseModel): class SbasStackDiscoverRequest(BaseModel): + sensor_family: str = Field(default="LT1", pattern="^(LT1|S1)$") source_roots: list[str] | None = None orbit_roots: list[str] | None = None min_scenes: int = Field(default=3, ge=2, le=100) @@ -43,7 +230,11 @@ class SbasStackDiscoverRequest(BaseModel): discovery_mode: str = Field(default="strict", pattern="^(strict|aoi)$") aoi_bbox: SbasAoiBbox | None = None min_aoi_coverage_ratio: float = Field(default=0.01, ge=0, le=1) - min_common_overlap_ratio: float = Field(default=0.0, ge=0, le=1) + min_common_overlap_ratio: float = Field( + default_factory=lambda: float(settings.GAMMA_SBAS_MIN_COMMON_OVERLAP_RATIO or 0.30), + ge=0, + le=1, + ) @field_validator("source_roots", "orbit_roots", mode="before") @classmethod @@ -57,6 +248,14 @@ class SbasStackDiscoverRequest(BaseModel): cleaned = [str(item or "").strip() for item in items if str(item or "").strip()] return cleaned or None + @field_validator("sensor_family", mode="before") + @classmethod + def _normalize_sensor_family(cls, value): + text = str(value or "LT1").strip().upper() + if text in {"SENTINEL1", "SENTINEL-1"}: + return "S1" + return text or "LT1" + @field_validator("platform", "relative_orbit", "orbit_direction", "admin_region", mode="before") @classmethod def _normalize_optional_text(cls, value): @@ -165,9 +364,122 @@ class SbasWorkflowJobRequest(SbasWorkflowPrepareRequest): timeout_seconds: int = Field(default=172800, ge=60, le=604800) +class LandsarSbasAutoWorkflowRequest(SbasStackDiscoverRequest): + sensor_family: str = Field(default="LT1", pattern="^LT1$") + require_orbits: bool = False + run_label: str | None = Field(default=None, max_length=160) + dem_path: str | None = Field(default=None, max_length=1024) + timeout_seconds: int | None = Field(default=None, ge=60, le=604800) + import_timeout_seconds: int | None = Field(default=None, ge=60, le=604800) + workflow_timeout_seconds: int | None = Field(default=None, ge=60, le=604800) + params: dict[str, object] = Field(default_factory=dict) + + @router.get("/capabilities") async def get_sbas_insar_capabilities(): - return sbas_insar_production_service.get_capabilities() + capabilities = sbas_insar_production_service.get_capabilities() + capabilities["processors"] = [ + { + "processor_code": capabilities.get("processor_code"), + "profile_code": "lt1_gamma_sbas", + "engine_code": capabilities.get("engine_code"), + "label": "Gamma / IPTA SBAS", + "enabled": bool((capabilities.get("runtime") or {}).get("enabled", True)), + }, + { + **landsar_sbas_service.get_capabilities(), + "label": "LandSAR SBAS", + }, + ] + return capabilities + + +@router.get("/landsar/capabilities") +async def get_landsar_sbas_capabilities(): + return landsar_sbas_service.get_capabilities() + + +@router.post("/landsar/workflows/auto", status_code=202) +async def submit_landsar_sbas_auto_workflow(request: LandsarSbasAutoWorkflowRequest): + try: + from ..services.job_handlers import JOB_TYPE_SBAS_LANDSAR_WORKFLOW + + selection_request = { + "run_label": request.run_label, + "source_roots": request.source_roots, + "orbit_roots": request.orbit_roots, + "min_scenes": request.min_scenes, + "discovery_mode": request.discovery_mode, + "admin_region": request.admin_region, + "aoi_bbox": request.aoi_bbox.model_dump() if request.aoi_bbox else None, + "min_aoi_coverage_ratio": request.min_aoi_coverage_ratio, + "min_common_overlap_ratio": request.min_common_overlap_ratio, + "limit": request.limit, + "dem_path": request.dem_path, + "timeout_seconds": request.timeout_seconds, + "import_timeout_seconds": request.import_timeout_seconds, + "params": dict(request.params or {}), + } + payload = { + "auto_select": True, + "selection_request": selection_request, + "timeout_seconds": request.workflow_timeout_seconds or request.timeout_seconds, + } + task_id = await task_service.create_task( + task_type=JOB_TYPE_SBAS_LANDSAR_WORKFLOW, + task_name="LandSAR SBAS Auto Workflow", + params=payload, + ) + job_id = await job_queue_service.create_job( + job_type=JOB_TYPE_SBAS_LANDSAR_WORKFLOW, + payload=payload, + task_id=task_id, + max_attempts=1, + ) + return { + "message": "LandSAR SBAS auto workflow job queued. Stack selection and input import will run in the background.", + "run_id": None, + "selection_pending": True, + "task_id": task_id, + "job_id": job_id, + "job_type": JOB_TYPE_SBAS_LANDSAR_WORKFLOW, + "status": "QUEUED", + } + except ValueError as exc: + message = str(exc) + status_code = 409 if "already running" in message.lower() or "conflict" in message.lower() or "任务冲突" in message else 400 + raise HTTPException(status_code=status_code, detail=message) from exc + + +@router.get("/landsar/runs") +async def list_landsar_sbas_runs(): + return await asyncio.to_thread(landsar_sbas_service.list_runs) + + +@router.get("/landsar/runs/{run_id}") +async def get_landsar_sbas_run(run_id: str): + try: + return await asyncio.to_thread(landsar_sbas_service.get_run_detail, run_id) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.get("/landsar/runs/{run_id}/artifacts/{relative_path:path}") +async def get_landsar_sbas_run_artifact(run_id: str, relative_path: str): + try: + artifact_path = await asyncio.to_thread(landsar_sbas_service.resolve_artifact, run_id, relative_path) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + media_type = mimetypes.guess_type(str(artifact_path))[0] or "application/octet-stream" + return FileResponse( + artifact_path, + media_type=media_type, + filename=artifact_path.name, + ) @router.post("/stacks/discover") @@ -175,6 +487,7 @@ async def discover_sbas_insar_stacks(request: SbasStackDiscoverRequest): try: return await asyncio.to_thread( sbas_insar_production_service.discover_stacks, + sensor_family=request.sensor_family, source_roots=request.source_roots, orbit_roots=request.orbit_roots, min_scenes=request.min_scenes, @@ -200,6 +513,7 @@ async def audit_sbas_insar_stack(stack_id: str, request: SbasStackDiscoverReques return await asyncio.to_thread( sbas_insar_production_service.audit_stack, stack_id, + sensor_family=request.sensor_family, source_roots=request.source_roots, orbit_roots=request.orbit_roots, min_scenes=request.min_scenes, @@ -222,6 +536,7 @@ async def submit_sbas_insar_run(stack_id: str, request: SbasRunSubmitRequest): return await asyncio.to_thread( sbas_insar_production_service.create_run, stack_id, + sensor_family=request.sensor_family, run_label=request.run_label, source_roots=request.source_roots, orbit_roots=request.orbit_roots, @@ -253,13 +568,57 @@ async def list_sbas_insar_runs(): @router.get("/runs/{run_id}") async def get_sbas_insar_run(run_id: str): try: - return await asyncio.to_thread(sbas_insar_production_service.get_run_detail, run_id) + detail = await asyncio.to_thread(sbas_insar_production_service.get_run_detail, run_id) + background_activity, wsl_processes = await asyncio.gather( + _load_run_background_activity(run_id), + asyncio.to_thread(_load_wsl_process_summary, run_id), + ) + runtime_status = dict(detail.get("runtime_status") or {}) + runtime_status["background_activity"] = background_activity + runtime_status["wsl_processes"] = wsl_processes + runtime_status["active"] = bool( + runtime_status.get("active") + or background_activity.get("active") + or any(item.get("matches_run") for item in wsl_processes.get("processes") or []) + ) + detail["runtime_status"] = runtime_status + return detail except FileNotFoundError as exc: raise HTTPException(status_code=404, detail=str(exc)) from exc except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc +@router.delete("/runs/{run_id}") +async def delete_sbas_insar_run( + run_id: str, + current_user: AuthUserORM = Depends(_require_admin), +): + _ = current_user + try: + async with _new_session() as db: + result = await sbas_insar_production_service.delete_run_record(run_id, db=db) + try: + from ..services.sbas_insar_catalog_service import sbas_insar_catalog_service + + catalog_result = await sbas_insar_catalog_service.rebuild_catalog(db, full_rebuild=True) + except Exception as catalog_exc: + catalog_result = { + "status": "WARN", + "message": f"SBAS catalog rebuild failed after run deletion: {catalog_exc}", + } + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + message = str(exc) + status_code = 409 if "active task/job" in message or "running" in message.lower() else 400 + raise HTTPException(status_code=status_code, detail=message) from exc + return { + **result, + "catalog": catalog_result, + } + + @router.post("/runs/{run_id}/workflow", status_code=202) async def prepare_sbas_insar_workflow(run_id: str, request: SbasWorkflowPrepareRequest): try: diff --git a/backend/app/routers/sbas_insar_products.py b/backend/app/routers/sbas_insar_products.py index e28ee65..d50225f 100644 --- a/backend/app/routers/sbas_insar_products.py +++ b/backend/app/routers/sbas_insar_products.py @@ -5,10 +5,11 @@ import os from fastapi import APIRouter, Depends, HTTPException, Request from fastapi.responses import FileResponse from pydantic import BaseModel +from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from ..database import get_db -from ..models import AuthUserORM +from ..models import AuthUserORM, SystemTaskORM from ..services.job_queue_service import job_queue_service from ..services.sbas_insar_catalog_service import ( JOB_TYPE_REBUILD_SBAS_INSAR_CATALOG, @@ -26,6 +27,24 @@ class SbasInsarCatalogRebuildRequest(BaseModel): full_rebuild: bool = True +class SbasInsarPointTimeseriesRequest(BaseModel): + lon: float + lat: float + + +async def _get_active_sbas_catalog_rebuild_task(db: AsyncSession) -> SystemTaskORM | None: + result = await db.execute( + select(SystemTaskORM) + .where( + SystemTaskORM.task_type == TASK_TYPE_REBUILD_SBAS_INSAR_CATALOG, + SystemTaskORM.status.in_(["PENDING", "RUNNING"]), + ) + .order_by(SystemTaskORM.created_at.desc(), SystemTaskORM.id.desc()) + .limit(1) + ) + return result.scalars().first() + + @router.get("/sbas-insar-products/catalog-status") async def get_sbas_insar_catalog_status( current_user: AuthUserORM = Depends(_get_current_user), @@ -43,12 +62,40 @@ async def queue_sbas_insar_catalog_rebuild( admin_user: AuthUserORM = Depends(_require_admin), ): _ = admin_user - task_id = await task_service.create_task( - TASK_TYPE_REBUILD_SBAS_INSAR_CATALOG, - "SBAS-InSAR result catalog rebuild", - params={"full_rebuild": request.full_rebuild}, - db=db, - ) + existing_task = await _get_active_sbas_catalog_rebuild_task(db) + if existing_task is not None: + await _add_operation_audit_log( + db, + request=http_request, + action="sbas_insar_catalog_rebuild_already_running", + resource="sbas-insar-products/rebuild", + detail={"task_id": existing_task.task_id, "full_rebuild": request.full_rebuild}, + ) + await db.commit() + return { + "message": "SBAS-InSAR result catalog rebuild is already queued or running.", + "task_id": existing_task.task_id, + "already_running": True, + } + + try: + task_id = await task_service.create_task( + TASK_TYPE_REBUILD_SBAS_INSAR_CATALOG, + "SBAS-InSAR result catalog rebuild", + params={"full_rebuild": request.full_rebuild}, + db=db, + ) + except ValueError as exc: + await db.rollback() + existing_task = await _get_active_sbas_catalog_rebuild_task(db) + if existing_task is not None: + return { + "message": "SBAS-InSAR result catalog rebuild is already queued or running.", + "task_id": existing_task.task_id, + "already_running": True, + } + raise HTTPException(status_code=409, detail=str(exc)) from exc + await job_queue_service.create_job( JOB_TYPE_REBUILD_SBAS_INSAR_CATALOG, payload={"full_rebuild": request.full_rebuild}, @@ -103,6 +150,32 @@ async def get_sbas_insar_product_detail( return detail +@router.post("/sbas-insar-products/{product_db_id}/point-timeseries") +async def query_sbas_insar_point_timeseries( + product_db_id: int, + request: SbasInsarPointTimeseriesRequest, + current_user: AuthUserORM = Depends(_get_current_user), + db: AsyncSession = Depends(get_db), +): + _ = current_user + try: + detail = await sbas_insar_catalog_service.query_point_timeseries( + db, + product_db_id=product_db_id, + lon=request.lon, + lat=request.lat, + ) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except RuntimeError as exc: + raise HTTPException(status_code=500, detail=str(exc)) from exc + if detail is None: + raise HTTPException(status_code=404, detail="SBAS-InSAR product not found") + return detail + + @router.get("/sbas-insar-products/{product_db_id}/preview") async def get_sbas_insar_product_preview( product_db_id: int, diff --git a/backend/app/services/dinsar_production_service.py b/backend/app/services/dinsar_production_service.py index c301802..4053f04 100644 --- a/backend/app/services/dinsar_production_service.py +++ b/backend/app/services/dinsar_production_service.py @@ -25,6 +25,8 @@ from ..models import ( WorkflowRunORM, WorkflowStepORM, ) +from ..utils import normalize_satellite_family +from .dinsar_naming import build_fallback_pair_key from .envi_service import RUNTIME_DIR, _collect_task_folders, _resolve_dinsar_pair_identity, _to_local_path from .task_service import task_service from .workflow_service import workflow_service @@ -33,6 +35,7 @@ from .workflow_service import workflow_service TASK_TYPE_DINSAR_PRODUCTION = "IDL_RUN_DINSAR" TASK_TYPE_ISCE2_DINSAR_PRODUCTION = "ISCE2_RUN" TASK_TYPE_PYINT_DINSAR_PRODUCTION = "PYINT_RUN" +TASK_TYPE_LANDSAR_DINSAR_PRODUCTION = "LANDSAR_RUN" RUN_STATUS_PENDING = "PENDING" RUN_STATUS_RUNNING = "RUNNING" RUN_STATUS_COMPLETED = "COMPLETED" @@ -83,6 +86,8 @@ def _task_type_for_engine(engine_code: str) -> str: return TASK_TYPE_ISCE2_DINSAR_PRODUCTION if normalized in {"pyint", "gamma"}: return TASK_TYPE_PYINT_DINSAR_PRODUCTION + if normalized == "landsar": + return TASK_TYPE_LANDSAR_DINSAR_PRODUCTION raise ValueError(f"Unsupported engine for D-InSAR production run: {engine_code}") @@ -94,6 +99,8 @@ def _workflow_name_for_engine(engine_code: str) -> str: return "dinsar_isce2_production" if normalized in {"pyint", "gamma"}: return "dinsar_pyint_gamma_production" + if normalized == "landsar": + return "dinsar_landsar_production" raise ValueError(f"Unsupported engine for D-InSAR production run: {engine_code}") @@ -105,6 +112,8 @@ def _workflow_step_name_for_engine(engine_code: str) -> str: return "Execute ISCE2 D-InSAR items" if normalized in {"pyint", "gamma"}: return "Execute PyINT/Gamma D-InSAR items" + if normalized == "landsar": + return "Execute LandSAR D-InSAR items" raise ValueError(f"Unsupported engine for D-InSAR production run: {engine_code}") @@ -150,6 +159,33 @@ def _looks_like_task_dir(path: str) -> bool: return os.path.isdir(os.path.join(path, "master")) and os.path.isdir(os.path.join(path, "slave")) +def _looks_like_landsar_task_dir(path: str) -> bool: + return os.path.isdir(os.path.join(path, "Input_Data")) + + +def _looks_like_landsar_raw_task_dir(path: str) -> bool: + normalized = os.path.normpath(os.path.abspath(_to_local_path(path))) + master_dir = os.path.join(normalized, "master") + slave_dir = os.path.join(normalized, "slave") + if not os.path.isdir(master_dir) or not os.path.isdir(slave_dir): + return False + + def _has_lt1_file(directory: str) -> bool: + try: + with os.scandir(directory) as entries: + for entry in entries: + if not entry.is_file(): + continue + name = entry.name.lower() + if name.startswith("lt1") and name.endswith((".xml", ".tif", ".tiff")): + return True + except OSError: + return False + return False + + return _has_lt1_file(master_dir) and _has_lt1_file(slave_dir) + + def _normalize_rerun_mode(value: Optional[str]) -> str: normalized = str(value or "").strip().lower() if normalized in VALID_RERUN_MODES: @@ -182,6 +218,64 @@ def _discover_run_items(root_dir: str) -> List[Dict[str, Any]]: return items +def _discover_landsar_run_items(root_dir: str) -> List[Dict[str, Any]]: + if _looks_like_landsar_task_dir(root_dir) or _looks_like_landsar_raw_task_dir(root_dir): + task_folders = [root_dir] + else: + task_folders = [ + folder + for folder in _collect_task_folders(root_dir) + if _looks_like_landsar_task_dir(folder) or _looks_like_landsar_raw_task_dir(folder) + ] + + items: List[Dict[str, Any]] = [] + try: + from ..dinsar_engines.landsar_engine import parse_lt1_slc_pair + except Exception: + parse_lt1_slc_pair = None + + for order_index, folder in enumerate(task_folders, start=1): + task_name = os.path.basename(folder) + task_alias, pair_key, pair_meta = _resolve_dinsar_pair_identity(folder, task_name) + has_input_data = _looks_like_landsar_task_dir(folder) + has_raw_input = _looks_like_landsar_raw_task_dir(folder) + pair = ( + parse_lt1_slc_pair(os.path.join(folder, "Input_Data")) + if parse_lt1_slc_pair is not None and has_input_data + else None + ) + if parse_lt1_slc_pair is not None and has_input_data and not pair and not has_raw_input: + continue + if pair and not pair_meta.get("pair_key"): + pair_key = build_fallback_pair_key( + task_alias, + "||".join([pair["master_xml"], pair["slave_xml"]]), + satellite_family=normalize_satellite_family("lt1"), + ) + elif not pair_meta.get("pair_key") and has_raw_input: + pair_key = build_fallback_pair_key( + task_alias, + folder, + satellite_family=normalize_satellite_family("lt1"), + ) + items.append( + { + "order_index": order_index, + "task_name": task_name, + "task_alias": task_alias, + "pair_key": pair_key, + "pair_uid": pair_meta.get("pair_uid") or pair_meta.get("scene_pair_uid"), + "network_run_id": pair_meta.get("network_run_id"), + "network_edge_id": pair_meta.get("network_edge_id"), + "policy_version": pair_meta.get("policy_version"), + "selection_strategy": pair_meta.get("selection_strategy"), + "source_task_dir": folder, + "results_root_dir": os.path.join(settings.DINSAR_PRODUCT_DIR, pair_key), + } + ) + return items + + def _current_pointer_path_for_root( results_root_dir: str, *, @@ -256,7 +350,12 @@ def _select_run_items( num_to_process: int, rerun_mode: Optional[str], ) -> Dict[str, Any]: - discovered_items = _discover_run_items(root_dir) + normalized_engine = str(engine_code or "").strip().lower() + discovered_items = ( + _discover_landsar_run_items(root_dir) + if normalized_engine == "landsar" + else _discover_run_items(root_dir) + ) normalized_mode = _normalize_rerun_mode(rerun_mode) skipped_completed_count = 0 diff --git a/backend/app/services/health_service.py b/backend/app/services/health_service.py index de0d5e0..fe8c09b 100644 --- a/backend/app/services/health_service.py +++ b/backend/app/services/health_service.py @@ -25,9 +25,15 @@ from ..models import ( from ..idl_service import get_idl_status from .product_package_schema import CANONICAL_PACKAGE_SCHEMA from .pairing_state_service import pairing_state_service +from .sbas_insar_catalog_service import sbas_insar_catalog_service from .wsl_runtime_registry import wsl_runtime_registry +ALLOWED_PRODUCT_PACKAGE_SCHEMAS = { + CANONICAL_PACKAGE_SCHEMA, + "insar.gamma-ipta-sbas-run/v1", + "insar.gamma-sbas-run/v1", +} DEFAULT_WORKER_TIMEOUT_SECONDS = 60 DEFAULT_SCHEMA_CACHE_SECONDS = 120 _SCHEMA_CACHE_LOCK = asyncio.Lock() @@ -202,13 +208,22 @@ def _build_catalog_status( catalog_name: str, storage_root: str, enabled: bool = True, + storage_roots: Optional[List[str]] = None, ) -> Dict[str, Any]: + normalized_roots = [str(item or "").strip() for item in (storage_roots or [storage_root]) if str(item or "").strip()] + if not normalized_roots and storage_root: + normalized_roots = [storage_root] return { "ok": False, "catalog_name": catalog_name, "enabled": enabled, "storage_root": storage_root, - "storage_root_exists": os.path.isdir(storage_root), + "storage_roots": normalized_roots, + "storage_root_exists": bool(storage_root) and os.path.isdir(storage_root), + "storage_roots_status": [ + {"path": root, "exists": os.path.isdir(root)} + for root in normalized_roots + ], "state_present": False, "catalog_status": None, "needs_rebuild": None, @@ -228,11 +243,13 @@ async def _check_catalog( catalog_name: str, storage_root: str, enabled: bool = True, + storage_roots: Optional[List[str]] = None, ) -> Dict[str, Any]: status = _build_catalog_status( catalog_name=catalog_name, storage_root=storage_root, enabled=enabled, + storage_roots=storage_roots, ) if not enabled: status["ok"] = True @@ -262,6 +279,16 @@ async def _check_catalog( status["last_message"] = state.last_message if state.storage_root: status["storage_root"] = state.storage_root + root_status_by_path = { + item["path"]: item + for item in status.get("storage_roots_status", []) + } + if state.storage_root not in root_status_by_path: + status.setdefault("storage_roots", []).insert(0, state.storage_root) + status.setdefault("storage_roots_status", []).insert( + 0, + {"path": state.storage_root, "exists": os.path.isdir(state.storage_root)}, + ) status["storage_root_exists"] = os.path.isdir(state.storage_root) count_result = await db.execute( @@ -273,7 +300,9 @@ async def _check_catalog( manifest_count = int(status["manifest_count"] or 0) needs_rebuild = bool(status["needs_rebuild"]) - status["ok"] = bool(status["storage_root_exists"]) and not ( + roots_status = status.get("storage_roots_status") or [] + roots_exist = all(bool(item.get("exists")) for item in roots_status) if roots_status else bool(status["storage_root_exists"]) + status["ok"] = roots_exist and not ( manifest_count > 0 and needs_rebuild ) except Exception as exc: @@ -299,10 +328,13 @@ async def _check_timeseries_result_catalog() -> Dict[str, Any]: async def _check_sbas_insar_result_catalog() -> Dict[str, Any]: + run_roots = sbas_insar_catalog_service.get_run_roots() + primary_root = run_roots[0] if run_roots else os.path.join(settings.GAMMA_SBAS_WORK_ROOT, "runs") return await _check_catalog( catalog_name="sbas_insar", - storage_root=os.path.join(settings.GAMMA_SBAS_WORK_ROOT, "runs"), - enabled=bool(settings.GAMMA_SBAS_ENABLED), + storage_root=primary_root, + storage_roots=run_roots, + enabled=bool(settings.GAMMA_SBAS_ENABLED or settings.LANDSAR_SBAS_ENABLED), ) @@ -389,6 +421,8 @@ def _sanitize_product_package_status(payload: Dict[str, Any]) -> Dict[str, Any]: "ok": bool(payload.get("ok")), "total_count": int(payload.get("total_count") or 0), "canonical_count": int(payload.get("canonical_count") or 0), + "valid_schema_count": int(payload.get("valid_schema_count") or 0), + "invalid_schema_count": int(payload.get("invalid_schema_count") or 0), "missing_manifest_count": int(payload.get("missing_manifest_count") or 0), "missing_publish_dir_count": int(payload.get("missing_publish_dir_count") or 0), "missing_processor_count": int(payload.get("missing_processor_count") or 0), @@ -1033,8 +1067,11 @@ async def _check_product_packages() -> Dict[str, Any]: status = { "ok": False, "canonical_schema": CANONICAL_PACKAGE_SCHEMA, + "allowed_schemas": sorted(ALLOWED_PRODUCT_PACKAGE_SCHEMAS), "total_count": 0, "canonical_count": 0, + "valid_schema_count": 0, + "invalid_schema_count": 0, "missing_manifest_count": 0, "missing_publish_dir_count": 0, "missing_processor_count": 0, @@ -1077,8 +1114,13 @@ async def _check_product_packages() -> Dict[str, Any]: status["by_family"][family_key] = int(status["by_family"].get(family_key, 0)) + 1 status["by_engine"][engine_key] = int(status["by_engine"].get(engine_key, 0)) + 1 - if str(package_schema or "").strip() == CANONICAL_PACKAGE_SCHEMA: + schema_key = str(package_schema or "").strip() + if schema_key == CANONICAL_PACKAGE_SCHEMA: status["canonical_count"] += 1 + if schema_key in ALLOWED_PRODUCT_PACKAGE_SCHEMAS: + status["valid_schema_count"] += 1 + else: + status["invalid_schema_count"] += 1 if not str(manifest_path or "").strip() or not os.path.isfile(str(manifest_path)): status["missing_manifest_count"] += 1 if not str(publish_dir or "").strip() or not os.path.isdir(str(publish_dir)): @@ -1097,7 +1139,7 @@ async def _check_product_packages() -> Dict[str, Any]: status["missing_processor_count"] == 0, status["missing_runtime_count"] == 0, status["missing_native_output_count"] == 0, - status["canonical_count"] == status["total_count"], + status["invalid_schema_count"] == 0, ] ) except Exception as exc: @@ -1432,7 +1474,8 @@ async def get_health_status( wsl_runtime_status.get("ok"), pairing_system_status.get("ok"), (not settings.TIMESERIES_ENABLED) or timeseries_result_catalog_status.get("ok"), - (not settings.GAMMA_SBAS_ENABLED) or sbas_insar_result_catalog_status.get("ok"), + (not (settings.GAMMA_SBAS_ENABLED or settings.LANDSAR_SBAS_ENABLED)) + or sbas_insar_result_catalog_status.get("ok"), ] ) diff --git a/backend/app/services/job_handlers.py b/backend/app/services/job_handlers.py index 900e71c..ba44fab 100644 --- a/backend/app/services/job_handlers.py +++ b/backend/app/services/job_handlers.py @@ -91,6 +91,7 @@ JOB_TYPE_GF3_SARSCAPE_SYNC = "GF3_SARSCAPE_SYNC" JOB_TYPE_GF3_SARSCAPE_CLEAN = "GF3_SARSCAPE_CLEAN" JOB_TYPE_ISCE2_RUN = "ISCE2_RUN" JOB_TYPE_PYINT_RUN = "PYINT_RUN" +JOB_TYPE_LANDSAR_RUN = "LANDSAR_RUN" JOB_TYPE_PUBLISH_DINSAR_PRODUCTS = "PUBLISH_DINSAR_PRODUCTS" JOB_TYPE_REBUILD_DINSAR_CATALOG = "REBUILD_DINSAR_CATALOG" JOB_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG" @@ -101,6 +102,7 @@ JOB_TYPE_SBAS_RDC_DEM = "SBAS_RDC_DEM" JOB_TYPE_SBAS_INTERFEROGRAMS = "SBAS_INTERFEROGRAMS" JOB_TYPE_SBAS_IPTA_TIMESERIES = "SBAS_IPTA_TIMESERIES" JOB_TYPE_SBAS_GAMMA_WORKFLOW = "SBAS_GAMMA_WORKFLOW" +JOB_TYPE_SBAS_LANDSAR_WORKFLOW = "SBAS_LANDSAR_WORKFLOW" COPY_ALLOWED_STATUSES = {"PENDING", "IN_PROGRESS", "COMPLETED", "FAILED"} @@ -2602,6 +2604,12 @@ async def _run_wsl_dinsar_production_controller( managed_run_dir = os.path.normpath(execution.output_dir) managed_native_output_dir = os.path.join(managed_run_dir, "native") + if engine_code == "landsar": + landsar_work_root = str(getattr(settings, "LANDSAR_WORK_ROOT", "") or "").strip() + if landsar_work_root: + managed_native_output_dir = os.path.normpath( + os.path.join(landsar_work_root, run_key, "native") + ) managed_work_dir = os.path.join(managed_native_output_dir, "workflow") managed_export_dir = os.path.join(managed_native_output_dir, "export") managed_orbit_output_dir = os.path.join(managed_work_dir, "orbits") @@ -2740,10 +2748,12 @@ async def _run_wsl_dinsar_production_controller( task_result = ((detail.get("task_results") or [{}])[0]) if result else {} try: - if not result or not result.success or not bool(task_result.get("success", result.success if result else False)): + result_error = str(result.error or "").strip() if result else "" + result_success = bool(result.success) if result else False + if not result or not result_success or not bool(task_result.get("success", result_success)): error_message = ( str(task_result.get("error") or "").strip() - or str(result.error or "").strip() + or result_error or run_exception_text or str(task_result.get("stderr_tail") or "").strip() or f"{engine_title} run failed." @@ -2853,19 +2863,19 @@ async def _run_wsl_dinsar_production_controller( await task_service.add_log( job.task_id, "INFO", - f"WSL command [{item_label}]: {task_result.get('command')}", + f"{engine_title} command [{item_label}]: {task_result.get('command')}", ) if task_result.get("stdout_tail"): await task_service.add_log( job.task_id, "INFO", - f"WSL stdout tail [{item_label}]:\n{task_result.get('stdout_tail')}", + f"{engine_title} stdout tail [{item_label}]:\n{task_result.get('stdout_tail')}", ) if task_result.get("stderr_tail"): await task_service.add_log( job.task_id, "WARNING", - f"WSL stderr tail [{item_label}]:\n{task_result.get('stderr_tail')}", + f"{engine_title} stderr tail [{item_label}]:\n{task_result.get('stderr_tail')}", ) publish_result = None @@ -3093,6 +3103,60 @@ async def _handle_pyint_run(job: SystemJobORM) -> None: ) +async def _handle_landsar_run(job: SystemJobORM) -> None: + production_run_id = str((job.payload or {}).get("production_run_id") or "").strip() + if production_run_id: + try: + await _run_wsl_dinsar_production_controller( + job, + engine_code="landsar", + engine_title="LandSAR", + fallback_timeout_seconds=int(getattr(settings, "LANDSAR_DINSAR_TIMEOUT_SECONDS", 0) or 43200), + ) + except Exception as exc: + latest_message = f"LandSAR D-InSAR production controller failed: {exc}" + try: + async with AsyncSessionLocal() as db: + run = await dinsar_production_service.get_run(production_run_id, db) + if run is not None and str(run.status or "").strip().upper() not in {"COMPLETED", "FAILED", "CANCELLED"}: + summary_payload = dict(run.summary_json or {}) + summary_payload["controller_error"] = str(exc) + await dinsar_production_service.finalize_run( + run, + db=db, + status="FAILED", + summary_payload=summary_payload, + latest_message=latest_message, + ) + dinsar_production_service.append_run_log( + run.run_id, + f"[controller-failed] {exc}", + ) + except Exception: + pass + + try: + current_task = await task_service.get_task(job.task_id) + if current_task and current_task.status not in {"COMPLETED", "FAILED", "CANCELLED"}: + await task_service.add_log(job.task_id, "ERROR", latest_message) + await task_service.update_task( + job.task_id, + status="FAILED", + progress=100, + message=latest_message, + ) + except Exception: + pass + raise + return + + await _handle_queued_engine_run( + job, + engine_title="LandSAR", + fallback_timeout_seconds=int(getattr(settings, "LANDSAR_DINSAR_TIMEOUT_SECONDS", 0) or 43200), + ) + + async def _handle_water_geocode(job: SystemJobORM) -> None: """单景 SAR 地理编码 job handler（多视 + 地理编码 + 辐射定标）。""" from .water_service import run_geocoding_workflow, WATER_RESULTS_DIR @@ -4963,6 +5027,187 @@ async def _handle_sbas_gamma_workflow(job: SystemJobORM) -> None: ) +async def _handle_sbas_landsar_workflow(job: SystemJobORM) -> None: + from .landsar_sbas_service import landsar_sbas_service + + payload = job.payload or {} + run_id = str(payload.get("run_id") or "").strip() + auto_select = bool(payload.get("auto_select")) + if not run_id and not auto_select: + raise ValueError("SBAS_LANDSAR_WORKFLOW requires run_id or auto_select") + timeout_seconds = _normalize_positive_int(payload.get("timeout_seconds")) or int( + getattr(settings, "LANDSAR_SBAS_TIMEOUT_SECONDS", 0) or 172800 + ) + + await task_service.start_task( + job.task_id, + message=( + "LandSAR SBAS auto workflow started: selecting LT-1 stack" + if auto_select + else f"LandSAR SBAS workflow started: {run_id}" + ), + ) + await task_service.update_task( + job.task_id, + progress=5, + message=( + "Using Gamma SBAS production-area stack discovery for LandSAR input selection..." + if auto_select + else f"Preparing LandSAR SBAS workflow: {run_id}" + ), + ) + + last_log_at = 0.0 + + loop = asyncio.get_running_loop() + + def _progress(event: dict[str, Any]) -> None: + nonlocal last_log_at + level = str(event.get("level") or "INFO").upper() + message = str(event.get("message") or "").strip() + if not message: + return + now = time.monotonic() + if level == "INFO" and now - last_log_at < 0.2: + return + last_log_at = now + try: + asyncio.run_coroutine_threadsafe( + task_service.add_log(job.task_id, level, message), + loop, + ) + except Exception: + pass + + if auto_select: + selection_request = dict(payload.get("selection_request") or {}) + selection_limit = selection_request.get("limit") + if selection_limit is None: + selection_limit = 30 + await task_service.add_log( + job.task_id, + "INFO", + ( + "LandSAR auto workflow is reusing Gamma stack discovery: " + f"admin_region={selection_request.get('admin_region') or '-'}, " + f"min_scenes={selection_request.get('min_scenes') or '-'}" + ), + ) + try: + detail = await asyncio.to_thread( + landsar_sbas_service.create_run_from_best_stack, + run_label=selection_request.get("run_label"), + source_roots=selection_request.get("source_roots"), + orbit_roots=selection_request.get("orbit_roots"), + min_scenes=selection_request.get("min_scenes"), + discovery_mode=selection_request.get("discovery_mode") or "strict", + admin_region=selection_request.get("admin_region"), + aoi_bbox=selection_request.get("aoi_bbox"), + min_aoi_coverage_ratio=selection_request.get("min_aoi_coverage_ratio", 0.01), + min_common_overlap_ratio=selection_request.get( + "min_common_overlap_ratio", + settings.GAMMA_SBAS_MIN_COMMON_OVERLAP_RATIO, + ), + limit=selection_limit, + dem_path=selection_request.get("dem_path"), + timeout_seconds=selection_request.get("timeout_seconds"), + import_timeout_seconds=selection_request.get("import_timeout_seconds"), + params=dict(selection_request.get("params") or {}), + ) + except Exception as exc: + await task_service.add_log(job.task_id, "ERROR", f"LandSAR auto stack selection failed: {exc}") + raise + + run_id = ( + (detail.get("run") or {}).get("run_id") + or (detail.get("manifest") or {}).get("run_id") + or "" + ) + if not run_id: + raise ValueError("LandSAR auto workflow did not create a run.") + selection = detail.get("selection") or {} + await task_service.add_log( + job.task_id, + "INFO", + ( + "LandSAR auto stack selected: " + f"stack_id={selection.get('selected_stack_id') or '-'}, run_id={run_id}" + ), + ) + await task_service.update_task( + job.task_id, + progress=15, + message=f"LandSAR SBAS Run created from Gamma-selected stack: {run_id}", + ) + + runner_task = asyncio.create_task( + asyncio.to_thread( + landsar_sbas_service.execute_run, + run_id, + timeout_seconds=timeout_seconds, + progress_callback=_progress, + ) + ) + + async def _keepalive() -> None: + while not runner_task.done(): + await asyncio.sleep(30) + await task_service.update_task( + job.task_id, + progress=50, + message=f"LandSAR SBAS workflow is still running: {run_id}", + ) + + keepalive_task = asyncio.create_task(_keepalive()) + try: + result = await runner_task + finally: + keepalive_task.cancel() + try: + await keepalive_task + except asyncio.CancelledError: + pass + + manifest = result.get("manifest") or {} + workflow = manifest.get("workflow") or {} + summary = workflow.get("summary") or {} + status = str(manifest.get("status") or "").strip().upper() + if status not in {"LANDSAR_SBAS_COMPLETED", "LANDSAR_SBAS_PARTIAL"}: + failed_count = summary.get("failed_count") or manifest.get("failed_task_count") or 0 + if status == "LANDSAR_SBAS_RUNTIME_UNSUPPORTED": + unsupported_count = summary.get("unsupported_proid_count") or 0 + configured_proid = manifest.get("proid") or "unknown" + message = ( + f"LandSAR SBAS runtime unsupported: configured proID {configured_proid} is not recognized by " + f"this LandSAR installation. failure_kind=unsupported_proid, " + f"next_stage={manifest.get('next_stage') or 'configure_landsar_sbas_runtime'}, " + f"unsupported_tasks={unsupported_count}" + ) + await task_service.add_log(job.task_id, "ERROR", message) + await task_service.update_task( + job.task_id, + status="FAILED", + progress=100, + message=message, + ) + raise RuntimeError(message) + raise RuntimeError( + "LandSAR SBAS workflow failed: " + f"status={status or 'UNKNOWN'}, failed={failed_count}" + ) + + await task_service.update_task( + job.task_id, + status="COMPLETED", + progress=100, + message=( + f"LandSAR SBAS workflow {status.lower()}: " + f"completed={summary.get('completed_count', 0)}, " + f"failed={summary.get('failed_count', 0)}" + ), + ) + + _HANDLERS = { JOB_TYPE_SCAN_DATA: _handle_scan_data, JOB_TYPE_SCAN_ASSET_INVENTORY: _handle_scan_asset_inventory, @@ -4993,6 +5238,7 @@ _HANDLERS = { JOB_TYPE_IDL_RUN_DINSAR: _handle_idl_run_dinsar, JOB_TYPE_ISCE2_RUN: _handle_isce2_run, JOB_TYPE_PYINT_RUN: _handle_pyint_run, + JOB_TYPE_LANDSAR_RUN: _handle_landsar_run, JOB_TYPE_WATER_GEOCODE: _handle_water_geocode, JOB_TYPE_SAR_SCENE_PREPROCESS: _handle_sar_scene_preprocess, JOB_TYPE_WATER_FLOOD: _handle_water_flood, @@ -5009,6 +5255,7 @@ _HANDLERS = { JOB_TYPE_SBAS_INTERFEROGRAMS: _handle_sbas_interferograms, JOB_TYPE_SBAS_IPTA_TIMESERIES: _handle_sbas_ipta_timeseries, JOB_TYPE_SBAS_GAMMA_WORKFLOW: _handle_sbas_gamma_workflow, + JOB_TYPE_SBAS_LANDSAR_WORKFLOW: _handle_sbas_landsar_workflow, } diff --git a/backend/app/services/landsar_sbas_service.py b/backend/app/services/landsar_sbas_service.py new file mode 100644 index 0000000..669951d --- /dev/null +++ b/backend/app/services/landsar_sbas_service.py @@ -0,0 +1,2293 @@ +from __future__ import annotations + +import json +import os +import queue +import re +import shutil +import subprocess +import threading +import time +import hashlib +from datetime import datetime +from pathlib import Path +from typing import Any, Callable, Optional + +from ..config import settings +from ..dinsar_engines.landsar_engine import ( + LandsarEngine, + _collect_tail, + _decode_line, + _extract_date, + _find_dll, + _landsar_process_env, + _norm_path, + _path_search_dirs, + _summarize_landsar_failure, +) + + +SBAS_PROID = str(settings.LANDSAR_SBAS_PROID or "280039").strip() or "280039" +SBAS_PROCESS_NAME = str(settings.LANDSAR_SBAS_PROCESS_NAME or "SBAS Stream").strip() or "SBAS Stream" +IMPORT_PROID = "100016" +PROCESSOR_CODE = "landsar_sbas" +PROFILE_CODE = "lt1_landsar_sbas" +ENGINE_CODE = "landsar" +WORKFLOW_CODE = "sbas_insar" +_DEFAULT_MIN_COMMON_OVERLAP_RATIO = 0.30 + +_SAFE_NAME_RE = re.compile(r"[^0-9A-Za-z._-]+") +_SUCCESS_RE = re.compile(r"(success|成功)", re.IGNORECASE) +_UNSUPPORTED_PROID_RE = re.compile(r"(Cannot read this ID|Unknown ID)", re.IGNORECASE) +_SBAS_EXTRA_DLLS = ( + "SAR_InSAR_MTInSARModel.dll", + "SAR_InSAR_PSInSAR_CSU.dll", + "SAR_InSAR_MBCP_MTInSARModel.dll", +) + +def _effective_min_common_overlap_ratio(value: Any) -> float: + try: + requested = float(value or 0.0) + except (TypeError, ValueError): + requested = 0.0 + try: + configured = float(settings.GAMMA_SBAS_MIN_COMMON_OVERLAP_RATIO or _DEFAULT_MIN_COMMON_OVERLAP_RATIO) + except (TypeError, ValueError): + configured = _DEFAULT_MIN_COMMON_OVERLAP_RATIO + requested = min(1.0, max(0.0, requested)) + configured = min(1.0, max(0.0, configured)) + return max(requested, configured) + + +_DEFAULT_PARAMS: dict[str, Any] = { + "dem_data_type": 1, + "dem_format": 4, + "do_select_intf": 1, + "intf_method": 0, + "perp_baseline": 200, + "time_baseline": 300, + "doppler_baseline": 100, + "do_multilook": 1, + "multi_pass": 0, + "az_looks": 3, + "rg_looks": 3, + "do_select_points": 1, + "da_threshold": 0.25, + "intensity_threshold": 0.0, + "calibration_method": 0, + "calibration_threshold": 0.4, + "fine_reg_window": 128, + "resample_factor": 2, + "do_coherent_intf": 1, + "do_coherent_diff": 1, + "remove_trend_phase": 0, + "do_build_network": 1, + "network_type": 0, + "max_arc_distance": 1000, + "do_arc_solve": 1, + "solve_method": 0, + "max_temporal_coh": 0.7, + "do_network_adjust": 1, + "ref_point_index": 0, + "do_spatial_filter": 1, + "spatial_filter_dist": 1000, + "do_phase_unwrap": 1, + "unwrap_ref_index": 0, + "do_nonlinear_deform": 1, + "time_filter_threshold": 0.3, + "do_deform_integrate": 1, + "do_los_output": 1, + "gen_vector_map": 0, + "gen_pre_raster": 0, + "gen_post_raster": 1, + "post_raster_res": 0, + "window_size": 0, +} + +_INT_PARAM_KEYS = { + "dem_data_type", + "dem_format", + "do_select_intf", + "intf_method", + "perp_baseline", + "time_baseline", + "doppler_baseline", + "do_multilook", + "multi_pass", + "az_looks", + "rg_looks", + "do_select_points", + "calibration_method", + "fine_reg_window", + "resample_factor", + "do_coherent_intf", + "do_coherent_diff", + "remove_trend_phase", + "do_build_network", + "network_type", + "max_arc_distance", + "do_arc_solve", + "solve_method", + "do_network_adjust", + "ref_point_index", + "do_spatial_filter", + "spatial_filter_dist", + "do_phase_unwrap", + "unwrap_ref_index", + "do_nonlinear_deform", + "do_deform_integrate", + "do_los_output", + "gen_vector_map", + "gen_pre_raster", + "gen_post_raster", + "post_raster_res", + "window_size", +} + +_FLOAT_PARAM_KEYS = { + "da_threshold", + "intensity_threshold", + "calibration_threshold", + "max_temporal_coh", + "time_filter_threshold", +} + + +def _utc_text(value: datetime | None = None) -> str: + return (value or datetime.utcnow()).isoformat(timespec="seconds") + "Z" + + +def _safe_name(value: Any, fallback: str = "task") -> str: + text = _SAFE_NAME_RE.sub("_", str(value or "").strip()).strip("._-") + return text or fallback + + +def _coerce_bool(value: Any) -> bool: + if isinstance(value, bool): + return value + text = str(value or "").strip().lower() + if text in {"1", "true", "yes", "y", "on"}: + return True + if text in {"0", "false", "no", "n", "off", ""}: + return False + return bool(value) + + +def _read_json(path: Path) -> dict[str, Any]: + with path.open("r", encoding="utf-8") as fp: + payload = json.load(fp) + return payload if isinstance(payload, dict) else {} + + +def _write_json(path: Path, payload: dict[str, Any]) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8") + return path + + +def _short_hash(value: Any, length: int = 10) -> str: + digest = hashlib.sha1(str(value or "").encode("utf-8", errors="ignore")).hexdigest() + return digest[: max(6, int(length or 10))] + + +def _split_config_paths(value: str) -> list[str]: + return [ + part.strip().strip('"').strip("'") + for part in str(value or "").replace(";", ",").split(",") + if part.strip().strip('"').strip("'") + ] + + +def format_stack_label(stack_manifest: dict[str, Any], fallback: str) -> str: + stack = stack_manifest.get("stack") or {} + dates = stack_manifest.get("dates") or [] + parts = [ + stack.get("satellite") or "LT1", + stack.get("orbit_direction"), + f"relOrbit {stack.get('relative_orbit')}" if stack.get("relative_orbit") else None, + dates[0] if dates else None, + ] + label = " ".join(str(item).strip() for item in parts if str(item or "").strip()) + return label or fallback + + +def _normalize_path_items(value: Any) -> list[str]: + if value is None: + return [] + if isinstance(value, str): + raw_items = re.split(r"[\r\n;,]+", value) + else: + raw_items = list(value) + return [_norm_path(item) for item in raw_items if str(item or "").strip()] + + +def count_landsar_slc_files(input_data_dir: str) -> tuple[int, list[dict[str, str]]]: + input_dir = _norm_path(input_data_dir) + if not os.path.isdir(input_dir): + return 0, [] + pairs: list[dict[str, str]] = [] + for xml_path in sorted(Path(input_dir).glob("LT1*_SLC.xml"), key=lambda item: item.name.lower()): + tif_path = xml_path.with_name(xml_path.name.replace("_SLC.xml", "_SLC.tif")) + if not tif_path.is_file(): + tif_path = xml_path.with_name(xml_path.name.replace("_SLC.xml", "_SLC.tiff")) + if not tif_path.is_file(): + continue + pairs.append( + { + "date": _extract_date(xml_path.name), + "xml": _norm_path(xml_path), + "tif": _norm_path(tif_path), + "name": xml_path.stem, + } + ) + return len(pairs), pairs + + +def _generate_sbas_param_file( + filepath: str, + *, + slc_folder: str, + dem_path: str, + output_dir: str, + project_name: str, + params: dict[str, Any], +) -> str: + def p(key: str, default: Any) -> Any: + return params.get(key, default) + + lines = [ + SBAS_PROCESS_NAME, + f"ID {SBAS_PROID}", + "", + "输入输出数据设置", + f"SLC文件夹路径 <{slc_folder}>", + f"参考DEM数据地址 <{dem_path}>", + f"参考DEM数据类型_0文件夹_1文件 {p('dem_data_type', 1)}", + f"参考DEM数据格式_0strm1*1deg/1strm5*5deg/2aster/3tandem/4coper {p('dem_format', 4)}", + f"项目名称 {project_name}", + f"项目输出根目录 <{output_dir}>", + "", + f"是否执行选取干涉对 {p('do_select_intf', 1)}", + f"干涉对选取方法_0single_1prim {p('intf_method', 0)}", + f"干涉对垂直基线阈值 {p('perp_baseline', 200)}", + f"干涉对时间基线阈值 {p('time_baseline', 300)}", + f"干涉对多普勒基线阈值 {p('doppler_baseline', 100)}", + "", + f"是否执行RSLC多视 {p('do_multilook', 1)}", + f"是否需要做多次 {p('multi_pass', 0)}", + f"方位向多视数 {p('az_looks', 3)}", + f"距离向多视数 {p('rg_looks', 3)}", + "", + f"是否执行选取相干点 {p('do_select_points', 1)}", + f"振幅差阈值最大 {p('da_threshold', 0.25)}", + f"强度最小阈值 {p('intensity_threshold', 0.0)}", + f"定标方法 {p('calibration_method', 0)}", + f"相干点强度定标阈值 {p('calibration_threshold', 0.4)}", + f"精配准窗口尺寸 {p('fine_reg_window', 128)}", + f"重采样因子 {p('resample_factor', 2)}", + "", + f"是否执行相干点干涉 {p('do_coherent_intf', 1)}", + f"是否执行相干点差分干涉 {p('do_coherent_diff', 1)}", + f"是否去除趋势性相位 {p('remove_trend_phase', 0)}", + "", + f"是否执行相干点网络构建 {p('do_build_network', 1)}", + f"网型_0delaunay_1star_2free {p('network_type', 0)}", + f"弧段最大距离阈值 {p('max_arc_distance', 1000)}", + "", + f"是否执行弧段模型解算 {p('do_arc_solve', 1)}", + f"解算方法_0periodogram_1lsm {p('solve_method', 0)}", + f"弧段最大时域相干性阈值 {p('max_temporal_coh', 0.7)}", + "", + f"是否执行全体点网平差 {p('do_network_adjust', 1)}", + f"参考点索引号 {p('ref_point_index', 0)}", + "", + f"是否执行相位矢量数据空间滤波 {p('do_spatial_filter', 1)}", + f"空间滤波距离阈值 {p('spatial_filter_dist', 1000)}", + "", + f"是否执行相干点相位解缠 {p('do_phase_unwrap', 1)}", + f"相位解缠参考点索引号 {p('unwrap_ref_index', 0)}", + "", + f"是否执行相干点非线性形变提取 {p('do_nonlinear_deform', 1)}", + f"时间滤波时间阈值 {p('time_filter_threshold', 0.3)}", + "", + f"是否执行相干点形变整合 {p('do_deform_integrate', 1)}", + f"是否执行LOS向时序文件输出 {p('do_los_output', 1)}", + f"是否生成矢量图 {p('gen_vector_map', 0)}", + f"是否生成编码前栅格图 {p('gen_pre_raster', 0)}", + f"是否生成编码后栅格图 {p('gen_post_raster', 1)}", + f"编码后分辨率 {p('post_raster_res', 0)}", + f"窗口大小 {p('window_size', 0)}", + ] + target = _norm_path(filepath) + os.makedirs(os.path.dirname(target), exist_ok=True) + with open(target, "w", encoding="utf-8") as fp: + fp.write("\n".join(lines) + "\n") + return target + + +def _generate_lt1_multiscene_import_param_file( + filepath: str, + *, + scene_dirs: list[str], + export_dir: str, + sat_mode: str = "MONO", +) -> str: + lines = [ + "卫星数据导入LT-1", + f"处理编号 {IMPORT_PROID}", + "设置数据导入形式_0文件夹导入_1数据导入 文件夹导入", + "读取成像参数文件_0否_1是 1", + "读取SLC数据文件_0否_1是 1", + "文件夹导入标识 TRUE", + f"文件夹导入个数 {len(scene_dirs)}", + ] + for index, scene_dir in enumerate(scene_dirs, start=1): + lines.append(f"文件夹{index}路径 <{scene_dir}>") + lines.extend( + [ + "数据导入 FALSE", + f"输入卫星数据格式 {sat_mode}", + "输入主影像成像参数文件路径 <>", + "输入主影像SLC数据文件路径 <>", + "输入主影像RPB数据文件路径 <>", + "输入辅影像成像参数文件路径 <>", + "输入辅影像SLC数据文件路径 <>", + "输入辅影像RPB数据文件路径 <>", + "设置数据导出目标路径_0原目录_1新目录 1", + f"设置输出文件目录 <{export_dir}>", + ] + ) + target = _norm_path(filepath) + os.makedirs(os.path.dirname(target), exist_ok=True) + with open(target, "w", encoding="utf-8") as fp: + fp.write("\n".join(lines) + "\n") + return target + + +class LandsarSbasService: + def __init__(self) -> None: + self.work_root = Path(settings.LANDSAR_SBAS_WORK_ROOT or Path(settings.LANDSAR_WORK_ROOT) / "sbas") + self.product_root = Path(settings.LANDSAR_SBAS_PRODUCT_ROOT or Path(settings.TIMESERIES_PRODUCT_DIR) / "sbas_landsar") + self._engine = LandsarEngine() + + def _allocate_work_run_root(self, run_id: str, created_at: datetime) -> Path: + short_parent = self.work_root / "x" + short_parent.mkdir(parents=True, exist_ok=True) + base_name = f"r{created_at.strftime('%y%m%d%H%M%S')}_{_short_hash(run_id, 8)}" + for index in range(1000): + suffix = "" if index == 0 else f"_{index}" + candidate = short_parent / f"{base_name}{suffix}" + if not candidate.exists(): + candidate.mkdir(parents=True, exist_ok=True) + return candidate + raise RuntimeError("Unable to allocate a short LandSAR SBAS work directory.") + + def get_run_root(self) -> str: + root = self.product_root / "runs" + root.mkdir(parents=True, exist_ok=True) + return _norm_path(root) + + def configured_run_root(self) -> str: + return _norm_path(self.product_root / "runs") + + def get_capabilities(self) -> dict[str, Any]: + availability = self.check_available() + return { + "workflow_code": WORKFLOW_CODE, + "processor_code": PROCESSOR_CODE, + "profile_code": PROFILE_CODE, + "engine_code": ENGINE_CODE, + "proid": SBAS_PROID, + "process_name": SBAS_PROCESS_NAME, + "enabled": bool(settings.LANDSAR_SBAS_ENABLED), + "available": availability["available"], + "status": availability["status"], + "message": availability["message"], + "checks": availability["checks"], + "work_root": str(self.work_root), + "product_root": str(self.product_root), + "source_roots": _split_config_paths(settings.LANDSAR_SBAS_SOURCE_ROOTS), + "default_dem_path": self.default_dem_path, + "default_timeout_seconds": settings.LANDSAR_SBAS_TIMEOUT_SECONDS, + "min_scenes": settings.LANDSAR_SBAS_MIN_SCENES, + "min_common_overlap_ratio": _effective_min_common_overlap_ratio(None), + "params_schema": self.params_schema(), + } + + @property + def default_dem_path(self) -> str: + dem_path = str(settings.LANDSAR_SBAS_DEM_PATH or settings.LANDSAR_DEM_PATH or "").strip() + return _norm_path(dem_path) if dem_path else "" + + def resolve_lt1_scene_dir(self, file_path: Any) -> str: + path = Path(_norm_path(file_path)) + candidates = [path] + if path.is_file(): + candidates.insert(0, path.parent) + for candidate in candidates: + if self._looks_like_lt1_scene_dir(candidate): + return _norm_path(candidate) + return "" + + @staticmethod + def _looks_like_lt1_scene_dir(path: Path) -> bool: + if not path.is_dir(): + return False + has_meta = any(path.glob("*.meta.xml")) + has_tif = bool(list(path.glob("*.tif")) + list(path.glob("*.tiff"))) + return has_meta and has_tif + + def params_schema(self) -> dict[str, Any]: + return { + "dem_path": {"label": "DEM 文件", "type": "string", "default": self.default_dem_path}, + "dem_format": {"label": "DEM 格式", "type": "number", "default": 4, "min": 0, "max": 4}, + "intf_method": {"label": "干涉对方法", "type": "select", "default": 0, "options": [{"value": 0, "label": "single"}, {"value": 1, "label": "prim"}]}, + "perp_baseline": {"label": "垂直基线阈值", "type": "number", "default": 200}, + "time_baseline": {"label": "时间基线阈值", "type": "number", "default": 300}, + "doppler_baseline": {"label": "多普勒基线阈值", "type": "number", "default": 100}, + "az_looks": {"label": "方位向多视", "type": "number", "default": 3, "min": 1}, + "rg_looks": {"label": "距离向多视", "type": "number", "default": 3, "min": 1}, + "da_threshold": {"label": "DA 阈值", "type": "number", "default": 0.25, "min": 0, "max": 1}, + "network_type": {"label": "网络类型", "type": "select", "default": 0, "options": [{"value": 0, "label": "Delaunay"}, {"value": 1, "label": "Star"}, {"value": 2, "label": "Free"}]}, + "solve_method": {"label": "解算方法", "type": "select", "default": 0, "options": [{"value": 0, "label": "Periodogram"}, {"value": 1, "label": "LSM"}]}, + "gen_vector_map": {"label": "生成矢量图", "type": "boolean", "default": False}, + "gen_post_raster": {"label": "生成编码后栅格", "type": "boolean", "default": True}, + } + + def check_available(self) -> dict[str, Any]: + engine_availability = self._engine.check_available() + console_path = self._engine._console_exe + home = self._engine._home + search_dirs = _path_search_dirs(os.path.dirname(_norm_path(console_path)), home) + dll_checks = [] + for name in _SBAS_EXTRA_DLLS: + path = _find_dll(name, search_dirs) + dll_checks.append( + { + "name": name, + "ok": bool(path), + "detail": path or "missing", + "optional": name != "SAR_InSAR_MTInSARModel.dll", + } + ) + required_extra_ok = all(item["ok"] or item.get("optional") for item in dll_checks) + enabled = bool(settings.LANDSAR_SBAS_ENABLED) + available = enabled and bool(engine_availability.available) and required_extra_ok + checks = [ + {"name": "LANDSAR_SBAS_ENABLED", "ok": enabled, "detail": str(enabled).lower()}, + *engine_availability.checks, + *dll_checks, + ] + message = "LandSAR SBAS console is available." if available else engine_availability.message + if bool(engine_availability.available) and not required_extra_ok: + missing = [item["name"] for item in dll_checks if not item["ok"] and not item.get("optional")] + message = f"LandSAR SBAS dependencies are missing: {', '.join(missing)}" + if not enabled: + message = "LandSAR SBAS is disabled." + return { + "available": available, + "status": "ok" if available else "unavailable", + "message": message, + "checks": checks, + } + + def normalize_params(self, extra: Optional[dict[str, Any]] = None) -> dict[str, Any]: + raw_extra = dict(extra or {}) + normalized = dict(_DEFAULT_PARAMS) + for key, value in raw_extra.items(): + if key in {"dem_path", "project_name"}: + continue + if key not in normalized: + continue + if value is None or (isinstance(value, str) and not value.strip()): + continue + if key in _INT_PARAM_KEYS: + try: + normalized[key] = int(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{key} must be an integer.") from exc + elif key in _FLOAT_PARAM_KEYS: + try: + normalized[key] = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{key} must be a number.") from exc + else: + normalized[key] = value + for key in ("gen_vector_map", "gen_pre_raster", "gen_post_raster", "do_los_output"): + if key in raw_extra: + normalized[key] = 1 if _coerce_bool(raw_extra.get(key)) else 0 + return normalized + + def _iter_candidate_task_dirs(self, root_dir: str) -> list[str]: + root = _norm_path(root_dir) + if not os.path.isdir(root): + return [] + if os.path.basename(root).lower() == "input_data": + parent = os.path.dirname(root) + return [_norm_path(parent)] if parent else [] + if os.path.isdir(os.path.join(root, "Input_Data")): + return [root] + return [ + _norm_path(entry.path) + for entry in sorted(os.scandir(root), key=lambda item: item.name.lower()) + if entry.is_dir() + and entry.name.lower().startswith("task_") + and os.path.isdir(os.path.join(entry.path, "Input_Data")) + ] + + def validate_root_dir( + self, + root_dir: str, + *, + min_scenes: int | None = None, + num_to_process: int = 0, + rerun_mode: str = "rerun_all", + ) -> dict[str, Any]: + normalized_root = _norm_path(root_dir) + if not normalized_root or not os.path.isdir(normalized_root): + raise ValueError(f"LandSAR SBAS root_dir does not exist or is not a directory: {root_dir}") + min_count = max(3, int(min_scenes or settings.LANDSAR_SBAS_MIN_SCENES or 3)) + candidates = self._iter_candidate_task_dirs(normalized_root) + valid: list[dict[str, Any]] = [] + invalid: list[dict[str, Any]] = [] + skipped_completed = 0 + for task_dir in candidates: + task_name = os.path.basename(task_dir) + input_dir = os.path.join(task_dir, "Input_Data") + slc_count, pairs = count_landsar_slc_files(input_dir) + if slc_count < min_count: + invalid.append( + { + "task_name": task_name, + "task_dir": task_dir, + "reason": f"Input_Data SLC pair count is {slc_count}, expected >= {min_count}", + } + ) + continue + if rerun_mode == "unfinished_only" and self._has_completed_output(os.path.join(task_dir, "Output_Data")): + skipped_completed += 1 + continue + dates = [item.get("date") for item in pairs if item.get("date")] + valid.append( + { + "task_name": task_name, + "task_dir": task_dir, + "input_data_dir": input_dir, + "slc_count": slc_count, + "dates": dates, + "date_start": dates[0] if dates else None, + "date_end": dates[-1] if dates else None, + "scenes": pairs, + } + ) + if num_to_process and num_to_process > 0: + valid = valid[: int(num_to_process)] + return { + "schema": "insar.landsar-sbas-task-discovery/v1", + "root_dir": normalized_root, + "min_scenes": min_count, + "candidate_count": len(candidates), + "task_count": len(valid), + "selected_task_count": len(valid), + "skipped_completed_count": skipped_completed, + "invalid_candidates": invalid, + "items": valid, + } + + def discover_tasks( + self, + *, + root_dir: str | None = None, + min_scenes: int | None = None, + num_to_process: int = 0, + rerun_mode: str = "rerun_all", + ) -> dict[str, Any]: + root_text = str(root_dir or "").strip() + roots = [root_text] if root_text else _split_config_paths(settings.LANDSAR_SBAS_SOURCE_ROOTS) + items: list[dict[str, Any]] = [] + invalid: list[dict[str, Any]] = [] + errors: list[dict[str, str]] = [] + for root in roots: + try: + result = self.validate_root_dir( + root, + min_scenes=min_scenes, + num_to_process=num_to_process, + rerun_mode=rerun_mode, + ) + items.extend(result.get("items") or []) + invalid.extend(result.get("invalid_candidates") or []) + except ValueError as exc: + errors.append({"root_dir": root, "error": str(exc)}) + if num_to_process and num_to_process > 0: + items = items[: int(num_to_process)] + return { + "schema": "insar.landsar-sbas-task-discovery/v1", + "generated_at": _utc_text(), + "root_dir": root_text, + "source_roots": roots, + "min_scenes": max(3, int(min_scenes or settings.LANDSAR_SBAS_MIN_SCENES or 3)), + "task_count": len(items), + "items": items, + "invalid_candidates": invalid, + "errors": errors, + } + + def materialize_stack( + self, + *, + source_dirs: Any, + dest_root: str | None = None, + task_name: str | None = None, + min_scenes: int | None = None, + max_scenes: int = 0, + overwrite: bool = False, + dry_run: bool = False, + ) -> dict[str, Any]: + sources = _normalize_path_items(source_dirs) + if not sources: + raise ValueError("LandSAR SBAS stack materialization requires at least one source directory.") + target_root_text = _norm_path(dest_root or (_split_config_paths(settings.LANDSAR_SBAS_SOURCE_ROOTS)[:1] or [self.work_root])[0]) + if not target_root_text: + raise ValueError("LandSAR SBAS stack materialization requires dest_root.") + min_count = max(3, int(min_scenes or settings.LANDSAR_SBAS_MIN_SCENES or 3)) + scenes, scan_errors, duplicate_count, missing_tif_count = self._collect_slc_scenes(sources) + if max_scenes and int(max_scenes) > 0: + scenes = scenes[: int(max_scenes)] + if len(scenes) < min_count: + raise ValueError(f"Only {len(scenes)} valid LT1 SLC scenes found, expected >= {min_count}.") + + dates = [scene.get("date") for scene in scenes if scene.get("date")] + date_start = dates[0] if dates else None + date_end = dates[-1] if dates else None + default_name = "Task_LandSAR_SBAS" + if date_start and date_end: + default_name = f"Task_{date_start}_{date_end}_SBAS" + normalized_task_name = _safe_name(task_name or default_name, default_name) + if not normalized_task_name.lower().startswith("task_"): + normalized_task_name = f"Task_{normalized_task_name}" + + target_root = Path(target_root_text) + task_dir = target_root / normalized_task_name + input_dir = task_dir / "Input_Data" + existing_files = list(input_dir.glob("*")) if input_dir.is_dir() else [] + if existing_files and not overwrite and not dry_run: + raise ValueError(f"Target Input_Data is not empty: {input_dir}. Enable overwrite to reuse it.") + + copied_files: list[dict[str, str]] = [] + skipped_existing = 0 + scene_records: list[dict[str, Any]] = [] + if not dry_run: + input_dir.mkdir(parents=True, exist_ok=True) + + for scene in scenes: + target_files: list[str] = [] + for source_file in scene["files"]: + target_file = input_dir / source_file.name + if not dry_run: + if target_file.exists() and not overwrite: + skipped_existing += 1 + else: + target_file.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source_file, target_file) + copied_files.append({"source": str(source_file), "target": str(target_file)}) + target_files.append(str(target_file)) + scene_records.append( + { + "date": scene.get("date"), + "name": scene.get("name"), + "source_dir": scene.get("source_dir"), + "source_xml": scene.get("xml"), + "source_tif": scene.get("tif"), + "target_files": target_files, + } + ) + + slc_count = len(scenes) if dry_run else count_landsar_slc_files(str(input_dir))[0] + task_item = { + "task_name": normalized_task_name, + "task_dir": str(task_dir), + "input_data_dir": str(input_dir), + "slc_count": slc_count, + "dates": dates, + "date_start": date_start, + "date_end": date_end, + "scenes": scene_records, + } + manifest = { + "schema": "insar.landsar-sbas-stack-materialization/v1", + "generated_at": _utc_text(), + "dry_run": dry_run, + "source_dirs": sources, + "dest_root": str(target_root), + "task": task_item, + "min_scenes": min_count, + "max_scenes": int(max_scenes or 0), + "overwrite": bool(overwrite), + "duplicate_scene_count": duplicate_count, + "missing_tif_count": missing_tif_count, + "scan_errors": scan_errors, + "copied_file_count": len(copied_files), + "skipped_existing_count": skipped_existing, + "copied_files": copied_files[:200], + } + if not dry_run: + _write_json(task_dir / "landsar_sbas_stack_manifest.json", manifest) + return { + "schema": "insar.landsar-sbas-stack-materialization-result/v1", + "ready": slc_count >= min_count, + "task": task_item, + "task_name": normalized_task_name, + "task_dir": str(task_dir), + "input_data_dir": str(input_dir), + "slc_count": slc_count, + "copied_file_count": len(copied_files), + "duplicate_scene_count": duplicate_count, + "missing_tif_count": missing_tif_count, + "scan_errors": scan_errors, + "manifest_path": None if dry_run else str(task_dir / "landsar_sbas_stack_manifest.json"), + } + + def _collect_slc_scenes(self, source_dirs: list[str]) -> tuple[list[dict[str, Any]], list[dict[str, str]], int, int]: + scenes_by_key: dict[str, dict[str, Any]] = {} + errors: list[dict[str, str]] = [] + duplicate_count = 0 + missing_tif_count = 0 + for source_dir in source_dirs: + if not os.path.isdir(source_dir): + errors.append({"source_dir": source_dir, "error": "source directory does not exist"}) + continue + for current_dir, _, filenames in os.walk(source_dir): + for filename in filenames: + upper = filename.upper() + if not (upper.startswith("LT1") and upper.endswith("_SLC.XML")): + continue + xml_path = Path(current_dir) / filename + tif_path = xml_path.with_name(f"{xml_path.stem}.tif") + if not tif_path.is_file(): + tif_path = xml_path.with_name(f"{xml_path.stem}.tiff") + if not tif_path.is_file(): + missing_tif_count += 1 + continue + key = xml_path.stem.lower() + if key in scenes_by_key: + duplicate_count += 1 + continue + files = sorted( + [path for path in xml_path.parent.glob(f"{xml_path.stem}.*") if path.is_file()], + key=lambda item: item.name.lower(), + ) + scenes_by_key[key] = { + "date": _extract_date(xml_path.name), + "name": xml_path.stem, + "source_dir": str(xml_path.parent), + "xml": str(xml_path), + "tif": str(tif_path), + "files": files, + } + return ( + sorted(scenes_by_key.values(), key=lambda item: (item.get("date") or "", item.get("name") or "")), + errors, + duplicate_count, + missing_tif_count, + ) + + def import_stack_scenes( + self, + *, + scenes: list[dict[str, Any]], + dest_root: str | None = None, + task_name: str | None = None, + min_scenes: int | None = None, + sat_mode: str = "MONO", + overwrite: bool = False, + timeout_seconds: int | None = None, + progress_callback: Optional[Callable[[dict[str, Any]], None]] = None, + ) -> dict[str, Any]: + min_count = max(3, int(min_scenes or settings.LANDSAR_SBAS_MIN_SCENES or 3)) + normalized_scenes: list[dict[str, Any]] = [] + seen_dirs: set[str] = set() + for scene in scenes: + scene_dir = self._resolve_import_scene_dir(scene) + if not scene_dir: + continue + if scene_dir in seen_dirs: + continue + seen_dirs.add(scene_dir) + normalized_scenes.append({**scene, "scene_dir": scene_dir}) + normalized_scenes.sort(key=lambda item: str(item.get("date") or item.get("imaging_date") or "")) + if len(normalized_scenes) < min_count: + raise ValueError(f"Only {len(normalized_scenes)} database LT-1 scenes selected, expected >= {min_count}.") + + target_root_text = _norm_path(dest_root or (_split_config_paths(settings.LANDSAR_SBAS_SOURCE_ROOTS)[:1] or [self.work_root])[0]) + dates = [str(scene.get("date") or scene.get("imaging_date") or "")[:8] for scene in normalized_scenes if str(scene.get("date") or scene.get("imaging_date") or "").strip()] + date_start = dates[0] if dates else None + date_end = dates[-1] if dates else None + default_name = f"Task_{date_start}_{date_end}_SBAS" if date_start and date_end else "Task_LandSAR_DB_SBAS" + normalized_task_name = _safe_name(task_name or default_name, default_name) + if not normalized_task_name.lower().startswith("task_"): + normalized_task_name = f"Task_{normalized_task_name}" + + task_dir = Path(target_root_text) / normalized_task_name + input_dir = task_dir / "Input_Data" + output_dir = task_dir / "Output_Data" + if input_dir.is_dir() and any(input_dir.iterdir()) and not overwrite: + existing_count, _ = count_landsar_slc_files(str(input_dir)) + if existing_count >= min_count: + return self._db_import_result( + task_dir=task_dir, + input_dir=input_dir, + scenes=normalized_scenes, + status="LANDSAR_SBAS_INPUT_READY", + returncode=0, + stdout_text="", + error="", + command=[], + param_file="", + skipped=True, + min_scenes=min_count, + sat_mode=sat_mode, + ) + raise ValueError(f"Target Input_Data exists but is incomplete: {input_dir}. Enable overwrite to retry.") + input_dir.mkdir(parents=True, exist_ok=True) + output_dir.mkdir(parents=True, exist_ok=True) + + availability = self.check_available() + if not availability["available"]: + raise ValueError(f"LandSAR import is not available: {availability['message']}") + console_path = self._engine._console_exe + home = self._engine._home + config_ok, config_detail = self._engine._ensure_config_csv() + if not config_ok: + raise ValueError(f"LandSAR config.csv is not ready: {config_detail}") + auth_ok, auth_detail = self._engine._start_auth_server_if_needed() + if not auth_ok: + raise ValueError(f"LandSAR network license server is not ready: {auth_detail}") + + timeout = max(60, int(timeout_seconds or settings.LANDSAR_SBAS_TIMEOUT_SECONDS or 172800)) + param_file = _generate_lt1_multiscene_import_param_file( + str(input_dir / f"{IMPORT_PROID}_timeseries.txt"), + scene_dirs=[scene["scene_dir"] for scene in normalized_scenes], + export_dir=str(input_dir), + sat_mode=str(sat_mode or "MONO").strip() or "MONO", + ) + command = [console_path, param_file] + self._emit(progress_callback, "INFO", f"LandSAR 100016 import started: {normalized_task_name}, scenes={len(normalized_scenes)}") + rc, stdout_text, timed_out = self._run_console( + command, + cwd=home if os.path.isdir(home) else os.path.dirname(console_path), + log_path=str(input_dir / f"{IMPORT_PROID}_timeseries_console.log"), + timeout=timeout, + progress_callback=progress_callback, + task_name=normalized_task_name, + ) + slc_count, _ = count_landsar_slc_files(str(input_dir)) + success = rc == 0 and slc_count >= min_count and self._has_completed_import(str(input_dir)) + error = "" + if timed_out: + error = f"LandSAR import timed out after {timeout}s." + success = False + elif rc != 0: + error = _summarize_landsar_failure(stdout_text, "LandSAR import", rc) + elif slc_count < min_count: + error = f"LandSAR import produced {slc_count} SLC scenes, expected >= {min_count}." + elif not self._has_completed_import(str(input_dir)): + error = "LandSAR import success marker is missing." + + status = "LANDSAR_SBAS_INPUT_READY" if success else "LANDSAR_SBAS_IMPORT_FAILED" + result = self._db_import_result( + task_dir=task_dir, + input_dir=input_dir, + scenes=normalized_scenes, + status=status, + returncode=rc, + stdout_text=stdout_text, + error=error, + command=command, + param_file=param_file, + skipped=False, + min_scenes=min_count, + sat_mode=sat_mode, + ) + if not success: + raise RuntimeError(error or "LandSAR import failed.") + self._emit(progress_callback, "INFO", f"LandSAR 100016 import completed: {normalized_task_name}, SLC={slc_count}") + return result + + def _create_run_from_gamma_stack( + self, + stack_id: str, + *, + sensor_family: str = "LT1", + run_label: str | None = None, + source_roots: list[str] | None = None, + orbit_roots: list[str] | None = None, + min_scenes: int | None = None, + require_orbits: bool = False, + discovery_mode: str = "strict", + admin_region: str | None = None, + aoi_bbox: dict[str, Any] | None = None, + min_aoi_coverage_ratio: float = 0.01, + min_common_overlap_ratio: float | None = None, + dem_path: str | None = None, + timeout_seconds: int | None = None, + import_timeout_seconds: int | None = None, + params: Optional[dict[str, Any]] = None, + task_name: str | None = None, + dest_root: str | None = None, + overwrite_input: bool = False, + created_by: str | None = None, + ) -> dict[str, Any]: + if not bool(settings.LANDSAR_SBAS_ENABLED): + raise ValueError("LandSAR SBAS is disabled.") + normalized_sensor = str(sensor_family or "LT1").strip().upper() + if normalized_sensor != "LT1": + raise ValueError("LandSAR SBAS currently supports LT-1 stacks only.") + + normalized_min = max(3, int(min_scenes or settings.LANDSAR_SBAS_MIN_SCENES or 3)) + min_common_overlap_ratio = _effective_min_common_overlap_ratio(min_common_overlap_ratio) + params_payload = self.normalize_params(params or {}) + normalized_dem = _norm_path(dem_path or self.default_dem_path) + if not normalized_dem or not os.path.isfile(normalized_dem): + raise ValueError(f"LandSAR SBAS DEM file is missing: {normalized_dem or '<empty>'}") + + from .sbas_insar_production_service import sbas_insar_production_service + + audit = sbas_insar_production_service.audit_stack( + stack_id, + sensor_family="LT1", + source_roots=source_roots, + orbit_roots=orbit_roots, + min_scenes=normalized_min, + require_orbits=bool(require_orbits), + discovery_mode=discovery_mode, + admin_region=admin_region, + aoi_bbox=aoi_bbox, + min_aoi_coverage_ratio=min_aoi_coverage_ratio, + min_common_overlap_ratio=min_common_overlap_ratio, + ) + stack_manifest = dict(audit.get("manifest") or {}) + if stack_manifest.get("status") == "BLOCKED": + blockers = "; ".join(str(item) for item in (stack_manifest.get("blockers") or [])) + raise ValueError(f"stack manifest is not ready for LandSAR input import: {blockers or 'blocked'}") + selected_scenes = self._normalize_stack_scenes_for_import(stack_manifest.get("scenes") or []) + if len(selected_scenes) < normalized_min: + raise ValueError(f"Only {len(selected_scenes)} LT-1 scenes selected, expected >= {normalized_min}.") + + dates = sorted({ + str(scene.get("date") or scene.get("imaging_date") or "")[:8] + for scene in selected_scenes + if str(scene.get("date") or scene.get("imaging_date") or "").strip() + }) + created_at = datetime.utcnow() + safe_stack = _safe_name(stack_id, "stack") + run_id = f"landsar_sbas_{created_at.strftime('%Y%m%dT%H%M%S%fZ')}_{safe_stack}" + run_dir = Path(self.get_run_root()) / run_id + work_run_root = self._allocate_work_run_root(run_id, created_at) + native_root = work_run_root / "n" + input_task_root = work_run_root / "i" + publish_root = run_dir / "publish" / "landsar" + for path in (native_root, input_task_root, publish_root): + path.mkdir(parents=True, exist_ok=True) + + default_task_name = ( + f"Task_{dates[0]}_{dates[-1]}_SBAS" + if dates + else f"Task_{safe_stack}_SBAS" + ) + normalized_task_name = _safe_name(task_name or default_task_name, default_task_name) + if not normalized_task_name.lower().startswith("task_"): + normalized_task_name = f"Task_{normalized_task_name}" + + import_dest_root = _norm_path(dest_root or input_task_root) + discovery_params = { + "sensor_family": "LT1", + "source_roots": source_roots, + "orbit_roots": orbit_roots, + "min_scenes": normalized_min, + "require_orbits": bool(require_orbits), + "discovery_mode": discovery_mode, + "admin_region": admin_region, + "aoi_bbox": aoi_bbox, + "min_aoi_coverage_ratio": min_aoi_coverage_ratio, + "min_common_overlap_ratio": min_common_overlap_ratio, + } + manifest = { + "schema": "insar.landsar-sbas-run/v1", + "run_id": run_id, + "run_label": run_label or f"LandSAR SBAS {format_stack_label(stack_manifest, stack_id)}", + "workflow_code": WORKFLOW_CODE, + "processor_code": PROCESSOR_CODE, + "profile_code": PROFILE_CODE, + "engine_code": ENGINE_CODE, + "proid": SBAS_PROID, + "process_name": SBAS_PROCESS_NAME, + "execution_mode": "landsar_stack_selection_import_then_sbas", + "source_mode": "gamma_production_area_stack_selection", + "status": "LANDSAR_SBAS_INPUT_PENDING", + "created_at": _utc_text(created_at), + "created_by": created_by, + "root_dir": import_dest_root, + "run_dir": str(run_dir), + "work_root": str(work_run_root), + "work_root_strategy": "short_landsar_execution_path", + "native_root": str(native_root), + "input_task_root": import_dest_root, + "publish_root": str(publish_root), + "dem_path": normalized_dem, + "params": params_payload, + "timeout_seconds": max(60, int(timeout_seconds or settings.LANDSAR_SBAS_TIMEOUT_SECONDS or 172800)), + "import_timeout_seconds": max(60, int(import_timeout_seconds or timeout_seconds or settings.LANDSAR_SBAS_TIMEOUT_SECONDS or 172800)), + "min_scenes": normalized_min, + "scene_count": len(selected_scenes), + "task_count": 0, + "pair_count": None, + "date_start": dates[0] if dates else None, + "date_end": dates[-1] if dates else None, + "dates": dates, + "tasks": [], + "next_stage": "import_landsar_input", + "input_import": { + "status": "PENDING", + "dest_root": import_dest_root, + "task_name": normalized_task_name, + "overwrite": bool(overwrite_input), + "sat_mode": "MONO", + }, + "source_stack": { + "stack_id": stack_id, + "source_system": "gamma_sbas_production_stack_discovery", + "audit_status": audit.get("status"), + "audit_manifest_path": audit.get("manifest_path"), + "pair_network_path": audit.get("pair_network_path"), + "discovery_params": discovery_params, + "stack": stack_manifest.get("stack") or {}, + "geographic_coverage": stack_manifest.get("geographic_coverage"), + "common_overlap_ratio": stack_manifest.get("common_overlap_ratio"), + "scenes": selected_scenes, + "scene_count": len(selected_scenes), + "dates": dates, + "warnings": stack_manifest.get("warnings") or [], + }, + "geographic_coverage": stack_manifest.get("geographic_coverage"), + } + _write_json(run_dir / "run_manifest.json", manifest) + _write_json( + run_dir / "stack_manifest.json", + { + "schema": "insar.landsar-sbas-stack/v1", + "run_id": run_id, + "status": "READY_FOR_LANDSAR_INPUT_IMPORT", + "source": "gamma_production_area_stack_selection", + "stack_id": stack_id, + "audit_manifest_path": audit.get("manifest_path"), + "dates": dates, + "scene_count": len(selected_scenes), + "geographic_coverage": stack_manifest.get("geographic_coverage"), + "scenes": selected_scenes, + }, + ) + _write_json( + run_dir / "workflow_summary.json", + { + "schema": "insar.landsar-sbas-workflow-summary/v1", + "run_id": run_id, + "process_name": SBAS_PROCESS_NAME, + "ready": False, + "status": "INPUT_PENDING", + "task_count": 0, + "completed_count": 0, + "failed_count": 0, + }, + ) + return self.get_run_detail(run_id) + + def select_best_stack_for_area( + self, + *, + source_roots: list[str] | None = None, + orbit_roots: list[str] | None = None, + min_scenes: int | None = None, + discovery_mode: str = "strict", + admin_region: str | None = None, + aoi_bbox: dict[str, Any] | None = None, + min_aoi_coverage_ratio: float = 0.01, + min_common_overlap_ratio: float | None = None, + limit: int = 30, + ) -> dict[str, Any]: + normalized_min = max(3, int(min_scenes or settings.LANDSAR_SBAS_MIN_SCENES or 3)) + min_common_overlap_ratio = _effective_min_common_overlap_ratio(min_common_overlap_ratio) + from .sbas_insar_production_service import sbas_insar_production_service + + discovery = sbas_insar_production_service.discover_stacks( + sensor_family="LT1", + source_roots=source_roots, + orbit_roots=orbit_roots, + min_scenes=normalized_min, + require_orbits=False, + include_scenes=False, + limit=0, + discovery_mode=discovery_mode, + admin_region=admin_region, + aoi_bbox=aoi_bbox, + min_aoi_coverage_ratio=min_aoi_coverage_ratio, + min_common_overlap_ratio=min_common_overlap_ratio, + ) + candidates = list(discovery.get("items") or []) + viable = [ + item for item in candidates + if item.get("status") == "READY" + and int(item.get("usable_scene_count") or item.get("scene_count") or 0) >= normalized_min + ] + if not viable: + blockers = [] + for item in candidates[:5]: + blockers.extend(str(blocker) for blocker in (item.get("blockers") or []) if blocker) + region_label = admin_region or (discovery.get("aoi") or {}).get("name") or "<all configured roots>" + detail = "; ".join(sorted(set(blockers))) if blockers else "no READY LT-1 stack matched the selected production area" + raise ValueError(f"No LandSAR SBAS-ready LT-1 stack found for {region_label}: {detail}") + + selected = max(viable, key=self._landsar_stack_candidate_score) + ranked = sorted(viable, key=self._landsar_stack_candidate_score, reverse=True) + return { + "schema": "insar.landsar-sbas-auto-selection/v1", + "generated_at": _utc_text(), + "selection_strategy": "gamma_production_area_max_scene_landsar_lt1_stack", + "source_system": "gamma_sbas_production_stack_discovery", + "processor_code": PROCESSOR_CODE, + "sensor_family": "LT1", + "min_scenes": normalized_min, + "requested_limit": int(limit or 0), + "discovery_limit": 0, + "discovery_mode": discovery.get("discovery_mode") or discovery_mode, + "admin_region": admin_region, + "aoi": discovery.get("aoi"), + "candidate_count": len(candidates), + "viable_count": len(viable), + "selected_stack_id": selected.get("stack_id"), + "selected_stack": selected, + "ranked_candidates": ranked[:10], + "discovery_snapshot_path": discovery.get("snapshot_path"), + "warnings": discovery.get("warnings") or [], + } + + def create_run_from_best_stack( + self, + *, + run_label: str | None = None, + source_roots: list[str] | None = None, + orbit_roots: list[str] | None = None, + min_scenes: int | None = None, + discovery_mode: str = "strict", + admin_region: str | None = None, + aoi_bbox: dict[str, Any] | None = None, + min_aoi_coverage_ratio: float = 0.01, + min_common_overlap_ratio: float | None = None, + limit: int = 30, + dem_path: str | None = None, + timeout_seconds: int | None = None, + import_timeout_seconds: int | None = None, + params: Optional[dict[str, Any]] = None, + created_by: str | None = None, + ) -> dict[str, Any]: + min_common_overlap_ratio = _effective_min_common_overlap_ratio(min_common_overlap_ratio) + selection = self.select_best_stack_for_area( + source_roots=source_roots, + orbit_roots=orbit_roots, + min_scenes=min_scenes, + discovery_mode=discovery_mode, + admin_region=admin_region, + aoi_bbox=aoi_bbox, + min_aoi_coverage_ratio=min_aoi_coverage_ratio, + min_common_overlap_ratio=min_common_overlap_ratio, + limit=limit, + ) + selected_stack_id = str(selection.get("selected_stack_id") or "").strip() + if not selected_stack_id: + raise ValueError("LandSAR auto stack selection did not return a stack id.") + selected_stack = selection.get("selected_stack") or {} + auto_label = run_label or f"LandSAR SBAS {format_stack_label({'stack': selected_stack, 'dates': selected_stack.get('dates') or []}, selected_stack_id)}" + detail = self._create_run_from_gamma_stack( + selected_stack_id, + sensor_family="LT1", + run_label=auto_label, + source_roots=source_roots, + orbit_roots=orbit_roots, + min_scenes=min_scenes, + require_orbits=False, + discovery_mode=discovery_mode, + admin_region=admin_region, + aoi_bbox=aoi_bbox, + min_aoi_coverage_ratio=min_aoi_coverage_ratio, + min_common_overlap_ratio=min_common_overlap_ratio, + dem_path=dem_path, + timeout_seconds=timeout_seconds, + import_timeout_seconds=import_timeout_seconds, + params=params, + created_by=created_by, + ) + run_id = (detail.get("run") or {}).get("run_id") or (detail.get("manifest") or {}).get("run_id") + if run_id: + run_dir = self._resolve_run_dir(str(run_id)) + manifest_path = run_dir / "run_manifest.json" + manifest = _read_json(manifest_path) + manifest["auto_selection"] = self._stack_selection_manifest(selection) + manifest["source_mode"] = "gamma_production_area_stack_selection" + source_stack = dict(manifest.get("source_stack") or {}) + source_stack["selection_strategy"] = selection.get("selection_strategy") + source_stack["selected_by"] = "system" + source_stack["source_system"] = "gamma_sbas_production_stack_discovery" + manifest["source_stack"] = source_stack + _write_json(manifest_path, manifest) + detail = self.get_run_detail(str(run_id)) + detail["selection"] = selection + return detail + + def _db_import_result( + self, + *, + task_dir: Path, + input_dir: Path, + scenes: list[dict[str, Any]], + status: str, + returncode: int, + stdout_text: str, + error: str, + command: list[str], + param_file: str, + skipped: bool, + min_scenes: int, + sat_mode: str, + ) -> dict[str, Any]: + slc_count, pairs = count_landsar_slc_files(str(input_dir)) + dates = [item.get("date") for item in pairs if item.get("date")] + task = { + "task_name": task_dir.name, + "task_dir": str(task_dir), + "input_data_dir": str(input_dir), + "slc_count": slc_count, + "dates": dates, + "date_start": dates[0] if dates else None, + "date_end": dates[-1] if dates else None, + "scenes": pairs, + } + manifest = { + "schema": "insar.landsar-sbas-db-import/v1", + "generated_at": _utc_text(), + "status": status, + "ready": status == "LANDSAR_SBAS_INPUT_READY", + "task": task, + "min_scenes": min_scenes, + "sat_mode": sat_mode, + "skipped": skipped, + "returncode": returncode, + "error": error, + "command": " ".join(command), + "param_file": param_file, + "stdout_tail": _collect_tail(stdout_text or "", 4000), + "source_scenes": [ + { + "radar_data_id": scene.get("radar_data_id") or scene.get("id"), + "unique_id": scene.get("unique_id"), + "date": scene.get("date") or scene.get("imaging_date"), + "scene_dir": scene.get("scene_dir"), + "file_path": scene.get("file_path"), + "relative_orbit": scene.get("relative_orbit"), + "orbit_direction": scene.get("orbit_direction"), + "polarization": scene.get("polarization"), + } + for scene in scenes + ], + } + _write_json(task_dir / "landsar_sbas_db_import_manifest.json", manifest) + return { + "schema": "insar.landsar-sbas-db-import-result/v1", + "ready": manifest["ready"], + "status": status, + "task": task, + "task_name": task["task_name"], + "task_dir": task["task_dir"], + "input_data_dir": task["input_data_dir"], + "slc_count": slc_count, + "manifest_path": str(task_dir / "landsar_sbas_db_import_manifest.json"), + "error": error, + } + + def _has_completed_import(self, output_dir: str) -> bool: + if not output_dir or not os.path.isdir(output_dir): + return False + log_candidates = [ + os.path.join(output_dir, f"{IMPORT_PROID}.log"), + os.path.join(output_dir, f"{IMPORT_PROID}_timeseries_console.log"), + os.path.join(output_dir, f"{IMPORT_PROID}_console.log"), + *[str(path) for path in Path(output_dir).glob(f"*{IMPORT_PROID}*.log")], + ] + for log_path in log_candidates: + if not os.path.isfile(log_path): + continue + try: + content = Path(log_path).read_text(encoding="utf-8", errors="ignore") + except OSError: + continue + lowered = content.lower() + if "console success" in lowered: + return True + if "lt-1" in lowered and _SUCCESS_RE.search(content): + return True + if "数据导入" in content and _SUCCESS_RE.search(content): + return True + return False + + def _resolve_import_scene_dir(self, scene: dict[str, Any]) -> str: + for key in ( + "scene_dir", + "scene_dir_windows", + "file_path", + "tiff_windows", + "meta_windows", + "source_dir", + ): + raw_value = scene.get(key) + if not raw_value: + continue + try: + resolved = self.resolve_lt1_scene_dir(raw_value) + except Exception: + resolved = "" + if resolved: + return resolved + return "" + + def _normalize_stack_scenes_for_import(self, scenes: list[dict[str, Any]]) -> list[dict[str, Any]]: + normalized: list[dict[str, Any]] = [] + seen_dirs: set[str] = set() + for scene in scenes: + if not isinstance(scene, dict): + continue + scene_dir = self._resolve_import_scene_dir(scene) + if not scene_dir or scene_dir in seen_dirs: + continue + seen_dirs.add(scene_dir) + normalized.append( + { + **scene, + "scene_dir": scene_dir, + "file_path": scene.get("file_path") or scene_dir, + "unique_id": scene.get("unique_id") or scene.get("scene_name"), + } + ) + normalized.sort(key=lambda item: str(item.get("date") or item.get("imaging_date") or "")) + return normalized + + @staticmethod + def _landsar_stack_candidate_score(candidate: dict[str, Any]) -> tuple[int, float, float, int, str]: + common_overlap = float(candidate.get("common_overlap_ratio") or 0.0) + aoi_overlap = float(candidate.get("aoi_overlap_ratio_mean") or 0.0) + usable_count = int(candidate.get("usable_scene_count") or candidate.get("scene_count") or 0) + temporal_gap = int(candidate.get("max_temporal_gap_days") or 9999) + date_start = str(candidate.get("date_start") or "") + return (usable_count, common_overlap, aoi_overlap, -temporal_gap, date_start) + + @staticmethod + def _stack_selection_manifest(selection: dict[str, Any]) -> dict[str, Any]: + ranked = [] + for item in list(selection.get("ranked_candidates") or [])[:10]: + ranked.append( + { + "stack_id": item.get("stack_id"), + "status": item.get("status"), + "satellite": item.get("satellite"), + "relative_orbit": item.get("relative_orbit"), + "orbit_direction": item.get("orbit_direction"), + "scene_count": item.get("scene_count"), + "usable_scene_count": item.get("usable_scene_count"), + "date_start": item.get("date_start"), + "date_end": item.get("date_end"), + "common_overlap_ratio": item.get("common_overlap_ratio"), + "aoi_overlap_ratio_mean": item.get("aoi_overlap_ratio_mean"), + "max_temporal_gap_days": item.get("max_temporal_gap_days"), + } + ) + selected = selection.get("selected_stack") or {} + return { + "schema": selection.get("schema"), + "generated_at": selection.get("generated_at"), + "selection_strategy": selection.get("selection_strategy"), + "selected_stack_id": selection.get("selected_stack_id"), + "selected_stack": ranked[0] if ranked else { + "stack_id": selected.get("stack_id"), + "status": selected.get("status"), + }, + "candidate_count": selection.get("candidate_count"), + "viable_count": selection.get("viable_count"), + "requested_limit": selection.get("requested_limit"), + "discovery_limit": selection.get("discovery_limit"), + "ranked_candidates": ranked, + "discovery_snapshot_path": selection.get("discovery_snapshot_path"), + "admin_region": selection.get("admin_region"), + "aoi": selection.get("aoi"), + "warnings": selection.get("warnings") or [], + } + + def _ensure_tasks_for_execution( + self, + *, + run_dir: Path, + manifest_path: Path, + manifest: dict[str, Any], + progress_callback: Optional[Callable[[dict[str, Any]], None]] = None, + ) -> tuple[dict[str, Any], list[dict[str, Any]]]: + tasks = list(manifest.get("tasks") or []) + if tasks: + return manifest, tasks + + source_stack = manifest.get("source_stack") or {} + scenes = self._normalize_stack_scenes_for_import(source_stack.get("scenes") or []) + if not scenes: + raise ValueError("LandSAR SBAS run has no selected tasks or source stack scenes.") + + input_import = dict(manifest.get("input_import") or {}) + manifest["status"] = "LANDSAR_SBAS_INPUT_IMPORTING" + manifest["next_stage"] = "import_landsar_input" + input_import["status"] = "RUNNING" + input_import["started_at"] = _utc_text() + manifest["input_import"] = input_import + _write_json(manifest_path, manifest) + + try: + result = self.import_stack_scenes( + scenes=scenes, + dest_root=input_import.get("dest_root") or manifest.get("input_task_root"), + task_name=input_import.get("task_name"), + min_scenes=manifest.get("min_scenes"), + sat_mode=input_import.get("sat_mode") or "MONO", + overwrite=bool(input_import.get("overwrite")), + timeout_seconds=manifest.get("import_timeout_seconds") or manifest.get("timeout_seconds"), + progress_callback=progress_callback, + ) + except Exception as exc: + input_import.update( + { + "status": "FAILED", + "ended_at": _utc_text(), + "error": str(exc), + } + ) + manifest["status"] = "LANDSAR_SBAS_IMPORT_FAILED" + manifest["next_stage"] = "inspect_landsar_import_logs" + manifest["input_import"] = input_import + _write_json(manifest_path, manifest) + raise + + task = result.get("task") or {} + tasks = [task] if task else [] + dates = [date for date in (task.get("dates") or []) if date] + input_import.update( + { + "status": result.get("status") or "LANDSAR_SBAS_INPUT_READY", + "ready": bool(result.get("ready")), + "ended_at": _utc_text(), + "result": result, + "manifest_path": result.get("manifest_path"), + "task_dir": result.get("task_dir"), + "input_data_dir": result.get("input_data_dir"), + "slc_count": result.get("slc_count"), + } + ) + manifest.update( + { + "status": "LANDSAR_SBAS_QUEUED", + "next_stage": "execute_landsar_sbas", + "root_dir": result.get("task_dir") or manifest.get("root_dir"), + "tasks": tasks, + "task_count": len(tasks), + "scene_count": int(result.get("slc_count") or len(scenes)), + "dates": dates or manifest.get("dates") or [], + "date_start": (dates or manifest.get("dates") or [None])[0], + "date_end": (dates or manifest.get("dates") or [None])[-1], + "input_import": input_import, + } + ) + _write_json(manifest_path, manifest) + _write_json( + run_dir / "stack_manifest.json", + { + "schema": "insar.landsar-sbas-stack/v1", + "run_id": manifest.get("run_id"), + "status": "READY_FOR_LANDSAR_SBAS", + "source": "sbas_stack_selection_import", + "stack_id": source_stack.get("stack_id"), + "task": task, + "tasks": tasks, + "dates": dates or manifest.get("dates") or [], + "scene_count": manifest.get("scene_count"), + "source_scenes": scenes, + }, + ) + return manifest, tasks + + def create_run( + self, + *, + root_dir: str, + run_label: str | None = None, + num_to_process: int = 0, + min_scenes: int | None = None, + rerun_mode: str = "rerun_all", + timeout_seconds: int | None = None, + extra: Optional[dict[str, Any]] = None, + created_by: str | None = None, + ) -> dict[str, Any]: + if not bool(settings.LANDSAR_SBAS_ENABLED): + raise ValueError("LandSAR SBAS is disabled.") + params = self.normalize_params(extra or {}) + dem_path = _norm_path((extra or {}).get("dem_path") or self.default_dem_path) + if not dem_path or not os.path.isfile(dem_path): + raise ValueError(f"LandSAR SBAS DEM file is missing: {dem_path or '<empty>'}") + validation = self.validate_root_dir( + root_dir, + min_scenes=min_scenes, + num_to_process=num_to_process, + rerun_mode=rerun_mode, + ) + tasks = list(validation.get("items") or []) + if not tasks: + raise ValueError("No valid LandSAR SBAS Task_* directories selected.") + + created_at = datetime.utcnow() + run_id = f"landsar_sbas_{created_at.strftime('%Y%m%dT%H%M%S%fZ')}_{_safe_name(tasks[0].get('task_name'), 'task')}" + run_dir = Path(self.get_run_root()) / run_id + work_run_root = self._allocate_work_run_root(run_id, created_at) + native_root = work_run_root / "n" + publish_root = run_dir / "publish" / "landsar" + native_root.mkdir(parents=True, exist_ok=True) + publish_root.mkdir(parents=True, exist_ok=True) + + all_dates = sorted({date for task in tasks for date in (task.get("dates") or []) if date}) + scene_count = sum(int(task.get("slc_count") or 0) for task in tasks) + manifest = { + "schema": "insar.landsar-sbas-run/v1", + "run_id": run_id, + "run_label": run_label or f"LandSAR SBAS {tasks[0].get('task_name')}", + "workflow_code": WORKFLOW_CODE, + "processor_code": PROCESSOR_CODE, + "profile_code": PROFILE_CODE, + "engine_code": ENGINE_CODE, + "proid": SBAS_PROID, + "process_name": SBAS_PROCESS_NAME, + "execution_mode": "landsar_console_sbas_process", + "status": "LANDSAR_SBAS_QUEUED", + "created_at": _utc_text(created_at), + "created_by": created_by, + "root_dir": _norm_path(root_dir), + "run_dir": str(run_dir), + "work_root": str(work_run_root), + "work_root_strategy": "short_landsar_execution_path", + "native_root": str(native_root), + "publish_root": str(publish_root), + "dem_path": dem_path, + "params": params, + "timeout_seconds": max(60, int(timeout_seconds or settings.LANDSAR_SBAS_TIMEOUT_SECONDS or 172800)), + "min_scenes": validation.get("min_scenes"), + "scene_count": scene_count, + "task_count": len(tasks), + "pair_count": None, + "date_start": all_dates[0] if all_dates else None, + "date_end": all_dates[-1] if all_dates else None, + "dates": all_dates, + "tasks": tasks, + "next_stage": "execute_landsar_sbas", + } + _write_json(run_dir / "run_manifest.json", manifest) + _write_json( + run_dir / "stack_manifest.json", + { + "schema": "insar.landsar-sbas-stack/v1", + "run_id": run_id, + "status": "READY_FOR_LANDSAR_SBAS", + "source": "Task_*/Input_Data", + "tasks": tasks, + "dates": all_dates, + "scene_count": scene_count, + }, + ) + _write_json( + run_dir / "workflow_summary.json", + { + "schema": "insar.landsar-sbas-workflow-summary/v1", + "run_id": run_id, + "process_name": SBAS_PROCESS_NAME, + "ready": False, + "status": "QUEUED", + "task_count": len(tasks), + "completed_count": 0, + "failed_count": 0, + }, + ) + return self.get_run_detail(run_id) + + def list_runs(self) -> dict[str, Any]: + run_root = Path(self.get_run_root()) + items: list[dict[str, Any]] = [] + for manifest_path in sorted(run_root.glob("*/run_manifest.json")): + try: + manifest = _read_json(manifest_path) + items.append(self._build_run_card(manifest_path.parent, manifest)) + except Exception as exc: + items.append({"run_id": manifest_path.parent.name, "status": "RUN_MANIFEST_UNREADABLE", "error": str(exc)}) + items.sort(key=lambda item: str(item.get("created_at") or ""), reverse=True) + return {"items": items, "count": len(items), "run_root": str(run_root)} + + def get_run_detail(self, run_id: str) -> dict[str, Any]: + run_dir = self._resolve_run_dir(run_id) + manifest = _read_json(run_dir / "run_manifest.json") + return { + "run": self._build_run_card(run_dir, manifest), + "manifest": manifest, + "command_manifest": self._read_optional_json(run_dir / "landsar_command_manifest.json"), + "workflow_manifest": self._read_optional_json(run_dir / "workflow_summary.json"), + "workflow_state": self._read_optional_json(run_dir / "workflow_summary.json"), + "geographic_coverage": manifest.get("geographic_coverage") or self._build_geographic_coverage(run_dir), + "artifacts": self._build_run_artifacts(run_dir), + } + + @staticmethod + def _classify_sbas_runtime_failure(stdout_text: str, returncode: int) -> tuple[str, str]: + if _UNSUPPORTED_PROID_RE.search(stdout_text or ""): + return ( + "unsupported_proid", + ( + f"LandSAR InSAR_Console does not support SBAS proID {SBAS_PROID}. " + f"Input import succeeded, but this LandSAR installation did not accept process '{SBAS_PROCESS_NAME}'." + ), + ) + return "console_failure", _summarize_landsar_failure(stdout_text, "LandSAR SBAS", returncode) + + def execute_run( + self, + run_id: str, + *, + timeout_seconds: int | None = None, + progress_callback: Optional[Callable[[dict[str, Any]], None]] = None, + ) -> dict[str, Any]: + run_dir = self._resolve_run_dir(run_id) + manifest_path = run_dir / "run_manifest.json" + manifest = _read_json(manifest_path) + availability = self.check_available() + if not availability["available"]: + raise ValueError(f"LandSAR SBAS is not available: {availability['message']}") + + console_path = self._engine._console_exe + home = self._engine._home + config_ok, config_detail = self._engine._ensure_config_csv() + if not config_ok: + raise ValueError(f"LandSAR config.csv is not ready: {config_detail}") + auth_ok, auth_detail = self._engine._start_auth_server_if_needed() + if not auth_ok: + raise ValueError(f"LandSAR network license server is not ready: {auth_detail}") + + timeout = max(60, int(timeout_seconds or manifest.get("timeout_seconds") or settings.LANDSAR_SBAS_TIMEOUT_SECONDS or 172800)) + manifest, tasks = self._ensure_tasks_for_execution( + run_dir=run_dir, + manifest_path=manifest_path, + manifest=manifest, + progress_callback=progress_callback, + ) + if not tasks: + raise ValueError("LandSAR SBAS run has no selected tasks.") + + started_at = _utc_text() + manifest["status"] = "LANDSAR_SBAS_RUNNING" + manifest["started_at"] = started_at + manifest["next_stage"] = "execute_landsar_sbas" + _write_json(manifest_path, manifest) + + task_results: list[dict[str, Any]] = [] + success_count = 0 + failed_count = 0 + skipped_count = 0 + primary_published = False + + for index, task in enumerate(tasks, start=1): + task_name = str(task.get("task_name") or f"Task_{index}") + task_alias = _safe_name(task_name, f"task_{index}") + input_dir = _norm_path(task.get("input_data_dir") or os.path.join(str(task.get("task_dir") or ""), "Input_Data")) + native_root = Path(str(manifest.get("native_root") or run_dir / "native")) + native_output_dir = native_root / task_alias / "Output_Data" + native_output_dir.mkdir(parents=True, exist_ok=True) + project_name = str((manifest.get("params") or {}).get("project_name") or task_alias).strip() or task_alias + param_file = _generate_sbas_param_file( + str(native_output_dir / f"{SBAS_PROID}.txt"), + slc_folder=input_dir, + dem_path=str(manifest.get("dem_path") or ""), + output_dir=str(native_output_dir), + project_name=project_name, + params=dict(manifest.get("params") or {}), + ) + command = [console_path, param_file] + self._emit(progress_callback, "INFO", f"[{index}/{len(tasks)}] LandSAR SBAS {task_name} started") + rc, stdout_text, timed_out = self._run_console( + command, + cwd=home if os.path.isdir(home) else os.path.dirname(console_path), + log_path=str(native_output_dir / f"{SBAS_PROID}_console.log"), + timeout=timeout, + progress_callback=progress_callback, + task_name=task_name, + ) + log_publish_result = self._copy_native_logs( + run_dir=run_dir, + task_alias=task_alias, + native_output_dir=native_output_dir, + ) + success = rc == 0 and self._has_completed_output(str(native_output_dir)) + error = "" + failure_kind = "" + if timed_out: + error = f"LandSAR SBAS timed out after {timeout}s." + failure_kind = "timeout" + success = False + elif rc != 0: + failure_kind, error = self._classify_sbas_runtime_failure(stdout_text, rc) + elif not success: + error = "LandSAR SBAS success marker or core output is missing." + failure_kind = "missing_outputs" + + publish_result: dict[str, Any] = {} + if success: + success_count += 1 + publish_result = self._publish_task_outputs( + run_dir=run_dir, + task_alias=task_alias, + native_output_dir=native_output_dir, + make_primary=not primary_published, + ) + primary_published = primary_published or bool(publish_result.get("primary_published")) + self._emit(progress_callback, "INFO", f"[{index}/{len(tasks)}] LandSAR SBAS {task_name} completed") + else: + failed_count += 1 + self._emit(progress_callback, "ERROR", f"[{index}/{len(tasks)}] LandSAR SBAS {task_name} failed: {error}") + + task_results.append( + { + "task_name": task_name, + "task_alias": task_alias, + "input_data_dir": input_dir, + "native_output_dir": str(native_output_dir), + "param_file": param_file, + "process_name": SBAS_PROCESS_NAME, + "command": " ".join(command), + "returncode": rc, + "success": success, + "timed_out": timed_out, + "failure_kind": failure_kind, + "error": error, + "stdout_tail": _collect_tail(stdout_text, 4000), + "native_logs": log_publish_result, + "publish": publish_result, + } + ) + + unsupported_count = sum(1 for item in task_results if item.get("failure_kind") == "unsupported_proid") + if success_count > 0 and failed_count == 0: + status = "LANDSAR_SBAS_COMPLETED" + next_stage = "review_landsar_products" + elif success_count > 0: + status = "LANDSAR_SBAS_PARTIAL" + next_stage = "review_landsar_products" + elif failed_count > 0 and unsupported_count == failed_count: + status = "LANDSAR_SBAS_RUNTIME_UNSUPPORTED" + next_stage = "configure_landsar_sbas_runtime" + else: + status = "LANDSAR_SBAS_FAILED" + next_stage = "inspect_landsar_logs" + + coverage = self._build_geographic_coverage(run_dir) + quality_summary = self._build_quality_summary(run_dir) + workflow_summary = { + "schema": "insar.landsar-sbas-workflow-summary/v1", + "run_id": run_id, + "process_name": SBAS_PROCESS_NAME, + "status": status, + "ready": status in {"LANDSAR_SBAS_COMPLETED", "LANDSAR_SBAS_PARTIAL"}, + "started_at": started_at, + "ended_at": _utc_text(), + "task_count": len(tasks), + "completed_count": success_count, + "failed_count": failed_count, + "skipped_count": skipped_count, + "unsupported_proid_count": unsupported_count, + "task_results": task_results, + } + product_summary = { + "schema": "insar.landsar-sbas-product-summary/v1", + "run_id": run_id, + "processor_code": PROCESSOR_CODE, + "engine_code": ENGINE_CODE, + "proid": SBAS_PROID, + "process_name": SBAS_PROCESS_NAME, + "default_los_product": "los_timeseries", + "los_sign_convention": "LandSAR LOS output; sign and rate/cumulative semantics require algorithm confirmation.", + "output_semantics_note": "Do not label LandSAR *.los.tif as annual velocity until verified by the algorithm owner.", + "task_count": len(tasks), + "completed_task_count": success_count, + "unsupported_proid_count": unsupported_count, + "primary_asset": "publish/landsar/los_timeseries.tif" if (run_dir / "publish" / "landsar" / "los_timeseries.tif").is_file() else None, + } + command_manifest = { + "schema": "insar.landsar-sbas-command-manifest/v1", + "run_id": run_id, + "engine": ENGINE_CODE, + "processor_code": PROCESSOR_CODE, + "proid": SBAS_PROID, + "process_name": SBAS_PROCESS_NAME, + "console_path": console_path, + "home": home, + "work_root": manifest.get("work_root"), + "work_root_strategy": manifest.get("work_root_strategy"), + "timeout_seconds": timeout, + "params": manifest.get("params") or {}, + "availability": availability, + "expected_outputs": [ + "publish/landsar/los_timeseries.tif", + "publish/landsar/post_raster.tif", + "publish/landsar/preview.png", + ], + } + _write_json(run_dir / "workflow_summary.json", workflow_summary) + _write_json(run_dir / "product_summary.json", product_summary) + _write_json(run_dir / "quality_summary.json", quality_summary) + _write_json(run_dir / "landsar_command_manifest.json", command_manifest) + + manifest.update( + { + "status": status, + "ended_at": workflow_summary["ended_at"], + "next_stage": next_stage, + "task_results": task_results, + "completed_task_count": success_count, + "failed_task_count": failed_count, + "geographic_coverage": coverage, + "workflow": {"status": status, "summary": workflow_summary}, + "product_summary_path": str(run_dir / "product_summary.json"), + } + ) + _write_json(manifest_path, manifest) + return self.get_run_detail(run_id) + + def resolve_artifact(self, run_id: str, relative_path: str) -> Path: + run_dir = self._resolve_run_dir(run_id) + normalized = str(relative_path or "").replace("\\", "/").strip("/") + target = (run_dir / normalized).resolve() + root = run_dir.resolve() + try: + target.relative_to(root) + except ValueError as exc: + raise ValueError("artifact path escapes run root") from exc + if not target.is_file(): + raise FileNotFoundError(f"artifact not found: {normalized}") + return target + + def _run_console( + self, + command: list[str], + *, + cwd: str, + log_path: str, + timeout: int, + progress_callback: Optional[Callable[[dict[str, Any]], None]], + task_name: str, + ) -> tuple[int, str, bool]: + os.makedirs(os.path.dirname(log_path), exist_ok=True) + with open(log_path, "a", encoding="utf-8", errors="replace") as log_fp: + log_fp.write(f"\n[{_utc_text()}] command: {' '.join(command)}\n") + process = subprocess.Popen( + command, + cwd=cwd, + env=_landsar_process_env(command[0], self._engine._home), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + stdin=subprocess.DEVNULL, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + output_lines: list[str] = [] + line_queue: queue.Queue[Any] = queue.Queue() + sentinel = object() + + def _reader() -> None: + try: + if process.stdout is None: + return + for raw_line in iter(process.stdout.readline, b""): + line_queue.put(raw_line) + finally: + line_queue.put(sentinel) + + reader = threading.Thread(target=_reader, name="landsar-sbas-console-reader", daemon=True) + reader.start() + started = time.monotonic() + timed_out = False + stdout_closed = False + while True: + try: + raw_item = line_queue.get(timeout=1) + except queue.Empty: + raw_item = None + if raw_item is sentinel: + stdout_closed = True + elif raw_item: + line = _decode_line(raw_item) + output_lines.append(line) + log_fp.write(line + "\n") + log_fp.flush() + if line.strip(): + self._emit(progress_callback, "INFO", f"{task_name}: {line.strip()}") + if process.poll() is not None and stdout_closed: + break + if time.monotonic() - started > timeout: + timed_out = True + process.kill() + break + if timed_out: + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + pass + reader.join(timeout=5) + while True: + try: + raw_item = line_queue.get_nowait() + except queue.Empty: + break + if raw_item is sentinel or not raw_item: + continue + line = _decode_line(raw_item) + output_lines.append(line) + log_fp.write(line + "\n") + if process.poll() is None: + timed_out = True + process.kill() + rc = int(process.poll() if process.poll() is not None else -9) + if timed_out: + log_fp.write(f"[{_utc_text()}] timeout after {timeout}s\n") + log_fp.write(f"[{_utc_text()}] returncode={rc}\n") + return rc, "\n".join(output_lines), timed_out + + def _has_completed_output(self, output_dir: str) -> bool: + if not output_dir or not os.path.isdir(output_dir): + return False + if not (self._select_los_file(output_dir) or self._select_raster_file(output_dir)): + return False + log_candidates = [ + os.path.join(output_dir, f"{SBAS_PROID}.log"), + os.path.join(output_dir, f"{SBAS_PROID}_console.log"), + *[str(path) for path in Path(output_dir).glob(f"*{SBAS_PROID}*.log")], + ] + for log_path in log_candidates: + if not os.path.isfile(log_path): + continue + try: + content = Path(log_path).read_text(encoding="utf-8", errors="ignore") + except OSError: + continue + lowered = content.lower() + if "console success" in lowered: + return True + if "sbas" in lowered and _SUCCESS_RE.search(content): + return True + return False + + def _select_los_file(self, output_dir: str) -> str: + return self._first_matching_file(output_dir, ["*.los.tif", "*.los.tiff", "*los*.tif", "*los*.tiff"]) + + def _select_raster_file(self, output_dir: str) -> str: + return self._first_matching_file(output_dir, ["*.raster.tif", "*.raster.tiff", "*raster*.tif", "*raster*.tiff"]) + + @staticmethod + def _first_matching_file(output_dir: str, patterns: list[str]) -> str: + root = Path(output_dir) + if not root.is_dir(): + return "" + for pattern in patterns: + for path in sorted(root.rglob(pattern), key=lambda item: str(item).lower()): + if path.is_file(): + return _norm_path(path) + return "" + + def _copy_native_logs(self, *, run_dir: Path, task_alias: str, native_output_dir: Path) -> dict[str, Any]: + native_logs = run_dir / "native_logs" / task_alias + native_logs.mkdir(parents=True, exist_ok=True) + copied: list[dict[str, str]] = [] + + def copy_named(src: str, target: Path) -> str: + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, target) + copied.append({"source": src, "target": str(target)}) + return str(target) + + for log_path in native_output_dir.glob(f"*{SBAS_PROID}*.log"): + if log_path.is_file(): + copy_named(str(log_path), native_logs / log_path.name) + param_path = native_output_dir / f"{SBAS_PROID}.txt" + if param_path.is_file(): + copy_named(str(param_path), native_logs / param_path.name) + return {"native_logs_dir": str(native_logs), "copied": copied} + + def _publish_task_outputs(self, *, run_dir: Path, task_alias: str, native_output_dir: Path, make_primary: bool) -> dict[str, Any]: + publish_dir = run_dir / "publish" / "landsar" + task_publish_dir = publish_dir / task_alias + native_logs = run_dir / "native_logs" / task_alias + task_publish_dir.mkdir(parents=True, exist_ok=True) + + los_src = self._select_los_file(str(native_output_dir)) + raster_src = self._select_raster_file(str(native_output_dir)) + copied: list[dict[str, str]] = [] + + def copy_named(src: str, target: Path) -> str: + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, target) + copied.append({"source": src, "target": str(target)}) + return str(target) + + task_los = copy_named(los_src, task_publish_dir / "los_timeseries.tif") if los_src else "" + task_raster = copy_named(raster_src, task_publish_dir / "post_raster.tif") if raster_src else "" + vector_dir = native_output_dir / "vector" + if vector_dir.is_dir(): + shutil.copytree(vector_dir, task_publish_dir / "vector", dirs_exist_ok=True) + + primary_published = False + preview_path = "" + if make_primary: + if task_los: + copy_named(task_los, publish_dir / "los_timeseries.tif") + primary_published = True + if task_raster: + copy_named(task_raster, publish_dir / "post_raster.tif") + primary_published = True + preview_source = task_los or task_raster + if preview_source: + preview_path = self._build_preview_png(preview_source, str(publish_dir / "preview.png")) or "" + if preview_path: + copied.append({"source": preview_source, "target": preview_path}) + return { + "task_publish_dir": str(task_publish_dir), + "native_logs_dir": str(native_logs), + "los_timeseries": task_los, + "post_raster": task_raster, + "preview": preview_path, + "primary_published": primary_published, + "copied": copied, + } + + def _build_preview_png(self, source: str, target: str) -> str | None: + try: + import numpy as np + import rasterio + from PIL import Image + from rasterio.enums import Resampling + + with rasterio.open(source) as src: + scale = max(src.width / 1200, src.height / 1200, 1) + out_w = max(1, int(src.width / scale)) + out_h = max(1, int(src.height / scale)) + data = src.read(1, out_shape=(out_h, out_w), resampling=Resampling.bilinear).astype("float32") + mask = src.dataset_mask(out_shape=(out_h, out_w)) > 0 + nodata = src.nodata + if nodata is not None: + mask &= data != nodata + finite = np.isfinite(data) + mask &= finite + if not np.any(mask): + return None + values = data[mask] + p2, p98 = np.nanpercentile(values, [2, 98]) + if not np.isfinite(p2) or not np.isfinite(p98) or p98 <= p2: + p2 = float(np.nanmin(values)) + p98 = float(np.nanmax(values)) + if p98 <= p2: + norm = np.zeros_like(data, dtype="float32") + else: + norm = np.clip((data - p2) / (p98 - p2), 0, 1) + gray = (norm * 255).astype("uint8") + rgba = np.dstack([gray, gray, gray, np.where(mask, 255, 0).astype("uint8")]) + image = Image.fromarray(rgba, mode="RGBA") + image.thumbnail((1200, 1200), Image.Resampling.LANCZOS) + out_path = Path(target) + out_path.parent.mkdir(parents=True, exist_ok=True) + image.save(out_path) + return str(out_path) + except Exception: + return None + + def _build_quality_summary(self, run_dir: Path) -> dict[str, Any]: + primary = run_dir / "publish" / "landsar" / "los_timeseries.tif" + stats = self._raster_stats(primary) if primary.is_file() else {} + return { + "schema": "insar.landsar-sbas-quality-summary/v1", + "generated_at": _utc_text(), + "primary_geotiff": stats, + } + + @staticmethod + def _raster_stats(path: Path) -> dict[str, Any]: + try: + import numpy as np + import rasterio + from rasterio.enums import Resampling + + with rasterio.open(path) as src: + scale = max(src.width / 2048, src.height / 2048, 1) + out_w = max(1, int(src.width / scale)) + out_h = max(1, int(src.height / scale)) + data = src.read(1, out_shape=(out_h, out_w), resampling=Resampling.nearest).astype("float64") + mask = src.dataset_mask(out_shape=(out_h, out_w)) > 0 + if src.nodata is not None: + mask &= data != src.nodata + values = data[mask & np.isfinite(data)] + if values.size == 0: + return {"exists": True, "valid_count": 0} + return { + "exists": True, + "sampled": scale > 1, + "width": src.width, + "height": src.height, + "crs": str(src.crs) if src.crs else None, + "valid_count": int(values.size), + "min": float(np.nanmin(values)), + "p05": float(np.nanpercentile(values, 5)), + "median": float(np.nanmedian(values)), + "p95": float(np.nanpercentile(values, 95)), + "max": float(np.nanmax(values)), + "mean": float(np.nanmean(values)), + "std": float(np.nanstd(values)), + } + except Exception as exc: + return {"exists": path.is_file(), "error": str(exc)} + + def _build_geographic_coverage(self, run_dir: Path) -> dict[str, Any]: + primary = run_dir / "publish" / "landsar" / "los_timeseries.tif" + if not primary.is_file(): + primary = run_dir / "publish" / "landsar" / "post_raster.tif" + bbox = None + crs = None + try: + import rasterio + from rasterio.warp import transform_bounds + + with rasterio.open(primary) as src: + crs = str(src.crs) if src.crs else None + bounds = src.bounds + if src.crs: + west, south, east, north = transform_bounds(src.crs, "EPSG:4326", *bounds, densify_pts=21) + else: + west, south, east, north = bounds.left, bounds.bottom, bounds.right, bounds.top + bbox = { + "min_lon": float(west), + "min_lat": float(south), + "max_lon": float(east), + "max_lat": float(north), + } + except Exception: + bbox = None + center = None + if bbox: + center = { + "lon": (bbox["min_lon"] + bbox["max_lon"]) / 2, + "lat": (bbox["min_lat"] + bbox["max_lat"]) / 2, + } + return { + "schema": "insar.landsar-sbas-geographic-coverage/v1", + "bbox": bbox, + "bbox_intersection": bbox, + "center": center, + "crs": crs, + "source": str(primary) if primary.is_file() else None, + } + + def _build_run_card(self, run_dir: Path, manifest: dict[str, Any]) -> dict[str, Any]: + coverage = manifest.get("geographic_coverage") or self._build_geographic_coverage(run_dir) + center = (coverage or {}).get("center") + return { + "run_id": manifest.get("run_id") or run_dir.name, + "run_label": manifest.get("run_label"), + "status": manifest.get("status") or "UNKNOWN", + "created_at": manifest.get("created_at"), + "workflow_code": manifest.get("workflow_code") or WORKFLOW_CODE, + "processor_code": manifest.get("processor_code") or PROCESSOR_CODE, + "engine_code": manifest.get("engine_code") or ENGINE_CODE, + "sensor_family": "LT1", + "profile_code": manifest.get("profile_code") or PROFILE_CODE, + "execution_enabled": True, + "stack_id": manifest.get("stack_id") or manifest.get("run_id"), + "scene_count": manifest.get("scene_count"), + "pair_count": manifest.get("pair_count"), + "task_count": manifest.get("task_count"), + "next_stage": manifest.get("next_stage"), + "platform": "LT1", + "reference_date": manifest.get("date_start"), + "date_start": manifest.get("date_start"), + "date_end": manifest.get("date_end"), + "center": center, + "run_dir": str(run_dir), + } + + def _build_run_artifacts(self, run_dir: Path) -> list[dict[str, Any]]: + artifacts: list[dict[str, Any]] = [] + for path in sorted(run_dir.rglob("*")): + if not path.is_file(): + continue + rel = str(path.relative_to(run_dir)).replace("\\", "/") + if path.stat().st_size <= 0: + continue + role = "artifact" + if rel == "run_manifest.json": + role = "run_manifest" + elif rel.endswith("_console.log") or rel.endswith(".log"): + role = "native_log" + elif rel.endswith(f"{SBAS_PROID}.txt"): + role = "parameter_file" + elif rel.endswith(".tif") or rel.endswith(".tiff"): + role = "primary_geotiff" if rel == "publish/landsar/los_timeseries.tif" else "geotiff" + elif rel.endswith(".png"): + role = "primary_preview" + artifacts.append( + { + "key": _safe_name(Path(rel).stem), + "label": rel, + "role": role, + "relative_path": rel, + "size_bytes": path.stat().st_size, + } + ) + return artifacts + + def _resolve_run_dir(self, run_id: str) -> Path: + clean_id = str(run_id or "").strip() + if not clean_id or Path(clean_id).name != clean_id: + raise ValueError("invalid LandSAR SBAS run id") + run_dir = (Path(self.get_run_root()) / clean_id).resolve() + root = Path(self.get_run_root()).resolve() + try: + run_dir.relative_to(root) + except ValueError as exc: + raise ValueError("run id escapes LandSAR SBAS run root") from exc + if not run_dir.is_dir(): + raise FileNotFoundError(f"LandSAR SBAS run not found: {clean_id}") + return run_dir + + @staticmethod + def _read_optional_json(path: Path) -> dict[str, Any] | None: + if not path.is_file(): + return None + return _read_json(path) + + @staticmethod + def _emit(callback: Optional[Callable[[dict[str, Any]], None]], level: str, message: str) -> None: + if not callable(callback): + return + try: + callback({"level": level, "message": message, "timestamp": _utc_text()}) + except Exception: + return + + +landsar_sbas_service = LandsarSbasService() diff --git a/backend/app/services/product_packaging.py b/backend/app/services/product_packaging.py index 6170f7a..6700105 100644 --- a/backend/app/services/product_packaging.py +++ b/backend/app/services/product_packaging.py @@ -17,7 +17,7 @@ def _clean_dict(payload: Optional[Dict[str, Any]]) -> Dict[str, Any]: def _kind_from_engine(engine_code: str) -> Optional[str]: normalized = str(engine_code or "").strip().lower() - if normalized in {"envi", "sarscape"}: + if normalized in {"envi", "sarscape", "landsar"}: return "windows" if normalized in {"isce2", "pyint", "gamma"}: return "wsl" diff --git a/backend/app/services/result_catalog_service.py b/backend/app/services/result_catalog_service.py index c8c3446..63fbb81 100644 --- a/backend/app/services/result_catalog_service.py +++ b/backend/app/services/result_catalog_service.py @@ -340,6 +340,8 @@ def _iter_flat_result_candidates(root_dir: str) -> Iterable[Dict[str, Any]]: if is_standard_isce2_disp_file(normalized_root, entry.path): source_dir = os.path.dirname(os.path.dirname(os.path.dirname(entry.path))) + run_meta = find_json_sidecar(source_dir, RUN_META_FILENAME, max_levels=0) or {} + engine_code = _first_text(run_meta.get("engine_code")) or "isce2" source_files = [entry.path] coh_candidates = ( os.path.join(source_dir, "assets", "coh", "coh.tif"), @@ -350,7 +352,7 @@ def _iter_flat_result_candidates(root_dir: str) -> Iterable[Dict[str, Any]]: source_files.append(coh_path) break yield { - "engine_code": "isce2", + "engine_code": engine_code, "name": os.path.splitext(entry.name)[0], "task_name": "", "source_dir": source_dir, diff --git a/backend/app/services/root_registry_service.py b/backend/app/services/root_registry_service.py index e2d9949..5554c09 100644 --- a/backend/app/services/root_registry_service.py +++ b/backend/app/services/root_registry_service.py @@ -359,6 +359,16 @@ def _build_root_specs_from_settings() -> List[RootSpec]: owner_engine="dinsar", ) ) + specs.extend( + _iter_single_root_specs( + env_var="LANDSAR_WORK_ROOT", + path=settings.LANDSAR_WORK_ROOT, + root_role="work_root_landsar", + display_name="LandSAR Work Root", + scan_mode="workspace", + owner_engine="landsar", + ) + ) specs.extend( _iter_single_root_specs( env_var="TIMESERIES_PRODUCT_DIR", diff --git a/backend/app/services/sbas_insar_catalog_service.py b/backend/app/services/sbas_insar_catalog_service.py index b667f85..56ea2df 100644 --- a/backend/app/services/sbas_insar_catalog_service.py +++ b/backend/app/services/sbas_insar_catalog_service.py @@ -1,10 +1,14 @@ from __future__ import annotations import asyncio +import gzip import hashlib import json +import math import mimetypes import os +import re +import struct from datetime import datetime from pathlib import Path from typing import Any, Optional @@ -17,6 +21,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from ..config import settings from ..models import ResultAssetORM, ResultCatalogStateORM, ResultIssueORM, ResultProductORM from .admin_region_lookup_service import lookup_admin_region_for_point +from .landsar_sbas_service import landsar_sbas_service from .sbas_insar_production_service import sbas_insar_production_service @@ -24,8 +29,19 @@ SBAS_INSAR_CATALOG_NAME = "sbas_insar" JOB_TYPE_REBUILD_SBAS_INSAR_CATALOG = "REBUILD_SBAS_INSAR_CATALOG" TASK_TYPE_REBUILD_SBAS_INSAR_CATALOG = "REBUILD_SBAS_INSAR_CATALOG" -_READY_STATUSES = {"PRODUCTS_READY", "MONITOR_POINTS_READY", "WORKFLOW_COMPLETED"} +_READY_STATUSES = { + "PRODUCTS_READY", + "MONITOR_POINTS_READY", + "WORKFLOW_COMPLETED", + "LANDSAR_SBAS_COMPLETED", + "LANDSAR_SBAS_PARTIAL", +} _REQUIRED_ASSET_ROLES = {"primary_geotiff", "quality_geotiff"} +_LANDSAR_REQUIRED_ASSET_ROLES = {"primary_geotiff"} +_WGS84_GEOGCS_WKT = ( + 'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563]],' + 'PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]]' +) _CORE_ASSETS = ( ("run_manifest", "Run manifest", "run_manifest.json", True, False), @@ -102,6 +118,117 @@ _CORE_ASSETS = ( ("height_correction", "Height correction GeoTIFF", "publish/geotiff/hgt_correction_m.tif", False, False), ) +_LANDSAR_CORE_ASSETS = ( + ("run_manifest", "Run manifest", "run_manifest.json", True, False), + ("stack_manifest", "Task/Input_Data manifest", "stack_manifest.json", True, False), + ("workflow_summary", "LandSAR SBAS workflow summary", "workflow_summary.json", True, False), + ("product_summary", "LandSAR SBAS product summary", "product_summary.json", False, False), + ("quality_summary", "LandSAR SBAS quality summary", "quality_summary.json", False, False), + ("command_manifest", "LandSAR SBAS command manifest", "landsar_command_manifest.json", False, False), + ("native_console_log", "LandSAR SBAS console log", "native_logs", False, False), + ("primary_preview", "LandSAR LOS preview", "publish/landsar/preview.png", False, True), + ("primary_geotiff", "LandSAR LOS time-series GeoTIFF", "publish/landsar/los_timeseries.tif", True, True), + ("secondary_geotiff", "LandSAR post-raster GeoTIFF", "publish/landsar/post_raster.tif", False, False), +) + +_EXPERT_GAMMA_CORE_ASSETS = ( + ("run_manifest", "Run manifest", "run_manifest.json", True, False), + ("stack_manifest", "Stack manifest", "stack_manifest.json", True, False), + ("workflow_summary", "Workflow summary", "workflow_summary.json", False, False), + ("monitor_points_summary", "Monitor points summary", "monitor_points_summary.json", False, False), + ( + "unwrapped_phase_summary", + "Expert Gamma geocoded unwrapped phase summary", + "publish/geotiff/unwrapped/unwrapped_phase_summary.json", + False, + False, + ), + ( + "unwrapped_phase_radar_colorbar", + "Expert Gamma rmg.cm unwrapped phase radar colorbar", + "publish/geotiff/unwrapped/unwrapped_phase_rmg_colorbar.png", + False, + False, + ), + ( + "point_vector_summary", + "Expert Gamma LOS point-vector summary", + "publish/vectors/los_rate_points_summary.json", + False, + False, + ), + ( + "point_vector_geojson_gz", + "Expert Gamma LOS point-vector GeoJSON.gz", + "publish/vectors/los_rate_points.geojson.gz", + False, + False, + ), + ( + "primary_geocoded_preview", + "Expert Gamma geo_los_def_rate RGB PNG preview", + "publish/geotiff/geo_los_def_rate_rgb_preview.png", + False, + True, + ), + ( + "primary_rate_color_preview", + "Expert Gamma pure geo_los_def_rate hls.cm PNG preview", + "publish/geotiff/geo_los_def_rate_pure_hls_preview.png", + False, + False, + ), + ( + "primary_geotiff", + "Expert Gamma geo_los_def_rate GeoTIFF", + "publish/geotiff/geo_los_def_rate.tif", + True, + True, + ), + ( + "primary_rgb_geotiff", + "Expert Gamma geo_los_def_rate RGB GeoTIFF", + "publish/geotiff/geo_los_def_rate_rgb.tif", + False, + False, + ), + ( + "primary_colorbar", + "Expert Gamma hls.cm deformation-rate colorbar", + "publish/geotiff/geo_los_def_rate_hls_colorbar.png", + False, + False, + ), + ( + "monitor_points", + "Expert Gamma disp_prt_2d point time series", + "publish/points/disp_point.txt", + False, + False, + ), + ( + "monitor_point_items", + "Expert Gamma disp_prt_2d column definitions", + "publish/points/items.txt", + False, + False, + ), + ( + "monitor_point_selection", + "Expert Gamma disp_prt_2d selected radar points", + "publish/points/disp_point_sel.txt", + False, + False, + ), + ( + "monitor_point_selection_metadata", + "Expert Gamma monitor point selection metadata", + "publish/points/disp_point_selection.json", + False, + False, + ), +) + def _utcnow() -> datetime: return datetime.utcnow() @@ -143,6 +270,35 @@ def _safe_int(value: Any) -> Optional[int]: return None +def _read_gamma_key_values(path: Path) -> dict[str, str]: + values: dict[str, str] = {} + if not path.is_file(): + return values + for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): + if ":" not in line: + continue + key, raw_value = line.split(":", 1) + key = key.strip() + value = raw_value.strip().split()[0] if raw_value.strip() else "" + if key: + values[key] = value + return values + + +def _read_gamma_int_param(path: Path, key: str) -> Optional[int]: + return _safe_int(_read_gamma_key_values(path).get(key)) + + +def _haversine_m(lon1: float, lat1: float, lon2: float, lat2: float) -> float: + radius_m = 6371008.8 + phi1 = math.radians(lat1) + phi2 = math.radians(lat2) + dphi = math.radians(lat2 - lat1) + dlambda = math.radians(lon2 - lon1) + a = math.sin(dphi / 2) ** 2 + math.cos(phi1) * math.cos(phi2) * math.sin(dlambda / 2) ** 2 + return 2 * radius_m * math.asin(min(1.0, math.sqrt(a))) + + def _parse_datetime(value: Any) -> Optional[datetime]: text = str(value or "").strip() if not text: @@ -200,6 +356,1591 @@ def _media_type(path: str) -> Optional[str]: return explicit.get(ext) or mimetypes.guess_type(path)[0] +def _build_rgb_geotiff_preview(source: Path, target: Path) -> Optional[str]: + if target.is_file(): + return str(target) + if not source.is_file(): + return None + try: + import numpy as np + import rasterio + from PIL import Image + from rasterio.enums import Resampling + except Exception: + return None + + try: + with rasterio.open(source) as src: + scale = max(src.width / 1600, src.height / 1600, 1) + out_w = max(1, int(src.width / scale)) + out_h = max(1, int(src.height / scale)) + mask = src.dataset_mask(out_shape=(out_h, out_w)) > 0 + if src.count >= 3: + data = src.read( + [1, 2, 3], + out_shape=(3, out_h, out_w), + resampling=Resampling.bilinear, + ) + rgb = np.moveaxis(data, 0, -1) + if rgb.dtype != np.uint8: + rgb = np.clip(rgb, 0, 255).astype("uint8") + else: + data = src.read(1, out_shape=(out_h, out_w), resampling=Resampling.bilinear).astype("float32") + finite = np.isfinite(data) + valid = finite & mask + if np.any(valid): + p2, p98 = np.nanpercentile(data[valid], [2, 98]) + if not np.isfinite(p2) or not np.isfinite(p98) or p98 <= p2: + p2 = float(np.nanmin(data[valid])) + p98 = float(np.nanmax(data[valid])) + norm = np.clip((data - p2) / max(p98 - p2, 1e-6), 0, 1) + else: + norm = np.zeros_like(data, dtype="float32") + gray = np.where(np.isfinite(norm), norm * 255, 0).astype("uint8") + rgb = np.dstack([gray, gray, gray]) + alpha = np.where(mask, 255, 0).astype("uint8") + image = Image.fromarray(np.dstack([rgb, alpha]), mode="RGBA") + resampling = getattr(getattr(Image, "Resampling", Image), "LANCZOS") + image.thumbnail((1600, 1600), resampling) + target.parent.mkdir(parents=True, exist_ok=True) + image.save(target, "PNG", optimize=True) + return str(target) + except Exception: + return None + + +def _find_gamma_hls_colormap() -> Optional[Path]: + candidates = [ + Path(r"\\wsl.localhost\Ubuntu-24.04\usr\local\GAMMA_SOFTWARE-20240627\DISP\cmaps\hls.cm"), + Path(r"\\wsl$\Ubuntu-24.04\usr\local\GAMMA_SOFTWARE-20240627\DISP\cmaps\hls.cm"), + ] + for candidate in candidates: + if candidate.is_file(): + return candidate + return None + + +def _find_gamma_colormap(name: str) -> Optional[Path]: + safe_name = Path(str(name or "")).name + if not safe_name: + return None + candidates = [ + Path(r"\\wsl.localhost\Ubuntu-24.04\usr\local\GAMMA_SOFTWARE-20240627\DISP\cmaps") / safe_name, + Path(r"\\wsl$\Ubuntu-24.04\usr\local\GAMMA_SOFTWARE-20240627\DISP\cmaps") / safe_name, + ] + for candidate in candidates: + if candidate.is_file(): + return candidate + return None + + +def _read_gamma_colormap(path: Path) -> list[tuple[int, int, int]]: + colors: list[tuple[int, int, int]] = [] + if not path.is_file(): + return colors + for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): + parts = line.strip().split() + if len(parts) < 3: + continue + try: + red, green, blue = (max(0, min(255, int(float(value)))) for value in parts[:3]) + except ValueError: + continue + colors.append((red, green, blue)) + return colors + + +def _build_gamma_colormap_colorbar( + target: Path, + *, + colormap_name: str, + min_value: float, + max_value: float, + unit: str, + title: str, +) -> Optional[str]: + if target.is_file(): + return str(target) + source = _find_gamma_colormap(colormap_name) + colors = _read_gamma_colormap(source) if source else [] + if not colors: + return None + try: + import numpy as np + from PIL import Image, ImageDraw, ImageFont + + width = 920 + bar_height = 34 + label_height = 78 + margin_x = 48 + margin_top = 14 + target.parent.mkdir(parents=True, exist_ok=True) + image = Image.new("RGBA", (width, bar_height + label_height), (255, 255, 255, 0)) + draw = ImageDraw.Draw(image) + gradient_width = width - margin_x * 2 + color_array = np.asarray(colors, dtype=np.uint8) + for x in range(gradient_width): + idx = int(round((x / max(1, gradient_width - 1)) * (len(color_array) - 1))) + draw.line( + [(margin_x + x, margin_top), (margin_x + x, margin_top + bar_height)], + fill=tuple(int(value) for value in color_array[idx]) + (255,), + ) + draw.rectangle( + [margin_x, margin_top, margin_x + gradient_width, margin_top + bar_height], + outline=(15, 23, 42, 180), + width=1, + ) + try: + font = ImageFont.truetype("arial.ttf", 14) + small_font = ImageFont.truetype("arial.ttf", 12) + except Exception: + font = ImageFont.load_default() + small_font = font + ticks = [ + (min_value, f"{min_value:g}"), + (0.0, "0"), + (max_value, f"{max_value:g}"), + ] + for value, label in ticks: + ratio = (value - min_value) / max(max_value - min_value, 1e-6) + x = margin_x + int(round(ratio * gradient_width)) + draw.line([(x, margin_top + bar_height), (x, margin_top + bar_height + 7)], fill=(15, 23, 42, 220), width=1) + text = f"{label} {unit}".strip() + bbox = draw.textbbox((0, 0), text, font=font) + draw.text((x - (bbox[2] - bbox[0]) / 2, margin_top + bar_height + 10), text, fill=(15, 23, 42, 255), font=font) + source_label = f"{title} / {colormap_name}" + source_bbox = draw.textbbox((0, 0), source_label, font=small_font) + draw.text( + (width - margin_x - (source_bbox[2] - source_bbox[0]), margin_top + bar_height + 34), + source_label, + fill=(71, 85, 105, 255), + font=small_font, + ) + image.save(target, "PNG", optimize=True) + return str(target) + except Exception: + return None + + +def _build_gamma_hls_colorbar(target: Path, *, min_mm_year: float = -80.0, max_mm_year: float = 80.0) -> Optional[str]: + if target.is_file(): + return str(target) + source = _find_gamma_hls_colormap() + colors = _read_gamma_colormap(source) if source else [] + if not colors: + return None + try: + import numpy as np + from PIL import Image, ImageDraw, ImageFont + + width = 900 + bar_height = 34 + label_height = 58 + margin_x = 46 + margin_top = 14 + target.parent.mkdir(parents=True, exist_ok=True) + image = Image.new("RGBA", (width, bar_height + label_height), (255, 255, 255, 0)) + draw = ImageDraw.Draw(image) + gradient_width = width - margin_x * 2 + color_array = np.asarray(colors, dtype=np.uint8) + for x in range(gradient_width): + idx = int(round((x / max(1, gradient_width - 1)) * (len(color_array) - 1))) + draw.line( + [(margin_x + x, margin_top), (margin_x + x, margin_top + bar_height)], + fill=tuple(int(value) for value in color_array[idx]) + (255,), + ) + draw.rectangle( + [margin_x, margin_top, margin_x + gradient_width, margin_top + bar_height], + outline=(15, 23, 42, 180), + width=1, + ) + ticks = [ + (min_mm_year, f"{min_mm_year:g}"), + (0.0, "0"), + (max_mm_year, f"{max_mm_year:g}"), + ] + try: + font = ImageFont.truetype("arial.ttf", 14) + small_font = ImageFont.truetype("arial.ttf", 12) + except Exception: + font = ImageFont.load_default() + small_font = font + for value, label in ticks: + ratio = (value - min_mm_year) / max(max_mm_year - min_mm_year, 1e-6) + x = margin_x + int(round(ratio * gradient_width)) + draw.line([(x, margin_top + bar_height), (x, margin_top + bar_height + 7)], fill=(15, 23, 42, 220), width=1) + text = f"{label} mm/yr" + bbox = draw.textbbox((0, 0), text, font=font) + draw.text((x - (bbox[2] - bbox[0]) / 2, margin_top + bar_height + 10), text, fill=(15, 23, 42, 255), font=font) + source_label = "Gamma hls.cm" + source_bbox = draw.textbbox((0, 0), source_label, font=small_font) + draw.text( + (width - margin_x - (source_bbox[2] - source_bbox[0]), margin_top + bar_height + 33), + source_label, + fill=(71, 85, 105, 255), + font=small_font, + ) + image.save(target, "PNG", optimize=True) + return str(target) + except Exception: + return None + + +def _build_gamma_hls_rate_preview( + source: Path, + target: Path, + *, + coverage_source: Optional[Path] = None, + min_native: float = -0.08, + max_native: float = 0.08, +) -> Optional[str]: + if target.is_file(): + return str(target) + if not source.is_file(): + return None + colormap_path = _find_gamma_hls_colormap() + colors = _read_gamma_colormap(colormap_path) if colormap_path else [] + if not colors: + return None + try: + import numpy as np + import rasterio + from PIL import Image + from rasterio.enums import Resampling + except Exception: + return None + + try: + with rasterio.open(source) as src: + scale = max(src.width / 1600, src.height / 1600, 1) + out_w = max(1, int(src.width / scale)) + out_h = max(1, int(src.height / scale)) + data = src.read(1, out_shape=(out_h, out_w), resampling=Resampling.nearest).astype("float32") + coverage = np.ones((out_h, out_w), dtype=bool) + if coverage_source and coverage_source.is_file(): + with rasterio.open(coverage_source) as coverage_src: + if coverage_src.width == src.width and coverage_src.height == src.height and coverage_src.count >= 3: + coverage_rgb = coverage_src.read( + [1, 2, 3], + out_shape=(3, out_h, out_w), + resampling=Resampling.nearest, + ) + coverage = np.any(coverage_rgb != 0, axis=0) + valid = np.isfinite(data) & coverage & (data != 0.0) + ratio = np.clip((data - min_native) / max(max_native - min_native, 1e-12), 0.0, 1.0) + color_array = np.asarray(colors, dtype=np.uint8) + indices = np.rint(ratio * (len(color_array) - 1)).astype(np.int32) + rgb = color_array[np.clip(indices, 0, len(color_array) - 1)] + alpha = np.where(valid, 255, 0).astype(np.uint8) + image = Image.fromarray(np.dstack([rgb, alpha]), mode="RGBA") + resampling = getattr(getattr(Image, "Resampling", Image), "LANCZOS") + image.thumbnail((1600, 1600), resampling) + target.parent.mkdir(parents=True, exist_ok=True) + image.save(target, "PNG", optimize=True) + return str(target) + except Exception: + return None + + +def _parse_expert_disp_point_table(disp_point_path: Path) -> list[dict[str, Any]]: + if not disp_point_path.is_file(): + return [] + import csv + + rows = list(csv.reader(disp_point_path.read_text(encoding="utf-8", errors="ignore").splitlines())) + if len(rows) < 2: + return [] + header = [str(value or "").strip() for value in rows[0]] + dates = [value for value in header[5:] if value] + points: list[dict[str, Any]] = [] + for index, row in enumerate(rows[1:], start=1): + if len(row) < 5: + continue + values = [str(value or "").strip() for value in row] + try: + img_x = int(float(values[0])) + img_y = int(float(values[1])) + except ValueError: + continue + displacements: list[dict[str, Any]] = [] + for date_text, value_text in zip(dates, values[5:]): + try: + displacement = float(value_text) + except ValueError: + continue + date_clean = date_text.strip() + if len(date_clean) == 8 and date_clean.isdigit(): + date_iso = f"{date_clean[0:4]}-{date_clean[4:6]}-{date_clean[6:8]}" + else: + date_iso = date_clean + displacements.append({"date": date_iso, "displacement_mm": displacement}) + points.append( + { + "point_id": f"expert_point_{index:03d}", + "img_x": img_x, + "img_y": img_y, + "height_m": _safe_float(values[2]) if len(values) > 2 else None, + "deformation_rate_mm_per_year": _safe_float(values[3]) if len(values) > 3 else None, + "stdev_residual_phase_rad": _safe_float(values[4]) if len(values) > 4 else None, + "displacements": displacements, + } + ) + return points + + +def _read_expert_sbas_dates(run_dir: Path) -> list[str]: + rmli_tab = run_dir / "sbas" / "RMLI_tab" + dates: list[str] = [] + if rmli_tab.is_file(): + for line in rmli_tab.read_text(encoding="utf-8", errors="ignore").splitlines(): + parts = line.split() + if not parts: + continue + match = re.search(r"(\d{8})", Path(parts[0]).name) + if match: + raw = match.group(1) + dates.append(f"{raw[0:4]}-{raw[4:6]}-{raw[6:8]}") + if dates: + return dates + points = _parse_expert_disp_point_table(run_dir / "publish" / "points" / "disp_point.txt") + for point in points: + displacements = point.get("displacements") or [] + if displacements: + return [str(item.get("date") or "") for item in displacements if item.get("date")] + return [] + + +def _read_radar_float32(path: Path, *, width: int, img_x: int, img_y: int) -> Optional[float]: + if width <= 0 or img_x < 0 or img_y < 0 or not path.is_file(): + return None + offset = (img_y * width + img_x) * 4 + try: + if offset < 0 or offset + 4 > path.stat().st_size: + return None + with path.open("rb") as fp: + fp.seek(offset) + payload = fp.read(4) + if len(payload) != 4: + return None + value = struct.unpack(">f", payload)[0] + return value if math.isfinite(value) else None + except Exception: + return None + + +def _read_geo_rate_window( + *, + source_path: Path, + coverage_path: Path, + center_row: int, + center_col: int, + radius: int, + width: int, + height: int, +) -> tuple[Any, Any, int, int]: + import numpy as np + import rasterio + + row0 = max(0, center_row - radius) + row1 = min(height - 1, center_row + radius) + col0 = max(0, center_col - radius) + col1 = min(width - 1, center_col + radius) + if row0 > row1 or col0 > col1: + return None, None, row0, col0 + window = rasterio.windows.Window(col0, row0, col1 - col0 + 1, row1 - row0 + 1) + with rasterio.open(source_path) as src: + rate = src.read(1, window=window) + coverage = None + if coverage_path.is_file(): + with rasterio.open(coverage_path) as cov: + if cov.width == width and cov.height == height and cov.count >= 3: + rgb = cov.read([1, 2, 3], window=window) + coverage = np.any(rgb != 0, axis=0) + return rate, coverage, row0, col0 + + +def _query_expert_gamma_point_timeseries(run_dir: Path, *, lon: float, lat: float) -> dict[str, Any]: + source_path = run_dir / "publish" / "geotiff" / "geo_los_def_rate.tif" + coverage_path = run_dir / "publish" / "geotiff" / "geo_los_def_rate_rgb.tif" + lt_path = run_dir / "dem" / f"{_safe_read_json(run_dir / 'run_manifest.json').get('reference_date') or ''}.lt_fine" + if not lt_path.is_file(): + candidates = sorted((run_dir / "dem").glob("*.lt_fine")) + lt_path = candidates[0] if candidates else lt_path + mli_par = run_dir / "sbas" / "mli.ave.par" + disp_tab = run_dir / "sbas" / "disp.TS_tab" + if not source_path.is_file(): + raise FileNotFoundError("geo_los_def_rate.tif is missing") + if not lt_path.is_file(): + raise FileNotFoundError("Gamma lookup table *.lt_fine is missing") + if not mli_par.is_file(): + raise FileNotFoundError("mli.ave.par is missing") + if not disp_tab.is_file(): + raise FileNotFoundError("disp.TS_tab is missing") + + try: + import numpy as np + import rasterio + except Exception as exc: + raise RuntimeError(f"point_query_dependency_unavailable: {exc}") from exc + + with rasterio.open(source_path) as src: + width = int(src.width) + height = int(src.height) + if lon < src.bounds.left or lon > src.bounds.right or lat < src.bounds.bottom or lat > src.bounds.top: + raise ValueError("requested WGS84 coordinate is outside product geocoded bounds") + center_row, center_col = src.index(lon, lat) + transform = src.transform + + valid_choice: Optional[dict[str, Any]] = None + for radius in [0, 1, 2, 4, 8, 16, 32, 64, 128, 256]: + rate, coverage, row0, col0 = _read_geo_rate_window( + source_path=source_path, + coverage_path=coverage_path, + center_row=int(center_row), + center_col=int(center_col), + radius=radius, + width=width, + height=height, + ) + if rate is None: + continue + valid = np.isfinite(rate) & (rate != 0.0) + if coverage is not None: + valid &= coverage + if not np.any(valid): + continue + rows, cols = np.where(valid) + abs_rows = rows + row0 + abs_cols = cols + col0 + distances = (abs_rows - int(center_row)) ** 2 + (abs_cols - int(center_col)) ** 2 + best_index = int(np.argmin(distances)) + matched_row = int(abs_rows[best_index]) + matched_col = int(abs_cols[best_index]) + matched_lon, matched_lat = rasterio.transform.xy(transform, matched_row, matched_col, offset="center") + rate_value = float(rate[rows[best_index], cols[best_index]]) + valid_choice = { + "geo_row": matched_row, + "geo_col": matched_col, + "lon": float(matched_lon), + "lat": float(matched_lat), + "los_rate_mm_per_year": rate_value * 1000.0, + "source_native_m_per_year": rate_value, + "search_radius_pixels": radius, + "distance_m": _haversine_m(lon, lat, float(matched_lon), float(matched_lat)), + } + break + + if valid_choice is None: + raise ValueError("no valid deformation pixel found near requested WGS84 coordinate") + + lt_offset = (int(valid_choice["geo_row"]) * width + int(valid_choice["geo_col"])) * 8 + if lt_offset < 0 or lt_offset + 8 > lt_path.stat().st_size: + raise ValueError("matched geocoded pixel is outside lookup table") + with lt_path.open("rb") as fp: + fp.seek(lt_offset) + range_value, azimuth_value = struct.unpack(">ff", fp.read(8)) + if not math.isfinite(range_value) or not math.isfinite(azimuth_value): + raise ValueError("lookup table returned invalid radar coordinates") + + radar_width = _read_gamma_int_param(mli_par, "range_samples") or _read_gamma_int_param(mli_par, "width") or 0 + radar_lines = _read_gamma_int_param(mli_par, "azimuth_lines") or _read_gamma_int_param(mli_par, "nlines") or 0 + img_x = int(round(range_value)) + img_y = int(round(azimuth_value)) + if radar_width <= 0 or radar_lines <= 0: + raise ValueError("invalid radar image dimensions") + img_x = min(max(0, img_x), radar_width - 1) + img_y = min(max(0, img_y), radar_lines - 1) + + raw_disp_paths = [ + Path(_wsl_path_to_windows(line.strip())) + for line in disp_tab.read_text(encoding="utf-8", errors="ignore").splitlines() + if line.strip() + ] + dates = _read_expert_sbas_dates(run_dir) + displacements: list[dict[str, Any]] = [] + for index, disp_path in enumerate(raw_disp_paths): + value_m = _read_radar_float32(disp_path, width=radar_width, img_x=img_x, img_y=img_y) + date = dates[index] if index < len(dates) else f"epoch_{index + 1:03d}" + if value_m is None: + displacements.append({"date": date, "displacement_mm": None}) + else: + displacements.append({"date": date, "displacement_mm": value_m * 1000.0}) + + if not any(item.get("displacement_mm") is not None for item in displacements): + raise ValueError("matched radar pixel has no readable displacement values") + + return { + "schema": "insar.gamma-sbas-point-query/v1", + "source_tool": "disp.TS_tab_radar_pixel_sample", + "query": {"lon": lon, "lat": lat}, + "matched": { + **valid_choice, + "used_nearest": int(valid_choice["geo_row"]) != int(center_row) + or int(valid_choice["geo_col"]) != int(center_col), + "input_geo_row": int(center_row), + "input_geo_col": int(center_col), + "radar_range": float(range_value), + "radar_azimuth": float(azimuth_value), + "img_x": img_x, + "img_y": img_y, + "radar_width": radar_width, + "radar_lines": radar_lines, + }, + "unit": "mm", + "rate_unit": "mm/yr", + "displacement_count": len(displacements), + "displacements": displacements, + } + + +def _locate_radar_points_in_geocoded_product(run_dir: Path, points: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + targets: list[tuple[str, float, float]] = [] + for point in points: + point_id = str(point.get("point_id") or "").strip() + img_x = _safe_float(point.get("img_x")) + img_y = _safe_float(point.get("img_y")) + if point_id and img_x is not None and img_y is not None: + targets.append((point_id, img_x, img_y)) + if not targets: + return {} + + source_path = run_dir / "publish" / "geotiff" / "geo_los_def_rate.tif" + coverage_path = run_dir / "publish" / "geotiff" / "geo_los_def_rate_rgb.tif" + manifest = _safe_read_json(run_dir / "run_manifest.json") + lt_path = run_dir / "dem" / f"{manifest.get('reference_date') or ''}.lt_fine" + if not lt_path.is_file(): + candidates = sorted((run_dir / "dem").glob("*.lt_fine")) + lt_path = candidates[0] if candidates else lt_path + if not source_path.is_file() or not lt_path.is_file(): + return {} + + try: + import numpy as np + import rasterio + from rasterio.windows import Window + except Exception: + return {} + + try: + with rasterio.open(source_path) as src: + width = int(src.width) + height = int(src.height) + transform = src.transform + expected_size = width * height * 2 * 4 + if lt_path.stat().st_size < expected_size: + return {} + + lookup = np.memmap(lt_path, dtype=">f4", mode="r", shape=(height, width, 2)) + coarse_step = 16 + coarse_best: dict[str, dict[str, Any]] = { + point_id: {"distance2": math.inf, "row": None, "col": None} + for point_id, _, _ in targets + } + for row0 in range(0, height, coarse_step * 128): + row1 = min(height, row0 + coarse_step * 128) + sampled = lookup[row0:row1:coarse_step, 0:width:coarse_step, :] + if sampled.size == 0: + continue + range_chunk = np.asarray(sampled[:, :, 0], dtype=np.float32) + azimuth_chunk = np.asarray(sampled[:, :, 1], dtype=np.float32) + valid = np.isfinite(range_chunk) & np.isfinite(azimuth_chunk) & (range_chunk >= 0.0) & (azimuth_chunk >= 0.0) + if not np.any(valid): + continue + for point_id, img_x, img_y in targets: + distances = (range_chunk - np.float32(img_x)) ** 2 + (azimuth_chunk - np.float32(img_y)) ** 2 + distances = np.where(valid, distances, np.float32(np.inf)) + flat_index = int(np.argmin(distances)) + value = float(distances.flat[flat_index]) + if value >= coarse_best[point_id]["distance2"]: + continue + local_row, local_col = np.unravel_index(flat_index, distances.shape) + coarse_best[point_id] = { + "distance2": value, + "row": int(row0 + local_row * coarse_step), + "col": int(local_col * coarse_step), + } + + results: dict[str, dict[str, Any]] = {} + coverage_src = None + if coverage_path.is_file(): + try: + candidate = rasterio.open(coverage_path) + if candidate.width == width and candidate.height == height and candidate.count >= 3: + coverage_src = candidate + else: + candidate.close() + except Exception: + coverage_src = None + + try: + refine_radius = coarse_step * 12 + for point_id, img_x, img_y in targets: + seed = coarse_best.get(point_id) or {} + seed_row = seed.get("row") + seed_col = seed.get("col") + if seed_row is None or seed_col is None: + continue + row0 = max(0, int(seed_row) - refine_radius) + row1 = min(height, int(seed_row) + refine_radius + 1) + col0 = max(0, int(seed_col) - refine_radius) + col1 = min(width, int(seed_col) + refine_radius + 1) + local = lookup[row0:row1, col0:col1, :] + range_local = np.asarray(local[:, :, 0], dtype=np.float32) + azimuth_local = np.asarray(local[:, :, 1], dtype=np.float32) + valid = np.isfinite(range_local) & np.isfinite(azimuth_local) & (range_local >= 0.0) & (azimuth_local >= 0.0) + if not np.any(valid): + continue + distances = (range_local - np.float32(img_x)) ** 2 + (azimuth_local - np.float32(img_y)) ** 2 + distances = np.where(valid, distances, np.float32(np.inf)) + flat_index = int(np.argmin(distances)) + value = float(distances.flat[flat_index]) + if not math.isfinite(value): + continue + local_row, local_col = np.unravel_index(flat_index, distances.shape) + row = int(row0 + local_row) + col = int(col0 + local_col) + lon, lat = rasterio.transform.xy(transform, row, col, offset="center") + rate_mm_per_year = None + coverage_valid = None + try: + rate_window = src.read(1, window=Window(col, row, 1, 1)) + rate_value = float(rate_window[0, 0]) + if math.isfinite(rate_value): + rate_mm_per_year = rate_value * 1000.0 + except Exception: + rate_mm_per_year = None + if coverage_src is not None: + try: + rgb = coverage_src.read([1, 2, 3], window=Window(col, row, 1, 1)) + coverage_valid = bool(np.any(rgb != 0)) + except Exception: + coverage_valid = None + results[point_id] = { + "lon": float(lon), + "lat": float(lat), + "geo_row": row, + "geo_col": col, + "geo_match_distance_px": math.sqrt(value), + "geo_los_rate_mm_per_year": rate_mm_per_year, + "geo_coverage_valid": coverage_valid, + "geo_source": "gamma_lt_fine_nearest_inverse", + } + finally: + if coverage_src is not None: + coverage_src.close() + del lookup + return results + except Exception: + return {} + + +def _file_summary(path: Path) -> dict[str, Any]: + return { + "path": str(path), + "exists": path.is_file(), + "size_bytes": path.stat().st_size if path.is_file() else None, + } + + +def _build_expert_gamma_primary_geotiff_stats(path: Path) -> dict[str, Any]: + stats = { + **_file_summary(path), + "schema": "insar.gamma-sbas-expert-primary-geotiff-stats/v1", + "source": "expert_gamma_geo_los_def_rate", + "unit": "mm/yr", + "native_unit": "m/yr", + "scale_to_unit": 1000.0, + "zero_is_valid": False, + "validity_rule": "expert_rgb_coverage_finite_nonzero_values", + } + if not path.is_file(): + return stats + + try: + import numpy as np + import rasterio + from rasterio.windows import Window + except Exception as exc: + stats["error"] = f"raster_stats_dependency_unavailable: {exc}" + return stats + + try: + coverage_path = path.with_name("geo_los_def_rate_rgb.tif") + coverage_mask_source = str(coverage_path) if coverage_path.is_file() else None + valid_count = 0 + zero_count = 0 + nonzero_count = 0 + sample_count = 0 + value_sum = 0.0 + value_sumsq = 0.0 + value_min: Optional[float] = None + value_max: Optional[float] = None + percentile_chunks: list[Any] = [] + max_percentile_samples = 5_000_000 + tile_size = 1024 + + with rasterio.open(path) as src: + coverage_src = None + if coverage_path.is_file(): + try: + candidate = rasterio.open(coverage_path) + if candidate.width == src.width and candidate.height == src.height and candidate.count >= 3: + coverage_src = candidate + else: + candidate.close() + except Exception: + coverage_src = None + total_count = int(src.width * src.height) + stats.update( + { + "width": int(src.width), + "height": int(src.height), + "band_count": int(src.count), + "dtype": str(src.dtypes[0]) if src.dtypes else None, + "crs": str(src.crs) if src.crs else None, + "nodata": _safe_float(src.nodata), + "metadata_nodata_applied": _safe_float(src.nodata) == 0.0, + "coverage_mask_source": coverage_mask_source if coverage_src is not None else None, + "bounds": { + "left": _safe_float(src.bounds.left), + "bottom": _safe_float(src.bounds.bottom), + "right": _safe_float(src.bounds.right), + "top": _safe_float(src.bounds.top), + }, + "total_count": total_count, + } + ) + for row_off in range(0, src.height, tile_size): + for col_off in range(0, src.width, tile_size): + window = Window( + col_off=col_off, + row_off=row_off, + width=min(tile_size, src.width - col_off), + height=min(tile_size, src.height - row_off), + ) + data = src.read(1, window=window, masked=False) + values = np.asarray(data, dtype=np.float64).reshape(-1) + if values.size == 0: + continue + finite = np.isfinite(values) + if coverage_src is not None: + coverage_rgb = coverage_src.read([1, 2, 3], window=window, masked=False) + coverage = np.any(coverage_rgb != 0, axis=0).reshape(-1) + else: + coverage = np.ones(values.shape, dtype=bool) + covered_values = values[finite & coverage] + zero_count += int(np.count_nonzero(covered_values == 0.0)) + values = values[finite & coverage & (values != 0.0)] + if values.size == 0: + continue + scaled = values * 1000.0 + valid_count += int(scaled.size) + nonzero_count += int(np.count_nonzero(values != 0.0)) + value_sum += float(scaled.sum(dtype=np.float64)) + value_sumsq += float(np.square(scaled, dtype=np.float64).sum(dtype=np.float64)) + chunk_min = float(scaled.min()) + chunk_max = float(scaled.max()) + value_min = chunk_min if value_min is None else min(value_min, chunk_min) + value_max = chunk_max if value_max is None else max(value_max, chunk_max) + if sample_count < max_percentile_samples: + remaining = max_percentile_samples - sample_count + if scaled.size <= remaining: + sample = scaled + else: + step = max(1, int(np.ceil(scaled.size / remaining))) + sample = scaled[::step][:remaining] + percentile_chunks.append(sample.astype(np.float64, copy=True)) + sample_count += int(sample.size) + + if coverage_src is not None: + coverage_src.close() + + stats["valid_count"] = valid_count + stats["zero_count"] = zero_count + stats["nonzero_count"] = nonzero_count + stats["valid_ratio"] = (valid_count / stats["total_count"]) if stats.get("total_count") else None + if valid_count <= 0: + return stats + + mean = value_sum / valid_count + variance = max((value_sumsq / valid_count) - (mean * mean), 0.0) + stats.update( + { + "min": value_min, + "max": value_max, + "mean": mean, + "stddev": float(variance ** 0.5), + "sample_count": sample_count, + "percentiles_sampled": sample_count < valid_count, + } + ) + + if percentile_chunks: + percentile_values = np.concatenate(percentile_chunks) + p01, p05, median, p95, p99 = np.percentile(percentile_values, [1, 5, 50, 95, 99]) + stats.update( + { + "p01": float(p01), + "p05": float(p05), + "median": float(median), + "p95": float(p95), + "p99": float(p99), + } + ) + return stats + except Exception as exc: + stats["error"] = f"raster_stats_failed: {exc}" + return stats + + +def _build_expert_gamma_quality_summary(run_dir: Path) -> dict[str, Any]: + primary_stats = _build_expert_gamma_primary_geotiff_stats( + run_dir / "publish" / "geotiff" / "geo_los_def_rate.tif" + ) + return { + "schema": "insar.gamma-sbas-derived-quality-summary/v1", + "source": "derived_from_expert_gamma_outputs", + "note": "Catalog/UI inspection stats derived from expert geo_los_def_rate.tif; quality_summary.json is not required for the expert workflow.", + "primary_geotiff": primary_stats, + } + + +def _pixel_center(transform: tuple[float, float, float, float, float, float], row: int, col: int) -> tuple[float, float]: + a, b, c, d, e, f = transform + x = c + (col + 0.5) * a + (row + 0.5) * b + y = f + (col + 0.5) * d + (row + 0.5) * e + return float(x), float(y) + + +def _normalize_crs_label(value: Any) -> Optional[str]: + text = str(value or "").strip() + if not text: + return None + upper = text.upper() + if "EPSG" in upper and "4326" in upper: + return "EPSG:4326" + if "WGS 84" in upper or "WGS_1984" in upper: + return "EPSG:4326" + return text[:240] + + +def _build_expert_gamma_point_vector(run_dir: Path, *, summary_context: Optional[dict[str, Any]] = None) -> dict[str, Any]: + source_path = run_dir / "publish" / "geotiff" / "geo_los_def_rate.tif" + coverage_path = run_dir / "publish" / "geotiff" / "geo_los_def_rate_rgb.tif" + output_path = run_dir / "publish" / "vectors" / "los_rate_points.geojson.gz" + summary_path = run_dir / "publish" / "vectors" / "los_rate_points_summary.json" + previous_summary = _safe_read_json(summary_path) + context = summary_context or {} + stack_dates = context.get("stack_dates") or [] + admin_region = context.get("admin_region") if isinstance(context.get("admin_region"), dict) else {} + admin_names = admin_region.get("names") if isinstance(admin_region.get("names"), dict) else {} + fields = [ + "run_id", + "row", + "col", + "lon", + "lat", + "los_rate_mm_per_year", + "source_native_m_per_year", + "date_start", + "date_end", + "reference_date", + "admin_province", + "admin_city", + ] + summary: dict[str, Any] = { + "schema": "insar.gamma-sbas-expert-point-vector-summary/v1", + "generated_at": previous_summary.get("generated_at") or _utcnow().isoformat(timespec="seconds") + "Z", + "ready": False, + "feature_count": 0, + "output_geojson_gz": str(output_path), + "output_size_bytes": output_path.stat().st_size if output_path.is_file() else 0, + "fields": fields, + "source_geotiffs": { + "geo_los_def_rate": str(source_path), + "coverage_mask": str(coverage_path) if coverage_path.is_file() else None, + }, + "unit": "mm/yr", + "native_unit": "m/yr", + "scale_to_unit": 1000.0, + "zero_is_valid": False, + "validity_rule": "expert_rgb_coverage_finite_nonzero_values", + "date_start": context.get("date_start") or (stack_dates[0] if stack_dates else None), + "date_end": context.get("date_end") or (stack_dates[-1] if stack_dates else None), + "reference_date": context.get("reference_date"), + "admin_region": { + "province": admin_names.get("province") or admin_region.get("province"), + "city": admin_names.get("city") or admin_region.get("city"), + }, + "los_convention": context.get("los_sign_convention") or "Gamma expert geo_los_def_rate output; sign follows the expert workflow.", + "frontend_policy": "download_only; do not render full point GeoJSON in browser", + } + if not source_path.is_file(): + _write_json_if_changed(summary_path, {**summary, "error": "source_geotiff_missing"}) + return summary + if output_path.is_file() and summary_path.is_file(): + input_paths = [source_path] + if coverage_path.is_file(): + input_paths.append(coverage_path) + output_mtime = output_path.stat().st_mtime + if ( + previous_summary.get("validity_rule") == "expert_rgb_coverage_finite_nonzero_values" + and previous_summary.get("zero_is_valid") is False + and all(output_mtime >= item.stat().st_mtime for item in input_paths) + ): + return previous_summary + + try: + import numpy as np + import rasterio + from rasterio.windows import Window + except Exception as exc: + summary["error"] = f"point_vector_dependency_unavailable: {exc}" + _write_json_if_changed(summary_path, summary) + return summary + + try: + output_path.parent.mkdir(parents=True, exist_ok=True) + feature_count = 0 + zero_count = 0 + tile_size = 512 + with rasterio.open(source_path) as src: + coverage_src = None + if coverage_path.is_file(): + candidate = rasterio.open(coverage_path) + if candidate.width == src.width and candidate.height == src.height and candidate.count >= 3: + coverage_src = candidate + else: + candidate.close() + transform = tuple(float(value) for value in src.transform.to_gdal()) + # Convert GDAL geotransform (c, a, b, f, d, e) to affine tuple used by _pixel_center. + transform = (transform[1], transform[2], transform[0], transform[4], transform[5], transform[3]) + nodata = _safe_float(src.nodata) + summary.update( + { + "width": int(src.width), + "height": int(src.height), + "crs": _normalize_crs_label(str(src.crs) if src.crs else None), + "nodata": nodata, + "metadata_nodata_applied": nodata == 0.0, + "coverage_mask_source": str(coverage_path) if coverage_src is not None else None, + "total_count": int(src.width * src.height), + } + ) + with gzip.open(output_path, "wt", encoding="utf-8", compresslevel=6) as handle: + handle.write('{"type":"FeatureCollection","features":[\n') + first = True + for row_off in range(0, src.height, tile_size): + for col_off in range(0, src.width, tile_size): + window = Window( + col_off=col_off, + row_off=row_off, + width=min(tile_size, src.width - col_off), + height=min(tile_size, src.height - row_off), + ) + data = src.read(1, window=window, masked=False).astype("float64", copy=False) + finite = np.isfinite(data) + coverage = np.ones(data.shape, dtype=bool) + if coverage_src is not None: + rgb = coverage_src.read([1, 2, 3], window=window, masked=False) + coverage = np.any(rgb != 0, axis=0) + covered = finite & coverage + zero_count += int(np.count_nonzero(data[covered] == 0.0)) + valid = covered & (data != 0.0) + rows, cols = np.where(valid) + for local_row, local_col in zip(rows.tolist(), cols.tolist()): + row = int(row_off + local_row) + col = int(col_off + local_col) + lon, lat = _pixel_center(transform, row, col) + native_value = float(data[local_row, local_col]) + properties = { + "run_id": context.get("run_id") or run_dir.name, + "row": row, + "col": col, + "lon": lon, + "lat": lat, + "los_rate_mm_per_year": native_value * 1000.0, + "source_native_m_per_year": native_value, + "date_start": summary.get("date_start"), + "date_end": summary.get("date_end"), + "reference_date": summary.get("reference_date"), + "admin_province": summary["admin_region"].get("province"), + "admin_city": summary["admin_region"].get("city"), + } + feature = { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [lon, lat]}, + "properties": properties, + } + if not first: + handle.write(",\n") + handle.write(json.dumps(feature, ensure_ascii=False, separators=(",", ":"))) + first = False + feature_count += 1 + handle.write("\n]}\n") + if coverage_src is not None: + coverage_src.close() + summary.update( + { + "ready": output_path.is_file() and output_path.stat().st_size > 0, + "feature_count": feature_count, + "zero_count": zero_count, + "output_size_bytes": output_path.stat().st_size if output_path.is_file() else 0, + } + ) + _write_json_if_changed(summary_path, summary) + return summary + except Exception as exc: + summary["error"] = f"point_vector_export_failed: {exc}" + _write_json_if_changed(summary_path, summary) + return summary + + +def _write_text_if_changed(path: Path, text: str) -> bool: + if path.is_file(): + try: + if path.read_text(encoding="utf-8", errors="ignore") == text: + return False + except Exception: + pass + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return True + + +def _write_json_if_changed(path: Path, payload: dict[str, Any]) -> bool: + text = json.dumps(payload, ensure_ascii=False, indent=2) + "\n" + return _write_text_if_changed(path, text) + + +_MONITOR_POINT_SELECTION_DEFINITIONS = [ + ( + "toward_high_rate_low_sigma", + "趋近雷达高形变低残差点", + "rate > 0，且绝对速率位于高分位，残差低，用于检查明显正向形变区域。", + ), + ( + "away_high_rate_low_sigma", + "远离雷达高形变低残差点", + "rate < 0，且绝对速率位于高分位，残差低，用于检查明显负向形变区域。", + ), + ( + "high_abs_rate_low_sigma", + "高绝对速率低残差点", + "不区分正负，优先选择绝对速率高且残差低的有效点。", + ), + ( + "stable_low_sigma", + "近零低残差代表点", + "绝对速率位于低分位且残差低，用于对照相对稳定区域。", + ), + ( + "center_valid", + "覆盖区中心有效点", + "从有效像元中选取最接近雷达网格中心的点，用于空间位置对照。", + ), +] + + +def _read_expert_monitor_point_selection(points_dir: Path) -> dict[tuple[int, int], dict[str, Any]]: + selection_path = points_dir / "disp_point_sel.txt" + rows: list[dict[str, Any]] = [] + if selection_path.is_file(): + for index, line in enumerate(selection_path.read_text(encoding="utf-8", errors="ignore").splitlines()): + parts = line.strip().split() + if len(parts) < 2: + continue + try: + img_x = int(float(parts[0])) + img_y = int(float(parts[1])) + except ValueError: + continue + key, label, description = ( + _MONITOR_POINT_SELECTION_DEFINITIONS[index] + if index < len(_MONITOR_POINT_SELECTION_DEFINITIONS) + else ( + f"extra_representative_{index + 1:03d}", + f"补充代表点 {index + 1}", + "自动选点数量超过内置策略说明时的补充代表点。", + ) + ) + rows.append( + { + "selection_rank": index + 1, + "selection_key": key, + "selection_label": label, + "selection_description": description, + "img_x": img_x, + "img_y": img_y, + } + ) + payload = { + "schema": "insar.gamma-sbas-expert-monitor-point-selection/v1", + "generated_at": _utcnow().isoformat(timespec="seconds") + "Z", + "source": "disp_point_sel.txt", + "selection_count": len(rows), + "strategy": "auto_representative_points", + "strategy_note": "自动选取趋近/远离雷达高形变、绝对高形变、近零稳定和中心有效点；时序仍由 Gamma disp_prt_2d 输出。", + "points": rows, + } + if rows: + _write_json_if_changed(points_dir / "disp_point_selection.json", payload) + return {(int(item["img_x"]), int(item["img_y"])): item for item in rows} + + +def _wsl_path_to_windows(path: str) -> str: + text = str(path or "").strip() + if text.startswith("/mnt/") and len(text) > 6 and text[6] == "/": + drive = text[5].upper() + return f"{drive}:{text[6:]}".replace("/", "\\") + return text + + +def _build_expert_unwrapped_phase_radar_browse( + run_dir: Path, + source_paths: list[Path], + *, + width: int, + lines: int, +) -> dict[str, dict[str, Any]]: + unwrapped_dir = run_dir / "publish" / "geotiff" / "unwrapped" + browse_by_pair: dict[str, dict[str, Any]] = {} + if not source_paths or width <= 0 or lines <= 0: + return browse_by_pair + try: + import numpy as np + from PIL import Image + except Exception: + return browse_by_pair + + colormap_path = _find_gamma_colormap("rmg.cm") + colors = _read_gamma_colormap(colormap_path) if colormap_path else [] + if not colors: + return browse_by_pair + color_array = np.asarray(colors, dtype=np.uint8) + unwrapped_dir.mkdir(parents=True, exist_ok=True) + _build_gamma_colormap_colorbar( + unwrapped_dir / "unwrapped_phase_rmg_colorbar.png", + colormap_name="rmg.cm", + min_value=-6.28, + max_value=6.28, + unit="rad", + title="Gamma unwrapped phase browse", + ) + for source in source_paths: + pair_id = source.name.replace(".unw.atmsub_1", "") + output_bmp = unwrapped_dir / f"{source.name}.rdc_rmg.bmp" + output_png = unwrapped_dir / f"{source.name}.rdc_rmg_preview.png" + item = { + "pair_id": pair_id, + "radar_coordinates": True, + "browse_command": "rasdt_pwr <unw.atmsub_1> mli.ave <width> 1 - 1 1 -6.28 6.28 1 rmg.cm ... 1.0 0.35 24", + "colormap": "Gamma rmg.cm", + "display_range_rad": [-6.28, 6.28], + "bmp": _file_summary(output_bmp), + "preview": _file_summary(output_png), + "ready": False, + } + if not source.is_file(): + item["error"] = "source_missing" + browse_by_pair[pair_id] = item + continue + if ( + output_png.is_file() + and output_png.stat().st_mtime >= source.stat().st_mtime + and (not output_bmp.is_file() or output_bmp.stat().st_mtime >= source.stat().st_mtime) + ): + item["bmp"] = _file_summary(output_bmp) + item["preview"] = _file_summary(output_png) + item["ready"] = output_png.stat().st_size > 0 + browse_by_pair[pair_id] = item + continue + try: + data = np.fromfile(source, dtype=">f4", count=int(width) * int(lines)).reshape((int(lines), int(width))) + valid = np.isfinite(data) & (data != 0.0) + wrapped = ((data + 6.28) % 12.56) - 6.28 + ratio = np.clip((wrapped + 6.28) / 12.56, 0.0, 1.0) + indices = np.rint(ratio * (len(color_array) - 1)).astype(np.int32) + rgb = color_array[np.clip(indices, 0, len(color_array) - 1)] + rgb = np.where(valid[..., None], rgb, 0).astype(np.uint8) + image = Image.fromarray(rgb, mode="RGB") + image.save(output_bmp, "BMP") + preview = image.copy() + resampling = getattr(getattr(Image, "Resampling", Image), "LANCZOS") + preview.thumbnail((1600, 1600), resampling) + preview.save(output_png, "PNG", optimize=True) + item.update( + { + "bmp": _file_summary(output_bmp), + "preview": _file_summary(output_png), + "valid_count": int(np.count_nonzero(valid)), + "ready": output_png.is_file() and output_png.stat().st_size > 0, + } + ) + except Exception as exc: + item["error"] = str(exc) + browse_by_pair[pair_id] = item + return browse_by_pair + + +def _build_expert_unwrapped_phase_derivatives(run_dir: Path) -> dict[str, Any]: + unwrapped_dir = run_dir / "publish" / "geotiff" / "unwrapped" + summary_path = unwrapped_dir / "unwrapped_phase_summary.json" + final_tab = run_dir / "sbas" / "final_unw_tab" + manifest = _safe_read_json(run_dir / "run_manifest.json") + stack_manifest = _safe_read_json(run_dir / "stack_manifest.json") + stack = stack_manifest.get("stack") if isinstance(stack_manifest.get("stack"), dict) else {} + reference_date = str( + manifest.get("reference_date") + or stack.get("reference_date") + or (manifest.get("coregistration") or {}).get("reference_date") + or "" + ).strip() + dem_par = run_dir / "dem" / f"{reference_date}_seg.dem_par" if reference_date else next(iter(sorted((run_dir / "dem").glob("*_seg.dem_par"))), run_dir / "dem" / "_missing_seg.dem_par") + lookup = run_dir / "dem" / f"{reference_date}.lt_fine" if reference_date else next(iter(sorted((run_dir / "dem").glob("*.lt_fine"))), run_dir / "dem" / "_missing.lt_fine") + mli_par = run_dir / "sbas" / "mli.ave.par" + previous = _safe_read_json(summary_path) + source_paths: list[Path] = [] + if final_tab.is_file(): + for line in final_tab.read_text(encoding="utf-8", errors="ignore").splitlines(): + text = line.strip().split() + if text: + source_paths.append(Path(_wsl_path_to_windows(text[0]))) + + summary: dict[str, Any] = { + "schema": "insar.gamma-sbas-expert-unwrapped-phase-summary/v1", + "generated_at": previous.get("generated_at") or _utcnow().isoformat(timespec="seconds") + "Z", + "source_stage": "final_unw_tab", + "source_tab": _file_summary(final_tab), + "source_count": len(source_paths), + "ready": False, + "products": [], + "note": "Geocoded GeoTIFF derivatives from the final unwrapped phase files consumed by the expert Gamma SBAS inversion.", + } + if not source_paths: + _write_json_if_changed(summary_path, summary) + return summary + + outputs = [ + unwrapped_dir / f"{path.name}.geo.tif" + for path in source_paths + ] + radar_outputs = [ + unwrapped_dir / f"{path.name}.rdc_rmg_preview.png" + for path in source_paths + ] + radar_colorbar = unwrapped_dir / "unwrapped_phase_rmg_colorbar.png" + previous_products = previous.get("products") if isinstance(previous.get("products"), list) else [] + previous_has_radar_browse = ( + len(previous_products) == len(source_paths) + and all((item.get("radar_browse") or {}).get("ready") for item in previous_products if isinstance(item, dict)) + ) + if ( + summary_path.is_file() + and all(output.is_file() for output in outputs) + and all(output.is_file() for output in radar_outputs) + and radar_colorbar.is_file() + and previous_has_radar_browse + and all( + output.stat().st_mtime >= source.stat().st_mtime + for source, output in zip(source_paths, outputs) + if source.is_file() + ) + and all( + output.stat().st_mtime >= source.stat().st_mtime + for source, output in zip(source_paths, radar_outputs) + if source.is_file() + ) + ): + return previous + + try: + import numpy as np + import rasterio + from PIL import Image + from rasterio.transform import from_origin + except Exception as exc: + summary["error"] = f"unwrapped_phase_dependency_unavailable: {exc}" + _write_json_if_changed(summary_path, summary) + return summary + + def read_gamma_param(path: Path, key: str) -> Optional[str]: + if not path.is_file(): + return None + for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): + parts = line.split() + if parts and parts[0].rstrip(":") == key.rstrip(":") and len(parts) > 1: + return parts[1] + return None + + try: + rdc_width = _safe_int(read_gamma_param(mli_par, "range_samples")) + rdc_lines = _safe_int(read_gamma_param(mli_par, "azimuth_lines")) + dem_width = _safe_int(read_gamma_param(dem_par, "width")) + dem_lines = _safe_int(read_gamma_param(dem_par, "nlines")) + corner_lon = _safe_float(read_gamma_param(dem_par, "corner_lon")) + corner_lat = _safe_float(read_gamma_param(dem_par, "corner_lat")) + post_lon = _safe_float(read_gamma_param(dem_par, "post_lon")) + post_lat = _safe_float(read_gamma_param(dem_par, "post_lat")) + if None in (rdc_width, rdc_lines, dem_width, dem_lines, corner_lon, corner_lat, post_lon, post_lat): + raise RuntimeError("required Gamma geometry parameters are missing") + if not lookup.is_file(): + raise FileNotFoundError(str(lookup)) + + unwrapped_dir.mkdir(parents=True, exist_ok=True) + radar_browse_by_pair = _build_expert_unwrapped_phase_radar_browse( + run_dir, + source_paths, + width=int(rdc_width), + lines=int(rdc_lines), + ) + lut = np.fromfile(lookup, dtype=">c8").reshape((int(dem_lines), int(dem_width))) + rng = np.rint(lut.real).astype(np.int32) + az = np.rint(lut.imag).astype(np.int32) + valid_lut = ( + np.isfinite(lut.real) + & np.isfinite(lut.imag) + & (rng >= 0) + & (rng < int(rdc_width)) + & (az >= 0) + & (az < int(rdc_lines)) + ) + del lut + transform = from_origin(float(corner_lon), float(corner_lat), abs(float(post_lon)), abs(float(post_lat))) + products: list[dict[str, Any]] = [] + preview_limit = 6 + for source in source_paths: + if not source.is_file(): + products.append({"source": str(source), "ready": False, "error": "source_missing"}) + continue + pair_id = source.name.replace(".unw.atmsub_1", "") + output_tif = unwrapped_dir / f"{source.name}.geo.tif" + output_preview = unwrapped_dir / f"{source.name}.geo_preview.png" + data = np.fromfile(source, dtype=">f4", count=int(rdc_width) * int(rdc_lines)).reshape((int(rdc_lines), int(rdc_width))) + geo = np.full((int(dem_lines), int(dem_width)), np.nan, dtype=np.float32) + geo[valid_lut] = data[az[valid_lut], rng[valid_lut]] + finite_nonzero = np.isfinite(geo) & (geo != 0.0) + with rasterio.open( + output_tif, + "w", + driver="GTiff", + width=int(dem_width), + height=int(dem_lines), + count=1, + dtype="float32", + crs=_WGS84_GEOGCS_WKT, + transform=transform, + nodata=np.nan, + compress="deflate", + ) as dst: + dst.write(geo, 1) + if np.any(finite_nonzero) and len(products) < preview_limit: + valid_values = geo[finite_nonzero] + p02, p98 = np.nanpercentile(valid_values, [2, 98]) + if not np.isfinite(p02) or not np.isfinite(p98) or p98 <= p02: + p02 = float(np.nanmin(valid_values)) + p98 = float(np.nanmax(valid_values)) + norm = np.clip((geo - p02) / max(p98 - p02, 1e-6), 0, 1) + gray = np.where(finite_nonzero, norm * 255, 0).astype("uint8") + alpha = np.where(finite_nonzero, 255, 0).astype("uint8") + rgba = np.dstack([gray, gray, gray, alpha]) + image = Image.fromarray(rgba, mode="RGBA") + resampling = getattr(getattr(Image, "Resampling", Image), "LANCZOS") + image.thumbnail((1600, 1600), resampling) + image.save(output_preview, "PNG", optimize=True) + products.append( + { + "pair_id": pair_id, + "source": str(source), + "geotiff": _file_summary(output_tif), + "preview": _file_summary(output_preview), + "radar_browse": radar_browse_by_pair.get(pair_id) or {}, + "valid_count": int(np.count_nonzero(finite_nonzero)), + "unit": "rad", + "description": "Final model-corrected unwrapped phase used by Gamma SBAS inversion, geocoded for inspection.", + "ready": output_tif.is_file() and output_tif.stat().st_size > 0, + } + ) + del data, geo, finite_nonzero + summary.update( + { + "generated_at": _utcnow().isoformat(timespec="seconds") + "Z", + "ready": bool(products) and all(item.get("ready") for item in products), + "products": products, + } + ) + _write_json_if_changed(summary_path, summary) + return summary + except Exception as exc: + summary["error"] = f"unwrapped_phase_export_failed: {exc}" + _write_json_if_changed(summary_path, summary) + return summary + + +def _write_expert_monitor_point_derivatives(run_dir: Path) -> dict[str, Any]: + points_dir = run_dir / "publish" / "points" + disp_point_path = points_dir / "disp_point.txt" + items_path = points_dir / "items.txt" + selection_by_xy = _read_expert_monitor_point_selection(points_dir) + points = _parse_expert_disp_point_table(disp_point_path) + geo_locations = _locate_radar_points_in_geocoded_product(run_dir, points) + out_dir = run_dir / "publish" / "monitor_points" + summary_path = run_dir / "monitor_points_summary.json" + monitor_outputs: list[dict[str, Any]] = [] + if points: + out_dir.mkdir(parents=True, exist_ok=True) + expected_names = set() + for index in range(1, len(points) + 1): + point_id = f"expert_point_{index:03d}" + expected_names.update( + { + f"{point_id}_timeseries.csv", + f"{point_id}_timeseries.png", + f"{point_id}_metadata.json", + } + ) + for old_path in out_dir.iterdir(): + if old_path.is_file() and old_path.name not in expected_names: + try: + old_path.unlink() + except Exception: + pass + + for point in points: + point_id = str(point["point_id"]) + csv_path = out_dir / f"{point_id}_timeseries.csv" + png_path = out_dir / f"{point_id}_timeseries.png" + metadata_path = out_dir / f"{point_id}_metadata.json" + displacements = point.get("displacements") or [] + csv_lines = ["date,displacement_mm"] + csv_lines.extend(f"{item['date']},{item['displacement_mm']:.6f}" for item in displacements) + curve_needs_refresh = _write_text_if_changed(csv_path, "\n".join(csv_lines) + "\n") + selection = selection_by_xy.get((int(point.get("img_x") or 0), int(point.get("img_y") or 0)), {}) + geo_location = geo_locations.get(point_id) or {} + + metadata = { + "schema": "insar.gamma-sbas-expert-monitor-point/v1", + "point_id": point_id, + "source_tool": "disp_prt_2d", + "selection_rank": selection.get("selection_rank"), + "selection_key": selection.get("selection_key"), + "selection_label": selection.get("selection_label"), + "selection_description": selection.get("selection_description"), + "img_x": point.get("img_x"), + "img_y": point.get("img_y"), + "lon": geo_location.get("lon"), + "lat": geo_location.get("lat"), + "geo_row": geo_location.get("geo_row"), + "geo_col": geo_location.get("geo_col"), + "geo_match_distance_px": geo_location.get("geo_match_distance_px"), + "geo_los_rate_mm_per_year": geo_location.get("geo_los_rate_mm_per_year"), + "geo_coverage_valid": geo_location.get("geo_coverage_valid"), + "geo_source": geo_location.get("geo_source"), + "height_m": point.get("height_m"), + "deformation_rate_mm_per_year": point.get("deformation_rate_mm_per_year"), + "stdev_residual_phase_rad": point.get("stdev_residual_phase_rad"), + "displacement_count": len(displacements), + "displacements": displacements, + "source_files": { + "items": str(items_path), + "disp_point": str(disp_point_path), + }, + } + _write_json_if_changed(metadata_path, metadata) + + try: + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + from matplotlib.dates import DateFormatter + + if curve_needs_refresh or not png_path.is_file(): + x_values = [datetime.fromisoformat(str(item["date"])) for item in displacements] + x_labels = [str(item["date"]) for item in displacements] + y_values = [float(item["displacement_mm"]) for item in displacements] + fig_width = max(8.0, len(x_values) * 1.2) + fig, ax = plt.subplots(figsize=(fig_width, 4.5), dpi=150) + ax.plot(x_values, y_values, marker="o", markersize=5, linewidth=1.8, color="#1d4ed8") + ax.axhline(0, color="#94a3b8", linewidth=0.8) + ax.set_title( + f"Gamma SBAS {point_id} rate={point.get('deformation_rate_mm_per_year') or 0:.3f} mm/yr", + fontsize=10, + ) + ax.set_xlabel("Date") + ax.set_ylabel("Displacement (mm)") + ax.grid(True, alpha=0.25) + ax.set_xticks(x_values) + ax.set_xticklabels(x_labels, rotation=35, ha="right") + ax.xaxis.set_major_formatter(DateFormatter("%Y-%m-%d")) + fig.tight_layout() + fig.savefig(png_path) + plt.close(fig) + except Exception: + if not png_path.is_file(): + png_path.write_bytes(b"") + + monitor_outputs.append( + { + "point_id": point_id, + "metadata": metadata, + "files": { + "png": _file_summary(png_path), + "csv": _file_summary(csv_path), + "metadata": _file_summary(metadata_path), + }, + } + ) + + summary = { + "schema": "insar.gamma-sbas-expert-monitor-points-summary/v1", + "generated_at": _utcnow().isoformat(timespec="seconds") + "Z", + "mode": "expert_disp_prt_2d", + "source_tool": "disp_prt_2d", + "source_files": { + "items": _file_summary(items_path), + "disp_point": _file_summary(disp_point_path), + "selection": _file_summary(points_dir / "disp_point_selection.json"), + }, + "monitor_points": [ + { + "point_id": point.get("point_id"), + "selection_rank": selection_by_xy.get((int(point.get("img_x") or 0), int(point.get("img_y") or 0)), {}).get("selection_rank"), + "selection_key": selection_by_xy.get((int(point.get("img_x") or 0), int(point.get("img_y") or 0)), {}).get("selection_key"), + "selection_label": selection_by_xy.get((int(point.get("img_x") or 0), int(point.get("img_y") or 0)), {}).get("selection_label"), + "selection_description": selection_by_xy.get((int(point.get("img_x") or 0), int(point.get("img_y") or 0)), {}).get("selection_description"), + "img_x": point.get("img_x"), + "img_y": point.get("img_y"), + "lon": (geo_locations.get(str(point.get("point_id"))) or {}).get("lon"), + "lat": (geo_locations.get(str(point.get("point_id"))) or {}).get("lat"), + "geo_row": (geo_locations.get(str(point.get("point_id"))) or {}).get("geo_row"), + "geo_col": (geo_locations.get(str(point.get("point_id"))) or {}).get("geo_col"), + "geo_match_distance_px": (geo_locations.get(str(point.get("point_id"))) or {}).get("geo_match_distance_px"), + "geo_los_rate_mm_per_year": (geo_locations.get(str(point.get("point_id"))) or {}).get("geo_los_rate_mm_per_year"), + "geo_coverage_valid": (geo_locations.get(str(point.get("point_id"))) or {}).get("geo_coverage_valid"), + "geo_source": (geo_locations.get(str(point.get("point_id"))) or {}).get("geo_source"), + "height_m": point.get("height_m"), + "deformation_rate_mm_per_year": point.get("deformation_rate_mm_per_year"), + "stdev_residual_phase_rad": point.get("stdev_residual_phase_rad"), + "displacement_count": len(point.get("displacements") or []), + "displacements": point.get("displacements") or [], + } + for point in points + ], + "monitor_outputs": monitor_outputs, + "ready": bool(monitor_outputs) + and all( + (item.get("files") or {}).get("png", {}).get("exists") + and (item.get("files") or {}).get("csv", {}).get("exists") + and (item.get("files") or {}).get("metadata", {}).get("exists") + for item in monitor_outputs + ), + } + if points: + _write_json_if_changed(summary_path, summary) + return summary + + def _bbox_polygon( min_lon: Optional[float], min_lat: Optional[float], @@ -246,22 +1987,63 @@ class SbasInsarCatalogService: run_root.mkdir(parents=True, exist_ok=True) return _normalize_path(run_root) + def get_run_roots(self) -> list[str]: + roots = [self.get_run_root()] + try: + landsar_root = landsar_sbas_service.configured_run_root() + if landsar_root not in roots: + roots.append(landsar_root) + except Exception: + pass + return roots + def _iter_run_manifest_paths(self, run_root: Optional[str] = None) -> list[str]: - root = Path(run_root or self.get_run_root()) - if not root.is_dir(): - return [] - return [ - _normalize_path(path) - for path in sorted(root.glob("*/run_manifest.json")) - if self._is_publish_ready(path.parent, _safe_read_json(path)) - ] + roots = [run_root] if run_root else self.get_run_roots() + manifest_paths: list[str] = [] + for raw_root in roots: + root = Path(raw_root) + if not root.is_dir(): + continue + manifest_paths.extend( + _normalize_path(path) + for path in sorted(root.glob("*/run_manifest.json")) + if self._is_publish_ready(path.parent, _safe_read_json(path)) + ) + return sorted(dict.fromkeys(manifest_paths)) + + @staticmethod + def _is_landsar_manifest(manifest: dict[str, Any]) -> bool: + return str(manifest.get("processor_code") or "").strip().lower() == "landsar_sbas" + + @staticmethod + def _is_expert_gamma_manifest(manifest: dict[str, Any], run_dir: Optional[Path] = None) -> bool: + execution_mode = str(manifest.get("execution_mode") or "").strip().lower() + if execution_mode == "expert_manifest_script_workflow": + return True + if run_dir is not None and (run_dir / "publish" / "geotiff" / "geo_los_def_rate.tif").is_file(): + return True + return False + + def _asset_definitions_for_manifest(self, manifest: dict[str, Any], run_dir: Optional[Path] = None): + if self._is_landsar_manifest(manifest): + return _LANDSAR_CORE_ASSETS + if self._is_expert_gamma_manifest(manifest, run_dir): + return _EXPERT_GAMMA_CORE_ASSETS + return _CORE_ASSETS + + def _required_roles_for_manifest(self, manifest: dict[str, Any], run_dir: Optional[Path] = None) -> set[str]: + if self._is_expert_gamma_manifest(manifest, run_dir): + return {"primary_geotiff"} + return _LANDSAR_REQUIRED_ASSET_ROLES if self._is_landsar_manifest(manifest) else _REQUIRED_ASSET_ROLES def _is_publish_ready(self, run_dir: Path, manifest: dict[str, Any]) -> bool: status = str(manifest.get("status") or "").strip().upper() + asset_defs = self._asset_definitions_for_manifest(manifest, run_dir) + required_roles = self._required_roles_for_manifest(manifest, run_dir) required_outputs_ready = all( (run_dir / relative_path).is_file() - for role, _name, relative_path, is_required, _is_primary in _CORE_ASSETS - if role in _REQUIRED_ASSET_ROLES and is_required + for role, _name, relative_path, is_required, _is_primary in asset_defs + if role in required_roles and is_required ) return status in _READY_STATUSES or required_outputs_ready @@ -270,13 +2052,36 @@ class SbasInsarCatalogService: for raw_path in manifest_paths: manifest_path = Path(raw_path) run_dir = manifest_path.parent + manifest_payload = _safe_read_json(manifest_path) tracked_paths = [ manifest_path, - run_dir / "product_summary.json", - run_dir / "quality_summary.json", run_dir / "monitor_points_summary.json", ] - tracked_paths.extend(run_dir / relative_path for _role, _name, relative_path, _required, _primary in _CORE_ASSETS) + if not self._is_expert_gamma_manifest(manifest_payload, run_dir): + tracked_paths.extend( + [ + run_dir / "product_summary.json", + run_dir / "quality_summary.json", + ] + ) + else: + tracked_paths.extend( + [ + run_dir / "diff_dir" / "bprep_file.png", + run_dir / "diff_dir" / "mean.cc_mask.bmp", + run_dir / "sbas" / "final_unw_tab", + ] + ) + diff_dir = run_dir / "diff_dir" + if diff_dir.is_dir(): + tracked_paths.extend(sorted(diff_dir.glob("*.adf.unw.bmp"))) + tracked_paths.extend( + run_dir / relative_path + for _role, _name, relative_path, _required, _primary in self._asset_definitions_for_manifest( + manifest_payload, run_dir + ) + if not (run_dir / relative_path).is_dir() + ) for path in tracked_paths: if not path.exists(): continue @@ -338,6 +2143,7 @@ class SbasInsarCatalogService: file_size=absolute_path.stat().st_size if exists else None, srid=4326 if ( (relative_path.lower().endswith((".tif", ".tiff")) and "/geotiff/" in relative_path) + or (relative_path.lower().endswith((".tif", ".tiff")) and "/landsar/" in relative_path) or relative_path.lower().endswith(".geojson.gz") ) else None, ) @@ -371,21 +2177,180 @@ class SbasInsarCatalogService: ) return rows + def _unwrapped_phase_asset_rows(self, run_dir: Path) -> list[ResultAssetORM]: + unwrapped_dir = run_dir / "publish" / "geotiff" / "unwrapped" + if not unwrapped_dir.is_dir(): + return [] + rows: list[ResultAssetORM] = [] + for path in sorted(unwrapped_dir.iterdir()): + if not path.is_file(): + continue + lowered = path.name.lower() + if lowered in {"unwrapped_phase_summary.json", "unwrapped_phase_rmg_colorbar.png"}: + continue + if lowered.endswith(".rdc_rmg_preview.png"): + role = "unwrapped_phase_radar_preview" + name = f"Radar-coordinate rmg unwrapped phase preview {path.name}" + elif lowered.endswith(".rdc_rmg.bmp"): + role = "unwrapped_phase_radar_bmp" + name = f"Gamma radar-coordinate rmg unwrapped phase BMP {path.name}" + elif lowered.endswith((".tif", ".tiff")): + role = "unwrapped_phase_geotiff" + name = f"Geocoded unwrapped phase {path.name}" + elif lowered.endswith(".png"): + role = "unwrapped_phase_preview" + name = f"Unwrapped phase preview {path.name}" + else: + continue + relative_path = str(path.relative_to(run_dir)).replace("\\", "/") + rows.append( + self._asset_row( + run_dir, + role=role, + name=name, + relative_path=relative_path, + is_required=False, + is_primary=False, + ) + ) + return rows + + def _gamma_intermediate_qc_asset_rows(self, run_dir: Path) -> list[ResultAssetORM]: + rows: list[ResultAssetORM] = [] + + static_assets = ( + ( + "gamma_qc_baseline_plot", + "Gamma baseline network plot", + run_dir / "diff_dir" / "bprep_file.png", + ), + ( + "gamma_qc_mean_coherence", + "Gamma mean coherence mask", + run_dir / "diff_dir" / "mean.cc_mask.bmp", + ), + ) + for role, name, path in static_assets: + if not path.is_file(): + continue + rows.append( + self._asset_row( + run_dir, + role=role, + name=f"{name} {path.name}", + relative_path=str(path.relative_to(run_dir)).replace("\\", "/"), + is_required=False, + is_primary=False, + ) + ) + + diff_dir = run_dir / "diff_dir" + final_tab = run_dir / "sbas" / "final_unw_tab" + pair_ids: list[str] = [] + if final_tab.is_file(): + try: + for line in final_tab.read_text(encoding="utf-8", errors="ignore").splitlines(): + raw_path = line.strip().split()[0] if line.strip() else "" + if not raw_path: + continue + name = Path(_wsl_path_to_windows(raw_path)).name + pair_id = name.replace(".unw.atmsub_1", "").replace(".unw", "") + if pair_id and pair_id not in pair_ids: + pair_ids.append(pair_id) + except OSError: + pair_ids = [] + + unwrapped_paths: list[Path] = [] + for pair_id in pair_ids: + path = diff_dir / f"{pair_id}.adf.unw.bmp" + if path.is_file(): + unwrapped_paths.append(path) + if not unwrapped_paths and diff_dir.is_dir(): + unwrapped_paths = sorted(diff_dir.glob("*.adf.unw.bmp")) + + if len(unwrapped_paths) > 3: + last_index = len(unwrapped_paths) - 1 + indexes = sorted({round(index * last_index / 2) for index in range(3)}) + unwrapped_paths = [unwrapped_paths[index] for index in indexes] + + for path in unwrapped_paths: + rows.append( + self._asset_row( + run_dir, + role="gamma_qc_unwrapped_phase", + name=f"Gamma representative filtered unwrapped phase {path.name}", + relative_path=str(path.relative_to(run_dir)).replace("\\", "/"), + is_required=False, + is_primary=False, + ) + ) + return rows + def _build_product(self, manifest_path: str) -> ResultProductORM: manifest_file = Path(manifest_path) run_dir = manifest_file.parent manifest = _read_json(manifest_file) if not self._is_publish_ready(run_dir, manifest): raise ValueError(f"run is not publish-ready: {manifest.get('status') or 'UNKNOWN'}") + if self._is_landsar_manifest(manifest): + return self._build_landsar_product(run_dir, manifest_file, manifest) detail = sbas_insar_production_service.get_run_detail(run_dir.name) coverage = detail.get("geographic_coverage") or {} stack_manifest = _safe_read_json(run_dir / "stack_manifest.json") - product_summary = _safe_read_json(run_dir / "product_summary.json") - quality_summary = _safe_read_json(run_dir / "quality_summary.json") monitor_summary = _safe_read_json(run_dir / "monitor_points_summary.json") - point_vector_summary = _safe_read_json(run_dir / "publish" / "vectors" / "los_rate_points_summary.json") workflow_summary = _safe_read_json(run_dir / "workflow_summary.json") + is_expert_gamma = self._is_expert_gamma_manifest(manifest, run_dir) + product_summary = {} if is_expert_gamma else _safe_read_json(run_dir / "product_summary.json") + quality_summary = ( + _build_expert_gamma_quality_summary(run_dir) + if is_expert_gamma + else _safe_read_json(run_dir / "quality_summary.json") + ) + point_vector_summary = _safe_read_json(run_dir / "publish" / "vectors" / "los_rate_points_summary.json") + asset_definitions = self._asset_definitions_for_manifest(manifest, run_dir) + if is_expert_gamma: + _build_rgb_geotiff_preview( + run_dir / "publish" / "geotiff" / "geo_los_def_rate_rgb.tif", + run_dir / "publish" / "geotiff" / "geo_los_def_rate_rgb_preview.png", + ) + _build_gamma_hls_rate_preview( + run_dir / "publish" / "geotiff" / "geo_los_def_rate.tif", + run_dir / "publish" / "geotiff" / "geo_los_def_rate_pure_hls_preview.png", + coverage_source=run_dir / "publish" / "geotiff" / "geo_los_def_rate_rgb.tif", + min_native=-0.08, + max_native=0.08, + ) + _build_gamma_hls_colorbar( + run_dir / "publish" / "geotiff" / "geo_los_def_rate_hls_colorbar.png", + min_mm_year=-80.0, + max_mm_year=80.0, + ) + point_vector_context = { + "run_id": str(manifest.get("run_id") or run_dir.name).strip() or run_dir.name, + "reference_date": str( + manifest.get("reference_date") + or (stack_manifest.get("stack") or {}).get("reference_date") + or "" + ).strip() or None, + "stack_dates": _stack_dates_from_manifest(stack_manifest, manifest, stack_manifest.get("stack") or {}), + "los_sign_convention": "Gamma expert geo_los_def_rate output; sign and unit semantics follow the expert workflow.", + "admin_region": coverage.get("admin_region"), + } + point_vector_context["date_start"] = ( + point_vector_context["stack_dates"][0] if point_vector_context["stack_dates"] else None + ) + point_vector_context["date_end"] = ( + point_vector_context["stack_dates"][-1] if point_vector_context["stack_dates"] else None + ) + point_vector_summary = _build_expert_gamma_point_vector( + run_dir, + summary_context=point_vector_context, + ) + unwrapped_phase_summary = _build_expert_unwrapped_phase_derivatives(run_dir) + monitor_summary = _write_expert_monitor_point_derivatives(run_dir) or monitor_summary + else: + unwrapped_phase_summary = {} bbox = coverage.get("bbox") or {} min_lon = _safe_float(bbox.get("min_lon")) @@ -418,12 +2383,36 @@ class SbasInsarCatalogService: is_required=is_required, is_primary=is_primary, ) - for role, name, relative_path, is_required, is_primary in _CORE_ASSETS + for role, name, relative_path, is_required, is_primary in asset_definitions ] assets.extend(self._monitor_asset_rows(run_dir)) + if is_expert_gamma: + assets.extend(self._unwrapped_phase_asset_rows(run_dir)) + assets.extend(self._gamma_intermediate_qc_asset_rows(run_dir)) preview_asset = next((asset for asset in assets if asset.asset_role == "primary_geocoded_preview" and asset.exists_flag), None) primary_asset = next((asset for asset in assets if asset.asset_role == "primary_geotiff" and asset.exists_flag), None) missing_required = [asset for asset in assets if asset.is_required and not asset.exists_flag] + default_los_product = product_summary.get("default_los_product") + los_sign_convention = product_summary.get("los_sign_convention") + if is_expert_gamma: + default_los_product = default_los_product or "geo_los_def_rate" + los_sign_convention = ( + los_sign_convention + or "Gamma expert geo_los_def_rate output; sign and unit semantics follow the expert workflow." + ) + color_policy = { + "schema": "insar.gamma-sbas-color-policy/v1", + "source": "expert_gamma_command", + "browse_command": "rasdt_pwr los_def_rate ... -0.08 0.08 0 hls.cm ... 24", + "colormap": "Gamma hls.cm", + "data_range_native": [-0.08, 0.08], + "display_range_mm_per_year": [-80.0, 80.0], + "note": "The RGB browse GeoTIFF is generated by Gamma with hls.cm. Treat it as the expert browse standard unless the project defines a separate cartographic standard.", + } + else: + default_los_product = default_los_product or "los_rate_toward_m_per_year" + los_sign_convention = los_sign_convention or "toward radar positive; away from radar negative" + color_policy = product_summary.get("color_policy") produced_at = ( _parse_datetime(monitor_summary.get("generated_at")) @@ -454,17 +2443,16 @@ class SbasInsarCatalogService: "pair_count": _safe_int(manifest.get("pair_count")), "status": manifest.get("status"), "next_stage": manifest.get("next_stage"), - "los_sign_convention": ( - product_summary.get("los_sign_convention") - or "toward radar positive; away from radar negative" - ), - "default_los_product": product_summary.get("default_los_product") or "los_rate_toward_m_per_year", + "los_sign_convention": los_sign_convention, + "default_los_product": default_los_product, + "color_policy": color_policy, "center": center or None, "admin_region": admin_region, "geographic_coverage": coverage, "quality": quality_summary, "monitor_points": monitor_summary, "point_vector": point_vector_summary, + "unwrapped_phase": unwrapped_phase_summary, "workflow": { "status": ((manifest.get("workflow") or {}).get("status")), "summary": ((manifest.get("workflow") or {}).get("summary")) or workflow_summary, @@ -502,6 +2490,7 @@ class SbasInsarCatalogService: "sensor": stack.get("satellite") or manifest.get("platform"), "orbit_direction": stack.get("orbit_direction") or manifest.get("direction"), "product": "Gamma SBAS", + "workflow_mode": "expert_document" if is_expert_gamma else "legacy_gamma", "admin_region": (admin_region or {}).get("display_name") if isinstance(admin_region, dict) else None, }, min_lon=min_lon, @@ -548,9 +2537,202 @@ class SbasInsarCatalogService: ) return product + def _build_landsar_product(self, run_dir: Path, manifest_file: Path, manifest: dict[str, Any]) -> ResultProductORM: + try: + detail = landsar_sbas_service.get_run_detail(run_dir.name) + except Exception: + detail = {} + coverage = detail.get("geographic_coverage") or manifest.get("geographic_coverage") or {} + stack_manifest = _safe_read_json(run_dir / "stack_manifest.json") + product_summary = _safe_read_json(run_dir / "product_summary.json") + quality_summary = _safe_read_json(run_dir / "quality_summary.json") + workflow_summary = _safe_read_json(run_dir / "workflow_summary.json") + + bbox = coverage.get("bbox") or {} + min_lon = _safe_float(bbox.get("min_lon")) + min_lat = _safe_float(bbox.get("min_lat")) + max_lon = _safe_float(bbox.get("max_lon")) + max_lat = _safe_float(bbox.get("max_lat")) + poly = _bbox_polygon(min_lon, min_lat, max_lon, max_lat) + + run_id = str(manifest.get("run_id") or run_dir.name).strip() or run_dir.name + stack_id = str(manifest.get("stack_id") or run_id).strip() + stack_dates = _stack_dates_from_manifest(stack_manifest, manifest, {}) + if not stack_dates: + stack_dates = [str(item or "").strip() for item in manifest.get("dates") or [] if str(item or "").strip()] + display_name = str(manifest.get("run_label") or f"LandSAR SBAS {run_id}").strip() + product_id = str(manifest.get("product_id") or "").strip() or f"landsar_sbas_{run_id}" + if len(product_id) > 64: + product_id = f"landsar_sbas_{_stable_digest(product_id, run_dir, length=32)}" + + assets: list[ResultAssetORM] = [ + self._asset_row( + run_dir, + role=role, + name=name, + relative_path=relative_path, + is_required=is_required, + is_primary=is_primary, + ) + for role, name, relative_path, is_required, is_primary in _LANDSAR_CORE_ASSETS + if not (run_dir / relative_path).is_dir() + ] + native_logs_dir = run_dir / "native_logs" + if native_logs_dir.is_dir(): + for path in sorted(native_logs_dir.rglob("*")): + if not path.is_file(): + continue + relative_path = str(path.relative_to(run_dir)).replace("\\", "/") + assets.append( + self._asset_row( + run_dir, + role="native_log" if path.suffix.lower() == ".log" else "native_parameter", + name=path.name, + relative_path=relative_path, + is_required=False, + is_primary=False, + ) + ) + task_publish_root = run_dir / "publish" / "landsar" + if task_publish_root.is_dir(): + for path in sorted(task_publish_root.rglob("*")): + if not path.is_file(): + continue + relative_path = str(path.relative_to(run_dir)).replace("\\", "/") + if relative_path in {item.relative_path for item in assets}: + continue + role = "landsar_task_geotiff" if path.suffix.lower() in {".tif", ".tiff"} else "landsar_task_asset" + assets.append( + self._asset_row( + run_dir, + role=role, + name=path.name, + relative_path=relative_path, + is_required=False, + is_primary=False, + ) + ) + + preview_asset = next((asset for asset in assets if asset.asset_role == "primary_preview" and asset.exists_flag), None) + primary_asset = next((asset for asset in assets if asset.asset_role == "primary_geotiff" and asset.exists_flag), None) + missing_required = [asset for asset in assets if asset.is_required and not asset.exists_flag] + + produced_at = ( + _parse_datetime(workflow_summary.get("ended_at")) + or _parse_datetime(product_summary.get("generated_at")) + or _parse_datetime(manifest.get("ended_at")) + or _parse_datetime(manifest.get("created_at")) + ) + center = coverage.get("center") or {} + admin_region = coverage.get("admin_region") or lookup_admin_region_for_point(center.get("lon"), center.get("lat")) + scene_count = _safe_int(manifest.get("scene_count")) or len(stack_dates) + task_count = _safe_int(manifest.get("task_count")) + + summary_json = { + "schema": "insar.landsar-sbas-result-catalog-summary/v1", + "run_id": run_id, + "stack_id": stack_id or None, + "reference_date": stack_dates[0] if stack_dates else None, + "stack_dates": stack_dates, + "stack_size": len(stack_dates), + "date_start": manifest.get("date_start") or (stack_dates[0] if stack_dates else None), + "date_end": manifest.get("date_end") or (stack_dates[-1] if stack_dates else None), + "scene_count": scene_count, + "task_count": task_count, + "pair_count": _safe_int(manifest.get("pair_count")), + "status": manifest.get("status"), + "next_stage": manifest.get("next_stage"), + "los_sign_convention": product_summary.get("los_sign_convention") or "LandSAR LOS output; semantics pending algorithm confirmation.", + "default_los_product": product_summary.get("default_los_product") or "los_timeseries", + "center": center or None, + "admin_region": admin_region, + "geographic_coverage": coverage, + "quality": quality_summary, + "workflow": workflow_summary, + "source_run_dir": str(run_dir), + "output_semantics_note": product_summary.get("output_semantics_note"), + } + + product = ResultProductORM( + product_id=product_id, + catalog_name=SBAS_INSAR_CATALOG_NAME, + product_family="timeseries", + product_type="sbas_insar", + display_name=display_name, + task_name="LandSAR SBAS-InSAR", + task_alias=run_id, + stack_key=stack_id or run_id, + run_key=run_id, + profile_code="lt1_landsar_sbas", + engine_code="landsar", + engine_version=None, + package_schema=str(manifest.get("schema") or "").strip() or "insar.landsar-sbas-run/v1", + package_layout="landsar_sbas_console_run", + processor_code="landsar_sbas", + runtime_id="landsar_console", + status="READY" if not missing_required else "INCOMPLETE", + health_status="OK" if not missing_required else "WARN", + publish_dir=_normalize_path(run_dir / "publish"), + manifest_path=_normalize_path(manifest_file), + source_primary_path=primary_asset.absolute_path if primary_asset else None, + native_output_dir=_normalize_path(run_dir), + preview_path=preview_asset.absolute_path if preview_asset else None, + primary_asset_path=primary_asset.absolute_path if primary_asset else None, + summary_json=summary_json, + tags_json={ + "sensor": "LT1", + "product": "LandSAR SBAS", + "processor_code": "landsar_sbas", + "admin_region": (admin_region or {}).get("display_name") if isinstance(admin_region, dict) else None, + }, + min_lon=min_lon, + min_lat=min_lat, + max_lon=max_lon, + max_lat=max_lat, + geom=from_shape(poly, srid=4326) if poly is not None else None, + coverage_polygon=(coverage.get("geojson") or coverage.get("scene_footprints_geojson")), + produced_at=produced_at, + published_at=produced_at, + ) + for asset in assets: + product.assets.append(asset) + if asset.is_required and not asset.exists_flag: + product.issues.append( + ResultIssueORM( + asset=asset, + issue_code="MISSING_REQUIRED_ASSET", + severity="ERROR", + status="OPEN", + scope="file", + message=f"Required LandSAR SBAS asset is missing: {asset.relative_path}", + ) + ) + if not preview_asset: + product.issues.append( + ResultIssueORM( + issue_code="MISSING_PREVIEW", + severity="WARN", + status="OPEN", + scope="product", + message="LandSAR SBAS preview PNG is missing.", + ) + ) + if poly is None: + product.issues.append( + ResultIssueORM( + issue_code="MISSING_COVERAGE", + severity="WARN", + status="OPEN", + scope="product", + message="No valid EPSG:4326 geographic coverage bbox was found.", + ) + ) + return product + async def rebuild_catalog(self, db: AsyncSession, *, full_rebuild: bool = True) -> dict[str, Any]: run_root = self.get_run_root() - manifest_paths = await asyncio.to_thread(self._iter_run_manifest_paths, run_root) + run_roots = self.get_run_roots() + manifest_paths = await asyncio.to_thread(self._iter_run_manifest_paths) fingerprint = await asyncio.to_thread(self._tree_fingerprint, manifest_paths) state = await self._get_or_create_catalog_state(db, storage_root=run_root) state.status = "REBUILDING" @@ -596,6 +2778,7 @@ class SbasInsarCatalogService: select(func.count(ResultProductORM.id)).where(ResultProductORM.catalog_name == SBAS_INSAR_CATALOG_NAME) ) db_count = int(db_count_result.scalar_one() or 0) + fingerprint = await asyncio.to_thread(self._tree_fingerprint, manifest_paths) state = await self._get_or_create_catalog_state(db, storage_root=run_root) state.manifest_count = len(manifest_paths) state.manifest_fingerprint = fingerprint @@ -614,6 +2797,7 @@ class SbasInsarCatalogService: return { "catalog_name": SBAS_INSAR_CATALOG_NAME, "storage_root": run_root, + "storage_roots": run_roots, "run_count": len(manifest_paths), "manifest_count": len(manifest_paths), "manifest_fingerprint": fingerprint, @@ -690,6 +2874,7 @@ class SbasInsarCatalogService: "scene_count": summary.get("scene_count"), "pair_count": summary.get("pair_count"), "los_sign_convention": summary.get("los_sign_convention"), + "color_policy": summary.get("color_policy"), "center": summary.get("center") or ((summary.get("geographic_coverage") or {}).get("center")), "admin_region": summary.get("admin_region") or ((summary.get("geographic_coverage") or {}).get("admin_region")), "min_lon": product.min_lon, @@ -756,9 +2941,11 @@ class SbasInsarCatalogService: "pair_count": summary.get("pair_count"), "los_sign_convention": summary.get("los_sign_convention"), "default_los_product": summary.get("default_los_product"), + "color_policy": summary.get("color_policy"), "quality": summary.get("quality"), "monitor_points": summary.get("monitor_points"), "point_vector": summary.get("point_vector"), + "unwrapped_phase": summary.get("unwrapped_phase"), "workflow": summary.get("workflow"), "geographic_coverage": summary.get("geographic_coverage"), "center": summary.get("center") or ((summary.get("geographic_coverage") or {}).get("center")), @@ -803,6 +2990,27 @@ class SbasInsarCatalogService: ], } + async def query_point_timeseries( + self, + db: AsyncSession, + *, + product_db_id: int, + lon: float, + lat: float, + ) -> Optional[dict[str, Any]]: + result = await db.execute(select(ResultProductORM).where(ResultProductORM.id == product_db_id)) + product = result.scalar_one_or_none() + if product is None or product.catalog_name != SBAS_INSAR_CATALOG_NAME: + return None + manifest_path = str(product.manifest_path or "").strip() + run_dir = Path(manifest_path).parent if manifest_path else Path(str(product.native_output_dir or "")) + if not run_dir.is_dir(): + raise FileNotFoundError("SBAS run directory not found") + manifest = _safe_read_json(run_dir / "run_manifest.json") + if not self._is_expert_gamma_manifest(manifest, run_dir): + raise ValueError("point time-series query is only available for expert Gamma SBAS products") + return _query_expert_gamma_point_timeseries(run_dir, lon=lon, lat=lat) + async def get_asset(self, db: AsyncSession, *, product_db_id: int, asset_id: int) -> Optional[ResultAssetORM]: result = await db.execute( select(ResultAssetORM) @@ -817,7 +3025,8 @@ class SbasInsarCatalogService: async def get_catalog_status(self, db: AsyncSession) -> dict[str, Any]: run_root = self.get_run_root() - manifest_paths = await asyncio.to_thread(self._iter_run_manifest_paths, run_root) + run_roots = self.get_run_roots() + manifest_paths = await asyncio.to_thread(self._iter_run_manifest_paths) fingerprint = await asyncio.to_thread(self._tree_fingerprint, manifest_paths) state = await self._get_or_create_catalog_state(db, storage_root=run_root) db_count_result = await db.execute( @@ -844,6 +3053,7 @@ class SbasInsarCatalogService: "catalog_name": state.catalog_name, "product_family": state.product_family, "storage_root": state.storage_root, + "storage_roots": run_roots, "status": state.status, "needs_rebuild": state.needs_rebuild, "run_count": len(manifest_paths), @@ -865,7 +3075,8 @@ class SbasInsarCatalogService: raise RuntimeError("Database session factory is not initialized.") async with AsyncSessionLocal() as db: run_root = self.get_run_root() - manifest_paths = await asyncio.to_thread(self._iter_run_manifest_paths, run_root) + run_roots = self.get_run_roots() + manifest_paths = await asyncio.to_thread(self._iter_run_manifest_paths) fingerprint = await asyncio.to_thread(self._tree_fingerprint, manifest_paths) state = await self._get_or_create_catalog_state(db, storage_root=run_root) db_count_result = await db.execute( @@ -897,6 +3108,7 @@ class SbasInsarCatalogService: return { "storage_root": run_root, + "storage_roots": run_roots, "manifest_count": len(manifest_paths), "current_manifest_fingerprint": fingerprint, "indexed_manifest_fingerprint": state.manifest_fingerprint, diff --git a/backend/app/services/sbas_insar_production_service.py b/backend/app/services/sbas_insar_production_service.py index 3c442c6..06fc0ad 100644 --- a/backend/app/services/sbas_insar_production_service.py +++ b/backend/app/services/sbas_insar_production_service.py @@ -8,12 +8,14 @@ import re import shutil import struct import subprocess +import zipfile from datetime import datetime from pathlib import Path from typing import Any from xml.etree import ElementTree as ET from shapely.geometry import box as shapely_box +from sqlalchemy import String, cast, delete, or_, select from ..config import settings from .admin_region_lookup_service import ( @@ -24,6 +26,30 @@ from .admin_region_lookup_service import ( PRODUCT_DEFINITIONS = ( + { + "key": "expert_geo_los_def_rate_tif", + "label": "Expert Gamma geo_los_def_rate GeoTIFF", + "role": "primary_geotiff", + "relative_path": "publish/geotiff/geo_los_def_rate.tif", + }, + { + "key": "expert_geo_los_def_rate_rgb_tif", + "label": "Expert Gamma geo_los_def_rate RGB GeoTIFF", + "role": "primary_rgb_geotiff", + "relative_path": "publish/geotiff/geo_los_def_rate_rgb.tif", + }, + { + "key": "expert_geo_los_def_rate_rgb_preview_png", + "label": "Expert Gamma geo_los_def_rate RGB PNG preview", + "role": "primary_geocoded_preview", + "relative_path": "publish/geotiff/geo_los_def_rate_rgb_preview.png", + }, + { + "key": "expert_disp_point_txt", + "label": "Expert Gamma disp_prt_2d point time series", + "role": "monitor_points", + "relative_path": "publish/points/disp_point.txt", + }, { "key": "los_rate_toward_m_per_year_hls_geo_preview_png", "label": "Expert HLS LOS velocity geocoded RGB preview, toward radar positive", @@ -159,36 +185,87 @@ IPTA_MB_MODE_DESCRIPTIONS = { 1: "allow missing unwrapped phase values with network connectivity", 2: "allow missing unwrapped phase values without network connectivity requirement", } +GAMMA_SBAS_FALLBACK_MIN_COMMON_OVERLAP_RATIO = 0.30 + +GAMMA_SBAS_FORBIDDEN_DEFAULT_TOOLS = { + "LT1_precision_orbit.py", + "SLC_coreg.py", + "gc_map1", + "phase_sim_orb", + "SLC_diff_intf", + "adf", + "cc_wave", + "mcf", +} + +GAMMA_SBAS_MANUAL_QC_TOOLS = { + "disSLC", + "dismph_fft", +} + +GAMMA_SBAS_BLOCKING_INTERACTIVE_TOOLS = { + "disSLC", + "dismph", + "dismph_fft", + "dispwr", + "disras", + "xterm", + "display", + "eog", + "gwenview", + "xdg-open", +} + +GAMMA_SBAS_UNATTENDED_POLICY = ( + "Backend Gamma SBAS production is non-interactive. Expert manual display/QC " + "commands are documented but are not executed by default; reviewable browse " + "assets are produced by raster/export commands and the publish step." +) + +GAMMA_SBAS_REQUIRED_STEP_TOOLS = { + "02_import_lt1_slc": {"par_LT1_SLC", "ORB_filt_spline.py"}, + "03_reference_mli": {"multi_look", "ras_dB", "SLC_corners"}, + "04_dem_lookup": {"dem_import", "fill_gaps", "gc_map2", "pixel_area", "gc_map_fine", "geocode", "geocode_back"}, + "06_coregister_scenes": {"create_offset", "init_offset_orbit", "init_offset", "offset_pwr", "offset_fit", "SLC_interp"}, + "07_rmli_average": {"mk_mli_all", "ras_dB"}, + "08_diff_network": {"base_calc", "base_plot", "mk_diff_2d"}, + "09_filter_unwrap": {"mk_adf_2d", "ave_image", "rascc_mask", "mk_unw_2d"}, + "10_detrend_atm": {"create_diff_par", "quad_fit", "quad_sub", "atm_mod_2d", "fill_gaps", "atm_sim_2d", "sub_phase"}, + "11_sbas_inversion": {"mb", "real_to_cpx", "unw_model"}, + "12_outputs_points": {"replace_values", "mask_data", "dispmap", "ts_rate", "geocode_back", "data2geotiff", "disp_prt_2d"}, +} GAMMA_STAGE_PLAN = ( { "stage_id": "prepare_slc", "label": "Prepare LT1 SLCs", - "gamma_tools": ["par_LT1_SLC", "LT1_precision_orbit.py", "multi_look"], + "gamma_tools": ["par_LT1_SLC", "ORB_filt_spline.py", "SLC_corners"], + "manual_qc_tools": ["disSLC", "dismph_fft"], + "unattended_policy": GAMMA_SBAS_UNATTENDED_POLICY, "status": "PLANNED", }, { "stage_id": "baseline_audit", "label": "Gamma baseline audit and itab approval", - "gamma_tools": ["base_calc"], + "gamma_tools": ["multi_look", "base_calc", "base_plot"], "status": "PENDING_REQUIRED_AUDIT", }, { "stage_id": "coregistration", "label": "Stack co-registration", - "gamma_tools": ["SLC_coreg.py"], + "gamma_tools": ["create_offset", "init_offset_orbit", "init_offset", "offset_pwr", "offset_fit", "SLC_interp"], "status": "PLANNED_AFTER_BASELINE_AUDIT", }, { "stage_id": "rdc_dem", "label": "RDC DEM and lookup table", - "gamma_tools": ["gc_map1", "geocode", "gc_map_fine"], + "gamma_tools": ["dem_import", "fill_gaps", "gc_map2", "pixel_area", "create_diff_par", "offset_pwrm", "offset_fitm", "gc_map_fine", "geocode", "geocode_back"], "status": "PLANNED_AFTER_BASELINE_AUDIT", }, { "stage_id": "interferograms", "label": "Differential interferograms", - "gamma_tools": ["phase_sim_orb", "SLC_diff_intf", "adf", "mcf"], + "gamma_tools": ["base_calc", "base_plot", "mk_diff_2d", "mk_adf_2d", "ave_image", "rascc_mask", "mk_unw_2d"], "status": "PLANNED_AFTER_BASELINE_AUDIT", }, { @@ -200,7 +277,7 @@ GAMMA_STAGE_PLAN = ( { "stage_id": "ipta_timeseries", "label": "IPTA SBAS time-series inversion", - "gamma_tools": ["mb", "ts_rate"], + "gamma_tools": ["mb", "real_to_cpx", "unw_model"], "status": "PLANNED_AFTER_DETREND_ATM", }, { @@ -247,7 +324,9 @@ GAMMA_SBAS_WORKFLOW_STEPS = ( "legacy_stage": "baseline_audit", "script_name": "02_import_lt1_slc.sh", "status": "PENDING", - "expert_tools": ["par_LT1_SLC", "ORB_filt_spline.py", "SLC_corners", "disSLC", "dismph_fft"], + "expert_tools": ["par_LT1_SLC", "ORB_filt_spline.py", "SLC_corners"], + "manual_qc_tools": ["disSLC", "dismph_fft"], + "unattended_policy": GAMMA_SBAS_UNATTENDED_POLICY, }, { "id": "03_reference_mli", @@ -320,7 +399,7 @@ GAMMA_SBAS_WORKFLOW_STEPS = ( "legacy_stage": "ipta_timeseries", "script_name": "11_sbas_inversion.sh", "status": "PENDING", - "expert_tools": ["mb", "unw_to_cpx", "unw_model", "ts_rate"], + "expert_tools": ["mb", "real_to_cpx", "unw_model", "ts_rate"], }, { "id": "12_outputs_points", @@ -339,7 +418,7 @@ GAMMA_SBAS_EXPERT_DOCUMENT_STEPS = ( "title": "Directory and LT1 data preparation", "document_section": "1. Directory and data preparation", "workflow_steps": ["01_workspace_data"], - "implementation_status": "implemented_bridge", + "implementation_status": "implemented", "commands": [ "mkdir -p RAW SLC dem rslc_prep mli_dir diff_dir diff1_dir sbas", "ls RAW/<date>/*.tiff", @@ -368,7 +447,7 @@ GAMMA_SBAS_EXPERT_DOCUMENT_STEPS = ( "title": "Reference MLI and footprint checks", "document_section": "3. Reference multilook and range check", "workflow_steps": ["03_reference_mli"], - "implementation_status": "implemented_bridge", + "implementation_status": "implemented", "commands": [ "multi_look <ref>.slc <ref>.slc.par <ref>_<rlks>_<azlks>.mli <ref>_<rlks>_<azlks>.mli.par <rlks> <azlks>", "grep range_samples <ref>.mli.par", @@ -383,7 +462,7 @@ GAMMA_SBAS_EXPERT_DOCUMENT_STEPS = ( "title": "DEM import and lookup table", "document_section": "4. DEM import and geocoding lookup table", "workflow_steps": ["04_dem_lookup"], - "implementation_status": "implemented_bridge", + "implementation_status": "implemented", "commands": [ "dem_import <dem>.tif SRTM.dem SRTM.dem.par ...", "fill_gaps SRTM.dem <dem_width> SRTM_dem_fill", @@ -403,7 +482,7 @@ GAMMA_SBAS_EXPERT_DOCUMENT_STEPS = ( "title": "SLC coregistration preparation", "document_section": "5. SLC coregistration preparation", "workflow_steps": ["05_coreg_prep"], - "implementation_status": "implemented_bridge", + "implementation_status": "implemented", "commands": [ "cp SLC/dates rslc_prep/dates", "cp <ref>.slc <ref>.rslc", @@ -416,9 +495,9 @@ GAMMA_SBAS_EXPERT_DOCUMENT_STEPS = ( "title": "Coregister every SLC to reference", "document_section": "6. Coregister scenes to reference geometry", "workflow_steps": ["06_coregister_scenes"], - "implementation_status": "implemented_bridge", + "implementation_status": "implemented", "commands": [ - "create_offset <ref>.rslc.par <date>.slc.par <ref>_<date>.off 1", + "create_offset <ref>.rslc.par <date>.slc.par <ref>_<date>.off 1 <rlks> <azlks> 0", "init_offset_orbit <ref>.rslc.par <date>.slc.par <ref>_<date>.off", "init_offset <ref>.rslc <date>.slc <ref>.rslc.par <date>.slc.par <ref>_<date>.off <rlks> <azlks>", "offset_pwr <ref>.rslc <date>.slc <ref>.rslc.par <date>.slc.par <ref>_<date>.off ...", @@ -433,7 +512,7 @@ GAMMA_SBAS_EXPERT_DOCUMENT_STEPS = ( "title": "RMLI stack and average intensity", "document_section": "7. Generate RMLI and average intensity", "workflow_steps": ["07_rmli_average"], - "implementation_status": "implemented_bridge", + "implementation_status": "implemented", "commands": [ "mk_mli_all rslc_tab . <rlks> <azlks> 1 1.0 0.4 mli.ave", "grep range_samples mli.ave.par", @@ -447,7 +526,7 @@ GAMMA_SBAS_EXPERT_DOCUMENT_STEPS = ( "title": "Interferogram network and differential phase", "document_section": "8. Interferogram generation and differential interferometry", "workflow_steps": ["08_diff_network"], - "implementation_status": "implemented_bridge", + "implementation_status": "implemented", "commands": [ "base_calc rslc_tab <ref>.rslc.par bprep_file itab 1 1 <bmin> <bmax> <tmin> <tmax> -", "base_plot rslc_tab <ref>.rslc.par itab bprep_file 1", @@ -462,7 +541,7 @@ GAMMA_SBAS_EXPERT_DOCUMENT_STEPS = ( "title": "Adaptive filtering, coherence mask and unwrap", "document_section": "9. Adaptive filtering, coherence mask and phase unwrapping", "workflow_steps": ["09_filter_unwrap"], - "implementation_status": "implemented_bridge", + "implementation_status": "implemented", "commands": [ "mk_adf_2d rslc_tab itab mli.ave . 5 0.6 32 8 -u", "ls *.adf.diff", @@ -479,7 +558,7 @@ GAMMA_SBAS_EXPERT_DOCUMENT_STEPS = ( "title": "Detrend and atmospheric phase removal", "document_section": "10. Detrending and atmospheric phase removal", "workflow_steps": ["10_detrend_atm"], - "implementation_status": "implemented_bridge", + "implementation_status": "implemented", "commands": [ "create_diff_par <pair>.off <pair>.off <pair>.diff_par 0 0", "quad_fit <pair>.adf.unw <pair>.diff_par 5 5 - - 3 <pair>.unw_linear", @@ -498,10 +577,10 @@ GAMMA_SBAS_EXPERT_DOCUMENT_STEPS = ( "title": "SBAS inversion", "document_section": "11. SBAS inversion", "workflow_steps": ["11_sbas_inversion"], - "implementation_status": "implemented_bridge", + "implementation_status": "implemented", "commands": [ "mb unw_atmsub_tab RMLI_tab itab - itab_ts ras/diff1 1 diff1.sigma_ts 1 - <r_ref> <a_ref> 15 15 0.0 mli.ave.par", - "unw_to_cpx <pair>.unw.atmsub <pair>.unw.atmsub.cpx <width>", + "real_to_cpx - <pair>.unw.atmsub <pair>.unw.atmsub.cpx <width> 1", "unw_model <pair>.unw.atmsub.cpx <pair>.unw.atmsub_sim <pair>.unw.atmsub_1 <width> <r_ref> <a_ref>", "mb unw.atmsub_1_tab RMLI_tab itab - itab_ts ras/diff2 1 diff2.sigma_ts 0 - <r_ref> <a_ref> 15 15 0.0 mli.ave.par", "mb final_unw_tab RMLI_tab itab - itab_ts ras/diff 0 diff.sigma_ts 0 - <r_ref> <a_ref> 15 15 0.5 mli.ave.par", @@ -513,7 +592,7 @@ GAMMA_SBAS_EXPERT_DOCUMENT_STEPS = ( "title": "Output, geocode and point time-series", "document_section": "12. Output, geocoding and point time-series", "workflow_steps": ["12_outputs_points"], - "implementation_status": "implemented_bridge", + "implementation_status": "implemented", "commands": [ "replace_values diff.sigma_ts 0.5 0.0 diff.sigma_ts.masked <width> 1 2 0", "rasdt_pwr diff.sigma_ts.masked - <width> 1 0 1 1 0.0 1.5 1 cc.cm diff.sigma_ts.masked.bmp 1.0 0.35 8", @@ -525,7 +604,7 @@ GAMMA_SBAS_EXPERT_DOCUMENT_STEPS = ( "data2geotiff <ref>_seg.dem_par geo_los_def_rate 2 geo_los_def_rate.tif", "geocode_back los_def_rate.bmp <width> <ref>.lt_fine geo_los_def_rate.bmp <dem_width> <dem_lines> 0 2", "data2geotiff <ref>_seg.dem_par geo_los_def_rate.bmp 0 geo_los_def_rate_rgb.tif", - "disp_prt_2d disp_geo.TS_tab RMLI_tab itab_ts - 3 disp_point.txt - geo_los_def_rate geo_diff.sigma_ts items.txt disp_tab.txt 3 1 0", + "disp_prt_2d disp.TS_tab RMLI_tab itab_ts - 3 disp_point_sel.txt <ref>.hgt los_def_rate diff.sigma_ts.masked items.txt disp_point.txt 3 1 0", ], }, ) @@ -544,6 +623,54 @@ LT1_SCENE_RE = re.compile( re.IGNORECASE, ) +S1_SOURCE_RE = re.compile( + r"^(?P<satellite>S1[A-Z])_" + r"(?P<mode>[A-Z0-9]+)_" + r"(?P<product>[A-Z0-9]+)_+" + r"(?P<class>[0-9A-Z]{4})_" + r"(?P<start>\d{8}T\d{6}(?:\.\d+)?)_" + r"(?P<stop>\d{8}T\d{6}(?:\.\d+)?)_" + r"(?P<absolute_orbit>\d+)_" + r"(?P<datatake>[0-9A-F]+)_" + r"(?P<product_uid>[0-9A-F]+)" + r"(?:\.SAFE|\.zip)?$", + re.IGNORECASE, +) + +S1_EOF_RE = re.compile( + r"^(?P<satellite>S1[A-Z])_OPER_" + r"(?P<orbit_type>AUX_[A-Z0-9]+)_" + r"(?P<provider>[A-Z0-9]+)_" + r"(?P<generation>\d{8}T\d{6})_" + r"V(?P<valid_start>\d{8}T\d{6})_" + r"(?P<valid_stop>\d{8}T\d{6})\.EOF$", + re.IGNORECASE, +) + +S1_GAMMA_SBAS_PLANNING_STEPS = ( + { + "id": "01_s1_stack_assets", + "name": "Sentinel-1 ZIP/SAFE and EOF stack audit", + "status": "PENDING", + "optional": False, + "notes": ["Implemented as planning metadata; no Gamma commands are executed."], + }, + { + "id": "02_s1_tops_import", + "name": "Sentinel-1 TOPS import and burst selection", + "status": "PLANNED", + "optional": False, + "notes": ["Pending verified Gamma TOPS import script."], + }, + { + "id": "03_s1_sbas_workflow", + "name": "Sentinel-1 Gamma SBAS workflow", + "status": "PLANNED", + "optional": False, + "notes": ["Pending Sentinel-1 specific co-registration, interferogram and IPTA scripts."], + }, +) + class SbasInsarProductionService: _WORKFLOW_BASELINE_DONE_STATUSES = { @@ -670,6 +797,7 @@ class SbasInsarProductionService: "implementation_state": "expert_manifest_script_runner_primary", "trial_root": str(self.trial_root), "production_root": str(self.production_root), + "min_common_overlap_ratio": self._effective_min_common_overlap_ratio(None), "workflow_runner": { "enabled": bool(settings.GAMMA_SBAS_ENABLED), "runtime_id": settings.GAMMA_SBAS_RUNTIME_ID, @@ -681,7 +809,21 @@ class SbasInsarProductionService: "style": "expert_document_manifest_and_scripts", }, "workflow_node_count": len(GAMMA_SBAS_WORKFLOW_STEPS), - "supported_sensors": ["LT1"], + "supported_sensors": ["LT1", "S1"], + "sensor_profiles": [ + { + "sensor_family": "LT1", + "profile_code": "lt1_gamma_sbas", + "execution_enabled": True, + "description": "LT-1 Gamma SBAS workflow generated from the expert command document.", + }, + { + "sensor_family": "S1", + "profile_code": "s1_gamma_sbas", + "execution_enabled": False, + "description": "Sentinel-1 stack discovery and planning only; Gamma TOPS/SBAS execution is not enabled.", + }, + ], "supported_products": [item["key"] for item in PRODUCT_DEFINITIONS], "run_submission": { "enabled": True, @@ -707,7 +849,7 @@ class SbasInsarProductionService: "execution_enabled": True, "execution_mode": "queued_background_task", "job_type": "SBAS_COREGISTRATION", - "default_strategy": "common_reference_to_stack_reference_date", + "default_strategy": "expert_create_offset_init_offset_slc_interp", "requires_status": "ITAB_APPROVED", }, "rdc_dem": { @@ -715,7 +857,7 @@ class SbasInsarProductionService: "execution_enabled": True, "execution_mode": "queued_background_task", "job_type": "SBAS_RDC_DEM", - "default_strategy": "gamma_gc_map_fine_reference_geometry", + "default_strategy": "expert_dem_import_gc_map2_pixel_area_gc_map_fine", "requires_status": "COREGISTRATION_READY", }, "interferograms": { @@ -723,7 +865,7 @@ class SbasInsarProductionService: "execution_enabled": True, "execution_mode": "queued_background_task", "job_type": "SBAS_INTERFEROGRAMS", - "default_strategy": "approved_itab_common_reference_diff_unwrap", + "default_strategy": "expert_mk_diff_2d_mk_adf_2d_mk_unw_2d", "requires_status": "RDC_DEM_READY", }, "detrend_atm": { @@ -739,7 +881,7 @@ class SbasInsarProductionService: "execution_enabled": True, "execution_mode": "queued_background_task", "job_type": "SBAS_IPTA_TIMESERIES", - "default_strategy": "gamma_mb_ts_rate_common_reference", + "default_strategy": "expert_three_pass_mb_real_to_cpx_unw_model", "default_mb_mode": DEFAULT_IPTA_MB_MODE, "mb_mode_description": IPTA_MB_MODE_DESCRIPTIONS[DEFAULT_IPTA_MB_MODE], "requires_status": "DETREND_ATM_READY", @@ -776,6 +918,7 @@ class SbasInsarProductionService: def discover_stacks( self, *, + sensor_family: str = "LT1", source_roots: list[str] | None = None, orbit_roots: list[str] | None = None, min_scenes: int = 3, @@ -789,25 +932,28 @@ class SbasInsarProductionService: discovery_mode: str = "strict", aoi_bbox: dict[str, Any] | None = None, min_aoi_coverage_ratio: float = 0.01, - min_common_overlap_ratio: float = 0.0, + min_common_overlap_ratio: float | None = None, force_refresh: bool = False, ) -> dict[str, Any]: - source_paths = self._resolve_source_roots(source_roots) - orbit_paths = self._resolve_orbit_roots(orbit_roots) + sensor_family = self._normalize_sensor_family(sensor_family) + source_paths = self._resolve_source_roots(source_roots, sensor_family=sensor_family) + orbit_paths = self._resolve_orbit_roots(orbit_roots, sensor_family=sensor_family) root_warnings = self._build_root_resolution_warnings( source_roots=source_roots, orbit_roots=orbit_roots, source_paths=source_paths, orbit_paths=orbit_paths, + sensor_family=sensor_family, ) normalized_mode = self._normalize_discovery_mode(discovery_mode) min_aoi_coverage_ratio = max(0.0, min(1.0, float(min_aoi_coverage_ratio or 0.0))) - min_common_overlap_ratio = max(0.0, min(1.0, float(min_common_overlap_ratio or 0.0))) + min_common_overlap_ratio = self._effective_min_common_overlap_ratio(min_common_overlap_ratio) discovery_aoi = self._build_discovery_aoi(admin_region=admin_region, aoi_bbox=aoi_bbox) effective_mode = "aoi" if normalized_mode == "aoi" and discovery_aoi.get("geometry") is not None else "strict" cache_key = self._discovery_cache_key( source_paths=source_paths, orbit_paths=orbit_paths, + sensor_family=sensor_family, min_scenes=min_scenes, require_orbits=require_orbits, include_scenes=include_scenes, @@ -820,6 +966,7 @@ class SbasInsarProductionService: aoi_bbox=aoi_bbox, min_aoi_coverage_ratio=min_aoi_coverage_ratio, min_common_overlap_ratio=min_common_overlap_ratio, + strategy_version="gamma-overlap-substack-v4", ) if not force_refresh: cached = self._read_discovery_cache(cache_key) @@ -838,11 +985,20 @@ class SbasInsarProductionService: for root in source_paths: try: - for scene_dir in self._iter_lt1_scene_dirs(root): + scene_iter = ( + self._iter_s1_scene_sources(root) + if sensor_family == "S1" + else self._iter_lt1_scene_dirs(root) + ) + for scene_source in scene_iter: try: - scene = self._parse_lt1_scene(scene_dir, orbit_paths) + scene = ( + self._parse_s1_scene(scene_source, orbit_paths) + if sensor_family == "S1" + else self._parse_lt1_scene(scene_source, orbit_paths) + ) except Exception as exc: - errors.append({"scene_dir": str(scene_dir), "error": str(exc)}) + errors.append({"scene_source": str(scene_source), "error": str(exc)}) continue if platform_filter and scene.get("satellite") != platform_filter: continue @@ -860,44 +1016,59 @@ class SbasInsarProductionService: except Exception as exc: errors.append({"source_root": str(root), "error": str(exc)}) + if sensor_family == "S1": + scenes = self._dedupe_s1_scenes(scenes) + grouped_initial: dict[str, list[dict[str, Any]]] = {} for scene in scenes: - group_key = self._aoi_stack_group_key(scene) if effective_mode == "aoi" else self._stack_group_key(scene) + group_key = ( + self._aoi_stack_group_key(scene) + if effective_mode == "aoi" + else self._stack_group_key(scene) + ) grouped_initial.setdefault(group_key, []).append(scene) - if effective_mode == "aoi": - grouped: dict[str, list[dict[str, Any]]] = {} - for observation_key, group_scenes in grouped_initial.items(): - for cluster in self._cluster_aoi_scenes(group_scenes): - cluster_key = self._aoi_cluster_key(observation_key, cluster) - clustered_scenes = [ - { - **scene, - "aoi_cluster_key": cluster_key, - "aoi_cluster_source": "footprint_common_overlap", - } - for scene in cluster - ] - grouped[cluster_key] = clustered_scenes - else: - grouped = grouped_initial + cluster_source = ( + "aoi_footprint_common_overlap" + if effective_mode == "aoi" + else "footprint_common_overlap" + ) + candidate_scene_groups: list[dict[str, Any]] = [] + for observation_key, group_scenes in grouped_initial.items(): + candidate_scene_groups.extend( + self._build_discovery_scene_groups( + observation_key=observation_key, + group_scenes=group_scenes, + discovery_mode=effective_mode, + require_orbits=require_orbits, + min_scenes=min_scenes, + min_common_overlap_ratio=min_common_overlap_ratio, + cluster_source=cluster_source, + ) + ) candidates = [ self._build_stack_candidate( - group_scenes, + scene_group["scenes"], min_scenes=min_scenes, require_orbits=require_orbits, discovery_mode=effective_mode, aoi_summary=discovery_aoi.get("summary"), min_common_overlap_ratio=min_common_overlap_ratio, ) - for group_scenes in grouped.values() + for scene_group in candidate_scene_groups + if scene_group.get("scenes") ] + candidates = self._dedupe_stack_candidates(candidates) if admin_region and effective_mode != "aoi": candidates = [ candidate for candidate in candidates if admin_region_matches(candidate.get("admin_region"), admin_region) ] + self._annotate_stack_candidate_identity( + candidates, + existing_run_index=self._existing_run_identity_index(), + ) candidates.sort( key=lambda item: ( int(item.get("status") != "READY"), @@ -915,6 +1086,7 @@ class SbasInsarProductionService: snapshot = { "schema": "insar.sbas-stack-discovery/v1", "generated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "sensor_family": sensor_family, "source_roots": [str(path) for path in source_paths], "orbit_roots": [str(path) for path in orbit_paths], "min_scenes": min_scenes, @@ -943,6 +1115,7 @@ class SbasInsarProductionService: self, stack_id: str, *, + sensor_family: str = "LT1", source_roots: list[str] | None = None, orbit_roots: list[str] | None = None, min_scenes: int = 3, @@ -951,9 +1124,11 @@ class SbasInsarProductionService: admin_region: str | None = None, aoi_bbox: dict[str, Any] | None = None, min_aoi_coverage_ratio: float = 0.01, - min_common_overlap_ratio: float = 0.0, + min_common_overlap_ratio: float | None = None, ) -> dict[str, Any]: + sensor_family = self._normalize_sensor_family(sensor_family) discovery = self.discover_stacks( + sensor_family=sensor_family, source_roots=source_roots, orbit_roots=orbit_roots, min_scenes=min_scenes, @@ -978,41 +1153,74 @@ class SbasInsarProductionService: if (scene.get("has_orbit") or not require_orbits) ] usable_scenes.sort(key=lambda item: str(item.get("date") or "")) + duplicate_audit = self._duplicate_scene_date_audit(usable_scenes) pairs = self._build_adjacent_pairs(usable_scenes) blockers: list[str] = [] warnings: list[str] = [] + for blocker in candidate.get("blockers") or []: + text = str(blocker or "").strip() + if text and text not in blockers: + blockers.append(text) if len(usable_scenes) < min_scenes: blockers.append( f"Only {len(usable_scenes)} usable scenes; minimum required is {min_scenes}." ) if require_orbits and candidate.get("missing_orbit_count"): + orbit_label = "EOF" if sensor_family == "S1" else "TXT" warnings.append( - f"{candidate.get('missing_orbit_count')} scenes are excluded because precise orbit TXT is missing." + f"{candidate.get('missing_orbit_count')} scenes are excluded because precise orbit {orbit_label} is missing." ) if len(pairs) < max(0, len(usable_scenes) - 1): blockers.append("Adjacent pair network is not fully connected.") + if duplicate_audit.get("has_duplicate_dates"): + blockers.append( + "Duplicate acquisition dates remain in the Gamma date-keyed stack; " + "each executable SBAS stack must contain one scene per date." + ) for pair in pairs: if int(pair.get("delta_days") or 0) > 180: warnings.append( f"Long temporal gap: {pair.get('master_date')} -> {pair.get('slave_date')} " f"({pair.get('delta_days')} days)." ) + candidate_duplicate_audit = candidate.get("date_keyed_duplicate_audit") or {} + if candidate_duplicate_audit.get("excluded_scene_count"): + warnings.append( + "Same-date LT1 scenes were reduced to one representative scene per date " + "for the Gamma date-keyed expert workflow." + ) timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%S%fZ") + ready_status = ( + "READY_FOR_S1_GAMMA_SBAS_PLANNING" + if sensor_family == "S1" + else "READY_FOR_GAMMA_BASELINE_AUDIT" + ) manifest = { "schema": "insar.gamma-ipta-sbas-stack-manifest/v1", "generated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", "stack_id": stack_id, + "sensor_family": sensor_family, + "profile_code": "s1_gamma_sbas" if sensor_family == "S1" else "lt1_gamma_sbas", "processor_code": "gamma_ipta_sbas", "engine_code": "gamma", - "workflow": "Gamma DIFF + IPTA mb/ts_rate", - "status": "READY_FOR_GAMMA_BASELINE_AUDIT" if not blockers else "BLOCKED", + "workflow": "Sentinel-1 Gamma SBAS planning" if sensor_family == "S1" else "LT1 Gamma SBAS expert command workflow", + "status": ready_status if not blockers else "BLOCKED", "require_orbits": require_orbits, "min_scenes": min_scenes, "discovery_mode": candidate.get("discovery_mode") or discovery.get("discovery_mode") or "strict", "aoi": candidate.get("aoi") or discovery.get("aoi"), "common_overlap_ratio": candidate.get("common_overlap_ratio"), + "min_common_overlap_ratio": discovery.get("min_common_overlap_ratio"), + "scene_identity_hash": candidate.get("scene_identity_hash"), + "scene_name_count": candidate.get("scene_name_count"), + "scene_name_preview": candidate.get("scene_name_preview") or [], + "scene_names": candidate.get("scene_names") or [], + "date_sequence_hash": candidate.get("date_sequence_hash"), + "same_date_sequence_candidate_count": candidate.get("same_date_sequence_candidate_count"), + "same_date_sequence_distinct_scene_group_count": candidate.get("same_date_sequence_distinct_scene_group_count"), + "existing_same_scene_runs": candidate.get("existing_same_scene_runs") or [], "stack": { key: candidate.get(key) for key in [ @@ -1032,15 +1240,22 @@ class SbasInsarProductionService: "excluded_scenes": [ scene for scene in candidate.get("scenes", []) if scene not in usable_scenes - ], + ] + (candidate.get("date_keyed_excluded_scenes") or []), + "date_keyed_duplicate_audit": candidate_duplicate_audit or duplicate_audit, "pair_network": { "strategy": "adjacent_temporal_initial", "gamma_baseline_status": "PENDING", + "execution_enabled": sensor_family != "S1", "pairs": pairs, }, "blockers": blockers, "warnings": sorted(set(warnings)), - "next_stage": "convert selected LT1 scenes with par_LT1_SLC, then run Gamma base_calc before final itab approval", + "execution_enabled": sensor_family != "S1", + "next_stage": ( + "Sentinel-1 stack is ready for planning; Gamma TOPS/SBAS scripts are not enabled yet." + if sensor_family == "S1" + else "run the LT1 Gamma SBAS expert command workflow and review the generated command audit" + ), } manifest_path = self._write_runtime_json( Path("stack_manifests") / stack_id, @@ -1064,6 +1279,7 @@ class SbasInsarProductionService: self, stack_id: str, *, + sensor_family: str = "LT1", run_label: str | None = None, source_roots: list[str] | None = None, orbit_roots: list[str] | None = None, @@ -1075,11 +1291,13 @@ class SbasInsarProductionService: admin_region: str | None = None, aoi_bbox: dict[str, Any] | None = None, min_aoi_coverage_ratio: float = 0.01, - min_common_overlap_ratio: float = 0.0, + min_common_overlap_ratio: float | None = None, dry_run: bool = True, ) -> dict[str, Any]: + sensor_family = self._normalize_sensor_family(sensor_family) audit = self.audit_stack( stack_id, + sensor_family=sensor_family, source_roots=source_roots, orbit_roots=orbit_roots, min_scenes=min_scenes, @@ -1091,8 +1309,13 @@ class SbasInsarProductionService: min_common_overlap_ratio=min_common_overlap_ratio, ) manifest = audit["manifest"] - if manifest.get("status") != "READY_FOR_GAMMA_BASELINE_AUDIT": - raise ValueError("stack manifest is not ready for run planning") + ready_statuses = {"READY_FOR_GAMMA_BASELINE_AUDIT", "READY_FOR_S1_GAMMA_SBAS_PLANNING"} + if manifest.get("status") not in ready_statuses: + blockers = "; ".join(str(item) for item in (manifest.get("blockers") or []) if item) + raise ValueError( + "stack manifest is not ready for run planning" + + (f": {blockers}" if blockers else "") + ) timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") run_id = self._stable_id(f"{stack_id}|{timestamp}|{run_label or ''}") @@ -1102,7 +1325,11 @@ class SbasInsarProductionService: log_dir = run_dir / "logs" for path in (work_dir, publish_dir, log_dir): path.mkdir(parents=True, exist_ok=True) - expert_workspace = self._ensure_expert_workspace(run_dir) + expert_workspace = ( + self._ensure_s1_planning_workspace(run_dir) + if sensor_family == "S1" + else self._ensure_expert_workspace(run_dir) + ) monitor_config = self._build_monitor_point_config( monitor_points=monitor_points, @@ -1116,13 +1343,24 @@ class SbasInsarProductionService: "workflow_code": "sbas_insar", "processor_code": "gamma_ipta_sbas", "engine_code": "gamma", - "execution_mode": "expert_manifest_script_workflow", - "status": "WORKFLOW_READY", + "sensor_family": sensor_family, + "profile_code": "s1_gamma_sbas" if sensor_family == "S1" else "lt1_gamma_sbas", + "execution_mode": "s1_gamma_sbas_planning_only" if sensor_family == "S1" else "expert_manifest_script_workflow", + "execution_enabled": sensor_family != "S1", + "status": "S1_GAMMA_SBAS_PLANNED" if sensor_family == "S1" else "WORKFLOW_READY", "created_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", "stack_id": stack_id, "discovery_mode": manifest.get("discovery_mode"), "aoi": manifest.get("aoi"), "common_overlap_ratio": manifest.get("common_overlap_ratio"), + "min_common_overlap_ratio": manifest.get("min_common_overlap_ratio"), + "scene_identity_hash": manifest.get("scene_identity_hash"), + "scene_name_count": manifest.get("scene_name_count"), + "scene_name_preview": manifest.get("scene_name_preview") or [], + "scene_names": manifest.get("scene_names") or [], + "date_sequence_hash": manifest.get("date_sequence_hash"), + "same_date_sequence_candidate_count": manifest.get("same_date_sequence_candidate_count"), + "same_date_sequence_distinct_scene_group_count": manifest.get("same_date_sequence_distinct_scene_group_count"), "stack_manifest_path": audit["manifest_path"], "pair_network_path": audit["pair_network_path"], "workflow_manifest_path": str(run_dir / "manifest.json"), @@ -1134,18 +1372,31 @@ class SbasInsarProductionService: "stack": manifest.get("stack") or {}, "scene_count": len(manifest.get("scenes") or []), "pair_count": len(((manifest.get("pair_network") or {}).get("pairs")) or []), - "next_stage": "workflow", + "next_stage": "implement_s1_gamma_sbas_scripts" if sensor_family == "S1" else "workflow", "requires_user_action": [ - "Review Gamma base_calc baseline table before approving final itab.", - "Confirm monitoring-point source: manual points, imported layer, or automatic sampler.", - "Confirm geocoded preview products are published from EPSG:4326 GeoTIFFs.", + *( + [ + "Review Sentinel-1 stack grouping, EOF coverage, subswath/burst policy, and common overlap before enabling execution.", + "Implement and verify Sentinel-1 Gamma TOPS/SBAS scripts before submitting workflow jobs.", + ] + if sensor_family == "S1" + else [ + "Review Gamma base_calc baseline table before approving final itab.", + "Confirm monitoring-point source: manual points, imported layer, or automatic sampler.", + "Confirm geocoded preview products are published from EPSG:4326 GeoTIFFs.", + ] + ), ], "monitor_points": monitor_config, "planning_only": True, "legacy_dry_run_request": bool(dry_run), } command_manifest = self._build_command_manifest(run_manifest, manifest) - workflow_manifest = self._build_workflow_manifest(run_dir, run_manifest, manifest) + workflow_manifest = ( + self._build_s1_workflow_manifest(run_dir, run_manifest, manifest) + if sensor_family == "S1" + else self._build_workflow_manifest(run_dir, run_manifest, manifest) + ) run_manifest_path = self._write_json(run_dir / "run_manifest.json", run_manifest) command_manifest_path = self._write_json(run_dir / "gamma_command_manifest.json", command_manifest) @@ -1213,11 +1464,245 @@ class SbasInsarProductionService: "command_manifest": command_manifest, "workflow_manifest": workflow_manifest, "workflow_state": workflow_state, + "runtime_status": self._build_runtime_status( + run_dir, + manifest=manifest, + workflow_manifest=workflow_manifest or {}, + workflow_state=workflow_state or {}, + ), "monitor_points": monitor_points, "geographic_coverage": geographic_coverage, "artifacts": self._build_run_artifacts(run_dir), } + def _build_runtime_status( + self, + run_dir: Path, + *, + manifest: dict[str, Any], + workflow_manifest: dict[str, Any], + workflow_state: dict[str, Any], + ) -> dict[str, Any]: + steps = workflow_state.get("steps") or {} + manifest_steps = workflow_manifest.get("steps") or [] + current_step = None + for step in manifest_steps: + step_id = str(step.get("id") or "") + state = steps.get(step_id) or {} + status = str(state.get("status") or step.get("status") or "").strip().upper() + if status == "RUNNING": + current_step = { + "id": step_id, + "name": state.get("name") or step.get("name") or step_id, + "status": status, + "started_at": state.get("started_at"), + "log": state.get("log") or step.get("log"), + "script": state.get("script") or step.get("script"), + } + break + if current_step is None: + for step in manifest_steps: + step_id = str(step.get("id") or "") + state = steps.get(step_id) or {} + status = str(state.get("status") or step.get("status") or "").strip().upper() + if status in {"FAILED", "PENDING", "SCRIPT_READY"}: + current_step = { + "id": step_id, + "name": state.get("name") or step.get("name") or step_id, + "status": status, + "started_at": state.get("started_at"), + "ended_at": state.get("ended_at"), + "log": state.get("log") or step.get("log"), + "script": state.get("script") or step.get("script"), + } + break + + workflow_summary = ( + self._summarize_workflow_state(workflow_manifest, workflow_state) + if workflow_manifest and workflow_state + else {} + ) + recent_logs = self._recent_run_logs(run_dir) + latest_log = recent_logs[0] if recent_logs else None + run_status = str(manifest.get("status") or "UNKNOWN").strip().upper() + return { + "schema": "insar.sbas-runtime-status/v1", + "run_id": manifest.get("run_id") or run_dir.name, + "run_status": run_status, + "active": "RUNNING" in run_status or bool(current_step and current_step.get("status") == "RUNNING"), + "current_step": current_step, + "workflow_updated_at": workflow_state.get("updated_at"), + "workflow_summary": workflow_summary, + "latest_log_updated_at": latest_log.get("modified_at") if latest_log else None, + "recent_logs": recent_logs, + "wsl": { + "distro": settings.GAMMA_SBAS_WSL_DISTRO, + "runtime_id": settings.GAMMA_SBAS_RUNTIME_ID, + "run_root": self._windows_path_to_wsl_mount(str(run_dir)), + }, + "overlap_gate": { + "common_overlap_ratio": manifest.get("common_overlap_ratio"), + "min_common_overlap_ratio": manifest.get("min_common_overlap_ratio"), + "passed": ( + float(manifest.get("common_overlap_ratio") or 0.0) + >= float(manifest.get("min_common_overlap_ratio") or 0.0) + ), + }, + } + + def _recent_run_logs(self, run_dir: Path, *, limit: int = 8, tail_chars: int = 1600) -> list[dict[str, Any]]: + log_dir = run_dir / "logs" + if not log_dir.is_dir(): + return [] + files = [path for path in log_dir.glob("*") if path.is_file()] + files.sort(key=lambda path: path.stat().st_mtime if path.exists() else 0.0, reverse=True) + logs: list[dict[str, Any]] = [] + for path in files[: max(1, int(limit))]: + try: + stat = path.stat() + tail = self._tail_text(path.read_text(encoding="utf-8", errors="replace"), tail_chars) + modified_at = datetime.utcfromtimestamp(stat.st_mtime).isoformat(timespec="seconds") + "Z" + logs.append( + { + "name": path.name, + "relative_path": str(path.relative_to(run_dir)).replace("\\", "/"), + "size_bytes": stat.st_size, + "modified_at": modified_at, + "tail": tail, + } + ) + except Exception as exc: + logs.append( + { + "name": path.name, + "relative_path": str(path.relative_to(run_dir)).replace("\\", "/"), + "error": str(exc), + } + ) + return logs + + async def delete_run_record(self, run_id: str, *, db: Any) -> dict[str, Any]: + from ..models import ( + ResultAssetORM, + ResultIssueORM, + ResultProductORM, + SystemJobORM, + SystemTaskORM, + TaskLogORM, + ) + + clean_id = str(run_id or "").strip() + run_dir = self._resolve_run_dir(clean_id) + manifest = self._read_json(run_dir / "run_manifest.json") + run_ids = { + clean_id, + str(manifest.get("run_id") or "").strip(), + str(manifest.get("workflow_run_id") or "").strip(), + } + run_ids = {item for item in run_ids if item} + + like_conditions = [ + cast(SystemTaskORM.params, String).ilike(f"%{item}%") + for item in run_ids + ] + task_conditions = list(like_conditions) + for item in run_ids: + task_conditions.append(SystemTaskORM.task_name.ilike(f"%{item}%")) + + tasks = [] + if task_conditions: + task_result = await db.execute(select(SystemTaskORM).where(or_(*task_conditions))) + tasks = list(task_result.scalars().all()) + task_ids = sorted({str(task.task_id or "").strip() for task in tasks if str(task.task_id or "").strip()}) + + job_conditions = [ + cast(SystemJobORM.payload, String).ilike(f"%{item}%") + for item in run_ids + ] + for item in run_ids: + job_conditions.append(SystemJobORM.workflow_run_id == item) + if task_ids: + job_conditions.append(SystemJobORM.task_id.in_(task_ids)) + + jobs = [] + if job_conditions: + job_result = await db.execute(select(SystemJobORM).where(or_(*job_conditions))) + jobs = list(job_result.scalars().all()) + + active_task_statuses = {"PENDING", "RUNNING"} + active_job_statuses = {"READY", "PENDING", "RUNNING", "RETRY"} + active_tasks = [ + task.task_id + for task in tasks + if str(task.status or "").strip().upper() in active_task_statuses + ] + active_jobs = [ + job.job_id + for job in jobs + if str(job.status or "").strip().upper() in active_job_statuses + ] + if active_tasks or active_jobs: + raise ValueError( + "Cannot delete an SBAS run with active task/job: " + f"tasks={active_tasks or []}, jobs={active_jobs or []}" + ) + + product_conditions = [ + ResultProductORM.catalog_name == "sbas_insar", + or_( + *[ + or_( + ResultProductORM.run_key == item, + ResultProductORM.product_id.ilike(f"%{item}%"), + ResultProductORM.manifest_path.ilike(f"%{item}%"), + ResultProductORM.publish_dir.ilike(f"%{item}%"), + ) + for item in run_ids + ] + ), + ] + product_result = await db.execute(select(ResultProductORM).where(*product_conditions)) + products = list(product_result.scalars().all()) + product_ids = [product.id for product in products] + if product_ids: + await db.execute(delete(ResultIssueORM).where(ResultIssueORM.product_ref_id.in_(product_ids))) + await db.execute(delete(ResultAssetORM).where(ResultAssetORM.product_ref_id.in_(product_ids))) + await db.execute(delete(ResultProductORM).where(ResultProductORM.id.in_(product_ids))) + + job_ids = sorted({str(job.job_id or "").strip() for job in jobs if str(job.job_id or "").strip()}) + if job_ids: + await db.execute(delete(SystemJobORM).where(SystemJobORM.job_id.in_(job_ids))) + if task_ids: + await db.execute(delete(TaskLogORM).where(TaskLogORM.task_id.in_(task_ids))) + await db.execute(delete(SystemTaskORM).where(SystemTaskORM.task_id.in_(task_ids))) + + deleted_stack_files: list[str] = [] + for key in ("stack_manifest_path", "pair_network_path"): + path = self._resolve_production_delete_path(manifest.get(key)) + if path is not None and path.is_file(): + path.unlink() + deleted_stack_files.append(str(path)) + try: + parent = path.parent + stack_root = (self.production_root / "stack_manifests").resolve() + parent.relative_to(stack_root) + if parent.is_dir() and not any(parent.iterdir()): + parent.rmdir() + except Exception: + pass + + shutil.rmtree(run_dir) + await db.commit() + return { + "run_id": clean_id, + "deleted": True, + "run_dir_deleted": str(run_dir), + "stack_files_deleted": deleted_stack_files, + "tasks_deleted": len(task_ids), + "jobs_deleted": len(job_ids), + "products_deleted": len(product_ids), + } + def run_baseline_audit( self, run_id: str, @@ -1231,6 +1716,7 @@ class SbasInsarProductionService: run_dir = self._resolve_run_dir(run_id) manifest_path = run_dir / "run_manifest.json" manifest = self._read_json(manifest_path) + self._ensure_lt1_execution_enabled(manifest) stack_manifest = self._read_json(run_dir / "stack_manifest.json") if manifest.get("status") not in { "PLANNED_GAMMA_BASELINE_AUDIT", @@ -1435,6 +1921,7 @@ class SbasInsarProductionService: run_dir = self._resolve_run_dir(run_id) manifest_path = run_dir / "run_manifest.json" manifest = self._read_json(manifest_path) + self._ensure_lt1_execution_enabled(manifest) if manifest.get("status") not in { "ITAB_APPROVED", "COREGISTRATION_SCRIPT_READY", @@ -1505,6 +1992,7 @@ class SbasInsarProductionService: run_dir = self._resolve_run_dir(run_id) manifest_path = run_dir / "run_manifest.json" manifest = self._read_json(manifest_path) + self._ensure_lt1_execution_enabled(manifest) status = str(manifest.get("status") or "").strip() if status == "COREGISTRATION_READY": return self.get_run_detail(run_id) @@ -1609,6 +2097,7 @@ class SbasInsarProductionService: run_dir = self._resolve_run_dir(run_id) manifest_path = run_dir / "run_manifest.json" manifest = self._read_json(manifest_path) + self._ensure_lt1_execution_enabled(manifest) status = str(manifest.get("status") or "").strip() if status == "RDC_DEM_READY": return self.get_run_detail(run_id) @@ -1684,6 +2173,7 @@ class SbasInsarProductionService: run_dir = self._resolve_run_dir(run_id) manifest_path = run_dir / "run_manifest.json" manifest = self._read_json(manifest_path) + self._ensure_lt1_execution_enabled(manifest) status = str(manifest.get("status") or "").strip() if status == "RDC_DEM_READY": return self.get_run_detail(run_id) @@ -1793,6 +2283,7 @@ class SbasInsarProductionService: run_dir = self._resolve_run_dir(run_id) manifest_path = run_dir / "run_manifest.json" manifest = self._read_json(manifest_path) + self._ensure_lt1_execution_enabled(manifest) status = str(manifest.get("status") or "").strip() if status == "INTERFEROGRAMS_READY": return self.get_run_detail(run_id) @@ -1889,6 +2380,7 @@ class SbasInsarProductionService: run_dir = self._resolve_run_dir(run_id) manifest_path = run_dir / "run_manifest.json" manifest = self._read_json(manifest_path) + self._ensure_lt1_execution_enabled(manifest) status = str(manifest.get("status") or "").strip() if status == "INTERFEROGRAMS_READY": return self.get_run_detail(run_id) @@ -2000,6 +2492,10 @@ class SbasInsarProductionService: reference_window: int = 16, coherence_min: float = 0.15, ) -> dict[str, Any]: + run_dir = self._resolve_run_dir(run_id) + manifest_path = run_dir / "run_manifest.json" + manifest = self._read_json(manifest_path) + self._ensure_lt1_execution_enabled(manifest) if execute: return self.execute_detrend_atm( run_id, @@ -2008,9 +2504,6 @@ class SbasInsarProductionService: coherence_min=coherence_min, ) - run_dir = self._resolve_run_dir(run_id) - manifest_path = run_dir / "run_manifest.json" - manifest = self._read_json(manifest_path) status = str(manifest.get("status") or "").strip() if status == "DETREND_ATM_READY": return self.get_run_detail(run_id) @@ -2123,6 +2616,7 @@ class SbasInsarProductionService: run_dir = self._resolve_run_dir(run_id) manifest_path = run_dir / "run_manifest.json" manifest = self._read_json(manifest_path) + self._ensure_lt1_execution_enabled(manifest) status = str(manifest.get("status") or "").strip() if status == "DETREND_ATM_READY": return self.get_run_detail(run_id) @@ -2487,6 +2981,7 @@ class SbasInsarProductionService: run_dir = self._resolve_run_dir(run_id) manifest_path = run_dir / "run_manifest.json" manifest = self._read_json(manifest_path) + self._ensure_lt1_execution_enabled(manifest) recovered = self._recover_workflow_resume_status(dict(manifest)) if recovered.get("status") != manifest.get("status"): manifest = recovered @@ -2611,6 +3106,7 @@ class SbasInsarProductionService: run_dir = self._resolve_run_dir(run_id) manifest_path = run_dir / "run_manifest.json" manifest = self._read_json(manifest_path) + self._ensure_lt1_execution_enabled(manifest) recovered = self._recover_workflow_resume_status(dict(manifest)) if recovered.get("status") != manifest.get("status"): manifest = recovered @@ -2725,6 +3221,7 @@ class SbasInsarProductionService: run_dir = self._resolve_run_dir(run_id) manifest_path = run_dir / "run_manifest.json" manifest = self._read_json(manifest_path) + self._ensure_lt1_execution_enabled(manifest) recovered = self._recover_workflow_resume_status(dict(manifest)) if recovered.get("status") != manifest.get("status"): manifest = recovered @@ -2953,20 +3450,23 @@ class SbasInsarProductionService: "artifacts": artifacts, "stage_contract": [ "par_LT1_SLC", - "LT1_precision_orbit.py", + "ORB_filt_spline.py", "multi_look", "base_calc", - "SLC_coreg.py", - "gc_map1/geocode/gc_map_fine", - "phase_sim_orb", - "SLC_diff_intf", - "adf", - "mcf", + "create_offset/init_offset_orbit/init_offset/offset_pwr/offset_fit/SLC_interp", + "dem_import/fill_gaps/gc_map2/pixel_area/gc_map_fine", + "mk_diff_2d", + "mk_adf_2d", + "mk_unw_2d", + "quad_fit/quad_sub/atm_mod_2d/atm_sim_2d/sub_phase", "mb", + "real_to_cpx", + "unw_model", "ts_rate", "geocode_back", "data2geotiff", - "LOS sign conversion", + "dispmap", + "disp_prt_2d", "monitoring point time series", ], } @@ -3021,7 +3521,9 @@ class SbasInsarProductionService: run_dir = self._resolve_run_dir(run_id) manifest_path = run_dir / "run_manifest.json" run_manifest = self._read_json(manifest_path) + self._ensure_lt1_execution_enabled(run_manifest) stack_manifest = self._read_json(run_dir / "stack_manifest.json") + self._ensure_gamma_date_keyed_stack(stack_manifest) self._ensure_expert_workspace(run_dir) run_manifest = self._recover_workflow_resume_status(run_manifest) self._write_json(manifest_path, run_manifest) @@ -3036,8 +3538,6 @@ class SbasInsarProductionService: maximum=256, ), } - self._prepare_reusable_stage_scripts(run_id, run_dir, run_manifest, params) - run_manifest = self._read_json(manifest_path) resume_stage_status = str(run_manifest.get("status") or "").strip() run_manifest["workflow"] = { **(run_manifest.get("workflow") or {}), @@ -3265,6 +3765,7 @@ class SbasInsarProductionService: run_dir = self._resolve_run_dir(run_id) manifest_path = run_dir / "run_manifest.json" run_manifest = self._read_json(manifest_path) + self._ensure_lt1_execution_enabled(run_manifest) if not (run_dir / "manifest.json").is_file(): self.prepare_workflow(run_id, force=force) run_manifest = self._read_json(manifest_path) @@ -3286,7 +3787,7 @@ class SbasInsarProductionService: } self._write_json(manifest_path, run_manifest) - execution_results = self._execute_workflow_bridge( + execution_results = self._execute_expert_workflow_scripts( run_id, run_dir, workflow_manifest=workflow_manifest, @@ -3305,7 +3806,7 @@ class SbasInsarProductionService: "returncode": returncode, "runtime_id": settings.GAMMA_SBAS_RUNTIME_ID, "distro": settings.GAMMA_SBAS_WSL_DISTRO, - "mode": "managed_python_bridge_to_expert_scripts", + "mode": "expert_document_scripts", "results": execution_results, "summary": summary, } @@ -3315,7 +3816,8 @@ class SbasInsarProductionService: "execution": execution, "summary": summary, } - if returncode == 0 and summary.get("ready"): + audit_summary = self._read_optional_json(run_dir / "expert_command_audit.json") or {} + if returncode == 0 and summary.get("ready") and audit_summary.get("ready"): run_manifest["status"] = "WORKFLOW_COMPLETED" run_manifest["next_stage"] = "review_publish_products" elif returncode == 0: @@ -3380,7 +3882,7 @@ class SbasInsarProductionService: previous = state["steps"].get(step_id) or {} if previous.get("status") == "COMPLETED" and not force: result = {**previous, "status": "SKIPPED", "skipped_reason": "already completed"} - state["steps"][step_id] = result + state["steps"][step_id] = previous results.append(result) continue @@ -3422,6 +3924,143 @@ class SbasInsarProductionService: results.append(result) return results + def _execute_expert_workflow_scripts( + self, + run_id: str, + run_dir: Path, + *, + workflow_manifest: dict[str, Any], + from_step: str | None, + to_step: str | None, + only_steps: list[str], + force: bool, + timeout_seconds: int, + ) -> list[dict[str, Any]]: + selected = self._select_workflow_steps( + workflow_manifest.get("steps") or [], + from_step=from_step, + to_step=to_step, + only_steps=only_steps, + ) + state_path = run_dir / "state" / "step_status.json" + state = self._read_optional_json(state_path) or self._initial_workflow_state( + self._read_json(run_dir / "run_manifest.json"), + workflow_manifest, + ) + state.setdefault("steps", {}) + results: list[dict[str, Any]] = [] + for step in selected: + step_id = str(step.get("id") or "") + if not step.get("enabled"): + result = self._workflow_step_result(step, status="PLANNED", skipped_reason="step planned but not enabled") + state["steps"][step_id] = result + results.append(result) + continue + previous = state["steps"].get(step_id) or {} + if previous.get("status") == "COMPLETED" and not force: + result = {**previous, "status": "SKIPPED", "skipped_reason": "already completed"} + state["steps"][step_id] = previous + results.append(result) + continue + + started_at = datetime.utcnow().isoformat(timespec="seconds") + "Z" + state["steps"][step_id] = { + "id": step_id, + "name": step.get("name") or step_id, + "enabled": bool(step.get("enabled")), + "optional": bool(step.get("optional")), + "status": "RUNNING", + "started_at": started_at, + "script": step.get("script"), + "log": step.get("log"), + } + state["updated_at"] = started_at + self._write_json(state_path, state) + try: + detail = self._execute_expert_workflow_step_script(run_dir, step, timeout_seconds=timeout_seconds) + result = { + "id": step_id, + "name": step.get("name") or step_id, + "status": "COMPLETED", + "started_at": started_at, + "ended_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "returncode": 0, + "detail": detail, + } + except Exception as exc: + result = { + "id": step_id, + "name": step.get("name") or step_id, + "status": "FAILED", + "started_at": started_at, + "ended_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "returncode": 1, + "error": str(exc), + } + state["steps"][step_id] = result + state["updated_at"] = datetime.utcnow().isoformat(timespec="seconds") + "Z" + self._write_json(state_path, state) + results.append(result) + break + state["steps"][step_id] = result + state["updated_at"] = datetime.utcnow().isoformat(timespec="seconds") + "Z" + self._write_json(state_path, state) + results.append(result) + return results + + def _execute_expert_workflow_step_script( + self, + run_dir: Path, + step: dict[str, Any], + *, + timeout_seconds: int, + ) -> dict[str, Any]: + step_id = str(step.get("id") or "") + script_path = Path(self._path_to_windows(str(step.get("script") or "")) or "") + if not script_path.is_file(): + raise FileNotFoundError(f"expert workflow script not found for {step_id}: {script_path}") + audit = self._audit_expert_step_script(step_id, script_path) + if not audit.get("ready"): + raise ValueError(f"expert command audit failed for {step_id}: {audit}") + command = self._script_execution_command(str(self._windows_path_to_wsl_mount(str(script_path)))) + completed = subprocess.run( + command, + cwd=str(run_dir), + text=True, + capture_output=True, + timeout=timeout_seconds, + check=False, + ) + log_path = run_dir / "logs" / f"{step_id}.runner.log" + log_text = "\n".join( + [ + "$ " + " ".join(command), + "", + "STDOUT:", + completed.stdout or "", + "", + "STDERR:", + completed.stderr or "", + "", + ] + ) + log_path.write_text(log_text, encoding="utf-8", newline="\n") + if completed.returncode != 0: + raise RuntimeError( + f"expert workflow step {step_id} failed with rc={completed.returncode}: " + f"{self._tail_text(completed.stderr or completed.stdout)}" + ) + return { + "step_id": step_id, + "script": str(script_path), + "command": command, + "returncode": completed.returncode, + "stdout_tail": self._tail_text(completed.stdout), + "stderr_tail": self._tail_text(completed.stderr), + "runner_log": str(log_path), + "command_audit": audit, + } + def _execute_workflow_step_bridge(self, run_id: str, step_id: str, *, timeout_seconds: int) -> dict[str, Any]: self._restore_stage_status_for_workflow_step(run_id) if step_id in {"01_workspace_data"}: @@ -3695,18 +4334,33 @@ class SbasInsarProductionService: break return selected - def _resolve_source_roots(self, roots: list[str] | None) -> list[Path]: - raw_values = roots or self._split_config_paths(settings.GAMMA_SBAS_SOURCE_ROOTS) + def _resolve_source_roots(self, roots: list[str] | None, *, sensor_family: str = "LT1") -> list[Path]: + sensor_family = self._normalize_sensor_family(sensor_family) + raw_values = roots or self._default_source_roots(sensor_family) if not raw_values: - raw_values = [r"D:\LuTan1_Image_Pool"] + raw_values = [r"D:\Sentinel1_Image_Pool"] if sensor_family == "S1" else [r"D:\LuTan1_Image_Pool"] return self._dedupe_existing_dirs(raw_values) - def _resolve_orbit_roots(self, roots: list[str] | None) -> list[Path]: - raw_values = roots or self._split_config_paths(settings.GAMMA_SBAS_ORBIT_ROOTS) + def _resolve_orbit_roots(self, roots: list[str] | None, *, sensor_family: str = "LT1") -> list[Path]: + sensor_family = self._normalize_sensor_family(sensor_family) + raw_values = roots or self._default_orbit_roots(sensor_family) if not raw_values: - raw_values = [r"D:\orbit_pools\envi"] + raw_values = [r"D:\Sentinel1_Orbit_Pool"] if sensor_family == "S1" else [r"D:\orbit_pools\envi"] return self._dedupe_existing_dirs(raw_values) + def _default_source_roots(self, sensor_family: str) -> list[str]: + if sensor_family == "S1": + return self._split_config_paths( + settings.SENTINEL1_STORAGE_DIRS, + settings.SOURCE_PRODUCT_DIRS, + ) + return self._split_config_paths(settings.GAMMA_SBAS_SOURCE_ROOTS) + + def _default_orbit_roots(self, sensor_family: str) -> list[str]: + if sensor_family == "S1": + return self._split_config_paths(settings.ORBIT_SOURCE_DIRS) + return self._split_config_paths(settings.GAMMA_SBAS_ORBIT_ROOTS) + def _build_root_resolution_warnings( self, *, @@ -3714,10 +4368,16 @@ class SbasInsarProductionService: orbit_roots: list[str] | None, source_paths: list[Path], orbit_paths: list[Path], + sensor_family: str = "LT1", ) -> list[dict[str, Any]]: warnings: list[dict[str, Any]] = [] - source_requested = source_roots or self._split_config_paths(settings.GAMMA_SBAS_SOURCE_ROOTS) or [r"D:\LuTan1_Image_Pool"] - orbit_requested = orbit_roots or self._split_config_paths(settings.GAMMA_SBAS_ORBIT_ROOTS) or [r"D:\orbit_pools\envi"] + sensor_family = self._normalize_sensor_family(sensor_family) + source_requested = source_roots or self._default_source_roots(sensor_family) or ( + [r"D:\Sentinel1_Image_Pool"] if sensor_family == "S1" else [r"D:\LuTan1_Image_Pool"] + ) + orbit_requested = orbit_roots or self._default_orbit_roots(sensor_family) or ( + [r"D:\Sentinel1_Orbit_Pool"] if sensor_family == "S1" else [r"D:\orbit_pools\envi"] + ) source_missing = self._missing_root_values(source_requested) orbit_missing = self._missing_root_values(orbit_requested) @@ -3764,6 +4424,7 @@ class SbasInsarProductionService: *, source_paths: list[Path], orbit_paths: list[Path], + sensor_family: str, min_scenes: int, require_orbits: bool, include_scenes: bool, @@ -3776,10 +4437,12 @@ class SbasInsarProductionService: aoi_bbox: dict[str, Any] | None, min_aoi_coverage_ratio: float, min_common_overlap_ratio: float, + strategy_version: str = "gamma-overlap-substack-v3", ) -> str: payload = { "source_paths": [os.path.normcase(str(path.resolve())) for path in source_paths], "orbit_paths": [os.path.normcase(str(path.resolve())) for path in orbit_paths], + "sensor_family": str(sensor_family or "LT1").strip().upper(), "source_mtime_ns": [ int(path.stat().st_mtime_ns) if path.exists() else 0 for path in source_paths @@ -3800,7 +4463,8 @@ class SbasInsarProductionService: "aoi_bbox": SbasInsarProductionService._normalize_bbox(aoi_bbox), "min_aoi_coverage_ratio": float(min_aoi_coverage_ratio), "min_common_overlap_ratio": float(min_common_overlap_ratio), - "response_shape": "aoi_discovery_v1", + "response_shape": "footprint_cluster_discovery_v3", + "strategy_version": strategy_version, } return hashlib.sha1(json.dumps(payload, sort_keys=True).encode("utf-8")).hexdigest()[:16] @@ -3927,6 +4591,415 @@ class SbasInsarProductionService: except OSError: return False + def _iter_s1_scene_sources(self, root: Path): + if self._looks_like_s1_source(root): + yield root + return + + stack: list[tuple[Path, int]] = [(root, 0)] + seen: set[str] = set() + while stack: + current, depth = stack.pop() + key = os.path.normcase(str(current.resolve())) + if key in seen: + continue + seen.add(key) + try: + children = list(current.iterdir()) + except OSError: + continue + for child in children: + name = child.name + if name.startswith((".", "_")): + continue + if self._looks_like_s1_source(child): + yield child + continue + if child.is_dir() and depth < 2: + stack.append((child, depth + 1)) + + @staticmethod + def _looks_like_s1_source(path: Path) -> bool: + name = path.name + upper = name.upper() + if path.is_file() and upper.startswith("S1") and upper.endswith(".ZIP"): + return S1_SOURCE_RE.match(name) is not None + if path.is_dir() and upper.startswith("S1") and upper.endswith(".SAFE"): + return S1_SOURCE_RE.match(name) is not None and (path / "manifest.safe").is_file() + return False + + def _parse_s1_scene(self, source_path: Path, orbit_roots: list[Path]) -> dict[str, Any]: + source_name = source_path.name + filename_meta = self._parse_s1_scene_name(source_name) + if not filename_meta: + raise ValueError(f"Cannot parse Sentinel-1 source name: {source_name}") + + manifest_meta = self._parse_s1_manifest(source_path) + meta = {**filename_meta, **{key: value for key, value in manifest_meta.items() if value not in (None, "", [])}} + start_dt = meta.get("start_time_utc_dt") + stop_dt = meta.get("stop_time_utc_dt") or start_dt + date = str(meta.get("date") or "")[:8] + satellite = str(meta.get("satellite") or "").upper() + orbit_path = self._find_s1_orbit(orbit_roots, satellite, start_dt, stop_dt) + + bbox = meta.get("bbox") + center_lon, center_lat = self._centroid_from_bbox(bbox) + if meta.get("center_lon") is not None: + center_lon = self._as_float(meta.get("center_lon")) + if meta.get("center_lat") is not None: + center_lat = self._as_float(meta.get("center_lat")) + + polarizations = meta.get("polarization_channels") or [] + polarization = "+".join(polarizations) if polarizations else str(meta.get("polarization") or "").upper() or None + source_format = "S1_SAFE_DIR" if source_path.is_dir() else "S1_ZIP" + source_windows = self._path_to_windows(str(source_path)) + source_wsl = self._windows_path_to_wsl_mount(str(source_path)) + return { + "scene_name": source_path.name, + "logical_product_uid": meta.get("logical_product_uid"), + "scene_dir_windows": source_windows if source_path.is_dir() else None, + "scene_dir_wsl": source_wsl if source_path.is_dir() else None, + "source_windows": source_windows, + "source_wsl": source_wsl, + "source_format": source_format, + "archive_windows": source_windows if source_path.is_file() else None, + "archive_wsl": source_wsl if source_path.is_file() else None, + "orbit_windows": self._path_to_windows(str(orbit_path)) if orbit_path else None, + "orbit_wsl": self._windows_path_to_wsl_mount(str(orbit_path)) if orbit_path else None, + "has_orbit": bool(orbit_path), + "date": date, + "satellite_family": "S1", + "satellite": satellite, + "satellite_mode": str(meta.get("product_type") or "").upper() or None, + "receiving_station": None, + "absolute_orbit": str(meta.get("absolute_orbit") or "") or None, + "relative_orbit": str(meta.get("relative_orbit") or "") or None, + "orbit_direction": str(meta.get("orbit_direction") or "").upper() or None, + "imaging_mode": str(meta.get("imaging_mode") or "").upper() or None, + "look_direction": None, + "polarization": polarization, + "product_type": str(meta.get("product_type") or "").upper() or None, + "product_level": "L1", + "source_product_token": meta.get("source_product_token"), + "center_lon": center_lon, + "center_lat": center_lat, + "center_bucket": self._center_bucket(center_lon, center_lat), + "bbox": bbox, + "start_time_utc": self._datetime_to_iso(start_dt), + "stop_time_utc": self._datetime_to_iso(stop_dt), + "start_time_utc_dt": start_dt, + "stop_time_utc_dt": stop_dt, + "manifest_path": meta.get("manifest_path"), + "polarization_channels": polarizations, + "execution_note": "Sentinel-1 SBAS is discovery/planning only; Gamma TOPS execution is not enabled.", + } + + @staticmethod + def _parse_s1_scene_name(source_name: str) -> dict[str, Any]: + base = SbasInsarProductionService._strip_s1_suffix(source_name) + match = S1_SOURCE_RE.match(base) + if not match: + return {} + data = match.groupdict() + class_token = str(data.get("class") or "").upper() + start_dt = SbasInsarProductionService._parse_s1_datetime(data.get("start")) + stop_dt = SbasInsarProductionService._parse_s1_datetime(data.get("stop")) + return { + "logical_product_uid": base, + "satellite": str(data.get("satellite") or "").upper(), + "imaging_mode": str(data.get("mode") or "").upper(), + "product_type": str(data.get("product") or "").upper(), + "source_product_token": class_token, + "polarization": class_token[-2:] if len(class_token) >= 2 else class_token, + "absolute_orbit": str(data.get("absolute_orbit") or "").lstrip("0") or data.get("absolute_orbit"), + "date": str(data.get("start") or "")[:8], + "start_time_utc_dt": start_dt, + "stop_time_utc_dt": stop_dt, + "filename_datatake": str(data.get("datatake") or "").upper(), + "filename_product_uid": str(data.get("product_uid") or "").upper(), + } + + def _parse_s1_manifest(self, source_path: Path) -> dict[str, Any]: + if source_path.is_dir(): + manifest_path = source_path / "manifest.safe" + if not manifest_path.is_file(): + return {"manifest_parse_status": "MISSING"} + try: + data = manifest_path.read_bytes() + except OSError as exc: + return {"manifest_parse_status": "FAILED", "manifest_parse_error": str(exc)} + return { + "manifest_parse_status": "OK", + "manifest_path": str(manifest_path), + **self._parse_s1_manifest_bytes(data), + } + + try: + with zipfile.ZipFile(source_path) as archive: + manifest_name = next( + ( + name for name in archive.namelist() + if name.lower().endswith("/manifest.safe") or name.lower() == "manifest.safe" + ), + None, + ) + if not manifest_name: + return {"manifest_parse_status": "MISSING"} + return { + "manifest_parse_status": "OK", + "manifest_path": manifest_name, + **self._parse_s1_manifest_bytes(archive.read(manifest_name)), + } + except Exception as exc: + return {"manifest_parse_status": "FAILED", "manifest_parse_error": str(exc)} + + def _parse_s1_manifest_bytes(self, data: bytes) -> dict[str, Any]: + try: + root = ET.fromstring(data) + except Exception as exc: + return {"manifest_parse_status": "FAILED", "manifest_parse_error": str(exc)} + start_dt = self._parse_s1_datetime(self._first_text_by_local_name(root, {"startTime"})) + stop_dt = self._parse_s1_datetime(self._first_text_by_local_name(root, {"stopTime"})) + pols = [ + str(item).strip().upper() + for item in self._texts_by_local_name(root, "transmitterReceiverPolarisation") + if str(item).strip() + ] + polygon = self._s1_polygon_from_coordinates(self._first_text_by_local_name(root, {"coordinates"})) + bbox = self._bbox_from_points(polygon) + center_lon, center_lat = self._centroid_from_points(polygon) + return { + "start_time_utc_dt": start_dt, + "stop_time_utc_dt": stop_dt, + "product_type": self._clean_upper(self._first_text_by_local_name(root, {"productType"})), + "imaging_mode": self._clean_upper(self._first_text_by_local_name(root, {"mode"})), + "orbit_direction": self._clean_upper(self._first_text_by_local_name(root, {"pass"})), + "polarization_channels": pols, + "absolute_orbit": self._clean_text(self._first_text_by_local_name(root, {"orbitNumber"})), + "relative_orbit": self._clean_text(self._first_text_by_local_name(root, {"relativeOrbitNumber"})), + "bbox": bbox, + "center_lon": center_lon, + "center_lat": center_lat, + "coverage_polygon": polygon, + } + + @staticmethod + def _first_text_by_local_name(root: ET.Element, names: set[str]) -> str | None: + wanted = {name.lower() for name in names} + for element in root.iter(): + tag = str(element.tag).split("}")[-1].lower() + if tag not in wanted: + continue + text = (element.text or "").strip() + if text: + return text + return None + + @staticmethod + def _texts_by_local_name(root: ET.Element, name: str) -> list[str]: + wanted = str(name or "").lower() + values: list[str] = [] + for element in root.iter(): + tag = str(element.tag).split("}")[-1].lower() + if tag != wanted: + continue + text = (element.text or "").strip() + if text and text not in values: + values.append(text) + return values + + @staticmethod + def _s1_polygon_from_coordinates(text: str | None) -> list[tuple[float, float]] | None: + if not text: + return None + points: list[tuple[float, float]] = [] + for token in re.split(r"\s+", text.strip()): + parts = [part for part in re.split(r"[,;]", token) if part] + if len(parts) < 2: + continue + try: + first = float(parts[0]) + second = float(parts[1]) + except ValueError: + continue + if abs(first) > 90.0 and abs(second) <= 90.0: + lon, lat = first, second + else: + lon, lat = second, first + points.append((lon, lat)) + if len(points) < 3: + return None + if points[0] != points[-1]: + points.append(points[0]) + return points + + @staticmethod + def _bbox_from_points(points: list[tuple[float, float]] | None) -> dict[str, float] | None: + if not points: + return None + lons = [float(point[0]) for point in points] + lats = [float(point[1]) for point in points] + return { + "min_lon": min(lons), + "min_lat": min(lats), + "max_lon": max(lons), + "max_lat": max(lats), + } + + @staticmethod + def _centroid_from_points(points: list[tuple[float, float]] | None) -> tuple[float | None, float | None]: + if not points: + return None, None + unique = points[:-1] if len(points) > 1 and points[0] == points[-1] else points + if not unique: + return None, None + return ( + sum(float(point[0]) for point in unique) / len(unique), + sum(float(point[1]) for point in unique) / len(unique), + ) + + @staticmethod + def _centroid_from_bbox(bbox: dict[str, Any] | None) -> tuple[float | None, float | None]: + if not bbox: + return None, None + try: + return ( + (float(bbox["min_lon"]) + float(bbox["max_lon"])) / 2, + (float(bbox["min_lat"]) + float(bbox["max_lat"])) / 2, + ) + except (KeyError, TypeError, ValueError): + return None, None + + @staticmethod + def _clean_text(value: Any) -> str | None: + text = str(value or "").strip() + return text or None + + @staticmethod + def _clean_upper(value: Any) -> str | None: + text = str(value or "").strip().upper() + return text or None + + @staticmethod + def _strip_s1_suffix(name: str) -> str: + lower = str(name or "").lower() + if lower.endswith(".zip"): + return str(name)[:-4] + if lower.endswith(".safe"): + return str(name)[:-5] + return str(name or "") + + @staticmethod + def _parse_s1_datetime(value: Any) -> datetime | None: + text = str(value or "").strip() + if not text: + return None + if text.startswith("UTC="): + text = text[4:] + text = text.rstrip("Z") + for fmt in ( + "%Y-%m-%dT%H:%M:%S.%f", + "%Y-%m-%dT%H:%M:%S", + "%Y%m%dT%H%M%S.%f", + "%Y%m%dT%H%M%S", + ): + try: + return datetime.strptime(text, fmt) + except ValueError: + continue + return None + + @staticmethod + def _datetime_to_iso(value: datetime | None) -> str | None: + if not value: + return None + return value.isoformat(timespec="seconds") + "Z" + + def _find_s1_orbit( + self, + orbit_roots: list[Path], + satellite: str, + start_dt: datetime | None, + stop_dt: datetime | None, + ) -> Path | None: + if not satellite or not start_dt: + return None + stop_dt = stop_dt or start_dt + candidates: list[tuple[int, float, Path]] = [] + for root in orbit_roots: + try: + paths = root.rglob("S1*.EOF") + except OSError: + continue + for path in paths: + parsed = self._parse_s1_eof_name(path.name) + if not parsed: + continue + if parsed.get("satellite") != satellite: + continue + valid_start = parsed.get("valid_start") + valid_stop = parsed.get("valid_stop") + if not valid_start or not valid_stop: + continue + if valid_start <= start_dt and valid_stop >= stop_dt: + quality_rank = 0 if "POEORB" in str(parsed.get("orbit_type") or "") else 1 + coverage_margin = (start_dt - valid_start).total_seconds() + (valid_stop - stop_dt).total_seconds() + candidates.append((quality_rank, -coverage_margin, path)) + if not candidates: + return None + candidates.sort(key=lambda item: (item[0], item[1], str(item[2]))) + return candidates[0][2] + + @staticmethod + def _parse_s1_eof_name(name: str) -> dict[str, Any] | None: + match = S1_EOF_RE.match(str(name or "")) + if not match: + return None + data = match.groupdict() + return { + "satellite": str(data.get("satellite") or "").upper(), + "orbit_type": str(data.get("orbit_type") or "").upper(), + "valid_start": SbasInsarProductionService._parse_s1_datetime(data.get("valid_start")), + "valid_stop": SbasInsarProductionService._parse_s1_datetime(data.get("valid_stop")), + "generation": SbasInsarProductionService._parse_s1_datetime(data.get("generation")), + } + + @staticmethod + def _dedupe_s1_scenes(scenes: list[dict[str, Any]]) -> list[dict[str, Any]]: + by_uid: dict[str, dict[str, Any]] = {} + for scene in scenes: + uid = str(scene.get("logical_product_uid") or scene.get("scene_name") or "").strip() + if not uid: + continue + current = by_uid.get(uid) + if current is None: + by_uid[uid] = scene + continue + current_score = 2 if current.get("source_format") == "S1_SAFE_DIR" else 1 + scene_score = 2 if scene.get("source_format") == "S1_SAFE_DIR" else 1 + if scene_score > current_score: + by_uid[uid] = scene + return list(by_uid.values()) + + @staticmethod + def _normalize_sensor_family(value: Any) -> str: + text = str(value or "LT1").strip().upper().replace("-", "") + if text in {"S1", "S1A", "S1B", "S1C", "SENTINEL1", "SENTINEL1A", "SENTINEL1B", "SENTINEL1C"}: + return "S1" + return "LT1" + + @staticmethod + def _ensure_lt1_execution_enabled(manifest: dict[str, Any]) -> None: + raw_profile = str(manifest.get("profile_code") or "").strip().lower() + sensor_family = SbasInsarProductionService._normalize_sensor_family( + manifest.get("sensor_family") or ((manifest.get("stack") or {}).get("satellite")) + ) + if raw_profile.startswith("s1_") or sensor_family == "S1" or manifest.get("execution_enabled") is False: + raise ValueError( + "Sentinel-1 Gamma SBAS is currently discovery/planning only; " + "Gamma TOPS/SBAS execution scripts are not enabled." + ) + def _parse_lt1_scene(self, scene_dir: Path, orbit_roots: list[Path]) -> dict[str, Any]: scene_name = scene_dir.name filename_meta = self._parse_lt1_scene_name(scene_name) @@ -4171,6 +5244,311 @@ class SbasInsarProductionService: f"Details: {detail}" ) + def _resolve_expert_dem_import_source(self, stack_manifest: dict[str, Any]) -> dict[str, Any]: + errors: list[str] = [] + explicit_candidates = [ + ("GAMMA_SBAS_DEM_PATH", settings.GAMMA_SBAS_DEM_PATH), + ("IDL_DINSAR_DEM_BASE_FILE", settings.IDL_DINSAR_DEM_BASE_FILE), + ("ISCE2_DEM_PATH", settings.ISCE2_DEM_PATH), + ("PYINT_PREPARED_DEM_PATH", settings.PYINT_PREPARED_DEM_PATH), + ("TIMESERIES_DEM_PATH", settings.TIMESERIES_DEM_PATH), + ] + stack_bbox = self._stack_bbox_union(stack_manifest) + if not stack_bbox: + raise ValueError("Expert Gamma SBAS DEM selection requires auditable scene bbox coverage.") + for label, raw_path in explicit_candidates: + for candidate in self._expert_dem_import_candidate_paths(raw_path): + if not candidate.is_file(): + continue + coverage = self._infer_dem_import_source_coverage(candidate) + if stack_bbox and coverage.get("min_lon") is None: + errors.append(f"{label} coverage is not auditable for full stack bbox: {candidate}") + continue + covers_stack_bbox = self._bbox_contains(coverage, stack_bbox, margin_degrees=0.05) if stack_bbox and coverage else None + if stack_bbox and coverage and not covers_stack_bbox: + errors.append(f"{label} does not cover full stack bbox: {candidate}") + continue + return { + "source_label": label, + "source_type": "dem_import_source", + "windows_path": str(candidate), + "wsl_path": self._windows_path_to_wsl_mount(str(candidate)), + "coverage": coverage, + "covers_stack_bbox": covers_stack_bbox, + "stack_bbox": stack_bbox, + "selection_note": "Selected DEM source for expert dem_import; runtime Gamma DEM cache is not accepted.", + } + if str(raw_path or "").strip(): + errors.append(f"{label} has no readable DEM import source: {raw_path}") + detail = "; ".join(errors) if errors else "no configured DEM source" + raise FileNotFoundError( + "No usable DEM source was found for expert Gamma SBAS dem_import. " + "Configure GAMMA_SBAS_DEM_PATH, IDL_DINSAR_DEM_BASE_FILE, ISCE2_DEM_PATH, PYINT_PREPARED_DEM_PATH, " + "or TIMESERIES_DEM_PATH to a readable source raster covering the full stack bbox. " + f"Details: {detail}" + ) + + def _materialize_expert_dem_import_source(self, run_dir: Path, dem_source: dict[str, Any]) -> dict[str, Any]: + stack_bbox = self._normalize_bbox(dem_source.get("stack_bbox")) + coverage = self._normalize_bbox(dem_source.get("coverage")) + raw_path = str(dem_source.get("windows_path") or dem_source.get("wsl_path") or "").strip() + source_path = Path(self._path_to_windows(raw_path) or raw_path) + if not stack_bbox or not coverage or not source_path.is_file(): + return dem_source + raster_suffix = source_path.suffix.lower() + direct_geotiff_source = raster_suffix in {".tif", ".tiff"} + if raster_suffix not in {"", ".tif", ".tiff", ".img", ".wgs84", ".vrt"}: + return dem_source + + source_area = self._bbox_area(coverage) + stack_area = self._bbox_area(stack_bbox) + if direct_geotiff_source and source_area <= max(stack_area * 8.0, 2.0): + return dem_source + + margin = 0.25 + clip_bbox = { + "min_lon": max(float(coverage["min_lon"]), float(stack_bbox["min_lon"]) - margin), + "min_lat": max(float(coverage["min_lat"]), float(stack_bbox["min_lat"]) - margin), + "max_lon": min(float(coverage["max_lon"]), float(stack_bbox["max_lon"]) + margin), + "max_lat": min(float(coverage["max_lat"]), float(stack_bbox["max_lat"]) + margin), + } + if not self._bbox_contains(clip_bbox, stack_bbox): + raise ValueError(f"DEM crop bbox does not cover stack bbox: crop={clip_bbox}, stack={stack_bbox}") + + clip_path = run_dir / "dem" / "expert_dem_import_clip.tif" + clip_meta_path = run_dir / "state" / "dem_import_clip.json" + if clip_path.is_file(): + clip_coverage = self._dem_coverage_from_raster(clip_path) + if clip_coverage.get("driver") == "GTiff" and self._bbox_contains(clip_coverage, stack_bbox, margin_degrees=0.02): + clipped = { + **dem_source, + "source_type": "dem_import_source_clip", + "original_windows_path": str(source_path), + "original_wsl_path": self._windows_path_to_wsl_mount(str(source_path)), + "windows_path": str(clip_path), + "wsl_path": self._windows_path_to_wsl_mount(str(clip_path)), + "coverage": clip_coverage, + "clip_bbox": clip_bbox, + "selection_note": "Selected run-local DEM clip for expert dem_import to avoid full-raster Gamma memory allocation.", + } + self._write_json(clip_meta_path, clipped) + return clipped + + self._crop_raster_dem_to_bbox(source_path, clip_path, clip_bbox) + clip_coverage = self._dem_coverage_from_raster(clip_path) + if not self._bbox_contains(clip_coverage, stack_bbox, margin_degrees=0.02): + raise ValueError( + "Run-local DEM clip does not cover stack bbox after raster crop: " + f"clip_coverage={clip_coverage}, stack={stack_bbox}" + ) + clipped = { + **dem_source, + "source_type": "dem_import_source_clip", + "original_windows_path": str(source_path), + "original_wsl_path": self._windows_path_to_wsl_mount(str(source_path)), + "windows_path": str(clip_path), + "wsl_path": self._windows_path_to_wsl_mount(str(clip_path)), + "coverage": clip_coverage, + "clip_bbox": clip_bbox, + "source_area_sq_deg": source_area, + "stack_area_sq_deg": stack_area, + "selection_note": "Selected run-local DEM clip for expert dem_import to avoid full-raster Gamma memory allocation.", + } + self._write_json(clip_meta_path, clipped) + return clipped + + @staticmethod + def _crop_raster_dem_to_bbox(source_path: Path, clip_path: Path, bbox_lonlat: dict[str, float]) -> None: + try: + import rasterio # type: ignore + from rasterio.warp import transform_bounds # type: ignore + from rasterio.windows import Window, from_bounds # type: ignore + except Exception as exc: + raise RuntimeError("rasterio is required to crop large Gamma SBAS DEM import sources") from exc + + clip_path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = clip_path.with_suffix(f"{clip_path.suffix}.tmp") + if tmp_path.exists(): + tmp_path.unlink() + with rasterio.open(source_path) as src: + if not src.crs: + raise ValueError(f"DEM raster has no CRS and cannot be cropped by stack bbox: {source_path}") + left = float(bbox_lonlat["min_lon"]) + bottom = float(bbox_lonlat["min_lat"]) + right = float(bbox_lonlat["max_lon"]) + top = float(bbox_lonlat["max_lat"]) + if not getattr(src.crs, "is_geographic", False): + left, bottom, right, top = transform_bounds( + "EPSG:4326", + src.crs, + left, + bottom, + right, + top, + densify_pts=21, + ) + raw_window = from_bounds(left, bottom, right, top, transform=src.transform) + col_off = max(0, int(math.floor(raw_window.col_off))) + row_off = max(0, int(math.floor(raw_window.row_off))) + col_end = min(src.width, int(math.ceil(raw_window.col_off + raw_window.width))) + row_end = min(src.height, int(math.ceil(raw_window.row_off + raw_window.height))) + width = col_end - col_off + height = row_end - row_off + if width <= 0 or height <= 0: + raise ValueError(f"DEM crop window is empty for bbox {bbox_lonlat}: {source_path}") + window = Window(col_off, row_off, width, height) + profile = src.profile.copy() + profile.update( + driver="GTiff", + width=width, + height=height, + transform=src.window_transform(window), + BIGTIFF="IF_SAFER", + ) + with rasterio.open(tmp_path, "w", **profile) as dst: + chunk_lines = 2048 + for local_row in range(0, height, chunk_lines): + rows = min(chunk_lines, height - local_row) + read_window = Window(col_off, row_off + local_row, width, rows) + write_window = Window(0, local_row, width, rows) + dst.write(src.read(window=read_window), window=write_window) + tmp_path.replace(clip_path) + + def _expert_dem_import_candidate_paths(self, raw_path: str | None) -> list[Path]: + text = str(raw_path or "").strip() + if not text: + return [] + win_text = self._path_to_windows(text) or text + base = Path(win_text) + suffixes = {".tif", ".tiff", ".dem", ".hgt", ".img", ".wgs84"} + candidates: list[Path] = [] + if base.is_dir(): + for pattern in ("*.tif", "*.tiff", "*.dem", "*.hgt", "*.img", "*.wgs84"): + candidates.extend(sorted(base.glob(pattern))) + else: + candidates.append(base) + if base.suffix.lower() not in suffixes: + candidates.extend(Path(f"{win_text}{suffix}") for suffix in (".tif", ".tiff", ".dem", ".wgs84")) + deduped: list[Path] = [] + seen: set[str] = set() + for candidate in candidates: + key = str(candidate) + if key in seen: + continue + seen.add(key) + deduped.append(candidate) + return deduped + + def _infer_dem_import_source_coverage(self, path: Path) -> dict[str, Any]: + par_candidates = [ + Path(f"{path}.par"), + path.with_suffix(f"{path.suffix}.par") if path.suffix else Path(f"{path}.par"), + path.with_suffix(".dem.par"), + path.with_suffix(".par"), + ] + for par_path in par_candidates: + params = self._parse_gamma_params(par_path) + coverage = self._dem_coverage_from_params(params) + if coverage.get("min_lon") is not None: + coverage["coverage_source"] = str(par_path) + return coverage + raster_coverage = self._dem_coverage_from_raster(path) + if raster_coverage.get("min_lon") is not None: + return raster_coverage + return {"coverage_source": "unavailable"} + + def _dem_coverage_from_raster(self, path: Path) -> dict[str, Any]: + try: + import rasterio # type: ignore + from rasterio.warp import transform_bounds # type: ignore + + with rasterio.open(path) as dataset: + bounds = dataset.bounds + crs = dataset.crs + if crs: + if crs.to_epsg() == 4326 or getattr(crs, "is_geographic", False): + min_lon, min_lat, max_lon, max_lat = bounds.left, bounds.bottom, bounds.right, bounds.top + else: + min_lon, min_lat, max_lon, max_lat = transform_bounds( + crs, + "EPSG:4326", + bounds.left, + bounds.bottom, + bounds.right, + bounds.top, + densify_pts=21, + ) + else: + min_lon, min_lat, max_lon, max_lat = bounds.left, bounds.bottom, bounds.right, bounds.top + return { + "coverage_source": f"rasterio:{path}", + "min_lon": float(min_lon), + "min_lat": float(min_lat), + "max_lon": float(max_lon), + "max_lat": float(max_lat), + "width": int(dataset.width), + "nlines": int(dataset.height), + "driver": str(getattr(dataset, "driver", "") or ""), + "crs": str(crs) if crs else None, + "area_sq_deg": max(0.0, (float(max_lon) - float(min_lon)) * (float(max_lat) - float(min_lat))), + } + except Exception: + pass + + try: + from osgeo import gdal, osr # type: ignore + + dataset = gdal.Open(str(path)) + if dataset is None: + return {"coverage_source": "unavailable"} + transform = dataset.GetGeoTransform(can_return_null=True) + if not transform: + return {"coverage_source": "unavailable"} + width = int(dataset.RasterXSize) + height = int(dataset.RasterYSize) + corners = [ + (0, 0), + (width, 0), + (width, height), + (0, height), + ] + points = [ + ( + transform[0] + col * transform[1] + row * transform[2], + transform[3] + col * transform[4] + row * transform[5], + ) + for col, row in corners + ] + projection = dataset.GetProjection() + if projection: + src = osr.SpatialReference() + src.ImportFromWkt(projection) + dst = osr.SpatialReference() + dst.ImportFromEPSG(4326) + transformer = osr.CoordinateTransformation(src, dst) + transformed = [] + for x, y in points: + lon, lat, *_ = transformer.TransformPoint(float(x), float(y)) + transformed.append((lon, lat)) + points = transformed + lons = [float(item[0]) for item in points] + lats = [float(item[1]) for item in points] + min_lon, max_lon = min(lons), max(lons) + min_lat, max_lat = min(lats), max(lats) + return { + "coverage_source": f"gdal:{path}", + "min_lon": min_lon, + "min_lat": min_lat, + "max_lon": max_lon, + "max_lat": max_lat, + "width": width, + "nlines": height, + "crs": projection or None, + "area_sq_deg": max(0.0, (max_lon - min_lon) * (max_lat - min_lat)), + } + except Exception: + return {"coverage_source": "unavailable"} + def _gamma_dem_candidate_paths(self, raw_path: str | None) -> list[Path]: text = str(raw_path or "").strip() if not text: @@ -4997,6 +6375,395 @@ class SbasInsarProductionService: return 0.0 return width * height if width > 0 and height > 0 else 0.0 + def _build_discovery_scene_groups( + self, + *, + observation_key: str, + group_scenes: list[dict[str, Any]], + discovery_mode: str, + require_orbits: bool, + min_scenes: int, + min_common_overlap_ratio: float, + cluster_source: str, + ) -> list[dict[str, Any]]: + mode = self._normalize_discovery_mode(discovery_mode) + clusters = self._cluster_aoi_scenes(group_scenes) + scene_groups: list[dict[str, Any]] = [] + seen_scene_keys: set[tuple[str, ...]] = set() + + for cluster_index, cluster in enumerate(clusters): + primary_key = None + if mode == "aoi" or len(clusters) > 1: + primary_key = self._aoi_cluster_key(observation_key, cluster) + if len(clusters) > 1: + primary_key = f"{primary_key}|cluster_{cluster_index + 1}" + primary_scenes = self._prepare_candidate_group_scenes( + cluster, + cluster_key=primary_key, + cluster_source=cluster_source, + variant="primary_cluster", + ) + primary_scenes = self._select_date_keyed_stack_scenes(primary_scenes) + self._append_discovery_scene_group( + scene_groups, + seen_scene_keys, + primary_scenes, + variant="primary_cluster", + ) + + for subgroup_index, subgroup in enumerate( + self._extract_common_overlap_subgroups( + cluster, + require_orbits=require_orbits, + min_scenes=min_scenes, + min_common_overlap_ratio=min_common_overlap_ratio, + ) + ): + subgroup_key = self._substack_group_key(observation_key, subgroup) + subgroup_scenes = self._prepare_candidate_group_scenes( + subgroup, + cluster_key=subgroup_key, + cluster_source=cluster_source, + variant=f"common_overlap_substack_{subgroup_index + 1}", + ) + self._append_discovery_scene_group( + scene_groups, + seen_scene_keys, + subgroup_scenes, + variant="common_overlap_substack", + ) + + return scene_groups + + def _append_discovery_scene_group( + self, + scene_groups: list[dict[str, Any]], + seen_scene_keys: set[tuple[str, ...]], + scenes: list[dict[str, Any]], + *, + variant: str, + ) -> None: + key = self._scene_identity_key(scenes) + if not key or key in seen_scene_keys: + return + seen_scene_keys.add(key) + scene_groups.append({"variant": variant, "scenes": scenes}) + + def _prepare_candidate_group_scenes( + self, + scenes: list[dict[str, Any]], + *, + cluster_key: str | None, + cluster_source: str, + variant: str, + ) -> list[dict[str, Any]]: + prepared = [] + for scene in scenes: + item = { + **scene, + "aoi_cluster_source": cluster_source, + "discovery_group_variant": variant, + } + if cluster_key: + item["aoi_cluster_key"] = cluster_key + else: + item.pop("aoi_cluster_key", None) + prepared.append(item) + return prepared + + @staticmethod + def _scene_identity_key(scenes: list[dict[str, Any]]) -> tuple[str, ...]: + values = [ + str(scene.get("scene_name") or scene.get("scene_dir_windows") or scene.get("date") or "").strip() + for scene in scenes + ] + return tuple(sorted(value for value in values if value)) + + @staticmethod + def _hash_identity_values(values: list[str] | tuple[str, ...]) -> str | None: + filtered = [str(value or "").strip() for value in values if str(value or "").strip()] + if not filtered: + return None + return hashlib.sha1("|".join(filtered).encode("utf-8", errors="ignore")).hexdigest() + + def _scene_identity_summary(self, scenes: list[dict[str, Any]]) -> dict[str, Any]: + scene_names = self._scene_identity_key(scenes) + dates = tuple(sorted( + str(scene.get("date") or "").strip() + for scene in scenes + if str(scene.get("date") or "").strip() + )) + return { + "scene_identity_key": scene_names, + "scene_identity_hash": self._hash_identity_values(scene_names), + "scene_name_count": len(scene_names), + "scene_name_preview": list(scene_names[:3]), + "scene_names": list(scene_names), + "date_sequence_key": dates, + "date_sequence_hash": self._hash_identity_values(dates), + } + + def _annotate_stack_candidate_identity( + self, + candidates: list[dict[str, Any]], + *, + existing_run_index: dict[str, list[dict[str, Any]]] | None = None, + ) -> None: + by_date_sequence: dict[str, list[dict[str, Any]]] = {} + for candidate in candidates: + identity = self._scene_identity_summary(candidate.get("scenes") or []) + candidate["scene_identity_hash"] = identity["scene_identity_hash"] + candidate["scene_name_count"] = identity["scene_name_count"] + candidate["scene_name_preview"] = identity["scene_name_preview"] + candidate["scene_names"] = identity["scene_names"] + candidate["date_sequence_hash"] = identity["date_sequence_hash"] + key = identity.get("date_sequence_hash") + if key: + by_date_sequence.setdefault(str(key), []).append(candidate) + + for group in by_date_sequence.values(): + group.sort(key=self._stack_candidate_rank, reverse=True) + scene_hashes = { + str(item.get("scene_identity_hash") or "") + for item in group + if item.get("scene_identity_hash") + } + for index, candidate in enumerate(group, start=1): + siblings = [ + { + "stack_id": item.get("stack_id"), + "scene_identity_hash": item.get("scene_identity_hash"), + "center_bucket": item.get("center_bucket"), + "center": item.get("center"), + "admin_region": item.get("admin_region"), + "common_overlap_ratio": item.get("common_overlap_ratio"), + } + for item in group + if item is not candidate + ] + candidate["same_date_sequence_candidate_count"] = len(group) + candidate["same_date_sequence_rank"] = index + candidate["same_date_sequence_distinct_scene_group_count"] = len(scene_hashes) + candidate["same_date_sequence_siblings"] = siblings[:12] + candidate["same_date_sequence_has_different_scene_groups"] = len(scene_hashes) > 1 + + run_index = existing_run_index or {} + for candidate in candidates: + scene_hash = str(candidate.get("scene_identity_hash") or "") + candidate["existing_same_scene_runs"] = run_index.get(scene_hash, []) if scene_hash else [] + + def _existing_run_identity_index(self) -> dict[str, list[dict[str, Any]]]: + run_root = self.production_root / "runs" + index: dict[str, list[dict[str, Any]]] = {} + if not run_root.exists(): + return index + for manifest_path in sorted(run_root.glob("*/run_manifest.json")): + try: + manifest = self._read_json(manifest_path) + stack_manifest = self._load_stack_manifest_for_run(manifest_path.parent, manifest) + identity = self._scene_identity_summary(stack_manifest.get("scenes") or []) + scene_hash = identity.get("scene_identity_hash") + if not scene_hash: + continue + stack = stack_manifest.get("stack") or manifest.get("stack") or {} + coverage = self._build_stack_geographic_coverage(stack_manifest) + index.setdefault(str(scene_hash), []).append( + { + "run_id": manifest.get("run_id") or manifest_path.parent.name, + "run_label": manifest.get("run_label"), + "status": manifest.get("status") or "UNKNOWN", + "created_at": manifest.get("created_at"), + "stack_id": manifest.get("stack_id") or stack_manifest.get("stack_id"), + "scene_count": manifest.get("scene_count") or identity.get("scene_name_count"), + "pair_count": manifest.get("pair_count"), + "center_bucket": stack.get("center_bucket"), + "date_start": coverage.get("date_start"), + "date_end": coverage.get("date_end"), + } + ) + except Exception: + continue + for runs in index.values(): + runs.sort(key=lambda item: str(item.get("created_at") or ""), reverse=True) + return index + + def _load_stack_manifest_for_run(self, run_dir: Path, run_manifest: dict[str, Any]) -> dict[str, Any]: + stack_manifest = self._read_optional_json(run_dir / "stack_manifest.json") + if stack_manifest: + return stack_manifest + stack_manifest_path = Path(str(run_manifest.get("stack_manifest_path") or "")) + if stack_manifest_path.is_file(): + return self._read_optional_json(stack_manifest_path) or {} + return {} + + def _substack_group_key(self, observation_key: str, scenes: list[dict[str, Any]]) -> str: + digest_source = "|".join(self._scene_identity_key(scenes)) + digest = hashlib.sha1(digest_source.encode("utf-8", errors="ignore")).hexdigest()[:12] + dates = [ + str(scene.get("date") or "").strip() + for scene in scenes + if str(scene.get("date") or "").strip() + ] + date_start = min(dates) if dates else "unknown" + date_end = max(dates) if dates else "unknown" + return f"{observation_key}|substack_{date_start}_{date_end}_{digest}" + + def _extract_common_overlap_subgroups( + self, + scenes: list[dict[str, Any]], + *, + require_orbits: bool, + min_scenes: int, + min_common_overlap_ratio: float, + ) -> list[list[dict[str, Any]]]: + threshold = max(0.0, float(min_common_overlap_ratio or 0.0)) + if threshold <= 0: + return [] + usable = [ + scene for scene in scenes + if scene.get("has_orbit") or not require_orbits + ] + usable = [ + scene for scene in usable + if self._normalize_bbox(scene.get("bbox")) and str(scene.get("date") or "").strip() + ] + if len(usable) < min_scenes: + return [] + + subgroups: list[list[dict[str, Any]]] = [] + seen: set[tuple[str, ...]] = set() + seeds = sorted( + usable, + key=lambda item: ( + str(item.get("date") or ""), + float(item.get("center_lon") or 0.0), + float(item.get("center_lat") or 0.0), + str(item.get("scene_name") or ""), + ), + ) + for seed in seeds: + subgroup = self._grow_common_overlap_subgroup( + seed=seed, + scenes=usable, + min_common_overlap_ratio=threshold, + ) + if len(subgroup) < min_scenes: + continue + if self._scene_common_overlap_ratio(subgroup) < threshold: + continue + key = self._scene_identity_key(subgroup) + if not key or key in seen: + continue + seen.add(key) + subgroups.append(subgroup) + + subgroups.sort( + key=lambda items: ( + -len(items), + -self._scene_common_overlap_ratio(items), + self._subgroup_temporal_gap_score(items), + self._scene_identity_key(items), + ) + ) + return subgroups[:12] + + def _grow_common_overlap_subgroup( + self, + *, + seed: dict[str, Any], + scenes: list[dict[str, Any]], + min_common_overlap_ratio: float, + ) -> list[dict[str, Any]]: + selected = [seed] + selected_names = {str(seed.get("scene_name") or "")} + + while True: + selected_dates = {str(scene.get("date") or "").strip() for scene in selected} + best_scene: dict[str, Any] | None = None + best_score: tuple[Any, ...] | None = None + for scene in scenes: + scene_name = str(scene.get("scene_name") or "") + if scene_name in selected_names: + continue + scene_date = str(scene.get("date") or "").strip() + if not scene_date or scene_date in selected_dates: + continue + trial = selected + [scene] + ratio = self._scene_common_overlap_ratio(trial) + if ratio < min_common_overlap_ratio: + continue + score = ( + len(trial), + ratio, + -self._scene_distance(seed, scene), + str(scene.get("date") or ""), + str(scene.get("scene_name") or ""), + ) + if best_score is None or score > best_score: + best_score = score + best_scene = scene + if best_scene is None: + break + selected.append(best_scene) + selected_names.add(str(best_scene.get("scene_name") or "")) + + return sorted(selected, key=lambda item: (str(item.get("date") or ""), str(item.get("scene_name") or ""))) + + def _scene_common_overlap_ratio(self, scenes: list[dict[str, Any]]) -> float: + bbox_intersection = self._bbox_intersection([scene.get("bbox") for scene in scenes]) + bbox_union = self._stack_bbox_union({"scenes": scenes}) + union_area = self._bbox_area(bbox_union) + if not bbox_intersection or union_area <= 0: + return 0.0 + return self._bbox_area(bbox_intersection) / union_area + + def _scene_distance(self, first: dict[str, Any], second: dict[str, Any]) -> float: + first_lon = self._as_float(first.get("center_lon")) or 0.0 + first_lat = self._as_float(first.get("center_lat")) or 0.0 + second_lon = self._as_float(second.get("center_lon")) or 0.0 + second_lat = self._as_float(second.get("center_lat")) or 0.0 + return math.hypot(first_lon - second_lon, first_lat - second_lat) + + def _subgroup_temporal_gap_score(self, scenes: list[dict[str, Any]]) -> int: + gaps = self._temporal_gaps([ + str(scene.get("date") or "") + for scene in scenes + if str(scene.get("date") or "").strip() + ]) + return max(gaps) if gaps else 0 + + def _dedupe_stack_candidates(self, candidates: list[dict[str, Any]]) -> list[dict[str, Any]]: + by_scene_key: dict[tuple[str, ...], dict[str, Any]] = {} + for candidate in candidates: + scene_key = self._scene_identity_key(candidate.get("scenes") or []) + if not scene_key: + continue + current = by_scene_key.get(scene_key) + if current is None or self._stack_candidate_rank(candidate) > self._stack_candidate_rank(current): + by_scene_key[scene_key] = candidate + + by_stack_id: dict[str, dict[str, Any]] = {} + for candidate in by_scene_key.values(): + stack_id = str(candidate.get("stack_id") or "") + if not stack_id: + continue + current = by_stack_id.get(stack_id) + if current is None or self._stack_candidate_rank(candidate) > self._stack_candidate_rank(current): + by_stack_id[stack_id] = candidate + return list(by_stack_id.values()) + + @staticmethod + def _stack_candidate_rank(candidate: dict[str, Any]) -> tuple[Any, ...]: + return ( + int(candidate.get("status") == "READY"), + int(candidate.get("usable_scene_count") or 0), + float(candidate.get("common_overlap_ratio") or 0.0), + -int(candidate.get("missing_orbit_count") or 0), + -int(candidate.get("max_temporal_gap_days") or 0), + str(candidate.get("date_start") or ""), + str(candidate.get("stack_id") or ""), + ) + def _cluster_aoi_scenes(self, scenes: list[dict[str, Any]]) -> list[list[dict[str, Any]]]: sorted_scenes = sorted( scenes, @@ -5047,6 +6814,108 @@ class SbasInsarProductionService: spatial_key = f"center_{self._center_bucket(lon, lat)}" return f"{observation_key}|{spatial_key}" + def _select_date_keyed_stack_scenes(self, scenes: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Gamma expert scripts key SLC/RSLC products by date, so a run can use one scene per date.""" + duplicate_audit = self._duplicate_scene_date_audit(scenes) + duplicate_dates = set(duplicate_audit.get("duplicate_dates") or []) + if not duplicate_dates: + return list(scenes) + + overlap = self._bbox_intersection([scene.get("bbox") for scene in scenes]) + if overlap: + target_lon = (overlap["min_lon"] + overlap["max_lon"]) / 2 + target_lat = (overlap["min_lat"] + overlap["max_lat"]) / 2 + else: + center = self._stack_center({"scenes": scenes}) or {} + target_lon = self._as_float(center.get("lon")) or 0.0 + target_lat = self._as_float(center.get("lat")) or 0.0 + + selected: list[dict[str, Any]] = [] + excluded: list[dict[str, Any]] = [] + by_date: dict[str, list[dict[str, Any]]] = {} + for scene in scenes: + date = str(scene.get("date") or "").strip() + if not date: + continue + by_date.setdefault(date, []).append(scene) + + def score(scene: dict[str, Any]) -> tuple[float, float, str]: + bbox = self._normalize_bbox(scene.get("bbox")) + if bbox and overlap: + common = self._bbox_intersection([bbox, overlap]) + overlap_area = self._bbox_area(common) + else: + overlap_area = 0.0 + lon = self._as_float(scene.get("center_lon")) or target_lon + lat = self._as_float(scene.get("center_lat")) or target_lat + center_distance = math.hypot(lon - target_lon, lat - target_lat) + return (overlap_area, -center_distance, str(scene.get("scene_name") or "")) + + for date in sorted(by_date): + candidates = by_date[date] + winner = max(candidates, key=score) + selected.append( + { + **winner, + "date_keyed_scene_selected": True, + "same_date_scene_count": len(candidates), + "same_date_selection_policy": "max_common_overlap_then_nearest_cluster_center", + } + ) + for candidate in candidates: + if candidate is winner: + continue + excluded.append( + { + **candidate, + "date_keyed_scene_excluded": True, + "exclude_reason": "same_date_scene_not_selected_for_gamma_date_keyed_stack", + "selected_scene_name": winner.get("scene_name"), + "same_date_scene_count": len(candidates), + } + ) + + duplicate_audit["policy"] = "one_scene_per_date" + duplicate_audit["selection_policy"] = "max_common_overlap_then_nearest_cluster_center" + duplicate_audit["excluded_scene_count"] = len(excluded) + for scene in selected: + scene["date_keyed_duplicate_audit"] = duplicate_audit + scene["date_keyed_excluded_scenes"] = excluded + return selected + + @staticmethod + def _duplicate_scene_date_audit(scenes: list[dict[str, Any]]) -> dict[str, Any]: + by_date: dict[str, list[dict[str, Any]]] = {} + for scene in scenes: + date = str(scene.get("date") or "").strip() + if date: + by_date.setdefault(date, []).append(scene) + duplicate_groups = [] + for date, items in sorted(by_date.items()): + if len(items) <= 1: + continue + duplicate_groups.append( + { + "date": date, + "count": len(items), + "scene_names": [str(item.get("scene_name") or "") for item in items], + "centers": [ + { + "lon": item.get("center_lon"), + "lat": item.get("center_lat"), + } + for item in items + ], + } + ) + return { + "has_duplicate_dates": bool(duplicate_groups), + "duplicate_dates": [item["date"] for item in duplicate_groups], + "duplicate_groups": duplicate_groups, + "scene_count": len(scenes), + "unique_date_count": len(by_date), + } + def _build_stack_candidate( self, scenes: list[dict[str, Any]], @@ -5055,9 +6924,24 @@ class SbasInsarProductionService: require_orbits: bool, discovery_mode: str = "strict", aoi_summary: dict[str, Any] | None = None, - min_common_overlap_ratio: float = 0.0, + min_common_overlap_ratio: float | None = None, ) -> dict[str, Any]: scenes = sorted(scenes, key=lambda item: str(item.get("date") or "")) + date_keyed_duplicate_audit = {} + date_keyed_excluded_scenes: list[dict[str, Any]] = [] + for scene in scenes: + if scene.get("date_keyed_duplicate_audit"): + date_keyed_duplicate_audit = scene.get("date_keyed_duplicate_audit") or {} + if scene.get("date_keyed_excluded_scenes"): + date_keyed_excluded_scenes = scene.get("date_keyed_excluded_scenes") or [] + scenes = [ + { + key: value + for key, value in scene.items() + if key not in {"date_keyed_duplicate_audit", "date_keyed_excluded_scenes"} + } + for scene in scenes + ] first = scenes[0] orbit_ready = [scene for scene in scenes if scene.get("has_orbit")] usable = orbit_ready if require_orbits else scenes @@ -5085,7 +6969,9 @@ class SbasInsarProductionService: ) if mode == "aoi" and usable and not bbox_intersection: blockers.append("no common overlap across usable scenes") - if mode == "aoi" and min_common_overlap_ratio > 0 and common_overlap_ratio < min_common_overlap_ratio: + if mode != "aoi" and usable and not bbox_intersection: + blockers.append("no common overlap across usable scenes") + if min_common_overlap_ratio > 0 and common_overlap_ratio < min_common_overlap_ratio: blockers.append( f"common_overlap_ratio {common_overlap_ratio:.3f} < min_common_overlap_ratio {min_common_overlap_ratio:.3f}" ) @@ -5106,6 +6992,7 @@ class SbasInsarProductionService: "discovery_mode": mode, "aoi": aoi_summary, "group_key": group_key, + "sensor_family": first.get("satellite_family") or self._normalize_sensor_family(first.get("satellite")), "hard_group_fields": [ "satellite", "satellite_mode", @@ -5113,17 +7000,18 @@ class SbasInsarProductionService: "orbit_direction", "imaging_mode", "polarization", + "footprint_common_overlap_cluster", ] if mode == "aoi" else [ "satellite", "satellite_mode", - "receiving_station", "relative_orbit", "orbit_direction", "imaging_mode", "polarization", - "center_bucket", + "footprint_common_overlap_cluster", ], - "soft_group_fields": ["receiving_station", "center_bucket"] if mode == "aoi" else [], + "soft_group_fields": ["receiving_station", "center_bucket"], + "grouping_strategy": first.get("aoi_cluster_source") or "footprint_common_overlap", "satellite": first.get("satellite"), "satellite_mode": first.get("satellite_mode"), "receiving_station": first.get("receiving_station"), @@ -5146,6 +7034,7 @@ class SbasInsarProductionService: "bbox": bbox_union, "bbox_intersection": bbox_intersection, "common_overlap_ratio": common_overlap_ratio, + "min_common_overlap_ratio": min_common_overlap_ratio, "aoi_overlap_ratio_min": min(aoi_overlap_values) if aoi_overlap_values else None, "aoi_overlap_ratio_max": max(aoi_overlap_values) if aoi_overlap_values else None, "aoi_overlap_ratio_mean": ( @@ -5155,6 +7044,8 @@ class SbasInsarProductionService: "center": center, "admin_region": admin_region, "scenes": scenes, + "date_keyed_duplicate_audit": date_keyed_duplicate_audit or self._duplicate_scene_date_audit(scenes), + "date_keyed_excluded_scenes": date_keyed_excluded_scenes, } @staticmethod @@ -5162,6 +7053,23 @@ class SbasInsarProductionService: digest = hashlib.sha1(value.encode("utf-8", errors="ignore")).hexdigest()[:12] return f"sbas_{digest}" + @staticmethod + def _effective_min_common_overlap_ratio(value: Any) -> float: + try: + requested = float(value or 0.0) + except (TypeError, ValueError): + requested = 0.0 + try: + configured = float( + settings.GAMMA_SBAS_MIN_COMMON_OVERLAP_RATIO + or GAMMA_SBAS_FALLBACK_MIN_COMMON_OVERLAP_RATIO + ) + except (TypeError, ValueError): + configured = GAMMA_SBAS_FALLBACK_MIN_COMMON_OVERLAP_RATIO + requested = min(1.0, max(0.0, requested)) + configured = min(1.0, max(0.0, configured)) + return max(requested, configured) + @staticmethod def _temporal_gaps(dates: list[str]) -> list[int]: parsed: list[datetime] = [] @@ -5240,6 +7148,14 @@ class SbasInsarProductionService: def _write_script(path: Path, lines: list[str]) -> Path: path.parent.mkdir(parents=True, exist_ok=True) text = "\n".join(lines) + commands = SbasInsarProductionService._extract_shell_command_tokens(text) + blocking_interactive = sorted(set(GAMMA_SBAS_BLOCKING_INTERACTIVE_TOOLS) & commands) + if blocking_interactive: + raise ValueError( + "Refusing to write an interactive Gamma SBAS production script: " + f"{path} contains {', '.join(blocking_interactive)}. " + f"{GAMMA_SBAS_UNATTENDED_POLICY}" + ) try: path.write_text(text, encoding="utf-8", newline="\n") return path @@ -5277,6 +7193,20 @@ class SbasInsarProductionService: raise FileNotFoundError(f"run not found: {clean_id}") return run_dir + def _resolve_production_delete_path(self, value: Any) -> Path | None: + text = str(value or "").strip() + if not text: + return None + path = Path(text) + if not path.is_absolute(): + path = self.production_root / path + resolved = path.resolve() + try: + resolved.relative_to(self.production_root.resolve()) + except ValueError as exc: + raise ValueError(f"refusing to delete path outside SBAS production root: {resolved}") from exc + return resolved + @staticmethod def _read_json(path: Path) -> dict[str, Any]: return json.loads(path.read_text(encoding="utf-8")) @@ -5317,6 +7247,71 @@ class SbasInsarProductionService: self._write_json(run_dir / "workspace.json", workspace) return workspace + def _ensure_s1_planning_workspace(self, run_dir: Path) -> dict[str, Any]: + dirs = ("RAW", "orbits", "planning", "logs", "scripts", "state", "publish") + created: dict[str, str] = {} + for dirname in dirs: + path = run_dir / dirname + path.mkdir(parents=True, exist_ok=True) + created[dirname] = str(path) + workspace = { + "schema": "insar.s1-gamma-sbas-planning-workspace/v1", + "run_root": str(run_dir), + "directories": created, + "layout_source": "Sentinel-1 Gamma SBAS planning profile", + "execution_enabled": False, + } + self._write_json(run_dir / "workspace.json", workspace) + return workspace + + def _build_s1_workflow_manifest( + self, + run_dir: Path, + run_manifest: dict[str, Any], + stack_manifest: dict[str, Any], + ) -> dict[str, Any]: + steps = [] + for template in S1_GAMMA_SBAS_PLANNING_STEPS: + steps.append( + { + **dict(template), + "enabled": False, + "script": None, + "script_wsl": None, + "log": str(run_dir / "logs" / f"{template['id']}.log"), + "log_wsl": self._windows_path_to_wsl_mount(str(run_dir / "logs" / f"{template['id']}.log")), + "expert_tools": [], + } + ) + return { + "schema": "insar.s1-gamma-sbas-workflow-planning/v1", + "run_id": run_manifest.get("run_id") or run_dir.name, + "workflow_code": "sbas_insar", + "processor_code": "gamma_ipta_sbas", + "engine_code": "gamma", + "profile_code": "s1_gamma_sbas", + "runtime_id": settings.GAMMA_SBAS_RUNTIME_ID, + "created_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "run_root": str(run_dir), + "run_root_wsl": self._windows_path_to_wsl_mount(str(run_dir)), + "execution_enabled": False, + "execution_blocker": "Sentinel-1 Gamma TOPS/SBAS scripts have not been verified.", + "stack": stack_manifest.get("stack") or {}, + "scenes": stack_manifest.get("scenes") or [], + "pair_network": stack_manifest.get("pair_network") or {}, + "directories": { + key: str(run_dir / key) + for key in ("RAW", "orbits", "planning", "logs", "scripts", "state", "publish") + }, + "steps": steps, + "expert_document": { + "schema": "insar.s1-gamma-sbas-design/v1", + "source": "docs/SENTINEL1_GAMMA_SBAS_NO_STITCH_DESIGN.md", + "section_count": 0, + "steps": [], + }, + } + def _build_workflow_manifest( self, run_dir: Path, @@ -5455,7 +7450,7 @@ class SbasInsarProductionService: logs.append(str(workflow_step.get("log"))) status = str(template.get("implementation_status") or "planned") if planned and status.startswith("implemented"): - status = "planned_bridge" + status = "planned" expert_steps.append( { "id": template.get("id"), @@ -5467,12 +7462,14 @@ class SbasInsarProductionService: "mapped_workflow_steps": mapped_workflow_steps, "enabled": enabled, "optional": optional, - "command_count": len(template.get("commands") or []), - "commands": list(template.get("commands") or []), - "scripts": scripts, - "logs": logs, - } - ) + "command_count": len(template.get("commands") or []), + "commands": list(template.get("commands") or []), + "manual_qc_tools": list(template.get("manual_qc_tools") or []), + "unattended_policy": template.get("unattended_policy"), + "scripts": scripts, + "logs": logs, + } + ) return expert_steps def _summarize_workflow_state(self, workflow_manifest: dict[str, Any], state: dict[str, Any]) -> dict[str, Any]: @@ -5537,142 +7534,59 @@ class SbasInsarProductionService: params: dict[str, Any], reference_date: str, ) -> dict[str, dict[str, Any]]: + self._ensure_gamma_date_keyed_stack(stack_manifest) + scenes = sorted(stack_manifest.get("scenes") or [], key=lambda item: str(item.get("date") or "")) + if not scenes: + raise ValueError("Gamma SBAS expert workflow requires at least one LT1 scene") + if not reference_date or reference_date not in {str(scene.get("date") or "") for scene in scenes}: + reference_date = str(scenes[len(scenes) // 2].get("date") or "").strip() + if not reference_date: + raise ValueError("Gamma SBAS expert workflow requires a reference date") + + rlks = self._bounded_int(params.get("rlks"), default=8, minimum=1, maximum=64) + azlks = self._bounded_int(params.get("azlks"), default=8, minimum=1, maximum=64) + reference_window = self._bounded_int(params.get("reference_window"), default=16, minimum=1, maximum=256) + dem_source = self._resolve_expert_dem_import_source(stack_manifest) + dem_source = self._materialize_expert_dem_import_source(run_dir, dem_source) + + writers = { + "01_workspace_data": self._write_expert_workspace_script, + "02_import_lt1_slc": self._write_expert_import_slc_script, + "03_reference_mli": self._write_expert_reference_mli_script, + "04_dem_lookup": self._write_expert_dem_lookup_script, + "05_coreg_prep": self._write_expert_coreg_prep_script, + "06_coregister_scenes": self._write_expert_coregister_scenes_script, + "07_rmli_average": self._write_expert_rmli_average_script, + "08_diff_network": self._write_expert_diff_network_script, + "09_filter_unwrap": self._write_expert_filter_unwrap_script, + "10_detrend_atm": self._write_expert_detrend_atm_script, + "11_sbas_inversion": self._write_expert_sbas_inversion_script, + "12_outputs_points": self._write_expert_outputs_points_script, + } + context = { + "run_dir": run_dir, + "scenes": scenes, + "reference_date": reference_date, + "rlks": rlks, + "azlks": azlks, + "reference_window": reference_window, + "dem_source": dem_source, + } script_records: dict[str, dict[str, Any]] = {} - - workspace_script = run_dir / "scripts" / "01_workspace_data.sh" - workspace_script.parent.mkdir(parents=True, exist_ok=True) - workspace_script.write_text( - "\n".join( - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - f'RUN_ROOT="{self._windows_path_to_wsl_mount(str(run_dir))}"', - 'mkdir -p "${RUN_ROOT}"/{RAW,SLC,dem,rslc_prep,mli_dir,diff_dir,diff1_dir,sbas,publish,logs,scripts,state}', - 'find "${RUN_ROOT}" -maxdepth 1 -type d -printf "%f\\n" | sort', - "", - ] - ), - encoding="utf-8", - newline="\n", - ) - script_records["01_workspace_data"] = self._script_record( - workspace_script, - notes=["Expert section 1 workspace/data-layout check."], - ) - - try: - baseline_script = self._write_baseline_audit_script( - run_dir, - stack_manifest=stack_manifest, - rlks=int(params.get("rlks") or 8), - azlks=int(params.get("azlks") or 8), - max_delta_n=1, - ) - for step_id, filename in ( - ("01_import_slc", "01_import_slc.sh"), - ("02_import_lt1_slc", "02_import_lt1_slc.sh"), - ("03_reference_mli", "03_reference_mli.sh"), - ): - target = run_dir / "scripts" / filename - self._copy_script_alias(baseline_script, target) - script_records[step_id] = self._script_record( - target, - notes=["Current bridge reuses verified baseline-audit import/multilook/base_calc script."], - ) - except Exception as exc: - script_records["01_import_slc"] = {"notes": [f"script not ready: {exc}"]} - script_records["02_import_lt1_slc"] = {"notes": [f"script not ready: {exc}"]} - script_records["03_reference_mli"] = {"notes": [f"script not ready: {exc}"]} - - coreg = run_manifest.get("coregistration") or {} - coreg_script = coreg.get("script_path") - if coreg_script: - source = Path(self._path_to_windows(coreg_script) or coreg_script) - for step_id, filename in ( - ("02_coregister_stack", "02_coregister_stack.sh"), - ("05_coreg_prep", "05_coreg_prep.sh"), - ("06_coregister_scenes", "06_coregister_scenes.sh"), - ("07_rmli_average", "07_rmli_average.sh"), - ): - target = run_dir / "scripts" / filename - self._copy_script_alias(source, target) - script_records[step_id] = self._script_record(target) - - rdc_dem = run_manifest.get("rdc_dem") or {} - rdc_script = rdc_dem.get("script_path") - if rdc_script: - source = Path(self._path_to_windows(rdc_script) or rdc_script) - for step_id, filename in ( - ("03_prepare_dem", "03_prepare_dem.sh"), - ("04_dem_lookup", "04_dem_lookup.sh"), - ): - target = run_dir / "scripts" / filename - self._copy_script_alias(source, target) - script_records[step_id] = self._script_record(target) - - interferograms = run_manifest.get("interferograms") or {} - intf_script = interferograms.get("script_path") - if intf_script: - source = Path(self._path_to_windows(intf_script) or intf_script) - for step_id, filename in ( - ("04_build_network_diff", "04_build_network_diff.sh"), - ("08_diff_network", "08_diff_network.sh"), - ("09_filter_unwrap", "09_filter_unwrap.sh"), - ): - target = run_dir / "scripts" / filename - self._copy_script_alias(source, target) - script_records[step_id] = self._script_record(target) - - detrend_atm = run_manifest.get("detrend_atm") or {} - detrend_script = detrend_atm.get("script_path") - if detrend_script: - source = Path(self._path_to_windows(detrend_script) or detrend_script) - for step_id, filename in ( - ("05_detrend_atm", "05_detrend_atm.sh"), - ("10_detrend_atm", "10_detrend_atm.sh"), - ): - target = run_dir / "scripts" / filename - self._copy_script_alias(source, target) - script_records[step_id] = self._script_record(target) - - ipta = run_manifest.get("ipta_timeseries") or {} - ipta_script = ipta.get("script_path") - if ipta_script: - source = Path(self._path_to_windows(ipta_script) or ipta_script) - for step_id, filename in ( - ("06_sbas_inversion", "06_sbas_inversion.sh"), - ("11_sbas_inversion", "11_sbas_inversion.sh"), - ): - target = run_dir / "scripts" / filename - self._copy_script_alias(source, target) - script_records[step_id] = self._script_record(target) - - publish = run_manifest.get("publish_products") or {} - publish_script = publish.get("script_path") - if publish_script: - target = run_dir / "scripts" / "07_publish_products.sh" - self._copy_script_alias(Path(self._path_to_windows(publish_script) or publish_script), target) - script_records["07_publish_products"] = self._script_record(target) - - monitor = run_manifest.get("monitor_point_products") or {} - monitor_script = monitor.get("script_path") - if monitor_script: - target = run_dir / "scripts" / "08_point_timeseries.sh" - self._copy_script_alias(Path(self._path_to_windows(monitor_script) or monitor_script), target) - script_records["08_point_timeseries"] = self._script_record(target) - if publish_script or monitor_script: - target = run_dir / "scripts" / "12_outputs_points.sh" - wrapper_lines = ["#!/usr/bin/env bash", "set -euo pipefail"] - if publish_script: - wrapper_lines.append(f'bash "{self._windows_path_to_wsl_mount(str(Path(self._path_to_windows(publish_script) or publish_script)))}"') - if monitor_script: - wrapper_lines.append(f'bash "{self._windows_path_to_wsl_mount(str(Path(self._path_to_windows(monitor_script) or monitor_script)))}"') - wrapper_lines.append("") - target.write_text("\n".join(wrapper_lines), encoding="utf-8", newline="\n") - script_records["12_outputs_points"] = self._script_record( - target, - notes=["Expert section 12 wrapper runs publish products followed by monitoring-point extraction when both scripts are available."], - ) + for template in GAMMA_SBAS_WORKFLOW_STEPS: + step_id = str(template.get("id") or "") + writer = writers.get(step_id) + if not writer: + continue + script_path = writer(**context) + audit = self._audit_expert_step_script(step_id, script_path) + script_records[step_id] = self._script_record(script_path, notes=audit.get("notes") or []) + script_records[step_id]["command_audit"] = audit + audit_summary = self._audit_expert_workflow_scripts(script_records) + self._write_json(run_dir / "expert_command_audit.json", audit_summary) + if not audit_summary.get("ready"): + problems = "; ".join(audit_summary.get("problems") or []) + raise ValueError(f"Gamma SBAS expert command audit failed: {problems}") return script_records def _script_record(self, path: Path, *, notes: list[str] | None = None) -> dict[str, Any]: @@ -5690,6 +7604,980 @@ class SbasInsarProductionService: return target.write_text(source.read_text(encoding="utf-8", errors="ignore"), encoding="utf-8", newline="\n") + @staticmethod + def _extract_shell_command_tokens(script_text: str) -> set[str]: + tokens: set[str] = set() + for line in script_text.splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + for separator in (";", "&&", "||"): + stripped = stripped.replace(separator, "\n") + for segment in stripped.splitlines(): + part = segment.strip() + if not part or part.startswith("#"): + continue + first_word = part.split(None, 1)[0] + normalized_first_word = first_word.strip("'\"{}()") + if normalized_first_word in { + "if", + "then", + "else", + "fi", + "for", + "while", + "do", + "done", + "{", + "}", + "local", + "test", + "echo", + "printf", + "cp", + "rm", + "mkdir", + "ln", + "cd", + "read", + "return", + "exit", + ":", + "source", + "set", + }: + continue + if re.match(r"^[A-Za-z_][A-Za-z0-9_]*=", part): + continue + if re.match(r"^[A-Za-z_][A-Za-z0-9_]*=", part.split("$(", 1)[0]): + continue + match = re.match(r'(?:"\$\{?[A-Za-z_][A-Za-z0-9_]*\}?"\s+)?([A-Za-z0-9_._-]+)', part) + if match: + tokens.add(match.group(1)) + return tokens + + def _audit_expert_step_script(self, step_id: str, script_path: Path) -> dict[str, Any]: + text = script_path.read_text(encoding="utf-8", errors="ignore") if script_path.is_file() else "" + commands = self._extract_shell_command_tokens(text) + required = set(GAMMA_SBAS_REQUIRED_STEP_TOOLS.get(step_id) or set()) + missing = sorted(required - commands) + forbidden = sorted(set(GAMMA_SBAS_FORBIDDEN_DEFAULT_TOOLS) & commands) + blocking_interactive = sorted(set(GAMMA_SBAS_BLOCKING_INTERACTIVE_TOOLS) & commands) + manual_qc_tools = set() + for template in GAMMA_SBAS_WORKFLOW_STEPS: + if str(template.get("id") or "") == step_id: + manual_qc_tools = set(template.get("manual_qc_tools") or []) + break + manual_qc_not_executed = sorted(manual_qc_tools - commands) + ready = not missing and not forbidden and not blocking_interactive + notes = [] + if ready: + notes.append("Expert command audit passed.") + if missing: + notes.append("Missing required expert commands: " + ", ".join(missing)) + if forbidden: + notes.append("Forbidden legacy commands present: " + ", ".join(forbidden)) + if blocking_interactive: + notes.append("Blocking interactive commands present: " + ", ".join(blocking_interactive)) + if manual_qc_not_executed: + notes.append( + "Manual QC display commands intentionally not executed in unattended production: " + + ", ".join(manual_qc_not_executed) + ) + return { + "schema": "insar.gamma-sbas-expert-step-command-audit/v1", + "step_id": step_id, + "script": str(script_path), + "ready": ready, + "commands": sorted(commands), + "required_commands": sorted(required), + "missing_required_commands": missing, + "forbidden_commands": forbidden, + "blocking_interactive_commands": blocking_interactive, + "manual_qc_tools": sorted(manual_qc_tools), + "manual_qc_tools_not_executed": manual_qc_not_executed, + "unattended_policy": GAMMA_SBAS_UNATTENDED_POLICY, + "notes": notes, + } + + def _audit_expert_workflow_scripts(self, script_records: dict[str, dict[str, Any]]) -> dict[str, Any]: + step_audits = { + step_id: record.get("command_audit") or {} + for step_id, record in script_records.items() + } + problems: list[str] = [] + for step_id, audit in step_audits.items(): + if not audit.get("ready"): + missing = ", ".join(audit.get("missing_required_commands") or []) + forbidden = ", ".join(audit.get("forbidden_commands") or []) + interactive = ", ".join(audit.get("blocking_interactive_commands") or []) + detail = "; ".join( + item + for item in [ + f"missing={missing}" if missing else "", + f"forbidden={forbidden}" if forbidden else "", + f"interactive={interactive}" if interactive else "", + ] + if item + ) + problems.append(f"{step_id}: {detail or 'command audit failed'}") + ready = not problems and len(step_audits) >= len(GAMMA_SBAS_WORKFLOW_STEPS) + if len(step_audits) < len(GAMMA_SBAS_WORKFLOW_STEPS): + problems.append("not all expert workflow scripts were materialized") + ready = False + return { + "schema": "insar.gamma-sbas-expert-workflow-command-audit/v1", + "generated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "ready": ready, + "step_count": len(step_audits), + "expected_step_count": len(GAMMA_SBAS_WORKFLOW_STEPS), + "problems": problems, + "steps": step_audits, + "forbidden_default_tools": sorted(GAMMA_SBAS_FORBIDDEN_DEFAULT_TOOLS), + "blocking_interactive_tools": sorted(GAMMA_SBAS_BLOCKING_INTERACTIVE_TOOLS), + "unattended_policy": GAMMA_SBAS_UNATTENDED_POLICY, + } + + def _expert_script_header(self, run_dir: Path, *, reference_date: str, rlks: int, azlks: int) -> list[str]: + env_script = ( + self._windows_path_to_wsl_mount(settings.GAMMA_SBAS_ENV_SCRIPT or settings.PYINT_GAMMA_ENV_SCRIPT) + or f"{self._windows_path_to_wsl_mount(settings.PROJECT_ROOT)}/deploy/wsl/profiles/gamma_env.sh" + ) + return [ + "#!/usr/bin/env bash", + "set -euo pipefail", + "", + f'RUN_ROOT="{self._windows_path_to_wsl_mount(str(run_dir))}"', + 'RAW_DIR="${RUN_ROOT}/RAW"', + 'SLC_DIR="${RUN_ROOT}/SLC"', + 'DEM_DIR="${RUN_ROOT}/dem"', + 'RSLC_DIR="${RUN_ROOT}/rslc_prep"', + 'MLI_DIR="${RUN_ROOT}/mli_dir"', + 'DIFF_DIR="${RUN_ROOT}/diff_dir"', + 'DIFF1_DIR="${RUN_ROOT}/diff1_dir"', + 'SBAS_DIR="${RUN_ROOT}/sbas"', + 'PUBLISH_DIR="${RUN_ROOT}/publish"', + 'LOG_DIR="${RUN_ROOT}/logs"', + 'STATE_DIR="${RUN_ROOT}/state"', + f'REF_DATE="{reference_date}"', + f'RLKS="{rlks}"', + f'AZLKS="{azlks}"', + f'source "{env_script}" >/dev/null 2>&1', + 'mkdir -p "${RAW_DIR}" "${SLC_DIR}" "${DEM_DIR}" "${RSLC_DIR}" "${MLI_DIR}" "${DIFF_DIR}" "${DIFF1_DIR}" "${SBAS_DIR}" "${PUBLISH_DIR}" "${LOG_DIR}" "${STATE_DIR}"', + "", + ] + + @staticmethod + def _bash_array(name: str, values: list[str]) -> list[str]: + return [f"{name}=("] + [f' "{value}"' for value in values] + [")"] + + @staticmethod + def _unique_scene_dates(scenes: list[dict[str, Any]]) -> list[str]: + dates: list[str] = [] + seen: set[str] = set() + for scene in scenes: + date = str(scene.get("date") or "").strip() + if date and date not in seen: + dates.append(date) + seen.add(date) + return dates + + @classmethod + def _ensure_gamma_date_keyed_stack(cls, stack_manifest: dict[str, Any]) -> None: + scenes = stack_manifest.get("scenes") or [] + audit = cls._duplicate_scene_date_audit(scenes) + if not audit.get("has_duplicate_dates"): + return + examples: list[str] = [] + for group in audit.get("duplicate_groups") or []: + names = [name for name in (group.get("scene_names") or []) if name] + label = f"{group.get('date')}({group.get('count')})" + if names: + label = f"{label}: {', '.join(names[:3])}" + examples.append(label) + detail = "; ".join(examples[:5]) + raise ValueError( + "Gamma SBAS expert workflow is date-keyed and cannot execute a stack with " + f"multiple scenes on the same acquisition date. Rebuild the stack as one scene per date. {detail}" + ) + + def _write_expert_workspace_script( + self, + *, + run_dir: Path, + scenes: list[dict[str, Any]], + reference_date: str, + rlks: int, + azlks: int, + reference_window: int, + dem_source: dict[str, Any], + ) -> Path: + lines = self._expert_script_header(run_dir, reference_date=reference_date, rlks=rlks, azlks=azlks) + lines.extend( + [ + 'find "${RUN_ROOT}" -maxdepth 1 -type d -printf "%f\\n" | sort >"${STATE_DIR}/expert_workspace_dirs.txt"', + ': >"${STATE_DIR}/scene_dates.txt"', + ] + ) + for scene in scenes: + lines.append(f'echo "{scene.get("date")}" >>"${{STATE_DIR}}/scene_dates.txt"') + lines.extend( + [ + f'echo "{dem_source.get("wsl_path")}" >"${{STATE_DIR}}/dem_import_source.txt"', + 'test "$(wc -l <"${STATE_DIR}/scene_dates.txt")" -gt 0', + "", + ] + ) + return self._write_script(run_dir / "scripts" / "01_workspace_data.sh", lines) + + def _write_expert_import_slc_script( + self, + *, + run_dir: Path, + scenes: list[dict[str, Any]], + reference_date: str, + rlks: int, + azlks: int, + reference_window: int, + dem_source: dict[str, Any], + ) -> Path: + lines = self._expert_script_header(run_dir, reference_date=reference_date, rlks=rlks, azlks=azlks) + lines.extend( + [ + 'run_scene() {', + ' local date="$1"', + ' local tiff="$2"', + ' local meta="$3"', + ' local slc="${SLC_DIR}/${date}.slc"', + ' local par="${SLC_DIR}/${date}.slc.par"', + ' local width=""', + ' {', + ' echo "== expert import LT1 SLC ${date} =="', + ' test -r "${tiff}"', + ' test -r "${meta}"', + ' par_LT1_SLC "${tiff}" "${meta}" "${par}" "${slc}" 0', + ' cp -f "${par}" "${par}.orig"', + ' ORB_filt_spline.py "${par}.orig" "${par}" --ignore_start 3 --ignore_end 17 --degree 5', + ' SLC_corners "${par}"', + ' width="$(awk \'$1 == "range_samples:" {print $2; exit}\' "${par}")"', + ' test -n "${width}"', + ' echo "manual QC display commands are skipped in unattended production: disSLC dismph_fft"', + ' test -s "${slc}"', + ' test -s "${par}"', + ' test -s "${par}.orig"', + ' } >"${LOG_DIR}/${date}_expert_import_slc.log" 2>&1', + '}', + "", + ] + ) + for scene in scenes: + lines.append( + "run_scene " + f'"{scene.get("date")}" ' + f'"{scene.get("tiff_wsl")}" ' + f'"{scene.get("meta_wsl")}"' + ) + lines.extend( + [ + ': >"${SLC_DIR}/SLC_tab"', + ] + ) + for scene in scenes: + date = str(scene.get("date") or "") + lines.append(f'printf "%s %s\\n" "${{SLC_DIR}}/{date}.slc" "${{SLC_DIR}}/{date}.slc.par" >>"${{SLC_DIR}}/SLC_tab"') + lines.extend( + [ + 'test "$(wc -l <"${SLC_DIR}/SLC_tab")" -eq ' + str(len(scenes)), + "", + ] + ) + return self._write_script(run_dir / "scripts" / "02_import_lt1_slc.sh", lines) + + def _write_expert_reference_mli_script( + self, + *, + run_dir: Path, + scenes: list[dict[str, Any]], + reference_date: str, + rlks: int, + azlks: int, + reference_window: int, + dem_source: dict[str, Any], + ) -> Path: + lines = self._expert_script_header(run_dir, reference_date=reference_date, rlks=rlks, azlks=azlks) + lines.extend( + [ + 'REF_SLC="${SLC_DIR}/${REF_DATE}.slc"', + 'REF_PAR="${SLC_DIR}/${REF_DATE}.slc.par"', + 'REF_MLI="${MLI_DIR}/${REF_DATE}_${RLKS}_${AZLKS}.mli"', + 'REF_MLI_PAR="${MLI_DIR}/${REF_DATE}_${RLKS}_${AZLKS}.mli.par"', + '{', + ' echo "== expert reference MLI ${REF_DATE} =="', + ' test -s "${REF_SLC}"', + ' test -s "${REF_PAR}"', + ' multi_look "${REF_SLC}" "${REF_PAR}" "${REF_MLI}" "${REF_MLI_PAR}" "${RLKS}" "${AZLKS}"', + ' width="$(grep range_samples "${REF_MLI_PAR}" | awk \'{print $2; exit}\')"', + ' lines="$(grep azimuth_lines "${REF_MLI_PAR}" | awk \'{print $2; exit}\')"', + ' test -n "${width}"', + ' test -n "${lines}"', + ' ras_dB "${REF_MLI}" "${width}" 1 0 1 1 - - gray.cm "${REF_MLI}.bmp" 0 1', + ' SLC_corners "${REF_MLI_PAR}"', + ' cp -f "${REF_MLI}" "${MLI_DIR}/${REF_DATE}.mli"', + ' cp -f "${REF_MLI_PAR}" "${MLI_DIR}/${REF_DATE}.mli.par"', + '} >"${LOG_DIR}/${REF_DATE}_expert_reference_mli.log" 2>&1', + "", + ] + ) + return self._write_script(run_dir / "scripts" / "03_reference_mli.sh", lines) + + def _write_expert_dem_lookup_script( + self, + *, + run_dir: Path, + scenes: list[dict[str, Any]], + reference_date: str, + rlks: int, + azlks: int, + reference_window: int, + dem_source: dict[str, Any], + ) -> Path: + dem_wsl = str(dem_source.get("wsl_path") or "").strip() + if not dem_wsl: + raise ValueError("expert DEM lookup requires a source DEM for dem_import") + lines = self._expert_script_header(run_dir, reference_date=reference_date, rlks=rlks, azlks=azlks) + lines.extend( + [ + f'DEM_SRC="{dem_wsl}"', + 'REF_MLI="${MLI_DIR}/${REF_DATE}.mli"', + 'REF_MLI_PAR="${MLI_DIR}/${REF_DATE}.mli.par"', + 'SRTM_DEM="${DEM_DIR}/SRTM.dem"', + 'SRTM_DEM_PAR="${DEM_DIR}/SRTM.dem.par"', + 'SRTM_DEM_FILL="${DEM_DIR}/SRTM_dem_fill"', + 'SEG_DEM_PAR="${DEM_DIR}/${REF_DATE}_seg.dem_par"', + 'SEG_DEM="${DEM_DIR}/${REF_DATE}_seg.dem"', + 'LT="${DEM_DIR}/${REF_DATE}.lt"', + 'LS_MAP="${DEM_DIR}/${REF_DATE}.ls_map"', + 'INC="${DEM_DIR}/${REF_DATE}.inc"', + 'PSI="${DEM_DIR}/${REF_DATE}.psi"', + 'PIX="${DEM_DIR}/${REF_DATE}.pix"', + 'GAMMA0="${DEM_DIR}/${REF_DATE}.gamma0"', + 'DIFF_PAR="${DEM_DIR}/${REF_DATE}.diff_par"', + 'OFFS="${DEM_DIR}/${REF_DATE}.offs"', + 'SNR="${DEM_DIR}/${REF_DATE}.snr"', + 'COFFS="${DEM_DIR}/${REF_DATE}.coffs"', + 'COFFSETS="${DEM_DIR}/${REF_DATE}.coffsets"', + 'LT_FINE="${DEM_DIR}/${REF_DATE}.lt_fine"', + 'HGT="${DEM_DIR}/${REF_DATE}.hgt"', + 'REF_GEO="${DEM_DIR}/${REF_DATE}.geo"', + 'BLANK="${DEM_DIR}/${REF_DATE}.blank"', + '{', + ' echo "== expert DEM import and lookup ${REF_DATE} =="', + ' test -s "${DEM_SRC}"', + ' test -s "${REF_MLI}"', + ' test -s "${REF_MLI_PAR}"', + ' dem_import "${DEM_SRC}" "${SRTM_DEM}" "${SRTM_DEM_PAR}" 0 1 0 - - - - - -', + ' dem_width="$(awk \'$1 == "width:" {print $2; exit}\' "${SRTM_DEM_PAR}")"', + ' dem_lines="$(awk \'$1 == "nlines:" {print $2; exit}\' "${SRTM_DEM_PAR}")"', + ' mli_width="$(awk \'$1 == "range_samples:" {print $2; exit}\' "${REF_MLI_PAR}")"', + ' mli_lines="$(awk \'$1 == "azimuth_lines:" {print $2; exit}\' "${REF_MLI_PAR}")"', + ' test -n "${dem_width}"', + ' test -n "${dem_lines}"', + ' test -n "${mli_width}"', + ' test -n "${mli_lines}"', + ' fill_gaps "${SRTM_DEM}" "${dem_width}" "${SRTM_DEM_FILL}" 0 4 0', + ' gc_map2 "${REF_MLI_PAR}" "${SRTM_DEM_PAR}" "${SRTM_DEM_FILL}" "${SEG_DEM_PAR}" "${SEG_DEM}" "${LT}" - - "${LS_MAP}" "${INC}" "${PSI}" "${PIX}" - 8 1', + ' seg_width="$(awk \'$1 == "width:" {print $2; exit}\' "${SEG_DEM_PAR}")"', + ' seg_lines="$(awk \'$1 == "nlines:" {print $2; exit}\' "${SEG_DEM_PAR}")"', + ' test -n "${seg_width}"', + ' test -n "${seg_lines}"', + ' pixel_area "${REF_MLI_PAR}" "${SEG_DEM_PAR}" "${SEG_DEM}" "${LT}" "${LS_MAP}" "${INC}" "${PIX}" "${GAMMA0}" - - 1', + ' : >"${BLANK}"', + ' create_diff_par "${REF_MLI_PAR}" - "${DIFF_PAR}" 1 0 <"${BLANK}"', + ' offset_pwrm "${GAMMA0}" "${REF_MLI}" "${DIFF_PAR}" "${OFFS}" "${SNR}" 256 256 "${DEM_DIR}/${REF_DATE}.offsets" 1 64 64 0.2', + ' offset_fitm "${OFFS}" "${SNR}" "${DIFF_PAR}" "${COFFS}" "${COFFSETS}" 0.2 1', + ' gc_map_fine "${LT}" "${seg_width}" "${DIFF_PAR}" "${LT_FINE}" 1', + ' geocode "${LT_FINE}" "${SEG_DEM}" "${seg_width}" "${HGT}" "${mli_width}" "${mli_lines}"', + ' geocode_back "${REF_MLI}" "${mli_width}" "${LT_FINE}" "${REF_GEO}" "${seg_width}" "${seg_lines}" 5 0', + ' test -s "${LT_FINE}"', + ' test -s "${HGT}"', + ' test -s "${SEG_DEM_PAR}"', + '} >"${LOG_DIR}/${REF_DATE}_expert_dem_lookup.log" 2>&1', + "", + ] + ) + return self._write_script(run_dir / "scripts" / "04_dem_lookup.sh", lines) + + def _write_expert_coreg_prep_script( + self, + *, + run_dir: Path, + scenes: list[dict[str, Any]], + reference_date: str, + rlks: int, + azlks: int, + reference_window: int, + dem_source: dict[str, Any], + ) -> Path: + dates = self._unique_scene_dates(scenes) + lines = self._expert_script_header(run_dir, reference_date=reference_date, rlks=rlks, azlks=azlks) + lines.extend( + [ + 'cp -f "${SLC_DIR}/SLC_tab" "${RSLC_DIR}/SLC_tab"', + ': >"${RSLC_DIR}/dates"', + ] + ) + for date in dates: + lines.append(f'echo "{date}" >>"${{RSLC_DIR}}/dates"') + lines.extend( + [ + 'cp -f "${SLC_DIR}/${REF_DATE}.slc" "${RSLC_DIR}/${REF_DATE}.rslc"', + 'cp -f "${SLC_DIR}/${REF_DATE}.slc.par" "${RSLC_DIR}/${REF_DATE}.rslc.par"', + ': >"${RSLC_DIR}/rslc_tab"', + 'printf "%s %s\\n" "${RSLC_DIR}/${REF_DATE}.rslc" "${RSLC_DIR}/${REF_DATE}.rslc.par" >>"${RSLC_DIR}/rslc_tab"', + 'test -s "${RSLC_DIR}/${REF_DATE}.rslc"', + 'test -s "${RSLC_DIR}/${REF_DATE}.rslc.par"', + "", + ] + ) + return self._write_script(run_dir / "scripts" / "05_coreg_prep.sh", lines) + + def _write_expert_coregister_scenes_script( + self, + *, + run_dir: Path, + scenes: list[dict[str, Any]], + reference_date: str, + rlks: int, + azlks: int, + reference_window: int, + dem_source: dict[str, Any], + ) -> Path: + dates = self._unique_scene_dates(scenes) + lines = self._expert_script_header(run_dir, reference_date=reference_date, rlks=rlks, azlks=azlks) + lines.extend(self._bash_array("DATES", dates)) + lines.extend( + [ + 'REF_RSLC="${RSLC_DIR}/${REF_DATE}.rslc"', + 'REF_RSLC_PAR="${RSLC_DIR}/${REF_DATE}.rslc.par"', + 'coreg_scene() {', + ' local date="$1"', + ' local slc="${SLC_DIR}/${date}.slc"', + ' local slc_par="${SLC_DIR}/${date}.slc.par"', + ' local off="${RSLC_DIR}/${REF_DATE}_${date}.off"', + ' local offs="${RSLC_DIR}/${REF_DATE}_${date}.offs"', + ' local snr="${RSLC_DIR}/${REF_DATE}_${date}.snr"', + ' local coffs="${RSLC_DIR}/${REF_DATE}_${date}.coffs"', + ' local coffsets="${RSLC_DIR}/${REF_DATE}_${date}.coffsets"', + ' local rslc="${RSLC_DIR}/${date}.rslc"', + ' local rslc_par="${RSLC_DIR}/${date}.rslc.par"', + ' {', + ' echo "== expert coreg ${date} -> ${REF_DATE} =="', + ' if [ "${date}" = "${REF_DATE}" ]; then echo "reference scene already prepared"; return; fi', + ' test -s "${REF_RSLC}"', + ' test -s "${REF_RSLC_PAR}"', + ' test -s "${slc}"', + ' test -s "${slc_par}"', + ' create_offset "${REF_RSLC_PAR}" "${slc_par}" "${off}" 1 "${RLKS}" "${AZLKS}" 0', + ' init_offset_orbit "${REF_RSLC_PAR}" "${slc_par}" "${off}"', + ' init_offset "${REF_RSLC}" "${slc}" "${REF_RSLC_PAR}" "${slc_par}" "${off}" "${RLKS}" "${AZLKS}"', + ' offset_pwr "${REF_RSLC}" "${slc}" "${REF_RSLC_PAR}" "${slc_par}" "${off}" "${offs}" "${snr}" 64 64 "${RSLC_DIR}/${REF_DATE}_${date}.offsets" 2 64 64 0.2', + ' offset_fit "${offs}" "${snr}" "${off}" "${coffs}" "${coffsets}" 0.2 1', + ' SLC_interp "${slc}" "${REF_RSLC_PAR}" "${slc_par}" "${off}" "${rslc}" "${rslc_par}"', + ' test -s "${rslc}"', + ' test -s "${rslc_par}"', + ' } >"${LOG_DIR}/${REF_DATE}_${date}_expert_coreg.log" 2>&1', + '}', + 'for date in "${DATES[@]}"; do coreg_scene "${date}"; done', + ': >"${RSLC_DIR}/rslc_tab"', + 'for date in "${DATES[@]}"; do printf "%s %s\\n" "${RSLC_DIR}/${date}.rslc" "${RSLC_DIR}/${date}.rslc.par" >>"${RSLC_DIR}/rslc_tab"; done', + 'test "$(wc -l <"${RSLC_DIR}/rslc_tab")" -eq "${#DATES[@]}"', + "", + ] + ) + return self._write_script(run_dir / "scripts" / "06_coregister_scenes.sh", lines) + + def _write_expert_rmli_average_script( + self, + *, + run_dir: Path, + scenes: list[dict[str, Any]], + reference_date: str, + rlks: int, + azlks: int, + reference_window: int, + dem_source: dict[str, Any], + ) -> Path: + lines = self._expert_script_header(run_dir, reference_date=reference_date, rlks=rlks, azlks=azlks) + lines.extend( + [ + 'cd "${RSLC_DIR}"', + 'mk_mli_all rslc_tab . "${RLKS}" "${AZLKS}" 1 1.0 0.4 mli.ave', + 'width="$(grep range_samples mli.ave.par | awk \'{print $2; exit}\')"', + 'lines="$(grep azimuth_lines mli.ave.par | awk \'{print $2; exit}\')"', + 'test -n "${width}"', + 'test -n "${lines}"', + 'ras_dB mli.ave "${width}" 1 0 1 1 - - gray.cm mli.ave.bmp 0 1', + 'cp -f mli.ave "${MLI_DIR}/mli.ave"', + 'cp -f mli.ave.par "${MLI_DIR}/mli.ave.par"', + 'cp -f mli.ave.bmp "${MLI_DIR}/mli.ave.bmp" || true', + ': >"${MLI_DIR}/RMLI_tab"', + 'while read -r rslc rslc_par; do', + ' date="$(basename "${rslc}" .rslc)"', + ' ln -sf "${RSLC_DIR}/${date}.rmli" "${MLI_DIR}/${date}.rmli"', + ' ln -sf "${RSLC_DIR}/${date}.rmli.par" "${MLI_DIR}/${date}.rmli.par"', + ' [ -f "${RSLC_DIR}/${date}.rmli.bmp" ] && ln -sf "${RSLC_DIR}/${date}.rmli.bmp" "${MLI_DIR}/${date}.rmli.bmp" || true', + ' printf "%s %s\\n" "${MLI_DIR}/${date}.rmli" "${MLI_DIR}/${date}.rmli.par" >>"${MLI_DIR}/RMLI_tab"', + 'done < rslc_tab', + 'test "$(wc -l <"${MLI_DIR}/RMLI_tab")" -gt 0', + "", + ] + ) + return self._write_script(run_dir / "scripts" / "07_rmli_average.sh", lines) + + def _write_expert_diff_network_script( + self, + *, + run_dir: Path, + scenes: list[dict[str, Any]], + reference_date: str, + rlks: int, + azlks: int, + reference_window: int, + dem_source: dict[str, Any], + ) -> Path: + lines = self._expert_script_header(run_dir, reference_date=reference_date, rlks=rlks, azlks=azlks) + lines.extend( + [ + 'cd "${DIFF_DIR}"', + 'ln -sf "${RSLC_DIR}/rslc_tab" rslc_tab', + 'ln -sf "${MLI_DIR}/mli.ave" mli.ave', + 'ln -sf "${MLI_DIR}/mli.ave.par" mli.ave.par', + 'ln -sf "${DEM_DIR}/${REF_DATE}.hgt" "${REF_DATE}.hgt"', + 'base_calc rslc_tab "${RSLC_DIR}/${REF_DATE}.rslc.par" bprep_file itab 1 1 - - 1 3650 1', + 'base_plot rslc_tab "${RSLC_DIR}/${REF_DATE}.rslc.par" itab bprep_file 1', + 'mk_diff_2d rslc_tab itab 0 "${REF_DATE}.hgt" - mli.ave "${MLI_DIR}" . "${RLKS}" "${AZLKS}" 3 1 1 0 -u', + 'test -s itab', + 'ls *.diff > diff.list', + 'test -s diff.list', + "", + ] + ) + return self._write_script(run_dir / "scripts" / "08_diff_network.sh", lines) + + def _write_expert_filter_unwrap_script( + self, + *, + run_dir: Path, + scenes: list[dict[str, Any]], + reference_date: str, + rlks: int, + azlks: int, + reference_window: int, + dem_source: dict[str, Any], + ) -> Path: + lines = self._expert_script_header(run_dir, reference_date=reference_date, rlks=rlks, azlks=azlks) + lines.extend( + [ + 'cd "${DIFF_DIR}"', + 'width="$(awk \'$1 == "range_samples:" {print $2; exit}\' "${MLI_DIR}/mli.ave.par")"', + 'lines="$(awk \'$1 == "azimuth_lines:" {print $2; exit}\' "${MLI_DIR}/mli.ave.par")"', + 'test -n "${width}"', + 'test -n "${lines}"', + 'r_seed="$(( width / 2 ))"', + 'a_seed="$(( lines / 2 ))"', + 'mk_adf_2d rslc_tab itab mli.ave . 5 0.6 32 8 -u', + 'ls *.adf.cc > cc.list', + 'test -s cc.list', + 'ave_image cc.list "${width}" mean.cc', + 'rascc_mask mean.cc - "${width}" 1 1 - 1 1 0.20', + 'mk_unw_2d rslc_tab itab mli.ave . 0.20 0 1 1 1 1 "${r_seed}" "${a_seed}" 1 -u', + 'mk_unw_2d rslc_tab itab mli.ave . - - 1 1 1 1 "${r_seed}" "${a_seed}" 1 mean.cc_mask.bmp -u', + ': > unw.list', + 'while read -r i1 i2 pair_idx use_flag; do', + ' [ "${use_flag}" = "1" ] || continue', + ' d1="$(awk -v n="${i1}" \'NR == n {print $1; exit}\' rslc_tab)"', + ' d2="$(awk -v n="${i2}" \'NR == n {print $1; exit}\' rslc_tab)"', + ' date1="$(basename "${d1}" .rslc)"', + ' date2="$(basename "${d2}" .rslc)"', + ' unw="${date1}_${date2}.adf.unw"', + ' test -s "${unw}"', + ' echo "${unw}" >> unw.list', + 'done < itab', + 'test -s unw.list', + "", + ] + ) + return self._write_script(run_dir / "scripts" / "09_filter_unwrap.sh", lines) + + def _write_expert_detrend_atm_script( + self, + *, + run_dir: Path, + scenes: list[dict[str, Any]], + reference_date: str, + rlks: int, + azlks: int, + reference_window: int, + dem_source: dict[str, Any], + ) -> Path: + lines = self._expert_script_header(run_dir, reference_date=reference_date, rlks=rlks, azlks=azlks) + lines.extend( + [ + 'cd "${DIFF_DIR}"', + 'width="$(awk \'$1 == "range_samples:" {print $2; exit}\' "${MLI_DIR}/mli.ave.par")"', + 'test -n "${width}"', + 'valid_float_count() {', + ' local path="$1"', + ' python - "${path}" <<\'PY\'', + 'import sys', + 'from pathlib import Path', + 'import numpy as np', + '', + 'path = Path(sys.argv[1])', + 'if not path.is_file():', + ' print(0)', + ' raise SystemExit(0)', + 'data = np.fromfile(path, dtype=">f4")', + 'valid = np.isfinite(data) & (data != 0.0) & (np.abs(data) < 1.0e20)', + 'print(int(valid.sum()))', + 'PY', + '}', + ': > unw_atmsub_tab', + 'while read -r unw; do', + ' test -s "${unw}"', + ' pair="${unw%.adf.unw}"', + ' off="${pair}.off"', + ' diff_par="${pair}.diff_par"', + ' create_diff_par "${off}" "${off}" "${diff_par}" 0 0', + ' quad_fit "${unw}" "${diff_par}" 5 5 - - 3 "${pair}.unw_linear"', + ' quad_sub "${unw}" "${diff_par}" "${pair}.unw_sub_linear" 0 0', + ' rasdt_pwr "${pair}.unw_sub_linear" mli.ave "${width}" 1 - 1 1 -6.28 6.28 1 rmg.cm "${pair}.unw_sub_linear.bmp" 1.0 0.35 8 || true', + ' unw_sub_linear_valid="$(valid_float_count "${pair}.unw_sub_linear")"', + ' if [ "${unw_sub_linear_valid}" -le 0 ]; then', + ' echo "no valid pixels in ${pair}.unw_sub_linear" >&2', + ' exit 1', + ' fi', + ' selected_mfrac=""', + ' for mfrac in 0.20 0.10 0.05; do', + ' echo "atm_mod_2d_attempt pair=${pair} mfrac=${mfrac}"', + ' rm -f "${pair}.a0" "${pair}.a1" "${pair}.atm_sigma" "${pair}.atm_sigma_h" "${pair}.atm_s1" "${pair}.a0_fill" "${pair}.a1_fill" "${pair}.atm_model" "${pair}.unw.atmsub"', + ' atm_rc=0', + ' atm_mod_2d "${pair}.unw_sub_linear" "${DEM_DIR}/${REF_DATE}.hgt" "${pair}.adf.cc" "${diff_par}" - 0 "${pair}.a0" "${pair}.a1" "${pair}.atm_sigma" "${pair}.atm_sigma_h" "${pair}.atm_s1" 512 512 64 64 7000 - 0.15 "${mfrac}" - - 1 || atm_rc=$?', + ' if [ "${atm_rc}" -ne 0 ]; then', + ' echo "atm_mod_2d failed pair=${pair} mfrac=${mfrac}" >&2', + ' continue', + ' fi', + ' a0_valid="$(valid_float_count "${pair}.a0")"', + ' a1_valid="$(valid_float_count "${pair}.a1")"', + ' if [ "${a0_valid}" -le 0 ] && [ "${a1_valid}" -le 0 ]; then', + ' echo "atm_mod_2d produced no nonzero model coefficients pair=${pair} mfrac=${mfrac} a0_valid=${a0_valid} a1_valid=${a1_valid}" >&2', + ' continue', + ' fi', + ' patch_width="$(awk \'$1 == "offset_estimation_range_samples:" {print $2; exit}\' "${diff_par}")"', + ' test -n "${patch_width}"', + ' fill_gaps "${pair}.a0" "${patch_width}" "${pair}.a0_fill" 0 4 0', + ' fill_gaps "${pair}.a1" "${patch_width}" "${pair}.a1_fill" 0 4 0', + ' atm_sim_2d "${diff_par}" "${DEM_DIR}/${REF_DATE}.hgt" "${pair}.a0_fill" "${pair}.a1_fill" "${pair}.atm_model" -', + ' atm_valid="$(valid_float_count "${pair}.atm_model")"', + ' if [ "${atm_valid}" -le 0 ]; then', + ' echo "atm_sim_2d produced no nonzero model pair=${pair} mfrac=${mfrac}" >&2', + ' continue', + ' fi', + ' sub_phase "${pair}.unw_sub_linear" "${pair}.atm_model" "${diff_par}" "${pair}.unw.atmsub" 0 0 0', + ' atmsub_valid="$(valid_float_count "${pair}.unw.atmsub")"', + ' if [ "${atmsub_valid}" -le 0 ]; then', + ' echo "sub_phase produced no valid pixels pair=${pair} mfrac=${mfrac}" >&2', + ' continue', + ' fi', + ' selected_mfrac="${mfrac}"', + ' echo "atm_correction_selected pair=${pair} mfrac=${selected_mfrac} unw_sub_linear_valid=${unw_sub_linear_valid} a0_valid=${a0_valid} a1_valid=${a1_valid} atm_valid=${atm_valid} atmsub_valid=${atmsub_valid}"', + ' break', + ' done', + ' if [ -z "${selected_mfrac}" ]; then', + ' echo "atmospheric correction failed for ${pair}; tried mfrac 0.20, 0.10, 0.05 and produced no valid ${pair}.unw.atmsub" >&2', + ' exit 1', + ' fi', + ' test -s "${pair}.unw.atmsub"', + ' echo "${DIFF_DIR}/${pair}.unw.atmsub" >> unw_atmsub_tab', + 'done < unw.list', + 'cp -f unw_atmsub_tab "${SBAS_DIR}/unw_atmsub_tab"', + 'cp -f itab "${SBAS_DIR}/itab"', + 'cp -f "${MLI_DIR}/RMLI_tab" "${SBAS_DIR}/RMLI_tab"', + 'test -s "${SBAS_DIR}/unw_atmsub_tab"', + "", + ] + ) + return self._write_script(run_dir / "scripts" / "10_detrend_atm.sh", lines) + + def _write_expert_sbas_inversion_script( + self, + *, + run_dir: Path, + scenes: list[dict[str, Any]], + reference_date: str, + rlks: int, + azlks: int, + reference_window: int, + dem_source: dict[str, Any], + ) -> Path: + reference_dt = None + try: + reference_dt = datetime.strptime(reference_date, "%Y%m%d") + except ValueError: + reference_dt = None + temporal_reference_date = reference_date + temporal_candidates: list[tuple[int, str]] = [] + for scene in scenes: + date = str(scene.get("date") or "").strip() + if not date or date == reference_date: + continue + if reference_dt is not None: + try: + delta = abs((datetime.strptime(date, "%Y%m%d") - reference_dt).days) + except ValueError: + delta = 999999 + else: + delta = len(temporal_candidates) + temporal_candidates.append((delta, date)) + if temporal_candidates: + temporal_reference_date = sorted(temporal_candidates, key=lambda item: (item[0], item[1]))[0][1] + + lines = self._expert_script_header(run_dir, reference_date=reference_date, rlks=rlks, azlks=azlks) + lines.extend( + [ + 'cd "${SBAS_DIR}"', + 'mkdir -p ras', + 'width="$(awk \'$1 == "range_samples:" {print $2; exit}\' "${MLI_DIR}/mli.ave.par")"', + 'lines="$(awk \'$1 == "azimuth_lines:" {print $2; exit}\' "${MLI_DIR}/mli.ave.par")"', + 'test -n "${width}"', + 'test -n "${lines}"', + f'REFERENCE_WINDOW="{reference_window}"', + f'GEOM_REF_MLI_PAR="${{MLI_DIR}}/{reference_date}.rmli.par"', + f'TREF_MLI_PAR="${{MLI_DIR}}/{temporal_reference_date}.rmli.par"', + 'test -s "${GEOM_REF_MLI_PAR}"', + 'test -s "${TREF_MLI_PAR}"', + 'cp -f "${MLI_DIR}/mli.ave.par" mli.ave.par', + 'rm -f diff1.sigma_ts diff2.sigma_ts diff.sigma_ts hgt_correction_1 itab_ts unw.atmsub_1_tab final_unw_tab', + 'rm -f ras/diff*.tab ras/diff*.diff ras/diff*.bmp', + 'WIDTH="${width}" LINES="${lines}" REFERENCE_WINDOW="${REFERENCE_WINDOW}" python - <<\'PY\' > reference_region.txt.tmp', + 'import os', + 'from pathlib import Path', + 'import numpy as np', + '', + 'width = int(os.environ["WIDTH"])', + 'lines = int(os.environ["LINES"])', + 'requested_window = int(os.environ["REFERENCE_WINDOW"])', + 'candidate_windows = [16, 8, 4]', + 'center_x = width // 2', + 'center_y = lines // 2', + 'pairs = [Path(line.strip()) for line in Path("unw_atmsub_tab").read_text().splitlines() if line.strip()]', + 'if not pairs:', + ' raise SystemExit("unw_atmsub_tab is empty")', + 'valid_layers = []', + 'for path in pairs:', + ' data = np.fromfile(path, dtype=">f4", count=width * lines)', + ' if data.size != width * lines:', + ' raise SystemExit(f"incomplete unwrapped phase file: {path}")', + ' arr = data.reshape((lines, width))', + ' valid = np.isfinite(arr) & (arr != 0.0) & (np.abs(arr) < 1.0e20)', + ' valid_layers.append(valid)', + '', + 'common_valid = np.logical_and.reduce(valid_layers)', + '', + 'def window_sums(mask, window):', + ' arr = mask.astype(np.uint8)', + ' integral = np.pad(arr, ((1, 0), (1, 0)), mode="constant").cumsum(axis=0).cumsum(axis=1)', + ' return integral[window:, window:] - integral[:-window, window:] - integral[window:, :-window] + integral[:-window, :-window]', + '', + 'def best_complete_window(sums, expected, window):', + ' complete = sums == expected', + ' if not bool(complete.any()):', + ' return None', + ' half = window // 2', + ' best = None', + ' for y0 in range(complete.shape[0]):', + ' xs = np.flatnonzero(complete[y0])', + ' if xs.size == 0:', + ' continue', + ' y = y0 + half', + ' distances = np.abs(xs + half - center_x) + abs(y - center_y)', + ' idx = int(np.argmin(distances))', + ' candidate = (int(distances[idx]), int(xs[idx] + half), int(y))', + ' if best is None or candidate < best:', + ' best = candidate', + ' return best', + '', + 'diagnostics = []', + 'for window in candidate_windows:', + ' expected = window * window', + ' sums = window_sums(common_valid, window)', + ' max_valid = int(sums.max()) if sums.size else 0', + ' diagnostics.append(f"{window}x{window}:max={max_valid}/{expected}")', + ' best = best_complete_window(sums, expected, window)', + ' if best is not None:', + ' _, x, y = best', + ' print(x, y, expected, expected * len(valid_layers), window)', + ' break', + 'else:', + ' raise SystemExit("no complete reference window found for allowed windows (minimum 4x4); tried " + ", ".join(diagnostics))', + 'PY', + 'mv -f reference_region.txt.tmp reference_region.txt', + 'read -r r_ref a_ref min_valid total_valid actual_reference_window < reference_region.txt', + 'echo "selected_reference_region range=${r_ref} azimuth=${a_ref} min_valid=${min_valid} total_valid=${total_valid} window=${actual_reference_window} requested_window=${REFERENCE_WINDOW} fallback_ladder=16,8,4"', + 'if [ "${actual_reference_window}" != "${REFERENCE_WINDOW}" ]; then', + ' echo "reference_window_degraded from=${REFERENCE_WINDOW} to=${actual_reference_window}" >&2', + 'fi', + 'mb unw_atmsub_tab RMLI_tab itab - itab_ts ras/diff1 1 diff1.sigma_ts 1 hgt_correction_1 "${r_ref}" "${a_ref}" "${actual_reference_window}" "${actual_reference_window}" 1.0 "${GEOM_REF_MLI_PAR}" "${TREF_MLI_PAR}" 0', + ': > unw.atmsub_1_tab', + 'while read -r unw; do', + ' test -s "${unw}"', + ' base="$(basename "${unw}" .unw.atmsub)"', + ' sim="${unw}_sim"', + ' test -s "${sim}"', + ' real_to_cpx - "${unw}" "${base}.unw.atmsub.cpx" "${width}" 1', + ' unw_model "${base}.unw.atmsub.cpx" "${sim}" "${base}.unw.atmsub_1" "${width}" "${r_ref}" "${a_ref}"', + ' echo "${SBAS_DIR}/${base}.unw.atmsub_1" >> unw.atmsub_1_tab', + 'done < unw_atmsub_tab', + 'mb unw.atmsub_1_tab RMLI_tab itab - itab_ts ras/diff2 1 diff2.sigma_ts 0 - "${r_ref}" "${a_ref}" "${actual_reference_window}" "${actual_reference_window}" 1.0 "${GEOM_REF_MLI_PAR}" "${TREF_MLI_PAR}" 0', + 'cp -f unw.atmsub_1_tab final_unw_tab', + 'mb final_unw_tab RMLI_tab itab - itab_ts ras/diff 0 diff.sigma_ts 0 - "${r_ref}" "${a_ref}" "${actual_reference_window}" "${actual_reference_window}" 0.5 "${GEOM_REF_MLI_PAR}" "${TREF_MLI_PAR}" 0', + 'find ras -maxdepth 1 -type f -name "diff_*.diff" | sort > ras/diff.tab', + 'test -s diff.sigma_ts', + 'test -s itab_ts', + 'test -s ras/diff.tab', + "", + ] + ) + return self._write_script(run_dir / "scripts" / "11_sbas_inversion.sh", lines) + + def _write_expert_outputs_points_script( + self, + *, + run_dir: Path, + scenes: list[dict[str, Any]], + reference_date: str, + rlks: int, + azlks: int, + reference_window: int, + dem_source: dict[str, Any], + ) -> Path: + lines = self._expert_script_header(run_dir, reference_date=reference_date, rlks=rlks, azlks=azlks) + lines.extend( + [ + 'cd "${SBAS_DIR}"', + 'mkdir -p "${PUBLISH_DIR}/geotiff" "${PUBLISH_DIR}/points"', + 'width="$(awk \'$1 == "range_samples:" {print $2; exit}\' mli.ave.par)"', + 'dem_width="$(awk \'$1 == "width:" {print $2; exit}\' "${DEM_DIR}/${REF_DATE}_seg.dem_par")"', + 'dem_lines="$(awk \'$1 == "nlines:" {print $2; exit}\' "${DEM_DIR}/${REF_DATE}_seg.dem_par")"', + 'test -n "${width}"', + 'test -n "${dem_width}"', + 'test -n "${dem_lines}"', + 'az_lines="$(awk \'$1 == "azimuth_lines:" {print $2; exit}\' mli.ave.par)"', + 'test -n "${az_lines}"', + 'replace_values diff.sigma_ts 0.5 0.0 diff.sigma_ts.masked "${width}" 1 2 0', + 'rasdt_pwr diff.sigma_ts.masked - "${width}" 1 0 1 1 0.0 1.5 1 cc.cm diff.sigma_ts.masked.bmp 1.0 0.35 8', + ': > disp.TS_tab', + 'while read -r item; do', + ' date="$(basename "${item}")"', + ' masked="ras/${date}.masked"', + ' mask_data "${item}" "${width}" "${masked}" diff.sigma_ts.masked.bmp 0', + ' dispmap "${masked}" - mli.ave.par - "ras/${date}.disp" 0 0', + ' echo "${SBAS_DIR}/ras/${date}.disp" >> disp.TS_tab', + 'done < ras/diff.tab', + 'ts_rate disp.TS_tab RMLI_tab itab_ts - los_def_rate los_def_const los_def_sigma 0', + 'rasdt_pwr los_def_rate "${MLI_DIR}/mli.ave" "${width}" 1 0 1 1 -0.08 0.08 0 hls.cm los_def_rate.bmp 1.0 0.35 24', + 'geocode_back los_def_rate "${width}" "${DEM_DIR}/${REF_DATE}.lt_fine" geo_los_def_rate "${dem_width}" "${dem_lines}" 5 0', + 'data2geotiff "${DEM_DIR}/${REF_DATE}_seg.dem_par" geo_los_def_rate 2 "${PUBLISH_DIR}/geotiff/geo_los_def_rate.tif"', + 'geocode_back los_def_rate.bmp "${width}" "${DEM_DIR}/${REF_DATE}.lt_fine" geo_los_def_rate.bmp "${dem_width}" "${dem_lines}" 0 2', + 'data2geotiff "${DEM_DIR}/${REF_DATE}_seg.dem_par" geo_los_def_rate.bmp 0 "${PUBLISH_DIR}/geotiff/geo_los_def_rate_rgb.tif"', + 'python3 - "${width}" "${az_lines}" "${PUBLISH_DIR}/points/disp_point_sel.txt" "${PUBLISH_DIR}/points/disp_point_selection.json" <<\'PY\'', + 'import sys', + 'import json', + 'from datetime import datetime', + 'import numpy as np', + '', + 'width = int(sys.argv[1])', + 'lines = int(sys.argv[2])', + 'selection_txt = sys.argv[3]', + 'selection_json = sys.argv[4]', + 'count = width * lines', + 'rate = np.fromfile("los_def_rate", dtype=">f4", count=count)', + 'sigma = np.fromfile("diff.sigma_ts.masked", dtype=">f4", count=count)', + 'count = min(rate.size, sigma.size, count)', + 'if count < width * lines:', + ' lines = count // width', + ' count = width * lines', + 'rate = rate[:count].reshape(lines, width)', + 'sigma = sigma[:count].reshape(lines, width)', + 'yy, xx = np.indices(rate.shape, dtype=np.float32)', + 'edge = (xx > width * 0.08) & (xx < width * 0.92) & (yy > lines * 0.08) & (yy < lines * 0.92)', + 'valid = np.isfinite(rate) & np.isfinite(sigma) & (rate != 0.0) & (sigma > 0.0) & edge', + 'abs_rate = np.abs(rate)', + 'definitions = [', + ' ("toward_high_rate_low_sigma", "趋近雷达高形变低残差点", "rate > 0，且绝对速率位于高分位，残差低，用于检查明显正向形变区域。"),', + ' ("away_high_rate_low_sigma", "远离雷达高形变低残差点", "rate < 0，且绝对速率位于高分位，残差低，用于检查明显负向形变区域。"),', + ' ("high_abs_rate_low_sigma", "高绝对速率低残差点", "不区分正负，优先选择绝对速率高且残差低的有效点。"),', + ' ("stable_low_sigma", "近零低残差代表点", "绝对速率位于低分位且残差低，用于对照相对稳定区域。"),', + ' ("center_valid", "覆盖区中心有效点", "从有效像元中选取最接近雷达网格中心的点，用于空间位置对照。"),', + ']', + 'selected = []', + 'min_dist2 = float(max(32, int(min(width, lines) * 0.08)) ** 2)', + 'def add_point(definition, candidate, score):', + ' if not np.any(candidate):', + ' return', + ' filtered = candidate.copy()', + ' for existing in selected:', + ' filtered &= ((xx - float(existing["img_x"])) ** 2 + (yy - float(existing["img_y"])) ** 2) >= min_dist2', + ' if not np.any(filtered):', + ' filtered = candidate', + ' safe_score = np.full(rate.shape, -np.inf, dtype=np.float64)', + ' safe_score[filtered] = score[filtered]', + ' if not np.any(np.isfinite(safe_score[filtered])):', + ' return', + ' y, x = np.unravel_index(int(np.nanargmax(safe_score)), safe_score.shape)', + ' point = (int(x), int(y))', + ' if not any(point[0] == item["img_x"] and point[1] == item["img_y"] for item in selected):', + ' key, label, description = definition', + ' selected.append({"img_x": point[0], "img_y": point[1], "selection_key": key, "selection_label": label, "selection_description": description})', + 'if np.any(valid):', + ' abs_valid = abs_rate[valid]', + ' sig_valid = sigma[valid]', + ' high_abs = float(np.percentile(abs_valid, 85))', + ' low_abs = float(np.percentile(abs_valid, 25))', + ' low_sigma = float(np.percentile(sig_valid, 40))', + ' low_sig = valid & (sigma <= low_sigma)', + ' if not np.any(low_sig):', + ' low_sig = valid', + ' denom = np.maximum(sigma.astype(np.float64), 1.0e-6)', + ' add_point(definitions[0], low_sig & (rate > 0.0) & (abs_rate >= high_abs), rate / denom)', + ' add_point(definitions[1], low_sig & (rate < 0.0) & (abs_rate >= high_abs), -rate / denom)', + ' add_point(definitions[2], low_sig & (abs_rate >= high_abs), abs_rate / denom)', + ' add_point(definitions[3], low_sig & (abs_rate <= low_abs), 1.0 / ((abs_rate + 1.0) * denom))', + ' cx, cy = (width - 1) / 2.0, (lines - 1) / 2.0', + ' add_point(definitions[4], valid, -((xx - cx) ** 2 + (yy - cy) ** 2))', + 'if not selected:', + ' key, label, description = definitions[4]', + ' selected.append({"img_x": width // 2, "img_y": lines // 2, "selection_key": key, "selection_label": label, "selection_description": description})', + 'selected = selected[:5]', + 'for index, item in enumerate(selected, start=1):', + ' item["selection_rank"] = index', + 'with open(selection_txt, "w", encoding="utf-8") as handle:', + ' for item in selected:', + ' handle.write(f"{item[\'img_x\']} {item[\'img_y\']}\\n")', + 'payload = {"schema": "insar.gamma-sbas-expert-monitor-point-selection/v1", "generated_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", "source": "auto_representative_points", "selection_count": len(selected), "strategy": "auto_representative_points", "strategy_note": "自动选取趋近/远离雷达高形变、绝对高形变、近零稳定和中心有效点；时序仍由 Gamma disp_prt_2d 输出。", "points": selected}', + 'with open(selection_json, "w", encoding="utf-8") as handle:', + ' json.dump(payload, handle, ensure_ascii=False, indent=2)', + 'PY', + 'disp_prt_2d disp.TS_tab RMLI_tab itab_ts - 3 "${PUBLISH_DIR}/points/disp_point_sel.txt" "${DEM_DIR}/${REF_DATE}.hgt" los_def_rate diff.sigma_ts.masked "${PUBLISH_DIR}/points/items.txt" "${PUBLISH_DIR}/points/disp_point.txt" 3 1 0', + 'test -s "${PUBLISH_DIR}/geotiff/geo_los_def_rate.tif"', + 'test -s "${PUBLISH_DIR}/geotiff/geo_los_def_rate_rgb.tif"', + 'test -s "${PUBLISH_DIR}/points/items.txt"', + 'test -s "${PUBLISH_DIR}/points/disp_point.txt"', + "", + ] + ) + return self._write_script(run_dir / "scripts" / "12_outputs_points.sh", lines) + def _build_monitor_point_config( self, *, @@ -5917,6 +8805,8 @@ class SbasInsarProductionService: rlks: int, azlks: int, ) -> Path: + # Legacy bridge writer retained for old stage endpoints; the default LT1 Gamma SBAS + # workflow now executes the expert-document scripts generated above. scripts_dir = run_dir / "scripts" script_path = scripts_dir / "02_coreg_common_ref.sh" gamma_root = run_dir / "work" / "gamma" @@ -6074,6 +8964,8 @@ class SbasInsarProductionService: rlks: int, dem_source: dict[str, Any], ) -> Path: + # Legacy bridge writer retained for old stage endpoints; not used by the + # default expert-document workflow. scripts_dir = run_dir / "scripts" script_path = scripts_dir / "03_prepare_rdc_dem.sh" gamma_root = run_dir / "work" / "gamma" @@ -6214,6 +9106,8 @@ class SbasInsarProductionService: azlks: int, unwrap_threshold: float, ) -> Path: + # Legacy bridge writer retained for old stage endpoints; not used by the + # default expert-document workflow. scripts_dir = run_dir / "scripts" script_path = scripts_dir / "04_diff_unwrap_common_ref.sh" gamma_root = run_dir / "work" / "gamma" @@ -6341,11 +9235,11 @@ class SbasInsarProductionService: ' "${diff}" "${RLKS}" "${AZLKS}" "${SPS_FLAG}" "${AZF_FLAG}" - 1 1', ' adf "${diff}" "${diff_filt}" "${cc}" "${width}" 0.4 - 5', ' cc_wave "${diff_filt}" "${mli1}" "${mli2}" "${cc}" "${width}" 5 5', - ' rasmph_pwr "${diff_filt}" "${mli1}" "${width}" - - - - - - - - - "${cc}" - 0.1', - ' rasdt_pwr "${cc}" "${mli1}" "${width}" 1 0 1 1 0.1 1.0 1', + ' rasmph_pwr "${diff_filt}" "${mli1}" "${width}" - - - - rmg.cm "${diff_filt}.bmp" 1.0 0.35 8', + ' rasdt_pwr "${cc}" "${mli1}" "${width}" 1 0 1 1 0.1 1.0 1 cc.cm "${cc}.bmp" 1.0 0.35 8', ' rascc_mask "${cc}" "${mli1}" "${width}" 1 1 0 1 1 "${UNWRAP_THRESHOLD}" 0.0 0.1 0.9 1 .35 1 "${mask}"', ' mcf "${diff_filt}" "${cc}" "${mask}" "${unw}" "${width}" 2 0 0 "${width}" "${lines}" 1 1 - "${r_ref}" "${a_ref}" 1', - ' rasdt_pwr "${unw}" "${mli1}" "${width}" 1 0 1 1 -3.14 3.14 1', + ' rasdt_pwr "${unw}" "${mli1}" "${width}" 1 0 1 1 -3.14 3.14 1 rmg.cm "${unw}.bmp" 1.0 0.35 8', ' ls -lh "${sim_unw}" "${diff}" "${diff_filt}" "${cc}" "${mask}" "${unw}"', ' } >"${LOG_DIR}/${pair}_diff_unwrap_common.log" 2>&1', "", @@ -8127,13 +11021,48 @@ class SbasInsarProductionService: def _build_command_manifest(self, run_manifest: dict[str, Any], stack_manifest: dict[str, Any]) -> dict[str, Any]: scenes = stack_manifest.get("scenes") or [] pair_network = stack_manifest.get("pair_network") or {} + sensor_family = self._normalize_sensor_family( + run_manifest.get("sensor_family") or stack_manifest.get("sensor_family") + ) + if sensor_family == "S1": + return { + "schema": "insar.gamma-command-manifest/v1", + "run_id": run_manifest["run_id"], + "engine": "gamma", + "processor_code": "gamma_ipta_sbas", + "profile_code": "s1_gamma_sbas", + "execution_enabled": False, + "reason_execution_disabled": "Sentinel-1 Gamma TOPS/SBAS execution scripts are not enabled yet.", + "stage_plan": [dict(item) for item in S1_GAMMA_SBAS_PLANNING_STEPS], + "inputs": { + "scene_count": len(scenes), + "scenes": [ + { + "date": scene.get("date"), + "scene_name": scene.get("scene_name"), + "source_format": scene.get("source_format"), + "source_wsl": scene.get("source_wsl"), + "orbit_wsl": scene.get("orbit_wsl"), + "relative_orbit": scene.get("relative_orbit"), + "orbit_direction": scene.get("orbit_direction"), + "imaging_mode": scene.get("imaging_mode"), + "polarization": scene.get("polarization"), + } + for scene in scenes + ], + "pair_count": len(pair_network.get("pairs") or []), + "pair_network_strategy": pair_network.get("strategy"), + }, + "expected_outputs": [], + "next_manual_review": "Verify Sentinel-1 subswath/burst policy and implement dedicated Gamma TOPS/SBAS scripts before enabling execution.", + } return { "schema": "insar.gamma-command-manifest/v1", "run_id": run_manifest["run_id"], "engine": "gamma", "processor_code": "gamma_ipta_sbas", - "execution_enabled": False, - "reason_execution_disabled": "The managed Gamma runner is intentionally not attached in this planning slice.", + "execution_enabled": True, + "reason_execution_disabled": None, "stage_plan": [dict(item) for item in GAMMA_STAGE_PLAN], "expert_document_steps": [dict(item) for item in GAMMA_SBAS_EXPERT_DOCUMENT_STEPS], "inputs": { @@ -8157,6 +11086,9 @@ class SbasInsarProductionService: def _build_run_card(self, run_dir: Path, manifest: dict[str, Any]) -> dict[str, Any]: stack = manifest.get("stack") or {} + stack_manifest = self._load_stack_manifest_for_run(run_dir, manifest) + identity = self._scene_identity_summary(stack_manifest.get("scenes") or []) + stack_from_manifest = stack_manifest.get("stack") or {} try: coverage = self._build_run_geographic_coverage(run_dir, manifest) except Exception: @@ -8169,6 +11101,9 @@ class SbasInsarProductionService: "workflow_code": manifest.get("workflow_code"), "processor_code": manifest.get("processor_code"), "engine_code": manifest.get("engine_code"), + "sensor_family": manifest.get("sensor_family") or self._normalize_sensor_family(stack.get("satellite")), + "profile_code": manifest.get("profile_code"), + "execution_enabled": manifest.get("execution_enabled", True), "stack_id": manifest.get("stack_id"), "scene_count": manifest.get("scene_count"), "pair_count": manifest.get("pair_count"), @@ -8176,11 +11111,17 @@ class SbasInsarProductionService: "discovery_mode": manifest.get("discovery_mode"), "aoi": manifest.get("aoi"), "common_overlap_ratio": manifest.get("common_overlap_ratio"), + "min_common_overlap_ratio": manifest.get("min_common_overlap_ratio"), + "scene_identity_hash": manifest.get("scene_identity_hash") or identity.get("scene_identity_hash"), + "scene_name_count": manifest.get("scene_name_count") or identity.get("scene_name_count"), + "scene_name_preview": manifest.get("scene_name_preview") or identity.get("scene_name_preview") or [], + "scene_names": manifest.get("scene_names") or identity.get("scene_names") or [], + "date_sequence_hash": manifest.get("date_sequence_hash") or identity.get("date_sequence_hash"), "platform": stack.get("satellite"), "relative_orbit": stack.get("relative_orbit"), "direction": stack.get("orbit_direction"), "polarization": stack.get("polarization"), - "center_bucket": stack.get("center_bucket"), + "center_bucket": stack.get("center_bucket") or stack_from_manifest.get("center_bucket"), "reference_date": stack.get("reference_date"), "date_start": coverage.get("date_start"), "date_end": coverage.get("date_end"), diff --git a/backend/app/services/spatial_service.py b/backend/app/services/spatial_service.py index 30e8912..d990e98 100644 --- a/backend/app/services/spatial_service.py +++ b/backend/app/services/spatial_service.py @@ -45,6 +45,7 @@ from .pairing_state_service import pairing_state_service PAIRING_POLICY_VERSION = "2026.05.raw-source.v2" PAIRING_WARNING_CANDIDATE_THRESHOLD = 3000 +PAIRING_ALL_STRATEGY_HARD_LIMIT = 20000 logger = logging.getLogger(__name__) @@ -144,6 +145,12 @@ class SpatialService: require_orbit_data=require_orbit_data, ) + if effective_params.strategy == "all" and len(candidate_pool) > PAIRING_ALL_STRATEGY_HARD_LIMIT: + raise RuntimeError( + f"全部配对命中 {len(candidate_pool)} 条候选边，超过系统一次性返回上限 " + f"{PAIRING_ALL_STRATEGY_HARD_LIMIT}。请改用 SBAS/Sequential 策略，或收紧 AOI、日期范围、重叠率。" + ) + if len(candidate_pool) > PAIRING_WARNING_CANDIDATE_THRESHOLD: warnings.append( f"候选配对数超过 {PAIRING_WARNING_CANDIDATE_THRESHOLD}（当前: {len(candidate_pool)}），建议收紧参数或缩小 AOI。" diff --git a/backend/tests/test_sbas_stack_discovery.py b/backend/tests/test_sbas_stack_discovery.py new file mode 100644 index 0000000..dd9b217 --- /dev/null +++ b/backend/tests/test_sbas_stack_discovery.py @@ -0,0 +1,164 @@ +from backend.app.services.sbas_insar_production_service import SbasInsarProductionService + + +def _scene(date, lon, lat, name=None, has_orbit=True): + width = 1.0 + height = 1.0 + return { + "scene_name": name or f"LT1B_MONO_SYC_STRIP1_000000_E{lon:.1f}_N{lat:.1f}_{date}_SLC_HH", + "date": date, + "satellite": "LT1B", + "satellite_mode": "MONO", + "receiving_station": "SYC", + "relative_orbit": "114", + "orbit_direction": "DESCENDING", + "imaging_mode": "STRIP1", + "polarization": "HH", + "center_lon": lon, + "center_lat": lat, + "center_bucket": f"E{lon:.1f}_N{lat:.1f}", + "has_orbit": has_orbit, + "bbox": { + "min_lon": lon - width / 2, + "min_lat": lat - height / 2, + "max_lon": lon + width / 2, + "max_lat": lat + height / 2, + }, + } + + +def test_common_overlap_subgroups_preserve_viable_date_keyed_stack(): + service = SbasInsarProductionService() + viable = [ + _scene("20240101", 129.20, 44.10), + _scene("20240201", 129.22, 44.08), + _scene("20240301", 129.18, 44.11), + _scene("20240401", 129.21, 44.09), + ] + distractors = [ + _scene("20240501", 129.80, 44.80), + _scene("20240601", 129.85, 44.78), + _scene("20240701", 129.82, 44.82), + ] + + groups = service._build_discovery_scene_groups( + observation_key="LT1B|MONO|SYC|114|DESCENDING|STRIP1|HH|E129.2_N44.1", + group_scenes=viable + distractors, + discovery_mode="strict", + require_orbits=True, + min_scenes=3, + min_common_overlap_ratio=0.30, + cluster_source="footprint_common_overlap", + ) + candidates = [ + service._build_stack_candidate( + group["scenes"], + min_scenes=3, + require_orbits=True, + discovery_mode="strict", + min_common_overlap_ratio=0.30, + ) + for group in groups + ] + ready = [candidate for candidate in candidates if candidate["status"] == "READY"] + + assert ready + assert any( + set(candidate["dates"]) == {scene["date"] for scene in viable} + and candidate["common_overlap_ratio"] >= 0.30 + for candidate in ready + ) + + +def test_common_overlap_subgroups_do_not_drop_same_date_viable_branch(): + service = SbasInsarProductionService() + viable = [ + _scene("20240101", 129.20, 44.10, name="viable_a"), + _scene("20240201", 129.22, 44.08, name="viable_b"), + _scene("20240301", 129.18, 44.11, name="viable_c"), + ] + same_date_distractors = [ + _scene("20240101", 130.00, 44.90, name="distractor_a"), + _scene("20240201", 130.02, 44.88, name="distractor_b"), + _scene("20240301", 129.98, 44.91, name="distractor_c"), + ] + + groups = service._build_discovery_scene_groups( + observation_key="LT1B|MONO|SYC|114|DESCENDING|STRIP1|HH|E129.2_N44.1", + group_scenes=viable + same_date_distractors, + discovery_mode="strict", + require_orbits=True, + min_scenes=3, + min_common_overlap_ratio=0.30, + cluster_source="footprint_common_overlap", + ) + + assert any( + {scene["scene_name"] for scene in group["scenes"]} == {"viable_a", "viable_b", "viable_c"} + for group in groups + ) + + +def test_candidate_identity_marks_same_dates_with_different_scene_names(): + service = SbasInsarProductionService() + first = [ + _scene("20240101", 129.20, 44.10, name="frame_a_20240101"), + _scene("20240201", 129.22, 44.08, name="frame_a_20240201"), + _scene("20240301", 129.18, 44.11, name="frame_a_20240301"), + ] + second = [ + _scene("20240101", 129.70, 44.60, name="frame_b_20240101"), + _scene("20240201", 129.72, 44.58, name="frame_b_20240201"), + _scene("20240301", 129.68, 44.61, name="frame_b_20240301"), + ] + candidates = [ + service._build_stack_candidate( + scenes, + min_scenes=3, + require_orbits=True, + discovery_mode="strict", + min_common_overlap_ratio=0.30, + ) + for scenes in (first, second) + ] + + service._annotate_stack_candidate_identity(candidates, existing_run_index={}) + + assert candidates[0]["date_sequence_hash"] == candidates[1]["date_sequence_hash"] + assert candidates[0]["scene_identity_hash"] != candidates[1]["scene_identity_hash"] + assert all(candidate["same_date_sequence_candidate_count"] == 2 for candidate in candidates) + assert all(candidate["same_date_sequence_distinct_scene_group_count"] == 2 for candidate in candidates) + assert all(candidate["same_date_sequence_has_different_scene_groups"] is True for candidate in candidates) + + +def test_candidate_identity_matches_existing_same_scene_run(): + service = SbasInsarProductionService() + scenes = [ + _scene("20240101", 129.20, 44.10, name="same_a"), + _scene("20240201", 129.22, 44.08, name="same_b"), + _scene("20240301", 129.18, 44.11, name="same_c"), + ] + candidate = service._build_stack_candidate( + scenes, + min_scenes=3, + require_orbits=True, + discovery_mode="strict", + min_common_overlap_ratio=0.30, + ) + scene_hash = service._scene_identity_summary(scenes)["scene_identity_hash"] + + service._annotate_stack_candidate_identity( + [candidate], + existing_run_index={ + scene_hash: [ + { + "run_id": "sbas_existing", + "status": "WORKFLOW_COMPLETED", + "stack_id": candidate["stack_id"], + } + ] + }, + ) + + assert candidate["scene_identity_hash"] == scene_hash + assert candidate["existing_same_scene_runs"][0]["run_id"] == "sbas_existing" diff --git a/deploy/wsl/profiles/gamma_env.sh b/deploy/wsl/profiles/gamma_env.sh index 1de7d76..8027257 100644 --- a/deploy/wsl/profiles/gamma_env.sh +++ b/deploy/wsl/profiles/gamma_env.sh @@ -62,6 +62,8 @@ done _gamma_profile_repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" _gamma_profile_pyint_dir="${_gamma_profile_repo_root}/third_party/PyINT/pyint" +_gamma_profile_python_dir="/home/administrator/miniconda3/envs/insar_wsl_v1/bin" +_gamma_profile_prepend_path "${_gamma_profile_python_dir}" _gamma_profile_prepend_path "${_gamma_profile_pyint_dir}" export PATH @@ -74,6 +76,7 @@ export PYTHONPATH=".:${GAMMA_HOME}${PYTHONPATH:+:${PYTHONPATH}}" unset _candidate unset _gamma_dir unset _gamma_profile_home +unset _gamma_profile_python_dir unset _gamma_profile_pyint_dir unset _gamma_profile_repo_root unset -f _gamma_profile_die diff --git a/docs/GAMMA_SBAS_EXPERT_CORRECT_IMPLEMENTATION_ROUTE_20260607.md b/docs/GAMMA_SBAS_EXPERT_CORRECT_IMPLEMENTATION_ROUTE_20260607.md new file mode 100644 index 0000000..8480aa5 --- /dev/null +++ b/docs/GAMMA_SBAS_EXPERT_CORRECT_IMPLEMENTATION_ROUTE_20260607.md @@ -0,0 +1,225 @@ +# Gamma SBAS 正确实现路线 2026-06-07 + +## 结论 + +当前系统的 Gamma SBAS 实现是错误模式，不是“不严格模式”。它把专家文档中的逐命令流程抽象成了另一条 Gamma/PyINT 混合链路，导致生成结果无法按专家命令行过程复核，也不能作为正确 SBAS 成果交付。 + +本次重构目标：默认 LT1 Gamma SBAS 只能执行专家文档 `LT1_GAMMA_SBAS_逐命令处理流程.docx` 对应的命令链。系统的职责是封装专家命令行流程，而不是替换成看起来相似的工具链。 + +## 错误来源 + +| 专家文档要求 | 当前错误实现 | 重构要求 | +| --- | --- | --- | +| `par_LT1_SLC` 后执行 `ORB_filt_spline.py` | 当前使用 `LT1_precision_orbit.py` 桥接 | 改为专家文档记录的 `ORB_filt_spline.py`，保留轨道输入审计 | +| `create_offset/init_offset_orbit/init_offset/offset_pwr/offset_fit/SLC_interp` | 当前使用 `SLC_coreg.py --init_offset` | 删除默认流程中的 `SLC_coreg.py`，逐场景展开专家命令 | +| `dem_import/fill_gaps/gc_map2/pixel_area/create_diff_par/offset_pwrm/offset_fitm/gc_map_fine/geocode/geocode_back` | 当前复用 PyINT DEM cache 并使用 `gc_map1` | DEM 从源 GeoTIFF 导入，必须覆盖完整 stack bbox，不允许只覆盖中心点 | +| `base_calc/base_plot/mk_diff_2d` | 当前使用 `phase_sim_orb/SLC_diff_intf` | 改为 `mk_diff_2d` | +| `mk_adf_2d/ave_image/rascc_mask/mk_unw_2d` | 当前使用 `adf/cc_wave/mcf` | 改为专家文档的 `mk_adf_2d` 和两次 `mk_unw_2d` | +| `quad_fit/quad_sub/atm_mod_2d/fill_gaps/atm_sim_2d/sub_phase` | 当前部分接近但输入来自错误解缠链 | 保留命令类型，输入统一改为专家链路产物 | +| 三轮 `mb` 加复数转换/`unw_model` | 当前一轮 `mb` 后直接 `ts_rate` | 改为专家文档三轮反演；本机 Gamma 环境未提供 `unw_to_cpx`，实际使用 `real_to_cpx - <unw> <cpx> <width> 1` 执行同一位置的实数到复数转换 | +| `replace_values/mask_data/dispmap/ts_rate/geocode_back/data2geotiff/disp_prt_2d` | 当前 Python 转 LOS 后发布多套派生产品 | 改为专家文档输出链，派生产品只能在专家产物之后追加，不能替代主产品 | + +## 正确流程 + +### 01 Workspace + +创建专家文档目录结构： + +- `RAW` +- `SLC` +- `dem` +- `rslc_prep` +- `mli_dir` +- `diff_dir` +- `diff1_dir` +- `sbas` +- `publish` +- `logs` +- `scripts` +- `state` + +验收条件：目录存在，场景清单和源文件路径写入 manifest。 + +### 02 Import LT1 SLC + +每景执行： + +```bash +par_LT1_SLC <scene>.tiff <scene>.meta.xml <date>.slc.par <date>.slc 0 +cp <date>.slc.par <date>.slc.par.orig +ORB_filt_spline.py <date>.slc.par.orig <date>.slc.par --ignore_start 3 --ignore_end 17 --degree 5 +SLC_corners <date>.slc.par +disSLC <date>.slc <width> ... +dismph_fft <date>.slc <width> ... +``` + +验收条件：每景 `.slc/.slc.par/.slc.par.orig` 存在，`SLC_tab` 行数等于场景数。 + +### 03 Reference MLI + +参考景执行： + +```bash +multi_look <ref>.slc <ref>.slc.par <ref>_<rlks>_<azlks>.mli <ref>_<rlks>_<azlks>.mli.par <rlks> <azlks> +grep range_samples <ref>.mli.par +grep azimuth_lines <ref>.mli.par +ras_dB <ref>.mli <width> ... +SLC_corners <ref>.mli.par +``` + +验收条件：参考 MLI、参数文件、宽高审计和 BMP 浏览图存在。 + +### 04 DEM Lookup + +执行专家 DEM 链： + +```bash +dem_import <dem>.tif SRTM.dem SRTM.dem.par ... +fill_gaps SRTM.dem <dem_width> SRTM_dem_fill +gc_map2 <ref>.mli.par SRTM.dem.par SRTM_dem_fill <ref>_seg.dem_par <ref>_seg.dem <ref>.lt ... +pixel_area <ref>.mli.par <ref>_seg.dem_par <ref>_seg.dem <ref>.lt ... +create_diff_par <ref>.mli.par - <ref>.diff_par 1 0 +offset_pwrm <ref>.gamma0 <ref>.mli <ref>.diff_par ... +offset_fitm <ref>.offs <ref>.snr <ref>.diff_par ... +gc_map_fine <ref>.lt <dem_width> <ref>.diff_par <ref>.lt_fine 1 +geocode <ref>.lt_fine <ref>_seg.dem <dem_width> <ref>.hgt <mli_width> <mli_lines> +geocode_back <ref>.mli <mli_width> <ref>.lt_fine <ref>.geo <dem_width> <dem_lines> 5 0 +``` + +验收条件：`<ref>.lt_fine`、`<ref>.hgt`、`<ref>_seg.dem_par` 存在；DEM 覆盖必须包含完整 stack bbox。 + +### 05 Coreg Prep + +执行： + +```bash +cp SLC/dates rslc_prep/dates +cp <ref>.slc <ref>.rslc +cp <ref>.slc.par <ref>.rslc.par +``` + +验收条件：参考 RSLC 和 `rslc_tab` 初始化完成。 + +### 06 Coregister Scenes + +非参考景逐景执行： + +```bash +create_offset <ref>.rslc.par <date>.slc.par <ref>_<date>.off 1 +init_offset_orbit <ref>.rslc.par <date>.slc.par <ref>_<date>.off +init_offset <ref>.rslc <date>.slc <ref>.rslc.par <date>.slc.par <ref>_<date>.off <rlks> <azlks> +offset_pwr <ref>.rslc <date>.slc <ref>.rslc.par <date>.slc.par <ref>_<date>.off ... +offset_fit <ref>_<date>.offs <ref>_<date>.snr <ref>_<date>.off ... +SLC_interp <date>.slc <ref>.rslc.par <date>.slc.par <ref>_<date>.off <date>.rslc <date>.rslc.par +``` + +验收条件：每景 `.rslc/.rslc.par/.off` 存在，`rslc_tab` 行数等于场景数。 + +### 07 RMLI Average + +执行： + +```bash +mk_mli_all rslc_tab . <rlks> <azlks> 1 1.0 0.4 mli.ave +grep range_samples mli.ave.par +grep azimuth_lines mli.ave.par +ras_dB mli.ave <width> ... +``` + +验收条件：`mli.ave/mli.ave.par/mli.ave.bmp` 存在。 + +### 08 Diff Network + +执行： + +```bash +base_calc rslc_tab <ref>.rslc.par bprep_file itab 1 1 <bmin> <bmax> <tmin> <tmax> - +base_plot rslc_tab <ref>.rslc.par itab bprep_file 1 +mk_diff_2d rslc_tab itab 0 <ref>.hgt - mli.ave mli_dir . <rlks> <azlks> 3 1 1 0 -u +``` + +验收条件：`itab`、`bprep_file`、每对 `.diff/.diff.bmp` 存在。 + +### 09 Filter Unwrap + +执行： + +```bash +mk_adf_2d rslc_tab itab mli.ave . 5 0.6 32 8 -u +ave_image cc.list <width> mean.cc +rascc_mask mean.cc - <width> 1 1 - 1 1 <threshold> +mk_unw_2d rslc_tab itab mli.ave . <threshold> 0 1 1 1 1 <r_seed> <a_seed> 1 -u +mk_unw_2d rslc_tab itab mli.ave . - - 1 1 1 1 <r_seed> <a_seed> 1 mean.cc_mask.bmp -u +``` + +验收条件：每对 `.adf.diff/.adf.cc/.adf.unw` 存在，`mean.cc_mask.bmp` 存在。 + +### 10 Detrend ATM + +每对执行： + +```bash +create_diff_par <pair>.off <pair>.off <pair>.diff_par 0 0 +quad_fit <pair>.adf.unw <pair>.diff_par 5 5 - - 3 <pair>.unw_linear +quad_sub <pair>.adf.unw <pair>.diff_par <pair>.unw_sub_linear 0 0 +atm_mod_2d <pair>.unw_sub_linear <ref>.hgt <pair>.adf.cc <pair>.diff_par - 0 <pair>.a0 <pair>.a1 ... +fill_gaps <pair>.a0 <model_width> <pair>.a0_fill ... +fill_gaps <pair>.a1 <model_width> <pair>.a1_fill ... +atm_sim_2d <pair>.diff_par <ref>.hgt <pair>.a0_fill <pair>.a1_fill <pair>.atm_model +sub_phase <pair>.unw_sub_linear <pair>.atm_model <pair>.diff_par <pair>.unw.atmsub 0 +``` + +验收条件：`unw_atmsub_tab` 行数等于 `itab` 行数。 + +### 11 SBAS Inversion + +执行： + +```bash +mb unw_atmsub_tab RMLI_tab itab - itab_ts ras/diff1 1 diff1.sigma_ts 1 - <r_ref> <a_ref> 15 15 0.0 mli.ave.par +real_to_cpx - <pair>.unw.atmsub <pair>.unw.atmsub.cpx <width> 1 +unw_model <pair>.unw.atmsub.cpx <pair>.unw.atmsub_sim <pair>.unw.atmsub_1 <width> <r_ref> <a_ref> +mb unw.atmsub_1_tab RMLI_tab itab - itab_ts ras/diff2 1 diff2.sigma_ts 0 - <r_ref> <a_ref> 15 15 0.0 mli.ave.par +mb final_unw_tab RMLI_tab itab - itab_ts ras/diff 0 diff.sigma_ts 0 - <r_ref> <a_ref> 15 15 0.5 mli.ave.par +``` + +验收条件：`ras/diff*.tab`、`diff.sigma_ts`、`itab_ts` 存在。 + +### 12 Outputs Points + +执行： + +```bash +replace_values diff.sigma_ts 0.5 0.0 diff.sigma_ts.masked <width> 1 2 0 +mask_data ras/diff_<date> <width> ras/diff_<date>.masked diff.sigma_ts.masked.bmp 0 +dispmap ras/<date>.disp.phase - mli.ave.par - ras/<date>.disp 0 0 +ts_rate disp.TS_tab RMLI_tab itab_ts - los_def_rate los_def_const los_def_sigma 0 +geocode_back los_def_rate <width> <ref>.lt_fine geo_los_def_rate <dem_width> <dem_lines> 5 0 +data2geotiff <ref>_seg.dem_par geo_los_def_rate 2 geo_los_def_rate.tif +disp_prt_2d disp_geo.TS_tab RMLI_tab itab_ts - 3 disp_point.txt - geo_los_def_rate geo_diff.sigma_ts items.txt disp_tab.txt 3 1 0 +``` + +验收条件：`geo_los_def_rate.tif` 和点时序输出来自专家输出链。 + +## 工程改造路线 + +1. 更新 stage plan：默认 `lt1_gamma_sbas` 的工具清单改为专家命令，删除 `SLC_coreg.py/gc_map1/SLC_diff_intf/mcf/单轮 mb` 作为默认阶段描述。 +2. 更新专家步骤状态：`implementation_status` 不再使用 `implemented_bridge`，只有 `implemented`、`planned`、`blocked`。 +3. 重写 `_materialize_workflow_scripts`：生成 12 个专家脚本，禁止复制旧桥接脚本作为专家步骤。 +4. 增加命令审计：从每个脚本提取命令，和专家步骤允许命令集比对；发现旧错误命令或缺少核心命令时，任务不能进入完成状态。 +5. 修正 DEM 选择：DEM 必须覆盖完整 stack bbox；只覆盖中心点不能通过。 +6. 重构各阶段执行：`prepare/execute_coregistration`、`prepare/execute_rdc_dem`、`prepare/execute_interferograms`、`prepare/execute_ipta_timeseries`、`prepare/execute_publish_products` 逐步改为调用对应专家脚本和专家输出路径。 +7. 修正完成判定：`WORKFLOW_COMPLETED` 必须同时满足专家脚本全部完成、命令审计通过、关键输出来自专家链路。 +8. 标记历史产物：旧错误链路产物不能被 catalog 当作有效 SBAS 成果。 + +## 本轮编码边界 + +本轮先完成默认模式的结构性改造： + +- 文档化正确流程和验收条件。 +- 修改 stage plan 和 manifest 口径。 +- 生成专家 12 步脚本，不再复制旧桥接脚本。 +- 增加命令审计，阻断旧错误命令链。 +- 修正 DEM 覆盖判定。 + +后续继续把每个 `execute_*` 阶段从旧阶段脚本迁移到 12 步专家脚本。迁移过程中只允许使用专家文档命令；任何替代命令必须显式标记为未启用，不能进入生产完成状态。 diff --git a/docs/GAMMA_SBAS_EXPERT_WORKFLOW_AUDIT_20260608.md b/docs/GAMMA_SBAS_EXPERT_WORKFLOW_AUDIT_20260608.md new file mode 100644 index 0000000..c0c0c1b --- /dev/null +++ b/docs/GAMMA_SBAS_EXPERT_WORKFLOW_AUDIT_20260608.md @@ -0,0 +1,84 @@ +# Gamma SBAS expert workflow audit 2026-06-08 + +## Scope + +Audited run: + +- `run_id`: `sbas_a5d51de3808a` +- workflow manifest: `backend/runtime/sbas_insar_production/runs/sbas_a5d51de3808a/manifest.json` +- command manifest: `backend/runtime/sbas_insar_production/runs/sbas_a5d51de3808a/gamma_command_manifest.json` +- scripts: `backend/runtime/sbas_insar_production/runs/sbas_a5d51de3808a/scripts/*.sh` +- result catalog product: `gamma_sbas_sbas_a5d51de3808a` + +The audit checks whether the production path follows `LT1_GAMMA_SBAS_逐命令处理流程.docx` as represented by `docs/GAMMA_SBAS_EXPERT_CORRECT_IMPLEMENTATION_ROUTE_20260607.md`. + +## Result + +The Gamma processing chain is now aligned with the expert-document workflow at the command-chain level. The run completed all 12 expert workflow steps and the result catalog registers only the expert-chain products as the primary SBAS result. + +This is not the old mixed Gamma/PyINT path. The old invalid run and old invalid catalog record were removed. + +## Step Mapping + +| Step | Expert workflow requirement | Current run evidence | Status | +| --- | --- | --- | --- | +| 01 Workspace | Expert directories: `RAW`, `SLC`, `dem`, `rslc_prep`, `mli_dir`, `diff_dir`, `diff1_dir`, `sbas`, `publish`, `logs`, `scripts`, `state` | All directories exist under the run root | Pass | +| 02 Import LT1 SLC | `par_LT1_SLC`, copy original par, `ORB_filt_spline.py`, `SLC_corners`, browse checks | Script uses `par_LT1_SLC` and `ORB_filt_spline.py`; SLC products exist | Pass | +| 03 Reference MLI | `multi_look`, width/line checks, `ras_dB`, `SLC_corners` | Command audit passed with `multi_look`, `ras_dB`, `SLC_corners`; `mli.ave` products exist | Pass | +| 04 DEM Lookup | `dem_import`, `fill_gaps`, `gc_map2`, `pixel_area`, `create_diff_par`, `offset_pwrm`, `offset_fitm`, `gc_map_fine`, `geocode`, `geocode_back` | Command audit passed; `20241007.lt_fine`, `20241007.hgt`, `20241007_seg.dem_par` exist | Pass | +| 05 Coreg Prep | Copy reference SLC to RSLC prep and initialize `rslc_tab` | `rslc_prep/rslc_tab` exists | Pass | +| 06 Coregister Scenes | `create_offset`, `init_offset_orbit`, `init_offset`, `offset_pwr`, `offset_fit`, `SLC_interp` | Command audit passed with all required commands | Pass | +| 07 RMLI Average | `mk_mli_all`, width/line checks, `ras_dB` | Command audit passed; `mli.ave`, `mli.ave.par` exist | Pass | +| 08 Diff Network | `base_calc`, `base_plot`, `mk_diff_2d` | Command audit passed; `itab`, `bprep_file` exist | Pass | +| 09 Filter Unwrap | `mk_adf_2d`, `ave_image`, `rascc_mask`, two `mk_unw_2d` passes | Command audit passed with required commands | Pass | +| 10 Detrend ATM | `create_diff_par`, `quad_fit`, `quad_sub`, `atm_mod_2d`, `fill_gaps`, `atm_sim_2d`, `sub_phase` | Command audit passed; `unw_atmsub_tab` exists | Pass | +| 11 SBAS Inversion | Three `mb` passes with complex conversion and `unw_model` correction | Script runs three `mb` calls, `real_to_cpx -`, `unw_model`; `diff.sigma_ts`, `itab_ts`, `ras/diff.tab` exist | Pass with documented compatibility note | +| 12 Outputs Points | `replace_values`, `mask_data`, `dispmap`, `ts_rate`, `geocode_back`, `data2geotiff`, `disp_prt_2d` | Command audit passed; `geo_los_def_rate.tif`, `geo_los_def_rate_rgb.tif`, `items.txt`, `disp_point.txt` exist | Pass | + +## Compatibility Note + +The expert route originally referenced `unw_to_cpx`. The installed Gamma environment used for this run does not provide `unw_to_cpx`, and the implementation uses: + +```bash +real_to_cpx - <pair>.unw.atmsub <pair>.unw.atmsub.cpx <width> 1 +``` + +This is used only at the same workflow point to create the complex input consumed by `unw_model`. The command manifest, production code, and audit route have been updated to state this explicitly. + +## Primary Products + +The valid expert-chain products are: + +- `publish/geotiff/geo_los_def_rate.tif` +- `publish/geotiff/geo_los_def_rate_rgb.tif` +- `publish/points/items.txt` +- `publish/points/disp_point.txt` + +System-generated visualization derivatives are: + +- `publish/geotiff/geo_los_def_rate_rgb_preview.png` +- `publish/geotiff/geo_los_def_rate_hls_colorbar.png` +- `publish/monitor_points/expert_point_001_timeseries.png` +- `publish/monitor_points/expert_point_001_timeseries.csv` +- `publish/monitor_points/expert_point_001_metadata.json` + +These derivatives are not substitutes for the expert Gamma products. They are catalog and UI display assets derived after the expert outputs exist. + +## Catalog Status + +After cleanup and rebuild: + +- SBAS catalog run count: 1 +- registered product: `gamma_sbas_sbas_a5d51de3808a` +- product status: `READY` +- health status: `OK` +- issue count: 0 + +## Residual Risk + +The command chain now follows the expert workflow. Remaining validation should be scientific review of parameter choices and result interpretation, especially: + +- reference window size and selected reference point +- baseline network selection and itab approval policy +- deformation sign convention for business reporting +- whether the expert accepts `real_to_cpx -` as the local Gamma equivalent of the documented complex conversion step diff --git a/docs/GAMMA_SBAS_RUNTIME_OBSERVATIONS_20260611.md b/docs/GAMMA_SBAS_RUNTIME_OBSERVATIONS_20260611.md new file mode 100644 index 0000000..a97739d --- /dev/null +++ b/docs/GAMMA_SBAS_RUNTIME_OBSERVATIONS_20260611.md @@ -0,0 +1,99 @@ +# Gamma SBAS runtime observations - 2026-06-11 + +## 1. Interactive display window blocks unattended runs + +Observed during run: + +```text +run_id: sbas_c65c7486f76e +workflow: SBAS_GAMMA_WORKFLOW +stage: 02_import_lt1_slc.sh +date observed: 2026-06-11 +``` + +User reported a Gamma/WSL display-style window during execution. The window only had a `Close` button. After the user clicked `Close`, the workflow continued. + +Runtime checks later showed the same run was still in `02_import_lt1_slc.sh`, processing LT-1 scenes with `par_LT1_SLC`. The first two scenes completed: + +```text +20230624 -> completed SLC import +20230726 -> completed SLC import +20230819 -> running at 2026-06-11 11:18 Asia/Shanghai +``` + +There was a large delay between the end of `20230726` and the start of `20230819`. This is consistent with a blocking interactive display process. The likely trigger is the browse/preview commands in the import step: + +```bash +disSLC ... +dismph_fft ... +``` + +These commands are useful for expert visual inspection, but they are not acceptable as blocking UI windows in a managed background production workflow. + +The same window was reported again later in the same run while the import step was still active. This confirms the issue is repeatable and should be treated as a workflow design defect, not a one-off operator/environment event. + +For the active run, do not modify the generated shell script in place unless the operator explicitly chooses to skip the remaining interactive preview commands. A targeted intervention is possible only when the active child process is a display command such as `disSLC` or `dismph_fft`; terminating `par_LT1_SLC` or the parent bash process would risk corrupting or interrupting the import product. + +## 2. Current operational decision + +The operator changed the decision after the window repeated: stop this run, clear the run records, and audit/fix the production workflow before re-running. + +Completed cleanup: + +```text +run_id: sbas_c65c7486f76e +processes: stopped +run directory: deleted +system_tasks deleted: 1 +system_jobs deleted: 1 +catalog products deleted: 0 +``` + +## 3. Required follow-up after this run + +The fix must preserve expert-command equivalence for calculation commands while removing interactive blocking behavior. + +Important distinction: + +- `disSLC` and `dismph_fft` are expert command-line manual QC/display commands. +- They are not required numerical SBAS calculation steps. +- In a managed backend production workflow they must not open a UI window. +- The system should expose reviewable assets through generated rasters/previews and frontend visualization instead. + +Implementation requirements: + +1. Keep expert preview/browse products where they are generated by non-interactive raster/export commands. +2. Do not execute expert manual display commands in default backend production. +3. Never require an operator to click `Close` for a backend job to continue. +4. Add a workflow-level guard that detects display commands likely to open an interactive window. +5. Update run status reporting so the frontend shows the actual executing step instead of leaving `02_import_lt1_slc` as `PENDING` while the WSL process is running. + +Candidate code area: + +```text +backend/app/services/sbas_insar_production_service.py + _write_import_lt1_slc_script / 02_import_lt1_slc.sh generation +``` + +Related generated commands observed in the current script: + +```bash +disSLC "${slc}" "${width}" 1 0 - - 1 1 "${SLC_DIR}/${date}.slc.bmp" || true +dismph_fft "${slc}" "${width}" 1 0 - - 1 1 "${SLC_DIR}/${date}.slc_fft.bmp" || true +``` + +Implementation decision: + +- Remove `disSLC` and `dismph_fft` from generated unattended production scripts. +- Keep them in expert-document metadata as `manual_qc_tools`. +- Add script-write and command-audit guards so future generated scripts fail before execution if blocking display tools are present. +- Keep `ras*` commands that write explicit output BMP/GeoTIFF preview files; those are batch product-generation commands, not interactive display windows. + +## 4. Acceptance criteria + +After the fix: + +- A full Gamma SBAS run can complete unattended from the web UI. +- No Gamma display window should require manual close. +- Preview BMP/PNG artifacts are still generated for inspection when feasible. +- The step status file and frontend task status reflect the active step during long-running shell scripts. diff --git a/docs/GAMMA_SBAS_ZERO_RATE_VALID_VISUALIZATION_DESIGN_20260608.md b/docs/GAMMA_SBAS_ZERO_RATE_VALID_VISUALIZATION_DESIGN_20260608.md new file mode 100644 index 0000000..e148e43 --- /dev/null +++ b/docs/GAMMA_SBAS_ZERO_RATE_VALID_VISUALIZATION_DESIGN_20260608.md @@ -0,0 +1,68 @@ +# Gamma SBAS 0 速率无效值与结果展示设计 + +## 当前口径 + +专家 Gamma SBAS 核心速率产品： + +- `publish/geotiff/geo_los_def_rate.tif` +- 原始单位按专家命令解释为 `m/yr` +- 前端展示统一换算为 `mm/yr` + +当前处理口径按用户确认执行：`0` 速率按无效值处理。 + +原因是当前 GeoTIFF 的覆盖区外和部分背景区域都可能被写为 0；在没有专家显式稳定区 mask 或质量 mask 前，把 0 解释为稳定值会把背景大量纳入统计，导致 P05/P50/P95 等摘要失真。因此本阶段先按无效值处理 0，后续若专家确认稳定区 mask，再调整。 + +## 有效性规则 + +1. 有效像元必须位于专家 RGB 覆盖区内。 +2. 有效像元必须是有限值：不是 `NaN`、`Inf`、`-Inf`。 +3. 有效像元必须满足 `geo_los_def_rate != 0`。 +4. `0` 不参与统计、不参与纯速率图渲染，显示为透明背景。 +5. GeoTIFF 元数据 `nodata=0.0` 与当前口径一致，但后端仍显式记录该规则，避免前端误解。 + +## 后端实现 + +### `_build_expert_gamma_primary_geotiff_stats` + +- 使用 `masked=False` 读取速率，避免 Rasterio 隐式规则不可见。 +- 使用 `geo_los_def_rate_rgb.tif` 非黑像元推断专家覆盖区。 +- 有效条件：`coverage & finite & (value != 0.0)`。 +- 输出字段： + - `zero_is_valid: false` + - `validity_rule: "expert_rgb_coverage_finite_nonzero_values"` + - `coverage_mask_source` + - `zero_count` + - `nonzero_count` + - `metadata_nodata_applied: true` + +### `_build_gamma_hls_rate_preview` + +- 使用同一覆盖区规则； +- `0` 速率透明； +- 非零速率按 Gamma `hls.cm` 和专家固定范围 `[-0.08, 0.08] m/yr` 着色。 + +## 前端展示 + +1. `LOS 速率纯色图` 文案说明： + - 由 `geo_los_def_rate.tif` 派生； + - 不叠加强度图或底图； + - 0 速率按无效值透明处理。 +2. 统计摘要显示： + - `0 速率：按无效值处理` + - `有效规则：专家覆盖区内有限非零值` + +## 后续增强 + +1. 若专家提供显式有效 mask 或稳定区定义，可将本规则切换为“mask 内 0 稳定、mask 外 0 无效”。 +2. 多点时序、速率直方图、剖面线和质量图应基于同一有效性规则生成。 +3. 所有派生图必须标注规则来源，避免与专家原始计算产物混淆。 + +## 验证标准 + +1. 后端语法检查通过。 +2. 纯速率图非透明比例应接近非零有效像元比例。 +3. API 产品详情： + - `zero_is_valid=false` + - `validity_rule="expert_rgb_coverage_finite_nonzero_values"` + - `valid_count == nonzero_count` +4. 前端构建通过。 diff --git a/docs/GLOBAL_TASK_STATUS_LOCK_REDESIGN_20260612.md b/docs/GLOBAL_TASK_STATUS_LOCK_REDESIGN_20260612.md new file mode 100644 index 0000000..89073a9 --- /dev/null +++ b/docs/GLOBAL_TASK_STATUS_LOCK_REDESIGN_20260612.md @@ -0,0 +1,696 @@ +# 全局任务状态与界面锁重构设计 + +最后更新：2026-06-12 + +## 1. 背景 + +当前前端存在一个“全局界面锁”机制：只要系统检测到仍有活跃任务，并且该任务不在前端非阻塞白名单内，就会把整个系统切到全局锁定状态，并弹出 `ActiveTasksOverlay` 全屏遮罩。 + +这个设计在早期可以避免用户在 ENVI / SARscape / IDL 长任务执行期间重复点击、切换入口或提交冲突任务。但随着系统扩展到 D-InSAR、多引擎生产、SBAS-InSAR、数据接入、资产扫描、洪涝分析、AI 诊断等多个相对独立的任务域，全局锁已经过粗： + +- SBAS Gamma / LandSAR SBAS 已经证明长任务可以不锁全局界面，只在模块内展示状态。 +- 后端 `task_service.create_task()` 已经按 `task_type` 做同类型 `PENDING/RUNNING` 互斥。 +- 很多任务只是扫描、解包、目录重建或产物刷新，不应阻断其他业务操作。 +- 全屏遮罩会遮住当前功能页，用户看不到任务详情，也无法继续浏览结果或管理其他独立任务。 + +因此，本设计将“全局界面锁”降级为“全局任务状态中心”，并要求每个功能模块设计自己的任务状态面板和局部操作约束。 + +## 2. 当前实现审计 + +### 2.1 前端锁入口 + +核心文件： + +- `frontend/src/hooks/useGlobalTaskControl.js` +- `frontend/src/hooks/useDinsarOperations.js` +- `frontend/src/components/ActiveTasksOverlay.jsx` +- `frontend/src/components/app/AppOverlays.jsx` +- `frontend/src/ProductionWorkspace.jsx` + +当前逻辑： + +1. `useGlobalTaskControl` 订阅 `/tasks/active` 或 SSE `/tasks/active/stream`。 +2. 前端维护 `pendingTaskIds` 和 `nonBlockingTaskIds`。 +3. 活跃任务中只要存在不属于 `NON_BLOCKING_TASK_TYPES` 的任务，就设置 `isGlobalLocked=true`。 +4. `AppOverlays` 根据 `isGlobalLocked` 渲染 `ActiveTasksOverlay`。 +5. `App.ensureCanOperate()` 会因为 `isGlobalLocked` 拒绝写操作。 + +### 2.2 当前非阻塞任务 + +当前白名单包括： + +- `UNPACK_ARCHIVES` +- `UNPACK_SENTINEL1` +- `GF3_UNPACK` +- `GF3_SARSCAPE_PRODUCE` +- `GF3_SARSCAPE_SYNC` +- `GF3_SARSCAPE_CLEAN` +- `SCAN_ASSET_INVENTORY` +- `COPY_DATA` +- `SBAS_GAMMA_WORKFLOW` +- `SBAS_LANDSAR_WORKFLOW` +- `SBAS_COREGISTRATION` +- `SBAS_RDC_DEM` +- `SBAS_INTERFEROGRAMS` +- `SBAS_IPTA_TIMESERIES` +- `REBUILD_SBAS_INSAR_CATALOG` + +这些任务已经以“不锁全局界面”的方式运行。 + +### 2.3 当前仍会锁全局界面的任务 + +主要原因通常不是任务本身必须锁，而是提交时没有传 `taskType` 或 `nonBlocking: true`。 + +已审计到的典型入口： + +- D-InSAR 生产任务 + - `ProductionWorkspace.handleDinsarRunQueued()` + - `DinsarProductionPanel.handleSubmit()` +- D-InSAR 结果扫描 + - `ProductionWorkspace.handleDinsarProductQueued()` + - `DinsarProductsPanel.handleScan()` +- IDL Import / IDL DInSAR + - `IDLAutomationPanel` +- AI 训练、全量预测、AI 诊断 + - `useDinsarOperations.handleTrainAi()` + - `useDinsarOperations.handlePredictAll()` + - `useDinsarOperations.handleAnalyzeResult()` +- 灾害点同步 + - `HazardPointPanel` +- 水体 / 洪涝检测任务 + - `WaterMonitorPanel` +- 部分数据监控任务 + - `GF3_BATCH_PROCESS` + - `SCAN_DATA` + - 手动 LT-1 / 精轨 / GF3 扫描 +- 预览缓存重建 + - `App.rebuildRadarPreviewCache()` 通过 `handleTaskStart(null, ...)` 触发临时全局锁 + +### 2.4 后端已有保护 + +后端 `backend/app/services/task_service.py` 在 `create_task()` 中已做同类型任务互斥： + +- 查询相同 `task_type` 且状态为 `PENDING` / `RUNNING` 的任务。 +- 如果存在，直接返回任务冲突错误。 + +这说明前端全局锁不是唯一安全机制。真正的任务并发保护应继续下沉到后端，并从 `task_type` 扩展到更精确的资源锁。 + +## 3. 设计目标 + +1. 取消“任意阻塞任务遮住整个系统”的交互模式。 +2. 全局层只负责展示所有活跃任务、最近任务、失败任务和快捷入口。 +3. 每个功能模块拥有自己的任务状态面板，只展示和本功能相关的任务。 +4. 默认任务不锁全局界面。 +5. 需要互斥的场景由后端资源锁保证，而不是靠前端遮罩保证。 +6. 前端只做局部禁用：禁用同一个功能里会造成重复提交或破坏状态的按钮。 +7. 保留管理员取消任务能力，但从“强制解锁”改为“取消/中止任务”。 + +## 4. 非目标 + +本设计不要求一次性重写所有任务系统。 + +不在本阶段处理： + +- 重做后端任务表结构。 +- 重写 job worker。 +- 改变现有任务 API 的基本返回格式。 +- 一次性把所有历史任务类型改名。 +- 删除所有旧 overlay 代码。 + +本设计优先保证渐进迁移。 + +## 5. 新架构概览 + +```text +后端任务系统 +├─ SystemTaskORM / TaskLogORM +├─ SystemJobORM +├─ task_type 同类型互斥 +└─ 后续扩展：resource_lock_key / resource_scope + +前端任务状态层 +├─ 全局任务状态中心 +│ ├─ 展示所有活跃任务 +│ ├─ 展示最近失败 / 完成任务 +│ ├─ 支持跳转到所属功能 +│ └─ 不遮住全系统 +├─ 功能级任务面板 +│ ├─ D-InSAR 生产任务面板 +│ ├─ SBAS-InSAR 生产任务面板 +│ ├─ 数据接入任务面板 +│ ├─ 资产库存任务面板 +│ ├─ 洪涝分析任务面板 +│ └─ AI / 灾害点任务面板 +└─ 局部操作约束 + ├─ 同类任务运行中，禁用同类提交按钮 + ├─ 目录重建中，禁用同一目录重建按钮 + └─ 其他功能仍可浏览和操作 +``` + +## 6. 全局任务状态中心 + +### 6.1 职责 + +全局任务状态中心只做观察和导航，不做全局阻断。 + +职责： + +- 汇总 `/tasks/active`。 +- 汇总最近任务 `/tasks/recent`。 +- 显示任务类型、状态、进度、开始时间、最近消息。 +- 支持按功能域筛选： + - 数据接入 + - D-InSAR + - SBAS-InSAR + - 洪涝 + - AI + - 运维 +- 支持点击任务跳转到所属功能页。 +- 支持查看任务日志。 +- 支持管理员取消任务。 + +### 6.2 交互形态 + +建议替换现有 `ActiveTasksOverlay`： + +- 不再使用全屏遮罩。 +- 使用顶部状态入口或右下角任务抽屉。 +- 有活跃任务时显示小型状态指示。 +- 点击后打开任务中心抽屉或弹层。 +- 弹层不阻止用户关闭和继续使用系统。 + +### 6.3 状态文案 + +旧文案： + +```text +为了保证数据一致性，耗时任务执行期间 UI 已锁定。 +``` + +应替换为： + +```text +后台任务正在执行。你可以继续使用其他功能；同类任务的重复提交会由系统自动限制。 +``` + +## 7. 功能级任务面板 + +每个功能页应自行展示本功能任务状态。这样用户在当前业务上下文里能直接看到“我刚提交的任务跑到哪一步”，而不是被全局遮罩挡住。 + +### 7.1 通用组件建议 + +新增通用组件： + +- `TaskStatusPanel` +- `TaskLogPanel` +- `TaskProgressRow` +- `TaskStatusBadge` +- `useTaskMonitor` + +建议参数： + +```ts +type TaskStatusPanelProps = { + title: string; + taskTypes?: string[]; + taskTypePrefixes?: string[]; + taskIds?: string[]; + showRecent?: boolean; + compact?: boolean; + onTaskClick?: (task) => void; +}; +``` + +`useTaskMonitor` 负责： + +- 按 `taskTypes` / `taskTypePrefixes` 查询活跃任务。 +- 轮询或复用全局 SSE 数据。 +- 拉取指定任务日志。 +- 返回 `activeTasks`、`recentTasks`、`latestTask`、`isBusy`。 + +### 7.2 D-InSAR 生产 + +任务类型： + +- `IDL_RUN_DINSAR` +- `ISCE2_RUN` +- `PYINT_RUN` +- `LANDSAR_RUN` + +面板位置： + +- `DinsarProductionPanel` 右侧或提交区下方。 + +局部约束： + +- 如果同一 engine 的任务正在运行，禁用同 engine 再提交。 +- 其他 engine 是否允许并行应由后端资源锁决定。 +- 用户仍可查看历史 run、日志、结果列表。 + +### 7.3 D-InSAR 产物 + +任务类型： + +- `SCAN_DINSAR` +- `DINSAR_RESULT_SCAN` +- `DINSAR_RESULT_PACKAGE` +- 以当前后端实际 task_type 为准。 + +面板位置： + +- `DinsarProductsPanel` 扫描按钮旁或结果列表顶部。 + +局部约束： + +- 同一 catalog 重建任务运行中，禁用重复重建按钮。 +- 不影响 SBAS 生产、D-InSAR 生产、结果浏览。 + +### 7.4 SBAS-InSAR 生产 + +任务类型： + +- `SBAS_GAMMA_WORKFLOW` +- `SBAS_LANDSAR_WORKFLOW` +- `SBAS_COREGISTRATION` +- `SBAS_RDC_DEM` +- `SBAS_INTERFEROGRAMS` +- `SBAS_IPTA_TIMESERIES` + +当前状态： + +- 已基本符合新设计。 +- 已在模块内展示 Runtime Status。 +- 已显式设置 `nonBlocking: true`。 + +后续调整： + +- 将 Runtime Status 抽成 `TaskStatusPanel` 风格组件。 +- 支持按 `run_id` 过滤关联任务。 +- 全局任务中心只显示摘要和跳转入口。 + +### 7.5 SBAS-InSAR 结果 + +任务类型： + +- `REBUILD_SBAS_INSAR_CATALOG` + +局部约束： + +- 重建中只禁用“重建目录”按钮。 +- 结果列表仍可浏览。 + +当前已接近目标。 + +### 7.6 数据接入与资产扫描 + +任务类型： + +- `UNPACK_ARCHIVES` +- `UNPACK_SENTINEL1` +- `GF3_UNPACK` +- `GF3_BATCH_PROCESS` +- `GF3_SARSCAPE_PRODUCE` +- `GF3_SARSCAPE_SYNC` +- `GF3_SARSCAPE_CLEAN` +- `SCAN_DATA` +- `SCAN_ASSET_INVENTORY` + +建议： + +- 数据接入页按卫星/流程展示独立状态卡。 +- `SCAN_DATA` 不应锁全局界面。 +- `GF3_BATCH_PROCESS` 是否需要局部锁取决于是否写共享目录；默认只锁 GF3 预处理按钮。 + +### 7.7 洪涝 / 水体分析 + +任务类型： + +- `WATER_GEOCODE_*` +- `WATER_DETECT_*` +- `WATER_FLOOD_*` +- `FLOOD_SCENE_PREPROCESS_*` +- `FLOOD_WATER_EXTRACTION_*` +- `FLOOD_DETECTION_*` +- `GF3_PROCESS_*` + +建议： + +- 洪涝工作台显示场景级任务状态。 +- 使用 task type prefix 匹配。 +- 不再触发全局锁。 +- 单个场景处理时，只禁用该场景相关按钮。 + +### 7.8 AI 与灾害点 + +任务类型： + +- `AI_TRAIN` +- `AI_PREDICT` +- `AI_ANALYZE` +- `AI_WARMUP` +- `SCAN_HAZARD` + +建议： + +- AI 质量页显示 AI 训练/预测任务状态。 +- AI 诊断页显示诊断任务状态。 +- `AI_TRAIN` 运行中禁用再次训练；不阻止浏览结果。 +- `AI_ANALYZE` 运行中只禁用同一结果的重复诊断。 +- `SCAN_HAZARD` 只影响灾害点同步按钮。 + +## 8. 任务分类模型 + +建议引入统一任务元数据配置，前端和后端可逐步共享。 + +```ts +type TaskUiPolicy = { + taskType: string; + featureScope: string; + label: string; + globalVisible: boolean; + globalBlocking: boolean; + localBlocking: boolean; + resourceScope?: string; + routeTarget?: string; +}; +``` + +默认策略： + +- `globalVisible=true` +- `globalBlocking=false` +- `localBlocking=true` + +也就是说，任务默认显示在全局任务中心，但不锁整个系统。 + +只有极少数任务可设置： + +```ts +globalBlocking=true +``` + +但这应作为过渡兼容，不作为长期设计。 + +## 9. 后端资源锁设计 + +后端当前只有 `task_type` 级互斥。这不足以表达以下场景： + +- 同一个 run 不能同时执行两个会修改同一状态文件的步骤。 +- 同一个 catalog 不能并发重建。 +- 同一个输出目录不能被两个生产任务同时写入。 +- 同一个 ENVI / SARscape 单实例资源不能并行调用。 + +建议新增资源锁概念。 + +### 9.1 资源锁 key + +示例： + +```text +sbas-run:{run_id} +sbas-catalog:{catalog_root} +dinsar-engine:{engine_code} +dinsar-root:{root_dir} +envi-runtime:{host_or_profile} +gf3-sarscape-root:{root_dir} +water-scene:{scene_id} +ai-model:{model_id} +``` + +### 9.2 后端行为 + +创建任务时检查： + +- 同 `task_type` 是否已有活跃任务。 +- 同 `resource_lock_key` 是否已有活跃任务。 + +如果冲突： + +- 返回 `409 Conflict`。 +- 返回冲突任务 ID、任务类型、状态、消息。 + +前端收到后： + +- 不弹全局锁。 +- 在当前功能面板提示“已有同资源任务运行中”。 +- 提供跳转到任务日志。 + +### 9.3 迁移方式 + +第一阶段不必改数据库结构，可把资源锁写入 `SystemTaskORM.params`： + +```json +{ + "resource_lock_key": "sbas-run:sbas_e21648e52bd4", + "feature_scope": "sbas_insar" +} +``` + +后续再考虑独立列或资源锁表。 + +## 10. 前端状态管理重构 + +### 10.1 保留内容 + +保留： + +- `/tasks/active` SSE / fallback polling。 +- `activeTasks` 全局缓存。 +- 任务完成后刷新相关数据的能力。 + +### 10.2 移除或降级内容 + +降级： + +- `isGlobalLocked` +- `pendingTaskIds` +- `nonBlockingTaskIds` +- `NON_BLOCKING_TASK_TYPES` +- `ActiveTasksOverlay` + +迁移后： + +- `isGlobalLocked` 不再驱动全屏遮罩。 +- `pendingTaskIds` 不再用于判断系统锁定。 +- `nonBlockingTaskIds` 不再需要。 +- `NON_BLOCKING_TASK_TYPES` 变成 `TASK_UI_POLICIES`。 +- `ActiveTasksOverlay` 替换为 `GlobalTaskCenter`。 + +### 10.3 新 hook + +建议新增： + +```text +frontend/src/hooks/useTaskCenter.js +frontend/src/hooks/useTaskMonitor.js +frontend/src/config/taskUiPolicies.js +``` + +`useTaskCenter`： + +- 订阅 active tasks。 +- 维护全局任务状态。 +- 提供任务完成事件分发。 + +`useTaskMonitor`： + +- 从全局任务状态中过滤当前功能相关任务。 +- 提供 `isBusy`、`latestTask`、`activeTasks`、`recentTasks`。 + +## 11. 任务完成后的刷新策略 + +当前全局锁解除时会调用 `initializeAppData({ refreshRadarSearch: true })`，这也过粗。 + +应改成按任务类型刷新： + +| 任务类型 | 刷新目标 | +|---|---| +| `UNPACK_ARCHIVES` | LT-1 数据检索选项、当前检索页 | +| `UNPACK_SENTINEL1` | Sentinel-1 资产状态 | +| `SCAN_ASSET_INVENTORY` | 资产库存状态 | +| `SBAS_*` | 对应 SBAS run detail 或产品 catalog | +| `REBUILD_SBAS_INSAR_CATALOG` | SBAS 产品列表 | +| `SCAN_DINSAR` | D-InSAR 产品列表 | +| `AI_TRAIN` / `AI_PREDICT` | AI 状态与结果质量 | +| `SCAN_HAZARD` | 灾害点列表 | +| `WATER_*` / `FLOOD_*` | 洪涝工作台当前场景/事件 | + +实现上可维护： + +```ts +TASK_COMPLETION_REFRESH_POLICIES +``` + +每个功能页也可以订阅自己的任务完成事件。 + +## 12. 迁移计划 + +### 阶段 1：文档与审计 + +状态：本设计文档。 + +输出： + +- 当前锁机制审计。 +- 新交互原则。 +- 迁移边界。 + +### 阶段 2：抽象任务 UI policy + +新增： + +- `frontend/src/config/taskUiPolicies.js` + +内容： + +- task type -> label +- task type -> feature scope +- task type -> route target +- task type -> local/global blocking policy + +替换： + +- `useGlobalTaskControl.NON_BLOCKING_TASK_TYPES` +- `useDinsarOperations.NON_BLOCKING_TASK_TYPES` +- `ActiveTasksOverlay.getTaskTypeLabel` + +验收： + +- 所有任务类型 label 来自同一配置。 +- 新任务默认不锁全局。 + +### 阶段 3：全局遮罩改为任务中心 + +新增： + +- `GlobalTaskCenter` +- `TaskCenterButton` 或顶部状态入口 + +替换： + +- `ActiveTasksOverlay` + +验收： + +- 有活跃任务时不再遮住全系统。 +- 用户可继续切换页面和浏览结果。 +- 管理员仍可取消任务。 + +### 阶段 4：功能级任务面板 + +优先级： + +1. D-InSAR 生产 +2. D-InSAR 产物 +3. 数据接入 +4. 洪涝分析 +5. AI / 灾害点 +6. SBAS 组件收敛到通用面板 + +验收： + +- 每个功能页能看到本功能任务状态。 +- 同类任务运行中，只禁用同类提交按钮。 + +### 阶段 5：后端资源锁 + +新增： + +- 任务 params 中写入 `feature_scope`、`resource_lock_key`。 +- `task_service.create_task()` 支持资源锁冲突检查。 + +验收： + +- 同 run、同目录、同 catalog 的冲突任务由后端返回 `409`。 +- 前端不靠全局遮罩防冲突。 + +### 阶段 6：删除旧全局锁状态 + +删除或废弃： + +- `isGlobalLocked` +- `pendingTaskIds` +- `nonBlockingTaskIds` +- `handleTaskStart(null, ...)` 触发全局锁的路径 + +验收： + +- 代码中不存在“无 taskId 触发全局锁”的逻辑。 +- 全局任务中心只展示状态，不控制系统可用性。 + +## 13. 风险与处理 + +### 13.1 后端资源冲突未覆盖 + +风险： + +- 去掉前端全局锁后，某些共享目录或单实例程序可能被并发调用。 + +处理： + +- 迁移初期保留少量 `globalBlocking=true` 兼容策略。 +- 优先给 ENVI / SARscape / 同目录生产补资源锁。 +- 对不确定任务先做局部锁，不做全局遮罩。 + +### 13.2 用户忽略后台任务 + +风险： + +- 没有全屏遮罩后，用户可能不知道任务仍在跑。 + +处理： + +- 顶部或右下角常驻任务指示。 +- 功能页内明确显示任务状态。 +- 失败任务有醒目提示。 + +### 13.3 任务完成刷新过少 + +风险： + +- 以前全局刷新掩盖了局部刷新缺失。 + +处理： + +- 建立 `TASK_COMPLETION_REFRESH_POLICIES`。 +- 逐功能补刷新策略。 +- 保留手动刷新入口。 + +## 14. 验收标准 + +完成重构后应满足： + +1. 任意 SBAS 任务运行时，系统其他功能可正常浏览和操作。 +2. D-InSAR 任务运行时，不再弹全屏锁；D-InSAR 面板显示任务进度。 +3. 同一个 D-InSAR engine 或同一输出目录的冲突提交由后端拒绝。 +4. 数据接入扫描/解包任务运行时，不影响生产管理和结果浏览。 +5. 洪涝场景级任务运行时，只影响对应场景按钮。 +6. AI 训练/预测运行时，不阻断地图、结果浏览和 SBAS/DInSAR 生产。 +7. 所有活跃任务都能在全局任务中心找到。 +8. 每个功能页能看到与本功能相关的任务。 +9. 管理员取消任务能力仍可用，但语义是“取消任务”，不是“强制解锁界面”。 +10. 代码中不再通过 `handleTaskStart(null, ...)` 触发全局锁。 + +## 15. 推荐首批改动清单 + +建议第一批代码改动只做前端交互，不动后端任务模型： + +1. 新建 `taskUiPolicies.js`，统一任务 label、scope、blocking 策略。 +2. 将 D-InSAR 生产、D-InSAR 产物、IDL、AI、灾害点、水体任务全部标为 `globalBlocking=false`。 +3. `ActiveTasksOverlay` 改为可关闭的 `GlobalTaskCenter`。 +4. `ensureCanOperate()` 不再读取 `isGlobalLocked`，只检查用户权限。 +5. `rebuildRadarPreviewCache()` 删除 `handleTaskStart(null, ...)`，改用局部 loading。 +6. D-InSAR 生产面板增加任务状态卡。 +7. D-InSAR 产物面板增加目录扫描任务状态卡。 + +第二批再做后端资源锁。 + +## 16. 结论 + +全局界面锁应退出核心设计。任务系统的正确边界应是： + +- 全局：看见所有任务。 +- 功能页：管理本功能任务。 +- 后端：保证资源互斥和并发安全。 + +前端全屏锁只应作为临时兼容手段，不应继续扩展。 diff --git a/docs/INDEX.md b/docs/INDEX.md index d641739..624b9c8 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -18,6 +18,9 @@ - [FRONTEND_NAVIGATION_ARCHITECTURE.md](FRONTEND_NAVIGATION_ARCHITECTURE.md) 当前左侧导航和生产管理工作台视图模型。 +- [GLOBAL_TASK_STATUS_LOCK_REDESIGN_20260612.md](GLOBAL_TASK_STATUS_LOCK_REDESIGN_20260612.md) + 全局界面锁降级为任务状态中心、功能级任务面板和后端资源锁的重构设计。 + ## 生产与结果 - [PRODUCTION_RESULTS_MULTI_ENGINE_DESIGN_20260423.md](PRODUCTION_RESULTS_MULTI_ENGINE_DESIGN_20260423.md) diff --git a/docs/LANDSAR_SBAS_ARCHIVE_20260606.md b/docs/LANDSAR_SBAS_ARCHIVE_20260606.md new file mode 100644 index 0000000..3590570 --- /dev/null +++ b/docs/LANDSAR_SBAS_ARCHIVE_20260606.md @@ -0,0 +1,56 @@ +# LandSAR SBAS-InSAR Archive Record - 2026-06-06 + +## Decision + +The current LandSAR license does not include SBAS-InSAR capability. LandSAR SBAS production is archived and should not be used as a production path unless the runtime/license is replaced with one that supports SBAS-InSAR. + +Future SBAS production work should focus on Gamma SBAS. + +## Archived Runs + +### Run 1 + +- Task ID: `2f772b33-e463-415d-a0e5-3153b092ed74` +- Job ID: `1e89b910-437d-4d0c-9957-a908bcbf0651` +- Task type: `SBAS_LANDSAR_WORKFLOW` +- Run ID: `landsar_sbas_20260605T175848009769Z_sbas_be0008c47ac5` +- Status: `FAILED` +- Started at: `2026-06-06 01:58:33` +- Ended at: `2026-06-06 02:42:14` +- Failure summary: LandSAR SBAS workflow failed after LT-1 import; `InSAR_Console.exe` returned `Cannot read this ID` for configured SBAS proID `280039`. + +### Run 2 + +- Task ID: `51adf126-3d4b-4404-94af-1c12c2103f11` +- Job ID: `03eee99a-3bbc-4d81-9fdc-bf6199751834` +- Task type: `SBAS_LANDSAR_WORKFLOW` +- Run ID: `landsar_sbas_20260606T083925060668Z_sbas_be0008c47ac5` +- Status: `FAILED` +- Started at: `2026-06-06 16:38:20` +- Ended at: `2026-06-06 17:03:29` +- Failure summary: LandSAR SBAS runtime unsupported. LT-1 import completed for 7 scenes, but `InSAR_Console.exe` did not accept process `SBAS Stream` / proID `280039`. + +## Data Stack + +- Stack ID: `sbas_be0008c47ac5` +- Scene count: `7` +- Dates: `20240516`, `20240711`, `20240905`, `20250417`, `20250612`, `20250807`, `20251002` +- DEM: `D:\DEM\HeiLongJiang10M_DEM.tif` + +## Cleanup Scope + +The following records and generated artifacts were removed after this archive note was created: + +- `system_tasks` records for the two task IDs above. +- `system_jobs` records for the two job IDs above. +- `task_logs` rows for the two task IDs above. +- Run result directories under `D:\production_results\timeseries\sbas_landsar\runs`. +- LandSAR working directories under `D:\LandSAR_Work\sbas`. + +No matching rows were found in `result_products`, `result_catalog_states`, or `ps_timeseries_runs`. + +## Follow-Up + +- Keep LandSAR SBAS disabled or clearly marked unsupported in production operations. +- Continue SBAS production through Gamma SBAS only. +- Revisit LandSAR SBAS only after a supported license/runtime is available and proID/process compatibility is verified before full production execution. diff --git a/docs/LANDSAR_SBAS_INSAR_INTEGRATION_DESIGN.md b/docs/LANDSAR_SBAS_INSAR_INTEGRATION_DESIGN.md new file mode 100644 index 0000000..8140d72 --- /dev/null +++ b/docs/LANDSAR_SBAS_INSAR_INTEGRATION_DESIGN.md @@ -0,0 +1,283 @@ +# LandSAR SBAS-InSAR 接入设计 + +记录时间：2026-06-04 + +## 1. 结论 + +LandSAR 支持 SBAS-InSAR。其 SBAS 是一体化流程，算法编号为 `280039`，参数文件入口是： + +```text +SBASProcess +ID 280039 +``` + +这条链路和当前系统里的 Gamma SBAS 不同。Gamma SBAS 是分阶段工作流，包含栈发现、baseline audit、共参考配准、RDC DEM、干涉图、IPTA 反演和发布阶段；LandSAR SBAS 更接近 LandSAR D-InSAR 的模式：准备好 `Task_*/Input_Data` 后，通过 `InSAR_Console.exe + 280039.txt` 一次执行。 + +因此接入建议是：在 SBAS-InSAR 页面新增 `LandSAR SBAS` 处理器分支，而不是把 LandSAR 硬塞进现有 Gamma 分阶段按钮。 + +## 2. LandSAR SBAS 输入输出 + +### 输入目录 + +LandSAR SBAS 扫描 `Task_*` 目录，每个任务目录要求： + +```text +Task_xxx +|-- Input_Data +| |-- LT1*_SLC.xml +| |-- LT1*_SLC.tif +| `-- ... +`-- Output_Data +``` + +核心条件： + +- `Input_Data` 下至少 3 景已导入的 LT-1 SLC。 +- 每景需要 `LT1*_SLC.xml` 与 `LT1*_SLC.tif/.tiff` 配对。 +- DEM 需要外部指定。 +- 精轨最好已经写入 XML 或已通过 LandSAR 精轨导入流程处理。 + +当前 LandSAR 项目文档说明，TimeSeriesBuilder 输出的 `Task_TS_*/Input_Data` 可直接被 PS/SBAS 页面扫描。 + +### 输出目录 + +LandSAR SBAS 输出在每个任务的 `Output_Data`： + +```text +Output_Data +|-- *.los.tif +|-- *.raster.tif +|-- vector/ +|-- 280039.log +`-- 280039_console.log +``` + +成功判定可参考 LandSAR GUI： + +- 日志包含 `SBAS` 和 `success` +- 或日志包含 `console success` +- 控制台返回码为 0 +- 同时存在核心输出，如 `*.los.tif` 或 `*.raster.tif` + +## 3. 与现有系统的差异 + +### 现有 Gamma SBAS + +当前 `/api/sbas-insar-production` 是 Gamma / IPTA SBAS 主线： + +- 从资产池发现 LT-1 或 Sentinel-1 候选栈。 +- 创建 `run_manifest.json`。 +- 分阶段执行 Gamma 脚本。 +- catalog 期望的核心产品是： + - `publish/geotiff/los_rate_toward_m_per_year.tif` + - `publish/geotiff/los_sigma_m_per_year.tif` + - 相关预览图、质量图和监测点曲线。 + +### LandSAR SBAS + +LandSAR SBAS 的输入不是系统当前的资产栈 manifest，而是 LandSAR 风格的 `Task_*/Input_Data`。 + +LandSAR SBAS 的输出语义也和 Gamma 不完全一致。`*.los.tif` 在文档里描述为 LOS 时序形变场，不应在没有验证前直接标成 Gamma 那种 `los_rate_toward_m_per_year` 速率产品。 + +因此 LandSAR SBAS 需要独立 processor 标识： + +```text +processor_code = landsar_sbas +profile_code = lt1_landsar_sbas +engine_code = landsar +proid = 280039 +``` + +## 4. 推荐接入路线 + +### 阶段 1：MVP，只接现成 Task/Input_Data + +目标：先让系统能扫描、提交、监控和归档 LandSAR SBAS，不负责从原始 LT-1 自动构建时序 Input_Data。 + +新增配置： + +```text +LANDSAR_SBAS_ENABLED=true +LANDSAR_SBAS_WORK_ROOT=D:\LandSAR_Work\sbas +LANDSAR_SBAS_DEM_PATH=D:\DEM\HeiLongJiang10M_DEM.tif +LANDSAR_SBAS_TIMEOUT_SECONDS=172800 +LANDSAR_SBAS_MIN_SCENES=3 +LANDSAR_SBAS_SOURCE_ROOTS=D:\Task_Pool\SBAS +``` + +后端新增： + +```text +backend/app/services/landsar_sbas_service.py +``` + +职责： + +- 检查 LandSAR runtime、授权服务、`InSAR_Console.exe`、SBAS 相关 DLL。 +- 扫描 root 下的 `Task_*` 或单个 `Task_*`。 +- 校验 `Input_Data` 内 SLC XML/TIF 数量。 +- 生成 `280039.txt` 参数文件。 +- 调用： + +```text +D:\LandSAR\InSAR_Console.exe D:\LandSAR_Work\sbas\<run_id>\native\<task>\Output_Data\280039.txt +``` + +- 把日志和核心 GeoTIFF 复制到系统标准结果目录。 +- 写入 `run_manifest.json`、`workflow_summary.json`、`product_summary.json`。 + +建议标准结果目录： + +```text +D:\production_results\timeseries\sbas\<run_id> +|-- run_manifest.json +|-- workflow_summary.json +|-- product_summary.json +|-- native_logs +| |-- 280039.txt +| |-- 280039.log +| `-- 280039_console.log +`-- publish + `-- landsar + |-- los_timeseries.tif + |-- post_raster.tif + `-- vector/ +``` + +任务队列新增： + +```text +JOB_TYPE_SBAS_LANDSAR_WORKFLOW = "SBAS_LANDSAR_WORKFLOW" +``` + +前端新增： + +- 在 `SBAS-InSAR 生产` 页面增加处理器选择： + - `Gamma / IPTA SBAS` + - `LandSAR SBAS` +- 选择 `LandSAR SBAS` 后显示： + - Task 根目录 + - DEM 路径 + - 最少景数 + - 干涉对策略：`single` / `prim` + - 垂直基线阈值 + - 时间基线阈值 + - 方位向/距离向多视 + - 输出 LOS 时序 + - 输出编码后栅格 +- 隐藏 Gamma 的 baseline/coreg/RDC DEM/IPTA 分阶段按钮。 + +### 阶段 2：接入 catalog 和结果页 + +当前 `sbas_insar_catalog_service.py` 主要按 Gamma 产品定义扫描。LandSAR 接入后有两种选择： + +1. 复用 `sbas_insar` catalog，但让资产定义按 `processor_code` 分支。 +2. 新建 `landsar_sbas` catalog。 + +建议选择 1。理由是前端结果页还是 SBAS-InSAR 结果，只是处理器不同。 + +需要调整： + +- `_READY_STATUSES` 增加 LandSAR 完成状态。 +- `_CORE_ASSETS` 改成按 processor 分支。 +- LandSAR 主资产角色： + +```text +primary_geotiff -> publish/landsar/los_timeseries.tif +secondary_geotiff -> publish/landsar/post_raster.tif +run_manifest -> run_manifest.json +workflow_summary -> workflow_summary.json +native_console_log -> native_logs/280039_console.log +``` + +注意：不要把 LandSAR `*.los.tif` 直接命名为 `los_rate_toward_m_per_year.tif`，除非算法工程师确认该文件确实是年速率图。 + +### 阶段 3：从资产池自动构建 LandSAR 时序 Input_Data + +阶段 1 只接现成 `Task_*/Input_Data`。如果需要从系统资产池直接生产 LandSAR SBAS，需要新增前处理： + +1. 从 LT-1 资产池选择同轨同极化多景。 +2. 调用 LandSAR LT-1 数据导入 `100016`，构建多景 `Input_Data`。 +3. 调用 LandSAR 精轨导入 `100206`，或确认精轨已进入 XML。 +4. 输出 `Task_TS_*/Input_Data`。 +5. 再调用 SBAS `280039`。 + +这一阶段风险比 MVP 高，建议在 LandSAR SBAS 一体化流程跑通后再做。 + +## 5. 参数建议 + +MVP 默认参数应与 LandSAR GUI 保持一致： + +```text +dem_data_type=1 # 文件。注意 SBAS 模板中 0 是目录，1 是文件 +dem_format=4 # COPERNICUS，需结合实际 DEM 测试 +intf_method=0 # single +perp_baseline=200 +time_baseline=300 +doppler_baseline=100 +az_looks=3 +rg_looks=3 +da_threshold=0.25 +intensity_threshold=0.0 +calibration_threshold=0.4 +fine_reg_window=128 +resample_factor=2 +network_type=0 # Delaunay +max_arc_distance=1000 +solve_method=0 # Periodogram +max_temporal_coh=0.7 +ref_point_index=0 +spatial_filter_dist=1000 +unwrap_ref_index=0 +time_filter_threshold=0.3 +do_los_output=1 +gen_vector_map=0 +gen_pre_raster=0 +gen_post_raster=1 +``` + +需要特别注意 DEM 参数。DInSAR 用 `HeiLongJiang10M_DEM.tif` 已跑通，但 SBAS 模板里的 DEM 类型字段和 DInSAR 不同： + +- SBAS：`dem_data_type=0` 表示目录，`1` 表示文件。 +- 当前 DEM 是 GeoTIFF 文件，所以应传 `1`。 + +## 6. 风险点 + +1. 输入结构风险 + + 现有 Gamma SBAS 的候选栈不是 LandSAR 的 `Task_*/Input_Data`。MVP 必须明确只支持 LandSAR 已导入后的时序任务目录。 + +2. 输出语义风险 + + LandSAR `*.los.tif` 是否为单幅速率图、累计形变图，还是多波段时序图，需要用一次真实输出确认。没有确认前不能按 Gamma 年速率产品入库。 + +3. DLL 风险 + + SBAS/PS 可能需要 DInSAR 之外的 DLL，例如： + + - `SAR_InSAR_MTInSARModel.dll` + - `SAR_InSAR_PSInSAR_CSU.dll` + - `SAR_InSAR_MBCP_MTInSARModel.dll` + + LandSAR SBAS availability check 应单独检查这些 DLL。 + +4. 运行时间风险 + + SBAS 是多景时序处理，默认 timeout 应明显大于 DInSAR，建议先设为 48 小时。 + +5. catalog 风险 + + 当前 SBAS catalog 以 Gamma 结果为主。LandSAR 接入时要做 processor-specific asset mapping，否则会出现“运行成功但结果页查不到”。 + +## 7. 建议下一步 + +先实现阶段 1： + +1. 新增 `landsar_sbas_service.py`，只支持扫描现成 `Task_*/Input_Data`。 +2. 新增 `SBAS_LANDSAR_WORKFLOW` 后台任务。 +3. 前端 SBAS 页面增加 `LandSAR SBAS` 分支。 +4. 先不改资产池自动构建，不接 Sentinel-1，不开放 GACOS。 +5. 用一个 3 景以上的 `Task_TS_*` 做首轮真实测试。 +6. 根据真实输出再接 catalog 和结果页。 + +这个路线保留 Gamma SBAS 现状，同时利用已经验证的 LandSAR runtime 和授权链路，风险最低。 diff --git a/docs/LandSAR_API服务接入采购需求说明书_20260604.md b/docs/LandSAR_API服务接入采购需求说明书_20260604.md new file mode 100644 index 0000000..a490b9d --- /dev/null +++ b/docs/LandSAR_API服务接入采购需求说明书_20260604.md @@ -0,0 +1,843 @@ +# LandSAR D-InSAR 与 SBAS-InSAR 服务接入采购需求说明书 + +编制日期：2026-06-04 +适用项目：InSAR 管理系统 v2 +重点数据：陆探一号（LT-1 / LuTan-1）SAR 数据 +文档用途：供应商技术沟通、采购询价、招标需求编制 + +## 1. 项目背景 + +现有 InSAR 管理系统已经具备雷达影像资产管理、任务队列、生产任务监控、结果目录管理、D-InSAR 结果入库、SBAS-InSAR 结果 catalog、地图预览和结果查询等工程能力。系统侧按“处理器 processor + 工作流 workflow + 结果包 manifest”的方式组织外部算法服务，要求外部服务能够以稳定接口接收任务、返回结构化状态并输出可入库的标准结果。 + +本次拟采购 LandSAR 服务版中的陆探一号 D-InSAR 与 SBAS-InSAR 两个处理服务模块。服务应以本地部署方式运行在生产服务器上，通过 HTTP API、消息队列 API 或二者结合的方式接收处理任务。业务系统负责选择数据、创建任务、轮询或订阅任务状态、读取结果清单并完成 catalog 入库；LandSAR 服务负责实际算法生产、运行日志、状态输出和结果文件组织。 + +## 2. 建设目标 + +采购目标是获得一套可本地部署、可长期稳定运行、可由现有系统调用的 LandSAR 服务版 API 模块，重点支撑陆探一号 D-InSAR 与 SBAS-InSAR 自动化生产。 + +核心建设目标如下： + +1. 支持陆探一号 D-InSAR 差分干涉生产，产出可入库、可预览、可下载的标准 GeoTIFF 结果。 +2. 支持陆探一号 SBAS-InSAR 时序形变生产，产出语义明确、可归档、可预览、可下载的栅格或点矢量结果。 +3. 支持 D-InSAR 与 SBAS-InSAR 所需的数据导入、轨道处理、DEM 处理、地理编码和结果发布能力。 +4. 提供完整的任务提交、排队、状态查询、日志查询、结果查询、任务取消和错误码机制。 +5. 与现有系统任务队列、数据目录、D-InSAR 结果 catalog、SBAS-InSAR 结果 catalog 和前端生产页面集成。 +6. 避免业务系统直接维护算法进程、底层授权细节和非结构化运行状态。 + +## 3. 系统现状 + +### 3.1 现有业务系统 + +现有系统主要技术栈： + +- 后端：FastAPI / Python。 +- 前端：React / Vite。 +- 任务队列：系统内置任务队列和任务日志表。 +- 数据库：系统已有结果 catalog 和任务状态表。 +- 文件组织：生产结果主要进入 `D:\production_results`，临时工作目录可配置。 + +### 3.2 现有系统接入边界 + +现有系统侧可提供以下集成条件： + +- 可配置的源数据目录、工作目录、结果目录和 DEM 路径。 +- 可按任务生成 `run_id`、`task_id`、`job_id` 并维护任务日志。 +- D-InSAR 结果目录支持按 pair/run 组织。 +- SBAS-InSAR 结果目录支持按 processor/run 组织。 +- 结果 catalog 可按 `processor_code`、`engine_code`、`profile_code` 区分不同处理器。 +- 前端生产页面可根据处理器能力显示不同参数和任务状态。 + +### 3.3 当前痛点 + +1. D-InSAR 和 SBAS-InSAR 长流程任务需要稳定的异步服务接口。 +2. 任务状态、阶段进度、日志和错误原因需要结构化输出。 +3. 错误码需要统一，便于系统侧自动诊断和前端展示。 +4. 多任务并发、排队、互斥和取消能力需要明确。 +5. SBAS-InSAR 输出语义、质量指标和结果文件组织需要供应商明确说明。 +6. D-InSAR 与 SBAS-InSAR 需要共用统一的 API、任务状态和结果 manifest 规范，降低系统侧维护成本。 + +## 4. 采购范围 + +本次采购范围分为必选能力和可选能力。 + +### 4.1 必选能力 + +1. LandSAR 本地 API 服务部署授权。 +2. 陆探一号 D-InSAR API。 +3. 陆探一号 SBAS-InSAR API。 +4. D-InSAR 与 SBAS-InSAR 所需的数据导入、轨道处理、DEM 支持和地理编码能力。 +5. 任务队列、任务状态、日志、结果查询、任务取消和错误码接口。 +6. API 调用文档、参数说明、返回值说明和示例代码。 +7. 服务部署脚本、启动脚本、停止脚本、健康检查接口。 +8. 授权服务部署说明和异常处理说明。 +9. D-InSAR 与 SBAS-InSAR 各至少一组陆探一号样例数据的端到端验收支持。 + +### 4.2 可选能力 + +1. 陆探一号独立预处理 API。 +2. 陆探一号 PS-InSAR API。 +3. 多任务并发执行能力。 +4. GPU 加速能力。 +5. 断点续跑能力。 +6. 结果自动发布为标准 GeoTIFF、COG 或瓦片服务。 +7. 与第三方消息队列对接能力，例如 RabbitMQ、Redis Stream、Kafka 或 ZeroMQ。 + +## 5. 总体架构要求 + +### 5.1 部署方式 + +LandSAR API 服务应部署在本地生产服务器，推荐形态： + +```text +InSAR 管理系统 + -> HTTP API / MQ + -> LandSAR API Service + -> LandSAR Engine + -> 本地文件系统结果目录 +``` + +服务应支持 Windows Server 环境部署，安装目录、工作目录、授权方式和监听端口均应可配置。 + +### 5.2 服务访问方式 + +服务应至少提供一种稳定接口方式： + +- HTTP REST API，本地端口访问，如 `http://127.0.0.1:<port>` +- 或消息队列接口，任务提交后异步回传状态 + +推荐同时支持： + +- HTTP API 用于任务提交、状态查询、日志查询、结果查询、健康检查。 +- 消息队列用于长任务异步调度和状态通知。 + +### 5.3 任务执行模式 + +所有生产任务均应采用异步任务模式： + +1. 系统提交任务。 +2. LandSAR API 返回 `job_id`。 +3. 系统定期查询状态或接收消息通知。 +4. 任务完成后系统读取结果清单并入库。 + +不建议采用一次 HTTP 请求长期阻塞等待处理完成的模式。 + +## 6. 数据范围要求 + +### 6.1 支持数据类型 + +本次重点支持陆探一号： + +- LT-1A +- LT-1B +- SLC 产品 +- HH 极化优先，后续可扩展 HV/VV/VH +- 同轨、同模式、同区域数据配对和时序处理 + +### 6.2 输入数据形态 + +服务应明确支持以下至少一种输入形态。 + +优先要求： + +```text +原始 LT-1 产品目录或压缩包 +``` + +同时兼容： + +```text +已导入的 LT1*_SLC.xml + LT1*_SLC.tif +``` + +对于 D-InSAR，服务应支持输入主影像和辅影像路径。 +对于 SBAS/PS，服务应支持输入多景时序数据目录。 + +### 6.3 辅助数据 + +服务应支持以下辅助数据配置： + +- DEM 文件或 DEM 目录。 +- 精密轨道文件目录。 +- 输出目录。 +- 临时工作目录。 +- 可选 GCP 文件。 +- 可选 GACOS 大气改正文件，若服务支持。 + +## 7. 功能需求 + +## 7.1 陆探一号支撑性预处理能力 + +### 7.1.1 功能目标 + +支撑性预处理能力用于满足 D-InSAR 和 SBAS-InSAR 生产前的数据导入、轨道处理、多视、地理编码和快视输出需要。若供应商提供独立预处理 API，应可作为后续扩展能力接入系统；若预处理仅作为 D-InSAR/SBAS-InSAR 内部阶段，也应在任务日志、阶段状态和结果 manifest 中体现。 + +### 7.1.2 必须支持的处理能力 + +1. 数据导入。 +2. 精密轨道导入或轨道参数更新。 +3. 多视处理。 +4. 辐射定标或强度图生成。 +5. 地形校正、正射校正或地理编码。 +6. 输出标准 GeoTIFF。 +7. 输出快视图。 +8. 输出完整处理日志。 + +### 7.1.3 输入参数要求 + +如提供独立预处理 API，预处理任务应至少支持以下参数： + +```json +{ + "job_type": "lt1_preprocess", + "input_path": "D:/data/LT1/scene", + "dem_path": "D:/DEM/HeiLongJiang10M_DEM.tif", + "orbit_path": "D:/orbit_pools/landsar", + "output_dir": "D:/production_results/landsar_preprocess/<job_id>", + "work_dir": "D:/LandSAR_Work/api/<job_id>", + "polarization": "HH", + "az_looks": 3, + "rg_looks": 3, + "geocode": true, + "orthorectify": true, + "output_format": "GeoTIFF" +} +``` + +### 7.1.4 输出结果要求 + +如提供独立预处理 API，预处理任务应输出： + +```text +output_dir +|-- manifest.json +|-- logs/ +|-- quicklook/ +|-- geotiff/ +| |-- intensity_geo.tif +| |-- amplitude_geo.tif +| `-- ... +`-- metadata/ +``` + +GeoTIFF 必须满足： + +- GDAL 可读。 +- 有 CRS。 +- 有 GeoTransform。 +- NoData 值明确。 +- 数据类型明确。 +- 可被 QGIS 打开。 +- 可被现有系统用于地图预览和后续洪涝分析。 + +## 7.2 陆探一号 D-InSAR API + +### 7.2.1 功能目标 + +D-InSAR API 用于对两景陆探一号 SLC 数据执行差分干涉处理，输出 LOS 形变、相干性、解缠相位和快视产品。 + +### 7.2.2 必须支持的处理能力 + +1. 主辅影像导入。 +2. 精密轨道处理。 +3. 配准。 +4. 重采样。 +5. 干涉图生成。 +6. 去平地和地形相位。 +7. Goldstein 或等效滤波。 +8. 相干性计算。 +9. 相位解缠。 +10. LOS 向形变计算。 +11. 地理编码。 +12. 结果 GeoTIFF 输出。 +13. 处理日志和参数文件输出。 + +### 7.2.3 可选处理能力 + +1. LOS 转垂直向形变。 +2. GCP 优化。 +3. 大气相位改正。 +4. 自定义解缠阈值。 +5. 自定义滤波参数。 + +大气相位改正如依赖 GACOS，应明确 GACOS 文件来源、格式和命名规则。不得在无 GACOS 文件时静默启用。 + +### 7.2.4 输入参数要求 + +D-InSAR API 应至少支持以下参数： + +```json +{ + "job_type": "lt1_dinsar", + "master": { + "xml": "D:/Task_Pool/DInSAR/Task_xxx/Input_Data/master.xml", + "slc": "D:/Task_Pool/DInSAR/Task_xxx/Input_Data/master.tif" + }, + "slave": { + "xml": "D:/Task_Pool/DInSAR/Task_xxx/Input_Data/slave.xml", + "slc": "D:/Task_Pool/DInSAR/Task_xxx/Input_Data/slave.tif" + }, + "dem_path": "D:/DEM/HeiLongJiang10M_DEM.tif", + "output_dir": "D:/production_results/dinsar/<pair_key>/<run_id>", + "work_dir": "D:/LandSAR_Work/api/<job_id>", + "az_looks": 3, + "rg_looks": 3, + "coh_mask_threshold": 0.3, + "unwrap_coh_threshold": 0.3, + "filter_alpha": 0.6, + "geocode": true, + "vertical_displacement": false, + "atmospheric_correction": false +} +``` + +### 7.2.5 输出结果要求 + +D-InSAR 结果应至少包含： + +```text +output_dir +|-- manifest.json +|-- logs/ +|-- geotiff/ +| |-- los_displacement.tif +| |-- coherence.tif +| |-- unwrapped_phase.tif +| |-- wrapped_phase.tif +| `-- vertical_displacement.tif +|-- quicklook/ +`-- metadata/ +``` + +其中 `vertical_displacement.tif` 如未启用垂直向形变，可不生成。 + +核心 GeoTIFF 要求： + +- 可被 GDAL/QGIS 读取。 +- 坐标系和仿射变换完整。 +- 单位明确，例如米、毫米或弧度。 +- NoData 值明确。 +- 方向约定明确，例如朝向雷达为正或远离雷达为正。 +- 输出文件命名稳定，不随 GUI 语言环境变化。 + +## 7.3 陆探一号 SBAS-InSAR API + +### 7.3.1 能力定位 + +SBAS-InSAR 是本次采购的必选服务模块。供应商应明确该模块是否已产品化支持 LT-1，并提供真实样例数据、处理报告、输出文件说明和接口调用示例。 + +### 7.3.2 基本要求 + +SBAS-InSAR API 应至少具备： + +1. 多景 LT-1 SLC 输入。 +2. 干涉对自动选择。 +3. 垂直基线阈值设置。 +4. 时间基线阈值设置。 +5. 多视参数设置。 +6. 相干点或有效像元筛选。 +7. 时序形变反演。 +8. LOS 时序或速率产品输出。 +9. 编码后栅格产品输出。 +10. 完整日志和质量指标输出。 + +### 7.3.3 输入参数要求 + +SBAS-InSAR API 应至少支持以下参数： + +```json +{ + "job_type": "lt1_sbas_insar", + "input_stack": { + "mode": "task_input_data", + "path": "D:/Task_Pool/SBAS/Task_xxx/Input_Data" + }, + "dem_path": "D:/DEM/HeiLongJiang10M_DEM.tif", + "orbit_path": "D:/orbit_pools/landsar", + "output_dir": "D:/production_results/timeseries/sbas_landsar/<run_id>", + "work_dir": "D:/LandSAR_Work/api/<job_id>", + "az_looks": 3, + "rg_looks": 3, + "intf_method": "single", + "perp_baseline": 200, + "time_baseline": 300, + "doppler_baseline": 100, + "network_type": "delaunay", + "solve_method": "periodogram", + "los_output": true, + "post_raster": true, + "vector_output": false +} +``` + +服务应明确支持以下至少一种多景输入形态： + +- 原始 LT-1 多景产品目录或压缩包列表。 +- 已导入的 `LT1*_SLC.xml + LT1*_SLC.tif` 多景目录。 +- LandSAR 服务约定的 `Task_*/Input_Data` 多景任务目录。 + +### 7.3.4 输出结果要求 + +SBAS-InSAR 结果应至少包含： + +```text +output_dir +|-- manifest.json +|-- logs/ +|-- geotiff/ +| |-- los_timeseries.tif +| |-- los_rate.tif +| |-- quality.tif +| `-- post_raster.tif +|-- vectors/ +|-- quicklook/ +`-- metadata/ +``` + +其中 `los_rate.tif`、`quality.tif`、`vectors/` 可按供应商算法实际输出调整，但 manifest 必须准确标明每个资产的角色、单位、维度和业务含义。 + +### 7.3.5 输出说明要求 + +供应商必须说明 SBAS 输出文件语义： + +- 输出是累计形变、平均速率还是多波段时序。 +- 单位是米、毫米、弧度还是其他。 +- 正负号方向约定。 +- 多波段时序的日期映射关系。 +- 质量图或相干性图的含义。 +- 参考点或参考区域信息。 +- 干涉网络信息，包括时间基线、垂直基线和选对策略。 + +在未明确输出语义前，业务系统只按 LandSAR 原始产品归档，不直接标记为业务级年速率产品。供应商如要求系统展示年速率产品，必须明确输出确为年速率图，并提供单位、正负号方向和质量控制说明。 + +## 7.4 陆探一号 PS-InSAR API + +### 7.4.1 能力定位 + +PS-InSAR 作为可选扩展能力。若供应商 API 服务支持，应明确接口成熟度和样例验证情况。 + +### 7.4.2 基本要求 + +如支持 PS-InSAR，应至少具备： + +1. 多景 LT-1 SLC 输入。 +2. PS 点选择。 +3. 网络构建。 +4. 参数估计。 +5. 大气/轨道残差处理。 +6. 时序形变反演。 +7. 点矢量结果输出。 +8. 栅格化或可视化输出。 +9. 点位时间序列导出。 + +### 7.4.3 输出要求 + +PS-InSAR 输出应至少包含： + +- PS 点 GeoJSON / Shapefile / CSV。 +- 点位形变速率。 +- 点位时序形变。 +- 质量指标。 +- 参考点信息。 +- 坐标系统说明。 + +## 8. API 通用接口要求 + +### 8.1 健康检查接口 + +服务应提供： + +```http +GET /health +GET /version +GET /capabilities +``` + +返回内容至少包括： + +- 服务状态。 +- LandSAR 引擎版本。 +- 授权状态。 +- 支持模块列表。 +- 支持数据类型。 +- 当前队列长度。 +- 当前运行任务数量。 + +### 8.2 任务提交接口 + +服务应提供统一任务提交接口： + +```http +POST /jobs +``` + +返回： + +```json +{ + "job_id": "string", + "status": "queued", + "message": "job accepted" +} +``` + +### 8.3 任务状态接口 + +```http +GET /jobs/{job_id} +``` + +返回： + +```json +{ + "job_id": "string", + "job_type": "lt1_dinsar", + "status": "running", + "progress": 45, + "stage": "geocoding", + "message": "processing geocoding", + "created_at": "2026-06-04T10:00:00Z", + "started_at": "2026-06-04T10:01:00Z", + "updated_at": "2026-06-04T10:30:00Z" +} +``` + +### 8.4 日志接口 + +```http +GET /jobs/{job_id}/logs +``` + +要求： + +- 支持获取完整日志。 +- 支持按 offset 或时间增量获取日志。 +- 日志级别包含 INFO、WARNING、ERROR。 +- 日志中应包含 LandSAR 原始错误信息。 + +### 8.5 结果接口 + +```http +GET /jobs/{job_id}/result +``` + +返回： + +```json +{ + "job_id": "string", + "status": "completed", + "output_dir": "D:/production_results/...", + "assets": [ + { + "role": "primary_geotiff", + "path": "D:/production_results/.../los_displacement.tif", + "format": "GeoTIFF", + "unit": "m", + "description": "LOS displacement" + } + ], + "logs": [], + "metadata": {} +} +``` + +### 8.6 任务取消接口 + +```http +POST /jobs/{job_id}/cancel +``` + +要求： + +- 支持取消排队任务。 +- 支持尽可能安全地中止运行中任务。 +- 被取消任务应有明确状态 `cancelled`。 +- 不得留下无法再次运行的锁文件或僵尸进程。 + +## 9. 状态码和错误码要求 + +服务必须提供稳定错误码，至少包括： + +| 错误码 | 含义 | +| --- | --- | +| LICENSE_UNAVAILABLE | 授权不可用 | +| ENGINE_NOT_READY | LandSAR 引擎不可用 | +| INVALID_INPUT | 输入参数错误 | +| INPUT_NOT_FOUND | 输入文件不存在 | +| DEM_NOT_FOUND | DEM 不存在 | +| ORBIT_NOT_FOUND | 精轨文件不存在 | +| UNSUPPORTED_SENSOR | 不支持的数据类型 | +| PROCESS_FAILED | 处理失败 | +| OUTPUT_MISSING | 处理完成但结果缺失 | +| TIMEOUT | 任务超时 | +| CANCELLED | 用户取消 | + +错误响应应包含： + +```json +{ + "error_code": "DEM_NOT_FOUND", + "message": "DEM file not found", + "detail": "D:/DEM/xxx.tif", + "recoverable": true +} +``` + +## 10. 与现有系统集成要求 + +### 10.1 任务队列集成 + +现有系统会将 LandSAR API 服务视为外部处理器。业务系统负责创建本地任务记录，LandSAR API 负责实际生产。 + +集成流程： + +```text +用户提交生产任务 +-> 系统创建任务记录 +-> 系统调用 LandSAR API +-> LandSAR API 返回 job_id +-> 系统轮询或订阅 job 状态 +-> job 完成 +-> 系统读取 result assets +-> 系统入库 catalog +-> 前端展示结果 +``` + +### 10.2 文件目录集成 + +建议目录： + +```text +D:\LandSAR_Work\api +D:\production_results\landsar_preprocess +D:\production_results\dinsar +D:\production_results\timeseries\sbas +D:\production_results\timeseries\psinsar +``` + +LandSAR API 应允许调用方指定： + +- `work_dir` +- `output_dir` +- `log_dir` +- `temp_dir` + +### 10.3 结果入库集成 + +LandSAR API 结果清单应便于系统入库。推荐每个任务输出： + +```text +manifest.json +``` + +manifest 至少包含： + +- job_id +- job_type +- processor_code +- input_files +- output_files +- parameters +- start_time +- end_time +- status +- CRS +- bbox +- unit +- NoData +- software_version +- license_mode + +## 11. 性能和稳定性要求 + +### 11.1 并发要求 + +供应商应明确： + +- 是否支持多任务并行。 +- 最大并发任务数。 +- 不同模块是否互斥。 +- D-InSAR 与预处理是否可同时执行。 +- API 服务是否支持排队。 + +如不支持并发，服务也必须支持内部排队或返回明确的忙碌状态。 + +### 11.2 超时要求 + +建议默认超时： + +- 预处理：6 小时。 +- D-InSAR：12 小时。 +- SBAS-InSAR：48 小时。 +- PS-InSAR：72 小时。 + +超时后应返回明确状态，并保留日志。 + +### 11.3 稳定性要求 + +服务应支持： + +- 长时间运行。 +- 进程异常退出后自动恢复。 +- 服务重启后查询历史任务。 +- 任务失败后保留工作目录和日志。 +- 任务成功后可按配置清理中间文件。 + +## 12. 安全和授权要求 + +1. API 服务应支持本机访问限制，默认只监听 `127.0.0.1`。 +2. 如监听局域网地址，应支持 Token 或 API Key。 +3. 授权异常应有明确错误码。 +4. 授权服务应支持开机自启动或由 API 服务托管启动。 +5. 授权有效期、授权模块列表应可查询。 + +## 13. 文档和交付物要求 + +供应商应提供： + +1. API 接口文档。 +2. OpenAPI / Swagger 文档。 +3. 参数说明表。 +4. 错误码说明表。 +5. 部署说明。 +6. 授权说明。 +7. 示例调用代码，至少包括 Python 示例。 +8. 示例数据处理报告。 +9. 结果文件格式说明。 +10. 运维手册。 + +## 14. 验收要求 + +### 14.1 支撑性预处理验收 + +如供应商提供独立预处理 API，使用至少 1 景 LT-1 数据完成预处理，验收项： + +- API 可提交任务。 +- 任务状态可查询。 +- 日志可查询。 +- 输出 GeoTIFF 可被 GDAL/QGIS 打开。 +- 输出包含 CRS 和 GeoTransform。 +- 系统可读取结果并生成预览。 + +### 14.2 D-InSAR 验收 + +使用至少 1 组 LT-1 主辅影像完成 D-InSAR，验收项： + +- API 可提交任务。 +- 可输出 LOS 形变 GeoTIFF。 +- 可输出相干性 GeoTIFF。 +- 可输出解缠相位或差分相位产品。 +- 可输出处理日志。 +- 结果可进入现有系统 D-InSAR 结果管理。 +- 任务失败时错误码明确。 + +### 14.3 SBAS-InSAR 验收 + +使用不少于 3 景 LT-1 数据完成 SBAS-InSAR 样例处理。若供应商建议更高的最小景数，应按供应商推荐值提供样例数据和验收结果。 + +验收项： + +- API 可提交异步 SBAS-InSAR 任务。 +- 任务状态、阶段进度和日志可查询。 +- 可设置时间基线、垂直基线、多视参数和干涉网络策略。 +- 可输出 LOS 时序、速率、累计形变或供应商算法定义的主产品。 +- 输出结果语义明确，包括单位、正负号方向、日期映射、参考点和质量指标。 +- 输出 GeoTIFF 或点矢量结果可被 GDAL/QGIS 读取。 +- 结果可进入现有系统 SBAS-InSAR 结果 catalog。 +- 任务失败、结果缺失、输入不足、DEM 缺失和授权异常时错误码明确。 + +### 14.4 PS-InSAR 可选验收 + +如采购包含 PS-InSAR： + +- 使用供应商建议的最小 LT-1 数据景数完成 PS 样例处理。 +- 输出结果语义明确。 +- 输出点/栅格产品可被 GIS 软件打开。 +- 系统能归档结果和日志。 + +## 15. 供应商需确认问题 + +请供应商在报价或技术响应中明确回答以下问题： + +1. API 服务是 HTTP、消息队列，还是二者都支持？ +2. API 服务是否可本地离线部署？ +3. 默认监听端口是多少，是否可配置？ +4. 是否提供 OpenAPI / Swagger 文档？ +5. 是否支持 LT-1 原始产品直接输入？ +6. 是否支持已导入的 `LT1*_SLC.xml + LT1*_SLC.tif` 输入？ +7. 预处理是否包含正射校正或地理编码？ +8. D-InSAR 输出的 LOS 形变单位和正负号约定是什么？ +9. 是否支持垂直向形变？ +10. 是否支持 GACOS 或其他大气改正？ +11. SBAS-InSAR 是否已产品化支持 LT-1？最小建议景数是多少？ +12. PS-InSAR 是否已产品化支持 LT-1？ +13. 支持的最大并发任务数是多少？ +14. 是否支持任务取消？ +15. 是否支持断点续跑？ +16. 是否支持服务重启后恢复任务状态？ +17. 授权服务如何部署，是否支持无加密狗本地授权服务？ +18. 授权模块是否区分预处理、D-InSAR、SBAS、PS？ +19. 是否提供示例数据和验收报告？ +20. 是否提供二次开发技术支持？ + +## 16. 采购实施建议 + +建议采用“两个必选服务模块、分阶段实施验收”的方式推进： + +### 第一阶段：D-InSAR 服务接入 + +必须交付： + +- LandSAR API 服务。 +- LT-1 D-InSAR API。 +- 任务状态、日志、结果和错误码接口。 +- D-InSAR 示例数据验收。 + +### 第二阶段：SBAS-InSAR + +必须交付： + +- LT-1 SBAS-InSAR API。 +- 多景时序输入。 +- 时序结果输出。 +- 结果语义说明和系统入库适配。 +- SBAS-InSAR 示例数据验收。 + +### 第三阶段：PS-InSAR + +可选扩展： + +- LT-1 PS-InSAR API。 +- PS 点结果输出。 +- 点位时序曲线输出。 +- 监测点分析接口。 + +## 17. 本系统侧预计改造内容 + +采购 LandSAR API 服务后，现有系统侧需要进行以下改造： + +1. 新增 LandSAR API 客户端模块。 +2. 新增 API 服务健康检查。 +3. 新增 D-InSAR 服务版处理器分支。 +4. 新增 SBAS-InSAR 服务版处理器分支。 +5. 新增 LandSAR API job_id 与系统 task_id/run_id 的绑定关系。 +6. 新增 LandSAR 结果 manifest 解析。 +7. 新增错误码映射。 +8. 新增前端参数页和任务监控展示。 +9. 新增中间文件清理策略。 +10. 按 `processor_code` 适配 D-InSAR 与 SBAS-InSAR 结果 catalog。 + +## 18. 当前建议结论 + +本次采购应要求供应商交付“可本地部署、可 API 调用、可异步任务化”的 LandSAR 服务版 D-InSAR 与 SBAS-InSAR 两个模块。D-InSAR 与 SBAS-InSAR 均应作为强制响应项和验收项，PS-InSAR 可作为可选扩展项。 + +原因是： + +- 现有系统已经具备 D-InSAR 和 SBAS-InSAR 生产调度、结果 catalog 和前端展示入口。 +- D-InSAR 是成对影像生产流程，结果需进入现有 D-InSAR 结果管理。 +- SBAS-InSAR 是多景时序生产流程，结果需进入现有 SBAS-InSAR 结果 catalog。 +- 两类服务应共享统一任务状态、日志、错误码、结果 manifest 和授权健康检查接口。 +- PS-InSAR 输出形态和业务展示方式与 SBAS/D-InSAR 差异较大，可在 D-InSAR 与 SBAS-InSAR 服务稳定后扩展。 + +建议招标时将“D-InSAR API + SBAS-InSAR API + 服务部署授权 + 任务状态/日志/结果/错误码接口”列为必须响应项，将“独立预处理 API、PS-InSAR API、GPU 加速、断点续跑、第三方消息队列”列为可选响应项。 diff --git a/docs/SBAS_INSAR_CURRENT_WORKFLOW.md b/docs/SBAS_INSAR_CURRENT_WORKFLOW.md index 313940a..4c96ea7 100644 --- a/docs/SBAS_INSAR_CURRENT_WORKFLOW.md +++ b/docs/SBAS_INSAR_CURRENT_WORKFLOW.md @@ -139,6 +139,8 @@ AOI 模式处理逻辑： `center_bucket` 和 `receiving_station` 是内部诊断字段，不作为用户生产入口展示。 +2026-06-09 修正：候选序列发现不再把 `center_bucket` 作为 strict 模式的硬分组字段。此前全库查找会按中心点小格拆成大量小序列，1500 景 LT-1 数据里最大候选只有 7 景；现在 strict 和 aoi 都先按同卫星/同模式/同相对轨道/同方向/同极化形成观测组，再按 footprint 公共重叠聚类。实际验证：全库 LT-1 候选从最大 7 景恢复到 32/29/28 景级别；牡丹江 AOI 候选也能返回 32/29/26 景级别。 + ## 6. 产物契约 一次完成的 SBAS Run 是一个结果产品包。核心资产包括： diff --git a/docs/SBAS_INSAR_GAMMA_EXPERT_WORKFLOW_REVIEW_20260603.md b/docs/SBAS_INSAR_GAMMA_EXPERT_WORKFLOW_REVIEW_20260603.md new file mode 100644 index 0000000..7b3c1b1 --- /dev/null +++ b/docs/SBAS_INSAR_GAMMA_EXPERT_WORKFLOW_REVIEW_20260603.md @@ -0,0 +1,454 @@ +# SBAS-InSAR Gamma 实现与专家文档对照审阅记录 + +审阅日期：2026-06-03 +审阅对象： + +- 专家文档：`D:\Code\Insar_management_system_v2\LT1_GAMMA_SBAS_逐命令处理流程.docx` +- 当前实现：`backend/app/services/sbas_insar_production_service.py` +- 当前入库：`backend/app/services/sbas_insar_catalog_service.py` +- 样本 Run：`backend/runtime/sbas_insar_production/runs/sbas_7537cc71c998` + +## 1. 审阅结论 + +当前 SBAS-InSAR Gamma 实现不能视为专家文档的逐命令复刻。 + +它更接近“参考专家文档后形成的 Gamma/PyINT 混合生产链路”。这条链路曾跑出完整产物，但从专家文档一致性、结果验收语义和空间范围表达看，存在需要优先修正的问题。专家反馈“结果和实现路径有问题”，从代码和样本产物看是有依据的。 + +核心判断： + +1. 生产坐标系中的 RDC/SAR 栅格本身是矩形。 +2. 系统展示和入库使用的是地理坐标 bbox/GeoTIFF，而不是 SAR 坐标矩形。 +3. 当前把“发现阶段影像范围”“最终地理编码外包矩形”“最终有效像元范围”混在一起，容易造成范围和质量误判。 +4. 样本 Run 的 DEM 没有覆盖完整 stack bbox，但仍被标记为 ready，这是结果边缘异常和黑边风险的直接原因。 + +## 2. 专家文档主流程 + +专家文档的 LT-1 Gamma SBAS 主链路可以概括为： + +```text +par_LT1_SLC +ORB_filt_spline.py +multi_look +dem_import / fill_gaps +gc_map2 / pixel_area / create_diff_par / offset_pwrm / offset_fitm / gc_map_fine / geocode +create_offset / init_offset_orbit / init_offset / offset_pwr / offset_fit / SLC_interp +mk_mli_all +base_calc / base_plot / mk_diff_2d +mk_adf_2d / ave_image / rascc_mask / mk_unw_2d +quad_fit / quad_sub / atm_mod_2d / fill_gaps / atm_sim_2d / sub_phase +mb / unw_to_cpx / unw_model / mb / mb +replace_values / mask_data / dispmap / ts_rate +geocode_back / data2geotiff / disp_prt_2d +``` + +专家文档强调正式运行前必须用实际数据替换日期、极化、宽度、行数、DEM 宽度、种子点、阈值和小基线阈值，并用 `grep`、`SLC_corners`、显示检查命令进行核对。 + +## 3. 当前实现主流程 + +当前系统实现的主要阶段是： + +```text +01_baseline_audit.sh +02_coreg_common_ref.sh +03_prepare_rdc_dem.sh +04_diff_unwrap_common_ref.sh +05_detrend_atm.sh +05_mb_ts_rate.sh +07_publish_products.sh +08_point_timeseries.sh +``` + +实现入口集中在： + +- `backend/app/services/sbas_insar_production_service.py` +- `backend/app/services/job_handlers.py` +- `deploy/wsl/runners/gamma_sbas_product_tools.py` + +当前文档化的 stage 名称与专家文档相近，但部分脚本内部命令不是专家文档原命令序列。 + +## 4. 主要问题 + +### 4.1 DEM 允许只覆盖中心点，不强制覆盖完整 stack + +严重级别：高 + +当前 DEM 选择逻辑允许以下任一条件成立即保留 DEM： + +```python +self._bbox_contains(coverage, stack_bbox, margin_degrees=0.05) +or self._bbox_contains_point(coverage, self._stack_center(stack_manifest), margin_degrees=0.05) +``` + +位置： + +- `backend/app/services/sbas_insar_production_service.py::_resolve_rdc_dem_source` + +样本 Run `sbas_7537cc71c998` 的证据： + +```json +"covers_stack_bbox": false, +"covers_stack_center": true, +"stack_bbox": { + "min_lon": 128.7690438245, + "min_lat": 43.7486321624, + "max_lon": 129.6293024728, + "max_lat": 44.3582486206 +} +``` + +同一 Run 的 DEM coverage： + +```json +"coverage": { + "min_lon": 127.99998768, + "max_lon": 130.99998756, + "min_lat": 44.00000064, + "max_lat": 46.000000560000004 +} +``` + +也就是说，stack 南界到 `43.7486`，DEM 南界只到约 `44.0000`。这会造成南侧边缘缺失、NoData、黑边或地理编码结果范围不足。 + +当前 summary 仍显示： + +```json +"ready": true +``` + +这是验收逻辑漏洞。 + +建议： + +- DEM 选择必须强制 `covers_stack_bbox=true`，并增加安全缓冲。 +- 如果 DEM 不覆盖完整 stack，应直接阻断 RDC DEM 阶段，不允许标记 ready。 +- `selection_note` 不应写“covering the SBAS stack extent”，除非确实覆盖完整 stack bbox。 + +### 4.2 干涉和解缠命令链与专家文档不一致 + +严重级别：高 + +专家文档： + +```text +mk_diff_2d +mk_adf_2d +ave_image +rascc_mask +mk_unw_2d +``` + +当前实现： + +```text +create_offset +phase_sim_orb +SLC_diff_intf +adf +cc_wave +rascc_mask +mcf +``` + +位置： + +- `backend/app/services/sbas_insar_production_service.py::_write_interferogram_script` + +这不是简单命令名称不同，而是处理策略不同。当前链路可能可以跑通，但不能直接说“与专家逐命令流程一致”。如果专家按文档检查结果，当前实现路径会对不上。 + +建议： + +- 保留当前混合链路时，应将 profile 标记为 `lt1_gamma_sbas_hybrid` 或 `experimental`。 +- 新增严格专家链路 profile，例如 `lt1_gamma_sbas_expert_v1`，按专家文档生成 `mk_diff_2d/mk_adf_2d/mk_unw_2d` 脚本。 + +### 4.3 SBAS 反演缺少专家文档中的二次修正链路 + +严重级别：高 + +专家文档在第一次 `mb` 后包含： + +```text +unw_to_cpx +unw_model +mb +mb +``` + +当前实现基本是： + +```text +mb +ts_rate +``` + +位置： + +- `backend/app/services/sbas_insar_production_service.py::_write_ipta_timeseries_script` + +这会影响 2π 跳变修正、最终反演稳定性和专家验收一致性。 + +建议： + +- 明确把第一次 `mb`、模型辅助解缠修正、第二次 `mb`、最终 `mb` 分成独立可审计步骤。 +- 每次 `mb` 输出都应记录输入列表、`itab`、参考点、窗口、阈值和输出统计。 + +### 4.4 配准实现不是专家文档的显式逐命令配准 + +严重级别：中高 + +专家文档配准链： + +```text +create_offset +init_offset_orbit +init_offset +offset_pwr +offset_fit +SLC_interp +``` + +当前实现调用： + +```text +SLC_coreg.py --init_offset +``` + +位置： + +- `backend/app/services/sbas_insar_production_service.py::_write_coregistration_script` + +如果 `SLC_coreg.py` 内部等价，仍需要把内部日志和参数展开到系统审计里。否则专家无法按逐命令流程核对。 + +建议： + +- 专家链路 profile 中显式生成 `create_offset/init_offset_orbit/init_offset/offset_pwr/offset_fit/SLC_interp`。 +- 混合链路可以保留 `SLC_coreg.py`，但必须与专家链路区分。 + +### 4.5 DEM 查找表链路与专家文档不一致 + +严重级别：中 + +专家文档： + +```text +dem_import +fill_gaps +gc_map2 +pixel_area +create_diff_par +offset_pwrm +offset_fitm +gc_map_fine +geocode +``` + +当前实现： + +```text +复用已有 Gamma DEM cache +replace_values +gc_map1 +geocode +create_diff_par +init_offsetm +offset_pwrm +offset_fitm +gc_map_fine +geocode +``` + +位置： + +- `backend/app/services/sbas_insar_production_service.py::_write_rdc_dem_script` + +当前方式可能是工程上可行的，但与专家文档命令链不一致；同时 DEM 覆盖检查还存在高风险漏洞。 + +建议： + +- 专家链路 profile 中按文档执行 `dem_import/gc_map2/pixel_area`。 +- 混合链路继续使用 DEM cache 时，必须加强 DEM 覆盖、坐标、分辨率、NoData 验收。 + +### 4.6 入库和展示范围来自 stack 元数据，不是最终产品有效范围 + +严重级别：中 + +当前 geographic coverage 构造来自 stack scenes 的 metadata bbox： + +- `backend/app/services/sbas_insar_production_service.py::_build_stack_geographic_coverage` + +入库时使用该 coverage 写入： + +- `min_lon` +- `min_lat` +- `max_lon` +- `max_lat` +- `geom` +- `coverage_polygon` + +位置： + +- `backend/app/services/sbas_insar_catalog_service.py::_build_product` + +这意味着系统中展示的是“发现阶段影像范围”，不是最终 GeoTIFF 的真实 footprint，更不是最终有效像元 footprint。 + +样本 Run 的最终 GeoTIFF `los_rate_toward_mm_per_year.tif` 信息： + +```text +Size is 966, 435 +Origin = (128.790404333349983,44.362917266649994) +Pixel Size = (0.000833333300000,-0.000833333300000) +Upper Left = (128.7904043, 44.3629173) +Lower Right = (129.5954043, 44.0004173) +NoData Value=0 +``` + +而 stack bbox 是： + +```json +{ + "min_lon": 128.7690438245, + "min_lat": 43.7486321624, + "max_lon": 129.6293024728, + "max_lat": 44.3582486206 +} +``` + +两者明显不同。 + +建议： + +- 入库时从主 GeoTIFF 读取外包矩形、CRS、transform 和 NoData。 +- 另行计算有效像元 footprint，或至少计算有效像元 bbox。 +- 前端明确区分： + - stack metadata footprint + - RDC/SAR processing grid + - geocoded raster extent + - valid-pixel footprint + +### 4.7 质量统计把 0 当成有效值 + +严重级别：中 + +当前 `_gamma_float32_stats` 使用 finite 像元作为 `valid_count`，只额外记录 `nonzero_count`。但 GeoTIFF 明确 `NoData Value=0`，因此黑边或无效像元会被 `valid_count` 掩盖。 + +位置： + +- `backend/app/services/sbas_insar_production_service.py::_gamma_float32_stats` + +样本质量统计： + +```json +"pixel_count": 11480259, +"valid_count": 11480259, +"nonzero_count": 1956770 +``` + +`nonzero_count` 只占约 17%，但 `valid_count` 却是 100%。这会误导验收。 + +建议： + +- 对最终产品统计必须把 NoData 排除。 +- 对 RDC 中间文件可以同时报告： + - finite_count + - nonzero_count + - nodata_count + - valid_pixel_ratio + - valid_bbox +- `ready` 不应只看文件尺寸和 finite 统计。 + +## 5. SAR 坐标矩形问题 + +专家反馈“SAR 坐标应该是矩形”，需要拆成两个层次理解。 + +### 5.1 处理坐标 + +RDC/SAR 处理网格应该是规则矩形。 + +样本 Run 中： + +```json +"reference_geometry": { + "range_samples": 2693, + "azimuth_lines": 4263, + "expected_float32_bytes": 45921036 +} +``` + +RDC 文件大小匹配 `2693 * 4263 * 4`，说明处理中间产品本身是矩形栅格。 + +### 5.2 展示和入库坐标 + +系统前端和 catalog 当前展示的是 EPSG:4326 的地理 bbox 或 scene bbox，不是 SAR 坐标矩形。 + +地理编码后的 GeoTIFF 是经纬度网格矩形，但有效像元可能因为 SAR 覆盖、DEM 覆盖、查找表外推、NoData 而不是满矩形。前端如果只画地理 bbox，会让用户误以为整块都有效。 + +### 5.3 结论 + +当前问题不是“RDC 文件不是矩形”，而是系统把以下内容混用了： + +1. SAR/RDC 处理矩形。 +2. 原始 scene metadata bbox。 +3. stack 多景 bbox union/intersection。 +4. 最终 GeoTIFF 地理外包矩形。 +5. 最终有效像元 footprint。 + +建议把这些空间语义拆开保存和展示。 + +## 6. 建议整改路线 + +### 第一阶段：先修验收与范围表达 + +目标：不改变核心 Gamma 命令，先避免错误结果被标记为 ready。 + +1. DEM 必须完整覆盖 stack bbox，加缓冲；否则阻断。 +2. `rdc_dem_summary.ready` 必须检查 DEM coverage。 +3. `publish_product_summary` 读取主 GeoTIFF 的真实 extent、NoData、valid pixel ratio。 +4. catalog 入库优先使用主 GeoTIFF extent 和 valid footprint，而不是 stack metadata bbox。 +5. 前端展示拆分为“数据发现范围”和“产品有效范围”。 + +### 第二阶段：建立专家文档严格链路 + +目标：给专家可逐命令审计的生产路径。 + +新增 profile： + +```text +lt1_gamma_sbas_expert_v1 +``` + +特性： + +- 按专家文档 12 节生成脚本。 +- 每节脚本命令、输入、输出、日志与专家文档一一对应。 +- 保留当前混合链路，但命名为 hybrid/experimental，不再和专家链路混称。 + +### 第三阶段：结果质量验收 + +目标：形成可解释的质量结论。 + +建议增加： + +- DEM 覆盖检查。 +- RDC 栅格尺寸检查。 +- GeoTIFF extent 检查。 +- NoData/valid-pixel ratio 检查。 +- 相干性统计。 +- 每对干涉图解缠覆盖率。 +- `mb` 输入层数、有效像元、参考点窗口记录。 +- LOS 速度范围和 sigma 分布阈值告警。 + +## 7. 当前不建议的做法 + +1. 不建议继续把当前链路称为“专家文档逐命令链路”。 +2. 不建议只看产物文件存在和文件大小判断成功。 +3. 不建议用 stack metadata bbox 代表最终产品有效范围。 +4. 不建议把 0 NoData 统计为有效像元。 +5. 不建议在 DEM 未覆盖完整 stack 的情况下继续标记 ready。 + +## 8. 后续需要专家确认的问题 + +1. 是否要求严格使用专家文档中的 `mk_diff_2d/mk_adf_2d/mk_unw_2d`，还是允许保留 `SLC_diff_intf/adf/mcf` 混合链路。 +2. DEM 来源是否必须每次从 GeoTIFF 通过 `dem_import` 重新导入，还是允许复用 Gamma DEM cache。 +3. `mb` 的三次反演和 `unw_model` 修正是否必须进入正式链路。 +4. 参考点 `R_REF/A_REF`、窗口、阈值是否由系统自动选择，还是必须由专家人工确认。 +5. 最终业务展示默认应展示 SAR/RDC 矩形、GeoTIFF extent，还是有效像元 footprint。 + diff --git a/docs/SENTINEL1_GAMMA_SBAS_NO_STITCH_DESIGN.md b/docs/SENTINEL1_GAMMA_SBAS_NO_STITCH_DESIGN.md new file mode 100644 index 0000000..0309717 --- /dev/null +++ b/docs/SENTINEL1_GAMMA_SBAS_NO_STITCH_DESIGN.md @@ -0,0 +1,541 @@ +# Sentinel-1 Gamma SBAS 无拼接接入设计 + +最后更新：2026-06-02 + +本文设计 `sbas-insar-production` 对 Sentinel-1 Gamma SBAS 的稳定接入方案。目标是复用现有 SBAS 生产管理框架，但不影响现有 LT-1 Gamma SBAS 链路；允许传感器专用逻辑冗余实现，以稳定性和可回退为第一优先级。 + +## 1. 术语说明 + +本文目标是 Sentinel-1 SAR 数据，不是 Sentinel-2。 + +Sentinel-2 是光学卫星，不具备 SAR 干涉相位，不能迁移到 Gamma SBAS-InSAR。如果后续业务说“哨兵2”，需要先确认是不是口误；系统实现应使用 `Sentinel-1`、`S1`、`s1_gamma_sbas` 这些明确命名，避免把 Sentinel-2 暗含进 InSAR 链路。 + +## 2. 设计结论 + +当前 Gamma SBAS 核心是 LT-1 专用实现，包含 LT-1 目录扫描、LT-1 元数据解析、LT-1 精轨脚本和 `par_LT1_SLC` 导入脚本。Sentinel-1 不应在这条链上硬改。 + +新增 Sentinel-1 支持时采用 profile 并列方案： + +```text +lt1_gamma_sbas # 现有链路，保持行为不变 +s1_gamma_sbas # 新增链路，独立发现、独立脚本、独立校验 +``` + +复用内容： + +- API 路由和生产 Run 生命周期。 +- Stack discovery / audit / create run 的外层流程。 +- Workflow/job 调度、日志、状态机。 +- Gamma 环境注入和 WSL runtime 管理。 +- 产品发布、catalog、预览、监测点、下载接口。 + +不复用或只抽象复用的内容： + +- 不复用 LT-1 场景扫描。 +- 不复用 LT-1 SLC 导入脚本。 +- 不复用 LT-1 精轨处理脚本。 +- 不复用 LT-1 专家文档中的传感器专有命令。 +- 不在 LT-1 脚本模板中加入 Sentinel-1 分支。 + +## 3. 目标与非目标 + +### 3.1 目标 + +1. 新增 Sentinel-1 Gamma SBAS profile，入口可发现 Sentinel-1 候选 stack。 +2. Sentinel-1 使用 ZIP/SAFE + EOF 资产，不走 LT-1 `tiff/meta.xml/txt orbit` 逻辑。 +3. 不支持拼接。第一阶段只支持单轨、同向、同 relative orbit、同 acquisition mode、同 polarization、同 subswath、同 burst 或同一稳定 burst key 的 stack。 +4. 对需要拼接才能覆盖 AOI 的数据，系统不自动拼接，改为拆成多个独立候选 stack 或直接标记为 `NOT_READY_REQUIRES_STITCHING`。 +5. LT-1 现有生产效果不变，默认入口仍可继续跑现有 LT-1 数据。 +6. Sentinel-1 先做严格、保守、可解释的生产链，允许代码冗余，避免为了共用而引入隐性耦合。 + +### 3.2 非目标 + +1. 不支持 Sentinel-2 光学时序。 +2. 不支持跨轨、跨 relative orbit、升降轨混合。 +3. 不支持跨 swath 拼接。 +4. 不支持跨 burst 拼接。 +5. 不支持相邻 Sentinel-1 slice/frame 自动拼接。 +6. 不把多个独立 Sentinel-1 SBAS 结果镶嵌成一张最终产品。 +7. 不改现有 LT-1 专家文档脚本的含义和输出。 + +## 4. 当前 LT-1 链路中不能直接复用的点 + +当前实现里有多处 LT-1 硬编码： + +```text +_iter_lt1_scene_dirs +_looks_like_lt1_scene_dir +_parse_lt1_scene +par_LT1_SLC +LT1_precision_orbit.py +Prepare LT1 SLCs +LT1_GAMMA_SBAS_逐命令处理流程.docx +layout_source = LT1_GAMMA_SBAS_expert_document +allowed_operations = lt1_gamma_sbas_workflow / lt1_gamma_sbas_step +``` + +这些不应扩展成大量 `if sensor == "S1"` 分支。否则 LT-1 的稳定链路会被 Sentinel-1 的 TOPS/burst 复杂性污染。 + +## 5. 总体架构 + +新增一个传感器 profile 适配层。现有 `SbasInsarProductionService` 保持外层协调角色，传感器专有逻辑下沉到 adapter。 + +建议模块： + +```text +backend/app/services/sbas_profiles/ + __init__.py + base.py + lt1_gamma_sbas_profile.py + s1_gamma_sbas_profile.py + +backend/app/services/sbas_script_templates/ + lt1_gamma_sbas_scripts.py + s1_gamma_sbas_scripts.py +``` + +核心接口建议： + +```python +class GammaSbasProfile: + profile_code: str + sensor_family: str + + def discover_scenes(source_roots, orbit_roots, filters) -> SceneDiscoveryResult: ... + def group_stack_candidates(scenes, aoi, options) -> list[StackCandidate]: ... + def audit_stack(stack_id, context) -> StackAudit: ... + def build_run_manifest(stack, options) -> dict: ... + def build_workflow_manifest(run_dir, run_manifest, options) -> dict: ... + def materialize_scripts(run_dir, workflow_manifest) -> list[ScriptArtifact]: ... + def validate_no_stitch_policy(stack) -> list[Issue]: ... +``` + +LT-1 profile 可以先只是封装现有函数，不改变行为。Sentinel-1 profile 独立实现。 + +## 6. API 和配置 + +### 6.1 API 参数 + +现有接口保持兼容，新增可选 `profile_code`： + +```json +{ + "profile_code": "lt1_gamma_sbas", + "source_roots": [], + "orbit_roots": [], + "admin_region": "", + "discovery_mode": "strict", + "aoi_bbox": null +} +``` + +默认值为 `lt1_gamma_sbas`。这样老前端和老调用不受影响。 + +新增 Sentinel-1 时使用： + +```json +{ + "profile_code": "s1_gamma_sbas", + "source_roots": ["D:\\Sentinel1_Image_Pool_ZIP"], + "orbit_roots": ["D:\\Sentinel1_Orbit_Pool"], + "admin_region": "..." +} +``` + +### 6.2 配置项 + +新增配置建议： + +```text +GAMMA_SBAS_PROFILES=lt1_gamma_sbas,s1_gamma_sbas +GAMMA_SBAS_DEFAULT_PROFILE=lt1_gamma_sbas + +GAMMA_SBAS_S1_ENABLED=false +GAMMA_SBAS_S1_SOURCE_ROOTS=D:\Sentinel1_Image_Pool_ZIP +GAMMA_SBAS_S1_ORBIT_ROOTS=D:\Sentinel1_Orbit_Pool +GAMMA_SBAS_S1_NO_STITCH=true +GAMMA_SBAS_S1_MIN_SCENES=8 +GAMMA_SBAS_S1_DEFAULT_SUBSWATH=IW2 +GAMMA_SBAS_S1_DEFAULT_POLARIZATION=VV +``` + +`GAMMA_SBAS_S1_ENABLED` 初始应为 `false`。完成样本验证后再开放。 + +### 6.3 Runtime 白名单 + +`wsl_runtime_registry.py` 需要新增 operation： + +```text +s1_gamma_sbas_workflow +s1_gamma_sbas_step +``` + +不要复用 `lt1_gamma_sbas_workflow` 的 operation 名称。 + +## 7. Sentinel-1 数据发现与分组 + +### 7.1 数据来源 + +优先复用资产库存层： + +- Sentinel-1 ZIP / SAFE 源产品资产。 +- Sentinel-1 EOF 精密轨道资产。 +- `logical_product_uid` 关联 ZIP 和 SAFE。 +- EOF 使用 validity window 匹配 scene。 + +如果资产库存不可用，S1 profile 可提供只读目录扫描兜底，但目录扫描结果必须写入同样的 `SceneDescriptor` 结构。 + +### 7.2 SceneDescriptor + +Sentinel-1 scene 描述结构至少包含： + +```json +{ + "sensor_family": "S1", + "satellite": "S1A", + "product_type": "SLC", + "acquisition_mode": "IW", + "polarization": "VV", + "orbit_direction": "ASCENDING", + "relative_orbit": "40", + "absolute_orbit": "...", + "start_time_utc": "...", + "stop_time_utc": "...", + "source_archive_path": "...zip", + "safe_dir": "...SAFE", + "manifest_path": "...manifest.safe", + "orbit_file_path": "...EOF", + "footprint": {}, + "available_subswaths": ["IW1", "IW2", "IW3"], + "burst_index_summary": {} +} +``` + +### 7.3 Stack 分组规则 + +Sentinel-1 stack candidate 必须满足： + +1. 同 `sensor_family = S1`。 +2. 同 `acquisition_mode = IW`。 +3. 同 `orbit_direction`。 +4. 同 `relative_orbit`。 +5. 同 polarization，第一阶段建议只支持 `VV`。 +6. 所有 scene 都有匹配 EOF。 +7. 所有 scene 与 AOI 有交集。 +8. 能解析出共同 subswath 和 burst key。 +9. 不需要跨 swath/burst/相邻 slice 拼接。 + +不满足第 8、9 条时，不应尝试自动修复，直接输出： + +```text +status = NOT_READY_REQUIRES_STITCHING +``` + +或者拆成多个候选： + +```text +s1_rel040_asc_iw2_burst_013 +s1_rel040_asc_iw2_burst_014 +``` + +每个候选独立生产，不做最终合成。 + +## 8. 无拼接策略 + +本文中的“不支持拼接”定义如下： + +1. 不把相邻 Sentinel-1 产品 slice 合成一个输入。 +2. 不把多个 subswath 合成一个输入。 +3. 不把多个 burst 的结果合成一个输出。 +4. 不把多个独立 SBAS run 的 GeoTIFF 合成一个产品。 + +第一阶段最稳策略是 `single_subswath_single_burst`： + +```text +stack geometry = relative orbit + direction + IW subswath + burst key +``` + +如果 AOI 跨多个 burst，系统给出多个独立候选。用户可以分别生产和查看，但系统不拼接。 + +这样牺牲覆盖范围，但能显著降低 TOPS 拼接、边界相位、burst overlap 和几何一致性的风险。 + +## 9. Sentinel-1 Workflow 阶段 + +新增 Sentinel-1 专用 workflow 模板，不修改 LT-1 模板。 + +建议阶段： + +```text +01_workspace_data +02_import_s1_slc +03_select_single_burst +04_reference_mli +05_baseline_audit +06_coregister_scenes +07_rdc_dem +08_diff_network +09_filter_unwrap +10_detrend_atm +11_sbas_inversion +12_publish_products +13_monitor_points +``` + +与 LT-1 的主要差异在前半段： + +- LT-1：`par_LT1_SLC` + LT-1 txt 精轨。 +- S1：ZIP/SAFE + EOF + TOPS/burst 选择。 + +后半段可复用 Gamma DIFF/IPTA 的思想，但脚本仍建议独立生成，避免 LT-1 和 S1 共用同一个 shell 模板。 + +## 10. 脚本隔离设计 + +LT-1 当前脚本路径保持不变： + +```text +scripts/01_workspace_data.sh +scripts/02_import_lt1_slc.sh +... +``` + +Sentinel-1 使用独立命名： + +```text +scripts/s1/01_workspace_data.sh +scripts/s1/02_import_s1_slc.sh +scripts/s1/03_select_single_burst.sh +scripts/s1/04_reference_mli.sh +... +``` + +`run_manifest.json` 中明确记录： + +```json +{ + "profile_code": "s1_gamma_sbas", + "sensor_family": "S1", + "stitching_policy": "disabled", + "stack_geometry_policy": "single_subswath_single_burst" +} +``` + +## 11. 产物与 Catalog + +Sentinel-1 产物仍进入 SBAS catalog，但必须带 profile 和 sensor 标签： + +```json +{ + "catalog_name": "sbas_insar", + "product_family": "timeseries", + "processor_code": "gamma_ipta_sbas", + "profile_code": "s1_gamma_sbas", + "sensor_family": "S1" +} +``` + +核心资产仍保持现有约定： + +```text +publish/geotiff/los_rate_toward_m_per_year.tif +publish/geotiff/los_rate_away_m_per_year.tif +publish/geotiff/los_sigma_m_per_year.tif +publish/geotiff/los_rate_toward_m_per_year.hls.geo_preview.png +publish/geotiff/los_sigma_m_per_year.cc.geo_preview.png +publish/vectors/los_rate_points.geojson.gz +publish/monitor_points/* +``` + +产品目录建议按 profile 分层，避免和 LT-1 混在一起： + +```text +D:\production_results\timeseries\sbas +|-- lt1_gamma_sbas +| `-- <run_id> +`-- s1_gamma_sbas + `-- <run_id> +``` + +如果短期不改目录，也必须在 manifest/catalog 中保留 `profile_code`，前端筛选时不能只看 product family。 + +## 12. 前端设计 + +`SBAS-InSAR Production` 增加 profile 选择： + +```text +数据类型: +[ LT-1 Gamma SBAS ] [ Sentinel-1 Gamma SBAS ] +``` + +默认仍是 LT-1。 + +Sentinel-1 页面提示： + +```text +当前 Sentinel-1 Gamma SBAS 使用无拼接策略。 +仅支持同轨同向、同 relative orbit、同 subswath、同 burst 的稳定候选序列。 +跨 burst / 跨 subswath / 相邻 slice 自动拼接暂不支持。 +``` + +候选列表增加字段： + +- sensor family +- satellite +- acquisition mode +- relative orbit +- orbit direction +- polarization +- subswath +- burst key +- stitching policy +- missing EOF count + +如果候选需要拼接，按钮置灰，原因显示为 `需要拼接，当前策略不支持`。 + +## 13. 稳定性护栏 + +### 13.1 不影响 LT-1 的护栏 + +1. 默认 profile 不变。 +2. LT-1 常量、模板、脚本文件名不改。 +3. Sentinel-1 代码放到新 adapter 和新脚本模板中。 +4. `GAMMA_SBAS_S1_ENABLED=false` 时前端不展示 S1。 +5. LT-1 的单元测试和脚本快照测试必须先通过。 + +### 13.2 Sentinel-1 提交前校验 + +提交生产前必须全部通过： + +```text +source ZIP/SAFE exists +EOF exists for every scene +same relative orbit +same orbit direction +same acquisition mode +same polarization +same subswath +same burst key +scene count >= minimum +no stitching required +DEM coverage exists +AOI intersects all selected scenes +``` + +任一失败，禁止提交 workflow job。 + +### 13.3 执行阶段校验 + +每阶段输出必须有 manifest 记录： + +```text +stage_status.json +stage stdout/stderr log +expected outputs +missing outputs +quality flags +``` + +Sentinel-1 首批样本不自动发布到正式 catalog。建议先生成 run artifact，人工确认后再开启 publish。 + +## 14. 实施阶段 + +### Phase 0：文档和开关 + +- 新增本文档。 +- 新增配置项设计。 +- S1 默认关闭。 + +### Phase 1：Profile 框架拆分 + +- 新增 `GammaSbasProfile` 基类。 +- 当前 LT-1 逻辑包一层 `Lt1GammaSbasProfile`。 +- 保证 LT-1 行为不变。 +- 增加 LT-1 脚本快照测试。 + +### Phase 2：Sentinel-1 Discovery + +- 实现 `S1GammaSbasProfile.discover_scenes`。 +- 优先读取 source/orbit asset inventory。 +- 输出 S1 stack candidates。 +- 对需要拼接的候选输出 `NOT_READY_REQUIRES_STITCHING`。 + +### Phase 3：Sentinel-1 Planning Run + +- 能创建 `s1_gamma_sbas` planning run。 +- 能生成 `run_manifest.json`、`stack_manifest.json`、`gamma_command_manifest.json`。 +- 只生成脚本，不执行 Gamma。 + +### Phase 4：Sentinel-1 Script Dry Run + +- 生成 `scripts/s1/*.sh`。 +- 在样本数据上执行到导入和 reference MLI。 +- 验证 no-stitch 限制是否真实有效。 + +### Phase 5：Sentinel-1 Workflow 执行 + +- 执行 coreg、RDC DEM、diff、unwrap、detrend、IPTA。 +- 先不自动发布正式 catalog。 +- 输出质量报告和人工验收包。 + +### Phase 6：Catalog 发布和前端展示 + +- Sentinel-1 样本通过后开启 publish。 +- 前端结果页支持 profile/sensor 筛选。 +- 产品详情显示 `stitching_policy=disabled`。 + +## 15. 验收标准 + +### 15.1 LT-1 回归 + +1. LT-1 discovery 结果不变。 +2. LT-1 run manifest 关键字段不变。 +3. LT-1 脚本输出与改造前一致。 +4. LT-1 样本能继续跑通。 +5. LT-1 catalog 结果不受 S1 profile 影响。 + +### 15.2 Sentinel-1 最小可用 + +1. 能发现 Sentinel-1 候选 stack。 +2. 能绑定 EOF。 +3. 能拒绝需要拼接的候选。 +4. 能生成 S1 planning run 和脚本。 +5. 能在一个单 subswath / 单 burst 样本上跑通 workflow。 +6. 输出标准 SBAS 产品资产。 +7. Catalog 中能按 `profile_code=s1_gamma_sbas` 查询。 + +## 16. 风险与取舍 + +### 16.1 覆盖范围变小 + +不拼接意味着 AOI 覆盖能力会变弱。跨 burst 或跨 subswath 的区域不会自动合成，只能拆成多个独立结果查看。 + +这是稳定性优先的取舍。 + +### 16.2 Sentinel-1 TOPS 复杂度高 + +Sentinel-1 TOPS 的 burst、Doppler、coregistration 对脚本稳定性要求高。第一阶段不应追求通用覆盖，应先让单一稳定样本跑通。 + +### 16.3 Gamma 命令版本差异 + +Sentinel-1 导入和 TOPS 处理命令需要以生产服务器安装的 Gamma 版本为准。脚本模板中所有 Sentinel-1 命令必须经过样本数据验证后再开放。 + +### 16.4 不共享脚本导致代码冗余 + +本设计接受冗余。相比强行抽象共用，冗余脚本更容易保证 LT-1 不被影响，也更容易单独回滚 Sentinel-1。 + +## 17. 回滚策略 + +如果 Sentinel-1 profile 出现问题： + +1. 设置 `GAMMA_SBAS_S1_ENABLED=false`。 +2. 前端隐藏 Sentinel-1 profile。 +3. Runtime 白名单保留不影响 LT-1。 +4. 已生成的 S1 run 保留为实验记录，不进入正式 catalog。 +5. LT-1 profile 不需要回滚。 + +## 18. 推荐下一步 + +1. 先实现 profile 拆分，但不改 LT-1 行为。 +2. 准备一个 Sentinel-1 最小样本集：同 relative orbit、同方向、同 polarization、同 subswath、同 burst key，至少 8 景。 +3. 只实现 discovery + planning run。 +4. 人工核对 `gamma_command_manifest.json` 和 `scripts/s1/*.sh`。 +5. 再进入 Gamma 实际执行阶段。 diff --git a/frontend/package-lock.json b/frontend/package-lock.json index ca79e8e..844279f 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -12,6 +12,7 @@ "chart.js": "^4.5.0", "chartjs-adapter-date-fns": "^3.0.0", "date-fns": "^4.1.0", + "echarts": "^6.1.0", "flatpickr": "^4.6.13", "html2canvas": "^1.4.1", "leaflet": "^1.9.4", @@ -2057,6 +2058,16 @@ "node": ">= 0.4" } }, + "node_modules/echarts": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/echarts/-/echarts-6.1.0.tgz", + "integrity": "sha512-q0yaFPggC9FUdsWH4blavRWFmxdrIodbkoKNAjJudAI6CA9gNPxHtV2RcZNEepZVlk4yvBYkOkbk6HIVpIyHZA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "2.3.0", + "zrender": "6.1.0" + } + }, "node_modules/electron-to-chromium": { "version": "1.5.227", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.227.tgz", @@ -4179,6 +4190,12 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/tslib": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.0.tgz", + "integrity": "sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg==", + "license": "0BSD" + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -4478,6 +4495,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/zrender": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/zrender/-/zrender-6.1.0.tgz", + "integrity": "sha512-oEGMDB6pOP2S6OwRR4PdVv610zrjnA3Bh+JnSG12fYJlBKjtNAoEb5fSUoCOOINlH96I2fU38/A2UpRKs67xYQ==", + "license": "BSD-3-Clause", + "dependencies": { + "tslib": "2.3.0" + } + }, "node_modules/zustand": { "version": "5.0.11", "resolved": "https://registry.npmjs.org/zustand/-/zustand-5.0.11.tgz", diff --git a/frontend/package.json b/frontend/package.json index 554325b..eda96ba 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -14,6 +14,7 @@ "chart.js": "^4.5.0", "chartjs-adapter-date-fns": "^3.0.0", "date-fns": "^4.1.0", + "echarts": "^6.1.0", "flatpickr": "^4.6.13", "html2canvas": "^1.4.1", "leaflet": "^1.9.4", diff --git a/frontend/src/AiAnalysisPanel.jsx b/frontend/src/AiAnalysisPanel.jsx index 1d0bad8..7659001 100644 --- a/frontend/src/AiAnalysisPanel.jsx +++ b/frontend/src/AiAnalysisPanel.jsx @@ -9,6 +9,8 @@ import { } from './api/ai'; import { getDinsarResults } from './api/dinsar'; import AiDiagnosisModal from './components/AiDiagnosisModal'; +import TaskStatusPanel from './components/tasks/TaskStatusPanel'; +import useTaskMonitor from './hooks/useTaskMonitor'; const cardStyle = { background: '#fff', @@ -20,6 +22,12 @@ const cardStyle = { export default function AiAnalysisPanel({ readOnly = false, onJobQueued }) { const { en } = useI18n(); + const aiTaskMonitor = useTaskMonitor({ + taskTypes: ['AI_ANALYZE'], + showRecent: true, + recentLimit: 1, + pollRecentMs: 10000, + }); // 状态 const [aiStatus, setAiStatus] = useState(null); @@ -209,6 +217,16 @@ export default function AiAnalysisPanel({ readOnly = false, onJobQueued }) { {en ? 'Create Diagnosis' : '创建诊断'} </h3> + <TaskStatusPanel + title={en ? 'AI Diagnosis Task' : 'AI 诊断任务'} + activeTasks={aiTaskMonitor.activeTasks} + recentTasks={aiTaskMonitor.recentTasks} + latestTask={aiTaskMonitor.latestTask} + isBusy={aiTaskMonitor.isBusy} + idleText={en ? 'No AI diagnosis task is running.' : '当前没有正在执行的 AI 诊断任务。'} + compact + /> + {/* D-InSAR Result Selection */} <div style={{ marginBottom: '12px' }}> <div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', marginBottom: '4px' }}> @@ -345,20 +363,20 @@ export default function AiAnalysisPanel({ readOnly = false, onJobQueued }) { {/* Submit Button */} <button onClick={handleCreateDiagnosis} - disabled={loading || !selectedResultId || !aiStatus?.ollama_online} + disabled={loading || aiTaskMonitor.isBusy || !selectedResultId || !aiStatus?.ollama_online} style={{ width: '100%', padding: '8px', - backgroundColor: loading || !selectedResultId || !aiStatus?.ollama_online ? '#cbd5e0' : '#3182ce', + backgroundColor: loading || aiTaskMonitor.isBusy || !selectedResultId || !aiStatus?.ollama_online ? '#cbd5e0' : '#3182ce', color: '#fff', border: 'none', borderRadius: '4px', fontSize: '14px', fontWeight: 500, - cursor: loading || !selectedResultId || !aiStatus?.ollama_online ? 'not-allowed' : 'pointer', + cursor: loading || aiTaskMonitor.isBusy || !selectedResultId || !aiStatus?.ollama_online ? 'not-allowed' : 'pointer', }} > - {loading ? (en ? 'Creating...' : '创建中...') : (en ? 'Create Diagnosis' : '创建诊断')} + {loading || aiTaskMonitor.isBusy ? (en ? 'Creating...' : '创建中...') : (en ? 'Create Diagnosis' : '创建诊断')} </button> {/* Message */} diff --git a/frontend/src/App.css b/frontend/src/App.css index 3daffb6..8a13ab3 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -1795,31 +1795,62 @@ input[type="checkbox"] { 100% { transform: rotate(360deg); } } -/* 全局任务锁定遮罩样式 */ +/* Global task center */ .global-task-overlay { position: fixed; - top: 0; - left: 0; - width: 100vw; - height: 100vh; - background-color: rgba(15, 23, 42, 0.75); - backdrop-filter: blur(6px); - z-index: 9999; - display: flex; - justify-content: center; + right: 18px; + bottom: 18px; + z-index: 1200; + color: #0f172a; + pointer-events: none; +} + +.task-center-button { + pointer-events: auto; + display: inline-flex; align-items: center; - color: white; + gap: 8px; + min-height: 38px; + padding: 8px 12px; + border: 1px solid #cbd5e1; + border-radius: 8px; + background: #ffffff; + color: #0f172a; + box-shadow: 0 8px 24px rgba(15, 23, 42, 0.18); + font-size: 12px; + cursor: pointer; +} + +.task-center-button strong { + color: #2563eb; + font-family: var(--font-mono); +} + +.task-center-dot { + width: 8px; + height: 8px; + border-radius: 999px; + background: #2563eb; + box-shadow: 0 0 0 4px rgba(37, 99, 235, 0.12); + animation: pulse 1.6s ease-in-out infinite; +} + +@keyframes pulse { + 0%, 100% { opacity: 0.55; } + 50% { opacity: 1; } } .overlay-content { - background: linear-gradient(180deg, #0f172a 0%, #111827 100%); - padding: 40px; - border-radius: 12px; - width: 500px; - max-width: 90%; - text-align: center; - box-shadow: 0 24px 40px rgba(15, 23, 42, 0.45); - border: 1px solid rgba(148, 163, 184, 0.2); + pointer-events: auto; + background: #ffffff; + padding: 14px; + border-radius: 8px; + width: min(440px, calc(100vw - 36px)); + max-height: min(70vh, 620px); + overflow: auto; + text-align: left; + box-shadow: 0 18px 44px rgba(15, 23, 42, 0.24); + border: 1px solid #cbd5e1; } .loading-spinner-large { @@ -1832,25 +1863,52 @@ input[type="checkbox"] { margin: 0 auto 20px; } -.overlay-content h3 { - margin: 0 0 25px 0; - font-size: 1.5rem; - letter-spacing: 1px; +.task-center-header { + display: flex; + justify-content: space-between; + gap: 12px; + align-items: flex-start; + margin-bottom: 12px; +} + +.task-center-header h3 { + margin: 0; + font-size: 15px; + color: #0f172a; +} + +.task-center-header p { + margin: 4px 0 0; + font-size: 12px; + line-height: 1.5; + color: #64748b; +} + +.task-center-close { + width: 28px; + height: 28px; + border: 1px solid #cbd5e1; + border-radius: 6px; + background: #f8fafc; + color: #334155; + cursor: pointer; + font-size: 18px; + line-height: 1; } .active-tasks-container { display: flex; flex-direction: column; - gap: 20px; - margin-bottom: 30px; + gap: 10px; + margin-bottom: 12px; text-align: left; } .task-progress-item { - background: rgba(148, 163, 184, 0.12); - padding: 15px; + background: #f8fafc; + padding: 10px; border-radius: 8px; - border: 1px solid rgba(148, 163, 184, 0.18); + border: 1px solid #e2e8f0; } .task-info-row { @@ -1862,7 +1920,7 @@ input[type="checkbox"] { .task-label { font-weight: 600; - color: #94a3b8; + color: #334155; } .task-percent { @@ -1872,8 +1930,8 @@ input[type="checkbox"] { } .task-progress-bar { - height: 8px; - background-color: rgba(148, 163, 184, 0.35); + height: 7px; + background-color: #e2e8f0; border-radius: 4px; overflow: hidden; margin-bottom: 8px; @@ -1889,7 +1947,7 @@ input[type="checkbox"] { .task-status-msg { margin: 0; font-size: 12px; - color: #e2e8f0; + color: #64748b; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; @@ -1897,7 +1955,7 @@ input[type="checkbox"] { .overlay-footer-hint { font-size: 12px; - color: #94a3b8; + color: #64748b; margin: 0; line-height: 1.5; } diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index a587e2c..632c19b 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -11,6 +11,7 @@ import AppSidePanel from './components/app/AppSidePanel'; import AppStatusHeader from './components/app/AppStatusHeader'; import { useI18n } from './i18n/I18nContext'; import apiClient from './api/client'; +import { getSbasInsarProductAssetUrl } from './api/sbasInsarProducts'; import { useAuthStore, useTaskStore, useUiStore, useRadarStore, useDinsarStore, useBatchStore, usePairingStore, useHazardStore, useMapStore, @@ -47,6 +48,138 @@ import { DINSAR_ENGINE_ALL, getDinsarEngineMeta } from './utils/dinsarEngines'; const NATIONAL_BOUNDARY_STATIC_URL = '/geojson/\u5168\u56fd\u884c\u653f\u533a.geojson'; +const toFiniteNumber = (value) => { + const numeric = Number(value); + return Number.isFinite(numeric) ? numeric : null; +}; + +const formatMapNumber = (value, digits = 2) => { + const numeric = Number(value); + return Number.isFinite(numeric) ? numeric.toFixed(digits) : '-'; +}; + +const getSbasProductBounds = (product) => { + const coverage = product?.geographic_coverage || {}; + const bbox = coverage.bbox || {}; + const minLon = toFiniteNumber(product?.min_lon ?? bbox.min_lon); + const minLat = toFiniteNumber(product?.min_lat ?? bbox.min_lat); + const maxLon = toFiniteNumber(product?.max_lon ?? bbox.max_lon); + const maxLat = toFiniteNumber(product?.max_lat ?? bbox.max_lat); + if ([minLon, minLat, maxLon, maxLat].some(value => value === null)) return null; + if (minLon >= maxLon || minLat >= maxLat) return null; + return [[minLat, minLon], [maxLat, maxLon]]; +}; + +const findSbasAsset = (detail, roles) => { + const roleSet = new Set(roles); + return (detail?.assets || []).find(asset => roleSet.has(asset.asset_role) && asset.exists_flag); +}; + +const sbasAssetCacheKey = (asset) => ( + [asset?.id, asset?.file_size, asset?.updated_at || asset?.created_at || asset?.relative_path] + .filter(Boolean) + .join(':') +); + +const sbasRateColor = (rate) => { + const numeric = Number(rate); + if (!Number.isFinite(numeric)) return '#64748b'; + if (numeric <= -30) return '#1d4ed8'; + if (numeric < -5) return '#38bdf8'; + if (numeric <= 5) return '#16a34a'; + if (numeric < 30) return '#f59e0b'; + return '#dc2626'; +}; + +const SBAS_OVERVIEW_COLORS = ['#1d4ed8', '#dc2626', '#059669', '#7c3aed', '#d97706', '#0f766e']; + +const normalizeSbasDisplacements = (rows) => (Array.isArray(rows) ? rows : []) + .map((item) => { + const date = String(item?.date || '').trim(); + const time = Date.parse(`${date}T00:00:00Z`); + const displacement = Number(item?.displacement_mm ?? item?.displacement ?? item?.value); + if (!date || !Number.isFinite(time) || !Number.isFinite(displacement)) return null; + return { date, time, displacement }; + }) + .filter(Boolean) + .sort((left, right) => left.time - right.time); + +const buildSbasSparklineSvg = (rows) => { + const values = normalizeSbasDisplacements(rows); + if (values.length < 2) return ''; + const width = 220; + const height = 72; + const padX = 12; + const padY = 10; + const minTime = Math.min(...values.map(item => item.time)); + const maxTime = Math.max(...values.map(item => item.time)); + const minValue = Math.min(...values.map(item => item.displacement), 0); + const maxValue = Math.max(...values.map(item => item.displacement), 0); + const timeSpan = Math.max(1, maxTime - minTime); + const valueSpan = Math.max(1e-9, maxValue - minValue); + const xScale = (time) => padX + ((time - minTime) / timeSpan) * (width - padX * 2); + const yScale = (value) => height - padY - ((value - minValue) / valueSpan) * (height - padY * 2); + const points = values.map(item => `${xScale(item.time).toFixed(1)},${yScale(item.displacement).toFixed(1)}`).join(' '); + const zeroY = yScale(0).toFixed(1); + const minLabel = escapeHtml(formatMapNumber(minValue, 1)); + const maxLabel = escapeHtml(formatMapNumber(maxValue, 1)); + return ` + <svg width="${width}" height="${height}" viewBox="0 0 ${width} ${height}" role="img" aria-label="SBAS displacement sparkline"> + <rect x="0" y="0" width="${width}" height="${height}" fill="#f8fafc" rx="6"></rect> + <line x1="${padX}" x2="${width - padX}" y1="${zeroY}" y2="${zeroY}" stroke="#94a3b8" stroke-width="1" stroke-dasharray="3 3"></line> + <polyline points="${points}" fill="none" stroke="#1d4ed8" stroke-width="2"></polyline> + ${values.map(item => `<circle cx="${xScale(item.time).toFixed(1)}" cy="${yScale(item.displacement).toFixed(1)}" r="2.4" fill="#1d4ed8"></circle>`).join('')} + <text x="${padX}" y="10" font-size="9" fill="#64748b">${maxLabel} mm</text> + <text x="${padX}" y="${height - 4}" font-size="9" fill="#64748b">${minLabel} mm</text> + </svg> + `; +}; + +const buildSbasPointPopupHtml = (point, options = {}) => { + const matched = point?.matched || {}; + const pointId = escapeHtml(point?.point_id || options.pointId || 'SBAS point'); + const label = escapeHtml(point?.selection_label || options.label || pointId); + const rate = point?.deformation_rate_mm_per_year ?? point?.los_rate_mm_per_year ?? matched.los_rate_mm_per_year; + const lon = point?.lon ?? matched.lon; + const lat = point?.lat ?? matched.lat; + const nearestNote = matched.used_nearest + ? `<div><strong>匹配:</strong> 最近有效像元，距离 ${escapeHtml(formatMapNumber(matched.distance_m, 1))} m</div>` + : ''; + return ` + <div class="sbas-popup" style="min-width:240px"> + <div style="font-weight:800;margin-bottom:6px">${label}</div> + <div><strong>ID:</strong> <span class="mono">${pointId}</span></div> + <div><strong>经纬度:</strong> ${escapeHtml(formatMapNumber(lon, 6))}, ${escapeHtml(formatMapNumber(lat, 6))}</div> + <div><strong>LOS速率:</strong> ${escapeHtml(formatMapNumber(rate, 2))} mm/yr</div> + ${nearestNote} + <div style="margin-top:8px">${buildSbasSparklineSvg(point?.displacements || options.displacements || [])}</div> + </div> + `; +}; + +const buildSbasOverviewPopupHtml = (product) => { + const title = escapeHtml(product?.display_name || product?.stack_key || product?.run_key || `SBAS #${product?.id ?? '-'}`); + const dateStart = escapeHtml(String(product?.date_start || '-').slice(0, 10)); + const dateEnd = escapeHtml(String(product?.date_end || '-').slice(0, 10)); + const stackSize = escapeHtml(product?.stack_size ?? product?.stack_dates?.length ?? '-'); + const status = escapeHtml(product?.status || '-'); + const health = escapeHtml(product?.health_status || '-'); + const runKey = escapeHtml(product?.run_key || '-'); + const stackKey = escapeHtml(product?.stack_key || '-'); + const region = product?.admin_region?.display_name || product?.admin_region?.name || product?.admin_region?.tree_id || '-'; + return ` + <div class="sbas-popup" style="min-width:260px"> + <div style="font-weight:850;margin-bottom:7px">${title}</div> + <div><strong>时间:</strong> ${dateStart} → ${dateEnd}</div> + <div><strong>栈期数:</strong> ${stackSize}</div> + <div><strong>状态:</strong> ${status} / ${health}</div> + <div><strong>区域:</strong> ${escapeHtml(region)}</div> + <div><strong>stack:</strong> <span class="mono">${stackKey}</span></div> + <div><strong>run:</strong> <span class="mono">${runKey}</span></div> + </div> + `; +}; + function App() { const { language, setLanguage } = useI18n(); @@ -77,21 +210,16 @@ function App() { setHealthError: state.setHealthError, }))); const { - activeTasks, setActiveTasks, isGlobalLocked, setIsGlobalLocked, + activeTasks, setActiveTasks, isCheckingTasks, setIsCheckingTasks, pendingTaskIds, setPendingTaskIds, - nonBlockingTaskIds, setNonBlockingTaskIds, } = useTaskStore(useShallow((state) => ({ activeTasks: state.activeTasks, setActiveTasks: state.setActiveTasks, - isGlobalLocked: state.isGlobalLocked, - setIsGlobalLocked: state.setIsGlobalLocked, isCheckingTasks: state.isCheckingTasks, setIsCheckingTasks: state.setIsCheckingTasks, pendingTaskIds: state.pendingTaskIds, setPendingTaskIds: state.setPendingTaskIds, - nonBlockingTaskIds: state.nonBlockingTaskIds, - setNonBlockingTaskIds: state.setNonBlockingTaskIds, }))); const { leftPanelTab, setLeftPanelTab, leftPanelWidth, setLeftPanelWidth, @@ -295,6 +423,7 @@ function App() { const foundPairsRef = useRef(foundPairs); const hazardLayersRef = useRef({}); const dinsarResultLayersRef = useRef({}); + const sbasAnalysisLayersRef = useRef({}); const resizeStateRef = useRef({ side: null, startX: 0, startLeft: 0, startRight: 0 }); const allDataRef = useRef(allData); const dinsarResultsRef = useRef(dinsarResults); @@ -324,6 +453,7 @@ function App() { activeLayersRef: activeLayersRef.current, hazardLayersGroupRef: hazardLayersGroupRef.current, dinsarResultLayersRef: dinsarResultLayersRef.current, + sbasAnalysisLayersRef: sbasAnalysisLayersRef.current, waterSceneLayersRef: waterSceneLayersRef.current, radarPreviewLayersRef: radarPreviewLayersRef.current, pairLayersRef: pairLayersRef.current, @@ -386,12 +516,8 @@ function App() { addLog('warn', '当前账号为只读用户，无法执行写操作。'); return false; } - if (isCheckingTasks || isGlobalLocked) { - addLog('warn', '系统正在处理任务，请稍候...'); - return false; - } return true; - }, [addLog, isAdmin, isCheckingTasks, isGlobalLocked]); + }, [addLog, isAdmin]); const clearRadarMapLayers = () => { cancelMapBatch(); @@ -412,6 +538,16 @@ function App() { radarPreviewLayersRef.current = {}; }; + const clearSbasAnalysisLayers = useCallback(() => { + Object.values(sbasAnalysisLayersRef.current).forEach((entry) => { + const layer = entry?.layer || entry; + if (layer?.remove) { + layer.remove(); + } + }); + sbasAnalysisLayersRef.current = {}; + }, []); + const clearRadarSearchResults = (options = {}) => { const nextLimit = Math.max( 1, @@ -613,7 +749,6 @@ function App() { setHasRadarSearched, setCurrentUser, setAuthChecked, - setIsGlobalLocked, setPendingTaskIds, setLicenseLoading, setLicenseStatus, @@ -798,11 +933,11 @@ function App() { handleTaskCompletionRef.current = handleTaskCompletion; const { - forceUnlockPwd, - setForceUnlockPwd, - showForceUnlock, - setShowForceUnlock, - handleForceUnlock, + cancelTaskPwd, + setCancelTaskPwd, + showCancelTask, + setShowCancelTask, + handleCancelActiveTasks, } = useGlobalTaskControl({ currentUser, licenseOk: !!licenseStatus?.ok, @@ -810,14 +945,8 @@ function App() { setActiveTasks, pendingTaskIds, setPendingTaskIds, - nonBlockingTaskIds, - setNonBlockingTaskIds, - isGlobalLocked, - setIsGlobalLocked, setIsCheckingTasks, handleTaskCompletionRef, - initializeAppDataRef, - addLog, }); const fetchRadarPreviewStatus = useCallback(async (itemId, options = {}) => { @@ -839,8 +968,7 @@ function App() { if (!ensureCanOperate()) return; if (rebuildingPreviewIds[itemId]) return; - // 立即锁定前端 - handleTaskStart(null, `正在生成影像 ${itemId} 的预览缓存...`); + addLog('info', `正在生成影像 ${itemId} 的预览缓存...`); setRebuildingPreviewIds(prev => ({ ...prev, [itemId]: true })); try { @@ -1197,6 +1325,219 @@ function App() { } }, [addLog, buildDinsarResultPopupHtml, showDates, updateLayerTooltip]); + const flyToSbasProduct = useCallback((product) => { + if (!mapRef.current || !product) return false; + const bounds = getSbasProductBounds(product); + if (!bounds) { + addLog('warn', '当前 SBAS 产品没有可定位的地理范围。'); + return false; + } + mapRef.current.flyToBounds(L.latLngBounds(bounds), { padding: [45, 45], maxZoom: 12 }); + return true; + }, [addLog]); + + const toggleSbasRateLayer = useCallback((detail, shouldBeVisible, opacity = 0.78) => { + if (!mapRef.current || !detail) return false; + const layerKey = `rate:${detail.id}`; + const existing = sbasAnalysisLayersRef.current[layerKey]?.layer; + if (!shouldBeVisible) { + if (existing) existing.remove(); + delete sbasAnalysisLayersRef.current[layerKey]; + return true; + } + + const asset = findSbasAsset(detail, ['primary_geocoded_preview', 'primary_rate_color_preview']); + const bounds = getSbasProductBounds(detail); + if (!asset || !bounds) { + addLog('warn', 'SBAS 产品缺少 LOS 速率图或地理范围，无法叠加到地图。'); + return false; + } + if (existing) { + if (!mapRef.current.hasLayer(existing)) existing.addTo(mapRef.current); + existing.setOpacity(opacity); + flyToSbasProduct(detail); + return true; + } + + const imageUrl = getSbasInsarProductAssetUrl(detail.id, asset.id, sbasAssetCacheKey(asset)); + const layer = L.imageOverlay(imageUrl, bounds, { + opacity, + interactive: true, + crossOrigin: true, + }).addTo(mapRef.current); + layer.bindPopup( + `<div class="sbas-popup"><div style="font-weight:800;margin-bottom:6px">${escapeHtml(detail.display_name || detail.run_key || 'Gamma SBAS')}</div>` + + `<div><strong>图层:</strong> LOS 速率图</div>` + + `<div><strong>色表:</strong> ${escapeHtml(detail.color_policy?.colormap || 'Gamma hls.cm')}</div>` + + `<div><strong>范围:</strong> ${escapeHtml((detail.color_policy?.display_range_mm_per_year || [-80, 80]).join(' 到 '))} mm/yr</div></div>`, + { maxWidth: 340 }, + ); + layer.on('load', () => addLog('success', `SBAS LOS 速率图已加载：${detail.display_name || detail.run_key || detail.id}`)); + layer.on('error', () => { + addLog('error', 'SBAS LOS 速率图加载失败。'); + layer.remove(); + delete sbasAnalysisLayersRef.current[layerKey]; + }); + sbasAnalysisLayersRef.current[layerKey] = { layer, kind: 'rate', productId: detail.id }; + flyToSbasProduct(detail); + return true; + }, [addLog, flyToSbasProduct]); + + const updateSbasRateOpacity = useCallback((opacity) => { + Object.values(sbasAnalysisLayersRef.current).forEach((entry) => { + if (entry?.kind === 'rate' && entry.layer?.setOpacity) { + entry.layer.setOpacity(opacity); + } + }); + }, []); + + const toggleSbasProductOverview = useCallback((products, shouldBeVisible) => { + if (!mapRef.current) return false; + const layerKey = 'overview'; + const existing = sbasAnalysisLayersRef.current[layerKey]?.layer; + if (!shouldBeVisible) { + if (existing) existing.remove(); + delete sbasAnalysisLayersRef.current[layerKey]; + return true; + } + if (existing) { + if (!mapRef.current.hasLayer(existing)) existing.addTo(mapRef.current); + const existingBounds = sbasAnalysisLayersRef.current[layerKey]?.bounds; + if (existingBounds) { + mapRef.current.flyToBounds(existingBounds, { padding: [55, 55], maxZoom: 10 }); + } + return true; + } + + const validProducts = (Array.isArray(products) ? products : []) + .map((product) => ({ product, bounds: getSbasProductBounds(product) })) + .filter(item => item.bounds); + if (!validProducts.length) { + addLog('warn', '当前没有可绘制范围的 SBAS 产品。'); + return false; + } + + const group = L.layerGroup(); + let allBounds = null; + validProducts.forEach(({ product, bounds }, index) => { + const color = SBAS_OVERVIEW_COLORS[index % SBAS_OVERVIEW_COLORS.length] || '#1d4ed8'; + const rectangle = L.rectangle(bounds, { + color, + weight: 2, + opacity: 0.95, + fillColor: color, + fillOpacity: 0.08, + dashArray: index % 2 === 0 ? undefined : '6 4', + interactive: true, + }); + rectangle.bindPopup(buildSbasOverviewPopupHtml(product), { maxWidth: 360 }); + rectangle.bindTooltip( + `${product.display_name || product.stack_key || product.run_key || product.id}<br>${String(product.date_start || '-').slice(0, 10)} → ${String(product.date_end || '-').slice(0, 10)}`, + { sticky: true, direction: 'top', opacity: 0.92 }, + ); + rectangle.addTo(group); + const nextBounds = L.latLngBounds(bounds); + allBounds = allBounds ? allBounds.extend(nextBounds) : nextBounds; + }); + group.addTo(mapRef.current); + sbasAnalysisLayersRef.current[layerKey] = { layer: group, kind: 'overview', bounds: allBounds }; + if (allBounds) { + mapRef.current.flyToBounds(allBounds, { padding: [55, 55], maxZoom: 10 }); + } + addLog('info', `已显示 ${validProducts.length} 个 SBAS 产品范围和时间。`); + return true; + }, [addLog]); + + const toggleSbasMonitorPoints = useCallback((detail, shouldBeVisible) => { + if (!mapRef.current || !detail) return false; + const layerKey = `points:${detail.id}`; + const existing = sbasAnalysisLayersRef.current[layerKey]?.layer; + if (!shouldBeVisible) { + if (existing) existing.remove(); + delete sbasAnalysisLayersRef.current[layerKey]; + return true; + } + if (existing) { + if (!mapRef.current.hasLayer(existing)) existing.addTo(mapRef.current); + flyToSbasProduct(detail); + return true; + } + const points = (detail.monitor_points?.monitor_points || []) + .filter(point => Number.isFinite(Number(point.lat)) && Number.isFinite(Number(point.lon))); + if (!points.length) { + addLog('warn', '当前 SBAS 监测点没有 WGS84 坐标，请重新注册资产后再显示。'); + return false; + } + const group = L.layerGroup(); + const latLngs = []; + points.forEach((point) => { + const lat = Number(point.lat); + const lon = Number(point.lon); + const rate = Number(point.deformation_rate_mm_per_year); + const color = sbasRateColor(rate); + const marker = L.circleMarker([lat, lon], { + radius: 7, + color: '#ffffff', + weight: 2, + fillColor: color, + fillOpacity: 0.92, + interactive: true, + }); + marker.bindPopup(buildSbasPointPopupHtml(point), { maxWidth: 320 }); + marker.addTo(group); + latLngs.push([lat, lon]); + }); + group.addTo(mapRef.current); + sbasAnalysisLayersRef.current[layerKey] = { layer: group, kind: 'points', productId: detail.id }; + if (latLngs.length) { + mapRef.current.flyToBounds(L.latLngBounds(latLngs), { padding: [55, 55], maxZoom: 13 }); + } + addLog('info', `已显示 ${points.length} 个 SBAS 监测点。`); + return true; + }, [addLog, flyToSbasProduct]); + + const showSbasQueryPoint = useCallback((result, detail) => { + if (!mapRef.current || !result?.matched) return false; + const matched = result.matched; + const lat = Number(matched.lat); + const lon = Number(matched.lon); + if (!Number.isFinite(lat) || !Number.isFinite(lon)) return false; + const layerKey = 'query'; + const existing = sbasAnalysisLayersRef.current[layerKey]?.layer; + if (existing) existing.remove(); + const marker = L.circleMarker([lat, lon], { + radius: 8, + color: '#111827', + weight: 2, + fillColor: matched.used_nearest ? '#f59e0b' : '#22c55e', + fillOpacity: 0.95, + interactive: true, + }).addTo(mapRef.current); + const point = { + point_id: matched.used_nearest ? 'query_nearest' : 'query_point', + selection_label: matched.used_nearest ? '查询点最近有效像元' : '查询点', + deformation_rate_mm_per_year: matched.los_rate_mm_per_year, + displacements: result.displacements || [], + matched, + lon, + lat, + }; + marker.bindPopup(buildSbasPointPopupHtml(point), { maxWidth: 320 }).openPopup(); + sbasAnalysisLayersRef.current[layerKey] = { layer: marker, kind: 'query', productId: detail?.id }; + mapRef.current.flyTo([lat, lon], Math.max(mapRef.current.getZoom(), 12), { animate: true }); + return true; + }, []); + + const sbasAnalysisPanel = { + onToggleRateLayer: toggleSbasRateLayer, + onRateOpacityChange: updateSbasRateOpacity, + onToggleMonitorPoints: toggleSbasMonitorPoints, + onToggleProductOverview: toggleSbasProductOverview, + onFlyToProduct: flyToSbasProduct, + onShowQueryPoint: showSbasQueryPoint, + onClearLayers: clearSbasAnalysisLayers, + }; + const toggleDinsarResultVisibility = useCallback((resultId) => { cancelMapBatch(); const currentResults = dinsarResultsRef.current; @@ -1702,10 +2043,10 @@ function App() { fetchDinsarResults({ offset: 0 }); }, [fetchDinsarResults]); - const handleCancelForceUnlock = useCallback(() => { - setShowForceUnlock(false); - setForceUnlockPwd(''); - }, [setShowForceUnlock, setForceUnlockPwd]); + const handleCloseCancelTask = useCallback(() => { + setShowCancelTask(false); + setCancelTaskPwd(''); + }, [setShowCancelTask, setCancelTaskPwd]); const radarPanel = { radarCurrentPage, @@ -1852,6 +2193,7 @@ function App() { aiPanel={aiPanel} pairsPanel={pairsPanel} psPanel={psPanel} + sbasAnalysisPanel={sbasAnalysisPanel} /> <div @@ -1916,14 +2258,13 @@ function App() { onRefreshLicenseStatus={fetchLicenseStatus} licenseFileName={licenseFileName} licenseUploadStatus={licenseUploadStatus} - isGlobalLocked={isGlobalLocked} activeTasks={activeTasks} - showForceUnlock={showForceUnlock} - forceUnlockPwd={forceUnlockPwd} - onShowForceUnlock={() => setShowForceUnlock(true)} - onForceUnlockPwdChange={setForceUnlockPwd} - onForceUnlockConfirm={handleForceUnlock} - onCancelForceUnlock={handleCancelForceUnlock} + showCancelTask={showCancelTask} + cancelTaskPwd={cancelTaskPwd} + onShowCancelTask={() => setShowCancelTask(true)} + onCancelTaskPwdChange={setCancelTaskPwd} + onCancelTaskConfirm={handleCancelActiveTasks} + onCloseCancelTask={handleCloseCancelTask} mapExport={mapExport} /> </div> diff --git a/frontend/src/DinsarProductionPanel.jsx b/frontend/src/DinsarProductionPanel.jsx index 9d3562d..9b1b148 100644 --- a/frontend/src/DinsarProductionPanel.jsx +++ b/frontend/src/DinsarProductionPanel.jsx @@ -1,8 +1,9 @@ import React, { useCallback, useEffect, useState } from 'react'; import { deleteRunLog, deleteRunRecord, getRunLog, listEngines, listRuns, previewPyintInputAssets, submitRun } from './api/dinsarProduction'; -import { clearTaskLogs, deleteTaskLog, deleteTaskRecord, getActiveTasks, getRecentTasks, getTaskLogs } from './api/tasks'; +import { clearTaskLogs, deleteTaskLog, deleteTaskRecord, getRecentTasks, getTaskLogs } from './api/tasks'; import { formatSatelliteFamilyLabel, inferSatelliteFamilyFromResultLike } from './utils/satelliteFamily'; +import useTaskMonitor from './hooks/useTaskMonitor'; const card = { background: '#fff', @@ -45,8 +46,10 @@ const ENGINE_LABEL = { const TASK_TYPE_LABEL = { ISCE2_RUN: 'ISCE2生产', PYINT_RUN: 'PyINT/Gamma生产', + LANDSAR_RUN: 'LandSAR生产', IDL_RUN_DINSAR: 'ENVI生产', }; +const DINSAR_PRODUCTION_TASK_TYPES = ['ISCE2_RUN', 'PYINT_RUN', 'LANDSAR_RUN', 'IDL_RUN_DINSAR']; const STATUS_LABEL = { PENDING: '等待中', @@ -111,6 +114,7 @@ function formatStatus(status) { function taskTypeToEngine(taskType) { if (taskType === 'ISCE2_RUN') return 'isce2'; if (taskType === 'PYINT_RUN') return 'pyint'; + if (taskType === 'LANDSAR_RUN') return 'landsar'; if (taskType === 'IDL_RUN_DINSAR') return 'sarscape'; return ''; } @@ -191,7 +195,7 @@ function mergeRunRows(productionRuns, recentTasks, limit = null) { ); (recentTasks || []).forEach(task => { - if (!['ISCE2_RUN', 'PYINT_RUN', 'IDL_RUN_DINSAR'].includes(task?.task_type)) { + if (!DINSAR_PRODUCTION_TASK_TYPES.includes(task?.task_type)) { return; } const taskId = String(task?.task_id || '').trim(); @@ -428,6 +432,7 @@ function ParamField({ name, schema, value, disabled, onChange }) { const description = schema.description || ''; const recommendation = schema.recommendation || ''; const isReadonly = !!schema.readonly; + const readonlyLabel = schema.readonly_label || '固定值'; const inputStyle = { width: '100%', padding: '5px 8px', @@ -441,29 +446,43 @@ function ParamField({ name, schema, value, disabled, onChange }) { if (schema.type === 'boolean') { return ( - <label style={{ display: 'flex', alignItems: 'center', gap: 8, fontSize: 12, color: '#0f172a' }}> - <input - type="checkbox" - checked={!!value} - disabled={disabled || isReadonly} - onChange={event => onChange(name, event.target.checked)} - /> - <span>{label}</span> - {isReadonly && ( - <span - style={{ - padding: '1px 6px', - borderRadius: 999, - background: '#e2e8f0', - color: '#475569', - fontSize: 11, - }} - > - 固定值 - </span> - )} - {description && <span style={{ color: '#64748b' }}>{description}</span>} - </label> + <div + style={{ + display: 'flex', + flexDirection: 'column', + gap: 4, + minWidth: 220, + flex: '1 1 260px', + fontSize: 12, + color: isReadonly ? '#475569' : '#0f172a', + }} + > + <label style={{ display: 'flex', alignItems: 'center', gap: 8 }}> + <input + type="checkbox" + checked={!!value} + disabled={disabled || isReadonly} + onChange={event => onChange(name, event.target.checked)} + /> + <span>{label}</span> + {isReadonly && ( + <span + style={{ + padding: '1px 6px', + borderRadius: 999, + background: '#e2e8f0', + color: '#475569', + fontSize: 11, + whiteSpace: 'nowrap', + }} + > + {readonlyLabel} + </span> + )} + </label> + {description && <div style={{ color: '#64748b', lineHeight: 1.45 }}>{description}</div>} + {recommendation && <div style={{ color: '#2563eb', lineHeight: 1.45 }}>推荐：{recommendation}</div>} + </div> ); } @@ -483,7 +502,7 @@ function ParamField({ name, schema, value, disabled, onChange }) { fontSize: 11, }} > - 固定值 + {readonlyLabel} </span> )} </div> @@ -517,7 +536,7 @@ function ParamField({ name, schema, value, disabled, onChange }) { fontSize: 11, }} > - 固定值 + {readonlyLabel} </span> )} </div> @@ -568,8 +587,6 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) loading: false, }); const [runLogDeletingId, setRunLogDeletingId] = useState(''); - const [activeTask, setActiveTask] = useState(null); - const [recentTask, setRecentTask] = useState(null); const [taskLogs, setTaskLogs] = useState([]); const [taskLogsLoading, setTaskLogsLoading] = useState(false); const [taskLogActionLoading, setTaskLogActionLoading] = useState(false); @@ -583,12 +600,19 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) const currentDefaultTimeoutSec = Number(currentEngineObj?.default_timeout_seconds || 0) || 0; const currentParamHelpText = selectedEngine === 'pyint' ? 'PyINT/Gamma 会按目标网格尺寸自动换算多视；新增 Gamma 残余重去平在解缠后执行 rascc_mask/quad_fit/quad_sub，再导出 native 和标准 GeoTIFF。' - : selectedEngine === 'isce2' + : selectedEngine === 'landsar' + ? 'LandSAR 当前使用已跑通的稳定参数。GACOS 大气相位改正需要外部大气延迟文件，未配置文件前不可启用；垂直向形变为可选输出，默认关闭。' + : selectedEngine === 'isce2' ? '这些参数现在按执行、交付、增强分组展示。结果异常时，优先尝试关闭增强项，再回看基础几何和配对质量。' : '这些参数影响当前引擎的生产模板。建议先使用默认值，只有在结果边界、噪声或几何表现异常时再逐项调整。'; const pyintPreviewBlocksSubmit = selectedEngine === 'pyint' && pyintPreview && pyintPreview.allow_submit === false; + const taskMonitor = useTaskMonitor({ + taskTypes: DINSAR_PRODUCTION_TASK_TYPES, + showRecent: true, + recentLimit: 1, + }); const latestRunWithTask = runs.find(run => run?.task_id) || null; - const monitoredTask = activeTask || recentTask || ( + const monitoredTask = taskMonitor.latestTask || ( latestRunWithTask ? { task_id: latestRunWithTask.task_id, @@ -597,7 +621,9 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) ? 'ISCE2_RUN' : latestRunWithTask.engine === 'pyint' ? 'PYINT_RUN' - : 'IDL_RUN_DINSAR', + : latestRunWithTask.engine === 'landsar' + ? 'LANDSAR_RUN' + : 'IDL_RUN_DINSAR', status: latestRunWithTask.raw_status || latestRunWithTask.status, progress: latestRunWithTask.raw_status === 'COMPLETED' || latestRunWithTask.status === 'success' ? 100 : null, message: latestRunWithTask.message || '最近一次任务', @@ -605,7 +631,7 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) : null ); const logTaskId = monitoredTask?.task_id || ''; - const showingRecentTask = !activeTask && !!monitoredTask; + const showingRecentTask = !taskMonitor.isBusy && !!monitoredTask; const loadEngines = useCallback(async () => { setEnginesLoading(true); @@ -640,7 +666,7 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) const allTasks = []; let offset = 0; while (true) { - const data = await getRecentTasks(['ISCE2_RUN', 'PYINT_RUN', 'IDL_RUN_DINSAR'], [], TASK_HISTORY_PAGE_SIZE, offset); + const data = await getRecentTasks(DINSAR_PRODUCTION_TASK_TYPES, [], TASK_HISTORY_PAGE_SIZE, offset); const pageTasks = Array.isArray(data) ? data : (data?.tasks || []); allTasks.push(...pageTasks); if (pageTasks.length < TASK_HISTORY_PAGE_SIZE) break; @@ -663,32 +689,6 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) } }, []); - const loadActiveTask = useCallback(async () => { - try { - const data = await getActiveTasks(); - const tasks = Array.isArray(data) ? data : (data?.tasks || []); - const relevantTask = tasks.find(task => ['ISCE2_RUN', 'PYINT_RUN', 'IDL_RUN_DINSAR'].includes(task.task_type)) || null; - setActiveTask(relevantTask); - return relevantTask; - } catch { - setActiveTask(null); - return null; - } - }, []); - - const loadRecentTask = useCallback(async () => { - try { - const data = await getRecentTasks(['ISCE2_RUN', 'PYINT_RUN', 'IDL_RUN_DINSAR'], [], 1, 0); - const tasks = Array.isArray(data) ? data : (data?.tasks || []); - const relevantTask = tasks[0] || null; - setRecentTask(relevantTask); - return relevantTask; - } catch { - setRecentTask(null); - return null; - } - }, []); - const loadTaskLogs = useCallback(async (taskId, options = {}) => { const silent = !!options.silent; if (!taskId) { @@ -744,18 +744,17 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) const refreshMonitor = useCallback(async (options = {}) => { const silent = !!options.silent; - const [nextRuns, nextActiveTask, nextRecentTask] = await Promise.all([ + const [nextRuns, nextRecentTasks] = await Promise.all([ loadRuns({ silent }), - loadActiveTask(), - loadRecentTask(), + taskMonitor.refreshRecentTasks(), ]); const fallbackTaskId = - nextActiveTask?.task_id - || nextRecentTask?.task_id + taskMonitor.activeTasks[0]?.task_id + || nextRecentTasks[0]?.task_id || nextRuns.find(run => run?.task_id)?.task_id || ''; await loadTaskLogs(fallbackTaskId, { silent }); - }, [loadActiveTask, loadRecentTask, loadRuns, loadTaskLogs]); + }, [loadRuns, loadTaskLogs, taskMonitor]); useEffect(() => { loadEngines(); @@ -763,12 +762,12 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) }, [loadEngines, refreshMonitor]); useEffect(() => { - const intervalMs = activeTask ? 5000 : 15000; + const intervalMs = taskMonitor.isBusy ? 5000 : 15000; const timer = window.setInterval(() => { refreshMonitor({ silent: true }); }, intervalMs); return () => window.clearInterval(timer); - }, [activeTask, refreshMonitor]); + }, [taskMonitor.isBusy, refreshMonitor]); useEffect(() => { if (currentProfiles.length > 0) { diff --git a/frontend/src/DinsarProductsPanel.jsx b/frontend/src/DinsarProductsPanel.jsx index cccb15c..096b425 100644 --- a/frontend/src/DinsarProductsPanel.jsx +++ b/frontend/src/DinsarProductsPanel.jsx @@ -2,8 +2,9 @@ import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { scanDinsarResults } from './api/dinsar'; import { extractDispResults } from './api/idl'; -import { clearTaskLogs, deleteTaskLog, getActiveTasks, getRecentTasks, getTaskLogs } from './api/tasks'; +import { clearTaskLogs, deleteTaskLog, getTaskLogs } from './api/tasks'; import DinsarCatalogPanel from './components/DinsarCatalogPanel'; +import useTaskMonitor from './hooks/useTaskMonitor'; const PRODUCT_TASK_TYPES = [ 'SCAN_DINSAR', @@ -55,42 +56,20 @@ export default function DinsarProductsPanel({ readOnly = false, onJobQueued }) { const [actionError, setActionError] = useState(false); const [scanning, setScanning] = useState(false); - const [activeTask, setActiveTask] = useState(null); - const [recentTask, setRecentTask] = useState(null); const [taskLogs, setTaskLogs] = useState([]); const [taskLogsLoading, setTaskLogsLoading] = useState(false); const [taskLogActionLoading, setTaskLogActionLoading] = useState(false); const [taskLogDeletingId, setTaskLogDeletingId] = useState(null); - const monitoredTask = activeTask || recentTask; + const taskMonitor = useTaskMonitor({ + taskTypes: PRODUCT_TASK_TYPES, + showRecent: true, + recentLimit: 1, + }); + const monitoredTask = taskMonitor.latestTask; const logTaskId = monitoredTask?.task_id || ''; - const showingRecentTask = !activeTask && !!recentTask; + const showingRecentTask = !taskMonitor.isBusy && !!monitoredTask; const actionTone = getMessageTone(actionMessage, actionError); - const loadActiveTask = useCallback(async () => { - try { - const data = await getActiveTasks(); - const tasks = Array.isArray(data) ? data : (data?.tasks || []); - const relevantTask = tasks.find((task) => PRODUCT_TASK_TYPES.includes(task.task_type)) || null; - setActiveTask(relevantTask); - return relevantTask; - } catch { - setActiveTask(null); - return null; - } - }, []); - - const loadRecentTask = useCallback(async () => { - try { - const tasks = await getRecentTasks(PRODUCT_TASK_TYPES, [], 1, 0); - const nextTask = Array.isArray(tasks) ? (tasks[0] || null) : null; - setRecentTask(nextTask); - return nextTask; - } catch { - setRecentTask(null); - return null; - } - }, []); - const loadTaskLogs = useCallback(async (taskId) => { if (!taskId) { setTaskLogs([]); @@ -108,17 +87,14 @@ export default function DinsarProductsPanel({ readOnly = false, onJobQueued }) { }, []); const refreshMonitor = useCallback(async () => { - const [nextActiveTask, nextRecentTask] = await Promise.all([ - loadActiveTask(), - loadRecentTask(), - ]); - const nextTaskId = nextActiveTask?.task_id || nextRecentTask?.task_id || ''; + const nextRecentTasks = await taskMonitor.refreshRecentTasks(); + const nextTaskId = taskMonitor.activeTasks[0]?.task_id || nextRecentTasks[0]?.task_id || logTaskId; await loadTaskLogs(nextTaskId); - }, [loadActiveTask, loadRecentTask, loadTaskLogs]); + }, [loadTaskLogs, logTaskId, taskMonitor]); useEffect(() => { - refreshMonitor(); - }, [refreshMonitor]); + loadTaskLogs(logTaskId); + }, [loadTaskLogs, logTaskId]); const handleDeleteTaskLog = useCallback(async (logId) => { const taskId = logTaskId; diff --git a/frontend/src/HazardPointPanel.jsx b/frontend/src/HazardPointPanel.jsx index fab0370..ceeef62 100644 --- a/frontend/src/HazardPointPanel.jsx +++ b/frontend/src/HazardPointPanel.jsx @@ -1,5 +1,7 @@ import React, { useState, useEffect, useRef } from 'react'; import apiClient from './api/client'; +import TaskStatusPanel from './components/tasks/TaskStatusPanel'; +import useTaskMonitor from './hooks/useTaskMonitor'; const HazardPointPanel = ({ onPointClick, onToggleVisibility, isVisible, onScanComplete, onTaskStart, points: externalPoints, readOnly = false }) => { const [points, setPoints] = useState(Array.isArray(externalPoints) ? externalPoints : []); @@ -7,6 +9,13 @@ const HazardPointPanel = ({ onPointClick, onToggleVisibility, isVisible, onScanC const [isLoading, setIsLoading] = useState(false); const [message, setMessage] = useState(''); const fetchPointsRef = useRef(null); + const scanTaskMonitor = useTaskMonitor({ + taskTypes: ['SCAN_HAZARD'], + showRecent: true, + recentLimit: 1, + pollRecentMs: 10000, + }); + const scanBusy = isLoading || scanTaskMonitor.isBusy; const fetchPoints = async () => { setIsLoading(true); @@ -84,12 +93,22 @@ const HazardPointPanel = ({ onPointClick, onToggleVisibility, isVisible, onScanC <button className="primary-btn" onClick={handleScan} - disabled={isLoading || readOnly} + disabled={scanBusy || readOnly} style={{ width: '100%', marginBottom: '10px' }} > - {isLoading ? '同步中...' : '同步 Shapefile 数据'} + {scanBusy ? '同步中...' : '同步 Shapefile 数据'} </button> + <TaskStatusPanel + title="灾害点同步任务" + activeTasks={scanTaskMonitor.activeTasks} + recentTasks={scanTaskMonitor.recentTasks} + latestTask={scanTaskMonitor.latestTask} + isBusy={scanTaskMonitor.isBusy} + idleText="当前没有正在执行的灾害点同步任务。" + compact + /> + <input type="text" placeholder="搜索编号、名称、市县..." diff --git a/frontend/src/HealthCheckPanel.jsx b/frontend/src/HealthCheckPanel.jsx index 17bf681..7966b78 100644 --- a/frontend/src/HealthCheckPanel.jsx +++ b/frontend/src/HealthCheckPanel.jsx @@ -857,7 +857,7 @@ const HealthCheckPanel = ({ language = 'zh', currentUser }) => { <span>{en ? 'Overall' : '总体状态'}</span> {renderBadge( productPackages.ok, - `${toNumber(productPackages.canonical_count)} / ${toNumber(productPackages.total_count)}` + `${toNumber(productPackages.valid_schema_count ?? productPackages.canonical_count)} / ${toNumber(productPackages.total_count)}` )} </div> <div className="health-card-row"> diff --git a/frontend/src/IDLAutomationPanel.jsx b/frontend/src/IDLAutomationPanel.jsx index 65f02a9..3c9b72e 100644 --- a/frontend/src/IDLAutomationPanel.jsx +++ b/frontend/src/IDLAutomationPanel.jsx @@ -8,7 +8,6 @@ import { queueImportJob, queueDinsarJob, getRecentRuns, - getActiveTasks, forceCancelTask, extractDispResults, getTaskOverview, @@ -16,6 +15,8 @@ import { deleteRun, } from './api/idl'; import { scanDinsarResults } from './api/dinsar'; +import TaskStatusPanel from './components/tasks/TaskStatusPanel'; +import useTaskMonitor from './hooks/useTaskMonitor'; import { getStatistics } from './api/stats'; @@ -34,9 +35,15 @@ function IDLAutomationPanel({ readOnly = false, onJobQueued }) { const [recentRuns, setRecentRuns] = useState([]); const [isBusy, setIsBusy] = useState(false); const [message, setMessage] = useState(''); - const [runningTask, setRunningTask] = useState(null); // active IDL task from backend - const [showUnlockInput, setShowUnlockInput] = useState(false); - const [unlockPassword, setUnlockPassword] = useState(''); + const [showCancelInput, setShowCancelInput] = useState(false); + const [cancelPassword, setCancelPassword] = useState(''); + const idlTaskMonitor = useTaskMonitor({ + taskTypes: ['IDL_IMPORT', 'IDL_DINSAR'], + showRecent: true, + recentLimit: 1, + pollRecentMs: 10000, + }); + const runningTask = idlTaskMonitor.activeTasks[0] || null; const [importRootDir, setImportRootDir] = useState(''); const [importNumToProcess, setImportNumToProcess] = useState(0); @@ -58,20 +65,12 @@ function IDLAutomationPanel({ readOnly = false, onJobQueued }) { const [dinsarInspect, setDinsarInspect] = useState(null); const refreshData = useCallback(async () => { - const [s, runs, tasks] = await Promise.all([ + const [s, runs] = await Promise.all([ getEnviStatus(), getRecentRuns(20), - getActiveTasks().catch(() => []), ]); setStatus(s); setRecentRuns(Array.isArray(runs?.runs) ? runs.runs : []); - // Find any running IDL task (IDL_IMPORT or IDL_DINSAR) - const taskList = Array.isArray(tasks) ? tasks : []; - const active = taskList.find( - (t) => ['IDL_IMPORT', 'IDL_DINSAR'].includes(t.task_type) && - ['PENDING', 'RUNNING'].includes(t.status) - ); - setRunningTask(active || null); }, []); useEffect(() => { @@ -186,17 +185,18 @@ function IDLAutomationPanel({ readOnly = false, onJobQueued }) { } }; - const handleForceUnlock = async () => { - if (!runningTask || !unlockPassword) return; + const handleCancelRunningTask = async () => { + if (!runningTask || !cancelPassword) return; try { - await forceCancelTask(runningTask.task_id, unlockPassword); - setMessage('任务已强制取消，前端已解锁。'); - setShowUnlockInput(false); - setUnlockPassword(''); + await forceCancelTask(runningTask.task_id, cancelPassword); + setMessage('任务取消请求已提交。'); + setShowCancelInput(false); + setCancelPassword(''); + await idlTaskMonitor.refreshRecentTasks(); await refreshData(); } catch (error) { - const detail = error?.response?.data?.detail || error?.message || '解锁失败'; - setMessage(`强制解锁失败: ${detail}`); + const detail = error?.response?.data?.detail || error?.message || '取消失败'; + setMessage(`取消任务失败: ${detail}`); } }; @@ -220,7 +220,7 @@ function IDLAutomationPanel({ readOnly = false, onJobQueued }) { ); }; - // Buttons are locked when: submitting API call, readOnly user, or a backend task is running + // Buttons are locally disabled when submitting API calls or an IDL task is already active. const isLocked = isBusy || !!runningTask; const demDisplay = status?.dem_base_file || '-'; @@ -245,71 +245,46 @@ function IDLAutomationPanel({ readOnly = false, onJobQueued }) { </div> </div> - {/* Running task indicator */} - {runningTask && ( - <div style={{ - ...cardStyle, - background: '#fffbeb', - borderColor: '#f59e0b', - }}> - <div style={{ display: 'flex', alignItems: 'center', gap: '10px' }}> - <span style={{ fontSize: '16px' }}>⚙</span> - <div style={{ flex: 1, fontSize: '13px' }}> - <strong style={{ color: '#b45309' }}> - {runningTask.task_type === 'IDL_IMPORT' ? 'Import' : 'D-InSAR'} 任务运行中 - </strong> - {runningTask.progress > 0 && ( - <span style={{ color: '#92400e', marginLeft: '8px', fontVariantNumeric: 'tabular-nums' }}> - {runningTask.progress}% - </span> - )} - <div style={{ color: '#92400e', marginTop: '3px', fontSize: '12px', wordBreak: 'break-all' }}> - {runningTask.message || runningTask.status} - </div> - {runningTask.progress > 0 && ( - <div style={{ marginTop: '5px', height: '6px', background: '#fde68a', borderRadius: '3px', overflow: 'hidden' }}> - <div style={{ - height: '100%', - width: `${runningTask.progress}%`, - background: '#f59e0b', - borderRadius: '3px', - transition: 'width 0.5s ease', - }} /> - </div> - )} - </div> - {!readOnly && !showUnlockInput && ( - <button - type="button" - onClick={() => setShowUnlockInput(true)} - style={{ - padding: '3px 10px', - borderRadius: '4px', - border: '1px solid #dc2626', - background: '#fef2f2', - color: '#dc2626', - fontSize: '12px', - cursor: 'pointer', - }} - > - 强制解锁 - </button> - )} - </div> - {showUnlockInput && ( + <TaskStatusPanel + title="ENVI / SARscape 任务" + activeTasks={idlTaskMonitor.activeTasks} + recentTasks={idlTaskMonitor.recentTasks} + latestTask={idlTaskMonitor.latestTask} + isBusy={idlTaskMonitor.isBusy} + idleText="当前没有正在执行的 ENVI / SARscape 任务。" + action={runningTask && !readOnly && !showCancelInput ? ( + <button + type="button" + onClick={() => setShowCancelInput(true)} + style={{ + padding: '3px 10px', + borderRadius: '4px', + border: '1px solid #dc2626', + background: '#fef2f2', + color: '#dc2626', + fontSize: '12px', + cursor: 'pointer', + }} + > + 取消任务 + </button> + ) : null} + footer={runningTask ? ( + <> + {showCancelInput && ( <div style={{ marginTop: '8px', display: 'flex', alignItems: 'center', gap: '8px' }}> <input type="password" placeholder="输入管理员密码" - value={unlockPassword} - onChange={(e) => setUnlockPassword(e.target.value)} - onKeyDown={(e) => e.key === 'Enter' && handleForceUnlock()} + value={cancelPassword} + onChange={(e) => setCancelPassword(e.target.value)} + onKeyDown={(e) => e.key === 'Enter' && handleCancelRunningTask()} style={{ padding: '4px 8px', fontSize: '12px', borderRadius: '4px', border: '1px solid #d1d5db', width: '160px' }} /> <button type="button" - onClick={handleForceUnlock} - disabled={!unlockPassword} + onClick={handleCancelRunningTask} + disabled={!cancelPassword} style={{ padding: '4px 10px', borderRadius: '4px', @@ -317,28 +292,29 @@ function IDLAutomationPanel({ readOnly = false, onJobQueued }) { background: '#dc2626', color: '#fff', fontSize: '12px', - cursor: unlockPassword ? 'pointer' : 'not-allowed', - opacity: unlockPassword ? 1 : 0.5, + cursor: cancelPassword ? 'pointer' : 'not-allowed', + opacity: cancelPassword ? 1 : 0.5, }} > 确认取消 </button> <button type="button" - onClick={() => { setShowUnlockInput(false); setUnlockPassword(''); }} + onClick={() => { setShowCancelInput(false); setCancelPassword(''); }} style={{ padding: '4px 10px', borderRadius: '4px', border: '1px solid #d1d5db', background: '#fff', fontSize: '12px', cursor: 'pointer' }} > 取消 </button> </div> )} - {!showUnlockInput && ( + {!showCancelInput && ( <div style={{ fontSize: '11px', color: '#92400e', marginTop: '4px', marginLeft: '26px' }}> - 按钮已锁定，等待任务完成 + 同类 ENVI/SARscape 任务运行中，当前提交按钮暂不可用。 </div> )} - </div> - )} + </> + ) : null} + /> {/* Task 状态总览 */} <div style={cardStyle}> diff --git a/frontend/src/ProductionWorkspace.jsx b/frontend/src/ProductionWorkspace.jsx index 5823792..6cc5c67 100644 --- a/frontend/src/ProductionWorkspace.jsx +++ b/frontend/src/ProductionWorkspace.jsx @@ -69,7 +69,10 @@ export default function ProductionWorkspace({ }; const handleSbasProductQueued = taskId => { - onTaskStart?.(taskId, 'SBAS-InSAR result catalog task queued.'); + onTaskStart?.(taskId, 'SBAS-InSAR result catalog task queued.', { + taskType: 'REBUILD_SBAS_INSAR_CATALOG', + nonBlocking: true, + }); }; return ( @@ -181,6 +184,7 @@ export default function ProductionWorkspace({ {activeView === 'sbas_insar_production' && ( <LazySbasInsarProductionPanel readOnly={readOnly} + onTaskStart={onTaskStart} /> )} {activeView === 'sbas_insar_products' && ( diff --git a/frontend/src/SbasInsarProductionPanel.jsx b/frontend/src/SbasInsarProductionPanel.jsx index 6eca859..77f1831 100644 --- a/frontend/src/SbasInsarProductionPanel.jsx +++ b/frontend/src/SbasInsarProductionPanel.jsx @@ -3,10 +3,14 @@ import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { auditSbasInsarStack, decideSbasInsarItab, + deleteSbasInsarRun, + getLandsarSbasRun, + getLandsarSbasRunArtifactUrl, discoverSbasInsarStacks, getSbasInsarCapabilities, getSbasInsarRun, getSbasInsarRunArtifactUrl, + listLandsarSbasRuns, listSbasInsarRuns, prepareSbasInsarCoregistration, prepareSbasInsarInterferograms, @@ -14,6 +18,7 @@ import { prepareSbasInsarRdcDem, prepareSbasInsarWorkflow, runSbasInsarBaselineAudit, + submitLandsarSbasAutoWorkflow, submitSbasInsarCoregistrationJob, submitSbasInsarInterferogramsJob, submitSbasInsarIptaTimeseriesJob, @@ -40,6 +45,8 @@ const mutedStyle = { lineHeight: 1.55, }; +const ACTIVE_RUN_REFRESH_INTERVAL_MS = 10 * 60 * 1000; + const labelStyle = { color: '#475569', fontSize: 12, @@ -51,19 +58,26 @@ const valueStyle = { fontWeight: 650, }; -const gridStyle = { - display: 'grid', - gridTemplateColumns: 'minmax(280px, 360px) minmax(0, 1fr)', - gap: 12, - alignItems: 'start', -}; - const metricGridStyle = { display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(150px, 1fr))', gap: 10, }; +const compactDetailsStyle = { + border: '1px solid #e2e8f0', + borderRadius: 8, + padding: '9px 10px', + background: '#ffffff', +}; + +const compactSummaryStyle = { + cursor: 'pointer', + color: '#0f172a', + fontSize: 13, + fontWeight: 700, +}; + const buttonBaseStyle = { width: '100%', textAlign: 'left', @@ -80,20 +94,6 @@ function formatValue(value, suffix = '') { return `${numeric.toFixed(Math.abs(numeric) >= 100 ? 1 : 2)}${suffix}`; } -function formatBytes(value) { - const size = Number(value || 0); - if (!Number.isFinite(size) || size <= 0) return '-'; - if (size < 1024) return `${size} B`; - const units = ['KB', 'MB', 'GB', 'TB']; - let current = size / 1024; - let index = 0; - while (current >= 1024 && index < units.length - 1) { - current /= 1024; - index += 1; - } - return `${current.toFixed(current >= 100 ? 0 : 1)} ${units[index]}`; -} - function normalizeBbox(bbox) { if (!bbox || typeof bbox !== 'object') return null; const minLon = Number(bbox.min_lon); @@ -150,7 +150,27 @@ function formatPercent(value) { return `${(numeric * 100).toFixed(numeric >= 0.1 ? 0 : 1)}%`; } +function shortHash(value) { + const text = String(value || '').trim(); + return text ? text.slice(0, 10) : '-'; +} + +function getSceneNames(stack) { + const sceneNames = Array.isArray(stack?.scene_names) ? stack.scene_names.filter(Boolean) : []; + if (sceneNames.length > 0) return sceneNames; + const scenes = Array.isArray(stack?.scenes) ? stack.scenes : []; + const names = scenes.map(scene => scene?.scene_name).filter(Boolean); + if (names.length > 0) return names; + return Array.isArray(stack?.scene_name_preview) ? stack.scene_name_preview.filter(Boolean) : []; +} + +function isActiveRunStatus(value) { + const text = String(value || '').toUpperCase(); + return text.includes('RUNNING') || ['READY', 'PENDING', 'RETRY'].includes(text); +} + function StatusBadge({ value }) { + const text = String(value || 'UNKNOWN').toUpperCase(); const okValues = new Set([ 'READY', 'READY_FOR_GAMMA_BASELINE_AUDIT', @@ -159,8 +179,10 @@ function StatusBadge({ value }) { 'COMPLETED', 'IPTA_TIMESERIES_READY', ]); - const isOk = okValues.has(value); - const color = isOk ? '#0f766e' : '#92400e'; + const isRunning = text.includes('RUNNING'); + const isOk = okValues.has(text); + const color = isRunning ? '#0369a1' : (isOk ? '#0f766e' : '#92400e'); + const background = isRunning ? '#e0f2fe' : (isOk ? '#ccfbf1' : '#fef3c7'); return ( <span style={{ @@ -170,7 +192,7 @@ function StatusBadge({ value }) { padding: '2px 8px', borderRadius: 999, color, - background: isOk ? '#ccfbf1' : '#fef3c7', + background, fontSize: 12, fontWeight: 700, }} @@ -198,6 +220,130 @@ function Metric({ label, value }) { ); } +function StackIdentityNotice({ stack }) { + if (!stack) return null; + const sameDateCount = Number(stack.same_date_sequence_candidate_count || 0); + const distinctSceneCount = Number(stack.same_date_sequence_distinct_scene_group_count || 0); + const sameSceneRuns = Array.isArray(stack.existing_same_scene_runs) ? stack.existing_same_scene_runs : []; + const showDateSequenceNotice = sameDateCount > 1 && distinctSceneCount > 1; + if (!showDateSequenceNotice && sameSceneRuns.length === 0) return null; + + return ( + <div style={{ display: 'grid', gap: 8 }}> + {showDateSequenceNotice && ( + <div style={{ border: '1px solid #fed7aa', borderRadius: 8, padding: 10, background: '#fff7ed', color: '#9a3412', fontSize: 12, lineHeight: 1.55 }}> + 同日期序列候选 {sameDateCount} 个，其中不同影像组 {distinctSceneCount} 个。判断是否同任务请以影像名称集合为准。 + </div> + )} + {sameSceneRuns.length > 0 && ( + <div style={{ border: '1px solid #fecaca', borderRadius: 8, padding: 10, background: '#fef2f2', color: '#991b1b', fontSize: 12, lineHeight: 1.55 }}> + 已存在同影像任务：{sameSceneRuns.map(item => `${item.run_id} (${item.status || '-'})`).join('；')} + </div> + )} + </div> + ); +} + +function SceneNamePanel({ stack }) { + if (!stack) return null; + const names = getSceneNames(stack); + return ( + <details style={compactDetailsStyle}> + <summary style={compactSummaryStyle}> + 影像名称 {names.length || stack.scene_name_count || 0} 景；影像组 {shortHash(stack.scene_identity_hash)} + </summary> + {names.length > 0 ? ( + <div style={{ display: 'grid', gap: 6, marginTop: 8 }}> + {names.map(name => ( + <div key={name} style={{ ...mutedStyle, wordBreak: 'break-all' }}> + {name} + </div> + ))} + </div> + ) : ( + <div style={{ ...mutedStyle, marginTop: 8 }}>当前候选未返回完整影像名称；请先执行审计或重新发现候选。</div> + )} + </details> + ); +} + +function RuntimeStatusPanel({ status }) { + if (!status) return null; + const background = status.background_activity || {}; + const tasks = Array.isArray(background.tasks) ? background.tasks : []; + const jobs = Array.isArray(background.jobs) ? background.jobs : []; + const taskLogs = Array.isArray(background.task_logs) ? background.task_logs : []; + const fileLogs = Array.isArray(status.recent_logs) ? status.recent_logs : []; + const wslProcesses = Array.isArray(status.wsl_processes?.processes) ? status.wsl_processes.processes : []; + const currentTask = tasks.find(item => ['PENDING', 'RUNNING'].includes(String(item.status || '').toUpperCase())) || tasks[0] || null; + const currentJob = jobs.find(item => ['READY', 'PENDING', 'RUNNING', 'RETRY'].includes(String(item.status || '').toUpperCase())) || jobs[0] || null; + const latestTaskLog = taskLogs[0] || null; + const latestFileLog = fileLogs[0] || null; + const gate = status.overlap_gate || {}; + + return ( + <div style={{ border: '1px solid #bae6fd', borderRadius: 8, padding: 10, background: '#f0f9ff' }}> + <div style={{ display: 'flex', justifyContent: 'space-between', gap: 10, alignItems: 'center', flexWrap: 'wrap' }}> + <div style={valueStyle}>Runtime Status</div> + <StatusBadge value={status.active ? 'RUNNING' : (status.run_status || 'IDLE')} /> + </div> + <div style={{ ...metricGridStyle, marginTop: 8 }}> + <Metric label="Current step" value={status.current_step?.id || '-'} /> + <Metric label="Workflow updated" value={status.workflow_updated_at || '-'} /> + <Metric label="Latest log" value={status.latest_log_updated_at || '-'} /> + <Metric + label="Common overlap" + value={`${formatPercent(gate.common_overlap_ratio)} / ${formatPercent(gate.min_common_overlap_ratio)}`} + /> + </div> + {(currentTask || currentJob) && ( + <div style={{ ...mutedStyle, marginTop: 8, wordBreak: 'break-word' }}> + Task: {currentTask ? `${currentTask.task_type || '-'} ${currentTask.status || '-'} ${currentTask.progress ?? 0}%` : '-'} + {'; '} + Job: {currentJob ? `${currentJob.job_type || '-'} ${currentJob.status || '-'}` : '-'} + </div> + )} + {latestTaskLog && ( + <div style={{ ...mutedStyle, marginTop: 6, whiteSpace: 'pre-wrap', wordBreak: 'break-word' }}> + DB log: [{latestTaskLog.level || 'INFO'}] {latestTaskLog.message} + </div> + )} + {latestFileLog?.tail && ( + <details style={{ ...compactDetailsStyle, marginTop: 8, borderColor: '#bae6fd' }}> + <summary style={compactSummaryStyle}>{latestFileLog.name || 'latest log'}</summary> + <pre + style={{ + margin: '8px 0 0', + maxHeight: 180, + overflow: 'auto', + whiteSpace: 'pre-wrap', + wordBreak: 'break-word', + fontSize: 11, + lineHeight: 1.45, + color: '#0f172a', + }} + > + {latestFileLog.tail} + </pre> + </details> + )} + <details style={{ ...compactDetailsStyle, marginTop: 8, borderColor: '#bae6fd' }}> + <summary style={compactSummaryStyle}>WSL processes ({wslProcesses.length})</summary> + <div style={{ display: 'grid', gap: 6, marginTop: 8 }}> + {wslProcesses.length === 0 && ( + <div style={mutedStyle}>{status.wsl_processes?.error || 'No matching WSL process reported.'}</div> + )} + {wslProcesses.map(item => ( + <div key={`${item.pid}-${item.command}`} style={{ ...mutedStyle, fontFamily: 'monospace', wordBreak: 'break-word' }}> + {item.pid} {item.etime} {item.stat} {item.command} + </div> + ))} + </div> + </details> + </div> + ); +} + function RunArtifactLink({ runId, artifact }) { const href = getSbasInsarRunArtifactUrl(runId, artifact.relative_path); return ( @@ -482,11 +628,34 @@ function UnusedGeographicCoveragePanel({ coverage }) { } */ -export default function SbasInsarProductionPanel({ readOnly = false }) { +export default function SbasInsarProductionPanel({ readOnly = false, onTaskStart }) { + const [processorMode, setProcessorMode] = useState('landsar'); const [capabilities, setCapabilities] = useState(null); const [runs, setRuns] = useState([]); const [selectedRunId, setSelectedRunId] = useState(''); const [runDetail, setRunDetail] = useState(null); + const [landsarRuns, setLandsarRuns] = useState([]); + const [selectedLandsarRunId, setSelectedLandsarRunId] = useState(''); + const [landsarRunDetail, setLandsarRunDetail] = useState(null); + const [landsarDemPath, setLandsarDemPath] = useState(''); + const [landsarMinScenes, setLandsarMinScenes] = useState(3); + const [landsarSubmitLoading, setLandsarSubmitLoading] = useState(false); + const [landsarWorkflowJobLoading, setLandsarWorkflowJobLoading] = useState(false); + const [landsarWorkflowJob, setLandsarWorkflowJob] = useState(null); + const [landsarParams, setLandsarParams] = useState({ + dem_format: 4, + intf_method: 0, + perp_baseline: 200, + time_baseline: 300, + doppler_baseline: 100, + az_looks: 3, + rg_looks: 3, + da_threshold: 0.25, + network_type: 0, + solve_method: 0, + gen_vector_map: false, + gen_post_raster: true, + }); const [loading, setLoading] = useState(false); const [runDetailLoading, setRunDetailLoading] = useState(false); const [error, setError] = useState(''); @@ -497,6 +666,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { const [stackAudit, setStackAudit] = useState(null); const [submitLoading, setSubmitLoading] = useState(false); const [stackAdminRegionQuery, setStackAdminRegionQuery] = useState(''); + const [selectedSensorFamily, setSelectedSensorFamily] = useState('LT1'); const [baselineAuditLoading, setBaselineAuditLoading] = useState(false); const [itabDecisionLoading, setItabDecisionLoading] = useState(false); const [coregistrationLoading, setCoregistrationLoading] = useState(false); @@ -514,20 +684,29 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { const [workflowLoading, setWorkflowLoading] = useState(false); const [workflowJobLoading, setWorkflowJobLoading] = useState(false); const [workflowJob, setWorkflowJob] = useState(null); + const [runDeleteLoading, setRunDeleteLoading] = useState(false); const stackDiscoveryPayload = useMemo(() => { const adminRegion = stackAdminRegionQuery.trim(); + const isLandsar = processorMode === 'landsar'; + const processorCapability = (capabilities?.processors || []).find(item => + isLandsar ? item.processor_code === 'landsar_sbas' : item.processor_code === 'gamma_ipta_sbas' + ); + const configuredMinCommonOverlap = Number( + processorCapability?.min_common_overlap_ratio ?? capabilities?.min_common_overlap_ratio ?? 0.3 + ); return { - min_scenes: 3, - require_orbits: true, + sensor_family: isLandsar ? 'LT1' : selectedSensorFamily, + min_scenes: isLandsar ? Math.max(3, Number(landsarMinScenes) || 3) : 3, + require_orbits: !isLandsar, include_scenes: false, - limit: 30, + limit: 0, discovery_mode: adminRegion ? 'aoi' : 'strict', admin_region: adminRegion || undefined, min_aoi_coverage_ratio: 0.01, - min_common_overlap_ratio: 0, + min_common_overlap_ratio: Number.isFinite(configuredMinCommonOverlap) ? configuredMinCommonOverlap : 0.3, }; - }, [stackAdminRegionQuery]); + }, [capabilities, landsarMinScenes, processorMode, selectedSensorFamily, stackAdminRegionQuery]); const loadProductionRuns = useCallback(async () => { setLoading(true); @@ -538,9 +717,19 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { listSbasInsarRuns(), ]); const runItems = Array.isArray(runData?.items) ? runData.items : []; + const activeRun = runItems.find(item => isActiveRunStatus(item.status)); setCapabilities(capabilityData); setRuns(runItems); - setSelectedRunId(current => current || runItems[0]?.run_id || ''); + setSelectedRunId(current => { + if (activeRun && current !== activeRun.run_id) return activeRun.run_id; + if (current && runItems.some(item => item.run_id === current)) return current; + return runItems[0]?.run_id || ''; + }); + const landsarCapability = (capabilityData?.processors || []).find(item => item.processor_code === 'landsar_sbas'); + if (landsarCapability) { + setLandsarDemPath(current => current || landsarCapability.default_dem_path || ''); + setLandsarMinScenes(current => Math.max(3, Number(current || landsarCapability.min_scenes || 3))); + } } catch (exc) { setError(exc?.response?.data?.detail || exc.message || 'SBAS-InSAR 列表加载失败'); setCapabilities(null); @@ -554,6 +743,15 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { loadProductionRuns(); }, [loadProductionRuns]); + useEffect(() => { + const hasActiveRun = runs.some(item => isActiveRunStatus(item.status)); + if (!hasActiveRun) return undefined; + const timer = window.setInterval(() => { + loadProductionRuns(); + }, ACTIVE_RUN_REFRESH_INTERVAL_MS); + return () => window.clearInterval(timer); + }, [loadProductionRuns, runs]); + const loadRunDetail = useCallback(async runId => { if (!runId) { setRunDetail(null); @@ -576,6 +774,57 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { loadRunDetail(selectedRunId); }, [loadRunDetail, selectedRunId]); + useEffect(() => { + const statusText = String(runDetail?.run?.status || runDetail?.manifest?.status || '').toUpperCase(); + const active = Boolean(runDetail?.runtime_status?.active) || statusText.includes('RUNNING'); + if (!selectedRunId || !active) return undefined; + const timer = window.setInterval(() => { + loadRunDetail(selectedRunId); + }, ACTIVE_RUN_REFRESH_INTERVAL_MS); + return () => window.clearInterval(timer); + }, [loadRunDetail, runDetail?.manifest?.status, runDetail?.run?.status, runDetail?.runtime_status?.active, selectedRunId]); + + const loadLandsarRuns = useCallback(async () => { + setError(''); + try { + const data = await listLandsarSbasRuns(); + const items = Array.isArray(data?.items) ? data.items : []; + setLandsarRuns(items); + setSelectedLandsarRunId(current => current || items[0]?.run_id || ''); + } catch (exc) { + setError(exc?.response?.data?.detail || exc.message || 'LandSAR SBAS Run 列表加载失败'); + setLandsarRuns([]); + } + }, []); + + useEffect(() => { + loadLandsarRuns(); + }, [loadLandsarRuns]); + + const loadLandsarRunDetail = useCallback(async runId => { + if (!runId) { + setLandsarRunDetail(null); + return; + } + setRunDetailLoading(true); + setError(''); + try { + const data = await getLandsarSbasRun(runId); + setLandsarRunDetail(data); + } catch (exc) { + setError(exc?.response?.data?.detail || exc.message || 'LandSAR SBAS Run 详情加载失败'); + setLandsarRunDetail(null); + } finally { + setRunDetailLoading(false); + } + }, []); + + useEffect(() => { + if (processorMode === 'landsar') { + loadLandsarRunDetail(selectedLandsarRunId); + } + }, [loadLandsarRunDetail, processorMode, selectedLandsarRunId]); + const handleDiscoverStacks = useCallback(async () => { setDiscovering(true); setError(''); @@ -620,7 +869,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { const data = await submitSbasInsarRun(selectedStackId, { ...stackDiscoveryPayload, run_label: candidate - ? `${candidate.satellite || 'LT1'} ${formatAdminRegion(candidate.admin_region)} relOrbit ${candidate.relative_orbit || ''}`.trim() + ? `${candidate.satellite || selectedSensorFamily} ${formatAdminRegion(candidate.admin_region)} relOrbit ${candidate.relative_orbit || ''}`.trim() : undefined, dry_run: false, monitor_point_strategy: 'auto_representative_points', @@ -638,7 +887,84 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { } finally { setSubmitLoading(false); } - }, [readOnly, selectedStackId, stackCandidates, stackDiscoveryPayload]); + }, [readOnly, selectedSensorFamily, selectedStackId, stackCandidates, stackDiscoveryPayload]); + + const handleDeleteRun = useCallback(async runId => { + if (!runId || readOnly || runDeleteLoading) return; + const target = runs.find(item => item.run_id === runId); + const label = target?.run_label || target?.run_id || runId; + const ok = window.confirm( + `确定删除生产 Run ${label}？\n\n会删除运行目录、关联任务/Job 记录和已登记的 SBAS 成果。正在运行的任务会被后端拒绝删除。` + ); + if (!ok) return; + setRunDeleteLoading(true); + setError(''); + try { + await deleteSbasInsarRun(runId); + const runData = await listSbasInsarRuns(); + const runItems = Array.isArray(runData?.items) ? runData.items : []; + setRuns(runItems); + const nextRunId = selectedRunId === runId ? (runItems[0]?.run_id || '') : selectedRunId; + setSelectedRunId(nextRunId); + if (nextRunId) { + const detailData = await getSbasInsarRun(nextRunId); + setRunDetail(detailData); + } else { + setRunDetail(null); + } + } catch (exc) { + setError(exc?.response?.data?.detail || exc.message || 'SBAS-InSAR 生产 Run 删除失败'); + } finally { + setRunDeleteLoading(false); + } + }, [readOnly, runDeleteLoading, runs, selectedRunId]); + + const handleSubmitLandsarAutoWorkflow = useCallback(async () => { + if (readOnly) return; + setLandsarWorkflowJobLoading(true); + setLandsarSubmitLoading(true); + setError(''); + setStackAudit(null); + try { + const data = await submitLandsarSbasAutoWorkflow({ + ...stackDiscoveryPayload, + sensor_family: 'LT1', + require_orbits: false, + include_scenes: false, + dem_path: landsarDemPath || undefined, + timeout_seconds: 172800, + import_timeout_seconds: 172800, + workflow_timeout_seconds: 172800, + params: landsarParams, + }); + const runId = data?.run_id || data?.run?.run_id || data?.manifest?.run_id; + const selection = data?.selection || {}; + const selected = selection.selected_stack; + if (selected?.stack_id) { + setStackCandidates(selection.ranked_candidates || [selected]); + setSelectedStackId(selected.stack_id); + } + if (runId) { + setSelectedLandsarRunId(runId); + const detailData = await getLandsarSbasRun(runId); + setLandsarRunDetail(detailData); + } + if (data?.task_id) { + onTaskStart?.(data.task_id, 'LandSAR SBAS workflow queued.', { + taskType: data.job_type || 'SBAS_LANDSAR_WORKFLOW', + nonBlocking: true, + }); + } + setLandsarWorkflowJob(data); + await loadLandsarRuns(); + } catch (exc) { + setError(exc?.response?.data?.detail || exc.message || 'LandSAR SBAS 自动生产提交失败'); + setLandsarWorkflowJob(null); + } finally { + setLandsarSubmitLoading(false); + setLandsarWorkflowJobLoading(false); + } + }, [landsarDemPath, landsarParams, loadLandsarRuns, onTaskStart, readOnly, stackDiscoveryPayload]); const workflowPayload = useMemo(() => ({ force: false, @@ -672,6 +998,12 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { try { const data = await submitSbasInsarWorkflowJob(selectedRunId, workflowPayload); setWorkflowJob(data); + if (data?.task_id) { + onTaskStart?.(data.task_id, 'Gamma SBAS workflow queued.', { + taskType: data.job_type || 'SBAS_GAMMA_WORKFLOW', + nonBlocking: true, + }); + } const detailData = await getSbasInsarRun(selectedRunId); setRunDetail(detailData); const runData = await listSbasInsarRuns(); @@ -682,7 +1014,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { } finally { setWorkflowJobLoading(false); } - }, [readOnly, selectedRunId, workflowPayload]); + }, [onTaskStart, readOnly, selectedRunId, workflowPayload]); const handleBaselineAudit = useCallback(async (execute = false) => { if (!selectedRunId || readOnly) return; @@ -759,6 +1091,12 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { timeout_seconds: 43200, }); setCoregistrationJob(data); + if (data?.task_id) { + onTaskStart?.(data.task_id, 'SBAS coregistration task queued.', { + taskType: data.job_type || 'SBAS_COREGISTRATION', + nonBlocking: true, + }); + } const detailData = await getSbasInsarRun(selectedRunId); setRunDetail(detailData); const runData = await listSbasInsarRuns(); @@ -769,7 +1107,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { } finally { setCoregistrationJobLoading(false); } - }, [readOnly, selectedRunId]); + }, [onTaskStart, readOnly, selectedRunId]); const handlePrepareRdcDem = useCallback(async () => { if (!selectedRunId || readOnly) return; @@ -800,6 +1138,12 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { timeout_seconds: 43200, }); setRdcDemJob(data); + if (data?.task_id) { + onTaskStart?.(data.task_id, 'SBAS RDC DEM task queued.', { + taskType: data.job_type || 'SBAS_RDC_DEM', + nonBlocking: true, + }); + } const detailData = await getSbasInsarRun(selectedRunId); setRunDetail(detailData); const runData = await listSbasInsarRuns(); @@ -810,7 +1154,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { } finally { setRdcDemJobLoading(false); } - }, [readOnly, selectedRunId]); + }, [onTaskStart, readOnly, selectedRunId]); const handlePrepareInterferograms = useCallback(async () => { if (!selectedRunId || readOnly) return; @@ -845,6 +1189,12 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { timeout_seconds: 43200, }); setInterferogramJob(data); + if (data?.task_id) { + onTaskStart?.(data.task_id, 'SBAS interferogram task queued.', { + taskType: data.job_type || 'SBAS_INTERFEROGRAMS', + nonBlocking: true, + }); + } const detailData = await getSbasInsarRun(selectedRunId); setRunDetail(detailData); const runData = await listSbasInsarRuns(); @@ -855,7 +1205,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { } finally { setInterferogramJobLoading(false); } - }, [readOnly, selectedRunId]); + }, [onTaskStart, readOnly, selectedRunId]); const handlePrepareIptaTimeseries = useCallback(async () => { if (!selectedRunId || readOnly) return; @@ -888,6 +1238,12 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { timeout_seconds: 43200, }); setIptaTimeseriesJob(data); + if (data?.task_id) { + onTaskStart?.(data.task_id, 'SBAS IPTA timeseries task queued.', { + taskType: data.job_type || 'SBAS_IPTA_TIMESERIES', + nonBlocking: true, + }); + } const detailData = await getSbasInsarRun(selectedRunId); setRunDetail(detailData); const runData = await listSbasInsarRuns(); @@ -898,13 +1254,14 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { } finally { setIptaTimeseriesJobLoading(false); } - }, [readOnly, selectedRunId]); + }, [onTaskStart, readOnly, selectedRunId]); const selectedStack = stackCandidates.find(item => item.stack_id === selectedStackId) || null; const run = runDetail?.run || null; const runManifest = runDetail?.manifest || {}; const workflowManifest = runDetail?.workflow_manifest || {}; const workflowState = runDetail?.workflow_state || {}; + const runtimeStatus = runDetail?.runtime_status || null; const workflowSteps = Array.isArray(workflowManifest.steps) ? workflowManifest.steps : []; const expertDocumentSteps = Array.isArray(workflowManifest.expert_document?.steps) ? workflowManifest.expert_document.steps @@ -922,20 +1279,397 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { const publishProductsPlan = runManifest.publish_products || null; const monitorProductsPlan = runManifest.monitor_point_products || null; const runGeographicCoverage = runDetail?.geographic_coverage || null; - const runPrimaryPreview = ( - runArtifacts.find(item => item.key === 'los_rate_toward_m_per_year_hls_geo_preview_png') - || runArtifacts.find(item => item.key === 'los_rate_toward_mm_per_year_geo_preview_png') - || runArtifacts.find(item => item.key === 'los_rate_toward_mm_per_year_bmp') - ); - const runSigmaPreview = ( - runArtifacts.find(item => item.key === 'los_sigma_m_per_year_cc_geo_preview_png') - || runArtifacts.find(item => item.key === 'los_sigma_mm_per_year_geo_preview_png') - || runArtifacts.find(item => item.key === 'los_sigma_mm_per_year_bmp') - ); - const runMonitorPreview = runArtifacts.find(item => item.role === 'monitor_point' && item.relative_path.endsWith('.png')); - const runMonitorCsv = runArtifacts.find(item => item.role === 'monitor_point' && item.relative_path.endsWith('.csv')); const itabApproved = itabDecision?.decision === 'approve' || runManifest.baseline_audit?.approved_for_next_stage === true; const itabRejected = itabDecision?.decision === 'reject'; + const runSensorFamily = String(run?.sensor_family || runManifest.sensor_family || runManifest.profile_code || '').toUpperCase(); + const runExecutionEnabled = run?.execution_enabled !== false && runManifest.execution_enabled !== false && !runSensorFamily.startsWith('S1'); + const showGammaAdvancedActions = runManifest.show_advanced_actions === true; + const landsarCapability = (capabilities?.processors || []).find(item => item.processor_code === 'landsar_sbas') || {}; + const landsarRun = landsarRunDetail?.run || null; + const landsarManifest = landsarRunDetail?.manifest || {}; + const landsarWorkflow = landsarRunDetail?.workflow_manifest || {}; + const landsarArtifacts = landsarRunDetail?.artifacts || []; + const landsarPrimaryPreview = landsarArtifacts.find(item => item.relative_path === 'publish/landsar/preview.png'); + const landsarPrimaryTif = landsarArtifacts.find(item => item.relative_path === 'publish/landsar/los_timeseries.tif'); + const activeGammaRun = runs.find(item => isActiveRunStatus(item.status)) || null; + const activeGammaRunNotice = activeGammaRun ? ( + <div style={{ marginTop: 10, border: '1px solid #bae6fd', borderRadius: 8, background: '#f0f9ff', padding: 10, display: 'flex', justifyContent: 'space-between', gap: 10, alignItems: 'center', flexWrap: 'wrap' }}> + <div> + <div style={valueStyle}>Gamma run is active</div> + <div style={{ ...mutedStyle, marginTop: 4 }}> + {activeGammaRun.run_label || activeGammaRun.run_id}；{activeGammaRun.status} + </div> + </div> + <button + type="button" + onClick={() => { + setSelectedRunId(activeGammaRun.run_id); + setProcessorMode('gamma'); + }} + style={{ border: '1px solid #0369a1', borderRadius: 8, background: '#e0f2fe', color: '#0369a1', padding: '7px 11px', fontWeight: 750 }} + > + Open Runtime Status + </button> + </div> + ) : null; + const updateLandsarParam = (key, value) => { + setLandsarParams(current => ({ ...current, [key]: value })); + }; + const processorSelector = ( + <div style={{ display: 'flex', gap: 8, flexWrap: 'wrap', marginTop: 12 }}> + {[ + ['landsar', 'LandSAR SBAS'], + ['gamma', 'Gamma / IPTA SBAS'], + ].map(([key, label]) => ( + <button + key={key} + type="button" + onClick={() => setProcessorMode(key)} + style={{ + border: `1px solid ${processorMode === key ? '#0f766e' : '#cbd5e1'}`, + borderRadius: 8, + background: processorMode === key ? '#ccfbf1' : '#ffffff', + color: processorMode === key ? '#0f766e' : '#334155', + padding: '8px 12px', + fontWeight: 750, + cursor: 'pointer', + }} + > + {label} + </button> + ))} + </div> + ); + + if (processorMode === 'landsar') { + return ( + <div style={shellStyle}> + <section style={sectionStyle}> + <div style={{ display: 'flex', justifyContent: 'space-between', gap: 12, alignItems: 'flex-start' }}> + <div> + <h2 style={{ margin: 0, fontSize: 20, color: '#0f172a' }}>SBAS-InSAR 生产</h2> + <div style={{ ...mutedStyle, marginTop: 6 }}> + LandSAR SBAS 按生产区域自动发现 LT-1 时序栈、创建 Run，并在后台导入场景后执行一体化流程。 + </div> + {processorSelector} + </div> + <button + type="button" + onClick={() => { + loadProductionRuns(); + loadLandsarRuns(); + }} + disabled={loading} + style={{ + border: '1px solid #0f766e', + borderRadius: 8, + background: '#f0fdfa', + color: '#0f766e', + padding: '8px 12px', + fontWeight: 700, + cursor: loading ? 'default' : 'pointer', + whiteSpace: 'nowrap', + }} + > + {loading ? '刷新中' : '刷新'} + </button> + </div> + <div style={{ ...metricGridStyle, marginTop: 12 }}> + <Metric label="处理器" value={landsarCapability.processor_code || 'landsar_sbas'} /> + <Metric label="引擎" value={landsarCapability.engine_code || 'landsar'} /> + <Metric label="状态" value={landsarCapability.status || '-'} /> + <Metric label="最少景数" value={landsarCapability.min_scenes || landsarMinScenes} /> + </div> + {error && ( + <div style={{ marginTop: 10, padding: '8px 10px', borderRadius: 8, border: '1px solid #fecaca', background: '#fef2f2', color: '#991b1b', fontSize: 13 }}> + {error} + </div> + )} + </section> + + <section style={sectionStyle}> + <div style={{ display: 'flex', justifyContent: 'space-between', gap: 12, alignItems: 'flex-start' }}> + <div> + <h3 style={{ margin: 0, fontSize: 15, color: '#0f172a' }}>SBAS 生产区域</h3> + <div style={{ ...mutedStyle, marginTop: 5 }}> + 只需要指定生产区域；系统会自动寻找满足覆盖和时序条件的 LT-1 栈。下方候选仅用于审计，不需要人工选序列。 + </div> + </div> + <div style={{ display: 'flex', gap: 8, alignItems: 'center', flexWrap: 'wrap', justifyContent: 'flex-end' }}> + <span style={{ ...labelStyle, border: '1px solid #cbd5e1', borderRadius: 8, padding: '8px 10px', background: '#fff' }}>LT-1</span> + <input + value={stackAdminRegionQuery} + onChange={event => { + setStackAdminRegionQuery(event.target.value); + setStackCandidates([]); + setSelectedStackId(''); + setStackAudit(null); + }} + onKeyDown={event => { + if (event.key === 'Enter') handleSubmitLandsarAutoWorkflow(); + }} + placeholder="输入行政区，例如 牡丹江 / 洛阳" + style={{ + border: '1px solid #cbd5e1', + borderRadius: 8, + padding: '8px 10px', + fontSize: 12, + minWidth: 180, + }} + /> + <button + type="button" + onClick={handleDiscoverStacks} + disabled={discovering} + style={{ + border: '1px solid #0f766e', + borderRadius: 8, + background: '#f0fdfa', + color: '#0f766e', + padding: '8px 12px', + fontWeight: 700, + cursor: discovering ? 'default' : 'pointer', + whiteSpace: 'nowrap', + }} + > + {discovering ? '审计中' : '审计候选'} + </button> + {!readOnly && ( + <button + type="button" + onClick={handleSubmitLandsarAutoWorkflow} + disabled={landsarWorkflowJobLoading || landsarSubmitLoading} + style={{ + border: '1px solid #7c3aed', + borderRadius: 8, + background: '#f5f3ff', + color: '#6d28d9', + padding: '8px 12px', + fontWeight: 700, + cursor: landsarWorkflowJobLoading || landsarSubmitLoading ? 'default' : 'pointer', + whiteSpace: 'nowrap', + }} + > + {landsarWorkflowJobLoading || landsarSubmitLoading ? '提交中' : '自动创建并提交'} + </button> + )} + </div> + </div> + + <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(150px, 1fr))', gap: 10, marginTop: 12 }}> + <label style={{ display: 'grid', gap: 5, gridColumn: 'span 2' }}> + <span style={labelStyle}>DEM 文件</span> + <input value={landsarDemPath} onChange={event => setLandsarDemPath(event.target.value)} placeholder="D:\\DEM\\HeiLongJiang10M_DEM.tif" style={{ border: '1px solid #cbd5e1', borderRadius: 8, padding: '8px 10px', fontSize: 12 }} /> + </label> + <label style={{ display: 'grid', gap: 5 }}> + <span style={labelStyle}>最少景数</span> + <input type="number" value={landsarMinScenes} min={3} onChange={event => setLandsarMinScenes(event.target.value)} style={{ border: '1px solid #cbd5e1', borderRadius: 8, padding: '8px 10px', fontSize: 12 }} /> + </label> + <label style={{ display: 'grid', gap: 5 }}> + <span style={labelStyle}>干涉对方法</span> + <select value={landsarParams.intf_method} onChange={event => updateLandsarParam('intf_method', Number(event.target.value))} style={{ border: '1px solid #cbd5e1', borderRadius: 8, padding: '8px 10px', fontSize: 12, background: '#fff' }}> + <option value={0}>single</option> + <option value={1}>prim</option> + </select> + </label> + <label style={{ display: 'grid', gap: 5 }}> + <span style={labelStyle}>垂直基线</span> + <input type="number" value={landsarParams.perp_baseline} onChange={event => updateLandsarParam('perp_baseline', Number(event.target.value))} style={{ border: '1px solid #cbd5e1', borderRadius: 8, padding: '8px 10px', fontSize: 12 }} /> + </label> + <label style={{ display: 'grid', gap: 5 }}> + <span style={labelStyle}>时间基线</span> + <input type="number" value={landsarParams.time_baseline} onChange={event => updateLandsarParam('time_baseline', Number(event.target.value))} style={{ border: '1px solid #cbd5e1', borderRadius: 8, padding: '8px 10px', fontSize: 12 }} /> + </label> + <label style={{ display: 'grid', gap: 5 }}> + <span style={labelStyle}>多普勒基线</span> + <input type="number" value={landsarParams.doppler_baseline} onChange={event => updateLandsarParam('doppler_baseline', Number(event.target.value))} style={{ border: '1px solid #cbd5e1', borderRadius: 8, padding: '8px 10px', fontSize: 12 }} /> + </label> + <label style={{ display: 'grid', gap: 5 }}> + <span style={labelStyle}>方位向多视</span> + <input type="number" min={1} value={landsarParams.az_looks} onChange={event => updateLandsarParam('az_looks', Number(event.target.value))} style={{ border: '1px solid #cbd5e1', borderRadius: 8, padding: '8px 10px', fontSize: 12 }} /> + </label> + <label style={{ display: 'grid', gap: 5 }}> + <span style={labelStyle}>距离向多视</span> + <input type="number" min={1} value={landsarParams.rg_looks} onChange={event => updateLandsarParam('rg_looks', Number(event.target.value))} style={{ border: '1px solid #cbd5e1', borderRadius: 8, padding: '8px 10px', fontSize: 12 }} /> + </label> + <label style={{ display: 'grid', gap: 5 }}> + <span style={labelStyle}>DA 阈值</span> + <input type="number" step="0.01" min={0} max={1} value={landsarParams.da_threshold} onChange={event => updateLandsarParam('da_threshold', Number(event.target.value))} style={{ border: '1px solid #cbd5e1', borderRadius: 8, padding: '8px 10px', fontSize: 12 }} /> + </label> + </div> + + {stackCandidates.length > 0 && ( + <div style={{ display: 'grid', gridTemplateColumns: 'minmax(260px, 420px) minmax(0, 1fr)', gap: 12, marginTop: 12 }}> + <div style={{ display: 'grid', gap: 8, maxHeight: 360, overflow: 'auto' }}> + {stackCandidates.map(item => { + const active = item.stack_id === selectedStackId; + return ( + <div + key={item.stack_id} + style={{ + ...buttonBaseStyle, + borderColor: active ? '#1d4ed8' : '#d8dee8', + background: active ? '#eff6ff' : '#ffffff', + cursor: 'default', + }} + > + <div style={{ display: 'flex', justifyContent: 'space-between', gap: 8 }}> + <strong style={{ color: '#0f172a', fontSize: 13 }}> + {item.satellite || 'LT1'} / {item.orbit_direction || '-'} / relOrbit {item.relative_orbit || '-'} + </strong> + <div style={{ display: 'flex', gap: 6, alignItems: 'center', flexWrap: 'wrap', justifyContent: 'flex-end' }}> + <StatusBadge value={item.sensor_family || 'LT1'} /> + <StatusBadge value={item.status} /> + </div> + </div> + <div style={{ ...mutedStyle, marginTop: 6 }}> + {item.date_start} 至 {item.date_end}，可用 {item.usable_scene_count}/{item.scene_count} 景，最大间隔 {item.max_temporal_gap_days} 天 + </div> + <div style={{ ...mutedStyle, marginTop: 4 }}> + 行政区：{formatAdminRegion(item.admin_region)}；公共重叠 {formatPercent(item.common_overlap_ratio)} + </div> + <div style={{ ...mutedStyle, marginTop: 4 }}> + 影像组 {shortHash(item.scene_identity_hash)}；同日期序列 {item.same_date_sequence_candidate_count || 1} 组 + </div> + </div> + ); + })} + </div> + <div style={{ display: 'grid', gap: 10 }}> + {selectedStack && ( + <> + <div style={metricGridStyle}> + <Metric label="平台/模式" value={`${selectedStack.satellite || '-'} / ${selectedStack.imaging_mode || '-'}`} /> + <Metric label="轨道方向" value={selectedStack.orbit_direction || '-'} /> + <Metric label="极化/接收站" value={`${selectedStack.polarization || '-'} / ${selectedStack.receiving_station || '-'}`} /> + <Metric label="建议参考日期" value={selectedStack.reference_date || '-'} /> + <Metric label="公共重叠" value={formatPercent(selectedStack.common_overlap_ratio)} /> + <Metric label="最低公共重叠" value={formatPercent(selectedStack.min_common_overlap_ratio)} /> + <Metric label="AOI 覆盖" value={formatPercent(selectedStack.aoi_overlap_ratio_mean)} /> + </div> + {(selectedStack.blockers || []).length > 0 && ( + <div style={{ marginTop: 8, border: '1px solid #fecaca', borderRadius: 8, padding: 9, background: '#fef2f2', color: '#991b1b', fontSize: 12, lineHeight: 1.5 }}> + Blocked: {selectedStack.blockers.join('; ')} + </div> + )} + <StackIdentityNotice stack={selectedStack} /> + <SceneNamePanel stack={selectedStack} /> + <LocationSummaryPanel + coverage={{ + bbox: selectedStack.bbox || selectedStack.bbox_intersection, + bbox_intersection: selectedStack.bbox_intersection, + center: selectedStack.center || bboxCenter(selectedStack.bbox || selectedStack.bbox_intersection), + admin_region: selectedStack.admin_region, + scene_bbox_count: selectedStack.usable_scene_count || selectedStack.scene_count || 0, + }} + /> + </> + )} + {stackAudit && ( + <div style={{ border: '1px solid #dbeafe', borderRadius: 8, padding: 10, background: '#eff6ff' }}> + <div style={valueStyle}>Manifest 已生成</div> + <div style={{ ...mutedStyle, marginTop: 6, wordBreak: 'break-all' }}> + {stackAudit.manifest_path} + </div> + <div style={{ ...mutedStyle, marginTop: 6 }}> + 状态：{stackAudit.status}；pair 数：{stackAudit.manifest?.pair_network?.pairs?.length || 0} + </div> + {(stackAudit.manifest?.warnings || []).length > 0 && ( + <div style={{ ...mutedStyle, marginTop: 6 }}> + 警告：{stackAudit.manifest.warnings.join('；')} + </div> + )} + </div> + )} + {landsarRun && ( + <div style={{ border: '1px solid #ddd6fe', borderRadius: 8, padding: 10, background: '#f5f3ff' }}> + <div style={valueStyle}>LandSAR Run</div> + <div style={{ ...mutedStyle, marginTop: 6 }}> + {landsarRun.run_id}；状态：{landsarRun.status}；下一步：{landsarRun.next_stage || '-'} + </div> + </div> + )} + </div> + </div> + )} + {!discovering && stackCandidates.length === 0 && ( + <div style={{ ...mutedStyle, padding: '10px 0', marginTop: 8 }}>提交后后台任务会自动发现并选择生产序列；候选审计只用于提前查看系统会如何筛选。</div> + )} + {landsarWorkflowJob?.selection_pending && ( + <div style={{ ...mutedStyle, border: '1px solid #bbf7d0', borderRadius: 8, padding: 10, background: '#f0fdf4', marginTop: 10 }}> + 已提交后台自动生产任务：{landsarWorkflowJob.task_id || '-'}；Job：{landsarWorkflowJob.job_id || '-'}。系统正在复用 Gamma 的生产区域栈发现与审计逻辑选择 LT-1 序列，随后自动导入并执行 LandSAR SBAS。 + </div> + )} + </section> + + <section style={sectionStyle}> + <div style={{ display: 'flex', justifyContent: 'space-between', gap: 12, alignItems: 'flex-start' }}> + <div> + <h3 style={{ margin: 0, fontSize: 15, color: '#0f172a' }}>LandSAR Run</h3> + <div style={{ ...mutedStyle, marginTop: 5 }}>自动生产任务会创建 Run、导入选中序列并执行 LandSAR SBAS，完成后结果会进入 SBAS-InSAR 结果目录。</div> + </div> + </div> + + <div style={{ display: 'grid', gridTemplateColumns: 'minmax(260px, 420px) minmax(0, 1fr)', gap: 12, marginTop: 12 }}> + <div style={{ display: 'grid', gap: 8, maxHeight: 320, overflow: 'auto' }}> + {landsarRuns.map(item => { + const active = item.run_id === selectedLandsarRunId; + return ( + <button key={item.run_id} type="button" onClick={() => setSelectedLandsarRunId(item.run_id)} style={{ ...buttonBaseStyle, borderColor: active ? '#7c3aed' : '#d8dee8', background: active ? '#f5f3ff' : '#fff' }}> + <div style={{ display: 'flex', justifyContent: 'space-between', gap: 8 }}> + <strong style={{ color: '#0f172a', fontSize: 13 }}>{item.run_label || item.run_id}</strong> + <StatusBadge value={item.status} /> + </div> + <div style={{ ...mutedStyle, marginTop: 6 }}>{item.scene_count || 0} 景，{item.task_count || 0} 个 Task，{item.date_start || '-'} 至 {item.date_end || '-'}</div> + <div style={{ ...mutedStyle, marginTop: 4, wordBreak: 'break-all' }}>{item.run_id}</div> + </button> + ); + })} + {!loading && landsarRuns.length === 0 && <div style={{ ...mutedStyle, padding: '10px 0' }}>暂无 LandSAR SBAS Run。</div>} + </div> + <div style={{ display: 'grid', gap: 10 }}> + {runDetailLoading && <div style={mutedStyle}>正在加载 Run 详情...</div>} + {!runDetailLoading && landsarRun && ( + <> + <div style={metricGridStyle}> + <Metric label="状态" value={landsarRun.status || '-'} /> + <Metric label="Task 数" value={landsarRun.task_count || landsarManifest.task_count || 0} /> + <Metric label="场景数" value={landsarRun.scene_count || landsarManifest.scene_count || 0} /> + <Metric label="下一阶段" value={landsarRun.next_stage || '-'} /> + </div> + {landsarPrimaryPreview && ( + <div style={{ border: '1px solid #e2e8f0', borderRadius: 8, overflow: 'hidden', background: '#0f172a', maxWidth: 520 }}> + <img src={getLandsarSbasRunArtifactUrl(landsarRun.run_id, landsarPrimaryPreview.relative_path)} alt={landsarRun.run_label || landsarRun.run_id} style={{ display: 'block', width: '100%', objectFit: 'contain' }} /> + </div> + )} + {landsarWorkflowJob && landsarWorkflowJob.run_id === landsarRun.run_id && ( + <div style={{ ...mutedStyle, border: '1px solid #bbf7d0', borderRadius: 8, padding: 10, background: '#f0fdf4' }}> + 已提交后台任务：{landsarWorkflowJob.task_id}；Job：{landsarWorkflowJob.job_id} + </div> + )} + <div style={{ display: 'grid', gap: 6 }}> + {(landsarArtifacts.slice(0, 24)).map(asset => ( + <div key={asset.relative_path} style={{ display: 'grid', gridTemplateColumns: 'minmax(130px, 180px) minmax(0, 1fr) auto', gap: 8, alignItems: 'center', border: '1px solid #e2e8f0', borderRadius: 8, padding: '7px 9px' }}> + <div style={{ fontSize: 12, fontWeight: 750, color: '#0f172a' }}>{asset.role}</div> + <div style={{ ...mutedStyle, wordBreak: 'break-all' }}>{asset.relative_path}</div> + <a href={getLandsarSbasRunArtifactUrl(landsarRun.run_id, asset.relative_path)} target="_blank" rel="noreferrer" style={{ color: '#1d4ed8', fontSize: 12, fontWeight: 750 }}>打开</a> + </div> + ))} + </div> + {landsarPrimaryTif && ( + <div style={mutedStyle}>主 GeoTIFF：{landsarPrimaryTif.relative_path}</div> + )} + {landsarWorkflow?.task_results && ( + <div style={mutedStyle}>完成 {landsarWorkflow.completed_count || 0}，失败 {landsarWorkflow.failed_count || 0}</div> + )} + </> + )} + </div> + </div> + </section> + </div> + ); + } return ( <div style={shellStyle}> @@ -946,6 +1680,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { <div style={{ ...mutedStyle, marginTop: 6 }}> Gamma IPTA SBAS 生产入口。当前阶段接入已验证的 LT1/Gamma 试验成果，作业提交在下一阶段开放。 </div> + {processorSelector} </div> <button type="button" @@ -993,6 +1728,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { {error} </div> )} + {activeGammaRunNotice} </section> <section style={sectionStyle}> @@ -1004,6 +1740,27 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { </div> </div> <div style={{ display: 'flex', gap: 8, alignItems: 'center', flexWrap: 'wrap', justifyContent: 'flex-end' }}> + <select + value={selectedSensorFamily} + onChange={event => { + setSelectedSensorFamily(event.target.value); + setStackCandidates([]); + setSelectedStackId(''); + setStackAudit(null); + }} + style={{ + border: '1px solid #cbd5e1', + borderRadius: 8, + padding: '8px 10px', + fontSize: 12, + minWidth: 130, + background: '#fff', + color: '#0f172a', + }} + > + <option value="LT1">LT-1</option> + <option value="S1">Sentinel-1</option> + </select> <input value={stackAdminRegionQuery} onChange={event => { @@ -1100,7 +1857,10 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { <strong style={{ color: '#0f172a', fontSize: 13 }}> {item.satellite || 'LT1'} / {item.orbit_direction || '-'} / relOrbit {item.relative_orbit || '-'} </strong> - <StatusBadge value={item.status} /> + <div style={{ display: 'flex', gap: 6, alignItems: 'center', flexWrap: 'wrap', justifyContent: 'flex-end' }}> + <StatusBadge value={item.sensor_family || selectedSensorFamily} /> + <StatusBadge value={item.status} /> + </div> </div> <div style={{ ...mutedStyle, marginTop: 6 }}> {item.date_start} 至 {item.date_end}，可用 {item.usable_scene_count}/{item.scene_count} 景， @@ -1112,6 +1872,9 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { <div style={{ ...mutedStyle, marginTop: 4 }}> 覆盖 {formatPercent(item.aoi_overlap_ratio_mean)}；中心点 {formatCenter(item.center)} </div> + <div style={{ ...mutedStyle, marginTop: 4 }}> + 影像组 {shortHash(item.scene_identity_hash)}；同日期序列 {item.same_date_sequence_candidate_count || 1} 组 + </div> </button> ); })} @@ -1125,8 +1888,16 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { <Metric label="极化/接收站" value={`${selectedStack.polarization || '-'} / ${selectedStack.receiving_station || '-'}`} /> <Metric label="建议参考日期" value={selectedStack.reference_date || '-'} /> <Metric label="公共重叠" value={formatPercent(selectedStack.common_overlap_ratio)} /> + <Metric label="最低公共重叠" value={formatPercent(selectedStack.min_common_overlap_ratio)} /> <Metric label="AOI 覆盖" value={formatPercent(selectedStack.aoi_overlap_ratio_mean)} /> </div> + {(selectedStack.blockers || []).length > 0 && ( + <div style={{ marginTop: 8, border: '1px solid #fecaca', borderRadius: 8, padding: 9, background: '#fef2f2', color: '#991b1b', fontSize: 12, lineHeight: 1.5 }}> + Blocked: {selectedStack.blockers.join('; ')} + </div> + )} + <StackIdentityNotice stack={selectedStack} /> + <SceneNamePanel stack={selectedStack} /> <LocationSummaryPanel coverage={{ bbox: selectedStack.bbox || selectedStack.bbox_intersection, @@ -1188,6 +1959,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { <div style={{ display: 'grid', gap: 8, maxHeight: 300, overflow: 'auto' }}> {runs.map(item => { const active = item.run_id === selectedRunId; + const running = isActiveRunStatus(item.status); return ( <button key={item.run_id} @@ -1208,14 +1980,17 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { <div style={{ ...mutedStyle, marginTop: 6 }}> {item.scene_count || 0} 景，{item.pair_count || 0} 对，下一步 {item.next_stage || '-'} </div> + <div style={{ ...mutedStyle, marginTop: 4 }}> + 影像组 {shortHash(item.scene_identity_hash)} + </div> + {running && ( + <div style={{ marginTop: 6, color: '#0369a1', fontSize: 12, fontWeight: 700 }}> + 正在运行，已自动打开右侧 Runtime Status + </div> + )} </button> ); })} - {runs.length > 0 && ( - <div style={{ ...mutedStyle, padding: '2px 0 6px' }}> - 当前 Run 列表已补充中心点行政区；筛选入口优先放在候选序列发现阶段。 - </div> - )} {!loading && runs.length === 0 && ( <div style={{ ...mutedStyle, padding: '10px 0' }}> 暂无计划 Run。先发现序列，再创建计划 Run。 @@ -1227,18 +2002,84 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { {runDetailLoading && <div style={mutedStyle}>正在加载 Run 详情...</div>} {!runDetailLoading && run && ( <> + <div + style={{ + border: '1px solid #e2e8f0', + borderRadius: 8, + padding: 10, + background: '#f8fafc', + display: 'flex', + justifyContent: 'space-between', + alignItems: 'center', + gap: 10, + flexWrap: 'wrap', + }} + > + <div> + <div style={valueStyle}>{run.run_label || run.run_id}</div> + <div style={{ ...mutedStyle, marginTop: 4, wordBreak: 'break-all' }}> + Run ID: {run.run_id} + </div> + </div> + {!readOnly && ( + <button + type="button" + onClick={() => handleDeleteRun(run.run_id)} + disabled={runDeleteLoading} + style={{ + border: '1px solid #b91c1c', + borderRadius: 8, + background: runDeleteLoading ? '#f8fafc' : '#fef2f2', + color: runDeleteLoading ? '#94a3b8' : '#b91c1c', + padding: '7px 11px', + fontWeight: 700, + cursor: runDeleteLoading ? 'default' : 'pointer', + }} + > + {runDeleteLoading ? '删除中' : '删除 Run'} + </button> + )} + </div> + <div style={metricGridStyle}> <Metric label="状态" value={run.status || '-'} /> <Metric label="参考日期" value={run.reference_date || '-'} /> <Metric label="场景/配对" value={`${run.scene_count || 0} / ${run.pair_count || 0}`} /> <Metric label="下一阶段" value={run.next_stage || '-'} /> + <Metric label="影像组" value={shortHash(run.scene_identity_hash)} /> + <Metric + label="公共重叠" + value={`${formatPercent(run.common_overlap_ratio)} / ${formatPercent(run.min_common_overlap_ratio)}`} + /> </div> - <LocationSummaryPanel coverage={runGeographicCoverage} /> + <SceneNamePanel stack={{ ...run, scenes: runManifest.scenes }} /> + + <RuntimeStatusPanel status={runtimeStatus} /> + + <details style={compactDetailsStyle}> + <summary style={compactSummaryStyle}>空间覆盖</summary> + <div style={{ marginTop: 10 }}> + <LocationSummaryPanel coverage={runGeographicCoverage} /> + </div> + </details> {!readOnly && ( <div style={{ border: '1px solid #bbf7d0', borderRadius: 8, padding: 10, background: '#f0fdf4' }}> <div style={valueStyle}>Gamma SBAS Workflow</div> + {!runExecutionEnabled && ( + <div style={{ + marginTop: 8, + padding: '8px 10px', + borderRadius: 8, + border: '1px solid #fed7aa', + background: '#fff7ed', + color: '#9a3412', + fontSize: 12, + }}> + Sentinel-1 Gamma SBAS is planning-only. Stack discovery, audit manifest and run record are enabled; Gamma execution is disabled until the S1 TOPS/SBAS scripts are verified. + </div> + )} <div style={{ ...mutedStyle, marginTop: 6 }}> 专家文档目录 + manifest + WSL runner 主路径。旧分阶段执行仅作为兼容桥接。 </div> @@ -1246,7 +2087,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { <button type="button" onClick={handlePrepareWorkflow} - disabled={workflowLoading} + disabled={workflowLoading || !runExecutionEnabled} style={{ border: '1px solid #15803d', borderRadius: 8, @@ -1254,7 +2095,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { color: '#166534', padding: '8px 12px', fontWeight: 700, - cursor: workflowLoading ? 'default' : 'pointer', + cursor: workflowLoading || !runExecutionEnabled ? 'default' : 'pointer', }} > {workflowLoading ? '生成中' : '生成 Workflow Manifest'} @@ -1262,7 +2103,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { <button type="button" onClick={handleSubmitWorkflowJob} - disabled={workflowJobLoading} + disabled={workflowJobLoading || !runExecutionEnabled} style={{ border: '1px solid #0f766e', borderRadius: 8, @@ -1270,7 +2111,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { color: '#0f766e', padding: '8px 12px', fontWeight: 700, - cursor: workflowJobLoading ? 'default' : 'pointer', + cursor: workflowJobLoading || !runExecutionEnabled ? 'default' : 'pointer', }} > {workflowJobLoading ? '提交中' : '提交 Gamma SBAS Workflow'} @@ -1282,7 +2123,9 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { </div> )} {workflowSteps.length > 0 && ( - <div style={{ display: 'grid', gap: 6, marginTop: 10 }}> + <details style={{ ...compactDetailsStyle, marginTop: 10, borderColor: '#bbf7d0' }}> + <summary style={compactSummaryStyle}>Workflow steps ({workflowSteps.length})</summary> + <div style={{ display: 'grid', gap: 6, marginTop: 10 }}> {workflowSteps.map(step => { const state = workflowStepState[step.id] || {}; return ( @@ -1311,13 +2154,14 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { </div> ); })} - </div> + </div> + </details> )} {expertDocumentSteps.length > 0 && ( - <div style={{ marginTop: 12 }}> - <div style={valueStyle}>Expert document path</div> + <details style={{ ...compactDetailsStyle, marginTop: 10 }}> + <summary style={compactSummaryStyle}>Expert document path ({expertDocumentSteps.length})</summary> <div style={{ ...mutedStyle, marginTop: 4 }}> - {expertDocumentSteps.length} sections from the LT1 Gamma SBAS expert document. Commands are shown as the acceptance checklist; implementation may be a bridge where the verified experiment already covers the same Gamma function. + {expertDocumentSteps.length} sections from the LT1 Gamma SBAS expert document. Commands are used as the acceptance checklist; completed workflow steps must pass the expert command audit. </div> <div style={{ display: 'grid', gap: 6, marginTop: 8 }}> {expertDocumentSteps.map(item => { @@ -1361,17 +2205,17 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { ); })} </div> - </div> + </details> )} </div> )} - {!readOnly && false && ( + {!readOnly && showGammaAdvancedActions && ( <div style={{ display: 'flex', gap: 8, flexWrap: 'wrap' }}> <button type="button" onClick={() => handleBaselineAudit(false)} - disabled={baselineAuditLoading} + disabled={baselineAuditLoading || !runExecutionEnabled} style={{ border: '1px solid #1d4ed8', borderRadius: 8, @@ -1379,7 +2223,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { color: '#1d4ed8', padding: '8px 12px', fontWeight: 700, - cursor: baselineAuditLoading ? 'default' : 'pointer', + cursor: baselineAuditLoading || !runExecutionEnabled ? 'default' : 'pointer', }} > {baselineAuditLoading ? '处理中' : '生成/解析 baseline audit'} @@ -1387,7 +2231,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { <button type="button" onClick={() => handleBaselineAudit(true)} - disabled={baselineAuditLoading} + disabled={baselineAuditLoading || !runExecutionEnabled} style={{ border: '1px solid #7c3aed', borderRadius: 8, @@ -1395,7 +2239,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { color: '#6d28d9', padding: '8px 12px', fontWeight: 700, - cursor: baselineAuditLoading ? 'default' : 'pointer', + cursor: baselineAuditLoading || !runExecutionEnabled ? 'default' : 'pointer', }} > 执行 Gamma baseline audit @@ -1403,7 +2247,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { <button type="button" onClick={handlePrepareCoregistration} - disabled={coregistrationLoading} + disabled={coregistrationLoading || !runExecutionEnabled} style={{ border: '1px solid #0f766e', borderRadius: 8, @@ -1411,7 +2255,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { color: '#0f766e', padding: '8px 12px', fontWeight: 700, - cursor: coregistrationLoading ? 'default' : 'pointer', + cursor: coregistrationLoading || !runExecutionEnabled ? 'default' : 'pointer', }} > {coregistrationLoading ? '生成中' : '生成共参考配准脚本'} @@ -1419,7 +2263,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { <button type="button" onClick={handleSubmitCoregistrationJob} - disabled={coregistrationJobLoading} + disabled={coregistrationJobLoading || !runExecutionEnabled} style={{ border: '1px solid #b45309', borderRadius: 8, @@ -1427,7 +2271,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { color: '#92400e', padding: '8px 12px', fontWeight: 700, - cursor: coregistrationJobLoading ? 'default' : 'pointer', + cursor: coregistrationJobLoading || !runExecutionEnabled ? 'default' : 'pointer', }} > {coregistrationJobLoading ? '提交中' : '提交共参考配准任务'} @@ -1435,7 +2279,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { <button type="button" onClick={handlePrepareRdcDem} - disabled={rdcDemLoading} + disabled={rdcDemLoading || !runExecutionEnabled} style={{ border: '1px solid #0369a1', borderRadius: 8, @@ -1443,7 +2287,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { color: '#0369a1', padding: '8px 12px', fontWeight: 700, - cursor: rdcDemLoading ? 'default' : 'pointer', + cursor: rdcDemLoading || !runExecutionEnabled ? 'default' : 'pointer', }} > {rdcDemLoading ? '生成中' : '生成 RDC DEM 脚本'} @@ -1451,7 +2295,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { <button type="button" onClick={handleSubmitRdcDemJob} - disabled={rdcDemJobLoading} + disabled={rdcDemJobLoading || !runExecutionEnabled} style={{ border: '1px solid #4338ca', borderRadius: 8, @@ -1459,7 +2303,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { color: '#3730a3', padding: '8px 12px', fontWeight: 700, - cursor: rdcDemJobLoading ? 'default' : 'pointer', + cursor: rdcDemJobLoading || !runExecutionEnabled ? 'default' : 'pointer', }} > {rdcDemJobLoading ? '提交中' : '提交 RDC DEM 任务'} @@ -1467,7 +2311,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { <button type="button" onClick={handlePrepareInterferograms} - disabled={interferogramLoading} + disabled={interferogramLoading || !runExecutionEnabled} style={{ border: '1px solid #7c2d12', borderRadius: 8, @@ -1475,7 +2319,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { color: '#7c2d12', padding: '8px 12px', fontWeight: 700, - cursor: interferogramLoading ? 'default' : 'pointer', + cursor: interferogramLoading || !runExecutionEnabled ? 'default' : 'pointer', }} > {interferogramLoading ? '生成中' : '生成干涉图脚本'} @@ -1483,7 +2327,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { <button type="button" onClick={handleSubmitInterferogramsJob} - disabled={interferogramJobLoading} + disabled={interferogramJobLoading || !runExecutionEnabled} style={{ border: '1px solid #be123c', borderRadius: 8, @@ -1491,7 +2335,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { color: '#be123c', padding: '8px 12px', fontWeight: 700, - cursor: interferogramJobLoading ? 'default' : 'pointer', + cursor: interferogramJobLoading || !runExecutionEnabled ? 'default' : 'pointer', }} > {interferogramJobLoading ? '提交中' : '提交干涉图任务'} @@ -1499,7 +2343,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { <button type="button" onClick={handlePrepareIptaTimeseries} - disabled={iptaTimeseriesLoading} + disabled={iptaTimeseriesLoading || !runExecutionEnabled} style={{ border: '1px solid #166534', borderRadius: 8, @@ -1507,7 +2351,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { color: '#166534', padding: '8px 12px', fontWeight: 700, - cursor: iptaTimeseriesLoading ? 'default' : 'pointer', + cursor: iptaTimeseriesLoading || !runExecutionEnabled ? 'default' : 'pointer', }} > {iptaTimeseriesLoading ? '生成中' : '生成 IPTA 脚本'} @@ -1515,7 +2359,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { <button type="button" onClick={handleSubmitIptaTimeseriesJob} - disabled={iptaTimeseriesJobLoading} + disabled={iptaTimeseriesJobLoading || !runExecutionEnabled} style={{ border: '1px solid #15803d', borderRadius: 8, @@ -1523,7 +2367,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { color: '#166534', padding: '8px 12px', fontWeight: 700, - cursor: iptaTimeseriesJobLoading ? 'default' : 'pointer', + cursor: iptaTimeseriesJobLoading || !runExecutionEnabled ? 'default' : 'pointer', }} > {iptaTimeseriesJobLoading ? '提交中' : '提交 IPTA 任务'} @@ -1531,6 +2375,9 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { </div> )} + <details style={compactDetailsStyle}> + <summary style={compactSummaryStyle}>阶段/脚本明细</summary> + <div style={{ display: 'grid', gap: 10, marginTop: 10 }}> <div style={{ border: '1px solid #e2e8f0', borderRadius: 8, padding: 10 }}> <div style={valueStyle}>Gamma 阶段计划</div> <div style={{ display: 'grid', gap: 6, marginTop: 8 }}> @@ -1588,7 +2435,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { <button type="button" onClick={() => handleItabDecision('approve')} - disabled={itabDecisionLoading || itabApproved} + disabled={itabDecisionLoading || itabApproved || !runExecutionEnabled} style={{ border: '1px solid #0f766e', borderRadius: 8, @@ -1596,7 +2443,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { color: itabApproved ? '#94a3b8' : '#0f766e', padding: '7px 11px', fontWeight: 700, - cursor: itabDecisionLoading || itabApproved ? 'default' : 'pointer', + cursor: itabDecisionLoading || itabApproved || !runExecutionEnabled ? 'default' : 'pointer', }} > {itabApproved ? 'itab 已批准' : '批准 itab'} @@ -1604,7 +2451,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { <button type="button" onClick={() => handleItabDecision('reject')} - disabled={itabDecisionLoading || itabApproved || itabRejected} + disabled={itabDecisionLoading || itabApproved || itabRejected || !runExecutionEnabled} style={{ border: '1px solid #b91c1c', borderRadius: 8, @@ -1612,7 +2459,7 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { color: itabApproved || itabRejected ? '#94a3b8' : '#b91c1c', padding: '7px 11px', fontWeight: 700, - cursor: itabDecisionLoading || itabApproved || itabRejected ? 'default' : 'pointer', + cursor: itabDecisionLoading || itabApproved || itabRejected || !runExecutionEnabled ? 'default' : 'pointer', }} > {itabRejected ? 'itab 已拒绝' : '拒绝 itab'} @@ -1933,76 +2780,13 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { )} </div> )} - - {(runPrimaryPreview || runSigmaPreview || runMonitorPreview) && ( - <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(260px, 1fr))', gap: 12 }}> - {runPrimaryPreview && ( - <div> - <div style={{ ...valueStyle, marginBottom: 6 }}>LOS Velocity</div> - <div style={{ ...mutedStyle, marginBottom: 6 }}> - {runPrimaryPreview.key.endsWith('_geo_preview_png') ? 'WGS84 geocoded preview' : 'RDC QA preview'} - </div> - <img - alt="Run LOS velocity toward radar positive" - src={getSbasInsarRunArtifactUrl(run.run_id, runPrimaryPreview.relative_path)} - style={{ - width: '100%', - aspectRatio: '4 / 3', - objectFit: 'contain', - border: '1px solid #d8dee8', - borderRadius: 8, - background: '#f8fafc', - }} - /> - </div> - )} - {runSigmaPreview && ( - <div> - <div style={{ ...valueStyle, marginBottom: 6 }}>LOS Sigma</div> - <div style={{ ...mutedStyle, marginBottom: 6 }}> - {runSigmaPreview.key.endsWith('_geo_preview_png') ? 'WGS84 geocoded preview' : 'RDC QA preview'} - </div> - <img - alt="Run LOS velocity sigma" - src={getSbasInsarRunArtifactUrl(run.run_id, runSigmaPreview.relative_path)} - style={{ - width: '100%', - aspectRatio: '4 / 3', - objectFit: 'contain', - border: '1px solid #d8dee8', - borderRadius: 8, - background: '#f8fafc', - }} - /> - </div> - )} - {runMonitorPreview && ( - <div> - <div style={{ ...valueStyle, marginBottom: 6 }}>Monitoring Curve</div> - <img - alt="Run monitoring point LOS displacement time series" - src={getSbasInsarRunArtifactUrl(run.run_id, runMonitorPreview.relative_path)} - style={{ - width: '100%', - aspectRatio: '4 / 3', - objectFit: 'contain', - border: '1px solid #d8dee8', - borderRadius: 8, - background: '#ffffff', - }} - /> - {runMonitorCsv && ( - <div style={{ ...mutedStyle, marginTop: 6 }}> - <RunArtifactLink runId={run.run_id} artifact={runMonitorCsv} /> - </div> - )} - </div> - )} </div> - )} + </details> {runArtifacts.length > 0 && ( - <div style={{ border: '1px solid #e2e8f0', borderRadius: 8, overflow: 'hidden' }}> + <details style={compactDetailsStyle}> + <summary style={compactSummaryStyle}>资产下载 ({runArtifacts.length})</summary> + <div style={{ border: '1px solid #e2e8f0', borderRadius: 8, overflow: 'hidden', marginTop: 10 }}> <table style={{ width: '100%', borderCollapse: 'collapse', fontSize: 13 }}> <thead> <tr style={{ background: '#f8fafc', color: '#475569' }}> @@ -2023,7 +2807,8 @@ export default function SbasInsarProductionPanel({ readOnly = false }) { ))} </tbody> </table> - </div> + </div> + </details> )} </> )} diff --git a/frontend/src/SbasInsarProductsPanel.jsx b/frontend/src/SbasInsarProductsPanel.jsx index 4894ed8..da02aa2 100644 --- a/frontend/src/SbasInsarProductsPanel.jsx +++ b/frontend/src/SbasInsarProductsPanel.jsx @@ -1,4 +1,15 @@ -import React, { useCallback, useEffect, useMemo, useState } from 'react'; +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { LineChart } from 'echarts/charts'; +import { + DataZoomComponent, + GridComponent, + LegendComponent, + MarkLineComponent, + TooltipComponent, + ToolboxComponent, +} from 'echarts/components'; +import * as echarts from 'echarts/core'; +import { CanvasRenderer } from 'echarts/renderers'; import { getSbasInsarCatalogStatus, @@ -6,9 +17,21 @@ import { getSbasInsarProductDetail, getSbasInsarProductPreviewUrl, listSbasInsarProducts, + querySbasInsarPointTimeseries, queueSbasInsarCatalogRebuild, } from './api/sbasInsarProducts'; +echarts.use([ + LineChart, + GridComponent, + TooltipComponent, + LegendComponent, + DataZoomComponent, + ToolboxComponent, + MarkLineComponent, + CanvasRenderer, +]); + const statusColors = { READY: '#15803d', WARN: '#b45309', @@ -150,7 +173,193 @@ function findAssets(assets, roles) { return assets.filter(asset => roleSet.has(asset.asset_role) && asset.exists_flag); } -function ProductPreview({ title, asset, productId }) { +function assetCacheKey(asset) { + if (!asset) return ''; + return [asset.id, asset.file_size, asset.updated_at || asset.created_at || asset.relative_path].filter(Boolean).join(':'); +} + +function productCacheKey(product) { + if (!product) return ''; + return [ + product.id, + product.status, + product.health_status, + product.updated_at || product.published_at || product.produced_at || product.manifest_fingerprint, + ].filter(Boolean).join(':'); +} + +const assetRoleInfo = { + run_manifest: { + label: '运行清单', + description: '记录本次 SBAS 运行的参数、状态、引擎信息和可追溯入口。', + }, + stack_manifest: { + label: '影像栈清单', + description: '记录参考景、参与景、日期序列、轨道和输入栈元数据。', + }, + workflow_summary: { + label: '处理流程摘要', + description: '记录生产流程、步骤状态、命令链路和关键执行信息。', + }, + product_summary: { + label: '产品摘要', + description: '旧版托管产品摘要；专家 Gamma 模式下不再作为必需产物。', + }, + quality_summary: { + label: '质量摘要', + description: '旧版质量统计摘要；专家 Gamma 模式下统计由 GeoTIFF 派生。', + }, + monitor_points_summary: { + label: '监测点摘要', + description: '记录监测点选择、曲线产物和点位时序文件。', + }, + monitor_point_selection_metadata: { + label: '监测点选点策略元数据', + description: '记录每个自动监测点的选点策略、顺序、说明和雷达坐标。', + }, + unwrapped_phase_summary: { + label: '解缠相位摘要', + description: '记录 final_unw_tab 中最终解缠相位文件的地理编码导出状态。', + }, + unwrapped_phase_geotiff: { + label: '解缠相位 GeoTIFF', + description: '由 Gamma SBAS 最终解缠相位文件地理编码得到的检查栅格，单位为 rad。', + }, + unwrapped_phase_preview: { + label: '解缠相位预览图', + description: '解缠相位 GeoTIFF 的快速浏览 PNG，用于检查空间连续性和异常条带。', + }, + unwrapped_phase_radar_preview: { + label: '解缠相位雷达坐标预览图', + description: '由 final_unw_tab 最终解缠相位在雷达坐标下渲染的彩色检查图；色表采用 Gamma rmg.cm。', + }, + unwrapped_phase_radar_bmp: { + label: '解缠相位雷达坐标 BMP', + description: 'Gamma 风格的雷达坐标解缠相位原始浏览 BMP，适合专家复核。', + }, + unwrapped_phase_radar_colorbar: { + label: '解缠相位色卡', + description: '解缠相位浏览色卡，Gamma rmg.cm 仅表示所用色表，显示范围为 -6.28 到 6.28 rad。', + }, + point_vector_summary: { + label: '速率点矢量摘要', + description: '记录 LOS 速率点矢量的字段、数量、坐标系和导出状态。', + }, + point_vector_geojson_gz: { + label: '速率点矢量 GeoJSON', + description: '用于 GIS 或前端叠加检视的压缩点矢量文件。', + }, + primary_geocoded_preview: { + label: 'LOS 速率预览图', + description: '面向快速检视的地理编码速率图，专家 Gamma 模式对应 geo_los_def_rate RGB 预览。', + }, + primary_rate_color_preview: { + label: 'LOS 速率纯色图', + description: '由 geo_los_def_rate.tif 按 Gamma hls.cm 直接上色的速率图，不叠加强度图或底图；0 速率按无效值透明处理。', + }, + primary_preview: { + label: 'LOS 结果预览图', + description: '面向快速检视的 SBAS 结果预览图。', + }, + quality_geocoded_preview: { + label: 'LOS Sigma 预览图', + description: '速度不确定性或残差质量图的预览图；专家 Gamma 当前链路未生成时不显示。', + }, + primary_geotiff: { + label: 'LOS 速率 GeoTIFF', + description: '核心结果栅格，可用于 QGIS、ArcGIS、Python 和后续统计分析。', + }, + alternate_geotiff: { + label: 'LOS 反向约定 GeoTIFF', + description: '旧版 away-from-radar 符号约定下的备用速率栅格。', + }, + quality_geotiff: { + label: 'LOS Sigma GeoTIFF', + description: '速度不确定性或残差质量栅格。', + }, + primary_rgb_geotiff: { + label: 'LOS 速率 RGB GeoTIFF', + description: '按 Gamma hls.cm 色表渲染后的 RGB 栅格，适合制图和浏览。', + }, + quality_rgb_geotiff: { + label: 'LOS Sigma RGB GeoTIFF', + description: '质量或 Sigma 栅格渲染后的 RGB 浏览文件。', + }, + primary_colorbar: { + label: 'LOS 速率色卡', + description: '与速率预览图一致的 Gamma hls.cm 色卡，用于解释颜色和速率范围。', + }, + gamma_phase_rate: { + label: 'Gamma 相位速率栅格', + description: 'Gamma 原生相位速率中间产物，用于专家复核。', + }, + gamma_sigma_rate: { + label: 'Gamma Sigma 速率栅格', + description: 'Gamma 原生 Sigma 速率中间产物，用于质量复核。', + }, + gamma_qc_baseline_plot: { + label: 'Gamma 基线网络图', + description: '由 base_plot 输出的干涉网络/基线分布图，用于检查参与干涉对的时空基线关系。', + }, + gamma_qc_mean_coherence: { + label: 'Gamma 平均相干掩膜', + description: '由解缠阶段使用的 mean.cc_mask.bmp，用于检查解缠有效区和低相干屏蔽范围。', + }, + gamma_qc_unwrapped_phase: { + label: 'Gamma 代表性解缠质控图', + description: '从进入 final_unw_tab 的干涉对中抽取的滤波解缠相位 BMP，用于快速复核中间过程。', + }, + height_correction: { + label: '高程改正栅格', + description: '高度误差改正相关产物，用于检查 DEM 或高程残差影响。', + }, + monitor_points: { + label: '监测点时序表', + description: '专家命令 disp_prt_2d 输出的监测点形变时序原始表。', + }, + monitor_point_items: { + label: '监测点字段说明', + description: 'disp_prt_2d 输出表的列定义和日期字段说明。', + }, + monitor_point_selection: { + label: '监测点选点文件', + description: '专家链路使用的雷达坐标监测点选择文件。', + }, + monitor_point_curve: { + label: '监测点形变曲线', + description: '由 disp_prt_2d 时序表派生的点位形变折线图。', + }, + monitor_point_csv: { + label: '监测点时序 CSV', + description: '单个监测点的日期-形变量表格，便于复核和二次绘图。', + }, + monitor_point_metadata: { + label: '监测点元数据', + description: '单个监测点的位置、高程、速率和残差信息。', + }, + command_manifest: { + label: '命令清单', + description: '记录外部处理器命令、参数和执行链路。', + }, + native_console_log: { + label: '原生日志目录', + description: '外部处理器的控制台日志或运行日志。', + }, + secondary_geotiff: { + label: '辅助 GeoTIFF', + description: '处理器导出的辅助栅格结果。', + }, +}; + +function getAssetRoleInfo(asset) { + const role = asset?.asset_role || ''; + return assetRoleInfo[role] || { + label: asset?.asset_name || role || '未命名资产', + description: '系统登记的补充资产，用于下载、归档或专家复核。', + }; +} + +function ProductPreview({ title, asset, productId, imageMaxHeight = 300, onOpen }) { if (!asset) { return ( <div style={{ border: '1px solid #e2e8f0', borderRadius: 8, padding: 10, background: '#f8fafc' }}> @@ -161,19 +370,167 @@ function ProductPreview({ title, asset, productId }) { } return ( <div style={{ border: '1px solid #e2e8f0', borderRadius: 8, overflow: 'hidden', background: '#ffffff' }}> - <div style={{ padding: '8px 10px', fontSize: 12, fontWeight: 700, color: '#0f172a', background: '#f8fafc' }}> - {title} + <div style={{ display: 'flex', justifyContent: 'space-between', gap: 10, alignItems: 'center', padding: '8px 10px', background: '#f8fafc' }}> + <div style={{ fontSize: 12, fontWeight: 700, color: '#0f172a' }}>{title}</div> + {onOpen && ( + <button type="button" onClick={onOpen} style={{ ...buttonStyle, padding: '4px 8px' }}> + 查看大图 + </button> + )} </div> <img - src={getSbasInsarProductAssetUrl(productId, asset.id)} + src={getSbasInsarProductAssetUrl(productId, asset.id, assetCacheKey(asset))} alt={title} - style={{ display: 'block', width: '100%', maxHeight: 300, objectFit: 'contain', background: '#0f172a' }} + onClick={onOpen} + style={{ + display: 'block', + width: '100%', + maxHeight: imageMaxHeight, + objectFit: 'contain', + background: '#ffffff', + cursor: onOpen ? 'zoom-in' : 'default', + }} /> <div style={{ ...mutedStyle, padding: '7px 10px', wordBreak: 'break-all' }}>{asset.relative_path}</div> </div> ); } +function ImageLightbox({ image, onClose }) { + if (!image) return null; + return ( + <div + role="dialog" + aria-modal="true" + onMouseDown={onClose} + style={{ + position: 'fixed', + inset: 0, + zIndex: 2000, + background: 'rgba(15, 23, 42, 0.76)', + display: 'grid', + placeItems: 'center', + padding: 20, + }} + > + <div + onMouseDown={event => event.stopPropagation()} + style={{ + width: 'min(1500px, 96vw)', + maxHeight: '94vh', + background: '#ffffff', + borderRadius: 8, + border: '1px solid #cbd5e1', + display: 'grid', + gridTemplateRows: 'auto minmax(0, 1fr) auto', + overflow: 'hidden', + }} + > + <div style={{ display: 'flex', justifyContent: 'space-between', gap: 12, alignItems: 'center', padding: '10px 12px', background: '#f8fafc', borderBottom: '1px solid #e2e8f0' }}> + <div style={{ fontSize: 13, fontWeight: 800, color: '#0f172a' }}>{image.title}</div> + <button type="button" onClick={onClose} style={buttonStyle}>关闭</button> + </div> + <div style={{ overflow: 'auto', background: '#ffffff', padding: 12 }}> + <img + src={image.src} + alt={image.title} + style={{ display: 'block', width: '100%', height: 'auto', objectFit: 'contain', background: '#ffffff' }} + /> + </div> + <div style={{ ...mutedStyle, padding: '8px 12px', borderTop: '1px solid #e2e8f0', wordBreak: 'break-all' }}> + {image.path || '-'} + </div> + </div> + </div> + ); +} + +function AssetActionLink({ label, asset, productId }) { + if (!asset?.exists_flag) return null; + return ( + <a + href={getSbasInsarProductAssetUrl(productId, asset.id, assetCacheKey(asset))} + target="_blank" + rel="noreferrer" + style={{ ...buttonStyle, textDecoration: 'none', display: 'inline-flex', justifyContent: 'center' }} + > + {label} + </a> + ); +} + +function VelocityInspectionPanel({ + asset, + colorbarAsset, + colorPolicy, + primaryGeotiff, + rgbGeotiff, + productId, + onOpen, +}) { + const range = Array.isArray(colorPolicy?.display_range_mm_per_year) ? colorPolicy.display_range_mm_per_year : null; + const rangeText = range && range.length >= 2 ? `${formatNumber(range[0], 0)} 到 ${formatNumber(range[1], 0)} mm/yr` : '-'; + const sourceText = colorPolicy?.source || 'expert_gamma_command'; + const browseText = colorPolicy?.browse_command || 'rasdt_pwr / geocode_back / data2geotiff'; + + if (!asset) { + return ( + <div style={{ border: '1px solid #e2e8f0', borderRadius: 8, padding: 12, background: '#f8fafc' }}> + <div style={{ fontSize: 13, fontWeight: 800, color: '#0f172a' }}>LOS 速率检视</div> + <div style={{ ...mutedStyle, marginTop: 6 }}>暂无预览。</div> + </div> + ); + } + + return ( + <div style={{ border: '1px solid #cbd5e1', borderRadius: 8, overflow: 'hidden', background: '#ffffff' }}> + <div style={{ display: 'flex', justifyContent: 'space-between', gap: 12, alignItems: 'flex-start', padding: '12px 14px', background: '#f8fafc', borderBottom: '1px solid #e2e8f0' }}> + <div> + <div style={{ fontSize: 15, fontWeight: 850, color: '#0f172a' }}>LOS 速率检视</div> + <div style={{ ...mutedStyle, marginTop: 4 }}>Gamma expert chain: {browseText}</div> + </div> + <div style={{ display: 'flex', flexWrap: 'wrap', gap: 8, justifyContent: 'flex-end' }}> + <button type="button" onClick={onOpen} style={buttonStyle}>查看大图</button> + <AssetActionLink label="打开 GeoTIFF" asset={primaryGeotiff} productId={productId} /> + <AssetActionLink label="打开 RGB" asset={rgbGeotiff} productId={productId} /> + </div> + </div> + + <div style={{ display: 'grid', gridTemplateColumns: 'minmax(0, 1fr) minmax(230px, 300px)', gap: 0 }}> + <div style={{ minWidth: 0, background: '#ffffff', borderRight: '1px solid #e2e8f0' }}> + <img + src={getSbasInsarProductAssetUrl(productId, asset.id, assetCacheKey(asset))} + alt="LOS 速率图" + onClick={onOpen} + style={{ display: 'block', width: '100%', maxHeight: 680, objectFit: 'contain', background: '#ffffff', cursor: 'zoom-in' }} + /> + </div> + + <div style={{ display: 'grid', alignContent: 'start', gap: 10, padding: 12, background: '#ffffff' }}> + <Metric label="色表" value={colorPolicy?.colormap || 'Gamma hls.cm'} /> + <Metric label="显示范围" value={rangeText} /> + <Metric label="来源" value={sourceText} /> + <Metric label="预览文件" value={formatBytes(asset.file_size)} /> + {colorbarAsset ? ( + <div> + <div style={{ fontSize: 12, fontWeight: 800, color: '#0f172a', marginBottom: 6 }}>速率色卡</div> + <img + src={getSbasInsarProductAssetUrl(productId, colorbarAsset.id, assetCacheKey(colorbarAsset))} + alt="Gamma hls.cm colorbar" + style={{ display: 'block', width: '100%', height: 'auto', background: '#ffffff', border: '1px solid #e2e8f0', borderRadius: 6 }} + /> + </div> + ) : null} + </div> + </div> + + <div style={{ ...mutedStyle, padding: '8px 12px', borderTop: '1px solid #e2e8f0', wordBreak: 'break-all' }}> + {asset.relative_path} + </div> + </div> + ); +} + function PointVectorDownload({ asset, summary, productId }) { if (!asset && !summary) return null; const fields = Array.isArray(summary?.fields) ? summary.fields : []; @@ -187,7 +544,7 @@ function PointVectorDownload({ asset, summary, productId }) { </div> </div> {asset ? ( - <a href={getSbasInsarProductAssetUrl(productId, asset.id)} target="_blank" rel="noreferrer" style={{ ...buttonStyle, textDecoration: 'none' }}> + <a href={getSbasInsarProductAssetUrl(productId, asset.id, assetCacheKey(asset))} target="_blank" rel="noreferrer" style={{ ...buttonStyle, textDecoration: 'none' }}> 下载 </a> ) : ( @@ -209,6 +566,842 @@ function PointVectorDownload({ asset, summary, productId }) { ); } +function GammaIntermediateQcPanel({ assets = [], productId, onOpen }) { + const [expanded, setExpanded] = useState(false); + const readyAssets = (Array.isArray(assets) ? assets : []).filter(asset => asset?.exists_flag); + if (!readyAssets.length) return null; + + const baseline = readyAssets.find(asset => asset.asset_role === 'gamma_qc_baseline_plot'); + const coherence = readyAssets.find(asset => asset.asset_role === 'gamma_qc_mean_coherence'); + const unwrapped = readyAssets.filter(asset => asset.asset_role === 'gamma_qc_unwrapped_phase'); + const previewAssets = [baseline, coherence, ...unwrapped].filter(Boolean); + + return ( + <div style={{ border: '1px solid #dbe3ef', borderRadius: 8, background: '#ffffff', overflow: 'hidden' }}> + <button + type="button" + onClick={() => setExpanded(value => !value)} + style={{ + width: '100%', + border: 0, + background: '#f8fafc', + cursor: 'pointer', + padding: '10px 12px', + display: 'flex', + justifyContent: 'space-between', + gap: 12, + alignItems: 'center', + textAlign: 'left', + }} + > + <span> + <span style={{ display: 'block', fontSize: 13, fontWeight: 850, color: '#0f172a' }}>Gamma 中间质控图</span> + <span style={{ display: 'block', ...mutedStyle, marginTop: 3 }}> + 只展示少量专家复核入口：基线网络、平均相干掩膜和代表性解缠相位；完整中间文件仍在资产下载中保留。 + </span> + </span> + <span style={{ color: '#334155', fontSize: 12, fontWeight: 800, whiteSpace: 'nowrap' }}> + {readyAssets.length} 项 / {expanded ? '收起' : '展开'} + </span> + </button> + + {expanded && ( + <div style={{ display: 'grid', gap: 10, padding: 12, borderTop: '1px solid #e2e8f0' }}> + <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(260px, 1fr))', gap: 10 }}> + {previewAssets.map(asset => { + const info = getAssetRoleInfo(asset); + const isLargeBmp = asset.format === 'BMP' || String(asset.relative_path || '').toLowerCase().endsWith('.bmp'); + return ( + <ProductPreview + key={asset.id} + title={info.label} + asset={asset} + productId={productId} + imageMaxHeight={isLargeBmp ? 360 : 300} + onOpen={() => onOpen?.(info.label, asset)} + /> + ); + })} + </div> + <div style={mutedStyle}> + 这些图来自 Gamma 中间阶段，主要用于判断网络、相干和解缠是否异常；它们不是最终形变速率成果，不能替代上方 LOS 速率图和 GeoTIFF。 + </div> + </div> + )} + </div> + ); +} + +function monitorPointIdFromAsset(asset) { + const text = [asset?.relative_path, asset?.asset_name].filter(Boolean).join(' '); + const match = text.match(/expert_point_\d{3}/i); + return match ? match[0] : ''; +} + +function pairIdFromUnwrappedAsset(asset) { + const text = [asset?.relative_path, asset?.asset_name].filter(Boolean).join(' '); + const match = text.match(/\d{8}_\d{8}/); + return match ? match[0] : text; +} + +function buildMonitorPointCards(monitorPoints, monitorOutputs, assets) { + const cards = new Map(); + const ensure = pointId => { + const id = String(pointId || '').trim(); + if (!id) return null; + if (!cards.has(id)) { + cards.set(id, { point_id: id, point: { point_id: id }, output: null, assets: {} }); + } + return cards.get(id); + }; + + (Array.isArray(monitorPoints) ? monitorPoints : []).forEach((point, index) => { + const fallbackId = `expert_point_${String(index + 1).padStart(3, '0')}`; + const card = ensure(point?.point_id || fallbackId); + if (card) card.point = { ...card.point, ...point, point_id: card.point_id }; + }); + + (Array.isArray(monitorOutputs) ? monitorOutputs : []).forEach((output, index) => { + const metadata = output?.metadata || {}; + const fallbackId = `expert_point_${String(index + 1).padStart(3, '0')}`; + const card = ensure(output?.point_id || metadata.point_id || fallbackId); + if (card) { + card.output = output; + card.point = { ...card.point, ...metadata, point_id: card.point_id }; + } + }); + + (Array.isArray(assets) ? assets : []).forEach(asset => { + if (!asset?.exists_flag) return; + if (!['monitor_point_curve', 'monitor_point_csv', 'monitor_point_metadata'].includes(asset.asset_role)) return; + const card = ensure(monitorPointIdFromAsset(asset)); + if (!card) return; + if (asset.asset_role === 'monitor_point_curve') card.assets.curve = asset; + if (asset.asset_role === 'monitor_point_csv') card.assets.csv = asset; + if (asset.asset_role === 'monitor_point_metadata') card.assets.metadata = asset; + }); + + return Array.from(cards.values()).sort((left, right) => { + const leftRank = Number(left.point?.selection_rank ?? 9999); + const rightRank = Number(right.point?.selection_rank ?? 9999); + if (leftRank !== rightRank) return leftRank - rightRank; + return left.point_id.localeCompare(right.point_id); + }); +} + +function buildUnwrappedPhaseCards(summary, previews, geotiffs, radarPreviews = [], radarBmps = []) { + const cards = new Map(); + const ensure = pairId => { + const id = String(pairId || '').trim(); + if (!id) return null; + if (!cards.has(id)) cards.set(id, { pair_id: id, summary: null, preview: null, geotiff: null, radarPreview: null, radarBmp: null }); + return cards.get(id); + }; + + (Array.isArray(summary?.products) ? summary.products : []).forEach((item, index) => { + const card = ensure(item?.pair_id || `unwrapped_${String(index + 1).padStart(3, '0')}`); + if (card) card.summary = item; + }); + (Array.isArray(previews) ? previews : []).forEach(asset => { + const card = ensure(pairIdFromUnwrappedAsset(asset)); + if (card) card.preview = asset; + }); + (Array.isArray(geotiffs) ? geotiffs : []).forEach(asset => { + const card = ensure(pairIdFromUnwrappedAsset(asset)); + if (card) card.geotiff = asset; + }); + (Array.isArray(radarPreviews) ? radarPreviews : []).forEach(asset => { + const card = ensure(pairIdFromUnwrappedAsset(asset)); + if (card) card.radarPreview = asset; + }); + (Array.isArray(radarBmps) ? radarBmps : []).forEach(asset => { + const card = ensure(pairIdFromUnwrappedAsset(asset)); + if (card) card.radarBmp = asset; + }); + return Array.from(cards.values()).sort((left, right) => left.pair_id.localeCompare(right.pair_id)); +} + +function InlineImageAsset({ title, asset, productId, imageMaxHeight = 260, onOpen }) { + if (!asset?.exists_flag) { + return ( + <div style={{ border: '1px dashed #cbd5e1', borderRadius: 6, padding: 10, background: '#f8fafc' }}> + <div style={{ fontSize: 12, fontWeight: 800, color: '#0f172a' }}>{title}</div> + <div style={{ ...mutedStyle, marginTop: 4 }}>暂无预览。</div> + </div> + ); + } + return ( + <div style={{ border: '1px solid #e2e8f0', borderRadius: 6, overflow: 'hidden', background: '#ffffff' }}> + <div style={{ display: 'flex', justifyContent: 'space-between', gap: 8, alignItems: 'center', padding: '7px 9px', background: '#f8fafc' }}> + <div style={{ fontSize: 12, fontWeight: 800, color: '#0f172a' }}>{title}</div> + {onOpen && ( + <button type="button" onClick={onOpen} style={{ ...buttonStyle, padding: '4px 8px' }}> + 查看大图 + </button> + )} + </div> + <img + src={getSbasInsarProductAssetUrl(productId, asset.id, assetCacheKey(asset))} + alt={title} + onClick={onOpen} + style={{ + display: 'block', + width: '100%', + maxHeight: imageMaxHeight, + objectFit: 'contain', + background: '#ffffff', + cursor: onOpen ? 'zoom-in' : 'default', + }} + /> + </div> + ); +} + +const monitorChartColors = ['#1d4ed8', '#dc2626', '#059669', '#7c3aed', '#d97706', '#0f766e']; + +function normalizeMonitorDate(value) { + const raw = String(value || '').trim(); + if (/^\d{8}$/.test(raw)) return `${raw.slice(0, 4)}-${raw.slice(4, 6)}-${raw.slice(6, 8)}`; + return raw.slice(0, 10); +} + +function parseMonitorDateMs(value) { + const date = normalizeMonitorDate(value); + const match = date.match(/^(\d{4})-(\d{2})-(\d{2})$/); + if (!match) { + const fallback = Date.parse(date); + return Number.isFinite(fallback) ? fallback : NaN; + } + return Date.UTC(Number(match[1]), Number(match[2]) - 1, Number(match[3])); +} + +function niceAxisStep(rawStep) { + if (!Number.isFinite(rawStep) || rawStep <= 0) return 1; + const exponent = Math.floor(Math.log10(rawStep)); + const base = rawStep / (10 ** exponent); + let niceBase = 10; + if (base <= 1) niceBase = 1; + else if (base <= 2) niceBase = 2; + else if (base <= 5) niceBase = 5; + return niceBase * (10 ** exponent); +} + +function buildLinearAxis(values, targetTicks = 6) { + const finiteValues = values.filter(Number.isFinite); + if (!finiteValues.length) return { min: -1, max: 1, step: 0.5, ticks: [-1, -0.5, 0, 0.5, 1] }; + + const dataMin = Math.min(...finiteValues); + const dataMax = Math.max(...finiteValues); + if (dataMin === 0 && dataMax === 0) return { min: -1, max: 1, step: 0.5, ticks: [-1, -0.5, 0, 0.5, 1] }; + + const dataRange = Math.max(dataMax - dataMin, Math.max(Math.abs(dataMin), Math.abs(dataMax)) * 0.2, 1); + let lower = Math.min(dataMin, 0); + let upper = Math.max(dataMax, 0); + const padding = dataRange * 0.06; + if (dataMin < 0) lower -= padding; + if (dataMax > 0) upper += padding; + if (lower === upper) { + lower -= 1; + upper += 1; + } + + const step = niceAxisStep((upper - lower) / Math.max(2, targetTicks - 1)); + const min = Math.floor(lower / step) * step; + const max = Math.ceil(upper / step) * step; + const ticks = []; + const count = Math.max(1, Math.round((max - min) / step)); + for (let index = 0; index <= count; index += 1) { + const value = min + step * index; + ticks.push(Math.abs(value) < Math.abs(step) * 1e-9 ? 0 : value); + } + if (!ticks.includes(0) && min < 0 && max > 0) ticks.push(0); + ticks.sort((left, right) => left - right); + return { min, max, step, ticks }; +} + +function formatAxisTick(value, step) { + const normalized = Math.abs(value) < Math.max(Math.abs(step), 1) * 1e-9 ? 0 : value; + const absStep = Math.abs(step); + const digits = absStep >= 1 ? 0 : absStep >= 0.1 ? 1 : absStep >= 0.01 ? 2 : 3; + return normalized.toFixed(digits); +} + +function formatUtcDate(value) { + const numeric = Number(value); + const date = new Date(numeric); + if (!Number.isFinite(numeric) || Number.isNaN(date.getTime())) return String(value ?? '-'); + const year = date.getUTCFullYear(); + const month = String(date.getUTCMonth() + 1).padStart(2, '0'); + const day = String(date.getUTCDate()).padStart(2, '0'); + return `${year}-${month}-${day}`; +} + +function normalizeMonitorDisplacements(point) { + const rows = Array.isArray(point?.displacements) ? point.displacements : []; + return rows + .map(item => { + const date = normalizeMonitorDate(item?.date); + const time = parseMonitorDateMs(date); + const displacement = Number(item?.displacement_mm ?? item?.displacement ?? item?.value); + if (!date || !Number.isFinite(time) || !Number.isFinite(displacement)) return null; + return { date, time, displacement }; + }) + .filter(Boolean) + .sort((left, right) => left.time - right.time); +} + +function EchartsTimeSeriesCanvas({ option }) { + const containerRef = useRef(null); + const chartRef = useRef(null); + + useEffect(() => { + if (!containerRef.current) return undefined; + const chart = echarts.init(containerRef.current, null, { renderer: 'canvas' }); + chartRef.current = chart; + + let resizeObserver = null; + if (typeof ResizeObserver !== 'undefined') { + resizeObserver = new ResizeObserver(() => chart.resize()); + resizeObserver.observe(containerRef.current); + } + const handleResize = () => chart.resize(); + window.addEventListener('resize', handleResize); + + return () => { + window.removeEventListener('resize', handleResize); + resizeObserver?.disconnect(); + chart.dispose(); + chartRef.current = null; + }; + }, []); + + useEffect(() => { + if (!chartRef.current || !option) return; + chartRef.current.setOption(option, true); + }, [option]); + + return <div ref={containerRef} style={{ width: '100%', minWidth: 720, height: 430 }} />; +} + +function CombinedMonitorPointChart({ cards, title = '监测点合并形变曲线', subtitle, emptyText }) { + const chart = useMemo(() => { + const series = (Array.isArray(cards) ? cards : []) + .map((card, index) => { + const point = card.point || {}; + const values = normalizeMonitorDisplacements(point); + const isQuery = Boolean(card.isQuery || point.is_query || point.matched); + return { + pointId: card.point_id || point.point_id || `point_${index + 1}`, + label: point.selection_label || card.point_id, + key: point.selection_key || '', + rate: Number(point.deformation_rate_mm_per_year), + matched: point.matched || null, + isQuery, + color: isQuery ? '#111827' : monitorChartColors[index % monitorChartColors.length], + values, + }; + }) + .filter(item => item.values.length > 0); + + const dateEntries = Array.from( + new Map( + series + .flatMap(item => item.values) + .map(value => [value.date, { date: value.date, time: value.time }]), + ).values(), + ).sort((left, right) => left.time - right.time); + const dates = dateEntries.map(item => item.date); + const times = dateEntries.map(item => item.time); + const allValues = series.flatMap(item => item.values.map(value => value.displacement)); + if (!series.length || !dates.length || !allValues.length) return { series: [], dates: [], yMin: -1, yMax: 1 }; + const yAxis = buildLinearAxis(allValues, 6); + return { + series, + dates, + dateEntries, + minTime: Math.min(...times), + maxTime: Math.max(...times), + yAxis, + yMin: yAxis.min, + yMax: yAxis.max, + }; + }, [cards]); + + const option = useMemo(() => { + if (!chart.series.length) return null; + const yAxis = chart.yAxis || { min: chart.yMin, max: chart.yMax, step: 0.5 }; + const oneDay = 24 * 60 * 60 * 1000; + const xMin = chart.minTime === chart.maxTime ? chart.minTime - oneDay : chart.minTime; + const xMax = chart.minTime === chart.maxTime ? chart.maxTime + oneDay : chart.maxTime; + + return { + animation: false, + backgroundColor: '#ffffff', + color: chart.series.map(item => item.color), + tooltip: { + trigger: 'axis', + confine: true, + axisPointer: { type: 'line', snap: true, lineStyle: { color: '#475569', width: 1 } }, + formatter: params => { + const rows = (Array.isArray(params) ? params : [params]).filter(item => Array.isArray(item?.value)); + if (!rows.length) return ''; + const date = rows[0]?.data?.date || formatUtcDate(rows[0].value[0]); + const body = rows + .map(item => { + const value = Number(item.value[1]); + return `${item.marker}<span style="font-weight:650">${item.seriesName}</span>: ${formatNumber(value, 2)} mm`; + }) + .join('<br/>'); + return `<div style="font-weight:750;margin-bottom:4px">${date}</div>${body}`; + }, + }, + legend: { + type: 'scroll', + top: 6, + left: 8, + right: 100, + itemWidth: 14, + itemHeight: 8, + textStyle: { color: '#334155', fontSize: 11 }, + }, + toolbox: { + top: 26, + right: 12, + itemSize: 14, + feature: { + dataZoom: { yAxisIndex: 'none', title: { zoom: '区域缩放', back: '缩放还原' } }, + restore: { title: '还原' }, + saveAsImage: { title: '保存图片', name: title }, + }, + }, + grid: { left: 72, right: 28, top: 64, bottom: 70 }, + xAxis: { + type: 'time', + min: xMin, + max: xMax, + name: 'SAR Date', + nameLocation: 'middle', + nameGap: 42, + axisLabel: { + color: '#64748b', + hideOverlap: true, + formatter: value => formatUtcDate(value), + }, + axisLine: { lineStyle: { color: '#94a3b8' } }, + splitLine: { show: true, lineStyle: { color: '#f1f5f9' } }, + }, + yAxis: { + type: 'value', + min: yAxis.min, + max: yAxis.max, + interval: yAxis.step, + name: '累计形变 (mm)', + nameLocation: 'middle', + nameGap: 52, + axisLabel: { + color: '#64748b', + formatter: value => formatAxisTick(Number(value), yAxis.step), + }, + axisLine: { show: true, lineStyle: { color: '#94a3b8' } }, + splitLine: { show: true, lineStyle: { color: '#e2e8f0' } }, + }, + dataZoom: [ + { type: 'inside', xAxisIndex: 0, filterMode: 'none' }, + { type: 'slider', xAxisIndex: 0, filterMode: 'none', height: 24, bottom: 18 }, + ], + series: chart.series.map((item, index) => ({ + name: item.label, + type: 'line', + data: item.values.map(value => ({ + value: [value.time, Number(value.displacement.toFixed(6))], + date: value.date, + })), + showSymbol: true, + symbol: 'circle', + symbolSize: item.isQuery ? 8 : 6, + smooth: false, + connectNulls: false, + emphasis: { focus: 'series' }, + lineStyle: { width: item.isQuery ? 3 : 2.2, color: item.color }, + itemStyle: { color: item.color, borderColor: '#ffffff', borderWidth: 1 }, + markLine: index === 0 ? { + symbol: 'none', + silent: true, + data: [{ yAxis: 0, name: '0 mm' }], + label: { formatter: '0 mm', color: '#475569' }, + lineStyle: { color: '#0f172a', opacity: 0.32, type: 'dashed', width: 1 }, + } : undefined, + })), + }; + }, [chart, title]); + + if (!chart.series.length) { + return ( + <div style={{ border: '1px dashed #cbd5e1', borderRadius: 8, padding: 12, background: '#f8fafc' }}> + <div style={{ fontSize: 13, fontWeight: 850, color: '#0f172a' }}>{title}</div> + <div style={{ ...mutedStyle, marginTop: 5 }}> + {emptyText || '当前摘要还没有内嵌日期-形变序列；重新注册资产后会由 `disp_prt_2d` 输出生成合并曲线。'} + </div> + </div> + ); + } + + return ( + <div style={{ border: '1px solid #cbd5e1', borderRadius: 8, background: '#ffffff', overflow: 'hidden' }}> + <div style={{ padding: '10px 12px', background: '#f8fafc', borderBottom: '1px solid #e2e8f0' }}> + <div style={{ display: 'flex', justifyContent: 'space-between', gap: 12, alignItems: 'flex-start', flexWrap: 'wrap' }}> + <div> + <div style={{ fontSize: 14, fontWeight: 850, color: '#0f172a' }}>{title}</div> + <div style={{ ...mutedStyle, marginTop: 4 }}> + {subtitle || '同一次 Gamma `disp_prt_2d` 时序输出，按自动选点策略叠加显示；横坐标按实际日期间隔缩放，纵坐标为累计形变 mm。'} + </div> + </div> + <div style={{ color: '#334155', fontSize: 12, fontWeight: 750 }}> + {chart.series.length} 条曲线 / {chart.dates.length} 期 + </div> + </div> + </div> + + <div style={{ padding: '10px 12px 0', overflowX: 'auto' }}> + <EchartsTimeSeriesCanvas option={option} /> + </div> + + <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(210px, 1fr))', gap: 8, padding: '8px 12px 12px' }}> + {chart.series.map(item => ( + <div key={`legend-${item.pointId}`} style={{ display: 'flex', gap: 8, alignItems: 'flex-start', minWidth: 0 }}> + <span style={{ width: 10, height: 10, borderRadius: 999, background: item.color, marginTop: 4, flex: '0 0 auto' }} /> + <div style={{ minWidth: 0 }}> + <div style={{ fontSize: 12, fontWeight: 800, color: '#0f172a', overflowWrap: 'anywhere' }}>{item.label}</div> + <div style={{ ...mutedStyle, marginTop: 1 }}> + {item.pointId}{item.key ? ` / ${item.key}` : ''}，速率 {Number.isFinite(item.rate) ? `${formatNumber(item.rate, 2)} mm/yr` : '-'} + {item.matched?.used_nearest ? `，最近邻 ${formatNumber(item.matched.distance_m, 1)} m` : ''} + </div> + </div> + </div> + ))} + </div> + </div> + ); +} + +function PointTimeseriesLookup({ productId, result, onResult, onClear }) { + const [lon, setLon] = useState(''); + const [lat, setLat] = useState(''); + const [loading, setLoading] = useState(false); + const [message, setMessage] = useState(''); + + const queryPoint = async () => { + const numericLon = Number(lon); + const numericLat = Number(lat); + if (!Number.isFinite(numericLon) || !Number.isFinite(numericLat)) { + setMessage('请输入有效的 WGS84 经度和纬度。'); + return; + } + if (numericLon < -180 || numericLon > 180 || numericLat < -90 || numericLat > 90) { + setMessage('经纬度超出 WGS84 范围。'); + return; + } + setLoading(true); + setMessage(''); + try { + const nextResult = await querySbasInsarPointTimeseries(productId, { lon: numericLon, lat: numericLat }); + onResult?.(nextResult); + setMessage('查询完成，结果已加入下方 ECharts 曲线。'); + } catch (error) { + onClear?.(); + setMessage(`查询失败：${error?.response?.data?.detail || error.message}`); + } finally { + setLoading(false); + } + }; + + const matched = result?.matched || {}; + + return ( + <div style={{ border: '1px solid #dbe3ef', borderRadius: 8, background: '#ffffff', overflow: 'hidden' }}> + <div style={{ display: 'grid', gap: 10, padding: 12, background: '#f8fafc', borderBottom: '1px solid #e2e8f0' }}> + <div style={{ display: 'flex', justifyContent: 'space-between', gap: 10, alignItems: 'flex-start', flexWrap: 'wrap' }}> + <div> + <div style={{ fontSize: 13, fontWeight: 850, color: '#0f172a' }}>按 WGS84 经纬度查询形变曲线</div> + <div style={{ ...mutedStyle, marginTop: 4 }}> + 输入经纬度后，系统会在有效覆盖区内取该位置或最近有效像元，并返回对应雷达像素的 SBAS 时序。 + </div> + </div> + </div> + <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(150px, 1fr)) minmax(150px, auto)', gap: 8, alignItems: 'center' }}> + <input + value={lon} + onChange={event => setLon(event.target.value)} + onKeyDown={event => { + if (event.key === 'Enter') queryPoint(); + }} + placeholder="经度 lon，如 129.18" + style={{ border: '1px solid #cbd5e1', borderRadius: 6, padding: '7px 9px', fontSize: 12 }} + /> + <input + value={lat} + onChange={event => setLat(event.target.value)} + onKeyDown={event => { + if (event.key === 'Enter') queryPoint(); + }} + placeholder="纬度 lat，如 44.05" + style={{ border: '1px solid #cbd5e1', borderRadius: 6, padding: '7px 9px', fontSize: 12 }} + /> + <div style={{ display: 'flex', gap: 8, flexWrap: 'wrap' }}> + <button type="button" onClick={queryPoint} disabled={loading || !productId} style={{ ...buttonStyle, opacity: loading ? 0.65 : 1 }}> + {loading ? '查询中...' : '查询曲线'} + </button> + {result && ( + <button + type="button" + onClick={() => { + onClear?.(); + setMessage('已清除查询曲线。'); + }} + style={buttonStyle} + > + 清除查询 + </button> + )} + </div> + </div> + {message && <div style={{ color: message.includes('失败') || message.includes('超出') || message.includes('有效') ? '#dc2626' : '#166534', fontSize: 12 }}>{message}</div>} + </div> + + {result && ( + <div style={{ display: 'grid', gap: 10, padding: 12 }}> + <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(150px, 1fr))', gap: 8 }}> + <Metric label="匹配方式" value={matched.used_nearest ? '最近有效像元' : '输入点有效像元'} accent={matched.used_nearest ? '#b45309' : '#15803d'} /> + <Metric label="匹配经纬度" value={`${formatNumber(matched.lon, 6)}, ${formatNumber(matched.lat, 6)}`} /> + <Metric label="距离" value={`${formatNumber(matched.distance_m, 1)} m`} /> + <Metric label="雷达坐标 x,y" value={`${matched.img_x ?? '-'}, ${matched.img_y ?? '-'}`} /> + <Metric label="LOS 速率" value={`${formatNumber(matched.los_rate_mm_per_year, 2)} mm/yr`} /> + </div> + <div style={mutedStyle}>查询点曲线已叠加到下方主图；若输入位置不是有效像元，图例会标注最近邻距离。</div> + </div> + )} + </div> + ); +} + +function MonitorPointInspection({ cards, productId, onOpen }) { + const [queryResult, setQueryResult] = useState(null); + + useEffect(() => { + setQueryResult(null); + }, [productId]); + + const queryCard = useMemo(() => { + if (!queryResult) return null; + const matched = queryResult.matched || {}; + const pointId = matched.used_nearest ? 'nearest_wgs84_query' : 'wgs84_query'; + return { + point_id: pointId, + isQuery: true, + point: { + point_id: pointId, + is_query: true, + selection_label: matched.used_nearest ? '查询点最近邻时序' : '查询点时序', + selection_key: `${formatNumber(matched.lon, 6)}, ${formatNumber(matched.lat, 6)}`, + deformation_rate_mm_per_year: matched.los_rate_mm_per_year, + matched, + displacements: queryResult.displacements || [], + }, + assets: {}, + }; + }, [queryResult]); + + const chartCards = useMemo(() => ( + queryCard ? [...cards, queryCard] : cards + ), [cards, queryCard]); + + if (!cards.length) { + return ( + <div style={{ display: 'grid', gap: 12 }}> + <PointTimeseriesLookup productId={productId} result={queryResult} onResult={setQueryResult} onClear={() => setQueryResult(null)} /> + <CombinedMonitorPointChart + cards={chartCards} + title="查询点形变曲线" + subtitle="暂无自动监测点时，仍可按 WGS84 经纬度查询有效覆盖区内或最近有效像元的 SBAS 时序。" + emptyText="暂无监测点或查询点曲线。" + /> + </div> + ); + } + + return ( + <div style={{ display: 'grid', gap: 12 }}> + <div style={mutedStyle}> + 这些点来自同一次 Gamma `disp_prt_2d` 时序结果，只是自动选点策略不同；主视图合并展示，单点 PNG、CSV 和元数据保留为复核入口。 + </div> + <PointTimeseriesLookup productId={productId} result={queryResult} onResult={setQueryResult} onClear={() => setQueryResult(null)} /> + <CombinedMonitorPointChart + cards={chartCards} + title="监测点和查询点形变曲线" + subtitle="同一次 Gamma `disp_prt_2d` 时序输出；自动选点和 WGS84 查询点共用 ECharts 时间轴，横坐标按实际日期间隔缩放，纵坐标为累计形变 mm。" + /> + <div style={{ border: '1px solid #e2e8f0', borderRadius: 8, overflow: 'hidden', background: '#ffffff' }}> + <div style={{ padding: '9px 12px', background: '#f8fafc', borderBottom: '1px solid #e2e8f0', fontSize: 13, fontWeight: 850, color: '#0f172a' }}> + 监测点明细和复核文件 + </div> + <div style={{ overflowX: 'auto' }}> + <div style={{ minWidth: 820 }}> + <div style={{ display: 'grid', gridTemplateColumns: '1.7fr 1fr 1fr 0.85fr 0.8fr 1.6fr', gap: 8, padding: '8px 12px', background: '#f8fafc', borderBottom: '1px solid #e2e8f0', color: '#475569', fontSize: 12, fontWeight: 800 }}> + <div>选点策略</div> + <div>雷达坐标</div> + <div>速率</div> + <div>残差</div> + <div>样本</div> + <div>复核文件</div> + </div> + {cards.map(card => { + const point = card.point || {}; + const curve = card.assets.curve; + const csv = card.assets.csv; + const metadata = card.assets.metadata; + const rank = Number(point.selection_rank); + const strategyLabel = point.selection_label || '未记录选点策略'; + const strategyKey = point.selection_key || '-'; + const strategyDescription = point.selection_description || '该点缺少选点策略元数据，建议重新生成监测点派生文件。'; + return ( + <div key={card.point_id} style={{ display: 'grid', gridTemplateColumns: '1.7fr 1fr 1fr 0.85fr 0.8fr 1.6fr', gap: 8, alignItems: 'center', padding: '10px 12px', borderBottom: '1px solid #edf2f7' }}> + <div style={{ minWidth: 0 }}> + <div style={{ display: 'flex', gap: 7, alignItems: 'center', flexWrap: 'wrap' }}> + <span style={{ fontSize: 12, fontWeight: 850, color: '#0f172a' }}>{strategyLabel}</span> + <span style={{ color: Number.isFinite(rank) ? '#1d4ed8' : '#64748b', fontSize: 12, fontWeight: 800 }}> + {Number.isFinite(rank) ? `策略 ${rank}` : '未排序'} + </span> + </div> + <div style={{ ...mutedStyle, marginTop: 2 }}>{card.point_id} / {strategyKey}</div> + <div style={{ ...mutedStyle, marginTop: 3, overflowWrap: 'anywhere' }}>{strategyDescription}</div> + </div> + <div style={{ color: '#0f172a', fontSize: 12, fontWeight: 700 }}>{point.img_x ?? '-'}, {point.img_y ?? '-'}</div> + <div style={{ color: '#0f172a', fontSize: 12, fontWeight: 700 }}>{formatNumber(point.deformation_rate_mm_per_year, 2)} mm/yr</div> + <div style={{ color: '#0f172a', fontSize: 12, fontWeight: 700 }}>{formatNumber(point.stdev_residual_phase_rad, 3)}</div> + <div style={{ color: '#0f172a', fontSize: 12, fontWeight: 700 }}>{point.displacement_count ?? '-'}</div> + <div style={{ display: 'flex', flexWrap: 'wrap', gap: 8 }}> + {curve && ( + <button type="button" onClick={() => onOpen?.(`${card.point_id} 形变曲线`, curve)} style={{ ...buttonStyle, padding: '5px 8px' }}> + 单点图 + </button> + )} + <AssetActionLink label="CSV" asset={csv} productId={productId} /> + <AssetActionLink label="元数据" asset={metadata} productId={productId} /> + </div> + </div> + ); + })} + </div> + </div> + </div> + </div> + ); +} + +function UnwrappedPhasePanel({ + summary, + previews = [], + geotiffs = [], + radarPreviews = [], + radarBmps = [], + radarColorbar, + productId, + onOpen, +}) { + const [expanded, setExpanded] = useState(false); + const cards = buildUnwrappedPhaseCards(summary, previews, geotiffs, radarPreviews, radarBmps); + if (!cards.length && !summary?.source_count && !summary?.error) return null; + return ( + <div style={{ border: '1px solid #e2e8f0', borderRadius: 8, background: '#ffffff', overflow: 'hidden' }}> + <button + type="button" + onClick={() => setExpanded(value => !value)} + style={{ + width: '100%', + border: 0, + background: '#f8fafc', + cursor: 'pointer', + padding: '10px 12px', + display: 'flex', + justifyContent: 'space-between', + gap: 12, + alignItems: 'center', + textAlign: 'left', + }} + > + <span> + <span style={{ display: 'block', fontSize: 13, fontWeight: 850, color: '#0f172a' }}>最终解缠相位检查</span> + <span style={{ display: 'block', ...mutedStyle, marginTop: 3 }}> + 来自 Gamma `final_unw_tab` 中进入 SBAS 反演的最终解缠相位文件；雷达坐标预览用于检查解缠连续性，GeoTIFF 用于 GIS 复核。 + </span> + </span> + <span style={{ display: 'flex', alignItems: 'center', gap: 8, color: '#334155', fontSize: 12, fontWeight: 800, whiteSpace: 'nowrap' }}> + <StatusBadge value={summary?.ready ? 'READY' : (summary?.error ? 'ERROR' : 'INCOMPLETE')} /> + {cards.length || radarPreviews.length || geotiffs.length} 项 / {expanded ? '收起' : '展开'} + </span> + </button> + + {expanded && ( + <div style={{ display: 'grid', gap: 10, padding: 12, borderTop: '1px solid #e2e8f0' }}> + <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(130px, 1fr))', gap: 8 }}> + <Metric label="源文件数" value={summary?.source_count ?? geotiffs.length ?? 0} /> + <Metric label="GeoTIFF" value={geotiffs.length || cards.filter(card => card.geotiff).length || 0} /> + <Metric label="雷达坐标预览" value={radarPreviews.length || cards.filter(card => card.radarPreview).length || 0} /> + <Metric label="单位" value="rad" /> + </div> + + {radarColorbar && ( + <div> + <div style={{ fontSize: 12, fontWeight: 800, color: '#0f172a', marginBottom: 6 }}>解缠相位色卡</div> + <div style={{ ...mutedStyle, marginBottom: 6 }}>色表：Gamma rmg.cm；显示范围：-6.28 到 6.28 rad。</div> + <img + src={getSbasInsarProductAssetUrl(productId, radarColorbar.id, assetCacheKey(radarColorbar))} + alt="Gamma rmg.cm unwrapped phase colorbar" + style={{ display: 'block', width: 'min(620px, 100%)', height: 'auto', background: '#ffffff', border: '1px solid #e2e8f0', borderRadius: 6 }} + /> + </div> + )} + + {summary?.error && ( + <div style={{ color: '#dc2626', fontSize: 12, wordBreak: 'break-all' }}>{summary.error}</div> + )} + + {cards.length > 0 && ( + <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(260px, 1fr))', gap: 10 }}> + {cards.map(card => ( + <div key={card.pair_id} style={{ border: '1px solid #dbe3ef', borderRadius: 8, background: '#ffffff', overflow: 'hidden' }}> + <div style={{ display: 'flex', justifyContent: 'space-between', gap: 8, padding: '8px 10px', background: '#ffffff', borderBottom: '1px solid #e2e8f0' }}> + <div> + <div style={{ fontSize: 12, fontWeight: 850, color: '#0f172a' }}>{card.pair_id}</div> + <div style={mutedStyle}>有效像元：{card.summary?.valid_count ?? '-'}</div> + </div> + <div style={{ display: 'flex', flexWrap: 'wrap', gap: 8, justifyContent: 'flex-end' }}> + <AssetActionLink label="打开 GeoTIFF" asset={card.geotiff} productId={productId} /> + <AssetActionLink label="打开雷达坐标 BMP" asset={card.radarBmp} productId={productId} /> + </div> + </div> + <InlineImageAsset + title="解缠相位雷达坐标预览图" + asset={card.radarPreview || card.preview} + productId={productId} + imageMaxHeight={310} + onOpen={(card.radarPreview || card.preview) ? () => onOpen?.(`${card.pair_id} 解缠相位`, card.radarPreview || card.preview) : undefined} + /> + {card.preview && card.radarPreview && ( + <div style={{ padding: '0 10px 10px' }}> + <AssetActionLink label="查看地理编码灰度预览" asset={card.preview} productId={productId} /> + </div> + )} + </div> + ))} + </div> + )} + </div> + )} + </div> + ); +} + export default function SbasInsarProductsPanel({ readOnly = false, onJobQueued }) { const [catalogStatus, setCatalogStatus] = useState(null); const [products, setProducts] = useState([]); @@ -220,6 +1413,8 @@ export default function SbasInsarProductsPanel({ readOnly = false, onJobQueued } const [detailLoading, setDetailLoading] = useState(false); const [actionLoading, setActionLoading] = useState(false); const [message, setMessage] = useState(''); + const [lightboxImage, setLightboxImage] = useState(null); + const [assetsExpanded, setAssetsExpanded] = useState(false); const loadCatalog = useCallback(async () => { setLoading(true); @@ -268,6 +1463,10 @@ export default function SbasInsarProductsPanel({ readOnly = false, onJobQueued } loadDetail(selectedId); }, [loadDetail, selectedId]); + useEffect(() => { + setAssetsExpanded(false); + }, [selectedId]); + const handleRebuild = async () => { if (readOnly) return; setActionLoading(true); @@ -284,24 +1483,63 @@ export default function SbasInsarProductsPanel({ readOnly = false, onJobQueued } } }; - const selectedAssets = Array.isArray(detail?.assets) ? detail.assets : []; - const selectedIssues = Array.isArray(detail?.issues) ? detail.issues : []; - const velocityPreview = useMemo(() => findFirstAsset(selectedAssets, ['primary_geocoded_preview']), [selectedAssets]); + const selectedAssets = useMemo(() => (Array.isArray(detail?.assets) ? detail.assets : []), [detail?.assets]); + const assetSummary = useMemo(() => { + const total = selectedAssets.length; + const ready = selectedAssets.filter(asset => asset?.exists_flag).length; + return { + total, + ready, + missing: Math.max(0, total - ready), + complete: total > 0 && ready === total, + }; + }, [selectedAssets]); + const selectedIssues = useMemo(() => (Array.isArray(detail?.issues) ? detail.issues : []), [detail?.issues]); + const velocityPreview = useMemo(() => findFirstAsset(selectedAssets, ['primary_geocoded_preview', 'primary_preview']), [selectedAssets]); + const velocityPureColorPreview = useMemo(() => findFirstAsset(selectedAssets, ['primary_rate_color_preview']), [selectedAssets]); + const velocityColorbar = useMemo(() => findFirstAsset(selectedAssets, ['primary_colorbar']), [selectedAssets]); + const primaryGeotiff = useMemo(() => findFirstAsset(selectedAssets, ['primary_geotiff']), [selectedAssets]); + const primaryRgbGeotiff = useMemo(() => findFirstAsset(selectedAssets, ['primary_rgb_geotiff']), [selectedAssets]); const sigmaPreview = useMemo(() => findFirstAsset(selectedAssets, ['quality_geocoded_preview']), [selectedAssets]); - const monitorPreviews = useMemo(() => findAssets(selectedAssets, ['monitor_point_curve']), [selectedAssets]); + const unwrappedPhasePreviews = useMemo(() => findAssets(selectedAssets, ['unwrapped_phase_preview']), [selectedAssets]); + const unwrappedPhaseGeotiffs = useMemo(() => findAssets(selectedAssets, ['unwrapped_phase_geotiff']), [selectedAssets]); + const unwrappedPhaseRadarPreviews = useMemo(() => findAssets(selectedAssets, ['unwrapped_phase_radar_preview']), [selectedAssets]); + const unwrappedPhaseRadarBmps = useMemo(() => findAssets(selectedAssets, ['unwrapped_phase_radar_bmp']), [selectedAssets]); + const unwrappedPhaseRadarColorbar = useMemo(() => findFirstAsset(selectedAssets, ['unwrapped_phase_radar_colorbar']), [selectedAssets]); + const gammaIntermediateQcAssets = useMemo( + () => findAssets(selectedAssets, ['gamma_qc_baseline_plot', 'gamma_qc_mean_coherence', 'gamma_qc_unwrapped_phase']), + [selectedAssets], + ); const pointVectorAsset = useMemo(() => findFirstAsset(selectedAssets, ['point_vector_geojson_gz']), [selectedAssets]); const pointVectorSummary = detail?.point_vector || {}; const monitorPoints = detail?.monitor_points?.monitor_points || detail?.geographic_coverage?.monitor_points || []; + const monitorOutputs = detail?.monitor_points?.monitor_outputs || []; + const monitorPointCards = useMemo( + () => buildMonitorPointCards(monitorPoints, monitorOutputs, selectedAssets), + [monitorPoints, monitorOutputs, selectedAssets], + ); + const unwrappedPhaseSummary = detail?.unwrapped_phase || {}; const coverage = detail?.geographic_coverage || {}; const center = detail?.center || coverage.center || bboxCenter(coverage.bbox); const adminRegion = detail?.admin_region || coverage.admin_region; const quality = detail?.quality || {}; - const rateStats = quality.los_rate_toward_mm_per_year_rdc || quality.los_rate_toward_m_per_year_rdc || {}; + const colorPolicy = detail?.color_policy || {}; + const rateStats = quality.los_rate_toward_mm_per_year_rdc || quality.los_rate_toward_m_per_year_rdc || quality.primary_geotiff || {}; const sigmaStats = quality.los_sigma_mm_per_year_rdc || quality.los_sigma_m_per_year_rdc || {}; + const hasSigmaStats = Object.keys(sigmaStats || {}).length > 0; const catalogColor = statusColors[catalogStatus?.status] || '#64748b'; + const openAssetLightbox = useCallback((title, asset) => { + if (!detail?.id || !asset) return; + setLightboxImage({ + title, + path: asset.relative_path, + src: getSbasInsarProductAssetUrl(detail.id, asset.id, assetCacheKey(asset)), + }); + }, [detail?.id]); return ( <div style={panelStyle}> + <ImageLightbox image={lightboxImage} onClose={() => setLightboxImage(null)} /> <section style={sectionStyle}> <div style={{ display: 'flex', justifyContent: 'space-between', gap: 12, alignItems: 'flex-start' }}> <div> @@ -423,7 +1661,7 @@ export default function SbasInsarProductsPanel({ readOnly = false, onJobQueued } <div style={{ display: 'grid', gridTemplateColumns: 'minmax(180px, 260px) minmax(0, 1fr)', gap: 14, alignItems: 'start' }}> <div style={{ border: '1px solid #e2e8f0', borderRadius: 8, overflow: 'hidden', background: '#0f172a' }}> <img - src={getSbasInsarProductPreviewUrl(detail.id)} + src={getSbasInsarProductPreviewUrl(detail.id, productCacheKey(detail))} alt={detail.display_name} style={{ display: 'block', width: '100%', minHeight: 150, objectFit: 'contain' }} /> @@ -465,26 +1703,52 @@ export default function SbasInsarProductsPanel({ readOnly = false, onJobQueued } <section style={sectionStyle}> <h4 style={{ margin: '0 0 10px', fontSize: 15, color: '#0f172a' }}>重要产物预览</h4> - <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(240px, 1fr))', gap: 12 }}> - <ProductPreview title="LOS 速率图" asset={velocityPreview} productId={detail.id} /> - <ProductPreview title="LOS Sigma 图" asset={sigmaPreview} productId={detail.id} /> + <div style={{ display: 'grid', gap: 12 }}> + <VelocityInspectionPanel + asset={velocityPreview} + colorbarAsset={velocityColorbar} + colorPolicy={colorPolicy} + primaryGeotiff={primaryGeotiff} + rgbGeotiff={primaryRgbGeotiff} + productId={detail.id} + onOpen={velocityPreview ? () => openAssetLightbox('LOS 速率图', velocityPreview) : undefined} + /> + {velocityPureColorPreview && ( + <ProductPreview + title="LOS 速率纯色图（无底图）" + asset={velocityPureColorPreview} + productId={detail.id} + imageMaxHeight={620} + onOpen={() => openAssetLightbox('LOS 速率纯色图（无底图）', velocityPureColorPreview)} + /> + )} + {sigmaPreview && <ProductPreview title="LOS Sigma 图" asset={sigmaPreview} productId={detail.id} />} </div> <div style={{ marginTop: 12 }}> <PointVectorDownload asset={pointVectorAsset} summary={pointVectorSummary} productId={detail.id} /> </div> <div style={{ marginTop: 12 }}> - <div style={{ fontSize: 13, fontWeight: 800, color: '#0f172a', marginBottom: 8 }}>监测点形变曲线</div> - {monitorPreviews.length === 0 ? ( - <div style={{ ...mutedStyle, border: '1px solid #e2e8f0', borderRadius: 8, padding: 10, background: '#f8fafc' }}> - 暂无监测点曲线。 - </div> - ) : ( - <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(240px, 1fr))', gap: 12 }}> - {monitorPreviews.map(asset => ( - <ProductPreview key={asset.id} title={asset.asset_name || '监测点曲线'} asset={asset} productId={detail.id} /> - ))} - </div> - )} + <UnwrappedPhasePanel + summary={unwrappedPhaseSummary} + previews={unwrappedPhasePreviews} + geotiffs={unwrappedPhaseGeotiffs} + radarPreviews={unwrappedPhaseRadarPreviews} + radarBmps={unwrappedPhaseRadarBmps} + radarColorbar={unwrappedPhaseRadarColorbar} + productId={detail.id} + onOpen={openAssetLightbox} + /> + </div> + <div style={{ marginTop: 12 }}> + <GammaIntermediateQcPanel + assets={gammaIntermediateQcAssets} + productId={detail.id} + onOpen={openAssetLightbox} + /> + </div> + <div style={{ marginTop: 12 }}> + <div style={{ fontSize: 13, fontWeight: 800, color: '#0f172a', marginBottom: 8 }}>监测点检查</div> + <MonitorPointInspection cards={monitorPointCards} productId={detail.id} onOpen={openAssetLightbox} /> </div> </section> @@ -493,43 +1757,83 @@ export default function SbasInsarProductsPanel({ readOnly = false, onJobQueued } <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(150px, 1fr))', gap: 8 }}> <Metric label="速率中位数" value={`${formatNumber(rateStats.median, 2)} mm/yr`} /> <Metric label="速率 P05 / P95" value={`${formatNumber(rateStats.p05, 2)} / ${formatNumber(rateStats.p95, 2)}`} /> - <Metric label="Sigma 中位数" value={`${formatNumber(sigmaStats.median, 2)} mm/yr`} /> + {hasSigmaStats && <Metric label="Sigma 中位数" value={`${formatNumber(sigmaStats.median, 2)} mm/yr`} />} <Metric label="有效像元" value={rateStats.valid_count ?? '-'} /> + <Metric label="0 速率" value={rateStats.zero_is_valid ? '按稳定值参与统计' : '按无效值处理'} /> + <Metric label="有效规则" value={rateStats.validity_rule === 'expert_rgb_coverage_finite_nonzero_values' ? '专家覆盖区内有限非零值' : (rateStats.validity_rule || '-')} /> </div> </section> <section style={sectionStyle}> - <h4 style={{ margin: '0 0 10px', fontSize: 15, color: '#0f172a' }}>资产下载</h4> - <div style={{ display: 'grid', gap: 7 }}> - {selectedAssets.map(asset => ( - <div - key={asset.id} + <div style={{ display: 'flex', justifyContent: 'space-between', gap: 12, alignItems: 'center', flexWrap: 'wrap' }}> + <div style={{ display: 'flex', alignItems: 'center', gap: 10, flexWrap: 'wrap' }}> + <h4 style={{ margin: 0, fontSize: 15, color: '#0f172a' }}>资产下载</h4> + <span style={{ - display: 'grid', - gridTemplateColumns: 'minmax(160px, 220px) minmax(0, 1fr) auto', - gap: 10, + display: 'inline-flex', alignItems: 'center', - border: '1px solid #e2e8f0', - borderRadius: 8, - padding: '8px 10px', - background: asset.exists_flag ? '#ffffff' : '#fef2f2', + gap: 6, + padding: '3px 9px', + borderRadius: 999, + background: assetSummary.complete ? '#dcfce7' : '#fef3c7', + color: assetSummary.complete ? '#166534' : '#92400e', + fontSize: 12, + fontWeight: 850, }} > - <div> - <div style={{ fontSize: 12, fontWeight: 750, color: '#0f172a' }}>{asset.asset_role}</div> - <div style={mutedStyle}>{formatBytes(asset.file_size)} / {asset.format || '-'}</div> - </div> - <div style={{ ...mutedStyle, wordBreak: 'break-all' }}>{asset.relative_path}</div> - {asset.exists_flag ? ( - <a href={getSbasInsarProductAssetUrl(detail.id, asset.id)} target="_blank" rel="noreferrer" style={{ color: '#1d4ed8', fontSize: 12, fontWeight: 750 }}> - 打开 - </a> - ) : ( - <span style={{ color: '#dc2626', fontSize: 12 }}>缺失</span> - )} - </div> - ))} + <span style={{ width: 7, height: 7, borderRadius: 999, background: assetSummary.complete ? '#16a34a' : '#f59e0b' }} /> + {assetSummary.complete ? '资产完备' : `缺失 ${assetSummary.missing} 项`} + </span> + <span style={{ color: '#334155', fontSize: 12, fontWeight: 800 }}> + {assetSummary.ready}/{assetSummary.total} + </span> + <span style={mutedStyle}>默认折叠，展开后查看全部下载文件。</span> + </div> + <button type="button" onClick={() => setAssetsExpanded(value => !value)} style={buttonStyle}> + {assetsExpanded ? '收起' : '展开'} + </button> </div> + {assetsExpanded && ( + <div style={{ display: 'grid', gap: 7, marginTop: 10 }}> + {selectedAssets.map(asset => { + const assetInfo = getAssetRoleInfo(asset); + return ( + <div + key={asset.id} + style={{ + display: 'grid', + gridTemplateColumns: 'minmax(220px, 300px) minmax(0, 1fr) auto', + gap: 12, + alignItems: 'center', + border: '1px solid #e2e8f0', + borderRadius: 8, + padding: '9px 10px', + background: asset.exists_flag ? '#ffffff' : '#fef2f2', + }} + > + <div> + <div style={{ fontSize: 13, fontWeight: 800, color: '#0f172a' }}>{assetInfo.label}</div> + <div style={{ ...mutedStyle, marginTop: 2 }}> + {formatBytes(asset.file_size)} / {asset.format || '-'} / {asset.exists_flag ? '已生成' : '缺失'} + </div> + <div style={{ ...mutedStyle, marginTop: 2 }}>内部角色：{asset.asset_role || '-'}</div> + </div> + <div> + <div style={{ color: '#334155', fontSize: 12, lineHeight: 1.55 }}>{assetInfo.description}</div> + <div style={{ ...mutedStyle, wordBreak: 'break-all', marginTop: 3 }}>{asset.relative_path}</div> + </div> + {asset.exists_flag ? ( + <a href={getSbasInsarProductAssetUrl(detail.id, asset.id, assetCacheKey(asset))} target="_blank" rel="noreferrer" style={{ color: '#1d4ed8', fontSize: 12, fontWeight: 750 }}> + 打开 + </a> + ) : ( + <span style={{ color: '#dc2626', fontSize: 12, fontWeight: 750 }}>缺失</span> + )} + </div> + ); + })} + </div> + )} </section> <section style={sectionStyle}> diff --git a/frontend/src/api/idl.js b/frontend/src/api/idl.js index 1e2dbf4..3a0f5d8 100644 --- a/frontend/src/api/idl.js +++ b/frontend/src/api/idl.js @@ -12,8 +12,6 @@ export const queueDinsarJob = (payload) => apiClient.post('/idl/jobs/dinsar', payload).then(r => r.data); export const getRecentRuns = (limit = 20) => apiClient.get(`/idl/jobs/recent?limit=${encodeURIComponent(limit)}`).then(r => r.data); -export const getActiveTasks = () => - apiClient.get('/tasks/active').then(r => r.data); export const getTaskLogs = (taskId, limit = 50, offset = 0) => apiClient.get(`/tasks/${taskId}/logs?limit=${encodeURIComponent(limit)}&offset=${encodeURIComponent(offset)}`).then(r => r.data); export const forceCancelTask = (taskId, password) => diff --git a/frontend/src/api/sbasInsarProduction.js b/frontend/src/api/sbasInsarProduction.js index 8d17cca..da9d7e5 100644 --- a/frontend/src/api/sbasInsarProduction.js +++ b/frontend/src/api/sbasInsarProduction.js @@ -24,6 +24,9 @@ export const listSbasInsarRuns = () => export const getSbasInsarRun = runId => apiClient.get(`/sbas-insar-production/runs/${encodeURIComponent(runId)}`).then(r => r.data); +export const deleteSbasInsarRun = runId => + apiClient.delete(`/sbas-insar-production/runs/${encodeURIComponent(runId)}`).then(r => r.data); + export const prepareSbasInsarWorkflow = (runId, payload = {}) => apiClient.post(`/sbas-insar-production/runs/${encodeURIComponent(runId)}/workflow`, payload).then(r => r.data); @@ -62,3 +65,18 @@ export const submitSbasInsarIptaTimeseriesJob = (runId, payload = {}) => export const getSbasInsarRunArtifactUrl = (runId, relativePath) => `/api/sbas-insar-production/runs/${encodeURIComponent(runId)}/artifacts/${encodeArtifactPath(relativePath)}`; + +export const getLandsarSbasCapabilities = () => + apiClient.get('/sbas-insar-production/landsar/capabilities').then(r => r.data); + +export const submitLandsarSbasAutoWorkflow = (payload = {}) => + apiClient.post('/sbas-insar-production/landsar/workflows/auto', payload).then(r => r.data); + +export const listLandsarSbasRuns = () => + apiClient.get('/sbas-insar-production/landsar/runs').then(r => r.data); + +export const getLandsarSbasRun = runId => + apiClient.get(`/sbas-insar-production/landsar/runs/${encodeURIComponent(runId)}`).then(r => r.data); + +export const getLandsarSbasRunArtifactUrl = (runId, relativePath) => + `/api/sbas-insar-production/landsar/runs/${encodeURIComponent(runId)}/artifacts/${encodeArtifactPath(relativePath)}`; diff --git a/frontend/src/api/sbasInsarProducts.js b/frontend/src/api/sbasInsarProducts.js index 0b2c441..ddd6d13 100644 --- a/frontend/src/api/sbasInsarProducts.js +++ b/frontend/src/api/sbasInsarProducts.js @@ -12,8 +12,20 @@ export const listSbasInsarProducts = (params = {}) => export const getSbasInsarProductDetail = productId => apiClient.get(`/sbas-insar-products/${encodeURIComponent(productId)}`).then(r => r.data); -export const getSbasInsarProductPreviewUrl = productId => - `${apiClient.defaults.baseURL || '/api'}/sbas-insar-products/${encodeURIComponent(productId)}/preview`; +export const querySbasInsarPointTimeseries = (productId, payload) => + apiClient.post(`/sbas-insar-products/${encodeURIComponent(productId)}/point-timeseries`, payload).then(r => r.data); -export const getSbasInsarProductAssetUrl = (productId, assetId) => - `${apiClient.defaults.baseURL || '/api'}/sbas-insar-products/${encodeURIComponent(productId)}/assets/${encodeURIComponent(assetId)}`; +const appendCacheKey = (url, cacheKey) => + cacheKey ? `${url}?v=${encodeURIComponent(cacheKey)}` : url; + +export const getSbasInsarProductPreviewUrl = (productId, cacheKey = '') => + appendCacheKey( + `${apiClient.defaults.baseURL || '/api'}/sbas-insar-products/${encodeURIComponent(productId)}/preview`, + cacheKey, + ); + +export const getSbasInsarProductAssetUrl = (productId, assetId, cacheKey = '') => + appendCacheKey( + `${apiClient.defaults.baseURL || '/api'}/sbas-insar-products/${encodeURIComponent(productId)}/assets/${encodeURIComponent(assetId)}`, + cacheKey, + ); diff --git a/frontend/src/components/ActiveTasksOverlay.jsx b/frontend/src/components/ActiveTasksOverlay.jsx deleted file mode 100644 index cbcea7e..0000000 --- a/frontend/src/components/ActiveTasksOverlay.jsx +++ /dev/null @@ -1,179 +0,0 @@ -const getTaskTypeLabel = (taskType) => { - if (taskType?.startsWith('WATER_GEOCODE_')) return '水体地理编码'; - if (taskType?.startsWith('WATER_FLOOD_')) return '洪涝检测'; - switch (taskType) { - case 'SCAN_DATA': - return '同步源数据'; - case 'SCAN_DINSAR': - return '扫描结果与自愈'; - case 'AI_TRAIN': - return '训练AI模型'; - case 'AI_PREDICT': - return '全量质量评估'; - case 'AI_ANALYZE': - return 'AI 智能诊断'; - case 'AI_WARMUP': - return 'AI 模型预热'; - case 'COPY_DATA': - return '数据分发拷贝'; - case 'SCAN_HAZARD': - return '灾害点同步'; - case 'UNPACK_ARCHIVES': - return 'LT-1 解包'; - case 'UNPACK_SENTINEL1': - return 'Sentinel-1 解包'; - case 'GF3_UNPACK': - return 'GF3 解包'; - case 'GF3_BATCH_PROCESS': - return 'GF3 预处理'; - case 'GF3_SARSCAPE_PRODUCE': - return 'GF3 SARscape 生产'; - case 'GF3_SARSCAPE_SYNC': - return 'GF3 SARscape 入库'; - case 'GF3_SARSCAPE_CLEAN': - return 'GF3 中间清理'; - case 'SCAN_ASSET_INVENTORY': - return '资产库存扫描'; - case 'IDL_IMPORT': - return 'ENVI 数据导入'; - case 'IDL_DINSAR': - return 'ENVI D-InSAR 生产'; - default: - return taskType; - } -}; - -export default function ActiveTasksOverlay({ - isVisible, - activeTasks, - t, - isAdmin, - showForceUnlock, - forceUnlockPwd, - onShowForceUnlock, - onForceUnlockPwdChange, - onForceUnlockConfirm, - onCancelForceUnlock, -}) { - if (!isVisible) { - return null; - } - - return ( - <div className="global-task-overlay"> - <div className="overlay-content"> - <div className="loading-spinner-large"></div> - <h3>系统任务执行中</h3> - <div className="active-tasks-container"> - {(() => { - const waterTasks = activeTasks.filter(t => - t.task_type?.startsWith('WATER_GEOCODE_') || t.task_type?.startsWith('WATER_FLOOD_') - ); - const otherTasks = activeTasks.filter(t => - !t.task_type?.startsWith('WATER_GEOCODE_') && !t.task_type?.startsWith('WATER_FLOOD_') - ); - const waterDone = waterTasks.filter(t => t.progress >= 100).length; - return ( - <> - {otherTasks.map((task) => ( - <div key={task.task_id} className="task-progress-item"> - <div className="task-info-row"> - <span className="task-label">{getTaskTypeLabel(task.task_type)}</span> - <span className="task-percent">{task.progress}%</span> - </div> - <div className="task-progress-bar"> - <div className="task-progress-fill" style={{ width: `${task.progress}%` }}></div> - </div> - <p className="task-status-msg">{t(task.message || '')}</p> - </div> - ))} - {waterTasks.length > 0 && ( - <div className="task-progress-item"> - <div className="task-info-row"> - <span className="task-label">水体处理（剩余 {waterTasks.length - waterDone} 景）</span> - </div> - </div> - )} - </> - ); - })()} - </div> - <p className="overlay-footer-hint">为了保证数据一致性，耗时任务执行期间 UI 已锁定。任务完成后将自动刷新页面数据。</p> - {isAdmin && ( - <div style={{ marginTop: '16px', textAlign: 'center' }}> - {!showForceUnlock ? ( - <button - onClick={onShowForceUnlock} - style={{ - padding: '6px 16px', - borderRadius: '6px', - border: '1px solid rgba(255,255,255,0.4)', - background: 'rgba(255,255,255,0.1)', - color: '#fff', - fontSize: '12px', - cursor: 'pointer', - }} - > - 管理员强制解锁 - </button> - ) : ( - <div style={{ display: 'inline-flex', alignItems: 'center', gap: '8px' }}> - <input - type="password" - placeholder="输入管理员密码" - value={forceUnlockPwd} - onChange={(e) => onForceUnlockPwdChange(e.target.value)} - onKeyDown={(e) => { - if (e.key === 'Enter') { - onForceUnlockConfirm(); - } - }} - style={{ - padding: '5px 10px', - fontSize: '12px', - borderRadius: '4px', - border: '1px solid rgba(255,255,255,0.4)', - background: 'rgba(255,255,255,0.15)', - color: '#fff', - width: '160px', - outline: 'none', - }} - /> - <button - disabled={!forceUnlockPwd} - onClick={onForceUnlockConfirm} - style={{ - padding: '5px 14px', - borderRadius: '4px', - border: '1px solid #dc2626', - background: '#dc2626', - color: '#fff', - fontSize: '12px', - cursor: forceUnlockPwd ? 'pointer' : 'not-allowed', - opacity: forceUnlockPwd ? 1 : 0.5, - }} - > - 确认解锁 - </button> - <button - onClick={onCancelForceUnlock} - style={{ - padding: '5px 10px', - borderRadius: '4px', - border: '1px solid rgba(255,255,255,0.4)', - background: 'transparent', - color: '#fff', - fontSize: '12px', - cursor: 'pointer', - }} - > - 取消 - </button> - </div> - )} - </div> - )} - </div> - </div> - ); -} diff --git a/frontend/src/components/GlobalTaskCenter.jsx b/frontend/src/components/GlobalTaskCenter.jsx new file mode 100644 index 0000000..77a2d94 --- /dev/null +++ b/frontend/src/components/GlobalTaskCenter.jsx @@ -0,0 +1,159 @@ +import { useState } from 'react'; +import { getTaskTypeLabel } from '../config/taskUiPolicies'; + +export default function GlobalTaskCenter({ + isVisible, + activeTasks, + t, + isAdmin, + showCancelTask, + cancelTaskPwd, + onShowCancelTask, + onCancelTaskPwdChange, + onCancelTaskConfirm, + onCloseCancelTask, +}) { + const [expanded, setExpanded] = useState(false); + if (!isVisible || activeTasks.length === 0) { + return null; + } + + const activeCount = activeTasks.length; + const avgProgress = Math.round( + activeTasks.reduce((sum, task) => sum + (Number(task.progress) || 0), 0) / Math.max(1, activeCount) + ); + + return ( + <div className="global-task-overlay"> + {!expanded && ( + <button className="task-center-button" onClick={() => setExpanded(true)}> + <span className="task-center-dot" /> + <span>后台任务 {activeCount}</span> + <strong>{avgProgress}%</strong> + </button> + )} + {expanded && ( + <div className="overlay-content"> + <div className="task-center-header"> + <div> + <h3>后台任务</h3> + <p>任务正在执行，你可以继续使用其他功能；同类重复提交由系统限制。</p> + </div> + <button className="task-center-close" onClick={() => setExpanded(false)} aria-label="关闭任务中心"> + × + </button> + </div> + <div className="active-tasks-container"> + {(() => { + const waterTasks = activeTasks.filter(task => + task.task_type?.startsWith('WATER_GEOCODE_') || task.task_type?.startsWith('WATER_FLOOD_') + ); + const otherTasks = activeTasks.filter(task => + !task.task_type?.startsWith('WATER_GEOCODE_') && !task.task_type?.startsWith('WATER_FLOOD_') + ); + const waterDone = waterTasks.filter(task => task.progress >= 100).length; + return ( + <> + {otherTasks.map((task) => ( + <div key={task.task_id} className="task-progress-item"> + <div className="task-info-row"> + <span className="task-label">{getTaskTypeLabel(task.task_type)}</span> + <span className="task-percent">{Number(task.progress) || 0}%</span> + </div> + <div className="task-progress-bar"> + <div className="task-progress-fill" style={{ width: `${Number(task.progress) || 0}%` }}></div> + </div> + <p className="task-status-msg">{t(task.message || '')}</p> + </div> + ))} + {waterTasks.length > 0 && ( + <div className="task-progress-item"> + <div className="task-info-row"> + <span className="task-label">水体处理（剩余 {waterTasks.length - waterDone} 景）</span> + </div> + </div> + )} + </> + ); + })()} + </div> + <p className="overlay-footer-hint">任务中心只展示状态，不再锁定整个界面。需要互斥的操作由功能页按钮和后端任务冲突检查处理。</p> + {isAdmin && ( + <div style={{ marginTop: '16px', textAlign: 'center' }}> + {!showCancelTask ? ( + <button + onClick={onShowCancelTask} + style={{ + padding: '6px 16px', + borderRadius: '6px', + border: '1px solid #fecaca', + background: '#fff1f2', + color: '#b91c1c', + fontSize: '12px', + cursor: 'pointer', + }} + > + 管理员取消任务 + </button> + ) : ( + <div style={{ display: 'inline-flex', alignItems: 'center', gap: '8px' }}> + <input + type="password" + placeholder="输入管理员密码" + value={cancelTaskPwd} + onChange={(e) => onCancelTaskPwdChange(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') { + onCancelTaskConfirm(); + } + }} + style={{ + padding: '5px 10px', + fontSize: '12px', + borderRadius: '4px', + border: '1px solid #cbd5e1', + background: '#ffffff', + color: '#0f172a', + width: '160px', + outline: 'none', + }} + /> + <button + disabled={!cancelTaskPwd} + onClick={onCancelTaskConfirm} + style={{ + padding: '5px 14px', + borderRadius: '4px', + border: '1px solid #dc2626', + background: '#dc2626', + color: '#fff', + fontSize: '12px', + cursor: cancelTaskPwd ? 'pointer' : 'not-allowed', + opacity: cancelTaskPwd ? 1 : 0.5, + }} + > + 确认取消 + </button> + <button + onClick={onCloseCancelTask} + style={{ + padding: '5px 10px', + borderRadius: '4px', + border: '1px solid #cbd5e1', + background: '#ffffff', + color: '#334155', + fontSize: '12px', + cursor: 'pointer', + }} + > + 关闭 + </button> + </div> + )} + </div> + )} + </div> + )} + </div> + ); +} diff --git a/frontend/src/components/PairingModal.jsx b/frontend/src/components/PairingModal.jsx index ed8a3c0..ab6ed2c 100644 --- a/frontend/src/components/PairingModal.jsx +++ b/frontend/src/components/PairingModal.jsx @@ -8,7 +8,7 @@ import { getAvailableSatellites } from '../api/radar'; // 配对策略说明 const STRATEGY_DESCRIPTIONS = { all: { - title: '全部配对（默认）', + title: '全部配对', description: '列出所有满足约束条件的候选干涉对，由用户自行筛选。', details: [ '• 系统遍历所有影像组合，保留满足时间基线和两景 footprint 最小重叠率的配对', @@ -19,7 +19,7 @@ const STRATEGY_DESCRIPTIONS = { params: '参数：时间基线范围、两景 footprint 最小重叠率、可选 footprint 中心距上限' }, sbas: { - title: 'SBAS (短基线子集)', + title: 'SBAS (短基线子集，推荐)', description: '基于短基线原则的配对策略，通过覆盖优化算法自动筛选配对。', details: [ '• 优先选择时间间隔较短、覆盖质量较好的配对；可按需启用 footprint 中心距限制', @@ -204,6 +204,11 @@ function PairingModal({ Star (星型) </label> </div> + {pairingParams.strategy === 'all' && ( + <div style={{ marginTop: 8, padding: '8px 10px', borderRadius: 8, background: '#fff7ed', border: '1px solid #fdba74', color: '#9a3412', fontSize: 12, lineHeight: 1.5 }}> + 全部配对会返回所有候选边；当前数据量较大时请先限定 AOI 或主/从影像时间范围。做 SBAS 生产建议使用“SBAS (短基线)”策略。 + </div> + )} </div> {/* 主影像时间范围 */} diff --git a/frontend/src/components/app/AppOverlays.jsx b/frontend/src/components/app/AppOverlays.jsx index 3a3e8c0..c9bb6a2 100644 --- a/frontend/src/components/app/AppOverlays.jsx +++ b/frontend/src/components/app/AppOverlays.jsx @@ -9,7 +9,7 @@ import { ModalLoadingFallback } from './AppLoadingFallbacks'; const LazyPairingModal = lazy(() => import('../PairingModal')); const LazyPsStackModal = lazy(() => import('../PsStackModal')); const LazyDataInfoModal = lazy(() => import('../DataInfoModal')); -const LazyActiveTasksOverlay = lazy(() => import('../ActiveTasksOverlay')); +const LazyGlobalTaskCenter = lazy(() => import('../GlobalTaskCenter')); const LazyStatisticsDashboard = lazy(() => import('../../StatisticsDashboard')); const LazyAiReportModal = lazy(() => import('../AiReportModal')); const LazyMapExportModal = lazy(() => import('../MapExportModal')); @@ -31,14 +31,13 @@ export default function AppOverlays({ onRefreshLicenseStatus, licenseFileName, licenseUploadStatus, - isGlobalLocked, activeTasks, - showForceUnlock, - forceUnlockPwd, - onShowForceUnlock, - onForceUnlockPwdChange, - onForceUnlockConfirm, - onCancelForceUnlock, + showCancelTask, + cancelTaskPwd, + onShowCancelTask, + onCancelTaskPwdChange, + onCancelTaskConfirm, + onCloseCancelTask, mapExport, }) { const { language, t } = useI18n(); @@ -126,19 +125,19 @@ export default function AppOverlays({ </Suspense> )} - {isGlobalLocked && ( - <Suspense fallback={<ModalLoadingFallback message="正在加载任务控制面板..." />}> - <LazyActiveTasksOverlay - isVisible={isGlobalLocked} + {activeTasks.length > 0 && ( + <Suspense fallback={<ModalLoadingFallback message="正在加载任务中心..." />}> + <LazyGlobalTaskCenter + isVisible={activeTasks.length > 0} activeTasks={activeTasks} t={t} isAdmin={isAdmin} - showForceUnlock={showForceUnlock} - forceUnlockPwd={forceUnlockPwd} - onShowForceUnlock={onShowForceUnlock} - onForceUnlockPwdChange={onForceUnlockPwdChange} - onForceUnlockConfirm={onForceUnlockConfirm} - onCancelForceUnlock={onCancelForceUnlock} + showCancelTask={showCancelTask} + cancelTaskPwd={cancelTaskPwd} + onShowCancelTask={onShowCancelTask} + onCancelTaskPwdChange={onCancelTaskPwdChange} + onCancelTaskConfirm={onCancelTaskConfirm} + onCloseCancelTask={onCloseCancelTask} /> </Suspense> )} diff --git a/frontend/src/components/app/AppSidePanel.jsx b/frontend/src/components/app/AppSidePanel.jsx index 1efaa5e..b785a22 100644 --- a/frontend/src/components/app/AppSidePanel.jsx +++ b/frontend/src/components/app/AppSidePanel.jsx @@ -30,6 +30,7 @@ const LazyBatchPanel = lazy(() => import('../../panels/BatchPanel')); const LazyPairsListPanel = lazy(() => import('../../panels/PairsListPanel')); const LazyPsResultsPanel = lazy(() => import('../../panels/PsResultsPanel')); const LazyPsinsarCatalogPanel = lazy(() => import('../PsinsarCatalogPanel')); +const LazySbasInsarMapAnalysisPanel = lazy(() => import('../../panels/SbasInsarMapAnalysisPanel')); const LazyProductionWorkspace = lazy(() => import('../../ProductionWorkspace')); export default function AppSidePanel({ @@ -62,6 +63,7 @@ export default function AppSidePanel({ aiPanel, pairsPanel, psPanel, + sbasAnalysisPanel, }) { const isProductionWorkspace = PRODUCTION_WORKSPACE_ROUTE_TABS.has(leftPanelTab); const activeLeftGroup = LEFT_TAB_GROUP[leftPanelTab] || 'data'; @@ -425,13 +427,12 @@ export default function AppSidePanel({ {leftPanelTab === 'psinsar_analysis' && ( <div className="panel-content" style={{ flex: '1 1 auto', padding: 0, overflow: 'auto' }}> - <div style={{ padding: '16px' }}> - <div className="empty-state"> - 时序InSAR 分析页已预留。 - <br /> - 后续可以在这里放置时序分析、速率分级、热点识别和专题统计能力。 - </div> - </div> + <Suspense fallback={<PanelLoadingBody message="正在加载时序InSAR地图分析..." />}> + <LazySbasInsarMapAnalysisPanel + readOnly={isReadOnlyUser} + {...sbasAnalysisPanel} + /> + </Suspense> </div> )} diff --git a/frontend/src/components/tasks/TaskStatusPanel.jsx b/frontend/src/components/tasks/TaskStatusPanel.jsx new file mode 100644 index 0000000..ed62084 --- /dev/null +++ b/frontend/src/components/tasks/TaskStatusPanel.jsx @@ -0,0 +1,118 @@ +import { getTaskTypeLabel } from '../../config/taskUiPolicies'; + +const toneColor = { + active: { + border: '#f59e0b', + bg: '#fffbeb', + text: '#92400e', + fill: '#f59e0b', + }, + recent: { + border: '#93c5fd', + bg: '#eff6ff', + text: '#1d4ed8', + fill: '#3b82f6', + }, + idle: { + border: '#e2e8f0', + bg: '#f8fafc', + text: '#64748b', + fill: '#94a3b8', + }, + error: { + border: '#fecaca', + bg: '#fef2f2', + text: '#b91c1c', + fill: '#dc2626', + }, +}; + +const normalizeProgress = (value) => { + const numeric = Number(value); + if (!Number.isFinite(numeric)) return null; + return Math.max(0, Math.min(100, numeric)); +}; + +const isFailed = (task) => String(task?.status || '').toUpperCase() === 'FAILED'; + +export default function TaskStatusPanel({ + title = '任务状态', + activeTasks = [], + recentTasks = [], + latestTask = null, + isBusy = false, + idleText = '当前没有正在执行的相关任务。', + compact = false, + action = null, + footer = null, +}) { + const task = latestTask || activeTasks[0] || recentTasks[0] || null; + const showingRecent = !isBusy && !!task; + const tone = task ? (isFailed(task) ? 'error' : (showingRecent ? 'recent' : 'active')) : 'idle'; + const colors = toneColor[tone]; + const progress = normalizeProgress(task?.progress); + + return ( + <div + className="task-status-panel" + style={{ + padding: compact ? 10 : 12, + borderRadius: 8, + border: `1px solid ${colors.border}`, + background: colors.bg, + marginBottom: 12, + }} + > + <div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', gap: 10 }}> + <div> + <strong style={{ color: colors.text, fontSize: compact ? 13 : 14 }}>{title}</strong> + {task && ( + <span style={{ marginLeft: 8, color: colors.text, fontSize: 12 }}> + {showingRecent ? '最近一次' : '运行中'} + </span> + )} + </div> + {action} + </div> + + {!task ? ( + <div style={{ marginTop: 7, color: colors.text, fontSize: 12 }}>{idleText}</div> + ) : ( + <> + <div style={{ marginTop: 8, color: colors.text, fontSize: 12, wordBreak: 'break-all' }}> + <span style={{ fontWeight: 700 }}>{getTaskTypeLabel(task.task_type)}</span> + <span style={{ margin: '0 6px' }}>·</span> + <span>{String(task.status || '-').toUpperCase()}</span> + {task.task_id && ( + <> + <span style={{ margin: '0 6px' }}>·</span> + <code style={{ fontSize: 11 }}>{task.task_id}</code> + </> + )} + </div> + <div style={{ marginTop: 5, color: colors.text, fontSize: 12, wordBreak: 'break-word' }}> + {task.message || '-'} + </div> + {progress !== null && ( + <div style={{ marginTop: 7, display: 'flex', alignItems: 'center', gap: 8 }}> + <div style={{ flex: 1, height: 7, background: '#ffffff', borderRadius: 999, overflow: 'hidden' }}> + <div + style={{ + height: '100%', + width: `${progress}%`, + background: colors.fill, + transition: 'width 0.25s ease', + }} + /> + </div> + <span style={{ color: colors.text, fontSize: 11, fontVariantNumeric: 'tabular-nums' }}> + {Math.round(progress)}% + </span> + </div> + )} + </> + )} + {footer && <div style={{ marginTop: 7 }}>{footer}</div>} + </div> + ); +} diff --git a/frontend/src/config/appConstants.js b/frontend/src/config/appConstants.js index 6a12e87..7ee4d34 100644 --- a/frontend/src/config/appConstants.js +++ b/frontend/src/config/appConstants.js @@ -127,7 +127,7 @@ export const LEFT_GROUP_SECTIONS = { { key: 'psinsar', label: '时序InSAR', - tabs: ['psinsar_results', 'psinsar_analysis'], + tabs: ['psinsar_analysis'], }, ], ai_analysis: [ diff --git a/frontend/src/config/taskUiPolicies.js b/frontend/src/config/taskUiPolicies.js new file mode 100644 index 0000000..cea94ee --- /dev/null +++ b/frontend/src/config/taskUiPolicies.js @@ -0,0 +1,76 @@ +const TASK_UI_POLICIES = { + SCAN_DATA: { label: '同步源数据', featureScope: 'data_monitor' }, + SCAN_DINSAR: { label: '扫描 D-InSAR 结果', featureScope: 'dinsar_products' }, + DINSAR_RESULT_SCAN: { label: 'D-InSAR 结果扫描', featureScope: 'dinsar_products' }, + AI_TRAIN: { label: '训练 AI 模型', featureScope: 'ai' }, + AI_PREDICT: { label: '全量质量评估', featureScope: 'ai' }, + AI_ANALYZE: { label: 'AI 智能诊断', featureScope: 'ai' }, + AI_WARMUP: { label: 'AI 模型预热', featureScope: 'ai' }, + COPY_DATA: { label: '数据分发拷贝', featureScope: 'data_monitor' }, + SCAN_HAZARD: { label: '灾害点同步', featureScope: 'hazard' }, + UNPACK_ARCHIVES: { label: 'LT-1 解包', featureScope: 'data_monitor' }, + UNPACK_SENTINEL1: { label: 'Sentinel-1 解包', featureScope: 'data_monitor' }, + GF3_UNPACK: { label: 'GF3 解包', featureScope: 'data_monitor' }, + GF3_BATCH_PROCESS: { label: 'GF3 预处理', featureScope: 'data_monitor' }, + GF3_SARSCAPE_PRODUCE: { label: 'GF3 SARscape 生产', featureScope: 'data_monitor' }, + GF3_SARSCAPE_SYNC: { label: 'GF3 SARscape 入库', featureScope: 'data_monitor' }, + GF3_SARSCAPE_CLEAN: { label: 'GF3 中间清理', featureScope: 'data_monitor' }, + SCAN_ASSET_INVENTORY: { label: '资产库存扫描', featureScope: 'asset_inventory' }, + IDL_IMPORT: { label: 'ENVI 数据导入', featureScope: 'dinsar_production' }, + IDL_DINSAR: { label: 'ENVI D-InSAR 生产', featureScope: 'dinsar_production' }, + IDL_RUN_DINSAR: { label: 'ENVI D-InSAR 生产', featureScope: 'dinsar_production' }, + ISCE2_RUN: { label: 'ISCE2 D-InSAR 生产', featureScope: 'dinsar_production' }, + PYINT_RUN: { label: 'PyINT D-InSAR 生产', featureScope: 'dinsar_production' }, + LANDSAR_RUN: { label: 'LandSAR D-InSAR 生产', featureScope: 'dinsar_production' }, + SBAS_GAMMA_WORKFLOW: { label: 'Gamma SBAS 工作流', featureScope: 'sbas_insar' }, + SBAS_LANDSAR_WORKFLOW: { label: 'LandSAR SBAS 工作流', featureScope: 'sbas_insar' }, + SBAS_COREGISTRATION: { label: 'SBAS 配准', featureScope: 'sbas_insar' }, + SBAS_RDC_DEM: { label: 'SBAS RDC DEM', featureScope: 'sbas_insar' }, + SBAS_INTERFEROGRAMS: { label: 'SBAS 干涉图', featureScope: 'sbas_insar' }, + SBAS_IPTA_TIMESERIES: { label: 'SBAS IPTA 时序', featureScope: 'sbas_insar' }, + REBUILD_SBAS_INSAR_CATALOG: { label: 'SBAS 结果目录重建', featureScope: 'sbas_products' }, +}; + +const PREFIX_POLICIES = [ + { prefix: 'WATER_GEOCODE_', label: '水体地理编码', featureScope: 'water' }, + { prefix: 'WATER_DETECT_', label: '水体检测', featureScope: 'water' }, + { prefix: 'WATER_FLOOD_', label: '洪涝检测', featureScope: 'water' }, + { prefix: 'FLOOD_SCENE_PREPROCESS_', label: '洪涝场景预处理', featureScope: 'flood' }, + { prefix: 'FLOOD_WATER_EXTRACTION_', label: '洪涝水体提取', featureScope: 'flood' }, + { prefix: 'FLOOD_DETECTION_', label: '洪涝检测', featureScope: 'flood' }, + { prefix: 'GF3_PROCESS_', label: 'GF3 场景处理', featureScope: 'water' }, +]; + +export function getTaskUiPolicy(taskType) { + const normalized = String(taskType || '').trim().toUpperCase(); + if (!normalized) { + return { + taskType: '', + label: '后台任务', + featureScope: 'unknown', + globalVisible: true, + globalBlocking: false, + localBlocking: true, + }; + } + const exact = TASK_UI_POLICIES[normalized]; + const prefix = PREFIX_POLICIES.find((item) => normalized.startsWith(item.prefix)); + const policy = exact || prefix || {}; + return { + taskType: normalized, + label: policy.label || normalized, + featureScope: policy.featureScope || 'unknown', + globalVisible: policy.globalVisible ?? true, + globalBlocking: policy.globalBlocking ?? false, + localBlocking: policy.localBlocking ?? true, + }; +} + +export function isTaskGloballyBlocking(taskType) { + return !!getTaskUiPolicy(taskType).globalBlocking; +} + +export function getTaskTypeLabel(taskType) { + return getTaskUiPolicy(taskType).label; +} + diff --git a/frontend/src/hooks/useAppAuthLifecycle.js b/frontend/src/hooks/useAppAuthLifecycle.js index d768da3..69c64f0 100644 --- a/frontend/src/hooks/useAppAuthLifecycle.js +++ b/frontend/src/hooks/useAppAuthLifecycle.js @@ -15,7 +15,6 @@ export default function useAppAuthLifecycle({ setHasRadarSearched, setCurrentUser, setAuthChecked, - setIsGlobalLocked, setPendingTaskIds, setLicenseLoading, setLicenseStatus, @@ -72,7 +71,6 @@ export default function useAppAuthLifecycle({ setHasRadarSearched(false); clearRadarSearchResults(); setCurrentUser(null); - setIsGlobalLocked(false); setPendingTaskIds([]); prevLicenseOkRef.current = false; setAuthChecked(true); @@ -82,7 +80,6 @@ export default function useAppAuthLifecycle({ radarSearchRequestSeqRef, setHasRadarSearched, setCurrentUser, - setIsGlobalLocked, setPendingTaskIds, prevLicenseOkRef, setAuthChecked, diff --git a/frontend/src/hooks/useDinsarOperations.js b/frontend/src/hooks/useDinsarOperations.js index f10bc0e..072105f 100644 --- a/frontend/src/hooks/useDinsarOperations.js +++ b/frontend/src/hooks/useDinsarOperations.js @@ -11,8 +11,6 @@ import { normalizePagePayload } from '../utils/appHelpers'; import { normalizeTaskStatus } from '../utils/appUiHelpers'; import { DEFAULT_LIST_PAGE_SIZE } from '../config/appConstants'; -const NON_BLOCKING_TASK_TYPES = new Set(['UNPACK_ARCHIVES', 'UNPACK_SENTINEL1', 'GF3_UNPACK', 'GF3_SARSCAPE_PRODUCE', 'GF3_SARSCAPE_SYNC', 'GF3_SARSCAPE_CLEAN', 'SCAN_ASSET_INVENTORY', 'COPY_DATA']); - export default function useDinsarOperations({ onCleanupDinsarLayers, fetchRadarImagingDates, @@ -26,7 +24,7 @@ export default function useDinsarOperations({ setAiStatus, setActiveAiReport, } = useDinsarStore(); const { setHazardPoints } = useHazardStore(); - const { setPendingTaskIds, setNonBlockingTaskIds, setIsGlobalLocked } = useTaskStore(); + const { setPendingTaskIds } = useTaskStore(); const { currentUser } = useAuthStore(); const { hasRadarSearched, radarPagination, @@ -123,16 +121,8 @@ export default function useDinsarOperations({ }; const handleTaskStart = (taskId, message, options = {}) => { - const taskType = String(options?.taskType || '').trim().toUpperCase(); - const isNonBlocking = !!options?.nonBlocking || NON_BLOCKING_TASK_TYPES.has(taskType); if (taskId) { setPendingTaskIds(prev => [...prev, taskId]); - if (isNonBlocking) { - setNonBlockingTaskIds(prev => [...new Set([...prev, taskId])]); - } - } - if (!isNonBlocking) { - setIsGlobalLocked(true); } if (message) addLog('info', message); }; @@ -313,7 +303,6 @@ export default function useDinsarOperations({ const handleAnalyzeResult = async (resultId) => { if (!ensureCanOperate()) return; addLog('info', `正在对结果 ID:${resultId} 发起 AI 智能诊断任务...`); - setIsGlobalLocked(true); try { const response = await apiClient.post(`/ai/analyze-result/${resultId}`); const taskId = response.data.task_id; @@ -322,7 +311,6 @@ export default function useDinsarOperations({ } catch (error) { const msg = error.response?.data?.detail || error.message; addLog('error', `发起 AI 诊断失败: ${msg}`); - setIsGlobalLocked(false); } }; diff --git a/frontend/src/hooks/useGlobalTaskControl.js b/frontend/src/hooks/useGlobalTaskControl.js index 4ffd778..592f28b 100644 --- a/frontend/src/hooks/useGlobalTaskControl.js +++ b/frontend/src/hooks/useGlobalTaskControl.js @@ -2,13 +2,6 @@ import { useCallback, useEffect, useRef, useState } from 'react'; import apiClient from '../api/client'; import { normalizeTaskStatus } from '../utils/appUiHelpers'; -const NON_BLOCKING_TASK_TYPES = new Set(['UNPACK_ARCHIVES', 'UNPACK_SENTINEL1', 'GF3_UNPACK', 'GF3_SARSCAPE_PRODUCE', 'GF3_SARSCAPE_SYNC', 'GF3_SARSCAPE_CLEAN', 'SCAN_ASSET_INVENTORY', 'COPY_DATA']); - -const isTaskNonBlocking = (taskId, taskType, nonBlockingTaskIds = []) => ( - NON_BLOCKING_TASK_TYPES.has(String(taskType || '').toUpperCase()) - || nonBlockingTaskIds.includes(taskId) -); - export default function useGlobalTaskControl({ currentUser, licenseOk, @@ -16,32 +9,15 @@ export default function useGlobalTaskControl({ setActiveTasks, pendingTaskIds, setPendingTaskIds, - nonBlockingTaskIds, - setNonBlockingTaskIds, - isGlobalLocked, - setIsGlobalLocked, setIsCheckingTasks, handleTaskCompletionRef, - initializeAppDataRef, - addLog, }) { - const [forceUnlockPwd, setForceUnlockPwd] = useState(''); - const [showForceUnlock, setShowForceUnlock] = useState(false); - const lastLockTimeRef = useRef(null); - - const isGlobalLockedRef = useRef(isGlobalLocked); - useEffect(() => { - isGlobalLockedRef.current = isGlobalLocked; - if (isGlobalLocked) { - lastLockTimeRef.current = Date.now(); - } - }, [isGlobalLocked]); + const [cancelTaskPwd, setCancelTaskPwd] = useState(''); + const [showCancelTask, setShowCancelTask] = useState(false); // Stable refs so SSE handler doesn't need to re-subscribe on every render const pendingTaskIdsRef = useRef(pendingTaskIds); useEffect(() => { pendingTaskIdsRef.current = pendingTaskIds; }, [pendingTaskIds]); - const nonBlockingTaskIdsRef = useRef(nonBlockingTaskIds); - useEffect(() => { nonBlockingTaskIdsRef.current = nonBlockingTaskIds; }, [nonBlockingTaskIds]); const handleTasksUpdate = useCallback(async (tasks) => { setActiveTasks(tasks); @@ -51,26 +27,13 @@ export default function useGlobalTaskControl({ setIsCheckingTasks(false); const currentPending = pendingTaskIdsRef.current; - let updatedPending = currentPending; - let effectiveNonBlockingTaskIds = Array.from(new Set([ - ...nonBlockingTaskIdsRef.current, - ...tasks - .filter((task) => NON_BLOCKING_TASK_TYPES.has(String(task.task_type || '').toUpperCase())) - .map((task) => task.task_id), - ])); // 如果 pendingTaskIds 为空，但 activeTasks 有任务，说明是刷新后的初始化 // 需要将 activeTasks 中的任务添加到 pendingTaskIds if (currentPending.length === 0 && hasRunningTasks) { const activeTaskIds = tasks.map((t) => t.task_id); - const nextNonBlockingIds = tasks - .filter((task) => NON_BLOCKING_TASK_TYPES.has(String(task.task_type || '').toUpperCase())) - .map((task) => task.task_id); console.log('初始化：将活跃任务添加到 pending 列表:', activeTaskIds); setPendingTaskIds(activeTaskIds); - setNonBlockingTaskIds(nextNonBlockingIds); - updatedPending = activeTaskIds; - effectiveNonBlockingTaskIds = nextNonBlockingIds; } if (currentPending.length > 0) { @@ -121,47 +84,16 @@ export default function useGlobalTaskControl({ if (reallyFinishedIds.length > 0) { console.log('真正完成的任务:', reallyFinishedIds); - setPendingTaskIds((prev) => { - const newPending = prev.filter((id) => !reallyFinishedIds.includes(id)); - updatedPending = newPending; - return newPending; - }); - setNonBlockingTaskIds((prev) => prev.filter((id) => !reallyFinishedIds.includes(id))); - effectiveNonBlockingTaskIds = effectiveNonBlockingTaskIds.filter((id) => !reallyFinishedIds.includes(id)); - } else { - // 没有真正完成的任务，保持 updatedPending 不变 - updatedPending = currentPending; + setPendingTaskIds((prev) => prev.filter((id) => !reallyFinishedIds.includes(id))); } } } - // 只有当没有运行中的任务且没有待处理的任务时，才解锁 - const blockingRunningTasks = tasks.filter((task) => ( - !isTaskNonBlocking(task.task_id, task.task_type, effectiveNonBlockingTaskIds) - )); - const blockingPendingTaskIds = updatedPending.filter((taskId) => ( - !effectiveNonBlockingTaskIds.includes(taskId) - )); - const shouldBeLocked = blockingRunningTasks.length > 0 || blockingPendingTaskIds.length > 0; - - if (shouldBeLocked !== isGlobalLockedRef.current) { - setIsGlobalLocked(shouldBeLocked); - if (!shouldBeLocked) { - addLog('success', '后台任务已完成，正在同步最新数据...'); - setTimeout(() => { - initializeAppDataRef.current?.({ refreshRadarSearch: true }); - }, 500); - } - } }, [ setActiveTasks, setIsCheckingTasks, handleTaskCompletionRef, setPendingTaskIds, - setNonBlockingTaskIds, - setIsGlobalLocked, - addLog, - initializeAppDataRef, ]); // Fallback polling (used when SSE is unavailable) @@ -216,22 +148,22 @@ export default function useGlobalTaskControl({ }; }, [currentUser, licenseOk, syncActiveTasks, handleTasksUpdate]); - const handleForceUnlock = useCallback(() => { - if (!forceUnlockPwd || activeTasks.length === 0) return; + const handleCancelActiveTasks = useCallback(() => { + if (!cancelTaskPwd || activeTasks.length === 0) return; Promise.all(activeTasks.map((task) => - apiClient.post(`/tasks/${task.task_id}/force-cancel`, { password: forceUnlockPwd }).catch(() => {}) + apiClient.post(`/tasks/${task.task_id}/force-cancel`, { password: cancelTaskPwd }).catch(() => {}) )).then(() => { - setForceUnlockPwd(''); - setShowForceUnlock(false); + setCancelTaskPwd(''); + setShowCancelTask(false); syncActiveTasks(); }); - }, [activeTasks, forceUnlockPwd, syncActiveTasks]); + }, [activeTasks, cancelTaskPwd, syncActiveTasks]); return { - forceUnlockPwd, - setForceUnlockPwd, - showForceUnlock, - setShowForceUnlock, - handleForceUnlock, + cancelTaskPwd, + setCancelTaskPwd, + showCancelTask, + setShowCancelTask, + handleCancelActiveTasks, }; } diff --git a/frontend/src/hooks/usePairingLogic.js b/frontend/src/hooks/usePairingLogic.js index f53daab..94f8864 100644 --- a/frontend/src/hooks/usePairingLogic.js +++ b/frontend/src/hooks/usePairingLogic.js @@ -5,6 +5,7 @@ * focusBatchAfterCreate, clearPsResults */ import apiClient from '../api/client'; +import { getPairingHealth } from '../api/pairing'; import { useUiStore, usePairingStore, useMapStore, useBatchStore, useAuthStore, } from '../store'; @@ -171,8 +172,42 @@ export default function usePairingLogic({ setPairingAlert({ warnings: [], fallbackUsed: false }); const formData = new FormData(); - for (const key in pairingParams) { - const value = pairingParams[key]; + const effectivePairingParams = { ...pairingParams }; + if (!effectivePairingParams.strategy) { + effectivePairingParams.strategy = 'sbas'; + } + if (effectivePairingParams.strategy === 'all') { + const hasDateWindow = Boolean( + effectivePairingParams.master_date_from + || effectivePairingParams.master_date_to + || effectivePairingParams.slave_date_from + || effectivePairingParams.slave_date_to + ); + if (!hasDateWindow && pairingAoiMode !== 'region' && !pairingFiles?.length) { + addLog('warn', '全部配对可能返回大量结果。请先限定行政区、上传 AOI 或设置主/从影像时间范围。'); + setIsLoading(false); + return; + } + } + + try { + const pairingHealth = await getPairingHealth(); + if (pairingHealth?.needs_rebuild || pairingHealth?.status !== 'READY') { + addLog( + 'warn', + `配对基础当前状态为 ${pairingHealth?.status || 'UNKNOWN'}，dirty 场景 ${Number(pairingHealth?.dirty_scene_count || 0)}。请先在“配对规划”页执行“修复配对基础”。` + ); + setIsLoading(false); + return; + } + } catch (error) { + addLog('warn', `配对基础状态检查失败: ${error.response?.data?.detail || error.message}`); + setIsLoading(false); + return; + } + + for (const key in effectivePairingParams) { + const value = effectivePairingParams[key]; // 跳过 null/undefined 值 if (value === null || value === undefined) continue; // allowed_satellites 是数组，需要序列化为 JSON diff --git a/frontend/src/hooks/useTaskMonitor.js b/frontend/src/hooks/useTaskMonitor.js new file mode 100644 index 0000000..25e39f6 --- /dev/null +++ b/frontend/src/hooks/useTaskMonitor.js @@ -0,0 +1,109 @@ +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { useTaskStore } from '../store'; +import { getRecentTasks, getTaskLogs } from '../api/tasks'; + +const normalizeList = (value) => (Array.isArray(value) ? value.filter(Boolean) : []); + +const matchesTask = (task, taskTypes, taskTypePrefixes, taskIds) => { + const taskId = String(task?.task_id || ''); + const taskType = String(task?.task_type || '').toUpperCase(); + if (taskIds.length && taskIds.includes(taskId)) return true; + if (taskTypes.length && taskTypes.includes(taskType)) return true; + if (taskTypePrefixes.length && taskTypePrefixes.some(prefix => taskType.startsWith(prefix))) return true; + return !taskTypes.length && !taskTypePrefixes.length && !taskIds.length; +}; + +export default function useTaskMonitor({ + taskTypes = [], + taskTypePrefixes = [], + taskIds = [], + showRecent = false, + recentLimit = 5, + pollRecentMs = 0, +} = {}) { + const activeTasks = useTaskStore((state) => state.activeTasks); + const normalizedTaskTypes = useMemo( + () => normalizeList(taskTypes).map(item => String(item).toUpperCase()), + [taskTypes], + ); + const normalizedPrefixes = useMemo( + () => normalizeList(taskTypePrefixes).map(item => String(item).toUpperCase()), + [taskTypePrefixes], + ); + const normalizedTaskIds = useMemo( + () => normalizeList(taskIds).map(item => String(item)), + [taskIds], + ); + const [recentTasks, setRecentTasks] = useState([]); + const [recentLoading, setRecentLoading] = useState(false); + const [recentError, setRecentError] = useState(''); + + const filteredActiveTasks = useMemo( + () => activeTasks.filter(task => matchesTask(task, normalizedTaskTypes, normalizedPrefixes, normalizedTaskIds)), + [activeTasks, normalizedTaskTypes, normalizedPrefixes, normalizedTaskIds], + ); + + const refreshRecentTasks = useCallback(async () => { + if (!showRecent) return []; + setRecentLoading(true); + setRecentError(''); + try { + if (!normalizedTaskTypes.length) { + setRecentTasks([]); + return []; + } + const data = await getRecentTasks(normalizedTaskTypes, [], recentLimit, 0); + const tasks = Array.isArray(data) ? data : (data?.tasks || []); + const filtered = tasks.filter(task => matchesTask(task, normalizedTaskTypes, normalizedPrefixes, normalizedTaskIds)); + setRecentTasks(filtered); + return filtered; + } catch (error) { + setRecentError(error?.response?.data?.detail || error?.message || '任务记录加载失败'); + setRecentTasks([]); + return []; + } finally { + setRecentLoading(false); + } + }, [normalizedTaskTypes, normalizedPrefixes, normalizedTaskIds, recentLimit, showRecent]); + + useEffect(() => { + if (!showRecent) { + setRecentTasks([]); + setRecentError(''); + return undefined; + } + void refreshRecentTasks(); + if (!pollRecentMs) return undefined; + const timer = window.setInterval(() => { + void refreshRecentTasks(); + }, pollRecentMs); + return () => window.clearInterval(timer); + }, [pollRecentMs, refreshRecentTasks, showRecent]); + + const latestTask = filteredActiveTasks[0] || recentTasks[0] || null; + const isBusy = filteredActiveTasks.length > 0; + + const loadTaskLogs = useCallback((taskId, limit = 50, offset = 0) => ( + getTaskLogs(taskId, limit, offset) + ), []); + + return useMemo(() => ({ + activeTasks: filteredActiveTasks, + recentTasks, + latestTask, + isBusy, + recentLoading, + recentError, + refreshRecentTasks, + loadTaskLogs, + }), [ + filteredActiveTasks, + recentTasks, + latestTask, + isBusy, + recentLoading, + recentError, + refreshRecentTasks, + loadTaskLogs, + ]); +} diff --git a/frontend/src/i18n/translations.js b/frontend/src/i18n/translations.js index 3a27f7f..437e808 100644 --- a/frontend/src/i18n/translations.js +++ b/frontend/src/i18n/translations.js @@ -231,10 +231,10 @@ { zh: '运行中', en: 'Running' }, { zh: '引擎', en: 'Engine' }, { zh: '任务运行中', en: 'Task Running' }, - { zh: '强制解锁', en: 'Force Unlock' }, + { zh: '取消任务', en: 'Cancel Task' }, { zh: '输入管理员密码', en: 'Enter admin password' }, { zh: '确认取消', en: 'Confirm Cancel' }, - { zh: '按钮已锁定，等待任务完成', en: 'Buttons locked, waiting for task completion' }, + { zh: '同类任务运行中，当前提交按钮暂不可用。', en: 'A similar task is running. Submit buttons are temporarily unavailable.' }, { zh: 'Task 状态总览', en: 'Task Status Overview' }, { zh: '加载中...', en: 'Loading...' }, { zh: '刷新', en: 'Refresh' }, @@ -300,8 +300,8 @@ { zh: '扫描入库触发失败', en: 'Result scan trigger failed' }, { zh: '提取完成', en: 'Extraction complete' }, { zh: '总览加载失败', en: 'Overview load failed' }, - { zh: '任务已强制取消，前端已解锁。', en: 'Task force-cancelled, UI unlocked.' }, - { zh: '强制解锁失败', en: 'Force unlock failed' }, + { zh: '任务取消请求已提交。', en: 'Task cancellation requested.' }, + { zh: '取消任务失败', en: 'Task cancellation failed' }, { zh: '任务已入队', en: 'Task queued' }, // --- HazardPointPanel --- diff --git a/frontend/src/panels/SbasInsarMapAnalysisPanel.jsx b/frontend/src/panels/SbasInsarMapAnalysisPanel.jsx new file mode 100644 index 0000000..e72e3d8 --- /dev/null +++ b/frontend/src/panels/SbasInsarMapAnalysisPanel.jsx @@ -0,0 +1,652 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { LineChart } from 'echarts/charts'; +import { + DataZoomComponent, + GridComponent, + LegendComponent, + MarkLineComponent, + TooltipComponent, +} from 'echarts/components'; +import * as echarts from 'echarts/core'; +import { CanvasRenderer } from 'echarts/renderers'; + +import { + getSbasInsarProductAssetUrl, + getSbasInsarProductDetail, + listSbasInsarProducts, + querySbasInsarPointTimeseries, +} from '../api/sbasInsarProducts'; + +echarts.use([ + LineChart, + GridComponent, + TooltipComponent, + LegendComponent, + DataZoomComponent, + MarkLineComponent, + CanvasRenderer, +]); + +const panelStyle = { display: 'grid', gap: 12, padding: 16 }; +const cardStyle = { + border: '1px solid #d8dee8', + borderRadius: 8, + background: '#ffffff', + overflow: 'hidden', +}; +const cardBodyStyle = { display: 'grid', gap: 10, padding: 12 }; +const mutedStyle = { color: '#64748b', fontSize: 12, lineHeight: 1.55 }; +const labelStyle = { color: '#475569', fontSize: 12, fontWeight: 750 }; +const inputStyle = { + width: '100%', + minWidth: 0, + border: '1px solid #cbd5e1', + borderRadius: 6, + padding: '7px 9px', + fontSize: 12, + boxSizing: 'border-box', +}; +const buttonStyle = { + border: '1px solid #cbd5e1', + borderRadius: 6, + background: '#ffffff', + color: '#0f172a', + cursor: 'pointer', + fontSize: 12, + fontWeight: 700, + padding: '7px 11px', +}; +const primaryButtonStyle = { + ...buttonStyle, + borderColor: '#1d4ed8', + background: '#1d4ed8', + color: '#ffffff', +}; +const chartColors = ['#1d4ed8', '#dc2626', '#059669', '#7c3aed', '#d97706', '#0f766e', '#111827']; + +function formatNumber(value, digits = 2) { + const numeric = Number(value); + if (!Number.isFinite(numeric)) return '-'; + return numeric.toFixed(digits); +} + +function formatDate(value) { + if (!value) return '-'; + return String(value).slice(0, 10); +} + +function normalizeDate(value) { + const text = String(value || '').trim(); + if (/^\d{8}$/.test(text)) { + return `${text.slice(0, 4)}-${text.slice(4, 6)}-${text.slice(6, 8)}`; + } + return text; +} + +function parseDateMs(value) { + const date = normalizeDate(value); + if (!date) return NaN; + const time = Date.parse(`${date}T00:00:00Z`); + return Number.isFinite(time) ? time : NaN; +} + +function normalizeDisplacements(rows) { + return (Array.isArray(rows) ? rows : []) + .map((item) => { + const date = normalizeDate(item?.date); + const time = parseDateMs(date); + const displacement = Number(item?.displacement_mm ?? item?.displacement ?? item?.value); + if (!date || !Number.isFinite(time) || !Number.isFinite(displacement)) return null; + return { date, time, displacement }; + }) + .filter(Boolean) + .sort((left, right) => left.time - right.time); +} + +function buildLinearAxis(values, targetTicks = 6) { + const finite = values.map(Number).filter(Number.isFinite); + if (!finite.length) return { min: -1, max: 1, step: 0.5 }; + let min = Math.min(...finite, 0); + let max = Math.max(...finite, 0); + if (min === max) { + const pad = Math.max(Math.abs(min) * 0.2, 1); + min -= pad; + max += pad; + } + const rawStep = (max - min) / Math.max(1, targetTicks); + const magnitude = 10 ** Math.floor(Math.log10(rawStep)); + const normalized = rawStep / magnitude; + const niceFactor = normalized <= 1 ? 1 : normalized <= 2 ? 2 : normalized <= 5 ? 5 : 10; + const step = niceFactor * magnitude; + return { + min: Math.floor(min / step) * step, + max: Math.ceil(max / step) * step, + step, + }; +} + +function formatAxisDate(value) { + const date = new Date(value); + if (Number.isNaN(date.getTime())) return ''; + return date.toISOString().slice(0, 10); +} + +function Metric({ label, value, accent }) { + return ( + <div style={{ border: '1px solid #e2e8f0', borderRadius: 8, padding: '8px 9px', background: '#f8fafc', minWidth: 0 }}> + <div style={{ color: '#64748b', fontSize: 12 }}>{label}</div> + <div style={{ color: accent || '#0f172a', fontSize: 14, fontWeight: 800, marginTop: 4, overflowWrap: 'anywhere' }}>{value}</div> + </div> + ); +} + +function StatusBadge({ value }) { + const color = value === 'READY' ? '#15803d' : value === 'INCOMPLETE' ? '#b45309' : value === 'ERROR' ? '#dc2626' : '#64748b'; + return ( + <span style={{ display: 'inline-flex', alignItems: 'center', gap: 6, color, fontSize: 12, fontWeight: 800 }}> + <span style={{ width: 7, height: 7, borderRadius: 999, background: color }} /> + {value || 'UNKNOWN'} + </span> + ); +} + +function findAsset(assets, roles) { + const roleSet = new Set(roles); + return (Array.isArray(assets) ? assets : []).find((asset) => roleSet.has(asset.asset_role) && asset.exists_flag); +} + +function assetCacheKey(asset) { + return [asset?.id, asset?.file_size, asset?.updated_at || asset?.created_at || asset?.relative_path] + .filter(Boolean) + .join(':'); +} + +function pointCardsFromDetail(detail, queryResult) { + const points = detail?.monitor_points?.monitor_points || []; + const cards = points.map((point, index) => ({ + id: point.point_id || `point_${index + 1}`, + name: point.selection_label || point.point_id || `监测点 ${index + 1}`, + subName: point.selection_key || '', + rate: Number(point.deformation_rate_mm_per_year), + values: normalizeDisplacements(point.displacements), + color: chartColors[index % chartColors.length], + point, + })); + if (queryResult) { + const matched = queryResult.matched || {}; + cards.push({ + id: matched.used_nearest ? 'query_nearest' : 'query_exact', + name: matched.used_nearest ? '查询点最近邻' : '查询点', + subName: `${formatNumber(matched.lon, 6)}, ${formatNumber(matched.lat, 6)}`, + rate: Number(matched.los_rate_mm_per_year), + values: normalizeDisplacements(queryResult.displacements), + color: '#111827', + point: { + point_id: matched.used_nearest ? 'query_nearest' : 'query_exact', + selection_label: matched.used_nearest ? '查询点最近邻' : '查询点', + selection_key: matched.used_nearest ? `最近邻 ${formatNumber(matched.distance_m, 1)} m` : '输入点有效像元', + deformation_rate_mm_per_year: matched.los_rate_mm_per_year, + displacements: queryResult.displacements || [], + matched, + lon: matched.lon, + lat: matched.lat, + }, + }); + } + return cards.filter((card) => card.values.length > 0); +} + +function EchartsCanvas({ option }) { + const containerRef = useRef(null); + const chartRef = useRef(null); + + useEffect(() => { + if (!containerRef.current) return undefined; + const chart = echarts.init(containerRef.current, null, { renderer: 'canvas' }); + chartRef.current = chart; + let resizeObserver = null; + if (typeof ResizeObserver !== 'undefined') { + resizeObserver = new ResizeObserver(() => chart.resize()); + resizeObserver.observe(containerRef.current); + } + const onResize = () => chart.resize(); + window.addEventListener('resize', onResize); + return () => { + window.removeEventListener('resize', onResize); + resizeObserver?.disconnect(); + chart.dispose(); + chartRef.current = null; + }; + }, []); + + useEffect(() => { + if (!chartRef.current || !option) return; + chartRef.current.setOption(option, true); + }, [option]); + + return <div ref={containerRef} style={{ width: '100%', minWidth: 640, height: 360 }} />; +} + +function TimeseriesChart({ cards }) { + const chart = useMemo(() => { + const validCards = (Array.isArray(cards) ? cards : []).filter((card) => card.values.length > 0); + const allValues = validCards.flatMap((card) => card.values.map((value) => value.displacement)); + const allTimes = validCards.flatMap((card) => card.values.map((value) => value.time)); + if (!validCards.length || !allTimes.length) return null; + return { + cards: validCards, + yAxis: buildLinearAxis(allValues, 6), + minTime: Math.min(...allTimes), + maxTime: Math.max(...allTimes), + }; + }, [cards]); + + const option = useMemo(() => { + if (!chart) return null; + const oneDay = 24 * 60 * 60 * 1000; + const xMin = chart.minTime === chart.maxTime ? chart.minTime - oneDay : chart.minTime; + const xMax = chart.minTime === chart.maxTime ? chart.maxTime + oneDay : chart.maxTime; + return { + animation: false, + backgroundColor: '#ffffff', + color: chart.cards.map((card) => card.color), + tooltip: { + trigger: 'axis', + confine: true, + axisPointer: { type: 'line', snap: true }, + formatter: (params) => { + const rows = (Array.isArray(params) ? params : [params]).filter((item) => Array.isArray(item?.value)); + if (!rows.length) return ''; + const date = rows[0]?.data?.date || formatAxisDate(rows[0].value[0]); + const body = rows.map((item) => `${item.marker}<b>${item.seriesName}</b>: ${formatNumber(item.value[1], 2)} mm`).join('<br/>'); + return `<div style="font-weight:750;margin-bottom:4px">${date}</div>${body}`; + }, + }, + legend: { + type: 'scroll', + top: 6, + left: 8, + right: 8, + itemWidth: 14, + itemHeight: 8, + textStyle: { color: '#334155', fontSize: 11 }, + }, + grid: { left: 72, right: 28, top: 58, bottom: 62 }, + xAxis: { + type: 'time', + min: xMin, + max: xMax, + name: 'SAR Date', + nameLocation: 'middle', + nameGap: 38, + axisLabel: { color: '#64748b', hideOverlap: true, formatter: formatAxisDate }, + splitLine: { show: true, lineStyle: { color: '#f1f5f9' } }, + }, + yAxis: { + type: 'value', + min: chart.yAxis.min, + max: chart.yAxis.max, + interval: chart.yAxis.step, + name: '累计形变 (mm)', + nameLocation: 'middle', + nameGap: 50, + axisLabel: { color: '#64748b' }, + splitLine: { show: true, lineStyle: { color: '#e2e8f0' } }, + }, + dataZoom: [ + { type: 'inside', xAxisIndex: 0, filterMode: 'none' }, + { type: 'slider', xAxisIndex: 0, filterMode: 'none', height: 22, bottom: 18 }, + ], + series: chart.cards.map((card, index) => ({ + name: card.name, + type: 'line', + data: card.values.map((value) => ({ value: [value.time, Number(value.displacement.toFixed(6))], date: value.date })), + showSymbol: true, + symbolSize: card.id.startsWith('query') ? 8 : 6, + smooth: false, + connectNulls: false, + lineStyle: { width: card.id.startsWith('query') ? 3 : 2.2, color: card.color }, + itemStyle: { color: card.color, borderColor: '#ffffff', borderWidth: 1 }, + markLine: index === 0 ? { + symbol: 'none', + silent: true, + data: [{ yAxis: 0, name: '0 mm' }], + label: { formatter: '0 mm', color: '#475569' }, + lineStyle: { color: '#0f172a', opacity: 0.32, type: 'dashed', width: 1 }, + } : undefined, + })), + }; + }, [chart]); + + if (!option) { + return ( + <div style={{ border: '1px dashed #cbd5e1', borderRadius: 8, padding: 12, background: '#f8fafc', ...mutedStyle }}> + 暂无可绘制的监测点或查询点时序。 + </div> + ); + } + + return ( + <div style={{ border: '1px solid #e2e8f0', borderRadius: 8, overflow: 'hidden', background: '#ffffff' }}> + <div style={{ padding: '9px 12px', background: '#f8fafc', borderBottom: '1px solid #e2e8f0' }}> + <div style={{ fontSize: 13, fontWeight: 850, color: '#0f172a' }}>监测点和查询点形变曲线</div> + <div style={{ ...mutedStyle, marginTop: 3 }}>横坐标按真实 SAR 日期间隔缩放，纵坐标为累计形变 mm。</div> + </div> + <div style={{ overflowX: 'auto', padding: '8px 10px 0' }}> + <EchartsCanvas option={option} /> + </div> + </div> + ); +} + +export default function SbasInsarMapAnalysisPanel({ + readOnly, + onToggleRateLayer, + onRateOpacityChange, + onToggleMonitorPoints, + onToggleProductOverview, + onFlyToProduct, + onShowQueryPoint, + onClearLayers, +}) { + const [products, setProducts] = useState([]); + const [selectedId, setSelectedId] = useState(''); + const [detail, setDetail] = useState(null); + const [loading, setLoading] = useState(false); + const [detailLoading, setDetailLoading] = useState(false); + const [message, setMessage] = useState(''); + const [overviewVisible, setOverviewVisible] = useState(false); + const [rateVisible, setRateVisible] = useState(false); + const [monitorVisible, setMonitorVisible] = useState(false); + const [opacity, setOpacity] = useState(0.78); + const [lon, setLon] = useState(''); + const [lat, setLat] = useState(''); + const [queryLoading, setQueryLoading] = useState(false); + const [queryResult, setQueryResult] = useState(null); + + const loadProducts = useCallback(async () => { + setLoading(true); + setMessage(''); + try { + const payload = await listSbasInsarProducts({ limit: 30, offset: 0, status: 'READY' }); + const nextItems = (payload?.items || []).filter((item) => ( + String(item.engine_code || '').toLowerCase() === 'gamma' + || String(item.processor_code || '').toLowerCase().includes('gamma') + )); + setProducts(nextItems); + setSelectedId((prev) => (prev && nextItems.some((item) => String(item.id) === String(prev)) ? prev : String(nextItems[0]?.id || ''))); + if (!nextItems.length) { + setMessage('暂无 READY 的 Gamma SBAS 产品；请先完成结果注册。'); + } + } catch (error) { + setMessage(`加载 SBAS 产品失败：${error?.response?.data?.detail || error.message}`); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { + void loadProducts(); + }, [loadProducts]); + + useEffect(() => { + if (!selectedId) { + setDetail(null); + return undefined; + } + let disposed = false; + setDetailLoading(true); + setMessage(''); + setOverviewVisible(false); + setRateVisible(false); + setMonitorVisible(false); + setQueryResult(null); + onClearLayers?.(); + getSbasInsarProductDetail(selectedId) + .then((payload) => { + if (!disposed) setDetail(payload); + }) + .catch((error) => { + if (!disposed) { + setDetail(null); + setMessage(`加载产品详情失败：${error?.response?.data?.detail || error.message}`); + } + }) + .finally(() => { + if (!disposed) setDetailLoading(false); + }); + return () => { + disposed = true; + }; + }, [selectedId, onClearLayers]); + + const selectedProduct = useMemo( + () => products.find((item) => String(item.id) === String(selectedId)) || detail, + [detail, products, selectedId], + ); + + const assets = detail?.assets || []; + const rateAsset = useMemo(() => findAsset(assets, ['primary_geocoded_preview', 'primary_rate_color_preview']), [assets]); + const colorbarAsset = useMemo(() => findAsset(assets, ['primary_colorbar']), [assets]); + const monitorPoints = detail?.monitor_points?.monitor_points || []; + const geocodedPointCount = monitorPoints.filter((point) => Number.isFinite(Number(point.lon)) && Number.isFinite(Number(point.lat))).length; + const chartCards = useMemo(() => pointCardsFromDetail(detail, queryResult), [detail, queryResult]); + const colorPolicy = detail?.color_policy || {}; + const range = colorPolicy?.display_range_mm_per_year || []; + const rateRangeText = Array.isArray(range) && range.length >= 2 + ? `${formatNumber(range[0], 0)} 到 ${formatNumber(range[1], 0)} mm/yr` + : '-80 到 80 mm/yr'; + + const toggleRate = () => { + if (!detail) return; + const nextVisible = !rateVisible; + const ok = onToggleRateLayer?.(detail, nextVisible, opacity); + if (ok !== false) setRateVisible(nextVisible); + }; + + const toggleMonitor = () => { + if (!detail) return; + const nextVisible = !monitorVisible; + const ok = onToggleMonitorPoints?.(detail, nextVisible); + if (ok !== false) setMonitorVisible(nextVisible); + }; + + const toggleOverview = () => { + const nextVisible = !overviewVisible; + const ok = onToggleProductOverview?.(products, nextVisible); + if (ok !== false) { + setOverviewVisible(nextVisible); + setMessage(nextVisible ? `已在地图显示 ${products.length} 个 SBAS 产品范围和时间。` : '已隐藏 SBAS 产品范围总览。'); + } + }; + + const changeOpacity = (event) => { + const next = Number(event.target.value); + setOpacity(next); + onRateOpacityChange?.(next); + }; + + const queryPoint = async () => { + if (!detail?.id) return; + const numericLon = Number(lon); + const numericLat = Number(lat); + if (!Number.isFinite(numericLon) || !Number.isFinite(numericLat)) { + setMessage('请输入有效 WGS84 经度和纬度。'); + return; + } + if (numericLon < -180 || numericLon > 180 || numericLat < -90 || numericLat > 90) { + setMessage('经纬度超出 WGS84 范围。'); + return; + } + setQueryLoading(true); + setMessage(''); + try { + const result = await querySbasInsarPointTimeseries(detail.id, { lon: numericLon, lat: numericLat }); + setQueryResult(result); + onShowQueryPoint?.(result, detail); + const matched = result?.matched || {}; + setMessage(matched.used_nearest ? `已使用最近有效像元，距离 ${formatNumber(matched.distance_m, 1)} m。` : '查询点位于有效像元，曲线已生成。'); + } catch (error) { + setQueryResult(null); + setMessage(`查询失败：${error?.response?.data?.detail || error.message}`); + } finally { + setQueryLoading(false); + } + }; + + return ( + <div style={panelStyle}> + <div style={{ display: 'grid', gap: 4 }}> + <div style={{ color: '#0f172a', fontSize: 16, fontWeight: 900 }}>时序InSAR地图分析</div> + <div style={mutedStyle}>只读检视 Gamma SBAS 成果：速率图叠加、自动监测点、WGS84 点查询和形变曲线。</div> + </div> + + <div style={cardStyle}> + <div style={cardBodyStyle}> + <div style={{ display: 'grid', gap: 6 }}> + <label style={labelStyle} htmlFor="sbas-map-product">SBAS 产品</label> + <select + id="sbas-map-product" + value={selectedId} + onChange={(event) => setSelectedId(event.target.value)} + disabled={loading || detailLoading || !products.length} + style={inputStyle} + > + {products.map((item) => ( + <option key={item.id} value={item.id}> + {item.display_name || item.run_key || item.id} / {formatDate(item.date_start)} → {formatDate(item.date_end)} + </option> + ))} + </select> + </div> + <div style={{ display: 'flex', gap: 8, flexWrap: 'wrap' }}> + <button type="button" onClick={loadProducts} disabled={loading} style={buttonStyle}>{loading ? '刷新中...' : '刷新产品'}</button> + <button + type="button" + onClick={toggleOverview} + disabled={!products.length} + style={overviewVisible ? primaryButtonStyle : buttonStyle} + > + {overviewVisible ? '隐藏全部范围' : `查看全部范围/时间 (${products.length})`} + </button> + <button type="button" onClick={() => onFlyToProduct?.(detail || selectedProduct)} disabled={!selectedProduct} style={buttonStyle}>定位成果范围</button> + <button + type="button" + onClick={() => { + setOverviewVisible(false); + setRateVisible(false); + setMonitorVisible(false); + setQueryResult(null); + onClearLayers?.(); + }} + style={buttonStyle} + > + 清除地图图层 + </button> + </div> + {message && ( + <div style={{ color: message.includes('失败') || message.includes('超出') || message.includes('暂无') ? '#dc2626' : '#166534', fontSize: 12 }}> + {message} + </div> + )} + </div> + </div> + + {detailLoading && <div className="empty-state">正在加载 SBAS 产品详情...</div>} + + {detail && ( + <> + <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(130px, 1fr))', gap: 8 }}> + <Metric label="状态" value={<StatusBadge value={detail.status} />} accent={detail.status === 'READY' ? '#15803d' : '#b45309'} /> + <Metric label="时间范围" value={`${formatDate(detail.date_start)} → ${formatDate(detail.date_end)}`} /> + <Metric label="栈期数" value={detail.stack_size ?? detail.stack_dates?.length ?? '-'} /> + <Metric label="监测点" value={`${geocodedPointCount}/${monitorPoints.length || 0} 有经纬度`} accent={geocodedPointCount ? '#15803d' : '#b45309'} /> + <Metric label="色表范围" value={rateRangeText} /> + </div> + + <div style={cardStyle}> + <div style={{ padding: '10px 12px', background: '#f8fafc', borderBottom: '1px solid #e2e8f0' }}> + <div style={{ fontSize: 13, fontWeight: 850, color: '#0f172a' }}>地图图层</div> + <div style={{ ...mutedStyle, marginTop: 3 }}>速率图使用专家链路生成的 Gamma hls.cm 浏览图；监测点使用 `disp_prt_2d` 自动选点结果。</div> + </div> + <div style={cardBodyStyle}> + <div style={{ display: 'flex', gap: 8, flexWrap: 'wrap', alignItems: 'center' }}> + <button type="button" onClick={toggleRate} disabled={!rateAsset} style={rateVisible ? primaryButtonStyle : buttonStyle}> + {rateVisible ? '隐藏 LOS 速率图' : '显示 LOS 速率图'} + </button> + <button type="button" onClick={toggleMonitor} disabled={!monitorPoints.length || !geocodedPointCount} style={monitorVisible ? primaryButtonStyle : buttonStyle}> + {monitorVisible ? '隐藏监测点' : '显示监测点'} + </button> + </div> + <div style={{ display: 'grid', gridTemplateColumns: '80px 1fr 44px', gap: 8, alignItems: 'center' }}> + <div style={labelStyle}>透明度</div> + <input type="range" min="0.2" max="1" step="0.02" value={opacity} onChange={changeOpacity} disabled={!rateAsset} /> + <div style={{ color: '#334155', fontSize: 12, fontWeight: 800 }}>{Math.round(opacity * 100)}%</div> + </div> + {colorbarAsset && ( + <div style={{ display: 'grid', gap: 5 }}> + <div style={labelStyle}>LOS 速率色卡</div> + <img + src={getSbasInsarProductAssetUrl(detail.id, colorbarAsset.id, assetCacheKey(colorbarAsset))} + alt="Gamma hls.cm LOS velocity colorbar" + style={{ width: '100%', maxHeight: 76, objectFit: 'contain', border: '1px solid #e2e8f0', borderRadius: 6, background: '#ffffff' }} + /> + </div> + )} + {!rateAsset && <div style={mutedStyle}>未找到可叠加的 LOS 速率预览资产。</div>} + {monitorPoints.length > 0 && geocodedPointCount === 0 && ( + <div style={{ color: '#b45309', fontSize: 12 }}> + 当前监测点摘要没有 WGS84 经纬度；重新注册资产后可在主地图落点。 + </div> + )} + </div> + </div> + + <div style={cardStyle}> + <div style={{ padding: '10px 12px', background: '#f8fafc', borderBottom: '1px solid #e2e8f0' }}> + <div style={{ fontSize: 13, fontWeight: 850, color: '#0f172a' }}>WGS84 点查询</div> + <div style={{ ...mutedStyle, marginTop: 3 }}>输入覆盖区内经纬度；若不是有效像元，系统会取最近有效像元并标注距离。</div> + </div> + <div style={cardBodyStyle}> + <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(150px, 1fr)) minmax(110px, auto)', gap: 8, alignItems: 'center' }}> + <input + value={lon} + onChange={(event) => setLon(event.target.value)} + onKeyDown={(event) => { if (event.key === 'Enter') queryPoint(); }} + placeholder="经度 lon" + style={inputStyle} + /> + <input + value={lat} + onChange={(event) => setLat(event.target.value)} + onKeyDown={(event) => { if (event.key === 'Enter') queryPoint(); }} + placeholder="纬度 lat" + style={inputStyle} + /> + <button type="button" onClick={queryPoint} disabled={queryLoading || !detail?.id} style={primaryButtonStyle}> + {queryLoading ? '查询中...' : '查询曲线'} + </button> + </div> + {queryResult && ( + <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(130px, 1fr))', gap: 8 }}> + <Metric label="匹配方式" value={queryResult.matched?.used_nearest ? '最近有效像元' : '输入点有效像元'} accent={queryResult.matched?.used_nearest ? '#b45309' : '#15803d'} /> + <Metric label="匹配经纬度" value={`${formatNumber(queryResult.matched?.lon, 6)}, ${formatNumber(queryResult.matched?.lat, 6)}`} /> + <Metric label="距离" value={`${formatNumber(queryResult.matched?.distance_m, 1)} m`} /> + <Metric label="LOS 速率" value={`${formatNumber(queryResult.matched?.los_rate_mm_per_year, 2)} mm/yr`} /> + </div> + )} + </div> + </div> + + <TimeseriesChart cards={chartCards} /> + </> + )} + + {!loading && !detailLoading && !detail && ( + <div className="empty-state"> + {readOnly ? '暂无可检视的 Gamma SBAS 产品。' : '暂无可检视的 Gamma SBAS 产品。'} + </div> + )} + </div> + ); +} diff --git a/frontend/src/store/pairingStore.js b/frontend/src/store/pairingStore.js index 35ffa37..f16c960 100644 --- a/frontend/src/store/pairingStore.js +++ b/frontend/src/store/pairingStore.js @@ -29,7 +29,7 @@ export const usePairingStore = create((set) => ({ master_date_to: '', slave_date_from: '', slave_date_to: '', - strategy: 'all', + strategy: 'sbas', num_connections: 1, reference_image_id: null, allowed_satellites: null, diff --git a/frontend/src/store/taskStore.js b/frontend/src/store/taskStore.js index 93d008d..287d76f 100644 --- a/frontend/src/store/taskStore.js +++ b/frontend/src/store/taskStore.js @@ -5,13 +5,9 @@ const s = (set, key) => (v) => export const useTaskStore = create((set) => ({ activeTasks: [], - isGlobalLocked: false, isCheckingTasks: true, // 初始化时假设正在检查任务，避免闪烁 pendingTaskIds: [], - nonBlockingTaskIds: [], setActiveTasks: s(set, 'activeTasks'), - setIsGlobalLocked: s(set, 'isGlobalLocked'), setIsCheckingTasks: s(set, 'isCheckingTasks'), setPendingTaskIds: s(set, 'pendingTaskIds'), - setNonBlockingTaskIds: s(set, 'nonBlockingTaskIds'), })); diff --git a/frontend/src/utils/mapExportHelpers.js b/frontend/src/utils/mapExportHelpers.js index ad9d6db..27a8801 100644 --- a/frontend/src/utils/mapExportHelpers.js +++ b/frontend/src/utils/mapExportHelpers.js @@ -54,6 +54,27 @@ const LAYER_DEFS = [ color: '#ff6b35', type: 'colorbar', label: { en: 'D-InSAR Displacement (m)', zh: 'D-InSAR 形变量 (m)' }, }, + { + id: 'sbas_rate', + ref: 'sbasAnalysisLayersRef', + detect: (layers) => layers && Object.values(layers).some(item => item?.kind === 'rate'), + color: '#1d4ed8', type: 'colorbar', + label: { en: 'SBAS LOS Velocity (mm/yr)', zh: 'SBAS LOS 速率 (mm/yr)' }, + }, + { + id: 'sbas_overview', + ref: 'sbasAnalysisLayersRef', + detect: (layers) => layers && Object.values(layers).some(item => item?.kind === 'overview'), + color: '#7c3aed', type: 'polygon', + label: { en: 'SBAS Product Footprints', zh: 'SBAS 产品范围' }, + }, + { + id: 'sbas_points', + ref: 'sbasAnalysisLayersRef', + detect: (layers) => layers && Object.values(layers).some(item => item?.kind === 'points' || item?.kind === 'query'), + color: '#16a34a', type: 'circle', + label: { en: 'SBAS Monitoring Points', zh: 'SBAS 监测点' }, + }, { id: 'water_scene', ref: 'waterSceneLayersRef', diff --git a/frontend/vite.config.js b/frontend/vite.config.js index abb4eab..d989486 100644 --- a/frontend/vite.config.js +++ b/frontend/vite.config.js @@ -71,6 +71,8 @@ export default defineConfig(({ mode }) => { '/api': { target: backendTarget, changeOrigin: true, + timeout: 300000, + proxyTimeout: 300000, } } } diff --git a/scripts/prepare_landsar_dem_int16.py b/scripts/prepare_landsar_dem_int16.py new file mode 100644 index 0000000..eda7884 --- /dev/null +++ b/scripts/prepare_landsar_dem_int16.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 +""" +Prepare reusable LandSAR DEM GeoTIFFs. + +The script converts large DEM rasters to uncompressed Int16 GeoTIFFs with a +stable nodata value. It streams data by windows, so it can process the 10 m +Heilongjiang DEM and the COPDEM China DEM without loading them into memory. +""" +from __future__ import annotations + +import argparse +import math +import os +import sys +from pathlib import Path +from typing import Iterable, Optional + +import numpy as np + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + + +def _configure_proj_lib() -> None: + candidates = [ + Path(sys.prefix) / "Library" / "share" / "proj", + Path(sys.prefix) / "lib" / "site-packages" / "rasterio" / "proj_data", + Path(sys.prefix) / "Lib" / "site-packages" / "rasterio" / "proj_data", + ] + for data_dir in candidates: + if Path(data_dir, "proj.db").is_file(): + os.environ["PROJ_LIB"] = str(data_dir) + os.environ["PROJ_DATA"] = str(data_dir) + return + + try: + import pyproj + + data_dir = pyproj.datadir.get_data_dir() + if data_dir and Path(data_dir, "proj.db").is_file(): + os.environ["PROJ_LIB"] = str(data_dir) + os.environ["PROJ_DATA"] = str(data_dir) + except Exception: + return + + +_configure_proj_lib() + + +try: + import rasterio + from rasterio.crs import CRS + from rasterio.transform import array_bounds + from rasterio.windows import Window, from_bounds +except Exception as exc: # pragma: no cover - CLI dependency guard + raise SystemExit( + "rasterio/numpy are required. Run this with the project Python, for example:\n" + r" C:\ProgramData\anaconda3\envs\InSAR\python.exe scripts\prepare_landsar_dem_int16.py" + ) from exc + + +DEFAULT_DEM_ROOT = Path(r"D:\DEM") +DEFAULT_OUTPUT_ROOT = DEFAULT_DEM_ROOT / "landsar_prepared" +DEFAULT_NODATA = -32768 +DEFAULT_CRS = CRS.from_epsg(4326) +HEILONGJIANG_10M_DEM = "HeiLongJiang10M_DEM.tif" +HEILONGJIANG_10M_ALIAS = "\u9ed1\u9f99\u6c5f\u770110M_DEM" + +SOURCE_ALIASES = { + "HeiLongJiang10M_DEM": HEILONGJIANG_10M_DEM, + "Heilongjiang10M_DEM": HEILONGJIANG_10M_DEM, + HEILONGJIANG_10M_ALIAS: HEILONGJIANG_10M_DEM, + "COPDEM_GLO30_China_4326_DEM": "COPDEM_GLO30_China_4326_DEM", +} + + +def _source_path(alias_or_path: str, dem_root: Path) -> Path: + text = str(alias_or_path or "").strip().strip('"') + if not text: + raise ValueError("source must not be empty") + + mapped = SOURCE_ALIASES.get(text, text) + candidate = Path(mapped) + if not candidate.is_absolute(): + candidate = dem_root / mapped + if candidate.exists(): + return candidate + + for suffix in (".tif", ".tiff", ".vrt", ".jp2"): + with_suffix = candidate.with_suffix(suffix) + if with_suffix.exists(): + return with_suffix + raise FileNotFoundError(f"DEM source not found: {alias_or_path} -> {candidate}") + + +def _safe_stem(alias_or_path: str, source: Path) -> str: + text = str(alias_or_path or "").strip() + if text in SOURCE_ALIASES: + return text + return source.stem or source.name + + +def _parse_bbox(value: str | None) -> Optional[tuple[float, float, float, float]]: + if not value: + return None + parts = [part.strip() for part in value.replace(";", ",").split(",") if part.strip()] + if len(parts) != 4: + raise ValueError("--bbox must be xmin,ymin,xmax,ymax") + xmin, ymin, xmax, ymax = (float(part) for part in parts) + if xmin >= xmax or ymin >= ymax: + raise ValueError("--bbox requires xmin < xmax and ymin < ymax") + return xmin, ymin, xmax, ymax + + +def _align_window(window: Window, width: int, height: int) -> Window: + col_off = max(0, int(math.floor(window.col_off))) + row_off = max(0, int(math.floor(window.row_off))) + col_stop = min(width, int(math.ceil(window.col_off + window.width))) + row_stop = min(height, int(math.ceil(window.row_off + window.height))) + if col_stop <= col_off or row_stop <= row_off: + raise ValueError("requested bbox does not overlap source raster") + return Window(col_off, row_off, col_stop - col_off, row_stop - row_off) + + +def _iter_windows(width: int, height: int, block_size: int) -> Iterable[Window]: + step = max(64, int(block_size)) + for row in range(0, height, step): + h = min(step, height - row) + for col in range(0, width, step): + w = min(step, width - col) + yield Window(col, row, w, h) + + +def _convert_array(data: np.ma.MaskedArray | np.ndarray, nodata: int) -> np.ndarray: + if isinstance(data, np.ma.MaskedArray): + mask = np.ma.getmaskarray(data) + array = np.asarray(data.filled(np.nan), dtype="float32") + else: + array = np.asarray(data, dtype="float32") + mask = np.zeros(array.shape, dtype=bool) + + invalid = mask | ~np.isfinite(array) + rounded = np.rint(array) + rounded = np.clip(rounded, nodata + 1, 32767) + out = rounded.astype("int16", copy=False) + if invalid.any(): + out = out.copy() + out[invalid] = nodata + return out + + +def _format_gib(byte_count: int) -> str: + return f"{byte_count / (1024 ** 3):.3f} GiB" + + +def convert_dem( + source_text: str, + *, + dem_root: Path, + output_root: Path, + bbox: Optional[tuple[float, float, float, float]], + suffix: str, + nodata: int, + block_size: int, + overwrite: bool, + dry_run: bool, +) -> Path: + source = _source_path(source_text, dem_root) + stem = _safe_stem(source_text, source) + if suffix: + stem = f"{stem}_{suffix.strip('_')}" + target = output_root / f"{stem}_int16.tif" + + with rasterio.open(source) as src: + src_crs = src.crs or DEFAULT_CRS + if bbox: + window = _align_window(from_bounds(*bbox, transform=src.transform), src.width, src.height) + else: + window = Window(0, 0, src.width, src.height) + window = Window(int(window.col_off), int(window.row_off), int(window.width), int(window.height)) + transform = src.window_transform(window) + bounds = array_bounds(int(window.height), int(window.width), transform) + estimated_bytes = int(window.width) * int(window.height) * np.dtype("int16").itemsize + + print(f"Source: {source}") + print(f" driver={src.driver} dtype={src.dtypes[0]} size={src.width}x{src.height} crs={src.crs or 'EPSG:4326 assumed'}") + print(f" output window={int(window.width)}x{int(window.height)} bounds={tuple(round(v, 8) for v in bounds)}") + print(f" target={target}") + print(f" estimated raw int16 size={_format_gib(estimated_bytes)}") + + if dry_run: + return target + output_root.mkdir(parents=True, exist_ok=True) + if target.exists() and not overwrite: + raise FileExistsError(f"target exists; pass --overwrite to replace it: {target}") + + profile = src.profile.copy() + profile.update( + driver="GTiff", + height=int(window.height), + width=int(window.width), + count=1, + dtype="int16", + crs=src_crs, + transform=transform, + nodata=nodata, + compress="NONE", + tiled=True, + blockxsize=512, + blockysize=512, + BIGTIFF="YES", + interleave="band", + ) + profile.pop("photometric", None) + profile.pop("predictor", None) + + if target.exists(): + target.unlink() + + with rasterio.open(target, "w", **profile) as dst: + total_pixels = int(window.width) * int(window.height) + done_pixels = 0 + last_percent = -1 + for rel_window in _iter_windows(int(window.width), int(window.height), block_size): + src_window = Window( + window.col_off + rel_window.col_off, + window.row_off + rel_window.row_off, + rel_window.width, + rel_window.height, + ) + data = src.read(1, window=src_window, masked=True) + dst.write(_convert_array(data, nodata), 1, window=rel_window) + done_pixels += int(rel_window.width) * int(rel_window.height) + percent = int(done_pixels * 100 / max(1, total_pixels)) + if percent != last_percent and (percent % 5 == 0 or percent == 100): + print(f" progress={percent}%") + last_percent = percent + + actual_size = target.stat().st_size if target.exists() else 0 + print(f"Done: {target} ({_format_gib(actual_size)})") + return target + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Convert large DEMs to reusable LandSAR Int16 GeoTIFFs." + ) + parser.add_argument( + "--source", + action="append", + default=[], + help=( + "Source alias/path. Can be repeated. Defaults to HeiLongJiang10M_DEM " + "and COPDEM_GLO30_China_4326_DEM." + ), + ) + parser.add_argument("--dem-root", default=str(DEFAULT_DEM_ROOT)) + parser.add_argument("--output-root", default=str(DEFAULT_OUTPUT_ROOT)) + parser.add_argument( + "--bbox", + default="", + help="Optional crop bounds as xmin,ymin,xmax,ymax in EPSG:4326. Omit to convert full raster.", + ) + parser.add_argument("--suffix", default="landsar") + parser.add_argument("--nodata", type=int, default=DEFAULT_NODATA) + parser.add_argument("--block-size", type=int, default=2048) + parser.add_argument("--overwrite", action="store_true") + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + sources = args.source or ["HeiLongJiang10M_DEM", "COPDEM_GLO30_China_4326_DEM"] + bbox = _parse_bbox(args.bbox) + for source in sources: + convert_dem( + source, + dem_root=Path(args.dem_root), + output_root=Path(args.output_root), + bbox=bbox, + suffix=args.suffix, + nodata=args.nodata, + block_size=args.block_size, + overwrite=args.overwrite, + dry_run=args.dry_run, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())