diff --git a/.codex_tmp/audit_lt1_dem_geometry_chain.py b/.codex_tmp/audit_lt1_dem_geometry_chain.py new file mode 100644 index 0000000..c12fe9a --- /dev/null +++ b/.codex_tmp/audit_lt1_dem_geometry_chain.py @@ -0,0 +1,403 @@ +from __future__ import annotations + +import json +import math +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np + + +MASTER_DATE = "20230726" +SLAVE_DATES = ("20230624", "20230920") +PATCH_SIZE = 512 +GAMMA_FLOAT32 = np.dtype(">f4") + + +@dataclass +class GammaImageShape: + width: int + lines: int + + +def parse_gamma_par_value(path: Path, key: str) -> str: + prefix = key.strip() + ":" + for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): + stripped = line.strip() + if stripped.startswith(prefix): + _, _, tail = stripped.partition(":") + return tail.strip().split()[0] + raise ValueError(f"Missing key '{key}' in {path}") + + +def parse_gamma_shape(path: Path) -> GammaImageShape: + width = int(float(parse_gamma_par_value(path, "range_samples"))) + lines = int(float(parse_gamma_par_value(path, "azimuth_lines"))) + return GammaImageShape(width=width, lines=lines) + + +def read_float32_image(path: Path, shape: GammaImageShape) -> np.ndarray: + arr = np.fromfile(path, dtype=GAMMA_FLOAT32) + expected = shape.width * shape.lines + if arr.size != expected: + raise ValueError(f"Unexpected size for {path}: expected {expected}, got {arr.size}") + return arr.reshape(shape.lines, shape.width) + + +def read_lt0_lookup(path: Path, shape: GammaImageShape) -> np.ndarray: + arr = np.fromfile(path, dtype=GAMMA_FLOAT32) + expected = shape.width * shape.lines * 2 + if arr.size != expected: + raise ValueError(f"Unexpected lt0 size for {path}: expected {expected}, got {arr.size}") + return arr.reshape(shape.lines, shape.width, 2) + + +def center_slice(size: int, patch: int) -> slice: + patch = min(size, patch) + start = max(0, (size - patch) // 2) + stop = start + patch + return slice(start, stop) + + +def safe_float(value: Any) -> float | None: + if value is None: + return None + try: + result = float(value) + except Exception: + return None + if math.isnan(result) or math.isinf(result): + return None + return result + + +def summarize_array(arr: np.ndarray, *, patch_size: int = PATCH_SIZE) -> dict[str, Any]: + finite = np.isfinite(arr) + zeros = finite & (arr == 0) + nonzero = finite & (arr != 0) + + ys = center_slice(arr.shape[0], patch_size) + xs = center_slice(arr.shape[1], patch_size) + patch = arr[ys, xs] + patch_finite = np.isfinite(patch) + patch_zeros = patch_finite & (patch == 0) + patch_nonzero = patch_finite & (patch != 0) + + nz = arr[nonzero] + patch_nz = patch[patch_nonzero] + stats: dict[str, Any] = { + "shape": [int(arr.shape[0]), int(arr.shape[1])], + "count": int(arr.size), + "finite_count": int(finite.sum()), + "zero_count": int(zeros.sum()), + "zero_ratio": safe_float(zeros.sum() / arr.size if arr.size else None), + "nonzero_count": int(nonzero.sum()), + "center_patch_shape": [int(patch.shape[0]), int(patch.shape[1])], + "center_patch_count": int(patch.size), + "center_patch_zero_count": int(patch_zeros.sum()), + "center_patch_zero_ratio": safe_float(patch_zeros.sum() / patch.size if patch.size else None), + "center_patch_nonzero_count": int(patch_nonzero.sum()), + } + if nz.size: + stats.update( + { + "min_nonzero": safe_float(nz.min()), + "max_nonzero": safe_float(nz.max()), + "mean_nonzero": safe_float(nz.mean()), + "std_nonzero": safe_float(nz.std()), + } + ) + if patch_nz.size: + stats.update( + { + "center_patch_min_nonzero": safe_float(patch_nz.min()), + "center_patch_max_nonzero": safe_float(patch_nz.max()), + "center_patch_mean_nonzero": safe_float(patch_nz.mean()), + "center_patch_std_nonzero": safe_float(patch_nz.std()), + } + ) + return stats + + +def summarize_lt0(arr: np.ndarray, *, patch_size: int = PATCH_SIZE) -> dict[str, Any]: + rng = arr[:, :, 0] + az = arr[:, :, 1] + finite = np.isfinite(rng) & np.isfinite(az) + zero_pair = finite & (rng == 0) & (az == 0) + valid_pair = finite & (~zero_pair) + magnitude = np.sqrt(np.square(rng, dtype=np.float64) + np.square(az, dtype=np.float64)) + + ys = center_slice(arr.shape[0], patch_size) + xs = center_slice(arr.shape[1], patch_size) + patch_valid = valid_pair[ys, xs] + patch_zero = zero_pair[ys, xs] + patch_mag = magnitude[ys, xs][patch_valid] + all_mag = magnitude[valid_pair] + + stats: dict[str, Any] = { + "shape": [int(arr.shape[0]), int(arr.shape[1]), 2], + "count": int(arr.shape[0] * arr.shape[1]), + "valid_pair_count": int(valid_pair.sum()), + "valid_pair_ratio": safe_float(valid_pair.sum() / valid_pair.size if valid_pair.size else None), + "zero_pair_count": int(zero_pair.sum()), + "zero_pair_ratio": safe_float(zero_pair.sum() / zero_pair.size if zero_pair.size else None), + "center_patch_shape": [int(patch_valid.shape[0]), int(patch_valid.shape[1])], + "center_patch_valid_pair_count": int(patch_valid.sum()), + "center_patch_valid_pair_ratio": safe_float(patch_valid.sum() / patch_valid.size if patch_valid.size else None), + "center_patch_zero_pair_count": int(patch_zero.sum()), + "center_patch_zero_pair_ratio": safe_float(patch_zero.sum() / patch_zero.size if patch_zero.size else None), + } + if all_mag.size: + stats.update( + { + "magnitude_min": safe_float(all_mag.min()), + "magnitude_max": safe_float(all_mag.max()), + "magnitude_mean": safe_float(all_mag.mean()), + "magnitude_std": safe_float(all_mag.std()), + } + ) + if patch_mag.size: + stats.update( + { + "center_patch_magnitude_min": safe_float(patch_mag.min()), + "center_patch_magnitude_max": safe_float(patch_mag.max()), + "center_patch_magnitude_mean": safe_float(patch_mag.mean()), + "center_patch_magnitude_std": safe_float(patch_mag.std()), + } + ) + return stats + + +def summarize_overlap(a: np.ndarray, b: np.ndarray, *, patch_size: int = PATCH_SIZE) -> dict[str, Any]: + if a.shape != b.shape: + raise ValueError(f"Shape mismatch for overlap: {a.shape} vs {b.shape}") + finite_a = np.isfinite(a) + finite_b = np.isfinite(b) + nz_a = finite_a & (a != 0) + nz_b = finite_b & (b != 0) + overlap = nz_a & nz_b + + ys = center_slice(a.shape[0], patch_size) + xs = center_slice(a.shape[1], patch_size) + patch_overlap = overlap[ys, xs] + patch_nz_a = nz_a[ys, xs] + patch_nz_b = nz_b[ys, xs] + + return { + "shape": [int(a.shape[0]), int(a.shape[1])], + "overlap_nonzero_count": int(overlap.sum()), + "overlap_nonzero_ratio": safe_float(overlap.sum() / overlap.size if overlap.size else None), + "a_nonzero_count": int(nz_a.sum()), + "b_nonzero_count": int(nz_b.sum()), + "center_patch_overlap_nonzero_count": int(patch_overlap.sum()), + "center_patch_overlap_nonzero_ratio": safe_float(patch_overlap.sum() / patch_overlap.size if patch_overlap.size else None), + "center_patch_a_nonzero_count": int(patch_nz_a.sum()), + "center_patch_b_nonzero_count": int(patch_nz_b.sum()), + } + + +def scale_to_u8(arr: np.ndarray) -> np.ndarray: + finite = np.isfinite(arr) + valid = arr[finite & (arr != 0)] + if valid.size == 0: + return np.zeros(arr.shape, dtype=np.uint8) + lo = np.percentile(valid, 1) + hi = np.percentile(valid, 99) + if not np.isfinite(lo) or not np.isfinite(hi) or hi <= lo: + lo = float(valid.min()) + hi = float(valid.max()) if valid.size else lo + 1.0 + if hi <= lo: + hi = lo + 1.0 + scaled = np.clip((arr - lo) / (hi - lo), 0, 1) + scaled[~finite] = 0 + scaled[arr == 0] = 0 + return np.round(scaled * 255.0).astype(np.uint8) + + +def write_pgm(path: Path, arr_u8: np.ndarray) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + header = f"P5\n{arr_u8.shape[1]} {arr_u8.shape[0]}\n255\n".encode("ascii") + with path.open("wb") as fp: + fp.write(header) + fp.write(arr_u8.tobytes()) + + +def write_quicklook(path: Path, arr: np.ndarray) -> None: + write_pgm(path, scale_to_u8(arr)) + + +def case_root(run_root: Path, case_name: str) -> Path: + return run_root / case_name / "pyint_stage" + + +def audit_case(run_root: Path, case_name: str, out_root: Path) -> dict[str, Any]: + case_dir = case_root(run_root, case_name) + dem_dir = case_dir / "DEM" + out_case_root = out_root / case_name + out_case_root.mkdir(parents=True, exist_ok=True) + + master_shape = parse_gamma_shape(dem_dir / f"{MASTER_DATE}_2rlks.amp.par") + hgtsim = read_float32_image(dem_dir / f"{MASTER_DATE}_2rlks.rdc.dem", master_shape) + lt0_quicklook_written = False + + result: dict[str, Any] = { + "case": case_name, + "master_date": MASTER_DATE, + "master_shape": {"width": master_shape.width, "lines": master_shape.lines}, + "dem": { + "hgtsim": summarize_array(hgtsim), + }, + "slaves": {}, + } + + write_quicklook(out_case_root / "hgtsim.pgm", hgtsim) + + for slave_date in SLAVE_DATES: + slc_dir = case_dir / "SLC" / slave_date + rslc_dir = case_dir / "RSLC" / slave_date + slave_shape = parse_gamma_shape(slc_dir / f"{slave_date}_2rlks.amp.par") + samp = read_float32_image(slc_dir / f"{slave_date}_2rlks.amp", slave_shape) + mli0 = read_float32_image(rslc_dir / "mli0", slave_shape) + lt0 = read_lt0_lookup(rslc_dir / "lt0", master_shape) + + lt0_mag = np.sqrt(np.square(lt0[:, :, 0], dtype=np.float64) + np.square(lt0[:, :, 1], dtype=np.float64)) + + slave_out = out_case_root / slave_date + slave_out.mkdir(parents=True, exist_ok=True) + write_quicklook(slave_out / "samp.pgm", samp) + write_quicklook(slave_out / "mli0.pgm", mli0) + if not lt0_quicklook_written: + write_quicklook(out_case_root / "lt0_magnitude.pgm", lt0_mag.astype(np.float32)) + lt0_quicklook_written = True + + slave_summary = { + "shape": {"width": slave_shape.width, "lines": slave_shape.lines}, + "samp": summarize_array(samp), + "mli0": summarize_array(mli0), + "lt0": summarize_lt0(lt0), + "mli0_samp_overlap": summarize_overlap(mli0, samp), + } + result["slaves"][slave_date] = slave_summary + + (slave_out / "summary.json").write_text( + json.dumps(slave_summary, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + + (out_case_root / "summary.json").write_text( + json.dumps(result, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + return result + + +def build_summary_rows(result: dict[str, Any]) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + rows.append( + { + "case": result["case"], + "date": result["master_date"], + "artifact": "hgtsim", + "zero_ratio": result["dem"]["hgtsim"].get("zero_ratio"), + "center_patch_zero_ratio": result["dem"]["hgtsim"].get("center_patch_zero_ratio"), + "overlap_ratio": None, + "center_patch_overlap_ratio": None, + } + ) + for slave_date, payload in result["slaves"].items(): + for artifact in ("samp", "mli0"): + stats = payload[artifact] + rows.append( + { + "case": result["case"], + "date": slave_date, + "artifact": artifact, + "zero_ratio": stats.get("zero_ratio"), + "center_patch_zero_ratio": stats.get("center_patch_zero_ratio"), + "overlap_ratio": None, + "center_patch_overlap_ratio": None, + } + ) + lt0 = payload["lt0"] + rows.append( + { + "case": result["case"], + "date": slave_date, + "artifact": "lt0", + "zero_ratio": lt0.get("zero_pair_ratio"), + "center_patch_zero_ratio": lt0.get("center_patch_zero_pair_ratio"), + "overlap_ratio": lt0.get("valid_pair_ratio"), + "center_patch_overlap_ratio": lt0.get("center_patch_valid_pair_ratio"), + } + ) + overlap = payload["mli0_samp_overlap"] + rows.append( + { + "case": result["case"], + "date": slave_date, + "artifact": "mli0_samp_overlap", + "zero_ratio": None, + "center_patch_zero_ratio": None, + "overlap_ratio": overlap.get("overlap_nonzero_ratio"), + "center_patch_overlap_ratio": overlap.get("center_patch_overlap_nonzero_ratio"), + } + ) + return rows + + +def write_summary_tsv(path: Path, rows: list[dict[str, Any]]) -> None: + header = [ + "case", + "date", + "artifact", + "zero_ratio", + "center_patch_zero_ratio", + "overlap_ratio", + "center_patch_overlap_ratio", + ] + lines = ["\t".join(header)] + for row in rows: + values = [] + for key in header: + value = row.get(key) + if isinstance(value, float): + values.append(f"{value:.6f}") + elif value is None: + values.append("") + else: + values.append(str(value)) + lines.append("\t".join(values)) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def main() -> int: + if len(sys.argv) < 2: + print("usage: audit_lt1_dem_geometry_chain.py [case ...]", file=sys.stderr) + return 2 + + run_root = Path(sys.argv[1]).resolve() + case_names = tuple(sys.argv[2:]) if len(sys.argv) > 2 else ("case_A_baseline", "case_C_precise_orbit_rewrite") + out_root = run_root / "audit_dem_geometry" + out_root.mkdir(parents=True, exist_ok=True) + + all_results = [] + all_rows: list[dict[str, Any]] = [] + for case_name in case_names: + result = audit_case(run_root, case_name, out_root) + all_results.append(result) + all_rows.extend(build_summary_rows(result)) + + (out_root / "audit_summary.json").write_text( + json.dumps({"run_root": str(run_root), "results": all_results}, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + write_summary_tsv(out_root / "audit_summary.tsv", all_rows) + + print(json.dumps({"run_root": str(run_root), "output_dir": str(out_root), "case_count": len(case_names)}, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.codex_tmp/audit_lt1_pool_multiscene_root.py b/.codex_tmp/audit_lt1_pool_multiscene_root.py new file mode 100644 index 0000000..741e637 --- /dev/null +++ b/.codex_tmp/audit_lt1_pool_multiscene_root.py @@ -0,0 +1,204 @@ +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path +from typing import Any + + +def load_base_module(script_path: Path): + spec = importlib.util.spec_from_file_location("audit_lt1_dem_geometry_chain_base", script_path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load base audit script: {script_path}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def build_summary_rows(result: dict[str, Any]) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + rows.append( + { + "root": result["root"], + "date": result["master_date"], + "artifact": "hgtsim", + "zero_ratio": result["dem"]["hgtsim"].get("zero_ratio"), + "center_patch_zero_ratio": result["dem"]["hgtsim"].get("center_patch_zero_ratio"), + "overlap_ratio": None, + "center_patch_overlap_ratio": None, + } + ) + for slave_date, payload in result["slaves"].items(): + for artifact in ("samp", "mli0"): + stats = payload[artifact] + rows.append( + { + "root": result["root"], + "date": slave_date, + "artifact": artifact, + "zero_ratio": stats.get("zero_ratio"), + "center_patch_zero_ratio": stats.get("center_patch_zero_ratio"), + "overlap_ratio": None, + "center_patch_overlap_ratio": None, + } + ) + lt0 = payload["lt0"] + rows.append( + { + "root": result["root"], + "date": slave_date, + "artifact": "lt0", + "zero_ratio": lt0.get("zero_pair_ratio"), + "center_patch_zero_ratio": lt0.get("center_patch_zero_pair_ratio"), + "overlap_ratio": lt0.get("valid_pair_ratio"), + "center_patch_overlap_ratio": lt0.get("center_patch_valid_pair_ratio"), + } + ) + overlap = payload["mli0_samp_overlap"] + rows.append( + { + "root": result["root"], + "date": slave_date, + "artifact": "mli0_samp_overlap", + "zero_ratio": None, + "center_patch_zero_ratio": None, + "overlap_ratio": overlap.get("overlap_nonzero_ratio"), + "center_patch_overlap_ratio": overlap.get("center_patch_overlap_nonzero_ratio"), + } + ) + return rows + + +def write_summary_tsv(path: Path, rows: list[dict[str, Any]]) -> None: + header = [ + "root", + "date", + "artifact", + "zero_ratio", + "center_patch_zero_ratio", + "overlap_ratio", + "center_patch_overlap_ratio", + ] + lines = ["\t".join(header)] + for row in rows: + values = [] + for key in header: + value = row.get(key) + if isinstance(value, float): + values.append(f"{value:.6f}") + elif value is None: + values.append("") + else: + values.append(str(value)) + lines.append("\t".join(values)) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def audit_root(run_root: Path, master_date: str, slave_dates: tuple[str, ...], project: str) -> dict[str, Any]: + base_script = Path(__file__).with_name("audit_lt1_dem_geometry_chain.py") + base = load_base_module(base_script) + + project_root = run_root / project + dem_dir = project_root / "DEM" + out_root = run_root / "audit_dem_geometry" + out_root.mkdir(parents=True, exist_ok=True) + + master_shape = base.parse_gamma_shape(dem_dir / f"{master_date}_2rlks.amp.par") + hgtsim = base.read_float32_image(dem_dir / f"{master_date}_2rlks.rdc.dem", master_shape) + + result: dict[str, Any] = { + "root": str(run_root), + "project": project, + "master_date": master_date, + "master_shape": {"width": master_shape.width, "lines": master_shape.lines}, + "dem": { + "hgtsim": base.summarize_array(hgtsim), + }, + "slaves": {}, + } + + base.write_quicklook(out_root / "hgtsim.pgm", hgtsim) + lt0_quicklook_written = False + + for slave_date in slave_dates: + slc_dir = project_root / "SLC" / slave_date + rslc_dir = project_root / "RSLC" / slave_date + slave_shape = base.parse_gamma_shape(slc_dir / f"{slave_date}_2rlks.amp.par") + samp = base.read_float32_image(slc_dir / f"{slave_date}_2rlks.amp", slave_shape) + mli0 = base.read_float32_image(rslc_dir / "mli0", slave_shape) + lt0 = base.read_lt0_lookup(rslc_dir / "lt0", master_shape) + + lt0_mag = (lt0[:, :, 0].astype("float64") ** 2 + lt0[:, :, 1].astype("float64") ** 2) ** 0.5 + + slave_out = out_root / slave_date + slave_out.mkdir(parents=True, exist_ok=True) + base.write_quicklook(slave_out / "samp.pgm", samp) + base.write_quicklook(slave_out / "mli0.pgm", mli0) + if not lt0_quicklook_written: + base.write_quicklook(out_root / "lt0_magnitude.pgm", lt0_mag.astype("float32")) + lt0_quicklook_written = True + + slave_summary = { + "shape": {"width": slave_shape.width, "lines": slave_shape.lines}, + "samp": base.summarize_array(samp), + "mli0": base.summarize_array(mli0), + "lt0": base.summarize_lt0(lt0), + "mli0_samp_overlap": base.summarize_overlap(mli0, samp), + } + result["slaves"][slave_date] = slave_summary + (slave_out / "summary.json").write_text( + json.dumps(slave_summary, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + + (out_root / "summary.json").write_text( + json.dumps(result, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + rows = build_summary_rows(result) + write_summary_tsv(out_root / "audit_summary.tsv", rows) + return result + + +def main() -> int: + if len(sys.argv) < 4: + print( + "usage: audit_lt1_pool_multiscene_root.py [slave_date ...] [--project name]", + file=sys.stderr, + ) + return 2 + + args = list(sys.argv[1:]) + project = "pyint_stage" + if "--project" in args: + idx = args.index("--project") + try: + project = args[idx + 1] + except IndexError as exc: + raise SystemExit("--project requires a value") from exc + del args[idx : idx + 2] + + run_root = Path(args[0]).resolve() + master_date = args[1] + slave_dates = tuple(args[2:]) + + result = audit_root(run_root, master_date, slave_dates, project) + print( + json.dumps( + { + "run_root": str(run_root), + "output_dir": str(run_root / "audit_dem_geometry"), + "master_date": master_date, + "slave_count": len(slave_dates), + "project": project, + }, + ensure_ascii=False, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.codex_tmp/compare_lt1_import_paths.sh b/.codex_tmp/compare_lt1_import_paths.sh new file mode 100644 index 0000000..00ece1f --- /dev/null +++ b/.codex_tmp/compare_lt1_import_paths.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ $# -lt 1 ]; then + echo "usage: $0 YYYYMMDD" >&2 + exit 1 +fi + +DATE_TEXT="$1" +ROOT='/mnt/d/PyINT_POOL_TEST/LT1A_MONO_SYC_STRIP1_E129.6_N45.0_3scene' +PROJECT_DIR="$ROOT/pyint_stage" +DOWNLOAD_DIR="$PROJECT_DIR/DOWNLOAD" +CURRENT_SLC_DIR="$PROJECT_DIR/SLC/$DATE_TEXT" +COMPARE_ROOT="$ROOT/import_compare/$DATE_TEXT" +LIST_FILE="$COMPARE_ROOT/t_$DATE_TEXT" +CURRENT_PAR="$CURRENT_SLC_DIR/$DATE_TEXT.slc.par" +GAMMA_ENV='/mnt/d/Code/Insar_management_system_v2/backend/app/pyint_pipeline/pyint_gamma_env.sh' + +die() { + echo "$1" >&2 + exit 1 +} + +[ -f "$CURRENT_PAR" ] || die "Current .slc.par not found: $CURRENT_PAR" +[ -f "$GAMMA_ENV" ] || die "Gamma env script not found: $GAMMA_ENV" + +SCENE_TIFF="$(find "$DOWNLOAD_DIR" -maxdepth 1 -type f -name "LT1*${DATE_TEXT}*.tiff" | sort | head -n 1)" +[ -n "$SCENE_TIFF" ] || die "Scene TIFF not found for date: $DATE_TEXT" + +. "$GAMMA_ENV" >/dev/null 2>&1 +mkdir -p "$COMPARE_ROOT" +printf '%s\n' "$SCENE_TIFF" >"$LIST_FILE" + +pushd "$COMPARE_ROOT" >/dev/null +SCENE_XML="${SCENE_TIFF%.tiff}.meta.xml" +[ -f "$SCENE_XML" ] || die "Scene meta xml not found: $SCENE_XML" +par_LT1_SLC "$SCENE_TIFF" "$SCENE_XML" "./$DATE_TEXT.slc.par" "./$DATE_TEXT.slc" > import.stdout.log 2> import.stderr.log +par_LT1_SLC_YSLi "$SCENE_TIFF" "$SCENE_XML" "./$DATE_TEXT.slc.update" "./$DATE_TEXT.slc.update.par" 0 >> import.stdout.log 2>> import.stderr.log +popd >/dev/null + +ALT_PAR="$COMPARE_ROOT/$DATE_TEXT.slc.par" +[ -f "$ALT_PAR" ] || die "Alternate .slc.par not found: $ALT_PAR" + +python3 - "$ALT_PAR" "$COMPARE_ROOT/$DATE_TEXT.slc.update.par" <<'PY' +import sys +from pathlib import Path + +main_path = Path(sys.argv[1]) +update_path = Path(sys.argv[2]) + +main_lines = main_path.read_text(encoding="utf-8", errors="ignore").splitlines() +update_lines = update_path.read_text(encoding="utf-8", errors="ignore").splitlines() + +prefixes = ("number_of_state_vectors:", "time_of_first_state_vector:", "state_vector_interval:", "state_vector_position_", "state_vector_velocity_") +main_prefix = ("number_of_state_vectors:", "time_of_first_state_vector:", "state_vector_interval:", "state_vector_position_", "state_vector_velocity_") + +filtered_main = [line for line in main_lines if not line.startswith(main_prefix)] +replacement = [line for line in update_lines if line.startswith(prefixes)] + +main_path.write_text("\n".join(filtered_main + replacement) + "\n", encoding="utf-8") +PY + +python3 - "$CURRENT_PAR" "$ALT_PAR" <<'PY' +import re +import sys +from pathlib import Path + +keys = [ + "center_latitude", + "center_longitude", + "start_time", + "center_time", + "end_time", + "azimuth_line_time", + "range_samples", + "azimuth_lines", + "range_pixel_spacing", + "azimuth_pixel_spacing", + "near_range_slc", + "center_range_slc", + "far_range_slc", + "incidence_angle", + "heading", + "prf", + "azimuth_proc_bandwidth", + "doppler_polynomial", + "number_of_state_vectors", + "time_of_first_state_vector", + "state_vector_interval", +] + +vector_patterns = [ + "state_vector_position_1", + "state_vector_velocity_1", + "state_vector_position_2", + "state_vector_velocity_2", + "state_vector_position_3", + "state_vector_velocity_3", +] + +def read_map(path: Path): + data = {} + for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): + for key in keys + vector_patterns: + if line.startswith(key + ":"): + data[key] = line.split(":", 1)[1].strip() + break + return data + +cur = read_map(Path(sys.argv[1])) +alt = read_map(Path(sys.argv[2])) +all_keys = keys + vector_patterns +for key in all_keys: + cur_v = cur.get(key, "") + alt_v = alt.get(key, "") + if cur_v != alt_v: + print(f"{key}\n current: {cur_v}\n alt: {alt_v}") +PY diff --git a/.codex_tmp/orbit_smoke/source/LT1A_GpsData_GAS_C_20250622.txt b/.codex_tmp/orbit_smoke/source/LT1A_GpsData_GAS_C_20250622.txt new file mode 100644 index 0000000..d2c1965 Binary files /dev/null and b/.codex_tmp/orbit_smoke/source/LT1A_GpsData_GAS_C_20250622.txt differ diff --git a/.codex_tmp/orbit_smoke/source/LT1B_GpsData_GAS_C_20250623.txt b/.codex_tmp/orbit_smoke/source/LT1B_GpsData_GAS_C_20250623.txt new file mode 100644 index 0000000..a0d0e95 --- /dev/null +++ b/.codex_tmp/orbit_smoke/source/LT1B_GpsData_GAS_C_20250623.txt @@ -0,0 +1 @@ +2025 6 23 13 34 55.000 -1307508.7503 -651238.6465 -6835090.2734 2779.6665514 6982.8147436 -12.1250000 diff --git a/.codex_tmp/orbit_smoke_41817fae19fd4434b5e63edb31a01910/source/LT1A_GpsData_GAS_C_20250622.txt b/.codex_tmp/orbit_smoke_41817fae19fd4434b5e63edb31a01910/source/LT1A_GpsData_GAS_C_20250622.txt new file mode 100644 index 0000000..d2c1965 Binary files /dev/null and b/.codex_tmp/orbit_smoke_41817fae19fd4434b5e63edb31a01910/source/LT1A_GpsData_GAS_C_20250622.txt differ diff --git a/.codex_tmp/orbit_smoke_41817fae19fd4434b5e63edb31a01910/source/LT1B_GpsData_GAS_C_20250623.txt b/.codex_tmp/orbit_smoke_41817fae19fd4434b5e63edb31a01910/source/LT1B_GpsData_GAS_C_20250623.txt new file mode 100644 index 0000000..a0d0e95 --- /dev/null +++ b/.codex_tmp/orbit_smoke_41817fae19fd4434b5e63edb31a01910/source/LT1B_GpsData_GAS_C_20250623.txt @@ -0,0 +1 @@ +2025 6 23 13 34 55.000 -1307508.7503 -651238.6465 -6835090.2734 2779.6665514 6982.8147436 -12.1250000 diff --git a/.codex_tmp/orbit_smoke_58525e754384446ca1e016affd4cd340/source/LT1A_GpsData_GAS_C_20250622.txt b/.codex_tmp/orbit_smoke_58525e754384446ca1e016affd4cd340/source/LT1A_GpsData_GAS_C_20250622.txt new file mode 100644 index 0000000..d2c1965 Binary files /dev/null and b/.codex_tmp/orbit_smoke_58525e754384446ca1e016affd4cd340/source/LT1A_GpsData_GAS_C_20250622.txt differ diff --git a/.codex_tmp/orbit_smoke_58525e754384446ca1e016affd4cd340/source/LT1B_GpsData_GAS_C_20250623.txt b/.codex_tmp/orbit_smoke_58525e754384446ca1e016affd4cd340/source/LT1B_GpsData_GAS_C_20250623.txt new file mode 100644 index 0000000..a0d0e95 --- /dev/null +++ b/.codex_tmp/orbit_smoke_58525e754384446ca1e016affd4cd340/source/LT1B_GpsData_GAS_C_20250623.txt @@ -0,0 +1 @@ +2025 6 23 13 34 55.000 -1307508.7503 -651238.6465 -6835090.2734 2779.6665514 6982.8147436 -12.1250000 diff --git a/.codex_tmp/pyint_variants/no_rescue/LICENSE b/.codex_tmp/pyint_variants/no_rescue/LICENSE new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/.codex_tmp/pyint_variants/no_rescue/README.md b/.codex_tmp/pyint_variants/no_rescue/README.md new file mode 100644 index 0000000..440ac43 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/README.md @@ -0,0 +1,99 @@ +# PyINT +[![Language](https://img.shields.io/badge/python-3.5%2B-blue.svg)](https://www.python.org/) +[![License](https://img.shields.io/badge/license-GPL-yellow.svg)](https://github.com/ymcmrs/PyINT/blob/master/LICENSE) + +### Single or time-series of interferograms processing based on python and GAMMA for all of the present SAR datasets. + +PYthon-based INterferometry Toolbox (PyINT) is an open-source package for single or time-series of interferograms processing from downloading data (or SLC) to generating differential-unwrapped interferograms by using GAMMA software. You can process in a routine way (e.g., raw2ifg.py, slc2ifg.py or pyintApp.py) or process step by step. There are many GAMMA-independent tools of PyINT could be useful for you no matter you use GAMMA or other interferometry softwares (e.g., ISCE, SNAP). Advantages include (but not limited to) download and update precise-orbit data automatically (support S1, ERS, ASAR), download and process 30m-SRTM dem automatically, cat multi-frames automatically, select swaths and bursts flexibly (for S1), extract the related S1 butsts for Coregistration automatically, etc. Welcome to contribute/improve PyINT. + + +### 1 Download + +Download the development version using git: + + cd ~/python + git clone https://github.com/ymcmrs/PyINT + + +### 2 Installation + + 1） To make pyint importable in python, by adding the path PyINT directory to your $PYTHONPATH + For csh/tcsh user, add to your **_~/.cshrc_** file for example: + + ############################ Python ############################### + if ( ! $?PYTHONPATH ) then + setenv PYTHONPATH "" + endif + + ##--------- Anaconda ---------------## + setenv PYTHON3DIR ~/python/anaconda3 + setenv PATH ${PATH}:${PYTHON3DIR}/bin + + ##--------- PyINT ------------------## + setenv PYINT_HOME ~/python/PyINT + setenv PYTHONPATH ${PYTHONPATH}:${PYINT_HOME} + setenv PATH ${PATH}:${PYINT_HOME}/pyint + + 2） Install gdal, elevation module using pip or conda for DEM processing. + + 3） Install [SSARA](https://github.com/bakerunavco/SSARA) and set account info for downloading data. [option] + + +### 3 Running PyINT + +1). $SCRATCHDIR and $TEMPLATEDIR should be available in your system environment. $SCRATCHDIR for processing, $TEMPLATEDIR for template files to set the related processing parameters, $DEMDIR for saving DEMs: + + setenv SCRATCHDIR /Users/Yunmeng/Documents/SCRATCH + setenv TEMPLATEDIR /Users/Yunmeng/Documents/development/TEMPLATEDIR + setenv DEMDIR /Users/Yunmeng/Documents/SCRATCH/DEM + +2). Preparing your template file, which should be saved in $TEMPLATEDIR, for setting some basic parameters (see the template file above).The template file should be named with a prefix of your project name: + + e.g., MexicoCityT143F529S1D.template [Region + Track + Frame + Satellite + Orbit] + + +3). Single interferogram processing: + + slc2ifg.py projectName Mdate Sdate # start from SLC to unwrapped-differential Ifg + raw2ifg.py projectName Mdate Sdate # start from raw data to unwrapped-differential Ifg + + e.g. : + slc2ifg.py HawaiiT87F526S1D 20150101 20160201 + raw2ifg.py HawaiiT87F526S1D 20150101 20160201 + +4). Time-series of interferograms processing. + + pyintApp.py projectName + + e.g. : + pyintApp.py MexicoCityT143F529S1D # template file MexicoCityT143F529S1D.template should be availabe in TEMPLATEDIR + + General work-flow: + + 1) download data : download SLCs using SSARA (please check https://github.com/bakerunavco/SSARA) + [You should provide Sensor, Track, Frame, or Time information in template] + 2) generate SLC : raw 2 slc (multi-frame processing is also supported) + [include orbit correction for S1,ASAR,ERS and burst-extraction for S1] + 3) generate DEM : reference image related geo-dem, rdc-dem, lookup table will be generated. + [SRTM-1 will be downloaded and processed automatically if not provided] + 4) coregister SLC : coregister SLCs to the reference SLC iamge. + [with assistant of DEM] + 5) select pairs : select interferometric pairs for time-series processing. + [networks of sbas, sequential, delaunay, and stars are supported] + 6) interferometry : generate unwrapped differential interferograms. + [include differential, unwrapping, and geocoding] + 7) load data : loading data for time-series analysis, mintPy is supported presently. + + Note: + + i ) Single interferogram processing please use slc2ifg.py or raw2ifg.py + ii ) Multi-processor parallel processing is supported, but keep in mind GAMMA calls multi-threads already. + iii) You can using pyintApp.py from downloading data to generate time-series of unwrapped-differential Ifgs, + or you also can process step by step. + + +Note: All of the above codes are based on the hypothesis that you have installed [GAMMA](https://www.gamma-rs.ch/). + +### 4 Citing this work + + Y.M., Cao, "PyINT: Python&GAMMA based interferometry toolbox", Remote Sensing Code Library, doi:10.21982/vd48-7p51, April, 2019. diff --git a/.codex_tmp/pyint_variants/no_rescue/VENDORED_FROM.md b/.codex_tmp/pyint_variants/no_rescue/VENDORED_FROM.md new file mode 100644 index 0000000..a8c3232 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/VENDORED_FROM.md @@ -0,0 +1,12 @@ +# Vendored PyINT + +- Source copied from local path: `D:\Code\PyINT` +- Vendored into this repository on: `2026-04-19` +- Scope kept in repo: `pyint/`, `template/`, `README.md`, `LICENSE` +- Excluded from vendored copy: `pyint_bk/`, `pyint.zip`, IDE metadata, `__pycache__` + +Notes: + +- This vendored copy is used by the local D-InSAR `PyINT / Gamma` integration. +- Runtime defaults now resolve `PYINT_HOME` to this directory when `.env` does not override it. +- If upstream `PyINT` is updated, sync this directory intentionally and record the source revision or date here. diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/20210110.slc.par b/.codex_tmp/pyint_variants/no_rescue/pyint/20210110.slc.par new file mode 100644 index 0000000..d571806 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/20210110.slc.par @@ -0,0 +1,81 @@ +Gamma Interferometric SAR Processor (ISP) - Image Parameter File + +title: s1a-iw1-slc-vv-20210110t110734-20210110t110759-036075-043a78-004.tiff S1A-IW-IW1-VV-36075 (software: Sentinel-1 IPF 003.31) +sensor: S1A IW IW1 VV +date: 2021 01 10 11 07 33.60437 +start_time: 40053.604372 s +center_time: 40067.080599 s +end_time: 40080.556826 s +azimuth_line_time: 2.05555630e-03 s +line_header_size: 0 +range_samples: 68467 +azimuth_lines: 13113 +range_looks: 1 +azimuth_looks: 1 +image_format: FCOMPLEX +image_geometry: SLANT_RANGE +polarization: VV +range_scale_factor: 1.0000000e+00 +azimuth_scale_factor: 1.0000000e+00 +center_latitude: 25.7907364 degrees +center_longitude: 104.4737803 degrees +heading: -12.5014428 degrees +range_pixel_spacing: 2.3295620 m +azimuth_pixel_spacing: 13.9946500 m +near_range_slc: 798980.1369 m +center_range_slc: 878728.0328 m +far_range_slc: 958475.9288 m +first_slant_range_polynomial: 0.00000 0.00000 0.00000e+00 0.00000e+00 0.00000e+00 0.00000e+00 s m 1 m^-1 m^-2 m^-3 +center_slant_range_polynomial: 0.00000 0.00000 0.00000e+00 0.00000e+00 0.00000e+00 0.00000e+00 s m 1 m^-1 m^-2 m^-3 +last_slant_range_polynomial: 0.00000 0.00000 0.00000e+00 0.00000e+00 0.00000e+00 0.00000e+00 s m 1 m^-1 m^-2 m^-3 +incidence_angle: 39.6640 degrees +azimuth_deskew: ON +azimuth_angle: 90.0000 degrees +radar_frequency: 5.4050005e+09 Hz +adc_sampling_rate: 6.4345241e+07 Hz +chirp_bandwidth: 5.6500000e+07 Hz +prf: 486.4863103 Hz +azimuth_proc_bandwidth: 327.00000 Hz +doppler_polynomial: 17.39054 -2.62817e-05 -8.00473e-11 0.00000e+00 Hz Hz/m Hz/m^2 Hz/m^3 +doppler_poly_dot: 0.00000e+00 0.00000e+00 0.00000e+00 0.00000e+00 Hz/s Hz/s/m Hz/s/m^2 Hz/s/m^3 +doppler_poly_ddot: 0.00000e+00 0.00000e+00 0.00000e+00 0.00000e+00 Hz/s^2 Hz/s^2/m Hz/s^2/m^2 Hz/s^2/m^3 +receiver_gain: 0.0000 dB +calibration_gain: 0.0000 dB +sar_to_earth_center: 7072935.2710 m +earth_radius_below_sensor: 6374216.3707 m +earth_semi_major_axis: 6378137.0000 m +earth_semi_minor_axis: 6356752.3141 m +number_of_state_vectors: 15 +time_of_first_state_vector: 40002.000000 s +state_vector_interval: 10.000000 s +state_vector_position_1: -1202502.7493 6500810.8061 2515842.3001 m m m +state_vector_velocity_1: 2079.47309 -2309.53890 6932.46052 m/s m/s m/s +state_vector_position_2: -1181658.0158 6477336.1615 2585023.4928 m m m +state_vector_velocity_2: 2089.41624 -2385.34851 6903.64775 m/s m/s m/s +state_vector_position_3: -1160715.5736 6453104.6806 2653912.6562 m m m +state_vector_velocity_3: 2099.01470 -2460.90464 6874.05521 m/s m/s m/s +state_vector_position_4: -1139678.8732 6428118.9436 2722502.0093 m m m +state_vector_velocity_4: 2108.26777 -2536.19825 6843.68628 m/s m/s m/s +state_vector_position_5: -1118551.3718 6402381.6207 2790783.8056 m m m +state_vector_velocity_5: 2117.17479 -2611.22032 6812.54442 m/s m/s m/s +state_vector_position_6: -1097336.5330 6375895.4722 2858750.3336 m m m +state_vector_velocity_6: 2125.73515 -2685.96187 6780.63319 m/s m/s m/s +state_vector_position_7: -1076037.8261 6348663.3481 2926393.9175 m m m +state_vector_velocity_7: 2133.94831 -2760.41395 6747.95623 m/s m/s m/s +state_vector_position_8: -1054658.7259 6320688.1875 2993706.9187 m m m +state_vector_velocity_8: 2141.81376 -2834.56766 6714.51728 m/s m/s m/s +state_vector_position_9: -1033202.7115 6291973.0188 3060681.7361 m m m +state_vector_velocity_9: 2149.33106 -2908.41412 6680.32013 m/s m/s m/s +state_vector_position_10: -1011673.2665 6262520.9586 3127310.8074 m m m +state_vector_velocity_10: 2156.49982 -2981.94447 6645.36870 m/s m/s m/s +state_vector_position_11: -990073.8782 6232335.2122 3193586.6096 m m m +state_vector_velocity_11: 2163.31969 -3055.14991 6609.66698 m/s m/s m/s +state_vector_position_12: -968408.0367 6201419.0725 3259501.6599 m m m +state_vector_velocity_12: 2169.79039 -3128.02167 6573.21903 m/s m/s m/s +state_vector_position_13: -946679.2350 6169775.9199 3325048.5169 m m m +state_vector_velocity_13: 2175.91169 -3200.55102 6536.02901 m/s m/s m/s +state_vector_position_14: -924890.9683 6137409.2222 3390219.7811 m m m +state_vector_velocity_14: 2181.68339 -3272.72926 6498.10116 m/s m/s m/s +state_vector_position_15: -903046.7331 6104322.5337 3455008.0956 m m m +state_vector_velocity_15: 2187.10536 -3344.54774 6459.43982 m/s m/s m/s + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/API_download_S1_SLC.py b/.codex_tmp/pyint_variants/no_rescue/pyint/API_download_S1_SLC.py new file mode 100644 index 0000000..9eb046a --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/API_download_S1_SLC.py @@ -0,0 +1,494 @@ +#! /usr/bin/env python3 +# -*- coding: utf-8 -*- + +########################################################################### +# Header information +########################################################################### + +"""API_download_S1_SLC.py: Script to download Sentinel-1 SLC images from the ASF mirror + Supports both bounding box and Shapefile for spatial filtering""" + +__author__ = "Alexis Hrysiewicz" +__copyright__ = "Copyright 2022" +__credits__ = ["Alexis Hrysiewicz"] +__license__ = "GPL" +__version__ = "2.0.0" +__maintainer__ = "Alexis Hrysiewicz" +__status__ = "Production" +__date__ = "Jan. 2022" + +########################################################################### +# Python packages +########################################################################### + +import os +import sys +import pandas as pd +import datetime +import os.path +import optparse + +# 尝试导入 geopandas 和 shapely 用于 Shapefile 支持 +try: + import geopandas as gpd + from shapely.geometry import box, Polygon, MultiPolygon + HAS_SHAPEFILE_SUPPORT = True +except ImportError: + HAS_SHAPEFILE_SUPPORT = False + print("Warning: geopandas/shapely not installed. Shapefile support disabled.") + print("Install with: pip install geopandas shapely") + +########################################################################### +########################################################################### + +class OptionParser (optparse.OptionParser): + + def check_required(self, opt): + option = self.get_option(opt) + if getattr(self.values, option.dest) is None: + self.error("%s option not supplied" % option) + +def bbox_from_shapefile(shapefile_path): + """ + 从 Shapefile 文件提取边界框 + + Args: + shapefile_path: Shapefile 文件路径（.shp） + + Returns: + 边界框字符串格式: "min_lon,min_lat,max_lon,max_lat" + """ + try: + # 读取 Shapefile + gdf = gpd.read_file(shapefile_path) + + # 计算总边界框 + total_bounds = gdf.total_bounds + # total_bounds = (minx, miny, maxx, maxy) + + # 转换为字符串格式 + bbox_str = f"{total_bounds[0]},{total_bounds[1]},{total_bounds[2]},{total_bounds[3]}" + + print(f"Shapefile bounding box: {bbox_str}") + return bbox_str + except Exception as e: + print(f"Error reading shapefile: {str(e)}") + return None + +def check_slc_exists(slc_name, path_SLC, path_RSLC, acquisition_date): + """ + 检查 SLC 文件是否已存在 + + Args: + slc_name: SLC 文件名 + path_SLC: SLC 存储路径 + path_RSLC: RSLC 存储路径 + acquisition_date: 采集日期字符串 + + Returns: + bool: 文件是否已存在 + """ + # 检查 .zip 文件 + if os.path.exists(os.path.join(path_SLC, slc_name + '.zip')): + return True + + # 检查 .rslc 文件 + try: + datei = datetime.datetime.strptime(acquisition_date.split('.')[0], '%Y-%m-%dT%H:%M:%S').strftime("%Y%m%d") + if os.path.exists(os.path.join(path_RSLC, datei + '.vv.rslc')): + return True + except: + pass + + return False + +def read_netrc(machine_name): + """ + 从 ~/.netrc 文件读取登录凭证 + + Args: + machine_name: 机器名称 (例如: urs.earthdata.nasa.gov) + + Returns: + tuple: (username, password) 或 (None, None) 如果未找到 + """ + netrc_path = os.path.expanduser('~/.netrc') + + if not os.path.exists(netrc_path): + return None, None + + try: + with open(netrc_path, 'r') as f: + lines = f.readlines() + i = 0 + while i < len(lines): + line = lines[i].strip() + + # 查找 machine 行 + if line.startswith('machine ' + machine_name): + username = None + password = None + + # 查找接下来的几行中的 login 和 password + j = i + 1 + while j < len(lines) and j < i + 5: # 最多查找5行 + next_line = lines[j].strip() + tokens = next_line.split() + + if len(tokens) >= 2: + if tokens[0] == 'login': + username = tokens[1] + elif tokens[0] == 'password': + password = tokens[1] + + # 如果找到下一个 machine，停止查找 + if next_line.startswith('machine '): + break + + j += 1 + + if username and password: + return username, password + + i += 1 + except Exception as e: + print(f"Warning: Error reading .netrc file: {str(e)}") + + return None, None + +def download_slc(url, username, password, path_SLC): + """ + 下载单个 SLC 文件 + + Args: + url: 下载 URL + username: ASF 用户名 + password: ASF 密码 + path_SLC: 存储路径 + + Returns: + bool: 下载是否成功 + """ + # 如果提供了用户名和密码，使用它们 + if username and password: + cmd = f'wget -c --http-user={username} --http-password={password} "{url}" -P {path_SLC}' + else: + # 否则使用 .netrc 文件 + # --auth-no-challenge: 立即发送认证信息，不等待服务器挑战 + # --netrc: 从 ~/.netrc 读取认证信息 + cmd = f'wget -c --auth-no-challenge --netrc "{url}" -P {path_SLC}' + + return os.system(cmd) == 0 + +def convert_yyyymmdd_to_iso(date_str): + """ + 将 YYYYMMDD 格式转换为 ISO 格式 (YYYY-MM-DDTHH:MM:SS) + + Args: + date_str: 日期字符串 (YYYYMMDD) + + Returns: + str: ISO 格式日期字符串 (YYYY-MM-DDT00:00:00) + """ + if len(date_str) == 8: + year = date_str[0:4] + month = date_str[4:6] + day = date_str[6:8] + return f"{year}-{month}-{day}T00:00:00" + else: + # 如果已经是其他格式，直接返回 + return date_str + +########################################################################### +########################################################################### + +if len(sys.argv) < 3: + prog = os.path.basename(sys.argv[0]) + print("="*80) + print("Sentinel-1 SLC Downloader from ASF") + print("="*80) + print("\nUsage examples:") + print("\n1. Using bounding box:") + print(" python3 " + prog + " -u username -p password -s . -r . \\") + print(" -b -10.78,51.27,-5.03,55.70 \\") + print(" -i 20170101 -j 20231231 \\") + print(f" -o 1 -f a -q IW -m csv -w n") + print("\n2. Using Shapefile (new feature):") + print(" python3 " + prog + " -u username -p password -s . -r . \\") + print(" --shp study_area.shp \\") + print(" -i 20170101 -j 20231231 \\") + print(f" -o 1 -f a -q IW -m csv -w n") + print("\nOptions:") + print(" -u, --username ASF username (optional, can use .netrc)") + print(" -p, --password ASF password (optional, can use .netrc)") + print(" -n, --netrc Machine name in ~/.netrc (default: urs.earthdata.nasa.gov)") + print(" -s, --path_SLC Output directory for SLC files") + print(" -r, --path_RSLC Directory to check for processed RSLC files") + print(" -b, --bbox Bounding box (min_lon,min_lat,max_lon,max_lat)") + print(" --shp Shapefile path (.shp) for spatial filtering") + print(" -i, --date_start Start date (YYYYMMDD)") + print(" -j, --date_end End date (YYYYMMDD)") + print(" -o, --orbit_relative Relative orbit number (optional)") + print(" --frame Frame number (optional)") + print(" -f, --flight_direction Flight direction (a=ascending, d=descending, or all)") + print(" -q, --q_acqui_mode Acquisition mode (IW, EW, SM, or all)") + print(" -m, --mode Output format (csv or kml)") + print(" -w, --write Download files? (Y/N)") + print(" --parallel Number of parallel downloads (default: 1, serial)") + print("\nNote: Shapefile support requires geopandas and shapely") + print("="*80) + sys.exit(-1) +else: + usage = "usage: %prog [options] " + parser = OptionParser(usage=usage) + parser.add_option("-u", "--username", action="store", type="string", default=None, + help="ASF username (can be read from ~/.netrc)") + parser.add_option("-p", "--password", action="store", type="string", default=None, + help="ASF password (can be read from ~/.netrc)") + parser.add_option("-n", "--netrc", action="store", type="string", default='urs.earthdata.nasa.gov', + help="Machine name in ~/.netrc file (default: urs.earthdata.nasa.gov)") + parser.add_option("-s", "--path_SLC", action="store", type="string", default='.') + parser.add_option("-r", "--path_RSLC", action="store", type="string", default='.') # Only available for GAMMA stack + parser.add_option("-b", "--bbox", action="store", type="string", default='-10.78,51.27,-5.03,55.70') + parser.add_option("--shp", "--shapefile", action="store", type="string", default=None, + help="Shapefile path for spatial filtering") + parser.add_option("-i", "--date_start", action="store", type="string", default='20170101') + parser.add_option("-j", "--date_end", action="store", type="string", default='20240101') + parser.add_option("-o", "--orbit_relative", action="store", type="float", default=None, + help="Relative orbit number (optional, if not specified will search all orbits)") + parser.add_option("--frame", action="store", type="int", default=None, + help="Frame number (optional, if not specified will search all frames)") + parser.add_option("-f", "--flight_direction", action="store", type="string", default='a', + help="Flight direction (a=ascending, d=descending, or 'all' for both)") + parser.add_option("-q", "--q_acqui_mode", action="store", type="string", default='IW') + parser.add_option("-m", "--mode", action="store", type="string", default='csv') + parser.add_option("-w", "--write", action="store", type="string", default='n') + parser.add_option("--parallel", action="store", type="int", default=1, + help="Number of parallel downloads (default: 1, serial)") + (options, args) = parser.parse_args() + +########################################################################### +# Main +########################################################################### + +date_format = "%Y-%m-%d" + +# 转换日期格式 +date_start_iso = convert_yyyymmdd_to_iso(options.date_start) +date_end_iso = convert_yyyymmdd_to_iso(options.date_end) + +# 处理用户名和密码：如果未提供，尝试从 .netrc 读取 +if options.username is None or options.password is None: + netrc_username, netrc_password = read_netrc(options.netrc) + if netrc_username and netrc_password: + if options.username is None: + options.username = netrc_username + print(f"Username read from ~/.netrc ({options.netrc})") + if options.password is None: + options.password = netrc_password + print(f"Password read from ~/.netrc ({options.netrc})") + else: + if options.username is None: + print("Error: Username not provided and not found in ~/.netrc") + print("Please provide username with -u option or configure ~/.netrc") + sys.exit(1) + if options.password is None: + print("Error: Password not provided and not found in ~/.netrc") + print("Please provide password with -p option or configure ~/.netrc") + sys.exit(1) + +# 处理空间范围：优先使用 Shapefile，否则使用 bounding box +if options.shp: + if not HAS_SHAPEFILE_SUPPORT: + print("Error: Shapefile support requires geopandas and shapely") + print("Install with: pip install geopandas shapely") + sys.exit(1) + + print(f"Using Shapefile: {options.shp}") + bbox = bbox_from_shapefile(options.shp) + if bbox is None: + print("Error: Failed to extract bounding box from shapefile") + sys.exit(1) +else: + bbox = options.bbox + print(f"Using bounding box: {bbox}") + +# 构建 API 查询命令 +# 轨道号、frame 和飞行方向都是可选的 +api_params = f"platform=s1&bbox={bbox}&start={date_start_iso}-UTC&end={date_end_iso}-UTC&processingLevel=SLC&maxResults=10000" + +# 如果指定了轨道号，添加到查询参数 +if options.orbit_relative is not None: + api_params += f"&relativeOrbit={int(options.orbit_relative)}" + +# 如果指定了 frame 号，添加到查询参数 +if hasattr(options, 'frame') and options.frame is not None: + api_params += f"&frame={options.frame}" + +# 如果指定了飞行方向且不是 'all'，添加到查询参数 +if options.flight_direction and options.flight_direction.upper() != 'ALL': + api_params += f"&flightDirection={options.flight_direction.upper()}" + +# 添加输出格式 +api_params += f"&output={options.mode.upper()}" + +cmd1 = f'curl "https://api.daac.asf.alaska.edu/services/search/param?{api_params}" > SLC_list.{options.mode.lower()}' + +print(f"Querying ASF API for SLC data...") +print(f" Bounding Box: {bbox}") +print(f" Date Range: {options.date_start} to {options.date_end}") +if options.orbit_relative is not None: + print(f" Orbit: {options.orbit_relative}") +else: + print(f" Orbit: All") +if hasattr(options, 'frame') and options.frame is not None: + print(f" Frame: {options.frame}") +else: + print(f" Frame: All") +if options.flight_direction and options.flight_direction.upper() != 'ALL': + print(f" Flight Direction: {options.flight_direction.upper()}") +else: + print(f" Flight Direction: All") +print(f" Mode: {options.q_acqui_mode.upper()}") +print() + +# 执行查询 +if options.mode.upper() == 'KML': + os.system(cmd1) +elif options.mode.upper() == 'CSV': + os.system(cmd1) +else: + print('Error: Please select a correct mode (csv or kml)...') + sys.exit(1) + +# 根据波束模式过滤列表 +print(f"Filtering results by acquisition mode: {options.q_acqui_mode.upper()}") +if os.path.exists("SLC_list.csv"): + os.rename("SLC_list.csv","SLC_list_orig.csv") + h = 0 + fout = open("SLC_list.csv",'w') + total_count = 0 + matched_count = 0 + + with open("SLC_list_orig.csv") as fi: + for li in fi: + total_count += 1 + if h > 0: + if options.q_acqui_mode.upper() == 'all': + fout.write(li) + matched_count += 1 + elif options.q_acqui_mode.upper() in li: + fout.write(li) + matched_count += 1 + else: + fout.write(li) + h = h + 1 + + fout.close() + os.remove("SLC_list_orig.csv") + + print(f" Total results: {total_count - 1}") + print(f" Filtered results: {matched_count}") +else: + print("Warning: SLC_list.csv not found") + sys.exit(1) + +# 下载文件 +if options.mode.upper() == 'CSV' and options.write.upper() == 'Y': + try: + listSLC = pd.read_csv("SLC_list.csv") + except Exception as e: + print(f"Error reading SLC_list.csv: {str(e)}") + sys.exit(1) + + print(f"\nFound {len(listSLC)} SLC files to process") + print("="*80) + + # 创建输出目录 + if not os.path.exists(options.path_SLC): + os.makedirs(options.path_SLC) + print(f"Created output directory: {options.path_SLC}") + + # 检查是否有并行参数 + parallel_numb = getattr(options, 'parallel', 1) + if parallel_numb is None: + parallel_numb = 1 + + # 准备下载数据列表 + download_tasks = [] + skipped_tasks = [] + + for h, slci in enumerate(listSLC['Granule Name']): + url = listSLC['URL'][h] + acquisition_date = listSLC['Acquisition Date'][h] + + # 检查文件是否已存在 + if check_slc_exists(slci, options.path_SLC, options.path_RSLC, acquisition_date): + skipped_tasks.append((h+1, slci, "Already exists")) + else: + download_tasks.append((h+1, slci, url, acquisition_date)) + + print(f"Files to download: {len(download_tasks)}") + print(f"Files skipped (already exists): {len(skipped_tasks)}") + + # 并行下载函数 + def download_worker(task): + idx, slci, url, acquisition_date = task + print(f"\n[{idx}/{len(listSLC)}] Downloading: {slci}") + + if download_slc(url, options.username, options.password, options.path_SLC): + print(f" Status: Download SUCCESS") + return (idx, slci, "SUCCESS") + else: + print(f" Status: Download FAILED") + return (idx, slci, "FAILED") + + # 执行下载 + downloaded = 0 + failed = 0 + + if parallel_numb > 1 and len(download_tasks) > 0: + # 并行下载 + from multiprocessing.pool import ThreadPool + print(f"\nStarting parallel download with {parallel_numb} workers...") + + pool = ThreadPool(parallel_numb) + results = pool.map(download_worker, download_tasks) + pool.close() + pool.join() + + # 统计结果 + for result in results: + if result[2] == "SUCCESS": + downloaded += 1 + else: + failed += 1 + elif len(download_tasks) > 0: + # 串行下载 + print("\nStarting serial download...") + for task in download_tasks: + result = download_worker(task) + if result[2] == "SUCCESS": + downloaded += 1 + else: + failed += 1 + + # 打印摘要 + print("\n" + "="*80) + print("DOWNLOAD SUMMARY") + print("="*80) + print(f"Total SLC files: {len(listSLC)}") + print(f"Downloaded: {downloaded}") + print(f"Skipped (already exists): {len(skipped_tasks)}") + print(f"Failed: {failed}") + print("="*80) + + if failed > 0: + print(f"\nWarning: {failed} downloads failed. Check the output above for details.") + sys.exit(1) + +elif options.mode.upper() == 'KML' and options.write.upper() == 'Y': + print('Please, select CSV mode to download..') +elif options.write.upper() != 'Y': + print(f"\nSLC list saved to: SLC_list.csv") + print("To download files, re-run with -w Y option") \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/ASAR_orb_cor.py b/.codex_tmp/pyint_variants/no_rescue/pyint/ASAR_orb_cor.py new file mode 100644 index 0000000..87eb3c5 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/ASAR_orb_cor.py @@ -0,0 +1,139 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Central South University & University of Miami ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import linecache +import datetime + +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +######################################################################### + +INTRODUCTION = ''' +############################################################################# + Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 + + Correct ASAR orbit using precise DORIS data. + +''' + +EXAMPLE = ''' + Usage: + ASAR_orb_cor.py projectName Date + + Examples: + ASAR_orb_cor.py AqabaERSA 960101 + +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Download (and correct) precise ERS orbit data.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='project name') + parser.add_argument('date',help='date of the ERS data') + + inps = parser.parse_args() + return inps + +################################################################################ + + +def main(argv): + + total = time.time() + inps = cmdLineParse() + + projectName = inps.projectName + DATE = inps.date + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + ASAROrbDir = os.getenv('ASARORBDIR') + + + slcDir = scratchDir + '/' + projectName + '/SLC' + projDir = scratchDir + '/' + projectName + + + if len(DATE)==8: + DATE8 = DATE + DATE6 = DATE[2:8] + else: + DATE8 = '20' + DATE + DATE6 = DATE + + + slcDir1 = slcDir + '/' + DATE6 + slcpar = slcDir1 + '/' + DATE6 + '.slc.par' + + + Year = DATE8[0:4] + Month = DATE8[4:6] + Day = DATE8[6:8] + + DATE1 = (datetime.date(int(Year),int(Month),int(Day)) + datetime.timedelta(days=-1)).strftime("%Y%m%d") + DATE2 = (datetime.date(int(Year),int(Month),int(Day)) + datetime.timedelta(days=1)).strftime("%Y%m%d") + + os.chdir(ASAROrbDir) + call_str = 'ls > t0' + os.system(call_str) + + call_str="grep " + DATE1 + " t0 | grep " + DATE2 + " >t01" + os.system(call_str) + + AA= np.loadtxt('t01',dtype=np.str) + AA_file = ASAROrbDir + '/' + str(AA) + + call_str = 'DORIS_vec ' + slcpar + ' ' + AA_file + os.system(call_str) + + + print("Using DORIS orbital data for %s is done." % DATE) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/ASAR_orb_cor_all .py b/.codex_tmp/pyint_variants/no_rescue/pyint/ASAR_orb_cor_all .py new file mode 100644 index 0000000..0676e4c --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/ASAR_orb_cor_all .py @@ -0,0 +1,125 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Central South University & University of Miami ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import linecache + +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +def is_number(s): + try: + int(s) + return True + except ValueError: + return False + +######################################################################### + +INTRODUCTION = ''' +############################################################################# + Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 + + Correct ASAR orbit using precise DORIS data. + +''' + +EXAMPLE = ''' + Usage: + ASAR_orb_cor_all.py projectName + + Examples: + ASAR_orb_cor_all.py AqabaERSA + +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Download (and correct) precise ERS orbit data.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='project name') + + inps = parser.parse_args() + return inps + +################################################################################ + + +def main(argv): + + total = time.time() + inps = cmdLineParse() + + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + slcDir = scratchDir + '/' + projectName + '/SLC' + + ListSLC = os.listdir(slcDir) + Datelist = [] + SLCfile = [] + SLCParfile = [] + + + for kk in range(len(ListSLC)): + if ( is_number(ListSLC[kk]) and len(ListSLC[kk])==6 ): # if SAR date number is 8, 6 should change to 8. + DD=ListSLC[kk] + Year=int(DD[0:2]) + Month = int(DD[2:4]) + Day = int(DD[4:6]) + Datelist.append(ListSLC[kk]) + N = len(Datelist) + + for i in range(N): + call_str = 'ERS_orb_cor.py ' + projectName + ' ' + Datelist[i] + print(call_str) + os.system(call_str) + + print("Using DEFT orbital data for project %s is done." % projectName) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/ASAR_orb_cor_par.py b/.codex_tmp/pyint_variants/no_rescue/pyint/ASAR_orb_cor_par.py new file mode 100644 index 0000000..78f2aea --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/ASAR_orb_cor_par.py @@ -0,0 +1,122 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Central South University & University of Miami ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import linecache +import datetime + +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +######################################################################### + +INTRODUCTION = ''' +############################################################################# + Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 + + Precise ENVISAT orbit data from Delft (http://www.deos.tudelft.nl/). + Correct the orbit parameters using DORIS_vec + +''' + +EXAMPLE = ''' + Usage: + ASAR_orb_cor_par.py slc_par + + Examples: + ASAR_orb_cor_par.py 960101.slc.par + +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Download (and correct) precise ERS orbit data.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('slc_par',help='slc_par file.') + + inps = parser.parse_args() + return inps + +################################################################################ + + +def main(argv): + + total = time.time() + inps = cmdLineParse() + + slcpar = inps.slc_par + ASAROrbDir = os.getenv('ASARORBDIR') + PATH =os.getcwd() + + DATE = UseGamma(slcpar,'read','date:') + Year = str(int(DATE[0:4])) + Month = str(int(DATE[4:8])) + Day = str(int(DATE[8:12])) + + DATE1 = (datetime.date(int(Year),int(Month),int(Day)) + datetime.timedelta(days=-1)).strftime("%Y%m%d") + DATE2 = (datetime.date(int(Year),int(Month),int(Day)) + datetime.timedelta(days=1)).strftime("%Y%m%d") + + os.chdir(ASAROrbDir) + call_str = 'ls > t0' + os.system(call_str) + + call_str="grep " + DATE1 + " t0 | grep " + DATE2 + " > t01" + os.system(call_str) + + AA= np.loadtxt('t01',dtype=np.str) + AA_file = ASAROrbDir + '/' + str(AA) + + os.chdir(PATH) + call_str = 'DORIS_vec ' + slcpar + ' ' + AA_file + ' 20' + os.system(call_str) + + + print("Using DORIS orbital data for %s is done." % slcpar) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/.gitignore b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/.gitignore new file mode 100644 index 0000000..946f285 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/.gitignore @@ -0,0 +1,167 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# mac +.DS_Store + + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ +tests + + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/LICENSE b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/LICENSE new file mode 100644 index 0000000..d470d14 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Fanchengyan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/README.md b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/README.md new file mode 100644 index 0000000..433a5ed --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/README.md @@ -0,0 +1,2 @@ +# AutoGACOS +A Python library for automatically submitting and downloading GACOS data diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/Makefile b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/requirements.txt b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/requirements.txt new file mode 100644 index 0000000..2a9aa68 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/requirements.txt @@ -0,0 +1,6 @@ +recommonmark +sphinx>=7 +myst-parser +myst_nb +sphinx_rtd_theme +Jinja2 diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/api/datasets/datasets.rst b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/api/datasets/datasets.rst new file mode 100644 index 0000000..f8fa7f7 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/api/datasets/datasets.rst @@ -0,0 +1,32 @@ +InSAR Datasets +============== + +To automatically submit requests to the GACOS service, the user needs to provide +the coordinates of the area of interest and the acquisition dates and times of +the SAR images. We provide a set of classes to automatically retrieve the +those information from well known InSAR datasets. + +Currently, the following InSAR Datasets are supported: + * :class:`.HyP3Dataset` + * :class:`.LiCSARDataset` + +HyP3Dataset +----------- + +.. autoclass:: gacos.HyP3Dataset + :members: + :undoc-members: + :member-order: bysource + :show-inheritance: + +LiCSARDataset +------------- + +.. autoclass:: gacos.LiCSARDataset + :members: + :undoc-members: + :member-order: bysource + :show-inheritance: + + + \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/api/index.rst b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/api/index.rst new file mode 100644 index 0000000..32d5ef4 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/api/index.rst @@ -0,0 +1,11 @@ +Python API Reference +==================== + + +.. toctree:: + + datasets/datasets + submit/submit + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/api/submit/submit.rst b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/api/submit/submit.rst new file mode 100644 index 0000000..a07379b --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/api/submit/submit.rst @@ -0,0 +1,15 @@ +Requests Submitter +================== + +The GACOS limits the number of acquisition requests that can be submitted at a time. +This is to prevent the system from being overloaded. But, if you have a large number +of acquisition requests to submit, you can submit them in batches. We designed a +:class:`gacos.Submitter` class to help you submit the requests in batches. The +acquisitions will be split into batches which not exceed the 20 acquisitions per +request. + +.. autoclass:: gacos.Submitter + :members: + :undoc-members: + :member-order: bysource + :show-inheritance: diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/conf.py b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/conf.py new file mode 100644 index 0000000..073055f --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/conf.py @@ -0,0 +1,58 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "FanSAR" +copyright = "2023, Fancy" +author = "Fancy" +release = "v1.0" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + "myst_nb", + "sphinx.ext.autodoc", + "sphinx.ext.doctest", + "sphinx.ext.intersphinx", + "sphinx.ext.napoleon", + "sphinx.ext.todo", +] +source_suffix = { + ".rst": "restructuredtext", + ".md": "markdown", + ".ipynb": "myst-nb", +} + +# templates_path = ['_templates'] +exclude_patterns = [] + +# to disable execution of notebooks +nb_execution_mode = "off" +nb_execution_excludepatterns = ["quickstart.ipynb"] + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "sphinx_rtd_theme" +# html_static_path = ['_static'] + +# General information about the project. +project = "faninsar" +copyright = "2023, Fancy" +author = "Chengyan Fan" + +video_enforce_extra_source = True + +autodoc_default_options = { + "members": True, + "undoc-members": True, + "member-order": "bysource", + "special-members": "__init__", + ":show-inheritance:": True, +} diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/index.rst b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/index.rst new file mode 100644 index 0000000..91f1318 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/index.rst @@ -0,0 +1,19 @@ +===================================== +Welcome to AutoGACOS's documentation! +===================================== + + +Introduction +------------ + +.. toctree:: + :maxdepth: 4 + :caption: Contents: + + intro + user_guide/quickstart + AutoGACOS API Reference + terminology + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/intro.rst b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/intro.rst new file mode 100644 index 0000000..e8b5eb2 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/intro.rst @@ -0,0 +1,3 @@ +Introduction +============ + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/terminology.rst b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/terminology.rst new file mode 100644 index 0000000..844e30d --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/terminology.rst @@ -0,0 +1,27 @@ +.. _terminology: + +Terminology +=========== + +.. glossary:: + + Acquisition + A single SAR acquisition, and is expressed as ``datetime.datetime`` object. + + Pair + A pair is a combination of two SAR acquisitions. + + Pairs + A collection of pairs. + + Loop + A loop + + Loops + A collection of loops. + + SBASNetwork + A collection of loops and pairs. + + CRS + A coordinate reference system (CRS) is a coordinate-based local, regional or global system used to locate geographical entities. In FanInSAR, the CRS is handled by the ``rasterio`` package. \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/user_guide/quickstart.ipynb b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/user_guide/quickstart.ipynb new file mode 100644 index 0000000..beb1bcd --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/docs/source/user_guide/quickstart.ipynb @@ -0,0 +1,635 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Quickstart for AutoGACOS\n", + "\n", + "**AutoGACOS** is a Python library for automatically submitting and downloading GACOS data" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import gacos" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "path = \"/Volumes/Data/Hyp3/descending_roi\"\n", + "ds = gacos.HyP3Dataset(path)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "check all acquisition dates in Hyp3 dateset" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "DatetimeIndex(['2015-11-12 23:26:31', '2015-12-06 23:26:31',\n", + " '2015-12-30 23:26:30', '2016-01-23 23:26:29',\n", + " '2016-02-16 23:26:28', '2016-03-11 23:26:28',\n", + " '2016-04-04 23:26:29', '2016-04-28 23:26:30',\n", + " '2016-05-22 23:26:34', '2016-06-15 23:26:36',\n", + " ...\n", + " '2023-01-10 23:27:15', '2023-01-22 23:27:15',\n", + " '2023-02-03 23:27:15', '2023-02-15 23:27:14',\n", + " '2023-02-27 23:27:15', '2023-03-11 23:27:14',\n", + " '2023-03-23 23:27:14', '2023-04-04 23:27:15',\n", + " '2023-08-14 23:27:22', '2023-09-07 23:27:23'],\n", + " dtype='datetime64[ns]', length=204, freq=None)\n", + "Index(['20151112', '20151206', '20151230', '20160123', '20160216', '20160311',\n", + " '20160404', '20160428', '20160522', '20160615',\n", + " ...\n", + " '20230110', '20230122', '20230203', '20230215', '20230227', '20230311',\n", + " '20230323', '20230404', '20230814', '20230907'],\n", + " dtype='object', length=204)\n", + "['23:25' '23:26' '23:27']\n" + ] + } + ], + "source": [ + "print(ds.date_times)\n", + "print(ds.dates)\n", + "print(ds.times)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Submit GACOS requests automatically" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "bf9575b9deea497089cc8167b8c8a438", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/3 [00:00>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 25, 'date': '20161007\\n20161031\\n20161124\\n20161218\\n20170111\\n20170204', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", + " sleeping for 546 seconds...\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "8fd9eb9543774f3b87058eef331792df", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/6 [00:00>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 26, 'date': '20151112\\n20151206\\n20151230\\n20160123\\n20160216\\n20160311\\n20160404\\n20160428\\n20160522\\n20160615\\n20160709\\n20160802\\n20160826\\n20160919\\n20170222\\n20170318\\n20170330', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", + ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 26, 'date': '20170411\\n20170423\\n20170505\\n20170517\\n20170529\\n20170610\\n20170622\\n20170704\\n20170716\\n20170728\\n20170809\\n20170821\\n20170902\\n20170914\\n20171008\\n20171020\\n20171101', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", + ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 26, 'date': '20171113\\n20171125\\n20171207\\n20171219\\n20171231\\n20180112\\n20180124\\n20180205\\n20180217\\n20180301\\n20180313\\n20180325\\n20180406\\n20180418\\n20180430\\n20180512\\n20180524', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", + ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 26, 'date': '20180605\\n20180617\\n20180629\\n20180711\\n20180723\\n20180804\\n20180816\\n20180921\\n20181003\\n20181015\\n20181027\\n20181108\\n20181120\\n20181202\\n20181214\\n20181226\\n20190107', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", + ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 26, 'date': '20190119\\n20190131\\n20190212\\n20190224\\n20190308\\n20190320\\n20190401\\n20190413\\n20190425\\n20190507\\n20190519\\n20190531\\n20190612\\n20190624\\n20190706\\n20190718\\n20190730', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", + ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 26, 'date': '20190811\\n20190823\\n20190904\\n20191221\\n20200102\\n20200114\\n20200126\\n20200207\\n20200219\\n20200302\\n20200314\\n20200326\\n20200407\\n20200419\\n20200501\\n20200513', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", + " sleeping for 288 seconds...\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4aa82286c5414723adcddcc8d864c133", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/5 [00:00>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 27, 'date': '20190916\\n20190928\\n20191010\\n20191022\\n20191103\\n20191115\\n20191127\\n20191209\\n20200525\\n20200606\\n20200618\\n20200630\\n20200712\\n20200724\\n20200805\\n20200817\\n20200829\\n20200910\\n20200922\\n20201004', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", + ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 27, 'date': '20201016\\n20201028\\n20201109\\n20201121\\n20201203\\n20201215\\n20201227\\n20210108\\n20210120\\n20210201\\n20210213\\n20210225\\n20210309\\n20210321\\n20210402\\n20210414\\n20210426\\n20210508\\n20210520\\n20210601', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", + ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 27, 'date': '20210613\\n20210625\\n20210707\\n20210719\\n20210731\\n20210812\\n20210824\\n20210905\\n20210917\\n20210929\\n20211011\\n20211023\\n20211104\\n20211116\\n20211128\\n20211210\\n20211222\\n20220103\\n20220115', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", + ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 27, 'date': '20220127\\n20220208\\n20220220\\n20220304\\n20220316\\n20220328\\n20220409\\n20220421\\n20220503\\n20220515\\n20220527\\n20220620\\n20220702\\n20220714\\n20220726\\n20220807\\n20220819\\n20220831\\n20220912', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", + ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 27, 'date': '20220924\\n20221006\\n20221018\\n20221030\\n20221111\\n20221123\\n20221205\\n20221217\\n20221229\\n20230110\\n20230122\\n20230203\\n20230215\\n20230227\\n20230311\\n20230323\\n20230404\\n20230814\\n20230907', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", + " sleeping for 743 seconds...\n" + ] + } + ], + "source": [ + "# initialize a submitter with the dataset and your email\n", + "submitter = gacos.Submitter(ds, email=\"your_email@xxx.com\")\n", + "\n", + "# submit all gacos requests for acquisitions in the Hyp3 dataset\n", + "submitter.post_requests()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# check the status of all requests\n", + "print(f\"Successfully submitted requests: \\n{submitter.succeed}\")\n", + "print(f\"Failed to submit requests: \\n{submitter.failed}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Retrieve GACOS results urls from your email automatically" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "667dccc73a434102a87226ebcb453389", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Retrieving GACOS urls: 0%| | 0/240 [00:00 None: + """Initialize SarDataset class + + Parameters + ---------- + bounds : tuple[float, float, float, float] + The bounding box of the dataset. + date_times : pd.DatetimeIndex + The datetime index of the dataset. + gacos_dir : Optional[Union[Path, str]], optional + The directory used to save gacos data. Used to check if the data is + already downloaded and avoid resubmitting. Default is None. + """ + self.bounds = bounds + self._date_times = date_times + + self._dates = date_times.strftime("%Y%m%d") + + if gacos_dir is not None: + self._dates_remain = self._get_dates_remain(gacos_dir) + else: + self._dates_remain = self.dates + + hour = date_times.hour + minute = np.round((date_times.second / 60) + date_times.minute).astype(int) + times = pd.Series([f"{h:02d}:{m:02d}" for h, m in zip(hour, minute)]) + self._times = times + + self._times_remain = self._get_times_remain() + + def __str__(self) -> str: + return ( + f"{self.__class__.__name__}(\n" + f" bounds={self.bounds}, \n" + f" times={len(self.times)}, \n" + f" dates={len(self.dates)}\n" + ")" + ) + + def __repr__(self) -> str: + return self.__str__() + + def _get_dates_remain(self, gacos_dir: Union[Path, str]): + """Get the dates that are not downloaded yet. + Parameters + ---------- + gacos_dir : Union[Path, str] + The directory used to save gacos data. Used to check if the data is + already downloaded and avoid resubmitting. + + Returns + ------- + dates_remain : np.ndarray + The dates that are not downloaded yet. + """ + gacos_files = list(Path(gacos_dir).rglob("*.ztd.tif")) + gacos_dates = [] + for i in gacos_files: + stem = i.stem.split(".")[0] + if len(stem) == 8: + gacos_dates.append(stem) + dates_remain = np.setdiff1d(self.dates, gacos_dates) + return dates_remain + + def _get_times_remain(self): + """Get the times corresponding to the dates that are not downloaded yet.""" + idx = np.where(np.isin(self.dates, self.dates_remain))[0] + times_remain = self._times[idx] + return times_remain + + @property + def dates(self): + """The dates (YYYYMMDD) of the acquisitions parsed from dataset.""" + return self._dates + + @property + def times(self): + """The times (HH:MM) of the acquisitions parsed from dataset.""" + return self._times.unique() + + @property + def date_times(self): + """The datetime of the acquisitions parsed from dataset.""" + return self._date_times + + @property + def dates_remain(self): + """The dates that are not downloaded yet. If gacos_dir is None, then + dates_remain is the same as dates.""" + return self._dates_remain + + @property + def times_remain(self): + """The times corresponding to the dates that are not downloaded yet.""" + return self._times_remain + + def gen_datetime_patches( + self, + mode: Literal["all", "remain"] = "remain", + ) -> dict: + """Generate datetime patches. + + Parameters + ---------- + mode : Literal["all", "remain"], optional + The mode to generate datetime patches. If "all", then generate all + the datetime patches. If "remain", then generate the datetime + patches of the dates that are not downloaded yet. Default is + "remain". + + Returns + ------- + datetime_patches : dict + The datetime patches. The key is the time (HH:MM) and the value is + the datetime patches. + """ + nums = 20 + datetime_patches = {} + + if mode == "all": + for _time in self.times: + _dts = self.dates[self._times == _time] + n_patch = np.ceil(len(_dts) / nums) + dates_patch = np.array_split(_dts, n_patch) + datetime_patches[_time] = dates_patch + elif mode == "remain": + for _time in self.times_remain: + _dts = self.dates_remain[self._times_remain == _time] + n_patch = np.ceil(len(_dts) / nums) + dates_patch = np.array_split(_dts, n_patch) + datetime_patches[_time] = dates_patch + + return datetime_patches + + def gen_post_data( + self, + dates: Union[list, np.ndarray], + times: Union[tuple[int, int], tuple[str, str]], + email: str, + ): + """Generate post data for gacos website. + + Parameters + ---------- + dates : list or np.ndarray + The list of dates. + times : tuple[int, int] + The time of the acquisition (hour, minute). + email : str + The email address to receive the gacos data. + + Returns + ------- + post_data : dict + The post data. + """ + if isinstance(dates, np.ndarray): + dates = dates.tolist() + times = [int(t) for t in times] + + post_data = { + "N": self.bounds[3], + "W": self.bounds[0], + "S": self.bounds[1], + "E": self.bounds[2], + "H": times[0], + "M": times[1], + "date": "\n".join(dates), + "type": "2", + "email": email, + } + return post_data + + +class LiCSARDataset(SarDataset): + def __init__( + self, + home_dir: Union[Path, str], + gacos_dir: Optional[Union[Path, str]] = None, + ) -> None: + """Initialize LiCSARDataset class + + Parameters + ---------- + home_dir : Union[Path, str] + The home directory of LiCSAR dataset. + gacos_dir : Optional[Union[Path, str]], optional + The directory used to save gacos data. Used to check if the data is + already downloaded and avoid resubmitting. Default is None. + """ + self.home_dir = Path(home_dir) + self.dataset = LiCSAR(home_dir) + bounds = self.dataset.bounds + time = self._get_time() + dates = self.dataset.pairs.dates + date_times = pd.to_datetime([f"{d} {time[0]}:{time[1]}:00" for d in dates]) + super().__init__(bounds, date_times, gacos_dir) + + def _get_time(self): + """Get the acquisition time of acquisitions. + + Returns + ------- + time: tuple[int, int] + A tuple of hour and minute representing the acquisition time. + + Raises + ------ + ValueError + If no center_time found in metadata.txt. + """ + meta_file = sorted(self.home_dir.rglob("metadata.txt"))[0] + + with open(meta_file) as f: + lines = f.readlines() + time = None + for line in lines: + line_split = line.split("=") + key, value = (line_split[0].strip(), line_split[1]) + if "center_time" == key: + center_time = value.strip() + hour, minute, second = center_time.split(":") + hour, minute, second = int(hour), int(minute), float(second) + minute = minute + int(np.round(second / 60, 0)) + return hour, minute + else: + continue + if time is None: + raise ValueError(f"No center_time found in {meta_file}") + + +class HyP3Dataset(SarDataset): + def __init__( + self, + home_dir: Union[Path, str], + gacos_dir: Optional[Union[Path, str]] = None, + ) -> None: + """Initialize HyP3Dataset class + + Parameters + ---------- + home_dir : Union[Path, str] + The home directory of HyP3 dataset. + gacos_dir : Optional[Union[Path, str]], optional + The directory used to save gacos data. Used to check if the data is + already downloaded and avoid resubmitting. Default is None. + """ + self.dataset = HyP3(home_dir) + bounds = self.dataset.bounds.to_crs("epsg:4326") + date_times = self.dataset.datetime + + super().__init__(bounds, date_times, gacos_dir) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/gacos/download.py b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/gacos/download.py new file mode 100644 index 0000000..5afcf60 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/gacos/download.py @@ -0,0 +1,158 @@ +import tarfile +from pathlib import Path +from typing import Optional, Union + +import numpy as np +import pandas as pd +from data_downloader import downloader +from faninsar.query import BoundingBox +from tqdm.auto import tqdm + +from .parse_email import GACOSEmail + + +class Downloader: + def __init__( + self, + url_file: Union[Path, str], + output_dir: Union[Path, str], + tar_gz_dir: Optional[Union[Path, str]] = None, + keep_original: bool = False, + times: Optional[Union[float, list[float]]] = None, + bounds: Optional[tuple[float, float, float, float]] = None, + ) -> None: + """Initialize Downloader class + + Parameters + ---------- + url_file : Union[Path, str] + Path to file containing URLs that created by :meth:`GACOSEmail.retrieve_gacos_urls` + output_dir : Union[Path, str] + directory to output gacos files + tar_gz_dir : Optional[Union[Path, str]], optional + directory to store downloaded *.tar.gz files. If None, then + `output_dir` is used. Default is None. + keep_original : bool, optional + Whether to keep original files (*.tar.gz). Default is False. + times : Optional[float], optional + times of acquisition, used to filter out files that are not needed. + this can be a single time or a list of times. times differ by less + than 10 minutes are considered the same. Default is None. + bounds : Optional[tuple[float, float, float, float]], optional + bounds of area of interest with order (W, S, E, N), used to filter + out files that are not needed. Default is None. + """ + self.url_file = Path(url_file) + self.output_dir = Path(output_dir) + if tar_gz_dir is None: + self.tar_gz_dir = self.output_dir + self.keep_original = keep_original + + if not self.url_file.exists(): + raise FileNotFoundError(f"{self.url_file} does not exist") + if not self.output_dir.exists(): + self.output_dir.mkdir(parents=True) + if not self.tar_gz_dir.exists(): + self.tar_gz_dir.mkdir(parents=True) + + self.df_urls = pd.read_csv(self.url_file, header=0) + + # only keep urls that intersect with bounds + if bounds is not None: + mask_bbox = self._bbox_mask(BoundingBox(*bounds)) + + # only keep urls that acquisition time is within 10 minutes of `time` + if times is not None: + if isinstance(times, float): + times = [times] + mask_time = self._time_mask(times) + + if bounds is not None and times is not None: + self.mask = mask_bbox & mask_time + elif bounds is not None: + self.mask = mask_bbox + elif times is not None: + self.mask = mask_time + else: + self.mask = np.ones(self.df_urls.shape[0], dtype=bool) + + self.mask = self.mask & self.date_mask + + def _bbox_mask(self, bounds) -> np.ndarray: + intersection_bbox = np.array( + [ + BoundingBox(*b).intersects(bounds) + for b in zip( + self.df_urls["south"].astype(float), + self.df_urls["west"].astype(float), + self.df_urls["north"].astype(float), + self.df_urls["east"].astype(float), + ) + ] + ) + return intersection_bbox + + def _time_mask(self, times) -> np.ndarray: + """Only keep urls that acquisition time is within 10 minutes of `time`""" + intersection_times = [] + for time in times: + intersection_times.append( + np.array( + np.abs(self.df_urls["time"].astype(float) - time) + <= 1 / 60 * 10 # 10 minutes + ) + ) + intersection_time = np.any(intersection_times, axis=0) + return intersection_time + + @property + def date_mask(self) -> np.ndarray: + """Remove urls that all acquisition dates have been downloaded""" + dates_urls = self.df_urls["date"].map(lambda x: eval(x)) + intersection_dates = [] + for dt_url in dates_urls: + intersection_dates.append(~np.all(np.isin(dt_url, self.dates_downloaded))) + return np.array(intersection_dates) + + @property + def dates_downloaded(self) -> np.ndarray: + """Return dates that have been downloaded""" + gacos_files = list(self.output_dir.rglob("*.ztd.tif")) + dates = [] + for i in gacos_files: + stem = i.stem.split(".")[0] + if len(stem) == 8: + dates.append(stem) + return np.array(dates) + + def download(self) -> None: + """Download GACOS files from URLs in file created by :meth:`GACOSEmail.retrieve_gacos_urls`""" + urls_used = self.df_urls[self.mask]["url"].values + + for url in tqdm(urls_used, unit="file", desc="Downloading GACOS files"): + gz_file = self.tar_gz_dir / Path(url).name + downloader.download_data(url, file_name=gz_file) + self._extract_tar_gz(gz_file) + if not self.keep_original: + self._delete_file(gz_file) + + def _extract_tar_gz(self, gz_file) -> None: + """Unzip/extract downloaded GACOS files + + Parameters + ---------- + gz_file : Path + path to downloaded GACOS file (*.tar.gz) + """ + with tarfile.open(gz_file, "r:gz") as tar: + tar.extractall(path=self.output_dir) + + def _delete_file(self, gz_file) -> None: + """Delete original GACOS files + + Parameters + ---------- + gz_file : Path + path to downloaded GACOS file (*.tar.gz) + """ + gz_file.unlink() diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/gacos/parse_email.py b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/gacos/parse_email.py new file mode 100644 index 0000000..f074cdf --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/gacos/parse_email.py @@ -0,0 +1,391 @@ +import email +import email.message +import getpass +import imaplib +import poplib +import re +from email.parser import Parser +from email.utils import parseaddr +from pathlib import Path +from typing import Literal, Optional, Union + +import pandas as pd +from tqdm.auto import tqdm + + +class GACOSEmail: + """a class to retrieve gacos urls from email. + + .. note:: + The IMAP server is used to retrieve content from email. Some email + service providers may need to enable the IMAP service in the settings. + **You are recommended to use a new email account to receive gacos urls to + avoid polluting your own email account**. + """ + + def __init__( + self, + username: str, + password: str, + host: str, + prompt: bool = False, + email_protocol: Literal["imap", "pop3"] = "imap", + port: Optional[int] = None, + gacos_email: str = "gacos2017@foxmail.com", + gacos_suffix: str = "tar.gz", + start_date: Optional[str] = None, + end_date: Optional[str] = None, + date_args: Optional[dict] = None, + ssl: bool = False, + ) -> None: + """Retrieve gacos urls from email. + + Parameters + ---------- + username : str + The username of the email address. + password : str + The password. + host : str + The host of the email address. For example, the host of gmail for + imap is "imap.gmail.com". You can find the host of your email + settings or search it on the Internet. + prompt: bool, optional + Prompt for username and/or password interactively when they are not + provided as keyword parameters. Default is False. + email_protocol : str, one of ["imap", "pop3"], optional + The protocol of the email. Default is "imap". + port : int, optional + The port of the host of your email. If None, the default port will be + used. Default is None. + gacos_email : str, optional + The email address of gacos. Default is "gacos2017@foxmail.com". + gacos_suffix : str, optional + The suffix of the gacos file url. Default is "tar.gz". The suffix is used + to filter urls in the email. This parameter is used to avoid the + situation that the email contains other urls. + start_date / end_date: str, optional + The start/end date of email. Used to filter the email. Default is None. Can be any format that can be parsed by pandas.to_datetime. + date_args : dict, optional + The arguments are passed to pandas.to_datetime. Default is None. + ssl : bool, optional + Whether to use SSL connection. Default is False. + """ + if prompt: + self.username = None + self.password = None + else: + self.username = username + self.password = password + self.host = host + self.email_protocol = email_protocol + self.port = port + self.gacos_email = gacos_email + self.gacos_suffix = gacos_suffix + self.ssl = ssl + + # date part + self.start_date = start_date + self.end_date = end_date + if date_args is None: + date_args = {} + self.date_args = date_args + + def _retrieve_gacos_urls_pop3(self): + server = login_in_email_pop3( + self.username, self.password, self.host, self.port, ssl=self.ssl + ) + print(server.getwelcome()) + + nums = server.stat()[0] + + gacos = [] + for i in tqdm(range(1, nums + 1), unit=" emails", desc="Retrieving GACOS Urls"): + response, msgLines, octets = server.retr(i) + msgLinesToStr = b"\r\n".join(msgLines).decode("utf8", "ignore") + messageObject = Parser().parsestr(msgLinesToStr) + + senderContent = messageObject["From"] + senderRealName, senderAdr = parseaddr(senderContent) + if senderAdr == self.gacos_email: + if not in_date_range( + pd.to_datetime(messageObject["Date"]).tz_localize(None), + self.start_date, + self.end_date, + self.date_args, + ): + continue + + msgBodyContents = get_content(messageObject) + info = parse_gacos_info( + msgBodyContents, + gacos_suffix=self.gacos_suffix, + ) + if info is not None: + gacos.append(info) + + server.quit() + + return gacos + + def _retrieve_gacos_urls_imap(self): + server = login_in_email_imap( + self.username, self.password, self.host, self.port, ssl=self.ssl + ) + if server is None: + print("IMAP server connection failed, skipping email check.") + return [] + server.select("inbox") + status, data = server.search(None, "ALL") + + gacos = [] + for i in tqdm(data[0].split(), unit=" emails", desc="Retrieving GACOS urls"): + res, msg = server.fetch(i, "(RFC822)") + for response_part in msg: + if isinstance(response_part, tuple): + msgLines = response_part[1].decode("utf8", "ignore") + break + + messageObject = Parser().parsestr(msgLines) + + senderContent = messageObject["From"] + senderRealName, senderAdr = parseaddr(senderContent) + if senderAdr == self.gacos_email: + if not in_date_range( + pd.to_datetime(messageObject["Date"]).tz_localize(None), + self.start_date, + self.end_date, + self.date_args, + ): + continue + + msgBodyContents = get_content(messageObject) + info = parse_gacos_info( + msgBodyContents, + gacos_suffix=self.gacos_suffix, + ) + if info is not None: + gacos.append(info) + + server.close() + + return gacos + + def retrieve_gacos_urls( + self, + output_file: Union[str, Path], + ): + """Retrieve gacos urls from username. + + Parameters + ---------- + output_file : str or Path + The output file used to save the gacos urls. + """ + if self.email_protocol == "pop3": + gacos = self._retrieve_gacos_urls_pop3() + elif self.email_protocol == "imap": + gacos = self._retrieve_gacos_urls_imap() + else: + raise ValueError("email_protocol must be 'pop3' or 'imap'.") + + cols = ["url", "south", "north", "west", "east", "time", "date"] + df_gacos = pd.DataFrame(gacos, columns=cols).drop_duplicates(subset="url") + + # save to file + try: + df_gacos.to_csv(output_file) + print(f"Save gacos urls to {output_file}") + except Exception as e: + self.df_gacos = df_gacos + print(e) + print("Save gacos urls failed") + print("You can access the gacos urls by `df_gacos` attribute.") + + +def in_date_range(date, start_date, end_date, date_args={}): + start_date = pd.to_datetime(start_date, **date_args) + end_date = pd.to_datetime(end_date, **date_args) + start_none = start_date is None or pd.isna(start_date) + end_none = end_date is None or pd.isna(end_date) + if start_none and end_none: + return True + elif start_none: + return date <= end_date + elif end_none: + return date >= start_date + else: + return (date >= start_date) and (date <= end_date) + + +def decodeBody(msgPart: email.message.Message): + """decode email body + + Parameters + ---------- + msgPart : email.message.Message + The email message object. + """ + contentType = msgPart.get_content_type() + textContent = "" + if contentType == "text/plain" or contentType == "text/html": + content = msgPart.get_payload(decode=True) + charset = msgPart.get_charset() + if charset is None: + contentType = msgPart.get("Content-Type", "").lower() + position = contentType.find("charset=") + if position >= 0: + charset = contentType[position + 8 :].strip() + if charset: + textContent = content.decode(charset) + return textContent + + +def get_content(messageObject): + msgBodyContents = [] + if messageObject.is_multipart(): # parse multipart email + messageParts = messageObject.get_payload() + for messagePart in messageParts: + bodyContent = decodeBody(messagePart) + if bodyContent: + msgBodyContents.append(bodyContent) + else: + bodyContent = decodeBody(messageObject) + if bodyContent: + msgBodyContents.append(bodyContent) + return msgBodyContents + + +def login_in_email_pop3(username, password, host, port, ssl=False): + try: + if username is None: + username = input("username: ") + if password is None: + password = getpass.getpass("password: ") + + if ssl: + if port is None: + port = 995 + server = poplib.POP3_SSL(host, port) + else: + if port is None: + port = 110 + server = poplib.POP3(host, port) + + server.user(username) + server.pass_(password) + return server + except Exception as e: + print(e) + print("login failed") + + +def login_in_email_imap(username, password, host, port, ssl=False): + try: + if username is None: + username = input("username: ") + if password is None: + password = getpass.getpass("password: ") + + if ssl: + if port is None: + port = 993 + else: + if port is None: + port = 143 + + # 强制 IPv4: monkey-patch getaddrinfo，避免 IPv6 不可达 + import socket as _socket + _orig_getaddrinfo = _socket.getaddrinfo + def _ipv4_only_getaddrinfo(*args, **kwargs): + return _orig_getaddrinfo(args[0], args[1], _socket.AF_INET, + *args[3:], **kwargs) + _socket.getaddrinfo = _ipv4_only_getaddrinfo + try: + if ssl: + server = imaplib.IMAP4_SSL(host, port) + else: + server = imaplib.IMAP4(host, port) + finally: + _socket.getaddrinfo = _orig_getaddrinfo + + server.login(username, password) + + # 163/126 等网易邮箱要求登录后发送 ID 命令才能执行 SELECT + if '163.com' in host or '126.com' in host or 'yeah.net' in host: + try: + tag = server._new_tag() + server.send(tag + b' ID ("name" "pyint" "version" "1.0" ' + b'"vendor" "pyint")\r\n') + while True: + resp = server.readline() + if resp.startswith(tag): + break + except Exception: + pass + + return server + except Exception as e: + print(f"IMAP login failed: {e}") + return None + + +def parse_gacos_info(msgBodyContents, gacos_suffix="tar.gz"): + """Parse gacos info from email body. + + Parameters + ---------- + msgBodyContents : list + The email body contents. + gacos_suffix : str, optional + The suffix of the gacos file url. Default is "tar.gz". The suffix is used + to filter urls in the email. This parameter is used to avoid the + situation that the email contains other urls. + """ + + url, south, north, west, east, _time, date_list = ( + None, + None, + None, + None, + None, + None, + None, + ) + date_list = [] + for contents in msgBodyContents: + lines = [i.strip() for i in contents.split("\n") if i] + for line in lines: + line = line.strip() + loc = line.split("=") + if len(loc) == 2: + parameter, value = loc + parameter, value = (parameter.strip(), value.strip()) + if "MinLat" == parameter: + south = float(value) + if "MaxLat" == parameter: + north = float(value) + if "MinLon" == parameter: + west = float(value) + if "MaxLon" == parameter: + east = float(value) + loc = line.split(":") + if len(loc) == 2: + parameter, value = loc + parameter, value = (parameter.strip(), value.strip()) + if "Time" == parameter: + _time = float(value) + + if len(line) == 8 and line.isdigit(): + date_list.append(line) + + for i in ["http", "ftp", "https"]: + result = re.search(f"\({i}.*{gacos_suffix}\)", line) + if result: + url = result.group()[1:-1] + break + + if url == south == north == west == east == _time: + return None + else: + return url, south, north, west, east, _time, date_list diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/gacos/submit.py b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/gacos/submit.py new file mode 100644 index 0000000..ce1d235 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/gacos/submit.py @@ -0,0 +1,83 @@ +import time +from pathlib import Path +from typing import Optional, Union + +import numpy as np +import requests +from tqdm.auto import tqdm + +from .datasets import SarDataset + + +class Submitter: + def __init__( + self, + dataset: SarDataset, + email: str, + sleep_time_range: tuple[int, int] = (60 / 2, 60 * 5), + gacos_url="http://www.gacos.net/M/action_page.php", + ) -> None: + """Initialize Submitter class + + Parameters + ---------- + dataset : SarDataset + The SarDataset object. + email : str + The email address to submit to gacos. + sleep_time_range : tuple[int, int], optional + The range of sleep time in seconds. Default is (60, 60 * 5). + gacos_url : str, optional + The url of gacos website. Default is "http://www.gacos.net/M/action_page.php". + """ + self.dataset = dataset + self.email = email + self.sleep_time_range = sleep_time_range + self.gacos_url = gacos_url + + self._failed = [] + self._succeed = [] + + def _post_data(self, data): + """Post data to gacos website.""" + r = requests.post(self.gacos_url, data=data) + return "Thanks for using GACOS!" in r.text + + def post_requests(self): + # post gacos info to website + datetime_patches = self.dataset.gen_datetime_patches() + for _key, _dates in tqdm( + datetime_patches.items(), + desc="submitting times", + unit="times", + ): + try: + for _dt in tqdm(_dates, desc="submitting dates", unit="dates"): + post_data = self.dataset.gen_post_data( + _dt, _key.split(":"), self.email + ) + status_ok = self._post_data(post_data) + if status_ok: + self._succeed.append(post_data) + tqdm.write(f">>> succeed post: {post_data}") + else: + self._failed.append(post_data) + tqdm.write(f">>> failed post: {post_data}") + + # wait to avoid be rejected + sleep_time = np.random.randint(*self.sleep_time_range) + tqdm.write(f" sleeping for {sleep_time} seconds...") + time.sleep(sleep_time) + except: + self._failed.append(post_data) + tqdm.write(f">>> failed post: {post_data}") + + @property + def failed(self): + """A list of failed post data.""" + return self._failed + + @property + def succeed(self): + """A list of succeed post data.""" + return self._succeed diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/pyproject.toml b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/pyproject.toml new file mode 100644 index 0000000..c1a0f62 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/AutoGACOS/pyproject.toml @@ -0,0 +1,23 @@ +[project] +name = "AutoGACOS" +authors = [{name = "Fancy", email = "fanchengyan@outlook.com"}] +description = "A Python library for automatically submitting and downloading GACOS data" +version = "0.1.0" +requires-python = ">=3.6" +dependencies = [ + "data_downloader", + "faninsar", +] +readme = "README.md" +license = {file = "LICENSE"} +keywords = ["GACOS", "data", "downloader"] + + +[project.optional-dependencies] +dev = [ + "pytest" +] + +[project.urls] +Homepage = "https://github.com/Fanchengyan/AutoGACOS" +Repository = "https://github.com/Fanchengyan/AutoGACOS" diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/DEM_DOWNLOAD_GUIDE.md b/.codex_tmp/pyint_variants/no_rescue/pyint/DEM_DOWNLOAD_GUIDE.md new file mode 100644 index 0000000..ecabdcf --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/DEM_DOWNLOAD_GUIDE.md @@ -0,0 +1,265 @@ +# DEM 数据下载指南 + +本文档介绍如何使用 `makedem.py` 下载和处理不同来源的 DEM 数据。 + +## 支持的 DEM 数据源 + +### 1. Copernicus DEM (默认推荐) + +**特点:** +- 分辨率: 30米 +- 覆盖: 全球 (60°S - 80°N) +- 数据质量: 高精度,全球一致性 +- 下载方式: 自动从 AWS/OpenTopography 下载 +- 费用: 免费 + +**使用方法:** +```bash +# 基本使用 +python makedem.py -r 116/117/39/40 --dem-source copernicus + +# 指定并行下载数量(加速下载) +python makedem.py -r 116/117/39/40 --dem-source copernicus --num-workers 8 + +# 指定输出目录 +python makedem.py -r 116/117/39/40 --dem-source copernicus --dir /path/to/output +``` + +**优点:** +- ✓ 全自动下载 +- ✓ 高精度数据 +- ✓ 全球覆盖 +- ✓ 支持并行下载加速 + +**缺点:** +- ✗ 需要网络连接 +- ✗ 下载时间取决于网络速度 + +--- + +### 2. NASADEM + +**特点:** +- 分辨率: 30米 +- 覆盖: 全球 (60°S - 60°N) +- 数据质量: SRTM + ICESat 数据融合 +- 下载方式: 通过 NASADEM Python 库自动下载 +- 费用: 免费 + +**依赖安装:** +```bash +pip install nasadem rasters +``` + +**使用方法:** +```bash +# 基本使用 +python makedem.py -r 116/117/39/40 --dem-source nasadem + +# 指定输出目录 +python makedem.py -r 116/117/39/40 --dem-source nasadem --dir /path/to/output +``` + +**优点:** +- ✓ 全自动下载 +- ✓ NASA官方数据 +- ✓ 融合多源数据,精度提升 + +**缺点:** +- ✗ 需要网络连接 +- ✗ 首次下载可能较慢 +- ✗ 需要额外安装库 +- ✗ 覆盖范围小于 Copernicus (南北纬60°限制) + +--- + +### 3. SRTM (需要预下载数据) + +**特点:** +- 分辨率: 30米 (SRTM 1 arc-second) 或 90米 (SRTM 3 arc-second) +- 覆盖: 全球 (60°S - 60°N) +- 数据质量: 良好 +- 下载方式: 需要手动预下载 .hgt 文件 +- 费用: 免费 + +**依赖安装:** +```bash +# 注意: srtm库需要 Python >= 3.12 +pip install srtm + +# 检查Python版本 +python --version # 应该显示 3.12 或更高版本 +``` + +**Python版本要求:** +- srtm 库要求 Python >= 3.12 +- 如果您的Python版本 < 3.12，建议使用 Copernicus 或 NASADEM + +**数据准备步骤:** + +1. **下载 SRTM .hgt 文件** + + 从以下网站之一下载所需区域的 .hgt 或 .hgt.zip 文件: + - **CSI-CGIAR SRTM**: https://srtm.csi.cgiar.org/ (推荐,无需注册) + - **USGS EarthExplorer**: https://earthexplorer.usgs.gov/ (需要注册) + - **NASA EarthData**: https://urs.earthdata.nasa.gov/ (需要注册) + +2. **文件命名规则** + + SRTM 文件按 1°×1° 分块命名: + - `N39E116.hgt` → 北纬39-40°, 东经116-117° + - `N39W110.hgt` → 北纬39-40°, 西经110-109° + - `S10E120.hgt` → 南纬10-9°, 东经120-121° + +3. **数据组织** + + 将所有 .hgt 文件放在同一目录下,例如: + ``` + /path/to/srtm_data/ + ├── N39E116.hgt + ├── N39E117.hgt + ├── N40E116.hgt + └── N40E117.hgt + ``` + +**使用方法:** +```bash +# 指定SRTM数据目录 +python makedem.py -r 116/117/39/40 --dem-source srtm --srtm-data-dir /path/to/srtm_data + +# 注意:如果不指定 --srtm-data-dir,程序会报错并提示 +python makedem.py -r 116/117/39/40 --dem-source srtm +# 错误: 使用 SRTM 数据源需要指定 --srtm-data-dir 参数 +``` + +**优点:** +- ✓ 离线使用(数据预下载后) +- ✓ 不依赖实时网络下载 +- ✓ 数据源多样化 + +**缺点:** +- ✗ 需要手动下载数据文件 +- ✗ 需要自行管理数据文件 +- ✗ 需要确保覆盖目标区域的所有分块 +- ✗ srtm 库对 .hgt.zip 压缩文件支持不稳定,建议解压后使用 + +--- + +## 使用建议 + +### 推荐使用顺序: + +1. **首选: Copernicus DEM** + - 全自动,高精度,全球覆盖 + - 使用 `--num-workers 8` 加速下载 + +2. **备选: NASADEM** + - 全自动,NASA官方数据 + - 适合北纬60°以南,南纬60°以北区域 + +3. **离线环境: SRTM** + - 仅在无网络环境或已有SRTM数据时使用 + - 需要提前准备数据文件 + +### 性能对比: + +| DEM源 | 下载速度 | 数据精度 | 全球覆盖 | 离线使用 | 推荐指数 | +|-------|---------|---------|---------|---------|---------| +| Copernicus | ★★★★☆ | ★★★★★ | ★★★★★ | ✗ | ★★★★★ | +| NASADEM | ★★★☆☆ | ★★★★☆ | ★★★☆☆ | ✗ | ★★★★☆ | +| SRTM | N/A (离线) | ★★★★☆ | ★★★☆☆ | ✓ | ★★★☆☆ | + +### 常见问题: + +**Q: 如何确定需要下载哪些 SRTM 文件?** + +A: 根据目标区域范围计算所需文件: +```python +# 示例: 区域 116/117/39/40 (东经116-117°, 北纬39-40°) +west, east, south, north = 116, 117, 39, 40 + +# 需要下载的文件: +# N39E116.hgt (39-40°N, 116-117°E) +# 如果区域更大,需要下载多个文件 +``` + +**Q: SRTM .hgt.zip 文件可以直接使用吗?** + +A: srtm 库对压缩文件支持不稳定,建议先解压: +```bash +cd /path/to/srtm_data/ +unzip "*.hgt.zip" +``` + +**Q: 如何加速 Copernicus DEM 下载?** + +A: 增加并行下载数量: +```bash +# 默认使用 4 个线程 +python makedem.py -r 116/117/39/40 --dem-source copernicus --num-workers 4 + +# 加速到 8-16 个线程(根据网络情况调整) +python makedem.py -r 116/117/39/40 --dem-source copernicus --num-workers 16 +``` + +**Q: 下载的数据格式是什么?** + +A: 所有DEM源最终都会转换为 GAMMA/ROI_PAC 格式: +- `.dem` 文件: 二进制高程数据 (big-endian for GAMMA) +- `.dem.par` 文件: 参数文件 + +--- + +## 示例工作流程 + +### 场景1: 处理Sentinel-1数据(推荐Copernicus) + +```bash +# 1. 确定研究区域范围 +# 使用 SLC 参数文件自动确定 +python makedem.py -s /path/to/slc.par --dem-source copernicus --num-workers 8 + +# 或手动指定区域 +python makedem.py -r 116/117/39/40 --dem-source copernicus --num-workers 8 +``` + +### 场景2: 离线处理(使用SRTM) + +```bash +# 1. 提前下载SRTM数据 +# 从 https://srtm.csi.cgiar.org/ 下载 N39E116.hgt, N39E117.hgt 等 + +# 2. 组织数据 +mkdir -p ~/data/SRTM +mv N*.hgt ~/data/SRTM/ + +# 3. 使用预下载数据生成DEM +python makedem.py -r 116/117/39/40 --dem-source srtm --srtm-data-dir ~/data/SRTM +``` + +### 场景3: NASADEM对比测试 + +```bash +# 下载Copernicus DEM +python makedem.py -r 116/117/39/40 --dem-source copernicus --dir ./copernicus + +# 下载NASADEM +python makedem.py -r 116/117/39/40 --dem-source nasadem --dir ./nasadem + +# 对比两个DEM文件 +# 使用 GMT 或其他工具进行可视化和分析 +``` + +--- + +## 参考资料 + +- **Copernicus DEM**: https://copernicus-dem-90m.s3.amazonaws.com/readme.html +- **NASADEM**: https://github.com/DFS-iData/NASADEM +- **SRTM CSI-CGIAR**: https://srtm.csi.cgiar.org/ +- **GAMMA Software**: https://gamma-rs.ch/ + +--- + +**最后更新**: 2026-03-13 +**作者**: iFlow CLI diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ALOS.py b/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ALOS.py new file mode 100644 index 0000000..f90aadf --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ALOS.py @@ -0,0 +1,193 @@ +#! /usr/bin/env python +#''' +################################################################################## +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : June, 2019 # +# # +# Generate SLC from SAR_IMS_P1 data # +# # +################################################################################## +#''' +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +from pyint import _utils as ut + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close + +def rm(TXT): + call_str = 'rm ' + TXT + os.system(call_str) + +def usage(): + print(''' +****************************************************************************************************** + + Generate SLC and SLC_par file for ERS/ENVISAT (SAR_IMS_1P format) + + usage: + + Down2SLC_ERS.py ProjectName DownName + + e.g. Down2SLC_ERS.py CotopaxiT120ERSA 910101 + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==3: + projectName = sys.argv[1] + Date = sys.argv[2] + else: + usage();sys.exit(1) + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + slcDir = scratchDir + '/' + projectName + '/SLC' + workflow = downDir + '/' + Date + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + if not os.path.isdir(slcDir): + call_str= 'mkdir ' +slcDir + os.system(call_str) + os.chdir(workflow) + tempfile=workflow + '/' + 'workreport' + if os.path.isfile(tempfile): + tempDict=ut.update_template(tempfile) + CEOS_SAR_leader = tempDict['Pdi_L10ProductFileName02'].split('"')[1] + CEOS_raw_data = tempDict['Pdi_L10ProductFileName03'].split('"')[1] + Date0 = Date +# if len(Date)==6: +# Date0 = Date +# elif len(Date)==8: +# Date0 = Date[2:8] + #Date0=Date +# else: +# print('The input Date is invalid.') +# sys.exit(1) + + #FileDir = downDir + '/' + downName + SAR_par='palsar_' +Date0 + '.par' + PROC_par='p' + Date0 + '.slc.par' + raw_out=Date0 + '.raw' + plot_data=Date0 +'.mlcc' + rspec_data=Date0 +'.rspec' + doppler_data=Date0 + '.dop' + rc_data=Date0 + '.rc' + autof_data=Date0 +'.autof' + slc_data=Date0 +'.slc' + slc_par=Date0 +'.slc.par' + call_str = 'PALSAR_proc ' + CEOS_SAR_leader + ' ' + SAR_par +' '+ PROC_par + ' '+ CEOS_raw_data + ' ' + raw_out + ' 0 0' + os.system(call_str) + palsar_ant_data='/home/chen/Software/InSAR/GAMMA/GAMMA_SOFTWARE-20180704/MSP/sensors/palsar_ant_20061024.dat' + call_str = 'cp ' + palsar_ant_data + ' .' + os.system(call_str) + call_str = 'PALSAR_antpat' + ' ' + SAR_par +' '+ PROC_par + ' ' + palsar_ant_data + ' palsar_antpat_msp.dat' + os.system(call_str) + call_str = 'doppler ' + SAR_par +' '+ PROC_par + ' ' + raw_out + ' ' + plot_data + os.system(call_str) + call_str = 'doppler ' + SAR_par +' '+ PROC_par + ' ' + raw_out + ' ' + doppler_data + os.system(call_str) + call_str = 'rspec_IQ ' + SAR_par +' '+ PROC_par + ' ' + raw_out + ' ' + rspec_data + os.system(call_str) + #call_str = 'rspec_JERS ' + SAR_par +' '+ PROC_par + ' ' + CEOS_raw_data + ' ' + rspec_data + ' - - - - - -' + #os.system(call_str) + call_str = 'pre_rc ' + SAR_par +' '+ PROC_par + ' ' + raw_out + ' ' + rc_data + os.system(call_str) + call_str = 'autof ' + SAR_par +' '+ PROC_par + ' ' + rc_data + ' ' + autof_data +' 5.0' + os.system(call_str) + call_str = 'autof ' + SAR_par +' '+ PROC_par + ' ' + rc_data + ' ' + autof_data +' 5.0' + os.system(call_str) + call_str = 'az_proc ' + SAR_par +' '+ PROC_par + ' ' + rc_data + ' ' + slc_data +' 16384' + os.system(call_str) + call_str = 'par_MSP ' + SAR_par +' '+ PROC_par + ' ' + slc_par + os.system(call_str) + else: + print('No data is found for date:' + Date) + sys.exit(1) + + call_str ="rename 's/VV.SLC/slc/g' *" + os.system(call_str) + + Date0 = Date + + dataDir = slcDir + '/' + Date0 + if not os.path.isdir(dataDir): + call_str = 'mkdir ' + dataDir + print('Generate SLC dir for date: ' + Date0) + os.system(call_str) + call_str = 'mv ' + Date0 + '.slc* ' + dataDir + os.system(call_str) + print("Down to SLC for %s is done! " % Date) + SslcImg = dataDir + '/'+ Date0 + '.slc' + SslcPar = dataDir + '/'+ Date0 + '.slc.par' + + SamprlksImg = dataDir + '/'+ Date0 + '_' + rlks + 'rlks' + '.amp' + SamprlksPar = dataDir + '/'+ Date0 + '_' + rlks + 'rlks' + '.amp.par' + call_str = 'multi_look ' + SslcImg + ' ' + SslcPar + ' ' + SamprlksImg + ' ' + SamprlksPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + + nWidth = ut.read_gamma_par(SamprlksPar, 'read', 'range_samples') + call_str = 'raspwr ' + SamprlksImg + ' ' + str(nWidth) + os.system(call_str) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ASAR_Cat.py b/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ASAR_Cat.py new file mode 100644 index 0000000..5594a52 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ASAR_Cat.py @@ -0,0 +1,307 @@ +#! /usr/bin/env python +#''' +################################################################################## +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : June, 2019 # +# # +# Generate SLC from SAR_IMS_P1 data # +# # +################################################################################## +#''' +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close + +def rm(TXT): + call_str = 'rm ' + TXT + os.system(call_str) + +def usage(): + print(''' +****************************************************************************************************** + + Generate SLC and SLC_par file for ERS/ENVISAT (SAR_IMS_1P format) + + usage: + + Down2SLC_ASAR_Cat.py ProjectName DownName + + e.g. Down2SLC_ASAR_Cat.py CotopaxiT120ERSA 910101 + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==3: + projectName = sys.argv[1] + Date = sys.argv[2] + else: + usage();sys.exit(1) + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + '/' + projectName + '.template' + templateContents=read_template(templateFile) + + if 'rlks4cor' in templateContents: rlks4cor = templateContents['rlks4cor'] + else: rlks4cor = '4' + if 'azlks4cor' in templateContents: azlks4cor = templateContents['azlks4cor'] + else: azlks4cor = '4' + + if 'rwin4cor' in templateContents: rwin4cor = templateContents['rwin4cor'] + else: rwin4cor = '128' + if 'azwin4cor' in templateContents: azwin4cor = templateContents['azwin4cor'] + else: azwin4cor = '128' + if 'rsample4cor' in templateContents: rsample4cor = templateContents['rsample4cor'] + else: rsample4cor = '64' + if 'azsample4cor' in templateContents: azsample4cor = templateContents['azsample4cor'] + else: azsample4cor = '64' + + if ' rpos4cor' in templateContents: rpos4cor = templateContents[' rpos4cor'] + else: rpos4cor = ' - ' + if 'azpos4cor' in templateContents: azpos4cor = templateContents['azpos4cor'] + else: azpos4cor = ' - ' + + + if 'rfwin4cor' in templateContents: rfwin4cor = templateContents['rfwin4cor'] + else: rfwin4cor = str(int(int(rwin4cor)/2)) + if 'azfwin4cor' in templateContents: azfwin4cor = templateContents['azfwin4cor'] + else: azfwin4cor = str(int(int(azwin4cor)/2)) + if 'rfsample4cor' in templateContents: rfsample4cor = templateContents['rfsample4cor'] + else: rfsample4cor = str(2*int(rsample4cor)) + if 'azfsample4cor' in templateContents: azfsample4cor = templateContents['azfsample4cor'] + else: azfsample4cor = str(2*int(azsample4cor)) + + if 'thresh4cor' in templateContents: thresh4cor = templateContents['thresh4cor'] + else: thresh4cor = ' 0.2 ' + + rlks = templateContents['Range_Looks'] + azlks = templateContents['Azimuth_Looks'] + + + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + slcDir = scratchDir + '/' + projectName + '/SLC' + + if not os.path.isdir(slcDir): + call_str= 'mkdir ' +slcDir + os.system(call_str) + os.chdir(downDir) + + t0 = 't0_' + Date + call_str = 'ls >' + t0 + os.system(call_str) + + tt = 'tt_' + Date + call_str = "grep " + Date + ' ' + t0 + '> ' + tt + os.system(call_str) + + te = 'te_' + Date + call_str = "grep ASA_IMS_1P " + tt + " > " + te + os.system(call_str) + + AA= np.loadtxt(te,dtype=np.str) + Na = AA.size + AA=AA.reshape(Na,) + + for i in range(Na): + Date0 = Date+'_' + str(i) + downName = str(AA[i]) + FileDir = downDir + '/' + downName + call_str = 'par_ASAR '+ FileDir + ' ' + Date0 + os.system(call_str) + + call_str ="rename 's/VV.SLC/slc/g' *" + os.system(call_str) + + slcpar = Date0 + '.slc.par' + call_str = 'ASAR_orb_cor_par.py ' + slcpar + os.system(call_str) + + Date0 = Date + if len(Date)==6: + Date0 = Date + Date6 = Date + elif len(Date)==8: + Date0 = Date[2:8] + Date6 = Date[2:8] + else: + print('The input Date is invalid.') + sys.exit(1) + + dataDir = slcDir + '/' + Date0 + if not os.path.isdir(dataDir): + call_str = 'mkdir ' + dataDir + print('Generate SLC dir for date: ' + Date0) + os.system(call_str) + call_str = 'mv ' + Date + '*.slc* ' + dataDir + os.system(call_str) + + os.chdir(dataDir) + for i in range(Na): + Date0 = Date+'_' + str(i) + downName = str(AA[i]) + + SLCm = Date0 + '.slc' + SLCm_par = Date0 + '.slc.par' + MamprlksImg = Date0 + '.amp' + MamprlksPar = Date0 + '.amp.par' + + call_str = 'multi_look ' + SLCm + ' ' + SLCm_par + ' ' + MamprlksImg + ' ' + MamprlksPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + nWidth = UseGamma(MamprlksPar, 'read', 'range_samples') + call_str = 'raspwr ' + MamprlksImg + ' ' + nWidth + os.system(call_str) + + SLC = Date6+'.slc' + SLCPAR = Date6 + '.slc.par' + AMP = Date6+'.amp' + AMPPAR = Date6+'.amp.par' + if Na==1: + os.rename(SLCm,SLC) + os.rename(SLCm_par,SLCPAR) + os.rename(MamprlksImg,AMP) + os.rename(MamprlksPar,AMPPAR) + + + for i in range(Na-1): + if i==0: + DateA = Date + '_0' + else: + DateA = Date + '_' + str(i-1) + str(i) + + SLCA = DateA + '.slc' + SLCA_par = DateA + '.slc.par' + + + DateB = Date + '_' +str(i+1) + SLCB = DateB + '.slc' + SLCB_par = DateB + '.slc.par' + + DateC = Date + '_' + str(i) + str(i+1) + SLCC = DateC + '.slc' + SLCC_par = DateC + '.slc.par' + + ##################################################################################### + MamprlksImg = Date + '.amp' + MamprlksPar = Date + '.amp.par' + + off = DateC + '.off' + offs = DateC + '.offs' + offsets = DateC + '.offsets' + coffs = DateC + '.coffs' + coffsets = DateC + '.coffsets' + snr = DateC + '.snr' + off_std = DateC + '.off_std' + + ########################## Generate off file ############################# + + call_str = "create_offset " + SLCA_par + " " + SLCB_par + " " + off + " 1 - - 0" + os.system(call_str) + call_str = 'init_offset_orbit '+ SLCA_par + " " + SLCB_par + ' ' + off + os.system(call_str) + + + call_str = 'init_offset '+ SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' ' + rlks4cor + ' ' + azlks4cor + ' ' + rpos4cor + ' ' + azpos4cor + os.system(call_str) + + call_str = 'init_offset '+ SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' 1 1 - - ' + os.system(call_str) + + call_str = "offset_pwr " + SLCA + " " + SLCB + " " + SLCA_par + " " + SLCB_par + " " + off + " " + offs + " " + snr + " " + rwin4cor + " " + azwin4cor + " " + offsets + " 2 "+ rsample4cor + " " + azsample4cor + os.system(call_str) + + call_str = "offset_fit " + offs + " " + snr + " " + off + " " + coffs + " " + coffsets + " " + thresh4cor +" 3" + os.system(call_str) + + call_str = "offset_pwr " +SLCA + " " + SLCB + " " + SLCA_par + " " + SLCB_par + " " + off + " " + offs + " " + snr + " " + rfwin4cor + " " + azfwin4cor + " " + offsets + " 2 " + rfsample4cor + " " + azfsample4cor + os.system(call_str) + + call_str = "offset_fit " + offs + " " + snr + " " + off + " " + coffs + " " + coffsets + " " + thresh4cor + " 4 >" + off_std + os.system(call_str) + + ######################################################################################## + + call_str = 'SLC_cat ' + SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' ' + SLCC + ' ' + SLCC_par + os.system(call_str) + + if i==(Na-2): + if len(Date)==6: + DD = Date + else: + DD = Date[2:8] + SLCm = DD + '.slc' + SLCm_par = DD + '.slc.par' + + + call_str = 'cp ' + SLCC + ' ' + SLCm + os.system(call_str) + + call_str = 'cp ' + SLCC_par + ' ' + SLCm_par + os.system(call_str) + + call_str = 'multi_look ' + SLCm + ' ' + SLCm_par + ' ' + MamprlksImg + ' ' + MamprlksPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + nWidth = UseGamma(MamprlksPar, 'read', 'range_samples') + call_str = 'raspwr ' + MamprlksImg + ' ' + nWidth + os.system(call_str) + + call_str = 'rm *.amp' + os.system(call_str) + + call_str = 'rm *_*.slc' + os.system(call_str) + + print("Down to SLC for %s is done! " % Date) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ASAR_Cat_All.py b/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ASAR_Cat_All.py new file mode 100644 index 0000000..cd7c04a --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ASAR_Cat_All.py @@ -0,0 +1,136 @@ +#! /usr/bin/env python +#''' +################################################################################## +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : March, 2017 # +# # +# Generate Sentinel SLC from the downloaded data # +# # +################################################################################## +#''' +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close + +def rm(TXT): + call_str = 'rm ' + TXT + os.system(call_str) + +def usage(): + print(''' +****************************************************************************************************** + + Downloading Sentinel-1A/B data based on ssara + + usage: + + Down2SLC_ERS_Cat_All.py ProjectName + + e.g. Down2SLC_ERS_Cat_All.py CotopaxiT120ERSA + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==2: + projectName = sys.argv[1] + else: + usage();sys.exit(1) + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + slcDir = scratchDir + '/' + projectName + '/SLC' + + if not os.path.isdir(slcDir): + call_str= 'mkdir ' +slcDir + os.system(call_str) + + os.chdir(downDir) + + call_str = 'ls > tt0' + os.system(call_str) + + call_str = 'grep ASA_IMS_1P tt0 >tt1' + os.system(call_str) + + call_str = "awk -F_ '{print $3}' tt1 > tt2 " + os.system(call_str) + + call_str = "awk -FSA '{print $2}' tt2 > ttt" + os.system(call_str) + + call_str = 'sort ttt | uniq > ttm' + os.system(call_str) + + AA= np.loadtxt('ttm',dtype=np.str) + Na = AA.size + + for i in range(Na): + call_str = 'Down2SLC_ASAR_Cat.py ' + projectName + ' ' + AA[i] + print(call_str) + call_str = 'Down2SLC_ASAR_Cat.py ' + projectName + ' ' + AA[i] + ' >/dev/null' + os.system(call_str) + + + print("Down to SLC for %s is done! " % projectName) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ERS.py b/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ERS.py new file mode 100644 index 0000000..868a153 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ERS.py @@ -0,0 +1,155 @@ +#! /usr/bin/env python +#''' +################################################################################## +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : June, 2019 # +# # +# Generate SLC from SAR_IMS_P1 data # +# # +################################################################################## +#''' +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close + +def rm(TXT): + call_str = 'rm ' + TXT + os.system(call_str) + +def usage(): + print(''' +****************************************************************************************************** + + Generate SLC and SLC_par file for ERS/ENVISAT (SAR_IMS_1P format) + + usage: + + Down2SLC_ERS.py ProjectName DownName + + e.g. Down2SLC_ERS.py CotopaxiT120ERSA 910101 + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==3: + projectName = sys.argv[1] + Date = sys.argv[2] + else: + usage();sys.exit(1) + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + slcDir = scratchDir + '/' + projectName + '/SLC' + + if not os.path.isdir(slcDir): + call_str= 'mkdir ' +slcDir + os.system(call_str) + os.chdir(downDir) + + t0 = 't0_' + Date + call_str = 'ls >' + t0 + os.system(call_str) + + tt = 'tt_' + Date + call_str = "grep " + Date + ' ' + t0 + '> ' + tt + os.system(call_str) + + te = 'te_' + Date + call_str = "grep SAR_IMS_1P " + tt + " > " + te + os.system(call_str) + + AA= np.loadtxt(te,dtype=np.str) + Na = AA.size + + if Na > 0: + downName = str(AA[0]) + FileDir = downDir + '/' + downName + call_str = 'par_ASAR '+ FileDir + ' ' + Date + os.system(call_str) + else: + print('No data is found for date:' + Date) + sys.exit(1) + + call_str ="rename 's/VV.SLC/slc/g' *" + os.system(call_str) + + Date0 = Date + if len(Date)==6: + Date0 = Date + elif len(Date)==8: + Date0 = Date[2:8] + else: + print('The input Date is invalid.') + sys.exit(1) + + dataDir = slcDir + '/' + Date0 + if not os.path.isdir(dataDir): + call_str = 'mkdir ' + dataDir + print('Generate SLC dir for date: ' + Date0) + os.system(call_str) + call_str = 'mv ' + Date + '.slc* ' + dataDir + os.system(call_str) + + print("Down to SLC for %s is done! " % Date) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ERS_All.py b/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ERS_All.py new file mode 100644 index 0000000..2eefb4a --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ERS_All.py @@ -0,0 +1,229 @@ +#! /usr/bin/env python +#''' +################################################################################## +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : March, 2017 # +# # +# Generate Sentinel SLC from the downloaded data # +# # +################################################################################## +#''' +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close + +def rm(TXT): + call_str = 'rm ' + TXT + os.system(call_str) + +def usage(): + print(''' +****************************************************************************************************** + + Downloading Sentinel-1A/B data based on ssara + + usage: + + Down2SLC_Sen_Gamma.py ProjectName DownName + + e.g. Down2SLC_Sen_Gamma.py CotopaxiT120SenVVA 170118 + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==3: + projectName = sys.argv[1] + Date = sys.argv[2] + else: + usage();sys.exit(1) + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + slcDir = scratchDir + '/' + projectName + '/SLC' + + if not os.path.isdir(slcDir): + call_str= 'mkdir ' +slcDir + os.system(call_str) + + os.chdir(downDir) + + + t0 = 't0_' + Date + call_str = 'ls >' + t0 + os.system(call_str) + + tt = 'tt_' + Date + call_str = "grep " + Date + ' ' + t0 + '> ' + tt + os.system(call_str) + + ts = 'ts_' + Date + call_str = "grep SAFE " + tt + ' >' + ts + os.system(call_str) + + tz = 'tz_' + Date + call_str = "grep zip " + tt + " > " + tz + os.system(call_str) + + A1= np.loadtxt(ts,dtype=np.str) + Na1 = A1.size + + A2= np.loadtxt(tz,dtype=np.str) + Na2 = A2.size + + rm(t0);rm(tt);rm(ts);rm(tz) + + if Na1 == 0: + if Na2 > 0: + if Na2 == 1: + downName = str(A2) + else: + downName = str(A2[0]) + FileDir = downDir + '/' + downName + RAWNAME = downName.split('.')[0]+'.SAFE' + call_str = 'unzip '+ FileDir + os.system(call_str) + + else: + if Na1 == 1: + RAWNAME = str(A1) + else: + RAWNAME = str(A1[0]) + + print(RAWNAME) + RAWFILEDir = downDir + '/'+str(RAWNAME) + + + Date = RAWNAME[19:25] + DateDir = slcDir + '/'+Date + + if not os.path.isdir(DateDir): + call_str='mkdir '+DateDir + os.system(call_str) + + measureDir = RAWFILEDir + '/measurement' + annotatDir = RAWFILEDir + '/annotation' + calibraDir = RAWFILEDir + '/annotation/calibration' + + MM = glob.glob(measureDir + '/*vv*tiff') +# MEASURE = glob.glob(measureDir + '/*vv*tiff') +# ANNOTAT = glob.glob(annotatDir + '/*vv*xml' ) +# CALIBRA = glob.glob(calibraDir+'/calibration*vv*') +# NOISE = glob.glob(calibraDir+'/noise*vv*') + + SLC_Tab = DateDir + '/' + Date+'_SLC_Tab' + TEST = DateDir + '/' + Date + '.IW1.slc' + + if not os.path.isfile(TEST): + if os.path.isfile(SLC_Tab): + os.remove(SLC_Tab) + for kk in range(len(MM)): + SLC = DateDir + '/' + Date + '.IW' + str(kk+1)+'.slc' + SLCPar = DateDir + '/' + Date + '.IW' + str(kk+1)+'.slc.par' + TOPPar = DateDir + '/' + Date + '.IW' + str(kk+1)+'.slc.TOPS_par' + BURST = DateDir + '/' + Date + '.IW' + str(kk+1)+'.burst.par' + + if os.path.isfile(BURST): + os.remove(BURST) + call_str = 'echo ' + SLC + ' ' + SLCPar + ' ' + TOPPar + ' >> ' + SLC_Tab + os.system(call_str) + + MEASURE = glob.glob(measureDir + '/*iw' + str(kk+1) + '*vv*tiff') + ANNOTAT = glob.glob(annotatDir + '/*iw' + str(kk+1) + '*vv*xml' ) + CALIBRA = glob.glob(calibraDir+'/calibration*'+ 'iw' + str(kk+1) + '*vv*') + NOISE = glob.glob(calibraDir+'/noise*' + 'iw' + str(kk+1) + '*vv*') + + #call_str = 'S1_burstloc ' + ANNOTAT[0] + '> ' +BURST + #os.system(call_str) + + if int(Date) > 180311: + call_str = 'par_S1_SLC ' + MEASURE[0] + ' ' + ANNOTAT[0] + ' ' + CALIBRA[0] + ' - ' + SLCPar + ' ' + SLC + ' ' + TOPPar + else: + call_str = 'par_S1_SLC ' + MEASURE[0] + ' ' + ANNOTAT[0] + ' ' + CALIBRA[0] + ' ' + NOISE[0] + ' ' + SLCPar + ' ' + SLC + ' ' + TOPPar + + os.system(call_str) + + call_str = 'SLC_burst_corners ' + SLCPar + ' ' + TOPPar + ' > ' +BURST + os.system(call_str) + + TSLC = DateDir + '/' + Date + '.slc' + TSLCPar = DateDir + '/' + Date + '.slc.par' + + TMLI = DateDir + '/' + Date + '_20rlks.amp' + TMLIPar = DateDir + '/' + Date + '_20rlks.amp.par' + + call_str = 'SLC_mosaic_S1_TOPS ' + SLC_Tab + ' ' + TSLC + ' ' + TSLCPar + ' 10 2' + os.system(call_str) + + call_str = 'multi_look ' + TSLC + ' ' + TSLCPar + ' ' + TMLI + ' ' + TMLIPar + ' 20 4' + os.system(call_str) + + nWidth = UseGamma(TMLIPar, 'read','range_samples:') + call_str = 'raspwr ' + TMLI + ' ' + nWidth + ' - - - - - - - ' + os.system(call_str) + + call_str = 'rm -rf ' + RAWFILEDir + os.system(call_str) + + print("Down to SLC for %s is done! " % Date) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ERS_Cat.py b/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ERS_Cat.py new file mode 100644 index 0000000..1b32f36 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ERS_Cat.py @@ -0,0 +1,306 @@ +#! /usr/bin/env python +#''' +################################################################################## +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : June, 2019 # +# # +# Generate SLC from SAR_IMS_P1 data # +# # +################################################################################## +#''' +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close + +def rm(TXT): + call_str = 'rm ' + TXT + os.system(call_str) + +def usage(): + print(''' +****************************************************************************************************** + + Generate SLC and SLC_par file for ERS/ENVISAT (SAR_IMS_1P format) + + usage: + + Down2SLC_ERS.py ProjectName DownName + + e.g. Down2SLC_ERS.py CotopaxiT120ERSA 910101 + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==3: + projectName = sys.argv[1] + Date = sys.argv[2] + else: + usage();sys.exit(1) + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + '/' + projectName + '.template' + templateContents=read_template(templateFile) + + if 'rlks4cor' in templateContents: rlks4cor = templateContents['rlks4cor'] + else: rlks4cor = '4' + if 'azlks4cor' in templateContents: azlks4cor = templateContents['azlks4cor'] + else: azlks4cor = '4' + + if 'rwin4cor' in templateContents: rwin4cor = templateContents['rwin4cor'] + else: rwin4cor = '128' + if 'azwin4cor' in templateContents: azwin4cor = templateContents['azwin4cor'] + else: azwin4cor = '128' + if 'rsample4cor' in templateContents: rsample4cor = templateContents['rsample4cor'] + else: rsample4cor = '64' + if 'azsample4cor' in templateContents: azsample4cor = templateContents['azsample4cor'] + else: azsample4cor = '64' + + if ' rpos4cor' in templateContents: rpos4cor = templateContents[' rpos4cor'] + else: rpos4cor = ' - ' + if 'azpos4cor' in templateContents: azpos4cor = templateContents['azpos4cor'] + else: azpos4cor = ' - ' + + + if 'rfwin4cor' in templateContents: rfwin4cor = templateContents['rfwin4cor'] + else: rfwin4cor = str(int(int(rwin4cor)/2)) + if 'azfwin4cor' in templateContents: azfwin4cor = templateContents['azfwin4cor'] + else: azfwin4cor = str(int(int(azwin4cor)/2)) + if 'rfsample4cor' in templateContents: rfsample4cor = templateContents['rfsample4cor'] + else: rfsample4cor = str(2*int(rsample4cor)) + if 'azfsample4cor' in templateContents: azfsample4cor = templateContents['azfsample4cor'] + else: azfsample4cor = str(2*int(azsample4cor)) + + if 'thresh4cor' in templateContents: thresh4cor = templateContents['thresh4cor'] + else: thresh4cor = ' 0.2 ' + + rlks = templateContents['Range_Looks'] + azlks = templateContents['Azimuth_Looks'] + + + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + slcDir = scratchDir + '/' + projectName + '/SLC' + + if not os.path.isdir(slcDir): + call_str= 'mkdir ' +slcDir + os.system(call_str) + os.chdir(downDir) + + t0 = 't0_' + Date + call_str = 'ls >' + t0 + os.system(call_str) + + tt = 'tt_' + Date + call_str = "grep " + Date + ' ' + t0 + '> ' + tt + os.system(call_str) + + te = 'te_' + Date + call_str = "grep SAR_IMS_1P " + tt + " > " + te + os.system(call_str) + + AA= np.loadtxt(te,dtype=np.str) + Na = AA.size + AA=AA.reshape(Na,) + + for i in range(Na): + Date0 = Date+'_' + str(i) + downName = str(AA[i]) + FileDir = downDir + '/' + downName + call_str = 'par_ASAR '+ FileDir + ' ' + Date0 + os.system(call_str) + + call_str ="rename 's/VV.SLC/slc/g' *" + os.system(call_str) + + slcpar = Date0 + '.slc.par' + call_str = 'ERS_orb_cor_par.py ' + slcpar + os.system(call_str) + + Date0 = Date + if len(Date)==6: + Date6 = Date + Date0 = Date + elif len(Date)==8: + Date0 = Date[2:8] + Date6 = Date[2:8] + else: + print('The input Date is invalid.') + sys.exit(1) + + dataDir = slcDir + '/' + Date0 + if not os.path.isdir(dataDir): + call_str = 'mkdir ' + dataDir + print('Generate SLC dir for date: ' + Date0) + os.system(call_str) + call_str = 'mv ' + Date + '*.slc* ' + dataDir + os.system(call_str) + + os.chdir(dataDir) + for i in range(Na): + Date0 = Date+'_' + str(i) + downName = str(AA[i]) + + SLCm = Date0 + '.slc' + SLCm_par = Date0 + '.slc.par' + MamprlksImg = Date0 + '.amp' + MamprlksPar = Date0 + '.amp.par' + + call_str = 'multi_look ' + SLCm + ' ' + SLCm_par + ' ' + MamprlksImg + ' ' + MamprlksPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + nWidth = UseGamma(MamprlksPar, 'read', 'range_samples') + call_str = 'raspwr ' + MamprlksImg + ' ' + nWidth + os.system(call_str) + + SLC = Date6+'.slc' + SLCPAR = Date6 + '.slc.par' + AMP = Date6+'.amp' + AMPPAR = Date6+'.amp.par' + if Na==1: + os.rename(SLCm,SLC) + os.rename(SLCm_par,SLCPAR) + os.rename(MamprlksImg,AMP) + os.rename(MamprlksPar,AMPPAR) + + for i in range(Na-1): + if i==0: + DateA = Date + '_0' + else: + DateA = Date + '_' + str(i-1) + str(i) + + SLCA = DateA + '.slc' + SLCA_par = DateA + '.slc.par' + + + DateB = Date + '_' +str(i+1) + SLCB = DateB + '.slc' + SLCB_par = DateB + '.slc.par' + + DateC = Date + '_' + str(i) + str(i+1) + SLCC = DateC + '.slc' + SLCC_par = DateC + '.slc.par' + + ##################################################################################### + MamprlksImg = Date + '.amp' + MamprlksPar = Date + '.amp.par' + + off = DateC + '.off' + offs = DateC + '.offs' + offsets = DateC + '.offsets' + coffs = DateC + '.coffs' + coffsets = DateC + '.coffsets' + snr = DateC + '.snr' + off_std = DateC + '.off_std' + + ########################## Generate off file ############################# + + call_str = "create_offset " + SLCA_par + " " + SLCB_par + " " + off + " 1 - - 0" + os.system(call_str) + call_str = 'init_offset_orbit '+ SLCA_par + " " + SLCB_par + ' ' + off + os.system(call_str) + + + call_str = 'init_offset '+ SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' ' + rlks4cor + ' ' + azlks4cor + ' ' + rpos4cor + ' ' + azpos4cor + os.system(call_str) + + call_str = 'init_offset '+ SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' 1 1 - - ' + os.system(call_str) + + call_str = "offset_pwr " + SLCA + " " + SLCB + " " + SLCA_par + " " + SLCB_par + " " + off + " " + offs + " " + snr + " " + rwin4cor + " " + azwin4cor + " " + offsets + " 2 "+ rsample4cor + " " + azsample4cor + os.system(call_str) + + call_str = "offset_fit " + offs + " " + snr + " " + off + " " + coffs + " " + coffsets + " " + thresh4cor +" 3" + os.system(call_str) + + call_str = "offset_pwr " +SLCA + " " + SLCB + " " + SLCA_par + " " + SLCB_par + " " + off + " " + offs + " " + snr + " " + rfwin4cor + " " + azfwin4cor + " " + offsets + " 2 " + rfsample4cor + " " + azfsample4cor + os.system(call_str) + + call_str = "offset_fit " + offs + " " + snr + " " + off + " " + coffs + " " + coffsets + " " + thresh4cor + " 4 >" + off_std + os.system(call_str) + + ######################################################################################## + + call_str = 'SLC_cat ' + SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' ' + SLCC + ' ' + SLCC_par + os.system(call_str) + + if i==(Na-2): + if len(Date)==6: + DD = Date + else: + DD = Date[2:8] + SLCm = DD + '.slc' + SLCm_par = DD + '.slc.par' + + + call_str = 'cp ' + SLCC + ' ' + SLCm + os.system(call_str) + + call_str = 'cp ' + SLCC_par + ' ' + SLCm_par + os.system(call_str) + + call_str = 'multi_look ' + SLCm + ' ' + SLCm_par + ' ' + MamprlksImg + ' ' + MamprlksPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + nWidth = UseGamma(MamprlksPar, 'read', 'range_samples') + call_str = 'raspwr ' + MamprlksImg + ' ' + nWidth + os.system(call_str) + + call_str = 'rm *.amp' + os.system(call_str) + + call_str = 'rm *_*.slc' + os.system(call_str) + + print("Down to SLC for %s is done! " % Date) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ERS_Cat_All.py b/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ERS_Cat_All.py new file mode 100644 index 0000000..d256db3 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/Down2SLC_ERS_Cat_All.py @@ -0,0 +1,136 @@ +#! /usr/bin/env python +#''' +################################################################################## +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : March, 2017 # +# # +# Generate Sentinel SLC from the downloaded data # +# # +################################################################################## +#''' +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close + +def rm(TXT): + call_str = 'rm ' + TXT + os.system(call_str) + +def usage(): + print(''' +****************************************************************************************************** + + Downloading Sentinel-1A/B data based on ssara + + usage: + + Down2SLC_ERS_Cat_All.py ProjectName + + e.g. Down2SLC_ERS_Cat_All.py CotopaxiT120ERSA + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==2: + projectName = sys.argv[1] + else: + usage();sys.exit(1) + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + slcDir = scratchDir + '/' + projectName + '/SLC' + + if not os.path.isdir(slcDir): + call_str= 'mkdir ' +slcDir + os.system(call_str) + + os.chdir(downDir) + + call_str = 'ls > tt0' + os.system(call_str) + + call_str = 'grep SAR_IMS_1P tt0 >tt1' + os.system(call_str) + + call_str = "awk -F_ '{print $3}' tt1 > tt2 " + os.system(call_str) + + call_str = "awk -FSA '{print $2}' tt2 > ttt" + os.system(call_str) + + call_str = 'sort ttt | uniq > ttm' + os.system(call_str) + + AA= np.loadtxt('ttm',dtype=np.str) + Na = AA.size + + for i in range(Na): + call_str = 'Down2SLC_ERS_Cat.py ' + projectName + ' ' + AA[i] + print(call_str) + call_str = 'Down2SLC_ERS_Cat.py ' + projectName + ' ' + AA[i] + ' >/dev/null' + os.system(call_str) + + + print("Down to SLC for %s is done! " % projectName) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/ERS_DEOS.py b/.codex_tmp/pyint_variants/no_rescue/pyint/ERS_DEOS.py new file mode 100644 index 0000000..f97c88e --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/ERS_DEOS.py @@ -0,0 +1,206 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Central South University & University of Miami ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import linecache + +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +######################################################################### + +INTRODUCTION = ''' +############################################################################# + Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 + + Download precise ERS orbit data from Delft (http://www.deos.tudelft.nl/). + Correct the orbit parameters using DELFT_vec2 + + ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-2/dgm-e04/arclist + + +''' + +EXAMPLE = ''' + Usage: + ERS_DEOS.py projectName Date + + Examples: + ERS_DEOS.py AqabaERSA 960101 + +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Download (and correct) precise ERS orbit data.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='project name') + parser.add_argument('date',help='date of the ERS data') + + inps = parser.parse_args() + return inps + +################################################################################ + + +def main(argv): + + total = time.time() + inps = cmdLineParse() + + projectName = inps.projectName + DATE = inps.date + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + slcDir = scratchDir + '/' + projectName + '/SLC' + projDir = scratchDir + '/' + projectName + + ERS1_Orb_Dir = scratchDir + '/' + projectName + '/ERS1_Orb' + ERS2_Orb_Dir = scratchDir + '/' + projectName + '/ERS2_Orb' + + if not os.path.isdir(ERS1_Orb_Dir): + call_str = 'mkdir ' + ERS1_Orb_Dir + os.system(call_str) + + if not os.path.isdir(ERS2_Orb_Dir): + call_str = 'mkdir ' + ERS2_Orb_Dir + os.system(call_str) + + ERS1_Orb_list = ERS1_Orb_Dir + '/ERS1_Orb_list' + ERS2_Orb_list = ERS2_Orb_Dir + '/ERS2_Orb_list' + + os.chdir(ERS1_Orb_Dir) + if not os.path.isfile(ERS1_Orb_list): + Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-1/dgm-e04/arclist' + call_str = 'wget ' + Url + ' -O ERS1_Orb_list' + os.system(call_str) + + call_str = 'cp ERS1_Orb_list arclist' + os.system(call_str) + + os.chdir(ERS2_Orb_Dir) + if not os.path.isfile(ERS2_Orb_list): + Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-2/dgm-e04/arclist' + call_str = 'wget ' + Url + ' -O ERS2_Orb_list' + os.system(call_str) + + call_str = 'cp ERS2_Orb_list arclist' + os.system(call_str) + + if len(DATE)==8: + DATE6 = DATE[2:8] + else: + DATE6 = DATE + + slcDir1 = slcDir + '/' + DATE6 + slcpar = slcDir1 + '/' + DATE6 + '.slc.par' + Title = UseGamma(slcpar, 'read', 'title:') + if 'E1' in Title: + Flag = 'ERS1' + Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-1/dgm-e04/' + List = ERS1_Orb_list + Dir = ERS1_Orb_Dir + elif 'E2' in Title: + Flag = 'ERS2' + Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-2/dgm-e04/' + List = ERS2_Orb_list + Dir = ERS2_Orb_Dir + else: + print('The SAR data is invalid.') + sys.exit(1) + + if len(DATE)==6: + A0 = int(DATE[0:2]) + if A0 < 50: + DATE = '20' + DATE + else: + DATA = '19' + DATE + + YYMM = DATE6[0:2] + DATE6[2:4] + + call_str = 'grep ' + YYMM + ' ' + List + ' > list0' + os.system(call_str) + + call_str = "awk '{print $12}' list0 > start_end" + os.system(call_str) + + call_str = "awk '{print $1}' list0 > vec_num" + os.system(call_str) + + AA= np.loadtxt('start_end',dtype=np.str) + Na = AA.size + + print(AA) + + A_num= np.loadtxt('vec_num',dtype=np.str) + Na_num = A_num.size + + for i in range(Na-1): + A0 = AA[i] + A1 = AA[i+1] + print(DATE6) + print(A0) + print(A1) + if int(DATE6)==int(A0): + m = i + elif int(DATE6)==int(A1): + m = i+1 + elif int(A0) < int(DATE6) < int(A1): + m = i + + FF = A_num[int(m)] + SS = Url + 'ODR.' + FF + print(SS) + + call_str = 'wget -q --no-check-certificate ' + SS + ' -P ' + Dir + os.system(call_str) + + + print("Download precise DEFT orbital data for %s is done." % DATE) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/ERS_orb_cor.py b/.codex_tmp/pyint_variants/no_rescue/pyint/ERS_orb_cor.py new file mode 100644 index 0000000..696cd99 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/ERS_orb_cor.py @@ -0,0 +1,132 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Central South University & University of Miami ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import linecache + +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +######################################################################### + +INTRODUCTION = ''' +############################################################################# + Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 + + Precise ERS orbit data from Delft (http://www.deos.tudelft.nl/). + Correct the orbit parameters using DELFT_vec2 + +''' + +EXAMPLE = ''' + Usage: + ERS_orb_cor.py projectName Date + + Examples: + ERS_orb_cor.py AqabaERSA 960101 + +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Download (and correct) precise ERS orbit data.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='project name') + parser.add_argument('date',help='date of the ERS data') + + inps = parser.parse_args() + return inps + +################################################################################ + + +def main(argv): + + total = time.time() + inps = cmdLineParse() + + projectName = inps.projectName + DATE = inps.date + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + Ers1OrbDir = os.getenv('ERS1ORBDIR') + Ers2OrbDir = os.getenv('ERS2ORBDIR') + + slcDir = scratchDir + '/' + projectName + '/SLC' + projDir = scratchDir + '/' + projectName + + + if len(DATE)==8: + DATE6 = DATE[2:8] + else: + DATE6 = DATE + + slcDir1 = slcDir + '/' + DATE6 + slcpar = slcDir1 + '/' + DATE6 + '.slc.par' + Title = UseGamma(slcpar, 'read', 'title:') + if 'E1' in Title: + Flag = 'ERS1' + #Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-1/dgm-e04/' + #List = ERS1_Orb_list + Dir = Ers1OrbDir + elif 'E2' in Title: + Flag = 'ERS2' + #Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-2/dgm-e04/' + #List = ERS2_Orb_list + Dir = Ers2OrbDir + else: + print('The SAR data is invalid.') + sys.exit(1) + + call_str = 'DELFT_vec2 ' + slcpar + ' ' + Dir + ' 30' + os.system(call_str) + + + print("Using DEFT orbital data for %s is done." % DATE) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/ERS_orb_cor_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/ERS_orb_cor_all.py new file mode 100644 index 0000000..ae81bb8 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/ERS_orb_cor_all.py @@ -0,0 +1,126 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Central South University & University of Miami ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import linecache + +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +def is_number(s): + try: + int(s) + return True + except ValueError: + return False + +######################################################################### + +INTRODUCTION = ''' +############################################################################# + Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 + + Precise ERS orbit data from Delft (http://www.deos.tudelft.nl/). + Correct the orbit parameters using DELFT_vec2 + +''' + +EXAMPLE = ''' + Usage: + ERS_orb_cor_all.py projectName + + Examples: + ERS_orb_cor_all.py AqabaERSA + +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Download (and correct) precise ERS orbit data.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='project name') + + inps = parser.parse_args() + return inps + +################################################################################ + + +def main(argv): + + total = time.time() + inps = cmdLineParse() + + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + slcDir = scratchDir + '/' + projectName + '/SLC' + + ListSLC = os.listdir(slcDir) + Datelist = [] + SLCfile = [] + SLCParfile = [] + + + for kk in range(len(ListSLC)): + if ( is_number(ListSLC[kk]) and len(ListSLC[kk])==6 ): # if SAR date number is 8, 6 should change to 8. + DD=ListSLC[kk] + Year=int(DD[0:2]) + Month = int(DD[2:4]) + Day = int(DD[4:6]) + Datelist.append(ListSLC[kk]) + N = len(Datelist) + + for i in range(N): + call_str = 'ERS_orb_cor.py ' + projectName + ' ' + Datelist[i] + print(call_str) + os.system(call_str) + + print("Using DEFT orbital data for project %s is done." % projectName) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/ERS_orb_cor_par.py b/.codex_tmp/pyint_variants/no_rescue/pyint/ERS_orb_cor_par.py new file mode 100644 index 0000000..2514d7c --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/ERS_orb_cor_par.py @@ -0,0 +1,118 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Central South University & University of Miami ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import linecache + +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +######################################################################### + +INTRODUCTION = ''' +############################################################################# + Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 + + Precise ERS orbit data from Delft (http://www.deos.tudelft.nl/). + Correct the orbit parameters using DELFT_vec2 + +''' + +EXAMPLE = ''' + Usage: + ERS_orb_cor_par.py SLC_par + + Examples: + ERS_orb_cor_par.py 960101.slc.par + +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Download (and correct) precise ERS orbit data.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('slc_par',help='slc par file') + + inps = parser.parse_args() + return inps + +################################################################################ + + +def main(argv): + + total = time.time() + inps = cmdLineParse() + + slcpar = inps.slc_par + + Ers1OrbDir = os.getenv('ERS1ORBDIR') + Ers2OrbDir = os.getenv('ERS2ORBDIR') + + Title = UseGamma(slcpar, 'read', 'title:') + if 'E1' in Title: + Flag = 'ERS1' + #Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-1/dgm-e04/' + #List = ERS1_Orb_list + Dir = Ers1OrbDir + elif 'E2' in Title: + Flag = 'ERS2' + #Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-2/dgm-e04/' + #List = ERS2_Orb_list + Dir = Ers2OrbDir + else: + print('The SAR data is invalid.') + sys.exit(1) + + call_str = 'DELFT_vec2 ' + slcpar + ' ' + Dir + os.system(call_str) + + + print("Using DEFT orbital data for %s is done." % slcpar) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/GACOS_correction.csh b/.codex_tmp/pyint_variants/no_rescue/pyint/GACOS_correction.csh new file mode 100644 index 0000000..882b07d --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/GACOS_correction.csh @@ -0,0 +1,131 @@ +#!/bin/csh -f +# $Id$ +#GACOS Correction +#Erik Rivas, Oct 18 2021 +# + +if ($#argv != 5) then + echo "" + echo "Usage: GACOS.csh list_interferograms full_path_to_GACOS_data full_path_topo reference_point incidence_angle" + echo "Script needs to be run inside the intf_all folder" + echo "Performs gacos correction" + echo "" + echo "list_interferograms: list of folders inside the intf_all folder containing the phases and coherence grids" + echo "" + echo "full_path_to_GACOS_data" + echo "Example: /home/erikr/gacos/" + echo "" + echo "full_path_topo" + echo "Example: /home/erikr/project/topo/" + echo "" + echo "the topo folder needs to have: dem.grd, master.PRM, and the correspondant .LED file" + echo "" + echo "list_interferograms: list of folders with interferograms created" + echo "" + echo "Reference point in lon lat coordinates (text file)" + echo "" + echo "Indicence angle in degrees from SAT_look (float/integer)" + echo "" + echo "Outputs: phasefilt.grd files corrected and added as additional products in each interferogram folder. These outputs should be used for the unwrap processing" + exit 1 +endif + +set list = $1 +set GACOS_dir = $2 +set topo_dir = $3 +set reference_point = $4 +set incidence = $5 + +#Checking inputs +if !(-e $list) then + echo "$list seems not to exist" + exit 1 +endif +if !(-d $GACOS_dir) then + echo "$GACOS_dir seems not to exist" + exit 1 +endif +if !(-d $topo_dir) then + echo "$topo_dir seems not to exist" + exit 1 +endif +if !(-f $reference_point) then + echo "Reference point file: $reference_point seems not to exist. Provide a text file with lon lat values" + exit 1 +endif + + +#PROJECT POINT FROM LON-LAT TO RADAR COORDINATES +set ref_llh = $topo_dir"ref.llh" +set out_ratll = $topo_dir"out.ratll" +set reference_point_ra = $topo_dir"ref_point.ra" +if (-f $ref_llh) then + echo "Removing old $ref_llh" + rm $ref_llh +endif +if (-f $out_ratll) then + echo "Removing old $out_ratll" + rm $out_ratll +endif +if (-f $reference_point_ra) then + echo "Removing old $reference_point_ra" + rm $reference_point_ra +endif + +gmt grdtrack $reference_point -G$topo_dir"dem.grd" >> $ref_llh +ln -s $topo_dir*.LED . +SAT_llt2rat $topo_dir"master.PRM" 0 < $topo_dir"ref.llh" > $out_ratll +rm *.LED +cat $out_ratll |awk '{print $1, $2}' > $reference_point_ra +set dem_grd = $topo_dir"dem.grd" +#----------------------------------------------# + +#FOR LOOP OVER LIST OF INTERFEROGRAMS +foreach dir (`awk '{print $1}' $list`) + if !(-d $dir) then + echo "$dir directory seems not to exist" + exit 1 + endif + + cd $dir + + #Check if there are only two SLC files (not sure if this step is neccessary) + if (`ls *.SLC|wc -l` != "2") then + echo "the number of SLC files is inconsistent" + endif + + #ls sorts content alphanumeric, therefore it is assume the first as the master on the list + set fst_date = `ls *.SLC|sed -n '1p'|awk '{print substr($1,4,8)}'` + set scd_date = `ls *.SLC|sed -n '2p'|awk '{print substr($1,4,8)}'` + + #Save directory of current intf + set intf_dir = `pwd` + + cd $GACOS_dir + + + #Check if the GACOS files are in the folder + if (-f $fst_date".ztd" && -f $fst_date".ztd.rsc" && -f $scd_date".ztd" && -f $scd_date".ztd.rsc") then + set first_ztd = $GACOS_dir$fst_date".ztd" + set first_rsc = $GACOS_dir$fst_date".ztd.rsc" + set second_ztd = $GACOS_dir$scd_date".ztd" + set second_rsc = $GACOS_dir$scd_date".ztd.rsc" + + #GACOS Correction + cd $intf_dir + #Link trans.dat to each folder. Neccesary to project ztd grids to radar coordinates + ln -s $topo_dir"trans.dat" + operation.csh $first_ztd $first_rsc $second_ztd $second_rsc $reference_point_ra $incidence $dem_grd + rm trans.dat + + else + + echo "GACOS files do not exist / Wrong directory" + exit 1 + + endif + + cd .. +end + +echo "GACOS correction done" diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/Get_off_std.py b/.codex_tmp/pyint_variants/no_rescue/pyint/Get_off_std.py new file mode 100644 index 0000000..18775a2 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/Get_off_std.py @@ -0,0 +1,145 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Central South University & University of Miami ### +################################################################# + + +import os +import sys +import glob +import time +import argparse + +import h5py +import numpy as np + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +def UseGamma2(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[2].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +def usage(): + print(''' +****************************************************************************************************** + + Get co-registration standard deviation of SLCs for one project + + usage: + + GenerateRSC_Gamma.py projectName + + e.g. Get_off_std.py GalapagosT061EnvA + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==2: + if argv[0] in ['-h','--help']: usage(); sys.exit(1) + else: projectName=sys.argv[1] + else: + usage();sys.exit(1) + + scratchDir = os.getenv('SCRATCHDIR') + OFFDir = scratchDir + '/' + projectName + '/RSLC' + OFFSTR = OFFDir + '/*.off_std' + OFFFile = glob.glob(OFFSTR) + STD_TXT ='COREG_STD_ALL' + os.chdir(OFFDir) + if os.path.isfile(STD_TXT): + os.remove(STD_TXT) + + for ff in OFFFile: + OFF_STD = ff + NM = os.path.basename(ff).split('.')[0] + RR = UseGamma(OFF_STD,'read','final range offset poly. coeff.:') + cor_rg = RR.split(' ')[0] + + AA = UseGamma(OFF_STD,'read','final azimuth offset poly. coeff.:') + cor_az = AA.split(' ')[0] + + STDRR = UseGamma(OFF_STD,'read','final model fit std. dev. (samples) range:') + std_rg=STDRR.split(' ')[0] + + std_az = UseGamma2(OFF_STD,'read','final model fit std. dev. (samples) range:') + + STR = NM + ' ' + cor_rg + ' ' + cor_az + ' ' + std_rg + ' ' + std_az + call_str ='echo ' + STR + ' >> ' + STD_TXT + os.system(call_str) + + + sys.exit(1) + + +if __name__ == '__main__': + main(sys.argv[1:]) + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/.gitignore b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/.gitignore new file mode 100644 index 0000000..764a6ac --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/.gitignore @@ -0,0 +1,5 @@ +*.swp +bin/ +*.tif +*.pyc +__pycache__/ diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/LICENSE b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_01_Read_Data.py b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_01_Read_Data.py new file mode 100644 index 0000000..a9815c1 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_01_Read_Data.py @@ -0,0 +1,582 @@ +#!/usr/bin/env python3 + +import os +import re +from datetime import datetime, timedelta +import numpy as np +import rasterio +from bin.multilook_w import multilook_w +from bin.multilook import multilook +import configparser +import sys +import warnings +import pickle +import glob + +warnings.filterwarnings("ignore", category=DeprecationWarning) + +####################################################################### + +HELP_TEXT = """ +PhaseBias_01_Read_Data.py + +This script automates the process of downloading GeoTIFF files for wrapped interferograms and coherence images. +The files can be retrieved either from the COMET-LiCS web portal or from a root path specified in the configuration file. + +Assumptions: +- Interferograms are organized in folders named as `yyyymmdd_yyyymmdd` (e.g., `20230101_20230107`). + +Files Downloaded: +1. `yyyymmdd_yyyymmdd.geo.diff_pha.tif`: + - Contains the wrapped phase image in radians. + - Values range from -3.14 to 3.14. + +2. `yyyymmdd_yyyymmdd.geo.cc.tif`: + - Contains the coherence image of the interferometric pair. + - Values range from 0 to 255, where: + - 0 represents the lowest coherence. + - 255 represents the highest coherence. + +Outputs: +1. `All_ifgs_start_end` (Pickle file, saved as `.pkl`): + - A dictionary containing all interferograms available between the specified start and end dates. +2. `All_coh_start_end` (Pickle file, saved as `.pkl`): + - A dictionary containing all coherence data available between the specified start and end dates. + +Storage Information: +- All output files are saved in the directory `Data/` under the `output_path` defined in the configuration file. + +Additional Output: +- A summary report is generated that includes: + - The number of available interferograms for each temporal baseline. + - The number of missing interferograms. + +""" + +if "--help" in sys.argv: + print(HELP_TEXT) + sys.exit(0) + +################### + +# Config file path +config_file = "config.txt" + +# Check if config file exists +if not os.path.exists(config_file): + raise FileNotFoundError( + f"Error: The configuration file '{config_file}' was not found in the script's directory." + ) + +# Initialize the configparser +config = configparser.ConfigParser() + +# Read the config.txt file +config.read(config_file) + +# Define required parameters +required_parameters = [ + "root_path", + "frame", + "start", + "end", + "interval", + "nlook", + "LiCSAR_data", +] + +# Initialize a dictionary to store parameters +parameters = {} + +# Attempt to retrieve each parameter and catch errors +try: + parameters["root_path"] = config.get("DEFAULT", "root_path") + parameters["output_path"] = config.get("DEFAULT", "output_path") + parameters["frame"] = config.get("DEFAULT", "frame") + parameters["start"] = config.get("DEFAULT", "start") + parameters["end"] = config.get("DEFAULT", "end") + parameters["interval"] = config.getint("DEFAULT", "interval") + parameters["nlook"] = config.getint( + "DEFAULT", "nlook") # Assuming 'nlook' should be an integer + parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") + +except configparser.NoOptionError as e: + # Error message if a required parameter is missing + missing_param = str(e).split(": ")[1] + raise ValueError( + f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." + ) +except configparser.Error as e: + # General error for any configparser-related issues + raise ValueError(f"Error while reading '{config_file}': {e}") + +# Assign to individual variables for easy access +root_path = parameters["root_path"] +output_path = parameters["output_path"] +frame = parameters["frame"] +start = parameters["start"] +end = parameters["end"] +interval = parameters["interval"] +nlook = parameters["nlook"] +LiCSAR_data = parameters["LiCSAR_data"] + +# hardcoded parameters, not included in the config file +landmask = 1 +filtered_ifgs = "yes" +min_baseline = 5 +max_baseline = 366 + +# Print the values to confirm they're being read correctly +print("Root Path:", root_path) +print("output Path:", output_path) +print("Frame:", frame) +print("Start Date:", start) +print("End Date:", end) +print("Interval:", interval) +print("nlook:", nlook) +print("LiCSAR_data", LiCSAR_data) + +################ +# all_ifgs = read_ifgs(frame, start, end, min_baseline, max_baseline, landmask, nlook, interval, LiCSAR_data, filtered_ifgs) + +def read_ifgs( + start, + end, + min_baseline, + max_baseline, + landmask, + nlook, + interval, + LiCSAR_data, + filtered_ifgs, +): + + start_date = datetime.strptime(start, "%Y%m%d") + end_date = datetime.strptime(end, "%Y%m%d") + # Check if the file ends with "geo.diff_pha.tif" + # if filename.endswith("geo.diff_unfiltered_pha.tif"): # if unfiltered data are used + if filtered_ifgs == "yes": # yes for the filtered ifgs, no for the unfiltered ifgs + req_file_name = "geo.diff_pha.tif" + else: + req_file_name = "geo.diff_unfiltered_pha.tif" + + ############ reading + + if LiCSAR_data == "yes": + track = frame[0:3] # extracting track number from frame id + if track[0] == "0": + track = track[1:3] + if track[0] == "0": + track = track[1:2] + + root_directory = f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}/interferograms/" + else: + root_directory = os.path.join(root_path, "interferograms") # Use os.path.join for consistent path construction + #root_directory = os.path.join(root_path, "") + + ##### Count total files matching the criteria for accurate progress calculation + print(f"Reading data from the path: {root_directory}") + + total_files = sum(1 for dirpath, _, filenames in os.walk(root_directory) + for filename in filenames + if filename.endswith(req_file_name)) + processed_files = 0 + + print("Reading wrapped interferograms: [", end="", flush=True) + ##### + + # Initialize an empty dictionary to store the arrays for each category + category_arrays = {} + last_date = {} + n = 0 + + if landmask is not None: + if LiCSAR_data == "yes": + landmask_path = f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}/metadata/{frame}.geo.landmask.tif" + else: + # Use glob to find the file ending with ".geo.landmask.tif" in the metadata directory + metadata_dir = os.path.join(root_path, "metadata") + landmask_files = glob.glob( + os.path.join(metadata_dir, "*.geo.landmask.tif")) + if landmask_files: + landmask_path = landmask_files[0] # Take the first match + else: + raise FileNotFoundError( + "No file ending with '.geo.landmask.tif' found in the metadata directory." + ) + + # Load the landmask file + with rasterio.open(landmask_path) as src: + array_landmask = src.read(1) + + n = 1 + # Traverse the root directory and its subdirectories + for dirpath, dirnames, filenames in os.walk(root_directory): + dirnames.sort( + ) # This ensures dirnames are processed in alphabetical order + # Iterate through the files in the current directory + for filename in filenames: + # Check if the file ends with "geo.diff_pha.tif" + # if filename.endswith("geo.diff_unfiltered_pha.tif"): # if unfiltered data are used + if (filtered_ifgs == "yes" + ): # yes for the filtered ifgs, no for the unfiltered ifgs + req_file_name = "geo.diff_pha.tif" + else: + req_file_name = "geo.diff_unfiltered_pha.tif" + + if filename.endswith(req_file_name): + file_path = os.path.join(dirpath, filename) + + #### for printing into the console + processed_files += 1 + percentage = (processed_files / total_files) * 100 + + # Clear previous percentage and print updated one + sys.stdout.write( + "\rReading wrapped phases: [{:<50}] {}%".format( + "=" * int(percentage // 2), int(percentage))) + sys.stdout.flush() + + ##### + + # Extract the dates from the file name + match = re.search(r"(\d+)_(\d+)", filename) + if match: + date1 = match.group(1) + date2 = match.group(2) + + # Convert the dates to datetime objects + date1_obj = datetime.strptime(date1, "%Y%m%d") + date2_obj = datetime.strptime(date2, "%Y%m%d") + + # Check if the dates fall within the specified range + category = (date2_obj - date1_obj).days + + if (start_date <= date1_obj <= end_date + and start_date <= date2_obj <= end_date + and category < max_baseline + and category > min_baseline): + if ( + n == 1 + ): # finding the first and last acquisiton in the time-series + first_acq = date1_obj + else: + last_acq = date2_obj + n = n + 1 + + # Calculate the difference in days between the two dates + + # Open the TIFF file + try: + with rasterio.open(file_path) as src: + array_data = src.read(1).astype(np.float32) + except Exception as e: + print(f"\u26a0\ufe0f Error reading {file_path}: {e}. Appending None instead.") + array_data = None + + #if array_data is not None: + array_data[array_data == 0] = np.nan ## converting the zerso values in wrapped phases into nan. This is helpful when calculating loop closures + + #array_data[np.abs(array_data) < 1e-8] = np.nan # as after filter i observed low vlaues outside the frame. I also wanted to nulify them. so used this insteadof above + + + if landmask != None: #and array_data is not None: + array_data[array_landmask != 1] = np.nan + + if ( + nlook != None and nlook != 1 #and array_data is not None + ): # incase of unwrap data because they are already multilooked to 10 we don't multilook them here + array_data = multilook_w(array_data, nlook) + + if category not in last_date: + last_date[category] = date1_obj + # if (date1_obj - date2_obj) > interval + + # appending None to the category_arrays where there are missing ifgs in the middle of time-series + diff_ifgs = ( + date1_obj - last_date[category] + ).days # /category #difference between the first epochs of two consecutive ifg + if diff_ifgs > interval: + n_no_acq = int(diff_ifgs / interval - + 1) ## number of missing ifgs + for i in range( + n_no_acq + ): # the number of none depends on the number of missing ifgs + # Append the none to the existing array + category_arrays[category].append(None) + last_date[category] = date1_obj + + # Check if the category is already in the dictionary + if category in category_arrays: + # If the category already exists, append the array to the existing array + category_arrays[category].append(array_data) + else: + # If the category doesn't exist, create a new list with the array + category_arrays[category] = [array_data] + + # appending None to the category_arrays where there are missing ifgs in the end of time-series + for category in sorted(category_arrays): + while (last_date[category] + timedelta(category)) < last_acq: + category_arrays[category].append(None) + last_date[category] = last_date[category] + timedelta(interval) + + # appending None to the category_arrays where there are missing ifgs in the begining of time-series + # max number of expected 6-day(i.e. interval) interferograms in the full time-series + max_ifg_number = abs(int((first_acq - last_acq).days / interval)) + + # all the categories in the data e.g. 6, 12, 18 etc + cat = [] + for category in category_arrays: + cat.append(category) + + for i in range(interval, max(cat)+1, interval): + if i in cat: + while len(category_arrays[i]) < (max_ifg_number + 1 - i/interval): + category_arrays[i].insert( 0, None) + + print("\nFinished reading all interferograms.") + + return category_arrays + + +########################################################################################################################### +########################################################################################################################## + + +def read_coh(start, end, min_baseline, max_baseline, landmask, nlook, interval, + LiCSAR_data): + + start_date = datetime.strptime(start, "%Y%m%d") + end_date = datetime.strptime(end, "%Y%m%d") + ############ reading + + if LiCSAR_data == "yes": + track = frame[0:3] + if track[0] == "0": + track = track[1:3] + if track[0] == "0": + track = track[1:2] + + root_directory = f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}/interferograms/" + else: + root_directory = os.path.join( + root_path, "interferograms" + ) # Use os.path.join for consistent path construction + + req_file_name = "geo.cc.tif" + + ##### Count total files matching the criteria for accurate progress calculation + + total_files = sum(1 for dirpath, _, filenames in os.walk(root_directory) + for filename in filenames + if filename.endswith(req_file_name)) + processed_files = 0 + + print("Reading coherence data: [", end="", flush=True) + ##### + + # Initialize an empty dictionary to store the arrays for each category + category_arrays = {} + last_date = {} + n = 0 + + if landmask is not None: + if LiCSAR_data == "yes": + landmask_path = f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}/metadata/{frame}.geo.landmask.tif" + else: + # Use glob to find the file ending with ".geo.landmask.tif" in the metadata directory + metadata_dir = os.path.join(root_path, "metadata") + landmask_files = glob.glob( + os.path.join(metadata_dir, "*.geo.landmask.tif")) + if landmask_files: + landmask_path = landmask_files[0] # Take the first match + else: + raise FileNotFoundError( + "No file ending with '.geo.landmask.tif' found in the metadata directory." + ) + + # Load the landmask file + with rasterio.open(landmask_path) as src: + array_landmask = src.read(1) + + n = 1 + # Traverse the root directory and its subdirectories + for dirpath, dirnames, filenames in os.walk(root_directory): + # Iterate through the files in the current directory + for filename in filenames: + # Check if the file ends with "geo.cc.tif" + + if filename.endswith(req_file_name): + file_path = os.path.join(dirpath, filename) + + #### for printing into the console + processed_files += 1 + percentage = (processed_files / total_files) * 100 + + # Clear previous percentage and print updated one + sys.stdout.write( + "\rReading coherence data: [{:<50}] {}%".format( + "=" * int(percentage // 2), int(percentage))) + sys.stdout.flush() + + ##### + + # Extract the dates from the file name + match = re.search(r"(\d+)_(\d+)", filename) + if match: + date1 = match.group(1) + date2 = match.group(2) + + # Convert the dates to datetime objects + date1_obj = datetime.strptime(date1, "%Y%m%d") + date2_obj = datetime.strptime(date2, "%Y%m%d") + + # Check if the dates fall within the specified range + category = (date2_obj - date1_obj).days + + if (start_date <= date1_obj <= end_date + and start_date <= date2_obj <= end_date + and category < max_baseline + and category > min_baseline): + if ( + n == 1 + ): # finding the first and last acquisiton in the time-series + first_acq = date1_obj + else: + last_acq = date2_obj + n = n + 1 + + # Calculate the difference in days between the two dates + + # Open the TIFF file + try: + with rasterio.open(file_path) as src: + array_data = src.read(1).astype(np.float32) + except Exception as e: + print(f"\u26a0\ufe0f Error reading {file_path}: {e}. Appending None instead.") + array_data = None + + if landmask != 0: + array_data[array_landmask != 1] = np.nan + + if ( + nlook != None and nlook != 1 + ): # incase of unwrap data because they are already multilooked to 10 we don't multilook them here + array_data = multilook(array_data, nlook) + + if category not in last_date: + last_date[category] = date1_obj + # if (date1_obj - date2_obj) > interval + + # appending None to the category_arrays where there are missing ifgs in the middle of time-series + diff_ifgs = ( + date1_obj - last_date[category] + ).days # /category #difference between the first epochs of two consecutive ifg + if diff_ifgs > interval: + n_no_acq = int(diff_ifgs / interval - + 1) ## number of missing ifgs + for i in range( + n_no_acq + ): # the number of none depends on the number of missing ifgs + # Append the none to the existing array + category_arrays[category].append(None) + last_date[category] = date1_obj + + # Check if the category is already in the dictionary + if category in category_arrays: + # If the category already exists, append the array to the existing array + category_arrays[category].append(array_data) + else: + # If the category doesn't exist, create a new list with the array + category_arrays[category] = [array_data] + + # appending None to the category_arrays where there are missing ifgs in the end of time-series + for category in sorted(category_arrays): + while (last_date[category] + timedelta(category)) < last_acq: + category_arrays[category].append(None) + last_date[category] = last_date[category] + timedelta(interval) + + # appending None to the category_arrays where there are missing ifgs in the begining of time-series + # max number of expected 6-day(i.e. interval) interferograms in the full time-series + max_ifg_number = abs(int((first_acq - last_acq).days / interval)) + + # all the categories in the data e.g. 6, 12, 18 etc + cat = [] + for category in category_arrays: + cat.append(category) + + for i in range(interval, max(cat)+1, interval): + if i in cat: + while len(category_arrays[i]) < (max_ifg_number + 1 - i/interval): + category_arrays[i].insert(0, None) + + print("\nFinished reading all coherence data.") + + return category_arrays + + +########################################################################################################################## +########################################################################################################################## + +all_ifgs = read_ifgs( + start, + end, + min_baseline, + max_baseline, + landmask, + nlook, + interval, + LiCSAR_data, + filtered_ifgs, +) +# Define the directory path +directory_path = os.path.join(output_path, "Data") + +# Create the directory if it doesn't exist +os.makedirs(directory_path, exist_ok=True) + +# Define the file path +#file_path = os.path.join(directory_path, f"_all_ifgs_{start}_{end}") + +# Remove all files inside the directory +if os.path.exists(directory_path): + files = glob.glob(os.path.join(directory_path, "*")) + for file in files: + os.remove(file) + + +# for writing the all_loops +file_path = output_path + "/Data/" + "All_ifgs_" + start + "_" + end + ".pkl" +with open(file_path, "wb") as file: # for writting a dictionary to a file + pickle.dump(all_ifgs, file) + + +################################################ + +# After processing all interferograms, generate the report +print("\nReport on the number of IFGs and missing IFGs per category:") +for category in sorted(all_ifgs): + num_ifgs = sum(1 for item in all_ifgs[category] + if item is not None) # Count valid IFGs + num_missing_ifgs = sum(1 for item in all_ifgs[category] + if item is None) # Count missing IFGs + print( + f"{category}-days: Number of IFGs: {num_ifgs} Number of missing IFGs: {num_missing_ifgs}" + ) + +################################################################ + +import gc +del all_ifgs +gc.collect() + + + +all_coh = read_coh(start, end, min_baseline, max_baseline, landmask, nlook, + interval, LiCSAR_data) + +## for writting the all_coh +file_path = output_path + "/Data/" + "All_coh_" + start + "_" + end + ".pkl" +with open(file_path, "wb") as file: # for writting a dictionary to a file + pickle.dump(all_coh, file) + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_02_Loop_Closures.py b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_02_Loop_Closures.py new file mode 100644 index 0000000..c7a14d8 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_02_Loop_Closures.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 + +import numpy as np +import pickle +import glob +import configparser +import os +import sys +import warnings +warnings.filterwarnings("ignore", category=DeprecationWarning) + +################################################################################################ + +HELP_TEXT = """ +PhaseBias_02_Loop_Closures.py + +This script calculates loop closures (Δφ) using the interferograms imported in the first step. + +Definition of Loop Closures: +Loop closures, Δφ, are calculated for the epochs between `i` and `k`, and are defined as: + Δφ_(i,k) = |φ_(i,k) - ∑_(t=i)^k φ_(t,t+1)|_2π + +Where: +- φ_(i,j): Represents the phase difference for a pixel in the interferogram formed between epochs `i` and `j`. +- |.|_2π: Indicates that the result is wrapped modulo 2π (i.e., values range from -π to π). + +Key Information: +- Nonzero closure phase is a by-product of spatial filtering/multilooking and is primarily associated with changes in the scattering and electrical properties of the ground surface. +- The calculated loop closures are based on the minimum temporal-baseline interferograms defined in the configuration file (parameter: `interval`) and are referred to as base interferograms. These may represent: + - 12- and 6-day closures (or 24- and 12-day closures), Δφ_(i,i+2) + - 18- and 6-day closures (or 36- and 12-day closures), Δφ_(i,i+3) + The distinction depends on whether the base interferograms are 6-day or 12-day intervals. + +Outputs: +- The calculated loop closures are stored as a dictionary in a file named: + `All_loops_start_end.pkl` +- The output is saved in the directory `Data/` under the `output_path` defined in the configuration file. + +Loop Closure Report: +At the end of the script, a loop closure report is displayed. This report includes: +- The number of generated and missing loop closures for short-interval closures, such as 12-6 and 18-6. +- The number of generated and missing long-interval closures, obtained using long-term interferograms, such as 204-6 and 204-12. + +Usage: +1. Ensure the interferograms from the first script are available. +2. Run the script to generate loop closures for the specified temporal baselines. + +Example: +```bash +python PhaseBias_02_Loop_Closures.py + + +""" +if "--help" in sys.argv: + print(HELP_TEXT) + sys.exit(0) + +################################################################################################ +# Config file path +config_file = "config.txt" + +# Check if config file exists +if not os.path.exists(config_file): + raise FileNotFoundError( + f"Error: The configuration file '{config_file}' was not found in the script's directory." + ) + +# Initialize the configparser +config = configparser.ConfigParser() + +# Read the config.txt file +config.read(config_file) + +# Define required parameters +required_parameters = [ + "root_path", + "frame", + "start", + "end", + "interval", + "nlook", + "LiCSAR_data", +] + +# Initialize a dictionary to store parameters +parameters = {} + +# Attempt to retrieve each parameter and catch errors +try: + parameters["root_path"] = config.get("DEFAULT", "root_path") + parameters["output_path"] = config.get("DEFAULT", "output_path") + parameters["frame"] = config.get("DEFAULT", "frame") + parameters["start"] = config.get("DEFAULT", "start") + parameters["end"] = config.get("DEFAULT", "end") + parameters["interval"] = config.getint("DEFAULT", "interval") + parameters["nlook"] = config.getint( + "DEFAULT", "nlook") # Assuming 'nlook' should be an integer + parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") + +except configparser.NoOptionError as e: + # Error message if a required parameter is missing + missing_param = str(e).split(": ")[1] + raise ValueError( + f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." + ) +except configparser.Error as e: + # General error for any configparser-related issues + raise ValueError(f"Error while reading '{config_file}': {e}") + +# Assign to individual variables for easy access +root_path = parameters["root_path"] +output_path = parameters["output_path"] +frame = parameters["frame"] +start = parameters["start"] +end = parameters["end"] +interval = parameters["interval"] +nlook = parameters["nlook"] +LiCSAR_data = parameters["LiCSAR_data"] + +# hardcoded parameters, not included in the config file +landmask = 1 +filtered_ifgs = "yes" +max_loop = 5 # in case of 3 all loops with 6,12,18-day will be calculated (e.g. (60,6), (60,12) and (60,18)) + + +############################################################################################# +################################################################################################# +def loop_calc(max_loop): + + ## Read ifgs: + + # Define the path components + sub_dir = "Data" + filename_pattern = "All_ifgs_*" # Pattern to match files starting with "All_ifgs" + + # Construct the full file path with the refined pattern + file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) + # Use glob to find the file and ensure it exists + file_list = glob.glob(file_path_pattern) + if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + all_ifgs = pickle.load(file) + else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) + + loop = {} + missing = {} + + cat = sorted(all_ifgs.keys()) # Get sorted categories + # print('cat = ', cat) + + # Initialize counters for reporting + loop_counts = { + category: { + cat[l]: 0 + for l in range(max_loop) + } + for category in sorted(all_ifgs.keys()) + } + missing_counts = { + category: { + cat[l]: 0 + for l in range(max_loop) + } + for category in sorted(all_ifgs.keys()) + } + + for category in cat: + loop[category] = { + } # Initialize the key in the loop dictionary with an inner dictionary + missing[category] = [] # second ver + for l in range( + max_loop + ): # it was up to 3 in the first version. but this allows to go for longer loops e.g. 288-72 (cat[11] is 72) + loop[category][cat[l]] = [ + ] # Initialize the key in the inner dictionary + # missing[cat[l]] = [] # first ver + # Calculate total iterations for progress percentage + total_iterations = sum( + len(all_ifgs[cat[i]]) for l in range(max_loop) + for i in range(1, len(cat)) + if cat[i] % cat[l] == 0 and cat[i] != cat[l]) + completed_iterations = 0 + + # Display the message before starting the progress bar + print( + f"Calculating all {interval}, {2 * interval}, and {3 * interval} loop closures." + ) + + for l in range( + 0, max_loop + ): # l could be 6, 12 and 18 to calculate e.g. 36-6 or 36-12 or 36-18 + for i in range(1, len(cat)): # index for the category e.g. 6, 12, 18 + if cat[i] % cat[l] == 0 and cat[i] != cat[ + l]: # such as 12/6 or 24/12 + for t in range(len(all_ifgs[ + cat[i]])): # index for the epochs in each category + + ####################progress bar + completed_iterations += 1 + percentage = (completed_iterations / + total_iterations) * 100 + + # Display progress percentage + sys.stdout.write("\rProgress: [{:<50}] {}%".format( + "=" * int(percentage // 2), int(percentage))) + sys.stdout.flush() + ################################## + + if cat[l] == interval: + end_index = t + int(cat[i] / cat[l]) + # print('end_index = ', end_index) + else: + end_index = t + int(cat[i] / cat[l]) * int( + cat[l] / interval) + + # recording the missing ifgs in each cat e.g. 6,12,18 + #if np.any(np.array(all_ifgs[cat[i]])[t] == None): + if np.any(np.array(all_ifgs[cat[i]], dtype=object)[t] == None): + missing[cat[i]].append( + t + ) # to record the missing ifgs index for each cateogory e.g. 18 + + + + # --- Check for overshoot(not enough data at the tail). this allows the calculate the missing in the next for loop + if end_index > len(all_ifgs[cat[l]]): + # Not enough short IFGs to form a valid closure → skip this loop + loop[cat[i]][cat[l]].append(None) + missing_counts[cat[i]][cat[l]] += 1 + continue + + + + for e in range(t, end_index, int(cat[l]/interval)): # to record the missing ifgs index for the period of each loop + #print(f" Long baseline (cat[i]): {cat[i]}") + #print(f" Short baseline (cat[l]): {cat[l]}") + #print(f" Index t: {t}, e: {e}, end_index: {end_index}") + #print(f" len(all_ifgs[cat[l]]): {len(all_ifgs[cat[l]])}") + + elem = all_ifgs[cat[l]][e] + + if elem is None: + missing[cat[l]].append(e) + + + + if all_ifgs[cat[i]][t] is None or any(all_ifgs[cat[l]][elem] is None for elem in range(t, end_index, int(cat[l] / interval))): + + loop[cat[i]][cat[l]].append(None) + missing_counts[cat[i]][cat[l]] += 1 # Increment missing loop counter + + else: + closure = np.angle( np.exp( 1j * ( np.array(all_ifgs[cat[i]][t], dtype=np.float32)- np.sum( np.array(all_ifgs[cat[l]][t:end_index:int(cat[l] / interval)], dtype=np.float32), axis=0)))) + + if cat[i] in loop: + loop[cat[i]][cat[l]].append(closure) + loop_counts[cat[i]][cat[l]] += 1 # Increment successful loop counter + + else: + loop[cat[i]][cat[l]] = [closure] + print("\n Finished calculating the loop closures.") + + # Generate report + print("\n=== Loop Closure Report ===") + for category in [ + 2 * interval, + 3 * interval, + ]: # Restrict to relevant categories e.g. 18-6 and 12-6 as the main observations + for l in [interval]: # Restrict to the loop levels of interest + if category in loop_counts and l in loop_counts[category]: + generated = loop_counts[category].get(l, 0) + missing = missing_counts[category].get(l, 0) + print( + f"Loop Closure {category} - {l}: Generated = {generated}, Missing = {missing}" + ) + + print("\n=== Loop Closure Report for Long-interval Loop Closures ===") + + # Iterate over categories greater than 200 + for category in cat: + if category > 200: # Check for long categories + for l in range( + 1, + 5 * interval): # Check loop levels less than 5 * interval + if category in loop_counts and l in loop_counts[category]: + generated = loop_counts[category].get(l, 0) + if generated >= 1: # Only report if at least one loop is generated + print( + f"Loop Closure {category} - {l}: Generated = {generated}" + ) + + return loop, missing + + +################################################################################################ + +all_loops, missing = loop_calc(max_loop) + +# Define the directory path +directory_path = os.path.join(output_path, "Data") + +# Create the directory if it doesn't exist +os.makedirs(directory_path, exist_ok=True) + +# Define the file path +file_path = os.path.join(directory_path, f"All_loops_{start}_{end}.pkl") + +with open(file_path, "wb") as file: # for writting a dictionary to a file + pickle.dump(all_loops, file) + +### + +# Define the file path +file_path = os.path.join(directory_path, f"missing_loops") + +with open(file_path, "wb") as file: # for writting a dictionary to a file + pickle.dump(missing, file) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_03_calibration_pars.py b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_03_calibration_pars.py new file mode 100644 index 0000000..63c5c34 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_03_calibration_pars.py @@ -0,0 +1,424 @@ +#!/usr/bin/env python3 + +import numpy as np +import matplotlib.pyplot as plt +from matplotlib.gridspec import GridSpec +import pickle +import os +import sys +import glob +import configparser +import warnings + +with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=DeprecationWarning) + +################################################################################ + +HELP_TEXT = """ +PhaseBias_03_calibration_pars.py + +This script estimates the calibration parameters a_n using the loop closures calculated in the previous step. + +The calibration parameters a_n are estimated by calculating the ratio of the long-term loop closure for the interferogram of interest to the long-term loop closure for the base interferogram. Assuming negligible bias in long-term interferograms, a_n is calculated as follows: + + a_n = |φ(i,i+l) - Σ(t=i to i+l)φ(t,t+1+n)|_2π / |φ(i,i+l) - Σ(t=i to i+l)φ(t,t+1)|_2π + +where: +- φ(i,i+l) is a long-term interferogram connecting epoch i to i+l. +- φ(t,t+1) represents the base interferogram. +- φ(t,t+1+n) is the short interferogram to be corrected using a_n. + +### Input: +1. Loop closures from the previous step (e.g., 216-6, 216-12, and 216-18 for 6-day acquisition intervals). +2. The long-term interferogram length (e.g., 216 days) must be specified at the beginning. +3. The number of a_n parameters to estimate, specified using `num_a`. For example: + - Setting `num_a=2` indicates that a1 and a2 will be estimated (used to correct 12- and 18-day interferograms). + - For correcting longer interferograms, `num_a` should be set accordingly. + +### Output: +1. All possible a_n values are calculated. +2. Visualizations: + - Example plots of a_n arrays and their corresponding histograms, illustrating spatial variations. + - Time-series plots showing how a_n values evolve over time. +3. Mean a_n values over time are output for further use. + +### File Outputs: +- Results are stored in the output directory defined in the `config.txt` file under `/Data/` as `Mean_an_values.pkl`. + +### Additional Notes: +- Users should ensure that `num_a` aligns with the desired temporal baselines for correction. + +""" +if "--help" in sys.argv: + print(HELP_TEXT) + sys.exit(0) + + +############################################################################### +# Config file path +config_file = "config.txt" + +# Check if config file exists +if not os.path.exists(config_file): + raise FileNotFoundError( + f"Error: The configuration file '{config_file}' was not found in the script's directory." + ) + +# Initialize the configparser +config = configparser.ConfigParser() + + +# Read the config.txt file +config.read(config_file) + +# Define required parameters +required_parameters = [ + "root_path", + "frame", + "start", + "end", + "interval", + "nlook", + "LiCSAR_data", +] + +# Initialize a dictionary to store parameters +parameters = {} + +# Attempt to retrieve each parameter and catch errors +try: + parameters["root_path"] = config.get("DEFAULT", "root_path") + parameters["output_path"] = config.get("DEFAULT", "output_path") + parameters["frame"] = config.get("DEFAULT", "frame") + parameters["start"] = config.get("DEFAULT", "start") + parameters["end"] = config.get("DEFAULT", "end") + parameters["interval"] = config.getint("DEFAULT", "interval") + parameters["nlook"] = config.getint( + "DEFAULT", "nlook" + ) # Assuming 'nlook' should be an integer + parameters["num_a"] = config.getint("DEFAULT", "num_a") + parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") + +except configparser.NoOptionError as e: + # Error message if a required parameter is missing + missing_param = str(e).split(": ")[1] + raise ValueError( + f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." + ) +except configparser.Error as e: + # General error for any configparser-related issues + raise ValueError(f"Error while reading '{config_file}': {e}") + +# Assign to individual variables for easy access +root_path = parameters["root_path"] +output_path = parameters["output_path"] +frame = parameters["frame"] +start = parameters["start"] +end = parameters["end"] +interval = parameters["interval"] +nlook = parameters["nlook"] +num_a = parameters["num_a"] +LiCSAR_data = parameters["LiCSAR_data"] + +# hardcoded parameters, not included in the config file +landmask = 1 +filtered_ifgs = "yes" +max_loop = 5 # in case of 3 all loops with 6,12,18-day will be calculated (e.g. (60,6), (60,12) and (60,18)) + + +##################################################### Loop Closure calculation ##################################################### +#################################################################################################################################### +track = frame[0:3] +if track[0] == "0": + track = track[1:3] +if track[0] == "0": + track = track[1:2] + + +###################### Read coh: +print("\nReading all coherence data...") +# Define the path components +sub_dir = "Data" +filename_pattern = "All_coh_*" # Pattern to match files starting with "All_ifgs" + +# Construct the full file path with the refined pattern +file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + coh = pickle.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) +print("Reading coherence data completed.") + + +##################### Read Loops + +print("\nReading all loop closures...") +filename_pattern = "All_loops_*" # Pattern to match files starting with "All_ifgs" + +# Construct the full file path with the refined pattern +file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + all_loops = pickle.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) + +print("Reading loop closures completed.") + + +##################################################### Loop Closure calculation ##################################################### +#################################################################################################################################### + + +long_baseline = 216 # 36,72,108,144,180,216,252,288,324 +coh_thresh = 25 + + +def calc_an(loop_360_n, loop_360_6): + """ + Calculates the a_n values from loop closures. + + Parameters: + loop_360_n (ndarray): Array of loop closures for n-day intervals. + loop_360_6 (ndarray): Array of loop closures for 6-day intervals. + + Returns: + ndarray: Mean a_n values for all loop closures. + list: List of all a_n arrays for further processing or plotting. + """ + ratios = [] + + # Loop through indices + for i in range(len(loop_360_6)): + if loop_360_6[i] is not None and loop_360_n[i] is not None: + # Calculate the ratio for non-NaN elements + ratio_i = np.divide( + loop_360_n[i], + loop_360_6[i], + out=np.full_like(loop_360_n[i], np.nan), + where=~np.isnan(loop_360_6[i]), + ) + ratios.append(ratio_i) + +# Check if `ratios` is empty before proceeding + if not ratios: + print("Warning: No valid a_n values could be calculated. Returning NaN values.") + return np.array([np.nan]), [] + + + # Convert the list of ratios to a numpy array + an = np.array(ratios, dtype=object) + mean_an_long_baseline = [] + + # Calculate mean values of each a_n + for arr in an: + mask = (arr > 0) & (arr < 1) # Apply mask to filter invalid values + arr[~mask] = np.nan + if not np.isnan(arr).all(): # Check if all elements are NaN + mean_an = np.nanmean(arr) + else: + mean_an = np.nan # Assign NaN for completely empty slices + + mean_an_long_baseline.append(mean_an) + + return np.array(mean_an_long_baseline), an + + +################################################################################## +############################ +def plot_an(an_arrays, mean_a_long_baseline, k): + """ + Plots the a_n arrays with the largest coverage and their histograms in a two-row layout, + as well as the mean values of calibration parameters a_n over time. + + Parameters: + an_arrays (list): List of a_n arrays. + mean_a_long_baseline (ndarray): Array of mean calibration parameter values for a_1, a_2, ..., a_n. + k (int): The current value of k (e.g., 2 for a1, 3 for a2). + """ + + # Calculate the coverage (number of non-NaN values) for each array + coverage = [] + valid_an_arrays = [] # Store only valid arrays for plotting + + for arr in an_arrays: + if ( + arr is not None and arr.ndim == 2 and not np.all(np.isnan(arr)) + ): # Check for valid 2D arrays + coverage.append(np.sum(~np.isnan(arr))) # Count non-NaN values + valid_an_arrays.append(arr) + + # Sort arrays by coverage in descending order + sorted_indices = np.argsort(coverage)[::-1] + selected_arrays = [ + valid_an_arrays[idx] for idx in sorted_indices[:6] + ] # Select top 6 arrays + + # Plot the a_n arrays and their histograms in a two-row layout + if selected_arrays: + num_arrays = len(selected_arrays) + fig = plt.figure(figsize=(4 * num_arrays, 8)) + gs = GridSpec( + 2, num_arrays + 1, width_ratios=[1] * num_arrays + [0.1] + ) # Add space for colorbar + + # Plot selected arrays + axes_img = [] + for i, arr in enumerate(selected_arrays): + ax_img = fig.add_subplot(gs[0, i]) + im = ax_img.imshow(arr, cmap="RdYlBu", vmin=-0.8, vmax=0.8) + ax_img.set_title(f"{i+1}th $a_{{{k-1}}}$ Array") # Dynamically set title + ax_img.axis("off") + axes_img.append(ax_img) + + # Add a single colorbar for all array plots + cbar_ax = fig.add_subplot(gs[0, -1]) + fig.colorbar(im, cax=cbar_ax, orientation="vertical", label="Color Scale") + + # Plot histograms for the selected arrays + for i, arr in enumerate(selected_arrays): + ax_hist = fig.add_subplot(gs[1, i]) + ax_hist.hist(arr.flatten(), bins=100, color="blue", alpha=0.7, density=True) + ax_hist.set_title( + f"{i+1}th $a_{{{k-1}}}$ Histogram" + ) # Dynamically set title + ax_hist.set_xlim([-3, 3]) # Adjust histogram range as needed + + plt.tight_layout() + plt.show() + else: + print("No valid a_n arrays available for plotting.") + + # Plot the mean values of a_n over time + x_values = np.arange(len(mean_a_long_baseline)) # Sequential indices for x-axis + fig, ax = plt.subplots(figsize=(6, 4)) + + + # Filter out NaN values to avoid gaps in the plot + valid_indices = ~np.isnan(mean_a_long_baseline) # Mask for valid (non-NaN) entries + x_valid = x_values[valid_indices] # Only use valid time steps + y_valid = np.array(mean_a_long_baseline)[valid_indices] # Only use valid a_n values + + + # Remove NaN values for trendline fitting + valid_indices = ~np.isnan(mean_a_long_baseline) # Mask for valid (non-NaN) entries + if ( + valid_indices.sum() > 1 + ): # Ensure there are at least two valid points for fitting +# x_valid = x_values[valid_indices] + y_valid = np.array(mean_a_long_baseline)[valid_indices] + x_valid = np.arange(len(y_valid)) + + # Fit and plot a trendline + coefficients = np.polyfit(x_valid, y_valid, 1) + trendline = np.polyval(coefficients, x_valid) + ax.plot(x_valid, trendline, color="red", label="Trendline") + + # Scatter plot for all points (including NaN if any) + ax.scatter( + #x_values, + #mean_a_long_baseline, + x_valid, # Only use valid time steps + y_valid, # Only use valid a_n values + + color="blue", + marker="o", + label="Mean Values", + s=9, + ) + + # Set labels and limits + ax.set_ylim(-3, 3) # Adjust limits as needed + ax.set_xlabel("Time Step") # Sequential index as x-axis + ax.set_ylabel("Mean Values", fontsize=12) + ax.legend() + + # Adjust layout and show the plot + plt.tight_layout() + plt.show() + + +## +######### forming 360-6 + +loop_360_6_all = np.array(all_loops[long_baseline][interval], dtype=object) +coh_360 = np.array(coh[long_baseline][:], dtype=object) + +loop_360_6 = np.full_like( + loop_360_6_all, np.nan +) # create a new array as the same size of loop_360_6_all with nan values + + +for i, arr in enumerate(loop_360_6_all): + if arr is not None: + (frame_row, frame_col) = np.shape(arr) + loop_360_6[i] = np.where((coh_360[i] > coh_thresh), arr, np.nan) + +###### forming 360-n (n=12,18,24 etc) +# mean_a_long_baseline = [] +final_mean_values = [] +an_labels = [] + +# Loop through the values of k corresponding to the number of calibration parameters +for k in range(2, 2 + num_a): # k=2 for a1, k=3 for a2, etc. + mean_an_long_baseline = [] + + # Forming 360-n (n=12, 18, 24, etc.) + loop_360_n_all = np.array(all_loops[long_baseline][k * interval], dtype=object) + loop_360_n = np.full_like(loop_360_n_all, np.nan) + + for i, arr in enumerate(loop_360_n_all): + if arr is not None: + # Apply coherence threshold filtering + loop_360_n[i] = np.where((coh_360[i] > coh_thresh), arr, np.nan) + + # Calculate a_n values + mean_an_long_baseline, an_arrays = calc_an(loop_360_n, loop_360_6) + + # If all values in mean_an_long_baseline are nan (empty ratios), stop execution + if np.isnan(mean_an_long_baseline).all(): + print("No valid a_n values could be calculated. Stopping execution.") + sys.exit(1) # Exit the script with an error code (1 means failure) + + # Calculate the mean of all mean values for this k and store it + overall_mean = np.nanmean(mean_an_long_baseline) + final_mean_values.append(overall_mean) + an_labels.append(f"a{k-1}") + + for idx, mean_a in enumerate(mean_an_long_baseline, start=1): + if not np.isnan(mean_a): # Check if the value is not NaN + print(f"Mean of the {idx}th a({k-1}) is {mean_a:.6f}") + + plot_an(an_arrays, mean_an_long_baseline, k) + + +# Print the final mean values across all k +for k_idx, final_mean in enumerate(final_mean_values, start=2): + print(f"Final mean value for a({k_idx-1}) is {final_mean:.6f}") + +# Write final mean values to a text file +output_path = os.path.join(parameters["output_path"], "Data") +os.makedirs(output_path, exist_ok=True) # Create directory if it doesn't exist +output_file = os.path.join(output_path, "an.txt") + +with open(output_file, "w") as file: + for label, mean_value in zip(an_labels, final_mean_values): + file.write(f"{label}={mean_value:.6f}\n") + +print(f"Final mean values written to {output_file}.") diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_04_Inversion.py b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_04_Inversion.py new file mode 100644 index 0000000..2913c3e --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_04_Inversion.py @@ -0,0 +1,683 @@ +#!/usr/bin/env python3 + +import numpy as np +import pickle +import os +from scipy.sparse import csc_array +from scipy.sparse.linalg import lsqr +import sys +import math +from bin.read_orig_ifgs_coh import read_orig_ifgs_coh +from bin.circular_mean_var import circular_mean_and_variance_over_epochs +import glob +import configparser +import warnings + +warnings.filterwarnings("ignore", category=DeprecationWarning) + +################################################################################ + +HELP_TEXT = """ +PhaseBias_04_Inversion.py + +This script estimates the phase bias terms for the base interferograms using short-term closure phases as input. +The bias terms derived in this step serve as the foundation for correcting other short-term interferograms. + +### Workflow: +1. **Masking Noisy Loop Closures:** + - A moving average is applied to the time-series of loop closures to account for seasonal fluctuations. + - A circular moving average (mean of complex values over time) is computed for each pixel. + - The distance of each loop closure from the moving average is calculated. If the distance exceeds a threshold (2σ), the point is masked as noisy. + - The circular standard deviation (σ) is computed using the von Mises distribution (circular normal distribution). + +2. **Refined Loop Closures:** + - Masked loop closures, denoted as 〖Δφ〗_(i,i+2)^r and 〖Δφ〗_(i,i+3)^r, are used as primary observations for phase bias estimation. + +3. **First Inversion (Without Temporal Smoothing Constraints):** + - Estimates the bias terms that can be corrected directly from the observed loop closures. + +4. **Second Inversion (With Temporal Smoothing Constraints):** + - Incorporates temporal smoothing constraints to estimate additional bias terms that cannot be directly derived from loop closures. + - The temporal smoothing minimizes differences between bias terms over time, ensuring consistency. + +5. **Combining Results:** + - Bias terms from the first inversion are combined with the results of the second inversion to include terms not estimated in the first round. + +### Outputs: +- The phase bias terms for the base interferograms are stored as a NumPy array file: + - `X_base_ifgs_biases.npy`, saved in the `Data` directory under the `output_path` defined in `config.txt`. + + +### Input Requirements: +- Short-term closure phases (from `PhaseBias_02_Loop_Closures.py`). +- Configuration parameters, including thresholds and temporal settings, defined in `config.txt`. + +### Output Files: +- `X_base_ifgs_biases.npy`: Stores the estimated bias terms for the base interferograms. +""" + +if "--help" in sys.argv: + print(HELP_TEXT) + sys.exit(0) + +############################################################################### +# Config file path +config_file = "config.txt" + +# Check if config file exists +if not os.path.exists(config_file): + raise FileNotFoundError( + f"Error: The configuration file '{config_file}' was not found in the script's directory." + ) + +# Initialize the configparser +config = configparser.ConfigParser() + +# Read the config.txt file +config.read(config_file) + +# Define required parameters +required_parameters = [ + "root_path", + "frame", + "start", + "end", + "interval", + "nlook", + "LiCSAR_data", + "a1_6_day", + "a2_6_day", + "a1_12_day", + "a2_12_day", + "estimate_an_values", +] + +# Initialize a dictionary to store parameters +parameters = {} + +# Attempt to retrieve each parameter and catch errors +try: + parameters["root_path"] = config.get("DEFAULT", "root_path") + parameters["output_path"] = config.get("DEFAULT", "output_path") + parameters["frame"] = config.get("DEFAULT", "frame") + parameters["start"] = config.get("DEFAULT", "start") + parameters["end"] = config.get("DEFAULT", "end") + parameters["interval"] = config.getint("DEFAULT", "interval") + parameters["nlook"] = config.getint( + "DEFAULT", "nlook" + ) # Assuming 'nlook' should be an integer + parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") + parameters["a1_6_day"] = config.getfloat("DEFAULT", "a1_6_day") + parameters["a2_6_day"] = config.getfloat("DEFAULT", "a2_6_day") + parameters["a1_12_day"] = config.getfloat("DEFAULT", "a1_12_day") + parameters["a2_12_day"] = config.getfloat("DEFAULT", "a2_12_day") + parameters["estimate_an_values"] = config.get("DEFAULT", "estimate_an_values") + +except configparser.NoOptionError as e: + # Error message if a required parameter is missing + missing_param = str(e).split(": ")[1] + raise ValueError( + f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." + ) +except configparser.Error as e: + # General error for any configparser-related issues + raise ValueError(f"Error while reading '{config_file}': {e}") + +# Assign to individual variables for easy access +root_path = parameters["root_path"] +output_path = parameters["output_path"] +frame = parameters["frame"] +start = parameters["start"] +end = parameters["end"] +interval = parameters["interval"] +nlook = parameters["nlook"] +LiCSAR_data = parameters["LiCSAR_data"] +estimate_an_values = parameters["estimate_an_values"] + + +# Initialize a1 and a2 with None for safety +a1 = None +a2 = None + + +print("estimate_an_values is ", estimate_an_values) +# Check if estimate_an_values is 'yes' +if estimate_an_values == "yes": + an_file_path = os.path.join(output_path, "Data", "an.txt") + + # Try reading the file + try: + if os.path.exists(an_file_path): + print(f"Reading 'a' values from {an_file_path}...") + + # Read the file and load the values + with open(an_file_path, "r") as file: + for line in file: + line = line.strip() + if line.startswith("a1="): + a1 = float(line.split("=")[1]) + elif line.startswith("a2="): + a2 = float(line.split("=")[1]) + + # Check if both a1 and a2 were found + if a1 is None or a2 is None: + print( + "Warning: Missing a1 or a2 in an.txt. Switching to default values from config file." + ) + raise ValueError("Incomplete a1 or a2 values.") + else: + print( + f"Warning: {an_file_path} does not exist. Switching to default values from config file." + ) + raise FileNotFoundError + + except (FileNotFoundError, ValueError): + # Fallback to default values from config file + if interval == 6: + a1 = parameters["a1_6_day"] + a2 = parameters["a2_6_day"] + else: + a1 = parameters["a1_12_day"] + a2 = parameters["a2_12_day"] +else: + # Use default values from the config file + print("Using default 'an' values from the config file.") + if interval == 6: + a1 = parameters["a1_6_day"] + a2 = parameters["a2_6_day"] + else: + a1 = parameters["a1_12_day"] + a2 = parameters["a2_12_day"] + +# Print the chosen values +print(f"Final values - a1: {a1}, a2: {a2}") + + +##################################################### Loop Closure calculation ##################################################### +#################################################################################################################################### +#################################################################################################################################### + +track = frame[0:3] +if track[0] == "0": + track = track[1:3] +if track[0] == "0": + track = track[1:2] + +with_temporals = "yes" # it should be yes, if you want to incorporate the temporal constraints +apply_to_all = "no" # in case of with_temporal='yes', we can decide if you want to use the temporal to all ifgs (i.e. apply_to_all='yes'), or just to those that can not be corrected (i.e. apply_to_all='no') +w = 0.1 # the weigth of the temporal smoothing constraints +####### +coh_thresh = 11 # threhsold on the coherence values. In this version it is off +min_num_eq = 10 # I used 10 in all my experiments. min number of equations in the least square inversion. + +############################### Reading input data ############################################# +#################### Read ifgs: +print("Reading all ifgs...") +# Define the path components +sub_dir = "Data" +filename_pattern = "All_ifgs_*" # Pattern to match files starting with "All_ifgs" + +# Construct the full file path with the refined pattern +file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + all_ifgs = pickle.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) +print("Reading ifgs completed.") + +###################### Read coh: +print("\nReading all coherence data...") +# Define the path components +sub_dir = "Data" +filename_pattern = "All_coh_*" # Pattern to match files starting with "All_ifgs" + +# Construct the full file path with the refined pattern +file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + all_coh = pickle.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) +print("Reading coherence data completed.") + +##################### Read Loops + +print("\nReading all loop closures...") +filename_pattern = "All_loops_*" # Pattern to match files starting with "All_ifgs" + +# Construct the full file path with the refined pattern +file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + all_loops = pickle.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) + +print("Reading loop closures completed.") + +all_cat = [] +for cat in all_loops: + all_cat.append(cat) + +### stacking all the ifgs 6/12/18 in a numpy var all_ifgs +print( + "Extracting the required interferograms(up to 3 epochs), and the corresponding loop closures for inversion..." +) +desired_categories = [interval, 2 * interval, 3 * interval] +all_ifgs, _, existing_ifgs_index = read_orig_ifgs_coh( + all_ifgs, all_coh, desired_categories +) + +######### forming 12-6 and 18-6 loop closures from the imported data +################################################################# +loop_12 = np.array(all_loops[2 * interval][interval], dtype=object) +#loop_12 = np.array(all_loops[2 * interval][interval]) +#coh_12 = np.array(all_coh[2 * interval][:]) +coh_12 = np.array(all_coh[2 * interval][:], dtype=object) + +#loop_18 = np.array(all_loops[3 * interval][interval]) +#coh_18 = np.array(all_coh[3 * interval][:]) +loop_18 = np.array(all_loops[3 * interval][interval], dtype=object) +coh_18 = np.array(all_coh[3 * interval][:], dtype=object) + +############## finding dynamic pixel-based thresholds to remove the noisy loops +loop_12_orig = [] +loop_18_orig = [] +none_indices12 = [] +none_indices18 = [] + +# refine loop_12 to exclude the none indices and create loop_12_orig +for i, arr in enumerate(loop_12): + if arr is not None: + (frame_row, frame_col) = np.shape(arr) + loop_12_orig.append(arr) + else: + none_indices12.append(i) + +# refine loop_18 to exclude the none indices and create loop_18_orig +for i, arr in enumerate(loop_18): + if arr is not None: + loop_18_orig.append(arr) + else: + none_indices18.append(i) + +loop_12_orig = np.array(loop_12_orig) +loop_18_orig = np.array(loop_18_orig) + + +### removing noisy loops +print( + "Masking noisy loop closures using circular moving average and standard deviaiton of the loop closures in time:" +) + +# using circular_mean to calculate the mean phase value using complex numbers, and von_mises_variance to calculate the second moment of the Von Mises distribution +_, thresh_loop12 = circular_mean_and_variance_over_epochs(loop_12_orig, axis=0) +_, thresh_loop18 = circular_mean_and_variance_over_epochs(loop_18_orig, axis=0) + + +# Add a new axis at the beginning (axis=0) of these arrays to have shape (1, ...) +# These are the threshold vlaues obtained for each pixel that will be used for masking later +thresh_loop12 = np.expand_dims(thresh_loop12, axis=0) +thresh_loop18 = np.expand_dims(thresh_loop18, axis=0) + + +# calculate a temporal moving average for wrapped data +def moving_average(arr, window_size): + # Initialize an empty array to store the temporal averages + moving_averages = np.zeros_like(arr) + half_window = window_size // 2 + ### miroring data at both ends for window_size/2 at each end + mirrored_shape = (arr.shape[0] + window_size, arr.shape[1], arr.shape[2]) + + # Create an empty array to hold the mirrored data + mirrored_arr = np.empty(mirrored_shape, dtype=arr.dtype) + + # Fill the mirrored array with the original data + mirrored_arr[window_size // 2 : -window_size // 2] = arr + + # Mirror the data at the beginning + for i in range(window_size // 2): +# mirrored_arr[i] = arr[window_size // 2 - i] + mirrored_arr[i] = arr[min(half_window - i, arr.shape[0] - 1)] + + # Mirror the data at the end + for i in range(window_size // 2): +# mirrored_arr[-(i + 1)] = arr[-(i + 1) - window_size // 2] + mirrored_arr[-(i + 1)] = arr[max(-1 - (half_window - i), 0)] + #### + + # Calculate temporal averages within each window + for t in range(half_window, mirrored_arr.shape[0] - half_window): + moving_averages[t - half_window, :, :] = np.angle( + np.nanmean( + np.exp(1j * (mirrored_arr[t - half_window : t + half_window, :, :])), + axis=0, + ) + ) + return moving_averages + + +# for loop 12 +# Calculate distance between each epoch and the moving average for each pixel +print( + f"Calculating the moving average for {2 * interval} loop closures, followed by masking noisy loop closures..." +) +print( + "This may take a while, as it depends on the number of loops and the size of your dataset.\n" +) + +window_size = 30 + +moving_avg = moving_average(loop_12_orig, window_size) +distances = np.abs(np.angle(np.exp(1j * (loop_12_orig - moving_avg)))) +mask = distances > 2 * thresh_loop12 # it is 4*sigma + +# Replace values with NaN where mask is True +loop_12_bk = loop_12_orig +loop_12_orig = np.where(mask, np.nan, loop_12_orig) +loop_12 = loop_12_orig.tolist() + +for idx in none_indices12: + loop_12.insert(idx, None) +loop_12 = np.array(loop_12, dtype=object) + +# for loop 18 +# Calculate distance between each epoch and the moving average for each pixel +print( + f"Calculating the moving average for {3 * interval} loop closures, followed by masking noisy loop closures..." +) +print( + "This may take a while, as it depends on the number of loops and the size of your dataset.\n" +) + +moving_avg = moving_average(loop_18_orig, window_size) +distances = np.abs(np.angle(np.exp(1j * (loop_18_orig - moving_avg)))) +mask = distances > 2 * thresh_loop18 # it is 4*sigma + +# Replace values with NaN where mask is True +loop_18_bk = loop_18_orig +loop_18_orig = np.where(mask, np.nan, loop_18_orig) +loop_18 = loop_18_orig.tolist() + +for idx in none_indices18: + loop_18.insert(idx, None) +loop_18 = np.array(loop_18, dtype=object) + +print("Masking of noisy loop closures is completed.") + +# ################ Apply coherece thresholding +# print('shape loop_12 = ', np.shape(loop_12)) +# print('shape loop_12_orig = ', np.shape(loop_12_orig)) +# print('shape coh_12 = ', np.shape(coh_12)) +# for i, arr in enumerate(loop_12): +# if arr is not None: +# (frame_row, frame_col) = np.shape(arr) +# loop_12[i] = np.where((coh_12[i] > coh_thresh), arr, np.nan) +# +# +# for i, arr in enumerate(loop_18): +# if arr is not None: +# loop_18[i] = np.where((coh_18[i] > coh_thresh), arr, np.nan) +# +# +# + +len12 = len(loop_12) # number of 12 day loops including None +len18 = len(loop_18) # number of 18 day loops including None + +######################## Forming the design matrix A ############################## +#################################################################################### + +print("Preparing the design matrix and observation vector for the inversion step...") + +b = [] +A = [] +n_unk = ( + len12 + 1 +) # this in an initial value(i.e. the num of 6 biases). will be changed by the actual number of unk after considering the None values/missing +# print('n_unk = ', n_unk) + +row = np.zeros( + n_unk, dtype=np.float32 +) # each row of the design matrix A for the 12-day +for i in range(len12): + b.append(loop_12[i]) + # print('np.shape(loop_12[i] = ', np.shape(loop_12[i])) + row[i : i + 2] = a1 - 1 + A.append(row) + row = np.zeros(n_unk, dtype=np.float32) + +row = np.zeros( + n_unk, dtype=np.float32 +) # each row of the design matrix A for the 18-day +for i in range(len18): + b.append(loop_18[i]) + row[i : i + 3] = a2 - 1 + A.append(row) + row = np.zeros(n_unk, dtype=np.float32) + +A = np.array(A) +b = np.array(b, dtype=object) + + +####################### removing the rows from b and A where b is None +mask = [] +mask = np.array([arr is not None for arr in b]) +# Convert the boolean mask to an integer mask +mask = np.nonzero(mask) + +A = A[mask] # Filter rows of A based on the mask +b = b[mask] + +b = b.tolist() + +############ removing the columns of A where are values are zero (these are the unkonws which doesn't fall in any equations nor 12 neither 18 and thus cannot be corrected) + +# Find the column indices where all values are zero +zero_columns = np.all(A == 0, axis=0) + +## Get the indices of the zero columns +zero_column_indices = np.where(zero_columns)[0] + +# Get the indices of the non-zero columns +non_zero_column_indices = np.where(~zero_columns)[ + 0 +] # the indices of the unknowns that can be corrected + +# Save the file +directory_path = os.path.join(output_path, "Data") +file_path = os.path.join(directory_path, f"indices_unknown_tobe_corrected.npy") +np.save(file_path, non_zero_column_indices) + + +### Adding all the existing 6-day ifgs indices to non_zero_column_indices. We want to use estimate their biases in the second step. + + +A_bk = A # to keep a copy of A in case we want to esimate all 6-day biases with smoothing constraints +b_bk = b +all_column_indices = np.array( + range(A_bk.shape[1]) +) # generating all indices in case we want to esimate all 6-day biases with smoothing constraints + +A = A[:, non_zero_column_indices] + +# finding the last column with value -1. this gives the number of 6-day biases that can be corrected +last_column_with_minus_1 = None + +# Iterate through the columns from right to left +for col in range(len(A[0]) - 1, -1, -1): + if -1 in [row[col] for row in A]: + last_column_with_minus_1 = col + break + +num_rows_before_temporals = A.shape[0] + +################################# Least Square inversion ############################### +######################################################################################## +# First Inversion: Without Temporal Smoothing Constraints +print("Starting the first inversion (without temporal smoothing constraints).") +print( + "This step estimates the bias terms that can be corrected based on observed loop closures." +) + +damp_factor = 0 + +if apply_to_all == "no": # this is without using any temporal constraints + + X1 = np.zeros((np.shape(A_bk)[1], frame_row, frame_col), dtype=np.float32) + # X1[:,:,:] = np.nan # after checking noticed this doesn't have any effect on the final vel + b_bk = np.array(b_bk) + + total_pixels = frame_row * frame_col # Total number of pixels for progress bar + processed_pixels = 0 # Counter for processed pixels + print("Progress: [", end="", flush=True) + + for row in range(frame_row): + for col in range(frame_col): + processed_pixels += 1 + # Update progress bar + percentage = (processed_pixels / total_pixels) * 100 + if processed_pixels % (total_pixels // 100) == 0: # Update every 1% + print(f"{int(percentage)}%", end="", flush=True) + sys.stdout.write( + "\rProgress: [" + + "=" * (int(percentage) // 2) + + " " * (50 - int(percentage) // 2) + + "]" + ) + + non_nan_mask = ~np.isnan(b_bk[:, row, col]) + bb = b_bk[non_nan_mask, row, col] # Remove NaN values + AA = A_bk[non_nan_mask, :] # Remove corresponding rows + if len(bb) > min_num_eq: # what is the min number of equation? + + x, istop, itn, normr, normr2 = lsqr( + csc_array(AA), bb, damp=damp_factor + )[ + :5 + ] # Using sparse matrix representation scipy.sparse.linalg + X1[:, row, col] = x + + +# if with_temporals == "no": # this solution will be the final answer to be saved as X +# # Fill X with values from X1 according to non_zero_column_indices +# for i, idx in enumerate(non_zero_column_indices): +# X[idx, :, :] = X1[i, :, :] + + + + +print("\nFirst inversion (without temporal smoothing constraints) completed.") + +########################### +# Second Inversion: With Temporal Smoothing Constraints +print("Starting the second inversion (with temporal smoothing constraints).") +print( + "This step estimates all bias terms, including those that cannot be corrected, using temporal smoothing constraints." +) + +if with_temporals == "yes": # This is using the temporal constranints on all unknowns + X2 = np.zeros((np.shape(A_bk)[1], frame_row, frame_col), dtype=np.float32) + # X2[:,:,:] = np.nan # after checking noticed this doesn't have any effect on the final vel + b_bk = np.array(b_bk) + + #### Adding temporal constraints + row = np.zeros( + A_bk.shape[1], dtype=np.float32 + ) # additional rows the design matrix A for temporal constraints, setting that to zero again + for i in range(len(all_column_indices) - 2): # for all unknowns + row[i] = -1 * w + row[i + 1] = 2 * w + row[i + 2] = -1 * w + A_bk = np.vstack((A_bk, row)) + row = np.zeros( + A_bk.shape[1], dtype=np.float32 + ) # additional rows the design matrix A for temporal constraints, setting that to zero again + num_temporals = len(A_bk) - len(b_bk) + # print('num_temporals is ', num_temporals) + + ######### Appending zeroes to b according to the added number of temporal constraints obtained by np.shape(A)[0] - np.shape(b)[0] + b_0 = np.zeros( + (np.shape(A_bk)[0] - np.shape(b)[0], frame_row, frame_col), dtype=np.float32 + ) + b_bk = np.vstack((b_bk, b_0)) + + thresh_n_eq = num_temporals # in case of using temporals, we don't want the nan pixels with zero equations with only using temporals + + processed_pixels = 0 # Reset counter for second inversion + print("Progress: [", end="", flush=True) + + for row in range(frame_row): + for col in range(frame_col): + processed_pixels += 1 + # Update progress bar + percentage = (processed_pixels / total_pixels) * 100 + if processed_pixels % (total_pixels // 100) == 0: # Update every 1% + print(f"{int(percentage)}%", end="", flush=True) + sys.stdout.write( + "\rProgress: [" + + "=" * (int(percentage) // 2) + + " " * (50 - int(percentage) // 2) + + "]" + ) + + non_nan_mask = ~np.isnan(b_bk[:, row, col]) + bb = b_bk[non_nan_mask, row, col] # Remove NaN values + AA = A_bk[non_nan_mask, :] # Remove corresponding rows + if len(bb) > thresh_n_eq: + + x, istop, itn, normr, normr2 = lsqr( + csc_array(AA), bb, damp=damp_factor + )[ + :5 + ] # Using sparse matrix representation scipy.sparse.linalg + X2[:, row, col] = x + + print("\nSecond inversion (with temporal smoothing constraints) completed.") + + ####### combining X1 and X2 into X + + X = np.zeros((np.shape(A_bk)[1], frame_row, frame_col), dtype=np.float32) + X[:, :, :] = np.nan + + # Fill X with values from X1 according to non_zero_column_indices + for i, idx in enumerate(non_zero_column_indices): + X[idx, :, :] = X1[i, :, :] + + # Fill in the missing values from X2 + for i, idx in enumerate(all_column_indices): + if idx not in non_zero_column_indices: + X[idx, :, :] = X2[i, :, :] + + # Ensure the dtype is appropriate for your data + X = X.astype(np.float32) + +directory_path = os.path.join(output_path, "Data") +file_path = os.path.join(directory_path, f"X_base_ifgs_biases.npy") +np.save(file_path, X) + +print("Inversion process completed. Results saved to:", file_path) + +########################################################################################################## diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_05_Correction.py b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_05_Correction.py new file mode 100644 index 0000000..14cbca0 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/PhaseBias_05_Correction.py @@ -0,0 +1,535 @@ +#!/usr/bin/env python3 + +import numpy as np +from datetime import datetime +import pickle +import time +import os +from bin.generate_ifgs_from_epochs import generate_ifg_pairs +import configparser +import glob +from bin.resample import resample_geotiff +import sys + +# sys.path.append(os.path.dirname(os.path.abspath(__file__))) +import bin +from bin.read_orig_ifgs_coh import read_orig_ifgs_coh + +from bin.ifg_cc_totif import ifg_cc_2tif +import gc + +############################################################################### + +HELP_TEXT = """ +PhaseBias_05_Correction.py + +This script applies phase bias correction to desired interferograms using the estimated bias terms for the base interferograms. The corrected interferograms are saved in GeoTIFF format for further analysis. + +### Workflow: +1. **Inputs:** + - **Original Wrapped Interferograms, and Coherence Data:** Obtained from step 1 (`PhaseBias_01_Read_Data.py`). + - **Estimated Bias Terms for Base Intierferograms:** Obtained from step 4 (`PhaseBias_04_Inversion.py`). + - **Calibration Parameters (*a_n*):** Estimated from step 3 (`PhaseBias_03_calibration_pars.py`). + - **max_con:** Defines the maximum number of connections to be corrected. For instance: + - `max_con=5` corrects interferograms up to 30-day intervals for a 6-day acquisition interval. + - Adjust `max_con` as needed. + +2. **Bias Term Estimation for Desired Interferograms:** + - Bias terms for desired interferograms, δ_(i,i+n+1), are estimated using the relationship: + δ_(i,i+n+1) = a_n * (∑_(t=i)^(i+n) δ_(t,t+1) ) + +3. **Correction of Interferograms:** + - Using the estimated bias terms, the interferograms are corrected as follows: + φ_(i,i+n+1)^c = φ_(i,i+n+1) - δ_(i,i+n+1) + + - Here, φ_(i,i+n+1) is the original interferogram, and φ_(i,i+n+1)^c is the corrected interferogram. + +4. **Outputs:** + - Corrected interferograms are saved in GeoTIFF format in the `GEOC` directory under the `output_path` specified in `config.txt`. + +### Output File Format: +- **Corrected Interferograms:** Saved in GeoTIFF format, organized by temporal baseline. + +### Input Requirements: +- Original wrapped interferograms from step 1. +- Bias terms for base interferograms from step 4. +- Configuration parameters (e.g., `output_path`, `interval`) defined in `config.txt`. + +### Output Directory: +- `GEOC` directory under `output_path`: Contains the corrected interferograms in GeoTIFF format, organized by temporal baseline. + +""" +if "--help" in sys.argv: + print(HELP_TEXT) + sys.exit(0) + + +################################################################################ + + +# Config file path +config_file = "config.txt" + +# Check if config file exists +if not os.path.exists(config_file): + raise FileNotFoundError( + f"Error: The configuration file '{config_file}' was not found in the script's directory." + ) + +# Initialize the configparser +config = configparser.ConfigParser() + + +# Read the config.txt file +config.read(config_file) + +print("Config keys:", list(config.defaults().keys())) + + +# Define required parameters +required_parameters = [ + "root_path", + "frame", + "start", + "end", + "interval", + "nlook", + "LiCSAR_data", + "a1_6_day", + "a2_6_day", + "a3_6_day", + "a4_6_day", + "a1_12_day", + "a2_12_day", + "a3_12_day", + "a4_12_day", + "estimate_an_values", +] + +# Initialize a dictionary to store parameters +parameters = {} + +# Attempt to retrieve each parameter and catch errors +try: + parameters["root_path"] = config.get("DEFAULT", "root_path") + parameters["output_path"] = config.get("DEFAULT", "output_path") + parameters["frame"] = config.get("DEFAULT", "frame") + parameters["start"] = config.get("DEFAULT", "start") + parameters["end"] = config.get("DEFAULT", "end") + parameters["interval"] = config.getint("DEFAULT", "interval") + parameters["nlook"] = config.getint( + "DEFAULT", "nlook" + ) # Assuming 'nlook' should be an integer + parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") + parameters["a1_6_day"] = config.getfloat("DEFAULT", "a1_6_day") + parameters["a2_6_day"] = config.getfloat("DEFAULT", "a2_6_day") + parameters["a3_6_day"] = config.getfloat("DEFAULT", "a3_6_day") + parameters["a4_6_day"] = config.getfloat("DEFAULT", "a4_6_day") + parameters["a1_12_day"] = config.getfloat("DEFAULT", "a1_12_day") + parameters["a2_12_day"] = config.getfloat("DEFAULT", "a2_12_day") + parameters["a3_12_day"] = config.getfloat("DEFAULT", "a3_12_day") + parameters["a4_12_day"] = config.getfloat("DEFAULT", "a4_12_day") + parameters["estimate_an_values"] = config.get("DEFAULT", "estimate_an_values") + +except configparser.NoOptionError as e: + # Error message if a required parameter is missing + missing_param = str(e).split(": ")[1] + raise ValueError( + f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." + ) +except configparser.Error as e: + # General error for any configparser-related issues + raise ValueError(f"Error while reading '{config_file}': {e}") + +# Assign to individual variables for easy access +root_path = parameters["root_path"] +output_path = parameters["output_path"] +frame = parameters["frame"] +start = parameters["start"] +end = parameters["end"] +interval = parameters["interval"] +nlook = parameters["nlook"] +LiCSAR_data = parameters["LiCSAR_data"] +estimate_an_values = parameters["estimate_an_values"] + + +start_date = datetime.strptime(start, "%Y%m%d") +end_date = datetime.strptime(end, "%Y%m%d") + +uncor_only = 1 # in case of 1 only uses 6-12-18 uncorrected ifgs. it is based on the desired_lengths ifgs +max_con = 5 # in case of 5, it will correct up to 6*5=30-day ifgs. + + +##################################################### +# Root directory where the category directories are located + +if LiCSAR_data == "yes": + track = frame[0:3] # extracting track number from frame id + if track[0] == "0": + track = track[1:3] + if track[0] == "0": + track = track[1:2] + + root_directory = ( + f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}" + ) +else: + root_directory = root_path + + +############################################ + +# Initialize the an values with None for safety +an_values = [None] * ( + max_con - 1 +) # For max_con=5, this will create [None, None, None, None] + +print("estimate_an_values is ", estimate_an_values) + +# Check if estimate_an_values is 'yes' +if estimate_an_values == "yes": + an_file_path = os.path.join(output_path, "Data", "an.txt") + + # Try reading the file + try: + if os.path.exists(an_file_path): + print(f"Reading 'a' values from {an_file_path}...") + + # Read the file and load the an values dynamically + with open(an_file_path, "r") as file: + for line in file: + line = line.strip() + for i in range( + 1, max_con + ): # Loop through a1, a2, ..., a(max_con-1) + param_name = f"a{i}" + if line.startswith(f"{param_name}="): + an_values[i - 1] = float(line.split("=")[1]) + + # Check if all required an values were found + if any(value is None for value in an_values): + print( + "Warning: Missing one or more an values in an.txt. Switching to default values from config file." + ) + raise ValueError("Incomplete an values.") + else: + print( + f"Warning: {an_file_path} does not exist. Switching to default values from config file." + ) + raise FileNotFoundError + + except (FileNotFoundError, ValueError): + # Fallback to default values from config file + print("Loading default 'an' values from config file...") + for i in range(1, max_con): + if interval == 6: + an_values[i - 1] = parameters[f"a{i}_6_day"] + else: + an_values[i - 1] = parameters[f"a{i}_12_day"] +else: + # Use default values from the config file + print("Using default 'an' values from the config file.") + for i in range(1, max_con): + if interval == 6: + an_values[i - 1] = parameters[f"a{i}_6_day"] + else: + an_values[i - 1] = parameters[f"a{i}_12_day"] + +# Print the chosen an values +for i, value in enumerate(an_values, start=1): + print(f"Final value - a{i}: {value}") + + +############## generating ifg pairs between two epochs + +# desired_lengths = [interval, 2*interval, 3*interval, 4*interval, 5*interval] +desired_lengths = [interval * i for i in range(1, max_con + 1)] # replacing the above + +all_ifgs_string = generate_ifg_pairs(start_date, end_date, interval, desired_lengths) + + +##### reading original ifgs/coh ############################# +#################################################################################### + +############################### Reading input data ############################################# +#################### Read ifgs: +print("Reading all ifgs...") +# Define the path components +sub_dir = "Data" +filename_pattern = "All_ifgs_*" # Pattern to match files starting with "All_ifgs" + +# Construct the full file path with the refined pattern +file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + ifgs = pickle.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) + + +###################### Read coh: +print("\nReading all coherence data...") +# Define the path components +sub_dir = "Data" +filename_pattern = "All_coh_*" # Pattern to match files starting with "All_ifgs" + +# Construct the full file path with the refined pattern +file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + coh = pickle.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) +print("\nReading data completed.") + + +######## stacking all the ifgs 6/12/18 in a numpy var all_ifgs +desired_categories = desired_lengths +print('stacking all the ifgs 6/12/18 in a numpy var all_ifgs') +all_ifgs, all_coh, existing_ifgs_index = read_orig_ifgs_coh( + ifgs, coh, desired_categories +) +print('stacking completed') + +del ifgs, coh +gc.collect() + +print('ifgs coh deleted') +##### Reading the estimated bias terms as well as the indices of unknowns that can be corrected obtained in step 03 ############## +############################################################################################## +print("Correcting the interferograms...") +# Define the path components +sub_dir = "Data" +filename_pattern_X = "X_base_ifgs_biases.npy" # Pattern to match files starting with "X_base" +filename_pattern_indices = "indices*" + +# Construct the full file path with the refined pattern +file_path_pattern_X = os.path.join(output_path, sub_dir, filename_pattern_X) +file_path_pattern_indices = os.path.join(output_path, sub_dir, filename_pattern_indices) + +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern_X) + +if file_list: + file_path = file_list[0] # string path to .npy file + X = np.load(file_path, mmap_mode="r") # ✅ correct way +else: + raise FileNotFoundError(f"No file matching '{filename_pattern_X}' was found in '{os.path.join(output_path, sub_dir)}'.") + + + +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern_indices) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + indices_unknown_tobe_corrected = np.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) + + +#### loading indices of unknown that can be corrected and the estimated unknowns + +indices_unknown_tobe_corrected_orig = indices_unknown_tobe_corrected.copy() + +len_6_biases = indices_unknown_tobe_corrected[-1] + 1 # number of 6-day biases + +all_column_indices = np.array( + range(len_6_biases) +) # generating all indices in case we want to esimate all 6-day biases with smoothing constraints. +indices_unknown_tobe_corrected = all_column_indices # in this case we had provided the corrections for all 6-day biases + + +# extending the indices_unknown_tobe_corrected in the original approach to have the indices of 12/18 correctable ifgs: +extended_indices = ( + indices_unknown_tobe_corrected.copy() +) # because in this case indices_unknown_tobe_corrected referes to all_column_indices +X_extended = ( + X.copy() +) # because in this case we have estimated all 6-day biases but we don't need them for the correction. we just need them to correct the existing 12/18 days based on the 6 days + +extended_indices = extended_indices.tolist() +X_extended = X_extended.tolist() +print('indices loaded') + +# Generalized loop to handle any max_con +for n in range(2, max_con + 1): # Start from 2*interval and go up to (max_con)*interval + print('n is ', n) + current_length = n * interval # Calculate the length (e.g., 12, 18, 24, ...) + + if current_length in desired_lengths: # Check if this length is required + for i in range(len(indices_unknown_tobe_corrected) - (n - 1)): + last_item = extended_indices[-1] # Get the last index + extended_indices.append(last_item + 1) # Append the new index + + # Sum up X values for the current length (n) +# X_sum = sum( +# X[i + j, :, :] for j in range(n) +# ) # Sum over n consecutive indices + + + X_sum = X[i : i + n].sum(axis=0) # more efficient + + + # Append the result to X_extended using the corresponding an_values + X_extended.append( + an_values[n - 2] # an_values is 0-based, so use n-2 + * np.angle(np.exp(complex(0 + 1j) * X_sum)) + ) + + +X = np.array(X_extended) +indices_unknown_tobe_corrected = extended_indices +del X_extended +print('X extended') + + +#### correcting the ifgs (6/12/18) ######################## +################################################################# + + +####### v3 +def wrap_to_pi_inplace(arr): + np.remainder(arr + np.pi, 2*np.pi, out=arr) + arr -= np.pi + return arr + +print(f"all_ifgs.shape: {all_ifgs.shape}") +print(f"len(indices_unknown_tobe_corrected): {len(indices_unknown_tobe_corrected)}") +print(f"len(X): {len(X)}") + +all_ifgs_cor = np.full(all_ifgs.shape, np.nan, dtype=np.float32) +if uncor_only == 0: # if we want to use the corrected ifgs (using wrapped phases) + batch_size = 1 # adjust based on memory + for i in range(0, len(indices_unknown_tobe_corrected), batch_size): + print('batch ', i) + idx = indices_unknown_tobe_corrected[i:i+batch_size] + arr = all_ifgs[idx] - X[i:i+batch_size] + wrap_to_pi_inplace(arr) + all_ifgs_cor[idx] = arr +else: + print("uncor_only = 1 → skipping correction, copying originals") + all_ifgs_cor[indices_unknown_tobe_corrected] = all_ifgs[indices_unknown_tobe_corrected] + +################## v2 +#def wrap_to_pi(x): +# """Wrap values to [-pi, pi).""" +# return (x + np.pi) % (2 * np.pi) - np.pi +# +## Preallocate directly as float array filled with NaN +#all_ifgs_cor = np.full(all_ifgs.shape, np.nan, dtype=np.float32) +# +#if uncor_only == 0: # use corrected ifgs (wrapped phases) +# print('uncor_only 0') +# all_ifgs_cor[indices_unknown_tobe_corrected] = wrap_to_pi(all_ifgs[indices_unknown_tobe_corrected] - X) +#else: # no correction, just copy +# all_ifgs_cor[indices_unknown_tobe_corrected] = all_ifgs[indices_unknown_tobe_corrected] + +############### # v1 +# this was less efficient and so I replaced it with above +#all_ifgs_cor = np.full_like(all_ifgs, None) +#all_ifgs_cor[:, :, :] = np.nan +# +#if uncor_only == 0: # if we want to use the corrected ifgs (using wrapped phases) +# all_ifgs_cor[indices_unknown_tobe_corrected, :, :] = np.angle( +# np.exp( +# complex(0 + 1j) +# * (all_ifgs[indices_unknown_tobe_corrected, :, :] - X[:, :, :]) +# ) +# ) +#else: # no need for correction +# all_ifgs_cor[indices_unknown_tobe_corrected, :, :] = all_ifgs[ +# indices_unknown_tobe_corrected, :, : +# ] +# +print ('before outputting') + +### Outputting to disk ################ +############################################################################# + + +# Create the directory (and parent directories) if they don't exist +GEOC_dir = "GEOC" +metadata_dir = "metadata" + +output_wrap_dir = os.path.join(output_path, GEOC_dir) +os.makedirs( + output_wrap_dir, exist_ok=True +) # create the GEOC directory if it doesn't exist + +template_tif_path = os.path.join(root_directory, metadata_dir, frame + ".geo.hgt.tif") +print('template_tif_path', template_tif_path) +if nlook != 1: + resampled_tif_path = os.path.join(output_wrap_dir, frame + ".geo.hgt.tif") + print('template_tif_path', template_tif_path) + print('resampled_tif_path', resampled_tif_path) + print('nlook ', nlook) + + resample_geotiff(template_tif_path, resampled_tif_path, nlook) + template_tif_path = resampled_tif_path + +print('resample complete') + +total_files = len(all_ifgs) # Total number of interferograms +processed_files = 0 # Counter for processed files + +print(f"Writing the corrected interferograms to {output_wrap_dir}") +print("Progress: [", end="", flush=True) +time.sleep(3) + + +for i in range(len(all_ifgs)): + if i in indices_unknown_tobe_corrected: + + # Writing progress bar + processed_files += 1 + percentage = (processed_files / total_files) * 100 + if processed_files % (total_files // 100) == 0: # Update progress every 1% + sys.stdout.write( + f"\rProgress: [" + + "=" * (int(percentage) // 2) + + " " * (50 - int(percentage) // 2) + + f"] {int(percentage)}%" + ) + sys.stdout.flush() + + inpha = all_ifgs_cor[i, :, :] # your phase input, float32 + incoh = all_coh[i, :, :] # your coh input. It can be either 0-255 or 0-1 + + output_phase_path = os.path.join(output_wrap_dir, all_ifgs_string[i]) + output_cc_path = os.path.join(output_wrap_dir, all_ifgs_string[i]) + + # output_phase_path = output_wrap_dir + all_ifgs_string[i] + # output_cc_path = output_wrap_dir + all_ifgs_string[i] + + os.makedirs(output_phase_path, exist_ok=True) + os.makedirs(output_cc_path, exist_ok=True) + + output_phase_file = ( + output_phase_path + "/" + all_ifgs_string[i] + ".geo.diff_pha.tif" + ) + + output_cc_file = output_cc_path + "/" + all_ifgs_string[i] + ".geo.cc.tif" + + ifg_cc_2tif(inpha, output_phase_file, template_tif_path) + ifg_cc_2tif(incoh, output_cc_file, template_tif_path) + +print("\nAll corrected interferograms have been written to disk.") diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/README.md b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/README.md new file mode 100644 index 0000000..aafee2e --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/README.md @@ -0,0 +1,39 @@ +# InSAR Phase Bias Correction +This repository contains scripts for mitigating phase bias in InSAR data. It includes tools for reading data, calculating loop closures, estimating calibration parameters, performing inversion, and correcting interferograms. + +## Directory Structure +- `PhaseBias_01_Read_Data.py`: Reads input wrapped ifgs and coherence data. +- `PhaseBias_02_Loop_Closures.py`: Calculates loop closures. +- `PhaseBias_03_calibration_pars.py`: Estimates the calibration parameters $a_n$. +- `PhaseBias_04_Inversion.py`: Performs inversion for phase bias estimation. +- `PhaseBias_05_Correction.py`: Corrects interferograms using estimated biases. +- `bin/`: Contains auxiliary functions used by the scripts. +- `config.txt`: Configuration file for setting parameters. + +## Usage +1. Clone the repository: + ```bash + git clone https://github.com/YourUsername/InSAR_PhaseBias_Correction.git + cd InSAR_PhaseBias_Correction + +## Test Dataset + +The test dataset used in this repository originates from one of the islands in the Azores. The full dataset can be accessed through the COMET LiCSAR frame **082D_05125_020000**, available at the following link: + +[https://gws-access.jasmin.ac.uk/public/nceo_geohazards/LiCSAR_products/82/082D_05125_020000/](https://gws-access.jasmin.ac.uk/public/nceo_geohazards/LiCSAR_products/82/082D_05125_020000/) + +Additionally, we provide a **sample dataset**, including example interferograms, which can be downloaded from Zenodo: + +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.14525360.svg)](https://doi.org/10.5281/zenodo.14525360) + + +# Citation +If you use this repository for your research, please cite the following paper: + +Maghsoudi, Y., Hooper, A.J., Wright, T.J., Lazecky, M., & Ansari, H. (2022). Characterizing and correcting phase biases in short-term, multilooked interferograms. Remote Sensing of Environment, 275, 113022. +https://doi.org/10.1016/j.rse.2022.113022 + + +# Acknowledgment +This work was funded by the European Space Agency (ESA) as part of the Phase Bias Correction Project. We gratefully acknowledge their support. + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_01_Read_Data.py b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_01_Read_Data.py new file mode 100644 index 0000000..5b72365 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_01_Read_Data.py @@ -0,0 +1,583 @@ +#!/usr/bin/env python3 + +import os +import re +from datetime import datetime, timedelta +import numpy as np +from osgeo import gdal +from bin.multilook_w import multilook_w +import configparser +import sys +import warnings +import pickle +import glob + +warnings.filterwarnings("ignore", category=np.VisibleDeprecationWarning) + +####################################################################### + +HELP_TEXT = """ +PhaseBias_01_Read_Data.py + +This script automates the process of downloading GeoTIFF files for wrapped interferograms and coherence images. +The files can be retrieved either from the COMET-LiCS web portal or from a root path specified in the configuration file. + +Assumptions: +- Interferograms are organized in folders named as `yyyymmdd_yyyymmdd` (e.g., `20230101_20230107`). + +Files Downloaded: +1. `yyyymmdd_yyyymmdd.geo.diff_pha.tif`: + - Contains the wrapped phase image in radians. + - Values range from -3.14 to 3.14. + +2. `yyyymmdd_yyyymmdd.geo.cc.tif`: + - Contains the coherence image of the interferometric pair. + - Values range from 0 to 255, where: + - 0 represents the lowest coherence. + - 255 represents the highest coherence. + +Outputs: +1. `All_ifgs_start_end` (Pickle file, saved as `.pkl`): + - A dictionary containing all interferograms available between the specified start and end dates. +2. `All_coh_start_end` (Pickle file, saved as `.pkl`): + - A dictionary containing all coherence data available between the specified start and end dates. + +Storage Information: +- All output files are saved in the directory `Data/` under the `output_path` defined in the configuration file. + +Additional Output: +- A summary report is generated that includes: + - The number of available interferograms for each temporal baseline. + - The number of missing interferograms. + +""" + +if "--help" in sys.argv: + print(HELP_TEXT) + sys.exit(0) + +################### + +# Config file path +config_file = "config.txt" + +# Check if config file exists +if not os.path.exists(config_file): + raise FileNotFoundError( + f"Error: The configuration file '{config_file}' was not found in the script's directory." + ) + +# Initialize the configparser +config = configparser.ConfigParser() + +# Read the config.txt file +config.read(config_file) + +# Define required parameters +required_parameters = [ + "root_path", + "frame", + "start", + "end", + "interval", + "nlook", + "LiCSAR_data", +] + +# Initialize a dictionary to store parameters +parameters = {} + +# Attempt to retrieve each parameter and catch errors +try: + parameters["root_path"] = config.get("DEFAULT", "root_path") + parameters["output_path"] = config.get("DEFAULT", "output_path") + parameters["frame"] = config.get("DEFAULT", "frame") + parameters["start"] = config.get("DEFAULT", "start") + parameters["end"] = config.get("DEFAULT", "end") + parameters["interval"] = config.getint("DEFAULT", "interval") + parameters["nlook"] = config.getint( + "DEFAULT", "nlook") # Assuming 'nlook' should be an integer + parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") + +except configparser.NoOptionError as e: + # Error message if a required parameter is missing + missing_param = str(e).split(": ")[1] + raise ValueError( + f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." + ) +except configparser.Error as e: + # General error for any configparser-related issues + raise ValueError(f"Error while reading '{config_file}': {e}") + +# Assign to individual variables for easy access +root_path = parameters["root_path"] +output_path = parameters["output_path"] +frame = parameters["frame"] +start = parameters["start"] +end = parameters["end"] +interval = parameters["interval"] +nlook = parameters["nlook"] +LiCSAR_data = parameters["LiCSAR_data"] + +# hardcoded parameters, not included in the config file +landmask = 1 +filtered_ifgs = "yes" +min_baseline = 5 +max_baseline = 366 + +# Print the values to confirm they're being read correctly +print("Root Path:", root_path) +print("output Path:", output_path) +print("Frame:", frame) +print("Start Date:", start) +print("End Date:", end) +print("Interval:", interval) +print("nlook:", nlook) +print("LiCSAR_data", LiCSAR_data) + +################ +# all_ifgs = read_ifgs(frame, start, end, min_baseline, max_baseline, landmask, nlook, interval, LiCSAR_data, filtered_ifgs) + +def read_ifgs( + start, + end, + min_baseline, + max_baseline, + landmask, + nlook, + interval, + LiCSAR_data, + filtered_ifgs, +): + + start_date = datetime.strptime(start, "%Y%m%d") + end_date = datetime.strptime(end, "%Y%m%d") + + # Check if the file ends with "geo.diff_pha.tif" + # if filename.endswith("geo.diff_unfiltered_pha.tif"): # if unfiltered data are used + if filtered_ifgs == "yes": # yes for the filtered ifgs, no for the unfiltered ifgs + req_file_name = "geo.diff_pha.tif" + else: + req_file_name = "geo.diff_unfiltered_pha.tif" + + ############ reading + + if LiCSAR_data == "yes": + track = frame[0:3] # extracting track number from frame id + if track[0] == "0": + track = track[1:3] + if track[0] == "0": + track = track[1:2] + + root_directory = f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}/interferograms/" + else: + root_directory = os.path.join( + root_path, "interferograms" + ) # Use os.path.join for consistent path construction + + ##### Count total files matching the criteria for accurate progress calculation + print(f"Reading data from the path: {root_directory}") + + total_files = sum(1 for dirpath, _, filenames in os.walk(root_directory) + for filename in filenames + if filename.endswith(req_file_name)) + processed_files = 0 + + print("Reading wrapped interferograms: [", end="", flush=True) + ##### + + # Initialize an empty dictionary to store the arrays for each category + category_arrays = {} + last_date = {} + n = 0 + + if landmask is not None: + if LiCSAR_data == "yes": + landmask_path = f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}/metadata/{frame}.geo.landmask.tif" + else: + # Use glob to find the file ending with ".geo.landmask.tif" in the metadata directory + metadata_dir = os.path.join(root_path, "metadata") + landmask_files = glob.glob( + os.path.join(metadata_dir, "*.geo.landmask.tif")) + if landmask_files: + landmask_path = landmask_files[0] # Take the first match + else: + raise FileNotFoundError( + "No file ending with '.geo.landmask.tif' found in the metadata directory." + ) + + # Load the landmask file + landmask_dataset = gdal.Open(landmask_path) + landmask_image = landmask_dataset.ReadAsArray() + array_landmask = np.array(landmask_image) + + n = 1 + # Traverse the root directory and its subdirectories + for dirpath, dirnames, filenames in os.walk(root_directory): + dirnames.sort( + ) # This ensures dirnames are processed in alphabetical order + # Iterate through the files in the current directory + for filename in filenames: + # Check if the file ends with "geo.diff_pha.tif" + # if filename.endswith("geo.diff_unfiltered_pha.tif"): # if unfiltered data are used + if (filtered_ifgs == "yes" + ): # yes for the filtered ifgs, no for the unfiltered ifgs + req_file_name = "geo.diff_pha.tif" + else: + req_file_name = "geo.diff_unfiltered_pha.tif" + + if filename.endswith(req_file_name): + file_path = os.path.join(dirpath, filename) + + #### for printing into the console + processed_files += 1 + percentage = (processed_files / total_files) * 100 + + # Clear previous percentage and print updated one + sys.stdout.write( + "\rReading wrapped phases: [{:<50}] {}%".format( + "=" * int(percentage // 2), int(percentage))) + sys.stdout.flush() + + ##### + + # Extract the dates from the file name + match = re.search(r"(\d+)_(\d+)", filename) + if match: + date1 = match.group(1) + date2 = match.group(2) + + # Convert the dates to datetime objects + date1_obj = datetime.strptime(date1, "%Y%m%d") + date2_obj = datetime.strptime(date2, "%Y%m%d") + + # Check if the dates fall within the specified range + category = (date2_obj - date1_obj).days + + if (start_date <= date1_obj <= end_date + and start_date <= date2_obj <= end_date + and category < max_baseline + and category > min_baseline): + if ( + n == 1 + ): # finding the first and last acquisiton in the time-series + first_acq = date1_obj + else: + last_acq = date2_obj + n = n + 1 + + # Calculate the difference in days between the two dates + + # Open the TIFF file + tiff_dataset = gdal.Open(file_path) + # Read the image data as a NumPy array + tiff_image = tiff_dataset.ReadAsArray() + + # Convert the image to a numpy array + array_data = np.array(tiff_image) + array_data[array_data == 0] = ( + np.nan + ) ## converting the zerso values in wrapped phases into nan. This is helpful when calculating loop closures + + if landmask != None: + array_data[array_landmask != 1] = np.nan + + if ( + nlook != None and nlook != 1 + ): # incase of unwrap data because they are already multilooked to 10 we don't multilook them here + array_data = multilook_w(array_data, nlook) + + if category not in last_date: + last_date[category] = date1_obj + # if (date1_obj - date2_obj) > interval + + # appending None to the category_arrays where there are missing ifgs in the middle of time-series + diff_ifgs = ( + date1_obj - last_date[category] + ).days # /category #difference between the first epochs of two consecutive ifg + if diff_ifgs > interval: + n_no_acq = int(diff_ifgs / interval - + 1) ## number of missing ifgs + for i in range( + n_no_acq + ): # the number of none depends on the number of missing ifgs + # Append the none to the existing array + category_arrays[category].append(None) + last_date[category] = date1_obj + + # Check if the category is already in the dictionary + if category in category_arrays: + # If the category already exists, append the array to the existing array + category_arrays[category].append(array_data) + else: + # If the category doesn't exist, create a new list with the array + category_arrays[category] = [array_data] + + # appending None to the category_arrays where there are missing ifgs in the end of time-series + for category in sorted(category_arrays): + while (last_date[category] + timedelta(category)) < last_acq: + category_arrays[category].append(None) + last_date[category] = last_date[category] + timedelta(interval) + + # appending None to the category_arrays where there are missing ifgs in the begining of time-series + # max number of expected 6-day(i.e. interval) interferograms in the full time-series + max_ifg_number = abs(int((first_acq - last_acq).days / interval)) + + # all the categories in the data e.g. 6, 12, 18 etc + cat = [] + for category in category_arrays: + cat.append(category) + + for i in range(interval, max(cat)+1, interval): + if i in cat: + while len(category_arrays[i]) < (max_ifg_number + 1 - i/interval): + category_arrays[i].insert( 0, None) + + print("\nFinished reading all interferograms.") + + return category_arrays + + +########################################################################################################################### +########################################################################################################################## + + +def read_coh(start, end, min_baseline, max_baseline, nlook, landmask, interval, + LiCSAR_data): + + start_date = datetime.strptime(start, "%Y%m%d") + end_date = datetime.strptime(end, "%Y%m%d") + + ############ reading + + if LiCSAR_data == "yes": + track = frame[0:3] + if track[0] == "0": + track = track[1:3] + if track[0] == "0": + track = track[1:2] + + root_directory = f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}/interferograms/" + else: + root_directory = os.path.join( + root_path, "interferograms" + ) # Use os.path.join for consistent path construction + + req_file_name = "geo.cc.tif" + + ##### Count total files matching the criteria for accurate progress calculation + + total_files = sum(1 for dirpath, _, filenames in os.walk(root_directory) + for filename in filenames + if filename.endswith(req_file_name)) + processed_files = 0 + + print("Reading coherence data: [", end="", flush=True) + ##### + + # Initialize an empty dictionary to store the arrays for each category + category_arrays = {} + last_date = {} + n = 0 + + if landmask is not None: + if LiCSAR_data == "yes": + landmask_path = f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}/metadata/{frame}.geo.landmask.tif" + else: + # Use glob to find the file ending with ".geo.landmask.tif" in the metadata directory + metadata_dir = os.path.join(root_path, "metadata") + landmask_files = glob.glob( + os.path.join(metadata_dir, "*.geo.landmask.tif")) + if landmask_files: + landmask_path = landmask_files[0] # Take the first match + else: + raise FileNotFoundError( + "No file ending with '.geo.landmask.tif' found in the metadata directory." + ) + + # Load the landmask file + landmask_dataset = gdal.Open(landmask_path) + landmask_image = landmask_dataset.ReadAsArray() + array_landmask = np.array(landmask_image) + + n = 1 + # Traverse the root directory and its subdirectories + for dirpath, dirnames, filenames in os.walk(root_directory): + # Iterate through the files in the current directory + for filename in filenames: + # Check if the file ends with "geo.diff_pha.tif" + + if filename.endswith(req_file_name): + file_path = os.path.join(dirpath, filename) + + #### for printing into the console + processed_files += 1 + percentage = (processed_files / total_files) * 100 + + # Clear previous percentage and print updated one + sys.stdout.write( + "\rReading coherence data: [{:<50}] {}%".format( + "=" * int(percentage // 2), int(percentage))) + sys.stdout.flush() + + ##### + + # Extract the dates from the file name + match = re.search(r"(\d+)_(\d+)", filename) + if match: + date1 = match.group(1) + date2 = match.group(2) + + # Convert the dates to datetime objects + date1_obj = datetime.strptime(date1, "%Y%m%d") + date2_obj = datetime.strptime(date2, "%Y%m%d") + + # Check if the dates fall within the specified range + category = (date2_obj - date1_obj).days + + if (start_date <= date1_obj <= end_date + and start_date <= date2_obj <= end_date + and category < max_baseline + and category > min_baseline): + if ( + n == 1 + ): # finding the first and last acquisiton in the time-series + first_acq = date1_obj + else: + last_acq = date2_obj + n = n + 1 + + # Calculate the difference in days between the two dates + + # Open the TIFF file + tiff_dataset = gdal.Open(file_path) + + # Read the image data as a NumPy array + tiff_image = tiff_dataset.ReadAsArray() + + # Convert the image to a numpy array + array_data = np.array(tiff_image) + # Convert array_data to a float type that can accommodate np.nan + array_data = array_data.astype(np.float32) + + if landmask != 0: + array_data[array_landmask != 1] = np.nan + + if ( + nlook != None and nlook != 1 + ): # incase of unwrap data because they are already multilooked to 10 we don't multilook them here + array_data = multilook(array_data, nlook) + + if category not in last_date: + last_date[category] = date1_obj + # if (date1_obj - date2_obj) > interval + + # appending None to the category_arrays where there are missing ifgs in the middle of time-series + diff_ifgs = ( + date1_obj - last_date[category] + ).days # /category #difference between the first epochs of two consecutive ifg + if diff_ifgs > interval: + n_no_acq = int(diff_ifgs / interval - + 1) ## number of missing ifgs + for i in range( + n_no_acq + ): # the number of none depends on the number of missing ifgs + # Append the none to the existing array + category_arrays[category].append(None) + last_date[category] = date1_obj + + # Check if the category is already in the dictionary + if category in category_arrays: + # If the category already exists, append the array to the existing array + category_arrays[category].append(array_data) + else: + # If the category doesn't exist, create a new list with the array + category_arrays[category] = [array_data] + + # appending None to the category_arrays where there are missing ifgs in the end of time-series + for category in sorted(category_arrays): + while (last_date[category] + timedelta(category)) < last_acq: + category_arrays[category].append(None) + last_date[category] = last_date[category] + timedelta(interval) + + # appending None to the category_arrays where there are missing ifgs in the begining of time-series + # max number of expected 6-day(i.e. interval) interferograms in the full time-series + max_ifg_number = abs(int((first_acq - last_acq).days / interval)) + + # all the categories in the data e.g. 6, 12, 18 etc + cat = [] + for category in category_arrays: + cat.append(category) + + for i in range(interval, max(cat)+1, interval): + if i in cat: + while len(category_arrays[i]) < (max_ifg_number + 1 - i/interval): + category_arrays[i].insert(0, None) + + print("\nFinished reading all coherence data.") + + return category_arrays + + +########################################################################################################################## +########################################################################################################################## + +all_ifgs = read_ifgs( + start, + end, + min_baseline, + max_baseline, + landmask, + nlook, + interval, + LiCSAR_data, + filtered_ifgs, +) +# Define the directory path +directory_path = os.path.join(output_path, "Data") + +# Create the directory if it doesn't exist +os.makedirs(directory_path, exist_ok=True) + +# Define the file path +#file_path = os.path.join(directory_path, f"_all_ifgs_{start}_{end}") + +# Remove all files inside the directory +if os.path.exists(directory_path): + files = glob.glob(os.path.join(directory_path, "*")) + for file in files: + os.remove(file) + + +# for writing the all_loops +file_path = output_path + "/Data/" + "All_ifgs_" + start + "_" + end + ".pkl" +with open(file_path, "wb") as file: # for writting a dictionary to a file + pickle.dump(all_ifgs, file) + +# Free memory before starting coherence +import gc +del all_ifgs +gc.collect() + +################################################################ + +all_coh = read_coh(start, end, min_baseline, max_baseline, landmask, nlook, + interval, LiCSAR_data) + +## for writting the all_coh +file_path = output_path + "/Data/" + "All_coh_" + start + "_" + end + ".pkl" +with open(file_path, "wb") as file: # for writting a dictionary to a file + pickle.dump(all_coh, file) + +################################################ + +# After processing all interferograms, generate the report +print("\nReport on the number of IFGs and missing IFGs per category:") +for category in sorted(all_ifgs): + num_ifgs = sum(1 for item in all_ifgs[category] + if item is not None) # Count valid IFGs + num_missing_ifgs = sum(1 for item in all_ifgs[category] + if item is None) # Count missing IFGs + print( + f"{category}-days: Number of IFGs: {num_ifgs} Number of missing IFGs: {num_missing_ifgs}" + ) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_02_Loop_Closures.py b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_02_Loop_Closures.py new file mode 100644 index 0000000..9c6165d --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_02_Loop_Closures.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 + +import numpy as np +import pickle +import glob +import configparser +import os +import sys +import warnings +warnings.filterwarnings("ignore", category=np.VisibleDeprecationWarning) + +################################################################################################ + +HELP_TEXT = """ +PhaseBias_02_Loop_Closures.py + +This script calculates loop closures (Δφ) using the interferograms imported in the first step. + +Definition of Loop Closures: +Loop closures, Δφ, are calculated for the epochs between `i` and `k`, and are defined as: + Δφ_(i,k) = |φ_(i,k) - ∑_(t=i)^k φ_(t,t+1)|_2π + +Where: +- φ_(i,j): Represents the phase difference for a pixel in the interferogram formed between epochs `i` and `j`. +- |.|_2π: Indicates that the result is wrapped modulo 2π (i.e., values range from -π to π). + +Key Information: +- Nonzero closure phase is a by-product of spatial filtering/multilooking and is primarily associated with changes in the scattering and electrical properties of the ground surface. +- The calculated loop closures are based on the minimum temporal-baseline interferograms defined in the configuration file (parameter: `interval`) and are referred to as base interferograms. These may represent: + - 12- and 6-day closures (or 24- and 12-day closures), Δφ_(i,i+2) + - 18- and 6-day closures (or 36- and 12-day closures), Δφ_(i,i+3) + The distinction depends on whether the base interferograms are 6-day or 12-day intervals. + +Outputs: +- The calculated loop closures are stored as a dictionary in a file named: + `All_loops_start_end.pkl` +- The output is saved in the directory `Data/` under the `output_path` defined in the configuration file. + +Loop Closure Report: +At the end of the script, a loop closure report is displayed. This report includes: +- The number of generated and missing loop closures for short-interval closures, such as 12-6 and 18-6. +- The number of generated and missing long-interval closures, obtained using long-term interferograms, such as 204-6 and 204-12. + +Usage: +1. Ensure the interferograms from the first script are available. +2. Run the script to generate loop closures for the specified temporal baselines. + +Example: +```bash +python PhaseBias_02_Loop_Closures.py + + +""" +if "--help" in sys.argv: + print(HELP_TEXT) + sys.exit(0) + +################################################################################################ +# Config file path +config_file = "config.txt" + +# Check if config file exists +if not os.path.exists(config_file): + raise FileNotFoundError( + f"Error: The configuration file '{config_file}' was not found in the script's directory." + ) + +# Initialize the configparser +config = configparser.ConfigParser() + +# Read the config.txt file +config.read(config_file) + +# Define required parameters +required_parameters = [ + "root_path", + "frame", + "start", + "end", + "interval", + "nlook", + "LiCSAR_data", +] + +# Initialize a dictionary to store parameters +parameters = {} + +# Attempt to retrieve each parameter and catch errors +try: + parameters["root_path"] = config.get("DEFAULT", "root_path") + parameters["output_path"] = config.get("DEFAULT", "output_path") + parameters["frame"] = config.get("DEFAULT", "frame") + parameters["start"] = config.get("DEFAULT", "start") + parameters["end"] = config.get("DEFAULT", "end") + parameters["interval"] = config.getint("DEFAULT", "interval") + parameters["nlook"] = config.getint( + "DEFAULT", "nlook") # Assuming 'nlook' should be an integer + parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") + +except configparser.NoOptionError as e: + # Error message if a required parameter is missing + missing_param = str(e).split(": ")[1] + raise ValueError( + f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." + ) +except configparser.Error as e: + # General error for any configparser-related issues + raise ValueError(f"Error while reading '{config_file}': {e}") + +# Assign to individual variables for easy access +root_path = parameters["root_path"] +output_path = parameters["output_path"] +frame = parameters["frame"] +start = parameters["start"] +end = parameters["end"] +interval = parameters["interval"] +nlook = parameters["nlook"] +LiCSAR_data = parameters["LiCSAR_data"] + +# hardcoded parameters, not included in the config file +landmask = 1 +filtered_ifgs = "yes" +max_loop = 5 # in case of 3 all loops with 6,12,18-day will be calculated (e.g. (60,6), (60,12) and (60,18)) + + +############################################################################################# +################################################################################################# +def loop_calc(max_loop): + + ## Read ifgs: + + # Define the path components + sub_dir = "Data" + filename_pattern = "All_ifgs_*" # Pattern to match files starting with "All_ifgs" + + # Construct the full file path with the refined pattern + file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) + # Use glob to find the file and ensure it exists + file_list = glob.glob(file_path_pattern) + if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + all_ifgs = pickle.load(file) + else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) + + loop = {} + missing = {} + + cat = sorted(all_ifgs.keys()) # Get sorted categories + # print('cat = ', cat) + + # Initialize counters for reporting + loop_counts = { + category: { + cat[l]: 0 + for l in range(max_loop) + } + for category in sorted(all_ifgs.keys()) + } + missing_counts = { + category: { + cat[l]: 0 + for l in range(max_loop) + } + for category in sorted(all_ifgs.keys()) + } + + for category in cat: + loop[category] = { + } # Initialize the key in the loop dictionary with an inner dictionary + missing[category] = [] # second ver + for l in range( + max_loop + ): # it was up to 3 in the first version. but this allows to go for longer loops e.g. 288-72 (cat[11] is 72) + loop[category][cat[l]] = [ + ] # Initialize the key in the inner dictionary + # missing[cat[l]] = [] # first ver + # Calculate total iterations for progress percentage + total_iterations = sum( + len(all_ifgs[cat[i]]) for l in range(max_loop) + for i in range(1, len(cat)) + if cat[i] % cat[l] == 0 and cat[i] != cat[l]) + completed_iterations = 0 + + # Display the message before starting the progress bar + print( + f"Calculating all {interval}, {2 * interval}, and {3 * interval} loop closures." + ) + + for l in range( + 0, max_loop + ): # l could be 6, 12 and 18 to calculate e.g. 36-6 or 36-12 or 36-18 + for i in range(1, len(cat)): # index for the category e.g. 6, 12, 18 + if cat[i] % cat[l] == 0 and cat[i] != cat[ + l]: # such as 12/6 or 24/12 + for t in range(len(all_ifgs[ + cat[i]])): # index for the epochs in each category + + ####################progress bar + completed_iterations += 1 + percentage = (completed_iterations / + total_iterations) * 100 + + # Display progress percentage + sys.stdout.write("\rProgress: [{:<50}] {}%".format( + "=" * int(percentage // 2), int(percentage))) + sys.stdout.flush() + ################################## + + if cat[l] == interval: + end_index = t + int(cat[i] / cat[l]) + # print('end_index = ', end_index) + else: + end_index = t + int(cat[i] / cat[l]) * int( + cat[l] / interval) + + # recording the missing ifgs in each cat e.g. 6,12,18 + #if np.any(np.array(all_ifgs[cat[i]])[t] == None): + if np.any(np.array(all_ifgs[cat[i]], dtype=object)[t] == None): + missing[cat[i]].append( + t + ) # to record the missing ifgs index for each cateogory e.g. 18 + + + for e in range(t, end_index, int(cat[l]/interval)): # to record the missing ifgs index for the period of each loop + elem = all_ifgs[cat[l]][e] + + if elem is None: + missing[cat[l]].append(e) + + + + if all_ifgs[cat[i]][t] is None or any(all_ifgs[cat[l]][elem] is None for elem in range(t, end_index, int(cat[l] / interval))): + + loop[cat[i]][cat[l]].append(None) + missing_counts[cat[i]][cat[l]] += 1 # Increment missing loop counter + + else: + closure = np.angle( np.exp( 1j * ( np.array(all_ifgs[cat[i]][t], dtype=np.float32)- np.sum( np.array(all_ifgs[cat[l]][t:end_index:int(cat[l] / interval)], dtype=np.float32), axis=0)))) + + + if cat[i] in loop: + loop[cat[i]][cat[l]].append(closure) + loop_counts[cat[i]][cat[ + l]] += 1 # Increment successful loop counter + + else: + loop[cat[i]][cat[l]] = [closure] + print("\n Finished calculating the loop closures.") + + # Generate report + print("\n=== Loop Closure Report ===") + for category in [ + 2 * interval, + 3 * interval, + ]: # Restrict to relevant categories e.g. 18-6 and 12-6 as the main observations + for l in [interval]: # Restrict to the loop levels of interest + if category in loop_counts and l in loop_counts[category]: + generated = loop_counts[category].get(l, 0) + missing = missing_counts[category].get(l, 0) + print( + f"Loop Closure {category} - {l}: Generated = {generated}, Missing = {missing}" + ) + + print("\n=== Loop Closure Report for Long-interval Loop Closures ===") + + # Iterate over categories greater than 200 + for category in cat: + if category > 200: # Check for long categories + for l in range( + 1, + 5 * interval): # Check loop levels less than 5 * interval + if category in loop_counts and l in loop_counts[category]: + generated = loop_counts[category].get(l, 0) + if generated >= 1: # Only report if at least one loop is generated + print( + f"Loop Closure {category} - {l}: Generated = {generated}" + ) + + return loop, missing + + +################################################################################################ + +all_loops, missing = loop_calc(max_loop) + +# Define the directory path +directory_path = os.path.join(output_path, "Data") + +# Create the directory if it doesn't exist +os.makedirs(directory_path, exist_ok=True) + +# Define the file path +file_path = os.path.join(directory_path, f"All_loops_{start}_{end}.pkl") + +with open(file_path, "wb") as file: # for writting a dictionary to a file + pickle.dump(all_loops, file) + +### + +# Define the file path +file_path = os.path.join(directory_path, f"missing_loops") + +with open(file_path, "wb") as file: # for writting a dictionary to a file + pickle.dump(missing, file) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_03_calibration_pars.py b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_03_calibration_pars.py new file mode 100644 index 0000000..ce3aba9 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_03_calibration_pars.py @@ -0,0 +1,424 @@ +#!/usr/bin/env python3 + +import numpy as np +import matplotlib.pyplot as plt +from matplotlib.gridspec import GridSpec +import pickle +import os +import sys +import glob +import configparser +import warnings + +with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=np.VisibleDeprecationWarning) + +################################################################################ + +HELP_TEXT = """ +PhaseBias_03_calibration_pars.py + +This script estimates the calibration parameters a_n using the loop closures calculated in the previous step. + +The calibration parameters a_n are estimated by calculating the ratio of the long-term loop closure for the interferogram of interest to the long-term loop closure for the base interferogram. Assuming negligible bias in long-term interferograms, a_n is calculated as follows: + + a_n = |φ(i,i+l) - Σ(t=i to i+l)φ(t,t+1+n)|_2π / |φ(i,i+l) - Σ(t=i to i+l)φ(t,t+1)|_2π + +where: +- φ(i,i+l) is a long-term interferogram connecting epoch i to i+l. +- φ(t,t+1) represents the base interferogram. +- φ(t,t+1+n) is the short interferogram to be corrected using a_n. + +### Input: +1. Loop closures from the previous step (e.g., 216-6, 216-12, and 216-18 for 6-day acquisition intervals). +2. The long-term interferogram length (e.g., 216 days) must be specified at the beginning. +3. The number of a_n parameters to estimate, specified using `num_a`. For example: + - Setting `num_a=2` indicates that a1 and a2 will be estimated (used to correct 12- and 18-day interferograms). + - For correcting longer interferograms, `num_a` should be set accordingly. + +### Output: +1. All possible a_n values are calculated. +2. Visualizations: + - Example plots of a_n arrays and their corresponding histograms, illustrating spatial variations. + - Time-series plots showing how a_n values evolve over time. +3. Mean a_n values over time are output for further use. + +### File Outputs: +- Results are stored in the output directory defined in the `config.txt` file under `/Data/` as `Mean_an_values.pkl`. + +### Additional Notes: +- Users should ensure that `num_a` aligns with the desired temporal baselines for correction. + +""" +if "--help" in sys.argv: + print(HELP_TEXT) + sys.exit(0) + + +############################################################################### +# Config file path +config_file = "config.txt" + +# Check if config file exists +if not os.path.exists(config_file): + raise FileNotFoundError( + f"Error: The configuration file '{config_file}' was not found in the script's directory." + ) + +# Initialize the configparser +config = configparser.ConfigParser() + + +# Read the config.txt file +config.read(config_file) + +# Define required parameters +required_parameters = [ + "root_path", + "frame", + "start", + "end", + "interval", + "nlook", + "LiCSAR_data", +] + +# Initialize a dictionary to store parameters +parameters = {} + +# Attempt to retrieve each parameter and catch errors +try: + parameters["root_path"] = config.get("DEFAULT", "root_path") + parameters["output_path"] = config.get("DEFAULT", "output_path") + parameters["frame"] = config.get("DEFAULT", "frame") + parameters["start"] = config.get("DEFAULT", "start") + parameters["end"] = config.get("DEFAULT", "end") + parameters["interval"] = config.getint("DEFAULT", "interval") + parameters["nlook"] = config.getint( + "DEFAULT", "nlook" + ) # Assuming 'nlook' should be an integer + parameters["num_a"] = config.getint("DEFAULT", "num_a") + parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") + +except configparser.NoOptionError as e: + # Error message if a required parameter is missing + missing_param = str(e).split(": ")[1] + raise ValueError( + f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." + ) +except configparser.Error as e: + # General error for any configparser-related issues + raise ValueError(f"Error while reading '{config_file}': {e}") + +# Assign to individual variables for easy access +root_path = parameters["root_path"] +output_path = parameters["output_path"] +frame = parameters["frame"] +start = parameters["start"] +end = parameters["end"] +interval = parameters["interval"] +nlook = parameters["nlook"] +num_a = parameters["num_a"] +LiCSAR_data = parameters["LiCSAR_data"] + +# hardcoded parameters, not included in the config file +landmask = 1 +filtered_ifgs = "yes" +max_loop = 5 # in case of 3 all loops with 6,12,18-day will be calculated (e.g. (60,6), (60,12) and (60,18)) + + +##################################################### Loop Closure calculation ##################################################### +#################################################################################################################################### +track = frame[0:3] +if track[0] == "0": + track = track[1:3] +if track[0] == "0": + track = track[1:2] + + +###################### Read coh: +print("\nReading all coherence data...") +# Define the path components +sub_dir = "Data" +filename_pattern = "All_coh_*" # Pattern to match files starting with "All_ifgs" + +# Construct the full file path with the refined pattern +file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + coh = pickle.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) +print("Reading coherence data completed.") + + +##################### Read Loops + +print("\nReading all loop closures...") +filename_pattern = "All_loops_*" # Pattern to match files starting with "All_ifgs" + +# Construct the full file path with the refined pattern +file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + all_loops = pickle.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) + +print("Reading loop closures completed.") + + +##################################################### Loop Closure calculation ##################################################### +#################################################################################################################################### + + +long_baseline = 216 # 36,72,108,144,180,216,252,288,324 +coh_thresh = 25 + + +def calc_an(loop_360_n, loop_360_6): + """ + Calculates the a_n values from loop closures. + + Parameters: + loop_360_n (ndarray): Array of loop closures for n-day intervals. + loop_360_6 (ndarray): Array of loop closures for 6-day intervals. + + Returns: + ndarray: Mean a_n values for all loop closures. + list: List of all a_n arrays for further processing or plotting. + """ + ratios = [] + + # Loop through indices + for i in range(len(loop_360_6)): + if loop_360_6[i] is not None and loop_360_n[i] is not None: + # Calculate the ratio for non-NaN elements + ratio_i = np.divide( + loop_360_n[i], + loop_360_6[i], + out=np.full_like(loop_360_n[i], np.nan), + where=~np.isnan(loop_360_6[i]), + ) + ratios.append(ratio_i) + +# Check if `ratios` is empty before proceeding + if not ratios: + print("Warning: No valid a_n values could be calculated. Returning NaN values.") + return np.array([np.nan]), [] + + + # Convert the list of ratios to a numpy array + an = np.array(ratios, dtype=object) + mean_an_long_baseline = [] + + # Calculate mean values of each a_n + for arr in an: + mask = (arr > 0) & (arr < 1) # Apply mask to filter invalid values + arr[~mask] = np.nan + if not np.isnan(arr).all(): # Check if all elements are NaN + mean_an = np.nanmean(arr) + else: + mean_an = np.nan # Assign NaN for completely empty slices + + mean_an_long_baseline.append(mean_an) + + return np.array(mean_an_long_baseline), an + + +################################################################################## +############################ +def plot_an(an_arrays, mean_a_long_baseline, k): + """ + Plots the a_n arrays with the largest coverage and their histograms in a two-row layout, + as well as the mean values of calibration parameters a_n over time. + + Parameters: + an_arrays (list): List of a_n arrays. + mean_a_long_baseline (ndarray): Array of mean calibration parameter values for a_1, a_2, ..., a_n. + k (int): The current value of k (e.g., 2 for a1, 3 for a2). + """ + + # Calculate the coverage (number of non-NaN values) for each array + coverage = [] + valid_an_arrays = [] # Store only valid arrays for plotting + + for arr in an_arrays: + if ( + arr is not None and arr.ndim == 2 and not np.all(np.isnan(arr)) + ): # Check for valid 2D arrays + coverage.append(np.sum(~np.isnan(arr))) # Count non-NaN values + valid_an_arrays.append(arr) + + # Sort arrays by coverage in descending order + sorted_indices = np.argsort(coverage)[::-1] + selected_arrays = [ + valid_an_arrays[idx] for idx in sorted_indices[:6] + ] # Select top 6 arrays + + # Plot the a_n arrays and their histograms in a two-row layout + if selected_arrays: + num_arrays = len(selected_arrays) + fig = plt.figure(figsize=(4 * num_arrays, 8)) + gs = GridSpec( + 2, num_arrays + 1, width_ratios=[1] * num_arrays + [0.1] + ) # Add space for colorbar + + # Plot selected arrays + axes_img = [] + for i, arr in enumerate(selected_arrays): + ax_img = fig.add_subplot(gs[0, i]) + im = ax_img.imshow(arr, cmap="RdYlBu", vmin=-0.8, vmax=0.8) + ax_img.set_title(f"{i+1}th $a_{{{k-1}}}$ Array") # Dynamically set title + ax_img.axis("off") + axes_img.append(ax_img) + + # Add a single colorbar for all array plots + cbar_ax = fig.add_subplot(gs[0, -1]) + fig.colorbar(im, cax=cbar_ax, orientation="vertical", label="Color Scale") + + # Plot histograms for the selected arrays + for i, arr in enumerate(selected_arrays): + ax_hist = fig.add_subplot(gs[1, i]) + ax_hist.hist(arr.flatten(), bins=100, color="blue", alpha=0.7, density=True) + ax_hist.set_title( + f"{i+1}th $a_{{{k-1}}}$ Histogram" + ) # Dynamically set title + ax_hist.set_xlim([-10, 10]) # Adjust histogram range as needed + + plt.tight_layout() + plt.show() + else: + print("No valid a_n arrays available for plotting.") + + # Plot the mean values of a_n over time + x_values = np.arange(len(mean_a_long_baseline)) # Sequential indices for x-axis + fig, ax = plt.subplots(figsize=(6, 4)) + + + # Filter out NaN values to avoid gaps in the plot + valid_indices = ~np.isnan(mean_a_long_baseline) # Mask for valid (non-NaN) entries + x_valid = x_values[valid_indices] # Only use valid time steps + y_valid = np.array(mean_a_long_baseline)[valid_indices] # Only use valid a_n values + + + # Remove NaN values for trendline fitting + valid_indices = ~np.isnan(mean_a_long_baseline) # Mask for valid (non-NaN) entries + if ( + valid_indices.sum() > 1 + ): # Ensure there are at least two valid points for fitting +# x_valid = x_values[valid_indices] + y_valid = np.array(mean_a_long_baseline)[valid_indices] + x_valid = np.arange(len(y_valid)) + + # Fit and plot a trendline + coefficients = np.polyfit(x_valid, y_valid, 1) + trendline = np.polyval(coefficients, x_valid) + ax.plot(x_valid, trendline, color="red", label="Trendline") + + # Scatter plot for all points (including NaN if any) + ax.scatter( + #x_values, + #mean_a_long_baseline, + x_valid, # Only use valid time steps + y_valid, # Only use valid a_n values + + color="blue", + marker="o", + label="Mean Values", + s=9, + ) + + # Set labels and limits + ax.set_ylim(-3, 3) # Adjust limits as needed + ax.set_xlabel("Time Step") # Sequential index as x-axis + ax.set_ylabel("Mean Values", fontsize=12) + ax.legend() + + # Adjust layout and show the plot + plt.tight_layout() + plt.show() + + +## +######### forming 360-6 + +loop_360_6_all = np.array(all_loops[long_baseline][interval], dtype=object) +coh_360 = np.array(coh[long_baseline][:], dtype=object) + +loop_360_6 = np.full_like( + loop_360_6_all, np.nan +) # create a new array as the same size of loop_360_6_all with nan values + + +for i, arr in enumerate(loop_360_6_all): + if arr is not None: + (frame_row, frame_col) = np.shape(arr) + loop_360_6[i] = np.where((coh_360[i] > coh_thresh), arr, np.nan) + +###### forming 360-n (n=12,18,24 etc) +# mean_a_long_baseline = [] +final_mean_values = [] +an_labels = [] + +# Loop through the values of k corresponding to the number of calibration parameters +for k in range(2, 2 + num_a): # k=2 for a1, k=3 for a2, etc. + mean_an_long_baseline = [] + + # Forming 360-n (n=12, 18, 24, etc.) + loop_360_n_all = np.array(all_loops[long_baseline][k * interval], dtype=object) + loop_360_n = np.full_like(loop_360_n_all, np.nan) + + for i, arr in enumerate(loop_360_n_all): + if arr is not None: + # Apply coherence threshold filtering + loop_360_n[i] = np.where((coh_360[i] > coh_thresh), arr, np.nan) + + # Calculate a_n values + mean_an_long_baseline, an_arrays = calc_an(loop_360_n, loop_360_6) + + # If all values in mean_an_long_baseline are nan (empty ratios), stop execution + if np.isnan(mean_an_long_baseline).all(): + print("No valid a_n values could be calculated. Stopping execution.") + sys.exit(1) # Exit the script with an error code (1 means failure) + + # Calculate the mean of all mean values for this k and store it + overall_mean = np.nanmean(mean_an_long_baseline) + final_mean_values.append(overall_mean) + an_labels.append(f"a{k-1}") + + for idx, mean_a in enumerate(mean_an_long_baseline, start=1): + if not np.isnan(mean_a): # Check if the value is not NaN + print(f"Mean of the {idx}th a({k-1}) is {mean_a:.6f}") + + plot_an(an_arrays, mean_an_long_baseline, k) + + +# Print the final mean values across all k +for k_idx, final_mean in enumerate(final_mean_values, start=2): + print(f"Final mean value for a({k_idx-1}) is {final_mean:.6f}") + +# Write final mean values to a text file +output_path = os.path.join(parameters["output_path"], "Data") +os.makedirs(output_path, exist_ok=True) # Create directory if it doesn't exist +output_file = os.path.join(output_path, "an.txt") + +with open(output_file, "w") as file: + for label, mean_value in zip(an_labels, final_mean_values): + file.write(f"{label}={mean_value:.6f}\n") + +print(f"Final mean values written to {output_file}.") diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_04_Inversion.py b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_04_Inversion.py new file mode 100644 index 0000000..822a56e --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_04_Inversion.py @@ -0,0 +1,673 @@ +#!/usr/bin/env python3 + +import numpy as np +import pickle +import os +from scipy.sparse import csc_array +from scipy.sparse.linalg import lsqr +import sys +import math +from bin.read_orig_ifgs_coh import read_orig_ifgs_coh +from bin.circular_mean_var import circular_mean_and_variance_over_epochs +import glob +import configparser +import warnings + +warnings.filterwarnings("ignore", category=np.VisibleDeprecationWarning) + +################################################################################ + +HELP_TEXT = """ +PhaseBias_04_Inversion.py + +This script estimates the phase bias terms for the base interferograms using short-term closure phases as input. +The bias terms derived in this step serve as the foundation for correcting other short-term interferograms. + +### Workflow: +1. **Masking Noisy Loop Closures:** + - A moving average is applied to the time-series of loop closures to account for seasonal fluctuations. + - A circular moving average (mean of complex values over time) is computed for each pixel. + - The distance of each loop closure from the moving average is calculated. If the distance exceeds a threshold (2σ), the point is masked as noisy. + - The circular standard deviation (σ) is computed using the von Mises distribution (circular normal distribution). + +2. **Refined Loop Closures:** + - Masked loop closures, denoted as 〖Δφ〗_(i,i+2)^r and 〖Δφ〗_(i,i+3)^r, are used as primary observations for phase bias estimation. + +3. **First Inversion (Without Temporal Smoothing Constraints):** + - Estimates the bias terms that can be corrected directly from the observed loop closures. + +4. **Second Inversion (With Temporal Smoothing Constraints):** + - Incorporates temporal smoothing constraints to estimate additional bias terms that cannot be directly derived from loop closures. + - The temporal smoothing minimizes differences between bias terms over time, ensuring consistency. + +5. **Combining Results:** + - Bias terms from the first inversion are combined with the results of the second inversion to include terms not estimated in the first round. + +### Outputs: +- The phase bias terms for the base interferograms are stored as a NumPy array file: + - `X_base_ifgs_biases.npy`, saved in the `Data` directory under the `output_path` defined in `config.txt`. + + +### Input Requirements: +- Short-term closure phases (from `PhaseBias_02_Loop_Closures.py`). +- Configuration parameters, including thresholds and temporal settings, defined in `config.txt`. + +### Output Files: +- `X_base_ifgs_biases.npy`: Stores the estimated bias terms for the base interferograms. +""" + +if "--help" in sys.argv: + print(HELP_TEXT) + sys.exit(0) + +############################################################################### +# Config file path +config_file = "config.txt" + +# Check if config file exists +if not os.path.exists(config_file): + raise FileNotFoundError( + f"Error: The configuration file '{config_file}' was not found in the script's directory." + ) + +# Initialize the configparser +config = configparser.ConfigParser() + +# Read the config.txt file +config.read(config_file) + +# Define required parameters +required_parameters = [ + "root_path", + "frame", + "start", + "end", + "interval", + "nlook", + "LiCSAR_data", + "a1_6_day", + "a2_6_day", + "a1_12_day", + "a2_12_day", + "estimate_an_values", +] + +# Initialize a dictionary to store parameters +parameters = {} + +# Attempt to retrieve each parameter and catch errors +try: + parameters["root_path"] = config.get("DEFAULT", "root_path") + parameters["output_path"] = config.get("DEFAULT", "output_path") + parameters["frame"] = config.get("DEFAULT", "frame") + parameters["start"] = config.get("DEFAULT", "start") + parameters["end"] = config.get("DEFAULT", "end") + parameters["interval"] = config.getint("DEFAULT", "interval") + parameters["nlook"] = config.getint( + "DEFAULT", "nlook" + ) # Assuming 'nlook' should be an integer + parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") + parameters["a1_6_day"] = config.getfloat("DEFAULT", "a1_6_day") + parameters["a2_6_day"] = config.getfloat("DEFAULT", "a2_6_day") + parameters["a1_12_day"] = config.getfloat("DEFAULT", "a1_12_day") + parameters["a2_12_day"] = config.getfloat("DEFAULT", "a2_12_day") + parameters["estimate_an_values"] = config.get("DEFAULT", "estimate_an_values") + +except configparser.NoOptionError as e: + # Error message if a required parameter is missing + missing_param = str(e).split(": ")[1] + raise ValueError( + f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." + ) +except configparser.Error as e: + # General error for any configparser-related issues + raise ValueError(f"Error while reading '{config_file}': {e}") + +# Assign to individual variables for easy access +root_path = parameters["root_path"] +output_path = parameters["output_path"] +frame = parameters["frame"] +start = parameters["start"] +end = parameters["end"] +interval = parameters["interval"] +nlook = parameters["nlook"] +LiCSAR_data = parameters["LiCSAR_data"] +estimate_an_values = parameters["estimate_an_values"] + + +# Initialize a1 and a2 with None for safety +a1 = None +a2 = None + + +print("estimate_an_values is ", estimate_an_values) +# Check if estimate_an_values is 'yes' +if estimate_an_values == "yes": + an_file_path = os.path.join(output_path, "Data", "an.txt") + + # Try reading the file + try: + if os.path.exists(an_file_path): + print(f"Reading 'a' values from {an_file_path}...") + + # Read the file and load the values + with open(an_file_path, "r") as file: + for line in file: + line = line.strip() + if line.startswith("a1="): + a1 = float(line.split("=")[1]) + elif line.startswith("a2="): + a2 = float(line.split("=")[1]) + + # Check if both a1 and a2 were found + if a1 is None or a2 is None: + print( + "Warning: Missing a1 or a2 in an.txt. Switching to default values from config file." + ) + raise ValueError("Incomplete a1 or a2 values.") + else: + print( + f"Warning: {an_file_path} does not exist. Switching to default values from config file." + ) + raise FileNotFoundError + + except (FileNotFoundError, ValueError): + # Fallback to default values from config file + if interval == 6: + a1 = parameters["a1_6_day"] + a2 = parameters["a2_6_day"] + else: + a1 = parameters["a1_12_day"] + a2 = parameters["a2_12_day"] +else: + # Use default values from the config file + print("Using default 'an' values from the config file.") + if interval == 6: + a1 = parameters["a1_6_day"] + a2 = parameters["a2_6_day"] + else: + a1 = parameters["a1_12_day"] + a2 = parameters["a2_12_day"] + +# Print the chosen values +print(f"Final values - a1: {a1}, a2: {a2}") + + +##################################################### Loop Closure calculation ##################################################### +#################################################################################################################################### +#################################################################################################################################### + +track = frame[0:3] +if track[0] == "0": + track = track[1:3] +if track[0] == "0": + track = track[1:2] + +with_temporals = "yes" +apply_to_all = "no" # in case of with_temporal='yes', we can decide if you want to use the temporal to all ifgs (i.e. apply_to_all='yes'), or just to those that can not be corrected (i.e. apply_to_all='no') +w = 0.1 # the weigth of the temporal smoothing constraints +####### +coh_thresh = 11 # threhsold on the coherence values. In this version it is off +min_num_eq = 10 # I used 10 in all my experiments. min number of equations in the least square inversion. + +############################### Reading input data ############################################# + +##################### Read ifgs: +#print("Reading all ifgs...") +## Define the path components +#sub_dir = "Data" +#filename_pattern = "All_ifgs_*" # Pattern to match files starting with "All_ifgs" +# +## Construct the full file path with the refined pattern +#file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +## Use glob to find the file and ensure it exists +#file_list = glob.glob(file_path_pattern) +#if file_list: +# file_path = file_list[0] # Get the single file path directly +# +# # Load the data from the file +# with open(file_path, "rb") as file: +# all_ifgs = pickle.load(file) +#else: +# raise FileNotFoundError( +# f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." +# ) +#print("Reading ifgs completed.") + + +####################### Read coh: +#print("\nReading all coherence data...") +## Define the path components +#sub_dir = "Data" +#filename_pattern = "All_coh_*" # Pattern to match files starting with "All_ifgs" +# +## Construct the full file path with the refined pattern +#file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +## Use glob to find the file and ensure it exists +#file_list = glob.glob(file_path_pattern) +#if file_list: +# file_path = file_list[0] # Get the single file path directly +# +# # Load the data from the file +# with open(file_path, "rb") as file: +# all_coh = pickle.load(file) +#else: +# raise FileNotFoundError( +# f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." +# ) +#print("Reading coherence data completed.") + +##################### Read Loops + +print("\nReading all loop closures...") +sub_dir = "Data" +filename_pattern = "All_loops_*" # Pattern to match files starting with "All_ifgs" + +# Construct the full file path with the refined pattern +file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + all_loops = pickle.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) + +print("Reading loop closures completed.") + +all_cat = [] +for cat in all_loops: + all_cat.append(cat) + +### stacking all the ifgs 6/12/18 in a numpy var all_ifgs +print( + "Extracting the required interferograms(up to 3 epochs), and the corresponding loop closures for inversion..." +) +#desired_categories = [interval, 2 * interval, 3 * interval] +#all_ifgs, _, existing_ifgs_index = read_orig_ifgs_coh( +# all_ifgs, all_coh, desired_categories +#) + +######### forming 12-6 and 18-6 loop closures from the imported data +################################################################# +loop_12 = np.array(all_loops[2 * interval][interval], dtype=object) +#coh_12 = np.array(all_coh[2 * interval][:], dtype=object) + +loop_18 = np.array(all_loops[3 * interval][interval], dtype=object) +#coh_18 = np.array(all_coh[3 * interval][:], dtype=object) + +############## finding dynamic pixel-based thresholds to remove the noisy loops +loop_12_orig = [] +loop_18_orig = [] +none_indices12 = [] +none_indices18 = [] + +# refine loop_12 to exclude the none indices and create loop_12_orig +for i, arr in enumerate(loop_12): + if arr is not None: + (frame_row, frame_col) = np.shape(arr) + loop_12_orig.append(arr) + else: + none_indices12.append(i) + +# refine loop_18 to exclude the none indices and create loop_18_orig +for i, arr in enumerate(loop_18): + if arr is not None: + loop_18_orig.append(arr) + else: + none_indices18.append(i) + +loop_12_orig = np.array(loop_12_orig) +loop_18_orig = np.array(loop_18_orig) + + +### removing noisy loops +print( + "Masking noisy loop closures using circular moving average and standard deviaiton of the loop closures in time:" +) + +# using circular_mean to calculate the mean phase value using complex numbers, and von_mises_variance to calculate the second moment of the Von Mises distribution +_, thresh_loop12 = circular_mean_and_variance_over_epochs(loop_12_orig, axis=0) +_, thresh_loop18 = circular_mean_and_variance_over_epochs(loop_18_orig, axis=0) + + +# Add a new axis at the beginning (axis=0) of these arrays to have shape (1, ...) +# These are the threshold vlaues obtained for each pixel that will be used for masking later +thresh_loop12 = np.expand_dims(thresh_loop12, axis=0) +thresh_loop18 = np.expand_dims(thresh_loop18, axis=0) + + +# calculate a temporal moving average for wrapped data +def moving_average(arr, window_size): + # Initialize an empty array to store the temporal averages + moving_averages = np.zeros_like(arr) + half_window = window_size // 2 + ### miroring data at both ends for window_size/2 at each end + mirrored_shape = (arr.shape[0] + window_size, arr.shape[1], arr.shape[2]) + + # Create an empty array to hold the mirrored data + mirrored_arr = np.empty(mirrored_shape, dtype=arr.dtype) + + # Fill the mirrored array with the original data + mirrored_arr[window_size // 2 : -window_size // 2] = arr + + # Mirror the data at the beginning + for i in range(window_size // 2): +# mirrored_arr[i] = arr[window_size // 2 - i] + mirrored_arr[i] = arr[min(half_window - i, arr.shape[0] - 1)] + + # Mirror the data at the end + for i in range(window_size // 2): +# mirrored_arr[-(i + 1)] = arr[-(i + 1) - window_size // 2] + mirrored_arr[-(i + 1)] = arr[max(-1 - (half_window - i), 0)] + #### + + # Calculate temporal averages within each window + for t in range(half_window, mirrored_arr.shape[0] - half_window): + moving_averages[t - half_window, :, :] = np.angle( + np.nanmean( + np.exp(1j * (mirrored_arr[t - half_window : t + half_window, :, :])), + axis=0, + ) + ) + return moving_averages + + +# for loop 12 +# Calculate distance between each epoch and the moving average for each pixel +print( + f"Calculating the moving average for {2 * interval} loop closures, followed by masking noisy loop closures..." +) +print( + "This may take a while, as it depends on the number of loops and the size of your dataset.\n" +) + +window_size = 30 + +moving_avg = moving_average(loop_12_orig, window_size) +distances = np.abs(np.angle(np.exp(1j * (loop_12_orig - moving_avg)))) +mask = distances > 4 * thresh_loop12 # it is 4*sigma + +# Replace values with NaN where mask is True +loop_12_bk = loop_12_orig +loop_12_orig = np.where(mask, np.nan, loop_12_orig) +loop_12 = loop_12_orig.tolist() + +for idx in none_indices12: + loop_12.insert(idx, None) +loop_12 = np.array(loop_12, dtype=object) + +# for loop 18 +# Calculate distance between each epoch and the moving average for each pixel +print( + f"Calculating the moving average for {3 * interval} loop closures, followed by masking noisy loop closures..." +) +print( + "This may take a while, as it depends on the number of loops and the size of your dataset.\n" +) + +moving_avg = moving_average(loop_18_orig, window_size) +distances = np.abs(np.angle(np.exp(1j * (loop_18_orig - moving_avg)))) +mask = distances > 4 * thresh_loop18 # it is 4*sigma + +# Replace values with NaN where mask is True +loop_18_bk = loop_18_orig +loop_18_orig = np.where(mask, np.nan, loop_18_orig) +loop_18 = loop_18_orig.tolist() + +for idx in none_indices18: + loop_18.insert(idx, None) +loop_18 = np.array(loop_18, dtype=object) + +print("Masking of noisy loop closures is completed.") + +# ################ Apply coherece thresholding +# print('shape loop_12 = ', np.shape(loop_12)) +# print('shape loop_12_orig = ', np.shape(loop_12_orig)) +# print('shape coh_12 = ', np.shape(coh_12)) +# for i, arr in enumerate(loop_12): +# if arr is not None: +# (frame_row, frame_col) = np.shape(arr) +# loop_12[i] = np.where((coh_12[i] > coh_thresh), arr, np.nan) +# +# +# for i, arr in enumerate(loop_18): +# if arr is not None: +# loop_18[i] = np.where((coh_18[i] > coh_thresh), arr, np.nan) +# +# +# + +len12 = len(loop_12) # number of 12 day loops including None +len18 = len(loop_18) # number of 18 day loops including None + +######################## Forming the design matrix A ############################## +#################################################################################### + +print("Preparing the design matrix and observation vector for the inversion step...") + +b = [] +A = [] +n_unk = ( + len12 + 1 +) # this in an initial value(i.e. the num of 6 biases). will be changed by the actual number of unk after considering the None values/missing +# print('n_unk = ', n_unk) + +row = np.zeros( + n_unk, dtype=np.float32 +) # each row of the design matrix A for the 12-day +for i in range(len12): + b.append(loop_12[i]) + # print('np.shape(loop_12[i] = ', np.shape(loop_12[i])) + row[i : i + 2] = a1 - 1 + A.append(row) + row = np.zeros(n_unk, dtype=np.float32) + +row = np.zeros( + n_unk, dtype=np.float32 +) # each row of the design matrix A for the 18-day +for i in range(len18): + b.append(loop_18[i]) + row[i : i + 3] = a2 - 1 + A.append(row) + row = np.zeros(n_unk, dtype=np.float32) + +A = np.array(A) +b = np.array(b, dtype=object) + + +####################### removing the rows from b and A where b is None +mask = [] +mask = np.array([arr is not None for arr in b]) +# Convert the boolean mask to an integer mask +mask = np.nonzero(mask) + +A = A[mask] # Filter rows of A based on the mask +b = b[mask] + +b = b.tolist() + +############ removing the columns of A where are values are zero (these are the unkonws which doesn't fall in any equations nor 12 neither 18 and thus cannot be corrected) + +# Find the column indices where all values are zero +zero_columns = np.all(A == 0, axis=0) + +## Get the indices of the zero columns +zero_column_indices = np.where(zero_columns)[0] + +# Get the indices of the non-zero columns +non_zero_column_indices = np.where(~zero_columns)[ + 0 +] # the indices of the unknowns that can be corrected + +# Save the file +directory_path = os.path.join(output_path, "Data") +file_path = os.path.join(directory_path, f"indices_unknown_tobe_corrected.npy") +np.save(file_path, non_zero_column_indices) + + +### Adding all the existing 6-day ifgs indices to non_zero_column_indices. We want to use estimate their biases in the second step. + + +A_bk = A # to keep a copy of A in case we want to esimate all 6-day biases with smoothing constraints +b_bk = b +all_column_indices = np.array( + range(A_bk.shape[1]) +) # generating all indices in case we want to esimate all 6-day biases with smoothing constraints + +A = A[:, non_zero_column_indices] + +# finding the last column with value -1. this gives the number of 6-day biases that can be corrected +last_column_with_minus_1 = None + +# Iterate through the columns from right to left +for col in range(len(A[0]) - 1, -1, -1): + if -1 in [row[col] for row in A]: + last_column_with_minus_1 = col + break + +num_rows_before_temporals = A.shape[0] + +################################# Least Square inversion ############################### +######################################################################################## +# First Inversion: Without Temporal Smoothing Constraints +print("Starting the first inversion (without temporal smoothing constraints).") +print( + "This step estimates the bias terms that can be corrected based on observed loop closures." +) + +damp_factor = 0 + +if apply_to_all == "no": # this is without using any temporal constraints + + X1 = np.zeros((np.shape(A_bk)[1], frame_row, frame_col), dtype=np.float32) + # X1[:,:,:] = np.nan # after checking noticed this doesn't have any effect on the final vel + b_bk = np.array(b_bk) + + total_pixels = frame_row * frame_col # Total number of pixels for progress bar + processed_pixels = 0 # Counter for processed pixels + print("Progress: [", end="", flush=True) + + for row in range(frame_row): + for col in range(frame_col): + processed_pixels += 1 + # Update progress bar + percentage = (processed_pixels / total_pixels) * 100 + if processed_pixels % (total_pixels // 100) == 0: # Update every 1% + print(f"{int(percentage)}%", end="", flush=True) + sys.stdout.write( + "\rProgress: [" + + "=" * (int(percentage) // 2) + + " " * (50 - int(percentage) // 2) + + "]" + ) + + non_nan_mask = ~np.isnan(b_bk[:, row, col]) + bb = b_bk[non_nan_mask, row, col] # Remove NaN values + AA = A_bk[non_nan_mask, :] # Remove corresponding rows + if len(bb) > min_num_eq: # what is the min number of equation? + + x, istop, itn, normr, normr2 = lsqr( + csc_array(AA), bb, damp=damp_factor + )[ + :5 + ] # Using sparse matrix representation scipy.sparse.linalg + X1[:, row, col] = x + +print("\nFirst inversion (without temporal smoothing constraints) completed.") + +########################### +# Second Inversion: With Temporal Smoothing Constraints +print("Starting the second inversion (with temporal smoothing constraints).") +print( + "This step estimates all bias terms, including those that cannot be corrected, using temporal smoothing constraints." +) + +if with_temporals == "yes": # This is using the temporal constranints on all unknowns + X2 = np.zeros((np.shape(A_bk)[1], frame_row, frame_col), dtype=np.float32) + # X2[:,:,:] = np.nan # after checking noticed this doesn't have any effect on the final vel + b_bk = np.array(b_bk) + + #### Adding temporal constraints + row = np.zeros( + A_bk.shape[1], dtype=np.float32 + ) # additional rows the design matrix A for temporal constraints, setting that to zero again + for i in range(len(all_column_indices) - 2): # for all unknowns + row[i] = -1 * w + row[i + 1] = 2 * w + row[i + 2] = -1 * w + A_bk = np.vstack((A_bk, row)) + row = np.zeros( + A_bk.shape[1], dtype=np.float32 + ) # additional rows the design matrix A for temporal constraints, setting that to zero again + num_temporals = len(A_bk) - len(b_bk) + # print('num_temporals is ', num_temporals) + + ######### Appending zeroes to b according to the added number of temporal constraints obtained by np.shape(A)[0] - np.shape(b)[0] + b_0 = np.zeros( + (np.shape(A_bk)[0] - np.shape(b)[0], frame_row, frame_col), dtype=np.float32 + ) + b_bk = np.vstack((b_bk, b_0)) + + thresh_n_eq = num_temporals # in case of using temporals, we don't want the nan pixels with zero equations with only using temporals + + processed_pixels = 0 # Reset counter for second inversion + print("Progress: [", end="", flush=True) + + for row in range(frame_row): + for col in range(frame_col): + processed_pixels += 1 + # Update progress bar + percentage = (processed_pixels / total_pixels) * 100 + if processed_pixels % (total_pixels // 100) == 0: # Update every 1% + print(f"{int(percentage)}%", end="", flush=True) + sys.stdout.write( + "\rProgress: [" + + "=" * (int(percentage) // 2) + + " " * (50 - int(percentage) // 2) + + "]" + ) + + non_nan_mask = ~np.isnan(b_bk[:, row, col]) + bb = b_bk[non_nan_mask, row, col] # Remove NaN values + AA = A_bk[non_nan_mask, :] # Remove corresponding rows + if len(bb) > thresh_n_eq: + + x, istop, itn, normr, normr2 = lsqr( + csc_array(AA), bb, damp=damp_factor + )[ + :5 + ] # Using sparse matrix representation scipy.sparse.linalg + X2[:, row, col] = x + + print("\nSecond inversion (with temporal smoothing constraints) completed.") + + ####### combining X1 and X2 into X + + X = np.zeros((np.shape(A_bk)[1], frame_row, frame_col), dtype=np.float32) + X[:, :, :] = np.nan + + # Fill X with values from X1 according to non_zero_column_indices + for i, idx in enumerate(non_zero_column_indices): + X[idx, :, :] = X1[i, :, :] + + # Fill in the missing values from X2 + for i, idx in enumerate(all_column_indices): + if idx not in non_zero_column_indices: + X[idx, :, :] = X2[i, :, :] + + # Ensure the dtype is appropriate for your data + X = X.astype(np.float32) + +directory_path = os.path.join(output_path, "Data") +file_path = os.path.join(directory_path, f"X_base_ifgs_biases.npy") +np.save(file_path, X) + +print("Inversion process completed. Results saved to:", file_path) + +########################################################################################################## diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_05_Correction.py b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_05_Correction.py new file mode 100644 index 0000000..a83200d --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_05_Correction.py @@ -0,0 +1,481 @@ +#!/usr/bin/env python3 + +import numpy as np +from datetime import datetime +import pickle +import time +import os +from bin.generate_ifgs_from_epochs import generate_ifg_pairs +import configparser +import glob +from bin.resample import resample_geotiff +import sys + +# sys.path.append(os.path.dirname(os.path.abspath(__file__))) +import bin +from bin.read_orig_ifgs_coh import read_orig_ifgs_coh + +from bin.ifg_cc_totif import ifg_cc_2tif +import gc + +############################################################################### + +HELP_TEXT = """ +PhaseBias_05_Correction.py + +This script applies phase bias correction to desired interferograms using the estimated bias terms for the base interferograms. The corrected interferograms are saved in GeoTIFF format for further analysis. + +### Workflow: +1. **Inputs:** + - **Original Wrapped Interferograms, and Coherence Data:** Obtained from step 1 (`PhaseBias_01_Read_Data.py`). + - **Estimated Bias Terms for Base Intierferograms:** Obtained from step 4 (`PhaseBias_04_Inversion.py`). + - **Calibration Parameters (*a_n*):** Estimated from step 3 (`PhaseBias_03_calibration_pars.py`). + - **max_con:** Defines the maximum number of connections to be corrected. For instance: + - `max_con=5` corrects interferograms up to 30-day intervals for a 6-day acquisition interval. + - Adjust `max_con` as needed. + +2. **Bias Term Estimation for Desired Interferograms:** + - Bias terms for desired interferograms, δ_(i,i+n+1), are estimated using the relationship: + δ_(i,i+n+1) = a_n * (∑_(t=i)^(i+n) δ_(t,t+1) ) + +3. **Correction of Interferograms:** + - Using the estimated bias terms, the interferograms are corrected as follows: + φ_(i,i+n+1)^c = φ_(i,i+n+1) - δ_(i,i+n+1) + + - Here, φ_(i,i+n+1) is the original interferogram, and φ_(i,i+n+1)^c is the corrected interferogram. + +4. **Outputs:** + - Corrected interferograms are saved in GeoTIFF format in the `GEOC` directory under the `output_path` specified in `config.txt`. + +### Output File Format: +- **Corrected Interferograms:** Saved in GeoTIFF format, organized by temporal baseline. + +### Input Requirements: +- Original wrapped interferograms from step 1. +- Bias terms for base interferograms from step 4. +- Configuration parameters (e.g., `output_path`, `interval`) defined in `config.txt`. + +### Output Directory: +- `GEOC` directory under `output_path`: Contains the corrected interferograms in GeoTIFF format, organized by temporal baseline. + +""" +if "--help" in sys.argv: + print(HELP_TEXT) + sys.exit(0) + + +################################################################################ + + +# Config file path +config_file = "config.txt" + +# Check if config file exists +if not os.path.exists(config_file): + raise FileNotFoundError( + f"Error: The configuration file '{config_file}' was not found in the script's directory." + ) + +# Initialize the configparser +config = configparser.ConfigParser() + + +# Read the config.txt file +config.read(config_file) + + +# Define required parameters +required_parameters = [ + "root_path", + "frame", + "start", + "end", + "interval", + "nlook", + "LiCSAR_data", + "a1_6_day", + "a2_6_day", + "a3_6_day", + "a4_6_day", + "a1_12_day", + "a2_12_day", + "a3_12_day", + "a4_12_day", + "estimate_an_values", +] + +# Initialize a dictionary to store parameters +parameters = {} + +# Attempt to retrieve each parameter and catch errors +try: + parameters["root_path"] = config.get("DEFAULT", "root_path") + parameters["output_path"] = config.get("DEFAULT", "output_path") + parameters["frame"] = config.get("DEFAULT", "frame") + parameters["start"] = config.get("DEFAULT", "start") + parameters["end"] = config.get("DEFAULT", "end") + parameters["interval"] = config.getint("DEFAULT", "interval") + parameters["nlook"] = config.getint( + "DEFAULT", "nlook" + ) # Assuming 'nlook' should be an integer + parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") + parameters["a1_6_day"] = config.getfloat("DEFAULT", "a1_6_day") + parameters["a2_6_day"] = config.getfloat("DEFAULT", "a2_6_day") + parameters["a3_6_day"] = config.getfloat("DEFAULT", "a3_6_day") + parameters["a4_6_day"] = config.getfloat("DEFAULT", "a4_6_day") + parameters["a1_12_day"] = config.getfloat("DEFAULT", "a1_12_day") + parameters["a2_12_day"] = config.getfloat("DEFAULT", "a2_12_day") + parameters["a3_12_day"] = config.getfloat("DEFAULT", "a3_12_day") + parameters["estimate_an_values"] = config.get("DEFAULT", "estimate_an_values") + +except configparser.NoOptionError as e: + # Error message if a required parameter is missing + missing_param = str(e).split(": ")[1] + raise ValueError( + f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." + ) +except configparser.Error as e: + # General error for any configparser-related issues + raise ValueError(f"Error while reading '{config_file}': {e}") + +# Assign to individual variables for easy access +root_path = parameters["root_path"] +output_path = parameters["output_path"] +frame = parameters["frame"] +start = parameters["start"] +end = parameters["end"] +interval = parameters["interval"] +nlook = parameters["nlook"] +LiCSAR_data = parameters["LiCSAR_data"] +estimate_an_values = parameters["estimate_an_values"] + + +start_date = datetime.strptime(start, "%Y%m%d") +end_date = datetime.strptime(end, "%Y%m%d") + +uncor_only = 0 # in case of 1 only uses 6-12-18 uncorrected ifgs. it is based on the desired_lengths ifgs +max_con = 5 # in case of 5, it will correct up to 6*5=30-day ifgs. + + +##################################################### +# Root directory where the category directories are located + +if LiCSAR_data == "yes": + track = frame[0:3] # extracting track number from frame id + if track[0] == "0": + track = track[1:3] + if track[0] == "0": + track = track[1:2] + + root_directory = ( + f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}" + ) +else: + root_directory = root_path + + +############################################ + +# Initialize the an values with None for safety +an_values = [None] * ( + max_con - 1 +) # For max_con=5, this will create [None, None, None, None] + +print("estimate_an_values is ", estimate_an_values) + +# Check if estimate_an_values is 'yes' +if estimate_an_values == "yes": + an_file_path = os.path.join(output_path, "Data", "an.txt") + + # Try reading the file + try: + if os.path.exists(an_file_path): + print(f"Reading 'a' values from {an_file_path}...") + + # Read the file and load the an values dynamically + with open(an_file_path, "r") as file: + for line in file: + line = line.strip() + for i in range( + 1, max_con + ): # Loop through a1, a2, ..., a(max_con-1) + param_name = f"a{i}" + if line.startswith(f"{param_name}="): + an_values[i - 1] = float(line.split("=")[1]) + + # Check if all required an values were found + if any(value is None for value in an_values): + print( + "Warning: Missing one or more an values in an.txt. Switching to default values from config file." + ) + raise ValueError("Incomplete an values.") + else: + print( + f"Warning: {an_file_path} does not exist. Switching to default values from config file." + ) + raise FileNotFoundError + + except (FileNotFoundError, ValueError): + # Fallback to default values from config file + print("Loading default 'an' values from config file...") + for i in range(1, max_con): + if interval == 6: + an_values[i - 1] = parameters[f"a{i}_6_day"] + else: + an_values[i - 1] = parameters[f"a{i}_12_day"] +else: + # Use default values from the config file + print("Using default 'an' values from the config file.") + for i in range(1, max_con): + if interval == 6: + an_values[i - 1] = parameters[f"a{i}_6_day"] + else: + an_values[i - 1] = parameters[f"a{i}_12_day"] + +# Print the chosen an values +for i, value in enumerate(an_values, start=1): + print(f"Final value - a{i}: {value}") + + +############## generating ifg pairs between two epochs + +# desired_lengths = [interval, 2*interval, 3*interval, 4*interval, 5*interval] +desired_lengths = [interval * i for i in range(1, max_con + 1)] # replacing the above + +all_ifgs_string = generate_ifg_pairs(start_date, end_date, interval, desired_lengths) + + +##### reading original ifgs/coh ############################# +#################################################################################### + +############################### Reading input data ############################################# +#################### Read ifgs: +print("Reading all ifgs...") +# Define the path components +sub_dir = "Data" +filename_pattern = "All_ifgs_*" # Pattern to match files starting with "All_ifgs" + +# Construct the full file path with the refined pattern +file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + ifgs = pickle.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) + + +###################### Read coh: +print("\nReading all coherence data...") +# Define the path components +sub_dir = "Data" +filename_pattern = "All_coh_*" # Pattern to match files starting with "All_ifgs" + +# Construct the full file path with the refined pattern +file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + coh = pickle.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) +print("\nReading data completed.") + + +######## stacking all the ifgs 6/12/18 in a numpy var all_ifgs +desired_categories = desired_lengths + +all_ifgs, all_coh, existing_ifgs_index = read_orig_ifgs_coh( + ifgs, coh, desired_categories +) + + +del ifgs, coh +gc.collect() + +##### Reading the estimated bias terms as well as the indices of unknowns that can be corrected obtained in step 03 ############## +############################################################################################## +print("Correcting the interferograms...") +# Define the path components +sub_dir = "Data" +filename_pattern_X = "X_base*" # Pattern to match files starting with "X_base" +filename_pattern_indices = "indices*" + +# Construct the full file path with the refined pattern +file_path_pattern_X = os.path.join(output_path, sub_dir, filename_pattern_X) +file_path_pattern_indices = os.path.join(output_path, sub_dir, filename_pattern_indices) + +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern_X) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + X = np.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) +# print('Reading ifgs completed.') + +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern_indices) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + indices_unknown_tobe_corrected = np.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) + + +#### loading indices of unknown that can be corrected and the estimated unknowns + +indices_unknown_tobe_corrected_orig = indices_unknown_tobe_corrected.copy() + +len_6_biases = indices_unknown_tobe_corrected[-1] + 1 # number of 6-day biases + +all_column_indices = np.array( + range(len_6_biases) +) # generating all indices in case we want to esimate all 6-day biases with smoothing constraints. +indices_unknown_tobe_corrected = all_column_indices # in this case we had provided the corrections for all 6-day biases + + +# extending the indices_unknown_tobe_corrected in the original approach to have the indices of 12/18 correctable ifgs: +extended_indices = ( + indices_unknown_tobe_corrected.copy() +) # because in this case indices_unknown_tobe_corrected referes to all_column_indices +X_extended = ( + X.copy() +) # because in this case we have estimated all 6-day biases but we don't need them for the correction. we just need them to correct the existing 12/18 days based on the 6 days + +extended_indices = extended_indices.tolist() +X_extended = X_extended.tolist() + + +# Generalized loop to handle any max_con +for n in range(2, max_con + 1): # Start from 2*interval and go up to (max_con)*interval + current_length = n * interval # Calculate the length (e.g., 12, 18, 24, ...) + + if current_length in desired_lengths: # Check if this length is required + for i in range(len(indices_unknown_tobe_corrected) - (n - 1)): + last_item = extended_indices[-1] # Get the last index + extended_indices.append(last_item + 1) # Append the new index + + # Sum up X values for the current length (n) + X_sum = sum( + X[i + j, :, :] for j in range(n) + ) # Sum over n consecutive indices + + # Append the result to X_extended using the corresponding an_values + X_extended.append( + an_values[n - 2] # an_values is 0-based, so use n-2 + * np.angle(np.exp(complex(0 + 1j) * X_sum)) + ) + + +X = np.array(X_extended) +indices_unknown_tobe_corrected = extended_indices +del X_extended + + +#### correcting the ifgs (6/12/18) ######################## +################################################################# +all_ifgs_cor = np.full_like(all_ifgs, None) +all_ifgs_cor[:, :, :] = np.nan + +if uncor_only == 0: # if we want to use the corrected ifgs (using wrapped phases) + all_ifgs_cor[indices_unknown_tobe_corrected, :, :] = np.angle( + np.exp( + complex(0 + 1j) + * (all_ifgs[indices_unknown_tobe_corrected, :, :] - X[:, :, :]) + ) + ) +else: # no need for correction + all_ifgs_cor[indices_unknown_tobe_corrected, :, :] = all_ifgs[ + indices_unknown_tobe_corrected, :, : + ] + + +### Outputting to disk ################ +############################################################################# + + +# Create the directory (and parent directories) if they don't exist +GEOC_dir = "GEOC" +metadata_dir = "metadata" + +output_wrap_dir = os.path.join(output_path, GEOC_dir) +os.makedirs( + output_wrap_dir, exist_ok=True +) # create the GEOC directory if it doesn't exist + +template_tif_path = os.path.join(root_directory, metadata_dir, frame + ".geo.hgt.tif") + +if nlook != 1: + resampled_tif_path = os.path.join(output_wrap_dir, frame + ".geo.hgt.tif") + resample_geotiff(template_tif_path, resampled_tif_path, nlook) + template_tif_path = resampled_tif_path + + +total_files = len(all_ifgs) # Total number of interferograms +processed_files = 0 # Counter for processed files + +print(f"Writing the corrected interferograms to {output_wrap_dir}") +print("Progress: [", end="", flush=True) +time.sleep(3) + + +for i in range(len(all_ifgs)): + if i in indices_unknown_tobe_corrected: + + # Writing progress bar + processed_files += 1 + percentage = (processed_files / total_files) * 100 + if processed_files % (total_files // 100) == 0: # Update progress every 1% + sys.stdout.write( + f"\rProgress: [" + + "=" * (int(percentage) // 2) + + " " * (50 - int(percentage) // 2) + + f"] {int(percentage)}%" + ) + sys.stdout.flush() + + inpha = all_ifgs_cor[i, :, :] # your phase input, float32 + incoh = all_coh[i, :, :] # your coh input. It can be either 0-255 or 0-1 + + output_phase_path = os.path.join(output_wrap_dir, all_ifgs_string[i]) + output_cc_path = os.path.join(output_wrap_dir, all_ifgs_string[i]) + + # output_phase_path = output_wrap_dir + all_ifgs_string[i] + # output_cc_path = output_wrap_dir + all_ifgs_string[i] + + os.makedirs(output_phase_path, exist_ok=True) + os.makedirs(output_cc_path, exist_ok=True) + + output_phase_file = ( + output_phase_path + "/" + all_ifgs_string[i] + ".geo.diff_pha.tif" + ) + + output_cc_file = output_cc_path + "/" + all_ifgs_string[i] + ".geo.cc.tif" + + ifg_cc_2tif(inpha, output_phase_file, template_tif_path) + ifg_cc_2tif(incoh, output_cc_file, template_tif_path) + +print("\nAll corrected interferograms have been written to disk.") diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/config.txt b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/config.txt new file mode 100644 index 0000000..15f08de --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/config.txt @@ -0,0 +1,43 @@ +[DEFAULT] +# That path to interferograms directory. In case of LiCSAR data set LiCSAR_public='yes', and provide the frame id +root_path = /gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products.public/44/044A_04728_131314 + +output_path = /work/scratch-pw4/earyma/ESA_project/044A_04728_131314 + +LiCSAR_data=no +frame=044A_04728_131314 + +# Provide the start and end dates of the time-series that you want to calculate the phase bias correction +start=20170513 +end=20210504 + +#Provide the data acquisition interval: 6-day or 12-day +interval=6 + + +# If you want to apply further multilooking to interferograms (expand the explanation) +nlook=10 + + +# The calibration parameters 'an' can be estimated from the data by specifying the number of parameters to be determined. For instance, setting num_a=2 indicates that +#a1 and a2 will be estimated. + +num_a=2 + + +### an parameters +# in case of yes the an values will be estimated from the data, otherwise the default values given below will be used +estimate_an_values=no + +# Using 6-days interval +a1_6_day=0.50 +a2_6_day=0.36 +a3_6_day=0.299 +a4_6_day=0.2476 + + +# using 12-day interval +a1_12_day=0.494 +a2_12_day=0.297 +a3_12_day=0.24 +a4_12_day=0.22 diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/config_12day.txt b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/config_12day.txt new file mode 100644 index 0000000..3c92abc --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/config_12day.txt @@ -0,0 +1,45 @@ +[DEFAULT] +# That path to interferograms directory. In case of LiCSAR data set LiCSAR_public='yes', and provide the frame id +root_path = /gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products.public/44/044A_04728_131314 + +output_path = /work/scratch-pw4/earyma/ESA_project/044A_04728_131314 + +LiCSAR_data=no +frame=044A_04728_131314 + +# Provide the start and end dates of the time-series that you want to calculate the phase bias correction +start=20170513 +end=20210504 + +#Provide the data acquisition interval: 6-day or 12-day +# Modified for 12-day and 24-day data +interval=12 + + +# If you want to apply further multilooking to interferograms (expand the explanation) +nlook=10 + +# The calibration parameters 'an' can be estimated from the data by specifying the number of parameters to be determined. For instance, setting num_a=2 indicates that +#a1 and a2 will be estimated. + +num_a=2 + + +### an parameters +# in case of yes the an values will be estimated from the data, otherwise the default values given below will be used +# Modified: estimate from data for 12-day intervals +estimate_an_values=yes + +# Using 6-days interval (original) +a1_6_day=0.50 +a2_6_day=0.36 +a3_6_day=0.299 +a4_6_day=0.2476 + + +# using 12-day interval (for 12-day base interferograms) +# These values will be used if estimate_an_values=no +a1_12_day=0.494 +a2_12_day=0.297 +a3_12_day=0.24 +a4_12_day=0.22 diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/.gitkeep b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/.gitkeep @@ -0,0 +1 @@ + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/loop_closure_time_series.png b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/loop_closure_time_series.png new file mode 100644 index 0000000..d5252b1 Binary files /dev/null and b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/loop_closure_time_series.png differ diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/spatial_distribution_a1.png b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/spatial_distribution_a1.png new file mode 100644 index 0000000..9f6fd52 Binary files /dev/null and b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/spatial_distribution_a1.png differ diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/spatial_distribution_a2.png b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/spatial_distribution_a2.png new file mode 100644 index 0000000..492ff32 Binary files /dev/null and b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/spatial_distribution_a2.png differ diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/timeseries_a1.png b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/timeseries_a1.png new file mode 100644 index 0000000..26cac51 Binary files /dev/null and b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/timeseries_a1.png differ diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/timeseries_a2.png b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/timeseries_a2.png new file mode 100644 index 0000000..7a8cbdf Binary files /dev/null and b/.codex_tmp/pyint_variants/no_rescue/pyint/InSAR_PhaseBias_Correction/images/timeseries_a2.png differ diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/LT1_import_SLC_from_zipfiles b/.codex_tmp/pyint_variants/no_rescue/pyint/LT1_import_SLC_from_zipfiles new file mode 100644 index 0000000..f953e16 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/LT1_import_SLC_from_zipfiles @@ -0,0 +1,121 @@ +#! /bin/csh -f +echo "*** S1_import_SLC_from_zipfiles: Script to read in and concatenate S1 TOPS SLC from zip files ***" +echo "*** Copyright 2023 Gamma Remote Sensing, v2.3 19-Apr-2023 uw/cm ***" +echo "" + +if ($#argv < 1)then + echo "usage: S1_import_SLC_from_zipfiles [burst_number_table_ref] [pol] [dtype] [swath_flag] [OPOD_dir] [cleaning] [noise_mode]" + echo " zipfile_list (input) ASCII file containing S1 zip filename(s) of one data take (one per line, in correct sequence)" + echo " (other values are vv, vh, hh, hv)" + echo " dtype output data type (enter - for default)" + echo " 0: FCOMPLEX (default)" + echo " 1: SCOMPLEX" + echo " " + echo "resulting files: burst SLC files (per polarization, with SLC_tab, SLC, SLC_par, TOPS_par and optionally SLC.noise)" + echo "(concatenated, empty bursts added where necessary) at selected polarizations" + echo " " + exit +endif + + + +# 6-Jan-2023: logfile name updated to be the same on every platform + +set zipfile_list = $1 + # in this case it generates temp.burst_number_table_ref +set dtype = "0" # 0: FCOMPLEX + + +if ($#argv >= 2) then + if ( "$2" != "-" ) then + set dtype = $2 + endif +endif + + + +set zipfile=`date "+%Y%m%d_%H%M%S"` +set logfile = "LT1_import_SLC_from_zipfiles.$zipfile.logfile" +echo "the processing is documented in the logfile: $logfile" + +echo "COMMAND_used: $0 $1 $2 " > $logfile +date >> $logfile +echo "" >> $logfile + +############################################################################## +if (-e "$1" == 0) then + echo "ERROR: zipfile_list ($1) does not exist"; exit(-1) +else + #determine number of rows of $zipfile_list + set zipfile=`awk '(NR>=1){print NF}' $zipfile_list` + set nrows=`echo "$zipfile" | awk '(NR==1){print NF}'` +endif + +set i="1" +while ( "$i" <= "$nrows" ) + # interpret names, check consistency of names + # read first zipfile name and reduce it to filename without path + set zipfile = `awk '(NR=='"$i"'){print $1}' $zipfile_list` + if ( -e $zipfile == "0" ) then + echo "ERROR: zipfile number $1 ($zipfile) does not exist"; exit(-1) + endif + + set zipfile = `awk '(NR=='"$i"'){print $1}' $zipfile_list` + # set l = `echo "$zipfile" | awk '{print length($1)}'` + set m = `echo "$zipfile tar.gz" | awk '{print match($1,$2)-71}'` + set filename_without_path = `echo "$zipfile $m" | awk '{print substr($1,$2,70)}'` + set path_only = `echo "$zipfile $m" | awk '{print substr($1,1,$2-1)}'` + # echo "$path_only $filename_without_path" + echo "$filename_without_path" + echo "$path_only" + + set sensor = `echo "$filename_without_path" | awk '{print substr($1,1,4)}'` + set mode = `echo "$filename_without_path" | awk '{print substr($1,15,6)}'` + set polar = `echo "$filename_without_path" | awk '{print substr($1,54,2)}'` + set level = `echo "$filename_without_path" | awk '{print substr($1,50,3)}'` + set date = `echo "$filename_without_path" | awk '{print substr($1,41,8)}'` + set orbit = `echo "$filename_without_path" | awk '{print substr($1,22,6)}'` + set product = `echo "$filename_without_path" | awk '{print substr($1,61,10)}'` + echo "sensor: $sensor mode: $mode polar: $polar level: $level date: $date orbit: $orbit product: $product" + + if ( "$level" != "SLC" ) then + echo "ERROR: type indicated in zipfile ($type) is not SLC"; exit(-1) + endif + + set GeoTIFF = `tar -tzf $zipfile | grep "tiff" | awk '{print $1}'` + set xml_name = `tar -tzf $zipfile | grep "meta.xml" | awk '{print $1}'` + set num = `grep "$date" $zipfile_list | wc -l ` + if ( ! -d $date ) then + mkdir $date + endif + if ( ! -d tmp_data_dir ) then + mkdir tmp_data_dir + endif + if ( $num>1 ) then + set SLC = ${date}_$product.slc + set SLC_par = ${date}_$product.slc.par + set SLC1 = ${date}_$product.slc.update + set SLC1_par = ${date}_$product.slc.update.par + else + set SLC = $date.slc + set SLC_par = $date.slc.par + set SLC1 = $date.slc.update + set SLC1_par = $date.slc.update.par + endif + tar -xzv -C tmp_data_dir -f $zipfile $GeoTIFF + tar -xzv -C tmp_data_dir -f $zipfile $xml_name + echo "COMMAND: par_LT1_SLC ./tmp_data_dir/$GeoTIFF ./tmp_data_dir/$xml_name ./$date/$SLC_par ./$date/$SLC " + par_LT1_SLC ./tmp_data_dir/$GeoTIFF ./tmp_data_dir/$xml_name ./$date/$SLC_par ./$date/$SLC + echo "COMMAND: par_LT1_SLC ./tmp_data_dir/$GeoTIFF ./tmp_data_dir/$xml_name ./$date/$SLC1 ./$date/$SLC1_par " + par_LT1_SLC_YSLi ./tmp_data_dir/$GeoTIFF ./tmp_data_dir/$xml_name ./$date/$SLC1 ./$date/$SLC1_par 0 + grep "state_vector" ./$date/$SLC1_par | awk -F ' ' '{printf " %s %.5f %.5f %.5f %s %s\n", $1, $2, $3, $4, $5,$6}' | awk '{if (NR%2==1){print $0,"m/s"}else{print $0,"m"}}' | awk 'NR<=3 {print $1,$2}' >tmp1 + grep "state_vector" ./$date/$SLC1_par | awk -F ' ' '{printf " %s %.5f %.5f %.5f %s %s\n", $1, $2, $3, $4, $5,$6}' | awk '{if (NR%2==1){print $0,"m/s"}else{print $0,"m"}}' | awk 'NR>3 {print $0}' >tmp2 + set histchars="" + awk '!/state_vector/ ' ./$date/$SLC_par >tmp + cat tmp tmp1 tmp2 > ./$date/$SLC_par + rm tmp1 tmp tmp2 + set i = `echo "$i" | awk '{printf "%d", $1+1}'` +end +rm -rf *.logfile tmp_data_dir + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/LT1_import_SLC_from_zipfiles1 b/.codex_tmp/pyint_variants/no_rescue/pyint/LT1_import_SLC_from_zipfiles1 new file mode 100644 index 0000000..7141360 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/LT1_import_SLC_from_zipfiles1 @@ -0,0 +1,126 @@ +#! /bin/csh -f +echo "*** S1_import_SLC_from_zipfiles: Script to read in and concatenate S1 TOPS SLC from zip files ***" +echo "*** Copyright 2023 Gamma Remote Sensing, v2.3 19-Apr-2023 uw/cm ***" +echo "" + +if ($#argv < 1)then + echo "usage: S1_import_SLC_from_zipfiles [burst_number_table_ref] [pol] [dtype] [swath_flag] [OPOD_dir] [cleaning] [noise_mode]" + echo " zipfile_list (input) ASCII file containing S1 zip filename(s) of one data take (one per line, in correct sequence)" + echo " (other values are vv, vh, hh, hv)" + echo " dtype output data type (enter - for default)" + echo " 0: FCOMPLEX (default)" + echo " 1: SCOMPLEX" + echo " " + echo "resulting files: burst SLC files (per polarization, with SLC_tab, SLC, SLC_par, TOPS_par and optionally SLC.noise)" + echo "(concatenated, empty bursts added where necessary) at selected polarizations" + echo " " + exit +endif + + + +# 6-Jan-2023: logfile name updated to be the same on every platform + +set zipfile_list = $1 + # in this case it generates temp.burst_number_table_ref +set dtype = "0" # 0: FCOMPLEX + + +if ($#argv >= 2) then + if ( "$2" != "-" ) then + set dtype = $2 + endif +endif + + + +set zipfile=`date "+%Y%m%d_%H%M%S"` +set logfile = "LT1_import_SLC_from_zipfiles.$zipfile.logfile" +echo "the processing is documented in the logfile: $logfile" + +echo "COMMAND_used: $0 $1 $2 " > $logfile +date >> $logfile +echo "" >> $logfile + +############################################################################## +if (-e "$1" == 0) then + echo "ERROR: zipfile_list ($1) does not exist"; exit(-1) +else + #determine number of rows of $zipfile_list + set zipfile=`awk '(NR>=1){print NF}' $zipfile_list` + set nrows=`echo "$zipfile" | awk '(NR==1){print NF}'` +endif + + # interpret names, check consistency of names + # read first zipfile name and reduce it to filename without path + set zipfile = `awk '(NR=='"1"'){print $1}' $zipfile_list` + if ( -e $zipfile == "0" ) then + echo "ERROR: zipfile number $1 ($zipfile) does not exist"; exit(-1) + endif + + set zipfile = `awk '(NR=='"1"'){print $1}' $zipfile_list` + set input_basename = `basename "$zipfile"` + set input_mode = "archive" + if ( `echo "$input_basename" | grep -c ".tar.gz$"` > 0 ) then + set filename_without_path = `echo "$input_basename" | sed 's/\.tar\.gz$//'` + set GeoTIFF = `tar -tzf $zipfile | grep "tiff" | awk '{print $1}'` + set xml_name = `tar -tzf $zipfile | grep "meta.xml" | awk '{print $1}'` + if ( ! -d tmp_data_dir ) then + mkdir tmp_data_dir + endif + tar -xzv -C tmp_data_dir -f $zipfile $GeoTIFF + tar -xzv -C tmp_data_dir -f $zipfile $xml_name + set input_tiff = ./tmp_data_dir/$GeoTIFF + set input_xml = ./tmp_data_dir/$xml_name + else if ( `echo "$input_basename" | grep -c ".tiff$"` > 0 ) then + set input_mode = "scene" + set filename_without_path = `echo "$input_basename" | sed 's/\.tiff$//'` + set input_tiff = $zipfile + set input_xml = `echo "$zipfile" | sed 's/\.tiff$/.meta.xml/'` + if ( -e "$input_xml" == 0 ) then + echo "ERROR: meta xml for scene input does not exist ($input_xml)"; exit(-1) + endif + else + echo "ERROR: unsupported LT-1 input ($zipfile), only .tar.gz or .tiff are supported"; exit(-1) + endif + # echo "$filename_without_path" + + set sensor = `echo "$filename_without_path" | awk '{print substr($1,1,3)}'` + set mode = `echo "$filename_without_path" | awk '{print substr($1,15,5)}'` + set polar = `echo "$filename_without_path" | awk '{print substr($1,54,2)}'` + set level = `echo "$filename_without_path" | awk '{print substr($1,50,3)}'` + set date = `echo "$filename_without_path" | awk '{print substr($1,41,8)}'` + set orbit = `echo "$filename_without_path" | awk '{print substr($1,22,5)}'` + set product = `echo "$filename_without_path" | awk '{print substr($1,61,10)}'` + echo "sensor: $sensor mode: $mode polar: $polar level: $level date: $date orbit: $orbit product: $product" + + if ( "$level" != "SLC" ) then + echo "ERROR: type indicated in zipfile ($type) is not SLC"; exit(-1) + endif + + set num = `grep "$date" t_$date | wc -l ` + + if ( $num>1 ) then + set SLC = ${date}_${product}.slc + set SLC_par = ${date}_${product}.slc.par + set SLC1 = ${date}_${product}.slc.update + set SLC1_par = ${date}_${product}.slc.update.par + else + set SLC = $date.slc + set SLC_par = $date.slc.par + set SLC1 = $date.slc.update + set SLC1_par = $date.slc.update.par + endif + + echo "COMMAND: par_LT1_SLC $input_tiff $input_xml ./$SLC_par ./$SLC " + par_LT1_SLC $input_tiff $input_xml ./$SLC_par ./$SLC + echo "COMMAND: par_LT1_SLC $input_tiff $input_xml ./$SLC1 ./$SLC1_par " + par_LT1_SLC_YSLi $input_tiff $input_xml ./$SLC1 ./$SLC1_par 0 + grep "state_vector" ./$SLC1_par | awk -F ' ' '{printf " %s %.5f %.5f %.5f %s %s\n", $1, $2, $3, $4, $5,$6}' | awk '{if (NR%2==1){print $0,"m/s"}else{print $0,"m"}}' | awk 'NR<=3 {print $1,$2}' >tmp1 + grep "state_vector" ./$SLC1_par | awk -F ' ' '{printf " %s %.5f %.5f %.5f %s %s\n", $1, $2, $3, $4, $5,$6}' | awk '{if (NR%2==1){print $0,"m/s"}else{print $0,"m"}}' | awk 'NR>3 {print $0}' >tmp2 + set histchars="" + awk '!/state_vector/ ' ./$SLC_par >tmp + cat tmp tmp1 tmp2 > ./$SLC_par + rm tmp1 tmp tmp2 +rm -rf *.logfile tmp_data_dir + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/MAI_SLC_Gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/MAI_SLC_Gamma.py new file mode 100644 index 0000000..340d6e6 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/MAI_SLC_Gamma.py @@ -0,0 +1,202 @@ +#! /usr/bin/env python +#''' +################################################################################## +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : February, 2017 # +# # +# Split beam of SLC: backward and forward SLC image generation # +# # +################################################################################## +#''' +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +from pyint import _utils as ut +import argparse + + + +INTRODUCTION = ''' +------------------------------------------------------------------------------------------ + Split beam of original SLC to generate sub-aperture SLC: backward- and forward-SLCs. + +''' + +EXAMPLE = ''' + Usage: + MAI_SLC_gamma.py projectName Mdate Sdate + MAI_SLC_gamma.py PacayaT163TsxHhA 20150102 20150601 +------------------------------------------------------------------------------------------ +''' + + + + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def ras2jpg(input, strTitle): + call_str = "convert " + input + ".ras " + input + ".jpg" + os.system(call_str) + call_str = "convert " + input + ".jpg -resize 250 " + input + ".thumb.jpg" + os.system(call_str) + call_str = "convert " + input + ".jpg -resize 500 " + input + ".bthumb.jpg" + os.system(call_str) + call_str = "$INT_SCR/addtitle2jpg.pl " + input + ".thumb.jpg 14 " + strTitle + os.system(call_str) + call_str = "$INT_SCR/addtitle2jpg.pl " + input + ".bthumb.jpg 24 " + strTitle + os.system(call_str) + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Split beam of original SLC to generate sub-aperture SLC: backward- and forward-SLCs',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('Mdate',help='Master date.') + parser.add_argument('Sdate',help='Slave date.') + + inps = parser.parse_args() + return inps + + + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + Mdate = inps.Mdate + Sdate = inps.Sdate + + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + coregCoarse = templateContents['Coreg_Coarse'] + if 'Squint_MAI' in templateContents: Squint = templateContents['Squint_MAI'] + else: Squint = '0.5' + + + + projectDir = scratchDir + '/' + projectName + demDir = scratchDir + '/' + projectName + '/DEM' + + slcDir = scratchDir + '/' + projectName + '/SLC' + rslcDir = scratchDir + '/' + projectName + '/RSLC' + ifgDir = projectDir + '/MAIifg' + if not os.path.isdir(ifgDir): os.mkdir(ifgDir) + + Pair = Mdate + '-' + Sdate + workDir = ifgDir + '/' + Pair + if not os.path.isdir(workDir): os.mkdir(workDir) + + +# input slcs + ####################################################################### + Mamp = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + Samp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + Mrslc = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc' + MrslcPar = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc.par' + Srslc = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc' + SrslcPar = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc.par' + + HGT = demDir + '/' + masterDate + '_' + rlks + 'rlks.rdc.dem' + + MasterPar = rslcDir + '/' + masterDate + '/' + masterDate + '.rslc.par' + + + + +# split slcs + + MFslcImg = workDir + "/" + Mdate + ".F.slc" + MFslcPar = workDir + "/" + Mdate + ".F.slc.par" + SFslcImg = workDir + "/" + Sdate + ".F.slc" + SFslcPar = workDir + "/" + Sdate + ".F.slc.par" + + MBslcImg = workDir + "/" + Mdate + ".B.slc" + MBslcPar = workDir + "/" + Mdate + ".B.slc.par" + SBslcImg = workDir + "/" + Sdate + ".B.slc" + SBslcPar = workDir + "/" + Sdate + ".B.slc.par" + + print(MFslcImg + " "+ MslcImg) + +# Multi-aperture processing + + call_str = '$GAMMA_BIN/sbi_filt '+ MslcImg + ' ' + MrslcPar + ' '+SrslcPar + ' ' + MFslcImg + ' '+ MFslcPar + ' ' + MBslcImg + ' ' + MBslcPar + ' ' + Squint + os.system(call_str) + print(call_str) + + call_str = '$GAMMA_BIN/sbi_filt '+ SslcImg + ' ' + SrslcPar + ' '+MrslcPar + ' ' + SFslcImg + ' '+ SFslcPar + ' ' + SBslcImg + ' ' + SBslcPar + ' ' + Squint + os.system(call_str) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/MAI_SLC_Gamma1.py b/.codex_tmp/pyint_variants/no_rescue/pyint/MAI_SLC_Gamma1.py new file mode 100644 index 0000000..bd35061 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/MAI_SLC_Gamma1.py @@ -0,0 +1,134 @@ +#! /usr/bin/env python +#''' +################################################################################## +# # +# Author: Wei Chen # +# Email : chenweicug@gmail.com # +# Date : May,25th 2021 # +# # +# Split beam of SLC: backward and forward SLC image generation # +# # +################################################################################## +#''' + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse + +from pyint import _utils as ut + + +def get_s1_date(raw_file): + file0 = os.path.basename(raw_file) + date = file0[17:25] + return date + +def get_satellite(raw_file): + if 'S1A_IW_SLC_' in raw_file: + s0 = 'A' + else: + s0 = 'B' + + return s0 + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLC from Sentinel-1 raw data with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='project name. e.g., ChangningT55S1A') + parser.add_argument('Mdate',help='date to be processed. e.g., 20180101') + parser.add_argument('Sdate',help='date to be processed. e.g., 20180113') + + inps = parser.parse_args() + + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Split beam of original SLC to generate sub-aperture SLC: backward- and forward-SLCs +''' + +EXAMPLE = """Usage: + + MAI_SLC_Gamma.py projectName Mdate Sdate + + MAI_SLC_Gamma.py ChangningT55S1A 20180517 20180529 + +------------------------------------------------------------------- +""" + +def main(argv): + inps = cmdLineParse() + projectName = inps.projectName + Mdate = inps.Mdate + Sdate = inps.Sdate + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + slcDir = projectDir + '/SLC' + down_dir = projectDir + '/DOWNLOAD' + workDir = projectDir + '/MAI' + + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Mdate = templateDict['masterDate'] + coregCoarse = templateDict['Coreg_Coarse'] + + if not os.path.isdir(workDir): + call_str="mkdir " + workDir + os.system(call_str) + if 'Squint_MAI' in templateDict: Squint = templateContents['Squint_MAI'] + else: Squint = '0.5' + + SslcDir = slcDir + "/" + Sdate + MslcDir = slcDir + "/" + Mdate + + MslcImg = MslcDir + "/" + Mdate + ".slc" + MslcPar = MslcDir + "/" + Mdate + ".slc.par" + SslcImg = SslcDir + "/" + Sdate + ".slc" + SslcPar = SslcDir + "/" + Sdate + ".slc.par" + +# split slcs + + MFslcImg = workDir + "/" + Mdate + ".F.slc" + MFslcPar = workDir + "/" + Mdate + ".F.slc.par" + SFslcImg = workDir + "/" + Sdate + ".F.slc" + SFslcPar = workDir + "/" + Sdate + ".F.slc.par" + + MBslcImg = workDir + "/" + Mdate + ".B.slc" + MBslcPar = workDir + "/" + Mdate + ".B.slc.par" + SBslcImg = workDir + "/" + Sdate + ".B.slc" + SBslcPar = workDir + "/" + Sdate + ".B.slc.par" + print(MFslcImg + " "+ MslcImg) + +# Multi-aperture processing + + call_str = 'sbi_filt '+ MslcImg + ' ' + MslcPar + ' '+SslcPar + ' ' + MFslcImg + ' '+ MFslcPar + ' ' + MBslcImg + ' ' + MBslcPar + ' ' + Squint + os.system(call_str) + print(call_str) + + call_str = 'sbi_filt '+ SslcImg + ' ' + SslcPar + ' '+MslcPar + ' ' + SFslcImg + ' '+ SFslcPar + ' ' + SBslcImg + ' ' + SBslcPar + ' ' + Squint + os.system(call_str) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/NameChange.py b/.codex_tmp/pyint_variants/no_rescue/pyint/NameChange.py new file mode 100644 index 0000000..6889a2d --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/NameChange.py @@ -0,0 +1,180 @@ +#! /usr/bin/env python +#''' +################################################################################### +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : Mar, 2017 # +# # +# Select Interferometry-Pairs from time series SAR images # +# # +################################################################################### +#''' +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + + +def is_number(s): + try: + int(s) + return True + except ValueError: + return False + +def add_zero(s): + if len(s)==1: + s="000"+s + elif len(s)==2: + s="00"+s + elif len(s)==3: + s="0"+s + return s + + +def usage(): + print(''' +****************************************************************************************************** + + Select interferometry pairs from time series SAR images + + usage: + + SelectPairs_Gamma.py ProjectName + + e.g. SelectPairs_Gamma.py PacayaT163TsxHhA + + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==2: + if argv[0] in ['-h','--help']: usage(); sys.exit(1) + else: projectName=sys.argv[1] + else: + usage();sys.exit(1) + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateContents=read_template(templateFile) + processDir = scratchDir + '/' + projectName + "/PROCESS" + slcDir = scratchDir + '/' + projectName + "/SLC" + + if not os.path.isdir(processDir): + call_str = 'mkdir ' + processDir + os.system(call_str) + + + if 'JOB' in templateContents: JOB = templateContents['JOB'] + else: JOB = 'IFG' + + INF=JOB + if INF=='IFG': + Suffix=[''] + print("Time series interferograms will be processed!") + elif INF=='MAI': + Suffix=['.F','.B'] + print("Time series multi-aperture interferograms will be processed!") + elif INF=='RSI': + Suffix=['.HF','.LF'] + print("Time series range split-spectrum interferograms will be processed!") + else: + print("The folder name %s cannot be identified !" % igramDir) + usage();sys.exit(1) + +# define files + + SLC_Tab = processDir + "/SLC_Tab" + TS_Berp = processDir + "/TS_Berp" + TS_Itab = processDir + "/TS_Itab" + itab_type = '1' + pltflg = '0' + + if 'Max_Spacial_Baseline' in templateContents: MaxSB=templateContents['Max_Spacial_Baseline'] + else: + print("Max_Spacial_Baseline is not found in template!! ") + print("500m is chosen as the threshold for spatial baseline!") + MaxSB = '500' + + if 'Max_Temporal_Baseline' in templateContents: MaxTB=templateContents['Max_Temporal_Baseline'] + else: + print("Max_Temporal_Baseline is not found in template!! ") + print("500 days is chosen as the threshold for temporal baseline!") + MaxTB = '500' + + +# extract available SAR images slc and slc_par + ListSLC = os.listdir(slcDir) + Datelist = [] + SLCfile = [] + SLCParfile = [] + + print("All of the available SAR acquisition datelist is :") + for kk in range(len(ListSLC)): + if ( is_number(ListSLC[kk]) and len(ListSLC[kk])==6 ): # if SAR date number is 8, 6 should change to 8. + DD=ListSLC[kk] + Year=int(DD[0:2]) + Month = int(DD[2:4]) + Day = int(DD[4:6]) + if ( 0 < Year < 20 and 0 < Month < 13 and 0 < Day < 32 ): + Datelist.append(ListSLC[kk]) + print(ListSLC[kk]) + DateDir = slcDir+'/'+ListSLC[kk] + SLC0 = glob.glob(DateDir+'/*slc')[0] + SLCPar0 = glob.glob(DateDir+'/*slc.par')[0] + + str_slc = slcDir + "/" + ListSLC[kk] +"/" + ListSLC[kk] + ".slc" + str_slc_par = slcDir + "/" + ListSLC[kk] +"/" + ListSLC[kk] + ".slc.par" + + call_str = 'mv ' + SLC0 + ' ' + str_slc + os.system(call_str) + + call_str = 'mv ' + SLCPar0 + ' ' + str_slc_par + os.system(call_str) + + SLCfile.append(str_slc) + SLCParfile.append(str_slc_par) + + + print("Change name of SLC file is done! ") + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/OPENTOPO_USAGE.md b/.codex_tmp/pyint_variants/no_rescue/pyint/OPENTOPO_USAGE.md new file mode 100644 index 0000000..bbf8be5 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/OPENTOPO_USAGE.md @@ -0,0 +1,83 @@ +# OpenTopography DEM 下载使用示例 + +## 快速开始 + +### 1. 获取免费API Key + +访问: https://opentopography.org/myOpenTopo +- 注册账户 (免费) +- 在 'My Account' 页面请求 API key +- API key 立即可用 + +### 2. 下载不同分辨率的 DEM + +#### SRTM 30m (推荐) +```bash +python makedem.py -r 116/117/39/40 --dem-source srtm \ + --opentopo-api-key YOUR_KEY \ + --opentopo-dem-type SRTMGL1 +``` + +#### SRTM 90m +```bash +python makedem.py -r 116/117/39/40 --dem-source srtm \ + --opentopo-api-key YOUR_KEY \ + --opentopo-dem-type SRTMGL3 +``` + +#### NASADEM 30m +```bash +python makedem.py -r 116/117/39/40 --dem-source srtm \ + --opentopo-api-key YOUR_KEY \ + --opentopo-dem-type NASADEM +``` + +#### Copernicus 30m +```bash +python makedem.py -r 116/117/39/40 --dem-source srtm \ + --opentopo-api-key YOUR_KEY \ + --opentopo-dem-type COP30 +``` + +#### Copernicus 90m +```bash +python makedem.py -r 116/117/39/40 --dem-source srtm \ + --opentopo-api-key YOUR_KEY \ + --opentopo-dem-type COP90 +``` + +## 数据对比 + +| DEM类型 | 分辨率 | 覆盖范围 | 推荐用途 | +|---------|--------|----------|----------| +| **SRTMGL1** | **30m** | 60°S-60°N | **SRTM研究首选** | +| SRTMGL3 | 90m | 60°S-60°N | 快速预览 | +| **NASADEM** | **30m** | 60°S-60°N | **高精度研究** | +| **COP30** | **30m** | 全球 | **全球覆盖首选** | +| COP90 | 90m | 全球 | 快速预览 | + +## 推荐选择 + +1. **SRTM研究**: SRTMGL1 (30m) +2. **高精度需求**: NASADEM 或 COP30 (30m) +3. **全球覆盖**: COP30 (30m) +4. **快速预览**: SRTMGL3 或 COP90 (90m) + +## 注意事项 + +- **SRTMGL1/COP30/NASADEM** (30m): 最大支持 450,000 km² +- **SRTMGL3/COP90** (90m): 最大支持 4,050,000 km² +- 所有数据集需要 OpenTopography API key +- 免费配额足够大部分科研用途 + +## 输出文件命名 + +- SRTMGL1_116_117_39_40.tif +- NASADEM_116_117_39_40.tif +- COP30_116_117_39_40.tif + +文件名格式: `{DEM类型}_{西经}_{东经}_{南纬}_{北纬}.tif` + +--- + +**更新日期**: 2026-03-13 diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/POT_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/POT_gamma.py new file mode 100644 index 0000000..4e87a87 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/POT_gamma.py @@ -0,0 +1,394 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Pixel Offset Tracking (POT) for a single pair ### +### Based on GAMMA offset_pwr_tracking / offset_pwr_tracking2### +### Author: ZYD / Cascade AI ### +################################################################# + +import os +import sys +import time +import argparse +import numpy as np + +from pyint import _utils as ut + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Pixel Offset Tracking (POT) for a single interferometric pair + using GAMMA software. + + Two-round estimation approach (Greenland tracking demo): + Round 1: Large search window for initial offset field + Post-processing: Quality check, outlier removal, gap filling + Round 2: Smaller window with conditioned Round 1 as initial + Final: Convert pixel offsets to displacement in meters + Geocode +''' + +EXAMPLE = ''' + Usage: + POT_gamma.py projectName Mdate Sdate + POT_gamma.py shanghaiT171F128S1A 20241105 20241117 +------------------------------------------------------------------- +''' + + +def run_cmd(cmd_str): + """执行 GAMMA 命令并打印""" + print(f' >> {cmd_str}') + return os.system(cmd_str) + + +def sanitize_gamma_float(filepath, valid_max=1e6): + """ + 清理 GAMMA 浮点数据文件中的无效值。 + GAMMA 对无效像素写入特殊标记值（~3.4e38 / NaN / Inf）， + 这些值无法被 single_class_mapping 的范围过滤器正确捕获。 + 将 NaN / Inf / |val| > valid_max 的像素替换为 0.0。 + """ + data = np.fromfile(filepath, dtype=np.float32) + bad_mask = ~np.isfinite(data) | (np.abs(data) > valid_max) + n_bad = int(np.sum(bad_mask)) + if n_bad > 0: + data[bad_mask] = 0.0 + data.tofile(filepath) + print(f' [sanitize] {os.path.basename(filepath)}: ' + f'清理 {n_bad} 个无效像素 (NaN/Inf/|val|>{valid_max})') + + +def postprocess_offsets(offs_cpx, ccp, mli, width, + ccp_thresh, roff_min, roff_max, + azoff_min, azoff_max, + drange_thresh, dazimuth_thresh, + median_win, median_nmin, + prefix): + """ + 偏移量场后处理流程（参考 GAMMA Greenland tracking demo）: + 1) 提取距离向/方位向分量 + 清理 GAMMA no-data 标记值 + 2) 初始掩膜: 互相关阈值 + 偏移量范围限制 + 3) 中值滤波 + 偏差计算 + 4) 偏差阈值精细掩膜 + 5) 空洞填充 + 6) 空间滤波 → 组合为 conditioned 复数偏移量 + + 返回: (condi, real_interp, imag_interp) + - condi: fspf 平滑后的复数偏移量（供 Round 2 初始值） + - real_interp / imag_interp: 填充后的偏移量（供最终米制转换） + """ + real_file = prefix + '.real' + imag_file = prefix + '.imag' + + # --- 1) 提取距离向 (real) 和方位向 (imag) --- + run_cmd(f'cpx_to_real {offs_cpx} {real_file} {width} 0') + run_cmd(f'cpx_to_real {offs_cpx} {imag_file} {width} 1') + + # 清理 GAMMA no-data 标记值（~3.4e38），避免污染后续掩膜和插值 + valid_max = max(abs(float(roff_max)), abs(float(roff_min)), + abs(float(azoff_max)), abs(float(azoff_min))) * 10 + sanitize_gamma_float(real_file, valid_max) + sanitize_gamma_float(imag_file, valid_max) + sanitize_gamma_float(ccp, 1.0) + + # --- 2) 初始掩膜: 互相关 + 偏移量范围 --- + mask1 = prefix + '.mask1.bmp' + real_m1 = prefix + '.real.masked1' + imag_m1 = prefix + '.imag.masked1' + + run_cmd(f'single_class_mapping 3 ' + f'{ccp} {ccp_thresh} 1.0 ' + f'{real_file} {roff_min} {roff_max} ' + f'{imag_file} {azoff_min} {azoff_max} ' + f'{mask1} {width} 1 0 1 1') + run_cmd(f'mask_class {mask1} {real_file} {real_m1} 0 1 1 1 0 0.0') + run_cmd(f'mask_class {mask1} {imag_file} {imag_m1} 0 1 1 1 0 0.0') + + # 初始掩膜后 BMP + run_cmd(f'rasdt_pwr {real_m1} {mli} {width} - - - - ' + f'{roff_min} {roff_max} 0 rmg.cm {real_m1}.bmp - - 24') + run_cmd(f'rasdt_pwr {imag_m1} {mli} {width} - - - - ' + f'{azoff_min} {azoff_max} 0 rmg.cm {imag_m1}.bmp - - 24') + + # --- 3) 中值滤波 + 偏差 --- + real_med = prefix + '.real.median' + imag_med = prefix + '.imag.median' + dreal = prefix + '.dreal' + dimag = prefix + '.dimag' + + run_cmd(f'median_filter {real_m1} {real_med} {width} ' + f'{median_win} {median_win} {median_nmin}') + run_cmd(f'lin_comb 2 {real_m1} {real_med} 0. 1. -1. ' + f'{dreal} {width} 1 0 1 1') + run_cmd(f'median_filter {imag_m1} {imag_med} {width} ' + f'{median_win} {median_win} {median_nmin}') + run_cmd(f'lin_comb 2 {imag_m1} {imag_med} 0. 1. -1. ' + f'{dimag} {width} 1 0 1 1') + + # --- 4) 偏差阈值精细掩膜 --- + mask2 = prefix + '.mask2.bmp' + real_masked = prefix + '.real.masked' + imag_masked = prefix + '.imag.masked' + + run_cmd(f'single_class_mapping 5 ' + f'{dreal} -{drange_thresh} {drange_thresh} ' + f'{dimag} -{dazimuth_thresh} {dazimuth_thresh} ' + f'{ccp} {ccp_thresh} 1.0 ' + f'{real_file} {roff_min} {roff_max} ' + f'{imag_file} {azoff_min} {azoff_max} ' + f'{mask2} {width} 1 0 1 1 5') + run_cmd(f'mask_class {mask2} {real_file} {real_masked} 0 1 1 1 0 0.0') + run_cmd(f'mask_class {mask2} {imag_file} {imag_masked} 0 1 1 1 0 0.0') + + # 精细掩膜后 BMP + run_cmd(f'rasdt_pwr {real_masked} {mli} {width} - - - - ' + f'{roff_min} {roff_max} 0 rmg.cm {real_masked}.bmp - - 24') + run_cmd(f'rasdt_pwr {imag_masked} {mli} {width} - - - - ' + f'{azoff_min} {azoff_max} 0 rmg.cm {imag_masked}.bmp - - 24') + + # --- 5) 空洞填充 --- + real_interp = prefix + '.real.interp' + imag_interp = prefix + '.imag.interp' + run_cmd(f'fill_gaps {real_masked} {width} {real_interp} 0 4 - 1') + run_cmd(f'fill_gaps {imag_masked} {width} {imag_interp} 0 4 - 1') + + # 清理 fill_gaps 插值可能引入的 NaN/Inf/极端值 + sanitize_gamma_float(real_interp, valid_max) + sanitize_gamma_float(imag_interp, valid_max) + + # 填充后 BMP + run_cmd(f'rasdt_pwr {real_interp} {mli} {width} - - - - ' + f'{roff_min} {roff_max} 0 rmg.cm {real_interp}.bmp - - 24') + run_cmd(f'rasdt_pwr {imag_interp} {mli} {width} - - - - ' + f'{azoff_min} {azoff_max} 0 rmg.cm {imag_interp}.bmp - - 24') + + # --- 6) 空间滤波 + 组合 conditioned --- + real_fspf = prefix + '.real.fspf' + imag_fspf = prefix + '.imag.fspf' + run_cmd(f'fspf {real_interp} {real_fspf} {width} 2 2 2') + run_cmd(f'fspf {imag_interp} {imag_fspf} {width} 2 2 2') + + condi = prefix + '.condi' + run_cmd(f'real_to_cpx {real_fspf} {imag_fspf} {condi} {width} 0') + + return condi, real_interp, imag_interp + + +def cmdLineParse(): + parser = argparse.ArgumentParser( + description='Pixel Offset Tracking for a single pair using GAMMA.', + formatter_class=argparse.RawTextHelpFormatter, + epilog=INTRODUCTION + '\n' + EXAMPLE) + + parser.add_argument('projectName', help='projectName for processing.') + parser.add_argument('Mdate', help='Master date.') + parser.add_argument('Sdate', help='Slave date.') + + inps = parser.parse_args() + return inps + + +def main(argv): + + start_time = time.time() + inps = cmdLineParse() + Mdate = inps.Mdate + Sdate = inps.Sdate + projectName = inps.projectName + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + '/' + projectName + '.template' + templateDict = ut.update_template(templateFile) + + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + + # ========== POT 参数 ========== + pot_rstep = templateDict['pot_rstep'] + pot_azstep = templateDict['pot_azstep'] + pot_rwin = templateDict['pot_rwin'] + pot_azwin = templateDict['pot_azwin'] + pot_rwin2 = templateDict['pot_rwin2'] + pot_azwin2 = templateDict['pot_azwin2'] + pot_ovr = templateDict['pot_ovr'] + pot_snr_thresh = templateDict['pot_snr_thresh'] + pot_ccp_thresh = templateDict['pot_ccp_thresh'] + pot_roff_min = templateDict['pot_roff_min'] + pot_roff_max = templateDict['pot_roff_max'] + pot_azoff_min = templateDict['pot_azoff_min'] + pot_azoff_max = templateDict['pot_azoff_max'] + pot_drange_thresh = templateDict['pot_drange_thresh'] + pot_dazimuth_thresh = templateDict['pot_dazimuth_thresh'] + pot_median_win = templateDict['pot_median_win'] + pot_median_nmin = templateDict['pot_median_nmin'] + pot_two_rounds = templateDict['pot_two_rounds'] + pot_geocode = templateDict['pot_geocode'] + pot_disp_max = templateDict['pot_disp_max'] + + # ========== 目录 ========== + projectDir = scratchDir + '/' + projectName + rslcDir = projectDir + '/RSLC' + demDir = projectDir + '/DEM' + potDir = projectDir + '/offsets' + if not os.path.isdir(potDir): + os.mkdir(potDir) + + Pair = Mdate + '-' + Sdate + workDir = potDir + '/' + Pair + if not os.path.isdir(workDir): + os.mkdir(workDir) + + # ========== 输入文件 ========== + Mrslc = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc' + MrslcPar = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc.par' + Srslc = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc' + SrslcPar = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc.par' + + slc_width = ut.read_gamma_par(MrslcPar, 'read', 'range_samples') + off_width = str(int(int(slc_width) // int(pot_rstep))) + + print('=' * 60) + print(f'Pixel Offset Tracking (POT): {Pair}') + print(f' SLC width : {slc_width}') + print(f' Offset width : {off_width}') + print(f' Step (r x az) : {pot_rstep} x {pot_azstep}') + print(f' R1 window : {pot_rwin} x {pot_azwin}') + if pot_two_rounds == '1': + print(f' R2 window : {pot_rwin2} x {pot_azwin2}') + print(f' Offset range : [{pot_roff_min}, {pot_roff_max}] r ' + f'[{pot_azoff_min}, {pot_azoff_max}] az') + print('=' * 60) + + ####################################################################### + # Step 1: 生成偏移量几何下的 MLI（背景图 + 尺寸参考） + ####################################################################### + print('\n[Step 1] 生成偏移量几何 MLI ...') + MLI_pot = workDir + '/' + Mdate + '.mli_pot' + MLI_pot_par = workDir + '/' + Mdate + '.mli_pot.par' + + run_cmd(f'multi_look {Mrslc} {MrslcPar} {MLI_pot} {MLI_pot_par} ' + f'{pot_rstep} {pot_azstep}') + run_cmd(f'raspwr {MLI_pot} {off_width} - - - - 1. .2 - {MLI_pot}.bmp') + + ####################################################################### + # Step 2: 创建偏移量参数文件 + ####################################################################### + print('\n[Step 2] 创建偏移量参数文件 ...') + OFF = workDir + '/' + Pair + '.off' + run_cmd(f'create_offset {MrslcPar} {SrslcPar} {OFF} 1 {rlks} {azlks} 0') + + ####################################################################### + # Step 3: Round 1 — 初始偏移量估计（大窗口） + ####################################################################### + print(f'\n[Step 3] Round 1 偏移量估计 ({pot_rwin}x{pot_azwin}) ...') + r1_tag = f'{pot_rwin}x{pot_azwin}' + r1_prefix = workDir + '/' + Pair + '.offs' + r1_tag + offs_r1 = r1_prefix + ccp_r1 = workDir + '/' + Pair + '.ccp' + r1_tag + + run_cmd(f'offset_pwr_tracking {Mrslc} {Srslc} {MrslcPar} {SrslcPar} ' + f'{OFF} {offs_r1} {ccp_r1} ' + f'{pot_rwin} {pot_azwin} - {pot_ovr} {pot_snr_thresh} ' + f'{pot_rstep} {pot_azstep}') + + ####################################################################### + # Step 4: Round 1 后处理 + ####################################################################### + print(f'\n[Step 4] Round 1 后处理 ...') + r1_condi, _, _ = postprocess_offsets( + offs_cpx=offs_r1, ccp=ccp_r1, mli=MLI_pot, width=off_width, + ccp_thresh=pot_ccp_thresh, + roff_min=pot_roff_min, roff_max=pot_roff_max, + azoff_min=pot_azoff_min, azoff_max=pot_azoff_max, + drange_thresh=pot_drange_thresh, dazimuth_thresh=pot_dazimuth_thresh, + median_win=pot_median_win, median_nmin=pot_median_nmin, + prefix=r1_prefix) + + # 默认使用 Round 1 结果 + final_prefix = r1_prefix + final_ccp = ccp_r1 + final_off = OFF + + ####################################################################### + # Step 5-6: Round 2 — 精细偏移量估计（小窗口，可选） + ####################################################################### + if pot_two_rounds == '1': + print(f'\n[Step 5] Round 2 偏移量估计 ({pot_rwin2}x{pot_azwin2}) ...') + OFF2 = workDir + '/' + Pair + '.off2' + run_cmd(f'create_offset {MrslcPar} {SrslcPar} {OFF2} 1 {rlks} {azlks} 0') + + r2_tag = f'{pot_rwin2}x{pot_azwin2}' + r2_prefix = workDir + '/' + Pair + '.offs' + r2_tag + offs_r2 = r2_prefix + ccp_r2 = workDir + '/' + Pair + '.ccp' + r2_tag + + run_cmd(f'offset_pwr_tracking2 {Mrslc} {Srslc} {MrslcPar} {SrslcPar} ' + f'{OFF2} {offs_r2} {ccp_r2} {OFF} {r1_condi} ' + f'{pot_rwin2} {pot_azwin2} - {pot_ovr} {pot_snr_thresh} ' + f'{pot_rstep} {pot_azstep}') + + print(f'\n[Step 6] Round 2 后处理 ...') + _, _, _ = postprocess_offsets( + offs_cpx=offs_r2, ccp=ccp_r2, mli=MLI_pot, width=off_width, + ccp_thresh=pot_ccp_thresh, + roff_min=pot_roff_min, roff_max=pot_roff_max, + azoff_min=pot_azoff_min, azoff_max=pot_azoff_max, + drange_thresh=pot_drange_thresh, dazimuth_thresh=pot_dazimuth_thresh, + median_win=pot_median_win, median_nmin=pot_median_nmin, + prefix=r2_prefix) + + final_prefix = r2_prefix + final_ccp = ccp_r2 + final_off = OFF2 + + ####################################################################### + # Step 7: 像素偏移量 → 米制位移量 + ####################################################################### + print('\n[Step 7] 像素偏移量转换为地面位移 (米) ...') + + # 用填充后（非 fspf 平滑）的偏移量组合复数，供 offset_tracking 使用 + final_real_interp = final_prefix + '.real.interp' + final_imag_interp = final_prefix + '.imag.interp' + final_offs_combined = final_prefix + '.offs_combined' + run_cmd(f'real_to_cpx {final_real_interp} {final_imag_interp} ' + f'{final_offs_combined} {off_width} 0') + + disp_map = workDir + '/' + Pair + '.disp_map' + run_cmd(f'offset_tracking {final_offs_combined} {final_ccp} ' + f'{MrslcPar} {final_off} {disp_map} - 2 {pot_ccp_thresh} 0') + + # 提取位移分量 + disp_real = disp_map + '.real' # 地距向位移 (米) + disp_imag = disp_map + '.imag' # 方位向位移 (米) + disp_mag = disp_map + '.mag' # 位移幅值 (米) + + run_cmd(f'cpx_to_real {disp_map} {disp_real} {off_width} 0') + run_cmd(f'cpx_to_real {disp_map} {disp_imag} {off_width} 1') + run_cmd(f'cpx_to_real {disp_map} {disp_mag} {off_width} 3') + + # 清理 offset_tracking 输出中的 NaN/Inf/极端值 + disp_max_m = float(pot_disp_max) * 10 + sanitize_gamma_float(disp_real, disp_max_m) + sanitize_gamma_float(disp_imag, disp_max_m) + sanitize_gamma_float(disp_mag, disp_max_m) + + # BMP 可视化 + run_cmd(f'rasdt_pwr {disp_real} {MLI_pot} {off_width} - - - - ' + f'-{pot_disp_max} {pot_disp_max} 1 rmg.cm {disp_real}.bmp - - 24') + run_cmd(f'rasdt_pwr {disp_imag} {MLI_pot} {off_width} - - - - ' + f'-{pot_disp_max} {pot_disp_max} 1 rmg.cm {disp_imag}.bmp - - 24') + run_cmd(f'rasdt_pwr {disp_mag} {MLI_pot} {off_width} - - - - ' + f'-{pot_disp_max} {pot_disp_max} 1 rmg.cm {disp_mag}.bmp - - 24') + + # 注意: 地理编码已移至 geocode_gamma.py --type pot + # 用法: geocode_gamma.py projectName Pair --type pot + + print(f"\nPixel Offset Tracking for {Pair} is done!") + ut.print_process_time(start_time, time.time()) + sys.exit(0) + + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/POT_gamma_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/POT_gamma_all.py new file mode 100644 index 0000000..07c365c --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/POT_gamma_all.py @@ -0,0 +1,148 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Batch Pixel Offset Tracking (POT) using GAMMA ### +### Author: ZYD / Cascade AI ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + + +def work(data0): + cmd = data0[0] + err_file = data0[1] + p = subprocess.run(cmd, shell=False, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa = stderr.decode("utf-8") + else: + aa = stderr + + if aa: + str0 = ' '.join(cmd) + '\n' + with open(err_file, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return + +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + Batch Pixel Offset Tracking for one project using GAMMA. + + Runs POT_gamma.py for each interferometric pair in parallel. + Skips pairs that already have completed results (disp_map.mag.bmp). +''' + +EXAMPLE = ''' + Usage: + POT_gamma_all.py projectName + POT_gamma_all.py projectName --parallel 4 + POT_gamma_all.py projectName --parallel 4 --ifgarmList-txt /test/ifgram_list.txt +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser( + description='Batch Pixel Offset Tracking for one project using GAMMA.', + formatter_class=argparse.RawTextHelpFormatter, + epilog=INTRODUCTION + '\n' + EXAMPLE) + + parser.add_argument('projectName', help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, + help='Enable parallel processing and specify the number of processors.') + parser.add_argument('--ifgarmList-txt', dest='ifgarmListTxt', + help='Provided ifgram_list_txt. Default: using ifgram_list.txt under projectName folder.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + projectDir = scratchDir + '/' + projectName + potDir = projectDir + '/offsets' + if not os.path.isdir(potDir): + os.mkdir(potDir) + + templateDict = ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + if inps.ifgarmListTxt: + ifgramList_txt = inps.ifgarmListTxt + else: + ifgramList_txt = projectDir + '/ifgram_list.txt' + + ifgList0 = ut.read_txt2array(ifgramList_txt) + + if len(ifgList0) == 3: + ifgList = ifgList0[0] + ifgList = [ifgList] + else: + ifgList = ifgList0[:, 0] + + err_txt = projectDir + '/POT_gamma_all.err' + if os.path.isfile(err_txt): + os.remove(err_txt) + + data_para = [] + skip_count = 0 + for i in range(len(ifgList)): + m0 = ut.yyyymmdd(ifgList[i].split('-')[0]) + s0 = ut.yyyymmdd(ifgList[i].split('-')[1]) + cmd0 = ['POT_gamma.py', projectName, m0, s0] + + # 检查完成标志: disp_map.mag.bmp + Pair = ifgList[i] + disp_bmp = potDir + '/' + Pair + '/' + Pair + '.disp_map.mag.bmp' + + k00 = 0 + if os.path.isfile(disp_bmp): + if os.path.getsize(disp_bmp) > 0: + k00 = 1 + skip_count += 1 + + if k00 == 0: + data0 = [cmd0, err_txt] + data_para.append(data0) + + total = len(ifgList) + todo = len(data_para) + print(f'Pixel Offset Tracking: {total} pairs total, ' + f'{skip_count} already done, {todo} to process') + print(f'Parallel processors: {inps.parallelNumb}') + print(f'Output directory: {potDir}') + print('=' * 60) + + if todo > 0: + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + + print("Pixel Offset Tracking for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(0) + + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/README.md b/.codex_tmp/pyint_variants/no_rescue/pyint/README.md new file mode 100644 index 0000000..1995412 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/README.md @@ -0,0 +1,26 @@ +## GACOS CORRECTION FOR GMTSAR +These software is intended to correct unwrapped phases using GACOS grids for GMTSAR. + +They have been tested with: GMT v6.1.0 GMTSAR v6.0 + +On "Test_against_matlab" folder, I perform comparisons with matlab with consistent results. +To reproduce the output just use: source command.txt in each folder. +There is a small README file in each folder + +### SINGLE INTERFEROGRAM +single_GACOS_correction.csh is intented to correct a single interferogram at a time with certain parameters + +### STACK OF INTERFEROGRAMS +GACOS_correction.csh and operation.csh are used to correct a stack of interferograms in a time series process + +Corrections are applied to the unwrap.grd files + +Feel free to use and edit the code if needed + +References: + +*Yu, C., Li, Z., Penna, N. T., & Crippa, P. (2018). Generic atmospheric correction model for Interferometric Synthetic Aperture Radar observations. Journal of Geophysical Research: Solid Earth, 123(10), 9202-9222.* + +*Yu, C., Li, Z., & Penna, N. T. (2018). Interferometric synthetic aperture radar atmospheric correction using a GPS-based iterative tropospheric decomposition model. Remote Sensing of Environment, 204, 109-121.* + +*Yu, C., Penna, N. T., & Li, Z. (2017). Generation of real‐time mode high‐resolution water vapor fields from GPS observations. Journal of Geophysical Research: Atmospheres, 122(3), 2008-2025.* diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/RSI_SLC_Gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/RSI_SLC_Gamma.py new file mode 100644 index 0000000..8dfb52f --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/RSI_SLC_Gamma.py @@ -0,0 +1,216 @@ +#! /usr/bin/env python +#''' +################################################################################### +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : March , 2017 # +# # +# Range Split Spectrum for SAR complex data based on GAMMA # +# # +################################################################################### +#''' +import numpy as np +import os +import pysar._readfile as readfile +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + + +def ras2jpg(input, strTitle): + call_str = "convert " + input + ".ras " + input + ".jpg" + os.system(call_str) + call_str = "convert " + input + ".jpg -resize 250 " + input + ".thumb.jpg" + os.system(call_str) + call_str = "convert " + input + ".jpg -resize 500 " + input + ".bthumb.jpg" + os.system(call_str) + call_str = "$INT_SCR/addtitle2jpg.pl " + input + ".thumb.jpg 14 " + strTitle + os.system(call_str) + call_str = "$INT_SCR/addtitle2jpg.pl " + input + ".bthumb.jpg 24 " + strTitle + os.system(call_str) + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +def geocode(inFile, outFile, UTMTORDC, nWidth, nWidthUTMDEM, nLineUTMDEM): + if inFile.rsplit('.')[1] == 'int': + call_str = '$GAMMA_BIN/geocode_back ' + inFile + ' ' + nWidth + ' ' + UTMTORDC + ' ' + outFile + ' ' + nWidthUTMDEM + ' ' + nLineUTMDEM + ' 0 1' + else: + call_str = '$GAMMA_BIN/geocode_back ' + inFile + ' ' + nWidth + ' ' + UTMTORDC + ' ' + outFile + ' ' + nWidthUTMDEM + ' ' + nLineUTMDEM + ' 0 0' + os.system(call_str) + + +def createBlankFile(strFile): + f = open(strFile,'w') + for i in range (10): + f.write('\n') + f.close() + + +def usage(): + print(''' +****************************************************************************************************** + + Split Spectrum on range direction to generate high-frequency- and low-frequency- SLCs + + usage: + + RSI_SLC_Gamma.py igramDir + + e.g. RSI_SLC_Gamma.py RSI_PacayaT163TsxHhA_131021-131101_0011_-0007 + + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==2: + if argv[0] in ['-h','--help']: usage(); sys.exit(1) + else: igramDir=sys.argv[1] + else: + usage();sys.exit(1) + + INF = igramDir.split('_')[0] + projectName = igramDir.split('_')[1] + IFGPair = igramDir.split(projectName+'_')[1].split('_')[0] + Mdate = IFGPair.split('-')[0] + Sdate = IFGPair.split('-')[1] + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + + processDir = scratchDir + '/' + projectName + "/PROCESS" + slcDir = scratchDir + '/' + projectName + "/SLC" + workdir = processDir + '/' + igramDir + + templateContents=read_template(templateFile) + rlks = templateContents['Range_Looks'] + azlks = templateContents['Azimuth_Looks'] + + if INF!='RSI': + usage();sys.exit(1) + + if not os.path.isdir(workdir): + call_str = 'mkdir ' + workdir + os.system(call_str) + + SslcDir = slcDir + "/" + Sdate + MslcDir = slcDir + "/" + Mdate + +# input slcs + + SslcDir = slcDir + "/" + Sdate + MslcDir = slcDir + "/" + Mdate + + MslcImg = MslcDir + "/" + Mdate + ".slc" + MslcPar = MslcDir + "/" + Mdate + ".slc.par" + SslcImg = SslcDir + "/" + Sdate + ".slc" + SslcPar = SslcDir + "/" + Sdate + ".slc.par" + + + MHslcImg = workdir + '/'+Mdate + '.HF.slc' + MHslcPar = workdir + '/'+Mdate + '.HF.slc.par' + call_str = 'cp ' + MslcPar + ' '+ MHslcPar + os.system(call_str) + + SHslcImg = workdir + '/'+Sdate + '.HF.slc' + SHslcPar = workdir + '/'+Sdate + '.HF.slc.par' + call_str = 'cp ' + SslcPar + ' '+ SHslcPar + os.system(call_str) + + MLslcImg = workdir + '/'+Mdate + '.LF.slc' + MLslcPar = workdir + '/'+Mdate + '.LF.slc.par' + call_str = 'cp ' + MslcPar + ' '+ MLslcPar + os.system(call_str) + + SLslcImg = workdir + '/'+Sdate + '.LF.slc' + SLslcPar = workdir + '/'+Sdate + '.LF.slc.par' + call_str = 'cp ' + SslcPar + ' '+ SLslcPar + os.system(call_str) + + nWidth = UseGamma(MslcPar, 'read','range_samples:') + + call_str= 'bpf ' + MslcImg + ' ' + MHslcImg + ' ' + nWidth + ' 0.25 0.5 0 1 0 0 - - 1' + os.system(call_str) + + call_str= 'bpf ' + MslcImg + ' ' + MLslcImg + ' ' + nWidth + ' -0.25 0.5 0 1 0 0 - - 1' + os.system(call_str) + + nWidth = UseGamma(SslcPar, 'read','range_samples:') + + call_str= 'bpf ' + SslcImg + ' ' + SHslcImg + ' ' + nWidth + ' 0.25 0.5 0 1 0 0 - - 1' + os.system(call_str) + + call_str= 'bpf ' + SslcImg + ' ' + SLslcImg + ' ' + nWidth + ' -0.25 0.5 0 1 0 0 - - 1' + os.system(call_str) + + + print("Split spectrum for both slave and master date is done!") + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + + + + + + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/Raw2SLC_ERS_Cat.py b/.codex_tmp/pyint_variants/no_rescue/pyint/Raw2SLC_ERS_Cat.py new file mode 100644 index 0000000..3bd572c --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/Raw2SLC_ERS_Cat.py @@ -0,0 +1,306 @@ +#! /usr/bin/env python +#''' +################################################################################## +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : June, 2019 # +# # +# Generate SLC from SAR_IMS_P1 data # +# # +################################################################################## +#''' +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close + +def rm(TXT): + call_str = 'rm ' + TXT + os.system(call_str) + +def usage(): + print(''' +****************************************************************************************************** + + Generate SLC and SLC_par file for ERS/ENVISAT (SAR_IMS_1P format) + + usage: + + Down2SLC_ERS.py ProjectName DownName + + e.g. Down2SLC_ERS.py CotopaxiT120ERSA 910101 + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==3: + projectName = sys.argv[1] + Date = sys.argv[2] + else: + usage();sys.exit(1) + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + '/' + projectName + '.template' + templateContents=read_template(templateFile) + + if 'rlks4cor' in templateContents: rlks4cor = templateContents['rlks4cor'] + else: rlks4cor = '4' + if 'azlks4cor' in templateContents: azlks4cor = templateContents['azlks4cor'] + else: azlks4cor = '4' + + if 'rwin4cor' in templateContents: rwin4cor = templateContents['rwin4cor'] + else: rwin4cor = '128' + if 'azwin4cor' in templateContents: azwin4cor = templateContents['azwin4cor'] + else: azwin4cor = '128' + if 'rsample4cor' in templateContents: rsample4cor = templateContents['rsample4cor'] + else: rsample4cor = '64' + if 'azsample4cor' in templateContents: azsample4cor = templateContents['azsample4cor'] + else: azsample4cor = '64' + + if ' rpos4cor' in templateContents: rpos4cor = templateContents[' rpos4cor'] + else: rpos4cor = ' - ' + if 'azpos4cor' in templateContents: azpos4cor = templateContents['azpos4cor'] + else: azpos4cor = ' - ' + + + if 'rfwin4cor' in templateContents: rfwin4cor = templateContents['rfwin4cor'] + else: rfwin4cor = str(int(int(rwin4cor)/2)) + if 'azfwin4cor' in templateContents: azfwin4cor = templateContents['azfwin4cor'] + else: azfwin4cor = str(int(int(azwin4cor)/2)) + if 'rfsample4cor' in templateContents: rfsample4cor = templateContents['rfsample4cor'] + else: rfsample4cor = str(2*int(rsample4cor)) + if 'azfsample4cor' in templateContents: azfsample4cor = templateContents['azfsample4cor'] + else: azfsample4cor = str(2*int(azsample4cor)) + + if 'thresh4cor' in templateContents: thresh4cor = templateContents['thresh4cor'] + else: thresh4cor = ' 0.2 ' + + rlks = templateContents['Range_Looks'] + azlks = templateContents['Azimuth_Looks'] + + + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + slcDir = scratchDir + '/' + projectName + '/SLC' + + if not os.path.isdir(slcDir): + call_str= 'mkdir ' +slcDir + os.system(call_str) + os.chdir(downDir) + + t0 = 't0_' + Date + call_str = 'ls >' + t0 + os.system(call_str) + + tt = 'tt_' + Date + call_str = "grep " + Date + ' ' + t0 + '> ' + tt + os.system(call_str) + + te = 'te_' + Date + call_str = "grep SAR_IM " + tt + " > " + te + os.system(call_str) + + AA= np.loadtxt(te,dtype=np.str) + Na = AA.size + AA=AA.reshape(Na,) + + for i in range(Na): + Date0 = Date+'_' + str(i) + downName = str(AA[i]) + FileDir = downDir + '/' + downName + call_str = 'raw2slc_ers_envisat.py '+ FileDir + ' -o ' + Date0 + os.system(call_str) + + #call_str ="rename 's/VV.SLC/slc/g' *" + #os.system(call_str) + + #slcpar = Date0 + '.slc.par' + #call_str = 'ERS_orb_cor_par.py ' + slcpar + #os.system(call_str) + + Date0 = Date + if len(Date)==6: + Date6 = Date + Date0 = Date + elif len(Date)==8: + Date0 = Date[2:8] + Date6 = Date[2:8] + else: + print('The input Date is invalid.') + sys.exit(1) + + dataDir = slcDir + '/' + Date0 + if not os.path.isdir(dataDir): + call_str = 'mkdir ' + dataDir + print('Generate SLC dir for date: ' + Date0) + os.system(call_str) + call_str = 'mv ' + Date + '*.slc* ' + dataDir + os.system(call_str) + + os.chdir(dataDir) + for i in range(Na): + Date0 = Date+'_' + str(i) + downName = str(AA[i]) + + SLCm = Date0 + '.slc' + SLCm_par = Date0 + '.slc.par' + MamprlksImg = Date0 + '.amp' + MamprlksPar = Date0 + '.amp.par' + + call_str = 'multi_look ' + SLCm + ' ' + SLCm_par + ' ' + MamprlksImg + ' ' + MamprlksPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + nWidth = UseGamma(MamprlksPar, 'read', 'range_samples') + call_str = 'raspwr ' + MamprlksImg + ' ' + nWidth + os.system(call_str) + + SLC = Date6+'.slc' + SLCPAR = Date6 + '.slc.par' + AMP = Date6+'.amp' + AMPPAR = Date6+'.amp.par' + if Na==1: + os.rename(SLCm,SLC) + os.rename(SLCm_par,SLCPAR) + os.rename(MamprlksImg,AMP) + os.rename(MamprlksPar,AMPPAR) + + for i in range(Na-1): + if i==0: + DateA = Date + '_0' + else: + DateA = Date + '_' + str(i-1) + str(i) + + SLCA = DateA + '.slc' + SLCA_par = DateA + '.slc.par' + + + DateB = Date + '_' +str(i+1) + SLCB = DateB + '.slc' + SLCB_par = DateB + '.slc.par' + + DateC = Date + '_' + str(i) + str(i+1) + SLCC = DateC + '.slc' + SLCC_par = DateC + '.slc.par' + + ##################################################################################### + MamprlksImg = Date + '.amp' + MamprlksPar = Date + '.amp.par' + + off = DateC + '.off' + offs = DateC + '.offs' + offsets = DateC + '.offsets' + coffs = DateC + '.coffs' + coffsets = DateC + '.coffsets' + snr = DateC + '.snr' + off_std = DateC + '.off_std' + + ########################## Generate off file ############################# + + call_str = "create_offset " + SLCA_par + " " + SLCB_par + " " + off + " 1 - - 0" + os.system(call_str) + call_str = 'init_offset_orbit '+ SLCA_par + " " + SLCB_par + ' ' + off + os.system(call_str) + + + call_str = 'init_offset '+ SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' ' + rlks4cor + ' ' + azlks4cor + ' ' + rpos4cor + ' ' + azpos4cor + os.system(call_str) + + call_str = 'init_offset '+ SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' 1 1 - - ' + os.system(call_str) + + call_str = "offset_pwr " + SLCA + " " + SLCB + " " + SLCA_par + " " + SLCB_par + " " + off + " " + offs + " " + snr + " " + rwin4cor + " " + azwin4cor + " " + offsets + " 2 "+ rsample4cor + " " + azsample4cor + os.system(call_str) + + call_str = "offset_fit " + offs + " " + snr + " " + off + " " + coffs + " " + coffsets + " " + thresh4cor +" 3" + os.system(call_str) + + call_str = "offset_pwr " +SLCA + " " + SLCB + " " + SLCA_par + " " + SLCB_par + " " + off + " " + offs + " " + snr + " " + rfwin4cor + " " + azfwin4cor + " " + offsets + " 2 " + rfsample4cor + " " + azfsample4cor + os.system(call_str) + + call_str = "offset_fit " + offs + " " + snr + " " + off + " " + coffs + " " + coffsets + " " + thresh4cor + " 4 >" + off_std + os.system(call_str) + + ######################################################################################## + + call_str = 'SLC_cat ' + SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' ' + SLCC + ' ' + SLCC_par + os.system(call_str) + + if i==(Na-2): + if len(Date)==6: + DD = Date + else: + DD = Date[2:8] + SLCm = DD + '.slc' + SLCm_par = DD + '.slc.par' + + + call_str = 'cp ' + SLCC + ' ' + SLCm + os.system(call_str) + + call_str = 'cp ' + SLCC_par + ' ' + SLCm_par + os.system(call_str) + + call_str = 'multi_look ' + SLCm + ' ' + SLCm_par + ' ' + MamprlksImg + ' ' + MamprlksPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + nWidth = UseGamma(MamprlksPar, 'read', 'range_samples') + call_str = 'raspwr ' + MamprlksImg + ' ' + nWidth + os.system(call_str) + + call_str = 'rm *.amp' + os.system(call_str) + + call_str = 'rm *_*.slc' + os.system(call_str) + + print("Down to SLC for %s is done! " % Date) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/Raw2SLC_ERS_Cat_All.py b/.codex_tmp/pyint_variants/no_rescue/pyint/Raw2SLC_ERS_Cat_All.py new file mode 100644 index 0000000..4586dc9 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/Raw2SLC_ERS_Cat_All.py @@ -0,0 +1,136 @@ +#! /usr/bin/env python +#''' +################################################################################## +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : March, 2017 # +# # +# Generate Sentinel SLC from the downloaded data # +# # +################################################################################## +#''' +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close + +def rm(TXT): + call_str = 'rm ' + TXT + os.system(call_str) + +def usage(): + print(''' +****************************************************************************************************** + + Cat ERS SLC from ERS raw data with ENVISAT format. + + usage: + + Raw2SLC_ERS_Cat_All.py ProjectName + + e.g. Raw2SLC_ERS_Cat_All.py CotopaxiT120ERSA + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==2: + projectName = sys.argv[1] + else: + usage();sys.exit(1) + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + slcDir = scratchDir + '/' + projectName + '/SLC' + + if not os.path.isdir(slcDir): + call_str= 'mkdir ' +slcDir + os.system(call_str) + + os.chdir(downDir) + + call_str = 'ls > tt0' + os.system(call_str) + + call_str = 'grep SAR_IM tt0 >tt1' + os.system(call_str) + + call_str = "awk -F0PWDSI '{print $2}' tt1 > tt2 " + os.system(call_str) + + call_str = "awk -F_ '{print $1}' tt2 > ttt" + os.system(call_str) + + call_str = 'sort ttt | uniq > ttm' + os.system(call_str) + + AA= np.loadtxt('ttm',dtype=np.str) + Na = AA.size + + for i in range(Na): + call_str = 'Raw2SLC_ERS_Cat.py ' + projectName + ' ' + AA[i] + print(call_str) + call_str = 'Raw2SLC_ERS_Cat.py ' + projectName + ' ' + AA[i] + ' >/dev/null' + os.system(call_str) + + + print("Down to SLC for %s is done! " % projectName) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/SAR2LATLON.py b/.codex_tmp/pyint_variants/no_rescue/pyint/SAR2LATLON.py new file mode 100644 index 0000000..35a6c3b --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/SAR2LATLON.py @@ -0,0 +1,144 @@ +#! /usr/bin/env python + +#''' +################################################################################## +### Author: Yun-Meng Cao ### +### Date : March, 2017 ### +### Email : ymcmrs@gmail.com ### +### Transform SAR coordinates into LAT/LON coordinates based on lookup table ### +################################################################################## +#''' + +import os +import sys +import glob +import time +import argparse + +import h5py +import numpy as np + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def read_data(inFile, dtype, nWidth, nLength): + data = np.fromfile(inFile, dtype, int(nLength)*int(nWidth)).reshape(int(nLength),int(nWidth)) + + return data + +def is_number(s): + try: + int(s) + return True + except ValueError: + return False + +def add_zero(s): + if len(s)==1: + s="000"+s + elif len(s)==2: + s="00"+s + elif len(s)==3: + s="0"+s + return s + + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + + +def usage(): + print(''' +****************************************************************************************************** + + Transform SAR coordinates into LAT/LON coordinates based on lookup table + + usage: + + SAR2LATLON.py Range Azimuth LookupTableFile UTMDEMpar + + e.g. SAR2LATLON.py 1500 1000 /Yunmeng/20201230.utm_to_rdc /Yunmeng/20201230.dem.utm.par + + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==5: + Range = sys.argv[1] + Azimuth = sys.argv[2] + LtFile = sys.argv[3] + UTMPAR = sys.argv[4] + else: + usage();sys.exit(1) + + nWidthUTM = UseGamma(UTMPAR, 'read', 'width:') + nLineUTM = UseGamma(UTMPAR, 'read', 'nlines:') + + Corner_LAT = UseGamma(UTMPAR, 'read', 'corner_lat:') + Corner_LON = UseGamma(UTMPAR, 'read', 'corner_lon:') + + Corner_LAT =Corner_LAT.split(' ')[0] + Corner_LON =Corner_LON.split(' ')[0] + + post_Lat = UseGamma(UTMPAR, 'read', 'post_lat:') + post_Lon = UseGamma(UTMPAR, 'read', 'post_lon:') + + post_Lat =post_Lat.split(' ')[0] + post_Lon =post_Lon.split(' ')[0] + data = read_data(LtFile,'>c8',nWidthUTM,nLineUTM) # real: range imaginary: azimuth + + Range_LT = data.real + Azimuth_LT = data.imag + + CPX_input =complex(Range + '+' + Azimuth+'j') + + DD = abs(CPX_input - data) + + LL= abs(DD) + IDX= np.where(LL == LL.min()) + Lat_out = float(Corner_LAT) + IDX[0]*float(post_Lat) + Lon_out = float(Corner_LON) + IDX[1]*float(post_Lon) + + print(' Range: ' + Range + ' ' + 'Azimuth: ' + Azimuth) + print(' Latitude: ' + str(Lat_out[0]) + ' ' + 'Longitude: ' + str(Lon_out[0])) + +############################################################################## +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/SRTM_AUTO_DOWNLOAD.md b/.codex_tmp/pyint_variants/no_rescue/pyint/SRTM_AUTO_DOWNLOAD.md new file mode 100644 index 0000000..439e3b4 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/SRTM_AUTO_DOWNLOAD.md @@ -0,0 +1,150 @@ +# SRTM 自动下载功能说明 + +## 功能概述 + +`makedem.py` 现在支持自动下载SRTM数据,无需手动准备数据文件! + +## 使用方法 + +### 方法1: 自动下载 (推荐) + +**步骤1: 获取OpenTopography API key** (免费) + +访问: https://opentopography.org/myOpenTopo +- 注册免费账户 +- 在 'My Account' 页面请求API key +- API key立即可用,免费配额足够大部分用途 + +**步骤2: 运行命令** + +```bash +# 使用API key自动下载SRTM数据 +python makedem.py -r 116/117/39/40 --dem-source srtm --opentopo-api-key YOUR_API_KEY + +# 或设置环境变量 +export OPENTOPO_API_KEY="your_api_key_here" +python makedem.py -r 116/117/39/40 --dem-source srtm +``` + +### 方法2: 使用预下载的.hgt文件 + +如果已有SRTM .hgt文件: + +```bash +python makedem.py -r 116/117/39/40 --dem-source srtm --srtm-data-dir /path/to/srtm_data +``` + +## SRTM数据规格 + +- **分辨率**: 90米 (SRTM GL3) +- **覆盖**: 全球 (60°S - 60°N) +- **数据源**: OpenTopography API +- **格式**: GeoTIFF +- **费用**: 免费 (需要免费API key) + +## 对比 + +| 方法 | 优点 | 缺点 | +|------|------|------| +| 自动下载 | ✓ 全自动
✓ 无需准备数据
✓ 快速便捷 | ✗ 需要API key
✗ 需要网络 | +| 预下载文件 | ✓ 离线使用
✓ 不依赖API | ✗ 需要手动下载
✗ 需要管理文件 | + +## 示例 + +### 北京地区 (116-117°E, 39-40°N) + +```bash +# 自动下载 +python makedem.py -r 116/117/39/40 --dem-source srtm --opentopo-api-key YOUR_KEY + +# 输出文件: SRTM_116_117_39_40.tif +``` + +### 使用SLC参数文件 + +```bash +# 自动确定区域范围 +python makedem.py -s /path/to/slc.par --dem-source srtm --opentopo-api-key YOUR_KEY +``` + +## 常见问题 + +**Q: 为什么需要API key?** + +A: OpenTopography要求API key来管理数据访问配额。免费API key配额足够大部分科研和教育用途。 + +**Q: 如何获取API key?** + +A: +1. 访问 https://opentopography.org/myOpenTopo +2. 注册账户 (免费) +3. 在账户页面点击 "Request API Key" +4. API key会立即显示,可以直接使用 + +**Q: 免费API key有什么限制?** + +A: +- SRTM GL3: 最大4,050,000 km² per request +- 对于大部分研究区域完全够用 +- 如果需要更大范围,可以分块下载 + +**Q: SRTM vs Copernicus vs NASADEM?** + +A: +- **Copernicus (推荐)**: 30m分辨率,全球覆盖,全自动下载 +- **NASADEM**: 30m分辨率,NASA官方,60°S-60°N覆盖 +- **SRTM**: 90m分辨率,历史数据,适合对比研究 + +**Q: 没有.hgt文件怎么办?** + +A: 直接使用方法1自动下载,无需准备任何数据文件! + +## 故障排除 + +### 错误: HTTP 401 + +**原因**: API key无效或已过期 + +**解决**: +- 检查API key是否正确 +- 确认账户状态正常 +- 如需要,重新生成API key + +### 错误: HTTP 403 + +**原因**: 超出配额或访问限制 + +**解决**: +- 检查请求区域大小 +- 分块下载大区域 +- 确认API key配额 + +### 错误: 文件下载失败 + +**原因**: 网络问题 + +**解决**: +- 检查网络连接 +- 重试命令 +- 使用预下载方法 + +## Python版本兼容性 + +- **方法1 (自动下载)**: 支持所有Python版本 +- **方法2 (使用.hgt文件)**: + - Python >= 3.12: 使用srtm库自动查询 + - Python < 3.12: 使用GDAL手动处理 + +两种方法结果相同,只是处理方式不同。 + +## 相关资源 + +- OpenTopography: https://opentopography.org/ +- API文档: https://portal.opentopography.org/apidocs/ +- 获取API key: https://opentopography.org/myOpenTopo +- SRTM数据介绍: https://www2.jpl.nasa.gov/srtm/ + +--- + +**更新日期**: 2026-03-13 +**功能版本**: v1.0 diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/UTM2SARPIX.py b/.codex_tmp/pyint_variants/no_rescue/pyint/UTM2SARPIX.py new file mode 100644 index 0000000..46fc6b8 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/UTM2SARPIX.py @@ -0,0 +1,128 @@ +#! /usr/bin/env python +#''' +################################################################################### +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : March, 2017 # +# # +# Transform lat and lon to SAR coordinates based on SLCPar and DEM # +# # +################################################################################### +#''' + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + + +def usage(): + print(''' +****************************************************************************************************** + + Transform lat and lon to SAR coordinates based on SLCPar and DEM + + usage: + + UTM2SARPIX.py latitude longitude SLCPar DEM + + e.g. UTM2SARPIX.py 31.1 -108.2 /Yunmeng/2010.slc.par /Yunmeng/2010.dem + +******************************************************************************************************* + ''') + + +def main(argv): + + if len(sys.argv)==5: + LAT = sys.argv[1] + LON = sys.argv[2] + PAR = sys.argv[3] + DEM = sys.argv[4] + else: + usage();sys.exit(1) + + DEMpar = DEM + '.par' + + DateFormat = UseGamma(DEMpar, 'read', 'data_format:') + + nWidth = UseGamma(DEMpar, 'read', 'width:') + nLength = UseGamma(DEMpar, 'read', 'nlines:') + + Corner_LAT = UseGamma(DEMpar, 'read', 'corner_lat:') + Corner_LON = UseGamma(DEMpar, 'read', 'corner_lon:') + + Corner_LAT =Corner_LAT.split(' ')[0] + Corner_LON =Corner_LON.split(' ')[0] + + post_Lat = UseGamma(DEMpar, 'read', 'post_lat:') + post_Lon = UseGamma(DEMpar, 'read', 'post_lon:') + + post_Lat =post_Lat.split(' ')[0] + post_Lon =post_Lon.split(' ')[0] + + if DateFormat =='INTEGER*2': + STR = '>i2' + else: + STR = '>f4' + + TXT = 'SARCOORD' + + DEMdate = np.fromfile(DEM,STR,int(nLength)*int(nWidth)).reshape(int(nLength),int(nWidth)) + + LAT = float(LAT); LON =float(LON) + nWidth=int(nWidth);nLength=int(nLength) + Corner_LAT = float(Corner_LAT); Corner_LON=float(Corner_LON) + post_Lat = float(post_Lat); post_Lon=float(post_Lon) + + XX = int (( LAT - Corner_LAT ) / post_Lat) # latitude width range + YY = int (( LON - Corner_LON ) / post_Lon) # longitude nline azimuth + + + ELEV = DEMdate[XX][YY] + + call_str = 'coord_to_sarpix ' + PAR + ' - - ' + str(LAT) + ' ' + str(LON) + ' ' + str(ELEV) + ' >' +TXT + os.system(call_str) + + call_str = 'cat ' + TXT + os.system(call_str) + + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + + + + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/_network.py b/.codex_tmp/pyint_variants/no_rescue/pyint/_network.py new file mode 100644 index 0000000..4a3bf94 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/_network.py @@ -0,0 +1,483 @@ +############################################################ +# Program is part of PyINT # +# Copyright 2019 Yunmeng Cao # +# Contact: ymcmrs@gmail.com # +############################################################ +# This program is modified from MintPy/select_network.py +# Copyright: 2017-2019, Yunjun Zhang +# Contact: yzhang@rsmas.miami.edu + +import os +import sys +import itertools +import h5py +import numpy as np +from scipy import sparse +from matplotlib import (colors, + dates as mdates, + pyplot as plt) +from matplotlib.tri import Triangulation +from mpl_toolkits.axes_grid1 import make_axes_locatable + + + + +################################################################## +def read_pairs_list(date12ListFile, dateList=[]): + """Read Pairs List file like below: + 070311-070426 + 070311-070611 + ... + """ + # Read date12 list file + date12List = sorted(list(np.loadtxt(date12ListFile, dtype=bytes).astype(str))) + + # Get dateList from date12List + if not dateList: + dateList = [] + for date12 in date12List: + dates = date12.split('-') + if not dates[0] in dateList: + dateList.append(dates[0]) + if not dates[1] in dateList: + dateList.append(dates[1]) + dateList.sort() + date6List = yymmdd(dateList) + + # Get pair index + pairs_idx = [] + for date12 in date12List: + dates = date12.split('-') + pair_idx = [date6List.index(dates[0]), date6List.index(dates[1])] + pairs_idx.append(pair_idx) + + return pairs_idx + +def get_date12_list(fname, dropIfgram=False): + """Read Date12 info from input file: Pairs.list or multi-group hdf5 file + Inputs: + fname - string, path/name of input multi-group hdf5 file or text file + dropIfgram - bool, check the "DROP_IFGRAM" attribute or not for multi-group hdf5 file + Output: + date12_list - list of string in YYMMDD-YYMMDD format + Example: + date12List = get_date12_list('ifgramStack.h5') + date12List = get_date12_list('ifgramStack.h5', dropIfgram=True) + date12List = get_date12_list('Pairs.list') + """ + date12_list = [] + ext = os.path.splitext(fname)[1].lower() + if ext == '.h5': + k = readfile.read_attribute(fname)['FILE_TYPE'] + if k == 'ifgramStack': + date12_list = ifgramStack(fname).get_date12_list(dropIfgram=dropIfgram) + else: + return None + else: + txtContent = np.loadtxt(fname, dtype=bytes).astype(str) + if len(txtContent.shape) == 1: + txtContent = txtContent.reshape(-1, 1) + date12_list = [i for i in txtContent[:, 0]] + date12_list = sorted(date12_list) + return date12_list + + +def igram_perp_baseline_list(fname): + """Get perpendicular baseline list from input multi_group hdf5 file""" + print(('read perp baseline info from '+fname)) + k = readfile.read_attribute(fname)['FILE_TYPE'] + h5 = h5py.File(fname, 'r') + epochList = sorted(h5[k].keys()) + p_baseline_list = [] + for epoch in epochList: + p_baseline = (float(h5[k][epoch].attrs['P_BASELINE_BOTTOM_HDR']) + + float(h5[k][epoch].attrs['P_BASELINE_TOP_HDR']))/2 + p_baseline_list.append(p_baseline) + h5.close() + return p_baseline_list + + +def select_pairs_sbas(date_list, date12_format='YYMMDD-YYMMDD'): + """Select All Possible Pairs/Interferograms + Input : date_list - list of date in YYMMDD/YYYYMMDD format + Output: date12_list - list date12 in YYMMDD-YYMMDD format + Reference: + Berardino, P., G. Fornaro, R. Lanari, and E. Sansosti (2002), A new algorithm for surface deformation monitoring + based on small baseline differential SAR interferograms, IEEE TGRS, 40(11), 2375-2383. + """ + date8_list = sorted(yyyymmdd(date_list)) + date6_list = yymmdd(date8_list) + date12_list = list(itertools.combinations(date6_list, 2)) + date12_list = [date12[0]+'-'+date12[1] for date12 in date12_list] + if date12_format == 'YYYYMMDD_YYYYMMDD': + date12_list = yyyymmdd_date12(date12_list) + return date12_list + + +def select_pairs_sequential(date_list, num_connection=2, date12_format='YYMMDD-YYMMDD'): + """Select Pairs in a Sequential way: + For each acquisition, find its num_connection nearest acquisitions in the past time. + Inputs: + date_list : list of date in YYMMDD/YYYYMMDD format + Reference: + Fattahi, H., and F. Amelung (2013), DEM Error Correction in InSAR Time Series, IEEE TGRS, 51(7), 4249-4259. + """ + date8_list = sorted(yyyymmdd(date_list)) + date6_list = yymmdd(date8_list) + date_idx_list = list(range(len(date6_list))) + + # Get pairs index list + date12_idx_list = [] + for date_idx in date_idx_list: + for i in range(num_connection): + if date_idx-i-1 >= 0: + date12_idx_list.append([date_idx-i-1, date_idx]) + date12_idx_list = [sorted(idx) for idx in sorted(date12_idx_list)] + + # Convert index into date12 + date12_list = [date6_list[idx[0]]+'-'+date6_list[idx[1]] + for idx in date12_idx_list] + if date12_format == 'YYYYMMDD_YYYYMMDD': + date12_list = yyyymmdd_date12(date12_list) + return date12_list + + +def select_pairs_hierarchical(date_list, pbase_list, temp_perp_list, date12_format='YYMMDD-YYMMDD'): + """Select Pairs in a hierarchical way using list of temporal and perpendicular baseline thresholds + For each temporal/perpendicular combination, select all possible pairs; and then merge all combination results + together for the final output (Zhao, 2015). + Inputs: + date_list : list of date in YYMMDD/YYYYMMDD format + pbase_list : list of float, perpendicular spatial baseline + temp_perp_list : list of list of 2 floats, for list of temporal/perp baseline, e.g. + [[32.0, 800.0], [48.0, 600.0], [64.0, 200.0]] + Examples: + pairs = select_pairs_hierarchical(date_list, pbase_list, [[32.0, 800.0], [48.0, 600.0], [64.0, 200.0]]) + Reference: + Zhao, W., (2015), Small deformation detected from InSAR time-series and their applications in geophysics, Doctoral + dissertation, Univ. of Miami, Section 6.3. + """ + # Get all date12 + date12_list_all = select_pairs_all(date_list) + + # Loop of Threshold + print('List of temporal and perpendicular spatial baseline thresholds:') + print(temp_perp_list) + date12_list = [] + for temp_perp in temp_perp_list: + tbase_max = temp_perp[0] + pbase_max = temp_perp[1] + date12_list_tmp = threshold_temporal_baseline(date12_list_all, + tbase_max, + keep_seasonal=False) + date12_list_tmp = threshold_perp_baseline(date12_list_tmp, + date_list, + pbase_list, + pbase_max) + date12_list += date12_list_tmp + date12_list = sorted(list(set(date12_list))) + if date12_format == 'YYYYMMDD_YYYYMMDD': + date12_list = yyyymmdd_date12(date12_list) + return date12_list + + +def select_pairs_delaunay(date_list, tbase_list, pbase_list, norm=True, date12_format='YYMMDD-YYMMDD'): + """Select Pairs using Delaunay Triangulation based on temporal/perpendicular baselines + Inputs: + date_list : list of date in YYMMDD/YYYYMMDD format + pbase_list : list of float, perpendicular spatial baseline + norm : normalize temporal baseline to perpendicular baseline + Key points + 1. Define a ratio between perpendicular and temporal baseline axis units (Pepe and Lanari, 2006, TGRS). + 2. Pairs with too large perpendicular / temporal baseline or Doppler centroid difference should be removed + after this, using a threshold, to avoid strong decorrelations (Zebker and Villasenor, 1992, TGRS). + Reference: + Pepe, A., and R. Lanari (2006), On the extension of the minimum cost flow algorithm for phase unwrapping + of multitemporal differential SAR interferograms, IEEE TGRS, 44(9), 2374-2383. + Zebker, H. A., and J. Villasenor (1992), Decorrelation in interferometric radar echoes, IEEE TGRS, 30(5), 950-959. + """ + # Get temporal baseline in days + date6_list = yymmdd(date_list) + date8_list = yyyymmdd(date_list) + #tbase_list = date_list2tbase(date8_list)[0] + + # Normalization (Pepe and Lanari, 2006, TGRS) + if norm: + temp2perp_scale = (max(pbase_list)-min(pbase_list)) / (max(tbase_list)-min(tbase_list)) + tbase_list = [tbase*temp2perp_scale for tbase in tbase_list] + + # Generate Delaunay Triangulation + date12_idx_list = Triangulation(tbase_list, pbase_list).edges.tolist() + date12_idx_list = [sorted(idx) for idx in sorted(date12_idx_list)] + + # Convert index into date12 + date12_list = [date6_list[idx[0]]+'-'+date6_list[idx[1]] + for idx in date12_idx_list] + if date12_format == 'YYYYMMDD_YYYYMMDD': + date12_list = yyyymmdd_date12(date12_list) + return date12_list + + +def select_pairs_mst(date_list, pbase_list, date12_format='YYMMDD-YYMMDD'): + """Select Pairs using Minimum Spanning Tree technique + Connection Cost is calculated using the baseline distance in perp and scaled temporal baseline (Pepe and Lanari, + 2006, TGRS) plane. + Inputs: + date_list : list of date in YYMMDD/YYYYMMDD format + pbase_list : list of float, perpendicular spatial baseline + References: + Pepe, A., and R. Lanari (2006), On the extension of the minimum cost flow algorithm for phase unwrapping + of multitemporal differential SAR interferograms, IEEE TGRS, 44(9), 2374-2383. + Perissin D., Wang T. (2012), Repeat-pass SAR interferometry with partially coherent targets. IEEE TGRS. 271-280 + """ + # Get temporal baseline in days + date6_list = yymmdd(date_list) + date8_list = yyyymmdd(date_list) + tbase_list = date_list2tbase(date8_list)[0] + # Normalization (Pepe and Lanari, 2006, TGRS) + temp2perp_scale = (max(pbase_list)-min(pbase_list)) / (max(tbase_list)-min(tbase_list)) + tbase_list = [tbase*temp2perp_scale for tbase in tbase_list] + + # Get weight matrix + ttMat1, ttMat2 = np.meshgrid(np.array(tbase_list), np.array(tbase_list)) + ppMat1, ppMat2 = np.meshgrid(np.array(pbase_list), np.array(pbase_list)) + ttMat = np.abs(ttMat1 - ttMat2) # temporal distance matrix + ppMat = np.abs(ppMat1 - ppMat2) # spatial distance matrix + + # 2D distance matrix in temp/perp domain + weightMat = np.sqrt(np.square(ttMat) + np.square(ppMat)) + weightMat = sparse.csr_matrix(weightMat) # compress sparse row matrix + + # MST path based on weight matrix + mstMat = sparse.csgraph.minimum_spanning_tree(weightMat) + + # Convert MST index matrix into date12 list + [s_idx_list, m_idx_list] = [date_idx_array.tolist() + for date_idx_array in sparse.find(mstMat)[0:2]] + date12_list = [] + for i in range(len(m_idx_list)): + idx = sorted([m_idx_list[i], s_idx_list[i]]) + date12 = date6_list[idx[0]]+'-'+date6_list[idx[1]] + date12_list.append(date12) + if date12_format == 'YYYYMMDD_YYYYMMDD': + date12_list = yyyymmdd_date12(date12_list) + return date12_list + + +def select_pairs_star(date_list, m_date=None, pbase_list=[], date12_format='YYMMDD-YYMMDD'): + """Select Star-like network/interferograms/pairs, it's a single master network, similar to PS approach. + Usage: + m_date : master date, choose it based on the following cretiria: + 1) near the center in temporal and spatial baseline + 2) prefer winter season than summer season for less temporal decorrelation + Reference: + Ferretti, A., C. Prati, and F. Rocca (2001), Permanent scatterers in SAR interferometry, IEEE TGRS, 39(1), 8-20. + """ + date8_list = sorted(yyyymmdd(date_list)) + date6_list = yymmdd(date8_list) + + # Select master date if not existed + if not m_date: + m_date = select_master_date(date8_list, pbase_list) + print(('auto select master date: '+m_date)) + + # Check input master date + m_date8 = yyyymmdd(m_date) + if m_date8 not in date8_list: + print('Input master date is not existed in date list!') + print(('Input master date: '+str(m_date8))) + print(('Input date list: '+str(date8_list))) + m_date8 = None + + # Generate star/ps network + m_idx = date8_list.index(m_date8) + date12_idx_list = [sorted([m_idx, s_idx]) for s_idx in range(len(date8_list)) + if s_idx is not m_idx] + date12_list = [date6_list[idx[0]]+'-'+date6_list[idx[1]] + for idx in date12_idx_list] + if date12_format == 'YYYYMMDD_YYYYMMDD': + date12_list = yyyymmdd_date12(date12_list) + return date12_list + + +def select_master_date(date_list, pbase_list=[]): + """Select super master date based on input temporal and/or perpendicular baseline info. + Return master date in YYYYMMDD format. + """ + date8_list = yyyymmdd(date_list) + if not pbase_list: + # Choose date in the middle + m_date8 = date8_list[int(len(date8_list)/2)] + else: + # Get temporal baseline list + tbase_list = date_list2tbase(date8_list)[0] + # Normalization (Pepe and Lanari, 2006, TGRS) + temp2perp_scale = (max(pbase_list)-min(pbase_list)) / (max(tbase_list)-min(tbase_list)) + tbase_list = [tbase*temp2perp_scale for tbase in tbase_list] + # Get distance matrix + ttMat1, ttMat2 = np.meshgrid(np.array(tbase_list), + np.array(tbase_list)) + ppMat1, ppMat2 = np.meshgrid(np.array(pbase_list), + np.array(pbase_list)) + ttMat = np.abs(ttMat1 - ttMat2) # temporal distance matrix + ppMat = np.abs(ppMat1 - ppMat2) # spatial distance matrix + # 2D distance matrix in temp/perp domain + disMat = np.sqrt(np.square(ttMat) + np.square(ppMat)) + + # Choose date minimize the total distance of temp/perp baseline + disMean = np.mean(disMat, 0) + m_idx = np.argmin(disMean) + m_date8 = date8_list[m_idx] + return m_date8 + + +def select_master_interferogram(date12_list, date_list, pbase_list, m_date=None): + """Select reference interferogram based on input temp/perp baseline info + If master_date is specified, select its closest slave_date, which is newer than master_date; + otherwise, choose the closest pair among all pairs as master interferogram. + Example: + master_date12 = pnet.select_master_ifgram(date12_list, date_list, pbase_list) + '080211-080326' = pnet.select_master_ifgram(date12_list, date_list, pbase_list, m_date='080211') + """ + pbase_array = np.array(pbase_list, dtype='float64') + # Get temporal baseline + date8_list = yyyymmdd(date_list) + date6_list = yymmdd(date8_list) + tbase_array = np.array(date_list2tbase(date8_list)[0], dtype='float64') + # Normalization (Pepe and Lanari, 2006, TGRS) + temp2perp_scale = (max(pbase_array)-min(pbase_array)) / (max(tbase_array)-min(tbase_array)) + tbase_array *= temp2perp_scale + + # Calculate sqrt of temp/perp baseline for input pairs + idx1 = np.array([date6_list.index(date12.split('-')[0]) for date12 in date12_list]) + idx2 = np.array([date6_list.index(date12.split('-')[1]) for date12 in date12_list]) + base_distance = np.sqrt((tbase_array[idx2] - tbase_array[idx1])**2 + + (pbase_array[idx2] - pbase_array[idx1])**2) + + # Get master interferogram index + if not m_date: + # Choose pair with shortest temp/perp baseline + m_date12_idx = np.argmin(base_distance) + else: + m_date = yymmdd(m_date) + # Choose pair contains m_date with shortest temp/perp baseline + m_date12_idx_array = np.array([date12_list.index(date12) for date12 in date12_list + if m_date+'-' in date12]) + min_base_distance = np.min(base_distance[m_date12_idx_array]) + m_date12_idx = np.where(base_distance == min_base_distance)[0][0] + + m_date12 = date12_list[m_date12_idx] + return m_date12 + + +########################################################## +def datenum2datetime(datenum): + """Convert Matlab datenum into Python datetime. + Parameters: datenum : Date in datenum format, i.e. 731763.5 + Returns: datetime: Date in datetime.datetime format, datetime.datetime(2003, 7, 1, 12, 0) + """ + return dt.fromordinal(int(datenum)) \ + + timedelta(days=datenum % 1) \ + - timedelta(days=366) + + +def decimal_year2datetime(years): + """read date in 2002.40657084 to datetime format + Parameters: years : (list of) float or str for years + Returns: years_dt : (list of) datetime.datetime objects + """ + def decimal_year2datetime1(x): + x = float(x) + year = np.floor(x).astype(int) + yday = np.floor((x - year) * 365.25).astype(int) + 1 + x2 = '{:d}-{:d}'.format(year, yday) + try: + xt = dt(*time.strptime(x2, "%Y-%j")[0:5]) + except: + raise ValueError('wrong format: ',x) + return xt + + if isinstance(years, (float, str)): + years_dt = decimal_year2datetime1(years) + + elif isinstance(years, list): + years_dt = [] + for year in years: + years_dt.append(decimal_year2datetime1(year)) + + else: + raise ValueError('unrecognized input format: {}. Only float/str/list are supported.'.format(type(years))) + return years_dt + + +def yyyymmdd2years(dates): + if isinstance(dates, str): + d = dt(*time.strptime(dates, "%Y%m%d")[0:5]) + yy = float(d.year)+float(d.timetuple().tm_yday-1)/365.25 + elif isinstance(dates, list): + yy = [] + for date in dates: + d = dt(*time.strptime(date, "%Y%m%d")[0:5]) + yy.append(float(d.year)+float(d.timetuple().tm_yday-1)/365.25) + else: + raise ValueError('Unrecognized date format. Only string and list supported.') + return yy + + +def yymmdd2yyyymmdd(date): + if date[0] == '9': + date = '19'+date + else: + date = '20'+date + return date + + +def yyyymmdd(dates): + if isinstance(dates, str): + if len(dates) == 6: + datesOut = yymmdd2yyyymmdd(dates) + else: + datesOut = dates + elif isinstance(dates, list): + datesOut = [] + for date in dates: + if len(date) == 6: + date = yymmdd2yyyymmdd(date) + datesOut.append(date) + else: + # print 'Un-recognized date input!' + return None + return datesOut + + +def yymmdd(dates): + if isinstance(dates, str): + if len(dates) == 8: + datesOut = dates[2:8] + else: + datesOut = dates + elif isinstance(dates, list): + datesOut = [] + for date in dates: + if len(date) == 8: + date = date[2:8] + datesOut.append(date) + else: + # print 'Un-recognized date input!' + return None + return datesOut + + +def yyyymmdd_date12(date12_list): + """Convert date12 into YYYYMMDD_YYYYMMDD format""" + m_dates = yyyymmdd([i.replace('-', '_').split('_')[0] for i in date12_list]) + s_dates = yyyymmdd([i.replace('-', '_').split('_')[1] for i in date12_list]) + date12_list = ['{}-{}'.format(m, s) for m, s in zip(m_dates, s_dates)] + return date12_list + +def yymmdd_date12(date12_list): + """Convert date12 into YYMMDD-YYMMDD format""" + m_dates = yymmdd([i.replace('-', '_').split('_')[0] for i in date12_list]) + s_dates = yymmdd([i.replace('-', '_').split('_')[1] for i in date12_list]) + date12_list = ['{}-{}'.format(m, s) for m, s in zip(m_dates, s_dates)] + return date12_list diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/_orbit_bridge.py b/.codex_tmp/pyint_variants/no_rescue/pyint/_orbit_bridge.py new file mode 100644 index 0000000..1a70887 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/_orbit_bridge.py @@ -0,0 +1,93 @@ +import os +import subprocess +import sys + + +TRUE_VALUES = {"1", "true", "yes", "on"} + + +def _read_bool_env(name, default=False): + value = os.getenv(name) + if value is None: + return default + return str(value).strip().lower() in TRUE_VALUES + + +def apply_precise_orbit(date_text, slc_par_paths, work_dir=None, operation_tag="raw2slc"): + helper_path = str(os.getenv("PYINT_LT1_PRECISE_ORBIT_HELPER") or "").strip() + manifest_json = str(os.getenv("PYINT_LT1_PRECISE_ORBIT_MANIFEST") or "").strip() + enabled = _read_bool_env("PYINT_LT1_PRECISE_ORBIT_ENABLED", False) + strict = _read_bool_env("PYINT_LT1_PRECISE_ORBIT_STRICT", True) + + result = { + "enabled": enabled, + "helper_path": helper_path, + "manifest_json": manifest_json, + "summary_json": "", + "returncode": 0, + "stdout": "", + "stderr": "", + "command": [], + "applied_files": [], + "status": "disabled", + } + if not enabled: + return result + if not helper_path or not os.path.isfile(helper_path): + message = "PYINT_LT1_PRECISE_ORBIT_HELPER is missing or does not exist." + result.update({"status": "missing_helper", "stderr": message}) + if strict: + raise RuntimeError(message) + return result + if not manifest_json: + message = "PYINT_LT1_PRECISE_ORBIT_MANIFEST is empty." + result.update({"status": "missing_manifest", "stderr": message}) + if strict: + raise RuntimeError(message) + return result + + files = [] + for path in slc_par_paths or []: + text = str(path or "").strip() + if text and os.path.isfile(text): + files.append(text) + result["applied_files"] = files + if not files: + result["status"] = "skipped" + return result + + summary_json = os.path.join(str(work_dir or os.getcwd()), "orbit_bridge_summary.json") + command = [ + sys.executable, + helper_path, + "--date", + str(date_text or "").strip(), + "--manifest-json", + manifest_json, + "--summary-json", + summary_json, + "--operation-tag", + str(operation_tag or "raw2slc"), + ] + for path in files: + command.extend(["--slc-par", path]) + result["summary_json"] = summary_json + result["command"] = command + + proc = subprocess.run( + command, + text=True, + capture_output=True, + check=False, + ) + result.update( + { + "returncode": int(proc.returncode), + "stdout": proc.stdout or "", + "stderr": proc.stderr or "", + "status": "applied" if proc.returncode == 0 else "failed", + } + ) + if proc.returncode != 0 and strict: + raise RuntimeError(result["stderr"] or result["stdout"] or "Precise orbit bridge failed.") + return result diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/_utils.py b/.codex_tmp/pyint_variants/no_rescue/pyint/_utils.py new file mode 100644 index 0000000..58f607d --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/_utils.py @@ -0,0 +1,689 @@ +############################################################ +# Program is part of PyINT V2.1 # +# Copyright 2017-2019 Yunmeng Cao # +# Contact: ymcmrs@gmail.com # +############################################################ +#from datetime import datetime +import datetime +import urllib.request +import os +import numpy as np +import random +import h5py +from pathlib import Path +import linecache +import time +import glob + +from tqdm import tqdm +from concurrent.futures import ProcessPoolExecutor, as_completed + +# 无条件清理 OMP_PLACES / OMP_PROC_BIND，避免 libgomp 报错 +for _k in ('OMP_PLACES', 'OMP_PROC_BIND'): + os.environ.pop(_k, None) + +####################### update template ############################# +def update_template(template_file): + + templateDict = {} + + ######### download using SSARA ###### + #templateDict['sensor'] = 'sentinel-1' + #templateDict['track'] = '14' + #templateDict['frame'] = '75' + templateDict['start_time'] = '1989-01-01' + templateDict['end_time'] = '2189-01-01' + + ######### download using ASF API ###### + templateDict['asf_username'] = '' # ASF username (can be read from ~/.netrc) + templateDict['asf_password'] = '' # ASF password (can be read from ~/.netrc) + templateDict['bbox'] = '' # bounding box: min_lon,min_lat,max_lon,max_lat + templateDict['shapefile'] = '' # shapefile path for spatial filtering + templateDict['relative_orbit'] = '' # relative orbit number (optional) + templateDict['frame'] = '' # frame number (optional) + templateDict['flight_direction'] = 'a' # flight direction: a=ascending, d=descending, all + templateDict['acquisition_mode'] = 'IW' # acquisition mode: IW, EW, SM, all + + ####### basic parameters for interferometry ###### + templateDict['start_swath'] = '1' + templateDict['end_swath'] = '3' + + templateDict['start_burst'] = '1' + templateDict['end_burst'] = '20' + + templateDict['dem_lat_ovr'] = '0.5' # get 30m resolution of lookup table, 0.5 to 60m, 2 to 15m + templateDict['dem_lon_ovr'] = '0.5' # get 30m resolution of lookup table + templateDict['dem_num_workers'] = '4' # number of parallel workers for DEM download + templateDict['dem_source'] = 'copernicus' # DEM data source: copernicus or nasadem + + templateDict['Igram_Spsflg'] = '1' # Range spectral filtering + templateDict['Igram_Azfflg'] = '1' # Azimuth common band filtering + + templateDict['rwin4cor'] = '256' # range window length for coregistration + templateDict['azwin4cor'] = '256' # azimuth window length for coregistration + + templateDict['rsample4cor'] = '32' # range samples used for fitting the coregistration parameters + templateDict['azsample4cor'] = '32' # azimuth samples used for fitting the coregistration parameters + + templateDict['thresh4cor'] = '0.15' # 2016 GAMMA or higher version, for 2015 GAMMA or lower version should be SNR + + templateDict['coreCoarse'] = 'both' # initial coregistration method, [options: orbit, ampcor, both] + templateDict['coreMethod'] = 'DEM' # coregistration method [option: DEM, init] DEM means with DEM assistant + + templateDict['Igram_Cor_rwin'] = '5' # used for cc_wave + templateDict['Igram_Cor_awin'] = '5' # used for cc_wave + + templateDict['Igram_Cor_Win'] = '5' # used for adf + templateDict['adf_alpha'] = '0.4' # used for adf + ######## sim phase ################## + templateDict['Igram_Flag_TDM'] = 'N' + templateDict['Simphase_rpos'] = '-' + templateDict['Simphase_azpos'] = '-' + templateDict['Simphase_rwin'] = '256' + templateDict['Simphase_azwin'] = '256' + templateDict['Simphase_thresh'] = '-' + + #### unwrap phase ########### + templateDict['auto_unw'] = '1' # auto reference-point selection + templateDict['make_mask'] = '1' # coherence-mask switch for unwrap + templateDict['init_flag'] = '1' # set reference-point phase to 0.0 + templateDict['r_refer'] = '-' # range reference pixel for manual unwrap + templateDict['a_refer'] = '-' # azimuth reference pixel for manual unwrap + templateDict['mcf_triangular'] = '0' # triangular type of mcf [0: regular; 1: delaunay;] + templateDict['unwrap_patr'] = '4' + templateDict['unwrap_pataz'] = '4' + templateDict['unwrapThreshold'] = '0.1' # minimum coherence used for unwrap + + #### geocode ######### + templateDict['geo_interp'] = '0' # [0: nearest; 1: bicubic spline] + templateDict['geocode_products'] = 'hyp3,licsbas' # 产品类型: hyp3,licsbas,pot (逗号分隔多选) + + ############## interferometry ################ + templateDict['int_flag'] = '1' # 1 means do interferometry + templateDict['diff_flag'] = '1' # differential process, i.e., remove DEM phase + templateDict['unw_flag'] = '1' # unwrap process + templateDict['geo_flag'] = '0' # geocode process + + ############## select network ############# + templateDict['endDate'] = '21000101' + templateDict['startDate'] = '19000101' + templateDict['network_method'] = 'sbas' # sbas, sequential, delaunay, stars + templateDict['conNumb'] = '2' # connect number for sequential + templateDict['max_tb'] = '50000' + templateDict['max_sb'] = '50000' + + ############## time-series ################ + templateDict['download_data'] = '0' # if 1, track, frame, or time informations should be provided + templateDict['down_parallel'] = '1' # multi-processor number used for downloading + + templateDict['raw2slc_all'] = '0' # i.e., download 2 slc + templateDict['raw2slc_all_parallel'] = '1' # multi-processor number used + + templateDict['extract_burst_all'] = '0' # for TOPS SLC only + templateDict['extract_all_parallel'] = '1' # multi-processor number used + + templateDict['coreg_all'] = '1' + templateDict['coreg_all_parallel'] = '1' # multi-processor number used [4 or 8] + + templateDict['select_pairs'] = '1' + + templateDict['diff_all'] = '1' + templateDict['diff_all_parallel'] = '1' # multi-processor number used [4 or 8] + + templateDict['unwrap_all'] = '1' + templateDict['unwrap_all_parallel'] = '1' # multi-processor number used [8 or 10] + + templateDict['atmcor_all'] = '0' # if 1, run atmospheric correction + templateDict['atmcor_all_parallel'] = '1' # multi-processor number used + + templateDict['geocode_all'] = '0' + templateDict['geocode_all_parallel'] = '1' # multi-processor number used + templateDict['gamma2licsbas_all'] = '0' # legacy conversion stage switch + templateDict['gamma2licsbas_all_parallel'] = '1' + templateDict['hyp3format_all'] = '0' # legacy conversion stage switch + templateDict['hyp3format_all_parallel'] = '1' + + ############## GACOS atmospheric correction ############# + templateDict['gacos_correction'] = '0' # if 1, apply GACOS atmospheric correction + templateDict['gacos_all_parallel'] = '1' # multi-processor number used for GACOS correction + templateDict['gacos_dir'] = '' # directory for GACOS ZTD files (default: projectName/GACOS) + templateDict['gacos_email'] = '' # email for GACOS auto-download + templateDict['gacos_email_user'] = '' # IMAP username + templateDict['gacos_email_pass'] = '' # IMAP authorization code + templateDict['gacos_email_host'] = 'imap.163.com' # IMAP server host + templateDict['gacos_email_port'] = '993' # IMAP server port + templateDict['gacos_email_ssl'] = '1' # Use SSL + templateDict['gacos_check_interval'] = '60' # Email check interval (seconds) + + ############## Pixel Offset Tracking (POT) ############# + templateDict['pot_all'] = '0' # if 1, run offset tracking + templateDict['pot_all_parallel'] = '1' # multi-processor number used + templateDict['pot_rstep'] = '50' # range step in SLC pixels + templateDict['pot_azstep'] = '10' # azimuth step in SLC pixels + templateDict['pot_rwin'] = '256' # range window size (Round 1) + templateDict['pot_azwin'] = '128' # azimuth window size (Round 1) + templateDict['pot_rwin2'] = '128' # range window size (Round 2) + templateDict['pot_azwin2'] = '64' # azimuth window size (Round 2) + templateDict['pot_ovr'] = '1' # SLC oversampling factor (1=no deramp needed) + templateDict['pot_snr_thresh'] = '0.01' # SNR threshold for offset_pwr_tracking + templateDict['pot_ccp_thresh'] = '0.05' # cross-correlation threshold for masking + templateDict['pot_roff_min'] = '-50' # min range offset (pixels) for masking + templateDict['pot_roff_max'] = '50' # max range offset (pixels) for masking + templateDict['pot_azoff_min'] = '-20' # min azimuth offset (pixels) for masking + templateDict['pot_azoff_max'] = '20' # max azimuth offset (pixels) for masking + templateDict['pot_drange_thresh'] = '2.0' # deviation threshold range (pixels) + templateDict['pot_dazimuth_thresh'] = '1.0' # deviation threshold azimuth (pixels) + templateDict['pot_median_win'] = '7' # median filter window size + templateDict['pot_median_nmin'] = '15' # min valid samples for median filter + templateDict['pot_two_rounds'] = '1' # 1=two-round estimation, 0=single round + templateDict['pot_geocode'] = '1' # 1=geocode displacement results + templateDict['pot_disp_max'] = '100' # max displacement for BMP display (meters) + + templateDict['load_data'] = '0' # loading data for mintpy processing + + templateDict0 = read_template(template_file, delimiter='=') + for key, value in templateDict0.items(): + templateDict[key] = str(value) + + return templateDict + +####################### time-related function ################ + +def is_number(s): + try: + int(s) + return True + except ValueError: + return False + +def print_process_time(start, end): + hours, rem = divmod(end-start, 3600) + minutes, seconds = divmod(rem, 60) + print("Total process time: "+"{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) + return + +def date_add1d(date0): + format_str = '%Y%m%d' # The format + date_obj = datetime.datetime.strptime(date0,format_str) + date1 = (date_obj + datetime.timedelta(days=1)).date().strftime('%Y%m%d') + return date1 + +def date_minus1d(date0): + format_str = '%Y%m%d' # The format + date_obj = datetime.datetime.strptime(date0,format_str) + date1 = (date_obj - datetime.timedelta(days=1)).date().strftime('%Y%m%d') + return date1 + + +def generate_random_name(sufix): + + nowTime = datetime.datetime.now().strftime("%Y%m%d%H%M%S") #generate the present time + randomNum = random.randint(0,100) + randomNum1 = random.randint(0,100) + Nm = nowTime + str(randomNum)+ str(randomNum1) + sufix + + return Nm + +def yyyymmdd2yyyyddd(date): + dt = datetime.datetime.strptime(date, "%Y%m%d") # get datetime object + day_of_year = (dt - datetime.datetime(dt.year, 1, 1)) # Jan the 1st is day 1 + doy = day_of_year.days + 1 + year = dt.year + + doy = str(doy) + if len(doy) ==1: + doy = '00' + doy + elif len(doy) ==2: + doy ='0'+ doy + year = str(year) + + return year, doy + +def get_txt_lines(txt): + count=0 + myfile=open(txt,"r") + for line in myfile: + count=count+1 + + return count + +def read_txt2list(txt): + A = np.loadtxt(txt,dtype=str) + if np.array(A).size ==1: + A = [A] + elif isinstance(A[0],bytes): + A = A.astype(str) + A = list(A) + return A + +def read_txt2array(txt): + A = np.loadtxt(txt,dtype=str) + if np.array(A).size ==1: + A = [A] + elif isinstance(A[0],bytes): + A = A.astype(str) + #A = list(A) + return A +def get_sardata_swath(start_swath,end_swath): + k_swath = '0' + if (start_swath == '1') and (end_swath == '1'): + k_swath = '1' + elif (start_swath == '2') and (end_swath == '2'): + k_swath = '2' + elif (start_swath == '3') and (end_swath == '3'): + k_swath = '3' + elif (start_swath == '1') and (end_swath == '2'): + k_swath = '4' + elif (start_swath == '2') and (end_swath == '3'): + k_swath = '5' + elif (start_swath == '2') and (end_swath == '3'): + k_swath = '-' + + return k_swath + +def get_filelist_filesize(url0): + ttt = generate_random_name('.txt') + call_str = 'curl -s ' + url0 + ' > ' + ttt + os.system(call_str) + A = read_txt2array(ttt) + + A_size = A[:,4] + A_size = A_size.astype(int) + A_size = A_size/1024/1024 #bytes to Mb + A_name = A[:,8] + if os.path.isfile(ttt): + os.remove(ttt) + + return A_name, A_size + +def yyyymmdd(date0): + if len(date0) ==6: + if float(date0[0:2]) > 90: + date1 = '19' + date0 + else: + date1 = '20' + date0 + elif len(date0) ==8: + date1 = date0 + else: + print('The input date is invalid!!') + date1 = '' + return date1 + +def yymmdd(date0): + if len(date0) ==6: + date1 = date0 + elif len(date0) ==8: + date1 = date0[2:8] + else: + print('The input date is invalid!!') + date1 = '' + return date1 + +def parallel_process(array, function, n_jobs=16, use_kwargs=False): + """ + A parallel version of the map function with a progress bar. + + Args: + array (array-like): An array to iterate over. + function (function): A python function to apply to the elements of array + n_jobs (int, default=16): The number of cores to use + use_kwargs (boolean, default=False): Whether to consider the elements of array as dictionaries of + keyword arguments to function + Returns: + [function(array[0]), function(array[1]), ...] + """ + #We run the first few iterations serially to catch bugs + #If we set n_jobs to 1, just run a list comprehension. This is useful for benchmarking and debugging. + if n_jobs==1: + return [function(**a) if use_kwargs else function(a) for a in tqdm(array[:])] + #Assemble the workers + with ProcessPoolExecutor(max_workers=n_jobs) as pool: + #Pass the elements of array into function + if use_kwargs: + futures = [pool.submit(function, **a) for a in array[:]] + else: + futures = [pool.submit(function, a) for a in array[:]] + kwargs = { + 'total': len(futures), + 'unit': 'it', + 'unit_scale': True, + 'leave': True + } + #Print out the progress as tasks complete + for f in tqdm(as_completed(futures), **kwargs): + pass + out = [] + #Get the results from the futures. + for i, future in tqdm(enumerate(futures)): + try: + out.append(future.result()) + except Exception as e: + out.append(e) + return out + +def sort_unique_list(numb_list): + list_out = sorted(set(numb_list)) + return list_out + + +############################# download data ################################### +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def download_s1_orbit(date,save_path,satellite='A'): + + DATE = date + if len(DATE)==6: + DATE = '20' + DATE + ST = satellite + YEAR = int(DATE[0:4]) + MON = int(DATE[4:6]) + DAY = int(DATE[6:8]) + + MON_DAY = [31,28,31,30,31,30,31,31,30,31,30,31] + + if YEAR%4==0: + MON_DAY[1]=29 + + if MON ==1 and DAY ==1: + DAY0 = 31 + MON0 = 12 + YEAR0 =YEAR -1 + elif MON!=1 and DAY ==1: + DAY0 = MON_DAY[MON-2] + MON0 = MON-1 + YEAR0 = YEAR + else: + DAY0 = DAY -1 + MON0 = MON + YEAR0 = YEAR + + MONDAY0 = MON_DAY[MON0-1] + + TT = [1,4,7,10,13,16,19,22,25,28,MONDAY0] + + T0 = [] + for k in range(len(TT)): + T0.append(TT[k]) + TT[k] = TT[k]-DAY0 + + for k in range(len(TT)): + if k == len(TT)-1: + if TT[k]==0: + ff = k-1 + else: + if TT[k]<=0 and TT[k+1]>0: + ff = k + + + DAY1 = T0[ff] + DAY2 = T0[ff+1] + + S1 = StrNum(YEAR0) + '-' + StrNum(MON0) + S2 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY1) + S3 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY2) + S4 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY0) + + #SS = 'https://qc.sentinel1.eo.esa.int/aux_poeorb/?mission=S1' + ST + '&validity_start_time=' + StrNum(YEAR0) + '&validity_start_time=' + S1 + '&validity_start_time=' + S2 + '..' + S3 + '&validity_start_time=' + S4 + SS = 'https://qc.sentinel1.eo.esa.int/aux_poeorb/?validity_start=' + StrNum(YEAR0) + '&validity_start=' + S1 + '&validity_start=' + S2 + '..' + S3 + '&validity_start=' +S4 + '&sentinel1__mission=S1'+ST+ '&sentinel1_mission=S1'+ST+ '&sentinel1_mission=S1'+ST+ '&sentinel1_mission=S1'+ST + + tt ='tt_orb_' + date + tt0 ='tt0_orb_' + date + tt00 ='tt00_orb_' + date + tt000 ='tt000_orb_' + date + SS = "'" + SS + "'" + #print(SS) + call_str = 'curl -s -l ' + SS + ' > ' + tt + os.system(call_str) + + call_str = "grep 'EOF' -C 0 " + tt + " >" + tt0 + os.system(call_str) + + call_str="awk -F'href=' '{print $2}' " + tt0 +' >' + tt00 + os.system(call_str) + + call_str= "awk -F'>' '{print $1}' " + tt00 + '> ' + tt000 + os.system(call_str) + + SS=linecache.getline(tt000, 1) + SS = SS.split('"')[1] + filename = os.path.basename(SS) + if not os.path.isfile(filename): + call_str = 'wget -q --no-check-certificate ' + SS + ' -O ' + save_path + '/' +filename + os.system(call_str) + call_str = "downlaod the precise Orbit of" + date + print(call_str) + + filename = os.path.basename(SS) + os.remove(tt) + os.remove(tt0) + os.remove(tt00) + os.remove(tt000) + + return filename + + +############################# write & read ##################################### +def copy_file(file0,file1): + call_str = 'cp ' + file0 + ' ' + file1 + os.system(call_str) + + return + +def get_project_slcList(projectName): + scratchDir = os.getenv('SCRATCHDIR') + slcDir = scratchDir + '/' + projectName + "/SLC" + #slcDir = scratchDir + '/SLC' + fn = os.listdir(slcDir) + slc_list0 = [os.path.basename(fname) for fname in sorted(fn)] + #slc_list0 = [os.path.basename(fname) for fname in sorted(glob.glob(slcDir + '/*'))] + + slc_list = [] + for k0 in slc_list0: + if is_number(k0): + slc_list.append(k0) + slc_list = sorted(slc_list) + + return slc_list + +def createBlankFile(strFile): + f = open(strFile,'w') + for i in range (10): + f.write('\n') + f.close() + +def read_attr(fname): + # read hdf5 + with h5py.File(fname, 'r') as f: + atr = dict(f.attrs) + + return atr + +def read_hdf5(fname, datasetName=None, box=None): + # read hdf5 + with h5py.File(fname, 'r') as f: + data = f[datasetName][:] + atr = dict(f.attrs) + + return data, atr + +def get_dataNames(FILE): + with h5py.File(FILE, 'r') as f: + dataNames = [] + for k0 in f.keys(): + dataNames.append(k0) + return dataNames + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def read_gamma_par(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close + + +def write_h5(datasetDict, out_file, metadata=None, ref_file=None, compression=None): + + if os.path.isfile(out_file): + print('delete exsited file: {}'.format(out_file)) + os.remove(out_file) + + print('create HDF5 file: {} with w mode'.format(out_file)) + dt = h5py.special_dtype(vlen=np.dtype('float64')) + + + with h5py.File(out_file, 'w') as f: + for dsName in datasetDict.keys(): + data = datasetDict[dsName] + ds = f.create_dataset(dsName, + data=data, + compression=compression) + + for key, value in metadata.items(): + f.attrs[key] = str(value) + #print(key + ': ' + value) + print('finished writing to {}'.format(out_file)) + + return out_file + +###################################################################### +class progressBar: + """Creates a text-based progress bar. Call the object with + the simple print command to see the progress bar, which looks + something like this: + [=======> 22% ] + You may specify the progress bar's min and max values on init. + + note: + modified from mintPy (https://github.com/insarlab/MintPy/wiki) + Code originally from http://code.activestate.com/recipes/168639/ + + example: + from mintpy.utils import ptime + date12_list = ptime.list_ifgram2date12(ifgram_list) + prog_bar = ptime.progressBar(maxValue=1000, prefix='calculating:') + for i in range(1000): + prog_bar.update(i+1, suffix=date) + prog_bar.update(i+1, suffix=date12_list[i]) + prog_bar.close() + """ + + def __init__(self, maxValue=100, prefix='', minValue=0, totalWidth=70, print_msg=True): + self.prog_bar = "[]" # This holds the progress bar string + self.min = minValue + self.max = maxValue + self.span = maxValue - minValue + self.suffix = '' + self.prefix = prefix + + self.print_msg = print_msg + ## calculate total width based on console width + #rows, columns = os.popen('stty size', 'r').read().split() + #self.width = round(int(columns) * 0.7 / 10) * 10 + self.width = totalWidth + self.reset() + + def reset(self): + self.start_time = time.time() + self.amount = 0 # When amount == max, we are 100% done + self.update_amount(0) # Build progress bar string + + def update_amount(self, newAmount=0, suffix=''): + """ Update the progress bar with the new amount (with min and max + values set at initialization; if it is over or under, it takes the + min or max value as a default. """ + if newAmount < self.min: + newAmount = self.min + if newAmount > self.max: + newAmount = self.max + self.amount = newAmount + + # Figure out the new percent done, round to an integer + diffFromMin = np.float(self.amount - self.min) + percentDone = (diffFromMin / np.float(self.span)) * 100.0 + percentDone = np.int(np.round(percentDone)) + + # Figure out how many hash bars the percentage should be + allFull = self.width - 2 - 18 + numHashes = (percentDone / 100.0) * allFull + numHashes = np.int(np.round(numHashes)) + + # Build a progress bar with an arrow of equal signs; special cases for + # empty and full + if numHashes == 0: + self.prog_bar = '%s[>%s]' % (self.prefix, ' '*(allFull-1)) + elif numHashes == allFull: + self.prog_bar = '%s[%s]' % (self.prefix, '='*allFull) + if suffix: + self.prog_bar += ' %s' % (suffix) + else: + self.prog_bar = '[%s>%s]' % ('='*(numHashes-1), ' '*(allFull-numHashes)) + # figure out where to put the percentage, roughly centered + percentPlace = int(len(self.prog_bar)/2 - len(str(percentDone))) + percentString = ' ' + str(percentDone) + '% ' + # slice the percentage into the bar + self.prog_bar = ''.join([self.prog_bar[0:percentPlace], + percentString, + self.prog_bar[percentPlace+len(percentString):]]) + # prefix and suffix + self.prog_bar = self.prefix + self.prog_bar + if suffix: + self.prog_bar += ' %s' % (suffix) + # time info - elapsed time and estimated remaining time + if percentDone > 0: + elapsed_time = time.time() - self.start_time + self.prog_bar += '%5ds / %5ds' % (int(elapsed_time), + int(elapsed_time * (100./percentDone-1))) + + def update(self, value, every=1, suffix=''): + """ Updates the amount, and writes to stdout. Prints a + carriage return first, so it will overwrite the current + line in stdout.""" + if value % every == 0 or value >= self.max: + self.update_amount(newAmount=value, suffix=suffix) + if self.print_msg: + sys.stdout.write('\r' + self.prog_bar) + sys.stdout.flush() + + def close(self): + """Prints a blank space at the end to ensure proper printing + of future statements.""" + if self.print_msg: + print(' ') diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/_utils_chen.py b/.codex_tmp/pyint_variants/no_rescue/pyint/_utils_chen.py new file mode 100644 index 0000000..dcb69f5 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/_utils_chen.py @@ -0,0 +1,641 @@ +############################################################ +# Program is part of PyINT V2.1 # +# Copyright 2017-2019 Yunmeng Cao # +# Contact: ymcmrs@gmail.com # +############################################################ +#from datetime import datetime +import datetime +import urllib.request +import os +import numpy as np +import random +import h5py +from pathlib import Path +import linecache +import time +import glob + +from tqdm import tqdm +from concurrent.futures import ProcessPoolExecutor, as_completed + +####################### update template ############################# +def update_template(template_file): + + templateDict = {} + + ######### download using SSARA ###### + #templateDict['sensor'] = 'sentinel-1' + #templateDict['track'] = '14' + #templateDict['frame'] = '75' + templateDict['start_time'] = '1989-01-01' + templateDict['end_time'] = '2189-01-01' + + ####### basic parameters for interferometry ###### + templateDict['start_swath'] = '1' + templateDict['end_swath'] = '3' + + templateDict['start_burst'] = '1' + templateDict['end_burst'] = '20' + + templateDict['dem_lat_ovr'] = '0.5' # get 30m resolution of lookup table, 0.5 to 60m, 2 to 15m + templateDict['dem_lon_ovr'] = '0.5' # get 30m resolution of lookup table + + templateDict['Igram_Spsflg'] = '1' # Range spectral filtering + templateDict['Igram_Azfflg'] = '1' # Azimuth common band filtering + + templateDict['rwin4cor'] = '256' # range window length for coregistration + templateDict['azwin4cor'] = '256' # azimuth window length for coregistration + + templateDict['rsample4cor'] = '32' # range samples used for fitting the coregistration parameters + templateDict['azsample4cor'] = '32' # azimuth samples used for fitting the coregistration parameters + + templateDict['thresh4cor'] = '0.15' # 2016 GAMMA or higher version, for 2015 GAMMA or lower version should be SNR + + templateDict['coreCoarse'] = 'both' # initial coregistration method, [options: orbit, ampcor, both] + templateDict['coreMethod'] = 'DEM' # coregistration method [option: DEM, init] DEM means with DEM assistant + + templateDict['Igram_Cor_rwin'] = '5' # used for cc_wave + templateDict['Igram_Cor_awin'] = '5' # used for cc_wave + + templateDict['Igram_Cor_Win'] = '5' # used for adf + templateDict['adf_alpha'] = '0.4' # used for adf + ######## sim phase ################## + templateDict['Igram_Flag_TDM'] = 'N' + templateDict['Simphase_rpos'] = '-' + templateDict['Simphase_azpos'] = '-' + templateDict['Simphase_rwin'] = '256' + templateDict['Simphase_azwin'] = '256' + templateDict['Simphase_thresh'] = '-' + + #### unwrap phase ########### + templateDict['mcf_triangular'] = '0' # triangular type of mcf [0: regular; 1: delaunay;] + templateDict['unwrap_patr'] = '1' + templateDict['unwrap_pataz'] = '1' + templateDict['unwrapThreshold'] = '0.1' # minimum coherence used for unwrap + + #### geocode ######### + templateDict['geo_interp'] = '0' # [0: nearest; 1: bicubic spline] + + ############## interferometry ################ + templateDict['int_flag'] = '1' # 1 means do interferometry + templateDict['diff_flag'] = '1' # differential process, i.e., remove DEM phase + templateDict['unw_flag'] = '1' # unwrap process + templateDict['geo_flag'] = '0' # geocode process + + ############## select network ############# + templateDict['endDate'] = '21000101' + templateDict['startDate'] = '19000101' + templateDict['network_method'] = 'sbas' # sbas, sequential, delaunay, stars + templateDict['conNumb'] = '2' # connect number for sequential + templateDict['max_tb'] = '50000' + templateDict['max_sb'] = '50000' + + ############## time-series ################ + templateDict['download_data'] = '0' # if 1, track, frame, or time informations should be provided + templateDict['down_parallel'] = '1' # multi-processor number used for downloading + + templateDict['raw2slc_all'] = '0' # i.e., download 2 slc + templateDict['raw2slc_all_parallel'] = '1' # multi-processor number used + + templateDict['extract_burst_all'] = '0' # for TOPS SLC only + templateDict['extract_all_parallel'] = '1' # multi-processor number used + + templateDict['coreg_all'] = '1' + templateDict['coreg_all_parallel'] = '1' # multi-processor number used [4 or 8] + + templateDict['select_pairs'] = '1' + + templateDict['diff_all'] = '1' + templateDict['diff_all_parallel'] = '1' # multi-processor number used [4 or 8] + + templateDict['unwrap_all'] = '1' + templateDict['unwrap_all_parallel'] = '1' # multi-processor number used [8 or 10] + + templateDict['geocode_all'] = '0' + templateDict['geocode_all_parallel'] = '1' # multi-processor number used + + templateDict['load_data'] = '0' # loading data for mintpy processing + + templateDict0 = read_template(template_file, delimiter='=') + for key, value in templateDict0.items(): + templateDict[key] = str(value) + + return templateDict + +####################### time-related function ################ + +def is_number(s): + try: + int(s) + return True + except ValueError: + return False + +def print_process_time(start, end): + hours, rem = divmod(end-start, 3600) + minutes, seconds = divmod(rem, 60) + print("Total process time: "+"{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) + return + +def date_add1d(date0): + format_str = '%Y%m%d' # The format + date_obj = datetime.datetime.strptime(date0,format_str) + date1 = (date_obj + datetime.timedelta(days=1)).date().strftime('%Y%m%d') + return date1 + +def date_minus1d(date0): + format_str = '%Y%m%d' # The format + date_obj = datetime.datetime.strptime(date0,format_str) + date1 = (date_obj - datetime.timedelta(days=1)).date().strftime('%Y%m%d') + return date1 + + +def generate_random_name(sufix): + + nowTime = datetime.datetime.now().strftime("%Y%m%d%H%M%S") #generate the present time + randomNum = random.randint(0,100) + randomNum1 = random.randint(0,100) + Nm = nowTime + str(randomNum)+ str(randomNum1) + sufix + + return Nm + +def yyyymmdd2yyyyddd(date): + dt = datetime.datetime.strptime(date, "%Y%m%d") # get datetime object + day_of_year = (dt - datetime.datetime(dt.year, 1, 1)) # Jan the 1st is day 1 + doy = day_of_year.days + 1 + year = dt.year + + doy = str(doy) + if len(doy) ==1: + doy = '00' + doy + elif len(doy) ==2: + doy ='0'+ doy + year = str(year) + + return year, doy + +def get_txt_lines(txt): + count=0 + myfile=open(txt,"r") + for line in myfile: + count=count+1 + + return count + +def read_txt2list(txt): + A = np.loadtxt(txt,dtype=np.str) + if np.array(A).size ==1: + A = [A] + elif isinstance(A[0],bytes): + A = A.astype(str) + A = list(A) + return A + +def read_txt2array(txt): + A = np.loadtxt(txt,dtype=np.str) + if np.array(A).size ==1: + A = [A] + elif isinstance(A[0],bytes): + A = A.astype(str) + #A = list(A) + return A +def get_sardata_swath(start_swath,end_swath): + k_swath = '0' + if (start_swath == '1') and (end_swath == '1'): + k_swath = '1' + elif (start_swath == '2') and (end_swath == '2'): + k_swath = '2' + elif (start_swath == '3') and (end_swath == '3'): + k_swath = '3' + elif (start_swath == '1') and (end_swath == '2'): + k_swath = '4' + elif (start_swath == '2') and (end_swath == '3'): + k_swath = '5' + elif (start_swath == '1') and (end_swath == '3'): + k_swath = '0' + + return k_swath + +def get_filelist_filesize(url0): + ttt = generate_random_name('.txt') + call_str = 'curl -s ' + url0 + ' > ' + ttt + os.system(call_str) + A = read_txt2array(ttt) + + A_size = A[:,4] + A_size = A_size.astype(int) + A_size = A_size/1024/1024 #bytes to Mb + A_name = A[:,8] + if os.path.isfile(ttt): + os.remove(ttt) + + return A_name, A_size + +def yyyymmdd(date0): + if len(date0) ==6: + if float(date0[0:2]) > 90: + date1 = '19' + date0 + else: + date1 = '20' + date0 + elif len(date0) ==8: + date1 = date0 + else: + print('The input date is invalid!!') + date1 = '' + return date1 + +def yymmdd(date0): + if len(date0) ==6: + date1 = date0 + elif len(date0) ==8: + date1 = date0[2:8] + else: + print('The input date is invalid!!') + date1 = '' + return date1 + +def parallel_process(array, function, n_jobs=16, use_kwargs=False): + """ + A parallel version of the map function with a progress bar. + + Args: + array (array-like): An array to iterate over. + function (function): A python function to apply to the elements of array + n_jobs (int, default=16): The number of cores to use + use_kwargs (boolean, default=False): Whether to consider the elements of array as dictionaries of + keyword arguments to function + Returns: + [function(array[0]), function(array[1]), ...] + """ + #We run the first few iterations serially to catch bugs + #If we set n_jobs to 1, just run a list comprehension. This is useful for benchmarking and debugging. + if n_jobs==1: + return [function(**a) if use_kwargs else function(a) for a in tqdm(array[:])] + #Assemble the workers + with ProcessPoolExecutor(max_workers=n_jobs) as pool: + #Pass the elements of array into function + if use_kwargs: + futures = [pool.submit(function, **a) for a in array[:]] + else: + futures = [pool.submit(function, a) for a in array[:]] + kwargs = { + 'total': len(futures), + 'unit': 'it', + 'unit_scale': True, + 'leave': True + } + #Print out the progress as tasks complete + for f in tqdm(as_completed(futures), **kwargs): + pass + out = [] + #Get the results from the futures. + for i, future in tqdm(enumerate(futures)): + try: + out.append(future.result()) + except Exception as e: + out.append(e) + return out + +def sort_unique_list(numb_list): + list_out = sorted(set(numb_list)) + return list_out + + +############################# download data ################################### +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def download_s1_orbit(date,save_path,satellite='A'): + + DATE = date + if len(DATE)==6: + DATE = '20' + DATE + ST = satellite + YEAR = int(DATE[0:4]) + MON = int(DATE[4:6]) + DAY = int(DATE[6:8]) + + MON_DAY = [31,28,31,30,31,30,31,31,30,31,30,31] + + if YEAR%4==0: + MON_DAY[1]=29 + + if MON ==1 and DAY ==1: + DAY0 = 31 + MON0 = 12 + YEAR0 =YEAR -1 + elif MON!=1 and DAY ==1: + DAY0 = MON_DAY[MON-2] + MON0 = MON-1 + YEAR0 = YEAR + else: + DAY0 = DAY -1 + MON0 = MON + YEAR0 = YEAR + + MONDAY0 = MON_DAY[MON0-1] + + TT = [1,4,7,10,13,16,19,22,25,28,MONDAY0] + + T0 = [] + for k in range(len(TT)): + T0.append(TT[k]) + TT[k] = TT[k]-DAY0 + + for k in range(len(TT)): + if k == len(TT)-1: + if TT[k]==0: + ff = k-1 + else: + if TT[k]<=0 and TT[k+1]>0: + ff = k + + + DAY1 = T0[ff] + DAY2 = T0[ff+1] + + S1 = StrNum(YEAR0) + '-' + StrNum(MON0) + S2 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY1) + S3 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY2) + S4 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY0) + + #SS = 'https://qc.sentinel1.eo.esa.int/aux_poeorb/?mission=S1' + ST + '&validity_start_time=' + StrNum(YEAR0) + '&validity_start_time=' + S1 + '&validity_start_time=' + S2 + '..' + S3 + '&validity_start_time=' + S4 + SS = 'https://qc.sentinel1.eo.esa.int/aux_poeorb/?validity_start=' + StrNum(YEAR0) + '&validity_start=' + S1 + '&validity_start=' + S2 + '..' + S3 + '&validity_start=' +S4 + '&sentinel1__mission=S1'+ST+ '&sentinel1_mission=S1'+ST+ '&sentinel1_mission=S1'+ST+ '&sentinel1_mission=S1'+ST + + tt ='tt_orb_' + date + tt0 ='tt0_orb_' + date + tt00 ='tt00_orb_' + date + tt000 ='tt000_orb_' + date + SS = "'" + SS + "'" + #print(SS) + call_str = 'curl -s -l ' + SS + ' > ' + tt + os.system(call_str) + + call_str = "grep 'EOF' -C 0 " + tt + " >" + tt0 + os.system(call_str) + + call_str="awk -F'href=' '{print $2}' " + tt0 +' >' + tt00 + os.system(call_str) + + call_str= "awk -F'>' '{print $1}' " + tt00 + '> ' + tt000 + os.system(call_str) + + SS=linecache.getline(tt000, 1) + SS = SS.split('"')[-1] + + filename = os.path.basename(SS) +# filename = os.path.basename(SS) +# SS1 = save_path + '/' + filename +# if os.path.isfile(SS1): +# if os.path.getsize(SS1) > 4409000: +# call_str = "downlaod the precise Orbit of" + date +# print(call_str) +# else: +# call_str = 'wget -q --no-check-certificate ' + SS + ' -O ' + save_path + '/' +filename +# os.system(call_str) +# call_str = "downlaod the precise Orbit of" + date +# print(call_str) +# else: +# call_str = 'wget -q --no-check-certificate ' + SS + ' -O ' + save_path + '/' +filename +# os.system(call_str) +# call_str = "downlaod the precise Orbit of" + date +# print(call_str) +# if not os.path.isfile(filename): + call_str = 'wget -q --no-check-certificate ' + SS + ' -O ' + save_path + '/' +filename + os.system(call_str) + call_str = "downlaod the precise Orbit of" + date + print(call_str) + + filename = os.path.basename(SS) + os.remove(tt) + os.remove(tt0) + os.remove(tt00) + os.remove(tt000) + + return filename + + +############################# write & read ##################################### +def copy_file(file0,file1): + call_str = 'cp ' + file0 + ' ' + file1 + os.system(call_str) + + return + +def get_project_slcList(projectName): + scratchDir = os.getenv('SCRATCHDIR') + slcDir = scratchDir + '/' + projectName + "/SLC" + #slcDir = scratchDir + '/SLC' + fn = os.listdir(slcDir) + slc_list0 = [os.path.basename(fname) for fname in sorted(fn)] + #slc_list0 = [os.path.basename(fname) for fname in sorted(glob.glob(slcDir + '/*'))] + + slc_list = [] + for k0 in slc_list0: + if is_number(k0): + slc_list.append(k0) + slc_list = sorted(slc_list) + + return slc_list + +def createBlankFile(strFile): + f = open(strFile,'w') + for i in range (10): + f.write('\n') + f.close() + +def read_attr(fname): + # read hdf5 + with h5py.File(fname, 'r') as f: + atr = dict(f.attrs) + + return atr + +def read_hdf5(fname, datasetName=None, box=None): + # read hdf5 + with h5py.File(fname, 'r') as f: + data = f[datasetName][:] + atr = dict(f.attrs) + + return data, atr + +def get_dataNames(FILE): + with h5py.File(FILE, 'r') as f: + dataNames = [] + for k0 in f.keys(): + dataNames.append(k0) + return dataNames + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def read_gamma_par(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close + + +def write_h5(datasetDict, out_file, metadata=None, ref_file=None, compression=None): + + if os.path.isfile(out_file): + print('delete exsited file: {}'.format(out_file)) + os.remove(out_file) + + print('create HDF5 file: {} with w mode'.format(out_file)) + dt = h5py.special_dtype(vlen=np.dtype('float64')) + + + with h5py.File(out_file, 'w') as f: + for dsName in datasetDict.keys(): + data = datasetDict[dsName] + ds = f.create_dataset(dsName, + data=data, + compression=compression) + + for key, value in metadata.items(): + f.attrs[key] = str(value) + #print(key + ': ' + value) + print('finished writing to {}'.format(out_file)) + + return out_file + +###################################################################### +class progressBar: + """Creates a text-based progress bar. Call the object with + the simple print command to see the progress bar, which looks + something like this: + [=======> 22% ] + You may specify the progress bar's min and max values on init. + + note: + modified from mintPy (https://github.com/insarlab/MintPy/wiki) + Code originally from http://code.activestate.com/recipes/168639/ + + example: + from mintpy.utils import ptime + date12_list = ptime.list_ifgram2date12(ifgram_list) + prog_bar = ptime.progressBar(maxValue=1000, prefix='calculating:') + for i in range(1000): + prog_bar.update(i+1, suffix=date) + prog_bar.update(i+1, suffix=date12_list[i]) + prog_bar.close() + """ + + def __init__(self, maxValue=100, prefix='', minValue=0, totalWidth=70, print_msg=True): + self.prog_bar = "[]" # This holds the progress bar string + self.min = minValue + self.max = maxValue + self.span = maxValue - minValue + self.suffix = '' + self.prefix = prefix + + self.print_msg = print_msg + ## calculate total width based on console width + #rows, columns = os.popen('stty size', 'r').read().split() + #self.width = round(int(columns) * 0.7 / 10) * 10 + self.width = totalWidth + self.reset() + + def reset(self): + self.start_time = time.time() + self.amount = 0 # When amount == max, we are 100% done + self.update_amount(0) # Build progress bar string + + def update_amount(self, newAmount=0, suffix=''): + """ Update the progress bar with the new amount (with min and max + values set at initialization; if it is over or under, it takes the + min or max value as a default. """ + if newAmount < self.min: + newAmount = self.min + if newAmount > self.max: + newAmount = self.max + self.amount = newAmount + + # Figure out the new percent done, round to an integer + diffFromMin = np.float(self.amount - self.min) + percentDone = (diffFromMin / np.float(self.span)) * 100.0 + percentDone = np.int(np.round(percentDone)) + + # Figure out how many hash bars the percentage should be + allFull = self.width - 2 - 18 + numHashes = (percentDone / 100.0) * allFull + numHashes = np.int(np.round(numHashes)) + + # Build a progress bar with an arrow of equal signs; special cases for + # empty and full + if numHashes == 0: + self.prog_bar = '%s[>%s]' % (self.prefix, ' '*(allFull-1)) + elif numHashes == allFull: + self.prog_bar = '%s[%s]' % (self.prefix, '='*allFull) + if suffix: + self.prog_bar += ' %s' % (suffix) + else: + self.prog_bar = '[%s>%s]' % ('='*(numHashes-1), ' '*(allFull-numHashes)) + # figure out where to put the percentage, roughly centered + percentPlace = int(len(self.prog_bar)/2 - len(str(percentDone))) + percentString = ' ' + str(percentDone) + '% ' + # slice the percentage into the bar + self.prog_bar = ''.join([self.prog_bar[0:percentPlace], + percentString, + self.prog_bar[percentPlace+len(percentString):]]) + # prefix and suffix + self.prog_bar = self.prefix + self.prog_bar + if suffix: + self.prog_bar += ' %s' % (suffix) + # time info - elapsed time and estimated remaining time + if percentDone > 0: + elapsed_time = time.time() - self.start_time + self.prog_bar += '%5ds / %5ds' % (int(elapsed_time), + int(elapsed_time * (100./percentDone-1))) + + def update(self, value, every=1, suffix=''): + """ Updates the amount, and writes to stdout. Prints a + carriage return first, so it will overwrite the current + line in stdout.""" + if value % every == 0 or value >= self.max: + self.update_amount(newAmount=value, suffix=suffix) + if self.print_msg: + sys.stdout.write('\r' + self.prog_bar) + sys.stdout.flush() + + def close(self): + """Prints a blank space at the end to ensure proper printing + of future statements.""" + if self.print_msg: + print(' ') diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/_utils_old.py b/.codex_tmp/pyint_variants/no_rescue/pyint/_utils_old.py new file mode 100644 index 0000000..2b2cfa5 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/_utils_old.py @@ -0,0 +1,623 @@ +############################################################ +# Program is part of PyINT V2.1 # +# Copyright 2017-2019 Yunmeng Cao # +# Contact: ymcmrs@gmail.com # +############################################################ +#from datetime import datetime +import datetime +import urllib.request +import os +import numpy as np +import random +import h5py +from pathlib import Path +import linecache +import time +import glob + +from tqdm import tqdm +from concurrent.futures import ProcessPoolExecutor, as_completed + +####################### update template ############################# +def update_template(template_file): + + templateDict = {} + + ######### download using SSARA ###### + #templateDict['sensor'] = 'sentinel-1' + #templateDict['track'] = '14' + #templateDict['frame'] = '75' + templateDict['start_time'] = '1989-01-01' + templateDict['end_time'] = '2189-01-01' + + ####### basic parameters for interferometry ###### + templateDict['start_swath'] = '1' + templateDict['end_swath'] = '3' + + templateDict['start_burst'] = '1' + templateDict['end_burst'] = '20' + + templateDict['dem_lat_ovr'] = '0.5' # get 30m resolution of lookup table, 0.5 to 60m, 2 to 15m + templateDict['dem_lon_ovr'] = '0.5' # get 30m resolution of lookup table + + templateDict['Igram_Spsflg'] = '1' # Range spectral filtering + templateDict['Igram_Azfflg'] = '1' # Azimuth common band filtering + + templateDict['rwin4cor'] = '256' # range window length for coregistration + templateDict['azwin4cor'] = '256' # azimuth window length for coregistration + + templateDict['rsample4cor'] = '32' # range samples used for fitting the coregistration parameters + templateDict['azsample4cor'] = '32' # azimuth samples used for fitting the coregistration parameters + + templateDict['thresh4cor'] = '0.15' # 2016 GAMMA or higher version, for 2015 GAMMA or lower version should be SNR + + templateDict['coreCoarse'] = 'both' # initial coregistration method, [options: orbit, ampcor, both] + templateDict['coreMethod'] = 'DEM' # coregistration method [option: DEM, init] DEM means with DEM assistant + + templateDict['Igram_Cor_rwin'] = '5' # used for cc_wave + templateDict['Igram_Cor_awin'] = '5' # used for cc_wave + + templateDict['Igram_Cor_Win'] = '5' # used for adf + templateDict['adf_alpha'] = '0.4' # used for adf + ######## sim phase ################## + templateDict['Igram_Flag_TDM'] = 'N' + templateDict['Simphase_rpos'] = '-' + templateDict['Simphase_azpos'] = '-' + templateDict['Simphase_rwin'] = '256' + templateDict['Simphase_azwin'] = '256' + templateDict['Simphase_thresh'] = '-' + + #### unwrap phase ########### + templateDict['mcf_triangular'] = '0' # triangular type of mcf [0: regular; 1: delaunay;] + templateDict['unwrap_patr'] = '1' + templateDict['unwrap_pataz'] = '1' + templateDict['unwrapThreshold'] = '0.1' # minimum coherence used for unwrap + + #### geocode ######### + templateDict['geo_interp'] = '0' # [0: nearest; 1: bicubic spline] + + ############## interferometry ################ + templateDict['int_flag'] = '1' # 1 means do interferometry + templateDict['diff_flag'] = '1' # differential process, i.e., remove DEM phase + templateDict['unw_flag'] = '1' # unwrap process + templateDict['geo_flag'] = '0' # geocode process + + ############## select network ############# + templateDict['endDate'] = '21000101' + templateDict['startDate'] = '19000101' + templateDict['network_method'] = 'sbas' # sbas, sequential, delaunay, stars + templateDict['conNumb'] = '2' # connect number for sequential + templateDict['max_tb'] = '50000' + templateDict['max_sb'] = '50000' + + ############## time-series ################ + templateDict['download_data'] = '0' # if 1, track, frame, or time informations should be provided + templateDict['down_parallel'] = '1' # multi-processor number used for downloading + + templateDict['raw2slc_all'] = '0' # i.e., download 2 slc + templateDict['raw2slc_all_parallel'] = '1' # multi-processor number used + + templateDict['extract_burst_all'] = '0' # for TOPS SLC only + templateDict['extract_all_parallel'] = '1' # multi-processor number used + + templateDict['coreg_all'] = '1' + templateDict['coreg_all_parallel'] = '1' # multi-processor number used [4 or 8] + + templateDict['select_pairs'] = '1' + + templateDict['diff_all'] = '1' + templateDict['diff_all_parallel'] = '1' # multi-processor number used [4 or 8] + + templateDict['unwrap_all'] = '1' + templateDict['unwrap_all_parallel'] = '1' # multi-processor number used [8 or 10] + + templateDict['geocode_all'] = '0' + templateDict['geocode_all_parallel'] = '1' # multi-processor number used + + templateDict['load_data'] = '0' # loading data for mintpy processing + + templateDict0 = read_template(template_file, delimiter='=') + for key, value in templateDict0.items(): + templateDict[key] = str(value) + + return templateDict + +####################### time-related function ################ + +def is_number(s): + try: + int(s) + return True + except ValueError: + return False + +def print_process_time(start, end): + hours, rem = divmod(end-start, 3600) + minutes, seconds = divmod(rem, 60) + print("Total process time: "+"{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) + return + +def date_add1d(date0): + format_str = '%Y%m%d' # The format + date_obj = datetime.datetime.strptime(date0,format_str) + date1 = (date_obj + datetime.timedelta(days=1)).date().strftime('%Y%m%d') + return date1 + +def date_minus1d(date0): + format_str = '%Y%m%d' # The format + date_obj = datetime.datetime.strptime(date0,format_str) + date1 = (date_obj - datetime.timedelta(days=1)).date().strftime('%Y%m%d') + return date1 + + +def generate_random_name(sufix): + + nowTime = datetime.datetime.now().strftime("%Y%m%d%H%M%S") #generate the present time + randomNum = random.randint(0,100) + randomNum1 = random.randint(0,100) + Nm = nowTime + str(randomNum)+ str(randomNum1) + sufix + + return Nm + +def yyyymmdd2yyyyddd(date): + dt = datetime.datetime.strptime(date, "%Y%m%d") # get datetime object + day_of_year = (dt - datetime.datetime(dt.year, 1, 1)) # Jan the 1st is day 1 + doy = day_of_year.days + 1 + year = dt.year + + doy = str(doy) + if len(doy) ==1: + doy = '00' + doy + elif len(doy) ==2: + doy ='0'+ doy + year = str(year) + + return year, doy + +def get_txt_lines(txt): + count=0 + myfile=open(txt,"r") + for line in myfile: + count=count+1 + + return count + +def read_txt2list(txt): + A = np.loadtxt(txt,dtype=np.str) + if np.array(A).size ==1: + A = [A] + elif isinstance(A[0],bytes): + A = A.astype(str) + A = list(A) + return A + +def read_txt2array(txt): + A = np.loadtxt(txt,dtype=np.str) + if np.array(A).size ==1: + A = [A] + elif isinstance(A[0],bytes): + A = A.astype(str) + #A = list(A) + return A +def get_sardata_swath(start_swath,end_swath): + k_swath = '0' + if (start_swath == '1') and (end_swath == '1'): + k_swath = '1' + elif (start_swath == '2') and (end_swath == '2'): + k_swath = '2' + elif (start_swath == '3') and (end_swath == '3'): + k_swath = '3' + elif (start_swath == '1') and (end_swath == '2'): + k_swath = '4' + elif (start_swath == '2') and (end_swath == '3'): + k_swath = '5' + elif (start_swath == '2') and (end_swath == '3'): + k_swath = '-' + + return k_swath + +def get_filelist_filesize(url0): + ttt = generate_random_name('.txt') + call_str = 'curl -s ' + url0 + ' > ' + ttt + os.system(call_str) + A = read_txt2array(ttt) + + A_size = A[:,4] + A_size = A_size.astype(int) + A_size = A_size/1024/1024 #bytes to Mb + A_name = A[:,8] + if os.path.isfile(ttt): + os.remove(ttt) + + return A_name, A_size + +def yyyymmdd(date0): + if len(date0) ==6: + if float(date0[0:2]) > 90: + date1 = '19' + date0 + else: + date1 = '20' + date0 + elif len(date0) ==8: + date1 = date0 + else: + print('The input date is invalid!!') + date1 = '' + return date1 + +def yymmdd(date0): + if len(date0) ==6: + date1 = date0 + elif len(date0) ==8: + date1 = date0[2:8] + else: + print('The input date is invalid!!') + date1 = '' + return date1 + +def parallel_process(array, function, n_jobs=16, use_kwargs=False): + """ + A parallel version of the map function with a progress bar. + + Args: + array (array-like): An array to iterate over. + function (function): A python function to apply to the elements of array + n_jobs (int, default=16): The number of cores to use + use_kwargs (boolean, default=False): Whether to consider the elements of array as dictionaries of + keyword arguments to function + Returns: + [function(array[0]), function(array[1]), ...] + """ + #We run the first few iterations serially to catch bugs + #If we set n_jobs to 1, just run a list comprehension. This is useful for benchmarking and debugging. + if n_jobs==1: + return [function(**a) if use_kwargs else function(a) for a in tqdm(array[:])] + #Assemble the workers + with ProcessPoolExecutor(max_workers=n_jobs) as pool: + #Pass the elements of array into function + if use_kwargs: + futures = [pool.submit(function, **a) for a in array[:]] + else: + futures = [pool.submit(function, a) for a in array[:]] + kwargs = { + 'total': len(futures), + 'unit': 'it', + 'unit_scale': True, + 'leave': True + } + #Print out the progress as tasks complete + for f in tqdm(as_completed(futures), **kwargs): + pass + out = [] + #Get the results from the futures. + for i, future in tqdm(enumerate(futures)): + try: + out.append(future.result()) + except Exception as e: + out.append(e) + return out + +def sort_unique_list(numb_list): + list_out = sorted(set(numb_list)) + return list_out + + +############################# download data ################################### +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def download_s1_orbit(date,save_path,satellite='A'): + + DATE = date + if len(DATE)==6: + DATE = '20' + DATE + ST = satellite + YEAR = int(DATE[0:4]) + MON = int(DATE[4:6]) + DAY = int(DATE[6:8]) + + MON_DAY = [31,28,31,30,31,30,31,31,30,31,30,31] + + if YEAR%4==0: + MON_DAY[1]=29 + + if MON ==1 and DAY ==1: + DAY0 = 31 + MON0 = 12 + YEAR0 =YEAR -1 + elif MON!=1 and DAY ==1: + DAY0 = MON_DAY[MON-2] + MON0 = MON-1 + YEAR0 = YEAR + else: + DAY0 = DAY -1 + MON0 = MON + YEAR0 = YEAR + + MONDAY0 = MON_DAY[MON0-1] + + TT = [1,4,7,10,13,16,19,22,25,28,MONDAY0] + + T0 = [] + for k in range(len(TT)): + T0.append(TT[k]) + TT[k] = TT[k]-DAY0 + + for k in range(len(TT)): + if k == len(TT)-1: + if TT[k]==0: + ff = k-1 + else: + if TT[k]<=0 and TT[k+1]>0: + ff = k + + + DAY1 = T0[ff] + DAY2 = T0[ff+1] + + S1 = StrNum(YEAR0) + '-' + StrNum(MON0) + S2 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY1) + S3 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY2) + S4 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY0) + + #SS = 'https://qc.sentinel1.eo.esa.int/aux_poeorb/?mission=S1' + ST + '&validity_start_time=' + StrNum(YEAR0) + '&validity_start_time=' + S1 + '&validity_start_time=' + S2 + '..' + S3 + '&validity_start_time=' + S4 + SS = 'https://qc.sentinel1.eo.esa.int/aux_poeorb/?validity_start=' + StrNum(YEAR0) + '&validity_start=' + S1 + '&validity_start=' + S2 + '..' + S3 + '&validity_start=' +S4 + '&sentinel1__mission=S1'+ST+ '&sentinel1_mission=S1'+ST+ '&sentinel1_mission=S1'+ST+ '&sentinel1_mission=S1'+ST + + tt ='tt_orb_' + date + tt0 ='tt0_orb_' + date + tt00 ='tt00_orb_' + date + tt000 ='tt000_orb_' + date + SS = "'" + SS + "'" + #print(SS) + call_str = 'curl -s -l ' + SS + ' > ' + tt + os.system(call_str) + + call_str = "grep 'EOF' -C 0 " + tt + " >" + tt0 + os.system(call_str) + + call_str="awk -F'href=' '{print $2}' " + tt0 +' >' + tt00 + os.system(call_str) + + call_str= "awk -F'>' '{print $1}' " + tt00 + '> ' + tt000 + os.system(call_str) + + SS=linecache.getline(tt000, 1) + SS = SS.split('"')[-1] + filename = os.path.basename(SS) + call_str = 'wget -q --no-check-certificate ' + SS + ' -O ' + save_path + '/' +filename + os.system(call_str) + call_str = "downlaod the precise Orbit of" + date + print(call_str) + + filename = os.path.basename(SS) + os.remove(tt) + os.remove(tt0) + os.remove(tt00) + os.remove(tt000) + + return filename + + +############################# write & read ##################################### +def copy_file(file0,file1): + call_str = 'cp ' + file0 + ' ' + file1 + os.system(call_str) + + return + +def get_project_slcList(projectName): + scratchDir = os.getenv('SCRATCHDIR') + slcDir = scratchDir + '/' + projectName + "/SLC" + #slcDir = scratchDir + '/SLC' + fn = os.listdir(slcDir) + slc_list0 = [os.path.basename(fname) for fname in sorted(fn)] + #slc_list0 = [os.path.basename(fname) for fname in sorted(glob.glob(slcDir + '/*'))] + + slc_list = [] + for k0 in slc_list0: + if is_number(k0): + slc_list.append(k0) + slc_list = sorted(slc_list) + + return slc_list + +def createBlankFile(strFile): + f = open(strFile,'w') + for i in range (10): + f.write('\n') + f.close() + +def read_attr(fname): + # read hdf5 + with h5py.File(fname, 'r') as f: + atr = dict(f.attrs) + + return atr + +def read_hdf5(fname, datasetName=None, box=None): + # read hdf5 + with h5py.File(fname, 'r') as f: + data = f[datasetName][:] + atr = dict(f.attrs) + + return data, atr + +def get_dataNames(FILE): + with h5py.File(FILE, 'r') as f: + dataNames = [] + for k0 in f.keys(): + dataNames.append(k0) + return dataNames + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def read_gamma_par(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close + + +def write_h5(datasetDict, out_file, metadata=None, ref_file=None, compression=None): + + if os.path.isfile(out_file): + print('delete exsited file: {}'.format(out_file)) + os.remove(out_file) + + print('create HDF5 file: {} with w mode'.format(out_file)) + dt = h5py.special_dtype(vlen=np.dtype('float64')) + + + with h5py.File(out_file, 'w') as f: + for dsName in datasetDict.keys(): + data = datasetDict[dsName] + ds = f.create_dataset(dsName, + data=data, + compression=compression) + + for key, value in metadata.items(): + f.attrs[key] = str(value) + #print(key + ': ' + value) + print('finished writing to {}'.format(out_file)) + + return out_file + +###################################################################### +class progressBar: + """Creates a text-based progress bar. Call the object with + the simple print command to see the progress bar, which looks + something like this: + [=======> 22% ] + You may specify the progress bar's min and max values on init. + + note: + modified from mintPy (https://github.com/insarlab/MintPy/wiki) + Code originally from http://code.activestate.com/recipes/168639/ + + example: + from mintpy.utils import ptime + date12_list = ptime.list_ifgram2date12(ifgram_list) + prog_bar = ptime.progressBar(maxValue=1000, prefix='calculating:') + for i in range(1000): + prog_bar.update(i+1, suffix=date) + prog_bar.update(i+1, suffix=date12_list[i]) + prog_bar.close() + """ + + def __init__(self, maxValue=100, prefix='', minValue=0, totalWidth=70, print_msg=True): + self.prog_bar = "[]" # This holds the progress bar string + self.min = minValue + self.max = maxValue + self.span = maxValue - minValue + self.suffix = '' + self.prefix = prefix + + self.print_msg = print_msg + ## calculate total width based on console width + #rows, columns = os.popen('stty size', 'r').read().split() + #self.width = round(int(columns) * 0.7 / 10) * 10 + self.width = totalWidth + self.reset() + + def reset(self): + self.start_time = time.time() + self.amount = 0 # When amount == max, we are 100% done + self.update_amount(0) # Build progress bar string + + def update_amount(self, newAmount=0, suffix=''): + """ Update the progress bar with the new amount (with min and max + values set at initialization; if it is over or under, it takes the + min or max value as a default. """ + if newAmount < self.min: + newAmount = self.min + if newAmount > self.max: + newAmount = self.max + self.amount = newAmount + + # Figure out the new percent done, round to an integer + diffFromMin = np.float(self.amount - self.min) + percentDone = (diffFromMin / np.float(self.span)) * 100.0 + percentDone = np.int(np.round(percentDone)) + + # Figure out how many hash bars the percentage should be + allFull = self.width - 2 - 18 + numHashes = (percentDone / 100.0) * allFull + numHashes = np.int(np.round(numHashes)) + + # Build a progress bar with an arrow of equal signs; special cases for + # empty and full + if numHashes == 0: + self.prog_bar = '%s[>%s]' % (self.prefix, ' '*(allFull-1)) + elif numHashes == allFull: + self.prog_bar = '%s[%s]' % (self.prefix, '='*allFull) + if suffix: + self.prog_bar += ' %s' % (suffix) + else: + self.prog_bar = '[%s>%s]' % ('='*(numHashes-1), ' '*(allFull-numHashes)) + # figure out where to put the percentage, roughly centered + percentPlace = int(len(self.prog_bar)/2 - len(str(percentDone))) + percentString = ' ' + str(percentDone) + '% ' + # slice the percentage into the bar + self.prog_bar = ''.join([self.prog_bar[0:percentPlace], + percentString, + self.prog_bar[percentPlace+len(percentString):]]) + # prefix and suffix + self.prog_bar = self.prefix + self.prog_bar + if suffix: + self.prog_bar += ' %s' % (suffix) + # time info - elapsed time and estimated remaining time + if percentDone > 0: + elapsed_time = time.time() - self.start_time + self.prog_bar += '%5ds / %5ds' % (int(elapsed_time), + int(elapsed_time * (100./percentDone-1))) + + def update(self, value, every=1, suffix=''): + """ Updates the amount, and writes to stdout. Prints a + carriage return first, so it will overwrite the current + line in stdout.""" + if value % every == 0 or value >= self.max: + self.update_amount(newAmount=value, suffix=suffix) + if self.print_msg: + sys.stdout.write('\r' + self.prog_bar) + sys.stdout.flush() + + def close(self): + """Prints a blank space at the end to ensure proper printing + of future statements.""" + if self.print_msg: + print(' ') diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/applygacos.py b/.codex_tmp/pyint_variants/no_rescue/pyint/applygacos.py new file mode 100644 index 0000000..e760869 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/applygacos.py @@ -0,0 +1,305 @@ +#!/usr/bin/env python3 +import os,sys,struct,math +import numpy as np +import argparse +from pyint import _utils as ut +import subprocess + + +#if len(sys.argv) != 5: +# print("Usage:") +# print("./applygacos.py inpfilename ztd1filename ztd2filename elevfilename") +# print(" inpfilenmae : input interferogram, a inpfilename.rsc is needed!") +# print(" ztd1filename: input GACOS ztd day1 filename, a ztd1filename.rsc is needed!") +# print(" ztd2filename: input GACOS ztd day2 filename, a ztd2filename.rsc is needed!") +# print(" elevfilename: elevation angle file, must be the same size with interferogram") +# print(" elevation=90-incidence ") +# print(" can be generated by look_vector in gamma") +# print("outputfile will be saved as inpfilename.gacos") +# print("!!Note, python3 is needed!!") +# exit() + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister SM mode SLC to a reference SLC image using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='Name of project.') + parser.add_argument('mdate', help='date of the slave SLC image') + parser.add_argument('sdate', help='date of the slave SLC image') + parser.add_argument('ztd1', help='input GACOS ztd day1 filename, a ztd2filename.rsc is needed!') + parser.add_argument('ztd2', help='input GACOS ztd day2 filename, a ztd2filename.rsc is needed!' ) + inps = parser.parse_args() + return inps + +INTRODUCTION = ''' +------------------------------------------------------------------- + create_files for psokinv software running file by generated by Gamma +''' + +EXAMPLE = """Usage: + + coreg_gamma.py projectName + + coreg_gamma.py PacayaT163TsxHhA +------------------------------------------------------------------- +""" + + + + +#filename=sys.argv[1] +#ztd1filename=sys.argv[2] +#ztd2filename=sys.argv[3] +#elevfilename=sys.argv[4] +#print("start processing "+ filename) + +class HEADER: + width = 0 + length = 0 + xfirst = 0.0 + yfirst = 0.0 + xstep = 0.0 + ystep = 0.0 + + + + + +def read_header(filename): + if not os.path.isfile(filename): + print(filename+" file not exit") + + header = HEADER() + with open(filename) as f: + for line in f: + data=line.split() + if data[0] == "WIDTH": + header.width=int(data[1]) + if data[0] == "FILE_LENGTH": + header.length=int(data[1]) + if data[0] == "X_FIRST": + header.xfirst=float(data[1]) + if data[0] == "Y_FIRST": + header.yfirst=float(data[1]) + if data[0] == "X_STEP": + header.xstep=float(data[1]) + if data[0] == "Y_STEP": + header.ystep=float(data[1]) + + return header + + +def cut_image2(filename,headername,yfirst_new,length_new,xfirst_new,width_new): + header=read_header(headername) + + #print(yfirst_new, header.yfirst) + #print(length_new, header.length) + #print(xfirst_new, header.xfirst) + #print(width_new, header.width) + + with open(filename, 'rb') as f: + data0 = np.fromfile(f, dtype=np.float32) + data = np.reshape(data0, (header.length, header.width)) + + + out=np.zeros((length_new,width_new),dtype=np.float32) + #out=np.zeros(width_new*length_new,dtype=np.float32) + for i in range(header.length): + lat=header.yfirst+header.ystep*i + row=int(round((lat-yfirst_new)/header.ystep)) + if(row<0 or row>=length_new): + continue + for j in range(header.width): + lon=header.xfirst+header.xstep*j + col=int(round((lon-xfirst_new)/header.xstep)) + if(col<0 or col>=width_new): + continue + #out[width_new*row+col]=data[header.width*i+j] + out[row,col]=data[i,j] + + out=np.where(out==0,np.nan,out) + #print(np.nanstd(out)) + out.tofile(filename+".cut") + f = open(filename+".cut.rsc", 'w') + f.write("WIDTH "+str(width_new)+"\n") + f.write("FILE_LENGTH "+str(length_new)+"\n") + f.write("X_FIRST "+ str(xfirst_new)+"\n") + f.write("Y_FIRST "+ str(yfirst_new)+"\n") + f.write("X_STEP "+ str(header.xstep)+"\n") + f.write("Y_STEP "+ str(header.ystep)+"\n") + f.close() + + + + +def make_correction(phsfilename,ztd1filename,ztd2filename,elevfilename): + header=read_header(phsfilename+".rsc") + if not os.path.isfile(ztd1filename+".cut"): + cut_image2(ztd1filename,ztd1filename+".rsc",header.yfirst,header.length,header.xfirst,header.width) + if not os.path.isfile(ztd2filename+".cut"): + cut_image2(ztd2filename,ztd2filename+".rsc",header.yfirst,header.length,header.xfirst,header.width) + + with open(phsfilename, 'rb') as f: + data = np.fromfile(f, dtype=np.float32) + phase = np.reshape(data, [header.length, header.width]) + + with open(ztd1filename+".cut", 'rb') as f: + data = np.fromfile(f, dtype=np.float32) + ztd1 = np.reshape(data, [header.length, header.width]) + + with open(ztd2filename+".cut", 'rb') as f: + data = np.fromfile(f, dtype=np.float32) + ztd2 = np.reshape(data, [header.length, header.width]) + + with open(elevfilename, 'rb') as f: + data = np.fromfile(f, dtype=np.float32) + elev = np.reshape(data, [header.length, header.width]) + + #os.remove(ztd1filename+".cut") + #os.remove(ztd2filename+".cut") + dztd=ztd2-ztd1 + dztd=dztd/0.0044138251819503 + dztd=dztd/np.sin(elev) + index=np.where(phase==0) + phase[index]=np.nan + phasemean=np.nanmean(phase) + print("before "+str(np.nanstd(phase))) + phase=phase-phasemean + #phase.tofile(phsfilename+".raw") + + + phase=phase-dztd + phase[index]=np.nan + phasemean=np.nanmean(phase) + print("after "+str(np.nanstd(phase))) + phase=phase-phasemean + phase[index]=0 + phase.tofile(phsfilename+".gacos") + +def main(argv): + inps = cmdLineParse() + projectName = inps.projectName + Sdate = inps.sdate + Mdate = inps.mdate + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Smdate = templateDict['masterDate'] + + demDir = scratchDir + '/' + projectName + '/DEM' + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + ifgramDir = scratchDir + '/' + projectName + "/ifgrams" + workDir = ifgramDir + '/' + Mdate + '-' + Sdate + MslcDir = rslcDir + '/' + Mdate + SslcDir = rslcDir + '/' + Sdate + offpar = SslcDir + '/' + Smdate + '_' + Sdate + '.off' + slcpar = MslcDir + '/' + Smdate + '.rslc.par' + dempar = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem.par' + dem = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem' + unwpar = demDir + '/' + Mdate + '_' + rlks + 'rlks.utm.dem.par' + unw = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.diff_filt.unw" + + os.chdir(workDir) + call_str = "look_vector " + slcpar+ " " + offpar + " " + dempar + " " + dem + ' lv_theta lv_phi' + os.system(call_str) + call_str = "grep 'corner_lat:' " + dempar + " | awk '{print $2}' " + North=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'corner_lon:' " + dempar + " | awk '{print $2}' " + West=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lat:' " + dempar + " | awk '{print $2}' " + posty=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lon:' " + dempar + " | awk '{print $2}' " + postx=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'width:' " + dempar + " | awk '{print $2}' " + width=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'nlines:' " + dempar + " | awk '{print $2}' " + length=subprocess.getstatusoutput(call_str)[1] + South=str(round(float(North)+(float(length)-1)*float(posty),7)) + East=str(round(float(West)+(float(width)-1)*float(postx),7)) + + call_str = "swap_bytes lv_theta lv_theta.phase_swap 4 >dinsar.log " + os.system(call_str) + call_str = "gmt xyz2grd lv_theta.phase_swap -Glv_theta.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -N0" + os.system(call_str) + call_str = "gmt grdmath 90 lv_theta.grd 3.1415926 DIV 180 MUL SUB = lv_theta_final.grd" + os.system(call_str) + call_str = "gmt grdmath 90 lv_theta_final.grd SUB = lv_elev.grd" + os.system(call_str) + call_str = "gmt grdsample lv_elev.grd -Glv_elev_3c.grd -I3c" + os.system(call_str) + call_str = "gmt grd2xyz lv_elev_3c.grd -ZTLf -N0 > " + Mdate + '-' + Sdate + '.gacos.elev' + os.system(call_str) + + + call_str = "swap_bytes " + unw+ " unw.phase_swap 4 >dinsar.log " + os.system(call_str) + call_str = "gmt xyz2grd unw.phase_swap -Gunw_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx +" -ZTLf -N0" + os.system(call_str) + call_str = "gmt grdsample unw_f.grd -Gunw.grd -I3c" + os.system(call_str) + call_str = "gmt grd2xyz unw.grd -ZTLf > " + Mdate + '-' + Sdate + '.gacos.unw' + os.system(call_str) + call_str = "gmt grdinfo unw.grd -C | awk '{print $10}' " + Width=subprocess.getstatusoutput(call_str)[1] + call_str = "gmt grdinfo unw.grd -C | awk '{print $11}' " + line=subprocess.getstatusoutput(call_str)[1] + call_str = "gmt grdinfo unw.grd -C | awk '{print $2}' " + West=subprocess.getstatusoutput(call_str)[1] + call_str = "gmt grdinfo unw.grd -C | awk '{print $5}' " + North=subprocess.getstatusoutput(call_str)[1] + ymax=str(int(round(float(line) ,7))) + xmax=str(int(round(float(Width) ,7))) + output=workDir + '/' + Mdate + '-' + Sdate + '.gacos.unw.rsc' + if os.path.exists(output) is True: + os.remove(output) + + fopen=open(output,'a+') + fopen.write('WIDTH ' + Width + '\n') + fopen.write('FILE_LENGTH ' + line + '\n') + fopen.write('XMIN 1' + '\n') + fopen.write('XMAX ' + xmax + '\n') + fopen.write('YMIN 1' + '\n') + fopen.write('YMAX ' + ymax + '\n') + fopen.write('X_FIRST ' + West + '\n') + fopen.write('Y_FIRST ' + North + '\n') + fopen.write('X_STEP 8.33333333E-04' + '\n') + fopen.write('Y_STEP -8.33333333E-04' + '\n') + fopen.write('X_UNIT degrees' + '\n') + fopen.write('Y_UNIT degrees' + '\n') + fopen.write('Z_OFFSET 0' + '\n') + fopen.write('Z_SCALE 1' + '\n') + fopen.write('PROJECTION LATLON' + '\n') + fopen.write('DATUM WGS84' + '\n') + print("convert gamma products to GACOS is done!") + + print("start make gacos correction!") + filename=workDir + '/' + Mdate + '-' + Sdate + '.gacos.unw' + ztd1=inps.ztd1 + ztd2=inps.ztd2 + ztd1filename=workDir + '/' + ztd1 + ztd2filename=workDir + '/' + ztd2 + elevfilename=workDir + '/' + Mdate + '-' + Sdate + '.gacos.elev' + make_correction(filename,ztd1filename,ztd2filename,elevfilename) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + +#make_correction(filename,ztd1filename,ztd2filename,elevfilename) + + + + +#exit() + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/applygacos1.py b/.codex_tmp/pyint_variants/no_rescue/pyint/applygacos1.py new file mode 100644 index 0000000..e8e48e0 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/applygacos1.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +import os,sys,struct,math +import numpy as np +import argparse +from pyint import _utils as ut +import subprocess + + +#if len(sys.argv) != 5: +# print("Usage:") +# print("./applygacos.py inpfilename ztd1filename ztd2filename elevfilename") +# print(" inpfilenmae : input interferogram, a inpfilename.rsc is needed!") +# print(" ztd1filename: input GACOS ztd day1 filename, a ztd1filename.rsc is needed!") +# print(" ztd2filename: input GACOS ztd day2 filename, a ztd2filename.rsc is needed!") +# print(" elevfilename: elevation angle file, must be the same size with interferogram") +# print(" elevation=90-incidence ") +# print(" can be generated by look_vector in gamma") +# print("outputfile will be saved as inpfilename.gacos") +# print("!!Note, python3 is needed!!") +# exit() + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister SM mode SLC to a reference SLC image using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='Name of project.') + parser.add_argument('mdate', help='date of the slave SLC image') + parser.add_argument('sdate', help='date of the slave SLC image') + parser.add_argument('ztd1', help='input GACOS ztd day1 filename, a ztd2filename.rsc is needed!') + parser.add_argument('ztd2', help='input GACOS ztd day2 filename, a ztd2filename.rsc is needed!' ) + inps = parser.parse_args() + return inps + +INTRODUCTION = ''' +------------------------------------------------------------------- + create_files for psokinv software running file by generated by Gamma +''' + +EXAMPLE = """Usage: + + coreg_gamma.py projectName + + coreg_gamma.py PacayaT163TsxHhA +------------------------------------------------------------------- +""" + + + + +#filename=sys.argv[1] +#ztd1filename=sys.argv[2] +#ztd2filename=sys.argv[3] +#elevfilename=sys.argv[4] +#print("start processing "+ filename) + +class HEADER: + width = 0 + length = 0 + xfirst = 0.0 + yfirst = 0.0 + xstep = 0.0 + ystep = 0.0 + + + + + +def read_header(filename): + if not os.path.isfile(filename): + print(filename+" file not exit") + + header = HEADER() + with open(filename) as f: + for line in f: + data=line.split() + if data[0] == "WIDTH": + header.width=int(data[1]) + if data[0] == "FILE_LENGTH": + header.length=int(data[1]) + if data[0] == "X_FIRST": + header.xfirst=float(data[1]) + if data[0] == "Y_FIRST": + header.yfirst=float(data[1]) + if data[0] == "X_STEP": + header.xstep=float(data[1]) + if data[0] == "Y_STEP": + header.ystep=float(data[1]) + + return header + + +def cut_image2(filename,headername,yfirst_new,length_new,xfirst_new,width_new): + header=read_header(headername) + + #print(yfirst_new, header.yfirst) + #print(length_new, header.length) + #print(xfirst_new, header.xfirst) + #print(width_new, header.width) + + with open(filename, 'rb') as f: + data0 = np.fromfile(f, dtype=np.float32) + data = np.reshape(data0, (header.length, header.width)) + + + out=np.zeros((length_new,width_new),dtype=np.float32) + #out=np.zeros(width_new*length_new,dtype=np.float32) + for i in range(header.length): + lat=header.yfirst+header.ystep*i + row=int(round((lat-yfirst_new)/header.ystep)) + if(row<0 or row>=length_new): + continue + for j in range(header.width): + lon=header.xfirst+header.xstep*j + col=int(round((lon-xfirst_new)/header.xstep)) + if(col<0 or col>=width_new): + continue + #out[width_new*row+col]=data[header.width*i+j] + out[row,col]=data[i,j] + + out=np.where(out==0,np.nan,out) + #print(np.nanstd(out)) + out.tofile(filename+".cut") + f = open(filename+".cut.rsc", 'w') + f.write("WIDTH "+str(width_new)+"\n") + f.write("FILE_LENGTH "+str(length_new)+"\n") + f.write("X_FIRST "+ str(xfirst_new)+"\n") + f.write("Y_FIRST "+ str(yfirst_new)+"\n") + f.write("X_STEP "+ str(header.xstep)+"\n") + f.write("Y_STEP "+ str(header.ystep)+"\n") + f.write("X_UNIT degree"+"\n") + f.write("Y_UNIT degree"+"\n") + f.write("Z_OFFSET 0"+"\n") + f.write("Z_SCALE 1"+"\n") + f.write("PROJECTION LATLON"+"\n") + f.write("DATUM WGS84"+"\n") + f.close() + +def make_correction(phsfilename,ztd1filename,ztd2filename,elevfilename): + header=read_header(phsfilename+".rsc") + #if not os.path.isfile(ztd1filename+".cut"): + cut_image2(ztd1filename,ztd1filename+".rsc",header.yfirst,header.length,header.xfirst,header.width) + #if not os.path.isfile(ztd2filename+".cut"): + cut_image2(ztd2filename,ztd2filename+".rsc",header.yfirst,header.length,header.xfirst,header.width) + + with open(phsfilename, 'rb') as f: + data = np.fromfile(f, dtype=np.float32) + phase = np.reshape(data, [header.length, header.width]) + + with open(ztd1filename+".cut", 'rb') as f: + data = np.fromfile(f, dtype=np.float32) + ztd1 = np.reshape(data, [header.length, header.width]) + + with open(ztd2filename+".cut", 'rb') as f: + data = np.fromfile(f, dtype=np.float32) + ztd2 = np.reshape(data, [header.length, header.width]) + + with open(elevfilename, 'rb') as f: + data = np.fromfile(f, dtype=np.float32) + elev = np.reshape(data, [header.length, header.width]) + + #os.remove(ztd1filename+".cut") + #os.remove(ztd2filename+".cut") + dztd=ztd2-ztd1 + #dztd=dztd/0.0044138251819503 + dztd=dztd/np.sin(elev) + index=np.where(phase==0) + phase[index]=np.nan + phasemean=np.nanmean(phase) + print("before "+str(np.nanstd(phase))) + phase=phase-phasemean + #phase.tofile(phsfilename+".raw") + + + phase=phase-dztd + phase[index]=np.nan + phasemean=np.nanmean(phase) + print("after "+str(np.nanstd(phase))) + phase=phase-phasemean + phase[index]=0 + phase.tofile(phsfilename+".gacos") + +def main(argv): + inps = cmdLineParse() + projectName = inps.projectName + Sdate = inps.sdate + Mdate = inps.mdate + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Smdate = templateDict['masterDate'] + + demDir = scratchDir + '/' + projectName + '/DEM' + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + ifgramDir = scratchDir + '/' + projectName + "/ifgrams" + workDir = ifgramDir + '/' + Mdate + '-' + Sdate + MslcDir = rslcDir + '/' + Mdate + SslcDir = rslcDir + '/' + Sdate + offpar = SslcDir + '/' + Smdate + '_' + Sdate + '.off' + slcpar = MslcDir + '/' + Smdate + '.rslc.par' + dempar = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem.par' + dem = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem' + unwpar = demDir + '/' + Mdate + '_' + rlks + 'rlks.utm.dem.par' + unw = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.diff_filt.unw" + + os.chdir(workDir) + call_str = "look_vector " + slcpar+ " " + offpar + " " + dempar + " " + dem + ' lv_theta lv_phi' + os.system(call_str) + call_str = "grep 'corner_lat:' " + dempar + " | awk '{print $2}' " + North=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'corner_lon:' " + dempar + " | awk '{print $2}' " + West=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lat:' " + dempar + " | awk '{print $2}' " + posty=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lon:' " + dempar + " | awk '{print $2}' " + postx=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'width:' " + dempar + " | awk '{print $2}' " + width=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'nlines:' " + dempar + " | awk '{print $2}' " + length=subprocess.getstatusoutput(call_str)[1] + South=str(round(float(North)+(float(length)-1)*float(posty),7)) + East=str(round(float(West)+(float(width)-1)*float(postx),7)) + + call_str = "swap_bytes lv_theta lv_theta.phase_swap 4 >dinsar.log " + os.system(call_str) + call_str = "xyz2grd lv_theta.phase_swap -Glv_theta.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -N0" + os.system(call_str) + call_str = "grdmath 90 lv_theta.grd 3.1415926 DIV 180 MUL SUB = lv_theta_final.grd" + os.system(call_str) + call_str = "grdmath 90 lv_theta_final.grd SUB = lv_elev.grd" + os.system(call_str) + call_str = "grdsample lv_elev.grd -Glv_elev_3c.grd -I3c" + # call_str = "grdsample lv_theta_final.grd -Glv_elev_3c.grd -I3c" + os.system(call_str) + call_str = "grd2xyz lv_elev_3c.grd -ZTLf -N0 > " + Mdate + '-' + Sdate + '.gacos.elev' + os.system(call_str) + + + call_str = "swap_bytes " + unw+ " unw.phase_swap 4 >dinsar.log " + os.system(call_str) + call_str = "xyz2grd unw.phase_swap -Gunw_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx +" -ZTLf -N0" + os.system(call_str) + call_str = "grdsample unw_f.grd -Gunw.grd -I3c" + os.system(call_str) + call_str = "grd2xyz unw.grd -ZTLf -N0 > " + Mdate + '-' + Sdate + '.gacos.unw' + os.system(call_str) + call_str = "grdinfo unw.grd -C | awk '{print $10}' " + Width=subprocess.getstatusoutput(call_str)[1] + call_str = "grdinfo unw.grd -C | awk '{print $11}' " + line=subprocess.getstatusoutput(call_str)[1] + call_str = "grdinfo unw.grd -C | awk '{print $2}' " + West=subprocess.getstatusoutput(call_str)[1] + call_str = "grdinfo unw.grd -C | awk '{print $5}' " + North=subprocess.getstatusoutput(call_str)[1] + ymax=str(int(round(float(line) ,7))) + xmax=str(int(round(float(Width) ,7))) + output=workDir + '/' + Mdate + '-' + Sdate + '.gacos.unw.rsc' + if os.path.exists(output) is True: + os.remove(output) + + fopen=open(output,'a+') + fopen.write('WIDTH ' + Width + '\n') + fopen.write('FILE_LENGTH ' + line + '\n') + fopen.write('XMIN 1' + '\n') + fopen.write('XMAX ' + xmax + '\n') + fopen.write('YMIN 1' + '\n') + fopen.write('YMAX ' + ymax + '\n') + fopen.write('X_FIRST ' + West + '\n') + fopen.write('Y_FIRST ' + North + '\n') + fopen.write('X_STEP 8.33333333E-04' + '\n') + fopen.write('Y_STEP -8.33333333E-04' + '\n') + fopen.write('X_UNIT degrees' + '\n') + fopen.write('Y_UNIT degrees' + '\n') + fopen.write('Z_OFFSET 0' + '\n') + fopen.write('Z_SCALE 1' + '\n') + fopen.write('PROJECTION LATLON' + '\n') + fopen.write('DATUM WGS84' + '\n') + fopen.close() + print("convert gamma products to GACOS is done!") + + print("start make gacos correction!") + filename=workDir + '/' + Mdate + '-' + Sdate + '.gacos.unw' + ztd1=inps.ztd1 + ztd2=inps.ztd2 + ztd1filename=workDir + '/' + ztd1 + ztd2filename=workDir + '/' + ztd2 + elevfilename=workDir + '/' + Mdate + '-' + Sdate + '.gacos.elev' + make_correction(filename,ztd1filename,ztd2filename,elevfilename) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + +#make_correction(filename,ztd1filename,ztd2filename,elevfilename) + + + + +#exit() + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/atm_correction_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/atm_correction_gamma.py new file mode 100644 index 0000000..169bad0 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/atm_correction_gamma.py @@ -0,0 +1,105 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +from PIL import Image +from pylab import * +import argparse + +from pyint import _utils as ut + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Unwrap differential interferogram using GAMMA. + [Only support mcf, not implement branch_cut yet] + +''' + +EXAMPLE = ''' + Usage: + unwrap_gamma.py projectName Mdate Sdate + unwrap_gamma.py PacayaT163TsxHhA 20150102 20150601 +------------------------------------------------------------------- +''' + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Unwrap differential interferogram using GAMMA-mcf method.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('Mdate',help='Master date.') + parser.add_argument('Sdate',help='Slave date.') + + inps = parser.parse_args() + return inps + + +def main(argv): + + inps = cmdLineParse() + Mdate = inps.Mdate + Sdate = inps.Sdate + + projectName = inps.projectName + Sdate = inps.Sdate + Mdate = inps.Mdate + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + auto_unw = templateDict['auto_unw'] + init_flag = templateDict['init_flag'] + r_refer = templateDict['r_refer'] + a_refer = templateDict['a_refer'] + make_mask = templateDict['make_mask'] + processDir = scratchDir + '/' + projectName + "/PROCESS" + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + '/RSLC' + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + masterDate = templateDict['masterDate'] + demDir = scratchDir + '/' + projectName + '/DEM' + Pair = Mdate + '-' + Sdate + workDir = ifgDir + '/' + Pair + + ################ prepare file for parallel processing ############### + HGTSIM = demDir + '/' + masterDate + '_' + rlks + 'rlks.rdc.dem' + Mamp = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + Samp = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + diff_par = workDir + '/' + Mdate + '_' + Sdate + '_diff_par' + UNWlks = workDir + '/' + Pair + '_' +rlks + 'rlks.diff_filt.unw' + CORMASK = workDir + '/' + Pair + '_' +rlks + 'rlks.diff_filt.cor' + ATM_PHASE = workDir + '/' + Pair + '_' +rlks + 'rlks.atm_phae' + ATMCOR_UNW = workDir + '/' + Pair + '_' +rlks + 'rlks.diff_filt.atmcor.unw' + + nWidth = ut.read_gamma_par(MampPar, 'read', 'range_samples') + nLine = ut.read_gamma_par(MampPar, 'read', 'azimuth_lines') + ############################################################### + call_str = "create_diff_par " + MampPar + " " + SampPar + " " + diff_par + " 1 0 " + os.system(call_str) + call_str = "atm_mod_2d " + UNWlks + " " + HGTSIM + " " + CORMASK + " " + diff_par + " " + " - 0 a0 a1 sigma sigma_h s1" + os.system(call_str) + call_str = "atm_sim_2d " + diff_par + " " + HGTSIM + " a0 a1 " + ATM_PHASE + os.system(call_str) + call_str = "sub_phase " + UNWlks + " " + ATM_PHASE + " " + diff_par + " " + ATMCOR_UNW + " 0 0 0 " + os.system(call_str) + call_str = 'rasrmg ' + ATMCOR_UNW + ' ' + Mamp + ' ' + nWidth + ' - - - - - - - - - - ' + os.system(call_str) + + print("Correct atmospheric phase is done!") + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/atm_correction_gamma_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/atm_correction_gamma_all.py new file mode 100644 index 0000000..5ad8a59 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/atm_correction_gamma_all.py @@ -0,0 +1,121 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + + +def work(data0): + cmd = data0[0] + err_file = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + cmd[3] + '\n' + #print(aa) + with open(err_file, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + Correct atmospheric phase for one project using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + atm_correction_gamma_all.py projectName + atm_correction_gamma_all.py projectName --parallel 4 + atm_correction_gamma_all.py projectName --parallel 4 --ifgramList-txt /test/ifgram_list.txt +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Unwrap differential interferograms for one project using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + parser.add_argument('--ifgarmList-txt', dest='ifgarmListTxt', help='provided ifgram_list_txt. default: using ifgram_list.txt under projectName folder.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + projectDir = scratchDir + '/' + projectName + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + if inps.ifgarmListTxt: ifgramList_txt = inps.ifgarmListTxt + else: ifgramList_txt = scratchDir + '/' + projectName + '/ifgram_list.txt' + ifgList0 = ut.read_txt2array(ifgramList_txt) +# ifgList = ifgList0[:,0] + if len(ifgList0)==3: + ifgList=ifgList0[0] + ifgList=[ifgList] + else: + ifgList=ifgList0[:,0] + + err_txt = scratchDir + '/' + projectName + '/unwrap_gamma_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + for i in range(len(ifgList)): + m0 = ut.yyyymmdd(ifgList[i].split('-')[0]) + s0 = ut.yyyymmdd(ifgList[i].split('-')[1]) + cmd0 = ['atm_correction_gamma.py',projectName, m0, s0] + unw_file0 = ifgDir + '/' + ifgList[i] + '/' + ifgList[i] + '_' + rlks + 'rlks.diff_filt.atmcor.unw.bmp' + data0 = [cmd0,err_txt] + + k00 = 0 + if os.path.isfile(unw_file0): + if os.path.getsize(unw_file0) > 0: + k00 = 1 + if k00==0: + data_para.append(data0) + + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + print("Correct atmospheric phase for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/change_Name_for_mintpy.py b/.codex_tmp/pyint_variants/no_rescue/pyint/change_Name_for_mintpy.py new file mode 100644 index 0000000..271157f --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/change_Name_for_mintpy.py @@ -0,0 +1,135 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Central South University & University of Miami ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + + +def is_number(s): + try: + int(s) + return True + except ValueError: + return False + +def add_zero(s): + if len(s)==1: + s="000"+s + elif len(s)==2: + s="00"+s + elif len(s)==3: + s="0"+s + return s + +def usage(): + print(''' +****************************************************************************************************** + + Select interferometry pairs from time series SAR images + + usage: + + Generate_IfgDir.py ProjectName IFG_List + + e.g. Generate_IfgDir.py PacayaT163TsxHhA ifg_list + + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==2: + if argv[0] in ['-h','--help']: usage(); sys.exit(1) + else: + projectName=sys.argv[1] + projectName=sys.argv[1] + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + processDir = scratchDir + '/' + projectName + "/ifgrams" + root_dir=os.getcwd() + os.chdir(processDir) + call_str="find . -name '[0-9]*.unw' -print >unw_file" + os.system(call_str) + file_obj=open('unw_file') + all_lines=file_obj.readlines() + for line in all_lines: + unwdate1=line.split("/")[2] + unwdate2=line.split("/")[1] + unwdate3=line.split("/")[0] + str1=unwdate1.split(".")[0] + str2=unwdate1.split(".")[1] + str3=unwdate1.split(".")[2] + unwdate4=str2 + '_' + str1 + '.' + str3 + line1=processDir + '/' + unwdate2+ '/' + unwdate4 + line2=processDir + '/'+ unwdate2 + '/' + unwdate1 + line2=line2.split('\n')[0] + line1=line1.split('\n')[0] + call_str='mv'+ ' ' + line2 + ' ' + line1 + os.system(call_str) + call_str="find . -name '[0-9]*.cor' -print >cor_file" + os.system(call_str) + + file_obj=open('cor_file') + all_lines=file_obj.readlines() + for line in all_lines: + cordate1=line.split("/")[2] + cordate2=line.split("/")[1] + cordate3=line.split("/")[0] + str1=cordate1.split(".")[0] + str2=cordate1.split(".")[1] + str3=cordate1.split(".")[2] + sub1=str2.split('_')[1] + cordate4=sub1 + '_' + str1 + '.' + str3 + line1=processDir + '/' + cordate2 + '/' + cordate4 + line1=line1.split('\n')[0] + line2=processDir + '/'+ cordate2 + '/' + cordate1 + line2=line2.split('\n')[0] + call_str='mv' + ' ' + line2 + ' ' + line1 + os.system(call_str) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/convert_phs_to_grd.csh b/.codex_tmp/pyint_variants/no_rescue/pyint/convert_phs_to_grd.csh new file mode 100644 index 0000000..ec8da05 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/convert_phs_to_grd.csh @@ -0,0 +1,23 @@ +#!/bin/csh -f + +if ($#argv != 2) then + echo "" + echo "Usage: read_phs_file.csh phase_file header.rsc" + echo "" + echo "Convert binary files to .grd file" + echo "" + echo "binary file and header in .rsc is needed" + exit 1 + endif + +#######File to grid######### +set x_first_d1 = `cat $2|grep X_FIRST|awk '{print $2}'` +set y_first_d1 = `cat $2|grep Y_FIRST|awk '{print $2}'` +set width_d1 = `cat $2|grep WIDTH|awk '{print $2}'` +set length_d1 = `cat $2|grep FILE_LENGTH|awk '{print $2}'` +set x_step_d1 = `cat $2|grep X_STEP|awk '{print $2}'` +set y_step_d1 = `cat $2|grep X_STEP|awk '{print $2}'` +set name = `echo $1| awk -F/ '{print $NF}'` +gmt xyz2grd $1 -G"$name.grd" -RLT$x_first_d1/$y_first_d1/$width_d1/$length_d1 -I$x_step_d1/$y_step_d1 -ZTLf -di0 -r + +echo "From $1, $name.grd has been produced" diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_gamma.py new file mode 100644 index 0000000..49adcad --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_gamma.py @@ -0,0 +1,242 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import argparse + +from pyint import _utils as ut + + +def _run_or_raise(call_str, stage): + rc = os.system(call_str) + if rc != 0: + raise RuntimeError('%s failed with rc=%s: %s' % (stage, rc, call_str)) + return rc + + +def _safe_remove(path): + if path and os.path.exists(path): + os.remove(path) + + +def _copy_file(src, dst): + ut.copy_file(src, dst) + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister SM mode SLC to a reference SLC image using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='Name of project.') + parser.add_argument('sdate', help='date of the slave SLC image. [mater date is read from template]') + inps = parser.parse_args() + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Coregister SM mode SLC to a reference SLC image using GAMMA. + [The reference date or master date will be read from the template file.] +''' + +EXAMPLE = """Usage: + + coreg_gamma.py projectName Sdate + + coreg_gamma.py PacayaT163TsxHhA 20150102 +------------------------------------------------------------------- +""" + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Sdate = inps.sdate + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + if not os.path.isdir(rslcDir): os.mkdir(rslcDir) + #workDir = processDir + '/' + igramDir + workDir = rslcDir + '/' + Sdate + if not os.path.isdir(workDir): os.mkdir(workDir) + + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Mdate = templateDict['masterDate'] + IFGPair = Mdate + '-' + Sdate + + demDir = scratchDir + '/' + projectName + '/DEM' + + SslcDir = slcDir + '/' + Sdate + Samp = slcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar = slcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + Sramp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + SrampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + Sslc = slcDir + '/' + Sdate + '/' + Sdate + '.slc' + SslcPar = slcDir + '/' + Sdate + '/' + Sdate + '.slc.par' + + SrslcDir = rslcDir + "/" + Sdate + + Srslc = SrslcDir + "/" + Sdate + ".rslc" + SrslcPar = SrslcDir + "/" + Sdate + ".rslc.par" + + Srslc0 = SrslcDir + "/" + Sdate + ".rslc0" + SrslcPar0 = SrslcDir + "/" + Sdate + ".rslc0.par" + +##################################################### +## copy all of the master files into slave folder for parallel processing + remove_file = [] + Mslc0 = slcDir + '/' + Mdate + '/' + Mdate + '.slc' + MslcPar0 = slcDir + '/' + Mdate + '/' + Mdate + '.slc.par' + Mamp0 = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar0 = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + Mamp0_dem = demDir + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar0_dem = demDir + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + HGTSIM0 = demDir + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' + if not os.path.isfile(HGTSIM0): + call_str = 'generate_rdc_dem.py ' + projectName + _run_or_raise(call_str, 'generate_rdc_dem') + if not os.path.isfile(Mamp0) and os.path.isfile(Mamp0_dem): + ut.copy_file(Mamp0_dem, Mamp0) + if not os.path.isfile(MampPar0) and os.path.isfile(MampPar0_dem): + ut.copy_file(MampPar0_dem, MampPar0) + if not os.path.isfile(Mamp0) or not os.path.isfile(MampPar0): + call_str = 'multi_look ' + Mslc0 + ' ' + MslcPar0 + ' ' + Mamp0 + ' ' + MampPar0 + ' ' + rlks + ' ' + azlks + _run_or_raise(call_str, 'master_multi_look') + if not os.path.isfile(Samp) or not os.path.isfile(SampPar): + call_str = 'multi_look ' + Sslc + ' ' + SslcPar + ' ' + Samp + ' ' + SampPar + ' ' + rlks + ' ' + azlks + _run_or_raise(call_str, 'slave_multi_look') + + Mslc = slcDir + '/' + Sdate + '/' + Mdate + '.slc' + MslcPar = slcDir + '/' + Sdate + '/' + Mdate + '.slc.par' + Mamp = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + HGTSIM = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' + + ut.copy_file(HGTSIM0,HGTSIM) + ut.copy_file(Mslc0,Mslc) + ut.copy_file(MslcPar0,MslcPar) + ut.copy_file(Mamp0,Mamp) + ut.copy_file(MampPar0,MampPar) + +####################################################### +# define process files # + lt0 = workDir + "/lt0" + lt1 = workDir + "/lt1" + mli0 = workDir + "/mli0" + diff0 = workDir + "/diff0" + offs0 = workDir + "/offs0" + snr0 = workDir + "/snr0" + offsets0 = workDir + "/offsets0" + coffs0 = workDir + "/coffs0" + coffsets0 = workDir + "/coffsets0" + off = workDir + "/" + IFGPair + ".off" + offs = workDir + "/offs" + snr = workDir + "/snr" + offsets = workDir + "/offsets" + coffs = workDir + "/coffs" + coffsets = workDir + "/coffsets" + OFFSTD = workDir + "/" + IFGPair + ".off_std" +############################################## + + call_str = "rdc_trans " + MampPar + " " + HGTSIM + " " + SampPar + " " + lt0 + _run_or_raise(call_str, 'rdc_trans') + + width_Mamp = ut.read_gamma_par(MampPar, 'read', 'range_samples') + width_Samp = ut.read_gamma_par(SampPar, 'read', 'range_samples') + line_Samp = ut.read_gamma_par(SampPar, 'read', 'azimuth_lines') + + call_str = "geocode " + lt0 + " " + Mamp + " " + width_Mamp + " " + mli0 + " " + width_Samp + " " + line_Samp + " 2 0" + _run_or_raise(call_str, 'geocode_lt0') + + call_str = "create_diff_par " + SampPar + " - " + diff0 + " 1 0" + _run_or_raise(call_str, 'create_diff_par') + + try: + call_str = "init_offsetm " + mli0 + " " + Samp + " " + diff0 + " 1 1" + _run_or_raise(call_str, 'init_offsetm') + + call_str = "offset_pwrm " + mli0 + " " + Samp + " " + diff0 + " " + offs0 + " " + snr0 + " 256 256 " + offsets0 + " 2 32 32" + _run_or_raise(call_str, 'offset_pwrm') + + call_str = "offset_fitm " + offs0 + " " + snr0 + " " + diff0 + " " + coffs0 + " " + coffsets0 + " - 4" + _run_or_raise(call_str, 'offset_fitm') + + call_str = "gc_map_fine " + lt0 + " " + width_Mamp + " " + diff0 + " " + lt1 + _run_or_raise(call_str, 'gc_map_fine') + except RuntimeError as exc: + print('ERROR: initial DEM-assisted offset refinement failed in strict no-rescue mode') + print(str(exc)) + raise + + + call_str = "SLC_interp_lt " + Sslc + " " + MslcPar + " " + SslcPar + " " + lt1 + " " + MampPar + " " + SampPar + " - " + Srslc0 + " " + SrslcPar0 + _run_or_raise(call_str, 'SLC_interp_lt_initial') + + +############################################ Refinement ############################################ + try: + call_str = "create_offset " + MslcPar + " " + SrslcPar0 + " " + off + " 1 - - 0" + _run_or_raise(call_str, 'create_offset') + + call_str = "offset_pwr " + Mslc + " " + Srslc0 + " " + MslcPar + " " + SrslcPar0 + " " + off + " " + offs + " " + snr + " " + templateDict['rwin4cor'] + " " + templateDict['azwin4cor'] + " " + offsets + " 2 " + templateDict['rsample4cor'] + " " + templateDict['azsample4cor'] + _run_or_raise(call_str, 'offset_pwr_coarse') + + call_str = "offset_fit " + offs + " " + snr + " " + off + " " + coffs + " " + coffsets + " - 3" + _run_or_raise(call_str, 'offset_fit_coarse') + + rfwin4cor = str(int(1/2*int(templateDict['rwin4cor']))) + azfwin4cor = str(int(1/2*int(templateDict['azwin4cor']))) + + rfsample4cor = str(2*int(templateDict['rsample4cor'])) + azfsample4cor = str(2*int(templateDict['azsample4cor'])) + + call_str = "offset_pwr " + Mslc + " " + Srslc0 + " " + MslcPar + " " + SrslcPar0 + " " + off + " " + offs + " " + snr + " " + rfwin4cor + " " + azfwin4cor + " " + offsets + " 2 " + rfsample4cor + " " + azfsample4cor + _run_or_raise(call_str, 'offset_pwr_fine') + + call_str = "offset_fit " + offs + " " + snr + " " + off + " " + coffs + " " + coffsets + " - 3 >" + OFFSTD + _run_or_raise(call_str, 'offset_fit_fine') + +############################################ Resampling ############################################ + + call_str = "SLC_interp_lt " + Sslc + " " + MslcPar + " " + SslcPar + " " + lt1 + " " + MampPar + " " + SampPar + " " + off + " " + Srslc + " " + SrslcPar + _run_or_raise(call_str, 'SLC_interp_lt_final') + except RuntimeError as exc: + print('ERROR: offset refinement failed in strict no-rescue mode') + print(str(exc)) + raise + + call_str = 'multi_look ' + Srslc + ' ' + SrslcPar + ' ' + Sramp + ' ' + SrampPar + ' ' + rlks + ' ' + azlks + _run_or_raise(call_str, 'multi_look_rslc') + + nWidth = ut.read_gamma_par(SrampPar, 'read', 'range_samples') + call_str = 'raspwr ' + Sramp + ' ' + nWidth + _run_or_raise(call_str, 'raspwr') + + for path in ( + lt0, lt1, mli0, diff0, offs0, snr0, offsets0, coffs0, coffsets0, + off, offs, snr, offsets, coffs, coffsets, Srslc0, SrslcPar0, + Mslc, Mamp, HGTSIM, + ): + _safe_remove(path) + + print("Coregistration with DEM is done!") + + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_gamma_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_gamma_all.py new file mode 100644 index 0000000..cf7dd4d --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_gamma_all.py @@ -0,0 +1,187 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse +import shutil + +import subprocess +from pyint import _utils as ut + +def work(data0): + cmd = data0[0] + err_txt = data0[1] + + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if type(stdout) == bytes: + bb=stdout.decode("utf-8", errors="replace") + else: + bb = stdout + + if p.returncode != 0: + detail_parts = [] + if bb: + detail_parts.append(bb) + if aa: + detail_parts.append(aa) + detail = '\n'.join(part.strip() for part in detail_parts if part and part.strip()) + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + with open(err_txt, 'a') as f: + f.write(str0) + if detail: + f.write(detail) + f.write('\n') + raise RuntimeError(str0.strip() + ' failed with rc=' + str(p.returncode) + ('\n' + detail if detail else '')) + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + with open(err_txt, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + + +def ensure_master_rslc(projectName): + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict = ut.update_template(templateFile) + master_date = templateDict['masterDate'] + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + slc_root = scratchDir + '/' + projectName + '/SLC/' + master_date + rslc_root = scratchDir + '/' + projectName + '/RSLC/' + master_date + if not os.path.isdir(rslc_root): + os.makedirs(rslc_root, exist_ok=True) + + src_slc = slc_root + '/' + master_date + '.slc' + src_slc_par = slc_root + '/' + master_date + '.slc.par' + src_amp = slc_root + '/' + master_date + '_' + rlks + 'rlks.amp' + src_amp_par = slc_root + '/' + master_date + '_' + rlks + 'rlks.amp.par' + + dst_rslc = rslc_root + '/' + master_date + '.rslc' + dst_rslc_par = rslc_root + '/' + master_date + '.rslc.par' + dst_amp = rslc_root + '/' + master_date + '_' + rlks + 'rlks.amp' + dst_amp_par = rslc_root + '/' + master_date + '_' + rlks + 'rlks.amp.par' + + if not os.path.isfile(src_slc) or not os.path.isfile(src_slc_par): + raise FileNotFoundError('Master SLC is missing under SLC/' + master_date) + + if not os.path.isfile(dst_rslc): + shutil.copyfile(src_slc, dst_rslc) + if not os.path.isfile(dst_rslc_par): + shutil.copyfile(src_slc_par, dst_rslc_par) + + if os.path.isfile(src_amp) and not os.path.isfile(dst_amp): + shutil.copyfile(src_amp, dst_amp) + if os.path.isfile(src_amp_par) and not os.path.isfile(dst_amp_par): + shutil.copyfile(src_amp_par, dst_amp_par) + + if not os.path.isfile(dst_amp) or not os.path.isfile(dst_amp_par): + cmd = [ + 'multi_look', + dst_rslc, + dst_rslc_par, + dst_amp, + dst_amp_par, + rlks, + azlks, + ] + p = subprocess.run(cmd, shell=False, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + if p.returncode != 0: + detail_parts = [] + if p.stdout: + detail_parts.append(p.stdout.decode("utf-8", errors="replace")) + if p.stderr: + detail_parts.append(p.stderr.decode("utf-8", errors="replace")) + detail = '\n'.join(part.strip() for part in detail_parts if part and part.strip()) + raise RuntimeError('multi_look master RSLC failed' + ('\n' + detail if detail else '')) + +INTRODUCTION = ''' +------------------------------------------------------------------- + Coregister all of the SLCs to the reference SAR image using GAMMA. + [with assistant of DEM] + +''' + +EXAMPLE = ''' + Usage: + coreg_gamma_all.py projectName + coreg_gamma_all.py projectName --parallel 4 +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister all of the SLCs to the reference SAR image using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + projectDir = scratchDir + '/' + projectName + slcDir = scratchDir + '/' + projectName + '/SLC' + rslcDir = scratchDir + '/' + projectName + '/RSLC' + if not os.path.isdir(rslcDir): os.mkdir(rslcDir) + ensure_master_rslc(projectName) + + if 'S1' in projectName: cmd_command = 'coreg_s1_gamma.py' + else: cmd_command = 'coreg_gamma.py' + + err_txt = scratchDir + '/' + projectName + '/coreg_gamma_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + #slc_list = [os.path.basename(fname) for fname in sorted(glob.glob(slcDir + '/*'))] + master_date = ut.update_template(templateDir + "/" + projectName + ".template")['masterDate'] + slc_list = [item for item in ut.get_project_slcList(projectName) if item != master_date] + for i in range(len(slc_list)): + cmd0 = [cmd_command,projectName,slc_list[i]] + data0 = [cmd0,err_txt] + data_para.append(data0) + + results = ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + failures = [str(item) for item in results if isinstance(item, Exception)] + if failures: + raise RuntimeError('\n\n'.join(failures)) + print("Coregister all of the SLCs %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_s1_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_s1_gamma.py new file mode 100644 index 0000000..97ebd10 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_s1_gamma.py @@ -0,0 +1,239 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import argparse + +from pyint import _utils as ut + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister TOPS S1-SLC to a reference S1-SLC using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='Name of project.') + parser.add_argument('sdate', help='date of the slave S1 image. [mater date is read from template]') + inps = parser.parse_args() + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Coregister TOPS S1-SLC to a reference S1-SLC using GAMMA. + [The reference date or master date will be read from the template file.] +''' + +EXAMPLE = """Usage: + + coreg_s1_gamma.py projectName Sdate + + coreg_s1_gamma.py PacayaT163TsxHhA 20150102 +------------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Sdate = inps.sdate + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + if not os.path.isdir(rslcDir): os.mkdir(rslcDir) + #workDir = processDir + '/' + igramDir + workDir = rslcDir + '/' + Sdate + if not os.path.isdir(workDir): os.mkdir(workDir) + + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Mdate = templateDict['masterDate'] + + demDir = scratchDir + '/' + projectName + '/DEM' + +# Definition of file + MslcDir = slcDir + '/' + Mdate + SslcDir = slcDir + '/' + Sdate + Samp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + Sampbmp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.bmp' + + Mslc = slcDir + '/' + Mdate + '/' + Mdate + '.slc' + Mslcpar = slcDir + '/' + Mdate + '/' + Mdate + '.slc.par' + Mamp = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + Mampbmp = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.bmp' + + SLC1_INF_tab0 = MslcDir + '/' + Mdate + '_SLC_Tab' + SLC2_INF_tab = SslcDir + '/' + Sdate + '_SLC_Tab' + RSLC_tab = SslcDir + '/' + Sdate + '_RSLC_Tab' + SLC1_INF_tab = SslcDir + '/' + Mdate + '_SLC_Tab_coreg' + + HGTSIM = demDir + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' + if not os.path.isfile(HGTSIM): + call_str = 'generate_rdc_dem.py ' + projectName + os.system(call_str) + + ############## copy master files into slave folder for parallel process ########### + + #if not templateDict['coreg_all_parallel'] == '1': + # Mslc = slcDir + '/' + Sdate + '/' + Mdate + '.slc' + # Mslcpar = slcDir + '/' + Sdate + '/' + Mdate + '.slc.par' + # Mamp = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + # MampPar = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + # HGTSIM = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' + + #else: + + # Mslc = Mslc0 + # Mslcpar = Mslcpar0 + # Mamp = Mamp0 + # MampPar = MampPar0 + # HGTSIM = HGTSIM0 + + + #Mslc = slcDir + '/' + Sdate + '/' + Mdate + '.slc' + #Mslcpar = slcDir + '/' + Sdate + '/' + Mdate + '.slc.par' + #Mamp = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + #MampPar = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + #HGTSIM = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' + SLC1_INF_tab1 = SslcDir + '/' + Mdate + '_SLC_Tab' + if SslcDir==MslcDir: + SLC1_INF_tab1=SLC1_INF_tab0 + else: + ut.copy_file(SLC1_INF_tab0,SLC1_INF_tab1) + ############################################################################## + #with open(SLC1_INF_tab1, "r") as f: + # lines = f.readlines() + + #with open(SLC1_INF_tab, "w") as fw: + # lines_coreg = [] + # for k0 in lines: + # k00 = k0.replace(MslcDir,SslcDir) + # lines_coreg.append(k00) + # fw.write(k00) + + S_IW = ut.read_txt2array(SLC1_INF_tab1) + S_IW = S_IW.flatten() + #M_IW = ut.read_txt2array(SLC1_INF_tab1) + #M_IW = M_IW.flatten() + #S_IW = ut.read_txt2array(SLC1_INF_tab) + #S_IW = S_IW.flatten() + + #RSLC_tab = workDir + '/' + Sdate + '_RSLC_tab' + #if os.path.isfile(RSLC_tab): + # os.remove(RSLC_tab) + + #BURST = SslcDir + '/' + Sdate + '_Burst_Tab' + #AA = np.loadtxt(BURST) + #if EW==SW: + # AA = AA.reshape([1,2]) + # + #for kk in range(int(EW)-int(SW)+1): + # ii = int(int(kk) + 1) + # SB2=AA[ii-1,0] + # EB2=AA[ii-1,1] + # call_str = 'echo ' + workDir + '/'+ Sdate+ '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+str(int(SW)+kk)+ '.rslc' + ' ' + workDir + '/' + Sdate + '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+ str(int(SW)+kk)+ '.rslc.par' + ' ' + workDir + '/'+ Sdate+'_'+ str(int(SB2)) + str(int(EB2)) + '.IW'+str(int(SW)+kk)+ '.rslc.TOPS_par >>' + RSLC_tab + # os.system(call_str) + + os.chdir(workDir) + #TEST = workDir + '/' + Sdate +'_' + rlks + 'rlks.amp.par' + TEST = workDir + '/' + Mdate + '_' + Sdate +'.diff' + + k0 = 0 + if os.path.isfile(TEST): + if os.path.getsize(TEST) > 0: + k0 = 1 + + if k0 == 0: + if not Mdate==Sdate: +# call_str = 'S1_coreg_TOPS ' + SLC1_INF_tab1 + ' ' + Mdate + ' ' + SLC2_INF_tab + ' ' + Sdate + ' ' + RSLC_tab + ' ' + HGTSIM + ' ' + rlks + ' ' + azlks + ' - - 0.8 0.01 1.2 1' + call_str = 'ScanSAR_coreg.py ' + SLC1_INF_tab1 + ' ' + Mdate + ' ' + SLC2_INF_tab + ' ' + Sdate + ' ' + RSLC_tab + ' ' + HGTSIM + ' ' + rlks + ' ' + azlks + ' --cc 0.8 --fraction 0.01 --ph_stdev 0.8 --num_ovr 0 --no_check ' + print(call_str) + ret = os.system(call_str) + + #### clean large file #### + mslc = workDir + '/' + Mdate + '.slc' + mrslc = workDir + '/' + Mdate + '.rslc' + sslc = workDir + '/' + Sdate + '.slc' + srslc = workDir + '/' + Sdate + '.rslc' + srslcPar = workDir + '/' + Sdate + '.rslc.par' + + # 检查 ScanSAR_coreg.py 是否成功生成 .rslc（不检查返回码，spectral diversity 步骤可能报错但 RSLC 已生成） + if not os.path.isfile(srslc) or os.path.getsize(srslc) == 0: + print('ERROR: ScanSAR_coreg.py did not produce a valid ' + srslc) + sys.exit(1) + + call_str = 'multi_look ' + srslc + ' ' + srslcPar + ' ' + Samp + ' ' + SampPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + nWIDTH = ut.read_gamma_par(SampPar,'read', 'range_samples') + if os.path.isfile(mslc): os.remove(mslc) + if os.path.isfile(mrslc): os.remove(mrslc) + if os.path.isfile(sslc): os.remove(sslc) + + call_str = 'raspwr ' + Samp + ' ' + nWIDTH + os.system(call_str) + + #call_str = 'rm *mli*' + #os.system(call_str) + + #call_str = 'rm *IW*' + #os.system(call_str) + + #call_str = 'rm *off*' + #os.system(call_str) + + #call_str = 'rm *diff' + #os.system(call_str) + + #call_str = 'rm *diff_par*' + #os.system(call_str) + + #call_str = 'rm ' + Mdate + '.*' + #os.system(call_str) + + else: + call_str = 'cp ' + Mslc + ' ' + workDir + '/' + Mdate + '.rslc' + os.system(call_str) + + call_str = 'cp ' + Mslcpar + ' ' + workDir+ '/' + Mdate + '.rslc.par' + os.system(call_str) + + call_str = 'cp ' + Mamp + ' ' + workDir+ '/' + Mdate + '_' + rlks + 'rlks.amp' + os.system(call_str) + + call_str = 'cp ' + MampPar + ' ' + workDir+ '/' + Mdate + '_' + rlks + 'rlks.amp.par' + os.system(call_str) + + ut.copy_file(Mampbmp,Sampbmp) + + ################ clean redundant files ############# + + #if not Mdate ==Sdate: + + # if not templateDict['diff_all_parallel'] == '1': + # for i in range(len(S_IW)): + # if os.path.isfile(S_IW[i]): + # os.remove(S_IW[i]) + # if os.path.isfile(Mslc): os.remove(Mslc) + # if os.path.isfile(Mslcpar): os.remove(Mslcpar) + # if os.path.isfile(Mamp): os.remove(Mamp) + # if os.path.isfile(HGTSIM): os.remove(HGTSIM) + else: + print('The SLC has already coregister Done') + sys.exit(1) + print("Coregister TOP SLC image to the reference TOPS image is done !!") + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_s1_gamma_old.py b/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_s1_gamma_old.py new file mode 100644 index 0000000..15f9e08 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_s1_gamma_old.py @@ -0,0 +1,232 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import argparse + +from pyint import _utils as ut + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister TOPS S1-SLC to a reference S1-SLC using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='Name of project.') + parser.add_argument('sdate', help='date of the slave S1 image. [mater date is read from template]') + inps = parser.parse_args() + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Coregister TOPS S1-SLC to a reference S1-SLC using GAMMA. + [The reference date or master date will be read from the template file.] +''' + +EXAMPLE = """Usage: + + coreg_s1_gamma.py projectName Sdate + + coreg_s1_gamma.py PacayaT163TsxHhA 20150102 +------------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Sdate = inps.sdate + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + if not os.path.isdir(rslcDir): os.mkdir(rslcDir) + #workDir = processDir + '/' + igramDir + workDir = rslcDir + '/' + Sdate + if not os.path.isdir(workDir): os.mkdir(workDir) + + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Mdate = templateDict['masterDate'] + + demDir = scratchDir + '/' + projectName + '/DEM' + +# Definition of file + MslcDir = slcDir + '/' + Mdate + SslcDir = slcDir + '/' + Sdate + Samp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + Sampbmp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.bmp' + + Mslc = slcDir + '/' + Mdate + '/' + Mdate + '.slc' + Mslcpar = slcDir + '/' + Mdate + '/' + Mdate + '.slc.par' + Mamp = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + Mampbmp = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.bmp' + + #SLC1_INF_tab0 = MslcDir + '/' + Mdate + '_SLC_Tab0' + SLC1_INF_tab0 = MslcDir + '/' + Mdate + '_SLC_Tab' + SLC2_INF_tab = SslcDir + '/' + Sdate + '_SLC_Tab' + RSLC_tab = SslcDir + '/' + Sdate + '_RSLC_Tab' + #SLC1_INF_tab = SslcDir + '/' + Mdate + '_SLC_Tab_coreg' + SLC1_INF_tab = SslcDir + '/' + Mdate + '_SLC_Tab' + + HGTSIM = demDir + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' + if not os.path.isfile(HGTSIM): + call_str = 'generate_rdc_dem.py ' + projectName + os.system(call_str) + + ############## copy master files into slave folder for parallel process ########### + + #if not templateDict['coreg_all_parallel'] == '1': + # Mslc = slcDir + '/' + Sdate + '/' + Mdate + '.slc' + # Mslcpar = slcDir + '/' + Sdate + '/' + Mdate + '.slc.par' + # Mamp = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + # MampPar = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + # HGTSIM = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' + + #else: + + # Mslc = Mslc0 + # Mslcpar = Mslcpar0 + # Mamp = Mamp0 + # MampPar = MampPar0 + # HGTSIM = HGTSIM0 + + + #Mslc = slcDir + '/' + Sdate + '/' + Mdate + '.slc' + #Mslcpar = slcDir + '/' + Sdate + '/' + Mdate + '.slc.par' + #Mamp = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + #MampPar = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + #HGTSIM = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' + SLC1_INF_tab = SslcDir + '/' + Mdate + '_SLC_Tab' + if SslcDir==MslcDir: + SLC1_INF_tab1=SLC1_INF_tab0 + else: + ut.copy_file(SLC1_INF_tab0,SLC1_INF_tab) + ############################################################################## + #with open(SLC1_INF_tab, "r") as f: + # lines = f.readlines() + + #with open(SLC1_INF_tab, "w") as fw: + # lines_coreg = [] + # for k0 in lines: + # k00 = k0.replace(MslcDir,SslcDir) + # lines_coreg.append(k00) + # fw.write(k00) + + S_IW = ut.read_txt2array(SLC1_INF_tab) + S_IW = S_IW.flatten() + #M_IW = ut.read_txt2array(SLC1_INF_tab1) + #M_IW = M_IW.flatten() + #S_IW = ut.read_txt2array(SLC1_INF_tab) + #S_IW = S_IW.flatten() + + #RSLC_tab = workDir + '/' + Sdate + '_RSLC_tab' + #if os.path.isfile(RSLC_tab): + # os.remove(RSLC_tab) + + #BURST = SslcDir + '/' + Sdate + '_Burst_Tab' + #AA = np.loadtxt(BURST) + #if EW==SW: + # AA = AA.reshape([1,2]) + # + #for kk in range(int(EW)-int(SW)+1): + # ii = int(int(kk) + 1) + # SB2=AA[ii-1,0] + # EB2=AA[ii-1,1] + # call_str = 'echo ' + workDir + '/'+ Sdate+ '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+str(int(SW)+kk)+ '.rslc' + ' ' + workDir + '/' + Sdate + '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+ str(int(SW)+kk)+ '.rslc.par' + ' ' + workDir + '/'+ Sdate+'_'+ str(int(SB2)) + str(int(EB2)) + '.IW'+str(int(SW)+kk)+ '.rslc.TOPS_par >>' + RSLC_tab + # os.system(call_str) + + os.chdir(workDir) + #TEST = workDir + '/' + Sdate +'_' + rlks + 'rlks.amp.par' + TEST = workDir + '/' + Sdate +'.rslc.par' + + k0 = 0 + if os.path.isfile(TEST): + if os.path.getsize(TEST) > 0: + k0 = 1 + + if k0==0: + if not Mdate ==Sdate: + call_str = 'S1_coreg_TOPS ' + SLC1_INF_tab + ' ' + Mdate + ' ' + SLC2_INF_tab + ' ' + Sdate + ' ' + RSLC_tab + ' ' + HGTSIM + ' ' + rlks + ' ' + azlks + ' - - 0.8 0.01 1.2 1' + os.system(call_str) + + #### clean large file #### + mslc = workDir + '/' + Mdate + '.slc' + mrslc = workDir + '/' + Mdate + '.rslc' + sslc = workDir + '/' + Sdate + '.slc' + srslc = workDir + '/' + Sdate + '.rslc' + srslcPar = workDir + '/' + Sdate + '.rslc.par' + + call_str = 'multi_look ' + srslc + ' ' + srslcPar + ' ' + Samp + ' ' + SampPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + nWIDTH = ut.read_gamma_par(SampPar,'read', 'range_samples') + if os.path.isfile(mslc): os.remove(mslc) + if os.path.isfile(mrslc): os.remove(mrslc) + if os.path.isfile(sslc): os.remove(sslc) + + call_str = 'raspwr ' + Samp + ' ' + nWIDTH + os.system(call_str) + + #call_str = 'rm *mli*' + #os.system(call_str) + + #call_str = 'rm *IW*' + #os.system(call_str) + + #call_str = 'rm *off*' + #os.system(call_str) + + #call_str = 'rm *diff' + #os.system(call_str) + + #call_str = 'rm *diff_par*' + #os.system(call_str) + + #call_str = 'rm ' + Mdate + '.*' + #os.system(call_str) + + else: + call_str = 'cp ' + Mslc + ' ' + workDir + '/' + Mdate + '.rslc' + os.system(call_str) + + call_str = 'cp ' + Mslcpar + ' ' + workDir+ '/' + Mdate + '.rslc.par' + os.system(call_str) + + call_str = 'cp ' + Mamp + ' ' + workDir+ '/' + Mdate + '_' + rlks + 'rlks.amp' + os.system(call_str) + + call_str = 'cp ' + MampPar + ' ' + workDir+ '/' + Mdate + '_' + rlks + 'rlks.amp.par' + os.system(call_str) + + ut.copy_file(Mampbmp,Sampbmp) + + ################ clean redundant files ############# + + #if not Mdate ==Sdate: + + # if not templateDict['diff_all_parallel'] == '1': + # for i in range(len(S_IW)): + # if os.path.isfile(S_IW[i]): + # os.remove(S_IW[i]) + # if os.path.isfile(Mslc): os.remove(Mslc) + # if os.path.isfile(Mslcpar): os.remove(Mslcpar) + # if os.path.isfile(Mamp): os.remove(Mamp) + # if os.path.isfile(HGTSIM): os.remove(HGTSIM) + + print("Coregister TOP SLC image to the reference TOPS image is done !!") + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_s1_gamma_pot.py b/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_s1_gamma_pot.py new file mode 100644 index 0000000..712cf57 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/coreg_s1_gamma_pot.py @@ -0,0 +1,370 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import argparse +import glob + +from pyint import _utils as ut + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister TOPS S1-SLC to a reference S1-SLC using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='Name of project.') + parser.add_argument('sdate', help='date of the slave S1 image. [mater date is read from template]') + inps = parser.parse_args() + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Coregister TOPS S1-SLC to a reference S1-SLC using GAMMA. + [The reference date or master date will be read from the template file.] +''' + +EXAMPLE = """Usage: + + coreg_s1_gamma.py projectName Sdate + + coreg_s1_gamma.py PacayaT163TsxHhA 20150102 +------------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Sdate = inps.sdate + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + if not os.path.isdir(rslcDir): os.mkdir(rslcDir) + #workDir = processDir + '/' + igramDir + workDir = rslcDir + '/' + Sdate + if not os.path.isdir(workDir): os.mkdir(workDir) + + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Mdate = templateDict['masterDate'] + deramp = templateDict['deramp'] + + if 'boi' in templateDict: + boi = templateDict['boi'] + else: + boi = '0' + + demDir = scratchDir + '/' + projectName + '/DEM' + +# Definition of file + MslcDir = slcDir + '/' + Mdate + + SslcDir = slcDir + '/' + Sdate + Samp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + Sampbmp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.bmp' + + Mslc = slcDir + '/' + Mdate + '/' + Mdate + '.slc' + Mslcpar = slcDir + '/' + Mdate + '/' + Mdate + '.slc.par' + Mamp = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + Mampbmp = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.bmp' + + #SLC1_INF_tab = MslcDir + '/' + Mdate + '_SLC_Tab' + SLC2_INF_tab = SslcDir + '/' + Sdate + '_SLC_Tab' + RSLC_tab = SslcDir + '/' + Sdate + '_RSLC_Tab' + RSLC_tab2 = workDir + '/' + Sdate + '_RSLC_Tab' + call_str = 'cp ' + RSLC_tab + ' ' + RSLC_tab2 + os.system(call_str) + + #SLC1_INF_tab = SslcDir + '/' + Mdate + '_SLC_Tab_coreg' + + HGTSIM = demDir + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' + if not os.path.isfile(HGTSIM): + call_str = 'generate_rdc_dem.py ' + projectName + os.system(call_str) + + srslc_fboi0 = workDir + '/' + Sdate + '_overlap.fwd.slc'; srslc_fboi = srslc_fboi0.replace('.slc','.rslc') + srslc_fboi_par0 = workDir + '/' + Sdate + '_overlap.fwd.slc.par'; srslc_fboi_par = srslc_fboi_par0.replace('.slc','.rslc') + + srslc_bboi0 = workDir + '/' + Sdate + '_overlap.bwd.slc'; srslc_bboi = srslc_bboi0.replace('.slc','.rslc') + srslc_bboi_par0 = workDir + '/' + Sdate + '_overlap.bwd.slc.par'; srslc_bboi_par = srslc_bboi_par0.replace('.slc','.rslc') + + samp_fboi = workDir + '/' + Sdate + '_overlap.fwd_' + rlks + 'rlks.amp' + samp_fboi_par = workDir + '/' + Sdate + '_overlap.fwd_' + rlks + 'rlks.amp.par' + + samp_bboi = workDir + '/' + Sdate + '_overlap.bwd_' + rlks + 'rlks.amp' + samp_bboi_par = workDir + '/' + Sdate + '_overlap.bwd_' + rlks + 'rlks.amp.par' + ############## copy master files into slave folder for parallel process ########### + + #if not templateDict['coreg_all_parallel'] == '1': + # Mslc = slcDir + '/' + Sdate + '/' + Mdate + '.slc' + # Mslcpar = slcDir + '/' + Sdate + '/' + Mdate + '.slc.par' + # Mamp = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + # MampPar = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + # HGTSIM = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' + + #else: + + # Mslc = Mslc0 + # Mslcpar = Mslcpar0 + # Mamp = Mamp0 + # MampPar = MampPar0 + # HGTSIM = HGTSIM0 + + + #Mslc = slcDir + '/' + Sdate + '/' + Mdate + '.slc' + #Mslcpar = slcDir + '/' + Sdate + '/' + Mdate + '.slc.par' + #Mamp = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + #MampPar = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + #HGTSIM = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' + #SLC1_INF_tab1 = SslcDir + '/' + Mdate + '_SLC_Tab' + #ut.copy_file(SLC1_INF_tab0,SLC1_INF_tab1) + ############################################################################## + #with open(SLC1_INF_tab1, "r") as f: + # lines = f.readlines() + + #with open(SLC1_INF_tab, "w") as fw: + # lines_coreg = [] + # for k0 in lines: + # k00 = k0.replace(MslcDir,SslcDir) + # lines_coreg.append(k00) + # fw.write(k00) + + #S_IW = ut.read_txt2array(SLC1_INF_tab1) + #S_IW = S_IW.flatten() + #M_IW = ut.read_txt2array(SLC1_INF_tab1) + #M_IW = M_IW.flatten()TSLC = slc_dir + '/' + date + '.slc' + #TSLCPar = slc_dir + '/' + date + '.slc.par' + + # + #S_IW = ut.read_txt2array(SLC1_INF_tab) + #S_IW = S_IW.flatten() + + #RSLC_tab = workDir + '/' + Sdate + '_RSLC_tab' + #if os.path.isfile(RSLC_tab): + # os.remove(RSLC_tab) + + #BURST = SslcDir + '/' + Sdate + '_Burst_Tab' + #AA = np.loadtxt(BURST) + #if EW==SW: + # AA = AA.reshape([1,2]) + # + #for kk in range(int(EW)-int(SW)+1): + # ii = int(int(kk) + 1) + # SB2=AA[ii-1,0] + # EB2=AA[ii-1,1] + # call_str = 'echo ' + workDir + '/'+ Sdate+ '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+str(int(SW)+kk)+ '.rslc' + ' ' + workDir + '/' + Sdate + '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+ str(int(SW)+kk)+ '.rslc.par' + ' ' + workDir + '/'+ Sdate+'_'+ str(int(SB2)) + str(int(EB2)) + '.IW'+str(int(SW)+kk)+ '.rslc.TOPS_par >>' + RSLC_tab + # os.system(call_str) + + os.chdir(workDir) + #TEST = workDir + '/' + Sdate +'_' + rlks + 'rlks.amp.par' + TEST = workDir + '/' + Mdate + '_' + Sdate +'.diff.bmp' + + k0 = 0 + if os.path.isfile(TEST): + if os.path.getsize(TEST) > 0: + k0 = 1 + + if k0==0: + if not Mdate ==Sdate: + + call_str = 'cp -rf ' + MslcDir + ' ' + workDir + os.system(call_str) + + MslcDir2 = workDir + '/' + Mdate + + SLC_list = glob.glob(MslcDir2 + '/*IW*.slc') + SLC_par_list = glob.glob(MslcDir2 + '/*IW*.slc.par') + TOP_par_list = glob.glob(MslcDir2 + '/*IW*.slc.TOPS_par') + + SLC1_INF_tab = MslcDir2 + '/' + Mdate + '_SLC_Tab' + + if os.path.isfile(SLC1_INF_tab): + os.remove(SLC1_INF_tab) + + SLC_list = sorted(SLC_list) + SLC_par_list = sorted(SLC_par_list) + TOP_par_list = sorted(TOP_par_list) + + for kk in range(len(SLC_list)): + call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' >> ' + SLC1_INF_tab + os.system(call_str) + + + #call_str = 'S1_coreg_TOPS ' + SLC1_INF_tab + ' ' + Mdate + ' ' + SLC2_INF_tab + ' ' + Sdate + ' ' + RSLC_tab + ' ' + HGTSIM + ' ' + rlks + ' ' + azlks + ' - - 0.8 0.01 1.2 1' + + call_str = 'ScanSAR_coreg.py ' + SLC1_INF_tab + ' ' + Mdate + ' ' + SLC2_INF_tab + ' ' + Sdate + ' ' + RSLC_tab + ' ' + HGTSIM + ' ' + rlks + ' ' + azlks + ' --cc 0.8 --fraction 0.01 --ph_stdev 0.8 --num_ovr 0 --no_check ' + print(call_str) + os.system(call_str) + + #### clean large file #### + mslc = workDir + '/' + Mdate + '.slc' + mrslc = workDir + '/' + Mdate + '.rslc' + sslc = workDir + '/' + Sdate + '.slc' + srslc = workDir + '/' + Sdate + '.rslc' + + srslcPar = workDir + '/' + Sdate + '.rslc.par' + + if deramp=='1': + call_str = 'S1_deramp_TOPS_slave ' + RSLC_tab2 + ' ' + Sdate + ' ' + SLC1_INF_tab + ' 10 2 1' + os.system(call_str) + + call_str = 'mv ' + srslc + '.deramp' + ' ' + srslc + os.system(call_str) + + call_str = 'mv ' + srslc + '.deramp.par' + ' ' + srslcPar + os.system(call_str) + + call_str = 'multi_look ' + srslc + ' ' + srslcPar + ' ' + Samp + ' ' + SampPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + + nWIDTH = ut.read_gamma_par(SampPar,'read', 'range_samples') + + if boi =='1': + call_str = 'ScanSAR_burst_overlap ' + RSLC_tab2 + ' ' + Sdate+'_overlap' + ' ' + rlks + ' ' + azlks + ' - - ' + SLC1_INF_tab + ' - -' + os.system(call_str) + + call_str = 'mv ' + srslc_fboi0 + ' ' + srslc_fboi; os.system(call_str) + call_str = 'mv ' + srslc_fboi_par0 + ' ' + srslc_fboi_par; os.system(call_str) + call_str = 'mv ' + srslc_bboi0 + ' ' + srslc_bboi; os.system(call_str) + call_str = 'mv ' + srslc_bboi_par0 + ' ' + srslc_bboi_par; os.system(call_str) + + call_str = 'multi_look ' + srslc_fboi + ' ' + srslc_fboi_par + ' ' + samp_fboi + ' ' + samp_fboi_par + ' ' + rlks + ' ' + azlks + os.system(call_str) + + call_str = 'multi_look ' + srslc_bboi + ' ' + srslc_bboi_par + ' ' + samp_bboi + ' ' + samp_bboi_par + ' ' + rlks + ' ' + azlks + os.system(call_str) + + call_str = 'raspwr ' + samp_fboi + ' ' + nWIDTH + os.system(call_str) + + call_str = 'raspwr ' + samp_bboi + ' ' + nWIDTH + os.system(call_str) + + if os.path.isfile(mslc): os.remove(mslc) + if os.path.isfile(mrslc): os.remove(mrslc) + if os.path.isfile(sslc): os.remove(sslc) + + call_str = 'raspwr ' + Samp + ' ' + nWIDTH + os.system(call_str) + + call_str = 'rm -rf ' + MslcDir2 + os.system(call_str) + + #call_str = 'rm *mli*' + #os.system(call_str) + + #call_str = 'rm *IW*' + #os.system(call_str) + + #call_str = 'rm *off*' + #os.system(call_str) + + #call_str = 'rm *diff' + #os.system(call_str) + + #call_str = 'rm *diff_par*' + #os.system(call_str) + + #call_str = 'rm ' + Mdate + '.*' + + #os.system(call_str) + + else: + + #generate amp file for check image quality + TSLC = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc' + TSLCPar = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc.par' + + TMLI = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + TMLIPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + if not os.path.isfile(TMLIPar): + A1 = np.loadtxt(SLC2_INF_tab,dtype='str'); A1 = A1.flatten() + A2 = np.loadtxt(RSLC_tab,dtype='str'); A2 = A2.flatten() + + nn = len(A1) + for i in range(nn): + call_str = 'cp ' + A1[i] + ' ' + A2[i] + os.system(call_str) + + if deramp=='1': + call_str = 'S1_deramp_TOPS_reference ' + RSLC_tab2 + os.system(call_str) + + SLC2_INF_tab = RSLC_tab2 + '.deramp' + else: + SLC2_INF_tab = RSLC_tab2 + + call_str = 'SLC_mosaic_S1_TOPS ' + SLC2_INF_tab + ' ' + TSLC + ' ' + TSLCPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + + call_str = 'multi_look ' + TSLC + ' ' + TSLCPar + ' ' + TMLI + ' ' + TMLIPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + + nWidth = ut.read_gamma_par(TMLIPar, 'read','range_samples:') + + if boi =='1': + call_str = 'ScanSAR_burst_overlap ' + RSLC_tab2 + ' ' + Sdate+'_overlap' + ' ' + rlks + ' ' + azlks + ' - - ' + SLC2_INF_tab + ' - -' + os.system(call_str) + + call_str = 'mv ' + srslc_fboi0 + ' ' + srslc_fboi; os.system(call_str) + call_str = 'mv ' + srslc_fboi_par0 + ' ' + srslc_fboi_par; os.system(call_str) + call_str = 'mv ' + srslc_bboi0 + ' ' + srslc_bboi; os.system(call_str) + call_str = 'mv ' + srslc_bboi_par0 + ' ' + srslc_bboi_par; os.system(call_str) + + call_str = 'multi_look ' + srslc_fboi + ' ' + srslc_fboi_par + ' ' + samp_fboi + ' ' + samp_fboi_par + ' ' + rlks + ' ' + azlks + os.system(call_str) + + call_str = 'multi_look ' + srslc_bboi + ' ' + srslc_bboi_par + ' ' + samp_bboi + ' ' + samp_bboi_par + ' ' + rlks + ' ' + azlks + os.system(call_str) + + call_str = 'raspwr ' + samp_fboi + ' ' + nWidth + os.system(call_str) + + call_str = 'raspwr ' + samp_bboi + ' ' + nWidth + os.system(call_str) + + + call_str = 'raspwr ' + TMLI + ' ' + nWidth + ' - - - - - - - ' + os.system(call_str) + rr = glob.glob(workDir + '/*IW*.slc') + if len(rr) > 0: + call_str = 'rm ' + workDir + '/*IW*.slc' + os.system(call_str) + uu = glob.glob(workDir + '/*IW*.rslc') + + if len(uu) > 0: + call_str = 'rm ' + workDir + '/*IW*.rslc' + os.system(call_str) + ################ clean redundant files ############# + + #if not Mdate ==Sdate: + + # if not templateDict['diff_all_parallel'] == '1': + # for i in range(len(S_IW)): + # if os.path.isfile(S_IW[i]): + # os.remove(S_IW[i]) + # if os.path.isfile(Mslc): os.remove(Mslc) + # if os.path.isfile(Mslcpar): os.remove(Mslcpar) + # if os.path.isfile(Mamp): os.remove(Mamp) + # if os.path.isfile(HGTSIM): os.remove(HGTSIM) + + print("Coregister TOP SLC image to the reference TOPS image is done !!") + #sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/corners.txt b/.codex_tmp/pyint_variants/no_rescue/pyint/corners.txt new file mode 100644 index 0000000..cb93de4 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/corners.txt @@ -0,0 +1,22 @@ +*** Calculate SLC/MLI image corners in geodetic latitude and longitude (deg.) *** +*** Copyright 2022, Gamma Remote Sensing, v2.2 8-Nov-2022 clw/awi/cm *** +latitude (deg.): 24.74699954 longitude (deg.): 103.30166172 +latitude (deg.): 25.16531434 longitude (deg.): 105.78287221 +latitude (deg.): 26.37363426 longitude (deg.): 102.95054159 +latitude (deg.): 26.78897220 longitude (deg.): 105.46762233 + +center latitude (deg.): 25.79112685 center longitude (deg.): 104.47614292 +min. latitude (deg.): 24.74699954 max. latitude (deg.): 26.78897220 +min. longitude (deg.): 102.95054159 max. longitude (deg.): 105.78287221 +delta latitude (deg.): 2.04197267 delta longitude (deg.): 2.83233062 + +upper left corner latitude, longitude (deg.): 26.83 102.91 +lower right corner latitude, longitude (deg.): 24.71 105.82 + +lower left corner longitude, latitude (deg.): 102.91 24.71 +upper right corner longitude, latitude (deg.): 105.82 26.83 + +user time (s): 0.000 +system time (s): 0.000 +elapsed time (s): 0.000 + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/correct_ifg_for_hpy3_from_murp.py b/.codex_tmp/pyint_variants/no_rescue/pyint/correct_ifg_for_hpy3_from_murp.py new file mode 100644 index 0000000..2ff3b68 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/correct_ifg_for_hpy3_from_murp.py @@ -0,0 +1,769 @@ +#!/usr/bin/env python3 +""" +修复版MuRP算法 - 包含Linux系统中文显示支持 +""" + +import xarray as xr +import rioxarray +import os +import numpy as np +import matplotlib.pyplot as plt +from glob import glob +import argparse +import sys +from datetime import datetime +import rasterio +import warnings +import matplotlib.font_manager as fm +import platform + +# 设置中文字体支持 +def setup_chinese_font(): + """配置中文字体支持""" + system = platform.system() + + if system == "Linux": + # Linux系统字体配置 + chinese_fonts = [ + 'WenQuanYi Micro Hei', # 文泉驿微米黑 + 'Noto Sans CJK SC', # Google思源黑体 + 'DejaVu Sans', # 备选字体 + 'Arial' # 最后备选 + ] + + # 查找可用的字体 + available_fonts = [] + for font in chinese_fonts: + if any(f.name == font for f in fm.fontManager.ttflist): + available_fonts.append(font) + + if available_fonts: + plt.rcParams['font.sans-serif'] = available_fonts + print(f"已设置中文字体: {available_fonts[0]}") + else: + plt.rcParams['font.sans-serif'] = ['DejaVu Sans'] + print("警告: 未找到中文字体，使用默认字体") + else: + # Windows或macOS + plt.rcParams['font.sans-serif'] = ['SimHei', 'Microsoft YaHei', 'Arial'] + + # 解决负号显示问题 + plt.rcParams['axes.unicode_minus'] = False + +# 调用字体设置函数 +setup_chinese_font() +warnings.filterwarnings('ignore') + +class FixedMuRP: + """ + 修复版MuRP算法 - 解决相干性数据形状不一致问题 + """ + + def __init__(self, random_seed=42): + """初始化MuRP处理器""" + self.random_seed = random_seed + np.random.seed(random_seed) + self.ds = None + self.ds_corrected = None + self.refs = [] + self.fits = [] + self.crs = None + self.transform = None + self.bounds = None + self.width = None + self.height = None + + def load_hyp3_data_robust(self, data_path): + """ + 健壮的数据加载函数，处理形状不一致问题 + """ + print(f"正在加载Hyp3数据从: {data_path}") + + # 查找数据文件 + unw_files = glob(f'{data_path}/*/*unw_phase*.tif') + corr_files = glob(f'{data_path}/*/*corr*.tif') + dem_files = glob(f'{data_path}/*/*dem*.tif') + + if not unw_files: + raise FileNotFoundError(f"在 {data_path} 中未找到解缠相位文件") + if not dem_files: + raise FileNotFoundError(f"在 {data_path} 中未找到DEM文件") + + print(f"找到解缠相位文件: {len(unw_files)} 个") + print(f"找到相干性文件: {len(corr_files)} 个") + print(f"找到DEM文件: {len(dem_files)} 个") + + # 使用第一个解缠相位文件作为参考 + reference_file = unw_files[0] + with rasterio.open(reference_file) as src: + self.crs = src.crs + self.transform = src.transform + self.width = src.width + self.height = src.height + self.bounds = src.bounds + + print(f"坐标参考系统: {self.crs}") + print(f"图像尺寸: {self.width} x {self.height}") + + # 读取DEM数据 - 使用第一个DEM文件 + print("正在加载DEM数据...") + with rasterio.open(dem_files[0]) as src: + # 确保DEM尺寸与参考文件一致 + if src.width == self.width and src.height == self.height: + elevation_data = src.read(1) + else: + print("警告: DEM尺寸不匹配，将使用零数组") + elevation_data = np.zeros((self.height, self.width)) + + # 读取解缠相位数据 + print("正在加载解缠相位数据...") + unw_phase_data = [] + granule_names = [] + + for unw_file in unw_files: + with rasterio.open(unw_file) as src: + # 检查尺寸是否匹配 + if src.width == self.width and src.height == self.height: + data = src.read(1) + else: + print(f"警告: {unw_file} 尺寸不匹配，跳过") + continue + + unw_phase_data.append(data) + granule_name = os.path.basename(unw_file).replace('.tif', '') + granule_names.append(granule_name) + print(f" 已加载: {granule_name}") + + if not unw_phase_data: + raise ValueError("没有成功加载任何解缠相位数据") + + # 转换为numpy数组 + unw_phase_array = np.array(unw_phase_data) + print(f"解缠相位数据形状: {unw_phase_array.shape}") + + # 处理相干性数据 - 确保与解缠相位数据对应 + print("正在处理相干性数据...") + coherence_data = [] + used_corr_files = [] + + # 为每个解缠相位文件寻找匹配的相干性文件 + for i, granule_name in enumerate(granule_names): + # 从granule名称提取基本名称 + base_name = granule_name.replace('_unw_phase', '') + + # 查找匹配的相干性文件 + matching_corr_files = [f for f in corr_files if base_name in f and f not in used_corr_files] + + if matching_corr_files: + # 使用第一个匹配的文件 + corr_file = matching_corr_files[0] + try: + with rasterio.open(corr_file) as src: + # 检查尺寸是否匹配 + if src.width == self.width and src.height == self.height: + corr_data = src.read(1) + coherence_data.append(corr_data) + used_corr_files.append(corr_file) + print(f" 已加载相干性: {os.path.basename(corr_file)}") + else: + print(f" 警告: 相干性文件尺寸不匹配，使用默认值") + coherence_data.append(np.ones((self.height, self.width))) + except Exception as e: + print(f" 加载相干性文件失败: {e}") + coherence_data.append(np.ones((self.height, self.width))) + else: + # 没有找到匹配的相干性文件，使用默认值 + print(f" 未找到匹配的相干性文件，使用默认值") + coherence_data.append(np.ones((self.height, self.width))) + + # 如果相干性数据数量与解缠相位不匹配，补充默认值 + while len(coherence_data) < len(unw_phase_data): + coherence_data.append(np.ones((self.height, self.width))) + + # 转换为numpy数组 + coherence_array = np.array(coherence_data) + print(f"相干性数据形状: {coherence_array.shape}") + + # 创建xarray数据集 + ds = xr.Dataset( + { + 'unw_phase': (['granule', 'y', 'x'], unw_phase_array), + 'coherence': (['granule', 'y', 'x'], coherence_array), + 'elevation': (['y', 'x'], elevation_data) + }, + coords={ + 'granule': granule_names, + 'y': np.arange(self.height), + 'x': np.arange(self.width) + } + ) + + # 设置地理参考属性 + try: + ds.rio.set_crs(self.crs) + ds.rio.write_transform(self.transform, inplace=True) + except: + print("警告: 无法设置地理参考信息") + + print(f"成功加载数据: {len(granule_names)} 个干涉图") + self.ds = ds + return ds + + def select_reference_points(self, corr_thresh=0.6, n_refs=1000): + """ + 选择参考点 + """ + if self.ds is None: + raise ValueError("请先加载数据") + + print(f"步骤 1: 选择参考点 (相干性阈值: {corr_thresh}, 数量: {n_refs})") + + # 计算平均相干性 + coh_mean = self.ds.coherence.mean(dim='granule').values + + # 获取高相干性像素 + valid_mask = coh_mean >= corr_thresh + valid_coords = np.argwhere(valid_mask) + + if len(valid_coords) == 0: + print(f"警告: 没有找到相干性大于{corr_thresh}的像素，降低阈值到0.3") + valid_mask = coh_mean >= 0.3 + valid_coords = np.argwhere(valid_mask) + + if len(valid_coords) == 0: + print("警告: 仍然没有找到高相干性像素，使用所有像素") + valid_coords = np.array([[i, j] for i in range(self.height) for j in range(self.width)]) + + print(f" 找到 {len(valid_coords)} 个候选像素点") + + # 调整参考点数量 + if len(valid_coords) < n_refs: + n_refs = len(valid_coords) + print(f" 可用点不足，调整为选择 {n_refs} 个参考点") + + # 使用网格策略选择参考点 + grid_size = int(np.sqrt(n_refs / 4)) + grid_size = max(5, min(grid_size, 50)) + + x_bins = np.linspace(0, self.width, grid_size + 1, dtype=int) + y_bins = np.linspace(0, self.height, grid_size + 1, dtype=int) + + ref_list = [] + points_per_cell = max(1, n_refs // (grid_size * grid_size)) + + for i in range(grid_size): + for j in range(grid_size): + x_min, x_max = x_bins[i], x_bins[i+1] + y_min, y_max = y_bins[j], y_bins[j+1] + + # 在当前网格内选择点 + cell_mask = ((valid_coords[:, 1] >= x_min) & (valid_coords[:, 1] < x_max) & + (valid_coords[:, 0] >= y_min) & (valid_coords[:, 0] < y_max)) + cell_points = valid_coords[cell_mask] + + if len(cell_points) > 0: + n_select = min(points_per_cell, len(cell_points)) + selected_indices = np.random.choice(len(cell_points), n_select, replace=False) + + for idx in selected_indices: + y, x = cell_points[idx] + ref_list.append([int(x), int(y)]) + + # 如果点数不足，随机补充 + if len(ref_list) < n_refs: + remaining = n_refs - len(ref_list) + additional_indices = np.random.choice(len(valid_coords), remaining, replace=False) + for idx in additional_indices: + y, x = valid_coords[idx] + if [int(x), int(y)] not in ref_list: + ref_list.append([int(x), int(y)]) + + self.refs = ref_list[:n_refs] + print(f" 成功选择 {len(self.refs)} 个参考点") + return self.refs + + def sample_reference_data(self): + """采样参考点数据""" + if not self.refs: + raise ValueError("请先选择参考点") + + print("步骤 2: 采样参考点数据") + + # 验证参考点坐标 + valid_refs = [] + for ref in self.refs: + x, y = ref[0], ref[1] + if 0 <= x < self.width and 0 <= y < self.height: + valid_refs.append(ref) + else: + print(f"警告: 参考点 ({x}, {y}) 超出图像范围") + + if not valid_refs: + raise ValueError("没有有效的参考点") + + # 提取坐标 + x_coords = [ref[0] for ref in valid_refs] + y_coords = [ref[1] for ref in valid_refs] + + # 采样高程数据 + ref_elevation = [] + for x, y in valid_refs: + elev_val = self.ds.elevation.isel(x=x, y=y).values + ref_elevation.append(float(elev_val)) + ref_elevation = np.array(ref_elevation) + + # 采样相位数据 + ref_values = [] + for i in range(len(self.ds.granule)): + granule_phases = [] + for x, y in valid_refs: + phase_val = self.ds.unw_phase.isel(granule=i, x=x, y=y).values + granule_phases.append(float(phase_val)) + ref_values.append(granule_phases) + ref_values = np.array(ref_values) + + print(f" 成功采样 {len(valid_refs)} 个参考点，{len(ref_values)} 个干涉图") + return ref_values, ref_elevation + + def numpy_linear_regression(self, x, y): + """ + 使用NumPy实现线性回归 + """ + # 移除NaN值 + valid_mask = ~(np.isnan(x) | np.isnan(y)) + x_valid = x[valid_mask] + y_valid = y[valid_mask] + + if len(x_valid) < 2: + return np.nan, np.nan, 0, 0 + + # 计算线性回归参数 + x_mean = np.mean(x_valid) + y_mean = np.mean(y_valid) + + # 计算协方差和方差 + cov_xy = np.mean((x_valid - x_mean) * (y_valid - y_mean)) + var_x = np.mean((x_valid - x_mean) ** 2) + + if var_x == 0: + return np.nan, np.nan, 0, 0 + + beta = cov_xy / var_x + alpha = y_mean - beta * x_mean + + # 计算R² + y_pred = alpha + beta * x_valid + ss_res = np.sum((y_valid - y_pred) ** 2) + ss_tot = np.sum((y_valid - y_mean) ** 2) + + if ss_tot == 0: + r_squared = 0 + else: + r_squared = 1 - (ss_res / ss_tot) + + n_points = len(x_valid) + + return beta, alpha, r_squared, n_points + + def perform_linear_fits(self, ref_values, ref_elevation): + """ + 执行线性拟合 + """ + print("步骤 3: 线性拟合") + + fits = [] + fit_metrics = [] + elevations = np.array(ref_elevation) + + for i in range(len(ref_values)): + phases = ref_values[i] + + # 执行线性回归 + slope, intercept, r2, n_points = self.numpy_linear_regression(elevations, phases) + + fits.append([slope, intercept]) + fit_metrics.append({ + 'r_squared': r2, + 'n_points': n_points + }) + + if (i + 1) % 10 == 0 or (i + 1) == len(ref_values): + print(f" 已完成 {i+1}/{len(ref_values)} 个干涉图") + + # 统计结果 + valid_fits = sum(1 for fit in fits if not np.isnan(fit[0])) + valid_r2 = [m['r_squared'] for m in fit_metrics if not np.isnan(m['r_squared'])] + avg_r2 = np.mean(valid_r2) if valid_r2 else 0 + + self.fits = fits + self.fit_metrics = fit_metrics + + print(f" 线性拟合完成: {valid_fits}/{len(fits)} 个成功, 平均R²: {avg_r2:.3f}") + return fits, fit_metrics + + def apply_correction(self, min_r2=0.0): + """应用相位校正""" + print("步骤 4: 应用相位校正") + + if not self.fits: + raise ValueError("请先进行线性拟合") + + # 过滤低质量拟合 + valid_fits = [] + valid_indices = [] + + for i, fit in enumerate(self.fits): + if np.isnan(fit[0]) or np.isnan(fit[1]): + continue + + if self.fit_metrics[i]['r_squared'] < min_r2: + continue + + valid_fits.append(fit) + valid_indices.append(i) + + if not valid_fits: + print("警告: 没有通过质量控制的拟合，使用所有有效拟合") + valid_fits = [f for f in self.fits if not np.isnan(f[0]) and not np.isnan(f[1])] + valid_indices = [i for i, f in enumerate(self.fits) + if not np.isnan(f[0]) and not np.isnan(f[1])] + + if not valid_fits: + raise ValueError("没有有效的拟合可用于校正") + + print(f" 使用 {len(valid_fits)}/{len(self.fits)} 个拟合进行校正") + + # 创建校正后的数据集 + self.ds_corrected = self.ds.copy() + + # 创建校正后的相位数组 + corrected_phase = np.zeros_like(self.ds.unw_phase.values) + + # 对每个干涉图应用校正 + for idx, granule_idx in enumerate(valid_indices): + slope, intercept = valid_fits[idx] + + # 计算校正量: phase = slope * elevation + intercept + elevation_data = self.ds.elevation.values + correction = elevation_data * slope + intercept + + # 应用校正: corrected_phase = original_phase - correction + original_phase = self.ds.unw_phase[granule_idx].values + corrected_phase[granule_idx] = original_phase - correction + + # 添加校正后的变量 + self.ds_corrected['unw_phase_corrected'] = (('granule', 'y', 'x'), corrected_phase) + + return self.ds_corrected + + def save_results_as_geotiff(self, output_dir='.'): + """保存结果为GeoTIFF格式""" + if self.ds_corrected is None: + raise ValueError("请先进行校正") + + print("步骤 5: 保存GeoTIFF格式结果") + os.makedirs(output_dir, exist_ok=True) + + # 保存校正后的每个干涉图 + corrected_dir = os.path.join(output_dir, "corrected_phase") + os.makedirs(corrected_dir, exist_ok=True) + + for i, granule in enumerate(self.ds_corrected.granule.values): + # 获取校正后的相位数据 + phase_corrected = self.ds_corrected.unw_phase_corrected[i].values + + # 创建输出文件名 + output_file = os.path.join(corrected_dir, f"{granule}_corrected.tif") + + # 使用rasterio保存为GeoTIFF + try: + with rasterio.open( + output_file, 'w', + driver='GTiff', + height=self.height, + width=self.width, + count=1, + dtype=phase_corrected.dtype, + crs=self.crs, + transform=self.transform + ) as dst: + dst.write(phase_corrected, 1) + + print(f" 已保存: {output_file}") + except Exception as e: + print(f" 保存失败 {output_file}: {e}") + + # 保存平均校正后相位 + try: + mean_corrected = self.ds_corrected.unw_phase_corrected.mean(dim='granule').values + mean_file = os.path.join(output_dir, "mean_corrected_phase.tif") + + with rasterio.open( + mean_file, 'w', + driver='GTiff', + height=self.height, + width=self.width, + count=1, + dtype=mean_corrected.dtype, + crs=self.crs, + transform=self.transform + ) as dst: + dst.write(mean_corrected, 1) + + print(f" 已保存平均校正相位: {mean_file}") + except Exception as e: + print(f" 保存平均校正相位失败: {e}") + + # 保存原始平均相位用于对比 + try: + mean_original = self.ds.unw_phase.mean(dim='granule').values + mean_orig_file = os.path.join(output_dir, "mean_original_phase.tif") + + with rasterio.open( + mean_orig_file, 'w', + driver='GTiff', + height=self.height, + width=self.width, + count=1, + dtype=mean_original.dtype, + crs=self.crs, + transform=self.transform + ) as dst: + dst.write(mean_original, 1) + + print(f" 已保存原始平均相位: {mean_orig_file}") + except Exception as e: + print(f" 保存原始平均相位失败: {e}") + + # 保存高程数据 + try: + elev_file = os.path.join(output_dir, "elevation.tif") + elevation_data = self.ds.elevation.values + + with rasterio.open( + elev_file, 'w', + driver='GTiff', + height=self.height, + width=self.width, + count=1, + dtype=elevation_data.dtype, + crs=self.crs, + transform=self.transform + ) as dst: + dst.write(elevation_data, 1) + + print(f" 已保存高程数据: {elev_file}") + except Exception as e: + print(f" 保存高程数据失败: {e}") + + print(f"所有GeoTIFF文件已保存至: {output_dir}") + + def create_diagnostic_plots(self, ref_values, ref_elevation, output_dir='.'): + """创建诊断图表""" + print("步骤 6: 生成诊断图表") + os.makedirs(output_dir, exist_ok=True) + + # 创建综合诊断图 + fig, axes = plt.subplots(2, 3, figsize=(15, 10)) + + # 1. 参考点分布 + ax1 = axes[0, 0] + elevation_data = self.ds.elevation.values + im1 = ax1.imshow(elevation_data, cmap='terrain') + if self.refs: + x_coords, y_coords = zip(*self.refs) + ax1.scatter(x_coords, y_coords, c='red', s=5, alpha=0.7, label='Reference Points') + ax1.legend() + ax1.set_title('Reference Points Distribution') + plt.colorbar(im1, ax=ax1, label='Elevation (m)') + + # 2. 第一个干涉图的相位-高程关系 + ax2 = axes[0, 1] + if len(ref_values) > 0: + phases = ref_values[0] + valid_mask = ~(np.isnan(ref_elevation) | np.isnan(phases)) + + if np.sum(valid_mask) > 0: + ax2.scatter(ref_elevation[valid_mask], phases[valid_mask], + alpha=0.5, s=10) + + # 绘制拟合线 + if not np.isnan(self.fits[0][0]): + elev_min, elev_max = np.min(ref_elevation[valid_mask]), np.max(ref_elevation[valid_mask]) + elev_range = np.linspace(elev_min, elev_max, 100) + phase_fit = self.fits[0][0] * elev_range + self.fits[0][1] + ax2.plot(elev_range, phase_fit, 'r-', linewidth=2, + label=f'Slope: {self.fits[0][0]:.4f}') + ax2.legend() + + ax2.set_xlabel('Elevation (m)') + ax2.set_ylabel('Phase (rad)') + ax2.set_title('Phase vs Elevation') + ax2.grid(True, alpha=0.3) + + # 3. 拟合斜率分布 + ax3 = axes[0, 2] + slopes = [f[0] for f in self.fits if not np.isnan(f[0])] + if slopes: + ax3.hist(slopes, bins=20, alpha=0.7, density=True) + ax3.axvline(np.mean(slopes), color='r', linestyle='--', + label=f'Mean: {np.mean(slopes):.4f}') + ax3.legend() + ax3.set_xlabel('Slope') + ax3.set_ylabel('Density') + ax3.set_title('Slope Distribution') + ax3.grid(True, alpha=0.3) + + # 4. 校正前后对比 + ax4 = axes[1, 0] + phase_before = self.ds.unw_phase.mean(dim='granule').values + vmin, vmax = -np.pi, np.pi + im4 = ax4.imshow(phase_before, cmap='RdBu', vmin=vmin, vmax=vmax) + ax4.set_title('Mean Phase (Before Correction)') + plt.colorbar(im4, ax=ax4, label='Phase (rad)') + + ax5 = axes[1, 1] + phase_after = self.ds_corrected.unw_phase_corrected.mean(dim='granule').values + im5 = ax5.imshow(phase_after, cmap='RdBu', vmin=vmin, vmax=vmax) + ax5.set_title('Mean Phase (After Correction)') + plt.colorbar(im5, ax=ax5, label='Phase (rad)') + + # 5. 校正量 + ax6 = axes[1, 2] + correction = phase_before - phase_after + im6 = ax6.imshow(correction, cmap='viridis') + ax6.set_title('Phase Correction') + plt.colorbar(im6, ax=ax6, label='Correction (rad)') + + plt.tight_layout() + plt.savefig(f'{output_dir}/MuRP_diagnostics.png', dpi=300, bbox_inches='tight') + plt.close() + + print(f" 诊断图已保存至: {output_dir}/MuRP_diagnostics.png") + + def calculate_improvement(self): + """计算改善统计""" + if self.ds_corrected is None: + return 0, 0, 0 + + # 计算时间序列标准差 + std_before = float(self.ds.unw_phase.std(dim='granule').mean().values) + std_after = float(self.ds_corrected.unw_phase_corrected.std(dim='granule').mean().values) + + improvement = (std_before - std_after) / std_before * 100 if std_before != 0 else 0 + + return improvement, std_before, std_after + + def run_murp_correction(self, data_path, corr_thresh=0.6, n_refs=1000, + output_dir='.', create_plots=True, save_geotiff=True): + """ + 运行完整的MuRP校正流程 + """ + start_time = datetime.now() + print("="*60) + print("修复版MuRP算法开始执行") + print(f"开始时间: {start_time.strftime('%Y-%m-%d %H:%M:%S')}") + print("="*60) + + try: + # 1. 加载数据 + self.load_hyp3_data_robust(data_path) + + # 2. 选择参考点 + self.select_reference_points(corr_thresh, n_refs) + + # 3. 采样参考点数据 + ref_values, ref_elevation = self.sample_reference_data() + + # 4. 线性拟合 + self.perform_linear_fits(ref_values, ref_elevation) + + # 5. 应用校正 + self.apply_correction() + + # 6. 生成诊断图表 + if create_plots: + self.create_diagnostic_plots(ref_values, ref_elevation, output_dir) + + # 7. 保存GeoTIFF格式结果 + if save_geotiff: + self.save_results_as_geotiff(output_dir) + + # 计算改善统计 + improvement, std_before, std_after = self.calculate_improvement() + + end_time = datetime.now() + duration = (end_time - start_time).total_seconds() + + print("="*60) + print("修复版MuRP算法执行完成") + print(f"结束时间: {end_time.strftime('%Y-%m-%d %H:%M:%S')}") + print(f"总耗时: {duration:.1f} 秒") + print(f"时间序列标准差改善: {improvement:.1f}%") + print(f" 校正前: {std_before:.4f} rad") + print(f" 校正后: {std_after:.4f} rad") + print("="*60) + + return self.ds_corrected + + except Exception as e: + print(f"算法执行失败: {e}") + import traceback + traceback.print_exc() + return None + +def main(): + """命令行接口主函数""" + parser = argparse.ArgumentParser( + description='修复版MuRP算法 - 解决相干性数据形状不一致问题', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=''' +使用示例: + python fixed_murp.py /path/to/hyp3/data + python fixed_murp.py /path/to/hyp3/data --corr_thresh 0.6 --n_refs 500 + python fixed_murp.py /path/to/hyp3/data --output_dir ./results + ''' + ) + + parser.add_argument('data_path', help='Hyp3数据目录路径') + parser.add_argument('--corr_thresh', type=float, default=0.6, + help='相干性阈值 (默认: 0.6)') + parser.add_argument('--n_refs', type=int, default=1000, + help='参考点数量 (默认: 1000)') + parser.add_argument('--output_dir', default='.', + help='输出目录 (默认: 当前目录)') + parser.add_argument('--no_plots', action='store_true', + help='不生成诊断图表') + parser.add_argument('--no_geotiff', action='store_true', + help='不保存GeoTIFF格式') + + args = parser.parse_args() + + try: + # 创建MuRP处理器 + murp = FixedMuRP(random_seed=42) + + # 运行算法 + ds_corrected = murp.run_murp_correction( + data_path=args.data_path, + corr_thresh=args.corr_thresh, + n_refs=args.n_refs, + output_dir=args.output_dir, + create_plots=not args.no_plots, + save_geotiff=not args.no_geotiff + ) + + if ds_corrected is not None: + print("算法执行成功!") + else: + print("算法执行失败!") + sys.exit(1) + + except Exception as e: + print(f"错误: {e}") + import traceback + traceback.print_exc() + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/create_gacos.py b/.codex_tmp/pyint_variants/no_rescue/pyint/create_gacos.py new file mode 100644 index 0000000..1d28de3 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/create_gacos.py @@ -0,0 +1,136 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Wei Chen ### +### Contact : chenweicug@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import argparse +import subprocess +from pyint import _utils as ut + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister SM mode SLC to a reference SLC image using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='Name of project.') + parser.add_argument('mdate', help='date of the slave SLC image') + parser.add_argument('sdate', help='date of the slave SLC image') + inps = parser.parse_args() + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + create_files for psokinv software running file by generated by Gamma +''' + +EXAMPLE = """Usage: + + coreg_gamma.py projectName + + coreg_gamma.py PacayaT163TsxHhA +------------------------------------------------------------------- +""" + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Sdate = inps.sdate + Mdate = inps.mdate + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Smdate = templateDict['masterDate'] + + demDir = scratchDir + '/' + projectName + '/DEM' + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + ifgramDir = scratchDir + '/' + projectName + "/ifgrams" + workDir = ifgramDir + '/' + Mdate + '-' + Sdate + MslcDir = rslcDir + '/' + Mdate + SslcDir = rslcDir + '/' + Sdate + + offpar = SslcDir + '/' + Smdate + '_' + Sdate + '.off' + slcpar = MslcDir + '/' + Smdate + '.rslc.par' + dempar = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem.par' + dem = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem' + unwpar = demDir + '/' + Mdate + '_' + rlks + 'rlks.utm.dem.par' + unw = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.diff_filt.unw" + + os.chdir(workDir) + call_str = "look_vector " + slcpar+ " " + offpar + " " + dempar + " " + dem + ' lv_theta lv_phi' + os.system(call_str) + call_str = "grep 'corner_lat:' " + dempar + " | awk '{print $2}' " + North=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'corner_lon:' " + dempar + " | awk '{print $2}' " + West=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lat:' " + dempar + " | awk '{print $2}' " + posty=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lon:' " + dempar + " | awk '{print $2}' " + postx=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'width:' " + dempar + " | awk '{print $2}' " + width=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'nlines:' " + dempar + " | awk '{print $2}' " + length=subprocess.getstatusoutput(call_str)[1] + South=str(round(float(North)+(float(length)-1)*float(posty),7)) + East=str(round(float(West)+(float(width)-1)*float(postx),7)) + + call_str = "swap_bytes " + unw+ " unw.phase_swap 4 >dinsar.log " + os.system(call_str) + call_str = "gmt xyz2grd unw.phase_swap -Gunw_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx +" -ZTLf -N0" + os.system(call_str) + call_str = "gmt grdsample unw_f.grd -Gunw.grd -I3c" + os.system(call_str) + call_str = "gmt grd2xyz unw.grd -ZTLf -N0 > " + Mdate + '-' + Sdate + '.unw' + os.system(call_str) + call_str = "gmt grdinfo unw.grd -C | awk '{print $10}' " + Width=subprocess.getstatusoutput(call_str)[1] + call_str = "gmt grdinfo unw.grd -C | awk '{print $11}' " + line=subprocess.getstatusoutput(call_str)[1] + call_str = "gmt grdinfo unw.grd -C | awk '{print $2}' " + West=subprocess.getstatusoutput(call_str)[1] + call_str = "gmt grdinfo unw.grd -C | awk '{print $5}' " + North=subprocess.getstatusoutput(call_str)[1] + ymax=str(int(round(float(line) ,7))) + xmax=str(int(round(float(Width) ,7))) + output=workDir + '/' + Mdate + '-' + Sdate + '.gacos.unw.rsc' + if os.path.exists(output) is True: + os.remove(output) + + fopen=open(output,'a+') + fopen.write('WIDTH ' + Width + '\n') + fopen.write('FILE_LENGTH ' + line + '\n') + fopen.write('XMIN 1' + '\n') + fopen.write('XMAX ' + xmax + '\n') + fopen.write('YMIN 1' + '\n') + fopen.write('YMAX ' + ymax + '\n') + fopen.write('X_FIRST ' + West + '\n') + fopen.write('Y_FIRST ' + North + '\n') + fopen.write('X_STEP 8.33333333E-04' + '\n') + fopen.write('Y_STEP -8.33333333E-04' + '\n') + fopen.write('X_UNIT degrees' + '\n') + fopen.write('Y_UNIT degrees' + '\n') + fopen.write('Z_OFFSET 0' + '\n') + fopen.write('Z_SCALE 1' + '\n') + fopen.write('PROJECTION LATLON' + '\n') + fopen.write('DATUM WGS84' + '\n') + + print("convert gamma products to GACOS is done!") + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/create_psokinv.py b/.codex_tmp/pyint_variants/no_rescue/pyint/create_psokinv.py new file mode 100644 index 0000000..fc812a8 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/create_psokinv.py @@ -0,0 +1,180 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import argparse +import subprocess +from pyint import _utils as ut + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister SM mode SLC to a reference SLC image using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='Name of project.') + parser.add_argument('mdate', help='date of the slave SLC image') + parser.add_argument('sdate', help='date of the slave SLC image') + inps = parser.parse_args() + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + create_files for psokinv software running file by generated by Gamma +''' + +EXAMPLE = """Usage: + + create_psokinv.py projectName mdata sdata lon_min lon_max lat_min lat_max flag + + create_psokinv.py PacayaT163TsxHhA 20241203 20241215 80 81 39 30 1 +------------------------------------------------------------------- +""" + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Sdate = inps.sdate + Mdate = inps.mdate + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + + + + + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Smdate = templateDict['masterDate'] + + demDir = scratchDir + '/' + projectName + '/DEM' + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + ifgramDir = scratchDir + '/' + projectName + "/ifgrams" + workDir = ifgramDir + '/' + Mdate + '-' + Sdate + MslcDir = rslcDir + '/' + Mdate + SslcDir = rslcDir + '/' + Sdate + + offpar = SslcDir + '/' + Smdate + '_' + Sdate + '.off' + slcpar = MslcDir + '/' + Smdate + '.rslc.par' + dempar = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem.par' + dem = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem' + unwpar = demDir + '/' + Mdate + '_' + rlks + 'rlks.utm.dem.par' + unw = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.diff_filt.unw" + los = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.dispmap" + cor = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.diff_filt.cor" + + os.chdir(workDir) + call_str = "look_vector " + slcpar+ " " + offpar + " " + dempar + " " + dem + ' lv_theta lv_phi' + os.system(call_str) + call_str = "grep 'corner_lat:' " + dempar + " | awk '{print $2}' " + North=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'corner_lon:' " + dempar + " | awk '{print $2}' " + West=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lat:' " + dempar + " | awk '{print $2}' " + posty=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lon:' " + dempar + " | awk '{print $2}' " + postx=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'width:' " + dempar + " | awk '{print $2}' " + width=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'nlines:' " + dempar + " | awk '{print $2}' " + length=subprocess.getstatusoutput(call_str)[1] + South=str(float(North)+float(posty)*(float(length)-1)) + East=str(float(West)+float(postx)*(float(width)-1)) + #East=str(round(float(West)+(float(width)-1)*float(postx),7)) + + + call_str = "swap_bytes lv_theta lv_theta.phase_swap 4 >dinsar.log " + os.system(call_str) + call_str = "gmt xyz2grd lv_theta.phase_swap -Glv_theta.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -di0" + os.system(call_str) + call_str = "gmt grdmath 90 lv_theta.grd 3.1415926 DIV 180 MUL SUB = lv_theta_final.grd" + os.system(call_str) + call_str = "gmt grd2xyz lv_theta_final.grd -ZTLf -N0 >" +Mdate + '-' + Sdate + '.inc' + os.system(call_str) + call_str = "swap_bytes lv_phi lv_phi.phase_swap 4 >dinsar.log " + os.system(call_str) + call_str = "gmt xyz2grd lv_phi.phase_swap -Glv_phi.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -di0" + os.system(call_str) + call_str = "gmt grdmath -90 lv_phi.grd 3.1415926 DIV 180 MUL SUB = lv_phi_final.grd" + os.system(call_str) + call_str = "gmt grd2xyz lv_phi_final.grd -ZTLf -N0 >" +Mdate + '-' + Sdate + '.azi' + os.system(call_str) + call_str = "swap_bytes " + unw+ " unw.phase_swap 4 >dinsar.log " + os.system(call_str) + call_str = "gmt xyz2grd unw.phase_swap -Gunw.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -di0" + os.system(call_str) + call_str = "gmt grdmath 0.0556 unw.grd 3.1415926 DIV -4 DIV MUL = los.grd" + os.system(call_str) + call_str = "gmt grd2xyz unw.grd -ZTLf -N0 > " + Mdate + '-' + Sdate + '.unw' + os.system(call_str) + call_str = "swap_bytes " + dem + " dem.phase_swap 4 >dinsar.log " + os.system(call_str) + #call_str = "xyz2grd dem.phase_swap -Gdem_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I0.001" + " -ZTLf -N0" + call_str = "gmt xyz2grd dem.phase_swap -Gdem_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -di0" + os.system(call_str) + call_str = "gmt grd2xyz dem_f.grd -ZTLf -N0 >" + Mdate + '-' + Sdate + '.utm.dem' + os.system(call_str) + call_str = "gmt grd2xyz los.grd -ZTLf -N0 >" + Mdate + '-' + Sdate + '.utm.los' + os.system(call_str) + call_str = "gmt grdinfo lv_theta_final.grd -C | awk '{print $10}' " + Width=subprocess.getstatusoutput(call_str)[1] + call_str = "gmt grdinfo lv_theta_final.grd -C | awk '{print $11}' " + line=subprocess.getstatusoutput(call_str)[1] + call_str = "gmt grdinfo lv_theta_final.grd -C | awk '{print $2}' " + West=subprocess.getstatusoutput(call_str)[1] + call_str = "gmt grdinfo lv_theta_final.grd -C | awk '{print $5}' " + North=subprocess.getstatusoutput(call_str)[1] + ymax=str(int(round(float(line)-1 ,7))) + xmax=str(int(round(float(Width)-1 ,7))) + output=workDir + '/' + Mdate + '-' + Sdate + '.unw.rsc' + if os.path.exists(output) is True: + os.remove(output) + fopen=open(output,'a+') + fopen.write('WIDTH ' + Width + '\n') + fopen.write('FILE_LENGTH ' + line + '\n') + fopen.write('XMIN 0' + '\n') + fopen.write('XMAX ' + xmax + '\n') + fopen.write('YMIN 0' + '\n') + fopen.write('YMAX ' + ymax + '\n') + fopen.write('RLOOKS 1' + '\n') + fopen.write('ALOOKS 1' + '\n') + fopen.write('X_FIRST ' + West + '\n') + fopen.write('X_STEP 8.33333333E-04' + '\n') + fopen.write('X_UNIT degrees' + '\n') + fopen.write('Y_FIRST ' + North + '\n') + fopen.write('Y_STEP -8.33333333E-04' + '\n') + fopen.write('Y_UNIT degrees' + '\n') + fopen.write('WAVELENGTH 0.0555041577' + width + '\n') + #fopen.write('WAVELENGTH 0.236' + width + '\n') + fopen.write('MOLDE insar') + fopen.close() + call_str = "swap_bytes " + Mdate + '-' + Sdate + '.unw' + " geo_" + Mdate + '-' + Sdate + "_dem_flat.unw 4 >dinsar.log " + os.system(call_str) + call_str = "swap_bytes " + Mdate + '-' + Sdate + '.utm.dem' + " geo_" + Mdate + '-' + Sdate + "_dem_flat.utm.dem 4 >dinsar.log " + os.system(call_str) + call_str = "cp -f " + Mdate + '-' + Sdate + ".unw.rsc " + Mdate + '-' + Sdate +".inc.rsc" + os.system(call_str) + call_str = "cp -f " + Mdate + '-' + Sdate + ".unw.rsc " + Mdate + '-' + Sdate +".azi.rsc" + os.system(call_str) + call_str = "cp -f " + Mdate + '-' + Sdate + ".unw.rsc " + " geo_" + Mdate + '-' + Sdate + "_dem_flat.unw.rsc" + os.system(call_str) + call_str = "cp -f " + Mdate + '-' + Sdate + ".unw.rsc " + " geo_" + Mdate + '-' + Sdate + "_dem_flat.utm.dem.rsc" + os.system(call_str) + call_str = "rm -rf swap*grd *phase_swap dinsar.log" + os.system(call_str) + print("convert gamma products to psokinv is done!") + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/create_psokinv_cut.py b/.codex_tmp/pyint_variants/no_rescue/pyint/create_psokinv_cut.py new file mode 100644 index 0000000..6affcfb --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/create_psokinv_cut.py @@ -0,0 +1,235 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import argparse +import subprocess +from pyint import _utils as ut + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister SM mode SLC to a reference SLC image using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='Name of project.') + parser.add_argument('mdate', help='date of the slave SLC image') + parser.add_argument('sdate', help='date of the slave SLC image') + parser.add_argument('lon_min', help='the min lon of cut image') + parser.add_argument('lon_max', help='the max lon of cut image') + parser.add_argument('lat_min', help='the min lat of cut image') + parser.add_argument('lat_max', help='the min lat of cut image') + parser.add_argument('flag', help="the cut image flag (1: do; 0:don't do)") + inps = parser.parse_args() + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + create_files for psokinv software running file by generated by Gamma +''' + +EXAMPLE = """Usage: + + create_psokinv.py projectName mdata sdata lon_min lon_max lat_min lat_max flag + + create_psokinv.py PacayaT163TsxHhA 20241203 20241215 80 81 39 30 1 +------------------------------------------------------------------- +""" + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Sdate = inps.sdate + Mdate = inps.mdate + lon_max = inps.lon_max + lon_min = inps.lon_min + lat_min = inps.lat_min + lat_max = inps.lat_max + flag = inps.flag + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + + + + + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Smdate = templateDict['masterDate'] + + demDir = scratchDir + '/' + projectName + '/DEM' + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + ifgramDir = scratchDir + '/' + projectName + "/ifgrams" + workDir = ifgramDir + '/' + Mdate + '-' + Sdate + MslcDir = rslcDir + '/' + Mdate + SslcDir = rslcDir + '/' + Sdate + + offpar = SslcDir + '/' + Smdate + '_' + Sdate + '.off' + slcpar = MslcDir + '/' + Smdate + '.rslc.par' + dempar = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem.par' + dem = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem' + unwpar = demDir + '/' + Mdate + '_' + rlks + 'rlks.utm.dem.par' + unw = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.diff_filt.unw" + los = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.dispmap" + cor = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.diff_filt.cor" + + os.chdir(workDir) + call_str = "look_vector " + slcpar+ " " + offpar + " " + dempar + " " + dem + ' lv_theta lv_phi' + os.system(call_str) + call_str = "grep 'corner_lat:' " + dempar + " | awk '{print $2}' " + North=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'corner_lon:' " + dempar + " | awk '{print $2}' " + West=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lat:' " + dempar + " | awk '{print $2}' " + posty=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lon:' " + dempar + " | awk '{print $2}' " + postx=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'width:' " + dempar + " | awk '{print $2}' " + width=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'nlines:' " + dempar + " | awk '{print $2}' " + length=subprocess.getstatusoutput(call_str)[1] + South=str(round(float(North)+(float(length)-1)*float(posty),7)) + East=str(round(float(West)+(float(width)-1)*float(postx),7)) + + + call_str = "swap_bytes lv_theta lv_theta.phase_swap 4 >dinsar.log " + os.system(call_str) + call_str = "gmt xyz2grd lv_theta.phase_swap -Glv_theta.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -N0" + os.system(call_str) + call_str = "gmt grdmath 90 lv_theta.grd 3.1415926 DIV 180 MUL SUB = lv_theta_final.grd" + os.system(call_str) + if flag == '0': + call_str = "gmt grdsample lv_theta_final.grd -Glv_theta.grd -I3c" + else: + call_str = "gmt grdsample lv_theta_final.grd -R" + lon_min + "/" + lon_max + "/" + lat_min + "/" + lat_max + " -Glv_theta.grd -I3c" + os.system(call_str) + call_str = "gmt grd2xyz lv_theta.grd -ZTLf -N0 >" +Mdate + '-' + Sdate + '.inc' + os.system(call_str) + call_str = "swap_bytes lv_phi lv_phi.phase_swap 4 >dinsar.log " + os.system(call_str) + call_str = "gmt xyz2grd lv_phi.phase_swap -Glv_phi.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -N0" + os.system(call_str) + call_str = "gmt grdmath -90 lv_phi.grd 3.1415926 DIV 180 MUL SUB = lv_phi_final.grd" + os.system(call_str) + if flag == '0': + call_str = "gmt grdsample lv_phi_final.grd -Glv_phi.grd -I3c" + else: + call_str = "gmt grdsample lv_phi_final.grd -R" + lon_min + "/" + lon_max + "/" + lat_min + "/" + lat_max + " -Glv_phi.grd -I3c" + os.system(call_str) + call_str = "gmt grd2xyz lv_phi.grd -ZTLf -N0 >" +Mdate + '-' + Sdate + '.azi' + os.system(call_str) + call_str = "swap_bytes " + unw+ " unw.phase_swap 4 >dinsar.log " + os.system(call_str) + call_str = "gmt xyz2grd unw.phase_swap -Gunw_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx +" -ZTLf -N0" + os.system(call_str) + call_str = "swap_bytes " + los + " los.swap 4 >dinsar.log " + os.system(call_str) + call_str = "gmt xyz2grd los.swap -Glos_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx +" -ZTLf -N0" + os.system(call_str) + call_str = "swap_bytes " + cor + " cor.swap 4 >dinsar.log " + os.system(call_str) + call_str = "gmt xyz2grd cor.swap -Gcor_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx +" -ZTLf -N0" + os.system(call_str) + + if flag == '0': + call_str = "gmt grdsample unw_f.grd -Gunw.grd -I3c" + else: + call_str = "gmt grdsample unw_f.grd -R" + lon_min + "/" + lon_max + "/" + lat_min + "/" + lat_max + " -Gunw.grd -I3c" + os.system(call_str) + call_str = "gmt grdmath 0.0556 unw_f.grd 3.1415926 DIV -4 DIV MUL = los.grd" + os.system(call_str) + call_str = "gmt grd2xyz unw.grd -ZTLf -N0 > " + Mdate + '-' + Sdate + '.unw' + os.system(call_str) + call_str = "swap_bytes " + dem + " dem.phase_swap 4 >dinsar.log " + os.system(call_str) + #call_str = "xyz2grd dem.phase_swap -Gdem_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I0.001" + " -ZTLf -N0" + call_str = "gmt xyz2grd dem.phase_swap -Gdem_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -N0" + os.system(call_str) + if flag == '0': + call_str = "gmt grdsample dem_f.grd -Gdem.grd -I3c" + else: + call_str = "gmt grdsample dem_f.grd -R" + lon_min + "/" + lon_max + "/" + lat_min + "/" + lat_max + " -Gdem.grd -I3c" + os.system(call_str) + + call_str = "gmt grd2xyz dem.grd -ZTLf -N0 >" + Mdate + '-' + Sdate + '.utm.dem' + os.system(call_str) + + if flag == '0': + call_str = "gmt grdsample los_f.grd -Glos1.grd -I3c" + else: + call_str = "gmt grdsample los_f.grd -R" + lon_min + "/" + lon_max + "/" + lat_min + "/" + lat_max + " -Glos1.grd -I3c" + os.system(call_str) + + call_str = "gmt grd2xyz los1.grd -ZTLf -N0 >" + Mdate + '-' + Sdate + '.utm.los' + os.system(call_str) + if flag == '0': + call_str = "gmt grdsample cor_f.grd -Gcor.grd -I3c" + else: + call_str = "gmt grdsample cor_f.grd -R" + lon_min + "/" + lon_max + "/" + lat_min + "/" + lat_max + " -Gcor.grd -I3c" + os.system(call_str) + + call_str = "gmt grd2xyz cor.grd -ZTLf -N0 >" + Mdate + '-' + Sdate + '.utm.cor' + os.system(call_str) + + call_str = "gmt grdinfo lv_theta.grd -C | awk '{print $10}' " + Width=subprocess.getstatusoutput(call_str)[1] + call_str = "gmt grdinfo lv_theta.grd -C | awk '{print $11}' " + line=subprocess.getstatusoutput(call_str)[1] + call_str = "gmt grdinfo lv_theta.grd -C | awk '{print $2}' " + West=subprocess.getstatusoutput(call_str)[1] + call_str = "gmt grdinfo lv_theta.grd -C | awk '{print $5}' " + North=subprocess.getstatusoutput(call_str)[1] + ymax=str(int(round(float(line)-1 ,7))) + xmax=str(int(round(float(Width)-1 ,7))) + output=workDir + '/' + Mdate + '-' + Sdate + '.unw.rsc' + if os.path.exists(output) is True: + os.remove(output) + fopen=open(output,'a+') + fopen.write('WIDTH ' + Width + '\n') + fopen.write('FILE_LENGTH ' + line + '\n') + fopen.write('XMIN 0' + '\n') + fopen.write('XMAX ' + xmax + '\n') + fopen.write('YMIN 0' + '\n') + fopen.write('YMAX ' + ymax + '\n') + fopen.write('RLOOKS 1' + '\n') + fopen.write('ALOOKS 1' + '\n') + fopen.write('X_FIRST ' + West + '\n') + fopen.write('X_STEP 8.33333333E-04' + '\n') + fopen.write('X_UNIT degrees' + '\n') + fopen.write('Y_FIRST ' + North + '\n') + fopen.write('Y_STEP -8.33333333E-04' + '\n') + fopen.write('Y_UNIT degrees' + '\n') + fopen.write('WAVELENGTH 0.0555041577' + width + '\n') + #fopen.write('WAVELENGTH 0.236' + width + '\n') + fopen.write('MOLDE insar') + fopen.close() + call_str = "swap_bytes " + Mdate + '-' + Sdate + '.unw' + " geo_" + Mdate + '-' + Sdate + "_dem_flat.unw 4 >dinsar.log " + os.system(call_str) + call_str = "swap_bytes " + Mdate + '-' + Sdate + '.utm.dem' + " geo_" + Mdate + '-' + Sdate + "_dem_flat.utm.dem 4 >dinsar.log " + os.system(call_str) + call_str = "cp -f " + Mdate + '-' + Sdate + ".unw.rsc " + Mdate + '-' + Sdate +".inc.rsc" + os.system(call_str) + call_str = "cp -f " + Mdate + '-' + Sdate + ".unw.rsc " + Mdate + '-' + Sdate +".azi.rsc" + os.system(call_str) + call_str = "cp -f " + Mdate + '-' + Sdate + ".unw.rsc " + " geo_" + Mdate + '-' + Sdate + "_dem_flat.unw.rsc" + os.system(call_str) + call_str = "cp -f " + Mdate + '-' + Sdate + ".unw.rsc " + " geo_" + Mdate + '-' + Sdate + "_dem_flat.utm.dem.rsc" + os.system(call_str) + call_str = "rm -rf swap*grd *phase_swap dinsar.log" + os.system(call_str) + print("convert gamma products to psokinv is done!") + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/diff_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/diff_gamma.py new file mode 100644 index 0000000..7396087 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/diff_gamma.py @@ -0,0 +1,183 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT ### +### Author: chen ### +### Contact : chenweicug@126.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +from pyint import _utils as ut + + +def _run_or_raise(call_str, stage): + rc = os.system(call_str) + if rc != 0: + raise RuntimeError('%s failed with rc=%s: %s' % (stage, rc, call_str)) + return rc + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Generate differential interferogram image from SLC using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + diff_gamma.py projectName Mdate Sdate + diff_gamma.py PacayaT163TsxHhA 20150102 20150601 +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister all of the SLCs to the reference SAR image using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('Mdate',help='Master date.') + parser.add_argument('Sdate',help='Slave date.') + + inps = parser.parse_args() + return inps + + +def main(argv): + + start_time = time.time() + inps = cmdLineParse() + Mdate = inps.Mdate + Sdate = inps.Sdate + + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + + projectDir = scratchDir + '/' + projectName + demDir = scratchDir + '/' + projectName + '/DEM' + + slcDir = scratchDir + '/' + projectName + '/SLC' + rslcDir = scratchDir + '/' + projectName + '/RSLC' + ifgDir = projectDir + '/ifgrams' + if not os.path.isdir(ifgDir): os.mkdir(ifgDir) + + Pair = Mdate + '-' + Sdate + workDir = ifgDir + '/' + Pair + if not os.path.isdir(workDir): os.mkdir(workDir) + + ####################################################################### + Mamp = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + Samp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + Mrslc = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc' + MrslcPar = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc.par' + Srslc = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc' + SrslcPar = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc.par' + + HGT = demDir + '/' + masterDate + '_' + rlks + 'rlks.rdc.dem' + + MasterPar = rslcDir + '/' + masterDate + '/' + masterDate + '.rslc.par' + + ################# copy file for parallel processing ########################## + #Mamp = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp' + #MampPar = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + #Samp = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp' + #SampPar = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + #if not templateDict['diff_all_parallel'] == '1': + + # Mrslc = workDir + '/' + Mdate + '.rslc' + # MrslcPar = workDir + '/' + Mdate + '.rslc.par' + # Srslc = workDir + '/' + Sdate + '.rslc' + # SrslcPar = workDir + '/' + Sdate + '.rslc.par' + # ut.copy_file(Mrslc0,Mrslc) + # ut.copy_file(MrslcPar0,MrslcPar) + # ut.copy_file(Srslc0,Srslc) + # ut.copy_file(SrslcPar0,SrslcPar) + + #else: + + # Mrslc = Mrslc0 + # MrslcPar = MrslcPar0 + # Srslc = Srslc0 + # SrslcPar = SrslcPar0 + # HGT = HGT0 + # MasterPar = MasterPar0 + + #ut.copy_file(Mamp0,Mamp) + #ut.copy_file(MampPar0,MampPar) + #ut.copy_file(Samp0,Samp) + #ut.copy_file(SampPar0,SampPar) + + #ut.copy_file(HGT0,HGT) + #ut.copy_file(MasterPar0,MasterPar) + + ############################################################################ + + OFF = workDir + '/' + Pair +'_' + rlks + 'rlks.off' + call_str = 'create_offset '+ MrslcPar + ' ' + SrslcPar + ' ' + OFF + ' 1 ' + rlks + ' ' + azlks + ' 0' + _run_or_raise(call_str, 'create_offset') + + SIM_UNW = workDir + '/' + Pair + '.sim_unw' + call_str = 'phase_sim_orb ' + MrslcPar + ' ' + SrslcPar + ' ' + OFF + ' ' + HGT + ' ' + SIM_UNW + ' ' + MasterPar + ' - - 1 1' + _run_or_raise(call_str, 'phase_sim_orb') + + DIFF_IFG = workDir + '/' + Pair + '_' + rlks + 'rlks.diff' + call_str = 'SLC_diff_intf ' + Mrslc + ' ' + Srslc + ' ' + MrslcPar + ' ' + SrslcPar + ' ' + OFF + ' ' + SIM_UNW + ' ' + DIFF_IFG + ' ' + rlks + ' ' + azlks + ' ' + templateDict['Igram_Spsflg'] + ' ' + templateDict['Igram_Azfflg'] + ' - 1 1' + _run_or_raise(call_str, 'SLC_diff_intf') + + ##### filtering process & coherence estimation ########### + DIFFFILT = workDir + '/' + Pair + '_' + rlks + 'rlks.diff_filt' + COHFILT = workDir + '/' + Pair + '_' + rlks + 'rlks.diff_filt.cor' + + nWIDTH = ut.read_gamma_par(OFF, 'read', 'interferogram_width') + call_str = 'adf ' + DIFF_IFG + ' ' + DIFFFILT + ' ' + COHFILT + ' ' + nWIDTH + ' ' + templateDict['adf_alpha'] + ' - ' + templateDict['Igram_Cor_Win'] + _run_or_raise(call_str, 'adf') + + ################# coherence estimation ##################### + call_str = 'cc_wave ' + DIFFFILT + ' ' + Mamp + ' ' + Samp + ' ' + COHFILT + ' ' + nWIDTH + ' ' + templateDict['Igram_Cor_rwin'] + ' ' + templateDict['Igram_Cor_awin'] + _run_or_raise(call_str, 'cc_wave') + + + ################ save images ##################### + call_str = 'rasmph_pwr ' + DIFFFILT + ' ' + Mamp + ' ' + nWIDTH + ' - - - - - - - - - ' + COHFILT + ' - 0.1' + _run_or_raise(call_str, 'rasmph_pwr_diff_filt') + + call_str = 'rasmph_pwr ' + DIFF_IFG + ' ' + Mamp + ' ' + nWIDTH + ' - - - - - - - - - ' + COHFILT + ' - 0.1' + _run_or_raise(call_str, 'rasmph_pwr_diff') + + call_str = 'rasdt_pwr ' + COHFILT + ' ' + Mamp + ' ' + nWIDTH + ' 1 0 1 1 0.1 1.0 1 ' + _run_or_raise(call_str, 'rasdt_pwr_coh') + + #os.remove(Mamp) + #os.remove(MampPar) + #os.remove(Samp) + #os.remove(SampPar) + + #if not templateDict['diff_all_parallel'] == '1': + # if os.path.isfile(Mrslc): os.remove(Mrslc) + # if os.path.isfile(Srslc):os.remove(Srslc) + + # if os.path.isfile(HGT):os.remove(HGT) + + print("Subtraction of topography and flattening phase is done!") + ut.print_process_time(start_time, time.time()) + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/diff_gamma_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/diff_gamma_all.py new file mode 100644 index 0000000..def2813 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/diff_gamma_all.py @@ -0,0 +1,144 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + + +def work(data0): + cmd = data0[0] + err_file = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if type(stdout) == bytes: + bb=stdout.decode("utf-8", errors="replace") + else: + bb = stdout + + if p.returncode != 0: + detail_parts = [] + if bb: + detail_parts.append(bb) + if aa: + detail_parts.append(aa) + detail = '\n'.join(part.strip() for part in detail_parts if part and part.strip()) + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + cmd[3] + '\n' + with open(err_file, 'a') as f: + f.write(str0) + if detail: + f.write(detail) + f.write('\n') + raise RuntimeError(str0.strip() + ' failed with rc=' + str(p.returncode) + ('\n' + detail if detail else '')) + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + cmd[3] + '\n' + with open(err_file, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + Geneate differential interferograms for one project using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + diff_gamma_all.py projectName + diff_gamma_all.py projectName --parallel 4 + diff_gamma_all.py projectName --parallel 4 --ifgramList-txt /test/ifgram_list.txt +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Geneate differential interferograms for one project using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + parser.add_argument('--ifgarmList-txt', dest='ifgarmListTxt', help='provided ifgram_list_txt. default: using ifgram_list.txt under projectName folder.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + projectDir = scratchDir + '/' + projectName + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + if inps.ifgarmListTxt: ifgramList_txt = inps.ifgarmListTxt + else: ifgramList_txt = scratchDir + '/' + projectName + '/ifgram_list.txt' + ifgList0 = ut.read_txt2array(ifgramList_txt) +# ifgList = ifgList0[:,0] + if len(ifgList0)==3: + ifgList=ifgList0[0] + ifgList=[ifgList] + + else: + ifgList=ifgList0[:,0] + + err_txt = scratchDir + '/' + projectName + '/diff_gamma_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + for i in range(len(ifgList)): + m0 = ut.yyyymmdd(ifgList[i].split('-')[0]) + s0 = ut.yyyymmdd(ifgList[i].split('-')[1]) + cmd0 = ['diff_gamma.py',projectName, m0, s0] + diff_file0 = ifgDir + '/' + ifgList[i] + '/' + ifgList[i] + '_' + rlks + 'rlks.diff_filt.bmp' + data0 = [cmd0,err_txt] + + k00 = 0 + if os.path.isfile(diff_file0): + if os.path.getsize(diff_file0) > 0: + k00 = 1 + if k00==0: + data_para.append(data0) + + results = ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + failures = [str(item) for item in results if isinstance(item, Exception)] + if failures: + raise RuntimeError('\n\n'.join(failures)) + print("Generate differential interferograms for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_LT1.py b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_LT1.py new file mode 100644 index 0000000..3c1f69f --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_LT1.py @@ -0,0 +1,228 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.2 ### +### Copy Right (c): 2017-2019, Chen Wei ### +### Author: chenwei ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import re +import shutil +import tarfile + +from pyint import _orbit_bridge as orbit_bridge +from pyint import _utils as ut + + +def get_LT1_date(raw_file): + file0 = os.path.basename(raw_file) + date = file0[41:48] + return date + +def get_satellite(raw_file): + if 'LT1A_MONO_' in raw_file: + s0 = 'A' + else: + s0 = 'B' + return s0 + + +def discover_lt1_inputs(down_dir, date): + candidates = [] + for pattern in ( + down_dir + '/LT1*' + date + '*.tar.gz', + down_dir + '/LT1*' + date + '*.tiff', + ): + candidates.extend(glob.glob(pattern)) + return sorted(set(candidates)) + + +def write_input_list(list_path, paths): + with open(list_path, 'w') as f: + for path in paths: + f.write(path + '\n') + + +def _run_checked(command, cwd=None): + result = subprocess.run( + command, + cwd=cwd, + text=True, + capture_output=True, + check=False, + ) + if result.returncode != 0: + detail = (result.stderr or result.stdout or "").strip() + raise RuntimeError( + 'Command failed (%s): %s%s' % ( + result.returncode, + ' '.join(command), + ('\n' + detail) if detail else '', + ) + ) + return result + + +def _cleanup_paths(paths): + for path in paths: + if not path: + continue + try: + if os.path.isdir(path): + shutil.rmtree(path) + elif os.path.isfile(path): + os.remove(path) + except OSError: + continue + + +def _resolve_lt1_input_scene(raw_path, work_dir): + cleanup_paths = [] + raw_lower = raw_path.lower() + if raw_lower.endswith('.tiff'): + input_xml = re.sub(r'\.tiff$', '.meta.xml', raw_path, flags=re.IGNORECASE) + if not os.path.isfile(input_xml): + raise FileNotFoundError('LT-1 meta xml does not exist: ' + input_xml) + return raw_path, input_xml, cleanup_paths + + if raw_lower.endswith('.tar.gz'): + temp_dir = os.path.join(work_dir, 'tmp_data_dir') + if os.path.isdir(temp_dir): + shutil.rmtree(temp_dir) + os.makedirs(temp_dir, exist_ok=True) + cleanup_paths.append(temp_dir) + with tarfile.open(raw_path, 'r:gz') as archive: + member_names = archive.getnames() + tiff_members = [name for name in member_names if name.lower().endswith('.tiff')] + xml_members = [name for name in member_names if name.lower().endswith('meta.xml')] + if not tiff_members or not xml_members: + raise RuntimeError('LT-1 archive is missing .tiff or meta.xml: ' + raw_path) + tiff_member = tiff_members[0] + xml_member = xml_members[0] + archive.extract(tiff_member, path=temp_dir) + archive.extract(xml_member, path=temp_dir) + return os.path.join(temp_dir, tiff_member), os.path.join(temp_dir, xml_member), cleanup_paths + + raise RuntimeError('Unsupported LT-1 input, only .tiff or .tar.gz are supported: ' + raw_path) + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLC from LT1 raw data with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='project name. e.g., ChangningT55S1A') + parser.add_argument('date',help='date to be processed. e.g., 20180101') + inps = parser.parse_args() + + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Generate SLC from Sentinel-1 raw data using S1_import_SLC_from_zipfiles with orbit correction. + [Precise orbit data will be downloaded automatically] +''' + +EXAMPLE = """Usage: + + down2slc_sen.py projectName date + + down2slc_sen.py ChangningT55S1A 20180517 + +------------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + date = inps.date + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + slc_dir = projectDir + '/SLC' + down_dir = projectDir + '/DOWNLOAD' + + if not os.path.isdir(slc_dir): + os.mkdir(slc_dir) + + work_dir = slc_dir + '/' + date + if not os.path.isdir(work_dir): + os.mkdir(work_dir) + + os.chdir(work_dir) + + t_date = 't_' + date + + input_files = discover_lt1_inputs(down_dir, date) + write_input_list(t_date, input_files) + raw_files = ut.read_txt2list(t_date) + if len(raw_files) == 0: + raise RuntimeError('No LT-1 inputs found for date: ' + date) + satellite = get_satellite(str(raw_files[0])) + + zipfile_ref=str(raw_files[0]) + outfile_name=zipfile_ref.split('/')[-1].split('.')[0] + outfile_name=zipfile_ref.split('/')[-1] + print(outfile_name) + + cleanup_paths = [] + try: + input_tiff, input_xml, cleanup_paths = _resolve_lt1_input_scene(zipfile_ref, work_dir) + slc_path = work_dir + '/' + date + '.slc' + slc_par_path = work_dir + '/' + date + '.slc.par' + _run_checked( + ['par_LT1_SLC', input_tiff, input_xml, slc_par_path, slc_path], + cwd=work_dir, + ) + if not os.path.isfile(slc_path) or not os.path.isfile(slc_par_path): + raise RuntimeError('LT-1 import produced no SLC outputs for date: ' + date) + bridge_result = orbit_bridge.apply_precise_orbit( + date, + [slc_par_path], + work_dir=work_dir, + operation_tag='lt1_import', + ) + if bridge_result.get('stdout'): + print(bridge_result['stdout']) + if bridge_result.get('stderr'): + print(bridge_result['stderr']) + finally: + _cleanup_paths(cleanup_paths) + + + SLC_Tab = work_dir + '/' + date+'_SLC_Tab' + SLC_list = sorted(glob.glob(work_dir + '/*.slc')) + SLC_par_list = sorted(glob.glob(work_dir + '/*.slc.par')) + if len(SLC_list) == 0 or len(SLC_par_list) == 0: + raise RuntimeError('No LT-1 SLC outputs were generated for date: ' + date) + if len(SLC_list) != len(SLC_par_list): + raise RuntimeError('LT-1 SLC and SLC parameter counts do not match for date: ' + date) + + if os.path.isfile(SLC_Tab): + os.remove(SLC_Tab) + + for kk in range(len(SLC_list)): + call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' >> ' + SLC_Tab + if os.system(call_str) != 0: + raise RuntimeError('Failed to write SLC tab for date: ' + date) + with open(work_dir + '/down2slc.dat', 'w') as f: + f.write('ok\n') + print("Down to SLC for %s is done! " % date) + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_LT1_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_LT1_all.py new file mode 100644 index 0000000..09e167f --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_LT1_all.py @@ -0,0 +1,185 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.2 ### +### Copy Right (c): 2017-2019, Chen Wei ### +### Author: chenwei ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse +import re + +import subprocess +from pyint import _utils as ut + +def get_LT1_date(raw_file): + file0 = os.path.basename(raw_file) + match = re.search(r'(20\d{6})', file0) + if match: + return match.group(1) + return '' + + +def discover_lt1_inputs(down_dir): + candidates = [] + for pattern in ('/LT1*.tar.gz', '/LT1*.tiff'): + candidates.extend(glob.glob(down_dir + pattern)) + return sorted(set(candidates)) + +def work(data0): + cmd = data0[0] + err_txt = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if type(stdout) == bytes: + bb=stdout.decode("utf-8", errors="replace") + else: + bb = stdout + + if p.returncode != 0: + detail_parts = [] + if bb: + detail_parts.append(bb) + if aa: + detail_parts.append(aa) + detail = '\n'.join(part.strip() for part in detail_parts if part and part.strip()) + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + with open(err_txt, 'a') as f: + f.write(str0) + if detail: + f.write(detail) + f.write('\n') + raise RuntimeError(str0.strip() + ' failed with rc=' + str(p.returncode) + ('\n' + detail if detail else '')) + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + with open(err_txt, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + down2slc_LT1_all.py projectName + down2slc_LT1_all.py projectName --parallel 4 + +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + slcDir = scratchDir + '/' + projectName + "/SLC" + raw_file_list = discover_lt1_inputs(downDir) + if len(raw_file_list) == 0: + raise RuntimeError('No LT-1 inputs found under: ' + downDir) + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + + date_list = [] + cat_list = [] + for kk in range(len(raw_file_list)): + date0 = get_LT1_date(os.path.basename(raw_file_list[kk])) + if not date0: + continue + if date0 not in date_list: + date_list.append(date0) + cat_list.append('0') + else: + cat_list[date_list.index(date0)]='1' +# date_list = set(date_list) +# date_list = sorted(date_list) + + print('Date to be processed:') + for k0 in date_list: + print(k0) + + err_txt = scratchDir + '/' + projectName + '/down2slc_sen_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + for i in range(len(date_list)): + if cat_list[i]=='0': + cmd0 = ['down2slc_LT1.py',projectName,date_list[i]] + work_dir = slcDir + '/' + date_list[i] + slc_file0 = work_dir + '/down2slc.dat' + data0 = [cmd0,err_txt] +# data_para.append(data0) + k00 = 0 + if os.path.isfile(slc_file0): + if os.path.getsize(slc_file0) > 0: + k00 = 1 + else: + k00 = 0 + if k00==0: + data_para.append(data0) + else: + cmd0 = ['down2slc_cat_LT1.py',projectName,date_list[i]] + work_dir = slcDir + '/' + date_list[i] + slc_file0 = work_dir + '/down2slc.dat' + data0 = [cmd0,err_txt] +# data_para.append(data0) + #data_para.append(data0) + k00 = 0 + if os.path.isfile(slc_file0): + if os.path.getsize(slc_file0) > 0: + k00 = 1 + else: + k00 = 0 + if k00==0: + data_para.append(data0) + results = ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + failures = [str(item) for item in results if isinstance(item, Exception)] + if failures: + raise RuntimeError('\n\n'.join(failures)) + os.chdir(downDir) + print("Down to SLC for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_alos_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_alos_all.py new file mode 100644 index 0000000..3f08918 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_alos_all.py @@ -0,0 +1,138 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + + +def get_s1_date(raw_file): + file0 = os.path.basename(raw_file) + date = file0[17:25] + return date + + +def work(data0): + cmd = data0[0] + err_txt = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + #print(aa) + with open(err_txt, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + down2slc_sen_all.py projectName + down2slc_sen_all.py projectName --parallel 4 + +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + + raw_file_list = glob.glob(downDir + '/ALPSRP*.zip') + slc_dir = scratchDir + '/' + projectName + '/SLC' + if not os.path.isdir(slc_dir): + os.mkdir(slc_dir) + + err_txt = scratchDir + '/' + projectName + '/unzip_alos_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) +########get the ALOS date name ############### + data_para = [] + date_list = [] + for i in range(len(raw_file_list)): + cmd0 = ['unzip',raw_file_list[i],'-d',downDir] + call_str='unzip ' + raw_file_list[i] + ' -d ' + downDir + os.system(call_str) + workflow_file=raw_file_list[i].split(".")[0] + '.' +raw_file_list[i].split(".")[1] + os.chdir(workflow_file) + tempfile=workflow_file + '/' + 'workreport' + tempDict=ut.update_template(tempfile) + SeneceData = tempDict['Img_SceneCenterDateTime'].split('"')[1].split(' ')[0] + SceneID = tempDict['Scs_SceneID'] + os.chdir(downDir) + call_str= 'mv ' + workflow_file + ' ' + SeneceData + os.system(call_str) + data0=[SeneceData, err_txt] + data_para.append(data0) + date0=SeneceData + date_list.append(date0) + date_list = set(date_list) + date_list = sorted(date_list) + + print('Date to be processed:') + for k0 in date_list: + print(k0) + + err_txt = scratchDir + '/' + projectName + '/down2slc_alos_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + for i in range(len(date_list)): + cmd0 = ['Down2SLC_ALOS.py',projectName,date_list[i]] + data0 = [cmd0,err_txt] + data_para.append(data0) + + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + os.chdir(downDir) + print("Down to SLC for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_cat_LT1.py b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_cat_LT1.py new file mode 100644 index 0000000..04e02fd --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_cat_LT1.py @@ -0,0 +1,202 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.2 ### +### Copy Right (c): 2017-2019, Chen Wei ### +### Author: chenwei ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import re + +from pyint import _orbit_bridge as orbit_bridge +from pyint import _utils as ut + + +def get_LT1_date(raw_file): + file0 = os.path.basename(raw_file) + date = file0[41:48] + return date + +def get_satellite(raw_file): + if 'LT1A_MONO_' in raw_file: + s0 = 'A' + else: + s0 = 'B' + return s0 + + +def discover_lt1_inputs(down_dir, date): + candidates = [] + for pattern in ( + down_dir + '/LT1*' + date + '*.tar.gz', + down_dir + '/LT1*' + date + '*.tiff', + ): + candidates.extend(glob.glob(pattern)) + return sorted(set(candidates)) + + +def write_input_list(list_path, paths): + with open(list_path, 'w') as f: + for path in paths: + f.write(path + '\n') + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLC from LT1 raw data with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='project name. e.g., ChangningT55S1A') + parser.add_argument('date',help='date to be processed. e.g., 20180101') + inps = parser.parse_args() + + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Generate SLC from Sentinel-1 raw data using S1_import_SLC_from_zipfiles with orbit correction. + [Precise orbit data will be downloaded automatically] +''' + +EXAMPLE = """Usage: + + down2slc_LT1.py projectName date + + down2slc_LT1.py ChangningT55S1A 20180517 + +------------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + date = inps.date + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + slc_dir = projectDir + '/SLC' + down_dir = projectDir + '/DOWNLOAD' + + if not os.path.isdir(slc_dir): + os.mkdir(slc_dir) + + work_dir = slc_dir + '/' + date + if not os.path.isdir(work_dir): + os.mkdir(work_dir) + + os.chdir(work_dir) + + t_date = 't_' + date + + input_files = discover_lt1_inputs(down_dir, date) + write_input_list(t_date, input_files) + raw_files = ut.read_txt2list(t_date) + if len(raw_files) == 0: + raise RuntimeError('No LT-1 inputs found for date: ' + date) + satellite = get_satellite(str(raw_files[0])) + raw_file_list = list(raw_files) + + + + file_num=len(raw_files) + for kk in range(file_num): + + zipfile_ref=str(raw_files[kk]) + outfile_name=zipfile_ref.split('/')[-1] + print(outfile_name) + before_slc = set(glob.glob(work_dir + '/*.slc')) + before_slc_par = set(glob.glob(work_dir + '/*.slc.par')) + before_update = set(glob.glob(work_dir + '/*.slc.update')) + before_update_par = set(glob.glob(work_dir + '/*.slc.update.par')) + call_str = "echo " + zipfile_ref + " >date" + if os.system(call_str) != 0: + raise RuntimeError('Failed to materialize LT-1 input list for date: ' + date) + call_str = 'LT1_import_SLC_from_zipfiles1 date 0 ' + rc = os.system(call_str) + if rc != 0: + raise RuntimeError('LT1_import_SLC_from_zipfiles1 failed for date %s with rc=%s' % (date, rc)) + after_slc = set(glob.glob(work_dir + '/*.slc')) + after_slc_par = set(glob.glob(work_dir + '/*.slc.par')) + after_update = set(glob.glob(work_dir + '/*.slc.update')) + after_update_par = set(glob.glob(work_dir + '/*.slc.update.par')) + new_slc = sorted(after_slc - before_slc) + new_slc_par = sorted(after_slc_par - before_slc_par) + new_update = sorted(after_update - before_update) + new_update_par = sorted(after_update_par - before_update_par) + if not new_slc or not new_slc_par: + raise RuntimeError('LT-1 import produced no SLC outputs for date: ' + date) + if len(new_update) != len(new_update_par): + raise RuntimeError('LT-1 import produced mismatched update SLC outputs for date: ' + date) + bridge_targets = sorted((after_slc_par - before_slc_par) | (after_update_par - before_update_par)) + if bridge_targets: + bridge_result = orbit_bridge.apply_precise_orbit( + date, + bridge_targets, + work_dir=work_dir, + operation_tag='lt1_import', + ) + if bridge_result.get('stdout'): + print(bridge_result['stdout']) + if bridge_result.get('stderr'): + print(bridge_result['stderr']) + + + SLC_Tab = work_dir + '/' + date+'_SLC_Tab' + SLC_Tab_update = work_dir + '/' + date+'_update_SLC_Tab' + SLC_update_list = sorted(glob.glob(work_dir + '/*.slc.update')) + SLC_update_par_list = sorted(glob.glob(work_dir + '/*.slc.update.par')) + SLC_list = sorted(glob.glob(work_dir + '/*.slc')) + SLC_par_list = sorted(glob.glob(work_dir + '/*.slc.par')) + if len(SLC_list) == 0 or len(SLC_par_list) == 0: + raise RuntimeError('No LT-1 SLC outputs were generated for date: ' + date) + if len(SLC_list) != len(SLC_par_list): + raise RuntimeError('LT-1 SLC and SLC parameter counts do not match for date: ' + date) + if len(SLC_update_list) != len(SLC_update_par_list): + raise RuntimeError('LT-1 update SLC and parameter counts do not match for date: ' + date) + + if os.path.isfile(SLC_Tab): + os.remove(SLC_Tab) + + for kk in range(len(SLC_list)): + call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' >> ' + SLC_Tab + if os.system(call_str) != 0: + raise RuntimeError('Failed to write SLC tab for date: ' + date) + call_str = 'echo ' + SLC_update_list[kk] + ' ' + SLC_update_par_list[kk] + ' >> ' + SLC_Tab_update + if os.system(call_str) != 0: + raise RuntimeError('Failed to write update SLC tab for date: ' + date) + call_str = 'SLC_cat_list.py ' + SLC_Tab_update + ' ' + date + '.slc ' + date + '.slc.par ' + if os.system(call_str) != 0: + raise RuntimeError('SLC_cat_list.py failed for date: ' + date) + if not os.path.isfile(work_dir + '/' + date + '.slc.par'): + raise RuntimeError('Final LT-1 concatenated SLC parameter file is missing for date: ' + date) + final_bridge_result = orbit_bridge.apply_precise_orbit( + date, + [work_dir + '/' + date + '.slc.par'], + work_dir=work_dir, + operation_tag='slc_cat_final', + ) + if final_bridge_result.get('stdout'): + print(final_bridge_result['stdout']) + if final_bridge_result.get('stderr'): + print(final_bridge_result['stderr']) + with open(work_dir + '/down2slc.dat', 'w') as f: + f.write('ok\n') + print("Down to SLC for %s is done! " % date) + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_cat_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_cat_all.py new file mode 100644 index 0000000..2c9fc68 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_cat_all.py @@ -0,0 +1,148 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Central South University & University of Miami ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def is_number(s): + try: + int(s) + return True + except ValueError: + return False + + +def ras2jpg(input, strTitle): + call_str = "convert " + input + ".ras " + input + ".jpg" + os.system(call_str) + call_str = "convert " + input + ".jpg -resize 250 " + input + ".thumb.jpg" + os.system(call_str) + call_str = "convert " + input + ".jpg -resize 500 " + input + ".bthumb.jpg" + os.system(call_str) + call_str = "$INT_SCR/addtitle2jpg.pl " + input + ".thumb.jpg 14 " + strTitle + os.system(call_str) + call_str = "$INT_SCR/addtitle2jpg.pl " + input + ".bthumb.jpg 24 " + strTitle + os.system(call_str) + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +def write_template(File, Str): + f = open(File,'a') + f.write(Str) + f.close() + +def write_run_coreg_all(projectName,master,slavelist,workdir): + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + run_coreg_all = projectDir + "/run_coreg_all" + f_coreg = open(run_coreg_all,'w') + + for kk in range(len(slavelist)): + str_coreg = "GenOff_Gamma.py " + projectName + ' ' + master + ' ' + slavelist[kk] + ' ' + workdir + '\n' + f_coreg.write(str_coreg) + f_coreg.close() + + +def usage(): + print(''' +****************************************************************************************************** + + SLC cat for time-series of SAR datasets. + + usage: + + down2slc_cat_all.py projectName + + e.g. down2slc_cat_all.py PacayaERST163A + down2slc_cat_all.py PacayaEnvT163A + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==2: + if argv[0] in ['-h','--help']: usage(); sys.exit(1) + else: projectName=sys.argv[1] + else: + usage();sys.exit(1) + + if 'ERS' in projectName: + call_str='Down2SLC_ERS_Cat_All.py ' + projectName + else: + call_str='Down2SLC_ASAR_Cat_All.py ' + projectName + + os.system(call_str) + + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + + + + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_cat_sen.py b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_cat_sen.py new file mode 100644 index 0000000..20e331c --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_cat_sen.py @@ -0,0 +1,216 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import csv + +from pyint import _utils as ut + + +def get_s1_date(raw_file): + file0 = os.path.basename(raw_file) + date = file0[17:25] + return date + +def get_s1_time(raw_file): + file0 = os.path.basename(raw_file) + times= file0[26:32] + return times + +def get_satellite(raw_file): + if 'S1A_IW_SLC_' in raw_file: + s0 = 'A' + else: + s0 = 'B' + + return s0 + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLC from Sentinel-1 raw data with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='project name. e.g., ChangningT55S1A') + parser.add_argument('date',help='date to be processed. e.g., 20180101') + + inps = parser.parse_args() + + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Cat two SLC from Sentinel-1 raw data using S1_import_SLC_from_zipfiles and SLC_cat_S1_TOPS with orbit correction. + [Precise orbit data will be downloaded automatically] +''' + +EXAMPLE = """Usage: + + down2slc_sen.py projectName date + + down2slc_sen.py ChangningT55S1A 20180517 + +------------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + date = inps.date + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + slc_dir = projectDir + '/SLC' + down_dir = projectDir + '/DOWNLOAD' + #opod_dir = projectDir + '/OPOD' + opod_dir = os.getenv('OPOD_DIR') + + + #if not os.path.isdir(opod_dir): + # os.mkdir(opod_dir) + + call_str = " eof --save-dir " + opod_dir + " -p " + down_dir + os.system(call_str) + if not os.path.isdir(slc_dir): + os.mkdir(slc_dir) + + work_dir = slc_dir + '/' + date + if not os.path.isdir(work_dir): + os.mkdir(work_dir) + + os.chdir(work_dir) + + t_date = 't_' + date + + call_str = 'ls ' + down_dir + '/S1*' + date + '* > ' + t_date + os.system(call_str) + + start_swath = templateDict['start_swath'] + end_swath = templateDict['end_swath'] + +# if (start_swath == '1') and (end_swath == '1'): +# k_swath = '1' +# elif (start_swath == '2') and (end_swath == '2'): +# k_swath = '2' +# elif (start_swath == '3') and (end_swath == '3'): +# k_swath = '3' +# elif (start_swath == '1') and (end_swath == '2'): +# k_swath = '4' +# elif (start_swath == '2') and (end_swath == '3'): +# k_swath = '5' +# elif (start_swath == '2') and (end_swath == '3'): +# k_swath = '-' + k_swath = ut.get_sardata_swath(start_swath,end_swath) + + raw_files = ut.read_txt2list(t_date) + satellite = get_satellite(str(raw_files[0])) + #orbit_file = ut.download_s1_orbit(date,opod_dir,satellite=satellite) + raw_file_list = glob.glob(down_dir + '/S1*' + date + '*.zip') + file_num=len(raw_files) + call_str = 'S1_BURST_tab_from_zipfile.py 3 --zip_ref_list ' + t_date + ' --zip_list ' + t_date + os.system(call_str) + + for kk in range(file_num): + zipfile_ref=str(raw_files[kk]) + outfile_name=zipfile_ref.split('/')[-1].split('.')[0] + burst_number_table_ref=outfile_name + '.BURST_tab' + #call_str = 'S1_BURST_tab_from_zipfile ' + t_date + ' ' + str(raw_files[kk]) + ' - 1' + #os.system(call_str) + #call_str = 'S1_import_SLC_from_zipfiles ' + t_date + ' ' + burst_number_table_ref + ' vv 0 ' + k_swath + ' ' + opod_dir + ' 1 1 ' + call_str = 'read_S1_TOPS_SLC.py ' + zipfile_ref + ' --burst_sel ' + burst_number_table_ref + ' --pol vv --root_name ' + date + ' --sw_start ' + start_swath + ' --swn ' + end_swath + ' --OPOD_dir ' + opod_dir + os.system(call_str) + os.chdir(work_dir) + call_str = "rename 's/vv.iw1.slc/iw1_" + str(kk)+".slc/g' *" + os.system(call_str) + call_str = "rename 's/vv.iw2.slc/iw2_" + str(kk)+".slc/g' *" + os.system(call_str) + call_str = "rename 's/vv.iw3.slc/iw3_" + str(kk)+".slc/g' *" + os.system(call_str) + SLC_Tab = work_dir + '/' + date + '_SLC_Tab' +str(kk) + SLC_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc')) + SLC_par_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc.par')) + TOP_par_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc.tops_par')) + + + cat_str = 'touch slc_list slc_par_list top_par_list' + os.system(call_str) + list_num=len(SLC_list) + for tt in range(list_num): + call_str = 'echo ' + SLC_list[tt] + ' >> slc_list' + os.system(call_str) + call_str = 'echo ' + SLC_par_list[tt] + ' >> slc_par_list' + os.system(call_str) + call_str = 'echo ' + TOP_par_list[tt] + ' >> top_par_list' + os.system(call_str) + call_str = ' paste slc_list slc_par_list top_par_list > ' + SLC_Tab + os.system(call_str) + call_str = 'rm slc_list slc_par_list top_par_list' + os.system(call_str) + SLC_list = sorted(glob.glob(work_dir + '/*iw*.slc')) + SLC_Tab = work_dir + '/' + date+'_SLC_Tab' + subswath=len(SLC_list)/2 + for kk in range(int(subswath)): + swath=kk+1 + call_str = 'echo ' + date + '.IW' + str(swath) + '.slc ' + date + '.IW' + str(swath) + '.slc.par ' + date + '.IW' + str(swath) + '.slc.TOPS_par ' + '>> ' + SLC_Tab + os.system(call_str) + call_str = 'SLC_cat_S1_TOPS ' +SLC_Tab+'0 ' + SLC_Tab +'1 ' + SLC_Tab + os.system(call_str) + + #call_str = "rename vv.slc.iw1 IW1.slc * " + #os.system(call_str) + #call_str = "rename 's/vv.slc.iw2/IW2.slc/g' *" + #call_str = "rename vv.slc.iw2 IW2.slc * " + #os.system(call_str) + #call_str = "rename 's/vv.slc.iw3/IW3.slc/g' *" + #call_str = "rename vv.slc.iw3 IW3.slc * " + #os.system(call_str) + #call_str = 'rm *.iw* *SLC_Tab*' + #os.system(call_str) + + SLC_Tab = work_dir + '/' + date+'_SLC_Tab' + SLC_list = sorted(glob.glob(work_dir + '/*IW*.slc')) + SLC_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.par')) + TOP_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.TOPS_par')) + + #call_str = 'rm *iw* *vv.SLC_tab' + #os.system(call_str) + if os.path.isfile(SLC_Tab): + os.remove(SLC_Tab) + + for kk in range(len(SLC_list)): + call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' >> ' + SLC_Tab + os.system(call_str) + + BURST = SLC_par_list[kk].replace('slc.par','burst.par') + call_str = 'SLC_burst_corners ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' > ' +BURST + os.system(call_str) + call_str = "echo 'SLC has already down' >down2slc.dat" + os.system(call_str) + print("Down to SLC for %s is done! " % date) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_csk.py b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_csk.py new file mode 100644 index 0000000..a9219cf --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_csk.py @@ -0,0 +1,199 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.0 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Now at KAUST ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import linecache +import datetime + +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + + +def check_ERS_par(SAR_IM_0P): + Name = os.path.basename(SAR_IM_0P) + ff = Name.split('.')[1] + date = (Name.split('SAR_IM__0PWDSI')[1]).split('_')[0] + + if ff == 'E1': + par = 'ERS1_ESA.par' + antenna = 'ERS1_antenna.gain' + orbdir = os.getenv('ERS1ORBDIR') + elif ff == 'E2': + par = 'ERS2_ESA.par' + antenna = 'ERS2_antenna.gain' + orbdir = os.getenv('ERS2ORBDIR') + else: + print('Invalid input SAR_IM_0P file.') + sys.exit(1) + + return par, antenna, orbdir, date + +######################################################################### + +INTRODUCTION = ''' +############################################################################# + Copy Right(c): 2017-2020, Yunmeng Cao @PyINT v2.0 + + Generate SLC for CSK raw data (with **.tar.gz format). + +''' + +EXAMPLE = ''' + Usage: + down2slc_csk.py projectName date + + Examples: + down2slc_csk.py KilaueaT10CskA 20180419 + +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLC for ERS raw data with ENVISAT format.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='project name.') + parser.add_argument('date',help='to be processed date.') + + inps = parser.parse_args() + return inps + +################################################################################ + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + date = inps.date + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + + if not os.path.isdir(slcDir): + call_str = 'mkdir ' + slcDir + os.system(call_str) + + slcDir1 = slcDir + '/' + date + if not os.path.isdir(slcDir1): + call_str = 'mkdir ' + slcDir1 + os.system(call_str) + + tar0 = glob.glob(downDir + '/*' + date + '*.tar.gz')[0] + rawDir = downDir + '/raw_' + date + + if not os.path.isdir(rawDir): + call_str = 'mkdir ' + downDir + '/raw_' + date + os.system(call_str) + + call_str = 'tar -xzf ' + tar0 + ' -C ' + rawDir + os.system(call_str) + + par = rawDir + '/' + date + '.sar_par' + raw = rawDir + '/' + date + '.raw' + azsp = rawDir + '/' + date + '.azsp' + dop = rawDir + '/' + date + '.dop' + rspec = rawDir + '/' + date + '.rspec' + rc = rawDir + '/' + date + '.rc' + autof = rawDir + '/' + date + '.autof' + dop_ambig = rawDir + '/' + date + '.dop_ambig' + + h5file = glob.glob(rawDir + '/*.h5')[0] + + pslc_par = rawDir + '/' + 'p' + date + '.slc.par' + slc_par = slcDir1 + '/' + date + '.slc.par' + slc = slcDir1 + '/' +date + '.slc' + mli_par = slcDir1 + '/' +date + '.mli.par' + mli = slcDir1 + '/' + date + '.mli' + + call_str = 'CS_proc ' + h5file + ' ' + par + ' ' + pslc_par + ' ' + raw + ' - - ' + os.system(call_str) + + cal_str = 'dop_ambig ' + par + ' ' + pslc_par + ' ' + raw + ' 2 - ' + dop_ambig + os.system(call_str) + + call_str = 'azsp_IQ ' + par + ' ' + pslc_par + ' ' + raw + ' ' + azsp + os.system(call_str) + + call_str = 'doppler ' + par + ' ' + pslc_par + ' ' + raw + ' ' + dop + os.system(call_str) + + call_str = 'rspec_IQ ' + par + ' ' + pslc_par + ' ' + raw + ' ' + rspec + os.system(call_str) + + call_str = 'pre_rc ' + par + ' ' + pslc_par + ' ' + raw + ' ' + rc + os.system(call_str) + + call_str = 'autof ' + par + ' ' + pslc_par + ' ' + rc + ' ' + autof + ' 2.0 ' + os.system(call_str) + + call_str = 'autof ' + par + ' ' + pslc_par + ' ' + rc + ' ' + autof + ' 2.0 ' + os.system(call_str) + + call_str = 'az_proc ' + par + ' ' + pslc_par + ' ' + rc + ' ' + slc + ' - 1 ' + ' - 0 2.120 ' + os.system(call_str) + + call_str = 'par_MSP ' + par + ' ' + pslc_par + ' ' + slc_par + ' 1' + os.system(call_str) + + call_str = 'multi_look ' + slc + ' ' + slc_par + ' ' + mli + ' ' + mli_par + ' 20 16' + os.system(call_str) + + + Width = UseGamma(mli_par, 'read', 'range_samples: ') + call_str = 'raspwr ' + mli + ' ' + Width + os.system(call_str) + + + print("Generate SLC for %s is done." % date) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_csk_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_csk_all.py new file mode 100644 index 0000000..7bd2d8e --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_csk_all.py @@ -0,0 +1,119 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + + +def get_csk_date(raw_file): + file0 = os.path.basename(raw_file) + date = file0[27:35] + return date + + +def work(data0): + cmd = data0[0] + err_txt = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + #print(aa) + with open(err_txt, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + down2slc_sen_all.py projectName + down2slc_sen_all.py projectName --parallel 4 + +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + raw_file_list = glob.glob(downDir + '/CSK*.tar.gz') + + # get the burst number table of the mater date + + date_list = [] + for kk in range(len(raw_file_list)): + date0 = get_csk_date(os.path.basename(raw_file_list[kk])) + date_list.append(date0) + + date_list = set(date_list) + date_list = sorted(date_list) + + print('Date to be processed:') + for k0 in date_list: + print(k0) + + err_txt = scratchDir + '/' + projectName + '/down2slc_csk_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + for i in range(len(date_list)): + cmd0 = ['down2slc_csk.py',projectName,date_list[i]] + data0 = [cmd0,err_txt] + data_para.append(data0) + + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + os.chdir(downDir) + print("Down to SLC for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_sen.py b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_sen.py new file mode 100644 index 0000000..953fe87 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_sen.py @@ -0,0 +1,167 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse + +from pyint import _utils as ut + + +def get_s1_date(raw_file): + file0 = os.path.basename(raw_file) + date = file0[17:25] + return date + +def get_satellite(raw_file): + if 'S1A_IW_SLC_' in raw_file: + s0 = 'A' + else: + s0 = 'B' + + return s0 + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLC from Sentinel-1 raw data with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='project name. e.g., ChangningT55S1A') + parser.add_argument('date',help='date to be processed. e.g., 20180101') + + inps = parser.parse_args() + + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Generate SLC from Sentinel-1 raw data using S1_import_SLC_from_zipfiles with orbit correction. + [Precise orbit data will be downloaded automatically] +''' + +EXAMPLE = """Usage: + + down2slc_sen.py projectName date + + down2slc_sen.py ChangningT55S1A 20180517 + +------------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + date = inps.date + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + slc_dir = projectDir + '/SLC' + down_dir = projectDir + '/DOWNLOAD' + #opod_dir = projectDir + '/OPOD' + opod_dir = os.getenv('OPOD_DIR') + if not os.path.isdir(slc_dir): + os.mkdir(slc_dir) + # if not os.path.isdir(opod_dir): + # os.mkdir(opod_dir) + + work_dir = slc_dir + '/' + date + if not os.path.isdir(work_dir): + os.mkdir(work_dir) + #if not os.path.isdir(opod_dir): + # os.mkdir(opod_dir) + + + call_str = " eof --save-dir " + opod_dir + " -p " + down_dir + os.system(call_str) + + os.chdir(work_dir) + + t_date = 't_' + date + + call_str = 'ls ' + down_dir + '/S1*' + date + '*.zip > ' + t_date + os.system(call_str) + + start_swath = templateDict['start_swath'] + end_swath = templateDict['end_swath'] + +# if (start_swath == '1') and (end_swath == '1'): +# k_swath = '1' +# elif (start_swath == '2') and (end_swath == '2'): +# k_swath = '2' +# elif (start_swath == '3') and (end_swath == '3'): +# k_swath = '3' +# elif (start_swath == '1') and (end_swath == '2'): +# k_swath = '4' +# elif (start_swath == '2') and (end_swath == '3'): +# k_swath = '5' +# elif (start_swath == '2') and (end_swath == '3'): +# k_swath = '-' + k_swath = ut.get_sardata_swath(start_swath,end_swath) + + raw_files = ut.read_txt2list(t_date) + satellite = get_satellite(str(raw_files[0])) + #orbit_file = ut.download_s1_orbit(date,opod_dir,satellite=satellite) + zipfile_ref=str(raw_files[0]) + outfile_name=zipfile_ref.split('/')[-1].split('.')[0] + burst_number_table_ref=outfile_name + '.BURST_tab' + + call_str = 'S1_BURST_tab_from_zipfile.py 3 --zip_ref_list ' + t_date + ' --zip_list ' + t_date + os.system(call_str) + + # call_str = 'S1_import_SLC_from_zipfiles ' + t_date + ' ' + burst_number_table_ref + ' vv 0 ' + k_swath + call_str = 'read_S1_TOPS_SLC.py ' + zipfile_ref + ' --burst_sel ' + burst_number_table_ref + ' --pol vv --root_name ' + date + ' --sw_start ' + start_swath + ' --swn ' + end_swath + ' --OPOD_dir ' + opod_dir + os.system(call_str) + + os.chdir(work_dir) + + call_str = "rename 's/vv.iw1.slc/IW1.slc/g' *" + #call_str = "rename vv.slc.iw1 IW1.slc * " + os.system(call_str) + call_str = "rename 's/vv.iw2.slc/IW2.slc/g' *" + #call_str = "rename vv.slc.iw2 IW2.slc * " + os.system(call_str) + call_str = "rename 's/vv.iw3.slc/IW3.slc/g' *" + #call_str = "rename vv.slc.iw3 IW3.slc * " + os.system(call_str) + call_str = "rename 's/tops_par/TOPS_par/g' *.tops_par " + os.system(call_str) + + SLC_Tab = work_dir + '/' + date+'_SLC_Tab' + SLC_list = sorted(glob.glob(work_dir + '/*IW*.slc')) + SLC_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.par')) + TOP_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.TOPS_par')) + + if os.path.isfile(SLC_Tab): + os.remove(SLC_Tab) + + for kk in range(len(SLC_list)): + call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' >> ' + SLC_Tab + os.system(call_str) + + BURST = SLC_par_list[kk].replace('slc.par','burst.par') + call_str = 'SLC_burst_corners ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' > ' +BURST + os.system(call_str) + call_str = "echo 'SLC has already down' >down2slc.dat" + os.system(call_str) + print("Down to SLC for %s is done! " % date) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_sen_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_sen_all.py new file mode 100644 index 0000000..15c7800 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_sen_all.py @@ -0,0 +1,151 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + + +def get_s1_date(raw_file): + file0 = os.path.basename(raw_file) + date = file0[17:25] + return date + + +def work(data0): + cmd = data0[0] + err_txt = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + #print(aa) + with open(err_txt, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + down2slc_sen_all.py projectName + down2slc_sen_all.py projectName --parallel 4 + +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + slcDir = scratchDir + '/' + projectName + "/SLC" + raw_file_list = glob.glob(downDir + '/S1*.zip') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + + date_list = [] + cat_list = [] + for kk in range(len(raw_file_list)): + date0 = get_s1_date(os.path.basename(raw_file_list[kk])) + if date0 not in date_list: + date_list.append(date0) + cat_list.append('0') + else: + cat_list[date_list.index(date0)]='1' +# date_list = set(date_list) +# date_list = sorted(date_list) + + print('Date to be processed:') + for k0 in date_list: + print(k0) + + err_txt = scratchDir + '/' + projectName + '/down2slc_sen_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + for i in range(len(date_list)): + if cat_list[i]=='0': + cmd0 = ['down2slc_sen.py',projectName,date_list[i]] + work_dir = slcDir + '/' + date_list[i] + slc_file0 = work_dir + '/down2slc.dat' + data0 = [cmd0,err_txt] +# data_para.append(data0) + k00 = 0 + if os.path.isfile(slc_file0): + if os.path.getsize(slc_file0) > 0: + k00 = 1 + else: + k00 = 0 + if k00==0: + data_para.append(data0) + else: + cmd0 = ['down2slc_cat_sen.py',projectName,date_list[i]] + work_dir = slcDir + '/' + date_list[i] + slc_file0 = work_dir + '/down2slc.dat' + data0 = [cmd0,err_txt] +# data_para.append(data0) + #data_para.append(data0) + k00 = 0 + if os.path.isfile(slc_file0): + if os.path.getsize(slc_file0) > 0: + k00 = 1 + else: + k00 = 0 + if k00==0: + data_para.append(data0) + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + os.chdir(downDir) + print("Down to SLC for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_sen_all_old.py b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_sen_all_old.py new file mode 100644 index 0000000..755cce1 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_sen_all_old.py @@ -0,0 +1,103 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + + +def work(data0): + cmd = data0[0] + err_txt = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + #print(aa) + with open(err_txt, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + down2slc_sen_all.py projectName + down2slc_sen_all.py projectName --parallel 4 + +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + raw_file_list = glob.glob(downDir + '/S1*.zip') + + slc_dir = scratchDir + '/' + projectName + '/SLC' + if not os.path.isdir(slc_dir): + os.mkdir(slc_dir) + + err_txt = scratchDir + '/' + projectName + '/down2slc_sen_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + for i in range(len(raw_file_list)): + cmd0 = ['down2slc_sen.py',raw_file_list[i],slc_dir] + data0 = [cmd0,err_txt] + data_para.append(data0) + + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + os.chdir(downDir) + print("Down to SLC for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_sen_old.py b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_sen_old.py new file mode 100644 index 0000000..af9fb03 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/down2slc_sen_old.py @@ -0,0 +1,180 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse + +from pyint import _utils as ut + + +def get_s1_date(raw_file): + file0 = os.path.basename(raw_file) + date = file0[17:25] + return date + +def get_satellite(raw_file): + if 'S1A_IW_SLC_' in raw_file: + s0 = 'A' + else: + s0 = 'B' + + return s0 + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLC from Sentinel-1 raw data with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('s1_raw', help='raw Sentinel-1 file. e.g., S1A*.zip') + parser.add_argument('root_path',help='root path for saving the SLC files') + + inps = parser.parse_args() + + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Generate SLC from Sentinel-1 raw data with orbit correction using GAMMA. + [Precise orbit data will be downloaded automatically] +''' + +EXAMPLE = """Usage: + + down2slc_sen.py S1A_IW_SLC_XXXX.zip /Yunmeng/S1_test + +------------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + raw_file = inps.s1_raw + root_dir = inps.root_path + satellite = get_satellite(raw_file) + date = get_s1_date(raw_file) + if not os.path.isdir(root_dir): + os.mkdir(root_dir) + + slc_dir = root_dir + '/' + date + if not os.path.isdir(slc_dir): + os.mkdir(slc_dir) + + if len(os.path.dirname(raw_file))==0: + raw_file_dir = os.getcwd() + else: + raw_file_dir = os.path.dirname(raw_file) + + raw_dir = raw_file.replace('.zip','.SAFE') + +# MEASURE = glob.glob(measureDir + '/*vv*tiff') +# ANNOTAT = glob.glob(annotatDir + '/*vv*xml' ) +# CALIBRA = glob.glob(calibraDir+'/calibration*vv*') +# NOISE = glob.glob(calibraDir+'/noise*vv*') + + SLC_Tab = slc_dir + '/' + date+'_SLC_Tab' + TEST = slc_dir + '/' + date + '.IW3.slc.par' + k0 = 0 + if os.path.isfile(TEST): + if os.path.getsize(TEST) > 0: + k0 = 1 + + if k0==0: + if not os.path.isdir(raw_dir): + call_str = 'unzip ' + raw_file + ' -d ' + raw_file_dir + os.system(call_str) + + measureDir = raw_dir + '/measurement' + annotatDir = raw_dir + '/annotation' + calibraDir = raw_dir + '/annotation/calibration' + MM = glob.glob(measureDir + '/*vv*tiff') + + if os.path.isfile(SLC_Tab): + os.remove(SLC_Tab) + for kk in range(len(MM)): + SLC = slc_dir + '/' + date + '.IW' + str(kk+1)+'.slc' + SLCPar = slc_dir + '/' + date + '.IW' + str(kk+1)+'.slc.par' + TOPPar = slc_dir + '/' + date + '.IW' + str(kk+1)+'.slc.TOPS_par' + BURST = slc_dir + '/' + date + '.IW' + str(kk+1)+'.burst.par' + + if os.path.isfile(BURST): + os.remove(BURST) + call_str = 'echo ' + SLC + ' ' + SLCPar + ' ' + TOPPar + ' >> ' + SLC_Tab + os.system(call_str) + + MEASURE = glob.glob(measureDir + '/*iw' + str(kk+1) + '*vv*tiff') + ANNOTAT = glob.glob(annotatDir + '/*iw' + str(kk+1) + '*vv*xml' ) + CALIBRA = glob.glob(calibraDir+'/calibration*'+ 'iw' + str(kk+1) + '*vv*') + NOISE = glob.glob(calibraDir+'/noise*' + 'iw' + str(kk+1) + '*vv*') + + #call_str = 'S1_burstloc ' + ANNOTAT[0] + '> ' +BURST + #os.system(call_str) + + if not os.path.isfile(NOISE[0]): + call_str = 'par_S1_SLC ' + MEASURE[0] + ' ' + ANNOTAT[0] + ' ' + CALIBRA[0] + ' - ' + SLCPar + ' ' + SLC + ' ' + TOPPar + else: + call_str = 'par_S1_SLC ' + MEASURE[0] + ' ' + ANNOTAT[0] + ' ' + CALIBRA[0] + ' ' + NOISE[0] + ' ' + SLCPar + ' ' + SLC + ' ' + TOPPar + + #if int(date) > 180311: + # call_str = 'par_S1_SLC ' + MEASURE[0] + ' ' + ANNOTAT[0] + ' ' + CALIBRA[0] + ' - ' + SLCPar + ' ' + SLC + ' ' + TOPPar + #else: + # call_str = 'par_S1_SLC ' + MEASURE[0] + ' ' + ANNOTAT[0] + ' ' + CALIBRA[0] + ' ' + NOISE[0] + ' ' + SLCPar + ' ' + SLC + ' ' + TOPPar + + os.system(call_str) + + call_str = 'SLC_burst_corners ' + SLCPar + ' ' + TOPPar + ' > ' +BURST + os.system(call_str) + + # orbit correction + slc_pars = glob.glob(slc_dir + '/*.IW*.slc.par') + orbit_file0 = ut.download_s1_orbit(date,slc_dir,satellite=satellite) + orbit_file = slc_dir + '/' + orbit_file0 + + for i in range(len(slc_pars)): + call_str = 'S1_OPOD_vec ' + slc_pars[i] + ' ' + orbit_file + os.system(call_str) + + # generate amp file for check image quality + #TSLC = slc_dir + '/' + date + '.slc' + #TSLCPar = slc_dir + '/' + date + '.slc.par' + + #TMLI = slc_dir + '/' + date + '_40rlks.amp' + #TMLIPar = slc_dir + '/' + date + '_40rlks.amp.par' + + #call_str = 'SLC_mosaic_S1_TOPS ' + SLC_Tab + ' ' + TSLC + ' ' + TSLCPar + ' 10 2' + #os.system(call_str) + + #call_str = 'multi_look ' + TSLC + ' ' + TSLCPar + ' ' + TMLI + ' ' + TMLIPar + ' 40 8' + #os.system(call_str) + + #nWidth = ut.read_gamma_par(TMLIPar, 'read','range_samples:') + #call_str = 'raspwr ' + TMLI + ' ' + nWidth + ' - - - - - - - ' + #os.system(call_str) + + if os.path.isdir(raw_dir): + call_str = 'rm -rf ' + raw_dir + os.system(call_str) + + print("Down to SLC for %s is done! " % date) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/download_ers_deos.py b/.codex_tmp/pyint_variants/no_rescue/pyint/download_ers_deos.py new file mode 100644 index 0000000..980d752 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/download_ers_deos.py @@ -0,0 +1,171 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Now at KAUST ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import linecache + +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def StrNum1(S): + S=str(S) + if len(S)==1: + S='00' +S + elif len(S)==2: + S = '0' +S + else: + S = S + return S + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +def print_progress(iteration, total, prefix='calculating:', suffix='complete', decimals=1, barLength=50, elapsed_time=None): + """Print iterations progress - Greenstick from Stack Overflow + Call in a loop to create terminal progress bar + @params: + iteration - Required : current iteration (Int) + total - Required : total iterations (Int) + prefix - Optional : prefix string (Str) + suffix - Optional : suffix string (Str) + decimals - Optional : number of decimals in percent complete (Int) + barLength - Optional : character length of bar (Int) + elapsed_time- Optional : elapsed time in seconds (Int/Float) + + Reference: http://stackoverflow.com/questions/3173320/text-progress-bar-in-the-console + """ + filledLength = int(round(barLength * iteration / float(total))) + percents = round(100.00 * (iteration / float(total)), decimals) + bar = '#' * filledLength + '-' * (barLength - filledLength) + if elapsed_time: + sys.stdout.write('%s [%s] %s%s %s %s secs\r' % (prefix, bar, percents, '%', suffix, int(elapsed_time))) + else: + sys.stdout.write('%s [%s] %s%s %s\r' % (prefix, bar, percents, '%', suffix)) + sys.stdout.flush() + if iteration == total: + print("\n") + + ''' + Sample Useage: + for i in range(len(dateList)): + print_progress(i+1,len(dateList)) + ''' + return + + +######################################################################### + +INTRODUCTION = ''' +############################################################################# + Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 + + Download precise ERS orbit data from Delft Institute for Earth-Oriented Space Research (http://www.deos.tudelft.nl/). + +''' + +EXAMPLE = ''' + Usage: + download_ers_deos.py ERS1 + download_ers_deos.py ERS2 + +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Download (and correct) precise ERS orbit data.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('sar',help='project name') + + inps = parser.parse_args() + return inps + +################################################################################ + + +def main(argv): + + total = time.time() + inps = cmdLineParse() + sar = inps.sar + PATH = os.getcwd() + listfile = 'arclist' + + if sar == 'ERS1': + Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-1/dgm-e04/' + Url_list = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-1/dgm-e04/arclist' + num_min = 6 + num_max = 511 + N_num = 506 + if not os.path.isfile(listfile): + call_str = 'wget -q ' + Url_list + os.system(call_str) + + elif sar =='ERS2': + Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-2/dgm-e04/' + Url_list = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-2/dgm-e04/arclist' + num_min = 3 + num_max = 867 + N_num = 865 + if not os.path.isfile(listfile): + call_str = 'wget -q ' + Url_list + os.system(call_str) + + else: + print('SAR name is invalid!') + sys.exit(1) + + for i in range(N_num): + kk = num_min + i + kk=int(kk) + SS = Url + 'ODR.' + StrNum1(kk) + print_progress(i+1, N_num, prefix='DEFT_vec: ', suffix=StrNum1(kk)) + + call_str = 'wget -q --no-check-certificate ' + SS + ' -P ' + PATH + os.system(call_str) + + + print("Download precise DEFT orbital data for %s is done." % sar) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/downloader_gmtchina.py b/.codex_tmp/pyint_variants/no_rescue/pyint/downloader_gmtchina.py new file mode 100644 index 0000000..f8015fa --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/downloader_gmtchina.py @@ -0,0 +1,388 @@ +# -*- coding: utf-8 -* +#''' +#This code is used to download image from google and amap + +#@date : 2020-3-13 +#@author: Zheng Jie +#@E-mail: zhengjie9510@qq.com + +#Modified by GMT-china.org +#''' + +import sys +import io +import math +import multiprocessing +import time +import urllib.request as ur +from math import floor, pi, log, tan, atan, exp +from threading import Thread + +import PIL.Image as pil +import cv2 +import numpy as np +from osgeo import gdal, osr + + +# ------------------Interchange between WGS-84 and Web Mercator------------------------- +# WGS-84 to Web Mercator +def wgs_to_mercator(x, y): + y = 85.0511287798 if y > 85.0511287798 else y + y = -85.0511287798 if y < -85.0511287798 else y + + x2 = x * 20037508.34 / 180 + y2 = log(tan((90 + y) * pi / 360)) / (pi / 180) + y2 = y2 * 20037508.34 / 180 + return x2, y2 + + +# Web Mercator to WGS-84 +def mercator_to_wgs(x, y): + x2 = x / 20037508.34 * 180 + y2 = y / 20037508.34 * 180 + y2 = 180 / pi * (2 * atan(exp(y2 * pi / 180)) - pi / 2) + return x2, y2 + + +# -------------------------------------------------------------------------------------- + +# -----------------Interchange between GCJ-02 to WGS-84--------------------------- +# All public geographic data in mainland China need to be encrypted with GCJ-02, introducing random bias +# This part of the code is used to remove the bias +def transformLat(x, y): + ret = -100.0 + 2.0 * x + 3.0 * y + 0.2 * y * y + 0.1 * x * y + 0.2 * math.sqrt(abs(x)) + ret += (20.0 * math.sin(6.0 * x * math.pi) + 20.0 * math.sin(2.0 * x * math.pi)) * 2.0 / 3.0 + ret += (20.0 * math.sin(y * math.pi) + 40.0 * math.sin(y / 3.0 * math.pi)) * 2.0 / 3.0 + ret += (160.0 * math.sin(y / 12.0 * math.pi) + 320 * math.sin(y * math.pi / 30.0)) * 2.0 / 3.0 + return ret + + +def transformLon(x, y): + ret = 300.0 + x + 2.0 * y + 0.1 * x * x + 0.1 * x * y + 0.1 * math.sqrt(abs(x)) + ret += (20.0 * math.sin(6.0 * x * math.pi) + 20.0 * math.sin(2.0 * x * math.pi)) * 2.0 / 3.0 + ret += (20.0 * math.sin(x * math.pi) + 40.0 * math.sin(x / 3.0 * math.pi)) * 2.0 / 3.0 + ret += (150.0 * math.sin(x / 12.0 * math.pi) + 300.0 * math.sin(x / 30.0 * math.pi)) * 2.0 / 3.0 + return ret + + +def delta(lat, lon): + ''' + Krasovsky 1940 + // + // a = 6378245.0, 1/f = 298.3 + // b = a * (1 - f) + // ee = (a^2 - b^2) / a^2; + ''' + a = 6378245.0 # a: Projection factor of satellite ellipsoidal coordinates projected onto a flat map coordinate system + ee = 0.00669342162296594323 # ee: Eccentricity of ellipsoid + dLat = transformLat(lon - 105.0, lat - 35.0) + dLon = transformLon(lon - 105.0, lat - 35.0) + radLat = lat / 180.0 * math.pi + magic = math.sin(radLat) + magic = 1 - ee * magic * magic + sqrtMagic = math.sqrt(magic) + dLat = (dLat * 180.0) / ((a * (1 - ee)) / (magic * sqrtMagic) * math.pi) + dLon = (dLon * 180.0) / (a / sqrtMagic * math.cos(radLat) * math.pi) + return {'lat': dLat, 'lon': dLon} + + +def outOfChina(lat, lon): + if (lon < 72.004 or lon > 137.8347): + return True + if (lat < 0.8293 or lat > 55.8271): + return True + return False + + +def gcj_to_wgs(gcjLon, gcjLat): + if outOfChina(gcjLat, gcjLon): + return (gcjLon, gcjLat) + d = delta(gcjLat, gcjLon) + return (gcjLon - d["lon"], gcjLat - d["lat"]) + + +def wgs_to_gcj(wgsLon, wgsLat): + if outOfChina(wgsLat, wgsLon): + return wgsLon, wgsLat + d = delta(wgsLat, wgsLon) + return wgsLon + d["lon"], wgsLat + d["lat"] + + +# -------------------------------------------------------------- + +# --------------------------------------------------------- +# Get tile coordinates in Google Maps based on latitude and longitude of WGS-84 +def wgs_to_tile(j, w, z): + ''' + Get google-style tile cooridinate from geographical coordinate + j : Longittude + w : Latitude + z : zoom + ''' + isnum = lambda x: isinstance(x, int) or isinstance(x, float) + if not (isnum(j) and isnum(w)): + raise TypeError("j and w must be int or float!") + + if not isinstance(z, int) or z < 0 or z > 22: + raise TypeError("z must be int and between 0 to 22.") + + if j < 0: + j = 180 + j + else: + j += 180 + j /= 360 # make j to (0,1) + + w = 85.0511287798 if w > 85.0511287798 else w + w = -85.0511287798 if w < -85.0511287798 else w + w = log(tan((90 + w) * pi / 360)) / (pi / 180) + w /= 180 # make w to (-1,1) + w = 1 - (w + 1) / 2 # make w to (0,1) and left top is 0-point + + num = 2 ** z + x = floor(j * num) + y = floor(w * num) + return x, y + + +def pixls_to_mercator(zb): + # Get the web Mercator projection coordinates of the four corners of the area according to the four corner coordinates of the tile + inx, iny = zb["LT"] # left top + inx2, iny2 = zb["RB"] # right bottom + length = 20037508.3427892 + sum = 2 ** zb["z"] + LTx = inx / sum * length * 2 - length + LTy = -(iny / sum * length * 2) + length + + RBx = (inx2 + 1) / sum * length * 2 - length + RBy = -((iny2 + 1) / sum * length * 2) + length + + # LT=left top,RB=right buttom + # Returns the projected coordinates of the four corners + res = {'LT': (LTx, LTy), 'RB': (RBx, RBy), + 'LB': (LTx, RBy), 'RT': (RBx, LTy)} + return res + + +def tile_to_pixls(zb): + # Tile coordinates are converted to pixel coordinates of the four corners + out = {} + width = (zb["RT"][0] - zb["LT"][0] + 1) * 256 + height = (zb["LB"][1] - zb["LT"][1] + 1) * 256 + out["LT"] = (0, 0) + out["RT"] = (width, 0) + out["LB"] = (0, -height) + out["RB"] = (width, -height) + return out + + +# ----------------------------------------------------------- + +# --------------------------------------------------------- +class Downloader(Thread): + # multiple threads downloader + def __init__(self, index, count, urls, datas): + # index represents the number of threads + # count represents the total number of threads + # urls represents the list of URLs nedd to be downloaded + # datas represents the list of data need to be returned. + super().__init__() + self.urls = urls + self.datas = datas + self.index = index + self.count = count + + def download(self, url): + HEADERS = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36 Edg/88.0.705.68'} + header = ur.Request(url, headers=HEADERS) + err = 0 + while (err < 3): + try: + data = ur.urlopen(header).read() + except: + err += 1 + else: + return data + raise Exception("Bad network link.") + + def run(self): + for i, url in enumerate(self.urls): + if i % self.count != self.index: + continue + self.datas[i] = self.download(url) + + +# --------------------------------------------------------- + +# --------------------------------------------------------- +def getExtent(x1, y1, x2, y2, z, source="amap"): + pos1x, pos1y = wgs_to_tile(x1, y1, z) + pos2x, pos2y = wgs_to_tile(x2, y2, z) + Xframe = pixls_to_mercator( + {"LT": (pos1x, pos1y), "RT": (pos2x, pos1y), "LB": (pos1x, pos2y), "RB": (pos2x, pos2y), "z": z}) + for i in ["LT", "LB", "RT", "RB"]: + Xframe[i] = mercator_to_wgs(*Xframe[i]) + if source == "google_sat": + pass + elif source == "amap" or source == "amap_sat" or source == "google": + for i in ["LT", "LB", "RT", "RB"]: + Xframe[i] = gcj_to_wgs(*Xframe[i]) + else: + raise Exception("Invalid argument: source.") + return Xframe + + +def saveTiff(r, g, b, gt, filePath): + fname_out = filePath + driver = gdal.GetDriverByName('GTiff') + # Create a 3-band dataset + dset_output = driver.Create(fname_out, r.shape[1], r.shape[0], 3, gdal.GDT_Byte) + dset_output.SetGeoTransform(gt) + try: + proj = osr.SpatialReference() + proj.ImportFromEPSG(4326) + dset_output.SetSpatialRef(proj) + except: + print("Error: Coordinate system setting failed") + dset_output.GetRasterBand(1).WriteArray(r) + dset_output.GetRasterBand(2).WriteArray(g) + dset_output.GetRasterBand(3).WriteArray(b) + dset_output.FlushCache() + dset_output = None + print("Image Saved") + + +# --------------------------------------------------------- + +# --------------------------------------------------------- +MAP_URLS = { + "google": "https://mt1.google.com/vt/lyrs=r&x={x}&y={y}&z={z}", + "google_sat": "https://mt1.google.com/vt/lyrs=s&x={x}&y={y}&z={z}", + "amap": "https://webrd02.is.autonavi.com/appmaptile?lang=zh_cn&size=1&scale=1&style=8&x={x}&y={y}&z={z}", + "amap_sat": "https://webst01.is.autonavi.com/appmaptile?style=6&x={x}&y={y}&z={z}" + } + + +def get_url(source, x, y, z, style): # + if source == 'amap': + url = MAP_URLS["amap"].format(x=x, y=y, z=z) + elif source == 'amap_sat': + url = MAP_URLS["amap_sat"].format(x=x, y=y, z=z) + elif source == 'google': + url = MAP_URLS["google"].format(x=x, y=y, z=z) + elif source == 'google_sat': + url = MAP_URLS["google_sat"].format(x=x, y=y, z=z) + else: + raise Exception("Unknown Map Source ! ") + return url + + +def get_urls(x1, y1, x2, y2, z, source, style): + pos1x, pos1y = wgs_to_tile(x1, y1, z) + pos2x, pos2y = wgs_to_tile(x2, y2, z) + lenx = pos2x - pos1x + 1 + leny = pos2y - pos1y + 1 + print("Total tiles number：{x} X {y}".format(x=lenx, y=leny)) + urls = [get_url(source, i, j, z, style) for j in range(pos1y, pos1y + leny) for i in range(pos1x, pos1x + lenx)] + return urls + + +# --------------------------------------------------------- + +# --------------------------------------------------------- +def merge_tiles(datas, x1, y1, x2, y2, z): + pos1x, pos1y = wgs_to_tile(x1, y1, z) + pos2x, pos2y = wgs_to_tile(x2, y2, z) + lenx = pos2x - pos1x + 1 + leny = pos2y - pos1y + 1 + outpic = pil.new('RGBA', (lenx * 256, leny * 256)) + for i, data in enumerate(datas): + picio = io.BytesIO(data) + small_pic = pil.open(picio) + y, x = i // lenx, i % lenx + outpic.paste(small_pic, (x * 256, y * 256)) + print('Tiles merge completed') + return outpic + + +def download_tiles(urls, multi=10): + url_len = len(urls) + datas = [None] * url_len + if multi < 1 or multi > 20 or not isinstance(multi, int): + raise Exception("multi of Downloader shuold be int and between 1 to 20.") + tasks = [Downloader(i, multi, urls, datas) for i in range(multi)] + for i in tasks: + i.start() + for i in tasks: + i.join() + return datas + + +# --------------------------------------------------------- + +# --------------------------------------------------------- +def main(left, top, right, bottom, zoom, filePath, style='s', server="amap"): + """ + Download images based on spatial extent. + + East longitude is positive and west longitude is negative. + North latitude is positive, south latitude is negative. + + Parameters + ---------- + left, top : left-top coordinate, for example (100.361,38.866) + + right, bottom : right-bottom coordinate + + z : zoom + + filePath : File path for storing results, TIFF format + + style : + m for map; + s for satellite; + y for satellite with label; + t for terrain; + p for terrain with label; + h for label; + + source : Google or amap + """ + # --------------------------------------------------------- + # Get the urls of all tiles in the extent + urls = get_urls(left, top, right, bottom, zoom, server, style) + + # Group URLs based on the number of CPU cores to achieve roughly equal amounts of tasks + urls_group = [urls[i:i + math.ceil(len(urls) / multiprocessing.cpu_count())] for i in + range(0, len(urls), math.ceil(len(urls) / multiprocessing.cpu_count()))] + + # Each set of URLs corresponds to a process for downloading tile maps + print('Tiles downloading......') + pool = multiprocessing.Pool(multiprocessing.cpu_count()) + results = pool.map(download_tiles, urls_group) + pool.close() + pool.join() + result = [x for j in results for x in j] + print('Tiles download complete') + + # Combine downloaded tile maps into one map + outpic = merge_tiles(result, left, top, right, bottom, zoom) + outpic = outpic.convert('RGB') + r, g, b = cv2.split(np.array(outpic)) + + # Get the spatial information of the four corners of the merged map and use it for outputting + extent = getExtent(left, top, right, bottom, zoom, server) + gt = (extent['LT'][0], (extent['RB'][0] - extent['LT'][0]) / r.shape[1], 0, extent['LT'][1], 0, + (extent['RB'][1] - extent['LT'][1]) / r.shape[0]) + saveTiff(r, g, b, gt, filePath) + + +# --------------------------------------------------------- +if __name__ == '__main__': + start_time = time.time() + + #main(118.055917, 24.559724, 118.244753, 24.399450, 16, r'google_sat.tif', server="google_sat") + main(float(sys.argv[1]), float(sys.argv[4]), float(sys.argv[2]), float(sys.argv[3]), int(sys.argv[5]), sys.argv[6], server=sys.argv[7]) + + end_time = time.time() + print('lasted a total of {:.2f} seconds'.format(end_time - start_time)) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/extract_s1_bursts.py b/.codex_tmp/pyint_variants/no_rescue/pyint/extract_s1_bursts.py new file mode 100644 index 0000000..2f03583 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/extract_s1_bursts.py @@ -0,0 +1,297 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import glob +import argparse + +from pyint import _utils as ut + +def common_burst(La_M,La_S): + La_M = [float(i) for i in La_M] + La_S = [float(i) for i in La_S] + Min = max(min(La_M),min(La_S)) + Max = min(max(La_M),max(La_S)) + + M_min = [] + M_max = [] + for xm in La_M: + k0_min = float(xm) - float(Min) + k0_max = float(xm) - float(Max) + M_min.append(abs(k0_min)) + M_max.append(abs(k0_max)) + + M_Index_min = M_min.index(min(M_min)) + 1 + M_Index_max = M_max.index(min(M_max)) + 1 + Mindex =[M_Index_min,M_Index_max] + + S_min = [] + S_max = [] + for xs in La_S: + k0_min = float(xs) - float(Min) + k0_max = float(xs) - float(Max) + S_min.append(abs(k0_min)) + S_max.append(abs(k0_max)) + + S_Index_min = S_min.index(min(S_min)) + 1 + S_Index_max = S_max.index(min(S_max)) + 1 + Sindex =[S_Index_min,S_Index_max] + + return min(Mindex) , max(Mindex), min(Sindex),max(Sindex) + +def get_common_burst(Mslc_dir,Sslc_dir,common_burst_txt): + Mpar_list = sorted(glob.glob(Mslc_dir+'/*.IW*.burst.par')) + Spar_list = sorted(glob.glob(Sslc_dir+'/*.IW*.burst.par')) + if os.path.isfile(common_burst_txt): + os.remove(common_burst_txt) + for kk in range(len(Mpar_list)): + MBURST = Mpar_list[kk] + SBURST = Spar_list[kk] + + Mtt = Sslc_dir + '/' + os.path.basename(MBURST.replace('burst.par','tt0')) + Stt = SBURST.replace('burst.par','tt0') + + call_str = "grep 'Burst:' " + MBURST + ' >' + Mtt + os.system(call_str) + call_str = "grep 'Burst:' " + SBURST + ' >' + Stt + os.system(call_str) + + MM = ut.read_txt2array(Mtt) + SM = ut.read_txt2array(Stt) + La_M = MM[:,2] + La_S = SM[:,2] + + PP = common_burst_Ref(La_M,La_S) + + print('Common bursts of swath' + str(kk+1) + ' : (master) ' + str(PP[0]) + ' ' + str(PP[1]) + ' (slave) ' + str(PP[2]) + ' ' + str(PP[3])) + call_str = 'echo ' + str(PP[0]) + ' ' + str(PP[1]) + ' ' + str(PP[2]) + ' ' + str(PP[3]) + ' >>' + common_burst_txt + os.system(call_str) + + return + +def common_burst_Ref(La_M,La_S): + Min = max(min(La_M),min(La_S)) + Max = min(max(La_M),max(La_S)) + + M_min = [] + M_max = [] + for xm in La_M: + k0_min = float(xm) - float(Min) + k0_max = float(xm) - float(Max) + M_min.append(abs(k0_min)) + M_max.append(abs(k0_max)) + + M_Index_min = M_min.index(min(M_min)) + 1 + M_Index_max = M_max.index(min(M_max)) + 1 + Mindex =[M_Index_min,M_Index_max] + + S_min = [] + S_max = [] + for xs in La_S: + k0_min = float(xs) - float(Min) + k0_max = float(xs) - float(Max) + S_min.append(abs(k0_min)) + S_max.append(abs(k0_max)) + + S_Index_min = S_min.index(min(S_min)) + 1 + S_Index_max = S_max.index(min(S_max)) + 1 + Sindex =[S_Index_min,S_Index_max] + #print La_M + #print La_S + #print min(M_min),min(M_max),min(S_min),min(S_max) + M1 = min(Mindex) + M2 = max(Mindex) + + S1 = min(Sindex) + S2 = max(Sindex) + + if M1==1: S1 = S1 + else: + S1=1-M1+1 + M1=1 + + + if M2 ==len(La_M): S2 = S2 + else: + S2 = S2 + len(La_M) - M2 + M2 = len(La_M) + + + return M1 , M2, S1, S2 +######################################################################### + +INTRODUCTION = ''' +############################################################################# + + Extract the common bursts for S1 TOPs based on one master image. +''' + +EXAMPLE = ''' + Usage: + extract_s1_bursts.py projectName 170115 + + Examples: + extract_s1_bursts.py PacayaT163TsxHhA 170115 +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Check common busrts for TOPS data.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='Name of project.') + parser.add_argument('Sdate',help='Slave date.') + + inps = parser.parse_args() + + return inps + +################################################################################ + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Sdate = inps.Sdate + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Mdate = templateDict['masterDate'] + + processDir = scratchDir + '/' + projectName + "/PROCESS" + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + + workDir0 = rslcDir + '/' + Sdate + #if not os.path.isdir(rslcDir): + # os.mkdir(rslcDir) + + MslcDir = slcDir + '/' + Mdate + SslcDir = slcDir + '/' + Sdate + #MBurst_Par = slcDir + '/' + Mdate + '/' + + workDir = slcDir + '/' + Sdate + + BURST = workDir + '/' + Mdate + '_' + Sdate + '.common_burst_ref' + + MslcImg = workDir + '/'+Mdate + '.slc' + MslcPar = workDir + '/'+Mdate + '.slc.par' + + MamprlksImg = workDir + '/'+Mdate + '_' + rlks + 'rlks' + '.amp' + MamprlksPar = workDir + '/'+Mdate + '_' + rlks +'rlks' + '.amp.par' + + SslcImg = workDir + '/'+ Sdate + '.slc' + SslcPar = workDir + '/'+ Sdate + '.slc.par' + + SamprlksImg = workDir + '/'+ Sdate + '_' + rlks + 'rlks' + '.amp' + SamprlksPar = workDir + '/'+ Sdate + '_' + rlks + 'rlks' + '.amp.par' + + if not os.path.isdir(workDir): + os.mkdir(workDir) + + get_common_burst(MslcDir,SslcDir,BURST) + AA = ut.read_txt2array(BURST) + + SW = templateDict['start_swath'] + EW = templateDict['end_swath'] + + SB = templateDict['start_burst'] + EB = templateDict['end_burst'] + + + SLC2_tab = workDir + '/' + Sdate + '_SLC_Tab0' + + + SLC2_INF_tab = workDir + '/' + Sdate + '_SLC_Tab' + SLC2_RSLC_tab = workDir + '/' + Sdate + '_RSLC_Tab' + + BURST2_tab = workDir + '/' + Sdate + '_Burst_Tab' + + if os.path.isfile(SLC2_tab): + os.remove(SLC2_tab) + + if os.path.isfile(SLC2_INF_tab): + os.remove(SLC2_INF_tab) + + if os.path.isfile(SLC2_RSLC_tab): + os.remove(SLC2_RSLC_tab) + + if os.path.isfile(BURST2_tab): + os.remove(BURST2_tab) + + for kk in range(int(EW)-int(SW)+1): + + call_str = 'echo ' + SslcDir + '/' + Sdate+'.IW'+str(int(SW)+kk) + '.slc' + ' ' + SslcDir + '/'+ Sdate + '.IW'+str(int(SW)+kk) +'.slc.par' + ' ' + SslcDir + '/'+ Sdate+'.IW'+str(int(SW)+kk) + '.slc.TOPS_par >>' + SLC2_tab + os.system(call_str) + + ii = kk + 1 + if (int(SW)==int(EW)): + SB1=int(AA[0]) + EB1=int(AA[1]) + SB2=int(AA[2]) + EB2=int(AA[3]) + else: + SB1=int(AA[ii-1,0]) + EB1=int(AA[ii-1,1]) + + SB2=int(AA[ii-1,2]) + EB2=int(AA[ii-1,3]) + + if not int(SB)==1: + SB2 = SB2 + int(SB)-int(SB1) + if not int(EB)==20: + EB2 = SB2 + int(EB) - int(SB) + + call_str = 'echo ' + workDir + '/'+ Sdate+ '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+str(int(SW)+kk)+ '.slc' + ' ' + workDir + '/' + Sdate + '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+ str(int(SW)+kk)+ '.slc.par' + ' ' + workDir + '/'+ Sdate+'_'+ str(int(SB2)) + str(int(EB2)) + '.IW'+str(int(SW)+kk)+ '.slc.TOPS_par >>' + SLC2_INF_tab + os.system(call_str) + + call_str = 'echo ' + workDir0 + '/'+ Sdate+ '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+str(int(SW)+kk)+ '.rslc' + ' ' + workDir0 + '/' + Sdate + '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+ str(int(SW)+kk)+ '.rslc.par' + ' ' + workDir0 + '/'+ Sdate+'_'+ str(int(SB2)) + str(int(EB2)) + '.IW'+str(int(SW)+kk)+ '.rslc.TOPS_par >>' + SLC2_RSLC_tab + os.system(call_str) + + call_str = 'echo ' + str(int(SB2)) + ' ' + str(int(EB2)) + ' >>' + BURST2_tab + os.system(call_str) + + + TEST = SamprlksPar + k0 = 0 + if os.path.isfile(TEST): + if os.path.getsize(TEST) > 0: + k0 = 1 + + if k0==0: + + call_str = 'SLC_copy_ScanSAR ' + SLC2_tab + ' ' + SLC2_INF_tab + ' ' + BURST2_tab + os.system(call_str) + + call_str = 'SLC_mosaic_ScanSAR ' + SLC2_INF_tab + ' ' + SslcImg + ' ' + SslcPar + ' ' + rlks + ' ' +azlks + os.system(call_str) + + call_str = 'multi_look ' + SslcImg + ' ' + SslcPar + ' ' + SamprlksImg + ' ' + SamprlksPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + + nWidth = ut.read_gamma_par(SamprlksPar, 'read', 'range_samples') + call_str = 'raspwr ' + SamprlksImg + ' ' + nWidth + os.system(call_str) + iw1 = Sdate + '.IW1.slc*' + iw2 = Sdate + '.IW2.slc*' + iw3 = Sdate + '.IW3.slc*' + call_str = 'rm '+ iw1 + ' ' + iw2 + ' '+ iw3 + os.system(call_str) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/extract_s1_bursts_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/extract_s1_bursts_all.py new file mode 100644 index 0000000..2a48bf8 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/extract_s1_bursts_all.py @@ -0,0 +1,96 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + + +def work(data0): + cmd = data0[0] + err_file = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + #print(aa) + with open(err_file, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + Extract reference TOPS related bursts for coregistration using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + extract_s1_bursts_all.py projectName + extract_s1_bursts_all.py projectName --parallel 4 +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Extract reference TOPS related bursts for coregistration using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + slcDir = scratchDir + '/' + projectName + '/SLC' + slc_list = [os.path.basename(fname) for fname in sorted(glob.glob(slcDir + '/*'))] + + err_txt = scratchDir + '/' + projectName + '/extract_s1_bursts_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + for i in range(len(slc_list)): + cmd0 = ['extract_s1_bursts.py',projectName,slc_list[i]] + data0 = [cmd0,err_txt] + data_para.append(data0) + + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + print("Extract TOPS bursts for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/extract_s1_bursts_old.py b/.codex_tmp/pyint_variants/no_rescue/pyint/extract_s1_bursts_old.py new file mode 100644 index 0000000..062469c --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/extract_s1_bursts_old.py @@ -0,0 +1,292 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import glob +import argparse + +from pyint import _utils as ut + +def common_burst(La_M,La_S): + La_M = [float(i) for i in La_M] + La_S = [float(i) for i in La_S] + Min = max(min(La_M),min(La_S)) + Max = min(max(La_M),max(La_S)) + + M_min = [] + M_max = [] + for xm in La_M: + k0_min = float(xm) - float(Min) + k0_max = float(xm) - float(Max) + M_min.append(abs(k0_min)) + M_max.append(abs(k0_max)) + + M_Index_min = M_min.index(min(M_min)) + 1 + M_Index_max = M_max.index(min(M_max)) + 1 + Mindex =[M_Index_min,M_Index_max] + + S_min = [] + S_max = [] + for xs in La_S: + k0_min = float(xs) - float(Min) + k0_max = float(xs) - float(Max) + S_min.append(abs(k0_min)) + S_max.append(abs(k0_max)) + + S_Index_min = S_min.index(min(S_min)) + 1 + S_Index_max = S_max.index(min(S_max)) + 1 + Sindex =[S_Index_min,S_Index_max] + return min(Mindex) , max(Mindex), min(Sindex),max(Sindex) + +def get_common_burst(Mslc_dir,Sslc_dir,common_burst_txt): + Mpar_list = sorted(glob.glob(Mslc_dir+'/*.IW*.burst.par')) + Spar_list = sorted(glob.glob(Sslc_dir+'/*.IW*.burst.par')) + if os.path.isfile(common_burst_txt): + os.remove(common_burst_txt) + for kk in range(len(Mpar_list)): + MBURST = Mpar_list[kk] + SBURST = Spar_list[kk] + + Mtt = Sslc_dir + '/' + os.path.basename(MBURST.replace('burst.par','tt0')) + Stt = SBURST.replace('burst.par','tt0') + call_str = "grep 'Burst:' " + MBURST + ' >' + Mtt + os.system(call_str) + + call_str = "grep 'Burst:' " + SBURST + ' >' + Stt + os.system(call_str) + + MM = ut.read_txt2array(Mtt) + SM = ut.read_txt2array(Stt) + La_M = MM[:,2] + La_S = SM[:,2] + + PP = common_burst_Ref(La_M,La_S) + + print('Common bursts of swath' + str(kk+1) + ' : (master) ' + str(PP[0]) + ' ' + str(PP[1]) + ' (slave) ' + str(PP[2]) + ' ' + str(PP[3])) + call_str = 'echo ' + str(PP[0]) + ' ' + str(PP[1]) + ' ' + str(PP[2]) + ' ' + str(PP[3]) + ' >>' + common_burst_txt + os.system(call_str) + + return + +def common_burst_Ref(La_M,La_S): + Min = max(min(La_M),min(La_S)) + Max = min(max(La_M),max(La_S)) + + M_min = [] + M_max = [] + for xm in La_M: + k0_min = float(xm) - float(Min) + k0_max = float(xm) - float(Max) + M_min.append(abs(k0_min)) + M_max.append(abs(k0_max)) + + M_Index_min = M_min.index(min(M_min)) + 1 + M_Index_max = M_max.index(min(M_max)) + 1 + Mindex =[M_Index_min,M_Index_max] + + S_min = [] + S_max = [] + for xs in La_S: + k0_min = float(xs) - float(Min) + k0_max = float(xs) - float(Max) + S_min.append(abs(k0_min)) + S_max.append(abs(k0_max)) + + S_Index_min = S_min.index(min(S_min)) + 1 + S_Index_max = S_max.index(min(S_max)) + 1 + Sindex =[S_Index_min,S_Index_max] + #print La_M + #print La_S + #print min(M_min),min(M_max),min(S_min),min(S_max) + M1 = min(Mindex) + M2 = max(Mindex) + + S1 = min(Sindex) + S2 = max(Sindex) + + if M1==1: S1 = S1 + else: + S1=1-M1+1 + M1=1 + + + if M2 ==len(La_M): S2 = S2 + else: + S2 = S2 + len(La_M) - M2 + M2 = len(La_M) + + + return M1 , M2, S1, S2 +######################################################################### + +INTRODUCTION = ''' +############################################################################# + + Extract the common bursts for S1 TOPs based on one master image. +''' + +EXAMPLE = ''' + Usage: + extract_s1_bursts.py projectName 170115 + + Examples: + extract_s1_bursts.py PacayaT163TsxHhA 170115 +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Check common busrts for TOPS data.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='Name of project.') + parser.add_argument('Sdate',help='Slave date.') + + inps = parser.parse_args() + + return inps + +################################################################################ + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Sdate = inps.Sdate + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Mdate = templateDict['masterDate'] + + processDir = scratchDir + '/' + projectName + "/PROCESS" + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + + workDir0 = rslcDir + '/' + Sdate + #if not os.path.isdir(rslcDir): + # os.mkdir(rslcDir) + + MslcDir = slcDir + '/' + Mdate + SslcDir = slcDir + '/' + Sdate + #MBurst_Par = slcDir + '/' + Mdate + '/' + + workDir = slcDir + '/' + Sdate + + BURST = workDir + '/' + Mdate + '_' + Sdate + '.common_burst_ref' + + MslcImg = workDir + '/'+Mdate + '.slc' + MslcPar = workDir + '/'+Mdate + '.slc.par' + + MamprlksImg = workDir + '/'+Mdate + '_' + rlks + 'rlks' + '.amp' + MamprlksPar = workDir + '/'+Mdate + '_' + rlks +'rlks' + '.amp.par' + + SslcImg = workDir + '/'+ Sdate + '.slc' + SslcPar = workDir + '/'+ Sdate + '.slc.par' + + SamprlksImg = workDir + '/'+ Sdate + '_' + rlks + 'rlks' + '.amp' + SamprlksPar = workDir + '/'+ Sdate + '_' + rlks + 'rlks' + '.amp.par' + + if not os.path.isdir(workDir): + os.mkdir(workDir) + + get_common_burst(MslcDir,SslcDir,BURST) + AA = ut.read_txt2array(BURST) + NL = ut.get_txt_lines(BURST) + + if NL ==1: + AA = AA.reshape(1,len(AA)) + + + SW = templateDict['start_swath'] + EW = templateDict['end_swath'] + + SB = templateDict['start_burst'] + EB = templateDict['end_burst'] + + + SLC2_tab = workDir + '/' + Sdate + '_SLC_Tab0' + + + SLC2_INF_tab = workDir + '/' + Sdate + '_SLC_Tab' + SLC2_RSLC_tab = workDir + '/' + Sdate + '_RSLC_Tab' + + BURST2_tab = workDir + '/' + Sdate + '_Burst_Tab' + + if os.path.isfile(SLC2_tab): + os.remove(SLC2_tab) + + if os.path.isfile(SLC2_INF_tab): + os.remove(SLC2_INF_tab) + + if os.path.isfile(SLC2_RSLC_tab): + os.remove(SLC2_RSLC_tab) + + if os.path.isfile(BURST2_tab): + os.remove(BURST2_tab) + + for kk in range(int(EW)-int(SW)+1): + + call_str = 'echo ' + SslcDir + '/' + Sdate+'.IW'+str(int(SW)+kk) + '.slc' + ' ' + SslcDir + '/'+ Sdate + '.IW'+str(int(SW)+kk) +'.slc.par' + ' ' + SslcDir + '/'+ Sdate+'.IW'+str(int(SW)+kk) + '.slc.TOPS_par >>' + SLC2_tab + os.system(call_str) + + ii = int(SW) + kk + SB1=int(AA[ii-1,0]) + EB1=int(AA[ii-1,1]) + + SB2=int(AA[ii-1,2]) + EB2=int(AA[ii-1,3]) + + if not int(SB)==1: + SB2 = SB2 + int(SB) - int(SB1) + if not int(EB)==100: + EB2 = SB2 + int(EB) - int(SB) + + call_str = 'echo ' + workDir + '/'+ Sdate+ '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+str(int(SW)+kk)+ '.slc' + ' ' + workDir + '/' + Sdate + '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+ str(int(SW)+kk)+ '.slc.par' + ' ' + workDir + '/'+ Sdate+'_'+ str(int(SB2)) + str(int(EB2)) + '.IW'+str(int(SW)+kk)+ '.slc.TOPS_par >>' + SLC2_INF_tab + os.system(call_str) + + call_str = 'echo ' + workDir0 + '/'+ Sdate+ '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+str(int(SW)+kk)+ '.rslc' + ' ' + workDir + '/' + Sdate + '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+ str(int(SW)+kk)+ '.rslc.par' + ' ' + workDir + '/'+ Sdate+'_'+ str(int(SB2)) + str(int(EB2)) + '.IW'+str(int(SW)+kk)+ '.rslc.TOPS_par >>' + SLC2_RSLC_tab + os.system(call_str) + + call_str = 'echo ' + str(int(SB2)) + ' ' + str(int(EB2)) + ' >>' + BURST2_tab + os.system(call_str) + + + TEST = SamprlksPar + k0 = 0 + if os.path.isfile(TEST): + if os.path.getsize(TEST) > 0: + k0 = 1 + + if k0==0: + +# call_str = 'SLC_copy_ScanSAR ' + SLC2_tab + ' ' + SLC2_INF_tab + ' ' + BURST2_tab +# os.system(call_str) + +# call_str = 'SLC_mosaic_S1_TOPS ' + SLC2_INF_tab + ' ' + SslcImg + ' ' + SslcPar + ' ' + rlks + ' ' +azlks + call_str = 'SLC_mosaic_S1_TOPS ' + SLC2_tab + ' ' + SslcImg + ' ' + SslcPar + ' ' + rlks + ' ' +azlks + os.system(call_str) + + call_str = 'multi_look ' + SslcImg + ' ' + SslcPar + ' ' + SamprlksImg + ' ' + SamprlksPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + + nWidth = ut.read_gamma_par(SamprlksPar, 'read', 'range_samples') + call_str = 'raspwr ' + SamprlksImg + ' ' + str(nWidth) + os.system(call_str) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/gacos_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/gacos_gamma.py new file mode 100644 index 0000000..75370f5 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/gacos_gamma.py @@ -0,0 +1,1186 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +""" +GACOS atmospheric correction for GAMMA interferograms. +This script applies GACOS-derived tropospheric corrections to unwrapped interferograms. + +GACOS (Generic Atmospheric Correction Online Service) provides zenith total delay (ZTD) +maps that can be used to correct for atmospheric phase delays in InSAR data. + +Usage: + gacos_gamma.py projectName Mdate-Sdate + gacos_gamma.py PacayaT163TsxHhA 20150102-20150601 +""" + +import numpy as np +import os +import sys +import argparse +import subprocess +import glob +import warnings +from pathlib import Path +from datetime import datetime +from scipy.interpolate import RegularGridInterpolator + +from pyint import _utils as ut + +warnings.filterwarnings('ignore', category=RuntimeWarning) + +# Try to import AutoGACOS modules +try: + from gacos import Downloader, Submitter, SarDataset + AUTOGACOS_AVAILABLE = True +except Exception as _e: + AUTOGACOS_AVAILABLE = False + print(f"[DEBUG] AutoGACOS import failed: {type(_e).__name__}: {_e}") + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Apply GACOS atmospheric correction to interferograms. + + This script: + 1. Generates elevation angle file from GAMMA products + 2. Downloads GACOS ZTD data for master and slave dates + 3. Applies tropospheric correction to the interferogram + + Prerequisites: + - GACOS data for the acquisition dates (or valid email for auto-download) + - Geocoded interferogram from geocode_gamma.py +------------------------------------------------------------------- +''' + +EXAMPLE = ''' + Usage: + gacos_gamma.py projectName Mdate-Sdate + gacos_gamma.py projectName Mdate-Sdate --ztd-dir /path/to/gacos/data + gacos_gamma.py PacayaT163TsxHhA 20150102-20150601 +------------------------------------------------------------------- +''' + + +class HEADER: + """Header class for GACOS file format""" + width = 0 + length = 0 + xfirst = 0.0 + yfirst = 0.0 + xstep = 0.0 + ystep = 0.0 + + +def read_header(filename): + """Read header information from GACOS .rsc file""" + if not os.path.isfile(filename): + print(filename + " file not exist") + return None + + header = HEADER() + with open(filename) as f: + for line in f: + data = line.split() + if len(data) >= 2: + if data[0] == "WIDTH": + header.width = int(data[1]) + if data[0] == "FILE_LENGTH": + header.length = int(data[1]) + if data[0] == "X_FIRST": + header.xfirst = float(data[1]) + if data[0] == "Y_FIRST": + header.yfirst = float(data[1]) + if data[0] == "X_STEP": + header.xstep = float(data[1]) + if data[0] == "Y_STEP": + header.ystep = float(data[1]) + return header + + +def cut_image2(filename, headername, yfirst_new, length_new, xfirst_new, width_new): + """Cut GACOS ZTD image to match interferogram extent""" + header = read_header(headername) + if header is None: + return None + + with open(filename, 'rb') as f: + data0 = np.fromfile(f, dtype=np.float32) + data = np.reshape(data0, (header.length, header.width)) + + out = np.zeros((length_new, width_new), dtype=np.float32) + + for i in range(header.length): + lat = header.yfirst + header.ystep * i + row = int(round((lat - yfirst_new) / header.ystep)) + if row < 0 or row >= length_new: + continue + for j in range(header.width): + lon = header.xfirst + header.xstep * j + col = int(round((lon - xfirst_new) / header.xstep)) + if col < 0 or col >= width_new: + continue + out[row, col] = data[i, j] + + out = np.where(out == 0, np.nan, out) + out.tofile(filename + ".cut") + + f = open(filename + ".cut.rsc", 'w') + f.write("WIDTH " + str(width_new) + "\n") + f.write("FILE_LENGTH " + str(length_new) + "\n") + f.write("X_FIRST " + str(xfirst_new) + "\n") + f.write("Y_FIRST " + str(yfirst_new) + "\n") + f.write("X_STEP " + str(header.xstep) + "\n") + f.write("Y_STEP " + str(header.ystep) + "\n") + f.close() + + return filename + ".cut" + + +def make_correction(phsfilename, ztd1filename, ztd2filename, elevfilename, wavelength=None): + """ + Apply GACOS tropospheric correction to interferogram. + + Parameters: + ----------- + phsfilename : str + Path to unwrapped phase file + ztd1filename : str + Path to GACOS ZTD file for master date + ztd2filename : str + Path to GACOS ZTD file for slave date + elevfilename : str + Path to elevation angle file + wavelength : float, optional + Radar wavelength in meters. If None, read from template. + """ + header = read_header(phsfilename + ".rsc") + if header is None: + print("Error reading phase header file") + return None + + # Cut ZTD files to match interferogram extent + if not os.path.isfile(ztd1filename + ".cut"): + cut_image2(ztd1filename, ztd1filename + ".rsc", header.yfirst, header.length, header.xfirst, header.width) + if not os.path.isfile(ztd2filename + ".cut"): + cut_image2(ztd2filename, ztd2filename + ".rsc", header.yfirst, header.length, header.xfirst, header.width) + + # Read phase data + with open(phsfilename, 'rb') as f: + data = np.fromfile(f, dtype=np.float32) + phase = np.reshape(data, [header.length, header.width]) + + # Read ZTD data + with open(ztd1filename + ".cut", 'rb') as f: + data = np.fromfile(f, dtype=np.float32) + ztd1 = np.reshape(data, [header.length, header.width]) + + with open(ztd2filename + ".cut", 'rb') as f: + data = np.fromfile(f, dtype=np.float32) + ztd2 = np.reshape(data, [header.length, header.width]) + + # Read elevation angle data + with open(elevfilename, 'rb') as f: + data = np.fromfile(f, dtype=np.float32) + elev = np.reshape(data, [header.length, header.width]) + + # Calculate tropospheric phase delay + # ZTD difference to phase: phase = 4*pi*dZTD / (wavelength * sin(elevation)) + # Default wavelength factor for Sentinel-1: 0.0044138251819503 = 4*pi/lambda + if wavelength: + ztd_factor = 4 * np.pi / wavelength + else: + ztd_factor = 1 / 0.0044138251819503 # Default factor + + dztd = ztd2 - ztd1 + dztd = dztd / 0.0044138251819503 # Convert ZTD to phase + dztd = dztd / np.sin(elev) # Incidence angle correction + + # Apply correction + index = np.where(phase == 0) + phase[index] = np.nan + phasemean = np.nanmean(phase) + print("Phase std before correction: " + str(np.nanstd(phase))) + + phase = phase - phasemean + phase = phase - dztd # Apply correction + phase[index] = np.nan + phasemean = np.nanmean(phase) + print("Phase std after correction: " + str(np.nanstd(phase))) + + phase = phase - phasemean + phase[index] = 0 + phase.tofile(phsfilename + ".gacos") + + return phsfilename + ".gacos" + + +def generate_elevation_angle(workDir, Mdate, Sdate, MampPar, offpar, dempar, dem, rlks): + """ + Generate elevation angle file from GAMMA products. + + The elevation angle (90 - incidence angle) is needed for + projecting ZTD to line-of-sight phase delay. + """ + os.chdir(workDir) + + # Generate look vector (incidence angle) + call_str = "look_vector " + MampPar + " " + offpar + " " + dempar + " " + dem + " lv_theta lv_phi" + os.system(call_str) + + # Get DEM parameters + call_str = "grep 'corner_lat:' " + dempar + " | awk '{print $2}' " + North = subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'corner_lon:' " + dempar + " | awk '{print $2}' " + West = subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lat:' " + dempar + " | awk '{print $2}' " + posty = subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lon:' " + dempar + " | awk '{print $2}' " + postx = subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'width:' " + dempar + " | awk '{print $2}' " + width = subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'nlines:' " + dempar + " | awk '{print $2}' " + length = subprocess.getstatusoutput(call_str)[1] + + South = str(round(float(North) + (float(length) - 1) * float(posty), 7)) + East = str(round(float(West) + (float(width) - 1) * float(postx), 7)) + + # Convert to elevation angle + call_str = "swap_bytes lv_theta lv_theta.phase_swap 4 > dinsar.log " + os.system(call_str) + + call_str = "gmt xyz2grd lv_theta.phase_swap -Glv_theta.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" + South + "/" + North + " -I" + postx + " -ZTLf -N0" + os.system(call_str) + + # Elevation angle = 90 - incidence angle + call_str = "gmt grdmath 90 lv_theta.grd 3.1415926 DIV 180 MUL SUB = lv_theta_final.grd" + os.system(call_str) + call_str = "gmt grdmath 90 lv_theta_final.grd SUB = lv_elev.grd" + os.system(call_str) + + # Resample to 3 arc-seconds + call_str = "gmt grdsample lv_elev.grd -Glv_elev_3c.grd -I3c" + os.system(call_str) + + # Convert to binary + call_str = "gmt grd2xyz lv_elev_3c.grd -ZTLf -N0 > " + Mdate + '-' + Sdate + '.gacos.elev' + os.system(call_str) + + return workDir + '/' + Mdate + '-' + Sdate + '.gacos.elev' + + +def prepare_unw_for_gacos(workDir, unw, Mdate, Sdate, dempar, rlks): + """ + Prepare unwrapped interferogram for GACOS correction. + + Converts GAMMA format to GACOS-compatible format with .rsc file. + """ + os.chdir(workDir) + + # Get DEM parameters + call_str = "grep 'corner_lat:' " + dempar + " | awk '{print $2}' " + North = subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'corner_lon:' " + dempar + " | awk '{print $2}' " + West = subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lat:' " + dempar + " | awk '{print $2}' " + posty = subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lon:' " + dempar + " | awk '{print $2}' " + postx = subprocess.getstatusoutput(call_str)[1] + + # Convert unw to grd + call_str = "swap_bytes " + unw + " unw.phase_swap 4 > dinsar.log " + os.system(call_str) + + # Get dimensions + call_str = "gmt grdinfo unw_f.grd -C" + result = subprocess.getstatusoutput(call_str)[1].split() + if len(result) >= 12: + Width = result[9] + line = result[10] + West = result[1] + North = result[4] + else: + # Fallback to DEM parameters + call_str = "grep 'width:' " + dempar + " | awk '{print $2}' " + Width = subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'nlines:' " + dempar + " | awk '{print $2}' " + line = subprocess.getstatusoutput(call_str)[1] + + ymax = str(int(round(float(line), 7))) + xmax = str(int(round(float(Width), 7))) + + # Create .rsc file + output = workDir + '/' + Mdate + '-' + Sdate + '.gacos.unw.rsc' + if os.path.exists(output): + os.remove(output) + + with open(output, 'a+') as f: + f.write('WIDTH ' + str(Width) + '\n') + f.write('FILE_LENGTH ' + str(line) + '\n') + f.write('XMIN 1' + '\n') + f.write('XMAX ' + xmax + '\n') + f.write('YMIN 1' + '\n') + f.write('YMAX ' + ymax + '\n') + f.write('X_FIRST ' + str(West) + '\n') + f.write('Y_FIRST ' + str(North) + '\n') + f.write('X_STEP 8.33333333E-04' + '\n') + f.write('Y_STEP -8.33333333E-04' + '\n') + f.write('X_UNIT degrees' + '\n') + f.write('Y_UNIT degrees' + '\n') + f.write('Z_OFFSET 0' + '\n') + f.write('Z_SCALE 1' + '\n') + f.write('PROJECTION LATLON' + '\n') + f.write('DATUM WGS84' + '\n') + + return workDir + '/' + Mdate + '-' + Sdate + '.gacos.unw' + + +def parse_dem_par(dempar): + """ + 从 GAMMA DEM par 文件读取网格参数（纯 Python，无需 grep/awk） + """ + info = {} + with open(dempar) as f: + for line in f: + parts = line.split(':') + if len(parts) >= 2: + key = parts[0].strip() + val = parts[1].strip().split()[0] if parts[1].strip() else '' + info[key] = val + return { + 'width': int(info['width']), + 'nlines': int(info['nlines']), + 'corner_lat': float(info['corner_lat']), + 'corner_lon': float(info['corner_lon']), + 'post_lat': float(info['post_lat']), + 'post_lon': float(info['post_lon']), + } + + +def resample_ztd_to_dem(ztd_file, rsc_file, dem_info): + """ + 读取 ZTD 数据并双线性插值重采样到 DEM 网格 + 修复: RegularGridInterpolator 要求坐标严格递增， + ZTD 纬度为递减（北→南），需翻转 + """ + # 读取 RSC 头文件 + rsc = {} + with open(rsc_file) as f: + for line in f: + parts = line.split() + if len(parts) >= 2: + rsc[parts[0]] = parts[1] + + ztd_w = int(rsc['WIDTH']) + ztd_h = int(rsc['FILE_LENGTH']) + ztd_x0 = float(rsc['X_FIRST']) + ztd_y0 = float(rsc['Y_FIRST']) + ztd_dx = float(rsc['X_STEP']) + ztd_dy = float(rsc['Y_STEP']) + + raw = np.fromfile(ztd_file, dtype=np.float32).reshape(ztd_h, ztd_w) + + # 构建 ZTD 网格坐标 + ztd_lats = ztd_y0 + np.arange(ztd_h) * ztd_dy + ztd_lons = ztd_x0 + np.arange(ztd_w) * ztd_dx + + # 翻转使坐标严格递增（RegularGridInterpolator 要求） + if ztd_lats[0] > ztd_lats[-1]: + ztd_lats = ztd_lats[::-1] + raw = raw[::-1, :] + if ztd_lons[0] > ztd_lons[-1]: + ztd_lons = ztd_lons[::-1] + raw = raw[:, ::-1] + + interp = RegularGridInterpolator( + (ztd_lats, ztd_lons), raw, + method='linear', bounds_error=False, fill_value=np.nan + ) + + # 构建 DEM 网格查询点 + dem_w = dem_info['width'] + dem_h = dem_info['nlines'] + dem_lats = dem_info['corner_lat'] + np.arange(dem_h) * dem_info['post_lat'] + dem_lons = dem_info['corner_lon'] + np.arange(dem_w) * dem_info['post_lon'] + + # 逐行插值（节省内存） + result = np.empty((dem_h, dem_w), dtype=np.float32) + for i in range(dem_h): + pts = np.column_stack([np.full(dem_w, dem_lats[i]), dem_lons]) + result[i] = interp(pts).astype(np.float32) + + return result + + +def convert_ztd_tif_to_binary(tif_file): + """ + 将 .ztd.tif (GeoTIFF) 转换为 .ztd (raw binary) + .ztd.rsc (header) + 返回 .ztd 文件路径 + """ + try: + import rasterio + except ImportError: + from osgeo import gdal + ds = gdal.Open(tif_file) + gt = ds.GetGeoTransform() + w, h = ds.RasterXSize, ds.RasterYSize + data = ds.GetRasterBand(1).ReadAsArray().astype(np.float32) + x0, y0, dx, dy = gt[0], gt[3], gt[1], gt[5] + ds = None + else: + with rasterio.open(tif_file) as src: + data = src.read(1).astype(np.float32) + w, h = src.width, src.height + x0 = src.transform[2] + y0 = src.transform[5] + dx = src.transform[0] + dy = src.transform[4] + + ztd_file = tif_file.replace('.ztd.tif', '.ztd') + rsc_file = tif_file.replace('.ztd.tif', '.ztd.rsc') + + data.tofile(ztd_file) + with open(rsc_file, 'w') as f: + f.write(f"WIDTH {w}\n") + f.write(f"FILE_LENGTH {h}\n") + f.write(f"X_FIRST {x0}\n") + f.write(f"Y_FIRST {y0}\n") + f.write(f"X_STEP {dx}\n") + f.write(f"Y_STEP {dy}\n") + f.write("X_UNIT degrees\n") + f.write("Y_UNIT degrees\n") + f.write("PROJECTION LATLON\n") + f.write("DATUM WGS84\n") + + return ztd_file + + +def apply_gacos_correction_python(geo_unw_file, ztd1_file, ztd1_rsc, ztd2_file, ztd2_rsc, + dempar, slcpar=None, wavelength=None, out_file=None, + figdir=None, pair_str=None): + """ + 纯 Python GACOS 大气校正（不依赖 GMT） + 修复原 make_correction 的所有 Bug: + - 极端值掩码（GAMMA 无效标记） + - sin/cos 弧度转换 + - cut_image2 慢速循环 → scipy 向量化重采样 + - 增加校正前后对比图 + + 参数: + ----- + geo_unw_file : str 地理编码后的解缠相位文件 + ztd1_file : str Master 日期 ZTD 二进制文件 + ztd1_rsc : str Master 日期 ZTD RSC 头文件 + ztd2_file : str Slave 日期 ZTD 二进制文件 + ztd2_rsc : str Slave 日期 ZTD RSC 头文件 + dempar : str DEM 参数文件 + slcpar : str SLC 参数文件（读取入射角，可选） + wavelength : float 雷达波长（m），None则从slcpar读取 + out_file : str 输出文件路径（默认 geo_unw_file + '.gacos'） + figdir : str 对比图保存目录（None则不绘图） + pair_str : str 干涉对名称如 '20241105-20241117' + + 返回: 输出文件路径 + """ + # 1. 读取 DEM 参数 + dem = parse_dem_par(dempar) + dem_w, dem_h = dem['width'], dem['nlines'] + + # 2. 读取入射角和波长 + inc_angle_deg = 39.0 # 默认值 + if slcpar and os.path.isfile(slcpar): + with open(slcpar) as f: + for line in f: + if line.startswith('incidence_angle:'): + inc_angle_deg = float(line.split(':')[1].strip().split()[0]) + if line.startswith('radar_frequency:') and wavelength is None: + freq = float(line.split(':')[1].strip().split()[0]) + wavelength = 299792458.0 / freq + + if wavelength is None: + wavelength = 0.0554657595 # Sentinel-1 C-band 默认值 + + cos_inc = np.cos(np.radians(inc_angle_deg)) + ztd_to_phase = 4.0 * np.pi / wavelength + + print(f" 入射角: {inc_angle_deg:.4f}°, cos={cos_inc:.6f}") + print(f" 波长: {wavelength:.10f} m") + + # 3. 读取地理编码后的解缠相位 + unw_data = np.fromfile(geo_unw_file, dtype=np.float32).reshape(dem_h, dem_w) + + # 4. 重采样 ZTD 到 DEM 网格 + print(" 重采样 ZTD Master...") + ztd_m = resample_ztd_to_dem(ztd1_file, ztd1_rsc, dem) + print(" 重采样 ZTD Slave...") + ztd_s = resample_ztd_to_dem(ztd2_file, ztd2_rsc, dem) + + # 5. 计算 LOS 相位校正量 + dztd = ztd_s - ztd_m + phase_correction = dztd * ztd_to_phase / cos_inc + + # 6. 构建有效像素掩码 + # 排除: 零值 | NaN | GAMMA 伪影/解缠错误（|x| >= 1000 rad ≈ 4.4m LOS） + PHASE_THRESH = 1000.0 + valid = ((unw_data != 0) + & np.isfinite(unw_data) + & (np.abs(unw_data) < PHASE_THRESH) + & np.isfinite(phase_correction)) + n_valid = np.sum(valid) + n_total = dem_w * dem_h + print(f" 有效像素: {n_valid}/{n_total} ({n_valid/n_total*100:.1f}%)") + + if n_valid == 0: + print(" 错误: 无有效像素，跳过校正") + return None + + # 7. 统计校正前 + std_before = float(np.std(unw_data[valid])) + + # 8. 应用校正（保留原始数据结构，仅修改有效像素） + corrected = unw_data.copy() + corrected[valid] = unw_data[valid] - phase_correction[valid] + + # 去均值（仅对有效像素） + mean_val = np.mean(corrected[valid]) + corrected[valid] -= mean_val + + std_after = float(np.std(corrected[valid])) + reduction = (1 - std_after / std_before) * 100 if std_before > 0 else 0 + + print(f" 校正前 std: {std_before:.3f} rad") + print(f" 校正后 std: {std_after:.3f} rad") + print(f" std 降低: {reduction:.1f}%") + + # 9. 保存 + if out_file is None: + out_file = geo_unw_file + '.gacos' + corrected.astype(np.float32).tofile(out_file) + print(f" 输出: {out_file}") + + # 10. 绘制对比图 + if figdir: + plot_gacos_comparison(unw_data, phase_correction, corrected, valid, + dem, std_before, std_after, pair_str, figdir) + + return out_file + + +def plot_gacos_comparison(unw_before, correction, unw_after, valid_mask, + dem_info, std_before, std_after, pair_str, figdir): + """ + 绘制 GACOS 校正前后相位对比图（三子图） + 版本控制: 自动追加 _v1, _v2... + """ + import matplotlib + matplotlib.use('Agg') + import matplotlib.pyplot as plt + import matplotlib.colors as mcolors + + # 中文字体配置 + plt.rcParams['font.sans-serif'] = ['Noto Serif CJK SC', 'Noto Sans CJK SC', + 'AR PL UMing CN', 'SimHei', 'DejaVu Sans'] + plt.rcParams['axes.unicode_minus'] = False + + figdir = Path(figdir) + figdir.mkdir(parents=True, exist_ok=True) + + # 版本控制 + base_name = f"GACOS_comparison_{pair_str}" if pair_str else "GACOS_comparison" + existing = sorted(figdir.glob(f"{base_name}_v*.png")) + ver = int(existing[-1].stem.split('_v')[-1]) + 1 if existing else 1 + out_path = figdir / f"{base_name}_v{ver}.png" + + # 准备显示数据（仅显示有效像素） + before_disp = np.full_like(unw_before, np.nan, dtype=np.float64) + before_disp[valid_mask] = unw_before[valid_mask] + corr_disp = np.full_like(correction, np.nan, dtype=np.float64) + corr_disp[valid_mask] = correction[valid_mask] + after_disp = np.full_like(unw_after, np.nan, dtype=np.float64) + after_disp[valid_mask] = unw_after[valid_mask] + + # 色标范围：使用有效数据的 P2/P98 + vals_b = unw_before[valid_mask] + vals_c = correction[valid_mask] + vals_a = unw_after[valid_mask] + vlim = max(abs(np.percentile(vals_b, 2)), abs(np.percentile(vals_b, 98)), 0.5) + vlim_c = max(abs(np.percentile(vals_c, 2)), abs(np.percentile(vals_c, 98)), 0.5) + vlim_a = max(abs(np.percentile(vals_a, 2)), abs(np.percentile(vals_a, 98)), 0.5) + + # 地理坐标范围 + w = dem_info['width'] + h = dem_info['nlines'] + extent = [dem_info['corner_lon'], + dem_info['corner_lon'] + w * dem_info['post_lon'], + dem_info['corner_lat'] + h * dem_info['post_lat'], + dem_info['corner_lat']] + + # 降采样显示 + step = max(1, h // 2000) + b_s = before_disp[::step, ::step] + c_s = corr_disp[::step, ::step] + a_s = after_disp[::step, ::step] + + # 使用带 NaN 灰色背景的 colormap + cmap_phase = plt.cm.RdBu_r.copy() + cmap_phase.set_bad(color='#E0E0E0') + cmap_corr = plt.cm.coolwarm.copy() + cmap_corr.set_bad(color='#E0E0E0') + + fig, axes = plt.subplots(1, 3, figsize=(18, 6), dpi=150) + for ax in axes: + ax.set_facecolor('#E0E0E0') + + im1 = axes[0].imshow(b_s, cmap=cmap_phase, vmin=-vlim, vmax=vlim, + extent=extent, aspect='auto', interpolation='nearest') + axes[0].set_title(f'Unwrapped Phase (std={std_before:.2f} rad)', fontsize=11, fontweight='bold') + plt.colorbar(im1, ax=axes[0], label='rad', shrink=0.8) + + im2 = axes[1].imshow(c_s, cmap=cmap_corr, vmin=-vlim_c, vmax=vlim_c, + extent=extent, aspect='auto', interpolation='nearest') + axes[1].set_title('GACOS Correction', fontsize=11, fontweight='bold') + plt.colorbar(im2, ax=axes[1], label='rad', shrink=0.8) + + im3 = axes[2].imshow(a_s, cmap=cmap_phase, vmin=-vlim_a, vmax=vlim_a, + extent=extent, aspect='auto', interpolation='nearest') + axes[2].set_title(f'Corrected (std={std_after:.2f} rad)', fontsize=11, fontweight='bold') + plt.colorbar(im3, ax=axes[2], label='rad', shrink=0.8) + + for ax in axes: + ax.set_xlabel('Longitude') + ax.set_ylabel('Latitude') + ax.tick_params(labelsize=9) + + reduction = (1 - std_after / std_before) * 100 if std_before > 0 else 0 + n_valid = np.sum(valid_mask) + if pair_str: + m, s = pair_str.split('-') + fig.suptitle(f'GACOS: {m}-{s} | valid={n_valid:,} px | std {reduction:+.1f}%', + fontsize=13, fontweight='bold', y=1.02) + plt.tight_layout() + fig.savefig(str(out_path), dpi=150, bbox_inches='tight', + facecolor='white', edgecolor='none') + plt.close(fig) + print(f" 对比图: {out_path}") + + +def find_existing_ztd(date, gacos_dir): + """ + Find existing GACOS ZTD file for a specific date. + + Returns: + -------- + ztd_file : str or None + Path to ZTD file (without extension), or None if not found + """ + # Check for .ztd + .ztd.rsc format (preferred) + ztd_pattern = os.path.join(gacos_dir, "**", date + "*.ztd") + ztd_files = glob.glob(ztd_pattern, recursive=True) + if ztd_files: + for ztd_file in ztd_files: + if ztd_file.endswith('.ztd.tif'): + continue + rsc_file = ztd_file + '.rsc' + if os.path.exists(rsc_file): + return ztd_file + + # Check for .ztd.tif format, auto-convert if needed + ztd_pattern = os.path.join(gacos_dir, "**", date + "*.ztd.tif") + tif_files = glob.glob(ztd_pattern, recursive=True) + if tif_files: + tif_file = tif_files[0] + ztd_bin = tif_file.replace('.ztd.tif', '.ztd') + rsc_file = tif_file.replace('.ztd.tif', '.ztd.rsc') + if not os.path.exists(ztd_bin) or not os.path.exists(rsc_file): + print(f" Auto-converting {os.path.basename(tif_file)} -> .ztd + .rsc") + convert_ztd_tif_to_binary(tif_file) + if os.path.exists(ztd_bin) and os.path.exists(rsc_file): + return ztd_bin + + return None + + +def submit_gacos_request(dates, bounds, email, gacos_dir, acquisition_time=None): + """ + Submit GACOS data request for multiple dates. + + Parameters: + ----------- + dates : list + List of acquisition dates in YYYYMMDD format + bounds : tuple + Bounding box (West, South, East, North) + email : str + Email address for GACOS submission + gacos_dir : str + Directory to save GACOS data + acquisition_time : str, optional + Acquisition time in HH:MM format + + Returns: + -------- + success : bool + True if submission successful + """ + if not AUTOGACOS_AVAILABLE: + print("AutoGACOS module not available. Please download GACOS data manually.") + return False + + try: + from gacos import Submitter, SarDataset + import pandas as pd + + # Create datetime index + if acquisition_time: + hour, minute = map(int, acquisition_time.split(':')) + else: + hour, minute = 10, 0 # Sentinel-1 升轨默认采集时间 ~10:00 UTC + + date_strings = [f"{d} {hour:02d}:{minute:02d}:00" for d in dates] + date_times = pd.to_datetime(date_strings) + + # Create dataset + dataset = SarDataset(bounds, date_times, gacos_dir) + + print(f"\nSubmitting GACOS request for {len(dates)} dates...") + print(f"Bounds: W={bounds[0]:.4f}, S={bounds[1]:.4f}, E={bounds[2]:.4f}, N={bounds[3]:.4f}") + print(f"Acquisition time: {hour:02d}:{minute:02d} UTC") + print(f"Email: {email}") + + # Submit request + submitter = Submitter(dataset, email) + submitter.post_requests() + + if submitter.succeed: + print(f"\nSuccessfully submitted {len(submitter.succeed)} requests.") + if submitter.failed: + print(f"\nFailed to submit {len(submitter.failed)} requests.") + + return len(submitter.succeed) > 0 + + except Exception as e: + print(f"Error in GACOS submission: {e}") + import traceback + traceback.print_exc() + return False + + +def download_gacos_from_email(email_config, gacos_dir, bounds=None, times=None, + submit_time=None): + """ + 从邮箱检索 gacos2017@foxmail.com 发来的 GACOS 下载链接并下载数据。 + + Parameters: + ----------- + email_config : dict + 邮箱配置: username, password, host, port, ssl + gacos_dir : str + GACOS 数据保存目录 + bounds : tuple, optional + 边界框过滤 (West, South, East, North) + times : list, optional + 采集时间过滤 + submit_time : str, optional + 提交时间（YYYY-MM-DD HH:MM:SS），仅检索此时间之后的邮件 + + Returns: + -------- + downloaded_files : list + 已下载的 ZTD 文件路径列表 + """ + if not AUTOGACOS_AVAILABLE: + print("AutoGACOS module not available.") + return [] + + try: + from gacos import Downloader, GACOSEmail + + # Step 1: 从邮箱检索 GACOS 下载链接 + print(" 检索 gacos2017@foxmail.com 邮件中的下载链接...") + + email_retriever = GACOSEmail( + username=email_config.get('username'), + password=email_config.get('password'), + host=email_config.get('host', 'imap.163.com'), + port=email_config.get('port'), + email_protocol=email_config.get('protocol', 'imap'), + ssl=email_config.get('ssl', True), + gacos_email='gacos2017@foxmail.com', + start_date=submit_time + ) + + url_file = os.path.join(gacos_dir, 'gacos_urls.csv') + email_retriever.retrieve_gacos_urls(url_file) + + if not os.path.exists(url_file): + print(" 未找到 GACOS 下载链接") + return [] + + # 检查 CSV 是否有内容 + import pandas as pd + try: + df = pd.read_csv(url_file) + if len(df) == 0: + print(" 邮件中未发现新的 GACOS 数据链接") + return [] + print(f" 找到 {len(df)} 个下载链接") + except Exception: + print(" URL 文件解析失败") + return [] + + # Step 2: 下载文件 + print(" 下载 GACOS 数据...") + + dl = Downloader( + url_file=url_file, + output_dir=gacos_dir, + bounds=bounds, + times=times, + keep_original=False + ) + + dl.download() + + # Step 3: 将下载的 .ztd.tif 转换为 .ztd + .rsc + downloaded = [] + for tif_file in glob.glob(os.path.join(gacos_dir, "*.ztd.tif")): + rsc_file = tif_file.replace('.ztd.tif', '.ztd.rsc') + if not os.path.exists(rsc_file): + ztd_file = convert_ztd_tif_to_binary(tif_file) + if ztd_file: + downloaded.append(ztd_file) + print(f" 转换: {os.path.basename(tif_file)} → .ztd + .rsc") + else: + downloaded.append(tif_file.replace('.ztd.tif', '.ztd')) + + return downloaded + + except Exception as e: + print(f" 邮箱检索/下载出错: {e}") + import traceback + traceback.print_exc() + return [] + + +def auto_gacos_workflow(dates, bounds, gacos_dir, email, email_config=None, + acquisition_time=None, wait_for_email=False, max_wait_hours=24, + check_interval=60): + """ + Complete automatic GACOS workflow: submit request, wait, and download. + + Parameters: + ----------- + dates : list + List of acquisition dates in YYYYMMDD format + bounds : tuple + Bounding box (West, South, East, North) + gacos_dir : str + Directory to save GACOS data + email : str + Email address for GACOS submission + email_config : dict, optional + Email configuration for downloading (username, password, host, etc.) + acquisition_time : str, optional + Acquisition time in HH:MM format + wait_for_email : bool, optional + Whether to wait for email notification before downloading + max_wait_hours : int, optional + Maximum hours to wait for email + + Returns: + -------- + ztd_files : dict + Dictionary mapping dates to ZTD file paths + """ + import time as time_module + from datetime import datetime, timedelta + + ztd_files = {} + missing_dates = [] + + # Step 1: Check existing files + print("\n" + "="*60) + print("Step 1: Checking existing GACOS data...") + print("="*60) + + for date in dates: + ztd_file = find_existing_ztd(date, gacos_dir) + if ztd_file: + ztd_files[date] = ztd_file + else: + missing_dates.append(date) + + if not missing_dates: + print("\nAll required GACOS data already exists!") + return ztd_files + + print(f"\nMissing GACOS data for {len(missing_dates)} dates:") + for d in missing_dates: + print(f" - {d}") + + # Step 2: Submit request for missing dates + print("\n" + "="*60) + print("Step 2: Submitting GACOS requests...") + print("="*60) + + submit_success = submit_gacos_request( + dates=missing_dates, + bounds=bounds, + email=email, + gacos_dir=gacos_dir, + acquisition_time=acquisition_time + ) + + if not submit_success: + print("\n提交失败（可能是重复提交或服务器繁忙），尝试从邮箱获取已有数据...") + else: + print("\nGACOS request submitted successfully!") + print("You will receive an email with download links when data is ready.") + + # Step 3: Wait and download from email (if configured) + if email_config and wait_for_email: + print("\n" + "="*60) + print("Step 3: 检查 gacos2017@foxmail.com 邮件并下载...") + print("="*60) + + # 记录时间基准，仅检索此时间之后的 GACOS 邮件，过滤掉旧邮件 + if submit_success: + submit_time_str = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + else: + # 提交失败（重复提交），向前回溯 1 小时查找已有邮件 + submit_time_str = (datetime.now() - timedelta(hours=1)).strftime('%Y-%m-%d %H:%M:%S') + print(f"邮件过滤起始时间: {submit_time_str}") + + start_time = datetime.now() + # 提交成功则等 3 分钟再检查；提交失败则立即检查邮箱 + first_wait = 180 if submit_success else 5 + retry_wait = 120 # 之后每 2 分钟 + attempt = 0 + + # 首次等待 + print(f"\n提交完成，等待 {first_wait//60} 分钟后首次检查邮箱...") + time_module.sleep(first_wait) + + while (datetime.now() - start_time) < timedelta(hours=max_wait_hours): + attempt += 1 + elapsed = datetime.now() - start_time + elapsed_min = int(elapsed.total_seconds() // 60) + elapsed_sec = int(elapsed.total_seconds() % 60) + print(f"\n[第 {attempt} 次检查] 已等待 {elapsed_min}m{elapsed_sec}s") + + downloaded = download_gacos_from_email( + email_config=email_config, + gacos_dir=gacos_dir, + bounds=bounds, + submit_time=submit_time_str + ) + + # 检查缺失日期是否已下载 + for date in missing_dates[:]: + ztd_file = find_existing_ztd(date, gacos_dir) + if ztd_file: + ztd_files[date] = ztd_file + missing_dates.remove(date) + print(f" ✓ 已获取: {date}") + + if not missing_dates: + print(f"\n所有 {len(dates)} 个日期的 GACOS 数据下载完成！") + break + + print(f" 仍缺 {len(missing_dates)} 个日期，{retry_wait//60} 分钟后重试...") + time_module.sleep(retry_wait) + + if missing_dates: + print(f"\n超时！仍缺 {len(missing_dates)} 个日期的 GACOS 数据。") + + return ztd_files + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Apply GACOS atmospheric correction to GAMMA interferograms.', + formatter_class=argparse.RawTextHelpFormatter, + epilog=INTRODUCTION + '\n' + EXAMPLE) + + parser.add_argument('projectName', help='projectName for processing.') + parser.add_argument('pair', help='Master-Slave, e.g., 20150101-20150106.') + parser.add_argument('--ztd-dir', dest='ztdDir', help='Directory containing GACOS ZTD files.') + parser.add_argument('--ztd1', dest='ztd1', help='GACOS ZTD file for master date.') + parser.add_argument('--ztd2', dest='ztd2', help='GACOS ZTD file for slave date.') + parser.add_argument('--email', dest='email', help='Email for GACOS auto-download.') + parser.add_argument('--wavelength', dest='wavelength', type=float, help='Radar wavelength in meters.') + + # Email configuration for auto-download + parser.add_argument('--email-user', dest='emailUser', help='Email username for downloading.') + parser.add_argument('--email-pass', dest='emailPass', help='Email password for downloading.') + parser.add_argument('--email-host', dest='emailHost', default='imap.gmail.com', help='Email IMAP host.') + parser.add_argument('--email-port', dest='emailPort', type=int, default=993, help='Email IMAP port.') + parser.add_argument('--email-ssl', dest='emailSsl', action='store_true', default=True, help='Use SSL for email.') + + # Auto-download options + parser.add_argument('--auto-download', dest='autoDownload', action='store_true', + help='Automatically submit request and download.') + parser.add_argument('--wait-hours', dest='waitHours', type=int, default=24, + help='Maximum hours to wait for email notification.') + + inps = parser.parse_args() + return inps + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Pair = inps.pair + + # Parse dates + Mdate = ut.yyyymmdd(Pair.split('-')[0]) + Sdate = ut.yyyymmdd(Pair.split('-')[1]) + + # Setup directories + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict = ut.update_template(templateFile) + + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + + ifgDir = scratchDir + '/' + projectName + "/ifgrams" + demDir = scratchDir + '/' + projectName + "/DEM" + slcDir = scratchDir + '/' + projectName + "/SLC" + workDir = ifgDir + '/' + Pair + + # GACOS data directory + if inps.ztdDir: + gacosDir = inps.ztdDir + else: + gacosDir = scratchDir + '/' + projectName + "/GACOS" + + if not os.path.exists(gacosDir): + os.makedirs(gacosDir) + + # Define input files + dempar = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem.par' + slcpar = slcDir + '/' + masterDate + '/' + masterDate + '.slc.par' + geo_unw = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.unw' + out_file = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.unw.gacos' + figdir = scratchDir + '/' + projectName + "/figure" + + # Check if geocoded interferogram exists + if not os.path.isfile(geo_unw): + print(f"Error: Geocoded interferogram not found: {geo_unw}") + print("Please run geocode_gamma.py first.") + sys.exit(1) + + print("="*60) + print("GACOS Atmospheric Correction (Pure Python)") + print(f"Project: {projectName}") + print(f"Pair: {Pair}") + print("="*60) + + # Step 1: Find GACOS ZTD files + print("\nStep 1: Checking GACOS ZTD files...") + + ztd1 = None + ztd2 = None + + if inps.ztd1 and inps.ztd2: + ztd1 = inps.ztd1 + ztd2 = inps.ztd2 + else: + ztd1 = find_existing_ztd(Mdate, gacosDir) + ztd2 = find_existing_ztd(Sdate, gacosDir) + + if ztd1 and ztd2: + print(f" Master ZTD: {ztd1}") + print(f" Slave ZTD: {ztd2}") + elif inps.email: + # Auto-download workflow (existing logic preserved) + dem_info = parse_dem_par(dempar) + North = dem_info['corner_lat'] + West = dem_info['corner_lon'] + South = North + (dem_info['nlines'] - 1) * dem_info['post_lat'] + East = West + (dem_info['width'] - 1) * dem_info['post_lon'] + bounds = (West, South, East, North) + + missing_dates = [] + if not ztd1: missing_dates.append(Mdate) + if not ztd2: missing_dates.append(Sdate) + + if inps.autoDownload: + email_config = None + if inps.emailUser and inps.emailPass: + email_config = { + 'username': inps.emailUser, 'password': inps.emailPass, + 'host': inps.emailHost, 'port': inps.emailPort, 'ssl': inps.emailSsl + } + ztd_files = auto_gacos_workflow( + dates=missing_dates, bounds=bounds, gacos_dir=gacosDir, + email=inps.email, email_config=email_config, + wait_for_email=(email_config is not None), max_wait_hours=inps.waitHours + ) + ztd1 = ztd1 or ztd_files.get(Mdate) + ztd2 = ztd2 or ztd_files.get(Sdate) + else: + submit_gacos_request(dates=missing_dates, bounds=bounds, + email=inps.email, gacos_dir=gacosDir) + print(f"\nGACOS request submitted for: {missing_dates}") + sys.exit(0) + + if ztd1 is None or ztd2 is None: + print("\nError: GACOS ZTD files not found.") + print(f" Missing: {Mdate if not ztd1 else ''} {Sdate if not ztd2 else ''}") + print(f" Search dir: {gacosDir}") + sys.exit(1) + + # Step 2: Apply GACOS correction (Pure Python, no GMT dependency) + print("\nStep 2: Applying GACOS correction...") + + ztd1_rsc = ztd1 + '.rsc' + ztd2_rsc = ztd2 + '.rsc' + + corrected_file = apply_gacos_correction_python( + geo_unw_file=geo_unw, + ztd1_file=ztd1, ztd1_rsc=ztd1_rsc, + ztd2_file=ztd2, ztd2_rsc=ztd2_rsc, + dempar=dempar, + slcpar=slcpar, + wavelength=inps.wavelength, + out_file=out_file, + figdir=figdir, + pair_str=Pair + ) + + if corrected_file: + # Step 3: Generate BMP preview using rasdt_pwr + print("\nStep 3: Generating BMP preview...") + geo_amp = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.amp' + nWidthUTMDEM = str(parse_dem_par(dempar)['width']) + + if os.path.isfile(geo_amp): + call_str = ('rasdt_pwr ' + corrected_file + ' ' + geo_amp + ' ' + + nWidthUTMDEM + ' - - - - -3.14 3.14 1 rmg.cm') + print(f" {call_str}") + os.system(call_str) + bmp_file = corrected_file + '.bmp' + if os.path.isfile(bmp_file): + print(f" BMP: {bmp_file}") + else: + print(" Warning: BMP file not generated") + else: + print(f" Warning: geo_amp not found: {geo_amp}") + + print(f"\nGACOS atmospheric correction completed successfully!") + else: + print("\nGACOS correction failed!") + sys.exit(1) + + sys.exit(0) + + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/gacos_gamma_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/gacos_gamma_all.py new file mode 100644 index 0000000..f1ff747 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/gacos_gamma_all.py @@ -0,0 +1,494 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import time +import glob +import argparse +import subprocess + +from pyint import _utils as ut + + +def work(data0): + """Worker function for parallel processing""" + cmd = data0[0] + err_file = data0[1] + p = subprocess.run(cmd, shell=False, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa = stderr.decode("utf-8") + else: + aa = stderr + + if aa: + print(aa) + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + with open(err_file, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Apply GACOS atmospheric correction to all interferograms + for one project using GAMMA. + + This script processes all interferogram pairs listed in + ifgram_list.txt and applies GACOS tropospheric correction. +------------------------------------------------------------------- +''' + +EXAMPLE = ''' + Usage: + gacos_gamma_all.py projectName + gacos_gamma_all.py projectName --parallel 4 + gacos_gamma_all.py projectName --parallel 4 --ztd-dir /path/to/gacos + gacos_gamma_all.py projectName --parallel 4 --ifgramList-txt /test/ifgram_list.txt +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Apply GACOS atmospheric correction to all interferograms for one project.', + formatter_class=argparse.RawTextHelpFormatter, + epilog=INTRODUCTION + '\n' + EXAMPLE) + + parser.add_argument('projectName', help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, + help='Enable parallel processing and Specify the number of processors.') + parser.add_argument('--ifgarmList-txt', dest='ifgarmListTxt', + help='provided ifgram_list_txt. default: using ifgram_list.txt under projectName folder.') + parser.add_argument('--ztd-dir', dest='ztdDir', + help='Directory containing GACOS ZTD files. Default: projectName/GACOS') + parser.add_argument('--email', dest='email', + help='Email for GACOS auto-download.') + parser.add_argument('--auto-download', dest='autoDownload', action='store_true', + help='Automatically submit and download missing GACOS data.') + parser.add_argument('--email-user', dest='emailUser', help='Email username for IMAP download.') + parser.add_argument('--email-pass', dest='emailPass', help='Email password for IMAP download.') + parser.add_argument('--email-host', dest='emailHost', default='imap.gmail.com', help='IMAP host.') + parser.add_argument('--email-port', dest='emailPort', type=int, default=None, help='IMAP port.') + parser.add_argument('--email-ssl', dest='emailSsl', action='store_true', default=True, help='Use SSL.') + parser.add_argument('--wait-hours', dest='waitHours', type=int, default=24, + help='Max hours to wait for GACOS email.') + parser.add_argument('--skip-existing', dest='skipExisting', action='store_true', + help='Skip pairs that already have corrected output.') + + inps = parser.parse_args() + return inps + + +def check_gacos_data_availability(dates, gacos_dir, bounds): + """ + Check if GACOS ZTD data is available for all dates. + + Parameters: + ----------- + dates : list + List of acquisition dates (YYYYMMDD format) + gacos_dir : str + Directory containing GACOS data + bounds : tuple + Bounding box (West, South, East, North) + + Returns: + -------- + missing_dates : list + List of dates that don't have GACOS data + """ + missing_dates = [] + + for date in dates: + # Check for ZTD files + ztd_pattern = os.path.join(gacos_dir, date + "*.ztd.tif") + ztd_files = glob.glob(ztd_pattern) + + if not ztd_files: + # Also check for .ztd format + ztd_pattern = os.path.join(gacos_dir, date + "*.ztd") + ztd_files = glob.glob(ztd_pattern) + + if not ztd_files: + missing_dates.append(date) + + return missing_dates + + +def get_project_dates(projectName, scratchDir): + """ + Get all acquisition dates from the project. + + Returns: + -------- + dates : list + List of unique acquisition dates + """ + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + + dates = set() + + # Check SLC directory + if os.path.exists(slcDir): + for item in os.listdir(slcDir): + if os.path.isdir(os.path.join(slcDir, item)): + if len(item) == 8 and item.isdigit(): + dates.add(item) + + # Check RSLC directory + if os.path.exists(rslcDir): + for item in os.listdir(rslcDir): + if os.path.isdir(os.path.join(rslcDir, item)): + if len(item) == 8 and item.isdigit(): + dates.add(item) + + return sorted(list(dates)) + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + projectDir = scratchDir + '/' + projectName + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + + templateDict = ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + + # GACOS data directory (CLI > template > default) + if inps.ztdDir: + gacosDir = inps.ztdDir + elif templateDict.get('gacos_dir', '').strip(): + gacosDir = templateDict['gacos_dir'].strip() + else: + gacosDir = projectDir + '/GACOS' + + if not os.path.exists(gacosDir): + os.makedirs(gacosDir) + print(f"Created GACOS directory: {gacosDir}") + + # 从模板读取邮箱配置（CLI 优先于模板） + gacos_email = (inps.email or templateDict.get('gacos_email', '').strip()) or None + gacos_email_user = (getattr(inps, 'emailUser', None) or templateDict.get('gacos_email_user', '').strip()) or None + gacos_email_pass = (getattr(inps, 'emailPass', None) or templateDict.get('gacos_email_pass', '').strip()) or None + gacos_email_host = (getattr(inps, 'emailHost', None) or templateDict.get('gacos_email_host', 'imap.163.com').strip()) + gacos_email_port = getattr(inps, 'emailPort', None) or int(templateDict.get('gacos_email_port', '993').strip()) + gacos_email_ssl = templateDict.get('gacos_email_ssl', '1').strip() == '1' + gacos_check_interval = int(templateDict.get('gacos_check_interval', '60').strip()) + + # 如果有用户名和密码，自动启用 auto-download + auto_download = getattr(inps, 'autoDownload', False) or (gacos_email_user and gacos_email_pass) + + # Key files + demDir = projectDir + '/DEM' + slcDir = projectDir + '/SLC' + dempar = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem.par' + slcpar = slcDir + '/' + masterDate + '/' + masterDate + '.slc.par' + figdir = projectDir + '/figure' + + # Import pure Python correction functions from gacos_gamma + from pyint.gacos_gamma import (parse_dem_par, resample_ztd_to_dem, + find_existing_ztd, apply_gacos_correction_python, + submit_gacos_request, auto_gacos_workflow) + + # Get interferogram list + if inps.ifgarmListTxt: + ifgramList_txt = inps.ifgarmListTxt + else: + ifgramList_txt = projectDir + '/ifgram_list.txt' + + if not os.path.isfile(ifgramList_txt): + print(f"Error: Interferogram list not found: {ifgramList_txt}") + sys.exit(1) + + ifgList0 = ut.read_txt2array(ifgramList_txt) + + import numpy as np + ifgList0 = np.atleast_2d(np.array(ifgList0)) + ifgList = list(ifgList0[:, 0]) + + # Collect all unique dates + all_dates = set() + for ifg in ifgList: + mdate = ut.yyyymmdd(ifg.split('-')[0]) + sdate = ut.yyyymmdd(ifg.split('-')[1]) + all_dates.add(mdate) + all_dates.add(sdate) + all_dates = sorted(list(all_dates)) + + # Read DEM parameters + dem_info = parse_dem_par(dempar) + + print("="*60) + print("GACOS Atmospheric Correction (Pure Python, ZTD Pre-cache)") + print(f"Project: {projectName}") + print(f"Interferograms: {len(ifgList)}, Dates: {len(all_dates)}") + print(f"DEM: {dem_info['width']}×{dem_info['nlines']}") + print(f"GACOS dir: {gacosDir}") + print("="*60) + + # Check GACOS data availability using updated find_existing_ztd + print("\nChecking GACOS data availability...") + missing_dates = [] + ztd_paths = {} # {date: ztd_file_path} + for d in all_dates: + ztd = find_existing_ztd(d, gacosDir) + if ztd: + ztd_paths[d] = ztd + else: + missing_dates.append(d) + + if missing_dates: + print(f"\n⚠ GACOS data missing for {len(missing_dates)} dates: {missing_dates[:5]}...") + print(f"ZTD available: {len(ztd_paths)}/{len(all_dates)} dates") + + # 自动提交/下载缺失的 GACOS 数据 + if gacos_email: + # 从 DEM 参数计算研究区边界框 + North = dem_info['corner_lat'] + West = dem_info['corner_lon'] + South = North + (dem_info['nlines'] - 1) * dem_info['post_lat'] + East = West + (dem_info['width'] - 1) * dem_info['post_lon'] + bounds = (West, South, East, North) + + # 获取 SAR 采集时间（GACOS 要求非零） + acq_time = templateDict.get('gacos_acq_time', '').strip() + if not acq_time and os.path.isfile(slcpar): + try: + with open(slcpar) as f: + for line in f: + if line.startswith('center_time:'): + secs = float(line.split()[1]) + h = int(secs // 3600) + m = int((secs % 3600) // 60) + acq_time = f"{h:02d}:{m:02d}" + print(f"从 SLC par 读取采集时间: {acq_time} UTC") + break + except Exception: + pass + if not acq_time: + acq_time = '10:00' + print(f"使用默认采集时间: {acq_time} UTC") + + if auto_download and gacos_email_user and gacos_email_pass: + # 完整自动流程：提交 → 等待邮件 → 下载 + email_config = { + 'username': gacos_email_user, 'password': gacos_email_pass, + 'host': gacos_email_host, 'port': gacos_email_port, 'ssl': gacos_email_ssl + } + wait_hours = getattr(inps, 'waitHours', 24) + ztd_files = auto_gacos_workflow( + dates=missing_dates, bounds=bounds, gacos_dir=gacosDir, + email=gacos_email, email_config=email_config, + acquisition_time=acq_time, + wait_for_email=True, max_wait_hours=wait_hours, + check_interval=gacos_check_interval + ) + # 重新检查已下载的日期 + for d in missing_dates[:]: + ztd = find_existing_ztd(d, gacosDir) + if ztd: + ztd_paths[d] = ztd + missing_dates.remove(d) + else: + # 仅提交请求，不等待下载 + submit_gacos_request( + dates=missing_dates, bounds=bounds, + email=gacos_email, gacos_dir=gacosDir, + acquisition_time=acq_time + ) + print(f"\nGACOS request submitted for {len(missing_dates)} dates.") + print("Please wait for email notification, then re-run this script.") + if not ztd_paths: + print("No ZTD data available yet. Exiting.") + sys.exit(0) + else: + print("\nTip: 在模板中配置 gacos_email 可自动提交 GACOS 请求") + print(" 配置 gacos_email_user + gacos_email_pass 可自动轮询下载") + + if missing_dates: + print(f"\n仍有 {len(missing_dates)} 个日期缺失 ZTD，相关干涉对将被跳过") + + print(f"\nZTD available: {len(ztd_paths)}/{len(all_dates)} dates") + + # Pre-cache all ZTD resampled to DEM grid (each date only once) + print(f"\nPre-loading {len(ztd_paths)} ZTD files to DEM grid...") + t0 = time.time() + ztd_cache = {} + for i, (d, ztd_file) in enumerate(sorted(ztd_paths.items())): + ztd_rsc = ztd_file + '.rsc' + ztd_cache[d] = resample_ztd_to_dem(ztd_file, ztd_rsc, dem_info) + if (i + 1) % 10 == 0 or i == 0: + print(f" [{i+1}/{len(ztd_paths)}] {d}") + t_cache = time.time() - t0 + mem_gb = sum(v.nbytes for v in ztd_cache.values()) / 1024**3 + print(f" Pre-load done: {t_cache:.1f}s, ~{mem_gb:.2f} GB") + + # Read incidence angle and wavelength from SLC par + import numpy as np + inc_angle_deg = 39.0 + wavelength = 0.0554657595 + if os.path.isfile(slcpar): + with open(slcpar) as f: + for line in f: + if line.startswith('incidence_angle:'): + inc_angle_deg = float(line.split(':')[1].strip().split()[0]) + if line.startswith('radar_frequency:'): + freq = float(line.split(':')[1].strip().split()[0]) + wavelength = 299792458.0 / freq + cos_inc = np.cos(np.radians(inc_angle_deg)) + ztd_to_phase = 4.0 * np.pi / wavelength + + print(f"\n入射角: {inc_angle_deg:.4f}°, 波长: {wavelength:.10f} m") + + # Process each pair + print(f"\nProcessing {len(ifgList)} interferograms...\n") + + ok_count = 0 + skip_count = 0 + err_count = 0 + errors = [] + plot_count = 0 + MAX_PLOTS = 5 # only plot first 5 pairs + + for i, ifg in enumerate(ifgList): + mdate = ut.yyyymmdd(ifg.split('-')[0]) + sdate = ut.yyyymmdd(ifg.split('-')[1]) + pair = mdate + '-' + sdate + + workDir = ifgDir + '/' + pair + unw_file = workDir + '/geo_' + pair + '_' + rlks + 'rlks.diff_filt.unw' + out_file = workDir + '/geo_' + pair + '_' + rlks + 'rlks.diff_filt.unw.gacos' + + # Skip if already exists + if inps.skipExisting and os.path.isfile(out_file) and os.path.getsize(out_file) > 0: + skip_count += 1 + continue + + # Skip if input missing + if not os.path.isfile(unw_file): + err_count += 1 + errors.append(f"{pair}: geo_unw not found") + continue + + # Skip if ZTD missing + if mdate not in ztd_cache or sdate not in ztd_cache: + err_count += 1 + errors.append(f"{pair}: ZTD missing for {mdate if mdate not in ztd_cache else sdate}") + continue + + try: + # Read unwrapped phase + unw_data = np.fromfile(unw_file, dtype=np.float32).reshape( + dem_info['nlines'], dem_info['width']) + + # Compute phase correction from cached ZTD + dztd = ztd_cache[sdate] - ztd_cache[mdate] + phase_correction = dztd * ztd_to_phase / cos_inc + + # Valid pixel mask (exclude zeros, NaN, GAMMA artifacts/unwrap errors) + PHASE_THRESH = 1000.0 + valid = ((unw_data != 0) + & np.isfinite(unw_data) + & (np.abs(unw_data) < PHASE_THRESH) + & np.isfinite(phase_correction)) + n_valid = np.sum(valid) + + if n_valid == 0: + err_count += 1 + errors.append(f"{pair}: no valid pixels") + continue + + std_before = float(np.std(unw_data[valid])) + + # Apply correction (preserve original data, only modify valid pixels) + corrected = unw_data.copy() + corrected[valid] = unw_data[valid] - phase_correction[valid] + + # Demean (only valid pixels) + mean_val = np.mean(corrected[valid]) + corrected[valid] -= mean_val + + std_after = float(np.std(corrected[valid])) + + # Save + corrected.astype(np.float32).tofile(out_file) + + # Generate BMP preview using rasdt_pwr + geo_amp = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.amp' + if os.path.isfile(geo_amp): + call_str = ('rasdt_pwr ' + out_file + ' ' + geo_amp + ' ' + + str(dem_info['width']) + + ' - - - - -3.14 3.14 1 rmg.cm') + os.system(call_str) + + reduction = (1 - std_after / std_before) * 100 if std_before > 0 else 0 + ok_count += 1 + + # Plot first N pairs + do_plot = (plot_count < MAX_PLOTS) + if do_plot: + from pyint.gacos_gamma import plot_gacos_comparison + plot_gacos_comparison(unw_data, phase_correction, corrected, + valid, dem_info, std_before, std_after, + pair, figdir) + plot_count += 1 + + if ok_count <= 5 or ok_count % 50 == 0: + print(f" [{i+1}/{len(ifgList)}] {pair}: ✅ std {std_before:.2f}->{std_after:.2f} ({reduction:.1f}%)") + + except Exception as e: + err_count += 1 + errors.append(f"{pair}: {str(e)}") + if err_count <= 3: + print(f" [{i+1}/{len(ifgList)}] {pair}: ❌ {e}") + + elapsed = time.time() - start_time + + # Summary + print("\n" + "="*60) + print("GACOS Atmospheric Correction Summary") + print("="*60) + print(f" Total: {len(ifgList)}") + print(f" Success: {ok_count}") + print(f" Skipped: {skip_count}") + print(f" Failed: {err_count}") + + if errors: + err_txt = projectDir + '/gacos_gamma_all.err' + with open(err_txt, 'w') as f: + for e in errors: + f.write(e + '\n') + print(f"\n Error log: {err_txt}") + for e in errors[:5]: + print(f" {e}") + + print(f"\n Time: {elapsed/60:.1f} min") + if plot_count > 0: + print(f" Comparison plots: {figdir}/GACOS_comparison_*.png") + print("="*60) + + sys.exit(0) + + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/gamma2licsbas_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/gamma2licsbas_gamma.py new file mode 100644 index 0000000..695f336 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/gamma2licsbas_gamma.py @@ -0,0 +1,319 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import os +import sys +import argparse + +from pyint import _utils as ut + + +INTRODUCTION = ''' +------------------------------------------------------------------- + 将 geocode_gamma.py 已地理编码的 GAMMA 二进制产品转换为 + LiCSBAS 所需的 GeoTIFF 格式, 并组织到 GEOC 目录结构。 + + 本脚本 **不做任何地理编码或计算**, 仅执行格式转换: + 1. data2geotiff : GAMMA 二进制 (EQA) → GeoTIFF + 2. GMT grdmath : 从 look_vector 计算 E/N/U 分量 + 3. 文件复制 : 组织到 GEOC/{Pair}/ 目录 + + 所有地理编码产品由 geocode_gamma.py 提供: + amp, corr, unw, diff_filt, diff, dem, lv_theta, lv_phi +''' + +EXAMPLE = ''' + Usage: + gamma2licsbas_gamma.py projectName Mdate-Sdate + gamma2licsbas_gamma.py PacayaT163TsxHhA 20150102-20150601 +------------------------------------------------------------------- +''' + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Convert geocoded GAMMA binaries to LiCSBAS GeoTIFF format.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('pair',help='Master-Slave, e.g., 20150101-20150106.') + + inps = parser.parse_args() + return inps + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Pair = inps.pair + + scratchDir = os.getenv('SCRATCHDIR') + ifgDir = scratchDir + '/' + projectName + "/ifgrams" + demDir = scratchDir + '/' + projectName + "/DEM" + workDir = ifgDir + '/' + Pair + + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + + ################################################################# + # DEM 参数 (直接引用, 不复制不删除) + ################################################################# + DEM_par = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem.par' + DEM = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem' + DEM_tif = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem.tif' + + # look_vector 文件 (由 geocode_gamma.py 生成) + lv_theta = workDir + '/' + 'lv_theta' + lv_phi = workDir + '/' + 'lv_phi' + lv_theta_tif = workDir + '/' + 'lv_theta.geo.tif' + lv_phi_tif = workDir + '/' + 'lv_phi.geo.tif' + lv_theta_grd = workDir + '/' + 'lv_theta.grd' + lv_theta_grd_final = workDir + '/' + 'inc_deg.nc' + lv_phi_grd = workDir + '/' + 'lv_phi.grd' + lv_phi_grd_final = workDir + '/' + 'azi_deg.nc' + UE_grd = workDir + '/' + 'ue.grd' + UN_grd = workDir + '/' + 'un.grd' + UU_grd = workDir + '/' + 'uu.grd' + UE_tif = workDir + '/' + 'ue.geo.E.tif' + UN_tif = workDir + '/' + 'un.geo.N.tif' + UU_tif = workDir + '/' + 'uu.geo.U.tif' + + if not os.path.exists(lv_theta) or not os.path.exists(lv_phi): + print(f"WARNING: look_vector 文件不存在, 请先运行 geocode_gamma.py {projectName} {Pair}") + print(f" 缺少: lv_theta={os.path.exists(lv_theta)}, lv_phi={os.path.exists(lv_phi)}") + + GEOC = scratchDir + '/' + projectName + '/GEOC' + if not os.path.isdir(GEOC): + os.mkdir(GEOC) + Store_dir = GEOC + '/' + Pair + if not os.path.isdir(Store_dir): + os.mkdir(Store_dir) + + # 判断DEM_tif是否存在 + if not os.path.exists(DEM_tif): + call_str = 'data2geotiff ' + DEM_par + ' ' + DEM + ' 2 ' + DEM_tif + os.system(call_str) + else: + print(f"File {DEM_tif} already exists, skipping generation.") + + # 判断lv_theta_tif是否存在 + if not os.path.exists(lv_theta_tif): + call_str = 'data2geotiff ' + DEM_par + ' ' + lv_theta + ' 2 ' + lv_theta_tif + os.system(call_str) + else: + print(f"File {lv_theta_tif} already exists, skipping generation.") + + # 判断lv_phi_tif是否存在 + if not os.path.exists(lv_phi_tif): + call_str = 'data2geotiff ' + DEM_par + ' ' + lv_phi + ' 2 ' + lv_phi_tif + os.system(call_str) + else: + print(f"File {lv_phi_tif} already exists, skipping generation.") + + # 判断lv_theta_grd是否存在 + if not os.path.exists(lv_theta_grd): + call_str = 'gdal_translate -of GSBG ' + lv_theta_tif + ' ' + lv_theta_grd + os.system(call_str) + else: + print(f"File {lv_theta_grd} already exists, skipping conversion.") + + # 判断lv_phi_grd是否存在 + if not os.path.exists(lv_phi_grd): + call_str = 'gdal_translate -of GSBG ' + lv_phi_tif + ' ' + lv_phi_grd + os.system(call_str) + else: + print(f"File {lv_phi_grd} already exists, skipping conversion.") + + # 判断lv_theta_grd_final是否存在 + if not os.path.exists(lv_theta_grd_final): + call_str = 'gmt grdmath 90 ' + lv_theta_grd + ' 3.1415926 DIV 180 MUL SUB = ' + lv_theta_grd_final + os.system(call_str) + else: + print(f"File {lv_theta_grd_final} already exists, skipping calculation.") + + # 判断lv_phi_grd_final是否存在 + if not os.path.exists(lv_phi_grd_final): + call_str = 'gmt grdmath -180 ' + lv_phi_grd + ' 3.1415926 DIV 180 MUL SUB = ' + lv_phi_grd_final + os.system(call_str) + else: + print(f"File {lv_phi_grd_final} already exists, skipping calculation.") + + # 判断UE_grd是否存在 + if not os.path.exists(UE_grd): + call_str = 'gmt grdmath ' + lv_phi_grd_final + ' COSD ' + lv_theta_grd_final + ' SIND MUL NEG = ' + UE_grd + os.system(call_str) + else: + print(f"File {UE_grd} already exists, skipping calculation.") + + # 判断UN_grd是否存在 + if not os.path.exists(UN_grd): + call_str = 'gmt grdmath ' + lv_phi_grd_final + ' SIND ' + lv_theta_grd_final + ' SIND MUL = ' + UN_grd + os.system(call_str) + else: + print(f"File {UN_grd} already exists, skipping calculation.") + + # 判断UU_grd是否存在 + if not os.path.exists(UU_grd): + call_str = 'gmt grdmath ' + lv_theta_grd_final + ' COSD = ' + UU_grd + os.system(call_str) + else: + print(f"File {UU_grd} already exists, skipping calculation.") + + # 判断UE_tif是否存在 + if not os.path.exists(UE_tif): + call_str = 'gdal_translate -of GTiff ' + UE_grd + ' ' + UE_tif + os.system(call_str) + else: + print(f"File {UE_tif} already exists, skipping conversion.") + + # 判断UN_tif是否存在 + if not os.path.exists(UN_tif): + call_str = 'gdal_translate -of GTiff ' + UN_grd + ' ' + UN_tif + os.system(call_str) + else: + print(f"File {UN_tif} already exists, skipping conversion.") + + # 判断UU_tif是否存在 + if not os.path.exists(UU_tif): + call_str = 'gdal_translate -of GTiff ' + UU_grd + ' ' + UU_tif + os.system(call_str) + else: + print(f"File {UU_tif} already exists, skipping conversion.") + +##### get unw.tif + GeoMamp = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.amp' + GeoMamp_tif = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.amp.tif' + GeoCOR = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.diff_filt.cor' + GeoCOR_tif = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.cor.tif' + GeoUNW_tif = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.unw.tif' + GeoUNW = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.unw' + GeoDIFF = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt' + GeoDIFF_tif = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.tif' + Geodiff = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff' + Geodiff_tif = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff.tif' + GeoATMCOR_UNW = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.unw.gacos' + GeoATMCOR_UNW_tif = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.unw.gacos.tif' + Geodem = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.hgt' + Geodem_tif = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.hgt.tif' + + # 判断GeoUNW_tif是否存在 + if not os.path.exists(GeoUNW_tif): + call_str = 'data2geotiff ' + DEM_par + ' ' + GeoUNW + ' ' + ' 2 ' + GeoUNW_tif + os.system(call_str) + else: + print(f"File {GeoUNW_tif} already exists, skipping conversion.") + + # 判断Geodem_tif是否存在 + if not os.path.exists(Geodem_tif): + call_str = 'data2geotiff ' + DEM_par + ' ' + Geodem + ' ' + ' 2 ' + Geodem_tif + os.system(call_str) + else: + print(f"File {Geodem_tif} already exists, skipping conversion.") + + # 判断GeoCOR_tif是否存在 + if not os.path.exists(GeoCOR_tif): + call_str = 'data2geotiff ' + DEM_par + ' ' + GeoCOR + ' ' + ' 2 ' + GeoCOR_tif + os.system(call_str) + else: + print(f"File {GeoCOR_tif} already exists, skipping conversion.") + + # 判断GeoMamp_tif是否存在 + if not os.path.exists(GeoMamp_tif): + call_str = 'data2geotiff ' + DEM_par + ' ' + GeoMamp + ' ' + ' 2 ' + GeoMamp_tif + os.system(call_str) + else: + print(f"File {GeoMamp_tif} already exists, skipping conversion.") + + # 判断GeoDIFF_tif是否存在 + if not os.path.exists(GeoDIFF_tif): + call_str = 'data2geotiff ' + DEM_par + ' ' + GeoDIFF + ' ' + ' 2 ' + GeoDIFF_tif + os.system(call_str) + else: + print(f"File {GeoDIFF_tif} already exists, skipping conversion.") + + # 判断Geodiff_tif是否存在 + if not os.path.exists(Geodiff_tif): + call_str = 'data2geotiff ' + DEM_par + ' ' +Geodiff + ' ' + ' 2 ' + Geodiff_tif + os.system(call_str) + else: + print(f"File {Geodiff_tif} already exists, skipping conversion.") + + # 判断GeoATMCOR_UNW_tif是否存在（GACOS校正后的解缠结果） + if os.path.exists(GeoATMCOR_UNW) and not os.path.exists(GeoATMCOR_UNW_tif): + call_str = 'data2geotiff ' + DEM_par + ' ' + GeoATMCOR_UNW + ' ' + ' 2 ' + GeoATMCOR_UNW_tif + os.system(call_str) + elif os.path.exists(GeoATMCOR_UNW_tif): + print(f"File {GeoATMCOR_UNW_tif} already exists, skipping conversion.") + + GeoCOR_tif0 = Store_dir + '/' + Pair + '.geo.cc.tif' + GeoUNW_tif0 = Store_dir + '/' + Pair + '.geo.unw.tif' + UU_tif0 = GEOC + '/' + 'uu.geo.U.tif' + UE_tif0 = GEOC + '/' + 'ue.geo.E.tif' + UN_tif0 = GEOC + '/' + 'un.geo.N.tif' + dem_tif0 = GEOC + '/' + masterDate + '.geo.hgt.tif' + amp_tif0 = GEOC + '/' + masterDate + '.geo.mli.tif' + GeoDIFF_tif0 = Store_dir + '/' + Pair + '.geo.diff_pha.tif' + Geodiff_tif0 = Store_dir + '/' + Pair + '.geo.diff_unfiltered_pha.tif' + GeoATMCOR_UNW_tif0 = Store_dir + '/' + Pair + '.geo.unw.gacos.tif' + # 判断并复制文件，如果目标文件已存在则不复制 + if not os.path.exists(GeoUNW_tif0): + ut.copy_file(GeoUNW_tif, GeoUNW_tif0) + else: + print(f"File {GeoUNW_tif0} already exists, skipping copy.") + + if not os.path.exists(GeoCOR_tif0): + ut.copy_file(GeoCOR_tif, GeoCOR_tif0) + else: + print(f"File {GeoCOR_tif0} already exists, skipping copy.") + + if not os.path.exists(UU_tif0): + ut.copy_file(UU_tif, UU_tif0) + else: + print(f"File {UU_tif0} already exists, skipping copy.") + + if not os.path.exists(UE_tif0): + ut.copy_file(UE_tif, UE_tif0) + else: + print(f"File {UE_tif0} already exists, skipping copy.") + + if not os.path.exists(UN_tif0): + ut.copy_file(UN_tif, UN_tif0) + else: + print(f"File {UN_tif0} already exists, skipping copy.") + if not os.path.exists(GeoDIFF_tif0): + ut.copy_file(GeoDIFF_tif, GeoDIFF_tif0) + else: + print(f"File {GeoDIFF_tif0} already exists, skipping copy.") + if not os.path.exists(Geodiff_tif0): + ut.copy_file(Geodiff_tif, Geodiff_tif0) + else: + print(f"File {Geodiff_tif0} already exists, skipping copy.") + + if os.path.exists(GeoATMCOR_UNW_tif) and not os.path.exists(GeoATMCOR_UNW_tif0): + ut.copy_file(GeoATMCOR_UNW_tif, GeoATMCOR_UNW_tif0) + elif os.path.exists(GeoATMCOR_UNW_tif0): + print(f"File {GeoATMCOR_UNW_tif0} already exists, skipping copy.") + + if not os.path.exists(dem_tif0): + ut.copy_file(Geodem_tif, dem_tif0) + else: + print(f"File {dem_tif0} already exists, skipping copy.") + + if not os.path.exists(amp_tif0): + ut.copy_file(GeoMamp_tif, amp_tif0) + else: + print(f"File {amp_tif0} already exists, skipping copy.") + + print("Convert to LiCSBAS format is done!") + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/gamma2licsbas_gamma_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/gamma2licsbas_gamma_all.py new file mode 100644 index 0000000..26f8916 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/gamma2licsbas_gamma_all.py @@ -0,0 +1,122 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + +def work(data0): + cmd = data0[0] + err_file = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if aa: + print(aa) + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + '\n' + with open(err_file, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + Geocode interferograms for one project using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + geocode_gamma_all.py projectName + geocode_gamma_all.py projectName --parallel 4 + geocode_gamma_all.py projectName --parallel 4 --ifgramList-txt /test/ifgram_list.txt +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Geocode interferograms for one project using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + parser.add_argument('--ifgarmList-txt', dest='ifgarmListTxt', help='provided ifgram_list_txt. default: using ifgram_list.txt under projectName folder.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + projectDir = scratchDir + '/' + projectName + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + GEOC = scratchDir + '/' + projectName + '/GEOC' + if not os.path.isdir(GEOC): + os.makedirs(GEOC, exist_ok=True) + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + if inps.ifgarmListTxt: ifgramList_txt = inps.ifgarmListTxt + else: ifgramList_txt = scratchDir + '/' + projectName + '/ifgram_list.txt' + ifgList0 = ut.read_txt2array(ifgramList_txt) +# ifgList = ifgList0[:,0] + if len(ifgList0)==3: + ifgList=ifgList0[0] + ifgList=[ifgList] + else: + ifgList=ifgList0[:,0] + + err_txt = scratchDir + '/' + projectName + '/gamma2licsbas_gamma_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + for i in range(len(ifgList)): + #m0 = ut.yyyymmdd(ifgList[i].split('-')[0]) + #s0 = ut.yyyymmdd(ifgList[i].split('-')[1]) + cmd0 = ['gamma2licsbas_gamma.py',projectName, ifgList[i]] + data0 = [cmd0,err_txt] + geo_file0 = GEOC + '/' + ifgList[i] + '/'+ ifgList[i] + '.geo.unw.tif' + + k00 = 0 + if os.path.isfile(geo_file0): + if os.path.getsize(geo_file0) > 0: + k00 = 1 + if k00==0: + data_para.append(data0) + + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + print("Geocode interferograms for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/generate_amp_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/generate_amp_all.py new file mode 100644 index 0000000..17d26bf --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/generate_amp_all.py @@ -0,0 +1,102 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + + +def work(data0): + cmd = data0[0] + err_txt = data0[1] + + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + #print(aa) + with open(err_txt, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + Generate multilooked amp for one project. + +''' + +EXAMPLE = ''' + Usage: + generate_amp_all.py projectName + generate_amp_all.py projectName --parallel 4 +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate multilooked amp for one project.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + slcDir = scratchDir + '/' + projectName + '/SLC' + rslcDir = scratchDir + '/' + projectName + '/RSLC' + if not os.path.isdir(rslcDir): os.mkdir(rslcDir) + + cmd_command = 'generate_multilook_amp.py' + + err_txt = scratchDir + '/' + projectName + '/generate_amp_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + slc_list = [os.path.basename(fname) for fname in sorted(glob.glob(rslcDir + '/*'))] + #slc_list = ut.get_project_slcList(projectName) + for i in range(len(slc_list)): + cmd0 = [cmd_command,projectName,slc_list[i]] + data0 = [cmd0,err_txt] + data_para.append(data0) + + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + print("Generate multilooked apm for all rslcs is done! ") + ut.print_process_time(start_time, time.time()) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/generate_bursts_par.py b/.codex_tmp/pyint_variants/no_rescue/pyint/generate_bursts_par.py new file mode 100644 index 0000000..a5fcdde --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/generate_bursts_par.py @@ -0,0 +1,81 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import glob +import argparse + +from pyint import _utils as ut + + + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Generate burst par files for a project. +''' + +EXAMPLE = """Usage: + + generate_bursts_par.py projectName + +------------------------------------------------------------------- +""" + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Check common busrts for TOPS data.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='Name of project.') + + inps = parser.parse_args() + + return inps + +################################################################################ + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Mdate = templateDict['masterDate'] + + processDir = scratchDir + '/' + projectName + "/PROCESS" + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + dateList = ut.get_project_slcList(projectName) + + for i in range(len(dateList)): + slc_dir = slcDir + '/' + dateList[i] + print(slc_dir) + for j in range(3): + kk = j + SLC = slc_dir + '/' + dateList[i] + '.IW' + str(kk+1)+'.slc' + SLCPar = slc_dir + '/' + dateList[i] + '.IW' + str(kk+1)+'.slc.par' + TOPPar = slc_dir + '/' + dateList[i] + '.IW' + str(kk+1)+'.slc.TOPS_par' + BURST = slc_dir + '/' + dateList[i] + '.IW' + str(kk+1)+'.burst.par' + + call_str = 'SLC_burst_corners ' + SLCPar + ' ' + TOPPar + ' > ' +BURST + os.system(call_str) + + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/generate_multilook_amp.py b/.codex_tmp/pyint_variants/no_rescue/pyint/generate_multilook_amp.py new file mode 100644 index 0000000..ceb392c --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/generate_multilook_amp.py @@ -0,0 +1,87 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import argparse + +from pyint import _utils as ut + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister TOPS S1-SLC to a reference S1-SLC using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='Name of project.') + parser.add_argument('date', help='date of the slave S1 image. [mater date is read from template]') + inps = parser.parse_args() + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Generate multilook amplitude images for coregistered SLCs. + +''' + +EXAMPLE = """Usage: + + generate_multilook_amp.py projectName Date + + generate_multilook_amp.py PacayaT163TsxHhA 20150102 +------------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Date = inps.date + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + if not os.path.isdir(rslcDir): os.mkdir(rslcDir) + #workDir = processDir + '/' + igramDir + workDir = rslcDir + '/' + Date + if not os.path.isdir(workDir): os.mkdir(workDir) + + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Mdate = templateDict['masterDate'] + + + rslc = workDir + '/' + Date + '.rslc' + rslcPar = workDir + '/' + Date + '.rslc.par' + + amp = workDir + '/' + Date + '_' + rlks + 'rlks.amp' + ampPar = workDir + '/' + Date + '_' + rlks + 'rlks.amp.par' + + k0 = 0 + if os.path.isfile(ampPar): + if os.path.getsize(ampPar) > 0: + k0 =1 + + if k0==0: + call_str = 'multi_look ' + rslc + ' ' + rslcPar + ' ' + amp + ' ' + ampPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + + nWIDTH = ut.read_gamma_par(ampPar,'read', 'range_samples') + + call_str = 'raspwr ' + amp + ' ' + nWIDTH + os.system(call_str) + + print("Generate amplitude image for RSLC %s is done !!" % Date) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/generate_rdc_dem.py b/.codex_tmp/pyint_variants/no_rescue/pyint/generate_rdc_dem.py new file mode 100644 index 0000000..12f16bd --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/generate_rdc_dem.py @@ -0,0 +1,204 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import argparse + +from pyint import _utils as ut + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate radar-coordinates based DEM.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='Name of project.') + inps = parser.parse_args() + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Generate radar-coordinates based DEM. + [Geo-coordinates DEM can be downloaded automatically if not provided.] +''' + +EXAMPLE = """Usage: + + generate_rdc_dem.py projectName + +------------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + Mdate = templateDict['masterDate'] + + DEMDir = os.getenv('DEMDIR') + + processDir = scratchDir + '/' + projectName + "/ifgrams" + slcDir = scratchDir + '/' + projectName + "/SLC" + + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + if not os.path.isdir(processDir): + call_str = 'mkdir ' + processDir + os.system(call_str) + + simDir = scratchDir + '/' + projectName + "/DEM" + if not os.path.isdir(simDir): + call_str='mkdir ' + simDir + + + workDir = simDir + + if 'DEM' in templateDict: + dem = templateDict['DEM'] + if not os.path.isfile(dem): + dem = DEMDir + '/' + projectName + '/' + projectName + '.dem' + call_str = 'echo DEM= ' + dem + ' >> ' + templateFile + os.system(call_str) + templateDict['DEM'] = dem + else: + dem = DEMDir + '/' + projectName + '/' + projectName + '.dem' + call_str = 'echo DEM = ' + dem + ' >> ' + templateFile + os.system(call_str) + + demPar = dem + ".par" + + if not os.path.isfile(dem): + call_str = 'makedem_pyint.py ' + projectName + os.system(call_str) + +# Parameter setting for simPhase + latovrSimphase = templateDict['dem_lat_ovr'] + lonovrSimphase = templateDict['dem_lon_ovr'] + + rposSimphase = templateDict['Simphase_rpos'] + azposSimphase = templateDict['Simphase_azpos'] + rwinSimphase = templateDict['Simphase_rwin'] + azwinSimphase = templateDict['Simphase_azwin'] + #rwinSimphase = '128' + #azwinSimphase = '128' + + threshSimphase = templateDict['Simphase_thresh'] + +# Definition of file + MslcDir = slcDir + '/' + Mdate + MslcImg = MslcDir + '/' + Mdate + '.slc' + MslcPar = MslcDir + '/' + Mdate + '.slc.par' + OFFSTD = workDir + '/' + Mdate + '_dem.off_std' + + + BLANK = workDir + '/' + Mdate + '.blk' + MamprlksImg = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp' + MamprlksPar = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + + + UTMDEMpar = simDir + '/'+ Mdate + '_'+ rlks + 'rlks.utm.dem.par' + UTMDEM = simDir + '/' + Mdate + '_'+ rlks + 'rlks.utm.dem' + UTM2RDC = simDir + '/' + Mdate + '_'+ rlks + 'rlks.utm_to_rdc0' + SIMSARUTM = simDir + '/' + Mdate + '_'+ rlks + 'rlks.sim_sar_utm' + PIX = simDir + '/' + Mdate + '_'+ rlks + 'rlks.pix' + LSMAP = simDir + '/' + Mdate + '_'+ rlks + 'rlks.ls_map' + SIMSARRDC = simDir + '/' + Mdate + '_'+ rlks + 'rlks.sim_sar_rdc' + SIMDIFFpar = simDir + '/' + Mdate + '_'+ rlks + 'rlks.diff_par' + SIMOFFS = simDir + '/' + Mdate + '_'+ rlks + 'rlks.offs' + SIMSNR = simDir + '/' + Mdate + '_'+ rlks + 'rlks.snr' + SIMOFFSET = simDir + '/' + Mdate + '_'+ rlks + 'rlks.offset' + SIMCOFF = simDir + '/' + Mdate + '_'+ rlks + 'rlks.coff' + SIMCOFFSETS = simDir + '/' + Mdate + '_'+ rlks + 'rlks.coffsets' + UTMTORDC = simDir + '/' + Mdate + '_'+ rlks + 'rlks.UTM_TO_RDC' + HGTSIM = simDir + '/' + Mdate + '_'+ rlks + 'rlks.rdc.dem' + + if not (os.path.isdir(simDir)): + os.makedirs(simDir) + + ut.createBlankFile(BLANK) + +### remove DEM look up table if it existed for considering gamma overlapping + + if os.path.isfile(UTMDEM): + os.remove(UTMDEM) + if os.path.isfile(UTMDEMpar): + os.remove(UTMDEMpar) + if os.path.isfile(UTM2RDC): + os.remove(UTM2RDC) + + nWidthUTMDEM0 = ut.read_gamma_par(demPar, 'read', 'width') + DateFormat = ut.read_gamma_par(demPar, 'read', 'data_format:') + + if DateFormat == 'INTEGER*2': + DF_type = '4' + else: + DF_type = '2' + + + tmp_dem = dem + '_tmp' + + if not os.path.isfile(tmp_dem): + call_str = 'replace_values ' + dem + ' -32767 0 ' + tmp_dem + ' ' + nWidthUTMDEM0 + ' 2 ' + DF_type + os.system(call_str) + call_str = 'cp ' + tmp_dem + ' ' + dem + os.system(call_str) + + call_str = "multi_look " + MslcImg + " " + MslcPar + " " + MamprlksImg + " " + MamprlksPar + " " + rlks + " " + azlks + os.system(call_str) + + call_str = 'gc_map1 ' + MamprlksPar + ' ' + '-' + ' ' + demPar + ' ' + dem + ' ' + UTMDEMpar + ' ' + UTMDEM + ' ' + UTM2RDC + ' ' + latovrSimphase + ' ' + lonovrSimphase + ' ' + SIMSARUTM + ' - - - - ' + PIX + ' ' + LSMAP + ' - 3 128' + #call_str = 'gc_map2 ' + MamprlksPar + ' ' + demPar + ' ' + dem + ' ' + UTMDEMpar + ' ' + UTMDEM + ' ' + UTM2RDC + ' ' + latovrSimphase + ' ' + lonovrSimphase + ' ' + ' ' + LSMAP + ' - - - - ' + SIMSARUTM + ' - - - ' + PIX + os.system(call_str) + + nWidthUTMDEM = ut.read_gamma_par(UTMDEMpar, 'read', 'width') + nLinePWR1 = ut.read_gamma_par(MamprlksPar, 'read', 'azimuth_lines') + nWidth = ut.read_gamma_par(MamprlksPar, 'read', 'range_samples') + + call_str = 'geocode ' + UTM2RDC + ' ' + SIMSARUTM + ' ' + nWidthUTMDEM + ' ' + SIMSARRDC + ' ' + nWidth + ' ' + nLinePWR1 + ' 0 0' + os.system(call_str) + + call_str = 'create_diff_par ' + MamprlksPar + ' ' + MamprlksPar + ' ' + SIMDIFFpar + ' 1 < ' + BLANK + os.system(call_str) + + call_str = 'init_offsetm ' + SIMSARRDC + ' ' + MamprlksImg + ' ' + SIMDIFFpar + ' 2 2 ' + rposSimphase + ' ' + azposSimphase #+ ' - - - 512' + os.system(call_str) + + call_str = 'offset_pwrm ' + SIMSARRDC + ' ' + MamprlksImg + ' ' + SIMDIFFpar + ' ' + SIMOFFS + ' ' + SIMSNR + ' ' + rwinSimphase + ' ' + azwinSimphase + ' ' + SIMOFFSET #+ ' - 128 128 ' + threshSimphase + os.system(call_str) + + call_str = 'offset_fitm ' + SIMOFFS + ' ' + SIMSNR + ' ' + SIMDIFFpar + ' ' + SIMCOFF + ' ' + SIMCOFFSETS + ' - > ' + OFFSTD + os.system(call_str) + + call_str = 'gc_map_fine ' + UTM2RDC + ' ' + nWidthUTMDEM + ' ' + SIMDIFFpar + ' ' + UTMTORDC + ' 1' + #print(call_str) + os.system(call_str) + + call_str = 'geocode ' + UTMTORDC + ' ' + UTMDEM + ' ' + nWidthUTMDEM + ' ' + HGTSIM + ' ' + nWidth + ' ' + nLinePWR1 + ' 0 0 - - 1 1 1' + os.system(call_str) + + + required_outputs = [UTMDEMpar, UTMDEM, UTMTORDC, HGTSIM] + missing = [path for path in required_outputs if not os.path.isfile(path)] + if missing: + raise RuntimeError('generate_rdc_dem is missing required outputs: ' + ', '.join(missing)) + + print("Create DEM in Radar Coordinates is done!") + + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/geocode_dolphin.py b/.codex_tmp/pyint_variants/no_rescue/pyint/geocode_dolphin.py new file mode 100644 index 0000000..ef5edc8 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/geocode_dolphin.py @@ -0,0 +1,500 @@ +#!/usr/bin/env python +""" +Geocode dolphin unwrapped phase results with gdal_translate multi-looking and optional visualization +Output file will have the same pixel dimensions as the input phase file +""" + +import os +import argparse +import numpy as np +import rasterio +from osgeo import gdal, osr +import tempfile +import shutil +import subprocess +import matplotlib.pyplot as plt +from matplotlib.colors import LinearSegmentedColormap + +def gdal_translate_multilook(input_file, output_file, target_width, target_height, resample_alg='average'): + """ + Apply multi-looking using gdal_translate with -outsize option + """ + + cmd = [ + 'gdal_translate', + '-outsize', str(target_width), str(target_height), + '-r', resample_alg, + '-co', 'COMPRESS=LZW', + input_file, + output_file + ] + + print(f"Running: {' '.join(cmd)}") + result = subprocess.run(cmd, capture_output=True, text=True) + + if result.returncode != 0: + print(f"Error in gdal_translate: {result.stderr}") + raise RuntimeError(f"gdal_translate failed with return code {result.returncode}") + + print(f"Multi-looking completed: {input_file} -> {output_file}") + +def geocode_phase_file_same_size(phase_file, lon_file, lat_file, out_file, + output_srs=4326): + """ + Geocode a phase file using corresponding lat, lon files while maintaining same pixel dimensions + + Parameters: + ----------- + phase_file : str + Input unwrapped phase file from dolphin + lon_file : str + Longitude file (should match phase file dimensions) + lat_file : str + Latitude file (should match phase file dimensions) + out_file : str + Output geocoded file + output_srs : int + Output spatial reference system EPSG code + """ + + # Template for VRT source description + sourcexmltmpl = ''' + {0} + {1} + ''' + + # Create a temporary VRT file + temp_dir = tempfile.mkdtemp() + tempvrtname = os.path.join(temp_dir, 'geocode_temp.vrt') + + try: + # Open input file to get dimensions + with rasterio.open(phase_file) as src: + x_size, y_size = src.width, src.height + phase_transform = src.transform + phase_crs = src.crs + + # Create VRT driver + driver = gdal.GetDriverByName('VRT') + tempds = driver.Create(tempvrtname, x_size, y_size, 0) + + # Add band to VRT + tempds.AddBand(gdal.GDT_Float32) + tempds.GetRasterBand(1).SetMetadata( + {'source_0': sourcexmltmpl.format(phase_file, 1)}, + 'vrt_sources' + ) + + # Set spatial reference (WGS84) + sref = osr.SpatialReference() + sref.ImportFromEPSG(4326) + srswkt = sref.ExportToWkt() + + # Set geolocation metadata + tempds.SetMetadata({ + 'SRS': srswkt, + 'X_DATASET': lon_file, + 'X_BAND': '1', + 'Y_DATASET': lat_file, + 'Y_BAND': '1', + 'PIXEL_OFFSET': '0', + 'LINE_OFFSET': '0', + 'PIXEL_STEP': '1', + 'LINE_STEP': '1' + }, 'GEOLOCATION') + + # Clean up + tempds = None + + # Set output SRS + out_sref = osr.SpatialReference() + out_sref.ImportFromEPSG(output_srs) + + # Calculate output bounds from lon/lat files to maintain same pixel dimensions + with rasterio.open(lon_file) as lon_src: + lon_data = lon_src.read(1) + lon_transform = lon_src.transform + + with rasterio.open(lat_file) as lat_src: + lat_data = lat_src.read(1) + lat_transform = lat_src.transform + + # Calculate approximate bounds + valid_mask = ~(np.isnan(lon_data) | np.isnan(lat_data)) + if np.any(valid_mask): + min_lon = np.min(lon_data[valid_mask]) + max_lon = np.max(lon_data[valid_mask]) + min_lat = np.min(lat_data[valid_mask]) + max_lat = np.max(lat_data[valid_mask]) + + # Calculate approximate resolution to maintain same dimensions + approx_res_x = (max_lon - min_lon) / x_size + approx_res_y = (max_lat - min_lat) / y_size + + # Use the larger resolution to ensure we don't oversample + target_res = max(approx_res_x, approx_res_y) + + # Set warp options with calculated resolution to maintain size + warp_options = gdal.WarpOptions( + format='GTiff', + width=x_size, + height=y_size, + dstSRS=out_sref, + resampleAlg='near', + geoloc=True, + creationOptions=['COMPRESS=LZW', 'TILED=YES'] + ) + else: + # Fallback: use default warp options + warp_options = gdal.WarpOptions( + format='GTiff', + dstSRS=out_sref, + resampleAlg='near', + geoloc=True, + creationOptions=['COMPRESS=LZW', 'TILED=YES'] + ) + + # Perform geocoding + print(f"Geocoding {phase_file} to {out_file}") + print(f"Target output dimensions: {x_size} x {y_size}") + gdal.Warp(out_file, tempvrtname, options=warp_options) + + # Verify output dimensions match input + with rasterio.open(out_file) as out_src: + out_width, out_height = out_src.width, out_src.height + if out_width == x_size and out_height == y_size: + print(f"Success: Output dimensions match input ({x_size} x {y_size})") + else: + print(f"Warning: Output dimensions ({out_width} x {out_height}) differ from input ({x_size} x {y_size})") + + except Exception as e: + print(f"Error during geocoding: {e}") + raise + finally: + # Clean up temporary directory + shutil.rmtree(temp_dir) + +def get_file_dimensions(file_path): + """ + Get dimensions of a raster file + """ + with rasterio.open(file_path) as src: + return src.width, src.height + +def check_existing_multilook_files(phase_file, multilook_dir): + """ + Check if downsampled lon/lat files already exist and match phase file dimensions + """ + if not os.path.exists(multilook_dir): + return None, None + + # Get phase file dimensions + phase_width, phase_height = get_file_dimensions(phase_file) + + # Look for common downsampled file patterns + possible_lon_files = [ + os.path.join(multilook_dir, "lon.multilook.tif"), + os.path.join(multilook_dir, "lon.multilook.vrt"), + os.path.join(multilook_dir, "lon.strided.tif"), + os.path.join(multilook_dir, "lon.strided.vrt"), + os.path.join(multilook_dir, "lon_downsampled.tif"), + os.path.join(multilook_dir, "lon_downsampled.vrt"), + ] + + possible_lat_files = [ + os.path.join(multilook_dir, "lat.multilook.tif"), + os.path.join(multilook_dir, "lat.multilook.vrt"), + os.path.join(multilook_dir, "lat.strided.tif"), + os.path.join(multilook_dir, "lat.strided.vrt"), + os.path.join(multilook_dir, "lat_downsampled.tif"), + os.path.join(multilook_dir, "lat_downsampled.vrt"), + ] + + # Check for existing files that match dimensions + for lon_file in possible_lon_files: + if os.path.exists(lon_file): + try: + lon_width, lon_height = get_file_dimensions(lon_file) + if lon_width == phase_width and lon_height == phase_height: + # Found matching lon file, now check for matching lat file + for lat_file in possible_lat_files: + if os.path.exists(lat_file): + lat_width, lat_height = get_file_dimensions(lat_file) + if lat_width == phase_width and lat_height == phase_height: + print(f"Found existing downsampled files:") + print(f" Longitude: {lon_file}") + print(f" Latitude: {lat_file}") + return lon_file, lat_file + except: + # Skip files that can't be read + continue + + return None, None + +def create_phase_colormap(): + """ + Create a colormap suitable for phase data (cyclic colormap) + """ + # Create a cyclic colormap for phase data + colors = [ + (0, 0, 0.5), # Dark blue + (0, 0, 1), # Blue + (0, 1, 1), # Cyan + (0.5, 1, 0.5), # Light green + (1, 1, 0), # Yellow + (1, 0.5, 0), # Orange + (1, 0, 0), # Red + (0.5, 0, 0.5), # Purple + (0, 0, 0.5) # Dark blue (back to start) + ] + + return LinearSegmentedColormap.from_list('phase_cmap', colors, N=256) + +def create_simple_visualization(phase_file, output_file, viz_dir): + """ + Create a simple visualization of the geocoded output + """ + os.makedirs(viz_dir, exist_ok=True) + + # Create phase colormap + phase_cmap = create_phase_colormap() + + # Read the input and output files + with rasterio.open(phase_file) as src: + phase_data = src.read(1) + phase_width, phase_height = src.width, src.height + + with rasterio.open(output_file) as src: + output_data = src.read(1) + output_width, output_height = src.width, src.height + + # Calculate statistics for display + phase_stats = { + 'min': np.nanmin(phase_data), + 'max': np.nanmax(phase_data), + 'mean': np.nanmean(phase_data) + } + + output_stats = { + 'min': np.nanmin(output_data), + 'max': np.nanmax(output_data), + 'mean': np.nanmean(output_data) + } + + # Create figure with subplots + fig, axes = plt.subplots(1, 2, figsize=(16, 6)) + fig.suptitle('Input vs Output Comparison', fontsize=16, fontweight='bold') + + # Original phase + im1 = axes[0].imshow(phase_data, cmap=phase_cmap, + vmin=phase_stats['min'], vmax=phase_stats['max']) + axes[0].set_title(f'Original Phase\nDimensions: {phase_width} x {phase_height}') + axes[0].set_xlabel('Range') + axes[0].set_ylabel('Azimuth') + plt.colorbar(im1, ax=axes[0], label='Phase (rad)') + + # Geocoded output + im2 = axes[1].imshow(output_data, cmap=phase_cmap, + vmin=output_stats['min'], vmax=output_stats['max']) + axes[1].set_title(f'Geocoded Phase\nDimensions: {output_width} x {output_height}') + axes[1].set_xlabel('Longitude') + axes[1].set_ylabel('Latitude') + plt.colorbar(im2, ax=axes[1], label='Phase (rad)') + + # Add statistics text + stats_text1 = f"Min: {phase_stats['min']:.3f}\nMax: {phase_stats['max']:.3f}\nMean: {phase_stats['mean']:.3f}" + axes[0].text(0.02, 0.98, stats_text1, transform=axes[0].transAxes, + verticalalignment='top', fontsize=10, + bbox=dict(boxstyle='round', facecolor='white', alpha=0.8)) + + stats_text2 = f"Min: {output_stats['min']:.3f}\nMax: {output_stats['max']:.3f}\nMean: {output_stats['mean']:.3f}" + axes[1].text(0.02, 0.98, stats_text2, transform=axes[1].transAxes, + verticalalignment='top', fontsize=10, + bbox=dict(boxstyle='round', facecolor='white', alpha=0.8)) + + plt.tight_layout() + plt.savefig(os.path.join(viz_dir, 'input_output_comparison.png'), dpi=300, bbox_inches='tight') + plt.close(fig) + + print(f"Visualization saved to: {os.path.join(viz_dir, 'input_output_comparison.png')}") + +def main(): + parser = argparse.ArgumentParser( + description='Geocode dolphin unwrapped phase results with gdal_translate multi-looking and optional visualization. Output will have same pixel dimensions as input.' + ) + parser.add_argument('-i', '--input', required=True, + help='Input dolphin unwrapped phase file (GeoTIFF)') + parser.add_argument('--lon', required=True, + help='Input longitude file (e.g., lon.rdr.full.vrt)') + parser.add_argument('--lat', required=True, + help='Input latitude file (e.g., lat.rdr.full.vrt)') + parser.add_argument('-o', '--output', default='unwrapped_phase_geocoded.tif', + help='Output geocoded file path') + parser.add_argument('--multilook-dir', default='multilook_geoms', + help='Directory for multi-looked geometry files') + parser.add_argument('--viz-dir', default='visualizations', + help='Directory for visualization images') + parser.add_argument('--output-srs', type=int, default=4326, + help='Output SRS EPSG code (default: 4326 for WGS84)') + parser.add_argument('--resample-alg', default='average', + choices=['near', 'average', 'bilinear', 'cubic', 'cubicspline', 'lanczos', 'mode'], + help='Resampling algorithm for multi-looking (default: average)') + parser.add_argument('--no-cleanup', action='store_true', + help='Keep temporary multi-looked files') + parser.add_argument('--force-multilook', action='store_true', + help='Force multi-looking even if downsampled files exist') + + # Visualization options + viz_group = parser.add_mutually_exclusive_group() + viz_group.add_argument('--no-viz', action='store_true', + help='Skip visualization') + viz_group.add_argument('--viz', action='store_true', + help='Create visualization (default)') + + args = parser.parse_args() + + # Default to visualization if no option is specified + if not (args.no_viz or args.viz): + args.viz = True + + # Check if input files exist + for f in [args.input, args.lon, args.lat]: + if not os.path.exists(f): + print(f"Error: Input file does not exist: {f}") + return 1 + + # Get dimensions from phase file + print("Getting input file dimensions...") + phase_width, phase_height = get_file_dimensions(args.input) + lon_width, lon_height = get_file_dimensions(args.lon) + lat_width, lat_height = get_file_dimensions(args.lat) + + print(f"Phase file dimensions: {phase_width} x {phase_height}") + print(f"Longitude file dimensions: {lon_width} x {lon_height}") + print(f"Latitude file dimensions: {lat_width} x {lat_height}") + + # Check if multi-looking is needed or if files already exist + if (phase_width, phase_height) == (lon_width, lon_height) == (lat_width, lat_height): + print("Files already have matching dimensions. Skipping multi-looking.") + lon_multilook = args.lon + lat_multilook = args.lat + elif not args.force_multilook: + # Check if downsampled files already exist + existing_lon, existing_lat = check_existing_multilook_files(args.input, args.multilook_dir) + if existing_lon and existing_lat: + print("Using existing downsampled files.") + lon_multilook = existing_lon + lat_multilook = existing_lat + else: + # Need to create downsampled files + print("Downsampled files not found or don't match. Creating new ones...") + # Step 1: Apply multi-looking to lon/lat files using gdal_translate + print("Step 1: Applying multi-looking to lon/lat files using gdal_translate...") + + # Create output directory + os.makedirs(args.multilook_dir, exist_ok=True) + + # Generate output file paths + lon_basename = os.path.splitext(os.path.basename(args.lon))[0] + lat_basename = os.path.splitext(os.path.basename(args.lat))[0] + + lon_multilook = os.path.join(args.multilook_dir, f"{lon_basename}_multilook.tif") + lat_multilook = os.path.join(args.multilook_dir, f"{lat_basename}_multilook.tif") + + # Apply multi-looking using gdal_translate + try: + gdal_translate_multilook( + args.lon, lon_multilook, phase_width, phase_height, args.resample_alg + ) + gdal_translate_multilook( + args.lat, lat_multilook, phase_width, phase_height, args.resample_alg + ) + except Exception as e: + print(f"Multi-looking failed: {e}") + return 1 + else: + # Force multi-looking even if files exist + print("Forcing multi-looking...") + # Step 1: Apply multi-looking to lon/lat files using gdal_translate + print("Step 1: Applying multi-looking to lon/lat files using gdal_translate...") + + # Create output directory + os.makedirs(args.multilook_dir, exist_ok=True) + + # Generate output file paths + lon_basename = os.path.splitext(os.path.basename(args.lon))[0] + lat_basename = os.path.splitext(os.path.basename(args.lat))[0] + + lon_multilook = os.path.join(args.multilook_dir, f"{lon_basename}_multilook.tif") + lat_multilook = os.path.join(args.multilook_dir, f"{lat_basename}_multilook.tif") + + # Apply multi-looking using gdal_translate + try: + gdal_translate_multilook( + args.lon, lon_multilook, phase_width, phase_height, args.resample_alg + ) + gdal_translate_multilook( + args.lat, lat_multilook, phase_width, phase_height, args.resample_alg + ) + except Exception as e: + print(f"Multi-looking failed: {e}") + return 1 + + # Step 2: Geocode the phase file using (multi-looked) lon/lat + print("Step 2: Geocoding phase file (maintaining same dimensions)...") + + # Create output directory if needed + output_dir = os.path.dirname(args.output) + if output_dir and not os.path.exists(output_dir): + os.makedirs(output_dir, exist_ok=True) + + try: + geocode_phase_file_same_size( + phase_file=args.input, + lon_file=lon_multilook, + lat_file=lat_multilook, + out_file=args.output, + output_srs=args.output_srs + ) + + print(f"\nGeocoding completed successfully!") + print(f"Geocoded phase file saved to: {args.output}") + + # Display some info about the output file + with rasterio.open(args.output) as src: + print(f"Output file info:") + print(f" Dimensions: {src.width} x {src.height}") + print(f" CRS: {src.crs}") + if src.res: + print(f" Resolution: {src.res[0]:.6f} x {src.res[1]:.6f} degrees") + + except Exception as e: + print(f"Geocoding failed: {e}") + return 1 + + # Step 3: Create visualization if requested + if args.viz: + print("Step 3: Creating visualization...") + try: + create_simple_visualization(args.input, args.output, args.viz_dir) + except Exception as e: + print(f"Visualization failed: {e}") + else: + print("Skipping visualization as requested.") + + # Clean up temporary files if requested + if not args.no_cleanup and (lon_multilook != args.lon or lat_multilook != args.lat): + # Only clean up if we created new files and they're not the original ones + if (lon_multilook != args.lon and lat_multilook != args.lat and + os.path.exists(args.multilook_dir) and + (lon_multilook.startswith(args.multilook_dir) or + lat_multilook.startswith(args.multilook_dir))): + print("Cleaning up temporary files...") + shutil.rmtree(args.multilook_dir) + else: + print(f"Multi-looked files kept in: {args.multilook_dir}") + + return 0 + +if __name__ == "__main__": + exit(main()) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/geocode_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/geocode_gamma.py new file mode 100644 index 0000000..1ea3cc7 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/geocode_gamma.py @@ -0,0 +1,371 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import os +import sys +import argparse +import numpy as np + +from pyint import _utils as ut + + +def sanitize_gamma_float(filepath, valid_max=1e6): + """清理 GAMMA 浮点数据文件中的无效值 (NaN/Inf/极端值 → 0.0)""" + data = np.fromfile(filepath, dtype=np.float32) + bad_mask = ~np.isfinite(data) | (np.abs(data) > valid_max) + n_bad = int(np.sum(bad_mask)) + if n_bad > 0: + data[bad_mask] = 0.0 + data.tofile(filepath) + print(f' [sanitize] {os.path.basename(filepath)}: ' + f'清理 {n_bad} 个无效像素') + +def geocode(inFile, outFile, UTMTORDC, nWidth, nWidthUTMDEM, nLineUTMDEM, geo_interp='0'): + + if '.unw' in os.path.basename(inFile): + call_str = 'geocode_back ' + inFile + ' ' + nWidth + ' ' + UTMTORDC + ' ' + outFile + ' ' + nWidthUTMDEM + ' ' + nLineUTMDEM + ' ' + geo_interp + ' 0' + elif '.amp' in os.path.basename(inFile): + call_str = 'geocode_back ' + inFile + ' ' + nWidth + ' ' + UTMTORDC + ' ' + outFile + ' ' + nWidthUTMDEM + ' ' + nLineUTMDEM + ' ' + geo_interp + ' 0' + elif '.cor' in os.path.basename(inFile): + call_str = 'geocode_back ' + inFile + ' ' + nWidth + ' ' + UTMTORDC + ' ' + outFile + ' ' + nWidthUTMDEM + ' ' + nLineUTMDEM + ' ' + geo_interp + ' 0' + elif '.dem' in os.path.basename(inFile): + call_str = 'geocode_back ' + inFile + ' ' + nWidth + ' ' + UTMTORDC + ' ' + outFile + ' ' + nWidthUTMDEM + ' ' + nLineUTMDEM + ' ' + geo_interp + ' 0' + else: + call_str = 'geocode_back ' + inFile + ' ' + nWidth + ' ' + UTMTORDC + ' ' + outFile + ' ' + nWidthUTMDEM + ' ' + nLineUTMDEM+ ' ' + geo_interp + ' 1' + + os.system(call_str) + + return + +INTRODUCTION = ''' +------------------------------------------------------------------- + Convert radar-coordinates products into geo-coordinates using GAMMA. + + 由模板参数 geocode_products 控制产品类型 (逗号分隔多选): + hyp3 : 基础产品 + dispmap(LOS/vert) + wrapped_phase + look_vector + licsbas : 基础产品 + look_vector + pot : Pixel Offset Tracking 位移图 (两步地理编码) + + 基础产品 (hyp3/licsbas 均需): amp, corr, unw, diff_filt, diff, dem + 一次调用自动处理所有选中的产品类型。 +''' + +EXAMPLE = ''' + Usage: + geocode_gamma.py projectName Mdate-Sdate + geocode_gamma.py PacayaT163TsxHhA 20150102-20150601 + + Template parameter: + geocode_products = hyp3,licsbas (default) + geocode_products = pot + geocode_products = hyp3,licsbas,pot +------------------------------------------------------------------- +''' + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Geocode radar-coordinate products using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('pair',help='Master-Slave, e.g., 20150101-20150106.') + + inps = parser.parse_args() + return inps + + +def geocode_pot(projectName, Pair, templateDict): + """POT 位移图地理编码: 偏移量几何 → MLI 几何 → EQA 几何""" + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + potDir = projectDir + '/offsets' + workDir = potDir + '/' + Pair + demDir = projectDir + '/DEM' + rslcDir = projectDir + '/RSLC' + slcDir = projectDir + '/SLC' + + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + pot_disp_max = float(templateDict.get('pot_disp_max', '10')) + + Mdate = Pair.split('-')[0] + + if not os.path.isdir(workDir): + print(f'ERROR: POT 目录不存在: {workDir}') + return + + # ===== 公共文件 ===== + MampPar = rslcDir + '/' + masterDate + '/' + masterDate + '_' + rlks + 'rlks.amp.par' + Mamp = rslcDir + '/' + masterDate + '/' + masterDate + '_' + rlks + 'rlks.amp' + UTMTORDC = demDir + '/' + masterDate + '_' + rlks + 'rlks.UTM_TO_RDC' + UTMDEMpar = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem.par' + UTMDEM = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem' + SLCpar = slcDir + '/' + Mdate + '/' + Mdate + '.slc.par' + + if not os.path.isfile(UTMTORDC) or not os.path.isfile(UTMDEMpar): + print(' WARNING: POT 地理编码跳过 — 缺少 UTM_TO_RDC 或 utm.dem.par') + return + if not os.path.isfile(MampPar): + print(' WARNING: POT 地理编码跳过 — 缺少 master amp.par') + return + + mli_width = ut.read_gamma_par(MampPar, 'read', 'range_samples') + mli_nlines = ut.read_gamma_par(MampPar, 'read', 'azimuth_lines') + eqa_width = ut.read_gamma_par(UTMDEMpar, 'read', 'width') + eqa_nlines = ut.read_gamma_par(UTMDEMpar, 'read', 'nlines') + + # ===== POT 特有文件 ===== + MLI_pot_par = workDir + '/' + Mdate + '.mli_pot.par' + MLI_pot = workDir + '/' + Mdate + '.mli_pot' + disp_map = workDir + '/' + Pair + '.disp_map' + + if not os.path.isfile(disp_map): + print(f' WARNING: disp_map 不存在: {disp_map}') + return + + # 偏移量几何宽度 + off_width = ut.read_gamma_par(MLI_pot_par, 'read', 'range_samples') + disp_max_m = pot_disp_max * 10 + + # ===== Step A: 偏移量几何 → MLI 几何 (rdc_trans) ===== + print(' [POT] Step A: 偏移量几何 → MLI 几何') + mli_to_pot_lt = workDir + '/mli_to_pot.lt' + disp_map_mli = workDir + '/' + Pair + '.disp_map_mli' + + if not os.path.isfile(mli_to_pot_lt): + os.system(f'rdc_trans {MampPar} 0.1 {MLI_pot_par} {mli_to_pot_lt}') + + if not os.path.isfile(disp_map_mli): + os.system(f'geocode_back {disp_map} {off_width} {mli_to_pot_lt} ' + f'{disp_map_mli} {mli_width} {mli_nlines} 1 1') + + # ===== Step B: MLI 几何 → EQA 几何 ===== + print(' [POT] Step B: MLI 几何 → EQA 几何') + geo_disp = workDir + '/geo_' + Pair + '.disp_map' + + if not os.path.isfile(geo_disp): + os.system(f'geocode_back {disp_map_mli} {mli_width} {UTMTORDC} ' + f'{geo_disp} {eqa_width} {eqa_nlines} 1 1') + + # ===== Step C: 提取地理编码位移分量 ===== + print(' [POT] Step C: 提取位移分量 (real/imag/mag)') + geo_real = geo_disp + '.real' + geo_imag = geo_disp + '.imag' + geo_mag = geo_disp + '.mag' + + if not os.path.isfile(geo_real): + os.system(f'cpx_to_real {geo_disp} {geo_real} {eqa_width} 0') + sanitize_gamma_float(geo_real, disp_max_m) + if not os.path.isfile(geo_imag): + os.system(f'cpx_to_real {geo_disp} {geo_imag} {eqa_width} 1') + sanitize_gamma_float(geo_imag, disp_max_m) + if not os.path.isfile(geo_mag): + os.system(f'cpx_to_real {geo_disp} {geo_mag} {eqa_width} 3') + sanitize_gamma_float(geo_mag, disp_max_m) + + # ===== Step D: 地理编码 MLI 背景 ===== + geo_mli = workDir + '/geo_' + masterDate + '.mli' + if not os.path.isfile(geo_mli): + os.system(f'geocode_back {Mamp} {mli_width} {UTMTORDC} ' + f'{geo_mli} {eqa_width} {eqa_nlines} 5 0') + + # ===== Step E: 地理编码 CCP (互相关系数) ===== + # 自动查找最终 ccp 文件 + ccp_files = sorted([f for f in os.listdir(workDir) if f.startswith(Pair + '.ccp')]) + if ccp_files: + ccp_final = workDir + '/' + ccp_files[-1] + geo_ccp = workDir + '/geo_' + Pair + '.ccp' + if not os.path.isfile(geo_ccp): + # ccp 在偏移量几何, 需要两步 + ccp_mli = workDir + '/' + Pair + '.ccp_mli' + os.system(f'geocode_back {ccp_final} {off_width} {mli_to_pot_lt} ' + f'{ccp_mli} {mli_width} {mli_nlines} 1 0') + os.system(f'geocode_back {ccp_mli} {mli_width} {UTMTORDC} ' + f'{geo_ccp} {eqa_width} {eqa_nlines} 1 0') + + # ===== Step F: 视角矢量 (look_vector) ===== + print(' [POT] Step F: 视角矢量') + lv_theta = workDir + '/lv_theta' + lv_phi = workDir + '/lv_phi' + OFFpar = workDir + '/' + Pair + '.off' + + if not os.path.isfile(lv_theta) or not os.path.isfile(lv_phi): + if os.path.isfile(SLCpar) and os.path.isfile(OFFpar): + os.system(f'look_vector {SLCpar} {OFFpar} {UTMDEMpar} {UTMDEM} {lv_theta} {lv_phi}') + + # ===== BMP 可视化 ===== + print(' [POT] 生成 BMP') + disp_max_str = str(pot_disp_max) + os.system(f'rasdt_pwr {geo_mag} {geo_mli} {eqa_width} - - - - ' + f'-{disp_max_str} {disp_max_str} 1 rmg.cm {geo_mag}.bmp - - 24') + os.system(f'rasdt_pwr {geo_real} {geo_mli} {eqa_width} - - - - ' + f'-{disp_max_str} {disp_max_str} 1 rmg.cm {geo_real}.bmp - - 24') + os.system(f'rasdt_pwr {geo_imag} {geo_mli} {eqa_width} - - - - ' + f'-{disp_max_str} {disp_max_str} 1 rmg.cm {geo_imag}.bmp - - 24') + + print(' [POT] 地理编码完成!') + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Pair = inps.pair + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + '/' + projectName + '.template' + templateDict = ut.update_template(templateFile) + + # ===== 解析产品类型 ===== + products_str = templateDict.get('geocode_products', 'hyp3,licsbas') + products = set(p.strip().lower() for p in products_str.split(',')) + need_ifg = ('hyp3' in products) or ('licsbas' in products) + need_hyp3 = 'hyp3' in products + need_licsbas = 'licsbas' in products + need_pot = 'pot' in products + + print(f'[geocode_gamma] {projectName} / {Pair}') + print(f' geocode_products: {products_str} → ifg={need_ifg}, hyp3={need_hyp3}, licsbas={need_licsbas}, pot={need_pot}') + + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + potDir = scratchDir + '/' + projectName + '/offsets' + demDir = scratchDir + '/' + projectName + '/DEM' + slcDir = scratchDir + '/' + projectName + '/SLC' + rslcDir = scratchDir + '/' + projectName + '/RSLC' + + # ========================================================= + # IFG 地理编码 (hyp3 和/或 licsbas 需要) + # ========================================================= + ifgWorkDir = ifgDir + '/' + Pair + if need_ifg and os.path.isdir(ifgWorkDir): + print('\n--- IFG 地理编码 ---') + workDir = ifgWorkDir + + ######### copy common file for parallel processing ############# + Mamp0 = rslcDir + '/' + masterDate + '/' + masterDate + '_' + rlks + 'rlks.amp' + MampPar0 = rslcDir + '/' + masterDate + '/' + masterDate + '_' + rlks + 'rlks.amp.par' + Mamp = workDir + '/' + masterDate + '_' + rlks + 'rlks.amp' + MampPar = workDir + '/' + masterDate + '_' + rlks + 'rlks.amp.par' + ut.copy_file(Mamp0, Mamp) + ut.copy_file(MampPar0, MampPar) + + ################################################################# + UNWIFG = workDir + '/' + Pair + '_' + rlks + 'rlks.diff_filt.unw' + ATMCOR_UNW = workDir + '/' + Pair + '_' + rlks + 'rlks.diff_filt.atmcor.unw' + DIFFIFG = workDir + '/' + Pair + '_' + rlks + 'rlks.diff_filt' + diffifg = workDir + '/' + Pair + '_' + rlks + 'rlks.diff' + CORIFG = workDir + '/' + Pair + '_' + rlks + 'rlks.diff_filt.cor' + rdcdem = demDir + '/' + masterDate + '_' + rlks + 'rlks.rdc.dem' + + GeoMamp = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.amp' + GeoCOR = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.diff_filt.cor' + GeoUNW = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.unw' + GeoDIFF = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt' + geodiff = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff' + GeoATMCOR_UNW = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.atmcor.unw' + Geodem = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.hgt' + + UTMTORDC0 = demDir + '/' + masterDate + '_' + rlks + 'rlks.UTM_TO_RDC' + UTMDEMpar0 = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem.par' + UTMTORDC = workDir + '/' + masterDate + '_' + rlks + 'rlks.UTM_TO_RDC' + UTMDEMpar = workDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem.par' + ut.copy_file(UTMTORDC0, UTMTORDC) + ut.copy_file(UTMDEMpar0, UTMDEMpar) + + nWidth = ut.read_gamma_par(MampPar, 'read', 'range_samples') + nWidthUTMDEM = ut.read_gamma_par(UTMDEMpar, 'read', 'width') + nLineUTMDEM = ut.read_gamma_par(UTMDEMpar, 'read', 'nlines') + + # --- 基础产品地理编码 (hyp3/licsbas 均需) --- + geo_interp = templateDict['geo_interp'] + geocode(Mamp, GeoMamp, UTMTORDC, nWidth, nWidthUTMDEM, nLineUTMDEM, geo_interp=geo_interp) + geocode(CORIFG, GeoCOR, UTMTORDC, nWidth, nWidthUTMDEM, nLineUTMDEM, geo_interp=geo_interp) + geocode(DIFFIFG, GeoDIFF, UTMTORDC, nWidth, nWidthUTMDEM, nLineUTMDEM, geo_interp=geo_interp) + geocode(UNWIFG, GeoUNW, UTMTORDC, nWidth, nWidthUTMDEM, nLineUTMDEM, geo_interp=geo_interp) + geocode(diffifg, geodiff, UTMTORDC, nWidth, nWidthUTMDEM, nLineUTMDEM, geo_interp=geo_interp) + geocode(rdcdem, Geodem, UTMTORDC, nWidth, nWidthUTMDEM, nLineUTMDEM, geo_interp=geo_interp) + + Mdate = Pair.split('-')[0] + SLCpar = slcDir + '/' + Mdate + '/' + Mdate + '.slc.par' + OFFpar = workDir + '/' + Pair + '_' + rlks + 'rlks.off' + UTMDEM = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem' + + # --- hyp3 专属: dispmap (LOS/vert 位移场) --- + if need_hyp3: + print(' [hyp3] dispmap + geocode 位移场') + los_disp_rdc = workDir + '/' + Pair + '_' + rlks + 'rlks.los_disp' + vert_disp_rdc = workDir + '/' + Pair + '_' + rlks + 'rlks.vert_disp' + + if os.path.isfile(UNWIFG) and os.path.isfile(SLCpar) and os.path.isfile(OFFpar): + hgt_arg = rdcdem if os.path.isfile(rdcdem) else '-' + if not os.path.isfile(los_disp_rdc): + os.system(f'dispmap {UNWIFG} {hgt_arg} {SLCpar} {OFFpar} {los_disp_rdc} 0') + if not os.path.isfile(vert_disp_rdc): + os.system(f'dispmap {UNWIFG} {hgt_arg} {SLCpar} {OFFpar} {vert_disp_rdc} 1') + + geo_los = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.los_disp' + geo_vert = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.vert_disp' + if os.path.isfile(los_disp_rdc) and not os.path.isfile(geo_los): + os.system(f'geocode_back {los_disp_rdc} {nWidth} {UTMTORDC} {geo_los} {nWidthUTMDEM} {nLineUTMDEM} 1 0') + if os.path.isfile(vert_disp_rdc) and not os.path.isfile(geo_vert): + os.system(f'geocode_back {vert_disp_rdc} {nWidth} {UTMTORDC} {geo_vert} {nWidthUTMDEM} {nLineUTMDEM} 1 0') + + # --- hyp3 专属: 缠绕相位 --- + if need_hyp3: + print(' [hyp3] 提取缠绕相位') + geo_wrapped_pha = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.pha' + if os.path.isfile(GeoDIFF) and not os.path.isfile(geo_wrapped_pha): + os.system(f'cpx_to_real {GeoDIFF} {geo_wrapped_pha} {nWidthUTMDEM} 4') + + # --- hyp3/licsbas 共需: look_vector --- + if need_hyp3 or need_licsbas: + print(' [hyp3/licsbas] look_vector') + lv_theta = workDir + '/lv_theta' + lv_phi = workDir + '/lv_phi' + if not os.path.isfile(lv_theta) or not os.path.isfile(lv_phi): + if os.path.isfile(SLCpar) and os.path.isfile(OFFpar) and os.path.isfile(UTMDEMpar0) and os.path.isfile(UTMDEM): + os.system(f'look_vector {SLCpar} {OFFpar} {UTMDEMpar0} {UTMDEM} {lv_theta} {lv_phi}') + + # --- BMP 可视化 --- + os.system('rasmph_pwr ' + GeoDIFF + ' ' + GeoMamp + ' ' + nWidthUTMDEM + ' - - - - ') + os.system('raspwr ' + GeoMamp + ' ' + nWidthUTMDEM + ' - - - - - - - - - - ') + os.system('rasdt_pwr ' + GeoUNW + ' ' + GeoMamp + ' ' + nWidthUTMDEM + ' - - - - -3.14 3.14 1 rmg.cm') + + if os.path.isfile(GeoATMCOR_UNW): + os.system('rasdt_pwr ' + GeoATMCOR_UNW + ' ' + GeoMamp + ' ' + nWidthUTMDEM + ' - - - - -3.14 3.14 1 rmg.cm') + + if os.path.isfile(UTMTORDC): + os.remove(UTMTORDC) + if os.path.isfile(UTMDEMpar): + os.remove(UTMDEMpar) + + print(' IFG 地理编码完成!') + + elif need_ifg: + print(f' WARNING: ifgrams/{Pair} 不存在, 跳过 IFG 地理编码') + + # ========================================================= + # POT 地理编码 (pot 需要) + # ========================================================= + potWorkDir = potDir + '/' + Pair + if need_pot and os.path.isdir(potWorkDir): + print('\n--- POT 地理编码 ---') + geocode_pot(projectName, Pair, templateDict) + elif need_pot: + print(f' WARNING: offsets/{Pair} 不存在, 跳过 POT 地理编码') + + print("\nGeocoding is done!") + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/geocode_gamma_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/geocode_gamma_all.py new file mode 100644 index 0000000..696a7d9 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/geocode_gamma_all.py @@ -0,0 +1,169 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + +def work(data0): + cmd = data0[0] + err_file = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if aa: + print(aa) + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + '\n' + with open(err_file, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + Geocode products for one project using GAMMA. + 由模板参数 geocode_products 控制产品类型 (hyp3,licsbas,pot). + 一次调用自动处理所有选中的产品类型. +''' + +EXAMPLE = ''' + Usage: + geocode_gamma_all.py projectName + geocode_gamma_all.py projectName --parallel 4 + geocode_gamma_all.py projectName --parallel 4 --ifgramList-txt /test/ifgram_list.txt +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Geocode interferograms for one project using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + parser.add_argument('--ifgarmList-txt', dest='ifgarmListTxt', help='provided ifgram_list_txt. default: using ifgram_list.txt under projectName folder.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + projectDir = scratchDir + '/' + projectName + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + potDir = scratchDir + '/' + projectName + '/offsets' + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + # ===== 解析产品类型 ===== + products_str = templateDict.get('geocode_products', 'hyp3,licsbas') + products = set(p.strip().lower() for p in products_str.split(',')) + need_ifg = ('hyp3' in products) or ('licsbas' in products) + need_hyp3 = 'hyp3' in products + need_pot = 'pot' in products + + print(f'geocode_products: {products_str}') + + if inps.ifgarmListTxt: ifgramList_txt = inps.ifgarmListTxt + else: ifgramList_txt = scratchDir + '/' + projectName + '/ifgram_list.txt' + ifgList0 = ut.read_txt2array(ifgramList_txt) +# ifgList = ifgList0[:,0] + if len(ifgList0)==3: + ifgList=ifgList0[0] + ifgList=[ifgList] + else: + ifgList=ifgList0[:,0] + + err_txt = scratchDir + '/' + projectName + '/geocode_gamma_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + skip_count = 0 + for i in range(len(ifgList)): + pair_i = ifgList[i] + cmd0 = ['geocode_gamma.py', projectName, pair_i] + data0 = [cmd0, err_txt] + + # 根据 geocode_products 组合检查是否已完成 + ifg_done = True + pot_done = True + + if need_ifg: + ifg_dir_i = ifgDir + '/' + pair_i + if os.path.isdir(ifg_dir_i): + # 基础产品检查: unw.bmp + unw_bmp = ifg_dir_i + '/geo_' + pair_i + '_' + rlks + 'rlks.diff_filt.unw.bmp' + if not (os.path.isfile(unw_bmp) and os.path.getsize(unw_bmp) > 0): + ifg_done = False + # hyp3 额外检查: los_disp + if need_hyp3 and ifg_done: + disp_f = ifg_dir_i + '/geo_' + pair_i + '_' + rlks + 'rlks.los_disp' + lv_f = ifg_dir_i + '/lv_theta' + if not (os.path.isfile(disp_f) and os.path.getsize(disp_f) > 0): + ifg_done = False + if not (os.path.isfile(lv_f) and os.path.getsize(lv_f) > 0): + ifg_done = False + else: + ifg_done = True # 目录不存在时不需要处理, geocode_gamma.py 会打 WARNING + + if need_pot: + pot_dir_i = potDir + '/' + pair_i + if os.path.isdir(pot_dir_i): + geo_mag_bmp = pot_dir_i + '/geo_' + pair_i + '.disp_map.mag.bmp' + lv_f = pot_dir_i + '/lv_theta' + if not (os.path.isfile(geo_mag_bmp) and os.path.getsize(geo_mag_bmp) > 0): + pot_done = False + if not (os.path.isfile(lv_f) and os.path.getsize(lv_f) > 0): + pot_done = False + else: + pot_done = True # 目录不存在时不需要处理 + + if ifg_done and pot_done: + skip_count += 1 + else: + data_para.append(data0) + + total = len(ifgList) + todo = len(data_para) + print(f'Geocode: {total} pairs total, {skip_count} done, {todo} to process') + print(f'Parallel processors: {inps.parallelNumb}') + print('=' * 60) + + if todo > 0: + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + + print("Geocode products for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/geotiff2grd.sh b/.codex_tmp/pyint_variants/no_rescue/pyint/geotiff2grd.sh new file mode 100644 index 0000000..a67fcef --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/geotiff2grd.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +if [#agrv < 3]; then + + echo" + Usage: geotiff2grd.sh a.tif b.grd + a.tif is a geotif file include nan data + b.grd is a grd file translate by gdal without nan data in it + + Example: geotiff2grd.sh 20240114-20240126_los.tif 20240114-20240126_los.grd + + + Chen Wei 2024/2/17 +" + +exit + +fi + +export input=$1 +export out_grdfile=$2 +gdal_calc.py -A $input --outfile=tmp.tif --calc='(A==0)*(-9999)+(A!=0)*A' +gdal_translate -a_nodata -9999 -of GSBG tmp.tif $out_grdfile +rm tmp.tif *.xml diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/geotiff_utm2geo.py b/.codex_tmp/pyint_variants/no_rescue/pyint/geotiff_utm2geo.py new file mode 100644 index 0000000..d575f6b --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/geotiff_utm2geo.py @@ -0,0 +1,76 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import argparse +import os +import warnings + +from osgeo import gdal, osr + +from mintpy.utils import plot as pp, readfile, utils0 as ut + +# link: https://gdal.org/drivers/raster/index.html +GDAL_DRIVER2EXT = { + 'GTiff' : '.tif', + 'ENVI' : '', + 'GMT' : '.grd', + 'GRIB' : '.grb', + 'JPEG' : '.jpg', + 'PNG' : '.png', +} + + +############################################################################## +def cmdLineParse(): + parser = argparse.ArgumentParser(description='convert the UTM geotiff to geography geotiff.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('origin', help='Name of origin hdf5 file .') + parser.add_argument('target', help='Name of target geotiff.') + #parser.add_argument('subdataset', help='the master date and slave date of the dataset') + inps = parser.parse_args() + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + create_files for psokinv software running file by generated by Gamma +''' + +EXAMPLE = """Usage: + + geotiff_utm2geo.py A.h5 result.tif 20210101_20240302 +------------------------------------------------------------------- +""" + + +def main(argv): + + inps = cmdLineParse() + origin_tif = inps.origin + target_tif = inps.target + #dateset = inps.subdataset + ftype = readfile.read_attribute(origin_tif)['FILE_TYPE'] + if ftype == 'timeseries' or ftype == 'ifgramStack': + call_str = 'save_gdal.py ' + origin_tif + ' -d ' + target_tif +' --of GTiff -o tmp.tif ' + os.system(call_str) + else: + call_str = 'save_gdal.py ' + origin_tif +' --of GTiff -o tmp.tif ' + os.system(call_str) + call_str = ' gdalwarp tmp.tif ' + target_tif + '.tif' + ' -t_srs "EPSG:4326" ' + os.system(call_str) + call_str = 'gdal_translate -of GSBG ' + target_tif + '.tif ' + target_tif + '.grd' + os.system(call_str) + + + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/get_master_burst_numb.py b/.codex_tmp/pyint_variants/no_rescue/pyint/get_master_burst_numb.py new file mode 100644 index 0000000..708f802 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/get_master_burst_numb.py @@ -0,0 +1,217 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse + +from pyint import _utils as ut + + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Get burst number table of the master date for TOPS SLC.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='project name. e.g., ChangningT55S1A') + + inps = parser.parse_args() + + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Get burst number table of the master date for TOPS SLC. +''' + +EXAMPLE = """Usage: + + get_master_burst_numb.py projectName + + get_master_burst_numb.py ChangningT55S1A + +------------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + masterDate = templateDict['masterDate'] + date = masterDate + + slc_dir = projectDir + '/SLC' + down_dir = projectDir + '/DOWNLOAD' + opod_dir = projectDir + '/OPOD' + + if not os.path.isdir(slc_dir): + os.mkdir(slc_dir) + if not os.path.isdir(opod_dir): + os.mkdir(opod_dir) + + work_dir = slc_dir + '/' + date + if not os.path.isdir(work_dir): + os.mkdir(work_dir) + + work_dir = down_dir + os.chdir(work_dir) + + t_date = down_dir + '/t_' + date + t_burst_numb = down_dir + '/t_burst_numb_' + date + + call_str = 'ls ' + down_dir + '/S1*' + date + '*.zip > ' + t_date + os.system(call_str) + + zip_files = ut.read_txt2list(t_date) + print(zip_files) + #for i in range(len(zip_files)): + # call_str = 'S1_BURST_tab_from_zipfile - ' + zip_files[i] + ' - ' + # os.system(call_str) + AA=''.join(str(zip_files)).split("'")[1] + call_str = 'S1_BURST_tab_from_zipfile - ' + AA + ' - ' + os.system(call_str) + + call_str = 'ls ' + down_dir + '/S1*' + date + '*.burst_number_table > ' + t_burst_numb + os.system(call_str) + burst_files = ut.read_txt2list(t_burst_numb) + + + iw1_burst_node = [] + iw2_burst_node = [] + iw3_burst_node = [] + + print(burst_files) + for i in range(len(burst_files)): + print(burst_files[i]) + iw1_first = ut.read_gamma_par(str(burst_files[i]),'read','iw1_first_burst') + iw1_last = ut.read_gamma_par(str(burst_files[i]),'read','iw1_last_burst') + iw1_burst_node.append(float(iw1_first)) + iw1_burst_node.append(float(iw1_last)) + + iw2_first = ut.read_gamma_par(str(burst_files[i]),'read','iw2_first_burst') + iw2_last = ut.read_gamma_par(str(burst_files[i]),'read','iw2_last_burst') + iw2_burst_node.append(float(iw2_first)) + iw2_burst_node.append(float(iw2_last)) + + iw3_first = ut.read_gamma_par(str(burst_files[i]),'read','iw3_first_burst') + iw3_last = ut.read_gamma_par(str(burst_files[i]),'read','iw3_last_burst') + iw3_burst_node.append(float(iw3_first)) + iw3_burst_node.append(float(iw3_last)) + + + if 'iw1_first_burst' in templateDict: IW1_0 = templateDict['iw1_first_burst'] + else: IW1_0 = '1' + if 'iw1_last_burst' in templateDict: IW1_1 = templateDict['iw1_last_burst'] + else: IW1_1 = str(round(max(iw1_burst_node) - min(iw1_burst_node) + 1)) + + if 'iw2_first_burst' in templateDict: IW2_0 = templateDict['iw2_first_burst'] + else: IW2_0 = '1' + if 'iw2_last_burst' in templateDict: IW2_1 = templateDict['iw2_last_burst'] + else: IW2_1 = str(round(max(iw2_burst_node) - min(iw2_burst_node) + 1)) + + if 'iw3_first_burst' in templateDict: IW3_0 = templateDict['iw3_first_burst'] + else: IW3_0 = '1' + if 'iw3_last_burst' in templateDict: IW3_1 = templateDict['iw3_last_burst'] + else: IW3_1 = str(round(max(iw3_burst_node) - min(iw3_burst_node) + 1)) + + if int(IW1_1) > round(max(iw1_burst_node) - min(iw1_burst_node) + 1): + IW1_1 = str(round(max(iw1_burst_node) - min(iw1_burst_node) + 1)) + + if int(IW2_1) > round(max(iw2_burst_node) - min(iw2_burst_node) + 1): + IW2_1 = str(round(max(iw2_burst_node) - min(iw2_burst_node) + 1)) + + if int(IW3_1) > round(max(iw3_burst_node) - min(iw3_burst_node) + 1): + IW3_1 = str(round(max(iw3_burst_node) - min(iw3_burst_node) + 1)) + + + D_IW1_first = int(IW1_0) - 1 + DD_IW1_numb = int(IW1_1) - int(IW1_0) + 1 + + D_IW2_first = int(IW2_0) - 1 + DD_IW2_numb = int(IW2_1) - int(IW2_0) + 1 + + D_IW3_first = int(IW3_0) - 1 + DD_IW3_numb = int(IW3_1) - int(IW3_0) + 1 + + master_busrt = down_dir + '/master.burst_numb_table' + if os.path.isfile(master_busrt): + os.remove(master_busrt) + + with open(master_busrt, 'a') as f: + STR0 = 'iw1_number_of_bursts: ' + str(DD_IW1_numb) + '\n' + STR00 = 'iw1_number_of_bursts: ' + str(DD_IW1_numb) + print(STR00) + f.write(STR0) + + STR0 = 'iw1_first_burst: ' + str(min(iw1_burst_node) + D_IW1_first) + '\n' + STR00 = 'iw1_first_burst: ' + str(min(iw1_burst_node) + D_IW1_first) + print(STR00) + f.write(STR0) + + STR0 = 'iw1_last_burst: ' + str(min(iw1_burst_node) + D_IW1_first + DD_IW1_numb - 1) + '\n' + STR00 = 'iw1_last_burst: ' + str(min(iw1_burst_node) + D_IW1_first + DD_IW1_numb - 1) + print(STR00) + f.write(STR0) + + STR0 = 'iw2_number_of_bursts: ' + str(DD_IW2_numb) + '\n' + STR00 = 'iw2_number_of_bursts: ' + str(DD_IW2_numb) + print(STR00) + f.write(STR0) + + STR0 = 'iw2_first_burst: ' + str(min(iw2_burst_node) + D_IW2_first) + '\n' + STR00 = 'iw2_first_burst: ' + str(min(iw2_burst_node) + D_IW2_first) + print(STR00) + f.write(STR0) + + STR0 = 'iw2_last_burst: ' + str(min(iw2_burst_node) + D_IW2_first + DD_IW2_numb - 1) + '\n' + STR00 = 'iw2_last_burst: ' + str(min(iw2_burst_node) + D_IW2_first + DD_IW2_numb - 1) + print(STR00) + f.write(STR0) + + + STR0 = 'iw3_number_of_bursts: ' + str(DD_IW3_numb) + '\n' + STR00 = 'iw3_number_of_bursts: ' + str(DD_IW3_numb) + print(STR00) + f.write(STR0) + + STR0 = 'iw3_first_burst: ' + str(min(iw3_burst_node) + D_IW3_first) + '\n' + STR00 = 'iw3_first_burst: ' + str(min(iw3_burst_node) + D_IW3_first) + print(STR00) + f.write(STR0) + + STR0 = 'iw3_last_burst: ' + str(min(iw3_burst_node) + D_IW3_first + DD_IW3_numb - 1) + '\n' + STR00 = 'iw3_last_burst: ' + str(min(iw3_burst_node) + D_IW3_first + DD_IW3_numb - 1) + print(STR00) + f.write(STR0) + + + print("Get burst number table for master date is done! ") + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/gmt_grdview.sh b/.codex_tmp/pyint_variants/no_rescue/pyint/gmt_grdview.sh new file mode 100644 index 0000000..39f32e8 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/gmt_grdview.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# GMT modern mode bash template +# Date: 2024-10-01 +#Purpose: quickly generate basic GMT plot +#Author: +#Dependencies: GNUPlot, GMT v6 +#Written for: GNU/Linux + + +if [ $# -lt 1 ]; then +more < 0 " | bc` == 1 ] && [ ` echo "$vmax > 1 " | bc` == 1 ]; then + echo "1" + export cmin=`echo | gmt grdinfo $grdfile | grep 'v_min' | awk '{print $5/$3 } '` + export max_num=`echo "scale=6; 1/$vmax" | bc ` + export min_num=`echo "$max_num*$vmin" | bc ` + gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -G$cmin/1 -Z >1.cpt +elif [ ` echo "$flag < 0 " | bc` == 1 ] && [ ` echo "$vmin < -1 " | bc` == 1 ]; then + echo "2" + export cmax=`echo | gmt grdinfo $grdfile | grep 'v_min' | awk '{print $3/$5*-1} '` + export min_num=`echo "scale=6; 1/$vmin" | bc` + export max_num=`echo "$min_num*$vmax" | bc` + gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -G-1/$cmax -Z >1.cpt +elif [ `echo " $flag > 0 " | bc` == 1 ] && [ ` echo "$vmax < 1 " | bc` == 1 ]; then + echo "3" + if [ ` echo "$vmax < 0.1 " | bc` == 1 ]; then + export vmin1=$(echo "scale=2; $vmin*10" | bc) + export vmax1=$(echo "scale=2; $vmax*10" | bc) + gmt makecpt -C$color_file -G$vmin1/$vmax1 -T$vmin/$vmax/0.01 -Z >1.cpt + else + gmt makecpt -C$color_file -G$vmin/$vmax -T$vmin/$vmax/0.01 -Z >1.cpt + fi +elif [ ` echo "$flag < 0 " | bc` == 1 ] && [ ` echo "$vmin > -1 " | bc` == 1 ]; then + if [ ` echo "$vmin >-0.1 " | bc` == 1 ]; then + echo "4" + export vmin1=$(awk "BEGIN{print($vmin*10)}") + export vmax1=$(awk "BEGIN{print($vmax*10)}") + gmt makecpt -C$color_file -G$vmin1/$vmax1 -T$vmin/$vmax/0.01 -Z >1.cpt + else + gmt makecpt -C$color_file -G$vmin/$vmax -T$vmin/$vmax/0.01 -Z >1.cpt + fi +else + echo "5" + gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -Z >1.cpt +fi +gmt begin ${gmt_basename} png + + gmt grdimage $grdfile -C1.cpt -JM7.3c -Bxa1 -Bya1 -BWSen -R$region + gmt colorbar -Dx3.5/-1c+w6c/0.25c+jBC+h -Bxa0.2f0.1 -C1.cpt +gmt end show diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/gmt_makecpt.sh b/.codex_tmp/pyint_variants/no_rescue/pyint/gmt_makecpt.sh new file mode 100644 index 0000000..48cb96e --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/gmt_makecpt.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# GMT modern mode bash template +# Date: 2024-10-01 +#Purpose: quickly generate basic GMT plot +#Author: +#Dependencies: GNUPlot, GMT v6 +#Written for: GNU/Linux + + +if [ $# -lt 2 ]; then +more <0 " | bc ` -eq 1 ] && [ ` echo "$vmax > 1 " | bc ` -eq 1 ]; then + echo "1" + export cmin=$(awk "BEGIN{print($vmin/$vmax)}") + gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -G$cmin/1 -Z >disp.cpt + elif [ ` echo "$flag < 0 " | bc ` -eq 1 ] && [ ` echo "$vmin < -1 " | bc ` -eq 1 ]; then + echo "2" + export cmax=$(awk "BEGIN{print(-$vmax/$vmin)}") + gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -G-1/$cmax -Z >disp.cpt + elif [ `echo "$flag > 0 " | bc ` -eq 1 ] && [ ` echo "$vmax < 1 " | bc ` -eq 1 ]; then + echo "3" + if [ ` echo "$vmax < 0.1 " | bc ` -eq 1 ]; then + export vmin1=$(awk "BEGIN{print($vmin*10)}") + export vmax1=$(awk "BEGIN{print($vmax*10)}") + gmt makecpt -C$color_file -G$vmin1/$vmax1 -T$vmin/$vmax/0.01 -Z >disp.cpt + else + gmt makecpt -C$color_file -G$vmin/$vmax -T$vmin/$vmax/0.01 -Z >disp.cpt + fi + elif [ ` echo "$flag < 0 " | bc ` -eq 1 ] && [ ` echo "$vmin > -1 " | bc ` -eq 1 ]; then + if [ ` echo "$vmin >-0.1 " | bc ` -eq 1 ]; then + echo "4" + export vmin1=$(awk "BEGIN{print($vmin*10)}") + export vmax1=$(awk "BEGIN{print($vmax*10)}") + gmt makecpt -C$color_file -G$vmin1/$vmax1 -T$vmin/$vmax/0.01 -Z >disp.cpt + else + gmt makecpt -C$color_file -G$vmin/$vmax -T$vmin/$vmax/0.01 -Z >disp.cpt + fi + else + echo "5" + gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -Z >disp.cpt + fi + +elif [ "$type" == "phase" ]; then + export inc=$(awk "BEGIN{print($vmax/10)}") + gmt makecpt -Crainbow -T$vmin/$vmax/$inc -Ww -Z >phase.cpt +else + echo "error in the type" + exit +fi +export inc=$(awk "BEGIN{print(($vmax-$vmin)/4)}") +export inc_c=$(awk "BEGIN{print(($vmax-$vmin)/4)}") + +# Draw GMT map diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/gmt_plot_interf.sh b/.codex_tmp/pyint_variants/no_rescue/pyint/gmt_plot_interf.sh new file mode 100644 index 0000000..a5befea --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/gmt_plot_interf.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# GMT modern mode bash template +# Date: 2024-10-01 +#Purpose: quickly generate basic GMT plot +#Author: +#Dependencies: GNUPlot, GMT v6 +#Written for: GNU/Linux + + +if [ $# -lt 2 ]; then +more <0 " | bc ` -eq 1 ] && [ ` echo "$vmax > 1 " | bc ` -eq 1 ]; then + echo "1" + export cmin=`echo | gmt grdinfo los.grd | grep 'v_min' | awk '{print $5/$3 } '` + export max_num=`echo "scale=6; 1/$vmax" | bc ` + export min_num=`echo "$max_num*$vmin" | bc ` + gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -G$cmin/1 -Z >1.cpt +elif [ ` echo "$flag < 0 " | bc ` -eq 1 ] && [ ` echo "$vmin < -1 " | bc ` -eq 1 ]; then + echo "2" + export cmax=`echo | gmt grdinfo los.grd | grep 'v_min' | awk '{print $3/$5*-1} '` + export min_num=`echo "scale=6; 1/$vmin" | bc` + export max_num=`echo "$min_num*$vmax" | bc` + gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -G-1/$cmax -Z >1.cpt +elif [ `echo "$flag > 0 " | bc ` -eq 1 ] && [ ` echo "$vmax < 1 " | bc ` -eq 1 ]; then + echo "3" + if [ ` echo "$vmax < 0.1 " | bc ` -eq 1 ]; then + export vmin1=$(echo "scale=2; $vmin*10" | bc) + export vmax1=$(echo "scale=2; $vmax*10" | bc) + gmt makecpt -C$color_file -G$vmin1/$vmax1 -T$vmin/$vmax/0.01 -Z >1.cpt + else + gmt makecpt -C$color_file -G$vmin/$vmax -T$vmin/$vmax/0.01 -Z >1.cpt + fi +elif [ ` echo "$flag < 0 " | bc ` -eq 1 ] && [ ` echo "$vmin > -1 " | bc ` -eq 1 ]; then + if [ ` echo "$vmin >-0.1 " | bc ` -eq 1 ]; then + echo "4" + export vmin1=$(awk "BEGIN{print($vmin*10)}") + export vmax1=$(awk "BEGIN{print($vmax*10)}") + echo $vmin1 + echo $vmax1 + gmt makecpt -C$color_file -G$vmin1/$vmax1 -T$vmin/$vmax/0.01 -Z >1.cpt + else + gmt makecpt -C$color_file -G$vmin/$vmax -T$vmin/$vmax/0.01 -Z >1.cpt + fi +else + echo "5" + gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -Z >1.cpt +fi + +#gmt makecpt -C$color_file -G$vmin/$vmax -T$vmin/$vmax/0.01 >1.cpt + gmt makecpt -Crainbow -T-3.14159265/3.14159265/0.31415926 -Z >2.cpt + + +# Draw GMT map +gmt begin insar_los_unw_${gmt_basename} png + gmt subplot begin 1x2 -Fs7.0c -A+jTR+gwhite+p1p+o0.05c/0.2c -M0.6c/0.25c -JM7.3c -Bxa1 -Bya1 -BWSen -Y10c -R$region + gmt subplot set 0 -A"Unw" + gmt grdimage unw_plot.grd -C2.cpt -Q + gmt colorbar -Dx3.5c/-1c+w6c/0.25c+jBC+h -Bxa3.14 -C2.cpt + gmt subplot set 1 -A"Los" + gmt grdimage los.grd -C1.cpt + gmt colorbar -Dx3.5/-1c+w6c/0.25c+jBC+h -Bxa0.06f0.02 -C1.cpt + gmt subplot end +gmt end show \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/hyp3_timeseries_utm2wgs84.py b/.codex_tmp/pyint_variants/no_rescue/pyint/hyp3_timeseries_utm2wgs84.py new file mode 100644 index 0000000..a1bbcb5 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/hyp3_timeseries_utm2wgs84.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +""" +将 HDF5 文件的坐标系统从 UTM 转换为 WGS84 地理坐标 +同时保持与 MintPy tsview.py 的兼容性 +""" + +import h5py +import numpy as np +import copy +import argparse +import os + +def utm_to_wgs84(easting, northing, zone, northern=True): + """ + 将 UTM 坐标转换为 WGS84 大地坐标 + """ + # UTM 参数 + a = 6378137.0 # WGS84 长半轴 + e = 0.081819190842622 # WGS84 第一偏心率 + k0 = 0.9996 # 比例因子 + + # 中央经线 + lon0 = (zone - 1) * 6 - 180 + 3 + + # 调整东伪偏移 + x = easting - 500000.0 + + # 如果是南半球，调整北伪偏移 + if not northern: + y = northing - 10000000.0 + else: + y = northing + + # M = y / k0 + mu = y / (a * (1 - e**2 / 4 - 3 * e**4 / 64 - 5 * e**6 / 256) * k0) + + e1 = (1 - np.sqrt(1 - e**2)) / (1 + np.sqrt(1 - e**2)) + + # 纬度计算 + N1 = a / np.sqrt(1 - e**2 * np.sin(mu)**2) + T1 = np.tan(mu)**2 + C1 = e1**2 * np.cos(mu)**2 + R1 = a * (1 - e**2) / (1 - e**2 * np.sin(mu)**2)**(3/2) + D = x / (N1 * k0) + + lat = mu - N1 * np.tan(mu) / R1 * (D**2 / 2 + (5 + 3 * T1 + 10 * C1 - 4 * C1**2 - 9 * e1**2) * D**4 / 24 + (61 + 90 * T1 + 298 * C1 + 45 * T1**2 - 252 * e1**2 - 3 * C1**2) * D**6 / 720) + + # 经度计算 + lon = lon0 * np.pi / 180 + D * (1 + (1 + 3 * e1**2 + 2 * e1**4) * D**2 / 6 + (2 - e1**2) * D**4 / 120) / np.cos(mu) + + # 转换为角度 + lat_deg = np.degrees(lat) + lon_deg = np.degrees(lon) + + return lon_deg, lat_deg + + +def main(): + parser = argparse.ArgumentParser( + description='将 HDF5 文件转换为 WGS84 地理坐标（与 MintPy 兼容）', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=''' +示例: + python convert_to_wgs84.py input.h5 -o output.h5 + +说明: + 脚本会将文件完全转换为 WGS84 地理坐标系统， + 同时设置正确的属性以保持与 MintPy 的兼容性。 + ''' + ) + parser.add_argument('input', help='输入 HDF5 文件路径') + parser.add_argument('-o', '--output', help='输出 HDF5 文件路径（默认：输入文件名_WGS84.h5）') + + args = parser.parse_args() + + # 输入和输出文件 + input_file = args.input + + if args.output: + output_file = args.output + else: + # 自动生成输出文件名 + base, ext = os.path.splitext(input_file) + output_file = f"{base}_WGS84{ext}" + + print(f"正在读取文件: {input_file}") + f_in = h5py.File(input_file, 'r') + + # 读取所有数据集 + datasets = {} + for key in f_in.keys(): + datasets[key] = f_in[key][:] + print(f" 读取数据集: {key}, 形状: {datasets[key].shape}") + + # 读取所有属性 + attrs = dict(f_in.attrs) + print(f"\n共有 {len(attrs)} 个属性") + + # 解析 UTM 坐标信息 + x_first = float(attrs['X_FIRST']) + y_first = float(attrs['Y_FIRST']) + x_step = float(attrs['X_STEP']) + y_step = float(attrs['Y_STEP']) + width = int(attrs['WIDTH']) + length = int(attrs['LENGTH']) + utm_zone = attrs['UTM_ZONE'] + + # 获取参考点位置（像素坐标） + ref_x = int(attrs.get('REF_X', 0)) + ref_y = int(attrs.get('REF_Y', 0)) + + # 转换 UTM Zone + zone_num = int(''.join(filter(str.isdigit, utm_zone))) + is_northern = 'N' in utm_zone.upper() + + print(f"\n原始坐标系统:") + print(f" UTM Zone: {utm_zone} (EPSG: {attrs['EPSG']})") + print(f" 左上角 UTM: ({x_first}, {y_first})") + print(f" 步长: X={x_step}m, Y={y_step}m") + print(f" 参考点像素: ({ref_x}, {ref_y})") + + # 生成网格坐标 + x_coords = x_first + np.arange(width) * x_step + y_coords = y_first + np.arange(length) * y_step + + # 转换为 WGS84 + print(f"\n正在转换为 WGS84...") + lon_grid, lat_grid = np.meshgrid(x_coords, y_coords) + wgs84_lon, wgs84_lat = utm_to_wgs84(lon_grid, lat_grid, zone_num, is_northern) + + # 计算新的坐标参数（注意：lon 是经度，lat 是纬度） + # X 对应经度，Y 对应纬度 + lon_first = wgs84_lon[0, 0] + lat_first = wgs84_lat[0, 0] + lon_step = abs(wgs84_lon[0, 1] - wgs84_lon[0, 0]) if width > 1 else 0 + lat_step = abs(wgs84_lat[1, 0] - wgs84_lat[0, 0]) if length > 1 else 0 + + # 计算参考点的经纬度 + # REF_X 对应 X 轴（经度），REF_Y 对应 Y 轴（纬度） + ref_lon = wgs84_lon[ref_y, ref_x] # 经度 + ref_lat = wgs84_lat[ref_y, ref_x] # 纬度 + + # 更新属性 - 完全转换为 WGS84 + attrs_new = copy.deepcopy(attrs) + + # 更新坐标系统相关属性 + attrs_new['EPSG'] = 4326 # WGS84 + attrs_new['X_FIRST'] = lon_first # 经度 + attrs_new['Y_FIRST'] = lat_first # 纬度 + attrs_new['X_STEP'] = lon_step + attrs_new['Y_STEP'] = -lat_step # 保持Y方向向下为负 + attrs_new['X_UNIT'] = 'degrees' + attrs_new['Y_UNIT'] = 'degrees' + + # 更新参考坐标 - 现在是真正的经纬度 + attrs_new['REF_LON'] = ref_lon + attrs_new['REF_LAT'] = ref_lat + + # 保留原始 UTM 信息作为备份（带 _ORIG 后缀） + attrs_new['UTM_ZONE_ORIG'] = attrs_new.get('UTM_ZONE', '') + attrs_new['EPSG_ORIG'] = attrs_new.get('EPSG', '') + + # 关键：移除或清空 UTM_ZONE 属性，让 MintPy 识别为地理坐标系统 + if 'UTM_ZONE' in attrs_new: + del attrs_new['UTM_ZONE'] + + # 更新角点坐标 + attrs_new['LAT_REF1'] = lat_first + attrs_new['LAT_REF2'] = wgs84_lat[0, -1] + attrs_new['LAT_REF3'] = wgs84_lat[-1, 0] + attrs_new['LAT_REF4'] = wgs84_lat[-1, -1] + attrs_new['LON_REF1'] = lon_first + attrs_new['LON_REF2'] = wgs84_lon[0, -1] + attrs_new['LON_REF3'] = wgs84_lon[-1, 0] + attrs_new['LON_REF4'] = wgs84_lon[-1, -1] + + # 设置其他相关属性 + attrs_new['COORD_SYSTEM'] = 'GEO' # 地理坐标系统 + + print(f"\n新坐标系统 (WGS84):") + print(f" EPSG: {attrs_new['EPSG']}") + print(f" 左上角: ({attrs_new['X_FIRST']:.6f}, {attrs_new['Y_FIRST']:.6f})") + print(f" 步长: X={attrs_new['X_STEP']:.6f}°, Y={attrs_new['Y_STEP']:.6f}°") + print(f" 参考点经纬度: ({attrs_new['REF_LON']:.6f}, {attrs_new['REF_LAT']:.6f})") + print(f" 参考点像素: ({ref_x}, {ref_y})") + + # 创建新的 HDF5 文件 + print(f"\n正在写入文件: {output_file}") + f_out = h5py.File(output_file, 'w') + + # 写入所有数据集 + for key, data in datasets.items(): + f_out.create_dataset(key, data=data) + print(f" 写入数据集: {key}") + + # 写入所有属性 + for key, value in attrs_new.items(): + f_out.attrs[key] = value + + f_in.close() + f_out.close() + + print(f"\n完成！转换后的文件已保存为: {output_file}") + print("\n说明:") + print(" - 文件已完全转换为 WGS84 地理坐标系统") + print(" - 所有坐标属性已更新为经纬度") + print(" - 原始 UTM 信息已保存为 _ORIG 后缀的属性") + print(" - 应该与 MintPy tsview.py 兼容") + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/hyp3format_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/hyp3format_gamma.py new file mode 100644 index 0000000..dbffcb1 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/hyp3format_gamma.py @@ -0,0 +1,275 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Modified: 2026, Z. Zou - HyP3 UTM GeoTIFF output ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import argparse +import time +from datetime import datetime + +from pyint import _utils as ut + + +INTRODUCTION = ''' +------------------------------------------------------------------- + 将 geocode_gamma.py 已地理编码的 GAMMA 二进制产品转换为 + ASF HyP3 兼容的 UTM GeoTIFF 格式, 用于 MintPy 时间序列分析。 + + 本脚本 **不做任何地理编码或计算**, 仅执行: + 1. data2geotiff : GAMMA 二进制 (EQA) → GeoTIFF (EQA) + 2. gdalwarp : GeoTIFF (EQA) → GeoTIFF (UTM) + 3. rasterio : 生成水体掩膜 + + 所有地理编码产品由 geocode_gamma.py 提供: + amp, corr, dem, unw_phase, wrapped_phase, + los_disp, vert_disp, lv_theta, lv_phi +''' + +EXAMPLE = ''' + Usage: + hyp3format_gamma.py projectName ifgPair + hyp3format_gamma.py shanghaiT171F128S1A 20241105-20241117 + hyp3format_gamma.py shanghaiT171F128S1A 20241105-20241117 --output-dir /path/to/output +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser( + description='Convert geocoded GAMMA binaries to HyP3 UTM GeoTIFF.', + formatter_class=argparse.RawTextHelpFormatter, + epilog=INTRODUCTION + '\n' + EXAMPLE) + parser.add_argument('projectName', help='项目名称') + parser.add_argument('ifgPair', help='干涉图对 (如 20241105-20241117)') + parser.add_argument('--output-dir', dest='output_dir', default=None, + help='输出目录. 默认: projectDir/Hyp3Products/') + inps = parser.parse_args() + return inps + + +def get_utm_epsg(lon, lat): + """根据中心经纬度自动计算 UTM 投影的 EPSG 代码""" + zone = int((lon + 180) / 6) + 1 + return 32600 + zone if lat >= 0 else 32700 + zone + + +def run_cmd(cmd, desc=''): + """执行 shell 命令并检查返回值""" + if desc: + print(f" [{desc}] {cmd}") + else: + print(f" {cmd}") + ret = os.system(cmd) + if ret != 0: + print(f" WARNING: 返回非零退出码 {ret}") + return ret + + +def main(argv): + + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + ifgPair = inps.ifgPair + + Mdate = ifgPair.split('-')[0] + Sdate = ifgPair.split('-')[1] + + # 验证日期并计算时间基线 + d1 = datetime.strptime(Mdate, '%Y%m%d') + d2 = datetime.strptime(Sdate, '%Y%m%d') + interval = abs((d2 - d1).days) + + # 读取模板参数 + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + '/' + projectName + '.template' + templateDict = ut.update_template(templateFile) + + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + + # 项目目录 + projectDir = scratchDir + '/' + projectName + ifgDir = projectDir + '/ifgrams' + demDir = projectDir + '/DEM' + + workDir = ifgDir + '/' + ifgPair + if not os.path.isdir(workDir): + print(f"ERROR: 干涉图目录不存在: {workDir}") + sys.exit(1) + + # 输出目录 + outputDir = inps.output_dir or (projectDir + '/Hyp3Products') + productDir = os.path.join(outputDir, ifgPair) + os.makedirs(productDir, exist_ok=True) + + # ============================================================ + # DEM 参数 (直接引用, 不复制不删除) + # ============================================================ + DEMpar = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem.par' + + nWidthDEM = ut.read_gamma_par(DEMpar, 'read', 'width') + nLineDEM = ut.read_gamma_par(DEMpar, 'read', 'nlines') + + # 自动检测 UTM 投影 + corner_lat = float(ut.read_gamma_par(DEMpar, 'read', 'corner_lat').split()[0]) + corner_lon = float(ut.read_gamma_par(DEMpar, 'read', 'corner_lon').split()[0]) + post_lat = float(ut.read_gamma_par(DEMpar, 'read', 'post_lat').split()[0]) + post_lon = float(ut.read_gamma_par(DEMpar, 'read', 'post_lon').split()[0]) + center_lat = corner_lat + float(nLineDEM) / 2 * post_lat + center_lon = corner_lon + float(nWidthDEM) / 2 * post_lon + utm_epsg = get_utm_epsg(center_lon, center_lat) + + # HyP3 文件名前缀 (含 YYYYMMDDTHHMMSS, 兼容 MintPy 日期解析) + hyp3_prefix = f"{projectName}_{Mdate}T000000_{Sdate}T000000" + + print("\n" + "=" * 70) + print(f"HyP3 格式转换: {projectName} / {ifgPair}") + print(f"时间基线: {interval} 天, DEM: {nWidthDEM}×{nLineDEM}") + print(f"中心: {center_lat:.4f}°N, {center_lon:.4f}°E → EPSG:{utm_epsg}") + print(f"输出: {productDir}") + print("=" * 70) + + # ============================================================ + # 已地理编码 (EQA) GAMMA 二进制文件 (由 geocode_gamma.py 生成) + # ============================================================ + geo_unw = workDir + '/geo_' + ifgPair + '_' + rlks + 'rlks.diff_filt.unw' + geo_wrapped_pha = workDir + '/geo_' + ifgPair + '_' + rlks + 'rlks.diff_filt.pha' + geo_los_disp = workDir + '/geo_' + ifgPair + '_' + rlks + 'rlks.los_disp' + geo_vert_disp = workDir + '/geo_' + ifgPair + '_' + rlks + 'rlks.vert_disp' + geo_amp = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.amp' + geo_hgt = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.hgt' + lv_theta = workDir + '/lv_theta' + lv_phi = workDir + '/lv_phi' + + # 相干性可能用 Pair 或 masterDate 命名 + geo_cor = workDir + '/geo_' + ifgPair + '_' + rlks + 'rlks.diff_filt.cor' + if not os.path.exists(geo_cor): + geo_cor = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.diff_filt.cor' + + # ============================================================ + # data2geotiff (EQA binary → EQA GeoTIFF) → gdalwarp (→ UTM) + # ============================================================ + print("\n--- GAMMA 二进制 → EQA GeoTIFF → UTM GeoTIFF ---") + + product_map = [ + ('amp', geo_amp, 'bilinear'), + ('corr', geo_cor, 'bilinear'), + ('dem', geo_hgt, 'bilinear'), + ('unw_phase', geo_unw, 'bilinear'), + ('wrapped_phase', geo_wrapped_pha, 'bilinear'), + ('los_disp', geo_los_disp, 'bilinear'), + ('vert_disp', geo_vert_disp, 'bilinear'), + ('lv_theta', lv_theta, 'bilinear'), + ('lv_phi', lv_phi, 'bilinear'), + ] + + generated = {} + + for name, geo_bin, interp in product_map: + utm_tif = os.path.join(productDir, f'{hyp3_prefix}_{name}.tif') + + # 跳过已存在且非空的输出 + if os.path.exists(utm_tif) and os.path.getsize(utm_tif) > 0: + print(f" {name}: 已存在，跳过") + generated[name] = utm_tif + continue + + if not os.path.exists(geo_bin): + print(f" {name}: 源文件不存在 ({os.path.basename(geo_bin)})，跳过") + continue + + eqa_tif = workDir + '/' + name + '_hyp3_eqa.tif' + + # GAMMA 二进制 → EQA GeoTIFF + run_cmd(f'data2geotiff {DEMpar} {geo_bin} 2 {eqa_tif}', f'{name} → EQA') + + if not os.path.exists(eqa_tif): + print(f" WARNING: {name} EQA GeoTIFF 生成失败") + continue + + # EQA → UTM (gdalwarp) + run_cmd(f'gdalwarp -t_srs EPSG:{utm_epsg} -r {interp} ' + f'-co COMPRESS=LZW -overwrite {eqa_tif} {utm_tif}', + f'{name} → UTM') + + # 清理 EQA 中间文件 + if os.path.exists(eqa_tif): + os.remove(eqa_tif) + + if os.path.exists(utm_tif) and os.path.getsize(utm_tif) > 0: + generated[name] = utm_tif + + # ============================================================ + # 生成水体掩膜 (相干性阈值) + # ============================================================ + print("\n--- 水体掩膜 ---") + water_mask_tif = os.path.join(productDir, f'{hyp3_prefix}_water_mask.tif') + corr_tif = generated.get('corr') + + if corr_tif and os.path.exists(corr_tif) and not os.path.exists(water_mask_tif): + try: + import rasterio + with rasterio.open(corr_tif) as src: + corr_data = src.read(1) + mask = np.ones_like(corr_data, dtype=np.uint8) + mask[corr_data < 0.05] = 0 + mask[np.isnan(corr_data)] = 0 + + profile = src.profile.copy() + profile.update(dtype=rasterio.uint8, count=1, compress='lzw', nodata=None) + with rasterio.open(water_mask_tif, 'w', **profile) as dst: + dst.write(mask, 1) + generated['water_mask'] = water_mask_tif + print(f" 水体掩膜已生成") + except Exception as e: + print(f" WARNING: 水体掩膜生成失败: {e}") + elif os.path.exists(water_mask_tif): + generated['water_mask'] = water_mask_tif + print(f" 水体掩膜已存在，跳过") + + # ============================================================ + # 生成入射角图 (inc_map / inc_map_ell, 复制自 lv_theta) + # ============================================================ + print("\n--- 入射角图 ---") + inc_map_tif = os.path.join(productDir, f'{hyp3_prefix}_inc_map.tif') + inc_map_ell_tif = os.path.join(productDir, f'{hyp3_prefix}_inc_map_ell.tif') + lv_theta_tif = generated.get('lv_theta') + + if lv_theta_tif and os.path.exists(lv_theta_tif): + if not os.path.exists(inc_map_tif): + os.system(f'cp {lv_theta_tif} {inc_map_tif}') + generated['inc_map'] = inc_map_tif + print(f" inc_map 已生成 (= lv_theta)") + if not os.path.exists(inc_map_ell_tif): + os.system(f'cp {lv_theta_tif} {inc_map_ell_tif}') + generated['inc_map_ell'] = inc_map_ell_tif + print(f" inc_map_ell 已生成 (近似 lv_theta)") + + # ============================================================ + # 输出摘要 + # ============================================================ + print("\n" + "=" * 70) + print(f"HyP3 格式转换完成: {ifgPair}") + print(f"共 {len(generated)} 个产品:") + for name, path in sorted(generated.items()): + if os.path.exists(path): + size_mb = os.path.getsize(path) / 1024 / 1024 + print(f" {name:20s} ({size_mb:.1f} MB)") + print("=" * 70) + + ut.print_process_time(start_time, time.time()) + sys.exit(0) + + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/hyp3format_gamma_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/hyp3format_gamma_all.py new file mode 100644 index 0000000..bce90ec --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/hyp3format_gamma_all.py @@ -0,0 +1,147 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Modified: 2026, Z. Zou - HyP3 UTM batch conversion ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import time +import glob +import argparse +import subprocess + +from pyint import _utils as ut + + +def work(data0): + """并行 worker: 执行单对 hyp3format_gamma.py""" + cmd = data0[0] + err_file = data0[1] + p = subprocess.run(cmd, shell=False, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stderr = p.stderr + + if type(stderr) == bytes: + aa = stderr.decode("utf-8") + else: + aa = stderr + + if aa: + str0 = ' '.join(cmd) + '\n' + with open(err_file, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return + +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + 批量将 GAMMA 处理结果转换为 HyP3 UTM GeoTIFF 格式, + 支持并行处理。 +''' + +EXAMPLE = ''' + Usage: + hyp3format_gamma_all.py projectName + hyp3format_gamma_all.py projectName --parallel 4 + hyp3format_gamma_all.py projectName --output-dir /path/to/output + hyp3format_gamma_all.py projectName --parallel 4 --ifgramList-txt /test/ifgram_list.txt +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser( + description='批量转换 GAMMA 输出为 HyP3 UTM GeoTIFF 格式.', + formatter_class=argparse.RawTextHelpFormatter, + epilog=INTRODUCTION + '\n' + EXAMPLE) + + parser.add_argument('projectName', help='项目名称') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, + help='并行处理器数量') + parser.add_argument('--ifgramList-txt', dest='ifgarmListTxt', + help='干涉图列表文件. 默认: projectName/ifgram_list.txt') + parser.add_argument('--output-dir', dest='output_dir', default=None, + help='输出目录. 默认: projectDir/Hyp3Products/') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + projectDir = scratchDir + '/' + projectName + ifgDir = projectDir + '/ifgrams' + templateDict = ut.update_template(templateFile) + masterDate = templateDict['masterDate'] + + # 输出目录 + output_dir = inps.output_dir or (projectDir + '/Hyp3Products') + os.makedirs(output_dir, exist_ok=True) + + # 读取干涉图列表 + if inps.ifgarmListTxt: + ifgramList_txt = inps.ifgarmListTxt + else: + ifgramList_txt = projectDir + '/ifgram_list.txt' + + ifgList0 = ut.read_txt2array(ifgramList_txt) + + if len(ifgList0) == 3: + ifgList = ifgList0[0] + ifgList = [ifgList] + else: + ifgList = ifgList0[:,0] + + # 错误日志 + err_txt = projectDir + '/hyp3format_gamma_all.err' + if os.path.isfile(err_txt): + os.remove(err_txt) + + # 构建并行命令 + data_para = [] + skip_count = 0 + for i in range(len(ifgList)): + Pair = ifgList[i] + + # 检查输出是否已存在 (用 corr.tif 作为完成标记) + pair_dir = output_dir + '/' + Pair + # 匹配 HyP3 命名: {prefix}_corr.tif + geo_file0 = pair_dir + '/' + projectName + '_' + Pair.replace('-', 'T000000_') + 'T000000_corr.tif' + + if os.path.isfile(geo_file0) and os.path.getsize(geo_file0) > 0: + skip_count += 1 + continue + + cmd0 = ['hyp3format_gamma.py', projectName, Pair] + if inps.output_dir: + cmd0.extend(['--output-dir', inps.output_dir]) + + data0 = [cmd0, err_txt] + data_para.append(data0) + + print(f"HyP3 格式转换: 共 {len(ifgList)} 对, 跳过 {skip_count} 对, 待处理 {len(data_para)} 对") + print(f"并行数: {inps.parallelNumb}, 输出目录: {output_dir}") + + # 并行执行 + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + + print("HyP3 格式转换完成: project %s" % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/ionosphere_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/ionosphere_gamma.py new file mode 100644 index 0000000..61ffab6 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/ionosphere_gamma.py @@ -0,0 +1,411 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +from pyint import _utils as ut + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Calculate the ionospheric phases in an interferogram based on RSI + +''' + +EXAMPLE = ''' + Usage: + ionosphere_gamma.py projectName Mdate Sdate + ionosphere_gamma.py PacayaT163TsxHhA 20150102 20150601 +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Calculate the ionospheric phases in an interferogram based on RSI.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('Mdate',help='Master date.') + parser.add_argument('Sdate',help='Slave date.') + + inps = parser.parse_args() + return inps + + +def main(argv): + + start_time = time.time() + inps = cmdLineParse() + Mdate = inps.Mdate + Sdate = inps.Sdate + + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + + projectDir = scratchDir + '/' + projectName + demDir = scratchDir + '/' + projectName + '/DEM' + + slcDir = scratchDir + '/' + projectName + '/SLC' + rslcDir = scratchDir + '/' + projectName + '/RSLC' + ifgDir = projectDir + '/ifgrams' + if not os.path.isdir(ifgDir): os.mkdir(ifgDir) + + Pair = Mdate + '-' + Sdate + workDir = ifgDir + '/' + Pair + if not os.path.isdir(workDir): os.mkdir(workDir) + + ####################################################################### + Mamp0 = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar0 = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + Samp0 = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar0 = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + Mrslc = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc' + MrslcPar = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc.par' + Srslc = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc' + SrslcPar = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc.par' + + HGT0 = demDir + '/' + masterDate + '_' + rlks + 'rlks.rdc.dem' + + MasterPar0 = rslcDir + '/' + masterDate + '/' + masterDate + '.rslc.par' + + ################# copy file for parallel processing ########################## + Mamp = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + Samp = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + #Mrslc = workDir + '/' + Mdate + '.rslc' + #MrslcPar = workDir + '/' + Mdate + '.rslc.par' + #Srslc = workDir + '/' + Sdate + '.rslc' + #SrslcPar = workDir + '/' + Sdate + '.rslc.par' + + HGT = workDir + '/' + masterDate + '_' + rlks + 'rlks.rdc.dem' + MasterPar = workDir + '/' + masterDate + '.rslc.par' + + if not os.path.isfile(Mamp):ut.copy_file(Mamp0,Mamp) + if not os.path.isfile(MampPar):ut.copy_file(MampPar0,MampPar) + if not os.path.isfile(Samp):ut.copy_file(Samp0,Samp) + if not os.path.isfile(SampPar):ut.copy_file(SampPar0,SampPar) + + #if not os.path.isfile(Mrslc): ut.copy_file(Mrslc0,Mrslc) + #if not os.path.isfile(MrslcPar): ut.copy_file(MrslcPar0,MrslcPar) + #if not os.path.isfile(Srslc):ut.copy_file(Srslc0,Srslc) + #if not os.path.isfile(SrslcPar):ut.copy_file(SrslcPar0,SrslcPar) + + if not os.path.isfile(HGT):ut.copy_file(HGT0,HGT) + if not os.path.isfile(MasterPar):ut.copy_file(MasterPar0,MasterPar) + + + nWidth = ut.read_gamma_par(MampPar, 'read', 'range_samples') + nLine = ut.read_gamma_par(MampPar, 'read', 'azimuth_lines') + + ###################### Output files ################################# + ionoDir = workDir + '/ionosphere' + if not os.path.isdir(ionoDir): + os.mkdir(ionoDir) + + workDir = ionoDir + Mrslc_low = ionoDir + '/' + Mdate + '.low.rslc' + MrslcPar_low = ionoDir + '/' + Mdate + '.low.rslc.par' + Mrslc_high = ionoDir + '/' + Mdate + '.high.rslc' + MrslcPar_high = ionoDir + '/' + Mdate + '.high.rslc.par' + + Srslc_low = ionoDir + '/' + Sdate + '.low.rslc' + SrslcPar_low = ionoDir + '/' + Sdate + '.low.rslc.par' + Srslc_high = ionoDir + '/' + Sdate + '.high.rslc' + SrslcPar_high = ionoDir + '/' + Sdate + '.high.rslc.par' +######################################################### + TEST0 = MrslcPar_high + k0 = 0 + if os.path.isfile(TEST0): + if os.path.getsize(TEST0) > 0: + k0 =1 + + TEST1 = SrslcPar_high + k1 = 0 + if os.path.isfile(TEST1): + if os.path.getsize(TEST1) > 0: + k1 =1 + + if k0 ==0: + call_str = 'bpf_ssi ' + Mrslc + ' ' + MrslcPar + ' ' + Mrslc_low + ' ' + MrslcPar_low + ' ' + Mrslc_high + ' ' + MrslcPar_high + os.system(call_str) + + if k1==0: + call_str = 'bpf_ssi ' + Srslc + ' ' + SrslcPar + ' ' + Srslc_low + ' ' + SrslcPar_low + ' ' + Srslc_high + ' ' + SrslcPar_high + os.system(call_str) + + ################## interferometry ##################### + + off_low = workDir + '/' + Pair +'_' + rlks + 'rlks.low.off' + call_str = 'create_offset '+ MrslcPar_low + ' ' + SrslcPar_low + ' ' + off_low + ' 1 ' + rlks + ' ' + azlks + ' 0' + os.system(call_str) + + sim_low_unw = workDir + '/' + Pair + '.low.sim_unw' + call_str = 'phase_sim_orb ' + MrslcPar_low + ' ' + SrslcPar_low + ' ' + off_low + ' ' + HGT + ' ' + sim_low_unw + ' ' + MasterPar + ' - - 1 1' + os.system(call_str) + + diff_low = workDir + '/' + Pair + '_' + rlks + 'rlks.low.diff' + call_str = 'SLC_diff_intf ' + Mrslc_low + ' ' + Srslc_low + ' ' + MrslcPar_low + ' ' + SrslcPar_low + ' ' + off_low + ' ' + sim_low_unw + ' ' + diff_low + ' ' + rlks + ' ' + azlks + ' ' + ' 1 0 0.25' + os.system(call_str) + + + off_high = workDir + '/' + Pair +'_' + rlks + 'rlks.high.off' + call_str = 'create_offset '+ MrslcPar_high + ' ' + SrslcPar_high + ' ' + off_high + ' 1 ' + rlks + ' ' + azlks + ' 0' + os.system(call_str) + + sim_high_unw = workDir + '/' + Pair + '.high.sim_unw' + call_str = 'phase_sim_orb ' + MrslcPar_high + ' ' + SrslcPar_high + ' ' + off_high + ' ' + HGT + ' ' + sim_high_unw + ' ' + MasterPar + ' - - 1 1' + os.system(call_str) + + diff_high = workDir + '/' + Pair + '_' + rlks + 'rlks.high.diff' + call_str = 'SLC_diff_intf ' + Mrslc_high + ' ' + Srslc_high + ' ' + MrslcPar_high + ' ' + SrslcPar_high + ' ' + off_high + ' ' + sim_high_unw + ' ' + diff_high + ' ' + rlks + ' ' + azlks + ' ' + ' 1 0 0.25' + os.system(call_str) + + ###################################################### + hl_diff = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff' + hl_diff_cc = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff.cc' + hl_diff_mask = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff.cc_mask.bmp' + + hl_diff1 = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff1' + hl_diff2 = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff2' + hl_diff3 = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff3' + hl_diff4 = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff4' + + hl_diff4_phase = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff4_phase' + + hl_diff_jpg = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff.jpg' + + diff_low_phase = workDir + '/' + Pair + '_' + rlks + 'rlks.low.diff_phase' + call_str = 'cpx_to_real ' + diff_low + ' ' + diff_low_phase + ' ' + nWidth + ' 4' + os.system(call_str) + + call_str = 'subtract_phase ' + diff_high + ' ' + diff_low_phase + ' ' + hl_diff + ' ' + nWidth + ' 1' + os.system(call_str) + + call_str = 'rasmph_pwr ' + Mamp + ' ' + hl_diff + ' ' + nWidth + #call_str = 'vismph_pwr.py ' + hl_diff + ' ' + Mamp + ' ' + nWidth + ' -f 1.0 ' + ' -z 1000 ' + ' -p ' + hl_diff_jpg + os.system(call_str) + + #### filter + + call_str = 'adf ' + hl_diff + ' ' + hl_diff1 + ' ' + hl_diff_cc + ' ' + nWidth + ' 0.2 512 7 128' + os.system(call_str) + + call_str = 'adf ' + hl_diff1 + ' ' + hl_diff2 + ' ' + hl_diff_cc + ' ' + nWidth + ' 0.3 256 7 64' + os.system(call_str) + + call_str = 'adf ' + hl_diff2 + ' ' + hl_diff3 + ' ' + hl_diff_cc + ' ' + nWidth + ' 0.3 128 7 64' + os.system(call_str) + + call_str = 'adf ' + hl_diff3 + ' ' + hl_diff4 + ' ' + hl_diff_cc + ' ' + nWidth + ' 0.3 128 7 16' + os.system(call_str) + + # unwrawp + call_str = 'cpx_to_real ' + hl_diff4 + ' ' + hl_diff4_phase + ' ' + nWidth + ' 4' + os.system(call_str) + + # shift to zero + # mask very low coherence + + call_str = 'cc_ad ' + hl_diff + ' ' + Mamp + ' ' + Samp + ' - - ' + hl_diff_cc + ' ' + nWidth + ' 3 9' + os.system(call_str) + + call_str = 'rascc_mask ' + hl_diff_cc + ' ' + Mamp + ' ' + nWidth + ' 1 1 0 1 1 0.15 0.0 0.1 0.9 1.0 0.35 1 ' + hl_diff_mask + os.system(call_str) + + nWidth_half = str(float(nWidth)/2) + image_report = workDir + '/' + Pair + '.image_stat.report' + + #call_str = 'image_stat ' + hl_diff4_phase + ' ' + nWidth + ' ' + nWidth_half + ' 200 200 ' + image_report + call_str = 'image_stat ' + hl_diff4_phase + ' ' + nWidth + ' 0 0 ' + ' 200 200 ' + image_report + os.system(call_str) + + mean_phase = ut.read_gamma_par(image_report, 'read', 'mean') + std_phase = ut.read_gamma_par(image_report, 'read', 'stdev') + + + hl_diff_phase_shift = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff.ph_shift' + hl_diff_shifted = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff4.shifted' + hl_diff_shifted_phase = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff4.shifted.phase' + call_str = 'lin_comb 1 ' + hl_diff4_phase + ' ' + mean_phase + ' 0.0 ' + hl_diff_phase_shift + ' ' + nWidth + ' 1 ' + nLine + os.system(call_str) + + call_str = 'subtract_phase ' + hl_diff4_phase + ' ' + hl_diff_phase_shift + ' ' + hl_diff_shifted + ' ' + nWidth + ' 1' + os.system(call_str) + + call_str = 'cpx_to_real ' + hl_diff_shifted + ' ' + hl_diff_shifted_phase + ' ' + nWidth + ' 4' + os.system(call_str) + + call_str = 'lin_comb 1 ' + hl_diff_shifted_phase + ' ' + mean_phase + ' 1.0 ' + hl_diff4_phase + ' ' + nWidth + ' 1 ' + nLine + os.system(call_str) + + ############# determine trend considering cc_mask using unwrapping with multi_cpx before unwrapping ### + + diff_par = workDir + '/' + Pair + '.ddiff.par' + call_str = 'create_diff_par ' + MampPar + ' - ' + diff_par + ' 1 0' + os.system(call_str) + + ddiff_phase_trend = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff4_phase.trend' + call_str = 'quad_fit ' + hl_diff4_phase + ' ' + diff_par + ' 5 5 ' + hl_diff_mask + ' - 3 ' + ddiff_phase_trend + os.system(call_str) + + #call_str = 'vistd_pwr.py ' + ddiff_phase_trend + ' ' + Mamp + ' ' + nWidth + ' -c 12.6 -z 1000 -f 1.0 -m rmg -p ' + ddiff_phase_trend + '.jpg' + #os.system(call_str) + + hl_diff_detrend = hl_diff+ '.detrend' + call_str = 'subtract_phase ' + hl_diff + ' ' + ddiff_phase_trend + ' ' + hl_diff_detrend + ' ' + nWidth + ' 1.' + os.system(call_str) + + #call_str = 'vismph_pwr.py ' + hl_diff_detrend + ' ' + Mamp + ' ' + nWidth + ' -f 1.0 -z 1000 -p ' + hl_diff_detrend+'.jpg' + #os.system(call_str) + + + call_str = 'adf ' + hl_diff_detrend + ' ' + hl_diff1 + ' ' + hl_diff+'.smcc' + ' ' + nWidth + ' 0.2 512 7 128' + os.system(call_str) + + call_str = 'adf ' + hl_diff1 + ' ' + hl_diff2 + ' ' + hl_diff+'.smcc' + ' ' + nWidth + ' 0.3 256 7 64' + os.system(call_str) + + call_str = 'adf ' + hl_diff2 + ' ' + hl_diff3 + ' ' + hl_diff+'.smcc' + ' ' + nWidth + ' 0.3 128 7 64' + os.system(call_str) + + hl_diff4_detrend = hl_diff4+ '.detrend' + hl_diff_cc = hl_diff+'.sm4.cc' + call_str = 'adf ' + hl_diff3 + ' ' + hl_diff4_detrend + ' ' + hl_diff_cc + ' ' + nWidth + ' 0.3 128 7 16' + os.system(call_str) + + off1 = workDir + '/' + Pair + '.off1' + call_str = 'create_offset ' + MampPar + ' ' + MampPar + ' ' + off1 + ' 1 1 1 0' + os.system(call_str) + + hl_diff4_mask = hl_diff+'.sm4.cc_mask.bmp' + call_str = 'rascc_mask ' + hl_diff_cc + ' ' + Mamp + ' ' + nWidth + ' 1 1 0 1 1 0.95 0.0 0.1 0.1 0.9 1.0 0.35 1 ' + hl_diff4_mask + os.system(call_str) + + call_str = 'mask_class ' + hl_diff_mask + ' ' + hl_diff4_detrend + ' ' + hl_diff4_detrend + '.masked.tmp ' + ' 1 1 1 1 0 0.0 0.0' + os.system(call_str) + + call_str = 'mask_class ' + hl_diff4_mask + ' ' + hl_diff4_detrend + '.masked.tmp ' + ' ' + hl_diff4_detrend + '.masked' + ' 1 1 1 1 0 0.0 0.0' + os.system(call_str) + + + ddiff5 = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff5' + off5 = workDir + '/' + Pair + '.off5' + call_str = 'multi_cpx ' + hl_diff4_detrend + '.masked' + ' ' + off1 + ddiff5 + ' ' + off5 + ' 5 5' + os.system(call_str) + + call_str = 'multi_real ' + Mamp + ' ' + off1 + ' ' + Mamp + '.5' + off5 + ' 5 5' + os.system(call_str) + + call_str = 'multi_real ' + hl_diff_cc + ' ' + off1 + ' ' + hl_diff_cc + '.5' + off5 + ' 5 5' + os.system(call_str) + + nWidth5 = ut.read_gamma_par(off5,'read','interferogram_width') + + ddiff5_phase_tmp = ddiff5 + '.phase.tmp' + ddiff5_phase_interp = ddiff5 + '.phase.interp' + call_str = 'cpx_to_real ' + ddiff5 + ' ' + ddiff5_phase_tmp + ' ' + nWidth5 + ' 4' + os.system(call_str) + + call_str = 'fill_gaps ' + ddiff5_phase_tmp + ' ' + nWidth5 + ' ' + ddiff5_phase_interp + ' 0 4 - 1 100 4 400' + os.system(call_str) + + + #### remove outliers + ddiff5_fspf = ddiff5_phase_interp + '.fspf' + call_str = 'fspf ' + ddiff5_phase_interp + ' ' + ddiff5_fspf + ' ' + nWidth5 + ' 2 64 3' + os.system(call_str) + + ddiff5_phase_interp_outliers = ddiff5_phase_interp + '.outliers' + call_str = 'lin_comb 2 ' + ddiff5_phase_interp + ' ' + ddiff5_fspf + ' 100 1.0 -1.0 ' + ddiff5_phase_interp_outliers + ' ' + nWidth5 + os.system(call_str) + + hl_diff_cc5 = hl_diff_cc + '.5' + hl_diff5_mask = ddiff5 + '.mask.bmp' + call_str = 'single_class_mapping 2 ' + ddiff5_phase_interp_outliers + ' 99.95 100.05 ' + hl_diff_cc5 + ' 0.15 1.0 ' + hl_diff5_mask + ' ' + nWidth5 + os.system(call_str) + + ddiff5_phase_tmp1 = ddiff5_phase_tmp + '1' + call_str = 'mask_class ' + hl_diff5_mask + ' ' + ddiff5_phase_tmp + ' ' + ddiff5_phase_tmp1 + ' 0 1 1 1 0 0.0 0.0' + os.system(call_str) + + call_str = 'fill_gaps ' + ddiff5_phase_tmp1 + ' ' + ddiff5_phase_interp + ' 0 4 - 1 100 4 400' + os.system(call_str) + + ddiff_detrend_interp_phase = hl_diff + '.detrend' + '.interp.phase' + call_str = 'multi_real ' + ddiff5_phase_interp + ' ' + off5 + ' ' + ddiff_detrend_interp_phase + ' ' + off1 + ' -5 -5' + os.system(call_str) + + call_str = 'fspf ' + ddiff_detrend_interp_phase + ' ' + ddiff_detrend_interp_phase + '.fspf' + ' ' + nWidth + ' 2 8 3' + os.system(call_str) + + #call_str = ' visdt_pwr.py ' + ddiff_detrend_interp_phase + '.fspf' + ' ' + Mamp + ' ' + nWidth + ' -c 1.6 -z 1000 -f 1.0 -m rmg -p ' + ddiff_detrend_interp_phase + '.fspf.jpg' + #os.system(call_str) + + + ###### add the solutions + + ddiff_phase_fspf = hl_diff + ' .phase.fspf' + call_str = 'lin_comb 2 ' + ddiff_phase_trend + ' ' + ddiff_detrend_interp_phase + '.fspf' + ' 0.0 1.0 1.0 ' + ddiff_phase_fspf + ' ' + nWidth + ' 1 ' + nLine + os.system(call_str) + + #call_str ='visdt_pwr.py ' + ddiff_phase_fspf + ' ' + Mamp + ' ' + nWidth + ' -c 1.6 -z 1000 -f 1.0 -m rmg -p ' + ddiff_phase_fspf + '.jpg' + #os.system(call_str) + + + ###################### determin scaling factor using bpf_ssi + + bpf_ssi_out = workDir + '/bpf_ssi.out' + call_str = 'bpf_ssi ' + Mrslc + ' ' + MrslcPar + ' - - - - 0.6666 > ' + bpf_ssi_out + os.system(call_str) + + a0 = ut.read_gamma_par(bpf_ssi_out,'read','a') + b0 = ut.read_gamma_par(bpf_ssi_out,'read','b') + x0 = ut.read_gamma_par(bpf_ssi_out,'read','x') + y0 = ut.read_gamma_par(bpf_ssi_out,'read','y') + z0 = ut.read_gamma_par(bpf_ssi_out,'read','z') + zz0 = str(float(z0)*2) + + ddiff_phase_fspf_scaled = ddiff_phase_fspf + '.scaled' + call_str = 'lin_comb 1 ' + ddiff_phase_fspf + ' 0.0 ' + zz0 + ' ' + ddiff_phase_fspf_scaled + ' ' + nWidth + os.system(call_str) + + call_str = 'rasrmg ' + ddiff_phase_fspf_scaled + ' ' + Mamp + ' ' + nWidth + #call_str = 'vismph_pwr.py ' + hl_diff + ' ' + Mamp + ' ' + nWidth + ' -f 1.0 ' + ' -z 1000 ' + ' -p ' + hl_diff_jpg + os.system(call_str) + + #call_str ='visdt_pwr.py ' + ddiff_phase_fspf_scaled + ' ' + Mamp + ' ' + nWidth + ' -c 1.6 -z 1000 -f 1.0 -m rmg -p ' + ddiff_phase_fspf_scaled + '.jpg' + #os.system(call_str) + + # 2_phi_iono = phi0 + zz * ddiff_phase_fspf + # 2_phi_non-dispersive = phi0 - zz * ddiff_phase_fspf + + #call_str = ' subtract_phase ' + + + + print("Estimating the scaled ionospheric phases is done!") + ut.print_process_time(start_time, time.time()) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/load_data_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/load_data_gamma.py new file mode 100644 index 0000000..d3a567d --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/load_data_gamma.py @@ -0,0 +1,643 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Central South University & University of Miami ### +################################################################# + +import os +import sys +import glob +import time +import argparse +from mintpy.utils import readfile, writefile, ptime, utils as ut +import h5py +import numpy as np + +def print_progress(iteration, total, prefix='calculating:', suffix='complete', decimals=1, barLength=50, elapsed_time=None): + """Print iterations progress - Greenstick from Stack Overflow + Call in a loop to create terminal progress bar + @params: + iteration - Required : current iteration (Int) + total - Required : total iterations (Int) + prefix - Optional : prefix string (Str) + suffix - Optional : suffix string (Str) + decimals - Optional : number of decimals in percent complete (Int) + barLength - Optional : character length of bar (Int) + elapsed_time- Optional : elapsed time in seconds (Int/Float) + + Reference: http://stackoverflow.com/questions/3173320/text-progress-bar-in-the-console + """ + filledLength = int(round(barLength * iteration / float(total))) + percents = round(100.00 * (iteration / float(total)), decimals) + bar = '#' * filledLength + '-' * (barLength - filledLength) + if elapsed_time: + sys.stdout.write('%s [%s] %s%s %s %s secs\r' % (prefix, bar, percents, '%', suffix, int(elapsed_time))) + else: + sys.stdout.write('%s [%s] %s%s %s\r' % (prefix, bar, percents, '%', suffix)) + sys.stdout.flush() + if iteration == total: + print("\n") + + ''' + Sample Useage: + for i in range(len(dateList)): + print_progress(i+1,len(dateList)) + ''' + return + + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def read_data(inFile, dtype, nWidth, nLength): + data = np.fromfile(inFile, dtype, int(nLength)*int(nWidth)).reshape(int(nLength),int(nWidth)) + return data + +def is_number(s): + try: + int(s) + return True + except ValueError: + return False + +def add_zero(s): + if len(s)==1: + s="000"+s + elif len(s)==2: + s="00"+s + elif len(s)==3: + s="0"+s + return s + +def read_list(ListName): + List=glob.glob(ListName) + return List + +def read_list_except(ListName,ExString): + List=glob.glob(ListName) + List_New = [] + for S in List: + if ExString not in os.path.basename(S): + List_New.append(S) + return List_New + + +def load_gamma2multi_group_h5(fileType, fileName, fileList, RSCList, datatype): + + RSC =RSCList[0] + rsc_dic = read_roipac_rsc(RSC) + nWidth = rsc_dic['WIDTH'] + nLine = rsc_dic['FILE_LENGTH'] + + H5FILE = fileName + '.h5' + fileNum = len(fileList) + print('Start to load ' + fileName + ' >>> %s %s files will be loaded for further process' % ( str(fileNum), fileType)) + f = h5py.File(H5FILE,'w') + gg=f.create_group(fileType) + + for i in range(len(fileList)): + S=fileList[i] + RSC =RSCList[i] + rsc_dic = read_roipac_rsc(RSC) + print_progress(i+1, fileNum, prefix='loading', suffix=os.path.basename(S)) + data = read_data(S,datatype,nWidth,nLine) + File = os.path.basename(S) + group = gg.create_group(File) + dset = group.create_dataset(File, data = data, compression='gzip') + for key,value in rsc_dic.items(): + group.attrs[key] = value + group.attrs['X_MIN'] = '0' + group.attrs['X_MAX'] = str(int(nWidth)-1) + group.attrs['Y_MIN'] = '0' + group.attrs['Y_MAX'] = str(int(nLine)-1) + group.attrs['DATE12']=File.split('_')[2] + group.attrs['DATATYPE'] = datatype + group.attrs['PROCCESSOR'] = 'gamma' + + f.close() + + +def Get_Datelist(projectName): + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + rslcDir = scratchDir + '/' + projectName + "/RSLC" + slcDir = scratchDir + '/' + projectName + "/SLC" + + ListSLC = os.listdir(slcDir) + Datelist = [] + + for kk in range(len(ListSLC)): + if ( is_number(ListSLC[kk]) and len(ListSLC[kk])==6 ): # if SAR date number is 8, 6 should change to 8. + DD=ListSLC[kk] + Year=int(DD[0:2]) + Month = int(DD[2:4]) + Day = int(DD[4:6]) + if ( 0 < Year < 20 and 0 < Month < 13 and 0 < Day < 32 ): + Datelist.append(ListSLC[kk]) + Datelist = list(map(int,Datelist)) + Datelist.sort() + Datelist = list(map(str,Datelist)) + return Datelist + +def Get_Inflist(projectName): + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + IFGRAM = scratchDir + '/' + projectName + "/PROCESS/IFG*" + IFGList = glob.glob(IFGRAM) + + ListSLC = os.listdir(slcDir) + Datelist = [] + + +def Get_PairName(projectName): + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + IFGRAM = scratchDir + '/' + projectName + "/PROCESS/IFG*" + IFGList = glob.glob(IFGRAM) + + PAIR_Name = [] + for kk in range(len(IFGList)): + SS = IFGList[kk] + IFGName = os.path.basename(SS) + PAIR=IFGName.split('_')[2] + PAIR_Name.append(PAIR) + + return PAIR_Name + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +def UseGamma2(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[2].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +def Remove_Inf_List(RemoveNumberStr): + AD=RemoveNumberStr + Addlist = [] + if len(AD)>0: + AD= AD.split('[')[1].split(']')[0] + if ',' in AD: + LL=AD.split(',') + for kk in range(len(LL)): + XX=LL[kk] + if is_number(XX): + Addlist.append(XX) + + else: + D1=XX.split(':')[0] + D2=XX.split(':')[1] + for jj in range(int(D1),int(D2)+1): + Addlist.append(str(jj)) + else: + LL = AD + if is_number(LL): + Addlist.append(LL) + else: + D1=LL.split(':')[0] + D2=LL.split(':')[1] + for jj in range(int(D1),int(D2)+1): + Addlist.append(str(jj)) + for ii in range(len(Addlist)): + Addlist[ii]=int(Addlist[ii])-1 + Addlist = list(map(str,Addlist)) + return Addlist + + +def read_roipac_rsc(File): + '''Read ROI_PAC .rsc file into a python dictionary structure.''' + rsc_dict = dict(np.loadtxt(File, dtype=str, usecols=(0,1))) + return rsc_dict + + +######################################################################### + +INTRODUCTION = ''' +############################################################################# + Copy Right(c): 2017, Yunmeng Cao @PyINT v1.0 + + Loading data for PyMIS processing. + +''' + +EXAMPLE= ''' + Usage: + load_data_pymis.py projectName + load_data_pymis.py projectName --demRdc + load_data_pymis.py projectName --wrapIfgram + + Examples: + load_data_pymis.py PacayaT163TsxHhA + load_data_pymis.py PacayaT163TsxHhA --demRdc + load_data_pymis.py PacayaT163TsxHhA --coherence + +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Loading data for PyMIS processing.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='project name of PyMIS.') + parser.add_argument('--coherence',action="store_true", default=False, help='Loading coherence file.') + parser.add_argument('--wrapIfgram',action="store_true", default=False, help='Loading wrapped interferograms.') + parser.add_argument('--unwrapIfgram',action="store_true", default=False, help='Loading unwrapped interferograms.') + parser.add_argument('--demRdc',action="store_true", default=False, help='Loading radar coordinates DEM.') + parser.add_argument('--demGeo',action="store_true", default=False, help='Loading GEO coordinates DEM.') + parser.add_argument('--geo2rdc',action="store_true", default=False, help='Loading geocoding lookup table.') + + inps = parser.parse_args() + + + + return inps + +################################################################################ + +def main(argv): + + + total = time.time() + inps = cmdLineParse() + projectName = inps.projectName + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateContents=read_template(templateFile) + + processDir = scratchDir + '/' + projectName + "/ifgrams" + #processDir = scratchDir + '/' + projectName + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + #simDir = scratchDir + '/' + projectName + "/PROCESS/DEM" + simDir = scratchDir + '/' + projectName + "/DEM" + + + masterDate = templateContents['masterDate'] + rlks = templateContents['range_looks'] + azlks = templateContents['azimuth_looks'] + + workDir = scratchDir + '/' + projectName + '/TSSAR' + if not os.path.isdir(workDir): + call_str='mkdir ' + workDir + os.system(call_str) + + if not os.path.isdir(workDir): + call_str='mkdir ' + workDir + os.system(call_str) + + print('Project : ' + projectName) + print('Loading data for timeseries processing >>>') + print('Change process Dir to :' + workDir) + os.chdir(workDir) + + +################# DEM Define ############ + + + + + UTM2RDC = simDir+'/' + masterDate + '_' + rlks + 'rlks.UTM_TO_RDC' + RDCDEM = simDir+'/' + masterDate + '_' + rlks + 'rlks.rdc.dem' + UTMDEM = simDir+'/' + masterDate + '_' + rlks + 'rlks.utm.dem' + UTMPAR = simDir+'/' + masterDate + '_' + rlks + 'rlks.utm.dem.par' + + + +# if not os.path.isfile(UTM2RDC): +# print 'Start to generate subset look-up table and subset-DEM >>>' +# call_str = 'CreateRdcDem_Sub_Gammapy.py ' + igramDir +# print call_str +# os.system(call_str) +# else: +# print 'Subet lookup table %s is existed!' % UTM2RDC + + Dem_Format = UseGamma(UTMPAR, 'read', 'data_format:') + + if Dem_Format=='REAL*4': + dtype_utmdem='f4' + else: + dtype_utmdem='i2' + + nWidthUTM = UseGamma(UTMPAR, 'read', 'width:') + nLineUTM = UseGamma(UTMPAR, 'read', 'nlines:') + + Corner_LAT = UseGamma(UTMPAR, 'read', 'corner_lat:') + Corner_LON = UseGamma(UTMPAR, 'read', 'corner_lon:') + + Corner_LAT =Corner_LAT.split(' ')[0] + Corner_LON =Corner_LON.split(' ')[0] + + post_Lat = UseGamma(UTMPAR, 'read', 'post_lat:') + post_Lon = UseGamma(UTMPAR, 'read', 'post_lon:') + + post_Lat =post_Lat.split(' ')[0] + post_Lon =post_Lon.split(' ')[0] + +############### timeseries file define ################## + + if 'IFG_list' in templateContents: IFG_list = templateContents['IFG_list'] + else: IFG_list = processDir + '/*/diff_filt*rlks.int' + if 'UNW_list' in templateContents: UNW_list = templateContents['UNW_list'] + else: UNW_list = processDir + '/*/diff_filt*rlks.unw' + if 'COR_list' in templateContents: COR_list = templateContents['COR_list'] + else: COR_list = processDir + '/*/filt*rlks.cor' + if 'MASK_list' in templateContents: MASK_list = templateContents['MASK_list'] + # else: MASK_list = processDir + '/*/diff_filt*rlks.cor_mask' + else: MASK_list = processDir + '/*/*.cor_mask.bmp' + if 'RSC_list' in templateContents: RSC_list = templateContents['RSC_list'] + else: RSC_list = processDir + '/*/diff_filt_*unw.rsc' + + MM=MASK_list[0] + if MM.split('.')[len(MM.split('.'))-1]=='bmp': + dtype_mask ='u1' + else: + dtype_mask ='f4' + + if 'Byte_order' in templateContents: byteoder = templateContents['Byte_order'] + else: byteorder = 'big' + + if byteorder =='big': + sign = '>' + else: + sign ='<' + + dtype_inf = sign + 'c8' + dtype_unw = sign + 'f4' + dtype_utmdem = sign + dtype_utmdem + dtype_rdcdem = sign + 'f4' + dtype_lt = sign + 'c8' + dtype_mask = sign + dtype_mask + dtype_cor = sign + 'f4' + +#################### start to load data ######################## + # InfList = read_list(IFG_list) + InfList = ut.get_file_list(IFG_list, abspath=True) + NInf = len(InfList) + + print('RSC_Name is : ' + RSC_list) + # RSCList = read_list(RSC_list) + RSCList = ut.get_file_list(RSC_list, abspath=True) + RSC = RSCList[0] + rsc_dic = read_roipac_rsc(RSC) + nWidth = rsc_dic['WIDTH'] + nLine = rsc_dic['FILE_LENGTH'] + UNWList = ut.get_file_list(UNW_list, abspath=True) + CORList = ut.get_file_list(COR_list, abspath=True) + # UNWList = read_list(UNW_list) + # CORList = read_list(COR_list) + Flag_K = 0 + if inps.coherence: + Flag_K = 1 + print('') + print('COR_Name is : ' + COR_list) + load_gamma2multi_group_h5('coherence','coherence', CORList, RSCList, dtype_cor) + elif inps.wrapIfgram: + Flag_K = 1 + print('') + print('IFG_Name is : ' + IFG_list) + load_gamma2multi_group_h5('wrapped','wrapIfgram', InfList, RSCList, dtype_inf) + elif inps.unwrapIfgram: + Flag_K = 1 + print('') + print('UNW_Name is : ' + UNW_list) + load_gamma2multi_group_h5('interferograms','unwrapIfgram', UNWList, RSCList, dtype_unw) + elif inps.demGeo: + Flag_K = 1 + print('') + print('Start to write GEO-DEM into h5 file >>> ' + UTMDEM) + data = read_data(UTMDEM, dtype_utmdem, nWidthUTM, nLineUTM) + H5FILE = 'demGeo.h5' + f =h5py.File(H5FILE,'w') + group=f.create_group('dem') + dset = group.create_dataset('dem', data=data, compression='gzip') + group.attrs['WIDTH'] = nWidthUTM + group.attrs['FILE_LENGTH'] = nLineUTM + group.attrs['X_FIRST'] = Corner_LON # X: latitude Y: longitude + group.attrs['Y_FIRST'] = Corner_LAT + group.attrs['X_STEP'] = post_Lon + group.attrs['Y_STEP'] = post_Lat + group.attrs['X_UNIT'] = 'degrees' + group.attrs['Y_UNIT'] = 'degrees' + group.attrs['X_MIN'] = '0' + group.attrs['X_MAX'] = str(int(nWidthUTM)-1) + group.attrs['Y_MIN'] = '0' + group.attrs['Y_MAX'] = str(int(nLineUTM)-1) + group.attrs['DATATYPE'] = dtype_utmdem + group.attrs['COORD'] = 'geo' + group.attrs['PROCCESSOR'] = 'gamma' + f.close() + elif inps.demRdc: + Flag_K = 1 + print('') + print('Start to write RDC-DEM into h5 file >>> ' + RDCDEM) + data = read_data(RDCDEM, dtype_rdcdem, nWidth, nLine) + H5FILE = 'demRdc.h5' + f =h5py.File(H5FILE,'w') + group=f.create_group('dem') + dset = group.create_dataset('dem', data=data, compression='gzip') + group.attrs['PROCCESSOR'] = 'gamma' + group.attrs['WIDTH'] = nWidth + group.attrs['FILE_LENGTH'] = nLine + group.attrs['X_MIN'] = '0' + group.attrs['X_MAX'] = str(int(nWidth)-1) + group.attrs['Y_MIN'] = '0' + group.attrs['Y_MAX'] = str(int(nLine)-1) + group.attrs['DATATYPE'] = dtype_rdcdem + group.attrs['COORD'] ='radar' + f.close() + elif inps.geo2rdc: + Flag_K = 1 + print('') + print('Start to write GEO2RDC into h5 file >>> ' + UTM2RDC) + data = read_data(UTM2RDC,dtype_lt,nWidthUTM,nLineUTM) # real: range imaginary: azimuth + H5FILE = 'geo2rdc.h5' + f =h5py.File(H5FILE,'w') + group=f.create_group('lt') + dset = group.create_dataset('lt', data=data, compression='gzip') + group.attrs['WIDTH'] = nWidthUTM + group.attrs['FILE_LENGTH'] = nLineUTM + group.attrs['X_FIRST'] = Corner_LON # X: latitude Y: longitude + group.attrs['Y_FIRST'] = Corner_LAT + group.attrs['X_STEP'] = post_Lon + group.attrs['Y_STEP'] = post_Lat + group.attrs['X_UNIT'] = 'degrees' + group.attrs['Y_UNIT'] = 'degrees' + group.attrs['X_MIN'] = '0' + group.attrs['X_MAX'] = str(int(nWidthUTM)-1) + group.attrs['Y_MIN'] = '0' + group.attrs['Y_MAX'] = str(int(nLineUTM)-1) + group.attrs['DATATYPE'] = dtype_utmdem + group.attrs['COORD'] = 'geo' + group.attrs['PROCCESSOR'] = 'gamma' + f.close() + + + + if not os.path.isfile('wrapIfgram.h5') and Flag_K==0: + print('') + print('IFG_Name is : ' + IFG_list) + load_gamma2multi_group_h5('wrapped','wrapIfgram', InfList, RSCList, dtype_inf) + else: + print('') + print('wrapIfgram.h5 has existed, loading wrapped interferograms is skipped.') + + + + if not os.path.isfile('unwrapIfgram.h5') and Flag_K==0: + print('') + print('UNW_Name is : ' + UNW_list) + load_gamma2multi_group_h5('interferograms','unwrapIfgram', UNWList, RSCList, dtype_unw) + else: + print('') + print('unwrapIfgram.h5 has existed, loading unwrapped interferograms is skipped.') + + + if not os.path.isfile('coherence.h5') and Flag_K==0: + print('') + print('COR_Name is : ' + COR_list) + load_gamma2multi_group_h5('coherence','coherence', CORList, RSCList, dtype_cor) + else: + print('') + print('coherence.h5 has existed, loading coherence is skipped.') + + +######### load DEM and lookup table ########## + + if not os.path.isfile('demGeo.h5') and Flag_K==0: + print('') + print('Start to write GEO-DEM into h5 file >>> ' + UTMDEM) + data = read_data(UTMDEM, dtype_utmdem, nWidthUTM, nLineUTM) + H5FILE = 'demGeo.h5' + f =h5py.File(H5FILE,'w') + group=f.create_group('dem') + dset = group.create_dataset('dem', data=data, compression='gzip') + group.attrs['WIDTH'] = nWidthUTM + group.attrs['FILE_LENGTH'] = nLineUTM + group.attrs['X_FIRST'] = Corner_LON # X: latitude Y: longitude + group.attrs['Y_FIRST'] = Corner_LAT + group.attrs['X_STEP'] = post_Lon + group.attrs['Y_STEP'] = post_Lat + group.attrs['X_UNIT'] = 'degrees' + group.attrs['Y_UNIT'] = 'degrees' + group.attrs['X_MIN'] = '0' + group.attrs['X_MAX'] = str(int(nWidthUTM)-1) + group.attrs['Y_MIN'] = '0' + group.attrs['Y_MAX'] = str(int(nLineUTM)-1) + group.attrs['DATATYPE'] = dtype_utmdem + group.attrs['COORD'] = 'geo' + group.attrs['PROCCESSOR'] = 'gamma' + f.close() + else: + print('') + print('demGeo.h5 has existed, loading GEO-DEM is skipped.') + + + if not os.path.isfile('demRdc.h5') and Flag_K==0: + print('') + print('Start to write RDC-DEM into h5 file >>> ' + RDCDEM) + data = read_data(RDCDEM, dtype_rdcdem, nWidth, nLine) + H5FILE = 'demRdc.h5' + f =h5py.File(H5FILE,'w') + group=f.create_group('dem') + dset = group.create_dataset('dem', data=data, compression='gzip') + group.attrs['PROCCESSOR'] = 'gamma' + group.attrs['WIDTH'] = nWidth + group.attrs['FILE_LENGTH'] = nLine + group.attrs['X_MIN'] = '0' + group.attrs['X_MAX'] = str(int(nWidth)-1) + group.attrs['Y_MIN'] = '0' + group.attrs['Y_MAX'] = str(int(nLine)-1) + group.attrs['DATATYPE'] = dtype_rdcdem + group.attrs['COORD'] ='radar' + f.close() + else: + print('') + print('demRdc.h5 has existed, loading RDC-DEM is skipped.') + + + if not os.path.isfile('geo2rdc.h5') and Flag_K==0: + print('') + print('Start to write GEO2RDC into h5 file >>> ' + UTM2RDC) + data = read_data(UTM2RDC,dtype_lt,nWidthUTM,nLineUTM) # real: range imaginary: azimuth + H5FILE = 'geo2rdc.h5' + f =h5py.File(H5FILE,'w') + group=f.create_group('lt') + dset = group.create_dataset('lt', data=data, compression='gzip') + group.attrs['WIDTH'] = nWidthUTM + group.attrs['FILE_LENGTH'] = nLineUTM + group.attrs['X_FIRST'] = Corner_LON # X: latitude Y: longitude + group.attrs['Y_FIRST'] = Corner_LAT + group.attrs['X_STEP'] = post_Lon + group.attrs['Y_STEP'] = post_Lat + group.attrs['X_UNIT'] = 'degrees' + group.attrs['Y_UNIT'] = 'degrees' + group.attrs['X_MIN'] = '0' + group.attrs['X_MAX'] = str(int(nWidthUTM)-1) + group.attrs['Y_MIN'] = '0' + group.attrs['Y_MAX'] = str(int(nLineUTM)-1) + group.attrs['DATATYPE'] = dtype_utmdem + group.attrs['COORD'] = 'geo' + group.attrs['PROCCESSOR'] = 'gamma' + f.close() + else: + print('') + print('geo2rdc.h5 has existed, loading lookup table is skipped.') + + print('') + print('Done.\nLoading data spend ' + str(time.time()-total) +' secs') + sys.exit(1) + + +############################################################################## +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/load_mintpy.py b/.codex_tmp/pyint_variants/no_rescue/pyint/load_mintpy.py new file mode 100644 index 0000000..5be813a --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/load_mintpy.py @@ -0,0 +1,159 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Purpose: loading PyINT products for mintPy ### +### Copy Right (c): 2019, Yunmeng Cao ### +### Author : Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import time +import argparse +import glob + +from pyint import _utils as ut + +def write_template(str0,templateFile): + with open(templateFile, 'a') as f: + f.write(str0 + '\n') + return + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Loading pyint products for mintPy time-series analysis.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='name of the project.') + #parser.add_argument('--data-type', dest='dataType', type=str, default='big_endian',choices={'big_endian', 'little_endian'},help='data type, big endian or little endian. [default: big_endian]') + inps = parser.parse_args() + + return inps + + +INTRODUCTION = ''' +----------------------------------------------------------------- + Loading pyint products for mintPy time-series analysis. + +''' + +EXAMPLE = """Usage: + + load_mintpy.py projectName +----------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + scratchDir0 = os.getenv('SCRATCHDIR0') + projectDir = scratchDir + '/' + projectName + projectDir0 = scratchDir0 + '/' + projectName + demDir = scratchDir + '/' + projectName + '/DEM' + rslcDir = scratchDir + '/' + projectName + '/RSLC' + + ifgDir = projectDir + '/ifgrams' + #unwFile = projectDir + '/ifgrams/*/2*rlks.diff_filt.unw' # considering geocode unw file + unwFile = projectDir + '/ifgrams/*/diff_filt_*.unw' # considering geocode unw file + corFile = projectDir + '/ifgrams/*/diff_filt_*rlks.cor' + #corFile = projectDir + '/ifgrams/*/filt_*.cor' + print(unwFile) + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + MampPar = rslcDir + '/' + masterDate + '/' + masterDate + '_' + rlks + 'rlks.amp.par' + date_list = ut.get_project_slcList(projectName) + date_list = sorted(date_list) + slc_par_list = [rslcDir + '/' + date0 + '/' + date0 + '.rlsc.par' for date0 in date_list ] + ifgdir_list = glob.glob(ifgDir + '/*') + for k0 in ifgdir_list: + pair0 = os.path.basename(k0) + #print(pair0) + m0 = pair0.split('-')[0] + s0 = pair0.split('-')[1] + + workDir = k0 + mamppar0 = rslcDir + '/' + m0 + '/' + m0 + '_' + rlks + 'rlks.amp.par' + samppar0 = rslcDir + '/' + s0 + '/' + s0 + '_' + rlks + 'rlks.amp.par' + mrslcpar0 = rslcDir + '/' + m0 + '/' + m0 + '.rslc.par' + srslcpar0 = rslcDir + '/' + s0 + '/' + s0 + '.rslc.par' + + mamppar = workDir + '/' + m0 + '_' + rlks + 'rlks.amp.par' + samppar = workDir + '/' + s0 + '_' + rlks + 'rlks.amp.par' + mrslcpar = workDir + '/' + m0 + '.rslc.par' + srslcpar = workDir + '/' + s0 + '.rslc.par' + + ut.copy_file(mamppar0, mamppar) + ut.copy_file(samppar0, samppar) + ut.copy_file(mrslcpar0,mrslcpar) + ut.copy_file(srslcpar0,srslcpar) + + #unw_list = glob.glob(ifgDir + '/*/*rlks.diff_filt.unw') + #cor_list = glob.glob(ifgDir + '/*/*rlks.diff_filt.cor') + + dem_geo = glob.glob(demDir + '/*rlks.utm.dem')[0] + geo_par = glob.glob(demDir + '/*rlks.utm.dem.par')[0] + + dem_rdc = glob.glob(demDir + '/*_' + rlks + 'rlks.rdc.dem')[0] + rdc_par = glob.glob(demDir + '/*_' + rlks + 'rlks.diff_par')[0] # diff_par + lt = glob.glob(demDir + '/*_' + rlks + 'rlks.UTM_TO_RDC')[0] + + strPro = 'mintpy.load.processor = gamma' + strUNW = "mintpy.load.unwFile = " + unwFile + strCOR = 'mintpy.load.corFile = ' + corFile + strCon = 'mintpy.load.connCompFile = auto' + strInt = 'mintpy.load.intFile = auto' + strIon = 'mintpy.load.ionoFile = auto' + + strDem = 'mintpy.load.demFile = ' + dem_rdc + strDemGeo = 'mintpy.load.demFile = ' + dem_geo + strLtY = 'mintpy.load.lookupYFile = ' + lt + strLtX = 'mintpy.load.lookupXFile = ' + lt + strInc = 'mintpy.load.incAngleFile = auto' + strAza = 'mintpy.load.azAngleFile = auto' + strSha = 'mintpy.load.shadowMaskFile = auto' + strWat = 'mintpy.load.waterMaskFile = auto' + strBrp = 'mintpy.load.bperpFile = auto' + + templateFile0 = projectDir + '/mintpy.template' + + if 'mintpy.load.processor' not in templateDict: write_template(strPro,templateFile0) + if 'mintpy.load.unwFile' not in templateDict: write_template(strUNW,templateFile0) + if 'mintpy.load.corFile' not in templateDict: write_template(strCOR,templateFile0) + if 'mintpy.load.demFile' not in templateDict: write_template(strDem,templateFile0) + if 'mintpy.load.lookupYFile' not in templateDict: write_template(strLtY,templateFile0) + if 'mintpy.load.lookupXFile' not in templateDict: write_template(strLtX,templateFile0) + + if 'mintpy.load.connCompFile' not in templateDict: write_template(strCon,templateFile0) + if 'mintpy.load.intFile' not in templateDict: write_template(strInt,templateFile0) + if 'mintpy.load.ionoFile' not in templateDict: write_template(strIon,templateFile0) + + if 'mintpy.load.incAngleFile' not in templateDict: write_template(strInc,templateFile0) + if 'mintpy.load.azAngleFile' not in templateDict: write_template(strAza,templateFile0) + if 'mintpy.load.shadowMaskFile' not in templateDict: write_template(strSha,templateFile0) + if 'mintpy.load.waterMaskFile' not in templateDict: write_template(strWat,templateFile0) + if 'mintpy.load.bperpFile' not in templateDict: write_template(strBrp,templateFile0) + + os.chdir(projectDir0) + call_str = 'load_data.py -t ' + templateFile0 + os.system(call_str) + + os.chdir(projectDir0) + write_template(strDemGeo,templateFile0) + call_str = 'load_data.py -t ' + templateFile0 + os.system(call_str) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/make_local_dem.py b/.codex_tmp/pyint_variants/no_rescue/pyint/make_local_dem.py new file mode 100644 index 0000000..0772460 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/make_local_dem.py @@ -0,0 +1,565 @@ +#! /usr/bin/env python +################################################################# +### 从本地 FABDEM 瓦片库生成 GAMMA 格式 DEM ### +### 输入: 经纬度范围 + 本地 FABDEM ZIP 目录 ### +### 输出: .dem + .dem.par (与 makedem.py 输出完全一致) ### +### Author: Cascade AI + zouyuandong ### +### Date : 2026-03-15 ### +################################################################# + +import os +import sys +import re +import math +import glob +import zipfile +import tempfile +import shutil +import argparse +import subprocess +import numpy as np +from pathlib import Path + + +# ========================= GAMMA 参数文件写入 ========================= + +def write_dempar_file(filepath, corner_lon, corner_lat, post_lon, post_lat, + width, nlines, data_format='REAL*4'): + """写入 GAMMA DEM 参数文件 (.dem.par)，与 makedem.py 格式完全一致""" + with open(filepath, 'w') as f: + f.write("Gamma DIFF&GEO DEM/MAP parameter file\n") + f.write("title:\tIMPORTED DEM FROM FABDEM V1-2\n") + f.write("DEM_projection: EQA\n") + f.write("data_format: %s\n" % data_format) + f.write("DEM_hgt_offset: 0.00000\n") + f.write("DEM_scale: 1.00000\n") + f.write("width: %s\n" % str(int(width))) + f.write("nlines: %s\n" % str(int(nlines))) + f.write("corner_lat: %s decimal degrees\n" % str(float(corner_lat))) + f.write("corner_lon: %s decimal degrees\n" % str(float(corner_lon))) + f.write("post_lat: %s decimal degrees\n" % str(float(post_lat))) + f.write("post_lon: %s decimal degrees\n" % str(float(post_lon))) + f.write("\n") + f.write("ellipsoid_name: WGS 84\n") + f.write("ellipsoid_ra: 6378137.000 m\n") + f.write("ellipsoid_reciprocal_flattening: 298.2572236\n") + f.write("\n") + f.write("datum_name: WGS 1984\n") + f.write("datum_shift_dx: 0.000 m\n") + f.write("datum_shift_dy: 0.000 m\n") + f.write("datum_shift_dz: 0.000 m\n") + f.write("datum_scale_m: 0.00000e+00\n") + f.write("datum_rotation_alpha: 0.00000e+00 arc-sec\n") + f.write("datum_rotation_beta: 0.00000e+00 arc-sec\n") + f.write("datum_rotation_gamma: 0.00000e+00 arc-sec\n") + f.write("datum_country_list Global Definition, WGS84, World\n") + f.write("\n") + + +# ========================= FABDEM 瓦片索引逻辑 ========================= + +def parse_fabdem_zip_name(zip_name): + """解析 FABDEM ZIP 文件名，提取覆盖的经纬度范围 + + 示例: N30E120-N40E130_FABDEM_V1-2.zip → (lat_min=30, lon_min=120, lat_max=40, lon_max=130) + S10W080-S00W070_FABDEM_V1-2.zip → (lat_min=-10, lon_min=-80, lat_max=0, lon_max=-70) + """ + basename = Path(zip_name).stem # N30E120-N40E130_FABDEM_V1-2 + match = re.match( + r'([NS])(\d+)([EW])(\d+)-([NS])(\d+)([EW])(\d+)_FABDEM', + basename + ) + if not match: + return None + + lat1 = int(match.group(2)) * (1 if match.group(1) == 'N' else -1) + lon1 = int(match.group(4)) * (1 if match.group(3) == 'E' else -1) + lat2 = int(match.group(6)) * (1 if match.group(5) == 'N' else -1) + lon2 = int(match.group(8)) * (1 if match.group(7) == 'E' else -1) + + return { + 'lat_min': min(lat1, lat2), + 'lat_max': max(lat1, lat2), + 'lon_min': min(lon1, lon2), + 'lon_max': max(lon1, lon2), + } + + +def tile_name_for_coord(lat, lon): + """根据经纬度生成 FABDEM 1°×1° 瓦片文件名 + + 参数 lat, lon 为瓦片左下角整数坐标 + 示例: (31, 121) → 'N31E121_FABDEM_V1-2.tif' + (-1, -70) → 'S01W070_FABDEM_V1-2.tif' + """ + lat_prefix = 'N' if lat >= 0 else 'S' + lon_prefix = 'E' if lon >= 0 else 'W' + return f"{lat_prefix}{abs(lat):02d}{lon_prefix}{abs(lon):03d}_FABDEM_V1-2.tif" + + +def find_needed_tiles(west, south, east, north): + """计算覆盖目标区域所需的全部 1°×1° 瓦片坐标列表 + + 返回: [(lat, lon), ...] 瓦片左下角坐标 + """ + # 向下取整到最近的整数度 + lat_start = math.floor(south) + lat_end = math.ceil(north) # 不含 + lon_start = math.floor(west) + lon_end = math.ceil(east) + + tiles = [] + for lat in range(lat_start, lat_end): + for lon in range(lon_start, lon_end): + tiles.append((lat, lon)) + return tiles + + +def find_zip_for_tile(lat, lon, fabdem_dir): + """查找包含指定 1°×1° 瓦片的 ZIP 文件 + + FABDEM ZIP 按 10°×10° 分块，文件名编码了覆盖范围 + """ + fabdem_path = Path(fabdem_dir) + zip_files = sorted(fabdem_path.glob('*_FABDEM_V1-2.zip')) + + for zf in zip_files: + bounds = parse_fabdem_zip_name(zf.name) + if bounds is None: + continue + if (bounds['lat_min'] <= lat < bounds['lat_max'] and + bounds['lon_min'] <= lon < bounds['lon_max']): + return zf + return None + + +def extract_tiles_from_zips(tile_coords, fabdem_dir, extract_dir): + """从 FABDEM ZIP 文件中提取所需的 1°×1° GeoTIFF 瓦片 + + 参数: + tile_coords: [(lat, lon), ...] 需要提取的瓦片坐标 + fabdem_dir: FABDEM ZIP 文件所在目录 + extract_dir: 解压目标目录 + + 返回: + 提取成功的 GeoTIFF 文件路径列表 + """ + extract_path = Path(extract_dir) + extract_path.mkdir(parents=True, exist_ok=True) + + # 按 ZIP 文件分组，避免重复打开同一个 ZIP + zip_to_tiles = {} + missing_tiles = [] + + for lat, lon in tile_coords: + zip_file = find_zip_for_tile(lat, lon, fabdem_dir) + if zip_file is None: + tile_name = tile_name_for_coord(lat, lon) + print(f" ⚠ 未找到包含 {tile_name} 的 ZIP 文件 (lat={lat}, lon={lon})") + missing_tiles.append((lat, lon)) + continue + + zip_key = str(zip_file) + if zip_key not in zip_to_tiles: + zip_to_tiles[zip_key] = [] + zip_to_tiles[zip_key].append((lat, lon)) + + # 逐个 ZIP 文件提取 + extracted_files = [] + for zip_path, coords in zip_to_tiles.items(): + zip_name = Path(zip_path).name + print(f" 📦 从 {zip_name} 提取 {len(coords)} 个瓦片...") + + try: + with zipfile.ZipFile(zip_path, 'r') as zf: + zip_contents = zf.namelist() + + for lat, lon in coords: + tile_name = tile_name_for_coord(lat, lon) + + if tile_name in zip_contents: + zf.extract(tile_name, extract_dir) + out_file = extract_path / tile_name + if out_file.exists() and out_file.stat().st_size > 0: + extracted_files.append(str(out_file)) + print(f" ✓ {tile_name}") + else: + print(f" ✗ {tile_name} 提取后为空") + else: + # 海洋区域可能没有对应瓦片（正常） + print(f" - {tile_name} 不在 ZIP 中（可能是海洋区域）") + except Exception as e: + print(f" ✗ 打开 {zip_name} 失败: {e}") + + if missing_tiles: + print(f"\n ⚠ 共 {len(missing_tiles)} 个瓦片未找到对应的 ZIP 文件") + + return extracted_files + + +# ========================= 瓦片拼接 + GAMMA 格式转换（一步完成） ========================= + +def _parse_gdalinfo(gdalinfo_text): + """从 gdalinfo 输出解析元数据""" + width = nlines = None + corner_lon = corner_lat = post_lon = post_lat = None + for line in gdalinfo_text.splitlines(): + if 'Size is' in line: + parts = line.split('Size is')[1].strip().split(',') + width = int(parts[0].strip()) + nlines = int(parts[1].strip()) + elif 'Origin =' in line: + parts = line.split('(')[1].split(')')[0].split(',') + corner_lon = float(parts[0].strip()) + corner_lat = float(parts[1].strip()) + elif 'Pixel Size =' in line: + parts = line.split('(')[1].split(')')[0].split(',') + post_lon = float(parts[0].strip()) + post_lat = float(parts[1].strip()) + return width, nlines, corner_lon, corner_lat, post_lon, post_lat + + +def tiles_to_gamma_dem(tif_files, output_name, west, south, east, north, byteorder='big'): + """从多个 GeoTIFF 瓦片生成 GAMMA 格式 DEM (.dem + .dem.par) + + 流程: srtm2dem 逐个转换 → mosaic 合并（全部使用 GAMMA 原生工具） + + 参数: + tif_files: GeoTIFF 瓦片文件路径列表 + output_name: 输出文件名前缀（不含扩展名） + west, south, east, north: 裁剪范围（度） + byteorder: 字节序（未使用，srtm2dem 自动处理大端） + 返回: + (dem_file, dem_par_file) 或 (None, None) + """ + if not tif_files: + print(" ✗ 没有可用的瓦片文件") + return None, None + + Path(output_name).parent.mkdir(parents=True, exist_ok=True) + dem_file = output_name + '.dem' + dem_par_file = output_name + '.dem.par' + + # 临时目录放 /tmp/（本地盘），srtm2dem 单瓦片很快 + import tempfile as _tf + _tmp_dir = _tf.mkdtemp(prefix='gamma_dem_conv_') + + # ---- 步骤1: srtm2dem 逐个将 GeoTIFF 转为 GAMMA 格式 ---- + n = len(tif_files) + print(f" [1/2] srtm2dem 转换 {n} 个瓦片...") + gamma_tiles = [] # [(dem_path, dem_par_path), ...] + + for i, tif in enumerate(tif_files): + tile_name = Path(tif).stem + tile_dem = os.path.join(_tmp_dir, f'{tile_name}.dem') + tile_par = os.path.join(_tmp_dir, f'{tile_name}.dem.par') + + # gflg=3: 不做大地水准面校正, NODATA 替换为 0.0 + cmd = f'srtm2dem {tif} {tile_dem} {tile_par} 3' + ret = os.system(cmd + ' > /dev/null 2>&1') + + if ret == 0 and os.path.exists(tile_dem) and os.path.getsize(tile_dem) > 0: + gamma_tiles.append((tile_dem, tile_par)) + print(f" ✓ [{i+1}/{n}] {tile_name}") + else: + print(f" ✗ [{i+1}/{n}] {tile_name} srtm2dem 失败") + + if not gamma_tiles: + print(" ✗ 所有瓦片转换失败") + shutil.rmtree(_tmp_dir, ignore_errors=True) + return None, None + + print(f" 成功转换 {len(gamma_tiles)}/{n} 个瓦片") + + # ---- 步骤2: mosaic 合并所有 GAMMA DEM 瓦片 ---- + if len(gamma_tiles) == 1: + # 只有一个瓦片，直接复制 + shutil.copy2(gamma_tiles[0][0], dem_file) + shutil.copy2(gamma_tiles[0][1], dem_par_file) + print(f" [2/2] 单瓦片，直接输出") + else: + print(f" [2/2] mosaic 合并 {len(gamma_tiles)} 个 GAMMA DEM...") + # 构建 mosaic 命令: mosaic nfiles dem1 par1 dem2 par2 ... dem_out par_out mode format + cmd_parts = ['mosaic', str(len(gamma_tiles))] + for tile_dem, tile_par in gamma_tiles: + cmd_parts.append(tile_dem) + cmd_parts.append(tile_par) + cmd_parts.extend([dem_file, dem_par_file, '1', '0']) + # mode=1: 重叠区取平均, format=0: FLOAT + + cmd_str = ' '.join(cmd_parts) + ret = os.system(cmd_str) + + if ret != 0 or not os.path.exists(dem_file) or os.path.getsize(dem_file) == 0: + print(f" ✗ mosaic 失败 (exit={ret})") + shutil.rmtree(_tmp_dir, ignore_errors=True) + return None, None + + # 清理临时目录 + shutil.rmtree(_tmp_dir, ignore_errors=True) + + size_mb = os.path.getsize(dem_file) / (1024 * 1024) + print(f" ✓ {dem_file} ({size_mb:.1f} MB)") + print(f" ✓ {dem_par_file}") + + return dem_file, dem_par_file + + +# ========================= 主流程 ========================= + +INTRODUCTION = ''' +================================================================================ + make_local_dem.py — 从本地 FABDEM 瓦片库生成 GAMMA 格式 DEM + + 功能: + 1. 根据经纬度范围自动查找所需的 FABDEM 1°×1° 瓦片 + 2. 从 10°×10° ZIP 包中提取瓦片 + 3. 使用 GDAL 拼接并裁剪到目标范围 + 4. 转换为 GAMMA 格式 (.dem + .dem.par) + + 输出与 makedem.py 完全一致，可直接用于 generate_rdc_dem.py +================================================================================ +''' + +EXAMPLE = """ +用法: + # 方式1: 指定经纬度范围 + make_local_dem.py -r 120/123/30/32 -f /mnt/ZYD/全球FABDEM -o output_dem + + # 方式2: 从 SLC 参数文件自动确定范围 + make_local_dem.py -s master.slc.par -f /mnt/ZYD/全球FABDEM -o output_dem + + # 方式3: 从 PyINT 模板文件读取（集成到工作流） + make_local_dem.py --template shanghaiT171F128S1A -f /mnt/ZYD/全球FABDEM +""" + + +def cmdLineParse(): + parser = argparse.ArgumentParser( + description='从本地 FABDEM 瓦片库生成 GAMMA 格式 DEM', + formatter_class=argparse.RawTextHelpFormatter, + epilog=INTRODUCTION + '\n' + EXAMPLE + ) + + parser.add_argument('-r', dest='region', + help='研究区域范围: west/east/south/north (如: 120/123/30/32)') + parser.add_argument('-s', dest='slc_par', + help='SLC 参数文件路径（自动从中提取研究区域范围）') + parser.add_argument('-f', '--fabdem-dir', dest='fabdem_dir', required=True, + help='本地 FABDEM ZIP 文件目录 (如: /mnt/ZYD/全球FABDEM)') + parser.add_argument('-o', '--output', dest='output_name', default=None, + help='输出文件名前缀（不含扩展名）[默认: 工作目录/out]') + parser.add_argument('--dir', dest='work_dir', default=None, + help='工作目录 [默认: 当前目录]') + parser.add_argument('--template', dest='template_name', default=None, + help='PyINT 项目名（从模板文件读取 DEM 路径和 SLC 位置）') + parser.add_argument('--byteorder', dest='byteorder', choices=['big', 'little'], + default='big', + help='输出 DEM 字节序 [默认: big (GAMMA 标准)]') + parser.add_argument('--margin', dest='margin', type=float, default=1.0, + help='在 SLC 覆盖范围外扩展的余量（度）[默认: 1.0]') + + return parser.parse_args() + + +def get_region_from_slc_par(slc_par_file): + """从 GAMMA SLC 参数文件提取研究区域范围""" + corners_txt = 'corners_tmp.txt' + call_str = f"SLC_corners {slc_par_file} > {corners_txt}" + os.system(call_str) + + if not os.path.isfile(corners_txt): + print(f"✗ SLC_corners 执行失败") + return None + + with open(corners_txt, 'r') as f: + lines = f.readlines() + + os.remove(corners_txt) + + # 解析 SLC_corners 输出（第 9-10 行包含 lat/lon 范围） + try: + lat_line = lines[8] + lon_line = lines[9] + min_lat = float(lat_line.split(':')[1].split('max.')[0].strip()) + max_lat = float(lat_line.split(':')[2].strip()) + min_lon = float(lon_line.split(':')[1].split('max.')[0].strip()) + max_lon = float(lon_line.split(':')[2].strip()) + return min_lon, min_lat, max_lon, max_lat + except (IndexError, ValueError) as e: + print(f"✗ 解析 SLC_corners 输出失败: {e}") + return None + + +def get_region_from_template(project_name, margin=1.0): + """从 PyINT 模板文件获取区域范围（通过 master SLC 参数文件）""" + try: + from pyint import _utils as ut + except ImportError: + print("✗ 无法导入 pyint._utils，请确保 PyINT 在 PYTHONPATH 中") + return None, None + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + if not scratchDir or not templateDir: + print("✗ 环境变量 SCRATCHDIR 或 TEMPLATEDIR 未设置") + return None, None + + template_file = Path(templateDir) / f"{project_name}.template" + if not template_file.exists(): + print(f"✗ 模板文件不存在: {template_file}") + return None, None + + template_dict = ut.update_template(str(template_file)) + master_date = template_dict['masterDate'] + slc_dir = Path(scratchDir) / project_name / 'SLC' + master_slc_par = slc_dir / master_date / f"{master_date}.slc.par" + + if not master_slc_par.exists(): + print(f"✗ Master SLC 参数文件不存在: {master_slc_par}") + return None, None + + region = get_region_from_slc_par(str(master_slc_par)) + if region is None: + return None, None + + west, south, east, north = region + # 外扩 margin 度并取整 + west = math.floor(west - margin) + south = math.floor(south - margin) + east = math.ceil(east + margin) + north = math.ceil(north + margin) + + # 确定输出路径（使用模板中 DEM 路径或默认路径） + dem_dir = os.getenv('DEMDIR', '') + if 'DEM' in template_dict and template_dict['DEM'].strip(): + output_name = template_dict['DEM'].replace('.dem', '') + else: + dem_out_dir = Path(dem_dir) / project_name + dem_out_dir.mkdir(parents=True, exist_ok=True) + output_name = str(dem_out_dir / project_name) + + return (west, south, east, north), output_name + + +def main(): + args = cmdLineParse() + + # 确定工作目录 + if args.work_dir: + work_dir = Path(args.work_dir) + else: + work_dir = Path.cwd() + work_dir.mkdir(parents=True, exist_ok=True) + + # 确定 FABDEM 目录 + fabdem_dir = Path(args.fabdem_dir) + if not fabdem_dir.is_dir(): + print(f"✗ FABDEM 目录不存在: {fabdem_dir}") + sys.exit(1) + + zip_count = len(list(fabdem_dir.glob('*_FABDEM_V1-2.zip'))) + print(f"✓ FABDEM 目录: {fabdem_dir} ({zip_count} 个 ZIP 文件)") + + # 确定研究区域 + output_name = args.output_name + region = None + + if args.template_name: + # 从 PyINT 模板获取 + result = get_region_from_template(args.template_name, args.margin) + if result[0] is not None: + region = result[0] + if output_name is None: + output_name = result[1] + else: + print("✗ 无法从模板文件获取区域范围") + sys.exit(1) + + elif args.slc_par: + # 从 SLC 参数文件获取 + slc_region = get_region_from_slc_par(args.slc_par) + if slc_region is None: + sys.exit(1) + west, south, east, north = slc_region + west = math.floor(west - args.margin) + south = math.floor(south - args.margin) + east = math.ceil(east + args.margin) + north = math.ceil(north + args.margin) + region = (west, south, east, north) + + elif args.region: + # 从命令行参数解析 + parts = args.region.split('/') + if len(parts) != 4: + print("✗ 区域格式错误，应为: west/east/south/north") + sys.exit(1) + west, east, south, north = [float(x) for x in parts] + region = (west, south, east, north) + + else: + print("✗ 必须指定研究区域: -r, -s 或 --template") + sys.exit(1) + + if output_name is None: + output_name = str(work_dir / 'out') + + # 确保输出目录存在 + Path(output_name).parent.mkdir(parents=True, exist_ok=True) + + west, south, east, north = region + + print(f"\n{'='*70}") + print(f" 从本地 FABDEM 生成 GAMMA DEM") + print(f"{'='*70}") + print(f" 区域范围: {west}°E ~ {east}°E, {south}°N ~ {north}°N") + print(f" FABDEM 目录: {fabdem_dir}") + print(f" 输出文件: {output_name}.dem / {output_name}.dem.par") + print(f" 字节序: {args.byteorder}") + print(f"{'='*70}\n") + + # 1. 计算所需瓦片 + tiles = find_needed_tiles(west, south, east, north) + print(f"[1/4] 需要 {len(tiles)} 个 1°×1° 瓦片\n") + + # 2. 从 ZIP 中提取瓦片 + print(f"[2/4] 从 FABDEM ZIP 文件中提取瓦片...") + temp_dir = tempfile.mkdtemp(prefix='fabdem_tiles_') + try: + tif_files = extract_tiles_from_zips(tiles, str(fabdem_dir), temp_dir) + + if not tif_files: + print("\n✗ 没有提取到任何瓦片文件，请检查 FABDEM 目录和区域范围") + shutil.rmtree(temp_dir, ignore_errors=True) + sys.exit(1) + + print(f"\n 共提取 {len(tif_files)}/{len(tiles)} 个瓦片\n") + + # 3-4. 一步完成: VRT → ENVI 二进制 → GAMMA .dem + print(f"[3/4] 拼接裁剪 + 转换 GAMMA 格式...") + dem_file, dem_par_file = tiles_to_gamma_dem( + tif_files, output_name, west, south, east, north, args.byteorder + ) + + if dem_file is None: + print("\n✗ DEM 生成失败") + shutil.rmtree(temp_dir, ignore_errors=True) + sys.exit(1) + + finally: + # 清理临时目录 + shutil.rmtree(temp_dir, ignore_errors=True) + + # 完成 + print(f"\n{'='*70}") + print(f" ✓ FABDEM → GAMMA DEM 转换完成!") + print(f"{'='*70}") + print(f" DEM 文件: {dem_file}") + print(f" 参数文件: {dem_par_file}") + print(f" 字节序: {args.byteorder} endian") + print(f" 可直接用于: generate_rdc_dem.py") + print(f"{'='*70}\n") + + +if __name__ == '__main__': + main() diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/makedem.py b/.codex_tmp/pyint_variants/no_rescue/pyint/makedem.py new file mode 100644 index 0000000..38a211f --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/makedem.py @@ -0,0 +1,1486 @@ +#! /usr/bin/env python +import os +import sys +import numpy as np +import argparse +import subprocess +import glob +from skimage import io + +# 尝试导入 NASADEM 库 +try: + from NASADEM import NASADEM + from rasters import RasterGrid + HAS_NASADEM = True +except ImportError: + HAS_NASADEM = False + print("Warning: NASADEM library not installed. NASADEM download will not be available.") + print("Install with: pip install nasadem rasters") + +# 尝试导入 srtm 库 +try: + import srtm + HAS_SRTM = True +except ImportError: + HAS_SRTM = False + print("Warning: srtm library not installed. SRTM download will not be available.") + print("Install with: pip install srtm (requires Python >= 3.12)") + print(f"Current Python version: {sys.version_info.major}.{sys.version_info.minor}") + +resolutions = 30 # 90 + +def write_demrsc_file(FILE, Corner_LON, Corner_LAT, X_STEP, Y_STEP, WIDTH, LENGTH): + """Write ROI_PAC DEM resource file (.dem.rsc) + + Args: + FILE (str): Output resource file path + Corner_LON (str): Corner longitude + Corner_LAT (str): Corner latitude + X_STEP (str): Longitude step size + Y_STEP (str): Latitude step size + WIDTH (str): Number of columns + LENGTH (str): Number of lines + """ + f = open(FILE, 'w') + f.write('DATE12 111111-222222\n') + f.write('FILE_LENGTH ' + str(int(LENGTH)) + '\n') + f.write('FILE_TYPE .dem\n') + f.write('PROCESSOR roipac\n') + f.write('PROJECTION LATLON\n') + f.write('RLOOKS 1\n') + f.write('WIDTH ' + str(int(WIDTH)) + '\n') + f.write('XMAX ' + str(int(int(WIDTH)-1)) + '\n') + f.write('XMIN 0\n') + f.write('X_FIRST ' + str(float(Corner_LON)) + '\n') + f.write('X_STEP ' + str(float(X_STEP)) + '\n') + f.write('X_UNIT degrees\n') + f.write('YMAX ' + str(int(int(LENGTH)-1)) + '\n') + f.write('YMIN 0\n') + f.write('Y_FIRST ' + str(float(Corner_LAT)) + '\n') + f.write('Y_STEP ' + str(float(Y_STEP)) + '\n') + f.write('Y_UNIT degrees\n') + f.write('Z_OFFSET 0\n') + f.write('Z_SCALE 1\n') + f.close() + +def write_dempar_file(FILE, Corner_LON, Corner_LAT, X_STEP, Y_STEP, WIDTH, LENGTH, DATA_FORMAT): + """Write Gamma DEM parameter file (.dem.par) + + Args: + FILE (str): Output parameter file path + Corner_LON (str): Corner longitude + Corner_LAT (str): Corner latitude + X_STEP (str): Longitude step size + Y_STEP (str): Latitude step size + WIDTH (str): Number of columns + LENGTH (str): Number of lines + DATA_FORMAT (str): Data format (INTEGER*2 or REAL*4) + """ + DEM_TYPE = 'Copernicus30' + Proj = 'EQA' + f = open(FILE, 'w') + f.write("Gamma DIFF&GEO DEM/MAP parameter file\n") + f.write("title:\tIMPORTED DEM FROM %s\n" % DEM_TYPE) + f.write("DEM_projection: %s\n" % Proj) + f.write("data_format: %s\n" % DATA_FORMAT) + f.write("DEM_hgt_offset: 0.00000\n") + f.write("DEM_scale: 1.00000\n") + f.write("width: %s\n" % WIDTH) + f.write("nlines: %s\n" % LENGTH) + f.write("corner_lat: %s decimal degrees\n" % Corner_LAT) + f.write("corner_lon: %s decimal degrees\n" % Corner_LON) + f.write("post_lat: %s decimal degrees\n" % Y_STEP) + f.write("post_lon: %s decimal degrees\n" % X_STEP) + f.write("\n") + f.write("ellipsoid_name: WGS 84\n") + f.write("ellipsoid_ra: 6378137.000 m\n") + f.write("ellipsoid_reciprocal_flattening: 298.2572236\n") + f.write("\n") + f.write("datum_name: WGS 1984\n") + f.write("datum_shift_dx: 0.000 m\n") + f.write("datum_shift_dy: 0.000 m\n") + f.write("datum_shift_dz: 0.000 m\n") + f.write("datum_scale_m: 0.00000e+00\n") + f.write("datum_rotation_alpha: 0.00000e+00 arc-sec\n") + f.write("datum_rotation_beta: 0.00000e+00 arc-sec\n") + f.write("datum_rotation_gamma: 0.00000e+00 arc-sec\n") + f.write("datum_country_list Global Definition, WGS84, World\n") + f.write("\n") + f.close() + +def convert_to_gamma(input_file, output_name, byteorder='big', processor='gamma'): + """Convert TIF file to Gamma or ROI_PAC format (.dem and .dem.par/.dem.rsc) + + Args: + input_file (str): Input TIF file path + output_name (str): Output name (without extension) + byteorder (str): Byte order ('big' or 'little') + processor (str): Processor type ('gamma' or 'roi_pac') + """ + processor_name = 'Gamma' if processor == 'gamma' else 'ROI_PAC' + print(f"\n开始转换到 {processor_name} 格式: {input_file}") + + # 读取 DEM 数据 - 优先使用 GDAL（更稳定），然后是 rasterio + dem_data = None + + # 方法1: 使用 GDAL（最稳定） + try: + from osgeo import gdal + print(" 尝试使用 GDAL 读取数据...") + ds = gdal.Open(input_file) + if ds is not None: + band = ds.GetRasterBand(1) + dem_data = band.ReadAsArray() + print(f" ✓ 使用 GDAL 读取成功") + ds = None # 关闭文件 + except Exception as e: + print(f" ✗ GDAL 读取失败: {e}") + + # 方法2: 如果GDAL失败，尝试rasterio + if dem_data is None: + try: + import rasterio + print(" 尝试使用 rasterio 读取数据...") + with rasterio.open(input_file) as src: + dem_data = src.read(1) + print(f" ✓ 使用 rasterio 读取成功") + except Exception as e: + print(f" ✗ rasterio 读取失败: {e}") + + # 方法3: 最后尝试skimage + if dem_data is None: + try: + print(" 尝试使用 skimage 读取数据...") + dem_data = io.imread(input_file) + print(f" ✓ 使用 skimage 读取成功") + except Exception as e: + print(f" ✗ skimage 读取失败: {e}") + + # 如果所有方法都失败 + if dem_data is None: + raise ValueError(f"无法读取DEM文件: {input_file}，所有读取方法都失败了") + + # 确定数据格式 + if dem_data.dtype == 'float32': + DATA_FORMAT = 'REAL*4' + else: + DATA_FORMAT = 'INTEGER*2' + + # 字节序转换 + if not sys.byteorder == byteorder: + dem_data.byteswap(True) + + # 输出文件路径 + dem_file = output_name + '.dem' + if processor == 'gamma': + dem_par_file = output_name + '.dem.par' + else: + dem_par_file = output_name + '.dem.rsc' + + # 写入二进制 DEM 数据 + dem_data.tofile(dem_file) + + # 使用 gdalinfo 获取地理信息 + info_file = 'temp_gdalinfo.txt' + cmd = f"gdalinfo {input_file} > {info_file}" + os.system(cmd) + + # 解析地理信息 + Corner_LON = None + Corner_LAT = None + Post_LON = None + Post_LAT = None + WIDTH = None + FILE_LENGTH = None + + with open(info_file, 'r') as f: + for line in f: + if 'Origin =' in line: + parts = line.split('Origin =')[1].strip().split('(')[1].split(')')[0].split(',') + Corner_LON = parts[0] + Corner_LAT = parts[1] + elif 'Pixel Size ' in line: + parts = line.split('Pixel Size =')[1].strip().split('(')[1].split(')')[0].split(',') + Post_LON = parts[0] + Post_LAT = parts[1] + elif 'Size is' in line: + parts = line.split('Size is')[1].strip().split(',') + WIDTH = parts[0] + FILE_LENGTH = parts[1] + + # 删除临时文件 + if os.path.exists(info_file): + os.remove(info_file) + + # 写入参数文件 + if processor == 'gamma': + write_dempar_file(dem_par_file, Corner_LON, Corner_LAT, Post_LON, Post_LAT, WIDTH, FILE_LENGTH, DATA_FORMAT) + else: + write_demrsc_file(dem_par_file, Corner_LON, Corner_LAT, Post_LON, Post_LAT, WIDTH, FILE_LENGTH) + + print(f"{byteorder} endian {dem_file} and {dem_par_file} are generated.") + print(f"{processor_name} 格式转换完成!") + +def get_sufix(STR): + """Get file extension""" + n = len(STR.split('.')) + SUFIX = STR.split('.')[n-1] + return SUFIX + +def read_region(STR): + """Parse region string 'west/east/south/north'""" + WEST = STR.split('/')[0] + EAST = STR.split('/')[1].split('/')[0] + SOUTH = STR.split(EAST+'/')[1].split('/')[0] + NORTH = STR.split(EAST+'/')[1].split('/')[1] + WEST = float(WEST) + SOUTH = float(SOUTH) + EAST = float(EAST) + NORTH = float(NORTH) + return WEST, SOUTH, EAST, NORTH + +def cmd_init(lon, lat, save_path): + s_lon = str(abs(lon)) + s_lat = str(abs(lat)) + if abs(lon) < 10: + s_lon = "00" + str(abs(lon)) + elif abs(lon) < 100: + s_lon = "0" + str(abs(lon)) + if abs(lat) < 10: + s_lat = "0" + str(abs(lat)) + if lon < 0: + c_lon = "W" + str(s_lon) + else: + c_lon = "E" + str(s_lon) + if lat < 0: + c_lat = "S" + str(s_lat) + else: + c_lat = "N" + str(s_lat) + cmd = "aws s3 cp --no-sign-request" + " s3://copernicus-dem-{0}m/Copernicus_DSM_COG_{1}_{2}_00_{3}_00_DEM/ {4} --recursive".format( + str(resolutions), str(int(resolutions / 3)), str(c_lat), str(c_lon), save_path) + return cmd + +def get_remote_file(lon, lat, save_path, max_retries=3): + """Get Copernicus Dem by lon and lat with retry mechanism + + Args: + lon (number): lontitude + lat (number): latitude + save_path (str): directory to save data + max_retries (int): maximum retry attempts + """ + lon = int(lon) + lat = int(lat) + cmd = cmd_init(lon, lat, save_path) + + for attempt in range(max_retries): + try: + result = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=300) + if result.returncode == 0: + print(f"✓ Successfully downloaded: lon={lon}, lat={lat}") + return True + else: + if attempt < max_retries - 1: + print(f"✗ Failed (attempt {attempt+1}/{max_retries}): lon={lon}, lat={lat}, retrying...") + import time + time.sleep(2) + else: + print(f"✗ Failed after {max_retries} attempts: lon={lon}, lat={lat}") + return False + except subprocess.TimeoutExpired: + if attempt < max_retries - 1: + print(f"⏱ Timeout (attempt {attempt+1}/{max_retries}): lon={lon}, lat={lat}, retrying...") + import time + time.sleep(2) + else: + print(f"✗ Timeout after {max_retries} attempts: lon={lon}, lat={lat}") + return False + except Exception as e: + print(f"✗ Exception occurred for lon={lon}, lat={lat}: {e}") + return False + + return False + +def get_remote_file_batch(west, east, south, north, save_path, num_workers=4): + """Get Copernicus Dems by WESN extent with parallel download + + Args: + west (number): western longitude boundary + east (number): eastern longitude boundary + south (number): southern latitude boundary + north (number): northern latitude boundary + save_path (str): directory to save data + num_workers (int): number of parallel download workers (default: 4) + """ + from concurrent.futures import ThreadPoolExecutor, as_completed + from tqdm import tqdm + + west = int(west) + east = int(east) + south = int(south) + north = int(north) + + # 生成所有需要下载的坐标对 + coords = [(lon, lat) for lon in range(west, east + 1) for lat in range(south, north + 1)] + total_tiles = len(coords) + + print(f"\n{'='*80}") + print(f"并行下载 Copernicus DEM 数据") + print(f"{'='*80}") + print(f"区域范围: {west}°E - {east}°E, {south}°N - {north}°N") + print(f"总瓦片数: {total_tiles}") + print(f"并行线程: {num_workers}") + print(f"{'='*80}\n") + + success_count = 0 + failed_count = 0 + + # 使用线程池并行下载 + with ThreadPoolExecutor(max_workers=num_workers) as executor: + # 提交所有下载任务 + future_to_coord = {executor.submit(get_remote_file, lon, lat, save_path): (lon, lat) + for lon, lat in coords} + + # 使用 tqdm 显示进度条 + with tqdm(total=total_tiles, desc="下载进度", unit="瓦片") as pbar: + for future in as_completed(future_to_coord): + coord = future_to_coord[future] + try: + result = future.result() + if result: + success_count += 1 + else: + failed_count += 1 + except Exception as e: + print(f"\n✗ 下载失败 {coord}: {e}") + failed_count += 1 + + pbar.update(1) + + # 打印统计信息 + print(f"\n{'='*80}") + print(f"下载完成统计") + print(f"{'='*80}") + print(f"成功: {success_count}/{total_tiles} 瓦片") + print(f"失败: {failed_count}/{total_tiles} 瓦片") + if failed_count > 0: + print(f"\n⚠️ 有 {failed_count} 个瓦片下载失败,可能会影响 DEM 完整性") + print(f"{'='*80}\n") + +def download_nasadem(west, south, east, north, save_path, cell_size_deg=0.000277778): + """Download NASADEM data using NASADEM library + + Args: + west (float): western longitude boundary + south (float): southern latitude boundary + east (float): eastern longitude boundary + north (float): northern latitude boundary + save_path (str): directory to save data + cell_size_deg (float): cell size in degrees (default: 0.000277778 ≈ 30m at equator) + + Returns: + str: path to the downloaded DEM file, or None if failed + """ + if not HAS_NASADEM: + print("✗ NASADEM library not available. Cannot download NASADEM.") + print(" Install with: pip install nasadem rasters") + return None + + print(f"\n{'='*80}") + print(f"下载 NASADEM 数据") + print(f"{'='*80}") + print(f"区域范围: {west}°E - {east}°E, {south}°N - {north}°N") + print(f"分辨率: {cell_size_deg}° (约 {cell_size_deg * 111000:.0f} 米)") + print(f"{'='*80}\n") + + try: + from NASADEM.NASADEM import NASADEMConnection + from rasters import RasterGrid + import time + + start_time = time.time() + + # 定义目标区域 + geometry = RasterGrid.from_bbox( + xmin=west, ymin=south, xmax=east, ymax=north, + cell_size=cell_size_deg, + crs="EPSG:4326" + ) + + print(f"RasterGrid 维度: {geometry.shape}") + print(f"正在从NASA服务器下载高程数据...") + print(f"提示: 首次下载需要从NASA服务器获取数据,可能需要几分钟时间...") + + # 创建NASA DEM连接并获取高程数据 + conn = NASADEMConnection() + elevation = conn.elevation_m(geometry) + + elapsed_time = time.time() - start_time + + print(f"\n✓ 下载完成!") + print(f" 高程范围: {float(elevation.min()):.1f} 到 {float(elevation.max()):.1f} 米") + print(f" 数据维度: {elevation.shape}") + print(f" 用时: {elapsed_time:.1f} 秒") + + # 保存为 GeoTIFF + output_file = os.path.join(save_path, f"NASADEM_{west}_{east}_{south}_{north}.tif") + print(f"\n保存到: {output_file}") + + # 检查elevation对象是否有save方法 + if hasattr(elevation, 'save'): + elevation.save(output_file) + print(f"✓ 文件保存成功: {output_file}") + if os.path.exists(output_file): + file_size = os.path.getsize(output_file) / (1024 * 1024) + print(f" 文件大小: {file_size:.2f} MB") + return output_file + + # 否则使用 rasterio 保存 + try: + import rasterio + from rasterio.crs import CRS + from rasterio.transform import from_bounds + + # 获取数据数组 + if hasattr(elevation, 'read'): + elev_array = elevation.read() + else: + elev_array = np.array(elevation) + + # 计算变换矩阵 + height, width = elev_array.shape + transform = from_bounds(west, south, east, north, width, height) + + # 写入文件 + with rasterio.open( + output_file, + 'w', + driver='GTiff', + height=height, + width=width, + count=1, + dtype=elev_array.dtype, + crs=CRS.from_epsg(4326), + transform=transform, + nodata=-32767, + compress='LZW' + ) as dst: + dst.write(elev_array, 1) + + print(f"✓ 文件保存成功: {output_file}") + if os.path.exists(output_file): + file_size = os.path.getsize(output_file) / (1024 * 1024) + print(f" 文件大小: {file_size:.2f} MB") + return output_file + + except ImportError: + print("⚠️ rasterio 未安装,尝试使用 GDAL...") + + # 使用 GDAL 创建 GeoTIFF + width, height = geometry.width, geometry.height + gdal_cmd = f"gdal_create -outsize {width} {height} -of GTiff -co COMPRESS=LZW -a_srs EPSG:4326 -a_ullr {west} {north} {east} {south} {output_file}" + result = os.system(gdal_cmd) + + if result == 0 and os.path.exists(output_file): + print(f"✓ 文件创建成功: {output_file}") + return output_file + else: + print(f"✗ 文件创建失败") + return None + + except Exception as e: + print(f"\n✗ 下载失败: {e}") + import traceback + traceback.print_exc() + return None + +def download_srtm(west, south, east, north, save_path, srtm_data_dir=None, cell_size=0.000277778): + """Download SRTM data using srtm library (requires pre-downloaded .hgt files) + + Args: + west (float): western longitude boundary + south (float): southern latitude boundary + east (float): eastern longitude boundary + north (float): northern latitude boundary + save_path (str): directory to save output DEM + srtm_data_dir (str): directory containing SRTM .hgt files (required) + cell_size (float): cell size in degrees (default: 0.000277778 ≈ 30m) + + Returns: + str: path to the generated DEM file, or None if failed + + Note: + This function requires SRTM .hgt files to be pre-downloaded. + You can download SRTM data from: + - https://srtm.csi.cgiar.org/ + - https://earthexplorer.usgs.gov/ + + Python version requirement: + - srtm library requires Python >= 3.12 + - For Python < 3.12, use process_srtm_hgt_files() instead + """ + if not HAS_SRTM: + print("✗ srtm library not available. Cannot use SRTM.") + print(" Install with: pip install srtm (requires Python >= 3.12)") + print(f" Current Python version: {sys.version_info.major}.{sys.version_info.minor}") + print(" Alternative: Use process_srtm_hgt_files() to manually process .hgt files") + return None + + if srtm_data_dir is None or not os.path.isdir(srtm_data_dir): + print("✗ SRTM data directory not provided or does not exist.") + print(" Please download SRTM .hgt files and specify the directory.") + print(" Download sources:") + print(" - https://srtm.csi.cgiar.org/") + print(" - https://earthexplorer.usgs.gov/") + return None + + print(f"\n{'='*80}") + print(f"使用 SRTM 数据生成 DEM") + print(f"{'='*80}") + print(f"区域范围: {west}°E - {east}°E, {south}°N - {north}°N") + print(f"SRTM数据目录: {srtm_data_dir}") + print(f"分辨率: {cell_size}° (约 {cell_size * 111000:.0f} 米)") + print(f"{'='*80}\n") + + try: + import time + from tqdm import tqdm + + start_time = time.time() + + # 创建 SrtmService + service = srtm.SrtmService(srtm_data_dir, cache_size=100) + print(f"✓ SrtmService 创建成功") + + # 生成坐标网格 + lats = np.arange(south, north + cell_size, cell_size) + lons = np.arange(west, east + cell_size, cell_size) + + print(f"生成网格: {len(lats)} × {len(lons)} = {len(lats) * len(lons)} 个点") + + # 批量查询高程 + coords = [(lat, lon) for lat in lats for lon in lons] + print(f"正在查询高程数据...") + + elevations = service.get_elevations_batch(coords, default=0) + + # 重塑为数组 + elevation_array = np.array(elevations, dtype=np.float32).reshape(len(lats), len(lons)) + + # 替换0值为NaN(可能是缺失数据) + elevation_array[elevation_array == 0] = np.nan + + elapsed_time = time.time() - start_time + + print(f"\n✓ 查询完成!") + print(f" 有效数据: {np.sum(~np.isnan(elevation_array))} / {elevation_array.size} 点") + if np.sum(~np.isnan(elevation_array)) > 0: + print(f" 高程范围: {np.nanmin(elevation_array):.1f} 到 {np.nanmax(elevation_array):.1f} 米") + print(f" 用时: {elapsed_time:.1f} 秒") + + # 保存为 GeoTIFF + output_file = os.path.join(save_path, f"SRTM_{west}_{east}_{south}_{north}.tif") + print(f"\n保存到: {output_file}") + + try: + import rasterio + from rasterio.crs import CRS + from rasterio.transform import from_bounds + + height, width = elevation_array.shape + transform = from_bounds(west, south, east, north, width, height) + + # 替换NaN为nodata值 + elevation_array[np.isnan(elevation_array)] = -32767 + + with rasterio.open( + output_file, + 'w', + driver='GTiff', + height=height, + width=width, + count=1, + dtype=np.float32, + crs=CRS.from_epsg(4326), + transform=transform, + nodata=-32767, + compress='LZW' + ) as dst: + dst.write(elevation_array.astype(np.float32), 1) + + print(f"✓ 文件保存成功: {output_file}") + if os.path.exists(output_file): + file_size = os.path.getsize(output_file) / (1024 * 1024) + print(f" 文件大小: {file_size:.2f} MB") + return output_file + + except ImportError: + print("⚠️ rasterio 未安装,无法保存文件") + return None + + except Exception as e: + print(f"\n✗ 生成失败: {e}") + import traceback + traceback.print_exc() + return None + +def split_large_region(west, south, east, north, tile_size=1.0): + """将大区域分割成小块 + + Args: + west, south, east, north: 区域边界 + tile_size: 分块大小（度），默认1°×1° + + Returns: + list: 分块列表 [(west, south, east, north), ...] + """ + tiles = [] + + w = west + while w < east: + e = min(w + tile_size, east) + + s = south + while s < north: + n = min(s + tile_size, north) + tiles.append((w, s, e, n)) + s += tile_size + + w += tile_size + + return tiles + + +def download_srtm_cgiar(west, south, east, north, save_path, api_key=None, num_workers=4, dem_type='SRTMGL1', auto_tile=True): + """Download DEM data from OpenTopography automatically + + Args: + west (float): western longitude boundary + south (float): southern latitude boundary + east (float): eastern longitude boundary + north (float): northern latitude boundary + save_path (str): directory to save output DEM + api_key (str): OpenTopography API key (optional, will prompt if not provided) + num_workers (int): number of parallel downloads (not used for API, kept for compatibility) + dem_type (str): DEM type (default: 'SRTMGL1') + - SRTMGL1: SRTM GL1 30m (recommended for SRTM) + - SRTMGL3: SRTM GL3 90m + - NASADEM: NASADEM 30m + - COP30: Copernicus 30m + - COP90: Copernicus 90m + auto_tile (bool): 自动分块下载大区域 + + Returns: + str: path to the generated DEM file, or None if failed + + Note: + OpenTopography API provides: + - SRTMGL1: SRTM GL1 30m (recommended) + - SRTMGL3: SRTM GL3 90m + - NASADEM: NASADEM Global DEM 30m + - COP30: Copernicus 30m + - COP90: Copernicus 90m + + API Documentation: https://portal.opentopography.org/apidocs/ + Get API key: https://opentopography.org/myOpenTopo + """ + # DEM类型信息 + dem_info = { + 'SRTMGL1': {'resolution': '30m', 'name': 'SRTM GL1'}, + 'SRTMGL3': {'resolution': '90m', 'name': 'SRTM GL3'}, + 'NASADEM': {'resolution': '30m', 'name': 'NASADEM'}, + 'COP30': {'resolution': '30m', 'name': 'Copernicus 30m'}, + 'COP90': {'resolution': '90m', 'name': 'Copernicus 90m'} + } + + info = dem_info.get(dem_type, {'resolution': 'unknown', 'name': dem_type}) + + print(f"\n{'='*80}") + print(f"从 OpenTopography 自动下载 DEM 数据") + print(f"{'='*80}") + print(f"区域范围: {west}°E - {east}°E, {south}°N - {north}°N") + print(f"数据源: {info['name']} ({info['resolution']} resolution)") + print(f"{'='*80}\n") + + # 计算区域面积 + region_area = (east - west) * (north - south) + + # 如果区域较大，自动分块 + if auto_tile and region_area > 4: + print(f"⚠️ 区域较大 ({region_area:.1f} 平方度)，将自动分块下载以提高稳定性") + + # 计算分块 + tiles = split_large_region(west, south, east, north, tile_size=1.0) + print(f" 将分成 {len(tiles)} 个 1°×1° 的区块下载\n") + + # 创建临时目录存放分块文件 + temp_dir = os.path.join(save_path, f"temp_tiles_{dem_type}_{west}_{east}_{south}_{north}") + if not os.path.exists(temp_dir): + os.makedirs(temp_dir) + + # 下载每个分块 + downloaded_files = [] + failed_tiles = [] + + for i, (w, s, e, n) in enumerate(tiles, 1): + print(f"\n{'='*80}") + print(f"下载分块 {i}/{len(tiles)}: {w}°E-{e}°E, {s}°N-{n}°N") + print(f"{'='*80}") + + tile_file = download_single_tile(w, s, e, n, temp_dir, api_key, dem_type, info) + + if tile_file: + downloaded_files.append(tile_file) + print(f"✓ 分块 {i}/{len(tiles)} 下载成功") + else: + failed_tiles.append((w, s, e, n)) + print(f"✗ 分块 {i}/{len(tiles)} 下载失败") + + # 如果有失败的分块 + if failed_tiles: + print(f"\n⚠️ 有 {len(failed_tiles)} 个分块下载失败") + print("失败的分块:") + for w, s, e, n in failed_tiles: + print(f" {w}°E-{e}°E, {s}°N-{n}°N") + + # 合并所有分块 + if downloaded_files: + print(f"\n{'='*80}") + print(f"合并 {len(downloaded_files)} 个分块...") + print(f"{'='*80}\n") + + merged_file = merge_dem_tiles(downloaded_files, save_path, dem_type, west, east, south, north) + + # 清理临时文件 + import shutil + try: + shutil.rmtree(temp_dir) + print("✓ 临时文件已清理") + except: + pass + + if merged_file: + print(f"\n✓ 最终输出文件: {merged_file}") + return merged_file + else: + print("✗ 合并失败") + return None + else: + print("✗ 所有分块下载失败") + return None + + # 小区域直接下载 + else: + return download_single_tile(west, south, east, north, save_path, api_key, dem_type, info) + + +def download_single_tile(west, south, east, north, save_path, api_key, dem_type, dem_info): + """下载单个DEM块 + + Args: + west, south, east, north: 区域边界 + save_path: 保存路径 + api_key: OpenTopography API key + dem_type: DEM类型 + dem_info: DEM信息字典 + + Returns: + str: 下载的文件路径，失败返回None + """ + try: + import time + import urllib.request + import urllib.parse + + start_time = time.time() + + # OpenTopography API endpoint + base_url = "https://portal.opentopography.org/API/globaldem" + + # 如果没有提供API key，提示用户获取 + if api_key is None: + print("⚠️ 需要OpenTopography API key才能下载SRTM数据") + print("\n获取免费API key的步骤:") + print(" 1. 访问: https://opentopography.org/myOpenTopo") + print(" 2. 注册免费账户") + print(" 3. 在 'My Account' 页面请求API key") + print(" 4. 使用 --opentopo-api-key 参数提供API key") + print("\n或者使用其他DEM数据源:") + print(" - Copernicus (推荐): --dem-source copernicus") + print(" - NASADEM: --dem-source nasadem") + return None + + # 构建请求参数 + params = { + 'demtype': dem_type, + 'south': south, + 'north': north, + 'west': west, + 'east': east, + 'outputFormat': 'GTiff', + 'API_Key': api_key + } + + # 构建完整URL + url = base_url + '?' + urllib.parse.urlencode(params) + + # 输出文件名 + output_file = os.path.join(save_path, f"{dem_type}_{west}_{east}_{south}_{north}.tif") + + # 下载文件 - 使用curl以获得更好的稳定性 + max_retries = 3 + retry_count = 0 + download_success = False + + while retry_count < max_retries and not download_success: + try: + retry_count += 1 + print(f"正在从OpenTopography下载数据... (尝试 {retry_count}/{max_retries})") + + # 尝试使用 curl 下载（更稳定） + try: + import subprocess + curl_cmd = ['curl', '-L', '-o', output_file, '-s', '--show-error', url] + result = subprocess.run(curl_cmd, capture_output=True, text=True, timeout=1800) # 30分钟超时 + + if result.returncode != 0: + raise Exception(f"curl下载失败: {result.stderr}") + + print(" ✓ curl下载完成") + download_success = True + + except (subprocess.TimeoutExpired, FileNotFoundError): + # 如果curl不可用或超时，使用urllib + print(" 使用urllib下载...") + urllib.request.urlretrieve(url, output_file) + download_success = True + + except urllib.error.HTTPError as e: + print(f"\n✗ HTTP错误: {e.code} {e.reason}") + if e.code == 401: + print(" API key无效或已过期") + print(" 请获取新的API key: https://opentopography.org/myOpenTopo") + elif e.code == 403: + print(" 访问被拒绝，可能是:") + print(" - 请求区域过大") + print(" - API key配额已用完") + print(" - 需要注册获取免费API key") + if retry_count < max_retries: + print(f" 将在3秒后重试...") + time.sleep(3) + continue + return None + except Exception as e: + print(f"\n✗ 下载失败: {e}") + if retry_count < max_retries: + print(f" 将在3秒后重试...") + time.sleep(3) + continue + return None + + # 检查文件是否有效 + if os.path.exists(output_file): + file_size = os.path.getsize(output_file) + + # 如果文件很小，可能是错误消息 + if file_size < 1000: + with open(output_file, 'r') as f: + error_msg = f.read() + print(f"\n✗ 下载失败: {error_msg}") + os.remove(output_file) + return None + + elapsed_time = time.time() - start_time + + print(f"\n✓ 下载完成!") + print(f" 输出文件: {output_file}") + print(f" 文件大小: {file_size / (1024 * 1024):.2f} MB") + print(f" 用时: {elapsed_time:.1f} 秒") + + # 验证文件完整性 + print("\n验证文件完整性...") + + try: + import rasterio + with rasterio.open(output_file) as src: + # 尝试读取多个位置的数据验证 + print(f" 文件大小: {src.width} x {src.height} 像素") + + # 读取多个测试点 + test_points = [ + (0, 0, min(100, src.height), min(100, src.width)), # 左上角 + (max(0, src.height-100), max(0, src.width-100), src.height, src.width), # 右下角 + (src.height//2-50, src.width//2-50, src.height//2+50, src.width//2+50), # 中间 + ] + + for i, (row_start, col_start, row_end, col_end) in enumerate(test_points): + try: + window = ((row_start, row_end), (col_start, col_end)) + test_data = src.read(1, window=window) + print(f" ✓ 测试点 {i+1}/3 验证成功") + except Exception as e: + print(f" ✗ 测试点 {i+1}/3 验证失败: {e}") + raise + + print(f" ✓ 所有验证点通过，文件完整") + return output_file + + except Exception as e: + print(f" ✗ 文件验证失败: {e}") + print(f" 文件可能已损坏，正在删除...") + try: + os.remove(output_file) + except: + pass + return None + else: + print("✗ 文件下载失败") + return None + + except Exception as e: + print(f"\n✗ 下载失败: {e}") + import traceback + traceback.print_exc() + return None + +def merge_dem_tiles(tile_files, save_path, dem_type, west, east, south, north): + """合并多个DEM分块文件 + + Args: + tile_files: 分块文件列表 + save_path: 保存路径 + dem_type: DEM类型 + west, east, south, north: 最终区域边界 + + Returns: + str: 合并后的文件路径，失败返回None + """ + if not tile_files: + print("✗ 没有文件需要合并") + return None + + try: + import rasterio + from rasterio.merge import merge + import time + + print(f"开始合并 {len(tile_files)} 个分块文件...") + + # 读取所有分块 + src_files_to_mosaic = [] + for tile_file in tile_files: + src = rasterio.open(tile_file) + src_files_to_mosaic.append(src) + + # 合并 + start_time = time.time() + mosaic, out_trans = merge(src_files_to_mosaic) + + # 获取输出元数据 + out_meta = src_files_to_mosaic[0].meta.copy() + out_meta.update({ + "driver": "GTiff", + "height": mosaic.shape[1], + "width": mosaic.shape[2], + "transform": out_trans, + "compress": "lzw" + }) + + # 输出文件名 + output_file = os.path.join(save_path, f"{dem_type}_{west}_{east}_{south}_{north}.tif") + + # 写入合并后的文件 + with rasterio.open(output_file, "w", **out_meta) as dest: + dest.write(mosaic) + + # 关闭所有源文件 + for src in src_files_to_mosaic: + src.close() + + elapsed_time = time.time() - start_time + file_size = os.path.getsize(output_file) / (1024 * 1024) + + print(f"✓ 合并完成!") + print(f" 输出文件: {output_file}") + print(f" 文件大小: {file_size:.2f} MB") + print(f" 用时: {elapsed_time:.1f} 秒") + + return output_file + + except ImportError: + print("✗ rasterio 未安装，无法合并文件") + print(" 安装方法: pip install rasterio") + return None + except Exception as e: + print(f"✗ 合并失败: {e}") + import traceback + traceback.print_exc() + return None + + +def download_srtm_cgiar_old(west, south, east, north, save_path, num_workers=4): + """Download SRTM data from CSI-CGIAR (DEPRECATED - tiles not accessible) + + This function is kept as backup but CSI-CGIAR tile downloads are not reliable. + Use download_srtm_cgiar() with OpenTopography API instead. + """ + print("警告: CSI-CGIAR瓦片下载不可靠，建议使用OpenTopography API") + return None + +def process_srtm_hgt_files(west, south, east, north, srtm_data_dir, save_path, output_name=None): + """Process SRTM .hgt files manually (alternative for Python < 3.12) + + Args: + west (float): western longitude boundary + south (float): southern latitude boundary + east (float): eastern longitude boundary + north (float): northern latitude boundary + srtm_data_dir (str): directory containing SRTM .hgt files + save_path (str): directory to save output DEM + output_name (str): output filename (without extension) + + Returns: + str: path to the generated DEM file, or None if failed + + Note: + SRTM .hgt files are 1°×1° tiles with 1201×1201 pixels (3 arc-second) or + 3601×3601 pixels (1 arc-second). This function reads and merges them. + """ + print(f"\n{'='*80}") + print(f"手动处理 SRTM .hgt 文件") + print(f"{'='*80}") + print(f"区域范围: {west}°E - {east}°E, {south}°N - {north}°N") + print(f"SRTM数据目录: {srtm_data_dir}") + print(f"{'='*80}\n") + + try: + import rasterio + from rasterio.merge import merge + from rasterio.crs import CRS + import time + + start_time = time.time() + + # 查找需要的SRTM文件 + hgt_files = [] + for lat in range(int(south), int(north) + 1): + for lon in range(int(west), int(east) + 1): + # 构造文件名 + lat_prefix = 'N' if lat >= 0 else 'S' + lon_prefix = 'E' if lon >= 0 else 'W' + + # SRTM文件名格式: N39E116.hgt + filename = f"{lat_prefix}{abs(lat):02d}{lon_prefix}{abs(lon):03d}.hgt" + filepath = os.path.join(srtm_data_dir, filename) + + if os.path.exists(filepath): + hgt_files.append(filepath) + print(f" ✓ 找到文件: {filename}") + else: + print(f" ⚠ 文件缺失: {filename}") + + if not hgt_files: + print("\n✗ 没有找到任何SRTM文件!") + return None + + print(f"\n找到 {len(hgt_files)} 个SRTM文件") + + # 使用GDAL合并.hgt文件 + merged_file = os.path.join(save_path, f"SRTM_merged_{west}_{east}_{south}_{north}.tif") + + # 构建gdal_merge命令 + gdal_merge_cmd = ['gdal_merge.py', '-o', merged_file, '-of', 'GTiff', '-co', 'COMPRESS=LZW'] + gdal_merge_cmd.extend(hgt_files) + + print(f"\n使用GDAL合并文件...") + result = subprocess.run(gdal_merge_cmd, capture_output=True, text=True) + + if result.returncode != 0 or not os.path.exists(merged_file): + print(f"✗ GDAL合并失败: {result.stderr}") + return None + + print(f"✓ 合并完成: {merged_file}") + + # 裁剪到目标范围 + if output_name is None: + output_name = os.path.join(save_path, f"SRTM_{west}_{east}_{south}_{north}") + + output_file = output_name + '.tif' + + gdal_warp_cmd = [ + 'gdalwarp', + '-te', str(west), str(south), str(east), str(north), + '-of', 'GTiff', + '-co', 'COMPRESS=LZW', + merged_file, + output_file + ] + + print(f"\n裁剪到目标范围...") + result = subprocess.run(gdal_warp_cmd, capture_output=True, text=True) + + if result.returncode != 0 or not os.path.exists(output_file): + print(f"✗ GDAL裁剪失败: {result.stderr}") + return None + + elapsed_time = time.time() - start_time + + print(f"\n✓ SRTM DEM 生成完成!") + print(f" 输出文件: {output_file}") + if os.path.exists(output_file): + file_size = os.path.getsize(output_file) / (1024 * 1024) + print(f" 文件大小: {file_size:.2f} MB") + print(f" 用时: {elapsed_time:.1f} 秒") + + # 清理临时文件 + if os.path.exists(merged_file) and merged_file != output_file: + os.remove(merged_file) + print(f" 清理临时文件: {merged_file}") + + return output_file + + except ImportError: + print("✗ rasterio 未安装,无法处理文件") + return None + except Exception as e: + print(f"\n✗ 处理失败: {e}") + import traceback + traceback.print_exc() + return None + +def merge_tif_files(input_pattern, output_file): + """合并多个 TIF 文件为一个文件 + + Args: + input_pattern (str): 输入 TIF 文件的匹配模式 + output_file (str): 输出合并后的 TIF 文件路径 + """ + # 查找所有匹配的 TIF 文件 + tif_files = sorted(glob.glob(input_pattern)) + + if not tif_files: + print(f"未找到匹配的 TIF 文件: {input_pattern}") + return False + + print(f"\n{'='*80}") + print(f"合并 DEM 瓦片") + print(f"{'='*80}") + print(f"找到 {len(tif_files)} 个 TIF 文件待合并") + print(f"输出文件: {output_file}") + print(f"{'='*80}\n") + + # 使用 gdal_merge.py 合并文件 + try: + start_time = __import__('time').time() + cmd = ["gdal_merge.py", "-o", output_file, "-co", "COMPRESS=LZW", "-co", "BIGTIFF=YES"] + tif_files + print(f"执行合并命令...") + + # 使用 tqdm 显示进度 + result = subprocess.run(cmd, capture_output=True, text=True) + + elapsed_time = __import__('time').time() - start_time + + if result.returncode == 0: + # 检查输出文件大小 + if os.path.exists(output_file): + file_size = os.path.getsize(output_file) / (1024 * 1024) # MB + print(f"\n✓ 成功合并文件!") + print(f" 输出文件: {output_file}") + print(f" 文件大小: {file_size:.2f} MB") + print(f" 用时: {elapsed_time:.1f} 秒") + print(f"{'='*80}\n") + return True + else: + print(f"\n✗ 合并失败: {result.stderr}") + print(f"{'='*80}\n") + return False + except Exception as e: + print(f"\n✗ 合并过程中发生错误: {e}") + print(f"{'='*80}\n") + return False + +def convert_format(input_file, output_file, format_type="GTiff", options=None): + """转换 TIF 文件格式 + + Args: + input_file (str): 输入文件路径 + output_file (str): 输出文件路径 + format_type (str): 输出格式类型 (默认: GTiff) + options (list): gdal_translate 的额外选项 + """ + if options is None: + options = [] + + try: + cmd = ["gdal_translate", "-of", format_type] + options + [input_file, output_file] + print(f"执行转换命令: {' '.join(cmd)}") + result = subprocess.run(cmd, capture_output=True, text=True) + + if result.returncode == 0: + print(f"成功转换文件到: {output_file}") + return True + else: + print(f"转换失败: {result.stderr}") + return False + except Exception as e: + print(f"转换过程中发生错误: {e}") + return False + +def process_dem_files(save_path, west, east, south, north, merge=True, convert=True, + output_format="GTiff", convert_options=None, gamma=False, gamma_byteorder='big', + processor='gamma', output_name=None, num_workers=4, + srtm_data_dir=None, opentopo_api_key=None, opentopo_dem_type='SRTMGL1', + fabdem_dir=None): + """处理下载的 DEM 文件：下载、合并和转换 + + Args: + save_path (str): 数据保存目录 + west (float): 西经边界 + east (float): 东经边界 + south (float): 南纬边界 + north (float): 北纬边界 + merge (bool): 是否合并文件 + convert (bool): 是否转换格式 + output_format (str): 输出格式 + convert_options (list): 转换选项 + gamma (bool): 是否转换为 Gamma/ROI_PAC 格式 + gamma_byteorder (str): 字节序 ('big' 或 'little') + processor (str): 处理器类型 ('gamma' 或 'roi_pac') + output_name (str): 输出文件名（不含扩展名） + num_workers (int): 并行下载线程数 (default: 4) + dem_source (str): DEM数据源 ('copernicus', 'nasadem' 或 'srtm', default: 'copernicus') + srtm_data_dir (str): SRTM数据目录 (仅srtm源需要) + """ + if convert_options is None: + convert_options = ["-co", "COMPRESS=LZW", "-co", "TILED=YES"] + + # ============ 优先尝试本地 FABDEM 瓦片库 ============ + if fabdem_dir and os.path.isdir(fabdem_dir): + print(f"\n{'='*80}") + print(f"优先使用本地 FABDEM 瓦片库: {fabdem_dir}") + print(f"{'='*80}") + try: + from make_local_dem import (find_needed_tiles, extract_tiles_from_zips, + tiles_to_gamma_dem) + import tempfile, shutil + + tiles = find_needed_tiles(west, south, east, north) + print(f"需要 {len(tiles)} 个 1°×1° FABDEM 瓦片") + + temp_dir = tempfile.mkdtemp(prefix='fabdem_tiles_') + try: + tif_files = extract_tiles_from_zips(tiles, fabdem_dir, temp_dir) + if tif_files: + # 一步完成: VRT → ENVI 二进制 → .dem（仅一次大文件写入） + out_name = output_name if output_name else os.path.join(save_path, 'out') + dem_file, dem_par_file = tiles_to_gamma_dem( + tif_files, out_name, west, south, east, north, gamma_byteorder) + if dem_file and dem_par_file: + print(f"\n✓ 本地 FABDEM 生成 GAMMA DEM 成功!") + print(f" DEM: {dem_file}") + print(f" PAR: {dem_par_file}") + shutil.rmtree(temp_dir, ignore_errors=True) + return + else: + print("⚠ 本地 FABDEM 转换失败，回退到网络下载") + else: + print("⚠ 未从本地 FABDEM 提取到瓦片，回退到网络下载") + finally: + shutil.rmtree(temp_dir, ignore_errors=True) + except ImportError as e: + print(f"⚠ 无法导入 make_local_dem 模块 ({e})，回退到网络下载") + except Exception as e: + print(f"⚠ 本地 FABDEM 处理异常 ({e})，回退到网络下载") + + # ============ 回退: 网络下载 ============ + # 检查是否使用预下载的SRTM数据 + if srtm_data_dir and os.path.isdir(srtm_data_dir): + print(f"\n{'='*80}") + print(f"使用预下载的 SRTM 数据") + print(f"{'='*80}") + + if HAS_SRTM: + # Python >= 3.12, 使用srtm库 + dem_file = download_srtm(west, south, east, north, save_path, srtm_data_dir) + else: + # Python < 3.12, 使用手动处理方法 + print("注意: srtm库需要Python >= 3.12,使用手动处理方法") + dem_file = process_srtm_hgt_files(west, south, east, north, srtm_data_dir, + save_path, output_name) + else: + # 自动从OpenTopography下载 + print(f"\n{'='*80}") + print(f"开始下载 DEM 数据...") + print(f"{'='*80}") + print("自动从OpenTopography下载DEM数据") + dem_file = download_srtm_cgiar(west, south, east, north, save_path, + api_key=opentopo_api_key, num_workers=num_workers, + dem_type=opentopo_dem_type) + + if dem_file is None: + print("✗ DEM 生成失败,退出") + return + + current_file = dem_file + + # 生成输出文件名 + if output_name is None: + output_name = os.path.join(save_path, f"DEM_W{int(west)}_E{int(east)}_S{int(south)}_N{int(north)}") + + # 转换到 Gamma/ROI_PAC 格式 + if gamma: + print("\n开始转换到 {} 格式...".format(processor.upper())) + convert_to_gamma(current_file, output_name, gamma_byteorder, processor) + + # 转换到其他格式（非 Gamma/ROI_PAC） + if not gamma and convert: + print("\n开始转换文件格式...") + converted_file = os.path.join(save_path, f"final_DEM_W{int(west)}_E{int(east)}_S{int(south)}_N{int(north)}.tif") + convert_format(current_file, converted_file, output_format, convert_options) + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description='Download DEM data from OpenTopography and convert to Gamma/ROI_PAC format.', + formatter_class=argparse.RawTextHelpFormatter + ) + + # 与 makedem.py 相同的参数 + parser.add_argument('-r', dest='region', help='Research region, west/east/south/north (e.g., 106/110/36/40)') + parser.add_argument('-d', dest='dem', help='Raw dem file that used for further processing') + parser.add_argument('-s', dest='par', help='SLC parameter file of SAR image used for determining research region') + parser.add_argument('-p', dest='processor', help='Interferometry processor. [ gamma or roi_pac ] [default: gamma]') + parser.add_argument('-o', dest='out', help='Output name of the generated DEM') + parser.add_argument('--byteorder', dest='byteorder', choices=['big', 'little'], + help='Byteorder of the generated DEM: big or little. [default: big for gamma and little for roi_pac]') + parser.add_argument('--dir', dest='PATH', help='Processing directory for generating DEM. [default: Current directory]') + parser.add_argument('--num-workers', dest='num_workers', type=int, default=4, + help='Number of parallel download workers (used for batch processing). [default: 4]') + parser.add_argument('--srtm-data-dir', dest='srtm_data_dir', default=None, + help='Directory containing SRTM .hgt files (optional).\n' + 'If not provided, DEM data will be automatically downloaded from OpenTopography.\n' + 'Download SRTM data from: https://srtm.csi.cgiar.org/') + parser.add_argument('--opentopo-api-key', dest='opentopo_api_key', + default='09ad77d34545607fdf5cb182b64ac64e', + help='OpenTopography API key for downloading DEM data.\n' + 'Get a free API key at: https://opentopography.org/myOpenTopo\n' + 'Default key is provided, but you can use your own key.') + parser.add_argument('--opentopo-dem-type', dest='opentopo_dem_type', + choices=['SRTMGL1', 'SRTMGL3', 'NASADEM', 'COP30', 'COP90'], + default='SRTMGL1', + help='DEM type for OpenTopography download. [default: SRTMGL1]\n' + ' - SRTMGL1: SRTM GL1 30m (recommended)\n' + ' - SRTMGL3: SRTM GL3 90m\n' + ' - NASADEM: NASADEM 30m\n' + ' - COP30: Copernicus 30m\n' + ' - COP90: Copernicus 90m') + parser.add_argument('--fabdem-dir', dest='fabdem_dir', default=None, + help='Directory containing local FABDEM ZIP tiles (e.g., /mnt/ZYD/全球FABDEM).\n' + 'If provided, local FABDEM tiles will be used FIRST before network download.\n' + 'Falls back to OpenTopography download if local tiles are unavailable.') + + args = parser.parse_args() + + # 确定工作目录 + if args.PATH: + workdir = args.PATH + else: + workdir = os.getcwd() + + os.chdir(workdir) + + # 确定处理器类型 + if args.processor: + processor = args.processor + else: + processor = 'gamma' + + # 确定字节序 + if args.byteorder: + Byteorder = args.byteorder + else: + if processor == 'gamma': + Byteorder = 'big' + else: + Byteorder = 'little' + + # 确定输出名称 + if args.out: + Name = args.out + else: + Name = "out" + + # 处理已有 DEM 文件的情况 + if args.dem: + dem = args.dem + print('Raw dem file is provided: %s.' % dem) + + # 转换为 TIF 格式（如果不是 TIF） + SUFIX = get_sufix(dem) + if SUFIX != 'tif': + DTIF = dem.replace('.' + SUFIX, '.tif') + call_str = f'gdal_translate {dem} -of GTiff {DTIF}' + os.system(call_str) + DEM = DTIF + else: + DEM = dem + + # 转换到 Gamma/ROI_PAC 格式 + convert_to_gamma(DEM, Name, Byteorder, processor) + + BB = Byteorder + ' endian' + print('') + print('%s %s and %s are generated.' % (BB, Name + '.dem', Name + ('.dem.par' if processor == 'gamma' else '.dem.rsc'))) + print('Congratulations! Done!') + sys.exit(0) + + # 处理从 SLC 参数文件确定区域的情况 + if args.par: + Par = args.par + print("SLC_par file is provided: %s" % Par) + print("DEM over research region will be downloaded automatically based on %s" % Par) + print("DEM data will be downloaded from OpenTopography") + call_str = "SLC_corners " + Par + " > corners.txt" + os.system(call_str) + + File = open("corners.txt", "r") + InfoLine = File.readlines()[8:10] + File.close() + + MinLat = float(InfoLine[0].split(':')[1].split(' max. ')[0]) + MaxLat = float(InfoLine[0].split(':')[2]) + MinLon = float(InfoLine[1].split(':')[1].split(' max. ')[0]) + MaxLon = float(InfoLine[1].split(':')[2]) + + north = int(MaxLat) + 2 + south = int(MinLat) + east = int(MaxLon) + 2 + west = int(MinLon) + elif args.region: + region = args.region + west, south, east, north = read_region(region) + else: + parser.print_usage() + sys.exit(os.path.basename(sys.argv[0]) + ': error: research region, raw_demfile and SLC parameter file, at least one is needed.') + + print('Research region: %s(west) %s(south) %s(east) %s(north)' % (west, south, east, north)) + print('>>> Ready to download DEM over research region.') + + # 处理数据（下载、合并和转换到 Gamma/ROI_PAC 格式） + process_dem_files( + save_path=workdir, + west=west, + east=east, + south=south, + north=north, + merge=True, + convert=False, + gamma=True, + gamma_byteorder=Byteorder, + processor=processor, + output_name=os.path.join(workdir, Name), + num_workers=args.num_workers, + srtm_data_dir=args.srtm_data_dir, + opentopo_api_key=args.opentopo_api_key, + opentopo_dem_type=args.opentopo_dem_type, + fabdem_dir=args.fabdem_dir + ) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/makedem_bk.py b/.codex_tmp/pyint_variants/no_rescue/pyint/makedem_bk.py new file mode 100644 index 0000000..df12a49 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/makedem_bk.py @@ -0,0 +1,302 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import getopt +import array +import argparse +from skimage import io + +from pyint import _utils as ut + +def get_sufix(STR): + n = len(STR.split('.')) + SUFIX = STR.split('.')[n-1] + + return SUFIX + +def read_region(STR): + WEST = STR.split('/')[0] + EAST = STR.split('/')[1].split('/')[0] + + SOUTH = STR.split(EAST+'/')[1].split('/')[0] + NORTH = STR.split(EAST+'/')[1].split('/')[1] + + WEST =float(WEST) + SOUTH=float(SOUTH) + EAST=float(EAST) + NORTH=float(NORTH) + return WEST,SOUTH,EAST,NORTH + + +def write_demrsc_file(FILE,Corner_LON,Corner_LAT,X_STEP,Y_STEP,WIDTH,LENGTH): + f = open(FILE,'w') + f.write('DATE12 111111-222222\n') + f.write('FILE_LENGTH ' + str(int(LENGTH)) + '\n') + f.write('FILE_TYPE .dem\n') + f.write('PROCESSOR roipac\n') + f.write('PROJECTION LATLON\n') + f.write('RLOOKS 1\n') + f.write('WIDTH ' + str(int(WIDTH)) + '\n') + f.write('XMAX ' + str(int(int(WIDTH)-1)) + '\n') + f.write('XMIN 0\n') + f.write('X_FIRST ' + str(float(Corner_LON)) + '\n') + f.write('X_STEP ' + str(float(X_STEP)) + '\n') + f.write('X_UNIT degrees\n') + f.write('YMAX ' + str(int(int(LENGTH)-1)) + '\n') + f.write('YMIN 0\n') + f.write('Y_FIRST ' + str(float(Corner_LAT)) + '\n') + f.write('Y_STEP ' + str(float(Y_STEP)) + '\n') + f.write('Y_UNIT degrees\n') + f.write('Z_OFFSET 0\n') + f.write('Z_SCALE 1\n') + f.close + + +def write_dempar_file(FILE,Corner_LON,Corner_LAT,X_STEP,Y_STEP,WIDTH,LENGTH,DATA_FORMAT): + DEM_TYPE = 'SRTM1' + Proj = 'EQA' + f=open(FILE,'w') + f.write("Gamma DIFF&GEO DEM/MAP parameter file\n") + f.write("title:\tIMPORTED DEM FROM %s\n" % DEM_TYPE) # SRTM1 (30m) or SRTM3 (90m) + f.write("DEM_projection: %s\n" % Proj) # Projection should be checked. + f.write("data_format: %s\n" % DATA_FORMAT) # INTEGER*2 OR REAL*4 should be modified + f.write("DEM_hgt_offset: 0.00000\n") + f.write("DEM_scale: 1.00000\n") + f.write("width: %s\n" % WIDTH) + f.write("nlines: %s\n" % LENGTH) + f.write("corner_lat: %s decimal degrees\n" % Corner_LAT) + f.write("corner_lon: %s decimal degrees\n" % Corner_LON) + f.write("post_lat: %s decimal degrees\n" % Y_STEP) + f.write("post_lon: %s decimal degrees\n" % X_STEP) + f.write("\n") + f.write("ellipsoid_name: WGS 84\n") + f.write("ellipsoid_ra: 6378137.000 m\n") + f.write("ellipsoid_reciprocal_flattening: 298.2572236\n") + f.write("\n") + f.write("datum_name: WGS 1984\n") + f.write("datum_shift_dx: 0.000 m\n") + f.write("datum_shift_dy: 0.000 m\n") + f.write("datum_shift_dz: 0.000 m\n") + f.write("datum_scale_m: 0.00000e+00\n") + f.write("datum_rotation_alpha: 0.00000e+00 arc-sec\n") + f.write("datum_rotation_beta: 0.00000e+00 arc-sec\n") + f.write("datum_rotation_gamma: 0.00000e+00 arc-sec\n") + f.write("datum_country_list Global Definition, WGS84, World\n") + f.write("\n") + + f.close() + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +######################################################################### + +INTRODUCTION = ''' +############################################################################# + Copy Right(c): 2017-2019, Yunmeng Cao [ymcmrs@gmail.com] + + Generating DEM used in interferometry both for GAMMA and ROI_PAC processor. + + 1) Available raw DEM files can be used, e.g., dem.tif, dem.grd; + 2) If no raw DEM is provided, SRTM-1 (30m) can be downloaded automatically. + + Requirement: + Python 2.7 or higher version. GDAL should be installed in your PC. +''' + +EXAMPLE = ''' + Usage: + makedem.py -r west/east/south/north -d raw_demfile -p processor -o output + makedem.py -r west/east/south/north -p processor + makedem.py -d raw_demfile --byteorder + + Examples: + makedem.py -r " -118/-116/33/34 " -p gamma -o SouthCalifornia + makedem.py -r " -118/-116/33/34 " --byteorder little + makedem.py -d dem.tif -p roi_pac --byteorder big + makedem.py -s 20101108.slc.par -p roi_pac +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate DEM for interferometry processing.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + #parser.add_argument('-r','--lalo', dest='region', nargs=4, type=float, help='lalo limit') + parser.add_argument('-r',dest = 'region',help='Research region, west/east/south/north.') + parser.add_argument('-d', dest='dem', help='Raw dem file that used for further processing.') + parser.add_argument('-s', dest='par', help='SLC parameter file of SAR image used for determining research region.') + parser.add_argument('-p', dest='processor', help='Interferometry processor. [ gamma or roi_pac ] [default: gamma]') + parser.add_argument('-o', dest='out', help='Output name of the generated DEM.') + parser.add_argument('--byteorder', dest='byteorder', help='Byteorder of the generated DEM: big or little. [default: big for gamma and little for roi_pac]') + parser.add_argument('--dir', dest='PATH', help='Processing directory for generating DEM. [default: Current directory]') + + inps = parser.parse_args() + + if not inps.region and not inps.dem and not inps.par: + parser.print_usage() + sys.exit(os.path.basename(sys.argv[0])+': error: research region, raw_demfile and SLC parameter file, at least one is needed.') + + return inps + +################################################################################ + +def main(argv): + + inps = cmdLineParse() + + if inps.par: + Par = inps.par + print("SLC_par file is provided: %s" % Par) + print("SRTM1 over research region will be downloaded automatically based on %s" % Par) + call_str = "SLC_corners "+ Par + " > corners.txt" + os.system(call_str) + + File = open("corners.txt","r") + InfoLine = File.readlines()[8:10] + File.close() + + MinLat = float(InfoLine[0].split(':')[1].split(' max. ')[0]) + MaxLat = float(InfoLine[0].split(':')[2]) + MinLon = float(InfoLine[1].split(':')[1].split(' max. ')[0]) + MaxLon = float(InfoLine[1].split(':')[2]) + + north = int(MaxLat) + 2 + south = int(MinLat) + east = int(MaxLon) + 2 + west = int(MinLon) + + + if inps.region: + region = inps.region + west,south,east,north = read_region(region) + + #print 'Research region: %s/%s/%s/%s' % west,south,east,north + + if inps.dem: + dem = inps.dem + print('Raw dem file is provided: %s .' % dem) + + if inps.out: Name = inps.out + else: Name = "out" + + if inps.processor: processor = inps.processor + else: processor ='gamma' + + if inps.byteorder: Byteorder = inps.byteorder + else: + if processor == 'gamma': Byteorder = 'big' + else: Byteorder ='little' + + if inps.PATH: workdir = inps.PATH + else: workdir = os.getcwd() + os.chdir(workdir) + + if not inps.dem: + + print('Research region: %s(west) %s(south) %s(east) %s(north)' % (west,south,east,north)) + print('>>> Ready to download SRTM1 dem over research region.') + #call_str='wget -q -O dem.tif "http://ot-data1.sdsc.edu:9090/otr/getdem?north=%f&south=%f&east=%f&west=%f&demtype=SRTMGL1"' % (north,south,east,west) + #os.system(call_str) + #call_str = 'eio --product SRTM1 clip -o dem.tif --bounds ' + str(west) + ' ' + str(south) + ' ' + str(east) + ' ' + str(north) + call_str = 'sardem --bbox ' + str(west) + ' ' + str(south) + ' ' + str(east) + ' ' + str(north) + ' -o ' + Name + '.dem' + os.system(call_str) + print('>>> DEM download finished.') + + DEM_TYPE='SRTM1' + Proj = 'EQA' + + DEM = 'dem.tif' + call_str = 'gdal_translate ' + Name + '.dem' + ' -of GTiff ' + DEM + os.system(call_str) + else: + DEM = inps.dem + + + + SUFIX = get_sufix(DEM) + SS ='.' + SUFIX + DTIF = DEM.replace(SS,'.tif') + + if not SUFIX == 'tif': + call_str = 'gdal_translate ' + DEM + ' -of GTiff ' + DTIF + os.system(call_str) + + + DEM = DTIF + call_str = 'gdalinfo ' + DEM + ' >ttt' + os.system(call_str) + + f = open('ttt') + for line in f: + if 'Origin =' in line: + STR1 = line + AA = STR1.split('Origin =')[1] + Corner_LON = AA.split('(')[1].split(',')[0] + Corner_LAT = AA.split('(')[1].split(',')[1].split(')')[0] + elif 'Pixel Size ' in line: + STR2 = line + AA = STR2.split('Pixel Size =')[1] + Post_LON = AA.split('(')[1].split(',')[0] + Post_LAT = AA.split('(')[1].split(',')[1].split(')')[0] + + elif 'Size is' in line: + STR3 = line + AA =STR3.split('Size is')[1] + WIDTH = AA.split(',')[0] + FILE_LENGTH = AA.split(',')[1] + f.close() + + dem_data = io.imread(DEM) + if dem_data.dtype=='float32': + DATA_FORMAT='REAL*4' + else: + DATA_FORMAT='INTEGER*2' + + + if not sys.byteorder == Byteorder: + dem_data.byteswap(True) + + if processor =='gamma': + DEMDATA = Name + '.dem' + DEMPAR = Name + '.dem.par' + elif processor =='roi_pac': + DEMDATA = Name + '.dem' + DEMPAR = Name + '.dem.rsc' + + dem_data.tofile(DEMDATA) + + if processor =='gamma': + write_dempar_file(DEMPAR,Corner_LON,Corner_LAT,Post_LON,Post_LAT,WIDTH,FILE_LENGTH,DATA_FORMAT) + elif processor =='roi_pac': + write_demrsc_file(DEMPAR,Corner_LON,Corner_LAT,Post_LON,Post_LAT,WIDTH,FILE_LENGTH) + + BB =Byteorder + ' endian' + print('') + print('%s %s and %s are generated.' % (BB,DEMDATA,DEMPAR)) + print('Congratulations! Done!') + #print "Generating %s processor %s and %s is done!" % (processor,DEMDATA,DEMPAR) + sys.exit(1) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/makedem_pyint.py b/.codex_tmp/pyint_variants/no_rescue/pyint/makedem_pyint.py new file mode 100644 index 0000000..1705b07 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/makedem_pyint.py @@ -0,0 +1,203 @@ +#! /usr/bin/env python +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from pathlib import Path + +from pyint import _utils as ut + + +INTRODUCTION = """ +------------------------------------------------------------------- + + Generate radar-coordinates based DEM. + [Geo-coordinates DEM can be downloaded automatically if not provided.] +""" + +EXAMPLE = """Usage: + + makedem_pyint.py projectName --processor gamma +""" + + +def cmdLineParse() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Generate radar-coordinates based DEM.", + formatter_class=argparse.RawTextHelpFormatter, + epilog=INTRODUCTION + "\n" + EXAMPLE, + ) + parser.add_argument("projectName", help="Name of project.") + parser.add_argument( + "-p", + "--processor", + dest="processor", + choices={"gamma", "roi_pac"}, + default="gamma", + help="Interferometry processor. [default: gamma]", + ) + return parser.parse_args() + + +def _run_checked(command: list[str], *, cwd: str | None = None) -> subprocess.CompletedProcess[str]: + result = subprocess.run( + command, + cwd=cwd, + text=True, + capture_output=True, + check=False, + ) + if result.returncode != 0: + detail = (result.stderr or result.stdout or "").strip() + raise RuntimeError(f"Command failed ({result.returncode}): {' '.join(command)}\n{detail}") + return result + + +def _resolve_existing_dem_open_path(source_dem: str) -> Path: + source_path = Path(source_dem) + if source_path.is_file() and source_path.suffix.lower() == ".vrt": + return source_path + + vrt_path = Path(str(source_path) + ".vrt") + if vrt_path.is_file(): + return vrt_path + + if source_path.is_file(): + return source_path + + raise FileNotFoundError(f"Prepared DEM source does not exist: {source_dem}") + + +def _resolve_research_bbox_from_slc_par(slc_par: str) -> tuple[int, int, int, int]: + result = _run_checked(["SLC_corners", slc_par]) + lines = result.stdout.splitlines() + if len(lines) < 10: + raise RuntimeError(f"Unexpected SLC_corners output for {slc_par}") + + lat_line = lines[8].rstrip() + lon_line = lines[9].rstrip() + min_lat = float(lat_line.split(":")[1].split(" max. ")[0]) + max_lat = float(lat_line.split(":")[2]) + min_lon = float(lon_line.split(":")[1].split(" max. ")[0]) + max_lon = float(lon_line.split(":")[2]) + + north = int(max_lat) + 2 + south = int(min_lat) + east = int(max_lon) + 2 + west = int(min_lon) + return west, south, east, north + + +def _cleanup_temp_outputs(paths: list[Path]) -> None: + for path in paths: + try: + if path.exists(): + path.unlink() + except OSError: + continue + + +def _build_dem_from_existing_source( + *, + project_name: str, + processor: str, + slc_par: str, + source_dem: str, + work_dir: Path, +) -> None: + west, south, east, north = _resolve_research_bbox_from_slc_par(slc_par) + source_open_path = _resolve_existing_dem_open_path(source_dem) + clipped_tif = work_dir / f"{project_name}.prepared_source_clip.tif" + clipped_aux = Path(str(clipped_tif) + ".aux.xml") + + print(f"Using prepared DEM source: {source_dem}") + print(f"Clipping prepared DEM window: west={west}, south={south}, east={east}, north={north}") + + _run_checked( + [ + "gdal_translate", + "-projwin", + str(west), + str(north), + str(east), + str(south), + "-of", + "GTiff", + str(source_open_path), + str(clipped_tif), + ], + cwd=str(work_dir), + ) + _run_checked( + [ + "makedem.py", + "-d", + str(clipped_tif), + "-p", + processor, + "-o", + project_name, + ], + cwd=str(work_dir), + ) + _cleanup_temp_outputs([clipped_tif, clipped_aux]) + + +def main(argv: list[str]) -> None: + inps = cmdLineParse() + projectName = inps.projectName + processor = inps.processor + + scratchDir = os.getenv("SCRATCHDIR") + slcDir = scratchDir + "/" + projectName + "/SLC" + templateDir = os.getenv("TEMPLATEDIR") + templateFile = templateDir + "/" + projectName + ".template" + templateDict = ut.update_template(templateFile) + + masterDate = templateDict["masterDate"] + SLC_PAR = slcDir + "/" + masterDate + "/" + masterDate + ".slc.par" + + demDir = os.getenv("DEMDIR") + demDir1 = demDir + "/" + projectName + if not os.path.isdir(demDir1): + os.mkdir(demDir1) + + os.chdir(demDir1) + work_dir = Path(demDir1) + + prepared_dem_source = str(templateDict.get("prepared_dem_source", "") or "").strip() + if prepared_dem_source not in {"", "-"}: + _build_dem_from_existing_source( + project_name=projectName, + processor=processor, + slc_par=SLC_PAR, + source_dem=prepared_dem_source, + work_dir=work_dir, + ) + print(f"Generate DEM for project {projectName} is done.") + sys.exit(0) + + call_str = "makedem.py " + "-s " + SLC_PAR + " -p gamma " + " -o " + projectName + + if "fabdem_dir" in templateDict and templateDict["fabdem_dir"].strip() not in ["", "-"]: + fabdem_dir = templateDict["fabdem_dir"].strip() + if os.path.isdir(fabdem_dir): + call_str += " --fabdem-dir " + fabdem_dir + print("Using local FABDEM directory: %s" % fabdem_dir) + + if "opentopo_api_key" in templateDict and templateDict["opentopo_api_key"].strip() not in ["", "-"]: + call_str += " --opentopo-api-key " + templateDict["opentopo_api_key"].strip() + if "opentopo_dem_type" in templateDict and templateDict["opentopo_dem_type"].strip() not in ["", "-"]: + call_str += " --opentopo-dem-type " + templateDict["opentopo_dem_type"].strip() + + print("Running: %s" % call_str) + os.system(call_str) + + print("Generate DEM for project %s is done." % projectName) + sys.exit(0) + + +if __name__ == "__main__": + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/mintpy_extract_timeseries_to_geptiff.py b/.codex_tmp/pyint_variants/no_rescue/pyint/mintpy_extract_timeseries_to_geptiff.py new file mode 100644 index 0000000..688b713 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/mintpy_extract_timeseries_to_geptiff.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +""" +SBAS-InSAR时间序列数据提取脚本 +从H5文件中提取时间序列数据、日期和基线信息 +""" + +import h5py +import numpy as np +import os +import argparse +from osgeo import gdal + + +def save_as_geotiff(data, output_file, nodata=-9999): + """ + 将数据保存为GeoTIFF格式 + + Parameters: + ----------- + data : numpy.ndarray + 要保存的数据（2D数组） + output_file : str + 输出文件路径 + nodata : float + 无效值 + """ + # 设置无效值 + data_masked = np.where(np.isnan(data), nodata, data) + + # 获取数据形状 + rows, cols = data_masked.shape + + # 创建GeoTIFF驱动 + driver = gdal.GetDriverByName('GTiff') + + # 创建数据集 + dataset = driver.Create(output_file, cols, rows, 1, gdal.GDT_Float32) + + # 设置波段 + band = dataset.GetRasterBand(1) + band.WriteArray(data_masked) + band.SetNoDataValue(nodata) + + # 设置无数据值 + band.FlushCache() + + # 关闭数据集 + dataset = None + + print(f" 已保存: {output_file}") + + +def extract_timeseries(h5_file, output_dir='./output'): + """ + 提取SBAS-InSAR时间序列数据 + + Parameters: + ----------- + h5_file : str + H5文件路径 + output_dir : str + 输出目录 + """ + + # 创建输出目录 + os.makedirs(output_dir, exist_ok=True) + + # 读取H5文件 + print(f"正在读取文件: {h5_file}") + with h5py.File(h5_file, 'r') as f: + # 提取日期信息 + dates = f['date'][:] + # 将bytes转为字符串 + dates_str = [d.decode('utf-8') if isinstance(d, bytes) else d for d in dates] + print(f"时间序列长度: {len(dates_str)}") + print(f"日期范围: {dates_str[0]} 到 {dates_str[-1]}") + + # 提取基线信息 + bperp = f['bperp'][:] + print(f"基线形状: {bperp.shape}") + + # 提取时间序列数据 + timeseries = f['timeseries'][:] + print(f"时间序列数据形状: {timeseries.shape} (时间, 行, 列)") + + # 保存日期信息 + date_file = os.path.join(output_dir, 'dates.txt') + with open(date_file, 'w') as f_out: + for i, date in enumerate(dates_str): + f_out.write(f"{i+1} {date}\n") + print(f"日期信息已保存到: {date_file}") + + # 保存基线信息 + bperp_file = os.path.join(output_dir, 'bperp.npy') + np.save(bperp_file, bperp) + print(f"基线信息已保存到: {bperp_file}") + + # 保存时间序列数据（numpy格式） + timeseries_file = os.path.join(output_dir, 'timeseries.npy') + np.save(timeseries_file, timeseries) + print(f"时间序列数据已保存到: {timeseries_file}") + + # 保存每个时间点为单独的GeoTIFF文件 + print("\n正在保存每个时间点的GeoTIFF文件...") + geotiff_dir = os.path.join(output_dir, 'geotiff') + os.makedirs(geotiff_dir, exist_ok=True) + + for i, date in enumerate(dates_str): + # 保存为GeoTIFF + geotiff_file = os.path.join(geotiff_dir, f'timeseries_{date}.tif') + save_as_geotiff(timeseries[i], geotiff_file) + print(f"已保存 {len(dates_str)} 个时间点的GeoTIFF文件") + + # 计算并保存累积形变 + print("\n计算累积形变...") + cumulative = np.cumsum(timeseries, axis=0) + cumulative_file = os.path.join(output_dir, 'cumulative_deformation.npy') + np.save(cumulative_file, cumulative) + print(f"累积形变已保存到: {cumulative_file}") + + # 保存累积形变为GeoTIFF + print("\n正在保存累积形变GeoTIFF文件...") + for i, date in enumerate(dates_str): + cumulative_file = os.path.join(geotiff_dir, f'cumulative_{date}.tif') + save_as_geotiff(cumulative[i], cumulative_file) + print(f"已保存 {len(dates_str)} 个累积形变GeoTIFF文件") + + # 保存统计信息 + stats_file = os.path.join(output_dir, 'statistics.txt') + with open(stats_file, 'w') as f_out: + f_out.write("SBAS-InSAR时间序列统计信息\n") + f_out.write("=" * 50 + "\n\n") + f_out.write(f"总时间点数: {len(dates_str)}\n") + f_out.write(f"影像尺寸: {timeseries.shape[1]} 行 x {timeseries.shape[2]} 列\n") + f_out.write(f"日期范围: {dates_str[0]} 到 {dates_str[-1]}\n\n") + f_out.write("形变统计 (单位: mm):\n") + f_out.write(f" 最小值: {np.nanmin(timeseries):.4f}\n") + f_out.write(f" 最大值: {np.nanmax(timeseries):.4f}\n") + f_out.write(f" 平均值: {np.nanmean(timeseries):.4f}\n") + f_out.write(f" 标准差: {np.nanstd(timeseries):.4f}\n") + print(f"统计信息已保存到: {stats_file}") + + print("\n数据提取完成！") + print(f"所有数据已保存到目录: {output_dir}") + + +if __name__ == '__main__': + # 解析命令行参数 + parser = argparse.ArgumentParser( + description='从SBAS-InSAR的H5文件中提取时间序列数据并保存为GeoTIFF格式' + ) + parser.add_argument( + 'h5_file', + type=str, + help='输入的H5文件路径' + ) + parser.add_argument( + '-o', '--output', + type=str, + default='./extracted_data', + help='输出目录路径 (默认: ./extracted_data)' + ) + + args = parser.parse_args() + + # 执行提取 + extract_timeseries(args.h5_file, args.output) \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/mintpy_h5_form_utm_to_wgs84.py b/.codex_tmp/pyint_variants/no_rescue/pyint/mintpy_h5_form_utm_to_wgs84.py new file mode 100644 index 0000000..a1bbcb5 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/mintpy_h5_form_utm_to_wgs84.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +""" +将 HDF5 文件的坐标系统从 UTM 转换为 WGS84 地理坐标 +同时保持与 MintPy tsview.py 的兼容性 +""" + +import h5py +import numpy as np +import copy +import argparse +import os + +def utm_to_wgs84(easting, northing, zone, northern=True): + """ + 将 UTM 坐标转换为 WGS84 大地坐标 + """ + # UTM 参数 + a = 6378137.0 # WGS84 长半轴 + e = 0.081819190842622 # WGS84 第一偏心率 + k0 = 0.9996 # 比例因子 + + # 中央经线 + lon0 = (zone - 1) * 6 - 180 + 3 + + # 调整东伪偏移 + x = easting - 500000.0 + + # 如果是南半球，调整北伪偏移 + if not northern: + y = northing - 10000000.0 + else: + y = northing + + # M = y / k0 + mu = y / (a * (1 - e**2 / 4 - 3 * e**4 / 64 - 5 * e**6 / 256) * k0) + + e1 = (1 - np.sqrt(1 - e**2)) / (1 + np.sqrt(1 - e**2)) + + # 纬度计算 + N1 = a / np.sqrt(1 - e**2 * np.sin(mu)**2) + T1 = np.tan(mu)**2 + C1 = e1**2 * np.cos(mu)**2 + R1 = a * (1 - e**2) / (1 - e**2 * np.sin(mu)**2)**(3/2) + D = x / (N1 * k0) + + lat = mu - N1 * np.tan(mu) / R1 * (D**2 / 2 + (5 + 3 * T1 + 10 * C1 - 4 * C1**2 - 9 * e1**2) * D**4 / 24 + (61 + 90 * T1 + 298 * C1 + 45 * T1**2 - 252 * e1**2 - 3 * C1**2) * D**6 / 720) + + # 经度计算 + lon = lon0 * np.pi / 180 + D * (1 + (1 + 3 * e1**2 + 2 * e1**4) * D**2 / 6 + (2 - e1**2) * D**4 / 120) / np.cos(mu) + + # 转换为角度 + lat_deg = np.degrees(lat) + lon_deg = np.degrees(lon) + + return lon_deg, lat_deg + + +def main(): + parser = argparse.ArgumentParser( + description='将 HDF5 文件转换为 WGS84 地理坐标（与 MintPy 兼容）', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=''' +示例: + python convert_to_wgs84.py input.h5 -o output.h5 + +说明: + 脚本会将文件完全转换为 WGS84 地理坐标系统， + 同时设置正确的属性以保持与 MintPy 的兼容性。 + ''' + ) + parser.add_argument('input', help='输入 HDF5 文件路径') + parser.add_argument('-o', '--output', help='输出 HDF5 文件路径（默认：输入文件名_WGS84.h5）') + + args = parser.parse_args() + + # 输入和输出文件 + input_file = args.input + + if args.output: + output_file = args.output + else: + # 自动生成输出文件名 + base, ext = os.path.splitext(input_file) + output_file = f"{base}_WGS84{ext}" + + print(f"正在读取文件: {input_file}") + f_in = h5py.File(input_file, 'r') + + # 读取所有数据集 + datasets = {} + for key in f_in.keys(): + datasets[key] = f_in[key][:] + print(f" 读取数据集: {key}, 形状: {datasets[key].shape}") + + # 读取所有属性 + attrs = dict(f_in.attrs) + print(f"\n共有 {len(attrs)} 个属性") + + # 解析 UTM 坐标信息 + x_first = float(attrs['X_FIRST']) + y_first = float(attrs['Y_FIRST']) + x_step = float(attrs['X_STEP']) + y_step = float(attrs['Y_STEP']) + width = int(attrs['WIDTH']) + length = int(attrs['LENGTH']) + utm_zone = attrs['UTM_ZONE'] + + # 获取参考点位置（像素坐标） + ref_x = int(attrs.get('REF_X', 0)) + ref_y = int(attrs.get('REF_Y', 0)) + + # 转换 UTM Zone + zone_num = int(''.join(filter(str.isdigit, utm_zone))) + is_northern = 'N' in utm_zone.upper() + + print(f"\n原始坐标系统:") + print(f" UTM Zone: {utm_zone} (EPSG: {attrs['EPSG']})") + print(f" 左上角 UTM: ({x_first}, {y_first})") + print(f" 步长: X={x_step}m, Y={y_step}m") + print(f" 参考点像素: ({ref_x}, {ref_y})") + + # 生成网格坐标 + x_coords = x_first + np.arange(width) * x_step + y_coords = y_first + np.arange(length) * y_step + + # 转换为 WGS84 + print(f"\n正在转换为 WGS84...") + lon_grid, lat_grid = np.meshgrid(x_coords, y_coords) + wgs84_lon, wgs84_lat = utm_to_wgs84(lon_grid, lat_grid, zone_num, is_northern) + + # 计算新的坐标参数（注意：lon 是经度，lat 是纬度） + # X 对应经度，Y 对应纬度 + lon_first = wgs84_lon[0, 0] + lat_first = wgs84_lat[0, 0] + lon_step = abs(wgs84_lon[0, 1] - wgs84_lon[0, 0]) if width > 1 else 0 + lat_step = abs(wgs84_lat[1, 0] - wgs84_lat[0, 0]) if length > 1 else 0 + + # 计算参考点的经纬度 + # REF_X 对应 X 轴（经度），REF_Y 对应 Y 轴（纬度） + ref_lon = wgs84_lon[ref_y, ref_x] # 经度 + ref_lat = wgs84_lat[ref_y, ref_x] # 纬度 + + # 更新属性 - 完全转换为 WGS84 + attrs_new = copy.deepcopy(attrs) + + # 更新坐标系统相关属性 + attrs_new['EPSG'] = 4326 # WGS84 + attrs_new['X_FIRST'] = lon_first # 经度 + attrs_new['Y_FIRST'] = lat_first # 纬度 + attrs_new['X_STEP'] = lon_step + attrs_new['Y_STEP'] = -lat_step # 保持Y方向向下为负 + attrs_new['X_UNIT'] = 'degrees' + attrs_new['Y_UNIT'] = 'degrees' + + # 更新参考坐标 - 现在是真正的经纬度 + attrs_new['REF_LON'] = ref_lon + attrs_new['REF_LAT'] = ref_lat + + # 保留原始 UTM 信息作为备份（带 _ORIG 后缀） + attrs_new['UTM_ZONE_ORIG'] = attrs_new.get('UTM_ZONE', '') + attrs_new['EPSG_ORIG'] = attrs_new.get('EPSG', '') + + # 关键：移除或清空 UTM_ZONE 属性，让 MintPy 识别为地理坐标系统 + if 'UTM_ZONE' in attrs_new: + del attrs_new['UTM_ZONE'] + + # 更新角点坐标 + attrs_new['LAT_REF1'] = lat_first + attrs_new['LAT_REF2'] = wgs84_lat[0, -1] + attrs_new['LAT_REF3'] = wgs84_lat[-1, 0] + attrs_new['LAT_REF4'] = wgs84_lat[-1, -1] + attrs_new['LON_REF1'] = lon_first + attrs_new['LON_REF2'] = wgs84_lon[0, -1] + attrs_new['LON_REF3'] = wgs84_lon[-1, 0] + attrs_new['LON_REF4'] = wgs84_lon[-1, -1] + + # 设置其他相关属性 + attrs_new['COORD_SYSTEM'] = 'GEO' # 地理坐标系统 + + print(f"\n新坐标系统 (WGS84):") + print(f" EPSG: {attrs_new['EPSG']}") + print(f" 左上角: ({attrs_new['X_FIRST']:.6f}, {attrs_new['Y_FIRST']:.6f})") + print(f" 步长: X={attrs_new['X_STEP']:.6f}°, Y={attrs_new['Y_STEP']:.6f}°") + print(f" 参考点经纬度: ({attrs_new['REF_LON']:.6f}, {attrs_new['REF_LAT']:.6f})") + print(f" 参考点像素: ({ref_x}, {ref_y})") + + # 创建新的 HDF5 文件 + print(f"\n正在写入文件: {output_file}") + f_out = h5py.File(output_file, 'w') + + # 写入所有数据集 + for key, data in datasets.items(): + f_out.create_dataset(key, data=data) + print(f" 写入数据集: {key}") + + # 写入所有属性 + for key, value in attrs_new.items(): + f_out.attrs[key] = value + + f_in.close() + f_out.close() + + print(f"\n完成！转换后的文件已保存为: {output_file}") + print("\n说明:") + print(" - 文件已完全转换为 WGS84 地理坐标系统") + print(" - 所有坐标属性已更新为经纬度") + print(" - 原始 UTM 信息已保存为 _ORIG 后缀的属性") + print(" - 应该与 MintPy tsview.py 兼容") + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/mintpy_ssa.py b/.codex_tmp/pyint_variants/no_rescue/pyint/mintpy_ssa.py new file mode 100644 index 0000000..6f92b0a --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/mintpy_ssa.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +import argparse +import sys +import h5py +import numpy as np +from ssa import SSA +import time, math +from mintpy.utils import readfile, writefile, ptime, utils as ut +from mintpy.objects import timeseries + +def create_parser(): + parser=argparse.ArgumentParser() + parser.add_argument('timeseries_file', + help='time-series file to do SSA \n' + 'i.e.:geo_timeseries_GACOS_ramp_demErr.h5 (MintPy)\n') + parser.add_argument("-w", "--window", + type=int, + help='Widow length (default is 7) ') + parser.add_argument('-o', '--output', dest='outfile', help='output file name (default ssa.h5)') + # args=parser.parse_args() + # computing + # parser = arg_group.add_memory_argument(parser) + return parser +def cmd_line_parse(iargs=None): + parser = create_parser() + inps = parser.parse_args(args=iargs) + inps.key = readfile.read_attribute(inps.timeseries_file)['FILE_TYPE'] + if inps.key not in ['timeseries', 'giantTimeseries', 'HDFEOS']: + raise Exception('input file is {}, NOT timeseries!'.format(inps.key)) + if not inps.window: + inps.window = int(7) + if not inps.outfile: + outname = 'ssa' + outname += '.h5' + inps.outfile = outname + tsobj = timeseries(inps.timeseries_file) + return inps +def is_number(s): + if s==0: + return False + elif math.isnan(s): + return False + else: + return True +def main(iargs=None): + # if cmd: + # iargs = cmd.split()[1:] + inps = cmd_line_parse(iargs) + date_list = timeseries(inps.timeseries_file).get_date_list() + data, atr = readfile.read(inps.timeseries_file) + length, width = int(atr['LENGTH']), int(atr['WIDTH']) + N_dates=len(date_list) + # with h5py.File(inps.timeseries_file,'r') as f: + # data = f['timeseries'][:] + # date_list = f['date'][:] + data_matrix = np.empty((len(date_list),length,width)) + data_matrix[:] = np.nan + start_time = time.time() + for i in range(length): + for j in range(width): + if is_number(sum(data[:,i,j])): + a=SSA(data[:,i,j], inps.window) + data_matrix[:,i,j]=a.reconstruct([0,1]) # Grouping recostructed components, by defaults it takes F1 and F2 as trend + m, s = divmod(time.time()-start_time, 60) + block= [0,N_dates,0,length,0,width] + writefile.layout_hdf5(inps.outfile, metadata=atr, ref_file=inps.timeseries_file) + writefile.write_hdf5_block(inps.outfile, + data=data_matrix, + datasetName='timeseries', + block=block, + print_msg=False) + + + print(inps.timeseries_file) + print(inps.window) + # print(date_list) + print(length,width) + print('time used: {:02.0f} mins {:02.1f} secs.'.format(m, s)) +if __name__ == '__main__': + main(sys.argv[1:]) + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/operation.csh b/.codex_tmp/pyint_variants/no_rescue/pyint/operation.csh new file mode 100644 index 0000000..691259d --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/operation.csh @@ -0,0 +1,126 @@ +#!/bin/csh -f + +if ($#argv != 7) then + echo "" + echo "Usage: operation.csh master_ztd master_ztd.rsc slave_ztd slave_ztd.rsc reference_point incidence_angle dem_grid" + echo " Performs gacos correction over the phasefilt.grd files for Sentinel 1" + echo "" + echo "It works in each directory where the phase grids are located. This script works jointly with GACOS_correction.csh" + echo "" + echo "Reference point in radar coordinates (text file)" + echo "" + echo "Indicence angle in degrees from SAT_look (float/integer)" + echo "" + exit 1 + endif + +#reference point/stable point in radar coordinates +set reference_point = $5 + +#incidence angle obtained with the "SAT_look" script in degrees +set incidence = $6 + +#wavelength for Sentinel 1 (m) +set wavelength = 0.0554658 +set pi = 3.141592653589793238462 + +#######FIRST ZTD to grid######### +set x_first_d1 = `cat $2|grep X_FIRST|awk '{print $2}'` +set y_first_d1 = `cat $2|grep Y_FIRST|awk '{print $2}'` +set width_d1 = `cat $2|grep WIDTH|awk '{print $2}'` +set length_d1 = `cat $2|grep FILE_LENGTH|awk '{print $2}'` +set x_step_d1 = `cat $2|grep X_STEP|awk '{print $2}'` +set y_step_d1 = `cat $2|grep X_STEP|awk '{print $2}'` +set date_ztd_d1 = `echo $1|awk -F/ '{print $NF}'|cut -c1-8` +gmt xyz2grd $1 -G"date1_ztd.grd" -RLT$x_first_d1/$y_first_d1/$width_d1/$length_d1 -I$x_step_d1/$y_step_d1 -ZTLf -di0 -r + +#######SECOND ZTD to grid######### +set x_first_d2 = `cat $4|grep X_FIRST|awk '{print $2}'` +set y_first_d2 = `cat $4|grep Y_FIRST|awk '{print $2}'` +set width_d2 = `cat $4|grep WIDTH|awk '{print $2}'` +set length_d2 = `cat $4|grep FILE_LENGTH|awk '{print $2}'` +set x_step_d2 = `cat $4|grep X_STEP|awk '{print $2}'` +set y_step_d2 = `cat $4|grep X_STEP|awk '{print $2}'` +set date_ztd_d2 = `echo $3|awk -F/ '{print $NF}'|cut -c1-8` +gmt xyz2grd $3 -G"date2_ztd.grd" -RLT$x_first_d2/$y_first_d2/$width_d2/$length_d2 -I$x_step_d2/$y_step_d2 -ZTLf -di0 -r + +#TIME DIFFERENCE +gmt grdmath date2_ztd.grd date1_ztd.grd SUB = zpddm.grd + +#Checking GACOS grids within the DEM boundaries +set xmin_dem = `gmt grdinfo -C $7|awk '{print $2}'` +set xmax_dem = `gmt grdinfo -C $7|awk '{print $3}'` +set ymin_dem = `gmt grdinfo -C $7|awk '{print $4}'` +set ymax_dem = `gmt grdinfo -C $7|awk '{print $5}'` + +set xmin_gacos = `gmt grdinfo -C zpddm.grd|awk '{print $2}'` +set xmax_gacos = `gmt grdinfo -C zpddm.grd|awk '{print $3}'` +set ymin_gacos = `gmt grdinfo -C zpddm.grd|awk '{print $4}'` +set ymax_gacos = `gmt grdinfo -C zpddm.grd|awk '{print $5}'` + +set check = `echo "$xmin_dem < $xmin_gacos && $ymin_dem < $ymin_gacos && $xmax_dem > $xmax_gacos && $ymax_dem > $ymax_gacos"| bc` + +if ($check == 0) then + echo "Seems like your DEM is not large enough. GACOS grids dimensions must be within the DEM." + echo "DEM dimensions: xmin: $xmin_dem xmax: $xmax_dem ymin: $ymin_dem ymax: $ymax_dem" + echo "GACOS dimensions: xmin: $xmin_gacos xmax: $xmax_gacos ymin: $ymin_gacos ymax: $ymax_gacos" + exit 1 +else + echo "GACOS grid dimensions within DEM boundaries... continue..." +endif + +#PROJECT TO RADAR COORDINATES +proj_ll2ra.csh trans.dat zpddm.grd zpddm_ra.grd + +#RESAMPLE ZTD FILES WITH UNWRAP GRID PARAMETERS +set xmin = `gmt grdinfo -C phasefilt.grd|awk '{print $2}'` +set xmax = `gmt grdinfo -C phasefilt.grd|awk '{print $3}'` +set ymin = `gmt grdinfo -C phasefilt.grd|awk '{print $4}'` +set ymax = `gmt grdinfo -C phasefilt.grd|awk '{print $5}'` +set xinc = `gmt grdinfo -C phasefilt.grd|awk '{print $8}'` +set yinc = `gmt grdinfo -C phasefilt.grd|awk '{print $9}'` +gmt grdsample zpddm_ra.grd -Gresample_zpddm.grd -R$xmin/$xmax/$ymin/$ymax -I$xinc/$yinc -r + + +#REFERENCE POINT +set ref_value = `gmt grdtrack $reference_point -Gresample_zpddm.grd -Z` +set ref_value_phase = `gmt grdtrack $reference_point -Gphasefilt.grd -Z` +gmt grdmath resample_zpddm.grd $ref_value SUB = szpddm.grd +gmt grdmath phasefilt.grd $ref_value_phase SUB = phasefilt_ref.grd + +#Checking reference point values +if ($ref_value == "" || $ref_value_phase == "") then + echo "Problems with the reference point. Is the reference point within the AOI?" + exit 1 +endif + +#FROM METER TO PHASE +#gmt grdmath szpddm.grd 4 MUL $pi MUL $wavelength DIV = szpddm_phase.grd +#multiplying by -4 instead of 4 (edition: 13/4/2023) +gmt grdmath szpddm.grd -4 MUL $pi MUL $wavelength DIV = szpddm_phase.grd + +#PROJECTION FROM ZENITH VIEW TO LOS +gmt grdmath szpddm_phase.grd $incidence COSD DIV = szpddm_phase_LOS.grd + +#CORRECTION WITH GACOS DATA +#attention: Here the phasefilt.grd is corrected, therefore for the unwrap process this output or the detrended output +#should be used. +gmt grdmath phasefilt_ref.grd szpddm_phase_LOS.grd SUB = phasefilt_GACOS_corrected.grd + +#DETRENDING +gmt grdtrend phasefilt_GACOS_corrected.grd -N3r -Dphasefilt_GACOS_corrected_detrended.grd + +#checking existence of final outputs +if !(-f phasefilt_GACOS_corrected.grd || -f phasefilt_GACOS_corrected_detrended.grd) then + echo "Seems like there was an issue correcting with GACOS $1 and $3" + exit 1 +endif + +#clean up +# +rm date1_ztd.grd date2_ztd.grd +rm zpddm.grd zpddm_ra.grd resample_zpddm.grd +rm szpddm_phase.grd +rm phasefilt_ref.grd +rm szpddm_phase_LOS.grd +echo "corrections done with $date_ztd_d1 and $date_ztd_d2 over phasefilt.grd" diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/out.dem b/.codex_tmp/pyint_variants/no_rescue/pyint/out.dem new file mode 100644 index 0000000..bd9b847 Binary files /dev/null and b/.codex_tmp/pyint_variants/no_rescue/pyint/out.dem differ diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/out.dem.par b/.codex_tmp/pyint_variants/no_rescue/pyint/out.dem.par new file mode 100644 index 0000000..dda0c34 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/out.dem.par @@ -0,0 +1,27 @@ +Gamma DIFF&GEO DEM/MAP parameter file +title: IMPORTED DEM FROM Copernicus30 +DEM_projection: EQA +data_format: INTEGER*2 +DEM_hgt_offset: 0.00000 +DEM_scale: 1.00000 +width: 4800 +nlines: 3600 +corner_lat: 26.000416666674390 decimal degrees +corner_lon: 99.999583333269726 decimal degrees +post_lat: -0.000833333333333 decimal degrees +post_lon: 0.000833333333333 decimal degrees + +ellipsoid_name: WGS 84 +ellipsoid_ra: 6378137.000 m +ellipsoid_reciprocal_flattening: 298.2572236 + +datum_name: WGS 1984 +datum_shift_dx: 0.000 m +datum_shift_dy: 0.000 m +datum_shift_dz: 0.000 m +datum_scale_m: 0.00000e+00 +datum_rotation_alpha: 0.00000e+00 arc-sec +datum_rotation_beta: 0.00000e+00 arc-sec +datum_rotation_gamma: 0.00000e+00 arc-sec +datum_country_list Global Definition, WGS84, World + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/phase2los.py b/.codex_tmp/pyint_variants/no_rescue/pyint/phase2los.py new file mode 100644 index 0000000..3cff4db --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/phase2los.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +#! /usr/bin/env python +################################################################# +### This program is part of PyINT ### +### Copy Right (c): 2019, Wei Chen ### +### Author: Wei Chen ### +### Contact : chenweicug@126.com ### +################################################################# +import numpy as np +import os +import sys +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + +INTRODUCTION = ''' +------------------------------------------------------------------- + convert interferogram to los displacement and get the inc angle and azi angle. + +''' + +EXAMPLE = ''' + Usage: + phase2los.py projectName Mdate Sdate + phase2los.py PacayaT163TsxHhA 20150102 20150601 +------------------------------------------------------------------- +''' + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='convert interferogram to los displacement and get the inc angle and azi angle.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('Mdate',help='Master date.') + parser.add_argument('Sdate',help='Slave date.') + + inps = parser.parse_args() + return inps + + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Mdate = inps.Mdate + Sdate= inps.Sdate + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + satelite = templateDict['satelite'] + rlks = templateDict['range_looks'] + + projectDir = scratchDir + '/' + projectName + ifgDir = projectDir + '/ifgrams' + simDir = scratchDir + '/' + projectName + "/DEM" + losDir = projectDir + '/LosResult' + slcDir = projectDir + '/SLC' + + + if not os.path.isdir(losDir): os.mkdir(losDir) + + Pair = Mdate + '-' + Sdate + workDir = losDir + '/' + Pair + if not os.path.isdir(workDir): os.mkdir(workDir) + workdir = ifgDir + '/' + Pair + UTMDEMpar = simDir + '/'+ Mdate + '_'+ rlks + 'rlks.utm.dem.par' + GeoUNW = workdir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.unw' + GeoATMCOR_UNW = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.atmcor.unw' + call_str = 'data2geotiff ' + UTMDEMpar + ' ' + GeoATMCOR_UNW + ' 2 ' + GeoATMCOR_UNW +'.tif' + os.system(call_str) + call_str = 'data2geotiff ' + UTMDEMpar + ' ' + GeoUNW + ' 2 ' + GeoUNW +'.tif' + os.system(call_str) + call_str = 'gdal_translate -of GSBG ' + ' ' + GeoUNW +'.tif' + ' ' + workDir + '/' + GeoUNW + '.grd' + os.system(call_str) + call_str = 'gdal_translate -of GSBG ' + ' ' + GeoATMCOR_UNW +'.tif' + ' ' + workDir + '/' + GeoATMCOR_UNW +'.grd' + os.system(call_str) + if satelite == 'CSK': + wavelength = 0.0312283810417 + elif satelite == 'TSX': + wavelength = 0.03106657823461874 + elif satelite == 'S1A': + wavelength = 0.0554657647 + elif satelite == 'ALOS2': + wavelength = 0.2424525 + elif satelite == 'ALOS': + wavelength = 0.2360571 + elif satelite == 'ENVISAT': + wavelength = 0.056 + call_str = 'grdmath ' + workDir + '/' + GeoUNW + '.grd' + str(wavelength) + ' 1 3.141592653589 MUL DIV MUL = ' + workDir + '/los.grd' + os.system(call_str) + call_str = 'grdmath ' + workDir + '/' + GeoATMCOR_UNW +'.grd' + str(wavelength) + ' 1 3.141592653589 MUL DIV MUL = ' + workDir + '/los_atmcor.grd' + os.system(call_str) + print('the los displacement of ' + Pair + " has successfully done") + print ("#####################run data2inc_azi###################") + Mslcpar = slcDir + '/' + Mdate + '/' + Mdate + '.slc.par' + UTMDEM = simDir + '/'+ Mdate + '_'+ rlks + 'rlks.utm.dem' + OFF = workdir + '/' + Pair +'_' + rlks + 'rlks.off' + os.chdir(workDir) + call_str = 'data2inc_azi ' + Mslcpar + ' ' + OFF + ' ' + UTMDEMpar + ' ' + UTMDEM + os.system(call_str) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/phase2los_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/phase2los_all.py new file mode 100644 index 0000000..38f9ad2 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/phase2los_all.py @@ -0,0 +1,111 @@ +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + + +def work(data0): + cmd = data0[0] + err_file = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + cmd[3] + '\n' + #print(aa) + with open(err_file, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + convert interferograms phase to los deformation for one project using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + phase2los_all.py projectName + phase2los_all.py projectName --parallel 4 + diff_gamma_all.py projectName --parallel 4 --ifgramList-txt /test/ifgram_list.txt +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Geneate differential interferograms for one project using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + parser.add_argument('--ifgarmList-txt', dest='ifgarmListTxt', help='provided ifgram_list_txt. default: using ifgram_list.txt under projectName folder.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + projectDir = scratchDir + '/' + projectName + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + if inps.ifgarmListTxt: ifgramList_txt = inps.ifgarmListTxt + else: ifgramList_txt = scratchDir + '/' + projectName + '/ifgram_list.txt' + ifgList0 = ut.read_txt2array(ifgramList_txt) +# ifgList = ifgList0[:,0] + if len(ifgList0)==3: + ifgList=ifgList0[0] + ifgList=[ifgList] + + else: + ifgList=ifgList0[:,0] + + err_txt = scratchDir + '/' + projectName + '/diff_gamma_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + for i in range(len(ifgList)): + m0 = ut.yyyymmdd(ifgList[i].split('-')[0]) + print(m0) + s0 = ut.yyyymmdd(ifgList[i].split('-')[1]) + cmd0 = ['diff_gamma.py',projectName, m0, s0] + diff_file0 = ifgDir + '/' + ifgList[i] + '/' + ifgList[i] + '_' + rlks + 'rlks.diff_filt.bmp' + data0 = [cmd0,err_txt] + + k00 = 0 + if os.path.isfile(diff_file0): + if os.path.getsize(diff_file0) > 0: + k00 = 1 + if k00==0: + data_para.append(data0) + + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + print("Generate differential interferograms for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/phasebias_correction_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/phasebias_correction_gamma.py new file mode 100644 index 0000000..51e824f --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/phasebias_correction_gamma.py @@ -0,0 +1,545 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Phase Bias Correction for GAMMA format interferograms ### +### Author: ZYD / Cascade AI ### +################################################################# + +import numpy as np +import os +import sys +import time +import argparse +from pathlib import Path +from datetime import datetime +from concurrent.futures import ProcessPoolExecutor, as_completed + +import rasterio +from rasterio.transform import Affine +from rasterio.crs import CRS + +from pyint import _utils as ut + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Apply phase bias correction to interferograms. + This script integrates the phase bias correction algorithm from: + InSAR_PhaseBias_Correction/ directory + + Based on the paper: + "Correcting InSAR phase bias caused by atmospheric delays + and other error sources using multiple aperture interferometry" + (Reference: 10.1016/j.remote.2022.100013) + + Algorithm Steps: + 1. Read interferogram data (PhaseBias_01_Read_Data.py) + 2. Calculate loop closures (PhaseBias_02_Loop_Closures.py) + 3. Estimate calibration parameters an (PhaseBias_03_calibration_pars.py) + 4. Inversion for phase bias terms (PhaseBias_04_Inversion.py) + 5. Apply correction (PhaseBias_05_Correction.py) + +''' + +EXAMPLE = ''' + Usage: + phasebias_correction_gamma.py projectName [options] + phasebias_correction_gamma.py PacayaT163TsxHhA --interval 12 + phasebias_correction_gamma.py PacayaT163TsxHhA --interval 12 --nlook 10 + phasebias_correction_gamma.py PacayaT163TsxHhA --interval 24 --num-a 2 +------------------------------------------------------------------- +''' + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Apply phase bias correction to interferograms.', + formatter_class=argparse.RawTextHelpFormatter, + epilog=INTRODUCTION+'\n'+EXAMPLE) + parser.add_argument('projectName', help='projectName for processing.') + parser.add_argument('--interval', type=int, default=12, + help='Data acquisition interval in days (6, 12, or 24). [default: 12]') + parser.add_argument('--nlook', type=int, default=10, + help='Number of looks for multilooking. [default: 10]') + parser.add_argument('--num-a', type=int, default=2, + help='Number of calibration parameters to estimate. [default: 2]') + parser.add_argument('--start', type=str, default=None, + help='Start date (YYYYMMDD). If not specified, will use all available data.') + parser.add_argument('--end', type=str, default=None, + help='End date (YYYYMMDD). If not specified, will use all available data.') + parser.add_argument('--estimate-an', dest='estimate_an', action='store_true', + help='Estimate calibration parameters from data instead of using defaults.') + parser.add_argument('--max-con', type=int, default=5, + help='Maximum number of connections to correct. [default: 5]') + parser.add_argument('--parallel', type=int, default=8, + help='Number of parallel workers for data conversion. [default: 8]') + parser.add_argument('--skip-convert', action='store_true', + help='Skip data conversion if GeoTIFF already exists.') + + inps = parser.parse_args() + return inps + + +def find_date_range(ifgDir, interval=12): + """Find the start and end dates from interferogram directory + + Args: + ifgDir: interferogram directory + interval: temporal baseline in days + + Returns: + tuple: (start_date, end_date) in YYYYMMDD format + """ + dates = [] + + all_dirs = sorted([d for d in os.listdir(ifgDir) if os.path.isdir(os.path.join(ifgDir, d))]) + + for dirname in all_dirs: + if '-' in dirname: + date_pair = dirname.split('-') + elif '_' in dirname: + date_pair = dirname.split('_') + else: + continue + + if len(date_pair) == 2: + try: + date1 = date_pair[0] + date2 = date_pair[1] + + d1 = datetime.strptime(date1, '%Y%m%d') + d2 = datetime.strptime(date2, '%Y%m%d') + days = abs((d2 - d1).days) + + if days == interval: + dates.extend([date1, date2]) + except: + continue + + if dates: + dates = sorted(set(dates)) + return dates[0], dates[-1] + + return None, None + + +# ===================================================================== +# GAMMA 二进制 → GeoTIFF 转换函数 +# ===================================================================== + +def parse_utm_dem_par(par_file): + """解析 GAMMA utm.dem.par 文件，提取地理编码参数""" + params = {} + with open(par_file) as f: + for line in f: + if ':' in line: + key, val = line.split(':', 1) + key = key.strip() + val = val.strip().split()[0] # 取第一个字段 + params[key] = val + return { + 'width': int(params['width']), + 'nlines': int(params['nlines']), + 'corner_lat': float(params['corner_lat']), + 'corner_lon': float(params['corner_lon']), + 'post_lat': float(params['post_lat']), + 'post_lon': float(params['post_lon']), + } + + +def write_geotiff(data, out_path, geo_info, dtype='float32', nodata=None): + """将 numpy 数组写入带地理信息的 GeoTIFF (rasterio)""" + height, width = data.shape + transform = Affine( + geo_info['post_lon'], 0, geo_info['corner_lon'], + 0, geo_info['post_lat'], geo_info['corner_lat'] + ) + profile = { + 'driver': 'GTiff', + 'dtype': dtype, + 'width': width, + 'height': height, + 'count': 1, + 'crs': CRS.from_epsg(4326), + 'transform': transform, + 'compress': 'lzw', + 'tiled': True, + } + if nodata is not None: + profile['nodata'] = nodata + with rasterio.open(str(out_path), 'w', **profile) as dst: + dst.write(data, 1) + + +def convert_one_pair(args): + """转换单个干涉对: GAMMA 二进制 → GeoTIFF + + 返回: (pair_name, True/False, message) + """ + pair_dir, out_ifg_dir, geo_info, master_date, rlks = args + pair_dir = Path(pair_dir) + pair_name = pair_dir.name # e.g. 20241105-20241117 + d1, d2 = pair_name.split('-') + pair_us = f'{d1}_{d2}' # 下划线分隔 + width = geo_info['width'] + nlines = geo_info['nlines'] + + out_dir = Path(out_ifg_dir) / pair_us + out_dir.mkdir(parents=True, exist_ok=True) + + pha_tif = out_dir / f'{pair_us}.geo.diff_pha.tif' + cc_tif = out_dir / f'{pair_us}.geo.cc.tif' + + # 如果两个文件已存在则跳过 + if pha_tif.exists() and cc_tif.exists(): + return (pair_name, True, '已存在,跳过') + + try: + # --- 缩缩相位: FCOMPLEX → phase --- + diff_filt = pair_dir / f'geo_{pair_name}_{rlks}rlks.diff_filt' + if not diff_filt.exists(): + return (pair_name, False, f'缺少 {diff_filt.name}') + + cpx = np.fromfile(str(diff_filt), dtype=np.complex64).reshape(nlines, width) + phase = np.angle(cpx).astype(np.float32) + phase[cpx == 0] = 0.0 # GAMMA 无数据区域设为 0 + del cpx + write_geotiff(phase, pha_tif, geo_info, dtype='float32', nodata=0) + del phase + + # --- 相干性: FLOAT → 0-255 uint8 --- + cor_file = pair_dir / f'geo_{master_date}_{rlks}rlks.diff_filt.cor' + if not cor_file.exists(): + # 尝试备选命名 + cor_file = pair_dir / f'{pair_name}_{rlks}rlks.diff_filt.cor' + if not cor_file.exists(): + return (pair_name, False, f'缺少 cor 文件') + + cor = np.fromfile(str(cor_file), dtype=np.float32).reshape(nlines, width) + # 清理无效值 + cor[~np.isfinite(cor)] = 0 + cor = np.clip(cor, 0, 1) + cc_uint8 = (cor * 255).astype(np.uint8) + del cor + write_geotiff(cc_uint8, cc_tif, geo_info, dtype='uint8', nodata=0) + del cc_uint8 + + return (pair_name, True, 'OK') + + except Exception as e: + return (pair_name, False, str(e)) + + +def create_landmask(amp_file, geo_info, out_path): + """从地理编码幅度图生成陆地掩膜 GeoTIFF""" + width = geo_info['width'] + nlines = geo_info['nlines'] + amp = np.fromfile(str(amp_file), dtype=np.float32).reshape(nlines, width) + mask = np.zeros_like(amp, dtype=np.uint8) + mask[amp > 0] = 1 + write_geotiff(mask, out_path, geo_info, dtype='uint8', nodata=0) + print(f' 陆地掩膜已生成: {out_path}') + print(f' 有效像素: {int(np.sum(mask))}/{mask.size} ({100*np.sum(mask)/mask.size:.1f}%)') + + +def prepare_gamma_data_for_phasebias(ifgDir, demDir, masterDate, rlks, + staging_dir, n_parallel=8, + skip_existing=True): + """将 GAMMA 格式地理编码干涉图转换为 PhaseBias 脚本期望的 GeoTIFF 格式 + + 目录结构: + staging_dir/ + interferograms/ + YYYYMMDD_YYYYMMDD/ + YYYYMMDD_YYYYMMDD.geo.diff_pha.tif + YYYYMMDD_YYYYMMDD.geo.cc.tif + metadata/ + frame.geo.landmask.tif + """ + staging = Path(staging_dir) + ifg_out = staging / 'interferograms' + meta_out = staging / 'metadata' + ifg_out.mkdir(parents=True, exist_ok=True) + meta_out.mkdir(parents=True, exist_ok=True) + + # 解析地理编码参数 + utm_par = Path(demDir) / f'{masterDate}_{rlks}rlks.utm.dem.par' + if not utm_par.exists(): + raise FileNotFoundError(f'utm.dem.par 不存在: {utm_par}') + geo_info = parse_utm_dem_par(str(utm_par)) + print(f' 地理编码参数: {geo_info["width"]}x{geo_info["nlines"]}, ' + f'corner=({geo_info["corner_lat"]:.4f}, {geo_info["corner_lon"]:.4f})') + + # 生成陆地掩膜 + landmask_tif = meta_out / 'frame.geo.landmask.tif' + if not landmask_tif.exists(): + # 从任意干涉对目录中取地理编码幅度图 + first_pair = sorted(Path(ifgDir).iterdir())[0] + amp_file = first_pair / f'geo_{masterDate}_{rlks}rlks.amp' + if not amp_file.exists(): + print(f' 警告: 未找到幅度图 {amp_file}，将创建全 1 掩膜') + mask = np.ones((geo_info['nlines'], geo_info['width']), dtype=np.uint8) + write_geotiff(mask, landmask_tif, geo_info, dtype='uint8') + print(f' 全 1 掩膜已生成: {landmask_tif}') + else: + create_landmask(amp_file, geo_info, landmask_tif) + else: + print(f' 陆地掩膜已存在: {landmask_tif}') + + # 收集需要转换的干涉对 + pair_dirs = sorted([d for d in Path(ifgDir).iterdir() + if d.is_dir() and '-' in d.name]) + print(f' 发现 {len(pair_dirs)} 个干涉对目录') + + # 并行转换 + tasks = [(str(pd), str(ifg_out), geo_info, masterDate, rlks) for pd in pair_dirs] + + done_ok = 0 + done_skip = 0 + done_fail = 0 + t0 = time.time() + + with ProcessPoolExecutor(max_workers=n_parallel) as pool: + futures = {pool.submit(convert_one_pair, t): t[0] for t in tasks} + for i, fut in enumerate(as_completed(futures), 1): + pair_name, success, msg = fut.result() + if success: + if '跳过' in msg: + done_skip += 1 + else: + done_ok += 1 + else: + done_fail += 1 + print(f' ✗ {pair_name}: {msg}') + + if i % 50 == 0 or i == len(tasks): + elapsed = time.time() - t0 + print(f' 进度: {i}/{len(tasks)} ' + f'(新转换={done_ok}, 跳过={done_skip}, ' + f'失败={done_fail}, 耗时={elapsed:.0f}s)') + + print(f'\n 转换完成: 新转换={done_ok}, 跳过={done_skip}, 失败={done_fail}') + if done_fail > 0: + print(f' 警告: {done_fail} 个干涉对转换失败，将被 PhaseBias 脚本跳过') + + return str(staging) + + +# ===================================================================== +# 配置文件生成 +# ===================================================================== + +def create_config_file(output_path, root_path, interval, nlook, num_a, + start_date, end_date, estimate_an): + """创建 PhaseBias 脚本的配置文件 + + Args: + output_path: 输出目录 + root_path: 包含 interferograms/ 和 metadata/ 子目录的根路径 + """ + config_content = f"""[DEFAULT] +# 包含 interferograms/ 和 metadata/ 子目录的根路径 +root_path = {root_path} + +output_path = {output_path} + +LiCSAR_data=no +frame=NA + +# Start and end dates +start={start_date} +end={end_date} + +# Data acquisition interval: 6-day or 12-day or 24-day +interval={interval} + +# Multilooking factor +nlook={nlook} + +# Number of calibration parameters to estimate +num_a={num_a} + +### an parameters +# Estimate from data or use default values +estimate_an_values={'yes' if estimate_an else 'no'} + +# Using 6-days interval +a1_6_day=0.50 +a2_6_day=0.36 +a3_6_day=0.299 +a4_6_day=0.2476 + +# Using 12-day interval +a1_12_day=0.494 +a2_12_day=0.297 +a3_12_day=0.24 +a4_12_day=0.22 + +# Using 24-day interval (estimated) +a1_24_day=0.48 +a2_24_day=0.28 +a3_24_day=0.22 +a4_24_day=0.20 +""" + config_file = os.path.join(output_path, 'config.txt') + with open(config_file, 'w') as f: + f.write(config_content) + return config_file + + +def run_phasebias_scripts(phasebias_dir, output_path, script_name): + """Run a PhaseBias script + + Args: + phasebias_dir: directory containing PhaseBias scripts + output_path: working directory + script_name: name of the script to run + """ + script_path = os.path.join(phasebias_dir, script_name) + + if not os.path.exists(script_path): + print(f"ERROR: Script not found: {script_path}") + return False + + print(f"\n{'='*80}") + print(f"Running {script_name}...") + print(f"{'='*80}") + + # Run the script + original_dir = os.getcwd() + os.chdir(output_path) + + try: + import subprocess + result = subprocess.run([sys.executable, script_path], + capture_output=True, text=True, check=True) + print(result.stdout) + if result.stderr: + print("STDERR:", result.stderr) + return True + except subprocess.CalledProcessError as e: + print(f"ERROR running {script_name}:") + print(e.stdout) + print(e.stderr) + return False + finally: + os.chdir(original_dir) + + +def main(argv): + + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + interval = inps.interval + nlook = inps.nlook + num_a = inps.num_a + estimate_an = inps.estimate_an + max_con = inps.max_con + n_parallel = inps.parallel + skip_convert = inps.skip_convert + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + + # PyINT 路径 + templateFile = templateDir + "/" + projectName + ".template" + templateDict = ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + + ifgDir = scratchDir + '/' + projectName + "/ifgrams" + demDir = scratchDir + '/' + projectName + "/DEM" + + # PhaseBias 脚本目录 + pyintDir = os.path.dirname(os.path.abspath(__file__)) + phasebiasDir = os.path.join(pyintDir, 'InSAR_PhaseBias_Correction') + + # 工作目录 + workDir = scratchDir + '/' + projectName + "/PhaseBiasCorrection" + os.makedirs(workDir, exist_ok=True) + os.makedirs(os.path.join(workDir, 'Data'), exist_ok=True) + + print("\n" + "="*80) + print(f"Phase Bias Correction for {projectName}") + print(f"Temporal interval: {interval} days") + print(f"Multilooking factor: {nlook}") + print(f"Number of calibration parameters: {num_a}") + print(f"Estimate an from data: {estimate_an}") + print(f"Max connections: {max_con}") + print(f"Parallel workers: {n_parallel}") + print("="*80) + + # 确定日期范围 + if inps.start and inps.end: + start_date = inps.start + end_date = inps.end + else: + print("\n自动检测日期范围...") + start_date, end_date = find_date_range(ifgDir, interval) + if not start_date or not end_date: + print("ERROR: 无法从干涉图目录确定日期范围!") + print("请指定 --start 和 --end 参数.") + sys.exit(1) + + print(f"\n处理日期范围: {start_date} ~ {end_date}") + + # ====== Step 0: GAMMA 二进制 → GeoTIFF 转换 ====== + print("\n" + "="*80) + print("Step 0: GAMMA 二进制 → GeoTIFF 数据转换") + print("="*80) + + staging_dir = os.path.join(workDir, 'GEOC_staging') + staging_root = prepare_gamma_data_for_phasebias( + ifgDir, demDir, masterDate, str(rlks), + staging_dir, n_parallel=n_parallel, + skip_existing=skip_convert + ) + + # ====== 创建配置文件 ====== + print("\n创建配置文件...") + # root_path 指向 staging 目录（包含 interferograms/ 和 metadata/） + config_file = create_config_file(workDir, staging_root, interval, nlook, + num_a, start_date, end_date, estimate_an) + print(f"配置文件: {config_file}") + + # ====== 运行 PhaseBias 流水线 ====== + print("\n" + "="*80) + print("运行 Phase Bias Correction 流水线") + print("="*80) + + # Step 1: 读取数据 + if not run_phasebias_scripts(phasebiasDir, workDir, 'PhaseBias_01_Read_Data.py'): + print("ERROR in Step 1: Read Data") + sys.exit(1) + + # Step 2: 计算闭合环 + if not run_phasebias_scripts(phasebiasDir, workDir, 'PhaseBias_02_Loop_Closures.py'): + print("ERROR in Step 2: Loop Closures") + sys.exit(1) + + # Step 3: 估计标定参数（可选） + if estimate_an: + if not run_phasebias_scripts(phasebiasDir, workDir, 'PhaseBias_03_calibration_pars.py'): + print("ERROR in Step 3: Calibration Parameters") + sys.exit(1) + + # Step 4: 反演 + if not run_phasebias_scripts(phasebiasDir, workDir, 'PhaseBias_04_Inversion.py'): + print("ERROR in Step 4: Inversion") + sys.exit(1) + + # Step 5: 应用校正 + if not run_phasebias_scripts(phasebiasDir, workDir, 'PhaseBias_05_Correction.py'): + print("ERROR in Step 5: Correction") + sys.exit(1) + + print("\n" + "="*80) + print("相位偏差校正完成!") + print(f"输出目录: {workDir}") + print("="*80) + + ut.print_process_time(start_time, time.time()) + + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/phasebias_correction_gamma_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/phasebias_correction_gamma_all.py new file mode 100644 index 0000000..4d5d2c8 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/phasebias_correction_gamma_all.py @@ -0,0 +1,232 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import time +import glob +import argparse +import subprocess +from datetime import datetime + +from pyint import _utils as ut + + +def find_ifgs_by_interval(ifgDir, interval=12): + """Find all interferograms with specified temporal interval + + Args: + ifgDir: interferogram directory + interval: temporal baseline in days (6, 12, or 24) + + Returns: + list: list of date pairs + """ + ifgs = [] + + all_dirs = sorted([d for d in os.listdir(ifgDir) if os.path.isdir(os.path.join(ifgDir, d))]) + + for dirname in all_dirs: + if '-' in dirname: + dates = dirname.split('-') + elif '_' in dirname: + dates = dirname.split('_') + else: + continue + + if len(dates) == 2: + try: + date1 = dates[0] + date2 = dates[1] + + d1 = datetime.strptime(date1, '%Y%m%d') + d2 = datetime.strptime(date2, '%Y%m%d') + days = abs((d2 - d1).days) + + if days == interval: + ifgs.append(dirname) + except ValueError: + continue + + return ifgs + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Apply phase bias correction to all interferograms with + specified temporal interval in a project. + + This script runs the full PhaseBias correction pipeline: + 1. Read interferogram data + 2. Calculate loop closures + 3. Estimate calibration parameters (optional) + 4. Inversion for phase bias terms + 5. Apply correction + +''' + +EXAMPLE = ''' + Usage: + phasebias_correction_gamma_all.py projectName + phasebias_correction_gamma_all.py projectName --interval 12 + phasebias_correction_gamma_all.py projectName --interval 12 --nlook 10 + phasebias_correction_gamma_all.py projectName --interval 24 --estimate-an +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Apply phase bias correction to interferograms.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='projectName for processing.') + parser.add_argument('--interval', type=int, default=12, + help='Data acquisition interval in days (6, 12, or 24). [default: 12]') + parser.add_argument('--nlook', type=int, default=10, + help='Number of looks for multilooking. [default: 10]') + parser.add_argument('--num-a', type=int, default=2, + help='Number of calibration parameters to estimate. [default: 2]') + parser.add_argument('--start', type=str, default=None, + help='Start date (YYYYMMDD). If not specified, will use all available data.') + parser.add_argument('--end', type=str, default=None, + help='End date (YYYYMMDD). If not specified, will use all available data.') + parser.add_argument('--estimate-an', dest='estimate_an', action='store_true', + help='Estimate calibration parameters from data instead of using defaults.') + parser.add_argument('--max-con', type=int, default=5, + help='Maximum number of connections to correct. [default: 5]') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + interval = inps.interval + nlook = inps.nlook + num_a = inps.num_a + estimate_an = inps.estimate_an + max_con = inps.max_con + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + projectDir = scratchDir + '/' + projectName + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + + templateDict = ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + # Check if interferograms with specified interval exist + ifgList = find_ifgs_by_interval(ifgDir, interval) + + print("\n" + "="*80) + print(f"Phase Bias Correction for Project: {projectName}") + print(f"Temporal interval: {interval} days") + print(f"Number of interferograms found: {len(ifgList)}") + print("="*80) + + if len(ifgList) == 0: + print(f"\nERROR: No interferograms found with {interval}-day interval!") + print("Available interferograms:") + all_dirs = sorted([d for d in os.listdir(ifgDir) if os.path.isdir(os.path.join(ifgDir, d))]) + for dirname in all_dirs[:10]: # Show first 10 + if '-' in dirname: + dates = dirname.split('-') + if len(dates) == 2: + try: + d1 = datetime.strptime(dates[0], '%Y%m%d') + d2 = datetime.strptime(dates[1], '%Y%m%d') + days = abs((d2 - d1).days) + print(f" {dirname}: {days} days") + except: + pass + sys.exit(1) + + # Determine date range + if inps.start and inps.end: + start_date = inps.start + end_date = inps.end + else: + # Extract dates from interferogram list + dates = [] + for pair in ifgList: + if '-' in pair: + date_pair = pair.split('-') + dates.extend(date_pair) + elif '_' in pair: + date_pair = pair.split('_') + dates.extend(date_pair) + + dates = sorted(set(dates)) + start_date = dates[0] + end_date = dates[-1] + + print(f"\nDate range: {start_date} to {end_date}") + print(f"Multilooking factor: {nlook}") + print(f"Number of calibration parameters: {num_a}") + print(f"Estimate an from data: {estimate_an}") + print(f"Max connections: {max_con}") + + # Check if correction has already been done + outputDir = projectDir + '/PhaseBiasCorrection' + configFile = outputDir + '/config.txt' + + if os.path.exists(configFile): + print("\n" + "="*80) + print("Found existing PhaseBiasCorrection directory") + print("This script will process ALL interferograms together using the") + print("InSAR_PhaseBias_Correction pipeline.") + print("="*80) + + # Build command + cmd_list = [ + 'phasebias_correction_gamma.py', + projectName, + '--interval', str(interval), + '--nlook', str(nlook), + '--num-a', str(num_a), + '--max-con', str(max_con) + ] + + if start_date: + cmd_list.extend(['--start', start_date]) + if end_date: + cmd_list.extend(['--end', end_date]) + if estimate_an: + cmd_list.append('--estimate-an') + + # Run the phase bias correction script + print("\n" + "="*80) + print("Running phase bias correction...") + print("="*80) + print(f"Command: {' '.join(cmd_list)}") + + try: + result = subprocess.run(cmd_list, check=True, capture_output=False) + print("\n" + "="*80) + print("Phase bias correction completed successfully!") + print(f"Output directory: {outputDir}") + print(f"Corrected interferograms: {outputDir}/GEOC/") + print("="*80) + except subprocess.CalledProcessError as e: + print("\n" + "="*80) + print("ERROR: Phase bias correction failed!") + print("="*80) + sys.exit(1) + + ut.print_process_time(start_time, time.time()) + sys.exit(1) + + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/plot_auto_grd.sh b/.codex_tmp/pyint_variants/no_rescue/pyint/plot_auto_grd.sh new file mode 100644 index 0000000..ac1e8ae --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/plot_auto_grd.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# 描述：自动根据GRD文件生成地图（依赖GMT 6+） +# 用法：./auto_plot_grd.sh <输入.grd> [输出文件名] [选项] + +# 默认参数 +input_grd=$1 +output="map.png" # 默认输出文件名 +projection="M15c" # 默认投影（Mercator，宽度15厘米） +colormap="viridis" # 默认色标 +title="Topography" # 默认标题 +annotate="a" # 默认标注间隔（自动） + +# 解析命令行参数 +while [[ $# -gt 0 ]]; do + case $1 in + -o|--output) output="$2"; shift ;; + -P|--projection) projection="$2"; shift ;; + -C|--colormap) colormap="$2"; shift ;; + -T|--title) title="$2"; shift ;; + -A|--annotate) annotate="$2"; shift ;; + *) ;; + esac + shift +done + +# 检查输入文件是否存在 +if [[ ! -f $input_grd ]]; then + echo "错误：输入文件 $input_grd 不存在！" + exit 1 +fi + +# 从GRD文件获取地理范围 +region=$(gmt grdinfo $input_grd -I- | awk -F'R' '{print $2}') + +# 生成临时CPT色标文件 +gmt makecpt -Cturbo -T$(gmt grdinfo $input_grd -T | awk -F'T' '{print $2}') > tmp.cpt + +# 开始绘图 +gmt begin ${output%.*} pdf,png # 去除扩展名 + # 绘制GRD数据 + gmt grdimage $input_grd -R$region -J$projection -Ctmp.cpt -B$annotate -BWSne+t"$title" + # 添加海岸线 + gmt coast -R$region -J$projection -W0.5p,black -Df + # 添加色标 + gmt colorbar -DJBC+w10c/0.5c+o0/1c -Bxaf -By+l"Elevation (m)" +gmt end show + +# 清理临时文件 +#rm -f tmp.cpt + +echo "地图已生成：$output" diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/plot_geotiff.py b/.codex_tmp/pyint_variants/no_rescue/pyint/plot_geotiff.py new file mode 100644 index 0000000..81fa575 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/plot_geotiff.py @@ -0,0 +1,229 @@ +#! /usr/bin/env python +import argparse +import os +import numpy as np +import matplotlib.pyplot as plt +import rasterio +from rasterio.plot import show +import matplotlib.colors as colors +from matplotlib import cm +import warnings +warnings.filterwarnings('ignore') + +def parse_arguments(): + """解析命令行参数""" + parser = argparse.ArgumentParser(description='绘制GeoTIFF文件') + + # 必需参数 + parser.add_argument('-i', '--input', required=True, + help='输入GeoTIFF文件路径') + + # 输出参数 + parser.add_argument('-o', '--output', + help='输出图片文件路径 (默认: 输入文件名.png)') + + # 绘图参数 + parser.add_argument('--dpi', type=int, default=300, + help='输出图片分辨率 (默认: 300)') + parser.add_argument('--cmap', default='viridis', + help='色彩映射 (默认: viridis)') + parser.add_argument('--title', + help='图片标题 (默认: 使用文件名)') + parser.add_argument('--band', type=int, default=1, + help='要绘制的波段 (默认: 1)') + + # 显示参数 + parser.add_argument('--vmin', type=float, + help='颜色范围最小值') + parser.add_argument('--vmax', type=float, + help='颜色范围最大值') + parser.add_argument('--log', action='store_true', + help='使用对数颜色标尺') + parser.add_argument('--percentile', type=float, nargs=2, + help='使用百分位数设置颜色范围，例如 --percentile 2 98') + + # 图片尺寸 + parser.add_argument('--width', type=float, default=10, + help='图片宽度 (英寸, 默认: 10)') + parser.add_argument('--height', type=float, default=8, + help='图片高度 (英寸, 默认: 8)') + + # 其他选项 + parser.add_argument('--no-colorbar', action='store_true', + help='不显示颜色条') + parser.add_argument('--show', action='store_true', + help='显示图片 (默认只保存)') + + return parser.parse_args() + +def get_default_output_filename(input_file): + """根据输入文件名生成默认输出文件名""" + dir_name = os.path.dirname(input_file) + base_name = os.path.basename(input_file) + name, ext = os.path.splitext(base_name) + return os.path.join(dir_name, f"{name}.png") + +def read_geotiff(file_path, band=1): + """读取GeoTIFF文件""" + if not os.path.exists(file_path): + raise FileNotFoundError(f"文件不存在: {file_path}") + + with rasterio.open(file_path) as src: + # 检查波段数量 + if band > src.count: + raise ValueError(f"文件只有 {src.count} 个波段，无法读取波段 {band}") + + data = src.read(band) + profile = src.profile + bounds = src.bounds + crs = src.crs + nodata = src.nodata + + # 如果有无效值，创建掩码 + if nodata is not None: + data = np.ma.masked_where(data == nodata, data) + + return data, profile, bounds, crs, nodata + +def plot_geotiff(args): + """主绘图函数""" + # 设置默认输出文件名 + if args.output is None: + args.output = get_default_output_filename(args.input) + + # 设置默认标题 + if args.title is None: + args.title = os.path.basename(args.input) + + print(f"读取文件: {args.input}") + print(f"输出文件: {args.output}") + print(f"使用波段: {args.band}") + + # 读取数据 + data, profile, bounds, crs, nodata = read_geotiff(args.input, args.band) + + print(f"数据形状: {data.shape}") + print(f"数据类型: {data.dtype}") + print(f"坐标参考系统: {crs}") + print(f"数据范围: {bounds}") + + if nodata is not None: + print(f"无效值: {nodata}") + valid_data = data[~data.mask] if hasattr(data, 'mask') else data[data != nodata] + else: + valid_data = data.compressed() if hasattr(data, 'mask') else data.flatten() + + print(f"有效数据点数: {len(valid_data)}") + print(f"数据范围: {valid_data.min():.6f} - {valid_data.max():.6f}") + + # 设置颜色范围 + if args.percentile: + vmin = np.percentile(valid_data, args.percentile[0]) + vmax = np.percentile(valid_data, args.percentile[1]) + print(f"使用百分位数范围: {args.percentile[0]}% - {args.percentile[1]}%") + print(f"对应数值范围: {vmin:.6f} - {vmax:.6f}") + elif args.vmin is not None and args.vmax is not None: + vmin, vmax = args.vmin, args.vmax + print(f"使用指定范围: {vmin} - {vmax}") + else: + vmin, vmax = valid_data.min(), valid_data.max() + print(f"使用数据范围: {vmin:.6f} - {vmax:.6f}") + + # 创建图形 + fig, ax = plt.subplots(1, 1, figsize=(args.width, args.height)) + + # 选择颜色映射 + try: + cmap = plt.get_cmap(args.cmap) + except: + print(f"警告: 色彩映射 '{args.cmap}' 不存在，使用默认的 'viridis'") + cmap = plt.get_cmap('viridis') + + # 选择颜色标准化方式 + if args.log: + norm = colors.LogNorm(vmin=vmin, vmax=vmax) + print("使用对数颜色标尺") + # 对于对数标准化，不能同时传递vmin/vmax参数 + show_kwargs = {'norm': norm, 'cmap': cmap} + else: + # 对于线性标准化，可以直接传递vmin/vmax参数 + show_kwargs = {'vmin': vmin, 'vmax': vmax, 'cmap': cmap} + + # 使用rasterio的show函数显示地理参考数据 + if crs is not None: + with rasterio.open(args.input) as src: + # 使用rasterio的show函数，它能够正确处理地理参考 + # 注意：show()返回的是Axes对象，不是mappable对象 + image_axes = show( + (src, args.band), + ax=ax, + **show_kwargs + ) + + # 从axes对象中获取图像对象 + if hasattr(image_axes, 'images') and len(image_axes.images) > 0: + im = image_axes.images[0] + else: + # 如果无法获取图像对象，创建一个ScalarMappable用于颜色条 + im = cm.ScalarMappable(norm=norm if args.log else colors.Normalize(vmin=vmin, vmax=vmax), + cmap=cmap) + im.set_array([]) # 设置一个空数组 + else: + # 如果没有地理参考，使用普通的imshow + if args.log: + im = ax.imshow(data, norm=norm, cmap=cmap) + else: + im = ax.imshow(data, vmin=vmin, vmax=vmax, cmap=cmap) + + # 设置标题和标签 + ax.set_title(args.title, fontsize=14, fontweight='bold') + + # 添加颜色条 + if not args.no_colorbar: + # 确保im是一个mappable对象 + if not hasattr(im, 'set_array'): + # 如果不是mappable对象，创建一个 + im = cm.ScalarMappable(norm=norm if args.log else colors.Normalize(vmin=vmin, vmax=vmax), + cmap=cmap) + im.set_array([]) # 设置一个空数组 + + cbar = plt.colorbar(im, ax=ax, shrink=0.8) + cbar.set_label('值', rotation=270, labelpad=15) + + # 添加网格 + ax.grid(True, alpha=0.3) + + # 设置坐标轴标签 + if crs is not None: + ax.set_xlabel('经度') + ax.set_ylabel('纬度') + else: + ax.set_xlabel('X') + ax.set_ylabel('Y') + + # 保存图片 + plt.tight_layout() + plt.savefig(args.output, dpi=args.dpi, bbox_inches='tight') + print(f"图片已保存: {args.output}") + + # 显示图片 + if args.show: + plt.show() + + plt.close() + + return args.output + +def main(): + """主函数""" + args = parse_arguments() + + try: + output_file = plot_geotiff(args) + print(f"成功生成图片: {output_file}") + except Exception as e: + print(f"处理过程中发生错误: {e}") + raise + +if __name__ == "__main__": + main() diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/pot_gamma_subset.py b/.codex_tmp/pyint_variants/no_rescue/pyint/pot_gamma_subset.py new file mode 100644 index 0000000..100c3f7 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/pot_gamma_subset.py @@ -0,0 +1,191 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2022, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +from pyint import _utils as ut + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Pixel offset tracking for co-registered Images. TOPs should be deramped in advance. + +''' + +EXAMPLE = ''' + Usage: + pot_gamma_subset.py projectName Mdate Sdate subset + pot_gamma_subset.py PacayaT163TsxHhA 20150102 20150601 0102 +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Pixel offset tracking based Azimuth/Range dispalcement estimation.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('Mdate',help='Master date.') + parser.add_argument('Sdate',help='Slave date.') + parser.add_argument('Subset',help='Subset name, e.g., 0102') + + + inps = parser.parse_args() + return inps + + +def main(argv): + + start_time = time.time() + inps = cmdLineParse() + Mdate = inps.Mdate + Sdate = inps.Sdate + subset = inps.Subset + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + + if 'POT_rwin' in templateDict: + POT_rwin = templateDict['POT_rwin'] + else: + POT_rwin = '160' + print('POT range window sise: ' + POT_rwin) + + if 'POT_awin' in templateDict: + POT_awin = templateDict['POT_awin'] + else: + POT_awin = '160' + print('POT azimuth window sise: ' + POT_awin) + + if 'POT_astep' in templateDict: + POT_astep = templateDict['POT_astep'] + else: + POT_astep = azlks + + if 'POT_rstep' in templateDict: + POT_rstep = templateDict['POT_rstep'] + else: + POT_rstep = rlks + + print('POT range steps: ' + POT_rstep) + print('POT azimuth steps: ' + POT_astep) + + projectDir = scratchDir + '/' + projectName + demDir = scratchDir + '/' + projectName + '/DEM' + + slcDir = scratchDir + '/' + projectName + '/SLC' + rslcDir = scratchDir + '/' + projectName + '/RSLC' + ifgDir = projectDir + '/ifgrams' + if not os.path.isdir(ifgDir): os.mkdir(ifgDir) + + Pair = Mdate + '-' + Sdate + workDir = ifgDir + '/' + Pair + if not os.path.isdir(workDir): os.mkdir(workDir) + + ####################################################################### + Mamp = rslcDir + '/' + Mdate + '/' + Mdate + '_' + subset + '_' + rlks + 'rlks.amp' + MampPar = rslcDir + '/' + Mdate + '/' + Mdate + '_' + subset + '_' + rlks + 'rlks.amp.par' + Samp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + subset + '_' + rlks + 'rlks.amp' + SampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + subset + '_' + rlks + 'rlks.amp.par' + + Mrslc = rslcDir + '/' + Mdate + '/' + Mdate + '_' + subset + '.rslc' + MrslcPar = rslcDir + '/' + Mdate + '/' + Mdate + '_' + subset + '.rslc.par' + Srslc = rslcDir + '/' + Sdate + '/' + Sdate + '_' + subset + '.rslc' + SrslcPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + subset + '.rslc.par' + + #HGT = demDir + '/' + masterDate + '_' + rlks + 'rlks.rdc.dem' + #MasterPar = rslcDir + '/' + masterDate + '/' + masterDate + '.rslc.par' + + ################# copy file for parallel processing ########################## + #Mamp = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp' + #MampPar = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + #Samp = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp' + #SampPar = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + #if not templateDict['diff_all_parallel'] == '1': + + # Mrslc = workDir + '/' + Mdate + '.rslc' + # MrslcPar = workDir + '/' + Mdate + '.rslc.par' + # Srslc = workDir + '/' + Sdate + '.rslc' + # SrslcPar = workDir + '/' + Sdate + '.rslc.par' + # ut.copy_file(Mrslc0,Mrslc) + # ut.copy_file(MrslcPar0,MrslcPar) + # ut.copy_file(Srslc0,Srslc) + # ut.copy_file(SrslcPar0,SrslcPar) + + #else: + + # Mrslc = Mrslc0 + # MrslcPar = MrslcPar0 + # Srslc = Srslc0 + # SrslcPar = SrslcPar0 + # HGT = HGT0 + # MasterPar = MasterPar0 + + #ut.copy_file(Mamp0,Mamp) + #ut.copy_file(MampPar0,MampPar) + #ut.copy_file(Samp0,Samp) + #ut.copy_file(SampPar0,SampPar) + + #ut.copy_file(HGT0,HGT) + #ut.copy_file(MasterPar0,MasterPar) + + ############################################################################ + + OFF = workDir + '/' + Pair + '_' + subset + '_' + rlks + 'rlks_pot.off' + OFFS = workDir + '/' + Pair + '_' + subset + '_' + rlks + 'rlks_pot.offs' + CCP = workDir + '/' + Pair + '_' + subset + '_' + rlks + 'rlks_pot.ccp' + COFFS = workDir + '/' + Pair + '_' + subset + '_' + rlks + 'rlks_pot.coffs' + COFFS_FILT = workDir + '/' + Pair + '_' + subset + '_' + rlks + 'rlks_pot_filt.coffs' + POTA = workDir + '/' + Pair + '_' + subset + '_' + rlks + 'rlks_pot.az' + POTR = workDir + '/' + Pair + '_' + subset + '_' + rlks + 'rlks_pot.rg' + + call_str = 'create_offset '+ MrslcPar + ' ' + SrslcPar + ' ' + OFF + ' 1 ' + rlks + ' ' + azlks + ' 0' + os.system(call_str) + + call_str = 'offset_pwr_tracking ' + Mrslc + ' ' + Srslc + ' ' + MrslcPar + ' ' + SrslcPar + ' ' + OFF + ' ' + OFFS + ' ' + CCP + ' ' + POT_rwin + ' ' + POT_awin + ' - 2 0.05 ' + POT_rstep + ' ' + POT_astep + ' - - - - - - ' + os.system(call_str) + + call_str = 'offset_tracking ' + OFFS + ' ' + CCP + ' ' + MrslcPar + ' ' + OFF + ' ' + COFFS + ' - 1 - 1' + os.system(call_str) + + nWidth = ut.read_gamma_par(OFF, 'read', 'interferogram_width') + + #call_str = 'adf ' + COFFS + ' ' + COFFS_FILT + ' ' + CCP + ' ' + nWidth + #os.system(call_str) + + call_str = 'cpx_to_real ' + COFFS + ' ' + POTR + ' ' + nWidth + ' 0 ' + os.system(call_str) + + call_str = 'cpx_to_real ' + COFFS + ' ' + POTA + ' ' + nWidth + ' 1 ' + os.system(call_str) + + call_str = 'rasdt_pwr ' + POTR + ' ' + Mamp + ' ' + nWidth + ' - - - - - - - BuYlRd.cm' + os.system(call_str) + + call_str = 'rasdt_pwr ' + POTA + ' ' + Mamp + ' ' + nWidth + ' - - - - - - - BuYlRd.cm' + os.system(call_str) + + print("Pixel offset tracking is Done!") + ut.print_process_time(start_time, time.time()) + #sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/pot_gamma_subset_combine.py b/.codex_tmp/pyint_variants/no_rescue/pyint/pot_gamma_subset_combine.py new file mode 100644 index 0000000..026b510 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/pot_gamma_subset_combine.py @@ -0,0 +1,372 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import time +import numpy as np +import os +import sys +import argparse +import h5py + +from pyint import _utils as ut +from scipy.interpolate import griddata +from scipy.interpolate import NearestNDInterpolator + +#def resamp2d_near(row0,col0,z0,row1,col1): + +# xx0, yy0 = np.meshgrid((np.arange(col0) + 1), (np.arange(row0) + 1)) +# xx1, yy1 = np.meshgrid((np.arange(col1) + 1), (np.arange(row1) + 1)) + +# xx00 = xx0/col0*col1; yy00 = yy0/row0*row1 +# xx = xx00.flatten(); yy = yy00.flatten(); zz = z0.flatten() +# interp = NearestNDInterpolator(list(zip(xx, yy)), zz) +# z1 = interp(xx1, yy1) + +# return z1 + +def generate_name(i,j): + + if len(str(int(i)))==1: + i0 = '0' + str(int(i)) + else: + i0 = str(int(i)) + + if len(str(int(j)))==1: + j0 = '0' + str(int(j)) + else: + j0 = str(int(j)) + + name0 = i0+j0 + return name0 + +def reduce_samp(xx,yy,zz,xg,yg,extend): + + max_x = np.max(xg)+extend; min_x = np.min(xg)-extend # e.g. extend 100 + max_y = np.max(yg)+extend; min_y = np.min(yg)-extend # e.g., extend 100 + + xx1 = xx[((min_x (2/3*int(awidth)): + Ap_samp1 = Ap_samp + else: + Ap_samp1 = Ap_samp[0:(Na0-1)] # last batch with '-' means to the end + + if LR_end > (2/3*int(rwidth)): + Rp_samp1 = Rp_samp + else: + Rp_samp1 = Rp_samp[0:(Nr0-1)] # last batch with '-' means to the end + return Ap_samp1, Rp_samp1 + +def subset2coord(subset, astep, rstep, nLine, nWidth, awidth, rwidth, extend): + # default awidth = 5000 rwidth = 5000 extend = 200 + # astep: azimuth multilook numbers + # rstep: range multilook numbers + extend = int(extend) + Ap_samp1, Rp_samp1 = get_startSamp(nLine, nWidth, awidth, rwidth) + Na = len(Ap_samp1); Nr = len(Rp_samp1) + ii = int(subset[0:2]); jj = int(subset[2:4]) + #print(ii); print(jj) + rstart = str(Rp_samp1[jj]); astart = str(Ap_samp1[ii]); + #print(nLine);print(Ap_samp1);print(Rp_samp1) + + if not ii==0: + astart0 = str(int(int(astart) - extend)) # extend 200 to avoid edge effect + else: + astart0 = astart + + if not jj==0: + rstart0 = str(int(int(rstart) - extend)) # extend 200 to avoid edge effect + else: + rstart0 = rstart + + #print('astart') + #print(astart);print(Ap_samp1[Na-1]) + if astart == str(Ap_samp1[Na-1]): + awidth0 = '-' + aend0 = str(nLine) + else: + awidth0 = str(int(int(awidth) + extend)) # extend 200 to avoid edge effect + aend0 = str(int(int(astart0) + int(awidth) + extend - 1)) + + if rstart == str(Rp_samp1[Nr-1]): + rwidth0 = '-' + rend0 = str(nWidth) + else: + rwidth0 = str(int(int(rwidth) + extend)) # extend 200 to avoid edge effect + rend0 = str(int(int(rstart0) + int(rwidth) + extend - 1)) + + xx0 = np.arange(int(rstart0), int(rend0), int(rstep)); Nx0 = len(xx0); xx1 = xx0[0:Nx0-1] + yy0 = np.arange(int(astart0), int(aend0), int(astep)); Ny0 = len(yy0); yy1 = yy0[1:Ny0] + #print(rstart0); print(rend0); print(astart0); print(aend0) + + rwidth_total = int(int(rend0) - int(rstart0) + 1) + awidth_total = int(int(aend0) - int(astart0) + 1) + if np.mod(rwidth_total,int(rstep))==0: + xx1 = xx0 + else: + xx1 = xx0[0:Nx0-1] + + if np.mod(awidth_total,int(astep))==0: + yy1 = yy0 + else: + yy1 =yy0[0:Ny0-1] + + return xx1,yy1 + +def read_gammadata(file0,nWidth0,nLength0): + data0 = np.fromfile(file0,dtype='>f4',count=int(nLength0)*int(nWidth0)).reshape(int(nLength0), int(nWidth0)) + return data0 + +def write_h5(datasetDict, out_file, metadata=None, ref_file=None, compression=None): + + if os.path.isfile(out_file): + print('delete exsited file: {}'.format(out_file)) + os.remove(out_file) + + print('create HDF5 file: {} with w mode'.format(out_file)) + dt = h5py.special_dtype(vlen=np.dtype('float64')) + + + with h5py.File(out_file, 'w') as f: + for dsName in datasetDict.keys(): + data = datasetDict[dsName] + ds = f.create_dataset(dsName, + data=data, + compression=compression) + + for key, value in metadata.items(): + f.attrs[key] = str(value) + #print(key + ': ' + value) + print('finished writing to {}'.format(out_file)) + + return out_file + +INTRODUCTION = ''' +------------------------------------------------------------------- + Combine subset of POT results from pot_gamma_subset_jobs.py +''' + +EXAMPLE = ''' + Usage: + pot_gamma_subset_combine.py projectName Mdate Sdate + pot_gamma_subset_combine.py PacayaT163TsxHhA 20150601 20150613 +------------------------------------------------------------------- +''' + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Combine subset of POT results from pot_gamma_subset_jobs.py',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('Mdate',help='First date.') + parser.add_argument('Sdate',help='Second date.') + #parser.add_argument('--out',dest ='out', help='Output file name.') + parser.add_argument('--rwidth',dest ='rwidth', default = '5000', help='Patch range size.') + parser.add_argument('--awidth',dest ='awidth', default = '5000', help='Patch azimuth size.') + parser.add_argument('--extend',dest ='extend', default = '200', help='Patch extend to ensure overlap regions.') + + inps = parser.parse_args() + return inps + + +def main(argv): + + inps = cmdLineParse() + start_time = time.time() + projectName = inps.projectName + rwidth = inps.rwidth; awidth = inps.awidth; extend = inps.extend + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + processDir = scratchDir + '/' + projectName + "/PROCESS" + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + '/RSLC' + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + Mdate = inps.Mdate; Sdate = inps.Sdate + Pair = Mdate + '-' + Sdate + workDir = ifgDir + '/' + Pair + MslcPar = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc.par' + + Samp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + SWidth = ut.read_gamma_par(SampPar, 'read', 'range_samples') + SLine = ut.read_gamma_par(SampPar, 'read', 'azimuth_lines') + SampData = read_gammadata(Samp,SWidth,SLine) + + nWidth = ut.read_gamma_par(MslcPar, 'read', 'range_samples'); Rp_samp_large0 = np.arange(1,int(nWidth),int(rlks)) + nLine = ut.read_gamma_par(MslcPar, 'read', 'azimuth_lines'); Ap_samp_large0 = np.arange(1,int(nLine),int(azlks)) + Nr0 = len(Rp_samp_large0); Na0 = len(Ap_samp_large0) + + if np.mod(int(nWidth),int(rlks))==0: + Rp_samp_large = Rp_samp_large0 + else: + Rp_samp_large = Rp_samp_large0[0:Nr0-1] + + if np.mod(int(nLine),int(azlks))==0: + Ap_samp_large= Ap_samp_large0 + else: + Ap_samp_large =Ap_samp_large0[0:Na0-1] + + rr,aa = np.meshgrid(Rp_samp_large,Ap_samp_large) + + Nr = len(Rp_samp_large); Na = len(Ap_samp_large) + print('Total samples along range and azimuth: ' + str(Nr) + ' ' + str(Na)) + + Ap_samp1,Rp_samp1 = get_startSamp(nLine, nWidth, awidth, rwidth) + Na = len(Ap_samp1); Nr = len(Rp_samp1) + + xx_total = []; yy_total = []; az_total = []; rg_total = []; cc_total = []; + for i in range(Na): + for j in range(Nr): + subset0 = generate_name(i,j) + xx0,yy0 = subset2coord(subset0, azlks, rlks, nLine, nWidth, awidth, rwidth, extend) + #print(len(xx0)); print(len(yy0)) + [xx0,yy0] = np.meshgrid(xx0,yy0) + az0 = workDir + '/' + Pair + '_' + subset0 + '_' + rlks + 'rlks_pot.az' + rg0 = workDir + '/' + Pair + '_' + subset0 + '_' + rlks + 'rlks_pot.rg' + cc0 = workDir + '/' + Pair + '_' + subset0 + '_' + rlks + 'rlks_pot.ccp' + off0 = workDir + '/' + Pair + '_' + subset0 + '_' + rlks + 'rlks_pot.off' + nLine0 = ut.read_gamma_par(off0, 'read', 'interferogram_azimuth_lines') + nWidth0 = ut.read_gamma_par(off0, 'read', 'interferogram_width') + #print(nWidth0);print(nLine0) + if os.path.isfile(az0): + if os.path.getsize(az0)>0: + az_data0 = read_gammadata(az0,nWidth0,nLine0) + rg_data0 = read_gammadata(rg0,nWidth0,nLine0) + cc_data0 = read_gammadata(cc0,nWidth0,nLine0) + + xx_total.extend(list(xx0.flatten()));yy_total.extend(list(yy0.flatten())) + az_total.extend(list(az_data0.flatten())) + rg_total.extend(list(rg_data0.flatten())) + cc_total.extend(list(cc_data0.flatten())) + else: + data0 = np.zeros((int(nWidth0),int(nLine0)),dtype='float32') + xx_total.extend(list(data0.flatten()));yy_total.extend(list(data0.flatten())) + az_total.extend(list(data0.flatten())) + rg_total.extend(list(data0.flatten())) + cc_total.extend(list(data0.flatten())) + + xx_total = np.asarray(xx_total); #print(xx_total.shape) + yy_total = np.asarray(yy_total); #print(yy_total.shape) + az_total = np.asarray(az_total); #print(az_total.shape) + rg_total = np.asarray(rg_total); #print(rg_total.shape) + cc_total = np.asarray(cc_total); #print(cc_total.shape) + + xxa = xx_total[az_total!=0]; yya = yy_total[az_total!=0]; az_total1 = az_total[az_total!=0] + xxr = xx_total[rg_total!=0]; yyr = yy_total[rg_total!=0]; rg_total1 = rg_total[rg_total!=0] + xxc = xx_total[cc_total!=0]; yyc = yy_total[cc_total!=0]; cc_total1 = cc_total[cc_total!=0] + + #start_time = time.time() + #az_grid_large = interp_split(xxa,yya,az_total1,Rp_samp_large,Ap_samp_large,2); print('1 finish') + #rg_grid_large = interp_split(xx_total,yy_total,rg_total1,Rp_samp_large,Ap_samp_large,2) + #cc_grid_large = interp_split(xx_total,yy_total,cc_total1,Rp_samp_large,Ap_samp_large,2) + + #start_time = time.time() + #interp_az = NearestNDInterpolator(list(zip(xxa, yya)), az_total1) + #az_grid_large = interp_az(rr, aa) + #interp_rg = NearestNDInterpolator(list(zip(xxr, yyr)), rg_total1) + #rg_grid_large = interp_rg(rr, aa) + #interp_cc = NearestNDInterpolator(list(zip(xxc, yyc)), cc_total1) + #cc_grid_large = interp_cc(rr, aa) + #ut.print_process_time(start_time, time.time()) + + method0 = 'nearest' # keep 0 values to make good mask + print('Start to combine all of the sub-patches ...') + start_time = time.time() + az_grid_large = griddata((xxa,yya),az_total1,(rr,aa),method=method0); print('Azimuth interpolate finish') + #ut.print_process_time(start_time, time.time()) + #start_time = time.time() + rg_grid_large = griddata((xxr,yyr),rg_total1,(rr,aa),method=method0); print('Range interpolate finish') + #ut.print_process_time(start_time, time.time()) + #start_time = time.time() + cc_grid_large = griddata((xxc,yyc),cc_total1,(rr,aa),method=method0); print('CCP interpolate finish') + ut.print_process_time(start_time, time.time()) + + az_grid_large[SampData==0]=0 + rg_grid_large[SampData==0]=0 + cc_grid_large[SampData==0]=0 + + row1,col1 = rr.shape + meta = dict() + meta['WIDTH'] = str(col1); meta['LENGTH'] = str(row1); meta['UNIT'] = 'm' + meta['FILE_TYPE'] ='offset_tracking'; meta['DATE12'] = Pair + + datasetDict = dict() + datasetDict['azimuth'] = az_grid_large + #datasetDict['range'] = rg_grid_large + #datasetDict['ccp'] = cc_grid_large + #if inps.out: + # out_file = inps.out + #else: + out_file = Pair + '_pot_az.h5' + write_h5(datasetDict, out_file, metadata=meta, ref_file=None, compression=None) + + datasetDict = dict() + #datasetDict['azimuth'] = az_grid_large + datasetDict['range'] = rg_grid_large + #datasetDict['ccp'] = cc_grid_large + out_file = Pair + '_pot_rg.h5' + write_h5(datasetDict, out_file, metadata=meta, ref_file=None, compression=None) + + + datasetDict = dict() + #datasetDict['azimuth'] = az_grid_large + #datasetDict['range'] = rg_grid_large + datasetDict['coherence'] = cc_grid_large + #meta['FILE_TYPE'] = 'coherence' + out_file = Pair + '_pot_cc.h5' + write_h5(datasetDict, out_file, metadata=meta, ref_file=None, compression=None) + print("POT calculation done: " + Pair) + #sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/pot_gamma_subset_jobs.py b/.codex_tmp/pyint_variants/no_rescue/pyint/pot_gamma_subset_jobs.py new file mode 100644 index 0000000..f6bd3c7 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/pot_gamma_subset_jobs.py @@ -0,0 +1,95 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2022, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +from pyint import _utils as ut + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Pixel offset tracking for co-registered Images. TOPs should be deramped in advance. + +''' + +EXAMPLE = ''' + Usage: + pot_gamma_subset_jobs.py projectName Mdate Sdate --memory memory_single_job --walltime walltime_single_job + pot_gamma_subset_jobs.py PacayaT163TsxHhA 20150102 20150601 --memory 5000 --walltime 00:30:00 +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Pixel offset tracking based Azimuth/Range dispalcement estimation.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('Mdate',help='Master date.') + parser.add_argument('Sdate',help='Slave date.') + parser.add_argument('--memory',dest = 'memory',default = '5000', help='memory to be allocated for one job, default: 5GB') + parser.add_argument('--walltime',dest = 'walltime',default = '00:30:00', help='walltime to be allocated for one single job') + inps = parser.parse_args() + inps = parser.parse_args() + return inps + + +def main(argv): + + start_time = time.time() + inps = cmdLineParse() + Mdate = inps.Mdate + Sdate = inps.Sdate + + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + + slcDir = scratchDir + '/' + projectName + '/SLC' + rslcDir = scratchDir + '/' + projectName + '/RSLC' + + MrslcDir = rslcDir + '/' + Mdate + SrslcDir = rslcDir + '/' + Sdate + Pair = Mdate + '-' + Sdate + batch_txt = scratchDir + '/' + projectName + '/ifgrams/' + Pair + '/Batch_POT' + workDir = scratchDir + '/' + projectName + '/ifgrams/' + Pair + if not os.path.isdir(workDir): + os.mkdir(workDir) + os.chdir(workDir) + if os.path.isfile(batch_txt): + os.remove(batch_txt) + + mrslc_list = glob.glob(MrslcDir + '/*_0*.rslc');mrslc_list.sort(); nn = len(mrslc_list) + print('Total patch numbers: ' + str(nn)) + + for i in range(nn): + base0 = os.path.basename(mrslc_list[i]); subset1 = base0.split('_')[1]; subset0 = subset1.split('.rslc')[0] + str0 = 'pot_gamma_subset.py ' + projectName + ' ' + Mdate + ' ' + Sdate + ' ' + subset0 + call_str = 'echo ' + str0 + ' >>' + batch_txt + os.system(call_str) + + call_str = 'sbatch_jobs.py ' + batch_txt + ' --memory ' + inps.memory + ' --walltime ' + inps.walltime + ' --job-name ' + Pair + '_POT' + os.system(call_str) + + + print("Pixel offset tracking is Done!") + ut.print_process_time(start_time, time.time()) + #sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/process_tsifg.py b/.codex_tmp/pyint_variants/no_rescue/pyint/process_tsifg.py new file mode 100644 index 0000000..7f3d95d --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/process_tsifg.py @@ -0,0 +1,148 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Central South University & University of Miami ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def is_number(s): + try: + int(s) + return True + except ValueError: + return False + + +def ras2jpg(input, strTitle): + call_str = "convert " + input + ".ras " + input + ".jpg" + os.system(call_str) + call_str = "convert " + input + ".jpg -resize 250 " + input + ".thumb.jpg" + os.system(call_str) + call_str = "convert " + input + ".jpg -resize 500 " + input + ".bthumb.jpg" + os.system(call_str) + call_str = "$INT_SCR/addtitle2jpg.pl " + input + ".thumb.jpg 14 " + strTitle + os.system(call_str) + call_str = "$INT_SCR/addtitle2jpg.pl " + input + ".bthumb.jpg 24 " + strTitle + os.system(call_str) + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +def write_template(File, Str): + f = open(File,'a') + f.write(Str) + f.close() + +def write_run_coreg_all(projectName,master,slavelist,workdir): + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + run_coreg_all = projectDir + "/run_coreg_all" + f_coreg = open(run_coreg_all,'w') + + for kk in range(len(slavelist)): + str_coreg = "GenOff_Gamma.py " + projectName + ' ' + master + ' ' + slavelist[kk] + ' ' + workdir + '\n' + f_coreg.write(str_coreg) + f_coreg.close() + + +def usage(): + print(''' +****************************************************************************************************** + + Process time series of interferograms from downloading data or SLC images. + + usage: + + process_tsifg.py projectName + + e.g. process_tsifg.py PacayaT163TsxHhA + process_tsifg.py PacayaT163S1A + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==2: + if argv[0] in ['-h','--help']: usage(); sys.exit(1) + else: projectName=sys.argv[1] + else: + usage();sys.exit(1) + + if 'S1' in projectName: + call_str='process_tsifg_sen.py ' + projectName + else: + call_str='process_tsifg_gamma.py ' + projectName + + os.system(call_str) + + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + + + + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/psokinv2sdm.py b/.codex_tmp/pyint_variants/no_rescue/pyint/psokinv2sdm.py new file mode 100644 index 0000000..2da39ea --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/psokinv2sdm.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python + +import numpy as np +import os +import sys +import argparse +import subprocess +from pyint import _utils as ut + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='convert psokinv quadtree result to SDM.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('inputfile', help='psokinv quadtree file name ') + parser.add_argument('outfile', help=' file name for SDM ') + inps = parser.parse_args() + return inps +INTRODUCTION = ''' +------------------------------------------------------------------- + create_files for SDM software running file by generated by psokinv +''' + +EXAMPLE = """Usage: + + PSOKINV2SDM.py inputfile outfile + + create_psokinv.py 20241015_20250107.inp 20241015_20250107_los_for_sdm +------------------------------------------------------------------- +""" + +def phitheta2neu(phi,theta): + # + import numpy as np + # + deg2rad=np.pi/180. + # + theta=theta*deg2rad; phi=phi*deg2rad; + # + vn= np.sin(theta)*np.sin(phi) + ve=-np.sin(theta)*np.cos(phi) + vu= np.cos(theta) + # + return vn,ve,vu +# +def neu2phitheta(vn,ve,vu): + # + import numpy as np + # + rad2deg=180./np.pi; + # + theta=np.arccos(vu)*rad2deg; phi=np.arctan2(vn,-ve)*rad2deg; + # + return theta,phi +# +def psokinv2sdm(file,file_out): + # + import numpy as np + # + data=np.genfromtxt(file); + length=len(data) + inc=[] + azi=[] + out_data={} + for i in range(length): + (inc,azi)=neu2phitheta(data[i,4],data[i,3],data[i,5]); + out_data[i]=(data[i,0],data[i,1],data[i,2],inc,azi); + data1=np.array(list(out_data.values())) + txt_data = np.savetxt(file_out,data1,fmt='%.8f') + + +def main(argv): + inps = cmdLineParse() + input_file = inps.inputfile + out_file = inps.outfile + psokinv2sdm(input_file, out_file) + print("convert psokinv products to sdm is done!") + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/pyint.template b/.codex_tmp/pyint_variants/no_rescue/pyint/pyint.template new file mode 100644 index 0000000..fe85f8b --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/pyint.template @@ -0,0 +1,206 @@ +# *********************************** PyINT Template *******************************************# +# Template for PyINT v2.1 +# Please fill in the parameters according to your project requirements + +# ========================= Basic parameters ======================== + +# DEM file path (if not provided, SRTM will be downloaded automatically from OpenTopography) +#DEM = /path/to/your/dem.dem + +# Master date for interferometric processing +masterDate = YYYYMMDD # Master date in YYYYMMDD format + +# Swath and burst parameters (for Sentinel-1 TOPS mode only) +start_swath = 1 # Starting swath number +end_swath = 3 # Ending swath number +start_burst = 1 # Starting burst position (relative to master image) +end_burst = 9 # Ending burst position (relative to master image) + +# Multilooking parameters +range_looks = 20 # Multi-look numbers in range direction +azimuth_looks = 4 # Multi-look numbers in azimuth direction + +# ========================= Download Data ======================== + +# Data source and search parameters +Data_Hub = ASF # Data download source: SSARA, Scihub, ASF +sensor = Sentinel-1A # Envisat, ERS-1/2, Sentinel-1A/B, etc. +track = 128 # Track number of the SAR image +frame = 89 # Frame number [defined according to ASF] +start_time = YYYY-MM-DD # Start date for data search +end_time = YYYY-MM-DD # End date for data search + +# ASF download specific parameters +asf_platform = Sentinel-1A # Platform: Sentinel-1A, Sentinel-1B +asf_beam_mode = IW # Beam mode: IW (Interferometric Wide) +asf_polarization = VV # Polarization: VV, VH, HH, HV +asf_flight_direction = ascending # Flight direction: ascending, descending +asf_bbox = W,S,E,N # Bounding box: West,South,East,North + +# ========================= DEM Generation ======================== + +# DEM download parameters (for OpenTopography) +opentopo_api_key = YOUR_API_KEY # OpenTopography API key +opentopo_dem_type = SRTMGL1 # DEM type: SRTMGL1(30m), SRTMGL3(90m), NASADEM(30m), COP30, COP90 +opentopo_auto_tile = 1 # Auto-tile large regions (recommended: 1) + +# DEM processing parameters +dem_lat_ovr = 1 # Latitude oversampling (as to SRTM-1: 0.5 for 60m, 2 for 15m) +dem_lon_ovr = 1 # Longitude oversampling + +# ========================= Interferometry Parameters ======================== + +# Spectral filtering +Igram_Spsflg = 1 # Implement range spectral filtering +rbw_min = 1 # Minimum range bandwidth fraction (0.1 --> 1.0) + +# Coregistration parameters +rwin4cor = 256 # Range window length for coregistration +azwin4cor = 256 # Azimuth window length for coregistration +rsample4cor = 32 # Range samples for fitting coregistration parameters +azsample4cor = 32 # Azimuth samples for fitting coregistration parameters +thresh4cor = 0.3 # Coherence threshold (GAMMA 2016+, SNR for older versions) + +# Coherence estimation +Igram_Cor_rwin = 5 # Range window length for cc_wave coherence estimation +Igram_Cor_awin = 5 # Azimuth window length for cc_wave coherence estimation +Igram_Cor_Win = 5 # Window for cc estimation in adf +adf_alpha = 0.4 # Alpha for Gold-Stein filtering + +# ========================= Simulation Phase ======================== + +Igram_Flag_TDM = N # Y for Tandem-X +Simphase_rpos = - +Simphase_azpos = - +Simphase_rwin = 256 +Simphase_azwin = 256 +Simphase_thresh = - + +# ========================= Unwrapping ======================== + +mcf_triangular = 1 # Triangular type of mcf [0: regular; 1: delaunay] +unwrap_patr = 1 # Unwrap patches in range direction +unwrap_pataz = 1 # Unwrap patches in azimuth direction +unwrapThreshold = 0.05 # Coherence threshold for unwrapping +auto_unw = 1 # Auto or manual reference point [1: auto; 0: manual] +make_mask = 0 # Make mask based on coherence [0: don't mask; 1: make mask] +init_flag = 1 # Flag to set phase at reference point [0: use initial; 1: set to 0.0] +r_refer = - # Phase reference point range offset +a_refer = - # Phase reference point azimuth offset + +# ========================= Geocoding ======================== + +geo_interp = 0 # Interpolation method [0: nearest; 1: bicubic spline] + +# ========================= Satellite Configuration ======================== + +satelite = S1A # Satellite: S1A, S1B, CSK, TSX, ALOS2, ALOS, ENVISAT + +# ========================= Pair Selection ======================== + +network_method = sbas # Network method: sbas, sequential, delaunay, stars +endDate = YYYYMMDD # Exclude dates after this date +startDate = YYYYMMDD # Exclude dates before this date +conNumb = 15 # Connect number for sequential network +max_tb = 365 # Maximum temporal baseline (days) +max_sb = 100 # Maximum spatial baseline (meters) +min_tb = 1 # Minimum temporal baseline (days) +min_sb = 0 # Minimum spatial baseline (meters) + +# ========================= Phase Gradient (Optional) ======================== + +dataformat = cpxfloat32 # Data format of interferogram +scale = 10000 # Original phase/scale +step_windows = 1 # Step length for spatial phase gradient +filter_wins = [7, 7] # Windows of median filter +directions = ['grad_east', 'grad_north', 'grad_northeast', 'grad_southeast'] + +# Mapping parameters for phase gradient +in_low_high = [0.001, 1] # Input range for mapping +out_low_high = [0.001, 1] # Output range for mapping +gamma = 0.8 # Gamma for curve shape +choose_linear = 'n' # Linear or nonlinear mapping ['y': linear; 'n': nonlinear] + +# ========================= Hyp3 Format Conversion (NEW) ======================== + +# Convert GAMMA outputs to Hyp3-compatible GeoTIFF format +hyp3format = 0 # Enable Hyp3 format conversion [0: skip; 1: process] +hyp3format_parallel = 4 # Number of parallel processors for Hyp3 conversion +hyp3_output_dir = - # Output directory for Hyp3 products [default: projectName/Hyp3Products/] + +# ========================= Phase Bias Correction (NEW) ======================== + +# Apply phase bias correction based on loop closures algorithm +# Reference: 10.1016/j.remote.2022.100013 + +phasebias = 0 # Enable phase bias correction [0: skip; 1: process] +phasebias_interval = 12 # Temporal baseline in days [6, 12, or 24] +phasebias_nlook = 10 # Number of looks for multilooking +phasebias_num_a = 2 # Number of calibration parameters to estimate +phasebias_estimate_an = 0 # Estimate an from data [0: use defaults; 1: estimate from loop closures] +phasebias_max_con = 5 # Maximum number of connections + +# Default an values (used if phasebias_estimate_an = 0) +a1_6_day = 0.50 +a2_6_day = 0.36 +a3_6_day = 0.299 +a4_6_day = 0.2476 + +a1_12_day = 0.494 +a2_12_day = 0.297 +a3_12_day = 0.24 +a4_12_day = 0.22 + +a1_24_day = 0.48 +a2_24_day = 0.28 +a3_24_day = 0.22 +a4_24_day = 0.20 + +phasebias_start = - # Start date YYYYMMDD [default: auto-detect] +phasebias_end = - # End date YYYYMMDD [default: auto-detect] + +# ========================= Atmospheric Correction ======================== + +# GACOS tropospheric correction +gacos_correction = 0 # Enable GACOS correction [0: skip; 1: process] +gacos_all_parallel = 1 # Number of parallel processors (1=sequential with ZTD pre-cache, recommended) +gacos_dir = # Directory for GACOS ZTD files [default: projectName/GACOS] +gacos_email = # Email for GACOS auto-download (optional) + +# Ionospheric correction +ionosphere_correction = 0 # Enable ionospheric correction [0: skip; 1: process] +ionosphere_parallel = 4 # Number of parallel processors + +# ========================= Processing Workflow Control ======================== + +download_data = 0 # Enable data download [0: skip; 1: process] +down_parallel = 4 # Number of parallel processors for download + +raw2slc_all = 1 # Convert raw to SLC [0: skip; 1: process] +raw2slc_all_parallel = 4 # Number of parallel processors + +extract_burst_all = 1 # Extract bursts [0: skip; 1: process] +extract_all_parallel = 4 # Number of parallel processors + +coreg_all = 1 # Coregister SLCs [0: skip; 1: process] +coreg_all_parallel = 4 # Number of parallel processors + +select_pairs = 1 # Select interferometric pairs [0: skip; 1: process] + +diff_all = 1 # Generate differential interferograms [0: skip; 1: process] +diff_all_parallel = 4 # Number of parallel processors + +unwrap_all = 1 # Unwrap interferograms [0: skip; 1: process] +unwrap_all_parallel = 4 # Number of parallel processors + +geocode_all = 1 # Geocode products [0: skip; 1: process] +geocode_all_parallel = 4 # Number of parallel processors + +hyp3format_all = 0 # Convert to Hyp3 format [0: skip; 1: process] +hyp3format_all_parallel = 4 # Number of parallel processors + +phasebias_all = 0 # Apply phase bias correction [0: skip; 1: process] + +load_data = 0 # Load data for MintPy processing [0: skip; 1: process] + +# ========================= Good Luck ======================================= diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/pyintApp.py b/.codex_tmp/pyint_variants/no_rescue/pyint/pyintApp.py new file mode 100644 index 0000000..1ad8c15 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/pyintApp.py @@ -0,0 +1,333 @@ +#! /usr/bin/env python +########################################################### +# Project: PyINT # +# Purpose: Interferograms process using python/GAMMA # +# Author: Yunmeng Cao # +# Created: Feb. 2017 # +# Contact : ymcmrs@gmail.com # +# Copy Right (c): 2017-2019, Yunmeng Cao # +########################################################### + +import numpy as np +import os +import sys +import subprocess +import time +import argparse + +from pyint import _utils as ut + + +def _run_or_raise(call_str, stage): + rc = os.system(call_str) + if rc != 0: + raise RuntimeError('%s failed with rc=%s: %s' % (stage, rc, call_str)) + return rc + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Interferograms processing using PyINT.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='name of the project.') + parser.add_argument('-g', '--generate', action='store_true', dest='generate_structure', \ + help='Generate project directory structure if not exists.') + + inps = parser.parse_args() + return inps + + +INTRODUCTION = ''' +----------------------------------------------------------------------------------- + + Single or time-series of interferometry processing for satellite based + Synthetic Aperture Radar (SAR) images start from downloading data to + generate unwrapped-differential interferograms. + + Details please check: https://github.com/ymcmrs/PyINT + + General work-flow: + + 1) download data : download SLCs using SSARA/Scihub/ASF + SSARA: https://github.com/bakerunavco/SSARA + Scihub: Copernicus Open Access Hub + ASF: Alaska Satellite Facility (for Sentinel-1) + [You should provide Sensor, Track, Frame, or Time information in template] + 2) generate SLC : raw 2 slc (multi-frame processing is also supported) + [include orbit correction for S1,ASAR,ERS and burst-extraction for S1] + 3) generate DEM : reference image related geo-dem, rdc-dem, lookup table will be generated. + [SRTM-1 will be downloaded and processed automatically if not provided] + 4) coregister SLC : coregister SLCs to the reference SLC iamge. + [with assistant of DEM] + 5) select pairs : select interferometric pairs for time-series processing. + [networks of sbas, sequential, delaunay, and stars are supported] + 6) interferometry : generate unwrapped differential interferograms. + [include differential, unwrapping, and geocoding] + 7) offset tracking: pixel offset tracking (POT) for displacement measurement. + [two-round estimation with GAMMA offset_pwr_tracking] + 8) GACOS correction: apply GACOS tropospheric atmospheric correction. + [requires GACOS ZTD data for master and slave dates] + + Note: + + i) Single interferogram processing please use slc2ifg.py or raw2ifg.py + ii) Multi-processor parallel processing is supported, but keep in mind GAMMA calls multi-threads already. + iii) ASF download supports both bounding box and Shapefile for spatial filtering. + iv) GACOS correction requires geocoded interferograms (set geocode_all=1 first). + +''' + +EXAMPLE = """Usage: + + pyintApp.py -h + pyintApp.py projectName #[projectName.template should be available in TEMPLATEDIR] + pyintApp.py -g projectName #Generate project directory structure if not exists + +------------------------------------------------------------------------------------ +""" + +def main(argv): + + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + templateDir = os.getenv('TEMPLATEDIR') + scratchDir = os.getenv('SCRATCHDIR') + workDir = scratchDir + '/' + projectName + + # Generate project directory structure if -g flag is set and directory doesn't exist + if inps.generate_structure: + if not os.path.isdir(workDir): + print('Generating project directory structure for: %s' % projectName) + os.makedirs(workDir, exist_ok=True) + os.makedirs(workDir + '/DOWNLOAD', exist_ok=True) + os.makedirs(workDir + '/SLC', exist_ok=True) + os.makedirs(workDir + '/RSLC', exist_ok=True) + os.makedirs(workDir + '/DEM', exist_ok=True) + os.makedirs(workDir + '/ifgrams', exist_ok=True) + print('Project directory structure created successfully.') + print('Directory structure:') + print(' %s/' % projectName) + print(' ├── DOWNLOAD/') + print(' ├── SLC/') + print(' ├── RSLC/') + print(' ├── DEM/') + print(' └── ifgrams/') + return + else: + print('Project directory already exists: %s' % workDir) + print('Skipping directory structure generation.') + return + + templateFile = templateDir + "/" + projectName + ".template" + templateDict = ut.update_template(templateFile) + masterDate = templateDict['masterDate'] + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + satelite = templateDict['satelite'] + HGTSIM = scratchDir + '/' + projectName + '/DEM/' + ut.yyyymmdd(masterDate) + '_' + rlks + 'rlks.rdc.dem' + DataDir = workDir + '/' + 'DOWNLOAD' + DEMDir = workDir + '/' + 'DEM' + os.chdir(scratchDir) + if not os.path.isdir(workDir): + os.mkdir(workDir) + os.chdir(workDir) + os.mkdir(DataDir) + os.mkdir(DEMDir) + os.chdir(DataDir) + else: + os.chdir(DataDir) + ### download data + if templateDict['download_data'] == '1': + if templateDict['Data_Hub'] == 'SSARA': + print('Start to download SAR data using SSARA...') + call_str = 'ssara_federated_query.py -p ' + templateDict['sensor'] + ' -r ' + templateDict['track'] + ' -f ' + templateDict['frame'] + ' -s ' + templateDict['start_time'] + ' -e ' + templateDict['end_time'] + ' --print --download --parallel ' + templateDict['down_parallel'] + print(call_str) + _run_or_raise(call_str, 'download_data_ssara') + elif templateDict['Data_Hub'] == 'Scihub': + print('Start to downlad SAR data from Scihub') + call_str = 'scihub_search_s1_data.py ' + projectName + ' -s ' + templateDict['start_time'] + ' -e ' + templateDict['end_time'] + ' -p ' + templateDict['producttype'] +' -r ' + templateDict['region_box'] + ' -n ' + templateDict['Orbit_number'] + ' -d ' + templateDict['Direction'] + ' -sm ' + templateDict['Sence_model'] + ' -o ' + templateDict['output_file'] + print(call_str) + _run_or_raise(call_str, 'download_data_scihub') + elif templateDict['Data_Hub'] == 'ASF': + print('Start to download Sentinel-1 SLC data from ASF...') + # 获取必要的参数 + path_SLC = DataDir + path_RSLC = workDir + '/RSLC' + date_start = templateDict.get('start_time', '20170101') + date_end = templateDict.get('end_time', '20240101') + + # 构建基本命令 + call_str = 'API_download_S1_SLC.py -s ' + path_SLC + ' -r ' + path_RSLC + ' -i ' + date_start + ' -j ' + date_end + + # 添加用户名和密码（如果提供） + if 'asf_username' in templateDict and templateDict['asf_username'].strip(): + call_str += ' -u ' + templateDict['asf_username'] + if 'asf_password' in templateDict and templateDict['asf_password'].strip(): + call_str += ' -p ' + templateDict['asf_password'] + + # 添加边界框或Shapefile + if 'shapefile' in templateDict and templateDict['shapefile'].strip(): + call_str += ' --shp ' + templateDict['shapefile'] + elif 'bbox' in templateDict and templateDict['bbox'].strip(): + call_str += ' -b ' + templateDict['bbox'] + elif 'asf_bbox' in templateDict and templateDict['asf_bbox'].strip(): + call_str += ' -b ' + templateDict['asf_bbox'] + + # 添加轨道号（可选） + if 'relative_orbit' in templateDict and templateDict['relative_orbit'].strip(): + call_str += ' -o ' + str(templateDict['relative_orbit']) + elif 'track' in templateDict and templateDict['track'].strip(): + call_str += ' -o ' + str(templateDict['track']) + + # 添加帧号（可选） + if 'frame' in templateDict and templateDict['frame'].strip(): + call_str += ' --frame ' + str(templateDict['frame']) + + # 添加飞行方向（可选，默认为升轨） + if 'flight_direction' in templateDict: + call_str += ' -f ' + templateDict['flight_direction'] + else: + call_str += ' -f a' # 默认升轨 + + # 添加波束模式（默认IW） + if 'acquisition_mode' in templateDict: + call_str += ' -q ' + templateDict['acquisition_mode'] + else: + call_str += ' -q IW' # 默认IW模式 + + # 输出格式和下载选项 + call_str += ' -m csv -w Y' + + # 添加并行下载参数 + if 'down_parallel' in templateDict and templateDict['down_parallel'].strip(): + call_str += ' --parallel ' + str(templateDict['down_parallel']) + + print(call_str) + _run_or_raise(call_str, 'download_data_asf') + ### raw 2 slc + if templateDict['raw2slc_all'] == '1': # only for S1 data now + print('Start to convert downloaded-raw data into SLC ...') + print('Number of processor: %s' % str(templateDict['raw2slc_all_parallel'])) + if satelite=='S1A': + call_str = 'down2slc_sen_all.py ' + projectName + ' --parallel ' + templateDict['raw2slc_all_parallel'] + _run_or_raise(call_str, 'raw2slc_s1') + elif satelite=='ALOS': + call_str = 'down2slc_alos_all.py ' + projectName + ' --parallel ' + templateDict['raw2slc_all_parallel'] + _run_or_raise(call_str, 'raw2slc_alos') + elif satelite=='LT': + call_str = 'down2slc_LT1_all.py ' + projectName + ' --parallel ' + templateDict['raw2slc_all_parallel'] + _run_or_raise(call_str, 'raw2slc_lt1') + ### extract bursts + if templateDict['extract_burst_all'] == '1': + print('Start to extract common bursts ...') + print('Number of processor: %s' % str(templateDict['extract_all_parallel'])) + call_str = 'extract_s1_bursts_all.py ' + projectName + ' --parallel ' + templateDict['extract_all_parallel'] + _run_or_raise(call_str, 'extract_burst_all') + + ### generate rdc_dem + if not os.path.isfile(HGTSIM): + print('Start to generate geometry file ...') + call_str = 'makedem_pyint.py ' + projectName + _run_or_raise(call_str, 'makedem_pyint') + call_str = 'generate_rdc_dem.py ' + projectName + _run_or_raise(call_str, 'generate_rdc_dem') + + ### coreg SLC + if templateDict['coreg_all'] == '1': + print('Start to coregister SLCs ...') + print('Number of processor: %s' % str(templateDict['coreg_all_parallel'])) + call_str = 'coreg_gamma_all.py ' + projectName + ' --parallel ' + templateDict['coreg_all_parallel'] + _run_or_raise(call_str, 'coreg_all') + + ### select interferometric pairs + if templateDict['select_pairs'] == '1': + print('Start to select interferometric pairs ...') + print('Network selection method: %s' % templateDict['network_method']) + #print('Meximum temporal baseline threshold: %s' % templateDict['max_tb']) + #print('Meximum spatial baseline threshold: %s' % templateDict['max_sb']) + call_str = 'select_pairs.py ' + projectName + _run_or_raise(call_str, 'select_pairs') + + ### diff ifg + if templateDict['diff_all'] == '1': + print('Start to generate differential interferograms ...') + print('Number of processor: %s' % str(templateDict['diff_all_parallel'])) + call_str = 'diff_gamma_all.py ' + projectName + ' --parallel ' + templateDict['diff_all_parallel'] + _run_or_raise(call_str, 'diff_all') + + ### Pixel Offset Tracking (POT) + if templateDict['pot_all'] == '1': + print('Start to run Pixel Offset Tracking (POT) ...') + print('Number of processor: %s' % str(templateDict['pot_all_parallel'])) + call_str = 'POT_gamma_all.py ' + projectName + ' --parallel ' + templateDict['pot_all_parallel'] + _run_or_raise(call_str, 'pot_all') + + ### unw ifg + if templateDict['unwrap_all'] == '1': + print('Start to unwrap interferometric phases ...') + print('Number of processor: %s' % str(templateDict['unwrap_all_parallel'])) + call_str = 'unwrap_gamma_all.py ' + projectName + ' --parallel ' + templateDict['unwrap_all_parallel'] + _run_or_raise(call_str, 'unwrap_all') + + ### atmcor ifg + if templateDict['atmcor_all'] == '1': + print('Start to correct atmospheric phase ...') + print('Number of processor: %s' % str(templateDict['atmcor_all_parallel'])) + call_str = 'atm_correction_gamma_all.py ' + projectName + ' --parallel ' + templateDict['atmcor_all_parallel'] + _run_or_raise(call_str, 'atmcor_all') + + ### geocode ifg + if templateDict['geocode_all'] == '1': + print('Start to geocode Ifgs ...') + print('Number of processor: %s' % str(templateDict['geocode_all_parallel'])) + call_str = 'geocode_gamma_all.py ' + projectName + ' --parallel ' + templateDict['geocode_all_parallel'] + _run_or_raise(call_str, 'geocode_all') + + ### Convert GAMMA outputs to LiCSBAS format + if templateDict.get('gamma2licsbas_all', '0') == '1': + print('Start to convert GAMMA outputs to LiCSBAS format ...') + print('Number of processor: %s' % str(templateDict.get('gamma2licsbas_all_parallel', '1'))) + call_str = 'gamma2licsbas_gamma_all.py ' + projectName + ' --parallel ' + templateDict.get('gamma2licsbas_all_parallel', '1') + _run_or_raise(call_str, 'gamma2licsbas_all') + + ### GACOS atmospheric correction (after geocoding) + if templateDict['gacos_correction'] == '1': + print('Start to apply GACOS atmospheric correction ...') + gacos_dir = templateDict.get('gacos_dir', '').strip() + gacos_email = templateDict.get('gacos_email', '').strip() + + call_str = 'gacos_gamma_all.py ' + projectName + ' --skip-existing' + if gacos_dir: + call_str += ' --ztd-dir ' + gacos_dir + if gacos_email: + call_str += ' --email ' + gacos_email + print(call_str) + _run_or_raise(call_str, 'gacos_correction') + + ### Convert GAMMA outputs to HyP3 UTM format + if templateDict.get('hyp3format_all', '0') == '1': + print('Start to convert GAMMA outputs to HyP3 UTM format ...') + print('Number of processor: %s' % str(templateDict.get('hyp3format_all_parallel', '1'))) + call_str = 'hyp3format_gamma_all.py ' + projectName + ' --parallel ' + templateDict.get('hyp3format_all_parallel', '1') + _run_or_raise(call_str, 'hyp3format_all') + + ### load data + if templateDict['load_data'] == '1': + print('Start to load data for mintPy time-series analysis ...') + call_str = 'load_mintpy.py ' + projectName + _run_or_raise(call_str, 'load_data') + + print("PyINT processing for project %s is done." % projectName) + ut.print_process_time(start_time, time.time()) + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/raw2ifg.py b/.codex_tmp/pyint_variants/no_rescue/pyint/raw2ifg.py new file mode 100644 index 0000000..0835ffe --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/raw2ifg.py @@ -0,0 +1,70 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import time +import argparse + +from pyint import _utils as ut + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate unwrapped differential Ifg from SLC-raw data using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='name of the project.') + parser.add_argument('Mdate',help='master date for interferometry.') + parser.add_argument('Sdate',help='slave date for interferometry.') + + inps = parser.parse_args() + + return inps + + +INTRODUCTION = ''' +-------------------------------------------------------------------------------------- + Generate unwrapped differential Ifg from SLC-raw data using GAMMA. + + Note: 1) Precise orbit data will be downloaded and processed automatically + 2) SRTM-1 will be downloaded and processed automatically if not provided + in the template file. + +''' + +EXAMPLE = """Usage: + + raw2ifg.py projectName Mdate Sdate +--------------------------------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + Mdate = inps.Mdate + Sdate = inps.Sdate + + projectName = inps.projectName + + if 'S1' in projectName: + call_str = 'raw2ifg_s1.py ' + projectName + ' ' + ut.yyyymmdd(Mdate) + ' ' + ut.yyyymmdd(Sdate) + os.system(call_str) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/raw2ifg_s1.py b/.codex_tmp/pyint_variants/no_rescue/pyint/raw2ifg_s1.py new file mode 100644 index 0000000..f192afd --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/raw2ifg_s1.py @@ -0,0 +1,132 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import time +import glob +import argparse + +from pyint import _utils as ut + + +def get_s1_date(raw_file): + file0 = os.path.basename(raw_file) + date = file0[17:25] + return date + +def get_satellite(raw_file): + if 'S1A_IW_SLC_' in raw_file: + s0 = 'A' + else: + s0 = 'B' + + return s0 + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate Ifg from Sentinel-1 raw data with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='name of the project.') + parser.add_argument('Mdate',help='master date for interferometry.') + parser.add_argument('Sdate',help='slave date for interferometry.') + + inps = parser.parse_args() + + return inps + + +INTRODUCTION = ''' +--------------------------------------------------------------------------------------------------- + Generate unwrapped differential Ifg from Sentinel-1 raw data with orbit correction using GAMMA. + + Note: 1) Precise orbit data will be downloaded and processed automatically + 2) SRTM-1 will be downloaded and processed automatically if not provided + in the template file. + +''' + +EXAMPLE = """Usage: + + raw2ifg_s1.py projectName Mdate Sdate +---------------------------------------------------------------------------------------------------- +""" + +def main(argv): + + start_time = time.time() + inps = cmdLineParse() + Mdate = inps.Mdate + Sdate = inps.Sdate + + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + downDir = scratchDir + '/' + projectName + '/DOWNLOAD' + M_raw = glob.glob(downDir + '/S1*_' + ut.yyyymmdd(Mdate)+'*')[0] + S_raw = glob.glob(downDir + '/S1*_' + ut.yyyymmdd(Sdate)+'*')[0] + slcDir = scratchDir + '/' + projectName + '/SLC' + + + ######### down 2 slc ############# + call_str = 'down2slc_sen.py ' + M_raw + ' ' + slcDir + os.system(call_str) + + call_str = 'down2slc_sen.py ' + S_raw + ' ' + slcDir + os.system(call_str) + + ########## extract common bursts ## + call_str = 'extract_s1_bursts.py ' + projectName + ' ' + Mdate + os.system(call_str) + + call_str = 'extract_s1_bursts.py ' + projectName + ' ' + Sdate + os.system(call_str) + + ######### generate rdc_dem ########## + call_str = 'generate_rdc_dem.py ' + projectName + os.system(call_str) + + ########## coregister SLC ######## + + call_str = 'coreg_s1_gamma.py ' + projectName + ' ' + Mdate + os.system(call_str) + + call_str = 'coreg_s1_gamma.py ' + projectName + ' ' + Sdate + os.system(call_str) + + ######## Interferometry process ########### + call_str = 'diff_gamma.py ' + projectName + ' ' + Mdate + ' ' + Sdate + os.system(call_str) + + call_str = 'unwrap_gamma.py ' + projectName + ' ' + Mdate + ' ' + Sdate + os.system(call_str) + + call_str = 'geocode_gamma.py ' + projectName + ' ' + Mdate + '-' + Sdate + os.system(call_str) + + print("Generate Ifg from raw-TOPs data is done! ") + ut.print_process_time(start_time, time.time()) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/raw2slc_ers_envisat.py b/.codex_tmp/pyint_variants/no_rescue/pyint/raw2slc_ers_envisat.py new file mode 100644 index 0000000..6384f3f --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/raw2slc_ers_envisat.py @@ -0,0 +1,186 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.0 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Now at KAUST ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import linecache +import datetime + +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + + +def check_ERS_par(SAR_IM_0P): + Name = os.path.basename(SAR_IM_0P) + ff = Name.split('.')[1] + date = (Name.split('SAR_IM__0PWDSI')[1]).split('_')[0] + + if ff == 'E1': + par = 'ERS1_ESA.par' + antenna = 'ERS1_antenna.gain' + orbdir = os.getenv('ERS1ORBDIR') + elif ff == 'E2': + par = 'ERS2_ESA.par' + antenna = 'ERS2_antenna.gain' + orbdir = os.getenv('ERS2ORBDIR') + else: + print('Invalid input SAR_IM_0P file.') + sys.exit(1) + + return par, antenna, orbdir, date + +######################################################################### + +INTRODUCTION = ''' +############################################################################# + Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 + + Generate SLC for ERS raw data with ENVISAT format + +''' + +EXAMPLE = ''' + Usage: + raw2slc_ers_envisat.py SAR_IM_OP -o output_prefix + + Examples: + raw2slc_ers_envisat.py SAR_IM_OP -o 19950102 + +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLC for ERS raw data with ENVISAT format.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('ers_envisat_raw',help='ERS raw data with ENVISAT format, e.g., SAR_IM_0PXXXX.E1') + parser.add_argument('-o',dest='out_put',help='prefix of the output file, e.g., output.slc, output.slc.par') + + inps = parser.parse_args() + return inps + +################################################################################ + + +def main(argv): + + inps = cmdLineParse() + SAR_IM_0P = inps.ers_envisat_raw + par,antenna,orbdir,date = check_ERS_par(SAR_IM_0P) + + raw = date + '.raw' + azsp = date + '.azsp' + dop = date + '.dop' + rspec = date + '.rspec' + rc = date + '.rc' + autof = date + '.autof' + dop_ambig = date + '.dop_ambig' + + if not os.path.isfile(par): + call_str = 'cp $GAMMA_HOME/MSP/sensors/' + par + ' .' + os.system(call_str) + if not os.path.isfile(antenna): + call_str = 'cp $GAMMA_HOME/MSP/sensors/' + antenna + ' .' + os.system(call_str) + + if inps.out_put: + pslc_par = 'p' + inps.out_put + '.slc.par' + slc_par = inps.out_put + '.slc.par' + slc = inps.out_put + '.slc' + mli_par = inps.out_put + '.mli.par' + mli = inps.out_put + '.mli' + else: + pslc_par = 'p' + date + '.slc.par' + slc_par = date + '.slc.par' + slc = date + '.slc' + mli_par = date + '.mli.par' + mli = date + '.mli' + + call_str = 'ERS_ENVISAT_proc ' + SAR_IM_0P + ' ' + par + ' ' + pslc_par + ' ' + raw + os.system(call_str) + + call_str = 'DELFT_proc2 ' + pslc_par + ' ' + orbdir + ' 20' + os.system(call_str) + + cal_str = 'dop_ambig ' + par + ' ' + pslc_par + ' ' + raw + ' 2 - ' + dop_ambig + os.system(call_str) + + call_str = 'azsp_IQ ' + par + ' ' + pslc_par + ' ' + raw + ' ' + azsp + os.system(call_str) + + call_str = 'doppler ' + par + ' ' + pslc_par + ' ' + raw + ' ' + dop + os.system(call_str) + + call_str = 'rspec_IQ ' + par + ' ' + pslc_par + ' ' + raw + ' ' + rspec + os.system(call_str) + + call_str = 'pre_rc ' + par + ' ' + pslc_par + ' ' + raw + ' ' + rc + os.system(call_str) + + call_str = 'autof ' + par + ' ' + pslc_par + ' ' + rc + ' ' + autof + ' 2.0 ' + os.system(call_str) + + call_str = 'autof ' + par + ' ' + pslc_par + ' ' + rc + ' ' + autof + ' 2.0 ' + os.system(call_str) + + call_str = 'az_proc ' + par + ' ' + pslc_par + ' ' + rc + ' ' + slc + ' 4096 1 ' + ' 57.2 0 2.120 ' + os.system(call_str) + + call_str = 'par_MSP ' + par + ' ' + pslc_par + ' ' + slc_par + ' 1' + os.system(call_str) + + call_str = 'multi_look ' + slc + ' ' + slc_par + ' ' + mli + ' ' + mli_par + ' 2 10' + os.system(call_str) + + + Width = UseGamma(mli_par, 'read', 'range_samples: ') + call_str = 'raspwr ' + mli + ' ' + Width + os.system(call_str) + + + print("Generate SLC from %s is done." % SAR_IM_0P) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/rslcCopy_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/rslcCopy_gamma.py new file mode 100644 index 0000000..455686f --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/rslcCopy_gamma.py @@ -0,0 +1,89 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import argparse + +from pyint import _utils as ut + + +INTRODUCTION = ''' +------------------------------------------------------------------- + copy SLC/RSLC from large SLC/RSLC + +''' + +EXAMPLE = ''' + Usage: + slcCopy_gamma.py projectName Sdate rstart rwidth astart awidth outname + slcCopy_gamma.py PacayaT163TsxHhA 20150601 100 345 200 500 20150601_subset +------------------------------------------------------------------- +''' + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Copy RSLC subset from a large SLC.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('Date',help='Master date.') + parser.add_argument('rstart',help='start point of range') + parser.add_argument('rwidth',help='width of range direction') + parser.add_argument('astart',help='start point of azimuth') + parser.add_argument('awidth',help='width of azimith direction') + parser.add_argument('name',help='output name. e.g., 20150101_subset, then .slc and .slc.par will be generated') + + inps = parser.parse_args() + return inps + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Date = inps.Date + rstart = inps.rstart; rwidth = inps.rwidth; astart = inps.astart; awidth = inps.awidth; name = inps.name + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + processDir = scratchDir + '/' + projectName + "/PROCESS" + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + '/RSLC' + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + + Dslc = rslcDir + '/' + Date + '/' + Date + '.rslc' + DslcPar = rslcDir + '/' + Date + '/' + Date + '.rslc.par' + + Oslc = rslcDir + '/' + Date + '/' + name + '.rslc' + OslcPar = rslcDir + '/' + Date + '/' + name + '.rslc.par' + + Oamp = rslcDir + '/' + Date + '/' + name + '_' + rlks + 'rlks.amp' + OampPar = rslcDir + '/' + Date + '/' + name + '_' + rlks + 'rlks.amp.par' + + + + call_str = 'SLC_copy ' + Dslc + ' ' + DslcPar + ' ' + Oslc + ' ' + OslcPar + ' - - ' + rstart + ' ' + rwidth + ' ' + astart + ' ' + awidth + ' - -' + os.system(call_str) + + call_str = 'multi_look ' + Oslc + ' ' + OslcPar + ' ' + Oamp + ' ' + OampPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + #os.remove(MampPar) + #os.remove(SampPar) + print(call_str) + print("RSLC copy done.") + #sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/rslcCopy_gamma_jobs.py b/.codex_tmp/pyint_variants/no_rescue/pyint/rslcCopy_gamma_jobs.py new file mode 100644 index 0000000..c4d728a --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/rslcCopy_gamma_jobs.py @@ -0,0 +1,135 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import argparse + +from pyint import _utils as ut + + +INTRODUCTION = ''' +------------------------------------------------------------------- + copy SLC/RSLC from large SLC/RSLC + +''' + +EXAMPLE = ''' + Usage: + slcCopy_gamma_jobs.py projectName Sdate rwidth awidth --memory memory_single_job --walltime walltime_single_job + slcCopy_gamma_jobs.py PacayaT163TsxHhA 20150601 5000 5000 --memory 5000 --walltime 00:30:00 +------------------------------------------------------------------- +''' + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Copy SLC subset from a large SLC.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('Date',help='Master date.') + parser.add_argument('rwidth',help='width of range direction') + parser.add_argument('awidth',help='width of azimith direction') + parser.add_argument('--memory',dest = 'memory',default = '2000', help='memory to be allocated for one job, default: 5GB') + parser.add_argument('--walltime',dest = 'walltime',default = '00:05:00', help='walltime to be allocated for one single job') + inps = parser.parse_args() + return inps + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Date = inps.Date; rwidth = inps.rwidth; awidth = inps.awidth; memory = inps.memory; walltime = inps.walltime + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + processDir = scratchDir + '/' + projectName + "/PROCESS" + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + '/RSLC' + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + + Dslc = rslcDir + '/' + Date + '/' + Date + '.rslc' + DslcPar = rslcDir + '/' + Date + '/' + Date + '.rslc.par' + + nWidth = ut.read_gamma_par(DslcPar, 'read', 'range_samples') + nLine = ut.read_gamma_par(DslcPar, 'read', 'azimuth_lines') + + Ap_samp = np.arange(1,int(nLine),int(awidth)); Na0 = len(Ap_samp); LA_end = int(nLine) - Ap_samp[Na0-1] + Rp_samp = np.arange(1,int(nWidth),int(rwidth)); Nr0 = len(Rp_samp); LR_end = int(nWidth) - Rp_samp[Nr0-1] + + if LA_end > (2/3*int(awidth)): + Ap_samp1 = Ap_samp + else: + Ap_samp1 = Ap_samp[0:(Na0-1)] # last batch with '-' means to the end + + if LR_end > (2/3*int(rwidth)): + Rp_samp1 = Rp_samp + else: + Rp_samp1 = Rp_samp[0:(Nr0-1)] # last batch with '-' means to the end + + Na = len(Ap_samp1); Nr = len(Rp_samp1); Ntotal = Na*Nr + print('Azimuth samples: ' + str(Na)) + print('Range samples: ' + str(Nr)) + print('Total batch numbers: ' + str(Ntotal)) + + batch_txt = scratchDir + '/' + projectName + '/RSLC/' + Date + '/Batch_copySLC' + workDir = scratchDir + '/' + projectName + '/RSLC/' + Date + if os.path.isfile(batch_txt): + os.remove(batch_txt) + for i in range(Na): + for j in range(Nr): + if len(str(i)) == 1: + i0 = '0' + str(i) + else: + i0 = str(i) + if len(str(j)) == 1: + j0 = '0' + str(j) + else: + j0 = str(j) + name0 = Date + '_' +i0 + j0 + rstart = str(Rp_samp1[j]); astart = str(Ap_samp1[i]) + if rstart == str(Rp_samp1[Nr-1]): + rwidth0 = '-' + else: + rwidth0 = str(int(int(rwidth) + 200)) # extend 200 to avoid edge effect + + if astart == str(Ap_samp1[Na-1]): + awidth0 = '-' + else: + awidth0 = str(int(int(awidth) + 200)) # extend 200 to avoid edge effect + + if not i==0: + astart0 = str(int(int(astart) - 200)) # extend 200 to avoid edge effect + else: + astart0 = astart + + if not j==0: + rstart0 = str(int(int(rstart) - 200)) # extend 200 to avoid edge effect + else: + rstart0 = rstart + + #str0 = 'SLC_copy ' + Dslc + ' ' + DslcPar + ' ' + workDir + '/' + name0 + '.rslc' + ' ' + workDir + '/' + name0 + '.rslc.par - - ' + rstart0 + ' ' + rwidth0 + ' ' + astart0 + ' ' + awidth0 + ' - -' + str0 = 'rslcCopy_gamma.py ' + projectName + ' ' + Date + ' ' + rstart0 + ' ' + rwidth0 + ' ' + astart0 + ' ' + awidth0 + ' ' + name0 + call_str = 'echo ' + str0 + ' >>' + batch_txt + os.system(call_str) + + call_str = 'sbatch_jobs.py ' + batch_txt + ' --memory ' + inps.memory + ' --walltime ' + inps.walltime + ' --job-name ' + Date + '_copySLC' + os.system(call_str) + + print("RSLC copy done: " + Date) + #sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/s1_orb_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/s1_orb_all.py new file mode 100644 index 0000000..fa225b8 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/s1_orb_all.py @@ -0,0 +1,114 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + + +def work(data0): + cmd = data0[0] + err_txt = data0[1] + + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + #print(aa) + with open(err_txt, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + Using precise orbit data for all rslcs. + +''' + +EXAMPLE = ''' + Usage: + generate_amp_all.py projectName + generate_amp_all.py projectName --parallel 4 +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Using precise orbit data for all rslcs.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + slcDir = scratchDir + '/' + projectName + '/SLC' + rslcDir = scratchDir + '/' + projectName + '/RSLC' + if not os.path.isdir(rslcDir): os.mkdir(rslcDir) + + cmd_command = 'generate_multilook_amp.py' + + err_txt = scratchDir + '/' + projectName + '/generate_amp_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + slc_list = [os.path.basename(fname) for fname in sorted(glob.glob(rslcDir + '/*'))] + #slc_list = ut.get_project_slcList(projectName) + for i in range(len(slc_list)): + rslcPar = rslcDir + '/' + slc_list[i] + '/' + slc_list[i] + '.rslc.par' + #print(rslcPar) + workDir0 = rslcDir + '/' + slc_list[i] + BB = glob.glob(workDir0 + '/*.EOF') + if len(BB)==0: + Sensor = ut.read_gamma_par(rslcPar,'read','sensor') + if 'A' in Sensor: satellite = 'A' + else: satellite = 'B' + ut.download_s1_orbit(slc_list[i],workDir0,satellite=satellite) + + BB = glob.glob(workDir0 + '/*.EOF') + if len(BB) > 0: + orb_file = BB[0] + + call_str = 'S1_OPOD_vec ' + rslcPar + ' ' + orb_file + ' 31' + os.system(call_str) + + print("Using precise orbit data for all rslcs is done! ") + ut.print_process_time(start_time, time.time()) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/scihub_search_s1_data.py b/.codex_tmp/pyint_variants/no_rescue/pyint/scihub_search_s1_data.py new file mode 100644 index 0000000..6a36c96 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/scihub_search_s1_data.py @@ -0,0 +1,220 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT ### +### Author: chen ### +### Contact : chenweicug@126.com ### +################################################################# + +import getopt +import time +import glob +import numpy as np +import os +import sys +import subprocess +import time +import argparse +from pyint import _utils as ut + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Generate differential interferogram image from SLC using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + scihub_search_s1_data.sh -options + scihub_search_s1_data.sh -s 2015-08-01 -e NOW -r "123.0/-123.3/40.0/40.2" -d Descending - +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister all of the SLCs to the reference SAR image using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('-s', dest='Start_time', help='start_time [yyyy-mm-dd or NOW]') + parser.add_argument('-e', dest='End_time', help='end_time [yyyy-mm-dd or NOW]') + parser.add_argument('-r', dest='Region_box',help='region_box "lonW/lonE/latS/latN"') + parser.add_argument('-p', dest='producttype', help='producttype "[SLC GRD OCN]"') + parser.add_argument('-n', dest='Orbit_number',help='orbit_number [0-175]') + parser.add_argument('-d', dest='Direction', help='direction [Ascending/Descending]') + parser.add_argument('-sm',dest='Sence_model', help='sence model [IW/EW/SM]') + parser.add_argument('-o', dest='output_file', help='output_file name') + inps = parser.parse_args() + return inps +def main(argv): + + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + projectDir = scratchDir + '/' + projectName + dataDir = scratchDir + '/' + projectName + '/' +'DOWNLOAD' + #dataDir="DOWNLOAD" + if not os.path.isdir(dataDir): + os.mkdir(dataDir) + os.chdir(projectDir) +# The basic stuff that doesn't change^^ + search_query="https://scihub.copernicus.eu/dhus/search?q=platformname:Sentinel-1" + + if inps.output_file: + Output_file=inps.output_file + else: + Output_file='search_result.txt' + +# If we are searching based on starttime and endtime + if inps.Start_time: + starttime=inps.Start_time + else: + starttime='1990-01-01' + if inps.End_time: + endtime=inps.End_time + else: + endtime='NOW' + search_query=search_query+" AND beginposition:[" + starttime + "T00:00:00.000Z TO "+ endtime + "T00:00:00.000Z]" + #print(search_query) + + if inps.Region_box: + Region=inps.Region_box + compments=Region.split('/') + lonW=compments[0] + lonE=compments[1] + latS=compments[2] + latN=compments[3] + lonw=float(lonW) + lone=float(lonE) + latn=float(latN) + lats=float(latS) + if lonW >=lonE: + print("Error! Longitudes not in increasing order!") + sys.exit(1) + elif lats >= latn: + print("Error! Latitudes not in increasing order!") + sys.exit(1) + elif lats >= 90 or latn >= 90: + print("Error! latS or latN not in bounds!") + sys.exit(1) + else: + parser.print_usage() + sys.exit(1) +# add the SAR covered area + search_query+=" AND footprint:\"intersects(POLYGON((" + lonW + " " + latN + "," + lonE + " " + latN + "," + lonE + " " + latS + "," + lonW + " " + latS + "," + lonW + " " + latN +")))\"" + + if inps.Direction: + direction=inps.Direction + else: + direction="Ascending" +# add the orbit direction + search_query+=" AND orbitdirection:" + direction + + if inps.Sence_model: + model=inps.Sence_model + else: + model='IW' +# add the sense operational mode + search_query+=" AND sensoroperationalmode:"+ model + + if inps.Orbit_number: + if int(inps.Orbit_number) >= 175: + print("Error: orbit must be between 0 and 175") + sys.exit(1) + else: + orbit=inps.Orbit_number + # add the relative orbit number + search_query+=" AND relativeorbitnumber:" + orbit + if inps.producttype: + producttype=inps.producttype + else: + producttype='SLC' +# add the orbit direction + search_query+=" AND producttype:" + producttype + + + + + + +# how many rows to display and where to start? +# Max rows = 100 (slightly annoying rule from the Copernicus server) + if os.path.isfile(Output_file): os.remove(Output_file) + search_query0=search_query+"&start=0&rows=100" + print("the current search_query is: ",search_query0) + #call_str="echo \"Input options:\" $@ > " +Output_file + #os.system(call_str) + call_str="echo \"wget --no-check-certificate --user=chenwei --password=cw1425 \""+ search_query0 + " >> " + Output_file + os.system(call_str) +# Execute the search using wget + search_query0="'"+search_query0+ "'" + call_str="wget --no-check-certificate --user=chenwei --password=cw1425 " + search_query0 + " -O ->> " + Output_file + os.system(call_str) + call_str="`grep \'title>S1\' " + Output_file +" | wc -l`" + num_results=os.system(call_str) + print(num_results) + if num_results >= 100 : + print("We have 100 results... automatically searching for results #100-200") + search_query1=search_query + "&start=100&rows=100" + search_query1="'"+search_query1+ "'" + call_str="wget --no-check-certificate --user=chenwei --password=cw1425 " + search_query1 + " -O ->> " + Output_file + os.system(call_str) + call_str="`grep \'title>S1\' " + Output_file +" | wc -l`" + num_results=os.system(call_str) + elif num_results >= 200 : + print("We have 200 results... automatically searching for results #100-200") + search_query2=search_query + "&start=100&rows=100" + search_query2="'"+search_query2+ "'" + call_str="wget --no-check-certificate --user=chenwei --password=cw1425 " + search_query2 + " -O ->> " + Output_file + os.system(call_str) + call_str="`grep \'title>S1\' " + Output_file +" | wc -l`" + num_results=os.system(call_str) + +# Displaying a summary of the results + call_str="grep \'title>S1\' " + Output_file # displaying the results + os.system(call_str) + print("number of total results is: ") + call_str="grep \'title>S1\' " + Output_file + " | wc -l" + os.system(call_str) + print("####################################################") + print("the search has done and start to download") + print("####################################################") + id_results='uuid_file.txt' + if os.path.isfile(id_results): + os.remove(id_results) + call_str="grep -E 'uuid|S1' " + Output_file + " >> " + id_results + os.system(call_str) + call_str="sed -i 's/<str name=\"uuid\">//g' " + id_results + os.system(call_str) + call_str="sed -i 's/<title>//g' " + id_results + os.system(call_str) + call_str="sed -i 's/<\/title>//g' " + id_results + os.system(call_str) + call_str="sed -i 's/<\/str>//g' "+ id_results + os.system(call_str) + infopen = open(id_results,'r',encoding='utf-8') + lines = infopen.readlines() + count=int(len(lines)/2-1) + for i in range(0,count,2): + j=i+1 + title=lines[i].strip("\n") + uuid=lines[j].strip("\n") + url_address="\"https://scihub.copernicus.eu/dhus/odata/v1/Products('" + uuid + "')/\$value\"" + call_str= "wget -c --no-check-certificate --user=chenwei --password=cw1425 -O " +dataDir + "/" + title + ".zip " + url_address + print(call_str) + os.system(call_str) + call_str=title+".zip download has done successfully!" + print(call_str) + print("#########################################################") + print("#########all the data download successfully!#############") + + + + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/select_pairs.py b/.codex_tmp/pyint_variants/no_rescue/pyint/select_pairs.py new file mode 100644 index 0000000..0db9cb2 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/select_pairs.py @@ -0,0 +1,297 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +# This program is modified from MintPy/select_network.py +# Copyright: 2017-2019, Yunjun Zhang +# Contact: yzhang@rsmas.miami.edu + +import os +import sys +import glob +import argparse +import datetime +import inspect +import numpy as np + +from pyint import _utils as ut +from pyint import _network as nt + +def get_datelist_bperplist(TS_Net): + + IFG_Flag=np.asarray(TS_Net[:,0]) + MDatelist=np.asarray(TS_Net[:,1]) + SDatelist=np.asarray(TS_Net[:,2]) + Berplist=np.asarray(TS_Net[:,3]) + TBaselist=np.asarray(TS_Net[:,4]) + + date_list = [] + date_list.append(MDatelist[0]) + for k0 in SDatelist: + date_list.append(k0) + bperp_list = [] + bperp_list.append(0) + for k0 in Berplist: + bperp_list.append(float(k0)) + + tbase_list = [] + tbase_list.append(0) + for k0 in TBaselist: + tbase_list.append(float(k0)) + return date_list, tbase_list, bperp_list + + +def prune_datelist(date_list, tbase_list, pbase_list, templateDict): + nn = len(date_list) + + date_list_out = [] + for k0 in date_list: + if (k0 not in templateDict['exclude_list']) and (float(templateDict['startDate']) < float(k0)) and (float(templateDict['endDate']) > float(k0)): + date_list_out.append(k0) + date_list_out = sorted(date_list_out) + tbase_list_out = [] + pbase_list_out = [] + for k0 in date_list_out: + tbase0 = tbase_list[date_list.index(k0)] + pbase0 = pbase_list[date_list.index(k0)] + tbase_list_out.append(float(tbase0)) + pbase_list_out.append(float(pbase0)) + + tbase_list_out0 = np.asarray(tbase_list_out) + pbase_list_out0 = np.asarray(pbase_list_out) + + tbase_list_out1 = tbase_list_out0 - tbase_list_out0[0] + pbase_list_out1 = pbase_list_out0 - pbase_list_out0[0] + + return date_list_out, list(tbase_list_out1), list(pbase_list_out1) + +def select_network_candidate(date_list,tbase_list,pbase_list,templateDict): + + method = templateDict['network_method'] + + # Pais selection from method + if method == 'sbas': + date12_list = nt.select_pairs_sbas(date_list) + elif method == 'delaunay': + date12_list = nt.select_pairs_delaunay(date_list, tbase_list, pbase_list, norm = True) + elif method == 'star': + date12_list = nt.select_pairs_star(date_list) + elif method == 'sequential': + date12_list = nt.select_pairs_sequential(date_list, int(templateDict['conNumb'])) + #elif method == 'hierarchical': + # date12_list = nt.select_pairs_hierarchical(date_list, pbase_list, inps.tempPerpList) + #elif method == 'mst': + # date12_list = nt.select_pairs_mst(date_list, pbase_list) + else: + raise Exception('Unrecoganized select method: '+ method) + + date12_list =nt.yyyymmdd_date12(date12_list) + nn = len(date12_list) + + ifgram_tbase_list = [] + ifgram_pbase_list = [] + for i in range(nn): + m0 = date12_list[i].split('-')[0] + s0 = date12_list[i].split('-')[1] + + tbase0 = float(tbase_list[date_list.index(s0)]) - float(tbase_list[date_list.index(m0)]) + pbase0 = float(pbase_list[date_list.index(s0)]) - float(pbase_list[date_list.index(m0)]) + + ifgram_tbase_list.append(tbase0) + ifgram_pbase_list.append(pbase0) + + return date12_list, ifgram_tbase_list, ifgram_pbase_list + + +def prune_network(date12_list, ifgram_tbase_list, ifgram_pbase_list, templateDict): + """Pruning network candidates based on temp/perp baseline""" + date12_list0 = date12_list.copy() + ifgram_tbase_list = [float(i) for i in ifgram_tbase_list] + ifgram_pbase_list = [float(i) for i in ifgram_pbase_list] + + ifgram_tbase_list_abs = [abs(i) for i in ifgram_tbase_list] + ifgram_pbase_list_abs = [abs(i) for i in ifgram_pbase_list] + + ifgram_tbase_list_abs = np.asarray(ifgram_tbase_list_abs) + ifgram_pbase_list_abs = np.asarray(ifgram_pbase_list_abs) + + date12_list0 = np.asarray(date12_list0) + ifgram_tbase_list = np.asarray(ifgram_tbase_list) + ifgram_pbase_list = np.asarray(ifgram_pbase_list) + +# date12_list_out = date12_list0[(ifgram_tbase_list_abs < float(templateDict['max_tb'])) & (ifgram_pbase_list_abs < float(templateDict['max_sb']))] + date12_list_out = date12_list0[(ifgram_tbase_list_abs < float(templateDict['max_tb'])) & (ifgram_tbase_list_abs > float(templateDict['min_tb'])) & (ifgram_pbase_list_abs < float(templateDict['max_sb']))] +# ifgram_tbase_list_out = ifgram_tbase_list[(ifgram_tbase_list_abs < float(templateDict['max_tb'])) & (ifgram_pbase_list_abs < float(templateDict['max_sb']))] +# ifgram_pbase_list_out = ifgram_pbase_list[(ifgram_tbase_list_abs < float(templateDict['max_tb'])) & (ifgram_pbase_list_abs < float(templateDict['max_sb']))] + ifgram_tbase_list_out = ifgram_tbase_list[(ifgram_tbase_list_abs < float(templateDict['max_tb'])) & (ifgram_tbase_list_abs > float(templateDict['min_tb'])) & (ifgram_pbase_list_abs < float(templateDict['max_sb']))] + ifgram_pbase_list_out = ifgram_pbase_list[(ifgram_tbase_list_abs < float(templateDict['max_tb'])) & (ifgram_tbase_list_abs > float(templateDict['min_tb'])) & (ifgram_pbase_list_abs < float(templateDict['max_sb']))] + + return list(date12_list_out), list(ifgram_tbase_list_out), list(ifgram_pbase_list_out) + + +def write_ifgram_list(date12_list,ifgram_tbase_list,ifgram_pbase_list, out_file): + # Output directory/filename + # Write txt file + f = open(out_file, 'w') + #f.write('#Interferograms configuration generated by select_network.py\n') + #f.write('# Date12 Btemp(days) Bperp(m) sim_coherence\n') + for i in range(len(date12_list)): + line = '{} {:6.0f} {:6.1f}'.format(date12_list[i], + ifgram_tbase_list[i], + ifgram_pbase_list[i]) + f.write(line+'\n') + f.close() + return out_file + +#---------------------------------------------------------------------------------------------------# +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Select interferometric pairs for time-series InSAR process.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('-o', '--outfile',dest = 'outfile',help='Output list file for network, ifgram_list.txt by default.') + parser.add_argument('--exclude',dest='excludeDate', nargs='*', default=[], + help='date(s) excluded for network selection, e.g. --exclude 060713 070831') + parser.add_argument('--start-date', dest='startDate', type=str, help='start/min date of network') + parser.add_argument('--end-date', dest='endDate', type=str, help='end/max date of network') + parser.add_argument('--max-sb', dest='maxSB', type=float, help='maximum spatial baseline') + parser.add_argument('--max-tb', dest='maxTB', type=str, help='maximum temporal baseline') + parser.add_argument('--min-tb', dest='minTB', type=str, help='minmum temporal baseline') + parser.add_argument('--method', dest='method', default='sbas', choices={'sbas', 'sequential', 'delaunay', 'stars'}, + help='network selection method:\n' + + 'sbas - select based on the threshold values of the spatio-temporal baselines\n' + + 'sequential - select based on the sequential of the SAR acquisitions\n' + + 'delaunay - select based on delaunay triangulars\n' + + 'stars - one master image network, like PS.') + parser.add_argument('--conNumb', dest='conNumb', type=int, default=2, + help='Number of the neibour-connected SAR images at one side for sequential method.') + parser.add_argument('--coreg', dest='coreg', action='store_true', help='Using rslc/rslcPar to check orbit history.') + + inps = parser.parse_args() + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Select interferometric pairs based on SLC_TAB. + [sbas, sequential, delaunay, stars are supported.] + +''' + +EXAMPLE = ''' + Examples: + select_pairs.py PacayaT163TsxHhA + select_pairs.py PacayaT163TsxHhA --max-tb 300 --min-tb 200 --max-sb 100 + select_pairs.py PacayaT163TsxHhA --method sequential --conNumb 3 + select_pairs.py PacayaT163TsxHhA --method denaulay --max-tb 100 --max-sb 100 + select_pairs.py PacayaT163TsxHhA --method stars + select_pairs.py PacayaT163TsxHhA --coreg +------------------------------------------------------------------- +''' + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + templateDict['network_method'] = inps.method + templateDict['conNumb'] = inps.conNumb + + processDir = scratchDir + '/' + projectName + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + '/RSLC' + + #slc_list = ut.get_project_slcList(projectName) + masterDate = templateDict['masterDate'] + + if inps.coreg: + slcDir = rslcDir + slc0 = '.rslc' + slcpar0 = '.rslc.par' + else: + slc0 = '.rslc' + slcpar0 = '.rslc.par' + + slc_list0 = [os.path.basename(fname) for fname in sorted(glob.glob(rslcDir + '/*'))] + slc_list = [] + for k0 in slc_list0: + if ut.is_number(k0): + slc_list.append(k0) + slc_list = sorted(slc_list) + + SLCfile = [] + SLCParfile = [] + for kk in range(len(slc_list)): + #str_slc = slcDir + "/" + slc_list[kk] +"/" + slc_list[kk] + slc0 + str_slc = rslcDir + "/" + slc_list[kk] +"/" + slc_list[kk] + slc0 + str_slc_par = rslcDir + "/" + slc_list[kk] +"/" + slc_list[kk] + slcpar0 + SLCfile.append(str_slc) + SLCParfile.append(str_slc_par) + + RefPar = rslcDir + "/" + masterDate +"/" + masterDate + slcpar0 + SLC_Tab = scratchDir + '/' + projectName + "/SLC_Tab" + + SLC_Tab_all = processDir + "/SLC_Tab_all" + TS_Berp_all = processDir + "/TS_Berp_all" + TS_Itab_all = processDir + "/TS_Itab_all" + itab_type = '1' + pltflg = '0' + + if os.path.isfile(SLC_Tab): os.remove(SLC_Tab) + with open(SLC_Tab, 'a') as f: + for kk in range(len(SLCfile)): + f.write(str(SLCfile[kk])+ ' '+str(SLCParfile[kk])+'\n') + + ## Get all of the pairs using GAMMA + call_str = "base_calc " + SLC_Tab + " " + RefPar + " " + TS_Berp_all + " " + TS_Itab_all + " " + '1 0 ' + '- - - - >/dev/null' + os.system(call_str) + + with open(TS_Berp_all, 'r') as f: + lines = f.readlines() + + TS_Net_all = ut.read_txt2array(TS_Berp_all) + if len(lines) ==1: + TS_Net_all = TS_Net_all.reshape(1,9) + + TS_Net = TS_Net_all[0:(len(SLCfile)-1),:] + date_list, tbase_list, pbase_list = get_datelist_bperplist(TS_Net) + bl_list_txt = scratchDir + '/' + projectName + '/bl_list.txt' + with open(bl_list_txt, 'w') as f: + for kk in range(len(SLCfile)): + f.write(str(date_list[kk])+ ' '+str(pbase_list[kk])+'\n') + + exclude_list = [] + if 'exclude_date' in templateDict: + exclude_list = templateDict['exclude_date'].split(',')[:] + if inps.excludeDate: + exclude_list0 = inps.excludeDate + for k0 in exclude_list0: + exclude_list.append(k0) + templateDict['exclude_list'] = exclude_list + + if inps.startDate: templateDict['startDate'] = inps.startDate + if inps.endDate: templateDict['endDate'] = inps.endDate + if inps.maxTB: templateDict['max_tb'] = inps.maxTB + if inps.maxSB: templateDict['max_sb'] = inps.maxSB + + date_list, tbase_list, pbase_list = prune_datelist(date_list, tbase_list, pbase_list, templateDict) + date12_list, ifgram_tbase_list, ifgram_pbase_list = select_network_candidate(date_list,tbase_list,pbase_list,templateDict) + date12_list_final, ifgram_tbase_list_final, ifgram_pbase_list_final = prune_network(date12_list, ifgram_tbase_list, ifgram_pbase_list, templateDict) + + if inps.outfile: out_file = inps.outfile + else: out_file = scratchDir + '/' + projectName + "/ifgram_list.txt" + + write_ifgram_list(date12_list_final,ifgram_tbase_list_final,ifgram_pbase_list_final, out_file) + sys.exit(1) + +#################################################################### +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/select_paris_by_cor.py b/.codex_tmp/pyint_variants/no_rescue/pyint/select_paris_by_cor.py new file mode 100644 index 0000000..ae3b0fc --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/select_paris_by_cor.py @@ -0,0 +1,139 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT ### +### Author: chen ### +### Contact : chenweicug@126.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse +from pyint import _utils as ut +import cv2 + + +INTRODUCTION = ''' +------------------------------------------------------------------- + compare two corherence for SAR images for delete the bad images . + +''' + +EXAMPLE = ''' + Usage: + cor_correlation.py projectName Mdate Sdate + cor_correlation.py PacayaT163TsxHhA 20150102 20150601 ifgrams_list.txt +------------------------------------------------------------------- +''' +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister all of the SLCs to the reference SAR image using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('Mdate',help='Master date.') + parser.add_argument('Sdate',help='Slave date.') + parser.add_argument('ifgs', help='provided ifgram_list_txt. default: using ifgram_list.txt under projectName folder.') + parser.add_argument('correlation_index', help='provided the accept correlation for the file . default: 0.55.') + + inps = parser.parse_args() + return inps + + +def main(argv): + + start_time = time.time() + inps = cmdLineParse() + Mdate = inps.Mdate + Sdate = inps.Sdate + + if inps.correlation_index: Score = inps.correlation_index + else: Score = 0.55 + + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + + projectDir = scratchDir + '/' + projectName + + slcDir = scratchDir + '/' + projectName + '/SLC' + rslcDir = scratchDir + '/' + projectName + '/RSLC' + ifgDir = projectDir + '/ifgrams' + badifgDir = projectDir + '/bad_ifgrams' + if not os.path.isdir(ifgDir): os.mkdir(ifgDir) + if not os.path.isdir(badifgDir): os.mkdir(badifgDir) + Pair = Mdate + '-' + Sdate + workDir = ifgDir + '/' + Pair + + ####################################################################### + Mamp = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + Samp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + Mrslc = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc' + MrslcPar = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc.par' + Srslc = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc' + SrslcPar = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc.par' + + if inps.ifgs: ifgramList_txt = inps.ifgs + else: ifgramList_txt = scratchDir + '/' + projectName + '/ifgram_list.txt' + MasterPar = rslcDir + '/' + masterDate + '/' + masterDate + '.rslc.par' + first_image= workDir + '/' + Pair + '_' + rlks + 'rlks.diff_filt.cor.bmp' + ifgList0 = ut.read_txt2array(ifgramList_txt) +# 加载第一张图 + image1 = cv2.imread(first_image) + ifgList0 = ut.read_txt2array(ifgramList_txt) +# ifgList = ifgList0[:,0] + if len(ifgList0)==3: + ifgList=ifgList0[0] + ifgList=[ifgList] + + else: + # ifgList=ifgList0[:,0] + ifgList=ifgList0 + err_txt = scratchDir + '/' + projectName + '/cor_correlation_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + out_file='cor_correltion.txt' + if os.path.isfile(out_file): os.remove(out_file) + for i in range(len(ifgList)): + m0 = ut.yyyymmdd(ifgList[i].split('-')[0]) + s0 = ut.yyyymmdd(ifgList[i].split('-')[1]) + Pair2 = m0 + '-' + s0 + workdir = ifgDir + '/' + Pair2 + # 加载第二张图片 + second_image = workdir + '/' + Pair2 + '_' + rlks + 'rlks.diff_filt.cor.bmp' + image2=cv2.imread(second_image) + # 将图片转换为灰度图像 + gray_image1 = cv2.cvtColor(image1, cv2.COLOR_BGR2GRAY) + gray_image2 = cv2.cvtColor(image2, cv2.COLOR_BGR2GRAY) + # 计算两张图片之间的结构相似性指数（SSIM） + + ssim_score = cv2.matchTemplate(gray_image1, gray_image2, cv2.TM_CCOEFF_NORMED) + score = ssim_score[0][0] + + if str(score) < Score: + print(m0, "-", s0, "相关性得分：", ssim_score[0][0]) + call_str=' mv ' + workdir + ' ' + badifgDir + '/' + os.system(call_str) + else: + print("This pair", m0,'-', s0,"is accept") + call_str = 'echo ' + m0 + '-' + s0 + ' ' + ' ' + str(score) + '>>' + out_file + os.system(call_str) + + print("Delete the bad interferograms for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/single_GACOS_correction.csh b/.codex_tmp/pyint_variants/no_rescue/pyint/single_GACOS_correction.csh new file mode 100644 index 0000000..09bea26 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/single_GACOS_correction.csh @@ -0,0 +1,126 @@ +#!/bin/csh -f + +if ($#argv != 7) then + echo "" + echo "Usage: single_GACOS_correction.csh master_ztd master_ztd.rsc slave_ztd slave_ztd.rsc los.grd reference_point incidence_angle" + echo "" + echo "Performs gacos correction for Sentinel 1 for a single interferogram in grd format" + echo "" + echo "Output: corrected unwrapped phase" + echo "" + echo "Reference point text file in lon lat format" + echo "" + echo "" + echo "Indicence angle in degrees (float or integer)" + echo "" + exit 1 + endif + +#Checking input files +if !(-e $1) then + echo "master_ztd binary file: $1 seems not to exist" + exit 1 +endif +if !(-e $2) then + echo "master_ztd.rsc file: $2 seems not to exist" + exit 1 +endif +if !(-e $3) then + echo "slave_ztd binary file: $3 seems not to exist" + exit 1 +endif +if !(-e $4) then + echo "slave_ztd.rsc file: $4 seems not to exist" + exit 1 +endif +if !(-e $5) then + echo "Interferogram to correct: $5 seems not to exist" + exit 1 +endif +if !(-e $6) then + echo "Reference point text file: $6 seems not to exist" + exit 1 +endif + + +#reference point/stable point in radar coordinates +set reference_point = $6 + +#incidence angle obtained with the "SAT_look" script in degrees +set incidence = $7 + +#wavelength for Sentinel 1 (m) +set wavelength = 0.055165 +set pi = 3.141592653589793238462 + +#######FIRST ZTD to grid######### +set x_first_d1 = `cat $2|grep X_FIRST|awk '{print $2}'` +set y_first_d1 = `cat $2|grep Y_FIRST|awk '{print $2}'` +set width_d1 = `cat $2|grep WIDTH|awk '{print $2}'` +set length_d1 = `cat $2|grep FILE_LENGTH|awk '{print $2}'` +set x_step_d1 = `cat $2|grep X_STEP|awk '{print $2}'` +set y_step_d1 = `cat $2|grep X_STEP|awk '{print $2}'` +gmt xyz2grd $1 -G"date1_ztd.grd" -RLT$x_first_d1/$y_first_d1/$width_d1/$length_d1 -I$x_step_d1/$y_step_d1 -ZTLf -di0 -r + + +#######SECOND ZTD to grid######### +set x_first_d2 = `cat $4|grep X_FIRST|awk '{print $2}'` +set y_first_d2 = `cat $4|grep Y_FIRST|awk '{print $2}'` +set width_d2 = `cat $4|grep WIDTH|awk '{print $2}'` +set length_d2 = `cat $4|grep FILE_LENGTH|awk '{print $2}'` +set x_step_d2 = `cat $4|grep X_STEP|awk '{print $2}'` +set y_step_d2 = `cat $4|grep X_STEP|awk '{print $2}'` +gmt xyz2grd $3 -G"date2_ztd.grd" -RLT$x_first_d2/$y_first_d2/$width_d2/$length_d2 -I$x_step_d2/$y_step_d2 -ZTLf -di0 -r +gmt grdsample -R"date1_ztd.grd" date2_ztd.grd -I3s -Gtmp.grd +mv tmp.grd date2_ztd.grd +#TIME DIFFERENCE +gmt grdmath date2_ztd.grd date1_ztd.grd SUB = zpddm.grd +#######RESAMPLE DIFFERENCE WITH THE INTERFEROGRAM PARAMETERS#### +set xmin = `gmt grdinfo -C $5|awk '{print $2}'` +set xmax = `gmt grdinfo -C $5|awk '{print $3}'` +set ymin = `gmt grdinfo -C $5|awk '{print $4}'` +set ymax = `gmt grdinfo -C $5|awk '{print $5}'` +set xinc = `gmt grdinfo -C $5|awk '{print $8}'` +set yinc = `gmt grdinfo -C $5|awk '{print $9}'` +#Resample +#gmt grdsample zpddm.grd -Gresample_zpddm.grd -R$xmin/$xmax/$ymin/$ymax -I$xinc/$yinc -r +#gmt grd2xyz zpddm.grd >tmp +#gmt xyz2grd -R$xmin/$xmax/$ymin/$ymax -I$xinc/$yinc tmp -Gresample_zpddm.grd +gmt grdsample zpddm.grd -Gresample_zpddm.grd -R$5 -I$xinc/$yinc + +#FROM METERS TO PHASE +#gmt grdmath resample_zpddm.grd $wavelength DIV 4 MUL $pi MUL = resample_zpddm_phs.grd +# multiplying by -4 (edition 13/4/2023) +gmt grdmath resample_zpddm.grd $wavelength DIV -4 MUL $pi MUL = resample_zpddm_phs.grd +#cp resample_zpddm.grd resample_zpddm_phs.grd +#REFERENCE POINT +set ref_value_zpddm = `gmt grdtrack $reference_point -Gresample_zpddm_phs.grd -Z` +set ref_value_phase = `gmt grdtrack $reference_point -G$5 -Z` +gmt grdmath resample_zpddm_phs.grd $ref_value_zpddm SUB = szpddm.grd +gmt grdmath $5 $ref_value_phase SUB = phase_ref.grd + +#FROM METER TO PHASE +#gmt grdmath szpddm.grd 4 MUL $pi MUL $wavelength DIV = szpddm_phase.grd +#gmt grdinfo szpddm.grd +#PROJECTION FROM ZENITH VIEW TO LOS +#gmt grdmath szpddm_phase.grd $incidence COSD DIV = szpddm_phase_LOS.grd +gmt grdmath szpddm.grd $incidence COSD DIV = szpddm_LOS.grd +#CORRECTION WITH GACOS DATA +#UNITS: Phase +gmt grdmath phase_ref.grd szpddm_LOS.grd SUB = phase_GACOS_corrected_phs.grd + +#DETRENDING +#UNITS: phase +gmt grdtrend phase_GACOS_corrected_phs.grd -N3r -Dphase_GACOS_corrected_phs_detrended.grd +gmt grdmath phase_GACOS_corrected_phs.grd $wavelength MUL -4 DIV $pi DIV = phase_GACOS_corrected_los.grd +gmt grdmath phase_GACOS_corrected_phs_detrended.grd $wavelength MUL -4 DIV $pi DIV = phase_GACOS_corrected_los_detrended.grd + +#clean up +# +rm date1_ztd.grd date2_ztd.grd +rm zpddm.grd +rm resample_zpddm.grd resample_zpddm_phs.grd +rm phase_ref.grd +rm szpddm.grd +rm szpddm_LOS.grd +echo "corrections done with GACOS files $1 and $3 over interferogram: $5" diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/slc2ifg.py b/.codex_tmp/pyint_variants/no_rescue/pyint/slc2ifg.py new file mode 100644 index 0000000..43d1d26 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/slc2ifg.py @@ -0,0 +1,138 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import time +import glob +import argparse + +from pyint import _utils as ut + + +def get_s1_date(raw_file): + file0 = os.path.basename(raw_file) + date = file0[17:25] + return date + +def get_satellite(raw_file): + if 'S1A_IW_SLC_' in raw_file: + s0 = 'A' + else: + s0 = 'B' + + return s0 + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate Ifg from SLC using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='name of the project.') + parser.add_argument('Mdate',help='master date for interferometry.') + parser.add_argument('Sdate',help='slave date for interferometry.') + + inps = parser.parse_args() + + return inps + + +INTRODUCTION = ''' +-------------------------------------------------------------- + Generate unwrapped differential Ifg from SLC using GAMMA. + + Note: SRTM-1 will be downloaded and processed automatically + if not provided in the template file. + +''' + +EXAMPLE = """Usage: + + slc2ifg.py projectName Mdate Sdate +-------------------------------------------------------------- +""" + +def main(argv): + + start_time = time.time() + inps = cmdLineParse() + Mdate = inps.Mdate + Sdate = inps.Sdate + + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + #downDir = scratchDir + '/' + projectName + '/DOWNLOAD' + #M_raw = glob.glob(downDir + '/S1*_' + ut.yyyymmdd(Mdate)+'*')[0] + #S_raw = glob.glob(downDir + '/S1*_' + ut.yyyymmdd(Sdate)+'*')[0] + slcDir = scratchDir + '/' + projectName + '/SLC' + + + ######### down 2 slc ############# + #call_str = 'down2slc_sen.py ' + M_raw + ' ' + slcDir + #os.system(call_str) + + #call_str = 'down2slc_sen.py ' + S_raw + ' ' + slcDir + #os.system(call_str) + + ########## extract common bursts ## + if 'S1' in projectName: + call_str = 'extract_s1_bursts.py ' + projectName + ' ' + Mdate + os.system(call_str) + + call_str = 'extract_s1_bursts.py ' + projectName + ' ' + Sdate + os.system(call_str) + + ######### generate rdc_dem ########## + call_str = 'generate_rdc_dem.py ' + projectName + os.system(call_str) + + ########## coregister SLC ######## + if 'S1' in projectName: + call_str = 'coreg_s1_gamma.py ' + projectName + ' ' + Mdate + os.system(call_str) + + call_str = 'coreg_s1_gamma.py ' + projectName + ' ' + Sdate + os.system(call_str) + else: + call_str = 'coreg_gamma.py ' + projectName + ' ' + Mdate + os.system(call_str) + + call_str = 'coreg_gamma.py ' + projectName + ' ' + Sdate + os.system(call_str) + + ######## Interferometry process ########### + call_str = 'diff_gamma.py ' + projectName + ' ' + Mdate + ' ' + Sdate + os.system(call_str) + + call_str = 'unwrap_gamma.py ' + projectName + ' ' + Mdate + ' ' + Sdate + os.system(call_str) + + call_str = 'geocode_gamma.py ' + projectName + ' ' + Mdate + '-' + Sdate + os.system(call_str) + + print("Generate Ifg from SLC data is done! ") + ut.print_process_time(start_time, time.time()) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/slc_sen_cat.py b/.codex_tmp/pyint_variants/no_rescue/pyint/slc_sen_cat.py new file mode 100644 index 0000000..e1ff019 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/slc_sen_cat.py @@ -0,0 +1,182 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +from pyint import _utils as ut +def get_s1_date(raw_file): + file0 = os.path.basename(raw_file) + date = file0[17:25] + return date + +def get_satellite(raw_file): + if 'S1A_IW_SLC_' in raw_file: + s0 = 'A' + else: + s0 = 'B' + + return s0 + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLC from Sentinel-1 raw data with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='project name. e.g., ChangningT55S1A') + parser.add_argument('date',help='date to be processed. e.g., 20180101') + + inps = parser.parse_args() + + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Generate SLC from Sentinel-1 raw data using S1_import_SLC_from_zipfiles with orbit correction. + [Precise orbit data will be downloaded automatically] +''' + +EXAMPLE = """Usage: + + slc_sen_cat.py projectName date + + slc_sen_cat.py ChangningT55S1A 20180517 + +------------------------------------------------------------------- +""" + + + + + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + date = inps.date + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + slc_dir = projectDir + '/SLC' + down_dir = scratchDir + '/' + projectName + "/DOWNLOAD" + opod_dir = projectDir + '/OPOD' + + if not os.path.isdir(slc_dir): + os.mkdir(slc_dir) + if not os.path.isdir(opod_dir): + os.mkdir(opod_dir) + + work_dir = slc_dir + '/' + date + if not os.path.isdir(work_dir): + os.mkdir(work_dir) + + os.chdir(work_dir) + + t_date = 't_' + date + + call_str = 'ls ' + down_dir + '/S1*' + date + '* > ' + t_date + os.system(call_str) + start_swath = templateDict['start_swath'] + end_swath = templateDict['end_swath'] + k_swath = ut.get_sardata_swath(start_swath,end_swath) + + call_str = 'grep ' + date + ' ' + t_date + ' > t1_' + date + os.system(call_str) + t1_date = 't1_' + date + raw_files = ut.read_txt2list(t1_date) + raw_files = sorted(raw_files) + satellite = get_satellite(str(raw_files[0])) + orbit_file = ut.download_s1_orbit(date,opod_dir,satellite=satellite) + + for i in range(len(raw_files)): + zipfile_ref=str(raw_files[i]) + outfile_name=zipfile_ref.split('/')[-1].split('.')[0] + burst_number_table_ref=outfile_name + '.burst_number_table' + call_str = 'S1_BURST_tab_from_zipfile ' + t1_date + ' ' + str(raw_files[0]) + ' - 1' + os.system(call_str) + + call_str = call_str = 'grep ' + outfile_name + ' ' + 't1_' + date + ' > t2_'+date + os.system(call_str) + + t2_date = 't2_' + date + #call_str = 'S1_import_SLC_from_zipfiles ' + t2_date + ' ' + burst_number_table_ref + ' vv 0 ' + k_swath + call_str = 'S1_import_SLC_from_zipfiles ' + t2_date + ' ' + burst_number_table_ref + ' vv 0 ' + k_swath + ' ' + opod_dir + ' 1 1 ' + os.system(call_str) + + os.chdir(work_dir) + #call_str = "rename vv.slc.iw1 IW1_" + str(i+1) + ".slc * " + call_str = "rename 's/vv.slc.iw1/IW1_" + str(i+1) + ".slc/g' *" + os.system(call_str) + call_str = "rename 's/vv.slc.iw2/IW2_" + str(i+1) + ".slc/g' *" + #call_str = "rename vv.slc.iw2 IW2_" + str(i+1) + ".slc * " + os.system(call_str) + call_str = "rename 's/vv.slc.iw3/IW3_" + str(i+1) + ".slc/g' *" + #call_str = "rename vv.slc.iw3 IW3_" + str(i+1) + ".slc * " + os.system(call_str) + SLC_Tab = work_dir + '/' + date + '_' + str(i+1) + '_SLC_Tab' + SLC_list = sorted(glob.glob(work_dir + '/*IW?_' + str(i+1) + '.slc')) + SLC_par_list = sorted(glob.glob(work_dir + '/*IW?_' + str(i+1) + '.slc.par')) + TOP_par_list = sorted(glob.glob(work_dir + '/*IW?_' + str(i+1) + '.slc.TOPS_par')) + + for kk in range(len(SLC_list)): + call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' >> ' + SLC_Tab + os.system(call_str) + + + SLC_Tab = work_dir + '/' + date + '_SLC_Tab' + SLC_list = len(raw_files) + SLC_tab1 = date + '_' + '1_SLC_Tab' + SLC_tab2 = date + '_' + '2_SLC_Tab' + #call_str = 'grep "IW*" ' + date +'_1_SLC_Tab | wc -l' + n_swath = len(open(SLC_tab1, 'r').readlines()) + for kk in range(n_swath): + call_str = 'echo ' + date + '.IW' + str(kk+1) + '.slc ' + date + '.IW' + str(kk+1) + '.slc.par ' + date + '.IW' + str(kk+1) + '.slc.TOPS_par ' + ' >> ' + SLC_Tab + os.system(call_str) + + call_str = 'SLC_cat_S1_TOPS ' + SLC_tab1 + ' ' + SLC_tab2 + ' ' + SLC_Tab + os.system(call_str) + SLC_list = sorted(glob.glob(work_dir + '/*IW?.slc')) + SLC_par_list = sorted(glob.glob(work_dir + '/*IW?.slc.par')) + TOP_par_list = sorted(glob.glob(work_dir + '/*IW?.slc.TOPS_par')) + for kk in range(len(SLC_list)): + BURST = SLC_par_list[kk].replace('slc.par','burst.par') + call_str = 'SLC_burst_corners ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' > ' +BURST + os.system(call_str) + print("Down to SLC for %s is done! " % date) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + + + + + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/test_srtm_manual.py b/.codex_tmp/pyint_variants/no_rescue/pyint/test_srtm_manual.py new file mode 100644 index 0000000..91bc344 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/test_srtm_manual.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +""" +测试手动处理 SRTM .hgt 文件的脚本 + +这个脚本演示如何在 Python < 3.12 环境下手动处理 SRTM 数据 +""" + +import os +import sys + +print("=" * 80) +print("SRTM 手动处理功能测试") +print("=" * 80) + +# 检查Python版本 +print(f"\nPython版本: {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}") + +# 导入处理函数 +from makedem import process_srtm_hgt_files, HAS_SRTM + +print(f"\nsrtm库状态: {'已安装' if HAS_SRTM else '未安装 (需要Python >= 3.12)'}") + +print("\n" + "=" * 80) +print("使用说明") +print("=" * 80) + +print(""" +步骤1: 下载SRTM .hgt文件 +--------------------------- +从以下网站下载所需区域的 .hgt 文件: + +1. CSI-CGIAR SRTM (推荐,无需注册) + https://srtm.csi.cgiar.org/ + +2. USGS EarthExplorer (需要注册) + https://earthexplorer.usgs.gov/ + +文件命名规则: +- N39E116.hgt → 北纬39-40°, 东经116-117° +- N39W110.hgt → 北纬39-40°, 西经110-109° +- S10E120.hgt → 南纬10-9°, 东经120-121° + +步骤2: 组织文件 +--------------------------- +将所有 .hgt 文件放在同一目录下: + +mkdir -p ~/data/SRTM +mv N*.hgt S*.hgt ~/data/SRTM/ + +步骤3: 运行命令 +--------------------------- +使用以下命令生成DEM: + +makedem.py -r 116/117/39/40 --dem-source srtm --srtm-data-dir ~/data/SRTM + +或者直接调用Python函数: + +from makedem import process_srtm_hgt_files + +dem_file = process_srtm_hgt_files( + west=116.0, + south=39.0, + east=117.0, + north=40.0, + srtm_data_dir='/path/to/srtm/data', + save_path='/path/to/output' +) + +""") + +print("=" * 80) +print("示例: 为北京地区(116-117°E, 39-40°N)下载SRTM数据") +print("=" * 80) + +print(""" +需要下载的文件: +1. 访问 https://srtm.csi.cgiar.org/ +2. 选择区域: 39-40°N, 116-117°E +3. 下载文件: N39E116.hgt + +下载命令示例: +-------------""") +print(f""" +# 创建数据目录 +mkdir -p ~/data/SRTM + +# 假设已下载 N39E116.hgt 到 ~/Downloads +mv ~/Downloads/N39E116.hgt ~/data/SRTM/ + +# 生成DEM +makedem.py -r 116/117/39/40 --dem-source srtm --srtm-data-dir ~/data/SRTM + +# 输出文件将保存为: SRTM_116_117_39_40.tif +""") + +print("=" * 80) +print("注意事项") +print("=" * 80) + +print(""" +1. SRTM文件必须覆盖整个目标区域 + - 对于区域 116-117°E, 39-40°N + - 需要 N39E116.hgt 文件 + +2. 文件格式 + - 支持 .hgt 文件 (未压缩) + - 如果下载的是 .hgt.zip, 请先解压 + +3. 空数据区域 + - SRTM数据在海洋区域可能为空值 + - 程序会自动处理空值区域 + +4. Python版本兼容性 + - Python >= 3.12: 可使用 srtm 库(自动查询) + - Python < 3.12: 使用手动处理方法(GDAL合并) + - 两种方法结果相同,只是处理方式不同 +""") + +print("=" * 80) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/test_tiled.dem b/.codex_tmp/pyint_variants/no_rescue/pyint/test_tiled.dem new file mode 100644 index 0000000..266abaa Binary files /dev/null and b/.codex_tmp/pyint_variants/no_rescue/pyint/test_tiled.dem differ diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/test_tiled.dem.par b/.codex_tmp/pyint_variants/no_rescue/pyint/test_tiled.dem.par new file mode 100644 index 0000000..3123a30 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/test_tiled.dem.par @@ -0,0 +1,27 @@ +Gamma DIFF&GEO DEM/MAP parameter file +title: IMPORTED DEM FROM Copernicus30 +DEM_projection: EQA +data_format: INTEGER*2 +DEM_hgt_offset: 0.00000 +DEM_scale: 1.00000 +width: 6000 +nlines: 4800 +corner_lat: 28.000416666673939 decimal degrees +corner_lon: 101.999583333269271 decimal degrees +post_lat: -0.000833333333333 decimal degrees +post_lon: 0.000833333333333 decimal degrees + +ellipsoid_name: WGS 84 +ellipsoid_ra: 6378137.000 m +ellipsoid_reciprocal_flattening: 298.2572236 + +datum_name: WGS 1984 +datum_shift_dx: 0.000 m +datum_shift_dy: 0.000 m +datum_shift_dz: 0.000 m +datum_scale_m: 0.00000e+00 +datum_rotation_alpha: 0.00000e+00 arc-sec +datum_rotation_beta: 0.00000e+00 arc-sec +datum_rotation_gamma: 0.00000e+00 arc-sec +datum_country_list Global Definition, WGS84, World + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/tsview_mintpy_ssa.py b/.codex_tmp/pyint_variants/no_rescue/pyint/tsview_mintpy_ssa.py new file mode 100644 index 0000000..4e24c1f --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/tsview_mintpy_ssa.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +# Load the usual suspects: +import os +import sys +import argparse +import pandas as pd +import numpy as np +import math +import matplotlib.pyplot as plt +from mintpy.utils import utils as ut +from ssa import SSA +from sklearn.metrics import mean_squared_error + + +def create_parser(): + parser = argparse.ArgumentParser(description='SSA time series analysis for a specific location') + parser.add_argument('ts_dir', help='Time series directory (e.g., ./SBAS_atm_gacos)') + parser.add_argument('ts_file', help='Time series file name (e.g., geo/geo_timeseries_SET_GACOS_ramp_demErr.h5)') + parser.add_argument('lat', type=float, help='Latitude of the point') + parser.add_argument('lon', type=float, help='Longitude of the point') + parser.add_argument('-L', '--window', type=int, default=8, help='Window length for SSA (default: 8)') + parser.add_argument('-o', '--output', default='ssa_ts.txt', help='Output file name (default: ssa_ts.txt)') + return parser + + +def parse_args(): + parser = create_parser() + return parser.parse_args() + + +def DecYr(x): + return x.dt.to_period('D').dt.to_timestamp().dt.year + x.dt.to_period('D').dt.to_timestamp().dt.dayofyear / 365.25 + + +def sigma_sum(Sigma): + sigma_sumsq = (Sigma**2).sum() + return Sigma**2 / sigma_sumsq * 100 + + +def main(): + args = parse_args() + + proj_dir = os.path.expanduser(args.ts_dir) + ts_file = os.path.join(proj_dir, args.ts_file) + geom_file = None + + print(f"Reading time series from: {ts_file}") + print(f"Location: lat={args.lat}, lon={args.lon}") + print(f"Window length: {args.window}") + + dates, dis, std = ut.read_timeseries_lalo(lat=args.lat, lon=args.lon, ts_file=ts_file, lookup_file=geom_file) + + # Convert from meter to mm and save to panda df + df = pd.DataFrame({'date': dates, 'dis': dis*1000}) + + # Decomposition + F_ssa = SSA(dis*1000, args.window) + contri = sigma_sum(F_ssa.Sigma) + df_ssa = pd.concat([F_ssa.components_to_df()], axis=1) + np.savetxt('sigma_ssa.txt', np.c_[np.arange(1, args.window+1), contri], fmt="%.2f") + df_comp = pd.concat([df['date'], F_ssa.components_to_df()], axis=1) + print('\n \n', 'Writing SSA all components to: ssa_all_compo.txt ', '\n') + df_comp.to_csv('ssa_all_compo.txt', index=False, header=True, sep='\t') + + plt.rcParams.update({'font.size': 10}) + fig, ax = plt.subplots(nrows=args.window, ncols=1, sharex=True, figsize=(5, 11*args.window)) + for i, column in enumerate(df_ssa.columns): + ax[i].plot(df_ssa.index, df_ssa[column], label=column) + ax[i].set_title(f'{column} ({contri[i]:.1f}%)') + plt.subplots_adjust(hspace=.5) + + # Reconstruction + trend_indices = list(range(min(2, args.window))) + noise_indices = list(range(2, args.window)) + + df['ssa_trend'] = pd.DataFrame(F_ssa.reconstruct(trend_indices)) + df['noise'] = pd.DataFrame(F_ssa.reconstruct(noise_indices)) if noise_indices else 0 + print('\n \n', 'Writing to: ', args.output, '\n') + df.to_csv(args.output, index=False, header=True, sep='\t') + + fig, ax = plt.subplots(nrows=3, ncols=1, sharex=True, figsize=(10, 10)) + + ax[0].plot(df['date'], df['dis'], label='InSAR') + ax[0].set_title('InSAR') + ax[1].plot(df['date'], df['noise'], label='Noise') + ax[1].set_title('Periodic+Noise') + ax[2].plot(df['date'], df['ssa_trend'], label='Trend') + ax[2].set_title('Trend') + plt.show() + + +if __name__ == '__main__': + main() diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/unwrap_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/unwrap_gamma.py new file mode 100644 index 0000000..0995fc0 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/unwrap_gamma.py @@ -0,0 +1,156 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +from PIL import Image +from pylab import * +import argparse + +from pyint import _utils as ut + + +def _run_or_raise(call_str, stage): + rc = os.system(call_str) + if rc != 0: + raise RuntimeError('%s failed with rc=%s: %s' % (stage, rc, call_str)) + return rc + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Unwrap differential interferogram using GAMMA. + [Only support mcf, not implement branch_cut yet] + +''' + +EXAMPLE = ''' + Usage: + unwrap_gamma.py projectName Mdate Sdate + unwrap_gamma.py PacayaT163TsxHhA 20150102 20150601 +------------------------------------------------------------------- +''' + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Unwrap differential interferogram using GAMMA-mcf method.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('Mdate',help='Master date.') + parser.add_argument('Sdate',help='Slave date.') + + inps = parser.parse_args() + return inps + + +def main(argv): + + inps = cmdLineParse() + Mdate = inps.Mdate + Sdate = inps.Sdate + + projectName = inps.projectName + Sdate = inps.Sdate + Mdate = inps.Mdate + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + auto_unw = templateDict['auto_unw'] + init_flag = templateDict['init_flag'] + r_refer = templateDict['r_refer'] + a_refer = templateDict['a_refer'] + make_mask = templateDict['make_mask'] + processDir = scratchDir + '/' + projectName + "/PROCESS" + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + '/RSLC' + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + + Pair = Mdate + '-' + Sdate + workDir = ifgDir + '/' + Pair + + ################ copy file for parallel processing ############### + Mamp0 = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar0 = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + Samp0 = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar0 = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + + Mamp = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + Samp = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + ut.copy_file(Mamp0,Mamp) + ut.copy_file(Samp0,Samp) + ut.copy_file(MampPar0,MampPar) + ut.copy_file(SampPar0,SampPar) + ############################################################### + + nWidth = ut.read_gamma_par(MampPar, 'read', 'range_samples') + nLine = ut.read_gamma_par(MampPar, 'read', 'azimuth_lines') + if auto_unw == '1': + if str(r_refer).strip() in ('', '-'): + r_refer = str(int(int(nWidth) / 2)) + if str(a_refer).strip() in ('', '-'): + a_refer = str(int(int(nLine) / 2)) + unwrap_window = '0 0 ' + nWidth + ' ' + nLine + + CORMASK = workDir + '/' + Pair + '_' +rlks + 'rlks.diff_filt.cor' + WRAPlks = workDir + '/' + Pair + '_' +rlks + 'rlks.diff_filt' + UNWlks = workDir + '/' + Pair + '_' +rlks + 'rlks.diff_filt.unw' + + CORMASKbmp = CORMASK.replace('.diff_filt.cor','.diff_filt.cor_mask.bmp') + + if os.path.isfile(CORMASKbmp): + os.remove(CORMASKbmp) + + call_str = 'rascc_mask ' + CORMASK + ' ' + Mamp + ' ' + nWidth + ' 1 1 0 1 1 ' + templateDict['unwrapThreshold'] + ' 0.0 0.1 0.9 1. .35 1 ' + CORMASKbmp # based on int coherence + _run_or_raise(call_str, 'rascc_mask') + + if auto_unw == '1': + if make_mask == "1": + call_str = 'mcf ' + WRAPlks + ' ' + CORMASK + ' ' + CORMASKbmp + ' ' + UNWlks + ' ' + nWidth + ' ' + templateDict['mcf_triangular'] + ' ' + unwrap_window + ' ' + templateDict['unwrap_patr'] + ' ' + templateDict['unwrap_pataz'] +' - '+ r_refer + ' ' + a_refer + ' ' + init_flag + print(call_str) + _run_or_raise(call_str, 'mcf_masked') + else: + call_str = 'mcf ' + WRAPlks + ' ' + CORMASK + ' - ' + ' ' + UNWlks + ' ' + nWidth + ' ' + templateDict['mcf_triangular'] + ' ' + unwrap_window + ' ' + templateDict['unwrap_patr'] + ' ' + templateDict['unwrap_pataz'] + ' - ' + r_refer + ' ' + a_refer + ' ' + init_flag + _run_or_raise(call_str, 'mcf_unmasked') + else: + im = array(Image.open(CORMASKbmp)) + imshow(im) + print('Please select the reference points,max click 10 times') + x =ginput(10) + print ('you clicked:',x) + ref_point=list(x[-1]) + r_init=int(ref_point[0]) + a_init=int(ref_point[1]) + if make_mask == '1': + call_str = 'mcf ' + WRAPlks + ' ' + CORMASK + ' ' + CORMASKbmp + ' ' + UNWlks + ' ' + nWidth + ' ' + templateDict['mcf_triangular'] + ' ' + unwrap_window + ' ' + templateDict['unwrap_patr'] + ' ' + templateDict['unwrap_pataz'] + ' - ' + str(r_init) + ' ' + str(a_init) + ' ' + init_flag + _run_or_raise(call_str, 'mcf_masked_manual') + else: + call_str = 'mcf ' + WRAPlks + ' - ' + ' - ' + ' ' + UNWlks + ' ' + nWidth + ' ' + templateDict['mcf_triangular'] + ' ' + unwrap_window + ' ' + templateDict['unwrap_patr'] + ' ' + templateDict['unwrap_pataz'] + ' - ' + str(r_init) + ' ' + str(a_init) + ' ' + init_flag + _run_or_raise(call_str, 'mcf_unmasked_manual') + call_str = 'rasdt_pwr ' + UNWlks + ' ' + Mamp + ' ' + nWidth + ' 1 0 1 1 -3.14 3.14 1' + _run_or_raise(call_str, 'rasdt_pwr_unw') + + if os.path.isfile(Mamp): + os.remove(Mamp) + if os.path.isfile(Samp): + os.remove(Samp) + #os.remove(MampPar) + #os.remove(SampPar) + print("Uwrapping interferometric phase is done!") + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/unwrap_gamma_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/unwrap_gamma_all.py new file mode 100644 index 0000000..0cf1749 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/unwrap_gamma_all.py @@ -0,0 +1,143 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + + +def work(data0): + cmd = data0[0] + err_file = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if type(stdout) == bytes: + bb=stdout.decode("utf-8", errors="replace") + else: + bb = stdout + + if p.returncode != 0: + detail_parts = [] + if bb: + detail_parts.append(bb) + if aa: + detail_parts.append(aa) + detail = '\n'.join(part.strip() for part in detail_parts if part and part.strip()) + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + cmd[3] + '\n' + with open(err_file, 'a') as f: + f.write(str0) + if detail: + f.write(detail) + f.write('\n') + raise RuntimeError(str0.strip() + ' failed with rc=' + str(p.returncode) + ('\n' + detail if detail else '')) + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + cmd[3] + '\n' + with open(err_file, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + Unwrap differential interferograms for one project using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + unwrap_gamma_all.py projectName + unwrap_gamma_all.py projectName --parallel 4 + unwrap_gamma_all.py projectName --parallel 4 --ifgramList-txt /test/ifgram_list.txt +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Unwrap differential interferograms for one project using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + parser.add_argument('--ifgarmList-txt', dest='ifgarmListTxt', help='provided ifgram_list_txt. default: using ifgram_list.txt under projectName folder.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + projectDir = scratchDir + '/' + projectName + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + if inps.ifgarmListTxt: ifgramList_txt = inps.ifgarmListTxt + else: ifgramList_txt = scratchDir + '/' + projectName + '/ifgram_list.txt' + ifgList0 = ut.read_txt2array(ifgramList_txt) +# ifgList = ifgList0[:,0] + if len(ifgList0)==3: + ifgList=ifgList0[0] + ifgList=[ifgList] + else: + ifgList=ifgList0[:,0] + + err_txt = scratchDir + '/' + projectName + '/unwrap_gamma_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + for i in range(len(ifgList)): + m0 = ut.yyyymmdd(ifgList[i].split('-')[0]) + s0 = ut.yyyymmdd(ifgList[i].split('-')[1]) + cmd0 = ['unwrap_gamma.py',projectName, m0, s0] + unw_file0 = ifgDir + '/' + ifgList[i] + '/' + ifgList[i] + '_' + rlks + 'rlks.diff_filt.unw.bmp' + data0 = [cmd0,err_txt] + + k00 = 0 + if os.path.isfile(unw_file0): + if os.path.getsize(unw_file0) > 0: + k00 = 1 + if k00==0: + data_para.append(data0) + + results = ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + failures = [str(item) for item in results if isinstance(item, Exception)] + if failures: + raise RuntimeError('\n\n'.join(failures)) + print("Unwrap differential interferograms for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/unwrap_snaphu_gamma.py b/.codex_tmp/pyint_variants/no_rescue/pyint/unwrap_snaphu_gamma.py new file mode 100644 index 0000000..9c51057 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/unwrap_snaphu_gamma.py @@ -0,0 +1,447 @@ +#! /usr/bin/env python + +################################################################# + +### This program is part of PyINT v2.1 ### + +### Copy Right (c): 2017-2019, Yunmeng Cao ### + +### Author: Yunmeng Cao ### + +### Contact : ymcmrs@gmail.com ### + +################################################################# + + + +import numpy as np + +import os + +import sys + +import argparse + +import subprocess + +import re + + + +from pyint import _utils as ut + + + +INTRODUCTION = ''' + +------------------------------------------------------------------- + + Unwrap differential interferogram using SNAPHU. + + + +''' + + + +EXAMPLE = ''' + + Usage: + + unwrap_snaphu.py projectName Mdate Sdate + + unwrap_snaphu.py PacayaT163TsxHhA 20150102 20150601 + +------------------------------------------------------------------- + +''' + + + +def cmdLineParse(): + + parser = argparse.ArgumentParser(description='Unwrap differential interferogram using SNAPHU.',\ + + formatter_class=argparse.RawTextHelpFormatter,\ + + epilog=INTRODUCTION+'\n'+EXAMPLE) + + + + parser.add_argument('projectName', help='projectName for processing.') + + parser.add_argument('Mdate', help='Master date.') + + parser.add_argument('Sdate', help='Slave date.') + + + + inps = parser.parse_args() + + return inps + + + +def run_command(cmd): + + print(f"Running: {cmd}") + + status = subprocess.call(cmd, shell=True) + + if status != 0: + + print(f"Error running command: {cmd}") + + sys.exit(1) + + + +def extract_value(s): + + """Extract numeric value from string that may contain units""" + + # Remove any non-numeric characters except decimal point and minus sign + + return re.sub(r'[^\d\.\-]', '', s) + + + +def main(argv): + + inps = cmdLineParse() + + projectName = inps.projectName + + Mdate = inps.Mdate + + Sdate = inps.Sdate + + + + scratchDir = os.getenv('SCRATCHDIR') + + templateDir = os.getenv('TEMPLATEDIR') + + templateFile = templateDir + "/" + projectName + ".template" + + templateDict = ut.update_template(templateFile) + + rlks = templateDict['range_looks'] + + azlks = templateDict['azimuth_looks'] + + + + processDir = scratchDir + '/' + projectName + "/PROCESS" + + slcDir = scratchDir + '/' + projectName + "/SLC" + + rslcDir = scratchDir + '/' + projectName + '/RSLC' + + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + + + + Pair = Mdate + '-' + Sdate + + workDir = ifgDir + '/' + Pair + + + + # Create working directory if not exists + + os.makedirs(workDir, exist_ok=True) + + + + # Copy required files for processing + + Mamp0 = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + + MampPar0 = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + + Samp0 = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + + SampPar0 = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + + + Mamp = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp' + + MampPar = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + + Samp = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp' + + SampPar = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + off_par = workDir + '/' + Pair + '_' + rlks + 'rlks.off' + + ut.copy_file(Mamp0, Mamp) + + ut.copy_file(Samp0, Samp) + + ut.copy_file(MampPar0, MampPar) + + ut.copy_file(SampPar0, SampPar) + + + + # Define input/output files + + int_dir = workDir + + rootname = Pair + '_' + rlks + 'rlks' + + int_file = int_dir + '/' + rootname + '.diff_filt' + + cor_file = int_dir + '/' + rootname + '.diff_filt.cor' + + + + # Read parameters from par files + + width = ut.read_gamma_par(off_par, 'read', 'interferogram_azimuth_lines:') + length = ut.read_gamma_par(off_par, 'read', 'interferogram_width:') + #length = ut.read_gamma_par(MampPar, 'read', 'azimuth_lines') + row = ut.read_gamma_par(off_par, 'read', 'interferogram_azimuth_lines:') + col = ut.read_gamma_par(off_par, 'read', 'interferogram_width:') + pos = ut.read_gamma_par(MampPar, 'read', 'sar_to_earth_center') + earth = ut.read_gamma_par(MampPar, 'read', 'earth_radius_below_sensor') + near = ut.read_gamma_par(MampPar, 'read', 'near_range_slc') + dr = ut.read_gamma_par(off_par, 'read', 'interferogram_range_pixel_spacing:') + da_flight = ut.read_gamma_par(off_par, 'read', 'interferogram_azimuth_pixel_spacing:') + rangeres = ut.read_gamma_par(MampPar, 'read', 'range_pixel_spacing') + azres = ut.read_gamma_par(MampPar, 'read', 'azimuth_pixel_spacing') + nrange = ut.read_gamma_par(off_par, 'read', 'interferogram_range_looks:') + nazi = ut.read_gamma_par(off_par, 'read', 'interferogram_azimuth_looks:') + + + # Extract numeric values + earth = extract_value(earth) + pos = extract_value(pos) + near = extract_value(near) + dr = extract_value(dr) + da_flight = extract_value(da_flight) + nrange = extract_value(nrange) + nazi = extract_value(nazi) + rangeres = extract_value(rangeres) + azres = extract_value(azres) + + alt = str(float(pos) - float(earth)) + da = str(float(da_flight) * float(earth) / (float(earth) + float(alt))) + ncor = str(float(dr) * float(da) / (float(rangeres) * float(azres))) + + + + + + + + + + + + # Create SNAPHU configuration file + + conf_file = workDir + '/' + rootname + '.snaphuconf' + + with open(conf_file, 'w') as f: + + f.write(f"STATCOSTMODE SMOOTH\n") + + f.write(f"INFILE {rootname}.int\n") + + f.write(f"LINELENGTH {width}\n") # 使用原始宽度 + + f.write(f"OUTFILE {rootname}.unw\n") + + f.write(f"CORRFILE {rootname}.cor\n") + + f.write(f"LOGFILE {rootname}.snaphulog\n") + + f.write(f"\n") + + f.write(f"PIECEFIRSTROW 1\n") + + f.write(f"PIECEFIRSTCOL 1\n") + + f.write(f"PIECENROW {length}\n") # 使用原始长度 + + f.write(f"PIECENCOL {width}\n") # 使用原始宽度 + + f.write(f"ALTITUDE {alt}\n") + + f.write(f"EARTHRADIUS {earth}\n") + + f.write(f"NEARRANGE {near}\n") + + f.write(f"BASELINE 0.000000\n") + + f.write(f"BASELINEANGLE_DEG 0.000000\n") + + f.write(f"TRANSMITMODE REPEATPASS\n") + + f.write(f"DR {dr}\n") + + f.write(f"DA {da}\n") + + f.write(f"RANGERES {rangeres}\n") + + f.write(f"AZRES {azres}\n") + + f.write(f"LAMBDA 0.0556\n") + + f.write(f"NLOOKSRANGE {nrange}\n") + + f.write(f"NLOOKSAZ {nazi}\n") + + f.write(f"NLOOKSOTHER 1\n") + + f.write(f"NCORRLOOKS {ncor}\n") + + f.write(f"\n") + + f.write(f"CONNCOMPFILE {rootname}.byt\n") + + f.write(f"MAXNCOMPS 32\n") + + f.write(f"\n") + + f.write(f"INFILEFORMAT COMPLEX_DATA\n") + + f.write(f"OUTFILEFORMAT ALT_LINE_DATA\n") + + f.write(f"CORRFILEFORMAT FLOAT_DATA\n") + + f.write(f"VERBOSE FALSE\n") + + + + # Swap bytes for input files + + run_command(f"swap_bytes {int_file} {rootname}.int 4") + + run_command(f"swap_bytes {cor_file} {rootname}.cor 4") + + + + # Run SNAPHU + + run_command(f"snaphu -f {conf_file}") + + + + # Post-processing steps + + xmin = "0" + + ymin = "0" + + xmax = width + + ymax = length + + + + # 获取SNAPHU输出文件的尺寸 + + unwrapped_file = f"{rootname}.unw" + new_width = str(int(xmax)-int(xmin)) + + # 计算填充量 + + npad_bottom = str(int(length) - int(ymax)) + + npad_right = str(int(width) - int(xmax)) + + + + print(f"Original dimensions: {width}x{length}") + + print(f"SNAPHU output dimensions: {new_width}x{new_width}") + + print(f"Padding: bottom={npad_bottom}, right={npad_right}") + + + + # Zero padding + + unw_zero_file = rootname + "_zeropad.unw" + + mask_zero_file = rootname + "_mask_zeropad.unw" + + mask_zero_file_msk = rootname + "_mask_zeropad.msk" + + tobegeocodedm = rootname + "_masked.unw" + + unw_file = rootname + "_msk.unw" + + + + run_command(f"zeropad_msk {rootname}.unw {width} {ymin} {xmin} {npad_right} {npad_bottom} {unw_zero_file} rmg") + + + + # Phase and magnitude processing + + run_command(f"rmg2mag_phs {unw_zero_file} /dev/null phs {width}") + + run_command(f"swap_bytes phs phs4 4") + + run_command(f"cpx2mag_phs {rootname}.int pwr /dev/null {width}") + + run_command(f"mag_phs2rmg pwr phs {unw_zero_file} {width}") + + + + # Mask processing + + run_command(f"zeropad_msk {rootname}.byt {width} {ymin} {xmin} {npad_right} {npad_bottom} {mask_zero_file_msk} msk") + + run_command(f"cpx2mag_phs {rootname}.int pwr phs {width}") + + run_command(f"mag_phs2rmg pwr {mask_zero_file_msk} {mask_zero_file} {width}") + + + + # Combine results + + run_command(f"rmg2mag_phs {unw_zero_file} pwr phs1 {width}") + + run_command(f"rmg2mag_phs {mask_zero_file} /dev/null phs2 {width}") + + run_command(f"add_phs phs1 phs2 phs3 {width} {length} 0 1") + + run_command(f"add_phs pwr phs2 pwr3 {width} {length} 0 1") + + run_command(f"mag_phs2rmg pwr3 phs3 {tobegeocodedm} {width}") + + run_command(f"swap_bytes phs3 {unw_file} 4") + + + + # Clean up temporary files + + run_command("rm -rf pwr phs phs1 phs2 phs3 pwr3") + + run_command(f"mv phs4 {unw_zero_file}") + + + + # Remove copied files + + os.remove(Mamp) + + os.remove(Samp) + + + + print("Unwrapping with SNAPHU is done!") + + sys.exit(0) + + + +if __name__ == '__main__': + + main(sys.argv[:]) diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/unzip_s1_all.py b/.codex_tmp/pyint_variants/no_rescue/pyint/unzip_s1_all.py new file mode 100644 index 0000000..bb8c162 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/unzip_s1_all.py @@ -0,0 +1,101 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + +def work(data0): + cmd = data0[0] + err_txt = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + cmd[3] + '\n' + #print(aa) + with open(err_txt, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + Unzip Sentinel-1 raw dataset for one project. + [unzip does not need much CPU, so we can use parallel processing] + +''' + +EXAMPLE = ''' + Usage: + unzip_s1_all.py projectName + unzip_s1_all.py projectName --parallel 4 +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Unzip Sentinel-1 raw dataset for one project.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + raw_file_list = glob.glob(downDir + '/S1*.zip') + + slc_dir = scratchDir + '/' + projectName + '/SLC' + if not os.path.isdir(slc_dir): + os.mkdir(slc_dir) + + err_txt = scratchDir + '/' + projectName + '/unzip_s1_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + for i in range(len(raw_file_list)): + cmd0 = ['unzip',raw_file_list[i],'-d',downDir] + data0 = [cmd0,err_txt] + data_para.append(data0) + + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + os.chdir(downDir) + print("Unzip Sentinel-1 raw dataset for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + \ No newline at end of file diff --git a/.codex_tmp/pyint_variants/no_rescue/pyint/utm2ll b/.codex_tmp/pyint_variants/no_rescue/pyint/utm2ll new file mode 100644 index 0000000..722d337 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/pyint/utm2ll @@ -0,0 +1,67 @@ +#!/bin/csh -f +# +#convet utm coord. to lon and lat +# +if ( $#argv < 5) then +more <<EOF + +utm2ll is a C-shell script which convets utm coord. to lon and lat + +Usage: utm2ll zone x0 y0 dx dy +Where zone is the conventional zone number=int(lon/6)+31 + x0,y0 is the origin of UTM projection at easting and northing(meter) + dx,dy is the offset from origin at meter unit + +Example: utm2ll 45 311072.36 3542183.56 27000 62000 + origin 85 32(lon,lat) + + Qiao Xuejun 2008/2/20 +EOF +exit +endif + +set zone = $argv[1] +set x0 = $argv[2] +set y0 = $argv[3] +set dx = $argv[4] +set dy = $argv[5] +set pi = 3.141592653589793 +set a = 6378137.00 +set b = 6356752.3142 +#echo $a $b +set k0 = 0.9996 +set x = `echo $x0 $dx |awk '{print 500000-($1+$2)}'` +set y = `echo $y0 $dy |awk '{print $1+$2}'` +set e = `echo $a $b |awk '{print sqrt(1-($2*$2)/($1*$1))}'` +set ep2 = `echo $e $a $b |awk '{print ($1*$2/$3)*($1*$2/$3)}'` +set sin1 = `echo $pi |awk '{print $1/(180*60*60)}'` +set m = `echo $y $k0|awk '{print $1/$2}'` +set mu = `echo $m $a $e|awk '{print $1/($2*(1-$3*$3/4-3*$3*$3*$3*$3/64-5*$3*$3*$3*$3*$3*$3/256))}'` +set e1 = `echo $e |awk '{print (1-sqrt(1-$1*$1))/ (1+sqrt(1-$1*$1))}'` +set j1 = `echo $e1 |awk '{print 3*$1/2-27*$1*$1*$1/32}'` +set j2 = `echo $e1 |awk '{print 21*$1*$1/16-55*$1*$1*$1*$1/32}'` +set j3 = `echo $e1 |awk '{print 151*$1*$1*$1/96}'` +set j4 = `echo $e1 |awk '{print 1097*$1*$1*$1*$1/512}'` +set fp = `echo $mu $j1 $j2 $j3 $j4|awk '{print $1+$2*sin(2*$1)+$3*sin(4*$1)+$4*sin(6*$1)+$5*sin(8*$1)}'` +set c1 = `echo $ep2 $fp|awk '{print $1*cos($2)*cos($2)}'` +set t1 = `echo $fp |awk '{print sin($1)*sin($1)/(cos($1)*cos($1))}'` +set r1 = `echo $a $e $fp |awk '{print $1*(1-$2*$2)/((1-$2*$2*sin($3)*sin($3))*sqrt(1-$2*$2*sin($3)*sin($3)))}'` +set n1 = `echo $a $e $fp |awk '{print $1/sqrt(1-$2*$2*sin($3)*sin($3))}'` +set d = `echo $x $n1 $k0 |awk '{print $1/($2*$3)}'` +set q1 = `echo $n1 $fp $r1|awk '{print $1*(sin($2)/cos($2))/$3}'` +set q2 = `echo $d |awk '{print $1*$1/2}'` +set q3 = `echo $t1 $c1 $ep2 $d |awk '{print (5+3*$1+10*$2-4*$2*$2-9*$3)*$4*$4*$4*$4/24 }'` +set q4 = `echo $t1 $c1 $ep2 $d |awk '{print (61+90*$1+298*$2+45*$1*$1-3*$2*$2-252*$3)*$4*$4*$4*$4*$4*$4/720 }'` +set q5 = $d +set q6 = `echo $t1 $c1 $d |awk '{print (1+2*$1+$2)*$3*$3*$3/6}'` +set q7 = `echo $c1 $t1 $ep2 $d |awk '{print (5-2*$1+28*$2-3*$1*$1+8*$3+24*$2*$2)*$4*$4*$4*$4*$4/120}'` +#echo $x $e $ep2 $sin1 $m $mu $e1 $j1 $j2 $j3 $j4 $fp $c1 $t1 $r1 $n1 $d $q1 $q2 $q3 $q4 $q5 $q6 $q7 +set dlong = `echo $q5 $q6 $q7 $fp|awk '{print ($1-$2+$3)/cos($4)}'` +set lat = `echo $fp $q1 $q2 $q3 $q4 |awk '{print $1-$2*($3-$4+$5)}'` +set lon0 = `echo $zone |awk '{print 6*$1-183}'` +set lon = `echo $lon0 $dlong $pi |awk '{print $1-$2*180/$3 }'` +set lat = `echo $lat $pi |awk '{print $1*180/$2 }'` +echo $lon $lat + + + diff --git a/.codex_tmp/pyint_variants/no_rescue/template/ensishenlongxiT134F058S1A.template b/.codex_tmp/pyint_variants/no_rescue/template/ensishenlongxiT134F058S1A.template new file mode 100644 index 0000000..24930b5 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/template/ensishenlongxiT134F058S1A.template @@ -0,0 +1,117 @@ +# *********************************** PyINT Template *******************************************# +# Modifed Sep. , 2019 - YM.Cao + +# ------------------- Basic parameters ----------------------- + +DEM = /data/insar/chenwei/SCRATCHDIR/ensishenlongxiT134F058S1A/DEM/srtm.dem +# if not provide, SRTM-1 will be downloaded and processed automatically. + +masterDate = 20180123 # [If not provided, the first date in the date_list will be selected as the master date] + +start_swath = 2 # [option (for Sentinel-1 only)] +end_swath = 3 # [option (for Sentinel-1 only)] +start_burst = 3 # [option: bursts position is based on the master image (for Sentinel-1 only)] +end_burst = 6 # [option: bursts position is based on the master image (for Sentinel-1 only)] + +range_looks = 20 # multi-look numbers in range direction +azimuth_looks = 4 # multi-look numbers in azimuth direction + +# ------------------- download data ----------------------- + +sensor = Sentinel-1A # Envisat, ERS-1/2, .. please check SSARA for details +track = 128 # track number of the SAR image +frame = 89 # frame number of the SAR image [ defined according to ASF ] +start_time = 2017-11-01 # +end_time = 2109-01-01 # + +# ------------------- basic parameters for interferometry ----------------------- + +dem_lat_ovr = 1 # as to SRTM-1, 0.5 for 60 m, 2 for 15 m +dem_lon_ovr = 1 + +Igram_Spsflg = 1 # Implement range spectral filtering +rbw_min = 1 # minimum range bandwidth fraction (0.1 --> 1.0)(default: - 0.250) + +rwin4cor = 256 # range window length for coregistration +azwin4cor = 256 # azimuth window length for coregistration +rsample4cor = 32 # range samples used for fitting the coregistration parameters +azsample4cor = 32 # azimuth samples used for fitting the coregistration parameters + +thresh4cor = 0.3 # 2016 GAMMA or higher version, for 2015 GAMMA or lower version should be SNR + +Igram_Cor_rwin = 5 # range window length for cc_wave coherence estimation +Igram_Cor_awin = 5 # azimuth window length for cc_wave coherence estimation + +Igram_Cor_Win = 5 # cc estimation in adf +adf_alpha = 0.4 # adf alpha for Gold-Stein filtering + +# ------------------- sim phase ----------------------- + +Igram_Flag_TDM = N # Y for Tandem-X +Simphase_rpos = - +Simphase_azpos = - +Simphase_rwin = 256 +Simphase_azwin = 256 +Simphase_thresh = - + +# ------------------- unwrap phase ----------------------- + +mcf_triangular = 1 # triangular type of mcf [0: regular; 1: delaunay;] +unwrap_patr = 1 # unwrap patches in range direction +unwrap_pataz = 1 # unwrap patches in azimuth direction +unwrapThreshold = 0.25 # coherence threshold used for unwrap +auto_unw = 1 # auto or manual to select the reference point. True:auto False: manual +make_mask = 1 # make a mask based on the cor file to mask the lower field 0: don't mask 1: make a mask +init_flag = 1 # flag to set phase at reference point. 0: use initial point phase value (default) 1: set phase to 0.0 at initial point +r_refer = 396 #phase reference point range offset (default(-): roff) +a_refer = 537 # phase reference point azimuth offset (default(-): loff) +# ------------------- geocode ----------------------- + +geo_interp = 0 # [0: nearest; 1: bicubic spline] + +#-------------------- trans phase to los displacement -------------- + +satelite = S1A # the value of satelite must be one in the list['S1A', 'CSK','TSX','ALSO2','ALOS','ENVISAT'] + +# ------------------- select interferometric pairs -------------- + +network_method = sbas # sbas, sequential, delaunay, stars +endDate = 20210809 # date before this date will be excluded +startDate = 20170320 # date after this date will be excluded +conNumb = 2 # connect number for sequential +max_tb = 100 # temporal baseline threshold in days +max_sb = 100 # spatial baseline threshold in meters +#exclude_date = # exclude the date which will not be used + +# ------------------- time-series process -------------- + +download_data = 0 # [0: skip; 1: process] +down_parallel = 0 # multi-processor number used for downloading + +raw2slc_all = 0 # i.e., download2slc [0: skip; 1: process] +raw2slc_all_parallel = 4 # multi-processor number used + +extract_burst_all = 0 # for TOPS SLC only [0: skip; 1: process] +extract_all_parallel = 1 # multi-processor number used + +coreg_all = 0 # do coregister [0: skip; 1: process] +coreg_all_parallel = 4 # multi-processor number used [4 or 8] + +select_pairs = 0 # if 0: skip, and use the ifgram_list.txt under PROJECTNAME folder + +diff_all = 0 # generate differential Ifg [0: skip; 1: process] +diff_all_parallel = 4 # multi-processor number used [4 or 8] + +unwrap_all = 1 # unwrap Ifg [0: skip; 1: process] +unwrap_all_parallel = 8 # multi-processor number used [8 or 10] + +geocode_all = 1 # geocode diff,unw products [0: skip; 1: process] +geocode_all_parallel = 4 # multi-processor number used + +load_data = 0 # loading data for mintpy processing [0: skip; 1: process] + +# ------------------- Good Luck -------------- +DEM= /data/insar/chenwei/SCRATCHDIR/ensishenlongxiT134F058S1A/DEM/ensishenlongxiT134F058S1A.dem +DEM= /data/insar/chenwei/SCRATCHDIR/DEM/ensishenlongxiT134F058S1A/ensishenlongxiT134F058S1A.dem +DEM= /data/insar/chenwei/SCRATCHDIR/DEM/ensishenlongxiT134F058S1A/ensishenlongxiT134F058S1A.dem +DEM= /data/insar/chenwei/SCRATCHDIR/DEM/ensishenlongxiT134F058S1A/ensishenlongxiT134F058S1A.dem diff --git a/.codex_tmp/pyint_variants/no_rescue/template/pyint.template b/.codex_tmp/pyint_variants/no_rescue/template/pyint.template new file mode 100644 index 0000000..6459886 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/template/pyint.template @@ -0,0 +1,204 @@ +# *********************************** PyINT Template *******************************************# +# Template for PyINT v2.1 +# Please fill in the parameters according to your project requirements + +# ========================= Basic parameters ======================== + +# DEM file path (if not provided, SRTM will be downloaded automatically from OpenTopography) +#DEM = /path/to/your/dem.dem + +# Master date for interferometric processing +masterDate = YYYYMMDD # Master date in YYYYMMDD format + +# Swath and burst parameters (for Sentinel-1 TOPS mode only) +start_swath = 1 # Starting swath number +end_swath = 3 # Ending swath number +start_burst = 1 # Starting burst position (relative to master image) +end_burst = 9 # Ending burst position (relative to master image) + +# Multilooking parameters +range_looks = 20 # Multi-look numbers in range direction +azimuth_looks = 4 # Multi-look numbers in azimuth direction + +# ========================= Download Data ======================== + +# Data source and search parameters +Data_Hub = ASF # Data download source: SSARA, Scihub, ASF +sensor = Sentinel-1A # Envisat, ERS-1/2, Sentinel-1A/B, etc. +track = 128 # Track number of the SAR image +frame = 89 # Frame number [defined according to ASF] +start_time = YYYY-MM-DD # Start date for data search +end_time = YYYY-MM-DD # End date for data search + +# ASF download specific parameters +asf_platform = Sentinel-1A # Platform: Sentinel-1A, Sentinel-1B +asf_beam_mode = IW # Beam mode: IW (Interferometric Wide) +asf_polarization = VV # Polarization: VV, VH, HH, HV +asf_flight_direction = ascending # Flight direction: ascending, descending +asf_bbox = W,S,E,N # Bounding box: West,South,East,North + +# ========================= DEM Generation ======================== + +# DEM download parameters (for OpenTopography) +opentopo_api_key = YOUR_API_KEY # OpenTopography API key +opentopo_dem_type = SRTMGL1 # DEM type: SRTMGL1(30m), SRTMGL3(90m), NASADEM(30m), COP30, COP90 +opentopo_auto_tile = 1 # Auto-tile large regions (recommended: 1) + +# DEM processing parameters +dem_lat_ovr = 1 # Latitude oversampling (as to SRTM-1: 0.5 for 60m, 2 for 15m) +dem_lon_ovr = 1 # Longitude oversampling + +# ========================= Interferometry Parameters ======================== + +# Spectral filtering +Igram_Spsflg = 1 # Implement range spectral filtering +rbw_min = 1 # Minimum range bandwidth fraction (0.1 --> 1.0) + +# Coregistration parameters +rwin4cor = 256 # Range window length for coregistration +azwin4cor = 256 # Azimuth window length for coregistration +rsample4cor = 32 # Range samples for fitting coregistration parameters +azsample4cor = 32 # Azimuth samples for fitting coregistration parameters +thresh4cor = 0.3 # Coherence threshold (GAMMA 2016+, SNR for older versions) + +# Coherence estimation +Igram_Cor_rwin = 5 # Range window length for cc_wave coherence estimation +Igram_Cor_awin = 5 # Azimuth window length for cc_wave coherence estimation +Igram_Cor_Win = 5 # Window for cc estimation in adf +adf_alpha = 0.4 # Alpha for Gold-Stein filtering + +# ========================= Simulation Phase ======================== + +Igram_Flag_TDM = N # Y for Tandem-X +Simphase_rpos = - +Simphase_azpos = - +Simphase_rwin = 256 +Simphase_azwin = 256 +Simphase_thresh = - + +# ========================= Unwrapping ======================== + +mcf_triangular = 1 # Triangular type of mcf [0: regular; 1: delaunay] +unwrap_patr = 1 # Unwrap patches in range direction +unwrap_pataz = 1 # Unwrap patches in azimuth direction +unwrapThreshold = 0.05 # Coherence threshold for unwrapping +auto_unw = 1 # Auto or manual reference point [1: auto; 0: manual] +make_mask = 0 # Make mask based on coherence [0: don't mask; 1: make mask] +init_flag = 1 # Flag to set phase at reference point [0: use initial; 1: set to 0.0] +r_refer = - # Phase reference point range offset +a_refer = - # Phase reference point azimuth offset + +# ========================= Geocoding ======================== + +geo_interp = 0 # Interpolation method [0: nearest; 1: bicubic spline] + +# ========================= Satellite Configuration ======================== + +satelite = S1A # Satellite: S1A, S1B, CSK, TSX, ALOS2, ALOS, ENVISAT + +# ========================= Pair Selection ======================== + +network_method = sbas # Network method: sbas, sequential, delaunay, stars +endDate = YYYYMMDD # Exclude dates after this date +startDate = YYYYMMDD # Exclude dates before this date +conNumb = 15 # Connect number for sequential network +max_tb = 365 # Maximum temporal baseline (days) +max_sb = 100 # Maximum spatial baseline (meters) +min_tb = 1 # Minimum temporal baseline (days) +min_sb = 0 # Minimum spatial baseline (meters) + +# ========================= Phase Gradient (Optional) ======================== + +dataformat = cpxfloat32 # Data format of interferogram +scale = 10000 # Original phase/scale +step_windows = 1 # Step length for spatial phase gradient +filter_wins = [7, 7] # Windows of median filter +directions = ['grad_east', 'grad_north', 'grad_northeast', 'grad_southeast'] + +# Mapping parameters for phase gradient +in_low_high = [0.001, 1] # Input range for mapping +out_low_high = [0.001, 1] # Output range for mapping +gamma = 0.8 # Gamma for curve shape +choose_linear = 'n' # Linear or nonlinear mapping ['y': linear; 'n': nonlinear] + +# ========================= Hyp3 Format Conversion (NEW) ======================== + +# Convert GAMMA outputs to Hyp3-compatible GeoTIFF format +hyp3format = 0 # Enable Hyp3 format conversion [0: skip; 1: process] +hyp3format_parallel = 4 # Number of parallel processors for Hyp3 conversion +hyp3_output_dir = - # Output directory for Hyp3 products [default: projectName/Hyp3Products/] + +# ========================= Phase Bias Correction (NEW) ======================== + +# Apply phase bias correction based on loop closures algorithm +# Reference: 10.1016/j.remote.2022.100013 + +phasebias = 0 # Enable phase bias correction [0: skip; 1: process] +phasebias_interval = 12 # Temporal baseline in days [6, 12, or 24] +phasebias_nlook = 10 # Number of looks for multilooking +phasebias_num_a = 2 # Number of calibration parameters to estimate +phasebias_estimate_an = 0 # Estimate an from data [0: use defaults; 1: estimate from loop closures] +phasebias_max_con = 5 # Maximum number of connections + +# Default an values (used if phasebias_estimate_an = 0) +a1_6_day = 0.50 +a2_6_day = 0.36 +a3_6_day = 0.299 +a4_6_day = 0.2476 + +a1_12_day = 0.494 +a2_12_day = 0.297 +a3_12_day = 0.24 +a4_12_day = 0.22 + +a1_24_day = 0.48 +a2_24_day = 0.28 +a3_24_day = 0.22 +a4_24_day = 0.20 + +phasebias_start = - # Start date YYYYMMDD [default: auto-detect] +phasebias_end = - # End date YYYYMMDD [default: auto-detect] + +# ========================= Atmospheric Correction ======================== + +# GACOS tropospheric correction +gacos_correction = 0 # Enable GACOS correction [0: skip; 1: process] +gacos_parallel = 4 # Number of parallel processors + +# Ionospheric correction +ionosphere_correction = 0 # Enable ionospheric correction [0: skip; 1: process] +ionosphere_parallel = 4 # Number of parallel processors + +# ========================= Processing Workflow Control ======================== + +download_data = 0 # Enable data download [0: skip; 1: process] +down_parallel = 4 # Number of parallel processors for download + +raw2slc_all = 1 # Convert raw to SLC [0: skip; 1: process] +raw2slc_all_parallel = 4 # Number of parallel processors + +extract_burst_all = 1 # Extract bursts [0: skip; 1: process] +extract_all_parallel = 4 # Number of parallel processors + +coreg_all = 1 # Coregister SLCs [0: skip; 1: process] +coreg_all_parallel = 4 # Number of parallel processors + +select_pairs = 1 # Select interferometric pairs [0: skip; 1: process] + +diff_all = 1 # Generate differential interferograms [0: skip; 1: process] +diff_all_parallel = 4 # Number of parallel processors + +unwrap_all = 1 # Unwrap interferograms [0: skip; 1: process] +unwrap_all_parallel = 4 # Number of parallel processors + +geocode_all = 1 # Geocode products [0: skip; 1: process] +geocode_all_parallel = 4 # Number of parallel processors + +hyp3format_all = 0 # Convert to Hyp3 format [0: skip; 1: process] +hyp3format_all_parallel = 4 # Number of parallel processors + +phasebias_all = 0 # Apply phase bias correction [0: skip; 1: process] + +load_data = 0 # Load data for MintPy processing [0: skip; 1: process] + +# ========================= Good Luck ======================================= diff --git a/.codex_tmp/pyint_variants/no_rescue/template/shanghaiT171F128S1A.template b/.codex_tmp/pyint_variants/no_rescue/template/shanghaiT171F128S1A.template new file mode 100644 index 0000000..d350dd9 --- /dev/null +++ b/.codex_tmp/pyint_variants/no_rescue/template/shanghaiT171F128S1A.template @@ -0,0 +1,204 @@ +# *********************************** PyINT Template *******************************************# +# Template for PyINT v2.1 +# Please fill in the parameters according to your project requirements + +# ========================= Basic parameters ======================== + +# DEM file path (if not provided, SRTM will be downloaded automatically from OpenTopography) +#DEM = /path/to/your/dem.dem + +# Master date for interferometric processing +masterDate = 20250711 # Master date in YYYYMMDD format + +# Swath and burst parameters (for Sentinel-1 TOPS mode only) +start_swath = 1 # Starting swath number +end_swath = 3 # Ending swath number +start_burst = 1 # Starting burst position (relative to master image) +end_burst = 9 # Ending burst position (relative to master image) + +# Multilooking parameters +range_looks = 10 # Multi-look numbers in range direction +azimuth_looks = 2 # Multi-look numbers in azimuth direction + +# ========================= Download Data ======================== + +# Data source and search parameters +Data_Hub = ASF # Data download source: SSARA, Scihub, ASF +sensor = Sentinel-1A # Envisat, ERS-1/2, Sentinel-1A/B, etc. +track = 171 # Track number of the SAR image +frame = 96 # Frame number [defined according to ASF] +start_time = 2024-11-01 # Start date for data search +end_time = 2026-02-04 # End date for data search + +# ASF download specific parameters +asf_platform = Sentinel-1A # Platform: Sentinel-1A, Sentinel-1B +asf_beam_mode = IW # Beam mode: IW (Interferometric Wide) +asf_polarization = VV # Polarization: VV, VH, HH, HV +asf_flight_direction = ascending # Flight direction: ascending, descending +asf_bbox = 120.1,30.7,120.2,30.9 # Bounding box: West,South,East,North + +# ========================= DEM Generation ======================== + +# DEM download parameters (for OpenTopography) +opentopo_api_key = 09ad77d34545607fdf5cb182b64ac64e # OpenTopography API key +opentopo_dem_type = SRTMGL1 # DEM type: SRTMGL1(30m), SRTMGL3(90m), NASADEM(30m), COP30, COP90 +opentopo_auto_tile = 1 # Auto-tile large regions (recommended: 1) + +# DEM processing parameters +dem_lat_ovr = 1 # Latitude oversampling (as to SRTM-1: 0.5 for 60m, 2 for 15m) +dem_lon_ovr = 1 # Longitude oversampling + +# ========================= Interferometry Parameters ======================== + +# Spectral filtering +Igram_Spsflg = 1 # Implement range spectral filtering +rbw_min = 1 # Minimum range bandwidth fraction (0.1 --> 1.0) + +# Coregistration parameters +rwin4cor = 256 # Range window length for coregistration +azwin4cor = 256 # Azimuth window length for coregistration +rsample4cor = 32 # Range samples for fitting coregistration parameters +azsample4cor = 32 # Azimuth samples for fitting coregistration parameters +thresh4cor = 0.3 # Coherence threshold (GAMMA 2016+, SNR for older versions) + +# Coherence estimation +Igram_Cor_rwin = 5 # Range window length for cc_wave coherence estimation +Igram_Cor_awin = 5 # Azimuth window length for cc_wave coherence estimation +Igram_Cor_Win = 5 # Window for cc estimation in adf +adf_alpha = 0.4 # Alpha for Gold-Stein filtering + +# ========================= Simulation Phase ======================== + +Igram_Flag_TDM = N # Y for Tandem-X +Simphase_rpos = - +Simphase_azpos = - +Simphase_rwin = 256 +Simphase_azwin = 256 +Simphase_thresh = - + +# ========================= Unwrapping ======================== + +mcf_triangular = 1 # Triangular type of mcf [0: regular; 1: delaunay] +unwrap_patr = 1 # Unwrap patches in range direction +unwrap_pataz = 1 # Unwrap patches in azimuth direction +unwrapThreshold = 0.05 # Coherence threshold for unwrapping +auto_unw = 1 # Auto or manual reference point [1: auto; 0: manual] +make_mask = 0 # Make mask based on coherence [0: don't mask; 1: make mask] +init_flag = 1 # Flag to set phase at reference point [0: use initial; 1: set to 0.0] +r_refer = - # Phase reference point range offset +a_refer = - # Phase reference point azimuth offset + +# ========================= Geocoding ======================== + +geo_interp = 0 # Interpolation method [0: nearest; 1: bicubic spline] + +# ========================= Satellite Configuration ======================== + +satelite = S1A # Satellite: S1A, S1B, CSK, TSX, ALOS2, ALOS, ENVISAT + +# ========================= Pair Selection ======================== + +network_method = sbas # Network method: sbas, sequential, delaunay, stars +endDate = YYYYMMDD # Exclude dates after this date +startDate = YYYYMMDD # Exclude dates before this date +conNumb = 15 # Connect number for sequential network +max_tb = 365 # Maximum temporal baseline (days) +max_sb = 100 # Maximum spatial baseline (meters) +min_tb = 1 # Minimum temporal baseline (days) +min_sb = 0 # Minimum spatial baseline (meters) + +# ========================= Phase Gradient (Optional) ======================== + +dataformat = cpxfloat32 # Data format of interferogram +scale = 10000 # Original phase/scale +step_windows = 1 # Step length for spatial phase gradient +filter_wins = [7, 7] # Windows of median filter +directions = ['grad_east', 'grad_north', 'grad_northeast', 'grad_southeast'] + +# Mapping parameters for phase gradient +in_low_high = [0.001, 1] # Input range for mapping +out_low_high = [0.001, 1] # Output range for mapping +gamma = 0.8 # Gamma for curve shape +choose_linear = 'n' # Linear or nonlinear mapping ['y': linear; 'n': nonlinear] + +# ========================= Hyp3 Format Conversion (NEW) ======================== + +# Convert GAMMA outputs to Hyp3-compatible GeoTIFF format +hyp3format = 0 # Enable Hyp3 format conversion [0: skip; 1: process] +hyp3format_parallel = 4 # Number of parallel processors for Hyp3 conversion +hyp3_output_dir = - # Output directory for Hyp3 products [default: projectName/Hyp3Products/] + +# ========================= Phase Bias Correction (NEW) ======================== + +# Apply phase bias correction based on loop closures algorithm +# Reference: 10.1016/j.remote.2022.100013 + +phasebias = 0 # Enable phase bias correction [0: skip; 1: process] +phasebias_interval = 12 # Temporal baseline in days [6, 12, or 24] +phasebias_nlook = 10 # Number of looks for multilooking +phasebias_num_a = 2 # Number of calibration parameters to estimate +phasebias_estimate_an = 0 # Estimate an from data [0: use defaults; 1: estimate from loop closures] +phasebias_max_con = 5 # Maximum number of connections + +# Default an values (used if phasebias_estimate_an = 0) +a1_6_day = 0.50 +a2_6_day = 0.36 +a3_6_day = 0.299 +a4_6_day = 0.2476 + +a1_12_day = 0.494 +a2_12_day = 0.297 +a3_12_day = 0.24 +a4_12_day = 0.22 + +a1_24_day = 0.48 +a2_24_day = 0.28 +a3_24_day = 0.22 +a4_24_day = 0.20 + +phasebias_start = - # Start date YYYYMMDD [default: auto-detect] +phasebias_end = - # End date YYYYMMDD [default: auto-detect] + +# ========================= Atmospheric Correction ======================== + +# GACOS tropospheric correction +gacos_correction = 0 # Enable GACOS correction [0: skip; 1: process] +gacos_parallel = 4 # Number of parallel processors + +# Ionospheric correction +ionosphere_correction = 0 # Enable ionospheric correction [0: skip; 1: process] +ionosphere_parallel = 4 # Number of parallel processors + +# ========================= Processing Workflow Control ======================== + +download_data = 1 # Enable data download [0: skip; 1: process] +down_parallel = 4 # Number of parallel processors for download + +raw2slc_all = 0 # Convert raw to SLC [0: skip; 1: process] +raw2slc_all_parallel = 4 # Number of parallel processors + +extract_burst_all = 0 # Extract bursts [0: skip; 1: process] +extract_all_parallel = 4 # Number of parallel processors + +coreg_all = 0 # Coregister SLCs [0: skip; 1: process] +coreg_all_parallel = 4 # Number of parallel processors + +select_pairs = 0 # Select interferometric pairs [0: skip; 1: process] + +diff_all = 0 # Generate differential interferograms [0: skip; 1: process] +diff_all_parallel = 4 # Number of parallel processors + +unwrap_all = 0 # Unwrap interferograms [0: skip; 1: process] +unwrap_all_parallel = 4 # Number of parallel processors + +geocode_all = 0 # Geocode products [0: skip; 1: process] +geocode_all_parallel = 4 # Number of parallel processors + +hyp3format_all = 0 # Convert to Hyp3 format [0: skip; 1: process] +hyp3format_all_parallel = 4 # Number of parallel processors + +phasebias_all = 0 # Apply phase bias correction [0: skip; 1: process] + +load_data = 0 # Load data for MintPy processing [0: skip; 1: process] + +# ========================= Good Luck ======================================= diff --git a/.codex_tmp/run_lt1_coreg_orbit_experiment.sh b/.codex_tmp/run_lt1_coreg_orbit_experiment.sh new file mode 100644 index 0000000..67809aa --- /dev/null +++ b/.codex_tmp/run_lt1_coreg_orbit_experiment.sh @@ -0,0 +1,291 @@ +#!/usr/bin/env bash +set -u +set -o pipefail + +BASE_ROOT='/mnt/d/PyINT_POOL_TEST/LT1A_MONO_SYC_STRIP1_E129.6_N45.0_3scene' +CASES_ROOT='/mnt/d/PyINT_POOL_TEST/LT1A_MONO_SYC_STRIP1_E129.6_N45.0_3scene_cases' +PROJECT='pyint_stage' +MASTER_DATE='20230726' +SLAVE_DATES=(20230624 20230920) +SATELLITE='LT1A' +ORBIT_DIR='/mnt/d/orbit_pools/envi/LT1A' + +REPO='/mnt/d/Code/Insar_management_system_v2' +PYINT_HOME="$REPO/third_party/PyINT" +PYINT_SCRIPT_DIR="$PYINT_HOME/pyint" +PYTHON_BIN='/home/administrator/miniconda3/envs/isce2/bin/python' +GAMMA_ENV="$REPO/backend/app/pyint_pipeline/pyint_gamma_env.sh" +ORBIT_HELPER="$REPO/backend/app/pyint_pipeline/apply_lt1_precise_orbit.py" +GAMMA_SCRIPT_DIR='/usr/local/GAMMA_SOFTWARE-20240627/ISP/scripts' + +RUN_STAMP="${1:-$(date -u +%Y%m%dT%H%M%SZ)}" +RUN_ROOT="$CASES_ROOT/run_$RUN_STAMP" +STATUS_FILE="$RUN_ROOT/stage_status.tsv" + +fail() { + echo "$1" >&2 + exit 1 +} + +ensure_file() { + local path="$1" + [ -f "$path" ] || fail "Required file not found: $path" +} + +ensure_dir() { + local path="$1" + [ -d "$path" ] || fail "Required directory not found: $path" +} + +write_status() { + local case_name="$1" + local stage="$2" + local rc="$3" + local stdout_log="$4" + local stderr_log="$5" + printf '%s\t%s\t%s\t%s\t%s\n' "$case_name" "$stage" "$rc" "$stdout_log" "$stderr_log" >>"$STATUS_FILE" +} + +run_stage() { + local case_name="$1" + local stage="$2" + shift 2 + local stdout_log="$RUN_ROOT/$case_name/logs/${stage}.stdout.log" + local stderr_log="$RUN_ROOT/$case_name/logs/${stage}.stderr.log" + local rc=0 + + echo "[stage] $case_name :: $stage" + "$@" >"$stdout_log" 2>"$stderr_log" || rc=$? + write_status "$case_name" "$stage" "$rc" "$stdout_log" "$stderr_log" + + if [ "$rc" -eq 0 ]; then + echo "[ok] $case_name :: $stage" + else + echo "[fail] $case_name :: $stage (rc=$rc)" >&2 + fi + return "$rc" +} + +link_or_copy_static() { + local src="$1" + local dst="$2" + if [ -e "$src" ]; then + ln -s "$src" "$dst" + fi +} + +build_scene_dir() { + local case_root="$1" + local scene_date="$2" + local base_dir="$BASE_ROOT/$PROJECT/SLC/$scene_date" + local case_dir="$case_root/$PROJECT/SLC/$scene_date" + + ensure_dir "$base_dir" + mkdir -p "$case_dir" + + ensure_file "$base_dir/$scene_date.slc" + ensure_file "$base_dir/$scene_date.slc.par" + + ln -s "$base_dir/$scene_date.slc" "$case_dir/$scene_date.slc" + cp "$base_dir/$scene_date.slc.par" "$case_dir/$scene_date.slc.par" + + link_or_copy_static "$base_dir/${scene_date}_2rlks.amp" "$case_dir/${scene_date}_2rlks.amp" + link_or_copy_static "$base_dir/${scene_date}_2rlks.amp.par" "$case_dir/${scene_date}_2rlks.amp.par" + link_or_copy_static "$base_dir/${scene_date}_SLC_Tab" "$case_dir/${scene_date}_SLC_Tab" + link_or_copy_static "$base_dir/down2slc.dat" "$case_dir/down2slc.dat" + link_or_copy_static "$base_dir/t_${scene_date}" "$case_dir/t_${scene_date}" +} + +build_case() { + local case_name="$1" + local case_root="$RUN_ROOT/$case_name" + + mkdir -p "$case_root/templates" "$case_root/logs" "$case_root/manifests" + mkdir -p "$case_root/$PROJECT/SLC" "$case_root/$PROJECT/DEM" "$case_root/$PROJECT/RSLC" "$case_root/$PROJECT/ifgrams" + mkdir -p "$case_root/dem_store" + + cp "$BASE_ROOT/templates/$PROJECT.template" "$case_root/templates/$PROJECT.template" + cp -a "$BASE_ROOT/dem_store/$PROJECT" "$case_root/dem_store/" + + build_scene_dir "$case_root" "$MASTER_DATE" + for slave_date in "${SLAVE_DATES[@]}"; do + build_scene_dir "$case_root" "$slave_date" + done + + printf '%s\n' "$case_root" +} + +write_manifest() { + local role="$1" + local date_text="$2" + local output_path="$3" + local orbit_path="$ORBIT_DIR/${SATELLITE}_GpsData_GAS_C_${date_text}.txt" + + ensure_file "$orbit_path" + cat >"$output_path" <<EOF +{ + "orbits": { + "$role": { + "satellite": "$SATELLITE", + "date": "$date_text", + "expected_name": "${SATELLITE}_GpsData_GAS_C_${date_text}.txt", + "path": "$orbit_path" + } + } +} +EOF +} + +apply_bridge_once() { + local case_name="$1" + local case_root="$2" + local stage_name="$3" + local date_text="$4" + local role="$5" + local validate_flag="$6" + local strict_flag="$7" + shift 7 + local manifest_path="$case_root/manifests/${role}_${date_text}.json" + local summary_path="$case_root/logs/${stage_name}.summary.json" + local -a cmd + + write_manifest "$role" "$date_text" "$manifest_path" + + cmd=( + "$PYTHON_BIN" "$ORBIT_HELPER" + "--date" "$date_text" + "--role" "$role" + "--manifest-json" "$manifest_path" + "--summary-json" "$summary_path" + "--backup" + ) + + if [ "$validate_flag" = "true" ]; then + cmd+=("--validate-with-orb-filt") + else + cmd+=("--no-validate-with-orb-filt") + fi + + if [ "$strict_flag" = "true" ]; then + cmd+=("--strict") + else + cmd+=("--no-strict") + fi + + while [ "$#" -gt 0 ]; do + cmd+=("--slc-par" "$1") + shift + done + + run_stage "$case_name" "$stage_name" "${cmd[@]}" + return $? +} + +run_generate_and_coreg() { + local case_name="$1" + local case_root="$2" + local rc=0 + + export SCRATCHDIR="$case_root" + export TEMPLATEDIR="$case_root/templates" + export DEMDIR="$case_root/dem_store" + + run_stage "$case_name" "generate_rdc_dem" "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/generate_rdc_dem.py" "$PROJECT" || return $? + + for slave_date in "${SLAVE_DATES[@]}"; do + run_stage "$case_name" "coreg_${slave_date}" "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/coreg_gamma.py" "$PROJECT" "$slave_date" || rc=1 + done + + return "$rc" +} + +run_case_a() { + local case_name='case_A_baseline' + local case_root + case_root="$(build_case "$case_name")" + run_generate_and_coreg "$case_name" "$case_root" || true +} + +run_case_c() { + local case_name='case_C_precise_orbit_rewrite' + local case_root + case_root="$(build_case "$case_name")" + + apply_bridge_once \ + "$case_name" "$case_root" "orbit_bridge_master" "$MASTER_DATE" "master" "false" "true" \ + "$case_root/$PROJECT/SLC/$MASTER_DATE/$MASTER_DATE.slc.par" || return 1 + + for slave_date in "${SLAVE_DATES[@]}"; do + apply_bridge_once \ + "$case_name" "$case_root" "orbit_bridge_${slave_date}" "$slave_date" "slave" "false" "true" \ + "$case_root/$PROJECT/SLC/$slave_date/$slave_date.slc.par" || return 1 + done + + run_generate_and_coreg "$case_name" "$case_root" || true +} + +run_case_d() { + local case_name='case_D_precise_orbit_validate' + local case_root + case_root="$(build_case "$case_name")" + + apply_bridge_once \ + "$case_name" "$case_root" "orbit_bridge_master" "$MASTER_DATE" "master" "false" "true" \ + "$case_root/$PROJECT/SLC/$MASTER_DATE/$MASTER_DATE.slc.par" || return 1 + + for slave_date in "${SLAVE_DATES[@]}"; do + apply_bridge_once \ + "$case_name" "$case_root" "orbit_bridge_${slave_date}" "$slave_date" "slave" "false" "true" \ + "$case_root/$PROJECT/SLC/$slave_date/$slave_date.slc.par" || return 1 + done + + apply_bridge_once \ + "$case_name" "$case_root" "orbit_validate_master" "$MASTER_DATE" "master" "true" "false" \ + "$case_root/$PROJECT/SLC/$MASTER_DATE/$MASTER_DATE.slc.par" || true + + for slave_date in "${SLAVE_DATES[@]}"; do + apply_bridge_once \ + "$case_name" "$case_root" "orbit_validate_${slave_date}" "$slave_date" "slave" "true" "false" \ + "$case_root/$PROJECT/SLC/$slave_date/$slave_date.slc.par" || true + done + + run_generate_and_coreg "$case_name" "$case_root" || true +} + +main() { + ensure_dir "$BASE_ROOT" + ensure_dir "$BASE_ROOT/$PROJECT/SLC" + ensure_dir "$BASE_ROOT/dem_store/$PROJECT" + ensure_file "$BASE_ROOT/templates/$PROJECT.template" + ensure_file "$GAMMA_ENV" + ensure_file "$ORBIT_HELPER" + ensure_file "$PYTHON_BIN" + + mkdir -p "$RUN_ROOT" + printf 'case\tstage\trc\tstdout_log\tstderr_log\n' >"$STATUS_FILE" + + . "$GAMMA_ENV" >/dev/null 2>&1 + export PATH="/home/administrator/miniconda3/envs/isce2/bin:$GAMMA_SCRIPT_DIR:$PYINT_SCRIPT_DIR:$PATH" + export PYTHONPATH="$PYINT_HOME${PYTHONPATH:+:$PYTHONPATH}" + + cat >"$RUN_ROOT/run_info.txt" <<EOF +run_root=$RUN_ROOT +base_root=$BASE_ROOT +project=$PROJECT +master=$MASTER_DATE +slaves=${SLAVE_DATES[*]} +satellite=$SATELLITE +orbit_dir=$ORBIT_DIR +python=$PYTHON_BIN +pyint_home=$PYINT_HOME +EOF + + run_case_a + run_case_c + run_case_d + + echo "[done] run_root=$RUN_ROOT" + echo "[done] status_file=$STATUS_FILE" +} + +main "$@" diff --git a/.codex_tmp/run_lt1_dem_source_experiment.sh b/.codex_tmp/run_lt1_dem_source_experiment.sh new file mode 100644 index 0000000..3669082 --- /dev/null +++ b/.codex_tmp/run_lt1_dem_source_experiment.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash +set -u +set -o pipefail + +BASE_ROOT='/mnt/d/PyINT_POOL_TEST/LT1A_MONO_SYC_STRIP1_E129.6_N45.0_3scene' +CASES_ROOT='/mnt/d/PyINT_POOL_TEST/LT1A_MONO_SYC_STRIP1_E129.6_N45.0_3scene_dem_cases' +PROJECT='pyint_stage' +MASTER_DATE='20230726' +SLAVE_DATES=(20230624 20230920) + +REPO='/mnt/d/Code/Insar_management_system_v2' +PYINT_HOME="$REPO/third_party/PyINT" +PYINT_SCRIPT_DIR="$PYINT_HOME/pyint" +PYTHON_BIN='/home/administrator/miniconda3/envs/isce2/bin/python' +GAMMA_ENV="$REPO/backend/app/pyint_pipeline/pyint_gamma_env.sh" +GAMMA_SCRIPT_DIR='/usr/local/GAMMA_SOFTWARE-20240627/ISP/scripts' + +RUN_STAMP="${1:-$(date -u +%Y%m%dT%H%M%SZ)}" +RUN_ROOT="$CASES_ROOT/run_$RUN_STAMP" +STATUS_FILE="$RUN_ROOT/stage_status.tsv" +CASE_FILTER="${CASE_FILTER:-}" + +fail() { + echo "$1" >&2 + exit 1 +} + +ensure_file() { + local path="$1" + [ -f "$path" ] || fail "Required file not found: $path" +} + +ensure_dir() { + local path="$1" + [ -d "$path" ] || fail "Required directory not found: $path" +} + +write_status() { + local case_name="$1" + local stage="$2" + local rc="$3" + local stdout_log="$4" + local stderr_log="$5" + printf '%s\t%s\t%s\t%s\t%s\n' "$case_name" "$stage" "$rc" "$stdout_log" "$stderr_log" >>"$STATUS_FILE" +} + +run_stage() { + local case_name="$1" + local stage="$2" + shift 2 + local stdout_log="$RUN_ROOT/$case_name/logs/${stage}.stdout.log" + local stderr_log="$RUN_ROOT/$case_name/logs/${stage}.stderr.log" + local rc=0 + + echo "[stage] $case_name :: $stage" + "$@" >"$stdout_log" 2>"$stderr_log" || rc=$? + write_status "$case_name" "$stage" "$rc" "$stdout_log" "$stderr_log" + + if [ "$rc" -eq 0 ]; then + echo "[ok] $case_name :: $stage" + else + echo "[fail] $case_name :: $stage (rc=$rc)" >&2 + fi + return "$rc" +} + +link_or_copy_static() { + local src="$1" + local dst="$2" + if [ -e "$src" ]; then + ln -s "$src" "$dst" + fi +} + +build_scene_dir() { + local case_root="$1" + local scene_date="$2" + local base_dir="$BASE_ROOT/$PROJECT/SLC/$scene_date" + local case_dir="$case_root/$PROJECT/SLC/$scene_date" + + ensure_dir "$base_dir" + mkdir -p "$case_dir" + + ensure_file "$base_dir/$scene_date.slc" + ensure_file "$base_dir/$scene_date.slc.par" + + ln -s "$base_dir/$scene_date.slc" "$case_dir/$scene_date.slc" + cp "$base_dir/$scene_date.slc.par" "$case_dir/$scene_date.slc.par" + + link_or_copy_static "$base_dir/${scene_date}_2rlks.amp" "$case_dir/${scene_date}_2rlks.amp" + link_or_copy_static "$base_dir/${scene_date}_2rlks.amp.par" "$case_dir/${scene_date}_2rlks.amp.par" + link_or_copy_static "$base_dir/${scene_date}_SLC_Tab" "$case_dir/${scene_date}_SLC_Tab" + link_or_copy_static "$base_dir/down2slc.dat" "$case_dir/down2slc.dat" + link_or_copy_static "$base_dir/t_${scene_date}" "$case_dir/t_${scene_date}" +} + +set_template_key() { + local template_path="$1" + local key="$2" + local value="$3" + + "$PYTHON_BIN" -c ' +from pathlib import Path +import sys + +path = Path(sys.argv[1]) +key = sys.argv[2] +value = sys.argv[3] +lines = path.read_text(encoding="utf-8").splitlines() +prefix = key + "=" +updated = False +for idx, line in enumerate(lines): + if line.startswith(prefix): + lines[idx] = prefix + value + updated = True + break +if not updated: + lines.append(prefix + value) +path.write_text("\n".join(lines) + "\n", encoding="utf-8") +' "$template_path" "$key" "$value" +} + +build_case() { + local case_name="$1" + local dem_source="$2" + local opentopo_dem_type="$3" + local case_root="$RUN_ROOT/$case_name" + local template_path + + mkdir -p "$case_root/templates" "$case_root/logs" + mkdir -p "$case_root/$PROJECT/SLC" "$case_root/$PROJECT/DEM" "$case_root/$PROJECT/RSLC" "$case_root/$PROJECT/ifgrams" + mkdir -p "$case_root/dem_store/$PROJECT" + + cp "$BASE_ROOT/templates/$PROJECT.template" "$case_root/templates/$PROJECT.template" + template_path="$case_root/templates/$PROJECT.template" + set_template_key "$template_path" "prepared_dem_source" "$dem_source" + set_template_key "$template_path" "fabdem_dir" "-" + set_template_key "$template_path" "opentopo_dem_type" "$opentopo_dem_type" + set_template_key "$template_path" "opentopo_api_key" "-" + + build_scene_dir "$case_root" "$MASTER_DATE" + for slave_date in "${SLAVE_DATES[@]}"; do + build_scene_dir "$case_root" "$slave_date" + done + + printf '%s\n' "$case_root" +} + +run_case() { + local case_name="$1" + local dem_source="$2" + local opentopo_dem_type="$3" + local case_root + local rc=0 + + case_root="$(build_case "$case_name" "$dem_source" "$opentopo_dem_type")" + + export SCRATCHDIR="$case_root" + export TEMPLATEDIR="$case_root/templates" + export DEMDIR="$case_root/dem_store" + + run_stage "$case_name" "makedem_pyint" "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/makedem_pyint.py" "$PROJECT" || return $? + run_stage "$case_name" "generate_rdc_dem" "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/generate_rdc_dem.py" "$PROJECT" || return $? + + for slave_date in "${SLAVE_DATES[@]}"; do + run_stage "$case_name" "coreg_${slave_date}" "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/coreg_gamma.py" "$PROJECT" "$slave_date" || rc=1 + done + + return "$rc" +} + +should_run_case() { + local case_name="$1" + if [ -z "$CASE_FILTER" ]; then + return 0 + fi + case ",$CASE_FILTER," in + *",$case_name,"*) return 0 ;; + *) return 1 ;; + esac +} + +main() { + ensure_dir "$BASE_ROOT" + ensure_dir "$BASE_ROOT/$PROJECT/SLC" + ensure_file "$BASE_ROOT/templates/$PROJECT.template" + ensure_file "$GAMMA_ENV" + ensure_file "$PYTHON_BIN" + + mkdir -p "$RUN_ROOT" + printf 'case\tstage\trc\tstdout_log\tstderr_log\n' >"$STATUS_FILE" + + . "$GAMMA_ENV" >/dev/null 2>&1 + export PATH="/home/administrator/miniconda3/envs/isce2/bin:$GAMMA_SCRIPT_DIR:$PYINT_SCRIPT_DIR:$PATH" + export PYTHONPATH="$PYINT_HOME${PYTHONPATH:+:$PYTHONPATH}" + + cat >"$RUN_ROOT/run_info.txt" <<EOF +run_root=$RUN_ROOT +base_root=$BASE_ROOT +project=$PROJECT +master=$MASTER_DATE +slaves=${SLAVE_DATES[*]} +python=$PYTHON_BIN +pyint_home=$PYINT_HOME +EOF + + if should_run_case 'case_A_copdem_baseline'; then + run_case 'case_A_copdem_baseline' '/mnt/d/DEM/COPDEM_GLO30_China_4326_DEM' '-' + fi + if should_run_case 'case_B_gmted2010_jp2'; then + run_case 'case_B_gmted2010_jp2' '/mnt/d/DEM/GMTED2010.jp2' '-' + fi + if should_run_case 'case_C_opentopo_srtmgl1'; then + run_case 'case_C_opentopo_srtmgl1' '-' 'SRTMGL1' + fi + + echo "[done] run_root=$RUN_ROOT" + echo "[done] status_file=$STATUS_FILE" +} + +main "$@" diff --git a/.codex_tmp/run_lt1_pool_multiscene_generic.sh b/.codex_tmp/run_lt1_pool_multiscene_generic.sh new file mode 100644 index 0000000..d1c601b --- /dev/null +++ b/.codex_tmp/run_lt1_pool_multiscene_generic.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="${ROOT:-${1:-}}" +PROJECT="${PROJECT:-pyint_stage}" +MASTER_DATE="${MASTER_DATE:-}" +START_DATE="${START_DATE:-}" +END_DATE="${END_DATE:-}" +PREPARED_DEM="${PREPARED_DEM:-/mnt/d/DEM/COPDEM_GLO30_China_4326_DEM}" + +SCRATCHDIR="$ROOT" +TEMPLATEDIR="$ROOT/templates" +DEMDIR="$ROOT/dem_store" +PROJECT_DIR="$SCRATCHDIR/$PROJECT" +LOG_DIR="$ROOT/logs" +PYINT_HOME='/mnt/d/Code/Insar_management_system_v2/third_party/PyINT' +PYINT_SCRIPT_DIR="$PYINT_HOME/pyint" +PYTHON_BIN='/home/administrator/miniconda3/envs/isce2/bin/python' +GAMMA_ENV='/mnt/d/Code/Insar_management_system_v2/backend/app/pyint_pipeline/pyint_gamma_env.sh' +TEMPLATE_PATH="$TEMPLATEDIR/$PROJECT.template" + +die() { + echo "$1" >&2 + exit 1 +} + +run_stage() { + local stage="$1" + shift + local stdout_log="$LOG_DIR/${stage}.stdout.log" + local stderr_log="$LOG_DIR/${stage}.stderr.log" + echo "[stage] $stage" + if "$@" >"$stdout_log" 2>"$stderr_log"; then + echo "[ok] $stage" + else + local rc=$? + echo "[fail] $stage (rc=$rc)" >&2 + echo "stdout: $stdout_log" >&2 + echo "stderr: $stderr_log" >&2 + exit "$rc" + fi +} + +run_single_coreg() { + local slave_date="$1" + mkdir -p "$PROJECT_DIR/SLC" "$PROJECT_DIR/RSLC" "$PROJECT_DIR/DEM" "$PROJECT_DIR/ifgrams" + run_stage "coreg_${slave_date}" "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/coreg_gamma.py" "$PROJECT" "$slave_date" +} + +[ -n "$ROOT" ] || die "ROOT is required" +[ -n "$MASTER_DATE" ] || die "MASTER_DATE is required" +[ -n "$START_DATE" ] || die "START_DATE is required" +[ -n "$END_DATE" ] || die "END_DATE is required" +[ -d "$ROOT" ] || die "Experiment root not found: $ROOT" +[ -d "$PROJECT_DIR/DOWNLOAD" ] || die "DOWNLOAD directory not found: $PROJECT_DIR/DOWNLOAD" +[ -f "$PREPARED_DEM" ] || die "Prepared DEM not found: $PREPARED_DEM" +[ -f "$GAMMA_ENV" ] || die "Gamma env script not found: $GAMMA_ENV" +[ -x "$PYTHON_BIN" ] || die "Python not found: $PYTHON_BIN" + +. "$GAMMA_ENV" >/dev/null 2>&1 +export SCRATCHDIR +export TEMPLATEDIR +export DEMDIR +export PATH="/home/administrator/miniconda3/envs/isce2/bin:$PYINT_SCRIPT_DIR:$PATH" +export PYTHONPATH="$PYINT_HOME${PYTHONPATH:+:$PYTHONPATH}" +export PYINT_LT1_PRECISE_ORBIT_ENABLED='false' +export PYINT_LT1_PRECISE_ORBIT_MODE='bridge' +export PYINT_LT1_PRECISE_ORBIT_STRICT='false' +export PYINT_LT1_PRECISE_ORBIT_VALIDATE_WITH_ORB_FILT='false' +export PYINT_LT1_PRECISE_ORBIT_BACKUP='false' +unset PYINT_LT1_PRECISE_ORBIT_HELPER +unset PYINT_LT1_PRECISE_ORBIT_MANIFEST + +mkdir -p "$TEMPLATEDIR" "$DEMDIR" "$LOG_DIR" +mkdir -p "$PROJECT_DIR/SLC" "$PROJECT_DIR/RSLC" "$PROJECT_DIR/DEM" "$PROJECT_DIR/ifgrams" + +cat >"$TEMPLATE_PATH" <<EOF +# Auto-generated LT-1 pool multiscene test +satelite=LT +masterDate=$MASTER_DATE +range_looks=2 +azimuth_looks=2 +download_data=0 +raw2slc_all=1 +raw2slc_all_parallel=1 +coreg_all=1 +coreg_all_parallel=1 +select_pairs=1 +network_method=sbas +startDate=$START_DATE +endDate=$END_DATE +max_tb=50000 +max_sb=50000 +min_tb=1 +diff_all=0 +unwrap_all=0 +geocode_all=0 +atmcor_all=0 +load_data=0 +prepared_dem_source=$PREPARED_DEM +fabdem_dir=- +opentopo_dem_type=- +opentopo_api_key=- +EOF + +if [ "${2:-}" = "--single-coreg" ] || [ "${1:-}" = "--single-coreg" ]; then + local_date="${3:-${2:-}}" + [ -n "$local_date" ] || die "Usage: ROOT=... MASTER_DATE=... START_DATE=... END_DATE=... $0 --single-coreg YYYYMMDD" + run_single_coreg "$local_date" + echo "[done] single_coreg=$local_date" + exit 0 +fi + +run_stage down2slc_all "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/down2slc_LT1_all.py" "$PROJECT" --parallel 1 +run_stage makedem_pyint "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/makedem_pyint.py" "$PROJECT" +run_stage generate_rdc_dem "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/generate_rdc_dem.py" "$PROJECT" +run_stage coreg_gamma_all "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/coreg_gamma_all.py" "$PROJECT" --parallel 1 +run_stage select_pairs "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/select_pairs.py" "$PROJECT" + +echo "[done] project=$PROJECT" +echo "[done] root=$ROOT" diff --git a/.codex_tmp/run_lt1_pool_multiscene_test.sh b/.codex_tmp/run_lt1_pool_multiscene_test.sh new file mode 100644 index 0000000..bdb9986 --- /dev/null +++ b/.codex_tmp/run_lt1_pool_multiscene_test.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT='/mnt/d/PyINT_POOL_TEST/LT1A_MONO_SYC_STRIP1_E129.6_N45.0_3scene' +PROJECT='pyint_stage' +SCRATCHDIR="$ROOT" +TEMPLATEDIR="$ROOT/templates" +DEMDIR="$ROOT/dem_store" +PROJECT_DIR="$SCRATCHDIR/$PROJECT" +LOG_DIR="$ROOT/logs" +PYINT_HOME='/mnt/d/Code/Insar_management_system_v2/third_party/PyINT' +PYINT_SCRIPT_DIR="$PYINT_HOME/pyint" +PYTHON_BIN='/home/administrator/miniconda3/envs/isce2/bin/python' +GAMMA_ENV='/mnt/d/Code/Insar_management_system_v2/backend/app/pyint_pipeline/pyint_gamma_env.sh' +TEMPLATE_PATH="$TEMPLATEDIR/$PROJECT.template" +PREPARED_DEM='/mnt/d/DEM/COPDEM_GLO30_China_4326_DEM' + +die() { + echo "$1" >&2 + exit 1 +} + +run_stage() { + local stage="$1" + shift + local stdout_log="$LOG_DIR/${stage}.stdout.log" + local stderr_log="$LOG_DIR/${stage}.stderr.log" + echo "[stage] $stage" + if "$@" >"$stdout_log" 2>"$stderr_log"; then + echo "[ok] $stage" + else + local rc=$? + echo "[fail] $stage (rc=$rc)" >&2 + echo "stdout: $stdout_log" >&2 + echo "stderr: $stderr_log" >&2 + exit "$rc" + fi +} + +run_single_coreg() { + local slave_date="$1" + mkdir -p "$PROJECT_DIR/SLC" "$PROJECT_DIR/RSLC" "$PROJECT_DIR/DEM" "$PROJECT_DIR/ifgrams" + run_stage "coreg_${slave_date}" "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/coreg_gamma.py" "$PROJECT" "$slave_date" +} + +[ -d "$ROOT" ] || die "Experiment root not found: $ROOT" +[ -d "$PROJECT_DIR/DOWNLOAD" ] || die "DOWNLOAD directory not found: $PROJECT_DIR/DOWNLOAD" +[ -f "$PREPARED_DEM" ] || die "Prepared DEM not found: $PREPARED_DEM" +[ -f "$GAMMA_ENV" ] || die "Gamma env script not found: $GAMMA_ENV" +[ -x "$PYTHON_BIN" ] || die "Python not found: $PYTHON_BIN" + +. "$GAMMA_ENV" >/dev/null 2>&1 +export SCRATCHDIR +export TEMPLATEDIR +export DEMDIR +export PATH="/home/administrator/miniconda3/envs/isce2/bin:$PYINT_SCRIPT_DIR:$PATH" +export PYTHONPATH="$PYINT_HOME${PYTHONPATH:+:$PYTHONPATH}" +export PYINT_LT1_PRECISE_ORBIT_ENABLED='false' +export PYINT_LT1_PRECISE_ORBIT_MODE='bridge' +export PYINT_LT1_PRECISE_ORBIT_STRICT='false' +export PYINT_LT1_PRECISE_ORBIT_VALIDATE_WITH_ORB_FILT='false' +export PYINT_LT1_PRECISE_ORBIT_BACKUP='false' +unset PYINT_LT1_PRECISE_ORBIT_HELPER +unset PYINT_LT1_PRECISE_ORBIT_MANIFEST + +mkdir -p "$TEMPLATEDIR" "$DEMDIR" "$LOG_DIR" +mkdir -p "$PROJECT_DIR/SLC" "$PROJECT_DIR/RSLC" "$PROJECT_DIR/DEM" "$PROJECT_DIR/ifgrams" + +cat >"$TEMPLATE_PATH" <<'EOF' +# Auto-generated LT-1 pool multiscene test +satelite=LT +masterDate=20230726 +range_looks=2 +azimuth_looks=2 +download_data=0 +raw2slc_all=1 +raw2slc_all_parallel=1 +coreg_all=1 +coreg_all_parallel=1 +select_pairs=1 +network_method=sbas +startDate=20230601 +endDate=20231001 +max_tb=50000 +max_sb=50000 +min_tb=1 +diff_all=0 +unwrap_all=0 +geocode_all=0 +atmcor_all=0 +load_data=0 +prepared_dem_source=/mnt/d/DEM/COPDEM_GLO30_China_4326_DEM +fabdem_dir=- +opentopo_dem_type=- +opentopo_api_key=- +EOF + +if [ "${1:-}" = "--single-coreg" ]; then + [ -n "${2:-}" ] || die "Usage: $0 --single-coreg YYYYMMDD" + run_single_coreg "$2" + echo "[done] single_coreg=$2" + exit 0 +fi + +run_stage down2slc_all "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/down2slc_LT1_all.py" "$PROJECT" --parallel 1 +run_stage makedem_pyint "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/makedem_pyint.py" "$PROJECT" +run_stage generate_rdc_dem "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/generate_rdc_dem.py" "$PROJECT" +run_stage coreg_gamma_all "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/coreg_gamma_all.py" "$PROJECT" --parallel 1 +run_stage select_pairs "$PYTHON_BIN" "$PYINT_SCRIPT_DIR/select_pairs.py" "$PROJECT" + +echo "[done] project=$PROJECT" +echo "[done] root=$ROOT" diff --git a/.codex_tmp/run_pyint_ab_case.sh b/.codex_tmp/run_pyint_ab_case.sh new file mode 100644 index 0000000..00d3f9d --- /dev/null +++ b/.codex_tmp/run_pyint_ab_case.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 3 ]]; then + echo "usage: run_pyint_ab_case.sh <case_name> <pyint_home> <orbit_enabled:true|false> [unwrap:true|false] [geocode:true|false]" >&2 + exit 2 +fi + +case_name=$1 +pyint_home=$2 +orbit_enabled=$3 +unwrap_enabled=${4:-false} +geocode_enabled=${5:-false} + +repo=/mnt/d/Code/Insar_management_system_v2 +task_name=Task_20230602_20230720 +pair_key=lt1_20230602_20230720_ef1cd36538 +task_dir=/mnt/d/Task_Pool/DInSAR/Task_260416_Gamma_PyINT/Task_20230602_20230720 +manifest=/mnt/d/Code/Insar_management_system_v2/backend/runtime/pyint_input_assets/$pair_key/run_20260419T065041Z_pyint_lt1_gamma_dinsar/task_manifest.json +input_assets_dir=$(dirname "$manifest") +python_bin=/home/administrator/miniconda3/envs/isce2/bin/python +script=$repo/backend/app/pyint_pipeline/run_lt1_pyint_pipeline.py +pyint_app=$pyint_home/pyint/pyintApp.py +gamma_env=$repo/backend/app/pyint_pipeline/pyint_gamma_env.sh +prepared_dem=/mnt/d/DEM/COPDEM_GLO30_China_4326_DEM + +case_root=/mnt/d/PyINT_AB/$case_name +run_root=$case_root/work +project_name=${pair_key}_${case_name} +project_dir=$run_root/$project_name +template_root=$run_root/templates +output_dir=$case_root/output +dem_root=$case_root/dem_cache + +rm -rf "$case_root" +mkdir -p "$case_root" + +cmd=( + "$python_bin" "$script" "$task_dir" + --project-dir "$project_dir" + --template-root "$template_root" + --output-dir "$output_dir" + --pyint-home "$pyint_home" + --pyint-app-script "$pyint_app" + --python "$python_bin" + --dem-root "$dem_root" + --dem-mode prepared_file + --prepared-dem-path "$prepared_dem" + --project-name "$project_name" + --gamma-env-script "$gamma_env" + --pair-key "$pair_key" + --task-alias "$task_name" + --orbit-policy require_txt + --input-assets-dir "$input_assets_dir" + --input-assets-json "$manifest" + --master-date 20230602 + --slave-date 20230720 + --time-baseline-days 48 + --range-looks 2 + --azimuth-looks 2 + --parallel-workers 1 + --lt1-precise-orbit-enabled "$orbit_enabled" +) + +if [[ "$unwrap_enabled" == "true" ]]; then + cmd+=(--unwrap) +else + cmd+=(--no-unwrap) +fi + +if [[ "$geocode_enabled" == "true" ]]; then + cmd+=(--geocode) +else + cmd+=(--no-geocode) +fi + +printf 'CASE=%s\nPYINT_HOME=%s\nORBIT=%s\nUNWRAP=%s\nGEOCODE=%s\nCASE_ROOT=%s\n' \ + "$case_name" "$pyint_home" "$orbit_enabled" "$unwrap_enabled" "$geocode_enabled" "$case_root" + +"${cmd[@]}" diff --git a/.codex_tmp/run_scan_init_offsetm_patch.sh b/.codex_tmp/run_scan_init_offsetm_patch.sh new file mode 100644 index 0000000..4c9c23b --- /dev/null +++ b/.codex_tmp/run_scan_init_offsetm_patch.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO='/mnt/d/Code/Insar_management_system_v2' +RUN_ROOT='/mnt/d/PyINT_POOL_TEST/LT1A_MONO_SYC_STRIP1_E129.6_N45.0_3scene_cases/run_20260420T093322Z' +PYTHON_BIN='/home/administrator/miniconda3/envs/isce2/bin/python' +SCAN_SCRIPT="$REPO/.codex_tmp/scan_init_offsetm_patch.py" +GAMMA_ENV="$REPO/backend/app/pyint_pipeline/pyint_gamma_env.sh" + +. "$GAMMA_ENV" >/dev/null 2>&1 +export PATH="/usr/local/GAMMA_SOFTWARE-20240627/ISP/bin:/usr/local/GAMMA_SOFTWARE-20240627/ISP/scripts:$PATH" + +"$PYTHON_BIN" "$SCAN_SCRIPT" "$RUN_ROOT" "$@" diff --git a/.codex_tmp/scan_init_offsetm_patch.py b/.codex_tmp/scan_init_offsetm_patch.py new file mode 100644 index 0000000..9df317c --- /dev/null +++ b/.codex_tmp/scan_init_offsetm_patch.py @@ -0,0 +1,317 @@ +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import numpy as np + + +GAMMA_FLOAT32 = np.dtype(">f4") +ZERO_ERROR_RE = re.compile(r"number of zero values\s+(\d+)\s+in MLI1 image patch exceeds threshold:\s+(\d+)", re.IGNORECASE) + + +@dataclass +class GammaShape: + width: int + lines: int + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Scan init_offsetm patch positions for LT-1 DEM-assisted coreg.") + parser.add_argument("run_root", help="Experiment run root, e.g. /mnt/d/.../run_20260420T093322Z") + parser.add_argument("--case", dest="cases", action="append", default=[], help="Case name to scan. Can be repeated.") + parser.add_argument("--date", dest="dates", action="append", default=[], help="Slave date to scan. Can be repeated.") + parser.add_argument("--project", default="pyint_stage") + parser.add_argument("--patch-size", type=int, default=512) + parser.add_argument("--stride", type=int, default=256) + parser.add_argument("--max-candidates", type=int, default=20) + return parser.parse_args() + + +def parse_gamma_par_value(path: Path, key: str) -> str: + prefix = key.strip() + ":" + for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): + stripped = line.strip() + if stripped.startswith(prefix): + _, _, tail = stripped.partition(":") + return tail.strip().split()[0] + raise ValueError(f"Missing key '{key}' in {path}") + + +def parse_gamma_shape(path: Path) -> GammaShape: + width = int(float(parse_gamma_par_value(path, "range_samples"))) + lines = int(float(parse_gamma_par_value(path, "azimuth_lines"))) + return GammaShape(width=width, lines=lines) + + +def build_valid_mask(path: Path, shape: GammaShape) -> np.ndarray: + arr = np.memmap(path, dtype=GAMMA_FLOAT32, mode="r", shape=(shape.lines, shape.width)) + valid = np.isfinite(arr) & (arr != 0) + return np.asarray(valid, dtype=np.uint8) + + +def patch_sum(integral: np.ndarray, top: int, left: int, bottom: int, right: int) -> int: + br = int(integral[bottom, right]) + tr = int(integral[top, right]) + bl = int(integral[bottom, left]) + tl = int(integral[top, left]) + return br - tr - bl + tl + + +def candidate_positions(mask: np.ndarray, *, patch_size: int, stride: int, max_candidates: int) -> list[dict[str, Any]]: + lines, width = mask.shape + patch_size = max(1, min(patch_size, lines, width)) + half = patch_size // 2 + valid_y = range(half, lines - (patch_size - half) + 1, max(1, stride)) + valid_x = range(half, width - (patch_size - half) + 1, max(1, stride)) + + integral = np.pad(mask.astype(np.int64), ((1, 0), (1, 0)), mode="constant") + integral = integral.cumsum(axis=0).cumsum(axis=1) + patch_area = patch_size * patch_size + + center_y = lines // 2 + center_x = width // 2 + center_key = (center_x, center_y) + + candidates: list[dict[str, Any]] = [] + seen: set[tuple[int, int]] = set() + + for y in valid_y: + for x in valid_x: + top = y - half + left = x - half + bottom = top + patch_size + right = left + patch_size + nonzero = patch_sum(integral, top, left, bottom, right) + ratio = float(nonzero / patch_area) + entry = { + "rpos": int(x), + "azpos": int(y), + "patch_nonzero_count": int(nonzero), + "patch_nonzero_ratio": ratio, + "is_center": bool((x, y) == center_key), + } + candidates.append(entry) + + candidates.sort(key=lambda item: item["patch_nonzero_ratio"], reverse=True) + + selected: list[dict[str, Any]] = [] + for entry in candidates: + key = (entry["rpos"], entry["azpos"]) + if key in seen: + continue + selected.append(entry) + seen.add(key) + if len(selected) >= max_candidates: + break + + if center_key not in seen: + top = center_y - half + left = center_x - half + top = max(0, min(lines - patch_size, top)) + left = max(0, min(width - patch_size, left)) + bottom = top + patch_size + right = left + patch_size + nonzero = patch_sum(integral, top, left, bottom, right) + selected.append( + { + "rpos": int(left + half), + "azpos": int(top + half), + "patch_nonzero_count": int(nonzero), + "patch_nonzero_ratio": float(nonzero / patch_area), + "is_center": True, + } + ) + + for index, entry in enumerate(selected, start=1): + entry["rank"] = index + return selected + + +def parse_zero_error(text: str) -> dict[str, Any]: + match = ZERO_ERROR_RE.search(text or "") + if not match: + return {} + return { + "zero_count": int(match.group(1)), + "zero_threshold": int(match.group(2)), + } + + +def run_init_offsetm( + *, + mli0: Path, + samp: Path, + diff0: Path, + output_dir: Path, + patch_size: int, + rpos: int, + azpos: int, +) -> dict[str, Any]: + output_dir.mkdir(parents=True, exist_ok=True) + diff_copy = output_dir / f"r{rpos}_a{azpos}.diff_par" + shutil.copy2(diff0, diff_copy) + + cmd = [ + "init_offsetm", + str(mli0), + str(samp), + str(diff_copy), + "1", + "1", + str(int(rpos)), + str(int(azpos)), + "-", + "-", + "-", + str(int(patch_size)), + "0", + ] + result = subprocess.run(cmd, text=True, capture_output=True, check=False) + combined = ((result.stdout or "") + "\n" + (result.stderr or "")).strip() + payload = { + "command": " ".join(cmd), + "returncode": int(result.returncode), + "ok": result.returncode == 0, + "stdout_tail": (result.stdout or "")[-4000:], + "stderr_tail": (result.stderr or "")[-4000:], + "diff_par_copy": str(diff_copy), + } + payload.update(parse_zero_error(combined)) + return payload + + +def scan_case_date( + *, + run_root: Path, + project: str, + case_name: str, + slave_date: str, + patch_size: int, + stride: int, + max_candidates: int, + output_root: Path, +) -> dict[str, Any]: + case_root = run_root / case_name / project + slc_dir = case_root / "SLC" / slave_date + rslc_dir = case_root / "RSLC" / slave_date + amp_par = slc_dir / f"{slave_date}_2rlks.amp.par" + shape = parse_gamma_shape(amp_par) + mask = build_valid_mask(rslc_dir / "mli0", shape) + + candidates = candidate_positions(mask, patch_size=patch_size, stride=stride, max_candidates=max_candidates) + output_dir = output_root / case_name / slave_date + results = [] + for candidate in candidates: + command_result = run_init_offsetm( + mli0=rslc_dir / "mli0", + samp=slc_dir / f"{slave_date}_2rlks.amp", + diff0=rslc_dir / "diff0", + output_dir=output_dir / "diff_par", + patch_size=patch_size, + rpos=int(candidate["rpos"]), + azpos=int(candidate["azpos"]), + ) + row = dict(candidate) + row.update(command_result) + results.append(row) + + success_count = sum(1 for item in results if item["ok"]) + best_ratio = max((float(item["patch_nonzero_ratio"]) for item in results), default=0.0) + best_zero = min((int(item.get("zero_count", 10**18)) for item in results if "zero_count" in item), default=None) + payload = { + "case": case_name, + "slave_date": slave_date, + "shape": {"width": shape.width, "lines": shape.lines}, + "patch_size": int(patch_size), + "stride": int(stride), + "candidate_count": len(results), + "success_count": int(success_count), + "best_patch_nonzero_ratio": best_ratio, + "best_zero_count": best_zero, + "results": results, + } + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "scan_summary.json").write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + return payload + + +def write_summary_tsv(path: Path, summaries: list[dict[str, Any]]) -> None: + header = [ + "case", + "slave_date", + "rank", + "is_center", + "rpos", + "azpos", + "patch_nonzero_ratio", + "returncode", + "ok", + "zero_count", + "zero_threshold", + ] + lines = ["\t".join(header)] + for summary in summaries: + for row in summary["results"]: + values: list[str] = [] + for key in header: + if key in {"case", "slave_date"}: + value = summary[key] + else: + value = row.get(key) + if isinstance(value, float): + values.append(f"{value:.6f}") + elif value is None: + values.append("") + else: + values.append(str(value)) + lines.append("\t".join(values)) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def main() -> int: + args = parse_args() + run_root = Path(args.run_root).resolve() + case_names = tuple(args.cases) if args.cases else ("case_A_baseline", "case_C_precise_orbit_rewrite") + slave_dates = tuple(args.dates) if args.dates else ("20230624", "20230920") + output_root = run_root / "scan_init_offsetm_patch" + output_root.mkdir(parents=True, exist_ok=True) + + if not shutil.which("init_offsetm"): + raise RuntimeError("init_offsetm is not available in PATH") + + summaries = [] + for case_name in case_names: + for slave_date in slave_dates: + summaries.append( + scan_case_date( + run_root=run_root, + project=args.project, + case_name=case_name, + slave_date=slave_date, + patch_size=int(args.patch_size), + stride=int(args.stride), + max_candidates=int(args.max_candidates), + output_root=output_root, + ) + ) + + (output_root / "scan_summary.json").write_text( + json.dumps({"run_root": str(run_root), "summaries": summaries}, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + write_summary_tsv(output_root / "scan_summary.tsv", summaries) + print(json.dumps({"run_root": str(run_root), "output_root": str(output_root), "scan_count": len(summaries)}, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.codex_tmp/setup_lt1_pool_multiscene_experiment.ps1 b/.codex_tmp/setup_lt1_pool_multiscene_experiment.ps1 new file mode 100644 index 0000000..65fb0cd --- /dev/null +++ b/.codex_tmp/setup_lt1_pool_multiscene_experiment.ps1 @@ -0,0 +1,87 @@ +param( + [Parameter(Mandatory = $true)] + [string]$Root, + + [Parameter(Mandatory = $true)] + [string]$SourcePool, + + [Parameter(Mandatory = $true)] + [string[]]$Scenes +) + +$ErrorActionPreference = "Stop" + +function Fail([string]$Message) { + throw $Message +} + +$rootPath = [System.IO.Path]::GetFullPath($Root) +$sourcePoolPath = [System.IO.Path]::GetFullPath($SourcePool) + +if (-not (Test-Path -LiteralPath $sourcePoolPath -PathType Container)) { + Fail "Source pool not found: $sourcePoolPath" +} + +if (Test-Path -LiteralPath $rootPath) { + Fail "Experiment root already exists: $rootPath" +} + +$downloadDir = Join-Path $rootPath "pyint_stage\DOWNLOAD" +$inputDir = Join-Path $rootPath "input" +$templatesDir = Join-Path $rootPath "templates" +$logsDir = Join-Path $rootPath "logs" +$demStoreDir = Join-Path $rootPath "dem_store" + +New-Item -ItemType Directory -Path $downloadDir -Force | Out-Null +New-Item -ItemType Directory -Path $inputDir -Force | Out-Null +New-Item -ItemType Directory -Path $templatesDir -Force | Out-Null +New-Item -ItemType Directory -Path $logsDir -Force | Out-Null +New-Item -ItemType Directory -Path $demStoreDir -Force | Out-Null + +$manifestLines = @( + "Experiment Root: $rootPath" + "Created: $(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')" + "Source Pool: $sourcePoolPath" + "Scenes:" +) + +$copiedScenes = @() + +foreach ($scene in $Scenes) { + $sceneSourceDir = Join-Path $sourcePoolPath $scene + if (-not (Test-Path -LiteralPath $sceneSourceDir -PathType Container)) { + Fail "Scene not found: $sceneSourceDir" + } + + $sceneInputDir = Join-Path $inputDir $scene + New-Item -ItemType Directory -Path $sceneInputDir -Force | Out-Null + + $files = Get-ChildItem -LiteralPath $sceneSourceDir -File + if ($files.Count -eq 0) { + Fail "No files found in scene: $sceneSourceDir" + } + + foreach ($file in $files) { + Copy-Item -LiteralPath $file.FullName -Destination (Join-Path $sceneInputDir $file.Name) + Copy-Item -LiteralPath $file.FullName -Destination (Join-Path $downloadDir $file.Name) + } + + $manifestLines += "$scene | files=$($files.Count)" + $copiedScenes += [pscustomobject]@{ + scene = $scene + file_count = $files.Count + } +} + +$manifestPath = Join-Path $rootPath "MANIFEST.txt" +Set-Content -LiteralPath $manifestPath -Value $manifestLines -Encoding UTF8 + +$result = [pscustomobject]@{ + root = $rootPath + download_dir = $downloadDir + input_dir = $inputDir + manifest = $manifestPath + scenes = $copiedScenes +} + +$result | ConvertTo-Json -Depth 4 diff --git a/.codex_tmp/summarize_pyint_case.py b/.codex_tmp/summarize_pyint_case.py new file mode 100644 index 0000000..b8b67a4 --- /dev/null +++ b/.codex_tmp/summarize_pyint_case.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import json +import math +import sys +from pathlib import Path + +import numpy as np + + +def summarize_float_raster(path: Path) -> dict[str, object]: + if not path.is_file(): + return {"exists": False} + + arr = np.fromfile(path, dtype=np.float32) + if arr.size == 0: + return {"exists": True, "count": 0} + + finite = arr[np.isfinite(arr)] + zeros = int((finite == 0).sum()) + nz = finite[finite != 0] + result: dict[str, object] = { + "exists": True, + "count": int(finite.size), + "zero_ratio": float(zeros / finite.size) if finite.size else None, + "nonzero_count": int(nz.size), + } + if nz.size: + result.update( + { + "min_nonzero": float(nz.min()), + "max_nonzero": float(nz.max()), + "mean_nonzero": float(nz.mean()), + "std_nonzero": float(nz.std()), + } + ) + return result + + +def main() -> int: + if len(sys.argv) != 2: + print("usage: summarize_pyint_case.py <case_root>", file=sys.stderr) + return 2 + + case_root = Path(sys.argv[1]).resolve() + output_root = case_root / "output" + summary_path = output_root / "pyint_run_summary.json" + if not summary_path.is_file(): + print(json.dumps({"case_root": str(case_root), "summary_exists": False}, ensure_ascii=False, indent=2)) + return 1 + + payload = json.loads(summary_path.read_text(encoding="utf-8")) + pair_dir = Path(payload["copied_outputs"]["pair_dir"]) + + pair_name = str(payload["pair_name"]) + rlks = int(payload["range_looks"]) + look_text = f"{rlks}rlks" + + files = { + "diff_filt": pair_dir / f"{pair_name}_{look_text}.diff_filt", + "cor": pair_dir / f"{pair_name}_{look_text}.diff_filt.cor", + "unw": pair_dir / f"{pair_name}_{look_text}.diff_filt.unw", + "los": pair_dir / f"{pair_name}_{look_text}.los_disp", + "vert": pair_dir / f"{pair_name}_{look_text}.vert_disp", + "geo_unw": pair_dir / f"geo_{pair_name}_{look_text}.diff_filt.unw", + "geo_los": pair_dir / f"geo_{pair_name}_{look_text}.los_disp", + "geo_vert": pair_dir / f"geo_{pair_name}_{look_text}.vert_disp", + } + + result = { + "case_root": str(case_root), + "ok": payload.get("ok"), + "project_name": payload.get("project_name"), + "pair_dir": str(pair_dir), + "files": {name: summarize_float_raster(path) for name, path in files.items()}, + } + print(json.dumps(result, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.env.example b/.env.example index bf7b013..e7011d4 100644 --- a/.env.example +++ b/.env.example @@ -124,14 +124,58 @@ ISCE2_PIPELINE_SCRIPT= ISCE2_DEM_PATH= ISCE2_WORK_ROOT=D:\isce2_work ISCE2_OUTPUT_ROOT=D:\isce2_output +ISCE2_PER_TASK_TIMEOUT_SECONDS=43200 ISCE2_SMOKE_TEST_ENABLED=false +# ----------------------------------------------------------------------------- +# PyINT / Gamma 引擎（需要 WSL + PyINT + GAMMA） +# ----------------------------------------------------------------------------- +PYINT_ENABLED=false +# 留空时默认跟随 ISCE2_WSL_DISTRO +PYINT_WSL_DISTRO= +# 留空时默认跟随 ISCE2_PYTHON +PYINT_WSL_PYTHON= +# 留空时默认使用项目内置的 third_party\PyINT +PYINT_HOME= +# 留空时默认使用 PYINT_HOME\pyint\pyintApp.py +PYINT_APP_SCRIPT= +PYINT_TEMPLATE_ROOT=D:\Code\Insar_management_system_v2\backend\runtime\pyint_templates +PYINT_WORK_ROOT=D:\Code\Insar_management_system_v2\backend\runtime\pyint_work +PYINT_OUTPUT_ROOT=D:\Code\Insar_management_system_v2\backend\runtime\pyint_output +PYINT_DEM_ROOT=D:\Code\Insar_management_system_v2\backend\runtime\pyint_dem +# 可选: local_fabdem / opentopo / prepared_file +PYINT_DEM_MODE=local_fabdem +PYINT_FABDEM_ROOT= +# 现有 DEM 复用模式下可指向： +# 1. 已有 Gamma DEM 基础文件（需同名 .par） +# 2. 系统现有 DEM 基础文件（需至少有 .xml/.hdr/.vrt，运行时按任务范围裁剪并转换为 Gamma DEM） +# 留空时会尝试回退到 ISCE2_DEM_PATH / IDL_DINSAR_DEM_BASE_FILE +PYINT_PREPARED_DEM_PATH= +PYINT_OPENTOPO_DEM_TYPE=SRTMGL1 +PYINT_DEM_STRICT=true +PYINT_ORBIT_POLICY=require_txt +PYINT_ORBIT_POOL_TXT= +PYINT_RECORD_INPUT_ASSETS=true +PYINT_LT1_PRECISE_ORBIT_ENABLED=true +PYINT_LT1_PRECISE_ORBIT_MODE=replace +PYINT_LT1_PRECISE_ORBIT_STRICT=true +PYINT_LT1_PRECISE_ORBIT_VALIDATE_WITH_ORB_FILT=false +PYINT_LT1_PRECISE_ORBIT_BACKUP=true +PYINT_LT1_PRECISE_ORBIT_ORB_FILT_DEGREE=5 +# 可选：source 该脚本后应能找到 create_offset / geocode_back / LT1_import_SLC_from_zipfiles1 +PYINT_GAMMA_ENV_SCRIPT= +PYINT_DEFAULT_TIMEOUT_SECONDS=43200 +PYINT_SMOKE_TEST_ENABLED=false + + # ----------------------------------------------------------------------------- # 前端 tile-server 接入 # 说明：frontend/vite.config.js 会同时读取项目根 .env 与 frontend/.env， # 这里配置的 VITE_TILE_* 会透传给前端，且优先级高于 frontend/.env 同名项 # ----------------------------------------------------------------------------- +# Tile Server +# ----------------------------------------------------------------------------- VITE_TILE_SERVER_URL=http://127.0.0.1:8910 VITE_TILE_SERVER_TOKEN=change_me @@ -159,6 +203,8 @@ DEFAULT_VLM_MODEL=qwen3-vl:30b UNPACK_MIN_DISK_SPACE_GB=50 UNPACK_SCAN_WORKERS=4 UNPACK_EXTRACT_WORKERS=4 +UNPACK_MAX_FILES_PER_RUN=100 +UNPACK_MAX_RUNTIME_MINUTES=360 UNPACK_DELETE_ARCHIVE=true UNPACK_TMP_SUFFIX=.unpack_tmp diff --git a/backend/app/config.py b/backend/app/config.py index 77e20b3..33dcb74 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -219,12 +219,40 @@ class Settings(BaseSettings): ISCE2_DEM_PATH: str = "D:\\SRTM30m\\SRTMDEM_RSP_SARscape.wgs84" ISCE2_WORK_ROOT: str = "" ISCE2_OUTPUT_ROOT: str = "" + ISCE2_PER_TASK_TIMEOUT_SECONDS: int = 43200 ISCE2_SMOKE_TEST_ENABLED: bool = False ISCE2_STRIPMAP_APP: str = ( "/home/administrator/miniconda3/envs/isce2/lib/python3.11/" "site-packages/isce/applications/stripmapApp.py" ) ISCE2_PIPELINE_SCRIPT: str = "" + PYINT_ENABLED: bool = False + PYINT_WSL_DISTRO: str = "" + PYINT_WSL_PYTHON: str = "" + PYINT_HOME: str = "" + PYINT_APP_SCRIPT: str = "" + PYINT_TEMPLATE_ROOT: str = "" + PYINT_WORK_ROOT: str = "" + PYINT_OUTPUT_ROOT: str = "" + PYINT_DEM_ROOT: str = "" + PYINT_DEM_MODE: str = "local_fabdem" + PYINT_FABDEM_ROOT: str = "" + PYINT_PREPARED_DEM_PATH: str = "" + PYINT_OPENTOPO_DEM_TYPE: str = "SRTMGL1" + PYINT_OPENTOPO_API_KEY: str = "" + PYINT_DEM_STRICT: bool = True + PYINT_ORBIT_POLICY: str = "require_txt" + PYINT_ORBIT_POOL_TXT: str = "" + PYINT_RECORD_INPUT_ASSETS: bool = True + PYINT_LT1_PRECISE_ORBIT_ENABLED: bool = True + PYINT_LT1_PRECISE_ORBIT_MODE: str = "replace" + PYINT_LT1_PRECISE_ORBIT_STRICT: bool = True + PYINT_LT1_PRECISE_ORBIT_VALIDATE_WITH_ORB_FILT: bool = False + PYINT_LT1_PRECISE_ORBIT_BACKUP: bool = True + PYINT_LT1_PRECISE_ORBIT_ORB_FILT_DEGREE: int = 5 + PYINT_GAMMA_ENV_SCRIPT: str = "" + PYINT_DEFAULT_TIMEOUT_SECONDS: int = 43200 + PYINT_SMOKE_TEST_ENABLED: bool = False JOB_WORKER_HEALTH_TIMEOUT: int = 60 JOB_WORKER_JOB_HEARTBEAT_INTERVAL: float = 5.0 JOB_WORKER_STALE_RECOVER_INTERVAL: float = 15.0 @@ -330,6 +358,67 @@ class Settings(BaseSettings): "ISCE2_PIPELINE_SCRIPT", _windows_path_to_wsl_mount(local_pipeline), ) + if not self.PYINT_WSL_DISTRO: + object.__setattr__(self, "PYINT_WSL_DISTRO", self.ISCE2_WSL_DISTRO) + if not self.PYINT_WSL_PYTHON: + object.__setattr__(self, "PYINT_WSL_PYTHON", self.ISCE2_PYTHON) + if not self.PYINT_HOME: + object.__setattr__( + self, + "PYINT_HOME", + os.path.join(project_root, "third_party", "PyINT"), + ) + if not self.PYINT_APP_SCRIPT and self.PYINT_HOME: + object.__setattr__( + self, + "PYINT_APP_SCRIPT", + os.path.join(self.PYINT_HOME, "pyint", "pyintApp.py"), + ) + if not self.PYINT_TEMPLATE_ROOT: + object.__setattr__( + self, + "PYINT_TEMPLATE_ROOT", + os.path.join(backend_dir, "runtime", "pyint_templates"), + ) + if not self.PYINT_WORK_ROOT: + object.__setattr__( + self, + "PYINT_WORK_ROOT", + os.path.join(backend_dir, "runtime", "pyint_work"), + ) + if not self.PYINT_OUTPUT_ROOT: + object.__setattr__( + self, + "PYINT_OUTPUT_ROOT", + os.path.join(backend_dir, "runtime", "pyint_output"), + ) + if not self.PYINT_DEM_ROOT: + object.__setattr__( + self, + "PYINT_DEM_ROOT", + os.path.join(backend_dir, "runtime", "pyint_dem"), + ) + pyint_dem_mode = str(self.PYINT_DEM_MODE or "local_fabdem").strip().lower() or "local_fabdem" + if pyint_dem_mode not in {"local_fabdem", "opentopo", "prepared_file"}: + pyint_dem_mode = "local_fabdem" + object.__setattr__(self, "PYINT_DEM_MODE", pyint_dem_mode) + if not self.PYINT_OPENTOPO_DEM_TYPE: + object.__setattr__(self, "PYINT_OPENTOPO_DEM_TYPE", "SRTMGL1") + pyint_orbit_policy = str(self.PYINT_ORBIT_POLICY or "require_txt").strip().lower() or "require_txt" + if pyint_orbit_policy not in {"validate_only", "require_txt", "stage_txt"}: + pyint_orbit_policy = "require_txt" + object.__setattr__(self, "PYINT_ORBIT_POLICY", pyint_orbit_policy) + pyint_precise_orbit_mode = str(self.PYINT_LT1_PRECISE_ORBIT_MODE or "replace").strip().lower() or "replace" + if pyint_precise_orbit_mode not in {"replace", "replace_and_validate"}: + pyint_precise_orbit_mode = "replace" + object.__setattr__(self, "PYINT_LT1_PRECISE_ORBIT_MODE", pyint_precise_orbit_mode) + object.__setattr__( + self, + "PYINT_LT1_PRECISE_ORBIT_ORB_FILT_DEGREE", + max(1, int(self.PYINT_LT1_PRECISE_ORBIT_ORB_FILT_DEGREE or 5)), + ) + if not self.PYINT_ORBIT_POOL_TXT: + object.__setattr__(self, "PYINT_ORBIT_POOL_TXT", self.ORBIT_POOL_ENVI) if not self.TIMESERIES_WSL_DISTRO: object.__setattr__(self, "TIMESERIES_WSL_DISTRO", self.ISCE2_WSL_DISTRO) if not self.TIMESERIES_ENV_NAME: @@ -431,6 +520,10 @@ class Settings(BaseSettings): os.makedirs(settings.DINSAR_PRODUCT_DIR, exist_ok=True) os.makedirs(settings.PSINSAR_PRODUCT_DIR, exist_ok=True) os.makedirs(settings.RESULT_QUARANTINE_ROOT, exist_ok=True) + os.makedirs(settings.PYINT_TEMPLATE_ROOT, exist_ok=True) + os.makedirs(settings.PYINT_WORK_ROOT, exist_ok=True) + os.makedirs(settings.PYINT_OUTPUT_ROOT, exist_ok=True) + os.makedirs(settings.PYINT_DEM_ROOT, exist_ok=True) os.makedirs(settings.TIMESERIES_WORK_ROOT, exist_ok=True) @@ -601,6 +694,92 @@ def validate_runtime_config() -> dict[str, Any]: if not settings.ORBIT_POOL_ISCE2: warnings.append("ISCE2_ENABLED=true 但 ORBIT_POOL_ISCE2 未配置。") + if settings.PYINT_ENABLED: + if not settings.PYINT_WSL_DISTRO: + errors.append("PYINT_ENABLED=true but PYINT_WSL_DISTRO is not configured.") + if not settings.PYINT_WSL_PYTHON: + errors.append("PYINT_ENABLED=true but PYINT_WSL_PYTHON is not configured.") + if not settings.PYINT_HOME: + errors.append("PYINT_ENABLED=true but PYINT_HOME is not configured.") + if not settings.PYINT_APP_SCRIPT: + errors.append("PYINT_ENABLED=true but PYINT_APP_SCRIPT is not configured.") + _check_path( + label="PYINT_HOME", + value=settings.PYINT_HOME, + errors=errors, + warnings=warnings, + expect_file=False, + ) + _check_path( + label="PYINT_APP_SCRIPT", + value=settings.PYINT_APP_SCRIPT, + errors=errors, + warnings=warnings, + expect_file=True, + ) + _check_path( + label="PYINT_TEMPLATE_ROOT", + value=settings.PYINT_TEMPLATE_ROOT, + errors=errors, + warnings=warnings, + expect_file=False, + ) + _check_path( + label="PYINT_WORK_ROOT", + value=settings.PYINT_WORK_ROOT, + errors=errors, + warnings=warnings, + expect_file=False, + ) + _check_path( + label="PYINT_OUTPUT_ROOT", + value=settings.PYINT_OUTPUT_ROOT, + errors=errors, + warnings=warnings, + expect_file=False, + ) + _check_path( + label="PYINT_DEM_ROOT", + value=settings.PYINT_DEM_ROOT, + errors=errors, + warnings=warnings, + expect_file=False, + ) + if settings.PYINT_DEM_MODE == "local_fabdem": + _check_path( + label="PYINT_FABDEM_ROOT", + value=settings.PYINT_FABDEM_ROOT, + errors=errors, + warnings=warnings, + expect_file=False, + ) + elif settings.PYINT_DEM_MODE == "prepared_file": + _check_path( + label="PYINT_PREPARED_DEM_PATH", + value=( + settings.PYINT_PREPARED_DEM_PATH + or settings.ISCE2_DEM_PATH + or settings.IDL_DINSAR_DEM_BASE_FILE + ), + errors=errors, + warnings=warnings, + expect_file=True, + ) + _check_path( + label="PYINT_ORBIT_POOL_TXT", + value=settings.PYINT_ORBIT_POOL_TXT, + errors=errors, + warnings=warnings, + expect_file=False, + ) + _check_path( + label="PYINT_GAMMA_ENV_SCRIPT", + value=settings.PYINT_GAMMA_ENV_SCRIPT, + errors=errors, + warnings=warnings, + expect_file=True, + ) + if settings.TIMESERIES_ENABLED: _check_path( label="TIMESERIES_PYTHON", diff --git a/backend/app/dinsar_engines/base.py b/backend/app/dinsar_engines/base.py index 37c299b..e792268 100644 --- a/backend/app/dinsar_engines/base.py +++ b/backend/app/dinsar_engines/base.py @@ -3,7 +3,7 @@ from __future__ import annotations from abc import ABC, abstractmethod from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional @dataclass @@ -38,6 +38,7 @@ class RunRequest: num_to_process: int = 0 timeout_seconds: Optional[int] = None extra: Dict[str, Any] = field(default_factory=dict) + progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None @dataclass @@ -80,12 +81,19 @@ class DinsarEngine(ABC): def run(self, request: RunRequest) -> RunResult: """Executes a production run synchronously.""" + @property + def default_timeout_seconds(self) -> Optional[int]: + """Returns the engine's default timeout when the caller leaves it empty.""" + + return None + def to_dict(self) -> Dict[str, Any]: """Serializes the engine definition for API responses.""" return { "engine_code": self.engine_code, "engine_label": self.engine_label, + "default_timeout_seconds": self.default_timeout_seconds, "profiles": [ { "code": profile.code, diff --git a/backend/app/dinsar_engines/isce2_engine.py b/backend/app/dinsar_engines/isce2_engine.py index 02c8581..37b6d0a 100644 --- a/backend/app/dinsar_engines/isce2_engine.py +++ b/backend/app/dinsar_engines/isce2_engine.py @@ -6,7 +6,7 @@ from datetime import datetime from pathlib import Path from typing import Any, Dict, List -from ..config import get_env_text, read_bool_env +from ..config import get_env_text, read_bool_env, settings from .base import DinsarEngine, EngineAvailability, EngineProfile, RunRequest, RunResult from ..services.dinsar_naming import ( PAIR_META_FILENAME, @@ -56,6 +56,10 @@ class Isce2Engine(DinsarEngine): def engine_label(self) -> str: return "ISCE2（WSL）" + @property + def default_timeout_seconds(self) -> int: + return max(60, int(settings.ISCE2_PER_TASK_TIMEOUT_SECONDS or 43200)) + # ------------------------------------------------------------------ # Config helpers # ------------------------------------------------------------------ @@ -410,11 +414,21 @@ class Isce2Engine(DinsarEngine): extra = self.normalize_extra(request.extra) validation = self.validate_root_dir(request.root_dir, request.num_to_process) task_dirs: List[str] = validation["task_dirs"] + total_tasks = len(task_dirs) run_started_at = datetime.utcnow() run_started_at_text = run_started_at.isoformat(timespec="seconds") + "Z" run_key = build_run_key(self.engine_code, request.profile, started_at=run_started_at) + progress_callback = request.progress_callback - timeout = request.timeout_seconds or 21600 + def emit_progress(event_type: str, **payload: Any) -> None: + if not callable(progress_callback): + return + try: + progress_callback({"event": event_type, **payload}) + except Exception: + return + + timeout = max(60, int(request.timeout_seconds or self.default_timeout_seconds)) force = bool(extra.get("force")) target_grid_size_m = int(extra.get("target_grid_size_m", DEFAULT_TARGET_GRID_SIZE_M)) bbox = extra.get("bbox", "") @@ -436,7 +450,7 @@ class Isce2Engine(DinsarEngine): pairs_processed = 0 pairs_failed = 0 - for task_dir in task_dirs: + for pair_index, task_dir in enumerate(task_dirs, start=1): task_name = os.path.basename(os.path.normpath(task_dir)) pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {} task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name @@ -450,8 +464,31 @@ class Isce2Engine(DinsarEngine): wsl_work_dir = windows_path_to_wsl(work_dir, distro=self._distro) wsl_output_dir = windows_path_to_wsl(output_dir, distro=self._distro) wsl_orbit_output_dir = windows_path_to_wsl(orbit_output_dir, distro=self._distro) + emit_progress( + "pair_started", + pair_index=pair_index, + pair_total=total_tasks, + task_name=task_name, + task_alias=task_alias, + pair_key=pair_key, + task_dir=task_dir, + work_dir=work_dir, + output_dir=output_dir, + ) if not wsl_task_dir: pairs_failed += 1 + error_text = f"Unable to convert task dir to WSL path: {task_dir}" + emit_progress( + "pair_finished", + pair_index=pair_index, + pair_total=total_tasks, + task_name=task_name, + task_alias=task_alias, + pair_key=pair_key, + success=False, + returncode=-2, + error=error_text, + ) task_results.append( { "task_name": task_name, @@ -463,7 +500,7 @@ class Isce2Engine(DinsarEngine): "output_dir": output_dir, "success": False, "returncode": -2, - "error": f"Unable to convert task dir to WSL path: {task_dir}", + "error": error_text, "stdout_tail": "", "stderr_tail": "", "command": "", @@ -475,6 +512,18 @@ class Isce2Engine(DinsarEngine): continue if not wsl_work_dir or not wsl_output_dir or not wsl_orbit_output_dir: pairs_failed += 1 + error_text = "Unable to convert ISCE2 work/output paths to WSL paths." + emit_progress( + "pair_finished", + pair_index=pair_index, + pair_total=total_tasks, + task_name=task_name, + task_alias=task_alias, + pair_key=pair_key, + success=False, + returncode=-2, + error=error_text, + ) task_results.append( { "task_name": task_name, @@ -486,7 +535,7 @@ class Isce2Engine(DinsarEngine): "output_dir": output_dir, "success": False, "returncode": -2, - "error": "Unable to convert ISCE2 work/output paths to WSL paths.", + "error": error_text, "stdout_tail": "", "stderr_tail": "", "command": "", @@ -585,6 +634,17 @@ class Isce2Engine(DinsarEngine): else: pairs_failed += 1 + emit_progress( + "pair_finished", + pair_index=pair_index, + pair_total=total_tasks, + task_name=task_name, + task_alias=task_alias, + pair_key=pair_key, + success=success, + returncode=rc, + error=stderr.strip() if stderr else "", + ) task_results.append( { "task_name": task_name, diff --git a/backend/app/dinsar_engines/pyint_engine.py b/backend/app/dinsar_engines/pyint_engine.py new file mode 100644 index 0000000..d856081 --- /dev/null +++ b/backend/app/dinsar_engines/pyint_engine.py @@ -0,0 +1,814 @@ +"""PyINT D-InSAR engine backed by a WSL wrapper pipeline.""" +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List + +from ..config import get_env_text, read_bool_env, settings +from ..services.dinsar_naming import write_run_metadata +from ..services.pyint_input_assets_service import ( + get_pyint_dem_summary, + get_pyint_orbit_context, + materialize_pyint_input_assets, + resolve_pyint_task_input_assets, +) +from ..services.pyint_service import ( + DEFAULT_AZIMUTH_LOOKS, + DEFAULT_PARALLEL_WORKERS, + DEFAULT_RANGE_LOOKS, + MAX_LOOKS, + MAX_PARALLEL_WORKERS, + build_project_name, + check_pyint_environment, + infer_scene_date_from_archives, + infer_task_identity, + quote_shell, + resolve_time_baseline_days, + to_wsl_path, + validate_pyint_root_dir, +) +from ..services.wsl_service import run_wsl_command +from .base import DinsarEngine, EngineAvailability, EngineProfile, RunRequest, RunResult + + +def _read_env(name: str, default: str = "") -> str: + return get_env_text(name, default) or default + + +def _read_bool_env(name: str, default: bool = False) -> bool: + return read_bool_env(name, default) + + +def _windows_path_to_wsl_mount(path: str) -> str: + text = str(path or "").strip() + if not text: + return "" + drive, tail = os.path.splitdrive(os.path.normpath(text)) + if not drive: + return text.replace("\\", "/") + drive_letter = drive.rstrip(":").lower() + normalized_tail = tail.replace("\\", "/") + return f"/mnt/{drive_letter}/{normalized_tail}" + + +class PyintEngine(DinsarEngine): + @property + def engine_code(self) -> str: + return "pyint" + + @property + def engine_label(self) -> str: + return "PyINT / Gamma" + + @property + def default_timeout_seconds(self) -> int: + return max(60, int(settings.PYINT_DEFAULT_TIMEOUT_SECONDS or 43200)) + + @property + def _enabled(self) -> bool: + return _read_bool_env("PYINT_ENABLED", False) + + @property + def _distro(self) -> str: + return _read_env("PYINT_WSL_DISTRO", settings.ISCE2_WSL_DISTRO) + + @property + def _python(self) -> str: + return _read_env("PYINT_WSL_PYTHON", settings.ISCE2_PYTHON) + + @property + def _pyint_home(self) -> str: + return _read_env("PYINT_HOME", "") + + @property + def _pyint_app_script(self) -> str: + explicit = _read_env("PYINT_APP_SCRIPT", "") + if explicit: + return explicit + home = self._pyint_home + if not home: + return "" + return os.path.join(home, "pyint", "pyintApp.py") + + @property + def _template_root(self) -> str: + return _read_env("PYINT_TEMPLATE_ROOT", "") + + @property + def _work_root(self) -> str: + return _read_env("PYINT_WORK_ROOT", "") + + @property + def _output_root(self) -> str: + return _read_env("PYINT_OUTPUT_ROOT", "") + + @property + def _dem_root(self) -> str: + return _read_env("PYINT_DEM_ROOT", "") + + @property + def _dem_mode(self) -> str: + return str(getattr(settings, "PYINT_DEM_MODE", "local_fabdem") or "local_fabdem").strip().lower() + + @property + def _fabdem_root(self) -> str: + return _read_env("PYINT_FABDEM_ROOT", "") + + @property + def _opentopo_dem_type(self) -> str: + return _read_env("PYINT_OPENTOPO_DEM_TYPE", "SRTMGL1") + + @property + def _opentopo_api_key(self) -> str: + return _read_env("PYINT_OPENTOPO_API_KEY", "") + + @property + def _orbit_policy(self) -> str: + return str(getattr(settings, "PYINT_ORBIT_POLICY", "require_txt") or "require_txt").strip().lower() + + @property + def _orbit_pool_txt(self) -> str: + return _read_env("PYINT_ORBIT_POOL_TXT", settings.ORBIT_POOL_ENVI) + + @property + def _record_input_assets(self) -> bool: + return _read_bool_env("PYINT_RECORD_INPUT_ASSETS", True) + + @property + def _gamma_env_script(self) -> str: + return _read_env("PYINT_GAMMA_ENV_SCRIPT", "") + + @property + def _lt1_precise_orbit_enabled(self) -> bool: + return _read_bool_env("PYINT_LT1_PRECISE_ORBIT_ENABLED", True) + + @property + def _lt1_precise_orbit_mode(self) -> str: + return str(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_MODE", "replace") or "replace").strip().lower() + + @property + def _lt1_precise_orbit_strict(self) -> bool: + return _read_bool_env("PYINT_LT1_PRECISE_ORBIT_STRICT", True) + + @property + def _lt1_precise_orbit_validate_with_orb_filt(self) -> bool: + return _read_bool_env("PYINT_LT1_PRECISE_ORBIT_VALIDATE_WITH_ORB_FILT", False) + + @property + def _lt1_precise_orbit_backup(self) -> bool: + return _read_bool_env("PYINT_LT1_PRECISE_ORBIT_BACKUP", True) + + @property + def _lt1_precise_orbit_orb_filt_degree(self) -> int: + return max(1, int(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_ORB_FILT_DEGREE", 5) or 5)) + + @property + def _smoke_test(self) -> bool: + return _read_bool_env("PYINT_SMOKE_TEST_ENABLED", False) + + @property + def _pipeline_script(self) -> str: + local_script = ( + Path(__file__).resolve().parent.parent + / "pyint_pipeline" + / "run_lt1_pyint_pipeline.py" + ) + return _windows_path_to_wsl_mount(str(local_script)) + + def get_profiles(self) -> List[EngineProfile]: + return [ + EngineProfile( + code="lt1_gamma_dinsar", + label="LT-1 Gamma D-InSAR", + description="Use PyINT + Gamma in WSL for single-pair LT-1 D-InSAR processing.", + params_schema={ + "force": { + "label": "强制重跑", + "type": "boolean", + "default": False, + "description": "删除当前 run_key 对应的工作区后重跑。", + }, + "range_looks": { + "label": "距离向多视", + "type": "number", + "default": DEFAULT_RANGE_LOOKS, + "step": 1, + "min": 1, + "max": MAX_LOOKS, + "description": "PyINT 模板中的 range_looks。", + }, + "azimuth_looks": { + "label": "方位向多视", + "type": "number", + "default": DEFAULT_AZIMUTH_LOOKS, + "step": 1, + "min": 1, + "max": MAX_LOOKS, + "description": "PyINT 模板中的 azimuth_looks。", + }, + "parallel_workers": { + "label": "并行数", + "type": "number", + "default": DEFAULT_PARALLEL_WORKERS, + "step": 1, + "min": 1, + "max": MAX_PARALLEL_WORKERS, + "description": "同步控制 raw2slc/coreg/diff/unwrap/geocode 的并行数。", + }, + "unwrap": { + "label": "执行解缠", + "type": "boolean", + "default": True, + "description": "关闭后仅做到差分干涉图，不做解缠。", + }, + "geocode": { + "label": "执行地理编码", + "type": "boolean", + "default": True, + "description": "关闭后不导出地理编码结果。", + }, + }, + ), + ] + + def normalize_extra(self, extra: Dict[str, Any] | None) -> Dict[str, Any]: + normalized: Dict[str, Any] = dict(extra or {}) + + def _coerce_bool(value: Any) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, (int, float)): + return bool(value) + text = str(value or "").strip().lower() + if text in {"1", "true", "yes", "on"}: + return True + if text in {"0", "false", "no", "off", ""}: + return False + return bool(value) + + for key in ("force", "unwrap", "geocode"): + if key in normalized: + normalized[key] = _coerce_bool(normalized[key]) + + for key, maximum, label in ( + ("range_looks", MAX_LOOKS, "距离向多视"), + ("azimuth_looks", MAX_LOOKS, "方位向多视"), + ("parallel_workers", MAX_PARALLEL_WORKERS, "并行数"), + ): + if key not in normalized or normalized[key] is None: + continue + try: + parsed = int(normalized[key]) + except (TypeError, ValueError) as exc: + raise ValueError(f"{label}必须为整数。") from exc + if parsed < 1 or parsed > maximum: + raise ValueError(f"{label}必须在 1 到 {maximum} 之间。") + normalized[key] = parsed + + return normalized + + def validate_root_dir(self, root_dir: str, num_to_process: int = 0) -> Dict[str, Any]: + return validate_pyint_root_dir(root_dir, num_to_process) + + def check_available(self) -> EngineAvailability: + report = check_pyint_environment( + enabled=self._enabled, + distro=self._distro, + python_cmd=self._python, + pyint_home=self._pyint_home, + pyint_app_script=self._pyint_app_script, + template_root=self._template_root, + work_root=self._work_root, + output_root=self._output_root, + dem_root=self._dem_root, + gamma_env_script=self._gamma_env_script, + smoke_test=self._smoke_test, + ) + checks_list = [ + { + "name": check.name, + "ok": check.ok, + "detail": check.detail, + "skipped": check.skipped, + } + for check in report.checks + ] + if report.overall_ok: + status = "ok" + available = True + else: + critical_failed = [check for check in report.checks if not check.ok and not check.skipped] + status = "degraded" if critical_failed else "unavailable" + available = False + return EngineAvailability( + engine_code=self.engine_code, + status=status, + available=available, + checks=checks_list, + message=report.message, + ) + + def run(self, request: RunRequest) -> RunResult: + if not self._enabled: + return RunResult( + success=False, + engine_code=self.engine_code, + profile=request.profile, + job_id=request.job_id, + error="PyINT is disabled.", + ) + + if request.profile != "lt1_gamma_dinsar": + return RunResult( + success=False, + engine_code=self.engine_code, + profile=request.profile, + job_id=request.job_id, + error=f"Unknown profile: {request.profile}", + ) + + return self._run_lt1_gamma_dinsar(request) + + def _run_lt1_gamma_dinsar(self, request: RunRequest) -> RunResult: + extra = self.normalize_extra(request.extra) + validation = self.validate_root_dir(request.root_dir, request.num_to_process) + task_dirs: List[str] = validation["task_dirs"] + total_tasks = len(task_dirs) + run_started_at = datetime.utcnow() + run_started_at_text = run_started_at.isoformat(timespec="seconds") + "Z" + run_key = f"run_{run_started_at.strftime('%Y%m%dT%H%M%SZ')}_{self.engine_code}_{request.profile}" + progress_callback = request.progress_callback + + def emit_progress(event_type: str, **payload: Any) -> None: + if not callable(progress_callback): + return + try: + progress_callback({"event": event_type, **payload}) + except Exception: + return + + timeout = max(60, int(request.timeout_seconds or self.default_timeout_seconds)) + force = bool(extra.get("force")) + range_looks = int(extra.get("range_looks", DEFAULT_RANGE_LOOKS)) + azimuth_looks = int(extra.get("azimuth_looks", DEFAULT_AZIMUTH_LOOKS)) + parallel_workers = int(extra.get("parallel_workers", DEFAULT_PARALLEL_WORKERS)) + unwrap = bool(extra.get("unwrap", True)) + geocode = bool(extra.get("geocode", True)) + + wsl_pyint_home = to_wsl_path(self._pyint_home) + wsl_pyint_app = to_wsl_path(self._pyint_app_script) + wsl_dem_root = to_wsl_path(self._dem_root) + wsl_fabdem_root = to_wsl_path(self._fabdem_root) if self._fabdem_root else "" + wsl_orbit_pool = to_wsl_path(self._orbit_pool_txt) if self._orbit_pool_txt else "" + shared_dem_summary = get_pyint_dem_summary() + prepared_dem_path = str(shared_dem_summary.get("prepared_dem_path") or "").strip() + prepared_dem_kind = str(shared_dem_summary.get("prepared_dem_kind") or "").strip() + wsl_prepared_dem_path = to_wsl_path(prepared_dem_path) if prepared_dem_path else "" + shared_orbit_context = get_pyint_orbit_context() + + task_results: List[Dict[str, Any]] = [] + output_dirs: List[str] = [] + pairs_processed = 0 + pairs_failed = 0 + + for pair_index, task_dir in enumerate(task_dirs, start=1): + task_identity = infer_task_identity(task_dir) + task_name = task_identity["task_name"] + task_alias = task_identity["task_alias"] + pair_key = task_identity["pair_key"] + pair_meta = task_identity["pair_meta"] + master_date = task_identity["master_date"] + slave_date = task_identity["slave_date"] + + work_run_root = os.path.normpath(os.path.join(self._work_root, pair_key, run_key)) + output_dir = os.path.normpath(os.path.join(self._output_root, pair_key, run_key, "native")) + template_root = os.path.normpath(os.path.join(self._template_root, pair_key, run_key)) + project_name = build_project_name(pair_key, run_key) + project_dir = os.path.join(work_run_root, project_name) + input_assets_dir = os.path.join(work_run_root, "input_assets") + + wsl_task_dir = to_wsl_path(task_dir) + wsl_project_dir = to_wsl_path(project_dir) + wsl_output_dir = to_wsl_path(output_dir) + wsl_template_root = to_wsl_path(template_root) + + emit_progress( + "pair_started", + pair_index=pair_index, + pair_total=total_tasks, + task_name=task_name, + task_alias=task_alias, + pair_key=pair_key, + task_dir=task_dir, + work_dir=work_run_root, + output_dir=output_dir, + ) + + if not all((wsl_task_dir, wsl_project_dir, wsl_output_dir, wsl_template_root, wsl_pyint_home, wsl_pyint_app, wsl_dem_root)): + pairs_failed += 1 + error_text = "Unable to convert PyINT paths to WSL paths." + emit_progress( + "pair_finished", + pair_index=pair_index, + pair_total=total_tasks, + task_name=task_name, + task_alias=task_alias, + pair_key=pair_key, + success=False, + returncode=-2, + error=error_text, + ) + task_results.append( + { + "task_name": task_name, + "task_alias": task_alias, + "pair_key": pair_key, + "run_key": run_key, + "task_dir": task_dir, + "work_dir": work_run_root, + "project_dir": project_dir, + "output_dir": output_dir, + "success": False, + "returncode": -2, + "error": error_text, + "stdout_tail": "", + "stderr_tail": "", + "command": "", + "wsl_task_dir": wsl_task_dir, + "wsl_project_dir": wsl_project_dir, + "wsl_output_dir": wsl_output_dir, + } + ) + continue + + archives = self._discover_archives(task_dir) + master_archives = archives.get("master", []) + slave_archives = archives.get("slave", []) + if not master_date: + master_date = infer_scene_date_from_archives(master_archives) + if not slave_date: + slave_date = infer_scene_date_from_archives(slave_archives) + time_baseline_days = resolve_time_baseline_days(master_date, slave_date, pair_meta) + try: + task_input_assets = resolve_pyint_task_input_assets( + task_dir, + dem_summary=shared_dem_summary, + orbit_context=shared_orbit_context, + ) + except Exception as exc: + pairs_failed += 1 + error_text = f"Failed to resolve PyINT input assets: {exc}" + emit_progress( + "pair_finished", + pair_index=pair_index, + pair_total=total_tasks, + task_name=task_name, + task_alias=task_alias, + pair_key=pair_key, + success=False, + returncode=-3, + error=error_text, + ) + task_results.append( + { + "task_name": task_name, + "task_alias": task_alias, + "pair_key": pair_key, + "run_key": run_key, + "task_dir": task_dir, + "work_dir": work_run_root, + "project_dir": project_dir, + "output_dir": output_dir, + "success": False, + "returncode": -3, + "error": error_text, + "stdout_tail": "", + "stderr_tail": "", + "command": "", + "wsl_task_dir": wsl_task_dir, + "wsl_project_dir": wsl_project_dir, + "wsl_output_dir": wsl_output_dir, + } + ) + continue + + if not task_input_assets.get("allow_submit"): + pairs_failed += 1 + error_text = "; ".join(task_input_assets.get("blockers") or []) or "PyINT input assets are incomplete." + emit_progress( + "pair_finished", + pair_index=pair_index, + pair_total=total_tasks, + task_name=task_name, + task_alias=task_alias, + pair_key=pair_key, + success=False, + returncode=-4, + error=error_text, + ) + task_results.append( + { + "task_name": task_name, + "task_alias": task_alias, + "pair_key": pair_key, + "run_key": run_key, + "task_dir": task_dir, + "work_dir": work_run_root, + "project_dir": project_dir, + "output_dir": output_dir, + "success": False, + "returncode": -4, + "error": error_text, + "stdout_tail": "", + "stderr_tail": "", + "command": "", + "input_assets": task_input_assets.get("input_assets"), + "wsl_task_dir": wsl_task_dir, + "wsl_project_dir": wsl_project_dir, + "wsl_output_dir": wsl_output_dir, + } + ) + continue + + try: + materialized_input_assets = materialize_pyint_input_assets( + task_summary=task_input_assets, + input_assets_dir=input_assets_dir, + project_name=project_name, + ) + except Exception as exc: + pairs_failed += 1 + error_text = f"Failed to materialize PyINT input assets: {exc}" + emit_progress( + "pair_finished", + pair_index=pair_index, + pair_total=total_tasks, + task_name=task_name, + task_alias=task_alias, + pair_key=pair_key, + success=False, + returncode=-5, + error=error_text, + ) + task_results.append( + { + "task_name": task_name, + "task_alias": task_alias, + "pair_key": pair_key, + "run_key": run_key, + "task_dir": task_dir, + "work_dir": work_run_root, + "project_dir": project_dir, + "output_dir": output_dir, + "success": False, + "returncode": -5, + "error": error_text, + "stdout_tail": "", + "stderr_tail": "", + "command": "", + "input_assets": task_input_assets.get("input_assets"), + "wsl_task_dir": wsl_task_dir, + "wsl_project_dir": wsl_project_dir, + "wsl_output_dir": wsl_output_dir, + } + ) + continue + + input_assets_summary = materialized_input_assets.get("input_assets") or task_input_assets.get("input_assets") or {} + wsl_input_assets_dir = ( + to_wsl_path(materialized_input_assets.get("input_assets_dir", "")) + if materialized_input_assets.get("input_assets_dir") + else "" + ) + wsl_input_assets_json = ( + to_wsl_path(materialized_input_assets.get("task_manifest_path", "")) + if materialized_input_assets.get("task_manifest_path") + else "" + ) + + cmd_parts = [ + f"{quote_shell(self._python)} {quote_shell(self._pipeline_script)} {quote_shell(wsl_task_dir)}", + f"--project-dir {quote_shell(wsl_project_dir)}", + f"--template-root {quote_shell(wsl_template_root)}", + f"--output-dir {quote_shell(wsl_output_dir)}", + f"--pyint-home {quote_shell(wsl_pyint_home)}", + f"--pyint-app-script {quote_shell(wsl_pyint_app)}", + f"--python {quote_shell(self._python)}", + f"--dem-root {quote_shell(wsl_dem_root)}", + f"--dem-mode {quote_shell(self._dem_mode)}", + f"--project-name {quote_shell(project_name)}", + f"--pair-key {quote_shell(pair_key)}", + f"--task-alias {quote_shell(task_alias)}", + f"--orbit-policy {quote_shell(self._orbit_policy)}", + f"--range-looks {range_looks}", + f"--azimuth-looks {azimuth_looks}", + f"--parallel-workers {parallel_workers}", + f"--master-date {quote_shell(master_date)}" if master_date else "", + f"--slave-date {quote_shell(slave_date)}" if slave_date else "", + f"--time-baseline-days {time_baseline_days}", + f"--input-assets-dir {quote_shell(wsl_input_assets_dir)}" if wsl_input_assets_dir else "", + f"--input-assets-json {quote_shell(wsl_input_assets_json)}" if wsl_input_assets_json else "", + f"--lt1-precise-orbit-enabled {'true' if self._lt1_precise_orbit_enabled else 'false'}", + f"--lt1-precise-orbit-mode {quote_shell(self._lt1_precise_orbit_mode)}", + f"--lt1-precise-orbit-strict {'true' if self._lt1_precise_orbit_strict else 'false'}", + ( + f"--lt1-precise-orbit-validate-with-orb-filt " + f"{'true' if self._lt1_precise_orbit_validate_with_orb_filt else 'false'}" + ), + f"--lt1-precise-orbit-backup {'true' if self._lt1_precise_orbit_backup else 'false'}", + f"--lt1-precise-orbit-orb-filt-degree {self._lt1_precise_orbit_orb_filt_degree}", + "--unwrap" if unwrap else "--no-unwrap", + "--geocode" if geocode else "--no-geocode", + ] + if self._dem_mode == "local_fabdem" and wsl_fabdem_root: + cmd_parts.append(f"--fabdem-root {quote_shell(wsl_fabdem_root)}") + if self._dem_mode == "prepared_file" and wsl_prepared_dem_path: + cmd_parts.append(f"--prepared-dem-path {quote_shell(wsl_prepared_dem_path)}") + if self._dem_mode == "opentopo": + if self._opentopo_dem_type: + cmd_parts.append(f"--opentopo-dem-type {quote_shell(self._opentopo_dem_type)}") + if self._opentopo_api_key: + cmd_parts.append(f"--opentopo-api-key {quote_shell(self._opentopo_api_key)}") + if self._gamma_env_script: + cmd_parts.append(f"--gamma-env-script {quote_shell(to_wsl_path(self._gamma_env_script))}") + if force: + cmd_parts.append("--force") + + cmd = " ".join(part for part in cmd_parts if part) + rc, stdout, stderr = run_wsl_command( + cmd, + distro=self._distro, + timeout=timeout, + ) + + success = rc == 0 + if success: + pairs_processed += 1 + os.makedirs(output_dir, exist_ok=True) + write_run_metadata( + output_dir, + { + "run_key": run_key, + "pair_key": pair_key, + "task_name": task_name, + "task_alias": task_alias, + "engine_code": self.engine_code, + "profile_code": request.profile, + "source_root": os.path.normpath(request.root_dir), + "task_dir": os.path.normpath(task_dir), + "work_dir": work_run_root, + "output_dir": output_dir, + "project_dir": project_dir, + "started_at": run_started_at_text, + "finished_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + "params": { + "force": force, + "range_looks": range_looks, + "azimuth_looks": azimuth_looks, + "parallel_workers": parallel_workers, + "unwrap": unwrap, + "geocode": geocode, + }, + "master_path": pair_meta.get("master_path"), + "slave_path": pair_meta.get("slave_path"), + "master_satellite": task_input_assets.get("master_satellite") or pair_meta.get("master_satellite"), + "slave_satellite": task_input_assets.get("slave_satellite") or pair_meta.get("slave_satellite"), + "master_imaging_date": pair_meta.get("master_imaging_date") or master_date, + "slave_imaging_date": pair_meta.get("slave_imaging_date") or slave_date, + "master_imaging_mode": pair_meta.get("master_imaging_mode"), + "slave_imaging_mode": pair_meta.get("slave_imaging_mode"), + "master_polarization": pair_meta.get("master_polarization"), + "slave_polarization": pair_meta.get("slave_polarization"), + "time_baseline_days": pair_meta.get("time_baseline_days") or time_baseline_days, + "spatial_baseline_meters": pair_meta.get("spatial_baseline_meters"), + "scene_pair_uid": pair_meta.get("scene_pair_uid") or pair_meta.get("pair_uid"), + "pair_uid": pair_meta.get("pair_uid") or pair_meta.get("scene_pair_uid"), + "network_run_id": pair_meta.get("network_run_id"), + "network_edge_id": pair_meta.get("network_edge_id"), + "policy_version": pair_meta.get("policy_version"), + "selection_strategy": pair_meta.get("selection_strategy"), + "input_assets": input_assets_summary, + }, + ) + output_dirs.append(output_dir) + else: + pairs_failed += 1 + + emit_progress( + "pair_finished", + pair_index=pair_index, + pair_total=total_tasks, + task_name=task_name, + task_alias=task_alias, + pair_key=pair_key, + success=success, + returncode=rc, + error=stderr.strip() if stderr else "", + ) + task_results.append( + { + "task_name": task_name, + "task_alias": task_alias, + "pair_key": pair_key, + "run_key": run_key, + "task_dir": task_dir, + "work_dir": work_run_root, + "project_dir": project_dir, + "output_dir": output_dir, + "command": cmd, + "success": success, + "returncode": rc, + "stdout_tail": stdout[-3000:] if stdout else "", + "stderr_tail": stderr[-3000:] if stderr else "", + "error": stderr.strip() if stderr else "", + "wsl_task_dir": wsl_task_dir, + "wsl_project_dir": wsl_project_dir, + "wsl_output_dir": wsl_output_dir, + "wsl_template_root": wsl_template_root, + "master_date": master_date, + "slave_date": slave_date, + "archive_counts": { + "master": len(master_archives), + "slave": len(slave_archives), + }, + "input_assets": input_assets_summary, + "wsl_input_assets_dir": wsl_input_assets_dir, + } + ) + + invalid_candidates = validation.get("invalid_candidates", []) + pairs_failed += len(invalid_candidates) + overall_success = pairs_processed > 0 or (pairs_processed == 0 and pairs_failed == 0) + failed_task_names = [ + item["task_name"] + for item in task_results + if not item.get("success") + ] + [item["name"] for item in invalid_candidates] + + error = None + if not overall_success: + if failed_task_names: + error = f"All PyINT tasks failed: {', '.join(failed_task_names[:10])}" + else: + error = "PyINT run failed." + + last_task_result = task_results[-1] if task_results else {} + return RunResult( + success=overall_success, + engine_code=self.engine_code, + profile=request.profile, + job_id=request.job_id, + pairs_processed=pairs_processed, + pairs_failed=pairs_failed, + output_dirs=output_dirs, + error=error, + detail={ + "mode": validation["mode"], + "task_count": len(task_dirs), + "selected_tasks": [item.get("task_alias") or item.get("task_name") for item in task_results], + "invalid_candidates": invalid_candidates, + "task_results": task_results, + "run_key": run_key, + "started_at": run_started_at_text, + "force": force, + "timeout_seconds": timeout, + "range_looks": range_looks, + "azimuth_looks": azimuth_looks, + "parallel_workers": parallel_workers, + "unwrap": unwrap, + "geocode": geocode, + "command": last_task_result.get("command", ""), + "stdout_tail": last_task_result.get("stdout_tail", ""), + "stderr_tail": last_task_result.get("stderr_tail", ""), + "wsl_task_dir": last_task_result.get("wsl_task_dir", ""), + "wsl_project_dir": last_task_result.get("wsl_project_dir", ""), + "wsl_output_dir": last_task_result.get("wsl_output_dir", ""), + "wsl_template_root": last_task_result.get("wsl_template_root", ""), + "wsl_dem_root": wsl_dem_root, + "wsl_dem": wsl_dem_root, + "wsl_pyint_home": wsl_pyint_home, + "wsl_orbit_pool": wsl_orbit_pool, + "wsl_work_root": to_wsl_path(self._work_root) if self._work_root else "", + "wsl_output_root": to_wsl_path(self._output_root) if self._output_root else "", + "dem_mode": self._dem_mode, + "prepared_dem_path": prepared_dem_path, + "prepared_dem_kind": prepared_dem_kind, + "wsl_prepared_dem_path": wsl_prepared_dem_path, + "orbit_policy": self._orbit_policy, + "lt1_precise_orbit_enabled": self._lt1_precise_orbit_enabled, + "lt1_precise_orbit_mode": self._lt1_precise_orbit_mode, + "lt1_precise_orbit_strict": self._lt1_precise_orbit_strict, + "lt1_precise_orbit_validate_with_orb_filt": self._lt1_precise_orbit_validate_with_orb_filt, + "lt1_precise_orbit_backup": self._lt1_precise_orbit_backup, + "lt1_precise_orbit_orb_filt_degree": self._lt1_precise_orbit_orb_filt_degree, + "record_input_assets": self._record_input_assets, + }, + ) + + @staticmethod + def _discover_archives(task_dir: str) -> Dict[str, List[str]]: + from ..services.pyint_service import discover_lt1_archives + + return discover_lt1_archives(task_dir) diff --git a/backend/app/dinsar_engines/registry.py b/backend/app/dinsar_engines/registry.py index 45d6bac..50703a2 100644 --- a/backend/app/dinsar_engines/registry.py +++ b/backend/app/dinsar_engines/registry.py @@ -31,9 +31,10 @@ def _bootstrap() -> None: from .isce2_engine import Isce2Engine from .landsar_engine import LandsarEngine + from .pyint_engine import PyintEngine from .sarscape_engine import SarscapeEngine - for engine in (SarscapeEngine(), Isce2Engine(), LandsarEngine()): + for engine in (SarscapeEngine(), Isce2Engine(), PyintEngine(), LandsarEngine()): register(engine) diff --git a/backend/app/models/schemas.py b/backend/app/models/schemas.py index 92d304d..300fa0b 100644 --- a/backend/app/models/schemas.py +++ b/backend/app/models/schemas.py @@ -71,10 +71,12 @@ class DinsarResult(BaseModel): task_alias: Optional[str] = None pair_key: Optional[str] = None pair_uid: Optional[str] = None + run_key: Optional[str] = None network_run_id: Optional[str] = None network_edge_id: Optional[int] = None policy_version: Optional[str] = None selection_strategy: Optional[str] = None + engine_code: Optional[str] = None file_path: str min_lon: float min_lat: float diff --git a/backend/app/pyint_pipeline/__init__.py b/backend/app/pyint_pipeline/__init__.py new file mode 100644 index 0000000..342b222 --- /dev/null +++ b/backend/app/pyint_pipeline/__init__.py @@ -0,0 +1 @@ +"""PyINT pipeline helpers.""" diff --git a/backend/app/pyint_pipeline/apply_lt1_precise_orbit.py b/backend/app/pyint_pipeline/apply_lt1_precise_orbit.py new file mode 100644 index 0000000..d4d2325 --- /dev/null +++ b/backend/app/pyint_pipeline/apply_lt1_precise_orbit.py @@ -0,0 +1,575 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import bisect +import json +import math +import os +import re +import shutil +import subprocess +import sys +from dataclasses import dataclass +from datetime import date, datetime, timedelta +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional + + +SCRIPT_DIR = Path(__file__).resolve().parent +ISCE2_PIPELINE_DIR = SCRIPT_DIR.parent / "isce2_pipeline" +if str(ISCE2_PIPELINE_DIR) not in sys.path: + sys.path.insert(0, str(ISCE2_PIPELINE_DIR)) + +from convert_lt1_orbit_to_isce_xml import StateVector, parse_orbit_file # type: ignore + + +TRUE_VALUES = {"1", "true", "yes", "on"} +VECTOR_POS_RE = re.compile(r"^state_vector_position_(\d+):") +VECTOR_VEL_RE = re.compile(r"^state_vector_velocity_(\d+):") + + +@dataclass +class ParsedSlcPar: + path: Path + lines: List[str] + trailing_newline: bool + acquisition_date: date + number_of_state_vectors: int + time_of_first_state_vector: float + state_vector_interval: float + position_line_indexes: Dict[int, int] + velocity_line_indexes: Dict[int, int] + + +def utc_now_text() -> str: + return datetime.utcnow().isoformat(timespec="seconds") + "Z" + + +def read_bool(value: Any, default: bool = False) -> bool: + if value is None: + return default + if isinstance(value, bool): + return value + if isinstance(value, (int, float)): + return bool(value) + return str(value).strip().lower() in TRUE_VALUES + + +def windows_path_to_wsl_mount(path: str) -> str: + text = str(path or "").strip().strip('"').strip("'") + if not text: + return "" + normalized = text.replace("\\", "/") + if normalized.startswith("/"): + return normalized + if normalized.startswith("//"): + return "" + match = re.match(r"^([A-Za-z]):/(.*)$", normalized) + if not match: + return normalized + drive_letter = match.group(1).lower() + tail = match.group(2).lstrip("/") + return f"/mnt/{drive_letter}/{tail}" + + +def resolve_existing_path(path: str) -> Optional[Path]: + text = str(path or "").strip() + if not text: + return None + direct = Path(text) + if direct.exists(): + return direct.resolve() + converted = windows_path_to_wsl_mount(text) + if converted: + candidate = Path(converted) + if candidate.exists(): + return candidate.resolve() + return None + + +def load_json_file(path: Path | None) -> Dict[str, Any]: + if path is None or not path.is_file(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8-sig")) + except Exception: + return {} + return payload if isinstance(payload, dict) else {} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Apply LT-1 precise orbit TXT to Gamma .slc.par state vectors.") + parser.add_argument("--date", required=True, help="Scene date in YYYYMMDD format.") + parser.add_argument("--manifest-json", default=os.getenv("PYINT_LT1_PRECISE_ORBIT_MANIFEST", ""), help="task_manifest.json path.") + parser.add_argument("--summary-json", default="", help="Summary JSON path. Defaults to <slc_dir>/orbit_bridge_summary.json.") + parser.add_argument("--role", choices=("auto", "master", "slave"), default="auto") + parser.add_argument("--operation-tag", default="raw2slc") + parser.add_argument("--mode", default=os.getenv("PYINT_LT1_PRECISE_ORBIT_MODE", "replace")) + parser.add_argument("--slc-par", dest="slc_par_files", action="append", default=[], help="Target .slc.par or .slc.update.par file.") + parser.add_argument("--backup", dest="backup", action="store_true") + parser.add_argument("--no-backup", dest="backup", action="store_false") + parser.add_argument("--strict", dest="strict", action="store_true") + parser.add_argument("--no-strict", dest="strict", action="store_false") + parser.add_argument("--validate-with-orb-filt", dest="validate_with_orb_filt", action="store_true") + parser.add_argument("--no-validate-with-orb-filt", dest="validate_with_orb_filt", action="store_false") + parser.add_argument( + "--orb-filt-degree", + type=int, + default=int(str(os.getenv("PYINT_LT1_PRECISE_ORBIT_ORB_FILT_DEGREE", "5")).strip() or "5"), + ) + parser.set_defaults( + backup=read_bool(os.getenv("PYINT_LT1_PRECISE_ORBIT_BACKUP"), True), + strict=read_bool(os.getenv("PYINT_LT1_PRECISE_ORBIT_STRICT"), True), + validate_with_orb_filt=read_bool(os.getenv("PYINT_LT1_PRECISE_ORBIT_VALIDATE_WITH_ORB_FILT"), False), + ) + return parser.parse_args() + + +def get_orbits_payload(manifest: Dict[str, Any]) -> Dict[str, Any]: + if isinstance(manifest.get("orbits"), dict): + return manifest["orbits"] + input_assets = manifest.get("input_assets") + if isinstance(input_assets, dict) and isinstance(input_assets.get("orbits"), dict): + return input_assets["orbits"] + return {} + + +def resolve_orbit_entry(manifest: Dict[str, Any], date_text: str, role: str) -> Dict[str, Any]: + orbits = get_orbits_payload(manifest) + candidates: List[tuple[str, Dict[str, Any]]] = [] + for role_name in ("master", "slave"): + item = orbits.get(role_name) + if isinstance(item, dict): + candidates.append((role_name, item)) + + if role in {"master", "slave"}: + item = dict(orbits.get(role) or {}) + if not item: + raise RuntimeError(f"Missing orbit entry for role={role}") + item["role"] = role + return item + + matched: List[Dict[str, Any]] = [] + for role_name, item in candidates: + item_date = str(item.get("date") or "").strip() + expected_name = str(item.get("expected_name") or "").strip() + if item_date == date_text or date_text in expected_name: + candidate = dict(item) + candidate["role"] = role_name + matched.append(candidate) + + if len(matched) == 1: + return matched[0] + if not matched: + raise RuntimeError(f"Unable to match precise orbit entry for date={date_text}") + raise RuntimeError(f"Ambiguous precise orbit entries for date={date_text}") + + +def resolve_orbit_txt_path(entry: Dict[str, Any]) -> Path: + for key in ("staged_path", "path"): + candidate = resolve_existing_path(str(entry.get(key) or "")) + if candidate is not None and candidate.is_file(): + return candidate + raise FileNotFoundError( + f"Precise orbit TXT does not exist: expected {entry.get('expected_name') or '<unknown>'}" + ) + + +def parse_float_field(lines: Iterable[str], prefix: str) -> float: + for line in lines: + stripped = line.strip() + if not stripped.startswith(prefix): + continue + _, _, value = stripped.partition(":") + first_token = value.strip().split()[0] + return float(first_token) + raise ValueError(f"Missing field: {prefix}") + + +def parse_int_field(lines: Iterable[str], prefix: str) -> int: + return int(round(parse_float_field(lines, prefix))) + + +def parse_slc_date(lines: Iterable[str]) -> date: + for line in lines: + stripped = line.strip() + if not stripped.startswith("date:"): + continue + match = re.match(r"^date:\s+(\d+)\s+(\d+)\s+(\d+)", stripped) + if not match: + raise ValueError(f"Unable to parse date line: {stripped}") + return date(int(match.group(1)), int(match.group(2)), int(match.group(3))) + raise ValueError("Missing date: field in .slc.par") + + +def parse_slc_par(path: Path) -> ParsedSlcPar: + raw_text = path.read_text(encoding="utf-8", errors="ignore") + trailing_newline = raw_text.endswith("\n") + lines = raw_text.splitlines() + acquisition_date = parse_slc_date(lines) + number_of_state_vectors = parse_int_field(lines, "number_of_state_vectors") + time_of_first_state_vector = parse_float_field(lines, "time_of_first_state_vector") + state_vector_interval = parse_float_field(lines, "state_vector_interval") + + position_line_indexes: Dict[int, int] = {} + velocity_line_indexes: Dict[int, int] = {} + for idx, line in enumerate(lines): + stripped = line.strip() + pos_match = VECTOR_POS_RE.match(stripped) + if pos_match: + position_line_indexes[int(pos_match.group(1))] = idx + continue + vel_match = VECTOR_VEL_RE.match(stripped) + if vel_match: + velocity_line_indexes[int(vel_match.group(1))] = idx + + missing_positions = [index for index in range(1, number_of_state_vectors + 1) if index not in position_line_indexes] + missing_velocities = [index for index in range(1, number_of_state_vectors + 1) if index not in velocity_line_indexes] + if missing_positions or missing_velocities: + raise ValueError( + "Incomplete state vector block in .slc.par: " + f"missing positions={missing_positions[:5]}, missing velocities={missing_velocities[:5]}" + ) + + return ParsedSlcPar( + path=path, + lines=lines, + trailing_newline=trailing_newline, + acquisition_date=acquisition_date, + number_of_state_vectors=number_of_state_vectors, + time_of_first_state_vector=time_of_first_state_vector, + state_vector_interval=state_vector_interval, + position_line_indexes=position_line_indexes, + velocity_line_indexes=velocity_line_indexes, + ) + + +def build_target_times(parsed: ParsedSlcPar) -> List[datetime]: + start_time = datetime(parsed.acquisition_date.year, parsed.acquisition_date.month, parsed.acquisition_date.day) + return [ + start_time + timedelta(seconds=parsed.time_of_first_state_vector + parsed.state_vector_interval * index) + for index in range(parsed.number_of_state_vectors) + ] + + +def norm3(values: Iterable[float]) -> float: + items = [float(item) for item in values] + return math.sqrt(sum(item * item for item in items)) + + +def interpolate_state_vector(target_time: datetime, vectors: List[StateVector]) -> StateVector: + if not vectors: + raise ValueError("No precise orbit vectors available for interpolation") + + times = [vector.time for vector in vectors] + if target_time < times[0] or target_time > times[-1]: + raise ValueError( + f"Target time {target_time.isoformat()} is outside orbit range {times[0].isoformat()} - {times[-1].isoformat()}" + ) + + right_index = bisect.bisect_left(times, target_time) + if right_index < len(vectors) and times[right_index] == target_time: + return vectors[right_index] + if right_index == 0: + return vectors[0] + if right_index >= len(vectors): + return vectors[-1] + + left = vectors[right_index - 1] + right = vectors[right_index] + interval_seconds = (right.time - left.time).total_seconds() + if interval_seconds <= 0: + raise ValueError("Orbit vectors are not strictly increasing in time") + + offset_seconds = (target_time - left.time).total_seconds() + u = offset_seconds / interval_seconds + + h00 = 2 * u * u * u - 3 * u * u + 1 + h10 = u * u * u - 2 * u * u + u + h01 = -2 * u * u * u + 3 * u * u + h11 = u * u * u - u * u + + dh00 = 6 * u * u - 6 * u + dh10 = 3 * u * u - 4 * u + 1 + dh01 = -6 * u * u + 6 * u + dh11 = 3 * u * u - 2 * u + + p0 = (left.x, left.y, left.z) + p1 = (right.x, right.y, right.z) + v0 = (left.vx, left.vy, left.vz) + v1 = (right.vx, right.vy, right.vz) + + position = [] + velocity = [] + for axis in range(3): + pos = ( + h00 * p0[axis] + + h10 * interval_seconds * v0[axis] + + h01 * p1[axis] + + h11 * interval_seconds * v1[axis] + ) + vel = ( + dh00 * p0[axis] + + dh10 * interval_seconds * v0[axis] + + dh01 * p1[axis] + + dh11 * interval_seconds * v1[axis] + ) / interval_seconds + position.append(pos) + velocity.append(vel) + + return StateVector( + time=target_time, + x=position[0], + y=position[1], + z=position[2], + vx=velocity[0], + vy=velocity[1], + vz=velocity[2], + ) + + +def format_position_line(index: int, vector: StateVector) -> str: + return ( + f"state_vector_position_{index}:" + f" {vector.x:14.4f} {vector.y:14.4f} {vector.z:14.4f} m m m" + ) + + +def format_velocity_line(index: int, vector: StateVector) -> str: + return ( + f"state_vector_velocity_{index}:" + f" {vector.vx:13.5f} {vector.vy:13.5f} {vector.vz:13.5f} m/s m/s m/s" + ) + + +def backup_slc_par(path: Path) -> str: + backup_path = path.with_name(path.name + ".orbit_bridge.bak") + if not backup_path.exists(): + shutil.copy2(path, backup_path) + return str(backup_path) + + +def write_bridged_slc_par( + parsed: ParsedSlcPar, + vectors: List[StateVector], + *, + backup_enabled: bool, +) -> Dict[str, Any]: + if len(vectors) != parsed.number_of_state_vectors: + raise ValueError("Interpolated vector count does not match .slc.par state vector count") + + backup_path = "" + if backup_enabled: + backup_path = backup_slc_par(parsed.path) + + updated_lines = list(parsed.lines) + for index, vector in enumerate(vectors, start=1): + updated_lines[parsed.position_line_indexes[index]] = format_position_line(index, vector) + updated_lines[parsed.velocity_line_indexes[index]] = format_velocity_line(index, vector) + + text = "\n".join(updated_lines) + if parsed.trailing_newline: + text += "\n" + parsed.path.write_text(text, encoding="utf-8") + + return { + "backup_path": backup_path, + } + + +def run_orb_filt_validation(path: Path, degree: int) -> Dict[str, Any]: + command = shutil.which("ORB_filt_spline.py") + if not command: + return { + "requested": True, + "ok": False, + "status": "missing_command", + "command": "ORB_filt_spline.py", + } + + validate_path = path.with_name(path.name + ".orb_filt_validate.par") + result = subprocess.run( + [command, str(path), str(validate_path), "--degree", str(int(degree))], + text=True, + capture_output=True, + check=False, + ) + if result.returncode != 0 or not validate_path.exists(): + return { + "requested": True, + "ok": False, + "status": "command_failed", + "command": " ".join(result.args), + "returncode": int(result.returncode), + "stdout": (result.stdout or "")[-2000:], + "stderr": (result.stderr or "")[-2000:], + "output_par": str(validate_path), + } + + current_parsed = parse_slc_par(path) + validated_parsed = parse_slc_par(validate_path) + position_corrections: List[float] = [] + velocity_corrections: List[float] = [] + for index in range(1, current_parsed.number_of_state_vectors + 1): + cur_position = parse_vector_values(current_parsed.lines[current_parsed.position_line_indexes[index]]) + val_position = parse_vector_values(validated_parsed.lines[validated_parsed.position_line_indexes[index]]) + cur_velocity = parse_vector_values(current_parsed.lines[current_parsed.velocity_line_indexes[index]]) + val_velocity = parse_vector_values(validated_parsed.lines[validated_parsed.velocity_line_indexes[index]]) + position_corrections.append(norm3([val_position[i] - cur_position[i] for i in range(3)])) + velocity_corrections.append(norm3([val_velocity[i] - cur_velocity[i] for i in range(3)])) + + return { + "requested": True, + "ok": True, + "status": "ok", + "command": " ".join(result.args), + "returncode": int(result.returncode), + "output_par": str(validate_path), + "max_position_correction_m": max(position_corrections) if position_corrections else 0.0, + "max_velocity_correction_mps": max(velocity_corrections) if velocity_corrections else 0.0, + } + + +def parse_vector_values(line: str) -> List[float]: + _, _, payload = line.partition(":") + values: List[float] = [] + for token in payload.split(): + try: + values.append(float(token)) + except ValueError: + break + if len(values) == 3: + break + if len(values) != 3: + raise ValueError(f"Unable to parse state vector values from line: {line}") + return values + + +def build_operation_record(args: argparse.Namespace, summary_path: Path, manifest_path: Path | None) -> Dict[str, Any]: + return { + "generated_at": utc_now_text(), + "date": str(args.date or "").strip(), + "role": args.role, + "operation_tag": str(args.operation_tag or "").strip(), + "mode": str(args.mode or "").strip(), + "strict": bool(args.strict), + "backup": bool(args.backup), + "validate_with_orb_filt": bool(args.validate_with_orb_filt), + "orb_filt_degree": int(args.orb_filt_degree), + "manifest_json": str(manifest_path) if manifest_path else "", + "summary_json": str(summary_path), + "slc_par_files": [str(path) for path in args.slc_par_files], + "ok": False, + "error": "", + "orbit_source": {}, + "results": [], + } + + +def append_operation_summary(summary_path: Path, operation: Dict[str, Any]) -> None: + existing = load_json_file(summary_path) + operations = existing.get("operations") + if not isinstance(operations, list): + operations = [] + operations.append(operation) + payload = { + "generated_at": existing.get("generated_at") or utc_now_text(), + "last_updated_at": utc_now_text(), + "ok": all(bool(item.get("ok")) for item in operations), + "operation_count": len(operations), + "operations": operations, + } + summary_path.parent.mkdir(parents=True, exist_ok=True) + summary_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def default_summary_path(slc_par_files: List[str]) -> Path: + first_path = Path(slc_par_files[0]).resolve() + return first_path.parent / "orbit_bridge_summary.json" + + +def main() -> int: + args = parse_args() + if not args.slc_par_files: + raise SystemExit("--slc-par must be specified at least once") + + manifest_path = resolve_existing_path(args.manifest_json) if args.manifest_json else None + summary_path = Path(args.summary_json).resolve() if args.summary_json else default_summary_path(args.slc_par_files) + operation = build_operation_record(args, summary_path, manifest_path) + + exit_code = 0 + try: + if manifest_path is None: + raise FileNotFoundError("Precise orbit manifest JSON is not available") + manifest = load_json_file(manifest_path) + orbit_entry = resolve_orbit_entry(manifest, str(args.date or "").strip(), args.role) + orbit_txt_path = resolve_orbit_txt_path(orbit_entry) + orbit_vectors = sorted(parse_orbit_file(orbit_txt_path), key=lambda item: item.time) + operation["orbit_source"] = { + "role": orbit_entry.get("role"), + "satellite": orbit_entry.get("satellite"), + "date": orbit_entry.get("date"), + "expected_name": orbit_entry.get("expected_name"), + "source_txt": str(orbit_txt_path), + "vector_count": len(orbit_vectors), + "time_start": orbit_vectors[0].time.isoformat() if orbit_vectors else "", + "time_stop": orbit_vectors[-1].time.isoformat() if orbit_vectors else "", + } + + results: List[Dict[str, Any]] = [] + for slc_par_text in args.slc_par_files: + slc_par_path = resolve_existing_path(slc_par_text) + if slc_par_path is None or not slc_par_path.is_file(): + raise FileNotFoundError(f"Target .slc.par does not exist: {slc_par_text}") + + parsed = parse_slc_par(slc_par_path) + target_times = build_target_times(parsed) + bridged_vectors = [interpolate_state_vector(target_time, orbit_vectors) for target_time in target_times] + write_info = write_bridged_slc_par(parsed, bridged_vectors, backup_enabled=bool(args.backup)) + validation = ( + run_orb_filt_validation(slc_par_path, args.orb_filt_degree) + if args.validate_with_orb_filt + else {"requested": False, "ok": True, "status": "skipped"} + ) + result_item = { + "path": str(slc_par_path), + "status": "applied", + "ok": bool(validation.get("ok", False)), + "backup_path": write_info.get("backup_path", ""), + "vector_count": parsed.number_of_state_vectors, + "time_of_first_state_vector": parsed.time_of_first_state_vector, + "state_vector_interval": parsed.state_vector_interval, + "validation": validation, + "first_target_time": target_times[0].isoformat() if target_times else "", + "last_target_time": target_times[-1].isoformat() if target_times else "", + "max_position_norm_m": max(norm3((vector.x, vector.y, vector.z)) for vector in bridged_vectors) if bridged_vectors else 0.0, + "max_velocity_norm_mps": max(norm3((vector.vx, vector.vy, vector.vz)) for vector in bridged_vectors) if bridged_vectors else 0.0, + } + results.append(result_item) + + operation["results"] = results + operation["ok"] = all(bool(item.get("ok")) for item in results) + if not operation["ok"]: + operation["error"] = "One or more target .slc.par files failed validation" + if args.strict: + exit_code = 1 + except Exception as exc: + operation["error"] = str(exc) + operation["ok"] = False + exit_code = 1 if args.strict else 0 + + append_operation_summary(summary_path, operation) + if operation.get("error"): + print(operation["error"], file=sys.stderr) + else: + applied_count = len(operation.get("results") or []) + print( + f"Applied LT-1 precise orbit bridge to {applied_count} file(s) for {operation.get('date')} " + f"[{operation.get('operation_tag')}]" + ) + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/app/pyint_pipeline/pyint_gamma_env.sh b/backend/app/pyint_pipeline/pyint_gamma_env.sh new file mode 100644 index 0000000..044b55d --- /dev/null +++ b/backend/app/pyint_pipeline/pyint_gamma_env.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash + +_pyint_gamma_die() { + echo "$1" >&2 + return 1 2>/dev/null || exit 1 +} + +_pyint_gamma_home="" +if [ -n "${PYINT_GAMMA_HOME:-}" ] && [ -d "${PYINT_GAMMA_HOME}" ]; then + _pyint_gamma_home="${PYINT_GAMMA_HOME}" +elif [ -n "${GAMMA_HOME:-}" ] && [ -d "${GAMMA_HOME}" ]; then + _pyint_gamma_home="${GAMMA_HOME}" +else + for _candidate in \ + /usr/local/GAMMA_SOFTWARE-20240627 \ + /usr/local/GAMMA_SOFTWARE-* \ + /opt/GAMMA_SOFTWARE-*; do + [ -d "${_candidate}" ] || continue + _pyint_gamma_home="${_candidate}" + break + done +fi + +[ -n "${_pyint_gamma_home}" ] || _pyint_gamma_die "Gamma home not found." + +export GAMMA_HOME="${_pyint_gamma_home}" +export MSP_HOME="${GAMMA_HOME}/MSP" +export ISP_HOME="${GAMMA_HOME}/ISP" +export DIFF_HOME="${GAMMA_HOME}/DIFF" +export DISP_HOME="${GAMMA_HOME}/DISP" +export LAT_HOME="${GAMMA_HOME}/LAT" +export IPTA_HOME="${GAMMA_HOME}/IPTA" +export GEO_HOME="${GAMMA_HOME}/GEO" + +_pyint_gamma_prepend_path() { + local _dir="$1" + [ -d "${_dir}" ] || return 0 + case ":${PATH}:" in + *":${_dir}:"*) ;; + *) PATH="${_dir}:${PATH}" ;; + esac +} + +for _gamma_dir in \ + "${MSP_HOME}/bin" \ + "${ISP_HOME}/bin" \ + "${DIFF_HOME}/bin" \ + "${DISP_HOME}/bin" \ + "${LAT_HOME}/bin" \ + "${IPTA_HOME}/bin" \ + "${GEO_HOME}/bin" \ + "${MSP_HOME}/scripts" \ + "${ISP_HOME}/scripts" \ + "${DIFF_HOME}/scripts" \ + "${DISP_HOME}/scripts" \ + "${LAT_HOME}/scripts" \ + "${IPTA_HOME}/scripts" \ + "${GEO_HOME}/scripts"; do + _pyint_gamma_prepend_path "${_gamma_dir}" +done + +export PATH +export OS="linux64" +export HDF5_DISABLE_VERSION_CHECK="1" +export GNUTERM="${GNUTERM:-qt}" +export GAMMA_RASTER="${GAMMA_RASTER:-BMP}" +export PYTHONPATH=".:${GAMMA_HOME}${PYTHONPATH:+:${PYTHONPATH}}" + +_pyint_repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +_pyint_script_dir="${_pyint_repo_root}/third_party/PyINT/pyint" +_pyint_gamma_prepend_path "${_pyint_script_dir}" + +unset _pyint_gamma_home +unset _gamma_dir +unset _pyint_repo_root +unset _pyint_script_dir +unset -f _pyint_gamma_prepend_path +unset -f _pyint_gamma_die diff --git a/backend/app/pyint_pipeline/run_lt1_pyint_pipeline.py b/backend/app/pyint_pipeline/run_lt1_pyint_pipeline.py new file mode 100644 index 0000000..5ad00a0 --- /dev/null +++ b/backend/app/pyint_pipeline/run_lt1_pyint_pipeline.py @@ -0,0 +1,847 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import os +import shutil +import stat +import subprocess +import sys +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Iterable, List + + +LT1_INPUT_GLOBS = ("LT1*.tar.gz", "LT1*.tiff") +PAIR_META_FILENAME = ".dinsar_pair.json" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Materialize a PyINT LT-1 workspace from an existing Task_xxx pair directory." + ) + parser.add_argument("task_dir", help="Task directory containing master/ and slave/ subdirectories.") + parser.add_argument("--project-dir", required=True, help="Workspace directory for the generated PyINT project.") + parser.add_argument("--template-root", required=True, help="Directory where the generated template will be written.") + parser.add_argument("--output-dir", required=True, help="Directory where normalized native outputs will be copied.") + parser.add_argument("--pyint-home", required=True, help="PyINT repository root inside WSL.") + parser.add_argument("--pyint-app-script", required=True, help="pyintApp.py path inside WSL.") + parser.add_argument("--python", required=True, help="Python interpreter used to run PyINT inside WSL.") + parser.add_argument("--dem-root", required=True, help="DEMDIR root used by PyINT.") + parser.add_argument("--dem-mode", default="local_fabdem", help="DEM strategy used for this run.") + parser.add_argument("--fabdem-root", default="", help="Optional FABDEM tile root inside WSL.") + parser.add_argument("--prepared-dem-path", default="", help="Optional existing DEM path inside WSL.") + parser.add_argument("--opentopo-dem-type", default="SRTMGL1", help="DEM type when using OpenTopography.") + parser.add_argument("--opentopo-api-key", default="", help="Optional OpenTopography API key.") + parser.add_argument("--project-name", required=True, help="Unique PyINT project name for this run.") + parser.add_argument("--gamma-env-script", default="", help="Optional shell script used to expose GAMMA commands.") + parser.add_argument("--pair-key", default="", help="Pair key recorded into the run summary.") + parser.add_argument("--task-alias", default="", help="Task alias recorded into the run summary.") + parser.add_argument("--orbit-policy", default="require_txt", help="Orbit governance policy recorded into the run summary.") + parser.add_argument("--input-assets-dir", default="", help="Optional input_assets directory for this run.") + parser.add_argument("--input-assets-json", default="", help="Optional task_manifest.json path for this run.") + parser.add_argument("--master-date", default="", help="Master date in YYYYMMDD format.") + parser.add_argument("--slave-date", default="", help="Slave date in YYYYMMDD format.") + parser.add_argument("--time-baseline-days", type=int, default=0, help="Time baseline to record in ifgram_list.txt.") + parser.add_argument("--range-looks", type=int, default=2) + parser.add_argument("--azimuth-looks", type=int, default=2) + parser.add_argument("--parallel-workers", type=int, default=1) + parser.add_argument("--lt1-precise-orbit-enabled", default="true", help="Enable LT-1 precise orbit bridge.") + parser.add_argument("--lt1-precise-orbit-mode", default="replace", help="LT-1 precise orbit bridge mode.") + parser.add_argument("--lt1-precise-orbit-strict", default="true", help="Fail the run if precise orbit bridge fails.") + parser.add_argument( + "--lt1-precise-orbit-validate-with-orb-filt", + default="false", + help="Run ORB_filt_spline.py on a validation copy after rewriting state vectors.", + ) + parser.add_argument("--lt1-precise-orbit-backup", default="true", help="Backup original .slc.par before rewrite.") + parser.add_argument("--lt1-precise-orbit-orb-filt-degree", type=int, default=5) + parser.add_argument("--unwrap", dest="unwrap", action="store_true") + parser.add_argument("--no-unwrap", dest="unwrap", action="store_false") + parser.add_argument("--geocode", dest="geocode", action="store_true") + parser.add_argument("--no-geocode", dest="geocode", action="store_false") + parser.add_argument("--force", action="store_true", help="Delete an existing run root before rebuilding it.") + parser.set_defaults(unwrap=True, geocode=True) + return parser.parse_args() + + +def normalize_date_text(value: Any) -> str: + text = str(value or "").strip() + digits = "".join(ch for ch in text if ch.isdigit()) + if len(digits) >= 8 and digits.startswith("20"): + return digits[:8] + return "" + + +def normalize_bool_text(value: Any, default: bool = False) -> bool: + if value is None: + return default + if isinstance(value, bool): + return value + if isinstance(value, (int, float)): + return bool(value) + return str(value).strip().lower() in {"1", "true", "yes", "on"} + + +def ensure_directory(path: Path) -> Path: + path.mkdir(parents=True, exist_ok=True) + return path + + +def safe_rmtree(path: Path) -> None: + if not path.exists(): + return + resolved = path.resolve() + if len(resolved.parts) < 4: + raise RuntimeError(f"Refusing to remove an unsafe path: {resolved}") + shutil.rmtree(resolved) + + +def load_pair_meta(task_dir: Path) -> Dict[str, Any]: + path = task_dir / PAIR_META_FILENAME + if not path.is_file(): + return {} + try: + return json.loads(path.read_text(encoding="utf-8")) + except Exception: + return {} + + +def load_json_file(path: Path | None) -> Dict[str, Any]: + if path is None or not path.is_file(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8-sig")) + except Exception: + return {} + return payload if isinstance(payload, dict) else {} + + +def discover_lt1_archives(scene_dir: Path) -> List[Path]: + if not scene_dir.is_dir(): + return [] + items: List[Path] = [] + for pattern in LT1_INPUT_GLOBS: + items.extend(path.resolve() for path in scene_dir.rglob(pattern) if path.is_file()) + return sorted(set(items)) + + +def infer_scene_date(paths: Iterable[Path]) -> str: + dates = { + normalize_date_text(path.name) + for path in paths + if normalize_date_text(path.name) + } + if len(dates) == 1: + return next(iter(dates)) + return "" + + +def hardlink_or_copy(src: Path, dst: Path) -> str: + ensure_directory(dst.parent) + if dst.exists(): + return "skipped" + try: + os.link(src, dst) + return "hardlinked" + except OSError: + pass + try: + dst.symlink_to(src) + return "symlinked" + except OSError: + pass + shutil.copy2(src, dst) + return "copied" + + +def collect_related_lt1_input_files(path: Path) -> List[Path]: + resolved = path.resolve() + if resolved.suffix.lower() != ".tiff": + return [resolved] + + stem = resolved.stem + files = [ + candidate.resolve() + for candidate in resolved.parent.iterdir() + if candidate.is_file() and (candidate.name == resolved.name or candidate.name.startswith(stem)) + ] + return sorted(set(files)) + + +def write_text(path: Path, content: str) -> Path: + ensure_directory(path.parent) + path.write_text(content, encoding="utf-8") + return path + + +def inspect_prepared_dem_path(path_text: str) -> Dict[str, str]: + text = str(path_text or "").strip() + if not text: + return { + "path": "", + "kind": "", + "direct_dem_path": "", + "source_dem_path": "", + "source_dem_open_path": "", + } + + path = Path(text) + try: + resolved_path = path.resolve() + except Exception: + resolved_path = path + + gamma_par_path = Path(str(resolved_path) + ".par") + vrt_path = Path(str(resolved_path) + ".vrt") + xml_path = Path(str(resolved_path) + ".xml") + hdr_path = Path(str(resolved_path) + ".hdr") + + if resolved_path.is_file() and gamma_par_path.is_file(): + return { + "path": str(resolved_path), + "kind": "gamma_ready", + "direct_dem_path": str(resolved_path), + "source_dem_path": "", + "source_dem_open_path": "", + } + + if resolved_path.is_file() and (vrt_path.is_file() or xml_path.is_file() or hdr_path.is_file()): + return { + "path": str(resolved_path), + "kind": "source_dem", + "direct_dem_path": "", + "source_dem_path": str(resolved_path), + "source_dem_open_path": str(vrt_path if vrt_path.is_file() else resolved_path), + } + + if resolved_path.suffix.lower() == ".vrt" and resolved_path.is_file(): + return { + "path": str(resolved_path), + "kind": "source_dem", + "direct_dem_path": "", + "source_dem_path": str(resolved_path), + "source_dem_open_path": str(resolved_path), + } + + return { + "path": str(resolved_path), + "kind": "", + "direct_dem_path": "", + "source_dem_path": "", + "source_dem_open_path": "", + } + + +def build_template_text( + *, + project_name: str, + master_date: str, + range_looks: int, + azimuth_looks: int, + parallel_workers: int, + unwrap: bool, + geocode: bool, + dem_mode: str, + fabdem_root: str, + prepared_dem_path: str, + opentopo_dem_type: str, + opentopo_api_key: str, +) -> str: + prepared_dem = inspect_prepared_dem_path(prepared_dem_path) if dem_mode == "prepared_file" else {} + lines = [ + f"# Auto-generated for {project_name}", + "satelite=LT", + f"masterDate={master_date}", + f"range_looks={int(range_looks)}", + f"azimuth_looks={int(azimuth_looks)}", + "download_data=0", + "raw2slc_all=1", + f"raw2slc_all_parallel={int(parallel_workers)}", + "extract_burst_all=0", + f"extract_all_parallel={int(parallel_workers)}", + "coreg_all=1", + f"coreg_all_parallel={int(parallel_workers)}", + "select_pairs=0", + "diff_all=1", + f"diff_all_parallel={int(parallel_workers)}", + "pot_all=0", + f"pot_all_parallel={int(parallel_workers)}", + f"unwrap_all={1 if unwrap else 0}", + f"unwrap_all_parallel={int(parallel_workers)}", + "atmcor_all=0", + f"atmcor_all_parallel={int(parallel_workers)}", + f"geocode_all={1 if geocode else 0}", + f"geocode_all_parallel={int(parallel_workers)}", + "gacos_correction=0", + "load_data=0", + "geocode_products=hyp3,licsbas", + ] + if dem_mode == "local_fabdem" and fabdem_root: + lines.append(f"fabdem_dir={fabdem_root}") + else: + lines.append("fabdem_dir=-") + if dem_mode == "prepared_file" and prepared_dem.get("kind") == "gamma_ready": + lines.append(f"DEM={prepared_dem['direct_dem_path']}") + if dem_mode == "prepared_file" and prepared_dem.get("kind") == "source_dem": + lines.append(f"prepared_dem_source={prepared_dem['source_dem_path']}") + else: + lines.append("prepared_dem_source=-") + if dem_mode == "opentopo": + lines.append(f"opentopo_dem_type={opentopo_dem_type or 'SRTMGL1'}") + lines.append(f"opentopo_api_key={opentopo_api_key or '-'}") + else: + lines.append("opentopo_dem_type=-") + lines.append("opentopo_api_key=-") + return "\n".join(lines) + "\n" + + +def write_ifgram_list(path: Path, master_date: str, slave_date: str, time_baseline_days: int) -> Path: + content = f"{master_date}-{slave_date} {int(time_baseline_days)} 0.0\n" + return write_text(path, content) + + +def write_wrapper_scripts( + *, + wrappers_dir: Path, + pyint_home: Path, + python_cmd: str, + gamma_env_script: str, +) -> List[Path]: + scripts_dir = pyint_home / "pyint" + if not scripts_dir.is_dir(): + raise FileNotFoundError(f"PyINT scripts directory not found: {scripts_dir}") + + ensure_directory(wrappers_dir) + created: List[Path] = [] + pyint_scripts = sorted(path for path in scripts_dir.glob("*.py") if path.is_file()) + for script_path in pyint_scripts: + wrapper_path = wrappers_dir / script_path.name + lines = [ + "#!/usr/bin/env bash", + "set -e", + ] + if gamma_env_script: + lines.append(f". '{gamma_env_script}' >/dev/null 2>&1") + lines.extend( + [ + f"export PATH='{wrappers_dir}':'{scripts_dir}':\"$PATH\"", + f"export PYTHONPATH='{pyint_home}':\"${{PYTHONPATH:-}}\"", + f"exec '{python_cmd}' '{script_path}' \"$@\"", + "", + ] + ) + write_text(wrapper_path, "\n".join(lines)) + wrapper_path.chmod(wrapper_path.stat().st_mode | stat.S_IEXEC) + created.append(wrapper_path) + return created + + +def run_logged(command: List[str], *, env: Dict[str, str], cwd: Path, stdout_path: Path, stderr_path: Path) -> subprocess.CompletedProcess[str]: + result = subprocess.run( + command, + cwd=str(cwd), + env=env, + text=True, + capture_output=True, + check=False, + ) + write_text(stdout_path, result.stdout or "") + write_text(stderr_path, result.stderr or "") + return result + + +def require_task_layout(task_dir: Path) -> None: + missing = [name for name in ("master", "slave") if not (task_dir / name).is_dir()] + if missing: + raise FileNotFoundError(f"Task directory is missing required subdirectories: {', '.join(missing)}") + + +def collect_expected_outputs(project_dir: Path, pair_name: str, range_looks: int) -> Dict[str, str]: + pair_dir = project_dir / "ifgrams" / pair_name + look_text = f"{int(range_looks)}rlks" + return { + "pair_dir": str(pair_dir), + "diff_filt": str(pair_dir / f"{pair_name}_{look_text}.diff_filt"), + "coh": str(pair_dir / f"{pair_name}_{look_text}.diff_filt.cor"), + "unw": str(pair_dir / f"{pair_name}_{look_text}.diff_filt.unw"), + "geo_unw": str(pair_dir / f"geo_{pair_name}_{look_text}.diff_filt.unw"), + "geo_los": str(pair_dir / f"geo_{pair_name}_{look_text}.los_disp"), + } + + +def assert_required_outputs(outputs: Dict[str, str], *, unwrap: bool, geocode: bool) -> None: + required = ["pair_dir", "diff_filt", "coh"] + if unwrap: + required.append("unw") + if geocode: + required.append("geo_unw") + missing = [name for name in required if not Path(outputs[name]).exists()] + if missing: + raise RuntimeError(f"PyINT run finished but required outputs are missing: {', '.join(missing)}") + + +def is_binary_all_zero(path: Path, *, chunk_size: int = 1024 * 1024) -> bool: + if not path.is_file(): + return False + with path.open("rb") as handle: + while True: + chunk = handle.read(chunk_size) + if not chunk: + return True + if any(chunk): + return False + + +def collect_output_sanity_checks( + outputs: Dict[str, str], + *, + unwrap: bool, + geocode: bool, +) -> List[Dict[str, Any]]: + targets = [ + ("diff_filt", "wrapped differential interferogram"), + ("coh", "coherence"), + ] + if unwrap: + targets.append(("unw", "unwrapped interferogram")) + if geocode: + targets.extend( + [ + ("geo_unw", "geocoded unwrapped interferogram"), + ("geo_los", "geocoded LOS displacement"), + ] + ) + + checks: List[Dict[str, Any]] = [] + for name, label in targets: + path = Path(outputs[name]) + exists = path.exists() + size_bytes = path.stat().st_size if exists else 0 + all_zero = exists and is_binary_all_zero(path) + checks.append( + { + "name": name, + "label": label, + "path": str(path), + "exists": exists, + "size_bytes": int(size_bytes), + "all_zero": bool(all_zero), + "ok": bool(exists and size_bytes > 0 and not all_zero), + } + ) + return checks + + +def assert_output_sanity(checks: List[Dict[str, Any]]) -> None: + failed = [item for item in checks if not item.get("ok")] + if not failed: + return + detail = ", ".join(f"{item['name']}={item['path']}" for item in failed) + raise RuntimeError(f"PyINT run produced invalid all-zero binary outputs: {detail}") + + +def collect_stage_error_logs(project_dir: Path) -> Dict[str, str]: + logs: Dict[str, str] = {} + for filename in ( + "coreg_gamma_all.err", + "diff_gamma_all.err", + "unwrap_gamma_all.err", + "geocode_gamma_all.err", + ): + path = project_dir / filename + if path.is_file(): + logs[filename] = str(path) + return logs + + +def copy_native_outputs( + *, + project_dir: Path, + output_dir: Path, + pair_name: str, + template_path: Path, + ifgram_list_path: Path, + stdout_path: Path, + stderr_path: Path, +) -> Dict[str, str]: + ensure_directory(output_dir) + native_pair_dir = project_dir / "ifgrams" / pair_name + target_pair_dir = output_dir / "ifgrams" / pair_name + if native_pair_dir.is_dir(): + shutil.copytree(native_pair_dir, target_pair_dir, dirs_exist_ok=True) + + target_template = output_dir / template_path.name + shutil.copy2(template_path, target_template) + target_ifgram_list = output_dir / ifgram_list_path.name + shutil.copy2(ifgram_list_path, target_ifgram_list) + target_stdout = output_dir / stdout_path.name + target_stderr = output_dir / stderr_path.name + shutil.copy2(stdout_path, target_stdout) + shutil.copy2(stderr_path, target_stderr) + + return { + "pair_dir": str(target_pair_dir), + "template_path": str(target_template), + "ifgram_list_path": str(target_ifgram_list), + "stdout_path": str(target_stdout), + "stderr_path": str(target_stderr), + } + + +def collect_orbit_bridge_summaries(project_dir: Path) -> List[Dict[str, Any]]: + summaries: List[Dict[str, Any]] = [] + slc_root = project_dir / "SLC" + if not slc_root.is_dir(): + return summaries + + for summary_path in sorted(slc_root.glob("*/orbit_bridge_summary.json")): + payload = load_json_file(summary_path) + operations = payload.get("operations") if isinstance(payload.get("operations"), list) else [] + failed_operations = [item for item in operations if not item.get("ok")] + summaries.append( + { + "path": str(summary_path), + "date_dir": summary_path.parent.name, + "ok": bool(payload.get("ok", not failed_operations)), + "operation_count": len(operations), + "failed_operation_count": len(failed_operations), + "operations": operations, + "payload": payload, + } + ) + return summaries + + +def copy_orbit_bridge_summaries(summaries: List[Dict[str, Any]], output_dir: Path) -> Dict[str, str]: + if not summaries: + return {} + + target_dir = ensure_directory(output_dir / "orbit_bridge") + copied: Dict[str, str] = {} + for item in summaries: + source_path = Path(item["path"]) + target_path = target_dir / f"{item['date_dir']}_orbit_bridge_summary.json" + shutil.copy2(source_path, target_path) + copied[item["date_dir"]] = str(target_path) + return copied + + +def assert_orbit_bridge_ok( + *, + enabled: bool, + strict: bool, + summaries: List[Dict[str, Any]], + expected_dates: Iterable[str], +) -> None: + if not enabled: + return + if not strict: + return + + expected = {str(item).strip() for item in expected_dates if str(item).strip()} + found = {str(item.get("date_dir") or "").strip() for item in summaries if str(item.get("date_dir") or "").strip()} + missing = sorted(expected - found) + if missing: + raise RuntimeError(f"LT-1 precise orbit bridge summary is missing for: {', '.join(missing)}") + + failed = [item for item in summaries if not item.get("ok")] + if failed: + failed_dates = ", ".join(sorted(str(item.get("date_dir") or "") for item in failed)) + raise RuntimeError(f"LT-1 precise orbit bridge reported failures for: {failed_dates}") + + +def main() -> int: + args = parse_args() + + task_dir = Path(args.task_dir).resolve() + project_dir = Path(args.project_dir).resolve() + run_root = project_dir.parent + template_root = Path(args.template_root).resolve() + output_dir = Path(args.output_dir).resolve() + pyint_home = Path(args.pyint_home).resolve() + pyint_app_script = Path(args.pyint_app_script).resolve() + dem_root = Path(args.dem_root).resolve() + input_assets_dir = Path(args.input_assets_dir).resolve() if args.input_assets_dir else None + input_assets_json = Path(args.input_assets_json).resolve() if args.input_assets_json else None + input_assets_payload = load_json_file(input_assets_json) + precise_orbit_enabled = normalize_bool_text(args.lt1_precise_orbit_enabled, True) + precise_orbit_strict = normalize_bool_text(args.lt1_precise_orbit_strict, True) + precise_orbit_validate_with_orb_filt = normalize_bool_text( + args.lt1_precise_orbit_validate_with_orb_filt, + False, + ) + precise_orbit_backup = normalize_bool_text(args.lt1_precise_orbit_backup, True) + precise_orbit_mode = str(args.lt1_precise_orbit_mode or "replace").strip().lower() or "replace" + precise_orbit_helper = (Path(__file__).resolve().parent / "apply_lt1_precise_orbit.py").resolve() + dem_mode = str(args.dem_mode or "local_fabdem").strip().lower() or "local_fabdem" + prepared_dem_info = inspect_prepared_dem_path(args.prepared_dem_path) if dem_mode == "prepared_file" else {} + + require_task_layout(task_dir) + if not pyint_app_script.is_file(): + raise FileNotFoundError(f"pyintApp.py not found: {pyint_app_script}") + if precise_orbit_enabled and not precise_orbit_helper.is_file(): + raise FileNotFoundError(f"Precise orbit bridge helper not found: {precise_orbit_helper}") + if precise_orbit_enabled and input_assets_json is None: + raise RuntimeError("LT-1 precise orbit bridge requires --input-assets-json.") + if dem_mode == "prepared_file" and not prepared_dem_info.get("kind"): + raise RuntimeError( + "Prepared DEM mode requires either a Gamma DEM with .par, " + "or a source DEM with .xml/.hdr/.vrt sidecars." + ) + + if args.force: + safe_rmtree(run_root) + safe_rmtree(template_root) + safe_rmtree(output_dir) + + if run_root.exists(): + raise RuntimeError(f"PyINT run root already exists, rerun with --force: {run_root}") + + pair_meta = load_pair_meta(task_dir) + master_archives = discover_lt1_archives(task_dir / "master") + slave_archives = discover_lt1_archives(task_dir / "slave") + if not master_archives: + raise FileNotFoundError(f"No LT1 archives found under: {task_dir / 'master'}") + if not slave_archives: + raise FileNotFoundError(f"No LT1 archives found under: {task_dir / 'slave'}") + + master_date = normalize_date_text(args.master_date) or normalize_date_text(pair_meta.get("master_imaging_date")) or infer_scene_date(master_archives) + slave_date = normalize_date_text(args.slave_date) or normalize_date_text(pair_meta.get("slave_imaging_date")) or infer_scene_date(slave_archives) + if not master_date or not slave_date: + raise RuntimeError("Unable to determine master/slave dates from pair metadata or archive names.") + + pair_name = f"{master_date}-{slave_date}" + task_alias = str(args.task_alias or pair_meta.get("task_alias") or task_dir.name).strip() or task_dir.name + pair_key = str(args.pair_key or pair_meta.get("pair_key") or "").strip() + time_baseline_days = int(args.time_baseline_days or pair_meta.get("time_baseline_days") or 0) + + ensure_directory(run_root) + ensure_directory(template_root) + ensure_directory(output_dir) + ensure_directory(dem_root) + + pyint_scripts_dir = pyint_home / "pyint" + wrappers_dir = ensure_directory(run_root / "wrappers") + write_wrapper_scripts( + wrappers_dir=wrappers_dir, + pyint_home=pyint_home, + python_cmd=args.python, + gamma_env_script=args.gamma_env_script, + ) + + template_path = write_text( + template_root / f"{args.project_name}.template", + build_template_text( + project_name=args.project_name, + master_date=master_date, + range_looks=args.range_looks, + azimuth_looks=args.azimuth_looks, + parallel_workers=args.parallel_workers, + unwrap=bool(args.unwrap), + geocode=bool(args.geocode), + dem_mode=dem_mode, + fabdem_root=str(args.fabdem_root or "").strip(), + prepared_dem_path=str(args.prepared_dem_path or "").strip(), + opentopo_dem_type=str(args.opentopo_dem_type or "SRTMGL1").strip(), + opentopo_api_key=str(args.opentopo_api_key or "").strip(), + ), + ) + + scratch_root = ensure_directory(project_dir.parent) + archive_materialization: List[Dict[str, str]] = [] + env = os.environ.copy() + env.update( + { + "SCRATCHDIR": str(scratch_root), + "TEMPLATEDIR": str(template_root), + "DEMDIR": str(dem_root), + "PATH": f"{wrappers_dir}:{pyint_scripts_dir}:{env.get('PATH', '')}", + "PYTHONPATH": f"{pyint_home}:{env.get('PYTHONPATH', '')}", + "PYINT_LT1_PRECISE_ORBIT_ENABLED": "true" if precise_orbit_enabled else "false", + "PYINT_LT1_PRECISE_ORBIT_MODE": precise_orbit_mode, + "PYINT_LT1_PRECISE_ORBIT_STRICT": "true" if precise_orbit_strict else "false", + "PYINT_LT1_PRECISE_ORBIT_VALIDATE_WITH_ORB_FILT": "true" if precise_orbit_validate_with_orb_filt else "false", + "PYINT_LT1_PRECISE_ORBIT_BACKUP": "true" if precise_orbit_backup else "false", + "PYINT_LT1_PRECISE_ORBIT_ORB_FILT_DEGREE": str(int(args.lt1_precise_orbit_orb_filt_degree)), + "PYINT_LT1_PRECISE_ORBIT_HELPER": str(precise_orbit_helper), + "PYINT_LT1_PRECISE_ORBIT_MANIFEST": str(input_assets_json) if input_assets_json else "", + } + ) + + generate_stdout = run_root / "pyint_generate.stdout.log" + generate_stderr = run_root / "pyint_generate.stderr.log" + generate_result = run_logged( + [str(wrappers_dir / "pyintApp.py"), "-g", args.project_name], + env=env, + cwd=scratch_root, + stdout_path=generate_stdout, + stderr_path=generate_stderr, + ) + if generate_result.returncode != 0: + raise RuntimeError( + f"pyintApp.py -g failed with rc={generate_result.returncode}: " + f"{(generate_result.stderr or generate_result.stdout or '').strip()}" + ) + + pyint_project_dir = project_dir + download_dir = ensure_directory(pyint_project_dir / "DOWNLOAD") + ifgram_list_path = write_ifgram_list(pyint_project_dir / "ifgram_list.txt", master_date, slave_date, time_baseline_days) + for role, archives in (("master", master_archives), ("slave", slave_archives)): + for src_path in archives: + related_files = collect_related_lt1_input_files(src_path) + for related_path in related_files: + target_path = download_dir / related_path.name + op = hardlink_or_copy(related_path, target_path) + archive_materialization.append( + { + "role": role, + "source": str(related_path), + "group_source": str(src_path), + "target": str(target_path), + "operation": op, + } + ) + + run_stdout = run_root / "pyint.stdout.log" + run_stderr = run_root / "pyint.stderr.log" + run_started_at = datetime.utcnow().isoformat(timespec="seconds") + "Z" + run_result = run_logged( + [str(wrappers_dir / "pyintApp.py"), args.project_name], + env=env, + cwd=scratch_root, + stdout_path=run_stdout, + stderr_path=run_stderr, + ) + if run_result.returncode != 0: + stage_error_logs = collect_stage_error_logs(pyint_project_dir) + detail_text = (run_result.stderr or run_result.stdout or "").strip() + if stage_error_logs: + log_text = ", ".join(f"{name}={path}" for name, path in stage_error_logs.items()) + detail_text = f"{detail_text}\nStage logs: {log_text}" if detail_text else f"Stage logs: {log_text}" + raise RuntimeError( + f"pyintApp.py failed with rc={run_result.returncode}: " + f"{detail_text}" + ) + + orbit_bridge_summaries = collect_orbit_bridge_summaries(pyint_project_dir) + assert_orbit_bridge_ok( + enabled=precise_orbit_enabled, + strict=precise_orbit_strict, + summaries=orbit_bridge_summaries, + expected_dates=(master_date, slave_date), + ) + + outputs = collect_expected_outputs(pyint_project_dir, pair_name, args.range_looks) + assert_required_outputs(outputs, unwrap=bool(args.unwrap), geocode=bool(args.geocode)) + output_sanity_checks = collect_output_sanity_checks( + outputs, + unwrap=bool(args.unwrap), + geocode=bool(args.geocode), + ) + assert_output_sanity(output_sanity_checks) + stage_error_logs = collect_stage_error_logs(pyint_project_dir) + + copied_paths = copy_native_outputs( + project_dir=pyint_project_dir, + output_dir=output_dir, + pair_name=pair_name, + template_path=template_path, + ifgram_list_path=ifgram_list_path, + stdout_path=run_stdout, + stderr_path=run_stderr, + ) + copied_orbit_bridge_paths = copy_orbit_bridge_summaries(orbit_bridge_summaries, output_dir) + + summary = { + "ok": True, + "task_dir": str(task_dir), + "task_alias": task_alias, + "pair_key": pair_key, + "project_name": args.project_name, + "project_dir": str(pyint_project_dir), + "run_root": str(run_root), + "template_root": str(template_root), + "output_dir": str(output_dir), + "pyint_home": str(pyint_home), + "pyint_app_script": str(pyint_app_script), + "gamma_env_script": args.gamma_env_script, + "dem": { + "mode": dem_mode, + "dem_root": str(dem_root), + "fabdem_root": str(args.fabdem_root or "").strip(), + "prepared_dem_path": str(args.prepared_dem_path or "").strip(), + "prepared_dem_kind": str(prepared_dem_info.get("kind") or ""), + "prepared_dem_direct_path": str(prepared_dem_info.get("direct_dem_path") or ""), + "prepared_dem_source_path": str(prepared_dem_info.get("source_dem_path") or ""), + "prepared_dem_open_path": str(prepared_dem_info.get("source_dem_open_path") or ""), + "opentopo_dem_type": str(args.opentopo_dem_type or "SRTMGL1").strip(), + "opentopo_api_key_configured": bool(str(args.opentopo_api_key or "").strip()), + }, + "orbit_policy": str(args.orbit_policy or "require_txt").strip().lower(), + "precise_orbit_bridge": { + "enabled": precise_orbit_enabled, + "mode": precise_orbit_mode, + "strict": precise_orbit_strict, + "validate_with_orb_filt": precise_orbit_validate_with_orb_filt, + "backup": precise_orbit_backup, + "orb_filt_degree": int(args.lt1_precise_orbit_orb_filt_degree), + "helper_path": str(precise_orbit_helper), + "manifest_json": str(input_assets_json) if input_assets_json else "", + "summaries": [ + { + "path": item["path"], + "date_dir": item["date_dir"], + "ok": item["ok"], + "operation_count": item["operation_count"], + "failed_operation_count": item["failed_operation_count"], + "copied_summary_path": copied_orbit_bridge_paths.get(item["date_dir"], ""), + } + for item in orbit_bridge_summaries + ], + }, + "input_assets_dir": str(input_assets_dir) if input_assets_dir else "", + "input_assets_json": str(input_assets_json) if input_assets_json else "", + "input_assets": input_assets_payload, + "master_date": master_date, + "slave_date": slave_date, + "pair_name": pair_name, + "time_baseline_days": time_baseline_days, + "range_looks": int(args.range_looks), + "azimuth_looks": int(args.azimuth_looks), + "parallel_workers": int(args.parallel_workers), + "unwrap": bool(args.unwrap), + "geocode": bool(args.geocode), + "archives": { + "master": [str(path) for path in master_archives], + "slave": [str(path) for path in slave_archives], + }, + "archive_materialization": archive_materialization, + "workspace_outputs": outputs, + "output_sanity_checks": output_sanity_checks, + "copied_outputs": copied_paths, + "copied_orbit_bridge_paths": copied_orbit_bridge_paths, + "logs": { + "generate_stdout": str(generate_stdout), + "generate_stderr": str(generate_stderr), + "run_stdout": str(run_stdout), + "run_stderr": str(run_stderr), + "stage_error_logs": stage_error_logs, + }, + "started_at": run_started_at, + "finished_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", + } + + summary_path = output_dir / "pyint_run_summary.json" + write_text(summary_path, json.dumps(summary, ensure_ascii=False, indent=2) + "\n") + print(json.dumps(summary, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as exc: + print(str(exc), file=sys.stderr) + raise diff --git a/backend/app/routers/dinsar.py b/backend/app/routers/dinsar.py index 1e6cfa8..58635ac 100644 --- a/backend/app/routers/dinsar.py +++ b/backend/app/routers/dinsar.py @@ -166,10 +166,12 @@ def _build_dinsar_result_payload(record: DinsarCatalogReadRecord) -> DinsarResul task_alias=product.task_alias, pair_key=product.pair_key, pair_uid=product.pair_uid, + run_key=product.run_key, network_run_id=product.network_run_id, network_edge_id=product.network_edge_id, policy_version=product.policy_version, selection_strategy=product.selection_strategy, + engine_code=product.engine_code, file_path=file_path, min_lon=float(min_lon or 0.0), min_lat=float(min_lat or 0.0), diff --git a/backend/app/routers/dinsar_production.py b/backend/app/routers/dinsar_production.py index f145dcc..ae729c2 100644 --- a/backend/app/routers/dinsar_production.py +++ b/backend/app/routers/dinsar_production.py @@ -13,6 +13,7 @@ from ..config import read_int_env, settings from ..models import AuthUserORM from ..services.dinsar_production_service import dinsar_production_service from ..services.job_queue_service import job_queue_service +from ..services.pyint_input_assets_service import build_pyint_input_preview, summarize_preview_blockers from ..services.task_service import task_service router = APIRouter(prefix="/dinsar-production", tags=["dinsar-production"]) @@ -29,11 +30,17 @@ ISCE2_PRODUCTION_JOB_MAX_ATTEMPTS = read_int_env( minimum=1, maximum=10, ) +PYINT_PRODUCTION_JOB_MAX_ATTEMPTS = read_int_env( + "PYINT_PRODUCTION_JOB_MAX_ATTEMPTS", + 1, + minimum=1, + maximum=10, +) class RunJobRequest(BaseModel): - engine_code: str = Field(..., description="Engine code: sarscape / isce2 / landsar") - profile: str = Field(..., description="Engine profile, for example custom6 / lt1_stripmap") + engine_code: str = Field(..., description="Engine code: sarscape / isce2 / pyint / landsar") + profile: str = Field(..., description="Engine profile, for example custom6 / lt1_stripmap / lt1_gamma_dinsar") root_dir: str = Field(..., description="Windows root directory") num_to_process: int = Field(default=0, ge=0, description="How many tasks to process; 0 means all") timeout_seconds: Optional[int] = Field(default=None, ge=60) @@ -45,6 +52,11 @@ class WslCheckRequest(BaseModel): smoke_test: bool = Field(default=False) +class PreviewInputAssetsRequest(BaseModel): + root_dir: str = Field(..., description="Windows root directory or a single Task_* directory") + num_to_process: int = Field(default=0, ge=0, description="How many tasks to preview; 0 means all") + + def _get_registry(): from ..dinsar_engines import registry @@ -116,6 +128,22 @@ async def run_wsl_check( return report.to_dict() +@router.post("/engines/pyint/preview-input-assets") +async def preview_pyint_input_assets( + req: PreviewInputAssetsRequest, + current_user: AuthUserORM = Depends(_get_current_user), +): + try: + preview = await asyncio.to_thread( + build_pyint_input_preview, + req.root_dir, + req.num_to_process, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return preview + + @router.post("/run") async def submit_run( req: RunJobRequest, @@ -144,7 +172,7 @@ async def submit_run( ) validation_summary = None - if req.engine_code == "isce2" and hasattr(engine, "validate_root_dir"): + if hasattr(engine, "validate_root_dir"): try: validation_summary = await asyncio.to_thread( engine.validate_root_dir, @@ -154,33 +182,69 @@ async def submit_run( except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc + effective_timeout_seconds = req.timeout_seconds + if effective_timeout_seconds is None: + engine_default_timeout = getattr(engine, "default_timeout_seconds", None) + if engine_default_timeout: + effective_timeout_seconds = int(engine_default_timeout) + + pyint_preview = None + if req.engine_code == "pyint": + try: + pyint_preview = await asyncio.to_thread( + build_pyint_input_preview, + req.root_dir, + req.num_to_process, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + if not pyint_preview.get("allow_submit"): + detail = summarize_preview_blockers(pyint_preview) + raise HTTPException( + status_code=400, + detail=f"PyINT 输入资产预检未通过: {detail or '请先修复阻塞项。'}", + ) + if not pyint_preview.get("allow_submit"): + detail = summarize_preview_blockers(pyint_preview) + raise HTTPException( + status_code=400, + detail=f"PyINT 输入资产预检未通过: {detail or '请先修复阻塞项。'}", + ) + payload = { "engine_code": req.engine_code, "profile": req.profile, "root_dir": req.root_dir, "num_to_process": req.num_to_process, - "timeout_seconds": req.timeout_seconds, + "timeout_seconds": effective_timeout_seconds, "extra": dict(req.extra or {}), } - from ..services.job_handlers import JOB_TYPE_IDL_RUN_DINSAR, JOB_TYPE_ISCE2_RUN + from ..services.job_handlers import JOB_TYPE_IDL_RUN_DINSAR, JOB_TYPE_ISCE2_RUN, JOB_TYPE_PYINT_RUN if req.engine_code == "sarscape": payload["mode"] = "custom" if req.profile == "custom6" else "metatask" job_type = JOB_TYPE_IDL_RUN_DINSAR max_attempts = DINSAR_PRODUCTION_JOB_MAX_ATTEMPTS - elif req.engine_code == "isce2": + elif req.engine_code in {"isce2", "pyint"}: if hasattr(engine, "normalize_extra"): try: payload["extra"] = engine.normalize_extra(payload["extra"]) except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc - job_type = JOB_TYPE_ISCE2_RUN - max_attempts = ISCE2_PRODUCTION_JOB_MAX_ATTEMPTS + if req.engine_code == "isce2": + job_type = JOB_TYPE_ISCE2_RUN + max_attempts = ISCE2_PRODUCTION_JOB_MAX_ATTEMPTS + else: + job_type = JOB_TYPE_PYINT_RUN + max_attempts = PYINT_PRODUCTION_JOB_MAX_ATTEMPTS if validation_summary is not None: + validated_task_count = validation_summary.get("task_count", 0) + if pyint_preview is not None: + validated_task_count = int(pyint_preview.get("selected_task_count", validated_task_count) or 0) payload["extra"].update( { - "__validated_task_count": validation_summary.get("task_count", 0), + "__validated_task_count": validated_task_count, "__validated_mode": validation_summary.get("mode", ""), } ) diff --git a/backend/app/routers/unpack.py b/backend/app/routers/unpack.py index 2f8481c..0c00962 100644 --- a/backend/app/routers/unpack.py +++ b/backend/app/routers/unpack.py @@ -1,32 +1,38 @@ from __future__ import annotations -from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException -from sqlalchemy.ext.asyncio import AsyncSession +from typing import Optional + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field -from ..database import get_db from ..models import AuthUserORM from ..services.job_queue_service import job_queue_service from ..services.task_service import task_service -from ..services.unpack_service import get_unpack_config +from ..services.unpack_service import build_unpack_run_config, get_unpack_config from .dependencies import _require_admin router = APIRouter() +class UnpackRunRequest(BaseModel): + max_files_per_run: Optional[int] = Field(default=None, ge=0) + max_runtime_minutes: Optional[int] = Field(default=None, ge=0) + + @router.get("/unpack/config") async def get_unpack_config_endpoint(): - """ - 获取解包配置 (来源 .env)。 - """ return get_unpack_config() @router.post("/unpack/run", status_code=202) -async def run_unpack_endpoint(background_tasks: BackgroundTasks, admin_user: AuthUserORM = Depends(_require_admin)): - """ - 触发一次解包任务。 - """ - config = get_unpack_config() +async def run_unpack_endpoint( + request: Optional[UnpackRunRequest] = None, + admin_user: AuthUserORM = Depends(_require_admin), +): + del admin_user + + overrides = request.model_dump(exclude_none=True) if request else None + config = build_unpack_run_config(overrides) if not config.get("source_dirs"): raise HTTPException(status_code=400, detail="UNPACK_SOURCE_DIRS is not configured.") @@ -36,7 +42,11 @@ async def run_unpack_endpoint(background_tasks: BackgroundTasks, admin_user: Aut "Archive unpack", params=config, ) - await job_queue_service.create_job("UNPACK_ARCHIVES", payload=config, task_id=task_id) + await job_queue_service.create_job( + "UNPACK_ARCHIVES", + payload=config, + task_id=task_id, + ) return {"message": "Unpack task queued", "task_id": task_id} - except ValueError as e: - raise HTTPException(status_code=409, detail=str(e)) + except ValueError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc diff --git a/backend/app/services/job_handlers.py b/backend/app/services/job_handlers.py index 1728cdf..c027248 100644 --- a/backend/app/services/job_handlers.py +++ b/backend/app/services/job_handlers.py @@ -74,6 +74,7 @@ JOB_TYPE_WATER_DETECT = "WATER_DETECT" JOB_TYPE_GF3_PROCESS = "GF3_PROCESS" JOB_TYPE_GF3_BATCH_PROCESS = "GF3_BATCH_PROCESS" JOB_TYPE_ISCE2_RUN = "ISCE2_RUN" +JOB_TYPE_PYINT_RUN = "PYINT_RUN" JOB_TYPE_PUBLISH_DINSAR_PRODUCTS = "PUBLISH_DINSAR_PRODUCTS" JOB_TYPE_REBUILD_DINSAR_CATALOG = "REBUILD_DINSAR_CATALOG" JOB_TYPE_REBUILD_PSINSAR_CATALOG = "REBUILD_PSINSAR_CATALOG" @@ -1878,8 +1879,13 @@ async def _handle_idl_run_dinsar(job: SystemJobORM) -> None: ) -async def _handle_isce2_run(job: SystemJobORM) -> None: - """ISCE2 生产任务 handler — 通过 WSL 执行 run_lt1_dinsar_pipeline.py。""" +async def _handle_queued_engine_run( + job: SystemJobORM, + *, + engine_title: str, + fallback_timeout_seconds: int, +) -> None: + """Run a queued D-InSAR engine task through the shared WSL execution path.""" payload = job.payload or {} engine_code = payload.get("engine_code", "isce2") profile = payload.get("profile", "lt1_stripmap") @@ -1887,17 +1893,26 @@ async def _handle_isce2_run(job: SystemJobORM) -> None: num_to_process = payload.get("num_to_process", 0) timeout_seconds = payload.get("timeout_seconds") extra = payload.get("extra", {}) + selected_task_count = max(1, int(extra.get("__validated_task_count") or 0 or 1)) + pair_timeout_seconds = int(timeout_seconds or fallback_timeout_seconds) await task_service.start_task( job.task_id, - message=f"[{engine_code}/{profile}] 启动 ISCE2 处理...", + message=f"[{engine_code}/{profile}] 启动 {engine_title} 处理...", ) await task_service.add_log( job.task_id, "INFO", - f"ISCE2 job accepted. root_dir={root_dir}, profile={profile}, timeout={timeout_seconds or 21600}s, extra={extra}", + f"{engine_title} job accepted. root_dir={root_dir}, profile={profile}, timeout={pair_timeout_seconds}s, extra={extra}", + ) + await task_service.add_log( + job.task_id, + "INFO", + ( + f"{engine_title} batch contains {selected_task_count} pair task(s). " + f"Pairs run sequentially and each pair uses timeout={pair_timeout_seconds}s." + ), ) - from ..dinsar_engines.base import RunRequest from ..dinsar_engines import registry @@ -1905,6 +1920,25 @@ async def _handle_isce2_run(job: SystemJobORM) -> None: if not engine: raise RuntimeError(f"引擎 '{engine_code}' 未注册") + loop = asyncio.get_running_loop() + progress_queue: asyncio.Queue[Optional[Dict[str, Any]]] = asyncio.Queue() + progress_state: Dict[str, Any] = { + "progress": 5, + "message": f"[{engine_code}/{profile}] Running in WSL...", + "pair_index": 0, + "pair_total": selected_task_count, + "pair_label": "", + "pair_started_monotonic": None, + } + + def _emit_progress(event: Dict[str, Any]) -> None: + if not event: + return + try: + loop.call_soon_threadsafe(progress_queue.put_nowait, dict(event)) + except RuntimeError: + return + request = RunRequest( engine_code=engine_code, profile=profile, @@ -1913,6 +1947,7 @@ async def _handle_isce2_run(job: SystemJobORM) -> None: num_to_process=num_to_process, timeout_seconds=timeout_seconds, extra=extra, + progress_callback=_emit_progress, ) await task_service.update_task( @@ -1921,18 +1956,119 @@ async def _handle_isce2_run(job: SystemJobORM) -> None: message=f"[{engine_code}/{profile}] 正在执行，请等待...", ) + async def _consume_progress() -> None: + while True: + event = await progress_queue.get() + if event is None: + return + + event_type = str(event.get("event") or "").strip().lower() + pair_total = max(1, int(event.get("pair_total") or progress_state["pair_total"] or 1)) + pair_index = max(0, int(event.get("pair_index") or 0)) + task_label = str(event.get("task_alias") or event.get("task_name") or "").strip() + + if event_type == "pair_started": + progress = min( + 90, + max( + int(progress_state["progress"] or 5), + 5 + int((max(pair_index - 1, 0) / pair_total) * 80), + ), + ) + progress_state.update( + { + "progress": progress, + "pair_index": pair_index, + "pair_total": pair_total, + "pair_label": task_label, + "pair_started_monotonic": time.monotonic(), + "message": f"[{engine_code}/{profile}] Running {pair_index}/{pair_total}: {task_label}", + } + ) + await task_service.add_log( + job.task_id, + "INFO", + ( + f"{engine_title} pair {pair_index}/{pair_total} started: {task_label} " + f"(work_dir={event.get('work_dir')})" + ), + ) + await task_service.update_task( + job.task_id, + progress=progress_state["progress"], + message=progress_state["message"], + ) + continue + + if event_type == "pair_finished": + success = bool(event.get("success")) + returncode = int(event.get("returncode") or 0) + progress = min( + 90, + max( + int(progress_state["progress"] or 5), + 5 + int((max(pair_index, 0) / pair_total) * 80) if success else int(progress_state["progress"] or 5), + ), + ) + progress_state.update( + { + "progress": progress, + "pair_index": pair_index, + "pair_total": pair_total, + "pair_label": task_label, + "pair_started_monotonic": None, + } + ) + if success: + progress_state["message"] = f"[{engine_code}/{profile}] Finished {pair_index}/{pair_total}: {task_label}" + await task_service.add_log( + job.task_id, + "INFO", + f"{engine_title} pair {pair_index}/{pair_total} completed: {task_label}", + ) + else: + error_text = str(event.get("error") or "").strip() + timeout_note = " (timeout)" if returncode == -1 else "" + progress_state["message"] = f"[{engine_code}/{profile}] Failed {pair_index}/{pair_total}: {task_label}" + await task_service.add_log( + job.task_id, + "WARNING", + ( + f"{engine_title} pair {pair_index}/{pair_total} failed{timeout_note}: " + f"{task_label} (rc={returncode})" + f"{f', error={error_text}' if error_text else ''}" + ), + ) + if pair_index < pair_total: + await task_service.add_log( + job.task_id, + "WARNING", + f"{engine_title} will continue with the next pair ({pair_index + 1}/{pair_total}).", + ) + await task_service.update_task( + job.task_id, + progress=progress_state["progress"], + message=progress_state["message"], + ) + async def _task_keepalive(): while True: await asyncio.sleep(30) try: + message = str(progress_state.get("message") or f"[{engine_code}/{profile}] Running in WSL...") + started_monotonic = progress_state.get("pair_started_monotonic") + if isinstance(started_monotonic, (int, float)): + elapsed_seconds = max(0, int(time.monotonic() - float(started_monotonic))) + message = f"{message} (elapsed={elapsed_seconds}s)" await task_service.update_task( job.task_id, - progress=5, - message=f"[{engine_code}/{profile}] Running in WSL...", + progress=int(progress_state.get("progress") or 5), + message=message, ) except Exception as exc: print(f"[keepalive] WARNING: failed to update task {job.task_id}: {exc}") + progress_task = asyncio.create_task(_consume_progress()) keepalive_task = asyncio.create_task(_task_keepalive()) try: result = await asyncio.to_thread(engine.run, request) @@ -1942,6 +2078,8 @@ async def _handle_isce2_run(job: SystemJobORM) -> None: await keepalive_task except asyncio.CancelledError: pass + await progress_queue.put(None) + await progress_task detail = result.detail or {} @@ -1949,7 +2087,7 @@ async def _handle_isce2_run(job: SystemJobORM) -> None: await task_service.add_log( job.task_id, "INFO", - f"ISCE2 run mode={detail.get('mode', 'unknown')}, task_count={detail.get('task_count', 0)}", + f"{engine_title} run mode={detail.get('mode', 'unknown')}, task_count={detail.get('task_count', 0)}", ) for invalid in detail.get("invalid_candidates", []) or []: @@ -2062,16 +2200,26 @@ async def _handle_isce2_run(job: SystemJobORM) -> None: job.task_id, "INFO", ( - f"Auto-published ISCE2 results from {len(output_dirs)} directory(s). " + f"Auto-published {engine_title} results from {len(output_dirs)} directory(s). " f"processed={publish_result.get('processed', 0)} " f"issues={rebuild_result.get('issue_count', 0) if rebuild_result else 0}" ), ) + if int(publish_result.get("processed", 0) or 0) <= 0: + await task_service.add_log( + job.task_id, + "WARNING", + ( + f"No publishable {engine_title} result bundle was detected under " + f"{len(output_dirs)} output director" + f"{'y' if len(output_dirs) == 1 else 'ies'}." + ), + ) except Exception as exc: await task_service.add_log( job.task_id, "WARNING", - f"Auto-publish ISCE2 results failed: {exc}", + f"Auto-publish {engine_title} results failed: {exc}", ) if result.success: @@ -2080,7 +2228,7 @@ async def _handle_isce2_run(job: SystemJobORM) -> None: status="COMPLETED", progress=100, message=( - f"[{engine_code}/{profile}] 完成 — " + f"[{engine_code}/{profile}] 完成，" f"成功 {result.pairs_processed} 对，失败 {result.pairs_failed} 对" ), ) @@ -2095,6 +2243,22 @@ async def _handle_isce2_run(job: SystemJobORM) -> None: ) +async def _handle_isce2_run(job: SystemJobORM) -> None: + await _handle_queued_engine_run( + job, + engine_title="ISCE2", + fallback_timeout_seconds=settings.ISCE2_PER_TASK_TIMEOUT_SECONDS, + ) + + +async def _handle_pyint_run(job: SystemJobORM) -> None: + await _handle_queued_engine_run( + job, + engine_title="PyINT", + fallback_timeout_seconds=settings.PYINT_DEFAULT_TIMEOUT_SECONDS, + ) + + async def _handle_water_geocode(job: SystemJobORM) -> None: """单景 SAR 地理编码 job handler（多视 + 地理编码 + 辐射定标）。""" from .water_service import run_geocoding_workflow, WATER_RESULTS_DIR @@ -2860,6 +3024,7 @@ _HANDLERS = { JOB_TYPE_IDL_RUN_IMPORT: _handle_idl_run_import, JOB_TYPE_IDL_RUN_DINSAR: _handle_idl_run_dinsar, JOB_TYPE_ISCE2_RUN: _handle_isce2_run, + JOB_TYPE_PYINT_RUN: _handle_pyint_run, JOB_TYPE_WATER_GEOCODE: _handle_water_geocode, JOB_TYPE_WATER_FLOOD: _handle_water_flood, JOB_TYPE_WATER_DETECT: _handle_water_detect, diff --git a/backend/app/services/pyint_input_assets_service.py b/backend/app/services/pyint_input_assets_service.py new file mode 100644 index 0000000..2dbe603 --- /dev/null +++ b/backend/app/services/pyint_input_assets_service.py @@ -0,0 +1,729 @@ +"""PyINT input-asset resolution and materialization helpers.""" +from __future__ import annotations + +import json +import os +import shutil +from datetime import datetime +from typing import Any, Dict, List + +from ..config import settings +from .orbit_converter import get_source_orbit_inventory +from .pyint_service import ( + discover_lt1_archives, + infer_scene_date_from_archives, + infer_task_identity, + validate_pyint_root_dir, +) + + +VALID_DEM_MODES = {"local_fabdem", "opentopo", "prepared_file"} +VALID_ORBIT_POLICIES = {"validate_only", "require_txt", "stage_txt"} +VALID_PRECISE_ORBIT_MODES = {"replace", "replace_and_validate"} + + +def _utc_now_text() -> str: + return datetime.utcnow().isoformat(timespec="seconds") + "Z" + + +def _normalize_path(path: Any) -> str: + text = str(path or "").strip().strip('"').strip("'") + if not text: + return "" + return os.path.normpath(os.path.abspath(text)) + + +def _copy_json_safe(value: Any) -> Any: + return json.loads(json.dumps(value, ensure_ascii=False, default=str)) + + +def _normalize_lt1_satellite(value: Any) -> str: + text = str(value or "").strip().upper().replace("-", "").replace("_", "") + if "LT1A" in text: + return "LT1A" + if "LT1B" in text: + return "LT1B" + if text in {"A", "LTA"}: + return "LT1A" + if text in {"B", "LTB"}: + return "LT1B" + return "" + + +def _infer_satellite_from_archives(paths: List[str]) -> str: + satellites = { + satellite + for path in paths + for satellite in [_normalize_lt1_satellite(os.path.basename(path))] + if satellite + } + if len(satellites) == 1: + return next(iter(satellites)) + return "" + + +def _get_dem_mode() -> str: + raw_mode = str(getattr(settings, "PYINT_DEM_MODE", "local_fabdem") or "local_fabdem").strip().lower() + if raw_mode not in VALID_DEM_MODES: + return "local_fabdem" + return raw_mode + + +def _get_orbit_policy() -> str: + raw_policy = str(getattr(settings, "PYINT_ORBIT_POLICY", "require_txt") or "require_txt").strip().lower() + if raw_policy not in VALID_ORBIT_POLICIES: + return "require_txt" + return raw_policy + + +def _get_orbit_pool_root() -> str: + explicit = _normalize_path(getattr(settings, "PYINT_ORBIT_POOL_TXT", "")) + if explicit: + return explicit + return _normalize_path(settings.ORBIT_POOL_ENVI) + + +def get_pyint_precise_orbit_bridge_summary() -> Dict[str, Any]: + mode = str(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_MODE", "replace") or "replace").strip().lower() + if mode not in VALID_PRECISE_ORBIT_MODES: + mode = "replace" + return { + "enabled": bool(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_ENABLED", True)), + "mode": mode, + "strict": bool(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_STRICT", True)), + "validate_with_orb_filt": bool(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_VALIDATE_WITH_ORB_FILT", False)), + "backup": bool(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_BACKUP", True)), + "orb_filt_degree": max(1, int(getattr(settings, "PYINT_LT1_PRECISE_ORBIT_ORB_FILT_DEGREE", 5) or 5)), + } + + +def _prepared_dem_variants(value: Any) -> List[str]: + text = str(value or "").strip().strip('"').strip("'") + if not text: + return [] + + normalized = _normalize_path(text) + if not normalized: + return [] + + candidates = [normalized] + root, ext = os.path.splitext(normalized) + if ext.lower() == ".wgs84": + candidates.append(root) + elif not ext: + candidates.append(normalized + ".wgs84") + + unique: List[str] = [] + seen: set[str] = set() + for candidate in candidates: + item = _normalize_path(candidate) + if not item or item in seen: + continue + seen.add(item) + unique.append(item) + return unique + + +def _resolve_prepared_dem_path() -> Dict[str, str]: + explicit_value = getattr(settings, "PYINT_PREPARED_DEM_PATH", "") + explicit_candidates = _prepared_dem_variants(explicit_value) + if str(explicit_value or "").strip(): + for candidate in explicit_candidates: + if os.path.isfile(candidate): + return { + "path": candidate, + "resolved_from": "explicit", + } + return { + "path": "", + "resolved_from": "explicit", + } + + sources = [ + ("isce2_dem_path", getattr(settings, "ISCE2_DEM_PATH", "")), + ("idl_dinsar_dem_base_file", getattr(settings, "IDL_DINSAR_DEM_BASE_FILE", "")), + ] + for source_name, raw_value in sources: + for candidate in _prepared_dem_variants(raw_value): + if os.path.isfile(candidate): + return { + "path": candidate, + "resolved_from": source_name, + } + return { + "path": "", + "resolved_from": "", + } + + +def _inspect_prepared_dem_path(path: Any) -> Dict[str, Any]: + normalized = _normalize_path(path) + if not normalized: + return { + "path": "", + "exists": False, + "kind": "", + "gamma_par_path": "", + "gamma_par_exists": False, + "xml_path": "", + "xml_exists": False, + "hdr_path": "", + "hdr_exists": False, + "vrt_path": "", + "vrt_exists": False, + "open_path": "", + } + + gamma_par_path = normalized + ".par" + xml_path = normalized + ".xml" + hdr_path = normalized + ".hdr" + vrt_path = normalized + ".vrt" + + path_exists = os.path.isfile(normalized) + gamma_par_exists = os.path.isfile(gamma_par_path) + xml_exists = os.path.isfile(xml_path) + hdr_exists = os.path.isfile(hdr_path) + vrt_exists = os.path.isfile(vrt_path) + + kind = "" + open_path = "" + if path_exists and gamma_par_exists: + kind = "gamma_ready" + open_path = normalized + elif path_exists and (xml_exists or hdr_exists or vrt_exists): + kind = "source_dem" + open_path = vrt_path if vrt_exists else normalized + + return { + "path": normalized, + "exists": path_exists, + "kind": kind, + "gamma_par_path": gamma_par_path, + "gamma_par_exists": gamma_par_exists, + "xml_path": xml_path, + "xml_exists": xml_exists, + "hdr_path": hdr_path, + "hdr_exists": hdr_exists, + "vrt_path": vrt_path, + "vrt_exists": vrt_exists, + "open_path": open_path, + } + + +def get_pyint_dem_summary() -> Dict[str, Any]: + mode = _get_dem_mode() + strict = bool(getattr(settings, "PYINT_DEM_STRICT", True)) + cache_root = _normalize_path(settings.PYINT_DEM_ROOT) + fabdem_root = _normalize_path(getattr(settings, "PYINT_FABDEM_ROOT", "")) + prepared_dem_resolution = _resolve_prepared_dem_path() + prepared_dem_info = _inspect_prepared_dem_path(prepared_dem_resolution.get("path")) + opentopo_dem_type = str(getattr(settings, "PYINT_OPENTOPO_DEM_TYPE", "SRTMGL1") or "SRTMGL1").strip() or "SRTMGL1" + opentopo_api_key = str(getattr(settings, "PYINT_OPENTOPO_API_KEY", "") or "").strip() + + warnings: List[str] = [] + blockers: List[str] = [] + + source_root = "" + source_exists = False + if mode == "local_fabdem": + source_root = fabdem_root + source_exists = bool(source_root and os.path.isdir(source_root)) + elif mode == "prepared_file": + source_root = str(prepared_dem_info.get("path") or "") + source_exists = bool(prepared_dem_info.get("exists")) + cache_root_exists = bool(cache_root and os.path.isdir(cache_root)) + + if mode == "local_fabdem": + if not fabdem_root: + message = "未配置 PYINT_FABDEM_ROOT。" + if strict: + blockers.append(message) + else: + warnings.append(message) + elif not os.path.isdir(fabdem_root): + message = f"本地 FABDEM 根目录不存在: {fabdem_root}" + if strict: + blockers.append(message) + else: + warnings.append(message) + elif mode == "opentopo": + if not opentopo_api_key: + message = "DEM 策略为 OpenTopography，但未配置 PYINT_OPENTOPO_API_KEY。" + if strict: + blockers.append(message) + else: + warnings.append(message) + elif mode == "prepared_file": + if not prepared_dem_info.get("path"): + if prepared_dem_resolution.get("resolved_from") == "explicit": + message = "PYINT_PREPARED_DEM_PATH 已配置，但目标文件不存在。" + else: + message = ( + "未配置 PYINT_PREPARED_DEM_PATH，且未能从 ISCE2_DEM_PATH / " + "IDL_DINSAR_DEM_BASE_FILE 解析现有 DEM。" + ) + if strict: + blockers.append(message) + else: + warnings.append(message) + elif prepared_dem_info.get("kind") not in {"gamma_ready", "source_dem"}: + message = ( + "现有 DEM 缺少可识别 sidecar，至少需要同名 .par，或 .xml/.hdr/.vrt 中的一个: " + + str(prepared_dem_info.get("path") or "") + ) + if strict: + blockers.append(message) + else: + warnings.append(message) + + if not cache_root: + blockers.append("未配置 PYINT_DEM_ROOT。") + elif not cache_root_exists: + warnings.append(f"DEM 缓存目录当前不存在，运行时将尝试创建: {cache_root}") + + status = "ok" + if blockers: + status = "blocked" + elif warnings: + status = "warning" + + if mode == "local_fabdem": + detail = "使用本地 FABDEM 瓦片目录，由 PyINT 在 DEMDIR 中生成运行期 DEM。" + elif mode == "prepared_file": + if prepared_dem_info.get("kind") == "gamma_ready": + detail = "使用现有 Gamma DEM，运行时将直接注入到 PyINT 模板。" + else: + detail = "使用现有系统 DEM，运行时将按任务覆盖区裁剪并转换为本次任务的 Gamma DEM。" + else: + detail = f"使用 OpenTopography 在线 DEM 源，DEM 类型为 {opentopo_dem_type}。" + + return { + "mode": mode, + "strict": strict, + "source_root": source_root, + "source_exists": source_exists, + "cache_root": cache_root, + "cache_root_exists": cache_root_exists, + "fabdem_root": fabdem_root, + "prepared_dem_path": str(prepared_dem_info.get("path") or ""), + "prepared_dem_resolved_from": str(prepared_dem_resolution.get("resolved_from") or ""), + "prepared_dem_kind": str(prepared_dem_info.get("kind") or ""), + "prepared_dem_open_path": str(prepared_dem_info.get("open_path") or ""), + "prepared_dem_support": { + "gamma_par_exists": bool(prepared_dem_info.get("gamma_par_exists")), + "xml_exists": bool(prepared_dem_info.get("xml_exists")), + "hdr_exists": bool(prepared_dem_info.get("hdr_exists")), + "vrt_exists": bool(prepared_dem_info.get("vrt_exists")), + }, + "opentopo_dem_type": opentopo_dem_type, + "opentopo_api_key_configured": bool(opentopo_api_key), + "status": status, + "detail": detail, + "warnings": warnings, + "blockers": blockers, + "allow_submit": not blockers, + } + + +def _load_orbit_inventory() -> Dict[str, Any]: + pool_root = _get_orbit_pool_root() + if not pool_root: + return { + "pool_root": "", + "pool_exists": False, + "files": {}, + "warnings": ["未配置 PYINT_ORBIT_POOL_TXT，且 ORBIT_POOL_ENVI 为空。"], + } + if not os.path.isdir(pool_root): + return { + "pool_root": pool_root, + "pool_exists": False, + "files": {}, + "warnings": [f"轨道池目录不存在: {pool_root}"], + } + + inventory = get_source_orbit_inventory(pool_root, recursive=True) + return { + "pool_root": pool_root, + "pool_exists": True, + "files": inventory.get("files", {}), + "warnings": list(inventory.get("errors", []) or []), + "duplicate_count": int(inventory.get("duplicate_count", 0) or 0), + } + + +def get_pyint_orbit_context() -> Dict[str, Any]: + return _load_orbit_inventory() + + +def _resolve_orbit_file( + *, + role: str, + satellite: str, + date_text: str, + pool_root: str, + orbit_files: Dict[str, Dict[str, Any]], +) -> Dict[str, Any]: + satellite_text = _normalize_lt1_satellite(satellite) + normalized_date = str(date_text or "").strip() + expected_name = ( + f"{satellite_text}_GpsData_GAS_C_{normalized_date}.txt" + if satellite_text and normalized_date + else "" + ) + result: Dict[str, Any] = { + "role": role, + "satellite": satellite_text, + "date": normalized_date, + "expected_name": expected_name, + "pool_root": pool_root, + "resolved": False, + "path": "", + "resolution_method": "", + "staged_path": "", + } + if not satellite_text: + result["error"] = f"{role} 场景未能识别 LT-1 卫星型号。" + return result + if not normalized_date: + result["error"] = f"{role} 场景未能识别成像日期。" + return result + + stem = os.path.splitext(expected_name)[0] + item = orbit_files.get(stem) + if item and os.path.isfile(item.get("path", "")): + result.update( + { + "resolved": True, + "path": _normalize_path(item["path"]), + "resolution_method": "indexed_pool_scan", + } + ) + return result + + direct_candidate = os.path.join(pool_root, satellite_text, expected_name) + if os.path.isfile(direct_candidate): + result.update( + { + "resolved": True, + "path": _normalize_path(direct_candidate), + "resolution_method": "direct_satellite_subdir", + } + ) + return result + + flat_candidate = os.path.join(pool_root, expected_name) + if os.path.isfile(flat_candidate): + result.update( + { + "resolved": True, + "path": _normalize_path(flat_candidate), + "resolution_method": "direct_pool_root", + } + ) + return result + + result["error"] = f"轨道池中缺少 {expected_name}" + return result + + +def resolve_pyint_task_input_assets( + task_dir: str, + *, + dem_summary: Dict[str, Any] | None = None, + orbit_context: Dict[str, Any] | None = None, +) -> Dict[str, Any]: + task_dir = _normalize_path(task_dir) + task_identity = infer_task_identity(task_dir) + pair_meta = task_identity["pair_meta"] + archives = discover_lt1_archives(task_dir) + master_archives = list(archives.get("master", []) or []) + slave_archives = list(archives.get("slave", []) or []) + + warnings: List[str] = [] + blockers: List[str] = [] + + master_date = task_identity["master_date"] or infer_scene_date_from_archives(master_archives) + slave_date = task_identity["slave_date"] or infer_scene_date_from_archives(slave_archives) + + master_satellite = _normalize_lt1_satellite(pair_meta.get("master_satellite")) or _infer_satellite_from_archives(master_archives) + slave_satellite = _normalize_lt1_satellite(pair_meta.get("slave_satellite")) or _infer_satellite_from_archives(slave_archives) + + if not master_archives: + blockers.append("master/ 下未发现 LT-1 原始输入（LT1*.tar.gz 或 LT1*.tiff）。") + if not slave_archives: + blockers.append("slave/ 下未发现 LT-1 原始输入（LT1*.tar.gz 或 LT1*.tiff）。") + if not master_date: + blockers.append("未能识别主影像日期。") + if not slave_date: + blockers.append("未能识别从影像日期。") + + orbit_policy = _get_orbit_policy() + orbit_context = orbit_context or get_pyint_orbit_context() + orbit_pool_root = orbit_context.get("pool_root", "") + orbit_pool_exists = bool(orbit_context.get("pool_exists")) + orbit_files = orbit_context.get("files", {}) or {} + + orbit_warnings: List[str] = [] + if orbit_context.get("warnings"): + orbit_warnings.extend(str(item) for item in orbit_context["warnings"] if item) + + master_orbit = _resolve_orbit_file( + role="master", + satellite=master_satellite, + date_text=master_date, + pool_root=orbit_pool_root, + orbit_files=orbit_files, + ) + slave_orbit = _resolve_orbit_file( + role="slave", + satellite=slave_satellite, + date_text=slave_date, + pool_root=orbit_pool_root, + orbit_files=orbit_files, + ) + + for orbit_item in (master_orbit, slave_orbit): + if orbit_item.get("resolved"): + continue + message = str(orbit_item.get("error") or f"{orbit_item.get('role')} 轨道缺失").strip() + if orbit_policy == "validate_only": + orbit_warnings.append(message) + else: + blockers.append(message) + + if not orbit_pool_root: + if orbit_policy == "validate_only": + orbit_warnings.append("轨道池未配置，当前仅记录警告。") + else: + blockers.append("轨道池未配置。") + elif not orbit_pool_exists: + if orbit_policy == "validate_only": + orbit_warnings.append(f"轨道池目录不可用: {orbit_pool_root}") + else: + blockers.append(f"轨道池目录不可用: {orbit_pool_root}") + + warnings.extend(orbit_warnings) + + task_source = { + "task_dir": task_dir, + "task_name": task_identity["task_name"], + "task_alias": task_identity["task_alias"], + "pair_key": task_identity["pair_key"], + "master_date": master_date, + "slave_date": slave_date, + "master_satellite": master_satellite, + "slave_satellite": slave_satellite, + "archives": { + "master": master_archives, + "slave": slave_archives, + }, + } + + precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary() + orbits_summary = { + "policy": orbit_policy, + "pool_root": orbit_pool_root, + "pool_exists": orbit_pool_exists, + "master": master_orbit, + "slave": slave_orbit, + "resolved_count": int(bool(master_orbit.get("resolved"))) + int(bool(slave_orbit.get("resolved"))), + "missing_count": int(not master_orbit.get("resolved")) + int(not slave_orbit.get("resolved")), + "warnings": orbit_warnings, + "stage_mode": "copy" if orbit_policy == "stage_txt" or precise_orbit_bridge.get("enabled") else "none", + "precise_orbit_bridge": precise_orbit_bridge, + } + + dem_payload = _copy_json_safe(dem_summary or get_pyint_dem_summary()) + allow_submit = not blockers and bool(dem_payload.get("allow_submit", True)) + + return { + "task_name": task_identity["task_name"], + "task_alias": task_identity["task_alias"], + "pair_key": task_identity["pair_key"], + "task_dir": task_dir, + "master_date": master_date, + "slave_date": slave_date, + "master_satellite": master_satellite, + "slave_satellite": slave_satellite, + "archive_counts": { + "master": len(master_archives), + "slave": len(slave_archives), + }, + "warnings": warnings, + "blockers": blockers, + "allow_submit": allow_submit, + "task_source": task_source, + "dem": dem_payload, + "orbit_resolution": { + "master": master_orbit, + "slave": slave_orbit, + }, + "input_assets": { + "task_source": task_source, + "dem": dem_payload, + "orbits": orbits_summary, + }, + } + + +def build_pyint_input_preview(root_dir: str, num_to_process: int = 0) -> Dict[str, Any]: + validation = validate_pyint_root_dir(root_dir, num_to_process) + dem_summary = get_pyint_dem_summary() + orbit_context = get_pyint_orbit_context() + + warnings: List[str] = list(dem_summary.get("warnings") or []) + blockers: List[str] = list(dem_summary.get("blockers") or []) + task_summaries: List[Dict[str, Any]] = [] + resolved_task_count = 0 + missing_task_count = 0 + + for task_dir in validation.get("task_dirs", []) or []: + task_summary = resolve_pyint_task_input_assets( + task_dir, + dem_summary=dem_summary, + orbit_context=orbit_context, + ) + task_summaries.append(task_summary) + if task_summary.get("warnings"): + warnings.extend( + f"{task_summary['task_alias']}: {item}" + for item in task_summary["warnings"] + ) + if task_summary.get("blockers"): + blockers.extend( + f"{task_summary['task_alias']}: {item}" + for item in task_summary["blockers"] + ) + if task_summary["input_assets"]["orbits"]["missing_count"] == 0: + resolved_task_count += 1 + else: + missing_task_count += 1 + + allow_submit = not blockers + precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary() + return { + "root_dir": validation["root_dir"], + "mode": validation["mode"], + "task_count": len(task_summaries), + "selected_task_count": len(task_summaries), + "allow_submit": allow_submit, + "warnings": warnings, + "blockers": blockers, + "invalid_candidates": validation.get("invalid_candidates", []), + "dem": dem_summary, + "orbits": { + "policy": _get_orbit_policy(), + "pool_root": orbit_context.get("pool_root", ""), + "pool_exists": bool(orbit_context.get("pool_exists")), + "resolved_task_count": resolved_task_count, + "missing_task_count": missing_task_count, + "duplicate_count": int(orbit_context.get("duplicate_count", 0) or 0), + "warnings": list(orbit_context.get("warnings") or []), + }, + "precise_orbit_bridge": precise_orbit_bridge, + "tasks": task_summaries, + } + + +def summarize_preview_blockers(preview: Dict[str, Any], limit: int = 8) -> str: + blockers = [str(item).strip() for item in (preview.get("blockers") or []) if str(item).strip()] + if not blockers: + return "" + if len(blockers) <= limit: + return "; ".join(blockers) + return "; ".join(blockers[:limit]) + f"; 其余 {len(blockers) - limit} 项已省略" + + +def materialize_pyint_input_assets( + *, + task_summary: Dict[str, Any], + input_assets_dir: str, + project_name: str = "", +) -> Dict[str, Any]: + input_assets_dir = _normalize_path(input_assets_dir) + os.makedirs(input_assets_dir, exist_ok=True) + + record_enabled = bool(getattr(settings, "PYINT_RECORD_INPUT_ASSETS", True)) + orbits_dir = os.path.join(input_assets_dir, "orbits") + dem_dir = os.path.join(input_assets_dir, "dem") + if record_enabled: + os.makedirs(orbits_dir, exist_ok=True) + os.makedirs(dem_dir, exist_ok=True) + + manifest = _copy_json_safe(task_summary.get("input_assets") or {}) + manifest["generated_at"] = _utc_now_text() + manifest["task_name"] = task_summary.get("task_name") + manifest["task_alias"] = task_summary.get("task_alias") + manifest["pair_key"] = task_summary.get("pair_key") + manifest["task_dir"] = task_summary.get("task_dir") + manifest["allow_submit"] = bool(task_summary.get("allow_submit")) + manifest["warnings"] = list(task_summary.get("warnings") or []) + manifest["blockers"] = list(task_summary.get("blockers") or []) + + dem_summary = manifest.get("dem") or {} + if project_name: + dem_summary["resolved_output_dir"] = os.path.join(_normalize_path(settings.PYINT_DEM_ROOT), project_name) + manifest["dem"] = dem_summary + + orbits_summary = manifest.get("orbits") or {} + staged_count = 0 + precise_orbit_bridge = get_pyint_precise_orbit_bridge_summary() + should_stage_orbits = record_enabled and ( + str(orbits_summary.get("policy") or "").strip().lower() == "stage_txt" + or precise_orbit_bridge.get("enabled") + ) + if should_stage_orbits: + for role in ("master", "slave"): + orbit_item = orbits_summary.get(role) or {} + orbit_path = _normalize_path(orbit_item.get("path")) + expected_name = str(orbit_item.get("expected_name") or "").strip() + if not orbit_item.get("resolved") or not orbit_path or not expected_name: + continue + target_path = os.path.join(orbits_dir, expected_name) + if not os.path.exists(target_path): + shutil.copy2(orbit_path, target_path) + orbit_item["staged_path"] = target_path + orbit_item["stage_operation"] = "copied" + orbit_item["stage_reason"] = "precise_orbit_bridge" if precise_orbit_bridge.get("enabled") else "stage_txt_policy" + staged_count += 1 + orbits_summary[role] = orbit_item + manifest["orbits"] = orbits_summary + + materialized = { + "input_assets_dir": input_assets_dir, + "record_enabled": record_enabled, + "orbits_dir": orbits_dir if record_enabled else "", + "dem_dir": dem_dir if record_enabled else "", + "orbits_staged_count": staged_count, + "task_manifest_path": "", + "dem_summary_path": "", + "orbit_summary_path": "", + "input_assets": manifest, + } + + if not record_enabled: + return materialized + + task_manifest_path = os.path.join(input_assets_dir, "task_manifest.json") + dem_summary_path = os.path.join(dem_dir, "dem_summary.json") + orbit_summary_path = os.path.join(orbits_dir, "orbit_summary.json") + + with open(task_manifest_path, "w", encoding="utf-8") as fp: + json.dump(manifest, fp, ensure_ascii=False, indent=2) + fp.write("\n") + with open(dem_summary_path, "w", encoding="utf-8") as fp: + json.dump(dem_summary, fp, ensure_ascii=False, indent=2) + fp.write("\n") + with open(orbit_summary_path, "w", encoding="utf-8") as fp: + json.dump(orbits_summary, fp, ensure_ascii=False, indent=2) + fp.write("\n") + + materialized.update( + { + "task_manifest_path": task_manifest_path, + "dem_summary_path": dem_summary_path, + "orbit_summary_path": orbit_summary_path, + } + ) + return materialized diff --git a/backend/app/services/pyint_service.py b/backend/app/services/pyint_service.py new file mode 100644 index 0000000..d2a0d10 --- /dev/null +++ b/backend/app/services/pyint_service.py @@ -0,0 +1,409 @@ +"""Helpers for integrating the external PyINT workflow.""" +from __future__ import annotations + +import os +import re +import shlex +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional + +from ..config import get_env_text, read_bool_env, settings +from .dinsar_naming import PAIR_META_FILENAME, build_fallback_pair_key, find_json_sidecar +from .wsl_service import run_wsl_command + + +LT1_INPUT_GLOBS = ("LT1*.tar.gz", "LT1*.tiff") +DEFAULT_RANGE_LOOKS = 2 +DEFAULT_AZIMUTH_LOOKS = 2 +DEFAULT_PARALLEL_WORKERS = 1 +MAX_LOOKS = 32 +MAX_PARALLEL_WORKERS = 16 + +_DATE_TOKEN_RE = re.compile(r"(20\d{6})") +_SAFE_TEXT_RE = re.compile(r"[^0-9A-Za-z._-]+") + + +@dataclass +class PyintCheck: + name: str + ok: bool + detail: str = "" + skipped: bool = False + + +@dataclass +class PyintEnvironmentReport: + overall_ok: bool + checks: List[PyintCheck] = field(default_factory=list) + message: str = "" + + +def _read_env(name: str, default: str = "") -> str: + return get_env_text(name, default) or default + + +def _read_bool_env(name: str, default: bool = False) -> bool: + return read_bool_env(name, default) + + +def normalize_date_text(value: Any) -> str: + text = str(value or "").strip() + if not text: + return "" + match = _DATE_TOKEN_RE.search(re.sub(r"\D", "", text)) + if match: + return match.group(1) + match = _DATE_TOKEN_RE.search(text) + if match: + return match.group(1) + return "" + + +def slugify_text(value: Any, *, default: str = "item", max_len: int = 96) -> str: + text = _SAFE_TEXT_RE.sub("_", str(value or "").strip()).strip("._") + if not text: + text = default + return text[:max_len] + + +def build_project_name(pair_key: str, run_key: str) -> str: + return slugify_text(f"{pair_key}_{run_key}", default="pyint_project", max_len=120) + + +def windows_path_to_wsl_mount(path: str) -> str: + text = str(path or "").strip().strip('"').strip("'") + if not text: + return "" + normalized = os.path.normpath(text) + if normalized.startswith("/"): + return normalized.replace("\\", "/") + if normalized.startswith("\\\\"): + return "" + drive, tail = os.path.splitdrive(normalized) + if not drive: + return normalized.replace("\\", "/") + drive_letter = drive.rstrip(":").lower() + normalized_tail = tail.replace("\\", "/") + return f"/mnt/{drive_letter}/{normalized_tail}" + + +def to_wsl_path(path: str) -> str: + return windows_path_to_wsl_mount(path) + + +def quote_shell(value: str) -> str: + return shlex.quote(str(value or "")) + + +def discover_lt1_archives(task_dir: str) -> Dict[str, List[str]]: + task_path = Path(os.path.normpath(os.path.abspath(str(task_dir or "").strip()))) + result: Dict[str, List[str]] = {"master": [], "slave": []} + for role in ("master", "slave"): + role_dir = task_path / role + if not role_dir.is_dir(): + continue + inputs = [] + for pattern in LT1_INPUT_GLOBS: + inputs.extend( + str(path.resolve()) + for path in role_dir.rglob(pattern) + if path.is_file() + ) + result[role] = sorted(set(inputs)) + return result + + +def infer_scene_date_from_archives(paths: Iterable[str]) -> str: + dates = { + date_text + for path in paths + for date_text in [normalize_date_text(os.path.basename(path))] + if date_text + } + if len(dates) == 1: + return next(iter(dates)) + return "" + + +def infer_task_identity(task_dir: str) -> Dict[str, Any]: + task_name = os.path.basename(os.path.normpath(task_dir)) + pair_meta = find_json_sidecar(task_dir, PAIR_META_FILENAME, max_levels=0) or {} + task_alias = str(pair_meta.get("task_alias") or task_name).strip() or task_name + pair_key = str(pair_meta.get("pair_key") or "").strip() or build_fallback_pair_key(task_alias, task_dir) + master_date = normalize_date_text(pair_meta.get("master_imaging_date")) + slave_date = normalize_date_text(pair_meta.get("slave_imaging_date")) + return { + "task_name": task_name, + "task_alias": task_alias, + "pair_key": pair_key, + "pair_meta": pair_meta, + "master_date": master_date, + "slave_date": slave_date, + } + + +def build_template_text( + *, + project_name: str, + master_date: str, + range_looks: int, + azimuth_looks: int, + parallel_workers: int, + unwrap: bool, + geocode: bool, +) -> str: + lines = [ + f"# Auto-generated for {project_name}", + "satelite=LT", + f"masterDate={master_date}", + f"range_looks={int(range_looks)}", + f"azimuth_looks={int(azimuth_looks)}", + "download_data=0", + "raw2slc_all=1", + f"raw2slc_all_parallel={int(parallel_workers)}", + "coreg_all=1", + f"coreg_all_parallel={int(parallel_workers)}", + "select_pairs=0", + "diff_all=1", + f"diff_all_parallel={int(parallel_workers)}", + f"unwrap_all={1 if unwrap else 0}", + f"unwrap_all_parallel={int(parallel_workers)}", + f"geocode_all={1 if geocode else 0}", + f"geocode_all_parallel={int(parallel_workers)}", + "geocode_products=hyp3,licsbas", + ] + return "\n".join(lines) + "\n" + + +def validate_pyint_root_dir(root_dir: str, num_to_process: int = 0) -> Dict[str, Any]: + normalized_root = os.path.normpath(os.path.abspath(str(root_dir or "").strip())) + if not root_dir or not os.path.isdir(normalized_root): + raise ValueError(f"PyINT root_dir does not exist or is not a directory: {root_dir}") + + def _missing_task_subdirs(task_dir: str) -> List[str]: + missing: List[str] = [] + for subdir in ("master", "slave"): + if not os.path.isdir(os.path.join(task_dir, subdir)): + missing.append(subdir) + return missing + + def _iter_child_dirs(directory: str): + with os.scandir(directory) as entries: + child_dirs = [entry for entry in entries if entry.is_dir()] + child_dirs.sort(key=lambda entry: entry.name.lower()) + return child_dirs + + if not _missing_task_subdirs(normalized_root): + task_dirs = [normalized_root] + invalid_candidates: List[Dict[str, Any]] = [] + mode = "single_task_dir" + else: + task_dirs = [] + invalid_candidates = [] + for entry in _iter_child_dirs(normalized_root): + if not entry.name.lower().startswith("task_"): + continue + missing = _missing_task_subdirs(entry.path) + if missing: + invalid_candidates.append( + {"name": entry.name, "path": entry.path, "missing_subdirs": missing} + ) + continue + task_dirs.append(os.path.normpath(entry.path)) + mode = "task_root_dir" + + if not task_dirs: + detail = "" + if invalid_candidates: + formatted = ", ".join( + f"{item['name']} missing {','.join(item['missing_subdirs'])}" + for item in invalid_candidates[:5] + ) + detail = f" Invalid candidates: {formatted}." + raise ValueError( + "PyINT root_dir must be either a single task directory containing " + "'master' and 'slave', or a parent directory containing valid Task_* subdirectories." + f"{detail}" + ) + + selected_count = int(num_to_process or 0) + if selected_count > 0: + task_dirs = task_dirs[:selected_count] + + return { + "root_dir": normalized_root, + "mode": mode, + "task_dirs": task_dirs, + "task_count": len(task_dirs), + "invalid_candidates": invalid_candidates, + } + + +def resolve_time_baseline_days(master_date: str, slave_date: str, pair_meta: Dict[str, Any]) -> int: + raw_days = pair_meta.get("time_baseline_days") + try: + if raw_days not in (None, ""): + return int(raw_days) + except (TypeError, ValueError): + pass + + if not master_date or not slave_date: + return 0 + try: + master_dt = datetime.strptime(master_date, "%Y%m%d") + slave_dt = datetime.strptime(slave_date, "%Y%m%d") + except ValueError: + return 0 + return (slave_dt - master_dt).days + + +def _gamma_prefix(gamma_env_script_wsl: str) -> str: + script = str(gamma_env_script_wsl or "").strip() + if not script: + return "" + return f". {quote_shell(script)} >/dev/null 2>&1 && " + + +def check_pyint_environment( + *, + enabled: Optional[bool] = None, + distro: Optional[str] = None, + python_cmd: Optional[str] = None, + pyint_home: Optional[str] = None, + pyint_app_script: Optional[str] = None, + template_root: Optional[str] = None, + work_root: Optional[str] = None, + output_root: Optional[str] = None, + dem_root: Optional[str] = None, + gamma_env_script: Optional[str] = None, + smoke_test: Optional[bool] = None, +) -> PyintEnvironmentReport: + enabled_value = _read_bool_env("PYINT_ENABLED", False) if enabled is None else bool(enabled) + if not enabled_value: + return PyintEnvironmentReport( + overall_ok=False, + checks=[PyintCheck(name="PYINT_ENABLED", ok=False, detail="PYINT_ENABLED=false")], + message="PyINT is disabled. Set PYINT_ENABLED=true to enable it.", + ) + + distro_value = str(distro or _read_env("PYINT_WSL_DISTRO", settings.ISCE2_WSL_DISTRO)).strip() + python_value = str(python_cmd or _read_env("PYINT_WSL_PYTHON", settings.ISCE2_PYTHON)).strip() + pyint_home_wsl = to_wsl_path(str(pyint_home or _read_env("PYINT_HOME", ""))) + pyint_app_wsl = to_wsl_path(str(pyint_app_script or _read_env("PYINT_APP_SCRIPT", ""))) + template_root_wsl = to_wsl_path(str(template_root or _read_env("PYINT_TEMPLATE_ROOT", ""))) + work_root_wsl = to_wsl_path(str(work_root or _read_env("PYINT_WORK_ROOT", ""))) + output_root_wsl = to_wsl_path(str(output_root or _read_env("PYINT_OUTPUT_ROOT", ""))) + dem_root_wsl = to_wsl_path(str(dem_root or _read_env("PYINT_DEM_ROOT", ""))) + gamma_env_wsl = to_wsl_path(str(gamma_env_script or _read_env("PYINT_GAMMA_ENV_SCRIPT", ""))) + smoke_enabled = _read_bool_env("PYINT_SMOKE_TEST_ENABLED", False) if smoke_test is None else bool(smoke_test) + precise_orbit_enabled = _read_bool_env("PYINT_LT1_PRECISE_ORBIT_ENABLED", True) + + checks: List[PyintCheck] = [] + + def add(name: str, ok: bool, detail: str = "", skipped: bool = False) -> None: + checks.append(PyintCheck(name=name, ok=ok, detail=detail, skipped=skipped)) + + rc, out, err = run_wsl_command("echo pyint_alive", distro=distro_value, timeout=15) + wsl_ok = rc == 0 and "pyint_alive" in out + add("WSL distro", wsl_ok, out or err or distro_value) + + if not wsl_ok: + return PyintEnvironmentReport( + overall_ok=False, + checks=checks, + message=f"WSL distro is unavailable: {distro_value}", + ) + + rc, out, err = run_wsl_command( + f"{quote_shell(python_value)} --version", + distro=distro_value, + timeout=15, + ) + add("WSL Python", rc == 0, out or err or python_value) + + if pyint_home_wsl: + rc, out, err = run_wsl_command( + f"test -d {quote_shell(pyint_home_wsl)} && echo ok", + distro=distro_value, + timeout=10, + ) + add("PYINT_HOME", rc == 0 and "ok" in out, pyint_home_wsl or err) + else: + add("PYINT_HOME", False, "PYINT_HOME is empty") + + if pyint_app_wsl: + rc, out, err = run_wsl_command( + f"test -f {quote_shell(pyint_app_wsl)} && echo ok", + distro=distro_value, + timeout=10, + ) + add("pyintApp.py", rc == 0 and "ok" in out, pyint_app_wsl or err) + else: + add("pyintApp.py", False, "PYINT_APP_SCRIPT is empty") + + for name, path_text in ( + ("PYINT_TEMPLATE_ROOT", template_root_wsl), + ("PYINT_WORK_ROOT", work_root_wsl), + ("PYINT_OUTPUT_ROOT", output_root_wsl), + ("PYINT_DEM_ROOT", dem_root_wsl), + ): + if not path_text: + add(name, False, f"{name} is empty") + continue + rc, out, err = run_wsl_command( + f"test -d {quote_shell(path_text)} && test -w {quote_shell(path_text)} && echo ok", + distro=distro_value, + timeout=10, + ) + add(name, rc == 0 and "ok" in out, path_text or err) + + if gamma_env_wsl: + rc, out, err = run_wsl_command( + f"test -f {quote_shell(gamma_env_wsl)} && echo ok", + distro=distro_value, + timeout=10, + ) + add("GAMMA env script", rc == 0 and "ok" in out, gamma_env_wsl or err) + else: + add("GAMMA env script", True, "Not configured; using current PATH", skipped=True) + + gamma_prefix = _gamma_prefix(gamma_env_wsl) + for name, command_name in ( + ("GAMMA LT1 import", "LT1_import_SLC_from_zipfiles1"), + ("GAMMA geocode_back", "geocode_back"), + ): + rc, out, err = run_wsl_command( + gamma_prefix + f"command -v {quote_shell(command_name)}", + distro=distro_value, + timeout=10, + ) + add(name, rc == 0 and bool(out.strip()), out or err or command_name) + + helper_path = ( + Path(__file__).resolve().parent.parent + / "pyint_pipeline" + / "apply_lt1_precise_orbit.py" + ) + if precise_orbit_enabled: + add("LT1 precise orbit bridge helper", helper_path.is_file(), str(helper_path)) + else: + add("LT1 precise orbit bridge helper", True, "Skipped", skipped=True) + + if smoke_enabled: + smoke_cmd = ( + f"export PYTHONPATH={quote_shell(pyint_home_wsl)}:$PYTHONPATH && " + + gamma_prefix + + f"{quote_shell(python_value)} {quote_shell(pyint_app_wsl)} -h >/dev/null" + ) + rc, out, err = run_wsl_command(smoke_cmd, distro=distro_value, timeout=60) + add("PyINT smoke test", rc == 0, out or err or "pyintApp.py -h") + else: + add("PyINT smoke test", True, "Skipped", skipped=True) + + required_checks = [check for check in checks if not check.skipped] + overall_ok = all(check.ok for check in required_checks) + failed_names = [check.name for check in required_checks if not check.ok] + message = "All PyINT checks passed." if overall_ok else f"Failed checks: {', '.join(failed_names)}" + return PyintEnvironmentReport(overall_ok=overall_ok, checks=checks, message=message) diff --git a/backend/app/services/unpack_service.py b/backend/app/services/unpack_service.py index 2f41e3d..5771049 100644 --- a/backend/app/services/unpack_service.py +++ b/backend/app/services/unpack_service.py @@ -54,12 +54,55 @@ def get_unpack_config() -> Dict[str, Any]: minimum=1, maximum=32, ), + "max_files_per_run": module.parse_int( + env.get("UNPACK_MAX_FILES_PER_RUN"), + default=0, + minimum=0, + ), + "max_runtime_minutes": module.parse_int( + env.get("UNPACK_MAX_RUNTIME_MINUTES"), + default=0, + minimum=0, + ), } -async def run_unpack_task(task_id: str): +def _normalize_unpack_run_limits(raw_config: Optional[Dict[str, Any]]) -> Dict[str, int]: + if not isinstance(raw_config, dict): + return {} + + module = _load_unpack_module() + normalized: Dict[str, int] = {} + + if raw_config.get("max_files_per_run") is not None: + normalized["max_files_per_run"] = module.parse_int( + raw_config.get("max_files_per_run"), + default=0, + minimum=0, + ) + if raw_config.get("max_runtime_minutes") is not None: + normalized["max_runtime_minutes"] = module.parse_int( + raw_config.get("max_runtime_minutes"), + default=0, + minimum=0, + ) + + return normalized + + +def build_unpack_run_config(overrides: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + config = get_unpack_config() + config.update(_normalize_unpack_run_limits(overrides)) + return config + + +async def run_unpack_task(task_id: str, task_config: Optional[Dict[str, Any]] = None): module = _load_unpack_module() loop = asyncio.get_running_loop() + config_overrides = _normalize_unpack_run_limits(task_config) + if not config_overrides: + task_record = await task_service.get_task(task_id) + config_overrides = _normalize_unpack_run_limits(getattr(task_record, "params", None)) def _submit(coro): try: @@ -88,14 +131,19 @@ async def run_unpack_task(task_id: str): module.run_unpack_job, log_callback=log_cb, progress_callback=progress_cb, + config_overrides=config_overrides or None, ) if not result: - result = {"processed": 0, "failed": 0, "skipped": 0, "total": 0} + result = {"processed": 0, "failed": 0, "skipped": 0, "total": 0, "remaining": 0, "message": "completed"} - summary = ( - "Unpack complete: processed {processed}, failed {failed}, skipped {skipped}" - ).format(**result) + summary = "Unpack complete: processed {processed}, failed {failed}, skipped {skipped}".format(**result) + remaining = int(result.get("remaining") or 0) + if remaining > 0: + summary = f"{summary}, remaining {remaining}" + message_text = str(result.get("message") or "").strip() + if message_text and message_text != "completed": + summary = f"{summary} ({message_text})" await task_service.update_task(task_id, status="COMPLETED", progress=100, message=summary) except Exception as exc: await task_service.update_task(task_id, status="FAILED", message=f"Unpack failed: {exc}") diff --git a/docs/INDEX.md b/docs/INDEX.md index 9ffa34a..6ebcfb2 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -28,6 +28,16 @@ 配对能力增强设计。属于未完全落地的专项设计文档。 - **[GAMMA_WSL2_INTEGRATION_PLAN.md](GAMMA_WSL2_INTEGRATION_PLAN.md)** GAMMA + WSL2 双引擎方案。属于后续扩展规划，不是当前默认运行链路。 +- **[PYINT_GAMMA_INTEGRATION_DESIGN_20260418.md](PYINT_GAMMA_INTEGRATION_DESIGN_20260418.md)** + PyINT 生产引擎与 Gamma 精配对总体设计。明确现有多引擎架构下的接入边界、配置管理、数据库策略、运维自检扩展与前端入口分布。 +- **[PYINT_GAMMA_IMPLEMENTATION_TODO_20260418.md](PYINT_GAMMA_IMPLEMENTATION_TODO_20260418.md)** + PyINT 生产引擎与 Gamma 精配对实施清单。按阶段拆分后端、前端、接口、运维与可选数据库任务，作为后续落地执行顺序。 +- **[PYINT_INPUT_ASSET_ADAPTATION_DESIGN_20260419.md](PYINT_INPUT_ASSET_ADAPTATION_DESIGN_20260419.md)** + PyINT 输入资产适配设计。聚焦 `Task_*` 路径如何映射到 PyINT 工作区，以及 DEM、LT-1 精密轨道、运维自检和前端入口应如何纳入系统托管治理。 +- **[PYINT_LT1_COREG_ORBIT_HYPOTHESIS_EXPERIMENT_20260420.md](PYINT_LT1_COREG_ORBIT_HYPOTHESIS_EXPERIMENT_20260420.md)** + LT-1 在 PyINT/Gamma 中 `coreg` 失败的轨道假设验证实验设计。固定输入和 DEM，只改变 `.slc.par` 的 state vector 处理方式，对照验证问题是否集中在导入后的轨道几何链条。 +- **[PYINT_LT1_DEM_GEOMETRY_CHAIN_EXPERIMENT_20260420.md](PYINT_LT1_DEM_GEOMETRY_CHAIN_EXPERIMENT_20260420.md)** + LT-1 在 PyINT/Gamma 中 `init_offsetm` 失败的 DEM 几何链定位实验。聚焦 `HGTSIM / lt0 / mli0 / Samp` 的中间产物，区分 DEM 本体问题、DEM 几何映射链问题和中心 patch 选取问题。 - **[WSL2_ISCE2_MINTPY_SBAS_INTEGRATION_PLAN_20260412.md](WSL2_ISCE2_MINTPY_SBAS_INTEGRATION_PLAN_20260412.md)** 基于本机 `Ubuntu-24.04` WSL2、`isce2` / `mintpy` / `isce2_mintpy_v1` 实际环境核对后的 SBAS 集成落地方案，明确推荐运行时、workflow 补全顺序和正式产品边界。 - **[ISCE2_SBAS_TIMESERIES_DESIGN.md](ISCE2_SBAS_TIMESERIES_DESIGN.md)** @@ -91,6 +101,11 @@ 4. [ISCE2_SBAS_TIMESERIES_DESIGN.md](ISCE2_SBAS_TIMESERIES_DESIGN.md) 5. [WSL2_ISCE2_MINTPY_SBAS_INTEGRATION_PLAN_20260412.md](WSL2_ISCE2_MINTPY_SBAS_INTEGRATION_PLAN_20260412.md) 6. [GAMMA_WSL2_INTEGRATION_PLAN.md](GAMMA_WSL2_INTEGRATION_PLAN.md) +7. [PYINT_GAMMA_INTEGRATION_DESIGN_20260418.md](PYINT_GAMMA_INTEGRATION_DESIGN_20260418.md) +8. [PYINT_GAMMA_IMPLEMENTATION_TODO_20260418.md](PYINT_GAMMA_IMPLEMENTATION_TODO_20260418.md) +9. [PYINT_INPUT_ASSET_ADAPTATION_DESIGN_20260419.md](PYINT_INPUT_ASSET_ADAPTATION_DESIGN_20260419.md) +10. [PYINT_LT1_COREG_ORBIT_HYPOTHESIS_EXPERIMENT_20260420.md](PYINT_LT1_COREG_ORBIT_HYPOTHESIS_EXPERIMENT_20260420.md) +11. [PYINT_LT1_DEM_GEOMETRY_CHAIN_EXPERIMENT_20260420.md](PYINT_LT1_DEM_GEOMETRY_CHAIN_EXPERIMENT_20260420.md) ## 4. 后续维护规则 diff --git a/docs/PYINT_GAMMA_AB_FINDINGS_20260420.md b/docs/PYINT_GAMMA_AB_FINDINGS_20260420.md new file mode 100644 index 0000000..18c943a --- /dev/null +++ b/docs/PYINT_GAMMA_AB_FINDINGS_20260420.md @@ -0,0 +1,74 @@ +# PyINT + Gamma A/B 排查结论 + +更新时间：2026-04-20 + +## 1. 排查目标 + +验证当前仓库内改过的 `PyINT/Gamma` 流程，是否只是“流程层修改”而没有影响科学结果；尤其要定位为什么同一组 LT-1 `Task` 在 ENVI/IDL 核心可产出结果，而当前 PyINT 结果为空。 + +本轮对照任务： + +- 任务目录：`D:\Task_Pool\DInSAR\Task_260416_Gamma_PyINT\Task_20230602_20230720` +- DEM：`D:\DEM\COPDEM_GLO30_China_4326_DEM` +- 实验根目录：`D:\PyINT_AB` +- 生产链 Python：`/home/administrator/miniconda3/envs/isce2/bin/python` + +## 2. 对照实验 + +### Case A：当前代码 + 轨道桥接开启 + rescue 开启 + +- case：`D:\PyINT_AB\current_orbit_rescue` +- 结果：流程可跑完到 `diff` +- 但关键中间结果全 0： + - `diff_filt.zero_ratio = 1.0` + - `cor.zero_ratio = 1.0` + +### Case B：当前代码 + 轨道桥接关闭 + rescue 开启 + +- case：`D:\PyINT_AB\no_orbit_rescue` +- 结果：流程同样可跑完到 `diff` +- 关键中间结果仍然全 0： + - `diff_filt.zero_ratio = 1.0` + - `cor.zero_ratio = 1.0` + +结论：轨道桥接开关不是这次“全 0 结果”的主责任点。 + +### Case C：轨道桥接开启 + 去掉 rescue + +- case：`D:\PyINT_AB\orbit_no_rescue` +- 结果：流程直接死在 `coreg` +- 关键报错： + - `init_offsetm failed` + - `ERROR: number of zero values 195367 in MLI1 image patch exceeds threshold: 32768` + +结论：当前仓库里的 `coreg rescue` 确实在掩盖真实失败。它让一个本应失败的配准任务继续往后执行，最终产生“流程成功但科学结果全 0”的假成功。 + +## 3. 直接结论 + +1. 不能再说“我们现在的改动只改流程、不影响结果”。当前实现已经改变了失败语义，导致无效结果被当成成功结果收口。 +2. 这次任务的主要问题点在 `coreg`，不是 GeoTIFF 导出，不是地理编码，也不是解缠。 +3. LT-1 精密轨道桥接不是这次空结果的主责任点；更深层的根因仍然要继续排查 LT-1 导入 / 配准链路与 ENVI/IDL 核心之间的差异。 + +## 4. 已落实的代码策略 + +为避免系统继续产出“成功但全 0”的无效结果，当前仓库已做两项收敛： + +1. `third_party/PyINT/pyint/coreg_gamma.py` + - 去掉两个 rescue/fallback + - `init_offsetm/offset_pwrm/offset_fitm/gc_map_fine` 失败时不再复制 `lt0 -> lt1` + - offset refinement 失败时不再把 `Srslc0` 直接提升为最终 `RSLC` + - 改为失败即退出 +2. `backend/app/pyint_pipeline/run_lt1_pyint_pipeline.py` + - 增加产物有效性检查 + - 如果 `diff_filt` / `coh` / `unw` / `geo_unw` / `geo_los` 出现“文件存在但二进制全 0”，直接判定此次运行失败 + - 运行失败时附带阶段错误日志路径，例如 `coreg_gamma_all.err` + +## 5. 后续真正要解决的问题 + +这次代码收敛只解决“假成功”问题，还没有解决“为什么 LT-1 在 PyINT/Gamma 下配不准”这个根因。下一阶段建议继续做以下对照： + +1. 对比 ENVI/IDL 成功任务与 PyINT 导入后的 `.slc.par`、多视幅度、DEM 配准输入是否一致。 +2. 对比 LT-1 导入脚本生成的 `SLC/MLI` 几何参数，尤其是时序、PRF、采样间隔、deskew、状态矢量相关字段。 +3. 对比 `mli0` 与目标 `Samp` 的重叠区域，确认 `init_offsetm` 为什么会在中心 patch 上出现大量 0 值。 + +当前判断：真正的科学问题仍在 LT-1 导入 / coreg 前置几何链路，而不是后面的 unwrap / geocode。 diff --git a/docs/PYINT_GAMMA_IMPLEMENTATION_TODO_20260418.md b/docs/PYINT_GAMMA_IMPLEMENTATION_TODO_20260418.md new file mode 100644 index 0000000..d66154c --- /dev/null +++ b/docs/PYINT_GAMMA_IMPLEMENTATION_TODO_20260418.md @@ -0,0 +1,390 @@ +# PyINT + Gamma 实施清单 + +更新日期：2026-04-18 + +关联设计文档： + +- [PYINT_GAMMA_INTEGRATION_DESIGN_20260418.md](PYINT_GAMMA_INTEGRATION_DESIGN_20260418.md) + +## 当前落地进度 + +- [x] 已完成第一批 `PyINT` 生产引擎接入：配置、引擎注册、任务队列、WSL 包装脚本、生产面板入口。 +- [x] 已完成 `PyINT` 运行目录规范化输出：生成 `.dinsar_run.json` 与 `pyint_run_summary.json`。 +- [x] 已完成 `PyINT` 基础环境检查：WSL、Python、`PYINT_HOME`、`pyintApp.py`、Gamma 命令可达性。 +- [x] 已将 `PyINT` 代码收编到仓库内 `third_party/PyINT`，不再依赖默认外部绝对路径。 +- [ ] 尚未完成 `PyINT` 结果目录自动发布兼容。当前原生输出已保存，但现有结果 catalog 仍主要面向 ENVI / ISCE2 栅格产物。 +- [ ] 尚未开始 `Gamma` 精配对后端与前端集成。 + +## 1. 文档定位 + +这份清单用于把 `PyINT` 生产引擎接入和 `Gamma` 精配对接入拆成可执行任务，作为后续实施顺序、联调顺序和验收顺序的统一依据。 + +本清单按以下原则编排： + +- 一期优先打通 `PyINT` 生产引擎 +- 二期再做 `Gamma` 精配对 MVP +- 一期不强制改数据库主结构 +- 运维自检只加状态，不把主要操作堆回健康页 + +## 2. 实施总顺序 + +推荐顺序： + +1. 先确认环境基线和配置项 +2. 先打通 `PyINT` 后端引擎与任务执行 +3. 再补结果归一化与目录扫描兼容 +4. 再补前端生产入口 +5. 然后做 `Gamma` 精配对 MVP +6. 最后补健康检查、烟测和治理 + +不建议顺序： + +- 先改数据库再写主流程 +- 先做健康页大改 +- 先把 `PyINT` 全部高级参数暴露到前端 + +## 3. Phase 0：环境基线确认 + +目标： + +- 确认当前机器上的 `PyINT + Gamma + WSL` 具备最小可执行条件 +- 把配置字段定清楚，但不把敏感信息写入仓库文档 + +### 任务 + +- [ ] 确认 `D:\Code\PyINT` 的实际可执行入口路径 +- [ ] 确认当前唯一 WSL distro 名称，默认与 `ISCE2_WSL_DISTRO` 对齐 +- [ ] 确认 WSL 中 `PyINT` 可用 Python 路径 +- [ ] 确认 `GAMMA_ENV_SCRIPT` 的实际路径 +- [ ] 确认 `base_calc` 在 WSL 中可执行 +- [ ] 确认 `pyintApp.py` 在 WSL 中可执行 +- [ ] 确认 `SCRATCHDIR` / `TEMPLATEDIR` / `DEMDIR` 对应的系统托管目录方案 +- [ ] 确认 PyINT 一期只支持的业务范围，建议锁定 `LT-1 + Gamma D-InSAR` + +### 涉及文件 + +- [ ] `backend/app/config.py` +- [ ] `.env.example` +- [ ] 根 `.env` 本机配置对照，不入库敏感值 + +### 阶段验收 + +- [ ] 可以给出完整的 PyINT 运行必需配置字段列表 +- [ ] 可以在 WSL 内成功跑通最小烟测命令 +- [ ] 不需要把管理员密码、邮箱密码、sudo 密码写入代码或文档 + +## 4. Phase 1：PyINT 后端引擎接入 + +目标： + +- 把 `PyINT` 作为新的 D-InSAR 引擎正式接入现有多引擎体系 + +### 4.1 配置层 + +- [ ] 在 `backend/app/config.py` 新增 `PYINT_*` 配置 +- [ ] 增加默认继承逻辑：`PYINT_WSL_DISTRO` 默认跟随 `ISCE2_WSL_DISTRO` +- [ ] 增加默认继承逻辑：`PYINT_WSL_PYTHON` 默认跟随 `ISCE2_PYTHON` +- [ ] 增加系统托管目录默认值： + - [ ] `PYINT_TEMPLATE_ROOT` + - [ ] `PYINT_WORK_ROOT` + - [ ] `PYINT_OUTPUT_ROOT` +- [ ] 在 `validate_runtime_config()` 中加入 PyINT 基础校验 +- [ ] 在 `.env.example` 中补齐非敏感 PyINT 配置示例 + +### 4.2 服务层 + +- [ ] 新增 `backend/app/services/pyint_service.py` +- [ ] 封装 WSL 执行逻辑，复用现有 `wsl_service.py` +- [ ] 实现 Windows 路径到 WSL 路径转换 +- [ ] 实现 PyINT 工作区目录初始化 +- [ ] 实现模板文件生成 +- [ ] 实现运行参数到模板字段的映射 +- [ ] 实现运行摘要 JSON 输出 +- [ ] 实现 PyINT 烟测函数 + +### 4.3 引擎层 + +- [ ] 新增 `backend/app/dinsar_engines/pyint_engine.py` +- [ ] 实现 `DinsarEngine` 接口 +- [ ] 定义 `engine_code=pyint` +- [ ] 定义一期唯一 profile，建议为 `lt1_gamma_dinsar` +- [ ] 定义最小参数 schema，避免一开始暴露过多 PyINT 原生参数 +- [ ] 实现 `check_available()` +- [ ] 实现 `run()` + +### 4.4 注册与任务调度 + +- [ ] 在 `backend/app/dinsar_engines/registry.py` 注册 `PyINT` +- [ ] 在 `backend/app/services/job_handlers.py` 增加 `JOB_TYPE_PYINT_RUN` +- [ ] 新增对应 handler +- [ ] 在 `backend/app/routers/dinsar_production.py` 允许 `engine_code=pyint` +- [ ] 让生产提交逻辑按 `pyint` 分派到新 job type + +### 4.5 WSL 包装脚本 + +- [ ] 新增 `backend/app/pyint_pipeline/run_lt1_pyint_pipeline.py` +- [ ] 负责把系统任务目录映射为 PyINT 项目目录 +- [ ] 负责设置 `SCRATCHDIR` / `TEMPLATEDIR` / `DEMDIR` +- [ ] 负责调用 `pyintApp.py` 或必要的细粒度 PyINT 脚本 +- [ ] 负责收集输出路径和运行摘要 + +### 涉及文件 + +- [ ] `backend/app/config.py` +- [ ] `backend/app/dinsar_engines/registry.py` +- [ ] `backend/app/dinsar_engines/pyint_engine.py` +- [ ] `backend/app/services/pyint_service.py` +- [ ] `backend/app/services/job_handlers.py` +- [ ] `backend/app/routers/dinsar_production.py` +- [ ] `backend/app/pyint_pipeline/run_lt1_pyint_pipeline.py` +- [ ] `.env.example` + +### 阶段验收 + +- [ ] `/dinsar-production/engines` 能返回 `pyint` +- [ ] `PyINT` 引擎可在后端被识别为可用/不可用 +- [ ] 可以成功提交一个 `pyint` 生产任务到队列 +- [ ] 任务日志、任务状态、错误信息可通过现有任务体系查看 + +## 5. Phase 2：PyINT 结果归一化与结果治理兼容 + +目标： + +- 保证 `PyINT` 输出能进入现有结果扫描、发布和 catalog 体系 + +### 任务 + +- [ ] 定义 PyINT 结果工作区与正式输出区的边界 +- [ ] 统一输出 bundle 元数据格式 +- [ ] 输出 engine/profile/run_key/task_name/pair trace 元数据 +- [ ] 补齐 pair 相关元数据： + - [ ] `pair_uid` + - [ ] `network_run_id` + - [ ] `network_edge_id` + - [ ] `policy_version` +- [ ] 让现有 `dinsar_scan_service` 可以识别 PyINT 结果 +- [ ] 验证现有 `result_catalog_service` 可处理 PyINT 产物 +- [ ] 验证桥接一致性逻辑不会把 PyINT 结果识别坏 + +### 涉及文件 + +- [ ] `backend/app/services/pyint_service.py` +- [ ] `backend/app/services/dinsar_scan_service.py` +- [ ] `backend/app/services/result_catalog_service.py` +- [ ] 可能涉及现有结果元数据写入辅助模块 + +### 阶段验收 + +- [ ] 跑完 PyINT 后可被系统扫描到 +- [ ] 可进入结果目录索引 +- [ ] 不影响现有 SARscape/ISCE2 结果扫描 + +## 6. Phase 3：前端生产页接入 PyINT + +目标： + +- 在现有生产页中把 `PyINT` 作为正式引擎展示和提交 + +### 任务 + +- [ ] 在 `frontend/src/DinsarProductionPanel.jsx` 中显示 `PyINT` 引擎卡片 +- [ ] 补充 `ENGINE_LABEL` / `TASK_TYPE_LABEL` +- [ ] 根据 `PyINT` profile 渲染参数输入项 +- [ ] 对不可用状态显示明确原因 +- [ ] 提交成功后沿用现有任务监控 +- [ ] 验证运行列表中能正确显示 `pyint` + +### 涉及文件 + +- [ ] `frontend/src/DinsarProductionPanel.jsx` +- [ ] `frontend/src/api/dinsarProduction.js` +- [ ] `frontend/src/utils/dinsarEngines.js` 如需要 + +### 阶段验收 + +- [ ] 前端可看到 `PyINT` +- [ ] 可提交 `PyINT` 任务 +- [ ] 可看到任务状态和日志 +- [ ] 不影响现有 SARscape/ISCE2 提交 + +## 7. Phase 4：Gamma 精配对 MVP + +目标： + +- 在现有配对规划体系上实现一版可用的 `Gamma` 精配对 + +### 7.1 后端能力 + +- [ ] 新增 `backend/app/services/pairing_refinement_service.py` +- [ ] 新增 `backend/app/services/gamma_pairing_service.py` 或同等职责模块 +- [ ] 实现基于现有 `network_run_id` 的场景集提取 +- [ ] 实现 PyINT/Gamma 配对工作区构建 +- [ ] 调用 `select_pairs.py` / `base_calc` +- [ ] 解析 `ifgram_list.txt` / baseline 输出 +- [ ] 生成新的 refined `network_run_id` +- [ ] 将精配对结果写入： + - [ ] `pairing_network_runs` + - [ ] `pairing_network_edges` + - [ ] `selection_meta_json` + +### 7.2 接口层 + +- [ ] 在 `backend/app/routers/pairing.py` 增加 `POST /pairing/refine-gamma` +- [ ] 设计请求体和响应体 +- [ ] 设计运行告警返回字段 +- [ ] 如需要，增加 refined artifacts 查询接口 + +### 7.3 存储策略 + +- [ ] 明确一期不改 `pairing_metric_cache` 语义 +- [ ] 明确只在 run/edge JSON 中落精配对元数据 +- [ ] 保留粗配对网络和精配对网络双轨并存 + +### 涉及文件 + +- [ ] `backend/app/routers/pairing.py` +- [ ] `backend/app/services/spatial_service.py` 如需复用 +- [ ] `backend/app/services/pairing_refinement_service.py` +- [ ] `backend/app/services/gamma_pairing_service.py` +- [ ] `backend/app/models/schemas.py` + +### 阶段验收 + +- [ ] 可基于一个已有 `network_run_id` 发起精配对 +- [ ] 返回新的 refined `network_run_id` +- [ ] 精配对结果可通过现有 network 查询接口查看 +- [ ] 不破坏原粗配对结果 + +## 8. Phase 5：前端配对规划页接入 Gamma 精配对 + +目标： + +- 在配对规划页提供精配对入口和结果摘要 + +### 任务 + +- [ ] 在 `frontend/src/panels/PairPlanningPanel.jsx` 新增 `Gamma 精配对` 区块 +- [ ] 展示当前粗配对网络摘要 +- [ ] 增加发起精配对按钮 +- [ ] 展示精配对结果摘要 +- [ ] 展示粗配对与精配对差异提示 +- [ ] 增加“采用哪一版网络继续生产”的状态表达 + +### 涉及文件 + +- [ ] `frontend/src/panels/PairPlanningPanel.jsx` +- [ ] `frontend/src/api/pairing.js` + +### 阶段验收 + +- [ ] 管理员可在配对规划页发起精配对 +- [ ] 能看到 refined 结果摘要 +- [ ] 不需要进入健康检查页做配对操作 + +## 9. Phase 6：运维自检与烟测补齐 + +目标： + +- 让 PyINT/Gamma 的环境状态可被健康检查观察 + +### 任务 + +- [ ] 在 `backend/app/services/health_service.py` 中纳入 PyINT 检查 +- [ ] 检查项至少包括： + - [ ] `PYINT_ENABLED` + - [ ] distro 可访问 + - [ ] WSL Python 可执行 + - [ ] `pyintApp.py` 存在 + - [ ] `GAMMA_ENV_SCRIPT` 存在 + - [ ] `base_calc` 可执行 + - [ ] 模板目录可读 + - [ ] 工作目录可写 +- [ ] 增加管理员烟测接口 +- [ ] 前端健康页只展示状态摘要，不加复杂操作区 + +### 涉及文件 + +- [ ] `backend/app/services/health_service.py` +- [ ] `backend/app/routers/dinsar_production.py` +- [ ] `frontend/src/HealthCheckPanel.jsx` + +### 阶段验收 + +- [ ] 健康页可看到 PyINT 状态 +- [ ] 可区分“引擎不可用”和“系统整体故障” +- [ ] 不把精配对主操作入口放回健康页 + +## 10. Phase 7：可选数据库结构化增强 + +目标： + +- 只有在业务确认需要更强的历史与运维管理时才进入本阶段 + +### 进入条件 + +- [ ] 需要独立查询精配对历史 +- [ ] 需要统计精配对失败率 +- [ ] 需要管理精配对 artifacts 生命周期 +- [ ] 需要构建更完整的后台管理页 + +### 任务 + +- [ ] 设计 `pairing_refinement_runs` 等新表 +- [ ] 新增迁移文件，例如 `007_pyint_gamma_integration.sql` +- [ ] 在 `backend/app/db_maintenance.py` 中加入迁移列表 +- [ ] 验证 `ensure_database_ready()` 启动自动迁移 +- [ ] 验证幂等执行 + +### 阶段验收 + +- [ ] 新表结构不破坏现有 pairing 逻辑 +- [ ] 启动时可自动应用迁移 +- [ ] 老数据和老接口保持兼容 + +## 11. 联调与验收矩阵 + +### 后端 + +- [ ] `py_compile` 或等价语法检查通过 +- [ ] 新增路由可正常注册 +- [ ] 新增引擎可正常列出 +- [ ] 任务队列能执行 `PyINT` +- [ ] 精配对接口能生成 refined network + +### 前端 + +- [ ] `npm run build` 通过 +- [ ] 生产页能显示 `PyINT` +- [ ] 配对规划页能显示 `Gamma 精配对` +- [ ] 健康页能显示 PyINT 状态 + +### 集成 + +- [ ] `PyINT` 单任务最小链路跑通 +- [ ] 结果能被系统扫描 +- [ ] 精配对 MVP 跑通 +- [ ] 现有 SARscape/ISCE2 不回归 + +## 12. 当前明确不做 + +- [ ] 一期不把 PyINT 的全部模板参数开放到前端 +- [ ] 一期不接入 GACOS 自动邮箱下载链路 +- [ ] 一期不接入 POT、phase bias、完整时序 MintPy 流程 +- [ ] 一期不改写现有 `pairing_metric_cache` 字段语义 +- [ ] 一期不在健康检查页增加主操作面板 +- [ ] 一期不做多 WSL distro 管理 + +## 13. 当前建议的首批落地包 + +建议第一轮直接落以下内容： + +- [ ] `config.py` + `.env.example` 的 `PYINT_*` 配置 +- [ ] `pyint_service.py` +- [ ] `pyint_engine.py` +- [ ] `registry.py` 注册 +- [ ] `job_handlers.py` 的 `JOB_TYPE_PYINT_RUN` +- [ ] `dinsar_production.py` 的 `pyint` 提交分派 +- [ ] `run_lt1_pyint_pipeline.py` +- [ ] `DinsarProductionPanel.jsx` 的 `PyINT` 引擎展示与提交 + +这批完成后，再进入 `Gamma` 精配对 MVP。 diff --git a/docs/PYINT_GAMMA_INTEGRATION_DESIGN_20260418.md b/docs/PYINT_GAMMA_INTEGRATION_DESIGN_20260418.md new file mode 100644 index 0000000..325b5e5 --- /dev/null +++ b/docs/PYINT_GAMMA_INTEGRATION_DESIGN_20260418.md @@ -0,0 +1,601 @@ +# PyINT + Gamma 集成总体设计 + +**日期**: 2026-04-18 +**状态**: 总体设计 +**范围**: D-InSAR 生产引擎接入、Gamma 精配对接入、配置管理、数据库策略、运维自检、前端入口 + +## 1. 结论 + +本次集成建议采用两条并行但相互衔接的路线： + +1. 将 `PyINT` 作为新的 D-InSAR 生产引擎接入现有多引擎框架，统一走现有任务队列、运行日志、结果登记与结果目录治理链路。 +2. 将 `Gamma` 配对能力接入现有“配对基础 -> 配对规划 -> 生产执行”链路，作为数据库粗配对结果之上的精化步骤，而不是替换当前配对基础缓存。 + +核心判断如下： + +- `PyINT` 更适合作为“受控外部引擎”集成，而不是直接作为后端内部 Python 库深度嵌入。 +- `Gamma` 精配对是可集成的，但更适合针对“已经筛出的场景集合/网络运行”做二次优化，不适合直接取代当前全库候选对缓存。 +- 一期集成建议不强制修改数据库主结构；优先复用现有 `pairing_network_runs` / `pairing_network_edges` 的 JSON 承载精配对元数据。 +- 如果二期需要对 Gamma 精配对历史做独立检索、统计和运维闭环，再引入单独迁移文件，并通过现有数据库自维护机制自动落库。 + +## 2. 现状与约束 + +### 2.1 当前系统已有基础 + +- 已有 D-InSAR 多引擎抽象：`backend/app/dinsar_engines/base.py` +- 已有引擎注册表：`backend/app/dinsar_engines/registry.py` +- 已有生产任务接口与队列：`backend/app/routers/dinsar_production.py` +- 已有配对基础缓存、网络运行与边追踪： + - `backend/app/models/orm.py` + - `backend/app/services/pairing_cache_service.py` + - `backend/app/services/pairing_state_service.py` + - `backend/app/services/spatial_service.py` +- 已有数据库自维护与 SQL 迁移自动执行：`backend/app/db_maintenance.py` +- 已有运维自检面板与健康检查汇总：`backend/app/services/health_service.py`、`frontend/src/HealthCheckPanel.jsx` +- 已有生产页与配对规划页： + - `frontend/src/DinsarProductionPanel.jsx` + - `frontend/src/panels/PairPlanningPanel.jsx` + +### 2.2 PyINT 项目特征 + +从 `D:\Code\PyINT` 现状看，`PyINT` 不是干净的 SDK，而是以模板和脚本为中心的流程编排层： + +- 主入口为 `pyint/pyintApp.py` +- 配对能力入口为 `pyint/select_pairs.py` +- 严重依赖环境变量： + - `SCRATCHDIR` + - `TEMPLATEDIR` + - `DEMDIR` +- 运行方式偏 Linux / WSL，广泛调用外部命令与 GAMMA CLI +- 更适合作为“流程执行器”被调用，而不是被后端直接 import 后逐步复用内部函数 + +### 2.3 明确约束 + +- 本机只有一个 WSL 环境，不需要设计多 distro 调度系统。 +- 系统级 Windows Python 解释器已经在根 `.env` 中维护，可复用，不应再为 PyINT 额外复制一套 Windows Python 配置。 +- 管理员口令不应进入设计文档、代码或 `.env.example`。权限控制继续复用现有登录态与管理员角色校验。 +- 现有运维自检面板已经较重，PyINT/Gamma 的“操作入口”不应继续堆在健康检查页里。 + +## 3. 总体集成架构 + +### 3.1 总体原则 + +- 不新建平行子系统，优先复用现有引擎、作业、配对、结果目录与目录扫描体系。 +- 不改变现有 `pairing_metric_cache.spatial_baseline_meters` 的语义。 +- 不把 Gamma 精配对结果直接覆盖数据库粗配对缓存。 +- 运维页只看状态，实际操作放在生产页与配对规划页。 + +### 3.2 架构分层 + +#### A. 生产引擎层 + +新增 `pyint` 引擎，挂到现有 `registry` 中，与 `sarscape` / `isce2` / `landsar` 并列。 + +#### B. WSL 执行适配层 + +新增受控执行服务，负责： + +- 读取 `.env` 配置 +- 复用现有 WSL 命令执行与路径转换能力 +- 组装 `PyINT` 所需环境变量 +- 生成模板文件和运行目录 +- 执行 `PyINT` 包装脚本 +- 将输出归一化到系统现有结果结构 + +#### C. 配对精化层 + +保留现有数据库候选对缓存与网络运行。 + +在此基础上新增“Gamma 精配对”步骤： + +1. 先由当前配对接口生成候选网络 +2. 再将该网络对应场景集送入 Gamma / PyINT 配对流程 +3. 生成新的精化网络结果 +4. 前端允许用户查看并选择使用精化后的网络结果 + +#### D. 结果治理层 + +PyINT/Gamma 原始工作目录不直接作为系统正式结果。 + +必须经过适配层输出统一结果包，保证继续兼容： + +- 结果目录扫描 +- 结果目录发布 +- 结果目录桥接一致性 +- 预览图/缩略图生成 +- AI 诊断与 catalog 追踪 + +## 4. PyINT 生产引擎设计 + +### 4.1 目标 + +目标不是把 `PyINT` 原封不动暴露给用户，而是把它包装成当前系统理解的“一个可选生产引擎”。 + +### 4.2 推荐实现方式 + +新增以下后端组件： + +- `backend/app/dinsar_engines/pyint_engine.py` +- `backend/app/services/pyint_service.py` +- `backend/app/pyint_pipeline/run_lt1_pyint_pipeline.py` + +职责划分： + +- `pyint_engine.py` + - 实现 `DinsarEngine` + - 暴露 `engine_code=pyint` + - 提供可用性检查、处理 profile、参数 schema +- `pyint_service.py` + - WSL 执行 + - 路径转换 + - 模板写入 + - 环境变量组装 + - 烟测检查 +- `run_lt1_pyint_pipeline.py` + - 作为受控包装脚本在 WSL 中运行 + - 负责把系统已有 `Task_*` / 配对任务目录映射成 PyINT 项目工作区 + - 调用 `pyintApp.py` 或更细粒度子脚本 + - 收集输出并生成系统结果清单 + +### 4.3 与当前生产链路的关系 + +沿用现有生产链路： + +`前端生产页 -> /dinsar-production/run -> job queue -> pyint_engine.run() -> WSL -> PyINT -> 统一结果包 -> catalog/scan` + +这样做的收益： + +- 不需要新增独立任务中心 +- 不需要新增另一套运行日志 +- 不需要新增另一套前端生产入口 +- 与当前 `DinsarProductionPanel.jsx` 的多引擎 UI 完全兼容 + +### 4.4 输入与工作区组织 + +建议一期仍以当前系统已有的任务目录为输入，不要求用户先手工构造原生 PyINT 项目。 + +推荐工作区结构： + +- 系统输入根目录：沿用当前生产面板 `root_dir` +- PyINT 工作根目录：系统管理目录，例如 `backend/runtime/pyint_work` +- 模板目录：系统管理目录，例如 `backend/runtime/pyint_templates` +- 每次运行独立 `run_key` +- 每个 pair/task 独立 workspace，避免相互污染 + +### 4.5 结果输出策略 + +PyINT 原始输出不能直接作为系统正式结果目录暴露。 + +推荐新增“输出归一化”步骤，将 PyINT/Gamma 输出转为系统现有 bundle 约定，至少包含： + +- 结果主清单 +- 关键输出文件路径 +- 运行元数据 +- pair trace 信息 +- engine/profile 信息 + +必须保证与现有结果目录扫描机制兼容。 + +### 4.6 Profile 设计建议 + +一期建议只开放一个稳定 profile： + +- `lt1_gamma_dinsar` + +不建议一开始把 `PyINT` 全部开关都暴露到前端。应只暴露对当前业务必要的参数，例如： + +- 是否强制重跑 +- 多视参数 +- 相干阈值 +- geocode 开关 +- unwrap 开关 +- 超时 + +其余细节由模板生成器按系统默认值填充。 + +## 5. Gamma 精配对设计 + +### 5.1 目标定位 + +Gamma 精配对不替代当前数据库候选对缓存，而是建立在现有候选网络之上的二次精化机制。 + +推荐定位为： + +- 当前数据库配对：全库级、粗筛级、可快速响应 +- Gamma 精配对：项目级、网络级、精筛级、可生成更可靠的时空基线网络 + +### 5.2 推荐流程 + +1. 用户在现有配对规划页完成粗配对查询 +2. 后端返回 `network_run_id` +3. 用户在“Gamma 精配对”区域发起精化 +4. 系统根据该网络运行对应的场景集合，构建 PyINT/Gamma 工作区 +5. 调用 `select_pairs.py` / `base_calc` 生成精配对网络 +6. 后端将结果落回系统网络结果表示 +7. 前端展示“粗配对结果”和“Gamma 精配对结果”的对比摘要 +8. 用户选择使用哪一版网络继续生产 + +### 5.3 为什么不能直接覆盖当前 pairing cache + +当前 `pairing_metric_cache` 里的 `spatial_baseline_meters` 已经在系统内承担既有语义与下游用途。 +Gamma 计算出的垂直基线/网络属性与当前字段不等价，直接覆盖会带来语义混乱和回归风险。 + +因此必须坚持： + +- 现有缓存保留原语义 +- Gamma 精配对结果单独存储 +- 精配对结果仅作为网络选择依据，不回写粗配对主缓存 + +### 5.4 一期存储策略 + +一期推荐不新建强结构化表，优先复用： + +- `pairing_network_runs.request_params_json` +- `pairing_network_edges.selection_meta_json` + +建议约定写入内容： + +- `refinement_engine: gamma_pyint` +- `refinement_source_run_id` +- `gamma_bperp_m` +- `gamma_tbase_days` +- `gamma_rank` +- `gamma_ifgram_list_path` +- `gamma_artifact_dir` +- `gamma_selection_reason` + +同时新增一个新的 `network_run_id`，把“精配对结果”作为新的网络运行保存，而不是修改原粗配对运行。 + +这样做的收益： + +- 一期可不改数据库结构 +- 保留粗配对和精配对双轨结果，便于审计与回退 +- 复用现有 network run / edge 追踪模型 + +### 5.5 二期可选扩展 + +如果后续有以下需求，再引入数据库迁移： + +- 精配对历史独立检索 +- 精配对任务状态长期统计 +- 精配对工作区清理与资产追踪 +- 精配对失败类型聚合运维 + +二期建议新增表，例如： + +- `pairing_refinement_runs` +- `pairing_refinement_artifacts` + +但这不是一期必须项。 + +## 6. 配置与运行管理方案 + +### 6.1 配置原则 + +- Windows 侧解释器继续复用根 `.env` 中已有的 `PYTHON_PATH` +- WSL 侧只维护 PyINT/Gamma 运行必须配置 +- 因为本机只有一个 WSL 环境，`PyINT` 与 `ISCE2` 默认共用 distro + +### 6.2 建议新增配置项 + +建议在 `.env` / `.env.example` / `backend/app/config.py` 中新增： + +```ini +PYINT_ENABLED=false +PYINT_WSL_DISTRO= +PYINT_WSL_PYTHON= +PYINT_HOME= +PYINT_APP_SCRIPT= +PYINT_TEMPLATE_ROOT= +PYINT_WORK_ROOT= +PYINT_OUTPUT_ROOT= +PYINT_DEM_ROOT= +PYINT_GAMMA_ENV_SCRIPT= +PYINT_DEFAULT_TIMEOUT_SECONDS=43200 +PYINT_SMOKE_TEST_ENABLED=false + +PAIRING_GAMMA_ENABLED=false +PAIRING_GAMMA_WORK_ROOT= +PAIRING_GAMMA_TEMPLATE_ROOT= +PAIRING_GAMMA_TIMEOUT_SECONDS=7200 +``` + +默认策略建议： + +- `PYINT_WSL_DISTRO` 为空时，默认取 `ISCE2_WSL_DISTRO` +- `PYINT_WSL_PYTHON` 为空时，默认取 `ISCE2_PYTHON` +- `PYINT_APP_SCRIPT` 指向 `pyintApp.py` +- `PYINT_WORK_ROOT` / `PAIRING_GAMMA_WORK_ROOT` 使用系统托管目录，不直接让用户任意指定 + +### 6.3 不建议写入设计或配置的内容 + +- 管理员明文密码 +- ASF/GACOS 邮箱密码 +- WSL sudo 密码 + +这些信息如确需使用，也应通过运行时安全注入或机器本地安全配置处理，不写入仓库文档。 + +### 6.4 管理与治理策略 + +建议增加以下治理规则： + +- 所有 PyINT/Gamma 工作目录按 `run_key` 或 `network_run_id` 分目录 +- 所有运行都必须写运行摘要 JSON +- 所有正式产物必须进入统一结果发布目录 +- 中间工作区可按保留策略定期清理 +- 清理动作仅允许管理员执行 + +## 7. 数据库与数据库自维护策略 + +### 7.1 一期结论 + +一期建议： + +- `PyINT` 生产引擎接入不强制改库 +- `Gamma` 精配对接入不强制改库 +- 优先复用现有 run/edge JSON 元数据承载扩展信息 + +### 7.2 二期改库触发条件 + +当满足以下任意条件时，再进入改库： + +- 需要独立查询 Gamma 精配对运行历史 +- 需要单独统计 Gamma 精配对失败率 +- 需要把精配对资产纳入长期运维对象 +- 需要做更细粒度的后台管理界面 + +### 7.3 改库时的落地方式 + +如果二期改库，必须沿用现有数据库自维护机制： + +1. 在 `backend/migrations/` 新增 SQL 迁移文件，例如 `007_pyint_gamma_integration.sql` +2. 在 `backend/app/db_maintenance.py` 的 `MIGRATION_FILES` 中追加文件名 +3. 由 `ensure_database_ready()` 在启动时自动执行迁移 + +约束： + +- 不修改既有字段语义 +- 不破坏现有 `pairing_metric_cache` / `pairing_network_*` 查询逻辑 +- 迁移必须支持重复执行幂等 + +## 8. 运维自检与健康检查设计 + +### 8.1 设计原则 + +运维自检页继续只做“状态观察”，不做主操作入口。 + +PyINT/Gamma 的正式操作入口放在： + +- 生产页 +- 配对规划页 + +### 8.2 健康检查应新增的内容 + +建议在引擎可用性检查中加入 PyINT 项： + +- `PYINT_ENABLED` +- WSL distro 可访问 +- WSL Python 可执行 +- `PYINT_HOME` 存在 +- `pyintApp.py` 存在 +- `GAMMA_ENV_SCRIPT` 可 source +- 关键命令如 `base_calc` 可执行 +- 模板目录可读 +- 工作目录可写 +- DEM 根目录可读 + +### 8.3 健康页展示策略 + +不建议在 `HealthCheckPanel.jsx` 再新增一大块复杂操作区。 + +建议只保留两类展示： + +1. 在现有 `D-InSAR 引擎` 卡片中自然显示 `PyINT` +2. 在健康详情或备注中显示 PyINT/Gamma 的简要检查摘要 + +不建议： + +- 在健康页提供精配对执行按钮 +- 在健康页提供模板编辑入口 +- 在健康页堆叠大量结果目录说明 + +### 8.4 运维修复入口位置 + +- 引擎级问题：在生产页提示不可用原因 +- 配对级问题：在配对规划页处理 +- 只有“环境诊断/烟测”可以保留在运维页 + +## 9. 后端接口设计 + +### 9.1 生产接口 + +现有 `/dinsar-production/engines` 和 `/dinsar-production/run` 可继续复用。 + +需要做的只是： + +- 在引擎注册表中加入 `pyint` +- `list_engines()` 自动返回 PyINT +- `submit_run()` 允许 `engine_code=pyint` + +### 9.2 配对接口 + +建议新增以下接口： + +- `POST /pairing/refine-gamma` + - 输入：`network_run_id` 或明确场景列表 + - 输出：新的精配对 `network_run_id`、摘要、警告、产物位置 +- `GET /pairing/networks/{network_run_id}` + - 继续复用现有接口查看粗配对/精配对网络详情 +- 可选：`GET /pairing/refine-gamma/{network_run_id}/artifacts` + - 用于查看 artifact 摘要，不建议一期必做 + +### 9.3 管理接口 + +建议增加一个轻量管理接口用于 PyINT/Gamma 环境烟测，例如： + +- `POST /dinsar-production/engines/pyint/smoke-check` + +用途仅限管理员环境校验，不参与正式生产提交。 + +## 10. 前端入口与交互布局 + +### 10.1 生产页 + +位置：`frontend/src/DinsarProductionPanel.jsx` + +建议改动： + +- 新增 `PyINT` 引擎卡片 +- 显示 PyINT 可用性状态 +- 根据 profile 展示少量必要参数 +- 保留当前“根目录 + 参数 + 提交任务”交互，不新造独立页面 + +### 10.2 配对规划页 + +位置：`frontend/src/panels/PairPlanningPanel.jsx` + +建议新增一个独立区域： + +- 标题：`Gamma 精配对` +- 放置位置：`配对基础` 卡片下方，`结果与刷新` 卡片上方 + +该区域建议包含： + +- 粗配对网络摘要 +- 发起 Gamma 精配对按钮 +- 精配对结果摘要 +- 粗配对 / 精配对差异提示 +- 选择采用哪一版网络继续生产 + +不建议把精配对塞进现有健康检查页。 + +### 10.3 健康检查页 + +位置：`frontend/src/HealthCheckPanel.jsx` + +建议只做最小改动： + +- 让 `D-InSAR 引擎` 卡片中自动出现 `PyINT` +- 如需要，增加一条 PyINT/Gamma 环境说明 + +不增加复杂控制区，避免界面继续变重。 + +## 11. 涉及改动位置 + +### 11.1 后端 + +- `backend/app/dinsar_engines/registry.py` +- `backend/app/dinsar_engines/pyint_engine.py` 新增 +- `backend/app/services/pyint_service.py` 新增 +- `backend/app/pyint_pipeline/run_lt1_pyint_pipeline.py` 新增 +- `backend/app/routers/dinsar_production.py` +- `backend/app/routers/pairing.py` +- `backend/app/services/health_service.py` +- `backend/app/config.py` +- `.env.example` + +### 11.2 前端 + +- `frontend/src/DinsarProductionPanel.jsx` +- `frontend/src/panels/PairPlanningPanel.jsx` +- `frontend/src/HealthCheckPanel.jsx` +- `frontend/src/api/dinsarProduction.js` +- `frontend/src/api/pairing.js` + +### 11.3 数据库 + +一期可不改。 + +二期若改，涉及： + +- `backend/migrations/007_pyint_gamma_integration.sql` 新增 +- `backend/app/db_maintenance.py` + +## 12. 分阶段实施建议 + +### Phase 1: PyINT 引擎接入 + +- 新增 `pyint_engine` +- 完成 WSL 可用性检查 +- 完成模板生成与工作目录治理 +- 完成生产页引擎选择 +- 完成结果归一化与目录扫描兼容 + +### Phase 2: Gamma 精配对 MVP + +- 新增 `/pairing/refine-gamma` +- 基于现有 `network_run_id` 做精化 +- 精配对结果复用现有 network run / edge 模型表达 +- 前端在配对规划页增加精配对区块 + +### Phase 3: 运维与治理补齐 + +- 增加烟测接口 +- 增加工作区清理策略 +- 增加 artifact 摘要与失败类型归档 + +### Phase 4: 二期结构化增强 + +- 若业务确认需要，再加数据库迁移 +- 把精配对历史与资产纳入更细粒度可检索对象 + +## 13. 风险与规避 + +### 13.1 PyINT 代码稳定性 + +风险： + +- 模板字段和脚本依赖较多 +- 对目录命名和环境变量较敏感 + +规避： + +- 不做深度 import 复用 +- 使用受控包装脚本 +- 限制一期只开放一个稳定 profile + +### 13.2 WSL 与路径问题 + +风险： + +- Windows 路径和 WSL 路径混用 +- 工作区权限与可写性问题 + +规避： + +- 所有路径统一通过适配层转换 +- 工作目录和模板目录由系统托管 + +### 13.3 配对语义污染 + +风险： + +- 把 Gamma 垂直基线直接混写进现有粗配对缓存字段 + +规避： + +- 明确不覆盖 `pairing_metric_cache` 语义 +- 精配对结果单独落在网络运行元数据中 + +### 13.4 前端继续膨胀 + +风险： + +- 把运维、配对、生产操作继续堆到健康检查页 + +规避： + +- 健康页只显示状态 +- 生产操作只放生产页 +- 配对操作只放配对规划页 + +## 14. 最终建议 + +建议按以下判断执行： + +- `PyINT` 生产引擎接入：必要，且应尽快按现有多引擎架构落地。 +- `Gamma` 精配对接入：可行，但应作为“粗配对之后的精化层”落地。 +- 数据库：一期不强制改库；二期若需更强管理能力，再走数据库自维护迁移。 +- 运维自检：只加状态，不加大块操作区。 +- 前端入口：生产页接 `PyINT`，配对规划页接 `Gamma 精配对`。 + diff --git a/docs/PYINT_INPUT_ASSET_ADAPTATION_DESIGN_20260419.md b/docs/PYINT_INPUT_ASSET_ADAPTATION_DESIGN_20260419.md new file mode 100644 index 0000000..ff2a5c9 --- /dev/null +++ b/docs/PYINT_INPUT_ASSET_ADAPTATION_DESIGN_20260419.md @@ -0,0 +1,554 @@ +# PyINT 输入资产适配设计 + +**日期**: 2026-04-19 +**状态**: 总体设计 +**范围**: `Task_*` 路径适配、PyINT DEM 管理、LT-1 精密轨道治理、Gamma 配对前置条件、运维自检与前端入口 + +## 1. 结论 + +本次设计的核心结论如下： + +1. 用户侧继续沿用现有的 `Task_*` 输入模式，不要求手工准备原生 PyINT 项目目录，也不允许直接把任意外部路径当作长期运行依赖。 +2. 需要在现有 `pyint_engine -> run_lt1_pyint_pipeline.py` 之间补一层“输入资产适配层”，把 `Task_*`、DEM、精密轨道统一解析为系统托管的运行输入。 +3. DEM 可以在一期做到“系统托管且真实参与计算”，推荐优先走“本地 FABDEM/DEM 瓦片源 + PyINT 本地生成 DEM 产物”的方案，而不是直接复用 ISCE2 的 `.wgs84` 成品 DEM。 +4. LT-1 精密轨道在当前 PyINT 原生 LT-1 导入链路里，还没有现成的“接入系统轨道池并直接参与计算”的钩子。一期先做“治理级校验 + 按任务解析 + 随跑记录 + 可选准入阻断”，二期再补“真正参与 PyINT/Gamma 计算”的桥接。 +5. 一期不必改数据库结构，先把输入资产记录写入 `.dinsar_run.json`、`pyint_run_summary.json` 和结果 manifest 的扩展摘要。二期只有在需要按 DEM/轨道版本检索历史时才改库，并且必须走现有数据库自维护迁移机制。 +6. 前端主入口应放在现有 D-InSAR 生产面板的 PyINT 引擎区域；运维自检面板只保留状态摘要，不再堆叠新的操作区。 + +## 2. 现状与缺口 + +### 2.1 已经具备的部分 + +- `PyINT` 代码已经收编到仓库内 `third_party/PyINT`，不再依赖仓库外绝对路径。 +- 当前后端已经支持： + - `root_dir` 为单个任务目录，或为包含多个 `Task_*` 子目录的父目录 + - 对每个任务递归发现 `master/`、`slave/` 下的 `LT1*.tar.gz` + - 自动生成 `ifgram_list.txt` + - 自动生成 PyINT template + - 在 `backend/runtime/pyint_work` 下构造 PyINT 工作区并调用 `pyintApp.py` +- 当前系统已有成型的精轨治理链路： + - `MONITOR_ORBIT_DIR` 作为源目录 + - `ORBIT_POOL_ENVI` 作为 LT-1 `.txt` 精轨池 + - `ORBIT_POOL_ISCE2` 作为 ISCE2 `.xml` 精轨池 + - `orbit_converter.py` 已支持同步、修复、隔离和一致性检查 +- 当前系统已有成型的健康检查和目录治理链路： + - `health_service.py` + - `root_registry_service.py` + - 结果目录扫描和 manifest catalog + +### 2.2 目前还没有解决的部分 + +- 当前 PyINT 集成只解决了“`Task_*` 到 PyINT 工作区”的映射，没有解决“系统托管 DEM / 系统托管精轨资产如何进入 PyINT”。 +- 当前 `PYINT_DEM_ROOT` 只是 PyINT 的运行目录或缓存目录，不等价于“系统已经为本次任务解析好了 DEM 输入策略”。 +- 当前 LT-1 PyINT 导入脚本并没有直接消费系统里的 `LT1*_GpsData_GAS_C_YYYYMMDD.txt` 精轨池。 +- 当前前端也没有给 PyINT 提供“提交前资产预检/预览”的位置。 + +### 2.3 一个必须明确的现实约束 + +当前 vendored `PyINT` 的 LT-1 流程里： + +- DEM 侧已有明确入口，`makedem_pyint.py` 可以走本地 `fabdem_dir` 或 OpenTopography。 +- 精轨侧对 LT-1 没有现成的“使用系统 `.txt` 精轨池”的显式接口，现有 LT-1 导入脚本更接近“从压缩包和 XML 元数据生成 SLC 参数”。 + +因此本方案必须分两层描述精轨： + +1. 治理层接入：系统知道本次任务应该使用哪份精轨，能阻断缺失任务，能把依赖记录下来。 +2. 计算层接入：该精轨是否真的被 PyINT/Gamma 的 LT-1 导入过程消费。 + +一期只能承诺第一层，第二层需要专门桥接。 + +## 3. 总体方案 + +### 3.1 新增一层输入资产适配服务 + +建议在 `pyint_service.py` 旁边新增或内聚出一层输入资产适配职责，例如： + +- `resolve_pyint_tasks(root_dir)` +- `resolve_pyint_dem_asset(task_context)` +- `resolve_pyint_orbit_assets(task_context)` +- `materialize_pyint_input_assets(run_context)` +- `build_pyint_input_preview(root_dir)` + +其职责不是替代 PyINT，而是在系统生产语义和 PyINT 原生语义之间做转换。 + +### 3.2 总体执行链路 + +建议链路如下： + +`前端生产面板 root_dir` +-> `validate_pyint_root_dir()` +-> `PyINT 输入资产适配层` +-> `每个 Task_* 解析任务身份、DEM、精轨` +-> `运行目录 materialize` +-> `run_lt1_pyint_pipeline.py` +-> `PyINT / Gamma` +-> `pyint_run_summary.json + .dinsar_run.json` +-> `结果发布 / catalog` + +### 3.3 不再要求用户准备 PyINT 原生目录 + +用户仍然只需要提供： + +- 单个 `Task_YYYYMMDD_YYYYMMDD` +- 或者一个包含多个 `Task_*` 的批次根目录 + +系统内部自行生成： + +- `project_name` +- `template` +- `DOWNLOAD/` +- `ifgram_list.txt` +- `input_assets/` +- `native output` + +这保证 PyINT 继续是“受控执行器”，不是“要求用户手工维护目录结构的第二套系统”。 + +## 4. 与现有 `Task_*` 路径的配合方式 + +### 4.1 用户输入模式 + +沿用当前模式，不新增新的路径输入方式： + +- 模式 A：直接选一个 `Task_*` +- 模式 B：选一个包含多个 `Task_*` 的父目录 + +任务目录仍要求至少满足： + +- `master/` +- `slave/` +- 目录下可递归发现 `LT1*.tar.gz` + +可选但推荐继续保留： + +- `.dinsar_pair.json` + +### 4.2 任务解析规则 + +建议继续沿用当前逻辑，并把它明确固化为正式约束： + +1. `Task_*` 是业务输入根，不是 PyINT 工作区。 +2. `master/`、`slave/` 下面允许多层子目录，但最终必须能发现原始压缩包。 +3. 任务身份优先从 `.dinsar_pair.json` 读取。 +4. 缺失时再从任务目录名和压缩包文件名推导： + - `task_alias` + - `pair_key` + - `master_date` + - `slave_date` + +### 4.3 运行期目录建议 + +建议把每次运行的托管结构固定为： + +```text +backend/runtime/pyint_work/<pair_key>/<run_key>/ + input_assets/ + task_manifest.json + orbits/ + dem/ + <project_name>/ + DOWNLOAD/ + ifgram_list.txt + ... + +backend/runtime/pyint_templates/<pair_key>/<run_key>/ + <project_name>.template + +backend/runtime/pyint_output/<pair_key>/<run_key>/native/ + pyint_run_summary.json + .dinsar_run.json + ifgrams/ + ... +``` + +原则： + +- 原始 `Task_*` 只读，不回写。 +- 每次运行独立目录，避免不同 run 相互污染。 +- DEM、轨道、任务解析结果要在 `input_assets/` 下留痕。 + +## 5. DEM 方案 + +### 5.1 不建议直接把 ISCE2 DEM 方案硬套给 PyINT + +当前系统已有 `ISCE2_DEM_PATH`，它对应的是 ISCE2 直接消费的成品 DEM。 + +但当前 PyINT 的 DEM 处理逻辑更接近： + +- 先根据 master SLC 范围解析 DEM 覆盖区域 +- 再通过 `makedem_pyint.py` +- 结合 `fabdem_dir` 或 OpenTopography +- 在 `DEMDIR` 下生成 PyINT / Gamma 所需的 DEM 产物 + +因此一期不建议把 `PYINT_DEM_SOURCE` 简单绑定为 `ISCE2_DEM_PATH`。 + +### 5.2 推荐的 DEM 分层 + +建议把 PyINT 的 DEM 分成三层： + +1. DEM 源 + - 本地 FABDEM/DEM 瓦片根目录 + - 或 OpenTopography 在线源 +2. DEM 运行缓存 + - 即当前 `PYINT_DEM_ROOT` +3. 本次任务解析后的 DEM 产物 + - 位于 `PYINT_DEM_ROOT/<project_name>/...` + - 被 `generate_rdc_dem.py`、`geocode_gamma.py` 等步骤消费 + +### 5.3 推荐配置 + +建议新增或明确以下配置： + +```ini +PYINT_DEM_MODE=local_fabdem|opentopo +PYINT_FABDEM_ROOT= +PYINT_OPENTOPO_DEM_TYPE=SRTMGL1 +PYINT_DEM_ROOT= +PYINT_DEM_STRICT=true +``` + +说明： + +- `PYINT_DEM_MODE=local_fabdem` 为推荐默认值。 +- `PYINT_FABDEM_ROOT` 指向本机统一维护的 FABDEM/DEM 瓦片目录。 +- `PYINT_DEM_ROOT` 继续作为 PyINT DEM 运行缓存根。 +- 若后续确实验证可直接复用某个成品 DEM，再新增单独模式，不要和一期混在一起。 + +### 5.4 运行时行为 + +当用户提交 PyINT 任务时： + +1. 适配层先解析 DEM 模式。 +2. 若为 `local_fabdem`： + - 把 `fabdem_dir` 写入本次运行生成的 template + - `DEMDIR` 指向本次受控缓存根 +3. 若为 `opentopo`： + - 只在运行时注入 API key，不把敏感值写入仓库文档或 `.env.example` +4. 运行完成后记录： + - DEM 模式 + - DEM 源根目录 + - 生成产物目录 + - 关键 DEM 文件是否生成成功 + +### 5.5 DEM 与前端的关系 + +不建议在前端让用户手工输入单次 DEM 路径。 + +推荐做法是： + +- 前端只展示“当前 DEM 策略” +- 例如： + - `本地 FABDEM` + - `OpenTopography` + - `未配置` +- 如果 DEM 不可用，则在 PyINT 引擎区阻断提交 + +## 6. 精密轨道方案 + +### 6.1 一期目标不是“假装已经真正进计算” + +当前 LT-1 PyINT 原生脚本没有明确消费系统精轨池的接口，因此一期要把目标定义准确： + +- 系统必须能按任务解析 master/slave 对应的精轨文件 +- 系统必须能知道精轨是否缺失 +- 系统必须把这次运行实际匹配到的精轨记录下来 +- 系统必须能根据策略决定“警告放行”还是“阻断提交” + +但不能在未完成桥接前，对外宣称“精轨已经真实参与 LT-1 PyINT 计算”。 + +### 6.2 一期建议的精轨策略 + +建议精轨配置分为： + +```ini +PYINT_ORBIT_POLICY=validate_only|require_txt|stage_txt +PYINT_ORBIT_POOL_TXT= +PYINT_RECORD_INPUT_ASSETS=true +``` + +默认建议： + +- `PYINT_ORBIT_POOL_TXT` 为空时默认继承 `ORBIT_POOL_ENVI` +- `PYINT_ORBIT_POLICY=require_txt` + +三种策略含义： + +- `validate_only` + - 找得到则记录 + - 找不到只警告 +- `require_txt` + - 找不到直接阻断运行 +- `stage_txt` + - 除了要求存在，还把匹配到的轨道文件复制或硬链接到本次运行目录 + +### 6.3 轨道解析规则 + +对每个 task，按如下顺序解析： + +1. 从 `.dinsar_pair.json`、原始压缩包文件名或元数据确定： + - 卫星 `LT1A/LT1B` + - `master_date` + - `slave_date` +2. 到系统轨道池中查找： + - `LT1A_GpsData_GAS_C_YYYYMMDD.txt` + - `LT1B_GpsData_GAS_C_YYYYMMDD.txt` +3. 分别解析 master/slave 结果 +4. 形成本次运行的轨道摘要 + +### 6.4 与现有轨道治理链路的关系 + +PyINT 不应新建第二套精轨目录。 + +应直接复用现有治理链路： + +- 源目录：`MONITOR_ORBIT_DIR` +- 运行池：`ORBIT_POOL_ENVI` +- 一致性修复：`orbit_converter.py` +- 健康检查：`health_service.py` + +也就是说： + +- PyINT 的精轨输入来源仍应是系统托管的轨道池 +- 不是让用户每次在前端再手工填一条轨道路径 + +### 6.5 一期的落地方式 + +建议每次运行都在 `input_assets/orbits/` 下落盘一个轨道摘要，例如： + +```json +{ + "policy": "require_txt", + "pool_root": "D:\\orbit_pools\\envi", + "master": { + "date": "20250112", + "satellite": "LT1A", + "path": "D:\\orbit_pools\\envi\\LT1A\\LT1A_GpsData_GAS_C_20250112.txt", + "staged_path": "...\\input_assets\\orbits\\LT1A_GpsData_GAS_C_20250112.txt", + "resolved": true + }, + "slave": { + "date": "20250309", + "satellite": "LT1A", + "path": "D:\\orbit_pools\\envi\\LT1A\\LT1A_GpsData_GAS_C_20250309.txt", + "staged_path": "...\\input_assets\\orbits\\LT1A_GpsData_GAS_C_20250309.txt", + "resolved": true + } +} +``` + +这一步先解决： + +- 任务是否可跑 +- 运行可追溯 +- 后续桥接可复用 + +### 6.6 二期的“真正参与计算”桥接 + +如果要让系统精轨真实参与 LT-1 PyINT/Gamma 计算，建议单独做一个技术 Spike，候选方向有两个： + +1. 修改或包装 PyINT 的 LT-1 导入步骤 + - 在 `down2slc_LT1.py` / `LT1_import_SLC_from_zipfiles1` 前后插入系统精轨桥接步骤 +2. 在 PyINT 前增加一个 LT-1 预处理适配器 + - 先把系统精轨和原始场景解析成更稳定的中间输入 + - 再把中间输入交给 PyINT 后续流程 + +建议优先方向是第 1 种，因为它改动面更小。 + +但在明确 Gamma 对 LT-1 外部精轨的实际消费方式之前，不建议直接承诺实现周期。 + +## 7. Gamma 配对集成的关系 + +`select_pairs.py` / Gamma 精配对本身是可集成的，但它不应绕开输入资产治理。 + +建议关系如下： + +1. 生产引擎侧先把 PyINT 的 DEM / 轨道输入治理打通。 +2. Gamma 精配对继续作为“配对规划之后的精化步骤”存在。 +3. 精配对任务默认继承同一套： + - WSL 环境 + - PyINT vendored 代码 + - 轨道治理配置 +4. 前端入口仍放在 `PairPlanningPanel`，不放进运维自检。 + +换句话说： + +- “生产引擎接入”是必要前置。 +- “Gamma 配对接入”是可行的，但它应复用同一套输入资产治理，而不是另起一套路径和配置。 + +## 8. 运行元数据、结果治理与数据库策略 + +### 8.1 一期不改数据库主结构 + +一期建议不改库，原因是： + +- 当前已有 `.dinsar_run.json` +- 当前已有 `pyint_run_summary.json` +- 当前已有结果 manifest / catalog + +这些已经足够承载输入资产摘要。 + +### 8.2 一期建议记录的内容 + +建议把以下信息写入运行元数据： + +- `input_assets.task_source` + - `root_dir` + - `task_dir` + - `archives.master[]` + - `archives.slave[]` +- `input_assets.dem` + - `mode` + - `source_root` + - `cache_root` + - `resolved_output_dir` + - `key_outputs` +- `input_assets.orbits` + - `policy` + - `pool_root` + - `master` + - `slave` + - `stage_mode` + +### 8.3 二期改库触发条件 + +只有出现以下需求时再改库： + +- 需要按 DEM 版本检索历史 PyINT 结果 +- 需要按轨道版本检索历史 PyINT 结果 +- 需要统计“某批结果使用了哪套轨道/DEM” +- 需要把 PyINT 输入资产做成后台长期查询对象 + +### 8.4 如果改库，必须走现有数据库自维护机制 + +如果进入二期改库，必须： + +1. 在 `backend/migrations/` 新增 SQL 迁移文件 +2. 在 `backend/app/db_maintenance.py` 的迁移列表中登记 +3. 让现有数据库自维护机制自动执行 + +不允许手工改表绕过现有机制。 + +## 9. 健康检查、接口与前端位置 + +### 9.1 运维自检面板只做状态，不做主操作入口 + +当前健康页已经比较重，因此新增内容应控制在“状态摘要”层面： + +- `PyINT enabled` +- `PyINT home` +- `PyINT WSL` +- `PyINT DEM strategy` +- `PyINT orbit policy` +- `PYINT_FABDEM_ROOT` 可读 +- `PYINT_ORBIT_POOL_TXT` / `ORBIT_POOL_ENVI` 可读 + +不建议新增： + +- 手工触发 PyINT 任务按钮 +- 手工填 DEM 路径 +- 手工填轨道路径 + +### 9.2 生产页的建议位置 + +在 `DinsarProductionPanel` 的 PyINT 引擎区域增加“输入资产预检摘要”，展示： + +- 识别到的任务数 +- 无效任务数 +- DEM 策略 +- 轨道策略 +- 已解析轨道数量 +- 缺失轨道数量 +- 是否允许提交 + +### 9.3 建议新增一个轻量预检接口 + +建议新增： + +- `POST /dinsar-production/engines/pyint/preview-input-assets` + +返回： + +- 任务解析结果 +- DEM 配置状态 +- 轨道解析状态 +- 阻断原因 +- 警告列表 + +这样前端可以在正式提交前给出明确反馈，而不是等任务进入队列后才失败。 + +### 9.4 PairPlanning 页的位置 + +Gamma 精配对仍建议放在 `PairPlanningPanel`： + +- 先显示已有粗配对网络 +- 再提供 Gamma 精配对入口 +- 不把它塞回健康检查页 + +## 10. 推荐实施顺序 + +### Phase 1：输入资产适配层 + +- 固化 `Task_*` 路径解析规则 +- 新增输入资产预检模型 +- 生成 `task_manifest.json` +- 生成 `input_assets/orbits/` 与 `input_assets/dem/` 目录 + +### Phase 2：DEM 正式接入 + +- 增加 `PYINT_DEM_MODE` +- 增加 `PYINT_FABDEM_ROOT` +- 在 template 生成时注入 `fabdem_dir` / `opentopo_*` +- 把 DEM 解析摘要写入 run metadata + +### Phase 3：精轨治理接入 + +- 增加 `PYINT_ORBIT_POLICY` +- 复用 `ORBIT_POOL_ENVI` +- 运行前做 master/slave 精轨解析 +- 缺失时按策略阻断 +- 把轨道文件 staging 到运行目录 + +### Phase 4：前端预检入口 + +- 生产面板显示资产预检摘要 +- 健康页只增加状态项 + +### Phase 5：精轨计算桥接 Spike + +- 研究 LT-1 PyINT/Gamma 当前导入链路如何真正消费外部精轨 +- 决定是补包装步骤还是补脚本修改 + +### Phase 6：Gamma 精配对集成 + +- 在配对规划页复用同一套 PyINT/Gamma 环境和资产治理策略 + +## 11. 最终建议 + +对于你提出的三个问题，建议明确回答如下： + +1. `Task_*` 路径怎么配合 + 继续沿用现在的任务目录，不改用户输入方式；系统内部新增适配层把任务目录转换为 PyINT 工作区。 + +2. DEM 怎么处理 + 一期就做系统托管，推荐以本地 FABDEM/DEM 源目录为标准输入，由 PyINT 在受控 `DEMDIR` 下生成本次任务真正使用的 DEM 产物。 + +3. 精密轨道怎么处理 + 一期先接入系统精轨池做校验、阻断、记录和 staging；二期再补“真实参与 LT-1 PyINT/Gamma 计算”的桥接。当前实现还不能直接把这一步视为已经完成。 + +基于当前代码现状，最稳妥的方向不是“删掉 Task_* 模式重新发明一套 PyINT 路径”，而是“把 Task_* 保留为业务输入，把 DEM/精轨补成系统托管资产适配层”。 + +## 补充：现有 DEM 复用实现（2026-04-19） + +当前代码已补充 `PYINT_DEM_MODE=prepared_file`，用于复用系统现有 DEM 资产。 + +- `PYINT_PREPARED_DEM_PATH` 可显式指定现有 DEM 基础文件。 +- 若该值为空，运行时会按顺序回退解析 `ISCE2_DEM_PATH` 和 `IDL_DINSAR_DEM_BASE_FILE`。 +- 如果目标文件同名存在 `.par`，则视为现成的 Gamma DEM，直接写入 PyINT 模板中的 `DEM=...`。 +- 如果目标文件没有 `.par`，但同名存在 `.xml`、`.hdr` 或 `.vrt`，则视为系统现有源 DEM。 +- 对“系统现有源 DEM”，PyINT 在 `makedem_pyint.py` 中会根据 `master` 的 `SLC_par` 覆盖范围先裁剪局部窗口，再转换为本次任务使用的 Gamma DEM。 +- 这样可以复用系统已经维护的中国区或全局 DEM，不必强制切回 FABDEM 或重新在线下载。 + +这个实现的约束也需要明确： + +- 现有源 DEM 仍必须是可被 GDAL 打开的本地文件。 +- 运行环境里仍需要 `gdal_translate` 可用，因为裁剪发生在 WSL/PyINT 侧。 +- 该模式的本质不是“直接把 ISCE2 DEM 原样交给 Gamma”，而是“把系统现有 DEM 当作受控源，再为每次 PyINT 任务生成 Gamma 可消费的局部 DEM 产物”。 diff --git a/docs/PYINT_LT1_COREG_ORBIT_HYPOTHESIS_EXPERIMENT_20260420.md b/docs/PYINT_LT1_COREG_ORBIT_HYPOTHESIS_EXPERIMENT_20260420.md new file mode 100644 index 0000000..e5c1e98 --- /dev/null +++ b/docs/PYINT_LT1_COREG_ORBIT_HYPOTHESIS_EXPERIMENT_20260420.md @@ -0,0 +1,343 @@ +# PyINT LT-1 `coreg` 失败轨道假设验证实验 + +**日期**: 2026-04-20 +**状态**: 实验设计 +**目标问题**: 验证当前 LT-1 在 PyINT/Gamma 中 `coreg/init_offsetm` 失败，是否主要由 `par_LT1_SLC` 导入后的 orbit/state vector 处理缺失导致 + +## 1. 结论先行 + +这次实验不再重复验证配对逻辑，也不再重复验证 `Task_*` 路径组织、DEM 来源或多景输入形态。 + +本实验只回答一个更窄的问题: + +> 在保持同一批 LT-1 影像、同一 master、同一 DEM、同一 `coreg` 参数不变的前提下，只改变 `.slc.par` 中的 state vector 处理方式，是否会显著改变 `coreg/init_offsetm` 的失败行为。 + +如果答案是“会”，则当前问题主要集中在 LT-1 导入后的轨道几何链条。 +如果答案是“不会”，则需要把排查重点转回 LT-1 导入本身的几何建模或 `MLI/SLC` 生成环节。 + +## 2. 已知事实 + +当前已经有一个可复现的 3 景隔离实验环境: + +- 实验根目录: `D:\PyINT_POOL_TEST\LT1A_MONO_SYC_STRIP1_E129.6_N45.0_3scene` +- master: `20230726` +- slave: + - `20230624` + - `20230920` +- 已确认成功的阶段: + - `down2slc_all` + - `makedem_pyint` + - `generate_rdc_dem` +- 已确认失败的阶段: + - `coreg_gamma_all` +- 当前稳定失败特征: + - `init_offsetm failed` + - `ERROR: number of zero values ... exceeds threshold: 32768` + +这说明: + +1. 输入组织已经足够让 PyINT 跑到 `coreg`。 +2. DEM 不构成这轮失败的主因。 +3. 问题更像是 `coreg` 所依赖的几何输入有问题，尤其是 `.slc.par` 的 orbit/state vector 链条。 + +## 3. 实验假设 + +### H1: 当前主假设 + +`par_LT1_SLC` 导入后的 `.slc.par` 还需要额外的 orbit/state vector 处理。 + +这个处理至少可能包括两类: + +1. 对现有 state vectors 做平滑/过滤 +2. 用系统已有 LT-1 精密轨道 TXT 重写 state vectors，再做校验 + +### H0: 零假设 + +即使对 state vectors 做上述处理，`coreg/init_offsetm` 的失败模式也基本不变。 +若如此，则缺陷更可能位于: + +- LT-1 导入后的几何参数生成 +- `MLI` 生成质量 +- `deskew` / 时序 / 采样参数 +- 或 `par_LT1_SLC` 本身不适用于当前这批数据 + +## 4. 实验原则 + +本实验必须严格控制变量，避免再把多种问题混在一起。 + +固定不变的部分: + +- 同一批 3 景 LT-1 数据 +- 同一个 master: `20230726` +- 同一套 `range_looks=2`, `azimuth_looks=2` +- 同一个 DEM 数据源 +- 同一个 `coreg_gamma.py` +- 同一个 PyINT/Gamma 环境 + +唯一允许变化的部分: + +- `.slc.par` 内 state vector 的处理方式 + +不在本轮变化范围内: + +- 不切换配对策略 +- 不切换 DEM +- 不改 `coreg rescue` +- 不改 `select_pairs` +- 不重做新的场景池选择 + +## 5. 分组设计 + +### A 组: 基线组 + +目的: 复现当前失败，作为所有对照的基准。 + +处理方式: + +- 使用 `par_LT1_SLC` 当前直接生成的 `.slc.par` +- 不做任何 orbit/state vector 改写 +- 重新执行: + - `generate_rdc_dem` + - `coreg` 到每个 slave + +预期: + +- 与现有日志一致 +- 两个 slave 都在 `init_offsetm` 附近失败 + +### B 组: 仅做 Gamma orbit filter + +目的: 验证“问题是否只是 state vector 需要过滤/平滑，而不一定需要外部精轨替换”。 + +处理方式: + +- 在 A 组同源 `.slc.par` 副本上，仅对 state vectors 做 Gamma orbit filtering +- 不引入系统外部 LT-1 精密轨道 TXT +- 之后重新执行: + - `generate_rdc_dem` + - `coreg` + +判定意义: + +- 如果 B 组明显优于 A 组，说明 `par_LT1_SLC` 的原始 state vectors 质量不足，但问题可能主要是“需要轨道过滤” +- 如果 B 组与 A 组几乎一样失败，则仅做过滤不够 + +### C 组: 精密轨道重写组 + +目的: 验证“问题是否是导入后缺少外部精密轨道替换”。 + +处理方式: + +- 使用系统已有 LT-1 精密轨道 TXT +- 通过 [apply_lt1_precise_orbit.py](/D:/Code/Insar_management_system_v2/backend/app/pyint_pipeline/apply_lt1_precise_orbit.py) 重写 `.slc.par` 的 `state_vector_*` +- 插值目标时间栅格沿用 `.slc.par` 自身: + - `number_of_state_vectors` + - `time_of_first_state_vector` + - `state_vector_interval` +- 重写后重新执行: + - `generate_rdc_dem` + - `coreg` + +判定意义: + +- 如果 C 组明显优于 A 组，而 B 组没有明显改善，则说明问题更偏向“缺少精密轨道重写” +- 如果 C 组和 B 组都改善，则说明“导入后的轨道链条不完整”成立，但是否必须引入外部精轨还需进一步量化 + +### D 组: 精密轨道重写后校验组 + +目的: 观察“精轨重写后，Gamma 自身的 spline 校验是否仍给出大修正量”。 + +处理方式: + +- 先执行 C 组重写 +- 再调用 `ORB_filt_spline.py` 生成验证副本 +- 不一定把验证副本作为正式输入使用，先记录校验结果 + +判定意义: + +- 如果校验修正量很小，说明 C 组重写后的 state vectors 与 Gamma 的平滑约束基本一致 +- 如果校验修正量仍然很大，说明即使引入精轨，state vector 时间栅格或插值方式仍可能有问题 + +## 6. 实验执行路径 + +推荐直接复用现有 3 景实验目录，不重新拷贝数据: + +- 基础目录: `D:\PyINT_POOL_TEST\LT1A_MONO_SYC_STRIP1_E129.6_N45.0_3scene` +- 现有日志目录: `D:\PyINT_POOL_TEST\LT1A_MONO_SYC_STRIP1_E129.6_N45.0_3scene\logs` +- 现有 SLC 目录: `D:\PyINT_POOL_TEST\LT1A_MONO_SYC_STRIP1_E129.6_N45.0_3scene\pyint_stage\SLC` + +推荐把每个实验组做成并列工作副本，例如: + +```text +D:\PyINT_POOL_TEST\LT1A_MONO_SYC_STRIP1_E129.6_N45.0_3scene_cases\ + case_A_baseline\ + case_B_orb_filt\ + case_C_precise_orbit_rewrite\ + case_D_precise_orbit_validate\ +``` + +每个 case 必须从同一份 `down2slc_all` 完成后的结果拷贝出来，避免导入过程本身再次引入差异。 + +## 7. 每组执行顺序 + +### Step 1: 冻结基线输入 + +先选取一个共同起点: + +- `down2slc_all` 已完成 +- `makedem_pyint` 已完成 +- `coreg` 尚未执行，或清理已有 `coreg` 中间产物 + +这样所有 case 都基于同一份: + +- `SLC/*.slc` +- `SLC/*.slc.par` +- `MLI` +- `DEM` + +### Step 2: 仅修改 `.slc.par` + +按分组分别对 `.slc.par` 做处理: + +- A 组: 不改 +- B 组: 仅 filter +- C 组: 精轨重写 +- D 组: 精轨重写后再做 Gamma 校验 + +注意: + +- 不要重新导入原始压缩包 +- 不要改模板参数 +- 不要改 `ifgram_list` + +### Step 3: 重做 `generate_rdc_dem` + +因为 `rdc_dem` 与几何参数耦合，改完 `.slc.par` 后必须重做一次。 + +### Step 4: 单独跑每个 slave 的 `coreg` + +推荐不要一开始就跑 `coreg_gamma_all.py`，而是先分别跑: + +- `20230624` +- `20230920` + +这样更容易判断某个处理是否对所有 slave 都有效。 + +### Step 5: 收集指标 + +每个 case 对每个 slave 都产出独立日志，并汇总到统一表格。 + +## 8. 观测指标 + +本实验不能只看“是否跑通”，还要看失败形态是否发生了有意义的变化。 + +核心指标: + +1. `coreg` 是否成功进入下一阶段 +2. `init_offsetm` 是否仍失败 +3. 报错中的 zero-value patch 数量 +4. 失败位置是否仍固定在 `init_offsetm` +5. 是否产生有效的 `RSLC` + +轨道相关指标: + +1. 每个 `.slc.par` 改写前后的 state vector 位置差范数 +2. 每个 `.slc.par` 改写前后的 state vector 速度差范数 +3. `ORB_filt_spline.py` 若运行成功，其输出副本相对输入的修正量 + +结果质量指标: + +1. 如果 `coreg` 成功，后续 `diff/cor` 是否仍全 0 +2. `diff_filt.zero_ratio` +3. `cor.zero_ratio` + +## 9. 判定标准 + +### 支持主假设的证据 + +以下任一情况都可视为强支持: + +1. B 组或 C 组能让两个 slave 中至少一个从 `init_offsetm` 失败变成成功 +2. B 组或 C 组虽然仍失败，但 zero-value patch 数量显著下降 +3. C 组优于 B 组，说明外部精轨重写比单纯过滤更关键 + +### 反对主假设的证据 + +以下情况说明当前假设不足: + +1. A/B/C/D 四组都在同一位置、以近似相同错误失败 +2. 改写前后 state vector 差异很大，但 `coreg` 行为几乎不变 +3. `ORB_filt_spline.py` 校验显示修正量不大，但 `coreg` 仍完全失败 + +若出现这些情况，下一轮应优先排查: + +- `par_LT1_SLC` 生成的几何字段是否本身错误 +- `MLI` 数据中大面积零值的真实来源 +- LT-1 导入后的采样/deskew/时序链条 + +## 10. 推荐的最小可执行版本 + +如果想尽快验证，不必一次把四组都跑全，先做最小闭环: + +1. A 组: 当前基线 +2. C 组: 精密轨道重写 +3. D 组: 精密轨道重写后做 `ORB_filt_spline.py` 校验 + +理由: + +- A 组已经稳定存在 +- C 组最直接验证“外部精轨重写是否必要” +- D 组可以帮助判断“即便重写了，Gamma 仍不认可的程度有多大” + +B 组可以作为补充组，用来区分“只需过滤”还是“必须引入精轨”。 + +## 11. 执行前提 + +执行本实验前，需要确认: + +1. WSL 中可调用 Gamma 工具 +2. `apply_lt1_precise_orbit.py` 可以访问本次实验所需的 LT-1 精密轨道 TXT +3. `ORB_filt_spline.py` 如 shebang 不可用，则通过明确的 Python 解释器调用 +4. 每个 case 使用独立日志目录，避免覆盖 + +## 12. 风险与注意事项 + +1. 不能在同一个 `SLC` 目录上反复覆盖做多组实验，否则会污染基线 +2. 改写 `.slc.par` 后如果不重做 `generate_rdc_dem`，实验结论不可靠 +3. 这轮实验只验证“轨道处理是不是主因”，不等于验证“最终科学结果已经正确” +4. 即使某组 `coreg` 成功，也必须继续检查后续 `diff/cor` 是否仍然全 0 + +## 13. 实验产出物 + +建议每组最终至少保留: + +- 处理后的 `.slc.par` +- `generate_rdc_dem` 日志 +- 每个 slave 的 `coreg` 日志 +- 一个汇总 JSON 或 Markdown 表 + +建议的汇总字段: + +```json +{ + "case": "case_C_precise_orbit_rewrite", + "master": "20230726", + "slave": "20230920", + "state_vector_mode": "precise_orbit_rewrite", + "coreg_success": false, + "failed_stage": "init_offsetm", + "zero_patch_count": 190155, + "rslc_exists": false, + "orb_validation_status": "not_run" +} +``` + +## 14. 推荐下一步 + +推荐按下面顺序执行: + +1. 先在现有 3 景实验目录上做 A/C/D 三组 +2. 如果 C 组明显改善，再补 B 组区分“过滤”与“精轨重写”的贡献 +3. 如果 A/B/C/D 全部失败，再正式转向 `par_LT1_SLC` 输出几何字段和 `MLI` 零值来源排查 + +这轮实验的价值不在于立刻修好流程，而在于把问题边界收窄到“轨道链条”还是“导入几何链条”。 diff --git a/docs/PYINT_LT1_DEM_GEOMETRY_CHAIN_EXPERIMENT_20260420.md b/docs/PYINT_LT1_DEM_GEOMETRY_CHAIN_EXPERIMENT_20260420.md new file mode 100644 index 0000000..ee4a67c --- /dev/null +++ b/docs/PYINT_LT1_DEM_GEOMETRY_CHAIN_EXPERIMENT_20260420.md @@ -0,0 +1,308 @@ +# PyINT LT-1 DEM 几何链定位实验 + +**日期**: 2026-04-20 +**状态**: 实验设计 +**目标问题**: 验证 LT-1 在 PyINT/Gamma 中 `coreg/init_offsetm` 失败，是否由 DEM 本体问题引起，还是由 DEM 参与的几何映射链条引起 + +## 1. 结论先行 + +这轮实验不再直接回答“轨道是不是问题”，而是专门回答下面这个问题: + +> `init_offsetm` 报错里的大量 `zero values in MLI1 image patch`，究竟是 DEM 文件本身有问题，还是 `DEM -> rdc_trans -> geocode -> mli0` 这一条几何映射链条有问题。 + +当前更值得优先验证的是: + +1. `HGTSIM` 是否本身就存在异常空洞或覆盖错误 +2. `lt0` 是否把参考图映射到了错误位置 +3. `mli0` 的中心 `512 x 512` patch 是否天然就是大面积 0 +4. `mli0` 与 `Samp` 的有效重叠区是否根本不在图像中心 + +## 2. 已知事实 + +当前 `coreg_gamma.py` 的关键顺序是: + +- `rdc_trans` +- `geocode` +- `create_diff_par` +- `init_offsetm mli0 Samp diff0 1 1` + +也就是说，`init_offsetm` 当前比较的是: + +- `MLI1 = mli0` +- `MLI2 = Samp` + +而不是直接比较 DEM。 + +已完成的 A/C/D 轨道实验说明: + +1. 只改 state vector，`init_offsetm` 的 zero-count 会下降 +2. 但下降后仍然失败 +3. `ORB_filt_spline.py` 已经基本认可重写后的轨道 + +这说明: + +- 轨道确实影响几何 +- 但当前失败不太像“只有轨道问题” +- DEM 相关的几何映射链条值得单独定位 + +## 3. 三类假设 + +### H1: DEM 本体问题 + +DEM 文件本身有问题，例如: + +- 覆盖范围不对 +- 裁剪窗口不对 +- sidecar / 投影信息不对 +- 高程值大面积异常或空洞 + +若 H1 成立，则更换 DEM 源后应显著改变 `HGTSIM` 和 `mli0` 的空洞模式。 + +### H2: DEM 几何链问题 + +DEM 文件本身可用，但 LT-1 导入几何、轨道或 `generate_rdc_dem` 中间步骤有问题，导致: + +- `rdc_trans` 查找表偏移 +- `geocode` 后的 `mli0` 被映射到错误位置 +- `mli0` 中心 patch 与 `Samp` 根本不重叠 + +若 H2 成立，则即使更换 DEM，本质失败形态也可能不变；但改变轨道或几何参数时，`mli0` 的零值分布会跟着变化。 + +### H3: patch 选取问题 + +`mli0` 与 `Samp` 不是完全不重叠，而是图像中心不是有效重叠区。 + +若 H3 成立，则: + +- 全图并非都坏 +- 换 `rpos/azpos` 后，`init_offsetm` 可能在其他 patch 上能工作 + +## 4. 实验原则 + +这轮实验尽量不跑整条生产链，只盯 `init_offsetm` 之前的中间产物。 + +固定不变: + +- 同一批 3 景 LT-1 数据 +- 同一个 master: `20230726` +- 同一套 looks 参数 +- 同一份 PyINT/Gamma 环境 + +允许变化: + +- DEM 来源 +- 是否启用精轨重写 +- `init_offsetm` 的 patch 位置 + +优先观测对象: + +- `HGTSIM` +- `lt0` +- `mli0` +- `Samp` +- `diff0` + +## 5. 分层实验设计 + +### Layer 1: 不改 DEM，先看中间产物 + +目的: 判断当前 DEM 链条到底在哪一步开始“空掉”。 + +#### 组 L1-A: 基线组 + +使用当前已经失败的 case，导出并统计: + +- `HGTSIM` +- `lt0` +- `mli0` +- `Samp` + +对每个文件都做: + +1. 全图零值比例 +2. 中心 `512 x 512` patch 零值比例 +3. 中心 patch 的最小值、最大值、均值 +4. 快速可视化图 + +判定意义: + +- 如果 `HGTSIM` 自身就明显异常，优先怀疑 DEM 或 `generate_rdc_dem` +- 如果 `HGTSIM` 正常而 `mli0` 异常，优先怀疑 `lt0/geocode` +- 如果 `mli0` 正常但中心 patch 不在有效重叠区，优先怀疑 patch 选取 + +#### 组 L1-B: 精轨重写对照组 + +复用已经跑过的精轨重写 case，只比较: + +- `HGTSIM` +- `mli0` +- `Samp` + +判定意义: + +- 如果只改轨道，`mli0` 的零值分布就跟着变，说明问题不在 DEM 文件本体 +- 如果 `HGTSIM` 也明显变化，说明 DEM 映射结果强依赖 `.slc.par` 几何 + +### Layer 2: 改 DEM 源，固定几何 + +目的: 判断 DEM 文件本身是否是主因。 + +#### 组 L2-A: 当前 DEM + +使用当前 `prepared_dem_source`，作为基线。 + +#### 组 L2-B: 替代 DEM + +推荐优先选一个同区域、同分辨率级别、不同来源的 DEM，例如: + +- 已有的另一份系统 DEM +- 或成功 ENVI/IDL 任务中使用过的 DEM + +要求: + +- 不改 `.slc.par` +- 不改轨道 +- 只重跑 `makedem_pyint -> generate_rdc_dem` + +判定意义: + +- 如果换 DEM 后 `HGTSIM/mli0` 明显改善，DEM 本体有嫌疑 +- 如果几乎不变，DEM 本体不是主因 + +#### 组 L2-C: 合成平坦 DEM + +用一个覆盖相同区域、常数高程的测试 DEM。 + +这组不是为了出正确结果，而是为了测试: + +- 失败是否强依赖真实地形起伏 +- 还是只要进入几何映射链就已经错位 + +判定意义: + +- 如果平坦 DEM 仍在同一位置失败，说明不是高程细节导致 +- 如果平坦 DEM 反而明显改善，则真实 DEM 参与的映射可能存在投影或裁剪问题 + +### Layer 3: 不改 DEM，扫描 patch 位置 + +目的: 判断中心 patch 是否只是选错了地方。 + +方法: + +- 保持 `mli0`、`Samp`、`diff0` 不变 +- 只改变 `init_offsetm` 的: + - `rpos` + - `azpos` +- 在图像中心周围做稀疏网格扫描 + +推荐: + +- 先做 `5 x 5` 或 `7 x 7` 网格 +- patch 大小仍保持默认 `512` + +每个点记录: + +- 是否报 zero-patch 错误 +- zero-count +- 是否能进入下一步 + +判定意义: + +- 如果某些位置能通过，说明不是整幅图都坏，而是中心 patch 选取不对 +- 如果所有位置都报大面积 0，更像几何链整体错位 + +## 6. 关键观测指标 + +### DEM 相关 + +1. `HGTSIM` 是否生成成功 +2. `HGTSIM` 的全图与中心 patch 零值比例 +3. `HGTSIM` 是否存在明显空洞、条带或边界错切 + +### 几何映射相关 + +1. `lt0` 是否可用 +2. `mli0` 的全图与中心 patch 零值比例 +3. `mli0` 与 `Samp` 的有效像元重叠比例 +4. `mli0` 与 `Samp` 是否在中心 patch 上具有相似纹理 + +### `init_offsetm` 相关 + +1. 是否仍失败在同一位置 +2. zero-count 是否显著下降 +3. 换 patch 位置后是否存在可工作区域 + +## 7. 推荐输出物 + +每轮实验至少产出: + +- 中间产物清单 +- 每个文件的统计 JSON +- 快速可视化 PNG/TIF +- 一张对照表 + +推荐的统计字段: + +```json +{ + "case": "L1-A_baseline", + "file": "mli0", + "width": 0, + "lines": 0, + "global_zero_ratio": 0.0, + "center_patch_zero_ratio": 0.0, + "center_patch_size": 512, + "nonzero_overlap_with_samp": 0.0 +} +``` + +## 8. 判定标准 + +### 支持“DEM 本体问题”的证据 + +1. 更换 DEM 后，`HGTSIM` 和 `mli0` 的空洞模式大幅变化 +2. 同一套几何下，只有某一份 DEM 会触发大面积零值 +3. 合成平坦 DEM 能显著改善中心 patch + +### 支持“DEM 几何链问题”的证据 + +1. `HGTSIM` 看起来基本正常，但 `mli0` 大面积为 0 +2. 只改轨道或 `.slc.par`，`mli0` 零值分布就发生变化 +3. 换 DEM 后失败模式基本不变 + +### 支持“patch 选取问题”的证据 + +1. 中心 patch 失败，但偏移后的 patch 可以工作 +2. `mli0` 与 `Samp` 在全图上存在局部重叠区，只是中心不对 + +## 9. 推荐的最小闭环 + +不建议一上来就换很多 DEM。最小闭环应按这个顺序: + +1. 先做 Layer 1 + - 对现有基线 case 和精轨重写 case 提取 `HGTSIM/lt0/mli0/Samp` 统计和快视图 +2. 再做 Layer 3 + - 扫描 `init_offsetm` 的 `rpos/azpos` +3. 只有在 Layer 1/3 仍无法判断时，再做 Layer 2 换 DEM + +原因: + +- 这能先区分“DEM 文件坏了”与“中心 patch 选错了” +- 也能避免过早把问题全部甩给 DEM + +## 10. 推荐下一步 + +推荐直接执行两个子实验: + +1. 中间产物审计实验 + - 把 A 组和 C 组的 `HGTSIM/lt0/mli0/Samp` 取出来做零值统计和快视图 +2. `init_offsetm` patch 扫描实验 + - 在同一 case 上只扫描 `rpos/azpos` + +如果这两步做完后发现: + +- `mli0` 全图都坏，再优先查 `generate_rdc_dem` 和 LT-1 导入几何 +- 只有中心 patch 坏，再优先查 patch 选取 +- 换 DEM 才有明显改善，再回到 DEM 本体问题 + +这轮实验的目的不是立刻修好 `coreg`，而是把“DEM 文件问题”和“DEM 几何链问题”从概念判断变成可测的证据链。 diff --git a/docs/PYINT_LT1_DEM_SOURCE_EXPERIMENT_RESULTS_20260420.md b/docs/PYINT_LT1_DEM_SOURCE_EXPERIMENT_RESULTS_20260420.md new file mode 100644 index 0000000..c256f30 --- /dev/null +++ b/docs/PYINT_LT1_DEM_SOURCE_EXPERIMENT_RESULTS_20260420.md @@ -0,0 +1,158 @@ +# PyINT LT-1 DEM Source Experiment Results + +**日期**: 2026-04-20 +**状态**: 已执行 +**目标问题**: `init_offsetm` 失败是否主要由 DEM 本体来源导致 + +## 1. 前置结论 + +在这轮 DEM 源对照之前，已经完成 `init_offsetm` patch 扫描。 + +- 基线 run root: + - `D:\PyINT_POOL_TEST\LT1A_MONO_SYC_STRIP1_E129.6_N45.0_3scene_cases\run_20260420T093322Z` +- 扫描结论: + - 更换 `rpos/azpos` 可以把中心 patch 的零值数从 `184099/190155` 降到约 `168322/166114` + - 但仍远高于 `32768` 阈值 + - 没有任何候选 patch 通过 `init_offsetm` + +因此，这里把“中心 patch 选错”降级为次要因素，继续验证 DEM 源本体是否主导失败。 + +## 2. 实验设计 + +固定条件: + +- 同一组 3 景 LT-1 数据 +- 同一 master: `20230726` +- 同一 baseline orbit 几何 +- 同一 PyINT/Gamma 处理链 + +只更换 DEM 来源: + +1. `COPDEM` + - `prepared_dem_source=/mnt/d/DEM/COPDEM_GLO30_China_4326_DEM` +2. `GMTED2010` + - `prepared_dem_source=/mnt/d/DEM/GMTED2010.jp2` +3. `OpenTopography SRTMGL1` + - 走 PyINT 默认下载路径 + +执行链路: + +- `makedem_pyint` +- `generate_rdc_dem` +- `coreg_gamma` +- `audit_lt1_dem_geometry_chain.py` + +## 3. 实验目录 + +本地 DEM 对照: + +- run root: + - `D:\PyINT_POOL_TEST\LT1A_MONO_SYC_STRIP1_E129.6_N45.0_3scene_dem_cases\run_20260420T152607Z` +- 审计输出: + - `D:\PyINT_POOL_TEST\LT1A_MONO_SYC_STRIP1_E129.6_N45.0_3scene_dem_cases\run_20260420T152607Z\audit_dem_geometry` + +OpenTopography SRTMGL1: + +- 首次尝试: + - `D:\PyINT_POOL_TEST\LT1A_MONO_SYC_STRIP1_E129.6_N45.0_3scene_dem_cases\run_20260420T170406Z` + - 失败原因: WSL `isce2` 环境缺少 `rasterio` +- 安装 `rasterio` 后重跑: + - `D:\PyINT_POOL_TEST\LT1A_MONO_SYC_STRIP1_E129.6_N45.0_3scene_dem_cases\run_20260420T220524Z` + - 审计输出: + - `D:\PyINT_POOL_TEST\LT1A_MONO_SYC_STRIP1_E129.6_N45.0_3scene_dem_cases\run_20260420T220524Z\audit_dem_geometry` + +## 4. 结果摘要 + +### 4.1 `coreg/init_offsetm` zero-count + +`20230624`: + +- `COPDEM`: `184099` +- `GMTED2010`: `262033` +- `SRTMGL1`: `183582` + +`20230920`: + +- `COPDEM`: `190155` +- `GMTED2010`: `262070` +- `SRTMGL1`: `189571` + +阈值均为 `32768`。 + +### 4.2 `HGTSIM` / `lt0` / `mli0_samp_overlap` + +`COPDEM`: + +- `hgtsim zero_ratio`: `0.967266` +- `lt0 valid_pair_ratio`: `0.032734` +- `20230624 center overlap`: `0.297718` +- `20230920 center overlap`: `0.274616` + +`GMTED2010`: + +- `hgtsim zero_ratio`: `0.999964` +- `lt0 valid_pair_ratio`: `0.000036` +- `20230624 center overlap`: `0.000423` +- `20230920 center overlap`: `0.000282` + +`SRTMGL1`: + +- `hgtsim zero_ratio`: `0.967264` +- `lt0 valid_pair_ratio`: `0.032736` +- `20230624 center overlap`: `0.299690` +- `20230920 center overlap`: `0.276844` + +## 5. 结论 + +### 5.1 当前 `COPDEM` 不是主要故障源 + +`SRTMGL1` 作为更接近原始 PyINT 默认路径的下载 DEM，跑出来的几何指标与当前 `COPDEM` 几乎一致: + +- `hgtsim zero_ratio` 基本相同 +- `lt0 valid_pair_ratio` 基本相同 +- `mli0_samp_overlap` 基本相同 +- `init_offsetm zero-count` 只改善了几百个像素，量级上没有本质变化 + +这说明: + +- 把当前系统 DEM 替换成原始 PyINT 默认下载 DEM +- 并不能把问题从 `184k/190k` 拉到 `32768` 阈值附近 + +### 5.2 更差的 DEM 会进一步恶化问题 + +`GMTED2010` 把几何链几乎压成全零: + +- `hgtsim zero_ratio` 逼近 `1.0` +- `lt0 valid_pair_ratio` 下降到 `0.000036` +- `init_offsetm zero-count` 直接升到 `262k` + +这说明 DEM 源会影响结果，但当前问题不是“现有 DEM 明显坏掉”，而是: + +- 当前几何链本来就已经非常稀疏 +- 更粗或不合适的 DEM 只会让它更差 + +### 5.3 当前最可疑的位置继续落在 LT-1 几何导入链 + +综合 patch 扫描和 DEM 源对照，当前更像是以下链路问题，而不是 DEM 文件本体问题: + +- `par_LT1_SLC / LT-1 导入几何` +- `generate_rdc_dem` +- `coreg_gamma` 中基于 DEM 的几何映射链 + +## 6. 补充记录 + +为完成 `OpenTopography SRTMGL1` 对照，已在 WSL `isce2` 环境安装: + +- `rasterio==1.4.4` + +安装原因不是业务修复，而是 PyINT 下载 DEM 路径在当前环境里依赖该包做分块 tif 校验与合并。 + +## 7. 建议下一步 + +下一轮不建议继续反复更换 DEM。 + +更值得做的是: + +1. 对比 `COPDEM` 与 `SRTMGL1` 生成出来的 `pyint_stage.dem.par`、`UTMDEMpar`、`UTM2RDC/UTMTORDC` 是否几乎一致 +2. 回到 LT-1 导入链，核查 `.slc.par` 中被 `gc_map1 / geocode / init_offsetm` 直接消费的几何字段 +3. 如果需要继续做 DEM 类实验，优先做“同一 DEM 下替换导入几何参数”，而不是继续换 DEM 本体 diff --git a/docs/PYINT_LT1_PAIR_SELECTION_EXPERIMENT_20260421.md b/docs/PYINT_LT1_PAIR_SELECTION_EXPERIMENT_20260421.md new file mode 100644 index 0000000..c9e34c7 --- /dev/null +++ b/docs/PYINT_LT1_PAIR_SELECTION_EXPERIMENT_20260421.md @@ -0,0 +1,145 @@ +# PyINT LT-1 Pair Selection Experiment + +**日期**: 2026-04-21 +**状态**: 已执行 +**目标问题**: 之前 `init_offsetm` 失败是否只是当前任务配对选得不好 + +## 1. 实验思路 + +上一轮根因定位主要围绕这一组 2023 年 SYC 数据: + +- `2023-06-24` +- `2023-07-26` +- `2023-09-20` + +其中: + +- `2023-06-24 -> 2023-07-26` +- `2023-07-26 -> 2023-09-20` + +两对都失败，且 `2023-06-24 -> 2023-07-26` 已经是较短时基。 + +为了验证是不是“只是这几对碰巧不行”，本轮换了一组新的、同条带同中心点的 2024 年三景，并改用中间时相作为 master。 + +## 2. 实验数据 + +实验根目录: + +- `D:\PyINT_POOL_TEST\LT1A_MONO_SYC_STRIP1_E129.4_N45.0_3scene_2024` + +从影像库复制的三景: + +1. `LT1A_MONO_SYC_STRIP1_012583_E129.4_N45.0_20240520_SLC_HH_S2A_0000402579` +2. `LT1A_MONO_SYC_STRIP1_013416_E129.4_N45.0_20240715_SLC_HH_S2A_0000454453` +3. `LT1A_MONO_SYC_STRIP1_014249_E129.4_N45.0_20240909_SLC_HH_S2A_0000505340` + +配置: + +- `masterDate=20240715` +- `startDate=20240501` +- `endDate=20240930` +- DEM 仍使用: + - `/mnt/d/DEM/COPDEM_GLO30_China_4326_DEM` + +## 3. 执行链路 + +执行脚本: + +- 复制实验根: + - `D:\Code\Insar_management_system_v2\.codex_tmp\setup_lt1_pool_multiscene_experiment.ps1` +- 跑三景最小链路: + - `D:\Code\Insar_management_system_v2\.codex_tmp\run_lt1_pool_multiscene_generic.sh` +- 审计几何中间产物: + - `D:\Code\Insar_management_system_v2\.codex_tmp\audit_lt1_pool_multiscene_root.py` + +实际运行结果: + +- `down2slc_all`: 成功 +- `makedem_pyint`: 成功 +- `generate_rdc_dem`: 成功 +- `coreg_gamma_all`: 失败 + +由于 `coreg_gamma_all` 在第一对失败后停止，又额外补跑了: + +- `coreg_20240909` + +这样两对都拿到了独立结果。 + +## 4. 结果 + +### 4.1 `init_offsetm` 失败情况 + +`20240520 <- 20240715(master)`: + +- `zero_count = 197702` +- `threshold = 32768` + +`20240909 <- 20240715(master)`: + +- `zero_count = 196712` +- `threshold = 32768` + +对应日志: + +- `D:\PyINT_POOL_TEST\LT1A_MONO_SYC_STRIP1_E129.4_N45.0_3scene_2024\logs\coreg_gamma_all.stderr.log` +- `D:\PyINT_POOL_TEST\LT1A_MONO_SYC_STRIP1_E129.4_N45.0_3scene_2024\logs\coreg_20240909.stderr.log` + +### 4.2 中间产物审计 + +审计汇总: + +- `D:\PyINT_POOL_TEST\LT1A_MONO_SYC_STRIP1_E129.4_N45.0_3scene_2024\audit_dem_geometry\audit_summary.tsv` + +关键指标: + +- `hgtsim zero_ratio = 0.974356` +- `lt0 valid_pair_ratio = 0.025644` +- `20240520 center overlap = 0.245827` +- `20240909 center overlap = 0.249603` + +## 5. 与 2023 年那组三景对比 + +2023 年 `E129.6_N45.0` 那组基线结果: + +- `20230624`: `zero_count = 184099` +- `20230920`: `zero_count = 190155` +- `hgtsim zero_ratio = 0.967266` +- `lt0 valid_pair_ratio = 0.032734` + +2024 年 `E129.4_N45.0` 新组三景结果: + +- `20240520`: `zero_count = 197702` +- `20240909`: `zero_count = 196712` +- `hgtsim zero_ratio = 0.974356` +- `lt0 valid_pair_ratio = 0.025644` + +## 6. 结论 + +这轮实验不支持“只是当前 pair 选坏了”这个解释。 + +原因很直接: + +1. 换了一整组三景 +2. 换了 master +3. 两对 slave 都单独跑到了 `init_offsetm` +4. 结果仍然失败 +5. 而且零值规模没有改善，反而比 2023 那组更差 + +因此当前更合理的判断是: + +- 问题不是单纯 pair selection +- 也不是只集中在 `2023-07-26` 这一景 +- 更像 LT-1 在 PyINT/Gamma 下的导入几何链存在系统性问题 + +## 7. 建议下一步 + +下一步不建议继续只靠“再换几对”来试。 + +更有价值的是继续往几何链前面查: + +1. 对比不同实验根生成的 `.slc.par` 几何字段 +2. 对比 `generate_rdc_dem` 产生的: + - `*.utm.dem.par` + - `*.UTM_TO_RDC` + - `*.rdc.dem` +3. 重点核查 LT-1 导入程序和 Gamma 实际消费字段之间是否存在系统性偏差 diff --git a/docs/PYINT_LT1_PRECISE_ORBIT_BRIDGE_DESIGN_20260419.md b/docs/PYINT_LT1_PRECISE_ORBIT_BRIDGE_DESIGN_20260419.md new file mode 100644 index 0000000..1a520bd --- /dev/null +++ b/docs/PYINT_LT1_PRECISE_ORBIT_BRIDGE_DESIGN_20260419.md @@ -0,0 +1,405 @@ +# PyINT LT-1 精密轨道桥接设计 + +**日期**: 2026-04-19 +**状态**: 方案设计 +**范围**: LT-1 精密轨道真正参与 PyINT / Gamma 计算、与现有 `Task_*` 输入模式协同、前后端与运维落点、分阶段实施 + +## 1. 结论 + +当前仓库已经完成了 LT-1 轨道 TXT 的治理级接入，但还没有完成“精密轨道真实参与 PyINT / Gamma 计算”这一层。 + +本次设计的核心结论如下： + +1. 不能把系统轨道池里的 `LT1*_GpsData_GAS_C_YYYYMMDD.txt` 简单当成 `par_LT1_SLC` 的直接输入，因为当前 PyINT / Gamma 的 LT-1 导入链并没有暴露这样的接口。 +2. 正确的桥接点是 LT-1 导入完成后生成的 `.slc.par` / `.slc.update.par` 里的 `state_vector_*` 段，而不是当前外层 `run_lt1_pyint_pipeline.py` 的任务参数层。 +3. 推荐方案是在现有 Windows 侧轨道治理不变的前提下，在 WSL 侧新增一个 LT-1 精轨桥接 helper，把系统选中的精轨 TXT 重采样到 Gamma 参数文件已有的时间栅格，再回写 `.slc.par`。 +4. 桥接动作必须发生在 LT-1 导入之后、DEM / coreg / 干涉处理之前；只做“提交前预检”或“运行记录留痕”是不够的。 +5. 一期先不改数据库，先把桥接结果写进 `input_assets`、`pyint_run_summary.json`、结果 manifest 和运行日志；只有在后续确实需要跨运行检索、统计、追责时，再通过现有数据库自维护机制补迁移。 + +## 2. 现状与依据 + +### 2.1 当前本地代码链路 + +现有 vendored PyINT 的 LT-1 流程为： + +`pyintApp.py` +-> `down2slc_LT1_all.py` +-> `down2slc_LT1.py` 或 `down2slc_cat_LT1.py` +-> `LT1_import_SLC_from_zipfiles1` +-> `par_LT1_SLC` / `par_LT1_SLC_YSLi` +-> 生成 `.slc.par` / `.slc.update.par` + +已确认的关键事实： + +- [pyintApp.py](/D:/Code/Insar_management_system_v2/third_party/PyINT/pyint/pyintApp.py) 会先执行 LT-1 `raw2slc`，然后再进入 DEM、coreg、差分干涉。 +- [down2slc_LT1.py](/D:/Code/Insar_management_system_v2/third_party/PyINT/pyint/down2slc_LT1.py) 与 [down2slc_cat_LT1.py](/D:/Code/Insar_management_system_v2/third_party/PyINT/pyint/down2slc_cat_LT1.py) 是当前 LT-1 Python 入口。 +- [LT1_import_SLC_from_zipfiles1](/D:/Code/Insar_management_system_v2/third_party/PyINT/pyint/LT1_import_SLC_from_zipfiles1) 已经显式处理 `state_vector_*`，说明轨道状态向量确实是 LT-1 导入链中的有效控制点。 +- [20210110.slc.par](/D:/Code/Insar_management_system_v2/third_party/PyINT/pyint/20210110.slc.par) 展示了 Gamma 参数文件中的状态向量布局，包括： + - `number_of_state_vectors` + - `time_of_first_state_vector` + - `state_vector_interval` + - `state_vector_position_i` + - `state_vector_velocity_i` + +### 2.2 当前系统已有能力 + +仓库已经具备以下基础： + +- [pyint_input_assets_service.py](/D:/Code/Insar_management_system_v2/backend/app/services/pyint_input_assets_service.py) 已能从 `ORBIT_POOL_ENVI` / `PYINT_ORBIT_POOL_TXT` 解析 master/slave 对应的 LT-1 精轨 TXT。 +- [run_lt1_pyint_pipeline.py](/D:/Code/Insar_management_system_v2/backend/app/pyint_pipeline/run_lt1_pyint_pipeline.py) 已能把 `Task_*` 目录物化成 PyINT 工作区，并记录 `orbit_policy` 与 `input_assets`。 +- [DinsarProductionPanel.jsx](/D:/Code/Insar_management_system_v2/frontend/src/DinsarProductionPanel.jsx) 已经有 PyINT 输入资产预检入口，能够把“轨道是否齐全”提前暴露给用户。 +- ISCE2 侧已经有 LT-1 轨道 TXT 解析链，可复用 [convert_lt1_orbit_to_isce_xml.py](/D:/Code/Insar_management_system_v2/backend/app/isce2_pipeline/convert_lt1_orbit_to_isce_xml.py) 与 [lt1_input_resolver.py](/D:/Code/Insar_management_system_v2/backend/app/isce2_pipeline/lt1_input_resolver.py) 中的 `parse_orbit_file`、时间窗口解析等逻辑。 + +### 2.3 Gamma 官方文档给出的关键约束 + +用户提供的 Gamma 官方文档是： + +- <https://www.gamma-rs.ch/uploads/media/2023-1_TR_China_LT1_Support_in_GAMMA.pdf> + +其中与本设计直接相关的结论有两点： + +1. 在 LT-1 repeat-pass DInSAR 流程中，Gamma 文档明确说明，`par_LT1_SLC` 导入后需要立即检查并过滤 orbit state vectors，并使用 `ORB_filt_spline.py` 做校验。 +2. 在 LT-1 tandem single-pass 流程中，文档同样建议“读入数据后立刻检查/过滤状态向量”，以确保后续 MLI 参数和几何步骤使用的是修正后的状态向量。 + +这意味着桥接点必须放在“LT-1 导入完成之后立刻执行”，而不是只在外层运行摘要里记录轨道来源。 + +## 3. 当前缺口 + +当前实现还缺以下一层： + +1. 系统已经知道“这次任务应该用哪份精轨 TXT”，但 PyINT / Gamma 还不知道。 +2. 预检面板只能阻断“轨道缺失”的任务，不能保证“轨道已进入计算”。 +3. 只改外层 `run_lt1_pyint_pipeline.py` 不够，因为 PyINT 内部会自己完成 `raw2slc -> dem -> coreg` 连续流程，桥接必须插在内部 `raw2slc` 之后。 +4. `cat` 场景不能只更新单个 `.slc.par`。当前 [down2slc_cat_LT1.py](/D:/Code/Insar_management_system_v2/third_party/PyINT/pyint/down2slc_cat_LT1.py) 最终拼接依赖 `*.slc.update.par`，所以方案必须覆盖： + - 每个分片导入后的参数文件 + - 最终拼接得到的 `<date>.slc.par` + +## 4. 目标与非目标 + +### 4.1 目标 + +- 保留当前 `Task_*` 输入模式，不要求用户维护第二套 PyINT 原生目录。 +- 让 LT-1 精轨 TXT 真正参与 PyINT / Gamma 计算，而不是只做治理留痕。 +- 同时覆盖单场景和多分片 `cat` 场景。 +- 与现有 DEM 策略、结果目录、运行日志、预检面板兼容。 +- 为后续 Gamma 配对集成保留复用路径。 + +### 4.2 非目标 + +- 一期不修改数据库主结构。 +- 一期不在运维自检页新增复杂操作区。 +- 一期不承诺完成 LT1A/LT1B tandem 单通道单程干涉生产链，只保证当前 repeat-pass PyINT 流程的精轨桥接。 +- 一期不让用户在前端手工输入单次轨道路径。 + +## 5. 推荐总体方案 + +### 5.1 分层思路 + +推荐把方案拆成“控制面”和“计算面”两层。 + +#### A. 控制面，继续由现有后端负责 + +控制面继续沿用现有资产治理链路： + +- 从 `Task_*` 解析 master/slave 的卫星与日期 +- 从 `PYINT_ORBIT_POOL_TXT` 或 `ORBIT_POOL_ENVI` 定位精轨 TXT +- 在 `input_assets/orbits/` 下留痕 +- 在预检接口中返回“是否可提交” + +这部分由现有 [pyint_input_assets_service.py](/D:/Code/Insar_management_system_v2/backend/app/services/pyint_input_assets_service.py) 继续承担。 + +#### B. 计算面，新增 WSL 侧精轨桥接 helper + +新增一个 helper，例如： + +- `backend/app/pyint_pipeline/apply_lt1_precise_orbit.py` + +其职责是： + +1. 读取当前任务已解析好的 orbit manifest / staged TXT。 +2. 读取目标 `.slc.par` 或 `.slc.update.par`。 +3. 复用现有 LT-1 TXT 解析逻辑，解析精轨状态向量。 +4. 以 Gamma 当前参数文件已有的时间栅格为目标，进行插值和回写。 +5. 备份原始参数文件。 +6. 可选调用 `ORB_filt_spline.py` 做二次校验或残差诊断。 +7. 输出 `orbit_bridge_summary.json`。 + +### 5.2 为什么目标时间栅格要复用 `.slc.par` 自身 + +推荐不要自己发明新的状态向量数量和时间间隔，而是直接复用当前 `.slc.par` 中已有的： + +- `number_of_state_vectors` +- `time_of_first_state_vector` +- `state_vector_interval` + +然后把系统精轨 TXT 插值到这个时间栅格上。 + +这样做的收益是： + +- 不改变 Gamma 已经生成的参数文件结构。 +- 不引入新的向量个数假设。 +- 更容易和 `ORB_filt_spline.py`、后续 DEM / coreg 步骤兼容。 +- 对 `cat` 场景、已有模板和下游脚本影响最小。 + +### 5.3 插值策略 + +推荐优先使用“基于位置和速度的 Hermite 插值”，原因是 LT-1 TXT 同时提供了位置和速度。 + +最小实现要求如下： + +- 先复用 [convert_lt1_orbit_to_isce_xml.py](/D:/Code/Insar_management_system_v2/backend/app/isce2_pipeline/convert_lt1_orbit_to_isce_xml.py) 的 `parse_orbit_file` 解析状态向量。 +- 根据 `.slc.par` 的采样时间点计算目标 UTC 时间序列。 +- 对目标时间序列进行状态向量重采样。 +- 回写 `state_vector_position_i` 和 `state_vector_velocity_i`。 + +如果一期为了稳妥，不想一次性引入更复杂的插值器，也可以先做： + +- 线性插值作为第一落地版 +- `ORB_filt_spline.py` 作为强校验 + +但长期建议还是切到 Hermite，以减少轨道形状失真。 + +## 6. 推荐挂接点 + +### 6.1 不推荐只改最外层 wrapper + +不推荐只在 [run_lt1_pyint_pipeline.py](/D:/Code/Insar_management_system_v2/backend/app/pyint_pipeline/run_lt1_pyint_pipeline.py) 里做轨道处理，因为它在 PyINT 看来只是外层启动器，无法插入到内部 `raw2slc` 与 `makedem` 之间。 + +### 6.2 推荐挂接点 + +推荐优先修改以下 vendored Python 脚本： + +- [down2slc_LT1.py](/D:/Code/Insar_management_system_v2/third_party/PyINT/pyint/down2slc_LT1.py) +- [down2slc_cat_LT1.py](/D:/Code/Insar_management_system_v2/third_party/PyINT/pyint/down2slc_cat_LT1.py) + +推荐执行时机： + +1. 每次 `LT1_import_SLC_from_zipfiles1` 完成后： + - 对当前生成的 `.slc.par` + - 对当前生成的 `.slc.update.par` + 执行一次桥接 +2. `SLC_cat_list.py` 生成最终 `<date>.slc.par` 后： + - 再对最终参数文件执行一次桥接或至少一次强校验 + +这样可以同时满足： + +- 符合 Gamma 文档“导入后立即检查/过滤状态向量”的原则 +- 覆盖单片和多分片场景 +- 避免只在最终产物上补丁而遗漏拼接过程 + +`LT1_import_SLC_from_zipfiles1` 本身先不作为一期主改点，除非后续验证发现必须把逻辑进一步下沉到 shell 层才能完全覆盖。 + +## 7. 运行时流程 + +推荐的整体执行顺序如下： + +1. 用户在生产面板选择 PyINT 引擎并填写 `root_dir` +2. 后端调用 PyINT 输入资产预检 +3. 系统为每个 `Task_*` 解析： + - master/slave 日期 + - master/slave 卫星 + - 对应精轨 TXT + - DEM 策略 +4. `run_lt1_pyint_pipeline.py` 物化： + - `input_assets/orbits/` + - `task_manifest.json` + - PyINT 工作区和模板 +5. 外层 wrapper 将 orbit manifest 路径、helper 路径和桥接开关注入 WSL 环境 +6. PyINT 执行 `down2slc_LT1.py` / `down2slc_cat_LT1.py` +7. 每个 LT-1 导入步骤完成后，调用 `apply_lt1_precise_orbit.py` +8. helper 回写 `.slc.par` +9. PyINT 继续执行 DEM、coreg、差分干涉、解缠、地理编码 +10. 系统输出： + - `orbit_bridge_summary.json` + - `pyint_run_summary.json` + - 结果 manifest 摘要 + +## 8. 建议新增配置 + +当前已有： + +```ini +PYINT_ORBIT_POLICY=require_txt +PYINT_ORBIT_POOL_TXT= +``` + +建议新增或明确以下配置： + +```ini +PYINT_LT1_PRECISE_ORBIT_ENABLED=true +PYINT_LT1_PRECISE_ORBIT_MODE=replace_and_validate +PYINT_LT1_PRECISE_ORBIT_STRICT=true +PYINT_LT1_PRECISE_ORBIT_MARGIN_SECONDS=120 +PYINT_LT1_PRECISE_ORBIT_VALIDATE_WITH_ORB_FILT=true +PYINT_LT1_PRECISE_ORBIT_BACKUP=true +``` + +建议含义如下： + +- `PYINT_LT1_PRECISE_ORBIT_ENABLED` + - 是否启用真实桥接 +- `PYINT_LT1_PRECISE_ORBIT_MODE` + - `replace_and_validate` 为推荐默认值 + - 后续也可扩展 `validate_only` +- `PYINT_LT1_PRECISE_ORBIT_STRICT` + - 桥接失败时是否阻断任务 +- `PYINT_LT1_PRECISE_ORBIT_MARGIN_SECONDS` + - 对场景时间窗口额外扩展的秒数 +- `PYINT_LT1_PRECISE_ORBIT_VALIDATE_WITH_ORB_FILT` + - 是否调用 `ORB_filt_spline.py` 做残差校验 +- `PYINT_LT1_PRECISE_ORBIT_BACKUP` + - 是否在回写前备份原始 `.slc.par` + +## 9. 元数据与落盘策略 + +一期建议不改数据库，先把桥接痕迹记录到运行产物里。 + +### 9.1 `input_assets` 侧 + +建议在 `input_assets/orbits/` 下保留： + +- 解析到的 master/slave 精轨 TXT +- `orbit_resolution.json` +- `orbit_bridge_request.json` + +### 9.2 运行摘要侧 + +建议在 `pyint_run_summary.json` 中新增： + +```json +{ + "orbit_bridge": { + "enabled": true, + "mode": "replace_and_validate", + "status": "applied", + "master": { + "orbit_txt": "..." + }, + "slave": { + "orbit_txt": "..." + }, + "applied_files": [ + { + "path": ".../20250309.slc.par", + "role": "slave", + "vector_count": 15, + "validated": true + } + ] + } +} +``` + +### 9.3 结果 manifest 侧 + +结果 manifest 只保留摘要，不重复放大块明细，建议记录： + +- 是否启用精轨桥接 +- 桥接状态 +- master/slave 轨道来源 stem +- 是否通过 `ORB_filt_spline.py` 校验 + +### 9.4 数据库策略 + +一期不改数据库。 + +如果后续明确需要： + +- 按轨道版本检索历史运行 +- 统计桥接失败原因 +- 审计某次产品到底使用了哪份精轨 + +再通过现有 [db_maintenance.py](/D:/Code/Insar_management_system_v2/backend/app/db_maintenance.py) 机制新增迁移。 + +## 10. 前端与运维落点 + +### 10.1 前端 + +前端主入口继续放在现有 PyINT 生产区域，不新增独立页面。 + +建议在 [DinsarProductionPanel.jsx](/D:/Code/Insar_management_system_v2/frontend/src/DinsarProductionPanel.jsx) 的 PyINT 输入资产预检卡中增加两类信息： + +- 全局级： + - `精轨桥接: 已启用 / 仅治理 / 未启用` + - `桥接模式` +- 任务级： + - master/slave 是否已解析精轨 + - 本次是否满足真实桥接前置条件 + +不要让用户手工输入 orbit 路径。 + +### 10.2 运维自检 + +运维自检页不再承载新的操作区，只保留状态摘要。 + +建议在引擎健康或 PyINT 健康项里补充： + +- `PYINT_LT1_PRECISE_ORBIT_ENABLED` +- helper 脚本是否存在 +- `PYINT_ORBIT_POOL_TXT` / `ORBIT_POOL_ENVI` 是否可读 +- `ORB_filt_spline.py` 是否可调用 + +不建议把桥接按钮堆进现有 [HealthCheckPanel.jsx](/D:/Code/Insar_management_system_v2/frontend/src/HealthCheckPanel.jsx)。 + +## 11. 与 Gamma 配对集成的关系 + +这套桥接不是只服务 D-InSAR 生产，也是在为后续 Gamma 配对打基础。 + +原因是： + +- 如果后续要把 Gamma / PyINT 配对结果真正纳入系统，配对阶段对 baseline 和场景几何的一致性要求会更高。 +- 只做“轨道存在性预检”仍然不够，仍然需要一条“导入后立即修正状态向量”的内部链路。 + +因此推荐把本次 helper 设计成通用能力： + +- 当前用于 `pyintApp.py` 的 LT-1 `raw2slc` +- 后续也可复用于 `select_pairs.py` 前的 LT-1 导入准备 + +## 12. 风险与未决问题 + +当前仍有几项需要在实现阶段验证： + +1. LT-1 TXT 的时间系统与 `.slc.par` 的 `date + seconds-of-day` 是否存在跨日边界问题。 +2. `ORB_filt_spline.py` 更适合用于“替换后校验”还是“替换后再执行一次修正”，需要先做小样本验证。 +3. `SLC_cat_list.py` 当前并未 vendored 到仓库中，实现阶段要进一步确认它对输入 `.par` 的依赖细节。 +4. 当前 repeat-pass 流程与 LT1A/LT1B tandem single-pass 流程并不完全等价，后者需要单独设计。 +5. 如果发现某些场景的 Gamma 原始导入时间栅格明显不合理，可能需要从“复用现有时间栅格”升级到“按 scene window 重新构造时间栅格”。 + +## 13. 分阶段实施建议 + +### Phase 1 + +- 新增 `apply_lt1_precise_orbit.py` +- 复用现有 LT-1 TXT 解析器 +- 实现 `.slc.par` 读取、备份、状态向量回写 +- 产出 `orbit_bridge_summary.json` + +### Phase 2 + +- 修改 `down2slc_LT1.py` 和 `down2slc_cat_LT1.py` +- 在导入后与最终拼接后调用 helper +- 跑通单任务 smoke test + +### Phase 3 + +- 在运行摘要和结果 manifest 中纳入桥接信息 +- 在生产面板预检区域增加“精轨桥接已启用”可见性 +- 在 health 中增加最小状态摘要 + +### Phase 4 + +- 用真实 LT-1 样本比较桥接前后： + - `ORB_filt_spline.py` 残差 + - 后续 coreg 质量 + - 干涉相位整体趋势 +- 评估是否为 Gamma 配对链复用相同 helper + +## 14. 最终建议 + +推荐按以下原则推进： + +1. 继续保留现有 `Task_*` 输入模式和轨道治理链。 +2. 把 LT-1 精轨接入点明确落到 `.slc.par` 的 `state_vector_*` 回写，不再停留在治理层。 +3. 一期先实现“桥接 helper + vendored LT-1 raw2slc 挂接 + 运行元数据留痕”。 +4. 数据库先不动，运维面板先不扩张。 +5. 桥接 helper 从第一天起就按“未来可复用于 Gamma 配对”来设计接口。 diff --git a/frontend/src/App.css b/frontend/src/App.css index 65f4b5b..673d1b4 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -56,6 +56,14 @@ html, body { overflow: hidden; } +.main-layout--standalone { + padding: 14px 16px 18px; + align-items: stretch; + background: + linear-gradient(180deg, rgba(255, 255, 255, 0.22), rgba(255, 255, 255, 0)) no-repeat, + radial-gradient(circle at top left, rgba(37, 99, 235, 0.08), transparent 36%); +} + .panel-resizer { flex: 0 0 6px; cursor: col-resize; @@ -3811,3 +3819,1182 @@ input[type="checkbox"] { opacity: 0.5; cursor: not-allowed; } + +/* ============ D-InSAR Results Workspace ============ */ + +.dinsar-results-toolbar { + gap: 12px; + padding: 12px 15px 14px; + background: + linear-gradient(180deg, rgba(255, 255, 255, 0.98), rgba(242, 246, 251, 0.98)), + var(--color-panel-muted); +} + +.dinsar-toolbar-grid { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 12px; +} + +.dinsar-toolbar-panel { + display: grid; + gap: 8px; + padding: 12px 14px; + border: 1px solid var(--color-border); + border-radius: 12px; + background: + linear-gradient(180deg, rgba(255, 255, 255, 0.98), rgba(247, 250, 252, 0.98)); + box-shadow: var(--shadow-soft); +} + +.dinsar-toolbar-kicker { + font-size: 11px; + font-weight: 700; + letter-spacing: 0.08em; + text-transform: uppercase; + color: var(--color-text-muted); +} + +.dinsar-toolbar-value { + font-size: 1.25rem; + color: var(--color-text-primary); +} + +.dinsar-toolbar-note { + margin: 0; + font-size: 12px; + line-height: 1.55; + color: var(--color-text-secondary); +} + +.dinsar-toolbar-chip-row { + display: flex; + flex-wrap: wrap; + gap: 8px; +} + +.dinsar-toolbar-chip { + display: inline-flex; + align-items: center; + gap: 6px; + padding: 4px 10px; + border-radius: 999px; + border: 1px solid var(--color-border); + background: rgba(255, 255, 255, 0.92); + color: var(--color-text-secondary); + font-size: 11px; + font-weight: 600; +} + +.dinsar-toolbar-actions { + display: flex; + flex-wrap: wrap; + gap: 8px; +} + +.dinsar-filter-layout { + display: grid; + grid-template-columns: minmax(0, 1.15fr) minmax(0, 1fr) minmax(0, 1fr) minmax(0, 1.6fr); + gap: 10px; +} + +.dinsar-filter-field { + display: grid; + gap: 6px; + min-width: 0; +} + +.dinsar-filter-field span, +.dinsar-pagination-field span { + font-size: 11px; + font-weight: 700; + color: var(--color-text-muted); +} + +.dinsar-filter-field input, +.dinsar-filter-field select, +.dinsar-pagination-field input, +.dinsar-pagination-field select, +.dinsar-catalog-filter-field input, +.dinsar-catalog-filter-field select, +.dinsar-catalog-manage-form textarea, +.dinsar-catalog-manage-form input, +.dinsar-products-field input { + width: 100%; + min-width: 0; + border: 1px solid var(--color-border); + border-radius: 8px; + padding: 8px 10px; + font-size: 13px; + font-family: var(--font-sans); + color: var(--color-text-primary); + background: #fff; +} + +.dinsar-filter-field input:focus, +.dinsar-filter-field select:focus, +.dinsar-pagination-field input:focus, +.dinsar-pagination-field select:focus, +.dinsar-catalog-filter-field input:focus, +.dinsar-catalog-filter-field select:focus, +.dinsar-catalog-manage-form textarea:focus, +.dinsar-catalog-manage-form input:focus, +.dinsar-products-field input:focus { + outline: none; + border-color: var(--color-accent); + box-shadow: 0 0 0 3px var(--color-accent-glow); +} + +.dinsar-score-filter { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + gap: 10px; + align-items: center; +} + +.dinsar-score-filter strong { + min-width: 42px; + text-align: center; + padding: 7px 8px; + border-radius: 8px; + background: var(--color-accent-soft); + color: var(--color-accent-strong); +} + +.dinsar-engine-filter-row { + display: flex; + flex-wrap: wrap; + gap: 8px; +} + +.dinsar-engine-filter-row button { + display: inline-flex; + align-items: center; + gap: 8px; + padding: 6px 12px; + border-radius: 999px; + background: rgba(255, 255, 255, 0.9); +} + +.dinsar-engine-filter-row button strong { + min-width: 24px; + text-align: center; + color: var(--color-text-primary); +} + +.dinsar-engine-filter-row button.active { + border-color: var(--color-accent); + background: var(--color-accent-soft); + color: var(--color-accent-strong); +} + +.dinsar-toolbar-footer { + display: grid; + gap: 8px; +} + +.dinsar-toolbar-footer-main { + display: flex; + flex-wrap: wrap; + align-items: flex-end; + gap: 10px; +} + +.dinsar-pagination-field { + display: grid; + gap: 6px; + min-width: 120px; +} + +.dinsar-pagination-field-jump { + min-width: 240px; +} + +.dinsar-page-jump-input { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + gap: 8px; +} + +.dinsar-page-jump-input input.has-error { + border-color: var(--color-danger); + box-shadow: 0 0 0 2px rgba(220, 38, 38, 0.12); +} + +.dinsar-toolbar-hint { + font-size: 12px; + color: var(--color-text-muted); +} + +.dinsar-toolbar-hint.error { + color: var(--color-danger); +} + +.dinsar-row-header { + display: flex; + width: 100%; + justify-content: space-between; + gap: 8px; + align-items: flex-start; +} + +.dinsar-item { + gap: 8px; + min-height: 118px; + padding-top: 10px; + padding-bottom: 10px; +} + +.dinsar-item .data-item-controls { + width: 100%; + justify-content: space-between; + flex-wrap: wrap; +} + +.dinsar-row-badges { + display: flex; + flex-wrap: wrap; + justify-content: flex-end; + gap: 6px; + flex-shrink: 0; +} + +.dinsar-engine-badge { + display: inline-flex; + align-items: center; + justify-content: center; + padding: 3px 9px; + border-radius: 999px; + border: 1px solid var(--color-border); + background: rgba(255, 255, 255, 0.9); + font-size: 11px; + font-weight: 700; + line-height: 1; +} + +.dinsar-engine-badge.tone-envi { + border-color: rgba(14, 165, 164, 0.34); + background: rgba(14, 165, 164, 0.12); + color: #0f766e; +} + +.dinsar-engine-badge.tone-isce2 { + border-color: rgba(37, 99, 235, 0.32); + background: rgba(37, 99, 235, 0.12); + color: #1d4ed8; +} + +.dinsar-engine-badge.tone-pyint { + border-color: rgba(190, 24, 93, 0.28); + background: rgba(244, 63, 94, 0.12); + color: #be123c; +} + +.dinsar-engine-badge.tone-landsar { + border-color: rgba(217, 119, 6, 0.32); + background: rgba(245, 158, 11, 0.14); + color: #b45309; +} + +.dinsar-engine-badge.tone-unknown { + border-color: rgba(100, 116, 139, 0.28); + background: rgba(148, 163, 184, 0.12); + color: #475569; +} + +.dinsar-trace-stat { + display: inline-flex; + align-items: center; + gap: 6px; + padding: 3px 8px; + border-radius: 999px; + background: rgba(148, 163, 184, 0.12); + color: var(--color-text-secondary); +} + +.dinsar-trace-stat strong { + font-size: 10px; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--color-text-muted); +} + +.dinsar-control-cluster { + display: flex; + align-items: center; + gap: 8px; +} + +.dinsar-toggle-label { + display: inline-flex; + align-items: center; + gap: 6px; + padding: 5px 8px; + border-radius: 8px; + border: 1px solid var(--color-border); + background: #fff; + color: var(--color-text-secondary); + font-size: 12px; + cursor: pointer; +} + +.dinsar-toggle-label input[type="checkbox"] { + accent-color: var(--color-accent); +} + +/* ============ D-InSAR Catalog ============ */ + +.dinsar-status-pill { + display: inline-flex; + align-items: center; + justify-content: center; + padding: 3px 10px; + border-radius: 999px; + border: 1px solid transparent; + font-size: 11px; + font-weight: 700; + line-height: 1.2; +} + +.dinsar-status-pill.tone-ready { + background: rgba(22, 163, 74, 0.12); + color: #166534; + border-color: rgba(22, 163, 74, 0.25); +} + +.dinsar-status-pill.tone-info { + background: rgba(14, 165, 233, 0.12); + color: #0369a1; + border-color: rgba(14, 165, 233, 0.24); +} + +.dinsar-status-pill.tone-warn { + background: rgba(245, 158, 11, 0.14); + color: #b45309; + border-color: rgba(245, 158, 11, 0.25); +} + +.dinsar-status-pill.tone-error { + background: rgba(220, 38, 38, 0.12); + color: #b91c1c; + border-color: rgba(220, 38, 38, 0.24); +} + +.dinsar-status-pill.tone-neutral { + background: rgba(148, 163, 184, 0.12); + color: #475569; + border-color: rgba(148, 163, 184, 0.24); +} + +.dinsar-catalog-shell, +.dinsar-products-card, +.dinsar-products-hero { + background: linear-gradient(180deg, rgba(255, 255, 255, 0.98), rgba(249, 251, 255, 0.98)); + border: 1px solid var(--color-border); + border-radius: 14px; + box-shadow: var(--shadow-soft); +} + +.dinsar-catalog-shell { + padding: 16px; + display: grid; + gap: 14px; +} + +.dinsar-catalog-header, +.dinsar-products-hero { + display: flex; + justify-content: space-between; + gap: 16px; + align-items: flex-start; +} + +.dinsar-catalog-header-copy strong, +.dinsar-products-hero strong { + display: block; + font-size: 16px; + color: var(--color-text-primary); +} + +.dinsar-catalog-header-copy p, +.dinsar-products-hero p { + margin: 6px 0 0; + max-width: 760px; + font-size: 13px; + line-height: 1.7; + color: var(--color-text-secondary); +} + +.dinsar-catalog-header-actions, +.dinsar-products-hero-badges { + display: flex; + flex-wrap: wrap; + gap: 8px; + align-items: center; +} + +.dinsar-catalog-summary { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 10px; +} + +.dinsar-catalog-stat-card { + display: grid; + gap: 6px; + padding: 12px; + border-radius: 12px; + border: 1px solid var(--color-border); + background: rgba(248, 250, 252, 0.95); +} + +.dinsar-catalog-stat-card span { + font-size: 11px; + font-weight: 700; + color: var(--color-text-muted); +} + +.dinsar-catalog-stat-card strong { + font-size: 18px; + color: var(--color-text-primary); +} + +.dinsar-catalog-stat-card small { + font-size: 12px; + color: var(--color-text-secondary); +} + +.dinsar-catalog-meta-strip { + display: grid; + gap: 6px; + padding: 12px; + border-radius: 12px; + background: rgba(239, 246, 255, 0.75); + color: var(--color-text-secondary); + font-size: 12px; + word-break: break-all; +} + +.dinsar-catalog-message, +.dinsar-products-message { + padding: 10px 12px; + border-radius: 12px; + font-size: 13px; + font-weight: 600; +} + +.dinsar-catalog-message.tone-success, +.dinsar-products-message.tone-success { + background: rgba(22, 163, 74, 0.12); + color: #166534; + border: 1px solid rgba(22, 163, 74, 0.24); +} + +.dinsar-catalog-message.tone-error, +.dinsar-products-message.tone-error { + background: rgba(220, 38, 38, 0.12); + color: #b91c1c; + border: 1px solid rgba(220, 38, 38, 0.24); +} + +.dinsar-catalog-manage { + display: grid; + grid-template-columns: minmax(0, 1.15fr) minmax(0, 1.85fr); + gap: 14px; + padding: 14px; + border: 1px solid var(--color-border); + border-radius: 14px; + background: rgba(248, 250, 252, 0.92); +} + +.dinsar-catalog-manage-copy strong { + display: block; + margin-bottom: 6px; + font-size: 14px; + color: var(--color-text-primary); +} + +.dinsar-catalog-manage-copy p { + margin: 0; + font-size: 13px; + line-height: 1.7; + color: var(--color-text-secondary); +} + +.dinsar-catalog-manage-form { + display: grid; + gap: 10px; +} + +.dinsar-catalog-manage-form textarea { + min-height: 94px; + resize: vertical; +} + +.dinsar-catalog-manage-actions, +.dinsar-products-actions, +.dinsar-catalog-filter-actions { + display: flex; + flex-wrap: wrap; + gap: 8px; +} + +.dinsar-catalog-manage-actions .primary, +.dinsar-products-actions .primary { + background: var(--color-accent); + color: #fff; + border-color: var(--color-accent); +} + +.dinsar-catalog-manage-actions .primary:hover:not(:disabled), +.dinsar-products-actions .primary:hover:not(:disabled) { + background: var(--color-accent-strong); +} + +.dinsar-catalog-workspace { + display: grid; + grid-template-columns: minmax(360px, 420px) minmax(0, 1fr); + gap: 14px; + min-height: 0; +} + +.panel--standalone .dinsar-catalog-workspace { + grid-template-columns: minmax(380px, 440px) minmax(0, 1fr); +} + +.dinsar-catalog-list-card, +.dinsar-catalog-detail-card { + min-height: 0; + border: 1px solid var(--color-border); + border-radius: 14px; + background: rgba(255, 255, 255, 0.95); + display: flex; + flex-direction: column; + overflow: hidden; +} + +.panel--standalone { + width: 100%; + border-right: none; + border-left: none; + box-shadow: none; + background: transparent; +} + +.panel--standalone .panel-tabs { + padding-left: 0; + padding-right: 0; + background: transparent; + border-bottom: none; +} + +.panel--standalone .panel-content { + overflow-y: auto; +} + +.panel-standalone-header { + display: flex; + justify-content: space-between; + align-items: flex-start; + gap: 18px; + padding: 4px 0 18px; +} + +.panel-standalone-header-main { + display: grid; + gap: 8px; + min-width: 0; +} + +.panel-standalone-eyebrow { + font-size: 11px; + font-weight: 800; + letter-spacing: 0.08em; + text-transform: uppercase; + color: var(--color-text-muted); +} + +.panel-standalone-header-main strong { + font-size: 24px; + line-height: 1.2; + color: var(--color-text-primary); +} + +.panel-standalone-header-main p { + margin: 0; + max-width: 860px; + font-size: 13px; + line-height: 1.7; + color: var(--color-text-secondary); +} + +.panel-standalone-actions { + display: flex; + flex-wrap: wrap; + justify-content: flex-end; + gap: 8px; +} + +.panel-standalone-actions button { + padding: 8px 14px; + border-radius: 999px; + background: rgba(255, 255, 255, 0.92); + border: 1px solid var(--color-border); + color: var(--color-text-secondary); +} + +.panel-standalone-actions button.panel-standalone-return { + background: rgba(15, 23, 42, 0.86); + border-color: rgba(15, 23, 42, 0.92); + color: #fff; +} + +.panel-standalone-actions button.active-tab { + background: var(--color-accent); + border-color: var(--color-accent); + color: #fff; +} + +.dinsar-catalog-card-head { + display: flex; + justify-content: space-between; + gap: 12px; + align-items: center; + padding: 12px 14px; + border-bottom: 1px solid var(--color-border); + background: rgba(248, 250, 252, 0.94); +} + +.dinsar-catalog-card-head strong { + display: block; + font-size: 14px; +} + +.dinsar-catalog-card-head span { + display: block; + margin-top: 4px; + font-size: 12px; + color: var(--color-text-muted); +} + +.dinsar-catalog-filter-bar { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(0, 1.4fr) auto; + gap: 10px; + padding: 12px 14px; + border-bottom: 1px solid var(--color-border); +} + +.dinsar-catalog-filter-field { + display: grid; + gap: 6px; +} + +.dinsar-catalog-filter-field span { + font-size: 11px; + font-weight: 700; + color: var(--color-text-muted); +} + +.dinsar-catalog-empty { + padding: 20px 14px; + color: var(--color-text-muted); + font-size: 13px; +} + +.dinsar-catalog-empty.inline { + padding: 8px 0 0; +} + +.dinsar-catalog-empty.error { + color: var(--color-danger); +} + +.dinsar-catalog-empty.ok { + color: #166534; +} + +.dinsar-catalog-list { + display: grid; + gap: 10px; + padding: 12px; + overflow-y: auto; +} + +.dinsar-catalog-list-item { + display: grid; + gap: 8px; + padding: 12px; + text-align: left; + border-radius: 12px; + border: 1px solid var(--color-border); + background: #fff; + box-shadow: none; +} + +.dinsar-catalog-list-item.active { + border-color: rgba(37, 99, 235, 0.3); + background: rgba(239, 246, 255, 0.92); + box-shadow: 0 0 0 2px rgba(37, 99, 235, 0.08); +} + +.dinsar-catalog-list-item-top, +.dinsar-catalog-asset-top, +.dinsar-catalog-issue-top, +.dinsar-monitor-top, +.dinsar-monitor-log-head { + display: flex; + justify-content: space-between; + gap: 10px; + align-items: center; +} + +.dinsar-catalog-list-item-top strong { + min-width: 0; + font-size: 13px; + line-height: 1.5; + word-break: break-word; +} + +.dinsar-catalog-list-item-badges, +.dinsar-catalog-hero-badges { + display: flex; + flex-wrap: wrap; + gap: 8px; + align-items: center; + font-size: 11px; + color: var(--color-text-muted); +} + +.dinsar-catalog-list-item-meta, +.dinsar-catalog-list-item-trace, +.dinsar-monitor-message, +.dinsar-monitor-log-message, +.dinsar-catalog-asset-item, +.dinsar-catalog-issue-item { + font-size: 12px; + line-height: 1.6; + color: var(--color-text-secondary); + word-break: break-all; +} + +.dinsar-catalog-list-item-trace { + color: var(--color-text-muted); +} + +.dinsar-catalog-detail-body { + padding: 14px; + display: grid; + gap: 14px; + overflow-y: auto; +} + +.dinsar-catalog-hero { + display: grid; + grid-template-columns: minmax(200px, 280px) minmax(0, 1fr); + gap: 14px; +} + +.dinsar-catalog-preview-frame { + overflow: hidden; + min-height: 180px; + border-radius: 14px; + border: 1px solid var(--color-border); + background: + linear-gradient(135deg, rgba(226, 232, 240, 0.86), rgba(241, 245, 249, 0.96)); +} + +.dinsar-catalog-preview-frame img { + display: block; + width: 100%; + height: 100%; + object-fit: cover; +} + +.dinsar-catalog-hero-meta { + display: grid; + gap: 12px; + min-width: 0; +} + +.dinsar-catalog-hero-title-row { + display: flex; + justify-content: space-between; + gap: 14px; + align-items: flex-start; +} + +.dinsar-catalog-hero-title-row h4 { + margin: 0; + font-size: 18px; +} + +.dinsar-catalog-hero-title-row p { + margin: 6px 0 0; + font-size: 12px; + color: var(--color-text-muted); +} + +.dinsar-catalog-kv-grid, +.dinsar-catalog-detail-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 12px; +} + +.dinsar-catalog-meta-field { + display: grid; + gap: 4px; + padding: 10px 12px; + border-radius: 12px; + border: 1px solid var(--color-border); + background: rgba(248, 250, 252, 0.88); +} + +.dinsar-catalog-meta-field span { + font-size: 11px; + font-weight: 700; + color: var(--color-text-muted); +} + +.dinsar-catalog-meta-field strong { + font-size: 12px; + line-height: 1.55; + color: var(--color-text-primary); +} + +.break-all { + word-break: break-all; +} + +.dinsar-catalog-section-card { + display: grid; + gap: 10px; + padding: 12px; + border-radius: 14px; + border: 1px solid var(--color-border); + background: rgba(255, 255, 255, 0.95); +} + +.dinsar-catalog-section-card.nested { + background: rgba(248, 250, 252, 0.92); +} + +.dinsar-catalog-section-title { + font-size: 13px; + font-weight: 700; + color: var(--color-text-primary); +} + +.dinsar-catalog-asset-list, +.dinsar-catalog-issue-list, +.dinsar-monitor-log-list { + display: grid; + gap: 10px; +} + +.dinsar-catalog-asset-item, +.dinsar-catalog-issue-item { + padding: 10px 12px; + border-radius: 12px; + border: 1px solid var(--color-border); + background: rgba(248, 250, 252, 0.92); +} + +.dinsar-catalog-asset-item.ok { + border-color: rgba(22, 163, 74, 0.22); +} + +.dinsar-catalog-asset-item.missing, +.dinsar-catalog-issue-item.error { + border-color: rgba(220, 38, 38, 0.2); + background: rgba(254, 242, 242, 0.92); +} + +.dinsar-catalog-issue-item.warn { + border-color: rgba(245, 158, 11, 0.22); + background: rgba(255, 247, 237, 0.92); +} + +.dinsar-catalog-asset-top span, +.dinsar-catalog-issue-top span, +.dinsar-catalog-issue-action, +.dinsar-monitor-log-time { + font-size: 11px; + color: var(--color-text-muted); +} + +.dinsar-products-page { + width: 100%; + max-width: 1680px; + margin: 0 auto; + padding: 6px 0 24px; + display: grid; + gap: 18px; +} + +.panel--standalone .dinsar-products-page { + max-width: none; +} + +.dinsar-products-hero { + padding: 16px 18px; +} + +.dinsar-products-top-grid { + display: grid; + grid-template-columns: minmax(420px, 0.95fr) minmax(460px, 1.05fr); + gap: 16px; + align-items: start; +} + +.dinsar-products-card { + padding: 14px; + display: grid; + gap: 12px; +} + +.dinsar-products-card.monitor.tone-warn { + border-color: rgba(245, 158, 11, 0.28); +} + +.dinsar-products-card-head { + display: flex; + justify-content: space-between; + gap: 12px; + align-items: center; +} + +.dinsar-products-card-head strong { + display: block; + font-size: 14px; +} + +.dinsar-products-card-head span { + display: block; + margin-top: 4px; + font-size: 12px; + color: var(--color-text-muted); +} + +.dinsar-products-form-grid { + display: grid; + grid-template-columns: minmax(0, 1.4fr) minmax(0, 1fr); + gap: 10px; +} + +.dinsar-products-field { + display: grid; + gap: 6px; +} + +.dinsar-products-field span { + font-size: 11px; + font-weight: 700; + color: var(--color-text-muted); +} + +.dinsar-products-result-card { + padding: 12px; + border-radius: 12px; + font-size: 13px; + line-height: 1.65; +} + +.dinsar-products-result-card.success { + background: rgba(22, 163, 74, 0.1); + color: #166534; +} + +.dinsar-products-result-card.error { + background: rgba(220, 38, 38, 0.1); + color: #b91c1c; +} + +.dinsar-products-empty { + color: var(--color-text-muted); + font-size: 13px; +} + +.dinsar-monitor-card { + display: grid; + gap: 12px; + padding: 12px; + border-radius: 14px; + border: 1px solid var(--color-border); + background: rgba(248, 250, 252, 0.92); +} + +.dinsar-monitor-top strong { + display: block; + font-size: 14px; +} + +.dinsar-monitor-top span, +.dinsar-monitor-task-id { + font-size: 12px; + color: var(--color-text-muted); +} + +.dinsar-monitor-task-id { + font-family: var(--font-mono); +} + +.dinsar-monitor-progress { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + gap: 10px; + align-items: center; +} + +.dinsar-monitor-progress-track { + height: 8px; + border-radius: 999px; + overflow: hidden; + background: rgba(148, 163, 184, 0.2); +} + +.dinsar-monitor-progress-bar { + height: 100%; + background: linear-gradient(90deg, #38bdf8, #2563eb); +} + +.dinsar-monitor-log-item { + display: flex; + justify-content: space-between; + gap: 10px; + align-items: flex-start; + padding: 10px 12px; + border-radius: 12px; + border: 1px solid var(--color-border); + background: rgba(255, 255, 255, 0.96); +} + +.dinsar-monitor-log-item.tone-error { + border-color: rgba(220, 38, 38, 0.22); + background: rgba(254, 242, 242, 0.92); +} + +.dinsar-monitor-log-item.tone-warn { + border-color: rgba(245, 158, 11, 0.22); + background: rgba(255, 247, 237, 0.92); +} + +.dinsar-monitor-log-item .danger { + color: #b91c1c; + border-color: rgba(220, 38, 38, 0.2); + background: rgba(254, 242, 242, 0.92); +} + +.dinsar-monitor-log-main { + min-width: 0; +} + +.export-result-item label { + display: flex; + align-items: center; + gap: 8px; +} + +.export-result-item .dinsar-engine-badge { + margin-left: auto; +} + +@media (max-width: 1200px) { + .dinsar-toolbar-grid, + .dinsar-catalog-summary, + .dinsar-products-top-grid, + .dinsar-catalog-manage, + .dinsar-catalog-hero, + .dinsar-catalog-detail-grid, + .dinsar-catalog-kv-grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + + .dinsar-filter-layout { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + + .dinsar-catalog-workspace { + grid-template-columns: minmax(0, 320px) minmax(0, 1fr); + } +} + +@media (max-width: 1500px) { + .panel--standalone .dinsar-products-top-grid { + grid-template-columns: 1fr; + } +} + +@media (max-width: 900px) { + .dinsar-toolbar-grid, + .dinsar-filter-layout, + .dinsar-catalog-summary, + .dinsar-catalog-workspace, + .dinsar-catalog-manage, + .dinsar-catalog-filter-bar, + .dinsar-catalog-hero, + .dinsar-catalog-detail-grid, + .dinsar-catalog-kv-grid, + .dinsar-products-top-grid, + .dinsar-products-form-grid { + grid-template-columns: 1fr; + } + + .dinsar-catalog-header, + .dinsar-products-hero, + .dinsar-catalog-card-head, + .dinsar-products-card-head, + .dinsar-catalog-hero-title-row, + .dinsar-monitor-top, + .dinsar-monitor-log-head, + .dinsar-row-header, + .dinsar-item .data-item-controls { + flex-direction: column; + align-items: flex-start; + } + + .dinsar-row-badges { + justify-content: flex-start; + } + + .dinsar-toolbar-footer-main { + align-items: stretch; + } + + .panel-standalone-header { + flex-direction: column; + } + + .panel-standalone-actions { + justify-content: flex-start; + } +} + +@media (max-width: 640px) { + .dinsar-products-page { + padding: 12px; + } + + .dinsar-results-toolbar, + .dinsar-catalog-shell, + .dinsar-products-card, + .dinsar-products-hero { + padding-left: 12px; + padding-right: 12px; + } + + .dinsar-engine-filter-row, + .dinsar-toolbar-actions, + .dinsar-catalog-manage-actions, + .dinsar-products-actions { + flex-direction: column; + } + + .dinsar-engine-filter-row button, + .dinsar-toolbar-actions button, + .dinsar-catalog-manage-actions button, + .dinsar-products-actions button { + width: 100%; + } +} diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index d9cac00..5d0c13e 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -31,6 +31,7 @@ import { NATIONAL_BOUNDARY_GEOJSON_URL, getBaseLayerConfig, DEFAULT_LIST_PAGE_SIZE, + FULL_WIDTH_LEFT_TABS, } from './config/appConstants'; import { escapeHtml, formatCoordinate } from './utils/appHelpers'; import { @@ -42,6 +43,7 @@ import { DINSAR_STRATEGY_ALL, filterDinsarResults, } from './utils/dinsarResultFilters'; +import { DINSAR_ENGINE_ALL, getDinsarEngineMeta } from './utils/dinsarEngines'; const NATIONAL_BOUNDARY_STATIC_URL = '/geojson/\u5168\u56fd\u884c\u653f\u533a.geojson'; @@ -78,6 +80,7 @@ function App() { activeTasks, setActiveTasks, isGlobalLocked, setIsGlobalLocked, isCheckingTasks, setIsCheckingTasks, pendingTaskIds, setPendingTaskIds, + nonBlockingTaskIds, setNonBlockingTaskIds, } = useTaskStore(useShallow((state) => ({ activeTasks: state.activeTasks, setActiveTasks: state.setActiveTasks, @@ -87,6 +90,8 @@ function App() { setIsCheckingTasks: state.setIsCheckingTasks, pendingTaskIds: state.pendingTaskIds, setPendingTaskIds: state.setPendingTaskIds, + nonBlockingTaskIds: state.nonBlockingTaskIds, + setNonBlockingTaskIds: state.setNonBlockingTaskIds, }))); const { leftPanelTab, setLeftPanelTab, leftPanelWidth, setLeftPanelWidth, @@ -170,7 +175,7 @@ function App() { const { dinsarResults, setDinsarResults, dinsarPagination, dinsarPageInput, setDinsarPageInput, dinsarPageInputTouched, setDinsarPageInputTouched, - aiStatus, scoreFilter, setScoreFilter, traceSearch, strategyFilter, + aiStatus, scoreFilter, setScoreFilter, engineFilter, traceSearch, strategyFilter, } = useDinsarStore(useShallow((state) => ({ dinsarResults: state.dinsarResults, setDinsarResults: state.setDinsarResults, @@ -182,6 +187,7 @@ function App() { aiStatus: state.aiStatus, scoreFilter: state.scoreFilter, setScoreFilter: state.setScoreFilter, + engineFilter: state.engineFilter, traceSearch: state.traceSearch, strategyFilter: state.strategyFilter, }))); @@ -291,6 +297,19 @@ function App() { const mapBatchRef = useRef({ frameId: null, token: 0 }); const isAdmin = currentUser?.role === 'admin'; const isReadOnlyUser = !!currentUser && !isAdmin; + const isStandaloneLeftPage = FULL_WIDTH_LEFT_TABS.has(leftPanelTab); + + useEffect(() => { + if (isStandaloneLeftPage || !mapRef.current) { + return undefined; + } + + const frameId = requestAnimationFrame(() => { + mapRef.current?.invalidateSize(false); + }); + + return () => cancelAnimationFrame(frameId); + }, [isStandaloneLeftPage, leftPanelWidth, rightPanelWidth]); const getVisibleLayerRefs = useCallback(() => ({ activeLayersRef: activeLayersRef.current, @@ -776,6 +795,8 @@ function App() { setActiveTasks, pendingTaskIds, setPendingTaskIds, + nonBlockingTaskIds, + setNonBlockingTaskIds, isGlobalLocked, setIsGlobalLocked, setIsCheckingTasks, @@ -926,7 +947,7 @@ function App() { }); allDataRef.current = newAllData; setAllData(newAllData); - addLog('info', `正在预览包含 ${stack.length} 个场景的PS时序栈。`); + addLog('info', `正在预览包含 ${stack.length} 个场景的时序InSAR候选栈。`); }; const updateLayerTooltip = useCallback((layer, result, show) => { @@ -973,10 +994,12 @@ function App() { const aiScore = result.ai_score === null || result.ai_score === undefined ? '-' : `${(Number(result.ai_score) * 100).toFixed(0)}%`; + const engine = getDinsarEngineMeta(result.engine_code); const strategy = escapeHtml(result.selection_strategy || 'legacy'); const taskAlias = escapeHtml(result.task_alias || result.task_name || result.name || '-'); const pairKey = escapeHtml(result.pair_key || '-'); const pairUid = escapeHtml(result.pair_uid || '-'); + const runKey = escapeHtml(result.run_key || '-'); const networkRunId = escapeHtml(result.network_run_id || '-'); const networkEdgeId = escapeHtml(result.network_edge_id ?? '-'); const policyVersion = escapeHtml(result.policy_version || '-'); @@ -990,9 +1013,11 @@ function App() { <div><strong>任务:</strong> ${taskAlias}</div> <div><strong>日期:</strong> ${dateText}</div> <div><strong>AI:</strong> ${aiScore}</div> + <div><strong>引擎:</strong> ${escapeHtml(engine.label)} <span class="mono">(${escapeHtml(engine.code)})</span></div> <div><strong>策略:</strong> ${strategy}</div> <div><strong>edge:</strong> ${networkEdgeId}</div> <div><strong>run:</strong> <span class="mono">${networkRunId}</span></div> + <div><strong>run_key:</strong> <span class="mono">${runKey}</span></div> <div><strong>pair_key:</strong> <span class="mono">${pairKey}</span></div> <div><strong>pair_uid:</strong> <span class="mono">${pairUid}</span></div> <div><strong>policy:</strong> ${policyVersion}</div> @@ -1071,6 +1096,7 @@ function App() { const currentResults = dinsarResultsRef.current; const currentFilters = { scoreFilter, + engineFilter, strategyFilter, traceSearch, focusedHazardPoint, @@ -1083,6 +1109,9 @@ function App() { if (scoreFilter > 0) { filterNotes.push(`AI>=${Math.round(scoreFilter * 100)}%`); } + if (engineFilter !== DINSAR_ENGINE_ALL) { + filterNotes.push(`引擎:${getDinsarEngineMeta(engineFilter).shortLabel}`); + } if (focusedHazardPoint?.hazard_name) { filterNotes.push(`点位:${focusedHazardPoint.hazard_name}`); } @@ -1124,7 +1153,7 @@ function App() { runMapBatch(changedResults, (result) => { updateDinsarLayer(result, result.isVisible); }); - }, [addLog, focusedHazardPoint, runMapBatch, scoreFilter, setDinsarResults, strategyFilter, traceSearch, updateDinsarLayer]); + }, [addLog, engineFilter, focusedHazardPoint, runMapBatch, scoreFilter, setDinsarResults, strategyFilter, traceSearch, updateDinsarLayer]); const handleScoreFilterChange = useCallback((e) => { cancelMapBatch(); @@ -1562,11 +1591,12 @@ function App() { onRefreshHealth={handleRefreshHealth} onLogout={handleLogout} /> - <div className="main-layout"> + <div className={`main-layout${isStandaloneLeftPage ? ' main-layout--standalone' : ''}`}> <AppSidePanel - leftPanelWidth={leftPanelWidth} + leftPanelWidth={isStandaloneLeftPage ? '100%' : leftPanelWidth} leftPanelTab={leftPanelTab} setLeftPanelTab={setLeftPanelTab} + isStandalone={isStandaloneLeftPage} isAdmin={isAdmin} isReadOnlyUser={isReadOnlyUser} currentUser={currentUser} @@ -1594,28 +1624,32 @@ function App() { psPanel={psPanel} /> - <div className="panel-resizer" onMouseDown={(event) => startResize('left', event)} /> + {!isStandaloneLeftPage && ( + <> + <div className="panel-resizer" onMouseDown={(event) => startResize('left', event)} /> - <AppMapWorkspace - language={language} - showMapRegionLocator={showMapRegionLocator} - toggleMapRegionLocator={toggleMapRegionLocator} - mapRegionOptions={mapRegionOptions} - mapRegionSelection={mapRegionSelection} - mapRegionLoading={mapRegionLoading} - mapRegionLocating={mapRegionLocating} - mapRegionError={mapRegionError} - mapRegionLocatedName={mapRegionLocatedName} - onMapRegionProvinceChange={handleMapRegionProvinceChange} - onMapRegionCityChange={handleMapRegionCityChange} - onLocateSelectedRegion={locateSelectedRegionOnMap} - onClearMapRegionHighlight={clearMapRegionHighlight} - baseLayerKey={baseLayerKey} - setBaseLayerKey={setBaseLayerKey} - onOpenExportModal={mapExport.openExportModal} - /> - <div className="panel-resizer" onMouseDown={(event) => startResize('right', event)} /> - <AppLogPanel width={rightPanelWidth} /> + <AppMapWorkspace + language={language} + showMapRegionLocator={showMapRegionLocator} + toggleMapRegionLocator={toggleMapRegionLocator} + mapRegionOptions={mapRegionOptions} + mapRegionSelection={mapRegionSelection} + mapRegionLoading={mapRegionLoading} + mapRegionLocating={mapRegionLocating} + mapRegionError={mapRegionError} + mapRegionLocatedName={mapRegionLocatedName} + onMapRegionProvinceChange={handleMapRegionProvinceChange} + onMapRegionCityChange={handleMapRegionCityChange} + onLocateSelectedRegion={locateSelectedRegionOnMap} + onClearMapRegionHighlight={clearMapRegionHighlight} + baseLayerKey={baseLayerKey} + setBaseLayerKey={setBaseLayerKey} + onOpenExportModal={mapExport.openExportModal} + /> + <div className="panel-resizer" onMouseDown={(event) => startResize('right', event)} /> + <AppLogPanel width={rightPanelWidth} /> + </> + )} </div> <AppOverlays diff --git a/frontend/src/DataMonitorPanel.jsx b/frontend/src/DataMonitorPanel.jsx index b844789..6bf6965 100644 --- a/frontend/src/DataMonitorPanel.jsx +++ b/frontend/src/DataMonitorPanel.jsx @@ -1,5 +1,5 @@ -import React, { useState, useEffect, useRef } from 'react'; -import './App.css'; // 复用现有样式 +import React, { useEffect, useRef, useState } from 'react'; +import './App.css'; import { useI18n } from './i18n/I18nContext'; const DEFAULT_MONITOR_CONFIG = { @@ -7,18 +7,42 @@ const DEFAULT_MONITOR_CONFIG = { orbit_dir: '', dinsar_dirs: [], gf3_source_dirs: [], - gf3_storage_dirs: [] + gf3_storage_dirs: [], }; const DEFAULT_UNPACK_CONFIG = { source_dirs: [], insar_storage_dirs: [], min_disk_space_gb: 50, + max_files_per_run: 0, + max_runtime_minutes: 0, delete_archive: true, tmp_suffix: '.unpack_tmp', - archive_exts: [] + archive_exts: [], }; +const toArray = (value) => (Array.isArray(value) ? value : []); + +const createUnpackRunOptions = (config = DEFAULT_UNPACK_CONFIG) => ({ + max_files_per_run: String(config?.max_files_per_run ?? 0), + max_runtime_minutes: String(config?.max_runtime_minutes ?? 0), +}); + +const parseUnpackRunValue = (rawValue, label) => { + const value = String(rawValue ?? '').trim(); + if (!value) { + return 0; + } + + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 0) { + throw new Error(`${label}必须是大于等于 0 的整数`); + } + return parsed; +}; + +const formatList = (list) => (Array.isArray(list) && list.length ? list.join('; ') : '未配置'); + const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled = true }) => { const { t } = useI18n(); const [config, setConfig] = useState(DEFAULT_MONITOR_CONFIG); @@ -30,10 +54,15 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled const [message, setMessage] = useState(''); const [unpackLoading, setUnpackLoading] = useState(false); const [unpackMessage, setUnpackMessage] = useState(''); + const [showUnpackDialog, setShowUnpackDialog] = useState(false); + const [unpackRunOptions, setUnpackRunOptions] = useState(() => createUnpackRunOptions()); + const [unpackDialogError, setUnpackDialogError] = useState(''); + const [unpackTaskId, setUnpackTaskId] = useState(''); + const [unpackTaskTerminal, setUnpackTaskTerminal] = useState(false); const [gf3Loading, setGf3Loading] = useState(false); const [gf3Message, setGf3Message] = useState(''); const logEndRef = useRef(null); - const toArray = (value) => (Array.isArray(value) ? value : []); + const parseJsonSafe = async (response, fallback) => { try { return await response.json(); @@ -41,12 +70,13 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled return fallback; } }; + const displayLogs = toArray(logs); const displayActiveTasks = toArray(activeTasks); + const unpackActiveTask = displayActiveTasks.find((task) => + task.task_id === unpackTaskId || task.task_type === 'UNPACK_ARCHIVES' + ); - const formatList = (list) => (list && list.length ? list.join('; ') : '未配置'); - - // 获取初始状态 useEffect(() => { if (!enabled) { setConfig(DEFAULT_MONITOR_CONFIG); @@ -63,21 +93,25 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled } return data; }) - .then(data => { - if (canceled) return; + .then((data) => { + if (canceled) { + return; + } setConfig({ ...DEFAULT_MONITOR_CONFIG, ...data, radar_dirs: toArray(data?.radar_dirs), dinsar_dirs: toArray(data?.dinsar_dirs), gf3_source_dirs: toArray(data?.gf3_source_dirs), - gf3_storage_dirs: toArray(data?.gf3_storage_dirs) + gf3_storage_dirs: toArray(data?.gf3_storage_dirs), }); setConfigLoaded(true); }) - .catch(err => { - if (canceled) return; - console.error("Failed to fetch monitor status:", err); + .catch((err) => { + if (canceled) { + return; + } + console.error('Failed to fetch monitor status:', err); setConfigLoaded(false); }); @@ -89,6 +123,9 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled useEffect(() => { if (!enabled) { setUnpackConfig(DEFAULT_UNPACK_CONFIG); + setShowUnpackDialog(false); + setUnpackRunOptions(createUnpackRunOptions()); + setUnpackDialogError(''); return; } @@ -101,18 +138,22 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled } return data; }) - .then(data => { - if (canceled) return; + .then((data) => { + if (canceled) { + return; + } setUnpackConfig({ ...DEFAULT_UNPACK_CONFIG, ...data, source_dirs: toArray(data?.source_dirs), - insar_storage_dirs: toArray(data?.insar_storage_dirs) + insar_storage_dirs: toArray(data?.insar_storage_dirs), }); }) - .catch(err => { - if (canceled) return; - console.error("Failed to fetch unpack config:", err); + .catch((err) => { + if (canceled) { + return; + } + console.error('Failed to fetch unpack config:', err); }); return () => { @@ -120,11 +161,12 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled }; }, [apiEndpoint, enabled]); - // 轮询日志 useEffect(() => { if (!enabled) { setLogs([]); setActiveTasks([]); + setUnpackTaskId(''); + setUnpackTaskTerminal(false); return; } @@ -133,59 +175,121 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled try { const [logsRes, tasksRes] = await Promise.all([ fetch(`${apiEndpoint}/monitor/logs`, { credentials: 'include' }), - fetch(`${apiEndpoint}/tasks/active`, { credentials: 'include' }) + fetch(`${apiEndpoint}/tasks/active`, { credentials: 'include' }), ]); const [logsData, tasksData] = await Promise.all([ parseJsonSafe(logsRes, {}), - parseJsonSafe(tasksRes, []) + parseJsonSafe(tasksRes, []), ]); - if (canceled) return; + if (canceled) { + return; + } setLogs(logsRes.ok ? toArray(logsData?.logs) : []); setActiveTasks(tasksRes.ok ? toArray(tasksData) : []); } catch (err) { - if (canceled) return; - console.error("Failed to fetch logs or tasks:", err); + if (canceled) { + return; + } + console.error('Failed to fetch logs or tasks:', err); } }; fetchLogsAndTasks(); const intervalId = setInterval(fetchLogsAndTasks, 2000); + return () => { canceled = true; clearInterval(intervalId); }; }, [apiEndpoint, enabled]); - // 自动滚动到底部 + useEffect(() => { + if (!enabled) { + setUnpackTaskId(''); + setUnpackTaskTerminal(false); + return; + } + + if (unpackActiveTask) { + if (!unpackTaskId) { + setUnpackTaskId(unpackActiveTask.task_id); + } + setUnpackTaskTerminal(false); + setUnpackMessage( + `运行中 (${unpackActiveTask.progress || 0}%): ${unpackActiveTask.message || '正在解包 LT-1 归档...'}` + ); + return; + } + + if (!unpackTaskId || unpackTaskTerminal) { + return; + } + + let canceled = false; + fetch(`${apiEndpoint}/tasks/${unpackTaskId}`, { credentials: 'include' }) + .then(async (res) => { + const data = await parseJsonSafe(res, null); + if (!res.ok) { + throw new Error(data?.detail || `HTTP ${res.status}`); + } + return data; + }) + .then((task) => { + if (canceled || !task) { + return; + } + if (task.status === 'COMPLETED') { + setUnpackMessage(task.message || 'LT-1 解包已完成。'); + setUnpackTaskTerminal(true); + } else if (task.status === 'FAILED') { + setUnpackMessage(`失败：${task.message || 'LT-1 解包任务失败'}`); + setUnpackTaskTerminal(true); + } + }) + .catch((err) => { + if (canceled) { + return; + } + console.error('Failed to fetch unpack task status:', err); + }); + + return () => { + canceled = true; + }; + }, [apiEndpoint, enabled, unpackActiveTask, unpackTaskId, unpackTaskTerminal]); + useEffect(() => { if (logEndRef.current) { - logEndRef.current.scrollIntoView({ behavior: "smooth" }); + logEndRef.current.scrollIntoView({ behavior: 'smooth' }); } }, [logs]); - const hasRadarDirs = Array.isArray(config.radar_dirs) && config.radar_dirs.length > 0; + const hasRadarDirs = config.radar_dirs.length > 0; const hasOrbitDir = typeof config.orbit_dir === 'string' && config.orbit_dir.trim() !== ''; - const hasDinsarDirs = Array.isArray(config.dinsar_dirs) && config.dinsar_dirs.length > 0; - const hasGf3SourceDirs = Array.isArray(config.gf3_source_dirs) && config.gf3_source_dirs.length > 0; - const hasGf3StorageDirs = Array.isArray(config.gf3_storage_dirs) && config.gf3_storage_dirs.length > 0; + const hasDinsarDirs = config.dinsar_dirs.length > 0; + const hasGf3SourceDirs = config.gf3_source_dirs.length > 0; + const hasGf3StorageDirs = config.gf3_storage_dirs.length > 0; + const canRunRadar = !readOnly && configLoaded && hasRadarDirs; const canRunOrbit = !readOnly && configLoaded && hasOrbitDir; const canRunDinsar = !readOnly && configLoaded && hasDinsarDirs; const canRunGf3Scan = !readOnly && configLoaded && hasGf3StorageDirs; const canRunGf3Process = !readOnly && configLoaded && hasGf3SourceDirs; + const canOpenUnpackDialog = !readOnly && unpackConfig.source_dirs.length > 0; const handleRunNow = async (target) => { if (readOnly) { - setMessage('当前账号为只读模式，无法触发扫描。'); + setMessage('当前账户为只读模式，无法触发扫描。'); return; } + setLoading(true); const targetMap = { - 'radar': 'LT-1 数据', - 'orbit': '精轨数据', - 'dinsar': 'D-InSAR 结果', - 'gf3': 'GF3 数据' + radar: 'LT-1 数据', + orbit: '精轨数据', + dinsar: 'D-InSAR 结果', + gf3: 'GF3 数据', }; setMessage(`正在触发${targetMap[target] || '全部'}手动扫描...`); @@ -193,12 +297,14 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled const url = target ? `${apiEndpoint}/monitor/run-now?target=${target}` : `${apiEndpoint}/monitor/run-now`; const res = await fetch(url, { method: 'POST', - credentials: 'include' + credentials: 'include', }); - const data = await res.json(); + const data = await parseJsonSafe(res, {}); if (res.ok) { - setMessage(data.message); - if (onTaskStart) onTaskStart(data.task_id, `已触发${targetMap[target] || '全部'}手动扫描...`); + setMessage(data.message || '扫描任务已启动。'); + if (onTaskStart) { + onTaskStart(data.task_id, `已触发${targetMap[target] || '全部'}手动扫描...`); + } } else { setMessage(`触发失败: ${data.detail || '未知错误'}`); } @@ -209,27 +315,83 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled } }; - const handleUnpackRun = async () => { + const handleOpenUnpackDialog = () => { if (readOnly) { - setUnpackMessage('当前账号为只读模式，无法触发解包任务。'); + setUnpackMessage('当前账户为只读模式，无法触发解包任务。'); return; } + setUnpackDialogError(''); + setUnpackRunOptions(createUnpackRunOptions(unpackConfig)); + setShowUnpackDialog(true); + }; + + const handleCloseUnpackDialog = () => { + if (unpackLoading) { + return; + } + setShowUnpackDialog(false); + setUnpackDialogError(''); + }; + + const handleUnpackOptionChange = (field, value) => { + setUnpackRunOptions((prev) => ({ + ...prev, + [field]: value, + })); + }; + + const handleUnpackRun = async () => { + if (readOnly) { + setUnpackMessage('当前账户为只读模式，无法触发解包任务。'); + return; + } + + let payload; + try { + payload = { + max_files_per_run: parseUnpackRunValue(unpackRunOptions.max_files_per_run, '单次解包数量'), + max_runtime_minutes: parseUnpackRunValue(unpackRunOptions.max_runtime_minutes, '最长运行时间'), + }; + } catch (err) { + const errorText = err instanceof Error ? err.message : '解包参数校验失败'; + setUnpackDialogError(errorText); + setUnpackMessage(`失败：${errorText}`); + return; + } + + setUnpackDialogError(''); setUnpackLoading(true); setUnpackMessage('LT-1 解包任务启动中...'); try { const res = await fetch(`${apiEndpoint}/unpack/run`, { method: 'POST', - credentials: 'include' + credentials: 'include', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(payload), }); - const data = await res.json(); + const data = await parseJsonSafe(res, {}); if (res.ok) { + setShowUnpackDialog(false); + setUnpackTaskId(data.task_id || ''); + setUnpackTaskTerminal(false); setUnpackMessage(data.message || 'LT-1 解包任务已启动'); - if (onTaskStart) onTaskStart(data.task_id, 'LT-1 解包任务已启动。'); + if (onTaskStart) { + onTaskStart(data.task_id, 'LT-1 解包任务已启动。', { + nonBlocking: true, + taskType: 'UNPACK_ARCHIVES', + }); + } } else { - setUnpackMessage(`失败：${data.detail || '未知错误'}`); + const errorText = data.detail || '未知错误'; + setUnpackDialogError(errorText); + setUnpackMessage(`失败：${errorText}`); } } catch (err) { - setUnpackMessage(`失败：${err.message}`); + const errorText = err.message || '未知错误'; + setUnpackDialogError(errorText); + setUnpackMessage(`失败：${errorText}`); } finally { setUnpackLoading(false); } @@ -237,7 +399,7 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled const handleGf3BatchProcess = async () => { if (readOnly) { - setGf3Message('当前账号为只读模式，无法触发 GF3 处理。'); + setGf3Message('当前账户为只读模式，无法触发 GF3 处理。'); return; } setGf3Loading(true); @@ -245,12 +407,14 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled try { const res = await fetch(`${apiEndpoint}/monitor/gf3-process`, { method: 'POST', - credentials: 'include' + credentials: 'include', }); - const data = await res.json(); + const data = await parseJsonSafe(res, {}); if (res.ok) { setGf3Message(data.message || 'GF3 批量处理任务已启动'); - if (onTaskStart) onTaskStart(data.task_id, 'GF3 L1A→L2 批量处理已启动。'); + if (onTaskStart) { + onTaskStart(data.task_id, 'GF3 L1A→L2 批量处理已启动。'); + } } else { setGf3Message(`失败：${data.detail || '未知错误'}`); } @@ -261,7 +425,14 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled } }; - const sectionStyle = { marginBottom: '12px', padding: '10px 12px', borderRadius: '10px', background: 'var(--color-panel-bg)', border: '1px solid var(--color-border)', boxShadow: 'var(--shadow-soft)' }; + const sectionStyle = { + marginBottom: '12px', + padding: '10px 12px', + borderRadius: '10px', + background: 'var(--color-panel-bg)', + border: '1px solid var(--color-border)', + boxShadow: 'var(--shadow-soft)', + }; const labelStyle = { minWidth: '100px', color: 'var(--color-text-muted)', flexShrink: 0 }; const rowStyle = { display: 'flex', gap: '8px' }; const gridStyle = { display: 'grid', rowGap: '6px', fontSize: '0.9em', color: 'var(--color-text-secondary)' }; @@ -273,33 +444,52 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled color: 'white', border: 'none', borderRadius: '4px', - cursor: (loading || !canRun) ? 'not-allowed' : 'pointer', - fontSize: '0.85em' + cursor: loading || !canRun ? 'not-allowed' : 'pointer', + fontSize: '0.85em', }); - const actionBtnStyle = (isLoading, isReadOnly) => ({ + const actionBtnStyle = (isLoading, isDisabled) => ({ padding: '6px 10px', backgroundColor: 'var(--color-accent)', color: 'white', border: 'none', borderRadius: '4px', - cursor: (isLoading || isReadOnly) ? 'not-allowed' : 'pointer', - fontSize: '0.85em' + cursor: isLoading || isDisabled ? 'not-allowed' : 'pointer', + fontSize: '0.85em', }); return ( - <div className="monitor-panel" style={{ padding: '15px', backgroundColor: 'var(--color-panel-bg)', borderTop: '1px solid var(--color-border)', display: 'flex', flexDirection: 'column', height: '100%', overflow: 'hidden' }}> + <div + className="monitor-panel" + style={{ + padding: '15px', + backgroundColor: 'var(--color-panel-bg)', + borderTop: '1px solid var(--color-border)', + display: 'flex', + flexDirection: 'column', + height: '100%', + overflow: 'hidden', + }} + > <h3 style={{ marginTop: 0, marginBottom: '8px', fontSize: '1.1em', flexShrink: 0 }}>数据监控面板</h3> - {/* 可滚动内容区 */} <div style={{ flex: 1, overflowY: 'auto', minHeight: 0, paddingRight: '4px' }}> - <div style={{ margin: '0 0 12px', padding: '10px 12px', borderRadius: '8px', background: 'linear-gradient(90deg, var(--color-accent-soft) 0%, #fff 70%)', border: '1px solid #c7ddff', color: 'var(--color-accent-strong)', fontSize: '0.9em' }}> + <div + style={{ + margin: '0 0 12px', + padding: '10px 12px', + borderRadius: '8px', + background: 'linear-gradient(90deg, var(--color-accent-soft) 0%, #fff 70%)', + border: '1px solid #c7ddff', + color: 'var(--color-accent-strong)', + fontSize: '0.9em', + }} + > {configLoaded ? '仅手动模式。路径从 .env 读取；如需修改请更新 .env 并重启后端。' : '未加载到监控状态，请检查后端 /api/monitor/status。'} </div> - {/* 路径摘要 — 按卫星分组 */} <div style={sectionStyle}> <div style={{ fontWeight: 'bold', marginBottom: '8px', color: 'var(--color-text-primary)' }}>路径摘要</div> <div style={gridStyle}> @@ -311,28 +501,34 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled </div> </div> - {/* LT-1 归档解包 */} <div style={sectionStyle}> <div style={{ fontWeight: 'bold', marginBottom: '8px', color: 'var(--color-text-primary)' }}>LT-1 归档解包</div> <div style={{ ...gridStyle, marginBottom: '8px' }}> <div style={rowStyle}><span style={labelStyle}>来源目录</span><span style={{ wordBreak: 'break-all' }}>{formatList(unpackConfig.source_dirs)}</span></div> <div style={rowStyle}><span style={labelStyle}>LT-1 存储</span><span style={{ wordBreak: 'break-all' }}>{formatList(unpackConfig.insar_storage_dirs)}</span></div> + <div style={rowStyle}><span style={labelStyle}>单次上限</span><span>{unpackConfig.max_files_per_run > 0 ? `${unpackConfig.max_files_per_run} 个压缩包` : '不限'}</span></div> + <div style={rowStyle}><span style={labelStyle}>最长运行</span><span>{unpackConfig.max_runtime_minutes > 0 ? `${unpackConfig.max_runtime_minutes} 分钟` : '不限'}</span></div> </div> <div style={{ display: 'flex', gap: '10px' }}> <button - onClick={handleUnpackRun} - disabled={unpackLoading || readOnly} - style={actionBtnStyle(unpackLoading, readOnly)} + onClick={handleOpenUnpackDialog} + disabled={unpackLoading || !canOpenUnpackDialog} + style={actionBtnStyle(unpackLoading, !canOpenUnpackDialog)} > {unpackLoading ? '运行中...' : (readOnly ? '只读模式' : 'LT-1 解包')} </button> - <div style={{ fontSize: '0.85em', color: unpackMessage.includes('失败') ? 'var(--color-danger)' : 'var(--color-text-muted)', alignSelf: 'center' }}> + <div + style={{ + fontSize: '0.85em', + color: unpackMessage.includes('失败') ? 'var(--color-danger)' : 'var(--color-text-muted)', + alignSelf: 'center', + }} + > {unpackMessage} </div> </div> </div> - {/* GF3 L1A→L2 处理 */} <div style={sectionStyle}> <div style={{ fontWeight: 'bold', marginBottom: '8px', color: 'var(--color-text-primary)' }}>GF3 L1A → L2 处理</div> <div style={{ ...gridStyle, marginBottom: '8px' }}> @@ -347,20 +543,25 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled > {gf3Loading ? '运行中...' : (readOnly ? '只读模式' : 'GF3 L1A→L2')} </button> - <div style={{ fontSize: '0.85em', color: gf3Message.includes('失败') ? 'var(--color-danger)' : 'var(--color-text-muted)', alignSelf: 'center' }}> + <div + style={{ + fontSize: '0.85em', + color: gf3Message.includes('失败') ? 'var(--color-danger)' : 'var(--color-text-muted)', + alignSelf: 'center', + }} + > {gf3Message} </div> </div> </div> - {/* 活动任务 */} <div style={sectionStyle}> <div style={{ fontWeight: 'bold', marginBottom: '8px', color: 'var(--color-text-primary)' }}>活动任务</div> {displayActiveTasks.length === 0 ? ( <div style={{ fontSize: '0.85em', color: 'var(--color-text-muted)' }}>当前无活动任务。</div> ) : ( <div style={{ display: 'grid', rowGap: '8px' }}> - {displayActiveTasks.slice(0, 4).map(task => ( + {displayActiveTasks.slice(0, 4).map((task) => ( <div key={task.task_id} style={{ fontSize: '0.85em', color: 'var(--color-text-secondary)' }}> <div style={{ display: 'flex', justifyContent: 'space-between', marginBottom: '4px' }}> <span>{task.task_type}</span> @@ -376,19 +577,20 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled )} </div> - {/* 实时日志 */} <div style={{ marginBottom: '4px' }}> <h4 style={{ margin: '0 0 5px 0', fontSize: '1em' }}>实时日志</h4> - <div style={{ - height: '160px', - overflowY: 'auto', - backgroundColor: '#0f172a', - color: '#22c55e', - padding: '10px', - fontFamily: 'monospace', - fontSize: '0.85em', - borderRadius: '4px' - }}> + <div + style={{ + height: '160px', + overflowY: 'auto', + backgroundColor: '#0f172a', + color: '#22c55e', + padding: '10px', + fontFamily: 'monospace', + fontSize: '0.85em', + borderRadius: '4px', + }} + > {displayLogs.length === 0 ? ( <div style={{ color: 'var(--color-text-muted)' }}>暂无日志...</div> ) : ( @@ -401,7 +603,6 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled </div> </div> - {/* 扫描按钮 — 固定在底部 */} <div style={{ flexShrink: 0, borderTop: '1px solid var(--color-border)', paddingTop: '10px', marginTop: '6px' }}> <div style={{ display: 'flex', gap: '8px', marginBottom: '6px' }}> <button onClick={() => handleRunNow('radar')} disabled={loading || !canRunRadar} style={scanBtnStyle(canRunRadar)}>扫描 LT-1</button> @@ -411,6 +612,86 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled </div> {message && <div style={{ color: message.includes('失败') ? 'red' : 'green', fontSize: '0.9em' }}>{message}</div>} </div> + + {showUnpackDialog && ( + <div className="modal-overlay visible" onClick={handleCloseUnpackDialog}> + <div className="modal-content" onClick={(event) => event.stopPropagation()}> + <h3 style={{ marginTop: 0 }}>LT-1 解包任务参数</h3> + <form + onSubmit={(event) => { + event.preventDefault(); + handleUnpackRun(); + }} + > + <div + style={{ + marginBottom: '14px', + padding: '10px 12px', + borderRadius: '8px', + background: 'var(--color-panel-muted)', + border: '1px solid var(--color-border)', + fontSize: '0.9em', + color: 'var(--color-text-secondary)', + lineHeight: 1.7, + }} + > + 本次填写的参数只覆盖当前这一次解包任务，不会修改 `.env` 默认值。输入 `0` 表示不限。 + </div> + + <div style={{ ...gridStyle, marginBottom: '14px' }}> + <div style={rowStyle}><span style={labelStyle}>来源目录</span><span style={{ wordBreak: 'break-all' }}>{formatList(unpackConfig.source_dirs)}</span></div> + <div style={rowStyle}><span style={labelStyle}>LT-1 存储</span><span style={{ wordBreak: 'break-all' }}>{formatList(unpackConfig.insar_storage_dirs)}</span></div> + </div> + + <div className="form-group"> + <label>单次最多解包数量</label> + <input + type="number" + min="0" + step="1" + value={unpackRunOptions.max_files_per_run} + onChange={(event) => handleUnpackOptionChange('max_files_per_run', event.target.value)} + disabled={unpackLoading} + /> + </div> + + <div className="form-group"> + <label>最长运行时间（分钟）</label> + <input + type="number" + min="0" + step="1" + value={unpackRunOptions.max_runtime_minutes} + onChange={(event) => handleUnpackOptionChange('max_runtime_minutes', event.target.value)} + disabled={unpackLoading} + /> + </div> + + {unpackDialogError && ( + <div + style={{ + marginTop: '6px', + color: 'var(--color-danger)', + fontSize: '0.9em', + wordBreak: 'break-all', + }} + > + {unpackDialogError} + </div> + )} + + <div className="modal-actions"> + <button type="button" onClick={handleCloseUnpackDialog} disabled={unpackLoading}> + 取消 + </button> + <button type="submit" disabled={unpackLoading}> + {unpackLoading ? '启动中...' : '启动解包任务'} + </button> + </div> + </form> + </div> + </div> + )} </div> ); }; diff --git a/frontend/src/DinsarProductionPanel.jsx b/frontend/src/DinsarProductionPanel.jsx index 2fbf1db..4149581 100644 --- a/frontend/src/DinsarProductionPanel.jsx +++ b/frontend/src/DinsarProductionPanel.jsx @@ -1,6 +1,6 @@ import React, { useCallback, useEffect, useState } from 'react'; -import { listEngines, listRuns, submitRun } from './api/dinsarProduction'; +import { listEngines, listRuns, previewPyintInputAssets, submitRun } from './api/dinsarProduction'; import { getJobLog } from './api/idl'; import { clearTaskLogs, deleteTaskLog, getActiveTasks, getTaskLogs } from './api/tasks'; @@ -34,11 +34,13 @@ const ENGINE_STATUS_LABEL = { const ENGINE_LABEL = { sarscape: 'SARscape', isce2: 'ISCE2', + pyint: 'PyINT / Gamma', landsar: 'LANDSAR', }; const TASK_TYPE_LABEL = { ISCE2_RUN: 'ISCE2生产', + PYINT_RUN: 'PyINT生产', IDL_RUN_DINSAR: 'ENVI生产', }; @@ -55,6 +57,23 @@ const STATUS_LABEL = { pending: '等待中', }; +const PYINT_DEM_MODE_LABEL = { + local_fabdem: '本地 FABDEM', + opentopo: 'OpenTopography', + prepared_file: '现有 DEM', +}; + +const PYINT_ORBIT_POLICY_LABEL = { + validate_only: '仅校验', + require_txt: '必须存在', + stage_txt: '校验并留痕', +}; + +const PYINT_PRECISE_ORBIT_MODE_LABEL = { + replace: '状态向量替换', + replace_and_validate: '替换并校验', +}; + function formatEngineLabel(engineCode, engineLabel = '') { return engineLabel || ENGINE_LABEL[engineCode] || engineCode || '-'; } @@ -67,6 +86,61 @@ function formatStatus(status) { return STATUS_LABEL[status] || status || '-'; } +function formatPyintDemMode(mode) { + return PYINT_DEM_MODE_LABEL[mode] || mode || '-'; +} + +function formatPyintOrbitPolicy(policy) { + return PYINT_ORBIT_POLICY_LABEL[policy] || policy || '-'; +} + +function formatPyintPreciseOrbitMode(mode) { + return PYINT_PRECISE_ORBIT_MODE_LABEL[mode] || mode || '-'; +} + +function PreviewIssueList({ title, items, tone = 'warning' }) { + if (!Array.isArray(items) || items.length === 0) { + return null; + } + + const palette = tone === 'error' + ? { + background: '#fef2f2', + border: '#fecaca', + title: '#b91c1c', + text: '#7f1d1d', + } + : { + background: '#fff7ed', + border: '#fed7aa', + title: '#c2410c', + text: '#9a3412', + }; + + return ( + <div + style={{ + padding: '8px 10px', + borderRadius: 6, + border: `1px solid ${palette.border}`, + background: palette.background, + }} + > + <div style={{ fontSize: 12, fontWeight: 600, color: palette.title, marginBottom: 6 }}>{title}</div> + <div style={{ display: 'flex', flexDirection: 'column', gap: 4 }}> + {items.slice(0, 6).map((item, index) => ( + <div key={`${title}-${index}`} style={{ fontSize: 11, lineHeight: 1.45, color: palette.text }}> + {item} + </div> + ))} + {items.length > 6 && ( + <div style={{ fontSize: 11, color: palette.text }}>其余 {items.length - 6} 项已折叠。</div> + )} + </div> + </div> + ); +} + function EngineStatusCard({ engine, onSelect, selected }) { const color = ENGINE_STATUS_COLOR[engine.status] || '#94a3b8'; const label = ENGINE_STATUS_LABEL[engine.status] || engine.status; @@ -238,6 +312,9 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) const [submitting, setSubmitting] = useState(false); const [submitMsg, setSubmitMsg] = useState(''); const [submitError, setSubmitError] = useState(false); + const [pyintPreview, setPyintPreview] = useState(null); + const [pyintPreviewLoading, setPyintPreviewLoading] = useState(false); + const [pyintPreviewFeedback, setPyintPreviewFeedback] = useState({ message: '', error: false }); const [runs, setRuns] = useState([]); const [runsLoading, setRunsLoading] = useState(false); @@ -253,12 +330,22 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) const currentProfiles = currentEngineObj?.profiles || EMPTY_ARRAY; const currentProfileObj = currentProfiles.find(profile => profile.code === selectedProfile) || null; const currentParamSchema = currentProfileObj?.params_schema || EMPTY_OBJECT; + const currentDefaultTimeoutSec = Number(currentEngineObj?.default_timeout_seconds || 0) || 0; + const currentParamHelpText = selectedEngine === 'pyint' + ? '这些参数影响 PyINT 的多视、并行度以及是否执行解缠/地理编码。建议先直接使用默认值，优先确认当前任务目录里的 LT-1 原始压缩包是否能被正常识别。' + : '这些参数主要影响目标网格大小、精裁剪范围、地理编码范围和位移结果掩膜。建议先使用默认值，通常优先只调整目标网格大小；只有在边缘被裁切、时间窗异常或噪声较多时，再继续调整其他参数。'; + const pyintPreviewBlocksSubmit = selectedEngine === 'pyint' && pyintPreview && pyintPreview.allow_submit === false; const latestRunWithTask = runs.find(run => run?.task_id) || null; const monitoredTask = activeTask || ( latestRunWithTask ? { task_id: latestRunWithTask.task_id, - task_type: latestRunWithTask.engine === 'isce2' ? 'ISCE2_RUN' : 'IDL_RUN_DINSAR', + task_type: + latestRunWithTask.engine === 'isce2' + ? 'ISCE2_RUN' + : latestRunWithTask.engine === 'pyint' + ? 'PYINT_RUN' + : 'IDL_RUN_DINSAR', status: latestRunWithTask.raw_status || latestRunWithTask.status, progress: latestRunWithTask.raw_status === 'COMPLETED' || latestRunWithTask.status === 'success' ? 100 : null, message: latestRunWithTask.message || '最近一次任务', @@ -299,7 +386,7 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) try { const data = await getActiveTasks(); const tasks = Array.isArray(data) ? data : (data?.tasks || []); - const relevantTask = tasks.find(task => ['ISCE2_RUN', 'IDL_RUN_DINSAR'].includes(task.task_type)) || null; + const relevantTask = tasks.find(task => ['ISCE2_RUN', 'PYINT_RUN', 'IDL_RUN_DINSAR'].includes(task.task_type)) || null; setActiveTask(relevantTask); return relevantTask; } catch { @@ -384,6 +471,19 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) setEngineExtraParams(buildDefaults(currentParamSchema)); }, [selectedEngine, selectedProfile, currentParamSchema]); + useEffect(() => { + if (currentDefaultTimeoutSec > 0) { + setTimeoutSec(String(currentDefaultTimeoutSec)); + return; + } + setTimeoutSec(''); + }, [selectedEngine, currentDefaultTimeoutSec]); + + useEffect(() => { + setPyintPreview(null); + setPyintPreviewFeedback({ message: '', error: false }); + }, [selectedEngine, rootDir, numToProcess]); + const handleParamChange = useCallback((name, value) => { setEngineExtraParams(current => ({ ...current, @@ -391,12 +491,49 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) })); }, []); + const handlePreviewPyint = useCallback(async () => { + if (!rootDir.trim()) { + setPyintPreview(null); + setPyintPreviewFeedback({ message: '请先输入根目录。', error: true }); + return; + } + + setPyintPreviewLoading(true); + setPyintPreviewFeedback({ message: '', error: false }); + try { + const data = await previewPyintInputAssets({ + root_dir: rootDir.trim(), + num_to_process: Number(numToProcess) || 0, + }); + setPyintPreview(data); + const taskCount = Number(data?.selected_task_count || data?.task_count || 0); + const orbitMissingCount = Number(data?.orbits?.missing_task_count || 0); + const detail = data?.allow_submit + ? `预检完成，可提交 ${taskCount} 个任务。` + : `预检完成，存在阻塞项；涉及 ${orbitMissingCount} 个轨道未齐全任务。`; + setPyintPreviewFeedback({ message: detail, error: !data?.allow_submit }); + } catch (err) { + setPyintPreview(null); + setPyintPreviewFeedback({ + message: `预检失败：${err?.response?.data?.detail || err.message}`, + error: true, + }); + } finally { + setPyintPreviewLoading(false); + } + }, [numToProcess, rootDir]); + const handleSubmit = async () => { if (!rootDir.trim()) { setSubmitError(true); setSubmitMsg('请输入根目录。'); return; } + if (pyintPreviewBlocksSubmit) { + setSubmitError(true); + setSubmitMsg('PyINT 输入资产预检未通过，请先修复阻塞项。'); + return; + } setSubmitting(true); setSubmitMsg(''); @@ -434,10 +571,10 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) } }; - const isSubmitDisabled = readOnly || submitting || !currentEngineObj?.available; + const isSubmitDisabled = readOnly || submitting || !currentEngineObj?.available || pyintPreviewBlocksSubmit; return ( - <div style={{ padding: '16px', maxWidth: 960 }}> + <div style={{ padding: '16px 0', width: '100%' }}> {logModal.open && ( <div style={{ @@ -585,17 +722,27 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) </div> <div style={{ minWidth: 140 }}> <label style={{ fontSize: 12, color: '#64748b', display: 'block', marginBottom: 4 }}> - 超时时间（秒，可选） + 超时时间（秒） </label> <input type="number" min={60} value={timeoutSec} onChange={event => setTimeoutSec(event.target.value)} - placeholder="默认" + placeholder={currentDefaultTimeoutSec > 0 ? `默认 ${currentDefaultTimeoutSec}` : '默认'} disabled={readOnly} style={{ width: 140, padding: '5px 8px', borderRadius: 4, border: '1px solid #e2e8f0', fontSize: 13 }} /> + {selectedEngine === 'isce2' && currentDefaultTimeoutSec > 0 && ( + <div style={{ fontSize: 11, color: '#94a3b8', marginTop: 4 }}> + ISCE2 默认按单对任务使用 {currentDefaultTimeoutSec} 秒；批量目录会串行逐对套用该超时。 + </div> + )} + {selectedEngine === 'pyint' && currentDefaultTimeoutSec > 0 && ( + <div style={{ fontSize: 11, color: '#94a3b8', marginTop: 4 }}> + PyINT 默认按单对任务使用 {currentDefaultTimeoutSec} 秒；当前会逐对串行创建工作区并运行外部 PyINT / Gamma 流程。 + </div> + )} </div> </div> @@ -622,7 +769,7 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) borderRadius: 6, }} > - 这些参数主要影响目标网格大小、精裁剪范围、地理编码范围和位移结果掩膜。建议先使用默认值，通常优先只调整目标网格大小；只有在边缘被裁切、时间窗异常或噪声较多时，再继续调整其他参数。 + {currentParamHelpText} </div> <div style={{ display: 'flex', gap: 12, flexWrap: 'wrap' }}> {Object.entries(currentParamSchema).map(([name, schema]) => ( @@ -639,6 +786,185 @@ export default function DinsarProductionPanel({ readOnly = false, onJobQueued }) </div> )} + {selectedEngine === 'pyint' && ( + <div + style={{ + marginBottom: 10, + padding: '10px 12px', + background: '#f8fafc', + borderRadius: 6, + border: '1px solid #e2e8f0', + }} + > + <div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', gap: 10, flexWrap: 'wrap', marginBottom: 8 }}> + <div> + <div style={{ fontSize: 12, color: '#0f172a', fontWeight: 600 }}>PyINT 输入资产预检</div> + <div style={{ fontSize: 11, color: '#64748b', marginTop: 4 }}> + 提交前检查 Task_* 结构、DEM 策略和 LT-1 轨道是否齐备。即使不手动预检，后端提交时也会做同样校验。 + </div> + </div> + <button + onClick={handlePreviewPyint} + disabled={readOnly || pyintPreviewLoading || !rootDir.trim()} + style={{ + fontSize: 12, + padding: '5px 12px', + borderRadius: 6, + border: '1px solid #cbd5e1', + cursor: readOnly || pyintPreviewLoading || !rootDir.trim() ? 'not-allowed' : 'pointer', + background: '#fff', + color: '#0f172a', + }} + > + {pyintPreviewLoading ? '预检中...' : '预检输入资产'} + </button> + </div> + + {pyintPreviewFeedback.message && ( + <div + style={{ + marginBottom: pyintPreview ? 10 : 0, + padding: '8px 10px', + borderRadius: 6, + border: `1px solid ${pyintPreviewFeedback.error ? '#fecaca' : '#bbf7d0'}`, + background: pyintPreviewFeedback.error ? '#fef2f2' : '#f0fdf4', + color: pyintPreviewFeedback.error ? '#b91c1c' : '#15803d', + fontSize: 12, + lineHeight: 1.5, + }} + > + {pyintPreviewFeedback.message} + </div> + )} + + {!pyintPreview && !pyintPreviewLoading && ( + <div style={{ fontSize: 11, color: '#94a3b8' }}> + 尚未执行预检。建议在首次处理新批次前先预检一次。 + </div> + )} + + {pyintPreview && ( + <> + <div + style={{ + display: 'grid', + gridTemplateColumns: 'repeat(auto-fit, minmax(140px, 1fr))', + gap: 8, + marginBottom: 10, + }} + > + {[ + { label: '任务数', value: pyintPreview.selected_task_count ?? pyintPreview.task_count ?? 0, color: '#0f172a' }, + { label: 'DEM 策略', value: formatPyintDemMode(pyintPreview?.dem?.mode), color: '#1d4ed8' }, + { label: '轨道策略', value: formatPyintOrbitPolicy(pyintPreview?.orbits?.policy), color: '#7c3aed' }, + { + label: '精轨桥接', + value: pyintPreview?.precise_orbit_bridge?.enabled + ? formatPyintPreciseOrbitMode(pyintPreview?.precise_orbit_bridge?.mode) + : '关闭', + color: pyintPreview?.precise_orbit_bridge?.enabled ? '#0f766e' : '#64748b', + }, + { label: '可提交', value: pyintPreview.allow_submit ? '是' : '否', color: pyintPreview.allow_submit ? '#15803d' : '#b91c1c' }, + { label: '缺轨道任务', value: pyintPreview?.orbits?.missing_task_count ?? 0, color: (pyintPreview?.orbits?.missing_task_count || 0) > 0 ? '#b91c1c' : '#475569' }, + { label: '无效目录', value: pyintPreview?.invalid_candidates?.length ?? 0, color: (pyintPreview?.invalid_candidates?.length || 0) > 0 ? '#c2410c' : '#475569' }, + ].map(item => ( + <div + key={item.label} + style={{ + padding: '8px 10px', + borderRadius: 6, + border: '1px solid #e2e8f0', + background: '#fff', + }} + > + <div style={{ fontSize: 11, color: '#64748b', marginBottom: 4 }}>{item.label}</div> + <div style={{ fontSize: 13, fontWeight: 600, color: item.color, wordBreak: 'break-word' }}>{item.value}</div> + </div> + ))} + </div> + + <div + style={{ + display: 'grid', + gridTemplateColumns: 'repeat(auto-fit, minmax(260px, 1fr))', + gap: 8, + marginBottom: 10, + }} + > + <PreviewIssueList title="阻塞项" items={pyintPreview.blockers || []} tone="error" /> + <PreviewIssueList title="警告" items={pyintPreview.warnings || []} tone="warning" /> + </div> + + <div style={{ fontSize: 12, color: '#475569', marginBottom: 6 }}>任务级预检结果</div> + <div + style={{ + border: '1px solid #e2e8f0', + borderRadius: 6, + overflowX: 'auto', + overflowY: 'hidden', + background: '#fff', + }} + > + <div + style={{ + display: 'grid', + gridTemplateColumns: 'minmax(180px, 1.6fr) minmax(70px, 0.8fr) minmax(220px, 1.5fr) minmax(70px, 0.7fr)', + minWidth: 560, + gap: 0, + background: '#f8fafc', + borderBottom: '1px solid #e2e8f0', + }} + > + {['任务', '源文件', '轨道解析', '提交'].map(header => ( + <div key={header} style={{ padding: '8px 10px', fontSize: 11, color: '#64748b', fontWeight: 600 }}> + {header} + </div> + ))} + </div> + {(pyintPreview.tasks || []).map(task => { + const masterOrbit = task?.orbit_resolution?.master; + const slaveOrbit = task?.orbit_resolution?.slave; + return ( + <div + key={task.task_dir} + style={{ + display: 'grid', + gridTemplateColumns: 'minmax(180px, 1.6fr) minmax(70px, 0.8fr) minmax(220px, 1.5fr) minmax(70px, 0.7fr)', + minWidth: 560, + borderBottom: '1px solid #f1f5f9', + }} + > + <div style={{ padding: '8px 10px', minWidth: 0 }}> + <div style={{ fontSize: 12, color: '#0f172a', fontWeight: 600 }}>{task.task_alias || task.task_name}</div> + <div style={{ fontSize: 11, color: '#64748b', marginTop: 4 }}> + {task.master_date || '-'} / {task.slave_date || '-'} + </div> + </div> + <div style={{ padding: '8px 10px', fontSize: 11, color: '#334155' }}> + M {task?.archive_counts?.master ?? 0} + <br /> + S {task?.archive_counts?.slave ?? 0} + </div> + <div style={{ padding: '8px 10px', fontSize: 11, lineHeight: 1.5 }}> + <div style={{ color: masterOrbit?.resolved ? '#15803d' : '#b91c1c' }}> + M {masterOrbit?.resolved ? `${masterOrbit.satellite}/${masterOrbit.date}` : '缺失'} + </div> + <div style={{ color: slaveOrbit?.resolved ? '#15803d' : '#b91c1c' }}> + S {slaveOrbit?.resolved ? `${slaveOrbit.satellite}/${slaveOrbit.date}` : '缺失'} + </div> + </div> + <div style={{ padding: '8px 10px', fontSize: 11, fontWeight: 600, color: task.allow_submit ? '#15803d' : '#b91c1c' }}> + {task.allow_submit ? '可提交' : '阻塞'} + </div> + </div> + ); + })} + </div> + </> + )} + </div> + )} + <div style={{ display: 'flex', alignItems: 'center', gap: 12 }}> <button onClick={handleSubmit} diff --git a/frontend/src/DinsarProductsPanel.rewrite.jsx b/frontend/src/DinsarProductsPanel.rewrite.jsx new file mode 100644 index 0000000..87d95a1 --- /dev/null +++ b/frontend/src/DinsarProductsPanel.rewrite.jsx @@ -0,0 +1,405 @@ +import React, { useCallback, useEffect, useMemo, useState } from 'react'; + +import { scanDinsarResults } from './api/dinsar'; +import { extractDispResults } from './api/idl'; +import { clearTaskLogs, deleteTaskLog, getActiveTasks, getRecentTasks, getTaskLogs } from './api/tasks'; +import DinsarCatalogPanel from './components/DinsarCatalogPanel.rewrite'; + +const PRODUCT_TASK_TYPES = [ + 'SCAN_DINSAR', + 'PUBLISH_DINSAR_PRODUCTS', + 'REBUILD_DINSAR_CATALOG', +]; + +const TASK_TYPE_LABEL = { + SCAN_DINSAR: 'D-InSAR 结果扫描', + PUBLISH_DINSAR_PRODUCTS: 'D-InSAR 产物发布', + REBUILD_DINSAR_CATALOG: 'D-InSAR 目录重建', +}; + +const STATUS_LABEL = { + PENDING: '等待中', + RUNNING: '运行中', + COMPLETED: '已完成', + FAILED: '失败', + CANCELLED: '已取消', + CANCELED: '已取消', +}; + +function formatTaskType(taskType) { + return TASK_TYPE_LABEL[taskType] || taskType || '-'; +} + +function formatStatus(status) { + return STATUS_LABEL[status] || status || '-'; +} + +function getMessageTone(message, fallbackError = false) { + if (fallbackError) return 'error'; + return /失败|error|Error|ERROR/.test(String(message || '')) ? 'error' : 'success'; +} + +function getLogTone(level) { + const normalized = String(level || '').toUpperCase(); + if (normalized === 'ERROR') return 'error'; + if (normalized === 'WARNING' || normalized === 'WARN') return 'warn'; + return 'info'; +} + +export default function DinsarProductsPanel({ readOnly = false, onJobQueued }) { + const [extractRootDir, setExtractRootDir] = useState(''); + const [extractDestDir, setExtractDestDir] = useState(''); + const [extractResult, setExtractResult] = useState(null); + const [extracting, setExtracting] = useState(false); + const [actionMessage, setActionMessage] = useState(''); + const [actionError, setActionError] = useState(false); + const [scanning, setScanning] = useState(false); + + const [activeTask, setActiveTask] = useState(null); + const [recentTask, setRecentTask] = useState(null); + const [taskLogs, setTaskLogs] = useState([]); + const [taskLogsLoading, setTaskLogsLoading] = useState(false); + const [taskLogActionLoading, setTaskLogActionLoading] = useState(false); + const [taskLogDeletingId, setTaskLogDeletingId] = useState(null); + const monitoredTask = activeTask || recentTask; + const logTaskId = monitoredTask?.task_id || ''; + const showingRecentTask = !activeTask && !!recentTask; + const actionTone = getMessageTone(actionMessage, actionError); + + const loadActiveTask = useCallback(async () => { + try { + const data = await getActiveTasks(); + const tasks = Array.isArray(data) ? data : (data?.tasks || []); + const relevantTask = tasks.find((task) => PRODUCT_TASK_TYPES.includes(task.task_type)) || null; + setActiveTask(relevantTask); + return relevantTask; + } catch { + setActiveTask(null); + return null; + } + }, []); + + const loadRecentTask = useCallback(async () => { + try { + const tasks = await getRecentTasks(PRODUCT_TASK_TYPES, [], 1, 0); + const nextTask = Array.isArray(tasks) ? (tasks[0] || null) : null; + setRecentTask(nextTask); + return nextTask; + } catch { + setRecentTask(null); + return null; + } + }, []); + + const loadTaskLogs = useCallback(async (taskId) => { + if (!taskId) { + setTaskLogs([]); + return; + } + setTaskLogsLoading(true); + try { + const data = await getTaskLogs(taskId, 50, 0); + setTaskLogs(data?.logs || []); + } catch { + setTaskLogs([]); + } finally { + setTaskLogsLoading(false); + } + }, []); + + const refreshMonitor = useCallback(async () => { + const [nextActiveTask, nextRecentTask] = await Promise.all([ + loadActiveTask(), + loadRecentTask(), + ]); + const nextTaskId = nextActiveTask?.task_id || nextRecentTask?.task_id || ''; + await loadTaskLogs(nextTaskId); + }, [loadActiveTask, loadRecentTask, loadTaskLogs]); + + useEffect(() => { + refreshMonitor(); + }, [refreshMonitor]); + + const handleDeleteTaskLog = useCallback(async (logId) => { + const taskId = logTaskId; + if (!taskId || !logId || taskLogActionLoading) return; + if (!window.confirm('确定要删除这条任务日志吗？')) return; + + setTaskLogDeletingId(logId); + setTaskLogActionLoading(true); + try { + await deleteTaskLog(taskId, logId); + await loadTaskLogs(taskId); + } catch (error) { + setActionMessage(`删除日志失败：${error?.response?.data?.detail || error.message}`); + setActionError(true); + } finally { + setTaskLogDeletingId(null); + setTaskLogActionLoading(false); + } + }, [logTaskId, loadTaskLogs, taskLogActionLoading]); + + const handleClearTaskLogs = useCallback(async () => { + const taskId = logTaskId; + if (!taskId || taskLogActionLoading || taskLogs.length === 0) return; + if (!window.confirm(`确定要清空任务 ${taskId} 的全部日志吗？`)) return; + + setTaskLogActionLoading(true); + try { + await clearTaskLogs(taskId); + await loadTaskLogs(taskId); + } catch (error) { + setActionMessage(`清空日志失败：${error?.response?.data?.detail || error.message}`); + setActionError(true); + } finally { + setTaskLogActionLoading(false); + } + }, [logTaskId, loadTaskLogs, taskLogActionLoading, taskLogs.length]); + + const handleExtract = async () => { + if (!extractRootDir.trim()) return; + setExtracting(true); + setExtractResult(null); + setActionMessage(''); + setActionError(false); + try { + const result = await extractDispResults(extractRootDir.trim(), extractDestDir.trim() || null); + setExtractResult(result); + } catch (err) { + setExtractResult({ error: err?.response?.data?.detail || err.message }); + } finally { + setExtracting(false); + } + }; + + const handleScan = async () => { + if (readOnly) return; + setScanning(true); + setActionMessage(''); + setActionError(false); + try { + const result = await scanDinsarResults(); + setActionMessage(result?.message || `D-InSAR 结果扫描任务已入队：${result?.task_id || '-'}`); + if (result?.task_id) { + onJobQueued?.(result.task_id); + } + await refreshMonitor(); + } catch (err) { + setActionError(true); + setActionMessage(err?.response?.data?.detail || err.message || 'D-InSAR 结果扫描失败'); + } finally { + setScanning(false); + } + }; + + const monitorTone = useMemo(() => { + if (!monitoredTask) return 'neutral'; + if (showingRecentTask) return 'info'; + return String(monitoredTask.status || '').toUpperCase() === 'RUNNING' ? 'warn' : 'neutral'; + }, [monitoredTask, showingRecentTask]); + + return ( + <div className="dinsar-products-page"> + <div className="dinsar-products-hero"> + <div> + <strong>D-InSAR 结果提取与标准目录</strong> + <p> + 这里负责把生产目录中的位移结果提取为标准成果包，并触发统一扫描、发布和编目。 + 生产运行与参数配置现已收口到“生产管理”工作台中的 “D-InSAR 运行” 子视图。 + </p> + </div> + <div className="dinsar-products-hero-badges"> + <span className={`dinsar-status-pill tone-${readOnly ? 'warn' : 'ready'}`}> + {readOnly ? '只读模式' : '可执行写操作'} + </span> + <span className="dinsar-status-pill tone-info">日志改为手动刷新</span> + </div> + </div> + + <div className="dinsar-products-top-grid"> + <section className="dinsar-products-card"> + <div className="dinsar-products-card-head"> + <div> + <strong>结果提取与重扫</strong> + <span>先提取标准结果包，再按统一目录登记</span> + </div> + </div> + + <div className="dinsar-products-form-grid"> + <label className="dinsar-products-field dinsar-products-field-wide"> + <span>结果根目录</span> + <input + value={extractRootDir} + onChange={(event) => setExtractRootDir(event.target.value)} + placeholder="例如：D:\\Task_Pool\\DInSAR" + /> + </label> + <label className="dinsar-products-field"> + <span>目标目录（可选）</span> + <input + value={extractDestDir} + onChange={(event) => setExtractDestDir(event.target.value)} + placeholder="留空则使用系统默认" + /> + </label> + </div> + + <div className="dinsar-products-actions"> + <button + type="button" + className="primary" + onClick={handleExtract} + disabled={extracting || !extractRootDir.trim()} + > + {extracting ? '提取中...' : '提取位移结果'} + </button> + <button + type="button" + onClick={handleScan} + disabled={readOnly || scanning} + > + {scanning ? '重扫中...' : '重扫结果'} + </button> + </div> + + {actionMessage && ( + <div className={`dinsar-products-message tone-${actionTone}`}> + {actionMessage} + </div> + )} + + {extractResult && ( + <div className={`dinsar-products-result-card ${extractResult.error ? 'error' : 'success'}`}> + {extractResult.error ? ( + <span>提取失败：{extractResult.error}</span> + ) : ( + <> + <div>提取完成：复制 {extractResult.copied || 0} 个文件，覆盖 {extractResult.overwritten || 0} 个文件。</div> + {extractResult.catalog?.attempted && extractResult.catalog?.status === 'ok' && ( + <div> + 标准结果目录已同步：发布 {extractResult.catalog?.publish?.processed || 0} 项， + 重建登记 {extractResult.catalog?.rebuild?.registered || 0} 项。 + </div> + )} + {extractResult.catalog?.attempted && extractResult.catalog?.status === 'error' && ( + <div>标准结果目录同步失败：{extractResult.catalog?.message}</div> + )} + </> + )} + </div> + )} + </section> + + <section className={`dinsar-products-card monitor tone-${monitorTone}`}> + <div className="dinsar-products-card-head"> + <div> + <strong>产物任务监控</strong> + <span>当前不轮询，按需手动刷新</span> + </div> + <button type="button" onClick={refreshMonitor}>刷新</button> + </div> + + {!monitoredTask ? ( + <div className="dinsar-products-empty">当前没有正在执行的产物处理任务。</div> + ) : ( + <div className="dinsar-monitor-card"> + <div className="dinsar-monitor-top"> + <div> + <strong>{showingRecentTask ? '最近一次任务' : '当前任务'}</strong> + <span>{formatTaskType(monitoredTask.task_type)}</span> + </div> + <StatusSummary status={monitoredTask.status} /> + </div> + + <div className="dinsar-monitor-task-id">{monitoredTask.task_id}</div> + <div className="dinsar-monitor-message">{monitoredTask.message || '-'}</div> + + {monitoredTask.progress != null && ( + <div className="dinsar-monitor-progress"> + <div className="dinsar-monitor-progress-track"> + <div + className="dinsar-monitor-progress-bar" + style={{ width: `${monitoredTask.progress}%` }} + /> + </div> + <span>{monitoredTask.progress}%</span> + </div> + )} + + <div className="dinsar-monitor-log-head"> + <strong>{showingRecentTask ? '最近一次任务日志' : '当前任务日志'}</strong> + {!readOnly && ( + <button + type="button" + onClick={handleClearTaskLogs} + disabled={taskLogActionLoading || taskLogs.length === 0} + > + {taskLogActionLoading && taskLogDeletingId == null ? '清空中...' : '清空日志'} + </button> + )} + </div> + + {taskLogsLoading ? ( + <div className="dinsar-products-empty">日志加载中...</div> + ) : taskLogs.length === 0 ? ( + <div className="dinsar-products-empty">暂无日志。</div> + ) : ( + <div className="dinsar-monitor-log-list"> + {taskLogs.map((log, index) => { + const tone = getLogTone(log.level); + return ( + <div + key={log.id || `${log.timestamp || 'log'}-${index}`} + className={`dinsar-monitor-log-item tone-${tone}`} + > + <div className="dinsar-monitor-log-main"> + <div className="dinsar-monitor-log-time"> + {(log.timestamp || '').replace('T', ' ').replace('Z', '')} [{log.level}] + </div> + <div className="dinsar-monitor-log-message">{log.message}</div> + </div> + {!readOnly && ( + <button + type="button" + className="danger" + onClick={() => handleDeleteTaskLog(log.id)} + disabled={taskLogActionLoading || !log.id} + > + {taskLogDeletingId === log.id ? '删除中...' : '删除'} + </button> + )} + </div> + ); + })} + </div> + )} + </div> + )} + </section> + </div> + + <DinsarCatalogPanel + readOnly={readOnly} + initialSourceDir={extractRootDir} + onTaskQueued={onJobQueued} + /> + </div> + ); +} + +function StatusSummary({ status }) { + const normalized = String(status || '').toUpperCase(); + const tone = normalized === 'RUNNING' + ? 'warn' + : normalized === 'FAILED' + ? 'error' + : normalized === 'COMPLETED' + ? 'ready' + : 'neutral'; + + return ( + <span className={`dinsar-status-pill tone-${tone}`}> + {formatStatus(status)} + </span> + ); +} diff --git a/frontend/src/ProductionWorkspace.jsx b/frontend/src/ProductionWorkspace.jsx new file mode 100644 index 0000000..d2cbea1 --- /dev/null +++ b/frontend/src/ProductionWorkspace.jsx @@ -0,0 +1,208 @@ +import { Suspense, lazy, useEffect, useMemo, useState } from 'react'; + +import { + PRODUCTION_WORKSPACE_ENTRY_TO_VIEW, + PRODUCTION_WORKSPACE_TAB, + PRODUCTION_WORKSPACE_VIEWS, +} from './config/appConstants'; +import { PanelLoadingBody } from './components/app/AppLoadingFallbacks'; + +const LazyDinsarProductionPanel = lazy(() => import('./DinsarProductionPanel')); +const LazyTimeseriesProductionPanel = lazy(() => import('./TimeseriesProductionPanel')); +const LazyDinsarProductsPanel = lazy(() => import('./DinsarProductsPanel.rewrite')); +const LazyPsinsarCatalogPanel = lazy(() => import('./components/PsinsarCatalogPanel')); + +const shellStyle = { + minHeight: '100%', + padding: '20px 24px 28px', + boxSizing: 'border-box', + background: 'linear-gradient(180deg, #f5f7fb 0%, #eef4ff 52%, #f8fafc 100%)', +}; + +const heroStyle = { + display: 'grid', + gridTemplateColumns: 'repeat(auto-fit, minmax(320px, 1fr))', + gap: 16, + marginBottom: 18, +}; + +const heroCardStyle = { + borderRadius: 24, + border: '1px solid #d7e0eb', + background: 'linear-gradient(135deg, #ffffff 0%, #f8fbff 56%, #eef6ff 100%)', + boxShadow: '0 16px 40px rgba(15, 23, 42, 0.06)', +}; + +const summaryCardStyle = { + padding: '12px 14px', + borderRadius: 18, + border: '1px solid #e2e8f0', + background: 'rgba(255, 255, 255, 0.82)', +}; + +function resolveView(entry) { + return PRODUCTION_WORKSPACE_ENTRY_TO_VIEW[entry] || PRODUCTION_WORKSPACE_ENTRY_TO_VIEW[PRODUCTION_WORKSPACE_TAB]; +} + +export default function ProductionWorkspace({ + activeEntry = PRODUCTION_WORKSPACE_TAB, + readOnly = false, + onTaskStart, +}) { + const [activeView, setActiveView] = useState(() => resolveView(activeEntry)); + + useEffect(() => { + setActiveView(resolveView(activeEntry)); + }, [activeEntry]); + + const activeViewMeta = useMemo( + () => PRODUCTION_WORKSPACE_VIEWS.find(view => view.key === activeView) || PRODUCTION_WORKSPACE_VIEWS[0], + [activeView] + ); + + const handleDinsarRunQueued = taskId => { + onTaskStart?.(taskId, 'D-InSAR 任务已入队，等待处理...'); + }; + + const handleTimeseriesRunQueued = taskId => { + onTaskStart?.(taskId, '时序InSAR 运行已入队，当前默认执行 SBAS 流程...'); + }; + + const handleDinsarProductQueued = taskId => { + onTaskStart?.(taskId, 'D-InSAR 产物任务已入队，等待处理...'); + }; + + const handleTimeseriesProductQueued = taskId => { + onTaskStart?.(taskId, '时序InSAR 产物目录任务已入队，等待处理...'); + }; + + return ( + <div style={shellStyle}> + <div style={heroStyle}> + <section style={{ ...heroCardStyle, padding: '22px 24px' }}> + <div style={{ fontSize: 12, fontWeight: 700, letterSpacing: '0.08em', color: '#1d4ed8', textTransform: 'uppercase' }}> + Production Management + </div> + <h2 style={{ margin: '10px 0 12px', fontSize: 32, lineHeight: 1.1, color: '#0f172a' }}>生产管理</h2> + <p style={{ margin: 0, maxWidth: 900, fontSize: 14, lineHeight: 1.8, color: '#475569' }}> + 这里统一承载 D-InSAR 与时序InSAR的运行和产物工作台。当前时序入口默认接入 SBAS 实现， + 后续可在同一界面继续扩展 PS-InSAR、SBAS-InSAR 以及更多 WSL2 引擎实例。 + </p> + </section> + + <section + style={{ + ...heroCardStyle, + padding: '18px', + display: 'grid', + gap: 12, + alignContent: 'start', + }} + > + <div style={summaryCardStyle}> + <div style={{ fontSize: 11, color: '#64748b', marginBottom: 6 }}>统一入口</div> + <div style={{ fontSize: 15, fontWeight: 700, color: '#0f172a' }}>运行与产物同域编排</div> + <div style={{ fontSize: 12, lineHeight: 1.6, color: '#475569', marginTop: 4 }}> + 生产运行、目录重建、产物编目全部收口到同一顶级工作区。 + </div> + </div> + <div style={summaryCardStyle}> + <div style={{ fontSize: 11, color: '#64748b', marginBottom: 6 }}>时序当前实现</div> + <div style={{ fontSize: 15, fontWeight: 700, color: '#0f172a' }}>SBAS 默认接入</div> + <div style={{ fontSize: 12, lineHeight: 1.6, color: '#475569', marginTop: 4 }}> + 现阶段实际运行链路为 SBAS，界面命名已统一为时序InSAR。 + </div> + </div> + <div style={summaryCardStyle}> + <div style={{ fontSize: 11, color: '#64748b', marginBottom: 6 }}>引擎预留</div> + <div style={{ fontSize: 15, fontWeight: 700, color: '#0f172a' }}>ISCE / Gamma 可扩</div> + <div style={{ fontSize: 12, lineHeight: 1.6, color: '#475569', marginTop: 4 }}> + 现有 WSL2 嵌入保持不变，后续增加 Gamma 时可直接挂入当前工作台。 + </div> + </div> + </section> + </div> + + <section + style={{ + ...heroCardStyle, + padding: '14px', + marginBottom: 18, + }} + > + <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(240px, 1fr))', gap: 12 }}> + {PRODUCTION_WORKSPACE_VIEWS.map(view => { + const isActive = view.key === activeView; + return ( + <button + key={view.key} + type="button" + onClick={() => setActiveView(view.key)} + style={{ + textAlign: 'left', + padding: '14px 16px', + borderRadius: 18, + border: `1px solid ${isActive ? '#93c5fd' : '#d7e0eb'}`, + background: isActive + ? 'linear-gradient(135deg, #eff6ff 0%, #f8fbff 100%)' + : 'rgba(255, 255, 255, 0.88)', + boxShadow: isActive ? '0 10px 24px rgba(37, 99, 235, 0.12)' : 'none', + cursor: 'pointer', + transition: 'all 0.2s ease', + }} + > + <div style={{ display: 'flex', alignItems: 'center', justifyContent: 'space-between', gap: 8, marginBottom: 8 }}> + <strong style={{ fontSize: 15, color: '#0f172a' }}>{view.label}</strong> + <span + style={{ + padding: '4px 8px', + borderRadius: 999, + fontSize: 11, + fontWeight: 700, + color: isActive ? '#1d4ed8' : '#64748b', + background: isActive ? '#dbeafe' : '#f1f5f9', + }} + > + {isActive ? '当前视图' : '切换'} + </span> + </div> + <div style={{ fontSize: 12, lineHeight: 1.7, color: '#475569' }}>{view.description}</div> + </button> + ); + })} + </div> + </section> + + <section> + <div style={{ marginBottom: 12, fontSize: 12, color: '#64748b' }}>{activeViewMeta.label}</div> + <Suspense fallback={<PanelLoadingBody message={`正在加载 ${activeViewMeta.label}...`} />}> + {activeView === 'dinsar_runs' && ( + <LazyDinsarProductionPanel + readOnly={readOnly} + onJobQueued={handleDinsarRunQueued} + /> + )} + {activeView === 'timeseries_runs' && ( + <LazyTimeseriesProductionPanel + readOnly={readOnly} + onJobQueued={handleTimeseriesRunQueued} + /> + )} + {activeView === 'dinsar_products' && ( + <LazyDinsarProductsPanel + readOnly={readOnly} + onJobQueued={handleDinsarProductQueued} + /> + )} + {activeView === 'timeseries_products' && ( + <LazyPsinsarCatalogPanel + readOnly={readOnly} + showActions + onTaskQueued={handleTimeseriesProductQueued} + /> + )} + </Suspense> + </section> + </div> + ); +} diff --git a/frontend/src/TimeseriesProductionPanel.jsx b/frontend/src/TimeseriesProductionPanel.jsx index e0ab91c..f7dfb8e 100644 --- a/frontend/src/TimeseriesProductionPanel.jsx +++ b/frontend/src/TimeseriesProductionPanel.jsx @@ -154,7 +154,7 @@ export default function TimeseriesProductionPanel({ readOnly = false, onJobQueue const handleSubmit = async () => { if (!selectedBatchId) { - setMessage('请先选择一个 PS 批次。'); + setMessage('请先选择一个时序批次。'); return; } setSubmitting(true); @@ -182,9 +182,9 @@ export default function TimeseriesProductionPanel({ readOnly = false, onJobQueue const workflowSteps = selectedRunDetail?.workflow?.steps || []; return ( - <div style={{ padding: '16px', maxWidth: 1080 }}> + <div style={{ padding: '16px 0', width: '100%' }}> <div style={card}> - <strong style={{ fontSize: 14, display: 'block', marginBottom: 10 }}>SBAS 生产入口</strong> + <strong style={{ fontSize: 14, display: 'block', marginBottom: 10 }}>时序InSAR 运行入口</strong> <div style={{ fontSize: 12, @@ -196,18 +196,18 @@ export default function TimeseriesProductionPanel({ readOnly = false, onJobQueue borderRadius: 6, }} > - 当前系统阶段已接入完整八步链路：prepare、stack_prep_initial、materialize、stack_prep_refresh、 - run_isce2_stack、run_mintpy_sbas、export_publish_bundle、register_psinsar_product。 - 提交后系统会依次生成选栈 manifest、物化 LT-1 SLC、执行 ISCE2 stack、运行 MintPy SBAS、 - 导出 publish bundle，并把结果注册进 PS-InSAR catalog。 + 当前接入实现为 SBAS。现阶段已连通完整八步链路：prepare、stack_prep_initial、materialize、 + stack_prep_refresh、run_isce2_stack、run_mintpy_sbas、export_publish_bundle、 + register_psinsar_product。提交后系统会依次生成选栈 manifest、物化 LT-1 SLC、执行 ISCE2 + stack、运行 MintPy SBAS、导出 publish bundle，并把结果注册进时序InSAR catalog。 </div> </div> <div style={card}> <strong style={{ fontSize: 14, display: 'block', marginBottom: 10 }}>新建运行</strong> - <div style={{ display: 'grid', gridTemplateColumns: 'repeat(2, minmax(0, 1fr))', gap: 10 }}> + <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(220px, 1fr))', gap: 10 }}> <div> - <div style={{ fontSize: 12, color: '#64748b', marginBottom: 4 }}>PS 批次</div> + <div style={{ fontSize: 12, color: '#64748b', marginBottom: 4 }}>时序批次</div> <select value={selectedBatchId} onChange={event => setSelectedBatchId(event.target.value)} @@ -298,7 +298,7 @@ export default function TimeseriesProductionPanel({ readOnly = false, onJobQueue cursor: readOnly ? 'not-allowed' : 'pointer', }} > - {submitting ? '提交中...' : '提交 SBAS 运行'} + {submitting ? '提交中...' : '提交时序运行（SBAS）'} </button> {message && ( <span style={{ fontSize: 12, color: message.includes('失败') ? '#dc2626' : '#166534' }}> @@ -321,13 +321,13 @@ export default function TimeseriesProductionPanel({ readOnly = false, onJobQueue </button> </div> - <div style={{ display: 'grid', gridTemplateColumns: 'minmax(280px, 360px) 1fr', gap: 12 }}> + <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(320px, 1fr))', gap: 12 }}> <div style={{ border: '1px solid #e2e8f0', borderRadius: 6, overflow: 'hidden' }}> <div style={{ padding: '8px 10px', background: '#f8fafc', fontSize: 12, fontWeight: 600 }}> 运行列表 ({runs.length}) </div> {runs.length === 0 ? ( - <div style={{ padding: '12px', fontSize: 12, color: '#94a3b8' }}>当前暂无 SBAS 运行记录。</div> + <div style={{ padding: '12px', fontSize: 12, color: '#94a3b8' }}>当前暂无时序InSAR运行记录。</div> ) : ( <div style={{ maxHeight: 420, overflowY: 'auto' }}> {runs.map(item => ( diff --git a/frontend/src/api/dinsarProduction.js b/frontend/src/api/dinsarProduction.js index 7bf57a4..7055e7c 100644 --- a/frontend/src/api/dinsarProduction.js +++ b/frontend/src/api/dinsarProduction.js @@ -18,3 +18,6 @@ export const submitRun = (payload) => // 运行历史 export const listRuns = (limit = 20) => apiClient.get(`/dinsar-production/runs?limit=${encodeURIComponent(limit)}`).then(r => r.data); + +export const previewPyintInputAssets = (payload) => + apiClient.post('/dinsar-production/engines/pyint/preview-input-assets', payload).then(r => r.data); diff --git a/frontend/src/components/ActiveTasksOverlay.jsx b/frontend/src/components/ActiveTasksOverlay.jsx index b398cee..df77c5e 100644 --- a/frontend/src/components/ActiveTasksOverlay.jsx +++ b/frontend/src/components/ActiveTasksOverlay.jsx @@ -19,7 +19,7 @@ const getTaskTypeLabel = (taskType) => { case 'SCAN_HAZARD': return '灾害点同步'; case 'UNPACK_ARCHIVES': - return 'Archive unpack'; + return 'LT-1 解包'; case 'IDL_IMPORT': return 'ENVI 数据导入'; case 'IDL_DINSAR': diff --git a/frontend/src/components/DinsarCatalogPanel.rewrite.jsx b/frontend/src/components/DinsarCatalogPanel.rewrite.jsx new file mode 100644 index 0000000..707a83b --- /dev/null +++ b/frontend/src/components/DinsarCatalogPanel.rewrite.jsx @@ -0,0 +1,581 @@ +import React, { useCallback, useEffect, useMemo, useState } from 'react'; + +import apiClient from '../api/client'; +import { + getDinsarCatalogStatus, + getDinsarProductDetail, + listDinsarProducts, + queueDinsarCatalogRebuild, + queueDinsarProductPublish, +} from '../api/dinsarProducts'; +import { + DINSAR_ENGINE_ALL, + buildDinsarEngineOptions, + getDinsarEngineMeta, +} from '../utils/dinsarEngines'; + +const STATUS_TONE_MAP = { + READY: 'ready', + PARTIAL: 'warn', + QUARANTINED: 'error', + WARN: 'warn', + ERROR: 'error', + REBUILDING: 'info', +}; + +function formatDateTime(value) { + if (!value) return '-'; + try { + return new Date(value).toLocaleString(); + } catch { + return String(value); + } +} + +function parseDirectoryList(value) { + return [...new Set( + String(value || '') + .split(/[\r\n,;]+/) + .map((item) => item.trim()) + .filter(Boolean) + )]; +} + +function getMessageTone(message) { + return /失败|error|Error|ERROR/.test(String(message || '')) ? 'error' : 'success'; +} + +function StatusPill({ label, tone = 'neutral' }) { + return <span className={`dinsar-status-pill tone-${tone}`}>{label}</span>; +} + +function MetaField({ label, value, multiline = false }) { + const displayValue = value === null || value === undefined || value === '' ? '-' : value; + return ( + <div className="dinsar-catalog-meta-field"> + <span>{label}</span> + <strong className={multiline ? 'break-all' : ''}>{displayValue}</strong> + </div> + ); +} + +export default function DinsarCatalogPanel({ + readOnly = false, + compact = false, + initialSourceDir = '', + onTaskQueued, +}) { + const [catalogStatus, setCatalogStatus] = useState(null); + const [products, setProducts] = useState([]); + const [selectedProductId, setSelectedProductId] = useState(null); + const [selectedProduct, setSelectedProduct] = useState(null); + const [loading, setLoading] = useState(false); + const [detailLoading, setDetailLoading] = useState(false); + const [actionLoading, setActionLoading] = useState(false); + const [actionMessage, setActionMessage] = useState(''); + const [sourceDirectoriesText, setSourceDirectoriesText] = useState(initialSourceDir || ''); + const [publishRoot, setPublishRoot] = useState(''); + const [engineFilter, setEngineFilter] = useState(DINSAR_ENGINE_ALL); + const [queryDraft, setQueryDraft] = useState(''); + const [queryApplied, setQueryApplied] = useState(''); + + const listLimit = compact ? 8 : 24; + const previewBaseUrl = apiClient.defaults.baseURL || '/api'; + + useEffect(() => { + if (!initialSourceDir) return; + setSourceDirectoriesText((current) => (current.trim() ? current : initialSourceDir)); + }, [initialSourceDir]); + + const sourceDirectories = useMemo( + () => parseDirectoryList(sourceDirectoriesText), + [sourceDirectoriesText] + ); + + const engineOptions = useMemo( + () => buildDinsarEngineOptions(products, { includeKnown: true }), + [products] + ); + const selectedEngineMeta = useMemo( + () => (engineFilter === DINSAR_ENGINE_ALL ? null : getDinsarEngineMeta(engineFilter)), + [engineFilter] + ); + + useEffect(() => { + if (engineFilter === DINSAR_ENGINE_ALL) return; + if (!engineOptions.some((option) => option.value === engineFilter)) { + setEngineFilter(DINSAR_ENGINE_ALL); + } + }, [engineFilter, engineOptions]); + + const loadCatalog = useCallback(async () => { + setLoading(true); + try { + const [statusData, productData] = await Promise.all([ + getDinsarCatalogStatus(), + listDinsarProducts({ + limit: listLimit, + offset: 0, + engine_code: engineFilter === DINSAR_ENGINE_ALL ? undefined : engineFilter, + query: queryApplied || undefined, + }), + ]); + setCatalogStatus(statusData); + const nextItems = Array.isArray(productData?.items) ? productData.items : []; + setProducts(nextItems); + setSelectedProductId((current) => { + if (current && nextItems.some((item) => item.id === current)) { + return current; + } + return nextItems[0]?.id ?? null; + }); + } catch (error) { + setActionMessage(`结果目录状态加载失败：${error?.response?.data?.detail || error.message}`); + setCatalogStatus(null); + setProducts([]); + setSelectedProductId(null); + } finally { + setLoading(false); + } + }, [engineFilter, listLimit, queryApplied]); + + const loadProductDetail = useCallback(async (productId) => { + if (!productId) { + setSelectedProduct(null); + return; + } + setSelectedProduct(null); + setDetailLoading(true); + try { + const detail = await getDinsarProductDetail(productId); + setSelectedProduct(detail); + } catch (error) { + setSelectedProduct({ + error: error?.response?.data?.detail || error.message || '结果详情加载失败', + }); + } finally { + setDetailLoading(false); + } + }, []); + + useEffect(() => { + loadCatalog(); + }, [loadCatalog]); + + useEffect(() => { + loadProductDetail(selectedProductId); + }, [loadProductDetail, selectedProductId]); + + const handleApplyFilters = useCallback(() => { + setQueryApplied(queryDraft.trim()); + }, [queryDraft]); + + const handleResetFilters = useCallback(() => { + setEngineFilter(DINSAR_ENGINE_ALL); + setQueryDraft(''); + setQueryApplied(''); + }, []); + + const handleQueuePublish = async () => { + if (readOnly || sourceDirectories.length === 0) return; + setActionLoading(true); + setActionMessage(''); + try { + const result = await queueDinsarProductPublish({ + source_directories: sourceDirectories, + publish_root: publishRoot.trim() || null, + rebuild_catalog: true, + }); + setActionMessage(`结果包发布任务已入队：${result.task_id}`); + onTaskQueued?.(result.task_id); + await loadCatalog(); + } catch (error) { + setActionMessage(`结果包发布失败：${error?.response?.data?.detail || error.message}`); + } finally { + setActionLoading(false); + } + }; + + const handleQueueRebuild = async () => { + if (readOnly) return; + setActionLoading(true); + setActionMessage(''); + try { + const result = await queueDinsarCatalogRebuild({ + publish_root: publishRoot.trim() || null, + full_rebuild: true, + }); + setActionMessage(`结果目录重建任务已入队：${result.task_id}`); + onTaskQueued?.(result.task_id); + await loadCatalog(); + } catch (error) { + setActionMessage(`结果目录重建失败：${error?.response?.data?.detail || error.message}`); + } finally { + setActionLoading(false); + } + }; + + const catalogTone = STATUS_TONE_MAP[catalogStatus?.status] || 'neutral'; + const actionTone = getMessageTone(actionMessage); + const selectedIssues = Array.isArray(selectedProduct?.issues) ? selectedProduct.issues : []; + const selectedAssets = Array.isArray(selectedProduct?.assets) ? selectedProduct.assets : []; + const selectedPairingTrace = selectedProduct?.pairing_trace || null; + const selectedPairingNetwork = selectedProduct?.pairing_network || null; + const selectedPairingRun = selectedPairingNetwork?.run || null; + const selectedPairingEdge = selectedPairingNetwork?.edge || null; + const selectedPairingMetric = selectedPairingNetwork?.metric || null; + const selectedProductEngine = getDinsarEngineMeta(selectedProduct?.engine_code); + const selectedStatusTone = STATUS_TONE_MAP[selectedProduct?.status] || 'neutral'; + + return ( + <div className={`dinsar-catalog-shell ${compact ? 'compact' : ''}`}> + <div className="dinsar-catalog-header"> + <div className="dinsar-catalog-header-copy"> + <strong>{compact ? '结果目录状态' : '标准结果包目录'}</strong> + <p> + {compact + ? '查看结果包目录与数据库索引是否一致。' + : '统一结果目录按 engine + pair + run 管理，便于同一对影像保留多套生产结果并行对比。'} + </p> + </div> + <div className="dinsar-catalog-header-actions"> + {selectedEngineMeta && ( + <span className={`dinsar-engine-badge tone-${selectedEngineMeta.tone}`}> + {selectedEngineMeta.shortLabel} + </span> + )} + <button onClick={loadCatalog} disabled={loading || actionLoading}> + {loading ? '刷新中...' : '刷新'} + </button> + </div> + </div> + + <div className="dinsar-catalog-summary"> + <div className="dinsar-catalog-stat-card"> + <span>目录状态</span> + <strong>{catalogStatus?.status || '未知'}</strong> + <StatusPill label={catalogStatus?.status || 'UNKNOWN'} tone={catalogTone} /> + </div> + <div className="dinsar-catalog-stat-card"> + <span>需要重建</span> + <strong>{catalogStatus?.needs_rebuild ? '是' : '否'}</strong> + <small>{catalogStatus?.needs_rebuild ? 'Manifest 与数据库存在漂移' : '目录登记正常'}</small> + </div> + <div className="dinsar-catalog-stat-card"> + <span>Manifest / 数据库</span> + <strong>{catalogStatus?.manifest_count ?? 0} / {catalogStatus?.db_count ?? 0}</strong> + <small>已登记结果包总量</small> + </div> + <div className="dinsar-catalog-stat-card"> + <span>问题数量</span> + <strong>{catalogStatus?.issue_count ?? 0}</strong> + <small>含缺失文件与健康异常</small> + </div> + </div> + + <div className="dinsar-catalog-meta-strip"> + <div><strong>结果包根目录：</strong>{catalogStatus?.storage_root || '-'}</div> + <div><strong>最近消息：</strong>{catalogStatus?.last_message || '-'}</div> + <div><strong>最近全量重建：</strong>{formatDateTime(catalogStatus?.last_full_rebuild_at)}</div> + </div> + + {actionMessage && ( + <div className={`dinsar-catalog-message tone-${actionTone}`}> + {actionMessage} + </div> + )} + + {!compact && ( + <div className="dinsar-catalog-manage"> + <div className="dinsar-catalog-manage-copy"> + <strong>手动发布与目录重建</strong> + <p> + 这里用于把既有结果目录重新发布为标准结果包，并按最新规则重建目录索引。 + 如果同一对影像存在 ENVI 与 ISCE2 两套结果，它们会依赖 `engine_code` 与 `run_key` 分别登记，不会互相覆盖。 + </p> + </div> + <div className="dinsar-catalog-manage-form"> + <textarea + value={sourceDirectoriesText} + onChange={(event) => setSourceDirectoriesText(event.target.value)} + placeholder="输入一个或多个结果源目录，支持换行、逗号或分号分隔" + disabled={readOnly || actionLoading} + /> + <input + value={publishRoot} + onChange={(event) => setPublishRoot(event.target.value)} + placeholder="可选：自定义标准结果包根目录，留空使用系统配置" + disabled={readOnly || actionLoading} + /> + <div className="dinsar-catalog-manage-actions"> + <button + type="button" + className="primary" + onClick={handleQueuePublish} + disabled={readOnly || actionLoading || sourceDirectories.length === 0} + > + {actionLoading ? '处理中...' : '发布结果包并重建'} + </button> + <button + type="button" + onClick={handleQueueRebuild} + disabled={readOnly || actionLoading} + > + 仅重建目录 + </button> + </div> + </div> + </div> + )} + + <div className={`dinsar-catalog-workspace ${compact ? 'compact' : ''}`}> + <aside className="dinsar-catalog-list-card"> + <div className="dinsar-catalog-card-head"> + <div> + <strong>结果包列表</strong> + <span> + {loading ? '加载中...' : `当前展示 ${products.length} 条`} + </span> + </div> + {queryApplied && <StatusPill label={`检索: ${queryApplied}`} tone="info" />} + </div> + + <div className="dinsar-catalog-filter-bar"> + <label className="dinsar-catalog-filter-field"> + <span>生产引擎</span> + <select value={engineFilter} onChange={(event) => setEngineFilter(event.target.value)}> + <option value={DINSAR_ENGINE_ALL}>全部引擎</option> + {engineOptions.map((option) => ( + <option key={option.value} value={option.value}> + {option.label} + </option> + ))} + </select> + </label> + + <label className="dinsar-catalog-filter-field search"> + <span>检索</span> + <input + value={queryDraft} + onChange={(event) => setQueryDraft(event.target.value)} + onKeyDown={(event) => { + if (event.key === 'Enter') { + event.preventDefault(); + handleApplyFilters(); + } + }} + placeholder="搜索任务名 / pair / run / 引擎" + /> + </label> + + <div className="dinsar-catalog-filter-actions"> + <button type="button" onClick={handleApplyFilters}>查询</button> + <button type="button" onClick={handleResetFilters}>重置</button> + </div> + </div> + + {products.length === 0 ? ( + <div className="dinsar-catalog-empty"> + {loading ? '正在加载结果包...' : '当前筛选条件下没有结果包。'} + </div> + ) : ( + <div className="dinsar-catalog-list"> + {products.map((item) => { + const tone = STATUS_TONE_MAP[item.status] || 'neutral'; + const engineMeta = getDinsarEngineMeta(item.engine_code); + return ( + <button + key={item.id} + type="button" + className={`dinsar-catalog-list-item ${selectedProductId === item.id ? 'active' : ''}`} + onClick={() => setSelectedProductId(item.id)} + > + <div className="dinsar-catalog-list-item-top"> + <strong>{item.display_name || item.product_id}</strong> + <StatusPill label={item.status || 'UNKNOWN'} tone={tone} /> + </div> + <div className="dinsar-catalog-list-item-badges"> + <span className={`dinsar-engine-badge tone-${engineMeta.tone}`}>{engineMeta.shortLabel}</span> + <span>{formatDateTime(item.published_at)}</span> + </div> + <div className="dinsar-catalog-list-item-meta"> + {(item.task_alias || item.task_name || '-')}{item.run_key ? ` / ${item.run_key}` : ''} + </div> + <div className="dinsar-catalog-list-item-meta"> + {item.pair_key || '-'} + </div> + {(item.selection_strategy || item.network_run_id || item.network_edge_id != null) && ( + <div className="dinsar-catalog-list-item-trace"> + {(item.selection_strategy || 'trace')} + {item.network_edge_id != null ? ` / edge ${item.network_edge_id}` : ''} + {item.network_run_id ? ` / ${item.network_run_id}` : ''} + </div> + )} + </button> + ); + })} + </div> + )} + </aside> + + <section className="dinsar-catalog-detail-card"> + <div className="dinsar-catalog-card-head"> + <div> + <strong>结果包详情</strong> + <span>查看选中结果的发布信息、配对溯源与资产健康</span> + </div> + </div> + + {!selectedProductId ? ( + <div className="dinsar-catalog-empty">请选择一个结果包查看详情。</div> + ) : detailLoading || !selectedProduct ? ( + <div className="dinsar-catalog-empty">正在加载详情...</div> + ) : selectedProduct?.error ? ( + <div className="dinsar-catalog-empty error">{selectedProduct.error}</div> + ) : ( + <div className="dinsar-catalog-detail-body"> + <div className="dinsar-catalog-hero"> + <div className="dinsar-catalog-preview-frame"> + <img + src={`${previewBaseUrl}/dinsar-products/${selectedProduct.id}/preview`} + alt={selectedProduct.display_name || selectedProduct.product_id} + /> + </div> + + <div className="dinsar-catalog-hero-meta"> + <div className="dinsar-catalog-hero-title-row"> + <div> + <h4>{selectedProduct.display_name || selectedProduct.product_id}</h4> + <p>{selectedProduct.task_alias || selectedProduct.task_name || '未命名任务'}</p> + </div> + <div className="dinsar-catalog-hero-badges"> + <span className={`dinsar-engine-badge tone-${selectedProductEngine.tone}`}> + {selectedProductEngine.shortLabel} + </span> + <StatusPill label={selectedProduct.status || 'UNKNOWN'} tone={selectedStatusTone} /> + </div> + </div> + + <div className="dinsar-catalog-kv-grid"> + <MetaField label="产品编号" value={selectedProduct.product_id} multiline /> + <MetaField label="配对标识" value={selectedProduct.pair_key} multiline /> + <MetaField label="场景配对 UID" value={selectedProduct.pair_uid} multiline /> + <MetaField label="运行标识" value={selectedProduct.run_key} multiline /> + <MetaField label="生产配置" value={selectedProduct.profile_code} /> + <MetaField label="健康状态" value={selectedProduct.health_status} /> + <MetaField label="主文件" value={selectedProduct.primary_asset_path} multiline /> + <MetaField label="源文件" value={selectedProduct.source_primary_path} multiline /> + <MetaField label="结果包目录" value={selectedProduct.publish_dir} multiline /> + </div> + </div> + </div> + + <div className="dinsar-catalog-detail-grid"> + <div className="dinsar-catalog-section-card"> + <div className="dinsar-catalog-section-title">时空概览</div> + <MetaField label="主影像日期" value={selectedProduct.profile?.master_imaging_date} /> + <MetaField label="从影像日期" value={selectedProduct.profile?.slave_imaging_date} /> + <MetaField label="时间基线" value={selectedProduct.profile?.time_baseline_days} /> + <MetaField label="空间基线" value={selectedProduct.profile?.spatial_baseline_meters} /> + </div> + <div className="dinsar-catalog-section-card"> + <div className="dinsar-catalog-section-title">空间范围</div> + <MetaField label="最小坐标" value={`${selectedProduct.min_lon ?? '-'}, ${selectedProduct.min_lat ?? '-'}`} /> + <MetaField label="最大坐标" value={`${selectedProduct.max_lon ?? '-'}, ${selectedProduct.max_lat ?? '-'}`} /> + <MetaField label="登记时间" value={formatDateTime(selectedProduct.registered_at)} /> + <MetaField label="发布时间" value={formatDateTime(selectedProduct.published_at)} /> + </div> + </div> + + <div className="dinsar-catalog-section-card"> + <div className="dinsar-catalog-section-title">配对追踪</div> + {!selectedPairingTrace?.network_run_id ? ( + <div className="dinsar-catalog-empty inline">当前结果未携带完整的配对网络追踪信息。</div> + ) : ( + <div className="dinsar-catalog-detail-grid"> + <div className="dinsar-catalog-section-card nested"> + <MetaField label="network_run_id" value={selectedPairingTrace.network_run_id} multiline /> + <MetaField label="network_edge_id" value={selectedPairingTrace.network_edge_id} /> + <MetaField label="pair_uid" value={selectedPairingTrace.pair_uid} multiline /> + <MetaField label="选择策略" value={selectedPairingTrace.selection_strategy} /> + <MetaField label="策略版本" value={selectedPairingTrace.policy_version} /> + </div> + <div className="dinsar-catalog-section-card nested"> + <MetaField label="网络记录" value={selectedPairingNetwork?.run_found ? '已找到' : '未找到'} /> + <MetaField label="边记录" value={selectedPairingNetwork?.edge_found ? '已找到' : '未找到'} /> + <MetaField label="运行状态" value={selectedPairingRun?.status} /> + <MetaField label="候选边数" value={selectedPairingRun?.candidate_count} /> + <MetaField label="入选边数" value={selectedPairingRun?.selected_edge_count} /> + <MetaField label="告警数" value={selectedPairingRun?.warning_count} /> + </div> + <div className="dinsar-catalog-section-card nested"> + <MetaField label="edge_rank" value={selectedPairingEdge?.edge_rank} /> + <MetaField label="selection_reason" value={selectedPairingEdge?.selection_reason} multiline /> + <MetaField label="selection_score" value={selectedPairingEdge?.selection_score} /> + <MetaField label="reference_edge" value={selectedPairingEdge?.is_reference_edge ? '是' : '否'} /> + <MetaField label="metric_cache_ref_id" value={selectedPairingEdge?.metric_cache_ref_id} /> + </div> + <div className="dinsar-catalog-section-card nested"> + <MetaField label="主从日期" value={`${selectedPairingMetric?.master_imaging_date || '-'} / ${selectedPairingMetric?.slave_imaging_date || '-'}`} /> + <MetaField label="主从卫星" value={`${selectedPairingMetric?.master_satellite || '-'} / ${selectedPairingMetric?.slave_satellite || '-'}`} /> + <MetaField label="主从模式" value={`${selectedPairingMetric?.master_imaging_mode || '-'} / ${selectedPairingMetric?.slave_imaging_mode || '-'}`} /> + <MetaField label="主从极化" value={`${selectedPairingMetric?.master_polarization || '-'} / ${selectedPairingMetric?.slave_polarization || '-'}`} /> + <MetaField label="时间基线" value={selectedPairingMetric?.time_baseline_days} /> + <MetaField label="空间基线" value={selectedPairingMetric?.spatial_baseline_meters} /> + </div> + </div> + )} + </div> + + <div className="dinsar-catalog-detail-grid"> + <div className="dinsar-catalog-section-card"> + <div className="dinsar-catalog-section-title">资产列表 ({selectedAssets.length})</div> + {selectedAssets.length === 0 ? ( + <div className="dinsar-catalog-empty inline">暂无资产记录。</div> + ) : ( + <div className="dinsar-catalog-asset-list"> + {selectedAssets.map((asset) => ( + <div key={asset.id} className={`dinsar-catalog-asset-item ${asset.exists_flag ? 'ok' : 'missing'}`}> + <div className="dinsar-catalog-asset-top"> + <strong>{asset.asset_role}</strong> + <span>{asset.exists_flag ? '文件存在' : '文件缺失'}</span> + </div> + <div>{asset.asset_name}</div> + <div className="break-all">{asset.absolute_path}</div> + </div> + ))} + </div> + )} + </div> + + <div className="dinsar-catalog-section-card"> + <div className="dinsar-catalog-section-title">问题列表 ({selectedIssues.length})</div> + {selectedIssues.length === 0 ? ( + <div className="dinsar-catalog-empty inline ok">当前没有登记问题。</div> + ) : ( + <div className="dinsar-catalog-issue-list"> + {selectedIssues.map((issue) => ( + <div key={issue.id} className={`dinsar-catalog-issue-item ${String(issue.severity || '').toUpperCase() === 'ERROR' ? 'error' : 'warn'}`}> + <div className="dinsar-catalog-issue-top"> + <strong>{issue.issue_code}</strong> + <span>{issue.severity}</span> + </div> + <div>{issue.message}</div> + {issue.repair_action && ( + <div className="dinsar-catalog-issue-action"> + 建议修复动作：{issue.repair_action} + </div> + )} + </div> + ))} + </div> + )} + </div> + </div> + </div> + )} + </section> + </div> + </div> + ); +} diff --git a/frontend/src/components/PsStackModal.jsx b/frontend/src/components/PsStackModal.jsx index dbb83be..bb9416e 100644 --- a/frontend/src/components/PsStackModal.jsx +++ b/frontend/src/components/PsStackModal.jsx @@ -27,7 +27,7 @@ function PsStackModal({ return ( <div className="modal-overlay visible"> <div className="modal-content"> - <h3>准备PS时序数据栈</h3> + <h3>准备时序InSAR候选栈</h3> <form onSubmit={onSubmit}> <div className="form-group"> <label>研究区域来源:</label> diff --git a/frontend/src/components/PsinsarCatalogPanel.jsx b/frontend/src/components/PsinsarCatalogPanel.jsx index 9a7a5a2..28b3754 100644 --- a/frontend/src/components/PsinsarCatalogPanel.jsx +++ b/frontend/src/components/PsinsarCatalogPanel.jsx @@ -96,7 +96,7 @@ export default function PsinsarCatalogPanel({ return nextItems[0]?.id ?? null; }); } catch (error) { - setActionMessage(`PS-InSAR 结果目录状态加载失败：${error?.response?.data?.detail || error.message}`); + setActionMessage(`时序InSAR产物目录状态加载失败：${error?.response?.data?.detail || error.message}`); setCatalogStatus(null); setProducts([]); setSelectedProductId(null); @@ -146,11 +146,11 @@ export default function PsinsarCatalogPanel({ publish_root: publishRoot.trim() || null, full_rebuild: true, }); - setActionMessage(`PS-InSAR 结果目录重建任务已入队：${result.task_id}`); + setActionMessage(`时序InSAR产物目录重建任务已入队：${result.task_id}`); onTaskQueued?.(result.task_id); await loadCatalog(); } catch (error) { - setActionMessage(`PS-InSAR 结果目录重建失败：${error?.response?.data?.detail || error.message}`); + setActionMessage(`时序InSAR产物目录重建失败：${error?.response?.data?.detail || error.message}`); } finally { setActionLoading(false); } @@ -164,9 +164,9 @@ export default function PsinsarCatalogPanel({ <div style={panelCardStyle}> <div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', gap: 12, marginBottom: 10 }}> <div> - <strong style={{ fontSize: 14 }}>PS-InSAR 结果目录</strong> + <strong style={{ fontSize: 14 }}>时序InSAR 产物目录</strong> <div style={{ fontSize: 11, color: '#64748b', marginTop: 2 }}> - 结果目录以 `psinsar.publish.v1` bundle 为事实源，数据库仅保存索引与展示信息。 + 当前默认登记 SBAS 流程产物。结果目录以 `psinsar.publish.v1` bundle 为事实源，数据库仅保存索引与展示信息，后续可继续兼容 PS-InSAR / SBAS-InSAR。 </div> </div> <button @@ -224,7 +224,7 @@ export default function PsinsarCatalogPanel({ <input value={publishRoot} onChange={event => setPublishRoot(event.target.value)} - placeholder="可选：自定义 PS-InSAR 发布根目录，留空使用系统默认目录" + placeholder="可选：自定义时序InSAR发布根目录，留空使用系统默认目录" disabled={readOnly || actionLoading} style={{ width: '100%', @@ -249,7 +249,7 @@ export default function PsinsarCatalogPanel({ fontSize: 12, }} > - {actionLoading ? '处理中...' : '重建 PS-InSAR 结果目录'} + {actionLoading ? '处理中...' : '重建时序InSAR产物目录'} </button> {actionMessage && ( <div style={{ marginTop: 8, fontSize: 12, color: actionMessage.includes('失败') ? '#dc2626' : '#166534' }}> @@ -259,14 +259,14 @@ export default function PsinsarCatalogPanel({ </div> )} - <div style={{ display: 'grid', gridTemplateColumns: 'minmax(260px, 360px) 1fr', gap: 12 }}> + <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(320px, 1fr))', gap: 12 }}> <div style={{ border: '1px solid #e2e8f0', borderRadius: 6, overflow: 'hidden' }}> <div style={{ padding: '8px 10px', background: '#f8fafc', fontSize: 12, fontWeight: 600 }}> 产品列表 ({products.length}) </div> {products.length === 0 ? ( <div style={{ padding: '12px', fontSize: 12, color: '#94a3b8' }}> - {loading ? '正在加载结果...' : '当前没有已登记的 PS-InSAR 产品。'} + {loading ? '正在加载结果...' : '当前没有已登记的时序InSAR产物。'} </div> ) : ( <div style={{ maxHeight: 420, overflowY: 'auto' }}> diff --git a/frontend/src/components/ResultExportModal.rewrite.jsx b/frontend/src/components/ResultExportModal.rewrite.jsx new file mode 100644 index 0000000..b94c4fe --- /dev/null +++ b/frontend/src/components/ResultExportModal.rewrite.jsx @@ -0,0 +1,161 @@ +import { useMemo, useState } from 'react'; +import { exportDinsarResults } from '../api/dinsar'; +import { getDinsarEngineMeta } from '../utils/dinsarEngines'; + +const EXAMPLE_TARGET_DIR = String.raw`例如: D:\Export\Results 或 \\server\share\results`; + +export default function ResultExportModal({ results = [], onClose }) { + const [targetDir, setTargetDir] = useState(''); + const [selectedIds, setSelectedIds] = useState(() => new Set(results.map((result) => result.id))); + const [exporting, setExporting] = useState(false); + const [exportResult, setExportResult] = useState(null); + const [error, setError] = useState(''); + + const selectedCount = selectedIds.size; + const sortedResults = useMemo(() => [...results], [results]); + + const toggleSelect = (id) => { + setSelectedIds((prev) => { + const next = new Set(prev); + if (next.has(id)) { + next.delete(id); + } else { + next.add(id); + } + return next; + }); + }; + + const toggleAll = () => { + if (selectedIds.size === results.length) { + setSelectedIds(new Set()); + return; + } + setSelectedIds(new Set(results.map((result) => result.id))); + }; + + const handleExport = async () => { + const dir = targetDir.trim(); + if (!dir) { + setError('请输入目标路径。'); + return; + } + if (selectedIds.size === 0) { + setError('请至少选择一个结果。'); + return; + } + + setError(''); + setExporting(true); + setExportResult(null); + try { + const response = await exportDinsarResults([...selectedIds], dir); + setExportResult(response); + } catch (eventualError) { + setError(eventualError.response?.data?.detail || eventualError.message || '提取失败。'); + } finally { + setExporting(false); + } + }; + + return ( + <div className="modal-overlay visible" onClick={onClose}> + <div className="modal-content result-export-modal" onClick={(event) => event.stopPropagation()}> + <div className="modal-header"> + <h3>提取 D-InSAR 结果</h3> + <button type="button" className="modal-close-btn" onClick={onClose} aria-label="关闭"> + × + </button> + </div> + + <div className="modal-body"> + <div className="export-path-section"> + <label>目标路径，支持本地盘符或 UNC 路径，例如 `D:\Export\Results`。</label> + <input + type="text" + value={targetDir} + onChange={(event) => setTargetDir(event.target.value)} + placeholder={EXAMPLE_TARGET_DIR} + disabled={exporting} + className="export-path-input" + /> + </div> + + <div className="export-select-section"> + <div className="export-select-header"> + <label> + <input + type="checkbox" + checked={selectedIds.size === results.length && results.length > 0} + onChange={toggleAll} + disabled={exporting || results.length === 0} + /> + 全选 ({selectedCount}/{results.length}) + </label> + </div> + + <div className="export-select-hint"> + 导出时会优先按任务名创建子目录；如果同名结果已存在且内容不同，会自动追加后缀避免覆盖。 + </div> + + <ul className="export-result-list"> + {sortedResults.map((result) => { + const engineMeta = getDinsarEngineMeta(result.engine_code); + return ( + <li key={result.id} className="export-result-item"> + <label> + <input + type="checkbox" + checked={selectedIds.has(result.id)} + onChange={() => toggleSelect(result.id)} + disabled={exporting} + /> + <span className="export-result-name" title={result.file_path || result.name}> + {result.name} + </span> + <span className={`dinsar-engine-badge tone-${engineMeta.tone}`}> + {engineMeta.shortLabel} + </span> + </label> + </li> + ); + })} + </ul> + </div> + + {error && <div className="export-error">{error}</div>} + + {exportResult && ( + <div className="export-summary"> + <div className="export-summary-title">提取完成</div> + <div className="export-summary-stats"> + <span className="stat-ok">复制: {exportResult.copied}</span> + <span className="stat-skip">跳过: {exportResult.skipped}</span> + {exportResult.failed > 0 && ( + <span className="stat-fail">失败: {exportResult.failed}</span> + )} + </div> + <div className="export-summary-dir"> + 目标目录: {exportResult.target_dir} + </div> + </div> + )} + </div> + + <div className="modal-footer"> + <button type="button" className="btn-secondary" onClick={onClose} disabled={exporting}> + 关闭 + </button> + <button + type="button" + onClick={handleExport} + disabled={exporting || selectedIds.size === 0 || !targetDir.trim()} + className="btn-primary" + > + {exporting ? '提取中...' : `确定提取 ${selectedIds.size} 个结果`} + </button> + </div> + </div> + </div> + ); +} diff --git a/frontend/src/components/app/AppSidePanel.jsx b/frontend/src/components/app/AppSidePanel.jsx index 75c4b80..53d1333 100644 --- a/frontend/src/components/app/AppSidePanel.jsx +++ b/frontend/src/components/app/AppSidePanel.jsx @@ -8,6 +8,7 @@ import { LEFT_GROUP_TABS, LEFT_TAB_GROUP, LEFT_TAB_SECTION, + PRODUCTION_WORKSPACE_ROUTE_TABS, } from '../../config/appConstants'; import { getLeftTabLabel } from '../../utils/appUiHelpers'; import { PanelLoadingBody, PanelLoadingPanel } from './AppLoadingFallbacks'; @@ -15,27 +16,26 @@ import { PanelLoadingBody, PanelLoadingPanel } from './AppLoadingFallbacks'; const LazyDataMonitorPanel = lazy(() => import('../../DataMonitorPanel')); const LazyDataCopierPanel = lazy(() => import('../../DataCopierPanel')); const LazyIDLAutomationPanel = lazy(() => import('../../IDLAutomationPanel')); -const LazyDinsarProductionPanel = lazy(() => import('../../DinsarProductionPanel')); -const LazyDinsarProductsPanel = lazy(() => import('../../DinsarProductsPanel')); const LazyHazardPointPanel = lazy(() => import('../../HazardPointPanel')); const LazyHealthCheckPanel = lazy(() => import('../../HealthCheckPanel')); -const LazyTimeseriesProductionPanel = lazy(() => import('../../TimeseriesProductionPanel')); const LazyWaterMonitorPanel = lazy(() => import('../../WaterMonitorPanel')); const LazyUserAdminPanel = lazy(() => import('../../UserAdminPanel')); const LazyAuditLogPanel = lazy(() => import('../../AuditLogPanel')); const LazyAiQualityPanel = lazy(() => import('../../panels/AiQualityPanel')); const LazyAiAnalysisPanel = lazy(() => import('../../AiAnalysisPanel')); const LazyPairingPanel = lazy(() => import('../../panels/PairPlanningPanel')); -const LazyDinsarResultPanel = lazy(() => import('../../panels/DinsarResultPanel')); +const LazyDinsarResultPanel = lazy(() => import('../../panels/DinsarResultPanel.rewrite')); const LazyBatchPanel = lazy(() => import('../../panels/BatchPanel')); const LazyPairsListPanel = lazy(() => import('../../panels/PairsListPanel')); const LazyPsResultsPanel = lazy(() => import('../../panels/PsResultsPanel')); const LazyPsinsarCatalogPanel = lazy(() => import('../PsinsarCatalogPanel')); +const LazyProductionWorkspace = lazy(() => import('../../ProductionWorkspace')); export default function AppSidePanel({ leftPanelWidth, leftPanelTab, setLeftPanelTab, + isStandalone, isAdmin, isReadOnlyUser, currentUser, @@ -62,7 +62,13 @@ export default function AppSidePanel({ pairsPanel, psPanel, }) { + const isProductionWorkspace = PRODUCTION_WORKSPACE_ROUTE_TABS.has(leftPanelTab); const activeLeftGroup = LEFT_TAB_GROUP[leftPanelTab] || 'data'; + const leftTabLabelContext = { + pairCount: foundPairs.length, + psResultCount: psResults ? Object.keys(psResults).length : 0, + dinsarTotal, + }; const getVisibleTabs = (tabs = []) => tabs.filter((tab) => isAdmin || !ADMIN_ONLY_TABS.has(tab)); const getVisibleSections = (groupKey) => ( (LEFT_GROUP_SECTIONS[groupKey] || []) @@ -79,6 +85,7 @@ export default function AppSidePanel({ } return getVisibleTabs(LEFT_GROUP_TABS[groupKey] || [])[0] || ''; }; + const mainWorkspaceTab = getDefaultGroupTab('data') || 'data'; const activeGroupSections = getVisibleSections(activeLeftGroup); const hasSectionNav = activeGroupSections.length > 0; const preferredActiveSection = LEFT_TAB_SECTION[leftPanelTab]; @@ -88,59 +95,105 @@ export default function AppSidePanel({ const activeLeafTabs = hasSectionNav ? (activeGroupSections.find((section) => section.key === activeLeftSection)?.tabs || []) : getVisibleTabs(LEFT_GROUP_TABS[activeLeftGroup] || []); - const psResultCount = psResults ? Object.keys(psResults).length : 0; + const standaloneSectionTabs = isProductionWorkspace + ? [] + : ( + hasSectionNav + ? (activeGroupSections.find((section) => section.key === activeLeftSection)?.tabs || activeLeafTabs) + : activeLeafTabs + ); + const standaloneEyebrow = [LEFT_GROUP_LABELS[activeLeftGroup], activeGroupSections.find((section) => section.key === activeLeftSection)?.label] + .filter(Boolean) + .join(' / '); + const standaloneTitle = isProductionWorkspace + ? '生产管理' + : getLeftTabLabel(leftPanelTab, leftTabLabelContext); + const standaloneDescription = isProductionWorkspace + ? '这里统一承载 D-InSAR 与时序InSAR的运行和产物页面，当前时序流程默认接入 SBAS，后续可继续扩展 PS-InSAR / SBAS-InSAR。' + : '当前模块已切换为独立工作区模式。'; return ( - <aside className="panel data-panel" style={{ display: 'flex', flexDirection: 'column', width: leftPanelWidth }}> - <div className="panel-tabs"> - <div className="tabs-header group-tabs"> - {Object.entries(LEFT_GROUP_LABELS) - .filter(([groupKey]) => { - if (isAdmin) return true; - return !!getDefaultGroupTab(groupKey); - }) - .map(([groupKey, label]) => ( - <button - key={groupKey} - className={activeLeftGroup === groupKey ? 'active-tab' : ''} - onClick={() => { - const nextTab = getDefaultGroupTab(groupKey); - if (nextTab) setLeftPanelTab(nextTab); - }} - > - {label} - </button> - ))} + <aside + className={`panel data-panel${isStandalone ? ' panel--standalone' : ''}`} + style={{ display: 'flex', flexDirection: 'column', width: leftPanelWidth }} + > + {isStandalone ? ( + <div className="panel-standalone-header"> + <div className="panel-standalone-header-main"> + <span className="panel-standalone-eyebrow">{standaloneEyebrow}</span> + <strong>{standaloneTitle}</strong> + <p>{standaloneDescription}</p> + </div> + <div className="panel-standalone-actions"> + <button + type="button" + className="panel-standalone-return" + onClick={() => setLeftPanelTab(mainWorkspaceTab)} + > + 返回主界面 + </button> + {standaloneSectionTabs.length > 1 && ( + <> + {standaloneSectionTabs.map((tabKey) => ( + <button + key={tabKey} + className={leftPanelTab === tabKey ? 'active-tab' : ''} + onClick={() => setLeftPanelTab(tabKey)} + > + {getLeftTabLabel(tabKey, leftTabLabelContext)} + </button> + ))} + </> + )} + </div> </div> - {hasSectionNav && ( - <div className="tabs-header section-tabs"> - {activeGroupSections.map((section) => ( + ) : ( + <div className="panel-tabs"> + <div className="tabs-header group-tabs"> + {Object.entries(LEFT_GROUP_LABELS) + .filter(([groupKey]) => { + if (isAdmin) return true; + return !!getDefaultGroupTab(groupKey); + }) + .map(([groupKey, label]) => ( + <button + key={groupKey} + className={activeLeftGroup === groupKey ? 'active-tab' : ''} + onClick={() => { + const nextTab = getDefaultGroupTab(groupKey); + if (nextTab) setLeftPanelTab(nextTab); + }} + > + {label} + </button> + ))} + </div> + {hasSectionNav && ( + <div className="tabs-header section-tabs"> + {activeGroupSections.map((section) => ( + <button + key={section.key} + className={activeLeftSection === section.key ? 'active-tab' : ''} + onClick={() => setLeftPanelTab(section.tabs[0])} + > + {section.label} + </button> + ))} + </div> + )} + <div className="tabs-header left-tabs sub-tabs"> + {activeLeafTabs.map((tabKey) => ( <button - key={section.key} - className={activeLeftSection === section.key ? 'active-tab' : ''} - onClick={() => setLeftPanelTab(section.tabs[0])} + key={tabKey} + className={leftPanelTab === tabKey ? 'active-tab' : ''} + onClick={() => setLeftPanelTab(tabKey)} > - {section.label} + {getLeftTabLabel(tabKey, leftTabLabelContext)} </button> ))} </div> - )} - <div className="tabs-header left-tabs sub-tabs"> - {activeLeafTabs.map((tabKey) => ( - <button - key={tabKey} - className={leftPanelTab === tabKey ? 'active-tab' : ''} - onClick={() => setLeftPanelTab(tabKey)} - > - {getLeftTabLabel(tabKey, { - pairCount: foundPairs.length, - psResultCount, - dinsarTotal, - })} - </button> - ))} </div> - </div> + )} {leftPanelTab === 'data' && ( <RadarDataPanel @@ -227,48 +280,14 @@ export default function AppSidePanel({ </div> )} - {leftPanelTab === 'dinsar_production' && ( + {isProductionWorkspace && ( <div className="panel-content" style={{ flex: '1 1 auto', padding: 0, overflow: 'auto' }}> - <Suspense fallback={<PanelLoadingBody message="正在加载 D-InSAR 生产面板..." />}> - <LazyDinsarProductionPanel - readOnly={isReadOnlyUser} - currentUser={currentUser} - onJobQueued={(taskId) => taskPanel.onTaskStart(taskId, 'D-InSAR 任务已入队，等待处理...')} - /> - </Suspense> - </div> - )} - - {leftPanelTab === 'dinsar_products' && ( - <div className="panel-content" style={{ flex: '1 1 auto', padding: 0, overflow: 'auto' }}> - <Suspense fallback={<PanelLoadingBody message="正在加载 D-InSAR 产物面板..." />}> - <LazyDinsarProductsPanel - readOnly={isReadOnlyUser} - onJobQueued={(taskId) => taskPanel.onTaskStart(taskId, 'D-InSAR 产物任务已入队，等待处理...')} - /> - </Suspense> - </div> - )} - - {leftPanelTab === 'ps_production' && ( - <div className="panel-content" style={{ flex: '1 1 auto', padding: 0, overflow: 'auto' }}> - <Suspense fallback={<PanelLoadingBody message="正在加载 PS-InSAR 生产面板..." />}> - <LazyTimeseriesProductionPanel - readOnly={isReadOnlyUser} - onJobQueued={(taskId) => taskPanel.onTaskStart(taskId, 'SBAS 运行已入队，正在执行 prepare...')} - /> - </Suspense> - </div> - )} - - {leftPanelTab === 'ps_products' && ( - <div className="panel-content" style={{ flex: '1 1 auto', padding: 0, overflow: 'auto' }}> - <Suspense fallback={<PanelLoadingBody message="正在加载 PS-InSAR 目录面板..." />}> - <div style={{ padding: '16px' }}> - <LazyPsinsarCatalogPanel + <Suspense fallback={<PanelLoadingBody message="正在加载生产管理工作台..." />}> + <div style={{ minHeight: '100%' }}> + <LazyProductionWorkspace + activeEntry={leftPanelTab} readOnly={isReadOnlyUser} - showActions - onTaskQueued={(taskId) => taskPanel.onTaskStart(taskId, 'PS-InSAR 结果目录任务已入队，等待处理...')} + onTaskStart={taskPanel.onTaskStart} /> </div> </Suspense> @@ -380,7 +399,7 @@ export default function AppSidePanel({ {leftPanelTab === 'psinsar_results' && ( <div className="panel-content" style={{ flex: '1 1 auto', padding: 0, overflow: 'auto' }}> - <Suspense fallback={<PanelLoadingBody message="正在加载 PS-InSAR 结果目录..." />}> + <Suspense fallback={<PanelLoadingBody message="正在加载时序InSAR结果目录..." />}> <div style={{ padding: '16px' }}> <LazyPsinsarCatalogPanel readOnly @@ -395,7 +414,7 @@ export default function AppSidePanel({ <div className="panel-content" style={{ flex: '1 1 auto', padding: 0, overflow: 'auto' }}> <div style={{ padding: '16px' }}> <div className="empty-state"> - PS-InSAR 分析页已预留。 + 时序InSAR 分析页已预留。 <br /> 后续可以在这里放置时序分析、速率分级、热点识别和专题统计能力。 </div> diff --git a/frontend/src/components/panels/DinsarResultRow.rewrite.jsx b/frontend/src/components/panels/DinsarResultRow.rewrite.jsx new file mode 100644 index 0000000..17bc893 --- /dev/null +++ b/frontend/src/components/panels/DinsarResultRow.rewrite.jsx @@ -0,0 +1,175 @@ +import { memo } from 'react'; +import { parseDatesFromName, formatYmd } from '../../utils/appUiHelpers'; +import { getDinsarEngineMeta } from '../../utils/dinsarEngines'; + +function truncateMiddle(value, maxLength = 28) { + const text = String(value || '').trim(); + if (!text || text.length <= maxLength) { + return text || '-'; + } + const sideLength = Math.max(6, Math.floor((maxLength - 3) / 2)); + return `${text.slice(0, sideLength)}...${text.slice(-sideLength)}`; +} + +function DinsarResultRow({ + result, + language, + showDates, + isLoading, + isReadOnlyUser, + onLabel, + onAnalyze, + onToggleVisibility, +}) { + const dates = showDates ? parseDatesFromName(result.name, (value) => formatYmd(value, language)) : null; + const engineMeta = getDinsarEngineMeta(result.engine_code); + const hasTrace = !!( + result.selection_strategy || + result.network_run_id || + result.network_edge_id || + result.pair_uid || + result.pair_key || + result.run_key + ); + + return ( + <li className="data-item dinsar-item"> + <div className="dinsar-row-header"> + <span className="data-item-name" title={result.name}> + {result.name} + </span> + <div className="dinsar-row-badges"> + <span + className={`dinsar-engine-badge tone-${engineMeta.tone}`} + title={`${language === 'en' ? 'Engine' : '生产引擎'}: ${engineMeta.label}`} + > + {engineMeta.shortLabel} + </span> + {result.ai_score !== null && ( + <span + className={`ai-score ${result.ai_score > 0.7 ? 'good' : (result.ai_score < 0.4 ? 'bad' : 'medium')}`} + title={language === 'en' ? 'AI quality score' : 'AI 质量评分'} + > + AI {Math.round(result.ai_score * 100)} + </span> + )} + </div> + </div> + + {hasTrace && ( + <div className="dinsar-trace-info"> + <div className="dinsar-trace-line"> + <span + className="dinsar-trace-pill" + title={language === 'en' ? 'Pairing selection strategy' : '配对选择策略'} + > + {result.selection_strategy || 'legacy'} + </span> + {result.run_key && ( + <span + className="dinsar-trace-stat" + title={`${language === 'en' ? 'Run key' : '运行标识'}: ${result.run_key}`} + > + <strong>{language === 'en' ? 'run' : '运行'}</strong> + <span>{truncateMiddle(result.run_key, 24)}</span> + </span> + )} + {result.network_edge_id != null && ( + <span + className="dinsar-trace-stat" + title={language === 'en' ? 'Network edge id' : '网络边编号'} + > + <strong>edge</strong> + <span>{result.network_edge_id}</span> + </span> + )} + {result.network_run_id && ( + <span + className="dinsar-trace-stat" + title={`${language === 'en' ? 'Network run id' : '网络运行编号'}: ${result.network_run_id}`} + > + <strong>{language === 'en' ? 'network' : '网络'}</strong> + <span>{truncateMiddle(result.network_run_id, 22)}</span> + </span> + )} + </div> + <div className="dinsar-trace-line"> + <span + className="dinsar-trace-stat" + title={result.pair_uid || result.pair_key || '-'} + > + <strong>{language === 'en' ? 'pair' : '配对'}</strong> + <span>{truncateMiddle(result.pair_uid || result.pair_key, 36)}</span> + </span> + {result.policy_version && ( + <span className="dinsar-trace-stat"> + <strong>{language === 'en' ? 'policy' : '策略版本'}</strong> + <span>{result.policy_version}</span> + </span> + )} + </div> + </div> + )} + + {dates && ( + <div className="date-info"> + <span className="date-tag master" title={language === 'en' ? 'Master date' : '主影像日期'}> + {dates.master} + </span> + <span className="date-arrow">-></span> + <span className="date-tag slave" title={language === 'en' ? 'Slave date' : '从影像日期'}> + {dates.slave} + </span> + </div> + )} + + <div className="data-item-controls"> + <div className="label-buttons"> + <button + className={`label-btn good ${result.user_label === 1 ? 'active' : ''}`} + onClick={() => onLabel(result.id, result.user_label === 1 ? null : 1)} + title={language === 'en' ? 'Mark as high quality' : '标记为高质量'} + disabled={isReadOnlyUser} + > + {language === 'en' ? 'Good' : '良好'} + </button> + <button + className={`label-btn bad ${result.user_label === 0 ? 'active' : ''}`} + onClick={() => onLabel(result.id, result.user_label === 0 ? null : 0)} + title={language === 'en' ? 'Mark as low quality' : '标记为低质量'} + disabled={isReadOnlyUser} + > + {language === 'en' ? 'Poor' : '较差'} + </button> + </div> + + <div className="dinsar-control-cluster"> + <button + className="ai-analyze-btn" + onClick={(event) => { + event.stopPropagation(); + onAnalyze(result.id); + }} + disabled={isLoading || isReadOnlyUser} + title={language === 'en' ? 'Use AI to analyze this result' : '使用 AI 分析该结果'} + > + {language === 'en' ? 'AI Diagnose' : 'AI 诊断'} + </button> + <label className="dinsar-toggle-label" title={language === 'en' ? 'Show or hide on map' : '在地图上显示或隐藏'}> + <input + type="checkbox" + checked={!!result.isVisible} + onChange={(event) => { + event.stopPropagation(); + onToggleVisibility(result.id); + }} + /> + <span>{language === 'en' ? 'Map' : '地图'}</span> + </label> + </div> + </div> + </li> + ); +} + +export default memo(DinsarResultRow); diff --git a/frontend/src/config/appConstants.js b/frontend/src/config/appConstants.js index 4640780..c0cc6f0 100644 --- a/frontend/src/config/appConstants.js +++ b/frontend/src/config/appConstants.js @@ -51,9 +51,54 @@ export const NATIONAL_BOUNDARY_GEOJSON_URL = buildTileServerUrl('/geojson/全国 export const getBaseLayerConfig = key => BASE_LAYERS[key] || BASE_LAYERS[TILE_LAYER_DEFAULT_KEY]; +export const PRODUCTION_WORKSPACE_TAB = 'production_management'; +export const PRODUCTION_WORKSPACE_LEGACY_TABS = [ + 'dinsar_production', + 'dinsar_products', + 'ps_production', + 'ps_products', +]; + +export const PRODUCTION_WORKSPACE_VIEWS = [ + { + key: 'dinsar_runs', + label: 'D-InSAR 运行', + description: '运行任务编排、引擎切换与过程监控', + }, + { + key: 'timeseries_runs', + label: '时序InSAR 运行', + description: '当前默认接入 SBAS 流程，后续可扩展 PS-InSAR / SBAS-InSAR', + }, + { + key: 'dinsar_products', + label: 'D-InSAR 产物', + description: '结果提取、标准目录发布与产物编目', + }, + { + key: 'timeseries_products', + label: '时序InSAR 产物', + description: '当前统一登记时序产物，后续兼容多类型时序成果', + }, +]; + +export const PRODUCTION_WORKSPACE_ENTRY_TO_VIEW = Object.freeze({ + [PRODUCTION_WORKSPACE_TAB]: 'dinsar_runs', + dinsar_production: 'dinsar_runs', + dinsar_products: 'dinsar_products', + ps_production: 'timeseries_runs', + ps_products: 'timeseries_products', +}); + +export const PRODUCTION_WORKSPACE_ROUTE_TABS = new Set([ + PRODUCTION_WORKSPACE_TAB, + ...PRODUCTION_WORKSPACE_LEGACY_TABS, +]); + export const LEFT_GROUP_LABELS = { data: '数据管理', - production: '生产规划', + production_planning: '生产规划', + production_management: '生产管理', insar_analysis: 'InSAR形变分析', ai_analysis: 'AI分析', water: '水体监测', @@ -61,7 +106,7 @@ export const LEFT_GROUP_LABELS = { }; export const LEFT_GROUP_SECTIONS = { - production: [ + production_planning: [ { key: 'planning', label: '规划编组', @@ -72,16 +117,6 @@ export const LEFT_GROUP_SECTIONS = { label: '数据分发', tabs: ['copier'], }, - { - key: 'dinsar', - label: 'D-InSAR', - tabs: ['dinsar_production', 'dinsar_products'], - }, - { - key: 'psinsar', - label: 'PS-InSAR', - tabs: ['ps_production', 'ps_products'], - }, ], insar_analysis: [ { @@ -91,7 +126,7 @@ export const LEFT_GROUP_SECTIONS = { }, { key: 'psinsar', - label: 'PS-InSAR', + label: '时序InSAR', tabs: ['psinsar_results', 'psinsar_analysis'], }, ], @@ -111,7 +146,8 @@ export const LEFT_GROUP_SECTIONS = { export const LEFT_GROUP_TABS = { data: ['ingest', 'data', 'hazard'], - production: LEFT_GROUP_SECTIONS.production.flatMap(section => section.tabs), + production_planning: LEFT_GROUP_SECTIONS.production_planning.flatMap(section => section.tabs), + production_management: [PRODUCTION_WORKSPACE_TAB], insar_analysis: LEFT_GROUP_SECTIONS.insar_analysis.flatMap(section => section.tabs), ai_analysis: LEFT_GROUP_SECTIONS.ai_analysis.flatMap(section => section.tabs), water: ['water'], @@ -125,6 +161,10 @@ export const LEFT_TAB_GROUP = Object.entries(LEFT_GROUP_TABS).reduce((acc, [grou return acc; }, {}); +PRODUCTION_WORKSPACE_LEGACY_TABS.forEach(tab => { + LEFT_TAB_GROUP[tab] = 'production_management'; +}); + export const LEFT_TAB_SECTION = Object.entries(LEFT_GROUP_SECTIONS).reduce((acc, [, sections]) => { sections.forEach(section => { section.tabs.forEach(tab => { @@ -134,6 +174,10 @@ export const LEFT_TAB_SECTION = Object.entries(LEFT_GROUP_SECTIONS).reduce((acc, return acc; }, {}); +export const FULL_WIDTH_LEFT_TABS = new Set([ + ...PRODUCTION_WORKSPACE_ROUTE_TABS, +]); + export const ADMIN_ONLY_TABS = new Set([ 'ingest', 'pairing', @@ -141,10 +185,8 @@ export const ADMIN_ONLY_TABS = new Set([ 'ps_results', 'batches', 'copier', - 'dinsar_production', - 'dinsar_products', - 'ps_production', - 'ps_products', + PRODUCTION_WORKSPACE_TAB, + ...PRODUCTION_WORKSPACE_LEGACY_TABS, 'water', 'users', 'audit', diff --git a/frontend/src/hooks/useDinsarOperations.js b/frontend/src/hooks/useDinsarOperations.js index c3c212a..4596f44 100644 --- a/frontend/src/hooks/useDinsarOperations.js +++ b/frontend/src/hooks/useDinsarOperations.js @@ -11,6 +11,8 @@ import { normalizePagePayload } from '../utils/appHelpers'; import { normalizeTaskStatus } from '../utils/appUiHelpers'; import { DEFAULT_LIST_PAGE_SIZE } from '../config/appConstants'; +const NON_BLOCKING_TASK_TYPES = new Set(['UNPACK_ARCHIVES']); + export default function useDinsarOperations({ onCleanupDinsarLayers, fetchRadarImagingDates, @@ -24,7 +26,7 @@ export default function useDinsarOperations({ setAiStatus, setActiveAiReport, } = useDinsarStore(); const { setHazardPoints } = useHazardStore(); - const { setPendingTaskIds, setIsGlobalLocked } = useTaskStore(); + const { setPendingTaskIds, setNonBlockingTaskIds, setIsGlobalLocked } = useTaskStore(); const { currentUser } = useAuthStore(); const { hasRadarSearched, radarPagination, @@ -120,17 +122,49 @@ export default function useDinsarOperations({ } }; - const handleTaskStart = (taskId, message) => { + const handleTaskStart = (taskId, message, options = {}) => { + const taskType = String(options?.taskType || '').trim().toUpperCase(); + const isNonBlocking = !!options?.nonBlocking || NON_BLOCKING_TASK_TYPES.has(taskType); if (taskId) { setPendingTaskIds(prev => [...prev, taskId]); + if (isNonBlocking) { + setNonBlockingTaskIds(prev => [...new Set([...prev, taskId])]); + } + } + if (!isNonBlocking) { + setIsGlobalLocked(true); } - setIsGlobalLocked(true); if (message) addLog('info', message); }; const handleTaskCompletion = (taskInfo) => { console.log("收到任务完成通知:", taskInfo); const taskStatus = normalizeTaskStatus(taskInfo?.status); + const syncRadarViewsAfterUnpack = async () => { + try { + await Promise.all([ + fetchRadarImagingDates(), + fetchRadarSearchOptions(), + ]); + if (hasRadarSearched) { + const requestId = radarSearchRequestSeqRef.current + 1; + radarSearchRequestSeqRef.current = requestId; + await fetchAllData({ + limit: radarPagination.limit, + offset: radarPagination.offset, + criteria: radarSearchApplied, + aoiMode: radarSearchAppliedAoiMode, + regionTreeId: radarSearchAppliedRegionTreeId, + aoiToken: radarSearchAoiToken, + files: null, + requestId, + }); + } + } catch (error) { + console.error('解包完成后刷新 LT-1 视图失败:', error); + addLog('warn', 'LT-1 解包已完成，但刷新数据视图时发生错误，请手动刷新。'); + } + }; if (taskInfo.task_type === 'AI_ANALYZE') { if (taskInfo.message) { @@ -187,6 +221,14 @@ export default function useDinsarOperations({ } else if (taskStatus === 'FAILED') { addLog('error', `灾害点同步失败: ${taskInfo.message || '未知错误'}`); } + } else if (taskInfo.task_type === 'UNPACK_ARCHIVES') { + if (taskStatus === 'COMPLETED') { + addLog('success', taskInfo.message || 'LT-1 解包完成。'); + addLog('info', '正在同步 LT-1 解包后的数据视图...'); + void syncRadarViewsAfterUnpack(); + } else if (taskStatus === 'FAILED') { + addLog('error', `LT-1 解包失败: ${taskInfo.message || '未知错误'}`); + } } }; diff --git a/frontend/src/hooks/useGlobalTaskControl.js b/frontend/src/hooks/useGlobalTaskControl.js index 4cb66d7..38b5134 100644 --- a/frontend/src/hooks/useGlobalTaskControl.js +++ b/frontend/src/hooks/useGlobalTaskControl.js @@ -2,6 +2,13 @@ import { useCallback, useEffect, useRef, useState } from 'react'; import apiClient from '../api/client'; import { normalizeTaskStatus } from '../utils/appUiHelpers'; +const NON_BLOCKING_TASK_TYPES = new Set(['UNPACK_ARCHIVES']); + +const isTaskNonBlocking = (taskId, taskType, nonBlockingTaskIds = []) => ( + NON_BLOCKING_TASK_TYPES.has(String(taskType || '').toUpperCase()) + || nonBlockingTaskIds.includes(taskId) +); + export default function useGlobalTaskControl({ currentUser, licenseOk, @@ -9,6 +16,8 @@ export default function useGlobalTaskControl({ setActiveTasks, pendingTaskIds, setPendingTaskIds, + nonBlockingTaskIds, + setNonBlockingTaskIds, isGlobalLocked, setIsGlobalLocked, setIsCheckingTasks, @@ -31,6 +40,8 @@ export default function useGlobalTaskControl({ // Stable refs so SSE handler doesn't need to re-subscribe on every render const pendingTaskIdsRef = useRef(pendingTaskIds); useEffect(() => { pendingTaskIdsRef.current = pendingTaskIds; }, [pendingTaskIds]); + const nonBlockingTaskIdsRef = useRef(nonBlockingTaskIds); + useEffect(() => { nonBlockingTaskIdsRef.current = nonBlockingTaskIds; }, [nonBlockingTaskIds]); const handleTasksUpdate = useCallback(async (tasks) => { setActiveTasks(tasks); @@ -41,14 +52,25 @@ export default function useGlobalTaskControl({ const currentPending = pendingTaskIdsRef.current; let updatedPending = currentPending; + let effectiveNonBlockingTaskIds = Array.from(new Set([ + ...nonBlockingTaskIdsRef.current, + ...tasks + .filter((task) => NON_BLOCKING_TASK_TYPES.has(String(task.task_type || '').toUpperCase())) + .map((task) => task.task_id), + ])); // 如果 pendingTaskIds 为空，但 activeTasks 有任务，说明是刷新后的初始化 // 需要将 activeTasks 中的任务添加到 pendingTaskIds if (currentPending.length === 0 && hasRunningTasks) { const activeTaskIds = tasks.map((t) => t.task_id); + const nextNonBlockingIds = tasks + .filter((task) => NON_BLOCKING_TASK_TYPES.has(String(task.task_type || '').toUpperCase())) + .map((task) => task.task_id); console.log('初始化：将活跃任务添加到 pending 列表:', activeTaskIds); setPendingTaskIds(activeTaskIds); + setNonBlockingTaskIds(nextNonBlockingIds); updatedPending = activeTaskIds; + effectiveNonBlockingTaskIds = nextNonBlockingIds; } if (currentPending.length > 0) { @@ -104,6 +126,8 @@ export default function useGlobalTaskControl({ updatedPending = newPending; return newPending; }); + setNonBlockingTaskIds((prev) => prev.filter((id) => !reallyFinishedIds.includes(id))); + effectiveNonBlockingTaskIds = effectiveNonBlockingTaskIds.filter((id) => !reallyFinishedIds.includes(id)); } else { // 没有真正完成的任务，保持 updatedPending 不变 updatedPending = currentPending; @@ -112,7 +136,13 @@ export default function useGlobalTaskControl({ } // 只有当没有运行中的任务且没有待处理的任务时，才解锁 - const shouldBeLocked = hasRunningTasks || updatedPending.length > 0; + const blockingRunningTasks = tasks.filter((task) => ( + !isTaskNonBlocking(task.task_id, task.task_type, effectiveNonBlockingTaskIds) + )); + const blockingPendingTaskIds = updatedPending.filter((taskId) => ( + !effectiveNonBlockingTaskIds.includes(taskId) + )); + const shouldBeLocked = blockingRunningTasks.length > 0 || blockingPendingTaskIds.length > 0; if (shouldBeLocked !== isGlobalLockedRef.current) { setIsGlobalLocked(shouldBeLocked); @@ -128,6 +158,7 @@ export default function useGlobalTaskControl({ setIsCheckingTasks, handleTaskCompletionRef, setPendingTaskIds, + setNonBlockingTaskIds, setIsGlobalLocked, addLog, initializeAppDataRef, diff --git a/frontend/src/hooks/usePairingLogic.js b/frontend/src/hooks/usePairingLogic.js index 723d5b3..ca752e4 100644 --- a/frontend/src/hooks/usePairingLogic.js +++ b/frontend/src/hooks/usePairingLogic.js @@ -59,13 +59,13 @@ export default function usePairingLogic({ name: `PS_${direction}_${new Date().toISOString().slice(0, 10)}` }); const batchId = response.data?.batch_id || ''; - addLog('success', `已创建 PS 批次: ${batchId || direction}`); + addLog('success', `已创建时序批次: ${batchId || direction}`); if (focusAfterCreate && batchId) { await focusBatchAfterCreate('ps', batchId); } } catch (error) { const errorMessage = error.response?.data?.detail || error.message || '未知错误'; - addLog('error', `PS 批次创建失败: ${errorMessage}`); + addLog('error', `时序批次创建失败: ${errorMessage}`); } }; @@ -97,7 +97,7 @@ export default function usePairingLogic({ setPsResults(null); onClearAoiLayer(); setAoiLayer(null); - addLog('info', 'PS-InSAR 结果已清空。'); + addLog('info', '时序InSAR 候选栈结果已清空。'); }; const findPairs = async (e, externalRequireOrbitRef) => { @@ -210,7 +210,7 @@ export default function usePairingLogic({ setShowPsModal(false); setIsLoading(true); - addLog('info', '开始准备PS时序数据栈...'); + addLog('info', '开始准备时序InSAR候选栈...'); const formData = new FormData(); for (const key in psParams) { @@ -258,7 +258,7 @@ export default function usePairingLogic({ setPsResults(processedResults); if (Object.keys(processedResults).length > 0) { - addLog('success', `成功找到 ${Object.keys(processedResults).length} 个PS时序栈。`); + addLog('success', `成功找到 ${Object.keys(processedResults).length} 个时序InSAR候选栈。`); setLeftPanelTab('ps_results'); for (const [direction, stack] of Object.entries(processedResults)) { await createPsBatch(direction, stack, { focusAfterCreate: false }); @@ -268,9 +268,9 @@ export default function usePairingLogic({ setLeftPanelTab('ps_results'); } } catch (error) { - console.error("PS时序准备失败:", error); + console.error('时序InSAR候选栈准备失败:', error); const errorMessage = error.response?.data?.detail || error.message || '未知错误'; - addLog('error', `PS时序准备失败: ${errorMessage}`); + addLog('error', `时序InSAR候选栈准备失败: ${errorMessage}`); } finally { setIsLoading(false); setPsFiles(null); diff --git a/frontend/src/i18n/translations.js b/frontend/src/i18n/translations.js index 22b1b60..9dec838 100644 --- a/frontend/src/i18n/translations.js +++ b/frontend/src/i18n/translations.js @@ -208,7 +208,7 @@ { zh: '取消', en: 'Cancel' }, // --- App.jsx: PS modal --- - { zh: '准备PS时序数据栈', en: 'Prepare PS Time-series Stack' }, + { zh: '准备时序InSAR候选栈', en: 'Prepare Time-series InSAR Stack' }, { zh: '研究区域来源:', en: 'Study Area Source:' }, { zh: '研究区域 (Shapefile):', en: 'Study Area (Shapefile):' }, { zh: '准备并导出', en: 'Prepare & Export' }, diff --git a/frontend/src/panels/DinsarResultPanel.jsx b/frontend/src/panels/DinsarResultPanel.jsx index 90ec3b3..1e51e89 100644 --- a/frontend/src/panels/DinsarResultPanel.jsx +++ b/frontend/src/panels/DinsarResultPanel.jsx @@ -7,6 +7,11 @@ import DinsarResultRow from '../components/panels/DinsarResultRow'; import ResultExportModal from '../components/ResultExportModal'; import { PAGE_SIZE_OPTIONS } from '../config/appConstants'; import { getPageHintText } from '../utils/appUiHelpers'; +import { + DINSAR_ENGINE_ALL, + buildDinsarEngineOptions, + getDinsarEngineMeta, +} from '../utils/dinsarEngines'; import { DINSAR_STRATEGY_ALL, buildDinsarStrategyOptions, @@ -14,8 +19,8 @@ import { } from '../utils/dinsarResultFilters'; const DINSAR_ROW_HEIGHT = { - compact: 110, - expanded: 138, + compact: 136, + expanded: 166, }; export default function DinsarResultPanel({ @@ -37,9 +42,11 @@ export default function DinsarResultPanel({ dinsarResults, dinsarPagination, scoreFilter, + engineFilter, traceSearch, strategyFilter, dinsarPageInput, + setEngineFilter, setTraceSearch, setStrategyFilter, setDinsarPageInput, @@ -48,9 +55,11 @@ export default function DinsarResultPanel({ dinsarResults: state.dinsarResults, dinsarPagination: state.dinsarPagination, scoreFilter: state.scoreFilter, + engineFilter: state.engineFilter, traceSearch: state.traceSearch, strategyFilter: state.strategyFilter, dinsarPageInput: state.dinsarPageInput, + setEngineFilter: state.setEngineFilter, setTraceSearch: state.setTraceSearch, setStrategyFilter: state.setStrategyFilter, setDinsarPageInput: state.setDinsarPageInput, @@ -73,6 +82,38 @@ export default function DinsarResultPanel({ () => buildDinsarStrategyOptions(dinsarResults), [dinsarResults] ); + const engineOptions = useMemo( + () => buildDinsarEngineOptions(dinsarResults), + [dinsarResults] + ); + const engineFilterOptions = useMemo( + () => [ + { + value: DINSAR_ENGINE_ALL, + label: language === 'en' ? 'All engines' : '全部引擎', + }, + ...engineOptions.map((option) => ({ + value: option.value, + label: option.label, + })), + ], + [engineOptions, language] + ); + const filteredEngineMeta = useMemo( + () => (engineFilter === DINSAR_ENGINE_ALL ? null : getDinsarEngineMeta(engineFilter)), + [engineFilter] + ); + const engineCounts = useMemo(() => { + const counts = new Map(); + dinsarResults.forEach((result) => { + const meta = getDinsarEngineMeta(result?.engine_code); + counts.set(meta.code, (counts.get(meta.code) || 0) + 1); + }); + return engineOptions.map((option) => ({ + ...option, + count: counts.get(option.code) || 0, + })); + }, [dinsarResults, engineOptions]); useEffect(() => { if (strategyFilter === DINSAR_STRATEGY_ALL) { @@ -83,17 +124,31 @@ export default function DinsarResultPanel({ } }, [setStrategyFilter, strategyFilter, strategyOptions]); + useEffect(() => { + if (engineFilter === DINSAR_ENGINE_ALL) { + return; + } + if (!engineOptions.some((option) => option.value === engineFilter)) { + setEngineFilter(DINSAR_ENGINE_ALL); + } + }, [engineFilter, engineOptions, setEngineFilter]); + const filteredResults = useMemo( () => filterDinsarResults(dinsarResults, { scoreFilter, + engineFilter, strategyFilter, traceSearch, focusedHazardPoint, }), - [dinsarResults, focusedHazardPoint, scoreFilter, strategyFilter, traceSearch] + [dinsarResults, engineFilter, focusedHazardPoint, scoreFilter, strategyFilter, traceSearch] ); + const scorePercent = Math.round(Number(scoreFilter || 0) * 100); const virtualRowHeight = showDates ? DINSAR_ROW_HEIGHT.expanded : DINSAR_ROW_HEIGHT.compact; + const pageSummaryText = language === 'en' + ? `Page ${dinsarCurrentPage}/${dinsarTotalPages} · ${dinsarPagination.total} total` + : `第 ${dinsarCurrentPage}/${dinsarTotalPages} 页 · 共 ${dinsarPagination.total} 条`; return ( <div className="panel-content"> diff --git a/frontend/src/panels/DinsarResultPanel.rewrite.jsx b/frontend/src/panels/DinsarResultPanel.rewrite.jsx new file mode 100644 index 0000000..edbb75c --- /dev/null +++ b/frontend/src/panels/DinsarResultPanel.rewrite.jsx @@ -0,0 +1,461 @@ +import { useEffect, useMemo, useState } from 'react'; +import { useShallow } from 'zustand/react/shallow'; +import { useDinsarStore, useHazardStore, useUiStore, useAuthStore } from '../store'; +import { useI18n } from '../i18n/I18nContext'; +import VirtualizedList from '../components/common/VirtualizedList'; +import DinsarResultRow from '../components/panels/DinsarResultRow.rewrite'; +import ResultExportModal from '../components/ResultExportModal.rewrite'; +import { PAGE_SIZE_OPTIONS } from '../config/appConstants'; +import { getPageHintText } from '../utils/appUiHelpers'; +import { + DINSAR_ENGINE_ALL, + buildDinsarEngineOptions, + getDinsarEngineMeta, +} from '../utils/dinsarEngines'; +import { + DINSAR_STRATEGY_ALL, + buildDinsarStrategyOptions, + filterDinsarResults, +} from '../utils/dinsarResultFilters'; + +const DINSAR_ROW_HEIGHT = { + compact: 136, + expanded: 166, +}; + +export default function DinsarResultPanel({ + dinsarCurrentPage, + dinsarTotalPages, + showDinsarPageInputError, + dinsarPageInputValidationError, + onSetAllVisibility, + onScoreFilterChange, + onPageChange, + onPageSizeChange, + onGoToPage, + onToggleVisibility, + onLabel, + onAnalyze, +}) { + const { language } = useI18n(); + const { + dinsarResults, + dinsarPagination, + scoreFilter, + engineFilter, + traceSearch, + strategyFilter, + dinsarPageInput, + setEngineFilter, + setTraceSearch, + setStrategyFilter, + setDinsarPageInput, + setDinsarPageInputTouched, + } = useDinsarStore(useShallow((state) => ({ + dinsarResults: state.dinsarResults, + dinsarPagination: state.dinsarPagination, + scoreFilter: state.scoreFilter, + engineFilter: state.engineFilter, + traceSearch: state.traceSearch, + strategyFilter: state.strategyFilter, + dinsarPageInput: state.dinsarPageInput, + setEngineFilter: state.setEngineFilter, + setTraceSearch: state.setTraceSearch, + setStrategyFilter: state.setStrategyFilter, + setDinsarPageInput: state.setDinsarPageInput, + setDinsarPageInputTouched: state.setDinsarPageInputTouched, + }))); + const { focusedHazardPoint, setFocusedHazardPoint } = useHazardStore(useShallow((state) => ({ + focusedHazardPoint: state.focusedHazardPoint, + setFocusedHazardPoint: state.setFocusedHazardPoint, + }))); + const { isLoading, showDates, setShowDates } = useUiStore(useShallow((state) => ({ + isLoading: state.isLoading, + showDates: state.showDates, + setShowDates: state.setShowDates, + }))); + const { currentUser } = useAuthStore(); + const isReadOnlyUser = !!currentUser && currentUser.role !== 'admin'; + const [showExportModal, setShowExportModal] = useState(false); + + const strategyOptions = useMemo( + () => buildDinsarStrategyOptions(dinsarResults), + [dinsarResults] + ); + const engineOptions = useMemo( + () => buildDinsarEngineOptions(dinsarResults), + [dinsarResults] + ); + const engineFilterOptions = useMemo( + () => [ + { + value: DINSAR_ENGINE_ALL, + label: language === 'en' ? 'All engines' : '全部引擎', + }, + ...engineOptions.map((option) => ({ + value: option.value, + label: option.label, + })), + ], + [engineOptions, language] + ); + const filteredEngineMeta = useMemo( + () => (engineFilter === DINSAR_ENGINE_ALL ? null : getDinsarEngineMeta(engineFilter)), + [engineFilter] + ); + const engineCounts = useMemo(() => { + const counts = new Map(); + dinsarResults.forEach((result) => { + const meta = getDinsarEngineMeta(result?.engine_code); + counts.set(meta.code, (counts.get(meta.code) || 0) + 1); + }); + return engineOptions.map((option) => ({ + ...option, + count: counts.get(option.code) || 0, + })); + }, [dinsarResults, engineOptions]); + + useEffect(() => { + if (strategyFilter === DINSAR_STRATEGY_ALL) { + return; + } + if (!strategyOptions.includes(strategyFilter)) { + setStrategyFilter(DINSAR_STRATEGY_ALL); + } + }, [setStrategyFilter, strategyFilter, strategyOptions]); + + useEffect(() => { + if (engineFilter === DINSAR_ENGINE_ALL) { + return; + } + if (!engineOptions.some((option) => option.value === engineFilter)) { + setEngineFilter(DINSAR_ENGINE_ALL); + } + }, [engineFilter, engineOptions, setEngineFilter]); + + const filteredResults = useMemo( + () => filterDinsarResults(dinsarResults, { + scoreFilter, + engineFilter, + strategyFilter, + traceSearch, + focusedHazardPoint, + }), + [dinsarResults, engineFilter, focusedHazardPoint, scoreFilter, strategyFilter, traceSearch] + ); + + const scorePercent = Math.round(Number(scoreFilter || 0) * 100); + const virtualRowHeight = showDates ? DINSAR_ROW_HEIGHT.expanded : DINSAR_ROW_HEIGHT.compact; + const pageSummaryText = language === 'en' + ? `Page ${dinsarCurrentPage}/${dinsarTotalPages} · ${dinsarPagination.total} total` + : `第 ${dinsarCurrentPage}/${dinsarTotalPages} 页 · 共 ${dinsarPagination.total} 条`; + + return ( + <div className="panel-content"> + {dinsarPagination.total === 0 ? ( + <p className="empty-state"> + {language === 'en' ? 'No D-InSAR results found.' : '未找到 D-InSAR 结果。'} + </p> + ) : ( + <> + <div className="list-toolbar column-layout dinsar-results-toolbar"> + {focusedHazardPoint && ( + <div className="filter-banner"> + <span> + {language === 'en' ? ( + <> + Viewing results covering <strong>{focusedHazardPoint.hazard_name}</strong> + </> + ) : ( + <> + 当前仅显示覆盖隐患点 <strong>{focusedHazardPoint.hazard_name}</strong> 的结果 + </> + )} + </span> + <button className="clear-filter-btn" onClick={() => setFocusedHazardPoint(null)}> + {language === 'en' ? 'Clear Filter' : '清除筛选'} + </button> + </div> + )} + + <div className="dinsar-toolbar-grid"> + <section className="dinsar-toolbar-panel"> + <span className="dinsar-toolbar-kicker"> + {language === 'en' ? 'Current page' : '当前页'} + </span> + <strong className="dinsar-toolbar-value"> + {filteredResults.length} / {dinsarResults.length} + </strong> + <p className="dinsar-toolbar-note"> + {language === 'en' + ? 'Results after local filtering on the current page' + : '当前页本地筛选后的结果数量'} + </p> + <div className="dinsar-toolbar-chip-row"> + <span className="dinsar-toolbar-chip"> + {language === 'en' ? 'AI score' : 'AI 分数'} + {' >= '} + {scorePercent} + </span> + <span className="dinsar-toolbar-chip"> + {language === 'en' ? 'Dates' : '日期'} + {showDates + ? (language === 'en' ? ': visible' : '：已展开') + : (language === 'en' ? ': hidden' : '：已收起')} + </span> + </div> + </section> + + <section className="dinsar-toolbar-panel"> + <span className="dinsar-toolbar-kicker"> + {language === 'en' ? 'Engine focus' : '当前引擎'} + </span> + <strong className="dinsar-toolbar-value"> + {filteredEngineMeta + ? filteredEngineMeta.shortLabel + : (language === 'en' ? 'All' : '全部')} + </strong> + <p className="dinsar-toolbar-note"> + {filteredEngineMeta + ? filteredEngineMeta.label + : (language === 'en' + ? 'Compare outputs from all registered engines' + : '同时查看所有登记引擎的结果')} + </p> + <div className="dinsar-toolbar-chip-row"> + <span className="dinsar-toolbar-chip"> + {language === 'en' ? 'Strategy' : '策略'}: + {' '} + {strategyFilter === DINSAR_STRATEGY_ALL + ? (language === 'en' ? 'All' : '全部') + : strategyFilter} + </span> + <span className="dinsar-toolbar-chip"> + {language === 'en' ? 'Trace search' : '检索词'}: + {' '} + {traceSearch.trim() || (language === 'en' ? 'None' : '未设置')} + </span> + </div> + </section> + + <section className="dinsar-toolbar-panel"> + <span className="dinsar-toolbar-kicker"> + {language === 'en' ? 'Page control' : '分页控制'} + </span> + <strong className="dinsar-toolbar-value">{pageSummaryText}</strong> + <p className="dinsar-toolbar-note"> + {language === 'en' + ? 'Use page size and jump controls below for large catalogs' + : '大规模结果集请结合页大小和跳页控制使用'} + </p> + <div className="dinsar-toolbar-actions"> + <button onClick={() => onSetAllVisibility(true)}> + {language === 'en' ? 'Show All' : '全部显示'} + </button> + <button onClick={() => onSetAllVisibility(false)}> + {language === 'en' ? 'Hide All' : '全部隐藏'} + </button> + <button onClick={() => setShowDates(!showDates)}> + {showDates + ? (language === 'en' ? 'Hide Dates' : '收起日期') + : (language === 'en' ? 'Show Dates' : '显示日期')} + </button> + <button + onClick={() => setShowExportModal(true)} + disabled={isLoading || filteredResults.length === 0} + title={language === 'en' + ? 'Export visible results in the current filter scope' + : '按当前筛选范围导出结果文件'} + > + {language === 'en' ? 'Export...' : '提取结果...'} + </button> + </div> + </section> + </div> + + <div className="dinsar-filter-layout"> + <label className="dinsar-filter-field"> + <span>{language === 'en' ? 'AI score floor' : 'AI 分数下限'}</span> + <div className="dinsar-score-filter"> + <input + type="range" + min="0" + max="1" + step="0.1" + value={scoreFilter} + onChange={onScoreFilterChange} + /> + <strong>{scorePercent}</strong> + </div> + </label> + + <label className="dinsar-filter-field"> + <span>{language === 'en' ? 'Pairing strategy' : '配对策略'}</span> + <select + value={strategyFilter} + onChange={(event) => setStrategyFilter(event.target.value)} + disabled={isLoading} + > + <option value={DINSAR_STRATEGY_ALL}> + {language === 'en' ? 'All strategies' : '全部策略'} + </option> + {strategyOptions + .filter((value) => value !== DINSAR_STRATEGY_ALL) + .map((value) => ( + <option key={value} value={value}>{value}</option> + ))} + </select> + </label> + + <label className="dinsar-filter-field"> + <span>{language === 'en' ? 'Production engine' : '生产引擎'}</span> + <select + value={engineFilter} + onChange={(event) => setEngineFilter(event.target.value)} + disabled={isLoading} + > + {engineFilterOptions.map((option) => ( + <option key={option.value} value={option.value}> + {option.label} + </option> + ))} + </select> + </label> + + <label className="dinsar-filter-field dinsar-filter-field-wide"> + <span>{language === 'en' ? 'Trace search' : 'Trace 检索'}</span> + <input + type="text" + value={traceSearch} + onChange={(event) => setTraceSearch(event.target.value)} + placeholder={language === 'en' + ? 'Search pair / run / policy / engine' + : '搜索 pair / run / policy / engine'} + disabled={isLoading} + /> + </label> + </div> + + <div className="dinsar-engine-filter-row"> + <button + type="button" + className={engineFilter === DINSAR_ENGINE_ALL ? 'active' : ''} + onClick={() => setEngineFilter(DINSAR_ENGINE_ALL)} + disabled={isLoading} + > + <span>{language === 'en' ? 'All engines' : '全部引擎'}</span> + <strong>{dinsarResults.length}</strong> + </button> + {engineCounts.map((option) => ( + <button + key={option.code} + type="button" + className={engineFilter === option.code ? 'active' : ''} + onClick={() => setEngineFilter(option.code)} + disabled={isLoading} + > + <span>{option.shortLabel}</span> + <strong>{option.count}</strong> + </button> + ))} + </div> + + <div className="dinsar-toolbar-footer"> + <div className="dinsar-toolbar-footer-main"> + <button + type="button" + onClick={() => onPageChange(-1)} + disabled={isLoading || dinsarPagination.offset <= 0} + > + {language === 'en' ? 'Previous' : '上一页'} + </button> + <button + type="button" + onClick={() => onPageChange(1)} + disabled={isLoading || !dinsarPagination.hasMore} + > + {language === 'en' ? 'Next' : '下一页'} + </button> + + <label className="dinsar-pagination-field"> + <span>{language === 'en' ? 'Per page' : '每页条数'}</span> + <select + value={dinsarPagination.limit} + onChange={onPageSizeChange} + disabled={isLoading} + > + {PAGE_SIZE_OPTIONS.map((size) => ( + <option key={size} value={size}>{size}</option> + ))} + </select> + </label> + + <label className="dinsar-pagination-field dinsar-pagination-field-jump"> + <span>{language === 'en' ? 'Jump to page' : '跳转页码'}</span> + <div className="dinsar-page-jump-input"> + <input + type="number" + min={1} + max={dinsarTotalPages} + value={dinsarPageInput} + onChange={(event) => { + setDinsarPageInput(event.target.value); + setDinsarPageInputTouched(false); + }} + onBlur={() => setDinsarPageInputTouched(true)} + onKeyDown={(event) => { + if (event.key === 'Enter') { + event.preventDefault(); + onGoToPage(); + } + }} + disabled={isLoading} + className={showDinsarPageInputError ? 'has-error' : ''} + /> + <button type="button" onClick={onGoToPage} disabled={isLoading}> + {language === 'en' ? 'Jump' : '跳转'} + </button> + </div> + </label> + </div> + + <div className={`dinsar-toolbar-hint ${showDinsarPageInputError ? 'error' : ''}`}> + {showDinsarPageInputError + ? dinsarPageInputValidationError + : getPageHintText(dinsarTotalPages, language)} + </div> + </div> + </div> + + <div className="panel-scroll-shell"> + <VirtualizedList + items={filteredResults} + itemHeight={virtualRowHeight} + getKey={(result) => result.id} + renderItem={(result, index, key) => ( + <DinsarResultRow + key={key || `${result.id}-${index}`} + result={result} + language={language} + showDates={showDates} + isLoading={isLoading} + isReadOnlyUser={isReadOnlyUser} + onLabel={onLabel} + onAnalyze={onAnalyze} + onToggleVisibility={onToggleVisibility} + /> + )} + /> + </div> + </> + )} + + {showExportModal && ( + <ResultExportModal + results={filteredResults} + onClose={() => setShowExportModal(false)} + /> + )} + </div> + ); +} diff --git a/frontend/src/panels/PsResultsPanel.jsx b/frontend/src/panels/PsResultsPanel.jsx index 1872b2b..27edfe7 100644 --- a/frontend/src/panels/PsResultsPanel.jsx +++ b/frontend/src/panels/PsResultsPanel.jsx @@ -18,7 +18,7 @@ function PsResultsPanel({ return ( <div className="panel-content panel-scroll-shell"> {psStacks.length === 0 ? ( - <p className="empty-state">未找到PS时序栈。</p> + <p className="empty-state">未找到时序InSAR候选栈。</p> ) : ( <> <div className="list-toolbar"> diff --git a/frontend/src/store/dinsarStore.js b/frontend/src/store/dinsarStore.js index 87e5cb4..f9e4aff 100644 --- a/frontend/src/store/dinsarStore.js +++ b/frontend/src/store/dinsarStore.js @@ -1,4 +1,5 @@ import { create } from 'zustand'; +import { DINSAR_ENGINE_ALL } from '../utils/dinsarEngines'; const s = (set, key) => (v) => set((state) => ({ [key]: typeof v === 'function' ? v(state[key]) : v })); @@ -12,6 +13,7 @@ export const useDinsarStore = create((set) => ({ dinsarPageInputTouched: false, aiStatus: null, scoreFilter: 0, + engineFilter: DINSAR_ENGINE_ALL, traceSearch: '', strategyFilter: '__ALL__', activeAiReport: null, @@ -21,6 +23,7 @@ export const useDinsarStore = create((set) => ({ setDinsarPageInputTouched: s(set, 'dinsarPageInputTouched'), setAiStatus: s(set, 'aiStatus'), setScoreFilter: s(set, 'scoreFilter'), + setEngineFilter: s(set, 'engineFilter'), setTraceSearch: s(set, 'traceSearch'), setStrategyFilter: s(set, 'strategyFilter'), setActiveAiReport: s(set, 'activeAiReport'), diff --git a/frontend/src/store/taskStore.js b/frontend/src/store/taskStore.js index 38917fa..93d008d 100644 --- a/frontend/src/store/taskStore.js +++ b/frontend/src/store/taskStore.js @@ -8,8 +8,10 @@ export const useTaskStore = create((set) => ({ isGlobalLocked: false, isCheckingTasks: true, // 初始化时假设正在检查任务，避免闪烁 pendingTaskIds: [], + nonBlockingTaskIds: [], setActiveTasks: s(set, 'activeTasks'), setIsGlobalLocked: s(set, 'isGlobalLocked'), setIsCheckingTasks: s(set, 'isCheckingTasks'), setPendingTaskIds: s(set, 'pendingTaskIds'), + setNonBlockingTaskIds: s(set, 'nonBlockingTaskIds'), })); diff --git a/frontend/src/utils/appUiHelpers.js b/frontend/src/utils/appUiHelpers.js index 93a6a10..21c5016 100644 --- a/frontend/src/utils/appUiHelpers.js +++ b/frontend/src/utils/appUiHelpers.js @@ -42,29 +42,31 @@ export const getLeftTabLabel = (tabKey, metrics = {}) => { case 'pairs': return `任务规划 (${pairCount})`; case 'ps_results': - return `PS时序栈 (${psResultCount})`; + return `时序候选栈 (${psResultCount})`; case 'batches': return '任务批次'; case 'copier': return '数据分发'; + case 'production_management': + return '生产管理'; case 'idl': return 'D-InSAR生产（旧）'; case 'dinsar_production': - return 'D-InSAR生产'; + return 'D-InSAR运行'; case 'dinsar_products': return 'D-InSAR产物'; case 'ps_production': - return 'PS-InSAR生产'; + return '时序InSAR运行'; case 'ps_products': - return 'PS-InSAR产物'; + return '时序InSAR产物'; case 'dinsar_results': return `D-InSAR结果 (${dinsarTotal})`; case 'dinsar_analysis': return 'D-InSAR分析'; case 'psinsar_results': - return 'PS-InSAR结果'; + return '时序InSAR结果'; case 'psinsar_analysis': - return 'PS-InSAR分析'; + return '时序InSAR分析'; case 'ai_quality': return 'AI质量评估'; case 'ai_diagnosis': diff --git a/frontend/src/utils/dinsarEngines.js b/frontend/src/utils/dinsarEngines.js new file mode 100644 index 0000000..cbc9a23 --- /dev/null +++ b/frontend/src/utils/dinsarEngines.js @@ -0,0 +1,83 @@ +export const DINSAR_ENGINE_ALL = '__ALL__'; + +export const KNOWN_DINSAR_ENGINE_CODES = ['sarscape', 'envi', 'isce2', 'pyint', 'landsar']; + +const DINSAR_ENGINE_META = { + sarscape: { + label: 'ENVI / SARscape', + shortLabel: 'ENVI', + tone: 'envi', + }, + envi: { + label: 'Legacy ENVI', + shortLabel: 'ENVI-L', + tone: 'envi', + }, + isce2: { + label: 'ISCE2', + shortLabel: 'ISCE2', + tone: 'isce2', + }, + pyint: { + label: 'PyINT / Gamma', + shortLabel: 'PyINT', + tone: 'pyint', + }, + landsar: { + label: 'LandSAR', + shortLabel: 'LandSAR', + tone: 'landsar', + }, +}; + +function normalizeEngineCode(value) { + return String(value || '').trim().toLowerCase(); +} + +export function getDinsarEngineMeta(engineCode) { + const normalizedCode = normalizeEngineCode(engineCode); + const matched = DINSAR_ENGINE_META[normalizedCode]; + if (matched) { + return { + code: normalizedCode, + ...matched, + }; + } + return { + code: normalizedCode || 'unknown', + label: normalizedCode || 'Unknown', + shortLabel: normalizedCode || 'Unknown', + tone: 'unknown', + }; +} + +export function buildDinsarEngineOptions(results = [], { includeKnown = false } = {}) { + const codes = new Set(includeKnown ? KNOWN_DINSAR_ENGINE_CODES : []); + + (Array.isArray(results) ? results : []).forEach((result) => { + const normalizedCode = normalizeEngineCode(result?.engine_code); + if (normalizedCode) { + codes.add(normalizedCode); + } + }); + + return Array.from(codes) + .sort((left, right) => { + const leftIndex = KNOWN_DINSAR_ENGINE_CODES.indexOf(left); + const rightIndex = KNOWN_DINSAR_ENGINE_CODES.indexOf(right); + if (leftIndex >= 0 && rightIndex >= 0) { + return leftIndex - rightIndex; + } + if (leftIndex >= 0) { + return -1; + } + if (rightIndex >= 0) { + return 1; + } + return left.localeCompare(right); + }) + .map((code) => ({ + value: code, + ...getDinsarEngineMeta(code), + })); +} diff --git a/frontend/src/utils/dinsarResultFilters.js b/frontend/src/utils/dinsarResultFilters.js index 9affd41..7a21d2a 100644 --- a/frontend/src/utils/dinsarResultFilters.js +++ b/frontend/src/utils/dinsarResultFilters.js @@ -1,3 +1,5 @@ +import { DINSAR_ENGINE_ALL } from './dinsarEngines'; + export const DINSAR_STRATEGY_ALL = '__ALL__'; function normalizeTraceSearch(value) { @@ -15,10 +17,12 @@ function buildTraceText(result = {}) { result.task_name, result.pair_key, result.pair_uid, + result.run_key, result.network_run_id, result.network_edge_id, result.policy_version, result.selection_strategy, + result.engine_code, ] .filter(Boolean) .join(' ') @@ -71,19 +75,22 @@ export function matchesDinsarResultFilters( result, { scoreFilter = 0, + engineFilter = DINSAR_ENGINE_ALL, strategyFilter = DINSAR_STRATEGY_ALL, traceSearch = '', focusedHazardPoint = null, } = {} ) { const matchesScore = !hasAiScore(result) || Number(result.ai_score) >= Number(scoreFilter || 0); + const engineValue = String(result?.engine_code || '').trim().toLowerCase(); + const matchesEngine = engineFilter === DINSAR_ENGINE_ALL || engineValue === engineFilter; const strategyValue = String(result?.selection_strategy || '').trim(); const matchesStrategy = strategyFilter === DINSAR_STRATEGY_ALL || strategyValue === strategyFilter; const normalizedTraceSearch = normalizeTraceSearch(traceSearch); const matchesTrace = !normalizedTraceSearch || buildTraceText(result).includes(normalizedTraceSearch); const matchesHazard = matchesFocusedHazardPoint(result, focusedHazardPoint); - return matchesScore && matchesStrategy && matchesTrace && matchesHazard; + return matchesScore && matchesEngine && matchesStrategy && matchesTrace && matchesHazard; } export function filterDinsarResults(results = [], filters = {}) { diff --git a/scripts/unpack_archives_parallel.py b/scripts/unpack_archives_parallel.py index ebe62c2..40b3996 100644 --- a/scripts/unpack_archives_parallel.py +++ b/scripts/unpack_archives_parallel.py @@ -4,6 +4,7 @@ import os import shutil import tarfile import threading +import time from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait from datetime import datetime @@ -86,6 +87,14 @@ def _default_extract_workers(): return max(1, min(cpu, 8)) +def _format_limit_reason(reason, limit_value): + if reason == "max_files_per_run": + return f"reached max files per run: {limit_value}" + if reason == "max_runtime_minutes": + return f"reached max runtime: {limit_value} minutes" + return str(reason or "stopped") + + def get_disk_usage(path): try: os.makedirs(path, exist_ok=True) @@ -528,7 +537,7 @@ def _configure_logging(): logging.warning("failed to open activity log '%s': %s", ACTIVITY_LOG, file_handler_error) -def run_unpack_job(env_path=None, log_callback=None, progress_callback=None): +def run_unpack_job(env_path=None, log_callback=None, progress_callback=None, config_overrides=None): def _log(level, message, *args): logging.log(level, message, *args) if log_callback: @@ -542,6 +551,7 @@ def run_unpack_job(env_path=None, log_callback=None, progress_callback=None): _configure_logging() env = load_env(env_path or ENV_PATH) + config_overrides = config_overrides if isinstance(config_overrides, dict) else {} source_dirs = parse_dirs(env.get("UNPACK_SOURCE_DIRS")) target_dirs = parse_dirs( env.get("INSAR_STORAGE_DIRS") @@ -564,8 +574,25 @@ def run_unpack_job(env_path=None, log_callback=None, progress_callback=None): minimum=1, maximum=32, ) + max_files_per_run = parse_int( + config_overrides.get("max_files_per_run", env.get("UNPACK_MAX_FILES_PER_RUN")), + default=0, + minimum=0, + ) + max_runtime_minutes = parse_int( + config_overrides.get("max_runtime_minutes", env.get("UNPACK_MAX_RUNTIME_MINUTES")), + default=0, + minimum=0, + ) _log(logging.INFO, "=== start unpack job ===") + if "max_files_per_run" in config_overrides or "max_runtime_minutes" in config_overrides: + _log( + logging.INFO, + "run overrides: max_files_per_run=%s, max_runtime_minutes=%s", + max_files_per_run, + max_runtime_minutes, + ) if not source_dirs: _log(logging.INFO, "no UNPACK_SOURCE_DIRS configured, exit") @@ -590,6 +617,7 @@ def run_unpack_job(env_path=None, log_callback=None, progress_callback=None): all_archives = find_archives(source_dirs, extensions, workers=scan_workers, log_fn=_log) files_to_process = [archive_path for archive_path in all_archives if archive_path not in processed_files] + total_pending_files = len(files_to_process) _log( logging.INFO, @@ -609,16 +637,32 @@ def run_unpack_job(env_path=None, log_callback=None, progress_callback=None): "message": "nothing to do", } + if max_files_per_run > 0 and len(files_to_process) > max_files_per_run: + _log( + logging.INFO, + "apply UNPACK_MAX_FILES_PER_RUN=%s, this run will process the first %s pending archives", + max_files_per_run, + max_files_per_run, + ) + files_to_process = files_to_process[:max_files_per_run] + min_space_bytes = min_disk_gb * (1024 ** 3) reservation_manager = DiskReservationManager(min_space_bytes) total_files = len(files_to_process) + remaining_backlog_count = max(0, total_pending_files - total_files) processed_count = 0 failed_count = 0 skipped_count = 0 completed_count = 0 stop_reason = None + stop_reason_limit = None jobs = list(enumerate(files_to_process, start=1)) + started_at = time.monotonic() + max_runtime_seconds = max_runtime_minutes * 60 + + def _runtime_limit_reached(): + return max_runtime_seconds > 0 and (time.monotonic() - started_at) >= max_runtime_seconds _progress(0, f"processing 0/{total_files}") @@ -688,6 +732,11 @@ def run_unpack_job(env_path=None, log_callback=None, progress_callback=None): pct = int((completed_count / max(total_files, 1)) * 100) _progress(pct, f"processed {completed_count}/{total_files}") + if not stop_reason and next_job_index < total_files and _runtime_limit_reached(): + stop_reason_limit = max_runtime_minutes + stop_reason = _format_limit_reason("max_runtime_minutes", max_runtime_minutes) + _log(logging.INFO, "stop scheduling new archives: %s", stop_reason) + while ( next_job_index < total_files and len(active_futures) < extract_workers @@ -712,15 +761,21 @@ def run_unpack_job(env_path=None, log_callback=None, progress_callback=None): active_futures[future] = (archive_index, archive_path) next_job_index += 1 + if not stop_reason and remaining_backlog_count > 0 and max_files_per_run > 0: + stop_reason_limit = max_files_per_run + stop_reason = _format_limit_reason("max_files_per_run", max_files_per_run) + if stop_reason: - remaining_count = max(0, total_files - completed_count) + remaining_count = remaining_backlog_count + max(0, total_files - completed_count) create_report(REPORT_FILE, stop_reason, processed_count, remaining_count) return { "processed": processed_count, "failed": failed_count, "skipped": skipped_count, "total": total_files, - "message": "insufficient free space", + "remaining": remaining_count, + "limit_value": stop_reason_limit, + "message": stop_reason, } if os.path.exists(REPORT_FILE): @@ -732,6 +787,7 @@ def run_unpack_job(env_path=None, log_callback=None, progress_callback=None): "failed": failed_count, "skipped": skipped_count, "total": total_files, + "remaining": 0, "message": "completed", } diff --git a/third_party/PyINT/LICENSE b/third_party/PyINT/LICENSE new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/third_party/PyINT/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + <program> Copyright (C) <year> <name of author> + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +<https://www.gnu.org/licenses/>. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +<https://www.gnu.org/licenses/why-not-lgpl.html>. diff --git a/third_party/PyINT/README.md b/third_party/PyINT/README.md new file mode 100644 index 0000000..440ac43 --- /dev/null +++ b/third_party/PyINT/README.md @@ -0,0 +1,99 @@ +# PyINT +[![Language](https://img.shields.io/badge/python-3.5%2B-blue.svg)](https://www.python.org/) +[![License](https://img.shields.io/badge/license-GPL-yellow.svg)](https://github.com/ymcmrs/PyINT/blob/master/LICENSE) + +### Single or time-series of interferograms processing based on python and GAMMA for all of the present SAR datasets. + +PYthon-based INterferometry Toolbox (PyINT) is an open-source package for single or time-series of interferograms processing from downloading data (or SLC) to generating differential-unwrapped interferograms by using GAMMA software. You can process in a routine way (e.g., raw2ifg.py, slc2ifg.py or pyintApp.py) or process step by step. There are many GAMMA-independent tools of PyINT could be useful for you no matter you use GAMMA or other interferometry softwares (e.g., ISCE, SNAP). Advantages include (but not limited to) download and update precise-orbit data automatically (support S1, ERS, ASAR), download and process 30m-SRTM dem automatically, cat multi-frames automatically, select swaths and bursts flexibly (for S1), extract the related S1 butsts for Coregistration automatically, etc. Welcome to contribute/improve PyINT. + + +### 1 Download + +Download the development version using git: + + cd ~/python + git clone https://github.com/ymcmrs/PyINT + + +### 2 Installation + + 1） To make pyint importable in python, by adding the path PyINT directory to your $PYTHONPATH + For csh/tcsh user, add to your **_~/.cshrc_** file for example: + + ############################ Python ############################### + if ( ! $?PYTHONPATH ) then + setenv PYTHONPATH "" + endif + + ##--------- Anaconda ---------------## + setenv PYTHON3DIR ~/python/anaconda3 + setenv PATH ${PATH}:${PYTHON3DIR}/bin + + ##--------- PyINT ------------------## + setenv PYINT_HOME ~/python/PyINT + setenv PYTHONPATH ${PYTHONPATH}:${PYINT_HOME} + setenv PATH ${PATH}:${PYINT_HOME}/pyint + + 2） Install gdal, elevation module using pip or conda for DEM processing. + + 3） Install [SSARA](https://github.com/bakerunavco/SSARA) and set account info for downloading data. [option] + + +### 3 Running PyINT + +1). $SCRATCHDIR and $TEMPLATEDIR should be available in your system environment. $SCRATCHDIR for processing, $TEMPLATEDIR for template files to set the related processing parameters, $DEMDIR for saving DEMs: + + setenv SCRATCHDIR /Users/Yunmeng/Documents/SCRATCH + setenv TEMPLATEDIR /Users/Yunmeng/Documents/development/TEMPLATEDIR + setenv DEMDIR /Users/Yunmeng/Documents/SCRATCH/DEM + +2). Preparing your template file, which should be saved in $TEMPLATEDIR, for setting some basic parameters (see the template file above).The template file should be named with a prefix of your project name: + + e.g., MexicoCityT143F529S1D.template [Region + Track + Frame + Satellite + Orbit] + + +3). Single interferogram processing: + + slc2ifg.py projectName Mdate Sdate # start from SLC to unwrapped-differential Ifg + raw2ifg.py projectName Mdate Sdate # start from raw data to unwrapped-differential Ifg + + e.g. : + slc2ifg.py HawaiiT87F526S1D 20150101 20160201 + raw2ifg.py HawaiiT87F526S1D 20150101 20160201 + +4). Time-series of interferograms processing. + + pyintApp.py projectName + + e.g. : + pyintApp.py MexicoCityT143F529S1D # template file MexicoCityT143F529S1D.template should be availabe in TEMPLATEDIR + + General work-flow: + + 1) download data : download SLCs using SSARA (please check https://github.com/bakerunavco/SSARA) + [You should provide Sensor, Track, Frame, or Time information in template] + 2) generate SLC : raw 2 slc (multi-frame processing is also supported) + [include orbit correction for S1,ASAR,ERS and burst-extraction for S1] + 3) generate DEM : reference image related geo-dem, rdc-dem, lookup table will be generated. + [SRTM-1 will be downloaded and processed automatically if not provided] + 4) coregister SLC : coregister SLCs to the reference SLC iamge. + [with assistant of DEM] + 5) select pairs : select interferometric pairs for time-series processing. + [networks of sbas, sequential, delaunay, and stars are supported] + 6) interferometry : generate unwrapped differential interferograms. + [include differential, unwrapping, and geocoding] + 7) load data : loading data for time-series analysis, mintPy is supported presently. + + Note: + + i ) Single interferogram processing please use slc2ifg.py or raw2ifg.py + ii ) Multi-processor parallel processing is supported, but keep in mind GAMMA calls multi-threads already. + iii) You can using pyintApp.py from downloading data to generate time-series of unwrapped-differential Ifgs, + or you also can process step by step. + + +Note: All of the above codes are based on the hypothesis that you have installed [GAMMA](https://www.gamma-rs.ch/). + +### 4 Citing this work + + Y.M., Cao, "PyINT: Python&GAMMA based interferometry toolbox", Remote Sensing Code Library, doi:10.21982/vd48-7p51, April, 2019. diff --git a/third_party/PyINT/VENDORED_FROM.md b/third_party/PyINT/VENDORED_FROM.md new file mode 100644 index 0000000..a8c3232 --- /dev/null +++ b/third_party/PyINT/VENDORED_FROM.md @@ -0,0 +1,12 @@ +# Vendored PyINT + +- Source copied from local path: `D:\Code\PyINT` +- Vendored into this repository on: `2026-04-19` +- Scope kept in repo: `pyint/`, `template/`, `README.md`, `LICENSE` +- Excluded from vendored copy: `pyint_bk/`, `pyint.zip`, IDE metadata, `__pycache__` + +Notes: + +- This vendored copy is used by the local D-InSAR `PyINT / Gamma` integration. +- Runtime defaults now resolve `PYINT_HOME` to this directory when `.env` does not override it. +- If upstream `PyINT` is updated, sync this directory intentionally and record the source revision or date here. diff --git a/third_party/PyINT/pyint/20210110.slc.par b/third_party/PyINT/pyint/20210110.slc.par new file mode 100644 index 0000000..d571806 --- /dev/null +++ b/third_party/PyINT/pyint/20210110.slc.par @@ -0,0 +1,81 @@ +Gamma Interferometric SAR Processor (ISP) - Image Parameter File + +title: s1a-iw1-slc-vv-20210110t110734-20210110t110759-036075-043a78-004.tiff S1A-IW-IW1-VV-36075 (software: Sentinel-1 IPF 003.31) +sensor: S1A IW IW1 VV +date: 2021 01 10 11 07 33.60437 +start_time: 40053.604372 s +center_time: 40067.080599 s +end_time: 40080.556826 s +azimuth_line_time: 2.05555630e-03 s +line_header_size: 0 +range_samples: 68467 +azimuth_lines: 13113 +range_looks: 1 +azimuth_looks: 1 +image_format: FCOMPLEX +image_geometry: SLANT_RANGE +polarization: VV +range_scale_factor: 1.0000000e+00 +azimuth_scale_factor: 1.0000000e+00 +center_latitude: 25.7907364 degrees +center_longitude: 104.4737803 degrees +heading: -12.5014428 degrees +range_pixel_spacing: 2.3295620 m +azimuth_pixel_spacing: 13.9946500 m +near_range_slc: 798980.1369 m +center_range_slc: 878728.0328 m +far_range_slc: 958475.9288 m +first_slant_range_polynomial: 0.00000 0.00000 0.00000e+00 0.00000e+00 0.00000e+00 0.00000e+00 s m 1 m^-1 m^-2 m^-3 +center_slant_range_polynomial: 0.00000 0.00000 0.00000e+00 0.00000e+00 0.00000e+00 0.00000e+00 s m 1 m^-1 m^-2 m^-3 +last_slant_range_polynomial: 0.00000 0.00000 0.00000e+00 0.00000e+00 0.00000e+00 0.00000e+00 s m 1 m^-1 m^-2 m^-3 +incidence_angle: 39.6640 degrees +azimuth_deskew: ON +azimuth_angle: 90.0000 degrees +radar_frequency: 5.4050005e+09 Hz +adc_sampling_rate: 6.4345241e+07 Hz +chirp_bandwidth: 5.6500000e+07 Hz +prf: 486.4863103 Hz +azimuth_proc_bandwidth: 327.00000 Hz +doppler_polynomial: 17.39054 -2.62817e-05 -8.00473e-11 0.00000e+00 Hz Hz/m Hz/m^2 Hz/m^3 +doppler_poly_dot: 0.00000e+00 0.00000e+00 0.00000e+00 0.00000e+00 Hz/s Hz/s/m Hz/s/m^2 Hz/s/m^3 +doppler_poly_ddot: 0.00000e+00 0.00000e+00 0.00000e+00 0.00000e+00 Hz/s^2 Hz/s^2/m Hz/s^2/m^2 Hz/s^2/m^3 +receiver_gain: 0.0000 dB +calibration_gain: 0.0000 dB +sar_to_earth_center: 7072935.2710 m +earth_radius_below_sensor: 6374216.3707 m +earth_semi_major_axis: 6378137.0000 m +earth_semi_minor_axis: 6356752.3141 m +number_of_state_vectors: 15 +time_of_first_state_vector: 40002.000000 s +state_vector_interval: 10.000000 s +state_vector_position_1: -1202502.7493 6500810.8061 2515842.3001 m m m +state_vector_velocity_1: 2079.47309 -2309.53890 6932.46052 m/s m/s m/s +state_vector_position_2: -1181658.0158 6477336.1615 2585023.4928 m m m +state_vector_velocity_2: 2089.41624 -2385.34851 6903.64775 m/s m/s m/s +state_vector_position_3: -1160715.5736 6453104.6806 2653912.6562 m m m +state_vector_velocity_3: 2099.01470 -2460.90464 6874.05521 m/s m/s m/s +state_vector_position_4: -1139678.8732 6428118.9436 2722502.0093 m m m +state_vector_velocity_4: 2108.26777 -2536.19825 6843.68628 m/s m/s m/s +state_vector_position_5: -1118551.3718 6402381.6207 2790783.8056 m m m +state_vector_velocity_5: 2117.17479 -2611.22032 6812.54442 m/s m/s m/s +state_vector_position_6: -1097336.5330 6375895.4722 2858750.3336 m m m +state_vector_velocity_6: 2125.73515 -2685.96187 6780.63319 m/s m/s m/s +state_vector_position_7: -1076037.8261 6348663.3481 2926393.9175 m m m +state_vector_velocity_7: 2133.94831 -2760.41395 6747.95623 m/s m/s m/s +state_vector_position_8: -1054658.7259 6320688.1875 2993706.9187 m m m +state_vector_velocity_8: 2141.81376 -2834.56766 6714.51728 m/s m/s m/s +state_vector_position_9: -1033202.7115 6291973.0188 3060681.7361 m m m +state_vector_velocity_9: 2149.33106 -2908.41412 6680.32013 m/s m/s m/s +state_vector_position_10: -1011673.2665 6262520.9586 3127310.8074 m m m +state_vector_velocity_10: 2156.49982 -2981.94447 6645.36870 m/s m/s m/s +state_vector_position_11: -990073.8782 6232335.2122 3193586.6096 m m m +state_vector_velocity_11: 2163.31969 -3055.14991 6609.66698 m/s m/s m/s +state_vector_position_12: -968408.0367 6201419.0725 3259501.6599 m m m +state_vector_velocity_12: 2169.79039 -3128.02167 6573.21903 m/s m/s m/s +state_vector_position_13: -946679.2350 6169775.9199 3325048.5169 m m m +state_vector_velocity_13: 2175.91169 -3200.55102 6536.02901 m/s m/s m/s +state_vector_position_14: -924890.9683 6137409.2222 3390219.7811 m m m +state_vector_velocity_14: 2181.68339 -3272.72926 6498.10116 m/s m/s m/s +state_vector_position_15: -903046.7331 6104322.5337 3455008.0956 m m m +state_vector_velocity_15: 2187.10536 -3344.54774 6459.43982 m/s m/s m/s + diff --git a/third_party/PyINT/pyint/API_download_S1_SLC.py b/third_party/PyINT/pyint/API_download_S1_SLC.py new file mode 100644 index 0000000..9eb046a --- /dev/null +++ b/third_party/PyINT/pyint/API_download_S1_SLC.py @@ -0,0 +1,494 @@ +#! /usr/bin/env python3 +# -*- coding: utf-8 -*- + +########################################################################### +# Header information +########################################################################### + +"""API_download_S1_SLC.py: Script to download Sentinel-1 SLC images from the ASF mirror + Supports both bounding box and Shapefile for spatial filtering""" + +__author__ = "Alexis Hrysiewicz" +__copyright__ = "Copyright 2022" +__credits__ = ["Alexis Hrysiewicz"] +__license__ = "GPL" +__version__ = "2.0.0" +__maintainer__ = "Alexis Hrysiewicz" +__status__ = "Production" +__date__ = "Jan. 2022" + +########################################################################### +# Python packages +########################################################################### + +import os +import sys +import pandas as pd +import datetime +import os.path +import optparse + +# 尝试导入 geopandas 和 shapely 用于 Shapefile 支持 +try: + import geopandas as gpd + from shapely.geometry import box, Polygon, MultiPolygon + HAS_SHAPEFILE_SUPPORT = True +except ImportError: + HAS_SHAPEFILE_SUPPORT = False + print("Warning: geopandas/shapely not installed. Shapefile support disabled.") + print("Install with: pip install geopandas shapely") + +########################################################################### +########################################################################### + +class OptionParser (optparse.OptionParser): + + def check_required(self, opt): + option = self.get_option(opt) + if getattr(self.values, option.dest) is None: + self.error("%s option not supplied" % option) + +def bbox_from_shapefile(shapefile_path): + """ + 从 Shapefile 文件提取边界框 + + Args: + shapefile_path: Shapefile 文件路径（.shp） + + Returns: + 边界框字符串格式: "min_lon,min_lat,max_lon,max_lat" + """ + try: + # 读取 Shapefile + gdf = gpd.read_file(shapefile_path) + + # 计算总边界框 + total_bounds = gdf.total_bounds + # total_bounds = (minx, miny, maxx, maxy) + + # 转换为字符串格式 + bbox_str = f"{total_bounds[0]},{total_bounds[1]},{total_bounds[2]},{total_bounds[3]}" + + print(f"Shapefile bounding box: {bbox_str}") + return bbox_str + except Exception as e: + print(f"Error reading shapefile: {str(e)}") + return None + +def check_slc_exists(slc_name, path_SLC, path_RSLC, acquisition_date): + """ + 检查 SLC 文件是否已存在 + + Args: + slc_name: SLC 文件名 + path_SLC: SLC 存储路径 + path_RSLC: RSLC 存储路径 + acquisition_date: 采集日期字符串 + + Returns: + bool: 文件是否已存在 + """ + # 检查 .zip 文件 + if os.path.exists(os.path.join(path_SLC, slc_name + '.zip')): + return True + + # 检查 .rslc 文件 + try: + datei = datetime.datetime.strptime(acquisition_date.split('.')[0], '%Y-%m-%dT%H:%M:%S').strftime("%Y%m%d") + if os.path.exists(os.path.join(path_RSLC, datei + '.vv.rslc')): + return True + except: + pass + + return False + +def read_netrc(machine_name): + """ + 从 ~/.netrc 文件读取登录凭证 + + Args: + machine_name: 机器名称 (例如: urs.earthdata.nasa.gov) + + Returns: + tuple: (username, password) 或 (None, None) 如果未找到 + """ + netrc_path = os.path.expanduser('~/.netrc') + + if not os.path.exists(netrc_path): + return None, None + + try: + with open(netrc_path, 'r') as f: + lines = f.readlines() + i = 0 + while i < len(lines): + line = lines[i].strip() + + # 查找 machine 行 + if line.startswith('machine ' + machine_name): + username = None + password = None + + # 查找接下来的几行中的 login 和 password + j = i + 1 + while j < len(lines) and j < i + 5: # 最多查找5行 + next_line = lines[j].strip() + tokens = next_line.split() + + if len(tokens) >= 2: + if tokens[0] == 'login': + username = tokens[1] + elif tokens[0] == 'password': + password = tokens[1] + + # 如果找到下一个 machine，停止查找 + if next_line.startswith('machine '): + break + + j += 1 + + if username and password: + return username, password + + i += 1 + except Exception as e: + print(f"Warning: Error reading .netrc file: {str(e)}") + + return None, None + +def download_slc(url, username, password, path_SLC): + """ + 下载单个 SLC 文件 + + Args: + url: 下载 URL + username: ASF 用户名 + password: ASF 密码 + path_SLC: 存储路径 + + Returns: + bool: 下载是否成功 + """ + # 如果提供了用户名和密码，使用它们 + if username and password: + cmd = f'wget -c --http-user={username} --http-password={password} "{url}" -P {path_SLC}' + else: + # 否则使用 .netrc 文件 + # --auth-no-challenge: 立即发送认证信息，不等待服务器挑战 + # --netrc: 从 ~/.netrc 读取认证信息 + cmd = f'wget -c --auth-no-challenge --netrc "{url}" -P {path_SLC}' + + return os.system(cmd) == 0 + +def convert_yyyymmdd_to_iso(date_str): + """ + 将 YYYYMMDD 格式转换为 ISO 格式 (YYYY-MM-DDTHH:MM:SS) + + Args: + date_str: 日期字符串 (YYYYMMDD) + + Returns: + str: ISO 格式日期字符串 (YYYY-MM-DDT00:00:00) + """ + if len(date_str) == 8: + year = date_str[0:4] + month = date_str[4:6] + day = date_str[6:8] + return f"{year}-{month}-{day}T00:00:00" + else: + # 如果已经是其他格式，直接返回 + return date_str + +########################################################################### +########################################################################### + +if len(sys.argv) < 3: + prog = os.path.basename(sys.argv[0]) + print("="*80) + print("Sentinel-1 SLC Downloader from ASF") + print("="*80) + print("\nUsage examples:") + print("\n1. Using bounding box:") + print(" python3 " + prog + " -u username -p password -s . -r . \\") + print(" -b -10.78,51.27,-5.03,55.70 \\") + print(" -i 20170101 -j 20231231 \\") + print(f" -o 1 -f a -q IW -m csv -w n") + print("\n2. Using Shapefile (new feature):") + print(" python3 " + prog + " -u username -p password -s . -r . \\") + print(" --shp study_area.shp \\") + print(" -i 20170101 -j 20231231 \\") + print(f" -o 1 -f a -q IW -m csv -w n") + print("\nOptions:") + print(" -u, --username ASF username (optional, can use .netrc)") + print(" -p, --password ASF password (optional, can use .netrc)") + print(" -n, --netrc Machine name in ~/.netrc (default: urs.earthdata.nasa.gov)") + print(" -s, --path_SLC Output directory for SLC files") + print(" -r, --path_RSLC Directory to check for processed RSLC files") + print(" -b, --bbox Bounding box (min_lon,min_lat,max_lon,max_lat)") + print(" --shp Shapefile path (.shp) for spatial filtering") + print(" -i, --date_start Start date (YYYYMMDD)") + print(" -j, --date_end End date (YYYYMMDD)") + print(" -o, --orbit_relative Relative orbit number (optional)") + print(" --frame Frame number (optional)") + print(" -f, --flight_direction Flight direction (a=ascending, d=descending, or all)") + print(" -q, --q_acqui_mode Acquisition mode (IW, EW, SM, or all)") + print(" -m, --mode Output format (csv or kml)") + print(" -w, --write Download files? (Y/N)") + print(" --parallel Number of parallel downloads (default: 1, serial)") + print("\nNote: Shapefile support requires geopandas and shapely") + print("="*80) + sys.exit(-1) +else: + usage = "usage: %prog [options] " + parser = OptionParser(usage=usage) + parser.add_option("-u", "--username", action="store", type="string", default=None, + help="ASF username (can be read from ~/.netrc)") + parser.add_option("-p", "--password", action="store", type="string", default=None, + help="ASF password (can be read from ~/.netrc)") + parser.add_option("-n", "--netrc", action="store", type="string", default='urs.earthdata.nasa.gov', + help="Machine name in ~/.netrc file (default: urs.earthdata.nasa.gov)") + parser.add_option("-s", "--path_SLC", action="store", type="string", default='.') + parser.add_option("-r", "--path_RSLC", action="store", type="string", default='.') # Only available for GAMMA stack + parser.add_option("-b", "--bbox", action="store", type="string", default='-10.78,51.27,-5.03,55.70') + parser.add_option("--shp", "--shapefile", action="store", type="string", default=None, + help="Shapefile path for spatial filtering") + parser.add_option("-i", "--date_start", action="store", type="string", default='20170101') + parser.add_option("-j", "--date_end", action="store", type="string", default='20240101') + parser.add_option("-o", "--orbit_relative", action="store", type="float", default=None, + help="Relative orbit number (optional, if not specified will search all orbits)") + parser.add_option("--frame", action="store", type="int", default=None, + help="Frame number (optional, if not specified will search all frames)") + parser.add_option("-f", "--flight_direction", action="store", type="string", default='a', + help="Flight direction (a=ascending, d=descending, or 'all' for both)") + parser.add_option("-q", "--q_acqui_mode", action="store", type="string", default='IW') + parser.add_option("-m", "--mode", action="store", type="string", default='csv') + parser.add_option("-w", "--write", action="store", type="string", default='n') + parser.add_option("--parallel", action="store", type="int", default=1, + help="Number of parallel downloads (default: 1, serial)") + (options, args) = parser.parse_args() + +########################################################################### +# Main +########################################################################### + +date_format = "%Y-%m-%d" + +# 转换日期格式 +date_start_iso = convert_yyyymmdd_to_iso(options.date_start) +date_end_iso = convert_yyyymmdd_to_iso(options.date_end) + +# 处理用户名和密码：如果未提供，尝试从 .netrc 读取 +if options.username is None or options.password is None: + netrc_username, netrc_password = read_netrc(options.netrc) + if netrc_username and netrc_password: + if options.username is None: + options.username = netrc_username + print(f"Username read from ~/.netrc ({options.netrc})") + if options.password is None: + options.password = netrc_password + print(f"Password read from ~/.netrc ({options.netrc})") + else: + if options.username is None: + print("Error: Username not provided and not found in ~/.netrc") + print("Please provide username with -u option or configure ~/.netrc") + sys.exit(1) + if options.password is None: + print("Error: Password not provided and not found in ~/.netrc") + print("Please provide password with -p option or configure ~/.netrc") + sys.exit(1) + +# 处理空间范围：优先使用 Shapefile，否则使用 bounding box +if options.shp: + if not HAS_SHAPEFILE_SUPPORT: + print("Error: Shapefile support requires geopandas and shapely") + print("Install with: pip install geopandas shapely") + sys.exit(1) + + print(f"Using Shapefile: {options.shp}") + bbox = bbox_from_shapefile(options.shp) + if bbox is None: + print("Error: Failed to extract bounding box from shapefile") + sys.exit(1) +else: + bbox = options.bbox + print(f"Using bounding box: {bbox}") + +# 构建 API 查询命令 +# 轨道号、frame 和飞行方向都是可选的 +api_params = f"platform=s1&bbox={bbox}&start={date_start_iso}-UTC&end={date_end_iso}-UTC&processingLevel=SLC&maxResults=10000" + +# 如果指定了轨道号，添加到查询参数 +if options.orbit_relative is not None: + api_params += f"&relativeOrbit={int(options.orbit_relative)}" + +# 如果指定了 frame 号，添加到查询参数 +if hasattr(options, 'frame') and options.frame is not None: + api_params += f"&frame={options.frame}" + +# 如果指定了飞行方向且不是 'all'，添加到查询参数 +if options.flight_direction and options.flight_direction.upper() != 'ALL': + api_params += f"&flightDirection={options.flight_direction.upper()}" + +# 添加输出格式 +api_params += f"&output={options.mode.upper()}" + +cmd1 = f'curl "https://api.daac.asf.alaska.edu/services/search/param?{api_params}" > SLC_list.{options.mode.lower()}' + +print(f"Querying ASF API for SLC data...") +print(f" Bounding Box: {bbox}") +print(f" Date Range: {options.date_start} to {options.date_end}") +if options.orbit_relative is not None: + print(f" Orbit: {options.orbit_relative}") +else: + print(f" Orbit: All") +if hasattr(options, 'frame') and options.frame is not None: + print(f" Frame: {options.frame}") +else: + print(f" Frame: All") +if options.flight_direction and options.flight_direction.upper() != 'ALL': + print(f" Flight Direction: {options.flight_direction.upper()}") +else: + print(f" Flight Direction: All") +print(f" Mode: {options.q_acqui_mode.upper()}") +print() + +# 执行查询 +if options.mode.upper() == 'KML': + os.system(cmd1) +elif options.mode.upper() == 'CSV': + os.system(cmd1) +else: + print('Error: Please select a correct mode (csv or kml)...') + sys.exit(1) + +# 根据波束模式过滤列表 +print(f"Filtering results by acquisition mode: {options.q_acqui_mode.upper()}") +if os.path.exists("SLC_list.csv"): + os.rename("SLC_list.csv","SLC_list_orig.csv") + h = 0 + fout = open("SLC_list.csv",'w') + total_count = 0 + matched_count = 0 + + with open("SLC_list_orig.csv") as fi: + for li in fi: + total_count += 1 + if h > 0: + if options.q_acqui_mode.upper() == 'all': + fout.write(li) + matched_count += 1 + elif options.q_acqui_mode.upper() in li: + fout.write(li) + matched_count += 1 + else: + fout.write(li) + h = h + 1 + + fout.close() + os.remove("SLC_list_orig.csv") + + print(f" Total results: {total_count - 1}") + print(f" Filtered results: {matched_count}") +else: + print("Warning: SLC_list.csv not found") + sys.exit(1) + +# 下载文件 +if options.mode.upper() == 'CSV' and options.write.upper() == 'Y': + try: + listSLC = pd.read_csv("SLC_list.csv") + except Exception as e: + print(f"Error reading SLC_list.csv: {str(e)}") + sys.exit(1) + + print(f"\nFound {len(listSLC)} SLC files to process") + print("="*80) + + # 创建输出目录 + if not os.path.exists(options.path_SLC): + os.makedirs(options.path_SLC) + print(f"Created output directory: {options.path_SLC}") + + # 检查是否有并行参数 + parallel_numb = getattr(options, 'parallel', 1) + if parallel_numb is None: + parallel_numb = 1 + + # 准备下载数据列表 + download_tasks = [] + skipped_tasks = [] + + for h, slci in enumerate(listSLC['Granule Name']): + url = listSLC['URL'][h] + acquisition_date = listSLC['Acquisition Date'][h] + + # 检查文件是否已存在 + if check_slc_exists(slci, options.path_SLC, options.path_RSLC, acquisition_date): + skipped_tasks.append((h+1, slci, "Already exists")) + else: + download_tasks.append((h+1, slci, url, acquisition_date)) + + print(f"Files to download: {len(download_tasks)}") + print(f"Files skipped (already exists): {len(skipped_tasks)}") + + # 并行下载函数 + def download_worker(task): + idx, slci, url, acquisition_date = task + print(f"\n[{idx}/{len(listSLC)}] Downloading: {slci}") + + if download_slc(url, options.username, options.password, options.path_SLC): + print(f" Status: Download SUCCESS") + return (idx, slci, "SUCCESS") + else: + print(f" Status: Download FAILED") + return (idx, slci, "FAILED") + + # 执行下载 + downloaded = 0 + failed = 0 + + if parallel_numb > 1 and len(download_tasks) > 0: + # 并行下载 + from multiprocessing.pool import ThreadPool + print(f"\nStarting parallel download with {parallel_numb} workers...") + + pool = ThreadPool(parallel_numb) + results = pool.map(download_worker, download_tasks) + pool.close() + pool.join() + + # 统计结果 + for result in results: + if result[2] == "SUCCESS": + downloaded += 1 + else: + failed += 1 + elif len(download_tasks) > 0: + # 串行下载 + print("\nStarting serial download...") + for task in download_tasks: + result = download_worker(task) + if result[2] == "SUCCESS": + downloaded += 1 + else: + failed += 1 + + # 打印摘要 + print("\n" + "="*80) + print("DOWNLOAD SUMMARY") + print("="*80) + print(f"Total SLC files: {len(listSLC)}") + print(f"Downloaded: {downloaded}") + print(f"Skipped (already exists): {len(skipped_tasks)}") + print(f"Failed: {failed}") + print("="*80) + + if failed > 0: + print(f"\nWarning: {failed} downloads failed. Check the output above for details.") + sys.exit(1) + +elif options.mode.upper() == 'KML' and options.write.upper() == 'Y': + print('Please, select CSV mode to download..') +elif options.write.upper() != 'Y': + print(f"\nSLC list saved to: SLC_list.csv") + print("To download files, re-run with -w Y option") \ No newline at end of file diff --git a/third_party/PyINT/pyint/ASAR_orb_cor.py b/third_party/PyINT/pyint/ASAR_orb_cor.py new file mode 100644 index 0000000..87eb3c5 --- /dev/null +++ b/third_party/PyINT/pyint/ASAR_orb_cor.py @@ -0,0 +1,139 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Central South University & University of Miami ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import linecache +import datetime + +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +######################################################################### + +INTRODUCTION = ''' +############################################################################# + Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 + + Correct ASAR orbit using precise DORIS data. + +''' + +EXAMPLE = ''' + Usage: + ASAR_orb_cor.py projectName Date + + Examples: + ASAR_orb_cor.py AqabaERSA 960101 + +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Download (and correct) precise ERS orbit data.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='project name') + parser.add_argument('date',help='date of the ERS data') + + inps = parser.parse_args() + return inps + +################################################################################ + + +def main(argv): + + total = time.time() + inps = cmdLineParse() + + projectName = inps.projectName + DATE = inps.date + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + ASAROrbDir = os.getenv('ASARORBDIR') + + + slcDir = scratchDir + '/' + projectName + '/SLC' + projDir = scratchDir + '/' + projectName + + + if len(DATE)==8: + DATE8 = DATE + DATE6 = DATE[2:8] + else: + DATE8 = '20' + DATE + DATE6 = DATE + + + slcDir1 = slcDir + '/' + DATE6 + slcpar = slcDir1 + '/' + DATE6 + '.slc.par' + + + Year = DATE8[0:4] + Month = DATE8[4:6] + Day = DATE8[6:8] + + DATE1 = (datetime.date(int(Year),int(Month),int(Day)) + datetime.timedelta(days=-1)).strftime("%Y%m%d") + DATE2 = (datetime.date(int(Year),int(Month),int(Day)) + datetime.timedelta(days=1)).strftime("%Y%m%d") + + os.chdir(ASAROrbDir) + call_str = 'ls > t0' + os.system(call_str) + + call_str="grep " + DATE1 + " t0 | grep " + DATE2 + " >t01" + os.system(call_str) + + AA= np.loadtxt('t01',dtype=np.str) + AA_file = ASAROrbDir + '/' + str(AA) + + call_str = 'DORIS_vec ' + slcpar + ' ' + AA_file + os.system(call_str) + + + print("Using DORIS orbital data for %s is done." % DATE) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + diff --git a/third_party/PyINT/pyint/ASAR_orb_cor_all .py b/third_party/PyINT/pyint/ASAR_orb_cor_all .py new file mode 100644 index 0000000..0676e4c --- /dev/null +++ b/third_party/PyINT/pyint/ASAR_orb_cor_all .py @@ -0,0 +1,125 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Central South University & University of Miami ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import linecache + +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +def is_number(s): + try: + int(s) + return True + except ValueError: + return False + +######################################################################### + +INTRODUCTION = ''' +############################################################################# + Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 + + Correct ASAR orbit using precise DORIS data. + +''' + +EXAMPLE = ''' + Usage: + ASAR_orb_cor_all.py projectName + + Examples: + ASAR_orb_cor_all.py AqabaERSA + +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Download (and correct) precise ERS orbit data.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='project name') + + inps = parser.parse_args() + return inps + +################################################################################ + + +def main(argv): + + total = time.time() + inps = cmdLineParse() + + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + slcDir = scratchDir + '/' + projectName + '/SLC' + + ListSLC = os.listdir(slcDir) + Datelist = [] + SLCfile = [] + SLCParfile = [] + + + for kk in range(len(ListSLC)): + if ( is_number(ListSLC[kk]) and len(ListSLC[kk])==6 ): # if SAR date number is 8, 6 should change to 8. + DD=ListSLC[kk] + Year=int(DD[0:2]) + Month = int(DD[2:4]) + Day = int(DD[4:6]) + Datelist.append(ListSLC[kk]) + N = len(Datelist) + + for i in range(N): + call_str = 'ERS_orb_cor.py ' + projectName + ' ' + Datelist[i] + print(call_str) + os.system(call_str) + + print("Using DEFT orbital data for project %s is done." % projectName) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + diff --git a/third_party/PyINT/pyint/ASAR_orb_cor_par.py b/third_party/PyINT/pyint/ASAR_orb_cor_par.py new file mode 100644 index 0000000..78f2aea --- /dev/null +++ b/third_party/PyINT/pyint/ASAR_orb_cor_par.py @@ -0,0 +1,122 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Central South University & University of Miami ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import linecache +import datetime + +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +######################################################################### + +INTRODUCTION = ''' +############################################################################# + Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 + + Precise ENVISAT orbit data from Delft (http://www.deos.tudelft.nl/). + Correct the orbit parameters using DORIS_vec + +''' + +EXAMPLE = ''' + Usage: + ASAR_orb_cor_par.py slc_par + + Examples: + ASAR_orb_cor_par.py 960101.slc.par + +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Download (and correct) precise ERS orbit data.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('slc_par',help='slc_par file.') + + inps = parser.parse_args() + return inps + +################################################################################ + + +def main(argv): + + total = time.time() + inps = cmdLineParse() + + slcpar = inps.slc_par + ASAROrbDir = os.getenv('ASARORBDIR') + PATH =os.getcwd() + + DATE = UseGamma(slcpar,'read','date:') + Year = str(int(DATE[0:4])) + Month = str(int(DATE[4:8])) + Day = str(int(DATE[8:12])) + + DATE1 = (datetime.date(int(Year),int(Month),int(Day)) + datetime.timedelta(days=-1)).strftime("%Y%m%d") + DATE2 = (datetime.date(int(Year),int(Month),int(Day)) + datetime.timedelta(days=1)).strftime("%Y%m%d") + + os.chdir(ASAROrbDir) + call_str = 'ls > t0' + os.system(call_str) + + call_str="grep " + DATE1 + " t0 | grep " + DATE2 + " > t01" + os.system(call_str) + + AA= np.loadtxt('t01',dtype=np.str) + AA_file = ASAROrbDir + '/' + str(AA) + + os.chdir(PATH) + call_str = 'DORIS_vec ' + slcpar + ' ' + AA_file + ' 20' + os.system(call_str) + + + print("Using DORIS orbital data for %s is done." % slcpar) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + diff --git a/third_party/PyINT/pyint/AutoGACOS/.gitignore b/third_party/PyINT/pyint/AutoGACOS/.gitignore new file mode 100644 index 0000000..946f285 --- /dev/null +++ b/third_party/PyINT/pyint/AutoGACOS/.gitignore @@ -0,0 +1,167 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# mac +.DS_Store + + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ +tests + + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + \ No newline at end of file diff --git a/third_party/PyINT/pyint/AutoGACOS/LICENSE b/third_party/PyINT/pyint/AutoGACOS/LICENSE new file mode 100644 index 0000000..d470d14 --- /dev/null +++ b/third_party/PyINT/pyint/AutoGACOS/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Fanchengyan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/third_party/PyINT/pyint/AutoGACOS/README.md b/third_party/PyINT/pyint/AutoGACOS/README.md new file mode 100644 index 0000000..433a5ed --- /dev/null +++ b/third_party/PyINT/pyint/AutoGACOS/README.md @@ -0,0 +1,2 @@ +# AutoGACOS +A Python library for automatically submitting and downloading GACOS data diff --git a/third_party/PyINT/pyint/AutoGACOS/docs/Makefile b/third_party/PyINT/pyint/AutoGACOS/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/third_party/PyINT/pyint/AutoGACOS/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/third_party/PyINT/pyint/AutoGACOS/docs/requirements.txt b/third_party/PyINT/pyint/AutoGACOS/docs/requirements.txt new file mode 100644 index 0000000..2a9aa68 --- /dev/null +++ b/third_party/PyINT/pyint/AutoGACOS/docs/requirements.txt @@ -0,0 +1,6 @@ +recommonmark +sphinx>=7 +myst-parser +myst_nb +sphinx_rtd_theme +Jinja2 diff --git a/third_party/PyINT/pyint/AutoGACOS/docs/source/api/datasets/datasets.rst b/third_party/PyINT/pyint/AutoGACOS/docs/source/api/datasets/datasets.rst new file mode 100644 index 0000000..f8fa7f7 --- /dev/null +++ b/third_party/PyINT/pyint/AutoGACOS/docs/source/api/datasets/datasets.rst @@ -0,0 +1,32 @@ +InSAR Datasets +============== + +To automatically submit requests to the GACOS service, the user needs to provide +the coordinates of the area of interest and the acquisition dates and times of +the SAR images. We provide a set of classes to automatically retrieve the +those information from well known InSAR datasets. + +Currently, the following InSAR Datasets are supported: + * :class:`.HyP3Dataset` + * :class:`.LiCSARDataset` + +HyP3Dataset +----------- + +.. autoclass:: gacos.HyP3Dataset + :members: + :undoc-members: + :member-order: bysource + :show-inheritance: + +LiCSARDataset +------------- + +.. autoclass:: gacos.LiCSARDataset + :members: + :undoc-members: + :member-order: bysource + :show-inheritance: + + + \ No newline at end of file diff --git a/third_party/PyINT/pyint/AutoGACOS/docs/source/api/index.rst b/third_party/PyINT/pyint/AutoGACOS/docs/source/api/index.rst new file mode 100644 index 0000000..32d5ef4 --- /dev/null +++ b/third_party/PyINT/pyint/AutoGACOS/docs/source/api/index.rst @@ -0,0 +1,11 @@ +Python API Reference +==================== + + +.. toctree:: + + datasets/datasets + submit/submit + + + diff --git a/third_party/PyINT/pyint/AutoGACOS/docs/source/api/submit/submit.rst b/third_party/PyINT/pyint/AutoGACOS/docs/source/api/submit/submit.rst new file mode 100644 index 0000000..a07379b --- /dev/null +++ b/third_party/PyINT/pyint/AutoGACOS/docs/source/api/submit/submit.rst @@ -0,0 +1,15 @@ +Requests Submitter +================== + +The GACOS limits the number of acquisition requests that can be submitted at a time. +This is to prevent the system from being overloaded. But, if you have a large number +of acquisition requests to submit, you can submit them in batches. We designed a +:class:`gacos.Submitter` class to help you submit the requests in batches. The +acquisitions will be split into batches which not exceed the 20 acquisitions per +request. + +.. autoclass:: gacos.Submitter + :members: + :undoc-members: + :member-order: bysource + :show-inheritance: diff --git a/third_party/PyINT/pyint/AutoGACOS/docs/source/conf.py b/third_party/PyINT/pyint/AutoGACOS/docs/source/conf.py new file mode 100644 index 0000000..073055f --- /dev/null +++ b/third_party/PyINT/pyint/AutoGACOS/docs/source/conf.py @@ -0,0 +1,58 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "FanSAR" +copyright = "2023, Fancy" +author = "Fancy" +release = "v1.0" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + "myst_nb", + "sphinx.ext.autodoc", + "sphinx.ext.doctest", + "sphinx.ext.intersphinx", + "sphinx.ext.napoleon", + "sphinx.ext.todo", +] +source_suffix = { + ".rst": "restructuredtext", + ".md": "markdown", + ".ipynb": "myst-nb", +} + +# templates_path = ['_templates'] +exclude_patterns = [] + +# to disable execution of notebooks +nb_execution_mode = "off" +nb_execution_excludepatterns = ["quickstart.ipynb"] + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "sphinx_rtd_theme" +# html_static_path = ['_static'] + +# General information about the project. +project = "faninsar" +copyright = "2023, Fancy" +author = "Chengyan Fan" + +video_enforce_extra_source = True + +autodoc_default_options = { + "members": True, + "undoc-members": True, + "member-order": "bysource", + "special-members": "__init__", + ":show-inheritance:": True, +} diff --git a/third_party/PyINT/pyint/AutoGACOS/docs/source/index.rst b/third_party/PyINT/pyint/AutoGACOS/docs/source/index.rst new file mode 100644 index 0000000..91f1318 --- /dev/null +++ b/third_party/PyINT/pyint/AutoGACOS/docs/source/index.rst @@ -0,0 +1,19 @@ +===================================== +Welcome to AutoGACOS's documentation! +===================================== + + +Introduction +------------ + +.. toctree:: + :maxdepth: 4 + :caption: Contents: + + intro + user_guide/quickstart + AutoGACOS API Reference <api/index> + terminology + + + diff --git a/third_party/PyINT/pyint/AutoGACOS/docs/source/intro.rst b/third_party/PyINT/pyint/AutoGACOS/docs/source/intro.rst new file mode 100644 index 0000000..e8b5eb2 --- /dev/null +++ b/third_party/PyINT/pyint/AutoGACOS/docs/source/intro.rst @@ -0,0 +1,3 @@ +Introduction +============ + diff --git a/third_party/PyINT/pyint/AutoGACOS/docs/source/terminology.rst b/third_party/PyINT/pyint/AutoGACOS/docs/source/terminology.rst new file mode 100644 index 0000000..844e30d --- /dev/null +++ b/third_party/PyINT/pyint/AutoGACOS/docs/source/terminology.rst @@ -0,0 +1,27 @@ +.. _terminology: + +Terminology +=========== + +.. glossary:: + + Acquisition + A single SAR acquisition, and is expressed as ``datetime.datetime`` object. + + Pair + A pair is a combination of two SAR acquisitions. + + Pairs + A collection of pairs. + + Loop + A loop + + Loops + A collection of loops. + + SBASNetwork + A collection of loops and pairs. + + CRS + A coordinate reference system (CRS) is a coordinate-based local, regional or global system used to locate geographical entities. In FanInSAR, the CRS is handled by the ``rasterio`` package. \ No newline at end of file diff --git a/third_party/PyINT/pyint/AutoGACOS/docs/source/user_guide/quickstart.ipynb b/third_party/PyINT/pyint/AutoGACOS/docs/source/user_guide/quickstart.ipynb new file mode 100644 index 0000000..beb1bcd --- /dev/null +++ b/third_party/PyINT/pyint/AutoGACOS/docs/source/user_guide/quickstart.ipynb @@ -0,0 +1,635 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Quickstart for AutoGACOS\n", + "\n", + "**AutoGACOS** is a Python library for automatically submitting and downloading GACOS data" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import gacos" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "path = \"/Volumes/Data/Hyp3/descending_roi\"\n", + "ds = gacos.HyP3Dataset(path)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "check all acquisition dates in Hyp3 dateset" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "DatetimeIndex(['2015-11-12 23:26:31', '2015-12-06 23:26:31',\n", + " '2015-12-30 23:26:30', '2016-01-23 23:26:29',\n", + " '2016-02-16 23:26:28', '2016-03-11 23:26:28',\n", + " '2016-04-04 23:26:29', '2016-04-28 23:26:30',\n", + " '2016-05-22 23:26:34', '2016-06-15 23:26:36',\n", + " ...\n", + " '2023-01-10 23:27:15', '2023-01-22 23:27:15',\n", + " '2023-02-03 23:27:15', '2023-02-15 23:27:14',\n", + " '2023-02-27 23:27:15', '2023-03-11 23:27:14',\n", + " '2023-03-23 23:27:14', '2023-04-04 23:27:15',\n", + " '2023-08-14 23:27:22', '2023-09-07 23:27:23'],\n", + " dtype='datetime64[ns]', length=204, freq=None)\n", + "Index(['20151112', '20151206', '20151230', '20160123', '20160216', '20160311',\n", + " '20160404', '20160428', '20160522', '20160615',\n", + " ...\n", + " '20230110', '20230122', '20230203', '20230215', '20230227', '20230311',\n", + " '20230323', '20230404', '20230814', '20230907'],\n", + " dtype='object', length=204)\n", + "['23:25' '23:26' '23:27']\n" + ] + } + ], + "source": [ + "print(ds.date_times)\n", + "print(ds.dates)\n", + "print(ds.times)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Submit GACOS requests automatically" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "bf9575b9deea497089cc8167b8c8a438", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/3 [00:00<?, ?it/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a9dafb6856a5407da0dbf6b965189f73", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/1 [00:00<?, ?it/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 25, 'date': '20161007\\n20161031\\n20161124\\n20161218\\n20170111\\n20170204', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", + " sleeping for 546 seconds...\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "8fd9eb9543774f3b87058eef331792df", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/6 [00:00<?, ?it/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 26, 'date': '20151112\\n20151206\\n20151230\\n20160123\\n20160216\\n20160311\\n20160404\\n20160428\\n20160522\\n20160615\\n20160709\\n20160802\\n20160826\\n20160919\\n20170222\\n20170318\\n20170330', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", + ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 26, 'date': '20170411\\n20170423\\n20170505\\n20170517\\n20170529\\n20170610\\n20170622\\n20170704\\n20170716\\n20170728\\n20170809\\n20170821\\n20170902\\n20170914\\n20171008\\n20171020\\n20171101', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", + ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 26, 'date': '20171113\\n20171125\\n20171207\\n20171219\\n20171231\\n20180112\\n20180124\\n20180205\\n20180217\\n20180301\\n20180313\\n20180325\\n20180406\\n20180418\\n20180430\\n20180512\\n20180524', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", + ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 26, 'date': '20180605\\n20180617\\n20180629\\n20180711\\n20180723\\n20180804\\n20180816\\n20180921\\n20181003\\n20181015\\n20181027\\n20181108\\n20181120\\n20181202\\n20181214\\n20181226\\n20190107', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", + ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 26, 'date': '20190119\\n20190131\\n20190212\\n20190224\\n20190308\\n20190320\\n20190401\\n20190413\\n20190425\\n20190507\\n20190519\\n20190531\\n20190612\\n20190624\\n20190706\\n20190718\\n20190730', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", + ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 26, 'date': '20190811\\n20190823\\n20190904\\n20191221\\n20200102\\n20200114\\n20200126\\n20200207\\n20200219\\n20200302\\n20200314\\n20200326\\n20200407\\n20200419\\n20200501\\n20200513', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", + " sleeping for 288 seconds...\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4aa82286c5414723adcddcc8d864c133", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/5 [00:00<?, ?it/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 27, 'date': '20190916\\n20190928\\n20191010\\n20191022\\n20191103\\n20191115\\n20191127\\n20191209\\n20200525\\n20200606\\n20200618\\n20200630\\n20200712\\n20200724\\n20200805\\n20200817\\n20200829\\n20200910\\n20200922\\n20201004', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", + ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 27, 'date': '20201016\\n20201028\\n20201109\\n20201121\\n20201203\\n20201215\\n20201227\\n20210108\\n20210120\\n20210201\\n20210213\\n20210225\\n20210309\\n20210321\\n20210402\\n20210414\\n20210426\\n20210508\\n20210520\\n20210601', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", + ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 27, 'date': '20210613\\n20210625\\n20210707\\n20210719\\n20210731\\n20210812\\n20210824\\n20210905\\n20210917\\n20210929\\n20211011\\n20211023\\n20211104\\n20211116\\n20211128\\n20211210\\n20211222\\n20220103\\n20220115', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", + ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 27, 'date': '20220127\\n20220208\\n20220220\\n20220304\\n20220316\\n20220328\\n20220409\\n20220421\\n20220503\\n20220515\\n20220527\\n20220620\\n20220702\\n20220714\\n20220726\\n20220807\\n20220819\\n20220831\\n20220912', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", + ">>> succeed post: {'N': 39.16527884776772, 'W': 98.34602960014247, 'S': 38.52041942953548, 'E': 99.41788862899489, 'H': 23, 'M': 27, 'date': '20220924\\n20221006\\n20221018\\n20221030\\n20221111\\n20221123\\n20221205\\n20221217\\n20221229\\n20230110\\n20230122\\n20230203\\n20230215\\n20230227\\n20230311\\n20230323\\n20230404\\n20230814\\n20230907', 'type': '2', 'email': 'fanchengyan2020@126.com'}\n", + " sleeping for 743 seconds...\n" + ] + } + ], + "source": [ + "# initialize a submitter with the dataset and your email\n", + "submitter = gacos.Submitter(ds, email=\"your_email@xxx.com\")\n", + "\n", + "# submit all gacos requests for acquisitions in the Hyp3 dataset\n", + "submitter.post_requests()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# check the status of all requests\n", + "print(f\"Successfully submitted requests: \\n{submitter.succeed}\")\n", + "print(f\"Failed to submit requests: \\n{submitter.failed}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Retrieve GACOS results urls from your email automatically" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "667dccc73a434102a87226ebcb453389", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Retrieving GACOS urls: 0%| | 0/240 [00:00<?, ? emails/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "output_file = \"/Volumes/Data/Hyp3/descending_gacos/gacos_urls.csv\"\n", + "\n", + "email = gacos.GACOSEmail(\n", + " username=\"your_email\",\n", + " password=\"password_of_your_email\",\n", + " host=\"imap.xxx.com\", # imap server of your email\n", + " start_date=\"2023-11-01\", # only download gacos results for the email after this date\n", + ")\n", + "email.retrieve_gacos_urls(output_file)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Download GACOS results using the urls " + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b99066adfc1c4f24aa2164540cd03ce7", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/24 [00:00<?, ?it/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ed443926d42e480ca0d5d835ee23b377", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "20231116T010337VzTsCTN1h.tar.gz: 0%| | 0.00/15.6M [00:00<?, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a40ea2c7a8da49eebc8dcb46d6e528ae", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "20231116T011307V7OXOyZBe.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "478bae1d3f9545e7936e0436e21f48f3", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "20231116T011337VC4LikOAs.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "35e60690588147028abbc997e1b0fdf6", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "20231116T011408V6h3BniJT.tar.gz: 0%| | 0.00/41.1M [00:00<?, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b5797b66e4044865acac24bb6bdbe4bc", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "20231116T011237Vy7m1l2al.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ab235b358f7d477c9a8f367b1c969e04", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "20231116T011738Vniy7uNPV.tar.gz: 0%| | 0.00/51.2M [00:00<?, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "edd581a5a24049cfa884c7d47ff5fd6c", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "20231116T011808VcEaTHazf.tar.gz: 0%| | 0.00/48.8M [00:00<?, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f4501c922a984f8e890e2196c8717742", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "20231116T011838VJaHXOaZJ.tar.gz: 0%| | 0.00/48.8M [00:00<?, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "025da6a325394f6195555963ebc48f30", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "20231116T142253VdcxwhZjh.tar.gz: 0%| | 0.00/15.6M [00:00<?, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1e10f5c2420047dd9a20476e990be884", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "20231116T142324VXIAESwIW.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7f048fd91a7246f1adaa81de3c33a6f8", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "20231116T142354VQj8kS9I0.tar.gz: 0%| | 0.00/43.6M [00:00<?, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b819434541c540898948615a24650eab", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "20231116T142424VMFCCq5W7.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9b2abbf34daf4b12b44a44dc6b902ed2", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "20231116T144425VSlcfYAm2.tar.gz: 0%| | 0.00/15.6M [00:00<?, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "89261f821810404281862ba336f56eb6", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "20231116T144455VCncFDojU.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d55f37d845454a0d90254ab82686b5d0", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "20231116T144525VisGJO2bU.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "75bd725bd9bf43b0bfe03682db91c6a2", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "20231116T144555VcE1J5wo4.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "53773e7ba770412da3b45fb63d014a98", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "20231116T145756VCHtI731i.tar.gz: 0%| | 0.00/15.6M [00:00<?, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b7848836c32244759118c616b458a3c4", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "20231116T145827VhbjZbz5b.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f4f396e98e134ea08c8b6f5f9aa6138f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "20231116T145857VUeNu4WnY.tar.gz: 0%| | 0.00/43.6M [00:00<?, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7ef6ff94335c4db38c7f4d20a07d9f29", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "20231116T145927V2I8j7eo6.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "1979604e57d3473b9ea7af1cd2e112bb", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "20231116T151057VQMukfvJ5.tar.gz: 0%| | 0.00/15.6M [00:00<?, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7fcd89254e0e4335a20440b2cff80bcc", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "20231116T151158Vx3KrT8Va.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7550835c8d834f8895efea89ede2d308", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "20231116T151228VcpWSrjqg.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d42edcb7dfd546aca10f4cd4635e3cc1", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "20231116T151258VfhhwgAli.tar.gz: 0%| | 0.00/43.7M [00:00<?, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "url_file = output_file\n", + "output_dir = \"/Volumes/Data/Hyp3/descending_gacos\"\n", + "\n", + "gacos_dl = gacos.Downloader(url_file, output_dir, time=23.43)\n", + "gacos_dl.download()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "geo", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/third_party/PyINT/pyint/AutoGACOS/gacos/__init__.py b/third_party/PyINT/pyint/AutoGACOS/gacos/__init__.py new file mode 100644 index 0000000..ac5662b --- /dev/null +++ b/third_party/PyINT/pyint/AutoGACOS/gacos/__init__.py @@ -0,0 +1,4 @@ +from .datasets import LiCSARDataset, SarDataset, HyP3Dataset +from .parse_email import GACOSEmail +from .submit import Submitter +from .download import Downloader diff --git a/third_party/PyINT/pyint/AutoGACOS/gacos/datasets.py b/third_party/PyINT/pyint/AutoGACOS/gacos/datasets.py new file mode 100644 index 0000000..dafca0f --- /dev/null +++ b/third_party/PyINT/pyint/AutoGACOS/gacos/datasets.py @@ -0,0 +1,276 @@ +import warnings +from pathlib import Path +from typing import Literal, Optional, Union + +import numpy as np +import pandas as pd +try: + from faninsar.datasets import HyP3, LiCSAR +except ImportError: + try: + from faninsar.datasets import hyp3 as HyP3, licsar as LiCSAR + except ImportError: + HyP3 = None + LiCSAR = None + +warnings.filterwarnings("ignore") + + +class SarDataset: + def __init__( + self, + bounds: tuple[float, float, float, float], + date_times: pd.DatetimeIndex, + gacos_dir: Optional[Union[Path, str]] = None, + ) -> None: + """Initialize SarDataset class + + Parameters + ---------- + bounds : tuple[float, float, float, float] + The bounding box of the dataset. + date_times : pd.DatetimeIndex + The datetime index of the dataset. + gacos_dir : Optional[Union[Path, str]], optional + The directory used to save gacos data. Used to check if the data is + already downloaded and avoid resubmitting. Default is None. + """ + self.bounds = bounds + self._date_times = date_times + + self._dates = date_times.strftime("%Y%m%d") + + if gacos_dir is not None: + self._dates_remain = self._get_dates_remain(gacos_dir) + else: + self._dates_remain = self.dates + + hour = date_times.hour + minute = np.round((date_times.second / 60) + date_times.minute).astype(int) + times = pd.Series([f"{h:02d}:{m:02d}" for h, m in zip(hour, minute)]) + self._times = times + + self._times_remain = self._get_times_remain() + + def __str__(self) -> str: + return ( + f"{self.__class__.__name__}(\n" + f" bounds={self.bounds}, \n" + f" times={len(self.times)}, \n" + f" dates={len(self.dates)}\n" + ")" + ) + + def __repr__(self) -> str: + return self.__str__() + + def _get_dates_remain(self, gacos_dir: Union[Path, str]): + """Get the dates that are not downloaded yet. + Parameters + ---------- + gacos_dir : Union[Path, str] + The directory used to save gacos data. Used to check if the data is + already downloaded and avoid resubmitting. + + Returns + ------- + dates_remain : np.ndarray + The dates that are not downloaded yet. + """ + gacos_files = list(Path(gacos_dir).rglob("*.ztd.tif")) + gacos_dates = [] + for i in gacos_files: + stem = i.stem.split(".")[0] + if len(stem) == 8: + gacos_dates.append(stem) + dates_remain = np.setdiff1d(self.dates, gacos_dates) + return dates_remain + + def _get_times_remain(self): + """Get the times corresponding to the dates that are not downloaded yet.""" + idx = np.where(np.isin(self.dates, self.dates_remain))[0] + times_remain = self._times[idx] + return times_remain + + @property + def dates(self): + """The dates (YYYYMMDD) of the acquisitions parsed from dataset.""" + return self._dates + + @property + def times(self): + """The times (HH:MM) of the acquisitions parsed from dataset.""" + return self._times.unique() + + @property + def date_times(self): + """The datetime of the acquisitions parsed from dataset.""" + return self._date_times + + @property + def dates_remain(self): + """The dates that are not downloaded yet. If gacos_dir is None, then + dates_remain is the same as dates.""" + return self._dates_remain + + @property + def times_remain(self): + """The times corresponding to the dates that are not downloaded yet.""" + return self._times_remain + + def gen_datetime_patches( + self, + mode: Literal["all", "remain"] = "remain", + ) -> dict: + """Generate datetime patches. + + Parameters + ---------- + mode : Literal["all", "remain"], optional + The mode to generate datetime patches. If "all", then generate all + the datetime patches. If "remain", then generate the datetime + patches of the dates that are not downloaded yet. Default is + "remain". + + Returns + ------- + datetime_patches : dict + The datetime patches. The key is the time (HH:MM) and the value is + the datetime patches. + """ + nums = 20 + datetime_patches = {} + + if mode == "all": + for _time in self.times: + _dts = self.dates[self._times == _time] + n_patch = np.ceil(len(_dts) / nums) + dates_patch = np.array_split(_dts, n_patch) + datetime_patches[_time] = dates_patch + elif mode == "remain": + for _time in self.times_remain: + _dts = self.dates_remain[self._times_remain == _time] + n_patch = np.ceil(len(_dts) / nums) + dates_patch = np.array_split(_dts, n_patch) + datetime_patches[_time] = dates_patch + + return datetime_patches + + def gen_post_data( + self, + dates: Union[list, np.ndarray], + times: Union[tuple[int, int], tuple[str, str]], + email: str, + ): + """Generate post data for gacos website. + + Parameters + ---------- + dates : list or np.ndarray + The list of dates. + times : tuple[int, int] + The time of the acquisition (hour, minute). + email : str + The email address to receive the gacos data. + + Returns + ------- + post_data : dict + The post data. + """ + if isinstance(dates, np.ndarray): + dates = dates.tolist() + times = [int(t) for t in times] + + post_data = { + "N": self.bounds[3], + "W": self.bounds[0], + "S": self.bounds[1], + "E": self.bounds[2], + "H": times[0], + "M": times[1], + "date": "\n".join(dates), + "type": "2", + "email": email, + } + return post_data + + +class LiCSARDataset(SarDataset): + def __init__( + self, + home_dir: Union[Path, str], + gacos_dir: Optional[Union[Path, str]] = None, + ) -> None: + """Initialize LiCSARDataset class + + Parameters + ---------- + home_dir : Union[Path, str] + The home directory of LiCSAR dataset. + gacos_dir : Optional[Union[Path, str]], optional + The directory used to save gacos data. Used to check if the data is + already downloaded and avoid resubmitting. Default is None. + """ + self.home_dir = Path(home_dir) + self.dataset = LiCSAR(home_dir) + bounds = self.dataset.bounds + time = self._get_time() + dates = self.dataset.pairs.dates + date_times = pd.to_datetime([f"{d} {time[0]}:{time[1]}:00" for d in dates]) + super().__init__(bounds, date_times, gacos_dir) + + def _get_time(self): + """Get the acquisition time of acquisitions. + + Returns + ------- + time: tuple[int, int] + A tuple of hour and minute representing the acquisition time. + + Raises + ------ + ValueError + If no center_time found in metadata.txt. + """ + meta_file = sorted(self.home_dir.rglob("metadata.txt"))[0] + + with open(meta_file) as f: + lines = f.readlines() + time = None + for line in lines: + line_split = line.split("=") + key, value = (line_split[0].strip(), line_split[1]) + if "center_time" == key: + center_time = value.strip() + hour, minute, second = center_time.split(":") + hour, minute, second = int(hour), int(minute), float(second) + minute = minute + int(np.round(second / 60, 0)) + return hour, minute + else: + continue + if time is None: + raise ValueError(f"No center_time found in {meta_file}") + + +class HyP3Dataset(SarDataset): + def __init__( + self, + home_dir: Union[Path, str], + gacos_dir: Optional[Union[Path, str]] = None, + ) -> None: + """Initialize HyP3Dataset class + + Parameters + ---------- + home_dir : Union[Path, str] + The home directory of HyP3 dataset. + gacos_dir : Optional[Union[Path, str]], optional + The directory used to save gacos data. Used to check if the data is + already downloaded and avoid resubmitting. Default is None. + """ + self.dataset = HyP3(home_dir) + bounds = self.dataset.bounds.to_crs("epsg:4326") + date_times = self.dataset.datetime + + super().__init__(bounds, date_times, gacos_dir) diff --git a/third_party/PyINT/pyint/AutoGACOS/gacos/download.py b/third_party/PyINT/pyint/AutoGACOS/gacos/download.py new file mode 100644 index 0000000..5afcf60 --- /dev/null +++ b/third_party/PyINT/pyint/AutoGACOS/gacos/download.py @@ -0,0 +1,158 @@ +import tarfile +from pathlib import Path +from typing import Optional, Union + +import numpy as np +import pandas as pd +from data_downloader import downloader +from faninsar.query import BoundingBox +from tqdm.auto import tqdm + +from .parse_email import GACOSEmail + + +class Downloader: + def __init__( + self, + url_file: Union[Path, str], + output_dir: Union[Path, str], + tar_gz_dir: Optional[Union[Path, str]] = None, + keep_original: bool = False, + times: Optional[Union[float, list[float]]] = None, + bounds: Optional[tuple[float, float, float, float]] = None, + ) -> None: + """Initialize Downloader class + + Parameters + ---------- + url_file : Union[Path, str] + Path to file containing URLs that created by :meth:`GACOSEmail.retrieve_gacos_urls` + output_dir : Union[Path, str] + directory to output gacos files + tar_gz_dir : Optional[Union[Path, str]], optional + directory to store downloaded *.tar.gz files. If None, then + `output_dir` is used. Default is None. + keep_original : bool, optional + Whether to keep original files (*.tar.gz). Default is False. + times : Optional[float], optional + times of acquisition, used to filter out files that are not needed. + this can be a single time or a list of times. times differ by less + than 10 minutes are considered the same. Default is None. + bounds : Optional[tuple[float, float, float, float]], optional + bounds of area of interest with order (W, S, E, N), used to filter + out files that are not needed. Default is None. + """ + self.url_file = Path(url_file) + self.output_dir = Path(output_dir) + if tar_gz_dir is None: + self.tar_gz_dir = self.output_dir + self.keep_original = keep_original + + if not self.url_file.exists(): + raise FileNotFoundError(f"{self.url_file} does not exist") + if not self.output_dir.exists(): + self.output_dir.mkdir(parents=True) + if not self.tar_gz_dir.exists(): + self.tar_gz_dir.mkdir(parents=True) + + self.df_urls = pd.read_csv(self.url_file, header=0) + + # only keep urls that intersect with bounds + if bounds is not None: + mask_bbox = self._bbox_mask(BoundingBox(*bounds)) + + # only keep urls that acquisition time is within 10 minutes of `time` + if times is not None: + if isinstance(times, float): + times = [times] + mask_time = self._time_mask(times) + + if bounds is not None and times is not None: + self.mask = mask_bbox & mask_time + elif bounds is not None: + self.mask = mask_bbox + elif times is not None: + self.mask = mask_time + else: + self.mask = np.ones(self.df_urls.shape[0], dtype=bool) + + self.mask = self.mask & self.date_mask + + def _bbox_mask(self, bounds) -> np.ndarray: + intersection_bbox = np.array( + [ + BoundingBox(*b).intersects(bounds) + for b in zip( + self.df_urls["south"].astype(float), + self.df_urls["west"].astype(float), + self.df_urls["north"].astype(float), + self.df_urls["east"].astype(float), + ) + ] + ) + return intersection_bbox + + def _time_mask(self, times) -> np.ndarray: + """Only keep urls that acquisition time is within 10 minutes of `time`""" + intersection_times = [] + for time in times: + intersection_times.append( + np.array( + np.abs(self.df_urls["time"].astype(float) - time) + <= 1 / 60 * 10 # 10 minutes + ) + ) + intersection_time = np.any(intersection_times, axis=0) + return intersection_time + + @property + def date_mask(self) -> np.ndarray: + """Remove urls that all acquisition dates have been downloaded""" + dates_urls = self.df_urls["date"].map(lambda x: eval(x)) + intersection_dates = [] + for dt_url in dates_urls: + intersection_dates.append(~np.all(np.isin(dt_url, self.dates_downloaded))) + return np.array(intersection_dates) + + @property + def dates_downloaded(self) -> np.ndarray: + """Return dates that have been downloaded""" + gacos_files = list(self.output_dir.rglob("*.ztd.tif")) + dates = [] + for i in gacos_files: + stem = i.stem.split(".")[0] + if len(stem) == 8: + dates.append(stem) + return np.array(dates) + + def download(self) -> None: + """Download GACOS files from URLs in file created by :meth:`GACOSEmail.retrieve_gacos_urls`""" + urls_used = self.df_urls[self.mask]["url"].values + + for url in tqdm(urls_used, unit="file", desc="Downloading GACOS files"): + gz_file = self.tar_gz_dir / Path(url).name + downloader.download_data(url, file_name=gz_file) + self._extract_tar_gz(gz_file) + if not self.keep_original: + self._delete_file(gz_file) + + def _extract_tar_gz(self, gz_file) -> None: + """Unzip/extract downloaded GACOS files + + Parameters + ---------- + gz_file : Path + path to downloaded GACOS file (*.tar.gz) + """ + with tarfile.open(gz_file, "r:gz") as tar: + tar.extractall(path=self.output_dir) + + def _delete_file(self, gz_file) -> None: + """Delete original GACOS files + + Parameters + ---------- + gz_file : Path + path to downloaded GACOS file (*.tar.gz) + """ + gz_file.unlink() diff --git a/third_party/PyINT/pyint/AutoGACOS/gacos/parse_email.py b/third_party/PyINT/pyint/AutoGACOS/gacos/parse_email.py new file mode 100644 index 0000000..f074cdf --- /dev/null +++ b/third_party/PyINT/pyint/AutoGACOS/gacos/parse_email.py @@ -0,0 +1,391 @@ +import email +import email.message +import getpass +import imaplib +import poplib +import re +from email.parser import Parser +from email.utils import parseaddr +from pathlib import Path +from typing import Literal, Optional, Union + +import pandas as pd +from tqdm.auto import tqdm + + +class GACOSEmail: + """a class to retrieve gacos urls from email. + + .. note:: + The IMAP server is used to retrieve content from email. Some email + service providers may need to enable the IMAP service in the settings. + **You are recommended to use a new email account to receive gacos urls to + avoid polluting your own email account**. + """ + + def __init__( + self, + username: str, + password: str, + host: str, + prompt: bool = False, + email_protocol: Literal["imap", "pop3"] = "imap", + port: Optional[int] = None, + gacos_email: str = "gacos2017@foxmail.com", + gacos_suffix: str = "tar.gz", + start_date: Optional[str] = None, + end_date: Optional[str] = None, + date_args: Optional[dict] = None, + ssl: bool = False, + ) -> None: + """Retrieve gacos urls from email. + + Parameters + ---------- + username : str + The username of the email address. + password : str + The password. + host : str + The host of the email address. For example, the host of gmail for + imap is "imap.gmail.com". You can find the host of your email + settings or search it on the Internet. + prompt: bool, optional + Prompt for username and/or password interactively when they are not + provided as keyword parameters. Default is False. + email_protocol : str, one of ["imap", "pop3"], optional + The protocol of the email. Default is "imap". + port : int, optional + The port of the host of your email. If None, the default port will be + used. Default is None. + gacos_email : str, optional + The email address of gacos. Default is "gacos2017@foxmail.com". + gacos_suffix : str, optional + The suffix of the gacos file url. Default is "tar.gz". The suffix is used + to filter urls in the email. This parameter is used to avoid the + situation that the email contains other urls. + start_date / end_date: str, optional + The start/end date of email. Used to filter the email. Default is None. Can be any format that can be parsed by pandas.to_datetime. + date_args : dict, optional + The arguments are passed to pandas.to_datetime. Default is None. + ssl : bool, optional + Whether to use SSL connection. Default is False. + """ + if prompt: + self.username = None + self.password = None + else: + self.username = username + self.password = password + self.host = host + self.email_protocol = email_protocol + self.port = port + self.gacos_email = gacos_email + self.gacos_suffix = gacos_suffix + self.ssl = ssl + + # date part + self.start_date = start_date + self.end_date = end_date + if date_args is None: + date_args = {} + self.date_args = date_args + + def _retrieve_gacos_urls_pop3(self): + server = login_in_email_pop3( + self.username, self.password, self.host, self.port, ssl=self.ssl + ) + print(server.getwelcome()) + + nums = server.stat()[0] + + gacos = [] + for i in tqdm(range(1, nums + 1), unit=" emails", desc="Retrieving GACOS Urls"): + response, msgLines, octets = server.retr(i) + msgLinesToStr = b"\r\n".join(msgLines).decode("utf8", "ignore") + messageObject = Parser().parsestr(msgLinesToStr) + + senderContent = messageObject["From"] + senderRealName, senderAdr = parseaddr(senderContent) + if senderAdr == self.gacos_email: + if not in_date_range( + pd.to_datetime(messageObject["Date"]).tz_localize(None), + self.start_date, + self.end_date, + self.date_args, + ): + continue + + msgBodyContents = get_content(messageObject) + info = parse_gacos_info( + msgBodyContents, + gacos_suffix=self.gacos_suffix, + ) + if info is not None: + gacos.append(info) + + server.quit() + + return gacos + + def _retrieve_gacos_urls_imap(self): + server = login_in_email_imap( + self.username, self.password, self.host, self.port, ssl=self.ssl + ) + if server is None: + print("IMAP server connection failed, skipping email check.") + return [] + server.select("inbox") + status, data = server.search(None, "ALL") + + gacos = [] + for i in tqdm(data[0].split(), unit=" emails", desc="Retrieving GACOS urls"): + res, msg = server.fetch(i, "(RFC822)") + for response_part in msg: + if isinstance(response_part, tuple): + msgLines = response_part[1].decode("utf8", "ignore") + break + + messageObject = Parser().parsestr(msgLines) + + senderContent = messageObject["From"] + senderRealName, senderAdr = parseaddr(senderContent) + if senderAdr == self.gacos_email: + if not in_date_range( + pd.to_datetime(messageObject["Date"]).tz_localize(None), + self.start_date, + self.end_date, + self.date_args, + ): + continue + + msgBodyContents = get_content(messageObject) + info = parse_gacos_info( + msgBodyContents, + gacos_suffix=self.gacos_suffix, + ) + if info is not None: + gacos.append(info) + + server.close() + + return gacos + + def retrieve_gacos_urls( + self, + output_file: Union[str, Path], + ): + """Retrieve gacos urls from username. + + Parameters + ---------- + output_file : str or Path + The output file used to save the gacos urls. + """ + if self.email_protocol == "pop3": + gacos = self._retrieve_gacos_urls_pop3() + elif self.email_protocol == "imap": + gacos = self._retrieve_gacos_urls_imap() + else: + raise ValueError("email_protocol must be 'pop3' or 'imap'.") + + cols = ["url", "south", "north", "west", "east", "time", "date"] + df_gacos = pd.DataFrame(gacos, columns=cols).drop_duplicates(subset="url") + + # save to file + try: + df_gacos.to_csv(output_file) + print(f"Save gacos urls to {output_file}") + except Exception as e: + self.df_gacos = df_gacos + print(e) + print("Save gacos urls failed") + print("You can access the gacos urls by `df_gacos` attribute.") + + +def in_date_range(date, start_date, end_date, date_args={}): + start_date = pd.to_datetime(start_date, **date_args) + end_date = pd.to_datetime(end_date, **date_args) + start_none = start_date is None or pd.isna(start_date) + end_none = end_date is None or pd.isna(end_date) + if start_none and end_none: + return True + elif start_none: + return date <= end_date + elif end_none: + return date >= start_date + else: + return (date >= start_date) and (date <= end_date) + + +def decodeBody(msgPart: email.message.Message): + """decode email body + + Parameters + ---------- + msgPart : email.message.Message + The email message object. + """ + contentType = msgPart.get_content_type() + textContent = "" + if contentType == "text/plain" or contentType == "text/html": + content = msgPart.get_payload(decode=True) + charset = msgPart.get_charset() + if charset is None: + contentType = msgPart.get("Content-Type", "").lower() + position = contentType.find("charset=") + if position >= 0: + charset = contentType[position + 8 :].strip() + if charset: + textContent = content.decode(charset) + return textContent + + +def get_content(messageObject): + msgBodyContents = [] + if messageObject.is_multipart(): # parse multipart email + messageParts = messageObject.get_payload() + for messagePart in messageParts: + bodyContent = decodeBody(messagePart) + if bodyContent: + msgBodyContents.append(bodyContent) + else: + bodyContent = decodeBody(messageObject) + if bodyContent: + msgBodyContents.append(bodyContent) + return msgBodyContents + + +def login_in_email_pop3(username, password, host, port, ssl=False): + try: + if username is None: + username = input("username: ") + if password is None: + password = getpass.getpass("password: ") + + if ssl: + if port is None: + port = 995 + server = poplib.POP3_SSL(host, port) + else: + if port is None: + port = 110 + server = poplib.POP3(host, port) + + server.user(username) + server.pass_(password) + return server + except Exception as e: + print(e) + print("login failed") + + +def login_in_email_imap(username, password, host, port, ssl=False): + try: + if username is None: + username = input("username: ") + if password is None: + password = getpass.getpass("password: ") + + if ssl: + if port is None: + port = 993 + else: + if port is None: + port = 143 + + # 强制 IPv4: monkey-patch getaddrinfo，避免 IPv6 不可达 + import socket as _socket + _orig_getaddrinfo = _socket.getaddrinfo + def _ipv4_only_getaddrinfo(*args, **kwargs): + return _orig_getaddrinfo(args[0], args[1], _socket.AF_INET, + *args[3:], **kwargs) + _socket.getaddrinfo = _ipv4_only_getaddrinfo + try: + if ssl: + server = imaplib.IMAP4_SSL(host, port) + else: + server = imaplib.IMAP4(host, port) + finally: + _socket.getaddrinfo = _orig_getaddrinfo + + server.login(username, password) + + # 163/126 等网易邮箱要求登录后发送 ID 命令才能执行 SELECT + if '163.com' in host or '126.com' in host or 'yeah.net' in host: + try: + tag = server._new_tag() + server.send(tag + b' ID ("name" "pyint" "version" "1.0" ' + b'"vendor" "pyint")\r\n') + while True: + resp = server.readline() + if resp.startswith(tag): + break + except Exception: + pass + + return server + except Exception as e: + print(f"IMAP login failed: {e}") + return None + + +def parse_gacos_info(msgBodyContents, gacos_suffix="tar.gz"): + """Parse gacos info from email body. + + Parameters + ---------- + msgBodyContents : list + The email body contents. + gacos_suffix : str, optional + The suffix of the gacos file url. Default is "tar.gz". The suffix is used + to filter urls in the email. This parameter is used to avoid the + situation that the email contains other urls. + """ + + url, south, north, west, east, _time, date_list = ( + None, + None, + None, + None, + None, + None, + None, + ) + date_list = [] + for contents in msgBodyContents: + lines = [i.strip() for i in contents.split("\n") if i] + for line in lines: + line = line.strip() + loc = line.split("=") + if len(loc) == 2: + parameter, value = loc + parameter, value = (parameter.strip(), value.strip()) + if "MinLat" == parameter: + south = float(value) + if "MaxLat" == parameter: + north = float(value) + if "MinLon" == parameter: + west = float(value) + if "MaxLon" == parameter: + east = float(value) + loc = line.split(":") + if len(loc) == 2: + parameter, value = loc + parameter, value = (parameter.strip(), value.strip()) + if "Time" == parameter: + _time = float(value) + + if len(line) == 8 and line.isdigit(): + date_list.append(line) + + for i in ["http", "ftp", "https"]: + result = re.search(f"\({i}.*{gacos_suffix}\)", line) + if result: + url = result.group()[1:-1] + break + + if url == south == north == west == east == _time: + return None + else: + return url, south, north, west, east, _time, date_list diff --git a/third_party/PyINT/pyint/AutoGACOS/gacos/submit.py b/third_party/PyINT/pyint/AutoGACOS/gacos/submit.py new file mode 100644 index 0000000..ce1d235 --- /dev/null +++ b/third_party/PyINT/pyint/AutoGACOS/gacos/submit.py @@ -0,0 +1,83 @@ +import time +from pathlib import Path +from typing import Optional, Union + +import numpy as np +import requests +from tqdm.auto import tqdm + +from .datasets import SarDataset + + +class Submitter: + def __init__( + self, + dataset: SarDataset, + email: str, + sleep_time_range: tuple[int, int] = (60 / 2, 60 * 5), + gacos_url="http://www.gacos.net/M/action_page.php", + ) -> None: + """Initialize Submitter class + + Parameters + ---------- + dataset : SarDataset + The SarDataset object. + email : str + The email address to submit to gacos. + sleep_time_range : tuple[int, int], optional + The range of sleep time in seconds. Default is (60, 60 * 5). + gacos_url : str, optional + The url of gacos website. Default is "http://www.gacos.net/M/action_page.php". + """ + self.dataset = dataset + self.email = email + self.sleep_time_range = sleep_time_range + self.gacos_url = gacos_url + + self._failed = [] + self._succeed = [] + + def _post_data(self, data): + """Post data to gacos website.""" + r = requests.post(self.gacos_url, data=data) + return "Thanks for using GACOS!" in r.text + + def post_requests(self): + # post gacos info to website + datetime_patches = self.dataset.gen_datetime_patches() + for _key, _dates in tqdm( + datetime_patches.items(), + desc="submitting times", + unit="times", + ): + try: + for _dt in tqdm(_dates, desc="submitting dates", unit="dates"): + post_data = self.dataset.gen_post_data( + _dt, _key.split(":"), self.email + ) + status_ok = self._post_data(post_data) + if status_ok: + self._succeed.append(post_data) + tqdm.write(f">>> succeed post: {post_data}") + else: + self._failed.append(post_data) + tqdm.write(f">>> failed post: {post_data}") + + # wait to avoid be rejected + sleep_time = np.random.randint(*self.sleep_time_range) + tqdm.write(f" sleeping for {sleep_time} seconds...") + time.sleep(sleep_time) + except: + self._failed.append(post_data) + tqdm.write(f">>> failed post: {post_data}") + + @property + def failed(self): + """A list of failed post data.""" + return self._failed + + @property + def succeed(self): + """A list of succeed post data.""" + return self._succeed diff --git a/third_party/PyINT/pyint/AutoGACOS/pyproject.toml b/third_party/PyINT/pyint/AutoGACOS/pyproject.toml new file mode 100644 index 0000000..c1a0f62 --- /dev/null +++ b/third_party/PyINT/pyint/AutoGACOS/pyproject.toml @@ -0,0 +1,23 @@ +[project] +name = "AutoGACOS" +authors = [{name = "Fancy", email = "fanchengyan@outlook.com"}] +description = "A Python library for automatically submitting and downloading GACOS data" +version = "0.1.0" +requires-python = ">=3.6" +dependencies = [ + "data_downloader", + "faninsar", +] +readme = "README.md" +license = {file = "LICENSE"} +keywords = ["GACOS", "data", "downloader"] + + +[project.optional-dependencies] +dev = [ + "pytest" +] + +[project.urls] +Homepage = "https://github.com/Fanchengyan/AutoGACOS" +Repository = "https://github.com/Fanchengyan/AutoGACOS" diff --git a/third_party/PyINT/pyint/DEM_DOWNLOAD_GUIDE.md b/third_party/PyINT/pyint/DEM_DOWNLOAD_GUIDE.md new file mode 100644 index 0000000..ecabdcf --- /dev/null +++ b/third_party/PyINT/pyint/DEM_DOWNLOAD_GUIDE.md @@ -0,0 +1,265 @@ +# DEM 数据下载指南 + +本文档介绍如何使用 `makedem.py` 下载和处理不同来源的 DEM 数据。 + +## 支持的 DEM 数据源 + +### 1. Copernicus DEM (默认推荐) + +**特点:** +- 分辨率: 30米 +- 覆盖: 全球 (60°S - 80°N) +- 数据质量: 高精度,全球一致性 +- 下载方式: 自动从 AWS/OpenTopography 下载 +- 费用: 免费 + +**使用方法:** +```bash +# 基本使用 +python makedem.py -r 116/117/39/40 --dem-source copernicus + +# 指定并行下载数量(加速下载) +python makedem.py -r 116/117/39/40 --dem-source copernicus --num-workers 8 + +# 指定输出目录 +python makedem.py -r 116/117/39/40 --dem-source copernicus --dir /path/to/output +``` + +**优点:** +- ✓ 全自动下载 +- ✓ 高精度数据 +- ✓ 全球覆盖 +- ✓ 支持并行下载加速 + +**缺点:** +- ✗ 需要网络连接 +- ✗ 下载时间取决于网络速度 + +--- + +### 2. NASADEM + +**特点:** +- 分辨率: 30米 +- 覆盖: 全球 (60°S - 60°N) +- 数据质量: SRTM + ICESat 数据融合 +- 下载方式: 通过 NASADEM Python 库自动下载 +- 费用: 免费 + +**依赖安装:** +```bash +pip install nasadem rasters +``` + +**使用方法:** +```bash +# 基本使用 +python makedem.py -r 116/117/39/40 --dem-source nasadem + +# 指定输出目录 +python makedem.py -r 116/117/39/40 --dem-source nasadem --dir /path/to/output +``` + +**优点:** +- ✓ 全自动下载 +- ✓ NASA官方数据 +- ✓ 融合多源数据,精度提升 + +**缺点:** +- ✗ 需要网络连接 +- ✗ 首次下载可能较慢 +- ✗ 需要额外安装库 +- ✗ 覆盖范围小于 Copernicus (南北纬60°限制) + +--- + +### 3. SRTM (需要预下载数据) + +**特点:** +- 分辨率: 30米 (SRTM 1 arc-second) 或 90米 (SRTM 3 arc-second) +- 覆盖: 全球 (60°S - 60°N) +- 数据质量: 良好 +- 下载方式: 需要手动预下载 .hgt 文件 +- 费用: 免费 + +**依赖安装:** +```bash +# 注意: srtm库需要 Python >= 3.12 +pip install srtm + +# 检查Python版本 +python --version # 应该显示 3.12 或更高版本 +``` + +**Python版本要求:** +- srtm 库要求 Python >= 3.12 +- 如果您的Python版本 < 3.12，建议使用 Copernicus 或 NASADEM + +**数据准备步骤:** + +1. **下载 SRTM .hgt 文件** + + 从以下网站之一下载所需区域的 .hgt 或 .hgt.zip 文件: + - **CSI-CGIAR SRTM**: https://srtm.csi.cgiar.org/ (推荐,无需注册) + - **USGS EarthExplorer**: https://earthexplorer.usgs.gov/ (需要注册) + - **NASA EarthData**: https://urs.earthdata.nasa.gov/ (需要注册) + +2. **文件命名规则** + + SRTM 文件按 1°×1° 分块命名: + - `N39E116.hgt` → 北纬39-40°, 东经116-117° + - `N39W110.hgt` → 北纬39-40°, 西经110-109° + - `S10E120.hgt` → 南纬10-9°, 东经120-121° + +3. **数据组织** + + 将所有 .hgt 文件放在同一目录下,例如: + ``` + /path/to/srtm_data/ + ├── N39E116.hgt + ├── N39E117.hgt + ├── N40E116.hgt + └── N40E117.hgt + ``` + +**使用方法:** +```bash +# 指定SRTM数据目录 +python makedem.py -r 116/117/39/40 --dem-source srtm --srtm-data-dir /path/to/srtm_data + +# 注意:如果不指定 --srtm-data-dir,程序会报错并提示 +python makedem.py -r 116/117/39/40 --dem-source srtm +# 错误: 使用 SRTM 数据源需要指定 --srtm-data-dir 参数 +``` + +**优点:** +- ✓ 离线使用(数据预下载后) +- ✓ 不依赖实时网络下载 +- ✓ 数据源多样化 + +**缺点:** +- ✗ 需要手动下载数据文件 +- ✗ 需要自行管理数据文件 +- ✗ 需要确保覆盖目标区域的所有分块 +- ✗ srtm 库对 .hgt.zip 压缩文件支持不稳定,建议解压后使用 + +--- + +## 使用建议 + +### 推荐使用顺序: + +1. **首选: Copernicus DEM** + - 全自动,高精度,全球覆盖 + - 使用 `--num-workers 8` 加速下载 + +2. **备选: NASADEM** + - 全自动,NASA官方数据 + - 适合北纬60°以南,南纬60°以北区域 + +3. **离线环境: SRTM** + - 仅在无网络环境或已有SRTM数据时使用 + - 需要提前准备数据文件 + +### 性能对比: + +| DEM源 | 下载速度 | 数据精度 | 全球覆盖 | 离线使用 | 推荐指数 | +|-------|---------|---------|---------|---------|---------| +| Copernicus | ★★★★☆ | ★★★★★ | ★★★★★ | ✗ | ★★★★★ | +| NASADEM | ★★★☆☆ | ★★★★☆ | ★★★☆☆ | ✗ | ★★★★☆ | +| SRTM | N/A (离线) | ★★★★☆ | ★★★☆☆ | ✓ | ★★★☆☆ | + +### 常见问题: + +**Q: 如何确定需要下载哪些 SRTM 文件?** + +A: 根据目标区域范围计算所需文件: +```python +# 示例: 区域 116/117/39/40 (东经116-117°, 北纬39-40°) +west, east, south, north = 116, 117, 39, 40 + +# 需要下载的文件: +# N39E116.hgt (39-40°N, 116-117°E) +# 如果区域更大,需要下载多个文件 +``` + +**Q: SRTM .hgt.zip 文件可以直接使用吗?** + +A: srtm 库对压缩文件支持不稳定,建议先解压: +```bash +cd /path/to/srtm_data/ +unzip "*.hgt.zip" +``` + +**Q: 如何加速 Copernicus DEM 下载?** + +A: 增加并行下载数量: +```bash +# 默认使用 4 个线程 +python makedem.py -r 116/117/39/40 --dem-source copernicus --num-workers 4 + +# 加速到 8-16 个线程(根据网络情况调整) +python makedem.py -r 116/117/39/40 --dem-source copernicus --num-workers 16 +``` + +**Q: 下载的数据格式是什么?** + +A: 所有DEM源最终都会转换为 GAMMA/ROI_PAC 格式: +- `.dem` 文件: 二进制高程数据 (big-endian for GAMMA) +- `.dem.par` 文件: 参数文件 + +--- + +## 示例工作流程 + +### 场景1: 处理Sentinel-1数据(推荐Copernicus) + +```bash +# 1. 确定研究区域范围 +# 使用 SLC 参数文件自动确定 +python makedem.py -s /path/to/slc.par --dem-source copernicus --num-workers 8 + +# 或手动指定区域 +python makedem.py -r 116/117/39/40 --dem-source copernicus --num-workers 8 +``` + +### 场景2: 离线处理(使用SRTM) + +```bash +# 1. 提前下载SRTM数据 +# 从 https://srtm.csi.cgiar.org/ 下载 N39E116.hgt, N39E117.hgt 等 + +# 2. 组织数据 +mkdir -p ~/data/SRTM +mv N*.hgt ~/data/SRTM/ + +# 3. 使用预下载数据生成DEM +python makedem.py -r 116/117/39/40 --dem-source srtm --srtm-data-dir ~/data/SRTM +``` + +### 场景3: NASADEM对比测试 + +```bash +# 下载Copernicus DEM +python makedem.py -r 116/117/39/40 --dem-source copernicus --dir ./copernicus + +# 下载NASADEM +python makedem.py -r 116/117/39/40 --dem-source nasadem --dir ./nasadem + +# 对比两个DEM文件 +# 使用 GMT 或其他工具进行可视化和分析 +``` + +--- + +## 参考资料 + +- **Copernicus DEM**: https://copernicus-dem-90m.s3.amazonaws.com/readme.html +- **NASADEM**: https://github.com/DFS-iData/NASADEM +- **SRTM CSI-CGIAR**: https://srtm.csi.cgiar.org/ +- **GAMMA Software**: https://gamma-rs.ch/ + +--- + +**最后更新**: 2026-03-13 +**作者**: iFlow CLI diff --git a/third_party/PyINT/pyint/Down2SLC_ALOS.py b/third_party/PyINT/pyint/Down2SLC_ALOS.py new file mode 100644 index 0000000..f90aadf --- /dev/null +++ b/third_party/PyINT/pyint/Down2SLC_ALOS.py @@ -0,0 +1,193 @@ +#! /usr/bin/env python +#''' +################################################################################## +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : June, 2019 # +# # +# Generate SLC from SAR_IMS_P1 data # +# # +################################################################################## +#''' +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +from pyint import _utils as ut + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close + +def rm(TXT): + call_str = 'rm ' + TXT + os.system(call_str) + +def usage(): + print(''' +****************************************************************************************************** + + Generate SLC and SLC_par file for ERS/ENVISAT (SAR_IMS_1P format) + + usage: + + Down2SLC_ERS.py ProjectName DownName + + e.g. Down2SLC_ERS.py CotopaxiT120ERSA 910101 + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==3: + projectName = sys.argv[1] + Date = sys.argv[2] + else: + usage();sys.exit(1) + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + slcDir = scratchDir + '/' + projectName + '/SLC' + workflow = downDir + '/' + Date + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + if not os.path.isdir(slcDir): + call_str= 'mkdir ' +slcDir + os.system(call_str) + os.chdir(workflow) + tempfile=workflow + '/' + 'workreport' + if os.path.isfile(tempfile): + tempDict=ut.update_template(tempfile) + CEOS_SAR_leader = tempDict['Pdi_L10ProductFileName02'].split('"')[1] + CEOS_raw_data = tempDict['Pdi_L10ProductFileName03'].split('"')[1] + Date0 = Date +# if len(Date)==6: +# Date0 = Date +# elif len(Date)==8: +# Date0 = Date[2:8] + #Date0=Date +# else: +# print('The input Date is invalid.') +# sys.exit(1) + + #FileDir = downDir + '/' + downName + SAR_par='palsar_' +Date0 + '.par' + PROC_par='p' + Date0 + '.slc.par' + raw_out=Date0 + '.raw' + plot_data=Date0 +'.mlcc' + rspec_data=Date0 +'.rspec' + doppler_data=Date0 + '.dop' + rc_data=Date0 + '.rc' + autof_data=Date0 +'.autof' + slc_data=Date0 +'.slc' + slc_par=Date0 +'.slc.par' + call_str = 'PALSAR_proc ' + CEOS_SAR_leader + ' ' + SAR_par +' '+ PROC_par + ' '+ CEOS_raw_data + ' ' + raw_out + ' 0 0' + os.system(call_str) + palsar_ant_data='/home/chen/Software/InSAR/GAMMA/GAMMA_SOFTWARE-20180704/MSP/sensors/palsar_ant_20061024.dat' + call_str = 'cp ' + palsar_ant_data + ' .' + os.system(call_str) + call_str = 'PALSAR_antpat' + ' ' + SAR_par +' '+ PROC_par + ' ' + palsar_ant_data + ' palsar_antpat_msp.dat' + os.system(call_str) + call_str = 'doppler ' + SAR_par +' '+ PROC_par + ' ' + raw_out + ' ' + plot_data + os.system(call_str) + call_str = 'doppler ' + SAR_par +' '+ PROC_par + ' ' + raw_out + ' ' + doppler_data + os.system(call_str) + call_str = 'rspec_IQ ' + SAR_par +' '+ PROC_par + ' ' + raw_out + ' ' + rspec_data + os.system(call_str) + #call_str = 'rspec_JERS ' + SAR_par +' '+ PROC_par + ' ' + CEOS_raw_data + ' ' + rspec_data + ' - - - - - -' + #os.system(call_str) + call_str = 'pre_rc ' + SAR_par +' '+ PROC_par + ' ' + raw_out + ' ' + rc_data + os.system(call_str) + call_str = 'autof ' + SAR_par +' '+ PROC_par + ' ' + rc_data + ' ' + autof_data +' 5.0' + os.system(call_str) + call_str = 'autof ' + SAR_par +' '+ PROC_par + ' ' + rc_data + ' ' + autof_data +' 5.0' + os.system(call_str) + call_str = 'az_proc ' + SAR_par +' '+ PROC_par + ' ' + rc_data + ' ' + slc_data +' 16384' + os.system(call_str) + call_str = 'par_MSP ' + SAR_par +' '+ PROC_par + ' ' + slc_par + os.system(call_str) + else: + print('No data is found for date:' + Date) + sys.exit(1) + + call_str ="rename 's/VV.SLC/slc/g' *" + os.system(call_str) + + Date0 = Date + + dataDir = slcDir + '/' + Date0 + if not os.path.isdir(dataDir): + call_str = 'mkdir ' + dataDir + print('Generate SLC dir for date: ' + Date0) + os.system(call_str) + call_str = 'mv ' + Date0 + '.slc* ' + dataDir + os.system(call_str) + print("Down to SLC for %s is done! " % Date) + SslcImg = dataDir + '/'+ Date0 + '.slc' + SslcPar = dataDir + '/'+ Date0 + '.slc.par' + + SamprlksImg = dataDir + '/'+ Date0 + '_' + rlks + 'rlks' + '.amp' + SamprlksPar = dataDir + '/'+ Date0 + '_' + rlks + 'rlks' + '.amp.par' + call_str = 'multi_look ' + SslcImg + ' ' + SslcPar + ' ' + SamprlksImg + ' ' + SamprlksPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + + nWidth = ut.read_gamma_par(SamprlksPar, 'read', 'range_samples') + call_str = 'raspwr ' + SamprlksImg + ' ' + str(nWidth) + os.system(call_str) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/third_party/PyINT/pyint/Down2SLC_ASAR_Cat.py b/third_party/PyINT/pyint/Down2SLC_ASAR_Cat.py new file mode 100644 index 0000000..5594a52 --- /dev/null +++ b/third_party/PyINT/pyint/Down2SLC_ASAR_Cat.py @@ -0,0 +1,307 @@ +#! /usr/bin/env python +#''' +################################################################################## +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : June, 2019 # +# # +# Generate SLC from SAR_IMS_P1 data # +# # +################################################################################## +#''' +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close + +def rm(TXT): + call_str = 'rm ' + TXT + os.system(call_str) + +def usage(): + print(''' +****************************************************************************************************** + + Generate SLC and SLC_par file for ERS/ENVISAT (SAR_IMS_1P format) + + usage: + + Down2SLC_ASAR_Cat.py ProjectName DownName + + e.g. Down2SLC_ASAR_Cat.py CotopaxiT120ERSA 910101 + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==3: + projectName = sys.argv[1] + Date = sys.argv[2] + else: + usage();sys.exit(1) + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + '/' + projectName + '.template' + templateContents=read_template(templateFile) + + if 'rlks4cor' in templateContents: rlks4cor = templateContents['rlks4cor'] + else: rlks4cor = '4' + if 'azlks4cor' in templateContents: azlks4cor = templateContents['azlks4cor'] + else: azlks4cor = '4' + + if 'rwin4cor' in templateContents: rwin4cor = templateContents['rwin4cor'] + else: rwin4cor = '128' + if 'azwin4cor' in templateContents: azwin4cor = templateContents['azwin4cor'] + else: azwin4cor = '128' + if 'rsample4cor' in templateContents: rsample4cor = templateContents['rsample4cor'] + else: rsample4cor = '64' + if 'azsample4cor' in templateContents: azsample4cor = templateContents['azsample4cor'] + else: azsample4cor = '64' + + if ' rpos4cor' in templateContents: rpos4cor = templateContents[' rpos4cor'] + else: rpos4cor = ' - ' + if 'azpos4cor' in templateContents: azpos4cor = templateContents['azpos4cor'] + else: azpos4cor = ' - ' + + + if 'rfwin4cor' in templateContents: rfwin4cor = templateContents['rfwin4cor'] + else: rfwin4cor = str(int(int(rwin4cor)/2)) + if 'azfwin4cor' in templateContents: azfwin4cor = templateContents['azfwin4cor'] + else: azfwin4cor = str(int(int(azwin4cor)/2)) + if 'rfsample4cor' in templateContents: rfsample4cor = templateContents['rfsample4cor'] + else: rfsample4cor = str(2*int(rsample4cor)) + if 'azfsample4cor' in templateContents: azfsample4cor = templateContents['azfsample4cor'] + else: azfsample4cor = str(2*int(azsample4cor)) + + if 'thresh4cor' in templateContents: thresh4cor = templateContents['thresh4cor'] + else: thresh4cor = ' 0.2 ' + + rlks = templateContents['Range_Looks'] + azlks = templateContents['Azimuth_Looks'] + + + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + slcDir = scratchDir + '/' + projectName + '/SLC' + + if not os.path.isdir(slcDir): + call_str= 'mkdir ' +slcDir + os.system(call_str) + os.chdir(downDir) + + t0 = 't0_' + Date + call_str = 'ls >' + t0 + os.system(call_str) + + tt = 'tt_' + Date + call_str = "grep " + Date + ' ' + t0 + '> ' + tt + os.system(call_str) + + te = 'te_' + Date + call_str = "grep ASA_IMS_1P " + tt + " > " + te + os.system(call_str) + + AA= np.loadtxt(te,dtype=np.str) + Na = AA.size + AA=AA.reshape(Na,) + + for i in range(Na): + Date0 = Date+'_' + str(i) + downName = str(AA[i]) + FileDir = downDir + '/' + downName + call_str = 'par_ASAR '+ FileDir + ' ' + Date0 + os.system(call_str) + + call_str ="rename 's/VV.SLC/slc/g' *" + os.system(call_str) + + slcpar = Date0 + '.slc.par' + call_str = 'ASAR_orb_cor_par.py ' + slcpar + os.system(call_str) + + Date0 = Date + if len(Date)==6: + Date0 = Date + Date6 = Date + elif len(Date)==8: + Date0 = Date[2:8] + Date6 = Date[2:8] + else: + print('The input Date is invalid.') + sys.exit(1) + + dataDir = slcDir + '/' + Date0 + if not os.path.isdir(dataDir): + call_str = 'mkdir ' + dataDir + print('Generate SLC dir for date: ' + Date0) + os.system(call_str) + call_str = 'mv ' + Date + '*.slc* ' + dataDir + os.system(call_str) + + os.chdir(dataDir) + for i in range(Na): + Date0 = Date+'_' + str(i) + downName = str(AA[i]) + + SLCm = Date0 + '.slc' + SLCm_par = Date0 + '.slc.par' + MamprlksImg = Date0 + '.amp' + MamprlksPar = Date0 + '.amp.par' + + call_str = 'multi_look ' + SLCm + ' ' + SLCm_par + ' ' + MamprlksImg + ' ' + MamprlksPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + nWidth = UseGamma(MamprlksPar, 'read', 'range_samples') + call_str = 'raspwr ' + MamprlksImg + ' ' + nWidth + os.system(call_str) + + SLC = Date6+'.slc' + SLCPAR = Date6 + '.slc.par' + AMP = Date6+'.amp' + AMPPAR = Date6+'.amp.par' + if Na==1: + os.rename(SLCm,SLC) + os.rename(SLCm_par,SLCPAR) + os.rename(MamprlksImg,AMP) + os.rename(MamprlksPar,AMPPAR) + + + for i in range(Na-1): + if i==0: + DateA = Date + '_0' + else: + DateA = Date + '_' + str(i-1) + str(i) + + SLCA = DateA + '.slc' + SLCA_par = DateA + '.slc.par' + + + DateB = Date + '_' +str(i+1) + SLCB = DateB + '.slc' + SLCB_par = DateB + '.slc.par' + + DateC = Date + '_' + str(i) + str(i+1) + SLCC = DateC + '.slc' + SLCC_par = DateC + '.slc.par' + + ##################################################################################### + MamprlksImg = Date + '.amp' + MamprlksPar = Date + '.amp.par' + + off = DateC + '.off' + offs = DateC + '.offs' + offsets = DateC + '.offsets' + coffs = DateC + '.coffs' + coffsets = DateC + '.coffsets' + snr = DateC + '.snr' + off_std = DateC + '.off_std' + + ########################## Generate off file ############################# + + call_str = "create_offset " + SLCA_par + " " + SLCB_par + " " + off + " 1 - - 0" + os.system(call_str) + call_str = 'init_offset_orbit '+ SLCA_par + " " + SLCB_par + ' ' + off + os.system(call_str) + + + call_str = 'init_offset '+ SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' ' + rlks4cor + ' ' + azlks4cor + ' ' + rpos4cor + ' ' + azpos4cor + os.system(call_str) + + call_str = 'init_offset '+ SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' 1 1 - - ' + os.system(call_str) + + call_str = "offset_pwr " + SLCA + " " + SLCB + " " + SLCA_par + " " + SLCB_par + " " + off + " " + offs + " " + snr + " " + rwin4cor + " " + azwin4cor + " " + offsets + " 2 "+ rsample4cor + " " + azsample4cor + os.system(call_str) + + call_str = "offset_fit " + offs + " " + snr + " " + off + " " + coffs + " " + coffsets + " " + thresh4cor +" 3" + os.system(call_str) + + call_str = "offset_pwr " +SLCA + " " + SLCB + " " + SLCA_par + " " + SLCB_par + " " + off + " " + offs + " " + snr + " " + rfwin4cor + " " + azfwin4cor + " " + offsets + " 2 " + rfsample4cor + " " + azfsample4cor + os.system(call_str) + + call_str = "offset_fit " + offs + " " + snr + " " + off + " " + coffs + " " + coffsets + " " + thresh4cor + " 4 >" + off_std + os.system(call_str) + + ######################################################################################## + + call_str = 'SLC_cat ' + SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' ' + SLCC + ' ' + SLCC_par + os.system(call_str) + + if i==(Na-2): + if len(Date)==6: + DD = Date + else: + DD = Date[2:8] + SLCm = DD + '.slc' + SLCm_par = DD + '.slc.par' + + + call_str = 'cp ' + SLCC + ' ' + SLCm + os.system(call_str) + + call_str = 'cp ' + SLCC_par + ' ' + SLCm_par + os.system(call_str) + + call_str = 'multi_look ' + SLCm + ' ' + SLCm_par + ' ' + MamprlksImg + ' ' + MamprlksPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + nWidth = UseGamma(MamprlksPar, 'read', 'range_samples') + call_str = 'raspwr ' + MamprlksImg + ' ' + nWidth + os.system(call_str) + + call_str = 'rm *.amp' + os.system(call_str) + + call_str = 'rm *_*.slc' + os.system(call_str) + + print("Down to SLC for %s is done! " % Date) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/Down2SLC_ASAR_Cat_All.py b/third_party/PyINT/pyint/Down2SLC_ASAR_Cat_All.py new file mode 100644 index 0000000..cd7c04a --- /dev/null +++ b/third_party/PyINT/pyint/Down2SLC_ASAR_Cat_All.py @@ -0,0 +1,136 @@ +#! /usr/bin/env python +#''' +################################################################################## +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : March, 2017 # +# # +# Generate Sentinel SLC from the downloaded data # +# # +################################################################################## +#''' +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close + +def rm(TXT): + call_str = 'rm ' + TXT + os.system(call_str) + +def usage(): + print(''' +****************************************************************************************************** + + Downloading Sentinel-1A/B data based on ssara + + usage: + + Down2SLC_ERS_Cat_All.py ProjectName + + e.g. Down2SLC_ERS_Cat_All.py CotopaxiT120ERSA + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==2: + projectName = sys.argv[1] + else: + usage();sys.exit(1) + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + slcDir = scratchDir + '/' + projectName + '/SLC' + + if not os.path.isdir(slcDir): + call_str= 'mkdir ' +slcDir + os.system(call_str) + + os.chdir(downDir) + + call_str = 'ls > tt0' + os.system(call_str) + + call_str = 'grep ASA_IMS_1P tt0 >tt1' + os.system(call_str) + + call_str = "awk -F_ '{print $3}' tt1 > tt2 " + os.system(call_str) + + call_str = "awk -FSA '{print $2}' tt2 > ttt" + os.system(call_str) + + call_str = 'sort ttt | uniq > ttm' + os.system(call_str) + + AA= np.loadtxt('ttm',dtype=np.str) + Na = AA.size + + for i in range(Na): + call_str = 'Down2SLC_ASAR_Cat.py ' + projectName + ' ' + AA[i] + print(call_str) + call_str = 'Down2SLC_ASAR_Cat.py ' + projectName + ' ' + AA[i] + ' >/dev/null' + os.system(call_str) + + + print("Down to SLC for %s is done! " % projectName) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/third_party/PyINT/pyint/Down2SLC_ERS.py b/third_party/PyINT/pyint/Down2SLC_ERS.py new file mode 100644 index 0000000..868a153 --- /dev/null +++ b/third_party/PyINT/pyint/Down2SLC_ERS.py @@ -0,0 +1,155 @@ +#! /usr/bin/env python +#''' +################################################################################## +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : June, 2019 # +# # +# Generate SLC from SAR_IMS_P1 data # +# # +################################################################################## +#''' +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close + +def rm(TXT): + call_str = 'rm ' + TXT + os.system(call_str) + +def usage(): + print(''' +****************************************************************************************************** + + Generate SLC and SLC_par file for ERS/ENVISAT (SAR_IMS_1P format) + + usage: + + Down2SLC_ERS.py ProjectName DownName + + e.g. Down2SLC_ERS.py CotopaxiT120ERSA 910101 + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==3: + projectName = sys.argv[1] + Date = sys.argv[2] + else: + usage();sys.exit(1) + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + slcDir = scratchDir + '/' + projectName + '/SLC' + + if not os.path.isdir(slcDir): + call_str= 'mkdir ' +slcDir + os.system(call_str) + os.chdir(downDir) + + t0 = 't0_' + Date + call_str = 'ls >' + t0 + os.system(call_str) + + tt = 'tt_' + Date + call_str = "grep " + Date + ' ' + t0 + '> ' + tt + os.system(call_str) + + te = 'te_' + Date + call_str = "grep SAR_IMS_1P " + tt + " > " + te + os.system(call_str) + + AA= np.loadtxt(te,dtype=np.str) + Na = AA.size + + if Na > 0: + downName = str(AA[0]) + FileDir = downDir + '/' + downName + call_str = 'par_ASAR '+ FileDir + ' ' + Date + os.system(call_str) + else: + print('No data is found for date:' + Date) + sys.exit(1) + + call_str ="rename 's/VV.SLC/slc/g' *" + os.system(call_str) + + Date0 = Date + if len(Date)==6: + Date0 = Date + elif len(Date)==8: + Date0 = Date[2:8] + else: + print('The input Date is invalid.') + sys.exit(1) + + dataDir = slcDir + '/' + Date0 + if not os.path.isdir(dataDir): + call_str = 'mkdir ' + dataDir + print('Generate SLC dir for date: ' + Date0) + os.system(call_str) + call_str = 'mv ' + Date + '.slc* ' + dataDir + os.system(call_str) + + print("Down to SLC for %s is done! " % Date) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/third_party/PyINT/pyint/Down2SLC_ERS_All.py b/third_party/PyINT/pyint/Down2SLC_ERS_All.py new file mode 100644 index 0000000..2eefb4a --- /dev/null +++ b/third_party/PyINT/pyint/Down2SLC_ERS_All.py @@ -0,0 +1,229 @@ +#! /usr/bin/env python +#''' +################################################################################## +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : March, 2017 # +# # +# Generate Sentinel SLC from the downloaded data # +# # +################################################################################## +#''' +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close + +def rm(TXT): + call_str = 'rm ' + TXT + os.system(call_str) + +def usage(): + print(''' +****************************************************************************************************** + + Downloading Sentinel-1A/B data based on ssara + + usage: + + Down2SLC_Sen_Gamma.py ProjectName DownName + + e.g. Down2SLC_Sen_Gamma.py CotopaxiT120SenVVA 170118 + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==3: + projectName = sys.argv[1] + Date = sys.argv[2] + else: + usage();sys.exit(1) + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + slcDir = scratchDir + '/' + projectName + '/SLC' + + if not os.path.isdir(slcDir): + call_str= 'mkdir ' +slcDir + os.system(call_str) + + os.chdir(downDir) + + + t0 = 't0_' + Date + call_str = 'ls >' + t0 + os.system(call_str) + + tt = 'tt_' + Date + call_str = "grep " + Date + ' ' + t0 + '> ' + tt + os.system(call_str) + + ts = 'ts_' + Date + call_str = "grep SAFE " + tt + ' >' + ts + os.system(call_str) + + tz = 'tz_' + Date + call_str = "grep zip " + tt + " > " + tz + os.system(call_str) + + A1= np.loadtxt(ts,dtype=np.str) + Na1 = A1.size + + A2= np.loadtxt(tz,dtype=np.str) + Na2 = A2.size + + rm(t0);rm(tt);rm(ts);rm(tz) + + if Na1 == 0: + if Na2 > 0: + if Na2 == 1: + downName = str(A2) + else: + downName = str(A2[0]) + FileDir = downDir + '/' + downName + RAWNAME = downName.split('.')[0]+'.SAFE' + call_str = 'unzip '+ FileDir + os.system(call_str) + + else: + if Na1 == 1: + RAWNAME = str(A1) + else: + RAWNAME = str(A1[0]) + + print(RAWNAME) + RAWFILEDir = downDir + '/'+str(RAWNAME) + + + Date = RAWNAME[19:25] + DateDir = slcDir + '/'+Date + + if not os.path.isdir(DateDir): + call_str='mkdir '+DateDir + os.system(call_str) + + measureDir = RAWFILEDir + '/measurement' + annotatDir = RAWFILEDir + '/annotation' + calibraDir = RAWFILEDir + '/annotation/calibration' + + MM = glob.glob(measureDir + '/*vv*tiff') +# MEASURE = glob.glob(measureDir + '/*vv*tiff') +# ANNOTAT = glob.glob(annotatDir + '/*vv*xml' ) +# CALIBRA = glob.glob(calibraDir+'/calibration*vv*') +# NOISE = glob.glob(calibraDir+'/noise*vv*') + + SLC_Tab = DateDir + '/' + Date+'_SLC_Tab' + TEST = DateDir + '/' + Date + '.IW1.slc' + + if not os.path.isfile(TEST): + if os.path.isfile(SLC_Tab): + os.remove(SLC_Tab) + for kk in range(len(MM)): + SLC = DateDir + '/' + Date + '.IW' + str(kk+1)+'.slc' + SLCPar = DateDir + '/' + Date + '.IW' + str(kk+1)+'.slc.par' + TOPPar = DateDir + '/' + Date + '.IW' + str(kk+1)+'.slc.TOPS_par' + BURST = DateDir + '/' + Date + '.IW' + str(kk+1)+'.burst.par' + + if os.path.isfile(BURST): + os.remove(BURST) + call_str = 'echo ' + SLC + ' ' + SLCPar + ' ' + TOPPar + ' >> ' + SLC_Tab + os.system(call_str) + + MEASURE = glob.glob(measureDir + '/*iw' + str(kk+1) + '*vv*tiff') + ANNOTAT = glob.glob(annotatDir + '/*iw' + str(kk+1) + '*vv*xml' ) + CALIBRA = glob.glob(calibraDir+'/calibration*'+ 'iw' + str(kk+1) + '*vv*') + NOISE = glob.glob(calibraDir+'/noise*' + 'iw' + str(kk+1) + '*vv*') + + #call_str = 'S1_burstloc ' + ANNOTAT[0] + '> ' +BURST + #os.system(call_str) + + if int(Date) > 180311: + call_str = 'par_S1_SLC ' + MEASURE[0] + ' ' + ANNOTAT[0] + ' ' + CALIBRA[0] + ' - ' + SLCPar + ' ' + SLC + ' ' + TOPPar + else: + call_str = 'par_S1_SLC ' + MEASURE[0] + ' ' + ANNOTAT[0] + ' ' + CALIBRA[0] + ' ' + NOISE[0] + ' ' + SLCPar + ' ' + SLC + ' ' + TOPPar + + os.system(call_str) + + call_str = 'SLC_burst_corners ' + SLCPar + ' ' + TOPPar + ' > ' +BURST + os.system(call_str) + + TSLC = DateDir + '/' + Date + '.slc' + TSLCPar = DateDir + '/' + Date + '.slc.par' + + TMLI = DateDir + '/' + Date + '_20rlks.amp' + TMLIPar = DateDir + '/' + Date + '_20rlks.amp.par' + + call_str = 'SLC_mosaic_S1_TOPS ' + SLC_Tab + ' ' + TSLC + ' ' + TSLCPar + ' 10 2' + os.system(call_str) + + call_str = 'multi_look ' + TSLC + ' ' + TSLCPar + ' ' + TMLI + ' ' + TMLIPar + ' 20 4' + os.system(call_str) + + nWidth = UseGamma(TMLIPar, 'read','range_samples:') + call_str = 'raspwr ' + TMLI + ' ' + nWidth + ' - - - - - - - ' + os.system(call_str) + + call_str = 'rm -rf ' + RAWFILEDir + os.system(call_str) + + print("Down to SLC for %s is done! " % Date) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/third_party/PyINT/pyint/Down2SLC_ERS_Cat.py b/third_party/PyINT/pyint/Down2SLC_ERS_Cat.py new file mode 100644 index 0000000..1b32f36 --- /dev/null +++ b/third_party/PyINT/pyint/Down2SLC_ERS_Cat.py @@ -0,0 +1,306 @@ +#! /usr/bin/env python +#''' +################################################################################## +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : June, 2019 # +# # +# Generate SLC from SAR_IMS_P1 data # +# # +################################################################################## +#''' +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close + +def rm(TXT): + call_str = 'rm ' + TXT + os.system(call_str) + +def usage(): + print(''' +****************************************************************************************************** + + Generate SLC and SLC_par file for ERS/ENVISAT (SAR_IMS_1P format) + + usage: + + Down2SLC_ERS.py ProjectName DownName + + e.g. Down2SLC_ERS.py CotopaxiT120ERSA 910101 + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==3: + projectName = sys.argv[1] + Date = sys.argv[2] + else: + usage();sys.exit(1) + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + '/' + projectName + '.template' + templateContents=read_template(templateFile) + + if 'rlks4cor' in templateContents: rlks4cor = templateContents['rlks4cor'] + else: rlks4cor = '4' + if 'azlks4cor' in templateContents: azlks4cor = templateContents['azlks4cor'] + else: azlks4cor = '4' + + if 'rwin4cor' in templateContents: rwin4cor = templateContents['rwin4cor'] + else: rwin4cor = '128' + if 'azwin4cor' in templateContents: azwin4cor = templateContents['azwin4cor'] + else: azwin4cor = '128' + if 'rsample4cor' in templateContents: rsample4cor = templateContents['rsample4cor'] + else: rsample4cor = '64' + if 'azsample4cor' in templateContents: azsample4cor = templateContents['azsample4cor'] + else: azsample4cor = '64' + + if ' rpos4cor' in templateContents: rpos4cor = templateContents[' rpos4cor'] + else: rpos4cor = ' - ' + if 'azpos4cor' in templateContents: azpos4cor = templateContents['azpos4cor'] + else: azpos4cor = ' - ' + + + if 'rfwin4cor' in templateContents: rfwin4cor = templateContents['rfwin4cor'] + else: rfwin4cor = str(int(int(rwin4cor)/2)) + if 'azfwin4cor' in templateContents: azfwin4cor = templateContents['azfwin4cor'] + else: azfwin4cor = str(int(int(azwin4cor)/2)) + if 'rfsample4cor' in templateContents: rfsample4cor = templateContents['rfsample4cor'] + else: rfsample4cor = str(2*int(rsample4cor)) + if 'azfsample4cor' in templateContents: azfsample4cor = templateContents['azfsample4cor'] + else: azfsample4cor = str(2*int(azsample4cor)) + + if 'thresh4cor' in templateContents: thresh4cor = templateContents['thresh4cor'] + else: thresh4cor = ' 0.2 ' + + rlks = templateContents['Range_Looks'] + azlks = templateContents['Azimuth_Looks'] + + + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + slcDir = scratchDir + '/' + projectName + '/SLC' + + if not os.path.isdir(slcDir): + call_str= 'mkdir ' +slcDir + os.system(call_str) + os.chdir(downDir) + + t0 = 't0_' + Date + call_str = 'ls >' + t0 + os.system(call_str) + + tt = 'tt_' + Date + call_str = "grep " + Date + ' ' + t0 + '> ' + tt + os.system(call_str) + + te = 'te_' + Date + call_str = "grep SAR_IMS_1P " + tt + " > " + te + os.system(call_str) + + AA= np.loadtxt(te,dtype=np.str) + Na = AA.size + AA=AA.reshape(Na,) + + for i in range(Na): + Date0 = Date+'_' + str(i) + downName = str(AA[i]) + FileDir = downDir + '/' + downName + call_str = 'par_ASAR '+ FileDir + ' ' + Date0 + os.system(call_str) + + call_str ="rename 's/VV.SLC/slc/g' *" + os.system(call_str) + + slcpar = Date0 + '.slc.par' + call_str = 'ERS_orb_cor_par.py ' + slcpar + os.system(call_str) + + Date0 = Date + if len(Date)==6: + Date6 = Date + Date0 = Date + elif len(Date)==8: + Date0 = Date[2:8] + Date6 = Date[2:8] + else: + print('The input Date is invalid.') + sys.exit(1) + + dataDir = slcDir + '/' + Date0 + if not os.path.isdir(dataDir): + call_str = 'mkdir ' + dataDir + print('Generate SLC dir for date: ' + Date0) + os.system(call_str) + call_str = 'mv ' + Date + '*.slc* ' + dataDir + os.system(call_str) + + os.chdir(dataDir) + for i in range(Na): + Date0 = Date+'_' + str(i) + downName = str(AA[i]) + + SLCm = Date0 + '.slc' + SLCm_par = Date0 + '.slc.par' + MamprlksImg = Date0 + '.amp' + MamprlksPar = Date0 + '.amp.par' + + call_str = 'multi_look ' + SLCm + ' ' + SLCm_par + ' ' + MamprlksImg + ' ' + MamprlksPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + nWidth = UseGamma(MamprlksPar, 'read', 'range_samples') + call_str = 'raspwr ' + MamprlksImg + ' ' + nWidth + os.system(call_str) + + SLC = Date6+'.slc' + SLCPAR = Date6 + '.slc.par' + AMP = Date6+'.amp' + AMPPAR = Date6+'.amp.par' + if Na==1: + os.rename(SLCm,SLC) + os.rename(SLCm_par,SLCPAR) + os.rename(MamprlksImg,AMP) + os.rename(MamprlksPar,AMPPAR) + + for i in range(Na-1): + if i==0: + DateA = Date + '_0' + else: + DateA = Date + '_' + str(i-1) + str(i) + + SLCA = DateA + '.slc' + SLCA_par = DateA + '.slc.par' + + + DateB = Date + '_' +str(i+1) + SLCB = DateB + '.slc' + SLCB_par = DateB + '.slc.par' + + DateC = Date + '_' + str(i) + str(i+1) + SLCC = DateC + '.slc' + SLCC_par = DateC + '.slc.par' + + ##################################################################################### + MamprlksImg = Date + '.amp' + MamprlksPar = Date + '.amp.par' + + off = DateC + '.off' + offs = DateC + '.offs' + offsets = DateC + '.offsets' + coffs = DateC + '.coffs' + coffsets = DateC + '.coffsets' + snr = DateC + '.snr' + off_std = DateC + '.off_std' + + ########################## Generate off file ############################# + + call_str = "create_offset " + SLCA_par + " " + SLCB_par + " " + off + " 1 - - 0" + os.system(call_str) + call_str = 'init_offset_orbit '+ SLCA_par + " " + SLCB_par + ' ' + off + os.system(call_str) + + + call_str = 'init_offset '+ SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' ' + rlks4cor + ' ' + azlks4cor + ' ' + rpos4cor + ' ' + azpos4cor + os.system(call_str) + + call_str = 'init_offset '+ SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' 1 1 - - ' + os.system(call_str) + + call_str = "offset_pwr " + SLCA + " " + SLCB + " " + SLCA_par + " " + SLCB_par + " " + off + " " + offs + " " + snr + " " + rwin4cor + " " + azwin4cor + " " + offsets + " 2 "+ rsample4cor + " " + azsample4cor + os.system(call_str) + + call_str = "offset_fit " + offs + " " + snr + " " + off + " " + coffs + " " + coffsets + " " + thresh4cor +" 3" + os.system(call_str) + + call_str = "offset_pwr " +SLCA + " " + SLCB + " " + SLCA_par + " " + SLCB_par + " " + off + " " + offs + " " + snr + " " + rfwin4cor + " " + azfwin4cor + " " + offsets + " 2 " + rfsample4cor + " " + azfsample4cor + os.system(call_str) + + call_str = "offset_fit " + offs + " " + snr + " " + off + " " + coffs + " " + coffsets + " " + thresh4cor + " 4 >" + off_std + os.system(call_str) + + ######################################################################################## + + call_str = 'SLC_cat ' + SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' ' + SLCC + ' ' + SLCC_par + os.system(call_str) + + if i==(Na-2): + if len(Date)==6: + DD = Date + else: + DD = Date[2:8] + SLCm = DD + '.slc' + SLCm_par = DD + '.slc.par' + + + call_str = 'cp ' + SLCC + ' ' + SLCm + os.system(call_str) + + call_str = 'cp ' + SLCC_par + ' ' + SLCm_par + os.system(call_str) + + call_str = 'multi_look ' + SLCm + ' ' + SLCm_par + ' ' + MamprlksImg + ' ' + MamprlksPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + nWidth = UseGamma(MamprlksPar, 'read', 'range_samples') + call_str = 'raspwr ' + MamprlksImg + ' ' + nWidth + os.system(call_str) + + call_str = 'rm *.amp' + os.system(call_str) + + call_str = 'rm *_*.slc' + os.system(call_str) + + print("Down to SLC for %s is done! " % Date) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/Down2SLC_ERS_Cat_All.py b/third_party/PyINT/pyint/Down2SLC_ERS_Cat_All.py new file mode 100644 index 0000000..d256db3 --- /dev/null +++ b/third_party/PyINT/pyint/Down2SLC_ERS_Cat_All.py @@ -0,0 +1,136 @@ +#! /usr/bin/env python +#''' +################################################################################## +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : March, 2017 # +# # +# Generate Sentinel SLC from the downloaded data # +# # +################################################################################## +#''' +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close + +def rm(TXT): + call_str = 'rm ' + TXT + os.system(call_str) + +def usage(): + print(''' +****************************************************************************************************** + + Downloading Sentinel-1A/B data based on ssara + + usage: + + Down2SLC_ERS_Cat_All.py ProjectName + + e.g. Down2SLC_ERS_Cat_All.py CotopaxiT120ERSA + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==2: + projectName = sys.argv[1] + else: + usage();sys.exit(1) + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + slcDir = scratchDir + '/' + projectName + '/SLC' + + if not os.path.isdir(slcDir): + call_str= 'mkdir ' +slcDir + os.system(call_str) + + os.chdir(downDir) + + call_str = 'ls > tt0' + os.system(call_str) + + call_str = 'grep SAR_IMS_1P tt0 >tt1' + os.system(call_str) + + call_str = "awk -F_ '{print $3}' tt1 > tt2 " + os.system(call_str) + + call_str = "awk -FSA '{print $2}' tt2 > ttt" + os.system(call_str) + + call_str = 'sort ttt | uniq > ttm' + os.system(call_str) + + AA= np.loadtxt('ttm',dtype=np.str) + Na = AA.size + + for i in range(Na): + call_str = 'Down2SLC_ERS_Cat.py ' + projectName + ' ' + AA[i] + print(call_str) + call_str = 'Down2SLC_ERS_Cat.py ' + projectName + ' ' + AA[i] + ' >/dev/null' + os.system(call_str) + + + print("Down to SLC for %s is done! " % projectName) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/third_party/PyINT/pyint/ERS_DEOS.py b/third_party/PyINT/pyint/ERS_DEOS.py new file mode 100644 index 0000000..f97c88e --- /dev/null +++ b/third_party/PyINT/pyint/ERS_DEOS.py @@ -0,0 +1,206 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Central South University & University of Miami ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import linecache + +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +######################################################################### + +INTRODUCTION = ''' +############################################################################# + Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 + + Download precise ERS orbit data from Delft (http://www.deos.tudelft.nl/). + Correct the orbit parameters using DELFT_vec2 + + ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-2/dgm-e04/arclist + + +''' + +EXAMPLE = ''' + Usage: + ERS_DEOS.py projectName Date + + Examples: + ERS_DEOS.py AqabaERSA 960101 + +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Download (and correct) precise ERS orbit data.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='project name') + parser.add_argument('date',help='date of the ERS data') + + inps = parser.parse_args() + return inps + +################################################################################ + + +def main(argv): + + total = time.time() + inps = cmdLineParse() + + projectName = inps.projectName + DATE = inps.date + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + slcDir = scratchDir + '/' + projectName + '/SLC' + projDir = scratchDir + '/' + projectName + + ERS1_Orb_Dir = scratchDir + '/' + projectName + '/ERS1_Orb' + ERS2_Orb_Dir = scratchDir + '/' + projectName + '/ERS2_Orb' + + if not os.path.isdir(ERS1_Orb_Dir): + call_str = 'mkdir ' + ERS1_Orb_Dir + os.system(call_str) + + if not os.path.isdir(ERS2_Orb_Dir): + call_str = 'mkdir ' + ERS2_Orb_Dir + os.system(call_str) + + ERS1_Orb_list = ERS1_Orb_Dir + '/ERS1_Orb_list' + ERS2_Orb_list = ERS2_Orb_Dir + '/ERS2_Orb_list' + + os.chdir(ERS1_Orb_Dir) + if not os.path.isfile(ERS1_Orb_list): + Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-1/dgm-e04/arclist' + call_str = 'wget ' + Url + ' -O ERS1_Orb_list' + os.system(call_str) + + call_str = 'cp ERS1_Orb_list arclist' + os.system(call_str) + + os.chdir(ERS2_Orb_Dir) + if not os.path.isfile(ERS2_Orb_list): + Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-2/dgm-e04/arclist' + call_str = 'wget ' + Url + ' -O ERS2_Orb_list' + os.system(call_str) + + call_str = 'cp ERS2_Orb_list arclist' + os.system(call_str) + + if len(DATE)==8: + DATE6 = DATE[2:8] + else: + DATE6 = DATE + + slcDir1 = slcDir + '/' + DATE6 + slcpar = slcDir1 + '/' + DATE6 + '.slc.par' + Title = UseGamma(slcpar, 'read', 'title:') + if 'E1' in Title: + Flag = 'ERS1' + Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-1/dgm-e04/' + List = ERS1_Orb_list + Dir = ERS1_Orb_Dir + elif 'E2' in Title: + Flag = 'ERS2' + Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-2/dgm-e04/' + List = ERS2_Orb_list + Dir = ERS2_Orb_Dir + else: + print('The SAR data is invalid.') + sys.exit(1) + + if len(DATE)==6: + A0 = int(DATE[0:2]) + if A0 < 50: + DATE = '20' + DATE + else: + DATA = '19' + DATE + + YYMM = DATE6[0:2] + DATE6[2:4] + + call_str = 'grep ' + YYMM + ' ' + List + ' > list0' + os.system(call_str) + + call_str = "awk '{print $12}' list0 > start_end" + os.system(call_str) + + call_str = "awk '{print $1}' list0 > vec_num" + os.system(call_str) + + AA= np.loadtxt('start_end',dtype=np.str) + Na = AA.size + + print(AA) + + A_num= np.loadtxt('vec_num',dtype=np.str) + Na_num = A_num.size + + for i in range(Na-1): + A0 = AA[i] + A1 = AA[i+1] + print(DATE6) + print(A0) + print(A1) + if int(DATE6)==int(A0): + m = i + elif int(DATE6)==int(A1): + m = i+1 + elif int(A0) < int(DATE6) < int(A1): + m = i + + FF = A_num[int(m)] + SS = Url + 'ODR.' + FF + print(SS) + + call_str = 'wget -q --no-check-certificate ' + SS + ' -P ' + Dir + os.system(call_str) + + + print("Download precise DEFT orbital data for %s is done." % DATE) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + diff --git a/third_party/PyINT/pyint/ERS_orb_cor.py b/third_party/PyINT/pyint/ERS_orb_cor.py new file mode 100644 index 0000000..696cd99 --- /dev/null +++ b/third_party/PyINT/pyint/ERS_orb_cor.py @@ -0,0 +1,132 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Central South University & University of Miami ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import linecache + +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +######################################################################### + +INTRODUCTION = ''' +############################################################################# + Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 + + Precise ERS orbit data from Delft (http://www.deos.tudelft.nl/). + Correct the orbit parameters using DELFT_vec2 + +''' + +EXAMPLE = ''' + Usage: + ERS_orb_cor.py projectName Date + + Examples: + ERS_orb_cor.py AqabaERSA 960101 + +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Download (and correct) precise ERS orbit data.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='project name') + parser.add_argument('date',help='date of the ERS data') + + inps = parser.parse_args() + return inps + +################################################################################ + + +def main(argv): + + total = time.time() + inps = cmdLineParse() + + projectName = inps.projectName + DATE = inps.date + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + Ers1OrbDir = os.getenv('ERS1ORBDIR') + Ers2OrbDir = os.getenv('ERS2ORBDIR') + + slcDir = scratchDir + '/' + projectName + '/SLC' + projDir = scratchDir + '/' + projectName + + + if len(DATE)==8: + DATE6 = DATE[2:8] + else: + DATE6 = DATE + + slcDir1 = slcDir + '/' + DATE6 + slcpar = slcDir1 + '/' + DATE6 + '.slc.par' + Title = UseGamma(slcpar, 'read', 'title:') + if 'E1' in Title: + Flag = 'ERS1' + #Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-1/dgm-e04/' + #List = ERS1_Orb_list + Dir = Ers1OrbDir + elif 'E2' in Title: + Flag = 'ERS2' + #Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-2/dgm-e04/' + #List = ERS2_Orb_list + Dir = Ers2OrbDir + else: + print('The SAR data is invalid.') + sys.exit(1) + + call_str = 'DELFT_vec2 ' + slcpar + ' ' + Dir + ' 30' + os.system(call_str) + + + print("Using DEFT orbital data for %s is done." % DATE) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + diff --git a/third_party/PyINT/pyint/ERS_orb_cor_all.py b/third_party/PyINT/pyint/ERS_orb_cor_all.py new file mode 100644 index 0000000..ae81bb8 --- /dev/null +++ b/third_party/PyINT/pyint/ERS_orb_cor_all.py @@ -0,0 +1,126 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Central South University & University of Miami ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import linecache + +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +def is_number(s): + try: + int(s) + return True + except ValueError: + return False + +######################################################################### + +INTRODUCTION = ''' +############################################################################# + Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 + + Precise ERS orbit data from Delft (http://www.deos.tudelft.nl/). + Correct the orbit parameters using DELFT_vec2 + +''' + +EXAMPLE = ''' + Usage: + ERS_orb_cor_all.py projectName + + Examples: + ERS_orb_cor_all.py AqabaERSA + +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Download (and correct) precise ERS orbit data.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='project name') + + inps = parser.parse_args() + return inps + +################################################################################ + + +def main(argv): + + total = time.time() + inps = cmdLineParse() + + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + slcDir = scratchDir + '/' + projectName + '/SLC' + + ListSLC = os.listdir(slcDir) + Datelist = [] + SLCfile = [] + SLCParfile = [] + + + for kk in range(len(ListSLC)): + if ( is_number(ListSLC[kk]) and len(ListSLC[kk])==6 ): # if SAR date number is 8, 6 should change to 8. + DD=ListSLC[kk] + Year=int(DD[0:2]) + Month = int(DD[2:4]) + Day = int(DD[4:6]) + Datelist.append(ListSLC[kk]) + N = len(Datelist) + + for i in range(N): + call_str = 'ERS_orb_cor.py ' + projectName + ' ' + Datelist[i] + print(call_str) + os.system(call_str) + + print("Using DEFT orbital data for project %s is done." % projectName) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + diff --git a/third_party/PyINT/pyint/ERS_orb_cor_par.py b/third_party/PyINT/pyint/ERS_orb_cor_par.py new file mode 100644 index 0000000..2514d7c --- /dev/null +++ b/third_party/PyINT/pyint/ERS_orb_cor_par.py @@ -0,0 +1,118 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Central South University & University of Miami ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import linecache + +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +######################################################################### + +INTRODUCTION = ''' +############################################################################# + Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 + + Precise ERS orbit data from Delft (http://www.deos.tudelft.nl/). + Correct the orbit parameters using DELFT_vec2 + +''' + +EXAMPLE = ''' + Usage: + ERS_orb_cor_par.py SLC_par + + Examples: + ERS_orb_cor_par.py 960101.slc.par + +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Download (and correct) precise ERS orbit data.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('slc_par',help='slc par file') + + inps = parser.parse_args() + return inps + +################################################################################ + + +def main(argv): + + total = time.time() + inps = cmdLineParse() + + slcpar = inps.slc_par + + Ers1OrbDir = os.getenv('ERS1ORBDIR') + Ers2OrbDir = os.getenv('ERS2ORBDIR') + + Title = UseGamma(slcpar, 'read', 'title:') + if 'E1' in Title: + Flag = 'ERS1' + #Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-1/dgm-e04/' + #List = ERS1_Orb_list + Dir = Ers1OrbDir + elif 'E2' in Title: + Flag = 'ERS2' + #Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-2/dgm-e04/' + #List = ERS2_Orb_list + Dir = Ers2OrbDir + else: + print('The SAR data is invalid.') + sys.exit(1) + + call_str = 'DELFT_vec2 ' + slcpar + ' ' + Dir + os.system(call_str) + + + print("Using DEFT orbital data for %s is done." % slcpar) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + diff --git a/third_party/PyINT/pyint/GACOS_correction.csh b/third_party/PyINT/pyint/GACOS_correction.csh new file mode 100644 index 0000000..882b07d --- /dev/null +++ b/third_party/PyINT/pyint/GACOS_correction.csh @@ -0,0 +1,131 @@ +#!/bin/csh -f +# $Id$ +#GACOS Correction +#Erik Rivas, Oct 18 2021 +# + +if ($#argv != 5) then + echo "" + echo "Usage: GACOS.csh list_interferograms full_path_to_GACOS_data full_path_topo reference_point incidence_angle" + echo "Script needs to be run inside the intf_all folder" + echo "Performs gacos correction" + echo "" + echo "list_interferograms: list of folders inside the intf_all folder containing the phases and coherence grids" + echo "" + echo "full_path_to_GACOS_data" + echo "Example: /home/erikr/gacos/" + echo "" + echo "full_path_topo" + echo "Example: /home/erikr/project/topo/" + echo "" + echo "the topo folder needs to have: dem.grd, master.PRM, and the correspondant .LED file" + echo "" + echo "list_interferograms: list of folders with interferograms created" + echo "" + echo "Reference point in lon lat coordinates (text file)" + echo "" + echo "Indicence angle in degrees from SAT_look (float/integer)" + echo "" + echo "Outputs: phasefilt.grd files corrected and added as additional products in each interferogram folder. These outputs should be used for the unwrap processing" + exit 1 +endif + +set list = $1 +set GACOS_dir = $2 +set topo_dir = $3 +set reference_point = $4 +set incidence = $5 + +#Checking inputs +if !(-e $list) then + echo "$list seems not to exist" + exit 1 +endif +if !(-d $GACOS_dir) then + echo "$GACOS_dir seems not to exist" + exit 1 +endif +if !(-d $topo_dir) then + echo "$topo_dir seems not to exist" + exit 1 +endif +if !(-f $reference_point) then + echo "Reference point file: $reference_point seems not to exist. Provide a text file with lon lat values" + exit 1 +endif + + +#PROJECT POINT FROM LON-LAT TO RADAR COORDINATES +set ref_llh = $topo_dir"ref.llh" +set out_ratll = $topo_dir"out.ratll" +set reference_point_ra = $topo_dir"ref_point.ra" +if (-f $ref_llh) then + echo "Removing old $ref_llh" + rm $ref_llh +endif +if (-f $out_ratll) then + echo "Removing old $out_ratll" + rm $out_ratll +endif +if (-f $reference_point_ra) then + echo "Removing old $reference_point_ra" + rm $reference_point_ra +endif + +gmt grdtrack $reference_point -G$topo_dir"dem.grd" >> $ref_llh +ln -s $topo_dir*.LED . +SAT_llt2rat $topo_dir"master.PRM" 0 < $topo_dir"ref.llh" > $out_ratll +rm *.LED +cat $out_ratll |awk '{print $1, $2}' > $reference_point_ra +set dem_grd = $topo_dir"dem.grd" +#----------------------------------------------# + +#FOR LOOP OVER LIST OF INTERFEROGRAMS +foreach dir (`awk '{print $1}' $list`) + if !(-d $dir) then + echo "$dir directory seems not to exist" + exit 1 + endif + + cd $dir + + #Check if there are only two SLC files (not sure if this step is neccessary) + if (`ls *.SLC|wc -l` != "2") then + echo "the number of SLC files is inconsistent" + endif + + #ls sorts content alphanumeric, therefore it is assume the first as the master on the list + set fst_date = `ls *.SLC|sed -n '1p'|awk '{print substr($1,4,8)}'` + set scd_date = `ls *.SLC|sed -n '2p'|awk '{print substr($1,4,8)}'` + + #Save directory of current intf + set intf_dir = `pwd` + + cd $GACOS_dir + + + #Check if the GACOS files are in the folder + if (-f $fst_date".ztd" && -f $fst_date".ztd.rsc" && -f $scd_date".ztd" && -f $scd_date".ztd.rsc") then + set first_ztd = $GACOS_dir$fst_date".ztd" + set first_rsc = $GACOS_dir$fst_date".ztd.rsc" + set second_ztd = $GACOS_dir$scd_date".ztd" + set second_rsc = $GACOS_dir$scd_date".ztd.rsc" + + #GACOS Correction + cd $intf_dir + #Link trans.dat to each folder. Neccesary to project ztd grids to radar coordinates + ln -s $topo_dir"trans.dat" + operation.csh $first_ztd $first_rsc $second_ztd $second_rsc $reference_point_ra $incidence $dem_grd + rm trans.dat + + else + + echo "GACOS files do not exist / Wrong directory" + exit 1 + + endif + + cd .. +end + +echo "GACOS correction done" diff --git a/third_party/PyINT/pyint/Get_off_std.py b/third_party/PyINT/pyint/Get_off_std.py new file mode 100644 index 0000000..18775a2 --- /dev/null +++ b/third_party/PyINT/pyint/Get_off_std.py @@ -0,0 +1,145 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Central South University & University of Miami ### +################################################################# + + +import os +import sys +import glob +import time +import argparse + +import h5py +import numpy as np + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +def UseGamma2(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[2].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +def usage(): + print(''' +****************************************************************************************************** + + Get co-registration standard deviation of SLCs for one project + + usage: + + GenerateRSC_Gamma.py projectName + + e.g. Get_off_std.py GalapagosT061EnvA + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==2: + if argv[0] in ['-h','--help']: usage(); sys.exit(1) + else: projectName=sys.argv[1] + else: + usage();sys.exit(1) + + scratchDir = os.getenv('SCRATCHDIR') + OFFDir = scratchDir + '/' + projectName + '/RSLC' + OFFSTR = OFFDir + '/*.off_std' + OFFFile = glob.glob(OFFSTR) + STD_TXT ='COREG_STD_ALL' + os.chdir(OFFDir) + if os.path.isfile(STD_TXT): + os.remove(STD_TXT) + + for ff in OFFFile: + OFF_STD = ff + NM = os.path.basename(ff).split('.')[0] + RR = UseGamma(OFF_STD,'read','final range offset poly. coeff.:') + cor_rg = RR.split(' ')[0] + + AA = UseGamma(OFF_STD,'read','final azimuth offset poly. coeff.:') + cor_az = AA.split(' ')[0] + + STDRR = UseGamma(OFF_STD,'read','final model fit std. dev. (samples) range:') + std_rg=STDRR.split(' ')[0] + + std_az = UseGamma2(OFF_STD,'read','final model fit std. dev. (samples) range:') + + STR = NM + ' ' + cor_rg + ' ' + cor_az + ' ' + std_rg + ' ' + std_az + call_str ='echo ' + STR + ' >> ' + STD_TXT + os.system(call_str) + + + sys.exit(1) + + +if __name__ == '__main__': + main(sys.argv[1:]) + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/.gitignore b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/.gitignore new file mode 100644 index 0000000..764a6ac --- /dev/null +++ b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/.gitignore @@ -0,0 +1,5 @@ +*.swp +bin/ +*.tif +*.pyc +__pycache__/ diff --git a/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/LICENSE b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/PhaseBias_01_Read_Data.py b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/PhaseBias_01_Read_Data.py new file mode 100644 index 0000000..a9815c1 --- /dev/null +++ b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/PhaseBias_01_Read_Data.py @@ -0,0 +1,582 @@ +#!/usr/bin/env python3 + +import os +import re +from datetime import datetime, timedelta +import numpy as np +import rasterio +from bin.multilook_w import multilook_w +from bin.multilook import multilook +import configparser +import sys +import warnings +import pickle +import glob + +warnings.filterwarnings("ignore", category=DeprecationWarning) + +####################################################################### + +HELP_TEXT = """ +PhaseBias_01_Read_Data.py + +This script automates the process of downloading GeoTIFF files for wrapped interferograms and coherence images. +The files can be retrieved either from the COMET-LiCS web portal or from a root path specified in the configuration file. + +Assumptions: +- Interferograms are organized in folders named as `yyyymmdd_yyyymmdd` (e.g., `20230101_20230107`). + +Files Downloaded: +1. `yyyymmdd_yyyymmdd.geo.diff_pha.tif`: + - Contains the wrapped phase image in radians. + - Values range from -3.14 to 3.14. + +2. `yyyymmdd_yyyymmdd.geo.cc.tif`: + - Contains the coherence image of the interferometric pair. + - Values range from 0 to 255, where: + - 0 represents the lowest coherence. + - 255 represents the highest coherence. + +Outputs: +1. `All_ifgs_start_end` (Pickle file, saved as `.pkl`): + - A dictionary containing all interferograms available between the specified start and end dates. +2. `All_coh_start_end` (Pickle file, saved as `.pkl`): + - A dictionary containing all coherence data available between the specified start and end dates. + +Storage Information: +- All output files are saved in the directory `Data/` under the `output_path` defined in the configuration file. + +Additional Output: +- A summary report is generated that includes: + - The number of available interferograms for each temporal baseline. + - The number of missing interferograms. + +""" + +if "--help" in sys.argv: + print(HELP_TEXT) + sys.exit(0) + +################### + +# Config file path +config_file = "config.txt" + +# Check if config file exists +if not os.path.exists(config_file): + raise FileNotFoundError( + f"Error: The configuration file '{config_file}' was not found in the script's directory." + ) + +# Initialize the configparser +config = configparser.ConfigParser() + +# Read the config.txt file +config.read(config_file) + +# Define required parameters +required_parameters = [ + "root_path", + "frame", + "start", + "end", + "interval", + "nlook", + "LiCSAR_data", +] + +# Initialize a dictionary to store parameters +parameters = {} + +# Attempt to retrieve each parameter and catch errors +try: + parameters["root_path"] = config.get("DEFAULT", "root_path") + parameters["output_path"] = config.get("DEFAULT", "output_path") + parameters["frame"] = config.get("DEFAULT", "frame") + parameters["start"] = config.get("DEFAULT", "start") + parameters["end"] = config.get("DEFAULT", "end") + parameters["interval"] = config.getint("DEFAULT", "interval") + parameters["nlook"] = config.getint( + "DEFAULT", "nlook") # Assuming 'nlook' should be an integer + parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") + +except configparser.NoOptionError as e: + # Error message if a required parameter is missing + missing_param = str(e).split(": ")[1] + raise ValueError( + f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." + ) +except configparser.Error as e: + # General error for any configparser-related issues + raise ValueError(f"Error while reading '{config_file}': {e}") + +# Assign to individual variables for easy access +root_path = parameters["root_path"] +output_path = parameters["output_path"] +frame = parameters["frame"] +start = parameters["start"] +end = parameters["end"] +interval = parameters["interval"] +nlook = parameters["nlook"] +LiCSAR_data = parameters["LiCSAR_data"] + +# hardcoded parameters, not included in the config file +landmask = 1 +filtered_ifgs = "yes" +min_baseline = 5 +max_baseline = 366 + +# Print the values to confirm they're being read correctly +print("Root Path:", root_path) +print("output Path:", output_path) +print("Frame:", frame) +print("Start Date:", start) +print("End Date:", end) +print("Interval:", interval) +print("nlook:", nlook) +print("LiCSAR_data", LiCSAR_data) + +################ +# all_ifgs = read_ifgs(frame, start, end, min_baseline, max_baseline, landmask, nlook, interval, LiCSAR_data, filtered_ifgs) + +def read_ifgs( + start, + end, + min_baseline, + max_baseline, + landmask, + nlook, + interval, + LiCSAR_data, + filtered_ifgs, +): + + start_date = datetime.strptime(start, "%Y%m%d") + end_date = datetime.strptime(end, "%Y%m%d") + # Check if the file ends with "geo.diff_pha.tif" + # if filename.endswith("geo.diff_unfiltered_pha.tif"): # if unfiltered data are used + if filtered_ifgs == "yes": # yes for the filtered ifgs, no for the unfiltered ifgs + req_file_name = "geo.diff_pha.tif" + else: + req_file_name = "geo.diff_unfiltered_pha.tif" + + ############ reading + + if LiCSAR_data == "yes": + track = frame[0:3] # extracting track number from frame id + if track[0] == "0": + track = track[1:3] + if track[0] == "0": + track = track[1:2] + + root_directory = f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}/interferograms/" + else: + root_directory = os.path.join(root_path, "interferograms") # Use os.path.join for consistent path construction + #root_directory = os.path.join(root_path, "") + + ##### Count total files matching the criteria for accurate progress calculation + print(f"Reading data from the path: {root_directory}") + + total_files = sum(1 for dirpath, _, filenames in os.walk(root_directory) + for filename in filenames + if filename.endswith(req_file_name)) + processed_files = 0 + + print("Reading wrapped interferograms: [", end="", flush=True) + ##### + + # Initialize an empty dictionary to store the arrays for each category + category_arrays = {} + last_date = {} + n = 0 + + if landmask is not None: + if LiCSAR_data == "yes": + landmask_path = f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}/metadata/{frame}.geo.landmask.tif" + else: + # Use glob to find the file ending with ".geo.landmask.tif" in the metadata directory + metadata_dir = os.path.join(root_path, "metadata") + landmask_files = glob.glob( + os.path.join(metadata_dir, "*.geo.landmask.tif")) + if landmask_files: + landmask_path = landmask_files[0] # Take the first match + else: + raise FileNotFoundError( + "No file ending with '.geo.landmask.tif' found in the metadata directory." + ) + + # Load the landmask file + with rasterio.open(landmask_path) as src: + array_landmask = src.read(1) + + n = 1 + # Traverse the root directory and its subdirectories + for dirpath, dirnames, filenames in os.walk(root_directory): + dirnames.sort( + ) # This ensures dirnames are processed in alphabetical order + # Iterate through the files in the current directory + for filename in filenames: + # Check if the file ends with "geo.diff_pha.tif" + # if filename.endswith("geo.diff_unfiltered_pha.tif"): # if unfiltered data are used + if (filtered_ifgs == "yes" + ): # yes for the filtered ifgs, no for the unfiltered ifgs + req_file_name = "geo.diff_pha.tif" + else: + req_file_name = "geo.diff_unfiltered_pha.tif" + + if filename.endswith(req_file_name): + file_path = os.path.join(dirpath, filename) + + #### for printing into the console + processed_files += 1 + percentage = (processed_files / total_files) * 100 + + # Clear previous percentage and print updated one + sys.stdout.write( + "\rReading wrapped phases: [{:<50}] {}%".format( + "=" * int(percentage // 2), int(percentage))) + sys.stdout.flush() + + ##### + + # Extract the dates from the file name + match = re.search(r"(\d+)_(\d+)", filename) + if match: + date1 = match.group(1) + date2 = match.group(2) + + # Convert the dates to datetime objects + date1_obj = datetime.strptime(date1, "%Y%m%d") + date2_obj = datetime.strptime(date2, "%Y%m%d") + + # Check if the dates fall within the specified range + category = (date2_obj - date1_obj).days + + if (start_date <= date1_obj <= end_date + and start_date <= date2_obj <= end_date + and category < max_baseline + and category > min_baseline): + if ( + n == 1 + ): # finding the first and last acquisiton in the time-series + first_acq = date1_obj + else: + last_acq = date2_obj + n = n + 1 + + # Calculate the difference in days between the two dates + + # Open the TIFF file + try: + with rasterio.open(file_path) as src: + array_data = src.read(1).astype(np.float32) + except Exception as e: + print(f"\u26a0\ufe0f Error reading {file_path}: {e}. Appending None instead.") + array_data = None + + #if array_data is not None: + array_data[array_data == 0] = np.nan ## converting the zerso values in wrapped phases into nan. This is helpful when calculating loop closures + + #array_data[np.abs(array_data) < 1e-8] = np.nan # as after filter i observed low vlaues outside the frame. I also wanted to nulify them. so used this insteadof above + + + if landmask != None: #and array_data is not None: + array_data[array_landmask != 1] = np.nan + + if ( + nlook != None and nlook != 1 #and array_data is not None + ): # incase of unwrap data because they are already multilooked to 10 we don't multilook them here + array_data = multilook_w(array_data, nlook) + + if category not in last_date: + last_date[category] = date1_obj + # if (date1_obj - date2_obj) > interval + + # appending None to the category_arrays where there are missing ifgs in the middle of time-series + diff_ifgs = ( + date1_obj - last_date[category] + ).days # /category #difference between the first epochs of two consecutive ifg + if diff_ifgs > interval: + n_no_acq = int(diff_ifgs / interval - + 1) ## number of missing ifgs + for i in range( + n_no_acq + ): # the number of none depends on the number of missing ifgs + # Append the none to the existing array + category_arrays[category].append(None) + last_date[category] = date1_obj + + # Check if the category is already in the dictionary + if category in category_arrays: + # If the category already exists, append the array to the existing array + category_arrays[category].append(array_data) + else: + # If the category doesn't exist, create a new list with the array + category_arrays[category] = [array_data] + + # appending None to the category_arrays where there are missing ifgs in the end of time-series + for category in sorted(category_arrays): + while (last_date[category] + timedelta(category)) < last_acq: + category_arrays[category].append(None) + last_date[category] = last_date[category] + timedelta(interval) + + # appending None to the category_arrays where there are missing ifgs in the begining of time-series + # max number of expected 6-day(i.e. interval) interferograms in the full time-series + max_ifg_number = abs(int((first_acq - last_acq).days / interval)) + + # all the categories in the data e.g. 6, 12, 18 etc + cat = [] + for category in category_arrays: + cat.append(category) + + for i in range(interval, max(cat)+1, interval): + if i in cat: + while len(category_arrays[i]) < (max_ifg_number + 1 - i/interval): + category_arrays[i].insert( 0, None) + + print("\nFinished reading all interferograms.") + + return category_arrays + + +########################################################################################################################### +########################################################################################################################## + + +def read_coh(start, end, min_baseline, max_baseline, landmask, nlook, interval, + LiCSAR_data): + + start_date = datetime.strptime(start, "%Y%m%d") + end_date = datetime.strptime(end, "%Y%m%d") + ############ reading + + if LiCSAR_data == "yes": + track = frame[0:3] + if track[0] == "0": + track = track[1:3] + if track[0] == "0": + track = track[1:2] + + root_directory = f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}/interferograms/" + else: + root_directory = os.path.join( + root_path, "interferograms" + ) # Use os.path.join for consistent path construction + + req_file_name = "geo.cc.tif" + + ##### Count total files matching the criteria for accurate progress calculation + + total_files = sum(1 for dirpath, _, filenames in os.walk(root_directory) + for filename in filenames + if filename.endswith(req_file_name)) + processed_files = 0 + + print("Reading coherence data: [", end="", flush=True) + ##### + + # Initialize an empty dictionary to store the arrays for each category + category_arrays = {} + last_date = {} + n = 0 + + if landmask is not None: + if LiCSAR_data == "yes": + landmask_path = f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}/metadata/{frame}.geo.landmask.tif" + else: + # Use glob to find the file ending with ".geo.landmask.tif" in the metadata directory + metadata_dir = os.path.join(root_path, "metadata") + landmask_files = glob.glob( + os.path.join(metadata_dir, "*.geo.landmask.tif")) + if landmask_files: + landmask_path = landmask_files[0] # Take the first match + else: + raise FileNotFoundError( + "No file ending with '.geo.landmask.tif' found in the metadata directory." + ) + + # Load the landmask file + with rasterio.open(landmask_path) as src: + array_landmask = src.read(1) + + n = 1 + # Traverse the root directory and its subdirectories + for dirpath, dirnames, filenames in os.walk(root_directory): + # Iterate through the files in the current directory + for filename in filenames: + # Check if the file ends with "geo.cc.tif" + + if filename.endswith(req_file_name): + file_path = os.path.join(dirpath, filename) + + #### for printing into the console + processed_files += 1 + percentage = (processed_files / total_files) * 100 + + # Clear previous percentage and print updated one + sys.stdout.write( + "\rReading coherence data: [{:<50}] {}%".format( + "=" * int(percentage // 2), int(percentage))) + sys.stdout.flush() + + ##### + + # Extract the dates from the file name + match = re.search(r"(\d+)_(\d+)", filename) + if match: + date1 = match.group(1) + date2 = match.group(2) + + # Convert the dates to datetime objects + date1_obj = datetime.strptime(date1, "%Y%m%d") + date2_obj = datetime.strptime(date2, "%Y%m%d") + + # Check if the dates fall within the specified range + category = (date2_obj - date1_obj).days + + if (start_date <= date1_obj <= end_date + and start_date <= date2_obj <= end_date + and category < max_baseline + and category > min_baseline): + if ( + n == 1 + ): # finding the first and last acquisiton in the time-series + first_acq = date1_obj + else: + last_acq = date2_obj + n = n + 1 + + # Calculate the difference in days between the two dates + + # Open the TIFF file + try: + with rasterio.open(file_path) as src: + array_data = src.read(1).astype(np.float32) + except Exception as e: + print(f"\u26a0\ufe0f Error reading {file_path}: {e}. Appending None instead.") + array_data = None + + if landmask != 0: + array_data[array_landmask != 1] = np.nan + + if ( + nlook != None and nlook != 1 + ): # incase of unwrap data because they are already multilooked to 10 we don't multilook them here + array_data = multilook(array_data, nlook) + + if category not in last_date: + last_date[category] = date1_obj + # if (date1_obj - date2_obj) > interval + + # appending None to the category_arrays where there are missing ifgs in the middle of time-series + diff_ifgs = ( + date1_obj - last_date[category] + ).days # /category #difference between the first epochs of two consecutive ifg + if diff_ifgs > interval: + n_no_acq = int(diff_ifgs / interval - + 1) ## number of missing ifgs + for i in range( + n_no_acq + ): # the number of none depends on the number of missing ifgs + # Append the none to the existing array + category_arrays[category].append(None) + last_date[category] = date1_obj + + # Check if the category is already in the dictionary + if category in category_arrays: + # If the category already exists, append the array to the existing array + category_arrays[category].append(array_data) + else: + # If the category doesn't exist, create a new list with the array + category_arrays[category] = [array_data] + + # appending None to the category_arrays where there are missing ifgs in the end of time-series + for category in sorted(category_arrays): + while (last_date[category] + timedelta(category)) < last_acq: + category_arrays[category].append(None) + last_date[category] = last_date[category] + timedelta(interval) + + # appending None to the category_arrays where there are missing ifgs in the begining of time-series + # max number of expected 6-day(i.e. interval) interferograms in the full time-series + max_ifg_number = abs(int((first_acq - last_acq).days / interval)) + + # all the categories in the data e.g. 6, 12, 18 etc + cat = [] + for category in category_arrays: + cat.append(category) + + for i in range(interval, max(cat)+1, interval): + if i in cat: + while len(category_arrays[i]) < (max_ifg_number + 1 - i/interval): + category_arrays[i].insert(0, None) + + print("\nFinished reading all coherence data.") + + return category_arrays + + +########################################################################################################################## +########################################################################################################################## + +all_ifgs = read_ifgs( + start, + end, + min_baseline, + max_baseline, + landmask, + nlook, + interval, + LiCSAR_data, + filtered_ifgs, +) +# Define the directory path +directory_path = os.path.join(output_path, "Data") + +# Create the directory if it doesn't exist +os.makedirs(directory_path, exist_ok=True) + +# Define the file path +#file_path = os.path.join(directory_path, f"_all_ifgs_{start}_{end}") + +# Remove all files inside the directory +if os.path.exists(directory_path): + files = glob.glob(os.path.join(directory_path, "*")) + for file in files: + os.remove(file) + + +# for writing the all_loops +file_path = output_path + "/Data/" + "All_ifgs_" + start + "_" + end + ".pkl" +with open(file_path, "wb") as file: # for writting a dictionary to a file + pickle.dump(all_ifgs, file) + + +################################################ + +# After processing all interferograms, generate the report +print("\nReport on the number of IFGs and missing IFGs per category:") +for category in sorted(all_ifgs): + num_ifgs = sum(1 for item in all_ifgs[category] + if item is not None) # Count valid IFGs + num_missing_ifgs = sum(1 for item in all_ifgs[category] + if item is None) # Count missing IFGs + print( + f"{category}-days: Number of IFGs: {num_ifgs} Number of missing IFGs: {num_missing_ifgs}" + ) + +################################################################ + +import gc +del all_ifgs +gc.collect() + + + +all_coh = read_coh(start, end, min_baseline, max_baseline, landmask, nlook, + interval, LiCSAR_data) + +## for writting the all_coh +file_path = output_path + "/Data/" + "All_coh_" + start + "_" + end + ".pkl" +with open(file_path, "wb") as file: # for writting a dictionary to a file + pickle.dump(all_coh, file) + diff --git a/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/PhaseBias_02_Loop_Closures.py b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/PhaseBias_02_Loop_Closures.py new file mode 100644 index 0000000..c7a14d8 --- /dev/null +++ b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/PhaseBias_02_Loop_Closures.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 + +import numpy as np +import pickle +import glob +import configparser +import os +import sys +import warnings +warnings.filterwarnings("ignore", category=DeprecationWarning) + +################################################################################################ + +HELP_TEXT = """ +PhaseBias_02_Loop_Closures.py + +This script calculates loop closures (Δφ) using the interferograms imported in the first step. + +Definition of Loop Closures: +Loop closures, Δφ, are calculated for the epochs between `i` and `k`, and are defined as: + Δφ_(i,k) = |φ_(i,k) - ∑_(t=i)^k φ_(t,t+1)|_2π + +Where: +- φ_(i,j): Represents the phase difference for a pixel in the interferogram formed between epochs `i` and `j`. +- |.|_2π: Indicates that the result is wrapped modulo 2π (i.e., values range from -π to π). + +Key Information: +- Nonzero closure phase is a by-product of spatial filtering/multilooking and is primarily associated with changes in the scattering and electrical properties of the ground surface. +- The calculated loop closures are based on the minimum temporal-baseline interferograms defined in the configuration file (parameter: `interval`) and are referred to as base interferograms. These may represent: + - 12- and 6-day closures (or 24- and 12-day closures), Δφ_(i,i+2) + - 18- and 6-day closures (or 36- and 12-day closures), Δφ_(i,i+3) + The distinction depends on whether the base interferograms are 6-day or 12-day intervals. + +Outputs: +- The calculated loop closures are stored as a dictionary in a file named: + `All_loops_start_end.pkl` +- The output is saved in the directory `Data/` under the `output_path` defined in the configuration file. + +Loop Closure Report: +At the end of the script, a loop closure report is displayed. This report includes: +- The number of generated and missing loop closures for short-interval closures, such as 12-6 and 18-6. +- The number of generated and missing long-interval closures, obtained using long-term interferograms, such as 204-6 and 204-12. + +Usage: +1. Ensure the interferograms from the first script are available. +2. Run the script to generate loop closures for the specified temporal baselines. + +Example: +```bash +python PhaseBias_02_Loop_Closures.py + + +""" +if "--help" in sys.argv: + print(HELP_TEXT) + sys.exit(0) + +################################################################################################ +# Config file path +config_file = "config.txt" + +# Check if config file exists +if not os.path.exists(config_file): + raise FileNotFoundError( + f"Error: The configuration file '{config_file}' was not found in the script's directory." + ) + +# Initialize the configparser +config = configparser.ConfigParser() + +# Read the config.txt file +config.read(config_file) + +# Define required parameters +required_parameters = [ + "root_path", + "frame", + "start", + "end", + "interval", + "nlook", + "LiCSAR_data", +] + +# Initialize a dictionary to store parameters +parameters = {} + +# Attempt to retrieve each parameter and catch errors +try: + parameters["root_path"] = config.get("DEFAULT", "root_path") + parameters["output_path"] = config.get("DEFAULT", "output_path") + parameters["frame"] = config.get("DEFAULT", "frame") + parameters["start"] = config.get("DEFAULT", "start") + parameters["end"] = config.get("DEFAULT", "end") + parameters["interval"] = config.getint("DEFAULT", "interval") + parameters["nlook"] = config.getint( + "DEFAULT", "nlook") # Assuming 'nlook' should be an integer + parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") + +except configparser.NoOptionError as e: + # Error message if a required parameter is missing + missing_param = str(e).split(": ")[1] + raise ValueError( + f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." + ) +except configparser.Error as e: + # General error for any configparser-related issues + raise ValueError(f"Error while reading '{config_file}': {e}") + +# Assign to individual variables for easy access +root_path = parameters["root_path"] +output_path = parameters["output_path"] +frame = parameters["frame"] +start = parameters["start"] +end = parameters["end"] +interval = parameters["interval"] +nlook = parameters["nlook"] +LiCSAR_data = parameters["LiCSAR_data"] + +# hardcoded parameters, not included in the config file +landmask = 1 +filtered_ifgs = "yes" +max_loop = 5 # in case of 3 all loops with 6,12,18-day will be calculated (e.g. (60,6), (60,12) and (60,18)) + + +############################################################################################# +################################################################################################# +def loop_calc(max_loop): + + ## Read ifgs: + + # Define the path components + sub_dir = "Data" + filename_pattern = "All_ifgs_*" # Pattern to match files starting with "All_ifgs" + + # Construct the full file path with the refined pattern + file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) + # Use glob to find the file and ensure it exists + file_list = glob.glob(file_path_pattern) + if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + all_ifgs = pickle.load(file) + else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) + + loop = {} + missing = {} + + cat = sorted(all_ifgs.keys()) # Get sorted categories + # print('cat = ', cat) + + # Initialize counters for reporting + loop_counts = { + category: { + cat[l]: 0 + for l in range(max_loop) + } + for category in sorted(all_ifgs.keys()) + } + missing_counts = { + category: { + cat[l]: 0 + for l in range(max_loop) + } + for category in sorted(all_ifgs.keys()) + } + + for category in cat: + loop[category] = { + } # Initialize the key in the loop dictionary with an inner dictionary + missing[category] = [] # second ver + for l in range( + max_loop + ): # it was up to 3 in the first version. but this allows to go for longer loops e.g. 288-72 (cat[11] is 72) + loop[category][cat[l]] = [ + ] # Initialize the key in the inner dictionary + # missing[cat[l]] = [] # first ver + # Calculate total iterations for progress percentage + total_iterations = sum( + len(all_ifgs[cat[i]]) for l in range(max_loop) + for i in range(1, len(cat)) + if cat[i] % cat[l] == 0 and cat[i] != cat[l]) + completed_iterations = 0 + + # Display the message before starting the progress bar + print( + f"Calculating all {interval}, {2 * interval}, and {3 * interval} loop closures." + ) + + for l in range( + 0, max_loop + ): # l could be 6, 12 and 18 to calculate e.g. 36-6 or 36-12 or 36-18 + for i in range(1, len(cat)): # index for the category e.g. 6, 12, 18 + if cat[i] % cat[l] == 0 and cat[i] != cat[ + l]: # such as 12/6 or 24/12 + for t in range(len(all_ifgs[ + cat[i]])): # index for the epochs in each category + + ####################progress bar + completed_iterations += 1 + percentage = (completed_iterations / + total_iterations) * 100 + + # Display progress percentage + sys.stdout.write("\rProgress: [{:<50}] {}%".format( + "=" * int(percentage // 2), int(percentage))) + sys.stdout.flush() + ################################## + + if cat[l] == interval: + end_index = t + int(cat[i] / cat[l]) + # print('end_index = ', end_index) + else: + end_index = t + int(cat[i] / cat[l]) * int( + cat[l] / interval) + + # recording the missing ifgs in each cat e.g. 6,12,18 + #if np.any(np.array(all_ifgs[cat[i]])[t] == None): + if np.any(np.array(all_ifgs[cat[i]], dtype=object)[t] == None): + missing[cat[i]].append( + t + ) # to record the missing ifgs index for each cateogory e.g. 18 + + + + # --- Check for overshoot(not enough data at the tail). this allows the calculate the missing in the next for loop + if end_index > len(all_ifgs[cat[l]]): + # Not enough short IFGs to form a valid closure → skip this loop + loop[cat[i]][cat[l]].append(None) + missing_counts[cat[i]][cat[l]] += 1 + continue + + + + for e in range(t, end_index, int(cat[l]/interval)): # to record the missing ifgs index for the period of each loop + #print(f" Long baseline (cat[i]): {cat[i]}") + #print(f" Short baseline (cat[l]): {cat[l]}") + #print(f" Index t: {t}, e: {e}, end_index: {end_index}") + #print(f" len(all_ifgs[cat[l]]): {len(all_ifgs[cat[l]])}") + + elem = all_ifgs[cat[l]][e] + + if elem is None: + missing[cat[l]].append(e) + + + + if all_ifgs[cat[i]][t] is None or any(all_ifgs[cat[l]][elem] is None for elem in range(t, end_index, int(cat[l] / interval))): + + loop[cat[i]][cat[l]].append(None) + missing_counts[cat[i]][cat[l]] += 1 # Increment missing loop counter + + else: + closure = np.angle( np.exp( 1j * ( np.array(all_ifgs[cat[i]][t], dtype=np.float32)- np.sum( np.array(all_ifgs[cat[l]][t:end_index:int(cat[l] / interval)], dtype=np.float32), axis=0)))) + + if cat[i] in loop: + loop[cat[i]][cat[l]].append(closure) + loop_counts[cat[i]][cat[l]] += 1 # Increment successful loop counter + + else: + loop[cat[i]][cat[l]] = [closure] + print("\n Finished calculating the loop closures.") + + # Generate report + print("\n=== Loop Closure Report ===") + for category in [ + 2 * interval, + 3 * interval, + ]: # Restrict to relevant categories e.g. 18-6 and 12-6 as the main observations + for l in [interval]: # Restrict to the loop levels of interest + if category in loop_counts and l in loop_counts[category]: + generated = loop_counts[category].get(l, 0) + missing = missing_counts[category].get(l, 0) + print( + f"Loop Closure {category} - {l}: Generated = {generated}, Missing = {missing}" + ) + + print("\n=== Loop Closure Report for Long-interval Loop Closures ===") + + # Iterate over categories greater than 200 + for category in cat: + if category > 200: # Check for long categories + for l in range( + 1, + 5 * interval): # Check loop levels less than 5 * interval + if category in loop_counts and l in loop_counts[category]: + generated = loop_counts[category].get(l, 0) + if generated >= 1: # Only report if at least one loop is generated + print( + f"Loop Closure {category} - {l}: Generated = {generated}" + ) + + return loop, missing + + +################################################################################################ + +all_loops, missing = loop_calc(max_loop) + +# Define the directory path +directory_path = os.path.join(output_path, "Data") + +# Create the directory if it doesn't exist +os.makedirs(directory_path, exist_ok=True) + +# Define the file path +file_path = os.path.join(directory_path, f"All_loops_{start}_{end}.pkl") + +with open(file_path, "wb") as file: # for writting a dictionary to a file + pickle.dump(all_loops, file) + +### + +# Define the file path +file_path = os.path.join(directory_path, f"missing_loops") + +with open(file_path, "wb") as file: # for writting a dictionary to a file + pickle.dump(missing, file) diff --git a/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/PhaseBias_03_calibration_pars.py b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/PhaseBias_03_calibration_pars.py new file mode 100644 index 0000000..63c5c34 --- /dev/null +++ b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/PhaseBias_03_calibration_pars.py @@ -0,0 +1,424 @@ +#!/usr/bin/env python3 + +import numpy as np +import matplotlib.pyplot as plt +from matplotlib.gridspec import GridSpec +import pickle +import os +import sys +import glob +import configparser +import warnings + +with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=DeprecationWarning) + +################################################################################ + +HELP_TEXT = """ +PhaseBias_03_calibration_pars.py + +This script estimates the calibration parameters a_n using the loop closures calculated in the previous step. + +The calibration parameters a_n are estimated by calculating the ratio of the long-term loop closure for the interferogram of interest to the long-term loop closure for the base interferogram. Assuming negligible bias in long-term interferograms, a_n is calculated as follows: + + a_n = |φ(i,i+l) - Σ(t=i to i+l)φ(t,t+1+n)|_2π / |φ(i,i+l) - Σ(t=i to i+l)φ(t,t+1)|_2π + +where: +- φ(i,i+l) is a long-term interferogram connecting epoch i to i+l. +- φ(t,t+1) represents the base interferogram. +- φ(t,t+1+n) is the short interferogram to be corrected using a_n. + +### Input: +1. Loop closures from the previous step (e.g., 216-6, 216-12, and 216-18 for 6-day acquisition intervals). +2. The long-term interferogram length (e.g., 216 days) must be specified at the beginning. +3. The number of a_n parameters to estimate, specified using `num_a`. For example: + - Setting `num_a=2` indicates that a1 and a2 will be estimated (used to correct 12- and 18-day interferograms). + - For correcting longer interferograms, `num_a` should be set accordingly. + +### Output: +1. All possible a_n values are calculated. +2. Visualizations: + - Example plots of a_n arrays and their corresponding histograms, illustrating spatial variations. + - Time-series plots showing how a_n values evolve over time. +3. Mean a_n values over time are output for further use. + +### File Outputs: +- Results are stored in the output directory defined in the `config.txt` file under `/Data/` as `Mean_an_values.pkl`. + +### Additional Notes: +- Users should ensure that `num_a` aligns with the desired temporal baselines for correction. + +""" +if "--help" in sys.argv: + print(HELP_TEXT) + sys.exit(0) + + +############################################################################### +# Config file path +config_file = "config.txt" + +# Check if config file exists +if not os.path.exists(config_file): + raise FileNotFoundError( + f"Error: The configuration file '{config_file}' was not found in the script's directory." + ) + +# Initialize the configparser +config = configparser.ConfigParser() + + +# Read the config.txt file +config.read(config_file) + +# Define required parameters +required_parameters = [ + "root_path", + "frame", + "start", + "end", + "interval", + "nlook", + "LiCSAR_data", +] + +# Initialize a dictionary to store parameters +parameters = {} + +# Attempt to retrieve each parameter and catch errors +try: + parameters["root_path"] = config.get("DEFAULT", "root_path") + parameters["output_path"] = config.get("DEFAULT", "output_path") + parameters["frame"] = config.get("DEFAULT", "frame") + parameters["start"] = config.get("DEFAULT", "start") + parameters["end"] = config.get("DEFAULT", "end") + parameters["interval"] = config.getint("DEFAULT", "interval") + parameters["nlook"] = config.getint( + "DEFAULT", "nlook" + ) # Assuming 'nlook' should be an integer + parameters["num_a"] = config.getint("DEFAULT", "num_a") + parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") + +except configparser.NoOptionError as e: + # Error message if a required parameter is missing + missing_param = str(e).split(": ")[1] + raise ValueError( + f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." + ) +except configparser.Error as e: + # General error for any configparser-related issues + raise ValueError(f"Error while reading '{config_file}': {e}") + +# Assign to individual variables for easy access +root_path = parameters["root_path"] +output_path = parameters["output_path"] +frame = parameters["frame"] +start = parameters["start"] +end = parameters["end"] +interval = parameters["interval"] +nlook = parameters["nlook"] +num_a = parameters["num_a"] +LiCSAR_data = parameters["LiCSAR_data"] + +# hardcoded parameters, not included in the config file +landmask = 1 +filtered_ifgs = "yes" +max_loop = 5 # in case of 3 all loops with 6,12,18-day will be calculated (e.g. (60,6), (60,12) and (60,18)) + + +##################################################### Loop Closure calculation ##################################################### +#################################################################################################################################### +track = frame[0:3] +if track[0] == "0": + track = track[1:3] +if track[0] == "0": + track = track[1:2] + + +###################### Read coh: +print("\nReading all coherence data...") +# Define the path components +sub_dir = "Data" +filename_pattern = "All_coh_*" # Pattern to match files starting with "All_ifgs" + +# Construct the full file path with the refined pattern +file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + coh = pickle.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) +print("Reading coherence data completed.") + + +##################### Read Loops + +print("\nReading all loop closures...") +filename_pattern = "All_loops_*" # Pattern to match files starting with "All_ifgs" + +# Construct the full file path with the refined pattern +file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + all_loops = pickle.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) + +print("Reading loop closures completed.") + + +##################################################### Loop Closure calculation ##################################################### +#################################################################################################################################### + + +long_baseline = 216 # 36,72,108,144,180,216,252,288,324 +coh_thresh = 25 + + +def calc_an(loop_360_n, loop_360_6): + """ + Calculates the a_n values from loop closures. + + Parameters: + loop_360_n (ndarray): Array of loop closures for n-day intervals. + loop_360_6 (ndarray): Array of loop closures for 6-day intervals. + + Returns: + ndarray: Mean a_n values for all loop closures. + list: List of all a_n arrays for further processing or plotting. + """ + ratios = [] + + # Loop through indices + for i in range(len(loop_360_6)): + if loop_360_6[i] is not None and loop_360_n[i] is not None: + # Calculate the ratio for non-NaN elements + ratio_i = np.divide( + loop_360_n[i], + loop_360_6[i], + out=np.full_like(loop_360_n[i], np.nan), + where=~np.isnan(loop_360_6[i]), + ) + ratios.append(ratio_i) + +# Check if `ratios` is empty before proceeding + if not ratios: + print("Warning: No valid a_n values could be calculated. Returning NaN values.") + return np.array([np.nan]), [] + + + # Convert the list of ratios to a numpy array + an = np.array(ratios, dtype=object) + mean_an_long_baseline = [] + + # Calculate mean values of each a_n + for arr in an: + mask = (arr > 0) & (arr < 1) # Apply mask to filter invalid values + arr[~mask] = np.nan + if not np.isnan(arr).all(): # Check if all elements are NaN + mean_an = np.nanmean(arr) + else: + mean_an = np.nan # Assign NaN for completely empty slices + + mean_an_long_baseline.append(mean_an) + + return np.array(mean_an_long_baseline), an + + +################################################################################## +############################ +def plot_an(an_arrays, mean_a_long_baseline, k): + """ + Plots the a_n arrays with the largest coverage and their histograms in a two-row layout, + as well as the mean values of calibration parameters a_n over time. + + Parameters: + an_arrays (list): List of a_n arrays. + mean_a_long_baseline (ndarray): Array of mean calibration parameter values for a_1, a_2, ..., a_n. + k (int): The current value of k (e.g., 2 for a1, 3 for a2). + """ + + # Calculate the coverage (number of non-NaN values) for each array + coverage = [] + valid_an_arrays = [] # Store only valid arrays for plotting + + for arr in an_arrays: + if ( + arr is not None and arr.ndim == 2 and not np.all(np.isnan(arr)) + ): # Check for valid 2D arrays + coverage.append(np.sum(~np.isnan(arr))) # Count non-NaN values + valid_an_arrays.append(arr) + + # Sort arrays by coverage in descending order + sorted_indices = np.argsort(coverage)[::-1] + selected_arrays = [ + valid_an_arrays[idx] for idx in sorted_indices[:6] + ] # Select top 6 arrays + + # Plot the a_n arrays and their histograms in a two-row layout + if selected_arrays: + num_arrays = len(selected_arrays) + fig = plt.figure(figsize=(4 * num_arrays, 8)) + gs = GridSpec( + 2, num_arrays + 1, width_ratios=[1] * num_arrays + [0.1] + ) # Add space for colorbar + + # Plot selected arrays + axes_img = [] + for i, arr in enumerate(selected_arrays): + ax_img = fig.add_subplot(gs[0, i]) + im = ax_img.imshow(arr, cmap="RdYlBu", vmin=-0.8, vmax=0.8) + ax_img.set_title(f"{i+1}th $a_{{{k-1}}}$ Array") # Dynamically set title + ax_img.axis("off") + axes_img.append(ax_img) + + # Add a single colorbar for all array plots + cbar_ax = fig.add_subplot(gs[0, -1]) + fig.colorbar(im, cax=cbar_ax, orientation="vertical", label="Color Scale") + + # Plot histograms for the selected arrays + for i, arr in enumerate(selected_arrays): + ax_hist = fig.add_subplot(gs[1, i]) + ax_hist.hist(arr.flatten(), bins=100, color="blue", alpha=0.7, density=True) + ax_hist.set_title( + f"{i+1}th $a_{{{k-1}}}$ Histogram" + ) # Dynamically set title + ax_hist.set_xlim([-3, 3]) # Adjust histogram range as needed + + plt.tight_layout() + plt.show() + else: + print("No valid a_n arrays available for plotting.") + + # Plot the mean values of a_n over time + x_values = np.arange(len(mean_a_long_baseline)) # Sequential indices for x-axis + fig, ax = plt.subplots(figsize=(6, 4)) + + + # Filter out NaN values to avoid gaps in the plot + valid_indices = ~np.isnan(mean_a_long_baseline) # Mask for valid (non-NaN) entries + x_valid = x_values[valid_indices] # Only use valid time steps + y_valid = np.array(mean_a_long_baseline)[valid_indices] # Only use valid a_n values + + + # Remove NaN values for trendline fitting + valid_indices = ~np.isnan(mean_a_long_baseline) # Mask for valid (non-NaN) entries + if ( + valid_indices.sum() > 1 + ): # Ensure there are at least two valid points for fitting +# x_valid = x_values[valid_indices] + y_valid = np.array(mean_a_long_baseline)[valid_indices] + x_valid = np.arange(len(y_valid)) + + # Fit and plot a trendline + coefficients = np.polyfit(x_valid, y_valid, 1) + trendline = np.polyval(coefficients, x_valid) + ax.plot(x_valid, trendline, color="red", label="Trendline") + + # Scatter plot for all points (including NaN if any) + ax.scatter( + #x_values, + #mean_a_long_baseline, + x_valid, # Only use valid time steps + y_valid, # Only use valid a_n values + + color="blue", + marker="o", + label="Mean Values", + s=9, + ) + + # Set labels and limits + ax.set_ylim(-3, 3) # Adjust limits as needed + ax.set_xlabel("Time Step") # Sequential index as x-axis + ax.set_ylabel("Mean Values", fontsize=12) + ax.legend() + + # Adjust layout and show the plot + plt.tight_layout() + plt.show() + + +## +######### forming 360-6 + +loop_360_6_all = np.array(all_loops[long_baseline][interval], dtype=object) +coh_360 = np.array(coh[long_baseline][:], dtype=object) + +loop_360_6 = np.full_like( + loop_360_6_all, np.nan +) # create a new array as the same size of loop_360_6_all with nan values + + +for i, arr in enumerate(loop_360_6_all): + if arr is not None: + (frame_row, frame_col) = np.shape(arr) + loop_360_6[i] = np.where((coh_360[i] > coh_thresh), arr, np.nan) + +###### forming 360-n (n=12,18,24 etc) +# mean_a_long_baseline = [] +final_mean_values = [] +an_labels = [] + +# Loop through the values of k corresponding to the number of calibration parameters +for k in range(2, 2 + num_a): # k=2 for a1, k=3 for a2, etc. + mean_an_long_baseline = [] + + # Forming 360-n (n=12, 18, 24, etc.) + loop_360_n_all = np.array(all_loops[long_baseline][k * interval], dtype=object) + loop_360_n = np.full_like(loop_360_n_all, np.nan) + + for i, arr in enumerate(loop_360_n_all): + if arr is not None: + # Apply coherence threshold filtering + loop_360_n[i] = np.where((coh_360[i] > coh_thresh), arr, np.nan) + + # Calculate a_n values + mean_an_long_baseline, an_arrays = calc_an(loop_360_n, loop_360_6) + + # If all values in mean_an_long_baseline are nan (empty ratios), stop execution + if np.isnan(mean_an_long_baseline).all(): + print("No valid a_n values could be calculated. Stopping execution.") + sys.exit(1) # Exit the script with an error code (1 means failure) + + # Calculate the mean of all mean values for this k and store it + overall_mean = np.nanmean(mean_an_long_baseline) + final_mean_values.append(overall_mean) + an_labels.append(f"a{k-1}") + + for idx, mean_a in enumerate(mean_an_long_baseline, start=1): + if not np.isnan(mean_a): # Check if the value is not NaN + print(f"Mean of the {idx}th a({k-1}) is {mean_a:.6f}") + + plot_an(an_arrays, mean_an_long_baseline, k) + + +# Print the final mean values across all k +for k_idx, final_mean in enumerate(final_mean_values, start=2): + print(f"Final mean value for a({k_idx-1}) is {final_mean:.6f}") + +# Write final mean values to a text file +output_path = os.path.join(parameters["output_path"], "Data") +os.makedirs(output_path, exist_ok=True) # Create directory if it doesn't exist +output_file = os.path.join(output_path, "an.txt") + +with open(output_file, "w") as file: + for label, mean_value in zip(an_labels, final_mean_values): + file.write(f"{label}={mean_value:.6f}\n") + +print(f"Final mean values written to {output_file}.") diff --git a/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/PhaseBias_04_Inversion.py b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/PhaseBias_04_Inversion.py new file mode 100644 index 0000000..2913c3e --- /dev/null +++ b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/PhaseBias_04_Inversion.py @@ -0,0 +1,683 @@ +#!/usr/bin/env python3 + +import numpy as np +import pickle +import os +from scipy.sparse import csc_array +from scipy.sparse.linalg import lsqr +import sys +import math +from bin.read_orig_ifgs_coh import read_orig_ifgs_coh +from bin.circular_mean_var import circular_mean_and_variance_over_epochs +import glob +import configparser +import warnings + +warnings.filterwarnings("ignore", category=DeprecationWarning) + +################################################################################ + +HELP_TEXT = """ +PhaseBias_04_Inversion.py + +This script estimates the phase bias terms for the base interferograms using short-term closure phases as input. +The bias terms derived in this step serve as the foundation for correcting other short-term interferograms. + +### Workflow: +1. **Masking Noisy Loop Closures:** + - A moving average is applied to the time-series of loop closures to account for seasonal fluctuations. + - A circular moving average (mean of complex values over time) is computed for each pixel. + - The distance of each loop closure from the moving average is calculated. If the distance exceeds a threshold (2σ), the point is masked as noisy. + - The circular standard deviation (σ) is computed using the von Mises distribution (circular normal distribution). + +2. **Refined Loop Closures:** + - Masked loop closures, denoted as 〖Δφ〗_(i,i+2)^r and 〖Δφ〗_(i,i+3)^r, are used as primary observations for phase bias estimation. + +3. **First Inversion (Without Temporal Smoothing Constraints):** + - Estimates the bias terms that can be corrected directly from the observed loop closures. + +4. **Second Inversion (With Temporal Smoothing Constraints):** + - Incorporates temporal smoothing constraints to estimate additional bias terms that cannot be directly derived from loop closures. + - The temporal smoothing minimizes differences between bias terms over time, ensuring consistency. + +5. **Combining Results:** + - Bias terms from the first inversion are combined with the results of the second inversion to include terms not estimated in the first round. + +### Outputs: +- The phase bias terms for the base interferograms are stored as a NumPy array file: + - `X_base_ifgs_biases.npy`, saved in the `Data` directory under the `output_path` defined in `config.txt`. + + +### Input Requirements: +- Short-term closure phases (from `PhaseBias_02_Loop_Closures.py`). +- Configuration parameters, including thresholds and temporal settings, defined in `config.txt`. + +### Output Files: +- `X_base_ifgs_biases.npy`: Stores the estimated bias terms for the base interferograms. +""" + +if "--help" in sys.argv: + print(HELP_TEXT) + sys.exit(0) + +############################################################################### +# Config file path +config_file = "config.txt" + +# Check if config file exists +if not os.path.exists(config_file): + raise FileNotFoundError( + f"Error: The configuration file '{config_file}' was not found in the script's directory." + ) + +# Initialize the configparser +config = configparser.ConfigParser() + +# Read the config.txt file +config.read(config_file) + +# Define required parameters +required_parameters = [ + "root_path", + "frame", + "start", + "end", + "interval", + "nlook", + "LiCSAR_data", + "a1_6_day", + "a2_6_day", + "a1_12_day", + "a2_12_day", + "estimate_an_values", +] + +# Initialize a dictionary to store parameters +parameters = {} + +# Attempt to retrieve each parameter and catch errors +try: + parameters["root_path"] = config.get("DEFAULT", "root_path") + parameters["output_path"] = config.get("DEFAULT", "output_path") + parameters["frame"] = config.get("DEFAULT", "frame") + parameters["start"] = config.get("DEFAULT", "start") + parameters["end"] = config.get("DEFAULT", "end") + parameters["interval"] = config.getint("DEFAULT", "interval") + parameters["nlook"] = config.getint( + "DEFAULT", "nlook" + ) # Assuming 'nlook' should be an integer + parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") + parameters["a1_6_day"] = config.getfloat("DEFAULT", "a1_6_day") + parameters["a2_6_day"] = config.getfloat("DEFAULT", "a2_6_day") + parameters["a1_12_day"] = config.getfloat("DEFAULT", "a1_12_day") + parameters["a2_12_day"] = config.getfloat("DEFAULT", "a2_12_day") + parameters["estimate_an_values"] = config.get("DEFAULT", "estimate_an_values") + +except configparser.NoOptionError as e: + # Error message if a required parameter is missing + missing_param = str(e).split(": ")[1] + raise ValueError( + f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." + ) +except configparser.Error as e: + # General error for any configparser-related issues + raise ValueError(f"Error while reading '{config_file}': {e}") + +# Assign to individual variables for easy access +root_path = parameters["root_path"] +output_path = parameters["output_path"] +frame = parameters["frame"] +start = parameters["start"] +end = parameters["end"] +interval = parameters["interval"] +nlook = parameters["nlook"] +LiCSAR_data = parameters["LiCSAR_data"] +estimate_an_values = parameters["estimate_an_values"] + + +# Initialize a1 and a2 with None for safety +a1 = None +a2 = None + + +print("estimate_an_values is ", estimate_an_values) +# Check if estimate_an_values is 'yes' +if estimate_an_values == "yes": + an_file_path = os.path.join(output_path, "Data", "an.txt") + + # Try reading the file + try: + if os.path.exists(an_file_path): + print(f"Reading 'a' values from {an_file_path}...") + + # Read the file and load the values + with open(an_file_path, "r") as file: + for line in file: + line = line.strip() + if line.startswith("a1="): + a1 = float(line.split("=")[1]) + elif line.startswith("a2="): + a2 = float(line.split("=")[1]) + + # Check if both a1 and a2 were found + if a1 is None or a2 is None: + print( + "Warning: Missing a1 or a2 in an.txt. Switching to default values from config file." + ) + raise ValueError("Incomplete a1 or a2 values.") + else: + print( + f"Warning: {an_file_path} does not exist. Switching to default values from config file." + ) + raise FileNotFoundError + + except (FileNotFoundError, ValueError): + # Fallback to default values from config file + if interval == 6: + a1 = parameters["a1_6_day"] + a2 = parameters["a2_6_day"] + else: + a1 = parameters["a1_12_day"] + a2 = parameters["a2_12_day"] +else: + # Use default values from the config file + print("Using default 'an' values from the config file.") + if interval == 6: + a1 = parameters["a1_6_day"] + a2 = parameters["a2_6_day"] + else: + a1 = parameters["a1_12_day"] + a2 = parameters["a2_12_day"] + +# Print the chosen values +print(f"Final values - a1: {a1}, a2: {a2}") + + +##################################################### Loop Closure calculation ##################################################### +#################################################################################################################################### +#################################################################################################################################### + +track = frame[0:3] +if track[0] == "0": + track = track[1:3] +if track[0] == "0": + track = track[1:2] + +with_temporals = "yes" # it should be yes, if you want to incorporate the temporal constraints +apply_to_all = "no" # in case of with_temporal='yes', we can decide if you want to use the temporal to all ifgs (i.e. apply_to_all='yes'), or just to those that can not be corrected (i.e. apply_to_all='no') +w = 0.1 # the weigth of the temporal smoothing constraints +####### +coh_thresh = 11 # threhsold on the coherence values. In this version it is off +min_num_eq = 10 # I used 10 in all my experiments. min number of equations in the least square inversion. + +############################### Reading input data ############################################# +#################### Read ifgs: +print("Reading all ifgs...") +# Define the path components +sub_dir = "Data" +filename_pattern = "All_ifgs_*" # Pattern to match files starting with "All_ifgs" + +# Construct the full file path with the refined pattern +file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + all_ifgs = pickle.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) +print("Reading ifgs completed.") + +###################### Read coh: +print("\nReading all coherence data...") +# Define the path components +sub_dir = "Data" +filename_pattern = "All_coh_*" # Pattern to match files starting with "All_ifgs" + +# Construct the full file path with the refined pattern +file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + all_coh = pickle.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) +print("Reading coherence data completed.") + +##################### Read Loops + +print("\nReading all loop closures...") +filename_pattern = "All_loops_*" # Pattern to match files starting with "All_ifgs" + +# Construct the full file path with the refined pattern +file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + all_loops = pickle.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) + +print("Reading loop closures completed.") + +all_cat = [] +for cat in all_loops: + all_cat.append(cat) + +### stacking all the ifgs 6/12/18 in a numpy var all_ifgs +print( + "Extracting the required interferograms(up to 3 epochs), and the corresponding loop closures for inversion..." +) +desired_categories = [interval, 2 * interval, 3 * interval] +all_ifgs, _, existing_ifgs_index = read_orig_ifgs_coh( + all_ifgs, all_coh, desired_categories +) + +######### forming 12-6 and 18-6 loop closures from the imported data +################################################################# +loop_12 = np.array(all_loops[2 * interval][interval], dtype=object) +#loop_12 = np.array(all_loops[2 * interval][interval]) +#coh_12 = np.array(all_coh[2 * interval][:]) +coh_12 = np.array(all_coh[2 * interval][:], dtype=object) + +#loop_18 = np.array(all_loops[3 * interval][interval]) +#coh_18 = np.array(all_coh[3 * interval][:]) +loop_18 = np.array(all_loops[3 * interval][interval], dtype=object) +coh_18 = np.array(all_coh[3 * interval][:], dtype=object) + +############## finding dynamic pixel-based thresholds to remove the noisy loops +loop_12_orig = [] +loop_18_orig = [] +none_indices12 = [] +none_indices18 = [] + +# refine loop_12 to exclude the none indices and create loop_12_orig +for i, arr in enumerate(loop_12): + if arr is not None: + (frame_row, frame_col) = np.shape(arr) + loop_12_orig.append(arr) + else: + none_indices12.append(i) + +# refine loop_18 to exclude the none indices and create loop_18_orig +for i, arr in enumerate(loop_18): + if arr is not None: + loop_18_orig.append(arr) + else: + none_indices18.append(i) + +loop_12_orig = np.array(loop_12_orig) +loop_18_orig = np.array(loop_18_orig) + + +### removing noisy loops +print( + "Masking noisy loop closures using circular moving average and standard deviaiton of the loop closures in time:" +) + +# using circular_mean to calculate the mean phase value using complex numbers, and von_mises_variance to calculate the second moment of the Von Mises distribution +_, thresh_loop12 = circular_mean_and_variance_over_epochs(loop_12_orig, axis=0) +_, thresh_loop18 = circular_mean_and_variance_over_epochs(loop_18_orig, axis=0) + + +# Add a new axis at the beginning (axis=0) of these arrays to have shape (1, ...) +# These are the threshold vlaues obtained for each pixel that will be used for masking later +thresh_loop12 = np.expand_dims(thresh_loop12, axis=0) +thresh_loop18 = np.expand_dims(thresh_loop18, axis=0) + + +# calculate a temporal moving average for wrapped data +def moving_average(arr, window_size): + # Initialize an empty array to store the temporal averages + moving_averages = np.zeros_like(arr) + half_window = window_size // 2 + ### miroring data at both ends for window_size/2 at each end + mirrored_shape = (arr.shape[0] + window_size, arr.shape[1], arr.shape[2]) + + # Create an empty array to hold the mirrored data + mirrored_arr = np.empty(mirrored_shape, dtype=arr.dtype) + + # Fill the mirrored array with the original data + mirrored_arr[window_size // 2 : -window_size // 2] = arr + + # Mirror the data at the beginning + for i in range(window_size // 2): +# mirrored_arr[i] = arr[window_size // 2 - i] + mirrored_arr[i] = arr[min(half_window - i, arr.shape[0] - 1)] + + # Mirror the data at the end + for i in range(window_size // 2): +# mirrored_arr[-(i + 1)] = arr[-(i + 1) - window_size // 2] + mirrored_arr[-(i + 1)] = arr[max(-1 - (half_window - i), 0)] + #### + + # Calculate temporal averages within each window + for t in range(half_window, mirrored_arr.shape[0] - half_window): + moving_averages[t - half_window, :, :] = np.angle( + np.nanmean( + np.exp(1j * (mirrored_arr[t - half_window : t + half_window, :, :])), + axis=0, + ) + ) + return moving_averages + + +# for loop 12 +# Calculate distance between each epoch and the moving average for each pixel +print( + f"Calculating the moving average for {2 * interval} loop closures, followed by masking noisy loop closures..." +) +print( + "This may take a while, as it depends on the number of loops and the size of your dataset.\n" +) + +window_size = 30 + +moving_avg = moving_average(loop_12_orig, window_size) +distances = np.abs(np.angle(np.exp(1j * (loop_12_orig - moving_avg)))) +mask = distances > 2 * thresh_loop12 # it is 4*sigma + +# Replace values with NaN where mask is True +loop_12_bk = loop_12_orig +loop_12_orig = np.where(mask, np.nan, loop_12_orig) +loop_12 = loop_12_orig.tolist() + +for idx in none_indices12: + loop_12.insert(idx, None) +loop_12 = np.array(loop_12, dtype=object) + +# for loop 18 +# Calculate distance between each epoch and the moving average for each pixel +print( + f"Calculating the moving average for {3 * interval} loop closures, followed by masking noisy loop closures..." +) +print( + "This may take a while, as it depends on the number of loops and the size of your dataset.\n" +) + +moving_avg = moving_average(loop_18_orig, window_size) +distances = np.abs(np.angle(np.exp(1j * (loop_18_orig - moving_avg)))) +mask = distances > 2 * thresh_loop18 # it is 4*sigma + +# Replace values with NaN where mask is True +loop_18_bk = loop_18_orig +loop_18_orig = np.where(mask, np.nan, loop_18_orig) +loop_18 = loop_18_orig.tolist() + +for idx in none_indices18: + loop_18.insert(idx, None) +loop_18 = np.array(loop_18, dtype=object) + +print("Masking of noisy loop closures is completed.") + +# ################ Apply coherece thresholding +# print('shape loop_12 = ', np.shape(loop_12)) +# print('shape loop_12_orig = ', np.shape(loop_12_orig)) +# print('shape coh_12 = ', np.shape(coh_12)) +# for i, arr in enumerate(loop_12): +# if arr is not None: +# (frame_row, frame_col) = np.shape(arr) +# loop_12[i] = np.where((coh_12[i] > coh_thresh), arr, np.nan) +# +# +# for i, arr in enumerate(loop_18): +# if arr is not None: +# loop_18[i] = np.where((coh_18[i] > coh_thresh), arr, np.nan) +# +# +# + +len12 = len(loop_12) # number of 12 day loops including None +len18 = len(loop_18) # number of 18 day loops including None + +######################## Forming the design matrix A ############################## +#################################################################################### + +print("Preparing the design matrix and observation vector for the inversion step...") + +b = [] +A = [] +n_unk = ( + len12 + 1 +) # this in an initial value(i.e. the num of 6 biases). will be changed by the actual number of unk after considering the None values/missing +# print('n_unk = ', n_unk) + +row = np.zeros( + n_unk, dtype=np.float32 +) # each row of the design matrix A for the 12-day +for i in range(len12): + b.append(loop_12[i]) + # print('np.shape(loop_12[i] = ', np.shape(loop_12[i])) + row[i : i + 2] = a1 - 1 + A.append(row) + row = np.zeros(n_unk, dtype=np.float32) + +row = np.zeros( + n_unk, dtype=np.float32 +) # each row of the design matrix A for the 18-day +for i in range(len18): + b.append(loop_18[i]) + row[i : i + 3] = a2 - 1 + A.append(row) + row = np.zeros(n_unk, dtype=np.float32) + +A = np.array(A) +b = np.array(b, dtype=object) + + +####################### removing the rows from b and A where b is None +mask = [] +mask = np.array([arr is not None for arr in b]) +# Convert the boolean mask to an integer mask +mask = np.nonzero(mask) + +A = A[mask] # Filter rows of A based on the mask +b = b[mask] + +b = b.tolist() + +############ removing the columns of A where are values are zero (these are the unkonws which doesn't fall in any equations nor 12 neither 18 and thus cannot be corrected) + +# Find the column indices where all values are zero +zero_columns = np.all(A == 0, axis=0) + +## Get the indices of the zero columns +zero_column_indices = np.where(zero_columns)[0] + +# Get the indices of the non-zero columns +non_zero_column_indices = np.where(~zero_columns)[ + 0 +] # the indices of the unknowns that can be corrected + +# Save the file +directory_path = os.path.join(output_path, "Data") +file_path = os.path.join(directory_path, f"indices_unknown_tobe_corrected.npy") +np.save(file_path, non_zero_column_indices) + + +### Adding all the existing 6-day ifgs indices to non_zero_column_indices. We want to use estimate their biases in the second step. + + +A_bk = A # to keep a copy of A in case we want to esimate all 6-day biases with smoothing constraints +b_bk = b +all_column_indices = np.array( + range(A_bk.shape[1]) +) # generating all indices in case we want to esimate all 6-day biases with smoothing constraints + +A = A[:, non_zero_column_indices] + +# finding the last column with value -1. this gives the number of 6-day biases that can be corrected +last_column_with_minus_1 = None + +# Iterate through the columns from right to left +for col in range(len(A[0]) - 1, -1, -1): + if -1 in [row[col] for row in A]: + last_column_with_minus_1 = col + break + +num_rows_before_temporals = A.shape[0] + +################################# Least Square inversion ############################### +######################################################################################## +# First Inversion: Without Temporal Smoothing Constraints +print("Starting the first inversion (without temporal smoothing constraints).") +print( + "This step estimates the bias terms that can be corrected based on observed loop closures." +) + +damp_factor = 0 + +if apply_to_all == "no": # this is without using any temporal constraints + + X1 = np.zeros((np.shape(A_bk)[1], frame_row, frame_col), dtype=np.float32) + # X1[:,:,:] = np.nan # after checking noticed this doesn't have any effect on the final vel + b_bk = np.array(b_bk) + + total_pixels = frame_row * frame_col # Total number of pixels for progress bar + processed_pixels = 0 # Counter for processed pixels + print("Progress: [", end="", flush=True) + + for row in range(frame_row): + for col in range(frame_col): + processed_pixels += 1 + # Update progress bar + percentage = (processed_pixels / total_pixels) * 100 + if processed_pixels % (total_pixels // 100) == 0: # Update every 1% + print(f"{int(percentage)}%", end="", flush=True) + sys.stdout.write( + "\rProgress: [" + + "=" * (int(percentage) // 2) + + " " * (50 - int(percentage) // 2) + + "]" + ) + + non_nan_mask = ~np.isnan(b_bk[:, row, col]) + bb = b_bk[non_nan_mask, row, col] # Remove NaN values + AA = A_bk[non_nan_mask, :] # Remove corresponding rows + if len(bb) > min_num_eq: # what is the min number of equation? + + x, istop, itn, normr, normr2 = lsqr( + csc_array(AA), bb, damp=damp_factor + )[ + :5 + ] # Using sparse matrix representation scipy.sparse.linalg + X1[:, row, col] = x + + +# if with_temporals == "no": # this solution will be the final answer to be saved as X +# # Fill X with values from X1 according to non_zero_column_indices +# for i, idx in enumerate(non_zero_column_indices): +# X[idx, :, :] = X1[i, :, :] + + + + +print("\nFirst inversion (without temporal smoothing constraints) completed.") + +########################### +# Second Inversion: With Temporal Smoothing Constraints +print("Starting the second inversion (with temporal smoothing constraints).") +print( + "This step estimates all bias terms, including those that cannot be corrected, using temporal smoothing constraints." +) + +if with_temporals == "yes": # This is using the temporal constranints on all unknowns + X2 = np.zeros((np.shape(A_bk)[1], frame_row, frame_col), dtype=np.float32) + # X2[:,:,:] = np.nan # after checking noticed this doesn't have any effect on the final vel + b_bk = np.array(b_bk) + + #### Adding temporal constraints + row = np.zeros( + A_bk.shape[1], dtype=np.float32 + ) # additional rows the design matrix A for temporal constraints, setting that to zero again + for i in range(len(all_column_indices) - 2): # for all unknowns + row[i] = -1 * w + row[i + 1] = 2 * w + row[i + 2] = -1 * w + A_bk = np.vstack((A_bk, row)) + row = np.zeros( + A_bk.shape[1], dtype=np.float32 + ) # additional rows the design matrix A for temporal constraints, setting that to zero again + num_temporals = len(A_bk) - len(b_bk) + # print('num_temporals is ', num_temporals) + + ######### Appending zeroes to b according to the added number of temporal constraints obtained by np.shape(A)[0] - np.shape(b)[0] + b_0 = np.zeros( + (np.shape(A_bk)[0] - np.shape(b)[0], frame_row, frame_col), dtype=np.float32 + ) + b_bk = np.vstack((b_bk, b_0)) + + thresh_n_eq = num_temporals # in case of using temporals, we don't want the nan pixels with zero equations with only using temporals + + processed_pixels = 0 # Reset counter for second inversion + print("Progress: [", end="", flush=True) + + for row in range(frame_row): + for col in range(frame_col): + processed_pixels += 1 + # Update progress bar + percentage = (processed_pixels / total_pixels) * 100 + if processed_pixels % (total_pixels // 100) == 0: # Update every 1% + print(f"{int(percentage)}%", end="", flush=True) + sys.stdout.write( + "\rProgress: [" + + "=" * (int(percentage) // 2) + + " " * (50 - int(percentage) // 2) + + "]" + ) + + non_nan_mask = ~np.isnan(b_bk[:, row, col]) + bb = b_bk[non_nan_mask, row, col] # Remove NaN values + AA = A_bk[non_nan_mask, :] # Remove corresponding rows + if len(bb) > thresh_n_eq: + + x, istop, itn, normr, normr2 = lsqr( + csc_array(AA), bb, damp=damp_factor + )[ + :5 + ] # Using sparse matrix representation scipy.sparse.linalg + X2[:, row, col] = x + + print("\nSecond inversion (with temporal smoothing constraints) completed.") + + ####### combining X1 and X2 into X + + X = np.zeros((np.shape(A_bk)[1], frame_row, frame_col), dtype=np.float32) + X[:, :, :] = np.nan + + # Fill X with values from X1 according to non_zero_column_indices + for i, idx in enumerate(non_zero_column_indices): + X[idx, :, :] = X1[i, :, :] + + # Fill in the missing values from X2 + for i, idx in enumerate(all_column_indices): + if idx not in non_zero_column_indices: + X[idx, :, :] = X2[i, :, :] + + # Ensure the dtype is appropriate for your data + X = X.astype(np.float32) + +directory_path = os.path.join(output_path, "Data") +file_path = os.path.join(directory_path, f"X_base_ifgs_biases.npy") +np.save(file_path, X) + +print("Inversion process completed. Results saved to:", file_path) + +########################################################################################################## diff --git a/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/PhaseBias_05_Correction.py b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/PhaseBias_05_Correction.py new file mode 100644 index 0000000..14cbca0 --- /dev/null +++ b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/PhaseBias_05_Correction.py @@ -0,0 +1,535 @@ +#!/usr/bin/env python3 + +import numpy as np +from datetime import datetime +import pickle +import time +import os +from bin.generate_ifgs_from_epochs import generate_ifg_pairs +import configparser +import glob +from bin.resample import resample_geotiff +import sys + +# sys.path.append(os.path.dirname(os.path.abspath(__file__))) +import bin +from bin.read_orig_ifgs_coh import read_orig_ifgs_coh + +from bin.ifg_cc_totif import ifg_cc_2tif +import gc + +############################################################################### + +HELP_TEXT = """ +PhaseBias_05_Correction.py + +This script applies phase bias correction to desired interferograms using the estimated bias terms for the base interferograms. The corrected interferograms are saved in GeoTIFF format for further analysis. + +### Workflow: +1. **Inputs:** + - **Original Wrapped Interferograms, and Coherence Data:** Obtained from step 1 (`PhaseBias_01_Read_Data.py`). + - **Estimated Bias Terms for Base Intierferograms:** Obtained from step 4 (`PhaseBias_04_Inversion.py`). + - **Calibration Parameters (*a_n*):** Estimated from step 3 (`PhaseBias_03_calibration_pars.py`). + - **max_con:** Defines the maximum number of connections to be corrected. For instance: + - `max_con=5` corrects interferograms up to 30-day intervals for a 6-day acquisition interval. + - Adjust `max_con` as needed. + +2. **Bias Term Estimation for Desired Interferograms:** + - Bias terms for desired interferograms, δ_(i,i+n+1), are estimated using the relationship: + δ_(i,i+n+1) = a_n * (∑_(t=i)^(i+n) δ_(t,t+1) ) + +3. **Correction of Interferograms:** + - Using the estimated bias terms, the interferograms are corrected as follows: + φ_(i,i+n+1)^c = φ_(i,i+n+1) - δ_(i,i+n+1) + + - Here, φ_(i,i+n+1) is the original interferogram, and φ_(i,i+n+1)^c is the corrected interferogram. + +4. **Outputs:** + - Corrected interferograms are saved in GeoTIFF format in the `GEOC` directory under the `output_path` specified in `config.txt`. + +### Output File Format: +- **Corrected Interferograms:** Saved in GeoTIFF format, organized by temporal baseline. + +### Input Requirements: +- Original wrapped interferograms from step 1. +- Bias terms for base interferograms from step 4. +- Configuration parameters (e.g., `output_path`, `interval`) defined in `config.txt`. + +### Output Directory: +- `GEOC` directory under `output_path`: Contains the corrected interferograms in GeoTIFF format, organized by temporal baseline. + +""" +if "--help" in sys.argv: + print(HELP_TEXT) + sys.exit(0) + + +################################################################################ + + +# Config file path +config_file = "config.txt" + +# Check if config file exists +if not os.path.exists(config_file): + raise FileNotFoundError( + f"Error: The configuration file '{config_file}' was not found in the script's directory." + ) + +# Initialize the configparser +config = configparser.ConfigParser() + + +# Read the config.txt file +config.read(config_file) + +print("Config keys:", list(config.defaults().keys())) + + +# Define required parameters +required_parameters = [ + "root_path", + "frame", + "start", + "end", + "interval", + "nlook", + "LiCSAR_data", + "a1_6_day", + "a2_6_day", + "a3_6_day", + "a4_6_day", + "a1_12_day", + "a2_12_day", + "a3_12_day", + "a4_12_day", + "estimate_an_values", +] + +# Initialize a dictionary to store parameters +parameters = {} + +# Attempt to retrieve each parameter and catch errors +try: + parameters["root_path"] = config.get("DEFAULT", "root_path") + parameters["output_path"] = config.get("DEFAULT", "output_path") + parameters["frame"] = config.get("DEFAULT", "frame") + parameters["start"] = config.get("DEFAULT", "start") + parameters["end"] = config.get("DEFAULT", "end") + parameters["interval"] = config.getint("DEFAULT", "interval") + parameters["nlook"] = config.getint( + "DEFAULT", "nlook" + ) # Assuming 'nlook' should be an integer + parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") + parameters["a1_6_day"] = config.getfloat("DEFAULT", "a1_6_day") + parameters["a2_6_day"] = config.getfloat("DEFAULT", "a2_6_day") + parameters["a3_6_day"] = config.getfloat("DEFAULT", "a3_6_day") + parameters["a4_6_day"] = config.getfloat("DEFAULT", "a4_6_day") + parameters["a1_12_day"] = config.getfloat("DEFAULT", "a1_12_day") + parameters["a2_12_day"] = config.getfloat("DEFAULT", "a2_12_day") + parameters["a3_12_day"] = config.getfloat("DEFAULT", "a3_12_day") + parameters["a4_12_day"] = config.getfloat("DEFAULT", "a4_12_day") + parameters["estimate_an_values"] = config.get("DEFAULT", "estimate_an_values") + +except configparser.NoOptionError as e: + # Error message if a required parameter is missing + missing_param = str(e).split(": ")[1] + raise ValueError( + f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." + ) +except configparser.Error as e: + # General error for any configparser-related issues + raise ValueError(f"Error while reading '{config_file}': {e}") + +# Assign to individual variables for easy access +root_path = parameters["root_path"] +output_path = parameters["output_path"] +frame = parameters["frame"] +start = parameters["start"] +end = parameters["end"] +interval = parameters["interval"] +nlook = parameters["nlook"] +LiCSAR_data = parameters["LiCSAR_data"] +estimate_an_values = parameters["estimate_an_values"] + + +start_date = datetime.strptime(start, "%Y%m%d") +end_date = datetime.strptime(end, "%Y%m%d") + +uncor_only = 1 # in case of 1 only uses 6-12-18 uncorrected ifgs. it is based on the desired_lengths ifgs +max_con = 5 # in case of 5, it will correct up to 6*5=30-day ifgs. + + +##################################################### +# Root directory where the category directories are located + +if LiCSAR_data == "yes": + track = frame[0:3] # extracting track number from frame id + if track[0] == "0": + track = track[1:3] + if track[0] == "0": + track = track[1:2] + + root_directory = ( + f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}" + ) +else: + root_directory = root_path + + +############################################ + +# Initialize the an values with None for safety +an_values = [None] * ( + max_con - 1 +) # For max_con=5, this will create [None, None, None, None] + +print("estimate_an_values is ", estimate_an_values) + +# Check if estimate_an_values is 'yes' +if estimate_an_values == "yes": + an_file_path = os.path.join(output_path, "Data", "an.txt") + + # Try reading the file + try: + if os.path.exists(an_file_path): + print(f"Reading 'a' values from {an_file_path}...") + + # Read the file and load the an values dynamically + with open(an_file_path, "r") as file: + for line in file: + line = line.strip() + for i in range( + 1, max_con + ): # Loop through a1, a2, ..., a(max_con-1) + param_name = f"a{i}" + if line.startswith(f"{param_name}="): + an_values[i - 1] = float(line.split("=")[1]) + + # Check if all required an values were found + if any(value is None for value in an_values): + print( + "Warning: Missing one or more an values in an.txt. Switching to default values from config file." + ) + raise ValueError("Incomplete an values.") + else: + print( + f"Warning: {an_file_path} does not exist. Switching to default values from config file." + ) + raise FileNotFoundError + + except (FileNotFoundError, ValueError): + # Fallback to default values from config file + print("Loading default 'an' values from config file...") + for i in range(1, max_con): + if interval == 6: + an_values[i - 1] = parameters[f"a{i}_6_day"] + else: + an_values[i - 1] = parameters[f"a{i}_12_day"] +else: + # Use default values from the config file + print("Using default 'an' values from the config file.") + for i in range(1, max_con): + if interval == 6: + an_values[i - 1] = parameters[f"a{i}_6_day"] + else: + an_values[i - 1] = parameters[f"a{i}_12_day"] + +# Print the chosen an values +for i, value in enumerate(an_values, start=1): + print(f"Final value - a{i}: {value}") + + +############## generating ifg pairs between two epochs + +# desired_lengths = [interval, 2*interval, 3*interval, 4*interval, 5*interval] +desired_lengths = [interval * i for i in range(1, max_con + 1)] # replacing the above + +all_ifgs_string = generate_ifg_pairs(start_date, end_date, interval, desired_lengths) + + +##### reading original ifgs/coh ############################# +#################################################################################### + +############################### Reading input data ############################################# +#################### Read ifgs: +print("Reading all ifgs...") +# Define the path components +sub_dir = "Data" +filename_pattern = "All_ifgs_*" # Pattern to match files starting with "All_ifgs" + +# Construct the full file path with the refined pattern +file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + ifgs = pickle.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) + + +###################### Read coh: +print("\nReading all coherence data...") +# Define the path components +sub_dir = "Data" +filename_pattern = "All_coh_*" # Pattern to match files starting with "All_ifgs" + +# Construct the full file path with the refined pattern +file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + coh = pickle.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) +print("\nReading data completed.") + + +######## stacking all the ifgs 6/12/18 in a numpy var all_ifgs +desired_categories = desired_lengths +print('stacking all the ifgs 6/12/18 in a numpy var all_ifgs') +all_ifgs, all_coh, existing_ifgs_index = read_orig_ifgs_coh( + ifgs, coh, desired_categories +) +print('stacking completed') + +del ifgs, coh +gc.collect() + +print('ifgs coh deleted') +##### Reading the estimated bias terms as well as the indices of unknowns that can be corrected obtained in step 03 ############## +############################################################################################## +print("Correcting the interferograms...") +# Define the path components +sub_dir = "Data" +filename_pattern_X = "X_base_ifgs_biases.npy" # Pattern to match files starting with "X_base" +filename_pattern_indices = "indices*" + +# Construct the full file path with the refined pattern +file_path_pattern_X = os.path.join(output_path, sub_dir, filename_pattern_X) +file_path_pattern_indices = os.path.join(output_path, sub_dir, filename_pattern_indices) + +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern_X) + +if file_list: + file_path = file_list[0] # string path to .npy file + X = np.load(file_path, mmap_mode="r") # ✅ correct way +else: + raise FileNotFoundError(f"No file matching '{filename_pattern_X}' was found in '{os.path.join(output_path, sub_dir)}'.") + + + +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern_indices) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + indices_unknown_tobe_corrected = np.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) + + +#### loading indices of unknown that can be corrected and the estimated unknowns + +indices_unknown_tobe_corrected_orig = indices_unknown_tobe_corrected.copy() + +len_6_biases = indices_unknown_tobe_corrected[-1] + 1 # number of 6-day biases + +all_column_indices = np.array( + range(len_6_biases) +) # generating all indices in case we want to esimate all 6-day biases with smoothing constraints. +indices_unknown_tobe_corrected = all_column_indices # in this case we had provided the corrections for all 6-day biases + + +# extending the indices_unknown_tobe_corrected in the original approach to have the indices of 12/18 correctable ifgs: +extended_indices = ( + indices_unknown_tobe_corrected.copy() +) # because in this case indices_unknown_tobe_corrected referes to all_column_indices +X_extended = ( + X.copy() +) # because in this case we have estimated all 6-day biases but we don't need them for the correction. we just need them to correct the existing 12/18 days based on the 6 days + +extended_indices = extended_indices.tolist() +X_extended = X_extended.tolist() +print('indices loaded') + +# Generalized loop to handle any max_con +for n in range(2, max_con + 1): # Start from 2*interval and go up to (max_con)*interval + print('n is ', n) + current_length = n * interval # Calculate the length (e.g., 12, 18, 24, ...) + + if current_length in desired_lengths: # Check if this length is required + for i in range(len(indices_unknown_tobe_corrected) - (n - 1)): + last_item = extended_indices[-1] # Get the last index + extended_indices.append(last_item + 1) # Append the new index + + # Sum up X values for the current length (n) +# X_sum = sum( +# X[i + j, :, :] for j in range(n) +# ) # Sum over n consecutive indices + + + X_sum = X[i : i + n].sum(axis=0) # more efficient + + + # Append the result to X_extended using the corresponding an_values + X_extended.append( + an_values[n - 2] # an_values is 0-based, so use n-2 + * np.angle(np.exp(complex(0 + 1j) * X_sum)) + ) + + +X = np.array(X_extended) +indices_unknown_tobe_corrected = extended_indices +del X_extended +print('X extended') + + +#### correcting the ifgs (6/12/18) ######################## +################################################################# + + +####### v3 +def wrap_to_pi_inplace(arr): + np.remainder(arr + np.pi, 2*np.pi, out=arr) + arr -= np.pi + return arr + +print(f"all_ifgs.shape: {all_ifgs.shape}") +print(f"len(indices_unknown_tobe_corrected): {len(indices_unknown_tobe_corrected)}") +print(f"len(X): {len(X)}") + +all_ifgs_cor = np.full(all_ifgs.shape, np.nan, dtype=np.float32) +if uncor_only == 0: # if we want to use the corrected ifgs (using wrapped phases) + batch_size = 1 # adjust based on memory + for i in range(0, len(indices_unknown_tobe_corrected), batch_size): + print('batch ', i) + idx = indices_unknown_tobe_corrected[i:i+batch_size] + arr = all_ifgs[idx] - X[i:i+batch_size] + wrap_to_pi_inplace(arr) + all_ifgs_cor[idx] = arr +else: + print("uncor_only = 1 → skipping correction, copying originals") + all_ifgs_cor[indices_unknown_tobe_corrected] = all_ifgs[indices_unknown_tobe_corrected] + +################## v2 +#def wrap_to_pi(x): +# """Wrap values to [-pi, pi).""" +# return (x + np.pi) % (2 * np.pi) - np.pi +# +## Preallocate directly as float array filled with NaN +#all_ifgs_cor = np.full(all_ifgs.shape, np.nan, dtype=np.float32) +# +#if uncor_only == 0: # use corrected ifgs (wrapped phases) +# print('uncor_only 0') +# all_ifgs_cor[indices_unknown_tobe_corrected] = wrap_to_pi(all_ifgs[indices_unknown_tobe_corrected] - X) +#else: # no correction, just copy +# all_ifgs_cor[indices_unknown_tobe_corrected] = all_ifgs[indices_unknown_tobe_corrected] + +############### # v1 +# this was less efficient and so I replaced it with above +#all_ifgs_cor = np.full_like(all_ifgs, None) +#all_ifgs_cor[:, :, :] = np.nan +# +#if uncor_only == 0: # if we want to use the corrected ifgs (using wrapped phases) +# all_ifgs_cor[indices_unknown_tobe_corrected, :, :] = np.angle( +# np.exp( +# complex(0 + 1j) +# * (all_ifgs[indices_unknown_tobe_corrected, :, :] - X[:, :, :]) +# ) +# ) +#else: # no need for correction +# all_ifgs_cor[indices_unknown_tobe_corrected, :, :] = all_ifgs[ +# indices_unknown_tobe_corrected, :, : +# ] +# +print ('before outputting') + +### Outputting to disk ################ +############################################################################# + + +# Create the directory (and parent directories) if they don't exist +GEOC_dir = "GEOC" +metadata_dir = "metadata" + +output_wrap_dir = os.path.join(output_path, GEOC_dir) +os.makedirs( + output_wrap_dir, exist_ok=True +) # create the GEOC directory if it doesn't exist + +template_tif_path = os.path.join(root_directory, metadata_dir, frame + ".geo.hgt.tif") +print('template_tif_path', template_tif_path) +if nlook != 1: + resampled_tif_path = os.path.join(output_wrap_dir, frame + ".geo.hgt.tif") + print('template_tif_path', template_tif_path) + print('resampled_tif_path', resampled_tif_path) + print('nlook ', nlook) + + resample_geotiff(template_tif_path, resampled_tif_path, nlook) + template_tif_path = resampled_tif_path + +print('resample complete') + +total_files = len(all_ifgs) # Total number of interferograms +processed_files = 0 # Counter for processed files + +print(f"Writing the corrected interferograms to {output_wrap_dir}") +print("Progress: [", end="", flush=True) +time.sleep(3) + + +for i in range(len(all_ifgs)): + if i in indices_unknown_tobe_corrected: + + # Writing progress bar + processed_files += 1 + percentage = (processed_files / total_files) * 100 + if processed_files % (total_files // 100) == 0: # Update progress every 1% + sys.stdout.write( + f"\rProgress: [" + + "=" * (int(percentage) // 2) + + " " * (50 - int(percentage) // 2) + + f"] {int(percentage)}%" + ) + sys.stdout.flush() + + inpha = all_ifgs_cor[i, :, :] # your phase input, float32 + incoh = all_coh[i, :, :] # your coh input. It can be either 0-255 or 0-1 + + output_phase_path = os.path.join(output_wrap_dir, all_ifgs_string[i]) + output_cc_path = os.path.join(output_wrap_dir, all_ifgs_string[i]) + + # output_phase_path = output_wrap_dir + all_ifgs_string[i] + # output_cc_path = output_wrap_dir + all_ifgs_string[i] + + os.makedirs(output_phase_path, exist_ok=True) + os.makedirs(output_cc_path, exist_ok=True) + + output_phase_file = ( + output_phase_path + "/" + all_ifgs_string[i] + ".geo.diff_pha.tif" + ) + + output_cc_file = output_cc_path + "/" + all_ifgs_string[i] + ".geo.cc.tif" + + ifg_cc_2tif(inpha, output_phase_file, template_tif_path) + ifg_cc_2tif(incoh, output_cc_file, template_tif_path) + +print("\nAll corrected interferograms have been written to disk.") diff --git a/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/README.md b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/README.md new file mode 100644 index 0000000..aafee2e --- /dev/null +++ b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/README.md @@ -0,0 +1,39 @@ +# InSAR Phase Bias Correction +This repository contains scripts for mitigating phase bias in InSAR data. It includes tools for reading data, calculating loop closures, estimating calibration parameters, performing inversion, and correcting interferograms. + +## Directory Structure +- `PhaseBias_01_Read_Data.py`: Reads input wrapped ifgs and coherence data. +- `PhaseBias_02_Loop_Closures.py`: Calculates loop closures. +- `PhaseBias_03_calibration_pars.py`: Estimates the calibration parameters $a_n$. +- `PhaseBias_04_Inversion.py`: Performs inversion for phase bias estimation. +- `PhaseBias_05_Correction.py`: Corrects interferograms using estimated biases. +- `bin/`: Contains auxiliary functions used by the scripts. +- `config.txt`: Configuration file for setting parameters. + +## Usage +1. Clone the repository: + ```bash + git clone https://github.com/YourUsername/InSAR_PhaseBias_Correction.git + cd InSAR_PhaseBias_Correction + +## Test Dataset + +The test dataset used in this repository originates from one of the islands in the Azores. The full dataset can be accessed through the COMET LiCSAR frame **082D_05125_020000**, available at the following link: + +[https://gws-access.jasmin.ac.uk/public/nceo_geohazards/LiCSAR_products/82/082D_05125_020000/](https://gws-access.jasmin.ac.uk/public/nceo_geohazards/LiCSAR_products/82/082D_05125_020000/) + +Additionally, we provide a **sample dataset**, including example interferograms, which can be downloaded from Zenodo: + +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.14525360.svg)](https://doi.org/10.5281/zenodo.14525360) + + +# Citation +If you use this repository for your research, please cite the following paper: + +Maghsoudi, Y., Hooper, A.J., Wright, T.J., Lazecky, M., & Ansari, H. (2022). Characterizing and correcting phase biases in short-term, multilooked interferograms. Remote Sensing of Environment, 275, 113022. +https://doi.org/10.1016/j.rse.2022.113022 + + +# Acknowledgment +This work was funded by the European Space Agency (ESA) as part of the Phase Bias Correction Project. We gratefully acknowledge their support. + diff --git a/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_01_Read_Data.py b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_01_Read_Data.py new file mode 100644 index 0000000..5b72365 --- /dev/null +++ b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_01_Read_Data.py @@ -0,0 +1,583 @@ +#!/usr/bin/env python3 + +import os +import re +from datetime import datetime, timedelta +import numpy as np +from osgeo import gdal +from bin.multilook_w import multilook_w +import configparser +import sys +import warnings +import pickle +import glob + +warnings.filterwarnings("ignore", category=np.VisibleDeprecationWarning) + +####################################################################### + +HELP_TEXT = """ +PhaseBias_01_Read_Data.py + +This script automates the process of downloading GeoTIFF files for wrapped interferograms and coherence images. +The files can be retrieved either from the COMET-LiCS web portal or from a root path specified in the configuration file. + +Assumptions: +- Interferograms are organized in folders named as `yyyymmdd_yyyymmdd` (e.g., `20230101_20230107`). + +Files Downloaded: +1. `yyyymmdd_yyyymmdd.geo.diff_pha.tif`: + - Contains the wrapped phase image in radians. + - Values range from -3.14 to 3.14. + +2. `yyyymmdd_yyyymmdd.geo.cc.tif`: + - Contains the coherence image of the interferometric pair. + - Values range from 0 to 255, where: + - 0 represents the lowest coherence. + - 255 represents the highest coherence. + +Outputs: +1. `All_ifgs_start_end` (Pickle file, saved as `.pkl`): + - A dictionary containing all interferograms available between the specified start and end dates. +2. `All_coh_start_end` (Pickle file, saved as `.pkl`): + - A dictionary containing all coherence data available between the specified start and end dates. + +Storage Information: +- All output files are saved in the directory `Data/` under the `output_path` defined in the configuration file. + +Additional Output: +- A summary report is generated that includes: + - The number of available interferograms for each temporal baseline. + - The number of missing interferograms. + +""" + +if "--help" in sys.argv: + print(HELP_TEXT) + sys.exit(0) + +################### + +# Config file path +config_file = "config.txt" + +# Check if config file exists +if not os.path.exists(config_file): + raise FileNotFoundError( + f"Error: The configuration file '{config_file}' was not found in the script's directory." + ) + +# Initialize the configparser +config = configparser.ConfigParser() + +# Read the config.txt file +config.read(config_file) + +# Define required parameters +required_parameters = [ + "root_path", + "frame", + "start", + "end", + "interval", + "nlook", + "LiCSAR_data", +] + +# Initialize a dictionary to store parameters +parameters = {} + +# Attempt to retrieve each parameter and catch errors +try: + parameters["root_path"] = config.get("DEFAULT", "root_path") + parameters["output_path"] = config.get("DEFAULT", "output_path") + parameters["frame"] = config.get("DEFAULT", "frame") + parameters["start"] = config.get("DEFAULT", "start") + parameters["end"] = config.get("DEFAULT", "end") + parameters["interval"] = config.getint("DEFAULT", "interval") + parameters["nlook"] = config.getint( + "DEFAULT", "nlook") # Assuming 'nlook' should be an integer + parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") + +except configparser.NoOptionError as e: + # Error message if a required parameter is missing + missing_param = str(e).split(": ")[1] + raise ValueError( + f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." + ) +except configparser.Error as e: + # General error for any configparser-related issues + raise ValueError(f"Error while reading '{config_file}': {e}") + +# Assign to individual variables for easy access +root_path = parameters["root_path"] +output_path = parameters["output_path"] +frame = parameters["frame"] +start = parameters["start"] +end = parameters["end"] +interval = parameters["interval"] +nlook = parameters["nlook"] +LiCSAR_data = parameters["LiCSAR_data"] + +# hardcoded parameters, not included in the config file +landmask = 1 +filtered_ifgs = "yes" +min_baseline = 5 +max_baseline = 366 + +# Print the values to confirm they're being read correctly +print("Root Path:", root_path) +print("output Path:", output_path) +print("Frame:", frame) +print("Start Date:", start) +print("End Date:", end) +print("Interval:", interval) +print("nlook:", nlook) +print("LiCSAR_data", LiCSAR_data) + +################ +# all_ifgs = read_ifgs(frame, start, end, min_baseline, max_baseline, landmask, nlook, interval, LiCSAR_data, filtered_ifgs) + +def read_ifgs( + start, + end, + min_baseline, + max_baseline, + landmask, + nlook, + interval, + LiCSAR_data, + filtered_ifgs, +): + + start_date = datetime.strptime(start, "%Y%m%d") + end_date = datetime.strptime(end, "%Y%m%d") + + # Check if the file ends with "geo.diff_pha.tif" + # if filename.endswith("geo.diff_unfiltered_pha.tif"): # if unfiltered data are used + if filtered_ifgs == "yes": # yes for the filtered ifgs, no for the unfiltered ifgs + req_file_name = "geo.diff_pha.tif" + else: + req_file_name = "geo.diff_unfiltered_pha.tif" + + ############ reading + + if LiCSAR_data == "yes": + track = frame[0:3] # extracting track number from frame id + if track[0] == "0": + track = track[1:3] + if track[0] == "0": + track = track[1:2] + + root_directory = f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}/interferograms/" + else: + root_directory = os.path.join( + root_path, "interferograms" + ) # Use os.path.join for consistent path construction + + ##### Count total files matching the criteria for accurate progress calculation + print(f"Reading data from the path: {root_directory}") + + total_files = sum(1 for dirpath, _, filenames in os.walk(root_directory) + for filename in filenames + if filename.endswith(req_file_name)) + processed_files = 0 + + print("Reading wrapped interferograms: [", end="", flush=True) + ##### + + # Initialize an empty dictionary to store the arrays for each category + category_arrays = {} + last_date = {} + n = 0 + + if landmask is not None: + if LiCSAR_data == "yes": + landmask_path = f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}/metadata/{frame}.geo.landmask.tif" + else: + # Use glob to find the file ending with ".geo.landmask.tif" in the metadata directory + metadata_dir = os.path.join(root_path, "metadata") + landmask_files = glob.glob( + os.path.join(metadata_dir, "*.geo.landmask.tif")) + if landmask_files: + landmask_path = landmask_files[0] # Take the first match + else: + raise FileNotFoundError( + "No file ending with '.geo.landmask.tif' found in the metadata directory." + ) + + # Load the landmask file + landmask_dataset = gdal.Open(landmask_path) + landmask_image = landmask_dataset.ReadAsArray() + array_landmask = np.array(landmask_image) + + n = 1 + # Traverse the root directory and its subdirectories + for dirpath, dirnames, filenames in os.walk(root_directory): + dirnames.sort( + ) # This ensures dirnames are processed in alphabetical order + # Iterate through the files in the current directory + for filename in filenames: + # Check if the file ends with "geo.diff_pha.tif" + # if filename.endswith("geo.diff_unfiltered_pha.tif"): # if unfiltered data are used + if (filtered_ifgs == "yes" + ): # yes for the filtered ifgs, no for the unfiltered ifgs + req_file_name = "geo.diff_pha.tif" + else: + req_file_name = "geo.diff_unfiltered_pha.tif" + + if filename.endswith(req_file_name): + file_path = os.path.join(dirpath, filename) + + #### for printing into the console + processed_files += 1 + percentage = (processed_files / total_files) * 100 + + # Clear previous percentage and print updated one + sys.stdout.write( + "\rReading wrapped phases: [{:<50}] {}%".format( + "=" * int(percentage // 2), int(percentage))) + sys.stdout.flush() + + ##### + + # Extract the dates from the file name + match = re.search(r"(\d+)_(\d+)", filename) + if match: + date1 = match.group(1) + date2 = match.group(2) + + # Convert the dates to datetime objects + date1_obj = datetime.strptime(date1, "%Y%m%d") + date2_obj = datetime.strptime(date2, "%Y%m%d") + + # Check if the dates fall within the specified range + category = (date2_obj - date1_obj).days + + if (start_date <= date1_obj <= end_date + and start_date <= date2_obj <= end_date + and category < max_baseline + and category > min_baseline): + if ( + n == 1 + ): # finding the first and last acquisiton in the time-series + first_acq = date1_obj + else: + last_acq = date2_obj + n = n + 1 + + # Calculate the difference in days between the two dates + + # Open the TIFF file + tiff_dataset = gdal.Open(file_path) + # Read the image data as a NumPy array + tiff_image = tiff_dataset.ReadAsArray() + + # Convert the image to a numpy array + array_data = np.array(tiff_image) + array_data[array_data == 0] = ( + np.nan + ) ## converting the zerso values in wrapped phases into nan. This is helpful when calculating loop closures + + if landmask != None: + array_data[array_landmask != 1] = np.nan + + if ( + nlook != None and nlook != 1 + ): # incase of unwrap data because they are already multilooked to 10 we don't multilook them here + array_data = multilook_w(array_data, nlook) + + if category not in last_date: + last_date[category] = date1_obj + # if (date1_obj - date2_obj) > interval + + # appending None to the category_arrays where there are missing ifgs in the middle of time-series + diff_ifgs = ( + date1_obj - last_date[category] + ).days # /category #difference between the first epochs of two consecutive ifg + if diff_ifgs > interval: + n_no_acq = int(diff_ifgs / interval - + 1) ## number of missing ifgs + for i in range( + n_no_acq + ): # the number of none depends on the number of missing ifgs + # Append the none to the existing array + category_arrays[category].append(None) + last_date[category] = date1_obj + + # Check if the category is already in the dictionary + if category in category_arrays: + # If the category already exists, append the array to the existing array + category_arrays[category].append(array_data) + else: + # If the category doesn't exist, create a new list with the array + category_arrays[category] = [array_data] + + # appending None to the category_arrays where there are missing ifgs in the end of time-series + for category in sorted(category_arrays): + while (last_date[category] + timedelta(category)) < last_acq: + category_arrays[category].append(None) + last_date[category] = last_date[category] + timedelta(interval) + + # appending None to the category_arrays where there are missing ifgs in the begining of time-series + # max number of expected 6-day(i.e. interval) interferograms in the full time-series + max_ifg_number = abs(int((first_acq - last_acq).days / interval)) + + # all the categories in the data e.g. 6, 12, 18 etc + cat = [] + for category in category_arrays: + cat.append(category) + + for i in range(interval, max(cat)+1, interval): + if i in cat: + while len(category_arrays[i]) < (max_ifg_number + 1 - i/interval): + category_arrays[i].insert( 0, None) + + print("\nFinished reading all interferograms.") + + return category_arrays + + +########################################################################################################################### +########################################################################################################################## + + +def read_coh(start, end, min_baseline, max_baseline, nlook, landmask, interval, + LiCSAR_data): + + start_date = datetime.strptime(start, "%Y%m%d") + end_date = datetime.strptime(end, "%Y%m%d") + + ############ reading + + if LiCSAR_data == "yes": + track = frame[0:3] + if track[0] == "0": + track = track[1:3] + if track[0] == "0": + track = track[1:2] + + root_directory = f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}/interferograms/" + else: + root_directory = os.path.join( + root_path, "interferograms" + ) # Use os.path.join for consistent path construction + + req_file_name = "geo.cc.tif" + + ##### Count total files matching the criteria for accurate progress calculation + + total_files = sum(1 for dirpath, _, filenames in os.walk(root_directory) + for filename in filenames + if filename.endswith(req_file_name)) + processed_files = 0 + + print("Reading coherence data: [", end="", flush=True) + ##### + + # Initialize an empty dictionary to store the arrays for each category + category_arrays = {} + last_date = {} + n = 0 + + if landmask is not None: + if LiCSAR_data == "yes": + landmask_path = f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}/metadata/{frame}.geo.landmask.tif" + else: + # Use glob to find the file ending with ".geo.landmask.tif" in the metadata directory + metadata_dir = os.path.join(root_path, "metadata") + landmask_files = glob.glob( + os.path.join(metadata_dir, "*.geo.landmask.tif")) + if landmask_files: + landmask_path = landmask_files[0] # Take the first match + else: + raise FileNotFoundError( + "No file ending with '.geo.landmask.tif' found in the metadata directory." + ) + + # Load the landmask file + landmask_dataset = gdal.Open(landmask_path) + landmask_image = landmask_dataset.ReadAsArray() + array_landmask = np.array(landmask_image) + + n = 1 + # Traverse the root directory and its subdirectories + for dirpath, dirnames, filenames in os.walk(root_directory): + # Iterate through the files in the current directory + for filename in filenames: + # Check if the file ends with "geo.diff_pha.tif" + + if filename.endswith(req_file_name): + file_path = os.path.join(dirpath, filename) + + #### for printing into the console + processed_files += 1 + percentage = (processed_files / total_files) * 100 + + # Clear previous percentage and print updated one + sys.stdout.write( + "\rReading coherence data: [{:<50}] {}%".format( + "=" * int(percentage // 2), int(percentage))) + sys.stdout.flush() + + ##### + + # Extract the dates from the file name + match = re.search(r"(\d+)_(\d+)", filename) + if match: + date1 = match.group(1) + date2 = match.group(2) + + # Convert the dates to datetime objects + date1_obj = datetime.strptime(date1, "%Y%m%d") + date2_obj = datetime.strptime(date2, "%Y%m%d") + + # Check if the dates fall within the specified range + category = (date2_obj - date1_obj).days + + if (start_date <= date1_obj <= end_date + and start_date <= date2_obj <= end_date + and category < max_baseline + and category > min_baseline): + if ( + n == 1 + ): # finding the first and last acquisiton in the time-series + first_acq = date1_obj + else: + last_acq = date2_obj + n = n + 1 + + # Calculate the difference in days between the two dates + + # Open the TIFF file + tiff_dataset = gdal.Open(file_path) + + # Read the image data as a NumPy array + tiff_image = tiff_dataset.ReadAsArray() + + # Convert the image to a numpy array + array_data = np.array(tiff_image) + # Convert array_data to a float type that can accommodate np.nan + array_data = array_data.astype(np.float32) + + if landmask != 0: + array_data[array_landmask != 1] = np.nan + + if ( + nlook != None and nlook != 1 + ): # incase of unwrap data because they are already multilooked to 10 we don't multilook them here + array_data = multilook(array_data, nlook) + + if category not in last_date: + last_date[category] = date1_obj + # if (date1_obj - date2_obj) > interval + + # appending None to the category_arrays where there are missing ifgs in the middle of time-series + diff_ifgs = ( + date1_obj - last_date[category] + ).days # /category #difference between the first epochs of two consecutive ifg + if diff_ifgs > interval: + n_no_acq = int(diff_ifgs / interval - + 1) ## number of missing ifgs + for i in range( + n_no_acq + ): # the number of none depends on the number of missing ifgs + # Append the none to the existing array + category_arrays[category].append(None) + last_date[category] = date1_obj + + # Check if the category is already in the dictionary + if category in category_arrays: + # If the category already exists, append the array to the existing array + category_arrays[category].append(array_data) + else: + # If the category doesn't exist, create a new list with the array + category_arrays[category] = [array_data] + + # appending None to the category_arrays where there are missing ifgs in the end of time-series + for category in sorted(category_arrays): + while (last_date[category] + timedelta(category)) < last_acq: + category_arrays[category].append(None) + last_date[category] = last_date[category] + timedelta(interval) + + # appending None to the category_arrays where there are missing ifgs in the begining of time-series + # max number of expected 6-day(i.e. interval) interferograms in the full time-series + max_ifg_number = abs(int((first_acq - last_acq).days / interval)) + + # all the categories in the data e.g. 6, 12, 18 etc + cat = [] + for category in category_arrays: + cat.append(category) + + for i in range(interval, max(cat)+1, interval): + if i in cat: + while len(category_arrays[i]) < (max_ifg_number + 1 - i/interval): + category_arrays[i].insert(0, None) + + print("\nFinished reading all coherence data.") + + return category_arrays + + +########################################################################################################################## +########################################################################################################################## + +all_ifgs = read_ifgs( + start, + end, + min_baseline, + max_baseline, + landmask, + nlook, + interval, + LiCSAR_data, + filtered_ifgs, +) +# Define the directory path +directory_path = os.path.join(output_path, "Data") + +# Create the directory if it doesn't exist +os.makedirs(directory_path, exist_ok=True) + +# Define the file path +#file_path = os.path.join(directory_path, f"_all_ifgs_{start}_{end}") + +# Remove all files inside the directory +if os.path.exists(directory_path): + files = glob.glob(os.path.join(directory_path, "*")) + for file in files: + os.remove(file) + + +# for writing the all_loops +file_path = output_path + "/Data/" + "All_ifgs_" + start + "_" + end + ".pkl" +with open(file_path, "wb") as file: # for writting a dictionary to a file + pickle.dump(all_ifgs, file) + +# Free memory before starting coherence +import gc +del all_ifgs +gc.collect() + +################################################################ + +all_coh = read_coh(start, end, min_baseline, max_baseline, landmask, nlook, + interval, LiCSAR_data) + +## for writting the all_coh +file_path = output_path + "/Data/" + "All_coh_" + start + "_" + end + ".pkl" +with open(file_path, "wb") as file: # for writting a dictionary to a file + pickle.dump(all_coh, file) + +################################################ + +# After processing all interferograms, generate the report +print("\nReport on the number of IFGs and missing IFGs per category:") +for category in sorted(all_ifgs): + num_ifgs = sum(1 for item in all_ifgs[category] + if item is not None) # Count valid IFGs + num_missing_ifgs = sum(1 for item in all_ifgs[category] + if item is None) # Count missing IFGs + print( + f"{category}-days: Number of IFGs: {num_ifgs} Number of missing IFGs: {num_missing_ifgs}" + ) diff --git a/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_02_Loop_Closures.py b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_02_Loop_Closures.py new file mode 100644 index 0000000..9c6165d --- /dev/null +++ b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_02_Loop_Closures.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 + +import numpy as np +import pickle +import glob +import configparser +import os +import sys +import warnings +warnings.filterwarnings("ignore", category=np.VisibleDeprecationWarning) + +################################################################################################ + +HELP_TEXT = """ +PhaseBias_02_Loop_Closures.py + +This script calculates loop closures (Δφ) using the interferograms imported in the first step. + +Definition of Loop Closures: +Loop closures, Δφ, are calculated for the epochs between `i` and `k`, and are defined as: + Δφ_(i,k) = |φ_(i,k) - ∑_(t=i)^k φ_(t,t+1)|_2π + +Where: +- φ_(i,j): Represents the phase difference for a pixel in the interferogram formed between epochs `i` and `j`. +- |.|_2π: Indicates that the result is wrapped modulo 2π (i.e., values range from -π to π). + +Key Information: +- Nonzero closure phase is a by-product of spatial filtering/multilooking and is primarily associated with changes in the scattering and electrical properties of the ground surface. +- The calculated loop closures are based on the minimum temporal-baseline interferograms defined in the configuration file (parameter: `interval`) and are referred to as base interferograms. These may represent: + - 12- and 6-day closures (or 24- and 12-day closures), Δφ_(i,i+2) + - 18- and 6-day closures (or 36- and 12-day closures), Δφ_(i,i+3) + The distinction depends on whether the base interferograms are 6-day or 12-day intervals. + +Outputs: +- The calculated loop closures are stored as a dictionary in a file named: + `All_loops_start_end.pkl` +- The output is saved in the directory `Data/` under the `output_path` defined in the configuration file. + +Loop Closure Report: +At the end of the script, a loop closure report is displayed. This report includes: +- The number of generated and missing loop closures for short-interval closures, such as 12-6 and 18-6. +- The number of generated and missing long-interval closures, obtained using long-term interferograms, such as 204-6 and 204-12. + +Usage: +1. Ensure the interferograms from the first script are available. +2. Run the script to generate loop closures for the specified temporal baselines. + +Example: +```bash +python PhaseBias_02_Loop_Closures.py + + +""" +if "--help" in sys.argv: + print(HELP_TEXT) + sys.exit(0) + +################################################################################################ +# Config file path +config_file = "config.txt" + +# Check if config file exists +if not os.path.exists(config_file): + raise FileNotFoundError( + f"Error: The configuration file '{config_file}' was not found in the script's directory." + ) + +# Initialize the configparser +config = configparser.ConfigParser() + +# Read the config.txt file +config.read(config_file) + +# Define required parameters +required_parameters = [ + "root_path", + "frame", + "start", + "end", + "interval", + "nlook", + "LiCSAR_data", +] + +# Initialize a dictionary to store parameters +parameters = {} + +# Attempt to retrieve each parameter and catch errors +try: + parameters["root_path"] = config.get("DEFAULT", "root_path") + parameters["output_path"] = config.get("DEFAULT", "output_path") + parameters["frame"] = config.get("DEFAULT", "frame") + parameters["start"] = config.get("DEFAULT", "start") + parameters["end"] = config.get("DEFAULT", "end") + parameters["interval"] = config.getint("DEFAULT", "interval") + parameters["nlook"] = config.getint( + "DEFAULT", "nlook") # Assuming 'nlook' should be an integer + parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") + +except configparser.NoOptionError as e: + # Error message if a required parameter is missing + missing_param = str(e).split(": ")[1] + raise ValueError( + f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." + ) +except configparser.Error as e: + # General error for any configparser-related issues + raise ValueError(f"Error while reading '{config_file}': {e}") + +# Assign to individual variables for easy access +root_path = parameters["root_path"] +output_path = parameters["output_path"] +frame = parameters["frame"] +start = parameters["start"] +end = parameters["end"] +interval = parameters["interval"] +nlook = parameters["nlook"] +LiCSAR_data = parameters["LiCSAR_data"] + +# hardcoded parameters, not included in the config file +landmask = 1 +filtered_ifgs = "yes" +max_loop = 5 # in case of 3 all loops with 6,12,18-day will be calculated (e.g. (60,6), (60,12) and (60,18)) + + +############################################################################################# +################################################################################################# +def loop_calc(max_loop): + + ## Read ifgs: + + # Define the path components + sub_dir = "Data" + filename_pattern = "All_ifgs_*" # Pattern to match files starting with "All_ifgs" + + # Construct the full file path with the refined pattern + file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) + # Use glob to find the file and ensure it exists + file_list = glob.glob(file_path_pattern) + if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + all_ifgs = pickle.load(file) + else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) + + loop = {} + missing = {} + + cat = sorted(all_ifgs.keys()) # Get sorted categories + # print('cat = ', cat) + + # Initialize counters for reporting + loop_counts = { + category: { + cat[l]: 0 + for l in range(max_loop) + } + for category in sorted(all_ifgs.keys()) + } + missing_counts = { + category: { + cat[l]: 0 + for l in range(max_loop) + } + for category in sorted(all_ifgs.keys()) + } + + for category in cat: + loop[category] = { + } # Initialize the key in the loop dictionary with an inner dictionary + missing[category] = [] # second ver + for l in range( + max_loop + ): # it was up to 3 in the first version. but this allows to go for longer loops e.g. 288-72 (cat[11] is 72) + loop[category][cat[l]] = [ + ] # Initialize the key in the inner dictionary + # missing[cat[l]] = [] # first ver + # Calculate total iterations for progress percentage + total_iterations = sum( + len(all_ifgs[cat[i]]) for l in range(max_loop) + for i in range(1, len(cat)) + if cat[i] % cat[l] == 0 and cat[i] != cat[l]) + completed_iterations = 0 + + # Display the message before starting the progress bar + print( + f"Calculating all {interval}, {2 * interval}, and {3 * interval} loop closures." + ) + + for l in range( + 0, max_loop + ): # l could be 6, 12 and 18 to calculate e.g. 36-6 or 36-12 or 36-18 + for i in range(1, len(cat)): # index for the category e.g. 6, 12, 18 + if cat[i] % cat[l] == 0 and cat[i] != cat[ + l]: # such as 12/6 or 24/12 + for t in range(len(all_ifgs[ + cat[i]])): # index for the epochs in each category + + ####################progress bar + completed_iterations += 1 + percentage = (completed_iterations / + total_iterations) * 100 + + # Display progress percentage + sys.stdout.write("\rProgress: [{:<50}] {}%".format( + "=" * int(percentage // 2), int(percentage))) + sys.stdout.flush() + ################################## + + if cat[l] == interval: + end_index = t + int(cat[i] / cat[l]) + # print('end_index = ', end_index) + else: + end_index = t + int(cat[i] / cat[l]) * int( + cat[l] / interval) + + # recording the missing ifgs in each cat e.g. 6,12,18 + #if np.any(np.array(all_ifgs[cat[i]])[t] == None): + if np.any(np.array(all_ifgs[cat[i]], dtype=object)[t] == None): + missing[cat[i]].append( + t + ) # to record the missing ifgs index for each cateogory e.g. 18 + + + for e in range(t, end_index, int(cat[l]/interval)): # to record the missing ifgs index for the period of each loop + elem = all_ifgs[cat[l]][e] + + if elem is None: + missing[cat[l]].append(e) + + + + if all_ifgs[cat[i]][t] is None or any(all_ifgs[cat[l]][elem] is None for elem in range(t, end_index, int(cat[l] / interval))): + + loop[cat[i]][cat[l]].append(None) + missing_counts[cat[i]][cat[l]] += 1 # Increment missing loop counter + + else: + closure = np.angle( np.exp( 1j * ( np.array(all_ifgs[cat[i]][t], dtype=np.float32)- np.sum( np.array(all_ifgs[cat[l]][t:end_index:int(cat[l] / interval)], dtype=np.float32), axis=0)))) + + + if cat[i] in loop: + loop[cat[i]][cat[l]].append(closure) + loop_counts[cat[i]][cat[ + l]] += 1 # Increment successful loop counter + + else: + loop[cat[i]][cat[l]] = [closure] + print("\n Finished calculating the loop closures.") + + # Generate report + print("\n=== Loop Closure Report ===") + for category in [ + 2 * interval, + 3 * interval, + ]: # Restrict to relevant categories e.g. 18-6 and 12-6 as the main observations + for l in [interval]: # Restrict to the loop levels of interest + if category in loop_counts and l in loop_counts[category]: + generated = loop_counts[category].get(l, 0) + missing = missing_counts[category].get(l, 0) + print( + f"Loop Closure {category} - {l}: Generated = {generated}, Missing = {missing}" + ) + + print("\n=== Loop Closure Report for Long-interval Loop Closures ===") + + # Iterate over categories greater than 200 + for category in cat: + if category > 200: # Check for long categories + for l in range( + 1, + 5 * interval): # Check loop levels less than 5 * interval + if category in loop_counts and l in loop_counts[category]: + generated = loop_counts[category].get(l, 0) + if generated >= 1: # Only report if at least one loop is generated + print( + f"Loop Closure {category} - {l}: Generated = {generated}" + ) + + return loop, missing + + +################################################################################################ + +all_loops, missing = loop_calc(max_loop) + +# Define the directory path +directory_path = os.path.join(output_path, "Data") + +# Create the directory if it doesn't exist +os.makedirs(directory_path, exist_ok=True) + +# Define the file path +file_path = os.path.join(directory_path, f"All_loops_{start}_{end}.pkl") + +with open(file_path, "wb") as file: # for writting a dictionary to a file + pickle.dump(all_loops, file) + +### + +# Define the file path +file_path = os.path.join(directory_path, f"missing_loops") + +with open(file_path, "wb") as file: # for writting a dictionary to a file + pickle.dump(missing, file) diff --git a/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_03_calibration_pars.py b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_03_calibration_pars.py new file mode 100644 index 0000000..ce3aba9 --- /dev/null +++ b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_03_calibration_pars.py @@ -0,0 +1,424 @@ +#!/usr/bin/env python3 + +import numpy as np +import matplotlib.pyplot as plt +from matplotlib.gridspec import GridSpec +import pickle +import os +import sys +import glob +import configparser +import warnings + +with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=np.VisibleDeprecationWarning) + +################################################################################ + +HELP_TEXT = """ +PhaseBias_03_calibration_pars.py + +This script estimates the calibration parameters a_n using the loop closures calculated in the previous step. + +The calibration parameters a_n are estimated by calculating the ratio of the long-term loop closure for the interferogram of interest to the long-term loop closure for the base interferogram. Assuming negligible bias in long-term interferograms, a_n is calculated as follows: + + a_n = |φ(i,i+l) - Σ(t=i to i+l)φ(t,t+1+n)|_2π / |φ(i,i+l) - Σ(t=i to i+l)φ(t,t+1)|_2π + +where: +- φ(i,i+l) is a long-term interferogram connecting epoch i to i+l. +- φ(t,t+1) represents the base interferogram. +- φ(t,t+1+n) is the short interferogram to be corrected using a_n. + +### Input: +1. Loop closures from the previous step (e.g., 216-6, 216-12, and 216-18 for 6-day acquisition intervals). +2. The long-term interferogram length (e.g., 216 days) must be specified at the beginning. +3. The number of a_n parameters to estimate, specified using `num_a`. For example: + - Setting `num_a=2` indicates that a1 and a2 will be estimated (used to correct 12- and 18-day interferograms). + - For correcting longer interferograms, `num_a` should be set accordingly. + +### Output: +1. All possible a_n values are calculated. +2. Visualizations: + - Example plots of a_n arrays and their corresponding histograms, illustrating spatial variations. + - Time-series plots showing how a_n values evolve over time. +3. Mean a_n values over time are output for further use. + +### File Outputs: +- Results are stored in the output directory defined in the `config.txt` file under `/Data/` as `Mean_an_values.pkl`. + +### Additional Notes: +- Users should ensure that `num_a` aligns with the desired temporal baselines for correction. + +""" +if "--help" in sys.argv: + print(HELP_TEXT) + sys.exit(0) + + +############################################################################### +# Config file path +config_file = "config.txt" + +# Check if config file exists +if not os.path.exists(config_file): + raise FileNotFoundError( + f"Error: The configuration file '{config_file}' was not found in the script's directory." + ) + +# Initialize the configparser +config = configparser.ConfigParser() + + +# Read the config.txt file +config.read(config_file) + +# Define required parameters +required_parameters = [ + "root_path", + "frame", + "start", + "end", + "interval", + "nlook", + "LiCSAR_data", +] + +# Initialize a dictionary to store parameters +parameters = {} + +# Attempt to retrieve each parameter and catch errors +try: + parameters["root_path"] = config.get("DEFAULT", "root_path") + parameters["output_path"] = config.get("DEFAULT", "output_path") + parameters["frame"] = config.get("DEFAULT", "frame") + parameters["start"] = config.get("DEFAULT", "start") + parameters["end"] = config.get("DEFAULT", "end") + parameters["interval"] = config.getint("DEFAULT", "interval") + parameters["nlook"] = config.getint( + "DEFAULT", "nlook" + ) # Assuming 'nlook' should be an integer + parameters["num_a"] = config.getint("DEFAULT", "num_a") + parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") + +except configparser.NoOptionError as e: + # Error message if a required parameter is missing + missing_param = str(e).split(": ")[1] + raise ValueError( + f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." + ) +except configparser.Error as e: + # General error for any configparser-related issues + raise ValueError(f"Error while reading '{config_file}': {e}") + +# Assign to individual variables for easy access +root_path = parameters["root_path"] +output_path = parameters["output_path"] +frame = parameters["frame"] +start = parameters["start"] +end = parameters["end"] +interval = parameters["interval"] +nlook = parameters["nlook"] +num_a = parameters["num_a"] +LiCSAR_data = parameters["LiCSAR_data"] + +# hardcoded parameters, not included in the config file +landmask = 1 +filtered_ifgs = "yes" +max_loop = 5 # in case of 3 all loops with 6,12,18-day will be calculated (e.g. (60,6), (60,12) and (60,18)) + + +##################################################### Loop Closure calculation ##################################################### +#################################################################################################################################### +track = frame[0:3] +if track[0] == "0": + track = track[1:3] +if track[0] == "0": + track = track[1:2] + + +###################### Read coh: +print("\nReading all coherence data...") +# Define the path components +sub_dir = "Data" +filename_pattern = "All_coh_*" # Pattern to match files starting with "All_ifgs" + +# Construct the full file path with the refined pattern +file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + coh = pickle.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) +print("Reading coherence data completed.") + + +##################### Read Loops + +print("\nReading all loop closures...") +filename_pattern = "All_loops_*" # Pattern to match files starting with "All_ifgs" + +# Construct the full file path with the refined pattern +file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + all_loops = pickle.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) + +print("Reading loop closures completed.") + + +##################################################### Loop Closure calculation ##################################################### +#################################################################################################################################### + + +long_baseline = 216 # 36,72,108,144,180,216,252,288,324 +coh_thresh = 25 + + +def calc_an(loop_360_n, loop_360_6): + """ + Calculates the a_n values from loop closures. + + Parameters: + loop_360_n (ndarray): Array of loop closures for n-day intervals. + loop_360_6 (ndarray): Array of loop closures for 6-day intervals. + + Returns: + ndarray: Mean a_n values for all loop closures. + list: List of all a_n arrays for further processing or plotting. + """ + ratios = [] + + # Loop through indices + for i in range(len(loop_360_6)): + if loop_360_6[i] is not None and loop_360_n[i] is not None: + # Calculate the ratio for non-NaN elements + ratio_i = np.divide( + loop_360_n[i], + loop_360_6[i], + out=np.full_like(loop_360_n[i], np.nan), + where=~np.isnan(loop_360_6[i]), + ) + ratios.append(ratio_i) + +# Check if `ratios` is empty before proceeding + if not ratios: + print("Warning: No valid a_n values could be calculated. Returning NaN values.") + return np.array([np.nan]), [] + + + # Convert the list of ratios to a numpy array + an = np.array(ratios, dtype=object) + mean_an_long_baseline = [] + + # Calculate mean values of each a_n + for arr in an: + mask = (arr > 0) & (arr < 1) # Apply mask to filter invalid values + arr[~mask] = np.nan + if not np.isnan(arr).all(): # Check if all elements are NaN + mean_an = np.nanmean(arr) + else: + mean_an = np.nan # Assign NaN for completely empty slices + + mean_an_long_baseline.append(mean_an) + + return np.array(mean_an_long_baseline), an + + +################################################################################## +############################ +def plot_an(an_arrays, mean_a_long_baseline, k): + """ + Plots the a_n arrays with the largest coverage and their histograms in a two-row layout, + as well as the mean values of calibration parameters a_n over time. + + Parameters: + an_arrays (list): List of a_n arrays. + mean_a_long_baseline (ndarray): Array of mean calibration parameter values for a_1, a_2, ..., a_n. + k (int): The current value of k (e.g., 2 for a1, 3 for a2). + """ + + # Calculate the coverage (number of non-NaN values) for each array + coverage = [] + valid_an_arrays = [] # Store only valid arrays for plotting + + for arr in an_arrays: + if ( + arr is not None and arr.ndim == 2 and not np.all(np.isnan(arr)) + ): # Check for valid 2D arrays + coverage.append(np.sum(~np.isnan(arr))) # Count non-NaN values + valid_an_arrays.append(arr) + + # Sort arrays by coverage in descending order + sorted_indices = np.argsort(coverage)[::-1] + selected_arrays = [ + valid_an_arrays[idx] for idx in sorted_indices[:6] + ] # Select top 6 arrays + + # Plot the a_n arrays and their histograms in a two-row layout + if selected_arrays: + num_arrays = len(selected_arrays) + fig = plt.figure(figsize=(4 * num_arrays, 8)) + gs = GridSpec( + 2, num_arrays + 1, width_ratios=[1] * num_arrays + [0.1] + ) # Add space for colorbar + + # Plot selected arrays + axes_img = [] + for i, arr in enumerate(selected_arrays): + ax_img = fig.add_subplot(gs[0, i]) + im = ax_img.imshow(arr, cmap="RdYlBu", vmin=-0.8, vmax=0.8) + ax_img.set_title(f"{i+1}th $a_{{{k-1}}}$ Array") # Dynamically set title + ax_img.axis("off") + axes_img.append(ax_img) + + # Add a single colorbar for all array plots + cbar_ax = fig.add_subplot(gs[0, -1]) + fig.colorbar(im, cax=cbar_ax, orientation="vertical", label="Color Scale") + + # Plot histograms for the selected arrays + for i, arr in enumerate(selected_arrays): + ax_hist = fig.add_subplot(gs[1, i]) + ax_hist.hist(arr.flatten(), bins=100, color="blue", alpha=0.7, density=True) + ax_hist.set_title( + f"{i+1}th $a_{{{k-1}}}$ Histogram" + ) # Dynamically set title + ax_hist.set_xlim([-10, 10]) # Adjust histogram range as needed + + plt.tight_layout() + plt.show() + else: + print("No valid a_n arrays available for plotting.") + + # Plot the mean values of a_n over time + x_values = np.arange(len(mean_a_long_baseline)) # Sequential indices for x-axis + fig, ax = plt.subplots(figsize=(6, 4)) + + + # Filter out NaN values to avoid gaps in the plot + valid_indices = ~np.isnan(mean_a_long_baseline) # Mask for valid (non-NaN) entries + x_valid = x_values[valid_indices] # Only use valid time steps + y_valid = np.array(mean_a_long_baseline)[valid_indices] # Only use valid a_n values + + + # Remove NaN values for trendline fitting + valid_indices = ~np.isnan(mean_a_long_baseline) # Mask for valid (non-NaN) entries + if ( + valid_indices.sum() > 1 + ): # Ensure there are at least two valid points for fitting +# x_valid = x_values[valid_indices] + y_valid = np.array(mean_a_long_baseline)[valid_indices] + x_valid = np.arange(len(y_valid)) + + # Fit and plot a trendline + coefficients = np.polyfit(x_valid, y_valid, 1) + trendline = np.polyval(coefficients, x_valid) + ax.plot(x_valid, trendline, color="red", label="Trendline") + + # Scatter plot for all points (including NaN if any) + ax.scatter( + #x_values, + #mean_a_long_baseline, + x_valid, # Only use valid time steps + y_valid, # Only use valid a_n values + + color="blue", + marker="o", + label="Mean Values", + s=9, + ) + + # Set labels and limits + ax.set_ylim(-3, 3) # Adjust limits as needed + ax.set_xlabel("Time Step") # Sequential index as x-axis + ax.set_ylabel("Mean Values", fontsize=12) + ax.legend() + + # Adjust layout and show the plot + plt.tight_layout() + plt.show() + + +## +######### forming 360-6 + +loop_360_6_all = np.array(all_loops[long_baseline][interval], dtype=object) +coh_360 = np.array(coh[long_baseline][:], dtype=object) + +loop_360_6 = np.full_like( + loop_360_6_all, np.nan +) # create a new array as the same size of loop_360_6_all with nan values + + +for i, arr in enumerate(loop_360_6_all): + if arr is not None: + (frame_row, frame_col) = np.shape(arr) + loop_360_6[i] = np.where((coh_360[i] > coh_thresh), arr, np.nan) + +###### forming 360-n (n=12,18,24 etc) +# mean_a_long_baseline = [] +final_mean_values = [] +an_labels = [] + +# Loop through the values of k corresponding to the number of calibration parameters +for k in range(2, 2 + num_a): # k=2 for a1, k=3 for a2, etc. + mean_an_long_baseline = [] + + # Forming 360-n (n=12, 18, 24, etc.) + loop_360_n_all = np.array(all_loops[long_baseline][k * interval], dtype=object) + loop_360_n = np.full_like(loop_360_n_all, np.nan) + + for i, arr in enumerate(loop_360_n_all): + if arr is not None: + # Apply coherence threshold filtering + loop_360_n[i] = np.where((coh_360[i] > coh_thresh), arr, np.nan) + + # Calculate a_n values + mean_an_long_baseline, an_arrays = calc_an(loop_360_n, loop_360_6) + + # If all values in mean_an_long_baseline are nan (empty ratios), stop execution + if np.isnan(mean_an_long_baseline).all(): + print("No valid a_n values could be calculated. Stopping execution.") + sys.exit(1) # Exit the script with an error code (1 means failure) + + # Calculate the mean of all mean values for this k and store it + overall_mean = np.nanmean(mean_an_long_baseline) + final_mean_values.append(overall_mean) + an_labels.append(f"a{k-1}") + + for idx, mean_a in enumerate(mean_an_long_baseline, start=1): + if not np.isnan(mean_a): # Check if the value is not NaN + print(f"Mean of the {idx}th a({k-1}) is {mean_a:.6f}") + + plot_an(an_arrays, mean_an_long_baseline, k) + + +# Print the final mean values across all k +for k_idx, final_mean in enumerate(final_mean_values, start=2): + print(f"Final mean value for a({k_idx-1}) is {final_mean:.6f}") + +# Write final mean values to a text file +output_path = os.path.join(parameters["output_path"], "Data") +os.makedirs(output_path, exist_ok=True) # Create directory if it doesn't exist +output_file = os.path.join(output_path, "an.txt") + +with open(output_file, "w") as file: + for label, mean_value in zip(an_labels, final_mean_values): + file.write(f"{label}={mean_value:.6f}\n") + +print(f"Final mean values written to {output_file}.") diff --git a/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_04_Inversion.py b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_04_Inversion.py new file mode 100644 index 0000000..822a56e --- /dev/null +++ b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_04_Inversion.py @@ -0,0 +1,673 @@ +#!/usr/bin/env python3 + +import numpy as np +import pickle +import os +from scipy.sparse import csc_array +from scipy.sparse.linalg import lsqr +import sys +import math +from bin.read_orig_ifgs_coh import read_orig_ifgs_coh +from bin.circular_mean_var import circular_mean_and_variance_over_epochs +import glob +import configparser +import warnings + +warnings.filterwarnings("ignore", category=np.VisibleDeprecationWarning) + +################################################################################ + +HELP_TEXT = """ +PhaseBias_04_Inversion.py + +This script estimates the phase bias terms for the base interferograms using short-term closure phases as input. +The bias terms derived in this step serve as the foundation for correcting other short-term interferograms. + +### Workflow: +1. **Masking Noisy Loop Closures:** + - A moving average is applied to the time-series of loop closures to account for seasonal fluctuations. + - A circular moving average (mean of complex values over time) is computed for each pixel. + - The distance of each loop closure from the moving average is calculated. If the distance exceeds a threshold (2σ), the point is masked as noisy. + - The circular standard deviation (σ) is computed using the von Mises distribution (circular normal distribution). + +2. **Refined Loop Closures:** + - Masked loop closures, denoted as 〖Δφ〗_(i,i+2)^r and 〖Δφ〗_(i,i+3)^r, are used as primary observations for phase bias estimation. + +3. **First Inversion (Without Temporal Smoothing Constraints):** + - Estimates the bias terms that can be corrected directly from the observed loop closures. + +4. **Second Inversion (With Temporal Smoothing Constraints):** + - Incorporates temporal smoothing constraints to estimate additional bias terms that cannot be directly derived from loop closures. + - The temporal smoothing minimizes differences between bias terms over time, ensuring consistency. + +5. **Combining Results:** + - Bias terms from the first inversion are combined with the results of the second inversion to include terms not estimated in the first round. + +### Outputs: +- The phase bias terms for the base interferograms are stored as a NumPy array file: + - `X_base_ifgs_biases.npy`, saved in the `Data` directory under the `output_path` defined in `config.txt`. + + +### Input Requirements: +- Short-term closure phases (from `PhaseBias_02_Loop_Closures.py`). +- Configuration parameters, including thresholds and temporal settings, defined in `config.txt`. + +### Output Files: +- `X_base_ifgs_biases.npy`: Stores the estimated bias terms for the base interferograms. +""" + +if "--help" in sys.argv: + print(HELP_TEXT) + sys.exit(0) + +############################################################################### +# Config file path +config_file = "config.txt" + +# Check if config file exists +if not os.path.exists(config_file): + raise FileNotFoundError( + f"Error: The configuration file '{config_file}' was not found in the script's directory." + ) + +# Initialize the configparser +config = configparser.ConfigParser() + +# Read the config.txt file +config.read(config_file) + +# Define required parameters +required_parameters = [ + "root_path", + "frame", + "start", + "end", + "interval", + "nlook", + "LiCSAR_data", + "a1_6_day", + "a2_6_day", + "a1_12_day", + "a2_12_day", + "estimate_an_values", +] + +# Initialize a dictionary to store parameters +parameters = {} + +# Attempt to retrieve each parameter and catch errors +try: + parameters["root_path"] = config.get("DEFAULT", "root_path") + parameters["output_path"] = config.get("DEFAULT", "output_path") + parameters["frame"] = config.get("DEFAULT", "frame") + parameters["start"] = config.get("DEFAULT", "start") + parameters["end"] = config.get("DEFAULT", "end") + parameters["interval"] = config.getint("DEFAULT", "interval") + parameters["nlook"] = config.getint( + "DEFAULT", "nlook" + ) # Assuming 'nlook' should be an integer + parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") + parameters["a1_6_day"] = config.getfloat("DEFAULT", "a1_6_day") + parameters["a2_6_day"] = config.getfloat("DEFAULT", "a2_6_day") + parameters["a1_12_day"] = config.getfloat("DEFAULT", "a1_12_day") + parameters["a2_12_day"] = config.getfloat("DEFAULT", "a2_12_day") + parameters["estimate_an_values"] = config.get("DEFAULT", "estimate_an_values") + +except configparser.NoOptionError as e: + # Error message if a required parameter is missing + missing_param = str(e).split(": ")[1] + raise ValueError( + f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." + ) +except configparser.Error as e: + # General error for any configparser-related issues + raise ValueError(f"Error while reading '{config_file}': {e}") + +# Assign to individual variables for easy access +root_path = parameters["root_path"] +output_path = parameters["output_path"] +frame = parameters["frame"] +start = parameters["start"] +end = parameters["end"] +interval = parameters["interval"] +nlook = parameters["nlook"] +LiCSAR_data = parameters["LiCSAR_data"] +estimate_an_values = parameters["estimate_an_values"] + + +# Initialize a1 and a2 with None for safety +a1 = None +a2 = None + + +print("estimate_an_values is ", estimate_an_values) +# Check if estimate_an_values is 'yes' +if estimate_an_values == "yes": + an_file_path = os.path.join(output_path, "Data", "an.txt") + + # Try reading the file + try: + if os.path.exists(an_file_path): + print(f"Reading 'a' values from {an_file_path}...") + + # Read the file and load the values + with open(an_file_path, "r") as file: + for line in file: + line = line.strip() + if line.startswith("a1="): + a1 = float(line.split("=")[1]) + elif line.startswith("a2="): + a2 = float(line.split("=")[1]) + + # Check if both a1 and a2 were found + if a1 is None or a2 is None: + print( + "Warning: Missing a1 or a2 in an.txt. Switching to default values from config file." + ) + raise ValueError("Incomplete a1 or a2 values.") + else: + print( + f"Warning: {an_file_path} does not exist. Switching to default values from config file." + ) + raise FileNotFoundError + + except (FileNotFoundError, ValueError): + # Fallback to default values from config file + if interval == 6: + a1 = parameters["a1_6_day"] + a2 = parameters["a2_6_day"] + else: + a1 = parameters["a1_12_day"] + a2 = parameters["a2_12_day"] +else: + # Use default values from the config file + print("Using default 'an' values from the config file.") + if interval == 6: + a1 = parameters["a1_6_day"] + a2 = parameters["a2_6_day"] + else: + a1 = parameters["a1_12_day"] + a2 = parameters["a2_12_day"] + +# Print the chosen values +print(f"Final values - a1: {a1}, a2: {a2}") + + +##################################################### Loop Closure calculation ##################################################### +#################################################################################################################################### +#################################################################################################################################### + +track = frame[0:3] +if track[0] == "0": + track = track[1:3] +if track[0] == "0": + track = track[1:2] + +with_temporals = "yes" +apply_to_all = "no" # in case of with_temporal='yes', we can decide if you want to use the temporal to all ifgs (i.e. apply_to_all='yes'), or just to those that can not be corrected (i.e. apply_to_all='no') +w = 0.1 # the weigth of the temporal smoothing constraints +####### +coh_thresh = 11 # threhsold on the coherence values. In this version it is off +min_num_eq = 10 # I used 10 in all my experiments. min number of equations in the least square inversion. + +############################### Reading input data ############################################# + +##################### Read ifgs: +#print("Reading all ifgs...") +## Define the path components +#sub_dir = "Data" +#filename_pattern = "All_ifgs_*" # Pattern to match files starting with "All_ifgs" +# +## Construct the full file path with the refined pattern +#file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +## Use glob to find the file and ensure it exists +#file_list = glob.glob(file_path_pattern) +#if file_list: +# file_path = file_list[0] # Get the single file path directly +# +# # Load the data from the file +# with open(file_path, "rb") as file: +# all_ifgs = pickle.load(file) +#else: +# raise FileNotFoundError( +# f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." +# ) +#print("Reading ifgs completed.") + + +####################### Read coh: +#print("\nReading all coherence data...") +## Define the path components +#sub_dir = "Data" +#filename_pattern = "All_coh_*" # Pattern to match files starting with "All_ifgs" +# +## Construct the full file path with the refined pattern +#file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +## Use glob to find the file and ensure it exists +#file_list = glob.glob(file_path_pattern) +#if file_list: +# file_path = file_list[0] # Get the single file path directly +# +# # Load the data from the file +# with open(file_path, "rb") as file: +# all_coh = pickle.load(file) +#else: +# raise FileNotFoundError( +# f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." +# ) +#print("Reading coherence data completed.") + +##################### Read Loops + +print("\nReading all loop closures...") +sub_dir = "Data" +filename_pattern = "All_loops_*" # Pattern to match files starting with "All_ifgs" + +# Construct the full file path with the refined pattern +file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + all_loops = pickle.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) + +print("Reading loop closures completed.") + +all_cat = [] +for cat in all_loops: + all_cat.append(cat) + +### stacking all the ifgs 6/12/18 in a numpy var all_ifgs +print( + "Extracting the required interferograms(up to 3 epochs), and the corresponding loop closures for inversion..." +) +#desired_categories = [interval, 2 * interval, 3 * interval] +#all_ifgs, _, existing_ifgs_index = read_orig_ifgs_coh( +# all_ifgs, all_coh, desired_categories +#) + +######### forming 12-6 and 18-6 loop closures from the imported data +################################################################# +loop_12 = np.array(all_loops[2 * interval][interval], dtype=object) +#coh_12 = np.array(all_coh[2 * interval][:], dtype=object) + +loop_18 = np.array(all_loops[3 * interval][interval], dtype=object) +#coh_18 = np.array(all_coh[3 * interval][:], dtype=object) + +############## finding dynamic pixel-based thresholds to remove the noisy loops +loop_12_orig = [] +loop_18_orig = [] +none_indices12 = [] +none_indices18 = [] + +# refine loop_12 to exclude the none indices and create loop_12_orig +for i, arr in enumerate(loop_12): + if arr is not None: + (frame_row, frame_col) = np.shape(arr) + loop_12_orig.append(arr) + else: + none_indices12.append(i) + +# refine loop_18 to exclude the none indices and create loop_18_orig +for i, arr in enumerate(loop_18): + if arr is not None: + loop_18_orig.append(arr) + else: + none_indices18.append(i) + +loop_12_orig = np.array(loop_12_orig) +loop_18_orig = np.array(loop_18_orig) + + +### removing noisy loops +print( + "Masking noisy loop closures using circular moving average and standard deviaiton of the loop closures in time:" +) + +# using circular_mean to calculate the mean phase value using complex numbers, and von_mises_variance to calculate the second moment of the Von Mises distribution +_, thresh_loop12 = circular_mean_and_variance_over_epochs(loop_12_orig, axis=0) +_, thresh_loop18 = circular_mean_and_variance_over_epochs(loop_18_orig, axis=0) + + +# Add a new axis at the beginning (axis=0) of these arrays to have shape (1, ...) +# These are the threshold vlaues obtained for each pixel that will be used for masking later +thresh_loop12 = np.expand_dims(thresh_loop12, axis=0) +thresh_loop18 = np.expand_dims(thresh_loop18, axis=0) + + +# calculate a temporal moving average for wrapped data +def moving_average(arr, window_size): + # Initialize an empty array to store the temporal averages + moving_averages = np.zeros_like(arr) + half_window = window_size // 2 + ### miroring data at both ends for window_size/2 at each end + mirrored_shape = (arr.shape[0] + window_size, arr.shape[1], arr.shape[2]) + + # Create an empty array to hold the mirrored data + mirrored_arr = np.empty(mirrored_shape, dtype=arr.dtype) + + # Fill the mirrored array with the original data + mirrored_arr[window_size // 2 : -window_size // 2] = arr + + # Mirror the data at the beginning + for i in range(window_size // 2): +# mirrored_arr[i] = arr[window_size // 2 - i] + mirrored_arr[i] = arr[min(half_window - i, arr.shape[0] - 1)] + + # Mirror the data at the end + for i in range(window_size // 2): +# mirrored_arr[-(i + 1)] = arr[-(i + 1) - window_size // 2] + mirrored_arr[-(i + 1)] = arr[max(-1 - (half_window - i), 0)] + #### + + # Calculate temporal averages within each window + for t in range(half_window, mirrored_arr.shape[0] - half_window): + moving_averages[t - half_window, :, :] = np.angle( + np.nanmean( + np.exp(1j * (mirrored_arr[t - half_window : t + half_window, :, :])), + axis=0, + ) + ) + return moving_averages + + +# for loop 12 +# Calculate distance between each epoch and the moving average for each pixel +print( + f"Calculating the moving average for {2 * interval} loop closures, followed by masking noisy loop closures..." +) +print( + "This may take a while, as it depends on the number of loops and the size of your dataset.\n" +) + +window_size = 30 + +moving_avg = moving_average(loop_12_orig, window_size) +distances = np.abs(np.angle(np.exp(1j * (loop_12_orig - moving_avg)))) +mask = distances > 4 * thresh_loop12 # it is 4*sigma + +# Replace values with NaN where mask is True +loop_12_bk = loop_12_orig +loop_12_orig = np.where(mask, np.nan, loop_12_orig) +loop_12 = loop_12_orig.tolist() + +for idx in none_indices12: + loop_12.insert(idx, None) +loop_12 = np.array(loop_12, dtype=object) + +# for loop 18 +# Calculate distance between each epoch and the moving average for each pixel +print( + f"Calculating the moving average for {3 * interval} loop closures, followed by masking noisy loop closures..." +) +print( + "This may take a while, as it depends on the number of loops and the size of your dataset.\n" +) + +moving_avg = moving_average(loop_18_orig, window_size) +distances = np.abs(np.angle(np.exp(1j * (loop_18_orig - moving_avg)))) +mask = distances > 4 * thresh_loop18 # it is 4*sigma + +# Replace values with NaN where mask is True +loop_18_bk = loop_18_orig +loop_18_orig = np.where(mask, np.nan, loop_18_orig) +loop_18 = loop_18_orig.tolist() + +for idx in none_indices18: + loop_18.insert(idx, None) +loop_18 = np.array(loop_18, dtype=object) + +print("Masking of noisy loop closures is completed.") + +# ################ Apply coherece thresholding +# print('shape loop_12 = ', np.shape(loop_12)) +# print('shape loop_12_orig = ', np.shape(loop_12_orig)) +# print('shape coh_12 = ', np.shape(coh_12)) +# for i, arr in enumerate(loop_12): +# if arr is not None: +# (frame_row, frame_col) = np.shape(arr) +# loop_12[i] = np.where((coh_12[i] > coh_thresh), arr, np.nan) +# +# +# for i, arr in enumerate(loop_18): +# if arr is not None: +# loop_18[i] = np.where((coh_18[i] > coh_thresh), arr, np.nan) +# +# +# + +len12 = len(loop_12) # number of 12 day loops including None +len18 = len(loop_18) # number of 18 day loops including None + +######################## Forming the design matrix A ############################## +#################################################################################### + +print("Preparing the design matrix and observation vector for the inversion step...") + +b = [] +A = [] +n_unk = ( + len12 + 1 +) # this in an initial value(i.e. the num of 6 biases). will be changed by the actual number of unk after considering the None values/missing +# print('n_unk = ', n_unk) + +row = np.zeros( + n_unk, dtype=np.float32 +) # each row of the design matrix A for the 12-day +for i in range(len12): + b.append(loop_12[i]) + # print('np.shape(loop_12[i] = ', np.shape(loop_12[i])) + row[i : i + 2] = a1 - 1 + A.append(row) + row = np.zeros(n_unk, dtype=np.float32) + +row = np.zeros( + n_unk, dtype=np.float32 +) # each row of the design matrix A for the 18-day +for i in range(len18): + b.append(loop_18[i]) + row[i : i + 3] = a2 - 1 + A.append(row) + row = np.zeros(n_unk, dtype=np.float32) + +A = np.array(A) +b = np.array(b, dtype=object) + + +####################### removing the rows from b and A where b is None +mask = [] +mask = np.array([arr is not None for arr in b]) +# Convert the boolean mask to an integer mask +mask = np.nonzero(mask) + +A = A[mask] # Filter rows of A based on the mask +b = b[mask] + +b = b.tolist() + +############ removing the columns of A where are values are zero (these are the unkonws which doesn't fall in any equations nor 12 neither 18 and thus cannot be corrected) + +# Find the column indices where all values are zero +zero_columns = np.all(A == 0, axis=0) + +## Get the indices of the zero columns +zero_column_indices = np.where(zero_columns)[0] + +# Get the indices of the non-zero columns +non_zero_column_indices = np.where(~zero_columns)[ + 0 +] # the indices of the unknowns that can be corrected + +# Save the file +directory_path = os.path.join(output_path, "Data") +file_path = os.path.join(directory_path, f"indices_unknown_tobe_corrected.npy") +np.save(file_path, non_zero_column_indices) + + +### Adding all the existing 6-day ifgs indices to non_zero_column_indices. We want to use estimate their biases in the second step. + + +A_bk = A # to keep a copy of A in case we want to esimate all 6-day biases with smoothing constraints +b_bk = b +all_column_indices = np.array( + range(A_bk.shape[1]) +) # generating all indices in case we want to esimate all 6-day biases with smoothing constraints + +A = A[:, non_zero_column_indices] + +# finding the last column with value -1. this gives the number of 6-day biases that can be corrected +last_column_with_minus_1 = None + +# Iterate through the columns from right to left +for col in range(len(A[0]) - 1, -1, -1): + if -1 in [row[col] for row in A]: + last_column_with_minus_1 = col + break + +num_rows_before_temporals = A.shape[0] + +################################# Least Square inversion ############################### +######################################################################################## +# First Inversion: Without Temporal Smoothing Constraints +print("Starting the first inversion (without temporal smoothing constraints).") +print( + "This step estimates the bias terms that can be corrected based on observed loop closures." +) + +damp_factor = 0 + +if apply_to_all == "no": # this is without using any temporal constraints + + X1 = np.zeros((np.shape(A_bk)[1], frame_row, frame_col), dtype=np.float32) + # X1[:,:,:] = np.nan # after checking noticed this doesn't have any effect on the final vel + b_bk = np.array(b_bk) + + total_pixels = frame_row * frame_col # Total number of pixels for progress bar + processed_pixels = 0 # Counter for processed pixels + print("Progress: [", end="", flush=True) + + for row in range(frame_row): + for col in range(frame_col): + processed_pixels += 1 + # Update progress bar + percentage = (processed_pixels / total_pixels) * 100 + if processed_pixels % (total_pixels // 100) == 0: # Update every 1% + print(f"{int(percentage)}%", end="", flush=True) + sys.stdout.write( + "\rProgress: [" + + "=" * (int(percentage) // 2) + + " " * (50 - int(percentage) // 2) + + "]" + ) + + non_nan_mask = ~np.isnan(b_bk[:, row, col]) + bb = b_bk[non_nan_mask, row, col] # Remove NaN values + AA = A_bk[non_nan_mask, :] # Remove corresponding rows + if len(bb) > min_num_eq: # what is the min number of equation? + + x, istop, itn, normr, normr2 = lsqr( + csc_array(AA), bb, damp=damp_factor + )[ + :5 + ] # Using sparse matrix representation scipy.sparse.linalg + X1[:, row, col] = x + +print("\nFirst inversion (without temporal smoothing constraints) completed.") + +########################### +# Second Inversion: With Temporal Smoothing Constraints +print("Starting the second inversion (with temporal smoothing constraints).") +print( + "This step estimates all bias terms, including those that cannot be corrected, using temporal smoothing constraints." +) + +if with_temporals == "yes": # This is using the temporal constranints on all unknowns + X2 = np.zeros((np.shape(A_bk)[1], frame_row, frame_col), dtype=np.float32) + # X2[:,:,:] = np.nan # after checking noticed this doesn't have any effect on the final vel + b_bk = np.array(b_bk) + + #### Adding temporal constraints + row = np.zeros( + A_bk.shape[1], dtype=np.float32 + ) # additional rows the design matrix A for temporal constraints, setting that to zero again + for i in range(len(all_column_indices) - 2): # for all unknowns + row[i] = -1 * w + row[i + 1] = 2 * w + row[i + 2] = -1 * w + A_bk = np.vstack((A_bk, row)) + row = np.zeros( + A_bk.shape[1], dtype=np.float32 + ) # additional rows the design matrix A for temporal constraints, setting that to zero again + num_temporals = len(A_bk) - len(b_bk) + # print('num_temporals is ', num_temporals) + + ######### Appending zeroes to b according to the added number of temporal constraints obtained by np.shape(A)[0] - np.shape(b)[0] + b_0 = np.zeros( + (np.shape(A_bk)[0] - np.shape(b)[0], frame_row, frame_col), dtype=np.float32 + ) + b_bk = np.vstack((b_bk, b_0)) + + thresh_n_eq = num_temporals # in case of using temporals, we don't want the nan pixels with zero equations with only using temporals + + processed_pixels = 0 # Reset counter for second inversion + print("Progress: [", end="", flush=True) + + for row in range(frame_row): + for col in range(frame_col): + processed_pixels += 1 + # Update progress bar + percentage = (processed_pixels / total_pixels) * 100 + if processed_pixels % (total_pixels // 100) == 0: # Update every 1% + print(f"{int(percentage)}%", end="", flush=True) + sys.stdout.write( + "\rProgress: [" + + "=" * (int(percentage) // 2) + + " " * (50 - int(percentage) // 2) + + "]" + ) + + non_nan_mask = ~np.isnan(b_bk[:, row, col]) + bb = b_bk[non_nan_mask, row, col] # Remove NaN values + AA = A_bk[non_nan_mask, :] # Remove corresponding rows + if len(bb) > thresh_n_eq: + + x, istop, itn, normr, normr2 = lsqr( + csc_array(AA), bb, damp=damp_factor + )[ + :5 + ] # Using sparse matrix representation scipy.sparse.linalg + X2[:, row, col] = x + + print("\nSecond inversion (with temporal smoothing constraints) completed.") + + ####### combining X1 and X2 into X + + X = np.zeros((np.shape(A_bk)[1], frame_row, frame_col), dtype=np.float32) + X[:, :, :] = np.nan + + # Fill X with values from X1 according to non_zero_column_indices + for i, idx in enumerate(non_zero_column_indices): + X[idx, :, :] = X1[i, :, :] + + # Fill in the missing values from X2 + for i, idx in enumerate(all_column_indices): + if idx not in non_zero_column_indices: + X[idx, :, :] = X2[i, :, :] + + # Ensure the dtype is appropriate for your data + X = X.astype(np.float32) + +directory_path = os.path.join(output_path, "Data") +file_path = os.path.join(directory_path, f"X_base_ifgs_biases.npy") +np.save(file_path, X) + +print("Inversion process completed. Results saved to:", file_path) + +########################################################################################################## diff --git a/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_05_Correction.py b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_05_Correction.py new file mode 100644 index 0000000..a83200d --- /dev/null +++ b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/bk_23Sep2025/PhaseBias_05_Correction.py @@ -0,0 +1,481 @@ +#!/usr/bin/env python3 + +import numpy as np +from datetime import datetime +import pickle +import time +import os +from bin.generate_ifgs_from_epochs import generate_ifg_pairs +import configparser +import glob +from bin.resample import resample_geotiff +import sys + +# sys.path.append(os.path.dirname(os.path.abspath(__file__))) +import bin +from bin.read_orig_ifgs_coh import read_orig_ifgs_coh + +from bin.ifg_cc_totif import ifg_cc_2tif +import gc + +############################################################################### + +HELP_TEXT = """ +PhaseBias_05_Correction.py + +This script applies phase bias correction to desired interferograms using the estimated bias terms for the base interferograms. The corrected interferograms are saved in GeoTIFF format for further analysis. + +### Workflow: +1. **Inputs:** + - **Original Wrapped Interferograms, and Coherence Data:** Obtained from step 1 (`PhaseBias_01_Read_Data.py`). + - **Estimated Bias Terms for Base Intierferograms:** Obtained from step 4 (`PhaseBias_04_Inversion.py`). + - **Calibration Parameters (*a_n*):** Estimated from step 3 (`PhaseBias_03_calibration_pars.py`). + - **max_con:** Defines the maximum number of connections to be corrected. For instance: + - `max_con=5` corrects interferograms up to 30-day intervals for a 6-day acquisition interval. + - Adjust `max_con` as needed. + +2. **Bias Term Estimation for Desired Interferograms:** + - Bias terms for desired interferograms, δ_(i,i+n+1), are estimated using the relationship: + δ_(i,i+n+1) = a_n * (∑_(t=i)^(i+n) δ_(t,t+1) ) + +3. **Correction of Interferograms:** + - Using the estimated bias terms, the interferograms are corrected as follows: + φ_(i,i+n+1)^c = φ_(i,i+n+1) - δ_(i,i+n+1) + + - Here, φ_(i,i+n+1) is the original interferogram, and φ_(i,i+n+1)^c is the corrected interferogram. + +4. **Outputs:** + - Corrected interferograms are saved in GeoTIFF format in the `GEOC` directory under the `output_path` specified in `config.txt`. + +### Output File Format: +- **Corrected Interferograms:** Saved in GeoTIFF format, organized by temporal baseline. + +### Input Requirements: +- Original wrapped interferograms from step 1. +- Bias terms for base interferograms from step 4. +- Configuration parameters (e.g., `output_path`, `interval`) defined in `config.txt`. + +### Output Directory: +- `GEOC` directory under `output_path`: Contains the corrected interferograms in GeoTIFF format, organized by temporal baseline. + +""" +if "--help" in sys.argv: + print(HELP_TEXT) + sys.exit(0) + + +################################################################################ + + +# Config file path +config_file = "config.txt" + +# Check if config file exists +if not os.path.exists(config_file): + raise FileNotFoundError( + f"Error: The configuration file '{config_file}' was not found in the script's directory." + ) + +# Initialize the configparser +config = configparser.ConfigParser() + + +# Read the config.txt file +config.read(config_file) + + +# Define required parameters +required_parameters = [ + "root_path", + "frame", + "start", + "end", + "interval", + "nlook", + "LiCSAR_data", + "a1_6_day", + "a2_6_day", + "a3_6_day", + "a4_6_day", + "a1_12_day", + "a2_12_day", + "a3_12_day", + "a4_12_day", + "estimate_an_values", +] + +# Initialize a dictionary to store parameters +parameters = {} + +# Attempt to retrieve each parameter and catch errors +try: + parameters["root_path"] = config.get("DEFAULT", "root_path") + parameters["output_path"] = config.get("DEFAULT", "output_path") + parameters["frame"] = config.get("DEFAULT", "frame") + parameters["start"] = config.get("DEFAULT", "start") + parameters["end"] = config.get("DEFAULT", "end") + parameters["interval"] = config.getint("DEFAULT", "interval") + parameters["nlook"] = config.getint( + "DEFAULT", "nlook" + ) # Assuming 'nlook' should be an integer + parameters["LiCSAR_data"] = config.get("DEFAULT", "LiCSAR_data") + parameters["a1_6_day"] = config.getfloat("DEFAULT", "a1_6_day") + parameters["a2_6_day"] = config.getfloat("DEFAULT", "a2_6_day") + parameters["a3_6_day"] = config.getfloat("DEFAULT", "a3_6_day") + parameters["a4_6_day"] = config.getfloat("DEFAULT", "a4_6_day") + parameters["a1_12_day"] = config.getfloat("DEFAULT", "a1_12_day") + parameters["a2_12_day"] = config.getfloat("DEFAULT", "a2_12_day") + parameters["a3_12_day"] = config.getfloat("DEFAULT", "a3_12_day") + parameters["estimate_an_values"] = config.get("DEFAULT", "estimate_an_values") + +except configparser.NoOptionError as e: + # Error message if a required parameter is missing + missing_param = str(e).split(": ")[1] + raise ValueError( + f"Error: Required parameter '{missing_param}' is missing in '{config_file}'. Please check the file contents." + ) +except configparser.Error as e: + # General error for any configparser-related issues + raise ValueError(f"Error while reading '{config_file}': {e}") + +# Assign to individual variables for easy access +root_path = parameters["root_path"] +output_path = parameters["output_path"] +frame = parameters["frame"] +start = parameters["start"] +end = parameters["end"] +interval = parameters["interval"] +nlook = parameters["nlook"] +LiCSAR_data = parameters["LiCSAR_data"] +estimate_an_values = parameters["estimate_an_values"] + + +start_date = datetime.strptime(start, "%Y%m%d") +end_date = datetime.strptime(end, "%Y%m%d") + +uncor_only = 0 # in case of 1 only uses 6-12-18 uncorrected ifgs. it is based on the desired_lengths ifgs +max_con = 5 # in case of 5, it will correct up to 6*5=30-day ifgs. + + +##################################################### +# Root directory where the category directories are located + +if LiCSAR_data == "yes": + track = frame[0:3] # extracting track number from frame id + if track[0] == "0": + track = track[1:3] + if track[0] == "0": + track = track[1:2] + + root_directory = ( + f"/gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products/{track}/{frame}" + ) +else: + root_directory = root_path + + +############################################ + +# Initialize the an values with None for safety +an_values = [None] * ( + max_con - 1 +) # For max_con=5, this will create [None, None, None, None] + +print("estimate_an_values is ", estimate_an_values) + +# Check if estimate_an_values is 'yes' +if estimate_an_values == "yes": + an_file_path = os.path.join(output_path, "Data", "an.txt") + + # Try reading the file + try: + if os.path.exists(an_file_path): + print(f"Reading 'a' values from {an_file_path}...") + + # Read the file and load the an values dynamically + with open(an_file_path, "r") as file: + for line in file: + line = line.strip() + for i in range( + 1, max_con + ): # Loop through a1, a2, ..., a(max_con-1) + param_name = f"a{i}" + if line.startswith(f"{param_name}="): + an_values[i - 1] = float(line.split("=")[1]) + + # Check if all required an values were found + if any(value is None for value in an_values): + print( + "Warning: Missing one or more an values in an.txt. Switching to default values from config file." + ) + raise ValueError("Incomplete an values.") + else: + print( + f"Warning: {an_file_path} does not exist. Switching to default values from config file." + ) + raise FileNotFoundError + + except (FileNotFoundError, ValueError): + # Fallback to default values from config file + print("Loading default 'an' values from config file...") + for i in range(1, max_con): + if interval == 6: + an_values[i - 1] = parameters[f"a{i}_6_day"] + else: + an_values[i - 1] = parameters[f"a{i}_12_day"] +else: + # Use default values from the config file + print("Using default 'an' values from the config file.") + for i in range(1, max_con): + if interval == 6: + an_values[i - 1] = parameters[f"a{i}_6_day"] + else: + an_values[i - 1] = parameters[f"a{i}_12_day"] + +# Print the chosen an values +for i, value in enumerate(an_values, start=1): + print(f"Final value - a{i}: {value}") + + +############## generating ifg pairs between two epochs + +# desired_lengths = [interval, 2*interval, 3*interval, 4*interval, 5*interval] +desired_lengths = [interval * i for i in range(1, max_con + 1)] # replacing the above + +all_ifgs_string = generate_ifg_pairs(start_date, end_date, interval, desired_lengths) + + +##### reading original ifgs/coh ############################# +#################################################################################### + +############################### Reading input data ############################################# +#################### Read ifgs: +print("Reading all ifgs...") +# Define the path components +sub_dir = "Data" +filename_pattern = "All_ifgs_*" # Pattern to match files starting with "All_ifgs" + +# Construct the full file path with the refined pattern +file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + ifgs = pickle.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) + + +###################### Read coh: +print("\nReading all coherence data...") +# Define the path components +sub_dir = "Data" +filename_pattern = "All_coh_*" # Pattern to match files starting with "All_ifgs" + +# Construct the full file path with the refined pattern +file_path_pattern = os.path.join(output_path, sub_dir, filename_pattern) +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + coh = pickle.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) +print("\nReading data completed.") + + +######## stacking all the ifgs 6/12/18 in a numpy var all_ifgs +desired_categories = desired_lengths + +all_ifgs, all_coh, existing_ifgs_index = read_orig_ifgs_coh( + ifgs, coh, desired_categories +) + + +del ifgs, coh +gc.collect() + +##### Reading the estimated bias terms as well as the indices of unknowns that can be corrected obtained in step 03 ############## +############################################################################################## +print("Correcting the interferograms...") +# Define the path components +sub_dir = "Data" +filename_pattern_X = "X_base*" # Pattern to match files starting with "X_base" +filename_pattern_indices = "indices*" + +# Construct the full file path with the refined pattern +file_path_pattern_X = os.path.join(output_path, sub_dir, filename_pattern_X) +file_path_pattern_indices = os.path.join(output_path, sub_dir, filename_pattern_indices) + +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern_X) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + X = np.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) +# print('Reading ifgs completed.') + +# Use glob to find the file and ensure it exists +file_list = glob.glob(file_path_pattern_indices) +if file_list: + file_path = file_list[0] # Get the single file path directly + + # Load the data from the file + with open(file_path, "rb") as file: + indices_unknown_tobe_corrected = np.load(file) +else: + raise FileNotFoundError( + f"No file matching '{filename_pattern}' was found in '{os.path.join(output_dir, sub_dir)}'." + ) + + +#### loading indices of unknown that can be corrected and the estimated unknowns + +indices_unknown_tobe_corrected_orig = indices_unknown_tobe_corrected.copy() + +len_6_biases = indices_unknown_tobe_corrected[-1] + 1 # number of 6-day biases + +all_column_indices = np.array( + range(len_6_biases) +) # generating all indices in case we want to esimate all 6-day biases with smoothing constraints. +indices_unknown_tobe_corrected = all_column_indices # in this case we had provided the corrections for all 6-day biases + + +# extending the indices_unknown_tobe_corrected in the original approach to have the indices of 12/18 correctable ifgs: +extended_indices = ( + indices_unknown_tobe_corrected.copy() +) # because in this case indices_unknown_tobe_corrected referes to all_column_indices +X_extended = ( + X.copy() +) # because in this case we have estimated all 6-day biases but we don't need them for the correction. we just need them to correct the existing 12/18 days based on the 6 days + +extended_indices = extended_indices.tolist() +X_extended = X_extended.tolist() + + +# Generalized loop to handle any max_con +for n in range(2, max_con + 1): # Start from 2*interval and go up to (max_con)*interval + current_length = n * interval # Calculate the length (e.g., 12, 18, 24, ...) + + if current_length in desired_lengths: # Check if this length is required + for i in range(len(indices_unknown_tobe_corrected) - (n - 1)): + last_item = extended_indices[-1] # Get the last index + extended_indices.append(last_item + 1) # Append the new index + + # Sum up X values for the current length (n) + X_sum = sum( + X[i + j, :, :] for j in range(n) + ) # Sum over n consecutive indices + + # Append the result to X_extended using the corresponding an_values + X_extended.append( + an_values[n - 2] # an_values is 0-based, so use n-2 + * np.angle(np.exp(complex(0 + 1j) * X_sum)) + ) + + +X = np.array(X_extended) +indices_unknown_tobe_corrected = extended_indices +del X_extended + + +#### correcting the ifgs (6/12/18) ######################## +################################################################# +all_ifgs_cor = np.full_like(all_ifgs, None) +all_ifgs_cor[:, :, :] = np.nan + +if uncor_only == 0: # if we want to use the corrected ifgs (using wrapped phases) + all_ifgs_cor[indices_unknown_tobe_corrected, :, :] = np.angle( + np.exp( + complex(0 + 1j) + * (all_ifgs[indices_unknown_tobe_corrected, :, :] - X[:, :, :]) + ) + ) +else: # no need for correction + all_ifgs_cor[indices_unknown_tobe_corrected, :, :] = all_ifgs[ + indices_unknown_tobe_corrected, :, : + ] + + +### Outputting to disk ################ +############################################################################# + + +# Create the directory (and parent directories) if they don't exist +GEOC_dir = "GEOC" +metadata_dir = "metadata" + +output_wrap_dir = os.path.join(output_path, GEOC_dir) +os.makedirs( + output_wrap_dir, exist_ok=True +) # create the GEOC directory if it doesn't exist + +template_tif_path = os.path.join(root_directory, metadata_dir, frame + ".geo.hgt.tif") + +if nlook != 1: + resampled_tif_path = os.path.join(output_wrap_dir, frame + ".geo.hgt.tif") + resample_geotiff(template_tif_path, resampled_tif_path, nlook) + template_tif_path = resampled_tif_path + + +total_files = len(all_ifgs) # Total number of interferograms +processed_files = 0 # Counter for processed files + +print(f"Writing the corrected interferograms to {output_wrap_dir}") +print("Progress: [", end="", flush=True) +time.sleep(3) + + +for i in range(len(all_ifgs)): + if i in indices_unknown_tobe_corrected: + + # Writing progress bar + processed_files += 1 + percentage = (processed_files / total_files) * 100 + if processed_files % (total_files // 100) == 0: # Update progress every 1% + sys.stdout.write( + f"\rProgress: [" + + "=" * (int(percentage) // 2) + + " " * (50 - int(percentage) // 2) + + f"] {int(percentage)}%" + ) + sys.stdout.flush() + + inpha = all_ifgs_cor[i, :, :] # your phase input, float32 + incoh = all_coh[i, :, :] # your coh input. It can be either 0-255 or 0-1 + + output_phase_path = os.path.join(output_wrap_dir, all_ifgs_string[i]) + output_cc_path = os.path.join(output_wrap_dir, all_ifgs_string[i]) + + # output_phase_path = output_wrap_dir + all_ifgs_string[i] + # output_cc_path = output_wrap_dir + all_ifgs_string[i] + + os.makedirs(output_phase_path, exist_ok=True) + os.makedirs(output_cc_path, exist_ok=True) + + output_phase_file = ( + output_phase_path + "/" + all_ifgs_string[i] + ".geo.diff_pha.tif" + ) + + output_cc_file = output_cc_path + "/" + all_ifgs_string[i] + ".geo.cc.tif" + + ifg_cc_2tif(inpha, output_phase_file, template_tif_path) + ifg_cc_2tif(incoh, output_cc_file, template_tif_path) + +print("\nAll corrected interferograms have been written to disk.") diff --git a/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/config.txt b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/config.txt new file mode 100644 index 0000000..15f08de --- /dev/null +++ b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/config.txt @@ -0,0 +1,43 @@ +[DEFAULT] +# That path to interferograms directory. In case of LiCSAR data set LiCSAR_public='yes', and provide the frame id +root_path = /gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products.public/44/044A_04728_131314 + +output_path = /work/scratch-pw4/earyma/ESA_project/044A_04728_131314 + +LiCSAR_data=no +frame=044A_04728_131314 + +# Provide the start and end dates of the time-series that you want to calculate the phase bias correction +start=20170513 +end=20210504 + +#Provide the data acquisition interval: 6-day or 12-day +interval=6 + + +# If you want to apply further multilooking to interferograms (expand the explanation) +nlook=10 + + +# The calibration parameters 'an' can be estimated from the data by specifying the number of parameters to be determined. For instance, setting num_a=2 indicates that +#a1 and a2 will be estimated. + +num_a=2 + + +### an parameters +# in case of yes the an values will be estimated from the data, otherwise the default values given below will be used +estimate_an_values=no + +# Using 6-days interval +a1_6_day=0.50 +a2_6_day=0.36 +a3_6_day=0.299 +a4_6_day=0.2476 + + +# using 12-day interval +a1_12_day=0.494 +a2_12_day=0.297 +a3_12_day=0.24 +a4_12_day=0.22 diff --git a/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/config_12day.txt b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/config_12day.txt new file mode 100644 index 0000000..3c92abc --- /dev/null +++ b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/config_12day.txt @@ -0,0 +1,45 @@ +[DEFAULT] +# That path to interferograms directory. In case of LiCSAR data set LiCSAR_public='yes', and provide the frame id +root_path = /gws/nopw/j04/nceo_geohazards_vol1/public/LiCSAR_products.public/44/044A_04728_131314 + +output_path = /work/scratch-pw4/earyma/ESA_project/044A_04728_131314 + +LiCSAR_data=no +frame=044A_04728_131314 + +# Provide the start and end dates of the time-series that you want to calculate the phase bias correction +start=20170513 +end=20210504 + +#Provide the data acquisition interval: 6-day or 12-day +# Modified for 12-day and 24-day data +interval=12 + + +# If you want to apply further multilooking to interferograms (expand the explanation) +nlook=10 + +# The calibration parameters 'an' can be estimated from the data by specifying the number of parameters to be determined. For instance, setting num_a=2 indicates that +#a1 and a2 will be estimated. + +num_a=2 + + +### an parameters +# in case of yes the an values will be estimated from the data, otherwise the default values given below will be used +# Modified: estimate from data for 12-day intervals +estimate_an_values=yes + +# Using 6-days interval (original) +a1_6_day=0.50 +a2_6_day=0.36 +a3_6_day=0.299 +a4_6_day=0.2476 + + +# using 12-day interval (for 12-day base interferograms) +# These values will be used if estimate_an_values=no +a1_12_day=0.494 +a2_12_day=0.297 +a3_12_day=0.24 +a4_12_day=0.22 diff --git a/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/images/.gitkeep b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/images/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/images/.gitkeep @@ -0,0 +1 @@ + diff --git a/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/images/loop_closure_time_series.png b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/images/loop_closure_time_series.png new file mode 100644 index 0000000..d5252b1 Binary files /dev/null and b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/images/loop_closure_time_series.png differ diff --git a/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/images/spatial_distribution_a1.png b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/images/spatial_distribution_a1.png new file mode 100644 index 0000000..9f6fd52 Binary files /dev/null and b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/images/spatial_distribution_a1.png differ diff --git a/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/images/spatial_distribution_a2.png b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/images/spatial_distribution_a2.png new file mode 100644 index 0000000..492ff32 Binary files /dev/null and b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/images/spatial_distribution_a2.png differ diff --git a/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/images/timeseries_a1.png b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/images/timeseries_a1.png new file mode 100644 index 0000000..26cac51 Binary files /dev/null and b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/images/timeseries_a1.png differ diff --git a/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/images/timeseries_a2.png b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/images/timeseries_a2.png new file mode 100644 index 0000000..7a8cbdf Binary files /dev/null and b/third_party/PyINT/pyint/InSAR_PhaseBias_Correction/images/timeseries_a2.png differ diff --git a/third_party/PyINT/pyint/LT1_import_SLC_from_zipfiles b/third_party/PyINT/pyint/LT1_import_SLC_from_zipfiles new file mode 100644 index 0000000..f953e16 --- /dev/null +++ b/third_party/PyINT/pyint/LT1_import_SLC_from_zipfiles @@ -0,0 +1,121 @@ +#! /bin/csh -f +echo "*** S1_import_SLC_from_zipfiles: Script to read in and concatenate S1 TOPS SLC from zip files ***" +echo "*** Copyright 2023 Gamma Remote Sensing, v2.3 19-Apr-2023 uw/cm ***" +echo "" + +if ($#argv < 1)then + echo "usage: S1_import_SLC_from_zipfiles <zipfile_list> [burst_number_table_ref] [pol] [dtype] [swath_flag] [OPOD_dir] [cleaning] [noise_mode]" + echo " zipfile_list (input) ASCII file containing S1 zip filename(s) of one data take (one per line, in correct sequence)" + echo " (other values are vv, vh, hh, hv)" + echo " dtype output data type (enter - for default)" + echo " 0: FCOMPLEX (default)" + echo " 1: SCOMPLEX" + echo " " + echo "resulting files: burst SLC files (per polarization, with SLC_tab, SLC, SLC_par, TOPS_par and optionally SLC.noise)" + echo "(concatenated, empty bursts added where necessary) at selected polarizations" + echo " " + exit +endif + + + +# 6-Jan-2023: logfile name updated to be the same on every platform + +set zipfile_list = $1 + # in this case it generates temp.burst_number_table_ref +set dtype = "0" # 0: FCOMPLEX + + +if ($#argv >= 2) then + if ( "$2" != "-" ) then + set dtype = $2 + endif +endif + + + +set zipfile=`date "+%Y%m%d_%H%M%S"` +set logfile = "LT1_import_SLC_from_zipfiles.$zipfile.logfile" +echo "the processing is documented in the logfile: $logfile" + +echo "COMMAND_used: $0 $1 $2 " > $logfile +date >> $logfile +echo "" >> $logfile + +############################################################################## +if (-e "$1" == 0) then + echo "ERROR: zipfile_list ($1) does not exist"; exit(-1) +else + #determine number of rows of $zipfile_list + set zipfile=`awk '(NR>=1){print NF}' $zipfile_list` + set nrows=`echo "$zipfile" | awk '(NR==1){print NF}'` +endif + +set i="1" +while ( "$i" <= "$nrows" ) + # interpret names, check consistency of names + # read first zipfile name and reduce it to filename without path + set zipfile = `awk '(NR=='"$i"'){print $1}' $zipfile_list` + if ( -e $zipfile == "0" ) then + echo "ERROR: zipfile number $1 ($zipfile) does not exist"; exit(-1) + endif + + set zipfile = `awk '(NR=='"$i"'){print $1}' $zipfile_list` + # set l = `echo "$zipfile" | awk '{print length($1)}'` + set m = `echo "$zipfile tar.gz" | awk '{print match($1,$2)-71}'` + set filename_without_path = `echo "$zipfile $m" | awk '{print substr($1,$2,70)}'` + set path_only = `echo "$zipfile $m" | awk '{print substr($1,1,$2-1)}'` + # echo "$path_only $filename_without_path" + echo "$filename_without_path" + echo "$path_only" + + set sensor = `echo "$filename_without_path" | awk '{print substr($1,1,4)}'` + set mode = `echo "$filename_without_path" | awk '{print substr($1,15,6)}'` + set polar = `echo "$filename_without_path" | awk '{print substr($1,54,2)}'` + set level = `echo "$filename_without_path" | awk '{print substr($1,50,3)}'` + set date = `echo "$filename_without_path" | awk '{print substr($1,41,8)}'` + set orbit = `echo "$filename_without_path" | awk '{print substr($1,22,6)}'` + set product = `echo "$filename_without_path" | awk '{print substr($1,61,10)}'` + echo "sensor: $sensor mode: $mode polar: $polar level: $level date: $date orbit: $orbit product: $product" + + if ( "$level" != "SLC" ) then + echo "ERROR: type indicated in zipfile ($type) is not SLC"; exit(-1) + endif + + set GeoTIFF = `tar -tzf $zipfile | grep "tiff" | awk '{print $1}'` + set xml_name = `tar -tzf $zipfile | grep "meta.xml" | awk '{print $1}'` + set num = `grep "$date" $zipfile_list | wc -l ` + if ( ! -d $date ) then + mkdir $date + endif + if ( ! -d tmp_data_dir ) then + mkdir tmp_data_dir + endif + if ( $num>1 ) then + set SLC = ${date}_$product.slc + set SLC_par = ${date}_$product.slc.par + set SLC1 = ${date}_$product.slc.update + set SLC1_par = ${date}_$product.slc.update.par + else + set SLC = $date.slc + set SLC_par = $date.slc.par + set SLC1 = $date.slc.update + set SLC1_par = $date.slc.update.par + endif + tar -xzv -C tmp_data_dir -f $zipfile $GeoTIFF + tar -xzv -C tmp_data_dir -f $zipfile $xml_name + echo "COMMAND: par_LT1_SLC ./tmp_data_dir/$GeoTIFF ./tmp_data_dir/$xml_name ./$date/$SLC_par ./$date/$SLC " + par_LT1_SLC ./tmp_data_dir/$GeoTIFF ./tmp_data_dir/$xml_name ./$date/$SLC_par ./$date/$SLC + echo "COMMAND: par_LT1_SLC ./tmp_data_dir/$GeoTIFF ./tmp_data_dir/$xml_name ./$date/$SLC1 ./$date/$SLC1_par " + par_LT1_SLC_YSLi ./tmp_data_dir/$GeoTIFF ./tmp_data_dir/$xml_name ./$date/$SLC1 ./$date/$SLC1_par 0 + grep "state_vector" ./$date/$SLC1_par | awk -F ' ' '{printf " %s %.5f %.5f %.5f %s %s\n", $1, $2, $3, $4, $5,$6}' | awk '{if (NR%2==1){print $0,"m/s"}else{print $0,"m"}}' | awk 'NR<=3 {print $1,$2}' >tmp1 + grep "state_vector" ./$date/$SLC1_par | awk -F ' ' '{printf " %s %.5f %.5f %.5f %s %s\n", $1, $2, $3, $4, $5,$6}' | awk '{if (NR%2==1){print $0,"m/s"}else{print $0,"m"}}' | awk 'NR>3 {print $0}' >tmp2 + set histchars="" + awk '!/state_vector/ ' ./$date/$SLC_par >tmp + cat tmp tmp1 tmp2 > ./$date/$SLC_par + rm tmp1 tmp tmp2 + set i = `echo "$i" | awk '{printf "%d", $1+1}'` +end +rm -rf *.logfile tmp_data_dir + + diff --git a/third_party/PyINT/pyint/LT1_import_SLC_from_zipfiles1 b/third_party/PyINT/pyint/LT1_import_SLC_from_zipfiles1 new file mode 100644 index 0000000..7141360 --- /dev/null +++ b/third_party/PyINT/pyint/LT1_import_SLC_from_zipfiles1 @@ -0,0 +1,126 @@ +#! /bin/csh -f +echo "*** S1_import_SLC_from_zipfiles: Script to read in and concatenate S1 TOPS SLC from zip files ***" +echo "*** Copyright 2023 Gamma Remote Sensing, v2.3 19-Apr-2023 uw/cm ***" +echo "" + +if ($#argv < 1)then + echo "usage: S1_import_SLC_from_zipfiles <zipfile_list> [burst_number_table_ref] [pol] [dtype] [swath_flag] [OPOD_dir] [cleaning] [noise_mode]" + echo " zipfile_list (input) ASCII file containing S1 zip filename(s) of one data take (one per line, in correct sequence)" + echo " (other values are vv, vh, hh, hv)" + echo " dtype output data type (enter - for default)" + echo " 0: FCOMPLEX (default)" + echo " 1: SCOMPLEX" + echo " " + echo "resulting files: burst SLC files (per polarization, with SLC_tab, SLC, SLC_par, TOPS_par and optionally SLC.noise)" + echo "(concatenated, empty bursts added where necessary) at selected polarizations" + echo " " + exit +endif + + + +# 6-Jan-2023: logfile name updated to be the same on every platform + +set zipfile_list = $1 + # in this case it generates temp.burst_number_table_ref +set dtype = "0" # 0: FCOMPLEX + + +if ($#argv >= 2) then + if ( "$2" != "-" ) then + set dtype = $2 + endif +endif + + + +set zipfile=`date "+%Y%m%d_%H%M%S"` +set logfile = "LT1_import_SLC_from_zipfiles.$zipfile.logfile" +echo "the processing is documented in the logfile: $logfile" + +echo "COMMAND_used: $0 $1 $2 " > $logfile +date >> $logfile +echo "" >> $logfile + +############################################################################## +if (-e "$1" == 0) then + echo "ERROR: zipfile_list ($1) does not exist"; exit(-1) +else + #determine number of rows of $zipfile_list + set zipfile=`awk '(NR>=1){print NF}' $zipfile_list` + set nrows=`echo "$zipfile" | awk '(NR==1){print NF}'` +endif + + # interpret names, check consistency of names + # read first zipfile name and reduce it to filename without path + set zipfile = `awk '(NR=='"1"'){print $1}' $zipfile_list` + if ( -e $zipfile == "0" ) then + echo "ERROR: zipfile number $1 ($zipfile) does not exist"; exit(-1) + endif + + set zipfile = `awk '(NR=='"1"'){print $1}' $zipfile_list` + set input_basename = `basename "$zipfile"` + set input_mode = "archive" + if ( `echo "$input_basename" | grep -c ".tar.gz$"` > 0 ) then + set filename_without_path = `echo "$input_basename" | sed 's/\.tar\.gz$//'` + set GeoTIFF = `tar -tzf $zipfile | grep "tiff" | awk '{print $1}'` + set xml_name = `tar -tzf $zipfile | grep "meta.xml" | awk '{print $1}'` + if ( ! -d tmp_data_dir ) then + mkdir tmp_data_dir + endif + tar -xzv -C tmp_data_dir -f $zipfile $GeoTIFF + tar -xzv -C tmp_data_dir -f $zipfile $xml_name + set input_tiff = ./tmp_data_dir/$GeoTIFF + set input_xml = ./tmp_data_dir/$xml_name + else if ( `echo "$input_basename" | grep -c ".tiff$"` > 0 ) then + set input_mode = "scene" + set filename_without_path = `echo "$input_basename" | sed 's/\.tiff$//'` + set input_tiff = $zipfile + set input_xml = `echo "$zipfile" | sed 's/\.tiff$/.meta.xml/'` + if ( -e "$input_xml" == 0 ) then + echo "ERROR: meta xml for scene input does not exist ($input_xml)"; exit(-1) + endif + else + echo "ERROR: unsupported LT-1 input ($zipfile), only .tar.gz or .tiff are supported"; exit(-1) + endif + # echo "$filename_without_path" + + set sensor = `echo "$filename_without_path" | awk '{print substr($1,1,3)}'` + set mode = `echo "$filename_without_path" | awk '{print substr($1,15,5)}'` + set polar = `echo "$filename_without_path" | awk '{print substr($1,54,2)}'` + set level = `echo "$filename_without_path" | awk '{print substr($1,50,3)}'` + set date = `echo "$filename_without_path" | awk '{print substr($1,41,8)}'` + set orbit = `echo "$filename_without_path" | awk '{print substr($1,22,5)}'` + set product = `echo "$filename_without_path" | awk '{print substr($1,61,10)}'` + echo "sensor: $sensor mode: $mode polar: $polar level: $level date: $date orbit: $orbit product: $product" + + if ( "$level" != "SLC" ) then + echo "ERROR: type indicated in zipfile ($type) is not SLC"; exit(-1) + endif + + set num = `grep "$date" t_$date | wc -l ` + + if ( $num>1 ) then + set SLC = ${date}_${product}.slc + set SLC_par = ${date}_${product}.slc.par + set SLC1 = ${date}_${product}.slc.update + set SLC1_par = ${date}_${product}.slc.update.par + else + set SLC = $date.slc + set SLC_par = $date.slc.par + set SLC1 = $date.slc.update + set SLC1_par = $date.slc.update.par + endif + + echo "COMMAND: par_LT1_SLC $input_tiff $input_xml ./$SLC_par ./$SLC " + par_LT1_SLC $input_tiff $input_xml ./$SLC_par ./$SLC + echo "COMMAND: par_LT1_SLC $input_tiff $input_xml ./$SLC1 ./$SLC1_par " + par_LT1_SLC_YSLi $input_tiff $input_xml ./$SLC1 ./$SLC1_par 0 + grep "state_vector" ./$SLC1_par | awk -F ' ' '{printf " %s %.5f %.5f %.5f %s %s\n", $1, $2, $3, $4, $5,$6}' | awk '{if (NR%2==1){print $0,"m/s"}else{print $0,"m"}}' | awk 'NR<=3 {print $1,$2}' >tmp1 + grep "state_vector" ./$SLC1_par | awk -F ' ' '{printf " %s %.5f %.5f %.5f %s %s\n", $1, $2, $3, $4, $5,$6}' | awk '{if (NR%2==1){print $0,"m/s"}else{print $0,"m"}}' | awk 'NR>3 {print $0}' >tmp2 + set histchars="" + awk '!/state_vector/ ' ./$SLC_par >tmp + cat tmp tmp1 tmp2 > ./$SLC_par + rm tmp1 tmp tmp2 +rm -rf *.logfile tmp_data_dir + diff --git a/third_party/PyINT/pyint/MAI_SLC_Gamma.py b/third_party/PyINT/pyint/MAI_SLC_Gamma.py new file mode 100644 index 0000000..340d6e6 --- /dev/null +++ b/third_party/PyINT/pyint/MAI_SLC_Gamma.py @@ -0,0 +1,202 @@ +#! /usr/bin/env python +#''' +################################################################################## +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : February, 2017 # +# # +# Split beam of SLC: backward and forward SLC image generation # +# # +################################################################################## +#''' +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +from pyint import _utils as ut +import argparse + + + +INTRODUCTION = ''' +------------------------------------------------------------------------------------------ + Split beam of original SLC to generate sub-aperture SLC: backward- and forward-SLCs. + +''' + +EXAMPLE = ''' + Usage: + MAI_SLC_gamma.py projectName Mdate Sdate + MAI_SLC_gamma.py PacayaT163TsxHhA 20150102 20150601 +------------------------------------------------------------------------------------------ +''' + + + + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def ras2jpg(input, strTitle): + call_str = "convert " + input + ".ras " + input + ".jpg" + os.system(call_str) + call_str = "convert " + input + ".jpg -resize 250 " + input + ".thumb.jpg" + os.system(call_str) + call_str = "convert " + input + ".jpg -resize 500 " + input + ".bthumb.jpg" + os.system(call_str) + call_str = "$INT_SCR/addtitle2jpg.pl " + input + ".thumb.jpg 14 " + strTitle + os.system(call_str) + call_str = "$INT_SCR/addtitle2jpg.pl " + input + ".bthumb.jpg 24 " + strTitle + os.system(call_str) + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Split beam of original SLC to generate sub-aperture SLC: backward- and forward-SLCs',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('Mdate',help='Master date.') + parser.add_argument('Sdate',help='Slave date.') + + inps = parser.parse_args() + return inps + + + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + Mdate = inps.Mdate + Sdate = inps.Sdate + + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + coregCoarse = templateContents['Coreg_Coarse'] + if 'Squint_MAI' in templateContents: Squint = templateContents['Squint_MAI'] + else: Squint = '0.5' + + + + projectDir = scratchDir + '/' + projectName + demDir = scratchDir + '/' + projectName + '/DEM' + + slcDir = scratchDir + '/' + projectName + '/SLC' + rslcDir = scratchDir + '/' + projectName + '/RSLC' + ifgDir = projectDir + '/MAIifg' + if not os.path.isdir(ifgDir): os.mkdir(ifgDir) + + Pair = Mdate + '-' + Sdate + workDir = ifgDir + '/' + Pair + if not os.path.isdir(workDir): os.mkdir(workDir) + + +# input slcs + ####################################################################### + Mamp = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + Samp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + Mrslc = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc' + MrslcPar = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc.par' + Srslc = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc' + SrslcPar = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc.par' + + HGT = demDir + '/' + masterDate + '_' + rlks + 'rlks.rdc.dem' + + MasterPar = rslcDir + '/' + masterDate + '/' + masterDate + '.rslc.par' + + + + +# split slcs + + MFslcImg = workDir + "/" + Mdate + ".F.slc" + MFslcPar = workDir + "/" + Mdate + ".F.slc.par" + SFslcImg = workDir + "/" + Sdate + ".F.slc" + SFslcPar = workDir + "/" + Sdate + ".F.slc.par" + + MBslcImg = workDir + "/" + Mdate + ".B.slc" + MBslcPar = workDir + "/" + Mdate + ".B.slc.par" + SBslcImg = workDir + "/" + Sdate + ".B.slc" + SBslcPar = workDir + "/" + Sdate + ".B.slc.par" + + print(MFslcImg + " "+ MslcImg) + +# Multi-aperture processing + + call_str = '$GAMMA_BIN/sbi_filt '+ MslcImg + ' ' + MrslcPar + ' '+SrslcPar + ' ' + MFslcImg + ' '+ MFslcPar + ' ' + MBslcImg + ' ' + MBslcPar + ' ' + Squint + os.system(call_str) + print(call_str) + + call_str = '$GAMMA_BIN/sbi_filt '+ SslcImg + ' ' + SrslcPar + ' '+MrslcPar + ' ' + SFslcImg + ' '+ SFslcPar + ' ' + SBslcImg + ' ' + SBslcPar + ' ' + Squint + os.system(call_str) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + + + + diff --git a/third_party/PyINT/pyint/MAI_SLC_Gamma1.py b/third_party/PyINT/pyint/MAI_SLC_Gamma1.py new file mode 100644 index 0000000..bd35061 --- /dev/null +++ b/third_party/PyINT/pyint/MAI_SLC_Gamma1.py @@ -0,0 +1,134 @@ +#! /usr/bin/env python +#''' +################################################################################## +# # +# Author: Wei Chen # +# Email : chenweicug@gmail.com # +# Date : May,25th 2021 # +# # +# Split beam of SLC: backward and forward SLC image generation # +# # +################################################################################## +#''' + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse + +from pyint import _utils as ut + + +def get_s1_date(raw_file): + file0 = os.path.basename(raw_file) + date = file0[17:25] + return date + +def get_satellite(raw_file): + if 'S1A_IW_SLC_' in raw_file: + s0 = 'A' + else: + s0 = 'B' + + return s0 + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLC from Sentinel-1 raw data with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='project name. e.g., ChangningT55S1A') + parser.add_argument('Mdate',help='date to be processed. e.g., 20180101') + parser.add_argument('Sdate',help='date to be processed. e.g., 20180113') + + inps = parser.parse_args() + + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Split beam of original SLC to generate sub-aperture SLC: backward- and forward-SLCs +''' + +EXAMPLE = """Usage: + + MAI_SLC_Gamma.py projectName Mdate Sdate + + MAI_SLC_Gamma.py ChangningT55S1A 20180517 20180529 + +------------------------------------------------------------------- +""" + +def main(argv): + inps = cmdLineParse() + projectName = inps.projectName + Mdate = inps.Mdate + Sdate = inps.Sdate + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + slcDir = projectDir + '/SLC' + down_dir = projectDir + '/DOWNLOAD' + workDir = projectDir + '/MAI' + + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Mdate = templateDict['masterDate'] + coregCoarse = templateDict['Coreg_Coarse'] + + if not os.path.isdir(workDir): + call_str="mkdir " + workDir + os.system(call_str) + if 'Squint_MAI' in templateDict: Squint = templateContents['Squint_MAI'] + else: Squint = '0.5' + + SslcDir = slcDir + "/" + Sdate + MslcDir = slcDir + "/" + Mdate + + MslcImg = MslcDir + "/" + Mdate + ".slc" + MslcPar = MslcDir + "/" + Mdate + ".slc.par" + SslcImg = SslcDir + "/" + Sdate + ".slc" + SslcPar = SslcDir + "/" + Sdate + ".slc.par" + +# split slcs + + MFslcImg = workDir + "/" + Mdate + ".F.slc" + MFslcPar = workDir + "/" + Mdate + ".F.slc.par" + SFslcImg = workDir + "/" + Sdate + ".F.slc" + SFslcPar = workDir + "/" + Sdate + ".F.slc.par" + + MBslcImg = workDir + "/" + Mdate + ".B.slc" + MBslcPar = workDir + "/" + Mdate + ".B.slc.par" + SBslcImg = workDir + "/" + Sdate + ".B.slc" + SBslcPar = workDir + "/" + Sdate + ".B.slc.par" + print(MFslcImg + " "+ MslcImg) + +# Multi-aperture processing + + call_str = 'sbi_filt '+ MslcImg + ' ' + MslcPar + ' '+SslcPar + ' ' + MFslcImg + ' '+ MFslcPar + ' ' + MBslcImg + ' ' + MBslcPar + ' ' + Squint + os.system(call_str) + print(call_str) + + call_str = 'sbi_filt '+ SslcImg + ' ' + SslcPar + ' '+MslcPar + ' ' + SFslcImg + ' '+ SFslcPar + ' ' + SBslcImg + ' ' + SBslcPar + ' ' + Squint + os.system(call_str) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + diff --git a/third_party/PyINT/pyint/NameChange.py b/third_party/PyINT/pyint/NameChange.py new file mode 100644 index 0000000..6889a2d --- /dev/null +++ b/third_party/PyINT/pyint/NameChange.py @@ -0,0 +1,180 @@ +#! /usr/bin/env python +#''' +################################################################################### +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : Mar, 2017 # +# # +# Select Interferometry-Pairs from time series SAR images # +# # +################################################################################### +#''' +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + + +def is_number(s): + try: + int(s) + return True + except ValueError: + return False + +def add_zero(s): + if len(s)==1: + s="000"+s + elif len(s)==2: + s="00"+s + elif len(s)==3: + s="0"+s + return s + + +def usage(): + print(''' +****************************************************************************************************** + + Select interferometry pairs from time series SAR images + + usage: + + SelectPairs_Gamma.py ProjectName + + e.g. SelectPairs_Gamma.py PacayaT163TsxHhA + + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==2: + if argv[0] in ['-h','--help']: usage(); sys.exit(1) + else: projectName=sys.argv[1] + else: + usage();sys.exit(1) + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateContents=read_template(templateFile) + processDir = scratchDir + '/' + projectName + "/PROCESS" + slcDir = scratchDir + '/' + projectName + "/SLC" + + if not os.path.isdir(processDir): + call_str = 'mkdir ' + processDir + os.system(call_str) + + + if 'JOB' in templateContents: JOB = templateContents['JOB'] + else: JOB = 'IFG' + + INF=JOB + if INF=='IFG': + Suffix=[''] + print("Time series interferograms will be processed!") + elif INF=='MAI': + Suffix=['.F','.B'] + print("Time series multi-aperture interferograms will be processed!") + elif INF=='RSI': + Suffix=['.HF','.LF'] + print("Time series range split-spectrum interferograms will be processed!") + else: + print("The folder name %s cannot be identified !" % igramDir) + usage();sys.exit(1) + +# define files + + SLC_Tab = processDir + "/SLC_Tab" + TS_Berp = processDir + "/TS_Berp" + TS_Itab = processDir + "/TS_Itab" + itab_type = '1' + pltflg = '0' + + if 'Max_Spacial_Baseline' in templateContents: MaxSB=templateContents['Max_Spacial_Baseline'] + else: + print("Max_Spacial_Baseline is not found in template!! ") + print("500m is chosen as the threshold for spatial baseline!") + MaxSB = '500' + + if 'Max_Temporal_Baseline' in templateContents: MaxTB=templateContents['Max_Temporal_Baseline'] + else: + print("Max_Temporal_Baseline is not found in template!! ") + print("500 days is chosen as the threshold for temporal baseline!") + MaxTB = '500' + + +# extract available SAR images slc and slc_par + ListSLC = os.listdir(slcDir) + Datelist = [] + SLCfile = [] + SLCParfile = [] + + print("All of the available SAR acquisition datelist is :") + for kk in range(len(ListSLC)): + if ( is_number(ListSLC[kk]) and len(ListSLC[kk])==6 ): # if SAR date number is 8, 6 should change to 8. + DD=ListSLC[kk] + Year=int(DD[0:2]) + Month = int(DD[2:4]) + Day = int(DD[4:6]) + if ( 0 < Year < 20 and 0 < Month < 13 and 0 < Day < 32 ): + Datelist.append(ListSLC[kk]) + print(ListSLC[kk]) + DateDir = slcDir+'/'+ListSLC[kk] + SLC0 = glob.glob(DateDir+'/*slc')[0] + SLCPar0 = glob.glob(DateDir+'/*slc.par')[0] + + str_slc = slcDir + "/" + ListSLC[kk] +"/" + ListSLC[kk] + ".slc" + str_slc_par = slcDir + "/" + ListSLC[kk] +"/" + ListSLC[kk] + ".slc.par" + + call_str = 'mv ' + SLC0 + ' ' + str_slc + os.system(call_str) + + call_str = 'mv ' + SLCPar0 + ' ' + str_slc_par + os.system(call_str) + + SLCfile.append(str_slc) + SLCParfile.append(str_slc_par) + + + print("Change name of SLC file is done! ") + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + diff --git a/third_party/PyINT/pyint/OPENTOPO_USAGE.md b/third_party/PyINT/pyint/OPENTOPO_USAGE.md new file mode 100644 index 0000000..bbf8be5 --- /dev/null +++ b/third_party/PyINT/pyint/OPENTOPO_USAGE.md @@ -0,0 +1,83 @@ +# OpenTopography DEM 下载使用示例 + +## 快速开始 + +### 1. 获取免费API Key + +访问: https://opentopography.org/myOpenTopo +- 注册账户 (免费) +- 在 'My Account' 页面请求 API key +- API key 立即可用 + +### 2. 下载不同分辨率的 DEM + +#### SRTM 30m (推荐) +```bash +python makedem.py -r 116/117/39/40 --dem-source srtm \ + --opentopo-api-key YOUR_KEY \ + --opentopo-dem-type SRTMGL1 +``` + +#### SRTM 90m +```bash +python makedem.py -r 116/117/39/40 --dem-source srtm \ + --opentopo-api-key YOUR_KEY \ + --opentopo-dem-type SRTMGL3 +``` + +#### NASADEM 30m +```bash +python makedem.py -r 116/117/39/40 --dem-source srtm \ + --opentopo-api-key YOUR_KEY \ + --opentopo-dem-type NASADEM +``` + +#### Copernicus 30m +```bash +python makedem.py -r 116/117/39/40 --dem-source srtm \ + --opentopo-api-key YOUR_KEY \ + --opentopo-dem-type COP30 +``` + +#### Copernicus 90m +```bash +python makedem.py -r 116/117/39/40 --dem-source srtm \ + --opentopo-api-key YOUR_KEY \ + --opentopo-dem-type COP90 +``` + +## 数据对比 + +| DEM类型 | 分辨率 | 覆盖范围 | 推荐用途 | +|---------|--------|----------|----------| +| **SRTMGL1** | **30m** | 60°S-60°N | **SRTM研究首选** | +| SRTMGL3 | 90m | 60°S-60°N | 快速预览 | +| **NASADEM** | **30m** | 60°S-60°N | **高精度研究** | +| **COP30** | **30m** | 全球 | **全球覆盖首选** | +| COP90 | 90m | 全球 | 快速预览 | + +## 推荐选择 + +1. **SRTM研究**: SRTMGL1 (30m) +2. **高精度需求**: NASADEM 或 COP30 (30m) +3. **全球覆盖**: COP30 (30m) +4. **快速预览**: SRTMGL3 或 COP90 (90m) + +## 注意事项 + +- **SRTMGL1/COP30/NASADEM** (30m): 最大支持 450,000 km² +- **SRTMGL3/COP90** (90m): 最大支持 4,050,000 km² +- 所有数据集需要 OpenTopography API key +- 免费配额足够大部分科研用途 + +## 输出文件命名 + +- SRTMGL1_116_117_39_40.tif +- NASADEM_116_117_39_40.tif +- COP30_116_117_39_40.tif + +文件名格式: `{DEM类型}_{西经}_{东经}_{南纬}_{北纬}.tif` + +--- + +**更新日期**: 2026-03-13 diff --git a/third_party/PyINT/pyint/POT_gamma.py b/third_party/PyINT/pyint/POT_gamma.py new file mode 100644 index 0000000..4e87a87 --- /dev/null +++ b/third_party/PyINT/pyint/POT_gamma.py @@ -0,0 +1,394 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Pixel Offset Tracking (POT) for a single pair ### +### Based on GAMMA offset_pwr_tracking / offset_pwr_tracking2### +### Author: ZYD / Cascade AI ### +################################################################# + +import os +import sys +import time +import argparse +import numpy as np + +from pyint import _utils as ut + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Pixel Offset Tracking (POT) for a single interferometric pair + using GAMMA software. + + Two-round estimation approach (Greenland tracking demo): + Round 1: Large search window for initial offset field + Post-processing: Quality check, outlier removal, gap filling + Round 2: Smaller window with conditioned Round 1 as initial + Final: Convert pixel offsets to displacement in meters + Geocode +''' + +EXAMPLE = ''' + Usage: + POT_gamma.py projectName Mdate Sdate + POT_gamma.py shanghaiT171F128S1A 20241105 20241117 +------------------------------------------------------------------- +''' + + +def run_cmd(cmd_str): + """执行 GAMMA 命令并打印""" + print(f' >> {cmd_str}') + return os.system(cmd_str) + + +def sanitize_gamma_float(filepath, valid_max=1e6): + """ + 清理 GAMMA 浮点数据文件中的无效值。 + GAMMA 对无效像素写入特殊标记值（~3.4e38 / NaN / Inf）， + 这些值无法被 single_class_mapping 的范围过滤器正确捕获。 + 将 NaN / Inf / |val| > valid_max 的像素替换为 0.0。 + """ + data = np.fromfile(filepath, dtype=np.float32) + bad_mask = ~np.isfinite(data) | (np.abs(data) > valid_max) + n_bad = int(np.sum(bad_mask)) + if n_bad > 0: + data[bad_mask] = 0.0 + data.tofile(filepath) + print(f' [sanitize] {os.path.basename(filepath)}: ' + f'清理 {n_bad} 个无效像素 (NaN/Inf/|val|>{valid_max})') + + +def postprocess_offsets(offs_cpx, ccp, mli, width, + ccp_thresh, roff_min, roff_max, + azoff_min, azoff_max, + drange_thresh, dazimuth_thresh, + median_win, median_nmin, + prefix): + """ + 偏移量场后处理流程（参考 GAMMA Greenland tracking demo）: + 1) 提取距离向/方位向分量 + 清理 GAMMA no-data 标记值 + 2) 初始掩膜: 互相关阈值 + 偏移量范围限制 + 3) 中值滤波 + 偏差计算 + 4) 偏差阈值精细掩膜 + 5) 空洞填充 + 6) 空间滤波 → 组合为 conditioned 复数偏移量 + + 返回: (condi, real_interp, imag_interp) + - condi: fspf 平滑后的复数偏移量（供 Round 2 初始值） + - real_interp / imag_interp: 填充后的偏移量（供最终米制转换） + """ + real_file = prefix + '.real' + imag_file = prefix + '.imag' + + # --- 1) 提取距离向 (real) 和方位向 (imag) --- + run_cmd(f'cpx_to_real {offs_cpx} {real_file} {width} 0') + run_cmd(f'cpx_to_real {offs_cpx} {imag_file} {width} 1') + + # 清理 GAMMA no-data 标记值（~3.4e38），避免污染后续掩膜和插值 + valid_max = max(abs(float(roff_max)), abs(float(roff_min)), + abs(float(azoff_max)), abs(float(azoff_min))) * 10 + sanitize_gamma_float(real_file, valid_max) + sanitize_gamma_float(imag_file, valid_max) + sanitize_gamma_float(ccp, 1.0) + + # --- 2) 初始掩膜: 互相关 + 偏移量范围 --- + mask1 = prefix + '.mask1.bmp' + real_m1 = prefix + '.real.masked1' + imag_m1 = prefix + '.imag.masked1' + + run_cmd(f'single_class_mapping 3 ' + f'{ccp} {ccp_thresh} 1.0 ' + f'{real_file} {roff_min} {roff_max} ' + f'{imag_file} {azoff_min} {azoff_max} ' + f'{mask1} {width} 1 0 1 1') + run_cmd(f'mask_class {mask1} {real_file} {real_m1} 0 1 1 1 0 0.0') + run_cmd(f'mask_class {mask1} {imag_file} {imag_m1} 0 1 1 1 0 0.0') + + # 初始掩膜后 BMP + run_cmd(f'rasdt_pwr {real_m1} {mli} {width} - - - - ' + f'{roff_min} {roff_max} 0 rmg.cm {real_m1}.bmp - - 24') + run_cmd(f'rasdt_pwr {imag_m1} {mli} {width} - - - - ' + f'{azoff_min} {azoff_max} 0 rmg.cm {imag_m1}.bmp - - 24') + + # --- 3) 中值滤波 + 偏差 --- + real_med = prefix + '.real.median' + imag_med = prefix + '.imag.median' + dreal = prefix + '.dreal' + dimag = prefix + '.dimag' + + run_cmd(f'median_filter {real_m1} {real_med} {width} ' + f'{median_win} {median_win} {median_nmin}') + run_cmd(f'lin_comb 2 {real_m1} {real_med} 0. 1. -1. ' + f'{dreal} {width} 1 0 1 1') + run_cmd(f'median_filter {imag_m1} {imag_med} {width} ' + f'{median_win} {median_win} {median_nmin}') + run_cmd(f'lin_comb 2 {imag_m1} {imag_med} 0. 1. -1. ' + f'{dimag} {width} 1 0 1 1') + + # --- 4) 偏差阈值精细掩膜 --- + mask2 = prefix + '.mask2.bmp' + real_masked = prefix + '.real.masked' + imag_masked = prefix + '.imag.masked' + + run_cmd(f'single_class_mapping 5 ' + f'{dreal} -{drange_thresh} {drange_thresh} ' + f'{dimag} -{dazimuth_thresh} {dazimuth_thresh} ' + f'{ccp} {ccp_thresh} 1.0 ' + f'{real_file} {roff_min} {roff_max} ' + f'{imag_file} {azoff_min} {azoff_max} ' + f'{mask2} {width} 1 0 1 1 5') + run_cmd(f'mask_class {mask2} {real_file} {real_masked} 0 1 1 1 0 0.0') + run_cmd(f'mask_class {mask2} {imag_file} {imag_masked} 0 1 1 1 0 0.0') + + # 精细掩膜后 BMP + run_cmd(f'rasdt_pwr {real_masked} {mli} {width} - - - - ' + f'{roff_min} {roff_max} 0 rmg.cm {real_masked}.bmp - - 24') + run_cmd(f'rasdt_pwr {imag_masked} {mli} {width} - - - - ' + f'{azoff_min} {azoff_max} 0 rmg.cm {imag_masked}.bmp - - 24') + + # --- 5) 空洞填充 --- + real_interp = prefix + '.real.interp' + imag_interp = prefix + '.imag.interp' + run_cmd(f'fill_gaps {real_masked} {width} {real_interp} 0 4 - 1') + run_cmd(f'fill_gaps {imag_masked} {width} {imag_interp} 0 4 - 1') + + # 清理 fill_gaps 插值可能引入的 NaN/Inf/极端值 + sanitize_gamma_float(real_interp, valid_max) + sanitize_gamma_float(imag_interp, valid_max) + + # 填充后 BMP + run_cmd(f'rasdt_pwr {real_interp} {mli} {width} - - - - ' + f'{roff_min} {roff_max} 0 rmg.cm {real_interp}.bmp - - 24') + run_cmd(f'rasdt_pwr {imag_interp} {mli} {width} - - - - ' + f'{azoff_min} {azoff_max} 0 rmg.cm {imag_interp}.bmp - - 24') + + # --- 6) 空间滤波 + 组合 conditioned --- + real_fspf = prefix + '.real.fspf' + imag_fspf = prefix + '.imag.fspf' + run_cmd(f'fspf {real_interp} {real_fspf} {width} 2 2 2') + run_cmd(f'fspf {imag_interp} {imag_fspf} {width} 2 2 2') + + condi = prefix + '.condi' + run_cmd(f'real_to_cpx {real_fspf} {imag_fspf} {condi} {width} 0') + + return condi, real_interp, imag_interp + + +def cmdLineParse(): + parser = argparse.ArgumentParser( + description='Pixel Offset Tracking for a single pair using GAMMA.', + formatter_class=argparse.RawTextHelpFormatter, + epilog=INTRODUCTION + '\n' + EXAMPLE) + + parser.add_argument('projectName', help='projectName for processing.') + parser.add_argument('Mdate', help='Master date.') + parser.add_argument('Sdate', help='Slave date.') + + inps = parser.parse_args() + return inps + + +def main(argv): + + start_time = time.time() + inps = cmdLineParse() + Mdate = inps.Mdate + Sdate = inps.Sdate + projectName = inps.projectName + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + '/' + projectName + '.template' + templateDict = ut.update_template(templateFile) + + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + + # ========== POT 参数 ========== + pot_rstep = templateDict['pot_rstep'] + pot_azstep = templateDict['pot_azstep'] + pot_rwin = templateDict['pot_rwin'] + pot_azwin = templateDict['pot_azwin'] + pot_rwin2 = templateDict['pot_rwin2'] + pot_azwin2 = templateDict['pot_azwin2'] + pot_ovr = templateDict['pot_ovr'] + pot_snr_thresh = templateDict['pot_snr_thresh'] + pot_ccp_thresh = templateDict['pot_ccp_thresh'] + pot_roff_min = templateDict['pot_roff_min'] + pot_roff_max = templateDict['pot_roff_max'] + pot_azoff_min = templateDict['pot_azoff_min'] + pot_azoff_max = templateDict['pot_azoff_max'] + pot_drange_thresh = templateDict['pot_drange_thresh'] + pot_dazimuth_thresh = templateDict['pot_dazimuth_thresh'] + pot_median_win = templateDict['pot_median_win'] + pot_median_nmin = templateDict['pot_median_nmin'] + pot_two_rounds = templateDict['pot_two_rounds'] + pot_geocode = templateDict['pot_geocode'] + pot_disp_max = templateDict['pot_disp_max'] + + # ========== 目录 ========== + projectDir = scratchDir + '/' + projectName + rslcDir = projectDir + '/RSLC' + demDir = projectDir + '/DEM' + potDir = projectDir + '/offsets' + if not os.path.isdir(potDir): + os.mkdir(potDir) + + Pair = Mdate + '-' + Sdate + workDir = potDir + '/' + Pair + if not os.path.isdir(workDir): + os.mkdir(workDir) + + # ========== 输入文件 ========== + Mrslc = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc' + MrslcPar = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc.par' + Srslc = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc' + SrslcPar = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc.par' + + slc_width = ut.read_gamma_par(MrslcPar, 'read', 'range_samples') + off_width = str(int(int(slc_width) // int(pot_rstep))) + + print('=' * 60) + print(f'Pixel Offset Tracking (POT): {Pair}') + print(f' SLC width : {slc_width}') + print(f' Offset width : {off_width}') + print(f' Step (r x az) : {pot_rstep} x {pot_azstep}') + print(f' R1 window : {pot_rwin} x {pot_azwin}') + if pot_two_rounds == '1': + print(f' R2 window : {pot_rwin2} x {pot_azwin2}') + print(f' Offset range : [{pot_roff_min}, {pot_roff_max}] r ' + f'[{pot_azoff_min}, {pot_azoff_max}] az') + print('=' * 60) + + ####################################################################### + # Step 1: 生成偏移量几何下的 MLI（背景图 + 尺寸参考） + ####################################################################### + print('\n[Step 1] 生成偏移量几何 MLI ...') + MLI_pot = workDir + '/' + Mdate + '.mli_pot' + MLI_pot_par = workDir + '/' + Mdate + '.mli_pot.par' + + run_cmd(f'multi_look {Mrslc} {MrslcPar} {MLI_pot} {MLI_pot_par} ' + f'{pot_rstep} {pot_azstep}') + run_cmd(f'raspwr {MLI_pot} {off_width} - - - - 1. .2 - {MLI_pot}.bmp') + + ####################################################################### + # Step 2: 创建偏移量参数文件 + ####################################################################### + print('\n[Step 2] 创建偏移量参数文件 ...') + OFF = workDir + '/' + Pair + '.off' + run_cmd(f'create_offset {MrslcPar} {SrslcPar} {OFF} 1 {rlks} {azlks} 0') + + ####################################################################### + # Step 3: Round 1 — 初始偏移量估计（大窗口） + ####################################################################### + print(f'\n[Step 3] Round 1 偏移量估计 ({pot_rwin}x{pot_azwin}) ...') + r1_tag = f'{pot_rwin}x{pot_azwin}' + r1_prefix = workDir + '/' + Pair + '.offs' + r1_tag + offs_r1 = r1_prefix + ccp_r1 = workDir + '/' + Pair + '.ccp' + r1_tag + + run_cmd(f'offset_pwr_tracking {Mrslc} {Srslc} {MrslcPar} {SrslcPar} ' + f'{OFF} {offs_r1} {ccp_r1} ' + f'{pot_rwin} {pot_azwin} - {pot_ovr} {pot_snr_thresh} ' + f'{pot_rstep} {pot_azstep}') + + ####################################################################### + # Step 4: Round 1 后处理 + ####################################################################### + print(f'\n[Step 4] Round 1 后处理 ...') + r1_condi, _, _ = postprocess_offsets( + offs_cpx=offs_r1, ccp=ccp_r1, mli=MLI_pot, width=off_width, + ccp_thresh=pot_ccp_thresh, + roff_min=pot_roff_min, roff_max=pot_roff_max, + azoff_min=pot_azoff_min, azoff_max=pot_azoff_max, + drange_thresh=pot_drange_thresh, dazimuth_thresh=pot_dazimuth_thresh, + median_win=pot_median_win, median_nmin=pot_median_nmin, + prefix=r1_prefix) + + # 默认使用 Round 1 结果 + final_prefix = r1_prefix + final_ccp = ccp_r1 + final_off = OFF + + ####################################################################### + # Step 5-6: Round 2 — 精细偏移量估计（小窗口，可选） + ####################################################################### + if pot_two_rounds == '1': + print(f'\n[Step 5] Round 2 偏移量估计 ({pot_rwin2}x{pot_azwin2}) ...') + OFF2 = workDir + '/' + Pair + '.off2' + run_cmd(f'create_offset {MrslcPar} {SrslcPar} {OFF2} 1 {rlks} {azlks} 0') + + r2_tag = f'{pot_rwin2}x{pot_azwin2}' + r2_prefix = workDir + '/' + Pair + '.offs' + r2_tag + offs_r2 = r2_prefix + ccp_r2 = workDir + '/' + Pair + '.ccp' + r2_tag + + run_cmd(f'offset_pwr_tracking2 {Mrslc} {Srslc} {MrslcPar} {SrslcPar} ' + f'{OFF2} {offs_r2} {ccp_r2} {OFF} {r1_condi} ' + f'{pot_rwin2} {pot_azwin2} - {pot_ovr} {pot_snr_thresh} ' + f'{pot_rstep} {pot_azstep}') + + print(f'\n[Step 6] Round 2 后处理 ...') + _, _, _ = postprocess_offsets( + offs_cpx=offs_r2, ccp=ccp_r2, mli=MLI_pot, width=off_width, + ccp_thresh=pot_ccp_thresh, + roff_min=pot_roff_min, roff_max=pot_roff_max, + azoff_min=pot_azoff_min, azoff_max=pot_azoff_max, + drange_thresh=pot_drange_thresh, dazimuth_thresh=pot_dazimuth_thresh, + median_win=pot_median_win, median_nmin=pot_median_nmin, + prefix=r2_prefix) + + final_prefix = r2_prefix + final_ccp = ccp_r2 + final_off = OFF2 + + ####################################################################### + # Step 7: 像素偏移量 → 米制位移量 + ####################################################################### + print('\n[Step 7] 像素偏移量转换为地面位移 (米) ...') + + # 用填充后（非 fspf 平滑）的偏移量组合复数，供 offset_tracking 使用 + final_real_interp = final_prefix + '.real.interp' + final_imag_interp = final_prefix + '.imag.interp' + final_offs_combined = final_prefix + '.offs_combined' + run_cmd(f'real_to_cpx {final_real_interp} {final_imag_interp} ' + f'{final_offs_combined} {off_width} 0') + + disp_map = workDir + '/' + Pair + '.disp_map' + run_cmd(f'offset_tracking {final_offs_combined} {final_ccp} ' + f'{MrslcPar} {final_off} {disp_map} - 2 {pot_ccp_thresh} 0') + + # 提取位移分量 + disp_real = disp_map + '.real' # 地距向位移 (米) + disp_imag = disp_map + '.imag' # 方位向位移 (米) + disp_mag = disp_map + '.mag' # 位移幅值 (米) + + run_cmd(f'cpx_to_real {disp_map} {disp_real} {off_width} 0') + run_cmd(f'cpx_to_real {disp_map} {disp_imag} {off_width} 1') + run_cmd(f'cpx_to_real {disp_map} {disp_mag} {off_width} 3') + + # 清理 offset_tracking 输出中的 NaN/Inf/极端值 + disp_max_m = float(pot_disp_max) * 10 + sanitize_gamma_float(disp_real, disp_max_m) + sanitize_gamma_float(disp_imag, disp_max_m) + sanitize_gamma_float(disp_mag, disp_max_m) + + # BMP 可视化 + run_cmd(f'rasdt_pwr {disp_real} {MLI_pot} {off_width} - - - - ' + f'-{pot_disp_max} {pot_disp_max} 1 rmg.cm {disp_real}.bmp - - 24') + run_cmd(f'rasdt_pwr {disp_imag} {MLI_pot} {off_width} - - - - ' + f'-{pot_disp_max} {pot_disp_max} 1 rmg.cm {disp_imag}.bmp - - 24') + run_cmd(f'rasdt_pwr {disp_mag} {MLI_pot} {off_width} - - - - ' + f'-{pot_disp_max} {pot_disp_max} 1 rmg.cm {disp_mag}.bmp - - 24') + + # 注意: 地理编码已移至 geocode_gamma.py --type pot + # 用法: geocode_gamma.py projectName Pair --type pot + + print(f"\nPixel Offset Tracking for {Pair} is done!") + ut.print_process_time(start_time, time.time()) + sys.exit(0) + + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/POT_gamma_all.py b/third_party/PyINT/pyint/POT_gamma_all.py new file mode 100644 index 0000000..07c365c --- /dev/null +++ b/third_party/PyINT/pyint/POT_gamma_all.py @@ -0,0 +1,148 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Batch Pixel Offset Tracking (POT) using GAMMA ### +### Author: ZYD / Cascade AI ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + + +def work(data0): + cmd = data0[0] + err_file = data0[1] + p = subprocess.run(cmd, shell=False, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa = stderr.decode("utf-8") + else: + aa = stderr + + if aa: + str0 = ' '.join(cmd) + '\n' + with open(err_file, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return + +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + Batch Pixel Offset Tracking for one project using GAMMA. + + Runs POT_gamma.py for each interferometric pair in parallel. + Skips pairs that already have completed results (disp_map.mag.bmp). +''' + +EXAMPLE = ''' + Usage: + POT_gamma_all.py projectName + POT_gamma_all.py projectName --parallel 4 + POT_gamma_all.py projectName --parallel 4 --ifgarmList-txt /test/ifgram_list.txt +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser( + description='Batch Pixel Offset Tracking for one project using GAMMA.', + formatter_class=argparse.RawTextHelpFormatter, + epilog=INTRODUCTION + '\n' + EXAMPLE) + + parser.add_argument('projectName', help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, + help='Enable parallel processing and specify the number of processors.') + parser.add_argument('--ifgarmList-txt', dest='ifgarmListTxt', + help='Provided ifgram_list_txt. Default: using ifgram_list.txt under projectName folder.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + projectDir = scratchDir + '/' + projectName + potDir = projectDir + '/offsets' + if not os.path.isdir(potDir): + os.mkdir(potDir) + + templateDict = ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + if inps.ifgarmListTxt: + ifgramList_txt = inps.ifgarmListTxt + else: + ifgramList_txt = projectDir + '/ifgram_list.txt' + + ifgList0 = ut.read_txt2array(ifgramList_txt) + + if len(ifgList0) == 3: + ifgList = ifgList0[0] + ifgList = [ifgList] + else: + ifgList = ifgList0[:, 0] + + err_txt = projectDir + '/POT_gamma_all.err' + if os.path.isfile(err_txt): + os.remove(err_txt) + + data_para = [] + skip_count = 0 + for i in range(len(ifgList)): + m0 = ut.yyyymmdd(ifgList[i].split('-')[0]) + s0 = ut.yyyymmdd(ifgList[i].split('-')[1]) + cmd0 = ['POT_gamma.py', projectName, m0, s0] + + # 检查完成标志: disp_map.mag.bmp + Pair = ifgList[i] + disp_bmp = potDir + '/' + Pair + '/' + Pair + '.disp_map.mag.bmp' + + k00 = 0 + if os.path.isfile(disp_bmp): + if os.path.getsize(disp_bmp) > 0: + k00 = 1 + skip_count += 1 + + if k00 == 0: + data0 = [cmd0, err_txt] + data_para.append(data0) + + total = len(ifgList) + todo = len(data_para) + print(f'Pixel Offset Tracking: {total} pairs total, ' + f'{skip_count} already done, {todo} to process') + print(f'Parallel processors: {inps.parallelNumb}') + print(f'Output directory: {potDir}') + print('=' * 60) + + if todo > 0: + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + + print("Pixel Offset Tracking for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(0) + + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/README.md b/third_party/PyINT/pyint/README.md new file mode 100644 index 0000000..1995412 --- /dev/null +++ b/third_party/PyINT/pyint/README.md @@ -0,0 +1,26 @@ +## GACOS CORRECTION FOR GMTSAR +These software is intended to correct unwrapped phases using GACOS grids for GMTSAR. + +They have been tested with: GMT v6.1.0 GMTSAR v6.0 + +On "Test_against_matlab" folder, I perform comparisons with matlab with consistent results. +To reproduce the output just use: source command.txt in each folder. +There is a small README file in each folder + +### SINGLE INTERFEROGRAM +single_GACOS_correction.csh is intented to correct a single interferogram at a time with certain parameters + +### STACK OF INTERFEROGRAMS +GACOS_correction.csh and operation.csh are used to correct a stack of interferograms in a time series process + +Corrections are applied to the unwrap.grd files + +Feel free to use and edit the code if needed + +References: + +*Yu, C., Li, Z., Penna, N. T., & Crippa, P. (2018). Generic atmospheric correction model for Interferometric Synthetic Aperture Radar observations. Journal of Geophysical Research: Solid Earth, 123(10), 9202-9222.* + +*Yu, C., Li, Z., & Penna, N. T. (2018). Interferometric synthetic aperture radar atmospheric correction using a GPS-based iterative tropospheric decomposition model. Remote Sensing of Environment, 204, 109-121.* + +*Yu, C., Penna, N. T., & Li, Z. (2017). Generation of real‐time mode high‐resolution water vapor fields from GPS observations. Journal of Geophysical Research: Atmospheres, 122(3), 2008-2025.* diff --git a/third_party/PyINT/pyint/RSI_SLC_Gamma.py b/third_party/PyINT/pyint/RSI_SLC_Gamma.py new file mode 100644 index 0000000..8dfb52f --- /dev/null +++ b/third_party/PyINT/pyint/RSI_SLC_Gamma.py @@ -0,0 +1,216 @@ +#! /usr/bin/env python +#''' +################################################################################### +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : March , 2017 # +# # +# Range Split Spectrum for SAR complex data based on GAMMA # +# # +################################################################################### +#''' +import numpy as np +import os +import pysar._readfile as readfile +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + + +def ras2jpg(input, strTitle): + call_str = "convert " + input + ".ras " + input + ".jpg" + os.system(call_str) + call_str = "convert " + input + ".jpg -resize 250 " + input + ".thumb.jpg" + os.system(call_str) + call_str = "convert " + input + ".jpg -resize 500 " + input + ".bthumb.jpg" + os.system(call_str) + call_str = "$INT_SCR/addtitle2jpg.pl " + input + ".thumb.jpg 14 " + strTitle + os.system(call_str) + call_str = "$INT_SCR/addtitle2jpg.pl " + input + ".bthumb.jpg 24 " + strTitle + os.system(call_str) + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +def geocode(inFile, outFile, UTMTORDC, nWidth, nWidthUTMDEM, nLineUTMDEM): + if inFile.rsplit('.')[1] == 'int': + call_str = '$GAMMA_BIN/geocode_back ' + inFile + ' ' + nWidth + ' ' + UTMTORDC + ' ' + outFile + ' ' + nWidthUTMDEM + ' ' + nLineUTMDEM + ' 0 1' + else: + call_str = '$GAMMA_BIN/geocode_back ' + inFile + ' ' + nWidth + ' ' + UTMTORDC + ' ' + outFile + ' ' + nWidthUTMDEM + ' ' + nLineUTMDEM + ' 0 0' + os.system(call_str) + + +def createBlankFile(strFile): + f = open(strFile,'w') + for i in range (10): + f.write('\n') + f.close() + + +def usage(): + print(''' +****************************************************************************************************** + + Split Spectrum on range direction to generate high-frequency- and low-frequency- SLCs + + usage: + + RSI_SLC_Gamma.py igramDir + + e.g. RSI_SLC_Gamma.py RSI_PacayaT163TsxHhA_131021-131101_0011_-0007 + + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==2: + if argv[0] in ['-h','--help']: usage(); sys.exit(1) + else: igramDir=sys.argv[1] + else: + usage();sys.exit(1) + + INF = igramDir.split('_')[0] + projectName = igramDir.split('_')[1] + IFGPair = igramDir.split(projectName+'_')[1].split('_')[0] + Mdate = IFGPair.split('-')[0] + Sdate = IFGPair.split('-')[1] + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + + processDir = scratchDir + '/' + projectName + "/PROCESS" + slcDir = scratchDir + '/' + projectName + "/SLC" + workdir = processDir + '/' + igramDir + + templateContents=read_template(templateFile) + rlks = templateContents['Range_Looks'] + azlks = templateContents['Azimuth_Looks'] + + if INF!='RSI': + usage();sys.exit(1) + + if not os.path.isdir(workdir): + call_str = 'mkdir ' + workdir + os.system(call_str) + + SslcDir = slcDir + "/" + Sdate + MslcDir = slcDir + "/" + Mdate + +# input slcs + + SslcDir = slcDir + "/" + Sdate + MslcDir = slcDir + "/" + Mdate + + MslcImg = MslcDir + "/" + Mdate + ".slc" + MslcPar = MslcDir + "/" + Mdate + ".slc.par" + SslcImg = SslcDir + "/" + Sdate + ".slc" + SslcPar = SslcDir + "/" + Sdate + ".slc.par" + + + MHslcImg = workdir + '/'+Mdate + '.HF.slc' + MHslcPar = workdir + '/'+Mdate + '.HF.slc.par' + call_str = 'cp ' + MslcPar + ' '+ MHslcPar + os.system(call_str) + + SHslcImg = workdir + '/'+Sdate + '.HF.slc' + SHslcPar = workdir + '/'+Sdate + '.HF.slc.par' + call_str = 'cp ' + SslcPar + ' '+ SHslcPar + os.system(call_str) + + MLslcImg = workdir + '/'+Mdate + '.LF.slc' + MLslcPar = workdir + '/'+Mdate + '.LF.slc.par' + call_str = 'cp ' + MslcPar + ' '+ MLslcPar + os.system(call_str) + + SLslcImg = workdir + '/'+Sdate + '.LF.slc' + SLslcPar = workdir + '/'+Sdate + '.LF.slc.par' + call_str = 'cp ' + SslcPar + ' '+ SLslcPar + os.system(call_str) + + nWidth = UseGamma(MslcPar, 'read','range_samples:') + + call_str= 'bpf ' + MslcImg + ' ' + MHslcImg + ' ' + nWidth + ' 0.25 0.5 0 1 0 0 - - 1' + os.system(call_str) + + call_str= 'bpf ' + MslcImg + ' ' + MLslcImg + ' ' + nWidth + ' -0.25 0.5 0 1 0 0 - - 1' + os.system(call_str) + + nWidth = UseGamma(SslcPar, 'read','range_samples:') + + call_str= 'bpf ' + SslcImg + ' ' + SHslcImg + ' ' + nWidth + ' 0.25 0.5 0 1 0 0 - - 1' + os.system(call_str) + + call_str= 'bpf ' + SslcImg + ' ' + SLslcImg + ' ' + nWidth + ' -0.25 0.5 0 1 0 0 - - 1' + os.system(call_str) + + + print("Split spectrum for both slave and master date is done!") + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + + + + + + + + + + + diff --git a/third_party/PyINT/pyint/Raw2SLC_ERS_Cat.py b/third_party/PyINT/pyint/Raw2SLC_ERS_Cat.py new file mode 100644 index 0000000..3bd572c --- /dev/null +++ b/third_party/PyINT/pyint/Raw2SLC_ERS_Cat.py @@ -0,0 +1,306 @@ +#! /usr/bin/env python +#''' +################################################################################## +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : June, 2019 # +# # +# Generate SLC from SAR_IMS_P1 data # +# # +################################################################################## +#''' +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close + +def rm(TXT): + call_str = 'rm ' + TXT + os.system(call_str) + +def usage(): + print(''' +****************************************************************************************************** + + Generate SLC and SLC_par file for ERS/ENVISAT (SAR_IMS_1P format) + + usage: + + Down2SLC_ERS.py ProjectName DownName + + e.g. Down2SLC_ERS.py CotopaxiT120ERSA 910101 + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==3: + projectName = sys.argv[1] + Date = sys.argv[2] + else: + usage();sys.exit(1) + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + '/' + projectName + '.template' + templateContents=read_template(templateFile) + + if 'rlks4cor' in templateContents: rlks4cor = templateContents['rlks4cor'] + else: rlks4cor = '4' + if 'azlks4cor' in templateContents: azlks4cor = templateContents['azlks4cor'] + else: azlks4cor = '4' + + if 'rwin4cor' in templateContents: rwin4cor = templateContents['rwin4cor'] + else: rwin4cor = '128' + if 'azwin4cor' in templateContents: azwin4cor = templateContents['azwin4cor'] + else: azwin4cor = '128' + if 'rsample4cor' in templateContents: rsample4cor = templateContents['rsample4cor'] + else: rsample4cor = '64' + if 'azsample4cor' in templateContents: azsample4cor = templateContents['azsample4cor'] + else: azsample4cor = '64' + + if ' rpos4cor' in templateContents: rpos4cor = templateContents[' rpos4cor'] + else: rpos4cor = ' - ' + if 'azpos4cor' in templateContents: azpos4cor = templateContents['azpos4cor'] + else: azpos4cor = ' - ' + + + if 'rfwin4cor' in templateContents: rfwin4cor = templateContents['rfwin4cor'] + else: rfwin4cor = str(int(int(rwin4cor)/2)) + if 'azfwin4cor' in templateContents: azfwin4cor = templateContents['azfwin4cor'] + else: azfwin4cor = str(int(int(azwin4cor)/2)) + if 'rfsample4cor' in templateContents: rfsample4cor = templateContents['rfsample4cor'] + else: rfsample4cor = str(2*int(rsample4cor)) + if 'azfsample4cor' in templateContents: azfsample4cor = templateContents['azfsample4cor'] + else: azfsample4cor = str(2*int(azsample4cor)) + + if 'thresh4cor' in templateContents: thresh4cor = templateContents['thresh4cor'] + else: thresh4cor = ' 0.2 ' + + rlks = templateContents['Range_Looks'] + azlks = templateContents['Azimuth_Looks'] + + + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + slcDir = scratchDir + '/' + projectName + '/SLC' + + if not os.path.isdir(slcDir): + call_str= 'mkdir ' +slcDir + os.system(call_str) + os.chdir(downDir) + + t0 = 't0_' + Date + call_str = 'ls >' + t0 + os.system(call_str) + + tt = 'tt_' + Date + call_str = "grep " + Date + ' ' + t0 + '> ' + tt + os.system(call_str) + + te = 'te_' + Date + call_str = "grep SAR_IM " + tt + " > " + te + os.system(call_str) + + AA= np.loadtxt(te,dtype=np.str) + Na = AA.size + AA=AA.reshape(Na,) + + for i in range(Na): + Date0 = Date+'_' + str(i) + downName = str(AA[i]) + FileDir = downDir + '/' + downName + call_str = 'raw2slc_ers_envisat.py '+ FileDir + ' -o ' + Date0 + os.system(call_str) + + #call_str ="rename 's/VV.SLC/slc/g' *" + #os.system(call_str) + + #slcpar = Date0 + '.slc.par' + #call_str = 'ERS_orb_cor_par.py ' + slcpar + #os.system(call_str) + + Date0 = Date + if len(Date)==6: + Date6 = Date + Date0 = Date + elif len(Date)==8: + Date0 = Date[2:8] + Date6 = Date[2:8] + else: + print('The input Date is invalid.') + sys.exit(1) + + dataDir = slcDir + '/' + Date0 + if not os.path.isdir(dataDir): + call_str = 'mkdir ' + dataDir + print('Generate SLC dir for date: ' + Date0) + os.system(call_str) + call_str = 'mv ' + Date + '*.slc* ' + dataDir + os.system(call_str) + + os.chdir(dataDir) + for i in range(Na): + Date0 = Date+'_' + str(i) + downName = str(AA[i]) + + SLCm = Date0 + '.slc' + SLCm_par = Date0 + '.slc.par' + MamprlksImg = Date0 + '.amp' + MamprlksPar = Date0 + '.amp.par' + + call_str = 'multi_look ' + SLCm + ' ' + SLCm_par + ' ' + MamprlksImg + ' ' + MamprlksPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + nWidth = UseGamma(MamprlksPar, 'read', 'range_samples') + call_str = 'raspwr ' + MamprlksImg + ' ' + nWidth + os.system(call_str) + + SLC = Date6+'.slc' + SLCPAR = Date6 + '.slc.par' + AMP = Date6+'.amp' + AMPPAR = Date6+'.amp.par' + if Na==1: + os.rename(SLCm,SLC) + os.rename(SLCm_par,SLCPAR) + os.rename(MamprlksImg,AMP) + os.rename(MamprlksPar,AMPPAR) + + for i in range(Na-1): + if i==0: + DateA = Date + '_0' + else: + DateA = Date + '_' + str(i-1) + str(i) + + SLCA = DateA + '.slc' + SLCA_par = DateA + '.slc.par' + + + DateB = Date + '_' +str(i+1) + SLCB = DateB + '.slc' + SLCB_par = DateB + '.slc.par' + + DateC = Date + '_' + str(i) + str(i+1) + SLCC = DateC + '.slc' + SLCC_par = DateC + '.slc.par' + + ##################################################################################### + MamprlksImg = Date + '.amp' + MamprlksPar = Date + '.amp.par' + + off = DateC + '.off' + offs = DateC + '.offs' + offsets = DateC + '.offsets' + coffs = DateC + '.coffs' + coffsets = DateC + '.coffsets' + snr = DateC + '.snr' + off_std = DateC + '.off_std' + + ########################## Generate off file ############################# + + call_str = "create_offset " + SLCA_par + " " + SLCB_par + " " + off + " 1 - - 0" + os.system(call_str) + call_str = 'init_offset_orbit '+ SLCA_par + " " + SLCB_par + ' ' + off + os.system(call_str) + + + call_str = 'init_offset '+ SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' ' + rlks4cor + ' ' + azlks4cor + ' ' + rpos4cor + ' ' + azpos4cor + os.system(call_str) + + call_str = 'init_offset '+ SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' 1 1 - - ' + os.system(call_str) + + call_str = "offset_pwr " + SLCA + " " + SLCB + " " + SLCA_par + " " + SLCB_par + " " + off + " " + offs + " " + snr + " " + rwin4cor + " " + azwin4cor + " " + offsets + " 2 "+ rsample4cor + " " + azsample4cor + os.system(call_str) + + call_str = "offset_fit " + offs + " " + snr + " " + off + " " + coffs + " " + coffsets + " " + thresh4cor +" 3" + os.system(call_str) + + call_str = "offset_pwr " +SLCA + " " + SLCB + " " + SLCA_par + " " + SLCB_par + " " + off + " " + offs + " " + snr + " " + rfwin4cor + " " + azfwin4cor + " " + offsets + " 2 " + rfsample4cor + " " + azfsample4cor + os.system(call_str) + + call_str = "offset_fit " + offs + " " + snr + " " + off + " " + coffs + " " + coffsets + " " + thresh4cor + " 4 >" + off_std + os.system(call_str) + + ######################################################################################## + + call_str = 'SLC_cat ' + SLCA + ' ' + SLCB + ' ' + SLCA_par + ' ' + SLCB_par + ' ' + off + ' ' + SLCC + ' ' + SLCC_par + os.system(call_str) + + if i==(Na-2): + if len(Date)==6: + DD = Date + else: + DD = Date[2:8] + SLCm = DD + '.slc' + SLCm_par = DD + '.slc.par' + + + call_str = 'cp ' + SLCC + ' ' + SLCm + os.system(call_str) + + call_str = 'cp ' + SLCC_par + ' ' + SLCm_par + os.system(call_str) + + call_str = 'multi_look ' + SLCm + ' ' + SLCm_par + ' ' + MamprlksImg + ' ' + MamprlksPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + nWidth = UseGamma(MamprlksPar, 'read', 'range_samples') + call_str = 'raspwr ' + MamprlksImg + ' ' + nWidth + os.system(call_str) + + call_str = 'rm *.amp' + os.system(call_str) + + call_str = 'rm *_*.slc' + os.system(call_str) + + print("Down to SLC for %s is done! " % Date) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/Raw2SLC_ERS_Cat_All.py b/third_party/PyINT/pyint/Raw2SLC_ERS_Cat_All.py new file mode 100644 index 0000000..4586dc9 --- /dev/null +++ b/third_party/PyINT/pyint/Raw2SLC_ERS_Cat_All.py @@ -0,0 +1,136 @@ +#! /usr/bin/env python +#''' +################################################################################## +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : March, 2017 # +# # +# Generate Sentinel SLC from the downloaded data # +# # +################################################################################## +#''' +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close + +def rm(TXT): + call_str = 'rm ' + TXT + os.system(call_str) + +def usage(): + print(''' +****************************************************************************************************** + + Cat ERS SLC from ERS raw data with ENVISAT format. + + usage: + + Raw2SLC_ERS_Cat_All.py ProjectName + + e.g. Raw2SLC_ERS_Cat_All.py CotopaxiT120ERSA + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==2: + projectName = sys.argv[1] + else: + usage();sys.exit(1) + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + slcDir = scratchDir + '/' + projectName + '/SLC' + + if not os.path.isdir(slcDir): + call_str= 'mkdir ' +slcDir + os.system(call_str) + + os.chdir(downDir) + + call_str = 'ls > tt0' + os.system(call_str) + + call_str = 'grep SAR_IM tt0 >tt1' + os.system(call_str) + + call_str = "awk -F0PWDSI '{print $2}' tt1 > tt2 " + os.system(call_str) + + call_str = "awk -F_ '{print $1}' tt2 > ttt" + os.system(call_str) + + call_str = 'sort ttt | uniq > ttm' + os.system(call_str) + + AA= np.loadtxt('ttm',dtype=np.str) + Na = AA.size + + for i in range(Na): + call_str = 'Raw2SLC_ERS_Cat.py ' + projectName + ' ' + AA[i] + print(call_str) + call_str = 'Raw2SLC_ERS_Cat.py ' + projectName + ' ' + AA[i] + ' >/dev/null' + os.system(call_str) + + + print("Down to SLC for %s is done! " % projectName) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/third_party/PyINT/pyint/SAR2LATLON.py b/third_party/PyINT/pyint/SAR2LATLON.py new file mode 100644 index 0000000..35a6c3b --- /dev/null +++ b/third_party/PyINT/pyint/SAR2LATLON.py @@ -0,0 +1,144 @@ +#! /usr/bin/env python + +#''' +################################################################################## +### Author: Yun-Meng Cao ### +### Date : March, 2017 ### +### Email : ymcmrs@gmail.com ### +### Transform SAR coordinates into LAT/LON coordinates based on lookup table ### +################################################################################## +#''' + +import os +import sys +import glob +import time +import argparse + +import h5py +import numpy as np + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def read_data(inFile, dtype, nWidth, nLength): + data = np.fromfile(inFile, dtype, int(nLength)*int(nWidth)).reshape(int(nLength),int(nWidth)) + + return data + +def is_number(s): + try: + int(s) + return True + except ValueError: + return False + +def add_zero(s): + if len(s)==1: + s="000"+s + elif len(s)==2: + s="00"+s + elif len(s)==3: + s="0"+s + return s + + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + + +def usage(): + print(''' +****************************************************************************************************** + + Transform SAR coordinates into LAT/LON coordinates based on lookup table + + usage: + + SAR2LATLON.py Range Azimuth LookupTableFile UTMDEMpar + + e.g. SAR2LATLON.py 1500 1000 /Yunmeng/20201230.utm_to_rdc /Yunmeng/20201230.dem.utm.par + + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==5: + Range = sys.argv[1] + Azimuth = sys.argv[2] + LtFile = sys.argv[3] + UTMPAR = sys.argv[4] + else: + usage();sys.exit(1) + + nWidthUTM = UseGamma(UTMPAR, 'read', 'width:') + nLineUTM = UseGamma(UTMPAR, 'read', 'nlines:') + + Corner_LAT = UseGamma(UTMPAR, 'read', 'corner_lat:') + Corner_LON = UseGamma(UTMPAR, 'read', 'corner_lon:') + + Corner_LAT =Corner_LAT.split(' ')[0] + Corner_LON =Corner_LON.split(' ')[0] + + post_Lat = UseGamma(UTMPAR, 'read', 'post_lat:') + post_Lon = UseGamma(UTMPAR, 'read', 'post_lon:') + + post_Lat =post_Lat.split(' ')[0] + post_Lon =post_Lon.split(' ')[0] + data = read_data(LtFile,'>c8',nWidthUTM,nLineUTM) # real: range imaginary: azimuth + + Range_LT = data.real + Azimuth_LT = data.imag + + CPX_input =complex(Range + '+' + Azimuth+'j') + + DD = abs(CPX_input - data) + + LL= abs(DD) + IDX= np.where(LL == LL.min()) + Lat_out = float(Corner_LAT) + IDX[0]*float(post_Lat) + Lon_out = float(Corner_LON) + IDX[1]*float(post_Lon) + + print(' Range: ' + Range + ' ' + 'Azimuth: ' + Azimuth) + print(' Latitude: ' + str(Lat_out[0]) + ' ' + 'Longitude: ' + str(Lon_out[0])) + +############################################################################## +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/third_party/PyINT/pyint/SRTM_AUTO_DOWNLOAD.md b/third_party/PyINT/pyint/SRTM_AUTO_DOWNLOAD.md new file mode 100644 index 0000000..439e3b4 --- /dev/null +++ b/third_party/PyINT/pyint/SRTM_AUTO_DOWNLOAD.md @@ -0,0 +1,150 @@ +# SRTM 自动下载功能说明 + +## 功能概述 + +`makedem.py` 现在支持自动下载SRTM数据,无需手动准备数据文件! + +## 使用方法 + +### 方法1: 自动下载 (推荐) + +**步骤1: 获取OpenTopography API key** (免费) + +访问: https://opentopography.org/myOpenTopo +- 注册免费账户 +- 在 'My Account' 页面请求API key +- API key立即可用,免费配额足够大部分用途 + +**步骤2: 运行命令** + +```bash +# 使用API key自动下载SRTM数据 +python makedem.py -r 116/117/39/40 --dem-source srtm --opentopo-api-key YOUR_API_KEY + +# 或设置环境变量 +export OPENTOPO_API_KEY="your_api_key_here" +python makedem.py -r 116/117/39/40 --dem-source srtm +``` + +### 方法2: 使用预下载的.hgt文件 + +如果已有SRTM .hgt文件: + +```bash +python makedem.py -r 116/117/39/40 --dem-source srtm --srtm-data-dir /path/to/srtm_data +``` + +## SRTM数据规格 + +- **分辨率**: 90米 (SRTM GL3) +- **覆盖**: 全球 (60°S - 60°N) +- **数据源**: OpenTopography API +- **格式**: GeoTIFF +- **费用**: 免费 (需要免费API key) + +## 对比 + +| 方法 | 优点 | 缺点 | +|------|------|------| +| 自动下载 | ✓ 全自动<br>✓ 无需准备数据<br>✓ 快速便捷 | ✗ 需要API key<br>✗ 需要网络 | +| 预下载文件 | ✓ 离线使用<br>✓ 不依赖API | ✗ 需要手动下载<br>✗ 需要管理文件 | + +## 示例 + +### 北京地区 (116-117°E, 39-40°N) + +```bash +# 自动下载 +python makedem.py -r 116/117/39/40 --dem-source srtm --opentopo-api-key YOUR_KEY + +# 输出文件: SRTM_116_117_39_40.tif +``` + +### 使用SLC参数文件 + +```bash +# 自动确定区域范围 +python makedem.py -s /path/to/slc.par --dem-source srtm --opentopo-api-key YOUR_KEY +``` + +## 常见问题 + +**Q: 为什么需要API key?** + +A: OpenTopography要求API key来管理数据访问配额。免费API key配额足够大部分科研和教育用途。 + +**Q: 如何获取API key?** + +A: +1. 访问 https://opentopography.org/myOpenTopo +2. 注册账户 (免费) +3. 在账户页面点击 "Request API Key" +4. API key会立即显示,可以直接使用 + +**Q: 免费API key有什么限制?** + +A: +- SRTM GL3: 最大4,050,000 km² per request +- 对于大部分研究区域完全够用 +- 如果需要更大范围,可以分块下载 + +**Q: SRTM vs Copernicus vs NASADEM?** + +A: +- **Copernicus (推荐)**: 30m分辨率,全球覆盖,全自动下载 +- **NASADEM**: 30m分辨率,NASA官方,60°S-60°N覆盖 +- **SRTM**: 90m分辨率,历史数据,适合对比研究 + +**Q: 没有.hgt文件怎么办?** + +A: 直接使用方法1自动下载,无需准备任何数据文件! + +## 故障排除 + +### 错误: HTTP 401 + +**原因**: API key无效或已过期 + +**解决**: +- 检查API key是否正确 +- 确认账户状态正常 +- 如需要,重新生成API key + +### 错误: HTTP 403 + +**原因**: 超出配额或访问限制 + +**解决**: +- 检查请求区域大小 +- 分块下载大区域 +- 确认API key配额 + +### 错误: 文件下载失败 + +**原因**: 网络问题 + +**解决**: +- 检查网络连接 +- 重试命令 +- 使用预下载方法 + +## Python版本兼容性 + +- **方法1 (自动下载)**: 支持所有Python版本 +- **方法2 (使用.hgt文件)**: + - Python >= 3.12: 使用srtm库自动查询 + - Python < 3.12: 使用GDAL手动处理 + +两种方法结果相同,只是处理方式不同。 + +## 相关资源 + +- OpenTopography: https://opentopography.org/ +- API文档: https://portal.opentopography.org/apidocs/ +- 获取API key: https://opentopography.org/myOpenTopo +- SRTM数据介绍: https://www2.jpl.nasa.gov/srtm/ + +--- + +**更新日期**: 2026-03-13 +**功能版本**: v1.0 diff --git a/third_party/PyINT/pyint/UTM2SARPIX.py b/third_party/PyINT/pyint/UTM2SARPIX.py new file mode 100644 index 0000000..46fc6b8 --- /dev/null +++ b/third_party/PyINT/pyint/UTM2SARPIX.py @@ -0,0 +1,128 @@ +#! /usr/bin/env python +#''' +################################################################################### +# # +# Author: Yun-Meng Cao # +# Email : ymcmrs@gmail.com # +# Date : March, 2017 # +# # +# Transform lat and lon to SAR coordinates based on SLCPar and DEM # +# # +################################################################################### +#''' + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + + +def usage(): + print(''' +****************************************************************************************************** + + Transform lat and lon to SAR coordinates based on SLCPar and DEM + + usage: + + UTM2SARPIX.py latitude longitude SLCPar DEM + + e.g. UTM2SARPIX.py 31.1 -108.2 /Yunmeng/2010.slc.par /Yunmeng/2010.dem + +******************************************************************************************************* + ''') + + +def main(argv): + + if len(sys.argv)==5: + LAT = sys.argv[1] + LON = sys.argv[2] + PAR = sys.argv[3] + DEM = sys.argv[4] + else: + usage();sys.exit(1) + + DEMpar = DEM + '.par' + + DateFormat = UseGamma(DEMpar, 'read', 'data_format:') + + nWidth = UseGamma(DEMpar, 'read', 'width:') + nLength = UseGamma(DEMpar, 'read', 'nlines:') + + Corner_LAT = UseGamma(DEMpar, 'read', 'corner_lat:') + Corner_LON = UseGamma(DEMpar, 'read', 'corner_lon:') + + Corner_LAT =Corner_LAT.split(' ')[0] + Corner_LON =Corner_LON.split(' ')[0] + + post_Lat = UseGamma(DEMpar, 'read', 'post_lat:') + post_Lon = UseGamma(DEMpar, 'read', 'post_lon:') + + post_Lat =post_Lat.split(' ')[0] + post_Lon =post_Lon.split(' ')[0] + + if DateFormat =='INTEGER*2': + STR = '>i2' + else: + STR = '>f4' + + TXT = 'SARCOORD' + + DEMdate = np.fromfile(DEM,STR,int(nLength)*int(nWidth)).reshape(int(nLength),int(nWidth)) + + LAT = float(LAT); LON =float(LON) + nWidth=int(nWidth);nLength=int(nLength) + Corner_LAT = float(Corner_LAT); Corner_LON=float(Corner_LON) + post_Lat = float(post_Lat); post_Lon=float(post_Lon) + + XX = int (( LAT - Corner_LAT ) / post_Lat) # latitude width range + YY = int (( LON - Corner_LON ) / post_Lon) # longitude nline azimuth + + + ELEV = DEMdate[XX][YY] + + call_str = 'coord_to_sarpix ' + PAR + ' - - ' + str(LAT) + ' ' + str(LON) + ' ' + str(ELEV) + ' >' +TXT + os.system(call_str) + + call_str = 'cat ' + TXT + os.system(call_str) + + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + + + + + + + + + diff --git a/third_party/PyINT/pyint/_network.py b/third_party/PyINT/pyint/_network.py new file mode 100644 index 0000000..4a3bf94 --- /dev/null +++ b/third_party/PyINT/pyint/_network.py @@ -0,0 +1,483 @@ +############################################################ +# Program is part of PyINT # +# Copyright 2019 Yunmeng Cao # +# Contact: ymcmrs@gmail.com # +############################################################ +# This program is modified from MintPy/select_network.py +# Copyright: 2017-2019, Yunjun Zhang +# Contact: yzhang@rsmas.miami.edu + +import os +import sys +import itertools +import h5py +import numpy as np +from scipy import sparse +from matplotlib import (colors, + dates as mdates, + pyplot as plt) +from matplotlib.tri import Triangulation +from mpl_toolkits.axes_grid1 import make_axes_locatable + + + + +################################################################## +def read_pairs_list(date12ListFile, dateList=[]): + """Read Pairs List file like below: + 070311-070426 + 070311-070611 + ... + """ + # Read date12 list file + date12List = sorted(list(np.loadtxt(date12ListFile, dtype=bytes).astype(str))) + + # Get dateList from date12List + if not dateList: + dateList = [] + for date12 in date12List: + dates = date12.split('-') + if not dates[0] in dateList: + dateList.append(dates[0]) + if not dates[1] in dateList: + dateList.append(dates[1]) + dateList.sort() + date6List = yymmdd(dateList) + + # Get pair index + pairs_idx = [] + for date12 in date12List: + dates = date12.split('-') + pair_idx = [date6List.index(dates[0]), date6List.index(dates[1])] + pairs_idx.append(pair_idx) + + return pairs_idx + +def get_date12_list(fname, dropIfgram=False): + """Read Date12 info from input file: Pairs.list or multi-group hdf5 file + Inputs: + fname - string, path/name of input multi-group hdf5 file or text file + dropIfgram - bool, check the "DROP_IFGRAM" attribute or not for multi-group hdf5 file + Output: + date12_list - list of string in YYMMDD-YYMMDD format + Example: + date12List = get_date12_list('ifgramStack.h5') + date12List = get_date12_list('ifgramStack.h5', dropIfgram=True) + date12List = get_date12_list('Pairs.list') + """ + date12_list = [] + ext = os.path.splitext(fname)[1].lower() + if ext == '.h5': + k = readfile.read_attribute(fname)['FILE_TYPE'] + if k == 'ifgramStack': + date12_list = ifgramStack(fname).get_date12_list(dropIfgram=dropIfgram) + else: + return None + else: + txtContent = np.loadtxt(fname, dtype=bytes).astype(str) + if len(txtContent.shape) == 1: + txtContent = txtContent.reshape(-1, 1) + date12_list = [i for i in txtContent[:, 0]] + date12_list = sorted(date12_list) + return date12_list + + +def igram_perp_baseline_list(fname): + """Get perpendicular baseline list from input multi_group hdf5 file""" + print(('read perp baseline info from '+fname)) + k = readfile.read_attribute(fname)['FILE_TYPE'] + h5 = h5py.File(fname, 'r') + epochList = sorted(h5[k].keys()) + p_baseline_list = [] + for epoch in epochList: + p_baseline = (float(h5[k][epoch].attrs['P_BASELINE_BOTTOM_HDR']) + + float(h5[k][epoch].attrs['P_BASELINE_TOP_HDR']))/2 + p_baseline_list.append(p_baseline) + h5.close() + return p_baseline_list + + +def select_pairs_sbas(date_list, date12_format='YYMMDD-YYMMDD'): + """Select All Possible Pairs/Interferograms + Input : date_list - list of date in YYMMDD/YYYYMMDD format + Output: date12_list - list date12 in YYMMDD-YYMMDD format + Reference: + Berardino, P., G. Fornaro, R. Lanari, and E. Sansosti (2002), A new algorithm for surface deformation monitoring + based on small baseline differential SAR interferograms, IEEE TGRS, 40(11), 2375-2383. + """ + date8_list = sorted(yyyymmdd(date_list)) + date6_list = yymmdd(date8_list) + date12_list = list(itertools.combinations(date6_list, 2)) + date12_list = [date12[0]+'-'+date12[1] for date12 in date12_list] + if date12_format == 'YYYYMMDD_YYYYMMDD': + date12_list = yyyymmdd_date12(date12_list) + return date12_list + + +def select_pairs_sequential(date_list, num_connection=2, date12_format='YYMMDD-YYMMDD'): + """Select Pairs in a Sequential way: + For each acquisition, find its num_connection nearest acquisitions in the past time. + Inputs: + date_list : list of date in YYMMDD/YYYYMMDD format + Reference: + Fattahi, H., and F. Amelung (2013), DEM Error Correction in InSAR Time Series, IEEE TGRS, 51(7), 4249-4259. + """ + date8_list = sorted(yyyymmdd(date_list)) + date6_list = yymmdd(date8_list) + date_idx_list = list(range(len(date6_list))) + + # Get pairs index list + date12_idx_list = [] + for date_idx in date_idx_list: + for i in range(num_connection): + if date_idx-i-1 >= 0: + date12_idx_list.append([date_idx-i-1, date_idx]) + date12_idx_list = [sorted(idx) for idx in sorted(date12_idx_list)] + + # Convert index into date12 + date12_list = [date6_list[idx[0]]+'-'+date6_list[idx[1]] + for idx in date12_idx_list] + if date12_format == 'YYYYMMDD_YYYYMMDD': + date12_list = yyyymmdd_date12(date12_list) + return date12_list + + +def select_pairs_hierarchical(date_list, pbase_list, temp_perp_list, date12_format='YYMMDD-YYMMDD'): + """Select Pairs in a hierarchical way using list of temporal and perpendicular baseline thresholds + For each temporal/perpendicular combination, select all possible pairs; and then merge all combination results + together for the final output (Zhao, 2015). + Inputs: + date_list : list of date in YYMMDD/YYYYMMDD format + pbase_list : list of float, perpendicular spatial baseline + temp_perp_list : list of list of 2 floats, for list of temporal/perp baseline, e.g. + [[32.0, 800.0], [48.0, 600.0], [64.0, 200.0]] + Examples: + pairs = select_pairs_hierarchical(date_list, pbase_list, [[32.0, 800.0], [48.0, 600.0], [64.0, 200.0]]) + Reference: + Zhao, W., (2015), Small deformation detected from InSAR time-series and their applications in geophysics, Doctoral + dissertation, Univ. of Miami, Section 6.3. + """ + # Get all date12 + date12_list_all = select_pairs_all(date_list) + + # Loop of Threshold + print('List of temporal and perpendicular spatial baseline thresholds:') + print(temp_perp_list) + date12_list = [] + for temp_perp in temp_perp_list: + tbase_max = temp_perp[0] + pbase_max = temp_perp[1] + date12_list_tmp = threshold_temporal_baseline(date12_list_all, + tbase_max, + keep_seasonal=False) + date12_list_tmp = threshold_perp_baseline(date12_list_tmp, + date_list, + pbase_list, + pbase_max) + date12_list += date12_list_tmp + date12_list = sorted(list(set(date12_list))) + if date12_format == 'YYYYMMDD_YYYYMMDD': + date12_list = yyyymmdd_date12(date12_list) + return date12_list + + +def select_pairs_delaunay(date_list, tbase_list, pbase_list, norm=True, date12_format='YYMMDD-YYMMDD'): + """Select Pairs using Delaunay Triangulation based on temporal/perpendicular baselines + Inputs: + date_list : list of date in YYMMDD/YYYYMMDD format + pbase_list : list of float, perpendicular spatial baseline + norm : normalize temporal baseline to perpendicular baseline + Key points + 1. Define a ratio between perpendicular and temporal baseline axis units (Pepe and Lanari, 2006, TGRS). + 2. Pairs with too large perpendicular / temporal baseline or Doppler centroid difference should be removed + after this, using a threshold, to avoid strong decorrelations (Zebker and Villasenor, 1992, TGRS). + Reference: + Pepe, A., and R. Lanari (2006), On the extension of the minimum cost flow algorithm for phase unwrapping + of multitemporal differential SAR interferograms, IEEE TGRS, 44(9), 2374-2383. + Zebker, H. A., and J. Villasenor (1992), Decorrelation in interferometric radar echoes, IEEE TGRS, 30(5), 950-959. + """ + # Get temporal baseline in days + date6_list = yymmdd(date_list) + date8_list = yyyymmdd(date_list) + #tbase_list = date_list2tbase(date8_list)[0] + + # Normalization (Pepe and Lanari, 2006, TGRS) + if norm: + temp2perp_scale = (max(pbase_list)-min(pbase_list)) / (max(tbase_list)-min(tbase_list)) + tbase_list = [tbase*temp2perp_scale for tbase in tbase_list] + + # Generate Delaunay Triangulation + date12_idx_list = Triangulation(tbase_list, pbase_list).edges.tolist() + date12_idx_list = [sorted(idx) for idx in sorted(date12_idx_list)] + + # Convert index into date12 + date12_list = [date6_list[idx[0]]+'-'+date6_list[idx[1]] + for idx in date12_idx_list] + if date12_format == 'YYYYMMDD_YYYYMMDD': + date12_list = yyyymmdd_date12(date12_list) + return date12_list + + +def select_pairs_mst(date_list, pbase_list, date12_format='YYMMDD-YYMMDD'): + """Select Pairs using Minimum Spanning Tree technique + Connection Cost is calculated using the baseline distance in perp and scaled temporal baseline (Pepe and Lanari, + 2006, TGRS) plane. + Inputs: + date_list : list of date in YYMMDD/YYYYMMDD format + pbase_list : list of float, perpendicular spatial baseline + References: + Pepe, A., and R. Lanari (2006), On the extension of the minimum cost flow algorithm for phase unwrapping + of multitemporal differential SAR interferograms, IEEE TGRS, 44(9), 2374-2383. + Perissin D., Wang T. (2012), Repeat-pass SAR interferometry with partially coherent targets. IEEE TGRS. 271-280 + """ + # Get temporal baseline in days + date6_list = yymmdd(date_list) + date8_list = yyyymmdd(date_list) + tbase_list = date_list2tbase(date8_list)[0] + # Normalization (Pepe and Lanari, 2006, TGRS) + temp2perp_scale = (max(pbase_list)-min(pbase_list)) / (max(tbase_list)-min(tbase_list)) + tbase_list = [tbase*temp2perp_scale for tbase in tbase_list] + + # Get weight matrix + ttMat1, ttMat2 = np.meshgrid(np.array(tbase_list), np.array(tbase_list)) + ppMat1, ppMat2 = np.meshgrid(np.array(pbase_list), np.array(pbase_list)) + ttMat = np.abs(ttMat1 - ttMat2) # temporal distance matrix + ppMat = np.abs(ppMat1 - ppMat2) # spatial distance matrix + + # 2D distance matrix in temp/perp domain + weightMat = np.sqrt(np.square(ttMat) + np.square(ppMat)) + weightMat = sparse.csr_matrix(weightMat) # compress sparse row matrix + + # MST path based on weight matrix + mstMat = sparse.csgraph.minimum_spanning_tree(weightMat) + + # Convert MST index matrix into date12 list + [s_idx_list, m_idx_list] = [date_idx_array.tolist() + for date_idx_array in sparse.find(mstMat)[0:2]] + date12_list = [] + for i in range(len(m_idx_list)): + idx = sorted([m_idx_list[i], s_idx_list[i]]) + date12 = date6_list[idx[0]]+'-'+date6_list[idx[1]] + date12_list.append(date12) + if date12_format == 'YYYYMMDD_YYYYMMDD': + date12_list = yyyymmdd_date12(date12_list) + return date12_list + + +def select_pairs_star(date_list, m_date=None, pbase_list=[], date12_format='YYMMDD-YYMMDD'): + """Select Star-like network/interferograms/pairs, it's a single master network, similar to PS approach. + Usage: + m_date : master date, choose it based on the following cretiria: + 1) near the center in temporal and spatial baseline + 2) prefer winter season than summer season for less temporal decorrelation + Reference: + Ferretti, A., C. Prati, and F. Rocca (2001), Permanent scatterers in SAR interferometry, IEEE TGRS, 39(1), 8-20. + """ + date8_list = sorted(yyyymmdd(date_list)) + date6_list = yymmdd(date8_list) + + # Select master date if not existed + if not m_date: + m_date = select_master_date(date8_list, pbase_list) + print(('auto select master date: '+m_date)) + + # Check input master date + m_date8 = yyyymmdd(m_date) + if m_date8 not in date8_list: + print('Input master date is not existed in date list!') + print(('Input master date: '+str(m_date8))) + print(('Input date list: '+str(date8_list))) + m_date8 = None + + # Generate star/ps network + m_idx = date8_list.index(m_date8) + date12_idx_list = [sorted([m_idx, s_idx]) for s_idx in range(len(date8_list)) + if s_idx is not m_idx] + date12_list = [date6_list[idx[0]]+'-'+date6_list[idx[1]] + for idx in date12_idx_list] + if date12_format == 'YYYYMMDD_YYYYMMDD': + date12_list = yyyymmdd_date12(date12_list) + return date12_list + + +def select_master_date(date_list, pbase_list=[]): + """Select super master date based on input temporal and/or perpendicular baseline info. + Return master date in YYYYMMDD format. + """ + date8_list = yyyymmdd(date_list) + if not pbase_list: + # Choose date in the middle + m_date8 = date8_list[int(len(date8_list)/2)] + else: + # Get temporal baseline list + tbase_list = date_list2tbase(date8_list)[0] + # Normalization (Pepe and Lanari, 2006, TGRS) + temp2perp_scale = (max(pbase_list)-min(pbase_list)) / (max(tbase_list)-min(tbase_list)) + tbase_list = [tbase*temp2perp_scale for tbase in tbase_list] + # Get distance matrix + ttMat1, ttMat2 = np.meshgrid(np.array(tbase_list), + np.array(tbase_list)) + ppMat1, ppMat2 = np.meshgrid(np.array(pbase_list), + np.array(pbase_list)) + ttMat = np.abs(ttMat1 - ttMat2) # temporal distance matrix + ppMat = np.abs(ppMat1 - ppMat2) # spatial distance matrix + # 2D distance matrix in temp/perp domain + disMat = np.sqrt(np.square(ttMat) + np.square(ppMat)) + + # Choose date minimize the total distance of temp/perp baseline + disMean = np.mean(disMat, 0) + m_idx = np.argmin(disMean) + m_date8 = date8_list[m_idx] + return m_date8 + + +def select_master_interferogram(date12_list, date_list, pbase_list, m_date=None): + """Select reference interferogram based on input temp/perp baseline info + If master_date is specified, select its closest slave_date, which is newer than master_date; + otherwise, choose the closest pair among all pairs as master interferogram. + Example: + master_date12 = pnet.select_master_ifgram(date12_list, date_list, pbase_list) + '080211-080326' = pnet.select_master_ifgram(date12_list, date_list, pbase_list, m_date='080211') + """ + pbase_array = np.array(pbase_list, dtype='float64') + # Get temporal baseline + date8_list = yyyymmdd(date_list) + date6_list = yymmdd(date8_list) + tbase_array = np.array(date_list2tbase(date8_list)[0], dtype='float64') + # Normalization (Pepe and Lanari, 2006, TGRS) + temp2perp_scale = (max(pbase_array)-min(pbase_array)) / (max(tbase_array)-min(tbase_array)) + tbase_array *= temp2perp_scale + + # Calculate sqrt of temp/perp baseline for input pairs + idx1 = np.array([date6_list.index(date12.split('-')[0]) for date12 in date12_list]) + idx2 = np.array([date6_list.index(date12.split('-')[1]) for date12 in date12_list]) + base_distance = np.sqrt((tbase_array[idx2] - tbase_array[idx1])**2 + + (pbase_array[idx2] - pbase_array[idx1])**2) + + # Get master interferogram index + if not m_date: + # Choose pair with shortest temp/perp baseline + m_date12_idx = np.argmin(base_distance) + else: + m_date = yymmdd(m_date) + # Choose pair contains m_date with shortest temp/perp baseline + m_date12_idx_array = np.array([date12_list.index(date12) for date12 in date12_list + if m_date+'-' in date12]) + min_base_distance = np.min(base_distance[m_date12_idx_array]) + m_date12_idx = np.where(base_distance == min_base_distance)[0][0] + + m_date12 = date12_list[m_date12_idx] + return m_date12 + + +########################################################## +def datenum2datetime(datenum): + """Convert Matlab datenum into Python datetime. + Parameters: datenum : Date in datenum format, i.e. 731763.5 + Returns: datetime: Date in datetime.datetime format, datetime.datetime(2003, 7, 1, 12, 0) + """ + return dt.fromordinal(int(datenum)) \ + + timedelta(days=datenum % 1) \ + - timedelta(days=366) + + +def decimal_year2datetime(years): + """read date in 2002.40657084 to datetime format + Parameters: years : (list of) float or str for years + Returns: years_dt : (list of) datetime.datetime objects + """ + def decimal_year2datetime1(x): + x = float(x) + year = np.floor(x).astype(int) + yday = np.floor((x - year) * 365.25).astype(int) + 1 + x2 = '{:d}-{:d}'.format(year, yday) + try: + xt = dt(*time.strptime(x2, "%Y-%j")[0:5]) + except: + raise ValueError('wrong format: ',x) + return xt + + if isinstance(years, (float, str)): + years_dt = decimal_year2datetime1(years) + + elif isinstance(years, list): + years_dt = [] + for year in years: + years_dt.append(decimal_year2datetime1(year)) + + else: + raise ValueError('unrecognized input format: {}. Only float/str/list are supported.'.format(type(years))) + return years_dt + + +def yyyymmdd2years(dates): + if isinstance(dates, str): + d = dt(*time.strptime(dates, "%Y%m%d")[0:5]) + yy = float(d.year)+float(d.timetuple().tm_yday-1)/365.25 + elif isinstance(dates, list): + yy = [] + for date in dates: + d = dt(*time.strptime(date, "%Y%m%d")[0:5]) + yy.append(float(d.year)+float(d.timetuple().tm_yday-1)/365.25) + else: + raise ValueError('Unrecognized date format. Only string and list supported.') + return yy + + +def yymmdd2yyyymmdd(date): + if date[0] == '9': + date = '19'+date + else: + date = '20'+date + return date + + +def yyyymmdd(dates): + if isinstance(dates, str): + if len(dates) == 6: + datesOut = yymmdd2yyyymmdd(dates) + else: + datesOut = dates + elif isinstance(dates, list): + datesOut = [] + for date in dates: + if len(date) == 6: + date = yymmdd2yyyymmdd(date) + datesOut.append(date) + else: + # print 'Un-recognized date input!' + return None + return datesOut + + +def yymmdd(dates): + if isinstance(dates, str): + if len(dates) == 8: + datesOut = dates[2:8] + else: + datesOut = dates + elif isinstance(dates, list): + datesOut = [] + for date in dates: + if len(date) == 8: + date = date[2:8] + datesOut.append(date) + else: + # print 'Un-recognized date input!' + return None + return datesOut + + +def yyyymmdd_date12(date12_list): + """Convert date12 into YYYYMMDD_YYYYMMDD format""" + m_dates = yyyymmdd([i.replace('-', '_').split('_')[0] for i in date12_list]) + s_dates = yyyymmdd([i.replace('-', '_').split('_')[1] for i in date12_list]) + date12_list = ['{}-{}'.format(m, s) for m, s in zip(m_dates, s_dates)] + return date12_list + +def yymmdd_date12(date12_list): + """Convert date12 into YYMMDD-YYMMDD format""" + m_dates = yymmdd([i.replace('-', '_').split('_')[0] for i in date12_list]) + s_dates = yymmdd([i.replace('-', '_').split('_')[1] for i in date12_list]) + date12_list = ['{}-{}'.format(m, s) for m, s in zip(m_dates, s_dates)] + return date12_list diff --git a/third_party/PyINT/pyint/_orbit_bridge.py b/third_party/PyINT/pyint/_orbit_bridge.py new file mode 100644 index 0000000..1a70887 --- /dev/null +++ b/third_party/PyINT/pyint/_orbit_bridge.py @@ -0,0 +1,93 @@ +import os +import subprocess +import sys + + +TRUE_VALUES = {"1", "true", "yes", "on"} + + +def _read_bool_env(name, default=False): + value = os.getenv(name) + if value is None: + return default + return str(value).strip().lower() in TRUE_VALUES + + +def apply_precise_orbit(date_text, slc_par_paths, work_dir=None, operation_tag="raw2slc"): + helper_path = str(os.getenv("PYINT_LT1_PRECISE_ORBIT_HELPER") or "").strip() + manifest_json = str(os.getenv("PYINT_LT1_PRECISE_ORBIT_MANIFEST") or "").strip() + enabled = _read_bool_env("PYINT_LT1_PRECISE_ORBIT_ENABLED", False) + strict = _read_bool_env("PYINT_LT1_PRECISE_ORBIT_STRICT", True) + + result = { + "enabled": enabled, + "helper_path": helper_path, + "manifest_json": manifest_json, + "summary_json": "", + "returncode": 0, + "stdout": "", + "stderr": "", + "command": [], + "applied_files": [], + "status": "disabled", + } + if not enabled: + return result + if not helper_path or not os.path.isfile(helper_path): + message = "PYINT_LT1_PRECISE_ORBIT_HELPER is missing or does not exist." + result.update({"status": "missing_helper", "stderr": message}) + if strict: + raise RuntimeError(message) + return result + if not manifest_json: + message = "PYINT_LT1_PRECISE_ORBIT_MANIFEST is empty." + result.update({"status": "missing_manifest", "stderr": message}) + if strict: + raise RuntimeError(message) + return result + + files = [] + for path in slc_par_paths or []: + text = str(path or "").strip() + if text and os.path.isfile(text): + files.append(text) + result["applied_files"] = files + if not files: + result["status"] = "skipped" + return result + + summary_json = os.path.join(str(work_dir or os.getcwd()), "orbit_bridge_summary.json") + command = [ + sys.executable, + helper_path, + "--date", + str(date_text or "").strip(), + "--manifest-json", + manifest_json, + "--summary-json", + summary_json, + "--operation-tag", + str(operation_tag or "raw2slc"), + ] + for path in files: + command.extend(["--slc-par", path]) + result["summary_json"] = summary_json + result["command"] = command + + proc = subprocess.run( + command, + text=True, + capture_output=True, + check=False, + ) + result.update( + { + "returncode": int(proc.returncode), + "stdout": proc.stdout or "", + "stderr": proc.stderr or "", + "status": "applied" if proc.returncode == 0 else "failed", + } + ) + if proc.returncode != 0 and strict: + raise RuntimeError(result["stderr"] or result["stdout"] or "Precise orbit bridge failed.") + return result diff --git a/third_party/PyINT/pyint/_utils.py b/third_party/PyINT/pyint/_utils.py new file mode 100644 index 0000000..58f607d --- /dev/null +++ b/third_party/PyINT/pyint/_utils.py @@ -0,0 +1,689 @@ +############################################################ +# Program is part of PyINT V2.1 # +# Copyright 2017-2019 Yunmeng Cao # +# Contact: ymcmrs@gmail.com # +############################################################ +#from datetime import datetime +import datetime +import urllib.request +import os +import numpy as np +import random +import h5py +from pathlib import Path +import linecache +import time +import glob + +from tqdm import tqdm +from concurrent.futures import ProcessPoolExecutor, as_completed + +# 无条件清理 OMP_PLACES / OMP_PROC_BIND，避免 libgomp 报错 +for _k in ('OMP_PLACES', 'OMP_PROC_BIND'): + os.environ.pop(_k, None) + +####################### update template ############################# +def update_template(template_file): + + templateDict = {} + + ######### download using SSARA ###### + #templateDict['sensor'] = 'sentinel-1' + #templateDict['track'] = '14' + #templateDict['frame'] = '75' + templateDict['start_time'] = '1989-01-01' + templateDict['end_time'] = '2189-01-01' + + ######### download using ASF API ###### + templateDict['asf_username'] = '' # ASF username (can be read from ~/.netrc) + templateDict['asf_password'] = '' # ASF password (can be read from ~/.netrc) + templateDict['bbox'] = '' # bounding box: min_lon,min_lat,max_lon,max_lat + templateDict['shapefile'] = '' # shapefile path for spatial filtering + templateDict['relative_orbit'] = '' # relative orbit number (optional) + templateDict['frame'] = '' # frame number (optional) + templateDict['flight_direction'] = 'a' # flight direction: a=ascending, d=descending, all + templateDict['acquisition_mode'] = 'IW' # acquisition mode: IW, EW, SM, all + + ####### basic parameters for interferometry ###### + templateDict['start_swath'] = '1' + templateDict['end_swath'] = '3' + + templateDict['start_burst'] = '1' + templateDict['end_burst'] = '20' + + templateDict['dem_lat_ovr'] = '0.5' # get 30m resolution of lookup table, 0.5 to 60m, 2 to 15m + templateDict['dem_lon_ovr'] = '0.5' # get 30m resolution of lookup table + templateDict['dem_num_workers'] = '4' # number of parallel workers for DEM download + templateDict['dem_source'] = 'copernicus' # DEM data source: copernicus or nasadem + + templateDict['Igram_Spsflg'] = '1' # Range spectral filtering + templateDict['Igram_Azfflg'] = '1' # Azimuth common band filtering + + templateDict['rwin4cor'] = '256' # range window length for coregistration + templateDict['azwin4cor'] = '256' # azimuth window length for coregistration + + templateDict['rsample4cor'] = '32' # range samples used for fitting the coregistration parameters + templateDict['azsample4cor'] = '32' # azimuth samples used for fitting the coregistration parameters + + templateDict['thresh4cor'] = '0.15' # 2016 GAMMA or higher version, for 2015 GAMMA or lower version should be SNR + + templateDict['coreCoarse'] = 'both' # initial coregistration method, [options: orbit, ampcor, both] + templateDict['coreMethod'] = 'DEM' # coregistration method [option: DEM, init] DEM means with DEM assistant + + templateDict['Igram_Cor_rwin'] = '5' # used for cc_wave + templateDict['Igram_Cor_awin'] = '5' # used for cc_wave + + templateDict['Igram_Cor_Win'] = '5' # used for adf + templateDict['adf_alpha'] = '0.4' # used for adf + ######## sim phase ################## + templateDict['Igram_Flag_TDM'] = 'N' + templateDict['Simphase_rpos'] = '-' + templateDict['Simphase_azpos'] = '-' + templateDict['Simphase_rwin'] = '256' + templateDict['Simphase_azwin'] = '256' + templateDict['Simphase_thresh'] = '-' + + #### unwrap phase ########### + templateDict['auto_unw'] = '1' # auto reference-point selection + templateDict['make_mask'] = '1' # coherence-mask switch for unwrap + templateDict['init_flag'] = '1' # set reference-point phase to 0.0 + templateDict['r_refer'] = '-' # range reference pixel for manual unwrap + templateDict['a_refer'] = '-' # azimuth reference pixel for manual unwrap + templateDict['mcf_triangular'] = '0' # triangular type of mcf [0: regular; 1: delaunay;] + templateDict['unwrap_patr'] = '4' + templateDict['unwrap_pataz'] = '4' + templateDict['unwrapThreshold'] = '0.1' # minimum coherence used for unwrap + + #### geocode ######### + templateDict['geo_interp'] = '0' # [0: nearest; 1: bicubic spline] + templateDict['geocode_products'] = 'hyp3,licsbas' # 产品类型: hyp3,licsbas,pot (逗号分隔多选) + + ############## interferometry ################ + templateDict['int_flag'] = '1' # 1 means do interferometry + templateDict['diff_flag'] = '1' # differential process, i.e., remove DEM phase + templateDict['unw_flag'] = '1' # unwrap process + templateDict['geo_flag'] = '0' # geocode process + + ############## select network ############# + templateDict['endDate'] = '21000101' + templateDict['startDate'] = '19000101' + templateDict['network_method'] = 'sbas' # sbas, sequential, delaunay, stars + templateDict['conNumb'] = '2' # connect number for sequential + templateDict['max_tb'] = '50000' + templateDict['max_sb'] = '50000' + + ############## time-series ################ + templateDict['download_data'] = '0' # if 1, track, frame, or time informations should be provided + templateDict['down_parallel'] = '1' # multi-processor number used for downloading + + templateDict['raw2slc_all'] = '0' # i.e., download 2 slc + templateDict['raw2slc_all_parallel'] = '1' # multi-processor number used + + templateDict['extract_burst_all'] = '0' # for TOPS SLC only + templateDict['extract_all_parallel'] = '1' # multi-processor number used + + templateDict['coreg_all'] = '1' + templateDict['coreg_all_parallel'] = '1' # multi-processor number used [4 or 8] + + templateDict['select_pairs'] = '1' + + templateDict['diff_all'] = '1' + templateDict['diff_all_parallel'] = '1' # multi-processor number used [4 or 8] + + templateDict['unwrap_all'] = '1' + templateDict['unwrap_all_parallel'] = '1' # multi-processor number used [8 or 10] + + templateDict['atmcor_all'] = '0' # if 1, run atmospheric correction + templateDict['atmcor_all_parallel'] = '1' # multi-processor number used + + templateDict['geocode_all'] = '0' + templateDict['geocode_all_parallel'] = '1' # multi-processor number used + templateDict['gamma2licsbas_all'] = '0' # legacy conversion stage switch + templateDict['gamma2licsbas_all_parallel'] = '1' + templateDict['hyp3format_all'] = '0' # legacy conversion stage switch + templateDict['hyp3format_all_parallel'] = '1' + + ############## GACOS atmospheric correction ############# + templateDict['gacos_correction'] = '0' # if 1, apply GACOS atmospheric correction + templateDict['gacos_all_parallel'] = '1' # multi-processor number used for GACOS correction + templateDict['gacos_dir'] = '' # directory for GACOS ZTD files (default: projectName/GACOS) + templateDict['gacos_email'] = '' # email for GACOS auto-download + templateDict['gacos_email_user'] = '' # IMAP username + templateDict['gacos_email_pass'] = '' # IMAP authorization code + templateDict['gacos_email_host'] = 'imap.163.com' # IMAP server host + templateDict['gacos_email_port'] = '993' # IMAP server port + templateDict['gacos_email_ssl'] = '1' # Use SSL + templateDict['gacos_check_interval'] = '60' # Email check interval (seconds) + + ############## Pixel Offset Tracking (POT) ############# + templateDict['pot_all'] = '0' # if 1, run offset tracking + templateDict['pot_all_parallel'] = '1' # multi-processor number used + templateDict['pot_rstep'] = '50' # range step in SLC pixels + templateDict['pot_azstep'] = '10' # azimuth step in SLC pixels + templateDict['pot_rwin'] = '256' # range window size (Round 1) + templateDict['pot_azwin'] = '128' # azimuth window size (Round 1) + templateDict['pot_rwin2'] = '128' # range window size (Round 2) + templateDict['pot_azwin2'] = '64' # azimuth window size (Round 2) + templateDict['pot_ovr'] = '1' # SLC oversampling factor (1=no deramp needed) + templateDict['pot_snr_thresh'] = '0.01' # SNR threshold for offset_pwr_tracking + templateDict['pot_ccp_thresh'] = '0.05' # cross-correlation threshold for masking + templateDict['pot_roff_min'] = '-50' # min range offset (pixels) for masking + templateDict['pot_roff_max'] = '50' # max range offset (pixels) for masking + templateDict['pot_azoff_min'] = '-20' # min azimuth offset (pixels) for masking + templateDict['pot_azoff_max'] = '20' # max azimuth offset (pixels) for masking + templateDict['pot_drange_thresh'] = '2.0' # deviation threshold range (pixels) + templateDict['pot_dazimuth_thresh'] = '1.0' # deviation threshold azimuth (pixels) + templateDict['pot_median_win'] = '7' # median filter window size + templateDict['pot_median_nmin'] = '15' # min valid samples for median filter + templateDict['pot_two_rounds'] = '1' # 1=two-round estimation, 0=single round + templateDict['pot_geocode'] = '1' # 1=geocode displacement results + templateDict['pot_disp_max'] = '100' # max displacement for BMP display (meters) + + templateDict['load_data'] = '0' # loading data for mintpy processing + + templateDict0 = read_template(template_file, delimiter='=') + for key, value in templateDict0.items(): + templateDict[key] = str(value) + + return templateDict + +####################### time-related function ################ + +def is_number(s): + try: + int(s) + return True + except ValueError: + return False + +def print_process_time(start, end): + hours, rem = divmod(end-start, 3600) + minutes, seconds = divmod(rem, 60) + print("Total process time: "+"{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) + return + +def date_add1d(date0): + format_str = '%Y%m%d' # The format + date_obj = datetime.datetime.strptime(date0,format_str) + date1 = (date_obj + datetime.timedelta(days=1)).date().strftime('%Y%m%d') + return date1 + +def date_minus1d(date0): + format_str = '%Y%m%d' # The format + date_obj = datetime.datetime.strptime(date0,format_str) + date1 = (date_obj - datetime.timedelta(days=1)).date().strftime('%Y%m%d') + return date1 + + +def generate_random_name(sufix): + + nowTime = datetime.datetime.now().strftime("%Y%m%d%H%M%S") #generate the present time + randomNum = random.randint(0,100) + randomNum1 = random.randint(0,100) + Nm = nowTime + str(randomNum)+ str(randomNum1) + sufix + + return Nm + +def yyyymmdd2yyyyddd(date): + dt = datetime.datetime.strptime(date, "%Y%m%d") # get datetime object + day_of_year = (dt - datetime.datetime(dt.year, 1, 1)) # Jan the 1st is day 1 + doy = day_of_year.days + 1 + year = dt.year + + doy = str(doy) + if len(doy) ==1: + doy = '00' + doy + elif len(doy) ==2: + doy ='0'+ doy + year = str(year) + + return year, doy + +def get_txt_lines(txt): + count=0 + myfile=open(txt,"r") + for line in myfile: + count=count+1 + + return count + +def read_txt2list(txt): + A = np.loadtxt(txt,dtype=str) + if np.array(A).size ==1: + A = [A] + elif isinstance(A[0],bytes): + A = A.astype(str) + A = list(A) + return A + +def read_txt2array(txt): + A = np.loadtxt(txt,dtype=str) + if np.array(A).size ==1: + A = [A] + elif isinstance(A[0],bytes): + A = A.astype(str) + #A = list(A) + return A +def get_sardata_swath(start_swath,end_swath): + k_swath = '0' + if (start_swath == '1') and (end_swath == '1'): + k_swath = '1' + elif (start_swath == '2') and (end_swath == '2'): + k_swath = '2' + elif (start_swath == '3') and (end_swath == '3'): + k_swath = '3' + elif (start_swath == '1') and (end_swath == '2'): + k_swath = '4' + elif (start_swath == '2') and (end_swath == '3'): + k_swath = '5' + elif (start_swath == '2') and (end_swath == '3'): + k_swath = '-' + + return k_swath + +def get_filelist_filesize(url0): + ttt = generate_random_name('.txt') + call_str = 'curl -s ' + url0 + ' > ' + ttt + os.system(call_str) + A = read_txt2array(ttt) + + A_size = A[:,4] + A_size = A_size.astype(int) + A_size = A_size/1024/1024 #bytes to Mb + A_name = A[:,8] + if os.path.isfile(ttt): + os.remove(ttt) + + return A_name, A_size + +def yyyymmdd(date0): + if len(date0) ==6: + if float(date0[0:2]) > 90: + date1 = '19' + date0 + else: + date1 = '20' + date0 + elif len(date0) ==8: + date1 = date0 + else: + print('The input date is invalid!!') + date1 = '' + return date1 + +def yymmdd(date0): + if len(date0) ==6: + date1 = date0 + elif len(date0) ==8: + date1 = date0[2:8] + else: + print('The input date is invalid!!') + date1 = '' + return date1 + +def parallel_process(array, function, n_jobs=16, use_kwargs=False): + """ + A parallel version of the map function with a progress bar. + + Args: + array (array-like): An array to iterate over. + function (function): A python function to apply to the elements of array + n_jobs (int, default=16): The number of cores to use + use_kwargs (boolean, default=False): Whether to consider the elements of array as dictionaries of + keyword arguments to function + Returns: + [function(array[0]), function(array[1]), ...] + """ + #We run the first few iterations serially to catch bugs + #If we set n_jobs to 1, just run a list comprehension. This is useful for benchmarking and debugging. + if n_jobs==1: + return [function(**a) if use_kwargs else function(a) for a in tqdm(array[:])] + #Assemble the workers + with ProcessPoolExecutor(max_workers=n_jobs) as pool: + #Pass the elements of array into function + if use_kwargs: + futures = [pool.submit(function, **a) for a in array[:]] + else: + futures = [pool.submit(function, a) for a in array[:]] + kwargs = { + 'total': len(futures), + 'unit': 'it', + 'unit_scale': True, + 'leave': True + } + #Print out the progress as tasks complete + for f in tqdm(as_completed(futures), **kwargs): + pass + out = [] + #Get the results from the futures. + for i, future in tqdm(enumerate(futures)): + try: + out.append(future.result()) + except Exception as e: + out.append(e) + return out + +def sort_unique_list(numb_list): + list_out = sorted(set(numb_list)) + return list_out + + +############################# download data ################################### +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def download_s1_orbit(date,save_path,satellite='A'): + + DATE = date + if len(DATE)==6: + DATE = '20' + DATE + ST = satellite + YEAR = int(DATE[0:4]) + MON = int(DATE[4:6]) + DAY = int(DATE[6:8]) + + MON_DAY = [31,28,31,30,31,30,31,31,30,31,30,31] + + if YEAR%4==0: + MON_DAY[1]=29 + + if MON ==1 and DAY ==1: + DAY0 = 31 + MON0 = 12 + YEAR0 =YEAR -1 + elif MON!=1 and DAY ==1: + DAY0 = MON_DAY[MON-2] + MON0 = MON-1 + YEAR0 = YEAR + else: + DAY0 = DAY -1 + MON0 = MON + YEAR0 = YEAR + + MONDAY0 = MON_DAY[MON0-1] + + TT = [1,4,7,10,13,16,19,22,25,28,MONDAY0] + + T0 = [] + for k in range(len(TT)): + T0.append(TT[k]) + TT[k] = TT[k]-DAY0 + + for k in range(len(TT)): + if k == len(TT)-1: + if TT[k]==0: + ff = k-1 + else: + if TT[k]<=0 and TT[k+1]>0: + ff = k + + + DAY1 = T0[ff] + DAY2 = T0[ff+1] + + S1 = StrNum(YEAR0) + '-' + StrNum(MON0) + S2 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY1) + S3 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY2) + S4 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY0) + + #SS = 'https://qc.sentinel1.eo.esa.int/aux_poeorb/?mission=S1' + ST + '&validity_start_time=' + StrNum(YEAR0) + '&validity_start_time=' + S1 + '&validity_start_time=' + S2 + '..' + S3 + '&validity_start_time=' + S4 + SS = 'https://qc.sentinel1.eo.esa.int/aux_poeorb/?validity_start=' + StrNum(YEAR0) + '&validity_start=' + S1 + '&validity_start=' + S2 + '..' + S3 + '&validity_start=' +S4 + '&sentinel1__mission=S1'+ST+ '&sentinel1_mission=S1'+ST+ '&sentinel1_mission=S1'+ST+ '&sentinel1_mission=S1'+ST + + tt ='tt_orb_' + date + tt0 ='tt0_orb_' + date + tt00 ='tt00_orb_' + date + tt000 ='tt000_orb_' + date + SS = "'" + SS + "'" + #print(SS) + call_str = 'curl -s -l ' + SS + ' > ' + tt + os.system(call_str) + + call_str = "grep 'EOF' -C 0 " + tt + " >" + tt0 + os.system(call_str) + + call_str="awk -F'href=' '{print $2}' " + tt0 +' >' + tt00 + os.system(call_str) + + call_str= "awk -F'>' '{print $1}' " + tt00 + '> ' + tt000 + os.system(call_str) + + SS=linecache.getline(tt000, 1) + SS = SS.split('"')[1] + filename = os.path.basename(SS) + if not os.path.isfile(filename): + call_str = 'wget -q --no-check-certificate ' + SS + ' -O ' + save_path + '/' +filename + os.system(call_str) + call_str = "downlaod the precise Orbit of" + date + print(call_str) + + filename = os.path.basename(SS) + os.remove(tt) + os.remove(tt0) + os.remove(tt00) + os.remove(tt000) + + return filename + + +############################# write & read ##################################### +def copy_file(file0,file1): + call_str = 'cp ' + file0 + ' ' + file1 + os.system(call_str) + + return + +def get_project_slcList(projectName): + scratchDir = os.getenv('SCRATCHDIR') + slcDir = scratchDir + '/' + projectName + "/SLC" + #slcDir = scratchDir + '/SLC' + fn = os.listdir(slcDir) + slc_list0 = [os.path.basename(fname) for fname in sorted(fn)] + #slc_list0 = [os.path.basename(fname) for fname in sorted(glob.glob(slcDir + '/*'))] + + slc_list = [] + for k0 in slc_list0: + if is_number(k0): + slc_list.append(k0) + slc_list = sorted(slc_list) + + return slc_list + +def createBlankFile(strFile): + f = open(strFile,'w') + for i in range (10): + f.write('\n') + f.close() + +def read_attr(fname): + # read hdf5 + with h5py.File(fname, 'r') as f: + atr = dict(f.attrs) + + return atr + +def read_hdf5(fname, datasetName=None, box=None): + # read hdf5 + with h5py.File(fname, 'r') as f: + data = f[datasetName][:] + atr = dict(f.attrs) + + return data, atr + +def get_dataNames(FILE): + with h5py.File(FILE, 'r') as f: + dataNames = [] + for k0 in f.keys(): + dataNames.append(k0) + return dataNames + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def read_gamma_par(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close + + +def write_h5(datasetDict, out_file, metadata=None, ref_file=None, compression=None): + + if os.path.isfile(out_file): + print('delete exsited file: {}'.format(out_file)) + os.remove(out_file) + + print('create HDF5 file: {} with w mode'.format(out_file)) + dt = h5py.special_dtype(vlen=np.dtype('float64')) + + + with h5py.File(out_file, 'w') as f: + for dsName in datasetDict.keys(): + data = datasetDict[dsName] + ds = f.create_dataset(dsName, + data=data, + compression=compression) + + for key, value in metadata.items(): + f.attrs[key] = str(value) + #print(key + ': ' + value) + print('finished writing to {}'.format(out_file)) + + return out_file + +###################################################################### +class progressBar: + """Creates a text-based progress bar. Call the object with + the simple print command to see the progress bar, which looks + something like this: + [=======> 22% ] + You may specify the progress bar's min and max values on init. + + note: + modified from mintPy (https://github.com/insarlab/MintPy/wiki) + Code originally from http://code.activestate.com/recipes/168639/ + + example: + from mintpy.utils import ptime + date12_list = ptime.list_ifgram2date12(ifgram_list) + prog_bar = ptime.progressBar(maxValue=1000, prefix='calculating:') + for i in range(1000): + prog_bar.update(i+1, suffix=date) + prog_bar.update(i+1, suffix=date12_list[i]) + prog_bar.close() + """ + + def __init__(self, maxValue=100, prefix='', minValue=0, totalWidth=70, print_msg=True): + self.prog_bar = "[]" # This holds the progress bar string + self.min = minValue + self.max = maxValue + self.span = maxValue - minValue + self.suffix = '' + self.prefix = prefix + + self.print_msg = print_msg + ## calculate total width based on console width + #rows, columns = os.popen('stty size', 'r').read().split() + #self.width = round(int(columns) * 0.7 / 10) * 10 + self.width = totalWidth + self.reset() + + def reset(self): + self.start_time = time.time() + self.amount = 0 # When amount == max, we are 100% done + self.update_amount(0) # Build progress bar string + + def update_amount(self, newAmount=0, suffix=''): + """ Update the progress bar with the new amount (with min and max + values set at initialization; if it is over or under, it takes the + min or max value as a default. """ + if newAmount < self.min: + newAmount = self.min + if newAmount > self.max: + newAmount = self.max + self.amount = newAmount + + # Figure out the new percent done, round to an integer + diffFromMin = np.float(self.amount - self.min) + percentDone = (diffFromMin / np.float(self.span)) * 100.0 + percentDone = np.int(np.round(percentDone)) + + # Figure out how many hash bars the percentage should be + allFull = self.width - 2 - 18 + numHashes = (percentDone / 100.0) * allFull + numHashes = np.int(np.round(numHashes)) + + # Build a progress bar with an arrow of equal signs; special cases for + # empty and full + if numHashes == 0: + self.prog_bar = '%s[>%s]' % (self.prefix, ' '*(allFull-1)) + elif numHashes == allFull: + self.prog_bar = '%s[%s]' % (self.prefix, '='*allFull) + if suffix: + self.prog_bar += ' %s' % (suffix) + else: + self.prog_bar = '[%s>%s]' % ('='*(numHashes-1), ' '*(allFull-numHashes)) + # figure out where to put the percentage, roughly centered + percentPlace = int(len(self.prog_bar)/2 - len(str(percentDone))) + percentString = ' ' + str(percentDone) + '% ' + # slice the percentage into the bar + self.prog_bar = ''.join([self.prog_bar[0:percentPlace], + percentString, + self.prog_bar[percentPlace+len(percentString):]]) + # prefix and suffix + self.prog_bar = self.prefix + self.prog_bar + if suffix: + self.prog_bar += ' %s' % (suffix) + # time info - elapsed time and estimated remaining time + if percentDone > 0: + elapsed_time = time.time() - self.start_time + self.prog_bar += '%5ds / %5ds' % (int(elapsed_time), + int(elapsed_time * (100./percentDone-1))) + + def update(self, value, every=1, suffix=''): + """ Updates the amount, and writes to stdout. Prints a + carriage return first, so it will overwrite the current + line in stdout.""" + if value % every == 0 or value >= self.max: + self.update_amount(newAmount=value, suffix=suffix) + if self.print_msg: + sys.stdout.write('\r' + self.prog_bar) + sys.stdout.flush() + + def close(self): + """Prints a blank space at the end to ensure proper printing + of future statements.""" + if self.print_msg: + print(' ') diff --git a/third_party/PyINT/pyint/_utils_chen.py b/third_party/PyINT/pyint/_utils_chen.py new file mode 100644 index 0000000..dcb69f5 --- /dev/null +++ b/third_party/PyINT/pyint/_utils_chen.py @@ -0,0 +1,641 @@ +############################################################ +# Program is part of PyINT V2.1 # +# Copyright 2017-2019 Yunmeng Cao # +# Contact: ymcmrs@gmail.com # +############################################################ +#from datetime import datetime +import datetime +import urllib.request +import os +import numpy as np +import random +import h5py +from pathlib import Path +import linecache +import time +import glob + +from tqdm import tqdm +from concurrent.futures import ProcessPoolExecutor, as_completed + +####################### update template ############################# +def update_template(template_file): + + templateDict = {} + + ######### download using SSARA ###### + #templateDict['sensor'] = 'sentinel-1' + #templateDict['track'] = '14' + #templateDict['frame'] = '75' + templateDict['start_time'] = '1989-01-01' + templateDict['end_time'] = '2189-01-01' + + ####### basic parameters for interferometry ###### + templateDict['start_swath'] = '1' + templateDict['end_swath'] = '3' + + templateDict['start_burst'] = '1' + templateDict['end_burst'] = '20' + + templateDict['dem_lat_ovr'] = '0.5' # get 30m resolution of lookup table, 0.5 to 60m, 2 to 15m + templateDict['dem_lon_ovr'] = '0.5' # get 30m resolution of lookup table + + templateDict['Igram_Spsflg'] = '1' # Range spectral filtering + templateDict['Igram_Azfflg'] = '1' # Azimuth common band filtering + + templateDict['rwin4cor'] = '256' # range window length for coregistration + templateDict['azwin4cor'] = '256' # azimuth window length for coregistration + + templateDict['rsample4cor'] = '32' # range samples used for fitting the coregistration parameters + templateDict['azsample4cor'] = '32' # azimuth samples used for fitting the coregistration parameters + + templateDict['thresh4cor'] = '0.15' # 2016 GAMMA or higher version, for 2015 GAMMA or lower version should be SNR + + templateDict['coreCoarse'] = 'both' # initial coregistration method, [options: orbit, ampcor, both] + templateDict['coreMethod'] = 'DEM' # coregistration method [option: DEM, init] DEM means with DEM assistant + + templateDict['Igram_Cor_rwin'] = '5' # used for cc_wave + templateDict['Igram_Cor_awin'] = '5' # used for cc_wave + + templateDict['Igram_Cor_Win'] = '5' # used for adf + templateDict['adf_alpha'] = '0.4' # used for adf + ######## sim phase ################## + templateDict['Igram_Flag_TDM'] = 'N' + templateDict['Simphase_rpos'] = '-' + templateDict['Simphase_azpos'] = '-' + templateDict['Simphase_rwin'] = '256' + templateDict['Simphase_azwin'] = '256' + templateDict['Simphase_thresh'] = '-' + + #### unwrap phase ########### + templateDict['mcf_triangular'] = '0' # triangular type of mcf [0: regular; 1: delaunay;] + templateDict['unwrap_patr'] = '1' + templateDict['unwrap_pataz'] = '1' + templateDict['unwrapThreshold'] = '0.1' # minimum coherence used for unwrap + + #### geocode ######### + templateDict['geo_interp'] = '0' # [0: nearest; 1: bicubic spline] + + ############## interferometry ################ + templateDict['int_flag'] = '1' # 1 means do interferometry + templateDict['diff_flag'] = '1' # differential process, i.e., remove DEM phase + templateDict['unw_flag'] = '1' # unwrap process + templateDict['geo_flag'] = '0' # geocode process + + ############## select network ############# + templateDict['endDate'] = '21000101' + templateDict['startDate'] = '19000101' + templateDict['network_method'] = 'sbas' # sbas, sequential, delaunay, stars + templateDict['conNumb'] = '2' # connect number for sequential + templateDict['max_tb'] = '50000' + templateDict['max_sb'] = '50000' + + ############## time-series ################ + templateDict['download_data'] = '0' # if 1, track, frame, or time informations should be provided + templateDict['down_parallel'] = '1' # multi-processor number used for downloading + + templateDict['raw2slc_all'] = '0' # i.e., download 2 slc + templateDict['raw2slc_all_parallel'] = '1' # multi-processor number used + + templateDict['extract_burst_all'] = '0' # for TOPS SLC only + templateDict['extract_all_parallel'] = '1' # multi-processor number used + + templateDict['coreg_all'] = '1' + templateDict['coreg_all_parallel'] = '1' # multi-processor number used [4 or 8] + + templateDict['select_pairs'] = '1' + + templateDict['diff_all'] = '1' + templateDict['diff_all_parallel'] = '1' # multi-processor number used [4 or 8] + + templateDict['unwrap_all'] = '1' + templateDict['unwrap_all_parallel'] = '1' # multi-processor number used [8 or 10] + + templateDict['geocode_all'] = '0' + templateDict['geocode_all_parallel'] = '1' # multi-processor number used + + templateDict['load_data'] = '0' # loading data for mintpy processing + + templateDict0 = read_template(template_file, delimiter='=') + for key, value in templateDict0.items(): + templateDict[key] = str(value) + + return templateDict + +####################### time-related function ################ + +def is_number(s): + try: + int(s) + return True + except ValueError: + return False + +def print_process_time(start, end): + hours, rem = divmod(end-start, 3600) + minutes, seconds = divmod(rem, 60) + print("Total process time: "+"{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) + return + +def date_add1d(date0): + format_str = '%Y%m%d' # The format + date_obj = datetime.datetime.strptime(date0,format_str) + date1 = (date_obj + datetime.timedelta(days=1)).date().strftime('%Y%m%d') + return date1 + +def date_minus1d(date0): + format_str = '%Y%m%d' # The format + date_obj = datetime.datetime.strptime(date0,format_str) + date1 = (date_obj - datetime.timedelta(days=1)).date().strftime('%Y%m%d') + return date1 + + +def generate_random_name(sufix): + + nowTime = datetime.datetime.now().strftime("%Y%m%d%H%M%S") #generate the present time + randomNum = random.randint(0,100) + randomNum1 = random.randint(0,100) + Nm = nowTime + str(randomNum)+ str(randomNum1) + sufix + + return Nm + +def yyyymmdd2yyyyddd(date): + dt = datetime.datetime.strptime(date, "%Y%m%d") # get datetime object + day_of_year = (dt - datetime.datetime(dt.year, 1, 1)) # Jan the 1st is day 1 + doy = day_of_year.days + 1 + year = dt.year + + doy = str(doy) + if len(doy) ==1: + doy = '00' + doy + elif len(doy) ==2: + doy ='0'+ doy + year = str(year) + + return year, doy + +def get_txt_lines(txt): + count=0 + myfile=open(txt,"r") + for line in myfile: + count=count+1 + + return count + +def read_txt2list(txt): + A = np.loadtxt(txt,dtype=np.str) + if np.array(A).size ==1: + A = [A] + elif isinstance(A[0],bytes): + A = A.astype(str) + A = list(A) + return A + +def read_txt2array(txt): + A = np.loadtxt(txt,dtype=np.str) + if np.array(A).size ==1: + A = [A] + elif isinstance(A[0],bytes): + A = A.astype(str) + #A = list(A) + return A +def get_sardata_swath(start_swath,end_swath): + k_swath = '0' + if (start_swath == '1') and (end_swath == '1'): + k_swath = '1' + elif (start_swath == '2') and (end_swath == '2'): + k_swath = '2' + elif (start_swath == '3') and (end_swath == '3'): + k_swath = '3' + elif (start_swath == '1') and (end_swath == '2'): + k_swath = '4' + elif (start_swath == '2') and (end_swath == '3'): + k_swath = '5' + elif (start_swath == '1') and (end_swath == '3'): + k_swath = '0' + + return k_swath + +def get_filelist_filesize(url0): + ttt = generate_random_name('.txt') + call_str = 'curl -s ' + url0 + ' > ' + ttt + os.system(call_str) + A = read_txt2array(ttt) + + A_size = A[:,4] + A_size = A_size.astype(int) + A_size = A_size/1024/1024 #bytes to Mb + A_name = A[:,8] + if os.path.isfile(ttt): + os.remove(ttt) + + return A_name, A_size + +def yyyymmdd(date0): + if len(date0) ==6: + if float(date0[0:2]) > 90: + date1 = '19' + date0 + else: + date1 = '20' + date0 + elif len(date0) ==8: + date1 = date0 + else: + print('The input date is invalid!!') + date1 = '' + return date1 + +def yymmdd(date0): + if len(date0) ==6: + date1 = date0 + elif len(date0) ==8: + date1 = date0[2:8] + else: + print('The input date is invalid!!') + date1 = '' + return date1 + +def parallel_process(array, function, n_jobs=16, use_kwargs=False): + """ + A parallel version of the map function with a progress bar. + + Args: + array (array-like): An array to iterate over. + function (function): A python function to apply to the elements of array + n_jobs (int, default=16): The number of cores to use + use_kwargs (boolean, default=False): Whether to consider the elements of array as dictionaries of + keyword arguments to function + Returns: + [function(array[0]), function(array[1]), ...] + """ + #We run the first few iterations serially to catch bugs + #If we set n_jobs to 1, just run a list comprehension. This is useful for benchmarking and debugging. + if n_jobs==1: + return [function(**a) if use_kwargs else function(a) for a in tqdm(array[:])] + #Assemble the workers + with ProcessPoolExecutor(max_workers=n_jobs) as pool: + #Pass the elements of array into function + if use_kwargs: + futures = [pool.submit(function, **a) for a in array[:]] + else: + futures = [pool.submit(function, a) for a in array[:]] + kwargs = { + 'total': len(futures), + 'unit': 'it', + 'unit_scale': True, + 'leave': True + } + #Print out the progress as tasks complete + for f in tqdm(as_completed(futures), **kwargs): + pass + out = [] + #Get the results from the futures. + for i, future in tqdm(enumerate(futures)): + try: + out.append(future.result()) + except Exception as e: + out.append(e) + return out + +def sort_unique_list(numb_list): + list_out = sorted(set(numb_list)) + return list_out + + +############################# download data ################################### +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def download_s1_orbit(date,save_path,satellite='A'): + + DATE = date + if len(DATE)==6: + DATE = '20' + DATE + ST = satellite + YEAR = int(DATE[0:4]) + MON = int(DATE[4:6]) + DAY = int(DATE[6:8]) + + MON_DAY = [31,28,31,30,31,30,31,31,30,31,30,31] + + if YEAR%4==0: + MON_DAY[1]=29 + + if MON ==1 and DAY ==1: + DAY0 = 31 + MON0 = 12 + YEAR0 =YEAR -1 + elif MON!=1 and DAY ==1: + DAY0 = MON_DAY[MON-2] + MON0 = MON-1 + YEAR0 = YEAR + else: + DAY0 = DAY -1 + MON0 = MON + YEAR0 = YEAR + + MONDAY0 = MON_DAY[MON0-1] + + TT = [1,4,7,10,13,16,19,22,25,28,MONDAY0] + + T0 = [] + for k in range(len(TT)): + T0.append(TT[k]) + TT[k] = TT[k]-DAY0 + + for k in range(len(TT)): + if k == len(TT)-1: + if TT[k]==0: + ff = k-1 + else: + if TT[k]<=0 and TT[k+1]>0: + ff = k + + + DAY1 = T0[ff] + DAY2 = T0[ff+1] + + S1 = StrNum(YEAR0) + '-' + StrNum(MON0) + S2 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY1) + S3 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY2) + S4 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY0) + + #SS = 'https://qc.sentinel1.eo.esa.int/aux_poeorb/?mission=S1' + ST + '&validity_start_time=' + StrNum(YEAR0) + '&validity_start_time=' + S1 + '&validity_start_time=' + S2 + '..' + S3 + '&validity_start_time=' + S4 + SS = 'https://qc.sentinel1.eo.esa.int/aux_poeorb/?validity_start=' + StrNum(YEAR0) + '&validity_start=' + S1 + '&validity_start=' + S2 + '..' + S3 + '&validity_start=' +S4 + '&sentinel1__mission=S1'+ST+ '&sentinel1_mission=S1'+ST+ '&sentinel1_mission=S1'+ST+ '&sentinel1_mission=S1'+ST + + tt ='tt_orb_' + date + tt0 ='tt0_orb_' + date + tt00 ='tt00_orb_' + date + tt000 ='tt000_orb_' + date + SS = "'" + SS + "'" + #print(SS) + call_str = 'curl -s -l ' + SS + ' > ' + tt + os.system(call_str) + + call_str = "grep 'EOF' -C 0 " + tt + " >" + tt0 + os.system(call_str) + + call_str="awk -F'href=' '{print $2}' " + tt0 +' >' + tt00 + os.system(call_str) + + call_str= "awk -F'>' '{print $1}' " + tt00 + '> ' + tt000 + os.system(call_str) + + SS=linecache.getline(tt000, 1) + SS = SS.split('"')[-1] + + filename = os.path.basename(SS) +# filename = os.path.basename(SS) +# SS1 = save_path + '/' + filename +# if os.path.isfile(SS1): +# if os.path.getsize(SS1) > 4409000: +# call_str = "downlaod the precise Orbit of" + date +# print(call_str) +# else: +# call_str = 'wget -q --no-check-certificate ' + SS + ' -O ' + save_path + '/' +filename +# os.system(call_str) +# call_str = "downlaod the precise Orbit of" + date +# print(call_str) +# else: +# call_str = 'wget -q --no-check-certificate ' + SS + ' -O ' + save_path + '/' +filename +# os.system(call_str) +# call_str = "downlaod the precise Orbit of" + date +# print(call_str) +# if not os.path.isfile(filename): + call_str = 'wget -q --no-check-certificate ' + SS + ' -O ' + save_path + '/' +filename + os.system(call_str) + call_str = "downlaod the precise Orbit of" + date + print(call_str) + + filename = os.path.basename(SS) + os.remove(tt) + os.remove(tt0) + os.remove(tt00) + os.remove(tt000) + + return filename + + +############################# write & read ##################################### +def copy_file(file0,file1): + call_str = 'cp ' + file0 + ' ' + file1 + os.system(call_str) + + return + +def get_project_slcList(projectName): + scratchDir = os.getenv('SCRATCHDIR') + slcDir = scratchDir + '/' + projectName + "/SLC" + #slcDir = scratchDir + '/SLC' + fn = os.listdir(slcDir) + slc_list0 = [os.path.basename(fname) for fname in sorted(fn)] + #slc_list0 = [os.path.basename(fname) for fname in sorted(glob.glob(slcDir + '/*'))] + + slc_list = [] + for k0 in slc_list0: + if is_number(k0): + slc_list.append(k0) + slc_list = sorted(slc_list) + + return slc_list + +def createBlankFile(strFile): + f = open(strFile,'w') + for i in range (10): + f.write('\n') + f.close() + +def read_attr(fname): + # read hdf5 + with h5py.File(fname, 'r') as f: + atr = dict(f.attrs) + + return atr + +def read_hdf5(fname, datasetName=None, box=None): + # read hdf5 + with h5py.File(fname, 'r') as f: + data = f[datasetName][:] + atr = dict(f.attrs) + + return data, atr + +def get_dataNames(FILE): + with h5py.File(FILE, 'r') as f: + dataNames = [] + for k0 in f.keys(): + dataNames.append(k0) + return dataNames + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def read_gamma_par(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close + + +def write_h5(datasetDict, out_file, metadata=None, ref_file=None, compression=None): + + if os.path.isfile(out_file): + print('delete exsited file: {}'.format(out_file)) + os.remove(out_file) + + print('create HDF5 file: {} with w mode'.format(out_file)) + dt = h5py.special_dtype(vlen=np.dtype('float64')) + + + with h5py.File(out_file, 'w') as f: + for dsName in datasetDict.keys(): + data = datasetDict[dsName] + ds = f.create_dataset(dsName, + data=data, + compression=compression) + + for key, value in metadata.items(): + f.attrs[key] = str(value) + #print(key + ': ' + value) + print('finished writing to {}'.format(out_file)) + + return out_file + +###################################################################### +class progressBar: + """Creates a text-based progress bar. Call the object with + the simple print command to see the progress bar, which looks + something like this: + [=======> 22% ] + You may specify the progress bar's min and max values on init. + + note: + modified from mintPy (https://github.com/insarlab/MintPy/wiki) + Code originally from http://code.activestate.com/recipes/168639/ + + example: + from mintpy.utils import ptime + date12_list = ptime.list_ifgram2date12(ifgram_list) + prog_bar = ptime.progressBar(maxValue=1000, prefix='calculating:') + for i in range(1000): + prog_bar.update(i+1, suffix=date) + prog_bar.update(i+1, suffix=date12_list[i]) + prog_bar.close() + """ + + def __init__(self, maxValue=100, prefix='', minValue=0, totalWidth=70, print_msg=True): + self.prog_bar = "[]" # This holds the progress bar string + self.min = minValue + self.max = maxValue + self.span = maxValue - minValue + self.suffix = '' + self.prefix = prefix + + self.print_msg = print_msg + ## calculate total width based on console width + #rows, columns = os.popen('stty size', 'r').read().split() + #self.width = round(int(columns) * 0.7 / 10) * 10 + self.width = totalWidth + self.reset() + + def reset(self): + self.start_time = time.time() + self.amount = 0 # When amount == max, we are 100% done + self.update_amount(0) # Build progress bar string + + def update_amount(self, newAmount=0, suffix=''): + """ Update the progress bar with the new amount (with min and max + values set at initialization; if it is over or under, it takes the + min or max value as a default. """ + if newAmount < self.min: + newAmount = self.min + if newAmount > self.max: + newAmount = self.max + self.amount = newAmount + + # Figure out the new percent done, round to an integer + diffFromMin = np.float(self.amount - self.min) + percentDone = (diffFromMin / np.float(self.span)) * 100.0 + percentDone = np.int(np.round(percentDone)) + + # Figure out how many hash bars the percentage should be + allFull = self.width - 2 - 18 + numHashes = (percentDone / 100.0) * allFull + numHashes = np.int(np.round(numHashes)) + + # Build a progress bar with an arrow of equal signs; special cases for + # empty and full + if numHashes == 0: + self.prog_bar = '%s[>%s]' % (self.prefix, ' '*(allFull-1)) + elif numHashes == allFull: + self.prog_bar = '%s[%s]' % (self.prefix, '='*allFull) + if suffix: + self.prog_bar += ' %s' % (suffix) + else: + self.prog_bar = '[%s>%s]' % ('='*(numHashes-1), ' '*(allFull-numHashes)) + # figure out where to put the percentage, roughly centered + percentPlace = int(len(self.prog_bar)/2 - len(str(percentDone))) + percentString = ' ' + str(percentDone) + '% ' + # slice the percentage into the bar + self.prog_bar = ''.join([self.prog_bar[0:percentPlace], + percentString, + self.prog_bar[percentPlace+len(percentString):]]) + # prefix and suffix + self.prog_bar = self.prefix + self.prog_bar + if suffix: + self.prog_bar += ' %s' % (suffix) + # time info - elapsed time and estimated remaining time + if percentDone > 0: + elapsed_time = time.time() - self.start_time + self.prog_bar += '%5ds / %5ds' % (int(elapsed_time), + int(elapsed_time * (100./percentDone-1))) + + def update(self, value, every=1, suffix=''): + """ Updates the amount, and writes to stdout. Prints a + carriage return first, so it will overwrite the current + line in stdout.""" + if value % every == 0 or value >= self.max: + self.update_amount(newAmount=value, suffix=suffix) + if self.print_msg: + sys.stdout.write('\r' + self.prog_bar) + sys.stdout.flush() + + def close(self): + """Prints a blank space at the end to ensure proper printing + of future statements.""" + if self.print_msg: + print(' ') diff --git a/third_party/PyINT/pyint/_utils_old.py b/third_party/PyINT/pyint/_utils_old.py new file mode 100644 index 0000000..2b2cfa5 --- /dev/null +++ b/third_party/PyINT/pyint/_utils_old.py @@ -0,0 +1,623 @@ +############################################################ +# Program is part of PyINT V2.1 # +# Copyright 2017-2019 Yunmeng Cao # +# Contact: ymcmrs@gmail.com # +############################################################ +#from datetime import datetime +import datetime +import urllib.request +import os +import numpy as np +import random +import h5py +from pathlib import Path +import linecache +import time +import glob + +from tqdm import tqdm +from concurrent.futures import ProcessPoolExecutor, as_completed + +####################### update template ############################# +def update_template(template_file): + + templateDict = {} + + ######### download using SSARA ###### + #templateDict['sensor'] = 'sentinel-1' + #templateDict['track'] = '14' + #templateDict['frame'] = '75' + templateDict['start_time'] = '1989-01-01' + templateDict['end_time'] = '2189-01-01' + + ####### basic parameters for interferometry ###### + templateDict['start_swath'] = '1' + templateDict['end_swath'] = '3' + + templateDict['start_burst'] = '1' + templateDict['end_burst'] = '20' + + templateDict['dem_lat_ovr'] = '0.5' # get 30m resolution of lookup table, 0.5 to 60m, 2 to 15m + templateDict['dem_lon_ovr'] = '0.5' # get 30m resolution of lookup table + + templateDict['Igram_Spsflg'] = '1' # Range spectral filtering + templateDict['Igram_Azfflg'] = '1' # Azimuth common band filtering + + templateDict['rwin4cor'] = '256' # range window length for coregistration + templateDict['azwin4cor'] = '256' # azimuth window length for coregistration + + templateDict['rsample4cor'] = '32' # range samples used for fitting the coregistration parameters + templateDict['azsample4cor'] = '32' # azimuth samples used for fitting the coregistration parameters + + templateDict['thresh4cor'] = '0.15' # 2016 GAMMA or higher version, for 2015 GAMMA or lower version should be SNR + + templateDict['coreCoarse'] = 'both' # initial coregistration method, [options: orbit, ampcor, both] + templateDict['coreMethod'] = 'DEM' # coregistration method [option: DEM, init] DEM means with DEM assistant + + templateDict['Igram_Cor_rwin'] = '5' # used for cc_wave + templateDict['Igram_Cor_awin'] = '5' # used for cc_wave + + templateDict['Igram_Cor_Win'] = '5' # used for adf + templateDict['adf_alpha'] = '0.4' # used for adf + ######## sim phase ################## + templateDict['Igram_Flag_TDM'] = 'N' + templateDict['Simphase_rpos'] = '-' + templateDict['Simphase_azpos'] = '-' + templateDict['Simphase_rwin'] = '256' + templateDict['Simphase_azwin'] = '256' + templateDict['Simphase_thresh'] = '-' + + #### unwrap phase ########### + templateDict['mcf_triangular'] = '0' # triangular type of mcf [0: regular; 1: delaunay;] + templateDict['unwrap_patr'] = '1' + templateDict['unwrap_pataz'] = '1' + templateDict['unwrapThreshold'] = '0.1' # minimum coherence used for unwrap + + #### geocode ######### + templateDict['geo_interp'] = '0' # [0: nearest; 1: bicubic spline] + + ############## interferometry ################ + templateDict['int_flag'] = '1' # 1 means do interferometry + templateDict['diff_flag'] = '1' # differential process, i.e., remove DEM phase + templateDict['unw_flag'] = '1' # unwrap process + templateDict['geo_flag'] = '0' # geocode process + + ############## select network ############# + templateDict['endDate'] = '21000101' + templateDict['startDate'] = '19000101' + templateDict['network_method'] = 'sbas' # sbas, sequential, delaunay, stars + templateDict['conNumb'] = '2' # connect number for sequential + templateDict['max_tb'] = '50000' + templateDict['max_sb'] = '50000' + + ############## time-series ################ + templateDict['download_data'] = '0' # if 1, track, frame, or time informations should be provided + templateDict['down_parallel'] = '1' # multi-processor number used for downloading + + templateDict['raw2slc_all'] = '0' # i.e., download 2 slc + templateDict['raw2slc_all_parallel'] = '1' # multi-processor number used + + templateDict['extract_burst_all'] = '0' # for TOPS SLC only + templateDict['extract_all_parallel'] = '1' # multi-processor number used + + templateDict['coreg_all'] = '1' + templateDict['coreg_all_parallel'] = '1' # multi-processor number used [4 or 8] + + templateDict['select_pairs'] = '1' + + templateDict['diff_all'] = '1' + templateDict['diff_all_parallel'] = '1' # multi-processor number used [4 or 8] + + templateDict['unwrap_all'] = '1' + templateDict['unwrap_all_parallel'] = '1' # multi-processor number used [8 or 10] + + templateDict['geocode_all'] = '0' + templateDict['geocode_all_parallel'] = '1' # multi-processor number used + + templateDict['load_data'] = '0' # loading data for mintpy processing + + templateDict0 = read_template(template_file, delimiter='=') + for key, value in templateDict0.items(): + templateDict[key] = str(value) + + return templateDict + +####################### time-related function ################ + +def is_number(s): + try: + int(s) + return True + except ValueError: + return False + +def print_process_time(start, end): + hours, rem = divmod(end-start, 3600) + minutes, seconds = divmod(rem, 60) + print("Total process time: "+"{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) + return + +def date_add1d(date0): + format_str = '%Y%m%d' # The format + date_obj = datetime.datetime.strptime(date0,format_str) + date1 = (date_obj + datetime.timedelta(days=1)).date().strftime('%Y%m%d') + return date1 + +def date_minus1d(date0): + format_str = '%Y%m%d' # The format + date_obj = datetime.datetime.strptime(date0,format_str) + date1 = (date_obj - datetime.timedelta(days=1)).date().strftime('%Y%m%d') + return date1 + + +def generate_random_name(sufix): + + nowTime = datetime.datetime.now().strftime("%Y%m%d%H%M%S") #generate the present time + randomNum = random.randint(0,100) + randomNum1 = random.randint(0,100) + Nm = nowTime + str(randomNum)+ str(randomNum1) + sufix + + return Nm + +def yyyymmdd2yyyyddd(date): + dt = datetime.datetime.strptime(date, "%Y%m%d") # get datetime object + day_of_year = (dt - datetime.datetime(dt.year, 1, 1)) # Jan the 1st is day 1 + doy = day_of_year.days + 1 + year = dt.year + + doy = str(doy) + if len(doy) ==1: + doy = '00' + doy + elif len(doy) ==2: + doy ='0'+ doy + year = str(year) + + return year, doy + +def get_txt_lines(txt): + count=0 + myfile=open(txt,"r") + for line in myfile: + count=count+1 + + return count + +def read_txt2list(txt): + A = np.loadtxt(txt,dtype=np.str) + if np.array(A).size ==1: + A = [A] + elif isinstance(A[0],bytes): + A = A.astype(str) + A = list(A) + return A + +def read_txt2array(txt): + A = np.loadtxt(txt,dtype=np.str) + if np.array(A).size ==1: + A = [A] + elif isinstance(A[0],bytes): + A = A.astype(str) + #A = list(A) + return A +def get_sardata_swath(start_swath,end_swath): + k_swath = '0' + if (start_swath == '1') and (end_swath == '1'): + k_swath = '1' + elif (start_swath == '2') and (end_swath == '2'): + k_swath = '2' + elif (start_swath == '3') and (end_swath == '3'): + k_swath = '3' + elif (start_swath == '1') and (end_swath == '2'): + k_swath = '4' + elif (start_swath == '2') and (end_swath == '3'): + k_swath = '5' + elif (start_swath == '2') and (end_swath == '3'): + k_swath = '-' + + return k_swath + +def get_filelist_filesize(url0): + ttt = generate_random_name('.txt') + call_str = 'curl -s ' + url0 + ' > ' + ttt + os.system(call_str) + A = read_txt2array(ttt) + + A_size = A[:,4] + A_size = A_size.astype(int) + A_size = A_size/1024/1024 #bytes to Mb + A_name = A[:,8] + if os.path.isfile(ttt): + os.remove(ttt) + + return A_name, A_size + +def yyyymmdd(date0): + if len(date0) ==6: + if float(date0[0:2]) > 90: + date1 = '19' + date0 + else: + date1 = '20' + date0 + elif len(date0) ==8: + date1 = date0 + else: + print('The input date is invalid!!') + date1 = '' + return date1 + +def yymmdd(date0): + if len(date0) ==6: + date1 = date0 + elif len(date0) ==8: + date1 = date0[2:8] + else: + print('The input date is invalid!!') + date1 = '' + return date1 + +def parallel_process(array, function, n_jobs=16, use_kwargs=False): + """ + A parallel version of the map function with a progress bar. + + Args: + array (array-like): An array to iterate over. + function (function): A python function to apply to the elements of array + n_jobs (int, default=16): The number of cores to use + use_kwargs (boolean, default=False): Whether to consider the elements of array as dictionaries of + keyword arguments to function + Returns: + [function(array[0]), function(array[1]), ...] + """ + #We run the first few iterations serially to catch bugs + #If we set n_jobs to 1, just run a list comprehension. This is useful for benchmarking and debugging. + if n_jobs==1: + return [function(**a) if use_kwargs else function(a) for a in tqdm(array[:])] + #Assemble the workers + with ProcessPoolExecutor(max_workers=n_jobs) as pool: + #Pass the elements of array into function + if use_kwargs: + futures = [pool.submit(function, **a) for a in array[:]] + else: + futures = [pool.submit(function, a) for a in array[:]] + kwargs = { + 'total': len(futures), + 'unit': 'it', + 'unit_scale': True, + 'leave': True + } + #Print out the progress as tasks complete + for f in tqdm(as_completed(futures), **kwargs): + pass + out = [] + #Get the results from the futures. + for i, future in tqdm(enumerate(futures)): + try: + out.append(future.result()) + except Exception as e: + out.append(e) + return out + +def sort_unique_list(numb_list): + list_out = sorted(set(numb_list)) + return list_out + + +############################# download data ################################### +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def download_s1_orbit(date,save_path,satellite='A'): + + DATE = date + if len(DATE)==6: + DATE = '20' + DATE + ST = satellite + YEAR = int(DATE[0:4]) + MON = int(DATE[4:6]) + DAY = int(DATE[6:8]) + + MON_DAY = [31,28,31,30,31,30,31,31,30,31,30,31] + + if YEAR%4==0: + MON_DAY[1]=29 + + if MON ==1 and DAY ==1: + DAY0 = 31 + MON0 = 12 + YEAR0 =YEAR -1 + elif MON!=1 and DAY ==1: + DAY0 = MON_DAY[MON-2] + MON0 = MON-1 + YEAR0 = YEAR + else: + DAY0 = DAY -1 + MON0 = MON + YEAR0 = YEAR + + MONDAY0 = MON_DAY[MON0-1] + + TT = [1,4,7,10,13,16,19,22,25,28,MONDAY0] + + T0 = [] + for k in range(len(TT)): + T0.append(TT[k]) + TT[k] = TT[k]-DAY0 + + for k in range(len(TT)): + if k == len(TT)-1: + if TT[k]==0: + ff = k-1 + else: + if TT[k]<=0 and TT[k+1]>0: + ff = k + + + DAY1 = T0[ff] + DAY2 = T0[ff+1] + + S1 = StrNum(YEAR0) + '-' + StrNum(MON0) + S2 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY1) + S3 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY2) + S4 = StrNum(YEAR0) + '-' + StrNum(MON0) + '-' + StrNum(DAY0) + + #SS = 'https://qc.sentinel1.eo.esa.int/aux_poeorb/?mission=S1' + ST + '&validity_start_time=' + StrNum(YEAR0) + '&validity_start_time=' + S1 + '&validity_start_time=' + S2 + '..' + S3 + '&validity_start_time=' + S4 + SS = 'https://qc.sentinel1.eo.esa.int/aux_poeorb/?validity_start=' + StrNum(YEAR0) + '&validity_start=' + S1 + '&validity_start=' + S2 + '..' + S3 + '&validity_start=' +S4 + '&sentinel1__mission=S1'+ST+ '&sentinel1_mission=S1'+ST+ '&sentinel1_mission=S1'+ST+ '&sentinel1_mission=S1'+ST + + tt ='tt_orb_' + date + tt0 ='tt0_orb_' + date + tt00 ='tt00_orb_' + date + tt000 ='tt000_orb_' + date + SS = "'" + SS + "'" + #print(SS) + call_str = 'curl -s -l ' + SS + ' > ' + tt + os.system(call_str) + + call_str = "grep 'EOF' -C 0 " + tt + " >" + tt0 + os.system(call_str) + + call_str="awk -F'href=' '{print $2}' " + tt0 +' >' + tt00 + os.system(call_str) + + call_str= "awk -F'>' '{print $1}' " + tt00 + '> ' + tt000 + os.system(call_str) + + SS=linecache.getline(tt000, 1) + SS = SS.split('"')[-1] + filename = os.path.basename(SS) + call_str = 'wget -q --no-check-certificate ' + SS + ' -O ' + save_path + '/' +filename + os.system(call_str) + call_str = "downlaod the precise Orbit of" + date + print(call_str) + + filename = os.path.basename(SS) + os.remove(tt) + os.remove(tt0) + os.remove(tt00) + os.remove(tt000) + + return filename + + +############################# write & read ##################################### +def copy_file(file0,file1): + call_str = 'cp ' + file0 + ' ' + file1 + os.system(call_str) + + return + +def get_project_slcList(projectName): + scratchDir = os.getenv('SCRATCHDIR') + slcDir = scratchDir + '/' + projectName + "/SLC" + #slcDir = scratchDir + '/SLC' + fn = os.listdir(slcDir) + slc_list0 = [os.path.basename(fname) for fname in sorted(fn)] + #slc_list0 = [os.path.basename(fname) for fname in sorted(glob.glob(slcDir + '/*'))] + + slc_list = [] + for k0 in slc_list0: + if is_number(k0): + slc_list.append(k0) + slc_list = sorted(slc_list) + + return slc_list + +def createBlankFile(strFile): + f = open(strFile,'w') + for i in range (10): + f.write('\n') + f.close() + +def read_attr(fname): + # read hdf5 + with h5py.File(fname, 'r') as f: + atr = dict(f.attrs) + + return atr + +def read_hdf5(fname, datasetName=None, box=None): + # read hdf5 + with h5py.File(fname, 'r') as f: + data = f[datasetName][:] + atr = dict(f.attrs) + + return data, atr + +def get_dataNames(FILE): + with h5py.File(FILE, 'r') as f: + dataNames = [] + for k0 in f.keys(): + dataNames.append(k0) + return dataNames + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def read_gamma_par(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close + + +def write_h5(datasetDict, out_file, metadata=None, ref_file=None, compression=None): + + if os.path.isfile(out_file): + print('delete exsited file: {}'.format(out_file)) + os.remove(out_file) + + print('create HDF5 file: {} with w mode'.format(out_file)) + dt = h5py.special_dtype(vlen=np.dtype('float64')) + + + with h5py.File(out_file, 'w') as f: + for dsName in datasetDict.keys(): + data = datasetDict[dsName] + ds = f.create_dataset(dsName, + data=data, + compression=compression) + + for key, value in metadata.items(): + f.attrs[key] = str(value) + #print(key + ': ' + value) + print('finished writing to {}'.format(out_file)) + + return out_file + +###################################################################### +class progressBar: + """Creates a text-based progress bar. Call the object with + the simple print command to see the progress bar, which looks + something like this: + [=======> 22% ] + You may specify the progress bar's min and max values on init. + + note: + modified from mintPy (https://github.com/insarlab/MintPy/wiki) + Code originally from http://code.activestate.com/recipes/168639/ + + example: + from mintpy.utils import ptime + date12_list = ptime.list_ifgram2date12(ifgram_list) + prog_bar = ptime.progressBar(maxValue=1000, prefix='calculating:') + for i in range(1000): + prog_bar.update(i+1, suffix=date) + prog_bar.update(i+1, suffix=date12_list[i]) + prog_bar.close() + """ + + def __init__(self, maxValue=100, prefix='', minValue=0, totalWidth=70, print_msg=True): + self.prog_bar = "[]" # This holds the progress bar string + self.min = minValue + self.max = maxValue + self.span = maxValue - minValue + self.suffix = '' + self.prefix = prefix + + self.print_msg = print_msg + ## calculate total width based on console width + #rows, columns = os.popen('stty size', 'r').read().split() + #self.width = round(int(columns) * 0.7 / 10) * 10 + self.width = totalWidth + self.reset() + + def reset(self): + self.start_time = time.time() + self.amount = 0 # When amount == max, we are 100% done + self.update_amount(0) # Build progress bar string + + def update_amount(self, newAmount=0, suffix=''): + """ Update the progress bar with the new amount (with min and max + values set at initialization; if it is over or under, it takes the + min or max value as a default. """ + if newAmount < self.min: + newAmount = self.min + if newAmount > self.max: + newAmount = self.max + self.amount = newAmount + + # Figure out the new percent done, round to an integer + diffFromMin = np.float(self.amount - self.min) + percentDone = (diffFromMin / np.float(self.span)) * 100.0 + percentDone = np.int(np.round(percentDone)) + + # Figure out how many hash bars the percentage should be + allFull = self.width - 2 - 18 + numHashes = (percentDone / 100.0) * allFull + numHashes = np.int(np.round(numHashes)) + + # Build a progress bar with an arrow of equal signs; special cases for + # empty and full + if numHashes == 0: + self.prog_bar = '%s[>%s]' % (self.prefix, ' '*(allFull-1)) + elif numHashes == allFull: + self.prog_bar = '%s[%s]' % (self.prefix, '='*allFull) + if suffix: + self.prog_bar += ' %s' % (suffix) + else: + self.prog_bar = '[%s>%s]' % ('='*(numHashes-1), ' '*(allFull-numHashes)) + # figure out where to put the percentage, roughly centered + percentPlace = int(len(self.prog_bar)/2 - len(str(percentDone))) + percentString = ' ' + str(percentDone) + '% ' + # slice the percentage into the bar + self.prog_bar = ''.join([self.prog_bar[0:percentPlace], + percentString, + self.prog_bar[percentPlace+len(percentString):]]) + # prefix and suffix + self.prog_bar = self.prefix + self.prog_bar + if suffix: + self.prog_bar += ' %s' % (suffix) + # time info - elapsed time and estimated remaining time + if percentDone > 0: + elapsed_time = time.time() - self.start_time + self.prog_bar += '%5ds / %5ds' % (int(elapsed_time), + int(elapsed_time * (100./percentDone-1))) + + def update(self, value, every=1, suffix=''): + """ Updates the amount, and writes to stdout. Prints a + carriage return first, so it will overwrite the current + line in stdout.""" + if value % every == 0 or value >= self.max: + self.update_amount(newAmount=value, suffix=suffix) + if self.print_msg: + sys.stdout.write('\r' + self.prog_bar) + sys.stdout.flush() + + def close(self): + """Prints a blank space at the end to ensure proper printing + of future statements.""" + if self.print_msg: + print(' ') diff --git a/third_party/PyINT/pyint/applygacos.py b/third_party/PyINT/pyint/applygacos.py new file mode 100644 index 0000000..e760869 --- /dev/null +++ b/third_party/PyINT/pyint/applygacos.py @@ -0,0 +1,305 @@ +#!/usr/bin/env python3 +import os,sys,struct,math +import numpy as np +import argparse +from pyint import _utils as ut +import subprocess + + +#if len(sys.argv) != 5: +# print("Usage:") +# print("./applygacos.py inpfilename ztd1filename ztd2filename elevfilename") +# print(" inpfilenmae : input interferogram, a inpfilename.rsc is needed!") +# print(" ztd1filename: input GACOS ztd day1 filename, a ztd1filename.rsc is needed!") +# print(" ztd2filename: input GACOS ztd day2 filename, a ztd2filename.rsc is needed!") +# print(" elevfilename: elevation angle file, must be the same size with interferogram") +# print(" elevation=90-incidence ") +# print(" can be generated by look_vector in gamma") +# print("outputfile will be saved as inpfilename.gacos") +# print("!!Note, python3 is needed!!") +# exit() + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister SM mode SLC to a reference SLC image using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='Name of project.') + parser.add_argument('mdate', help='date of the slave SLC image') + parser.add_argument('sdate', help='date of the slave SLC image') + parser.add_argument('ztd1', help='input GACOS ztd day1 filename, a ztd2filename.rsc is needed!') + parser.add_argument('ztd2', help='input GACOS ztd day2 filename, a ztd2filename.rsc is needed!' ) + inps = parser.parse_args() + return inps + +INTRODUCTION = ''' +------------------------------------------------------------------- + create_files for psokinv software running file by generated by Gamma +''' + +EXAMPLE = """Usage: + + coreg_gamma.py projectName + + coreg_gamma.py PacayaT163TsxHhA +------------------------------------------------------------------- +""" + + + + +#filename=sys.argv[1] +#ztd1filename=sys.argv[2] +#ztd2filename=sys.argv[3] +#elevfilename=sys.argv[4] +#print("start processing "+ filename) + +class HEADER: + width = 0 + length = 0 + xfirst = 0.0 + yfirst = 0.0 + xstep = 0.0 + ystep = 0.0 + + + + + +def read_header(filename): + if not os.path.isfile(filename): + print(filename+" file not exit") + + header = HEADER() + with open(filename) as f: + for line in f: + data=line.split() + if data[0] == "WIDTH": + header.width=int(data[1]) + if data[0] == "FILE_LENGTH": + header.length=int(data[1]) + if data[0] == "X_FIRST": + header.xfirst=float(data[1]) + if data[0] == "Y_FIRST": + header.yfirst=float(data[1]) + if data[0] == "X_STEP": + header.xstep=float(data[1]) + if data[0] == "Y_STEP": + header.ystep=float(data[1]) + + return header + + +def cut_image2(filename,headername,yfirst_new,length_new,xfirst_new,width_new): + header=read_header(headername) + + #print(yfirst_new, header.yfirst) + #print(length_new, header.length) + #print(xfirst_new, header.xfirst) + #print(width_new, header.width) + + with open(filename, 'rb') as f: + data0 = np.fromfile(f, dtype=np.float32) + data = np.reshape(data0, (header.length, header.width)) + + + out=np.zeros((length_new,width_new),dtype=np.float32) + #out=np.zeros(width_new*length_new,dtype=np.float32) + for i in range(header.length): + lat=header.yfirst+header.ystep*i + row=int(round((lat-yfirst_new)/header.ystep)) + if(row<0 or row>=length_new): + continue + for j in range(header.width): + lon=header.xfirst+header.xstep*j + col=int(round((lon-xfirst_new)/header.xstep)) + if(col<0 or col>=width_new): + continue + #out[width_new*row+col]=data[header.width*i+j] + out[row,col]=data[i,j] + + out=np.where(out==0,np.nan,out) + #print(np.nanstd(out)) + out.tofile(filename+".cut") + f = open(filename+".cut.rsc", 'w') + f.write("WIDTH "+str(width_new)+"\n") + f.write("FILE_LENGTH "+str(length_new)+"\n") + f.write("X_FIRST "+ str(xfirst_new)+"\n") + f.write("Y_FIRST "+ str(yfirst_new)+"\n") + f.write("X_STEP "+ str(header.xstep)+"\n") + f.write("Y_STEP "+ str(header.ystep)+"\n") + f.close() + + + + +def make_correction(phsfilename,ztd1filename,ztd2filename,elevfilename): + header=read_header(phsfilename+".rsc") + if not os.path.isfile(ztd1filename+".cut"): + cut_image2(ztd1filename,ztd1filename+".rsc",header.yfirst,header.length,header.xfirst,header.width) + if not os.path.isfile(ztd2filename+".cut"): + cut_image2(ztd2filename,ztd2filename+".rsc",header.yfirst,header.length,header.xfirst,header.width) + + with open(phsfilename, 'rb') as f: + data = np.fromfile(f, dtype=np.float32) + phase = np.reshape(data, [header.length, header.width]) + + with open(ztd1filename+".cut", 'rb') as f: + data = np.fromfile(f, dtype=np.float32) + ztd1 = np.reshape(data, [header.length, header.width]) + + with open(ztd2filename+".cut", 'rb') as f: + data = np.fromfile(f, dtype=np.float32) + ztd2 = np.reshape(data, [header.length, header.width]) + + with open(elevfilename, 'rb') as f: + data = np.fromfile(f, dtype=np.float32) + elev = np.reshape(data, [header.length, header.width]) + + #os.remove(ztd1filename+".cut") + #os.remove(ztd2filename+".cut") + dztd=ztd2-ztd1 + dztd=dztd/0.0044138251819503 + dztd=dztd/np.sin(elev) + index=np.where(phase==0) + phase[index]=np.nan + phasemean=np.nanmean(phase) + print("before "+str(np.nanstd(phase))) + phase=phase-phasemean + #phase.tofile(phsfilename+".raw") + + + phase=phase-dztd + phase[index]=np.nan + phasemean=np.nanmean(phase) + print("after "+str(np.nanstd(phase))) + phase=phase-phasemean + phase[index]=0 + phase.tofile(phsfilename+".gacos") + +def main(argv): + inps = cmdLineParse() + projectName = inps.projectName + Sdate = inps.sdate + Mdate = inps.mdate + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Smdate = templateDict['masterDate'] + + demDir = scratchDir + '/' + projectName + '/DEM' + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + ifgramDir = scratchDir + '/' + projectName + "/ifgrams" + workDir = ifgramDir + '/' + Mdate + '-' + Sdate + MslcDir = rslcDir + '/' + Mdate + SslcDir = rslcDir + '/' + Sdate + offpar = SslcDir + '/' + Smdate + '_' + Sdate + '.off' + slcpar = MslcDir + '/' + Smdate + '.rslc.par' + dempar = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem.par' + dem = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem' + unwpar = demDir + '/' + Mdate + '_' + rlks + 'rlks.utm.dem.par' + unw = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.diff_filt.unw" + + os.chdir(workDir) + call_str = "look_vector " + slcpar+ " " + offpar + " " + dempar + " " + dem + ' lv_theta lv_phi' + os.system(call_str) + call_str = "grep 'corner_lat:' " + dempar + " | awk '{print $2}' " + North=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'corner_lon:' " + dempar + " | awk '{print $2}' " + West=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lat:' " + dempar + " | awk '{print $2}' " + posty=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lon:' " + dempar + " | awk '{print $2}' " + postx=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'width:' " + dempar + " | awk '{print $2}' " + width=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'nlines:' " + dempar + " | awk '{print $2}' " + length=subprocess.getstatusoutput(call_str)[1] + South=str(round(float(North)+(float(length)-1)*float(posty),7)) + East=str(round(float(West)+(float(width)-1)*float(postx),7)) + + call_str = "swap_bytes lv_theta lv_theta.phase_swap 4 >dinsar.log " + os.system(call_str) + call_str = "gmt xyz2grd lv_theta.phase_swap -Glv_theta.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -N0" + os.system(call_str) + call_str = "gmt grdmath 90 lv_theta.grd 3.1415926 DIV 180 MUL SUB = lv_theta_final.grd" + os.system(call_str) + call_str = "gmt grdmath 90 lv_theta_final.grd SUB = lv_elev.grd" + os.system(call_str) + call_str = "gmt grdsample lv_elev.grd -Glv_elev_3c.grd -I3c" + os.system(call_str) + call_str = "gmt grd2xyz lv_elev_3c.grd -ZTLf -N0 > " + Mdate + '-' + Sdate + '.gacos.elev' + os.system(call_str) + + + call_str = "swap_bytes " + unw+ " unw.phase_swap 4 >dinsar.log " + os.system(call_str) + call_str = "gmt xyz2grd unw.phase_swap -Gunw_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx +" -ZTLf -N0" + os.system(call_str) + call_str = "gmt grdsample unw_f.grd -Gunw.grd -I3c" + os.system(call_str) + call_str = "gmt grd2xyz unw.grd -ZTLf > " + Mdate + '-' + Sdate + '.gacos.unw' + os.system(call_str) + call_str = "gmt grdinfo unw.grd -C | awk '{print $10}' " + Width=subprocess.getstatusoutput(call_str)[1] + call_str = "gmt grdinfo unw.grd -C | awk '{print $11}' " + line=subprocess.getstatusoutput(call_str)[1] + call_str = "gmt grdinfo unw.grd -C | awk '{print $2}' " + West=subprocess.getstatusoutput(call_str)[1] + call_str = "gmt grdinfo unw.grd -C | awk '{print $5}' " + North=subprocess.getstatusoutput(call_str)[1] + ymax=str(int(round(float(line) ,7))) + xmax=str(int(round(float(Width) ,7))) + output=workDir + '/' + Mdate + '-' + Sdate + '.gacos.unw.rsc' + if os.path.exists(output) is True: + os.remove(output) + + fopen=open(output,'a+') + fopen.write('WIDTH ' + Width + '\n') + fopen.write('FILE_LENGTH ' + line + '\n') + fopen.write('XMIN 1' + '\n') + fopen.write('XMAX ' + xmax + '\n') + fopen.write('YMIN 1' + '\n') + fopen.write('YMAX ' + ymax + '\n') + fopen.write('X_FIRST ' + West + '\n') + fopen.write('Y_FIRST ' + North + '\n') + fopen.write('X_STEP 8.33333333E-04' + '\n') + fopen.write('Y_STEP -8.33333333E-04' + '\n') + fopen.write('X_UNIT degrees' + '\n') + fopen.write('Y_UNIT degrees' + '\n') + fopen.write('Z_OFFSET 0' + '\n') + fopen.write('Z_SCALE 1' + '\n') + fopen.write('PROJECTION LATLON' + '\n') + fopen.write('DATUM WGS84' + '\n') + print("convert gamma products to GACOS is done!") + + print("start make gacos correction!") + filename=workDir + '/' + Mdate + '-' + Sdate + '.gacos.unw' + ztd1=inps.ztd1 + ztd2=inps.ztd2 + ztd1filename=workDir + '/' + ztd1 + ztd2filename=workDir + '/' + ztd2 + elevfilename=workDir + '/' + Mdate + '-' + Sdate + '.gacos.elev' + make_correction(filename,ztd1filename,ztd2filename,elevfilename) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + +#make_correction(filename,ztd1filename,ztd2filename,elevfilename) + + + + +#exit() + + diff --git a/third_party/PyINT/pyint/applygacos1.py b/third_party/PyINT/pyint/applygacos1.py new file mode 100644 index 0000000..e8e48e0 --- /dev/null +++ b/third_party/PyINT/pyint/applygacos1.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +import os,sys,struct,math +import numpy as np +import argparse +from pyint import _utils as ut +import subprocess + + +#if len(sys.argv) != 5: +# print("Usage:") +# print("./applygacos.py inpfilename ztd1filename ztd2filename elevfilename") +# print(" inpfilenmae : input interferogram, a inpfilename.rsc is needed!") +# print(" ztd1filename: input GACOS ztd day1 filename, a ztd1filename.rsc is needed!") +# print(" ztd2filename: input GACOS ztd day2 filename, a ztd2filename.rsc is needed!") +# print(" elevfilename: elevation angle file, must be the same size with interferogram") +# print(" elevation=90-incidence ") +# print(" can be generated by look_vector in gamma") +# print("outputfile will be saved as inpfilename.gacos") +# print("!!Note, python3 is needed!!") +# exit() + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister SM mode SLC to a reference SLC image using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='Name of project.') + parser.add_argument('mdate', help='date of the slave SLC image') + parser.add_argument('sdate', help='date of the slave SLC image') + parser.add_argument('ztd1', help='input GACOS ztd day1 filename, a ztd2filename.rsc is needed!') + parser.add_argument('ztd2', help='input GACOS ztd day2 filename, a ztd2filename.rsc is needed!' ) + inps = parser.parse_args() + return inps + +INTRODUCTION = ''' +------------------------------------------------------------------- + create_files for psokinv software running file by generated by Gamma +''' + +EXAMPLE = """Usage: + + coreg_gamma.py projectName + + coreg_gamma.py PacayaT163TsxHhA +------------------------------------------------------------------- +""" + + + + +#filename=sys.argv[1] +#ztd1filename=sys.argv[2] +#ztd2filename=sys.argv[3] +#elevfilename=sys.argv[4] +#print("start processing "+ filename) + +class HEADER: + width = 0 + length = 0 + xfirst = 0.0 + yfirst = 0.0 + xstep = 0.0 + ystep = 0.0 + + + + + +def read_header(filename): + if not os.path.isfile(filename): + print(filename+" file not exit") + + header = HEADER() + with open(filename) as f: + for line in f: + data=line.split() + if data[0] == "WIDTH": + header.width=int(data[1]) + if data[0] == "FILE_LENGTH": + header.length=int(data[1]) + if data[0] == "X_FIRST": + header.xfirst=float(data[1]) + if data[0] == "Y_FIRST": + header.yfirst=float(data[1]) + if data[0] == "X_STEP": + header.xstep=float(data[1]) + if data[0] == "Y_STEP": + header.ystep=float(data[1]) + + return header + + +def cut_image2(filename,headername,yfirst_new,length_new,xfirst_new,width_new): + header=read_header(headername) + + #print(yfirst_new, header.yfirst) + #print(length_new, header.length) + #print(xfirst_new, header.xfirst) + #print(width_new, header.width) + + with open(filename, 'rb') as f: + data0 = np.fromfile(f, dtype=np.float32) + data = np.reshape(data0, (header.length, header.width)) + + + out=np.zeros((length_new,width_new),dtype=np.float32) + #out=np.zeros(width_new*length_new,dtype=np.float32) + for i in range(header.length): + lat=header.yfirst+header.ystep*i + row=int(round((lat-yfirst_new)/header.ystep)) + if(row<0 or row>=length_new): + continue + for j in range(header.width): + lon=header.xfirst+header.xstep*j + col=int(round((lon-xfirst_new)/header.xstep)) + if(col<0 or col>=width_new): + continue + #out[width_new*row+col]=data[header.width*i+j] + out[row,col]=data[i,j] + + out=np.where(out==0,np.nan,out) + #print(np.nanstd(out)) + out.tofile(filename+".cut") + f = open(filename+".cut.rsc", 'w') + f.write("WIDTH "+str(width_new)+"\n") + f.write("FILE_LENGTH "+str(length_new)+"\n") + f.write("X_FIRST "+ str(xfirst_new)+"\n") + f.write("Y_FIRST "+ str(yfirst_new)+"\n") + f.write("X_STEP "+ str(header.xstep)+"\n") + f.write("Y_STEP "+ str(header.ystep)+"\n") + f.write("X_UNIT degree"+"\n") + f.write("Y_UNIT degree"+"\n") + f.write("Z_OFFSET 0"+"\n") + f.write("Z_SCALE 1"+"\n") + f.write("PROJECTION LATLON"+"\n") + f.write("DATUM WGS84"+"\n") + f.close() + +def make_correction(phsfilename,ztd1filename,ztd2filename,elevfilename): + header=read_header(phsfilename+".rsc") + #if not os.path.isfile(ztd1filename+".cut"): + cut_image2(ztd1filename,ztd1filename+".rsc",header.yfirst,header.length,header.xfirst,header.width) + #if not os.path.isfile(ztd2filename+".cut"): + cut_image2(ztd2filename,ztd2filename+".rsc",header.yfirst,header.length,header.xfirst,header.width) + + with open(phsfilename, 'rb') as f: + data = np.fromfile(f, dtype=np.float32) + phase = np.reshape(data, [header.length, header.width]) + + with open(ztd1filename+".cut", 'rb') as f: + data = np.fromfile(f, dtype=np.float32) + ztd1 = np.reshape(data, [header.length, header.width]) + + with open(ztd2filename+".cut", 'rb') as f: + data = np.fromfile(f, dtype=np.float32) + ztd2 = np.reshape(data, [header.length, header.width]) + + with open(elevfilename, 'rb') as f: + data = np.fromfile(f, dtype=np.float32) + elev = np.reshape(data, [header.length, header.width]) + + #os.remove(ztd1filename+".cut") + #os.remove(ztd2filename+".cut") + dztd=ztd2-ztd1 + #dztd=dztd/0.0044138251819503 + dztd=dztd/np.sin(elev) + index=np.where(phase==0) + phase[index]=np.nan + phasemean=np.nanmean(phase) + print("before "+str(np.nanstd(phase))) + phase=phase-phasemean + #phase.tofile(phsfilename+".raw") + + + phase=phase-dztd + phase[index]=np.nan + phasemean=np.nanmean(phase) + print("after "+str(np.nanstd(phase))) + phase=phase-phasemean + phase[index]=0 + phase.tofile(phsfilename+".gacos") + +def main(argv): + inps = cmdLineParse() + projectName = inps.projectName + Sdate = inps.sdate + Mdate = inps.mdate + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Smdate = templateDict['masterDate'] + + demDir = scratchDir + '/' + projectName + '/DEM' + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + ifgramDir = scratchDir + '/' + projectName + "/ifgrams" + workDir = ifgramDir + '/' + Mdate + '-' + Sdate + MslcDir = rslcDir + '/' + Mdate + SslcDir = rslcDir + '/' + Sdate + offpar = SslcDir + '/' + Smdate + '_' + Sdate + '.off' + slcpar = MslcDir + '/' + Smdate + '.rslc.par' + dempar = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem.par' + dem = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem' + unwpar = demDir + '/' + Mdate + '_' + rlks + 'rlks.utm.dem.par' + unw = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.diff_filt.unw" + + os.chdir(workDir) + call_str = "look_vector " + slcpar+ " " + offpar + " " + dempar + " " + dem + ' lv_theta lv_phi' + os.system(call_str) + call_str = "grep 'corner_lat:' " + dempar + " | awk '{print $2}' " + North=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'corner_lon:' " + dempar + " | awk '{print $2}' " + West=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lat:' " + dempar + " | awk '{print $2}' " + posty=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lon:' " + dempar + " | awk '{print $2}' " + postx=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'width:' " + dempar + " | awk '{print $2}' " + width=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'nlines:' " + dempar + " | awk '{print $2}' " + length=subprocess.getstatusoutput(call_str)[1] + South=str(round(float(North)+(float(length)-1)*float(posty),7)) + East=str(round(float(West)+(float(width)-1)*float(postx),7)) + + call_str = "swap_bytes lv_theta lv_theta.phase_swap 4 >dinsar.log " + os.system(call_str) + call_str = "xyz2grd lv_theta.phase_swap -Glv_theta.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -N0" + os.system(call_str) + call_str = "grdmath 90 lv_theta.grd 3.1415926 DIV 180 MUL SUB = lv_theta_final.grd" + os.system(call_str) + call_str = "grdmath 90 lv_theta_final.grd SUB = lv_elev.grd" + os.system(call_str) + call_str = "grdsample lv_elev.grd -Glv_elev_3c.grd -I3c" + # call_str = "grdsample lv_theta_final.grd -Glv_elev_3c.grd -I3c" + os.system(call_str) + call_str = "grd2xyz lv_elev_3c.grd -ZTLf -N0 > " + Mdate + '-' + Sdate + '.gacos.elev' + os.system(call_str) + + + call_str = "swap_bytes " + unw+ " unw.phase_swap 4 >dinsar.log " + os.system(call_str) + call_str = "xyz2grd unw.phase_swap -Gunw_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx +" -ZTLf -N0" + os.system(call_str) + call_str = "grdsample unw_f.grd -Gunw.grd -I3c" + os.system(call_str) + call_str = "grd2xyz unw.grd -ZTLf -N0 > " + Mdate + '-' + Sdate + '.gacos.unw' + os.system(call_str) + call_str = "grdinfo unw.grd -C | awk '{print $10}' " + Width=subprocess.getstatusoutput(call_str)[1] + call_str = "grdinfo unw.grd -C | awk '{print $11}' " + line=subprocess.getstatusoutput(call_str)[1] + call_str = "grdinfo unw.grd -C | awk '{print $2}' " + West=subprocess.getstatusoutput(call_str)[1] + call_str = "grdinfo unw.grd -C | awk '{print $5}' " + North=subprocess.getstatusoutput(call_str)[1] + ymax=str(int(round(float(line) ,7))) + xmax=str(int(round(float(Width) ,7))) + output=workDir + '/' + Mdate + '-' + Sdate + '.gacos.unw.rsc' + if os.path.exists(output) is True: + os.remove(output) + + fopen=open(output,'a+') + fopen.write('WIDTH ' + Width + '\n') + fopen.write('FILE_LENGTH ' + line + '\n') + fopen.write('XMIN 1' + '\n') + fopen.write('XMAX ' + xmax + '\n') + fopen.write('YMIN 1' + '\n') + fopen.write('YMAX ' + ymax + '\n') + fopen.write('X_FIRST ' + West + '\n') + fopen.write('Y_FIRST ' + North + '\n') + fopen.write('X_STEP 8.33333333E-04' + '\n') + fopen.write('Y_STEP -8.33333333E-04' + '\n') + fopen.write('X_UNIT degrees' + '\n') + fopen.write('Y_UNIT degrees' + '\n') + fopen.write('Z_OFFSET 0' + '\n') + fopen.write('Z_SCALE 1' + '\n') + fopen.write('PROJECTION LATLON' + '\n') + fopen.write('DATUM WGS84' + '\n') + fopen.close() + print("convert gamma products to GACOS is done!") + + print("start make gacos correction!") + filename=workDir + '/' + Mdate + '-' + Sdate + '.gacos.unw' + ztd1=inps.ztd1 + ztd2=inps.ztd2 + ztd1filename=workDir + '/' + ztd1 + ztd2filename=workDir + '/' + ztd2 + elevfilename=workDir + '/' + Mdate + '-' + Sdate + '.gacos.elev' + make_correction(filename,ztd1filename,ztd2filename,elevfilename) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + +#make_correction(filename,ztd1filename,ztd2filename,elevfilename) + + + + +#exit() + + diff --git a/third_party/PyINT/pyint/atm_correction_gamma.py b/third_party/PyINT/pyint/atm_correction_gamma.py new file mode 100644 index 0000000..169bad0 --- /dev/null +++ b/third_party/PyINT/pyint/atm_correction_gamma.py @@ -0,0 +1,105 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +from PIL import Image +from pylab import * +import argparse + +from pyint import _utils as ut + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Unwrap differential interferogram using GAMMA. + [Only support mcf, not implement branch_cut yet] + +''' + +EXAMPLE = ''' + Usage: + unwrap_gamma.py projectName Mdate Sdate + unwrap_gamma.py PacayaT163TsxHhA 20150102 20150601 +------------------------------------------------------------------- +''' + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Unwrap differential interferogram using GAMMA-mcf method.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('Mdate',help='Master date.') + parser.add_argument('Sdate',help='Slave date.') + + inps = parser.parse_args() + return inps + + +def main(argv): + + inps = cmdLineParse() + Mdate = inps.Mdate + Sdate = inps.Sdate + + projectName = inps.projectName + Sdate = inps.Sdate + Mdate = inps.Mdate + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + auto_unw = templateDict['auto_unw'] + init_flag = templateDict['init_flag'] + r_refer = templateDict['r_refer'] + a_refer = templateDict['a_refer'] + make_mask = templateDict['make_mask'] + processDir = scratchDir + '/' + projectName + "/PROCESS" + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + '/RSLC' + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + masterDate = templateDict['masterDate'] + demDir = scratchDir + '/' + projectName + '/DEM' + Pair = Mdate + '-' + Sdate + workDir = ifgDir + '/' + Pair + + ################ prepare file for parallel processing ############### + HGTSIM = demDir + '/' + masterDate + '_' + rlks + 'rlks.rdc.dem' + Mamp = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + Samp = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + diff_par = workDir + '/' + Mdate + '_' + Sdate + '_diff_par' + UNWlks = workDir + '/' + Pair + '_' +rlks + 'rlks.diff_filt.unw' + CORMASK = workDir + '/' + Pair + '_' +rlks + 'rlks.diff_filt.cor' + ATM_PHASE = workDir + '/' + Pair + '_' +rlks + 'rlks.atm_phae' + ATMCOR_UNW = workDir + '/' + Pair + '_' +rlks + 'rlks.diff_filt.atmcor.unw' + + nWidth = ut.read_gamma_par(MampPar, 'read', 'range_samples') + nLine = ut.read_gamma_par(MampPar, 'read', 'azimuth_lines') + ############################################################### + call_str = "create_diff_par " + MampPar + " " + SampPar + " " + diff_par + " 1 0 " + os.system(call_str) + call_str = "atm_mod_2d " + UNWlks + " " + HGTSIM + " " + CORMASK + " " + diff_par + " " + " - 0 a0 a1 sigma sigma_h s1" + os.system(call_str) + call_str = "atm_sim_2d " + diff_par + " " + HGTSIM + " a0 a1 " + ATM_PHASE + os.system(call_str) + call_str = "sub_phase " + UNWlks + " " + ATM_PHASE + " " + diff_par + " " + ATMCOR_UNW + " 0 0 0 " + os.system(call_str) + call_str = 'rasrmg ' + ATMCOR_UNW + ' ' + Mamp + ' ' + nWidth + ' - - - - - - - - - - ' + os.system(call_str) + + print("Correct atmospheric phase is done!") + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/atm_correction_gamma_all.py b/third_party/PyINT/pyint/atm_correction_gamma_all.py new file mode 100644 index 0000000..5ad8a59 --- /dev/null +++ b/third_party/PyINT/pyint/atm_correction_gamma_all.py @@ -0,0 +1,121 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + + +def work(data0): + cmd = data0[0] + err_file = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + cmd[3] + '\n' + #print(aa) + with open(err_file, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + Correct atmospheric phase for one project using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + atm_correction_gamma_all.py projectName + atm_correction_gamma_all.py projectName --parallel 4 + atm_correction_gamma_all.py projectName --parallel 4 --ifgramList-txt /test/ifgram_list.txt +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Unwrap differential interferograms for one project using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + parser.add_argument('--ifgarmList-txt', dest='ifgarmListTxt', help='provided ifgram_list_txt. default: using ifgram_list.txt under projectName folder.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + projectDir = scratchDir + '/' + projectName + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + if inps.ifgarmListTxt: ifgramList_txt = inps.ifgarmListTxt + else: ifgramList_txt = scratchDir + '/' + projectName + '/ifgram_list.txt' + ifgList0 = ut.read_txt2array(ifgramList_txt) +# ifgList = ifgList0[:,0] + if len(ifgList0)==3: + ifgList=ifgList0[0] + ifgList=[ifgList] + else: + ifgList=ifgList0[:,0] + + err_txt = scratchDir + '/' + projectName + '/unwrap_gamma_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + for i in range(len(ifgList)): + m0 = ut.yyyymmdd(ifgList[i].split('-')[0]) + s0 = ut.yyyymmdd(ifgList[i].split('-')[1]) + cmd0 = ['atm_correction_gamma.py',projectName, m0, s0] + unw_file0 = ifgDir + '/' + ifgList[i] + '/' + ifgList[i] + '_' + rlks + 'rlks.diff_filt.atmcor.unw.bmp' + data0 = [cmd0,err_txt] + + k00 = 0 + if os.path.isfile(unw_file0): + if os.path.getsize(unw_file0) > 0: + k00 = 1 + if k00==0: + data_para.append(data0) + + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + print("Correct atmospheric phase for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/third_party/PyINT/pyint/change_Name_for_mintpy.py b/third_party/PyINT/pyint/change_Name_for_mintpy.py new file mode 100644 index 0000000..271157f --- /dev/null +++ b/third_party/PyINT/pyint/change_Name_for_mintpy.py @@ -0,0 +1,135 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Central South University & University of Miami ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + + +def is_number(s): + try: + int(s) + return True + except ValueError: + return False + +def add_zero(s): + if len(s)==1: + s="000"+s + elif len(s)==2: + s="00"+s + elif len(s)==3: + s="0"+s + return s + +def usage(): + print(''' +****************************************************************************************************** + + Select interferometry pairs from time series SAR images + + usage: + + Generate_IfgDir.py ProjectName IFG_List + + e.g. Generate_IfgDir.py PacayaT163TsxHhA ifg_list + + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==2: + if argv[0] in ['-h','--help']: usage(); sys.exit(1) + else: + projectName=sys.argv[1] + projectName=sys.argv[1] + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + processDir = scratchDir + '/' + projectName + "/ifgrams" + root_dir=os.getcwd() + os.chdir(processDir) + call_str="find . -name '[0-9]*.unw' -print >unw_file" + os.system(call_str) + file_obj=open('unw_file') + all_lines=file_obj.readlines() + for line in all_lines: + unwdate1=line.split("/")[2] + unwdate2=line.split("/")[1] + unwdate3=line.split("/")[0] + str1=unwdate1.split(".")[0] + str2=unwdate1.split(".")[1] + str3=unwdate1.split(".")[2] + unwdate4=str2 + '_' + str1 + '.' + str3 + line1=processDir + '/' + unwdate2+ '/' + unwdate4 + line2=processDir + '/'+ unwdate2 + '/' + unwdate1 + line2=line2.split('\n')[0] + line1=line1.split('\n')[0] + call_str='mv'+ ' ' + line2 + ' ' + line1 + os.system(call_str) + call_str="find . -name '[0-9]*.cor' -print >cor_file" + os.system(call_str) + + file_obj=open('cor_file') + all_lines=file_obj.readlines() + for line in all_lines: + cordate1=line.split("/")[2] + cordate2=line.split("/")[1] + cordate3=line.split("/")[0] + str1=cordate1.split(".")[0] + str2=cordate1.split(".")[1] + str3=cordate1.split(".")[2] + sub1=str2.split('_')[1] + cordate4=sub1 + '_' + str1 + '.' + str3 + line1=processDir + '/' + cordate2 + '/' + cordate4 + line1=line1.split('\n')[0] + line2=processDir + '/'+ cordate2 + '/' + cordate1 + line2=line2.split('\n')[0] + call_str='mv' + ' ' + line2 + ' ' + line1 + os.system(call_str) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + diff --git a/third_party/PyINT/pyint/convert_phs_to_grd.csh b/third_party/PyINT/pyint/convert_phs_to_grd.csh new file mode 100644 index 0000000..ec8da05 --- /dev/null +++ b/third_party/PyINT/pyint/convert_phs_to_grd.csh @@ -0,0 +1,23 @@ +#!/bin/csh -f + +if ($#argv != 2) then + echo "" + echo "Usage: read_phs_file.csh phase_file header.rsc" + echo "" + echo "Convert binary files to .grd file" + echo "" + echo "binary file and header in .rsc is needed" + exit 1 + endif + +#######File to grid######### +set x_first_d1 = `cat $2|grep X_FIRST|awk '{print $2}'` +set y_first_d1 = `cat $2|grep Y_FIRST|awk '{print $2}'` +set width_d1 = `cat $2|grep WIDTH|awk '{print $2}'` +set length_d1 = `cat $2|grep FILE_LENGTH|awk '{print $2}'` +set x_step_d1 = `cat $2|grep X_STEP|awk '{print $2}'` +set y_step_d1 = `cat $2|grep X_STEP|awk '{print $2}'` +set name = `echo $1| awk -F/ '{print $NF}'` +gmt xyz2grd $1 -G"$name.grd" -RLT$x_first_d1/$y_first_d1/$width_d1/$length_d1 -I$x_step_d1/$y_step_d1 -ZTLf -di0 -r + +echo "From $1, $name.grd has been produced" diff --git a/third_party/PyINT/pyint/coreg_gamma.py b/third_party/PyINT/pyint/coreg_gamma.py new file mode 100644 index 0000000..0b0fae8 --- /dev/null +++ b/third_party/PyINT/pyint/coreg_gamma.py @@ -0,0 +1,242 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import argparse + +from pyint import _utils as ut + + +def _run_or_raise(call_str, stage): + rc = os.system(call_str) + if rc != 0: + raise RuntimeError('%s failed with rc=%s: %s' % (stage, rc, call_str)) + return rc + + +def _safe_remove(path): + if path and os.path.exists(path): + os.remove(path) + + +def _copy_file(src, dst): + ut.copy_file(src, dst) + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister SM mode SLC to a reference SLC image using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='Name of project.') + parser.add_argument('sdate', help='date of the slave SLC image. [mater date is read from template]') + inps = parser.parse_args() + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Coregister SM mode SLC to a reference SLC image using GAMMA. + [The reference date or master date will be read from the template file.] +''' + +EXAMPLE = """Usage: + + coreg_gamma.py projectName Sdate + + coreg_gamma.py PacayaT163TsxHhA 20150102 +------------------------------------------------------------------- +""" + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Sdate = inps.sdate + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + if not os.path.isdir(rslcDir): os.mkdir(rslcDir) + #workDir = processDir + '/' + igramDir + workDir = rslcDir + '/' + Sdate + if not os.path.isdir(workDir): os.mkdir(workDir) + + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Mdate = templateDict['masterDate'] + IFGPair = Mdate + '-' + Sdate + + demDir = scratchDir + '/' + projectName + '/DEM' + + SslcDir = slcDir + '/' + Sdate + Samp = slcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar = slcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + Sramp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + SrampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + Sslc = slcDir + '/' + Sdate + '/' + Sdate + '.slc' + SslcPar = slcDir + '/' + Sdate + '/' + Sdate + '.slc.par' + + SrslcDir = rslcDir + "/" + Sdate + + Srslc = SrslcDir + "/" + Sdate + ".rslc" + SrslcPar = SrslcDir + "/" + Sdate + ".rslc.par" + + Srslc0 = SrslcDir + "/" + Sdate + ".rslc0" + SrslcPar0 = SrslcDir + "/" + Sdate + ".rslc0.par" + +##################################################### +## copy all of the master files into slave folder for parallel processing + remove_file = [] + Mslc0 = slcDir + '/' + Mdate + '/' + Mdate + '.slc' + MslcPar0 = slcDir + '/' + Mdate + '/' + Mdate + '.slc.par' + Mamp0 = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar0 = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + Mamp0_dem = demDir + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar0_dem = demDir + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + HGTSIM0 = demDir + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' + if not os.path.isfile(HGTSIM0): + call_str = 'generate_rdc_dem.py ' + projectName + _run_or_raise(call_str, 'generate_rdc_dem') + if not os.path.isfile(Mamp0) and os.path.isfile(Mamp0_dem): + ut.copy_file(Mamp0_dem, Mamp0) + if not os.path.isfile(MampPar0) and os.path.isfile(MampPar0_dem): + ut.copy_file(MampPar0_dem, MampPar0) + if not os.path.isfile(Mamp0) or not os.path.isfile(MampPar0): + call_str = 'multi_look ' + Mslc0 + ' ' + MslcPar0 + ' ' + Mamp0 + ' ' + MampPar0 + ' ' + rlks + ' ' + azlks + _run_or_raise(call_str, 'master_multi_look') + if not os.path.isfile(Samp) or not os.path.isfile(SampPar): + call_str = 'multi_look ' + Sslc + ' ' + SslcPar + ' ' + Samp + ' ' + SampPar + ' ' + rlks + ' ' + azlks + _run_or_raise(call_str, 'slave_multi_look') + + Mslc = slcDir + '/' + Sdate + '/' + Mdate + '.slc' + MslcPar = slcDir + '/' + Sdate + '/' + Mdate + '.slc.par' + Mamp = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + HGTSIM = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' + + ut.copy_file(HGTSIM0,HGTSIM) + ut.copy_file(Mslc0,Mslc) + ut.copy_file(MslcPar0,MslcPar) + ut.copy_file(Mamp0,Mamp) + ut.copy_file(MampPar0,MampPar) + +####################################################### +# define process files # + lt0 = workDir + "/lt0" + lt1 = workDir + "/lt1" + mli0 = workDir + "/mli0" + diff0 = workDir + "/diff0" + offs0 = workDir + "/offs0" + snr0 = workDir + "/snr0" + offsets0 = workDir + "/offsets0" + coffs0 = workDir + "/coffs0" + coffsets0 = workDir + "/coffsets0" + off = workDir + "/" + IFGPair + ".off" + offs = workDir + "/offs" + snr = workDir + "/snr" + offsets = workDir + "/offsets" + coffs = workDir + "/coffs" + coffsets = workDir + "/coffsets" + OFFSTD = workDir + "/" + IFGPair + ".off_std" +############################################## + + call_str = "rdc_trans " + MampPar + " " + HGTSIM + " " + SampPar + " " + lt0 + _run_or_raise(call_str, 'rdc_trans') + + width_Mamp = ut.read_gamma_par(MampPar, 'read', 'range_samples') + width_Samp = ut.read_gamma_par(SampPar, 'read', 'range_samples') + line_Samp = ut.read_gamma_par(SampPar, 'read', 'azimuth_lines') + + call_str = "geocode " + lt0 + " " + Mamp + " " + width_Mamp + " " + mli0 + " " + width_Samp + " " + line_Samp + " 2 0" + _run_or_raise(call_str, 'geocode_lt0') + + call_str = "create_diff_par " + SampPar + " - " + diff0 + " 1 0" + _run_or_raise(call_str, 'create_diff_par') + + try: + call_str = "init_offsetm " + mli0 + " " + Samp + " " + diff0 + " 1 1" + _run_or_raise(call_str, 'init_offsetm') + + call_str = "offset_pwrm " + mli0 + " " + Samp + " " + diff0 + " " + offs0 + " " + snr0 + " 256 256 " + offsets0 + " 2 32 32" + _run_or_raise(call_str, 'offset_pwrm') + + call_str = "offset_fitm " + offs0 + " " + snr0 + " " + diff0 + " " + coffs0 + " " + coffsets0 + " - 4" + _run_or_raise(call_str, 'offset_fitm') + + call_str = "gc_map_fine " + lt0 + " " + width_Mamp + " " + diff0 + " " + lt1 + _run_or_raise(call_str, 'gc_map_fine') + except RuntimeError as exc: + print('ERROR: initial DEM-assisted offset refinement failed; aborting coregistration instead of falling back') + print(str(exc)) + raise + + + call_str = "SLC_interp_lt " + Sslc + " " + MslcPar + " " + SslcPar + " " + lt1 + " " + MampPar + " " + SampPar + " - " + Srslc0 + " " + SrslcPar0 + _run_or_raise(call_str, 'SLC_interp_lt_initial') + + +############################################ Refinement ############################################ + try: + call_str = "create_offset " + MslcPar + " " + SrslcPar0 + " " + off + " 1 - - 0" + _run_or_raise(call_str, 'create_offset') + + call_str = "offset_pwr " + Mslc + " " + Srslc0 + " " + MslcPar + " " + SrslcPar0 + " " + off + " " + offs + " " + snr + " " + templateDict['rwin4cor'] + " " + templateDict['azwin4cor'] + " " + offsets + " 2 " + templateDict['rsample4cor'] + " " + templateDict['azsample4cor'] + _run_or_raise(call_str, 'offset_pwr_coarse') + + call_str = "offset_fit " + offs + " " + snr + " " + off + " " + coffs + " " + coffsets + " - 3" + _run_or_raise(call_str, 'offset_fit_coarse') + + rfwin4cor = str(int(1/2*int(templateDict['rwin4cor']))) + azfwin4cor = str(int(1/2*int(templateDict['azwin4cor']))) + + rfsample4cor = str(2*int(templateDict['rsample4cor'])) + azfsample4cor = str(2*int(templateDict['azsample4cor'])) + + call_str = "offset_pwr " + Mslc + " " + Srslc0 + " " + MslcPar + " " + SrslcPar0 + " " + off + " " + offs + " " + snr + " " + rfwin4cor + " " + azfwin4cor + " " + offsets + " 2 " + rfsample4cor + " " + azfsample4cor + _run_or_raise(call_str, 'offset_pwr_fine') + + call_str = "offset_fit " + offs + " " + snr + " " + off + " " + coffs + " " + coffsets + " - 3 >" + OFFSTD + _run_or_raise(call_str, 'offset_fit_fine') + +############################################ Resampling ############################################ + + call_str = "SLC_interp_lt " + Sslc + " " + MslcPar + " " + SslcPar + " " + lt1 + " " + MampPar + " " + SampPar + " " + off + " " + Srslc + " " + SrslcPar + _run_or_raise(call_str, 'SLC_interp_lt_final') + except RuntimeError as exc: + print('ERROR: offset refinement failed; aborting coregistration instead of promoting the provisional RSLC') + print(str(exc)) + raise + + call_str = 'multi_look ' + Srslc + ' ' + SrslcPar + ' ' + Sramp + ' ' + SrampPar + ' ' + rlks + ' ' + azlks + _run_or_raise(call_str, 'multi_look_rslc') + + nWidth = ut.read_gamma_par(SrampPar, 'read', 'range_samples') + call_str = 'raspwr ' + Sramp + ' ' + nWidth + _run_or_raise(call_str, 'raspwr') + + for path in ( + lt0, lt1, mli0, diff0, offs0, snr0, offsets0, coffs0, coffsets0, + off, offs, snr, offsets, coffs, coffsets, Srslc0, SrslcPar0, + Mslc, Mamp, HGTSIM, + ): + _safe_remove(path) + + print("Coregistration with DEM is done!") + + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/coreg_gamma_all.py b/third_party/PyINT/pyint/coreg_gamma_all.py new file mode 100644 index 0000000..cf7dd4d --- /dev/null +++ b/third_party/PyINT/pyint/coreg_gamma_all.py @@ -0,0 +1,187 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse +import shutil + +import subprocess +from pyint import _utils as ut + +def work(data0): + cmd = data0[0] + err_txt = data0[1] + + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if type(stdout) == bytes: + bb=stdout.decode("utf-8", errors="replace") + else: + bb = stdout + + if p.returncode != 0: + detail_parts = [] + if bb: + detail_parts.append(bb) + if aa: + detail_parts.append(aa) + detail = '\n'.join(part.strip() for part in detail_parts if part and part.strip()) + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + with open(err_txt, 'a') as f: + f.write(str0) + if detail: + f.write(detail) + f.write('\n') + raise RuntimeError(str0.strip() + ' failed with rc=' + str(p.returncode) + ('\n' + detail if detail else '')) + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + with open(err_txt, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + + +def ensure_master_rslc(projectName): + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict = ut.update_template(templateFile) + master_date = templateDict['masterDate'] + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + slc_root = scratchDir + '/' + projectName + '/SLC/' + master_date + rslc_root = scratchDir + '/' + projectName + '/RSLC/' + master_date + if not os.path.isdir(rslc_root): + os.makedirs(rslc_root, exist_ok=True) + + src_slc = slc_root + '/' + master_date + '.slc' + src_slc_par = slc_root + '/' + master_date + '.slc.par' + src_amp = slc_root + '/' + master_date + '_' + rlks + 'rlks.amp' + src_amp_par = slc_root + '/' + master_date + '_' + rlks + 'rlks.amp.par' + + dst_rslc = rslc_root + '/' + master_date + '.rslc' + dst_rslc_par = rslc_root + '/' + master_date + '.rslc.par' + dst_amp = rslc_root + '/' + master_date + '_' + rlks + 'rlks.amp' + dst_amp_par = rslc_root + '/' + master_date + '_' + rlks + 'rlks.amp.par' + + if not os.path.isfile(src_slc) or not os.path.isfile(src_slc_par): + raise FileNotFoundError('Master SLC is missing under SLC/' + master_date) + + if not os.path.isfile(dst_rslc): + shutil.copyfile(src_slc, dst_rslc) + if not os.path.isfile(dst_rslc_par): + shutil.copyfile(src_slc_par, dst_rslc_par) + + if os.path.isfile(src_amp) and not os.path.isfile(dst_amp): + shutil.copyfile(src_amp, dst_amp) + if os.path.isfile(src_amp_par) and not os.path.isfile(dst_amp_par): + shutil.copyfile(src_amp_par, dst_amp_par) + + if not os.path.isfile(dst_amp) or not os.path.isfile(dst_amp_par): + cmd = [ + 'multi_look', + dst_rslc, + dst_rslc_par, + dst_amp, + dst_amp_par, + rlks, + azlks, + ] + p = subprocess.run(cmd, shell=False, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + if p.returncode != 0: + detail_parts = [] + if p.stdout: + detail_parts.append(p.stdout.decode("utf-8", errors="replace")) + if p.stderr: + detail_parts.append(p.stderr.decode("utf-8", errors="replace")) + detail = '\n'.join(part.strip() for part in detail_parts if part and part.strip()) + raise RuntimeError('multi_look master RSLC failed' + ('\n' + detail if detail else '')) + +INTRODUCTION = ''' +------------------------------------------------------------------- + Coregister all of the SLCs to the reference SAR image using GAMMA. + [with assistant of DEM] + +''' + +EXAMPLE = ''' + Usage: + coreg_gamma_all.py projectName + coreg_gamma_all.py projectName --parallel 4 +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister all of the SLCs to the reference SAR image using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + projectDir = scratchDir + '/' + projectName + slcDir = scratchDir + '/' + projectName + '/SLC' + rslcDir = scratchDir + '/' + projectName + '/RSLC' + if not os.path.isdir(rslcDir): os.mkdir(rslcDir) + ensure_master_rslc(projectName) + + if 'S1' in projectName: cmd_command = 'coreg_s1_gamma.py' + else: cmd_command = 'coreg_gamma.py' + + err_txt = scratchDir + '/' + projectName + '/coreg_gamma_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + #slc_list = [os.path.basename(fname) for fname in sorted(glob.glob(slcDir + '/*'))] + master_date = ut.update_template(templateDir + "/" + projectName + ".template")['masterDate'] + slc_list = [item for item in ut.get_project_slcList(projectName) if item != master_date] + for i in range(len(slc_list)): + cmd0 = [cmd_command,projectName,slc_list[i]] + data0 = [cmd0,err_txt] + data_para.append(data0) + + results = ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + failures = [str(item) for item in results if isinstance(item, Exception)] + if failures: + raise RuntimeError('\n\n'.join(failures)) + print("Coregister all of the SLCs %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/third_party/PyINT/pyint/coreg_s1_gamma.py b/third_party/PyINT/pyint/coreg_s1_gamma.py new file mode 100644 index 0000000..97ebd10 --- /dev/null +++ b/third_party/PyINT/pyint/coreg_s1_gamma.py @@ -0,0 +1,239 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import argparse + +from pyint import _utils as ut + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister TOPS S1-SLC to a reference S1-SLC using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='Name of project.') + parser.add_argument('sdate', help='date of the slave S1 image. [mater date is read from template]') + inps = parser.parse_args() + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Coregister TOPS S1-SLC to a reference S1-SLC using GAMMA. + [The reference date or master date will be read from the template file.] +''' + +EXAMPLE = """Usage: + + coreg_s1_gamma.py projectName Sdate + + coreg_s1_gamma.py PacayaT163TsxHhA 20150102 +------------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Sdate = inps.sdate + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + if not os.path.isdir(rslcDir): os.mkdir(rslcDir) + #workDir = processDir + '/' + igramDir + workDir = rslcDir + '/' + Sdate + if not os.path.isdir(workDir): os.mkdir(workDir) + + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Mdate = templateDict['masterDate'] + + demDir = scratchDir + '/' + projectName + '/DEM' + +# Definition of file + MslcDir = slcDir + '/' + Mdate + SslcDir = slcDir + '/' + Sdate + Samp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + Sampbmp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.bmp' + + Mslc = slcDir + '/' + Mdate + '/' + Mdate + '.slc' + Mslcpar = slcDir + '/' + Mdate + '/' + Mdate + '.slc.par' + Mamp = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + Mampbmp = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.bmp' + + SLC1_INF_tab0 = MslcDir + '/' + Mdate + '_SLC_Tab' + SLC2_INF_tab = SslcDir + '/' + Sdate + '_SLC_Tab' + RSLC_tab = SslcDir + '/' + Sdate + '_RSLC_Tab' + SLC1_INF_tab = SslcDir + '/' + Mdate + '_SLC_Tab_coreg' + + HGTSIM = demDir + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' + if not os.path.isfile(HGTSIM): + call_str = 'generate_rdc_dem.py ' + projectName + os.system(call_str) + + ############## copy master files into slave folder for parallel process ########### + + #if not templateDict['coreg_all_parallel'] == '1': + # Mslc = slcDir + '/' + Sdate + '/' + Mdate + '.slc' + # Mslcpar = slcDir + '/' + Sdate + '/' + Mdate + '.slc.par' + # Mamp = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + # MampPar = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + # HGTSIM = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' + + #else: + + # Mslc = Mslc0 + # Mslcpar = Mslcpar0 + # Mamp = Mamp0 + # MampPar = MampPar0 + # HGTSIM = HGTSIM0 + + + #Mslc = slcDir + '/' + Sdate + '/' + Mdate + '.slc' + #Mslcpar = slcDir + '/' + Sdate + '/' + Mdate + '.slc.par' + #Mamp = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + #MampPar = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + #HGTSIM = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' + SLC1_INF_tab1 = SslcDir + '/' + Mdate + '_SLC_Tab' + if SslcDir==MslcDir: + SLC1_INF_tab1=SLC1_INF_tab0 + else: + ut.copy_file(SLC1_INF_tab0,SLC1_INF_tab1) + ############################################################################## + #with open(SLC1_INF_tab1, "r") as f: + # lines = f.readlines() + + #with open(SLC1_INF_tab, "w") as fw: + # lines_coreg = [] + # for k0 in lines: + # k00 = k0.replace(MslcDir,SslcDir) + # lines_coreg.append(k00) + # fw.write(k00) + + S_IW = ut.read_txt2array(SLC1_INF_tab1) + S_IW = S_IW.flatten() + #M_IW = ut.read_txt2array(SLC1_INF_tab1) + #M_IW = M_IW.flatten() + #S_IW = ut.read_txt2array(SLC1_INF_tab) + #S_IW = S_IW.flatten() + + #RSLC_tab = workDir + '/' + Sdate + '_RSLC_tab' + #if os.path.isfile(RSLC_tab): + # os.remove(RSLC_tab) + + #BURST = SslcDir + '/' + Sdate + '_Burst_Tab' + #AA = np.loadtxt(BURST) + #if EW==SW: + # AA = AA.reshape([1,2]) + # + #for kk in range(int(EW)-int(SW)+1): + # ii = int(int(kk) + 1) + # SB2=AA[ii-1,0] + # EB2=AA[ii-1,1] + # call_str = 'echo ' + workDir + '/'+ Sdate+ '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+str(int(SW)+kk)+ '.rslc' + ' ' + workDir + '/' + Sdate + '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+ str(int(SW)+kk)+ '.rslc.par' + ' ' + workDir + '/'+ Sdate+'_'+ str(int(SB2)) + str(int(EB2)) + '.IW'+str(int(SW)+kk)+ '.rslc.TOPS_par >>' + RSLC_tab + # os.system(call_str) + + os.chdir(workDir) + #TEST = workDir + '/' + Sdate +'_' + rlks + 'rlks.amp.par' + TEST = workDir + '/' + Mdate + '_' + Sdate +'.diff' + + k0 = 0 + if os.path.isfile(TEST): + if os.path.getsize(TEST) > 0: + k0 = 1 + + if k0 == 0: + if not Mdate==Sdate: +# call_str = 'S1_coreg_TOPS ' + SLC1_INF_tab1 + ' ' + Mdate + ' ' + SLC2_INF_tab + ' ' + Sdate + ' ' + RSLC_tab + ' ' + HGTSIM + ' ' + rlks + ' ' + azlks + ' - - 0.8 0.01 1.2 1' + call_str = 'ScanSAR_coreg.py ' + SLC1_INF_tab1 + ' ' + Mdate + ' ' + SLC2_INF_tab + ' ' + Sdate + ' ' + RSLC_tab + ' ' + HGTSIM + ' ' + rlks + ' ' + azlks + ' --cc 0.8 --fraction 0.01 --ph_stdev 0.8 --num_ovr 0 --no_check ' + print(call_str) + ret = os.system(call_str) + + #### clean large file #### + mslc = workDir + '/' + Mdate + '.slc' + mrslc = workDir + '/' + Mdate + '.rslc' + sslc = workDir + '/' + Sdate + '.slc' + srslc = workDir + '/' + Sdate + '.rslc' + srslcPar = workDir + '/' + Sdate + '.rslc.par' + + # 检查 ScanSAR_coreg.py 是否成功生成 .rslc（不检查返回码，spectral diversity 步骤可能报错但 RSLC 已生成） + if not os.path.isfile(srslc) or os.path.getsize(srslc) == 0: + print('ERROR: ScanSAR_coreg.py did not produce a valid ' + srslc) + sys.exit(1) + + call_str = 'multi_look ' + srslc + ' ' + srslcPar + ' ' + Samp + ' ' + SampPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + nWIDTH = ut.read_gamma_par(SampPar,'read', 'range_samples') + if os.path.isfile(mslc): os.remove(mslc) + if os.path.isfile(mrslc): os.remove(mrslc) + if os.path.isfile(sslc): os.remove(sslc) + + call_str = 'raspwr ' + Samp + ' ' + nWIDTH + os.system(call_str) + + #call_str = 'rm *mli*' + #os.system(call_str) + + #call_str = 'rm *IW*' + #os.system(call_str) + + #call_str = 'rm *off*' + #os.system(call_str) + + #call_str = 'rm *diff' + #os.system(call_str) + + #call_str = 'rm *diff_par*' + #os.system(call_str) + + #call_str = 'rm ' + Mdate + '.*' + #os.system(call_str) + + else: + call_str = 'cp ' + Mslc + ' ' + workDir + '/' + Mdate + '.rslc' + os.system(call_str) + + call_str = 'cp ' + Mslcpar + ' ' + workDir+ '/' + Mdate + '.rslc.par' + os.system(call_str) + + call_str = 'cp ' + Mamp + ' ' + workDir+ '/' + Mdate + '_' + rlks + 'rlks.amp' + os.system(call_str) + + call_str = 'cp ' + MampPar + ' ' + workDir+ '/' + Mdate + '_' + rlks + 'rlks.amp.par' + os.system(call_str) + + ut.copy_file(Mampbmp,Sampbmp) + + ################ clean redundant files ############# + + #if not Mdate ==Sdate: + + # if not templateDict['diff_all_parallel'] == '1': + # for i in range(len(S_IW)): + # if os.path.isfile(S_IW[i]): + # os.remove(S_IW[i]) + # if os.path.isfile(Mslc): os.remove(Mslc) + # if os.path.isfile(Mslcpar): os.remove(Mslcpar) + # if os.path.isfile(Mamp): os.remove(Mamp) + # if os.path.isfile(HGTSIM): os.remove(HGTSIM) + else: + print('The SLC has already coregister Done') + sys.exit(1) + print("Coregister TOP SLC image to the reference TOPS image is done !!") + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/coreg_s1_gamma_old.py b/third_party/PyINT/pyint/coreg_s1_gamma_old.py new file mode 100644 index 0000000..15f9e08 --- /dev/null +++ b/third_party/PyINT/pyint/coreg_s1_gamma_old.py @@ -0,0 +1,232 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import argparse + +from pyint import _utils as ut + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister TOPS S1-SLC to a reference S1-SLC using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='Name of project.') + parser.add_argument('sdate', help='date of the slave S1 image. [mater date is read from template]') + inps = parser.parse_args() + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Coregister TOPS S1-SLC to a reference S1-SLC using GAMMA. + [The reference date or master date will be read from the template file.] +''' + +EXAMPLE = """Usage: + + coreg_s1_gamma.py projectName Sdate + + coreg_s1_gamma.py PacayaT163TsxHhA 20150102 +------------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Sdate = inps.sdate + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + if not os.path.isdir(rslcDir): os.mkdir(rslcDir) + #workDir = processDir + '/' + igramDir + workDir = rslcDir + '/' + Sdate + if not os.path.isdir(workDir): os.mkdir(workDir) + + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Mdate = templateDict['masterDate'] + + demDir = scratchDir + '/' + projectName + '/DEM' + +# Definition of file + MslcDir = slcDir + '/' + Mdate + SslcDir = slcDir + '/' + Sdate + Samp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + Sampbmp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.bmp' + + Mslc = slcDir + '/' + Mdate + '/' + Mdate + '.slc' + Mslcpar = slcDir + '/' + Mdate + '/' + Mdate + '.slc.par' + Mamp = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + Mampbmp = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.bmp' + + #SLC1_INF_tab0 = MslcDir + '/' + Mdate + '_SLC_Tab0' + SLC1_INF_tab0 = MslcDir + '/' + Mdate + '_SLC_Tab' + SLC2_INF_tab = SslcDir + '/' + Sdate + '_SLC_Tab' + RSLC_tab = SslcDir + '/' + Sdate + '_RSLC_Tab' + #SLC1_INF_tab = SslcDir + '/' + Mdate + '_SLC_Tab_coreg' + SLC1_INF_tab = SslcDir + '/' + Mdate + '_SLC_Tab' + + HGTSIM = demDir + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' + if not os.path.isfile(HGTSIM): + call_str = 'generate_rdc_dem.py ' + projectName + os.system(call_str) + + ############## copy master files into slave folder for parallel process ########### + + #if not templateDict['coreg_all_parallel'] == '1': + # Mslc = slcDir + '/' + Sdate + '/' + Mdate + '.slc' + # Mslcpar = slcDir + '/' + Sdate + '/' + Mdate + '.slc.par' + # Mamp = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + # MampPar = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + # HGTSIM = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' + + #else: + + # Mslc = Mslc0 + # Mslcpar = Mslcpar0 + # Mamp = Mamp0 + # MampPar = MampPar0 + # HGTSIM = HGTSIM0 + + + #Mslc = slcDir + '/' + Sdate + '/' + Mdate + '.slc' + #Mslcpar = slcDir + '/' + Sdate + '/' + Mdate + '.slc.par' + #Mamp = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + #MampPar = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + #HGTSIM = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' + SLC1_INF_tab = SslcDir + '/' + Mdate + '_SLC_Tab' + if SslcDir==MslcDir: + SLC1_INF_tab1=SLC1_INF_tab0 + else: + ut.copy_file(SLC1_INF_tab0,SLC1_INF_tab) + ############################################################################## + #with open(SLC1_INF_tab, "r") as f: + # lines = f.readlines() + + #with open(SLC1_INF_tab, "w") as fw: + # lines_coreg = [] + # for k0 in lines: + # k00 = k0.replace(MslcDir,SslcDir) + # lines_coreg.append(k00) + # fw.write(k00) + + S_IW = ut.read_txt2array(SLC1_INF_tab) + S_IW = S_IW.flatten() + #M_IW = ut.read_txt2array(SLC1_INF_tab1) + #M_IW = M_IW.flatten() + #S_IW = ut.read_txt2array(SLC1_INF_tab) + #S_IW = S_IW.flatten() + + #RSLC_tab = workDir + '/' + Sdate + '_RSLC_tab' + #if os.path.isfile(RSLC_tab): + # os.remove(RSLC_tab) + + #BURST = SslcDir + '/' + Sdate + '_Burst_Tab' + #AA = np.loadtxt(BURST) + #if EW==SW: + # AA = AA.reshape([1,2]) + # + #for kk in range(int(EW)-int(SW)+1): + # ii = int(int(kk) + 1) + # SB2=AA[ii-1,0] + # EB2=AA[ii-1,1] + # call_str = 'echo ' + workDir + '/'+ Sdate+ '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+str(int(SW)+kk)+ '.rslc' + ' ' + workDir + '/' + Sdate + '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+ str(int(SW)+kk)+ '.rslc.par' + ' ' + workDir + '/'+ Sdate+'_'+ str(int(SB2)) + str(int(EB2)) + '.IW'+str(int(SW)+kk)+ '.rslc.TOPS_par >>' + RSLC_tab + # os.system(call_str) + + os.chdir(workDir) + #TEST = workDir + '/' + Sdate +'_' + rlks + 'rlks.amp.par' + TEST = workDir + '/' + Sdate +'.rslc.par' + + k0 = 0 + if os.path.isfile(TEST): + if os.path.getsize(TEST) > 0: + k0 = 1 + + if k0==0: + if not Mdate ==Sdate: + call_str = 'S1_coreg_TOPS ' + SLC1_INF_tab + ' ' + Mdate + ' ' + SLC2_INF_tab + ' ' + Sdate + ' ' + RSLC_tab + ' ' + HGTSIM + ' ' + rlks + ' ' + azlks + ' - - 0.8 0.01 1.2 1' + os.system(call_str) + + #### clean large file #### + mslc = workDir + '/' + Mdate + '.slc' + mrslc = workDir + '/' + Mdate + '.rslc' + sslc = workDir + '/' + Sdate + '.slc' + srslc = workDir + '/' + Sdate + '.rslc' + srslcPar = workDir + '/' + Sdate + '.rslc.par' + + call_str = 'multi_look ' + srslc + ' ' + srslcPar + ' ' + Samp + ' ' + SampPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + nWIDTH = ut.read_gamma_par(SampPar,'read', 'range_samples') + if os.path.isfile(mslc): os.remove(mslc) + if os.path.isfile(mrslc): os.remove(mrslc) + if os.path.isfile(sslc): os.remove(sslc) + + call_str = 'raspwr ' + Samp + ' ' + nWIDTH + os.system(call_str) + + #call_str = 'rm *mli*' + #os.system(call_str) + + #call_str = 'rm *IW*' + #os.system(call_str) + + #call_str = 'rm *off*' + #os.system(call_str) + + #call_str = 'rm *diff' + #os.system(call_str) + + #call_str = 'rm *diff_par*' + #os.system(call_str) + + #call_str = 'rm ' + Mdate + '.*' + #os.system(call_str) + + else: + call_str = 'cp ' + Mslc + ' ' + workDir + '/' + Mdate + '.rslc' + os.system(call_str) + + call_str = 'cp ' + Mslcpar + ' ' + workDir+ '/' + Mdate + '.rslc.par' + os.system(call_str) + + call_str = 'cp ' + Mamp + ' ' + workDir+ '/' + Mdate + '_' + rlks + 'rlks.amp' + os.system(call_str) + + call_str = 'cp ' + MampPar + ' ' + workDir+ '/' + Mdate + '_' + rlks + 'rlks.amp.par' + os.system(call_str) + + ut.copy_file(Mampbmp,Sampbmp) + + ################ clean redundant files ############# + + #if not Mdate ==Sdate: + + # if not templateDict['diff_all_parallel'] == '1': + # for i in range(len(S_IW)): + # if os.path.isfile(S_IW[i]): + # os.remove(S_IW[i]) + # if os.path.isfile(Mslc): os.remove(Mslc) + # if os.path.isfile(Mslcpar): os.remove(Mslcpar) + # if os.path.isfile(Mamp): os.remove(Mamp) + # if os.path.isfile(HGTSIM): os.remove(HGTSIM) + + print("Coregister TOP SLC image to the reference TOPS image is done !!") + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/coreg_s1_gamma_pot.py b/third_party/PyINT/pyint/coreg_s1_gamma_pot.py new file mode 100644 index 0000000..712cf57 --- /dev/null +++ b/third_party/PyINT/pyint/coreg_s1_gamma_pot.py @@ -0,0 +1,370 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import argparse +import glob + +from pyint import _utils as ut + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister TOPS S1-SLC to a reference S1-SLC using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='Name of project.') + parser.add_argument('sdate', help='date of the slave S1 image. [mater date is read from template]') + inps = parser.parse_args() + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Coregister TOPS S1-SLC to a reference S1-SLC using GAMMA. + [The reference date or master date will be read from the template file.] +''' + +EXAMPLE = """Usage: + + coreg_s1_gamma.py projectName Sdate + + coreg_s1_gamma.py PacayaT163TsxHhA 20150102 +------------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Sdate = inps.sdate + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + if not os.path.isdir(rslcDir): os.mkdir(rslcDir) + #workDir = processDir + '/' + igramDir + workDir = rslcDir + '/' + Sdate + if not os.path.isdir(workDir): os.mkdir(workDir) + + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Mdate = templateDict['masterDate'] + deramp = templateDict['deramp'] + + if 'boi' in templateDict: + boi = templateDict['boi'] + else: + boi = '0' + + demDir = scratchDir + '/' + projectName + '/DEM' + +# Definition of file + MslcDir = slcDir + '/' + Mdate + + SslcDir = slcDir + '/' + Sdate + Samp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + Sampbmp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.bmp' + + Mslc = slcDir + '/' + Mdate + '/' + Mdate + '.slc' + Mslcpar = slcDir + '/' + Mdate + '/' + Mdate + '.slc.par' + Mamp = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + Mampbmp = slcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.bmp' + + #SLC1_INF_tab = MslcDir + '/' + Mdate + '_SLC_Tab' + SLC2_INF_tab = SslcDir + '/' + Sdate + '_SLC_Tab' + RSLC_tab = SslcDir + '/' + Sdate + '_RSLC_Tab' + RSLC_tab2 = workDir + '/' + Sdate + '_RSLC_Tab' + call_str = 'cp ' + RSLC_tab + ' ' + RSLC_tab2 + os.system(call_str) + + #SLC1_INF_tab = SslcDir + '/' + Mdate + '_SLC_Tab_coreg' + + HGTSIM = demDir + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' + if not os.path.isfile(HGTSIM): + call_str = 'generate_rdc_dem.py ' + projectName + os.system(call_str) + + srslc_fboi0 = workDir + '/' + Sdate + '_overlap.fwd.slc'; srslc_fboi = srslc_fboi0.replace('.slc','.rslc') + srslc_fboi_par0 = workDir + '/' + Sdate + '_overlap.fwd.slc.par'; srslc_fboi_par = srslc_fboi_par0.replace('.slc','.rslc') + + srslc_bboi0 = workDir + '/' + Sdate + '_overlap.bwd.slc'; srslc_bboi = srslc_bboi0.replace('.slc','.rslc') + srslc_bboi_par0 = workDir + '/' + Sdate + '_overlap.bwd.slc.par'; srslc_bboi_par = srslc_bboi_par0.replace('.slc','.rslc') + + samp_fboi = workDir + '/' + Sdate + '_overlap.fwd_' + rlks + 'rlks.amp' + samp_fboi_par = workDir + '/' + Sdate + '_overlap.fwd_' + rlks + 'rlks.amp.par' + + samp_bboi = workDir + '/' + Sdate + '_overlap.bwd_' + rlks + 'rlks.amp' + samp_bboi_par = workDir + '/' + Sdate + '_overlap.bwd_' + rlks + 'rlks.amp.par' + ############## copy master files into slave folder for parallel process ########### + + #if not templateDict['coreg_all_parallel'] == '1': + # Mslc = slcDir + '/' + Sdate + '/' + Mdate + '.slc' + # Mslcpar = slcDir + '/' + Sdate + '/' + Mdate + '.slc.par' + # Mamp = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + # MampPar = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + # HGTSIM = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' + + #else: + + # Mslc = Mslc0 + # Mslcpar = Mslcpar0 + # Mamp = Mamp0 + # MampPar = MampPar0 + # HGTSIM = HGTSIM0 + + + #Mslc = slcDir + '/' + Sdate + '/' + Mdate + '.slc' + #Mslcpar = slcDir + '/' + Sdate + '/' + Mdate + '.slc.par' + #Mamp = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + #MampPar = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + #HGTSIM = slcDir + '/' + Sdate + '/' + Mdate + '_' + rlks + 'rlks.rdc.dem' + #SLC1_INF_tab1 = SslcDir + '/' + Mdate + '_SLC_Tab' + #ut.copy_file(SLC1_INF_tab0,SLC1_INF_tab1) + ############################################################################## + #with open(SLC1_INF_tab1, "r") as f: + # lines = f.readlines() + + #with open(SLC1_INF_tab, "w") as fw: + # lines_coreg = [] + # for k0 in lines: + # k00 = k0.replace(MslcDir,SslcDir) + # lines_coreg.append(k00) + # fw.write(k00) + + #S_IW = ut.read_txt2array(SLC1_INF_tab1) + #S_IW = S_IW.flatten() + #M_IW = ut.read_txt2array(SLC1_INF_tab1) + #M_IW = M_IW.flatten()TSLC = slc_dir + '/' + date + '.slc' + #TSLCPar = slc_dir + '/' + date + '.slc.par' + + # + #S_IW = ut.read_txt2array(SLC1_INF_tab) + #S_IW = S_IW.flatten() + + #RSLC_tab = workDir + '/' + Sdate + '_RSLC_tab' + #if os.path.isfile(RSLC_tab): + # os.remove(RSLC_tab) + + #BURST = SslcDir + '/' + Sdate + '_Burst_Tab' + #AA = np.loadtxt(BURST) + #if EW==SW: + # AA = AA.reshape([1,2]) + # + #for kk in range(int(EW)-int(SW)+1): + # ii = int(int(kk) + 1) + # SB2=AA[ii-1,0] + # EB2=AA[ii-1,1] + # call_str = 'echo ' + workDir + '/'+ Sdate+ '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+str(int(SW)+kk)+ '.rslc' + ' ' + workDir + '/' + Sdate + '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+ str(int(SW)+kk)+ '.rslc.par' + ' ' + workDir + '/'+ Sdate+'_'+ str(int(SB2)) + str(int(EB2)) + '.IW'+str(int(SW)+kk)+ '.rslc.TOPS_par >>' + RSLC_tab + # os.system(call_str) + + os.chdir(workDir) + #TEST = workDir + '/' + Sdate +'_' + rlks + 'rlks.amp.par' + TEST = workDir + '/' + Mdate + '_' + Sdate +'.diff.bmp' + + k0 = 0 + if os.path.isfile(TEST): + if os.path.getsize(TEST) > 0: + k0 = 1 + + if k0==0: + if not Mdate ==Sdate: + + call_str = 'cp -rf ' + MslcDir + ' ' + workDir + os.system(call_str) + + MslcDir2 = workDir + '/' + Mdate + + SLC_list = glob.glob(MslcDir2 + '/*IW*.slc') + SLC_par_list = glob.glob(MslcDir2 + '/*IW*.slc.par') + TOP_par_list = glob.glob(MslcDir2 + '/*IW*.slc.TOPS_par') + + SLC1_INF_tab = MslcDir2 + '/' + Mdate + '_SLC_Tab' + + if os.path.isfile(SLC1_INF_tab): + os.remove(SLC1_INF_tab) + + SLC_list = sorted(SLC_list) + SLC_par_list = sorted(SLC_par_list) + TOP_par_list = sorted(TOP_par_list) + + for kk in range(len(SLC_list)): + call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' >> ' + SLC1_INF_tab + os.system(call_str) + + + #call_str = 'S1_coreg_TOPS ' + SLC1_INF_tab + ' ' + Mdate + ' ' + SLC2_INF_tab + ' ' + Sdate + ' ' + RSLC_tab + ' ' + HGTSIM + ' ' + rlks + ' ' + azlks + ' - - 0.8 0.01 1.2 1' + + call_str = 'ScanSAR_coreg.py ' + SLC1_INF_tab + ' ' + Mdate + ' ' + SLC2_INF_tab + ' ' + Sdate + ' ' + RSLC_tab + ' ' + HGTSIM + ' ' + rlks + ' ' + azlks + ' --cc 0.8 --fraction 0.01 --ph_stdev 0.8 --num_ovr 0 --no_check ' + print(call_str) + os.system(call_str) + + #### clean large file #### + mslc = workDir + '/' + Mdate + '.slc' + mrslc = workDir + '/' + Mdate + '.rslc' + sslc = workDir + '/' + Sdate + '.slc' + srslc = workDir + '/' + Sdate + '.rslc' + + srslcPar = workDir + '/' + Sdate + '.rslc.par' + + if deramp=='1': + call_str = 'S1_deramp_TOPS_slave ' + RSLC_tab2 + ' ' + Sdate + ' ' + SLC1_INF_tab + ' 10 2 1' + os.system(call_str) + + call_str = 'mv ' + srslc + '.deramp' + ' ' + srslc + os.system(call_str) + + call_str = 'mv ' + srslc + '.deramp.par' + ' ' + srslcPar + os.system(call_str) + + call_str = 'multi_look ' + srslc + ' ' + srslcPar + ' ' + Samp + ' ' + SampPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + + nWIDTH = ut.read_gamma_par(SampPar,'read', 'range_samples') + + if boi =='1': + call_str = 'ScanSAR_burst_overlap ' + RSLC_tab2 + ' ' + Sdate+'_overlap' + ' ' + rlks + ' ' + azlks + ' - - ' + SLC1_INF_tab + ' - -' + os.system(call_str) + + call_str = 'mv ' + srslc_fboi0 + ' ' + srslc_fboi; os.system(call_str) + call_str = 'mv ' + srslc_fboi_par0 + ' ' + srslc_fboi_par; os.system(call_str) + call_str = 'mv ' + srslc_bboi0 + ' ' + srslc_bboi; os.system(call_str) + call_str = 'mv ' + srslc_bboi_par0 + ' ' + srslc_bboi_par; os.system(call_str) + + call_str = 'multi_look ' + srslc_fboi + ' ' + srslc_fboi_par + ' ' + samp_fboi + ' ' + samp_fboi_par + ' ' + rlks + ' ' + azlks + os.system(call_str) + + call_str = 'multi_look ' + srslc_bboi + ' ' + srslc_bboi_par + ' ' + samp_bboi + ' ' + samp_bboi_par + ' ' + rlks + ' ' + azlks + os.system(call_str) + + call_str = 'raspwr ' + samp_fboi + ' ' + nWIDTH + os.system(call_str) + + call_str = 'raspwr ' + samp_bboi + ' ' + nWIDTH + os.system(call_str) + + if os.path.isfile(mslc): os.remove(mslc) + if os.path.isfile(mrslc): os.remove(mrslc) + if os.path.isfile(sslc): os.remove(sslc) + + call_str = 'raspwr ' + Samp + ' ' + nWIDTH + os.system(call_str) + + call_str = 'rm -rf ' + MslcDir2 + os.system(call_str) + + #call_str = 'rm *mli*' + #os.system(call_str) + + #call_str = 'rm *IW*' + #os.system(call_str) + + #call_str = 'rm *off*' + #os.system(call_str) + + #call_str = 'rm *diff' + #os.system(call_str) + + #call_str = 'rm *diff_par*' + #os.system(call_str) + + #call_str = 'rm ' + Mdate + '.*' + + #os.system(call_str) + + else: + + #generate amp file for check image quality + TSLC = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc' + TSLCPar = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc.par' + + TMLI = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + TMLIPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + if not os.path.isfile(TMLIPar): + A1 = np.loadtxt(SLC2_INF_tab,dtype='str'); A1 = A1.flatten() + A2 = np.loadtxt(RSLC_tab,dtype='str'); A2 = A2.flatten() + + nn = len(A1) + for i in range(nn): + call_str = 'cp ' + A1[i] + ' ' + A2[i] + os.system(call_str) + + if deramp=='1': + call_str = 'S1_deramp_TOPS_reference ' + RSLC_tab2 + os.system(call_str) + + SLC2_INF_tab = RSLC_tab2 + '.deramp' + else: + SLC2_INF_tab = RSLC_tab2 + + call_str = 'SLC_mosaic_S1_TOPS ' + SLC2_INF_tab + ' ' + TSLC + ' ' + TSLCPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + + call_str = 'multi_look ' + TSLC + ' ' + TSLCPar + ' ' + TMLI + ' ' + TMLIPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + + nWidth = ut.read_gamma_par(TMLIPar, 'read','range_samples:') + + if boi =='1': + call_str = 'ScanSAR_burst_overlap ' + RSLC_tab2 + ' ' + Sdate+'_overlap' + ' ' + rlks + ' ' + azlks + ' - - ' + SLC2_INF_tab + ' - -' + os.system(call_str) + + call_str = 'mv ' + srslc_fboi0 + ' ' + srslc_fboi; os.system(call_str) + call_str = 'mv ' + srslc_fboi_par0 + ' ' + srslc_fboi_par; os.system(call_str) + call_str = 'mv ' + srslc_bboi0 + ' ' + srslc_bboi; os.system(call_str) + call_str = 'mv ' + srslc_bboi_par0 + ' ' + srslc_bboi_par; os.system(call_str) + + call_str = 'multi_look ' + srslc_fboi + ' ' + srslc_fboi_par + ' ' + samp_fboi + ' ' + samp_fboi_par + ' ' + rlks + ' ' + azlks + os.system(call_str) + + call_str = 'multi_look ' + srslc_bboi + ' ' + srslc_bboi_par + ' ' + samp_bboi + ' ' + samp_bboi_par + ' ' + rlks + ' ' + azlks + os.system(call_str) + + call_str = 'raspwr ' + samp_fboi + ' ' + nWidth + os.system(call_str) + + call_str = 'raspwr ' + samp_bboi + ' ' + nWidth + os.system(call_str) + + + call_str = 'raspwr ' + TMLI + ' ' + nWidth + ' - - - - - - - ' + os.system(call_str) + rr = glob.glob(workDir + '/*IW*.slc') + if len(rr) > 0: + call_str = 'rm ' + workDir + '/*IW*.slc' + os.system(call_str) + uu = glob.glob(workDir + '/*IW*.rslc') + + if len(uu) > 0: + call_str = 'rm ' + workDir + '/*IW*.rslc' + os.system(call_str) + ################ clean redundant files ############# + + #if not Mdate ==Sdate: + + # if not templateDict['diff_all_parallel'] == '1': + # for i in range(len(S_IW)): + # if os.path.isfile(S_IW[i]): + # os.remove(S_IW[i]) + # if os.path.isfile(Mslc): os.remove(Mslc) + # if os.path.isfile(Mslcpar): os.remove(Mslcpar) + # if os.path.isfile(Mamp): os.remove(Mamp) + # if os.path.isfile(HGTSIM): os.remove(HGTSIM) + + print("Coregister TOP SLC image to the reference TOPS image is done !!") + #sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/corners.txt b/third_party/PyINT/pyint/corners.txt new file mode 100644 index 0000000..cb93de4 --- /dev/null +++ b/third_party/PyINT/pyint/corners.txt @@ -0,0 +1,22 @@ +*** Calculate SLC/MLI image corners in geodetic latitude and longitude (deg.) *** +*** Copyright 2022, Gamma Remote Sensing, v2.2 8-Nov-2022 clw/awi/cm *** +latitude (deg.): 24.74699954 longitude (deg.): 103.30166172 +latitude (deg.): 25.16531434 longitude (deg.): 105.78287221 +latitude (deg.): 26.37363426 longitude (deg.): 102.95054159 +latitude (deg.): 26.78897220 longitude (deg.): 105.46762233 + +center latitude (deg.): 25.79112685 center longitude (deg.): 104.47614292 +min. latitude (deg.): 24.74699954 max. latitude (deg.): 26.78897220 +min. longitude (deg.): 102.95054159 max. longitude (deg.): 105.78287221 +delta latitude (deg.): 2.04197267 delta longitude (deg.): 2.83233062 + +upper left corner latitude, longitude (deg.): 26.83 102.91 +lower right corner latitude, longitude (deg.): 24.71 105.82 + +lower left corner longitude, latitude (deg.): 102.91 24.71 +upper right corner longitude, latitude (deg.): 105.82 26.83 + +user time (s): 0.000 +system time (s): 0.000 +elapsed time (s): 0.000 + diff --git a/third_party/PyINT/pyint/correct_ifg_for_hpy3_from_murp.py b/third_party/PyINT/pyint/correct_ifg_for_hpy3_from_murp.py new file mode 100644 index 0000000..2ff3b68 --- /dev/null +++ b/third_party/PyINT/pyint/correct_ifg_for_hpy3_from_murp.py @@ -0,0 +1,769 @@ +#!/usr/bin/env python3 +""" +修复版MuRP算法 - 包含Linux系统中文显示支持 +""" + +import xarray as xr +import rioxarray +import os +import numpy as np +import matplotlib.pyplot as plt +from glob import glob +import argparse +import sys +from datetime import datetime +import rasterio +import warnings +import matplotlib.font_manager as fm +import platform + +# 设置中文字体支持 +def setup_chinese_font(): + """配置中文字体支持""" + system = platform.system() + + if system == "Linux": + # Linux系统字体配置 + chinese_fonts = [ + 'WenQuanYi Micro Hei', # 文泉驿微米黑 + 'Noto Sans CJK SC', # Google思源黑体 + 'DejaVu Sans', # 备选字体 + 'Arial' # 最后备选 + ] + + # 查找可用的字体 + available_fonts = [] + for font in chinese_fonts: + if any(f.name == font for f in fm.fontManager.ttflist): + available_fonts.append(font) + + if available_fonts: + plt.rcParams['font.sans-serif'] = available_fonts + print(f"已设置中文字体: {available_fonts[0]}") + else: + plt.rcParams['font.sans-serif'] = ['DejaVu Sans'] + print("警告: 未找到中文字体，使用默认字体") + else: + # Windows或macOS + plt.rcParams['font.sans-serif'] = ['SimHei', 'Microsoft YaHei', 'Arial'] + + # 解决负号显示问题 + plt.rcParams['axes.unicode_minus'] = False + +# 调用字体设置函数 +setup_chinese_font() +warnings.filterwarnings('ignore') + +class FixedMuRP: + """ + 修复版MuRP算法 - 解决相干性数据形状不一致问题 + """ + + def __init__(self, random_seed=42): + """初始化MuRP处理器""" + self.random_seed = random_seed + np.random.seed(random_seed) + self.ds = None + self.ds_corrected = None + self.refs = [] + self.fits = [] + self.crs = None + self.transform = None + self.bounds = None + self.width = None + self.height = None + + def load_hyp3_data_robust(self, data_path): + """ + 健壮的数据加载函数，处理形状不一致问题 + """ + print(f"正在加载Hyp3数据从: {data_path}") + + # 查找数据文件 + unw_files = glob(f'{data_path}/*/*unw_phase*.tif') + corr_files = glob(f'{data_path}/*/*corr*.tif') + dem_files = glob(f'{data_path}/*/*dem*.tif') + + if not unw_files: + raise FileNotFoundError(f"在 {data_path} 中未找到解缠相位文件") + if not dem_files: + raise FileNotFoundError(f"在 {data_path} 中未找到DEM文件") + + print(f"找到解缠相位文件: {len(unw_files)} 个") + print(f"找到相干性文件: {len(corr_files)} 个") + print(f"找到DEM文件: {len(dem_files)} 个") + + # 使用第一个解缠相位文件作为参考 + reference_file = unw_files[0] + with rasterio.open(reference_file) as src: + self.crs = src.crs + self.transform = src.transform + self.width = src.width + self.height = src.height + self.bounds = src.bounds + + print(f"坐标参考系统: {self.crs}") + print(f"图像尺寸: {self.width} x {self.height}") + + # 读取DEM数据 - 使用第一个DEM文件 + print("正在加载DEM数据...") + with rasterio.open(dem_files[0]) as src: + # 确保DEM尺寸与参考文件一致 + if src.width == self.width and src.height == self.height: + elevation_data = src.read(1) + else: + print("警告: DEM尺寸不匹配，将使用零数组") + elevation_data = np.zeros((self.height, self.width)) + + # 读取解缠相位数据 + print("正在加载解缠相位数据...") + unw_phase_data = [] + granule_names = [] + + for unw_file in unw_files: + with rasterio.open(unw_file) as src: + # 检查尺寸是否匹配 + if src.width == self.width and src.height == self.height: + data = src.read(1) + else: + print(f"警告: {unw_file} 尺寸不匹配，跳过") + continue + + unw_phase_data.append(data) + granule_name = os.path.basename(unw_file).replace('.tif', '') + granule_names.append(granule_name) + print(f" 已加载: {granule_name}") + + if not unw_phase_data: + raise ValueError("没有成功加载任何解缠相位数据") + + # 转换为numpy数组 + unw_phase_array = np.array(unw_phase_data) + print(f"解缠相位数据形状: {unw_phase_array.shape}") + + # 处理相干性数据 - 确保与解缠相位数据对应 + print("正在处理相干性数据...") + coherence_data = [] + used_corr_files = [] + + # 为每个解缠相位文件寻找匹配的相干性文件 + for i, granule_name in enumerate(granule_names): + # 从granule名称提取基本名称 + base_name = granule_name.replace('_unw_phase', '') + + # 查找匹配的相干性文件 + matching_corr_files = [f for f in corr_files if base_name in f and f not in used_corr_files] + + if matching_corr_files: + # 使用第一个匹配的文件 + corr_file = matching_corr_files[0] + try: + with rasterio.open(corr_file) as src: + # 检查尺寸是否匹配 + if src.width == self.width and src.height == self.height: + corr_data = src.read(1) + coherence_data.append(corr_data) + used_corr_files.append(corr_file) + print(f" 已加载相干性: {os.path.basename(corr_file)}") + else: + print(f" 警告: 相干性文件尺寸不匹配，使用默认值") + coherence_data.append(np.ones((self.height, self.width))) + except Exception as e: + print(f" 加载相干性文件失败: {e}") + coherence_data.append(np.ones((self.height, self.width))) + else: + # 没有找到匹配的相干性文件，使用默认值 + print(f" 未找到匹配的相干性文件，使用默认值") + coherence_data.append(np.ones((self.height, self.width))) + + # 如果相干性数据数量与解缠相位不匹配，补充默认值 + while len(coherence_data) < len(unw_phase_data): + coherence_data.append(np.ones((self.height, self.width))) + + # 转换为numpy数组 + coherence_array = np.array(coherence_data) + print(f"相干性数据形状: {coherence_array.shape}") + + # 创建xarray数据集 + ds = xr.Dataset( + { + 'unw_phase': (['granule', 'y', 'x'], unw_phase_array), + 'coherence': (['granule', 'y', 'x'], coherence_array), + 'elevation': (['y', 'x'], elevation_data) + }, + coords={ + 'granule': granule_names, + 'y': np.arange(self.height), + 'x': np.arange(self.width) + } + ) + + # 设置地理参考属性 + try: + ds.rio.set_crs(self.crs) + ds.rio.write_transform(self.transform, inplace=True) + except: + print("警告: 无法设置地理参考信息") + + print(f"成功加载数据: {len(granule_names)} 个干涉图") + self.ds = ds + return ds + + def select_reference_points(self, corr_thresh=0.6, n_refs=1000): + """ + 选择参考点 + """ + if self.ds is None: + raise ValueError("请先加载数据") + + print(f"步骤 1: 选择参考点 (相干性阈值: {corr_thresh}, 数量: {n_refs})") + + # 计算平均相干性 + coh_mean = self.ds.coherence.mean(dim='granule').values + + # 获取高相干性像素 + valid_mask = coh_mean >= corr_thresh + valid_coords = np.argwhere(valid_mask) + + if len(valid_coords) == 0: + print(f"警告: 没有找到相干性大于{corr_thresh}的像素，降低阈值到0.3") + valid_mask = coh_mean >= 0.3 + valid_coords = np.argwhere(valid_mask) + + if len(valid_coords) == 0: + print("警告: 仍然没有找到高相干性像素，使用所有像素") + valid_coords = np.array([[i, j] for i in range(self.height) for j in range(self.width)]) + + print(f" 找到 {len(valid_coords)} 个候选像素点") + + # 调整参考点数量 + if len(valid_coords) < n_refs: + n_refs = len(valid_coords) + print(f" 可用点不足，调整为选择 {n_refs} 个参考点") + + # 使用网格策略选择参考点 + grid_size = int(np.sqrt(n_refs / 4)) + grid_size = max(5, min(grid_size, 50)) + + x_bins = np.linspace(0, self.width, grid_size + 1, dtype=int) + y_bins = np.linspace(0, self.height, grid_size + 1, dtype=int) + + ref_list = [] + points_per_cell = max(1, n_refs // (grid_size * grid_size)) + + for i in range(grid_size): + for j in range(grid_size): + x_min, x_max = x_bins[i], x_bins[i+1] + y_min, y_max = y_bins[j], y_bins[j+1] + + # 在当前网格内选择点 + cell_mask = ((valid_coords[:, 1] >= x_min) & (valid_coords[:, 1] < x_max) & + (valid_coords[:, 0] >= y_min) & (valid_coords[:, 0] < y_max)) + cell_points = valid_coords[cell_mask] + + if len(cell_points) > 0: + n_select = min(points_per_cell, len(cell_points)) + selected_indices = np.random.choice(len(cell_points), n_select, replace=False) + + for idx in selected_indices: + y, x = cell_points[idx] + ref_list.append([int(x), int(y)]) + + # 如果点数不足，随机补充 + if len(ref_list) < n_refs: + remaining = n_refs - len(ref_list) + additional_indices = np.random.choice(len(valid_coords), remaining, replace=False) + for idx in additional_indices: + y, x = valid_coords[idx] + if [int(x), int(y)] not in ref_list: + ref_list.append([int(x), int(y)]) + + self.refs = ref_list[:n_refs] + print(f" 成功选择 {len(self.refs)} 个参考点") + return self.refs + + def sample_reference_data(self): + """采样参考点数据""" + if not self.refs: + raise ValueError("请先选择参考点") + + print("步骤 2: 采样参考点数据") + + # 验证参考点坐标 + valid_refs = [] + for ref in self.refs: + x, y = ref[0], ref[1] + if 0 <= x < self.width and 0 <= y < self.height: + valid_refs.append(ref) + else: + print(f"警告: 参考点 ({x}, {y}) 超出图像范围") + + if not valid_refs: + raise ValueError("没有有效的参考点") + + # 提取坐标 + x_coords = [ref[0] for ref in valid_refs] + y_coords = [ref[1] for ref in valid_refs] + + # 采样高程数据 + ref_elevation = [] + for x, y in valid_refs: + elev_val = self.ds.elevation.isel(x=x, y=y).values + ref_elevation.append(float(elev_val)) + ref_elevation = np.array(ref_elevation) + + # 采样相位数据 + ref_values = [] + for i in range(len(self.ds.granule)): + granule_phases = [] + for x, y in valid_refs: + phase_val = self.ds.unw_phase.isel(granule=i, x=x, y=y).values + granule_phases.append(float(phase_val)) + ref_values.append(granule_phases) + ref_values = np.array(ref_values) + + print(f" 成功采样 {len(valid_refs)} 个参考点，{len(ref_values)} 个干涉图") + return ref_values, ref_elevation + + def numpy_linear_regression(self, x, y): + """ + 使用NumPy实现线性回归 + """ + # 移除NaN值 + valid_mask = ~(np.isnan(x) | np.isnan(y)) + x_valid = x[valid_mask] + y_valid = y[valid_mask] + + if len(x_valid) < 2: + return np.nan, np.nan, 0, 0 + + # 计算线性回归参数 + x_mean = np.mean(x_valid) + y_mean = np.mean(y_valid) + + # 计算协方差和方差 + cov_xy = np.mean((x_valid - x_mean) * (y_valid - y_mean)) + var_x = np.mean((x_valid - x_mean) ** 2) + + if var_x == 0: + return np.nan, np.nan, 0, 0 + + beta = cov_xy / var_x + alpha = y_mean - beta * x_mean + + # 计算R² + y_pred = alpha + beta * x_valid + ss_res = np.sum((y_valid - y_pred) ** 2) + ss_tot = np.sum((y_valid - y_mean) ** 2) + + if ss_tot == 0: + r_squared = 0 + else: + r_squared = 1 - (ss_res / ss_tot) + + n_points = len(x_valid) + + return beta, alpha, r_squared, n_points + + def perform_linear_fits(self, ref_values, ref_elevation): + """ + 执行线性拟合 + """ + print("步骤 3: 线性拟合") + + fits = [] + fit_metrics = [] + elevations = np.array(ref_elevation) + + for i in range(len(ref_values)): + phases = ref_values[i] + + # 执行线性回归 + slope, intercept, r2, n_points = self.numpy_linear_regression(elevations, phases) + + fits.append([slope, intercept]) + fit_metrics.append({ + 'r_squared': r2, + 'n_points': n_points + }) + + if (i + 1) % 10 == 0 or (i + 1) == len(ref_values): + print(f" 已完成 {i+1}/{len(ref_values)} 个干涉图") + + # 统计结果 + valid_fits = sum(1 for fit in fits if not np.isnan(fit[0])) + valid_r2 = [m['r_squared'] for m in fit_metrics if not np.isnan(m['r_squared'])] + avg_r2 = np.mean(valid_r2) if valid_r2 else 0 + + self.fits = fits + self.fit_metrics = fit_metrics + + print(f" 线性拟合完成: {valid_fits}/{len(fits)} 个成功, 平均R²: {avg_r2:.3f}") + return fits, fit_metrics + + def apply_correction(self, min_r2=0.0): + """应用相位校正""" + print("步骤 4: 应用相位校正") + + if not self.fits: + raise ValueError("请先进行线性拟合") + + # 过滤低质量拟合 + valid_fits = [] + valid_indices = [] + + for i, fit in enumerate(self.fits): + if np.isnan(fit[0]) or np.isnan(fit[1]): + continue + + if self.fit_metrics[i]['r_squared'] < min_r2: + continue + + valid_fits.append(fit) + valid_indices.append(i) + + if not valid_fits: + print("警告: 没有通过质量控制的拟合，使用所有有效拟合") + valid_fits = [f for f in self.fits if not np.isnan(f[0]) and not np.isnan(f[1])] + valid_indices = [i for i, f in enumerate(self.fits) + if not np.isnan(f[0]) and not np.isnan(f[1])] + + if not valid_fits: + raise ValueError("没有有效的拟合可用于校正") + + print(f" 使用 {len(valid_fits)}/{len(self.fits)} 个拟合进行校正") + + # 创建校正后的数据集 + self.ds_corrected = self.ds.copy() + + # 创建校正后的相位数组 + corrected_phase = np.zeros_like(self.ds.unw_phase.values) + + # 对每个干涉图应用校正 + for idx, granule_idx in enumerate(valid_indices): + slope, intercept = valid_fits[idx] + + # 计算校正量: phase = slope * elevation + intercept + elevation_data = self.ds.elevation.values + correction = elevation_data * slope + intercept + + # 应用校正: corrected_phase = original_phase - correction + original_phase = self.ds.unw_phase[granule_idx].values + corrected_phase[granule_idx] = original_phase - correction + + # 添加校正后的变量 + self.ds_corrected['unw_phase_corrected'] = (('granule', 'y', 'x'), corrected_phase) + + return self.ds_corrected + + def save_results_as_geotiff(self, output_dir='.'): + """保存结果为GeoTIFF格式""" + if self.ds_corrected is None: + raise ValueError("请先进行校正") + + print("步骤 5: 保存GeoTIFF格式结果") + os.makedirs(output_dir, exist_ok=True) + + # 保存校正后的每个干涉图 + corrected_dir = os.path.join(output_dir, "corrected_phase") + os.makedirs(corrected_dir, exist_ok=True) + + for i, granule in enumerate(self.ds_corrected.granule.values): + # 获取校正后的相位数据 + phase_corrected = self.ds_corrected.unw_phase_corrected[i].values + + # 创建输出文件名 + output_file = os.path.join(corrected_dir, f"{granule}_corrected.tif") + + # 使用rasterio保存为GeoTIFF + try: + with rasterio.open( + output_file, 'w', + driver='GTiff', + height=self.height, + width=self.width, + count=1, + dtype=phase_corrected.dtype, + crs=self.crs, + transform=self.transform + ) as dst: + dst.write(phase_corrected, 1) + + print(f" 已保存: {output_file}") + except Exception as e: + print(f" 保存失败 {output_file}: {e}") + + # 保存平均校正后相位 + try: + mean_corrected = self.ds_corrected.unw_phase_corrected.mean(dim='granule').values + mean_file = os.path.join(output_dir, "mean_corrected_phase.tif") + + with rasterio.open( + mean_file, 'w', + driver='GTiff', + height=self.height, + width=self.width, + count=1, + dtype=mean_corrected.dtype, + crs=self.crs, + transform=self.transform + ) as dst: + dst.write(mean_corrected, 1) + + print(f" 已保存平均校正相位: {mean_file}") + except Exception as e: + print(f" 保存平均校正相位失败: {e}") + + # 保存原始平均相位用于对比 + try: + mean_original = self.ds.unw_phase.mean(dim='granule').values + mean_orig_file = os.path.join(output_dir, "mean_original_phase.tif") + + with rasterio.open( + mean_orig_file, 'w', + driver='GTiff', + height=self.height, + width=self.width, + count=1, + dtype=mean_original.dtype, + crs=self.crs, + transform=self.transform + ) as dst: + dst.write(mean_original, 1) + + print(f" 已保存原始平均相位: {mean_orig_file}") + except Exception as e: + print(f" 保存原始平均相位失败: {e}") + + # 保存高程数据 + try: + elev_file = os.path.join(output_dir, "elevation.tif") + elevation_data = self.ds.elevation.values + + with rasterio.open( + elev_file, 'w', + driver='GTiff', + height=self.height, + width=self.width, + count=1, + dtype=elevation_data.dtype, + crs=self.crs, + transform=self.transform + ) as dst: + dst.write(elevation_data, 1) + + print(f" 已保存高程数据: {elev_file}") + except Exception as e: + print(f" 保存高程数据失败: {e}") + + print(f"所有GeoTIFF文件已保存至: {output_dir}") + + def create_diagnostic_plots(self, ref_values, ref_elevation, output_dir='.'): + """创建诊断图表""" + print("步骤 6: 生成诊断图表") + os.makedirs(output_dir, exist_ok=True) + + # 创建综合诊断图 + fig, axes = plt.subplots(2, 3, figsize=(15, 10)) + + # 1. 参考点分布 + ax1 = axes[0, 0] + elevation_data = self.ds.elevation.values + im1 = ax1.imshow(elevation_data, cmap='terrain') + if self.refs: + x_coords, y_coords = zip(*self.refs) + ax1.scatter(x_coords, y_coords, c='red', s=5, alpha=0.7, label='Reference Points') + ax1.legend() + ax1.set_title('Reference Points Distribution') + plt.colorbar(im1, ax=ax1, label='Elevation (m)') + + # 2. 第一个干涉图的相位-高程关系 + ax2 = axes[0, 1] + if len(ref_values) > 0: + phases = ref_values[0] + valid_mask = ~(np.isnan(ref_elevation) | np.isnan(phases)) + + if np.sum(valid_mask) > 0: + ax2.scatter(ref_elevation[valid_mask], phases[valid_mask], + alpha=0.5, s=10) + + # 绘制拟合线 + if not np.isnan(self.fits[0][0]): + elev_min, elev_max = np.min(ref_elevation[valid_mask]), np.max(ref_elevation[valid_mask]) + elev_range = np.linspace(elev_min, elev_max, 100) + phase_fit = self.fits[0][0] * elev_range + self.fits[0][1] + ax2.plot(elev_range, phase_fit, 'r-', linewidth=2, + label=f'Slope: {self.fits[0][0]:.4f}') + ax2.legend() + + ax2.set_xlabel('Elevation (m)') + ax2.set_ylabel('Phase (rad)') + ax2.set_title('Phase vs Elevation') + ax2.grid(True, alpha=0.3) + + # 3. 拟合斜率分布 + ax3 = axes[0, 2] + slopes = [f[0] for f in self.fits if not np.isnan(f[0])] + if slopes: + ax3.hist(slopes, bins=20, alpha=0.7, density=True) + ax3.axvline(np.mean(slopes), color='r', linestyle='--', + label=f'Mean: {np.mean(slopes):.4f}') + ax3.legend() + ax3.set_xlabel('Slope') + ax3.set_ylabel('Density') + ax3.set_title('Slope Distribution') + ax3.grid(True, alpha=0.3) + + # 4. 校正前后对比 + ax4 = axes[1, 0] + phase_before = self.ds.unw_phase.mean(dim='granule').values + vmin, vmax = -np.pi, np.pi + im4 = ax4.imshow(phase_before, cmap='RdBu', vmin=vmin, vmax=vmax) + ax4.set_title('Mean Phase (Before Correction)') + plt.colorbar(im4, ax=ax4, label='Phase (rad)') + + ax5 = axes[1, 1] + phase_after = self.ds_corrected.unw_phase_corrected.mean(dim='granule').values + im5 = ax5.imshow(phase_after, cmap='RdBu', vmin=vmin, vmax=vmax) + ax5.set_title('Mean Phase (After Correction)') + plt.colorbar(im5, ax=ax5, label='Phase (rad)') + + # 5. 校正量 + ax6 = axes[1, 2] + correction = phase_before - phase_after + im6 = ax6.imshow(correction, cmap='viridis') + ax6.set_title('Phase Correction') + plt.colorbar(im6, ax=ax6, label='Correction (rad)') + + plt.tight_layout() + plt.savefig(f'{output_dir}/MuRP_diagnostics.png', dpi=300, bbox_inches='tight') + plt.close() + + print(f" 诊断图已保存至: {output_dir}/MuRP_diagnostics.png") + + def calculate_improvement(self): + """计算改善统计""" + if self.ds_corrected is None: + return 0, 0, 0 + + # 计算时间序列标准差 + std_before = float(self.ds.unw_phase.std(dim='granule').mean().values) + std_after = float(self.ds_corrected.unw_phase_corrected.std(dim='granule').mean().values) + + improvement = (std_before - std_after) / std_before * 100 if std_before != 0 else 0 + + return improvement, std_before, std_after + + def run_murp_correction(self, data_path, corr_thresh=0.6, n_refs=1000, + output_dir='.', create_plots=True, save_geotiff=True): + """ + 运行完整的MuRP校正流程 + """ + start_time = datetime.now() + print("="*60) + print("修复版MuRP算法开始执行") + print(f"开始时间: {start_time.strftime('%Y-%m-%d %H:%M:%S')}") + print("="*60) + + try: + # 1. 加载数据 + self.load_hyp3_data_robust(data_path) + + # 2. 选择参考点 + self.select_reference_points(corr_thresh, n_refs) + + # 3. 采样参考点数据 + ref_values, ref_elevation = self.sample_reference_data() + + # 4. 线性拟合 + self.perform_linear_fits(ref_values, ref_elevation) + + # 5. 应用校正 + self.apply_correction() + + # 6. 生成诊断图表 + if create_plots: + self.create_diagnostic_plots(ref_values, ref_elevation, output_dir) + + # 7. 保存GeoTIFF格式结果 + if save_geotiff: + self.save_results_as_geotiff(output_dir) + + # 计算改善统计 + improvement, std_before, std_after = self.calculate_improvement() + + end_time = datetime.now() + duration = (end_time - start_time).total_seconds() + + print("="*60) + print("修复版MuRP算法执行完成") + print(f"结束时间: {end_time.strftime('%Y-%m-%d %H:%M:%S')}") + print(f"总耗时: {duration:.1f} 秒") + print(f"时间序列标准差改善: {improvement:.1f}%") + print(f" 校正前: {std_before:.4f} rad") + print(f" 校正后: {std_after:.4f} rad") + print("="*60) + + return self.ds_corrected + + except Exception as e: + print(f"算法执行失败: {e}") + import traceback + traceback.print_exc() + return None + +def main(): + """命令行接口主函数""" + parser = argparse.ArgumentParser( + description='修复版MuRP算法 - 解决相干性数据形状不一致问题', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=''' +使用示例: + python fixed_murp.py /path/to/hyp3/data + python fixed_murp.py /path/to/hyp3/data --corr_thresh 0.6 --n_refs 500 + python fixed_murp.py /path/to/hyp3/data --output_dir ./results + ''' + ) + + parser.add_argument('data_path', help='Hyp3数据目录路径') + parser.add_argument('--corr_thresh', type=float, default=0.6, + help='相干性阈值 (默认: 0.6)') + parser.add_argument('--n_refs', type=int, default=1000, + help='参考点数量 (默认: 1000)') + parser.add_argument('--output_dir', default='.', + help='输出目录 (默认: 当前目录)') + parser.add_argument('--no_plots', action='store_true', + help='不生成诊断图表') + parser.add_argument('--no_geotiff', action='store_true', + help='不保存GeoTIFF格式') + + args = parser.parse_args() + + try: + # 创建MuRP处理器 + murp = FixedMuRP(random_seed=42) + + # 运行算法 + ds_corrected = murp.run_murp_correction( + data_path=args.data_path, + corr_thresh=args.corr_thresh, + n_refs=args.n_refs, + output_dir=args.output_dir, + create_plots=not args.no_plots, + save_geotiff=not args.no_geotiff + ) + + if ds_corrected is not None: + print("算法执行成功!") + else: + print("算法执行失败!") + sys.exit(1) + + except Exception as e: + print(f"错误: {e}") + import traceback + traceback.print_exc() + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/third_party/PyINT/pyint/create_gacos.py b/third_party/PyINT/pyint/create_gacos.py new file mode 100644 index 0000000..1d28de3 --- /dev/null +++ b/third_party/PyINT/pyint/create_gacos.py @@ -0,0 +1,136 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Wei Chen ### +### Contact : chenweicug@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import argparse +import subprocess +from pyint import _utils as ut + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister SM mode SLC to a reference SLC image using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='Name of project.') + parser.add_argument('mdate', help='date of the slave SLC image') + parser.add_argument('sdate', help='date of the slave SLC image') + inps = parser.parse_args() + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + create_files for psokinv software running file by generated by Gamma +''' + +EXAMPLE = """Usage: + + coreg_gamma.py projectName + + coreg_gamma.py PacayaT163TsxHhA +------------------------------------------------------------------- +""" + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Sdate = inps.sdate + Mdate = inps.mdate + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Smdate = templateDict['masterDate'] + + demDir = scratchDir + '/' + projectName + '/DEM' + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + ifgramDir = scratchDir + '/' + projectName + "/ifgrams" + workDir = ifgramDir + '/' + Mdate + '-' + Sdate + MslcDir = rslcDir + '/' + Mdate + SslcDir = rslcDir + '/' + Sdate + + offpar = SslcDir + '/' + Smdate + '_' + Sdate + '.off' + slcpar = MslcDir + '/' + Smdate + '.rslc.par' + dempar = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem.par' + dem = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem' + unwpar = demDir + '/' + Mdate + '_' + rlks + 'rlks.utm.dem.par' + unw = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.diff_filt.unw" + + os.chdir(workDir) + call_str = "look_vector " + slcpar+ " " + offpar + " " + dempar + " " + dem + ' lv_theta lv_phi' + os.system(call_str) + call_str = "grep 'corner_lat:' " + dempar + " | awk '{print $2}' " + North=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'corner_lon:' " + dempar + " | awk '{print $2}' " + West=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lat:' " + dempar + " | awk '{print $2}' " + posty=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lon:' " + dempar + " | awk '{print $2}' " + postx=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'width:' " + dempar + " | awk '{print $2}' " + width=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'nlines:' " + dempar + " | awk '{print $2}' " + length=subprocess.getstatusoutput(call_str)[1] + South=str(round(float(North)+(float(length)-1)*float(posty),7)) + East=str(round(float(West)+(float(width)-1)*float(postx),7)) + + call_str = "swap_bytes " + unw+ " unw.phase_swap 4 >dinsar.log " + os.system(call_str) + call_str = "gmt xyz2grd unw.phase_swap -Gunw_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx +" -ZTLf -N0" + os.system(call_str) + call_str = "gmt grdsample unw_f.grd -Gunw.grd -I3c" + os.system(call_str) + call_str = "gmt grd2xyz unw.grd -ZTLf -N0 > " + Mdate + '-' + Sdate + '.unw' + os.system(call_str) + call_str = "gmt grdinfo unw.grd -C | awk '{print $10}' " + Width=subprocess.getstatusoutput(call_str)[1] + call_str = "gmt grdinfo unw.grd -C | awk '{print $11}' " + line=subprocess.getstatusoutput(call_str)[1] + call_str = "gmt grdinfo unw.grd -C | awk '{print $2}' " + West=subprocess.getstatusoutput(call_str)[1] + call_str = "gmt grdinfo unw.grd -C | awk '{print $5}' " + North=subprocess.getstatusoutput(call_str)[1] + ymax=str(int(round(float(line) ,7))) + xmax=str(int(round(float(Width) ,7))) + output=workDir + '/' + Mdate + '-' + Sdate + '.gacos.unw.rsc' + if os.path.exists(output) is True: + os.remove(output) + + fopen=open(output,'a+') + fopen.write('WIDTH ' + Width + '\n') + fopen.write('FILE_LENGTH ' + line + '\n') + fopen.write('XMIN 1' + '\n') + fopen.write('XMAX ' + xmax + '\n') + fopen.write('YMIN 1' + '\n') + fopen.write('YMAX ' + ymax + '\n') + fopen.write('X_FIRST ' + West + '\n') + fopen.write('Y_FIRST ' + North + '\n') + fopen.write('X_STEP 8.33333333E-04' + '\n') + fopen.write('Y_STEP -8.33333333E-04' + '\n') + fopen.write('X_UNIT degrees' + '\n') + fopen.write('Y_UNIT degrees' + '\n') + fopen.write('Z_OFFSET 0' + '\n') + fopen.write('Z_SCALE 1' + '\n') + fopen.write('PROJECTION LATLON' + '\n') + fopen.write('DATUM WGS84' + '\n') + + print("convert gamma products to GACOS is done!") + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + diff --git a/third_party/PyINT/pyint/create_psokinv.py b/third_party/PyINT/pyint/create_psokinv.py new file mode 100644 index 0000000..fc812a8 --- /dev/null +++ b/third_party/PyINT/pyint/create_psokinv.py @@ -0,0 +1,180 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import argparse +import subprocess +from pyint import _utils as ut + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister SM mode SLC to a reference SLC image using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='Name of project.') + parser.add_argument('mdate', help='date of the slave SLC image') + parser.add_argument('sdate', help='date of the slave SLC image') + inps = parser.parse_args() + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + create_files for psokinv software running file by generated by Gamma +''' + +EXAMPLE = """Usage: + + create_psokinv.py projectName mdata sdata lon_min lon_max lat_min lat_max flag + + create_psokinv.py PacayaT163TsxHhA 20241203 20241215 80 81 39 30 1 +------------------------------------------------------------------- +""" + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Sdate = inps.sdate + Mdate = inps.mdate + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + + + + + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Smdate = templateDict['masterDate'] + + demDir = scratchDir + '/' + projectName + '/DEM' + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + ifgramDir = scratchDir + '/' + projectName + "/ifgrams" + workDir = ifgramDir + '/' + Mdate + '-' + Sdate + MslcDir = rslcDir + '/' + Mdate + SslcDir = rslcDir + '/' + Sdate + + offpar = SslcDir + '/' + Smdate + '_' + Sdate + '.off' + slcpar = MslcDir + '/' + Smdate + '.rslc.par' + dempar = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem.par' + dem = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem' + unwpar = demDir + '/' + Mdate + '_' + rlks + 'rlks.utm.dem.par' + unw = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.diff_filt.unw" + los = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.dispmap" + cor = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.diff_filt.cor" + + os.chdir(workDir) + call_str = "look_vector " + slcpar+ " " + offpar + " " + dempar + " " + dem + ' lv_theta lv_phi' + os.system(call_str) + call_str = "grep 'corner_lat:' " + dempar + " | awk '{print $2}' " + North=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'corner_lon:' " + dempar + " | awk '{print $2}' " + West=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lat:' " + dempar + " | awk '{print $2}' " + posty=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lon:' " + dempar + " | awk '{print $2}' " + postx=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'width:' " + dempar + " | awk '{print $2}' " + width=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'nlines:' " + dempar + " | awk '{print $2}' " + length=subprocess.getstatusoutput(call_str)[1] + South=str(float(North)+float(posty)*(float(length)-1)) + East=str(float(West)+float(postx)*(float(width)-1)) + #East=str(round(float(West)+(float(width)-1)*float(postx),7)) + + + call_str = "swap_bytes lv_theta lv_theta.phase_swap 4 >dinsar.log " + os.system(call_str) + call_str = "gmt xyz2grd lv_theta.phase_swap -Glv_theta.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -di0" + os.system(call_str) + call_str = "gmt grdmath 90 lv_theta.grd 3.1415926 DIV 180 MUL SUB = lv_theta_final.grd" + os.system(call_str) + call_str = "gmt grd2xyz lv_theta_final.grd -ZTLf -N0 >" +Mdate + '-' + Sdate + '.inc' + os.system(call_str) + call_str = "swap_bytes lv_phi lv_phi.phase_swap 4 >dinsar.log " + os.system(call_str) + call_str = "gmt xyz2grd lv_phi.phase_swap -Glv_phi.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -di0" + os.system(call_str) + call_str = "gmt grdmath -90 lv_phi.grd 3.1415926 DIV 180 MUL SUB = lv_phi_final.grd" + os.system(call_str) + call_str = "gmt grd2xyz lv_phi_final.grd -ZTLf -N0 >" +Mdate + '-' + Sdate + '.azi' + os.system(call_str) + call_str = "swap_bytes " + unw+ " unw.phase_swap 4 >dinsar.log " + os.system(call_str) + call_str = "gmt xyz2grd unw.phase_swap -Gunw.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -di0" + os.system(call_str) + call_str = "gmt grdmath 0.0556 unw.grd 3.1415926 DIV -4 DIV MUL = los.grd" + os.system(call_str) + call_str = "gmt grd2xyz unw.grd -ZTLf -N0 > " + Mdate + '-' + Sdate + '.unw' + os.system(call_str) + call_str = "swap_bytes " + dem + " dem.phase_swap 4 >dinsar.log " + os.system(call_str) + #call_str = "xyz2grd dem.phase_swap -Gdem_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I0.001" + " -ZTLf -N0" + call_str = "gmt xyz2grd dem.phase_swap -Gdem_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -di0" + os.system(call_str) + call_str = "gmt grd2xyz dem_f.grd -ZTLf -N0 >" + Mdate + '-' + Sdate + '.utm.dem' + os.system(call_str) + call_str = "gmt grd2xyz los.grd -ZTLf -N0 >" + Mdate + '-' + Sdate + '.utm.los' + os.system(call_str) + call_str = "gmt grdinfo lv_theta_final.grd -C | awk '{print $10}' " + Width=subprocess.getstatusoutput(call_str)[1] + call_str = "gmt grdinfo lv_theta_final.grd -C | awk '{print $11}' " + line=subprocess.getstatusoutput(call_str)[1] + call_str = "gmt grdinfo lv_theta_final.grd -C | awk '{print $2}' " + West=subprocess.getstatusoutput(call_str)[1] + call_str = "gmt grdinfo lv_theta_final.grd -C | awk '{print $5}' " + North=subprocess.getstatusoutput(call_str)[1] + ymax=str(int(round(float(line)-1 ,7))) + xmax=str(int(round(float(Width)-1 ,7))) + output=workDir + '/' + Mdate + '-' + Sdate + '.unw.rsc' + if os.path.exists(output) is True: + os.remove(output) + fopen=open(output,'a+') + fopen.write('WIDTH ' + Width + '\n') + fopen.write('FILE_LENGTH ' + line + '\n') + fopen.write('XMIN 0' + '\n') + fopen.write('XMAX ' + xmax + '\n') + fopen.write('YMIN 0' + '\n') + fopen.write('YMAX ' + ymax + '\n') + fopen.write('RLOOKS 1' + '\n') + fopen.write('ALOOKS 1' + '\n') + fopen.write('X_FIRST ' + West + '\n') + fopen.write('X_STEP 8.33333333E-04' + '\n') + fopen.write('X_UNIT degrees' + '\n') + fopen.write('Y_FIRST ' + North + '\n') + fopen.write('Y_STEP -8.33333333E-04' + '\n') + fopen.write('Y_UNIT degrees' + '\n') + fopen.write('WAVELENGTH 0.0555041577' + width + '\n') + #fopen.write('WAVELENGTH 0.236' + width + '\n') + fopen.write('MOLDE insar') + fopen.close() + call_str = "swap_bytes " + Mdate + '-' + Sdate + '.unw' + " geo_" + Mdate + '-' + Sdate + "_dem_flat.unw 4 >dinsar.log " + os.system(call_str) + call_str = "swap_bytes " + Mdate + '-' + Sdate + '.utm.dem' + " geo_" + Mdate + '-' + Sdate + "_dem_flat.utm.dem 4 >dinsar.log " + os.system(call_str) + call_str = "cp -f " + Mdate + '-' + Sdate + ".unw.rsc " + Mdate + '-' + Sdate +".inc.rsc" + os.system(call_str) + call_str = "cp -f " + Mdate + '-' + Sdate + ".unw.rsc " + Mdate + '-' + Sdate +".azi.rsc" + os.system(call_str) + call_str = "cp -f " + Mdate + '-' + Sdate + ".unw.rsc " + " geo_" + Mdate + '-' + Sdate + "_dem_flat.unw.rsc" + os.system(call_str) + call_str = "cp -f " + Mdate + '-' + Sdate + ".unw.rsc " + " geo_" + Mdate + '-' + Sdate + "_dem_flat.utm.dem.rsc" + os.system(call_str) + call_str = "rm -rf swap*grd *phase_swap dinsar.log" + os.system(call_str) + print("convert gamma products to psokinv is done!") + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/create_psokinv_cut.py b/third_party/PyINT/pyint/create_psokinv_cut.py new file mode 100644 index 0000000..6affcfb --- /dev/null +++ b/third_party/PyINT/pyint/create_psokinv_cut.py @@ -0,0 +1,235 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import argparse +import subprocess +from pyint import _utils as ut + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister SM mode SLC to a reference SLC image using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='Name of project.') + parser.add_argument('mdate', help='date of the slave SLC image') + parser.add_argument('sdate', help='date of the slave SLC image') + parser.add_argument('lon_min', help='the min lon of cut image') + parser.add_argument('lon_max', help='the max lon of cut image') + parser.add_argument('lat_min', help='the min lat of cut image') + parser.add_argument('lat_max', help='the min lat of cut image') + parser.add_argument('flag', help="the cut image flag (1: do; 0:don't do)") + inps = parser.parse_args() + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + create_files for psokinv software running file by generated by Gamma +''' + +EXAMPLE = """Usage: + + create_psokinv.py projectName mdata sdata lon_min lon_max lat_min lat_max flag + + create_psokinv.py PacayaT163TsxHhA 20241203 20241215 80 81 39 30 1 +------------------------------------------------------------------- +""" + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Sdate = inps.sdate + Mdate = inps.mdate + lon_max = inps.lon_max + lon_min = inps.lon_min + lat_min = inps.lat_min + lat_max = inps.lat_max + flag = inps.flag + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + + + + + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Smdate = templateDict['masterDate'] + + demDir = scratchDir + '/' + projectName + '/DEM' + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + ifgramDir = scratchDir + '/' + projectName + "/ifgrams" + workDir = ifgramDir + '/' + Mdate + '-' + Sdate + MslcDir = rslcDir + '/' + Mdate + SslcDir = rslcDir + '/' + Sdate + + offpar = SslcDir + '/' + Smdate + '_' + Sdate + '.off' + slcpar = MslcDir + '/' + Smdate + '.rslc.par' + dempar = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem.par' + dem = demDir + '/' + Smdate + '_' + rlks + 'rlks.utm.dem' + unwpar = demDir + '/' + Mdate + '_' + rlks + 'rlks.utm.dem.par' + unw = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.diff_filt.unw" + los = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.dispmap" + cor = workDir + '/geo_' + Mdate + '-' + Sdate + "_" + rlks+"rlks.diff_filt.cor" + + os.chdir(workDir) + call_str = "look_vector " + slcpar+ " " + offpar + " " + dempar + " " + dem + ' lv_theta lv_phi' + os.system(call_str) + call_str = "grep 'corner_lat:' " + dempar + " | awk '{print $2}' " + North=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'corner_lon:' " + dempar + " | awk '{print $2}' " + West=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lat:' " + dempar + " | awk '{print $2}' " + posty=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lon:' " + dempar + " | awk '{print $2}' " + postx=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'width:' " + dempar + " | awk '{print $2}' " + width=subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'nlines:' " + dempar + " | awk '{print $2}' " + length=subprocess.getstatusoutput(call_str)[1] + South=str(round(float(North)+(float(length)-1)*float(posty),7)) + East=str(round(float(West)+(float(width)-1)*float(postx),7)) + + + call_str = "swap_bytes lv_theta lv_theta.phase_swap 4 >dinsar.log " + os.system(call_str) + call_str = "gmt xyz2grd lv_theta.phase_swap -Glv_theta.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -N0" + os.system(call_str) + call_str = "gmt grdmath 90 lv_theta.grd 3.1415926 DIV 180 MUL SUB = lv_theta_final.grd" + os.system(call_str) + if flag == '0': + call_str = "gmt grdsample lv_theta_final.grd -Glv_theta.grd -I3c" + else: + call_str = "gmt grdsample lv_theta_final.grd -R" + lon_min + "/" + lon_max + "/" + lat_min + "/" + lat_max + " -Glv_theta.grd -I3c" + os.system(call_str) + call_str = "gmt grd2xyz lv_theta.grd -ZTLf -N0 >" +Mdate + '-' + Sdate + '.inc' + os.system(call_str) + call_str = "swap_bytes lv_phi lv_phi.phase_swap 4 >dinsar.log " + os.system(call_str) + call_str = "gmt xyz2grd lv_phi.phase_swap -Glv_phi.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -N0" + os.system(call_str) + call_str = "gmt grdmath -90 lv_phi.grd 3.1415926 DIV 180 MUL SUB = lv_phi_final.grd" + os.system(call_str) + if flag == '0': + call_str = "gmt grdsample lv_phi_final.grd -Glv_phi.grd -I3c" + else: + call_str = "gmt grdsample lv_phi_final.grd -R" + lon_min + "/" + lon_max + "/" + lat_min + "/" + lat_max + " -Glv_phi.grd -I3c" + os.system(call_str) + call_str = "gmt grd2xyz lv_phi.grd -ZTLf -N0 >" +Mdate + '-' + Sdate + '.azi' + os.system(call_str) + call_str = "swap_bytes " + unw+ " unw.phase_swap 4 >dinsar.log " + os.system(call_str) + call_str = "gmt xyz2grd unw.phase_swap -Gunw_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx +" -ZTLf -N0" + os.system(call_str) + call_str = "swap_bytes " + los + " los.swap 4 >dinsar.log " + os.system(call_str) + call_str = "gmt xyz2grd los.swap -Glos_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx +" -ZTLf -N0" + os.system(call_str) + call_str = "swap_bytes " + cor + " cor.swap 4 >dinsar.log " + os.system(call_str) + call_str = "gmt xyz2grd cor.swap -Gcor_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx +" -ZTLf -N0" + os.system(call_str) + + if flag == '0': + call_str = "gmt grdsample unw_f.grd -Gunw.grd -I3c" + else: + call_str = "gmt grdsample unw_f.grd -R" + lon_min + "/" + lon_max + "/" + lat_min + "/" + lat_max + " -Gunw.grd -I3c" + os.system(call_str) + call_str = "gmt grdmath 0.0556 unw_f.grd 3.1415926 DIV -4 DIV MUL = los.grd" + os.system(call_str) + call_str = "gmt grd2xyz unw.grd -ZTLf -N0 > " + Mdate + '-' + Sdate + '.unw' + os.system(call_str) + call_str = "swap_bytes " + dem + " dem.phase_swap 4 >dinsar.log " + os.system(call_str) + #call_str = "xyz2grd dem.phase_swap -Gdem_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I0.001" + " -ZTLf -N0" + call_str = "gmt xyz2grd dem.phase_swap -Gdem_f.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" +South + "/" +North + " -I" + postx + " -ZTLf -N0" + os.system(call_str) + if flag == '0': + call_str = "gmt grdsample dem_f.grd -Gdem.grd -I3c" + else: + call_str = "gmt grdsample dem_f.grd -R" + lon_min + "/" + lon_max + "/" + lat_min + "/" + lat_max + " -Gdem.grd -I3c" + os.system(call_str) + + call_str = "gmt grd2xyz dem.grd -ZTLf -N0 >" + Mdate + '-' + Sdate + '.utm.dem' + os.system(call_str) + + if flag == '0': + call_str = "gmt grdsample los_f.grd -Glos1.grd -I3c" + else: + call_str = "gmt grdsample los_f.grd -R" + lon_min + "/" + lon_max + "/" + lat_min + "/" + lat_max + " -Glos1.grd -I3c" + os.system(call_str) + + call_str = "gmt grd2xyz los1.grd -ZTLf -N0 >" + Mdate + '-' + Sdate + '.utm.los' + os.system(call_str) + if flag == '0': + call_str = "gmt grdsample cor_f.grd -Gcor.grd -I3c" + else: + call_str = "gmt grdsample cor_f.grd -R" + lon_min + "/" + lon_max + "/" + lat_min + "/" + lat_max + " -Gcor.grd -I3c" + os.system(call_str) + + call_str = "gmt grd2xyz cor.grd -ZTLf -N0 >" + Mdate + '-' + Sdate + '.utm.cor' + os.system(call_str) + + call_str = "gmt grdinfo lv_theta.grd -C | awk '{print $10}' " + Width=subprocess.getstatusoutput(call_str)[1] + call_str = "gmt grdinfo lv_theta.grd -C | awk '{print $11}' " + line=subprocess.getstatusoutput(call_str)[1] + call_str = "gmt grdinfo lv_theta.grd -C | awk '{print $2}' " + West=subprocess.getstatusoutput(call_str)[1] + call_str = "gmt grdinfo lv_theta.grd -C | awk '{print $5}' " + North=subprocess.getstatusoutput(call_str)[1] + ymax=str(int(round(float(line)-1 ,7))) + xmax=str(int(round(float(Width)-1 ,7))) + output=workDir + '/' + Mdate + '-' + Sdate + '.unw.rsc' + if os.path.exists(output) is True: + os.remove(output) + fopen=open(output,'a+') + fopen.write('WIDTH ' + Width + '\n') + fopen.write('FILE_LENGTH ' + line + '\n') + fopen.write('XMIN 0' + '\n') + fopen.write('XMAX ' + xmax + '\n') + fopen.write('YMIN 0' + '\n') + fopen.write('YMAX ' + ymax + '\n') + fopen.write('RLOOKS 1' + '\n') + fopen.write('ALOOKS 1' + '\n') + fopen.write('X_FIRST ' + West + '\n') + fopen.write('X_STEP 8.33333333E-04' + '\n') + fopen.write('X_UNIT degrees' + '\n') + fopen.write('Y_FIRST ' + North + '\n') + fopen.write('Y_STEP -8.33333333E-04' + '\n') + fopen.write('Y_UNIT degrees' + '\n') + fopen.write('WAVELENGTH 0.0555041577' + width + '\n') + #fopen.write('WAVELENGTH 0.236' + width + '\n') + fopen.write('MOLDE insar') + fopen.close() + call_str = "swap_bytes " + Mdate + '-' + Sdate + '.unw' + " geo_" + Mdate + '-' + Sdate + "_dem_flat.unw 4 >dinsar.log " + os.system(call_str) + call_str = "swap_bytes " + Mdate + '-' + Sdate + '.utm.dem' + " geo_" + Mdate + '-' + Sdate + "_dem_flat.utm.dem 4 >dinsar.log " + os.system(call_str) + call_str = "cp -f " + Mdate + '-' + Sdate + ".unw.rsc " + Mdate + '-' + Sdate +".inc.rsc" + os.system(call_str) + call_str = "cp -f " + Mdate + '-' + Sdate + ".unw.rsc " + Mdate + '-' + Sdate +".azi.rsc" + os.system(call_str) + call_str = "cp -f " + Mdate + '-' + Sdate + ".unw.rsc " + " geo_" + Mdate + '-' + Sdate + "_dem_flat.unw.rsc" + os.system(call_str) + call_str = "cp -f " + Mdate + '-' + Sdate + ".unw.rsc " + " geo_" + Mdate + '-' + Sdate + "_dem_flat.utm.dem.rsc" + os.system(call_str) + call_str = "rm -rf swap*grd *phase_swap dinsar.log" + os.system(call_str) + print("convert gamma products to psokinv is done!") + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/diff_gamma.py b/third_party/PyINT/pyint/diff_gamma.py new file mode 100644 index 0000000..7396087 --- /dev/null +++ b/third_party/PyINT/pyint/diff_gamma.py @@ -0,0 +1,183 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT ### +### Author: chen ### +### Contact : chenweicug@126.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +from pyint import _utils as ut + + +def _run_or_raise(call_str, stage): + rc = os.system(call_str) + if rc != 0: + raise RuntimeError('%s failed with rc=%s: %s' % (stage, rc, call_str)) + return rc + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Generate differential interferogram image from SLC using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + diff_gamma.py projectName Mdate Sdate + diff_gamma.py PacayaT163TsxHhA 20150102 20150601 +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister all of the SLCs to the reference SAR image using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('Mdate',help='Master date.') + parser.add_argument('Sdate',help='Slave date.') + + inps = parser.parse_args() + return inps + + +def main(argv): + + start_time = time.time() + inps = cmdLineParse() + Mdate = inps.Mdate + Sdate = inps.Sdate + + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + + projectDir = scratchDir + '/' + projectName + demDir = scratchDir + '/' + projectName + '/DEM' + + slcDir = scratchDir + '/' + projectName + '/SLC' + rslcDir = scratchDir + '/' + projectName + '/RSLC' + ifgDir = projectDir + '/ifgrams' + if not os.path.isdir(ifgDir): os.mkdir(ifgDir) + + Pair = Mdate + '-' + Sdate + workDir = ifgDir + '/' + Pair + if not os.path.isdir(workDir): os.mkdir(workDir) + + ####################################################################### + Mamp = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + Samp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + Mrslc = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc' + MrslcPar = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc.par' + Srslc = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc' + SrslcPar = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc.par' + + HGT = demDir + '/' + masterDate + '_' + rlks + 'rlks.rdc.dem' + + MasterPar = rslcDir + '/' + masterDate + '/' + masterDate + '.rslc.par' + + ################# copy file for parallel processing ########################## + #Mamp = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp' + #MampPar = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + #Samp = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp' + #SampPar = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + #if not templateDict['diff_all_parallel'] == '1': + + # Mrslc = workDir + '/' + Mdate + '.rslc' + # MrslcPar = workDir + '/' + Mdate + '.rslc.par' + # Srslc = workDir + '/' + Sdate + '.rslc' + # SrslcPar = workDir + '/' + Sdate + '.rslc.par' + # ut.copy_file(Mrslc0,Mrslc) + # ut.copy_file(MrslcPar0,MrslcPar) + # ut.copy_file(Srslc0,Srslc) + # ut.copy_file(SrslcPar0,SrslcPar) + + #else: + + # Mrslc = Mrslc0 + # MrslcPar = MrslcPar0 + # Srslc = Srslc0 + # SrslcPar = SrslcPar0 + # HGT = HGT0 + # MasterPar = MasterPar0 + + #ut.copy_file(Mamp0,Mamp) + #ut.copy_file(MampPar0,MampPar) + #ut.copy_file(Samp0,Samp) + #ut.copy_file(SampPar0,SampPar) + + #ut.copy_file(HGT0,HGT) + #ut.copy_file(MasterPar0,MasterPar) + + ############################################################################ + + OFF = workDir + '/' + Pair +'_' + rlks + 'rlks.off' + call_str = 'create_offset '+ MrslcPar + ' ' + SrslcPar + ' ' + OFF + ' 1 ' + rlks + ' ' + azlks + ' 0' + _run_or_raise(call_str, 'create_offset') + + SIM_UNW = workDir + '/' + Pair + '.sim_unw' + call_str = 'phase_sim_orb ' + MrslcPar + ' ' + SrslcPar + ' ' + OFF + ' ' + HGT + ' ' + SIM_UNW + ' ' + MasterPar + ' - - 1 1' + _run_or_raise(call_str, 'phase_sim_orb') + + DIFF_IFG = workDir + '/' + Pair + '_' + rlks + 'rlks.diff' + call_str = 'SLC_diff_intf ' + Mrslc + ' ' + Srslc + ' ' + MrslcPar + ' ' + SrslcPar + ' ' + OFF + ' ' + SIM_UNW + ' ' + DIFF_IFG + ' ' + rlks + ' ' + azlks + ' ' + templateDict['Igram_Spsflg'] + ' ' + templateDict['Igram_Azfflg'] + ' - 1 1' + _run_or_raise(call_str, 'SLC_diff_intf') + + ##### filtering process & coherence estimation ########### + DIFFFILT = workDir + '/' + Pair + '_' + rlks + 'rlks.diff_filt' + COHFILT = workDir + '/' + Pair + '_' + rlks + 'rlks.diff_filt.cor' + + nWIDTH = ut.read_gamma_par(OFF, 'read', 'interferogram_width') + call_str = 'adf ' + DIFF_IFG + ' ' + DIFFFILT + ' ' + COHFILT + ' ' + nWIDTH + ' ' + templateDict['adf_alpha'] + ' - ' + templateDict['Igram_Cor_Win'] + _run_or_raise(call_str, 'adf') + + ################# coherence estimation ##################### + call_str = 'cc_wave ' + DIFFFILT + ' ' + Mamp + ' ' + Samp + ' ' + COHFILT + ' ' + nWIDTH + ' ' + templateDict['Igram_Cor_rwin'] + ' ' + templateDict['Igram_Cor_awin'] + _run_or_raise(call_str, 'cc_wave') + + + ################ save images ##################### + call_str = 'rasmph_pwr ' + DIFFFILT + ' ' + Mamp + ' ' + nWIDTH + ' - - - - - - - - - ' + COHFILT + ' - 0.1' + _run_or_raise(call_str, 'rasmph_pwr_diff_filt') + + call_str = 'rasmph_pwr ' + DIFF_IFG + ' ' + Mamp + ' ' + nWIDTH + ' - - - - - - - - - ' + COHFILT + ' - 0.1' + _run_or_raise(call_str, 'rasmph_pwr_diff') + + call_str = 'rasdt_pwr ' + COHFILT + ' ' + Mamp + ' ' + nWIDTH + ' 1 0 1 1 0.1 1.0 1 ' + _run_or_raise(call_str, 'rasdt_pwr_coh') + + #os.remove(Mamp) + #os.remove(MampPar) + #os.remove(Samp) + #os.remove(SampPar) + + #if not templateDict['diff_all_parallel'] == '1': + # if os.path.isfile(Mrslc): os.remove(Mrslc) + # if os.path.isfile(Srslc):os.remove(Srslc) + + # if os.path.isfile(HGT):os.remove(HGT) + + print("Subtraction of topography and flattening phase is done!") + ut.print_process_time(start_time, time.time()) + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/diff_gamma_all.py b/third_party/PyINT/pyint/diff_gamma_all.py new file mode 100644 index 0000000..def2813 --- /dev/null +++ b/third_party/PyINT/pyint/diff_gamma_all.py @@ -0,0 +1,144 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + + +def work(data0): + cmd = data0[0] + err_file = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if type(stdout) == bytes: + bb=stdout.decode("utf-8", errors="replace") + else: + bb = stdout + + if p.returncode != 0: + detail_parts = [] + if bb: + detail_parts.append(bb) + if aa: + detail_parts.append(aa) + detail = '\n'.join(part.strip() for part in detail_parts if part and part.strip()) + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + cmd[3] + '\n' + with open(err_file, 'a') as f: + f.write(str0) + if detail: + f.write(detail) + f.write('\n') + raise RuntimeError(str0.strip() + ' failed with rc=' + str(p.returncode) + ('\n' + detail if detail else '')) + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + cmd[3] + '\n' + with open(err_file, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + Geneate differential interferograms for one project using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + diff_gamma_all.py projectName + diff_gamma_all.py projectName --parallel 4 + diff_gamma_all.py projectName --parallel 4 --ifgramList-txt /test/ifgram_list.txt +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Geneate differential interferograms for one project using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + parser.add_argument('--ifgarmList-txt', dest='ifgarmListTxt', help='provided ifgram_list_txt. default: using ifgram_list.txt under projectName folder.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + projectDir = scratchDir + '/' + projectName + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + if inps.ifgarmListTxt: ifgramList_txt = inps.ifgarmListTxt + else: ifgramList_txt = scratchDir + '/' + projectName + '/ifgram_list.txt' + ifgList0 = ut.read_txt2array(ifgramList_txt) +# ifgList = ifgList0[:,0] + if len(ifgList0)==3: + ifgList=ifgList0[0] + ifgList=[ifgList] + + else: + ifgList=ifgList0[:,0] + + err_txt = scratchDir + '/' + projectName + '/diff_gamma_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + for i in range(len(ifgList)): + m0 = ut.yyyymmdd(ifgList[i].split('-')[0]) + s0 = ut.yyyymmdd(ifgList[i].split('-')[1]) + cmd0 = ['diff_gamma.py',projectName, m0, s0] + diff_file0 = ifgDir + '/' + ifgList[i] + '/' + ifgList[i] + '_' + rlks + 'rlks.diff_filt.bmp' + data0 = [cmd0,err_txt] + + k00 = 0 + if os.path.isfile(diff_file0): + if os.path.getsize(diff_file0) > 0: + k00 = 1 + if k00==0: + data_para.append(data0) + + results = ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + failures = [str(item) for item in results if isinstance(item, Exception)] + if failures: + raise RuntimeError('\n\n'.join(failures)) + print("Generate differential interferograms for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/third_party/PyINT/pyint/down2slc_LT1.py b/third_party/PyINT/pyint/down2slc_LT1.py new file mode 100644 index 0000000..3c1f69f --- /dev/null +++ b/third_party/PyINT/pyint/down2slc_LT1.py @@ -0,0 +1,228 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.2 ### +### Copy Right (c): 2017-2019, Chen Wei ### +### Author: chenwei ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import re +import shutil +import tarfile + +from pyint import _orbit_bridge as orbit_bridge +from pyint import _utils as ut + + +def get_LT1_date(raw_file): + file0 = os.path.basename(raw_file) + date = file0[41:48] + return date + +def get_satellite(raw_file): + if 'LT1A_MONO_' in raw_file: + s0 = 'A' + else: + s0 = 'B' + return s0 + + +def discover_lt1_inputs(down_dir, date): + candidates = [] + for pattern in ( + down_dir + '/LT1*' + date + '*.tar.gz', + down_dir + '/LT1*' + date + '*.tiff', + ): + candidates.extend(glob.glob(pattern)) + return sorted(set(candidates)) + + +def write_input_list(list_path, paths): + with open(list_path, 'w') as f: + for path in paths: + f.write(path + '\n') + + +def _run_checked(command, cwd=None): + result = subprocess.run( + command, + cwd=cwd, + text=True, + capture_output=True, + check=False, + ) + if result.returncode != 0: + detail = (result.stderr or result.stdout or "").strip() + raise RuntimeError( + 'Command failed (%s): %s%s' % ( + result.returncode, + ' '.join(command), + ('\n' + detail) if detail else '', + ) + ) + return result + + +def _cleanup_paths(paths): + for path in paths: + if not path: + continue + try: + if os.path.isdir(path): + shutil.rmtree(path) + elif os.path.isfile(path): + os.remove(path) + except OSError: + continue + + +def _resolve_lt1_input_scene(raw_path, work_dir): + cleanup_paths = [] + raw_lower = raw_path.lower() + if raw_lower.endswith('.tiff'): + input_xml = re.sub(r'\.tiff$', '.meta.xml', raw_path, flags=re.IGNORECASE) + if not os.path.isfile(input_xml): + raise FileNotFoundError('LT-1 meta xml does not exist: ' + input_xml) + return raw_path, input_xml, cleanup_paths + + if raw_lower.endswith('.tar.gz'): + temp_dir = os.path.join(work_dir, 'tmp_data_dir') + if os.path.isdir(temp_dir): + shutil.rmtree(temp_dir) + os.makedirs(temp_dir, exist_ok=True) + cleanup_paths.append(temp_dir) + with tarfile.open(raw_path, 'r:gz') as archive: + member_names = archive.getnames() + tiff_members = [name for name in member_names if name.lower().endswith('.tiff')] + xml_members = [name for name in member_names if name.lower().endswith('meta.xml')] + if not tiff_members or not xml_members: + raise RuntimeError('LT-1 archive is missing .tiff or meta.xml: ' + raw_path) + tiff_member = tiff_members[0] + xml_member = xml_members[0] + archive.extract(tiff_member, path=temp_dir) + archive.extract(xml_member, path=temp_dir) + return os.path.join(temp_dir, tiff_member), os.path.join(temp_dir, xml_member), cleanup_paths + + raise RuntimeError('Unsupported LT-1 input, only .tiff or .tar.gz are supported: ' + raw_path) + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLC from LT1 raw data with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='project name. e.g., ChangningT55S1A') + parser.add_argument('date',help='date to be processed. e.g., 20180101') + inps = parser.parse_args() + + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Generate SLC from Sentinel-1 raw data using S1_import_SLC_from_zipfiles with orbit correction. + [Precise orbit data will be downloaded automatically] +''' + +EXAMPLE = """Usage: + + down2slc_sen.py projectName date + + down2slc_sen.py ChangningT55S1A 20180517 + +------------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + date = inps.date + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + slc_dir = projectDir + '/SLC' + down_dir = projectDir + '/DOWNLOAD' + + if not os.path.isdir(slc_dir): + os.mkdir(slc_dir) + + work_dir = slc_dir + '/' + date + if not os.path.isdir(work_dir): + os.mkdir(work_dir) + + os.chdir(work_dir) + + t_date = 't_' + date + + input_files = discover_lt1_inputs(down_dir, date) + write_input_list(t_date, input_files) + raw_files = ut.read_txt2list(t_date) + if len(raw_files) == 0: + raise RuntimeError('No LT-1 inputs found for date: ' + date) + satellite = get_satellite(str(raw_files[0])) + + zipfile_ref=str(raw_files[0]) + outfile_name=zipfile_ref.split('/')[-1].split('.')[0] + outfile_name=zipfile_ref.split('/')[-1] + print(outfile_name) + + cleanup_paths = [] + try: + input_tiff, input_xml, cleanup_paths = _resolve_lt1_input_scene(zipfile_ref, work_dir) + slc_path = work_dir + '/' + date + '.slc' + slc_par_path = work_dir + '/' + date + '.slc.par' + _run_checked( + ['par_LT1_SLC', input_tiff, input_xml, slc_par_path, slc_path], + cwd=work_dir, + ) + if not os.path.isfile(slc_path) or not os.path.isfile(slc_par_path): + raise RuntimeError('LT-1 import produced no SLC outputs for date: ' + date) + bridge_result = orbit_bridge.apply_precise_orbit( + date, + [slc_par_path], + work_dir=work_dir, + operation_tag='lt1_import', + ) + if bridge_result.get('stdout'): + print(bridge_result['stdout']) + if bridge_result.get('stderr'): + print(bridge_result['stderr']) + finally: + _cleanup_paths(cleanup_paths) + + + SLC_Tab = work_dir + '/' + date+'_SLC_Tab' + SLC_list = sorted(glob.glob(work_dir + '/*.slc')) + SLC_par_list = sorted(glob.glob(work_dir + '/*.slc.par')) + if len(SLC_list) == 0 or len(SLC_par_list) == 0: + raise RuntimeError('No LT-1 SLC outputs were generated for date: ' + date) + if len(SLC_list) != len(SLC_par_list): + raise RuntimeError('LT-1 SLC and SLC parameter counts do not match for date: ' + date) + + if os.path.isfile(SLC_Tab): + os.remove(SLC_Tab) + + for kk in range(len(SLC_list)): + call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' >> ' + SLC_Tab + if os.system(call_str) != 0: + raise RuntimeError('Failed to write SLC tab for date: ' + date) + with open(work_dir + '/down2slc.dat', 'w') as f: + f.write('ok\n') + print("Down to SLC for %s is done! " % date) + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/down2slc_LT1_all.py b/third_party/PyINT/pyint/down2slc_LT1_all.py new file mode 100644 index 0000000..09e167f --- /dev/null +++ b/third_party/PyINT/pyint/down2slc_LT1_all.py @@ -0,0 +1,185 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.2 ### +### Copy Right (c): 2017-2019, Chen Wei ### +### Author: chenwei ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse +import re + +import subprocess +from pyint import _utils as ut + +def get_LT1_date(raw_file): + file0 = os.path.basename(raw_file) + match = re.search(r'(20\d{6})', file0) + if match: + return match.group(1) + return '' + + +def discover_lt1_inputs(down_dir): + candidates = [] + for pattern in ('/LT1*.tar.gz', '/LT1*.tiff'): + candidates.extend(glob.glob(down_dir + pattern)) + return sorted(set(candidates)) + +def work(data0): + cmd = data0[0] + err_txt = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if type(stdout) == bytes: + bb=stdout.decode("utf-8", errors="replace") + else: + bb = stdout + + if p.returncode != 0: + detail_parts = [] + if bb: + detail_parts.append(bb) + if aa: + detail_parts.append(aa) + detail = '\n'.join(part.strip() for part in detail_parts if part and part.strip()) + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + with open(err_txt, 'a') as f: + f.write(str0) + if detail: + f.write(detail) + f.write('\n') + raise RuntimeError(str0.strip() + ' failed with rc=' + str(p.returncode) + ('\n' + detail if detail else '')) + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + with open(err_txt, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + down2slc_LT1_all.py projectName + down2slc_LT1_all.py projectName --parallel 4 + +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + slcDir = scratchDir + '/' + projectName + "/SLC" + raw_file_list = discover_lt1_inputs(downDir) + if len(raw_file_list) == 0: + raise RuntimeError('No LT-1 inputs found under: ' + downDir) + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + + date_list = [] + cat_list = [] + for kk in range(len(raw_file_list)): + date0 = get_LT1_date(os.path.basename(raw_file_list[kk])) + if not date0: + continue + if date0 not in date_list: + date_list.append(date0) + cat_list.append('0') + else: + cat_list[date_list.index(date0)]='1' +# date_list = set(date_list) +# date_list = sorted(date_list) + + print('Date to be processed:') + for k0 in date_list: + print(k0) + + err_txt = scratchDir + '/' + projectName + '/down2slc_sen_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + for i in range(len(date_list)): + if cat_list[i]=='0': + cmd0 = ['down2slc_LT1.py',projectName,date_list[i]] + work_dir = slcDir + '/' + date_list[i] + slc_file0 = work_dir + '/down2slc.dat' + data0 = [cmd0,err_txt] +# data_para.append(data0) + k00 = 0 + if os.path.isfile(slc_file0): + if os.path.getsize(slc_file0) > 0: + k00 = 1 + else: + k00 = 0 + if k00==0: + data_para.append(data0) + else: + cmd0 = ['down2slc_cat_LT1.py',projectName,date_list[i]] + work_dir = slcDir + '/' + date_list[i] + slc_file0 = work_dir + '/down2slc.dat' + data0 = [cmd0,err_txt] +# data_para.append(data0) + #data_para.append(data0) + k00 = 0 + if os.path.isfile(slc_file0): + if os.path.getsize(slc_file0) > 0: + k00 = 1 + else: + k00 = 0 + if k00==0: + data_para.append(data0) + results = ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + failures = [str(item) for item in results if isinstance(item, Exception)] + if failures: + raise RuntimeError('\n\n'.join(failures)) + os.chdir(downDir) + print("Down to SLC for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/third_party/PyINT/pyint/down2slc_alos_all.py b/third_party/PyINT/pyint/down2slc_alos_all.py new file mode 100644 index 0000000..3f08918 --- /dev/null +++ b/third_party/PyINT/pyint/down2slc_alos_all.py @@ -0,0 +1,138 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + + +def get_s1_date(raw_file): + file0 = os.path.basename(raw_file) + date = file0[17:25] + return date + + +def work(data0): + cmd = data0[0] + err_txt = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + #print(aa) + with open(err_txt, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + down2slc_sen_all.py projectName + down2slc_sen_all.py projectName --parallel 4 + +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + + raw_file_list = glob.glob(downDir + '/ALPSRP*.zip') + slc_dir = scratchDir + '/' + projectName + '/SLC' + if not os.path.isdir(slc_dir): + os.mkdir(slc_dir) + + err_txt = scratchDir + '/' + projectName + '/unzip_alos_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) +########get the ALOS date name ############### + data_para = [] + date_list = [] + for i in range(len(raw_file_list)): + cmd0 = ['unzip',raw_file_list[i],'-d',downDir] + call_str='unzip ' + raw_file_list[i] + ' -d ' + downDir + os.system(call_str) + workflow_file=raw_file_list[i].split(".")[0] + '.' +raw_file_list[i].split(".")[1] + os.chdir(workflow_file) + tempfile=workflow_file + '/' + 'workreport' + tempDict=ut.update_template(tempfile) + SeneceData = tempDict['Img_SceneCenterDateTime'].split('"')[1].split(' ')[0] + SceneID = tempDict['Scs_SceneID'] + os.chdir(downDir) + call_str= 'mv ' + workflow_file + ' ' + SeneceData + os.system(call_str) + data0=[SeneceData, err_txt] + data_para.append(data0) + date0=SeneceData + date_list.append(date0) + date_list = set(date_list) + date_list = sorted(date_list) + + print('Date to be processed:') + for k0 in date_list: + print(k0) + + err_txt = scratchDir + '/' + projectName + '/down2slc_alos_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + for i in range(len(date_list)): + cmd0 = ['Down2SLC_ALOS.py',projectName,date_list[i]] + data0 = [cmd0,err_txt] + data_para.append(data0) + + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + os.chdir(downDir) + print("Down to SLC for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/third_party/PyINT/pyint/down2slc_cat_LT1.py b/third_party/PyINT/pyint/down2slc_cat_LT1.py new file mode 100644 index 0000000..04e02fd --- /dev/null +++ b/third_party/PyINT/pyint/down2slc_cat_LT1.py @@ -0,0 +1,202 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.2 ### +### Copy Right (c): 2017-2019, Chen Wei ### +### Author: chenwei ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import re + +from pyint import _orbit_bridge as orbit_bridge +from pyint import _utils as ut + + +def get_LT1_date(raw_file): + file0 = os.path.basename(raw_file) + date = file0[41:48] + return date + +def get_satellite(raw_file): + if 'LT1A_MONO_' in raw_file: + s0 = 'A' + else: + s0 = 'B' + return s0 + + +def discover_lt1_inputs(down_dir, date): + candidates = [] + for pattern in ( + down_dir + '/LT1*' + date + '*.tar.gz', + down_dir + '/LT1*' + date + '*.tiff', + ): + candidates.extend(glob.glob(pattern)) + return sorted(set(candidates)) + + +def write_input_list(list_path, paths): + with open(list_path, 'w') as f: + for path in paths: + f.write(path + '\n') + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLC from LT1 raw data with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='project name. e.g., ChangningT55S1A') + parser.add_argument('date',help='date to be processed. e.g., 20180101') + inps = parser.parse_args() + + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Generate SLC from Sentinel-1 raw data using S1_import_SLC_from_zipfiles with orbit correction. + [Precise orbit data will be downloaded automatically] +''' + +EXAMPLE = """Usage: + + down2slc_LT1.py projectName date + + down2slc_LT1.py ChangningT55S1A 20180517 + +------------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + date = inps.date + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + slc_dir = projectDir + '/SLC' + down_dir = projectDir + '/DOWNLOAD' + + if not os.path.isdir(slc_dir): + os.mkdir(slc_dir) + + work_dir = slc_dir + '/' + date + if not os.path.isdir(work_dir): + os.mkdir(work_dir) + + os.chdir(work_dir) + + t_date = 't_' + date + + input_files = discover_lt1_inputs(down_dir, date) + write_input_list(t_date, input_files) + raw_files = ut.read_txt2list(t_date) + if len(raw_files) == 0: + raise RuntimeError('No LT-1 inputs found for date: ' + date) + satellite = get_satellite(str(raw_files[0])) + raw_file_list = list(raw_files) + + + + file_num=len(raw_files) + for kk in range(file_num): + + zipfile_ref=str(raw_files[kk]) + outfile_name=zipfile_ref.split('/')[-1] + print(outfile_name) + before_slc = set(glob.glob(work_dir + '/*.slc')) + before_slc_par = set(glob.glob(work_dir + '/*.slc.par')) + before_update = set(glob.glob(work_dir + '/*.slc.update')) + before_update_par = set(glob.glob(work_dir + '/*.slc.update.par')) + call_str = "echo " + zipfile_ref + " >date" + if os.system(call_str) != 0: + raise RuntimeError('Failed to materialize LT-1 input list for date: ' + date) + call_str = 'LT1_import_SLC_from_zipfiles1 date 0 ' + rc = os.system(call_str) + if rc != 0: + raise RuntimeError('LT1_import_SLC_from_zipfiles1 failed for date %s with rc=%s' % (date, rc)) + after_slc = set(glob.glob(work_dir + '/*.slc')) + after_slc_par = set(glob.glob(work_dir + '/*.slc.par')) + after_update = set(glob.glob(work_dir + '/*.slc.update')) + after_update_par = set(glob.glob(work_dir + '/*.slc.update.par')) + new_slc = sorted(after_slc - before_slc) + new_slc_par = sorted(after_slc_par - before_slc_par) + new_update = sorted(after_update - before_update) + new_update_par = sorted(after_update_par - before_update_par) + if not new_slc or not new_slc_par: + raise RuntimeError('LT-1 import produced no SLC outputs for date: ' + date) + if len(new_update) != len(new_update_par): + raise RuntimeError('LT-1 import produced mismatched update SLC outputs for date: ' + date) + bridge_targets = sorted((after_slc_par - before_slc_par) | (after_update_par - before_update_par)) + if bridge_targets: + bridge_result = orbit_bridge.apply_precise_orbit( + date, + bridge_targets, + work_dir=work_dir, + operation_tag='lt1_import', + ) + if bridge_result.get('stdout'): + print(bridge_result['stdout']) + if bridge_result.get('stderr'): + print(bridge_result['stderr']) + + + SLC_Tab = work_dir + '/' + date+'_SLC_Tab' + SLC_Tab_update = work_dir + '/' + date+'_update_SLC_Tab' + SLC_update_list = sorted(glob.glob(work_dir + '/*.slc.update')) + SLC_update_par_list = sorted(glob.glob(work_dir + '/*.slc.update.par')) + SLC_list = sorted(glob.glob(work_dir + '/*.slc')) + SLC_par_list = sorted(glob.glob(work_dir + '/*.slc.par')) + if len(SLC_list) == 0 or len(SLC_par_list) == 0: + raise RuntimeError('No LT-1 SLC outputs were generated for date: ' + date) + if len(SLC_list) != len(SLC_par_list): + raise RuntimeError('LT-1 SLC and SLC parameter counts do not match for date: ' + date) + if len(SLC_update_list) != len(SLC_update_par_list): + raise RuntimeError('LT-1 update SLC and parameter counts do not match for date: ' + date) + + if os.path.isfile(SLC_Tab): + os.remove(SLC_Tab) + + for kk in range(len(SLC_list)): + call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' >> ' + SLC_Tab + if os.system(call_str) != 0: + raise RuntimeError('Failed to write SLC tab for date: ' + date) + call_str = 'echo ' + SLC_update_list[kk] + ' ' + SLC_update_par_list[kk] + ' >> ' + SLC_Tab_update + if os.system(call_str) != 0: + raise RuntimeError('Failed to write update SLC tab for date: ' + date) + call_str = 'SLC_cat_list.py ' + SLC_Tab_update + ' ' + date + '.slc ' + date + '.slc.par ' + if os.system(call_str) != 0: + raise RuntimeError('SLC_cat_list.py failed for date: ' + date) + if not os.path.isfile(work_dir + '/' + date + '.slc.par'): + raise RuntimeError('Final LT-1 concatenated SLC parameter file is missing for date: ' + date) + final_bridge_result = orbit_bridge.apply_precise_orbit( + date, + [work_dir + '/' + date + '.slc.par'], + work_dir=work_dir, + operation_tag='slc_cat_final', + ) + if final_bridge_result.get('stdout'): + print(final_bridge_result['stdout']) + if final_bridge_result.get('stderr'): + print(final_bridge_result['stderr']) + with open(work_dir + '/down2slc.dat', 'w') as f: + f.write('ok\n') + print("Down to SLC for %s is done! " % date) + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/down2slc_cat_all.py b/third_party/PyINT/pyint/down2slc_cat_all.py new file mode 100644 index 0000000..2c9fc68 --- /dev/null +++ b/third_party/PyINT/pyint/down2slc_cat_all.py @@ -0,0 +1,148 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Central South University & University of Miami ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def is_number(s): + try: + int(s) + return True + except ValueError: + return False + + +def ras2jpg(input, strTitle): + call_str = "convert " + input + ".ras " + input + ".jpg" + os.system(call_str) + call_str = "convert " + input + ".jpg -resize 250 " + input + ".thumb.jpg" + os.system(call_str) + call_str = "convert " + input + ".jpg -resize 500 " + input + ".bthumb.jpg" + os.system(call_str) + call_str = "$INT_SCR/addtitle2jpg.pl " + input + ".thumb.jpg 14 " + strTitle + os.system(call_str) + call_str = "$INT_SCR/addtitle2jpg.pl " + input + ".bthumb.jpg 24 " + strTitle + os.system(call_str) + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +def write_template(File, Str): + f = open(File,'a') + f.write(Str) + f.close() + +def write_run_coreg_all(projectName,master,slavelist,workdir): + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + run_coreg_all = projectDir + "/run_coreg_all" + f_coreg = open(run_coreg_all,'w') + + for kk in range(len(slavelist)): + str_coreg = "GenOff_Gamma.py " + projectName + ' ' + master + ' ' + slavelist[kk] + ' ' + workdir + '\n' + f_coreg.write(str_coreg) + f_coreg.close() + + +def usage(): + print(''' +****************************************************************************************************** + + SLC cat for time-series of SAR datasets. + + usage: + + down2slc_cat_all.py projectName + + e.g. down2slc_cat_all.py PacayaERST163A + down2slc_cat_all.py PacayaEnvT163A + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==2: + if argv[0] in ['-h','--help']: usage(); sys.exit(1) + else: projectName=sys.argv[1] + else: + usage();sys.exit(1) + + if 'ERS' in projectName: + call_str='Down2SLC_ERS_Cat_All.py ' + projectName + else: + call_str='Down2SLC_ASAR_Cat_All.py ' + projectName + + os.system(call_str) + + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + + + + + + + + + diff --git a/third_party/PyINT/pyint/down2slc_cat_sen.py b/third_party/PyINT/pyint/down2slc_cat_sen.py new file mode 100644 index 0000000..20e331c --- /dev/null +++ b/third_party/PyINT/pyint/down2slc_cat_sen.py @@ -0,0 +1,216 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import csv + +from pyint import _utils as ut + + +def get_s1_date(raw_file): + file0 = os.path.basename(raw_file) + date = file0[17:25] + return date + +def get_s1_time(raw_file): + file0 = os.path.basename(raw_file) + times= file0[26:32] + return times + +def get_satellite(raw_file): + if 'S1A_IW_SLC_' in raw_file: + s0 = 'A' + else: + s0 = 'B' + + return s0 + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLC from Sentinel-1 raw data with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='project name. e.g., ChangningT55S1A') + parser.add_argument('date',help='date to be processed. e.g., 20180101') + + inps = parser.parse_args() + + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Cat two SLC from Sentinel-1 raw data using S1_import_SLC_from_zipfiles and SLC_cat_S1_TOPS with orbit correction. + [Precise orbit data will be downloaded automatically] +''' + +EXAMPLE = """Usage: + + down2slc_sen.py projectName date + + down2slc_sen.py ChangningT55S1A 20180517 + +------------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + date = inps.date + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + slc_dir = projectDir + '/SLC' + down_dir = projectDir + '/DOWNLOAD' + #opod_dir = projectDir + '/OPOD' + opod_dir = os.getenv('OPOD_DIR') + + + #if not os.path.isdir(opod_dir): + # os.mkdir(opod_dir) + + call_str = " eof --save-dir " + opod_dir + " -p " + down_dir + os.system(call_str) + if not os.path.isdir(slc_dir): + os.mkdir(slc_dir) + + work_dir = slc_dir + '/' + date + if not os.path.isdir(work_dir): + os.mkdir(work_dir) + + os.chdir(work_dir) + + t_date = 't_' + date + + call_str = 'ls ' + down_dir + '/S1*' + date + '* > ' + t_date + os.system(call_str) + + start_swath = templateDict['start_swath'] + end_swath = templateDict['end_swath'] + +# if (start_swath == '1') and (end_swath == '1'): +# k_swath = '1' +# elif (start_swath == '2') and (end_swath == '2'): +# k_swath = '2' +# elif (start_swath == '3') and (end_swath == '3'): +# k_swath = '3' +# elif (start_swath == '1') and (end_swath == '2'): +# k_swath = '4' +# elif (start_swath == '2') and (end_swath == '3'): +# k_swath = '5' +# elif (start_swath == '2') and (end_swath == '3'): +# k_swath = '-' + k_swath = ut.get_sardata_swath(start_swath,end_swath) + + raw_files = ut.read_txt2list(t_date) + satellite = get_satellite(str(raw_files[0])) + #orbit_file = ut.download_s1_orbit(date,opod_dir,satellite=satellite) + raw_file_list = glob.glob(down_dir + '/S1*' + date + '*.zip') + file_num=len(raw_files) + call_str = 'S1_BURST_tab_from_zipfile.py 3 --zip_ref_list ' + t_date + ' --zip_list ' + t_date + os.system(call_str) + + for kk in range(file_num): + zipfile_ref=str(raw_files[kk]) + outfile_name=zipfile_ref.split('/')[-1].split('.')[0] + burst_number_table_ref=outfile_name + '.BURST_tab' + #call_str = 'S1_BURST_tab_from_zipfile ' + t_date + ' ' + str(raw_files[kk]) + ' - 1' + #os.system(call_str) + #call_str = 'S1_import_SLC_from_zipfiles ' + t_date + ' ' + burst_number_table_ref + ' vv 0 ' + k_swath + ' ' + opod_dir + ' 1 1 ' + call_str = 'read_S1_TOPS_SLC.py ' + zipfile_ref + ' --burst_sel ' + burst_number_table_ref + ' --pol vv --root_name ' + date + ' --sw_start ' + start_swath + ' --swn ' + end_swath + ' --OPOD_dir ' + opod_dir + os.system(call_str) + os.chdir(work_dir) + call_str = "rename 's/vv.iw1.slc/iw1_" + str(kk)+".slc/g' *" + os.system(call_str) + call_str = "rename 's/vv.iw2.slc/iw2_" + str(kk)+".slc/g' *" + os.system(call_str) + call_str = "rename 's/vv.iw3.slc/iw3_" + str(kk)+".slc/g' *" + os.system(call_str) + SLC_Tab = work_dir + '/' + date + '_SLC_Tab' +str(kk) + SLC_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc')) + SLC_par_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc.par')) + TOP_par_list = sorted(glob.glob(work_dir + '/*iw*_' + str(kk) +'.slc.tops_par')) + + + cat_str = 'touch slc_list slc_par_list top_par_list' + os.system(call_str) + list_num=len(SLC_list) + for tt in range(list_num): + call_str = 'echo ' + SLC_list[tt] + ' >> slc_list' + os.system(call_str) + call_str = 'echo ' + SLC_par_list[tt] + ' >> slc_par_list' + os.system(call_str) + call_str = 'echo ' + TOP_par_list[tt] + ' >> top_par_list' + os.system(call_str) + call_str = ' paste slc_list slc_par_list top_par_list > ' + SLC_Tab + os.system(call_str) + call_str = 'rm slc_list slc_par_list top_par_list' + os.system(call_str) + SLC_list = sorted(glob.glob(work_dir + '/*iw*.slc')) + SLC_Tab = work_dir + '/' + date+'_SLC_Tab' + subswath=len(SLC_list)/2 + for kk in range(int(subswath)): + swath=kk+1 + call_str = 'echo ' + date + '.IW' + str(swath) + '.slc ' + date + '.IW' + str(swath) + '.slc.par ' + date + '.IW' + str(swath) + '.slc.TOPS_par ' + '>> ' + SLC_Tab + os.system(call_str) + call_str = 'SLC_cat_S1_TOPS ' +SLC_Tab+'0 ' + SLC_Tab +'1 ' + SLC_Tab + os.system(call_str) + + #call_str = "rename vv.slc.iw1 IW1.slc * " + #os.system(call_str) + #call_str = "rename 's/vv.slc.iw2/IW2.slc/g' *" + #call_str = "rename vv.slc.iw2 IW2.slc * " + #os.system(call_str) + #call_str = "rename 's/vv.slc.iw3/IW3.slc/g' *" + #call_str = "rename vv.slc.iw3 IW3.slc * " + #os.system(call_str) + #call_str = 'rm *.iw* *SLC_Tab*' + #os.system(call_str) + + SLC_Tab = work_dir + '/' + date+'_SLC_Tab' + SLC_list = sorted(glob.glob(work_dir + '/*IW*.slc')) + SLC_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.par')) + TOP_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.TOPS_par')) + + #call_str = 'rm *iw* *vv.SLC_tab' + #os.system(call_str) + if os.path.isfile(SLC_Tab): + os.remove(SLC_Tab) + + for kk in range(len(SLC_list)): + call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' >> ' + SLC_Tab + os.system(call_str) + + BURST = SLC_par_list[kk].replace('slc.par','burst.par') + call_str = 'SLC_burst_corners ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' > ' +BURST + os.system(call_str) + call_str = "echo 'SLC has already down' >down2slc.dat" + os.system(call_str) + print("Down to SLC for %s is done! " % date) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/third_party/PyINT/pyint/down2slc_csk.py b/third_party/PyINT/pyint/down2slc_csk.py new file mode 100644 index 0000000..a9219cf --- /dev/null +++ b/third_party/PyINT/pyint/down2slc_csk.py @@ -0,0 +1,199 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.0 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Now at KAUST ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import linecache +import datetime + +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + + +def check_ERS_par(SAR_IM_0P): + Name = os.path.basename(SAR_IM_0P) + ff = Name.split('.')[1] + date = (Name.split('SAR_IM__0PWDSI')[1]).split('_')[0] + + if ff == 'E1': + par = 'ERS1_ESA.par' + antenna = 'ERS1_antenna.gain' + orbdir = os.getenv('ERS1ORBDIR') + elif ff == 'E2': + par = 'ERS2_ESA.par' + antenna = 'ERS2_antenna.gain' + orbdir = os.getenv('ERS2ORBDIR') + else: + print('Invalid input SAR_IM_0P file.') + sys.exit(1) + + return par, antenna, orbdir, date + +######################################################################### + +INTRODUCTION = ''' +############################################################################# + Copy Right(c): 2017-2020, Yunmeng Cao @PyINT v2.0 + + Generate SLC for CSK raw data (with **.tar.gz format). + +''' + +EXAMPLE = ''' + Usage: + down2slc_csk.py projectName date + + Examples: + down2slc_csk.py KilaueaT10CskA 20180419 + +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLC for ERS raw data with ENVISAT format.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='project name.') + parser.add_argument('date',help='to be processed date.') + + inps = parser.parse_args() + return inps + +################################################################################ + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + date = inps.date + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + + if not os.path.isdir(slcDir): + call_str = 'mkdir ' + slcDir + os.system(call_str) + + slcDir1 = slcDir + '/' + date + if not os.path.isdir(slcDir1): + call_str = 'mkdir ' + slcDir1 + os.system(call_str) + + tar0 = glob.glob(downDir + '/*' + date + '*.tar.gz')[0] + rawDir = downDir + '/raw_' + date + + if not os.path.isdir(rawDir): + call_str = 'mkdir ' + downDir + '/raw_' + date + os.system(call_str) + + call_str = 'tar -xzf ' + tar0 + ' -C ' + rawDir + os.system(call_str) + + par = rawDir + '/' + date + '.sar_par' + raw = rawDir + '/' + date + '.raw' + azsp = rawDir + '/' + date + '.azsp' + dop = rawDir + '/' + date + '.dop' + rspec = rawDir + '/' + date + '.rspec' + rc = rawDir + '/' + date + '.rc' + autof = rawDir + '/' + date + '.autof' + dop_ambig = rawDir + '/' + date + '.dop_ambig' + + h5file = glob.glob(rawDir + '/*.h5')[0] + + pslc_par = rawDir + '/' + 'p' + date + '.slc.par' + slc_par = slcDir1 + '/' + date + '.slc.par' + slc = slcDir1 + '/' +date + '.slc' + mli_par = slcDir1 + '/' +date + '.mli.par' + mli = slcDir1 + '/' + date + '.mli' + + call_str = 'CS_proc ' + h5file + ' ' + par + ' ' + pslc_par + ' ' + raw + ' - - ' + os.system(call_str) + + cal_str = 'dop_ambig ' + par + ' ' + pslc_par + ' ' + raw + ' 2 - ' + dop_ambig + os.system(call_str) + + call_str = 'azsp_IQ ' + par + ' ' + pslc_par + ' ' + raw + ' ' + azsp + os.system(call_str) + + call_str = 'doppler ' + par + ' ' + pslc_par + ' ' + raw + ' ' + dop + os.system(call_str) + + call_str = 'rspec_IQ ' + par + ' ' + pslc_par + ' ' + raw + ' ' + rspec + os.system(call_str) + + call_str = 'pre_rc ' + par + ' ' + pslc_par + ' ' + raw + ' ' + rc + os.system(call_str) + + call_str = 'autof ' + par + ' ' + pslc_par + ' ' + rc + ' ' + autof + ' 2.0 ' + os.system(call_str) + + call_str = 'autof ' + par + ' ' + pslc_par + ' ' + rc + ' ' + autof + ' 2.0 ' + os.system(call_str) + + call_str = 'az_proc ' + par + ' ' + pslc_par + ' ' + rc + ' ' + slc + ' - 1 ' + ' - 0 2.120 ' + os.system(call_str) + + call_str = 'par_MSP ' + par + ' ' + pslc_par + ' ' + slc_par + ' 1' + os.system(call_str) + + call_str = 'multi_look ' + slc + ' ' + slc_par + ' ' + mli + ' ' + mli_par + ' 20 16' + os.system(call_str) + + + Width = UseGamma(mli_par, 'read', 'range_samples: ') + call_str = 'raspwr ' + mli + ' ' + Width + os.system(call_str) + + + print("Generate SLC for %s is done." % date) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + diff --git a/third_party/PyINT/pyint/down2slc_csk_all.py b/third_party/PyINT/pyint/down2slc_csk_all.py new file mode 100644 index 0000000..7bd2d8e --- /dev/null +++ b/third_party/PyINT/pyint/down2slc_csk_all.py @@ -0,0 +1,119 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + + +def get_csk_date(raw_file): + file0 = os.path.basename(raw_file) + date = file0[27:35] + return date + + +def work(data0): + cmd = data0[0] + err_txt = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + #print(aa) + with open(err_txt, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + down2slc_sen_all.py projectName + down2slc_sen_all.py projectName --parallel 4 + +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + raw_file_list = glob.glob(downDir + '/CSK*.tar.gz') + + # get the burst number table of the mater date + + date_list = [] + for kk in range(len(raw_file_list)): + date0 = get_csk_date(os.path.basename(raw_file_list[kk])) + date_list.append(date0) + + date_list = set(date_list) + date_list = sorted(date_list) + + print('Date to be processed:') + for k0 in date_list: + print(k0) + + err_txt = scratchDir + '/' + projectName + '/down2slc_csk_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + for i in range(len(date_list)): + cmd0 = ['down2slc_csk.py',projectName,date_list[i]] + data0 = [cmd0,err_txt] + data_para.append(data0) + + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + os.chdir(downDir) + print("Down to SLC for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/third_party/PyINT/pyint/down2slc_sen.py b/third_party/PyINT/pyint/down2slc_sen.py new file mode 100644 index 0000000..953fe87 --- /dev/null +++ b/third_party/PyINT/pyint/down2slc_sen.py @@ -0,0 +1,167 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse + +from pyint import _utils as ut + + +def get_s1_date(raw_file): + file0 = os.path.basename(raw_file) + date = file0[17:25] + return date + +def get_satellite(raw_file): + if 'S1A_IW_SLC_' in raw_file: + s0 = 'A' + else: + s0 = 'B' + + return s0 + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLC from Sentinel-1 raw data with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='project name. e.g., ChangningT55S1A') + parser.add_argument('date',help='date to be processed. e.g., 20180101') + + inps = parser.parse_args() + + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Generate SLC from Sentinel-1 raw data using S1_import_SLC_from_zipfiles with orbit correction. + [Precise orbit data will be downloaded automatically] +''' + +EXAMPLE = """Usage: + + down2slc_sen.py projectName date + + down2slc_sen.py ChangningT55S1A 20180517 + +------------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + date = inps.date + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + slc_dir = projectDir + '/SLC' + down_dir = projectDir + '/DOWNLOAD' + #opod_dir = projectDir + '/OPOD' + opod_dir = os.getenv('OPOD_DIR') + if not os.path.isdir(slc_dir): + os.mkdir(slc_dir) + # if not os.path.isdir(opod_dir): + # os.mkdir(opod_dir) + + work_dir = slc_dir + '/' + date + if not os.path.isdir(work_dir): + os.mkdir(work_dir) + #if not os.path.isdir(opod_dir): + # os.mkdir(opod_dir) + + + call_str = " eof --save-dir " + opod_dir + " -p " + down_dir + os.system(call_str) + + os.chdir(work_dir) + + t_date = 't_' + date + + call_str = 'ls ' + down_dir + '/S1*' + date + '*.zip > ' + t_date + os.system(call_str) + + start_swath = templateDict['start_swath'] + end_swath = templateDict['end_swath'] + +# if (start_swath == '1') and (end_swath == '1'): +# k_swath = '1' +# elif (start_swath == '2') and (end_swath == '2'): +# k_swath = '2' +# elif (start_swath == '3') and (end_swath == '3'): +# k_swath = '3' +# elif (start_swath == '1') and (end_swath == '2'): +# k_swath = '4' +# elif (start_swath == '2') and (end_swath == '3'): +# k_swath = '5' +# elif (start_swath == '2') and (end_swath == '3'): +# k_swath = '-' + k_swath = ut.get_sardata_swath(start_swath,end_swath) + + raw_files = ut.read_txt2list(t_date) + satellite = get_satellite(str(raw_files[0])) + #orbit_file = ut.download_s1_orbit(date,opod_dir,satellite=satellite) + zipfile_ref=str(raw_files[0]) + outfile_name=zipfile_ref.split('/')[-1].split('.')[0] + burst_number_table_ref=outfile_name + '.BURST_tab' + + call_str = 'S1_BURST_tab_from_zipfile.py 3 --zip_ref_list ' + t_date + ' --zip_list ' + t_date + os.system(call_str) + + # call_str = 'S1_import_SLC_from_zipfiles ' + t_date + ' ' + burst_number_table_ref + ' vv 0 ' + k_swath + call_str = 'read_S1_TOPS_SLC.py ' + zipfile_ref + ' --burst_sel ' + burst_number_table_ref + ' --pol vv --root_name ' + date + ' --sw_start ' + start_swath + ' --swn ' + end_swath + ' --OPOD_dir ' + opod_dir + os.system(call_str) + + os.chdir(work_dir) + + call_str = "rename 's/vv.iw1.slc/IW1.slc/g' *" + #call_str = "rename vv.slc.iw1 IW1.slc * " + os.system(call_str) + call_str = "rename 's/vv.iw2.slc/IW2.slc/g' *" + #call_str = "rename vv.slc.iw2 IW2.slc * " + os.system(call_str) + call_str = "rename 's/vv.iw3.slc/IW3.slc/g' *" + #call_str = "rename vv.slc.iw3 IW3.slc * " + os.system(call_str) + call_str = "rename 's/tops_par/TOPS_par/g' *.tops_par " + os.system(call_str) + + SLC_Tab = work_dir + '/' + date+'_SLC_Tab' + SLC_list = sorted(glob.glob(work_dir + '/*IW*.slc')) + SLC_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.par')) + TOP_par_list = sorted(glob.glob(work_dir + '/*IW*.slc.TOPS_par')) + + if os.path.isfile(SLC_Tab): + os.remove(SLC_Tab) + + for kk in range(len(SLC_list)): + call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' >> ' + SLC_Tab + os.system(call_str) + + BURST = SLC_par_list[kk].replace('slc.par','burst.par') + call_str = 'SLC_burst_corners ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' > ' +BURST + os.system(call_str) + call_str = "echo 'SLC has already down' >down2slc.dat" + os.system(call_str) + print("Down to SLC for %s is done! " % date) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/down2slc_sen_all.py b/third_party/PyINT/pyint/down2slc_sen_all.py new file mode 100644 index 0000000..15c7800 --- /dev/null +++ b/third_party/PyINT/pyint/down2slc_sen_all.py @@ -0,0 +1,151 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + + +def get_s1_date(raw_file): + file0 = os.path.basename(raw_file) + date = file0[17:25] + return date + + +def work(data0): + cmd = data0[0] + err_txt = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + #print(aa) + with open(err_txt, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + down2slc_sen_all.py projectName + down2slc_sen_all.py projectName --parallel 4 + +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + slcDir = scratchDir + '/' + projectName + "/SLC" + raw_file_list = glob.glob(downDir + '/S1*.zip') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + + date_list = [] + cat_list = [] + for kk in range(len(raw_file_list)): + date0 = get_s1_date(os.path.basename(raw_file_list[kk])) + if date0 not in date_list: + date_list.append(date0) + cat_list.append('0') + else: + cat_list[date_list.index(date0)]='1' +# date_list = set(date_list) +# date_list = sorted(date_list) + + print('Date to be processed:') + for k0 in date_list: + print(k0) + + err_txt = scratchDir + '/' + projectName + '/down2slc_sen_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + for i in range(len(date_list)): + if cat_list[i]=='0': + cmd0 = ['down2slc_sen.py',projectName,date_list[i]] + work_dir = slcDir + '/' + date_list[i] + slc_file0 = work_dir + '/down2slc.dat' + data0 = [cmd0,err_txt] +# data_para.append(data0) + k00 = 0 + if os.path.isfile(slc_file0): + if os.path.getsize(slc_file0) > 0: + k00 = 1 + else: + k00 = 0 + if k00==0: + data_para.append(data0) + else: + cmd0 = ['down2slc_cat_sen.py',projectName,date_list[i]] + work_dir = slcDir + '/' + date_list[i] + slc_file0 = work_dir + '/down2slc.dat' + data0 = [cmd0,err_txt] +# data_para.append(data0) + #data_para.append(data0) + k00 = 0 + if os.path.isfile(slc_file0): + if os.path.getsize(slc_file0) > 0: + k00 = 1 + else: + k00 = 0 + if k00==0: + data_para.append(data0) + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + os.chdir(downDir) + print("Down to SLC for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/third_party/PyINT/pyint/down2slc_sen_all_old.py b/third_party/PyINT/pyint/down2slc_sen_all_old.py new file mode 100644 index 0000000..755cce1 --- /dev/null +++ b/third_party/PyINT/pyint/down2slc_sen_all_old.py @@ -0,0 +1,103 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + + +def work(data0): + cmd = data0[0] + err_txt = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + #print(aa) + with open(err_txt, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + down2slc_sen_all.py projectName + down2slc_sen_all.py projectName --parallel 4 + +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLCs from Sentinel-1 raw dataset with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + raw_file_list = glob.glob(downDir + '/S1*.zip') + + slc_dir = scratchDir + '/' + projectName + '/SLC' + if not os.path.isdir(slc_dir): + os.mkdir(slc_dir) + + err_txt = scratchDir + '/' + projectName + '/down2slc_sen_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + for i in range(len(raw_file_list)): + cmd0 = ['down2slc_sen.py',raw_file_list[i],slc_dir] + data0 = [cmd0,err_txt] + data_para.append(data0) + + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + os.chdir(downDir) + print("Down to SLC for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + \ No newline at end of file diff --git a/third_party/PyINT/pyint/down2slc_sen_old.py b/third_party/PyINT/pyint/down2slc_sen_old.py new file mode 100644 index 0000000..af9fb03 --- /dev/null +++ b/third_party/PyINT/pyint/down2slc_sen_old.py @@ -0,0 +1,180 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse + +from pyint import _utils as ut + + +def get_s1_date(raw_file): + file0 = os.path.basename(raw_file) + date = file0[17:25] + return date + +def get_satellite(raw_file): + if 'S1A_IW_SLC_' in raw_file: + s0 = 'A' + else: + s0 = 'B' + + return s0 + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLC from Sentinel-1 raw data with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('s1_raw', help='raw Sentinel-1 file. e.g., S1A*.zip') + parser.add_argument('root_path',help='root path for saving the SLC files') + + inps = parser.parse_args() + + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Generate SLC from Sentinel-1 raw data with orbit correction using GAMMA. + [Precise orbit data will be downloaded automatically] +''' + +EXAMPLE = """Usage: + + down2slc_sen.py S1A_IW_SLC_XXXX.zip /Yunmeng/S1_test + +------------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + raw_file = inps.s1_raw + root_dir = inps.root_path + satellite = get_satellite(raw_file) + date = get_s1_date(raw_file) + if not os.path.isdir(root_dir): + os.mkdir(root_dir) + + slc_dir = root_dir + '/' + date + if not os.path.isdir(slc_dir): + os.mkdir(slc_dir) + + if len(os.path.dirname(raw_file))==0: + raw_file_dir = os.getcwd() + else: + raw_file_dir = os.path.dirname(raw_file) + + raw_dir = raw_file.replace('.zip','.SAFE') + +# MEASURE = glob.glob(measureDir + '/*vv*tiff') +# ANNOTAT = glob.glob(annotatDir + '/*vv*xml' ) +# CALIBRA = glob.glob(calibraDir+'/calibration*vv*') +# NOISE = glob.glob(calibraDir+'/noise*vv*') + + SLC_Tab = slc_dir + '/' + date+'_SLC_Tab' + TEST = slc_dir + '/' + date + '.IW3.slc.par' + k0 = 0 + if os.path.isfile(TEST): + if os.path.getsize(TEST) > 0: + k0 = 1 + + if k0==0: + if not os.path.isdir(raw_dir): + call_str = 'unzip ' + raw_file + ' -d ' + raw_file_dir + os.system(call_str) + + measureDir = raw_dir + '/measurement' + annotatDir = raw_dir + '/annotation' + calibraDir = raw_dir + '/annotation/calibration' + MM = glob.glob(measureDir + '/*vv*tiff') + + if os.path.isfile(SLC_Tab): + os.remove(SLC_Tab) + for kk in range(len(MM)): + SLC = slc_dir + '/' + date + '.IW' + str(kk+1)+'.slc' + SLCPar = slc_dir + '/' + date + '.IW' + str(kk+1)+'.slc.par' + TOPPar = slc_dir + '/' + date + '.IW' + str(kk+1)+'.slc.TOPS_par' + BURST = slc_dir + '/' + date + '.IW' + str(kk+1)+'.burst.par' + + if os.path.isfile(BURST): + os.remove(BURST) + call_str = 'echo ' + SLC + ' ' + SLCPar + ' ' + TOPPar + ' >> ' + SLC_Tab + os.system(call_str) + + MEASURE = glob.glob(measureDir + '/*iw' + str(kk+1) + '*vv*tiff') + ANNOTAT = glob.glob(annotatDir + '/*iw' + str(kk+1) + '*vv*xml' ) + CALIBRA = glob.glob(calibraDir+'/calibration*'+ 'iw' + str(kk+1) + '*vv*') + NOISE = glob.glob(calibraDir+'/noise*' + 'iw' + str(kk+1) + '*vv*') + + #call_str = 'S1_burstloc ' + ANNOTAT[0] + '> ' +BURST + #os.system(call_str) + + if not os.path.isfile(NOISE[0]): + call_str = 'par_S1_SLC ' + MEASURE[0] + ' ' + ANNOTAT[0] + ' ' + CALIBRA[0] + ' - ' + SLCPar + ' ' + SLC + ' ' + TOPPar + else: + call_str = 'par_S1_SLC ' + MEASURE[0] + ' ' + ANNOTAT[0] + ' ' + CALIBRA[0] + ' ' + NOISE[0] + ' ' + SLCPar + ' ' + SLC + ' ' + TOPPar + + #if int(date) > 180311: + # call_str = 'par_S1_SLC ' + MEASURE[0] + ' ' + ANNOTAT[0] + ' ' + CALIBRA[0] + ' - ' + SLCPar + ' ' + SLC + ' ' + TOPPar + #else: + # call_str = 'par_S1_SLC ' + MEASURE[0] + ' ' + ANNOTAT[0] + ' ' + CALIBRA[0] + ' ' + NOISE[0] + ' ' + SLCPar + ' ' + SLC + ' ' + TOPPar + + os.system(call_str) + + call_str = 'SLC_burst_corners ' + SLCPar + ' ' + TOPPar + ' > ' +BURST + os.system(call_str) + + # orbit correction + slc_pars = glob.glob(slc_dir + '/*.IW*.slc.par') + orbit_file0 = ut.download_s1_orbit(date,slc_dir,satellite=satellite) + orbit_file = slc_dir + '/' + orbit_file0 + + for i in range(len(slc_pars)): + call_str = 'S1_OPOD_vec ' + slc_pars[i] + ' ' + orbit_file + os.system(call_str) + + # generate amp file for check image quality + #TSLC = slc_dir + '/' + date + '.slc' + #TSLCPar = slc_dir + '/' + date + '.slc.par' + + #TMLI = slc_dir + '/' + date + '_40rlks.amp' + #TMLIPar = slc_dir + '/' + date + '_40rlks.amp.par' + + #call_str = 'SLC_mosaic_S1_TOPS ' + SLC_Tab + ' ' + TSLC + ' ' + TSLCPar + ' 10 2' + #os.system(call_str) + + #call_str = 'multi_look ' + TSLC + ' ' + TSLCPar + ' ' + TMLI + ' ' + TMLIPar + ' 40 8' + #os.system(call_str) + + #nWidth = ut.read_gamma_par(TMLIPar, 'read','range_samples:') + #call_str = 'raspwr ' + TMLI + ' ' + nWidth + ' - - - - - - - ' + #os.system(call_str) + + if os.path.isdir(raw_dir): + call_str = 'rm -rf ' + raw_dir + os.system(call_str) + + print("Down to SLC for %s is done! " % date) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/third_party/PyINT/pyint/download_ers_deos.py b/third_party/PyINT/pyint/download_ers_deos.py new file mode 100644 index 0000000..980d752 --- /dev/null +++ b/third_party/PyINT/pyint/download_ers_deos.py @@ -0,0 +1,171 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Now at KAUST ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import linecache + +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def StrNum1(S): + S=str(S) + if len(S)==1: + S='00' +S + elif len(S)==2: + S = '0' +S + else: + S = S + return S + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +def print_progress(iteration, total, prefix='calculating:', suffix='complete', decimals=1, barLength=50, elapsed_time=None): + """Print iterations progress - Greenstick from Stack Overflow + Call in a loop to create terminal progress bar + @params: + iteration - Required : current iteration (Int) + total - Required : total iterations (Int) + prefix - Optional : prefix string (Str) + suffix - Optional : suffix string (Str) + decimals - Optional : number of decimals in percent complete (Int) + barLength - Optional : character length of bar (Int) + elapsed_time- Optional : elapsed time in seconds (Int/Float) + + Reference: http://stackoverflow.com/questions/3173320/text-progress-bar-in-the-console + """ + filledLength = int(round(barLength * iteration / float(total))) + percents = round(100.00 * (iteration / float(total)), decimals) + bar = '#' * filledLength + '-' * (barLength - filledLength) + if elapsed_time: + sys.stdout.write('%s [%s] %s%s %s %s secs\r' % (prefix, bar, percents, '%', suffix, int(elapsed_time))) + else: + sys.stdout.write('%s [%s] %s%s %s\r' % (prefix, bar, percents, '%', suffix)) + sys.stdout.flush() + if iteration == total: + print("\n") + + ''' + Sample Useage: + for i in range(len(dateList)): + print_progress(i+1,len(dateList)) + ''' + return + + +######################################################################### + +INTRODUCTION = ''' +############################################################################# + Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 + + Download precise ERS orbit data from Delft Institute for Earth-Oriented Space Research (http://www.deos.tudelft.nl/). + +''' + +EXAMPLE = ''' + Usage: + download_ers_deos.py ERS1 + download_ers_deos.py ERS2 + +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Download (and correct) precise ERS orbit data.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('sar',help='project name') + + inps = parser.parse_args() + return inps + +################################################################################ + + +def main(argv): + + total = time.time() + inps = cmdLineParse() + sar = inps.sar + PATH = os.getcwd() + listfile = 'arclist' + + if sar == 'ERS1': + Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-1/dgm-e04/' + Url_list = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-1/dgm-e04/arclist' + num_min = 6 + num_max = 511 + N_num = 506 + if not os.path.isfile(listfile): + call_str = 'wget -q ' + Url_list + os.system(call_str) + + elif sar =='ERS2': + Url = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-2/dgm-e04/' + Url_list = 'ftp://dutlru2.lr.tudelft.nl/pub/orbits/ODR.ERS-2/dgm-e04/arclist' + num_min = 3 + num_max = 867 + N_num = 865 + if not os.path.isfile(listfile): + call_str = 'wget -q ' + Url_list + os.system(call_str) + + else: + print('SAR name is invalid!') + sys.exit(1) + + for i in range(N_num): + kk = num_min + i + kk=int(kk) + SS = Url + 'ODR.' + StrNum1(kk) + print_progress(i+1, N_num, prefix='DEFT_vec: ', suffix=StrNum1(kk)) + + call_str = 'wget -q --no-check-certificate ' + SS + ' -P ' + PATH + os.system(call_str) + + + print("Download precise DEFT orbital data for %s is done." % sar) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + diff --git a/third_party/PyINT/pyint/downloader_gmtchina.py b/third_party/PyINT/pyint/downloader_gmtchina.py new file mode 100644 index 0000000..f8015fa --- /dev/null +++ b/third_party/PyINT/pyint/downloader_gmtchina.py @@ -0,0 +1,388 @@ +# -*- coding: utf-8 -* +#''' +#This code is used to download image from google and amap + +#@date : 2020-3-13 +#@author: Zheng Jie +#@E-mail: zhengjie9510@qq.com + +#Modified by GMT-china.org +#''' + +import sys +import io +import math +import multiprocessing +import time +import urllib.request as ur +from math import floor, pi, log, tan, atan, exp +from threading import Thread + +import PIL.Image as pil +import cv2 +import numpy as np +from osgeo import gdal, osr + + +# ------------------Interchange between WGS-84 and Web Mercator------------------------- +# WGS-84 to Web Mercator +def wgs_to_mercator(x, y): + y = 85.0511287798 if y > 85.0511287798 else y + y = -85.0511287798 if y < -85.0511287798 else y + + x2 = x * 20037508.34 / 180 + y2 = log(tan((90 + y) * pi / 360)) / (pi / 180) + y2 = y2 * 20037508.34 / 180 + return x2, y2 + + +# Web Mercator to WGS-84 +def mercator_to_wgs(x, y): + x2 = x / 20037508.34 * 180 + y2 = y / 20037508.34 * 180 + y2 = 180 / pi * (2 * atan(exp(y2 * pi / 180)) - pi / 2) + return x2, y2 + + +# -------------------------------------------------------------------------------------- + +# -----------------Interchange between GCJ-02 to WGS-84--------------------------- +# All public geographic data in mainland China need to be encrypted with GCJ-02, introducing random bias +# This part of the code is used to remove the bias +def transformLat(x, y): + ret = -100.0 + 2.0 * x + 3.0 * y + 0.2 * y * y + 0.1 * x * y + 0.2 * math.sqrt(abs(x)) + ret += (20.0 * math.sin(6.0 * x * math.pi) + 20.0 * math.sin(2.0 * x * math.pi)) * 2.0 / 3.0 + ret += (20.0 * math.sin(y * math.pi) + 40.0 * math.sin(y / 3.0 * math.pi)) * 2.0 / 3.0 + ret += (160.0 * math.sin(y / 12.0 * math.pi) + 320 * math.sin(y * math.pi / 30.0)) * 2.0 / 3.0 + return ret + + +def transformLon(x, y): + ret = 300.0 + x + 2.0 * y + 0.1 * x * x + 0.1 * x * y + 0.1 * math.sqrt(abs(x)) + ret += (20.0 * math.sin(6.0 * x * math.pi) + 20.0 * math.sin(2.0 * x * math.pi)) * 2.0 / 3.0 + ret += (20.0 * math.sin(x * math.pi) + 40.0 * math.sin(x / 3.0 * math.pi)) * 2.0 / 3.0 + ret += (150.0 * math.sin(x / 12.0 * math.pi) + 300.0 * math.sin(x / 30.0 * math.pi)) * 2.0 / 3.0 + return ret + + +def delta(lat, lon): + ''' + Krasovsky 1940 + // + // a = 6378245.0, 1/f = 298.3 + // b = a * (1 - f) + // ee = (a^2 - b^2) / a^2; + ''' + a = 6378245.0 # a: Projection factor of satellite ellipsoidal coordinates projected onto a flat map coordinate system + ee = 0.00669342162296594323 # ee: Eccentricity of ellipsoid + dLat = transformLat(lon - 105.0, lat - 35.0) + dLon = transformLon(lon - 105.0, lat - 35.0) + radLat = lat / 180.0 * math.pi + magic = math.sin(radLat) + magic = 1 - ee * magic * magic + sqrtMagic = math.sqrt(magic) + dLat = (dLat * 180.0) / ((a * (1 - ee)) / (magic * sqrtMagic) * math.pi) + dLon = (dLon * 180.0) / (a / sqrtMagic * math.cos(radLat) * math.pi) + return {'lat': dLat, 'lon': dLon} + + +def outOfChina(lat, lon): + if (lon < 72.004 or lon > 137.8347): + return True + if (lat < 0.8293 or lat > 55.8271): + return True + return False + + +def gcj_to_wgs(gcjLon, gcjLat): + if outOfChina(gcjLat, gcjLon): + return (gcjLon, gcjLat) + d = delta(gcjLat, gcjLon) + return (gcjLon - d["lon"], gcjLat - d["lat"]) + + +def wgs_to_gcj(wgsLon, wgsLat): + if outOfChina(wgsLat, wgsLon): + return wgsLon, wgsLat + d = delta(wgsLat, wgsLon) + return wgsLon + d["lon"], wgsLat + d["lat"] + + +# -------------------------------------------------------------- + +# --------------------------------------------------------- +# Get tile coordinates in Google Maps based on latitude and longitude of WGS-84 +def wgs_to_tile(j, w, z): + ''' + Get google-style tile cooridinate from geographical coordinate + j : Longittude + w : Latitude + z : zoom + ''' + isnum = lambda x: isinstance(x, int) or isinstance(x, float) + if not (isnum(j) and isnum(w)): + raise TypeError("j and w must be int or float!") + + if not isinstance(z, int) or z < 0 or z > 22: + raise TypeError("z must be int and between 0 to 22.") + + if j < 0: + j = 180 + j + else: + j += 180 + j /= 360 # make j to (0,1) + + w = 85.0511287798 if w > 85.0511287798 else w + w = -85.0511287798 if w < -85.0511287798 else w + w = log(tan((90 + w) * pi / 360)) / (pi / 180) + w /= 180 # make w to (-1,1) + w = 1 - (w + 1) / 2 # make w to (0,1) and left top is 0-point + + num = 2 ** z + x = floor(j * num) + y = floor(w * num) + return x, y + + +def pixls_to_mercator(zb): + # Get the web Mercator projection coordinates of the four corners of the area according to the four corner coordinates of the tile + inx, iny = zb["LT"] # left top + inx2, iny2 = zb["RB"] # right bottom + length = 20037508.3427892 + sum = 2 ** zb["z"] + LTx = inx / sum * length * 2 - length + LTy = -(iny / sum * length * 2) + length + + RBx = (inx2 + 1) / sum * length * 2 - length + RBy = -((iny2 + 1) / sum * length * 2) + length + + # LT=left top,RB=right buttom + # Returns the projected coordinates of the four corners + res = {'LT': (LTx, LTy), 'RB': (RBx, RBy), + 'LB': (LTx, RBy), 'RT': (RBx, LTy)} + return res + + +def tile_to_pixls(zb): + # Tile coordinates are converted to pixel coordinates of the four corners + out = {} + width = (zb["RT"][0] - zb["LT"][0] + 1) * 256 + height = (zb["LB"][1] - zb["LT"][1] + 1) * 256 + out["LT"] = (0, 0) + out["RT"] = (width, 0) + out["LB"] = (0, -height) + out["RB"] = (width, -height) + return out + + +# ----------------------------------------------------------- + +# --------------------------------------------------------- +class Downloader(Thread): + # multiple threads downloader + def __init__(self, index, count, urls, datas): + # index represents the number of threads + # count represents the total number of threads + # urls represents the list of URLs nedd to be downloaded + # datas represents the list of data need to be returned. + super().__init__() + self.urls = urls + self.datas = datas + self.index = index + self.count = count + + def download(self, url): + HEADERS = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36 Edg/88.0.705.68'} + header = ur.Request(url, headers=HEADERS) + err = 0 + while (err < 3): + try: + data = ur.urlopen(header).read() + except: + err += 1 + else: + return data + raise Exception("Bad network link.") + + def run(self): + for i, url in enumerate(self.urls): + if i % self.count != self.index: + continue + self.datas[i] = self.download(url) + + +# --------------------------------------------------------- + +# --------------------------------------------------------- +def getExtent(x1, y1, x2, y2, z, source="amap"): + pos1x, pos1y = wgs_to_tile(x1, y1, z) + pos2x, pos2y = wgs_to_tile(x2, y2, z) + Xframe = pixls_to_mercator( + {"LT": (pos1x, pos1y), "RT": (pos2x, pos1y), "LB": (pos1x, pos2y), "RB": (pos2x, pos2y), "z": z}) + for i in ["LT", "LB", "RT", "RB"]: + Xframe[i] = mercator_to_wgs(*Xframe[i]) + if source == "google_sat": + pass + elif source == "amap" or source == "amap_sat" or source == "google": + for i in ["LT", "LB", "RT", "RB"]: + Xframe[i] = gcj_to_wgs(*Xframe[i]) + else: + raise Exception("Invalid argument: source.") + return Xframe + + +def saveTiff(r, g, b, gt, filePath): + fname_out = filePath + driver = gdal.GetDriverByName('GTiff') + # Create a 3-band dataset + dset_output = driver.Create(fname_out, r.shape[1], r.shape[0], 3, gdal.GDT_Byte) + dset_output.SetGeoTransform(gt) + try: + proj = osr.SpatialReference() + proj.ImportFromEPSG(4326) + dset_output.SetSpatialRef(proj) + except: + print("Error: Coordinate system setting failed") + dset_output.GetRasterBand(1).WriteArray(r) + dset_output.GetRasterBand(2).WriteArray(g) + dset_output.GetRasterBand(3).WriteArray(b) + dset_output.FlushCache() + dset_output = None + print("Image Saved") + + +# --------------------------------------------------------- + +# --------------------------------------------------------- +MAP_URLS = { + "google": "https://mt1.google.com/vt/lyrs=r&x={x}&y={y}&z={z}", + "google_sat": "https://mt1.google.com/vt/lyrs=s&x={x}&y={y}&z={z}", + "amap": "https://webrd02.is.autonavi.com/appmaptile?lang=zh_cn&size=1&scale=1&style=8&x={x}&y={y}&z={z}", + "amap_sat": "https://webst01.is.autonavi.com/appmaptile?style=6&x={x}&y={y}&z={z}" + } + + +def get_url(source, x, y, z, style): # + if source == 'amap': + url = MAP_URLS["amap"].format(x=x, y=y, z=z) + elif source == 'amap_sat': + url = MAP_URLS["amap_sat"].format(x=x, y=y, z=z) + elif source == 'google': + url = MAP_URLS["google"].format(x=x, y=y, z=z) + elif source == 'google_sat': + url = MAP_URLS["google_sat"].format(x=x, y=y, z=z) + else: + raise Exception("Unknown Map Source ! ") + return url + + +def get_urls(x1, y1, x2, y2, z, source, style): + pos1x, pos1y = wgs_to_tile(x1, y1, z) + pos2x, pos2y = wgs_to_tile(x2, y2, z) + lenx = pos2x - pos1x + 1 + leny = pos2y - pos1y + 1 + print("Total tiles number：{x} X {y}".format(x=lenx, y=leny)) + urls = [get_url(source, i, j, z, style) for j in range(pos1y, pos1y + leny) for i in range(pos1x, pos1x + lenx)] + return urls + + +# --------------------------------------------------------- + +# --------------------------------------------------------- +def merge_tiles(datas, x1, y1, x2, y2, z): + pos1x, pos1y = wgs_to_tile(x1, y1, z) + pos2x, pos2y = wgs_to_tile(x2, y2, z) + lenx = pos2x - pos1x + 1 + leny = pos2y - pos1y + 1 + outpic = pil.new('RGBA', (lenx * 256, leny * 256)) + for i, data in enumerate(datas): + picio = io.BytesIO(data) + small_pic = pil.open(picio) + y, x = i // lenx, i % lenx + outpic.paste(small_pic, (x * 256, y * 256)) + print('Tiles merge completed') + return outpic + + +def download_tiles(urls, multi=10): + url_len = len(urls) + datas = [None] * url_len + if multi < 1 or multi > 20 or not isinstance(multi, int): + raise Exception("multi of Downloader shuold be int and between 1 to 20.") + tasks = [Downloader(i, multi, urls, datas) for i in range(multi)] + for i in tasks: + i.start() + for i in tasks: + i.join() + return datas + + +# --------------------------------------------------------- + +# --------------------------------------------------------- +def main(left, top, right, bottom, zoom, filePath, style='s', server="amap"): + """ + Download images based on spatial extent. + + East longitude is positive and west longitude is negative. + North latitude is positive, south latitude is negative. + + Parameters + ---------- + left, top : left-top coordinate, for example (100.361,38.866) + + right, bottom : right-bottom coordinate + + z : zoom + + filePath : File path for storing results, TIFF format + + style : + m for map; + s for satellite; + y for satellite with label; + t for terrain; + p for terrain with label; + h for label; + + source : Google or amap + """ + # --------------------------------------------------------- + # Get the urls of all tiles in the extent + urls = get_urls(left, top, right, bottom, zoom, server, style) + + # Group URLs based on the number of CPU cores to achieve roughly equal amounts of tasks + urls_group = [urls[i:i + math.ceil(len(urls) / multiprocessing.cpu_count())] for i in + range(0, len(urls), math.ceil(len(urls) / multiprocessing.cpu_count()))] + + # Each set of URLs corresponds to a process for downloading tile maps + print('Tiles downloading......') + pool = multiprocessing.Pool(multiprocessing.cpu_count()) + results = pool.map(download_tiles, urls_group) + pool.close() + pool.join() + result = [x for j in results for x in j] + print('Tiles download complete') + + # Combine downloaded tile maps into one map + outpic = merge_tiles(result, left, top, right, bottom, zoom) + outpic = outpic.convert('RGB') + r, g, b = cv2.split(np.array(outpic)) + + # Get the spatial information of the four corners of the merged map and use it for outputting + extent = getExtent(left, top, right, bottom, zoom, server) + gt = (extent['LT'][0], (extent['RB'][0] - extent['LT'][0]) / r.shape[1], 0, extent['LT'][1], 0, + (extent['RB'][1] - extent['LT'][1]) / r.shape[0]) + saveTiff(r, g, b, gt, filePath) + + +# --------------------------------------------------------- +if __name__ == '__main__': + start_time = time.time() + + #main(118.055917, 24.559724, 118.244753, 24.399450, 16, r'google_sat.tif', server="google_sat") + main(float(sys.argv[1]), float(sys.argv[4]), float(sys.argv[2]), float(sys.argv[3]), int(sys.argv[5]), sys.argv[6], server=sys.argv[7]) + + end_time = time.time() + print('lasted a total of {:.2f} seconds'.format(end_time - start_time)) diff --git a/third_party/PyINT/pyint/extract_s1_bursts.py b/third_party/PyINT/pyint/extract_s1_bursts.py new file mode 100644 index 0000000..2f03583 --- /dev/null +++ b/third_party/PyINT/pyint/extract_s1_bursts.py @@ -0,0 +1,297 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import glob +import argparse + +from pyint import _utils as ut + +def common_burst(La_M,La_S): + La_M = [float(i) for i in La_M] + La_S = [float(i) for i in La_S] + Min = max(min(La_M),min(La_S)) + Max = min(max(La_M),max(La_S)) + + M_min = [] + M_max = [] + for xm in La_M: + k0_min = float(xm) - float(Min) + k0_max = float(xm) - float(Max) + M_min.append(abs(k0_min)) + M_max.append(abs(k0_max)) + + M_Index_min = M_min.index(min(M_min)) + 1 + M_Index_max = M_max.index(min(M_max)) + 1 + Mindex =[M_Index_min,M_Index_max] + + S_min = [] + S_max = [] + for xs in La_S: + k0_min = float(xs) - float(Min) + k0_max = float(xs) - float(Max) + S_min.append(abs(k0_min)) + S_max.append(abs(k0_max)) + + S_Index_min = S_min.index(min(S_min)) + 1 + S_Index_max = S_max.index(min(S_max)) + 1 + Sindex =[S_Index_min,S_Index_max] + + return min(Mindex) , max(Mindex), min(Sindex),max(Sindex) + +def get_common_burst(Mslc_dir,Sslc_dir,common_burst_txt): + Mpar_list = sorted(glob.glob(Mslc_dir+'/*.IW*.burst.par')) + Spar_list = sorted(glob.glob(Sslc_dir+'/*.IW*.burst.par')) + if os.path.isfile(common_burst_txt): + os.remove(common_burst_txt) + for kk in range(len(Mpar_list)): + MBURST = Mpar_list[kk] + SBURST = Spar_list[kk] + + Mtt = Sslc_dir + '/' + os.path.basename(MBURST.replace('burst.par','tt0')) + Stt = SBURST.replace('burst.par','tt0') + + call_str = "grep 'Burst:' " + MBURST + ' >' + Mtt + os.system(call_str) + call_str = "grep 'Burst:' " + SBURST + ' >' + Stt + os.system(call_str) + + MM = ut.read_txt2array(Mtt) + SM = ut.read_txt2array(Stt) + La_M = MM[:,2] + La_S = SM[:,2] + + PP = common_burst_Ref(La_M,La_S) + + print('Common bursts of swath' + str(kk+1) + ' : (master) ' + str(PP[0]) + ' ' + str(PP[1]) + ' (slave) ' + str(PP[2]) + ' ' + str(PP[3])) + call_str = 'echo ' + str(PP[0]) + ' ' + str(PP[1]) + ' ' + str(PP[2]) + ' ' + str(PP[3]) + ' >>' + common_burst_txt + os.system(call_str) + + return + +def common_burst_Ref(La_M,La_S): + Min = max(min(La_M),min(La_S)) + Max = min(max(La_M),max(La_S)) + + M_min = [] + M_max = [] + for xm in La_M: + k0_min = float(xm) - float(Min) + k0_max = float(xm) - float(Max) + M_min.append(abs(k0_min)) + M_max.append(abs(k0_max)) + + M_Index_min = M_min.index(min(M_min)) + 1 + M_Index_max = M_max.index(min(M_max)) + 1 + Mindex =[M_Index_min,M_Index_max] + + S_min = [] + S_max = [] + for xs in La_S: + k0_min = float(xs) - float(Min) + k0_max = float(xs) - float(Max) + S_min.append(abs(k0_min)) + S_max.append(abs(k0_max)) + + S_Index_min = S_min.index(min(S_min)) + 1 + S_Index_max = S_max.index(min(S_max)) + 1 + Sindex =[S_Index_min,S_Index_max] + #print La_M + #print La_S + #print min(M_min),min(M_max),min(S_min),min(S_max) + M1 = min(Mindex) + M2 = max(Mindex) + + S1 = min(Sindex) + S2 = max(Sindex) + + if M1==1: S1 = S1 + else: + S1=1-M1+1 + M1=1 + + + if M2 ==len(La_M): S2 = S2 + else: + S2 = S2 + len(La_M) - M2 + M2 = len(La_M) + + + return M1 , M2, S1, S2 +######################################################################### + +INTRODUCTION = ''' +############################################################################# + + Extract the common bursts for S1 TOPs based on one master image. +''' + +EXAMPLE = ''' + Usage: + extract_s1_bursts.py projectName 170115 + + Examples: + extract_s1_bursts.py PacayaT163TsxHhA 170115 +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Check common busrts for TOPS data.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='Name of project.') + parser.add_argument('Sdate',help='Slave date.') + + inps = parser.parse_args() + + return inps + +################################################################################ + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Sdate = inps.Sdate + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Mdate = templateDict['masterDate'] + + processDir = scratchDir + '/' + projectName + "/PROCESS" + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + + workDir0 = rslcDir + '/' + Sdate + #if not os.path.isdir(rslcDir): + # os.mkdir(rslcDir) + + MslcDir = slcDir + '/' + Mdate + SslcDir = slcDir + '/' + Sdate + #MBurst_Par = slcDir + '/' + Mdate + '/' + + workDir = slcDir + '/' + Sdate + + BURST = workDir + '/' + Mdate + '_' + Sdate + '.common_burst_ref' + + MslcImg = workDir + '/'+Mdate + '.slc' + MslcPar = workDir + '/'+Mdate + '.slc.par' + + MamprlksImg = workDir + '/'+Mdate + '_' + rlks + 'rlks' + '.amp' + MamprlksPar = workDir + '/'+Mdate + '_' + rlks +'rlks' + '.amp.par' + + SslcImg = workDir + '/'+ Sdate + '.slc' + SslcPar = workDir + '/'+ Sdate + '.slc.par' + + SamprlksImg = workDir + '/'+ Sdate + '_' + rlks + 'rlks' + '.amp' + SamprlksPar = workDir + '/'+ Sdate + '_' + rlks + 'rlks' + '.amp.par' + + if not os.path.isdir(workDir): + os.mkdir(workDir) + + get_common_burst(MslcDir,SslcDir,BURST) + AA = ut.read_txt2array(BURST) + + SW = templateDict['start_swath'] + EW = templateDict['end_swath'] + + SB = templateDict['start_burst'] + EB = templateDict['end_burst'] + + + SLC2_tab = workDir + '/' + Sdate + '_SLC_Tab0' + + + SLC2_INF_tab = workDir + '/' + Sdate + '_SLC_Tab' + SLC2_RSLC_tab = workDir + '/' + Sdate + '_RSLC_Tab' + + BURST2_tab = workDir + '/' + Sdate + '_Burst_Tab' + + if os.path.isfile(SLC2_tab): + os.remove(SLC2_tab) + + if os.path.isfile(SLC2_INF_tab): + os.remove(SLC2_INF_tab) + + if os.path.isfile(SLC2_RSLC_tab): + os.remove(SLC2_RSLC_tab) + + if os.path.isfile(BURST2_tab): + os.remove(BURST2_tab) + + for kk in range(int(EW)-int(SW)+1): + + call_str = 'echo ' + SslcDir + '/' + Sdate+'.IW'+str(int(SW)+kk) + '.slc' + ' ' + SslcDir + '/'+ Sdate + '.IW'+str(int(SW)+kk) +'.slc.par' + ' ' + SslcDir + '/'+ Sdate+'.IW'+str(int(SW)+kk) + '.slc.TOPS_par >>' + SLC2_tab + os.system(call_str) + + ii = kk + 1 + if (int(SW)==int(EW)): + SB1=int(AA[0]) + EB1=int(AA[1]) + SB2=int(AA[2]) + EB2=int(AA[3]) + else: + SB1=int(AA[ii-1,0]) + EB1=int(AA[ii-1,1]) + + SB2=int(AA[ii-1,2]) + EB2=int(AA[ii-1,3]) + + if not int(SB)==1: + SB2 = SB2 + int(SB)-int(SB1) + if not int(EB)==20: + EB2 = SB2 + int(EB) - int(SB) + + call_str = 'echo ' + workDir + '/'+ Sdate+ '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+str(int(SW)+kk)+ '.slc' + ' ' + workDir + '/' + Sdate + '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+ str(int(SW)+kk)+ '.slc.par' + ' ' + workDir + '/'+ Sdate+'_'+ str(int(SB2)) + str(int(EB2)) + '.IW'+str(int(SW)+kk)+ '.slc.TOPS_par >>' + SLC2_INF_tab + os.system(call_str) + + call_str = 'echo ' + workDir0 + '/'+ Sdate+ '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+str(int(SW)+kk)+ '.rslc' + ' ' + workDir0 + '/' + Sdate + '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+ str(int(SW)+kk)+ '.rslc.par' + ' ' + workDir0 + '/'+ Sdate+'_'+ str(int(SB2)) + str(int(EB2)) + '.IW'+str(int(SW)+kk)+ '.rslc.TOPS_par >>' + SLC2_RSLC_tab + os.system(call_str) + + call_str = 'echo ' + str(int(SB2)) + ' ' + str(int(EB2)) + ' >>' + BURST2_tab + os.system(call_str) + + + TEST = SamprlksPar + k0 = 0 + if os.path.isfile(TEST): + if os.path.getsize(TEST) > 0: + k0 = 1 + + if k0==0: + + call_str = 'SLC_copy_ScanSAR ' + SLC2_tab + ' ' + SLC2_INF_tab + ' ' + BURST2_tab + os.system(call_str) + + call_str = 'SLC_mosaic_ScanSAR ' + SLC2_INF_tab + ' ' + SslcImg + ' ' + SslcPar + ' ' + rlks + ' ' +azlks + os.system(call_str) + + call_str = 'multi_look ' + SslcImg + ' ' + SslcPar + ' ' + SamprlksImg + ' ' + SamprlksPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + + nWidth = ut.read_gamma_par(SamprlksPar, 'read', 'range_samples') + call_str = 'raspwr ' + SamprlksImg + ' ' + nWidth + os.system(call_str) + iw1 = Sdate + '.IW1.slc*' + iw2 = Sdate + '.IW2.slc*' + iw3 = Sdate + '.IW3.slc*' + call_str = 'rm '+ iw1 + ' ' + iw2 + ' '+ iw3 + os.system(call_str) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/extract_s1_bursts_all.py b/third_party/PyINT/pyint/extract_s1_bursts_all.py new file mode 100644 index 0000000..2a48bf8 --- /dev/null +++ b/third_party/PyINT/pyint/extract_s1_bursts_all.py @@ -0,0 +1,96 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + + +def work(data0): + cmd = data0[0] + err_file = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + #print(aa) + with open(err_file, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + Extract reference TOPS related bursts for coregistration using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + extract_s1_bursts_all.py projectName + extract_s1_bursts_all.py projectName --parallel 4 +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Extract reference TOPS related bursts for coregistration using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + slcDir = scratchDir + '/' + projectName + '/SLC' + slc_list = [os.path.basename(fname) for fname in sorted(glob.glob(slcDir + '/*'))] + + err_txt = scratchDir + '/' + projectName + '/extract_s1_bursts_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + for i in range(len(slc_list)): + cmd0 = ['extract_s1_bursts.py',projectName,slc_list[i]] + data0 = [cmd0,err_txt] + data_para.append(data0) + + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + print("Extract TOPS bursts for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + \ No newline at end of file diff --git a/third_party/PyINT/pyint/extract_s1_bursts_old.py b/third_party/PyINT/pyint/extract_s1_bursts_old.py new file mode 100644 index 0000000..062469c --- /dev/null +++ b/third_party/PyINT/pyint/extract_s1_bursts_old.py @@ -0,0 +1,292 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import glob +import argparse + +from pyint import _utils as ut + +def common_burst(La_M,La_S): + La_M = [float(i) for i in La_M] + La_S = [float(i) for i in La_S] + Min = max(min(La_M),min(La_S)) + Max = min(max(La_M),max(La_S)) + + M_min = [] + M_max = [] + for xm in La_M: + k0_min = float(xm) - float(Min) + k0_max = float(xm) - float(Max) + M_min.append(abs(k0_min)) + M_max.append(abs(k0_max)) + + M_Index_min = M_min.index(min(M_min)) + 1 + M_Index_max = M_max.index(min(M_max)) + 1 + Mindex =[M_Index_min,M_Index_max] + + S_min = [] + S_max = [] + for xs in La_S: + k0_min = float(xs) - float(Min) + k0_max = float(xs) - float(Max) + S_min.append(abs(k0_min)) + S_max.append(abs(k0_max)) + + S_Index_min = S_min.index(min(S_min)) + 1 + S_Index_max = S_max.index(min(S_max)) + 1 + Sindex =[S_Index_min,S_Index_max] + return min(Mindex) , max(Mindex), min(Sindex),max(Sindex) + +def get_common_burst(Mslc_dir,Sslc_dir,common_burst_txt): + Mpar_list = sorted(glob.glob(Mslc_dir+'/*.IW*.burst.par')) + Spar_list = sorted(glob.glob(Sslc_dir+'/*.IW*.burst.par')) + if os.path.isfile(common_burst_txt): + os.remove(common_burst_txt) + for kk in range(len(Mpar_list)): + MBURST = Mpar_list[kk] + SBURST = Spar_list[kk] + + Mtt = Sslc_dir + '/' + os.path.basename(MBURST.replace('burst.par','tt0')) + Stt = SBURST.replace('burst.par','tt0') + call_str = "grep 'Burst:' " + MBURST + ' >' + Mtt + os.system(call_str) + + call_str = "grep 'Burst:' " + SBURST + ' >' + Stt + os.system(call_str) + + MM = ut.read_txt2array(Mtt) + SM = ut.read_txt2array(Stt) + La_M = MM[:,2] + La_S = SM[:,2] + + PP = common_burst_Ref(La_M,La_S) + + print('Common bursts of swath' + str(kk+1) + ' : (master) ' + str(PP[0]) + ' ' + str(PP[1]) + ' (slave) ' + str(PP[2]) + ' ' + str(PP[3])) + call_str = 'echo ' + str(PP[0]) + ' ' + str(PP[1]) + ' ' + str(PP[2]) + ' ' + str(PP[3]) + ' >>' + common_burst_txt + os.system(call_str) + + return + +def common_burst_Ref(La_M,La_S): + Min = max(min(La_M),min(La_S)) + Max = min(max(La_M),max(La_S)) + + M_min = [] + M_max = [] + for xm in La_M: + k0_min = float(xm) - float(Min) + k0_max = float(xm) - float(Max) + M_min.append(abs(k0_min)) + M_max.append(abs(k0_max)) + + M_Index_min = M_min.index(min(M_min)) + 1 + M_Index_max = M_max.index(min(M_max)) + 1 + Mindex =[M_Index_min,M_Index_max] + + S_min = [] + S_max = [] + for xs in La_S: + k0_min = float(xs) - float(Min) + k0_max = float(xs) - float(Max) + S_min.append(abs(k0_min)) + S_max.append(abs(k0_max)) + + S_Index_min = S_min.index(min(S_min)) + 1 + S_Index_max = S_max.index(min(S_max)) + 1 + Sindex =[S_Index_min,S_Index_max] + #print La_M + #print La_S + #print min(M_min),min(M_max),min(S_min),min(S_max) + M1 = min(Mindex) + M2 = max(Mindex) + + S1 = min(Sindex) + S2 = max(Sindex) + + if M1==1: S1 = S1 + else: + S1=1-M1+1 + M1=1 + + + if M2 ==len(La_M): S2 = S2 + else: + S2 = S2 + len(La_M) - M2 + M2 = len(La_M) + + + return M1 , M2, S1, S2 +######################################################################### + +INTRODUCTION = ''' +############################################################################# + + Extract the common bursts for S1 TOPs based on one master image. +''' + +EXAMPLE = ''' + Usage: + extract_s1_bursts.py projectName 170115 + + Examples: + extract_s1_bursts.py PacayaT163TsxHhA 170115 +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Check common busrts for TOPS data.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='Name of project.') + parser.add_argument('Sdate',help='Slave date.') + + inps = parser.parse_args() + + return inps + +################################################################################ + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Sdate = inps.Sdate + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Mdate = templateDict['masterDate'] + + processDir = scratchDir + '/' + projectName + "/PROCESS" + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + + workDir0 = rslcDir + '/' + Sdate + #if not os.path.isdir(rslcDir): + # os.mkdir(rslcDir) + + MslcDir = slcDir + '/' + Mdate + SslcDir = slcDir + '/' + Sdate + #MBurst_Par = slcDir + '/' + Mdate + '/' + + workDir = slcDir + '/' + Sdate + + BURST = workDir + '/' + Mdate + '_' + Sdate + '.common_burst_ref' + + MslcImg = workDir + '/'+Mdate + '.slc' + MslcPar = workDir + '/'+Mdate + '.slc.par' + + MamprlksImg = workDir + '/'+Mdate + '_' + rlks + 'rlks' + '.amp' + MamprlksPar = workDir + '/'+Mdate + '_' + rlks +'rlks' + '.amp.par' + + SslcImg = workDir + '/'+ Sdate + '.slc' + SslcPar = workDir + '/'+ Sdate + '.slc.par' + + SamprlksImg = workDir + '/'+ Sdate + '_' + rlks + 'rlks' + '.amp' + SamprlksPar = workDir + '/'+ Sdate + '_' + rlks + 'rlks' + '.amp.par' + + if not os.path.isdir(workDir): + os.mkdir(workDir) + + get_common_burst(MslcDir,SslcDir,BURST) + AA = ut.read_txt2array(BURST) + NL = ut.get_txt_lines(BURST) + + if NL ==1: + AA = AA.reshape(1,len(AA)) + + + SW = templateDict['start_swath'] + EW = templateDict['end_swath'] + + SB = templateDict['start_burst'] + EB = templateDict['end_burst'] + + + SLC2_tab = workDir + '/' + Sdate + '_SLC_Tab0' + + + SLC2_INF_tab = workDir + '/' + Sdate + '_SLC_Tab' + SLC2_RSLC_tab = workDir + '/' + Sdate + '_RSLC_Tab' + + BURST2_tab = workDir + '/' + Sdate + '_Burst_Tab' + + if os.path.isfile(SLC2_tab): + os.remove(SLC2_tab) + + if os.path.isfile(SLC2_INF_tab): + os.remove(SLC2_INF_tab) + + if os.path.isfile(SLC2_RSLC_tab): + os.remove(SLC2_RSLC_tab) + + if os.path.isfile(BURST2_tab): + os.remove(BURST2_tab) + + for kk in range(int(EW)-int(SW)+1): + + call_str = 'echo ' + SslcDir + '/' + Sdate+'.IW'+str(int(SW)+kk) + '.slc' + ' ' + SslcDir + '/'+ Sdate + '.IW'+str(int(SW)+kk) +'.slc.par' + ' ' + SslcDir + '/'+ Sdate+'.IW'+str(int(SW)+kk) + '.slc.TOPS_par >>' + SLC2_tab + os.system(call_str) + + ii = int(SW) + kk + SB1=int(AA[ii-1,0]) + EB1=int(AA[ii-1,1]) + + SB2=int(AA[ii-1,2]) + EB2=int(AA[ii-1,3]) + + if not int(SB)==1: + SB2 = SB2 + int(SB) - int(SB1) + if not int(EB)==100: + EB2 = SB2 + int(EB) - int(SB) + + call_str = 'echo ' + workDir + '/'+ Sdate+ '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+str(int(SW)+kk)+ '.slc' + ' ' + workDir + '/' + Sdate + '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+ str(int(SW)+kk)+ '.slc.par' + ' ' + workDir + '/'+ Sdate+'_'+ str(int(SB2)) + str(int(EB2)) + '.IW'+str(int(SW)+kk)+ '.slc.TOPS_par >>' + SLC2_INF_tab + os.system(call_str) + + call_str = 'echo ' + workDir0 + '/'+ Sdate+ '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+str(int(SW)+kk)+ '.rslc' + ' ' + workDir + '/' + Sdate + '_'+ str(int(SB2)) + str(int(EB2)) +'.IW'+ str(int(SW)+kk)+ '.rslc.par' + ' ' + workDir + '/'+ Sdate+'_'+ str(int(SB2)) + str(int(EB2)) + '.IW'+str(int(SW)+kk)+ '.rslc.TOPS_par >>' + SLC2_RSLC_tab + os.system(call_str) + + call_str = 'echo ' + str(int(SB2)) + ' ' + str(int(EB2)) + ' >>' + BURST2_tab + os.system(call_str) + + + TEST = SamprlksPar + k0 = 0 + if os.path.isfile(TEST): + if os.path.getsize(TEST) > 0: + k0 = 1 + + if k0==0: + +# call_str = 'SLC_copy_ScanSAR ' + SLC2_tab + ' ' + SLC2_INF_tab + ' ' + BURST2_tab +# os.system(call_str) + +# call_str = 'SLC_mosaic_S1_TOPS ' + SLC2_INF_tab + ' ' + SslcImg + ' ' + SslcPar + ' ' + rlks + ' ' +azlks + call_str = 'SLC_mosaic_S1_TOPS ' + SLC2_tab + ' ' + SslcImg + ' ' + SslcPar + ' ' + rlks + ' ' +azlks + os.system(call_str) + + call_str = 'multi_look ' + SslcImg + ' ' + SslcPar + ' ' + SamprlksImg + ' ' + SamprlksPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + + nWidth = ut.read_gamma_par(SamprlksPar, 'read', 'range_samples') + call_str = 'raspwr ' + SamprlksImg + ' ' + str(nWidth) + os.system(call_str) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/gacos_gamma.py b/third_party/PyINT/pyint/gacos_gamma.py new file mode 100644 index 0000000..75370f5 --- /dev/null +++ b/third_party/PyINT/pyint/gacos_gamma.py @@ -0,0 +1,1186 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +""" +GACOS atmospheric correction for GAMMA interferograms. +This script applies GACOS-derived tropospheric corrections to unwrapped interferograms. + +GACOS (Generic Atmospheric Correction Online Service) provides zenith total delay (ZTD) +maps that can be used to correct for atmospheric phase delays in InSAR data. + +Usage: + gacos_gamma.py projectName Mdate-Sdate + gacos_gamma.py PacayaT163TsxHhA 20150102-20150601 +""" + +import numpy as np +import os +import sys +import argparse +import subprocess +import glob +import warnings +from pathlib import Path +from datetime import datetime +from scipy.interpolate import RegularGridInterpolator + +from pyint import _utils as ut + +warnings.filterwarnings('ignore', category=RuntimeWarning) + +# Try to import AutoGACOS modules +try: + from gacos import Downloader, Submitter, SarDataset + AUTOGACOS_AVAILABLE = True +except Exception as _e: + AUTOGACOS_AVAILABLE = False + print(f"[DEBUG] AutoGACOS import failed: {type(_e).__name__}: {_e}") + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Apply GACOS atmospheric correction to interferograms. + + This script: + 1. Generates elevation angle file from GAMMA products + 2. Downloads GACOS ZTD data for master and slave dates + 3. Applies tropospheric correction to the interferogram + + Prerequisites: + - GACOS data for the acquisition dates (or valid email for auto-download) + - Geocoded interferogram from geocode_gamma.py +------------------------------------------------------------------- +''' + +EXAMPLE = ''' + Usage: + gacos_gamma.py projectName Mdate-Sdate + gacos_gamma.py projectName Mdate-Sdate --ztd-dir /path/to/gacos/data + gacos_gamma.py PacayaT163TsxHhA 20150102-20150601 +------------------------------------------------------------------- +''' + + +class HEADER: + """Header class for GACOS file format""" + width = 0 + length = 0 + xfirst = 0.0 + yfirst = 0.0 + xstep = 0.0 + ystep = 0.0 + + +def read_header(filename): + """Read header information from GACOS .rsc file""" + if not os.path.isfile(filename): + print(filename + " file not exist") + return None + + header = HEADER() + with open(filename) as f: + for line in f: + data = line.split() + if len(data) >= 2: + if data[0] == "WIDTH": + header.width = int(data[1]) + if data[0] == "FILE_LENGTH": + header.length = int(data[1]) + if data[0] == "X_FIRST": + header.xfirst = float(data[1]) + if data[0] == "Y_FIRST": + header.yfirst = float(data[1]) + if data[0] == "X_STEP": + header.xstep = float(data[1]) + if data[0] == "Y_STEP": + header.ystep = float(data[1]) + return header + + +def cut_image2(filename, headername, yfirst_new, length_new, xfirst_new, width_new): + """Cut GACOS ZTD image to match interferogram extent""" + header = read_header(headername) + if header is None: + return None + + with open(filename, 'rb') as f: + data0 = np.fromfile(f, dtype=np.float32) + data = np.reshape(data0, (header.length, header.width)) + + out = np.zeros((length_new, width_new), dtype=np.float32) + + for i in range(header.length): + lat = header.yfirst + header.ystep * i + row = int(round((lat - yfirst_new) / header.ystep)) + if row < 0 or row >= length_new: + continue + for j in range(header.width): + lon = header.xfirst + header.xstep * j + col = int(round((lon - xfirst_new) / header.xstep)) + if col < 0 or col >= width_new: + continue + out[row, col] = data[i, j] + + out = np.where(out == 0, np.nan, out) + out.tofile(filename + ".cut") + + f = open(filename + ".cut.rsc", 'w') + f.write("WIDTH " + str(width_new) + "\n") + f.write("FILE_LENGTH " + str(length_new) + "\n") + f.write("X_FIRST " + str(xfirst_new) + "\n") + f.write("Y_FIRST " + str(yfirst_new) + "\n") + f.write("X_STEP " + str(header.xstep) + "\n") + f.write("Y_STEP " + str(header.ystep) + "\n") + f.close() + + return filename + ".cut" + + +def make_correction(phsfilename, ztd1filename, ztd2filename, elevfilename, wavelength=None): + """ + Apply GACOS tropospheric correction to interferogram. + + Parameters: + ----------- + phsfilename : str + Path to unwrapped phase file + ztd1filename : str + Path to GACOS ZTD file for master date + ztd2filename : str + Path to GACOS ZTD file for slave date + elevfilename : str + Path to elevation angle file + wavelength : float, optional + Radar wavelength in meters. If None, read from template. + """ + header = read_header(phsfilename + ".rsc") + if header is None: + print("Error reading phase header file") + return None + + # Cut ZTD files to match interferogram extent + if not os.path.isfile(ztd1filename + ".cut"): + cut_image2(ztd1filename, ztd1filename + ".rsc", header.yfirst, header.length, header.xfirst, header.width) + if not os.path.isfile(ztd2filename + ".cut"): + cut_image2(ztd2filename, ztd2filename + ".rsc", header.yfirst, header.length, header.xfirst, header.width) + + # Read phase data + with open(phsfilename, 'rb') as f: + data = np.fromfile(f, dtype=np.float32) + phase = np.reshape(data, [header.length, header.width]) + + # Read ZTD data + with open(ztd1filename + ".cut", 'rb') as f: + data = np.fromfile(f, dtype=np.float32) + ztd1 = np.reshape(data, [header.length, header.width]) + + with open(ztd2filename + ".cut", 'rb') as f: + data = np.fromfile(f, dtype=np.float32) + ztd2 = np.reshape(data, [header.length, header.width]) + + # Read elevation angle data + with open(elevfilename, 'rb') as f: + data = np.fromfile(f, dtype=np.float32) + elev = np.reshape(data, [header.length, header.width]) + + # Calculate tropospheric phase delay + # ZTD difference to phase: phase = 4*pi*dZTD / (wavelength * sin(elevation)) + # Default wavelength factor for Sentinel-1: 0.0044138251819503 = 4*pi/lambda + if wavelength: + ztd_factor = 4 * np.pi / wavelength + else: + ztd_factor = 1 / 0.0044138251819503 # Default factor + + dztd = ztd2 - ztd1 + dztd = dztd / 0.0044138251819503 # Convert ZTD to phase + dztd = dztd / np.sin(elev) # Incidence angle correction + + # Apply correction + index = np.where(phase == 0) + phase[index] = np.nan + phasemean = np.nanmean(phase) + print("Phase std before correction: " + str(np.nanstd(phase))) + + phase = phase - phasemean + phase = phase - dztd # Apply correction + phase[index] = np.nan + phasemean = np.nanmean(phase) + print("Phase std after correction: " + str(np.nanstd(phase))) + + phase = phase - phasemean + phase[index] = 0 + phase.tofile(phsfilename + ".gacos") + + return phsfilename + ".gacos" + + +def generate_elevation_angle(workDir, Mdate, Sdate, MampPar, offpar, dempar, dem, rlks): + """ + Generate elevation angle file from GAMMA products. + + The elevation angle (90 - incidence angle) is needed for + projecting ZTD to line-of-sight phase delay. + """ + os.chdir(workDir) + + # Generate look vector (incidence angle) + call_str = "look_vector " + MampPar + " " + offpar + " " + dempar + " " + dem + " lv_theta lv_phi" + os.system(call_str) + + # Get DEM parameters + call_str = "grep 'corner_lat:' " + dempar + " | awk '{print $2}' " + North = subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'corner_lon:' " + dempar + " | awk '{print $2}' " + West = subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lat:' " + dempar + " | awk '{print $2}' " + posty = subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lon:' " + dempar + " | awk '{print $2}' " + postx = subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'width:' " + dempar + " | awk '{print $2}' " + width = subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'nlines:' " + dempar + " | awk '{print $2}' " + length = subprocess.getstatusoutput(call_str)[1] + + South = str(round(float(North) + (float(length) - 1) * float(posty), 7)) + East = str(round(float(West) + (float(width) - 1) * float(postx), 7)) + + # Convert to elevation angle + call_str = "swap_bytes lv_theta lv_theta.phase_swap 4 > dinsar.log " + os.system(call_str) + + call_str = "gmt xyz2grd lv_theta.phase_swap -Glv_theta.grd -Ddegree/degree/cm/1/0/=/= -R" + West + "/" + East + "/" + South + "/" + North + " -I" + postx + " -ZTLf -N0" + os.system(call_str) + + # Elevation angle = 90 - incidence angle + call_str = "gmt grdmath 90 lv_theta.grd 3.1415926 DIV 180 MUL SUB = lv_theta_final.grd" + os.system(call_str) + call_str = "gmt grdmath 90 lv_theta_final.grd SUB = lv_elev.grd" + os.system(call_str) + + # Resample to 3 arc-seconds + call_str = "gmt grdsample lv_elev.grd -Glv_elev_3c.grd -I3c" + os.system(call_str) + + # Convert to binary + call_str = "gmt grd2xyz lv_elev_3c.grd -ZTLf -N0 > " + Mdate + '-' + Sdate + '.gacos.elev' + os.system(call_str) + + return workDir + '/' + Mdate + '-' + Sdate + '.gacos.elev' + + +def prepare_unw_for_gacos(workDir, unw, Mdate, Sdate, dempar, rlks): + """ + Prepare unwrapped interferogram for GACOS correction. + + Converts GAMMA format to GACOS-compatible format with .rsc file. + """ + os.chdir(workDir) + + # Get DEM parameters + call_str = "grep 'corner_lat:' " + dempar + " | awk '{print $2}' " + North = subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'corner_lon:' " + dempar + " | awk '{print $2}' " + West = subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lat:' " + dempar + " | awk '{print $2}' " + posty = subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'post_lon:' " + dempar + " | awk '{print $2}' " + postx = subprocess.getstatusoutput(call_str)[1] + + # Convert unw to grd + call_str = "swap_bytes " + unw + " unw.phase_swap 4 > dinsar.log " + os.system(call_str) + + # Get dimensions + call_str = "gmt grdinfo unw_f.grd -C" + result = subprocess.getstatusoutput(call_str)[1].split() + if len(result) >= 12: + Width = result[9] + line = result[10] + West = result[1] + North = result[4] + else: + # Fallback to DEM parameters + call_str = "grep 'width:' " + dempar + " | awk '{print $2}' " + Width = subprocess.getstatusoutput(call_str)[1] + call_str = "grep 'nlines:' " + dempar + " | awk '{print $2}' " + line = subprocess.getstatusoutput(call_str)[1] + + ymax = str(int(round(float(line), 7))) + xmax = str(int(round(float(Width), 7))) + + # Create .rsc file + output = workDir + '/' + Mdate + '-' + Sdate + '.gacos.unw.rsc' + if os.path.exists(output): + os.remove(output) + + with open(output, 'a+') as f: + f.write('WIDTH ' + str(Width) + '\n') + f.write('FILE_LENGTH ' + str(line) + '\n') + f.write('XMIN 1' + '\n') + f.write('XMAX ' + xmax + '\n') + f.write('YMIN 1' + '\n') + f.write('YMAX ' + ymax + '\n') + f.write('X_FIRST ' + str(West) + '\n') + f.write('Y_FIRST ' + str(North) + '\n') + f.write('X_STEP 8.33333333E-04' + '\n') + f.write('Y_STEP -8.33333333E-04' + '\n') + f.write('X_UNIT degrees' + '\n') + f.write('Y_UNIT degrees' + '\n') + f.write('Z_OFFSET 0' + '\n') + f.write('Z_SCALE 1' + '\n') + f.write('PROJECTION LATLON' + '\n') + f.write('DATUM WGS84' + '\n') + + return workDir + '/' + Mdate + '-' + Sdate + '.gacos.unw' + + +def parse_dem_par(dempar): + """ + 从 GAMMA DEM par 文件读取网格参数（纯 Python，无需 grep/awk） + """ + info = {} + with open(dempar) as f: + for line in f: + parts = line.split(':') + if len(parts) >= 2: + key = parts[0].strip() + val = parts[1].strip().split()[0] if parts[1].strip() else '' + info[key] = val + return { + 'width': int(info['width']), + 'nlines': int(info['nlines']), + 'corner_lat': float(info['corner_lat']), + 'corner_lon': float(info['corner_lon']), + 'post_lat': float(info['post_lat']), + 'post_lon': float(info['post_lon']), + } + + +def resample_ztd_to_dem(ztd_file, rsc_file, dem_info): + """ + 读取 ZTD 数据并双线性插值重采样到 DEM 网格 + 修复: RegularGridInterpolator 要求坐标严格递增， + ZTD 纬度为递减（北→南），需翻转 + """ + # 读取 RSC 头文件 + rsc = {} + with open(rsc_file) as f: + for line in f: + parts = line.split() + if len(parts) >= 2: + rsc[parts[0]] = parts[1] + + ztd_w = int(rsc['WIDTH']) + ztd_h = int(rsc['FILE_LENGTH']) + ztd_x0 = float(rsc['X_FIRST']) + ztd_y0 = float(rsc['Y_FIRST']) + ztd_dx = float(rsc['X_STEP']) + ztd_dy = float(rsc['Y_STEP']) + + raw = np.fromfile(ztd_file, dtype=np.float32).reshape(ztd_h, ztd_w) + + # 构建 ZTD 网格坐标 + ztd_lats = ztd_y0 + np.arange(ztd_h) * ztd_dy + ztd_lons = ztd_x0 + np.arange(ztd_w) * ztd_dx + + # 翻转使坐标严格递增（RegularGridInterpolator 要求） + if ztd_lats[0] > ztd_lats[-1]: + ztd_lats = ztd_lats[::-1] + raw = raw[::-1, :] + if ztd_lons[0] > ztd_lons[-1]: + ztd_lons = ztd_lons[::-1] + raw = raw[:, ::-1] + + interp = RegularGridInterpolator( + (ztd_lats, ztd_lons), raw, + method='linear', bounds_error=False, fill_value=np.nan + ) + + # 构建 DEM 网格查询点 + dem_w = dem_info['width'] + dem_h = dem_info['nlines'] + dem_lats = dem_info['corner_lat'] + np.arange(dem_h) * dem_info['post_lat'] + dem_lons = dem_info['corner_lon'] + np.arange(dem_w) * dem_info['post_lon'] + + # 逐行插值（节省内存） + result = np.empty((dem_h, dem_w), dtype=np.float32) + for i in range(dem_h): + pts = np.column_stack([np.full(dem_w, dem_lats[i]), dem_lons]) + result[i] = interp(pts).astype(np.float32) + + return result + + +def convert_ztd_tif_to_binary(tif_file): + """ + 将 .ztd.tif (GeoTIFF) 转换为 .ztd (raw binary) + .ztd.rsc (header) + 返回 .ztd 文件路径 + """ + try: + import rasterio + except ImportError: + from osgeo import gdal + ds = gdal.Open(tif_file) + gt = ds.GetGeoTransform() + w, h = ds.RasterXSize, ds.RasterYSize + data = ds.GetRasterBand(1).ReadAsArray().astype(np.float32) + x0, y0, dx, dy = gt[0], gt[3], gt[1], gt[5] + ds = None + else: + with rasterio.open(tif_file) as src: + data = src.read(1).astype(np.float32) + w, h = src.width, src.height + x0 = src.transform[2] + y0 = src.transform[5] + dx = src.transform[0] + dy = src.transform[4] + + ztd_file = tif_file.replace('.ztd.tif', '.ztd') + rsc_file = tif_file.replace('.ztd.tif', '.ztd.rsc') + + data.tofile(ztd_file) + with open(rsc_file, 'w') as f: + f.write(f"WIDTH {w}\n") + f.write(f"FILE_LENGTH {h}\n") + f.write(f"X_FIRST {x0}\n") + f.write(f"Y_FIRST {y0}\n") + f.write(f"X_STEP {dx}\n") + f.write(f"Y_STEP {dy}\n") + f.write("X_UNIT degrees\n") + f.write("Y_UNIT degrees\n") + f.write("PROJECTION LATLON\n") + f.write("DATUM WGS84\n") + + return ztd_file + + +def apply_gacos_correction_python(geo_unw_file, ztd1_file, ztd1_rsc, ztd2_file, ztd2_rsc, + dempar, slcpar=None, wavelength=None, out_file=None, + figdir=None, pair_str=None): + """ + 纯 Python GACOS 大气校正（不依赖 GMT） + 修复原 make_correction 的所有 Bug: + - 极端值掩码（GAMMA 无效标记） + - sin/cos 弧度转换 + - cut_image2 慢速循环 → scipy 向量化重采样 + - 增加校正前后对比图 + + 参数: + ----- + geo_unw_file : str 地理编码后的解缠相位文件 + ztd1_file : str Master 日期 ZTD 二进制文件 + ztd1_rsc : str Master 日期 ZTD RSC 头文件 + ztd2_file : str Slave 日期 ZTD 二进制文件 + ztd2_rsc : str Slave 日期 ZTD RSC 头文件 + dempar : str DEM 参数文件 + slcpar : str SLC 参数文件（读取入射角，可选） + wavelength : float 雷达波长（m），None则从slcpar读取 + out_file : str 输出文件路径（默认 geo_unw_file + '.gacos'） + figdir : str 对比图保存目录（None则不绘图） + pair_str : str 干涉对名称如 '20241105-20241117' + + 返回: 输出文件路径 + """ + # 1. 读取 DEM 参数 + dem = parse_dem_par(dempar) + dem_w, dem_h = dem['width'], dem['nlines'] + + # 2. 读取入射角和波长 + inc_angle_deg = 39.0 # 默认值 + if slcpar and os.path.isfile(slcpar): + with open(slcpar) as f: + for line in f: + if line.startswith('incidence_angle:'): + inc_angle_deg = float(line.split(':')[1].strip().split()[0]) + if line.startswith('radar_frequency:') and wavelength is None: + freq = float(line.split(':')[1].strip().split()[0]) + wavelength = 299792458.0 / freq + + if wavelength is None: + wavelength = 0.0554657595 # Sentinel-1 C-band 默认值 + + cos_inc = np.cos(np.radians(inc_angle_deg)) + ztd_to_phase = 4.0 * np.pi / wavelength + + print(f" 入射角: {inc_angle_deg:.4f}°, cos={cos_inc:.6f}") + print(f" 波长: {wavelength:.10f} m") + + # 3. 读取地理编码后的解缠相位 + unw_data = np.fromfile(geo_unw_file, dtype=np.float32).reshape(dem_h, dem_w) + + # 4. 重采样 ZTD 到 DEM 网格 + print(" 重采样 ZTD Master...") + ztd_m = resample_ztd_to_dem(ztd1_file, ztd1_rsc, dem) + print(" 重采样 ZTD Slave...") + ztd_s = resample_ztd_to_dem(ztd2_file, ztd2_rsc, dem) + + # 5. 计算 LOS 相位校正量 + dztd = ztd_s - ztd_m + phase_correction = dztd * ztd_to_phase / cos_inc + + # 6. 构建有效像素掩码 + # 排除: 零值 | NaN | GAMMA 伪影/解缠错误（|x| >= 1000 rad ≈ 4.4m LOS） + PHASE_THRESH = 1000.0 + valid = ((unw_data != 0) + & np.isfinite(unw_data) + & (np.abs(unw_data) < PHASE_THRESH) + & np.isfinite(phase_correction)) + n_valid = np.sum(valid) + n_total = dem_w * dem_h + print(f" 有效像素: {n_valid}/{n_total} ({n_valid/n_total*100:.1f}%)") + + if n_valid == 0: + print(" 错误: 无有效像素，跳过校正") + return None + + # 7. 统计校正前 + std_before = float(np.std(unw_data[valid])) + + # 8. 应用校正（保留原始数据结构，仅修改有效像素） + corrected = unw_data.copy() + corrected[valid] = unw_data[valid] - phase_correction[valid] + + # 去均值（仅对有效像素） + mean_val = np.mean(corrected[valid]) + corrected[valid] -= mean_val + + std_after = float(np.std(corrected[valid])) + reduction = (1 - std_after / std_before) * 100 if std_before > 0 else 0 + + print(f" 校正前 std: {std_before:.3f} rad") + print(f" 校正后 std: {std_after:.3f} rad") + print(f" std 降低: {reduction:.1f}%") + + # 9. 保存 + if out_file is None: + out_file = geo_unw_file + '.gacos' + corrected.astype(np.float32).tofile(out_file) + print(f" 输出: {out_file}") + + # 10. 绘制对比图 + if figdir: + plot_gacos_comparison(unw_data, phase_correction, corrected, valid, + dem, std_before, std_after, pair_str, figdir) + + return out_file + + +def plot_gacos_comparison(unw_before, correction, unw_after, valid_mask, + dem_info, std_before, std_after, pair_str, figdir): + """ + 绘制 GACOS 校正前后相位对比图（三子图） + 版本控制: 自动追加 _v1, _v2... + """ + import matplotlib + matplotlib.use('Agg') + import matplotlib.pyplot as plt + import matplotlib.colors as mcolors + + # 中文字体配置 + plt.rcParams['font.sans-serif'] = ['Noto Serif CJK SC', 'Noto Sans CJK SC', + 'AR PL UMing CN', 'SimHei', 'DejaVu Sans'] + plt.rcParams['axes.unicode_minus'] = False + + figdir = Path(figdir) + figdir.mkdir(parents=True, exist_ok=True) + + # 版本控制 + base_name = f"GACOS_comparison_{pair_str}" if pair_str else "GACOS_comparison" + existing = sorted(figdir.glob(f"{base_name}_v*.png")) + ver = int(existing[-1].stem.split('_v')[-1]) + 1 if existing else 1 + out_path = figdir / f"{base_name}_v{ver}.png" + + # 准备显示数据（仅显示有效像素） + before_disp = np.full_like(unw_before, np.nan, dtype=np.float64) + before_disp[valid_mask] = unw_before[valid_mask] + corr_disp = np.full_like(correction, np.nan, dtype=np.float64) + corr_disp[valid_mask] = correction[valid_mask] + after_disp = np.full_like(unw_after, np.nan, dtype=np.float64) + after_disp[valid_mask] = unw_after[valid_mask] + + # 色标范围：使用有效数据的 P2/P98 + vals_b = unw_before[valid_mask] + vals_c = correction[valid_mask] + vals_a = unw_after[valid_mask] + vlim = max(abs(np.percentile(vals_b, 2)), abs(np.percentile(vals_b, 98)), 0.5) + vlim_c = max(abs(np.percentile(vals_c, 2)), abs(np.percentile(vals_c, 98)), 0.5) + vlim_a = max(abs(np.percentile(vals_a, 2)), abs(np.percentile(vals_a, 98)), 0.5) + + # 地理坐标范围 + w = dem_info['width'] + h = dem_info['nlines'] + extent = [dem_info['corner_lon'], + dem_info['corner_lon'] + w * dem_info['post_lon'], + dem_info['corner_lat'] + h * dem_info['post_lat'], + dem_info['corner_lat']] + + # 降采样显示 + step = max(1, h // 2000) + b_s = before_disp[::step, ::step] + c_s = corr_disp[::step, ::step] + a_s = after_disp[::step, ::step] + + # 使用带 NaN 灰色背景的 colormap + cmap_phase = plt.cm.RdBu_r.copy() + cmap_phase.set_bad(color='#E0E0E0') + cmap_corr = plt.cm.coolwarm.copy() + cmap_corr.set_bad(color='#E0E0E0') + + fig, axes = plt.subplots(1, 3, figsize=(18, 6), dpi=150) + for ax in axes: + ax.set_facecolor('#E0E0E0') + + im1 = axes[0].imshow(b_s, cmap=cmap_phase, vmin=-vlim, vmax=vlim, + extent=extent, aspect='auto', interpolation='nearest') + axes[0].set_title(f'Unwrapped Phase (std={std_before:.2f} rad)', fontsize=11, fontweight='bold') + plt.colorbar(im1, ax=axes[0], label='rad', shrink=0.8) + + im2 = axes[1].imshow(c_s, cmap=cmap_corr, vmin=-vlim_c, vmax=vlim_c, + extent=extent, aspect='auto', interpolation='nearest') + axes[1].set_title('GACOS Correction', fontsize=11, fontweight='bold') + plt.colorbar(im2, ax=axes[1], label='rad', shrink=0.8) + + im3 = axes[2].imshow(a_s, cmap=cmap_phase, vmin=-vlim_a, vmax=vlim_a, + extent=extent, aspect='auto', interpolation='nearest') + axes[2].set_title(f'Corrected (std={std_after:.2f} rad)', fontsize=11, fontweight='bold') + plt.colorbar(im3, ax=axes[2], label='rad', shrink=0.8) + + for ax in axes: + ax.set_xlabel('Longitude') + ax.set_ylabel('Latitude') + ax.tick_params(labelsize=9) + + reduction = (1 - std_after / std_before) * 100 if std_before > 0 else 0 + n_valid = np.sum(valid_mask) + if pair_str: + m, s = pair_str.split('-') + fig.suptitle(f'GACOS: {m}-{s} | valid={n_valid:,} px | std {reduction:+.1f}%', + fontsize=13, fontweight='bold', y=1.02) + plt.tight_layout() + fig.savefig(str(out_path), dpi=150, bbox_inches='tight', + facecolor='white', edgecolor='none') + plt.close(fig) + print(f" 对比图: {out_path}") + + +def find_existing_ztd(date, gacos_dir): + """ + Find existing GACOS ZTD file for a specific date. + + Returns: + -------- + ztd_file : str or None + Path to ZTD file (without extension), or None if not found + """ + # Check for .ztd + .ztd.rsc format (preferred) + ztd_pattern = os.path.join(gacos_dir, "**", date + "*.ztd") + ztd_files = glob.glob(ztd_pattern, recursive=True) + if ztd_files: + for ztd_file in ztd_files: + if ztd_file.endswith('.ztd.tif'): + continue + rsc_file = ztd_file + '.rsc' + if os.path.exists(rsc_file): + return ztd_file + + # Check for .ztd.tif format, auto-convert if needed + ztd_pattern = os.path.join(gacos_dir, "**", date + "*.ztd.tif") + tif_files = glob.glob(ztd_pattern, recursive=True) + if tif_files: + tif_file = tif_files[0] + ztd_bin = tif_file.replace('.ztd.tif', '.ztd') + rsc_file = tif_file.replace('.ztd.tif', '.ztd.rsc') + if not os.path.exists(ztd_bin) or not os.path.exists(rsc_file): + print(f" Auto-converting {os.path.basename(tif_file)} -> .ztd + .rsc") + convert_ztd_tif_to_binary(tif_file) + if os.path.exists(ztd_bin) and os.path.exists(rsc_file): + return ztd_bin + + return None + + +def submit_gacos_request(dates, bounds, email, gacos_dir, acquisition_time=None): + """ + Submit GACOS data request for multiple dates. + + Parameters: + ----------- + dates : list + List of acquisition dates in YYYYMMDD format + bounds : tuple + Bounding box (West, South, East, North) + email : str + Email address for GACOS submission + gacos_dir : str + Directory to save GACOS data + acquisition_time : str, optional + Acquisition time in HH:MM format + + Returns: + -------- + success : bool + True if submission successful + """ + if not AUTOGACOS_AVAILABLE: + print("AutoGACOS module not available. Please download GACOS data manually.") + return False + + try: + from gacos import Submitter, SarDataset + import pandas as pd + + # Create datetime index + if acquisition_time: + hour, minute = map(int, acquisition_time.split(':')) + else: + hour, minute = 10, 0 # Sentinel-1 升轨默认采集时间 ~10:00 UTC + + date_strings = [f"{d} {hour:02d}:{minute:02d}:00" for d in dates] + date_times = pd.to_datetime(date_strings) + + # Create dataset + dataset = SarDataset(bounds, date_times, gacos_dir) + + print(f"\nSubmitting GACOS request for {len(dates)} dates...") + print(f"Bounds: W={bounds[0]:.4f}, S={bounds[1]:.4f}, E={bounds[2]:.4f}, N={bounds[3]:.4f}") + print(f"Acquisition time: {hour:02d}:{minute:02d} UTC") + print(f"Email: {email}") + + # Submit request + submitter = Submitter(dataset, email) + submitter.post_requests() + + if submitter.succeed: + print(f"\nSuccessfully submitted {len(submitter.succeed)} requests.") + if submitter.failed: + print(f"\nFailed to submit {len(submitter.failed)} requests.") + + return len(submitter.succeed) > 0 + + except Exception as e: + print(f"Error in GACOS submission: {e}") + import traceback + traceback.print_exc() + return False + + +def download_gacos_from_email(email_config, gacos_dir, bounds=None, times=None, + submit_time=None): + """ + 从邮箱检索 gacos2017@foxmail.com 发来的 GACOS 下载链接并下载数据。 + + Parameters: + ----------- + email_config : dict + 邮箱配置: username, password, host, port, ssl + gacos_dir : str + GACOS 数据保存目录 + bounds : tuple, optional + 边界框过滤 (West, South, East, North) + times : list, optional + 采集时间过滤 + submit_time : str, optional + 提交时间（YYYY-MM-DD HH:MM:SS），仅检索此时间之后的邮件 + + Returns: + -------- + downloaded_files : list + 已下载的 ZTD 文件路径列表 + """ + if not AUTOGACOS_AVAILABLE: + print("AutoGACOS module not available.") + return [] + + try: + from gacos import Downloader, GACOSEmail + + # Step 1: 从邮箱检索 GACOS 下载链接 + print(" 检索 gacos2017@foxmail.com 邮件中的下载链接...") + + email_retriever = GACOSEmail( + username=email_config.get('username'), + password=email_config.get('password'), + host=email_config.get('host', 'imap.163.com'), + port=email_config.get('port'), + email_protocol=email_config.get('protocol', 'imap'), + ssl=email_config.get('ssl', True), + gacos_email='gacos2017@foxmail.com', + start_date=submit_time + ) + + url_file = os.path.join(gacos_dir, 'gacos_urls.csv') + email_retriever.retrieve_gacos_urls(url_file) + + if not os.path.exists(url_file): + print(" 未找到 GACOS 下载链接") + return [] + + # 检查 CSV 是否有内容 + import pandas as pd + try: + df = pd.read_csv(url_file) + if len(df) == 0: + print(" 邮件中未发现新的 GACOS 数据链接") + return [] + print(f" 找到 {len(df)} 个下载链接") + except Exception: + print(" URL 文件解析失败") + return [] + + # Step 2: 下载文件 + print(" 下载 GACOS 数据...") + + dl = Downloader( + url_file=url_file, + output_dir=gacos_dir, + bounds=bounds, + times=times, + keep_original=False + ) + + dl.download() + + # Step 3: 将下载的 .ztd.tif 转换为 .ztd + .rsc + downloaded = [] + for tif_file in glob.glob(os.path.join(gacos_dir, "*.ztd.tif")): + rsc_file = tif_file.replace('.ztd.tif', '.ztd.rsc') + if not os.path.exists(rsc_file): + ztd_file = convert_ztd_tif_to_binary(tif_file) + if ztd_file: + downloaded.append(ztd_file) + print(f" 转换: {os.path.basename(tif_file)} → .ztd + .rsc") + else: + downloaded.append(tif_file.replace('.ztd.tif', '.ztd')) + + return downloaded + + except Exception as e: + print(f" 邮箱检索/下载出错: {e}") + import traceback + traceback.print_exc() + return [] + + +def auto_gacos_workflow(dates, bounds, gacos_dir, email, email_config=None, + acquisition_time=None, wait_for_email=False, max_wait_hours=24, + check_interval=60): + """ + Complete automatic GACOS workflow: submit request, wait, and download. + + Parameters: + ----------- + dates : list + List of acquisition dates in YYYYMMDD format + bounds : tuple + Bounding box (West, South, East, North) + gacos_dir : str + Directory to save GACOS data + email : str + Email address for GACOS submission + email_config : dict, optional + Email configuration for downloading (username, password, host, etc.) + acquisition_time : str, optional + Acquisition time in HH:MM format + wait_for_email : bool, optional + Whether to wait for email notification before downloading + max_wait_hours : int, optional + Maximum hours to wait for email + + Returns: + -------- + ztd_files : dict + Dictionary mapping dates to ZTD file paths + """ + import time as time_module + from datetime import datetime, timedelta + + ztd_files = {} + missing_dates = [] + + # Step 1: Check existing files + print("\n" + "="*60) + print("Step 1: Checking existing GACOS data...") + print("="*60) + + for date in dates: + ztd_file = find_existing_ztd(date, gacos_dir) + if ztd_file: + ztd_files[date] = ztd_file + else: + missing_dates.append(date) + + if not missing_dates: + print("\nAll required GACOS data already exists!") + return ztd_files + + print(f"\nMissing GACOS data for {len(missing_dates)} dates:") + for d in missing_dates: + print(f" - {d}") + + # Step 2: Submit request for missing dates + print("\n" + "="*60) + print("Step 2: Submitting GACOS requests...") + print("="*60) + + submit_success = submit_gacos_request( + dates=missing_dates, + bounds=bounds, + email=email, + gacos_dir=gacos_dir, + acquisition_time=acquisition_time + ) + + if not submit_success: + print("\n提交失败（可能是重复提交或服务器繁忙），尝试从邮箱获取已有数据...") + else: + print("\nGACOS request submitted successfully!") + print("You will receive an email with download links when data is ready.") + + # Step 3: Wait and download from email (if configured) + if email_config and wait_for_email: + print("\n" + "="*60) + print("Step 3: 检查 gacos2017@foxmail.com 邮件并下载...") + print("="*60) + + # 记录时间基准，仅检索此时间之后的 GACOS 邮件，过滤掉旧邮件 + if submit_success: + submit_time_str = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + else: + # 提交失败（重复提交），向前回溯 1 小时查找已有邮件 + submit_time_str = (datetime.now() - timedelta(hours=1)).strftime('%Y-%m-%d %H:%M:%S') + print(f"邮件过滤起始时间: {submit_time_str}") + + start_time = datetime.now() + # 提交成功则等 3 分钟再检查；提交失败则立即检查邮箱 + first_wait = 180 if submit_success else 5 + retry_wait = 120 # 之后每 2 分钟 + attempt = 0 + + # 首次等待 + print(f"\n提交完成，等待 {first_wait//60} 分钟后首次检查邮箱...") + time_module.sleep(first_wait) + + while (datetime.now() - start_time) < timedelta(hours=max_wait_hours): + attempt += 1 + elapsed = datetime.now() - start_time + elapsed_min = int(elapsed.total_seconds() // 60) + elapsed_sec = int(elapsed.total_seconds() % 60) + print(f"\n[第 {attempt} 次检查] 已等待 {elapsed_min}m{elapsed_sec}s") + + downloaded = download_gacos_from_email( + email_config=email_config, + gacos_dir=gacos_dir, + bounds=bounds, + submit_time=submit_time_str + ) + + # 检查缺失日期是否已下载 + for date in missing_dates[:]: + ztd_file = find_existing_ztd(date, gacos_dir) + if ztd_file: + ztd_files[date] = ztd_file + missing_dates.remove(date) + print(f" ✓ 已获取: {date}") + + if not missing_dates: + print(f"\n所有 {len(dates)} 个日期的 GACOS 数据下载完成！") + break + + print(f" 仍缺 {len(missing_dates)} 个日期，{retry_wait//60} 分钟后重试...") + time_module.sleep(retry_wait) + + if missing_dates: + print(f"\n超时！仍缺 {len(missing_dates)} 个日期的 GACOS 数据。") + + return ztd_files + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Apply GACOS atmospheric correction to GAMMA interferograms.', + formatter_class=argparse.RawTextHelpFormatter, + epilog=INTRODUCTION + '\n' + EXAMPLE) + + parser.add_argument('projectName', help='projectName for processing.') + parser.add_argument('pair', help='Master-Slave, e.g., 20150101-20150106.') + parser.add_argument('--ztd-dir', dest='ztdDir', help='Directory containing GACOS ZTD files.') + parser.add_argument('--ztd1', dest='ztd1', help='GACOS ZTD file for master date.') + parser.add_argument('--ztd2', dest='ztd2', help='GACOS ZTD file for slave date.') + parser.add_argument('--email', dest='email', help='Email for GACOS auto-download.') + parser.add_argument('--wavelength', dest='wavelength', type=float, help='Radar wavelength in meters.') + + # Email configuration for auto-download + parser.add_argument('--email-user', dest='emailUser', help='Email username for downloading.') + parser.add_argument('--email-pass', dest='emailPass', help='Email password for downloading.') + parser.add_argument('--email-host', dest='emailHost', default='imap.gmail.com', help='Email IMAP host.') + parser.add_argument('--email-port', dest='emailPort', type=int, default=993, help='Email IMAP port.') + parser.add_argument('--email-ssl', dest='emailSsl', action='store_true', default=True, help='Use SSL for email.') + + # Auto-download options + parser.add_argument('--auto-download', dest='autoDownload', action='store_true', + help='Automatically submit request and download.') + parser.add_argument('--wait-hours', dest='waitHours', type=int, default=24, + help='Maximum hours to wait for email notification.') + + inps = parser.parse_args() + return inps + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Pair = inps.pair + + # Parse dates + Mdate = ut.yyyymmdd(Pair.split('-')[0]) + Sdate = ut.yyyymmdd(Pair.split('-')[1]) + + # Setup directories + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict = ut.update_template(templateFile) + + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + + ifgDir = scratchDir + '/' + projectName + "/ifgrams" + demDir = scratchDir + '/' + projectName + "/DEM" + slcDir = scratchDir + '/' + projectName + "/SLC" + workDir = ifgDir + '/' + Pair + + # GACOS data directory + if inps.ztdDir: + gacosDir = inps.ztdDir + else: + gacosDir = scratchDir + '/' + projectName + "/GACOS" + + if not os.path.exists(gacosDir): + os.makedirs(gacosDir) + + # Define input files + dempar = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem.par' + slcpar = slcDir + '/' + masterDate + '/' + masterDate + '.slc.par' + geo_unw = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.unw' + out_file = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.unw.gacos' + figdir = scratchDir + '/' + projectName + "/figure" + + # Check if geocoded interferogram exists + if not os.path.isfile(geo_unw): + print(f"Error: Geocoded interferogram not found: {geo_unw}") + print("Please run geocode_gamma.py first.") + sys.exit(1) + + print("="*60) + print("GACOS Atmospheric Correction (Pure Python)") + print(f"Project: {projectName}") + print(f"Pair: {Pair}") + print("="*60) + + # Step 1: Find GACOS ZTD files + print("\nStep 1: Checking GACOS ZTD files...") + + ztd1 = None + ztd2 = None + + if inps.ztd1 and inps.ztd2: + ztd1 = inps.ztd1 + ztd2 = inps.ztd2 + else: + ztd1 = find_existing_ztd(Mdate, gacosDir) + ztd2 = find_existing_ztd(Sdate, gacosDir) + + if ztd1 and ztd2: + print(f" Master ZTD: {ztd1}") + print(f" Slave ZTD: {ztd2}") + elif inps.email: + # Auto-download workflow (existing logic preserved) + dem_info = parse_dem_par(dempar) + North = dem_info['corner_lat'] + West = dem_info['corner_lon'] + South = North + (dem_info['nlines'] - 1) * dem_info['post_lat'] + East = West + (dem_info['width'] - 1) * dem_info['post_lon'] + bounds = (West, South, East, North) + + missing_dates = [] + if not ztd1: missing_dates.append(Mdate) + if not ztd2: missing_dates.append(Sdate) + + if inps.autoDownload: + email_config = None + if inps.emailUser and inps.emailPass: + email_config = { + 'username': inps.emailUser, 'password': inps.emailPass, + 'host': inps.emailHost, 'port': inps.emailPort, 'ssl': inps.emailSsl + } + ztd_files = auto_gacos_workflow( + dates=missing_dates, bounds=bounds, gacos_dir=gacosDir, + email=inps.email, email_config=email_config, + wait_for_email=(email_config is not None), max_wait_hours=inps.waitHours + ) + ztd1 = ztd1 or ztd_files.get(Mdate) + ztd2 = ztd2 or ztd_files.get(Sdate) + else: + submit_gacos_request(dates=missing_dates, bounds=bounds, + email=inps.email, gacos_dir=gacosDir) + print(f"\nGACOS request submitted for: {missing_dates}") + sys.exit(0) + + if ztd1 is None or ztd2 is None: + print("\nError: GACOS ZTD files not found.") + print(f" Missing: {Mdate if not ztd1 else ''} {Sdate if not ztd2 else ''}") + print(f" Search dir: {gacosDir}") + sys.exit(1) + + # Step 2: Apply GACOS correction (Pure Python, no GMT dependency) + print("\nStep 2: Applying GACOS correction...") + + ztd1_rsc = ztd1 + '.rsc' + ztd2_rsc = ztd2 + '.rsc' + + corrected_file = apply_gacos_correction_python( + geo_unw_file=geo_unw, + ztd1_file=ztd1, ztd1_rsc=ztd1_rsc, + ztd2_file=ztd2, ztd2_rsc=ztd2_rsc, + dempar=dempar, + slcpar=slcpar, + wavelength=inps.wavelength, + out_file=out_file, + figdir=figdir, + pair_str=Pair + ) + + if corrected_file: + # Step 3: Generate BMP preview using rasdt_pwr + print("\nStep 3: Generating BMP preview...") + geo_amp = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.amp' + nWidthUTMDEM = str(parse_dem_par(dempar)['width']) + + if os.path.isfile(geo_amp): + call_str = ('rasdt_pwr ' + corrected_file + ' ' + geo_amp + ' ' + + nWidthUTMDEM + ' - - - - -3.14 3.14 1 rmg.cm') + print(f" {call_str}") + os.system(call_str) + bmp_file = corrected_file + '.bmp' + if os.path.isfile(bmp_file): + print(f" BMP: {bmp_file}") + else: + print(" Warning: BMP file not generated") + else: + print(f" Warning: geo_amp not found: {geo_amp}") + + print(f"\nGACOS atmospheric correction completed successfully!") + else: + print("\nGACOS correction failed!") + sys.exit(1) + + sys.exit(0) + + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/gacos_gamma_all.py b/third_party/PyINT/pyint/gacos_gamma_all.py new file mode 100644 index 0000000..f1ff747 --- /dev/null +++ b/third_party/PyINT/pyint/gacos_gamma_all.py @@ -0,0 +1,494 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import time +import glob +import argparse +import subprocess + +from pyint import _utils as ut + + +def work(data0): + """Worker function for parallel processing""" + cmd = data0[0] + err_file = data0[1] + p = subprocess.run(cmd, shell=False, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa = stderr.decode("utf-8") + else: + aa = stderr + + if aa: + print(aa) + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + with open(err_file, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Apply GACOS atmospheric correction to all interferograms + for one project using GAMMA. + + This script processes all interferogram pairs listed in + ifgram_list.txt and applies GACOS tropospheric correction. +------------------------------------------------------------------- +''' + +EXAMPLE = ''' + Usage: + gacos_gamma_all.py projectName + gacos_gamma_all.py projectName --parallel 4 + gacos_gamma_all.py projectName --parallel 4 --ztd-dir /path/to/gacos + gacos_gamma_all.py projectName --parallel 4 --ifgramList-txt /test/ifgram_list.txt +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Apply GACOS atmospheric correction to all interferograms for one project.', + formatter_class=argparse.RawTextHelpFormatter, + epilog=INTRODUCTION + '\n' + EXAMPLE) + + parser.add_argument('projectName', help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, + help='Enable parallel processing and Specify the number of processors.') + parser.add_argument('--ifgarmList-txt', dest='ifgarmListTxt', + help='provided ifgram_list_txt. default: using ifgram_list.txt under projectName folder.') + parser.add_argument('--ztd-dir', dest='ztdDir', + help='Directory containing GACOS ZTD files. Default: projectName/GACOS') + parser.add_argument('--email', dest='email', + help='Email for GACOS auto-download.') + parser.add_argument('--auto-download', dest='autoDownload', action='store_true', + help='Automatically submit and download missing GACOS data.') + parser.add_argument('--email-user', dest='emailUser', help='Email username for IMAP download.') + parser.add_argument('--email-pass', dest='emailPass', help='Email password for IMAP download.') + parser.add_argument('--email-host', dest='emailHost', default='imap.gmail.com', help='IMAP host.') + parser.add_argument('--email-port', dest='emailPort', type=int, default=None, help='IMAP port.') + parser.add_argument('--email-ssl', dest='emailSsl', action='store_true', default=True, help='Use SSL.') + parser.add_argument('--wait-hours', dest='waitHours', type=int, default=24, + help='Max hours to wait for GACOS email.') + parser.add_argument('--skip-existing', dest='skipExisting', action='store_true', + help='Skip pairs that already have corrected output.') + + inps = parser.parse_args() + return inps + + +def check_gacos_data_availability(dates, gacos_dir, bounds): + """ + Check if GACOS ZTD data is available for all dates. + + Parameters: + ----------- + dates : list + List of acquisition dates (YYYYMMDD format) + gacos_dir : str + Directory containing GACOS data + bounds : tuple + Bounding box (West, South, East, North) + + Returns: + -------- + missing_dates : list + List of dates that don't have GACOS data + """ + missing_dates = [] + + for date in dates: + # Check for ZTD files + ztd_pattern = os.path.join(gacos_dir, date + "*.ztd.tif") + ztd_files = glob.glob(ztd_pattern) + + if not ztd_files: + # Also check for .ztd format + ztd_pattern = os.path.join(gacos_dir, date + "*.ztd") + ztd_files = glob.glob(ztd_pattern) + + if not ztd_files: + missing_dates.append(date) + + return missing_dates + + +def get_project_dates(projectName, scratchDir): + """ + Get all acquisition dates from the project. + + Returns: + -------- + dates : list + List of unique acquisition dates + """ + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + + dates = set() + + # Check SLC directory + if os.path.exists(slcDir): + for item in os.listdir(slcDir): + if os.path.isdir(os.path.join(slcDir, item)): + if len(item) == 8 and item.isdigit(): + dates.add(item) + + # Check RSLC directory + if os.path.exists(rslcDir): + for item in os.listdir(rslcDir): + if os.path.isdir(os.path.join(rslcDir, item)): + if len(item) == 8 and item.isdigit(): + dates.add(item) + + return sorted(list(dates)) + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + projectDir = scratchDir + '/' + projectName + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + + templateDict = ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + + # GACOS data directory (CLI > template > default) + if inps.ztdDir: + gacosDir = inps.ztdDir + elif templateDict.get('gacos_dir', '').strip(): + gacosDir = templateDict['gacos_dir'].strip() + else: + gacosDir = projectDir + '/GACOS' + + if not os.path.exists(gacosDir): + os.makedirs(gacosDir) + print(f"Created GACOS directory: {gacosDir}") + + # 从模板读取邮箱配置（CLI 优先于模板） + gacos_email = (inps.email or templateDict.get('gacos_email', '').strip()) or None + gacos_email_user = (getattr(inps, 'emailUser', None) or templateDict.get('gacos_email_user', '').strip()) or None + gacos_email_pass = (getattr(inps, 'emailPass', None) or templateDict.get('gacos_email_pass', '').strip()) or None + gacos_email_host = (getattr(inps, 'emailHost', None) or templateDict.get('gacos_email_host', 'imap.163.com').strip()) + gacos_email_port = getattr(inps, 'emailPort', None) or int(templateDict.get('gacos_email_port', '993').strip()) + gacos_email_ssl = templateDict.get('gacos_email_ssl', '1').strip() == '1' + gacos_check_interval = int(templateDict.get('gacos_check_interval', '60').strip()) + + # 如果有用户名和密码，自动启用 auto-download + auto_download = getattr(inps, 'autoDownload', False) or (gacos_email_user and gacos_email_pass) + + # Key files + demDir = projectDir + '/DEM' + slcDir = projectDir + '/SLC' + dempar = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem.par' + slcpar = slcDir + '/' + masterDate + '/' + masterDate + '.slc.par' + figdir = projectDir + '/figure' + + # Import pure Python correction functions from gacos_gamma + from pyint.gacos_gamma import (parse_dem_par, resample_ztd_to_dem, + find_existing_ztd, apply_gacos_correction_python, + submit_gacos_request, auto_gacos_workflow) + + # Get interferogram list + if inps.ifgarmListTxt: + ifgramList_txt = inps.ifgarmListTxt + else: + ifgramList_txt = projectDir + '/ifgram_list.txt' + + if not os.path.isfile(ifgramList_txt): + print(f"Error: Interferogram list not found: {ifgramList_txt}") + sys.exit(1) + + ifgList0 = ut.read_txt2array(ifgramList_txt) + + import numpy as np + ifgList0 = np.atleast_2d(np.array(ifgList0)) + ifgList = list(ifgList0[:, 0]) + + # Collect all unique dates + all_dates = set() + for ifg in ifgList: + mdate = ut.yyyymmdd(ifg.split('-')[0]) + sdate = ut.yyyymmdd(ifg.split('-')[1]) + all_dates.add(mdate) + all_dates.add(sdate) + all_dates = sorted(list(all_dates)) + + # Read DEM parameters + dem_info = parse_dem_par(dempar) + + print("="*60) + print("GACOS Atmospheric Correction (Pure Python, ZTD Pre-cache)") + print(f"Project: {projectName}") + print(f"Interferograms: {len(ifgList)}, Dates: {len(all_dates)}") + print(f"DEM: {dem_info['width']}×{dem_info['nlines']}") + print(f"GACOS dir: {gacosDir}") + print("="*60) + + # Check GACOS data availability using updated find_existing_ztd + print("\nChecking GACOS data availability...") + missing_dates = [] + ztd_paths = {} # {date: ztd_file_path} + for d in all_dates: + ztd = find_existing_ztd(d, gacosDir) + if ztd: + ztd_paths[d] = ztd + else: + missing_dates.append(d) + + if missing_dates: + print(f"\n⚠ GACOS data missing for {len(missing_dates)} dates: {missing_dates[:5]}...") + print(f"ZTD available: {len(ztd_paths)}/{len(all_dates)} dates") + + # 自动提交/下载缺失的 GACOS 数据 + if gacos_email: + # 从 DEM 参数计算研究区边界框 + North = dem_info['corner_lat'] + West = dem_info['corner_lon'] + South = North + (dem_info['nlines'] - 1) * dem_info['post_lat'] + East = West + (dem_info['width'] - 1) * dem_info['post_lon'] + bounds = (West, South, East, North) + + # 获取 SAR 采集时间（GACOS 要求非零） + acq_time = templateDict.get('gacos_acq_time', '').strip() + if not acq_time and os.path.isfile(slcpar): + try: + with open(slcpar) as f: + for line in f: + if line.startswith('center_time:'): + secs = float(line.split()[1]) + h = int(secs // 3600) + m = int((secs % 3600) // 60) + acq_time = f"{h:02d}:{m:02d}" + print(f"从 SLC par 读取采集时间: {acq_time} UTC") + break + except Exception: + pass + if not acq_time: + acq_time = '10:00' + print(f"使用默认采集时间: {acq_time} UTC") + + if auto_download and gacos_email_user and gacos_email_pass: + # 完整自动流程：提交 → 等待邮件 → 下载 + email_config = { + 'username': gacos_email_user, 'password': gacos_email_pass, + 'host': gacos_email_host, 'port': gacos_email_port, 'ssl': gacos_email_ssl + } + wait_hours = getattr(inps, 'waitHours', 24) + ztd_files = auto_gacos_workflow( + dates=missing_dates, bounds=bounds, gacos_dir=gacosDir, + email=gacos_email, email_config=email_config, + acquisition_time=acq_time, + wait_for_email=True, max_wait_hours=wait_hours, + check_interval=gacos_check_interval + ) + # 重新检查已下载的日期 + for d in missing_dates[:]: + ztd = find_existing_ztd(d, gacosDir) + if ztd: + ztd_paths[d] = ztd + missing_dates.remove(d) + else: + # 仅提交请求，不等待下载 + submit_gacos_request( + dates=missing_dates, bounds=bounds, + email=gacos_email, gacos_dir=gacosDir, + acquisition_time=acq_time + ) + print(f"\nGACOS request submitted for {len(missing_dates)} dates.") + print("Please wait for email notification, then re-run this script.") + if not ztd_paths: + print("No ZTD data available yet. Exiting.") + sys.exit(0) + else: + print("\nTip: 在模板中配置 gacos_email 可自动提交 GACOS 请求") + print(" 配置 gacos_email_user + gacos_email_pass 可自动轮询下载") + + if missing_dates: + print(f"\n仍有 {len(missing_dates)} 个日期缺失 ZTD，相关干涉对将被跳过") + + print(f"\nZTD available: {len(ztd_paths)}/{len(all_dates)} dates") + + # Pre-cache all ZTD resampled to DEM grid (each date only once) + print(f"\nPre-loading {len(ztd_paths)} ZTD files to DEM grid...") + t0 = time.time() + ztd_cache = {} + for i, (d, ztd_file) in enumerate(sorted(ztd_paths.items())): + ztd_rsc = ztd_file + '.rsc' + ztd_cache[d] = resample_ztd_to_dem(ztd_file, ztd_rsc, dem_info) + if (i + 1) % 10 == 0 or i == 0: + print(f" [{i+1}/{len(ztd_paths)}] {d}") + t_cache = time.time() - t0 + mem_gb = sum(v.nbytes for v in ztd_cache.values()) / 1024**3 + print(f" Pre-load done: {t_cache:.1f}s, ~{mem_gb:.2f} GB") + + # Read incidence angle and wavelength from SLC par + import numpy as np + inc_angle_deg = 39.0 + wavelength = 0.0554657595 + if os.path.isfile(slcpar): + with open(slcpar) as f: + for line in f: + if line.startswith('incidence_angle:'): + inc_angle_deg = float(line.split(':')[1].strip().split()[0]) + if line.startswith('radar_frequency:'): + freq = float(line.split(':')[1].strip().split()[0]) + wavelength = 299792458.0 / freq + cos_inc = np.cos(np.radians(inc_angle_deg)) + ztd_to_phase = 4.0 * np.pi / wavelength + + print(f"\n入射角: {inc_angle_deg:.4f}°, 波长: {wavelength:.10f} m") + + # Process each pair + print(f"\nProcessing {len(ifgList)} interferograms...\n") + + ok_count = 0 + skip_count = 0 + err_count = 0 + errors = [] + plot_count = 0 + MAX_PLOTS = 5 # only plot first 5 pairs + + for i, ifg in enumerate(ifgList): + mdate = ut.yyyymmdd(ifg.split('-')[0]) + sdate = ut.yyyymmdd(ifg.split('-')[1]) + pair = mdate + '-' + sdate + + workDir = ifgDir + '/' + pair + unw_file = workDir + '/geo_' + pair + '_' + rlks + 'rlks.diff_filt.unw' + out_file = workDir + '/geo_' + pair + '_' + rlks + 'rlks.diff_filt.unw.gacos' + + # Skip if already exists + if inps.skipExisting and os.path.isfile(out_file) and os.path.getsize(out_file) > 0: + skip_count += 1 + continue + + # Skip if input missing + if not os.path.isfile(unw_file): + err_count += 1 + errors.append(f"{pair}: geo_unw not found") + continue + + # Skip if ZTD missing + if mdate not in ztd_cache or sdate not in ztd_cache: + err_count += 1 + errors.append(f"{pair}: ZTD missing for {mdate if mdate not in ztd_cache else sdate}") + continue + + try: + # Read unwrapped phase + unw_data = np.fromfile(unw_file, dtype=np.float32).reshape( + dem_info['nlines'], dem_info['width']) + + # Compute phase correction from cached ZTD + dztd = ztd_cache[sdate] - ztd_cache[mdate] + phase_correction = dztd * ztd_to_phase / cos_inc + + # Valid pixel mask (exclude zeros, NaN, GAMMA artifacts/unwrap errors) + PHASE_THRESH = 1000.0 + valid = ((unw_data != 0) + & np.isfinite(unw_data) + & (np.abs(unw_data) < PHASE_THRESH) + & np.isfinite(phase_correction)) + n_valid = np.sum(valid) + + if n_valid == 0: + err_count += 1 + errors.append(f"{pair}: no valid pixels") + continue + + std_before = float(np.std(unw_data[valid])) + + # Apply correction (preserve original data, only modify valid pixels) + corrected = unw_data.copy() + corrected[valid] = unw_data[valid] - phase_correction[valid] + + # Demean (only valid pixels) + mean_val = np.mean(corrected[valid]) + corrected[valid] -= mean_val + + std_after = float(np.std(corrected[valid])) + + # Save + corrected.astype(np.float32).tofile(out_file) + + # Generate BMP preview using rasdt_pwr + geo_amp = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.amp' + if os.path.isfile(geo_amp): + call_str = ('rasdt_pwr ' + out_file + ' ' + geo_amp + ' ' + + str(dem_info['width']) + + ' - - - - -3.14 3.14 1 rmg.cm') + os.system(call_str) + + reduction = (1 - std_after / std_before) * 100 if std_before > 0 else 0 + ok_count += 1 + + # Plot first N pairs + do_plot = (plot_count < MAX_PLOTS) + if do_plot: + from pyint.gacos_gamma import plot_gacos_comparison + plot_gacos_comparison(unw_data, phase_correction, corrected, + valid, dem_info, std_before, std_after, + pair, figdir) + plot_count += 1 + + if ok_count <= 5 or ok_count % 50 == 0: + print(f" [{i+1}/{len(ifgList)}] {pair}: ✅ std {std_before:.2f}->{std_after:.2f} ({reduction:.1f}%)") + + except Exception as e: + err_count += 1 + errors.append(f"{pair}: {str(e)}") + if err_count <= 3: + print(f" [{i+1}/{len(ifgList)}] {pair}: ❌ {e}") + + elapsed = time.time() - start_time + + # Summary + print("\n" + "="*60) + print("GACOS Atmospheric Correction Summary") + print("="*60) + print(f" Total: {len(ifgList)}") + print(f" Success: {ok_count}") + print(f" Skipped: {skip_count}") + print(f" Failed: {err_count}") + + if errors: + err_txt = projectDir + '/gacos_gamma_all.err' + with open(err_txt, 'w') as f: + for e in errors: + f.write(e + '\n') + print(f"\n Error log: {err_txt}") + for e in errors[:5]: + print(f" {e}") + + print(f"\n Time: {elapsed/60:.1f} min") + if plot_count > 0: + print(f" Comparison plots: {figdir}/GACOS_comparison_*.png") + print("="*60) + + sys.exit(0) + + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/gamma2licsbas_gamma.py b/third_party/PyINT/pyint/gamma2licsbas_gamma.py new file mode 100644 index 0000000..695f336 --- /dev/null +++ b/third_party/PyINT/pyint/gamma2licsbas_gamma.py @@ -0,0 +1,319 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import os +import sys +import argparse + +from pyint import _utils as ut + + +INTRODUCTION = ''' +------------------------------------------------------------------- + 将 geocode_gamma.py 已地理编码的 GAMMA 二进制产品转换为 + LiCSBAS 所需的 GeoTIFF 格式, 并组织到 GEOC 目录结构。 + + 本脚本 **不做任何地理编码或计算**, 仅执行格式转换: + 1. data2geotiff : GAMMA 二进制 (EQA) → GeoTIFF + 2. GMT grdmath : 从 look_vector 计算 E/N/U 分量 + 3. 文件复制 : 组织到 GEOC/{Pair}/ 目录 + + 所有地理编码产品由 geocode_gamma.py 提供: + amp, corr, unw, diff_filt, diff, dem, lv_theta, lv_phi +''' + +EXAMPLE = ''' + Usage: + gamma2licsbas_gamma.py projectName Mdate-Sdate + gamma2licsbas_gamma.py PacayaT163TsxHhA 20150102-20150601 +------------------------------------------------------------------- +''' + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Convert geocoded GAMMA binaries to LiCSBAS GeoTIFF format.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('pair',help='Master-Slave, e.g., 20150101-20150106.') + + inps = parser.parse_args() + return inps + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Pair = inps.pair + + scratchDir = os.getenv('SCRATCHDIR') + ifgDir = scratchDir + '/' + projectName + "/ifgrams" + demDir = scratchDir + '/' + projectName + "/DEM" + workDir = ifgDir + '/' + Pair + + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + + ################################################################# + # DEM 参数 (直接引用, 不复制不删除) + ################################################################# + DEM_par = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem.par' + DEM = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem' + DEM_tif = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem.tif' + + # look_vector 文件 (由 geocode_gamma.py 生成) + lv_theta = workDir + '/' + 'lv_theta' + lv_phi = workDir + '/' + 'lv_phi' + lv_theta_tif = workDir + '/' + 'lv_theta.geo.tif' + lv_phi_tif = workDir + '/' + 'lv_phi.geo.tif' + lv_theta_grd = workDir + '/' + 'lv_theta.grd' + lv_theta_grd_final = workDir + '/' + 'inc_deg.nc' + lv_phi_grd = workDir + '/' + 'lv_phi.grd' + lv_phi_grd_final = workDir + '/' + 'azi_deg.nc' + UE_grd = workDir + '/' + 'ue.grd' + UN_grd = workDir + '/' + 'un.grd' + UU_grd = workDir + '/' + 'uu.grd' + UE_tif = workDir + '/' + 'ue.geo.E.tif' + UN_tif = workDir + '/' + 'un.geo.N.tif' + UU_tif = workDir + '/' + 'uu.geo.U.tif' + + if not os.path.exists(lv_theta) or not os.path.exists(lv_phi): + print(f"WARNING: look_vector 文件不存在, 请先运行 geocode_gamma.py {projectName} {Pair}") + print(f" 缺少: lv_theta={os.path.exists(lv_theta)}, lv_phi={os.path.exists(lv_phi)}") + + GEOC = scratchDir + '/' + projectName + '/GEOC' + if not os.path.isdir(GEOC): + os.mkdir(GEOC) + Store_dir = GEOC + '/' + Pair + if not os.path.isdir(Store_dir): + os.mkdir(Store_dir) + + # 判断DEM_tif是否存在 + if not os.path.exists(DEM_tif): + call_str = 'data2geotiff ' + DEM_par + ' ' + DEM + ' 2 ' + DEM_tif + os.system(call_str) + else: + print(f"File {DEM_tif} already exists, skipping generation.") + + # 判断lv_theta_tif是否存在 + if not os.path.exists(lv_theta_tif): + call_str = 'data2geotiff ' + DEM_par + ' ' + lv_theta + ' 2 ' + lv_theta_tif + os.system(call_str) + else: + print(f"File {lv_theta_tif} already exists, skipping generation.") + + # 判断lv_phi_tif是否存在 + if not os.path.exists(lv_phi_tif): + call_str = 'data2geotiff ' + DEM_par + ' ' + lv_phi + ' 2 ' + lv_phi_tif + os.system(call_str) + else: + print(f"File {lv_phi_tif} already exists, skipping generation.") + + # 判断lv_theta_grd是否存在 + if not os.path.exists(lv_theta_grd): + call_str = 'gdal_translate -of GSBG ' + lv_theta_tif + ' ' + lv_theta_grd + os.system(call_str) + else: + print(f"File {lv_theta_grd} already exists, skipping conversion.") + + # 判断lv_phi_grd是否存在 + if not os.path.exists(lv_phi_grd): + call_str = 'gdal_translate -of GSBG ' + lv_phi_tif + ' ' + lv_phi_grd + os.system(call_str) + else: + print(f"File {lv_phi_grd} already exists, skipping conversion.") + + # 判断lv_theta_grd_final是否存在 + if not os.path.exists(lv_theta_grd_final): + call_str = 'gmt grdmath 90 ' + lv_theta_grd + ' 3.1415926 DIV 180 MUL SUB = ' + lv_theta_grd_final + os.system(call_str) + else: + print(f"File {lv_theta_grd_final} already exists, skipping calculation.") + + # 判断lv_phi_grd_final是否存在 + if not os.path.exists(lv_phi_grd_final): + call_str = 'gmt grdmath -180 ' + lv_phi_grd + ' 3.1415926 DIV 180 MUL SUB = ' + lv_phi_grd_final + os.system(call_str) + else: + print(f"File {lv_phi_grd_final} already exists, skipping calculation.") + + # 判断UE_grd是否存在 + if not os.path.exists(UE_grd): + call_str = 'gmt grdmath ' + lv_phi_grd_final + ' COSD ' + lv_theta_grd_final + ' SIND MUL NEG = ' + UE_grd + os.system(call_str) + else: + print(f"File {UE_grd} already exists, skipping calculation.") + + # 判断UN_grd是否存在 + if not os.path.exists(UN_grd): + call_str = 'gmt grdmath ' + lv_phi_grd_final + ' SIND ' + lv_theta_grd_final + ' SIND MUL = ' + UN_grd + os.system(call_str) + else: + print(f"File {UN_grd} already exists, skipping calculation.") + + # 判断UU_grd是否存在 + if not os.path.exists(UU_grd): + call_str = 'gmt grdmath ' + lv_theta_grd_final + ' COSD = ' + UU_grd + os.system(call_str) + else: + print(f"File {UU_grd} already exists, skipping calculation.") + + # 判断UE_tif是否存在 + if not os.path.exists(UE_tif): + call_str = 'gdal_translate -of GTiff ' + UE_grd + ' ' + UE_tif + os.system(call_str) + else: + print(f"File {UE_tif} already exists, skipping conversion.") + + # 判断UN_tif是否存在 + if not os.path.exists(UN_tif): + call_str = 'gdal_translate -of GTiff ' + UN_grd + ' ' + UN_tif + os.system(call_str) + else: + print(f"File {UN_tif} already exists, skipping conversion.") + + # 判断UU_tif是否存在 + if not os.path.exists(UU_tif): + call_str = 'gdal_translate -of GTiff ' + UU_grd + ' ' + UU_tif + os.system(call_str) + else: + print(f"File {UU_tif} already exists, skipping conversion.") + +##### get unw.tif + GeoMamp = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.amp' + GeoMamp_tif = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.amp.tif' + GeoCOR = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.diff_filt.cor' + GeoCOR_tif = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.cor.tif' + GeoUNW_tif = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.unw.tif' + GeoUNW = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.unw' + GeoDIFF = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt' + GeoDIFF_tif = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.tif' + Geodiff = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff' + Geodiff_tif = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff.tif' + GeoATMCOR_UNW = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.unw.gacos' + GeoATMCOR_UNW_tif = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.unw.gacos.tif' + Geodem = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.hgt' + Geodem_tif = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.hgt.tif' + + # 判断GeoUNW_tif是否存在 + if not os.path.exists(GeoUNW_tif): + call_str = 'data2geotiff ' + DEM_par + ' ' + GeoUNW + ' ' + ' 2 ' + GeoUNW_tif + os.system(call_str) + else: + print(f"File {GeoUNW_tif} already exists, skipping conversion.") + + # 判断Geodem_tif是否存在 + if not os.path.exists(Geodem_tif): + call_str = 'data2geotiff ' + DEM_par + ' ' + Geodem + ' ' + ' 2 ' + Geodem_tif + os.system(call_str) + else: + print(f"File {Geodem_tif} already exists, skipping conversion.") + + # 判断GeoCOR_tif是否存在 + if not os.path.exists(GeoCOR_tif): + call_str = 'data2geotiff ' + DEM_par + ' ' + GeoCOR + ' ' + ' 2 ' + GeoCOR_tif + os.system(call_str) + else: + print(f"File {GeoCOR_tif} already exists, skipping conversion.") + + # 判断GeoMamp_tif是否存在 + if not os.path.exists(GeoMamp_tif): + call_str = 'data2geotiff ' + DEM_par + ' ' + GeoMamp + ' ' + ' 2 ' + GeoMamp_tif + os.system(call_str) + else: + print(f"File {GeoMamp_tif} already exists, skipping conversion.") + + # 判断GeoDIFF_tif是否存在 + if not os.path.exists(GeoDIFF_tif): + call_str = 'data2geotiff ' + DEM_par + ' ' + GeoDIFF + ' ' + ' 2 ' + GeoDIFF_tif + os.system(call_str) + else: + print(f"File {GeoDIFF_tif} already exists, skipping conversion.") + + # 判断Geodiff_tif是否存在 + if not os.path.exists(Geodiff_tif): + call_str = 'data2geotiff ' + DEM_par + ' ' +Geodiff + ' ' + ' 2 ' + Geodiff_tif + os.system(call_str) + else: + print(f"File {Geodiff_tif} already exists, skipping conversion.") + + # 判断GeoATMCOR_UNW_tif是否存在（GACOS校正后的解缠结果） + if os.path.exists(GeoATMCOR_UNW) and not os.path.exists(GeoATMCOR_UNW_tif): + call_str = 'data2geotiff ' + DEM_par + ' ' + GeoATMCOR_UNW + ' ' + ' 2 ' + GeoATMCOR_UNW_tif + os.system(call_str) + elif os.path.exists(GeoATMCOR_UNW_tif): + print(f"File {GeoATMCOR_UNW_tif} already exists, skipping conversion.") + + GeoCOR_tif0 = Store_dir + '/' + Pair + '.geo.cc.tif' + GeoUNW_tif0 = Store_dir + '/' + Pair + '.geo.unw.tif' + UU_tif0 = GEOC + '/' + 'uu.geo.U.tif' + UE_tif0 = GEOC + '/' + 'ue.geo.E.tif' + UN_tif0 = GEOC + '/' + 'un.geo.N.tif' + dem_tif0 = GEOC + '/' + masterDate + '.geo.hgt.tif' + amp_tif0 = GEOC + '/' + masterDate + '.geo.mli.tif' + GeoDIFF_tif0 = Store_dir + '/' + Pair + '.geo.diff_pha.tif' + Geodiff_tif0 = Store_dir + '/' + Pair + '.geo.diff_unfiltered_pha.tif' + GeoATMCOR_UNW_tif0 = Store_dir + '/' + Pair + '.geo.unw.gacos.tif' + # 判断并复制文件，如果目标文件已存在则不复制 + if not os.path.exists(GeoUNW_tif0): + ut.copy_file(GeoUNW_tif, GeoUNW_tif0) + else: + print(f"File {GeoUNW_tif0} already exists, skipping copy.") + + if not os.path.exists(GeoCOR_tif0): + ut.copy_file(GeoCOR_tif, GeoCOR_tif0) + else: + print(f"File {GeoCOR_tif0} already exists, skipping copy.") + + if not os.path.exists(UU_tif0): + ut.copy_file(UU_tif, UU_tif0) + else: + print(f"File {UU_tif0} already exists, skipping copy.") + + if not os.path.exists(UE_tif0): + ut.copy_file(UE_tif, UE_tif0) + else: + print(f"File {UE_tif0} already exists, skipping copy.") + + if not os.path.exists(UN_tif0): + ut.copy_file(UN_tif, UN_tif0) + else: + print(f"File {UN_tif0} already exists, skipping copy.") + if not os.path.exists(GeoDIFF_tif0): + ut.copy_file(GeoDIFF_tif, GeoDIFF_tif0) + else: + print(f"File {GeoDIFF_tif0} already exists, skipping copy.") + if not os.path.exists(Geodiff_tif0): + ut.copy_file(Geodiff_tif, Geodiff_tif0) + else: + print(f"File {Geodiff_tif0} already exists, skipping copy.") + + if os.path.exists(GeoATMCOR_UNW_tif) and not os.path.exists(GeoATMCOR_UNW_tif0): + ut.copy_file(GeoATMCOR_UNW_tif, GeoATMCOR_UNW_tif0) + elif os.path.exists(GeoATMCOR_UNW_tif0): + print(f"File {GeoATMCOR_UNW_tif0} already exists, skipping copy.") + + if not os.path.exists(dem_tif0): + ut.copy_file(Geodem_tif, dem_tif0) + else: + print(f"File {dem_tif0} already exists, skipping copy.") + + if not os.path.exists(amp_tif0): + ut.copy_file(GeoMamp_tif, amp_tif0) + else: + print(f"File {amp_tif0} already exists, skipping copy.") + + print("Convert to LiCSBAS format is done!") + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/gamma2licsbas_gamma_all.py b/third_party/PyINT/pyint/gamma2licsbas_gamma_all.py new file mode 100644 index 0000000..26f8916 --- /dev/null +++ b/third_party/PyINT/pyint/gamma2licsbas_gamma_all.py @@ -0,0 +1,122 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + +def work(data0): + cmd = data0[0] + err_file = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if aa: + print(aa) + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + '\n' + with open(err_file, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + Geocode interferograms for one project using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + geocode_gamma_all.py projectName + geocode_gamma_all.py projectName --parallel 4 + geocode_gamma_all.py projectName --parallel 4 --ifgramList-txt /test/ifgram_list.txt +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Geocode interferograms for one project using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + parser.add_argument('--ifgarmList-txt', dest='ifgarmListTxt', help='provided ifgram_list_txt. default: using ifgram_list.txt under projectName folder.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + projectDir = scratchDir + '/' + projectName + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + GEOC = scratchDir + '/' + projectName + '/GEOC' + if not os.path.isdir(GEOC): + os.makedirs(GEOC, exist_ok=True) + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + if inps.ifgarmListTxt: ifgramList_txt = inps.ifgarmListTxt + else: ifgramList_txt = scratchDir + '/' + projectName + '/ifgram_list.txt' + ifgList0 = ut.read_txt2array(ifgramList_txt) +# ifgList = ifgList0[:,0] + if len(ifgList0)==3: + ifgList=ifgList0[0] + ifgList=[ifgList] + else: + ifgList=ifgList0[:,0] + + err_txt = scratchDir + '/' + projectName + '/gamma2licsbas_gamma_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + for i in range(len(ifgList)): + #m0 = ut.yyyymmdd(ifgList[i].split('-')[0]) + #s0 = ut.yyyymmdd(ifgList[i].split('-')[1]) + cmd0 = ['gamma2licsbas_gamma.py',projectName, ifgList[i]] + data0 = [cmd0,err_txt] + geo_file0 = GEOC + '/' + ifgList[i] + '/'+ ifgList[i] + '.geo.unw.tif' + + k00 = 0 + if os.path.isfile(geo_file0): + if os.path.getsize(geo_file0) > 0: + k00 = 1 + if k00==0: + data_para.append(data0) + + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + print("Geocode interferograms for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/third_party/PyINT/pyint/generate_amp_all.py b/third_party/PyINT/pyint/generate_amp_all.py new file mode 100644 index 0000000..17d26bf --- /dev/null +++ b/third_party/PyINT/pyint/generate_amp_all.py @@ -0,0 +1,102 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + + +def work(data0): + cmd = data0[0] + err_txt = data0[1] + + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + #print(aa) + with open(err_txt, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + Generate multilooked amp for one project. + +''' + +EXAMPLE = ''' + Usage: + generate_amp_all.py projectName + generate_amp_all.py projectName --parallel 4 +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate multilooked amp for one project.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + slcDir = scratchDir + '/' + projectName + '/SLC' + rslcDir = scratchDir + '/' + projectName + '/RSLC' + if not os.path.isdir(rslcDir): os.mkdir(rslcDir) + + cmd_command = 'generate_multilook_amp.py' + + err_txt = scratchDir + '/' + projectName + '/generate_amp_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + slc_list = [os.path.basename(fname) for fname in sorted(glob.glob(rslcDir + '/*'))] + #slc_list = ut.get_project_slcList(projectName) + for i in range(len(slc_list)): + cmd0 = [cmd_command,projectName,slc_list[i]] + data0 = [cmd0,err_txt] + data_para.append(data0) + + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + print("Generate multilooked apm for all rslcs is done! ") + ut.print_process_time(start_time, time.time()) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + \ No newline at end of file diff --git a/third_party/PyINT/pyint/generate_bursts_par.py b/third_party/PyINT/pyint/generate_bursts_par.py new file mode 100644 index 0000000..a5fcdde --- /dev/null +++ b/third_party/PyINT/pyint/generate_bursts_par.py @@ -0,0 +1,81 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import glob +import argparse + +from pyint import _utils as ut + + + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Generate burst par files for a project. +''' + +EXAMPLE = """Usage: + + generate_bursts_par.py projectName + +------------------------------------------------------------------- +""" + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Check common busrts for TOPS data.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='Name of project.') + + inps = parser.parse_args() + + return inps + +################################################################################ + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Mdate = templateDict['masterDate'] + + processDir = scratchDir + '/' + projectName + "/PROCESS" + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + dateList = ut.get_project_slcList(projectName) + + for i in range(len(dateList)): + slc_dir = slcDir + '/' + dateList[i] + print(slc_dir) + for j in range(3): + kk = j + SLC = slc_dir + '/' + dateList[i] + '.IW' + str(kk+1)+'.slc' + SLCPar = slc_dir + '/' + dateList[i] + '.IW' + str(kk+1)+'.slc.par' + TOPPar = slc_dir + '/' + dateList[i] + '.IW' + str(kk+1)+'.slc.TOPS_par' + BURST = slc_dir + '/' + dateList[i] + '.IW' + str(kk+1)+'.burst.par' + + call_str = 'SLC_burst_corners ' + SLCPar + ' ' + TOPPar + ' > ' +BURST + os.system(call_str) + + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/generate_multilook_amp.py b/third_party/PyINT/pyint/generate_multilook_amp.py new file mode 100644 index 0000000..ceb392c --- /dev/null +++ b/third_party/PyINT/pyint/generate_multilook_amp.py @@ -0,0 +1,87 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import argparse + +from pyint import _utils as ut + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister TOPS S1-SLC to a reference S1-SLC using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='Name of project.') + parser.add_argument('date', help='date of the slave S1 image. [mater date is read from template]') + inps = parser.parse_args() + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Generate multilook amplitude images for coregistered SLCs. + +''' + +EXAMPLE = """Usage: + + generate_multilook_amp.py projectName Date + + generate_multilook_amp.py PacayaT163TsxHhA 20150102 +------------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Date = inps.date + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + if not os.path.isdir(rslcDir): os.mkdir(rslcDir) + #workDir = processDir + '/' + igramDir + workDir = rslcDir + '/' + Date + if not os.path.isdir(workDir): os.mkdir(workDir) + + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + Mdate = templateDict['masterDate'] + + + rslc = workDir + '/' + Date + '.rslc' + rslcPar = workDir + '/' + Date + '.rslc.par' + + amp = workDir + '/' + Date + '_' + rlks + 'rlks.amp' + ampPar = workDir + '/' + Date + '_' + rlks + 'rlks.amp.par' + + k0 = 0 + if os.path.isfile(ampPar): + if os.path.getsize(ampPar) > 0: + k0 =1 + + if k0==0: + call_str = 'multi_look ' + rslc + ' ' + rslcPar + ' ' + amp + ' ' + ampPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + + nWIDTH = ut.read_gamma_par(ampPar,'read', 'range_samples') + + call_str = 'raspwr ' + amp + ' ' + nWIDTH + os.system(call_str) + + print("Generate amplitude image for RSLC %s is done !!" % Date) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/generate_rdc_dem.py b/third_party/PyINT/pyint/generate_rdc_dem.py new file mode 100644 index 0000000..12f16bd --- /dev/null +++ b/third_party/PyINT/pyint/generate_rdc_dem.py @@ -0,0 +1,204 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import argparse + +from pyint import _utils as ut + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate radar-coordinates based DEM.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='Name of project.') + inps = parser.parse_args() + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Generate radar-coordinates based DEM. + [Geo-coordinates DEM can be downloaded automatically if not provided.] +''' + +EXAMPLE = """Usage: + + generate_rdc_dem.py projectName + +------------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + Mdate = templateDict['masterDate'] + + DEMDir = os.getenv('DEMDIR') + + processDir = scratchDir + '/' + projectName + "/ifgrams" + slcDir = scratchDir + '/' + projectName + "/SLC" + + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + if not os.path.isdir(processDir): + call_str = 'mkdir ' + processDir + os.system(call_str) + + simDir = scratchDir + '/' + projectName + "/DEM" + if not os.path.isdir(simDir): + call_str='mkdir ' + simDir + + + workDir = simDir + + if 'DEM' in templateDict: + dem = templateDict['DEM'] + if not os.path.isfile(dem): + dem = DEMDir + '/' + projectName + '/' + projectName + '.dem' + call_str = 'echo DEM= ' + dem + ' >> ' + templateFile + os.system(call_str) + templateDict['DEM'] = dem + else: + dem = DEMDir + '/' + projectName + '/' + projectName + '.dem' + call_str = 'echo DEM = ' + dem + ' >> ' + templateFile + os.system(call_str) + + demPar = dem + ".par" + + if not os.path.isfile(dem): + call_str = 'makedem_pyint.py ' + projectName + os.system(call_str) + +# Parameter setting for simPhase + latovrSimphase = templateDict['dem_lat_ovr'] + lonovrSimphase = templateDict['dem_lon_ovr'] + + rposSimphase = templateDict['Simphase_rpos'] + azposSimphase = templateDict['Simphase_azpos'] + rwinSimphase = templateDict['Simphase_rwin'] + azwinSimphase = templateDict['Simphase_azwin'] + #rwinSimphase = '128' + #azwinSimphase = '128' + + threshSimphase = templateDict['Simphase_thresh'] + +# Definition of file + MslcDir = slcDir + '/' + Mdate + MslcImg = MslcDir + '/' + Mdate + '.slc' + MslcPar = MslcDir + '/' + Mdate + '.slc.par' + OFFSTD = workDir + '/' + Mdate + '_dem.off_std' + + + BLANK = workDir + '/' + Mdate + '.blk' + MamprlksImg = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp' + MamprlksPar = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + + + UTMDEMpar = simDir + '/'+ Mdate + '_'+ rlks + 'rlks.utm.dem.par' + UTMDEM = simDir + '/' + Mdate + '_'+ rlks + 'rlks.utm.dem' + UTM2RDC = simDir + '/' + Mdate + '_'+ rlks + 'rlks.utm_to_rdc0' + SIMSARUTM = simDir + '/' + Mdate + '_'+ rlks + 'rlks.sim_sar_utm' + PIX = simDir + '/' + Mdate + '_'+ rlks + 'rlks.pix' + LSMAP = simDir + '/' + Mdate + '_'+ rlks + 'rlks.ls_map' + SIMSARRDC = simDir + '/' + Mdate + '_'+ rlks + 'rlks.sim_sar_rdc' + SIMDIFFpar = simDir + '/' + Mdate + '_'+ rlks + 'rlks.diff_par' + SIMOFFS = simDir + '/' + Mdate + '_'+ rlks + 'rlks.offs' + SIMSNR = simDir + '/' + Mdate + '_'+ rlks + 'rlks.snr' + SIMOFFSET = simDir + '/' + Mdate + '_'+ rlks + 'rlks.offset' + SIMCOFF = simDir + '/' + Mdate + '_'+ rlks + 'rlks.coff' + SIMCOFFSETS = simDir + '/' + Mdate + '_'+ rlks + 'rlks.coffsets' + UTMTORDC = simDir + '/' + Mdate + '_'+ rlks + 'rlks.UTM_TO_RDC' + HGTSIM = simDir + '/' + Mdate + '_'+ rlks + 'rlks.rdc.dem' + + if not (os.path.isdir(simDir)): + os.makedirs(simDir) + + ut.createBlankFile(BLANK) + +### remove DEM look up table if it existed for considering gamma overlapping + + if os.path.isfile(UTMDEM): + os.remove(UTMDEM) + if os.path.isfile(UTMDEMpar): + os.remove(UTMDEMpar) + if os.path.isfile(UTM2RDC): + os.remove(UTM2RDC) + + nWidthUTMDEM0 = ut.read_gamma_par(demPar, 'read', 'width') + DateFormat = ut.read_gamma_par(demPar, 'read', 'data_format:') + + if DateFormat == 'INTEGER*2': + DF_type = '4' + else: + DF_type = '2' + + + tmp_dem = dem + '_tmp' + + if not os.path.isfile(tmp_dem): + call_str = 'replace_values ' + dem + ' -32767 0 ' + tmp_dem + ' ' + nWidthUTMDEM0 + ' 2 ' + DF_type + os.system(call_str) + call_str = 'cp ' + tmp_dem + ' ' + dem + os.system(call_str) + + call_str = "multi_look " + MslcImg + " " + MslcPar + " " + MamprlksImg + " " + MamprlksPar + " " + rlks + " " + azlks + os.system(call_str) + + call_str = 'gc_map1 ' + MamprlksPar + ' ' + '-' + ' ' + demPar + ' ' + dem + ' ' + UTMDEMpar + ' ' + UTMDEM + ' ' + UTM2RDC + ' ' + latovrSimphase + ' ' + lonovrSimphase + ' ' + SIMSARUTM + ' - - - - ' + PIX + ' ' + LSMAP + ' - 3 128' + #call_str = 'gc_map2 ' + MamprlksPar + ' ' + demPar + ' ' + dem + ' ' + UTMDEMpar + ' ' + UTMDEM + ' ' + UTM2RDC + ' ' + latovrSimphase + ' ' + lonovrSimphase + ' ' + ' ' + LSMAP + ' - - - - ' + SIMSARUTM + ' - - - ' + PIX + os.system(call_str) + + nWidthUTMDEM = ut.read_gamma_par(UTMDEMpar, 'read', 'width') + nLinePWR1 = ut.read_gamma_par(MamprlksPar, 'read', 'azimuth_lines') + nWidth = ut.read_gamma_par(MamprlksPar, 'read', 'range_samples') + + call_str = 'geocode ' + UTM2RDC + ' ' + SIMSARUTM + ' ' + nWidthUTMDEM + ' ' + SIMSARRDC + ' ' + nWidth + ' ' + nLinePWR1 + ' 0 0' + os.system(call_str) + + call_str = 'create_diff_par ' + MamprlksPar + ' ' + MamprlksPar + ' ' + SIMDIFFpar + ' 1 < ' + BLANK + os.system(call_str) + + call_str = 'init_offsetm ' + SIMSARRDC + ' ' + MamprlksImg + ' ' + SIMDIFFpar + ' 2 2 ' + rposSimphase + ' ' + azposSimphase #+ ' - - - 512' + os.system(call_str) + + call_str = 'offset_pwrm ' + SIMSARRDC + ' ' + MamprlksImg + ' ' + SIMDIFFpar + ' ' + SIMOFFS + ' ' + SIMSNR + ' ' + rwinSimphase + ' ' + azwinSimphase + ' ' + SIMOFFSET #+ ' - 128 128 ' + threshSimphase + os.system(call_str) + + call_str = 'offset_fitm ' + SIMOFFS + ' ' + SIMSNR + ' ' + SIMDIFFpar + ' ' + SIMCOFF + ' ' + SIMCOFFSETS + ' - > ' + OFFSTD + os.system(call_str) + + call_str = 'gc_map_fine ' + UTM2RDC + ' ' + nWidthUTMDEM + ' ' + SIMDIFFpar + ' ' + UTMTORDC + ' 1' + #print(call_str) + os.system(call_str) + + call_str = 'geocode ' + UTMTORDC + ' ' + UTMDEM + ' ' + nWidthUTMDEM + ' ' + HGTSIM + ' ' + nWidth + ' ' + nLinePWR1 + ' 0 0 - - 1 1 1' + os.system(call_str) + + + required_outputs = [UTMDEMpar, UTMDEM, UTMTORDC, HGTSIM] + missing = [path for path in required_outputs if not os.path.isfile(path)] + if missing: + raise RuntimeError('generate_rdc_dem is missing required outputs: ' + ', '.join(missing)) + + print("Create DEM in Radar Coordinates is done!") + + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/geocode_dolphin.py b/third_party/PyINT/pyint/geocode_dolphin.py new file mode 100644 index 0000000..ef5edc8 --- /dev/null +++ b/third_party/PyINT/pyint/geocode_dolphin.py @@ -0,0 +1,500 @@ +#!/usr/bin/env python +""" +Geocode dolphin unwrapped phase results with gdal_translate multi-looking and optional visualization +Output file will have the same pixel dimensions as the input phase file +""" + +import os +import argparse +import numpy as np +import rasterio +from osgeo import gdal, osr +import tempfile +import shutil +import subprocess +import matplotlib.pyplot as plt +from matplotlib.colors import LinearSegmentedColormap + +def gdal_translate_multilook(input_file, output_file, target_width, target_height, resample_alg='average'): + """ + Apply multi-looking using gdal_translate with -outsize option + """ + + cmd = [ + 'gdal_translate', + '-outsize', str(target_width), str(target_height), + '-r', resample_alg, + '-co', 'COMPRESS=LZW', + input_file, + output_file + ] + + print(f"Running: {' '.join(cmd)}") + result = subprocess.run(cmd, capture_output=True, text=True) + + if result.returncode != 0: + print(f"Error in gdal_translate: {result.stderr}") + raise RuntimeError(f"gdal_translate failed with return code {result.returncode}") + + print(f"Multi-looking completed: {input_file} -> {output_file}") + +def geocode_phase_file_same_size(phase_file, lon_file, lat_file, out_file, + output_srs=4326): + """ + Geocode a phase file using corresponding lat, lon files while maintaining same pixel dimensions + + Parameters: + ----------- + phase_file : str + Input unwrapped phase file from dolphin + lon_file : str + Longitude file (should match phase file dimensions) + lat_file : str + Latitude file (should match phase file dimensions) + out_file : str + Output geocoded file + output_srs : int + Output spatial reference system EPSG code + """ + + # Template for VRT source description + sourcexmltmpl = ''' <SimpleSource> + <SourceFilename>{0}</SourceFilename> + <SourceBand>{1}</SourceBand> + </SimpleSource>''' + + # Create a temporary VRT file + temp_dir = tempfile.mkdtemp() + tempvrtname = os.path.join(temp_dir, 'geocode_temp.vrt') + + try: + # Open input file to get dimensions + with rasterio.open(phase_file) as src: + x_size, y_size = src.width, src.height + phase_transform = src.transform + phase_crs = src.crs + + # Create VRT driver + driver = gdal.GetDriverByName('VRT') + tempds = driver.Create(tempvrtname, x_size, y_size, 0) + + # Add band to VRT + tempds.AddBand(gdal.GDT_Float32) + tempds.GetRasterBand(1).SetMetadata( + {'source_0': sourcexmltmpl.format(phase_file, 1)}, + 'vrt_sources' + ) + + # Set spatial reference (WGS84) + sref = osr.SpatialReference() + sref.ImportFromEPSG(4326) + srswkt = sref.ExportToWkt() + + # Set geolocation metadata + tempds.SetMetadata({ + 'SRS': srswkt, + 'X_DATASET': lon_file, + 'X_BAND': '1', + 'Y_DATASET': lat_file, + 'Y_BAND': '1', + 'PIXEL_OFFSET': '0', + 'LINE_OFFSET': '0', + 'PIXEL_STEP': '1', + 'LINE_STEP': '1' + }, 'GEOLOCATION') + + # Clean up + tempds = None + + # Set output SRS + out_sref = osr.SpatialReference() + out_sref.ImportFromEPSG(output_srs) + + # Calculate output bounds from lon/lat files to maintain same pixel dimensions + with rasterio.open(lon_file) as lon_src: + lon_data = lon_src.read(1) + lon_transform = lon_src.transform + + with rasterio.open(lat_file) as lat_src: + lat_data = lat_src.read(1) + lat_transform = lat_src.transform + + # Calculate approximate bounds + valid_mask = ~(np.isnan(lon_data) | np.isnan(lat_data)) + if np.any(valid_mask): + min_lon = np.min(lon_data[valid_mask]) + max_lon = np.max(lon_data[valid_mask]) + min_lat = np.min(lat_data[valid_mask]) + max_lat = np.max(lat_data[valid_mask]) + + # Calculate approximate resolution to maintain same dimensions + approx_res_x = (max_lon - min_lon) / x_size + approx_res_y = (max_lat - min_lat) / y_size + + # Use the larger resolution to ensure we don't oversample + target_res = max(approx_res_x, approx_res_y) + + # Set warp options with calculated resolution to maintain size + warp_options = gdal.WarpOptions( + format='GTiff', + width=x_size, + height=y_size, + dstSRS=out_sref, + resampleAlg='near', + geoloc=True, + creationOptions=['COMPRESS=LZW', 'TILED=YES'] + ) + else: + # Fallback: use default warp options + warp_options = gdal.WarpOptions( + format='GTiff', + dstSRS=out_sref, + resampleAlg='near', + geoloc=True, + creationOptions=['COMPRESS=LZW', 'TILED=YES'] + ) + + # Perform geocoding + print(f"Geocoding {phase_file} to {out_file}") + print(f"Target output dimensions: {x_size} x {y_size}") + gdal.Warp(out_file, tempvrtname, options=warp_options) + + # Verify output dimensions match input + with rasterio.open(out_file) as out_src: + out_width, out_height = out_src.width, out_src.height + if out_width == x_size and out_height == y_size: + print(f"Success: Output dimensions match input ({x_size} x {y_size})") + else: + print(f"Warning: Output dimensions ({out_width} x {out_height}) differ from input ({x_size} x {y_size})") + + except Exception as e: + print(f"Error during geocoding: {e}") + raise + finally: + # Clean up temporary directory + shutil.rmtree(temp_dir) + +def get_file_dimensions(file_path): + """ + Get dimensions of a raster file + """ + with rasterio.open(file_path) as src: + return src.width, src.height + +def check_existing_multilook_files(phase_file, multilook_dir): + """ + Check if downsampled lon/lat files already exist and match phase file dimensions + """ + if not os.path.exists(multilook_dir): + return None, None + + # Get phase file dimensions + phase_width, phase_height = get_file_dimensions(phase_file) + + # Look for common downsampled file patterns + possible_lon_files = [ + os.path.join(multilook_dir, "lon.multilook.tif"), + os.path.join(multilook_dir, "lon.multilook.vrt"), + os.path.join(multilook_dir, "lon.strided.tif"), + os.path.join(multilook_dir, "lon.strided.vrt"), + os.path.join(multilook_dir, "lon_downsampled.tif"), + os.path.join(multilook_dir, "lon_downsampled.vrt"), + ] + + possible_lat_files = [ + os.path.join(multilook_dir, "lat.multilook.tif"), + os.path.join(multilook_dir, "lat.multilook.vrt"), + os.path.join(multilook_dir, "lat.strided.tif"), + os.path.join(multilook_dir, "lat.strided.vrt"), + os.path.join(multilook_dir, "lat_downsampled.tif"), + os.path.join(multilook_dir, "lat_downsampled.vrt"), + ] + + # Check for existing files that match dimensions + for lon_file in possible_lon_files: + if os.path.exists(lon_file): + try: + lon_width, lon_height = get_file_dimensions(lon_file) + if lon_width == phase_width and lon_height == phase_height: + # Found matching lon file, now check for matching lat file + for lat_file in possible_lat_files: + if os.path.exists(lat_file): + lat_width, lat_height = get_file_dimensions(lat_file) + if lat_width == phase_width and lat_height == phase_height: + print(f"Found existing downsampled files:") + print(f" Longitude: {lon_file}") + print(f" Latitude: {lat_file}") + return lon_file, lat_file + except: + # Skip files that can't be read + continue + + return None, None + +def create_phase_colormap(): + """ + Create a colormap suitable for phase data (cyclic colormap) + """ + # Create a cyclic colormap for phase data + colors = [ + (0, 0, 0.5), # Dark blue + (0, 0, 1), # Blue + (0, 1, 1), # Cyan + (0.5, 1, 0.5), # Light green + (1, 1, 0), # Yellow + (1, 0.5, 0), # Orange + (1, 0, 0), # Red + (0.5, 0, 0.5), # Purple + (0, 0, 0.5) # Dark blue (back to start) + ] + + return LinearSegmentedColormap.from_list('phase_cmap', colors, N=256) + +def create_simple_visualization(phase_file, output_file, viz_dir): + """ + Create a simple visualization of the geocoded output + """ + os.makedirs(viz_dir, exist_ok=True) + + # Create phase colormap + phase_cmap = create_phase_colormap() + + # Read the input and output files + with rasterio.open(phase_file) as src: + phase_data = src.read(1) + phase_width, phase_height = src.width, src.height + + with rasterio.open(output_file) as src: + output_data = src.read(1) + output_width, output_height = src.width, src.height + + # Calculate statistics for display + phase_stats = { + 'min': np.nanmin(phase_data), + 'max': np.nanmax(phase_data), + 'mean': np.nanmean(phase_data) + } + + output_stats = { + 'min': np.nanmin(output_data), + 'max': np.nanmax(output_data), + 'mean': np.nanmean(output_data) + } + + # Create figure with subplots + fig, axes = plt.subplots(1, 2, figsize=(16, 6)) + fig.suptitle('Input vs Output Comparison', fontsize=16, fontweight='bold') + + # Original phase + im1 = axes[0].imshow(phase_data, cmap=phase_cmap, + vmin=phase_stats['min'], vmax=phase_stats['max']) + axes[0].set_title(f'Original Phase\nDimensions: {phase_width} x {phase_height}') + axes[0].set_xlabel('Range') + axes[0].set_ylabel('Azimuth') + plt.colorbar(im1, ax=axes[0], label='Phase (rad)') + + # Geocoded output + im2 = axes[1].imshow(output_data, cmap=phase_cmap, + vmin=output_stats['min'], vmax=output_stats['max']) + axes[1].set_title(f'Geocoded Phase\nDimensions: {output_width} x {output_height}') + axes[1].set_xlabel('Longitude') + axes[1].set_ylabel('Latitude') + plt.colorbar(im2, ax=axes[1], label='Phase (rad)') + + # Add statistics text + stats_text1 = f"Min: {phase_stats['min']:.3f}\nMax: {phase_stats['max']:.3f}\nMean: {phase_stats['mean']:.3f}" + axes[0].text(0.02, 0.98, stats_text1, transform=axes[0].transAxes, + verticalalignment='top', fontsize=10, + bbox=dict(boxstyle='round', facecolor='white', alpha=0.8)) + + stats_text2 = f"Min: {output_stats['min']:.3f}\nMax: {output_stats['max']:.3f}\nMean: {output_stats['mean']:.3f}" + axes[1].text(0.02, 0.98, stats_text2, transform=axes[1].transAxes, + verticalalignment='top', fontsize=10, + bbox=dict(boxstyle='round', facecolor='white', alpha=0.8)) + + plt.tight_layout() + plt.savefig(os.path.join(viz_dir, 'input_output_comparison.png'), dpi=300, bbox_inches='tight') + plt.close(fig) + + print(f"Visualization saved to: {os.path.join(viz_dir, 'input_output_comparison.png')}") + +def main(): + parser = argparse.ArgumentParser( + description='Geocode dolphin unwrapped phase results with gdal_translate multi-looking and optional visualization. Output will have same pixel dimensions as input.' + ) + parser.add_argument('-i', '--input', required=True, + help='Input dolphin unwrapped phase file (GeoTIFF)') + parser.add_argument('--lon', required=True, + help='Input longitude file (e.g., lon.rdr.full.vrt)') + parser.add_argument('--lat', required=True, + help='Input latitude file (e.g., lat.rdr.full.vrt)') + parser.add_argument('-o', '--output', default='unwrapped_phase_geocoded.tif', + help='Output geocoded file path') + parser.add_argument('--multilook-dir', default='multilook_geoms', + help='Directory for multi-looked geometry files') + parser.add_argument('--viz-dir', default='visualizations', + help='Directory for visualization images') + parser.add_argument('--output-srs', type=int, default=4326, + help='Output SRS EPSG code (default: 4326 for WGS84)') + parser.add_argument('--resample-alg', default='average', + choices=['near', 'average', 'bilinear', 'cubic', 'cubicspline', 'lanczos', 'mode'], + help='Resampling algorithm for multi-looking (default: average)') + parser.add_argument('--no-cleanup', action='store_true', + help='Keep temporary multi-looked files') + parser.add_argument('--force-multilook', action='store_true', + help='Force multi-looking even if downsampled files exist') + + # Visualization options + viz_group = parser.add_mutually_exclusive_group() + viz_group.add_argument('--no-viz', action='store_true', + help='Skip visualization') + viz_group.add_argument('--viz', action='store_true', + help='Create visualization (default)') + + args = parser.parse_args() + + # Default to visualization if no option is specified + if not (args.no_viz or args.viz): + args.viz = True + + # Check if input files exist + for f in [args.input, args.lon, args.lat]: + if not os.path.exists(f): + print(f"Error: Input file does not exist: {f}") + return 1 + + # Get dimensions from phase file + print("Getting input file dimensions...") + phase_width, phase_height = get_file_dimensions(args.input) + lon_width, lon_height = get_file_dimensions(args.lon) + lat_width, lat_height = get_file_dimensions(args.lat) + + print(f"Phase file dimensions: {phase_width} x {phase_height}") + print(f"Longitude file dimensions: {lon_width} x {lon_height}") + print(f"Latitude file dimensions: {lat_width} x {lat_height}") + + # Check if multi-looking is needed or if files already exist + if (phase_width, phase_height) == (lon_width, lon_height) == (lat_width, lat_height): + print("Files already have matching dimensions. Skipping multi-looking.") + lon_multilook = args.lon + lat_multilook = args.lat + elif not args.force_multilook: + # Check if downsampled files already exist + existing_lon, existing_lat = check_existing_multilook_files(args.input, args.multilook_dir) + if existing_lon and existing_lat: + print("Using existing downsampled files.") + lon_multilook = existing_lon + lat_multilook = existing_lat + else: + # Need to create downsampled files + print("Downsampled files not found or don't match. Creating new ones...") + # Step 1: Apply multi-looking to lon/lat files using gdal_translate + print("Step 1: Applying multi-looking to lon/lat files using gdal_translate...") + + # Create output directory + os.makedirs(args.multilook_dir, exist_ok=True) + + # Generate output file paths + lon_basename = os.path.splitext(os.path.basename(args.lon))[0] + lat_basename = os.path.splitext(os.path.basename(args.lat))[0] + + lon_multilook = os.path.join(args.multilook_dir, f"{lon_basename}_multilook.tif") + lat_multilook = os.path.join(args.multilook_dir, f"{lat_basename}_multilook.tif") + + # Apply multi-looking using gdal_translate + try: + gdal_translate_multilook( + args.lon, lon_multilook, phase_width, phase_height, args.resample_alg + ) + gdal_translate_multilook( + args.lat, lat_multilook, phase_width, phase_height, args.resample_alg + ) + except Exception as e: + print(f"Multi-looking failed: {e}") + return 1 + else: + # Force multi-looking even if files exist + print("Forcing multi-looking...") + # Step 1: Apply multi-looking to lon/lat files using gdal_translate + print("Step 1: Applying multi-looking to lon/lat files using gdal_translate...") + + # Create output directory + os.makedirs(args.multilook_dir, exist_ok=True) + + # Generate output file paths + lon_basename = os.path.splitext(os.path.basename(args.lon))[0] + lat_basename = os.path.splitext(os.path.basename(args.lat))[0] + + lon_multilook = os.path.join(args.multilook_dir, f"{lon_basename}_multilook.tif") + lat_multilook = os.path.join(args.multilook_dir, f"{lat_basename}_multilook.tif") + + # Apply multi-looking using gdal_translate + try: + gdal_translate_multilook( + args.lon, lon_multilook, phase_width, phase_height, args.resample_alg + ) + gdal_translate_multilook( + args.lat, lat_multilook, phase_width, phase_height, args.resample_alg + ) + except Exception as e: + print(f"Multi-looking failed: {e}") + return 1 + + # Step 2: Geocode the phase file using (multi-looked) lon/lat + print("Step 2: Geocoding phase file (maintaining same dimensions)...") + + # Create output directory if needed + output_dir = os.path.dirname(args.output) + if output_dir and not os.path.exists(output_dir): + os.makedirs(output_dir, exist_ok=True) + + try: + geocode_phase_file_same_size( + phase_file=args.input, + lon_file=lon_multilook, + lat_file=lat_multilook, + out_file=args.output, + output_srs=args.output_srs + ) + + print(f"\nGeocoding completed successfully!") + print(f"Geocoded phase file saved to: {args.output}") + + # Display some info about the output file + with rasterio.open(args.output) as src: + print(f"Output file info:") + print(f" Dimensions: {src.width} x {src.height}") + print(f" CRS: {src.crs}") + if src.res: + print(f" Resolution: {src.res[0]:.6f} x {src.res[1]:.6f} degrees") + + except Exception as e: + print(f"Geocoding failed: {e}") + return 1 + + # Step 3: Create visualization if requested + if args.viz: + print("Step 3: Creating visualization...") + try: + create_simple_visualization(args.input, args.output, args.viz_dir) + except Exception as e: + print(f"Visualization failed: {e}") + else: + print("Skipping visualization as requested.") + + # Clean up temporary files if requested + if not args.no_cleanup and (lon_multilook != args.lon or lat_multilook != args.lat): + # Only clean up if we created new files and they're not the original ones + if (lon_multilook != args.lon and lat_multilook != args.lat and + os.path.exists(args.multilook_dir) and + (lon_multilook.startswith(args.multilook_dir) or + lat_multilook.startswith(args.multilook_dir))): + print("Cleaning up temporary files...") + shutil.rmtree(args.multilook_dir) + else: + print(f"Multi-looked files kept in: {args.multilook_dir}") + + return 0 + +if __name__ == "__main__": + exit(main()) diff --git a/third_party/PyINT/pyint/geocode_gamma.py b/third_party/PyINT/pyint/geocode_gamma.py new file mode 100644 index 0000000..1ea3cc7 --- /dev/null +++ b/third_party/PyINT/pyint/geocode_gamma.py @@ -0,0 +1,371 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import os +import sys +import argparse +import numpy as np + +from pyint import _utils as ut + + +def sanitize_gamma_float(filepath, valid_max=1e6): + """清理 GAMMA 浮点数据文件中的无效值 (NaN/Inf/极端值 → 0.0)""" + data = np.fromfile(filepath, dtype=np.float32) + bad_mask = ~np.isfinite(data) | (np.abs(data) > valid_max) + n_bad = int(np.sum(bad_mask)) + if n_bad > 0: + data[bad_mask] = 0.0 + data.tofile(filepath) + print(f' [sanitize] {os.path.basename(filepath)}: ' + f'清理 {n_bad} 个无效像素') + +def geocode(inFile, outFile, UTMTORDC, nWidth, nWidthUTMDEM, nLineUTMDEM, geo_interp='0'): + + if '.unw' in os.path.basename(inFile): + call_str = 'geocode_back ' + inFile + ' ' + nWidth + ' ' + UTMTORDC + ' ' + outFile + ' ' + nWidthUTMDEM + ' ' + nLineUTMDEM + ' ' + geo_interp + ' 0' + elif '.amp' in os.path.basename(inFile): + call_str = 'geocode_back ' + inFile + ' ' + nWidth + ' ' + UTMTORDC + ' ' + outFile + ' ' + nWidthUTMDEM + ' ' + nLineUTMDEM + ' ' + geo_interp + ' 0' + elif '.cor' in os.path.basename(inFile): + call_str = 'geocode_back ' + inFile + ' ' + nWidth + ' ' + UTMTORDC + ' ' + outFile + ' ' + nWidthUTMDEM + ' ' + nLineUTMDEM + ' ' + geo_interp + ' 0' + elif '.dem' in os.path.basename(inFile): + call_str = 'geocode_back ' + inFile + ' ' + nWidth + ' ' + UTMTORDC + ' ' + outFile + ' ' + nWidthUTMDEM + ' ' + nLineUTMDEM + ' ' + geo_interp + ' 0' + else: + call_str = 'geocode_back ' + inFile + ' ' + nWidth + ' ' + UTMTORDC + ' ' + outFile + ' ' + nWidthUTMDEM + ' ' + nLineUTMDEM+ ' ' + geo_interp + ' 1' + + os.system(call_str) + + return + +INTRODUCTION = ''' +------------------------------------------------------------------- + Convert radar-coordinates products into geo-coordinates using GAMMA. + + 由模板参数 geocode_products 控制产品类型 (逗号分隔多选): + hyp3 : 基础产品 + dispmap(LOS/vert) + wrapped_phase + look_vector + licsbas : 基础产品 + look_vector + pot : Pixel Offset Tracking 位移图 (两步地理编码) + + 基础产品 (hyp3/licsbas 均需): amp, corr, unw, diff_filt, diff, dem + 一次调用自动处理所有选中的产品类型。 +''' + +EXAMPLE = ''' + Usage: + geocode_gamma.py projectName Mdate-Sdate + geocode_gamma.py PacayaT163TsxHhA 20150102-20150601 + + Template parameter: + geocode_products = hyp3,licsbas (default) + geocode_products = pot + geocode_products = hyp3,licsbas,pot +------------------------------------------------------------------- +''' + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Geocode radar-coordinate products using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('pair',help='Master-Slave, e.g., 20150101-20150106.') + + inps = parser.parse_args() + return inps + + +def geocode_pot(projectName, Pair, templateDict): + """POT 位移图地理编码: 偏移量几何 → MLI 几何 → EQA 几何""" + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + potDir = projectDir + '/offsets' + workDir = potDir + '/' + Pair + demDir = projectDir + '/DEM' + rslcDir = projectDir + '/RSLC' + slcDir = projectDir + '/SLC' + + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + pot_disp_max = float(templateDict.get('pot_disp_max', '10')) + + Mdate = Pair.split('-')[0] + + if not os.path.isdir(workDir): + print(f'ERROR: POT 目录不存在: {workDir}') + return + + # ===== 公共文件 ===== + MampPar = rslcDir + '/' + masterDate + '/' + masterDate + '_' + rlks + 'rlks.amp.par' + Mamp = rslcDir + '/' + masterDate + '/' + masterDate + '_' + rlks + 'rlks.amp' + UTMTORDC = demDir + '/' + masterDate + '_' + rlks + 'rlks.UTM_TO_RDC' + UTMDEMpar = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem.par' + UTMDEM = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem' + SLCpar = slcDir + '/' + Mdate + '/' + Mdate + '.slc.par' + + if not os.path.isfile(UTMTORDC) or not os.path.isfile(UTMDEMpar): + print(' WARNING: POT 地理编码跳过 — 缺少 UTM_TO_RDC 或 utm.dem.par') + return + if not os.path.isfile(MampPar): + print(' WARNING: POT 地理编码跳过 — 缺少 master amp.par') + return + + mli_width = ut.read_gamma_par(MampPar, 'read', 'range_samples') + mli_nlines = ut.read_gamma_par(MampPar, 'read', 'azimuth_lines') + eqa_width = ut.read_gamma_par(UTMDEMpar, 'read', 'width') + eqa_nlines = ut.read_gamma_par(UTMDEMpar, 'read', 'nlines') + + # ===== POT 特有文件 ===== + MLI_pot_par = workDir + '/' + Mdate + '.mli_pot.par' + MLI_pot = workDir + '/' + Mdate + '.mli_pot' + disp_map = workDir + '/' + Pair + '.disp_map' + + if not os.path.isfile(disp_map): + print(f' WARNING: disp_map 不存在: {disp_map}') + return + + # 偏移量几何宽度 + off_width = ut.read_gamma_par(MLI_pot_par, 'read', 'range_samples') + disp_max_m = pot_disp_max * 10 + + # ===== Step A: 偏移量几何 → MLI 几何 (rdc_trans) ===== + print(' [POT] Step A: 偏移量几何 → MLI 几何') + mli_to_pot_lt = workDir + '/mli_to_pot.lt' + disp_map_mli = workDir + '/' + Pair + '.disp_map_mli' + + if not os.path.isfile(mli_to_pot_lt): + os.system(f'rdc_trans {MampPar} 0.1 {MLI_pot_par} {mli_to_pot_lt}') + + if not os.path.isfile(disp_map_mli): + os.system(f'geocode_back {disp_map} {off_width} {mli_to_pot_lt} ' + f'{disp_map_mli} {mli_width} {mli_nlines} 1 1') + + # ===== Step B: MLI 几何 → EQA 几何 ===== + print(' [POT] Step B: MLI 几何 → EQA 几何') + geo_disp = workDir + '/geo_' + Pair + '.disp_map' + + if not os.path.isfile(geo_disp): + os.system(f'geocode_back {disp_map_mli} {mli_width} {UTMTORDC} ' + f'{geo_disp} {eqa_width} {eqa_nlines} 1 1') + + # ===== Step C: 提取地理编码位移分量 ===== + print(' [POT] Step C: 提取位移分量 (real/imag/mag)') + geo_real = geo_disp + '.real' + geo_imag = geo_disp + '.imag' + geo_mag = geo_disp + '.mag' + + if not os.path.isfile(geo_real): + os.system(f'cpx_to_real {geo_disp} {geo_real} {eqa_width} 0') + sanitize_gamma_float(geo_real, disp_max_m) + if not os.path.isfile(geo_imag): + os.system(f'cpx_to_real {geo_disp} {geo_imag} {eqa_width} 1') + sanitize_gamma_float(geo_imag, disp_max_m) + if not os.path.isfile(geo_mag): + os.system(f'cpx_to_real {geo_disp} {geo_mag} {eqa_width} 3') + sanitize_gamma_float(geo_mag, disp_max_m) + + # ===== Step D: 地理编码 MLI 背景 ===== + geo_mli = workDir + '/geo_' + masterDate + '.mli' + if not os.path.isfile(geo_mli): + os.system(f'geocode_back {Mamp} {mli_width} {UTMTORDC} ' + f'{geo_mli} {eqa_width} {eqa_nlines} 5 0') + + # ===== Step E: 地理编码 CCP (互相关系数) ===== + # 自动查找最终 ccp 文件 + ccp_files = sorted([f for f in os.listdir(workDir) if f.startswith(Pair + '.ccp')]) + if ccp_files: + ccp_final = workDir + '/' + ccp_files[-1] + geo_ccp = workDir + '/geo_' + Pair + '.ccp' + if not os.path.isfile(geo_ccp): + # ccp 在偏移量几何, 需要两步 + ccp_mli = workDir + '/' + Pair + '.ccp_mli' + os.system(f'geocode_back {ccp_final} {off_width} {mli_to_pot_lt} ' + f'{ccp_mli} {mli_width} {mli_nlines} 1 0') + os.system(f'geocode_back {ccp_mli} {mli_width} {UTMTORDC} ' + f'{geo_ccp} {eqa_width} {eqa_nlines} 1 0') + + # ===== Step F: 视角矢量 (look_vector) ===== + print(' [POT] Step F: 视角矢量') + lv_theta = workDir + '/lv_theta' + lv_phi = workDir + '/lv_phi' + OFFpar = workDir + '/' + Pair + '.off' + + if not os.path.isfile(lv_theta) or not os.path.isfile(lv_phi): + if os.path.isfile(SLCpar) and os.path.isfile(OFFpar): + os.system(f'look_vector {SLCpar} {OFFpar} {UTMDEMpar} {UTMDEM} {lv_theta} {lv_phi}') + + # ===== BMP 可视化 ===== + print(' [POT] 生成 BMP') + disp_max_str = str(pot_disp_max) + os.system(f'rasdt_pwr {geo_mag} {geo_mli} {eqa_width} - - - - ' + f'-{disp_max_str} {disp_max_str} 1 rmg.cm {geo_mag}.bmp - - 24') + os.system(f'rasdt_pwr {geo_real} {geo_mli} {eqa_width} - - - - ' + f'-{disp_max_str} {disp_max_str} 1 rmg.cm {geo_real}.bmp - - 24') + os.system(f'rasdt_pwr {geo_imag} {geo_mli} {eqa_width} - - - - ' + f'-{disp_max_str} {disp_max_str} 1 rmg.cm {geo_imag}.bmp - - 24') + + print(' [POT] 地理编码完成!') + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Pair = inps.pair + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + '/' + projectName + '.template' + templateDict = ut.update_template(templateFile) + + # ===== 解析产品类型 ===== + products_str = templateDict.get('geocode_products', 'hyp3,licsbas') + products = set(p.strip().lower() for p in products_str.split(',')) + need_ifg = ('hyp3' in products) or ('licsbas' in products) + need_hyp3 = 'hyp3' in products + need_licsbas = 'licsbas' in products + need_pot = 'pot' in products + + print(f'[geocode_gamma] {projectName} / {Pair}') + print(f' geocode_products: {products_str} → ifg={need_ifg}, hyp3={need_hyp3}, licsbas={need_licsbas}, pot={need_pot}') + + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + potDir = scratchDir + '/' + projectName + '/offsets' + demDir = scratchDir + '/' + projectName + '/DEM' + slcDir = scratchDir + '/' + projectName + '/SLC' + rslcDir = scratchDir + '/' + projectName + '/RSLC' + + # ========================================================= + # IFG 地理编码 (hyp3 和/或 licsbas 需要) + # ========================================================= + ifgWorkDir = ifgDir + '/' + Pair + if need_ifg and os.path.isdir(ifgWorkDir): + print('\n--- IFG 地理编码 ---') + workDir = ifgWorkDir + + ######### copy common file for parallel processing ############# + Mamp0 = rslcDir + '/' + masterDate + '/' + masterDate + '_' + rlks + 'rlks.amp' + MampPar0 = rslcDir + '/' + masterDate + '/' + masterDate + '_' + rlks + 'rlks.amp.par' + Mamp = workDir + '/' + masterDate + '_' + rlks + 'rlks.amp' + MampPar = workDir + '/' + masterDate + '_' + rlks + 'rlks.amp.par' + ut.copy_file(Mamp0, Mamp) + ut.copy_file(MampPar0, MampPar) + + ################################################################# + UNWIFG = workDir + '/' + Pair + '_' + rlks + 'rlks.diff_filt.unw' + ATMCOR_UNW = workDir + '/' + Pair + '_' + rlks + 'rlks.diff_filt.atmcor.unw' + DIFFIFG = workDir + '/' + Pair + '_' + rlks + 'rlks.diff_filt' + diffifg = workDir + '/' + Pair + '_' + rlks + 'rlks.diff' + CORIFG = workDir + '/' + Pair + '_' + rlks + 'rlks.diff_filt.cor' + rdcdem = demDir + '/' + masterDate + '_' + rlks + 'rlks.rdc.dem' + + GeoMamp = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.amp' + GeoCOR = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.diff_filt.cor' + GeoUNW = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.unw' + GeoDIFF = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt' + geodiff = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff' + GeoATMCOR_UNW = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.atmcor.unw' + Geodem = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.hgt' + + UTMTORDC0 = demDir + '/' + masterDate + '_' + rlks + 'rlks.UTM_TO_RDC' + UTMDEMpar0 = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem.par' + UTMTORDC = workDir + '/' + masterDate + '_' + rlks + 'rlks.UTM_TO_RDC' + UTMDEMpar = workDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem.par' + ut.copy_file(UTMTORDC0, UTMTORDC) + ut.copy_file(UTMDEMpar0, UTMDEMpar) + + nWidth = ut.read_gamma_par(MampPar, 'read', 'range_samples') + nWidthUTMDEM = ut.read_gamma_par(UTMDEMpar, 'read', 'width') + nLineUTMDEM = ut.read_gamma_par(UTMDEMpar, 'read', 'nlines') + + # --- 基础产品地理编码 (hyp3/licsbas 均需) --- + geo_interp = templateDict['geo_interp'] + geocode(Mamp, GeoMamp, UTMTORDC, nWidth, nWidthUTMDEM, nLineUTMDEM, geo_interp=geo_interp) + geocode(CORIFG, GeoCOR, UTMTORDC, nWidth, nWidthUTMDEM, nLineUTMDEM, geo_interp=geo_interp) + geocode(DIFFIFG, GeoDIFF, UTMTORDC, nWidth, nWidthUTMDEM, nLineUTMDEM, geo_interp=geo_interp) + geocode(UNWIFG, GeoUNW, UTMTORDC, nWidth, nWidthUTMDEM, nLineUTMDEM, geo_interp=geo_interp) + geocode(diffifg, geodiff, UTMTORDC, nWidth, nWidthUTMDEM, nLineUTMDEM, geo_interp=geo_interp) + geocode(rdcdem, Geodem, UTMTORDC, nWidth, nWidthUTMDEM, nLineUTMDEM, geo_interp=geo_interp) + + Mdate = Pair.split('-')[0] + SLCpar = slcDir + '/' + Mdate + '/' + Mdate + '.slc.par' + OFFpar = workDir + '/' + Pair + '_' + rlks + 'rlks.off' + UTMDEM = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem' + + # --- hyp3 专属: dispmap (LOS/vert 位移场) --- + if need_hyp3: + print(' [hyp3] dispmap + geocode 位移场') + los_disp_rdc = workDir + '/' + Pair + '_' + rlks + 'rlks.los_disp' + vert_disp_rdc = workDir + '/' + Pair + '_' + rlks + 'rlks.vert_disp' + + if os.path.isfile(UNWIFG) and os.path.isfile(SLCpar) and os.path.isfile(OFFpar): + hgt_arg = rdcdem if os.path.isfile(rdcdem) else '-' + if not os.path.isfile(los_disp_rdc): + os.system(f'dispmap {UNWIFG} {hgt_arg} {SLCpar} {OFFpar} {los_disp_rdc} 0') + if not os.path.isfile(vert_disp_rdc): + os.system(f'dispmap {UNWIFG} {hgt_arg} {SLCpar} {OFFpar} {vert_disp_rdc} 1') + + geo_los = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.los_disp' + geo_vert = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.vert_disp' + if os.path.isfile(los_disp_rdc) and not os.path.isfile(geo_los): + os.system(f'geocode_back {los_disp_rdc} {nWidth} {UTMTORDC} {geo_los} {nWidthUTMDEM} {nLineUTMDEM} 1 0') + if os.path.isfile(vert_disp_rdc) and not os.path.isfile(geo_vert): + os.system(f'geocode_back {vert_disp_rdc} {nWidth} {UTMTORDC} {geo_vert} {nWidthUTMDEM} {nLineUTMDEM} 1 0') + + # --- hyp3 专属: 缠绕相位 --- + if need_hyp3: + print(' [hyp3] 提取缠绕相位') + geo_wrapped_pha = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.pha' + if os.path.isfile(GeoDIFF) and not os.path.isfile(geo_wrapped_pha): + os.system(f'cpx_to_real {GeoDIFF} {geo_wrapped_pha} {nWidthUTMDEM} 4') + + # --- hyp3/licsbas 共需: look_vector --- + if need_hyp3 or need_licsbas: + print(' [hyp3/licsbas] look_vector') + lv_theta = workDir + '/lv_theta' + lv_phi = workDir + '/lv_phi' + if not os.path.isfile(lv_theta) or not os.path.isfile(lv_phi): + if os.path.isfile(SLCpar) and os.path.isfile(OFFpar) and os.path.isfile(UTMDEMpar0) and os.path.isfile(UTMDEM): + os.system(f'look_vector {SLCpar} {OFFpar} {UTMDEMpar0} {UTMDEM} {lv_theta} {lv_phi}') + + # --- BMP 可视化 --- + os.system('rasmph_pwr ' + GeoDIFF + ' ' + GeoMamp + ' ' + nWidthUTMDEM + ' - - - - ') + os.system('raspwr ' + GeoMamp + ' ' + nWidthUTMDEM + ' - - - - - - - - - - ') + os.system('rasdt_pwr ' + GeoUNW + ' ' + GeoMamp + ' ' + nWidthUTMDEM + ' - - - - -3.14 3.14 1 rmg.cm') + + if os.path.isfile(GeoATMCOR_UNW): + os.system('rasdt_pwr ' + GeoATMCOR_UNW + ' ' + GeoMamp + ' ' + nWidthUTMDEM + ' - - - - -3.14 3.14 1 rmg.cm') + + if os.path.isfile(UTMTORDC): + os.remove(UTMTORDC) + if os.path.isfile(UTMDEMpar): + os.remove(UTMDEMpar) + + print(' IFG 地理编码完成!') + + elif need_ifg: + print(f' WARNING: ifgrams/{Pair} 不存在, 跳过 IFG 地理编码') + + # ========================================================= + # POT 地理编码 (pot 需要) + # ========================================================= + potWorkDir = potDir + '/' + Pair + if need_pot and os.path.isdir(potWorkDir): + print('\n--- POT 地理编码 ---') + geocode_pot(projectName, Pair, templateDict) + elif need_pot: + print(f' WARNING: offsets/{Pair} 不存在, 跳过 POT 地理编码') + + print("\nGeocoding is done!") + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/geocode_gamma_all.py b/third_party/PyINT/pyint/geocode_gamma_all.py new file mode 100644 index 0000000..696a7d9 --- /dev/null +++ b/third_party/PyINT/pyint/geocode_gamma_all.py @@ -0,0 +1,169 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + +def work(data0): + cmd = data0[0] + err_file = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if aa: + print(aa) + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + '\n' + with open(err_file, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + Geocode products for one project using GAMMA. + 由模板参数 geocode_products 控制产品类型 (hyp3,licsbas,pot). + 一次调用自动处理所有选中的产品类型. +''' + +EXAMPLE = ''' + Usage: + geocode_gamma_all.py projectName + geocode_gamma_all.py projectName --parallel 4 + geocode_gamma_all.py projectName --parallel 4 --ifgramList-txt /test/ifgram_list.txt +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Geocode interferograms for one project using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + parser.add_argument('--ifgarmList-txt', dest='ifgarmListTxt', help='provided ifgram_list_txt. default: using ifgram_list.txt under projectName folder.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + projectDir = scratchDir + '/' + projectName + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + potDir = scratchDir + '/' + projectName + '/offsets' + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + # ===== 解析产品类型 ===== + products_str = templateDict.get('geocode_products', 'hyp3,licsbas') + products = set(p.strip().lower() for p in products_str.split(',')) + need_ifg = ('hyp3' in products) or ('licsbas' in products) + need_hyp3 = 'hyp3' in products + need_pot = 'pot' in products + + print(f'geocode_products: {products_str}') + + if inps.ifgarmListTxt: ifgramList_txt = inps.ifgarmListTxt + else: ifgramList_txt = scratchDir + '/' + projectName + '/ifgram_list.txt' + ifgList0 = ut.read_txt2array(ifgramList_txt) +# ifgList = ifgList0[:,0] + if len(ifgList0)==3: + ifgList=ifgList0[0] + ifgList=[ifgList] + else: + ifgList=ifgList0[:,0] + + err_txt = scratchDir + '/' + projectName + '/geocode_gamma_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + skip_count = 0 + for i in range(len(ifgList)): + pair_i = ifgList[i] + cmd0 = ['geocode_gamma.py', projectName, pair_i] + data0 = [cmd0, err_txt] + + # 根据 geocode_products 组合检查是否已完成 + ifg_done = True + pot_done = True + + if need_ifg: + ifg_dir_i = ifgDir + '/' + pair_i + if os.path.isdir(ifg_dir_i): + # 基础产品检查: unw.bmp + unw_bmp = ifg_dir_i + '/geo_' + pair_i + '_' + rlks + 'rlks.diff_filt.unw.bmp' + if not (os.path.isfile(unw_bmp) and os.path.getsize(unw_bmp) > 0): + ifg_done = False + # hyp3 额外检查: los_disp + if need_hyp3 and ifg_done: + disp_f = ifg_dir_i + '/geo_' + pair_i + '_' + rlks + 'rlks.los_disp' + lv_f = ifg_dir_i + '/lv_theta' + if not (os.path.isfile(disp_f) and os.path.getsize(disp_f) > 0): + ifg_done = False + if not (os.path.isfile(lv_f) and os.path.getsize(lv_f) > 0): + ifg_done = False + else: + ifg_done = True # 目录不存在时不需要处理, geocode_gamma.py 会打 WARNING + + if need_pot: + pot_dir_i = potDir + '/' + pair_i + if os.path.isdir(pot_dir_i): + geo_mag_bmp = pot_dir_i + '/geo_' + pair_i + '.disp_map.mag.bmp' + lv_f = pot_dir_i + '/lv_theta' + if not (os.path.isfile(geo_mag_bmp) and os.path.getsize(geo_mag_bmp) > 0): + pot_done = False + if not (os.path.isfile(lv_f) and os.path.getsize(lv_f) > 0): + pot_done = False + else: + pot_done = True # 目录不存在时不需要处理 + + if ifg_done and pot_done: + skip_count += 1 + else: + data_para.append(data0) + + total = len(ifgList) + todo = len(data_para) + print(f'Geocode: {total} pairs total, {skip_count} done, {todo} to process') + print(f'Parallel processors: {inps.parallelNumb}') + print('=' * 60) + + if todo > 0: + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + + print("Geocode products for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/third_party/PyINT/pyint/geotiff2grd.sh b/third_party/PyINT/pyint/geotiff2grd.sh new file mode 100644 index 0000000..a67fcef --- /dev/null +++ b/third_party/PyINT/pyint/geotiff2grd.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +if [#agrv < 3]; then + + echo" + Usage: geotiff2grd.sh a.tif b.grd + a.tif is a geotif file include nan data + b.grd is a grd file translate by gdal without nan data in it + + Example: geotiff2grd.sh 20240114-20240126_los.tif 20240114-20240126_los.grd + + + Chen Wei 2024/2/17 +" + +exit + +fi + +export input=$1 +export out_grdfile=$2 +gdal_calc.py -A $input --outfile=tmp.tif --calc='(A==0)*(-9999)+(A!=0)*A' +gdal_translate -a_nodata -9999 -of GSBG tmp.tif $out_grdfile +rm tmp.tif *.xml diff --git a/third_party/PyINT/pyint/geotiff_utm2geo.py b/third_party/PyINT/pyint/geotiff_utm2geo.py new file mode 100644 index 0000000..d575f6b --- /dev/null +++ b/third_party/PyINT/pyint/geotiff_utm2geo.py @@ -0,0 +1,76 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import argparse +import os +import warnings + +from osgeo import gdal, osr + +from mintpy.utils import plot as pp, readfile, utils0 as ut + +# link: https://gdal.org/drivers/raster/index.html +GDAL_DRIVER2EXT = { + 'GTiff' : '.tif', + 'ENVI' : '', + 'GMT' : '.grd', + 'GRIB' : '.grb', + 'JPEG' : '.jpg', + 'PNG' : '.png', +} + + +############################################################################## +def cmdLineParse(): + parser = argparse.ArgumentParser(description='convert the UTM geotiff to geography geotiff.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('origin', help='Name of origin hdf5 file .') + parser.add_argument('target', help='Name of target geotiff.') + #parser.add_argument('subdataset', help='the master date and slave date of the dataset') + inps = parser.parse_args() + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + create_files for psokinv software running file by generated by Gamma +''' + +EXAMPLE = """Usage: + + geotiff_utm2geo.py A.h5 result.tif 20210101_20240302 +------------------------------------------------------------------- +""" + + +def main(argv): + + inps = cmdLineParse() + origin_tif = inps.origin + target_tif = inps.target + #dateset = inps.subdataset + ftype = readfile.read_attribute(origin_tif)['FILE_TYPE'] + if ftype == 'timeseries' or ftype == 'ifgramStack': + call_str = 'save_gdal.py ' + origin_tif + ' -d ' + target_tif +' --of GTiff -o tmp.tif ' + os.system(call_str) + else: + call_str = 'save_gdal.py ' + origin_tif +' --of GTiff -o tmp.tif ' + os.system(call_str) + call_str = ' gdalwarp tmp.tif ' + target_tif + '.tif' + ' -t_srs "EPSG:4326" ' + os.system(call_str) + call_str = 'gdal_translate -of GSBG ' + target_tif + '.tif ' + target_tif + '.grd' + os.system(call_str) + + + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/get_master_burst_numb.py b/third_party/PyINT/pyint/get_master_burst_numb.py new file mode 100644 index 0000000..708f802 --- /dev/null +++ b/third_party/PyINT/pyint/get_master_burst_numb.py @@ -0,0 +1,217 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse + +from pyint import _utils as ut + + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Get burst number table of the master date for TOPS SLC.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='project name. e.g., ChangningT55S1A') + + inps = parser.parse_args() + + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Get burst number table of the master date for TOPS SLC. +''' + +EXAMPLE = """Usage: + + get_master_burst_numb.py projectName + + get_master_burst_numb.py ChangningT55S1A + +------------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + masterDate = templateDict['masterDate'] + date = masterDate + + slc_dir = projectDir + '/SLC' + down_dir = projectDir + '/DOWNLOAD' + opod_dir = projectDir + '/OPOD' + + if not os.path.isdir(slc_dir): + os.mkdir(slc_dir) + if not os.path.isdir(opod_dir): + os.mkdir(opod_dir) + + work_dir = slc_dir + '/' + date + if not os.path.isdir(work_dir): + os.mkdir(work_dir) + + work_dir = down_dir + os.chdir(work_dir) + + t_date = down_dir + '/t_' + date + t_burst_numb = down_dir + '/t_burst_numb_' + date + + call_str = 'ls ' + down_dir + '/S1*' + date + '*.zip > ' + t_date + os.system(call_str) + + zip_files = ut.read_txt2list(t_date) + print(zip_files) + #for i in range(len(zip_files)): + # call_str = 'S1_BURST_tab_from_zipfile - ' + zip_files[i] + ' - ' + # os.system(call_str) + AA=''.join(str(zip_files)).split("'")[1] + call_str = 'S1_BURST_tab_from_zipfile - ' + AA + ' - ' + os.system(call_str) + + call_str = 'ls ' + down_dir + '/S1*' + date + '*.burst_number_table > ' + t_burst_numb + os.system(call_str) + burst_files = ut.read_txt2list(t_burst_numb) + + + iw1_burst_node = [] + iw2_burst_node = [] + iw3_burst_node = [] + + print(burst_files) + for i in range(len(burst_files)): + print(burst_files[i]) + iw1_first = ut.read_gamma_par(str(burst_files[i]),'read','iw1_first_burst') + iw1_last = ut.read_gamma_par(str(burst_files[i]),'read','iw1_last_burst') + iw1_burst_node.append(float(iw1_first)) + iw1_burst_node.append(float(iw1_last)) + + iw2_first = ut.read_gamma_par(str(burst_files[i]),'read','iw2_first_burst') + iw2_last = ut.read_gamma_par(str(burst_files[i]),'read','iw2_last_burst') + iw2_burst_node.append(float(iw2_first)) + iw2_burst_node.append(float(iw2_last)) + + iw3_first = ut.read_gamma_par(str(burst_files[i]),'read','iw3_first_burst') + iw3_last = ut.read_gamma_par(str(burst_files[i]),'read','iw3_last_burst') + iw3_burst_node.append(float(iw3_first)) + iw3_burst_node.append(float(iw3_last)) + + + if 'iw1_first_burst' in templateDict: IW1_0 = templateDict['iw1_first_burst'] + else: IW1_0 = '1' + if 'iw1_last_burst' in templateDict: IW1_1 = templateDict['iw1_last_burst'] + else: IW1_1 = str(round(max(iw1_burst_node) - min(iw1_burst_node) + 1)) + + if 'iw2_first_burst' in templateDict: IW2_0 = templateDict['iw2_first_burst'] + else: IW2_0 = '1' + if 'iw2_last_burst' in templateDict: IW2_1 = templateDict['iw2_last_burst'] + else: IW2_1 = str(round(max(iw2_burst_node) - min(iw2_burst_node) + 1)) + + if 'iw3_first_burst' in templateDict: IW3_0 = templateDict['iw3_first_burst'] + else: IW3_0 = '1' + if 'iw3_last_burst' in templateDict: IW3_1 = templateDict['iw3_last_burst'] + else: IW3_1 = str(round(max(iw3_burst_node) - min(iw3_burst_node) + 1)) + + if int(IW1_1) > round(max(iw1_burst_node) - min(iw1_burst_node) + 1): + IW1_1 = str(round(max(iw1_burst_node) - min(iw1_burst_node) + 1)) + + if int(IW2_1) > round(max(iw2_burst_node) - min(iw2_burst_node) + 1): + IW2_1 = str(round(max(iw2_burst_node) - min(iw2_burst_node) + 1)) + + if int(IW3_1) > round(max(iw3_burst_node) - min(iw3_burst_node) + 1): + IW3_1 = str(round(max(iw3_burst_node) - min(iw3_burst_node) + 1)) + + + D_IW1_first = int(IW1_0) - 1 + DD_IW1_numb = int(IW1_1) - int(IW1_0) + 1 + + D_IW2_first = int(IW2_0) - 1 + DD_IW2_numb = int(IW2_1) - int(IW2_0) + 1 + + D_IW3_first = int(IW3_0) - 1 + DD_IW3_numb = int(IW3_1) - int(IW3_0) + 1 + + master_busrt = down_dir + '/master.burst_numb_table' + if os.path.isfile(master_busrt): + os.remove(master_busrt) + + with open(master_busrt, 'a') as f: + STR0 = 'iw1_number_of_bursts: ' + str(DD_IW1_numb) + '\n' + STR00 = 'iw1_number_of_bursts: ' + str(DD_IW1_numb) + print(STR00) + f.write(STR0) + + STR0 = 'iw1_first_burst: ' + str(min(iw1_burst_node) + D_IW1_first) + '\n' + STR00 = 'iw1_first_burst: ' + str(min(iw1_burst_node) + D_IW1_first) + print(STR00) + f.write(STR0) + + STR0 = 'iw1_last_burst: ' + str(min(iw1_burst_node) + D_IW1_first + DD_IW1_numb - 1) + '\n' + STR00 = 'iw1_last_burst: ' + str(min(iw1_burst_node) + D_IW1_first + DD_IW1_numb - 1) + print(STR00) + f.write(STR0) + + STR0 = 'iw2_number_of_bursts: ' + str(DD_IW2_numb) + '\n' + STR00 = 'iw2_number_of_bursts: ' + str(DD_IW2_numb) + print(STR00) + f.write(STR0) + + STR0 = 'iw2_first_burst: ' + str(min(iw2_burst_node) + D_IW2_first) + '\n' + STR00 = 'iw2_first_burst: ' + str(min(iw2_burst_node) + D_IW2_first) + print(STR00) + f.write(STR0) + + STR0 = 'iw2_last_burst: ' + str(min(iw2_burst_node) + D_IW2_first + DD_IW2_numb - 1) + '\n' + STR00 = 'iw2_last_burst: ' + str(min(iw2_burst_node) + D_IW2_first + DD_IW2_numb - 1) + print(STR00) + f.write(STR0) + + + STR0 = 'iw3_number_of_bursts: ' + str(DD_IW3_numb) + '\n' + STR00 = 'iw3_number_of_bursts: ' + str(DD_IW3_numb) + print(STR00) + f.write(STR0) + + STR0 = 'iw3_first_burst: ' + str(min(iw3_burst_node) + D_IW3_first) + '\n' + STR00 = 'iw3_first_burst: ' + str(min(iw3_burst_node) + D_IW3_first) + print(STR00) + f.write(STR0) + + STR0 = 'iw3_last_burst: ' + str(min(iw3_burst_node) + D_IW3_first + DD_IW3_numb - 1) + '\n' + STR00 = 'iw3_last_burst: ' + str(min(iw3_burst_node) + D_IW3_first + DD_IW3_numb - 1) + print(STR00) + f.write(STR0) + + + print("Get burst number table for master date is done! ") + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/third_party/PyINT/pyint/gmt_grdview.sh b/third_party/PyINT/pyint/gmt_grdview.sh new file mode 100644 index 0000000..39f32e8 --- /dev/null +++ b/third_party/PyINT/pyint/gmt_grdview.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# GMT modern mode bash template +# Date: 2024-10-01 +#Purpose: quickly generate basic GMT plot +#Author: <chenweicug@126.com> +#Dependencies: GNUPlot, GMT v6 +#Written for: GNU/Linux + + +if [ $# -lt 1 ]; then +more <<EOF + +gmt_plot_interf.sh is quickly plot the unwrapped phase interferograms and the Los displacement. + +Usage: gmt_plot_interf.sh grdfile satellite +where + grdfile: grd file which contains unwrapped phase + satellite: satellite names: ALOS, S1 + +Example: + bash gmt_plot_interf.sh s1a_20200530_103025_scn1_unw_ifgramPhase.grd S1 + bash gmt_plot_interf.sh ALOS_PHDR_unw_V10_1089_S1.grd ALOS + +EOF +exit +fi + +export GMT_SESSION_NAME=$$ # Set a unique session name +gmt set FONT 14p +gmt set FONT_LABEL 10P + +export grdfile=$1 +export xmin=`gmt grdinfo $grdfile | grep 'x_min' | awk '{print $3}'` +export xmax=`gmt grdinfo $grdfile | grep 'x_min' | awk '{print $5}'` +export ymin=`gmt grdinfo $grdfile | grep 'y_min' | awk '{print $3}'` +export ymax=`gmt grdinfo $grdfile | grep 'y_min' | awk '{print $5}'` +export vmin=`gmt grdinfo $grdfile | grep 'v_min' | awk '{print $3}'` +export vmax=`gmt grdinfo $grdfile | grep 'v_min' | awk '{print $5}'` +export flag=`gmt grdinfo $grdfile | grep 'v_min' | awk '{print $5+$3}'` +export region="$xmin/$xmax/$ymin/$ymax" +export color_file=polar +echo "the plot region is $region" +export gmt_basename=`basename ${grdfile}` +echo " flag is $flag and vmin is $vmin and vmax is $vmax " + if [ ` echo "$flag > 0 " | bc` == 1 ] && [ ` echo "$vmax > 1 " | bc` == 1 ]; then + echo "1" + export cmin=`echo | gmt grdinfo $grdfile | grep 'v_min' | awk '{print $5/$3 } '` + export max_num=`echo "scale=6; 1/$vmax" | bc ` + export min_num=`echo "$max_num*$vmin" | bc ` + gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -G$cmin/1 -Z >1.cpt +elif [ ` echo "$flag < 0 " | bc` == 1 ] && [ ` echo "$vmin < -1 " | bc` == 1 ]; then + echo "2" + export cmax=`echo | gmt grdinfo $grdfile | grep 'v_min' | awk '{print $3/$5*-1} '` + export min_num=`echo "scale=6; 1/$vmin" | bc` + export max_num=`echo "$min_num*$vmax" | bc` + gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -G-1/$cmax -Z >1.cpt +elif [ `echo " $flag > 0 " | bc` == 1 ] && [ ` echo "$vmax < 1 " | bc` == 1 ]; then + echo "3" + if [ ` echo "$vmax < 0.1 " | bc` == 1 ]; then + export vmin1=$(echo "scale=2; $vmin*10" | bc) + export vmax1=$(echo "scale=2; $vmax*10" | bc) + gmt makecpt -C$color_file -G$vmin1/$vmax1 -T$vmin/$vmax/0.01 -Z >1.cpt + else + gmt makecpt -C$color_file -G$vmin/$vmax -T$vmin/$vmax/0.01 -Z >1.cpt + fi +elif [ ` echo "$flag < 0 " | bc` == 1 ] && [ ` echo "$vmin > -1 " | bc` == 1 ]; then + if [ ` echo "$vmin >-0.1 " | bc` == 1 ]; then + echo "4" + export vmin1=$(awk "BEGIN{print($vmin*10)}") + export vmax1=$(awk "BEGIN{print($vmax*10)}") + gmt makecpt -C$color_file -G$vmin1/$vmax1 -T$vmin/$vmax/0.01 -Z >1.cpt + else + gmt makecpt -C$color_file -G$vmin/$vmax -T$vmin/$vmax/0.01 -Z >1.cpt + fi +else + echo "5" + gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -Z >1.cpt +fi +gmt begin ${gmt_basename} png + + gmt grdimage $grdfile -C1.cpt -JM7.3c -Bxa1 -Bya1 -BWSen -R$region + gmt colorbar -Dx3.5/-1c+w6c/0.25c+jBC+h -Bxa0.2f0.1 -C1.cpt +gmt end show diff --git a/third_party/PyINT/pyint/gmt_makecpt.sh b/third_party/PyINT/pyint/gmt_makecpt.sh new file mode 100644 index 0000000..48cb96e --- /dev/null +++ b/third_party/PyINT/pyint/gmt_makecpt.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# GMT modern mode bash template +# Date: 2024-10-01 +#Purpose: quickly generate basic GMT plot +#Author: <chenweicug@126.com> +#Dependencies: GNUPlot, GMT v6 +#Written for: GNU/Linux + + +if [ $# -lt 2 ]; then +more <<EOF + +gmt_makecpt.sh: make color file for user defined data. for phase is wrrapperd color cycle and for displacement is defined the blue for negative and red for positive with the defined vmin and vmax. + The Zero will be white. + +Usage: gmt_makecpt.sh vmin vmax type +Where: + vmin is the minimum value of color bar. + vmax is the maximum value of color bar. + type = los or phase + +Example: + gmt_makecpt.sh -100 300 los # make color file for displacement plot the result is disp.cpt + gmt_makecpt.sh -90 180 phase # make color file for phase plot the result is phase.cpt + +Author: chenweicug@126.com + +EOF +exit +fi + +export GMT_SESSION_NAME=$$ # Set a unique session name +gmt set FONT 14p +gmt set FONT_LABEL 10P + +export color_file="polar" +export vmin=$1 +export vmax=$2 +export type=$3 # type is for dispalcement or phase + +export flag=$(awk "BEGIN{print($vmin+$vmax)}") +echo $flag +# GMT plotting +if [ "$type" == "los" ]; then + if [ ` echo "$flag >0 " | bc ` -eq 1 ] && [ ` echo "$vmax > 1 " | bc ` -eq 1 ]; then + echo "1" + export cmin=$(awk "BEGIN{print($vmin/$vmax)}") + gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -G$cmin/1 -Z >disp.cpt + elif [ ` echo "$flag < 0 " | bc ` -eq 1 ] && [ ` echo "$vmin < -1 " | bc ` -eq 1 ]; then + echo "2" + export cmax=$(awk "BEGIN{print(-$vmax/$vmin)}") + gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -G-1/$cmax -Z >disp.cpt + elif [ `echo "$flag > 0 " | bc ` -eq 1 ] && [ ` echo "$vmax < 1 " | bc ` -eq 1 ]; then + echo "3" + if [ ` echo "$vmax < 0.1 " | bc ` -eq 1 ]; then + export vmin1=$(awk "BEGIN{print($vmin*10)}") + export vmax1=$(awk "BEGIN{print($vmax*10)}") + gmt makecpt -C$color_file -G$vmin1/$vmax1 -T$vmin/$vmax/0.01 -Z >disp.cpt + else + gmt makecpt -C$color_file -G$vmin/$vmax -T$vmin/$vmax/0.01 -Z >disp.cpt + fi + elif [ ` echo "$flag < 0 " | bc ` -eq 1 ] && [ ` echo "$vmin > -1 " | bc ` -eq 1 ]; then + if [ ` echo "$vmin >-0.1 " | bc ` -eq 1 ]; then + echo "4" + export vmin1=$(awk "BEGIN{print($vmin*10)}") + export vmax1=$(awk "BEGIN{print($vmax*10)}") + gmt makecpt -C$color_file -G$vmin1/$vmax1 -T$vmin/$vmax/0.01 -Z >disp.cpt + else + gmt makecpt -C$color_file -G$vmin/$vmax -T$vmin/$vmax/0.01 -Z >disp.cpt + fi + else + echo "5" + gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -Z >disp.cpt + fi + +elif [ "$type" == "phase" ]; then + export inc=$(awk "BEGIN{print($vmax/10)}") + gmt makecpt -Crainbow -T$vmin/$vmax/$inc -Ww -Z >phase.cpt +else + echo "error in the type" + exit +fi +export inc=$(awk "BEGIN{print(($vmax-$vmin)/4)}") +export inc_c=$(awk "BEGIN{print(($vmax-$vmin)/4)}") + +# Draw GMT map diff --git a/third_party/PyINT/pyint/gmt_plot_interf.sh b/third_party/PyINT/pyint/gmt_plot_interf.sh new file mode 100644 index 0000000..a5befea --- /dev/null +++ b/third_party/PyINT/pyint/gmt_plot_interf.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# GMT modern mode bash template +# Date: 2024-10-01 +#Purpose: quickly generate basic GMT plot +#Author: <chenweicug@126.com> +#Dependencies: GNUPlot, GMT v6 +#Written for: GNU/Linux + + +if [ $# -lt 2 ]; then +more <<EOF + +gmt_plot_interf.sh is quickly plot the unwrapped phase interferograms and the Los displacement. + +Usage: gmt_plot_interf.sh grdfile satellite +where + grdfile: grd file which contains unwrapped phase + satellite: satellite names: ALOS, S1 + +Example: + bash gmt_plot_interf.sh s1a_20200530_103025_scn1_unw_ifgramPhase.grd S1 + bash gmt_plot_interf.sh ALOS_PHDR_unw_V10_1089_S1.grd ALOS + +EOF +exit +fi + +export GMT_SESSION_NAME=$$ # Set a unique session name +gmt set FONT 14p +gmt set FONT_LABEL 10P + + +export grdfile=$1 +export sat=$2 +export xmin=`gmt grdinfo $grdfile | grep 'x_min' | awk '{print $3}'` +echo "$xmin" +export xmax=`gmt grdinfo $grdfile | grep 'x_min' | awk '{print $5}'` +export ymin=`gmt grdinfo $grdfile | grep 'y_min' | awk '{print $3}'` +export ymax=`gmt grdinfo $grdfile | grep 'y_min' | awk '{print $5}'` +export vmin=`gmt grdinfo $grdfile | grep 'v_min' | awk '{print $3}'` +export vmax=`gmt grdinfo $grdfile | grep 'v_min' | awk '{print $5}'` + +export region="$xmin/$xmax/$ymin/$ymax" +export color_file=polar +echo "the plot region is $region" +export gmt_basename=`basename ${grdfile}` + + +if [ $sat=='S1' ]; then + gmt grdmath $grdfile 0.0555041577 MUL = tmp.grd + elif [ $sat=='ALOS' ]; then + gmt grdmath $grdfile 0.236 MUL = tmp.grd + else + gmt grdmath $grdfile 0.0311 MUL = tmp.grd +fi + + gmt grdmath tmp.grd -12.5663704 DIV = los.grd + rm tmp.grd + gmt grdmath $grdfile 3.1415926 FMOD = unw_plot.grd + +# GMT plotting + +export flag=`gmt grdinfo los.grd | grep 'v_min' | awk '{print $5+$3}'` +export vmin=`gmt grdinfo los.grd | grep 'v_min' | awk '{print $3}'` +export vmax=`gmt grdinfo los.grd | grep 'v_min' | awk '{print $5}'` +#export min_num=`echo "scale=6; 1/$vmin" | bc ` +#export max_num=`echo "scale=6; 1/$vmax" | bc ` +echo " flag is $flag and vmin is $vmin and vmax is $vmax " + if [ ` echo "$flag >0 " | bc ` -eq 1 ] && [ ` echo "$vmax > 1 " | bc ` -eq 1 ]; then + echo "1" + export cmin=`echo | gmt grdinfo los.grd | grep 'v_min' | awk '{print $5/$3 } '` + export max_num=`echo "scale=6; 1/$vmax" | bc ` + export min_num=`echo "$max_num*$vmin" | bc ` + gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -G$cmin/1 -Z >1.cpt +elif [ ` echo "$flag < 0 " | bc ` -eq 1 ] && [ ` echo "$vmin < -1 " | bc ` -eq 1 ]; then + echo "2" + export cmax=`echo | gmt grdinfo los.grd | grep 'v_min' | awk '{print $3/$5*-1} '` + export min_num=`echo "scale=6; 1/$vmin" | bc` + export max_num=`echo "$min_num*$vmax" | bc` + gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -G-1/$cmax -Z >1.cpt +elif [ `echo "$flag > 0 " | bc ` -eq 1 ] && [ ` echo "$vmax < 1 " | bc ` -eq 1 ]; then + echo "3" + if [ ` echo "$vmax < 0.1 " | bc ` -eq 1 ]; then + export vmin1=$(echo "scale=2; $vmin*10" | bc) + export vmax1=$(echo "scale=2; $vmax*10" | bc) + gmt makecpt -C$color_file -G$vmin1/$vmax1 -T$vmin/$vmax/0.01 -Z >1.cpt + else + gmt makecpt -C$color_file -G$vmin/$vmax -T$vmin/$vmax/0.01 -Z >1.cpt + fi +elif [ ` echo "$flag < 0 " | bc ` -eq 1 ] && [ ` echo "$vmin > -1 " | bc ` -eq 1 ]; then + if [ ` echo "$vmin >-0.1 " | bc ` -eq 1 ]; then + echo "4" + export vmin1=$(awk "BEGIN{print($vmin*10)}") + export vmax1=$(awk "BEGIN{print($vmax*10)}") + echo $vmin1 + echo $vmax1 + gmt makecpt -C$color_file -G$vmin1/$vmax1 -T$vmin/$vmax/0.01 -Z >1.cpt + else + gmt makecpt -C$color_file -G$vmin/$vmax -T$vmin/$vmax/0.01 -Z >1.cpt + fi +else + echo "5" + gmt makecpt -C$color_file -T$vmin/$vmax/0.01 -Z >1.cpt +fi + +#gmt makecpt -C$color_file -G$vmin/$vmax -T$vmin/$vmax/0.01 >1.cpt + gmt makecpt -Crainbow -T-3.14159265/3.14159265/0.31415926 -Z >2.cpt + + +# Draw GMT map +gmt begin insar_los_unw_${gmt_basename} png + gmt subplot begin 1x2 -Fs7.0c -A+jTR+gwhite+p1p+o0.05c/0.2c -M0.6c/0.25c -JM7.3c -Bxa1 -Bya1 -BWSen -Y10c -R$region + gmt subplot set 0 -A"Unw" + gmt grdimage unw_plot.grd -C2.cpt -Q + gmt colorbar -Dx3.5c/-1c+w6c/0.25c+jBC+h -Bxa3.14 -C2.cpt + gmt subplot set 1 -A"Los" + gmt grdimage los.grd -C1.cpt + gmt colorbar -Dx3.5/-1c+w6c/0.25c+jBC+h -Bxa0.06f0.02 -C1.cpt + gmt subplot end +gmt end show \ No newline at end of file diff --git a/third_party/PyINT/pyint/hyp3_timeseries_utm2wgs84.py b/third_party/PyINT/pyint/hyp3_timeseries_utm2wgs84.py new file mode 100644 index 0000000..a1bbcb5 --- /dev/null +++ b/third_party/PyINT/pyint/hyp3_timeseries_utm2wgs84.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +""" +将 HDF5 文件的坐标系统从 UTM 转换为 WGS84 地理坐标 +同时保持与 MintPy tsview.py 的兼容性 +""" + +import h5py +import numpy as np +import copy +import argparse +import os + +def utm_to_wgs84(easting, northing, zone, northern=True): + """ + 将 UTM 坐标转换为 WGS84 大地坐标 + """ + # UTM 参数 + a = 6378137.0 # WGS84 长半轴 + e = 0.081819190842622 # WGS84 第一偏心率 + k0 = 0.9996 # 比例因子 + + # 中央经线 + lon0 = (zone - 1) * 6 - 180 + 3 + + # 调整东伪偏移 + x = easting - 500000.0 + + # 如果是南半球，调整北伪偏移 + if not northern: + y = northing - 10000000.0 + else: + y = northing + + # M = y / k0 + mu = y / (a * (1 - e**2 / 4 - 3 * e**4 / 64 - 5 * e**6 / 256) * k0) + + e1 = (1 - np.sqrt(1 - e**2)) / (1 + np.sqrt(1 - e**2)) + + # 纬度计算 + N1 = a / np.sqrt(1 - e**2 * np.sin(mu)**2) + T1 = np.tan(mu)**2 + C1 = e1**2 * np.cos(mu)**2 + R1 = a * (1 - e**2) / (1 - e**2 * np.sin(mu)**2)**(3/2) + D = x / (N1 * k0) + + lat = mu - N1 * np.tan(mu) / R1 * (D**2 / 2 + (5 + 3 * T1 + 10 * C1 - 4 * C1**2 - 9 * e1**2) * D**4 / 24 + (61 + 90 * T1 + 298 * C1 + 45 * T1**2 - 252 * e1**2 - 3 * C1**2) * D**6 / 720) + + # 经度计算 + lon = lon0 * np.pi / 180 + D * (1 + (1 + 3 * e1**2 + 2 * e1**4) * D**2 / 6 + (2 - e1**2) * D**4 / 120) / np.cos(mu) + + # 转换为角度 + lat_deg = np.degrees(lat) + lon_deg = np.degrees(lon) + + return lon_deg, lat_deg + + +def main(): + parser = argparse.ArgumentParser( + description='将 HDF5 文件转换为 WGS84 地理坐标（与 MintPy 兼容）', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=''' +示例: + python convert_to_wgs84.py input.h5 -o output.h5 + +说明: + 脚本会将文件完全转换为 WGS84 地理坐标系统， + 同时设置正确的属性以保持与 MintPy 的兼容性。 + ''' + ) + parser.add_argument('input', help='输入 HDF5 文件路径') + parser.add_argument('-o', '--output', help='输出 HDF5 文件路径（默认：输入文件名_WGS84.h5）') + + args = parser.parse_args() + + # 输入和输出文件 + input_file = args.input + + if args.output: + output_file = args.output + else: + # 自动生成输出文件名 + base, ext = os.path.splitext(input_file) + output_file = f"{base}_WGS84{ext}" + + print(f"正在读取文件: {input_file}") + f_in = h5py.File(input_file, 'r') + + # 读取所有数据集 + datasets = {} + for key in f_in.keys(): + datasets[key] = f_in[key][:] + print(f" 读取数据集: {key}, 形状: {datasets[key].shape}") + + # 读取所有属性 + attrs = dict(f_in.attrs) + print(f"\n共有 {len(attrs)} 个属性") + + # 解析 UTM 坐标信息 + x_first = float(attrs['X_FIRST']) + y_first = float(attrs['Y_FIRST']) + x_step = float(attrs['X_STEP']) + y_step = float(attrs['Y_STEP']) + width = int(attrs['WIDTH']) + length = int(attrs['LENGTH']) + utm_zone = attrs['UTM_ZONE'] + + # 获取参考点位置（像素坐标） + ref_x = int(attrs.get('REF_X', 0)) + ref_y = int(attrs.get('REF_Y', 0)) + + # 转换 UTM Zone + zone_num = int(''.join(filter(str.isdigit, utm_zone))) + is_northern = 'N' in utm_zone.upper() + + print(f"\n原始坐标系统:") + print(f" UTM Zone: {utm_zone} (EPSG: {attrs['EPSG']})") + print(f" 左上角 UTM: ({x_first}, {y_first})") + print(f" 步长: X={x_step}m, Y={y_step}m") + print(f" 参考点像素: ({ref_x}, {ref_y})") + + # 生成网格坐标 + x_coords = x_first + np.arange(width) * x_step + y_coords = y_first + np.arange(length) * y_step + + # 转换为 WGS84 + print(f"\n正在转换为 WGS84...") + lon_grid, lat_grid = np.meshgrid(x_coords, y_coords) + wgs84_lon, wgs84_lat = utm_to_wgs84(lon_grid, lat_grid, zone_num, is_northern) + + # 计算新的坐标参数（注意：lon 是经度，lat 是纬度） + # X 对应经度，Y 对应纬度 + lon_first = wgs84_lon[0, 0] + lat_first = wgs84_lat[0, 0] + lon_step = abs(wgs84_lon[0, 1] - wgs84_lon[0, 0]) if width > 1 else 0 + lat_step = abs(wgs84_lat[1, 0] - wgs84_lat[0, 0]) if length > 1 else 0 + + # 计算参考点的经纬度 + # REF_X 对应 X 轴（经度），REF_Y 对应 Y 轴（纬度） + ref_lon = wgs84_lon[ref_y, ref_x] # 经度 + ref_lat = wgs84_lat[ref_y, ref_x] # 纬度 + + # 更新属性 - 完全转换为 WGS84 + attrs_new = copy.deepcopy(attrs) + + # 更新坐标系统相关属性 + attrs_new['EPSG'] = 4326 # WGS84 + attrs_new['X_FIRST'] = lon_first # 经度 + attrs_new['Y_FIRST'] = lat_first # 纬度 + attrs_new['X_STEP'] = lon_step + attrs_new['Y_STEP'] = -lat_step # 保持Y方向向下为负 + attrs_new['X_UNIT'] = 'degrees' + attrs_new['Y_UNIT'] = 'degrees' + + # 更新参考坐标 - 现在是真正的经纬度 + attrs_new['REF_LON'] = ref_lon + attrs_new['REF_LAT'] = ref_lat + + # 保留原始 UTM 信息作为备份（带 _ORIG 后缀） + attrs_new['UTM_ZONE_ORIG'] = attrs_new.get('UTM_ZONE', '') + attrs_new['EPSG_ORIG'] = attrs_new.get('EPSG', '') + + # 关键：移除或清空 UTM_ZONE 属性，让 MintPy 识别为地理坐标系统 + if 'UTM_ZONE' in attrs_new: + del attrs_new['UTM_ZONE'] + + # 更新角点坐标 + attrs_new['LAT_REF1'] = lat_first + attrs_new['LAT_REF2'] = wgs84_lat[0, -1] + attrs_new['LAT_REF3'] = wgs84_lat[-1, 0] + attrs_new['LAT_REF4'] = wgs84_lat[-1, -1] + attrs_new['LON_REF1'] = lon_first + attrs_new['LON_REF2'] = wgs84_lon[0, -1] + attrs_new['LON_REF3'] = wgs84_lon[-1, 0] + attrs_new['LON_REF4'] = wgs84_lon[-1, -1] + + # 设置其他相关属性 + attrs_new['COORD_SYSTEM'] = 'GEO' # 地理坐标系统 + + print(f"\n新坐标系统 (WGS84):") + print(f" EPSG: {attrs_new['EPSG']}") + print(f" 左上角: ({attrs_new['X_FIRST']:.6f}, {attrs_new['Y_FIRST']:.6f})") + print(f" 步长: X={attrs_new['X_STEP']:.6f}°, Y={attrs_new['Y_STEP']:.6f}°") + print(f" 参考点经纬度: ({attrs_new['REF_LON']:.6f}, {attrs_new['REF_LAT']:.6f})") + print(f" 参考点像素: ({ref_x}, {ref_y})") + + # 创建新的 HDF5 文件 + print(f"\n正在写入文件: {output_file}") + f_out = h5py.File(output_file, 'w') + + # 写入所有数据集 + for key, data in datasets.items(): + f_out.create_dataset(key, data=data) + print(f" 写入数据集: {key}") + + # 写入所有属性 + for key, value in attrs_new.items(): + f_out.attrs[key] = value + + f_in.close() + f_out.close() + + print(f"\n完成！转换后的文件已保存为: {output_file}") + print("\n说明:") + print(" - 文件已完全转换为 WGS84 地理坐标系统") + print(" - 所有坐标属性已更新为经纬度") + print(" - 原始 UTM 信息已保存为 _ORIG 后缀的属性") + print(" - 应该与 MintPy tsview.py 兼容") + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/third_party/PyINT/pyint/hyp3format_gamma.py b/third_party/PyINT/pyint/hyp3format_gamma.py new file mode 100644 index 0000000..dbffcb1 --- /dev/null +++ b/third_party/PyINT/pyint/hyp3format_gamma.py @@ -0,0 +1,275 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Modified: 2026, Z. Zou - HyP3 UTM GeoTIFF output ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import argparse +import time +from datetime import datetime + +from pyint import _utils as ut + + +INTRODUCTION = ''' +------------------------------------------------------------------- + 将 geocode_gamma.py 已地理编码的 GAMMA 二进制产品转换为 + ASF HyP3 兼容的 UTM GeoTIFF 格式, 用于 MintPy 时间序列分析。 + + 本脚本 **不做任何地理编码或计算**, 仅执行: + 1. data2geotiff : GAMMA 二进制 (EQA) → GeoTIFF (EQA) + 2. gdalwarp : GeoTIFF (EQA) → GeoTIFF (UTM) + 3. rasterio : 生成水体掩膜 + + 所有地理编码产品由 geocode_gamma.py 提供: + amp, corr, dem, unw_phase, wrapped_phase, + los_disp, vert_disp, lv_theta, lv_phi +''' + +EXAMPLE = ''' + Usage: + hyp3format_gamma.py projectName ifgPair + hyp3format_gamma.py shanghaiT171F128S1A 20241105-20241117 + hyp3format_gamma.py shanghaiT171F128S1A 20241105-20241117 --output-dir /path/to/output +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser( + description='Convert geocoded GAMMA binaries to HyP3 UTM GeoTIFF.', + formatter_class=argparse.RawTextHelpFormatter, + epilog=INTRODUCTION + '\n' + EXAMPLE) + parser.add_argument('projectName', help='项目名称') + parser.add_argument('ifgPair', help='干涉图对 (如 20241105-20241117)') + parser.add_argument('--output-dir', dest='output_dir', default=None, + help='输出目录. 默认: projectDir/Hyp3Products/') + inps = parser.parse_args() + return inps + + +def get_utm_epsg(lon, lat): + """根据中心经纬度自动计算 UTM 投影的 EPSG 代码""" + zone = int((lon + 180) / 6) + 1 + return 32600 + zone if lat >= 0 else 32700 + zone + + +def run_cmd(cmd, desc=''): + """执行 shell 命令并检查返回值""" + if desc: + print(f" [{desc}] {cmd}") + else: + print(f" {cmd}") + ret = os.system(cmd) + if ret != 0: + print(f" WARNING: 返回非零退出码 {ret}") + return ret + + +def main(argv): + + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + ifgPair = inps.ifgPair + + Mdate = ifgPair.split('-')[0] + Sdate = ifgPair.split('-')[1] + + # 验证日期并计算时间基线 + d1 = datetime.strptime(Mdate, '%Y%m%d') + d2 = datetime.strptime(Sdate, '%Y%m%d') + interval = abs((d2 - d1).days) + + # 读取模板参数 + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + '/' + projectName + '.template' + templateDict = ut.update_template(templateFile) + + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + + # 项目目录 + projectDir = scratchDir + '/' + projectName + ifgDir = projectDir + '/ifgrams' + demDir = projectDir + '/DEM' + + workDir = ifgDir + '/' + ifgPair + if not os.path.isdir(workDir): + print(f"ERROR: 干涉图目录不存在: {workDir}") + sys.exit(1) + + # 输出目录 + outputDir = inps.output_dir or (projectDir + '/Hyp3Products') + productDir = os.path.join(outputDir, ifgPair) + os.makedirs(productDir, exist_ok=True) + + # ============================================================ + # DEM 参数 (直接引用, 不复制不删除) + # ============================================================ + DEMpar = demDir + '/' + masterDate + '_' + rlks + 'rlks.utm.dem.par' + + nWidthDEM = ut.read_gamma_par(DEMpar, 'read', 'width') + nLineDEM = ut.read_gamma_par(DEMpar, 'read', 'nlines') + + # 自动检测 UTM 投影 + corner_lat = float(ut.read_gamma_par(DEMpar, 'read', 'corner_lat').split()[0]) + corner_lon = float(ut.read_gamma_par(DEMpar, 'read', 'corner_lon').split()[0]) + post_lat = float(ut.read_gamma_par(DEMpar, 'read', 'post_lat').split()[0]) + post_lon = float(ut.read_gamma_par(DEMpar, 'read', 'post_lon').split()[0]) + center_lat = corner_lat + float(nLineDEM) / 2 * post_lat + center_lon = corner_lon + float(nWidthDEM) / 2 * post_lon + utm_epsg = get_utm_epsg(center_lon, center_lat) + + # HyP3 文件名前缀 (含 YYYYMMDDTHHMMSS, 兼容 MintPy 日期解析) + hyp3_prefix = f"{projectName}_{Mdate}T000000_{Sdate}T000000" + + print("\n" + "=" * 70) + print(f"HyP3 格式转换: {projectName} / {ifgPair}") + print(f"时间基线: {interval} 天, DEM: {nWidthDEM}×{nLineDEM}") + print(f"中心: {center_lat:.4f}°N, {center_lon:.4f}°E → EPSG:{utm_epsg}") + print(f"输出: {productDir}") + print("=" * 70) + + # ============================================================ + # 已地理编码 (EQA) GAMMA 二进制文件 (由 geocode_gamma.py 生成) + # ============================================================ + geo_unw = workDir + '/geo_' + ifgPair + '_' + rlks + 'rlks.diff_filt.unw' + geo_wrapped_pha = workDir + '/geo_' + ifgPair + '_' + rlks + 'rlks.diff_filt.pha' + geo_los_disp = workDir + '/geo_' + ifgPair + '_' + rlks + 'rlks.los_disp' + geo_vert_disp = workDir + '/geo_' + ifgPair + '_' + rlks + 'rlks.vert_disp' + geo_amp = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.amp' + geo_hgt = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.hgt' + lv_theta = workDir + '/lv_theta' + lv_phi = workDir + '/lv_phi' + + # 相干性可能用 Pair 或 masterDate 命名 + geo_cor = workDir + '/geo_' + ifgPair + '_' + rlks + 'rlks.diff_filt.cor' + if not os.path.exists(geo_cor): + geo_cor = workDir + '/geo_' + masterDate + '_' + rlks + 'rlks.diff_filt.cor' + + # ============================================================ + # data2geotiff (EQA binary → EQA GeoTIFF) → gdalwarp (→ UTM) + # ============================================================ + print("\n--- GAMMA 二进制 → EQA GeoTIFF → UTM GeoTIFF ---") + + product_map = [ + ('amp', geo_amp, 'bilinear'), + ('corr', geo_cor, 'bilinear'), + ('dem', geo_hgt, 'bilinear'), + ('unw_phase', geo_unw, 'bilinear'), + ('wrapped_phase', geo_wrapped_pha, 'bilinear'), + ('los_disp', geo_los_disp, 'bilinear'), + ('vert_disp', geo_vert_disp, 'bilinear'), + ('lv_theta', lv_theta, 'bilinear'), + ('lv_phi', lv_phi, 'bilinear'), + ] + + generated = {} + + for name, geo_bin, interp in product_map: + utm_tif = os.path.join(productDir, f'{hyp3_prefix}_{name}.tif') + + # 跳过已存在且非空的输出 + if os.path.exists(utm_tif) and os.path.getsize(utm_tif) > 0: + print(f" {name}: 已存在，跳过") + generated[name] = utm_tif + continue + + if not os.path.exists(geo_bin): + print(f" {name}: 源文件不存在 ({os.path.basename(geo_bin)})，跳过") + continue + + eqa_tif = workDir + '/' + name + '_hyp3_eqa.tif' + + # GAMMA 二进制 → EQA GeoTIFF + run_cmd(f'data2geotiff {DEMpar} {geo_bin} 2 {eqa_tif}', f'{name} → EQA') + + if not os.path.exists(eqa_tif): + print(f" WARNING: {name} EQA GeoTIFF 生成失败") + continue + + # EQA → UTM (gdalwarp) + run_cmd(f'gdalwarp -t_srs EPSG:{utm_epsg} -r {interp} ' + f'-co COMPRESS=LZW -overwrite {eqa_tif} {utm_tif}', + f'{name} → UTM') + + # 清理 EQA 中间文件 + if os.path.exists(eqa_tif): + os.remove(eqa_tif) + + if os.path.exists(utm_tif) and os.path.getsize(utm_tif) > 0: + generated[name] = utm_tif + + # ============================================================ + # 生成水体掩膜 (相干性阈值) + # ============================================================ + print("\n--- 水体掩膜 ---") + water_mask_tif = os.path.join(productDir, f'{hyp3_prefix}_water_mask.tif') + corr_tif = generated.get('corr') + + if corr_tif and os.path.exists(corr_tif) and not os.path.exists(water_mask_tif): + try: + import rasterio + with rasterio.open(corr_tif) as src: + corr_data = src.read(1) + mask = np.ones_like(corr_data, dtype=np.uint8) + mask[corr_data < 0.05] = 0 + mask[np.isnan(corr_data)] = 0 + + profile = src.profile.copy() + profile.update(dtype=rasterio.uint8, count=1, compress='lzw', nodata=None) + with rasterio.open(water_mask_tif, 'w', **profile) as dst: + dst.write(mask, 1) + generated['water_mask'] = water_mask_tif + print(f" 水体掩膜已生成") + except Exception as e: + print(f" WARNING: 水体掩膜生成失败: {e}") + elif os.path.exists(water_mask_tif): + generated['water_mask'] = water_mask_tif + print(f" 水体掩膜已存在，跳过") + + # ============================================================ + # 生成入射角图 (inc_map / inc_map_ell, 复制自 lv_theta) + # ============================================================ + print("\n--- 入射角图 ---") + inc_map_tif = os.path.join(productDir, f'{hyp3_prefix}_inc_map.tif') + inc_map_ell_tif = os.path.join(productDir, f'{hyp3_prefix}_inc_map_ell.tif') + lv_theta_tif = generated.get('lv_theta') + + if lv_theta_tif and os.path.exists(lv_theta_tif): + if not os.path.exists(inc_map_tif): + os.system(f'cp {lv_theta_tif} {inc_map_tif}') + generated['inc_map'] = inc_map_tif + print(f" inc_map 已生成 (= lv_theta)") + if not os.path.exists(inc_map_ell_tif): + os.system(f'cp {lv_theta_tif} {inc_map_ell_tif}') + generated['inc_map_ell'] = inc_map_ell_tif + print(f" inc_map_ell 已生成 (近似 lv_theta)") + + # ============================================================ + # 输出摘要 + # ============================================================ + print("\n" + "=" * 70) + print(f"HyP3 格式转换完成: {ifgPair}") + print(f"共 {len(generated)} 个产品:") + for name, path in sorted(generated.items()): + if os.path.exists(path): + size_mb = os.path.getsize(path) / 1024 / 1024 + print(f" {name:20s} ({size_mb:.1f} MB)") + print("=" * 70) + + ut.print_process_time(start_time, time.time()) + sys.exit(0) + + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/hyp3format_gamma_all.py b/third_party/PyINT/pyint/hyp3format_gamma_all.py new file mode 100644 index 0000000..bce90ec --- /dev/null +++ b/third_party/PyINT/pyint/hyp3format_gamma_all.py @@ -0,0 +1,147 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Modified: 2026, Z. Zou - HyP3 UTM batch conversion ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import time +import glob +import argparse +import subprocess + +from pyint import _utils as ut + + +def work(data0): + """并行 worker: 执行单对 hyp3format_gamma.py""" + cmd = data0[0] + err_file = data0[1] + p = subprocess.run(cmd, shell=False, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stderr = p.stderr + + if type(stderr) == bytes: + aa = stderr.decode("utf-8") + else: + aa = stderr + + if aa: + str0 = ' '.join(cmd) + '\n' + with open(err_file, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return + +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + 批量将 GAMMA 处理结果转换为 HyP3 UTM GeoTIFF 格式, + 支持并行处理。 +''' + +EXAMPLE = ''' + Usage: + hyp3format_gamma_all.py projectName + hyp3format_gamma_all.py projectName --parallel 4 + hyp3format_gamma_all.py projectName --output-dir /path/to/output + hyp3format_gamma_all.py projectName --parallel 4 --ifgramList-txt /test/ifgram_list.txt +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser( + description='批量转换 GAMMA 输出为 HyP3 UTM GeoTIFF 格式.', + formatter_class=argparse.RawTextHelpFormatter, + epilog=INTRODUCTION + '\n' + EXAMPLE) + + parser.add_argument('projectName', help='项目名称') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, + help='并行处理器数量') + parser.add_argument('--ifgramList-txt', dest='ifgarmListTxt', + help='干涉图列表文件. 默认: projectName/ifgram_list.txt') + parser.add_argument('--output-dir', dest='output_dir', default=None, + help='输出目录. 默认: projectDir/Hyp3Products/') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + projectDir = scratchDir + '/' + projectName + ifgDir = projectDir + '/ifgrams' + templateDict = ut.update_template(templateFile) + masterDate = templateDict['masterDate'] + + # 输出目录 + output_dir = inps.output_dir or (projectDir + '/Hyp3Products') + os.makedirs(output_dir, exist_ok=True) + + # 读取干涉图列表 + if inps.ifgarmListTxt: + ifgramList_txt = inps.ifgarmListTxt + else: + ifgramList_txt = projectDir + '/ifgram_list.txt' + + ifgList0 = ut.read_txt2array(ifgramList_txt) + + if len(ifgList0) == 3: + ifgList = ifgList0[0] + ifgList = [ifgList] + else: + ifgList = ifgList0[:,0] + + # 错误日志 + err_txt = projectDir + '/hyp3format_gamma_all.err' + if os.path.isfile(err_txt): + os.remove(err_txt) + + # 构建并行命令 + data_para = [] + skip_count = 0 + for i in range(len(ifgList)): + Pair = ifgList[i] + + # 检查输出是否已存在 (用 corr.tif 作为完成标记) + pair_dir = output_dir + '/' + Pair + # 匹配 HyP3 命名: {prefix}_corr.tif + geo_file0 = pair_dir + '/' + projectName + '_' + Pair.replace('-', 'T000000_') + 'T000000_corr.tif' + + if os.path.isfile(geo_file0) and os.path.getsize(geo_file0) > 0: + skip_count += 1 + continue + + cmd0 = ['hyp3format_gamma.py', projectName, Pair] + if inps.output_dir: + cmd0.extend(['--output-dir', inps.output_dir]) + + data0 = [cmd0, err_txt] + data_para.append(data0) + + print(f"HyP3 格式转换: 共 {len(ifgList)} 对, 跳过 {skip_count} 对, 待处理 {len(data_para)} 对") + print(f"并行数: {inps.parallelNumb}, 输出目录: {output_dir}") + + # 并行执行 + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + + print("HyP3 格式转换完成: project %s" % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/ionosphere_gamma.py b/third_party/PyINT/pyint/ionosphere_gamma.py new file mode 100644 index 0000000..61ffab6 --- /dev/null +++ b/third_party/PyINT/pyint/ionosphere_gamma.py @@ -0,0 +1,411 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +from pyint import _utils as ut + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Calculate the ionospheric phases in an interferogram based on RSI + +''' + +EXAMPLE = ''' + Usage: + ionosphere_gamma.py projectName Mdate Sdate + ionosphere_gamma.py PacayaT163TsxHhA 20150102 20150601 +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Calculate the ionospheric phases in an interferogram based on RSI.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('Mdate',help='Master date.') + parser.add_argument('Sdate',help='Slave date.') + + inps = parser.parse_args() + return inps + + +def main(argv): + + start_time = time.time() + inps = cmdLineParse() + Mdate = inps.Mdate + Sdate = inps.Sdate + + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + + projectDir = scratchDir + '/' + projectName + demDir = scratchDir + '/' + projectName + '/DEM' + + slcDir = scratchDir + '/' + projectName + '/SLC' + rslcDir = scratchDir + '/' + projectName + '/RSLC' + ifgDir = projectDir + '/ifgrams' + if not os.path.isdir(ifgDir): os.mkdir(ifgDir) + + Pair = Mdate + '-' + Sdate + workDir = ifgDir + '/' + Pair + if not os.path.isdir(workDir): os.mkdir(workDir) + + ####################################################################### + Mamp0 = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar0 = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + Samp0 = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar0 = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + Mrslc = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc' + MrslcPar = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc.par' + Srslc = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc' + SrslcPar = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc.par' + + HGT0 = demDir + '/' + masterDate + '_' + rlks + 'rlks.rdc.dem' + + MasterPar0 = rslcDir + '/' + masterDate + '/' + masterDate + '.rslc.par' + + ################# copy file for parallel processing ########################## + Mamp = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + Samp = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + #Mrslc = workDir + '/' + Mdate + '.rslc' + #MrslcPar = workDir + '/' + Mdate + '.rslc.par' + #Srslc = workDir + '/' + Sdate + '.rslc' + #SrslcPar = workDir + '/' + Sdate + '.rslc.par' + + HGT = workDir + '/' + masterDate + '_' + rlks + 'rlks.rdc.dem' + MasterPar = workDir + '/' + masterDate + '.rslc.par' + + if not os.path.isfile(Mamp):ut.copy_file(Mamp0,Mamp) + if not os.path.isfile(MampPar):ut.copy_file(MampPar0,MampPar) + if not os.path.isfile(Samp):ut.copy_file(Samp0,Samp) + if not os.path.isfile(SampPar):ut.copy_file(SampPar0,SampPar) + + #if not os.path.isfile(Mrslc): ut.copy_file(Mrslc0,Mrslc) + #if not os.path.isfile(MrslcPar): ut.copy_file(MrslcPar0,MrslcPar) + #if not os.path.isfile(Srslc):ut.copy_file(Srslc0,Srslc) + #if not os.path.isfile(SrslcPar):ut.copy_file(SrslcPar0,SrslcPar) + + if not os.path.isfile(HGT):ut.copy_file(HGT0,HGT) + if not os.path.isfile(MasterPar):ut.copy_file(MasterPar0,MasterPar) + + + nWidth = ut.read_gamma_par(MampPar, 'read', 'range_samples') + nLine = ut.read_gamma_par(MampPar, 'read', 'azimuth_lines') + + ###################### Output files ################################# + ionoDir = workDir + '/ionosphere' + if not os.path.isdir(ionoDir): + os.mkdir(ionoDir) + + workDir = ionoDir + Mrslc_low = ionoDir + '/' + Mdate + '.low.rslc' + MrslcPar_low = ionoDir + '/' + Mdate + '.low.rslc.par' + Mrslc_high = ionoDir + '/' + Mdate + '.high.rslc' + MrslcPar_high = ionoDir + '/' + Mdate + '.high.rslc.par' + + Srslc_low = ionoDir + '/' + Sdate + '.low.rslc' + SrslcPar_low = ionoDir + '/' + Sdate + '.low.rslc.par' + Srslc_high = ionoDir + '/' + Sdate + '.high.rslc' + SrslcPar_high = ionoDir + '/' + Sdate + '.high.rslc.par' +######################################################### + TEST0 = MrslcPar_high + k0 = 0 + if os.path.isfile(TEST0): + if os.path.getsize(TEST0) > 0: + k0 =1 + + TEST1 = SrslcPar_high + k1 = 0 + if os.path.isfile(TEST1): + if os.path.getsize(TEST1) > 0: + k1 =1 + + if k0 ==0: + call_str = 'bpf_ssi ' + Mrslc + ' ' + MrslcPar + ' ' + Mrslc_low + ' ' + MrslcPar_low + ' ' + Mrslc_high + ' ' + MrslcPar_high + os.system(call_str) + + if k1==0: + call_str = 'bpf_ssi ' + Srslc + ' ' + SrslcPar + ' ' + Srslc_low + ' ' + SrslcPar_low + ' ' + Srslc_high + ' ' + SrslcPar_high + os.system(call_str) + + ################## interferometry ##################### + + off_low = workDir + '/' + Pair +'_' + rlks + 'rlks.low.off' + call_str = 'create_offset '+ MrslcPar_low + ' ' + SrslcPar_low + ' ' + off_low + ' 1 ' + rlks + ' ' + azlks + ' 0' + os.system(call_str) + + sim_low_unw = workDir + '/' + Pair + '.low.sim_unw' + call_str = 'phase_sim_orb ' + MrslcPar_low + ' ' + SrslcPar_low + ' ' + off_low + ' ' + HGT + ' ' + sim_low_unw + ' ' + MasterPar + ' - - 1 1' + os.system(call_str) + + diff_low = workDir + '/' + Pair + '_' + rlks + 'rlks.low.diff' + call_str = 'SLC_diff_intf ' + Mrslc_low + ' ' + Srslc_low + ' ' + MrslcPar_low + ' ' + SrslcPar_low + ' ' + off_low + ' ' + sim_low_unw + ' ' + diff_low + ' ' + rlks + ' ' + azlks + ' ' + ' 1 0 0.25' + os.system(call_str) + + + off_high = workDir + '/' + Pair +'_' + rlks + 'rlks.high.off' + call_str = 'create_offset '+ MrslcPar_high + ' ' + SrslcPar_high + ' ' + off_high + ' 1 ' + rlks + ' ' + azlks + ' 0' + os.system(call_str) + + sim_high_unw = workDir + '/' + Pair + '.high.sim_unw' + call_str = 'phase_sim_orb ' + MrslcPar_high + ' ' + SrslcPar_high + ' ' + off_high + ' ' + HGT + ' ' + sim_high_unw + ' ' + MasterPar + ' - - 1 1' + os.system(call_str) + + diff_high = workDir + '/' + Pair + '_' + rlks + 'rlks.high.diff' + call_str = 'SLC_diff_intf ' + Mrslc_high + ' ' + Srslc_high + ' ' + MrslcPar_high + ' ' + SrslcPar_high + ' ' + off_high + ' ' + sim_high_unw + ' ' + diff_high + ' ' + rlks + ' ' + azlks + ' ' + ' 1 0 0.25' + os.system(call_str) + + ###################################################### + hl_diff = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff' + hl_diff_cc = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff.cc' + hl_diff_mask = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff.cc_mask.bmp' + + hl_diff1 = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff1' + hl_diff2 = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff2' + hl_diff3 = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff3' + hl_diff4 = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff4' + + hl_diff4_phase = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff4_phase' + + hl_diff_jpg = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff.jpg' + + diff_low_phase = workDir + '/' + Pair + '_' + rlks + 'rlks.low.diff_phase' + call_str = 'cpx_to_real ' + diff_low + ' ' + diff_low_phase + ' ' + nWidth + ' 4' + os.system(call_str) + + call_str = 'subtract_phase ' + diff_high + ' ' + diff_low_phase + ' ' + hl_diff + ' ' + nWidth + ' 1' + os.system(call_str) + + call_str = 'rasmph_pwr ' + Mamp + ' ' + hl_diff + ' ' + nWidth + #call_str = 'vismph_pwr.py ' + hl_diff + ' ' + Mamp + ' ' + nWidth + ' -f 1.0 ' + ' -z 1000 ' + ' -p ' + hl_diff_jpg + os.system(call_str) + + #### filter + + call_str = 'adf ' + hl_diff + ' ' + hl_diff1 + ' ' + hl_diff_cc + ' ' + nWidth + ' 0.2 512 7 128' + os.system(call_str) + + call_str = 'adf ' + hl_diff1 + ' ' + hl_diff2 + ' ' + hl_diff_cc + ' ' + nWidth + ' 0.3 256 7 64' + os.system(call_str) + + call_str = 'adf ' + hl_diff2 + ' ' + hl_diff3 + ' ' + hl_diff_cc + ' ' + nWidth + ' 0.3 128 7 64' + os.system(call_str) + + call_str = 'adf ' + hl_diff3 + ' ' + hl_diff4 + ' ' + hl_diff_cc + ' ' + nWidth + ' 0.3 128 7 16' + os.system(call_str) + + # unwrawp + call_str = 'cpx_to_real ' + hl_diff4 + ' ' + hl_diff4_phase + ' ' + nWidth + ' 4' + os.system(call_str) + + # shift to zero + # mask very low coherence + + call_str = 'cc_ad ' + hl_diff + ' ' + Mamp + ' ' + Samp + ' - - ' + hl_diff_cc + ' ' + nWidth + ' 3 9' + os.system(call_str) + + call_str = 'rascc_mask ' + hl_diff_cc + ' ' + Mamp + ' ' + nWidth + ' 1 1 0 1 1 0.15 0.0 0.1 0.9 1.0 0.35 1 ' + hl_diff_mask + os.system(call_str) + + nWidth_half = str(float(nWidth)/2) + image_report = workDir + '/' + Pair + '.image_stat.report' + + #call_str = 'image_stat ' + hl_diff4_phase + ' ' + nWidth + ' ' + nWidth_half + ' 200 200 ' + image_report + call_str = 'image_stat ' + hl_diff4_phase + ' ' + nWidth + ' 0 0 ' + ' 200 200 ' + image_report + os.system(call_str) + + mean_phase = ut.read_gamma_par(image_report, 'read', 'mean') + std_phase = ut.read_gamma_par(image_report, 'read', 'stdev') + + + hl_diff_phase_shift = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff.ph_shift' + hl_diff_shifted = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff4.shifted' + hl_diff_shifted_phase = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff4.shifted.phase' + call_str = 'lin_comb 1 ' + hl_diff4_phase + ' ' + mean_phase + ' 0.0 ' + hl_diff_phase_shift + ' ' + nWidth + ' 1 ' + nLine + os.system(call_str) + + call_str = 'subtract_phase ' + hl_diff4_phase + ' ' + hl_diff_phase_shift + ' ' + hl_diff_shifted + ' ' + nWidth + ' 1' + os.system(call_str) + + call_str = 'cpx_to_real ' + hl_diff_shifted + ' ' + hl_diff_shifted_phase + ' ' + nWidth + ' 4' + os.system(call_str) + + call_str = 'lin_comb 1 ' + hl_diff_shifted_phase + ' ' + mean_phase + ' 1.0 ' + hl_diff4_phase + ' ' + nWidth + ' 1 ' + nLine + os.system(call_str) + + ############# determine trend considering cc_mask using unwrapping with multi_cpx before unwrapping ### + + diff_par = workDir + '/' + Pair + '.ddiff.par' + call_str = 'create_diff_par ' + MampPar + ' - ' + diff_par + ' 1 0' + os.system(call_str) + + ddiff_phase_trend = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff4_phase.trend' + call_str = 'quad_fit ' + hl_diff4_phase + ' ' + diff_par + ' 5 5 ' + hl_diff_mask + ' - 3 ' + ddiff_phase_trend + os.system(call_str) + + #call_str = 'vistd_pwr.py ' + ddiff_phase_trend + ' ' + Mamp + ' ' + nWidth + ' -c 12.6 -z 1000 -f 1.0 -m rmg -p ' + ddiff_phase_trend + '.jpg' + #os.system(call_str) + + hl_diff_detrend = hl_diff+ '.detrend' + call_str = 'subtract_phase ' + hl_diff + ' ' + ddiff_phase_trend + ' ' + hl_diff_detrend + ' ' + nWidth + ' 1.' + os.system(call_str) + + #call_str = 'vismph_pwr.py ' + hl_diff_detrend + ' ' + Mamp + ' ' + nWidth + ' -f 1.0 -z 1000 -p ' + hl_diff_detrend+'.jpg' + #os.system(call_str) + + + call_str = 'adf ' + hl_diff_detrend + ' ' + hl_diff1 + ' ' + hl_diff+'.smcc' + ' ' + nWidth + ' 0.2 512 7 128' + os.system(call_str) + + call_str = 'adf ' + hl_diff1 + ' ' + hl_diff2 + ' ' + hl_diff+'.smcc' + ' ' + nWidth + ' 0.3 256 7 64' + os.system(call_str) + + call_str = 'adf ' + hl_diff2 + ' ' + hl_diff3 + ' ' + hl_diff+'.smcc' + ' ' + nWidth + ' 0.3 128 7 64' + os.system(call_str) + + hl_diff4_detrend = hl_diff4+ '.detrend' + hl_diff_cc = hl_diff+'.sm4.cc' + call_str = 'adf ' + hl_diff3 + ' ' + hl_diff4_detrend + ' ' + hl_diff_cc + ' ' + nWidth + ' 0.3 128 7 16' + os.system(call_str) + + off1 = workDir + '/' + Pair + '.off1' + call_str = 'create_offset ' + MampPar + ' ' + MampPar + ' ' + off1 + ' 1 1 1 0' + os.system(call_str) + + hl_diff4_mask = hl_diff+'.sm4.cc_mask.bmp' + call_str = 'rascc_mask ' + hl_diff_cc + ' ' + Mamp + ' ' + nWidth + ' 1 1 0 1 1 0.95 0.0 0.1 0.1 0.9 1.0 0.35 1 ' + hl_diff4_mask + os.system(call_str) + + call_str = 'mask_class ' + hl_diff_mask + ' ' + hl_diff4_detrend + ' ' + hl_diff4_detrend + '.masked.tmp ' + ' 1 1 1 1 0 0.0 0.0' + os.system(call_str) + + call_str = 'mask_class ' + hl_diff4_mask + ' ' + hl_diff4_detrend + '.masked.tmp ' + ' ' + hl_diff4_detrend + '.masked' + ' 1 1 1 1 0 0.0 0.0' + os.system(call_str) + + + ddiff5 = workDir + '/' + Pair + '_' + rlks + 'rlks.hl.diff5' + off5 = workDir + '/' + Pair + '.off5' + call_str = 'multi_cpx ' + hl_diff4_detrend + '.masked' + ' ' + off1 + ddiff5 + ' ' + off5 + ' 5 5' + os.system(call_str) + + call_str = 'multi_real ' + Mamp + ' ' + off1 + ' ' + Mamp + '.5' + off5 + ' 5 5' + os.system(call_str) + + call_str = 'multi_real ' + hl_diff_cc + ' ' + off1 + ' ' + hl_diff_cc + '.5' + off5 + ' 5 5' + os.system(call_str) + + nWidth5 = ut.read_gamma_par(off5,'read','interferogram_width') + + ddiff5_phase_tmp = ddiff5 + '.phase.tmp' + ddiff5_phase_interp = ddiff5 + '.phase.interp' + call_str = 'cpx_to_real ' + ddiff5 + ' ' + ddiff5_phase_tmp + ' ' + nWidth5 + ' 4' + os.system(call_str) + + call_str = 'fill_gaps ' + ddiff5_phase_tmp + ' ' + nWidth5 + ' ' + ddiff5_phase_interp + ' 0 4 - 1 100 4 400' + os.system(call_str) + + + #### remove outliers + ddiff5_fspf = ddiff5_phase_interp + '.fspf' + call_str = 'fspf ' + ddiff5_phase_interp + ' ' + ddiff5_fspf + ' ' + nWidth5 + ' 2 64 3' + os.system(call_str) + + ddiff5_phase_interp_outliers = ddiff5_phase_interp + '.outliers' + call_str = 'lin_comb 2 ' + ddiff5_phase_interp + ' ' + ddiff5_fspf + ' 100 1.0 -1.0 ' + ddiff5_phase_interp_outliers + ' ' + nWidth5 + os.system(call_str) + + hl_diff_cc5 = hl_diff_cc + '.5' + hl_diff5_mask = ddiff5 + '.mask.bmp' + call_str = 'single_class_mapping 2 ' + ddiff5_phase_interp_outliers + ' 99.95 100.05 ' + hl_diff_cc5 + ' 0.15 1.0 ' + hl_diff5_mask + ' ' + nWidth5 + os.system(call_str) + + ddiff5_phase_tmp1 = ddiff5_phase_tmp + '1' + call_str = 'mask_class ' + hl_diff5_mask + ' ' + ddiff5_phase_tmp + ' ' + ddiff5_phase_tmp1 + ' 0 1 1 1 0 0.0 0.0' + os.system(call_str) + + call_str = 'fill_gaps ' + ddiff5_phase_tmp1 + ' ' + ddiff5_phase_interp + ' 0 4 - 1 100 4 400' + os.system(call_str) + + ddiff_detrend_interp_phase = hl_diff + '.detrend' + '.interp.phase' + call_str = 'multi_real ' + ddiff5_phase_interp + ' ' + off5 + ' ' + ddiff_detrend_interp_phase + ' ' + off1 + ' -5 -5' + os.system(call_str) + + call_str = 'fspf ' + ddiff_detrend_interp_phase + ' ' + ddiff_detrend_interp_phase + '.fspf' + ' ' + nWidth + ' 2 8 3' + os.system(call_str) + + #call_str = ' visdt_pwr.py ' + ddiff_detrend_interp_phase + '.fspf' + ' ' + Mamp + ' ' + nWidth + ' -c 1.6 -z 1000 -f 1.0 -m rmg -p ' + ddiff_detrend_interp_phase + '.fspf.jpg' + #os.system(call_str) + + + ###### add the solutions + + ddiff_phase_fspf = hl_diff + ' .phase.fspf' + call_str = 'lin_comb 2 ' + ddiff_phase_trend + ' ' + ddiff_detrend_interp_phase + '.fspf' + ' 0.0 1.0 1.0 ' + ddiff_phase_fspf + ' ' + nWidth + ' 1 ' + nLine + os.system(call_str) + + #call_str ='visdt_pwr.py ' + ddiff_phase_fspf + ' ' + Mamp + ' ' + nWidth + ' -c 1.6 -z 1000 -f 1.0 -m rmg -p ' + ddiff_phase_fspf + '.jpg' + #os.system(call_str) + + + ###################### determin scaling factor using bpf_ssi + + bpf_ssi_out = workDir + '/bpf_ssi.out' + call_str = 'bpf_ssi ' + Mrslc + ' ' + MrslcPar + ' - - - - 0.6666 > ' + bpf_ssi_out + os.system(call_str) + + a0 = ut.read_gamma_par(bpf_ssi_out,'read','a') + b0 = ut.read_gamma_par(bpf_ssi_out,'read','b') + x0 = ut.read_gamma_par(bpf_ssi_out,'read','x') + y0 = ut.read_gamma_par(bpf_ssi_out,'read','y') + z0 = ut.read_gamma_par(bpf_ssi_out,'read','z') + zz0 = str(float(z0)*2) + + ddiff_phase_fspf_scaled = ddiff_phase_fspf + '.scaled' + call_str = 'lin_comb 1 ' + ddiff_phase_fspf + ' 0.0 ' + zz0 + ' ' + ddiff_phase_fspf_scaled + ' ' + nWidth + os.system(call_str) + + call_str = 'rasrmg ' + ddiff_phase_fspf_scaled + ' ' + Mamp + ' ' + nWidth + #call_str = 'vismph_pwr.py ' + hl_diff + ' ' + Mamp + ' ' + nWidth + ' -f 1.0 ' + ' -z 1000 ' + ' -p ' + hl_diff_jpg + os.system(call_str) + + #call_str ='visdt_pwr.py ' + ddiff_phase_fspf_scaled + ' ' + Mamp + ' ' + nWidth + ' -c 1.6 -z 1000 -f 1.0 -m rmg -p ' + ddiff_phase_fspf_scaled + '.jpg' + #os.system(call_str) + + # 2_phi_iono = phi0 + zz * ddiff_phase_fspf + # 2_phi_non-dispersive = phi0 - zz * ddiff_phase_fspf + + #call_str = ' subtract_phase ' + + + + print("Estimating the scaled ionospheric phases is done!") + ut.print_process_time(start_time, time.time()) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/load_data_gamma.py b/third_party/PyINT/pyint/load_data_gamma.py new file mode 100644 index 0000000..d3a567d --- /dev/null +++ b/third_party/PyINT/pyint/load_data_gamma.py @@ -0,0 +1,643 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Central South University & University of Miami ### +################################################################# + +import os +import sys +import glob +import time +import argparse +from mintpy.utils import readfile, writefile, ptime, utils as ut +import h5py +import numpy as np + +def print_progress(iteration, total, prefix='calculating:', suffix='complete', decimals=1, barLength=50, elapsed_time=None): + """Print iterations progress - Greenstick from Stack Overflow + Call in a loop to create terminal progress bar + @params: + iteration - Required : current iteration (Int) + total - Required : total iterations (Int) + prefix - Optional : prefix string (Str) + suffix - Optional : suffix string (Str) + decimals - Optional : number of decimals in percent complete (Int) + barLength - Optional : character length of bar (Int) + elapsed_time- Optional : elapsed time in seconds (Int/Float) + + Reference: http://stackoverflow.com/questions/3173320/text-progress-bar-in-the-console + """ + filledLength = int(round(barLength * iteration / float(total))) + percents = round(100.00 * (iteration / float(total)), decimals) + bar = '#' * filledLength + '-' * (barLength - filledLength) + if elapsed_time: + sys.stdout.write('%s [%s] %s%s %s %s secs\r' % (prefix, bar, percents, '%', suffix, int(elapsed_time))) + else: + sys.stdout.write('%s [%s] %s%s %s\r' % (prefix, bar, percents, '%', suffix)) + sys.stdout.flush() + if iteration == total: + print("\n") + + ''' + Sample Useage: + for i in range(len(dateList)): + print_progress(i+1,len(dateList)) + ''' + return + + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def read_data(inFile, dtype, nWidth, nLength): + data = np.fromfile(inFile, dtype, int(nLength)*int(nWidth)).reshape(int(nLength),int(nWidth)) + return data + +def is_number(s): + try: + int(s) + return True + except ValueError: + return False + +def add_zero(s): + if len(s)==1: + s="000"+s + elif len(s)==2: + s="00"+s + elif len(s)==3: + s="0"+s + return s + +def read_list(ListName): + List=glob.glob(ListName) + return List + +def read_list_except(ListName,ExString): + List=glob.glob(ListName) + List_New = [] + for S in List: + if ExString not in os.path.basename(S): + List_New.append(S) + return List_New + + +def load_gamma2multi_group_h5(fileType, fileName, fileList, RSCList, datatype): + + RSC =RSCList[0] + rsc_dic = read_roipac_rsc(RSC) + nWidth = rsc_dic['WIDTH'] + nLine = rsc_dic['FILE_LENGTH'] + + H5FILE = fileName + '.h5' + fileNum = len(fileList) + print('Start to load ' + fileName + ' >>> %s %s files will be loaded for further process' % ( str(fileNum), fileType)) + f = h5py.File(H5FILE,'w') + gg=f.create_group(fileType) + + for i in range(len(fileList)): + S=fileList[i] + RSC =RSCList[i] + rsc_dic = read_roipac_rsc(RSC) + print_progress(i+1, fileNum, prefix='loading', suffix=os.path.basename(S)) + data = read_data(S,datatype,nWidth,nLine) + File = os.path.basename(S) + group = gg.create_group(File) + dset = group.create_dataset(File, data = data, compression='gzip') + for key,value in rsc_dic.items(): + group.attrs[key] = value + group.attrs['X_MIN'] = '0' + group.attrs['X_MAX'] = str(int(nWidth)-1) + group.attrs['Y_MIN'] = '0' + group.attrs['Y_MAX'] = str(int(nLine)-1) + group.attrs['DATE12']=File.split('_')[2] + group.attrs['DATATYPE'] = datatype + group.attrs['PROCCESSOR'] = 'gamma' + + f.close() + + +def Get_Datelist(projectName): + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + rslcDir = scratchDir + '/' + projectName + "/RSLC" + slcDir = scratchDir + '/' + projectName + "/SLC" + + ListSLC = os.listdir(slcDir) + Datelist = [] + + for kk in range(len(ListSLC)): + if ( is_number(ListSLC[kk]) and len(ListSLC[kk])==6 ): # if SAR date number is 8, 6 should change to 8. + DD=ListSLC[kk] + Year=int(DD[0:2]) + Month = int(DD[2:4]) + Day = int(DD[4:6]) + if ( 0 < Year < 20 and 0 < Month < 13 and 0 < Day < 32 ): + Datelist.append(ListSLC[kk]) + Datelist = list(map(int,Datelist)) + Datelist.sort() + Datelist = list(map(str,Datelist)) + return Datelist + +def Get_Inflist(projectName): + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + IFGRAM = scratchDir + '/' + projectName + "/PROCESS/IFG*" + IFGList = glob.glob(IFGRAM) + + ListSLC = os.listdir(slcDir) + Datelist = [] + + +def Get_PairName(projectName): + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + IFGRAM = scratchDir + '/' + projectName + "/PROCESS/IFG*" + IFGList = glob.glob(IFGRAM) + + PAIR_Name = [] + for kk in range(len(IFGList)): + SS = IFGList[kk] + IFGName = os.path.basename(SS) + PAIR=IFGName.split('_')[2] + PAIR_Name.append(PAIR) + + return PAIR_Name + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +def UseGamma2(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[2].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +def Remove_Inf_List(RemoveNumberStr): + AD=RemoveNumberStr + Addlist = [] + if len(AD)>0: + AD= AD.split('[')[1].split(']')[0] + if ',' in AD: + LL=AD.split(',') + for kk in range(len(LL)): + XX=LL[kk] + if is_number(XX): + Addlist.append(XX) + + else: + D1=XX.split(':')[0] + D2=XX.split(':')[1] + for jj in range(int(D1),int(D2)+1): + Addlist.append(str(jj)) + else: + LL = AD + if is_number(LL): + Addlist.append(LL) + else: + D1=LL.split(':')[0] + D2=LL.split(':')[1] + for jj in range(int(D1),int(D2)+1): + Addlist.append(str(jj)) + for ii in range(len(Addlist)): + Addlist[ii]=int(Addlist[ii])-1 + Addlist = list(map(str,Addlist)) + return Addlist + + +def read_roipac_rsc(File): + '''Read ROI_PAC .rsc file into a python dictionary structure.''' + rsc_dict = dict(np.loadtxt(File, dtype=str, usecols=(0,1))) + return rsc_dict + + +######################################################################### + +INTRODUCTION = ''' +############################################################################# + Copy Right(c): 2017, Yunmeng Cao @PyINT v1.0 + + Loading data for PyMIS processing. + +''' + +EXAMPLE= ''' + Usage: + load_data_pymis.py projectName + load_data_pymis.py projectName --demRdc + load_data_pymis.py projectName --wrapIfgram + + Examples: + load_data_pymis.py PacayaT163TsxHhA + load_data_pymis.py PacayaT163TsxHhA --demRdc + load_data_pymis.py PacayaT163TsxHhA --coherence + +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Loading data for PyMIS processing.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='project name of PyMIS.') + parser.add_argument('--coherence',action="store_true", default=False, help='Loading coherence file.') + parser.add_argument('--wrapIfgram',action="store_true", default=False, help='Loading wrapped interferograms.') + parser.add_argument('--unwrapIfgram',action="store_true", default=False, help='Loading unwrapped interferograms.') + parser.add_argument('--demRdc',action="store_true", default=False, help='Loading radar coordinates DEM.') + parser.add_argument('--demGeo',action="store_true", default=False, help='Loading GEO coordinates DEM.') + parser.add_argument('--geo2rdc',action="store_true", default=False, help='Loading geocoding lookup table.') + + inps = parser.parse_args() + + + + return inps + +################################################################################ + +def main(argv): + + + total = time.time() + inps = cmdLineParse() + projectName = inps.projectName + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateContents=read_template(templateFile) + + processDir = scratchDir + '/' + projectName + "/ifgrams" + #processDir = scratchDir + '/' + projectName + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + "/RSLC" + #simDir = scratchDir + '/' + projectName + "/PROCESS/DEM" + simDir = scratchDir + '/' + projectName + "/DEM" + + + masterDate = templateContents['masterDate'] + rlks = templateContents['range_looks'] + azlks = templateContents['azimuth_looks'] + + workDir = scratchDir + '/' + projectName + '/TSSAR' + if not os.path.isdir(workDir): + call_str='mkdir ' + workDir + os.system(call_str) + + if not os.path.isdir(workDir): + call_str='mkdir ' + workDir + os.system(call_str) + + print('Project : ' + projectName) + print('Loading data for timeseries processing >>>') + print('Change process Dir to :' + workDir) + os.chdir(workDir) + + +################# DEM Define ############ + + + + + UTM2RDC = simDir+'/' + masterDate + '_' + rlks + 'rlks.UTM_TO_RDC' + RDCDEM = simDir+'/' + masterDate + '_' + rlks + 'rlks.rdc.dem' + UTMDEM = simDir+'/' + masterDate + '_' + rlks + 'rlks.utm.dem' + UTMPAR = simDir+'/' + masterDate + '_' + rlks + 'rlks.utm.dem.par' + + + +# if not os.path.isfile(UTM2RDC): +# print 'Start to generate subset look-up table and subset-DEM >>>' +# call_str = 'CreateRdcDem_Sub_Gammapy.py ' + igramDir +# print call_str +# os.system(call_str) +# else: +# print 'Subet lookup table %s is existed!' % UTM2RDC + + Dem_Format = UseGamma(UTMPAR, 'read', 'data_format:') + + if Dem_Format=='REAL*4': + dtype_utmdem='f4' + else: + dtype_utmdem='i2' + + nWidthUTM = UseGamma(UTMPAR, 'read', 'width:') + nLineUTM = UseGamma(UTMPAR, 'read', 'nlines:') + + Corner_LAT = UseGamma(UTMPAR, 'read', 'corner_lat:') + Corner_LON = UseGamma(UTMPAR, 'read', 'corner_lon:') + + Corner_LAT =Corner_LAT.split(' ')[0] + Corner_LON =Corner_LON.split(' ')[0] + + post_Lat = UseGamma(UTMPAR, 'read', 'post_lat:') + post_Lon = UseGamma(UTMPAR, 'read', 'post_lon:') + + post_Lat =post_Lat.split(' ')[0] + post_Lon =post_Lon.split(' ')[0] + +############### timeseries file define ################## + + if 'IFG_list' in templateContents: IFG_list = templateContents['IFG_list'] + else: IFG_list = processDir + '/*/diff_filt*rlks.int' + if 'UNW_list' in templateContents: UNW_list = templateContents['UNW_list'] + else: UNW_list = processDir + '/*/diff_filt*rlks.unw' + if 'COR_list' in templateContents: COR_list = templateContents['COR_list'] + else: COR_list = processDir + '/*/filt*rlks.cor' + if 'MASK_list' in templateContents: MASK_list = templateContents['MASK_list'] + # else: MASK_list = processDir + '/*/diff_filt*rlks.cor_mask' + else: MASK_list = processDir + '/*/*.cor_mask.bmp' + if 'RSC_list' in templateContents: RSC_list = templateContents['RSC_list'] + else: RSC_list = processDir + '/*/diff_filt_*unw.rsc' + + MM=MASK_list[0] + if MM.split('.')[len(MM.split('.'))-1]=='bmp': + dtype_mask ='u1' + else: + dtype_mask ='f4' + + if 'Byte_order' in templateContents: byteoder = templateContents['Byte_order'] + else: byteorder = 'big' + + if byteorder =='big': + sign = '>' + else: + sign ='<' + + dtype_inf = sign + 'c8' + dtype_unw = sign + 'f4' + dtype_utmdem = sign + dtype_utmdem + dtype_rdcdem = sign + 'f4' + dtype_lt = sign + 'c8' + dtype_mask = sign + dtype_mask + dtype_cor = sign + 'f4' + +#################### start to load data ######################## + # InfList = read_list(IFG_list) + InfList = ut.get_file_list(IFG_list, abspath=True) + NInf = len(InfList) + + print('RSC_Name is : ' + RSC_list) + # RSCList = read_list(RSC_list) + RSCList = ut.get_file_list(RSC_list, abspath=True) + RSC = RSCList[0] + rsc_dic = read_roipac_rsc(RSC) + nWidth = rsc_dic['WIDTH'] + nLine = rsc_dic['FILE_LENGTH'] + UNWList = ut.get_file_list(UNW_list, abspath=True) + CORList = ut.get_file_list(COR_list, abspath=True) + # UNWList = read_list(UNW_list) + # CORList = read_list(COR_list) + Flag_K = 0 + if inps.coherence: + Flag_K = 1 + print('') + print('COR_Name is : ' + COR_list) + load_gamma2multi_group_h5('coherence','coherence', CORList, RSCList, dtype_cor) + elif inps.wrapIfgram: + Flag_K = 1 + print('') + print('IFG_Name is : ' + IFG_list) + load_gamma2multi_group_h5('wrapped','wrapIfgram', InfList, RSCList, dtype_inf) + elif inps.unwrapIfgram: + Flag_K = 1 + print('') + print('UNW_Name is : ' + UNW_list) + load_gamma2multi_group_h5('interferograms','unwrapIfgram', UNWList, RSCList, dtype_unw) + elif inps.demGeo: + Flag_K = 1 + print('') + print('Start to write GEO-DEM into h5 file >>> ' + UTMDEM) + data = read_data(UTMDEM, dtype_utmdem, nWidthUTM, nLineUTM) + H5FILE = 'demGeo.h5' + f =h5py.File(H5FILE,'w') + group=f.create_group('dem') + dset = group.create_dataset('dem', data=data, compression='gzip') + group.attrs['WIDTH'] = nWidthUTM + group.attrs['FILE_LENGTH'] = nLineUTM + group.attrs['X_FIRST'] = Corner_LON # X: latitude Y: longitude + group.attrs['Y_FIRST'] = Corner_LAT + group.attrs['X_STEP'] = post_Lon + group.attrs['Y_STEP'] = post_Lat + group.attrs['X_UNIT'] = 'degrees' + group.attrs['Y_UNIT'] = 'degrees' + group.attrs['X_MIN'] = '0' + group.attrs['X_MAX'] = str(int(nWidthUTM)-1) + group.attrs['Y_MIN'] = '0' + group.attrs['Y_MAX'] = str(int(nLineUTM)-1) + group.attrs['DATATYPE'] = dtype_utmdem + group.attrs['COORD'] = 'geo' + group.attrs['PROCCESSOR'] = 'gamma' + f.close() + elif inps.demRdc: + Flag_K = 1 + print('') + print('Start to write RDC-DEM into h5 file >>> ' + RDCDEM) + data = read_data(RDCDEM, dtype_rdcdem, nWidth, nLine) + H5FILE = 'demRdc.h5' + f =h5py.File(H5FILE,'w') + group=f.create_group('dem') + dset = group.create_dataset('dem', data=data, compression='gzip') + group.attrs['PROCCESSOR'] = 'gamma' + group.attrs['WIDTH'] = nWidth + group.attrs['FILE_LENGTH'] = nLine + group.attrs['X_MIN'] = '0' + group.attrs['X_MAX'] = str(int(nWidth)-1) + group.attrs['Y_MIN'] = '0' + group.attrs['Y_MAX'] = str(int(nLine)-1) + group.attrs['DATATYPE'] = dtype_rdcdem + group.attrs['COORD'] ='radar' + f.close() + elif inps.geo2rdc: + Flag_K = 1 + print('') + print('Start to write GEO2RDC into h5 file >>> ' + UTM2RDC) + data = read_data(UTM2RDC,dtype_lt,nWidthUTM,nLineUTM) # real: range imaginary: azimuth + H5FILE = 'geo2rdc.h5' + f =h5py.File(H5FILE,'w') + group=f.create_group('lt') + dset = group.create_dataset('lt', data=data, compression='gzip') + group.attrs['WIDTH'] = nWidthUTM + group.attrs['FILE_LENGTH'] = nLineUTM + group.attrs['X_FIRST'] = Corner_LON # X: latitude Y: longitude + group.attrs['Y_FIRST'] = Corner_LAT + group.attrs['X_STEP'] = post_Lon + group.attrs['Y_STEP'] = post_Lat + group.attrs['X_UNIT'] = 'degrees' + group.attrs['Y_UNIT'] = 'degrees' + group.attrs['X_MIN'] = '0' + group.attrs['X_MAX'] = str(int(nWidthUTM)-1) + group.attrs['Y_MIN'] = '0' + group.attrs['Y_MAX'] = str(int(nLineUTM)-1) + group.attrs['DATATYPE'] = dtype_utmdem + group.attrs['COORD'] = 'geo' + group.attrs['PROCCESSOR'] = 'gamma' + f.close() + + + + if not os.path.isfile('wrapIfgram.h5') and Flag_K==0: + print('') + print('IFG_Name is : ' + IFG_list) + load_gamma2multi_group_h5('wrapped','wrapIfgram', InfList, RSCList, dtype_inf) + else: + print('') + print('wrapIfgram.h5 has existed, loading wrapped interferograms is skipped.') + + + + if not os.path.isfile('unwrapIfgram.h5') and Flag_K==0: + print('') + print('UNW_Name is : ' + UNW_list) + load_gamma2multi_group_h5('interferograms','unwrapIfgram', UNWList, RSCList, dtype_unw) + else: + print('') + print('unwrapIfgram.h5 has existed, loading unwrapped interferograms is skipped.') + + + if not os.path.isfile('coherence.h5') and Flag_K==0: + print('') + print('COR_Name is : ' + COR_list) + load_gamma2multi_group_h5('coherence','coherence', CORList, RSCList, dtype_cor) + else: + print('') + print('coherence.h5 has existed, loading coherence is skipped.') + + +######### load DEM and lookup table ########## + + if not os.path.isfile('demGeo.h5') and Flag_K==0: + print('') + print('Start to write GEO-DEM into h5 file >>> ' + UTMDEM) + data = read_data(UTMDEM, dtype_utmdem, nWidthUTM, nLineUTM) + H5FILE = 'demGeo.h5' + f =h5py.File(H5FILE,'w') + group=f.create_group('dem') + dset = group.create_dataset('dem', data=data, compression='gzip') + group.attrs['WIDTH'] = nWidthUTM + group.attrs['FILE_LENGTH'] = nLineUTM + group.attrs['X_FIRST'] = Corner_LON # X: latitude Y: longitude + group.attrs['Y_FIRST'] = Corner_LAT + group.attrs['X_STEP'] = post_Lon + group.attrs['Y_STEP'] = post_Lat + group.attrs['X_UNIT'] = 'degrees' + group.attrs['Y_UNIT'] = 'degrees' + group.attrs['X_MIN'] = '0' + group.attrs['X_MAX'] = str(int(nWidthUTM)-1) + group.attrs['Y_MIN'] = '0' + group.attrs['Y_MAX'] = str(int(nLineUTM)-1) + group.attrs['DATATYPE'] = dtype_utmdem + group.attrs['COORD'] = 'geo' + group.attrs['PROCCESSOR'] = 'gamma' + f.close() + else: + print('') + print('demGeo.h5 has existed, loading GEO-DEM is skipped.') + + + if not os.path.isfile('demRdc.h5') and Flag_K==0: + print('') + print('Start to write RDC-DEM into h5 file >>> ' + RDCDEM) + data = read_data(RDCDEM, dtype_rdcdem, nWidth, nLine) + H5FILE = 'demRdc.h5' + f =h5py.File(H5FILE,'w') + group=f.create_group('dem') + dset = group.create_dataset('dem', data=data, compression='gzip') + group.attrs['PROCCESSOR'] = 'gamma' + group.attrs['WIDTH'] = nWidth + group.attrs['FILE_LENGTH'] = nLine + group.attrs['X_MIN'] = '0' + group.attrs['X_MAX'] = str(int(nWidth)-1) + group.attrs['Y_MIN'] = '0' + group.attrs['Y_MAX'] = str(int(nLine)-1) + group.attrs['DATATYPE'] = dtype_rdcdem + group.attrs['COORD'] ='radar' + f.close() + else: + print('') + print('demRdc.h5 has existed, loading RDC-DEM is skipped.') + + + if not os.path.isfile('geo2rdc.h5') and Flag_K==0: + print('') + print('Start to write GEO2RDC into h5 file >>> ' + UTM2RDC) + data = read_data(UTM2RDC,dtype_lt,nWidthUTM,nLineUTM) # real: range imaginary: azimuth + H5FILE = 'geo2rdc.h5' + f =h5py.File(H5FILE,'w') + group=f.create_group('lt') + dset = group.create_dataset('lt', data=data, compression='gzip') + group.attrs['WIDTH'] = nWidthUTM + group.attrs['FILE_LENGTH'] = nLineUTM + group.attrs['X_FIRST'] = Corner_LON # X: latitude Y: longitude + group.attrs['Y_FIRST'] = Corner_LAT + group.attrs['X_STEP'] = post_Lon + group.attrs['Y_STEP'] = post_Lat + group.attrs['X_UNIT'] = 'degrees' + group.attrs['Y_UNIT'] = 'degrees' + group.attrs['X_MIN'] = '0' + group.attrs['X_MAX'] = str(int(nWidthUTM)-1) + group.attrs['Y_MIN'] = '0' + group.attrs['Y_MAX'] = str(int(nLineUTM)-1) + group.attrs['DATATYPE'] = dtype_utmdem + group.attrs['COORD'] = 'geo' + group.attrs['PROCCESSOR'] = 'gamma' + f.close() + else: + print('') + print('geo2rdc.h5 has existed, loading lookup table is skipped.') + + print('') + print('Done.\nLoading data spend ' + str(time.time()-total) +' secs') + sys.exit(1) + + +############################################################################## +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/third_party/PyINT/pyint/load_mintpy.py b/third_party/PyINT/pyint/load_mintpy.py new file mode 100644 index 0000000..5be813a --- /dev/null +++ b/third_party/PyINT/pyint/load_mintpy.py @@ -0,0 +1,159 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Purpose: loading PyINT products for mintPy ### +### Copy Right (c): 2019, Yunmeng Cao ### +### Author : Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import time +import argparse +import glob + +from pyint import _utils as ut + +def write_template(str0,templateFile): + with open(templateFile, 'a') as f: + f.write(str0 + '\n') + return + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Loading pyint products for mintPy time-series analysis.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='name of the project.') + #parser.add_argument('--data-type', dest='dataType', type=str, default='big_endian',choices={'big_endian', 'little_endian'},help='data type, big endian or little endian. [default: big_endian]') + inps = parser.parse_args() + + return inps + + +INTRODUCTION = ''' +----------------------------------------------------------------- + Loading pyint products for mintPy time-series analysis. + +''' + +EXAMPLE = """Usage: + + load_mintpy.py projectName +----------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + scratchDir0 = os.getenv('SCRATCHDIR0') + projectDir = scratchDir + '/' + projectName + projectDir0 = scratchDir0 + '/' + projectName + demDir = scratchDir + '/' + projectName + '/DEM' + rslcDir = scratchDir + '/' + projectName + '/RSLC' + + ifgDir = projectDir + '/ifgrams' + #unwFile = projectDir + '/ifgrams/*/2*rlks.diff_filt.unw' # considering geocode unw file + unwFile = projectDir + '/ifgrams/*/diff_filt_*.unw' # considering geocode unw file + corFile = projectDir + '/ifgrams/*/diff_filt_*rlks.cor' + #corFile = projectDir + '/ifgrams/*/filt_*.cor' + print(unwFile) + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + MampPar = rslcDir + '/' + masterDate + '/' + masterDate + '_' + rlks + 'rlks.amp.par' + date_list = ut.get_project_slcList(projectName) + date_list = sorted(date_list) + slc_par_list = [rslcDir + '/' + date0 + '/' + date0 + '.rlsc.par' for date0 in date_list ] + ifgdir_list = glob.glob(ifgDir + '/*') + for k0 in ifgdir_list: + pair0 = os.path.basename(k0) + #print(pair0) + m0 = pair0.split('-')[0] + s0 = pair0.split('-')[1] + + workDir = k0 + mamppar0 = rslcDir + '/' + m0 + '/' + m0 + '_' + rlks + 'rlks.amp.par' + samppar0 = rslcDir + '/' + s0 + '/' + s0 + '_' + rlks + 'rlks.amp.par' + mrslcpar0 = rslcDir + '/' + m0 + '/' + m0 + '.rslc.par' + srslcpar0 = rslcDir + '/' + s0 + '/' + s0 + '.rslc.par' + + mamppar = workDir + '/' + m0 + '_' + rlks + 'rlks.amp.par' + samppar = workDir + '/' + s0 + '_' + rlks + 'rlks.amp.par' + mrslcpar = workDir + '/' + m0 + '.rslc.par' + srslcpar = workDir + '/' + s0 + '.rslc.par' + + ut.copy_file(mamppar0, mamppar) + ut.copy_file(samppar0, samppar) + ut.copy_file(mrslcpar0,mrslcpar) + ut.copy_file(srslcpar0,srslcpar) + + #unw_list = glob.glob(ifgDir + '/*/*rlks.diff_filt.unw') + #cor_list = glob.glob(ifgDir + '/*/*rlks.diff_filt.cor') + + dem_geo = glob.glob(demDir + '/*rlks.utm.dem')[0] + geo_par = glob.glob(demDir + '/*rlks.utm.dem.par')[0] + + dem_rdc = glob.glob(demDir + '/*_' + rlks + 'rlks.rdc.dem')[0] + rdc_par = glob.glob(demDir + '/*_' + rlks + 'rlks.diff_par')[0] # diff_par + lt = glob.glob(demDir + '/*_' + rlks + 'rlks.UTM_TO_RDC')[0] + + strPro = 'mintpy.load.processor = gamma' + strUNW = "mintpy.load.unwFile = " + unwFile + strCOR = 'mintpy.load.corFile = ' + corFile + strCon = 'mintpy.load.connCompFile = auto' + strInt = 'mintpy.load.intFile = auto' + strIon = 'mintpy.load.ionoFile = auto' + + strDem = 'mintpy.load.demFile = ' + dem_rdc + strDemGeo = 'mintpy.load.demFile = ' + dem_geo + strLtY = 'mintpy.load.lookupYFile = ' + lt + strLtX = 'mintpy.load.lookupXFile = ' + lt + strInc = 'mintpy.load.incAngleFile = auto' + strAza = 'mintpy.load.azAngleFile = auto' + strSha = 'mintpy.load.shadowMaskFile = auto' + strWat = 'mintpy.load.waterMaskFile = auto' + strBrp = 'mintpy.load.bperpFile = auto' + + templateFile0 = projectDir + '/mintpy.template' + + if 'mintpy.load.processor' not in templateDict: write_template(strPro,templateFile0) + if 'mintpy.load.unwFile' not in templateDict: write_template(strUNW,templateFile0) + if 'mintpy.load.corFile' not in templateDict: write_template(strCOR,templateFile0) + if 'mintpy.load.demFile' not in templateDict: write_template(strDem,templateFile0) + if 'mintpy.load.lookupYFile' not in templateDict: write_template(strLtY,templateFile0) + if 'mintpy.load.lookupXFile' not in templateDict: write_template(strLtX,templateFile0) + + if 'mintpy.load.connCompFile' not in templateDict: write_template(strCon,templateFile0) + if 'mintpy.load.intFile' not in templateDict: write_template(strInt,templateFile0) + if 'mintpy.load.ionoFile' not in templateDict: write_template(strIon,templateFile0) + + if 'mintpy.load.incAngleFile' not in templateDict: write_template(strInc,templateFile0) + if 'mintpy.load.azAngleFile' not in templateDict: write_template(strAza,templateFile0) + if 'mintpy.load.shadowMaskFile' not in templateDict: write_template(strSha,templateFile0) + if 'mintpy.load.waterMaskFile' not in templateDict: write_template(strWat,templateFile0) + if 'mintpy.load.bperpFile' not in templateDict: write_template(strBrp,templateFile0) + + os.chdir(projectDir0) + call_str = 'load_data.py -t ' + templateFile0 + os.system(call_str) + + os.chdir(projectDir0) + write_template(strDemGeo,templateFile0) + call_str = 'load_data.py -t ' + templateFile0 + os.system(call_str) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/third_party/PyINT/pyint/make_local_dem.py b/third_party/PyINT/pyint/make_local_dem.py new file mode 100644 index 0000000..0772460 --- /dev/null +++ b/third_party/PyINT/pyint/make_local_dem.py @@ -0,0 +1,565 @@ +#! /usr/bin/env python +################################################################# +### 从本地 FABDEM 瓦片库生成 GAMMA 格式 DEM ### +### 输入: 经纬度范围 + 本地 FABDEM ZIP 目录 ### +### 输出: .dem + .dem.par (与 makedem.py 输出完全一致) ### +### Author: Cascade AI + zouyuandong ### +### Date : 2026-03-15 ### +################################################################# + +import os +import sys +import re +import math +import glob +import zipfile +import tempfile +import shutil +import argparse +import subprocess +import numpy as np +from pathlib import Path + + +# ========================= GAMMA 参数文件写入 ========================= + +def write_dempar_file(filepath, corner_lon, corner_lat, post_lon, post_lat, + width, nlines, data_format='REAL*4'): + """写入 GAMMA DEM 参数文件 (.dem.par)，与 makedem.py 格式完全一致""" + with open(filepath, 'w') as f: + f.write("Gamma DIFF&GEO DEM/MAP parameter file\n") + f.write("title:\tIMPORTED DEM FROM FABDEM V1-2\n") + f.write("DEM_projection: EQA\n") + f.write("data_format: %s\n" % data_format) + f.write("DEM_hgt_offset: 0.00000\n") + f.write("DEM_scale: 1.00000\n") + f.write("width: %s\n" % str(int(width))) + f.write("nlines: %s\n" % str(int(nlines))) + f.write("corner_lat: %s decimal degrees\n" % str(float(corner_lat))) + f.write("corner_lon: %s decimal degrees\n" % str(float(corner_lon))) + f.write("post_lat: %s decimal degrees\n" % str(float(post_lat))) + f.write("post_lon: %s decimal degrees\n" % str(float(post_lon))) + f.write("\n") + f.write("ellipsoid_name: WGS 84\n") + f.write("ellipsoid_ra: 6378137.000 m\n") + f.write("ellipsoid_reciprocal_flattening: 298.2572236\n") + f.write("\n") + f.write("datum_name: WGS 1984\n") + f.write("datum_shift_dx: 0.000 m\n") + f.write("datum_shift_dy: 0.000 m\n") + f.write("datum_shift_dz: 0.000 m\n") + f.write("datum_scale_m: 0.00000e+00\n") + f.write("datum_rotation_alpha: 0.00000e+00 arc-sec\n") + f.write("datum_rotation_beta: 0.00000e+00 arc-sec\n") + f.write("datum_rotation_gamma: 0.00000e+00 arc-sec\n") + f.write("datum_country_list Global Definition, WGS84, World\n") + f.write("\n") + + +# ========================= FABDEM 瓦片索引逻辑 ========================= + +def parse_fabdem_zip_name(zip_name): + """解析 FABDEM ZIP 文件名，提取覆盖的经纬度范围 + + 示例: N30E120-N40E130_FABDEM_V1-2.zip → (lat_min=30, lon_min=120, lat_max=40, lon_max=130) + S10W080-S00W070_FABDEM_V1-2.zip → (lat_min=-10, lon_min=-80, lat_max=0, lon_max=-70) + """ + basename = Path(zip_name).stem # N30E120-N40E130_FABDEM_V1-2 + match = re.match( + r'([NS])(\d+)([EW])(\d+)-([NS])(\d+)([EW])(\d+)_FABDEM', + basename + ) + if not match: + return None + + lat1 = int(match.group(2)) * (1 if match.group(1) == 'N' else -1) + lon1 = int(match.group(4)) * (1 if match.group(3) == 'E' else -1) + lat2 = int(match.group(6)) * (1 if match.group(5) == 'N' else -1) + lon2 = int(match.group(8)) * (1 if match.group(7) == 'E' else -1) + + return { + 'lat_min': min(lat1, lat2), + 'lat_max': max(lat1, lat2), + 'lon_min': min(lon1, lon2), + 'lon_max': max(lon1, lon2), + } + + +def tile_name_for_coord(lat, lon): + """根据经纬度生成 FABDEM 1°×1° 瓦片文件名 + + 参数 lat, lon 为瓦片左下角整数坐标 + 示例: (31, 121) → 'N31E121_FABDEM_V1-2.tif' + (-1, -70) → 'S01W070_FABDEM_V1-2.tif' + """ + lat_prefix = 'N' if lat >= 0 else 'S' + lon_prefix = 'E' if lon >= 0 else 'W' + return f"{lat_prefix}{abs(lat):02d}{lon_prefix}{abs(lon):03d}_FABDEM_V1-2.tif" + + +def find_needed_tiles(west, south, east, north): + """计算覆盖目标区域所需的全部 1°×1° 瓦片坐标列表 + + 返回: [(lat, lon), ...] 瓦片左下角坐标 + """ + # 向下取整到最近的整数度 + lat_start = math.floor(south) + lat_end = math.ceil(north) # 不含 + lon_start = math.floor(west) + lon_end = math.ceil(east) + + tiles = [] + for lat in range(lat_start, lat_end): + for lon in range(lon_start, lon_end): + tiles.append((lat, lon)) + return tiles + + +def find_zip_for_tile(lat, lon, fabdem_dir): + """查找包含指定 1°×1° 瓦片的 ZIP 文件 + + FABDEM ZIP 按 10°×10° 分块，文件名编码了覆盖范围 + """ + fabdem_path = Path(fabdem_dir) + zip_files = sorted(fabdem_path.glob('*_FABDEM_V1-2.zip')) + + for zf in zip_files: + bounds = parse_fabdem_zip_name(zf.name) + if bounds is None: + continue + if (bounds['lat_min'] <= lat < bounds['lat_max'] and + bounds['lon_min'] <= lon < bounds['lon_max']): + return zf + return None + + +def extract_tiles_from_zips(tile_coords, fabdem_dir, extract_dir): + """从 FABDEM ZIP 文件中提取所需的 1°×1° GeoTIFF 瓦片 + + 参数: + tile_coords: [(lat, lon), ...] 需要提取的瓦片坐标 + fabdem_dir: FABDEM ZIP 文件所在目录 + extract_dir: 解压目标目录 + + 返回: + 提取成功的 GeoTIFF 文件路径列表 + """ + extract_path = Path(extract_dir) + extract_path.mkdir(parents=True, exist_ok=True) + + # 按 ZIP 文件分组，避免重复打开同一个 ZIP + zip_to_tiles = {} + missing_tiles = [] + + for lat, lon in tile_coords: + zip_file = find_zip_for_tile(lat, lon, fabdem_dir) + if zip_file is None: + tile_name = tile_name_for_coord(lat, lon) + print(f" ⚠ 未找到包含 {tile_name} 的 ZIP 文件 (lat={lat}, lon={lon})") + missing_tiles.append((lat, lon)) + continue + + zip_key = str(zip_file) + if zip_key not in zip_to_tiles: + zip_to_tiles[zip_key] = [] + zip_to_tiles[zip_key].append((lat, lon)) + + # 逐个 ZIP 文件提取 + extracted_files = [] + for zip_path, coords in zip_to_tiles.items(): + zip_name = Path(zip_path).name + print(f" 📦 从 {zip_name} 提取 {len(coords)} 个瓦片...") + + try: + with zipfile.ZipFile(zip_path, 'r') as zf: + zip_contents = zf.namelist() + + for lat, lon in coords: + tile_name = tile_name_for_coord(lat, lon) + + if tile_name in zip_contents: + zf.extract(tile_name, extract_dir) + out_file = extract_path / tile_name + if out_file.exists() and out_file.stat().st_size > 0: + extracted_files.append(str(out_file)) + print(f" ✓ {tile_name}") + else: + print(f" ✗ {tile_name} 提取后为空") + else: + # 海洋区域可能没有对应瓦片（正常） + print(f" - {tile_name} 不在 ZIP 中（可能是海洋区域）") + except Exception as e: + print(f" ✗ 打开 {zip_name} 失败: {e}") + + if missing_tiles: + print(f"\n ⚠ 共 {len(missing_tiles)} 个瓦片未找到对应的 ZIP 文件") + + return extracted_files + + +# ========================= 瓦片拼接 + GAMMA 格式转换（一步完成） ========================= + +def _parse_gdalinfo(gdalinfo_text): + """从 gdalinfo 输出解析元数据""" + width = nlines = None + corner_lon = corner_lat = post_lon = post_lat = None + for line in gdalinfo_text.splitlines(): + if 'Size is' in line: + parts = line.split('Size is')[1].strip().split(',') + width = int(parts[0].strip()) + nlines = int(parts[1].strip()) + elif 'Origin =' in line: + parts = line.split('(')[1].split(')')[0].split(',') + corner_lon = float(parts[0].strip()) + corner_lat = float(parts[1].strip()) + elif 'Pixel Size =' in line: + parts = line.split('(')[1].split(')')[0].split(',') + post_lon = float(parts[0].strip()) + post_lat = float(parts[1].strip()) + return width, nlines, corner_lon, corner_lat, post_lon, post_lat + + +def tiles_to_gamma_dem(tif_files, output_name, west, south, east, north, byteorder='big'): + """从多个 GeoTIFF 瓦片生成 GAMMA 格式 DEM (.dem + .dem.par) + + 流程: srtm2dem 逐个转换 → mosaic 合并（全部使用 GAMMA 原生工具） + + 参数: + tif_files: GeoTIFF 瓦片文件路径列表 + output_name: 输出文件名前缀（不含扩展名） + west, south, east, north: 裁剪范围（度） + byteorder: 字节序（未使用，srtm2dem 自动处理大端） + 返回: + (dem_file, dem_par_file) 或 (None, None) + """ + if not tif_files: + print(" ✗ 没有可用的瓦片文件") + return None, None + + Path(output_name).parent.mkdir(parents=True, exist_ok=True) + dem_file = output_name + '.dem' + dem_par_file = output_name + '.dem.par' + + # 临时目录放 /tmp/（本地盘），srtm2dem 单瓦片很快 + import tempfile as _tf + _tmp_dir = _tf.mkdtemp(prefix='gamma_dem_conv_') + + # ---- 步骤1: srtm2dem 逐个将 GeoTIFF 转为 GAMMA 格式 ---- + n = len(tif_files) + print(f" [1/2] srtm2dem 转换 {n} 个瓦片...") + gamma_tiles = [] # [(dem_path, dem_par_path), ...] + + for i, tif in enumerate(tif_files): + tile_name = Path(tif).stem + tile_dem = os.path.join(_tmp_dir, f'{tile_name}.dem') + tile_par = os.path.join(_tmp_dir, f'{tile_name}.dem.par') + + # gflg=3: 不做大地水准面校正, NODATA 替换为 0.0 + cmd = f'srtm2dem {tif} {tile_dem} {tile_par} 3' + ret = os.system(cmd + ' > /dev/null 2>&1') + + if ret == 0 and os.path.exists(tile_dem) and os.path.getsize(tile_dem) > 0: + gamma_tiles.append((tile_dem, tile_par)) + print(f" ✓ [{i+1}/{n}] {tile_name}") + else: + print(f" ✗ [{i+1}/{n}] {tile_name} srtm2dem 失败") + + if not gamma_tiles: + print(" ✗ 所有瓦片转换失败") + shutil.rmtree(_tmp_dir, ignore_errors=True) + return None, None + + print(f" 成功转换 {len(gamma_tiles)}/{n} 个瓦片") + + # ---- 步骤2: mosaic 合并所有 GAMMA DEM 瓦片 ---- + if len(gamma_tiles) == 1: + # 只有一个瓦片，直接复制 + shutil.copy2(gamma_tiles[0][0], dem_file) + shutil.copy2(gamma_tiles[0][1], dem_par_file) + print(f" [2/2] 单瓦片，直接输出") + else: + print(f" [2/2] mosaic 合并 {len(gamma_tiles)} 个 GAMMA DEM...") + # 构建 mosaic 命令: mosaic nfiles dem1 par1 dem2 par2 ... dem_out par_out mode format + cmd_parts = ['mosaic', str(len(gamma_tiles))] + for tile_dem, tile_par in gamma_tiles: + cmd_parts.append(tile_dem) + cmd_parts.append(tile_par) + cmd_parts.extend([dem_file, dem_par_file, '1', '0']) + # mode=1: 重叠区取平均, format=0: FLOAT + + cmd_str = ' '.join(cmd_parts) + ret = os.system(cmd_str) + + if ret != 0 or not os.path.exists(dem_file) or os.path.getsize(dem_file) == 0: + print(f" ✗ mosaic 失败 (exit={ret})") + shutil.rmtree(_tmp_dir, ignore_errors=True) + return None, None + + # 清理临时目录 + shutil.rmtree(_tmp_dir, ignore_errors=True) + + size_mb = os.path.getsize(dem_file) / (1024 * 1024) + print(f" ✓ {dem_file} ({size_mb:.1f} MB)") + print(f" ✓ {dem_par_file}") + + return dem_file, dem_par_file + + +# ========================= 主流程 ========================= + +INTRODUCTION = ''' +================================================================================ + make_local_dem.py — 从本地 FABDEM 瓦片库生成 GAMMA 格式 DEM + + 功能: + 1. 根据经纬度范围自动查找所需的 FABDEM 1°×1° 瓦片 + 2. 从 10°×10° ZIP 包中提取瓦片 + 3. 使用 GDAL 拼接并裁剪到目标范围 + 4. 转换为 GAMMA 格式 (.dem + .dem.par) + + 输出与 makedem.py 完全一致，可直接用于 generate_rdc_dem.py +================================================================================ +''' + +EXAMPLE = """ +用法: + # 方式1: 指定经纬度范围 + make_local_dem.py -r 120/123/30/32 -f /mnt/ZYD/全球FABDEM -o output_dem + + # 方式2: 从 SLC 参数文件自动确定范围 + make_local_dem.py -s master.slc.par -f /mnt/ZYD/全球FABDEM -o output_dem + + # 方式3: 从 PyINT 模板文件读取（集成到工作流） + make_local_dem.py --template shanghaiT171F128S1A -f /mnt/ZYD/全球FABDEM +""" + + +def cmdLineParse(): + parser = argparse.ArgumentParser( + description='从本地 FABDEM 瓦片库生成 GAMMA 格式 DEM', + formatter_class=argparse.RawTextHelpFormatter, + epilog=INTRODUCTION + '\n' + EXAMPLE + ) + + parser.add_argument('-r', dest='region', + help='研究区域范围: west/east/south/north (如: 120/123/30/32)') + parser.add_argument('-s', dest='slc_par', + help='SLC 参数文件路径（自动从中提取研究区域范围）') + parser.add_argument('-f', '--fabdem-dir', dest='fabdem_dir', required=True, + help='本地 FABDEM ZIP 文件目录 (如: /mnt/ZYD/全球FABDEM)') + parser.add_argument('-o', '--output', dest='output_name', default=None, + help='输出文件名前缀（不含扩展名）[默认: 工作目录/out]') + parser.add_argument('--dir', dest='work_dir', default=None, + help='工作目录 [默认: 当前目录]') + parser.add_argument('--template', dest='template_name', default=None, + help='PyINT 项目名（从模板文件读取 DEM 路径和 SLC 位置）') + parser.add_argument('--byteorder', dest='byteorder', choices=['big', 'little'], + default='big', + help='输出 DEM 字节序 [默认: big (GAMMA 标准)]') + parser.add_argument('--margin', dest='margin', type=float, default=1.0, + help='在 SLC 覆盖范围外扩展的余量（度）[默认: 1.0]') + + return parser.parse_args() + + +def get_region_from_slc_par(slc_par_file): + """从 GAMMA SLC 参数文件提取研究区域范围""" + corners_txt = 'corners_tmp.txt' + call_str = f"SLC_corners {slc_par_file} > {corners_txt}" + os.system(call_str) + + if not os.path.isfile(corners_txt): + print(f"✗ SLC_corners 执行失败") + return None + + with open(corners_txt, 'r') as f: + lines = f.readlines() + + os.remove(corners_txt) + + # 解析 SLC_corners 输出（第 9-10 行包含 lat/lon 范围） + try: + lat_line = lines[8] + lon_line = lines[9] + min_lat = float(lat_line.split(':')[1].split('max.')[0].strip()) + max_lat = float(lat_line.split(':')[2].strip()) + min_lon = float(lon_line.split(':')[1].split('max.')[0].strip()) + max_lon = float(lon_line.split(':')[2].strip()) + return min_lon, min_lat, max_lon, max_lat + except (IndexError, ValueError) as e: + print(f"✗ 解析 SLC_corners 输出失败: {e}") + return None + + +def get_region_from_template(project_name, margin=1.0): + """从 PyINT 模板文件获取区域范围（通过 master SLC 参数文件）""" + try: + from pyint import _utils as ut + except ImportError: + print("✗ 无法导入 pyint._utils，请确保 PyINT 在 PYTHONPATH 中") + return None, None + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + if not scratchDir or not templateDir: + print("✗ 环境变量 SCRATCHDIR 或 TEMPLATEDIR 未设置") + return None, None + + template_file = Path(templateDir) / f"{project_name}.template" + if not template_file.exists(): + print(f"✗ 模板文件不存在: {template_file}") + return None, None + + template_dict = ut.update_template(str(template_file)) + master_date = template_dict['masterDate'] + slc_dir = Path(scratchDir) / project_name / 'SLC' + master_slc_par = slc_dir / master_date / f"{master_date}.slc.par" + + if not master_slc_par.exists(): + print(f"✗ Master SLC 参数文件不存在: {master_slc_par}") + return None, None + + region = get_region_from_slc_par(str(master_slc_par)) + if region is None: + return None, None + + west, south, east, north = region + # 外扩 margin 度并取整 + west = math.floor(west - margin) + south = math.floor(south - margin) + east = math.ceil(east + margin) + north = math.ceil(north + margin) + + # 确定输出路径（使用模板中 DEM 路径或默认路径） + dem_dir = os.getenv('DEMDIR', '') + if 'DEM' in template_dict and template_dict['DEM'].strip(): + output_name = template_dict['DEM'].replace('.dem', '') + else: + dem_out_dir = Path(dem_dir) / project_name + dem_out_dir.mkdir(parents=True, exist_ok=True) + output_name = str(dem_out_dir / project_name) + + return (west, south, east, north), output_name + + +def main(): + args = cmdLineParse() + + # 确定工作目录 + if args.work_dir: + work_dir = Path(args.work_dir) + else: + work_dir = Path.cwd() + work_dir.mkdir(parents=True, exist_ok=True) + + # 确定 FABDEM 目录 + fabdem_dir = Path(args.fabdem_dir) + if not fabdem_dir.is_dir(): + print(f"✗ FABDEM 目录不存在: {fabdem_dir}") + sys.exit(1) + + zip_count = len(list(fabdem_dir.glob('*_FABDEM_V1-2.zip'))) + print(f"✓ FABDEM 目录: {fabdem_dir} ({zip_count} 个 ZIP 文件)") + + # 确定研究区域 + output_name = args.output_name + region = None + + if args.template_name: + # 从 PyINT 模板获取 + result = get_region_from_template(args.template_name, args.margin) + if result[0] is not None: + region = result[0] + if output_name is None: + output_name = result[1] + else: + print("✗ 无法从模板文件获取区域范围") + sys.exit(1) + + elif args.slc_par: + # 从 SLC 参数文件获取 + slc_region = get_region_from_slc_par(args.slc_par) + if slc_region is None: + sys.exit(1) + west, south, east, north = slc_region + west = math.floor(west - args.margin) + south = math.floor(south - args.margin) + east = math.ceil(east + args.margin) + north = math.ceil(north + args.margin) + region = (west, south, east, north) + + elif args.region: + # 从命令行参数解析 + parts = args.region.split('/') + if len(parts) != 4: + print("✗ 区域格式错误，应为: west/east/south/north") + sys.exit(1) + west, east, south, north = [float(x) for x in parts] + region = (west, south, east, north) + + else: + print("✗ 必须指定研究区域: -r, -s 或 --template") + sys.exit(1) + + if output_name is None: + output_name = str(work_dir / 'out') + + # 确保输出目录存在 + Path(output_name).parent.mkdir(parents=True, exist_ok=True) + + west, south, east, north = region + + print(f"\n{'='*70}") + print(f" 从本地 FABDEM 生成 GAMMA DEM") + print(f"{'='*70}") + print(f" 区域范围: {west}°E ~ {east}°E, {south}°N ~ {north}°N") + print(f" FABDEM 目录: {fabdem_dir}") + print(f" 输出文件: {output_name}.dem / {output_name}.dem.par") + print(f" 字节序: {args.byteorder}") + print(f"{'='*70}\n") + + # 1. 计算所需瓦片 + tiles = find_needed_tiles(west, south, east, north) + print(f"[1/4] 需要 {len(tiles)} 个 1°×1° 瓦片\n") + + # 2. 从 ZIP 中提取瓦片 + print(f"[2/4] 从 FABDEM ZIP 文件中提取瓦片...") + temp_dir = tempfile.mkdtemp(prefix='fabdem_tiles_') + try: + tif_files = extract_tiles_from_zips(tiles, str(fabdem_dir), temp_dir) + + if not tif_files: + print("\n✗ 没有提取到任何瓦片文件，请检查 FABDEM 目录和区域范围") + shutil.rmtree(temp_dir, ignore_errors=True) + sys.exit(1) + + print(f"\n 共提取 {len(tif_files)}/{len(tiles)} 个瓦片\n") + + # 3-4. 一步完成: VRT → ENVI 二进制 → GAMMA .dem + print(f"[3/4] 拼接裁剪 + 转换 GAMMA 格式...") + dem_file, dem_par_file = tiles_to_gamma_dem( + tif_files, output_name, west, south, east, north, args.byteorder + ) + + if dem_file is None: + print("\n✗ DEM 生成失败") + shutil.rmtree(temp_dir, ignore_errors=True) + sys.exit(1) + + finally: + # 清理临时目录 + shutil.rmtree(temp_dir, ignore_errors=True) + + # 完成 + print(f"\n{'='*70}") + print(f" ✓ FABDEM → GAMMA DEM 转换完成!") + print(f"{'='*70}") + print(f" DEM 文件: {dem_file}") + print(f" 参数文件: {dem_par_file}") + print(f" 字节序: {args.byteorder} endian") + print(f" 可直接用于: generate_rdc_dem.py") + print(f"{'='*70}\n") + + +if __name__ == '__main__': + main() diff --git a/third_party/PyINT/pyint/makedem.py b/third_party/PyINT/pyint/makedem.py new file mode 100644 index 0000000..38a211f --- /dev/null +++ b/third_party/PyINT/pyint/makedem.py @@ -0,0 +1,1486 @@ +#! /usr/bin/env python +import os +import sys +import numpy as np +import argparse +import subprocess +import glob +from skimage import io + +# 尝试导入 NASADEM 库 +try: + from NASADEM import NASADEM + from rasters import RasterGrid + HAS_NASADEM = True +except ImportError: + HAS_NASADEM = False + print("Warning: NASADEM library not installed. NASADEM download will not be available.") + print("Install with: pip install nasadem rasters") + +# 尝试导入 srtm 库 +try: + import srtm + HAS_SRTM = True +except ImportError: + HAS_SRTM = False + print("Warning: srtm library not installed. SRTM download will not be available.") + print("Install with: pip install srtm (requires Python >= 3.12)") + print(f"Current Python version: {sys.version_info.major}.{sys.version_info.minor}") + +resolutions = 30 # 90 + +def write_demrsc_file(FILE, Corner_LON, Corner_LAT, X_STEP, Y_STEP, WIDTH, LENGTH): + """Write ROI_PAC DEM resource file (.dem.rsc) + + Args: + FILE (str): Output resource file path + Corner_LON (str): Corner longitude + Corner_LAT (str): Corner latitude + X_STEP (str): Longitude step size + Y_STEP (str): Latitude step size + WIDTH (str): Number of columns + LENGTH (str): Number of lines + """ + f = open(FILE, 'w') + f.write('DATE12 111111-222222\n') + f.write('FILE_LENGTH ' + str(int(LENGTH)) + '\n') + f.write('FILE_TYPE .dem\n') + f.write('PROCESSOR roipac\n') + f.write('PROJECTION LATLON\n') + f.write('RLOOKS 1\n') + f.write('WIDTH ' + str(int(WIDTH)) + '\n') + f.write('XMAX ' + str(int(int(WIDTH)-1)) + '\n') + f.write('XMIN 0\n') + f.write('X_FIRST ' + str(float(Corner_LON)) + '\n') + f.write('X_STEP ' + str(float(X_STEP)) + '\n') + f.write('X_UNIT degrees\n') + f.write('YMAX ' + str(int(int(LENGTH)-1)) + '\n') + f.write('YMIN 0\n') + f.write('Y_FIRST ' + str(float(Corner_LAT)) + '\n') + f.write('Y_STEP ' + str(float(Y_STEP)) + '\n') + f.write('Y_UNIT degrees\n') + f.write('Z_OFFSET 0\n') + f.write('Z_SCALE 1\n') + f.close() + +def write_dempar_file(FILE, Corner_LON, Corner_LAT, X_STEP, Y_STEP, WIDTH, LENGTH, DATA_FORMAT): + """Write Gamma DEM parameter file (.dem.par) + + Args: + FILE (str): Output parameter file path + Corner_LON (str): Corner longitude + Corner_LAT (str): Corner latitude + X_STEP (str): Longitude step size + Y_STEP (str): Latitude step size + WIDTH (str): Number of columns + LENGTH (str): Number of lines + DATA_FORMAT (str): Data format (INTEGER*2 or REAL*4) + """ + DEM_TYPE = 'Copernicus30' + Proj = 'EQA' + f = open(FILE, 'w') + f.write("Gamma DIFF&GEO DEM/MAP parameter file\n") + f.write("title:\tIMPORTED DEM FROM %s\n" % DEM_TYPE) + f.write("DEM_projection: %s\n" % Proj) + f.write("data_format: %s\n" % DATA_FORMAT) + f.write("DEM_hgt_offset: 0.00000\n") + f.write("DEM_scale: 1.00000\n") + f.write("width: %s\n" % WIDTH) + f.write("nlines: %s\n" % LENGTH) + f.write("corner_lat: %s decimal degrees\n" % Corner_LAT) + f.write("corner_lon: %s decimal degrees\n" % Corner_LON) + f.write("post_lat: %s decimal degrees\n" % Y_STEP) + f.write("post_lon: %s decimal degrees\n" % X_STEP) + f.write("\n") + f.write("ellipsoid_name: WGS 84\n") + f.write("ellipsoid_ra: 6378137.000 m\n") + f.write("ellipsoid_reciprocal_flattening: 298.2572236\n") + f.write("\n") + f.write("datum_name: WGS 1984\n") + f.write("datum_shift_dx: 0.000 m\n") + f.write("datum_shift_dy: 0.000 m\n") + f.write("datum_shift_dz: 0.000 m\n") + f.write("datum_scale_m: 0.00000e+00\n") + f.write("datum_rotation_alpha: 0.00000e+00 arc-sec\n") + f.write("datum_rotation_beta: 0.00000e+00 arc-sec\n") + f.write("datum_rotation_gamma: 0.00000e+00 arc-sec\n") + f.write("datum_country_list Global Definition, WGS84, World\n") + f.write("\n") + f.close() + +def convert_to_gamma(input_file, output_name, byteorder='big', processor='gamma'): + """Convert TIF file to Gamma or ROI_PAC format (.dem and .dem.par/.dem.rsc) + + Args: + input_file (str): Input TIF file path + output_name (str): Output name (without extension) + byteorder (str): Byte order ('big' or 'little') + processor (str): Processor type ('gamma' or 'roi_pac') + """ + processor_name = 'Gamma' if processor == 'gamma' else 'ROI_PAC' + print(f"\n开始转换到 {processor_name} 格式: {input_file}") + + # 读取 DEM 数据 - 优先使用 GDAL（更稳定），然后是 rasterio + dem_data = None + + # 方法1: 使用 GDAL（最稳定） + try: + from osgeo import gdal + print(" 尝试使用 GDAL 读取数据...") + ds = gdal.Open(input_file) + if ds is not None: + band = ds.GetRasterBand(1) + dem_data = band.ReadAsArray() + print(f" ✓ 使用 GDAL 读取成功") + ds = None # 关闭文件 + except Exception as e: + print(f" ✗ GDAL 读取失败: {e}") + + # 方法2: 如果GDAL失败，尝试rasterio + if dem_data is None: + try: + import rasterio + print(" 尝试使用 rasterio 读取数据...") + with rasterio.open(input_file) as src: + dem_data = src.read(1) + print(f" ✓ 使用 rasterio 读取成功") + except Exception as e: + print(f" ✗ rasterio 读取失败: {e}") + + # 方法3: 最后尝试skimage + if dem_data is None: + try: + print(" 尝试使用 skimage 读取数据...") + dem_data = io.imread(input_file) + print(f" ✓ 使用 skimage 读取成功") + except Exception as e: + print(f" ✗ skimage 读取失败: {e}") + + # 如果所有方法都失败 + if dem_data is None: + raise ValueError(f"无法读取DEM文件: {input_file}，所有读取方法都失败了") + + # 确定数据格式 + if dem_data.dtype == 'float32': + DATA_FORMAT = 'REAL*4' + else: + DATA_FORMAT = 'INTEGER*2' + + # 字节序转换 + if not sys.byteorder == byteorder: + dem_data.byteswap(True) + + # 输出文件路径 + dem_file = output_name + '.dem' + if processor == 'gamma': + dem_par_file = output_name + '.dem.par' + else: + dem_par_file = output_name + '.dem.rsc' + + # 写入二进制 DEM 数据 + dem_data.tofile(dem_file) + + # 使用 gdalinfo 获取地理信息 + info_file = 'temp_gdalinfo.txt' + cmd = f"gdalinfo {input_file} > {info_file}" + os.system(cmd) + + # 解析地理信息 + Corner_LON = None + Corner_LAT = None + Post_LON = None + Post_LAT = None + WIDTH = None + FILE_LENGTH = None + + with open(info_file, 'r') as f: + for line in f: + if 'Origin =' in line: + parts = line.split('Origin =')[1].strip().split('(')[1].split(')')[0].split(',') + Corner_LON = parts[0] + Corner_LAT = parts[1] + elif 'Pixel Size ' in line: + parts = line.split('Pixel Size =')[1].strip().split('(')[1].split(')')[0].split(',') + Post_LON = parts[0] + Post_LAT = parts[1] + elif 'Size is' in line: + parts = line.split('Size is')[1].strip().split(',') + WIDTH = parts[0] + FILE_LENGTH = parts[1] + + # 删除临时文件 + if os.path.exists(info_file): + os.remove(info_file) + + # 写入参数文件 + if processor == 'gamma': + write_dempar_file(dem_par_file, Corner_LON, Corner_LAT, Post_LON, Post_LAT, WIDTH, FILE_LENGTH, DATA_FORMAT) + else: + write_demrsc_file(dem_par_file, Corner_LON, Corner_LAT, Post_LON, Post_LAT, WIDTH, FILE_LENGTH) + + print(f"{byteorder} endian {dem_file} and {dem_par_file} are generated.") + print(f"{processor_name} 格式转换完成!") + +def get_sufix(STR): + """Get file extension""" + n = len(STR.split('.')) + SUFIX = STR.split('.')[n-1] + return SUFIX + +def read_region(STR): + """Parse region string 'west/east/south/north'""" + WEST = STR.split('/')[0] + EAST = STR.split('/')[1].split('/')[0] + SOUTH = STR.split(EAST+'/')[1].split('/')[0] + NORTH = STR.split(EAST+'/')[1].split('/')[1] + WEST = float(WEST) + SOUTH = float(SOUTH) + EAST = float(EAST) + NORTH = float(NORTH) + return WEST, SOUTH, EAST, NORTH + +def cmd_init(lon, lat, save_path): + s_lon = str(abs(lon)) + s_lat = str(abs(lat)) + if abs(lon) < 10: + s_lon = "00" + str(abs(lon)) + elif abs(lon) < 100: + s_lon = "0" + str(abs(lon)) + if abs(lat) < 10: + s_lat = "0" + str(abs(lat)) + if lon < 0: + c_lon = "W" + str(s_lon) + else: + c_lon = "E" + str(s_lon) + if lat < 0: + c_lat = "S" + str(s_lat) + else: + c_lat = "N" + str(s_lat) + cmd = "aws s3 cp --no-sign-request" + " s3://copernicus-dem-{0}m/Copernicus_DSM_COG_{1}_{2}_00_{3}_00_DEM/ {4} --recursive".format( + str(resolutions), str(int(resolutions / 3)), str(c_lat), str(c_lon), save_path) + return cmd + +def get_remote_file(lon, lat, save_path, max_retries=3): + """Get Copernicus Dem by lon and lat with retry mechanism + + Args: + lon (number): lontitude + lat (number): latitude + save_path (str): directory to save data + max_retries (int): maximum retry attempts + """ + lon = int(lon) + lat = int(lat) + cmd = cmd_init(lon, lat, save_path) + + for attempt in range(max_retries): + try: + result = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=300) + if result.returncode == 0: + print(f"✓ Successfully downloaded: lon={lon}, lat={lat}") + return True + else: + if attempt < max_retries - 1: + print(f"✗ Failed (attempt {attempt+1}/{max_retries}): lon={lon}, lat={lat}, retrying...") + import time + time.sleep(2) + else: + print(f"✗ Failed after {max_retries} attempts: lon={lon}, lat={lat}") + return False + except subprocess.TimeoutExpired: + if attempt < max_retries - 1: + print(f"⏱ Timeout (attempt {attempt+1}/{max_retries}): lon={lon}, lat={lat}, retrying...") + import time + time.sleep(2) + else: + print(f"✗ Timeout after {max_retries} attempts: lon={lon}, lat={lat}") + return False + except Exception as e: + print(f"✗ Exception occurred for lon={lon}, lat={lat}: {e}") + return False + + return False + +def get_remote_file_batch(west, east, south, north, save_path, num_workers=4): + """Get Copernicus Dems by WESN extent with parallel download + + Args: + west (number): western longitude boundary + east (number): eastern longitude boundary + south (number): southern latitude boundary + north (number): northern latitude boundary + save_path (str): directory to save data + num_workers (int): number of parallel download workers (default: 4) + """ + from concurrent.futures import ThreadPoolExecutor, as_completed + from tqdm import tqdm + + west = int(west) + east = int(east) + south = int(south) + north = int(north) + + # 生成所有需要下载的坐标对 + coords = [(lon, lat) for lon in range(west, east + 1) for lat in range(south, north + 1)] + total_tiles = len(coords) + + print(f"\n{'='*80}") + print(f"并行下载 Copernicus DEM 数据") + print(f"{'='*80}") + print(f"区域范围: {west}°E - {east}°E, {south}°N - {north}°N") + print(f"总瓦片数: {total_tiles}") + print(f"并行线程: {num_workers}") + print(f"{'='*80}\n") + + success_count = 0 + failed_count = 0 + + # 使用线程池并行下载 + with ThreadPoolExecutor(max_workers=num_workers) as executor: + # 提交所有下载任务 + future_to_coord = {executor.submit(get_remote_file, lon, lat, save_path): (lon, lat) + for lon, lat in coords} + + # 使用 tqdm 显示进度条 + with tqdm(total=total_tiles, desc="下载进度", unit="瓦片") as pbar: + for future in as_completed(future_to_coord): + coord = future_to_coord[future] + try: + result = future.result() + if result: + success_count += 1 + else: + failed_count += 1 + except Exception as e: + print(f"\n✗ 下载失败 {coord}: {e}") + failed_count += 1 + + pbar.update(1) + + # 打印统计信息 + print(f"\n{'='*80}") + print(f"下载完成统计") + print(f"{'='*80}") + print(f"成功: {success_count}/{total_tiles} 瓦片") + print(f"失败: {failed_count}/{total_tiles} 瓦片") + if failed_count > 0: + print(f"\n⚠️ 有 {failed_count} 个瓦片下载失败,可能会影响 DEM 完整性") + print(f"{'='*80}\n") + +def download_nasadem(west, south, east, north, save_path, cell_size_deg=0.000277778): + """Download NASADEM data using NASADEM library + + Args: + west (float): western longitude boundary + south (float): southern latitude boundary + east (float): eastern longitude boundary + north (float): northern latitude boundary + save_path (str): directory to save data + cell_size_deg (float): cell size in degrees (default: 0.000277778 ≈ 30m at equator) + + Returns: + str: path to the downloaded DEM file, or None if failed + """ + if not HAS_NASADEM: + print("✗ NASADEM library not available. Cannot download NASADEM.") + print(" Install with: pip install nasadem rasters") + return None + + print(f"\n{'='*80}") + print(f"下载 NASADEM 数据") + print(f"{'='*80}") + print(f"区域范围: {west}°E - {east}°E, {south}°N - {north}°N") + print(f"分辨率: {cell_size_deg}° (约 {cell_size_deg * 111000:.0f} 米)") + print(f"{'='*80}\n") + + try: + from NASADEM.NASADEM import NASADEMConnection + from rasters import RasterGrid + import time + + start_time = time.time() + + # 定义目标区域 + geometry = RasterGrid.from_bbox( + xmin=west, ymin=south, xmax=east, ymax=north, + cell_size=cell_size_deg, + crs="EPSG:4326" + ) + + print(f"RasterGrid 维度: {geometry.shape}") + print(f"正在从NASA服务器下载高程数据...") + print(f"提示: 首次下载需要从NASA服务器获取数据,可能需要几分钟时间...") + + # 创建NASA DEM连接并获取高程数据 + conn = NASADEMConnection() + elevation = conn.elevation_m(geometry) + + elapsed_time = time.time() - start_time + + print(f"\n✓ 下载完成!") + print(f" 高程范围: {float(elevation.min()):.1f} 到 {float(elevation.max()):.1f} 米") + print(f" 数据维度: {elevation.shape}") + print(f" 用时: {elapsed_time:.1f} 秒") + + # 保存为 GeoTIFF + output_file = os.path.join(save_path, f"NASADEM_{west}_{east}_{south}_{north}.tif") + print(f"\n保存到: {output_file}") + + # 检查elevation对象是否有save方法 + if hasattr(elevation, 'save'): + elevation.save(output_file) + print(f"✓ 文件保存成功: {output_file}") + if os.path.exists(output_file): + file_size = os.path.getsize(output_file) / (1024 * 1024) + print(f" 文件大小: {file_size:.2f} MB") + return output_file + + # 否则使用 rasterio 保存 + try: + import rasterio + from rasterio.crs import CRS + from rasterio.transform import from_bounds + + # 获取数据数组 + if hasattr(elevation, 'read'): + elev_array = elevation.read() + else: + elev_array = np.array(elevation) + + # 计算变换矩阵 + height, width = elev_array.shape + transform = from_bounds(west, south, east, north, width, height) + + # 写入文件 + with rasterio.open( + output_file, + 'w', + driver='GTiff', + height=height, + width=width, + count=1, + dtype=elev_array.dtype, + crs=CRS.from_epsg(4326), + transform=transform, + nodata=-32767, + compress='LZW' + ) as dst: + dst.write(elev_array, 1) + + print(f"✓ 文件保存成功: {output_file}") + if os.path.exists(output_file): + file_size = os.path.getsize(output_file) / (1024 * 1024) + print(f" 文件大小: {file_size:.2f} MB") + return output_file + + except ImportError: + print("⚠️ rasterio 未安装,尝试使用 GDAL...") + + # 使用 GDAL 创建 GeoTIFF + width, height = geometry.width, geometry.height + gdal_cmd = f"gdal_create -outsize {width} {height} -of GTiff -co COMPRESS=LZW -a_srs EPSG:4326 -a_ullr {west} {north} {east} {south} {output_file}" + result = os.system(gdal_cmd) + + if result == 0 and os.path.exists(output_file): + print(f"✓ 文件创建成功: {output_file}") + return output_file + else: + print(f"✗ 文件创建失败") + return None + + except Exception as e: + print(f"\n✗ 下载失败: {e}") + import traceback + traceback.print_exc() + return None + +def download_srtm(west, south, east, north, save_path, srtm_data_dir=None, cell_size=0.000277778): + """Download SRTM data using srtm library (requires pre-downloaded .hgt files) + + Args: + west (float): western longitude boundary + south (float): southern latitude boundary + east (float): eastern longitude boundary + north (float): northern latitude boundary + save_path (str): directory to save output DEM + srtm_data_dir (str): directory containing SRTM .hgt files (required) + cell_size (float): cell size in degrees (default: 0.000277778 ≈ 30m) + + Returns: + str: path to the generated DEM file, or None if failed + + Note: + This function requires SRTM .hgt files to be pre-downloaded. + You can download SRTM data from: + - https://srtm.csi.cgiar.org/ + - https://earthexplorer.usgs.gov/ + + Python version requirement: + - srtm library requires Python >= 3.12 + - For Python < 3.12, use process_srtm_hgt_files() instead + """ + if not HAS_SRTM: + print("✗ srtm library not available. Cannot use SRTM.") + print(" Install with: pip install srtm (requires Python >= 3.12)") + print(f" Current Python version: {sys.version_info.major}.{sys.version_info.minor}") + print(" Alternative: Use process_srtm_hgt_files() to manually process .hgt files") + return None + + if srtm_data_dir is None or not os.path.isdir(srtm_data_dir): + print("✗ SRTM data directory not provided or does not exist.") + print(" Please download SRTM .hgt files and specify the directory.") + print(" Download sources:") + print(" - https://srtm.csi.cgiar.org/") + print(" - https://earthexplorer.usgs.gov/") + return None + + print(f"\n{'='*80}") + print(f"使用 SRTM 数据生成 DEM") + print(f"{'='*80}") + print(f"区域范围: {west}°E - {east}°E, {south}°N - {north}°N") + print(f"SRTM数据目录: {srtm_data_dir}") + print(f"分辨率: {cell_size}° (约 {cell_size * 111000:.0f} 米)") + print(f"{'='*80}\n") + + try: + import time + from tqdm import tqdm + + start_time = time.time() + + # 创建 SrtmService + service = srtm.SrtmService(srtm_data_dir, cache_size=100) + print(f"✓ SrtmService 创建成功") + + # 生成坐标网格 + lats = np.arange(south, north + cell_size, cell_size) + lons = np.arange(west, east + cell_size, cell_size) + + print(f"生成网格: {len(lats)} × {len(lons)} = {len(lats) * len(lons)} 个点") + + # 批量查询高程 + coords = [(lat, lon) for lat in lats for lon in lons] + print(f"正在查询高程数据...") + + elevations = service.get_elevations_batch(coords, default=0) + + # 重塑为数组 + elevation_array = np.array(elevations, dtype=np.float32).reshape(len(lats), len(lons)) + + # 替换0值为NaN(可能是缺失数据) + elevation_array[elevation_array == 0] = np.nan + + elapsed_time = time.time() - start_time + + print(f"\n✓ 查询完成!") + print(f" 有效数据: {np.sum(~np.isnan(elevation_array))} / {elevation_array.size} 点") + if np.sum(~np.isnan(elevation_array)) > 0: + print(f" 高程范围: {np.nanmin(elevation_array):.1f} 到 {np.nanmax(elevation_array):.1f} 米") + print(f" 用时: {elapsed_time:.1f} 秒") + + # 保存为 GeoTIFF + output_file = os.path.join(save_path, f"SRTM_{west}_{east}_{south}_{north}.tif") + print(f"\n保存到: {output_file}") + + try: + import rasterio + from rasterio.crs import CRS + from rasterio.transform import from_bounds + + height, width = elevation_array.shape + transform = from_bounds(west, south, east, north, width, height) + + # 替换NaN为nodata值 + elevation_array[np.isnan(elevation_array)] = -32767 + + with rasterio.open( + output_file, + 'w', + driver='GTiff', + height=height, + width=width, + count=1, + dtype=np.float32, + crs=CRS.from_epsg(4326), + transform=transform, + nodata=-32767, + compress='LZW' + ) as dst: + dst.write(elevation_array.astype(np.float32), 1) + + print(f"✓ 文件保存成功: {output_file}") + if os.path.exists(output_file): + file_size = os.path.getsize(output_file) / (1024 * 1024) + print(f" 文件大小: {file_size:.2f} MB") + return output_file + + except ImportError: + print("⚠️ rasterio 未安装,无法保存文件") + return None + + except Exception as e: + print(f"\n✗ 生成失败: {e}") + import traceback + traceback.print_exc() + return None + +def split_large_region(west, south, east, north, tile_size=1.0): + """将大区域分割成小块 + + Args: + west, south, east, north: 区域边界 + tile_size: 分块大小（度），默认1°×1° + + Returns: + list: 分块列表 [(west, south, east, north), ...] + """ + tiles = [] + + w = west + while w < east: + e = min(w + tile_size, east) + + s = south + while s < north: + n = min(s + tile_size, north) + tiles.append((w, s, e, n)) + s += tile_size + + w += tile_size + + return tiles + + +def download_srtm_cgiar(west, south, east, north, save_path, api_key=None, num_workers=4, dem_type='SRTMGL1', auto_tile=True): + """Download DEM data from OpenTopography automatically + + Args: + west (float): western longitude boundary + south (float): southern latitude boundary + east (float): eastern longitude boundary + north (float): northern latitude boundary + save_path (str): directory to save output DEM + api_key (str): OpenTopography API key (optional, will prompt if not provided) + num_workers (int): number of parallel downloads (not used for API, kept for compatibility) + dem_type (str): DEM type (default: 'SRTMGL1') + - SRTMGL1: SRTM GL1 30m (recommended for SRTM) + - SRTMGL3: SRTM GL3 90m + - NASADEM: NASADEM 30m + - COP30: Copernicus 30m + - COP90: Copernicus 90m + auto_tile (bool): 自动分块下载大区域 + + Returns: + str: path to the generated DEM file, or None if failed + + Note: + OpenTopography API provides: + - SRTMGL1: SRTM GL1 30m (recommended) + - SRTMGL3: SRTM GL3 90m + - NASADEM: NASADEM Global DEM 30m + - COP30: Copernicus 30m + - COP90: Copernicus 90m + + API Documentation: https://portal.opentopography.org/apidocs/ + Get API key: https://opentopography.org/myOpenTopo + """ + # DEM类型信息 + dem_info = { + 'SRTMGL1': {'resolution': '30m', 'name': 'SRTM GL1'}, + 'SRTMGL3': {'resolution': '90m', 'name': 'SRTM GL3'}, + 'NASADEM': {'resolution': '30m', 'name': 'NASADEM'}, + 'COP30': {'resolution': '30m', 'name': 'Copernicus 30m'}, + 'COP90': {'resolution': '90m', 'name': 'Copernicus 90m'} + } + + info = dem_info.get(dem_type, {'resolution': 'unknown', 'name': dem_type}) + + print(f"\n{'='*80}") + print(f"从 OpenTopography 自动下载 DEM 数据") + print(f"{'='*80}") + print(f"区域范围: {west}°E - {east}°E, {south}°N - {north}°N") + print(f"数据源: {info['name']} ({info['resolution']} resolution)") + print(f"{'='*80}\n") + + # 计算区域面积 + region_area = (east - west) * (north - south) + + # 如果区域较大，自动分块 + if auto_tile and region_area > 4: + print(f"⚠️ 区域较大 ({region_area:.1f} 平方度)，将自动分块下载以提高稳定性") + + # 计算分块 + tiles = split_large_region(west, south, east, north, tile_size=1.0) + print(f" 将分成 {len(tiles)} 个 1°×1° 的区块下载\n") + + # 创建临时目录存放分块文件 + temp_dir = os.path.join(save_path, f"temp_tiles_{dem_type}_{west}_{east}_{south}_{north}") + if not os.path.exists(temp_dir): + os.makedirs(temp_dir) + + # 下载每个分块 + downloaded_files = [] + failed_tiles = [] + + for i, (w, s, e, n) in enumerate(tiles, 1): + print(f"\n{'='*80}") + print(f"下载分块 {i}/{len(tiles)}: {w}°E-{e}°E, {s}°N-{n}°N") + print(f"{'='*80}") + + tile_file = download_single_tile(w, s, e, n, temp_dir, api_key, dem_type, info) + + if tile_file: + downloaded_files.append(tile_file) + print(f"✓ 分块 {i}/{len(tiles)} 下载成功") + else: + failed_tiles.append((w, s, e, n)) + print(f"✗ 分块 {i}/{len(tiles)} 下载失败") + + # 如果有失败的分块 + if failed_tiles: + print(f"\n⚠️ 有 {len(failed_tiles)} 个分块下载失败") + print("失败的分块:") + for w, s, e, n in failed_tiles: + print(f" {w}°E-{e}°E, {s}°N-{n}°N") + + # 合并所有分块 + if downloaded_files: + print(f"\n{'='*80}") + print(f"合并 {len(downloaded_files)} 个分块...") + print(f"{'='*80}\n") + + merged_file = merge_dem_tiles(downloaded_files, save_path, dem_type, west, east, south, north) + + # 清理临时文件 + import shutil + try: + shutil.rmtree(temp_dir) + print("✓ 临时文件已清理") + except: + pass + + if merged_file: + print(f"\n✓ 最终输出文件: {merged_file}") + return merged_file + else: + print("✗ 合并失败") + return None + else: + print("✗ 所有分块下载失败") + return None + + # 小区域直接下载 + else: + return download_single_tile(west, south, east, north, save_path, api_key, dem_type, info) + + +def download_single_tile(west, south, east, north, save_path, api_key, dem_type, dem_info): + """下载单个DEM块 + + Args: + west, south, east, north: 区域边界 + save_path: 保存路径 + api_key: OpenTopography API key + dem_type: DEM类型 + dem_info: DEM信息字典 + + Returns: + str: 下载的文件路径，失败返回None + """ + try: + import time + import urllib.request + import urllib.parse + + start_time = time.time() + + # OpenTopography API endpoint + base_url = "https://portal.opentopography.org/API/globaldem" + + # 如果没有提供API key，提示用户获取 + if api_key is None: + print("⚠️ 需要OpenTopography API key才能下载SRTM数据") + print("\n获取免费API key的步骤:") + print(" 1. 访问: https://opentopography.org/myOpenTopo") + print(" 2. 注册免费账户") + print(" 3. 在 'My Account' 页面请求API key") + print(" 4. 使用 --opentopo-api-key 参数提供API key") + print("\n或者使用其他DEM数据源:") + print(" - Copernicus (推荐): --dem-source copernicus") + print(" - NASADEM: --dem-source nasadem") + return None + + # 构建请求参数 + params = { + 'demtype': dem_type, + 'south': south, + 'north': north, + 'west': west, + 'east': east, + 'outputFormat': 'GTiff', + 'API_Key': api_key + } + + # 构建完整URL + url = base_url + '?' + urllib.parse.urlencode(params) + + # 输出文件名 + output_file = os.path.join(save_path, f"{dem_type}_{west}_{east}_{south}_{north}.tif") + + # 下载文件 - 使用curl以获得更好的稳定性 + max_retries = 3 + retry_count = 0 + download_success = False + + while retry_count < max_retries and not download_success: + try: + retry_count += 1 + print(f"正在从OpenTopography下载数据... (尝试 {retry_count}/{max_retries})") + + # 尝试使用 curl 下载（更稳定） + try: + import subprocess + curl_cmd = ['curl', '-L', '-o', output_file, '-s', '--show-error', url] + result = subprocess.run(curl_cmd, capture_output=True, text=True, timeout=1800) # 30分钟超时 + + if result.returncode != 0: + raise Exception(f"curl下载失败: {result.stderr}") + + print(" ✓ curl下载完成") + download_success = True + + except (subprocess.TimeoutExpired, FileNotFoundError): + # 如果curl不可用或超时，使用urllib + print(" 使用urllib下载...") + urllib.request.urlretrieve(url, output_file) + download_success = True + + except urllib.error.HTTPError as e: + print(f"\n✗ HTTP错误: {e.code} {e.reason}") + if e.code == 401: + print(" API key无效或已过期") + print(" 请获取新的API key: https://opentopography.org/myOpenTopo") + elif e.code == 403: + print(" 访问被拒绝，可能是:") + print(" - 请求区域过大") + print(" - API key配额已用完") + print(" - 需要注册获取免费API key") + if retry_count < max_retries: + print(f" 将在3秒后重试...") + time.sleep(3) + continue + return None + except Exception as e: + print(f"\n✗ 下载失败: {e}") + if retry_count < max_retries: + print(f" 将在3秒后重试...") + time.sleep(3) + continue + return None + + # 检查文件是否有效 + if os.path.exists(output_file): + file_size = os.path.getsize(output_file) + + # 如果文件很小，可能是错误消息 + if file_size < 1000: + with open(output_file, 'r') as f: + error_msg = f.read() + print(f"\n✗ 下载失败: {error_msg}") + os.remove(output_file) + return None + + elapsed_time = time.time() - start_time + + print(f"\n✓ 下载完成!") + print(f" 输出文件: {output_file}") + print(f" 文件大小: {file_size / (1024 * 1024):.2f} MB") + print(f" 用时: {elapsed_time:.1f} 秒") + + # 验证文件完整性 + print("\n验证文件完整性...") + + try: + import rasterio + with rasterio.open(output_file) as src: + # 尝试读取多个位置的数据验证 + print(f" 文件大小: {src.width} x {src.height} 像素") + + # 读取多个测试点 + test_points = [ + (0, 0, min(100, src.height), min(100, src.width)), # 左上角 + (max(0, src.height-100), max(0, src.width-100), src.height, src.width), # 右下角 + (src.height//2-50, src.width//2-50, src.height//2+50, src.width//2+50), # 中间 + ] + + for i, (row_start, col_start, row_end, col_end) in enumerate(test_points): + try: + window = ((row_start, row_end), (col_start, col_end)) + test_data = src.read(1, window=window) + print(f" ✓ 测试点 {i+1}/3 验证成功") + except Exception as e: + print(f" ✗ 测试点 {i+1}/3 验证失败: {e}") + raise + + print(f" ✓ 所有验证点通过，文件完整") + return output_file + + except Exception as e: + print(f" ✗ 文件验证失败: {e}") + print(f" 文件可能已损坏，正在删除...") + try: + os.remove(output_file) + except: + pass + return None + else: + print("✗ 文件下载失败") + return None + + except Exception as e: + print(f"\n✗ 下载失败: {e}") + import traceback + traceback.print_exc() + return None + +def merge_dem_tiles(tile_files, save_path, dem_type, west, east, south, north): + """合并多个DEM分块文件 + + Args: + tile_files: 分块文件列表 + save_path: 保存路径 + dem_type: DEM类型 + west, east, south, north: 最终区域边界 + + Returns: + str: 合并后的文件路径，失败返回None + """ + if not tile_files: + print("✗ 没有文件需要合并") + return None + + try: + import rasterio + from rasterio.merge import merge + import time + + print(f"开始合并 {len(tile_files)} 个分块文件...") + + # 读取所有分块 + src_files_to_mosaic = [] + for tile_file in tile_files: + src = rasterio.open(tile_file) + src_files_to_mosaic.append(src) + + # 合并 + start_time = time.time() + mosaic, out_trans = merge(src_files_to_mosaic) + + # 获取输出元数据 + out_meta = src_files_to_mosaic[0].meta.copy() + out_meta.update({ + "driver": "GTiff", + "height": mosaic.shape[1], + "width": mosaic.shape[2], + "transform": out_trans, + "compress": "lzw" + }) + + # 输出文件名 + output_file = os.path.join(save_path, f"{dem_type}_{west}_{east}_{south}_{north}.tif") + + # 写入合并后的文件 + with rasterio.open(output_file, "w", **out_meta) as dest: + dest.write(mosaic) + + # 关闭所有源文件 + for src in src_files_to_mosaic: + src.close() + + elapsed_time = time.time() - start_time + file_size = os.path.getsize(output_file) / (1024 * 1024) + + print(f"✓ 合并完成!") + print(f" 输出文件: {output_file}") + print(f" 文件大小: {file_size:.2f} MB") + print(f" 用时: {elapsed_time:.1f} 秒") + + return output_file + + except ImportError: + print("✗ rasterio 未安装，无法合并文件") + print(" 安装方法: pip install rasterio") + return None + except Exception as e: + print(f"✗ 合并失败: {e}") + import traceback + traceback.print_exc() + return None + + +def download_srtm_cgiar_old(west, south, east, north, save_path, num_workers=4): + """Download SRTM data from CSI-CGIAR (DEPRECATED - tiles not accessible) + + This function is kept as backup but CSI-CGIAR tile downloads are not reliable. + Use download_srtm_cgiar() with OpenTopography API instead. + """ + print("警告: CSI-CGIAR瓦片下载不可靠，建议使用OpenTopography API") + return None + +def process_srtm_hgt_files(west, south, east, north, srtm_data_dir, save_path, output_name=None): + """Process SRTM .hgt files manually (alternative for Python < 3.12) + + Args: + west (float): western longitude boundary + south (float): southern latitude boundary + east (float): eastern longitude boundary + north (float): northern latitude boundary + srtm_data_dir (str): directory containing SRTM .hgt files + save_path (str): directory to save output DEM + output_name (str): output filename (without extension) + + Returns: + str: path to the generated DEM file, or None if failed + + Note: + SRTM .hgt files are 1°×1° tiles with 1201×1201 pixels (3 arc-second) or + 3601×3601 pixels (1 arc-second). This function reads and merges them. + """ + print(f"\n{'='*80}") + print(f"手动处理 SRTM .hgt 文件") + print(f"{'='*80}") + print(f"区域范围: {west}°E - {east}°E, {south}°N - {north}°N") + print(f"SRTM数据目录: {srtm_data_dir}") + print(f"{'='*80}\n") + + try: + import rasterio + from rasterio.merge import merge + from rasterio.crs import CRS + import time + + start_time = time.time() + + # 查找需要的SRTM文件 + hgt_files = [] + for lat in range(int(south), int(north) + 1): + for lon in range(int(west), int(east) + 1): + # 构造文件名 + lat_prefix = 'N' if lat >= 0 else 'S' + lon_prefix = 'E' if lon >= 0 else 'W' + + # SRTM文件名格式: N39E116.hgt + filename = f"{lat_prefix}{abs(lat):02d}{lon_prefix}{abs(lon):03d}.hgt" + filepath = os.path.join(srtm_data_dir, filename) + + if os.path.exists(filepath): + hgt_files.append(filepath) + print(f" ✓ 找到文件: {filename}") + else: + print(f" ⚠ 文件缺失: {filename}") + + if not hgt_files: + print("\n✗ 没有找到任何SRTM文件!") + return None + + print(f"\n找到 {len(hgt_files)} 个SRTM文件") + + # 使用GDAL合并.hgt文件 + merged_file = os.path.join(save_path, f"SRTM_merged_{west}_{east}_{south}_{north}.tif") + + # 构建gdal_merge命令 + gdal_merge_cmd = ['gdal_merge.py', '-o', merged_file, '-of', 'GTiff', '-co', 'COMPRESS=LZW'] + gdal_merge_cmd.extend(hgt_files) + + print(f"\n使用GDAL合并文件...") + result = subprocess.run(gdal_merge_cmd, capture_output=True, text=True) + + if result.returncode != 0 or not os.path.exists(merged_file): + print(f"✗ GDAL合并失败: {result.stderr}") + return None + + print(f"✓ 合并完成: {merged_file}") + + # 裁剪到目标范围 + if output_name is None: + output_name = os.path.join(save_path, f"SRTM_{west}_{east}_{south}_{north}") + + output_file = output_name + '.tif' + + gdal_warp_cmd = [ + 'gdalwarp', + '-te', str(west), str(south), str(east), str(north), + '-of', 'GTiff', + '-co', 'COMPRESS=LZW', + merged_file, + output_file + ] + + print(f"\n裁剪到目标范围...") + result = subprocess.run(gdal_warp_cmd, capture_output=True, text=True) + + if result.returncode != 0 or not os.path.exists(output_file): + print(f"✗ GDAL裁剪失败: {result.stderr}") + return None + + elapsed_time = time.time() - start_time + + print(f"\n✓ SRTM DEM 生成完成!") + print(f" 输出文件: {output_file}") + if os.path.exists(output_file): + file_size = os.path.getsize(output_file) / (1024 * 1024) + print(f" 文件大小: {file_size:.2f} MB") + print(f" 用时: {elapsed_time:.1f} 秒") + + # 清理临时文件 + if os.path.exists(merged_file) and merged_file != output_file: + os.remove(merged_file) + print(f" 清理临时文件: {merged_file}") + + return output_file + + except ImportError: + print("✗ rasterio 未安装,无法处理文件") + return None + except Exception as e: + print(f"\n✗ 处理失败: {e}") + import traceback + traceback.print_exc() + return None + +def merge_tif_files(input_pattern, output_file): + """合并多个 TIF 文件为一个文件 + + Args: + input_pattern (str): 输入 TIF 文件的匹配模式 + output_file (str): 输出合并后的 TIF 文件路径 + """ + # 查找所有匹配的 TIF 文件 + tif_files = sorted(glob.glob(input_pattern)) + + if not tif_files: + print(f"未找到匹配的 TIF 文件: {input_pattern}") + return False + + print(f"\n{'='*80}") + print(f"合并 DEM 瓦片") + print(f"{'='*80}") + print(f"找到 {len(tif_files)} 个 TIF 文件待合并") + print(f"输出文件: {output_file}") + print(f"{'='*80}\n") + + # 使用 gdal_merge.py 合并文件 + try: + start_time = __import__('time').time() + cmd = ["gdal_merge.py", "-o", output_file, "-co", "COMPRESS=LZW", "-co", "BIGTIFF=YES"] + tif_files + print(f"执行合并命令...") + + # 使用 tqdm 显示进度 + result = subprocess.run(cmd, capture_output=True, text=True) + + elapsed_time = __import__('time').time() - start_time + + if result.returncode == 0: + # 检查输出文件大小 + if os.path.exists(output_file): + file_size = os.path.getsize(output_file) / (1024 * 1024) # MB + print(f"\n✓ 成功合并文件!") + print(f" 输出文件: {output_file}") + print(f" 文件大小: {file_size:.2f} MB") + print(f" 用时: {elapsed_time:.1f} 秒") + print(f"{'='*80}\n") + return True + else: + print(f"\n✗ 合并失败: {result.stderr}") + print(f"{'='*80}\n") + return False + except Exception as e: + print(f"\n✗ 合并过程中发生错误: {e}") + print(f"{'='*80}\n") + return False + +def convert_format(input_file, output_file, format_type="GTiff", options=None): + """转换 TIF 文件格式 + + Args: + input_file (str): 输入文件路径 + output_file (str): 输出文件路径 + format_type (str): 输出格式类型 (默认: GTiff) + options (list): gdal_translate 的额外选项 + """ + if options is None: + options = [] + + try: + cmd = ["gdal_translate", "-of", format_type] + options + [input_file, output_file] + print(f"执行转换命令: {' '.join(cmd)}") + result = subprocess.run(cmd, capture_output=True, text=True) + + if result.returncode == 0: + print(f"成功转换文件到: {output_file}") + return True + else: + print(f"转换失败: {result.stderr}") + return False + except Exception as e: + print(f"转换过程中发生错误: {e}") + return False + +def process_dem_files(save_path, west, east, south, north, merge=True, convert=True, + output_format="GTiff", convert_options=None, gamma=False, gamma_byteorder='big', + processor='gamma', output_name=None, num_workers=4, + srtm_data_dir=None, opentopo_api_key=None, opentopo_dem_type='SRTMGL1', + fabdem_dir=None): + """处理下载的 DEM 文件：下载、合并和转换 + + Args: + save_path (str): 数据保存目录 + west (float): 西经边界 + east (float): 东经边界 + south (float): 南纬边界 + north (float): 北纬边界 + merge (bool): 是否合并文件 + convert (bool): 是否转换格式 + output_format (str): 输出格式 + convert_options (list): 转换选项 + gamma (bool): 是否转换为 Gamma/ROI_PAC 格式 + gamma_byteorder (str): 字节序 ('big' 或 'little') + processor (str): 处理器类型 ('gamma' 或 'roi_pac') + output_name (str): 输出文件名（不含扩展名） + num_workers (int): 并行下载线程数 (default: 4) + dem_source (str): DEM数据源 ('copernicus', 'nasadem' 或 'srtm', default: 'copernicus') + srtm_data_dir (str): SRTM数据目录 (仅srtm源需要) + """ + if convert_options is None: + convert_options = ["-co", "COMPRESS=LZW", "-co", "TILED=YES"] + + # ============ 优先尝试本地 FABDEM 瓦片库 ============ + if fabdem_dir and os.path.isdir(fabdem_dir): + print(f"\n{'='*80}") + print(f"优先使用本地 FABDEM 瓦片库: {fabdem_dir}") + print(f"{'='*80}") + try: + from make_local_dem import (find_needed_tiles, extract_tiles_from_zips, + tiles_to_gamma_dem) + import tempfile, shutil + + tiles = find_needed_tiles(west, south, east, north) + print(f"需要 {len(tiles)} 个 1°×1° FABDEM 瓦片") + + temp_dir = tempfile.mkdtemp(prefix='fabdem_tiles_') + try: + tif_files = extract_tiles_from_zips(tiles, fabdem_dir, temp_dir) + if tif_files: + # 一步完成: VRT → ENVI 二进制 → .dem（仅一次大文件写入） + out_name = output_name if output_name else os.path.join(save_path, 'out') + dem_file, dem_par_file = tiles_to_gamma_dem( + tif_files, out_name, west, south, east, north, gamma_byteorder) + if dem_file and dem_par_file: + print(f"\n✓ 本地 FABDEM 生成 GAMMA DEM 成功!") + print(f" DEM: {dem_file}") + print(f" PAR: {dem_par_file}") + shutil.rmtree(temp_dir, ignore_errors=True) + return + else: + print("⚠ 本地 FABDEM 转换失败，回退到网络下载") + else: + print("⚠ 未从本地 FABDEM 提取到瓦片，回退到网络下载") + finally: + shutil.rmtree(temp_dir, ignore_errors=True) + except ImportError as e: + print(f"⚠ 无法导入 make_local_dem 模块 ({e})，回退到网络下载") + except Exception as e: + print(f"⚠ 本地 FABDEM 处理异常 ({e})，回退到网络下载") + + # ============ 回退: 网络下载 ============ + # 检查是否使用预下载的SRTM数据 + if srtm_data_dir and os.path.isdir(srtm_data_dir): + print(f"\n{'='*80}") + print(f"使用预下载的 SRTM 数据") + print(f"{'='*80}") + + if HAS_SRTM: + # Python >= 3.12, 使用srtm库 + dem_file = download_srtm(west, south, east, north, save_path, srtm_data_dir) + else: + # Python < 3.12, 使用手动处理方法 + print("注意: srtm库需要Python >= 3.12,使用手动处理方法") + dem_file = process_srtm_hgt_files(west, south, east, north, srtm_data_dir, + save_path, output_name) + else: + # 自动从OpenTopography下载 + print(f"\n{'='*80}") + print(f"开始下载 DEM 数据...") + print(f"{'='*80}") + print("自动从OpenTopography下载DEM数据") + dem_file = download_srtm_cgiar(west, south, east, north, save_path, + api_key=opentopo_api_key, num_workers=num_workers, + dem_type=opentopo_dem_type) + + if dem_file is None: + print("✗ DEM 生成失败,退出") + return + + current_file = dem_file + + # 生成输出文件名 + if output_name is None: + output_name = os.path.join(save_path, f"DEM_W{int(west)}_E{int(east)}_S{int(south)}_N{int(north)}") + + # 转换到 Gamma/ROI_PAC 格式 + if gamma: + print("\n开始转换到 {} 格式...".format(processor.upper())) + convert_to_gamma(current_file, output_name, gamma_byteorder, processor) + + # 转换到其他格式（非 Gamma/ROI_PAC） + if not gamma and convert: + print("\n开始转换文件格式...") + converted_file = os.path.join(save_path, f"final_DEM_W{int(west)}_E{int(east)}_S{int(south)}_N{int(north)}.tif") + convert_format(current_file, converted_file, output_format, convert_options) + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description='Download DEM data from OpenTopography and convert to Gamma/ROI_PAC format.', + formatter_class=argparse.RawTextHelpFormatter + ) + + # 与 makedem.py 相同的参数 + parser.add_argument('-r', dest='region', help='Research region, west/east/south/north (e.g., 106/110/36/40)') + parser.add_argument('-d', dest='dem', help='Raw dem file that used for further processing') + parser.add_argument('-s', dest='par', help='SLC parameter file of SAR image used for determining research region') + parser.add_argument('-p', dest='processor', help='Interferometry processor. [ gamma or roi_pac ] [default: gamma]') + parser.add_argument('-o', dest='out', help='Output name of the generated DEM') + parser.add_argument('--byteorder', dest='byteorder', choices=['big', 'little'], + help='Byteorder of the generated DEM: big or little. [default: big for gamma and little for roi_pac]') + parser.add_argument('--dir', dest='PATH', help='Processing directory for generating DEM. [default: Current directory]') + parser.add_argument('--num-workers', dest='num_workers', type=int, default=4, + help='Number of parallel download workers (used for batch processing). [default: 4]') + parser.add_argument('--srtm-data-dir', dest='srtm_data_dir', default=None, + help='Directory containing SRTM .hgt files (optional).\n' + 'If not provided, DEM data will be automatically downloaded from OpenTopography.\n' + 'Download SRTM data from: https://srtm.csi.cgiar.org/') + parser.add_argument('--opentopo-api-key', dest='opentopo_api_key', + default='09ad77d34545607fdf5cb182b64ac64e', + help='OpenTopography API key for downloading DEM data.\n' + 'Get a free API key at: https://opentopography.org/myOpenTopo\n' + 'Default key is provided, but you can use your own key.') + parser.add_argument('--opentopo-dem-type', dest='opentopo_dem_type', + choices=['SRTMGL1', 'SRTMGL3', 'NASADEM', 'COP30', 'COP90'], + default='SRTMGL1', + help='DEM type for OpenTopography download. [default: SRTMGL1]\n' + ' - SRTMGL1: SRTM GL1 30m (recommended)\n' + ' - SRTMGL3: SRTM GL3 90m\n' + ' - NASADEM: NASADEM 30m\n' + ' - COP30: Copernicus 30m\n' + ' - COP90: Copernicus 90m') + parser.add_argument('--fabdem-dir', dest='fabdem_dir', default=None, + help='Directory containing local FABDEM ZIP tiles (e.g., /mnt/ZYD/全球FABDEM).\n' + 'If provided, local FABDEM tiles will be used FIRST before network download.\n' + 'Falls back to OpenTopography download if local tiles are unavailable.') + + args = parser.parse_args() + + # 确定工作目录 + if args.PATH: + workdir = args.PATH + else: + workdir = os.getcwd() + + os.chdir(workdir) + + # 确定处理器类型 + if args.processor: + processor = args.processor + else: + processor = 'gamma' + + # 确定字节序 + if args.byteorder: + Byteorder = args.byteorder + else: + if processor == 'gamma': + Byteorder = 'big' + else: + Byteorder = 'little' + + # 确定输出名称 + if args.out: + Name = args.out + else: + Name = "out" + + # 处理已有 DEM 文件的情况 + if args.dem: + dem = args.dem + print('Raw dem file is provided: %s.' % dem) + + # 转换为 TIF 格式（如果不是 TIF） + SUFIX = get_sufix(dem) + if SUFIX != 'tif': + DTIF = dem.replace('.' + SUFIX, '.tif') + call_str = f'gdal_translate {dem} -of GTiff {DTIF}' + os.system(call_str) + DEM = DTIF + else: + DEM = dem + + # 转换到 Gamma/ROI_PAC 格式 + convert_to_gamma(DEM, Name, Byteorder, processor) + + BB = Byteorder + ' endian' + print('') + print('%s %s and %s are generated.' % (BB, Name + '.dem', Name + ('.dem.par' if processor == 'gamma' else '.dem.rsc'))) + print('Congratulations! Done!') + sys.exit(0) + + # 处理从 SLC 参数文件确定区域的情况 + if args.par: + Par = args.par + print("SLC_par file is provided: %s" % Par) + print("DEM over research region will be downloaded automatically based on %s" % Par) + print("DEM data will be downloaded from OpenTopography") + call_str = "SLC_corners " + Par + " > corners.txt" + os.system(call_str) + + File = open("corners.txt", "r") + InfoLine = File.readlines()[8:10] + File.close() + + MinLat = float(InfoLine[0].split(':')[1].split(' max. ')[0]) + MaxLat = float(InfoLine[0].split(':')[2]) + MinLon = float(InfoLine[1].split(':')[1].split(' max. ')[0]) + MaxLon = float(InfoLine[1].split(':')[2]) + + north = int(MaxLat) + 2 + south = int(MinLat) + east = int(MaxLon) + 2 + west = int(MinLon) + elif args.region: + region = args.region + west, south, east, north = read_region(region) + else: + parser.print_usage() + sys.exit(os.path.basename(sys.argv[0]) + ': error: research region, raw_demfile and SLC parameter file, at least one is needed.') + + print('Research region: %s(west) %s(south) %s(east) %s(north)' % (west, south, east, north)) + print('>>> Ready to download DEM over research region.') + + # 处理数据（下载、合并和转换到 Gamma/ROI_PAC 格式） + process_dem_files( + save_path=workdir, + west=west, + east=east, + south=south, + north=north, + merge=True, + convert=False, + gamma=True, + gamma_byteorder=Byteorder, + processor=processor, + output_name=os.path.join(workdir, Name), + num_workers=args.num_workers, + srtm_data_dir=args.srtm_data_dir, + opentopo_api_key=args.opentopo_api_key, + opentopo_dem_type=args.opentopo_dem_type, + fabdem_dir=args.fabdem_dir + ) diff --git a/third_party/PyINT/pyint/makedem_bk.py b/third_party/PyINT/pyint/makedem_bk.py new file mode 100644 index 0000000..df12a49 --- /dev/null +++ b/third_party/PyINT/pyint/makedem_bk.py @@ -0,0 +1,302 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import getopt +import array +import argparse +from skimage import io + +from pyint import _utils as ut + +def get_sufix(STR): + n = len(STR.split('.')) + SUFIX = STR.split('.')[n-1] + + return SUFIX + +def read_region(STR): + WEST = STR.split('/')[0] + EAST = STR.split('/')[1].split('/')[0] + + SOUTH = STR.split(EAST+'/')[1].split('/')[0] + NORTH = STR.split(EAST+'/')[1].split('/')[1] + + WEST =float(WEST) + SOUTH=float(SOUTH) + EAST=float(EAST) + NORTH=float(NORTH) + return WEST,SOUTH,EAST,NORTH + + +def write_demrsc_file(FILE,Corner_LON,Corner_LAT,X_STEP,Y_STEP,WIDTH,LENGTH): + f = open(FILE,'w') + f.write('DATE12 111111-222222\n') + f.write('FILE_LENGTH ' + str(int(LENGTH)) + '\n') + f.write('FILE_TYPE .dem\n') + f.write('PROCESSOR roipac\n') + f.write('PROJECTION LATLON\n') + f.write('RLOOKS 1\n') + f.write('WIDTH ' + str(int(WIDTH)) + '\n') + f.write('XMAX ' + str(int(int(WIDTH)-1)) + '\n') + f.write('XMIN 0\n') + f.write('X_FIRST ' + str(float(Corner_LON)) + '\n') + f.write('X_STEP ' + str(float(X_STEP)) + '\n') + f.write('X_UNIT degrees\n') + f.write('YMAX ' + str(int(int(LENGTH)-1)) + '\n') + f.write('YMIN 0\n') + f.write('Y_FIRST ' + str(float(Corner_LAT)) + '\n') + f.write('Y_STEP ' + str(float(Y_STEP)) + '\n') + f.write('Y_UNIT degrees\n') + f.write('Z_OFFSET 0\n') + f.write('Z_SCALE 1\n') + f.close + + +def write_dempar_file(FILE,Corner_LON,Corner_LAT,X_STEP,Y_STEP,WIDTH,LENGTH,DATA_FORMAT): + DEM_TYPE = 'SRTM1' + Proj = 'EQA' + f=open(FILE,'w') + f.write("Gamma DIFF&GEO DEM/MAP parameter file\n") + f.write("title:\tIMPORTED DEM FROM %s\n" % DEM_TYPE) # SRTM1 (30m) or SRTM3 (90m) + f.write("DEM_projection: %s\n" % Proj) # Projection should be checked. + f.write("data_format: %s\n" % DATA_FORMAT) # INTEGER*2 OR REAL*4 should be modified + f.write("DEM_hgt_offset: 0.00000\n") + f.write("DEM_scale: 1.00000\n") + f.write("width: %s\n" % WIDTH) + f.write("nlines: %s\n" % LENGTH) + f.write("corner_lat: %s decimal degrees\n" % Corner_LAT) + f.write("corner_lon: %s decimal degrees\n" % Corner_LON) + f.write("post_lat: %s decimal degrees\n" % Y_STEP) + f.write("post_lon: %s decimal degrees\n" % X_STEP) + f.write("\n") + f.write("ellipsoid_name: WGS 84\n") + f.write("ellipsoid_ra: 6378137.000 m\n") + f.write("ellipsoid_reciprocal_flattening: 298.2572236\n") + f.write("\n") + f.write("datum_name: WGS 1984\n") + f.write("datum_shift_dx: 0.000 m\n") + f.write("datum_shift_dy: 0.000 m\n") + f.write("datum_shift_dz: 0.000 m\n") + f.write("datum_scale_m: 0.00000e+00\n") + f.write("datum_rotation_alpha: 0.00000e+00 arc-sec\n") + f.write("datum_rotation_beta: 0.00000e+00 arc-sec\n") + f.write("datum_rotation_gamma: 0.00000e+00 arc-sec\n") + f.write("datum_country_list Global Definition, WGS84, World\n") + f.write("\n") + + f.close() + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +######################################################################### + +INTRODUCTION = ''' +############################################################################# + Copy Right(c): 2017-2019, Yunmeng Cao [ymcmrs@gmail.com] + + Generating DEM used in interferometry both for GAMMA and ROI_PAC processor. + + 1) Available raw DEM files can be used, e.g., dem.tif, dem.grd; + 2) If no raw DEM is provided, SRTM-1 (30m) can be downloaded automatically. + + Requirement: + Python 2.7 or higher version. GDAL should be installed in your PC. +''' + +EXAMPLE = ''' + Usage: + makedem.py -r west/east/south/north -d raw_demfile -p processor -o output + makedem.py -r west/east/south/north -p processor <gamma or roi_pac> + makedem.py -d raw_demfile --byteorder <little or big> + + Examples: + makedem.py -r " -118/-116/33/34 " -p gamma -o SouthCalifornia + makedem.py -r " -118/-116/33/34 " --byteorder little + makedem.py -d dem.tif -p roi_pac --byteorder big + makedem.py -s 20101108.slc.par -p roi_pac +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate DEM for interferometry processing.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + #parser.add_argument('-r','--lalo', dest='region', nargs=4, type=float, help='lalo limit') + parser.add_argument('-r',dest = 'region',help='Research region, west/east/south/north.') + parser.add_argument('-d', dest='dem', help='Raw dem file that used for further processing.') + parser.add_argument('-s', dest='par', help='SLC parameter file of SAR image used for determining research region.') + parser.add_argument('-p', dest='processor', help='Interferometry processor. [ gamma or roi_pac ] [default: gamma]') + parser.add_argument('-o', dest='out', help='Output name of the generated DEM.') + parser.add_argument('--byteorder', dest='byteorder', help='Byteorder of the generated DEM: big or little. [default: big for gamma and little for roi_pac]') + parser.add_argument('--dir', dest='PATH', help='Processing directory for generating DEM. [default: Current directory]') + + inps = parser.parse_args() + + if not inps.region and not inps.dem and not inps.par: + parser.print_usage() + sys.exit(os.path.basename(sys.argv[0])+': error: research region, raw_demfile and SLC parameter file, at least one is needed.') + + return inps + +################################################################################ + +def main(argv): + + inps = cmdLineParse() + + if inps.par: + Par = inps.par + print("SLC_par file is provided: %s" % Par) + print("SRTM1 over research region will be downloaded automatically based on %s" % Par) + call_str = "SLC_corners "+ Par + " > corners.txt" + os.system(call_str) + + File = open("corners.txt","r") + InfoLine = File.readlines()[8:10] + File.close() + + MinLat = float(InfoLine[0].split(':')[1].split(' max. ')[0]) + MaxLat = float(InfoLine[0].split(':')[2]) + MinLon = float(InfoLine[1].split(':')[1].split(' max. ')[0]) + MaxLon = float(InfoLine[1].split(':')[2]) + + north = int(MaxLat) + 2 + south = int(MinLat) + east = int(MaxLon) + 2 + west = int(MinLon) + + + if inps.region: + region = inps.region + west,south,east,north = read_region(region) + + #print 'Research region: %s/%s/%s/%s' % west,south,east,north + + if inps.dem: + dem = inps.dem + print('Raw dem file is provided: %s .' % dem) + + if inps.out: Name = inps.out + else: Name = "out" + + if inps.processor: processor = inps.processor + else: processor ='gamma' + + if inps.byteorder: Byteorder = inps.byteorder + else: + if processor == 'gamma': Byteorder = 'big' + else: Byteorder ='little' + + if inps.PATH: workdir = inps.PATH + else: workdir = os.getcwd() + os.chdir(workdir) + + if not inps.dem: + + print('Research region: %s(west) %s(south) %s(east) %s(north)' % (west,south,east,north)) + print('>>> Ready to download SRTM1 dem over research region.') + #call_str='wget -q -O dem.tif "http://ot-data1.sdsc.edu:9090/otr/getdem?north=%f&south=%f&east=%f&west=%f&demtype=SRTMGL1"' % (north,south,east,west) + #os.system(call_str) + #call_str = 'eio --product SRTM1 clip -o dem.tif --bounds ' + str(west) + ' ' + str(south) + ' ' + str(east) + ' ' + str(north) + call_str = 'sardem --bbox ' + str(west) + ' ' + str(south) + ' ' + str(east) + ' ' + str(north) + ' -o ' + Name + '.dem' + os.system(call_str) + print('>>> DEM download finished.') + + DEM_TYPE='SRTM1' + Proj = 'EQA' + + DEM = 'dem.tif' + call_str = 'gdal_translate ' + Name + '.dem' + ' -of GTiff ' + DEM + os.system(call_str) + else: + DEM = inps.dem + + + + SUFIX = get_sufix(DEM) + SS ='.' + SUFIX + DTIF = DEM.replace(SS,'.tif') + + if not SUFIX == 'tif': + call_str = 'gdal_translate ' + DEM + ' -of GTiff ' + DTIF + os.system(call_str) + + + DEM = DTIF + call_str = 'gdalinfo ' + DEM + ' >ttt' + os.system(call_str) + + f = open('ttt') + for line in f: + if 'Origin =' in line: + STR1 = line + AA = STR1.split('Origin =')[1] + Corner_LON = AA.split('(')[1].split(',')[0] + Corner_LAT = AA.split('(')[1].split(',')[1].split(')')[0] + elif 'Pixel Size ' in line: + STR2 = line + AA = STR2.split('Pixel Size =')[1] + Post_LON = AA.split('(')[1].split(',')[0] + Post_LAT = AA.split('(')[1].split(',')[1].split(')')[0] + + elif 'Size is' in line: + STR3 = line + AA =STR3.split('Size is')[1] + WIDTH = AA.split(',')[0] + FILE_LENGTH = AA.split(',')[1] + f.close() + + dem_data = io.imread(DEM) + if dem_data.dtype=='float32': + DATA_FORMAT='REAL*4' + else: + DATA_FORMAT='INTEGER*2' + + + if not sys.byteorder == Byteorder: + dem_data.byteswap(True) + + if processor =='gamma': + DEMDATA = Name + '.dem' + DEMPAR = Name + '.dem.par' + elif processor =='roi_pac': + DEMDATA = Name + '.dem' + DEMPAR = Name + '.dem.rsc' + + dem_data.tofile(DEMDATA) + + if processor =='gamma': + write_dempar_file(DEMPAR,Corner_LON,Corner_LAT,Post_LON,Post_LAT,WIDTH,FILE_LENGTH,DATA_FORMAT) + elif processor =='roi_pac': + write_demrsc_file(DEMPAR,Corner_LON,Corner_LAT,Post_LON,Post_LAT,WIDTH,FILE_LENGTH) + + BB =Byteorder + ' endian' + print('') + print('%s %s and %s are generated.' % (BB,DEMDATA,DEMPAR)) + print('Congratulations! Done!') + #print "Generating %s processor %s and %s is done!" % (processor,DEMDATA,DEMPAR) + sys.exit(1) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/third_party/PyINT/pyint/makedem_pyint.py b/third_party/PyINT/pyint/makedem_pyint.py new file mode 100644 index 0000000..1705b07 --- /dev/null +++ b/third_party/PyINT/pyint/makedem_pyint.py @@ -0,0 +1,203 @@ +#! /usr/bin/env python +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from pathlib import Path + +from pyint import _utils as ut + + +INTRODUCTION = """ +------------------------------------------------------------------- + + Generate radar-coordinates based DEM. + [Geo-coordinates DEM can be downloaded automatically if not provided.] +""" + +EXAMPLE = """Usage: + + makedem_pyint.py projectName --processor gamma +""" + + +def cmdLineParse() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Generate radar-coordinates based DEM.", + formatter_class=argparse.RawTextHelpFormatter, + epilog=INTRODUCTION + "\n" + EXAMPLE, + ) + parser.add_argument("projectName", help="Name of project.") + parser.add_argument( + "-p", + "--processor", + dest="processor", + choices={"gamma", "roi_pac"}, + default="gamma", + help="Interferometry processor. [default: gamma]", + ) + return parser.parse_args() + + +def _run_checked(command: list[str], *, cwd: str | None = None) -> subprocess.CompletedProcess[str]: + result = subprocess.run( + command, + cwd=cwd, + text=True, + capture_output=True, + check=False, + ) + if result.returncode != 0: + detail = (result.stderr or result.stdout or "").strip() + raise RuntimeError(f"Command failed ({result.returncode}): {' '.join(command)}\n{detail}") + return result + + +def _resolve_existing_dem_open_path(source_dem: str) -> Path: + source_path = Path(source_dem) + if source_path.is_file() and source_path.suffix.lower() == ".vrt": + return source_path + + vrt_path = Path(str(source_path) + ".vrt") + if vrt_path.is_file(): + return vrt_path + + if source_path.is_file(): + return source_path + + raise FileNotFoundError(f"Prepared DEM source does not exist: {source_dem}") + + +def _resolve_research_bbox_from_slc_par(slc_par: str) -> tuple[int, int, int, int]: + result = _run_checked(["SLC_corners", slc_par]) + lines = result.stdout.splitlines() + if len(lines) < 10: + raise RuntimeError(f"Unexpected SLC_corners output for {slc_par}") + + lat_line = lines[8].rstrip() + lon_line = lines[9].rstrip() + min_lat = float(lat_line.split(":")[1].split(" max. ")[0]) + max_lat = float(lat_line.split(":")[2]) + min_lon = float(lon_line.split(":")[1].split(" max. ")[0]) + max_lon = float(lon_line.split(":")[2]) + + north = int(max_lat) + 2 + south = int(min_lat) + east = int(max_lon) + 2 + west = int(min_lon) + return west, south, east, north + + +def _cleanup_temp_outputs(paths: list[Path]) -> None: + for path in paths: + try: + if path.exists(): + path.unlink() + except OSError: + continue + + +def _build_dem_from_existing_source( + *, + project_name: str, + processor: str, + slc_par: str, + source_dem: str, + work_dir: Path, +) -> None: + west, south, east, north = _resolve_research_bbox_from_slc_par(slc_par) + source_open_path = _resolve_existing_dem_open_path(source_dem) + clipped_tif = work_dir / f"{project_name}.prepared_source_clip.tif" + clipped_aux = Path(str(clipped_tif) + ".aux.xml") + + print(f"Using prepared DEM source: {source_dem}") + print(f"Clipping prepared DEM window: west={west}, south={south}, east={east}, north={north}") + + _run_checked( + [ + "gdal_translate", + "-projwin", + str(west), + str(north), + str(east), + str(south), + "-of", + "GTiff", + str(source_open_path), + str(clipped_tif), + ], + cwd=str(work_dir), + ) + _run_checked( + [ + "makedem.py", + "-d", + str(clipped_tif), + "-p", + processor, + "-o", + project_name, + ], + cwd=str(work_dir), + ) + _cleanup_temp_outputs([clipped_tif, clipped_aux]) + + +def main(argv: list[str]) -> None: + inps = cmdLineParse() + projectName = inps.projectName + processor = inps.processor + + scratchDir = os.getenv("SCRATCHDIR") + slcDir = scratchDir + "/" + projectName + "/SLC" + templateDir = os.getenv("TEMPLATEDIR") + templateFile = templateDir + "/" + projectName + ".template" + templateDict = ut.update_template(templateFile) + + masterDate = templateDict["masterDate"] + SLC_PAR = slcDir + "/" + masterDate + "/" + masterDate + ".slc.par" + + demDir = os.getenv("DEMDIR") + demDir1 = demDir + "/" + projectName + if not os.path.isdir(demDir1): + os.mkdir(demDir1) + + os.chdir(demDir1) + work_dir = Path(demDir1) + + prepared_dem_source = str(templateDict.get("prepared_dem_source", "") or "").strip() + if prepared_dem_source not in {"", "-"}: + _build_dem_from_existing_source( + project_name=projectName, + processor=processor, + slc_par=SLC_PAR, + source_dem=prepared_dem_source, + work_dir=work_dir, + ) + print(f"Generate DEM for project {projectName} is done.") + sys.exit(0) + + call_str = "makedem.py " + "-s " + SLC_PAR + " -p gamma " + " -o " + projectName + + if "fabdem_dir" in templateDict and templateDict["fabdem_dir"].strip() not in ["", "-"]: + fabdem_dir = templateDict["fabdem_dir"].strip() + if os.path.isdir(fabdem_dir): + call_str += " --fabdem-dir " + fabdem_dir + print("Using local FABDEM directory: %s" % fabdem_dir) + + if "opentopo_api_key" in templateDict and templateDict["opentopo_api_key"].strip() not in ["", "-"]: + call_str += " --opentopo-api-key " + templateDict["opentopo_api_key"].strip() + if "opentopo_dem_type" in templateDict and templateDict["opentopo_dem_type"].strip() not in ["", "-"]: + call_str += " --opentopo-dem-type " + templateDict["opentopo_dem_type"].strip() + + print("Running: %s" % call_str) + os.system(call_str) + + print("Generate DEM for project %s is done." % projectName) + sys.exit(0) + + +if __name__ == "__main__": + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/mintpy_extract_timeseries_to_geptiff.py b/third_party/PyINT/pyint/mintpy_extract_timeseries_to_geptiff.py new file mode 100644 index 0000000..688b713 --- /dev/null +++ b/third_party/PyINT/pyint/mintpy_extract_timeseries_to_geptiff.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +""" +SBAS-InSAR时间序列数据提取脚本 +从H5文件中提取时间序列数据、日期和基线信息 +""" + +import h5py +import numpy as np +import os +import argparse +from osgeo import gdal + + +def save_as_geotiff(data, output_file, nodata=-9999): + """ + 将数据保存为GeoTIFF格式 + + Parameters: + ----------- + data : numpy.ndarray + 要保存的数据（2D数组） + output_file : str + 输出文件路径 + nodata : float + 无效值 + """ + # 设置无效值 + data_masked = np.where(np.isnan(data), nodata, data) + + # 获取数据形状 + rows, cols = data_masked.shape + + # 创建GeoTIFF驱动 + driver = gdal.GetDriverByName('GTiff') + + # 创建数据集 + dataset = driver.Create(output_file, cols, rows, 1, gdal.GDT_Float32) + + # 设置波段 + band = dataset.GetRasterBand(1) + band.WriteArray(data_masked) + band.SetNoDataValue(nodata) + + # 设置无数据值 + band.FlushCache() + + # 关闭数据集 + dataset = None + + print(f" 已保存: {output_file}") + + +def extract_timeseries(h5_file, output_dir='./output'): + """ + 提取SBAS-InSAR时间序列数据 + + Parameters: + ----------- + h5_file : str + H5文件路径 + output_dir : str + 输出目录 + """ + + # 创建输出目录 + os.makedirs(output_dir, exist_ok=True) + + # 读取H5文件 + print(f"正在读取文件: {h5_file}") + with h5py.File(h5_file, 'r') as f: + # 提取日期信息 + dates = f['date'][:] + # 将bytes转为字符串 + dates_str = [d.decode('utf-8') if isinstance(d, bytes) else d for d in dates] + print(f"时间序列长度: {len(dates_str)}") + print(f"日期范围: {dates_str[0]} 到 {dates_str[-1]}") + + # 提取基线信息 + bperp = f['bperp'][:] + print(f"基线形状: {bperp.shape}") + + # 提取时间序列数据 + timeseries = f['timeseries'][:] + print(f"时间序列数据形状: {timeseries.shape} (时间, 行, 列)") + + # 保存日期信息 + date_file = os.path.join(output_dir, 'dates.txt') + with open(date_file, 'w') as f_out: + for i, date in enumerate(dates_str): + f_out.write(f"{i+1} {date}\n") + print(f"日期信息已保存到: {date_file}") + + # 保存基线信息 + bperp_file = os.path.join(output_dir, 'bperp.npy') + np.save(bperp_file, bperp) + print(f"基线信息已保存到: {bperp_file}") + + # 保存时间序列数据（numpy格式） + timeseries_file = os.path.join(output_dir, 'timeseries.npy') + np.save(timeseries_file, timeseries) + print(f"时间序列数据已保存到: {timeseries_file}") + + # 保存每个时间点为单独的GeoTIFF文件 + print("\n正在保存每个时间点的GeoTIFF文件...") + geotiff_dir = os.path.join(output_dir, 'geotiff') + os.makedirs(geotiff_dir, exist_ok=True) + + for i, date in enumerate(dates_str): + # 保存为GeoTIFF + geotiff_file = os.path.join(geotiff_dir, f'timeseries_{date}.tif') + save_as_geotiff(timeseries[i], geotiff_file) + print(f"已保存 {len(dates_str)} 个时间点的GeoTIFF文件") + + # 计算并保存累积形变 + print("\n计算累积形变...") + cumulative = np.cumsum(timeseries, axis=0) + cumulative_file = os.path.join(output_dir, 'cumulative_deformation.npy') + np.save(cumulative_file, cumulative) + print(f"累积形变已保存到: {cumulative_file}") + + # 保存累积形变为GeoTIFF + print("\n正在保存累积形变GeoTIFF文件...") + for i, date in enumerate(dates_str): + cumulative_file = os.path.join(geotiff_dir, f'cumulative_{date}.tif') + save_as_geotiff(cumulative[i], cumulative_file) + print(f"已保存 {len(dates_str)} 个累积形变GeoTIFF文件") + + # 保存统计信息 + stats_file = os.path.join(output_dir, 'statistics.txt') + with open(stats_file, 'w') as f_out: + f_out.write("SBAS-InSAR时间序列统计信息\n") + f_out.write("=" * 50 + "\n\n") + f_out.write(f"总时间点数: {len(dates_str)}\n") + f_out.write(f"影像尺寸: {timeseries.shape[1]} 行 x {timeseries.shape[2]} 列\n") + f_out.write(f"日期范围: {dates_str[0]} 到 {dates_str[-1]}\n\n") + f_out.write("形变统计 (单位: mm):\n") + f_out.write(f" 最小值: {np.nanmin(timeseries):.4f}\n") + f_out.write(f" 最大值: {np.nanmax(timeseries):.4f}\n") + f_out.write(f" 平均值: {np.nanmean(timeseries):.4f}\n") + f_out.write(f" 标准差: {np.nanstd(timeseries):.4f}\n") + print(f"统计信息已保存到: {stats_file}") + + print("\n数据提取完成！") + print(f"所有数据已保存到目录: {output_dir}") + + +if __name__ == '__main__': + # 解析命令行参数 + parser = argparse.ArgumentParser( + description='从SBAS-InSAR的H5文件中提取时间序列数据并保存为GeoTIFF格式' + ) + parser.add_argument( + 'h5_file', + type=str, + help='输入的H5文件路径' + ) + parser.add_argument( + '-o', '--output', + type=str, + default='./extracted_data', + help='输出目录路径 (默认: ./extracted_data)' + ) + + args = parser.parse_args() + + # 执行提取 + extract_timeseries(args.h5_file, args.output) \ No newline at end of file diff --git a/third_party/PyINT/pyint/mintpy_h5_form_utm_to_wgs84.py b/third_party/PyINT/pyint/mintpy_h5_form_utm_to_wgs84.py new file mode 100644 index 0000000..a1bbcb5 --- /dev/null +++ b/third_party/PyINT/pyint/mintpy_h5_form_utm_to_wgs84.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +""" +将 HDF5 文件的坐标系统从 UTM 转换为 WGS84 地理坐标 +同时保持与 MintPy tsview.py 的兼容性 +""" + +import h5py +import numpy as np +import copy +import argparse +import os + +def utm_to_wgs84(easting, northing, zone, northern=True): + """ + 将 UTM 坐标转换为 WGS84 大地坐标 + """ + # UTM 参数 + a = 6378137.0 # WGS84 长半轴 + e = 0.081819190842622 # WGS84 第一偏心率 + k0 = 0.9996 # 比例因子 + + # 中央经线 + lon0 = (zone - 1) * 6 - 180 + 3 + + # 调整东伪偏移 + x = easting - 500000.0 + + # 如果是南半球，调整北伪偏移 + if not northern: + y = northing - 10000000.0 + else: + y = northing + + # M = y / k0 + mu = y / (a * (1 - e**2 / 4 - 3 * e**4 / 64 - 5 * e**6 / 256) * k0) + + e1 = (1 - np.sqrt(1 - e**2)) / (1 + np.sqrt(1 - e**2)) + + # 纬度计算 + N1 = a / np.sqrt(1 - e**2 * np.sin(mu)**2) + T1 = np.tan(mu)**2 + C1 = e1**2 * np.cos(mu)**2 + R1 = a * (1 - e**2) / (1 - e**2 * np.sin(mu)**2)**(3/2) + D = x / (N1 * k0) + + lat = mu - N1 * np.tan(mu) / R1 * (D**2 / 2 + (5 + 3 * T1 + 10 * C1 - 4 * C1**2 - 9 * e1**2) * D**4 / 24 + (61 + 90 * T1 + 298 * C1 + 45 * T1**2 - 252 * e1**2 - 3 * C1**2) * D**6 / 720) + + # 经度计算 + lon = lon0 * np.pi / 180 + D * (1 + (1 + 3 * e1**2 + 2 * e1**4) * D**2 / 6 + (2 - e1**2) * D**4 / 120) / np.cos(mu) + + # 转换为角度 + lat_deg = np.degrees(lat) + lon_deg = np.degrees(lon) + + return lon_deg, lat_deg + + +def main(): + parser = argparse.ArgumentParser( + description='将 HDF5 文件转换为 WGS84 地理坐标（与 MintPy 兼容）', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=''' +示例: + python convert_to_wgs84.py input.h5 -o output.h5 + +说明: + 脚本会将文件完全转换为 WGS84 地理坐标系统， + 同时设置正确的属性以保持与 MintPy 的兼容性。 + ''' + ) + parser.add_argument('input', help='输入 HDF5 文件路径') + parser.add_argument('-o', '--output', help='输出 HDF5 文件路径（默认：输入文件名_WGS84.h5）') + + args = parser.parse_args() + + # 输入和输出文件 + input_file = args.input + + if args.output: + output_file = args.output + else: + # 自动生成输出文件名 + base, ext = os.path.splitext(input_file) + output_file = f"{base}_WGS84{ext}" + + print(f"正在读取文件: {input_file}") + f_in = h5py.File(input_file, 'r') + + # 读取所有数据集 + datasets = {} + for key in f_in.keys(): + datasets[key] = f_in[key][:] + print(f" 读取数据集: {key}, 形状: {datasets[key].shape}") + + # 读取所有属性 + attrs = dict(f_in.attrs) + print(f"\n共有 {len(attrs)} 个属性") + + # 解析 UTM 坐标信息 + x_first = float(attrs['X_FIRST']) + y_first = float(attrs['Y_FIRST']) + x_step = float(attrs['X_STEP']) + y_step = float(attrs['Y_STEP']) + width = int(attrs['WIDTH']) + length = int(attrs['LENGTH']) + utm_zone = attrs['UTM_ZONE'] + + # 获取参考点位置（像素坐标） + ref_x = int(attrs.get('REF_X', 0)) + ref_y = int(attrs.get('REF_Y', 0)) + + # 转换 UTM Zone + zone_num = int(''.join(filter(str.isdigit, utm_zone))) + is_northern = 'N' in utm_zone.upper() + + print(f"\n原始坐标系统:") + print(f" UTM Zone: {utm_zone} (EPSG: {attrs['EPSG']})") + print(f" 左上角 UTM: ({x_first}, {y_first})") + print(f" 步长: X={x_step}m, Y={y_step}m") + print(f" 参考点像素: ({ref_x}, {ref_y})") + + # 生成网格坐标 + x_coords = x_first + np.arange(width) * x_step + y_coords = y_first + np.arange(length) * y_step + + # 转换为 WGS84 + print(f"\n正在转换为 WGS84...") + lon_grid, lat_grid = np.meshgrid(x_coords, y_coords) + wgs84_lon, wgs84_lat = utm_to_wgs84(lon_grid, lat_grid, zone_num, is_northern) + + # 计算新的坐标参数（注意：lon 是经度，lat 是纬度） + # X 对应经度，Y 对应纬度 + lon_first = wgs84_lon[0, 0] + lat_first = wgs84_lat[0, 0] + lon_step = abs(wgs84_lon[0, 1] - wgs84_lon[0, 0]) if width > 1 else 0 + lat_step = abs(wgs84_lat[1, 0] - wgs84_lat[0, 0]) if length > 1 else 0 + + # 计算参考点的经纬度 + # REF_X 对应 X 轴（经度），REF_Y 对应 Y 轴（纬度） + ref_lon = wgs84_lon[ref_y, ref_x] # 经度 + ref_lat = wgs84_lat[ref_y, ref_x] # 纬度 + + # 更新属性 - 完全转换为 WGS84 + attrs_new = copy.deepcopy(attrs) + + # 更新坐标系统相关属性 + attrs_new['EPSG'] = 4326 # WGS84 + attrs_new['X_FIRST'] = lon_first # 经度 + attrs_new['Y_FIRST'] = lat_first # 纬度 + attrs_new['X_STEP'] = lon_step + attrs_new['Y_STEP'] = -lat_step # 保持Y方向向下为负 + attrs_new['X_UNIT'] = 'degrees' + attrs_new['Y_UNIT'] = 'degrees' + + # 更新参考坐标 - 现在是真正的经纬度 + attrs_new['REF_LON'] = ref_lon + attrs_new['REF_LAT'] = ref_lat + + # 保留原始 UTM 信息作为备份（带 _ORIG 后缀） + attrs_new['UTM_ZONE_ORIG'] = attrs_new.get('UTM_ZONE', '') + attrs_new['EPSG_ORIG'] = attrs_new.get('EPSG', '') + + # 关键：移除或清空 UTM_ZONE 属性，让 MintPy 识别为地理坐标系统 + if 'UTM_ZONE' in attrs_new: + del attrs_new['UTM_ZONE'] + + # 更新角点坐标 + attrs_new['LAT_REF1'] = lat_first + attrs_new['LAT_REF2'] = wgs84_lat[0, -1] + attrs_new['LAT_REF3'] = wgs84_lat[-1, 0] + attrs_new['LAT_REF4'] = wgs84_lat[-1, -1] + attrs_new['LON_REF1'] = lon_first + attrs_new['LON_REF2'] = wgs84_lon[0, -1] + attrs_new['LON_REF3'] = wgs84_lon[-1, 0] + attrs_new['LON_REF4'] = wgs84_lon[-1, -1] + + # 设置其他相关属性 + attrs_new['COORD_SYSTEM'] = 'GEO' # 地理坐标系统 + + print(f"\n新坐标系统 (WGS84):") + print(f" EPSG: {attrs_new['EPSG']}") + print(f" 左上角: ({attrs_new['X_FIRST']:.6f}, {attrs_new['Y_FIRST']:.6f})") + print(f" 步长: X={attrs_new['X_STEP']:.6f}°, Y={attrs_new['Y_STEP']:.6f}°") + print(f" 参考点经纬度: ({attrs_new['REF_LON']:.6f}, {attrs_new['REF_LAT']:.6f})") + print(f" 参考点像素: ({ref_x}, {ref_y})") + + # 创建新的 HDF5 文件 + print(f"\n正在写入文件: {output_file}") + f_out = h5py.File(output_file, 'w') + + # 写入所有数据集 + for key, data in datasets.items(): + f_out.create_dataset(key, data=data) + print(f" 写入数据集: {key}") + + # 写入所有属性 + for key, value in attrs_new.items(): + f_out.attrs[key] = value + + f_in.close() + f_out.close() + + print(f"\n完成！转换后的文件已保存为: {output_file}") + print("\n说明:") + print(" - 文件已完全转换为 WGS84 地理坐标系统") + print(" - 所有坐标属性已更新为经纬度") + print(" - 原始 UTM 信息已保存为 _ORIG 后缀的属性") + print(" - 应该与 MintPy tsview.py 兼容") + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/third_party/PyINT/pyint/mintpy_ssa.py b/third_party/PyINT/pyint/mintpy_ssa.py new file mode 100644 index 0000000..6f92b0a --- /dev/null +++ b/third_party/PyINT/pyint/mintpy_ssa.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +import argparse +import sys +import h5py +import numpy as np +from ssa import SSA +import time, math +from mintpy.utils import readfile, writefile, ptime, utils as ut +from mintpy.objects import timeseries + +def create_parser(): + parser=argparse.ArgumentParser() + parser.add_argument('timeseries_file', + help='time-series file to do SSA \n' + 'i.e.:geo_timeseries_GACOS_ramp_demErr.h5 (MintPy)\n') + parser.add_argument("-w", "--window", + type=int, + help='Widow length (default is 7) ') + parser.add_argument('-o', '--output', dest='outfile', help='output file name (default ssa.h5)') + # args=parser.parse_args() + # computing + # parser = arg_group.add_memory_argument(parser) + return parser +def cmd_line_parse(iargs=None): + parser = create_parser() + inps = parser.parse_args(args=iargs) + inps.key = readfile.read_attribute(inps.timeseries_file)['FILE_TYPE'] + if inps.key not in ['timeseries', 'giantTimeseries', 'HDFEOS']: + raise Exception('input file is {}, NOT timeseries!'.format(inps.key)) + if not inps.window: + inps.window = int(7) + if not inps.outfile: + outname = 'ssa' + outname += '.h5' + inps.outfile = outname + tsobj = timeseries(inps.timeseries_file) + return inps +def is_number(s): + if s==0: + return False + elif math.isnan(s): + return False + else: + return True +def main(iargs=None): + # if cmd: + # iargs = cmd.split()[1:] + inps = cmd_line_parse(iargs) + date_list = timeseries(inps.timeseries_file).get_date_list() + data, atr = readfile.read(inps.timeseries_file) + length, width = int(atr['LENGTH']), int(atr['WIDTH']) + N_dates=len(date_list) + # with h5py.File(inps.timeseries_file,'r') as f: + # data = f['timeseries'][:] + # date_list = f['date'][:] + data_matrix = np.empty((len(date_list),length,width)) + data_matrix[:] = np.nan + start_time = time.time() + for i in range(length): + for j in range(width): + if is_number(sum(data[:,i,j])): + a=SSA(data[:,i,j], inps.window) + data_matrix[:,i,j]=a.reconstruct([0,1]) # Grouping recostructed components, by defaults it takes F1 and F2 as trend + m, s = divmod(time.time()-start_time, 60) + block= [0,N_dates,0,length,0,width] + writefile.layout_hdf5(inps.outfile, metadata=atr, ref_file=inps.timeseries_file) + writefile.write_hdf5_block(inps.outfile, + data=data_matrix, + datasetName='timeseries', + block=block, + print_msg=False) + + + print(inps.timeseries_file) + print(inps.window) + # print(date_list) + print(length,width) + print('time used: {:02.0f} mins {:02.1f} secs.'.format(m, s)) +if __name__ == '__main__': + main(sys.argv[1:]) + diff --git a/third_party/PyINT/pyint/operation.csh b/third_party/PyINT/pyint/operation.csh new file mode 100644 index 0000000..691259d --- /dev/null +++ b/third_party/PyINT/pyint/operation.csh @@ -0,0 +1,126 @@ +#!/bin/csh -f + +if ($#argv != 7) then + echo "" + echo "Usage: operation.csh master_ztd master_ztd.rsc slave_ztd slave_ztd.rsc reference_point incidence_angle dem_grid" + echo " Performs gacos correction over the phasefilt.grd files for Sentinel 1" + echo "" + echo "It works in each directory where the phase grids are located. This script works jointly with GACOS_correction.csh" + echo "" + echo "Reference point in radar coordinates (text file)" + echo "" + echo "Indicence angle in degrees from SAT_look (float/integer)" + echo "" + exit 1 + endif + +#reference point/stable point in radar coordinates +set reference_point = $5 + +#incidence angle obtained with the "SAT_look" script in degrees +set incidence = $6 + +#wavelength for Sentinel 1 (m) +set wavelength = 0.0554658 +set pi = 3.141592653589793238462 + +#######FIRST ZTD to grid######### +set x_first_d1 = `cat $2|grep X_FIRST|awk '{print $2}'` +set y_first_d1 = `cat $2|grep Y_FIRST|awk '{print $2}'` +set width_d1 = `cat $2|grep WIDTH|awk '{print $2}'` +set length_d1 = `cat $2|grep FILE_LENGTH|awk '{print $2}'` +set x_step_d1 = `cat $2|grep X_STEP|awk '{print $2}'` +set y_step_d1 = `cat $2|grep X_STEP|awk '{print $2}'` +set date_ztd_d1 = `echo $1|awk -F/ '{print $NF}'|cut -c1-8` +gmt xyz2grd $1 -G"date1_ztd.grd" -RLT$x_first_d1/$y_first_d1/$width_d1/$length_d1 -I$x_step_d1/$y_step_d1 -ZTLf -di0 -r + +#######SECOND ZTD to grid######### +set x_first_d2 = `cat $4|grep X_FIRST|awk '{print $2}'` +set y_first_d2 = `cat $4|grep Y_FIRST|awk '{print $2}'` +set width_d2 = `cat $4|grep WIDTH|awk '{print $2}'` +set length_d2 = `cat $4|grep FILE_LENGTH|awk '{print $2}'` +set x_step_d2 = `cat $4|grep X_STEP|awk '{print $2}'` +set y_step_d2 = `cat $4|grep X_STEP|awk '{print $2}'` +set date_ztd_d2 = `echo $3|awk -F/ '{print $NF}'|cut -c1-8` +gmt xyz2grd $3 -G"date2_ztd.grd" -RLT$x_first_d2/$y_first_d2/$width_d2/$length_d2 -I$x_step_d2/$y_step_d2 -ZTLf -di0 -r + +#TIME DIFFERENCE +gmt grdmath date2_ztd.grd date1_ztd.grd SUB = zpddm.grd + +#Checking GACOS grids within the DEM boundaries +set xmin_dem = `gmt grdinfo -C $7|awk '{print $2}'` +set xmax_dem = `gmt grdinfo -C $7|awk '{print $3}'` +set ymin_dem = `gmt grdinfo -C $7|awk '{print $4}'` +set ymax_dem = `gmt grdinfo -C $7|awk '{print $5}'` + +set xmin_gacos = `gmt grdinfo -C zpddm.grd|awk '{print $2}'` +set xmax_gacos = `gmt grdinfo -C zpddm.grd|awk '{print $3}'` +set ymin_gacos = `gmt grdinfo -C zpddm.grd|awk '{print $4}'` +set ymax_gacos = `gmt grdinfo -C zpddm.grd|awk '{print $5}'` + +set check = `echo "$xmin_dem < $xmin_gacos && $ymin_dem < $ymin_gacos && $xmax_dem > $xmax_gacos && $ymax_dem > $ymax_gacos"| bc` + +if ($check == 0) then + echo "Seems like your DEM is not large enough. GACOS grids dimensions must be within the DEM." + echo "DEM dimensions: xmin: $xmin_dem xmax: $xmax_dem ymin: $ymin_dem ymax: $ymax_dem" + echo "GACOS dimensions: xmin: $xmin_gacos xmax: $xmax_gacos ymin: $ymin_gacos ymax: $ymax_gacos" + exit 1 +else + echo "GACOS grid dimensions within DEM boundaries... continue..." +endif + +#PROJECT TO RADAR COORDINATES +proj_ll2ra.csh trans.dat zpddm.grd zpddm_ra.grd + +#RESAMPLE ZTD FILES WITH UNWRAP GRID PARAMETERS +set xmin = `gmt grdinfo -C phasefilt.grd|awk '{print $2}'` +set xmax = `gmt grdinfo -C phasefilt.grd|awk '{print $3}'` +set ymin = `gmt grdinfo -C phasefilt.grd|awk '{print $4}'` +set ymax = `gmt grdinfo -C phasefilt.grd|awk '{print $5}'` +set xinc = `gmt grdinfo -C phasefilt.grd|awk '{print $8}'` +set yinc = `gmt grdinfo -C phasefilt.grd|awk '{print $9}'` +gmt grdsample zpddm_ra.grd -Gresample_zpddm.grd -R$xmin/$xmax/$ymin/$ymax -I$xinc/$yinc -r + + +#REFERENCE POINT +set ref_value = `gmt grdtrack $reference_point -Gresample_zpddm.grd -Z` +set ref_value_phase = `gmt grdtrack $reference_point -Gphasefilt.grd -Z` +gmt grdmath resample_zpddm.grd $ref_value SUB = szpddm.grd +gmt grdmath phasefilt.grd $ref_value_phase SUB = phasefilt_ref.grd + +#Checking reference point values +if ($ref_value == "" || $ref_value_phase == "") then + echo "Problems with the reference point. Is the reference point within the AOI?" + exit 1 +endif + +#FROM METER TO PHASE +#gmt grdmath szpddm.grd 4 MUL $pi MUL $wavelength DIV = szpddm_phase.grd +#multiplying by -4 instead of 4 (edition: 13/4/2023) +gmt grdmath szpddm.grd -4 MUL $pi MUL $wavelength DIV = szpddm_phase.grd + +#PROJECTION FROM ZENITH VIEW TO LOS +gmt grdmath szpddm_phase.grd $incidence COSD DIV = szpddm_phase_LOS.grd + +#CORRECTION WITH GACOS DATA +#attention: Here the phasefilt.grd is corrected, therefore for the unwrap process this output or the detrended output +#should be used. +gmt grdmath phasefilt_ref.grd szpddm_phase_LOS.grd SUB = phasefilt_GACOS_corrected.grd + +#DETRENDING +gmt grdtrend phasefilt_GACOS_corrected.grd -N3r -Dphasefilt_GACOS_corrected_detrended.grd + +#checking existence of final outputs +if !(-f phasefilt_GACOS_corrected.grd || -f phasefilt_GACOS_corrected_detrended.grd) then + echo "Seems like there was an issue correcting with GACOS $1 and $3" + exit 1 +endif + +#clean up +# +rm date1_ztd.grd date2_ztd.grd +rm zpddm.grd zpddm_ra.grd resample_zpddm.grd +rm szpddm_phase.grd +rm phasefilt_ref.grd +rm szpddm_phase_LOS.grd +echo "corrections done with $date_ztd_d1 and $date_ztd_d2 over phasefilt.grd" diff --git a/third_party/PyINT/pyint/out.dem b/third_party/PyINT/pyint/out.dem new file mode 100644 index 0000000..bd9b847 Binary files /dev/null and b/third_party/PyINT/pyint/out.dem differ diff --git a/third_party/PyINT/pyint/out.dem.par b/third_party/PyINT/pyint/out.dem.par new file mode 100644 index 0000000..dda0c34 --- /dev/null +++ b/third_party/PyINT/pyint/out.dem.par @@ -0,0 +1,27 @@ +Gamma DIFF&GEO DEM/MAP parameter file +title: IMPORTED DEM FROM Copernicus30 +DEM_projection: EQA +data_format: INTEGER*2 +DEM_hgt_offset: 0.00000 +DEM_scale: 1.00000 +width: 4800 +nlines: 3600 +corner_lat: 26.000416666674390 decimal degrees +corner_lon: 99.999583333269726 decimal degrees +post_lat: -0.000833333333333 decimal degrees +post_lon: 0.000833333333333 decimal degrees + +ellipsoid_name: WGS 84 +ellipsoid_ra: 6378137.000 m +ellipsoid_reciprocal_flattening: 298.2572236 + +datum_name: WGS 1984 +datum_shift_dx: 0.000 m +datum_shift_dy: 0.000 m +datum_shift_dz: 0.000 m +datum_scale_m: 0.00000e+00 +datum_rotation_alpha: 0.00000e+00 arc-sec +datum_rotation_beta: 0.00000e+00 arc-sec +datum_rotation_gamma: 0.00000e+00 arc-sec +datum_country_list Global Definition, WGS84, World + diff --git a/third_party/PyINT/pyint/phase2los.py b/third_party/PyINT/pyint/phase2los.py new file mode 100644 index 0000000..3cff4db --- /dev/null +++ b/third_party/PyINT/pyint/phase2los.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +#! /usr/bin/env python +################################################################# +### This program is part of PyINT ### +### Copy Right (c): 2019, Wei Chen ### +### Author: Wei Chen ### +### Contact : chenweicug@126.com ### +################################################################# +import numpy as np +import os +import sys +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + +INTRODUCTION = ''' +------------------------------------------------------------------- + convert interferogram to los displacement and get the inc angle and azi angle. + +''' + +EXAMPLE = ''' + Usage: + phase2los.py projectName Mdate Sdate + phase2los.py PacayaT163TsxHhA 20150102 20150601 +------------------------------------------------------------------- +''' + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='convert interferogram to los displacement and get the inc angle and azi angle.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('Mdate',help='Master date.') + parser.add_argument('Sdate',help='Slave date.') + + inps = parser.parse_args() + return inps + + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Mdate = inps.Mdate + Sdate= inps.Sdate + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + satelite = templateDict['satelite'] + rlks = templateDict['range_looks'] + + projectDir = scratchDir + '/' + projectName + ifgDir = projectDir + '/ifgrams' + simDir = scratchDir + '/' + projectName + "/DEM" + losDir = projectDir + '/LosResult' + slcDir = projectDir + '/SLC' + + + if not os.path.isdir(losDir): os.mkdir(losDir) + + Pair = Mdate + '-' + Sdate + workDir = losDir + '/' + Pair + if not os.path.isdir(workDir): os.mkdir(workDir) + workdir = ifgDir + '/' + Pair + UTMDEMpar = simDir + '/'+ Mdate + '_'+ rlks + 'rlks.utm.dem.par' + GeoUNW = workdir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.unw' + GeoATMCOR_UNW = workDir + '/geo_' + Pair + '_' + rlks + 'rlks.diff_filt.atmcor.unw' + call_str = 'data2geotiff ' + UTMDEMpar + ' ' + GeoATMCOR_UNW + ' 2 ' + GeoATMCOR_UNW +'.tif' + os.system(call_str) + call_str = 'data2geotiff ' + UTMDEMpar + ' ' + GeoUNW + ' 2 ' + GeoUNW +'.tif' + os.system(call_str) + call_str = 'gdal_translate -of GSBG ' + ' ' + GeoUNW +'.tif' + ' ' + workDir + '/' + GeoUNW + '.grd' + os.system(call_str) + call_str = 'gdal_translate -of GSBG ' + ' ' + GeoATMCOR_UNW +'.tif' + ' ' + workDir + '/' + GeoATMCOR_UNW +'.grd' + os.system(call_str) + if satelite == 'CSK': + wavelength = 0.0312283810417 + elif satelite == 'TSX': + wavelength = 0.03106657823461874 + elif satelite == 'S1A': + wavelength = 0.0554657647 + elif satelite == 'ALOS2': + wavelength = 0.2424525 + elif satelite == 'ALOS': + wavelength = 0.2360571 + elif satelite == 'ENVISAT': + wavelength = 0.056 + call_str = 'grdmath ' + workDir + '/' + GeoUNW + '.grd' + str(wavelength) + ' 1 3.141592653589 MUL DIV MUL = ' + workDir + '/los.grd' + os.system(call_str) + call_str = 'grdmath ' + workDir + '/' + GeoATMCOR_UNW +'.grd' + str(wavelength) + ' 1 3.141592653589 MUL DIV MUL = ' + workDir + '/los_atmcor.grd' + os.system(call_str) + print('the los displacement of ' + Pair + " has successfully done") + print ("#####################run data2inc_azi###################") + Mslcpar = slcDir + '/' + Mdate + '/' + Mdate + '.slc.par' + UTMDEM = simDir + '/'+ Mdate + '_'+ rlks + 'rlks.utm.dem' + OFF = workdir + '/' + Pair +'_' + rlks + 'rlks.off' + os.chdir(workDir) + call_str = 'data2inc_azi ' + Mslcpar + ' ' + OFF + ' ' + UTMDEMpar + ' ' + UTMDEM + os.system(call_str) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/third_party/PyINT/pyint/phase2los_all.py b/third_party/PyINT/pyint/phase2los_all.py new file mode 100644 index 0000000..38f9ad2 --- /dev/null +++ b/third_party/PyINT/pyint/phase2los_all.py @@ -0,0 +1,111 @@ +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + + +def work(data0): + cmd = data0[0] + err_file = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + cmd[3] + '\n' + #print(aa) + with open(err_file, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + convert interferograms phase to los deformation for one project using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + phase2los_all.py projectName + phase2los_all.py projectName --parallel 4 + diff_gamma_all.py projectName --parallel 4 --ifgramList-txt /test/ifgram_list.txt +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Geneate differential interferograms for one project using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + parser.add_argument('--ifgarmList-txt', dest='ifgarmListTxt', help='provided ifgram_list_txt. default: using ifgram_list.txt under projectName folder.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + projectDir = scratchDir + '/' + projectName + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + if inps.ifgarmListTxt: ifgramList_txt = inps.ifgarmListTxt + else: ifgramList_txt = scratchDir + '/' + projectName + '/ifgram_list.txt' + ifgList0 = ut.read_txt2array(ifgramList_txt) +# ifgList = ifgList0[:,0] + if len(ifgList0)==3: + ifgList=ifgList0[0] + ifgList=[ifgList] + + else: + ifgList=ifgList0[:,0] + + err_txt = scratchDir + '/' + projectName + '/diff_gamma_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + for i in range(len(ifgList)): + m0 = ut.yyyymmdd(ifgList[i].split('-')[0]) + print(m0) + s0 = ut.yyyymmdd(ifgList[i].split('-')[1]) + cmd0 = ['diff_gamma.py',projectName, m0, s0] + diff_file0 = ifgDir + '/' + ifgList[i] + '/' + ifgList[i] + '_' + rlks + 'rlks.diff_filt.bmp' + data0 = [cmd0,err_txt] + + k00 = 0 + if os.path.isfile(diff_file0): + if os.path.getsize(diff_file0) > 0: + k00 = 1 + if k00==0: + data_para.append(data0) + + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + print("Generate differential interferograms for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/phasebias_correction_gamma.py b/third_party/PyINT/pyint/phasebias_correction_gamma.py new file mode 100644 index 0000000..51e824f --- /dev/null +++ b/third_party/PyINT/pyint/phasebias_correction_gamma.py @@ -0,0 +1,545 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Phase Bias Correction for GAMMA format interferograms ### +### Author: ZYD / Cascade AI ### +################################################################# + +import numpy as np +import os +import sys +import time +import argparse +from pathlib import Path +from datetime import datetime +from concurrent.futures import ProcessPoolExecutor, as_completed + +import rasterio +from rasterio.transform import Affine +from rasterio.crs import CRS + +from pyint import _utils as ut + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Apply phase bias correction to interferograms. + This script integrates the phase bias correction algorithm from: + InSAR_PhaseBias_Correction/ directory + + Based on the paper: + "Correcting InSAR phase bias caused by atmospheric delays + and other error sources using multiple aperture interferometry" + (Reference: 10.1016/j.remote.2022.100013) + + Algorithm Steps: + 1. Read interferogram data (PhaseBias_01_Read_Data.py) + 2. Calculate loop closures (PhaseBias_02_Loop_Closures.py) + 3. Estimate calibration parameters an (PhaseBias_03_calibration_pars.py) + 4. Inversion for phase bias terms (PhaseBias_04_Inversion.py) + 5. Apply correction (PhaseBias_05_Correction.py) + +''' + +EXAMPLE = ''' + Usage: + phasebias_correction_gamma.py projectName [options] + phasebias_correction_gamma.py PacayaT163TsxHhA --interval 12 + phasebias_correction_gamma.py PacayaT163TsxHhA --interval 12 --nlook 10 + phasebias_correction_gamma.py PacayaT163TsxHhA --interval 24 --num-a 2 +------------------------------------------------------------------- +''' + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Apply phase bias correction to interferograms.', + formatter_class=argparse.RawTextHelpFormatter, + epilog=INTRODUCTION+'\n'+EXAMPLE) + parser.add_argument('projectName', help='projectName for processing.') + parser.add_argument('--interval', type=int, default=12, + help='Data acquisition interval in days (6, 12, or 24). [default: 12]') + parser.add_argument('--nlook', type=int, default=10, + help='Number of looks for multilooking. [default: 10]') + parser.add_argument('--num-a', type=int, default=2, + help='Number of calibration parameters to estimate. [default: 2]') + parser.add_argument('--start', type=str, default=None, + help='Start date (YYYYMMDD). If not specified, will use all available data.') + parser.add_argument('--end', type=str, default=None, + help='End date (YYYYMMDD). If not specified, will use all available data.') + parser.add_argument('--estimate-an', dest='estimate_an', action='store_true', + help='Estimate calibration parameters from data instead of using defaults.') + parser.add_argument('--max-con', type=int, default=5, + help='Maximum number of connections to correct. [default: 5]') + parser.add_argument('--parallel', type=int, default=8, + help='Number of parallel workers for data conversion. [default: 8]') + parser.add_argument('--skip-convert', action='store_true', + help='Skip data conversion if GeoTIFF already exists.') + + inps = parser.parse_args() + return inps + + +def find_date_range(ifgDir, interval=12): + """Find the start and end dates from interferogram directory + + Args: + ifgDir: interferogram directory + interval: temporal baseline in days + + Returns: + tuple: (start_date, end_date) in YYYYMMDD format + """ + dates = [] + + all_dirs = sorted([d for d in os.listdir(ifgDir) if os.path.isdir(os.path.join(ifgDir, d))]) + + for dirname in all_dirs: + if '-' in dirname: + date_pair = dirname.split('-') + elif '_' in dirname: + date_pair = dirname.split('_') + else: + continue + + if len(date_pair) == 2: + try: + date1 = date_pair[0] + date2 = date_pair[1] + + d1 = datetime.strptime(date1, '%Y%m%d') + d2 = datetime.strptime(date2, '%Y%m%d') + days = abs((d2 - d1).days) + + if days == interval: + dates.extend([date1, date2]) + except: + continue + + if dates: + dates = sorted(set(dates)) + return dates[0], dates[-1] + + return None, None + + +# ===================================================================== +# GAMMA 二进制 → GeoTIFF 转换函数 +# ===================================================================== + +def parse_utm_dem_par(par_file): + """解析 GAMMA utm.dem.par 文件，提取地理编码参数""" + params = {} + with open(par_file) as f: + for line in f: + if ':' in line: + key, val = line.split(':', 1) + key = key.strip() + val = val.strip().split()[0] # 取第一个字段 + params[key] = val + return { + 'width': int(params['width']), + 'nlines': int(params['nlines']), + 'corner_lat': float(params['corner_lat']), + 'corner_lon': float(params['corner_lon']), + 'post_lat': float(params['post_lat']), + 'post_lon': float(params['post_lon']), + } + + +def write_geotiff(data, out_path, geo_info, dtype='float32', nodata=None): + """将 numpy 数组写入带地理信息的 GeoTIFF (rasterio)""" + height, width = data.shape + transform = Affine( + geo_info['post_lon'], 0, geo_info['corner_lon'], + 0, geo_info['post_lat'], geo_info['corner_lat'] + ) + profile = { + 'driver': 'GTiff', + 'dtype': dtype, + 'width': width, + 'height': height, + 'count': 1, + 'crs': CRS.from_epsg(4326), + 'transform': transform, + 'compress': 'lzw', + 'tiled': True, + } + if nodata is not None: + profile['nodata'] = nodata + with rasterio.open(str(out_path), 'w', **profile) as dst: + dst.write(data, 1) + + +def convert_one_pair(args): + """转换单个干涉对: GAMMA 二进制 → GeoTIFF + + 返回: (pair_name, True/False, message) + """ + pair_dir, out_ifg_dir, geo_info, master_date, rlks = args + pair_dir = Path(pair_dir) + pair_name = pair_dir.name # e.g. 20241105-20241117 + d1, d2 = pair_name.split('-') + pair_us = f'{d1}_{d2}' # 下划线分隔 + width = geo_info['width'] + nlines = geo_info['nlines'] + + out_dir = Path(out_ifg_dir) / pair_us + out_dir.mkdir(parents=True, exist_ok=True) + + pha_tif = out_dir / f'{pair_us}.geo.diff_pha.tif' + cc_tif = out_dir / f'{pair_us}.geo.cc.tif' + + # 如果两个文件已存在则跳过 + if pha_tif.exists() and cc_tif.exists(): + return (pair_name, True, '已存在,跳过') + + try: + # --- 缩缩相位: FCOMPLEX → phase --- + diff_filt = pair_dir / f'geo_{pair_name}_{rlks}rlks.diff_filt' + if not diff_filt.exists(): + return (pair_name, False, f'缺少 {diff_filt.name}') + + cpx = np.fromfile(str(diff_filt), dtype=np.complex64).reshape(nlines, width) + phase = np.angle(cpx).astype(np.float32) + phase[cpx == 0] = 0.0 # GAMMA 无数据区域设为 0 + del cpx + write_geotiff(phase, pha_tif, geo_info, dtype='float32', nodata=0) + del phase + + # --- 相干性: FLOAT → 0-255 uint8 --- + cor_file = pair_dir / f'geo_{master_date}_{rlks}rlks.diff_filt.cor' + if not cor_file.exists(): + # 尝试备选命名 + cor_file = pair_dir / f'{pair_name}_{rlks}rlks.diff_filt.cor' + if not cor_file.exists(): + return (pair_name, False, f'缺少 cor 文件') + + cor = np.fromfile(str(cor_file), dtype=np.float32).reshape(nlines, width) + # 清理无效值 + cor[~np.isfinite(cor)] = 0 + cor = np.clip(cor, 0, 1) + cc_uint8 = (cor * 255).astype(np.uint8) + del cor + write_geotiff(cc_uint8, cc_tif, geo_info, dtype='uint8', nodata=0) + del cc_uint8 + + return (pair_name, True, 'OK') + + except Exception as e: + return (pair_name, False, str(e)) + + +def create_landmask(amp_file, geo_info, out_path): + """从地理编码幅度图生成陆地掩膜 GeoTIFF""" + width = geo_info['width'] + nlines = geo_info['nlines'] + amp = np.fromfile(str(amp_file), dtype=np.float32).reshape(nlines, width) + mask = np.zeros_like(amp, dtype=np.uint8) + mask[amp > 0] = 1 + write_geotiff(mask, out_path, geo_info, dtype='uint8', nodata=0) + print(f' 陆地掩膜已生成: {out_path}') + print(f' 有效像素: {int(np.sum(mask))}/{mask.size} ({100*np.sum(mask)/mask.size:.1f}%)') + + +def prepare_gamma_data_for_phasebias(ifgDir, demDir, masterDate, rlks, + staging_dir, n_parallel=8, + skip_existing=True): + """将 GAMMA 格式地理编码干涉图转换为 PhaseBias 脚本期望的 GeoTIFF 格式 + + 目录结构: + staging_dir/ + interferograms/ + YYYYMMDD_YYYYMMDD/ + YYYYMMDD_YYYYMMDD.geo.diff_pha.tif + YYYYMMDD_YYYYMMDD.geo.cc.tif + metadata/ + frame.geo.landmask.tif + """ + staging = Path(staging_dir) + ifg_out = staging / 'interferograms' + meta_out = staging / 'metadata' + ifg_out.mkdir(parents=True, exist_ok=True) + meta_out.mkdir(parents=True, exist_ok=True) + + # 解析地理编码参数 + utm_par = Path(demDir) / f'{masterDate}_{rlks}rlks.utm.dem.par' + if not utm_par.exists(): + raise FileNotFoundError(f'utm.dem.par 不存在: {utm_par}') + geo_info = parse_utm_dem_par(str(utm_par)) + print(f' 地理编码参数: {geo_info["width"]}x{geo_info["nlines"]}, ' + f'corner=({geo_info["corner_lat"]:.4f}, {geo_info["corner_lon"]:.4f})') + + # 生成陆地掩膜 + landmask_tif = meta_out / 'frame.geo.landmask.tif' + if not landmask_tif.exists(): + # 从任意干涉对目录中取地理编码幅度图 + first_pair = sorted(Path(ifgDir).iterdir())[0] + amp_file = first_pair / f'geo_{masterDate}_{rlks}rlks.amp' + if not amp_file.exists(): + print(f' 警告: 未找到幅度图 {amp_file}，将创建全 1 掩膜') + mask = np.ones((geo_info['nlines'], geo_info['width']), dtype=np.uint8) + write_geotiff(mask, landmask_tif, geo_info, dtype='uint8') + print(f' 全 1 掩膜已生成: {landmask_tif}') + else: + create_landmask(amp_file, geo_info, landmask_tif) + else: + print(f' 陆地掩膜已存在: {landmask_tif}') + + # 收集需要转换的干涉对 + pair_dirs = sorted([d for d in Path(ifgDir).iterdir() + if d.is_dir() and '-' in d.name]) + print(f' 发现 {len(pair_dirs)} 个干涉对目录') + + # 并行转换 + tasks = [(str(pd), str(ifg_out), geo_info, masterDate, rlks) for pd in pair_dirs] + + done_ok = 0 + done_skip = 0 + done_fail = 0 + t0 = time.time() + + with ProcessPoolExecutor(max_workers=n_parallel) as pool: + futures = {pool.submit(convert_one_pair, t): t[0] for t in tasks} + for i, fut in enumerate(as_completed(futures), 1): + pair_name, success, msg = fut.result() + if success: + if '跳过' in msg: + done_skip += 1 + else: + done_ok += 1 + else: + done_fail += 1 + print(f' ✗ {pair_name}: {msg}') + + if i % 50 == 0 or i == len(tasks): + elapsed = time.time() - t0 + print(f' 进度: {i}/{len(tasks)} ' + f'(新转换={done_ok}, 跳过={done_skip}, ' + f'失败={done_fail}, 耗时={elapsed:.0f}s)') + + print(f'\n 转换完成: 新转换={done_ok}, 跳过={done_skip}, 失败={done_fail}') + if done_fail > 0: + print(f' 警告: {done_fail} 个干涉对转换失败，将被 PhaseBias 脚本跳过') + + return str(staging) + + +# ===================================================================== +# 配置文件生成 +# ===================================================================== + +def create_config_file(output_path, root_path, interval, nlook, num_a, + start_date, end_date, estimate_an): + """创建 PhaseBias 脚本的配置文件 + + Args: + output_path: 输出目录 + root_path: 包含 interferograms/ 和 metadata/ 子目录的根路径 + """ + config_content = f"""[DEFAULT] +# 包含 interferograms/ 和 metadata/ 子目录的根路径 +root_path = {root_path} + +output_path = {output_path} + +LiCSAR_data=no +frame=NA + +# Start and end dates +start={start_date} +end={end_date} + +# Data acquisition interval: 6-day or 12-day or 24-day +interval={interval} + +# Multilooking factor +nlook={nlook} + +# Number of calibration parameters to estimate +num_a={num_a} + +### an parameters +# Estimate from data or use default values +estimate_an_values={'yes' if estimate_an else 'no'} + +# Using 6-days interval +a1_6_day=0.50 +a2_6_day=0.36 +a3_6_day=0.299 +a4_6_day=0.2476 + +# Using 12-day interval +a1_12_day=0.494 +a2_12_day=0.297 +a3_12_day=0.24 +a4_12_day=0.22 + +# Using 24-day interval (estimated) +a1_24_day=0.48 +a2_24_day=0.28 +a3_24_day=0.22 +a4_24_day=0.20 +""" + config_file = os.path.join(output_path, 'config.txt') + with open(config_file, 'w') as f: + f.write(config_content) + return config_file + + +def run_phasebias_scripts(phasebias_dir, output_path, script_name): + """Run a PhaseBias script + + Args: + phasebias_dir: directory containing PhaseBias scripts + output_path: working directory + script_name: name of the script to run + """ + script_path = os.path.join(phasebias_dir, script_name) + + if not os.path.exists(script_path): + print(f"ERROR: Script not found: {script_path}") + return False + + print(f"\n{'='*80}") + print(f"Running {script_name}...") + print(f"{'='*80}") + + # Run the script + original_dir = os.getcwd() + os.chdir(output_path) + + try: + import subprocess + result = subprocess.run([sys.executable, script_path], + capture_output=True, text=True, check=True) + print(result.stdout) + if result.stderr: + print("STDERR:", result.stderr) + return True + except subprocess.CalledProcessError as e: + print(f"ERROR running {script_name}:") + print(e.stdout) + print(e.stderr) + return False + finally: + os.chdir(original_dir) + + +def main(argv): + + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + interval = inps.interval + nlook = inps.nlook + num_a = inps.num_a + estimate_an = inps.estimate_an + max_con = inps.max_con + n_parallel = inps.parallel + skip_convert = inps.skip_convert + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + + # PyINT 路径 + templateFile = templateDir + "/" + projectName + ".template" + templateDict = ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + + ifgDir = scratchDir + '/' + projectName + "/ifgrams" + demDir = scratchDir + '/' + projectName + "/DEM" + + # PhaseBias 脚本目录 + pyintDir = os.path.dirname(os.path.abspath(__file__)) + phasebiasDir = os.path.join(pyintDir, 'InSAR_PhaseBias_Correction') + + # 工作目录 + workDir = scratchDir + '/' + projectName + "/PhaseBiasCorrection" + os.makedirs(workDir, exist_ok=True) + os.makedirs(os.path.join(workDir, 'Data'), exist_ok=True) + + print("\n" + "="*80) + print(f"Phase Bias Correction for {projectName}") + print(f"Temporal interval: {interval} days") + print(f"Multilooking factor: {nlook}") + print(f"Number of calibration parameters: {num_a}") + print(f"Estimate an from data: {estimate_an}") + print(f"Max connections: {max_con}") + print(f"Parallel workers: {n_parallel}") + print("="*80) + + # 确定日期范围 + if inps.start and inps.end: + start_date = inps.start + end_date = inps.end + else: + print("\n自动检测日期范围...") + start_date, end_date = find_date_range(ifgDir, interval) + if not start_date or not end_date: + print("ERROR: 无法从干涉图目录确定日期范围!") + print("请指定 --start 和 --end 参数.") + sys.exit(1) + + print(f"\n处理日期范围: {start_date} ~ {end_date}") + + # ====== Step 0: GAMMA 二进制 → GeoTIFF 转换 ====== + print("\n" + "="*80) + print("Step 0: GAMMA 二进制 → GeoTIFF 数据转换") + print("="*80) + + staging_dir = os.path.join(workDir, 'GEOC_staging') + staging_root = prepare_gamma_data_for_phasebias( + ifgDir, demDir, masterDate, str(rlks), + staging_dir, n_parallel=n_parallel, + skip_existing=skip_convert + ) + + # ====== 创建配置文件 ====== + print("\n创建配置文件...") + # root_path 指向 staging 目录（包含 interferograms/ 和 metadata/） + config_file = create_config_file(workDir, staging_root, interval, nlook, + num_a, start_date, end_date, estimate_an) + print(f"配置文件: {config_file}") + + # ====== 运行 PhaseBias 流水线 ====== + print("\n" + "="*80) + print("运行 Phase Bias Correction 流水线") + print("="*80) + + # Step 1: 读取数据 + if not run_phasebias_scripts(phasebiasDir, workDir, 'PhaseBias_01_Read_Data.py'): + print("ERROR in Step 1: Read Data") + sys.exit(1) + + # Step 2: 计算闭合环 + if not run_phasebias_scripts(phasebiasDir, workDir, 'PhaseBias_02_Loop_Closures.py'): + print("ERROR in Step 2: Loop Closures") + sys.exit(1) + + # Step 3: 估计标定参数（可选） + if estimate_an: + if not run_phasebias_scripts(phasebiasDir, workDir, 'PhaseBias_03_calibration_pars.py'): + print("ERROR in Step 3: Calibration Parameters") + sys.exit(1) + + # Step 4: 反演 + if not run_phasebias_scripts(phasebiasDir, workDir, 'PhaseBias_04_Inversion.py'): + print("ERROR in Step 4: Inversion") + sys.exit(1) + + # Step 5: 应用校正 + if not run_phasebias_scripts(phasebiasDir, workDir, 'PhaseBias_05_Correction.py'): + print("ERROR in Step 5: Correction") + sys.exit(1) + + print("\n" + "="*80) + print("相位偏差校正完成!") + print(f"输出目录: {workDir}") + print("="*80) + + ut.print_process_time(start_time, time.time()) + + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/phasebias_correction_gamma_all.py b/third_party/PyINT/pyint/phasebias_correction_gamma_all.py new file mode 100644 index 0000000..4d5d2c8 --- /dev/null +++ b/third_party/PyINT/pyint/phasebias_correction_gamma_all.py @@ -0,0 +1,232 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import time +import glob +import argparse +import subprocess +from datetime import datetime + +from pyint import _utils as ut + + +def find_ifgs_by_interval(ifgDir, interval=12): + """Find all interferograms with specified temporal interval + + Args: + ifgDir: interferogram directory + interval: temporal baseline in days (6, 12, or 24) + + Returns: + list: list of date pairs + """ + ifgs = [] + + all_dirs = sorted([d for d in os.listdir(ifgDir) if os.path.isdir(os.path.join(ifgDir, d))]) + + for dirname in all_dirs: + if '-' in dirname: + dates = dirname.split('-') + elif '_' in dirname: + dates = dirname.split('_') + else: + continue + + if len(dates) == 2: + try: + date1 = dates[0] + date2 = dates[1] + + d1 = datetime.strptime(date1, '%Y%m%d') + d2 = datetime.strptime(date2, '%Y%m%d') + days = abs((d2 - d1).days) + + if days == interval: + ifgs.append(dirname) + except ValueError: + continue + + return ifgs + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Apply phase bias correction to all interferograms with + specified temporal interval in a project. + + This script runs the full PhaseBias correction pipeline: + 1. Read interferogram data + 2. Calculate loop closures + 3. Estimate calibration parameters (optional) + 4. Inversion for phase bias terms + 5. Apply correction + +''' + +EXAMPLE = ''' + Usage: + phasebias_correction_gamma_all.py projectName + phasebias_correction_gamma_all.py projectName --interval 12 + phasebias_correction_gamma_all.py projectName --interval 12 --nlook 10 + phasebias_correction_gamma_all.py projectName --interval 24 --estimate-an +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Apply phase bias correction to interferograms.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='projectName for processing.') + parser.add_argument('--interval', type=int, default=12, + help='Data acquisition interval in days (6, 12, or 24). [default: 12]') + parser.add_argument('--nlook', type=int, default=10, + help='Number of looks for multilooking. [default: 10]') + parser.add_argument('--num-a', type=int, default=2, + help='Number of calibration parameters to estimate. [default: 2]') + parser.add_argument('--start', type=str, default=None, + help='Start date (YYYYMMDD). If not specified, will use all available data.') + parser.add_argument('--end', type=str, default=None, + help='End date (YYYYMMDD). If not specified, will use all available data.') + parser.add_argument('--estimate-an', dest='estimate_an', action='store_true', + help='Estimate calibration parameters from data instead of using defaults.') + parser.add_argument('--max-con', type=int, default=5, + help='Maximum number of connections to correct. [default: 5]') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + interval = inps.interval + nlook = inps.nlook + num_a = inps.num_a + estimate_an = inps.estimate_an + max_con = inps.max_con + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + projectDir = scratchDir + '/' + projectName + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + + templateDict = ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + # Check if interferograms with specified interval exist + ifgList = find_ifgs_by_interval(ifgDir, interval) + + print("\n" + "="*80) + print(f"Phase Bias Correction for Project: {projectName}") + print(f"Temporal interval: {interval} days") + print(f"Number of interferograms found: {len(ifgList)}") + print("="*80) + + if len(ifgList) == 0: + print(f"\nERROR: No interferograms found with {interval}-day interval!") + print("Available interferograms:") + all_dirs = sorted([d for d in os.listdir(ifgDir) if os.path.isdir(os.path.join(ifgDir, d))]) + for dirname in all_dirs[:10]: # Show first 10 + if '-' in dirname: + dates = dirname.split('-') + if len(dates) == 2: + try: + d1 = datetime.strptime(dates[0], '%Y%m%d') + d2 = datetime.strptime(dates[1], '%Y%m%d') + days = abs((d2 - d1).days) + print(f" {dirname}: {days} days") + except: + pass + sys.exit(1) + + # Determine date range + if inps.start and inps.end: + start_date = inps.start + end_date = inps.end + else: + # Extract dates from interferogram list + dates = [] + for pair in ifgList: + if '-' in pair: + date_pair = pair.split('-') + dates.extend(date_pair) + elif '_' in pair: + date_pair = pair.split('_') + dates.extend(date_pair) + + dates = sorted(set(dates)) + start_date = dates[0] + end_date = dates[-1] + + print(f"\nDate range: {start_date} to {end_date}") + print(f"Multilooking factor: {nlook}") + print(f"Number of calibration parameters: {num_a}") + print(f"Estimate an from data: {estimate_an}") + print(f"Max connections: {max_con}") + + # Check if correction has already been done + outputDir = projectDir + '/PhaseBiasCorrection' + configFile = outputDir + '/config.txt' + + if os.path.exists(configFile): + print("\n" + "="*80) + print("Found existing PhaseBiasCorrection directory") + print("This script will process ALL interferograms together using the") + print("InSAR_PhaseBias_Correction pipeline.") + print("="*80) + + # Build command + cmd_list = [ + 'phasebias_correction_gamma.py', + projectName, + '--interval', str(interval), + '--nlook', str(nlook), + '--num-a', str(num_a), + '--max-con', str(max_con) + ] + + if start_date: + cmd_list.extend(['--start', start_date]) + if end_date: + cmd_list.extend(['--end', end_date]) + if estimate_an: + cmd_list.append('--estimate-an') + + # Run the phase bias correction script + print("\n" + "="*80) + print("Running phase bias correction...") + print("="*80) + print(f"Command: {' '.join(cmd_list)}") + + try: + result = subprocess.run(cmd_list, check=True, capture_output=False) + print("\n" + "="*80) + print("Phase bias correction completed successfully!") + print(f"Output directory: {outputDir}") + print(f"Corrected interferograms: {outputDir}/GEOC/") + print("="*80) + except subprocess.CalledProcessError as e: + print("\n" + "="*80) + print("ERROR: Phase bias correction failed!") + print("="*80) + sys.exit(1) + + ut.print_process_time(start_time, time.time()) + sys.exit(1) + + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/plot_auto_grd.sh b/third_party/PyINT/pyint/plot_auto_grd.sh new file mode 100644 index 0000000..ac1e8ae --- /dev/null +++ b/third_party/PyINT/pyint/plot_auto_grd.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# 描述：自动根据GRD文件生成地图（依赖GMT 6+） +# 用法：./auto_plot_grd.sh <输入.grd> [输出文件名] [选项] + +# 默认参数 +input_grd=$1 +output="map.png" # 默认输出文件名 +projection="M15c" # 默认投影（Mercator，宽度15厘米） +colormap="viridis" # 默认色标 +title="Topography" # 默认标题 +annotate="a" # 默认标注间隔（自动） + +# 解析命令行参数 +while [[ $# -gt 0 ]]; do + case $1 in + -o|--output) output="$2"; shift ;; + -P|--projection) projection="$2"; shift ;; + -C|--colormap) colormap="$2"; shift ;; + -T|--title) title="$2"; shift ;; + -A|--annotate) annotate="$2"; shift ;; + *) ;; + esac + shift +done + +# 检查输入文件是否存在 +if [[ ! -f $input_grd ]]; then + echo "错误：输入文件 $input_grd 不存在！" + exit 1 +fi + +# 从GRD文件获取地理范围 +region=$(gmt grdinfo $input_grd -I- | awk -F'R' '{print $2}') + +# 生成临时CPT色标文件 +gmt makecpt -Cturbo -T$(gmt grdinfo $input_grd -T | awk -F'T' '{print $2}') > tmp.cpt + +# 开始绘图 +gmt begin ${output%.*} pdf,png # 去除扩展名 + # 绘制GRD数据 + gmt grdimage $input_grd -R$region -J$projection -Ctmp.cpt -B$annotate -BWSne+t"$title" + # 添加海岸线 + gmt coast -R$region -J$projection -W0.5p,black -Df + # 添加色标 + gmt colorbar -DJBC+w10c/0.5c+o0/1c -Bxaf -By+l"Elevation (m)" +gmt end show + +# 清理临时文件 +#rm -f tmp.cpt + +echo "地图已生成：$output" diff --git a/third_party/PyINT/pyint/plot_geotiff.py b/third_party/PyINT/pyint/plot_geotiff.py new file mode 100644 index 0000000..81fa575 --- /dev/null +++ b/third_party/PyINT/pyint/plot_geotiff.py @@ -0,0 +1,229 @@ +#! /usr/bin/env python +import argparse +import os +import numpy as np +import matplotlib.pyplot as plt +import rasterio +from rasterio.plot import show +import matplotlib.colors as colors +from matplotlib import cm +import warnings +warnings.filterwarnings('ignore') + +def parse_arguments(): + """解析命令行参数""" + parser = argparse.ArgumentParser(description='绘制GeoTIFF文件') + + # 必需参数 + parser.add_argument('-i', '--input', required=True, + help='输入GeoTIFF文件路径') + + # 输出参数 + parser.add_argument('-o', '--output', + help='输出图片文件路径 (默认: 输入文件名.png)') + + # 绘图参数 + parser.add_argument('--dpi', type=int, default=300, + help='输出图片分辨率 (默认: 300)') + parser.add_argument('--cmap', default='viridis', + help='色彩映射 (默认: viridis)') + parser.add_argument('--title', + help='图片标题 (默认: 使用文件名)') + parser.add_argument('--band', type=int, default=1, + help='要绘制的波段 (默认: 1)') + + # 显示参数 + parser.add_argument('--vmin', type=float, + help='颜色范围最小值') + parser.add_argument('--vmax', type=float, + help='颜色范围最大值') + parser.add_argument('--log', action='store_true', + help='使用对数颜色标尺') + parser.add_argument('--percentile', type=float, nargs=2, + help='使用百分位数设置颜色范围，例如 --percentile 2 98') + + # 图片尺寸 + parser.add_argument('--width', type=float, default=10, + help='图片宽度 (英寸, 默认: 10)') + parser.add_argument('--height', type=float, default=8, + help='图片高度 (英寸, 默认: 8)') + + # 其他选项 + parser.add_argument('--no-colorbar', action='store_true', + help='不显示颜色条') + parser.add_argument('--show', action='store_true', + help='显示图片 (默认只保存)') + + return parser.parse_args() + +def get_default_output_filename(input_file): + """根据输入文件名生成默认输出文件名""" + dir_name = os.path.dirname(input_file) + base_name = os.path.basename(input_file) + name, ext = os.path.splitext(base_name) + return os.path.join(dir_name, f"{name}.png") + +def read_geotiff(file_path, band=1): + """读取GeoTIFF文件""" + if not os.path.exists(file_path): + raise FileNotFoundError(f"文件不存在: {file_path}") + + with rasterio.open(file_path) as src: + # 检查波段数量 + if band > src.count: + raise ValueError(f"文件只有 {src.count} 个波段，无法读取波段 {band}") + + data = src.read(band) + profile = src.profile + bounds = src.bounds + crs = src.crs + nodata = src.nodata + + # 如果有无效值，创建掩码 + if nodata is not None: + data = np.ma.masked_where(data == nodata, data) + + return data, profile, bounds, crs, nodata + +def plot_geotiff(args): + """主绘图函数""" + # 设置默认输出文件名 + if args.output is None: + args.output = get_default_output_filename(args.input) + + # 设置默认标题 + if args.title is None: + args.title = os.path.basename(args.input) + + print(f"读取文件: {args.input}") + print(f"输出文件: {args.output}") + print(f"使用波段: {args.band}") + + # 读取数据 + data, profile, bounds, crs, nodata = read_geotiff(args.input, args.band) + + print(f"数据形状: {data.shape}") + print(f"数据类型: {data.dtype}") + print(f"坐标参考系统: {crs}") + print(f"数据范围: {bounds}") + + if nodata is not None: + print(f"无效值: {nodata}") + valid_data = data[~data.mask] if hasattr(data, 'mask') else data[data != nodata] + else: + valid_data = data.compressed() if hasattr(data, 'mask') else data.flatten() + + print(f"有效数据点数: {len(valid_data)}") + print(f"数据范围: {valid_data.min():.6f} - {valid_data.max():.6f}") + + # 设置颜色范围 + if args.percentile: + vmin = np.percentile(valid_data, args.percentile[0]) + vmax = np.percentile(valid_data, args.percentile[1]) + print(f"使用百分位数范围: {args.percentile[0]}% - {args.percentile[1]}%") + print(f"对应数值范围: {vmin:.6f} - {vmax:.6f}") + elif args.vmin is not None and args.vmax is not None: + vmin, vmax = args.vmin, args.vmax + print(f"使用指定范围: {vmin} - {vmax}") + else: + vmin, vmax = valid_data.min(), valid_data.max() + print(f"使用数据范围: {vmin:.6f} - {vmax:.6f}") + + # 创建图形 + fig, ax = plt.subplots(1, 1, figsize=(args.width, args.height)) + + # 选择颜色映射 + try: + cmap = plt.get_cmap(args.cmap) + except: + print(f"警告: 色彩映射 '{args.cmap}' 不存在，使用默认的 'viridis'") + cmap = plt.get_cmap('viridis') + + # 选择颜色标准化方式 + if args.log: + norm = colors.LogNorm(vmin=vmin, vmax=vmax) + print("使用对数颜色标尺") + # 对于对数标准化，不能同时传递vmin/vmax参数 + show_kwargs = {'norm': norm, 'cmap': cmap} + else: + # 对于线性标准化，可以直接传递vmin/vmax参数 + show_kwargs = {'vmin': vmin, 'vmax': vmax, 'cmap': cmap} + + # 使用rasterio的show函数显示地理参考数据 + if crs is not None: + with rasterio.open(args.input) as src: + # 使用rasterio的show函数，它能够正确处理地理参考 + # 注意：show()返回的是Axes对象，不是mappable对象 + image_axes = show( + (src, args.band), + ax=ax, + **show_kwargs + ) + + # 从axes对象中获取图像对象 + if hasattr(image_axes, 'images') and len(image_axes.images) > 0: + im = image_axes.images[0] + else: + # 如果无法获取图像对象，创建一个ScalarMappable用于颜色条 + im = cm.ScalarMappable(norm=norm if args.log else colors.Normalize(vmin=vmin, vmax=vmax), + cmap=cmap) + im.set_array([]) # 设置一个空数组 + else: + # 如果没有地理参考，使用普通的imshow + if args.log: + im = ax.imshow(data, norm=norm, cmap=cmap) + else: + im = ax.imshow(data, vmin=vmin, vmax=vmax, cmap=cmap) + + # 设置标题和标签 + ax.set_title(args.title, fontsize=14, fontweight='bold') + + # 添加颜色条 + if not args.no_colorbar: + # 确保im是一个mappable对象 + if not hasattr(im, 'set_array'): + # 如果不是mappable对象，创建一个 + im = cm.ScalarMappable(norm=norm if args.log else colors.Normalize(vmin=vmin, vmax=vmax), + cmap=cmap) + im.set_array([]) # 设置一个空数组 + + cbar = plt.colorbar(im, ax=ax, shrink=0.8) + cbar.set_label('值', rotation=270, labelpad=15) + + # 添加网格 + ax.grid(True, alpha=0.3) + + # 设置坐标轴标签 + if crs is not None: + ax.set_xlabel('经度') + ax.set_ylabel('纬度') + else: + ax.set_xlabel('X') + ax.set_ylabel('Y') + + # 保存图片 + plt.tight_layout() + plt.savefig(args.output, dpi=args.dpi, bbox_inches='tight') + print(f"图片已保存: {args.output}") + + # 显示图片 + if args.show: + plt.show() + + plt.close() + + return args.output + +def main(): + """主函数""" + args = parse_arguments() + + try: + output_file = plot_geotiff(args) + print(f"成功生成图片: {output_file}") + except Exception as e: + print(f"处理过程中发生错误: {e}") + raise + +if __name__ == "__main__": + main() diff --git a/third_party/PyINT/pyint/pot_gamma_subset.py b/third_party/PyINT/pyint/pot_gamma_subset.py new file mode 100644 index 0000000..100c3f7 --- /dev/null +++ b/third_party/PyINT/pyint/pot_gamma_subset.py @@ -0,0 +1,191 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2022, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +from pyint import _utils as ut + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Pixel offset tracking for co-registered Images. TOPs should be deramped in advance. + +''' + +EXAMPLE = ''' + Usage: + pot_gamma_subset.py projectName Mdate Sdate subset + pot_gamma_subset.py PacayaT163TsxHhA 20150102 20150601 0102 +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Pixel offset tracking based Azimuth/Range dispalcement estimation.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('Mdate',help='Master date.') + parser.add_argument('Sdate',help='Slave date.') + parser.add_argument('Subset',help='Subset name, e.g., 0102') + + + inps = parser.parse_args() + return inps + + +def main(argv): + + start_time = time.time() + inps = cmdLineParse() + Mdate = inps.Mdate + Sdate = inps.Sdate + subset = inps.Subset + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + + if 'POT_rwin' in templateDict: + POT_rwin = templateDict['POT_rwin'] + else: + POT_rwin = '160' + print('POT range window sise: ' + POT_rwin) + + if 'POT_awin' in templateDict: + POT_awin = templateDict['POT_awin'] + else: + POT_awin = '160' + print('POT azimuth window sise: ' + POT_awin) + + if 'POT_astep' in templateDict: + POT_astep = templateDict['POT_astep'] + else: + POT_astep = azlks + + if 'POT_rstep' in templateDict: + POT_rstep = templateDict['POT_rstep'] + else: + POT_rstep = rlks + + print('POT range steps: ' + POT_rstep) + print('POT azimuth steps: ' + POT_astep) + + projectDir = scratchDir + '/' + projectName + demDir = scratchDir + '/' + projectName + '/DEM' + + slcDir = scratchDir + '/' + projectName + '/SLC' + rslcDir = scratchDir + '/' + projectName + '/RSLC' + ifgDir = projectDir + '/ifgrams' + if not os.path.isdir(ifgDir): os.mkdir(ifgDir) + + Pair = Mdate + '-' + Sdate + workDir = ifgDir + '/' + Pair + if not os.path.isdir(workDir): os.mkdir(workDir) + + ####################################################################### + Mamp = rslcDir + '/' + Mdate + '/' + Mdate + '_' + subset + '_' + rlks + 'rlks.amp' + MampPar = rslcDir + '/' + Mdate + '/' + Mdate + '_' + subset + '_' + rlks + 'rlks.amp.par' + Samp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + subset + '_' + rlks + 'rlks.amp' + SampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + subset + '_' + rlks + 'rlks.amp.par' + + Mrslc = rslcDir + '/' + Mdate + '/' + Mdate + '_' + subset + '.rslc' + MrslcPar = rslcDir + '/' + Mdate + '/' + Mdate + '_' + subset + '.rslc.par' + Srslc = rslcDir + '/' + Sdate + '/' + Sdate + '_' + subset + '.rslc' + SrslcPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + subset + '.rslc.par' + + #HGT = demDir + '/' + masterDate + '_' + rlks + 'rlks.rdc.dem' + #MasterPar = rslcDir + '/' + masterDate + '/' + masterDate + '.rslc.par' + + ################# copy file for parallel processing ########################## + #Mamp = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp' + #MampPar = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + #Samp = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp' + #SampPar = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + #if not templateDict['diff_all_parallel'] == '1': + + # Mrslc = workDir + '/' + Mdate + '.rslc' + # MrslcPar = workDir + '/' + Mdate + '.rslc.par' + # Srslc = workDir + '/' + Sdate + '.rslc' + # SrslcPar = workDir + '/' + Sdate + '.rslc.par' + # ut.copy_file(Mrslc0,Mrslc) + # ut.copy_file(MrslcPar0,MrslcPar) + # ut.copy_file(Srslc0,Srslc) + # ut.copy_file(SrslcPar0,SrslcPar) + + #else: + + # Mrslc = Mrslc0 + # MrslcPar = MrslcPar0 + # Srslc = Srslc0 + # SrslcPar = SrslcPar0 + # HGT = HGT0 + # MasterPar = MasterPar0 + + #ut.copy_file(Mamp0,Mamp) + #ut.copy_file(MampPar0,MampPar) + #ut.copy_file(Samp0,Samp) + #ut.copy_file(SampPar0,SampPar) + + #ut.copy_file(HGT0,HGT) + #ut.copy_file(MasterPar0,MasterPar) + + ############################################################################ + + OFF = workDir + '/' + Pair + '_' + subset + '_' + rlks + 'rlks_pot.off' + OFFS = workDir + '/' + Pair + '_' + subset + '_' + rlks + 'rlks_pot.offs' + CCP = workDir + '/' + Pair + '_' + subset + '_' + rlks + 'rlks_pot.ccp' + COFFS = workDir + '/' + Pair + '_' + subset + '_' + rlks + 'rlks_pot.coffs' + COFFS_FILT = workDir + '/' + Pair + '_' + subset + '_' + rlks + 'rlks_pot_filt.coffs' + POTA = workDir + '/' + Pair + '_' + subset + '_' + rlks + 'rlks_pot.az' + POTR = workDir + '/' + Pair + '_' + subset + '_' + rlks + 'rlks_pot.rg' + + call_str = 'create_offset '+ MrslcPar + ' ' + SrslcPar + ' ' + OFF + ' 1 ' + rlks + ' ' + azlks + ' 0' + os.system(call_str) + + call_str = 'offset_pwr_tracking ' + Mrslc + ' ' + Srslc + ' ' + MrslcPar + ' ' + SrslcPar + ' ' + OFF + ' ' + OFFS + ' ' + CCP + ' ' + POT_rwin + ' ' + POT_awin + ' - 2 0.05 ' + POT_rstep + ' ' + POT_astep + ' - - - - - - ' + os.system(call_str) + + call_str = 'offset_tracking ' + OFFS + ' ' + CCP + ' ' + MrslcPar + ' ' + OFF + ' ' + COFFS + ' - 1 - 1' + os.system(call_str) + + nWidth = ut.read_gamma_par(OFF, 'read', 'interferogram_width') + + #call_str = 'adf ' + COFFS + ' ' + COFFS_FILT + ' ' + CCP + ' ' + nWidth + #os.system(call_str) + + call_str = 'cpx_to_real ' + COFFS + ' ' + POTR + ' ' + nWidth + ' 0 ' + os.system(call_str) + + call_str = 'cpx_to_real ' + COFFS + ' ' + POTA + ' ' + nWidth + ' 1 ' + os.system(call_str) + + call_str = 'rasdt_pwr ' + POTR + ' ' + Mamp + ' ' + nWidth + ' - - - - - - - BuYlRd.cm' + os.system(call_str) + + call_str = 'rasdt_pwr ' + POTA + ' ' + Mamp + ' ' + nWidth + ' - - - - - - - BuYlRd.cm' + os.system(call_str) + + print("Pixel offset tracking is Done!") + ut.print_process_time(start_time, time.time()) + #sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/pot_gamma_subset_combine.py b/third_party/PyINT/pyint/pot_gamma_subset_combine.py new file mode 100644 index 0000000..026b510 --- /dev/null +++ b/third_party/PyINT/pyint/pot_gamma_subset_combine.py @@ -0,0 +1,372 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import time +import numpy as np +import os +import sys +import argparse +import h5py + +from pyint import _utils as ut +from scipy.interpolate import griddata +from scipy.interpolate import NearestNDInterpolator + +#def resamp2d_near(row0,col0,z0,row1,col1): + +# xx0, yy0 = np.meshgrid((np.arange(col0) + 1), (np.arange(row0) + 1)) +# xx1, yy1 = np.meshgrid((np.arange(col1) + 1), (np.arange(row1) + 1)) + +# xx00 = xx0/col0*col1; yy00 = yy0/row0*row1 +# xx = xx00.flatten(); yy = yy00.flatten(); zz = z0.flatten() +# interp = NearestNDInterpolator(list(zip(xx, yy)), zz) +# z1 = interp(xx1, yy1) + +# return z1 + +def generate_name(i,j): + + if len(str(int(i)))==1: + i0 = '0' + str(int(i)) + else: + i0 = str(int(i)) + + if len(str(int(j)))==1: + j0 = '0' + str(int(j)) + else: + j0 = str(int(j)) + + name0 = i0+j0 + return name0 + +def reduce_samp(xx,yy,zz,xg,yg,extend): + + max_x = np.max(xg)+extend; min_x = np.min(xg)-extend # e.g. extend 100 + max_y = np.max(yg)+extend; min_y = np.min(yg)-extend # e.g., extend 100 + + xx1 = xx[((min_x<xx) & (xx<max_x) & (min_y<yy) & (yy<max_y))] + yy1 = yy[((min_x<xx) & (xx<max_x) & (min_y<yy) & (yy<max_y))] + zz1 = zz[((min_x<xx) & (xx<max_x) & (min_y<yy) & (yy<max_y))] + xx1 = np.reshape(xx1,(len(xx1),)) + yy1 = np.reshape(yy1,(len(yy1),)) + zz1 = np.reshape(zz1,(len(zz1),)) + + return xx1,yy1,zz1 + +def interp_split(xx,yy,zz,xg,yg,split_numb): + row0 = len(yg); col0 =len(xg); data_total = np.zeros((row0,col0),dtype='float32') + drow = int(row0/split_numb); dcol = int(col0/split_numb) + + for ki in range(split_numb): + if ki==(split_numb-1): + rr = np.arange(ki*drow,row0) + else: + rr = np.arange(ki*drow,(ki+1)*drow) + for kj in range(split_numb): + if kj == (split_numb-1): + cc = np.arange(kj*dcol,col0) + else: + cc = np.arange(kj*dcol,(ki+1)*dcol) + xx1,yy1,zz1 = reduce_samp(xx,yy,zz,xg,yg,100) + xgg1,ygg1 = np.meshgrid(xg,yg) + xgg2 = xgg1[rr[0]:(rr[len(rr)-1]+1),cc[0]:(cc[len(cc)-1]+1)] + ygg2 = ygg1[rr[0]:(rr[len(rr)-1]+1),cc[0]:(cc[len(cc)-1]+1)] + #points0 = np.zeros((len(xx1),2),dtype='float32'); points0[:,0]=xx1; points0[:,1] = yy1 + #points0=(xx1,yy1);print(points0.shape) + #print(xx1.shape);print(yy1.shape);print(zz1.shape);print(xgg1.shape);print(xgg2.shape); + data0 = griddata((xx1,yy1),zz1,(xgg2,ygg2),method='linear') + #data0 = griddata(points0,zz1,(xgg2,ygg2),method='linear') + print(data0.shape) + data_total[rr[0]:(rr[len(rr)-1]+1),cc[0]:(cc[len(cc)-1]+1)] = data0 + + return data_total + +def get_startSamp(nLine, nWidth, awidth, rwidth): + Ap_samp = np.arange(1,int(nLine),int(awidth)); Na0 = len(Ap_samp); LA_end = int(nLine) - Ap_samp[Na0-1] + Rp_samp = np.arange(1,int(nWidth),int(rwidth)); Nr0 = len(Rp_samp); LR_end = int(nWidth) - Rp_samp[Nr0-1] + + if LA_end > (2/3*int(awidth)): + Ap_samp1 = Ap_samp + else: + Ap_samp1 = Ap_samp[0:(Na0-1)] # last batch with '-' means to the end + + if LR_end > (2/3*int(rwidth)): + Rp_samp1 = Rp_samp + else: + Rp_samp1 = Rp_samp[0:(Nr0-1)] # last batch with '-' means to the end + return Ap_samp1, Rp_samp1 + +def subset2coord(subset, astep, rstep, nLine, nWidth, awidth, rwidth, extend): + # default awidth = 5000 rwidth = 5000 extend = 200 + # astep: azimuth multilook numbers + # rstep: range multilook numbers + extend = int(extend) + Ap_samp1, Rp_samp1 = get_startSamp(nLine, nWidth, awidth, rwidth) + Na = len(Ap_samp1); Nr = len(Rp_samp1) + ii = int(subset[0:2]); jj = int(subset[2:4]) + #print(ii); print(jj) + rstart = str(Rp_samp1[jj]); astart = str(Ap_samp1[ii]); + #print(nLine);print(Ap_samp1);print(Rp_samp1) + + if not ii==0: + astart0 = str(int(int(astart) - extend)) # extend 200 to avoid edge effect + else: + astart0 = astart + + if not jj==0: + rstart0 = str(int(int(rstart) - extend)) # extend 200 to avoid edge effect + else: + rstart0 = rstart + + #print('astart') + #print(astart);print(Ap_samp1[Na-1]) + if astart == str(Ap_samp1[Na-1]): + awidth0 = '-' + aend0 = str(nLine) + else: + awidth0 = str(int(int(awidth) + extend)) # extend 200 to avoid edge effect + aend0 = str(int(int(astart0) + int(awidth) + extend - 1)) + + if rstart == str(Rp_samp1[Nr-1]): + rwidth0 = '-' + rend0 = str(nWidth) + else: + rwidth0 = str(int(int(rwidth) + extend)) # extend 200 to avoid edge effect + rend0 = str(int(int(rstart0) + int(rwidth) + extend - 1)) + + xx0 = np.arange(int(rstart0), int(rend0), int(rstep)); Nx0 = len(xx0); xx1 = xx0[0:Nx0-1] + yy0 = np.arange(int(astart0), int(aend0), int(astep)); Ny0 = len(yy0); yy1 = yy0[1:Ny0] + #print(rstart0); print(rend0); print(astart0); print(aend0) + + rwidth_total = int(int(rend0) - int(rstart0) + 1) + awidth_total = int(int(aend0) - int(astart0) + 1) + if np.mod(rwidth_total,int(rstep))==0: + xx1 = xx0 + else: + xx1 = xx0[0:Nx0-1] + + if np.mod(awidth_total,int(astep))==0: + yy1 = yy0 + else: + yy1 =yy0[0:Ny0-1] + + return xx1,yy1 + +def read_gammadata(file0,nWidth0,nLength0): + data0 = np.fromfile(file0,dtype='>f4',count=int(nLength0)*int(nWidth0)).reshape(int(nLength0), int(nWidth0)) + return data0 + +def write_h5(datasetDict, out_file, metadata=None, ref_file=None, compression=None): + + if os.path.isfile(out_file): + print('delete exsited file: {}'.format(out_file)) + os.remove(out_file) + + print('create HDF5 file: {} with w mode'.format(out_file)) + dt = h5py.special_dtype(vlen=np.dtype('float64')) + + + with h5py.File(out_file, 'w') as f: + for dsName in datasetDict.keys(): + data = datasetDict[dsName] + ds = f.create_dataset(dsName, + data=data, + compression=compression) + + for key, value in metadata.items(): + f.attrs[key] = str(value) + #print(key + ': ' + value) + print('finished writing to {}'.format(out_file)) + + return out_file + +INTRODUCTION = ''' +------------------------------------------------------------------- + Combine subset of POT results from pot_gamma_subset_jobs.py +''' + +EXAMPLE = ''' + Usage: + pot_gamma_subset_combine.py projectName Mdate Sdate + pot_gamma_subset_combine.py PacayaT163TsxHhA 20150601 20150613 +------------------------------------------------------------------- +''' + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Combine subset of POT results from pot_gamma_subset_jobs.py',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('Mdate',help='First date.') + parser.add_argument('Sdate',help='Second date.') + #parser.add_argument('--out',dest ='out', help='Output file name.') + parser.add_argument('--rwidth',dest ='rwidth', default = '5000', help='Patch range size.') + parser.add_argument('--awidth',dest ='awidth', default = '5000', help='Patch azimuth size.') + parser.add_argument('--extend',dest ='extend', default = '200', help='Patch extend to ensure overlap regions.') + + inps = parser.parse_args() + return inps + + +def main(argv): + + inps = cmdLineParse() + start_time = time.time() + projectName = inps.projectName + rwidth = inps.rwidth; awidth = inps.awidth; extend = inps.extend + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + processDir = scratchDir + '/' + projectName + "/PROCESS" + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + '/RSLC' + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + Mdate = inps.Mdate; Sdate = inps.Sdate + Pair = Mdate + '-' + Sdate + workDir = ifgDir + '/' + Pair + MslcPar = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc.par' + + Samp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + SWidth = ut.read_gamma_par(SampPar, 'read', 'range_samples') + SLine = ut.read_gamma_par(SampPar, 'read', 'azimuth_lines') + SampData = read_gammadata(Samp,SWidth,SLine) + + nWidth = ut.read_gamma_par(MslcPar, 'read', 'range_samples'); Rp_samp_large0 = np.arange(1,int(nWidth),int(rlks)) + nLine = ut.read_gamma_par(MslcPar, 'read', 'azimuth_lines'); Ap_samp_large0 = np.arange(1,int(nLine),int(azlks)) + Nr0 = len(Rp_samp_large0); Na0 = len(Ap_samp_large0) + + if np.mod(int(nWidth),int(rlks))==0: + Rp_samp_large = Rp_samp_large0 + else: + Rp_samp_large = Rp_samp_large0[0:Nr0-1] + + if np.mod(int(nLine),int(azlks))==0: + Ap_samp_large= Ap_samp_large0 + else: + Ap_samp_large =Ap_samp_large0[0:Na0-1] + + rr,aa = np.meshgrid(Rp_samp_large,Ap_samp_large) + + Nr = len(Rp_samp_large); Na = len(Ap_samp_large) + print('Total samples along range and azimuth: ' + str(Nr) + ' ' + str(Na)) + + Ap_samp1,Rp_samp1 = get_startSamp(nLine, nWidth, awidth, rwidth) + Na = len(Ap_samp1); Nr = len(Rp_samp1) + + xx_total = []; yy_total = []; az_total = []; rg_total = []; cc_total = []; + for i in range(Na): + for j in range(Nr): + subset0 = generate_name(i,j) + xx0,yy0 = subset2coord(subset0, azlks, rlks, nLine, nWidth, awidth, rwidth, extend) + #print(len(xx0)); print(len(yy0)) + [xx0,yy0] = np.meshgrid(xx0,yy0) + az0 = workDir + '/' + Pair + '_' + subset0 + '_' + rlks + 'rlks_pot.az' + rg0 = workDir + '/' + Pair + '_' + subset0 + '_' + rlks + 'rlks_pot.rg' + cc0 = workDir + '/' + Pair + '_' + subset0 + '_' + rlks + 'rlks_pot.ccp' + off0 = workDir + '/' + Pair + '_' + subset0 + '_' + rlks + 'rlks_pot.off' + nLine0 = ut.read_gamma_par(off0, 'read', 'interferogram_azimuth_lines') + nWidth0 = ut.read_gamma_par(off0, 'read', 'interferogram_width') + #print(nWidth0);print(nLine0) + if os.path.isfile(az0): + if os.path.getsize(az0)>0: + az_data0 = read_gammadata(az0,nWidth0,nLine0) + rg_data0 = read_gammadata(rg0,nWidth0,nLine0) + cc_data0 = read_gammadata(cc0,nWidth0,nLine0) + + xx_total.extend(list(xx0.flatten()));yy_total.extend(list(yy0.flatten())) + az_total.extend(list(az_data0.flatten())) + rg_total.extend(list(rg_data0.flatten())) + cc_total.extend(list(cc_data0.flatten())) + else: + data0 = np.zeros((int(nWidth0),int(nLine0)),dtype='float32') + xx_total.extend(list(data0.flatten()));yy_total.extend(list(data0.flatten())) + az_total.extend(list(data0.flatten())) + rg_total.extend(list(data0.flatten())) + cc_total.extend(list(data0.flatten())) + + xx_total = np.asarray(xx_total); #print(xx_total.shape) + yy_total = np.asarray(yy_total); #print(yy_total.shape) + az_total = np.asarray(az_total); #print(az_total.shape) + rg_total = np.asarray(rg_total); #print(rg_total.shape) + cc_total = np.asarray(cc_total); #print(cc_total.shape) + + xxa = xx_total[az_total!=0]; yya = yy_total[az_total!=0]; az_total1 = az_total[az_total!=0] + xxr = xx_total[rg_total!=0]; yyr = yy_total[rg_total!=0]; rg_total1 = rg_total[rg_total!=0] + xxc = xx_total[cc_total!=0]; yyc = yy_total[cc_total!=0]; cc_total1 = cc_total[cc_total!=0] + + #start_time = time.time() + #az_grid_large = interp_split(xxa,yya,az_total1,Rp_samp_large,Ap_samp_large,2); print('1 finish') + #rg_grid_large = interp_split(xx_total,yy_total,rg_total1,Rp_samp_large,Ap_samp_large,2) + #cc_grid_large = interp_split(xx_total,yy_total,cc_total1,Rp_samp_large,Ap_samp_large,2) + + #start_time = time.time() + #interp_az = NearestNDInterpolator(list(zip(xxa, yya)), az_total1) + #az_grid_large = interp_az(rr, aa) + #interp_rg = NearestNDInterpolator(list(zip(xxr, yyr)), rg_total1) + #rg_grid_large = interp_rg(rr, aa) + #interp_cc = NearestNDInterpolator(list(zip(xxc, yyc)), cc_total1) + #cc_grid_large = interp_cc(rr, aa) + #ut.print_process_time(start_time, time.time()) + + method0 = 'nearest' # keep 0 values to make good mask + print('Start to combine all of the sub-patches ...') + start_time = time.time() + az_grid_large = griddata((xxa,yya),az_total1,(rr,aa),method=method0); print('Azimuth interpolate finish') + #ut.print_process_time(start_time, time.time()) + #start_time = time.time() + rg_grid_large = griddata((xxr,yyr),rg_total1,(rr,aa),method=method0); print('Range interpolate finish') + #ut.print_process_time(start_time, time.time()) + #start_time = time.time() + cc_grid_large = griddata((xxc,yyc),cc_total1,(rr,aa),method=method0); print('CCP interpolate finish') + ut.print_process_time(start_time, time.time()) + + az_grid_large[SampData==0]=0 + rg_grid_large[SampData==0]=0 + cc_grid_large[SampData==0]=0 + + row1,col1 = rr.shape + meta = dict() + meta['WIDTH'] = str(col1); meta['LENGTH'] = str(row1); meta['UNIT'] = 'm' + meta['FILE_TYPE'] ='offset_tracking'; meta['DATE12'] = Pair + + datasetDict = dict() + datasetDict['azimuth'] = az_grid_large + #datasetDict['range'] = rg_grid_large + #datasetDict['ccp'] = cc_grid_large + #if inps.out: + # out_file = inps.out + #else: + out_file = Pair + '_pot_az.h5' + write_h5(datasetDict, out_file, metadata=meta, ref_file=None, compression=None) + + datasetDict = dict() + #datasetDict['azimuth'] = az_grid_large + datasetDict['range'] = rg_grid_large + #datasetDict['ccp'] = cc_grid_large + out_file = Pair + '_pot_rg.h5' + write_h5(datasetDict, out_file, metadata=meta, ref_file=None, compression=None) + + + datasetDict = dict() + #datasetDict['azimuth'] = az_grid_large + #datasetDict['range'] = rg_grid_large + datasetDict['coherence'] = cc_grid_large + #meta['FILE_TYPE'] = 'coherence' + out_file = Pair + '_pot_cc.h5' + write_h5(datasetDict, out_file, metadata=meta, ref_file=None, compression=None) + print("POT calculation done: " + Pair) + #sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/pot_gamma_subset_jobs.py b/third_party/PyINT/pyint/pot_gamma_subset_jobs.py new file mode 100644 index 0000000..f6bd3c7 --- /dev/null +++ b/third_party/PyINT/pyint/pot_gamma_subset_jobs.py @@ -0,0 +1,95 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2022, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +from pyint import _utils as ut + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Pixel offset tracking for co-registered Images. TOPs should be deramped in advance. + +''' + +EXAMPLE = ''' + Usage: + pot_gamma_subset_jobs.py projectName Mdate Sdate --memory memory_single_job --walltime walltime_single_job + pot_gamma_subset_jobs.py PacayaT163TsxHhA 20150102 20150601 --memory 5000 --walltime 00:30:00 +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Pixel offset tracking based Azimuth/Range dispalcement estimation.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('Mdate',help='Master date.') + parser.add_argument('Sdate',help='Slave date.') + parser.add_argument('--memory',dest = 'memory',default = '5000', help='memory to be allocated for one job, default: 5GB') + parser.add_argument('--walltime',dest = 'walltime',default = '00:30:00', help='walltime to be allocated for one single job') + inps = parser.parse_args() + inps = parser.parse_args() + return inps + + +def main(argv): + + start_time = time.time() + inps = cmdLineParse() + Mdate = inps.Mdate + Sdate = inps.Sdate + + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + + + slcDir = scratchDir + '/' + projectName + '/SLC' + rslcDir = scratchDir + '/' + projectName + '/RSLC' + + MrslcDir = rslcDir + '/' + Mdate + SrslcDir = rslcDir + '/' + Sdate + Pair = Mdate + '-' + Sdate + batch_txt = scratchDir + '/' + projectName + '/ifgrams/' + Pair + '/Batch_POT' + workDir = scratchDir + '/' + projectName + '/ifgrams/' + Pair + if not os.path.isdir(workDir): + os.mkdir(workDir) + os.chdir(workDir) + if os.path.isfile(batch_txt): + os.remove(batch_txt) + + mrslc_list = glob.glob(MrslcDir + '/*_0*.rslc');mrslc_list.sort(); nn = len(mrslc_list) + print('Total patch numbers: ' + str(nn)) + + for i in range(nn): + base0 = os.path.basename(mrslc_list[i]); subset1 = base0.split('_')[1]; subset0 = subset1.split('.rslc')[0] + str0 = 'pot_gamma_subset.py ' + projectName + ' ' + Mdate + ' ' + Sdate + ' ' + subset0 + call_str = 'echo ' + str0 + ' >>' + batch_txt + os.system(call_str) + + call_str = 'sbatch_jobs.py ' + batch_txt + ' --memory ' + inps.memory + ' --walltime ' + inps.walltime + ' --job-name ' + Pair + '_POT' + os.system(call_str) + + + print("Pixel offset tracking is Done!") + ut.print_process_time(start_time, time.time()) + #sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/process_tsifg.py b/third_party/PyINT/pyint/process_tsifg.py new file mode 100644 index 0000000..7f3d95d --- /dev/null +++ b/third_party/PyINT/pyint/process_tsifg.py @@ -0,0 +1,148 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v1.0 ### +### Copy Right (c): 2017, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Central South University & University of Miami ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob + +def check_variable_name(path): + s=path.split("/")[0] + if len(s)>0 and s[0]=="$": + p0=os.getenv(s[1:]) + path=path.replace(path.split("/")[0],p0) + return path + +def read_template(File, delimiter='='): + '''Reads the template file into a python dictionary structure. + Input : string, full path to the template file + Output: dictionary, pysar template content + Example: + tmpl = read_template(KyushuT424F610_640AlosA.template) + tmpl = read_template(R1_54014_ST5_L0_F898.000.pi, ':') + ''' + template_dict = {} + for line in open(File): + line = line.strip() + c = [i.strip() for i in line.split(delimiter, 1)] #split on the 1st occurrence of delimiter + if len(c) < 2 or line.startswith('%') or line.startswith('#'): + next #ignore commented lines or those without variables + else: + atrName = c[0] + atrValue = str.replace(c[1],'\n','').split("#")[0].strip() + atrValue = check_variable_name(atrValue) + template_dict[atrName] = atrValue + return template_dict + +def is_number(s): + try: + int(s) + return True + except ValueError: + return False + + +def ras2jpg(input, strTitle): + call_str = "convert " + input + ".ras " + input + ".jpg" + os.system(call_str) + call_str = "convert " + input + ".jpg -resize 250 " + input + ".thumb.jpg" + os.system(call_str) + call_str = "convert " + input + ".jpg -resize 500 " + input + ".bthumb.jpg" + os.system(call_str) + call_str = "$INT_SCR/addtitle2jpg.pl " + input + ".thumb.jpg 14 " + strTitle + os.system(call_str) + call_str = "$INT_SCR/addtitle2jpg.pl " + input + ".bthumb.jpg 24 " + strTitle + os.system(call_str) + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + +def write_template(File, Str): + f = open(File,'a') + f.write(Str) + f.close() + +def write_run_coreg_all(projectName,master,slavelist,workdir): + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + run_coreg_all = projectDir + "/run_coreg_all" + f_coreg = open(run_coreg_all,'w') + + for kk in range(len(slavelist)): + str_coreg = "GenOff_Gamma.py " + projectName + ' ' + master + ' ' + slavelist[kk] + ' ' + workdir + '\n' + f_coreg.write(str_coreg) + f_coreg.close() + + +def usage(): + print(''' +****************************************************************************************************** + + Process time series of interferograms from downloading data or SLC images. + + usage: + + process_tsifg.py projectName + + e.g. process_tsifg.py PacayaT163TsxHhA + process_tsifg.py PacayaT163S1A + +******************************************************************************************************* + ''') + +def main(argv): + + if len(sys.argv)==2: + if argv[0] in ['-h','--help']: usage(); sys.exit(1) + else: projectName=sys.argv[1] + else: + usage();sys.exit(1) + + if 'S1' in projectName: + call_str='process_tsifg_sen.py ' + projectName + else: + call_str='process_tsifg_gamma.py ' + projectName + + os.system(call_str) + + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + + + + + + + + + diff --git a/third_party/PyINT/pyint/psokinv2sdm.py b/third_party/PyINT/pyint/psokinv2sdm.py new file mode 100644 index 0000000..2da39ea --- /dev/null +++ b/third_party/PyINT/pyint/psokinv2sdm.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python + +import numpy as np +import os +import sys +import argparse +import subprocess +from pyint import _utils as ut + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='convert psokinv quadtree result to SDM.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('inputfile', help='psokinv quadtree file name ') + parser.add_argument('outfile', help=' file name for SDM ') + inps = parser.parse_args() + return inps +INTRODUCTION = ''' +------------------------------------------------------------------- + create_files for SDM software running file by generated by psokinv +''' + +EXAMPLE = """Usage: + + PSOKINV2SDM.py inputfile outfile + + create_psokinv.py 20241015_20250107.inp 20241015_20250107_los_for_sdm +------------------------------------------------------------------- +""" + +def phitheta2neu(phi,theta): + # + import numpy as np + # + deg2rad=np.pi/180. + # + theta=theta*deg2rad; phi=phi*deg2rad; + # + vn= np.sin(theta)*np.sin(phi) + ve=-np.sin(theta)*np.cos(phi) + vu= np.cos(theta) + # + return vn,ve,vu +# +def neu2phitheta(vn,ve,vu): + # + import numpy as np + # + rad2deg=180./np.pi; + # + theta=np.arccos(vu)*rad2deg; phi=np.arctan2(vn,-ve)*rad2deg; + # + return theta,phi +# +def psokinv2sdm(file,file_out): + # + import numpy as np + # + data=np.genfromtxt(file); + length=len(data) + inc=[] + azi=[] + out_data={} + for i in range(length): + (inc,azi)=neu2phitheta(data[i,4],data[i,3],data[i,5]); + out_data[i]=(data[i,0],data[i,1],data[i,2],inc,azi); + data1=np.array(list(out_data.values())) + txt_data = np.savetxt(file_out,data1,fmt='%.8f') + + +def main(argv): + inps = cmdLineParse() + input_file = inps.inputfile + out_file = inps.outfile + psokinv2sdm(input_file, out_file) + print("convert psokinv products to sdm is done!") + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + diff --git a/third_party/PyINT/pyint/pyint.template b/third_party/PyINT/pyint/pyint.template new file mode 100644 index 0000000..fe85f8b --- /dev/null +++ b/third_party/PyINT/pyint/pyint.template @@ -0,0 +1,206 @@ +# *********************************** PyINT Template *******************************************# +# Template for PyINT v2.1 +# Please fill in the parameters according to your project requirements + +# ========================= Basic parameters ======================== + +# DEM file path (if not provided, SRTM will be downloaded automatically from OpenTopography) +#DEM = /path/to/your/dem.dem + +# Master date for interferometric processing +masterDate = YYYYMMDD # Master date in YYYYMMDD format + +# Swath and burst parameters (for Sentinel-1 TOPS mode only) +start_swath = 1 # Starting swath number +end_swath = 3 # Ending swath number +start_burst = 1 # Starting burst position (relative to master image) +end_burst = 9 # Ending burst position (relative to master image) + +# Multilooking parameters +range_looks = 20 # Multi-look numbers in range direction +azimuth_looks = 4 # Multi-look numbers in azimuth direction + +# ========================= Download Data ======================== + +# Data source and search parameters +Data_Hub = ASF # Data download source: SSARA, Scihub, ASF +sensor = Sentinel-1A # Envisat, ERS-1/2, Sentinel-1A/B, etc. +track = 128 # Track number of the SAR image +frame = 89 # Frame number [defined according to ASF] +start_time = YYYY-MM-DD # Start date for data search +end_time = YYYY-MM-DD # End date for data search + +# ASF download specific parameters +asf_platform = Sentinel-1A # Platform: Sentinel-1A, Sentinel-1B +asf_beam_mode = IW # Beam mode: IW (Interferometric Wide) +asf_polarization = VV # Polarization: VV, VH, HH, HV +asf_flight_direction = ascending # Flight direction: ascending, descending +asf_bbox = W,S,E,N # Bounding box: West,South,East,North + +# ========================= DEM Generation ======================== + +# DEM download parameters (for OpenTopography) +opentopo_api_key = YOUR_API_KEY # OpenTopography API key +opentopo_dem_type = SRTMGL1 # DEM type: SRTMGL1(30m), SRTMGL3(90m), NASADEM(30m), COP30, COP90 +opentopo_auto_tile = 1 # Auto-tile large regions (recommended: 1) + +# DEM processing parameters +dem_lat_ovr = 1 # Latitude oversampling (as to SRTM-1: 0.5 for 60m, 2 for 15m) +dem_lon_ovr = 1 # Longitude oversampling + +# ========================= Interferometry Parameters ======================== + +# Spectral filtering +Igram_Spsflg = 1 # Implement range spectral filtering +rbw_min = 1 # Minimum range bandwidth fraction (0.1 --> 1.0) + +# Coregistration parameters +rwin4cor = 256 # Range window length for coregistration +azwin4cor = 256 # Azimuth window length for coregistration +rsample4cor = 32 # Range samples for fitting coregistration parameters +azsample4cor = 32 # Azimuth samples for fitting coregistration parameters +thresh4cor = 0.3 # Coherence threshold (GAMMA 2016+, SNR for older versions) + +# Coherence estimation +Igram_Cor_rwin = 5 # Range window length for cc_wave coherence estimation +Igram_Cor_awin = 5 # Azimuth window length for cc_wave coherence estimation +Igram_Cor_Win = 5 # Window for cc estimation in adf +adf_alpha = 0.4 # Alpha for Gold-Stein filtering + +# ========================= Simulation Phase ======================== + +Igram_Flag_TDM = N # Y for Tandem-X +Simphase_rpos = - +Simphase_azpos = - +Simphase_rwin = 256 +Simphase_azwin = 256 +Simphase_thresh = - + +# ========================= Unwrapping ======================== + +mcf_triangular = 1 # Triangular type of mcf [0: regular; 1: delaunay] +unwrap_patr = 1 # Unwrap patches in range direction +unwrap_pataz = 1 # Unwrap patches in azimuth direction +unwrapThreshold = 0.05 # Coherence threshold for unwrapping +auto_unw = 1 # Auto or manual reference point [1: auto; 0: manual] +make_mask = 0 # Make mask based on coherence [0: don't mask; 1: make mask] +init_flag = 1 # Flag to set phase at reference point [0: use initial; 1: set to 0.0] +r_refer = - # Phase reference point range offset +a_refer = - # Phase reference point azimuth offset + +# ========================= Geocoding ======================== + +geo_interp = 0 # Interpolation method [0: nearest; 1: bicubic spline] + +# ========================= Satellite Configuration ======================== + +satelite = S1A # Satellite: S1A, S1B, CSK, TSX, ALOS2, ALOS, ENVISAT + +# ========================= Pair Selection ======================== + +network_method = sbas # Network method: sbas, sequential, delaunay, stars +endDate = YYYYMMDD # Exclude dates after this date +startDate = YYYYMMDD # Exclude dates before this date +conNumb = 15 # Connect number for sequential network +max_tb = 365 # Maximum temporal baseline (days) +max_sb = 100 # Maximum spatial baseline (meters) +min_tb = 1 # Minimum temporal baseline (days) +min_sb = 0 # Minimum spatial baseline (meters) + +# ========================= Phase Gradient (Optional) ======================== + +dataformat = cpxfloat32 # Data format of interferogram +scale = 10000 # Original phase/scale +step_windows = 1 # Step length for spatial phase gradient +filter_wins = [7, 7] # Windows of median filter +directions = ['grad_east', 'grad_north', 'grad_northeast', 'grad_southeast'] + +# Mapping parameters for phase gradient +in_low_high = [0.001, 1] # Input range for mapping +out_low_high = [0.001, 1] # Output range for mapping +gamma = 0.8 # Gamma for curve shape +choose_linear = 'n' # Linear or nonlinear mapping ['y': linear; 'n': nonlinear] + +# ========================= Hyp3 Format Conversion (NEW) ======================== + +# Convert GAMMA outputs to Hyp3-compatible GeoTIFF format +hyp3format = 0 # Enable Hyp3 format conversion [0: skip; 1: process] +hyp3format_parallel = 4 # Number of parallel processors for Hyp3 conversion +hyp3_output_dir = - # Output directory for Hyp3 products [default: projectName/Hyp3Products/] + +# ========================= Phase Bias Correction (NEW) ======================== + +# Apply phase bias correction based on loop closures algorithm +# Reference: 10.1016/j.remote.2022.100013 + +phasebias = 0 # Enable phase bias correction [0: skip; 1: process] +phasebias_interval = 12 # Temporal baseline in days [6, 12, or 24] +phasebias_nlook = 10 # Number of looks for multilooking +phasebias_num_a = 2 # Number of calibration parameters to estimate +phasebias_estimate_an = 0 # Estimate an from data [0: use defaults; 1: estimate from loop closures] +phasebias_max_con = 5 # Maximum number of connections + +# Default an values (used if phasebias_estimate_an = 0) +a1_6_day = 0.50 +a2_6_day = 0.36 +a3_6_day = 0.299 +a4_6_day = 0.2476 + +a1_12_day = 0.494 +a2_12_day = 0.297 +a3_12_day = 0.24 +a4_12_day = 0.22 + +a1_24_day = 0.48 +a2_24_day = 0.28 +a3_24_day = 0.22 +a4_24_day = 0.20 + +phasebias_start = - # Start date YYYYMMDD [default: auto-detect] +phasebias_end = - # End date YYYYMMDD [default: auto-detect] + +# ========================= Atmospheric Correction ======================== + +# GACOS tropospheric correction +gacos_correction = 0 # Enable GACOS correction [0: skip; 1: process] +gacos_all_parallel = 1 # Number of parallel processors (1=sequential with ZTD pre-cache, recommended) +gacos_dir = # Directory for GACOS ZTD files [default: projectName/GACOS] +gacos_email = # Email for GACOS auto-download (optional) + +# Ionospheric correction +ionosphere_correction = 0 # Enable ionospheric correction [0: skip; 1: process] +ionosphere_parallel = 4 # Number of parallel processors + +# ========================= Processing Workflow Control ======================== + +download_data = 0 # Enable data download [0: skip; 1: process] +down_parallel = 4 # Number of parallel processors for download + +raw2slc_all = 1 # Convert raw to SLC [0: skip; 1: process] +raw2slc_all_parallel = 4 # Number of parallel processors + +extract_burst_all = 1 # Extract bursts [0: skip; 1: process] +extract_all_parallel = 4 # Number of parallel processors + +coreg_all = 1 # Coregister SLCs [0: skip; 1: process] +coreg_all_parallel = 4 # Number of parallel processors + +select_pairs = 1 # Select interferometric pairs [0: skip; 1: process] + +diff_all = 1 # Generate differential interferograms [0: skip; 1: process] +diff_all_parallel = 4 # Number of parallel processors + +unwrap_all = 1 # Unwrap interferograms [0: skip; 1: process] +unwrap_all_parallel = 4 # Number of parallel processors + +geocode_all = 1 # Geocode products [0: skip; 1: process] +geocode_all_parallel = 4 # Number of parallel processors + +hyp3format_all = 0 # Convert to Hyp3 format [0: skip; 1: process] +hyp3format_all_parallel = 4 # Number of parallel processors + +phasebias_all = 0 # Apply phase bias correction [0: skip; 1: process] + +load_data = 0 # Load data for MintPy processing [0: skip; 1: process] + +# ========================= Good Luck ======================================= diff --git a/third_party/PyINT/pyint/pyintApp.py b/third_party/PyINT/pyint/pyintApp.py new file mode 100644 index 0000000..1ad8c15 --- /dev/null +++ b/third_party/PyINT/pyint/pyintApp.py @@ -0,0 +1,333 @@ +#! /usr/bin/env python +########################################################### +# Project: PyINT # +# Purpose: Interferograms process using python/GAMMA # +# Author: Yunmeng Cao # +# Created: Feb. 2017 # +# Contact : ymcmrs@gmail.com # +# Copy Right (c): 2017-2019, Yunmeng Cao # +########################################################### + +import numpy as np +import os +import sys +import subprocess +import time +import argparse + +from pyint import _utils as ut + + +def _run_or_raise(call_str, stage): + rc = os.system(call_str) + if rc != 0: + raise RuntimeError('%s failed with rc=%s: %s' % (stage, rc, call_str)) + return rc + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Interferograms processing using PyINT.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='name of the project.') + parser.add_argument('-g', '--generate', action='store_true', dest='generate_structure', \ + help='Generate project directory structure if not exists.') + + inps = parser.parse_args() + return inps + + +INTRODUCTION = ''' +----------------------------------------------------------------------------------- + + Single or time-series of interferometry processing for satellite based + Synthetic Aperture Radar (SAR) images start from downloading data to + generate unwrapped-differential interferograms. + + Details please check: https://github.com/ymcmrs/PyINT + + General work-flow: + + 1) download data : download SLCs using SSARA/Scihub/ASF + SSARA: https://github.com/bakerunavco/SSARA + Scihub: Copernicus Open Access Hub + ASF: Alaska Satellite Facility (for Sentinel-1) + [You should provide Sensor, Track, Frame, or Time information in template] + 2) generate SLC : raw 2 slc (multi-frame processing is also supported) + [include orbit correction for S1,ASAR,ERS and burst-extraction for S1] + 3) generate DEM : reference image related geo-dem, rdc-dem, lookup table will be generated. + [SRTM-1 will be downloaded and processed automatically if not provided] + 4) coregister SLC : coregister SLCs to the reference SLC iamge. + [with assistant of DEM] + 5) select pairs : select interferometric pairs for time-series processing. + [networks of sbas, sequential, delaunay, and stars are supported] + 6) interferometry : generate unwrapped differential interferograms. + [include differential, unwrapping, and geocoding] + 7) offset tracking: pixel offset tracking (POT) for displacement measurement. + [two-round estimation with GAMMA offset_pwr_tracking] + 8) GACOS correction: apply GACOS tropospheric atmospheric correction. + [requires GACOS ZTD data for master and slave dates] + + Note: + + i) Single interferogram processing please use slc2ifg.py or raw2ifg.py + ii) Multi-processor parallel processing is supported, but keep in mind GAMMA calls multi-threads already. + iii) ASF download supports both bounding box and Shapefile for spatial filtering. + iv) GACOS correction requires geocoded interferograms (set geocode_all=1 first). + +''' + +EXAMPLE = """Usage: + + pyintApp.py -h + pyintApp.py projectName #[projectName.template should be available in TEMPLATEDIR] + pyintApp.py -g projectName #Generate project directory structure if not exists + +------------------------------------------------------------------------------------ +""" + +def main(argv): + + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + templateDir = os.getenv('TEMPLATEDIR') + scratchDir = os.getenv('SCRATCHDIR') + workDir = scratchDir + '/' + projectName + + # Generate project directory structure if -g flag is set and directory doesn't exist + if inps.generate_structure: + if not os.path.isdir(workDir): + print('Generating project directory structure for: %s' % projectName) + os.makedirs(workDir, exist_ok=True) + os.makedirs(workDir + '/DOWNLOAD', exist_ok=True) + os.makedirs(workDir + '/SLC', exist_ok=True) + os.makedirs(workDir + '/RSLC', exist_ok=True) + os.makedirs(workDir + '/DEM', exist_ok=True) + os.makedirs(workDir + '/ifgrams', exist_ok=True) + print('Project directory structure created successfully.') + print('Directory structure:') + print(' %s/' % projectName) + print(' ├── DOWNLOAD/') + print(' ├── SLC/') + print(' ├── RSLC/') + print(' ├── DEM/') + print(' └── ifgrams/') + return + else: + print('Project directory already exists: %s' % workDir) + print('Skipping directory structure generation.') + return + + templateFile = templateDir + "/" + projectName + ".template" + templateDict = ut.update_template(templateFile) + masterDate = templateDict['masterDate'] + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + satelite = templateDict['satelite'] + HGTSIM = scratchDir + '/' + projectName + '/DEM/' + ut.yyyymmdd(masterDate) + '_' + rlks + 'rlks.rdc.dem' + DataDir = workDir + '/' + 'DOWNLOAD' + DEMDir = workDir + '/' + 'DEM' + os.chdir(scratchDir) + if not os.path.isdir(workDir): + os.mkdir(workDir) + os.chdir(workDir) + os.mkdir(DataDir) + os.mkdir(DEMDir) + os.chdir(DataDir) + else: + os.chdir(DataDir) + ### download data + if templateDict['download_data'] == '1': + if templateDict['Data_Hub'] == 'SSARA': + print('Start to download SAR data using SSARA...') + call_str = 'ssara_federated_query.py -p ' + templateDict['sensor'] + ' -r ' + templateDict['track'] + ' -f ' + templateDict['frame'] + ' -s ' + templateDict['start_time'] + ' -e ' + templateDict['end_time'] + ' --print --download --parallel ' + templateDict['down_parallel'] + print(call_str) + _run_or_raise(call_str, 'download_data_ssara') + elif templateDict['Data_Hub'] == 'Scihub': + print('Start to downlad SAR data from Scihub') + call_str = 'scihub_search_s1_data.py ' + projectName + ' -s ' + templateDict['start_time'] + ' -e ' + templateDict['end_time'] + ' -p ' + templateDict['producttype'] +' -r ' + templateDict['region_box'] + ' -n ' + templateDict['Orbit_number'] + ' -d ' + templateDict['Direction'] + ' -sm ' + templateDict['Sence_model'] + ' -o ' + templateDict['output_file'] + print(call_str) + _run_or_raise(call_str, 'download_data_scihub') + elif templateDict['Data_Hub'] == 'ASF': + print('Start to download Sentinel-1 SLC data from ASF...') + # 获取必要的参数 + path_SLC = DataDir + path_RSLC = workDir + '/RSLC' + date_start = templateDict.get('start_time', '20170101') + date_end = templateDict.get('end_time', '20240101') + + # 构建基本命令 + call_str = 'API_download_S1_SLC.py -s ' + path_SLC + ' -r ' + path_RSLC + ' -i ' + date_start + ' -j ' + date_end + + # 添加用户名和密码（如果提供） + if 'asf_username' in templateDict and templateDict['asf_username'].strip(): + call_str += ' -u ' + templateDict['asf_username'] + if 'asf_password' in templateDict and templateDict['asf_password'].strip(): + call_str += ' -p ' + templateDict['asf_password'] + + # 添加边界框或Shapefile + if 'shapefile' in templateDict and templateDict['shapefile'].strip(): + call_str += ' --shp ' + templateDict['shapefile'] + elif 'bbox' in templateDict and templateDict['bbox'].strip(): + call_str += ' -b ' + templateDict['bbox'] + elif 'asf_bbox' in templateDict and templateDict['asf_bbox'].strip(): + call_str += ' -b ' + templateDict['asf_bbox'] + + # 添加轨道号（可选） + if 'relative_orbit' in templateDict and templateDict['relative_orbit'].strip(): + call_str += ' -o ' + str(templateDict['relative_orbit']) + elif 'track' in templateDict and templateDict['track'].strip(): + call_str += ' -o ' + str(templateDict['track']) + + # 添加帧号（可选） + if 'frame' in templateDict and templateDict['frame'].strip(): + call_str += ' --frame ' + str(templateDict['frame']) + + # 添加飞行方向（可选，默认为升轨） + if 'flight_direction' in templateDict: + call_str += ' -f ' + templateDict['flight_direction'] + else: + call_str += ' -f a' # 默认升轨 + + # 添加波束模式（默认IW） + if 'acquisition_mode' in templateDict: + call_str += ' -q ' + templateDict['acquisition_mode'] + else: + call_str += ' -q IW' # 默认IW模式 + + # 输出格式和下载选项 + call_str += ' -m csv -w Y' + + # 添加并行下载参数 + if 'down_parallel' in templateDict and templateDict['down_parallel'].strip(): + call_str += ' --parallel ' + str(templateDict['down_parallel']) + + print(call_str) + _run_or_raise(call_str, 'download_data_asf') + ### raw 2 slc + if templateDict['raw2slc_all'] == '1': # only for S1 data now + print('Start to convert downloaded-raw data into SLC ...') + print('Number of processor: %s' % str(templateDict['raw2slc_all_parallel'])) + if satelite=='S1A': + call_str = 'down2slc_sen_all.py ' + projectName + ' --parallel ' + templateDict['raw2slc_all_parallel'] + _run_or_raise(call_str, 'raw2slc_s1') + elif satelite=='ALOS': + call_str = 'down2slc_alos_all.py ' + projectName + ' --parallel ' + templateDict['raw2slc_all_parallel'] + _run_or_raise(call_str, 'raw2slc_alos') + elif satelite=='LT': + call_str = 'down2slc_LT1_all.py ' + projectName + ' --parallel ' + templateDict['raw2slc_all_parallel'] + _run_or_raise(call_str, 'raw2slc_lt1') + ### extract bursts + if templateDict['extract_burst_all'] == '1': + print('Start to extract common bursts ...') + print('Number of processor: %s' % str(templateDict['extract_all_parallel'])) + call_str = 'extract_s1_bursts_all.py ' + projectName + ' --parallel ' + templateDict['extract_all_parallel'] + _run_or_raise(call_str, 'extract_burst_all') + + ### generate rdc_dem + if not os.path.isfile(HGTSIM): + print('Start to generate geometry file ...') + call_str = 'makedem_pyint.py ' + projectName + _run_or_raise(call_str, 'makedem_pyint') + call_str = 'generate_rdc_dem.py ' + projectName + _run_or_raise(call_str, 'generate_rdc_dem') + + ### coreg SLC + if templateDict['coreg_all'] == '1': + print('Start to coregister SLCs ...') + print('Number of processor: %s' % str(templateDict['coreg_all_parallel'])) + call_str = 'coreg_gamma_all.py ' + projectName + ' --parallel ' + templateDict['coreg_all_parallel'] + _run_or_raise(call_str, 'coreg_all') + + ### select interferometric pairs + if templateDict['select_pairs'] == '1': + print('Start to select interferometric pairs ...') + print('Network selection method: %s' % templateDict['network_method']) + #print('Meximum temporal baseline threshold: %s' % templateDict['max_tb']) + #print('Meximum spatial baseline threshold: %s' % templateDict['max_sb']) + call_str = 'select_pairs.py ' + projectName + _run_or_raise(call_str, 'select_pairs') + + ### diff ifg + if templateDict['diff_all'] == '1': + print('Start to generate differential interferograms ...') + print('Number of processor: %s' % str(templateDict['diff_all_parallel'])) + call_str = 'diff_gamma_all.py ' + projectName + ' --parallel ' + templateDict['diff_all_parallel'] + _run_or_raise(call_str, 'diff_all') + + ### Pixel Offset Tracking (POT) + if templateDict['pot_all'] == '1': + print('Start to run Pixel Offset Tracking (POT) ...') + print('Number of processor: %s' % str(templateDict['pot_all_parallel'])) + call_str = 'POT_gamma_all.py ' + projectName + ' --parallel ' + templateDict['pot_all_parallel'] + _run_or_raise(call_str, 'pot_all') + + ### unw ifg + if templateDict['unwrap_all'] == '1': + print('Start to unwrap interferometric phases ...') + print('Number of processor: %s' % str(templateDict['unwrap_all_parallel'])) + call_str = 'unwrap_gamma_all.py ' + projectName + ' --parallel ' + templateDict['unwrap_all_parallel'] + _run_or_raise(call_str, 'unwrap_all') + + ### atmcor ifg + if templateDict['atmcor_all'] == '1': + print('Start to correct atmospheric phase ...') + print('Number of processor: %s' % str(templateDict['atmcor_all_parallel'])) + call_str = 'atm_correction_gamma_all.py ' + projectName + ' --parallel ' + templateDict['atmcor_all_parallel'] + _run_or_raise(call_str, 'atmcor_all') + + ### geocode ifg + if templateDict['geocode_all'] == '1': + print('Start to geocode Ifgs ...') + print('Number of processor: %s' % str(templateDict['geocode_all_parallel'])) + call_str = 'geocode_gamma_all.py ' + projectName + ' --parallel ' + templateDict['geocode_all_parallel'] + _run_or_raise(call_str, 'geocode_all') + + ### Convert GAMMA outputs to LiCSBAS format + if templateDict.get('gamma2licsbas_all', '0') == '1': + print('Start to convert GAMMA outputs to LiCSBAS format ...') + print('Number of processor: %s' % str(templateDict.get('gamma2licsbas_all_parallel', '1'))) + call_str = 'gamma2licsbas_gamma_all.py ' + projectName + ' --parallel ' + templateDict.get('gamma2licsbas_all_parallel', '1') + _run_or_raise(call_str, 'gamma2licsbas_all') + + ### GACOS atmospheric correction (after geocoding) + if templateDict['gacos_correction'] == '1': + print('Start to apply GACOS atmospheric correction ...') + gacos_dir = templateDict.get('gacos_dir', '').strip() + gacos_email = templateDict.get('gacos_email', '').strip() + + call_str = 'gacos_gamma_all.py ' + projectName + ' --skip-existing' + if gacos_dir: + call_str += ' --ztd-dir ' + gacos_dir + if gacos_email: + call_str += ' --email ' + gacos_email + print(call_str) + _run_or_raise(call_str, 'gacos_correction') + + ### Convert GAMMA outputs to HyP3 UTM format + if templateDict.get('hyp3format_all', '0') == '1': + print('Start to convert GAMMA outputs to HyP3 UTM format ...') + print('Number of processor: %s' % str(templateDict.get('hyp3format_all_parallel', '1'))) + call_str = 'hyp3format_gamma_all.py ' + projectName + ' --parallel ' + templateDict.get('hyp3format_all_parallel', '1') + _run_or_raise(call_str, 'hyp3format_all') + + ### load data + if templateDict['load_data'] == '1': + print('Start to load data for mintPy time-series analysis ...') + call_str = 'load_mintpy.py ' + projectName + _run_or_raise(call_str, 'load_data') + + print("PyINT processing for project %s is done." % projectName) + ut.print_process_time(start_time, time.time()) + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/third_party/PyINT/pyint/raw2ifg.py b/third_party/PyINT/pyint/raw2ifg.py new file mode 100644 index 0000000..0835ffe --- /dev/null +++ b/third_party/PyINT/pyint/raw2ifg.py @@ -0,0 +1,70 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import time +import argparse + +from pyint import _utils as ut + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate unwrapped differential Ifg from SLC-raw data using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='name of the project.') + parser.add_argument('Mdate',help='master date for interferometry.') + parser.add_argument('Sdate',help='slave date for interferometry.') + + inps = parser.parse_args() + + return inps + + +INTRODUCTION = ''' +-------------------------------------------------------------------------------------- + Generate unwrapped differential Ifg from SLC-raw data using GAMMA. + + Note: 1) Precise orbit data will be downloaded and processed automatically + 2) SRTM-1 will be downloaded and processed automatically if not provided + in the template file. + +''' + +EXAMPLE = """Usage: + + raw2ifg.py projectName Mdate Sdate +--------------------------------------------------------------------------------------- +""" + +def main(argv): + + inps = cmdLineParse() + Mdate = inps.Mdate + Sdate = inps.Sdate + + projectName = inps.projectName + + if 'S1' in projectName: + call_str = 'raw2ifg_s1.py ' + projectName + ' ' + ut.yyyymmdd(Mdate) + ' ' + ut.yyyymmdd(Sdate) + os.system(call_str) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/third_party/PyINT/pyint/raw2ifg_s1.py b/third_party/PyINT/pyint/raw2ifg_s1.py new file mode 100644 index 0000000..f192afd --- /dev/null +++ b/third_party/PyINT/pyint/raw2ifg_s1.py @@ -0,0 +1,132 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import time +import glob +import argparse + +from pyint import _utils as ut + + +def get_s1_date(raw_file): + file0 = os.path.basename(raw_file) + date = file0[17:25] + return date + +def get_satellite(raw_file): + if 'S1A_IW_SLC_' in raw_file: + s0 = 'A' + else: + s0 = 'B' + + return s0 + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate Ifg from Sentinel-1 raw data with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='name of the project.') + parser.add_argument('Mdate',help='master date for interferometry.') + parser.add_argument('Sdate',help='slave date for interferometry.') + + inps = parser.parse_args() + + return inps + + +INTRODUCTION = ''' +--------------------------------------------------------------------------------------------------- + Generate unwrapped differential Ifg from Sentinel-1 raw data with orbit correction using GAMMA. + + Note: 1) Precise orbit data will be downloaded and processed automatically + 2) SRTM-1 will be downloaded and processed automatically if not provided + in the template file. + +''' + +EXAMPLE = """Usage: + + raw2ifg_s1.py projectName Mdate Sdate +---------------------------------------------------------------------------------------------------- +""" + +def main(argv): + + start_time = time.time() + inps = cmdLineParse() + Mdate = inps.Mdate + Sdate = inps.Sdate + + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + downDir = scratchDir + '/' + projectName + '/DOWNLOAD' + M_raw = glob.glob(downDir + '/S1*_' + ut.yyyymmdd(Mdate)+'*')[0] + S_raw = glob.glob(downDir + '/S1*_' + ut.yyyymmdd(Sdate)+'*')[0] + slcDir = scratchDir + '/' + projectName + '/SLC' + + + ######### down 2 slc ############# + call_str = 'down2slc_sen.py ' + M_raw + ' ' + slcDir + os.system(call_str) + + call_str = 'down2slc_sen.py ' + S_raw + ' ' + slcDir + os.system(call_str) + + ########## extract common bursts ## + call_str = 'extract_s1_bursts.py ' + projectName + ' ' + Mdate + os.system(call_str) + + call_str = 'extract_s1_bursts.py ' + projectName + ' ' + Sdate + os.system(call_str) + + ######### generate rdc_dem ########## + call_str = 'generate_rdc_dem.py ' + projectName + os.system(call_str) + + ########## coregister SLC ######## + + call_str = 'coreg_s1_gamma.py ' + projectName + ' ' + Mdate + os.system(call_str) + + call_str = 'coreg_s1_gamma.py ' + projectName + ' ' + Sdate + os.system(call_str) + + ######## Interferometry process ########### + call_str = 'diff_gamma.py ' + projectName + ' ' + Mdate + ' ' + Sdate + os.system(call_str) + + call_str = 'unwrap_gamma.py ' + projectName + ' ' + Mdate + ' ' + Sdate + os.system(call_str) + + call_str = 'geocode_gamma.py ' + projectName + ' ' + Mdate + '-' + Sdate + os.system(call_str) + + print("Generate Ifg from raw-TOPs data is done! ") + ut.print_process_time(start_time, time.time()) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/third_party/PyINT/pyint/raw2slc_ers_envisat.py b/third_party/PyINT/pyint/raw2slc_ers_envisat.py new file mode 100644 index 0000000..6384f3f --- /dev/null +++ b/third_party/PyINT/pyint/raw2slc_ers_envisat.py @@ -0,0 +1,186 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.0 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Email : ymcmrs@gmail.com ### +### Univ. : Now at KAUST ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +import linecache +import datetime + +def StrNum(S): + S = str(S) + if len(S)==1: + S='0' +S + return S + +def UseGamma(inFile, task, keyword): + if task == "read": + f = open(inFile, "r") + while 1: + line = f.readline() + if not line: break + if line.count(keyword) == 1: + strtemp = line.split(":") + value = strtemp[1].strip() + return value + print("Keyword " + keyword + " doesn't exist in " + inFile) + f.close() + + +def check_ERS_par(SAR_IM_0P): + Name = os.path.basename(SAR_IM_0P) + ff = Name.split('.')[1] + date = (Name.split('SAR_IM__0PWDSI')[1]).split('_')[0] + + if ff == 'E1': + par = 'ERS1_ESA.par' + antenna = 'ERS1_antenna.gain' + orbdir = os.getenv('ERS1ORBDIR') + elif ff == 'E2': + par = 'ERS2_ESA.par' + antenna = 'ERS2_antenna.gain' + orbdir = os.getenv('ERS2ORBDIR') + else: + print('Invalid input SAR_IM_0P file.') + sys.exit(1) + + return par, antenna, orbdir, date + +######################################################################### + +INTRODUCTION = ''' +############################################################################# + Copy Right(c): 2017-2019, Yunmeng Cao @PyINT v1.0 + + Generate SLC for ERS raw data with ENVISAT format + +''' + +EXAMPLE = ''' + Usage: + raw2slc_ers_envisat.py SAR_IM_OP -o output_prefix + + Examples: + raw2slc_ers_envisat.py SAR_IM_OP -o 19950102 + +############################################################################## +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLC for ERS raw data with ENVISAT format.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('ers_envisat_raw',help='ERS raw data with ENVISAT format, e.g., SAR_IM_0PXXXX.E1') + parser.add_argument('-o',dest='out_put',help='prefix of the output file, e.g., output.slc, output.slc.par') + + inps = parser.parse_args() + return inps + +################################################################################ + + +def main(argv): + + inps = cmdLineParse() + SAR_IM_0P = inps.ers_envisat_raw + par,antenna,orbdir,date = check_ERS_par(SAR_IM_0P) + + raw = date + '.raw' + azsp = date + '.azsp' + dop = date + '.dop' + rspec = date + '.rspec' + rc = date + '.rc' + autof = date + '.autof' + dop_ambig = date + '.dop_ambig' + + if not os.path.isfile(par): + call_str = 'cp $GAMMA_HOME/MSP/sensors/' + par + ' .' + os.system(call_str) + if not os.path.isfile(antenna): + call_str = 'cp $GAMMA_HOME/MSP/sensors/' + antenna + ' .' + os.system(call_str) + + if inps.out_put: + pslc_par = 'p' + inps.out_put + '.slc.par' + slc_par = inps.out_put + '.slc.par' + slc = inps.out_put + '.slc' + mli_par = inps.out_put + '.mli.par' + mli = inps.out_put + '.mli' + else: + pslc_par = 'p' + date + '.slc.par' + slc_par = date + '.slc.par' + slc = date + '.slc' + mli_par = date + '.mli.par' + mli = date + '.mli' + + call_str = 'ERS_ENVISAT_proc ' + SAR_IM_0P + ' ' + par + ' ' + pslc_par + ' ' + raw + os.system(call_str) + + call_str = 'DELFT_proc2 ' + pslc_par + ' ' + orbdir + ' 20' + os.system(call_str) + + cal_str = 'dop_ambig ' + par + ' ' + pslc_par + ' ' + raw + ' 2 - ' + dop_ambig + os.system(call_str) + + call_str = 'azsp_IQ ' + par + ' ' + pslc_par + ' ' + raw + ' ' + azsp + os.system(call_str) + + call_str = 'doppler ' + par + ' ' + pslc_par + ' ' + raw + ' ' + dop + os.system(call_str) + + call_str = 'rspec_IQ ' + par + ' ' + pslc_par + ' ' + raw + ' ' + rspec + os.system(call_str) + + call_str = 'pre_rc ' + par + ' ' + pslc_par + ' ' + raw + ' ' + rc + os.system(call_str) + + call_str = 'autof ' + par + ' ' + pslc_par + ' ' + rc + ' ' + autof + ' 2.0 ' + os.system(call_str) + + call_str = 'autof ' + par + ' ' + pslc_par + ' ' + rc + ' ' + autof + ' 2.0 ' + os.system(call_str) + + call_str = 'az_proc ' + par + ' ' + pslc_par + ' ' + rc + ' ' + slc + ' 4096 1 ' + ' 57.2 0 2.120 ' + os.system(call_str) + + call_str = 'par_MSP ' + par + ' ' + pslc_par + ' ' + slc_par + ' 1' + os.system(call_str) + + call_str = 'multi_look ' + slc + ' ' + slc_par + ' ' + mli + ' ' + mli_par + ' 2 10' + os.system(call_str) + + + Width = UseGamma(mli_par, 'read', 'range_samples: ') + call_str = 'raspwr ' + mli + ' ' + Width + os.system(call_str) + + + print("Generate SLC from %s is done." % SAR_IM_0P) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + diff --git a/third_party/PyINT/pyint/rslcCopy_gamma.py b/third_party/PyINT/pyint/rslcCopy_gamma.py new file mode 100644 index 0000000..455686f --- /dev/null +++ b/third_party/PyINT/pyint/rslcCopy_gamma.py @@ -0,0 +1,89 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import argparse + +from pyint import _utils as ut + + +INTRODUCTION = ''' +------------------------------------------------------------------- + copy SLC/RSLC from large SLC/RSLC + +''' + +EXAMPLE = ''' + Usage: + slcCopy_gamma.py projectName Sdate rstart rwidth astart awidth outname + slcCopy_gamma.py PacayaT163TsxHhA 20150601 100 345 200 500 20150601_subset +------------------------------------------------------------------- +''' + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Copy RSLC subset from a large SLC.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('Date',help='Master date.') + parser.add_argument('rstart',help='start point of range') + parser.add_argument('rwidth',help='width of range direction') + parser.add_argument('astart',help='start point of azimuth') + parser.add_argument('awidth',help='width of azimith direction') + parser.add_argument('name',help='output name. e.g., 20150101_subset, then .slc and .slc.par will be generated') + + inps = parser.parse_args() + return inps + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Date = inps.Date + rstart = inps.rstart; rwidth = inps.rwidth; astart = inps.astart; awidth = inps.awidth; name = inps.name + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + processDir = scratchDir + '/' + projectName + "/PROCESS" + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + '/RSLC' + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + + Dslc = rslcDir + '/' + Date + '/' + Date + '.rslc' + DslcPar = rslcDir + '/' + Date + '/' + Date + '.rslc.par' + + Oslc = rslcDir + '/' + Date + '/' + name + '.rslc' + OslcPar = rslcDir + '/' + Date + '/' + name + '.rslc.par' + + Oamp = rslcDir + '/' + Date + '/' + name + '_' + rlks + 'rlks.amp' + OampPar = rslcDir + '/' + Date + '/' + name + '_' + rlks + 'rlks.amp.par' + + + + call_str = 'SLC_copy ' + Dslc + ' ' + DslcPar + ' ' + Oslc + ' ' + OslcPar + ' - - ' + rstart + ' ' + rwidth + ' ' + astart + ' ' + awidth + ' - -' + os.system(call_str) + + call_str = 'multi_look ' + Oslc + ' ' + OslcPar + ' ' + Oamp + ' ' + OampPar + ' ' + rlks + ' ' + azlks + os.system(call_str) + #os.remove(MampPar) + #os.remove(SampPar) + print(call_str) + print("RSLC copy done.") + #sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/rslcCopy_gamma_jobs.py b/third_party/PyINT/pyint/rslcCopy_gamma_jobs.py new file mode 100644 index 0000000..c4d728a --- /dev/null +++ b/third_party/PyINT/pyint/rslcCopy_gamma_jobs.py @@ -0,0 +1,135 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import argparse + +from pyint import _utils as ut + + +INTRODUCTION = ''' +------------------------------------------------------------------- + copy SLC/RSLC from large SLC/RSLC + +''' + +EXAMPLE = ''' + Usage: + slcCopy_gamma_jobs.py projectName Sdate rwidth awidth --memory memory_single_job --walltime walltime_single_job + slcCopy_gamma_jobs.py PacayaT163TsxHhA 20150601 5000 5000 --memory 5000 --walltime 00:30:00 +------------------------------------------------------------------- +''' + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Copy SLC subset from a large SLC.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('Date',help='Master date.') + parser.add_argument('rwidth',help='width of range direction') + parser.add_argument('awidth',help='width of azimith direction') + parser.add_argument('--memory',dest = 'memory',default = '2000', help='memory to be allocated for one job, default: 5GB') + parser.add_argument('--walltime',dest = 'walltime',default = '00:05:00', help='walltime to be allocated for one single job') + inps = parser.parse_args() + return inps + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + Date = inps.Date; rwidth = inps.rwidth; awidth = inps.awidth; memory = inps.memory; walltime = inps.walltime + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + processDir = scratchDir + '/' + projectName + "/PROCESS" + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + '/RSLC' + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + + Dslc = rslcDir + '/' + Date + '/' + Date + '.rslc' + DslcPar = rslcDir + '/' + Date + '/' + Date + '.rslc.par' + + nWidth = ut.read_gamma_par(DslcPar, 'read', 'range_samples') + nLine = ut.read_gamma_par(DslcPar, 'read', 'azimuth_lines') + + Ap_samp = np.arange(1,int(nLine),int(awidth)); Na0 = len(Ap_samp); LA_end = int(nLine) - Ap_samp[Na0-1] + Rp_samp = np.arange(1,int(nWidth),int(rwidth)); Nr0 = len(Rp_samp); LR_end = int(nWidth) - Rp_samp[Nr0-1] + + if LA_end > (2/3*int(awidth)): + Ap_samp1 = Ap_samp + else: + Ap_samp1 = Ap_samp[0:(Na0-1)] # last batch with '-' means to the end + + if LR_end > (2/3*int(rwidth)): + Rp_samp1 = Rp_samp + else: + Rp_samp1 = Rp_samp[0:(Nr0-1)] # last batch with '-' means to the end + + Na = len(Ap_samp1); Nr = len(Rp_samp1); Ntotal = Na*Nr + print('Azimuth samples: ' + str(Na)) + print('Range samples: ' + str(Nr)) + print('Total batch numbers: ' + str(Ntotal)) + + batch_txt = scratchDir + '/' + projectName + '/RSLC/' + Date + '/Batch_copySLC' + workDir = scratchDir + '/' + projectName + '/RSLC/' + Date + if os.path.isfile(batch_txt): + os.remove(batch_txt) + for i in range(Na): + for j in range(Nr): + if len(str(i)) == 1: + i0 = '0' + str(i) + else: + i0 = str(i) + if len(str(j)) == 1: + j0 = '0' + str(j) + else: + j0 = str(j) + name0 = Date + '_' +i0 + j0 + rstart = str(Rp_samp1[j]); astart = str(Ap_samp1[i]) + if rstart == str(Rp_samp1[Nr-1]): + rwidth0 = '-' + else: + rwidth0 = str(int(int(rwidth) + 200)) # extend 200 to avoid edge effect + + if astart == str(Ap_samp1[Na-1]): + awidth0 = '-' + else: + awidth0 = str(int(int(awidth) + 200)) # extend 200 to avoid edge effect + + if not i==0: + astart0 = str(int(int(astart) - 200)) # extend 200 to avoid edge effect + else: + astart0 = astart + + if not j==0: + rstart0 = str(int(int(rstart) - 200)) # extend 200 to avoid edge effect + else: + rstart0 = rstart + + #str0 = 'SLC_copy ' + Dslc + ' ' + DslcPar + ' ' + workDir + '/' + name0 + '.rslc' + ' ' + workDir + '/' + name0 + '.rslc.par - - ' + rstart0 + ' ' + rwidth0 + ' ' + astart0 + ' ' + awidth0 + ' - -' + str0 = 'rslcCopy_gamma.py ' + projectName + ' ' + Date + ' ' + rstart0 + ' ' + rwidth0 + ' ' + astart0 + ' ' + awidth0 + ' ' + name0 + call_str = 'echo ' + str0 + ' >>' + batch_txt + os.system(call_str) + + call_str = 'sbatch_jobs.py ' + batch_txt + ' --memory ' + inps.memory + ' --walltime ' + inps.walltime + ' --job-name ' + Date + '_copySLC' + os.system(call_str) + + print("RSLC copy done: " + Date) + #sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/s1_orb_all.py b/third_party/PyINT/pyint/s1_orb_all.py new file mode 100644 index 0000000..fa225b8 --- /dev/null +++ b/third_party/PyINT/pyint/s1_orb_all.py @@ -0,0 +1,114 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + + +def work(data0): + cmd = data0[0] + err_txt = data0[1] + + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + '\n' + #print(aa) + with open(err_txt, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + Using precise orbit data for all rslcs. + +''' + +EXAMPLE = ''' + Usage: + generate_amp_all.py projectName + generate_amp_all.py projectName --parallel 4 +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Using precise orbit data for all rslcs.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + slcDir = scratchDir + '/' + projectName + '/SLC' + rslcDir = scratchDir + '/' + projectName + '/RSLC' + if not os.path.isdir(rslcDir): os.mkdir(rslcDir) + + cmd_command = 'generate_multilook_amp.py' + + err_txt = scratchDir + '/' + projectName + '/generate_amp_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + slc_list = [os.path.basename(fname) for fname in sorted(glob.glob(rslcDir + '/*'))] + #slc_list = ut.get_project_slcList(projectName) + for i in range(len(slc_list)): + rslcPar = rslcDir + '/' + slc_list[i] + '/' + slc_list[i] + '.rslc.par' + #print(rslcPar) + workDir0 = rslcDir + '/' + slc_list[i] + BB = glob.glob(workDir0 + '/*.EOF') + if len(BB)==0: + Sensor = ut.read_gamma_par(rslcPar,'read','sensor') + if 'A' in Sensor: satellite = 'A' + else: satellite = 'B' + ut.download_s1_orbit(slc_list[i],workDir0,satellite=satellite) + + BB = glob.glob(workDir0 + '/*.EOF') + if len(BB) > 0: + orb_file = BB[0] + + call_str = 'S1_OPOD_vec ' + rslcPar + ' ' + orb_file + ' 31' + os.system(call_str) + + print("Using precise orbit data for all rslcs is done! ") + ut.print_process_time(start_time, time.time()) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + \ No newline at end of file diff --git a/third_party/PyINT/pyint/scihub_search_s1_data.py b/third_party/PyINT/pyint/scihub_search_s1_data.py new file mode 100644 index 0000000..6a36c96 --- /dev/null +++ b/third_party/PyINT/pyint/scihub_search_s1_data.py @@ -0,0 +1,220 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT ### +### Author: chen ### +### Contact : chenweicug@126.com ### +################################################################# + +import getopt +import time +import glob +import numpy as np +import os +import sys +import subprocess +import time +import argparse +from pyint import _utils as ut + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Generate differential interferogram image from SLC using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + scihub_search_s1_data.sh -options + scihub_search_s1_data.sh -s 2015-08-01 -e NOW -r "123.0/-123.3/40.0/40.2" -d Descending - +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister all of the SLCs to the reference SAR image using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('-s', dest='Start_time', help='start_time [yyyy-mm-dd or NOW]') + parser.add_argument('-e', dest='End_time', help='end_time [yyyy-mm-dd or NOW]') + parser.add_argument('-r', dest='Region_box',help='region_box "lonW/lonE/latS/latN"') + parser.add_argument('-p', dest='producttype', help='producttype "[SLC GRD OCN]"') + parser.add_argument('-n', dest='Orbit_number',help='orbit_number [0-175]') + parser.add_argument('-d', dest='Direction', help='direction [Ascending/Descending]') + parser.add_argument('-sm',dest='Sence_model', help='sence model [IW/EW/SM]') + parser.add_argument('-o', dest='output_file', help='output_file name') + inps = parser.parse_args() + return inps +def main(argv): + + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + projectDir = scratchDir + '/' + projectName + dataDir = scratchDir + '/' + projectName + '/' +'DOWNLOAD' + #dataDir="DOWNLOAD" + if not os.path.isdir(dataDir): + os.mkdir(dataDir) + os.chdir(projectDir) +# The basic stuff that doesn't change^^ + search_query="https://scihub.copernicus.eu/dhus/search?q=platformname:Sentinel-1" + + if inps.output_file: + Output_file=inps.output_file + else: + Output_file='search_result.txt' + +# If we are searching based on starttime and endtime + if inps.Start_time: + starttime=inps.Start_time + else: + starttime='1990-01-01' + if inps.End_time: + endtime=inps.End_time + else: + endtime='NOW' + search_query=search_query+" AND beginposition:[" + starttime + "T00:00:00.000Z TO "+ endtime + "T00:00:00.000Z]" + #print(search_query) + + if inps.Region_box: + Region=inps.Region_box + compments=Region.split('/') + lonW=compments[0] + lonE=compments[1] + latS=compments[2] + latN=compments[3] + lonw=float(lonW) + lone=float(lonE) + latn=float(latN) + lats=float(latS) + if lonW >=lonE: + print("Error! Longitudes not in increasing order!") + sys.exit(1) + elif lats >= latn: + print("Error! Latitudes not in increasing order!") + sys.exit(1) + elif lats >= 90 or latn >= 90: + print("Error! latS or latN not in bounds!") + sys.exit(1) + else: + parser.print_usage() + sys.exit(1) +# add the SAR covered area + search_query+=" AND footprint:\"intersects(POLYGON((" + lonW + " " + latN + "," + lonE + " " + latN + "," + lonE + " " + latS + "," + lonW + " " + latS + "," + lonW + " " + latN +")))\"" + + if inps.Direction: + direction=inps.Direction + else: + direction="Ascending" +# add the orbit direction + search_query+=" AND orbitdirection:" + direction + + if inps.Sence_model: + model=inps.Sence_model + else: + model='IW' +# add the sense operational mode + search_query+=" AND sensoroperationalmode:"+ model + + if inps.Orbit_number: + if int(inps.Orbit_number) >= 175: + print("Error: orbit must be between 0 and 175") + sys.exit(1) + else: + orbit=inps.Orbit_number + # add the relative orbit number + search_query+=" AND relativeorbitnumber:" + orbit + if inps.producttype: + producttype=inps.producttype + else: + producttype='SLC' +# add the orbit direction + search_query+=" AND producttype:" + producttype + + + + + + +# how many rows to display and where to start? +# Max rows = 100 (slightly annoying rule from the Copernicus server) + if os.path.isfile(Output_file): os.remove(Output_file) + search_query0=search_query+"&start=0&rows=100" + print("the current search_query is: ",search_query0) + #call_str="echo \"Input options:\" $@ > " +Output_file + #os.system(call_str) + call_str="echo \"wget --no-check-certificate --user=chenwei --password=cw1425 \""+ search_query0 + " >> " + Output_file + os.system(call_str) +# Execute the search using wget + search_query0="'"+search_query0+ "'" + call_str="wget --no-check-certificate --user=chenwei --password=cw1425 " + search_query0 + " -O ->> " + Output_file + os.system(call_str) + call_str="`grep \'title>S1\' " + Output_file +" | wc -l`" + num_results=os.system(call_str) + print(num_results) + if num_results >= 100 : + print("We have 100 results... automatically searching for results #100-200") + search_query1=search_query + "&start=100&rows=100" + search_query1="'"+search_query1+ "'" + call_str="wget --no-check-certificate --user=chenwei --password=cw1425 " + search_query1 + " -O ->> " + Output_file + os.system(call_str) + call_str="`grep \'title>S1\' " + Output_file +" | wc -l`" + num_results=os.system(call_str) + elif num_results >= 200 : + print("We have 200 results... automatically searching for results #100-200") + search_query2=search_query + "&start=100&rows=100" + search_query2="'"+search_query2+ "'" + call_str="wget --no-check-certificate --user=chenwei --password=cw1425 " + search_query2 + " -O ->> " + Output_file + os.system(call_str) + call_str="`grep \'title>S1\' " + Output_file +" | wc -l`" + num_results=os.system(call_str) + +# Displaying a summary of the results + call_str="grep \'title>S1\' " + Output_file # displaying the results + os.system(call_str) + print("number of total results is: ") + call_str="grep \'title>S1\' " + Output_file + " | wc -l" + os.system(call_str) + print("####################################################") + print("the search has done and start to download") + print("####################################################") + id_results='uuid_file.txt' + if os.path.isfile(id_results): + os.remove(id_results) + call_str="grep -E 'uuid|<title>S1' " + Output_file + " >> " + id_results + os.system(call_str) + call_str="sed -i 's/<str name=\"uuid\">//g' " + id_results + os.system(call_str) + call_str="sed -i 's/<title>//g' " + id_results + os.system(call_str) + call_str="sed -i 's/<\/title>//g' " + id_results + os.system(call_str) + call_str="sed -i 's/<\/str>//g' "+ id_results + os.system(call_str) + infopen = open(id_results,'r',encoding='utf-8') + lines = infopen.readlines() + count=int(len(lines)/2-1) + for i in range(0,count,2): + j=i+1 + title=lines[i].strip("\n") + uuid=lines[j].strip("\n") + url_address="\"https://scihub.copernicus.eu/dhus/odata/v1/Products('" + uuid + "')/\$value\"" + call_str= "wget -c --no-check-certificate --user=chenwei --password=cw1425 -O " +dataDir + "/" + title + ".zip " + url_address + print(call_str) + os.system(call_str) + call_str=title+".zip download has done successfully!" + print(call_str) + print("#########################################################") + print("#########all the data download successfully!#############") + + + + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/select_pairs.py b/third_party/PyINT/pyint/select_pairs.py new file mode 100644 index 0000000..0db9cb2 --- /dev/null +++ b/third_party/PyINT/pyint/select_pairs.py @@ -0,0 +1,297 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +# This program is modified from MintPy/select_network.py +# Copyright: 2017-2019, Yunjun Zhang +# Contact: yzhang@rsmas.miami.edu + +import os +import sys +import glob +import argparse +import datetime +import inspect +import numpy as np + +from pyint import _utils as ut +from pyint import _network as nt + +def get_datelist_bperplist(TS_Net): + + IFG_Flag=np.asarray(TS_Net[:,0]) + MDatelist=np.asarray(TS_Net[:,1]) + SDatelist=np.asarray(TS_Net[:,2]) + Berplist=np.asarray(TS_Net[:,3]) + TBaselist=np.asarray(TS_Net[:,4]) + + date_list = [] + date_list.append(MDatelist[0]) + for k0 in SDatelist: + date_list.append(k0) + bperp_list = [] + bperp_list.append(0) + for k0 in Berplist: + bperp_list.append(float(k0)) + + tbase_list = [] + tbase_list.append(0) + for k0 in TBaselist: + tbase_list.append(float(k0)) + return date_list, tbase_list, bperp_list + + +def prune_datelist(date_list, tbase_list, pbase_list, templateDict): + nn = len(date_list) + + date_list_out = [] + for k0 in date_list: + if (k0 not in templateDict['exclude_list']) and (float(templateDict['startDate']) < float(k0)) and (float(templateDict['endDate']) > float(k0)): + date_list_out.append(k0) + date_list_out = sorted(date_list_out) + tbase_list_out = [] + pbase_list_out = [] + for k0 in date_list_out: + tbase0 = tbase_list[date_list.index(k0)] + pbase0 = pbase_list[date_list.index(k0)] + tbase_list_out.append(float(tbase0)) + pbase_list_out.append(float(pbase0)) + + tbase_list_out0 = np.asarray(tbase_list_out) + pbase_list_out0 = np.asarray(pbase_list_out) + + tbase_list_out1 = tbase_list_out0 - tbase_list_out0[0] + pbase_list_out1 = pbase_list_out0 - pbase_list_out0[0] + + return date_list_out, list(tbase_list_out1), list(pbase_list_out1) + +def select_network_candidate(date_list,tbase_list,pbase_list,templateDict): + + method = templateDict['network_method'] + + # Pais selection from method + if method == 'sbas': + date12_list = nt.select_pairs_sbas(date_list) + elif method == 'delaunay': + date12_list = nt.select_pairs_delaunay(date_list, tbase_list, pbase_list, norm = True) + elif method == 'star': + date12_list = nt.select_pairs_star(date_list) + elif method == 'sequential': + date12_list = nt.select_pairs_sequential(date_list, int(templateDict['conNumb'])) + #elif method == 'hierarchical': + # date12_list = nt.select_pairs_hierarchical(date_list, pbase_list, inps.tempPerpList) + #elif method == 'mst': + # date12_list = nt.select_pairs_mst(date_list, pbase_list) + else: + raise Exception('Unrecoganized select method: '+ method) + + date12_list =nt.yyyymmdd_date12(date12_list) + nn = len(date12_list) + + ifgram_tbase_list = [] + ifgram_pbase_list = [] + for i in range(nn): + m0 = date12_list[i].split('-')[0] + s0 = date12_list[i].split('-')[1] + + tbase0 = float(tbase_list[date_list.index(s0)]) - float(tbase_list[date_list.index(m0)]) + pbase0 = float(pbase_list[date_list.index(s0)]) - float(pbase_list[date_list.index(m0)]) + + ifgram_tbase_list.append(tbase0) + ifgram_pbase_list.append(pbase0) + + return date12_list, ifgram_tbase_list, ifgram_pbase_list + + +def prune_network(date12_list, ifgram_tbase_list, ifgram_pbase_list, templateDict): + """Pruning network candidates based on temp/perp baseline""" + date12_list0 = date12_list.copy() + ifgram_tbase_list = [float(i) for i in ifgram_tbase_list] + ifgram_pbase_list = [float(i) for i in ifgram_pbase_list] + + ifgram_tbase_list_abs = [abs(i) for i in ifgram_tbase_list] + ifgram_pbase_list_abs = [abs(i) for i in ifgram_pbase_list] + + ifgram_tbase_list_abs = np.asarray(ifgram_tbase_list_abs) + ifgram_pbase_list_abs = np.asarray(ifgram_pbase_list_abs) + + date12_list0 = np.asarray(date12_list0) + ifgram_tbase_list = np.asarray(ifgram_tbase_list) + ifgram_pbase_list = np.asarray(ifgram_pbase_list) + +# date12_list_out = date12_list0[(ifgram_tbase_list_abs < float(templateDict['max_tb'])) & (ifgram_pbase_list_abs < float(templateDict['max_sb']))] + date12_list_out = date12_list0[(ifgram_tbase_list_abs < float(templateDict['max_tb'])) & (ifgram_tbase_list_abs > float(templateDict['min_tb'])) & (ifgram_pbase_list_abs < float(templateDict['max_sb']))] +# ifgram_tbase_list_out = ifgram_tbase_list[(ifgram_tbase_list_abs < float(templateDict['max_tb'])) & (ifgram_pbase_list_abs < float(templateDict['max_sb']))] +# ifgram_pbase_list_out = ifgram_pbase_list[(ifgram_tbase_list_abs < float(templateDict['max_tb'])) & (ifgram_pbase_list_abs < float(templateDict['max_sb']))] + ifgram_tbase_list_out = ifgram_tbase_list[(ifgram_tbase_list_abs < float(templateDict['max_tb'])) & (ifgram_tbase_list_abs > float(templateDict['min_tb'])) & (ifgram_pbase_list_abs < float(templateDict['max_sb']))] + ifgram_pbase_list_out = ifgram_pbase_list[(ifgram_tbase_list_abs < float(templateDict['max_tb'])) & (ifgram_tbase_list_abs > float(templateDict['min_tb'])) & (ifgram_pbase_list_abs < float(templateDict['max_sb']))] + + return list(date12_list_out), list(ifgram_tbase_list_out), list(ifgram_pbase_list_out) + + +def write_ifgram_list(date12_list,ifgram_tbase_list,ifgram_pbase_list, out_file): + # Output directory/filename + # Write txt file + f = open(out_file, 'w') + #f.write('#Interferograms configuration generated by select_network.py\n') + #f.write('# Date12 Btemp(days) Bperp(m) sim_coherence\n') + for i in range(len(date12_list)): + line = '{} {:6.0f} {:6.1f}'.format(date12_list[i], + ifgram_tbase_list[i], + ifgram_pbase_list[i]) + f.write(line+'\n') + f.close() + return out_file + +#---------------------------------------------------------------------------------------------------# +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Select interferometric pairs for time-series InSAR process.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('-o', '--outfile',dest = 'outfile',help='Output list file for network, ifgram_list.txt by default.') + parser.add_argument('--exclude',dest='excludeDate', nargs='*', default=[], + help='date(s) excluded for network selection, e.g. --exclude 060713 070831') + parser.add_argument('--start-date', dest='startDate', type=str, help='start/min date of network') + parser.add_argument('--end-date', dest='endDate', type=str, help='end/max date of network') + parser.add_argument('--max-sb', dest='maxSB', type=float, help='maximum spatial baseline') + parser.add_argument('--max-tb', dest='maxTB', type=str, help='maximum temporal baseline') + parser.add_argument('--min-tb', dest='minTB', type=str, help='minmum temporal baseline') + parser.add_argument('--method', dest='method', default='sbas', choices={'sbas', 'sequential', 'delaunay', 'stars'}, + help='network selection method:\n' + + 'sbas - select based on the threshold values of the spatio-temporal baselines\n' + + 'sequential - select based on the sequential of the SAR acquisitions\n' + + 'delaunay - select based on delaunay triangulars\n' + + 'stars - one master image network, like PS.') + parser.add_argument('--conNumb', dest='conNumb', type=int, default=2, + help='Number of the neibour-connected SAR images at one side for sequential method.') + parser.add_argument('--coreg', dest='coreg', action='store_true', help='Using rslc/rslcPar to check orbit history.') + + inps = parser.parse_args() + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Select interferometric pairs based on SLC_TAB. + [sbas, sequential, delaunay, stars are supported.] + +''' + +EXAMPLE = ''' + Examples: + select_pairs.py PacayaT163TsxHhA + select_pairs.py PacayaT163TsxHhA --max-tb 300 --min-tb 200 --max-sb 100 + select_pairs.py PacayaT163TsxHhA --method sequential --conNumb 3 + select_pairs.py PacayaT163TsxHhA --method denaulay --max-tb 100 --max-sb 100 + select_pairs.py PacayaT163TsxHhA --method stars + select_pairs.py PacayaT163TsxHhA --coreg +------------------------------------------------------------------- +''' + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + templateDict['network_method'] = inps.method + templateDict['conNumb'] = inps.conNumb + + processDir = scratchDir + '/' + projectName + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + '/RSLC' + + #slc_list = ut.get_project_slcList(projectName) + masterDate = templateDict['masterDate'] + + if inps.coreg: + slcDir = rslcDir + slc0 = '.rslc' + slcpar0 = '.rslc.par' + else: + slc0 = '.rslc' + slcpar0 = '.rslc.par' + + slc_list0 = [os.path.basename(fname) for fname in sorted(glob.glob(rslcDir + '/*'))] + slc_list = [] + for k0 in slc_list0: + if ut.is_number(k0): + slc_list.append(k0) + slc_list = sorted(slc_list) + + SLCfile = [] + SLCParfile = [] + for kk in range(len(slc_list)): + #str_slc = slcDir + "/" + slc_list[kk] +"/" + slc_list[kk] + slc0 + str_slc = rslcDir + "/" + slc_list[kk] +"/" + slc_list[kk] + slc0 + str_slc_par = rslcDir + "/" + slc_list[kk] +"/" + slc_list[kk] + slcpar0 + SLCfile.append(str_slc) + SLCParfile.append(str_slc_par) + + RefPar = rslcDir + "/" + masterDate +"/" + masterDate + slcpar0 + SLC_Tab = scratchDir + '/' + projectName + "/SLC_Tab" + + SLC_Tab_all = processDir + "/SLC_Tab_all" + TS_Berp_all = processDir + "/TS_Berp_all" + TS_Itab_all = processDir + "/TS_Itab_all" + itab_type = '1' + pltflg = '0' + + if os.path.isfile(SLC_Tab): os.remove(SLC_Tab) + with open(SLC_Tab, 'a') as f: + for kk in range(len(SLCfile)): + f.write(str(SLCfile[kk])+ ' '+str(SLCParfile[kk])+'\n') + + ## Get all of the pairs using GAMMA + call_str = "base_calc " + SLC_Tab + " " + RefPar + " " + TS_Berp_all + " " + TS_Itab_all + " " + '1 0 ' + '- - - - >/dev/null' + os.system(call_str) + + with open(TS_Berp_all, 'r') as f: + lines = f.readlines() + + TS_Net_all = ut.read_txt2array(TS_Berp_all) + if len(lines) ==1: + TS_Net_all = TS_Net_all.reshape(1,9) + + TS_Net = TS_Net_all[0:(len(SLCfile)-1),:] + date_list, tbase_list, pbase_list = get_datelist_bperplist(TS_Net) + bl_list_txt = scratchDir + '/' + projectName + '/bl_list.txt' + with open(bl_list_txt, 'w') as f: + for kk in range(len(SLCfile)): + f.write(str(date_list[kk])+ ' '+str(pbase_list[kk])+'\n') + + exclude_list = [] + if 'exclude_date' in templateDict: + exclude_list = templateDict['exclude_date'].split(',')[:] + if inps.excludeDate: + exclude_list0 = inps.excludeDate + for k0 in exclude_list0: + exclude_list.append(k0) + templateDict['exclude_list'] = exclude_list + + if inps.startDate: templateDict['startDate'] = inps.startDate + if inps.endDate: templateDict['endDate'] = inps.endDate + if inps.maxTB: templateDict['max_tb'] = inps.maxTB + if inps.maxSB: templateDict['max_sb'] = inps.maxSB + + date_list, tbase_list, pbase_list = prune_datelist(date_list, tbase_list, pbase_list, templateDict) + date12_list, ifgram_tbase_list, ifgram_pbase_list = select_network_candidate(date_list,tbase_list,pbase_list,templateDict) + date12_list_final, ifgram_tbase_list_final, ifgram_pbase_list_final = prune_network(date12_list, ifgram_tbase_list, ifgram_pbase_list, templateDict) + + if inps.outfile: out_file = inps.outfile + else: out_file = scratchDir + '/' + projectName + "/ifgram_list.txt" + + write_ifgram_list(date12_list_final,ifgram_tbase_list_final,ifgram_pbase_list_final, out_file) + sys.exit(1) + +#################################################################### +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/third_party/PyINT/pyint/select_paris_by_cor.py b/third_party/PyINT/pyint/select_paris_by_cor.py new file mode 100644 index 0000000..ae3b0fc --- /dev/null +++ b/third_party/PyINT/pyint/select_paris_by_cor.py @@ -0,0 +1,139 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT ### +### Author: chen ### +### Contact : chenweicug@126.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse +from pyint import _utils as ut +import cv2 + + +INTRODUCTION = ''' +------------------------------------------------------------------- + compare two corherence for SAR images for delete the bad images . + +''' + +EXAMPLE = ''' + Usage: + cor_correlation.py projectName Mdate Sdate + cor_correlation.py PacayaT163TsxHhA 20150102 20150601 ifgrams_list.txt +------------------------------------------------------------------- +''' +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Coregister all of the SLCs to the reference SAR image using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('Mdate',help='Master date.') + parser.add_argument('Sdate',help='Slave date.') + parser.add_argument('ifgs', help='provided ifgram_list_txt. default: using ifgram_list.txt under projectName folder.') + parser.add_argument('correlation_index', help='provided the accept correlation for the file . default: 0.55.') + + inps = parser.parse_args() + return inps + + +def main(argv): + + start_time = time.time() + inps = cmdLineParse() + Mdate = inps.Mdate + Sdate = inps.Sdate + + if inps.correlation_index: Score = inps.correlation_index + else: Score = 0.55 + + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + + projectDir = scratchDir + '/' + projectName + + slcDir = scratchDir + '/' + projectName + '/SLC' + rslcDir = scratchDir + '/' + projectName + '/RSLC' + ifgDir = projectDir + '/ifgrams' + badifgDir = projectDir + '/bad_ifgrams' + if not os.path.isdir(ifgDir): os.mkdir(ifgDir) + if not os.path.isdir(badifgDir): os.mkdir(badifgDir) + Pair = Mdate + '-' + Sdate + workDir = ifgDir + '/' + Pair + + ####################################################################### + Mamp = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + Samp = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + Mrslc = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc' + MrslcPar = rslcDir + '/' + Mdate + '/' + Mdate + '.rslc.par' + Srslc = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc' + SrslcPar = rslcDir + '/' + Sdate + '/' + Sdate + '.rslc.par' + + if inps.ifgs: ifgramList_txt = inps.ifgs + else: ifgramList_txt = scratchDir + '/' + projectName + '/ifgram_list.txt' + MasterPar = rslcDir + '/' + masterDate + '/' + masterDate + '.rslc.par' + first_image= workDir + '/' + Pair + '_' + rlks + 'rlks.diff_filt.cor.bmp' + ifgList0 = ut.read_txt2array(ifgramList_txt) +# 加载第一张图 + image1 = cv2.imread(first_image) + ifgList0 = ut.read_txt2array(ifgramList_txt) +# ifgList = ifgList0[:,0] + if len(ifgList0)==3: + ifgList=ifgList0[0] + ifgList=[ifgList] + + else: + # ifgList=ifgList0[:,0] + ifgList=ifgList0 + err_txt = scratchDir + '/' + projectName + '/cor_correlation_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + out_file='cor_correltion.txt' + if os.path.isfile(out_file): os.remove(out_file) + for i in range(len(ifgList)): + m0 = ut.yyyymmdd(ifgList[i].split('-')[0]) + s0 = ut.yyyymmdd(ifgList[i].split('-')[1]) + Pair2 = m0 + '-' + s0 + workdir = ifgDir + '/' + Pair2 + # 加载第二张图片 + second_image = workdir + '/' + Pair2 + '_' + rlks + 'rlks.diff_filt.cor.bmp' + image2=cv2.imread(second_image) + # 将图片转换为灰度图像 + gray_image1 = cv2.cvtColor(image1, cv2.COLOR_BGR2GRAY) + gray_image2 = cv2.cvtColor(image2, cv2.COLOR_BGR2GRAY) + # 计算两张图片之间的结构相似性指数（SSIM） + + ssim_score = cv2.matchTemplate(gray_image1, gray_image2, cv2.TM_CCOEFF_NORMED) + score = ssim_score[0][0] + + if str(score) < Score: + print(m0, "-", s0, "相关性得分：", ssim_score[0][0]) + call_str=' mv ' + workdir + ' ' + badifgDir + '/' + os.system(call_str) + else: + print("This pair", m0,'-', s0,"is accept") + call_str = 'echo ' + m0 + '-' + s0 + ' ' + ' ' + str(score) + '>>' + out_file + os.system(call_str) + + print("Delete the bad interferograms for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/third_party/PyINT/pyint/single_GACOS_correction.csh b/third_party/PyINT/pyint/single_GACOS_correction.csh new file mode 100644 index 0000000..09bea26 --- /dev/null +++ b/third_party/PyINT/pyint/single_GACOS_correction.csh @@ -0,0 +1,126 @@ +#!/bin/csh -f + +if ($#argv != 7) then + echo "" + echo "Usage: single_GACOS_correction.csh master_ztd master_ztd.rsc slave_ztd slave_ztd.rsc los.grd reference_point incidence_angle" + echo "" + echo "Performs gacos correction for Sentinel 1 for a single interferogram in grd format" + echo "" + echo "Output: corrected unwrapped phase" + echo "" + echo "Reference point text file in lon lat format" + echo "" + echo "" + echo "Indicence angle in degrees (float or integer)" + echo "" + exit 1 + endif + +#Checking input files +if !(-e $1) then + echo "master_ztd binary file: $1 seems not to exist" + exit 1 +endif +if !(-e $2) then + echo "master_ztd.rsc file: $2 seems not to exist" + exit 1 +endif +if !(-e $3) then + echo "slave_ztd binary file: $3 seems not to exist" + exit 1 +endif +if !(-e $4) then + echo "slave_ztd.rsc file: $4 seems not to exist" + exit 1 +endif +if !(-e $5) then + echo "Interferogram to correct: $5 seems not to exist" + exit 1 +endif +if !(-e $6) then + echo "Reference point text file: $6 seems not to exist" + exit 1 +endif + + +#reference point/stable point in radar coordinates +set reference_point = $6 + +#incidence angle obtained with the "SAT_look" script in degrees +set incidence = $7 + +#wavelength for Sentinel 1 (m) +set wavelength = 0.055165 +set pi = 3.141592653589793238462 + +#######FIRST ZTD to grid######### +set x_first_d1 = `cat $2|grep X_FIRST|awk '{print $2}'` +set y_first_d1 = `cat $2|grep Y_FIRST|awk '{print $2}'` +set width_d1 = `cat $2|grep WIDTH|awk '{print $2}'` +set length_d1 = `cat $2|grep FILE_LENGTH|awk '{print $2}'` +set x_step_d1 = `cat $2|grep X_STEP|awk '{print $2}'` +set y_step_d1 = `cat $2|grep X_STEP|awk '{print $2}'` +gmt xyz2grd $1 -G"date1_ztd.grd" -RLT$x_first_d1/$y_first_d1/$width_d1/$length_d1 -I$x_step_d1/$y_step_d1 -ZTLf -di0 -r + + +#######SECOND ZTD to grid######### +set x_first_d2 = `cat $4|grep X_FIRST|awk '{print $2}'` +set y_first_d2 = `cat $4|grep Y_FIRST|awk '{print $2}'` +set width_d2 = `cat $4|grep WIDTH|awk '{print $2}'` +set length_d2 = `cat $4|grep FILE_LENGTH|awk '{print $2}'` +set x_step_d2 = `cat $4|grep X_STEP|awk '{print $2}'` +set y_step_d2 = `cat $4|grep X_STEP|awk '{print $2}'` +gmt xyz2grd $3 -G"date2_ztd.grd" -RLT$x_first_d2/$y_first_d2/$width_d2/$length_d2 -I$x_step_d2/$y_step_d2 -ZTLf -di0 -r +gmt grdsample -R"date1_ztd.grd" date2_ztd.grd -I3s -Gtmp.grd +mv tmp.grd date2_ztd.grd +#TIME DIFFERENCE +gmt grdmath date2_ztd.grd date1_ztd.grd SUB = zpddm.grd +#######RESAMPLE DIFFERENCE WITH THE INTERFEROGRAM PARAMETERS#### +set xmin = `gmt grdinfo -C $5|awk '{print $2}'` +set xmax = `gmt grdinfo -C $5|awk '{print $3}'` +set ymin = `gmt grdinfo -C $5|awk '{print $4}'` +set ymax = `gmt grdinfo -C $5|awk '{print $5}'` +set xinc = `gmt grdinfo -C $5|awk '{print $8}'` +set yinc = `gmt grdinfo -C $5|awk '{print $9}'` +#Resample +#gmt grdsample zpddm.grd -Gresample_zpddm.grd -R$xmin/$xmax/$ymin/$ymax -I$xinc/$yinc -r +#gmt grd2xyz zpddm.grd >tmp +#gmt xyz2grd -R$xmin/$xmax/$ymin/$ymax -I$xinc/$yinc tmp -Gresample_zpddm.grd +gmt grdsample zpddm.grd -Gresample_zpddm.grd -R$5 -I$xinc/$yinc + +#FROM METERS TO PHASE +#gmt grdmath resample_zpddm.grd $wavelength DIV 4 MUL $pi MUL = resample_zpddm_phs.grd +# multiplying by -4 (edition 13/4/2023) +gmt grdmath resample_zpddm.grd $wavelength DIV -4 MUL $pi MUL = resample_zpddm_phs.grd +#cp resample_zpddm.grd resample_zpddm_phs.grd +#REFERENCE POINT +set ref_value_zpddm = `gmt grdtrack $reference_point -Gresample_zpddm_phs.grd -Z` +set ref_value_phase = `gmt grdtrack $reference_point -G$5 -Z` +gmt grdmath resample_zpddm_phs.grd $ref_value_zpddm SUB = szpddm.grd +gmt grdmath $5 $ref_value_phase SUB = phase_ref.grd + +#FROM METER TO PHASE +#gmt grdmath szpddm.grd 4 MUL $pi MUL $wavelength DIV = szpddm_phase.grd +#gmt grdinfo szpddm.grd +#PROJECTION FROM ZENITH VIEW TO LOS +#gmt grdmath szpddm_phase.grd $incidence COSD DIV = szpddm_phase_LOS.grd +gmt grdmath szpddm.grd $incidence COSD DIV = szpddm_LOS.grd +#CORRECTION WITH GACOS DATA +#UNITS: Phase +gmt grdmath phase_ref.grd szpddm_LOS.grd SUB = phase_GACOS_corrected_phs.grd + +#DETRENDING +#UNITS: phase +gmt grdtrend phase_GACOS_corrected_phs.grd -N3r -Dphase_GACOS_corrected_phs_detrended.grd +gmt grdmath phase_GACOS_corrected_phs.grd $wavelength MUL -4 DIV $pi DIV = phase_GACOS_corrected_los.grd +gmt grdmath phase_GACOS_corrected_phs_detrended.grd $wavelength MUL -4 DIV $pi DIV = phase_GACOS_corrected_los_detrended.grd + +#clean up +# +rm date1_ztd.grd date2_ztd.grd +rm zpddm.grd +rm resample_zpddm.grd resample_zpddm_phs.grd +rm phase_ref.grd +rm szpddm.grd +rm szpddm_LOS.grd +echo "corrections done with GACOS files $1 and $3 over interferogram: $5" diff --git a/third_party/PyINT/pyint/slc2ifg.py b/third_party/PyINT/pyint/slc2ifg.py new file mode 100644 index 0000000..43d1d26 --- /dev/null +++ b/third_party/PyINT/pyint/slc2ifg.py @@ -0,0 +1,138 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import subprocess +import time +import glob +import argparse + +from pyint import _utils as ut + + +def get_s1_date(raw_file): + file0 = os.path.basename(raw_file) + date = file0[17:25] + return date + +def get_satellite(raw_file): + if 'S1A_IW_SLC_' in raw_file: + s0 = 'A' + else: + s0 = 'B' + + return s0 + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate Ifg from SLC using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='name of the project.') + parser.add_argument('Mdate',help='master date for interferometry.') + parser.add_argument('Sdate',help='slave date for interferometry.') + + inps = parser.parse_args() + + return inps + + +INTRODUCTION = ''' +-------------------------------------------------------------- + Generate unwrapped differential Ifg from SLC using GAMMA. + + Note: SRTM-1 will be downloaded and processed automatically + if not provided in the template file. + +''' + +EXAMPLE = """Usage: + + slc2ifg.py projectName Mdate Sdate +-------------------------------------------------------------- +""" + +def main(argv): + + start_time = time.time() + inps = cmdLineParse() + Mdate = inps.Mdate + Sdate = inps.Sdate + + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + masterDate = templateDict['masterDate'] + #downDir = scratchDir + '/' + projectName + '/DOWNLOAD' + #M_raw = glob.glob(downDir + '/S1*_' + ut.yyyymmdd(Mdate)+'*')[0] + #S_raw = glob.glob(downDir + '/S1*_' + ut.yyyymmdd(Sdate)+'*')[0] + slcDir = scratchDir + '/' + projectName + '/SLC' + + + ######### down 2 slc ############# + #call_str = 'down2slc_sen.py ' + M_raw + ' ' + slcDir + #os.system(call_str) + + #call_str = 'down2slc_sen.py ' + S_raw + ' ' + slcDir + #os.system(call_str) + + ########## extract common bursts ## + if 'S1' in projectName: + call_str = 'extract_s1_bursts.py ' + projectName + ' ' + Mdate + os.system(call_str) + + call_str = 'extract_s1_bursts.py ' + projectName + ' ' + Sdate + os.system(call_str) + + ######### generate rdc_dem ########## + call_str = 'generate_rdc_dem.py ' + projectName + os.system(call_str) + + ########## coregister SLC ######## + if 'S1' in projectName: + call_str = 'coreg_s1_gamma.py ' + projectName + ' ' + Mdate + os.system(call_str) + + call_str = 'coreg_s1_gamma.py ' + projectName + ' ' + Sdate + os.system(call_str) + else: + call_str = 'coreg_gamma.py ' + projectName + ' ' + Mdate + os.system(call_str) + + call_str = 'coreg_gamma.py ' + projectName + ' ' + Sdate + os.system(call_str) + + ######## Interferometry process ########### + call_str = 'diff_gamma.py ' + projectName + ' ' + Mdate + ' ' + Sdate + os.system(call_str) + + call_str = 'unwrap_gamma.py ' + projectName + ' ' + Mdate + ' ' + Sdate + os.system(call_str) + + call_str = 'geocode_gamma.py ' + projectName + ' ' + Mdate + '-' + Sdate + os.system(call_str) + + print("Generate Ifg from SLC data is done! ") + ut.print_process_time(start_time, time.time()) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + diff --git a/third_party/PyINT/pyint/slc_sen_cat.py b/third_party/PyINT/pyint/slc_sen_cat.py new file mode 100644 index 0000000..e1ff019 --- /dev/null +++ b/third_party/PyINT/pyint/slc_sen_cat.py @@ -0,0 +1,182 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# +import numpy as np +import os +import sys +import subprocess +import getopt +import time +import glob +import argparse +from pyint import _utils as ut +def get_s1_date(raw_file): + file0 = os.path.basename(raw_file) + date = file0[17:25] + return date + +def get_satellite(raw_file): + if 'S1A_IW_SLC_' in raw_file: + s0 = 'A' + else: + s0 = 'B' + + return s0 + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Generate SLC from Sentinel-1 raw data with orbit correction using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName', help='project name. e.g., ChangningT55S1A') + parser.add_argument('date',help='date to be processed. e.g., 20180101') + + inps = parser.parse_args() + + return inps + + +INTRODUCTION = ''' +------------------------------------------------------------------- + + Generate SLC from Sentinel-1 raw data using S1_import_SLC_from_zipfiles with orbit correction. + [Precise orbit data will be downloaded automatically] +''' + +EXAMPLE = """Usage: + + slc_sen_cat.py projectName date + + slc_sen_cat.py ChangningT55S1A 20180517 + +------------------------------------------------------------------- +""" + + + + + + +def main(argv): + + inps = cmdLineParse() + projectName = inps.projectName + date = inps.date + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + slc_dir = projectDir + '/SLC' + down_dir = scratchDir + '/' + projectName + "/DOWNLOAD" + opod_dir = projectDir + '/OPOD' + + if not os.path.isdir(slc_dir): + os.mkdir(slc_dir) + if not os.path.isdir(opod_dir): + os.mkdir(opod_dir) + + work_dir = slc_dir + '/' + date + if not os.path.isdir(work_dir): + os.mkdir(work_dir) + + os.chdir(work_dir) + + t_date = 't_' + date + + call_str = 'ls ' + down_dir + '/S1*' + date + '* > ' + t_date + os.system(call_str) + start_swath = templateDict['start_swath'] + end_swath = templateDict['end_swath'] + k_swath = ut.get_sardata_swath(start_swath,end_swath) + + call_str = 'grep ' + date + ' ' + t_date + ' > t1_' + date + os.system(call_str) + t1_date = 't1_' + date + raw_files = ut.read_txt2list(t1_date) + raw_files = sorted(raw_files) + satellite = get_satellite(str(raw_files[0])) + orbit_file = ut.download_s1_orbit(date,opod_dir,satellite=satellite) + + for i in range(len(raw_files)): + zipfile_ref=str(raw_files[i]) + outfile_name=zipfile_ref.split('/')[-1].split('.')[0] + burst_number_table_ref=outfile_name + '.burst_number_table' + call_str = 'S1_BURST_tab_from_zipfile ' + t1_date + ' ' + str(raw_files[0]) + ' - 1' + os.system(call_str) + + call_str = call_str = 'grep ' + outfile_name + ' ' + 't1_' + date + ' > t2_'+date + os.system(call_str) + + t2_date = 't2_' + date + #call_str = 'S1_import_SLC_from_zipfiles ' + t2_date + ' ' + burst_number_table_ref + ' vv 0 ' + k_swath + call_str = 'S1_import_SLC_from_zipfiles ' + t2_date + ' ' + burst_number_table_ref + ' vv 0 ' + k_swath + ' ' + opod_dir + ' 1 1 ' + os.system(call_str) + + os.chdir(work_dir) + #call_str = "rename vv.slc.iw1 IW1_" + str(i+1) + ".slc * " + call_str = "rename 's/vv.slc.iw1/IW1_" + str(i+1) + ".slc/g' *" + os.system(call_str) + call_str = "rename 's/vv.slc.iw2/IW2_" + str(i+1) + ".slc/g' *" + #call_str = "rename vv.slc.iw2 IW2_" + str(i+1) + ".slc * " + os.system(call_str) + call_str = "rename 's/vv.slc.iw3/IW3_" + str(i+1) + ".slc/g' *" + #call_str = "rename vv.slc.iw3 IW3_" + str(i+1) + ".slc * " + os.system(call_str) + SLC_Tab = work_dir + '/' + date + '_' + str(i+1) + '_SLC_Tab' + SLC_list = sorted(glob.glob(work_dir + '/*IW?_' + str(i+1) + '.slc')) + SLC_par_list = sorted(glob.glob(work_dir + '/*IW?_' + str(i+1) + '.slc.par')) + TOP_par_list = sorted(glob.glob(work_dir + '/*IW?_' + str(i+1) + '.slc.TOPS_par')) + + for kk in range(len(SLC_list)): + call_str = 'echo ' + SLC_list[kk] + ' ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' >> ' + SLC_Tab + os.system(call_str) + + + SLC_Tab = work_dir + '/' + date + '_SLC_Tab' + SLC_list = len(raw_files) + SLC_tab1 = date + '_' + '1_SLC_Tab' + SLC_tab2 = date + '_' + '2_SLC_Tab' + #call_str = 'grep "IW*" ' + date +'_1_SLC_Tab | wc -l' + n_swath = len(open(SLC_tab1, 'r').readlines()) + for kk in range(n_swath): + call_str = 'echo ' + date + '.IW' + str(kk+1) + '.slc ' + date + '.IW' + str(kk+1) + '.slc.par ' + date + '.IW' + str(kk+1) + '.slc.TOPS_par ' + ' >> ' + SLC_Tab + os.system(call_str) + + call_str = 'SLC_cat_S1_TOPS ' + SLC_tab1 + ' ' + SLC_tab2 + ' ' + SLC_Tab + os.system(call_str) + SLC_list = sorted(glob.glob(work_dir + '/*IW?.slc')) + SLC_par_list = sorted(glob.glob(work_dir + '/*IW?.slc.par')) + TOP_par_list = sorted(glob.glob(work_dir + '/*IW?.slc.TOPS_par')) + for kk in range(len(SLC_list)): + BURST = SLC_par_list[kk].replace('slc.par','burst.par') + call_str = 'SLC_burst_corners ' + SLC_par_list[kk] + ' ' + TOP_par_list[kk] + ' > ' +BURST + os.system(call_str) + print("Down to SLC for %s is done! " % date) + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + + + + + + + + + + + + + + + + + diff --git a/third_party/PyINT/pyint/test_srtm_manual.py b/third_party/PyINT/pyint/test_srtm_manual.py new file mode 100644 index 0000000..91bc344 --- /dev/null +++ b/third_party/PyINT/pyint/test_srtm_manual.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +""" +测试手动处理 SRTM .hgt 文件的脚本 + +这个脚本演示如何在 Python < 3.12 环境下手动处理 SRTM 数据 +""" + +import os +import sys + +print("=" * 80) +print("SRTM 手动处理功能测试") +print("=" * 80) + +# 检查Python版本 +print(f"\nPython版本: {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}") + +# 导入处理函数 +from makedem import process_srtm_hgt_files, HAS_SRTM + +print(f"\nsrtm库状态: {'已安装' if HAS_SRTM else '未安装 (需要Python >= 3.12)'}") + +print("\n" + "=" * 80) +print("使用说明") +print("=" * 80) + +print(""" +步骤1: 下载SRTM .hgt文件 +--------------------------- +从以下网站下载所需区域的 .hgt 文件: + +1. CSI-CGIAR SRTM (推荐,无需注册) + https://srtm.csi.cgiar.org/ + +2. USGS EarthExplorer (需要注册) + https://earthexplorer.usgs.gov/ + +文件命名规则: +- N39E116.hgt → 北纬39-40°, 东经116-117° +- N39W110.hgt → 北纬39-40°, 西经110-109° +- S10E120.hgt → 南纬10-9°, 东经120-121° + +步骤2: 组织文件 +--------------------------- +将所有 .hgt 文件放在同一目录下: + +mkdir -p ~/data/SRTM +mv N*.hgt S*.hgt ~/data/SRTM/ + +步骤3: 运行命令 +--------------------------- +使用以下命令生成DEM: + +makedem.py -r 116/117/39/40 --dem-source srtm --srtm-data-dir ~/data/SRTM + +或者直接调用Python函数: + +from makedem import process_srtm_hgt_files + +dem_file = process_srtm_hgt_files( + west=116.0, + south=39.0, + east=117.0, + north=40.0, + srtm_data_dir='/path/to/srtm/data', + save_path='/path/to/output' +) + +""") + +print("=" * 80) +print("示例: 为北京地区(116-117°E, 39-40°N)下载SRTM数据") +print("=" * 80) + +print(""" +需要下载的文件: +1. 访问 https://srtm.csi.cgiar.org/ +2. 选择区域: 39-40°N, 116-117°E +3. 下载文件: N39E116.hgt + +下载命令示例: +-------------""") +print(f""" +# 创建数据目录 +mkdir -p ~/data/SRTM + +# 假设已下载 N39E116.hgt 到 ~/Downloads +mv ~/Downloads/N39E116.hgt ~/data/SRTM/ + +# 生成DEM +makedem.py -r 116/117/39/40 --dem-source srtm --srtm-data-dir ~/data/SRTM + +# 输出文件将保存为: SRTM_116_117_39_40.tif +""") + +print("=" * 80) +print("注意事项") +print("=" * 80) + +print(""" +1. SRTM文件必须覆盖整个目标区域 + - 对于区域 116-117°E, 39-40°N + - 需要 N39E116.hgt 文件 + +2. 文件格式 + - 支持 .hgt 文件 (未压缩) + - 如果下载的是 .hgt.zip, 请先解压 + +3. 空数据区域 + - SRTM数据在海洋区域可能为空值 + - 程序会自动处理空值区域 + +4. Python版本兼容性 + - Python >= 3.12: 可使用 srtm 库(自动查询) + - Python < 3.12: 使用手动处理方法(GDAL合并) + - 两种方法结果相同,只是处理方式不同 +""") + +print("=" * 80) diff --git a/third_party/PyINT/pyint/test_tiled.dem b/third_party/PyINT/pyint/test_tiled.dem new file mode 100644 index 0000000..266abaa Binary files /dev/null and b/third_party/PyINT/pyint/test_tiled.dem differ diff --git a/third_party/PyINT/pyint/test_tiled.dem.par b/third_party/PyINT/pyint/test_tiled.dem.par new file mode 100644 index 0000000..3123a30 --- /dev/null +++ b/third_party/PyINT/pyint/test_tiled.dem.par @@ -0,0 +1,27 @@ +Gamma DIFF&GEO DEM/MAP parameter file +title: IMPORTED DEM FROM Copernicus30 +DEM_projection: EQA +data_format: INTEGER*2 +DEM_hgt_offset: 0.00000 +DEM_scale: 1.00000 +width: 6000 +nlines: 4800 +corner_lat: 28.000416666673939 decimal degrees +corner_lon: 101.999583333269271 decimal degrees +post_lat: -0.000833333333333 decimal degrees +post_lon: 0.000833333333333 decimal degrees + +ellipsoid_name: WGS 84 +ellipsoid_ra: 6378137.000 m +ellipsoid_reciprocal_flattening: 298.2572236 + +datum_name: WGS 1984 +datum_shift_dx: 0.000 m +datum_shift_dy: 0.000 m +datum_shift_dz: 0.000 m +datum_scale_m: 0.00000e+00 +datum_rotation_alpha: 0.00000e+00 arc-sec +datum_rotation_beta: 0.00000e+00 arc-sec +datum_rotation_gamma: 0.00000e+00 arc-sec +datum_country_list Global Definition, WGS84, World + diff --git a/third_party/PyINT/pyint/tsview_mintpy_ssa.py b/third_party/PyINT/pyint/tsview_mintpy_ssa.py new file mode 100644 index 0000000..4e24c1f --- /dev/null +++ b/third_party/PyINT/pyint/tsview_mintpy_ssa.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +# Load the usual suspects: +import os +import sys +import argparse +import pandas as pd +import numpy as np +import math +import matplotlib.pyplot as plt +from mintpy.utils import utils as ut +from ssa import SSA +from sklearn.metrics import mean_squared_error + + +def create_parser(): + parser = argparse.ArgumentParser(description='SSA time series analysis for a specific location') + parser.add_argument('ts_dir', help='Time series directory (e.g., ./SBAS_atm_gacos)') + parser.add_argument('ts_file', help='Time series file name (e.g., geo/geo_timeseries_SET_GACOS_ramp_demErr.h5)') + parser.add_argument('lat', type=float, help='Latitude of the point') + parser.add_argument('lon', type=float, help='Longitude of the point') + parser.add_argument('-L', '--window', type=int, default=8, help='Window length for SSA (default: 8)') + parser.add_argument('-o', '--output', default='ssa_ts.txt', help='Output file name (default: ssa_ts.txt)') + return parser + + +def parse_args(): + parser = create_parser() + return parser.parse_args() + + +def DecYr(x): + return x.dt.to_period('D').dt.to_timestamp().dt.year + x.dt.to_period('D').dt.to_timestamp().dt.dayofyear / 365.25 + + +def sigma_sum(Sigma): + sigma_sumsq = (Sigma**2).sum() + return Sigma**2 / sigma_sumsq * 100 + + +def main(): + args = parse_args() + + proj_dir = os.path.expanduser(args.ts_dir) + ts_file = os.path.join(proj_dir, args.ts_file) + geom_file = None + + print(f"Reading time series from: {ts_file}") + print(f"Location: lat={args.lat}, lon={args.lon}") + print(f"Window length: {args.window}") + + dates, dis, std = ut.read_timeseries_lalo(lat=args.lat, lon=args.lon, ts_file=ts_file, lookup_file=geom_file) + + # Convert from meter to mm and save to panda df + df = pd.DataFrame({'date': dates, 'dis': dis*1000}) + + # Decomposition + F_ssa = SSA(dis*1000, args.window) + contri = sigma_sum(F_ssa.Sigma) + df_ssa = pd.concat([F_ssa.components_to_df()], axis=1) + np.savetxt('sigma_ssa.txt', np.c_[np.arange(1, args.window+1), contri], fmt="%.2f") + df_comp = pd.concat([df['date'], F_ssa.components_to_df()], axis=1) + print('\n \n', 'Writing SSA all components to: ssa_all_compo.txt ', '\n') + df_comp.to_csv('ssa_all_compo.txt', index=False, header=True, sep='\t') + + plt.rcParams.update({'font.size': 10}) + fig, ax = plt.subplots(nrows=args.window, ncols=1, sharex=True, figsize=(5, 11*args.window)) + for i, column in enumerate(df_ssa.columns): + ax[i].plot(df_ssa.index, df_ssa[column], label=column) + ax[i].set_title(f'{column} ({contri[i]:.1f}%)') + plt.subplots_adjust(hspace=.5) + + # Reconstruction + trend_indices = list(range(min(2, args.window))) + noise_indices = list(range(2, args.window)) + + df['ssa_trend'] = pd.DataFrame(F_ssa.reconstruct(trend_indices)) + df['noise'] = pd.DataFrame(F_ssa.reconstruct(noise_indices)) if noise_indices else 0 + print('\n \n', 'Writing to: ', args.output, '\n') + df.to_csv(args.output, index=False, header=True, sep='\t') + + fig, ax = plt.subplots(nrows=3, ncols=1, sharex=True, figsize=(10, 10)) + + ax[0].plot(df['date'], df['dis'], label='InSAR') + ax[0].set_title('InSAR') + ax[1].plot(df['date'], df['noise'], label='Noise') + ax[1].set_title('Periodic+Noise') + ax[2].plot(df['date'], df['ssa_trend'], label='Trend') + ax[2].set_title('Trend') + plt.show() + + +if __name__ == '__main__': + main() diff --git a/third_party/PyINT/pyint/unwrap_gamma.py b/third_party/PyINT/pyint/unwrap_gamma.py new file mode 100644 index 0000000..0995fc0 --- /dev/null +++ b/third_party/PyINT/pyint/unwrap_gamma.py @@ -0,0 +1,156 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +from PIL import Image +from pylab import * +import argparse + +from pyint import _utils as ut + + +def _run_or_raise(call_str, stage): + rc = os.system(call_str) + if rc != 0: + raise RuntimeError('%s failed with rc=%s: %s' % (stage, rc, call_str)) + return rc + + +INTRODUCTION = ''' +------------------------------------------------------------------- + Unwrap differential interferogram using GAMMA. + [Only support mcf, not implement branch_cut yet] + +''' + +EXAMPLE = ''' + Usage: + unwrap_gamma.py projectName Mdate Sdate + unwrap_gamma.py PacayaT163TsxHhA 20150102 20150601 +------------------------------------------------------------------- +''' + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Unwrap differential interferogram using GAMMA-mcf method.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('Mdate',help='Master date.') + parser.add_argument('Sdate',help='Slave date.') + + inps = parser.parse_args() + return inps + + +def main(argv): + + inps = cmdLineParse() + Mdate = inps.Mdate + Sdate = inps.Sdate + + projectName = inps.projectName + Sdate = inps.Sdate + Mdate = inps.Mdate + + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + auto_unw = templateDict['auto_unw'] + init_flag = templateDict['init_flag'] + r_refer = templateDict['r_refer'] + a_refer = templateDict['a_refer'] + make_mask = templateDict['make_mask'] + processDir = scratchDir + '/' + projectName + "/PROCESS" + slcDir = scratchDir + '/' + projectName + "/SLC" + rslcDir = scratchDir + '/' + projectName + '/RSLC' + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + + Pair = Mdate + '-' + Sdate + workDir = ifgDir + '/' + Pair + + ################ copy file for parallel processing ############### + Mamp0 = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar0 = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + Samp0 = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar0 = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + + Mamp = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp' + MampPar = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + Samp = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp' + SampPar = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + ut.copy_file(Mamp0,Mamp) + ut.copy_file(Samp0,Samp) + ut.copy_file(MampPar0,MampPar) + ut.copy_file(SampPar0,SampPar) + ############################################################### + + nWidth = ut.read_gamma_par(MampPar, 'read', 'range_samples') + nLine = ut.read_gamma_par(MampPar, 'read', 'azimuth_lines') + if auto_unw == '1': + if str(r_refer).strip() in ('', '-'): + r_refer = str(int(int(nWidth) / 2)) + if str(a_refer).strip() in ('', '-'): + a_refer = str(int(int(nLine) / 2)) + unwrap_window = '0 0 ' + nWidth + ' ' + nLine + + CORMASK = workDir + '/' + Pair + '_' +rlks + 'rlks.diff_filt.cor' + WRAPlks = workDir + '/' + Pair + '_' +rlks + 'rlks.diff_filt' + UNWlks = workDir + '/' + Pair + '_' +rlks + 'rlks.diff_filt.unw' + + CORMASKbmp = CORMASK.replace('.diff_filt.cor','.diff_filt.cor_mask.bmp') + + if os.path.isfile(CORMASKbmp): + os.remove(CORMASKbmp) + + call_str = 'rascc_mask ' + CORMASK + ' ' + Mamp + ' ' + nWidth + ' 1 1 0 1 1 ' + templateDict['unwrapThreshold'] + ' 0.0 0.1 0.9 1. .35 1 ' + CORMASKbmp # based on int coherence + _run_or_raise(call_str, 'rascc_mask') + + if auto_unw == '1': + if make_mask == "1": + call_str = 'mcf ' + WRAPlks + ' ' + CORMASK + ' ' + CORMASKbmp + ' ' + UNWlks + ' ' + nWidth + ' ' + templateDict['mcf_triangular'] + ' ' + unwrap_window + ' ' + templateDict['unwrap_patr'] + ' ' + templateDict['unwrap_pataz'] +' - '+ r_refer + ' ' + a_refer + ' ' + init_flag + print(call_str) + _run_or_raise(call_str, 'mcf_masked') + else: + call_str = 'mcf ' + WRAPlks + ' ' + CORMASK + ' - ' + ' ' + UNWlks + ' ' + nWidth + ' ' + templateDict['mcf_triangular'] + ' ' + unwrap_window + ' ' + templateDict['unwrap_patr'] + ' ' + templateDict['unwrap_pataz'] + ' - ' + r_refer + ' ' + a_refer + ' ' + init_flag + _run_or_raise(call_str, 'mcf_unmasked') + else: + im = array(Image.open(CORMASKbmp)) + imshow(im) + print('Please select the reference points,max click 10 times') + x =ginput(10) + print ('you clicked:',x) + ref_point=list(x[-1]) + r_init=int(ref_point[0]) + a_init=int(ref_point[1]) + if make_mask == '1': + call_str = 'mcf ' + WRAPlks + ' ' + CORMASK + ' ' + CORMASKbmp + ' ' + UNWlks + ' ' + nWidth + ' ' + templateDict['mcf_triangular'] + ' ' + unwrap_window + ' ' + templateDict['unwrap_patr'] + ' ' + templateDict['unwrap_pataz'] + ' - ' + str(r_init) + ' ' + str(a_init) + ' ' + init_flag + _run_or_raise(call_str, 'mcf_masked_manual') + else: + call_str = 'mcf ' + WRAPlks + ' - ' + ' - ' + ' ' + UNWlks + ' ' + nWidth + ' ' + templateDict['mcf_triangular'] + ' ' + unwrap_window + ' ' + templateDict['unwrap_patr'] + ' ' + templateDict['unwrap_pataz'] + ' - ' + str(r_init) + ' ' + str(a_init) + ' ' + init_flag + _run_or_raise(call_str, 'mcf_unmasked_manual') + call_str = 'rasdt_pwr ' + UNWlks + ' ' + Mamp + ' ' + nWidth + ' 1 0 1 1 -3.14 3.14 1' + _run_or_raise(call_str, 'rasdt_pwr_unw') + + if os.path.isfile(Mamp): + os.remove(Mamp) + if os.path.isfile(Samp): + os.remove(Samp) + #os.remove(MampPar) + #os.remove(SampPar) + print("Uwrapping interferometric phase is done!") + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/unwrap_gamma_all.py b/third_party/PyINT/pyint/unwrap_gamma_all.py new file mode 100644 index 0000000..0cf1749 --- /dev/null +++ b/third_party/PyINT/pyint/unwrap_gamma_all.py @@ -0,0 +1,143 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + + +def work(data0): + cmd = data0[0] + err_file = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if type(stdout) == bytes: + bb=stdout.decode("utf-8", errors="replace") + else: + bb = stdout + + if p.returncode != 0: + detail_parts = [] + if bb: + detail_parts.append(bb) + if aa: + detail_parts.append(aa) + detail = '\n'.join(part.strip() for part in detail_parts if part and part.strip()) + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + cmd[3] + '\n' + with open(err_file, 'a') as f: + f.write(str0) + if detail: + f.write(detail) + f.write('\n') + raise RuntimeError(str0.strip() + ' failed with rc=' + str(p.returncode) + ('\n' + detail if detail else '')) + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + cmd[3] + '\n' + with open(err_file, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + Unwrap differential interferograms for one project using GAMMA. + +''' + +EXAMPLE = ''' + Usage: + unwrap_gamma_all.py projectName + unwrap_gamma_all.py projectName --parallel 4 + unwrap_gamma_all.py projectName --parallel 4 --ifgramList-txt /test/ifgram_list.txt +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Unwrap differential interferograms for one project using GAMMA.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + parser.add_argument('--ifgarmList-txt', dest='ifgarmListTxt', help='provided ifgram_list_txt. default: using ifgram_list.txt under projectName folder.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + templateDir = os.getenv('TEMPLATEDIR') + templateFile = templateDir + "/" + projectName + ".template" + projectDir = scratchDir + '/' + projectName + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + templateDict=ut.update_template(templateFile) + rlks = templateDict['range_looks'] + azlks = templateDict['azimuth_looks'] + + if inps.ifgarmListTxt: ifgramList_txt = inps.ifgarmListTxt + else: ifgramList_txt = scratchDir + '/' + projectName + '/ifgram_list.txt' + ifgList0 = ut.read_txt2array(ifgramList_txt) +# ifgList = ifgList0[:,0] + if len(ifgList0)==3: + ifgList=ifgList0[0] + ifgList=[ifgList] + else: + ifgList=ifgList0[:,0] + + err_txt = scratchDir + '/' + projectName + '/unwrap_gamma_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + for i in range(len(ifgList)): + m0 = ut.yyyymmdd(ifgList[i].split('-')[0]) + s0 = ut.yyyymmdd(ifgList[i].split('-')[1]) + cmd0 = ['unwrap_gamma.py',projectName, m0, s0] + unw_file0 = ifgDir + '/' + ifgList[i] + '/' + ifgList[i] + '_' + rlks + 'rlks.diff_filt.unw.bmp' + data0 = [cmd0,err_txt] + + k00 = 0 + if os.path.isfile(unw_file0): + if os.path.getsize(unw_file0) > 0: + k00 = 1 + if k00==0: + data_para.append(data0) + + results = ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + failures = [str(item) for item in results if isinstance(item, Exception)] + if failures: + raise RuntimeError('\n\n'.join(failures)) + print("Unwrap differential interferograms for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(0) + +if __name__ == '__main__': + main(sys.argv[:]) + diff --git a/third_party/PyINT/pyint/unwrap_snaphu_gamma.py b/third_party/PyINT/pyint/unwrap_snaphu_gamma.py new file mode 100644 index 0000000..9c51057 --- /dev/null +++ b/third_party/PyINT/pyint/unwrap_snaphu_gamma.py @@ -0,0 +1,447 @@ +#! /usr/bin/env python + +################################################################# + +### This program is part of PyINT v2.1 ### + +### Copy Right (c): 2017-2019, Yunmeng Cao ### + +### Author: Yunmeng Cao ### + +### Contact : ymcmrs@gmail.com ### + +################################################################# + + + +import numpy as np + +import os + +import sys + +import argparse + +import subprocess + +import re + + + +from pyint import _utils as ut + + + +INTRODUCTION = ''' + +------------------------------------------------------------------- + + Unwrap differential interferogram using SNAPHU. + + + +''' + + + +EXAMPLE = ''' + + Usage: + + unwrap_snaphu.py projectName Mdate Sdate + + unwrap_snaphu.py PacayaT163TsxHhA 20150102 20150601 + +------------------------------------------------------------------- + +''' + + + +def cmdLineParse(): + + parser = argparse.ArgumentParser(description='Unwrap differential interferogram using SNAPHU.',\ + + formatter_class=argparse.RawTextHelpFormatter,\ + + epilog=INTRODUCTION+'\n'+EXAMPLE) + + + + parser.add_argument('projectName', help='projectName for processing.') + + parser.add_argument('Mdate', help='Master date.') + + parser.add_argument('Sdate', help='Slave date.') + + + + inps = parser.parse_args() + + return inps + + + +def run_command(cmd): + + print(f"Running: {cmd}") + + status = subprocess.call(cmd, shell=True) + + if status != 0: + + print(f"Error running command: {cmd}") + + sys.exit(1) + + + +def extract_value(s): + + """Extract numeric value from string that may contain units""" + + # Remove any non-numeric characters except decimal point and minus sign + + return re.sub(r'[^\d\.\-]', '', s) + + + +def main(argv): + + inps = cmdLineParse() + + projectName = inps.projectName + + Mdate = inps.Mdate + + Sdate = inps.Sdate + + + + scratchDir = os.getenv('SCRATCHDIR') + + templateDir = os.getenv('TEMPLATEDIR') + + templateFile = templateDir + "/" + projectName + ".template" + + templateDict = ut.update_template(templateFile) + + rlks = templateDict['range_looks'] + + azlks = templateDict['azimuth_looks'] + + + + processDir = scratchDir + '/' + projectName + "/PROCESS" + + slcDir = scratchDir + '/' + projectName + "/SLC" + + rslcDir = scratchDir + '/' + projectName + '/RSLC' + + ifgDir = scratchDir + '/' + projectName + '/ifgrams' + + + + Pair = Mdate + '-' + Sdate + + workDir = ifgDir + '/' + Pair + + + + # Create working directory if not exists + + os.makedirs(workDir, exist_ok=True) + + + + # Copy required files for processing + + Mamp0 = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp' + + MampPar0 = rslcDir + '/' + Mdate + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + + Samp0 = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp' + + SampPar0 = rslcDir + '/' + Sdate + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + + + Mamp = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp' + + MampPar = workDir + '/' + Mdate + '_' + rlks + 'rlks.amp.par' + + Samp = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp' + + SampPar = workDir + '/' + Sdate + '_' + rlks + 'rlks.amp.par' + + off_par = workDir + '/' + Pair + '_' + rlks + 'rlks.off' + + ut.copy_file(Mamp0, Mamp) + + ut.copy_file(Samp0, Samp) + + ut.copy_file(MampPar0, MampPar) + + ut.copy_file(SampPar0, SampPar) + + + + # Define input/output files + + int_dir = workDir + + rootname = Pair + '_' + rlks + 'rlks' + + int_file = int_dir + '/' + rootname + '.diff_filt' + + cor_file = int_dir + '/' + rootname + '.diff_filt.cor' + + + + # Read parameters from par files + + width = ut.read_gamma_par(off_par, 'read', 'interferogram_azimuth_lines:') + length = ut.read_gamma_par(off_par, 'read', 'interferogram_width:') + #length = ut.read_gamma_par(MampPar, 'read', 'azimuth_lines') + row = ut.read_gamma_par(off_par, 'read', 'interferogram_azimuth_lines:') + col = ut.read_gamma_par(off_par, 'read', 'interferogram_width:') + pos = ut.read_gamma_par(MampPar, 'read', 'sar_to_earth_center') + earth = ut.read_gamma_par(MampPar, 'read', 'earth_radius_below_sensor') + near = ut.read_gamma_par(MampPar, 'read', 'near_range_slc') + dr = ut.read_gamma_par(off_par, 'read', 'interferogram_range_pixel_spacing:') + da_flight = ut.read_gamma_par(off_par, 'read', 'interferogram_azimuth_pixel_spacing:') + rangeres = ut.read_gamma_par(MampPar, 'read', 'range_pixel_spacing') + azres = ut.read_gamma_par(MampPar, 'read', 'azimuth_pixel_spacing') + nrange = ut.read_gamma_par(off_par, 'read', 'interferogram_range_looks:') + nazi = ut.read_gamma_par(off_par, 'read', 'interferogram_azimuth_looks:') + + + # Extract numeric values + earth = extract_value(earth) + pos = extract_value(pos) + near = extract_value(near) + dr = extract_value(dr) + da_flight = extract_value(da_flight) + nrange = extract_value(nrange) + nazi = extract_value(nazi) + rangeres = extract_value(rangeres) + azres = extract_value(azres) + + alt = str(float(pos) - float(earth)) + da = str(float(da_flight) * float(earth) / (float(earth) + float(alt))) + ncor = str(float(dr) * float(da) / (float(rangeres) * float(azres))) + + + + + + + + + + + + # Create SNAPHU configuration file + + conf_file = workDir + '/' + rootname + '.snaphuconf' + + with open(conf_file, 'w') as f: + + f.write(f"STATCOSTMODE SMOOTH\n") + + f.write(f"INFILE {rootname}.int\n") + + f.write(f"LINELENGTH {width}\n") # 使用原始宽度 + + f.write(f"OUTFILE {rootname}.unw\n") + + f.write(f"CORRFILE {rootname}.cor\n") + + f.write(f"LOGFILE {rootname}.snaphulog\n") + + f.write(f"\n") + + f.write(f"PIECEFIRSTROW 1\n") + + f.write(f"PIECEFIRSTCOL 1\n") + + f.write(f"PIECENROW {length}\n") # 使用原始长度 + + f.write(f"PIECENCOL {width}\n") # 使用原始宽度 + + f.write(f"ALTITUDE {alt}\n") + + f.write(f"EARTHRADIUS {earth}\n") + + f.write(f"NEARRANGE {near}\n") + + f.write(f"BASELINE 0.000000\n") + + f.write(f"BASELINEANGLE_DEG 0.000000\n") + + f.write(f"TRANSMITMODE REPEATPASS\n") + + f.write(f"DR {dr}\n") + + f.write(f"DA {da}\n") + + f.write(f"RANGERES {rangeres}\n") + + f.write(f"AZRES {azres}\n") + + f.write(f"LAMBDA 0.0556\n") + + f.write(f"NLOOKSRANGE {nrange}\n") + + f.write(f"NLOOKSAZ {nazi}\n") + + f.write(f"NLOOKSOTHER 1\n") + + f.write(f"NCORRLOOKS {ncor}\n") + + f.write(f"\n") + + f.write(f"CONNCOMPFILE {rootname}.byt\n") + + f.write(f"MAXNCOMPS 32\n") + + f.write(f"\n") + + f.write(f"INFILEFORMAT COMPLEX_DATA\n") + + f.write(f"OUTFILEFORMAT ALT_LINE_DATA\n") + + f.write(f"CORRFILEFORMAT FLOAT_DATA\n") + + f.write(f"VERBOSE FALSE\n") + + + + # Swap bytes for input files + + run_command(f"swap_bytes {int_file} {rootname}.int 4") + + run_command(f"swap_bytes {cor_file} {rootname}.cor 4") + + + + # Run SNAPHU + + run_command(f"snaphu -f {conf_file}") + + + + # Post-processing steps + + xmin = "0" + + ymin = "0" + + xmax = width + + ymax = length + + + + # 获取SNAPHU输出文件的尺寸 + + unwrapped_file = f"{rootname}.unw" + new_width = str(int(xmax)-int(xmin)) + + # 计算填充量 + + npad_bottom = str(int(length) - int(ymax)) + + npad_right = str(int(width) - int(xmax)) + + + + print(f"Original dimensions: {width}x{length}") + + print(f"SNAPHU output dimensions: {new_width}x{new_width}") + + print(f"Padding: bottom={npad_bottom}, right={npad_right}") + + + + # Zero padding + + unw_zero_file = rootname + "_zeropad.unw" + + mask_zero_file = rootname + "_mask_zeropad.unw" + + mask_zero_file_msk = rootname + "_mask_zeropad.msk" + + tobegeocodedm = rootname + "_masked.unw" + + unw_file = rootname + "_msk.unw" + + + + run_command(f"zeropad_msk {rootname}.unw {width} {ymin} {xmin} {npad_right} {npad_bottom} {unw_zero_file} rmg") + + + + # Phase and magnitude processing + + run_command(f"rmg2mag_phs {unw_zero_file} /dev/null phs {width}") + + run_command(f"swap_bytes phs phs4 4") + + run_command(f"cpx2mag_phs {rootname}.int pwr /dev/null {width}") + + run_command(f"mag_phs2rmg pwr phs {unw_zero_file} {width}") + + + + # Mask processing + + run_command(f"zeropad_msk {rootname}.byt {width} {ymin} {xmin} {npad_right} {npad_bottom} {mask_zero_file_msk} msk") + + run_command(f"cpx2mag_phs {rootname}.int pwr phs {width}") + + run_command(f"mag_phs2rmg pwr {mask_zero_file_msk} {mask_zero_file} {width}") + + + + # Combine results + + run_command(f"rmg2mag_phs {unw_zero_file} pwr phs1 {width}") + + run_command(f"rmg2mag_phs {mask_zero_file} /dev/null phs2 {width}") + + run_command(f"add_phs phs1 phs2 phs3 {width} {length} 0 1") + + run_command(f"add_phs pwr phs2 pwr3 {width} {length} 0 1") + + run_command(f"mag_phs2rmg pwr3 phs3 {tobegeocodedm} {width}") + + run_command(f"swap_bytes phs3 {unw_file} 4") + + + + # Clean up temporary files + + run_command("rm -rf pwr phs phs1 phs2 phs3 pwr3") + + run_command(f"mv phs4 {unw_zero_file}") + + + + # Remove copied files + + os.remove(Mamp) + + os.remove(Samp) + + + + print("Unwrapping with SNAPHU is done!") + + sys.exit(0) + + + +if __name__ == '__main__': + + main(sys.argv[:]) diff --git a/third_party/PyINT/pyint/unzip_s1_all.py b/third_party/PyINT/pyint/unzip_s1_all.py new file mode 100644 index 0000000..bb8c162 --- /dev/null +++ b/third_party/PyINT/pyint/unzip_s1_all.py @@ -0,0 +1,101 @@ +#! /usr/bin/env python +################################################################# +### This program is part of PyINT v2.1 ### +### Copy Right (c): 2017-2019, Yunmeng Cao ### +### Author: Yunmeng Cao ### +### Contact : ymcmrs@gmail.com ### +################################################################# + +import numpy as np +import os +import sys +import getopt +import time +import glob +import argparse + +import subprocess +from pyint import _utils as ut + +def work(data0): + cmd = data0[0] + err_txt = data0[1] + p = subprocess.run(cmd, shell=False,stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout = p.stdout + stderr = p.stderr + + if type(stderr) == bytes: + aa=stderr.decode("utf-8") + else: + aa = stderr + + if aa: + str0 = cmd[0] + ' ' + cmd[1] + ' ' + cmd[2] + ' ' + cmd[3] + '\n' + #print(aa) + with open(err_txt, 'a') as f: + f.write(str0) + f.write(aa) + f.write('\n') + + return +######################################################################### + +INTRODUCTION = ''' +------------------------------------------------------------------- + Unzip Sentinel-1 raw dataset for one project. + [unzip does not need much CPU, so we can use parallel processing] + +''' + +EXAMPLE = ''' + Usage: + unzip_s1_all.py projectName + unzip_s1_all.py projectName --parallel 4 +------------------------------------------------------------------- +''' + + +def cmdLineParse(): + parser = argparse.ArgumentParser(description='Unzip Sentinel-1 raw dataset for one project.',\ + formatter_class=argparse.RawTextHelpFormatter,\ + epilog=INTRODUCTION+'\n'+EXAMPLE) + + parser.add_argument('projectName',help='projectName for processing.') + parser.add_argument('--parallel', dest='parallelNumb', type=int, default=1, help='Enable parallel processing and Specify the number of processors.') + + inps = parser.parse_args() + return inps + + +def main(argv): + start_time = time.time() + inps = cmdLineParse() + projectName = inps.projectName + scratchDir = os.getenv('SCRATCHDIR') + projectDir = scratchDir + '/' + projectName + downDir = scratchDir + '/' + projectName + "/DOWNLOAD" + raw_file_list = glob.glob(downDir + '/S1*.zip') + + slc_dir = scratchDir + '/' + projectName + '/SLC' + if not os.path.isdir(slc_dir): + os.mkdir(slc_dir) + + err_txt = scratchDir + '/' + projectName + '/unzip_s1_all.err' + if os.path.isfile(err_txt): os.remove(err_txt) + + data_para = [] + for i in range(len(raw_file_list)): + cmd0 = ['unzip',raw_file_list[i],'-d',downDir] + data0 = [cmd0,err_txt] + data_para.append(data0) + + ut.parallel_process(data_para, work, n_jobs=inps.parallelNumb, use_kwargs=False) + os.chdir(downDir) + print("Unzip Sentinel-1 raw dataset for project %s is done! " % projectName) + ut.print_process_time(start_time, time.time()) + + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[:]) + \ No newline at end of file diff --git a/third_party/PyINT/pyint/utm2ll b/third_party/PyINT/pyint/utm2ll new file mode 100644 index 0000000..722d337 --- /dev/null +++ b/third_party/PyINT/pyint/utm2ll @@ -0,0 +1,67 @@ +#!/bin/csh -f +# +#convet utm coord. to lon and lat +# +if ( $#argv < 5) then +more <<EOF + +utm2ll is a C-shell script which convets utm coord. to lon and lat + +Usage: utm2ll zone x0 y0 dx dy +Where zone is the conventional zone number=int(lon/6)+31 + x0,y0 is the origin of UTM projection at easting and northing(meter) + dx,dy is the offset from origin at meter unit + +Example: utm2ll 45 311072.36 3542183.56 27000 62000 + origin 85 32(lon,lat) + + Qiao Xuejun 2008/2/20 +EOF +exit +endif + +set zone = $argv[1] +set x0 = $argv[2] +set y0 = $argv[3] +set dx = $argv[4] +set dy = $argv[5] +set pi = 3.141592653589793 +set a = 6378137.00 +set b = 6356752.3142 +#echo $a $b +set k0 = 0.9996 +set x = `echo $x0 $dx |awk '{print 500000-($1+$2)}'` +set y = `echo $y0 $dy |awk '{print $1+$2}'` +set e = `echo $a $b |awk '{print sqrt(1-($2*$2)/($1*$1))}'` +set ep2 = `echo $e $a $b |awk '{print ($1*$2/$3)*($1*$2/$3)}'` +set sin1 = `echo $pi |awk '{print $1/(180*60*60)}'` +set m = `echo $y $k0|awk '{print $1/$2}'` +set mu = `echo $m $a $e|awk '{print $1/($2*(1-$3*$3/4-3*$3*$3*$3*$3/64-5*$3*$3*$3*$3*$3*$3/256))}'` +set e1 = `echo $e |awk '{print (1-sqrt(1-$1*$1))/ (1+sqrt(1-$1*$1))}'` +set j1 = `echo $e1 |awk '{print 3*$1/2-27*$1*$1*$1/32}'` +set j2 = `echo $e1 |awk '{print 21*$1*$1/16-55*$1*$1*$1*$1/32}'` +set j3 = `echo $e1 |awk '{print 151*$1*$1*$1/96}'` +set j4 = `echo $e1 |awk '{print 1097*$1*$1*$1*$1/512}'` +set fp = `echo $mu $j1 $j2 $j3 $j4|awk '{print $1+$2*sin(2*$1)+$3*sin(4*$1)+$4*sin(6*$1)+$5*sin(8*$1)}'` +set c1 = `echo $ep2 $fp|awk '{print $1*cos($2)*cos($2)}'` +set t1 = `echo $fp |awk '{print sin($1)*sin($1)/(cos($1)*cos($1))}'` +set r1 = `echo $a $e $fp |awk '{print $1*(1-$2*$2)/((1-$2*$2*sin($3)*sin($3))*sqrt(1-$2*$2*sin($3)*sin($3)))}'` +set n1 = `echo $a $e $fp |awk '{print $1/sqrt(1-$2*$2*sin($3)*sin($3))}'` +set d = `echo $x $n1 $k0 |awk '{print $1/($2*$3)}'` +set q1 = `echo $n1 $fp $r1|awk '{print $1*(sin($2)/cos($2))/$3}'` +set q2 = `echo $d |awk '{print $1*$1/2}'` +set q3 = `echo $t1 $c1 $ep2 $d |awk '{print (5+3*$1+10*$2-4*$2*$2-9*$3)*$4*$4*$4*$4/24 }'` +set q4 = `echo $t1 $c1 $ep2 $d |awk '{print (61+90*$1+298*$2+45*$1*$1-3*$2*$2-252*$3)*$4*$4*$4*$4*$4*$4/720 }'` +set q5 = $d +set q6 = `echo $t1 $c1 $d |awk '{print (1+2*$1+$2)*$3*$3*$3/6}'` +set q7 = `echo $c1 $t1 $ep2 $d |awk '{print (5-2*$1+28*$2-3*$1*$1+8*$3+24*$2*$2)*$4*$4*$4*$4*$4/120}'` +#echo $x $e $ep2 $sin1 $m $mu $e1 $j1 $j2 $j3 $j4 $fp $c1 $t1 $r1 $n1 $d $q1 $q2 $q3 $q4 $q5 $q6 $q7 +set dlong = `echo $q5 $q6 $q7 $fp|awk '{print ($1-$2+$3)/cos($4)}'` +set lat = `echo $fp $q1 $q2 $q3 $q4 |awk '{print $1-$2*($3-$4+$5)}'` +set lon0 = `echo $zone |awk '{print 6*$1-183}'` +set lon = `echo $lon0 $dlong $pi |awk '{print $1-$2*180/$3 }'` +set lat = `echo $lat $pi |awk '{print $1*180/$2 }'` +echo $lon $lat + + + diff --git a/third_party/PyINT/template/ensishenlongxiT134F058S1A.template b/third_party/PyINT/template/ensishenlongxiT134F058S1A.template new file mode 100644 index 0000000..24930b5 --- /dev/null +++ b/third_party/PyINT/template/ensishenlongxiT134F058S1A.template @@ -0,0 +1,117 @@ +# *********************************** PyINT Template *******************************************# +# Modifed Sep. , 2019 - YM.Cao + +# ------------------- Basic parameters ----------------------- + +DEM = /data/insar/chenwei/SCRATCHDIR/ensishenlongxiT134F058S1A/DEM/srtm.dem +# if not provide, SRTM-1 will be downloaded and processed automatically. + +masterDate = 20180123 # [If not provided, the first date in the date_list will be selected as the master date] + +start_swath = 2 # [option (for Sentinel-1 only)] +end_swath = 3 # [option (for Sentinel-1 only)] +start_burst = 3 # [option: bursts position is based on the master image (for Sentinel-1 only)] +end_burst = 6 # [option: bursts position is based on the master image (for Sentinel-1 only)] + +range_looks = 20 # multi-look numbers in range direction +azimuth_looks = 4 # multi-look numbers in azimuth direction + +# ------------------- download data ----------------------- + +sensor = Sentinel-1A # Envisat, ERS-1/2, .. please check SSARA for details +track = 128 # track number of the SAR image +frame = 89 # frame number of the SAR image [ defined according to ASF ] +start_time = 2017-11-01 # +end_time = 2109-01-01 # + +# ------------------- basic parameters for interferometry ----------------------- + +dem_lat_ovr = 1 # as to SRTM-1, 0.5 for 60 m, 2 for 15 m +dem_lon_ovr = 1 + +Igram_Spsflg = 1 # Implement range spectral filtering +rbw_min = 1 # minimum range bandwidth fraction (0.1 --> 1.0)(default: - 0.250) + +rwin4cor = 256 # range window length for coregistration +azwin4cor = 256 # azimuth window length for coregistration +rsample4cor = 32 # range samples used for fitting the coregistration parameters +azsample4cor = 32 # azimuth samples used for fitting the coregistration parameters + +thresh4cor = 0.3 # 2016 GAMMA or higher version, for 2015 GAMMA or lower version should be SNR + +Igram_Cor_rwin = 5 # range window length for cc_wave coherence estimation +Igram_Cor_awin = 5 # azimuth window length for cc_wave coherence estimation + +Igram_Cor_Win = 5 # cc estimation in adf +adf_alpha = 0.4 # adf alpha for Gold-Stein filtering + +# ------------------- sim phase ----------------------- + +Igram_Flag_TDM = N # Y for Tandem-X +Simphase_rpos = - +Simphase_azpos = - +Simphase_rwin = 256 +Simphase_azwin = 256 +Simphase_thresh = - + +# ------------------- unwrap phase ----------------------- + +mcf_triangular = 1 # triangular type of mcf [0: regular; 1: delaunay;] +unwrap_patr = 1 # unwrap patches in range direction +unwrap_pataz = 1 # unwrap patches in azimuth direction +unwrapThreshold = 0.25 # coherence threshold used for unwrap +auto_unw = 1 # auto or manual to select the reference point. True:auto False: manual +make_mask = 1 # make a mask based on the cor file to mask the lower field 0: don't mask 1: make a mask +init_flag = 1 # flag to set phase at reference point. 0: use initial point phase value (default) 1: set phase to 0.0 at initial point +r_refer = 396 #phase reference point range offset (default(-): roff) +a_refer = 537 # phase reference point azimuth offset (default(-): loff) +# ------------------- geocode ----------------------- + +geo_interp = 0 # [0: nearest; 1: bicubic spline] + +#-------------------- trans phase to los displacement -------------- + +satelite = S1A # the value of satelite must be one in the list['S1A', 'CSK','TSX','ALSO2','ALOS','ENVISAT'] + +# ------------------- select interferometric pairs -------------- + +network_method = sbas # sbas, sequential, delaunay, stars +endDate = 20210809 # date before this date will be excluded +startDate = 20170320 # date after this date will be excluded +conNumb = 2 # connect number for sequential +max_tb = 100 # temporal baseline threshold in days +max_sb = 100 # spatial baseline threshold in meters +#exclude_date = # exclude the date which will not be used + +# ------------------- time-series process -------------- + +download_data = 0 # [0: skip; 1: process] +down_parallel = 0 # multi-processor number used for downloading + +raw2slc_all = 0 # i.e., download2slc [0: skip; 1: process] +raw2slc_all_parallel = 4 # multi-processor number used + +extract_burst_all = 0 # for TOPS SLC only [0: skip; 1: process] +extract_all_parallel = 1 # multi-processor number used + +coreg_all = 0 # do coregister [0: skip; 1: process] +coreg_all_parallel = 4 # multi-processor number used [4 or 8] + +select_pairs = 0 # if 0: skip, and use the ifgram_list.txt under PROJECTNAME folder + +diff_all = 0 # generate differential Ifg [0: skip; 1: process] +diff_all_parallel = 4 # multi-processor number used [4 or 8] + +unwrap_all = 1 # unwrap Ifg [0: skip; 1: process] +unwrap_all_parallel = 8 # multi-processor number used [8 or 10] + +geocode_all = 1 # geocode diff,unw products [0: skip; 1: process] +geocode_all_parallel = 4 # multi-processor number used + +load_data = 0 # loading data for mintpy processing [0: skip; 1: process] + +# ------------------- Good Luck -------------- +DEM= /data/insar/chenwei/SCRATCHDIR/ensishenlongxiT134F058S1A/DEM/ensishenlongxiT134F058S1A.dem +DEM= /data/insar/chenwei/SCRATCHDIR/DEM/ensishenlongxiT134F058S1A/ensishenlongxiT134F058S1A.dem +DEM= /data/insar/chenwei/SCRATCHDIR/DEM/ensishenlongxiT134F058S1A/ensishenlongxiT134F058S1A.dem +DEM= /data/insar/chenwei/SCRATCHDIR/DEM/ensishenlongxiT134F058S1A/ensishenlongxiT134F058S1A.dem diff --git a/third_party/PyINT/template/pyint.template b/third_party/PyINT/template/pyint.template new file mode 100644 index 0000000..6459886 --- /dev/null +++ b/third_party/PyINT/template/pyint.template @@ -0,0 +1,204 @@ +# *********************************** PyINT Template *******************************************# +# Template for PyINT v2.1 +# Please fill in the parameters according to your project requirements + +# ========================= Basic parameters ======================== + +# DEM file path (if not provided, SRTM will be downloaded automatically from OpenTopography) +#DEM = /path/to/your/dem.dem + +# Master date for interferometric processing +masterDate = YYYYMMDD # Master date in YYYYMMDD format + +# Swath and burst parameters (for Sentinel-1 TOPS mode only) +start_swath = 1 # Starting swath number +end_swath = 3 # Ending swath number +start_burst = 1 # Starting burst position (relative to master image) +end_burst = 9 # Ending burst position (relative to master image) + +# Multilooking parameters +range_looks = 20 # Multi-look numbers in range direction +azimuth_looks = 4 # Multi-look numbers in azimuth direction + +# ========================= Download Data ======================== + +# Data source and search parameters +Data_Hub = ASF # Data download source: SSARA, Scihub, ASF +sensor = Sentinel-1A # Envisat, ERS-1/2, Sentinel-1A/B, etc. +track = 128 # Track number of the SAR image +frame = 89 # Frame number [defined according to ASF] +start_time = YYYY-MM-DD # Start date for data search +end_time = YYYY-MM-DD # End date for data search + +# ASF download specific parameters +asf_platform = Sentinel-1A # Platform: Sentinel-1A, Sentinel-1B +asf_beam_mode = IW # Beam mode: IW (Interferometric Wide) +asf_polarization = VV # Polarization: VV, VH, HH, HV +asf_flight_direction = ascending # Flight direction: ascending, descending +asf_bbox = W,S,E,N # Bounding box: West,South,East,North + +# ========================= DEM Generation ======================== + +# DEM download parameters (for OpenTopography) +opentopo_api_key = YOUR_API_KEY # OpenTopography API key +opentopo_dem_type = SRTMGL1 # DEM type: SRTMGL1(30m), SRTMGL3(90m), NASADEM(30m), COP30, COP90 +opentopo_auto_tile = 1 # Auto-tile large regions (recommended: 1) + +# DEM processing parameters +dem_lat_ovr = 1 # Latitude oversampling (as to SRTM-1: 0.5 for 60m, 2 for 15m) +dem_lon_ovr = 1 # Longitude oversampling + +# ========================= Interferometry Parameters ======================== + +# Spectral filtering +Igram_Spsflg = 1 # Implement range spectral filtering +rbw_min = 1 # Minimum range bandwidth fraction (0.1 --> 1.0) + +# Coregistration parameters +rwin4cor = 256 # Range window length for coregistration +azwin4cor = 256 # Azimuth window length for coregistration +rsample4cor = 32 # Range samples for fitting coregistration parameters +azsample4cor = 32 # Azimuth samples for fitting coregistration parameters +thresh4cor = 0.3 # Coherence threshold (GAMMA 2016+, SNR for older versions) + +# Coherence estimation +Igram_Cor_rwin = 5 # Range window length for cc_wave coherence estimation +Igram_Cor_awin = 5 # Azimuth window length for cc_wave coherence estimation +Igram_Cor_Win = 5 # Window for cc estimation in adf +adf_alpha = 0.4 # Alpha for Gold-Stein filtering + +# ========================= Simulation Phase ======================== + +Igram_Flag_TDM = N # Y for Tandem-X +Simphase_rpos = - +Simphase_azpos = - +Simphase_rwin = 256 +Simphase_azwin = 256 +Simphase_thresh = - + +# ========================= Unwrapping ======================== + +mcf_triangular = 1 # Triangular type of mcf [0: regular; 1: delaunay] +unwrap_patr = 1 # Unwrap patches in range direction +unwrap_pataz = 1 # Unwrap patches in azimuth direction +unwrapThreshold = 0.05 # Coherence threshold for unwrapping +auto_unw = 1 # Auto or manual reference point [1: auto; 0: manual] +make_mask = 0 # Make mask based on coherence [0: don't mask; 1: make mask] +init_flag = 1 # Flag to set phase at reference point [0: use initial; 1: set to 0.0] +r_refer = - # Phase reference point range offset +a_refer = - # Phase reference point azimuth offset + +# ========================= Geocoding ======================== + +geo_interp = 0 # Interpolation method [0: nearest; 1: bicubic spline] + +# ========================= Satellite Configuration ======================== + +satelite = S1A # Satellite: S1A, S1B, CSK, TSX, ALOS2, ALOS, ENVISAT + +# ========================= Pair Selection ======================== + +network_method = sbas # Network method: sbas, sequential, delaunay, stars +endDate = YYYYMMDD # Exclude dates after this date +startDate = YYYYMMDD # Exclude dates before this date +conNumb = 15 # Connect number for sequential network +max_tb = 365 # Maximum temporal baseline (days) +max_sb = 100 # Maximum spatial baseline (meters) +min_tb = 1 # Minimum temporal baseline (days) +min_sb = 0 # Minimum spatial baseline (meters) + +# ========================= Phase Gradient (Optional) ======================== + +dataformat = cpxfloat32 # Data format of interferogram +scale = 10000 # Original phase/scale +step_windows = 1 # Step length for spatial phase gradient +filter_wins = [7, 7] # Windows of median filter +directions = ['grad_east', 'grad_north', 'grad_northeast', 'grad_southeast'] + +# Mapping parameters for phase gradient +in_low_high = [0.001, 1] # Input range for mapping +out_low_high = [0.001, 1] # Output range for mapping +gamma = 0.8 # Gamma for curve shape +choose_linear = 'n' # Linear or nonlinear mapping ['y': linear; 'n': nonlinear] + +# ========================= Hyp3 Format Conversion (NEW) ======================== + +# Convert GAMMA outputs to Hyp3-compatible GeoTIFF format +hyp3format = 0 # Enable Hyp3 format conversion [0: skip; 1: process] +hyp3format_parallel = 4 # Number of parallel processors for Hyp3 conversion +hyp3_output_dir = - # Output directory for Hyp3 products [default: projectName/Hyp3Products/] + +# ========================= Phase Bias Correction (NEW) ======================== + +# Apply phase bias correction based on loop closures algorithm +# Reference: 10.1016/j.remote.2022.100013 + +phasebias = 0 # Enable phase bias correction [0: skip; 1: process] +phasebias_interval = 12 # Temporal baseline in days [6, 12, or 24] +phasebias_nlook = 10 # Number of looks for multilooking +phasebias_num_a = 2 # Number of calibration parameters to estimate +phasebias_estimate_an = 0 # Estimate an from data [0: use defaults; 1: estimate from loop closures] +phasebias_max_con = 5 # Maximum number of connections + +# Default an values (used if phasebias_estimate_an = 0) +a1_6_day = 0.50 +a2_6_day = 0.36 +a3_6_day = 0.299 +a4_6_day = 0.2476 + +a1_12_day = 0.494 +a2_12_day = 0.297 +a3_12_day = 0.24 +a4_12_day = 0.22 + +a1_24_day = 0.48 +a2_24_day = 0.28 +a3_24_day = 0.22 +a4_24_day = 0.20 + +phasebias_start = - # Start date YYYYMMDD [default: auto-detect] +phasebias_end = - # End date YYYYMMDD [default: auto-detect] + +# ========================= Atmospheric Correction ======================== + +# GACOS tropospheric correction +gacos_correction = 0 # Enable GACOS correction [0: skip; 1: process] +gacos_parallel = 4 # Number of parallel processors + +# Ionospheric correction +ionosphere_correction = 0 # Enable ionospheric correction [0: skip; 1: process] +ionosphere_parallel = 4 # Number of parallel processors + +# ========================= Processing Workflow Control ======================== + +download_data = 0 # Enable data download [0: skip; 1: process] +down_parallel = 4 # Number of parallel processors for download + +raw2slc_all = 1 # Convert raw to SLC [0: skip; 1: process] +raw2slc_all_parallel = 4 # Number of parallel processors + +extract_burst_all = 1 # Extract bursts [0: skip; 1: process] +extract_all_parallel = 4 # Number of parallel processors + +coreg_all = 1 # Coregister SLCs [0: skip; 1: process] +coreg_all_parallel = 4 # Number of parallel processors + +select_pairs = 1 # Select interferometric pairs [0: skip; 1: process] + +diff_all = 1 # Generate differential interferograms [0: skip; 1: process] +diff_all_parallel = 4 # Number of parallel processors + +unwrap_all = 1 # Unwrap interferograms [0: skip; 1: process] +unwrap_all_parallel = 4 # Number of parallel processors + +geocode_all = 1 # Geocode products [0: skip; 1: process] +geocode_all_parallel = 4 # Number of parallel processors + +hyp3format_all = 0 # Convert to Hyp3 format [0: skip; 1: process] +hyp3format_all_parallel = 4 # Number of parallel processors + +phasebias_all = 0 # Apply phase bias correction [0: skip; 1: process] + +load_data = 0 # Load data for MintPy processing [0: skip; 1: process] + +# ========================= Good Luck ======================================= diff --git a/third_party/PyINT/template/shanghaiT171F128S1A.template b/third_party/PyINT/template/shanghaiT171F128S1A.template new file mode 100644 index 0000000..d350dd9 --- /dev/null +++ b/third_party/PyINT/template/shanghaiT171F128S1A.template @@ -0,0 +1,204 @@ +# *********************************** PyINT Template *******************************************# +# Template for PyINT v2.1 +# Please fill in the parameters according to your project requirements + +# ========================= Basic parameters ======================== + +# DEM file path (if not provided, SRTM will be downloaded automatically from OpenTopography) +#DEM = /path/to/your/dem.dem + +# Master date for interferometric processing +masterDate = 20250711 # Master date in YYYYMMDD format + +# Swath and burst parameters (for Sentinel-1 TOPS mode only) +start_swath = 1 # Starting swath number +end_swath = 3 # Ending swath number +start_burst = 1 # Starting burst position (relative to master image) +end_burst = 9 # Ending burst position (relative to master image) + +# Multilooking parameters +range_looks = 10 # Multi-look numbers in range direction +azimuth_looks = 2 # Multi-look numbers in azimuth direction + +# ========================= Download Data ======================== + +# Data source and search parameters +Data_Hub = ASF # Data download source: SSARA, Scihub, ASF +sensor = Sentinel-1A # Envisat, ERS-1/2, Sentinel-1A/B, etc. +track = 171 # Track number of the SAR image +frame = 96 # Frame number [defined according to ASF] +start_time = 2024-11-01 # Start date for data search +end_time = 2026-02-04 # End date for data search + +# ASF download specific parameters +asf_platform = Sentinel-1A # Platform: Sentinel-1A, Sentinel-1B +asf_beam_mode = IW # Beam mode: IW (Interferometric Wide) +asf_polarization = VV # Polarization: VV, VH, HH, HV +asf_flight_direction = ascending # Flight direction: ascending, descending +asf_bbox = 120.1,30.7,120.2,30.9 # Bounding box: West,South,East,North + +# ========================= DEM Generation ======================== + +# DEM download parameters (for OpenTopography) +opentopo_api_key = 09ad77d34545607fdf5cb182b64ac64e # OpenTopography API key +opentopo_dem_type = SRTMGL1 # DEM type: SRTMGL1(30m), SRTMGL3(90m), NASADEM(30m), COP30, COP90 +opentopo_auto_tile = 1 # Auto-tile large regions (recommended: 1) + +# DEM processing parameters +dem_lat_ovr = 1 # Latitude oversampling (as to SRTM-1: 0.5 for 60m, 2 for 15m) +dem_lon_ovr = 1 # Longitude oversampling + +# ========================= Interferometry Parameters ======================== + +# Spectral filtering +Igram_Spsflg = 1 # Implement range spectral filtering +rbw_min = 1 # Minimum range bandwidth fraction (0.1 --> 1.0) + +# Coregistration parameters +rwin4cor = 256 # Range window length for coregistration +azwin4cor = 256 # Azimuth window length for coregistration +rsample4cor = 32 # Range samples for fitting coregistration parameters +azsample4cor = 32 # Azimuth samples for fitting coregistration parameters +thresh4cor = 0.3 # Coherence threshold (GAMMA 2016+, SNR for older versions) + +# Coherence estimation +Igram_Cor_rwin = 5 # Range window length for cc_wave coherence estimation +Igram_Cor_awin = 5 # Azimuth window length for cc_wave coherence estimation +Igram_Cor_Win = 5 # Window for cc estimation in adf +adf_alpha = 0.4 # Alpha for Gold-Stein filtering + +# ========================= Simulation Phase ======================== + +Igram_Flag_TDM = N # Y for Tandem-X +Simphase_rpos = - +Simphase_azpos = - +Simphase_rwin = 256 +Simphase_azwin = 256 +Simphase_thresh = - + +# ========================= Unwrapping ======================== + +mcf_triangular = 1 # Triangular type of mcf [0: regular; 1: delaunay] +unwrap_patr = 1 # Unwrap patches in range direction +unwrap_pataz = 1 # Unwrap patches in azimuth direction +unwrapThreshold = 0.05 # Coherence threshold for unwrapping +auto_unw = 1 # Auto or manual reference point [1: auto; 0: manual] +make_mask = 0 # Make mask based on coherence [0: don't mask; 1: make mask] +init_flag = 1 # Flag to set phase at reference point [0: use initial; 1: set to 0.0] +r_refer = - # Phase reference point range offset +a_refer = - # Phase reference point azimuth offset + +# ========================= Geocoding ======================== + +geo_interp = 0 # Interpolation method [0: nearest; 1: bicubic spline] + +# ========================= Satellite Configuration ======================== + +satelite = S1A # Satellite: S1A, S1B, CSK, TSX, ALOS2, ALOS, ENVISAT + +# ========================= Pair Selection ======================== + +network_method = sbas # Network method: sbas, sequential, delaunay, stars +endDate = YYYYMMDD # Exclude dates after this date +startDate = YYYYMMDD # Exclude dates before this date +conNumb = 15 # Connect number for sequential network +max_tb = 365 # Maximum temporal baseline (days) +max_sb = 100 # Maximum spatial baseline (meters) +min_tb = 1 # Minimum temporal baseline (days) +min_sb = 0 # Minimum spatial baseline (meters) + +# ========================= Phase Gradient (Optional) ======================== + +dataformat = cpxfloat32 # Data format of interferogram +scale = 10000 # Original phase/scale +step_windows = 1 # Step length for spatial phase gradient +filter_wins = [7, 7] # Windows of median filter +directions = ['grad_east', 'grad_north', 'grad_northeast', 'grad_southeast'] + +# Mapping parameters for phase gradient +in_low_high = [0.001, 1] # Input range for mapping +out_low_high = [0.001, 1] # Output range for mapping +gamma = 0.8 # Gamma for curve shape +choose_linear = 'n' # Linear or nonlinear mapping ['y': linear; 'n': nonlinear] + +# ========================= Hyp3 Format Conversion (NEW) ======================== + +# Convert GAMMA outputs to Hyp3-compatible GeoTIFF format +hyp3format = 0 # Enable Hyp3 format conversion [0: skip; 1: process] +hyp3format_parallel = 4 # Number of parallel processors for Hyp3 conversion +hyp3_output_dir = - # Output directory for Hyp3 products [default: projectName/Hyp3Products/] + +# ========================= Phase Bias Correction (NEW) ======================== + +# Apply phase bias correction based on loop closures algorithm +# Reference: 10.1016/j.remote.2022.100013 + +phasebias = 0 # Enable phase bias correction [0: skip; 1: process] +phasebias_interval = 12 # Temporal baseline in days [6, 12, or 24] +phasebias_nlook = 10 # Number of looks for multilooking +phasebias_num_a = 2 # Number of calibration parameters to estimate +phasebias_estimate_an = 0 # Estimate an from data [0: use defaults; 1: estimate from loop closures] +phasebias_max_con = 5 # Maximum number of connections + +# Default an values (used if phasebias_estimate_an = 0) +a1_6_day = 0.50 +a2_6_day = 0.36 +a3_6_day = 0.299 +a4_6_day = 0.2476 + +a1_12_day = 0.494 +a2_12_day = 0.297 +a3_12_day = 0.24 +a4_12_day = 0.22 + +a1_24_day = 0.48 +a2_24_day = 0.28 +a3_24_day = 0.22 +a4_24_day = 0.20 + +phasebias_start = - # Start date YYYYMMDD [default: auto-detect] +phasebias_end = - # End date YYYYMMDD [default: auto-detect] + +# ========================= Atmospheric Correction ======================== + +# GACOS tropospheric correction +gacos_correction = 0 # Enable GACOS correction [0: skip; 1: process] +gacos_parallel = 4 # Number of parallel processors + +# Ionospheric correction +ionosphere_correction = 0 # Enable ionospheric correction [0: skip; 1: process] +ionosphere_parallel = 4 # Number of parallel processors + +# ========================= Processing Workflow Control ======================== + +download_data = 1 # Enable data download [0: skip; 1: process] +down_parallel = 4 # Number of parallel processors for download + +raw2slc_all = 0 # Convert raw to SLC [0: skip; 1: process] +raw2slc_all_parallel = 4 # Number of parallel processors + +extract_burst_all = 0 # Extract bursts [0: skip; 1: process] +extract_all_parallel = 4 # Number of parallel processors + +coreg_all = 0 # Coregister SLCs [0: skip; 1: process] +coreg_all_parallel = 4 # Number of parallel processors + +select_pairs = 0 # Select interferometric pairs [0: skip; 1: process] + +diff_all = 0 # Generate differential interferograms [0: skip; 1: process] +diff_all_parallel = 4 # Number of parallel processors + +unwrap_all = 0 # Unwrap interferograms [0: skip; 1: process] +unwrap_all_parallel = 4 # Number of parallel processors + +geocode_all = 0 # Geocode products [0: skip; 1: process] +geocode_all_parallel = 4 # Number of parallel processors + +hyp3format_all = 0 # Convert to Hyp3 format [0: skip; 1: process] +hyp3format_all_parallel = 4 # Number of parallel processors + +phasebias_all = 0 # Apply phase bias correction [0: skip; 1: process] + +load_data = 0 # Load data for MintPy processing [0: skip; 1: process] + +# ========================= Good Luck =======================================