Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor local InSAR asset and production workflows

Browse Source
This commit is contained in:
Harmon
2026-06-21 12:30:21 +08:00
parent 65a8cc4eac
commit 71c524967c
88 changed files with 11165 additions and 3017 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/prepare_landsar_dem_int16.py
+106 -30
View File
@@ -2,16 +2,21 @@
"""
Prepare reusable LandSAR DEM GeoTIFFs.
The script converts large DEM rasters to uncompressed Int16 GeoTIFFs with a
stable nodata value. It streams data by windows, so it can process the 10 m
Heilongjiang DEM and the COPDEM China DEM without loading them into memory.
The script has two explicit modes:
1. Convert a large DEM raster once to an uncompressed Int16 GeoTIFF.
2. Crop-copy a regional DEM from that prepared Int16 GeoTIFF without changing
pixel values.
Both modes stream data by windows, so they do not load large DEMs into memory.
"""
from __future__ import annotations
import argparse
import json
import math
import os
import sys
import tempfile
from pathlib import Path
from typing import Iterable, Optional
@@ -160,21 +165,34 @@ def convert_dem(
*,
dem_root: Path,
output_root: Path,
target_path: Optional[Path],
bbox: Optional[tuple[float, float, float, float]],
suffix: str,
nodata: int,
block_size: int,
overwrite: bool,
dry_run: bool,
crop_only: bool,
) -> Path:
source = _source_path(source_text, dem_root)
stem = _safe_stem(source_text, source)
if suffix:
stem = f"{stem}_{suffix.strip('_')}"
target = output_root / f"{stem}_int16.tif"
if target_path:
target = target_path if target_path.is_absolute() else output_root / target_path
else:
target = output_root / f"{stem}_int16.tif"
if crop_only and not bbox:
raise ValueError("--crop-only requires --bbox because full-size crop-copy is not useful")
with rasterio.open(source) as src:
src_crs = src.crs or DEFAULT_CRS
source_dtype = str(src.dtypes[0]).lower()
if crop_only and source_dtype != "int16":
raise ValueError(
f"--crop-only requires an already prepared Int16 GeoTIFF; got dtype={src.dtypes[0]} from {source}"
)
if bbox:
window = _align_window(from_bounds(*bbox, transform=src.transform), src.width, src.height)
else:
@@ -182,17 +200,21 @@ def convert_dem(
window = Window(int(window.col_off), int(window.row_off), int(window.width), int(window.height))
transform = src.window_transform(window)
bounds = array_bounds(int(window.height), int(window.width), transform)
estimated_bytes = int(window.width) * int(window.height) * np.dtype("int16").itemsize
target_dtype = source_dtype if crop_only else "int16"
target_nodata = src.nodata if crop_only else nodata
mode = "crop-copy-int16" if crop_only else "convert-int16"
estimated_bytes = int(window.width) * int(window.height) * np.dtype(target_dtype).itemsize
print(f"Source: {source}")
print(f" mode={mode}")
print(f" driver={src.driver} dtype={src.dtypes[0]} size={src.width}x{src.height} crs={src.crs or 'EPSG:4326 assumed'}")
print(f" output window={int(window.width)}x{int(window.height)} bounds={tuple(round(v, 8) for v in bounds)}")
print(f" target={target}")
print(f" estimated raw int16 size={_format_gib(estimated_bytes)}")
print(f" estimated raw {target_dtype} size={_format_gib(estimated_bytes)}")
if dry_run:
return target
output_root.mkdir(parents=True, exist_ok=True)
target.parent.mkdir(parents=True, exist_ok=True)
if target.exists() and not overwrite:
raise FileExistsError(f"target exists; pass --overwrite to replace it: {target}")
@@ -202,10 +224,10 @@ def convert_dem(
height=int(window.height),
width=int(window.width),
count=1,
dtype="int16",
dtype=target_dtype,
crs=src_crs,
transform=transform,
nodata=nodata,
nodata=target_nodata,
compress="NONE",
tiled=True,
blockxsize=512,
@@ -216,27 +238,66 @@ def convert_dem(
profile.pop("photometric", None)
profile.pop("predictor", None)
if target.exists():
target.unlink()
temp_path: Optional[Path] = None
with rasterio.open(target, "w", **profile) as dst:
total_pixels = int(window.width) * int(window.height)
done_pixels = 0
last_percent = -1
for rel_window in _iter_windows(int(window.width), int(window.height), block_size):
src_window = Window(
window.col_off + rel_window.col_off,
window.row_off + rel_window.row_off,
rel_window.width,
rel_window.height,
)
data = src.read(1, window=src_window, masked=True)
dst.write(_convert_array(data, nodata), 1, window=rel_window)
done_pixels += int(rel_window.width) * int(rel_window.height)
percent = int(done_pixels * 100 / max(1, total_pixels))
if percent != last_percent and (percent % 5 == 0 or percent == 100):
print(f" progress={percent}%")
last_percent = percent
try:
with tempfile.NamedTemporaryFile(
prefix=f"{target.stem}.",
suffix=".tmp.tif",
dir=str(target.parent),
delete=False,
) as tmp:
temp_path = Path(tmp.name)
with rasterio.open(temp_path, "w", **profile) as dst:
total_pixels = int(window.width) * int(window.height)
done_pixels = 0
last_percent = -1
for rel_window in _iter_windows(int(window.width), int(window.height), block_size):
src_window = Window(
window.col_off + rel_window.col_off,
window.row_off + rel_window.row_off,
rel_window.width,
rel_window.height,
)
if crop_only:
data = src.read(1, window=src_window, masked=False)
else:
data = _convert_array(src.read(1, window=src_window, masked=True), nodata)
dst.write(data, 1, window=rel_window)
done_pixels += int(rel_window.width) * int(rel_window.height)
percent = int(done_pixels * 100 / max(1, total_pixels))
if percent != last_percent and (percent % 5 == 0 or percent == 100):
print(f" progress={percent}%")
last_percent = percent
os.replace(temp_path, target)
temp_path = None
manifest = target.with_suffix(target.suffix + ".json")
manifest.write_text(
json.dumps(
{
"source": str(source),
"target": str(target),
"mode": mode,
"bbox": list(bbox) if bbox else None,
"bounds": [float(value) for value in bounds],
"width": int(window.width),
"height": int(window.height),
"source_dtype": src.dtypes[0],
"target_dtype": target_dtype,
"source_nodata": src.nodata,
"target_nodata": target_nodata,
"big_tiff": True,
"compress": "NONE",
},
ensure_ascii=False,
indent=2,
),
encoding="utf-8",
)
finally:
if temp_path and temp_path.exists():
temp_path.unlink(missing_ok=True)
actual_size = target.stat().st_size if target.exists() else 0
print(f"Done: {target} ({_format_gib(actual_size)})")
@@ -245,7 +306,7 @@ def convert_dem(
def main() -> int:
parser = argparse.ArgumentParser(
description="Convert large DEMs to reusable LandSAR Int16 GeoTIFFs."
description="Prepare reusable LandSAR Int16 GeoTIFFs and regional crop copies."
)
parser.add_argument(
"--source",
@@ -258,6 +319,11 @@ def main() -> int:
)
parser.add_argument("--dem-root", default=str(DEFAULT_DEM_ROOT))
parser.add_argument("--output-root", default=str(DEFAULT_OUTPUT_ROOT))
parser.add_argument(
"--target",
default="",
help="Optional exact target path. Only valid with one --source.",
)
parser.add_argument(
"--bbox",
default="",
@@ -268,21 +334,31 @@ def main() -> int:
parser.add_argument("--block-size", type=int, default=2048)
parser.add_argument("--overwrite", action="store_true")
parser.add_argument("--dry-run", action="store_true")
parser.add_argument(
"--crop-only",
action="store_true",
help="Copy a bbox window from an already prepared Int16 GeoTIFF without value conversion.",
)
args = parser.parse_args()
sources = args.source or ["HeiLongJiang10M_DEM", "COPDEM_GLO30_China_4326_DEM"]
if args.target and len(sources) != 1:
raise ValueError("--target can only be used with exactly one --source")
bbox = _parse_bbox(args.bbox)
target_path = Path(args.target) if args.target else None
for source in sources:
convert_dem(
source,
dem_root=Path(args.dem_root),
output_root=Path(args.output_root),
target_path=target_path,
bbox=bbox,
suffix=args.suffix,
nodata=args.nodata,
block_size=args.block_size,
overwrite=args.overwrite,
dry_run=args.dry_run,
crop_only=args.crop_only,
)
return 0