chore: 合并 GF3_L1A_To_L2_pipeline 历史

This commit is contained in:
Harmon
2026-10-16 18:59:39 +08:00
41 changed files with 3082 additions and 0 deletions
+16
View File
@@ -0,0 +1,16 @@
* text=auto
.gitattributes text eol=lf
.gitignore text eol=lf
*.md text eol=lf
*.py text eol=lf
*.go text eol=lf
*.mod text eol=lf
*.toml text eol=lf
*.yml text eol=lf
*.idl text eol=lf
*.pro text eol=lf
*.cmd text eol=crlf
*.exe binary
*.sav binary
+27
View File
@@ -0,0 +1,27 @@
.venv/
__pycache__/
*.py[cod]
.pytest_cache/
.mypy_cache/
.ruff_cache/
/.tmp_tests/
logs/
GMTED2010.jp2
# Local wrapper state
gf3wrapper.json
.gf3_extract/
.gf3_runtime/
# Production data and generated outputs
/data/
/outputs/
*.tar.gz
*.tgz
*.zip
*.tif
*.tiff
*.img
*.dat
*.sml
*.hdr
+30
View File
@@ -0,0 +1,30 @@
"""Compatibility wrapper for archive extraction."""
from __future__ import annotations
import argparse
import sys
from pathlib import Path
_SRC = Path(__file__).resolve().parent / "src"
if _SRC.is_dir():
sys.path.insert(0, str(_SRC))
from gf3_l1a2l2.decompress import extract_compressed_files
def main() -> int:
parser = argparse.ArgumentParser(description="Extract GF3 archives")
parser.add_argument("input", type=Path, help="folder containing .tar.gz or .zip archives")
parser.add_argument("--overwrite", action="store_true", help="overwrite existing extracted folders")
args = parser.parse_args()
results = extract_compressed_files(args.input, overwrite=args.overwrite)
failed = [item for item in results if item.status == "failed"]
for item in results:
print(f"{item.status}\t{item.archive}\t{item.output_dir}\t{item.message}")
return 1 if failed else 0
if __name__ == "__main__":
raise SystemExit(main())
+103
View File
@@ -0,0 +1,103 @@
# GF3 L1A to L2 Pipeline
This repository contains two GF-3 L1A to L2 processing paths:
- **SARscape Go wrapper**: Windows-oriented production wrapper around ENVI/IDL Runtime and SARscape. This is the recommended path when SARscape is available.
- **Python pipeline**: GDAL/Rasterio/NumPy implementation kept as a pure Python package and CLI.
The SARscape wrapper currently has the more complete operational workflow for GF-3 scene import, multilook/filter/geocode processing, batch directory discovery, saved defaults, and progress reporting.
## Repository Layout
```text
dist/windows/ Prebuilt Windows executable
docs/ Operational documentation
src/gf3_l1a2l2/ Python package
tests/ Python unit tests and small fixtures
tools/gf3_sarscape_wrapper/ Go wrapper source and embedded SAV asset
tools/gf3_sarscape_wrapper/idl/ IDL/SARscape source scripts
```
Large GF-3 archives, extracted scenes, SARscape outputs, and local runtime files are intentionally not tracked in Git.
## Recommended: SARscape Go Wrapper
Use the prebuilt executable:
```powershell
.\dist\windows\gf3wrapper.exe `
-input "D:\GF3\L1A_BATCH" `
-output "E:\GF3\L2_SARscape" `
-dem "D:\DEM\COPDEM_GLO30_China_4326_DEM" `
-pol "HH,HV"
```
Supported input forms:
- A single `*.tar.gz` GF-3 L1A package.
- A single `*.meta.xml` scene metadata file.
- A directory containing multiple `*.tar.gz` or `*.meta.xml` scenes.
The wrapper writes one output subdirectory per scene. On successful runs it saves defaults next to the executable in `gf3wrapper.json`, so double-click interactive mode can reuse the previous input/output/DEM/polarization settings.
Batch directory runs continue after per-scene failures. If one archive is truncated, invalid, missing metadata, or fails during SARscape processing, the wrapper prints the failed scene and reason, skips that scene, and continues with the remaining scenes. At the end it prints a batch summary and returns a non-zero exit code when any scene failed.
Before processing each scene, the wrapper checks the scene output directory for completed geocoded products for the requested polarizations. Already completed scenes are skipped, so rerunning the same input/output pair only processes missing or incomplete scenes.
More details: [docs/sarscape_go_wrapper.md](docs/sarscape_go_wrapper.md)
## Python Pipeline
Use an environment that already has working GDAL/OSGeo Python bindings. On Windows, Conda/Miniforge is usually the least fragile route.
```powershell
conda env create -f environment.yml
conda activate gf3-l1a2l2
pip install -e . --no-deps
```
Run:
```powershell
gf3-l1a2l2 run --input D:\GF3\L1A_BATCH --output E:\GF3\L2_Image --dem .\GMTED2010.jp2
```
Common options:
```powershell
gf3-l1a2l2 run `
--input D:\GF3\L1A_BATCH `
--output E:\GF3\L2_Image `
--dem .\GMTED2010.jp2 `
--workers 1 `
--x-res 0.0002 `
--y-res 0.0002 `
--dst-srs EPSG:4326
```
Legacy wrappers remain available:
```powershell
python gf3_L1A_To_L2.py run --input D:\GF3\L1A_BATCH --output E:\GF3\L2_Image --dem .\GMTED2010.jp2
python Decompression.py D:\GF3\L1A_BATCH
```
## Development Checks
Python:
```powershell
python -m pytest
```
Go wrapper:
```powershell
cd .\tools\gf3_sarscape_wrapper
$env:GOTELEMETRY = "off"
go build -o ..\..\dist\windows\gf3wrapper.exe .
```
## Data Policy
Do not commit production GF-3 archives or generated products. Keep large data on NAS/storage and document the path in run notes. Only small sanitized fixtures should be added under `tests/fixtures`.
+21
View File
@@ -0,0 +1,21 @@
# Windows Distribution
`gf3wrapper.exe` is the prebuilt SARscape Go wrapper.
Run from the repository root:
```powershell
.\dist\windows\gf3wrapper.exe `
-input "D:\GF3\L1A_BATCH" `
-output "E:\GF3\L2_SARscape" `
-dem "D:\DEM\COPDEM_GLO30_China_4326_DEM" `
-pol "HH,HV"
```
The executable requires ENVI/IDL Runtime and SARscape on the target machine.
In directory mode, failed scenes are skipped and listed in a final batch summary. The wrapper continues with remaining scenes, but exits non-zero if any scene failed.
Rerunning the same input/output pair skips scenes with complete geocoded outputs for the requested polarizations and processes only missing or incomplete scenes.
After successful runs, `gf3wrapper.json` is created in this directory to store defaults. That file is local runtime state and is ignored by Git.
Binary file not shown.
@@ -0,0 +1,45 @@
# Repository Layout
```text
.
|-- dist/
| `-- windows/
| `-- gf3wrapper.exe
|-- docs/
| |-- repository_layout.md
| `-- sarscape_go_wrapper.md
|-- src/
| `-- gf3_l1a2l2/
|-- tests/
| `-- fixtures/
|-- tools/
| `-- gf3_sarscape_wrapper/
| |-- assets/
| |-- idl/
| |-- go.mod
| `-- main.go
|-- README.md
|-- environment.yml
`-- pyproject.toml
```
## What Belongs In Git
- Python package source.
- Go wrapper source.
- IDL/SARscape source scripts.
- Small embedded runtime assets needed to build the wrapper.
- Small sanitized tests and fixtures.
- Documentation and reproducible build instructions.
- Small convenience binaries when the team explicitly wants a ready-to-run Windows artifact.
## What Does Not Belong In Git
- Raw GF-3 production archives.
- Extracted scenes.
- SARscape output directories.
- Local wrapper config files such as `gf3wrapper.json`.
- Runtime folders such as `.gf3_extract` and `.gf3_runtime`.
- DEM files and other large environment-specific dependencies.
Use NAS or object storage for production data and record exact paths in run logs or issue notes.
@@ -0,0 +1,142 @@
# SARscape Go Wrapper
`gf3wrapper.exe` is a Windows command-line wrapper for the SARscape/IDL GF-3 processing workflow. It embeds `gf3_sarscape_cli.sav`, starts `idlrt.exe`, and passes scene metadata, output directory, DEM, and polarization arguments to SARscape.
This path is intended for operational runs where ENVI/IDL Runtime and SARscape are installed and licensed.
## Requirements
- Windows host.
- ENVI/IDL Runtime, default path:
`C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlrt.exe`
- SARscape available to the IDL Runtime environment.
- DEM file, for example:
`D:\DEM\COPDEM_GLO30_China_4326_DEM`
- GF-3 L1A input as `*.tar.gz`, `*.meta.xml`, or a directory containing multiple scenes.
The IDL Runtime path can be overridden with `-idlrt` or the `IDLRT_PATH` environment variable.
## Quick Start
Use the prebuilt executable:
```powershell
.\dist\windows\gf3wrapper.exe `
-input "D:\GF3\L1A_BATCH" `
-output "E:\GF3\L2_SARscape" `
-dem "D:\DEM\COPDEM_GLO30_China_4326_DEM" `
-pol "HH,HV"
```
Double-click mode is also supported. When launched without arguments, the wrapper prompts for input, output, DEM, polarizations, and IDL Runtime path. Press Enter to reuse saved defaults.
## Inputs
The wrapper supports:
- A single GF-3 archive: `GF3_....tar.gz`
- A single extracted scene metadata file: `GF3_....meta.xml`
- A directory containing many archives and/or extracted scenes
Directory mode recursively scans for `*.tar.gz`, `*.tgz`, and `*.meta.xml`. If both archive and metadata exist for the same scene name, the archive is preferred. Internal `.gf3_extract` folders are skipped.
## Batch Failure Handling
Directory runs are fault-tolerant at the scene level. If a single scene cannot be extracted or processed, the wrapper records that scene as failed, prints the input path and error reason, and continues with the next scene.
Archive read errors include the archive path, archive size when available, and the archive member being read when the failure happened. For example, a truncated `.tar.gz` reports that the archive appears incomplete and should be re-copied or re-downloaded.
At the end of the run, the wrapper prints a summary:
```text
Batch summary: 3 succeeded, 1 failed, 4 total
Failed scenes:
- GF3_... (<archive path>): <failure reason>
```
If any scene failed, the process exits with a non-zero status after the summary. This lets scheduled jobs detect that the batch needs attention while still preserving successful outputs from other scenes.
## Resume Behavior
Before extracting or processing a scene, the wrapper checks the scene output directory for completed geocoded SARscape products for every requested polarization. A product is considered complete when both the final `*_geo.sml` metadata file and its matching data file exist and are non-empty.
Already completed scenes are skipped:
```text
SKIPPED GF3_...
Input D:\GF3\L1A_BATCH\GF3_....tar.gz
Output already complete E:\GF3\L2_SARscape\GF3_...
```
This is intended for interrupted or partially failed batch runs. For example, after replacing a truncated archive in the original input directory, rerun the same command with the same output directory; scenes that already have complete HH/HV geocoded outputs are skipped, and only missing or incomplete scenes are processed.
## Outputs
Each scene is processed into its own output directory:
```text
E:\GF3\L2_SARscape\
GF3_SCENE_NAME\
.gf3_extract\ Extracted archive, when input was .tar.gz
*_slc.sml
*_ml.sml
*_filt.sml
*_geo.sml
```
The embedded SAV is materialized under:
```text
<output>\.gf3_runtime\gf3_sarscape_cli.sav
```
## Saved Defaults
After a successful run, the wrapper writes `gf3wrapper.json` next to the executable:
```json
{
"idlrt_path": "C:\\Program Files\\Harris\\ENVI56\\IDL88\\bin\\bin.x86_64\\idlrt.exe",
"dem_file": "D:\\DEM\\COPDEM_GLO30_China_4326_DEM",
"polarizations": "HH,HV",
"last_input": "D:\\GF3\\L1A_BATCH",
"last_output": "E:\\GF3\\L2_SARscape",
"keep_extracted": true
}
```
Command-line flags override saved defaults. If `-input`, `-output`, `-dem`, or `-pol` are omitted, saved values are reused when available.
## Build
From the repository root:
```powershell
cd .\tools\gf3_sarscape_wrapper
$env:GOTELEMETRY = "off"
go build -o ..\..\dist\windows\gf3wrapper.exe .
```
If Go cannot write to the default user cache directory, set local cache paths:
```powershell
$env:GOCACHE = "C:\Temp\go-build-cache"
$env:APPDATA = "C:\Temp\go-appdata"
```
## Source Files
```text
tools/gf3_sarscape_wrapper/main.go
tools/gf3_sarscape_wrapper/assets/gf3_sarscape_cli.sav
tools/gf3_sarscape_wrapper/idl/gf3_sarscape_cli.pro
tools/gf3_sarscape_wrapper/idl/gf3_sarscape_pipeline.pro
```
`assets/gf3_sarscape_cli.sav` is embedded into the Go executable at build time.
## Notes
- Keep production archives and generated products outside Git.
- Keep only small sanitized fixtures in `tests/fixtures`.
- The prebuilt executable in `dist/windows` is a convenience artifact for Windows users. Rebuild it after changing `main.go` or the embedded `.sav`.
+10
View File
@@ -0,0 +1,10 @@
name: gf3-l1a2l2
channels:
- conda-forge
dependencies:
- python=3.11
- gdal
- rasterio
- numpy
- tqdm
- pip
+16
View File
@@ -0,0 +1,16 @@
"""Compatibility wrapper for the packaged GF3 L1A to L2 pipeline."""
from __future__ import annotations
import sys
from pathlib import Path
_SRC = Path(__file__).resolve().parent / "src"
if _SRC.is_dir():
sys.path.insert(0, str(_SRC))
from gf3_l1a2l2.cli import main
if __name__ == "__main__":
raise SystemExit(main())
+24
View File
@@ -0,0 +1,24 @@
[build-system]
requires = ["setuptools>=68", "wheel"]
build-backend = "setuptools.build_meta"
[project]
name = "gf3-l1a2l2-pipeline"
version = "0.1.0"
description = "GF3 L1A to L2 processing pipeline"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"numpy>=1.22",
"rasterio>=1.3",
"tqdm>=4.64",
]
[project.scripts]
gf3-l1a2l2 = "gf3_l1a2l2.cli:main"
[tool.setuptools]
package-dir = {"" = "src"}
[tool.setuptools.packages.find]
where = ["src"]
@@ -0,0 +1,16 @@
"""GF3 L1A to L2 processing pipeline."""
__all__ = ["PipelineConfig", "run_pipeline"]
__version__ = "0.1.0"
def __getattr__(name: str):
if name == "PipelineConfig":
from gf3_l1a2l2.config import PipelineConfig
return PipelineConfig
if name == "run_pipeline":
from gf3_l1a2l2.pipeline import run_pipeline
return run_pipeline
raise AttributeError(name)
@@ -0,0 +1,5 @@
from gf3_l1a2l2.cli import main
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,88 @@
from __future__ import annotations
import logging
import math
from pathlib import Path
from typing import Iterator, Optional
import numpy as np
import rasterio
from rasterio.windows import Window
from gf3_l1a2l2.errors import ProcessingError
from gf3_l1a2l2.models import CalibrationValue
from gf3_l1a2l2.paths import output_name
def radiometric_calibrate_l1a_to_l1b(
image_path: Path | str,
output_dir: Path | str,
calibration: CalibrationValue,
*,
block_size: int,
overwrite: bool = False,
logger: Optional[logging.Logger] = None,
) -> Path:
source = Path(image_path)
out_dir = Path(output_dir)
out_dir.mkdir(parents=True, exist_ok=True)
l1b_path = out_dir / output_name(source, "L1A", "L1B")
if l1b_path.exists() and not overwrite:
if logger:
logger.info("Skip existing L1B: %s", l1b_path)
return l1b_path
if not math.isfinite(calibration.qualify_value):
raise ProcessingError(f"Invalid QualifyValue for {source.name}: {calibration.qualify_value}")
if not math.isfinite(calibration.calibration_const):
raise ProcessingError(
f"Invalid CalibrationConst for {source.name}: {calibration.calibration_const}"
)
if logger:
logger.info("Calibrating %s -> %s", source, l1b_path)
try:
with rasterio.open(source) as src:
if src.count < 2:
raise ProcessingError(f"Expected at least 2 bands in L1A image: {source}")
profile = src.profile.copy()
profile.update(
driver="GTiff",
count=1,
dtype="float32",
compress="lzw",
BIGTIFF="YES",
tiled=True,
)
with rasterio.open(l1b_path, "w", **profile) as dst:
for window in _iter_windows(src.width, src.height, block_size):
real = src.read(1, window=window, out_dtype="float32")
imag = src.read(2, window=window, out_dtype="float32")
db = _calibrate_window(real, imag, calibration)
dst.write(db, 1, window=window)
except ProcessingError:
raise
except Exception as exc:
raise ProcessingError(f"Failed to calibrate {source}") from exc
return l1b_path
def _calibrate_window(real: np.ndarray, imag: np.ndarray, calibration: CalibrationValue) -> np.ndarray:
amplitude = np.hypot(real, imag)
scaled = (amplitude / np.float32(32767.0)) * np.float32(calibration.qualify_value)
with np.errstate(divide="ignore", invalid="ignore"):
db = np.float32(20.0) * np.log10(scaled) - np.float32(calibration.calibration_const)
return db.astype("float32", copy=False)
def _iter_windows(width: int, height: int, block_size: int) -> Iterator[Window]:
for row_off in range(0, height, block_size):
window_height = min(block_size, height - row_off)
for col_off in range(0, width, block_size):
window_width = min(block_size, width - col_off)
yield Window(col_off, row_off, window_width, window_height)
+107
View File
@@ -0,0 +1,107 @@
from __future__ import annotations
import argparse
import sys
from pathlib import Path
from gf3_l1a2l2.config import PipelineConfig
from gf3_l1a2l2.decompress import extract_compressed_files
from gf3_l1a2l2.discovery import discover_scenes
from gf3_l1a2l2.errors import GF3PipelineError
def main(argv: list[str] | None = None) -> int:
parser = build_parser()
args = parser.parse_args(argv)
if not hasattr(args, "handler"):
parser.print_help()
return 2
try:
return args.handler(args)
except GF3PipelineError as exc:
print(f"ERROR: {exc}", file=sys.stderr)
return 1
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(prog="gf3-l1a2l2", description="GF3 L1A to L2 pipeline")
subparsers = parser.add_subparsers(dest="command")
run_parser = subparsers.add_parser("run", help="extract and process GF3 L1A products")
run_parser.add_argument("--input", required=True, type=Path, help="input folder containing archives or scenes")
run_parser.add_argument("--output", required=True, type=Path, help="output folder")
run_parser.add_argument("--dem", type=Path, default=None, help="DEM file for RPC orthorectification")
run_parser.add_argument("--x-res", type=float, default=0.0002, help="output x resolution")
run_parser.add_argument("--y-res", type=float, default=0.0002, help="output y resolution")
run_parser.add_argument("--dst-srs", default="EPSG:4326", help="destination spatial reference")
run_parser.add_argument("--block-size", type=int, default=2048, help="calibration block size in pixels")
run_parser.add_argument("--workers", type=int, default=1, help="number of scenes to process concurrently")
run_parser.add_argument("--skip-decompress", action="store_true", help="do not extract archives before processing")
run_parser.add_argument("--overwrite", action="store_true", help="overwrite existing extraction and output files")
run_parser.add_argument("--flat-output", action="store_true", help="write all outputs directly to output folder")
run_parser.add_argument("--fail-fast", action="store_true", help="stop on the first failed scene or polarization")
run_parser.add_argument("--log-level", default="INFO", choices=["DEBUG", "INFO", "WARNING", "ERROR"])
run_parser.set_defaults(handler=_run)
decompress_parser = subparsers.add_parser("decompress", help="extract .tar.gz and .zip archives")
decompress_parser.add_argument("input", type=Path, help="folder containing archives")
decompress_parser.add_argument("--overwrite", action="store_true", help="overwrite existing extracted folders")
decompress_parser.set_defaults(handler=_decompress)
discover_parser = subparsers.add_parser("discover", help="list discovered GF3 scenes")
discover_parser.add_argument("input", type=Path, help="input folder")
discover_parser.set_defaults(handler=_discover)
return parser
def _run(args: argparse.Namespace) -> int:
from gf3_l1a2l2.pipeline import run_pipeline
dem_path = args.dem or _default_dem_path()
config = PipelineConfig(
input_dir=args.input,
output_dir=args.output,
dem_path=dem_path,
x_res=args.x_res,
y_res=args.y_res,
dst_srs=args.dst_srs,
block_size=args.block_size,
workers=args.workers,
skip_decompress=args.skip_decompress,
overwrite=args.overwrite,
flat_output=args.flat_output,
fail_fast=args.fail_fast,
log_level=args.log_level,
)
summary = run_pipeline(config)
return 1 if summary.failed else 0
def _decompress(args: argparse.Namespace) -> int:
results = extract_compressed_files(args.input, overwrite=args.overwrite)
failed = [item for item in results if item.status == "failed"]
for item in results:
print(f"{item.status}\t{item.archive}\t{item.output_dir}\t{item.message}")
return 1 if failed else 0
def _discover(args: argparse.Namespace) -> int:
scenes = discover_scenes(args.input)
for scene in scenes:
polarizations = ",".join(scene.available_polarizations)
print(f"{scene.root}\tmetadata={scene.metadata_xml.name}\tpol={polarizations}")
return 0
def _default_dem_path() -> Path | None:
candidates = [
Path.cwd() / "GMTED2010.jp2",
Path(__file__).resolve().parents[2] / "GMTED2010.jp2",
]
for candidate in candidates:
if candidate.is_file():
return candidate
return None
@@ -0,0 +1,41 @@
from __future__ import annotations
from dataclasses import dataclass, replace
from pathlib import Path
from typing import Optional
from gf3_l1a2l2.constants import (
DEFAULT_BLOCK_SIZE,
DEFAULT_DST_SRS,
DEFAULT_X_RES,
DEFAULT_Y_RES,
)
@dataclass(frozen=True)
class PipelineConfig:
input_dir: Path
output_dir: Path
dem_path: Optional[Path] = None
x_res: float = DEFAULT_X_RES
y_res: float = DEFAULT_Y_RES
dst_srs: str = DEFAULT_DST_SRS
block_size: int = DEFAULT_BLOCK_SIZE
workers: int = 1
skip_decompress: bool = False
overwrite: bool = False
flat_output: bool = False
keep_l1b: bool = True
fail_fast: bool = False
log_level: str = "INFO"
def resolved(self) -> "PipelineConfig":
dem_path = self.dem_path.resolve() if self.dem_path else None
return replace(
self,
input_dir=self.input_dir.resolve(),
output_dir=self.output_dir.resolve(),
dem_path=dem_path,
workers=max(1, int(self.workers)),
block_size=max(128, int(self.block_size)),
)
@@ -0,0 +1,6 @@
POLARIZATIONS = ("HH", "HV", "VH", "VV")
DEFAULT_X_RES = 0.0002
DEFAULT_Y_RES = 0.0002
DEFAULT_DST_SRS = "EPSG:4326"
DEFAULT_BLOCK_SIZE = 2048
@@ -0,0 +1,112 @@
from __future__ import annotations
import logging
import shutil
import tarfile
import zipfile
from dataclasses import dataclass
from pathlib import Path
from typing import Iterable, Optional
try:
from tqdm import tqdm
except Exception: # pragma: no cover - only used when tqdm is unavailable
def tqdm(iterable: Iterable, **_: object) -> Iterable:
return iterable
from gf3_l1a2l2.errors import ProcessingError
@dataclass(frozen=True)
class ExtractionResult:
archive: Path
output_dir: Path
status: str
message: str = ""
def extract_compressed_files(
folder_path: Path | str,
*,
overwrite: bool = False,
logger: Optional[logging.Logger] = None,
) -> list[ExtractionResult]:
folder = Path(folder_path).resolve()
if not folder.is_dir():
raise ProcessingError(f"Input folder does not exist: {folder}")
archives = sorted(
path
for path in folder.iterdir()
if path.is_file()
and (path.name.lower().endswith(".tar.gz") or path.suffix.lower() == ".zip")
)
results: list[ExtractionResult] = []
if not archives:
if logger:
logger.info("No .tar.gz or .zip archives found in %s", folder)
return results
for archive in tqdm(archives, desc="Extract archives", unit="file"):
output_dir = _archive_output_dir(archive)
try:
if output_dir.exists():
if not overwrite:
results.append(ExtractionResult(archive, output_dir, "skipped", "output exists"))
if logger:
logger.info("Skip existing extraction: %s", output_dir)
continue
shutil.rmtree(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
if archive.name.lower().endswith(".tar.gz"):
_extract_tar_gz(archive, output_dir)
elif archive.suffix.lower() == ".zip":
_extract_zip(archive, output_dir)
else:
continue
results.append(ExtractionResult(archive, output_dir, "done"))
if logger:
logger.info("Extracted %s -> %s", archive, output_dir)
except Exception as exc:
results.append(ExtractionResult(archive, output_dir, "failed", str(exc)))
if logger:
logger.exception("Failed to extract %s", archive)
return results
def _archive_output_dir(archive: Path) -> Path:
if archive.name.lower().endswith(".tar.gz"):
return archive.with_name(archive.name[:-7])
return archive.with_suffix("")
def _extract_tar_gz(archive: Path, output_dir: Path) -> None:
base = output_dir.resolve()
with tarfile.open(archive, "r:gz") as tar:
members = tar.getmembers()
for member in tqdm(members, desc=f"Extract {archive.name}", leave=False, unit="file"):
if member.issym() or member.islnk():
raise ProcessingError(f"Refusing to extract link from archive: {member.name}")
_ensure_safe_target(base, member.name)
tar.extract(member, path=base)
def _extract_zip(archive: Path, output_dir: Path) -> None:
base = output_dir.resolve()
with zipfile.ZipFile(archive, "r") as zip_ref:
names = zip_ref.namelist()
for name in tqdm(names, desc=f"Extract {archive.name}", leave=False, unit="file"):
_ensure_safe_target(base, name)
zip_ref.extract(name, base)
def _ensure_safe_target(base: Path, member_name: str) -> None:
target = (base / member_name).resolve()
if target == base:
return
if base not in target.parents:
raise ProcessingError(f"Unsafe archive path: {member_name}")
@@ -0,0 +1,90 @@
from __future__ import annotations
import logging
from pathlib import Path
from typing import Iterable, Optional
from gf3_l1a2l2.errors import DiscoveryError
from gf3_l1a2l2.models import SceneFiles
from gf3_l1a2l2.paths import detect_polarization
SKIP_DIR_NAMES = {".git", ".venv", ".tmp_tests", "__pycache__", ".pytest_cache", "logs"}
def discover_scenes(input_dir: Path | str, logger: Optional[logging.Logger] = None) -> list[SceneFiles]:
root_dir = Path(input_dir).resolve()
if not root_dir.is_dir():
raise DiscoveryError(f"Input directory does not exist: {root_dir}")
scenes: list[SceneFiles] = []
for directory in _walk_directories(root_dir):
try:
files = [path for path in directory.iterdir() if path.is_file()]
except OSError as exc:
if logger:
logger.warning("Skipping unreadable directory %s: %s", directory, exc)
continue
metadata_files = sorted(path for path in files if path.name.lower().endswith(".meta.xml"))
image_files = sorted(
path
for path in files
if path.suffix.lower() in (".tif", ".tiff") and detect_polarization(path)
)
if not metadata_files or not image_files:
continue
metadata_xml = metadata_files[0]
if len(metadata_files) > 1 and logger:
logger.warning("Multiple metadata files in %s; using %s", directory, metadata_xml.name)
images = _collect_by_polarization(image_files, logger)
rpcs = _collect_by_polarization(
sorted(path for path in files if path.suffix.lower() in (".rpc", ".rpb")),
logger,
)
scenes.append(SceneFiles(root=directory, metadata_xml=metadata_xml, images=images, rpcs=rpcs))
scenes.sort(key=lambda scene: str(scene.root).lower())
if logger:
logger.info("Discovered %s scene(s) under %s", len(scenes), root_dir)
return scenes
def _walk_directories(root_dir: Path) -> Iterable[Path]:
stack = [root_dir]
while stack:
directory = stack.pop()
if directory.name in SKIP_DIR_NAMES:
continue
yield directory
try:
children = list(directory.iterdir())
except OSError:
continue
stack.extend(path for path in children if path.is_dir() and path.name not in SKIP_DIR_NAMES)
def _collect_by_polarization(
files: Iterable[Path],
logger: Optional[logging.Logger] = None,
) -> dict[str, Path]:
collected: dict[str, Path] = {}
for path in files:
polarization = detect_polarization(path)
if not polarization:
continue
if polarization in collected:
if logger:
logger.warning(
"Duplicate %s file in %s; using %s and ignoring %s",
polarization,
path.parent,
collected[polarization].name,
path.name,
)
continue
collected[polarization] = path
return collected
@@ -0,0 +1,18 @@
class GF3PipelineError(Exception):
"""Base error for pipeline failures."""
class DiscoveryError(GF3PipelineError):
"""Raised when input products cannot be discovered."""
class MetadataError(GF3PipelineError):
"""Raised when GF3 metadata cannot be parsed."""
class RpcError(GF3PipelineError):
"""Raised when RPC metadata cannot be parsed."""
class ProcessingError(GF3PipelineError):
"""Raised when image processing fails."""
@@ -0,0 +1,78 @@
from __future__ import annotations
import logging
from pathlib import Path
from typing import Optional
from gf3_l1a2l2.errors import ProcessingError
from gf3_l1a2l2.paths import output_name
from gf3_l1a2l2.rpc import read_rpc_metadata
def geometric_correction(
l1b_path: Path | str,
rpc_path: Path | str,
output_dir: Path | str,
*,
dem_path: Optional[Path],
x_res: float,
y_res: float,
dst_srs: str,
overwrite: bool = False,
logger: Optional[logging.Logger] = None,
) -> Path:
source = Path(l1b_path)
out_dir = Path(output_dir)
out_dir.mkdir(parents=True, exist_ok=True)
l2_path = out_dir / output_name(source, "L1B", "L2")
if l2_path.exists() and not overwrite:
if logger:
logger.info("Skip existing L2: %s", l2_path)
return l2_path
if dem_path and not Path(dem_path).is_file():
raise ProcessingError(f"DEM file does not exist: {dem_path}")
rpc_metadata = read_rpc_metadata(rpc_path)
try:
from osgeo import gdal
except Exception as exc:
raise ProcessingError("Cannot import osgeo.gdal. Install GDAL Python bindings.") from exc
if logger:
logger.info("Geocorrecting %s -> %s", source, l2_path)
dataset = gdal.Open(str(source), gdal.GA_Update)
if dataset is None:
raise ProcessingError(f"GDAL cannot open L1B image: {source}")
try:
dataset.SetMetadata(rpc_metadata, "RPC")
transformer_options = []
if dem_path:
transformer_options.append(f"RPC_DEM={Path(dem_path)}")
warp_options = gdal.WarpOptions(
dstSRS=dst_srs,
xRes=x_res,
yRes=y_res,
rpc=True,
multithread=True,
transformerOptions=transformer_options,
creationOptions=["COMPRESS=LZW", "TILED=YES", "BIGTIFF=YES"],
)
result = gdal.Warp(str(l2_path), dataset, options=warp_options)
if result is None:
raise ProcessingError(f"GDAL Warp failed for {source}")
result.FlushCache()
result = None
except ProcessingError:
raise
except Exception as exc:
raise ProcessingError(f"Failed to geocorrect {source}") from exc
finally:
dataset = None
return l2_path
@@ -0,0 +1,34 @@
from __future__ import annotations
import logging
from datetime import datetime
from pathlib import Path
def configure_logging(output_dir: Path, level: str = "INFO") -> logging.Logger:
output_dir.mkdir(parents=True, exist_ok=True)
log_dir = output_dir / "logs"
log_dir.mkdir(parents=True, exist_ok=True)
logger = logging.getLogger("gf3_l1a2l2")
logger.setLevel(getattr(logging, level.upper(), logging.INFO))
logger.propagate = False
logger.handlers.clear()
formatter = logging.Formatter(
fmt="%(asctime)s %(levelname)s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
console = logging.StreamHandler()
console.setFormatter(formatter)
console.setLevel(getattr(logging, level.upper(), logging.INFO))
logger.addHandler(console)
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
file_handler = logging.FileHandler(log_dir / f"gf3_l1a2l2_{timestamp}.log", encoding="utf-8")
file_handler.setFormatter(formatter)
file_handler.setLevel(logging.DEBUG)
logger.addHandler(file_handler)
return logger
@@ -0,0 +1,145 @@
from __future__ import annotations
import math
import xml.etree.ElementTree as ET
from pathlib import Path
from typing import Iterable, Optional
from gf3_l1a2l2.constants import POLARIZATIONS
from gf3_l1a2l2.errors import MetadataError
from gf3_l1a2l2.models import CalibrationMetadata, CalibrationValue
from gf3_l1a2l2.paths import detect_polarization
QUALIFY_KEYWORDS = ("qualifyvalue", "qualityvalue")
CALIBRATION_KEYWORDS = ("calibrationconst", "calibrationconstant")
def read_calibration_metadata(xml_path: Path | str) -> CalibrationMetadata:
path = Path(xml_path)
try:
root = ET.parse(path).getroot()
except ET.ParseError as exc:
raise MetadataError(f"Invalid metadata XML: {path}") from exc
except OSError as exc:
raise MetadataError(f"Cannot read metadata XML: {path}") from exc
qualify_values = _find_values(root, QUALIFY_KEYWORDS) or _legacy_values(root, (17, 13))
calibration_values = _find_values(root, CALIBRATION_KEYWORDS) or _legacy_values(root, (18, 3))
if not qualify_values:
raise MetadataError(f"Cannot find QualifyValue values in {path}")
if not calibration_values:
raise MetadataError(f"Cannot find CalibrationConst values in {path}")
values: dict[str, CalibrationValue] = {}
for polarization in POLARIZATIONS:
if polarization not in qualify_values or polarization not in calibration_values:
continue
values[polarization] = CalibrationValue(
qualify_value=qualify_values[polarization],
calibration_const=calibration_values[polarization],
)
if not values:
raise MetadataError(f"No usable calibration values found in {path}")
return CalibrationMetadata(values=values)
def _find_values(root: ET.Element, keywords: Iterable[str]) -> Optional[dict[str, float]]:
lowered = tuple(keyword.lower() for keyword in keywords)
for element in root.iter():
name = _local_name(element.tag).lower()
if not any(keyword in name for keyword in lowered):
continue
values = _values_by_child_polarization(element)
if values:
return values
values = _values_by_child_order(element)
if values:
return values
values = _values_from_text(element.text)
if values:
return values
return None
def _legacy_values(root: ET.Element, indexes: tuple[int, int]) -> Optional[dict[str, float]]:
try:
node = root
for index in indexes:
node = list(node)[index]
values = [_parse_float(child.text) for child in list(node)[:4]]
except (IndexError, TypeError, ValueError):
return None
if len(values) < 4:
return None
return dict(zip(POLARIZATIONS, values))
def _values_by_child_polarization(element: ET.Element) -> Optional[dict[str, float]]:
values: dict[str, float] = {}
for child in list(element):
polarization = _element_polarization(child)
if not polarization:
continue
try:
values[polarization] = _parse_float(child.text)
except ValueError:
continue
return values if values else None
def _values_by_child_order(element: ET.Element) -> Optional[dict[str, float]]:
numeric_values: list[float] = []
for child in list(element):
try:
numeric_values.append(_parse_float(child.text))
except ValueError:
continue
if len(numeric_values) < 4:
return None
return dict(zip(POLARIZATIONS, numeric_values[:4]))
def _values_from_text(text: Optional[str]) -> Optional[dict[str, float]]:
if not text:
return None
normalized = text.replace(",", " ").replace(";", " ")
parts = [part for part in normalized.split() if part]
if len(parts) < 4:
return None
try:
values = [_parse_float(part) for part in parts[:4]]
except ValueError:
return None
return dict(zip(POLARIZATIONS, values))
def _element_polarization(element: ET.Element) -> Optional[str]:
for value in element.attrib.values():
polarization = detect_polarization(value)
if polarization:
return polarization
return detect_polarization(_local_name(element.tag))
def _parse_float(text: Optional[str]) -> float:
if text is None:
raise ValueError("missing value")
value = text.strip().strip("=")
if not value or value.upper() == "NULL":
return math.nan
return float(value)
def _local_name(tag: str) -> str:
if "}" in tag:
return tag.rsplit("}", 1)[1]
return tag
@@ -0,0 +1,66 @@
from __future__ import annotations
from dataclasses import dataclass, field
from pathlib import Path
from typing import Mapping, Optional
@dataclass(frozen=True)
class CalibrationValue:
qualify_value: float
calibration_const: float
@dataclass(frozen=True)
class CalibrationMetadata:
values: Mapping[str, CalibrationValue]
def for_polarization(self, polarization: str) -> CalibrationValue:
return self.values[polarization]
@dataclass(frozen=True)
class SceneFiles:
root: Path
metadata_xml: Path
images: Mapping[str, Path]
rpcs: Mapping[str, Path]
@property
def name(self) -> str:
return self.root.name
@property
def available_polarizations(self) -> tuple[str, ...]:
return tuple(sorted(self.images))
@dataclass(frozen=True)
class TaskResult:
scene: str
polarization: str
status: str
l1b_path: Optional[Path] = None
l2_path: Optional[Path] = None
message: str = ""
@dataclass
class PipelineSummary:
results: list[TaskResult] = field(default_factory=list)
@property
def succeeded(self) -> int:
return sum(1 for item in self.results if item.status == "done")
@property
def skipped(self) -> int:
return sum(1 for item in self.results if item.status == "skipped")
@property
def failed(self) -> int:
return sum(1 for item in self.results if item.status == "failed")
@property
def total(self) -> int:
return len(self.results)
@@ -0,0 +1,43 @@
from __future__ import annotations
from pathlib import Path
from typing import Optional
from gf3_l1a2l2.constants import POLARIZATIONS
def detect_polarization(path_or_name: Path | str) -> Optional[str]:
name = Path(path_or_name).name.upper()
for polarization in POLARIZATIONS:
if polarization in name:
return polarization
return None
def output_name(source: Path, source_level: str, target_level: str) -> str:
name = source.name
index = name.lower().find(source_level.lower())
if index >= 0:
return name[:index] + target_level + name[index + len(source_level) :]
stem = source.stem
suffix = "".join(source.suffixes)
if suffix and stem.endswith(suffix):
stem = stem[: -len(suffix)]
return f"{stem}_{target_level}{source.suffix}"
def scene_output_dir(input_dir: Path, output_dir: Path, scene_dir: Path, flat: bool) -> Path:
if flat:
return output_dir
try:
relative = scene_dir.relative_to(input_dir)
except ValueError:
relative = Path(scene_dir.name)
if str(relative) in ("", "."):
relative = Path(scene_dir.name)
safe_name = "_".join(part for part in relative.parts if part not in ("", "."))
return output_dir / safe_name
@@ -0,0 +1,182 @@
from __future__ import annotations
import concurrent.futures
import logging
from pathlib import Path
from typing import Iterable
from gf3_l1a2l2.calibration import radiometric_calibrate_l1a_to_l1b
from gf3_l1a2l2.config import PipelineConfig
from gf3_l1a2l2.constants import POLARIZATIONS
from gf3_l1a2l2.decompress import extract_compressed_files
from gf3_l1a2l2.discovery import discover_scenes
from gf3_l1a2l2.errors import GF3PipelineError
from gf3_l1a2l2.geocorrection import geometric_correction
from gf3_l1a2l2.log import configure_logging
from gf3_l1a2l2.metadata import read_calibration_metadata
from gf3_l1a2l2.models import PipelineSummary, SceneFiles, TaskResult
from gf3_l1a2l2.paths import scene_output_dir
def run_pipeline(config: PipelineConfig) -> PipelineSummary:
config = config.resolved()
logger = configure_logging(config.output_dir, config.log_level)
_validate_config(config)
logger.info("Input: %s", config.input_dir)
logger.info("Output: %s", config.output_dir)
if config.dem_path:
logger.info("DEM: %s", config.dem_path)
if not config.skip_decompress:
extract_compressed_files(config.input_dir, overwrite=config.overwrite, logger=logger)
scenes = discover_scenes(config.input_dir, logger=logger)
if not scenes:
logger.warning("No GF3 scenes found under %s", config.input_dir)
return PipelineSummary()
summary = PipelineSummary()
for results in _process_scenes(scenes, config, logger):
summary.results.extend(results)
logger.info(
"Pipeline finished: total=%s done=%s skipped=%s failed=%s",
summary.total,
summary.succeeded,
summary.skipped,
summary.failed,
)
return summary
def _validate_config(config: PipelineConfig) -> None:
if not config.input_dir.is_dir():
raise GF3PipelineError(f"Input directory does not exist: {config.input_dir}")
if config.dem_path and not config.dem_path.is_file():
raise GF3PipelineError(f"DEM file does not exist: {config.dem_path}")
if config.x_res <= 0 or config.y_res <= 0:
raise GF3PipelineError("x_res and y_res must be positive")
def _process_scenes(
scenes: list[SceneFiles],
config: PipelineConfig,
logger: logging.Logger,
) -> Iterable[list[TaskResult]]:
if config.workers == 1:
for scene in scenes:
yield _process_scene(scene, config, logger)
return
with concurrent.futures.ThreadPoolExecutor(max_workers=config.workers) as executor:
future_map = {
executor.submit(_process_scene, scene, config, logger): scene
for scene in scenes
}
for future in concurrent.futures.as_completed(future_map):
scene = future_map[future]
try:
yield future.result()
except Exception as exc:
logger.exception("Scene failed: %s", scene.root)
if config.fail_fast:
raise
yield [
TaskResult(
scene=scene.name,
polarization=polarization,
status="failed",
message=str(exc),
)
for polarization in scene.available_polarizations
]
def _process_scene(
scene: SceneFiles,
config: PipelineConfig,
logger: logging.Logger,
) -> list[TaskResult]:
logger.info("Processing scene: %s", scene.root)
out_dir = scene_output_dir(config.input_dir, config.output_dir, scene.root, config.flat_output)
out_dir.mkdir(parents=True, exist_ok=True)
try:
metadata = read_calibration_metadata(scene.metadata_xml)
except Exception as exc:
logger.exception("Failed to parse metadata for %s", scene.root)
if config.fail_fast:
raise
return [
TaskResult(scene=scene.name, polarization=pol, status="failed", message=str(exc))
for pol in scene.available_polarizations
]
results: list[TaskResult] = []
for polarization in POLARIZATIONS:
image_path = scene.images.get(polarization)
if not image_path:
continue
rpc_path = scene.rpcs.get(polarization)
if not rpc_path:
message = f"Missing RPC/RPB file for {polarization}"
logger.error("%s in %s", message, scene.root)
results.append(TaskResult(scene=scene.name, polarization=polarization, status="failed", message=message))
if config.fail_fast:
raise GF3PipelineError(message)
continue
if polarization not in metadata.values:
message = f"Missing calibration metadata for {polarization}"
logger.error("%s in %s", message, scene.metadata_xml)
results.append(TaskResult(scene=scene.name, polarization=polarization, status="failed", message=message))
if config.fail_fast:
raise GF3PipelineError(message)
continue
try:
calibration = metadata.for_polarization(polarization)
l1b_path = radiometric_calibrate_l1a_to_l1b(
image_path,
out_dir,
calibration,
block_size=config.block_size,
overwrite=config.overwrite,
logger=logger,
)
l2_path = geometric_correction(
l1b_path,
rpc_path,
out_dir,
dem_path=config.dem_path,
x_res=config.x_res,
y_res=config.y_res,
dst_srs=config.dst_srs,
overwrite=config.overwrite,
logger=logger,
)
results.append(
TaskResult(
scene=scene.name,
polarization=polarization,
status="done",
l1b_path=l1b_path,
l2_path=l2_path,
)
)
except Exception as exc:
logger.exception("Failed %s %s", scene.name, polarization)
results.append(
TaskResult(
scene=scene.name,
polarization=polarization,
status="failed",
message=str(exc),
)
)
if config.fail_fast:
raise
return results
@@ -0,0 +1,70 @@
from __future__ import annotations
import re
from pathlib import Path
from gf3_l1a2l2.errors import RpcError
RPC_KEY_MAP = {
"errBias": "ERR_BIAS",
"errRand": "ERR_RAND",
"lineOffset": "LINE_OFF",
"sampOffset": "SAMP_OFF",
"latOffset": "LAT_OFF",
"longOffset": "LONG_OFF",
"heightOffset": "HEIGHT_OFF",
"lineScale": "LINE_SCALE",
"sampScale": "SAMP_SCALE",
"latScale": "LAT_SCALE",
"longScale": "LONG_SCALE",
"heightScale": "HEIGHT_SCALE",
"lineNumCoef": "LINE_NUM_COEFF",
"lineDenCoef": "LINE_DEN_COEFF",
"sampNumCoef": "SAMP_NUM_COEFF",
"sampDenCoef": "SAMP_DEN_COEFF",
}
COEFFICIENT_KEYS = {
"lineNumCoef",
"lineDenCoef",
"sampNumCoef",
"sampDenCoef",
}
def read_rpc_metadata(rpc_path: Path | str) -> dict[str, str]:
path = Path(rpc_path)
try:
text = path.read_text(encoding="utf-8", errors="ignore")
except OSError as exc:
raise RpcError(f"Cannot read RPC file: {path}") from exc
metadata: dict[str, str] = {}
missing: list[str] = []
for source_key, target_key in RPC_KEY_MAP.items():
value = _find_rpc_value(text, source_key)
if value is None:
missing.append(source_key)
continue
metadata[target_key] = _normalize_rpc_value(source_key, value)
if missing:
raise RpcError(f"RPC file {path} is missing fields: {', '.join(missing)}")
return metadata
def _find_rpc_value(text: str, key: str) -> str | None:
pattern = re.compile(rf"\b{re.escape(key)}\b\s*[:=]?\s*(.*?);", re.IGNORECASE | re.DOTALL)
match = pattern.search(text)
if not match:
return None
return match.group(1)
def _normalize_rpc_value(key: str, raw_value: str) -> str:
value = raw_value.strip().strip("=")
if key in COEFFICIENT_KEYS:
value = value.replace("(", " ").replace(")", " ").replace(",", " ")
return " ".join(value.split())
+8
View File
@@ -0,0 +1,8 @@
<root>
<QualifyValue>
<HH>1.0</HH><HV>2.0</HV><VH>NULL</VH><VV>4.0</VV>
</QualifyValue>
<CalibrationConst>
<HH>10.0</HH><HV>20.0</HV><VH>30.0</VH><VV>40.0</VV>
</CalibrationConst>
</root>
+16
View File
@@ -0,0 +1,16 @@
errBias = -1;
errRand = -1;
lineOffset = 10;
sampOffset = 20;
latOffset = 30;
longOffset = 40;
heightOffset = 50;
lineScale = 60;
sampScale = 70;
latScale = 80;
longScale = 90;
heightScale = 100;
lineNumCoef = (1, 2, 3);
lineDenCoef = (4, 5, 6);
sampNumCoef = (7, 8, 9);
sampDenCoef = (10, 11, 12);
@@ -0,0 +1,19 @@
import math
import unittest
from pathlib import Path
from gf3_l1a2l2.metadata import read_calibration_metadata
class MetadataTests(unittest.TestCase):
def test_reads_named_calibration_values(self):
path = Path(__file__).parent / "fixtures" / "sample.meta.xml"
metadata = read_calibration_metadata(path)
self.assertEqual(metadata.for_polarization("HH").qualify_value, 1.0)
self.assertEqual(metadata.for_polarization("HV").calibration_const, 20.0)
self.assertTrue(math.isnan(metadata.for_polarization("VH").qualify_value))
if __name__ == "__main__":
unittest.main()
+18
View File
@@ -0,0 +1,18 @@
import unittest
from pathlib import Path
from gf3_l1a2l2.rpc import read_rpc_metadata
class RpcTests(unittest.TestCase):
def test_reads_rpc_metadata(self):
path = Path(__file__).parent / "fixtures" / "sample.rpb"
metadata = read_rpc_metadata(path)
self.assertEqual(metadata["ERR_BIAS"], "-1")
self.assertEqual(metadata["LINE_OFF"], "10")
self.assertEqual(metadata["LINE_NUM_COEFF"], "1 2 3")
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,51 @@
# GF-3 SARscape Go Wrapper
This wrapper embeds `assets/gf3_sarscape_cli.sav` and calls the local ENVI/IDL Runtime:
```text
idlrt.exe gf3_sarscape_cli.sav -args input_meta_xml out_dir dem_file HH,HV
```
It is the operational Windows path for SARscape-based GF-3 processing.
## Build
From this directory:
```powershell
$env:GOTELEMETRY = "off"
go build -o ..\..\dist\windows\gf3wrapper.exe .
```
## Run
Single archive:
```powershell
..\..\dist\windows\gf3wrapper.exe `
-input "D:\GF3\GF3_SCENE.tar.gz" `
-output "E:\GF3\L2_SARscape" `
-dem "D:\DEM\COPDEM_GLO30_China_4326_DEM" `
-pol "HH,HV"
```
Batch directory:
```powershell
..\..\dist\windows\gf3wrapper.exe `
-input "D:\GF3\L1A_BATCH" `
-output "E:\GF3\L2_SARscape"
```
If `gf3wrapper.json` exists next to the executable, omitted `-input`, `-output`, `-dem`, and `-pol` values are loaded from that file.
In batch directory mode, a failed scene does not stop the whole run. Invalid or truncated archives, missing metadata, and SARscape failures are reported with the scene input path and reason, then the wrapper continues with the next scene. A final summary lists succeeded and failed scenes, and the process exits non-zero when any scene failed.
Reruns with the same output directory skip scenes that already have complete `*_geo.sml` outputs and matching data files for the requested polarizations. This lets a repaired archive be placed back into the original input folder without reprocessing scenes that already finished.
## Notes
- The server still needs ENVI, IDL Runtime, and SARscape installed and licensed.
- `.tar.gz` inputs are extracted under each scene output directory in `.gf3_extract`.
- The embedded SAV is extracted to `<output>\.gf3_runtime\gf3_sarscape_cli.sav`.
- More operational details are in `docs/sarscape_go_wrapper.md`.
@@ -0,0 +1,3 @@
module gf3wrapper
go 1.22
@@ -0,0 +1,5 @@
.compile 'C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\gf3_sarscape_pipeline.pro'
.compile 'C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\gf3_sarscape_cli.pro'
resolve_all, skip_routines='envi', /continue_on_error
save, /routines, filename='C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\gf3_sarscape_cli.sav'
exit
@@ -0,0 +1,5 @@
.f
.compile 'C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\gf3_sarscape_pipeline.pro'
.compile 'C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\gf3_sarscape_cli.pro'
save, /routines, filename='C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\gf3_sarscape_cli.sav'
exit
@@ -0,0 +1,79 @@
function gf3_cli_split_csv, value
compile_opt idl2
text = strtrim(value, 2)
if text eq '' then return, ['HH', 'HV']
parts = ['']
first = 1
while 1 do begin
pos = strpos(text, ',')
if pos lt 0 then begin
token = strtrim(text, 2)
if first then parts = [token] else parts = [parts, token]
break
endif
token = strtrim(strmid(text, 0, pos), 2)
if first then begin
parts = [token]
first = 0
endif else begin
parts = [parts, token]
endelse
text = strtrim(strmid(text, pos + 1), 2)
endwhile
return, parts
end
pro gf3_sarscape_cli
compile_opt idl2
args = command_line_args(count=argc)
if argc lt 3 then begin
print, 'Usage: idlrt.exe gf3_sarscape_cli.sav -args input_meta_xml out_dir dem_file [polarizations]'
print, 'Example polarizations: HH,HV or HH or HV'
return
endif
input_meta_xml = args[0]
out_dir = args[1]
dem_file = args[2]
file_mkdir, out_dir
log_file = out_dir + '\gf3_sarscape_cli.log'
openw, lun, log_file, /get_lun
catch, err
if err ne 0 then begin
printf, lun, 'ERROR: ', !error_state.msg
free_lun, lun
catch, /cancel
return
endif
printf, lun, 'CLI start'
printf, lun, 'input_meta_xml=', input_meta_xml
printf, lun, 'out_dir=', out_dir
printf, lun, 'dem_file=', dem_file
if argc ge 4 then begin
polarizations = gf3_cli_split_csv(args[3])
endif else begin
polarizations = ['HH', 'HV']
endelse
printf, lun, 'polarizations=', strjoin(polarizations, ',')
e = envi(/headless)
printf, lun, 'ENVI headless initialized'
gf3_sarscape_run, $
input_meta_xml=input_meta_xml, $
out_dir=out_dir, $
dem_file=dem_file, $
polarizations=polarizations
printf, lun, 'CLI end'
free_lun, lun
end
@@ -0,0 +1,506 @@
function gf3_normalize_errmsg, value
compile_opt idl2
if n_elements(value) eq 0 then return, ''
sz = size(value, /tname)
if sz eq 'STRING' then begin
if n_elements(value) eq 1 then return, value
return, strjoin(value, ' ')
endif
if sz eq 'BYTE' then begin
if n_elements(value) eq 0 then return, ''
return, string(value)
endif
return, string(value)
end
function gf3_float_str, value
compile_opt idl2
return, string(double(value), format='(F0.1)')
end
function gf3_base_root, polarization
compile_opt idl2
common gf3_pipeline_cfg, cfg
pol = strupcase(strtrim(polarization, 2))
case pol of
'HH': return, cfg.imported_hh_root
'HV': return, cfg.imported_hv_root
else: message, 'Unsupported polarization: ' + pol
endcase
end
function gf3_stage_root, polarization, stage_name
compile_opt idl2
common gf3_pipeline_cfg, cfg
pol = strlowcase(strtrim(polarization, 2))
prefix = cfg.output_prefix
if prefix eq '' then prefix = 'gf3'
case strlowcase(strtrim(stage_name, 2)) of
'multilook': return, cfg.out_dir + '\' + prefix + '_' + pol + '_ml'
'filter': return, cfg.out_dir + '\' + prefix + '_' + pol + '_filt'
'geocode': return, cfg.out_dir + '\' + prefix + '_' + pol + '_geo'
else: message, 'Unsupported stage: ' + stage_name
endcase
end
function gf3_scene_token, meta_xml
compile_opt idl2
base = file_basename(meta_xml)
pos = strpos(strlowcase(base), '.meta.xml')
if pos ge 0 then base = strmid(base, 0, pos)
return, base
end
function gf3_imported_root_for, polarization
compile_opt idl2
common gf3_pipeline_cfg, cfg
pol = strupcase(strtrim(polarization, 2))
pattern = cfg.out_dir + '\*_' + pol + '_slc.sml'
matches = file_search(pattern, count=count)
if count eq 0 then return, ''
newest = matches[0]
newest_time = (file_info(newest)).mtime
for i = 1, count - 1 do begin
info = file_info(matches[i])
if info.mtime gt newest_time then begin
newest = matches[i]
newest_time = info.mtime
endif
endfor
return, strmid(newest, 0, strlen(newest) - 4)
end
pro gf3_refresh_import_roots
compile_opt idl2
common gf3_pipeline_cfg, cfg
hh = gf3_imported_root_for('HH')
hv = gf3_imported_root_for('HV')
if hh ne '' then cfg.imported_hh_root = hh
if hv ne '' then cfg.imported_hv_root = hv
end
pro gf3_run_module, module_name, set_params_proc, errmsg=errmsg, working_directory=working_directory
compile_opt idl2
errmsg = ''
ob = obj_new('SARscapeBatch', Module=module_name)
if ~obj_valid(ob) then begin
errmsg = 'Create object failed: ' + module_name
print, errmsg
return
endif
ob->SetParam, 'GENERAL_PARAMETERS_CMD.AVAILABLE_MEMORY_SIZE_GB', '8.0000000'
ob->SetParam, 'GENERAL_PARAMETERS_CMD.LOAD_IMAGES', 'OK'
ob->SetParam, 'GENERAL_PARAMETERS_CMD.DELETE_TEMPORARY_FILES', 'OK'
ob->SetParam, 'GENERAL_PARAMETERS_CMD.MAKE_TIFF', 'OK'
ob->SetParam, 'GENERAL_PARAMETERS_CMD.QUICK_LOOK_FORMAT', 'ql_tiff'
ob->SetParam, 'GENERAL_PARAMETERS_CMD.SARSCAPE_TRACE_LEVEL', '3.0000000'
ob->SetParam, 'GENERAL_PARAMETERS_CMD.RENAME_THE_FILE_USING_PARAMETERS_FLAG', 'OK'
ob->SetParam, 'GENERAL_PARAMETERS_CMD.INSERT_GEO_POINTS_FLAG', 'NotOK'
ob->SetParam, 'GENERAL_PARAMETERS_CMD.VERBOSE_TRACE', 'OK'
ob->SetParam, 'GENERAL_PARAMETERS_CMD.VERBOSE_STEP', 'OK'
ob->SetParam, 'GENERAL_PARAMETERS_CMD.VERBOSE_BAR', 'OK'
ob->SetParam, 'GENERAL_PARAMETERS_CMD.APPLY_CALIBRATION_CONSTANT_FLAG', 'OK'
ob->SetParam, 'GENERAL_PARAMETERS_CMD.FILL_DUMMY_DURING_IMPORT', 'OK'
call_procedure, set_params_proc, ob
ok = ob->VerifyParams(Silent=0)
if ~ok then begin
errmsg = 'VerifyParams failed: ' + module_name
print, errmsg
return
endif
if n_elements(working_directory) eq 0 then begin
ok = ob->Execute()
endif else begin
exec_errmsg = ''
ok = ob->Execute(WORKING_DIRECTORY=working_directory, ERRMSG=exec_errmsg)
errmsg = gf3_normalize_errmsg(exec_errmsg)
endelse
if ok then begin
print, 'Success: ' + module_name
endif else begin
if errmsg eq '' then begin
err_code = ''
err_text = get_SARscape_error_string('OK', ERROR_CODE=err_code)
errmsg = 'Execution failed [' + module_name + '] EC [' + err_code + ']: ' + err_text
endif
print, errmsg
endelse
end
pro gf3_pipeline_set_import_params, ob
compile_opt idl2
common gf3_pipeline_cfg, cfg
ob->SetParam, 'MAIN_BASIC_IMPORT_GAOFEN3_CMD_CMD.SARSCAPEENVIRONMENT', 'IDL_ENVI_ENV'
ob->SetParam, 'MAIN_BASIC_IMPORT_GAOFEN3_CMD_CMD.INPUT_FILE_LIST', [cfg.input_meta_xml]
ob->SetParam, 'MAIN_BASIC_IMPORT_GAOFEN3_CMD_CMD.OUTPUT_FILE_LIST', [cfg.import_root]
ob->SetParam, 'MAIN_BASIC_IMPORT_GAOFEN3_CMD_CMD.CROSS_COPOLARIZATION_FLAG', cfg.import_polarization_mode
ob->SetParam, 'MAIN_BASIC_IMPORT_GAOFEN3_CMD_CMD.DOPP_ESTIMATION_FROM_RASTER_FLAG', 'OK'
ob->SetParam, 'MAIN_BASIC_IMPORT_GAOFEN3_CMD_CMD.KEEP_ALL_LINES_FLAG', 'NotOK'
end
pro gf3_pipeline_set_multilook_params, ob
compile_opt idl2
common gf3_pipeline_cfg, cfg
common gf3_pipeline_state, state
ob->SetParam, 'MAIN_BASIC_MULTILOOKING.SARSCAPEENVIRONMENT', 'IDL_ENVI_ENV'
ob->SetParam, 'MAIN_BASIC_MULTILOOKING.INPUT_FILE_LIST', [state.current_input_root]
ob->SetParam, 'MAIN_BASIC_MULTILOOKING.OUTPUT_FILE_LIST', [state.current_output_root]
ob->SetParam, 'MAIN_BASIC_MULTILOOKING.MULTILOOK_METHOD', cfg.multilook_method
ob->SetParam, 'MAIN_BASIC_MULTILOOKING.RANGE_MULTILOOK', gf3_float_str(cfg.range_looks)
ob->SetParam, 'MAIN_BASIC_MULTILOOKING.AZIMUTH_MULTILOOK', gf3_float_str(cfg.azimuth_looks)
ob->SetParam, 'MAIN_BASIC_MULTILOOKING.GRID_SIZE_FOR_SUGGESTED_LOOKS', gf3_float_str(cfg.grid_size_for_suggested_looks)
ob->SetParam, 'MAIN_BASIC_MULTILOOKING.FILL_DUMMY_FLAG', 'OK'
ob->SetParam, 'MAIN_BASIC_MULTILOOKING.FILL_DUMMY_METHOD', 'min_image_value'
ob->SetParam, 'MAIN_BASIC_MULTILOOKING.ROWS_WINDOW_NUMBER', '3.0000000'
ob->SetParam, 'MAIN_BASIC_MULTILOOKING.COLS_WINDOW_NUMBER', '3.0000000'
end
pro gf3_pipeline_set_filter_params, ob
compile_opt idl2
common gf3_pipeline_cfg, cfg
common gf3_pipeline_state, state
ob->SetParam, 'MAIN_BASIC_DESPECKLE_CONVENTIONAL_CMD.SARSCAPEENVIRONMENT', 'IDL_ENVI_ENV'
ob->SetParam, 'MAIN_BASIC_DESPECKLE_CONVENTIONAL_CMD.INPUT_FILE_LIST', [state.current_input_root]
ob->SetParam, 'MAIN_BASIC_DESPECKLE_CONVENTIONAL_CMD.OUTPUT_FILE_LIST', [state.current_output_root]
ob->SetParam, 'MAIN_BASIC_DESPECKLE_CONVENTIONAL_CMD.FILT_TYPE', cfg.filter_type
ob->SetParam, 'MAIN_BASIC_DESPECKLE_CONVENTIONAL_CMD.EQUIVALENT_LOOKS', '-1.0000000'
ob->SetParam, 'PARAMETERS_GENERIC_FILTERS.WIN_SIZE', gf3_float_str(cfg.filter_window)
ob->SetParam, 'PARAMETERS_GENERIC_FILTERS.ROWS_WINDOW_NUMBER', gf3_float_str(cfg.filter_window)
ob->SetParam, 'PARAMETERS_GENERIC_FILTERS.COLS_WINDOW_NUMBER', gf3_float_str(cfg.filter_window)
ob->SetParam, 'PARAMETERS_GENERIC_FILTERS.WIN_MODE_SIZE', gf3_float_str(cfg.filter_window)
end
pro gf3_pipeline_set_geocode_params, ob
compile_opt idl2
common gf3_pipeline_cfg, cfg
common gf3_pipeline_state, state
ob->SetParam, 'OUT_CARTOGRAPHIC_SYSTEM.OCS_CARTOGRAPHIC_SYSTEM_SPECIFICATION', ''
ob->SetParam, 'OUT_CARTOGRAPHIC_SYSTEM.OCS_STATE', cfg.ocs_state
ob->SetParam, 'OUT_CARTOGRAPHIC_SYSTEM.OCS_HEMISPHERE', cfg.ocs_hemisphere
ob->SetParam, 'OUT_CARTOGRAPHIC_SYSTEM.OCS_PROJECTION', cfg.ocs_projection
ob->SetParam, 'OUT_CARTOGRAPHIC_SYSTEM.OCS_ELLIPSOID', 'WGS84'
ob->SetParam, 'OUT_CARTOGRAPHIC_SYSTEM.OCS_REFERENCE_HEIGHT', '0.0'
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.SARSCAPEENVIRONMENT', 'IDL_ENVI_ENV'
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.INPUT_FILE_LIST', [state.current_input_root]
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.OUTPUT_FILE_LIST', [state.current_output_root]
if cfg.dem_file ne '' then ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.DEM_FILE_NAME', cfg.dem_file
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.GEOCODE_GRID_SIZE_X', gf3_float_str(cfg.geocode_grid_x)
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.GEOCODE_GRID_SIZE_Y', gf3_float_str(cfg.geocode_grid_y)
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.GEOCODE_RESAMPLING_TYPE', cfg.geocode_resampling
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.CALIBRATION_FLAG', cfg.calibration_flag
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.GEO_SCATTERING_AREA_METHOD', 'sine_area_estimation'
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.TRUE_AREA_EXPONENT', '1.0000000'
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.MAX_VALUE_IN_CALIBRATION', '100.00000'
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.RAD_NORMALIZATION_FLAG', cfg.rad_normalization_flag
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.GEOCODE_RAD_NORM_DEG', '2.0000000'
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.GEOCODE_RAD_NORM_ANG', '-1.0000000'
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.GEOCODE_SIGMA_FLAG', cfg.output_sigma_flag
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.GEOCODE_GAMMA_FLAG', cfg.output_gamma_flag
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.GEOCODE_BETA_FLAG', cfg.output_beta_flag
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.GEOCODE_COREG_IMAGES_FLAG', 'NotOK'
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.GENERATE_LIA_FLAG', 'NotOK'
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.OUTPUT_TYPE', cfg.output_type
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.DUMMY_REMOVAL_FLAG', 'NotOK'
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.GEOCODE_BLOCK_SIZE', '30000.000'
end
pro gf3_init_default_cfg
compile_opt idl2
common gf3_pipeline_cfg, cfg
cfg = { $
input_meta_xml: 'C:\Users\Administrator\Desktop\GF3_Test\GF3_scene_20260513\GF3_SYC_FSI_051370_E124.6_N51.7_20260513_L1A_HHHV_L10007355955.meta.xml', $
work_dir: 'C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\work', $
temp_dir: 'C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\temp', $
out_dir: 'C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\out', $
output_prefix: '', $
dem_file: 'D:\DEM\COPDEM_GLO30_China_4326_DEM', $
import_root: 'C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\out\gf3_import', $
import_polarization_mode: 'ALL_POL', $
imported_hh_root: 'C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\out\gaofer3_20260513_214340051_D_HH_slc', $
imported_hv_root: 'C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\out\gaofer3_20260513_214340051_D_HV_slc', $
multilook_method: 'time_domain', $
range_looks: 2.0, $
azimuth_looks: 2.0, $
grid_size_for_suggested_looks: 15.0, $
filter_type: 'lee', $
filter_window: 5.0, $
geocode_grid_x: 15.0, $
geocode_grid_y: 15.0, $
geocode_resampling: '4th_order_cc', $
calibration_flag: 'OK', $
rad_normalization_flag: 'OK', $
output_sigma_flag: 'OK', $
output_gamma_flag: 'NotOK', $
output_beta_flag: 'NotOK', $
output_type: 'output_type_linear', $
ocs_state: 'GEO-GLOBAL', $
ocs_hemisphere: 'NONE', $
ocs_projection: 'GEO' $
}
end
pro gf3_configure, input_meta_xml=input_meta_xml, out_dir=out_dir, dem_file=dem_file, $
work_dir=work_dir, temp_dir=temp_dir, range_looks=range_looks, $
azimuth_looks=azimuth_looks, filter_type=filter_type, filter_window=filter_window, $
geocode_grid_x=geocode_grid_x, geocode_grid_y=geocode_grid_y, $
imported_hh_root=imported_hh_root, imported_hv_root=imported_hv_root
compile_opt idl2
common gf3_pipeline_cfg, cfg
gf3_init_default_cfg
if n_elements(input_meta_xml) gt 0 then cfg.input_meta_xml = input_meta_xml
if n_elements(out_dir) gt 0 then cfg.out_dir = out_dir
if n_elements(dem_file) gt 0 then cfg.dem_file = dem_file
if n_elements(work_dir) gt 0 then cfg.work_dir = work_dir
if n_elements(temp_dir) gt 0 then cfg.temp_dir = temp_dir
if n_elements(range_looks) gt 0 then cfg.range_looks = double(range_looks)
if n_elements(azimuth_looks) gt 0 then cfg.azimuth_looks = double(azimuth_looks)
if n_elements(filter_type) gt 0 then cfg.filter_type = filter_type
if n_elements(filter_window) gt 0 then cfg.filter_window = double(filter_window)
if n_elements(geocode_grid_x) gt 0 then cfg.geocode_grid_x = double(geocode_grid_x)
if n_elements(geocode_grid_y) gt 0 then cfg.geocode_grid_y = double(geocode_grid_y)
if n_elements(imported_hh_root) gt 0 then cfg.imported_hh_root = imported_hh_root
if n_elements(imported_hv_root) gt 0 then cfg.imported_hv_root = imported_hv_root
cfg.import_root = cfg.out_dir + '\gf3_import'
cfg.output_prefix = gf3_scene_token(cfg.input_meta_xml)
if n_elements(work_dir) eq 0 then cfg.work_dir = cfg.out_dir + '\work'
if n_elements(temp_dir) eq 0 then cfg.temp_dir = cfg.out_dir + '\temp'
if n_elements(imported_hh_root) eq 0 then cfg.imported_hh_root = ''
if n_elements(imported_hv_root) eq 0 then cfg.imported_hv_root = ''
end
pro gf3_prepare_dirs
compile_opt idl2
common gf3_pipeline_cfg, cfg
file_mkdir, cfg.work_dir
file_mkdir, cfg.temp_dir
file_mkdir, cfg.out_dir
end
pro gf3_start_batch
compile_opt idl2
common gf3_pipeline_cfg, cfg
SARscape_Batch_Init, Temp_Directory=cfg.temp_dir
end
pro gf3_stop_batch
compile_opt idl2
SARscape_Batch_Exit
end
pro gf3_run_import, errmsg=errmsg
compile_opt idl2
common gf3_pipeline_cfg, cfg
errmsg = ''
gf3_run_module, 'ImportGaofen3', 'gf3_pipeline_set_import_params', errmsg=errmsg, working_directory=cfg.work_dir
end
pro gf3_run_multilook, polarization, errmsg=errmsg
compile_opt idl2
common gf3_pipeline_cfg, cfg
common gf3_pipeline_state, state
state = {current_input_root: gf3_base_root(polarization), current_output_root: gf3_stage_root(polarization, 'multilook')}
errmsg = ''
gf3_run_module, 'BaseMultilooking', 'gf3_pipeline_set_multilook_params', errmsg=errmsg, working_directory=cfg.work_dir
end
pro gf3_run_filter, polarization, errmsg=errmsg
compile_opt idl2
common gf3_pipeline_cfg, cfg
common gf3_pipeline_state, state
state = {current_input_root: gf3_stage_root(polarization, 'multilook'), current_output_root: gf3_stage_root(polarization, 'filter')}
errmsg = ''
gf3_run_module, 'DespeckleConventionalSingle', 'gf3_pipeline_set_filter_params', errmsg=errmsg, working_directory=cfg.work_dir
end
pro gf3_run_geocode, polarization, errmsg=errmsg
compile_opt idl2
common gf3_pipeline_cfg, cfg
common gf3_pipeline_state, state
state = {current_input_root: gf3_stage_root(polarization, 'filter'), current_output_root: gf3_stage_root(polarization, 'geocode')}
errmsg = ''
gf3_run_module, 'BasicGeocoding', 'gf3_pipeline_set_geocode_params', errmsg=errmsg, working_directory=cfg.work_dir
end
pro gf3_run_single_pol_pipeline, polarization, do_import=do_import, errmsg=errmsg
compile_opt idl2
errmsg = ''
if keyword_set(do_import) then begin
gf3_run_import, errmsg=errmsg
if errmsg ne '' then return
gf3_refresh_import_roots
endif
gf3_run_multilook, polarization, errmsg=errmsg
if errmsg ne '' then return
gf3_run_filter, polarization, errmsg=errmsg
if errmsg ne '' then return
gf3_run_geocode, polarization, errmsg=errmsg
end
function gf3_should_run_pol, polarizations, polarization
compile_opt idl2
if n_elements(polarizations) eq 0 then return, 1
wanted = strupcase(strtrim(polarizations, 2))
pol = strupcase(strtrim(polarization, 2))
for i = 0, n_elements(wanted) - 1 do begin
if wanted[i] eq 'ALL' then return, 1
if wanted[i] eq pol then return, 1
endfor
return, 0
end
pro gf3_sarscape_run, input_meta_xml=input_meta_xml, out_dir=out_dir, dem_file=dem_file, $
polarizations=polarizations, skip_import=skip_import, work_dir=work_dir, temp_dir=temp_dir, $
range_looks=range_looks, azimuth_looks=azimuth_looks, filter_type=filter_type, $
filter_window=filter_window, geocode_grid_x=geocode_grid_x, geocode_grid_y=geocode_grid_y
compile_opt idl2
gf3_configure, input_meta_xml=input_meta_xml, out_dir=out_dir, dem_file=dem_file, $
work_dir=work_dir, temp_dir=temp_dir, range_looks=range_looks, $
azimuth_looks=azimuth_looks, filter_type=filter_type, filter_window=filter_window, $
geocode_grid_x=geocode_grid_x, geocode_grid_y=geocode_grid_y
gf3_prepare_dirs
gf3_start_batch
errmsg = ''
if ~keyword_set(skip_import) then begin
gf3_run_import, errmsg=errmsg
if errmsg eq '' then gf3_refresh_import_roots
endif else begin
gf3_refresh_import_roots
endelse
if errmsg eq '' && gf3_should_run_pol(polarizations, 'HH') then gf3_run_single_pol_pipeline, 'HH', errmsg=errmsg
if errmsg eq '' && gf3_should_run_pol(polarizations, 'HV') then gf3_run_single_pol_pipeline, 'HV', errmsg=errmsg
gf3_stop_batch
if errmsg ne '' then print, 'Pipeline stopped: ' + errmsg
end
pro gf3_sarscape_pipeline
compile_opt idl2
common gf3_pipeline_cfg, cfg
gf3_init_default_cfg
gf3_prepare_dirs
gf3_start_batch
errmsg = ''
gf3_run_import, errmsg=errmsg
if errmsg eq '' then gf3_refresh_import_roots
if errmsg eq '' then gf3_run_single_pol_pipeline, 'HH', errmsg=errmsg
if errmsg eq '' then gf3_run_single_pol_pipeline, 'HV', errmsg=errmsg
gf3_stop_batch
if errmsg ne '' then print, 'Pipeline stopped: ' + errmsg
end
pro gf3_sarscape_import_only
compile_opt idl2
gf3_init_default_cfg
gf3_prepare_dirs
gf3_start_batch
errmsg = ''
gf3_run_import, errmsg=errmsg
gf3_stop_batch
if errmsg ne '' then print, 'Import stopped: ' + errmsg
end
pro gf3_sarscape_hh_only
compile_opt idl2
gf3_init_default_cfg
gf3_prepare_dirs
gf3_start_batch
errmsg = ''
gf3_run_single_pol_pipeline, 'HH', do_import=1, errmsg=errmsg
gf3_stop_batch
if errmsg ne '' then print, 'HH pipeline stopped: ' + errmsg
end
pro gf3_sarscape_hv_only
compile_opt idl2
gf3_init_default_cfg
gf3_prepare_dirs
gf3_start_batch
errmsg = ''
gf3_run_single_pol_pipeline, 'HV', do_import=1, errmsg=errmsg
gf3_stop_batch
if errmsg ne '' then print, 'HV pipeline stopped: ' + errmsg
end
@@ -0,0 +1,8 @@
@echo off
set IDLRT=C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlrt.exe
set SAV=C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\gf3_sarscape_cli.sav
set META=C:\Users\Administrator\Desktop\GF3_Test\GF3_scene_20260513\GF3_SYC_FSI_051370_E124.6_N51.7_20260513_L1A_HHHV_L10007355955.meta.xml
set OUT=C:\Users\Administrator\Desktop\GF3_Test\GF3_cli_test_out_20260513
set DEM=D:\DEM\COPDEM_GLO30_China_4326_DEM
"%IDLRT%" "%SAV%" -args "%META%" "%OUT%" "%DEM%" "HH,HV"
@@ -0,0 +1,768 @@
package main
import (
"archive/tar"
"bufio"
"compress/gzip"
"embed"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"io/fs"
"os"
"os/exec"
"path/filepath"
"sort"
"strings"
"time"
)
//go:embed assets/gf3_sarscape_cli.sav
var embedded embed.FS
type config struct {
configPath string `json:"-"`
input string `json:"input"`
output string `json:"output"`
dem string `json:"dem"`
polarizations string `json:"polarizations"`
idlrt string `json:"idlrt"`
keepExtracted bool `json:"keep_extracted"`
}
type scene struct {
metaPath string
archivePath string
name string
}
type sceneFailure struct {
name string
input string
err error
}
type batchError struct {
failures []sceneFailure
}
func (e batchError) Error() string {
return fmt.Sprintf("batch completed with %d failed scene(s)", len(e.failures))
}
type completedProduct struct {
polarization string
path string
}
func main() {
cfg := parseFlags()
if len(os.Args) == 1 {
var err error
cfg, err = promptConfig(cfg)
if err != nil {
fmt.Printf("ERROR: %v\n", err)
waitForEnter()
os.Exit(1)
}
}
if err := run(cfg); err != nil {
fmt.Printf("ERROR: %v\n", err)
if len(os.Args) == 1 {
waitForEnter()
}
os.Exit(1)
}
if err := saveConfig(cfg.configPath, cfg); err != nil {
fmt.Printf("WARNING: save config: %v\n", err)
}
if len(os.Args) == 1 {
fmt.Println("Done.")
waitForEnter()
}
}
func parseFlags() config {
cfg := defaultConfig()
configPath := preScanConfigPath()
cfg.configPath = configPath
_ = loadConfig(configPath, &cfg)
flag.StringVar(&cfg.configPath, "config", configPath, "Config file path")
flag.StringVar(&cfg.input, "input", cfg.input, "GF-3 input: .meta.xml, .tar.gz, or directory")
flag.StringVar(&cfg.output, "output", cfg.output, "Output directory")
flag.StringVar(&cfg.dem, "dem", cfg.dem, "SARscape DEM file")
flag.StringVar(&cfg.polarizations, "pol", cfg.polarizations, "Polarizations: HH, HV, or HH,HV")
flag.StringVar(&cfg.idlrt, "idlrt", cfg.idlrt, "Path to idlrt.exe")
flag.BoolVar(&cfg.keepExtracted, "keep-extracted", cfg.keepExtracted, "Keep extracted archives under output/.gf3_extract")
flag.Parse()
return cfg
}
func defaultConfig() config {
return config{
dem: `D:\DEM\COPDEM_GLO30_China_4326_DEM`,
polarizations: "HH,HV",
idlrt: defaultIDLRT(),
keepExtracted: true,
}
}
func preScanConfigPath() string {
defaultPath := defaultConfigPath()
for i := 1; i < len(os.Args)-1; i++ {
if os.Args[i] == "-config" {
return os.Args[i+1]
}
}
return defaultPath
}
func defaultConfigPath() string {
exe, err := os.Executable()
if err != nil {
return "gf3wrapper.json"
}
return filepath.Join(filepath.Dir(exe), "gf3wrapper.json")
}
func loadConfig(path string, cfg *config) error {
data, err := os.ReadFile(path)
if err != nil {
return err
}
var stored struct {
IDLRTPath string `json:"idlrt_path"`
DemFile string `json:"dem_file"`
Polarizations string `json:"polarizations"`
LastInput string `json:"last_input"`
LastOutput string `json:"last_output"`
IDLRT string `json:"idlrt"`
Dem string `json:"dem"`
Input string `json:"input"`
Output string `json:"output"`
KeepExtracted *bool `json:"keep_extracted"`
}
if err := json.Unmarshal(data, &stored); err != nil {
return err
}
if stored.IDLRTPath != "" {
cfg.idlrt = stored.IDLRTPath
} else if stored.IDLRT != "" {
cfg.idlrt = stored.IDLRT
}
if stored.DemFile != "" {
cfg.dem = stored.DemFile
} else if stored.Dem != "" {
cfg.dem = stored.Dem
}
if stored.Polarizations != "" {
cfg.polarizations = stored.Polarizations
}
if stored.LastInput != "" {
cfg.input = stored.LastInput
} else if stored.Input != "" {
cfg.input = stored.Input
}
if stored.LastOutput != "" {
cfg.output = stored.LastOutput
} else if stored.Output != "" {
cfg.output = stored.Output
}
if stored.KeepExtracted != nil {
cfg.keepExtracted = *stored.KeepExtracted
}
cfg.configPath = path
return nil
}
func saveConfig(path string, cfg config) error {
if path == "" {
path = defaultConfigPath()
}
stored := struct {
IDLRTPath string `json:"idlrt_path"`
DemFile string `json:"dem_file"`
Polarizations string `json:"polarizations"`
LastInput string `json:"last_input"`
LastOutput string `json:"last_output"`
KeepExtracted bool `json:"keep_extracted"`
}{
IDLRTPath: cfg.idlrt,
DemFile: cfg.dem,
Polarizations: cfg.polarizations,
LastInput: cfg.input,
LastOutput: cfg.output,
KeepExtracted: cfg.keepExtracted,
}
data, err := json.MarshalIndent(stored, "", " ")
if err != nil {
return err
}
return os.WriteFile(path, data, 0o644)
}
func promptConfig(defaults config) (config, error) {
reader := bufio.NewReader(os.Stdin)
cfg := defaults
fmt.Println("GF-3 SARscape Wrapper")
fmt.Println()
cfg.input = prompt(reader, "Input .tar.gz / .meta.xml / folder", cfg.input)
cfg.output = prompt(reader, "Output folder", cfg.output)
cfg.dem = prompt(reader, "DEM file", cfg.dem)
cfg.polarizations = prompt(reader, "Polarizations", cfg.polarizations)
cfg.idlrt = prompt(reader, "IDL Runtime", cfg.idlrt)
if cfg.input == "" || cfg.output == "" || cfg.dem == "" {
return cfg, errors.New("input, output, and DEM are required")
}
return cfg, nil
}
func prompt(reader *bufio.Reader, label, defaultValue string) string {
if defaultValue != "" {
fmt.Printf("%s [%s]: ", label, defaultValue)
} else {
fmt.Printf("%s: ", label)
}
text, _ := reader.ReadString('\n')
text = strings.TrimSpace(text)
if text == "" {
return defaultValue
}
return text
}
func waitForEnter() {
fmt.Println()
fmt.Print("Press Enter to exit...")
_, _ = bufio.NewReader(os.Stdin).ReadString('\n')
}
func run(cfg config) error {
if cfg.input == "" || cfg.output == "" || cfg.dem == "" {
return errors.New("required flags: -input, -output, -dem")
}
if _, err := os.Stat(cfg.idlrt); err != nil {
return fmt.Errorf("idlrt not found: %s: %w", cfg.idlrt, err)
}
if _, err := os.Stat(cfg.dem); err != nil {
return fmt.Errorf("DEM not found: %s: %w", cfg.dem, err)
}
if err := os.MkdirAll(cfg.output, 0o755); err != nil {
return err
}
scenes, err := discoverScenes(cfg)
if err != nil {
return err
}
if len(scenes) == 0 {
return fmt.Errorf("no *.meta.xml scenes found in %s", cfg.input)
}
failures := make([]sceneFailure, 0)
processed := 0
skipped := 0
savPath := ""
for _, sc := range scenes {
sceneOut := filepath.Join(cfg.output, sc.name)
if products, ok := completedSceneOutputs(sceneOut, sc.name, cfg.polarizations); ok {
recordSceneSkip(sc, sceneOut, products)
skipped++
continue
}
if err := os.MkdirAll(sceneOut, 0o755); err != nil {
recordSceneFailure(&failures, sc, err)
continue
}
if sc.archivePath != "" {
extractDir := filepath.Join(sceneOut, ".gf3_extract")
fmt.Printf("Extracting %s\n", sc.archivePath)
if err := extractTarGz(sc.archivePath, extractDir); err != nil {
recordSceneFailure(&failures, sc, fmt.Errorf("failed to extract archive before processing scene %s: %w", sc.archivePath, err))
continue
}
extractedScenes, err := findMetaXML(extractDir)
if err != nil {
recordSceneFailure(&failures, sc, err)
continue
}
if len(extractedScenes) == 0 {
recordSceneFailure(&failures, sc, fmt.Errorf("no *.meta.xml found in archive: %s", sc.archivePath))
continue
}
sc.metaPath = extractedScenes[0].metaPath
sc.name = sceneName(sc.metaPath)
}
if savPath == "" {
var err error
savPath, err = materializeSAV(cfg.output)
if err != nil {
recordSceneFailure(&failures, sc, err)
continue
}
}
if err := runScene(cfg, savPath, sc, sceneOut); err != nil {
recordSceneFailure(&failures, sc, err)
continue
}
processed++
}
printBatchSummary(len(scenes), processed, skipped, failures)
if len(failures) > 0 {
return batchError{failures: failures}
}
return nil
}
func sceneInput(sc scene) string {
if sc.archivePath != "" {
return sc.archivePath
}
if sc.metaPath != "" {
return sc.metaPath
}
return sc.name
}
func recordSceneFailure(failures *[]sceneFailure, sc scene, err error) {
input := sceneInput(sc)
failure := sceneFailure{name: sc.name, input: input, err: err}
*failures = append(*failures, failure)
fmt.Printf("FAILED %s\n", sc.name)
fmt.Printf("Input %s\n", input)
fmt.Printf("Reason: %v\n", err)
fmt.Println("Continuing with next scene.")
}
func recordSceneSkip(sc scene, sceneOut string, products []completedProduct) {
parts := make([]string, 0, len(products))
for _, product := range products {
parts = append(parts, product.polarization+"="+product.path)
}
fmt.Printf("SKIPPED %s\n", sc.name)
fmt.Printf("Input %s\n", sceneInput(sc))
fmt.Printf("Output already complete %s\n", sceneOut)
fmt.Printf("Products %s\n", strings.Join(parts, "; "))
}
func printBatchSummary(total, processed, skipped int, failures []sceneFailure) {
fmt.Println()
fmt.Printf("Batch summary: %d processed, %d skipped, %d failed, %d total\n", processed, skipped, len(failures), total)
if len(failures) == 0 {
return
}
fmt.Println("Failed scenes:")
for _, failure := range failures {
fmt.Printf("- %s (%s): %v\n", failure.name, failure.input, failure.err)
}
}
func materializeSAV(outputDir string) (string, error) {
data, err := embedded.ReadFile("assets/gf3_sarscape_cli.sav")
if err != nil {
return "", err
}
runtimeDir := filepath.Join(outputDir, ".gf3_runtime")
if err := os.MkdirAll(runtimeDir, 0o755); err != nil {
return "", err
}
savPath := filepath.Join(runtimeDir, "gf3_sarscape_cli.sav")
if err := os.WriteFile(savPath, data, 0o644); err != nil {
return "", err
}
return savPath, nil
}
func discoverScenes(cfg config) ([]scene, error) {
info, err := os.Stat(cfg.input)
if err != nil {
return nil, err
}
if info.IsDir() {
return discoverScenesInDir(cfg.input, cfg.output)
}
lower := strings.ToLower(cfg.input)
if strings.HasSuffix(lower, ".meta.xml") {
return []scene{{metaPath: cfg.input, name: sceneName(cfg.input)}}, nil
}
if strings.HasSuffix(lower, ".tar.gz") || strings.HasSuffix(lower, ".tgz") {
return []scene{{archivePath: cfg.input, name: safeName(trimArchiveExt(filepath.Base(cfg.input)))}}, nil
}
return nil, fmt.Errorf("unsupported input type: %s", cfg.input)
}
func discoverScenesInDir(inputDir, outputDir string) ([]scene, error) {
_ = outputDir
scenesByName := make(map[string]scene)
err := filepath.WalkDir(inputDir, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
if d.Name() == ".gf3_extract" {
return filepath.SkipDir
}
return nil
}
lower := strings.ToLower(d.Name())
switch {
case strings.HasSuffix(lower, ".meta.xml"):
sc := scene{metaPath: path, name: sceneName(path)}
key := strings.ToLower(sc.name)
if _, ok := scenesByName[key]; !ok {
scenesByName[key] = sc
}
case strings.HasSuffix(lower, ".tar.gz") || strings.HasSuffix(lower, ".tgz"):
sc := scene{archivePath: path, name: safeName(trimArchiveExt(filepath.Base(path)))}
scenesByName[strings.ToLower(sc.name)] = sc
}
return nil
})
if err != nil {
return nil, err
}
scenes := make([]scene, 0, len(scenesByName))
for _, sc := range scenesByName {
scenes = append(scenes, sc)
}
sort.Slice(scenes, func(i, j int) bool {
return strings.ToLower(scenes[i].name) < strings.ToLower(scenes[j].name)
})
return scenes, nil
}
func findMetaXML(root string) ([]scene, error) {
var scenes []scene
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
if strings.HasSuffix(strings.ToLower(d.Name()), ".meta.xml") {
scenes = append(scenes, scene{metaPath: path, name: sceneName(path)})
}
return nil
})
return scenes, err
}
func uniqueScenes(scenes []scene) []scene {
seen := make(map[string]bool, len(scenes))
out := make([]scene, 0, len(scenes))
for _, sc := range scenes {
key := filepath.Clean(sc.metaPath)
if seen[key] {
continue
}
seen[key] = true
out = append(out, sc)
}
return out
}
func runScene(cfg config, savPath string, sc scene, sceneOut string) error {
fmt.Printf("Processing %s\n", sc.metaPath)
fmt.Printf("Output %s\n", sceneOut)
cmd := exec.Command(cfg.idlrt, savPath, "-args", sc.metaPath, sceneOut, cfg.dem, cfg.polarizations)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Dir = sceneOut
if err := cmd.Start(); err != nil {
return err
}
done := make(chan error, 1)
go func() {
done <- cmd.Wait()
}()
ticker := time.NewTicker(2 * time.Second)
defer ticker.Stop()
for {
select {
case err := <-done:
fmt.Printf("\r%-100s\r", "")
if err != nil {
return fmt.Errorf("scene failed %s: %w", sc.metaPath, err)
}
fmt.Println("Completed", sc.name)
return nil
case <-ticker.C:
fmt.Printf("\r%s", progressLine(sceneOut, sc.name, cfg.polarizations))
}
}
}
func progressLine(sceneOut, sceneNameValue, polarizations string) string {
stage, done, total := progressStage(sceneOut, sceneNameValue, polarizations)
width := 28
filled := 0
if total > 0 {
filled = done * width / total
}
if filled > width {
filled = width
}
bar := strings.Repeat("#", filled) + strings.Repeat("-", width-filled)
return fmt.Sprintf("[%s] %2d/%2d %s", bar, done, total, stage)
}
func progressStage(sceneOut, sceneNameValue, polarizations string) (string, int, int) {
pols := requestedPolarizations(polarizations)
checks := make([]struct {
label string
path string
}, 0, len(pols)*4)
for _, pol := range pols {
checks = append(checks, struct {
label string
path string
}{"import " + pol, filepath.Join(sceneOut, "*_"+pol+"_slc.sml")})
}
for _, pol := range pols {
lower := strings.ToLower(pol)
prefix := sceneNameValue + "_" + lower
checks = append(checks,
struct {
label string
path string
}{"multilook " + pol, filepath.Join(sceneOut, prefix+"_ml.sml")},
struct {
label string
path string
}{"filter " + pol, filepath.Join(sceneOut, prefix+"_filt.sml")},
struct {
label string
path string
}{"geocode " + pol, filepath.Join(sceneOut, prefix+"_geo.sml")},
)
}
done := 0
for _, check := range checks {
if matched(check.path) {
done++
continue
}
return check.label, done, len(checks)
}
return "finishing", done, len(checks)
}
func completedSceneOutputs(sceneOut, sceneNameValue, polarizations string) ([]completedProduct, bool) {
info, err := os.Stat(sceneOut)
if err != nil || !info.IsDir() {
return nil, false
}
pols := requestedPolarizations(polarizations)
products := make([]completedProduct, 0, len(pols))
for _, pol := range pols {
path, ok := completedGeoProduct(sceneOut, sceneNameValue, pol)
if !ok {
return nil, false
}
products = append(products, completedProduct{polarization: pol, path: path})
}
return products, true
}
func completedGeoProduct(sceneOut, sceneNameValue, polarization string) (string, bool) {
lower := strings.ToLower(polarization)
exact := filepath.Join(sceneOut, sceneNameValue+"_"+lower+"_geo.sml")
if completeGeoSML(exact) {
return exact, true
}
matches, err := filepath.Glob(filepath.Join(sceneOut, "*_"+lower+"_geo.sml"))
if err != nil {
return "", false
}
sort.Strings(matches)
for _, match := range matches {
if completeGeoSML(match) {
return match, true
}
}
return "", false
}
func completeGeoSML(path string) bool {
info, err := os.Stat(path)
if err != nil || info.IsDir() || info.Size() == 0 {
return false
}
dataPath := strings.TrimSuffix(path, ".sml")
dataInfo, err := os.Stat(dataPath)
return err == nil && !dataInfo.IsDir() && dataInfo.Size() > 0
}
func requestedPolarizations(value string) []string {
parts := strings.Split(value, ",")
out := make([]string, 0, len(parts))
for _, part := range parts {
pol := strings.ToUpper(strings.TrimSpace(part))
if pol == "" {
continue
}
if pol == "ALL" {
return []string{"HH", "HV"}
}
out = append(out, pol)
}
if len(out) == 0 {
return []string{"HH", "HV"}
}
return out
}
func matched(pattern string) bool {
matches, err := filepath.Glob(pattern)
if err != nil {
return false
}
return len(matches) > 0
}
func extractTarGz(src, dst string) error {
dst = filepath.Clean(dst)
tmpDst := dst + ".tmp"
if err := os.RemoveAll(tmpDst); err != nil {
return fmt.Errorf("remove stale temporary extraction directory %s: %w", tmpDst, err)
}
if err := os.MkdirAll(tmpDst, 0o755); err != nil {
return err
}
if err := extractTarGzToDir(src, tmpDst); err != nil {
_ = os.RemoveAll(tmpDst)
return err
}
if err := os.RemoveAll(dst); err != nil {
_ = os.RemoveAll(tmpDst)
return fmt.Errorf("replace extraction directory %s: %w", dst, err)
}
if err := os.Rename(tmpDst, dst); err != nil {
_ = os.RemoveAll(tmpDst)
return fmt.Errorf("move completed extraction into %s: %w", dst, err)
}
return nil
}
func extractTarGzToDir(src, dst string) error {
f, err := os.Open(src)
if err != nil {
return err
}
defer f.Close()
gz, err := gzip.NewReader(f)
if err != nil {
return archiveReadError(src, "", err)
}
defer gz.Close()
tr := tar.NewReader(gz)
var currentEntry string
for {
hdr, err := tr.Next()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return archiveReadError(src, currentEntry, err)
}
currentEntry = hdr.Name
target := filepath.Join(dst, filepath.Clean(hdr.Name))
if !strings.HasPrefix(target, filepath.Clean(dst)+string(os.PathSeparator)) && filepath.Clean(target) != filepath.Clean(dst) {
return fmt.Errorf("archive entry escapes destination: %s", hdr.Name)
}
switch hdr.Typeflag {
case tar.TypeDir:
if err := os.MkdirAll(target, 0o755); err != nil {
return err
}
case tar.TypeReg:
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
return err
}
out, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
if err != nil {
return err
}
_, copyErr := io.Copy(out, tr)
closeErr := out.Close()
if copyErr != nil {
return archiveReadError(src, hdr.Name, copyErr)
}
if closeErr != nil {
return closeErr
}
}
}
return nil
}
func archiveReadError(src, entry string, err error) error {
detail := src
if info, statErr := os.Stat(src); statErr == nil {
detail = fmt.Sprintf("%s (%d bytes)", src, info.Size())
}
if entry != "" {
detail = fmt.Sprintf("%s while reading %s", detail, entry)
}
if errors.Is(err, io.ErrUnexpectedEOF) || strings.Contains(strings.ToLower(err.Error()), "unexpected eof") {
return fmt.Errorf("archive appears truncated or incomplete: %s: %w; re-copy or re-download the .tar.gz", detail, err)
}
return fmt.Errorf("read archive %s: %w", detail, err)
}
func sceneName(metaPath string) string {
base := filepath.Base(metaPath)
base = strings.TrimSuffix(base, ".meta.xml")
base = strings.TrimSuffix(base, ".META.XML")
return safeName(base)
}
func safeName(s string) string {
replacer := strings.NewReplacer(" ", "_", ":", "_", "/", "_", "\\", "_")
return replacer.Replace(s)
}
func trimArchiveExt(name string) string {
lower := strings.ToLower(name)
switch {
case strings.HasSuffix(lower, ".tar.gz"):
return name[:len(name)-7]
case strings.HasSuffix(lower, ".tgz"):
return name[:len(name)-4]
default:
return strings.TrimSuffix(name, filepath.Ext(name))
}
}
func defaultIDLRT() string {
if v := os.Getenv("IDLRT_PATH"); v != "" {
return v
}
return `C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlrt.exe`
}
@@ -0,0 +1,61 @@
package main
import (
"os"
"path/filepath"
"testing"
)
func TestCompletedSceneOutputsRequiresGeoSMLAndDataForAllPolarizations(t *testing.T) {
dir := t.TempDir()
sceneName := "GF3_TEST_SCENE"
writeFile(t, filepath.Join(dir, sceneName+"_hh_geo.sml"), "hh sml")
writeFile(t, filepath.Join(dir, sceneName+"_hh_geo"), "hh data")
writeFile(t, filepath.Join(dir, sceneName+"_hv_geo.sml"), "hv sml")
writeFile(t, filepath.Join(dir, sceneName+"_hv_geo"), "hv data")
products, ok := completedSceneOutputs(dir, sceneName, "HH,HV")
if !ok {
t.Fatal("expected completed scene outputs")
}
if len(products) != 2 {
t.Fatalf("expected 2 completed products, got %d", len(products))
}
}
func TestCompletedSceneOutputsRejectsMissingDataFile(t *testing.T) {
dir := t.TempDir()
sceneName := "GF3_TEST_SCENE"
writeFile(t, filepath.Join(dir, sceneName+"_hh_geo.sml"), "hh sml")
writeFile(t, filepath.Join(dir, sceneName+"_hv_geo.sml"), "hv sml")
writeFile(t, filepath.Join(dir, sceneName+"_hv_geo"), "hv data")
if _, ok := completedSceneOutputs(dir, sceneName, "HH,HV"); ok {
t.Fatal("expected incomplete scene when one geocoded data file is missing")
}
}
func TestCompletedSceneOutputsRequiresRequestedPolarizations(t *testing.T) {
dir := t.TempDir()
sceneName := "GF3_TEST_SCENE"
writeFile(t, filepath.Join(dir, sceneName+"_hh_geo.sml"), "hh sml")
writeFile(t, filepath.Join(dir, sceneName+"_hh_geo"), "hh data")
if _, ok := completedSceneOutputs(dir, sceneName, "HH,HV"); ok {
t.Fatal("expected incomplete scene when HV output is missing")
}
if _, ok := completedSceneOutputs(dir, sceneName, "HH"); !ok {
t.Fatal("expected completed scene for HH-only request")
}
}
func writeFile(t *testing.T, path, value string) {
t.Helper()
if err := os.WriteFile(path, []byte(value), 0o644); err != nil {
t.Fatal(err)
}
}