chore: 合并 GF3_L1A_To_L2_pipeline 历史
This commit is contained in:
Harmon
@@ -0,0 +1,16 @@
|
||||
* text=auto
|
||||
|
||||
.gitattributes text eol=lf
|
||||
.gitignore text eol=lf
|
||||
*.md text eol=lf
|
||||
*.py text eol=lf
|
||||
*.go text eol=lf
|
||||
*.mod text eol=lf
|
||||
*.toml text eol=lf
|
||||
*.yml text eol=lf
|
||||
*.idl text eol=lf
|
||||
*.pro text eol=lf
|
||||
*.cmd text eol=crlf
|
||||
|
||||
*.exe binary
|
||||
*.sav binary
|
||||
@@ -0,0 +1,27 @@
|
||||
.venv/
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
/.tmp_tests/
|
||||
logs/
|
||||
GMTED2010.jp2
|
||||
|
||||
# Local wrapper state
|
||||
gf3wrapper.json
|
||||
.gf3_extract/
|
||||
.gf3_runtime/
|
||||
|
||||
# Production data and generated outputs
|
||||
/data/
|
||||
/outputs/
|
||||
*.tar.gz
|
||||
*.tgz
|
||||
*.zip
|
||||
*.tif
|
||||
*.tiff
|
||||
*.img
|
||||
*.dat
|
||||
*.sml
|
||||
*.hdr
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Compatibility wrapper for archive extraction."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
_SRC = Path(__file__).resolve().parent / "src"
|
||||
if _SRC.is_dir():
|
||||
sys.path.insert(0, str(_SRC))
|
||||
|
||||
from gf3_l1a2l2.decompress import extract_compressed_files
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Extract GF3 archives")
|
||||
parser.add_argument("input", type=Path, help="folder containing .tar.gz or .zip archives")
|
||||
parser.add_argument("--overwrite", action="store_true", help="overwrite existing extracted folders")
|
||||
args = parser.parse_args()
|
||||
|
||||
results = extract_compressed_files(args.input, overwrite=args.overwrite)
|
||||
failed = [item for item in results if item.status == "failed"]
|
||||
for item in results:
|
||||
print(f"{item.status}\t{item.archive}\t{item.output_dir}\t{item.message}")
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,103 @@
|
||||
# GF3 L1A to L2 Pipeline
|
||||
|
||||
This repository contains two GF-3 L1A to L2 processing paths:
|
||||
|
||||
- **SARscape Go wrapper**: Windows-oriented production wrapper around ENVI/IDL Runtime and SARscape. This is the recommended path when SARscape is available.
|
||||
- **Python pipeline**: GDAL/Rasterio/NumPy implementation kept as a pure Python package and CLI.
|
||||
|
||||
The SARscape wrapper currently has the more complete operational workflow for GF-3 scene import, multilook/filter/geocode processing, batch directory discovery, saved defaults, and progress reporting.
|
||||
|
||||
## Repository Layout
|
||||
|
||||
```text
|
||||
dist/windows/ Prebuilt Windows executable
|
||||
docs/ Operational documentation
|
||||
src/gf3_l1a2l2/ Python package
|
||||
tests/ Python unit tests and small fixtures
|
||||
tools/gf3_sarscape_wrapper/ Go wrapper source and embedded SAV asset
|
||||
tools/gf3_sarscape_wrapper/idl/ IDL/SARscape source scripts
|
||||
```
|
||||
|
||||
Large GF-3 archives, extracted scenes, SARscape outputs, and local runtime files are intentionally not tracked in Git.
|
||||
|
||||
## Recommended: SARscape Go Wrapper
|
||||
|
||||
Use the prebuilt executable:
|
||||
|
||||
```powershell
|
||||
.\dist\windows\gf3wrapper.exe `
|
||||
-input "D:\GF3\L1A_BATCH" `
|
||||
-output "E:\GF3\L2_SARscape" `
|
||||
-dem "D:\DEM\COPDEM_GLO30_China_4326_DEM" `
|
||||
-pol "HH,HV"
|
||||
```
|
||||
|
||||
Supported input forms:
|
||||
|
||||
- A single `*.tar.gz` GF-3 L1A package.
|
||||
- A single `*.meta.xml` scene metadata file.
|
||||
- A directory containing multiple `*.tar.gz` or `*.meta.xml` scenes.
|
||||
|
||||
The wrapper writes one output subdirectory per scene. On successful runs it saves defaults next to the executable in `gf3wrapper.json`, so double-click interactive mode can reuse the previous input/output/DEM/polarization settings.
|
||||
|
||||
Batch directory runs continue after per-scene failures. If one archive is truncated, invalid, missing metadata, or fails during SARscape processing, the wrapper prints the failed scene and reason, skips that scene, and continues with the remaining scenes. At the end it prints a batch summary and returns a non-zero exit code when any scene failed.
|
||||
|
||||
Before processing each scene, the wrapper checks the scene output directory for completed geocoded products for the requested polarizations. Already completed scenes are skipped, so rerunning the same input/output pair only processes missing or incomplete scenes.
|
||||
|
||||
More details: [docs/sarscape_go_wrapper.md](docs/sarscape_go_wrapper.md)
|
||||
|
||||
## Python Pipeline
|
||||
|
||||
Use an environment that already has working GDAL/OSGeo Python bindings. On Windows, Conda/Miniforge is usually the least fragile route.
|
||||
|
||||
```powershell
|
||||
conda env create -f environment.yml
|
||||
conda activate gf3-l1a2l2
|
||||
pip install -e . --no-deps
|
||||
```
|
||||
|
||||
Run:
|
||||
|
||||
```powershell
|
||||
gf3-l1a2l2 run --input D:\GF3\L1A_BATCH --output E:\GF3\L2_Image --dem .\GMTED2010.jp2
|
||||
```
|
||||
|
||||
Common options:
|
||||
|
||||
```powershell
|
||||
gf3-l1a2l2 run `
|
||||
--input D:\GF3\L1A_BATCH `
|
||||
--output E:\GF3\L2_Image `
|
||||
--dem .\GMTED2010.jp2 `
|
||||
--workers 1 `
|
||||
--x-res 0.0002 `
|
||||
--y-res 0.0002 `
|
||||
--dst-srs EPSG:4326
|
||||
```
|
||||
|
||||
Legacy wrappers remain available:
|
||||
|
||||
```powershell
|
||||
python gf3_L1A_To_L2.py run --input D:\GF3\L1A_BATCH --output E:\GF3\L2_Image --dem .\GMTED2010.jp2
|
||||
python Decompression.py D:\GF3\L1A_BATCH
|
||||
```
|
||||
|
||||
## Development Checks
|
||||
|
||||
Python:
|
||||
|
||||
```powershell
|
||||
python -m pytest
|
||||
```
|
||||
|
||||
Go wrapper:
|
||||
|
||||
```powershell
|
||||
cd .\tools\gf3_sarscape_wrapper
|
||||
$env:GOTELEMETRY = "off"
|
||||
go build -o ..\..\dist\windows\gf3wrapper.exe .
|
||||
```
|
||||
|
||||
## Data Policy
|
||||
|
||||
Do not commit production GF-3 archives or generated products. Keep large data on NAS/storage and document the path in run notes. Only small sanitized fixtures should be added under `tests/fixtures`.
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
# Windows Distribution
|
||||
|
||||
`gf3wrapper.exe` is the prebuilt SARscape Go wrapper.
|
||||
|
||||
Run from the repository root:
|
||||
|
||||
```powershell
|
||||
.\dist\windows\gf3wrapper.exe `
|
||||
-input "D:\GF3\L1A_BATCH" `
|
||||
-output "E:\GF3\L2_SARscape" `
|
||||
-dem "D:\DEM\COPDEM_GLO30_China_4326_DEM" `
|
||||
-pol "HH,HV"
|
||||
```
|
||||
|
||||
The executable requires ENVI/IDL Runtime and SARscape on the target machine.
|
||||
|
||||
In directory mode, failed scenes are skipped and listed in a final batch summary. The wrapper continues with remaining scenes, but exits non-zero if any scene failed.
|
||||
|
||||
Rerunning the same input/output pair skips scenes with complete geocoded outputs for the requested polarizations and processes only missing or incomplete scenes.
|
||||
|
||||
After successful runs, `gf3wrapper.json` is created in this directory to store defaults. That file is local runtime state and is ignored by Git.
|
||||
Binary file not shown.
@@ -0,0 +1,45 @@
|
||||
# Repository Layout
|
||||
|
||||
```text
|
||||
.
|
||||
|-- dist/
|
||||
| `-- windows/
|
||||
| `-- gf3wrapper.exe
|
||||
|-- docs/
|
||||
| |-- repository_layout.md
|
||||
| `-- sarscape_go_wrapper.md
|
||||
|-- src/
|
||||
| `-- gf3_l1a2l2/
|
||||
|-- tests/
|
||||
| `-- fixtures/
|
||||
|-- tools/
|
||||
| `-- gf3_sarscape_wrapper/
|
||||
| |-- assets/
|
||||
| |-- idl/
|
||||
| |-- go.mod
|
||||
| `-- main.go
|
||||
|-- README.md
|
||||
|-- environment.yml
|
||||
`-- pyproject.toml
|
||||
```
|
||||
|
||||
## What Belongs In Git
|
||||
|
||||
- Python package source.
|
||||
- Go wrapper source.
|
||||
- IDL/SARscape source scripts.
|
||||
- Small embedded runtime assets needed to build the wrapper.
|
||||
- Small sanitized tests and fixtures.
|
||||
- Documentation and reproducible build instructions.
|
||||
- Small convenience binaries when the team explicitly wants a ready-to-run Windows artifact.
|
||||
|
||||
## What Does Not Belong In Git
|
||||
|
||||
- Raw GF-3 production archives.
|
||||
- Extracted scenes.
|
||||
- SARscape output directories.
|
||||
- Local wrapper config files such as `gf3wrapper.json`.
|
||||
- Runtime folders such as `.gf3_extract` and `.gf3_runtime`.
|
||||
- DEM files and other large environment-specific dependencies.
|
||||
|
||||
Use NAS or object storage for production data and record exact paths in run logs or issue notes.
|
||||
@@ -0,0 +1,142 @@
|
||||
# SARscape Go Wrapper
|
||||
|
||||
`gf3wrapper.exe` is a Windows command-line wrapper for the SARscape/IDL GF-3 processing workflow. It embeds `gf3_sarscape_cli.sav`, starts `idlrt.exe`, and passes scene metadata, output directory, DEM, and polarization arguments to SARscape.
|
||||
|
||||
This path is intended for operational runs where ENVI/IDL Runtime and SARscape are installed and licensed.
|
||||
|
||||
## Requirements
|
||||
|
||||
- Windows host.
|
||||
- ENVI/IDL Runtime, default path:
|
||||
`C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlrt.exe`
|
||||
- SARscape available to the IDL Runtime environment.
|
||||
- DEM file, for example:
|
||||
`D:\DEM\COPDEM_GLO30_China_4326_DEM`
|
||||
- GF-3 L1A input as `*.tar.gz`, `*.meta.xml`, or a directory containing multiple scenes.
|
||||
|
||||
The IDL Runtime path can be overridden with `-idlrt` or the `IDLRT_PATH` environment variable.
|
||||
|
||||
## Quick Start
|
||||
|
||||
Use the prebuilt executable:
|
||||
|
||||
```powershell
|
||||
.\dist\windows\gf3wrapper.exe `
|
||||
-input "D:\GF3\L1A_BATCH" `
|
||||
-output "E:\GF3\L2_SARscape" `
|
||||
-dem "D:\DEM\COPDEM_GLO30_China_4326_DEM" `
|
||||
-pol "HH,HV"
|
||||
```
|
||||
|
||||
Double-click mode is also supported. When launched without arguments, the wrapper prompts for input, output, DEM, polarizations, and IDL Runtime path. Press Enter to reuse saved defaults.
|
||||
|
||||
## Inputs
|
||||
|
||||
The wrapper supports:
|
||||
|
||||
- A single GF-3 archive: `GF3_....tar.gz`
|
||||
- A single extracted scene metadata file: `GF3_....meta.xml`
|
||||
- A directory containing many archives and/or extracted scenes
|
||||
|
||||
Directory mode recursively scans for `*.tar.gz`, `*.tgz`, and `*.meta.xml`. If both archive and metadata exist for the same scene name, the archive is preferred. Internal `.gf3_extract` folders are skipped.
|
||||
|
||||
## Batch Failure Handling
|
||||
|
||||
Directory runs are fault-tolerant at the scene level. If a single scene cannot be extracted or processed, the wrapper records that scene as failed, prints the input path and error reason, and continues with the next scene.
|
||||
|
||||
Archive read errors include the archive path, archive size when available, and the archive member being read when the failure happened. For example, a truncated `.tar.gz` reports that the archive appears incomplete and should be re-copied or re-downloaded.
|
||||
|
||||
At the end of the run, the wrapper prints a summary:
|
||||
|
||||
```text
|
||||
Batch summary: 3 succeeded, 1 failed, 4 total
|
||||
Failed scenes:
|
||||
- GF3_... (<archive path>): <failure reason>
|
||||
```
|
||||
|
||||
If any scene failed, the process exits with a non-zero status after the summary. This lets scheduled jobs detect that the batch needs attention while still preserving successful outputs from other scenes.
|
||||
|
||||
## Resume Behavior
|
||||
|
||||
Before extracting or processing a scene, the wrapper checks the scene output directory for completed geocoded SARscape products for every requested polarization. A product is considered complete when both the final `*_geo.sml` metadata file and its matching data file exist and are non-empty.
|
||||
|
||||
Already completed scenes are skipped:
|
||||
|
||||
```text
|
||||
SKIPPED GF3_...
|
||||
Input D:\GF3\L1A_BATCH\GF3_....tar.gz
|
||||
Output already complete E:\GF3\L2_SARscape\GF3_...
|
||||
```
|
||||
|
||||
This is intended for interrupted or partially failed batch runs. For example, after replacing a truncated archive in the original input directory, rerun the same command with the same output directory; scenes that already have complete HH/HV geocoded outputs are skipped, and only missing or incomplete scenes are processed.
|
||||
|
||||
## Outputs
|
||||
|
||||
Each scene is processed into its own output directory:
|
||||
|
||||
```text
|
||||
E:\GF3\L2_SARscape\
|
||||
GF3_SCENE_NAME\
|
||||
.gf3_extract\ Extracted archive, when input was .tar.gz
|
||||
*_slc.sml
|
||||
*_ml.sml
|
||||
*_filt.sml
|
||||
*_geo.sml
|
||||
```
|
||||
|
||||
The embedded SAV is materialized under:
|
||||
|
||||
```text
|
||||
<output>\.gf3_runtime\gf3_sarscape_cli.sav
|
||||
```
|
||||
|
||||
## Saved Defaults
|
||||
|
||||
After a successful run, the wrapper writes `gf3wrapper.json` next to the executable:
|
||||
|
||||
```json
|
||||
{
|
||||
"idlrt_path": "C:\\Program Files\\Harris\\ENVI56\\IDL88\\bin\\bin.x86_64\\idlrt.exe",
|
||||
"dem_file": "D:\\DEM\\COPDEM_GLO30_China_4326_DEM",
|
||||
"polarizations": "HH,HV",
|
||||
"last_input": "D:\\GF3\\L1A_BATCH",
|
||||
"last_output": "E:\\GF3\\L2_SARscape",
|
||||
"keep_extracted": true
|
||||
}
|
||||
```
|
||||
|
||||
Command-line flags override saved defaults. If `-input`, `-output`, `-dem`, or `-pol` are omitted, saved values are reused when available.
|
||||
|
||||
## Build
|
||||
|
||||
From the repository root:
|
||||
|
||||
```powershell
|
||||
cd .\tools\gf3_sarscape_wrapper
|
||||
$env:GOTELEMETRY = "off"
|
||||
go build -o ..\..\dist\windows\gf3wrapper.exe .
|
||||
```
|
||||
|
||||
If Go cannot write to the default user cache directory, set local cache paths:
|
||||
|
||||
```powershell
|
||||
$env:GOCACHE = "C:\Temp\go-build-cache"
|
||||
$env:APPDATA = "C:\Temp\go-appdata"
|
||||
```
|
||||
|
||||
## Source Files
|
||||
|
||||
```text
|
||||
tools/gf3_sarscape_wrapper/main.go
|
||||
tools/gf3_sarscape_wrapper/assets/gf3_sarscape_cli.sav
|
||||
tools/gf3_sarscape_wrapper/idl/gf3_sarscape_cli.pro
|
||||
tools/gf3_sarscape_wrapper/idl/gf3_sarscape_pipeline.pro
|
||||
```
|
||||
|
||||
`assets/gf3_sarscape_cli.sav` is embedded into the Go executable at build time.
|
||||
|
||||
## Notes
|
||||
|
||||
- Keep production archives and generated products outside Git.
|
||||
- Keep only small sanitized fixtures in `tests/fixtures`.
|
||||
- The prebuilt executable in `dist/windows` is a convenience artifact for Windows users. Rebuild it after changing `main.go` or the embedded `.sav`.
|
||||
@@ -0,0 +1,10 @@
|
||||
name: gf3-l1a2l2
|
||||
channels:
|
||||
- conda-forge
|
||||
dependencies:
|
||||
- python=3.11
|
||||
- gdal
|
||||
- rasterio
|
||||
- numpy
|
||||
- tqdm
|
||||
- pip
|
||||
@@ -0,0 +1,16 @@
|
||||
"""Compatibility wrapper for the packaged GF3 L1A to L2 pipeline."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
_SRC = Path(__file__).resolve().parent / "src"
|
||||
if _SRC.is_dir():
|
||||
sys.path.insert(0, str(_SRC))
|
||||
|
||||
from gf3_l1a2l2.cli import main
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,24 @@
|
||||
[build-system]
|
||||
requires = ["setuptools>=68", "wheel"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "gf3-l1a2l2-pipeline"
|
||||
version = "0.1.0"
|
||||
description = "GF3 L1A to L2 processing pipeline"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"numpy>=1.22",
|
||||
"rasterio>=1.3",
|
||||
"tqdm>=4.64",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
gf3-l1a2l2 = "gf3_l1a2l2.cli:main"
|
||||
|
||||
[tool.setuptools]
|
||||
package-dir = {"" = "src"}
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["src"]
|
||||
@@ -0,0 +1,16 @@
|
||||
"""GF3 L1A to L2 processing pipeline."""
|
||||
|
||||
__all__ = ["PipelineConfig", "run_pipeline"]
|
||||
__version__ = "0.1.0"
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
if name == "PipelineConfig":
|
||||
from gf3_l1a2l2.config import PipelineConfig
|
||||
|
||||
return PipelineConfig
|
||||
if name == "run_pipeline":
|
||||
from gf3_l1a2l2.pipeline import run_pipeline
|
||||
|
||||
return run_pipeline
|
||||
raise AttributeError(name)
|
||||
@@ -0,0 +1,5 @@
|
||||
from gf3_l1a2l2.cli import main
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,88 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
from pathlib import Path
|
||||
from typing import Iterator, Optional
|
||||
|
||||
import numpy as np
|
||||
import rasterio
|
||||
from rasterio.windows import Window
|
||||
|
||||
from gf3_l1a2l2.errors import ProcessingError
|
||||
from gf3_l1a2l2.models import CalibrationValue
|
||||
from gf3_l1a2l2.paths import output_name
|
||||
|
||||
|
||||
def radiometric_calibrate_l1a_to_l1b(
|
||||
image_path: Path | str,
|
||||
output_dir: Path | str,
|
||||
calibration: CalibrationValue,
|
||||
*,
|
||||
block_size: int,
|
||||
overwrite: bool = False,
|
||||
logger: Optional[logging.Logger] = None,
|
||||
) -> Path:
|
||||
source = Path(image_path)
|
||||
out_dir = Path(output_dir)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
l1b_path = out_dir / output_name(source, "L1A", "L1B")
|
||||
|
||||
if l1b_path.exists() and not overwrite:
|
||||
if logger:
|
||||
logger.info("Skip existing L1B: %s", l1b_path)
|
||||
return l1b_path
|
||||
|
||||
if not math.isfinite(calibration.qualify_value):
|
||||
raise ProcessingError(f"Invalid QualifyValue for {source.name}: {calibration.qualify_value}")
|
||||
if not math.isfinite(calibration.calibration_const):
|
||||
raise ProcessingError(
|
||||
f"Invalid CalibrationConst for {source.name}: {calibration.calibration_const}"
|
||||
)
|
||||
|
||||
if logger:
|
||||
logger.info("Calibrating %s -> %s", source, l1b_path)
|
||||
|
||||
try:
|
||||
with rasterio.open(source) as src:
|
||||
if src.count < 2:
|
||||
raise ProcessingError(f"Expected at least 2 bands in L1A image: {source}")
|
||||
|
||||
profile = src.profile.copy()
|
||||
profile.update(
|
||||
driver="GTiff",
|
||||
count=1,
|
||||
dtype="float32",
|
||||
compress="lzw",
|
||||
BIGTIFF="YES",
|
||||
tiled=True,
|
||||
)
|
||||
|
||||
with rasterio.open(l1b_path, "w", **profile) as dst:
|
||||
for window in _iter_windows(src.width, src.height, block_size):
|
||||
real = src.read(1, window=window, out_dtype="float32")
|
||||
imag = src.read(2, window=window, out_dtype="float32")
|
||||
db = _calibrate_window(real, imag, calibration)
|
||||
dst.write(db, 1, window=window)
|
||||
except ProcessingError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise ProcessingError(f"Failed to calibrate {source}") from exc
|
||||
|
||||
return l1b_path
|
||||
|
||||
|
||||
def _calibrate_window(real: np.ndarray, imag: np.ndarray, calibration: CalibrationValue) -> np.ndarray:
|
||||
amplitude = np.hypot(real, imag)
|
||||
scaled = (amplitude / np.float32(32767.0)) * np.float32(calibration.qualify_value)
|
||||
with np.errstate(divide="ignore", invalid="ignore"):
|
||||
db = np.float32(20.0) * np.log10(scaled) - np.float32(calibration.calibration_const)
|
||||
return db.astype("float32", copy=False)
|
||||
|
||||
|
||||
def _iter_windows(width: int, height: int, block_size: int) -> Iterator[Window]:
|
||||
for row_off in range(0, height, block_size):
|
||||
window_height = min(block_size, height - row_off)
|
||||
for col_off in range(0, width, block_size):
|
||||
window_width = min(block_size, width - col_off)
|
||||
yield Window(col_off, row_off, window_width, window_height)
|
||||
@@ -0,0 +1,107 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from gf3_l1a2l2.config import PipelineConfig
|
||||
from gf3_l1a2l2.decompress import extract_compressed_files
|
||||
from gf3_l1a2l2.discovery import discover_scenes
|
||||
from gf3_l1a2l2.errors import GF3PipelineError
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = build_parser()
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if not hasattr(args, "handler"):
|
||||
parser.print_help()
|
||||
return 2
|
||||
|
||||
try:
|
||||
return args.handler(args)
|
||||
except GF3PipelineError as exc:
|
||||
print(f"ERROR: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(prog="gf3-l1a2l2", description="GF3 L1A to L2 pipeline")
|
||||
subparsers = parser.add_subparsers(dest="command")
|
||||
|
||||
run_parser = subparsers.add_parser("run", help="extract and process GF3 L1A products")
|
||||
run_parser.add_argument("--input", required=True, type=Path, help="input folder containing archives or scenes")
|
||||
run_parser.add_argument("--output", required=True, type=Path, help="output folder")
|
||||
run_parser.add_argument("--dem", type=Path, default=None, help="DEM file for RPC orthorectification")
|
||||
run_parser.add_argument("--x-res", type=float, default=0.0002, help="output x resolution")
|
||||
run_parser.add_argument("--y-res", type=float, default=0.0002, help="output y resolution")
|
||||
run_parser.add_argument("--dst-srs", default="EPSG:4326", help="destination spatial reference")
|
||||
run_parser.add_argument("--block-size", type=int, default=2048, help="calibration block size in pixels")
|
||||
run_parser.add_argument("--workers", type=int, default=1, help="number of scenes to process concurrently")
|
||||
run_parser.add_argument("--skip-decompress", action="store_true", help="do not extract archives before processing")
|
||||
run_parser.add_argument("--overwrite", action="store_true", help="overwrite existing extraction and output files")
|
||||
run_parser.add_argument("--flat-output", action="store_true", help="write all outputs directly to output folder")
|
||||
run_parser.add_argument("--fail-fast", action="store_true", help="stop on the first failed scene or polarization")
|
||||
run_parser.add_argument("--log-level", default="INFO", choices=["DEBUG", "INFO", "WARNING", "ERROR"])
|
||||
run_parser.set_defaults(handler=_run)
|
||||
|
||||
decompress_parser = subparsers.add_parser("decompress", help="extract .tar.gz and .zip archives")
|
||||
decompress_parser.add_argument("input", type=Path, help="folder containing archives")
|
||||
decompress_parser.add_argument("--overwrite", action="store_true", help="overwrite existing extracted folders")
|
||||
decompress_parser.set_defaults(handler=_decompress)
|
||||
|
||||
discover_parser = subparsers.add_parser("discover", help="list discovered GF3 scenes")
|
||||
discover_parser.add_argument("input", type=Path, help="input folder")
|
||||
discover_parser.set_defaults(handler=_discover)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def _run(args: argparse.Namespace) -> int:
|
||||
from gf3_l1a2l2.pipeline import run_pipeline
|
||||
|
||||
dem_path = args.dem or _default_dem_path()
|
||||
config = PipelineConfig(
|
||||
input_dir=args.input,
|
||||
output_dir=args.output,
|
||||
dem_path=dem_path,
|
||||
x_res=args.x_res,
|
||||
y_res=args.y_res,
|
||||
dst_srs=args.dst_srs,
|
||||
block_size=args.block_size,
|
||||
workers=args.workers,
|
||||
skip_decompress=args.skip_decompress,
|
||||
overwrite=args.overwrite,
|
||||
flat_output=args.flat_output,
|
||||
fail_fast=args.fail_fast,
|
||||
log_level=args.log_level,
|
||||
)
|
||||
summary = run_pipeline(config)
|
||||
return 1 if summary.failed else 0
|
||||
|
||||
|
||||
def _decompress(args: argparse.Namespace) -> int:
|
||||
results = extract_compressed_files(args.input, overwrite=args.overwrite)
|
||||
failed = [item for item in results if item.status == "failed"]
|
||||
for item in results:
|
||||
print(f"{item.status}\t{item.archive}\t{item.output_dir}\t{item.message}")
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
def _discover(args: argparse.Namespace) -> int:
|
||||
scenes = discover_scenes(args.input)
|
||||
for scene in scenes:
|
||||
polarizations = ",".join(scene.available_polarizations)
|
||||
print(f"{scene.root}\tmetadata={scene.metadata_xml.name}\tpol={polarizations}")
|
||||
return 0
|
||||
|
||||
|
||||
def _default_dem_path() -> Path | None:
|
||||
candidates = [
|
||||
Path.cwd() / "GMTED2010.jp2",
|
||||
Path(__file__).resolve().parents[2] / "GMTED2010.jp2",
|
||||
]
|
||||
for candidate in candidates:
|
||||
if candidate.is_file():
|
||||
return candidate
|
||||
return None
|
||||
@@ -0,0 +1,41 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, replace
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from gf3_l1a2l2.constants import (
|
||||
DEFAULT_BLOCK_SIZE,
|
||||
DEFAULT_DST_SRS,
|
||||
DEFAULT_X_RES,
|
||||
DEFAULT_Y_RES,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PipelineConfig:
|
||||
input_dir: Path
|
||||
output_dir: Path
|
||||
dem_path: Optional[Path] = None
|
||||
x_res: float = DEFAULT_X_RES
|
||||
y_res: float = DEFAULT_Y_RES
|
||||
dst_srs: str = DEFAULT_DST_SRS
|
||||
block_size: int = DEFAULT_BLOCK_SIZE
|
||||
workers: int = 1
|
||||
skip_decompress: bool = False
|
||||
overwrite: bool = False
|
||||
flat_output: bool = False
|
||||
keep_l1b: bool = True
|
||||
fail_fast: bool = False
|
||||
log_level: str = "INFO"
|
||||
|
||||
def resolved(self) -> "PipelineConfig":
|
||||
dem_path = self.dem_path.resolve() if self.dem_path else None
|
||||
return replace(
|
||||
self,
|
||||
input_dir=self.input_dir.resolve(),
|
||||
output_dir=self.output_dir.resolve(),
|
||||
dem_path=dem_path,
|
||||
workers=max(1, int(self.workers)),
|
||||
block_size=max(128, int(self.block_size)),
|
||||
)
|
||||
@@ -0,0 +1,6 @@
|
||||
POLARIZATIONS = ("HH", "HV", "VH", "VV")
|
||||
|
||||
DEFAULT_X_RES = 0.0002
|
||||
DEFAULT_Y_RES = 0.0002
|
||||
DEFAULT_DST_SRS = "EPSG:4326"
|
||||
DEFAULT_BLOCK_SIZE = 2048
|
||||
@@ -0,0 +1,112 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import shutil
|
||||
import tarfile
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Optional
|
||||
|
||||
try:
|
||||
from tqdm import tqdm
|
||||
except Exception: # pragma: no cover - only used when tqdm is unavailable
|
||||
def tqdm(iterable: Iterable, **_: object) -> Iterable:
|
||||
return iterable
|
||||
|
||||
from gf3_l1a2l2.errors import ProcessingError
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ExtractionResult:
|
||||
archive: Path
|
||||
output_dir: Path
|
||||
status: str
|
||||
message: str = ""
|
||||
|
||||
|
||||
def extract_compressed_files(
|
||||
folder_path: Path | str,
|
||||
*,
|
||||
overwrite: bool = False,
|
||||
logger: Optional[logging.Logger] = None,
|
||||
) -> list[ExtractionResult]:
|
||||
folder = Path(folder_path).resolve()
|
||||
if not folder.is_dir():
|
||||
raise ProcessingError(f"Input folder does not exist: {folder}")
|
||||
|
||||
archives = sorted(
|
||||
path
|
||||
for path in folder.iterdir()
|
||||
if path.is_file()
|
||||
and (path.name.lower().endswith(".tar.gz") or path.suffix.lower() == ".zip")
|
||||
)
|
||||
|
||||
results: list[ExtractionResult] = []
|
||||
if not archives:
|
||||
if logger:
|
||||
logger.info("No .tar.gz or .zip archives found in %s", folder)
|
||||
return results
|
||||
|
||||
for archive in tqdm(archives, desc="Extract archives", unit="file"):
|
||||
output_dir = _archive_output_dir(archive)
|
||||
try:
|
||||
if output_dir.exists():
|
||||
if not overwrite:
|
||||
results.append(ExtractionResult(archive, output_dir, "skipped", "output exists"))
|
||||
if logger:
|
||||
logger.info("Skip existing extraction: %s", output_dir)
|
||||
continue
|
||||
shutil.rmtree(output_dir)
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
if archive.name.lower().endswith(".tar.gz"):
|
||||
_extract_tar_gz(archive, output_dir)
|
||||
elif archive.suffix.lower() == ".zip":
|
||||
_extract_zip(archive, output_dir)
|
||||
else:
|
||||
continue
|
||||
|
||||
results.append(ExtractionResult(archive, output_dir, "done"))
|
||||
if logger:
|
||||
logger.info("Extracted %s -> %s", archive, output_dir)
|
||||
except Exception as exc:
|
||||
results.append(ExtractionResult(archive, output_dir, "failed", str(exc)))
|
||||
if logger:
|
||||
logger.exception("Failed to extract %s", archive)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _archive_output_dir(archive: Path) -> Path:
|
||||
if archive.name.lower().endswith(".tar.gz"):
|
||||
return archive.with_name(archive.name[:-7])
|
||||
return archive.with_suffix("")
|
||||
|
||||
|
||||
def _extract_tar_gz(archive: Path, output_dir: Path) -> None:
|
||||
base = output_dir.resolve()
|
||||
with tarfile.open(archive, "r:gz") as tar:
|
||||
members = tar.getmembers()
|
||||
for member in tqdm(members, desc=f"Extract {archive.name}", leave=False, unit="file"):
|
||||
if member.issym() or member.islnk():
|
||||
raise ProcessingError(f"Refusing to extract link from archive: {member.name}")
|
||||
_ensure_safe_target(base, member.name)
|
||||
tar.extract(member, path=base)
|
||||
|
||||
|
||||
def _extract_zip(archive: Path, output_dir: Path) -> None:
|
||||
base = output_dir.resolve()
|
||||
with zipfile.ZipFile(archive, "r") as zip_ref:
|
||||
names = zip_ref.namelist()
|
||||
for name in tqdm(names, desc=f"Extract {archive.name}", leave=False, unit="file"):
|
||||
_ensure_safe_target(base, name)
|
||||
zip_ref.extract(name, base)
|
||||
|
||||
|
||||
def _ensure_safe_target(base: Path, member_name: str) -> None:
|
||||
target = (base / member_name).resolve()
|
||||
if target == base:
|
||||
return
|
||||
if base not in target.parents:
|
||||
raise ProcessingError(f"Unsafe archive path: {member_name}")
|
||||
@@ -0,0 +1,90 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Optional
|
||||
|
||||
from gf3_l1a2l2.errors import DiscoveryError
|
||||
from gf3_l1a2l2.models import SceneFiles
|
||||
from gf3_l1a2l2.paths import detect_polarization
|
||||
|
||||
SKIP_DIR_NAMES = {".git", ".venv", ".tmp_tests", "__pycache__", ".pytest_cache", "logs"}
|
||||
|
||||
|
||||
def discover_scenes(input_dir: Path | str, logger: Optional[logging.Logger] = None) -> list[SceneFiles]:
|
||||
root_dir = Path(input_dir).resolve()
|
||||
if not root_dir.is_dir():
|
||||
raise DiscoveryError(f"Input directory does not exist: {root_dir}")
|
||||
|
||||
scenes: list[SceneFiles] = []
|
||||
for directory in _walk_directories(root_dir):
|
||||
try:
|
||||
files = [path for path in directory.iterdir() if path.is_file()]
|
||||
except OSError as exc:
|
||||
if logger:
|
||||
logger.warning("Skipping unreadable directory %s: %s", directory, exc)
|
||||
continue
|
||||
|
||||
metadata_files = sorted(path for path in files if path.name.lower().endswith(".meta.xml"))
|
||||
image_files = sorted(
|
||||
path
|
||||
for path in files
|
||||
if path.suffix.lower() in (".tif", ".tiff") and detect_polarization(path)
|
||||
)
|
||||
|
||||
if not metadata_files or not image_files:
|
||||
continue
|
||||
|
||||
metadata_xml = metadata_files[0]
|
||||
if len(metadata_files) > 1 and logger:
|
||||
logger.warning("Multiple metadata files in %s; using %s", directory, metadata_xml.name)
|
||||
|
||||
images = _collect_by_polarization(image_files, logger)
|
||||
rpcs = _collect_by_polarization(
|
||||
sorted(path for path in files if path.suffix.lower() in (".rpc", ".rpb")),
|
||||
logger,
|
||||
)
|
||||
|
||||
scenes.append(SceneFiles(root=directory, metadata_xml=metadata_xml, images=images, rpcs=rpcs))
|
||||
|
||||
scenes.sort(key=lambda scene: str(scene.root).lower())
|
||||
if logger:
|
||||
logger.info("Discovered %s scene(s) under %s", len(scenes), root_dir)
|
||||
return scenes
|
||||
|
||||
|
||||
def _walk_directories(root_dir: Path) -> Iterable[Path]:
|
||||
stack = [root_dir]
|
||||
while stack:
|
||||
directory = stack.pop()
|
||||
if directory.name in SKIP_DIR_NAMES:
|
||||
continue
|
||||
yield directory
|
||||
try:
|
||||
children = list(directory.iterdir())
|
||||
except OSError:
|
||||
continue
|
||||
stack.extend(path for path in children if path.is_dir() and path.name not in SKIP_DIR_NAMES)
|
||||
|
||||
|
||||
def _collect_by_polarization(
|
||||
files: Iterable[Path],
|
||||
logger: Optional[logging.Logger] = None,
|
||||
) -> dict[str, Path]:
|
||||
collected: dict[str, Path] = {}
|
||||
for path in files:
|
||||
polarization = detect_polarization(path)
|
||||
if not polarization:
|
||||
continue
|
||||
if polarization in collected:
|
||||
if logger:
|
||||
logger.warning(
|
||||
"Duplicate %s file in %s; using %s and ignoring %s",
|
||||
polarization,
|
||||
path.parent,
|
||||
collected[polarization].name,
|
||||
path.name,
|
||||
)
|
||||
continue
|
||||
collected[polarization] = path
|
||||
return collected
|
||||
@@ -0,0 +1,18 @@
|
||||
class GF3PipelineError(Exception):
|
||||
"""Base error for pipeline failures."""
|
||||
|
||||
|
||||
class DiscoveryError(GF3PipelineError):
|
||||
"""Raised when input products cannot be discovered."""
|
||||
|
||||
|
||||
class MetadataError(GF3PipelineError):
|
||||
"""Raised when GF3 metadata cannot be parsed."""
|
||||
|
||||
|
||||
class RpcError(GF3PipelineError):
|
||||
"""Raised when RPC metadata cannot be parsed."""
|
||||
|
||||
|
||||
class ProcessingError(GF3PipelineError):
|
||||
"""Raised when image processing fails."""
|
||||
@@ -0,0 +1,78 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from gf3_l1a2l2.errors import ProcessingError
|
||||
from gf3_l1a2l2.paths import output_name
|
||||
from gf3_l1a2l2.rpc import read_rpc_metadata
|
||||
|
||||
|
||||
def geometric_correction(
|
||||
l1b_path: Path | str,
|
||||
rpc_path: Path | str,
|
||||
output_dir: Path | str,
|
||||
*,
|
||||
dem_path: Optional[Path],
|
||||
x_res: float,
|
||||
y_res: float,
|
||||
dst_srs: str,
|
||||
overwrite: bool = False,
|
||||
logger: Optional[logging.Logger] = None,
|
||||
) -> Path:
|
||||
source = Path(l1b_path)
|
||||
out_dir = Path(output_dir)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
l2_path = out_dir / output_name(source, "L1B", "L2")
|
||||
|
||||
if l2_path.exists() and not overwrite:
|
||||
if logger:
|
||||
logger.info("Skip existing L2: %s", l2_path)
|
||||
return l2_path
|
||||
|
||||
if dem_path and not Path(dem_path).is_file():
|
||||
raise ProcessingError(f"DEM file does not exist: {dem_path}")
|
||||
|
||||
rpc_metadata = read_rpc_metadata(rpc_path)
|
||||
|
||||
try:
|
||||
from osgeo import gdal
|
||||
except Exception as exc:
|
||||
raise ProcessingError("Cannot import osgeo.gdal. Install GDAL Python bindings.") from exc
|
||||
|
||||
if logger:
|
||||
logger.info("Geocorrecting %s -> %s", source, l2_path)
|
||||
|
||||
dataset = gdal.Open(str(source), gdal.GA_Update)
|
||||
if dataset is None:
|
||||
raise ProcessingError(f"GDAL cannot open L1B image: {source}")
|
||||
|
||||
try:
|
||||
dataset.SetMetadata(rpc_metadata, "RPC")
|
||||
transformer_options = []
|
||||
if dem_path:
|
||||
transformer_options.append(f"RPC_DEM={Path(dem_path)}")
|
||||
|
||||
warp_options = gdal.WarpOptions(
|
||||
dstSRS=dst_srs,
|
||||
xRes=x_res,
|
||||
yRes=y_res,
|
||||
rpc=True,
|
||||
multithread=True,
|
||||
transformerOptions=transformer_options,
|
||||
creationOptions=["COMPRESS=LZW", "TILED=YES", "BIGTIFF=YES"],
|
||||
)
|
||||
result = gdal.Warp(str(l2_path), dataset, options=warp_options)
|
||||
if result is None:
|
||||
raise ProcessingError(f"GDAL Warp failed for {source}")
|
||||
result.FlushCache()
|
||||
result = None
|
||||
except ProcessingError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise ProcessingError(f"Failed to geocorrect {source}") from exc
|
||||
finally:
|
||||
dataset = None
|
||||
|
||||
return l2_path
|
||||
@@ -0,0 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def configure_logging(output_dir: Path, level: str = "INFO") -> logging.Logger:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
log_dir = output_dir / "logs"
|
||||
log_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger = logging.getLogger("gf3_l1a2l2")
|
||||
logger.setLevel(getattr(logging, level.upper(), logging.INFO))
|
||||
logger.propagate = False
|
||||
logger.handlers.clear()
|
||||
|
||||
formatter = logging.Formatter(
|
||||
fmt="%(asctime)s %(levelname)s %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
|
||||
console = logging.StreamHandler()
|
||||
console.setFormatter(formatter)
|
||||
console.setLevel(getattr(logging, level.upper(), logging.INFO))
|
||||
logger.addHandler(console)
|
||||
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
file_handler = logging.FileHandler(log_dir / f"gf3_l1a2l2_{timestamp}.log", encoding="utf-8")
|
||||
file_handler.setFormatter(formatter)
|
||||
file_handler.setLevel(logging.DEBUG)
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
return logger
|
||||
@@ -0,0 +1,145 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Optional
|
||||
|
||||
from gf3_l1a2l2.constants import POLARIZATIONS
|
||||
from gf3_l1a2l2.errors import MetadataError
|
||||
from gf3_l1a2l2.models import CalibrationMetadata, CalibrationValue
|
||||
from gf3_l1a2l2.paths import detect_polarization
|
||||
|
||||
|
||||
QUALIFY_KEYWORDS = ("qualifyvalue", "qualityvalue")
|
||||
CALIBRATION_KEYWORDS = ("calibrationconst", "calibrationconstant")
|
||||
|
||||
|
||||
def read_calibration_metadata(xml_path: Path | str) -> CalibrationMetadata:
|
||||
path = Path(xml_path)
|
||||
try:
|
||||
root = ET.parse(path).getroot()
|
||||
except ET.ParseError as exc:
|
||||
raise MetadataError(f"Invalid metadata XML: {path}") from exc
|
||||
except OSError as exc:
|
||||
raise MetadataError(f"Cannot read metadata XML: {path}") from exc
|
||||
|
||||
qualify_values = _find_values(root, QUALIFY_KEYWORDS) or _legacy_values(root, (17, 13))
|
||||
calibration_values = _find_values(root, CALIBRATION_KEYWORDS) or _legacy_values(root, (18, 3))
|
||||
|
||||
if not qualify_values:
|
||||
raise MetadataError(f"Cannot find QualifyValue values in {path}")
|
||||
if not calibration_values:
|
||||
raise MetadataError(f"Cannot find CalibrationConst values in {path}")
|
||||
|
||||
values: dict[str, CalibrationValue] = {}
|
||||
for polarization in POLARIZATIONS:
|
||||
if polarization not in qualify_values or polarization not in calibration_values:
|
||||
continue
|
||||
values[polarization] = CalibrationValue(
|
||||
qualify_value=qualify_values[polarization],
|
||||
calibration_const=calibration_values[polarization],
|
||||
)
|
||||
|
||||
if not values:
|
||||
raise MetadataError(f"No usable calibration values found in {path}")
|
||||
|
||||
return CalibrationMetadata(values=values)
|
||||
|
||||
|
||||
def _find_values(root: ET.Element, keywords: Iterable[str]) -> Optional[dict[str, float]]:
|
||||
lowered = tuple(keyword.lower() for keyword in keywords)
|
||||
for element in root.iter():
|
||||
name = _local_name(element.tag).lower()
|
||||
if not any(keyword in name for keyword in lowered):
|
||||
continue
|
||||
|
||||
values = _values_by_child_polarization(element)
|
||||
if values:
|
||||
return values
|
||||
|
||||
values = _values_by_child_order(element)
|
||||
if values:
|
||||
return values
|
||||
|
||||
values = _values_from_text(element.text)
|
||||
if values:
|
||||
return values
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _legacy_values(root: ET.Element, indexes: tuple[int, int]) -> Optional[dict[str, float]]:
|
||||
try:
|
||||
node = root
|
||||
for index in indexes:
|
||||
node = list(node)[index]
|
||||
values = [_parse_float(child.text) for child in list(node)[:4]]
|
||||
except (IndexError, TypeError, ValueError):
|
||||
return None
|
||||
|
||||
if len(values) < 4:
|
||||
return None
|
||||
return dict(zip(POLARIZATIONS, values))
|
||||
|
||||
|
||||
def _values_by_child_polarization(element: ET.Element) -> Optional[dict[str, float]]:
|
||||
values: dict[str, float] = {}
|
||||
for child in list(element):
|
||||
polarization = _element_polarization(child)
|
||||
if not polarization:
|
||||
continue
|
||||
try:
|
||||
values[polarization] = _parse_float(child.text)
|
||||
except ValueError:
|
||||
continue
|
||||
return values if values else None
|
||||
|
||||
|
||||
def _values_by_child_order(element: ET.Element) -> Optional[dict[str, float]]:
|
||||
numeric_values: list[float] = []
|
||||
for child in list(element):
|
||||
try:
|
||||
numeric_values.append(_parse_float(child.text))
|
||||
except ValueError:
|
||||
continue
|
||||
if len(numeric_values) < 4:
|
||||
return None
|
||||
return dict(zip(POLARIZATIONS, numeric_values[:4]))
|
||||
|
||||
|
||||
def _values_from_text(text: Optional[str]) -> Optional[dict[str, float]]:
|
||||
if not text:
|
||||
return None
|
||||
normalized = text.replace(",", " ").replace(";", " ")
|
||||
parts = [part for part in normalized.split() if part]
|
||||
if len(parts) < 4:
|
||||
return None
|
||||
try:
|
||||
values = [_parse_float(part) for part in parts[:4]]
|
||||
except ValueError:
|
||||
return None
|
||||
return dict(zip(POLARIZATIONS, values))
|
||||
|
||||
|
||||
def _element_polarization(element: ET.Element) -> Optional[str]:
|
||||
for value in element.attrib.values():
|
||||
polarization = detect_polarization(value)
|
||||
if polarization:
|
||||
return polarization
|
||||
return detect_polarization(_local_name(element.tag))
|
||||
|
||||
|
||||
def _parse_float(text: Optional[str]) -> float:
|
||||
if text is None:
|
||||
raise ValueError("missing value")
|
||||
value = text.strip().strip("=")
|
||||
if not value or value.upper() == "NULL":
|
||||
return math.nan
|
||||
return float(value)
|
||||
|
||||
|
||||
def _local_name(tag: str) -> str:
|
||||
if "}" in tag:
|
||||
return tag.rsplit("}", 1)[1]
|
||||
return tag
|
||||
@@ -0,0 +1,66 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Mapping, Optional
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CalibrationValue:
|
||||
qualify_value: float
|
||||
calibration_const: float
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CalibrationMetadata:
|
||||
values: Mapping[str, CalibrationValue]
|
||||
|
||||
def for_polarization(self, polarization: str) -> CalibrationValue:
|
||||
return self.values[polarization]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SceneFiles:
|
||||
root: Path
|
||||
metadata_xml: Path
|
||||
images: Mapping[str, Path]
|
||||
rpcs: Mapping[str, Path]
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self.root.name
|
||||
|
||||
@property
|
||||
def available_polarizations(self) -> tuple[str, ...]:
|
||||
return tuple(sorted(self.images))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TaskResult:
|
||||
scene: str
|
||||
polarization: str
|
||||
status: str
|
||||
l1b_path: Optional[Path] = None
|
||||
l2_path: Optional[Path] = None
|
||||
message: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class PipelineSummary:
|
||||
results: list[TaskResult] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def succeeded(self) -> int:
|
||||
return sum(1 for item in self.results if item.status == "done")
|
||||
|
||||
@property
|
||||
def skipped(self) -> int:
|
||||
return sum(1 for item in self.results if item.status == "skipped")
|
||||
|
||||
@property
|
||||
def failed(self) -> int:
|
||||
return sum(1 for item in self.results if item.status == "failed")
|
||||
|
||||
@property
|
||||
def total(self) -> int:
|
||||
return len(self.results)
|
||||
@@ -0,0 +1,43 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from gf3_l1a2l2.constants import POLARIZATIONS
|
||||
|
||||
|
||||
def detect_polarization(path_or_name: Path | str) -> Optional[str]:
|
||||
name = Path(path_or_name).name.upper()
|
||||
for polarization in POLARIZATIONS:
|
||||
if polarization in name:
|
||||
return polarization
|
||||
return None
|
||||
|
||||
|
||||
def output_name(source: Path, source_level: str, target_level: str) -> str:
|
||||
name = source.name
|
||||
index = name.lower().find(source_level.lower())
|
||||
if index >= 0:
|
||||
return name[:index] + target_level + name[index + len(source_level) :]
|
||||
|
||||
stem = source.stem
|
||||
suffix = "".join(source.suffixes)
|
||||
if suffix and stem.endswith(suffix):
|
||||
stem = stem[: -len(suffix)]
|
||||
return f"{stem}_{target_level}{source.suffix}"
|
||||
|
||||
|
||||
def scene_output_dir(input_dir: Path, output_dir: Path, scene_dir: Path, flat: bool) -> Path:
|
||||
if flat:
|
||||
return output_dir
|
||||
|
||||
try:
|
||||
relative = scene_dir.relative_to(input_dir)
|
||||
except ValueError:
|
||||
relative = Path(scene_dir.name)
|
||||
|
||||
if str(relative) in ("", "."):
|
||||
relative = Path(scene_dir.name)
|
||||
|
||||
safe_name = "_".join(part for part in relative.parts if part not in ("", "."))
|
||||
return output_dir / safe_name
|
||||
@@ -0,0 +1,182 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
from gf3_l1a2l2.calibration import radiometric_calibrate_l1a_to_l1b
|
||||
from gf3_l1a2l2.config import PipelineConfig
|
||||
from gf3_l1a2l2.constants import POLARIZATIONS
|
||||
from gf3_l1a2l2.decompress import extract_compressed_files
|
||||
from gf3_l1a2l2.discovery import discover_scenes
|
||||
from gf3_l1a2l2.errors import GF3PipelineError
|
||||
from gf3_l1a2l2.geocorrection import geometric_correction
|
||||
from gf3_l1a2l2.log import configure_logging
|
||||
from gf3_l1a2l2.metadata import read_calibration_metadata
|
||||
from gf3_l1a2l2.models import PipelineSummary, SceneFiles, TaskResult
|
||||
from gf3_l1a2l2.paths import scene_output_dir
|
||||
|
||||
|
||||
def run_pipeline(config: PipelineConfig) -> PipelineSummary:
|
||||
config = config.resolved()
|
||||
logger = configure_logging(config.output_dir, config.log_level)
|
||||
_validate_config(config)
|
||||
|
||||
logger.info("Input: %s", config.input_dir)
|
||||
logger.info("Output: %s", config.output_dir)
|
||||
if config.dem_path:
|
||||
logger.info("DEM: %s", config.dem_path)
|
||||
|
||||
if not config.skip_decompress:
|
||||
extract_compressed_files(config.input_dir, overwrite=config.overwrite, logger=logger)
|
||||
|
||||
scenes = discover_scenes(config.input_dir, logger=logger)
|
||||
if not scenes:
|
||||
logger.warning("No GF3 scenes found under %s", config.input_dir)
|
||||
return PipelineSummary()
|
||||
|
||||
summary = PipelineSummary()
|
||||
for results in _process_scenes(scenes, config, logger):
|
||||
summary.results.extend(results)
|
||||
|
||||
logger.info(
|
||||
"Pipeline finished: total=%s done=%s skipped=%s failed=%s",
|
||||
summary.total,
|
||||
summary.succeeded,
|
||||
summary.skipped,
|
||||
summary.failed,
|
||||
)
|
||||
return summary
|
||||
|
||||
|
||||
def _validate_config(config: PipelineConfig) -> None:
|
||||
if not config.input_dir.is_dir():
|
||||
raise GF3PipelineError(f"Input directory does not exist: {config.input_dir}")
|
||||
if config.dem_path and not config.dem_path.is_file():
|
||||
raise GF3PipelineError(f"DEM file does not exist: {config.dem_path}")
|
||||
if config.x_res <= 0 or config.y_res <= 0:
|
||||
raise GF3PipelineError("x_res and y_res must be positive")
|
||||
|
||||
|
||||
def _process_scenes(
|
||||
scenes: list[SceneFiles],
|
||||
config: PipelineConfig,
|
||||
logger: logging.Logger,
|
||||
) -> Iterable[list[TaskResult]]:
|
||||
if config.workers == 1:
|
||||
for scene in scenes:
|
||||
yield _process_scene(scene, config, logger)
|
||||
return
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=config.workers) as executor:
|
||||
future_map = {
|
||||
executor.submit(_process_scene, scene, config, logger): scene
|
||||
for scene in scenes
|
||||
}
|
||||
for future in concurrent.futures.as_completed(future_map):
|
||||
scene = future_map[future]
|
||||
try:
|
||||
yield future.result()
|
||||
except Exception as exc:
|
||||
logger.exception("Scene failed: %s", scene.root)
|
||||
if config.fail_fast:
|
||||
raise
|
||||
yield [
|
||||
TaskResult(
|
||||
scene=scene.name,
|
||||
polarization=polarization,
|
||||
status="failed",
|
||||
message=str(exc),
|
||||
)
|
||||
for polarization in scene.available_polarizations
|
||||
]
|
||||
|
||||
|
||||
def _process_scene(
|
||||
scene: SceneFiles,
|
||||
config: PipelineConfig,
|
||||
logger: logging.Logger,
|
||||
) -> list[TaskResult]:
|
||||
logger.info("Processing scene: %s", scene.root)
|
||||
out_dir = scene_output_dir(config.input_dir, config.output_dir, scene.root, config.flat_output)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
metadata = read_calibration_metadata(scene.metadata_xml)
|
||||
except Exception as exc:
|
||||
logger.exception("Failed to parse metadata for %s", scene.root)
|
||||
if config.fail_fast:
|
||||
raise
|
||||
return [
|
||||
TaskResult(scene=scene.name, polarization=pol, status="failed", message=str(exc))
|
||||
for pol in scene.available_polarizations
|
||||
]
|
||||
|
||||
results: list[TaskResult] = []
|
||||
for polarization in POLARIZATIONS:
|
||||
image_path = scene.images.get(polarization)
|
||||
if not image_path:
|
||||
continue
|
||||
|
||||
rpc_path = scene.rpcs.get(polarization)
|
||||
if not rpc_path:
|
||||
message = f"Missing RPC/RPB file for {polarization}"
|
||||
logger.error("%s in %s", message, scene.root)
|
||||
results.append(TaskResult(scene=scene.name, polarization=polarization, status="failed", message=message))
|
||||
if config.fail_fast:
|
||||
raise GF3PipelineError(message)
|
||||
continue
|
||||
|
||||
if polarization not in metadata.values:
|
||||
message = f"Missing calibration metadata for {polarization}"
|
||||
logger.error("%s in %s", message, scene.metadata_xml)
|
||||
results.append(TaskResult(scene=scene.name, polarization=polarization, status="failed", message=message))
|
||||
if config.fail_fast:
|
||||
raise GF3PipelineError(message)
|
||||
continue
|
||||
|
||||
try:
|
||||
calibration = metadata.for_polarization(polarization)
|
||||
l1b_path = radiometric_calibrate_l1a_to_l1b(
|
||||
image_path,
|
||||
out_dir,
|
||||
calibration,
|
||||
block_size=config.block_size,
|
||||
overwrite=config.overwrite,
|
||||
logger=logger,
|
||||
)
|
||||
l2_path = geometric_correction(
|
||||
l1b_path,
|
||||
rpc_path,
|
||||
out_dir,
|
||||
dem_path=config.dem_path,
|
||||
x_res=config.x_res,
|
||||
y_res=config.y_res,
|
||||
dst_srs=config.dst_srs,
|
||||
overwrite=config.overwrite,
|
||||
logger=logger,
|
||||
)
|
||||
results.append(
|
||||
TaskResult(
|
||||
scene=scene.name,
|
||||
polarization=polarization,
|
||||
status="done",
|
||||
l1b_path=l1b_path,
|
||||
l2_path=l2_path,
|
||||
)
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.exception("Failed %s %s", scene.name, polarization)
|
||||
results.append(
|
||||
TaskResult(
|
||||
scene=scene.name,
|
||||
polarization=polarization,
|
||||
status="failed",
|
||||
message=str(exc),
|
||||
)
|
||||
)
|
||||
if config.fail_fast:
|
||||
raise
|
||||
|
||||
return results
|
||||
@@ -0,0 +1,70 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from gf3_l1a2l2.errors import RpcError
|
||||
|
||||
|
||||
RPC_KEY_MAP = {
|
||||
"errBias": "ERR_BIAS",
|
||||
"errRand": "ERR_RAND",
|
||||
"lineOffset": "LINE_OFF",
|
||||
"sampOffset": "SAMP_OFF",
|
||||
"latOffset": "LAT_OFF",
|
||||
"longOffset": "LONG_OFF",
|
||||
"heightOffset": "HEIGHT_OFF",
|
||||
"lineScale": "LINE_SCALE",
|
||||
"sampScale": "SAMP_SCALE",
|
||||
"latScale": "LAT_SCALE",
|
||||
"longScale": "LONG_SCALE",
|
||||
"heightScale": "HEIGHT_SCALE",
|
||||
"lineNumCoef": "LINE_NUM_COEFF",
|
||||
"lineDenCoef": "LINE_DEN_COEFF",
|
||||
"sampNumCoef": "SAMP_NUM_COEFF",
|
||||
"sampDenCoef": "SAMP_DEN_COEFF",
|
||||
}
|
||||
|
||||
COEFFICIENT_KEYS = {
|
||||
"lineNumCoef",
|
||||
"lineDenCoef",
|
||||
"sampNumCoef",
|
||||
"sampDenCoef",
|
||||
}
|
||||
|
||||
|
||||
def read_rpc_metadata(rpc_path: Path | str) -> dict[str, str]:
|
||||
path = Path(rpc_path)
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8", errors="ignore")
|
||||
except OSError as exc:
|
||||
raise RpcError(f"Cannot read RPC file: {path}") from exc
|
||||
|
||||
metadata: dict[str, str] = {}
|
||||
missing: list[str] = []
|
||||
for source_key, target_key in RPC_KEY_MAP.items():
|
||||
value = _find_rpc_value(text, source_key)
|
||||
if value is None:
|
||||
missing.append(source_key)
|
||||
continue
|
||||
metadata[target_key] = _normalize_rpc_value(source_key, value)
|
||||
|
||||
if missing:
|
||||
raise RpcError(f"RPC file {path} is missing fields: {', '.join(missing)}")
|
||||
|
||||
return metadata
|
||||
|
||||
|
||||
def _find_rpc_value(text: str, key: str) -> str | None:
|
||||
pattern = re.compile(rf"\b{re.escape(key)}\b\s*[:=]?\s*(.*?);", re.IGNORECASE | re.DOTALL)
|
||||
match = pattern.search(text)
|
||||
if not match:
|
||||
return None
|
||||
return match.group(1)
|
||||
|
||||
|
||||
def _normalize_rpc_value(key: str, raw_value: str) -> str:
|
||||
value = raw_value.strip().strip("=")
|
||||
if key in COEFFICIENT_KEYS:
|
||||
value = value.replace("(", " ").replace(")", " ").replace(",", " ")
|
||||
return " ".join(value.split())
|
||||
@@ -0,0 +1,8 @@
|
||||
<root>
|
||||
<QualifyValue>
|
||||
<HH>1.0</HH><HV>2.0</HV><VH>NULL</VH><VV>4.0</VV>
|
||||
</QualifyValue>
|
||||
<CalibrationConst>
|
||||
<HH>10.0</HH><HV>20.0</HV><VH>30.0</VH><VV>40.0</VV>
|
||||
</CalibrationConst>
|
||||
</root>
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
errBias = -1;
|
||||
errRand = -1;
|
||||
lineOffset = 10;
|
||||
sampOffset = 20;
|
||||
latOffset = 30;
|
||||
longOffset = 40;
|
||||
heightOffset = 50;
|
||||
lineScale = 60;
|
||||
sampScale = 70;
|
||||
latScale = 80;
|
||||
longScale = 90;
|
||||
heightScale = 100;
|
||||
lineNumCoef = (1, 2, 3);
|
||||
lineDenCoef = (4, 5, 6);
|
||||
sampNumCoef = (7, 8, 9);
|
||||
sampDenCoef = (10, 11, 12);
|
||||
@@ -0,0 +1,19 @@
|
||||
import math
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from gf3_l1a2l2.metadata import read_calibration_metadata
|
||||
|
||||
|
||||
class MetadataTests(unittest.TestCase):
|
||||
def test_reads_named_calibration_values(self):
|
||||
path = Path(__file__).parent / "fixtures" / "sample.meta.xml"
|
||||
metadata = read_calibration_metadata(path)
|
||||
|
||||
self.assertEqual(metadata.for_polarization("HH").qualify_value, 1.0)
|
||||
self.assertEqual(metadata.for_polarization("HV").calibration_const, 20.0)
|
||||
self.assertTrue(math.isnan(metadata.for_polarization("VH").qualify_value))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,18 @@
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from gf3_l1a2l2.rpc import read_rpc_metadata
|
||||
|
||||
|
||||
class RpcTests(unittest.TestCase):
|
||||
def test_reads_rpc_metadata(self):
|
||||
path = Path(__file__).parent / "fixtures" / "sample.rpb"
|
||||
metadata = read_rpc_metadata(path)
|
||||
|
||||
self.assertEqual(metadata["ERR_BIAS"], "-1")
|
||||
self.assertEqual(metadata["LINE_OFF"], "10")
|
||||
self.assertEqual(metadata["LINE_NUM_COEFF"], "1 2 3")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,51 @@
|
||||
# GF-3 SARscape Go Wrapper
|
||||
|
||||
This wrapper embeds `assets/gf3_sarscape_cli.sav` and calls the local ENVI/IDL Runtime:
|
||||
|
||||
```text
|
||||
idlrt.exe gf3_sarscape_cli.sav -args input_meta_xml out_dir dem_file HH,HV
|
||||
```
|
||||
|
||||
It is the operational Windows path for SARscape-based GF-3 processing.
|
||||
|
||||
## Build
|
||||
|
||||
From this directory:
|
||||
|
||||
```powershell
|
||||
$env:GOTELEMETRY = "off"
|
||||
go build -o ..\..\dist\windows\gf3wrapper.exe .
|
||||
```
|
||||
|
||||
## Run
|
||||
|
||||
Single archive:
|
||||
|
||||
```powershell
|
||||
..\..\dist\windows\gf3wrapper.exe `
|
||||
-input "D:\GF3\GF3_SCENE.tar.gz" `
|
||||
-output "E:\GF3\L2_SARscape" `
|
||||
-dem "D:\DEM\COPDEM_GLO30_China_4326_DEM" `
|
||||
-pol "HH,HV"
|
||||
```
|
||||
|
||||
Batch directory:
|
||||
|
||||
```powershell
|
||||
..\..\dist\windows\gf3wrapper.exe `
|
||||
-input "D:\GF3\L1A_BATCH" `
|
||||
-output "E:\GF3\L2_SARscape"
|
||||
```
|
||||
|
||||
If `gf3wrapper.json` exists next to the executable, omitted `-input`, `-output`, `-dem`, and `-pol` values are loaded from that file.
|
||||
|
||||
In batch directory mode, a failed scene does not stop the whole run. Invalid or truncated archives, missing metadata, and SARscape failures are reported with the scene input path and reason, then the wrapper continues with the next scene. A final summary lists succeeded and failed scenes, and the process exits non-zero when any scene failed.
|
||||
|
||||
Reruns with the same output directory skip scenes that already have complete `*_geo.sml` outputs and matching data files for the requested polarizations. This lets a repaired archive be placed back into the original input folder without reprocessing scenes that already finished.
|
||||
|
||||
## Notes
|
||||
|
||||
- The server still needs ENVI, IDL Runtime, and SARscape installed and licensed.
|
||||
- `.tar.gz` inputs are extracted under each scene output directory in `.gf3_extract`.
|
||||
- The embedded SAV is extracted to `<output>\.gf3_runtime\gf3_sarscape_cli.sav`.
|
||||
- More operational details are in `docs/sarscape_go_wrapper.md`.
|
||||
Binary file not shown.
@@ -0,0 +1,3 @@
|
||||
module gf3wrapper
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,5 @@
|
||||
.compile 'C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\gf3_sarscape_pipeline.pro'
|
||||
.compile 'C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\gf3_sarscape_cli.pro'
|
||||
resolve_all, skip_routines='envi', /continue_on_error
|
||||
save, /routines, filename='C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\gf3_sarscape_cli.sav'
|
||||
exit
|
||||
@@ -0,0 +1,5 @@
|
||||
.f
|
||||
.compile 'C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\gf3_sarscape_pipeline.pro'
|
||||
.compile 'C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\gf3_sarscape_cli.pro'
|
||||
save, /routines, filename='C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\gf3_sarscape_cli.sav'
|
||||
exit
|
||||
@@ -0,0 +1,79 @@
|
||||
function gf3_cli_split_csv, value
|
||||
compile_opt idl2
|
||||
|
||||
text = strtrim(value, 2)
|
||||
if text eq '' then return, ['HH', 'HV']
|
||||
|
||||
parts = ['']
|
||||
first = 1
|
||||
while 1 do begin
|
||||
pos = strpos(text, ',')
|
||||
if pos lt 0 then begin
|
||||
token = strtrim(text, 2)
|
||||
if first then parts = [token] else parts = [parts, token]
|
||||
break
|
||||
endif
|
||||
token = strtrim(strmid(text, 0, pos), 2)
|
||||
if first then begin
|
||||
parts = [token]
|
||||
first = 0
|
||||
endif else begin
|
||||
parts = [parts, token]
|
||||
endelse
|
||||
text = strtrim(strmid(text, pos + 1), 2)
|
||||
endwhile
|
||||
|
||||
return, parts
|
||||
end
|
||||
|
||||
|
||||
pro gf3_sarscape_cli
|
||||
compile_opt idl2
|
||||
|
||||
args = command_line_args(count=argc)
|
||||
if argc lt 3 then begin
|
||||
print, 'Usage: idlrt.exe gf3_sarscape_cli.sav -args input_meta_xml out_dir dem_file [polarizations]'
|
||||
print, 'Example polarizations: HH,HV or HH or HV'
|
||||
return
|
||||
endif
|
||||
|
||||
input_meta_xml = args[0]
|
||||
out_dir = args[1]
|
||||
dem_file = args[2]
|
||||
file_mkdir, out_dir
|
||||
|
||||
log_file = out_dir + '\gf3_sarscape_cli.log'
|
||||
openw, lun, log_file, /get_lun
|
||||
catch, err
|
||||
if err ne 0 then begin
|
||||
printf, lun, 'ERROR: ', !error_state.msg
|
||||
free_lun, lun
|
||||
catch, /cancel
|
||||
return
|
||||
endif
|
||||
|
||||
printf, lun, 'CLI start'
|
||||
printf, lun, 'input_meta_xml=', input_meta_xml
|
||||
printf, lun, 'out_dir=', out_dir
|
||||
printf, lun, 'dem_file=', dem_file
|
||||
|
||||
if argc ge 4 then begin
|
||||
polarizations = gf3_cli_split_csv(args[3])
|
||||
endif else begin
|
||||
polarizations = ['HH', 'HV']
|
||||
endelse
|
||||
|
||||
printf, lun, 'polarizations=', strjoin(polarizations, ',')
|
||||
|
||||
e = envi(/headless)
|
||||
printf, lun, 'ENVI headless initialized'
|
||||
|
||||
gf3_sarscape_run, $
|
||||
input_meta_xml=input_meta_xml, $
|
||||
out_dir=out_dir, $
|
||||
dem_file=dem_file, $
|
||||
polarizations=polarizations
|
||||
|
||||
printf, lun, 'CLI end'
|
||||
free_lun, lun
|
||||
end
|
||||
@@ -0,0 +1,506 @@
|
||||
function gf3_normalize_errmsg, value
|
||||
compile_opt idl2
|
||||
|
||||
if n_elements(value) eq 0 then return, ''
|
||||
|
||||
sz = size(value, /tname)
|
||||
|
||||
if sz eq 'STRING' then begin
|
||||
if n_elements(value) eq 1 then return, value
|
||||
return, strjoin(value, ' ')
|
||||
endif
|
||||
|
||||
if sz eq 'BYTE' then begin
|
||||
if n_elements(value) eq 0 then return, ''
|
||||
return, string(value)
|
||||
endif
|
||||
|
||||
return, string(value)
|
||||
end
|
||||
|
||||
|
||||
function gf3_float_str, value
|
||||
compile_opt idl2
|
||||
return, string(double(value), format='(F0.1)')
|
||||
end
|
||||
|
||||
|
||||
function gf3_base_root, polarization
|
||||
compile_opt idl2
|
||||
common gf3_pipeline_cfg, cfg
|
||||
|
||||
pol = strupcase(strtrim(polarization, 2))
|
||||
case pol of
|
||||
'HH': return, cfg.imported_hh_root
|
||||
'HV': return, cfg.imported_hv_root
|
||||
else: message, 'Unsupported polarization: ' + pol
|
||||
endcase
|
||||
end
|
||||
|
||||
|
||||
function gf3_stage_root, polarization, stage_name
|
||||
compile_opt idl2
|
||||
common gf3_pipeline_cfg, cfg
|
||||
|
||||
pol = strlowcase(strtrim(polarization, 2))
|
||||
prefix = cfg.output_prefix
|
||||
if prefix eq '' then prefix = 'gf3'
|
||||
|
||||
case strlowcase(strtrim(stage_name, 2)) of
|
||||
'multilook': return, cfg.out_dir + '\' + prefix + '_' + pol + '_ml'
|
||||
'filter': return, cfg.out_dir + '\' + prefix + '_' + pol + '_filt'
|
||||
'geocode': return, cfg.out_dir + '\' + prefix + '_' + pol + '_geo'
|
||||
else: message, 'Unsupported stage: ' + stage_name
|
||||
endcase
|
||||
end
|
||||
|
||||
|
||||
function gf3_scene_token, meta_xml
|
||||
compile_opt idl2
|
||||
|
||||
base = file_basename(meta_xml)
|
||||
pos = strpos(strlowcase(base), '.meta.xml')
|
||||
if pos ge 0 then base = strmid(base, 0, pos)
|
||||
return, base
|
||||
end
|
||||
|
||||
|
||||
function gf3_imported_root_for, polarization
|
||||
compile_opt idl2
|
||||
common gf3_pipeline_cfg, cfg
|
||||
|
||||
pol = strupcase(strtrim(polarization, 2))
|
||||
pattern = cfg.out_dir + '\*_' + pol + '_slc.sml'
|
||||
matches = file_search(pattern, count=count)
|
||||
if count eq 0 then return, ''
|
||||
|
||||
newest = matches[0]
|
||||
newest_time = (file_info(newest)).mtime
|
||||
for i = 1, count - 1 do begin
|
||||
info = file_info(matches[i])
|
||||
if info.mtime gt newest_time then begin
|
||||
newest = matches[i]
|
||||
newest_time = info.mtime
|
||||
endif
|
||||
endfor
|
||||
|
||||
return, strmid(newest, 0, strlen(newest) - 4)
|
||||
end
|
||||
|
||||
|
||||
pro gf3_refresh_import_roots
|
||||
compile_opt idl2
|
||||
common gf3_pipeline_cfg, cfg
|
||||
|
||||
hh = gf3_imported_root_for('HH')
|
||||
hv = gf3_imported_root_for('HV')
|
||||
|
||||
if hh ne '' then cfg.imported_hh_root = hh
|
||||
if hv ne '' then cfg.imported_hv_root = hv
|
||||
end
|
||||
|
||||
|
||||
pro gf3_run_module, module_name, set_params_proc, errmsg=errmsg, working_directory=working_directory
|
||||
compile_opt idl2
|
||||
|
||||
errmsg = ''
|
||||
ob = obj_new('SARscapeBatch', Module=module_name)
|
||||
if ~obj_valid(ob) then begin
|
||||
errmsg = 'Create object failed: ' + module_name
|
||||
print, errmsg
|
||||
return
|
||||
endif
|
||||
|
||||
ob->SetParam, 'GENERAL_PARAMETERS_CMD.AVAILABLE_MEMORY_SIZE_GB', '8.0000000'
|
||||
ob->SetParam, 'GENERAL_PARAMETERS_CMD.LOAD_IMAGES', 'OK'
|
||||
ob->SetParam, 'GENERAL_PARAMETERS_CMD.DELETE_TEMPORARY_FILES', 'OK'
|
||||
ob->SetParam, 'GENERAL_PARAMETERS_CMD.MAKE_TIFF', 'OK'
|
||||
ob->SetParam, 'GENERAL_PARAMETERS_CMD.QUICK_LOOK_FORMAT', 'ql_tiff'
|
||||
ob->SetParam, 'GENERAL_PARAMETERS_CMD.SARSCAPE_TRACE_LEVEL', '3.0000000'
|
||||
ob->SetParam, 'GENERAL_PARAMETERS_CMD.RENAME_THE_FILE_USING_PARAMETERS_FLAG', 'OK'
|
||||
ob->SetParam, 'GENERAL_PARAMETERS_CMD.INSERT_GEO_POINTS_FLAG', 'NotOK'
|
||||
ob->SetParam, 'GENERAL_PARAMETERS_CMD.VERBOSE_TRACE', 'OK'
|
||||
ob->SetParam, 'GENERAL_PARAMETERS_CMD.VERBOSE_STEP', 'OK'
|
||||
ob->SetParam, 'GENERAL_PARAMETERS_CMD.VERBOSE_BAR', 'OK'
|
||||
ob->SetParam, 'GENERAL_PARAMETERS_CMD.APPLY_CALIBRATION_CONSTANT_FLAG', 'OK'
|
||||
ob->SetParam, 'GENERAL_PARAMETERS_CMD.FILL_DUMMY_DURING_IMPORT', 'OK'
|
||||
|
||||
call_procedure, set_params_proc, ob
|
||||
|
||||
ok = ob->VerifyParams(Silent=0)
|
||||
if ~ok then begin
|
||||
errmsg = 'VerifyParams failed: ' + module_name
|
||||
print, errmsg
|
||||
return
|
||||
endif
|
||||
|
||||
if n_elements(working_directory) eq 0 then begin
|
||||
ok = ob->Execute()
|
||||
endif else begin
|
||||
exec_errmsg = ''
|
||||
ok = ob->Execute(WORKING_DIRECTORY=working_directory, ERRMSG=exec_errmsg)
|
||||
errmsg = gf3_normalize_errmsg(exec_errmsg)
|
||||
endelse
|
||||
|
||||
if ok then begin
|
||||
print, 'Success: ' + module_name
|
||||
endif else begin
|
||||
if errmsg eq '' then begin
|
||||
err_code = ''
|
||||
err_text = get_SARscape_error_string('OK', ERROR_CODE=err_code)
|
||||
errmsg = 'Execution failed [' + module_name + '] EC [' + err_code + ']: ' + err_text
|
||||
endif
|
||||
print, errmsg
|
||||
endelse
|
||||
end
|
||||
|
||||
|
||||
pro gf3_pipeline_set_import_params, ob
|
||||
compile_opt idl2
|
||||
common gf3_pipeline_cfg, cfg
|
||||
|
||||
ob->SetParam, 'MAIN_BASIC_IMPORT_GAOFEN3_CMD_CMD.SARSCAPEENVIRONMENT', 'IDL_ENVI_ENV'
|
||||
ob->SetParam, 'MAIN_BASIC_IMPORT_GAOFEN3_CMD_CMD.INPUT_FILE_LIST', [cfg.input_meta_xml]
|
||||
ob->SetParam, 'MAIN_BASIC_IMPORT_GAOFEN3_CMD_CMD.OUTPUT_FILE_LIST', [cfg.import_root]
|
||||
ob->SetParam, 'MAIN_BASIC_IMPORT_GAOFEN3_CMD_CMD.CROSS_COPOLARIZATION_FLAG', cfg.import_polarization_mode
|
||||
ob->SetParam, 'MAIN_BASIC_IMPORT_GAOFEN3_CMD_CMD.DOPP_ESTIMATION_FROM_RASTER_FLAG', 'OK'
|
||||
ob->SetParam, 'MAIN_BASIC_IMPORT_GAOFEN3_CMD_CMD.KEEP_ALL_LINES_FLAG', 'NotOK'
|
||||
end
|
||||
|
||||
|
||||
pro gf3_pipeline_set_multilook_params, ob
|
||||
compile_opt idl2
|
||||
common gf3_pipeline_cfg, cfg
|
||||
common gf3_pipeline_state, state
|
||||
|
||||
ob->SetParam, 'MAIN_BASIC_MULTILOOKING.SARSCAPEENVIRONMENT', 'IDL_ENVI_ENV'
|
||||
ob->SetParam, 'MAIN_BASIC_MULTILOOKING.INPUT_FILE_LIST', [state.current_input_root]
|
||||
ob->SetParam, 'MAIN_BASIC_MULTILOOKING.OUTPUT_FILE_LIST', [state.current_output_root]
|
||||
ob->SetParam, 'MAIN_BASIC_MULTILOOKING.MULTILOOK_METHOD', cfg.multilook_method
|
||||
ob->SetParam, 'MAIN_BASIC_MULTILOOKING.RANGE_MULTILOOK', gf3_float_str(cfg.range_looks)
|
||||
ob->SetParam, 'MAIN_BASIC_MULTILOOKING.AZIMUTH_MULTILOOK', gf3_float_str(cfg.azimuth_looks)
|
||||
ob->SetParam, 'MAIN_BASIC_MULTILOOKING.GRID_SIZE_FOR_SUGGESTED_LOOKS', gf3_float_str(cfg.grid_size_for_suggested_looks)
|
||||
ob->SetParam, 'MAIN_BASIC_MULTILOOKING.FILL_DUMMY_FLAG', 'OK'
|
||||
ob->SetParam, 'MAIN_BASIC_MULTILOOKING.FILL_DUMMY_METHOD', 'min_image_value'
|
||||
ob->SetParam, 'MAIN_BASIC_MULTILOOKING.ROWS_WINDOW_NUMBER', '3.0000000'
|
||||
ob->SetParam, 'MAIN_BASIC_MULTILOOKING.COLS_WINDOW_NUMBER', '3.0000000'
|
||||
end
|
||||
|
||||
|
||||
pro gf3_pipeline_set_filter_params, ob
|
||||
compile_opt idl2
|
||||
common gf3_pipeline_cfg, cfg
|
||||
common gf3_pipeline_state, state
|
||||
|
||||
ob->SetParam, 'MAIN_BASIC_DESPECKLE_CONVENTIONAL_CMD.SARSCAPEENVIRONMENT', 'IDL_ENVI_ENV'
|
||||
ob->SetParam, 'MAIN_BASIC_DESPECKLE_CONVENTIONAL_CMD.INPUT_FILE_LIST', [state.current_input_root]
|
||||
ob->SetParam, 'MAIN_BASIC_DESPECKLE_CONVENTIONAL_CMD.OUTPUT_FILE_LIST', [state.current_output_root]
|
||||
ob->SetParam, 'MAIN_BASIC_DESPECKLE_CONVENTIONAL_CMD.FILT_TYPE', cfg.filter_type
|
||||
ob->SetParam, 'MAIN_BASIC_DESPECKLE_CONVENTIONAL_CMD.EQUIVALENT_LOOKS', '-1.0000000'
|
||||
ob->SetParam, 'PARAMETERS_GENERIC_FILTERS.WIN_SIZE', gf3_float_str(cfg.filter_window)
|
||||
ob->SetParam, 'PARAMETERS_GENERIC_FILTERS.ROWS_WINDOW_NUMBER', gf3_float_str(cfg.filter_window)
|
||||
ob->SetParam, 'PARAMETERS_GENERIC_FILTERS.COLS_WINDOW_NUMBER', gf3_float_str(cfg.filter_window)
|
||||
ob->SetParam, 'PARAMETERS_GENERIC_FILTERS.WIN_MODE_SIZE', gf3_float_str(cfg.filter_window)
|
||||
end
|
||||
|
||||
|
||||
pro gf3_pipeline_set_geocode_params, ob
|
||||
compile_opt idl2
|
||||
common gf3_pipeline_cfg, cfg
|
||||
common gf3_pipeline_state, state
|
||||
|
||||
ob->SetParam, 'OUT_CARTOGRAPHIC_SYSTEM.OCS_CARTOGRAPHIC_SYSTEM_SPECIFICATION', ''
|
||||
ob->SetParam, 'OUT_CARTOGRAPHIC_SYSTEM.OCS_STATE', cfg.ocs_state
|
||||
ob->SetParam, 'OUT_CARTOGRAPHIC_SYSTEM.OCS_HEMISPHERE', cfg.ocs_hemisphere
|
||||
ob->SetParam, 'OUT_CARTOGRAPHIC_SYSTEM.OCS_PROJECTION', cfg.ocs_projection
|
||||
ob->SetParam, 'OUT_CARTOGRAPHIC_SYSTEM.OCS_ELLIPSOID', 'WGS84'
|
||||
ob->SetParam, 'OUT_CARTOGRAPHIC_SYSTEM.OCS_REFERENCE_HEIGHT', '0.0'
|
||||
|
||||
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.SARSCAPEENVIRONMENT', 'IDL_ENVI_ENV'
|
||||
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.INPUT_FILE_LIST', [state.current_input_root]
|
||||
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.OUTPUT_FILE_LIST', [state.current_output_root]
|
||||
if cfg.dem_file ne '' then ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.DEM_FILE_NAME', cfg.dem_file
|
||||
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.GEOCODE_GRID_SIZE_X', gf3_float_str(cfg.geocode_grid_x)
|
||||
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.GEOCODE_GRID_SIZE_Y', gf3_float_str(cfg.geocode_grid_y)
|
||||
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.GEOCODE_RESAMPLING_TYPE', cfg.geocode_resampling
|
||||
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.CALIBRATION_FLAG', cfg.calibration_flag
|
||||
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.GEO_SCATTERING_AREA_METHOD', 'sine_area_estimation'
|
||||
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.TRUE_AREA_EXPONENT', '1.0000000'
|
||||
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.MAX_VALUE_IN_CALIBRATION', '100.00000'
|
||||
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.RAD_NORMALIZATION_FLAG', cfg.rad_normalization_flag
|
||||
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.GEOCODE_RAD_NORM_DEG', '2.0000000'
|
||||
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.GEOCODE_RAD_NORM_ANG', '-1.0000000'
|
||||
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.GEOCODE_SIGMA_FLAG', cfg.output_sigma_flag
|
||||
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.GEOCODE_GAMMA_FLAG', cfg.output_gamma_flag
|
||||
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.GEOCODE_BETA_FLAG', cfg.output_beta_flag
|
||||
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.GEOCODE_COREG_IMAGES_FLAG', 'NotOK'
|
||||
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.GENERATE_LIA_FLAG', 'NotOK'
|
||||
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.OUTPUT_TYPE', cfg.output_type
|
||||
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.DUMMY_REMOVAL_FLAG', 'NotOK'
|
||||
ob->SetParam, 'MAIN_BASIC_CALIBRATION_AND_GEO_CMD.GEOCODE_BLOCK_SIZE', '30000.000'
|
||||
end
|
||||
|
||||
|
||||
pro gf3_init_default_cfg
|
||||
compile_opt idl2
|
||||
common gf3_pipeline_cfg, cfg
|
||||
|
||||
cfg = { $
|
||||
input_meta_xml: 'C:\Users\Administrator\Desktop\GF3_Test\GF3_scene_20260513\GF3_SYC_FSI_051370_E124.6_N51.7_20260513_L1A_HHHV_L10007355955.meta.xml', $
|
||||
work_dir: 'C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\work', $
|
||||
temp_dir: 'C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\temp', $
|
||||
out_dir: 'C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\out', $
|
||||
output_prefix: '', $
|
||||
dem_file: 'D:\DEM\COPDEM_GLO30_China_4326_DEM', $
|
||||
import_root: 'C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\out\gf3_import', $
|
||||
import_polarization_mode: 'ALL_POL', $
|
||||
imported_hh_root: 'C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\out\gaofer3_20260513_214340051_D_HH_slc', $
|
||||
imported_hv_root: 'C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\out\gaofer3_20260513_214340051_D_HV_slc', $
|
||||
multilook_method: 'time_domain', $
|
||||
range_looks: 2.0, $
|
||||
azimuth_looks: 2.0, $
|
||||
grid_size_for_suggested_looks: 15.0, $
|
||||
filter_type: 'lee', $
|
||||
filter_window: 5.0, $
|
||||
geocode_grid_x: 15.0, $
|
||||
geocode_grid_y: 15.0, $
|
||||
geocode_resampling: '4th_order_cc', $
|
||||
calibration_flag: 'OK', $
|
||||
rad_normalization_flag: 'OK', $
|
||||
output_sigma_flag: 'OK', $
|
||||
output_gamma_flag: 'NotOK', $
|
||||
output_beta_flag: 'NotOK', $
|
||||
output_type: 'output_type_linear', $
|
||||
ocs_state: 'GEO-GLOBAL', $
|
||||
ocs_hemisphere: 'NONE', $
|
||||
ocs_projection: 'GEO' $
|
||||
}
|
||||
end
|
||||
|
||||
|
||||
pro gf3_configure, input_meta_xml=input_meta_xml, out_dir=out_dir, dem_file=dem_file, $
|
||||
work_dir=work_dir, temp_dir=temp_dir, range_looks=range_looks, $
|
||||
azimuth_looks=azimuth_looks, filter_type=filter_type, filter_window=filter_window, $
|
||||
geocode_grid_x=geocode_grid_x, geocode_grid_y=geocode_grid_y, $
|
||||
imported_hh_root=imported_hh_root, imported_hv_root=imported_hv_root
|
||||
|
||||
compile_opt idl2
|
||||
common gf3_pipeline_cfg, cfg
|
||||
|
||||
gf3_init_default_cfg
|
||||
|
||||
if n_elements(input_meta_xml) gt 0 then cfg.input_meta_xml = input_meta_xml
|
||||
if n_elements(out_dir) gt 0 then cfg.out_dir = out_dir
|
||||
if n_elements(dem_file) gt 0 then cfg.dem_file = dem_file
|
||||
if n_elements(work_dir) gt 0 then cfg.work_dir = work_dir
|
||||
if n_elements(temp_dir) gt 0 then cfg.temp_dir = temp_dir
|
||||
if n_elements(range_looks) gt 0 then cfg.range_looks = double(range_looks)
|
||||
if n_elements(azimuth_looks) gt 0 then cfg.azimuth_looks = double(azimuth_looks)
|
||||
if n_elements(filter_type) gt 0 then cfg.filter_type = filter_type
|
||||
if n_elements(filter_window) gt 0 then cfg.filter_window = double(filter_window)
|
||||
if n_elements(geocode_grid_x) gt 0 then cfg.geocode_grid_x = double(geocode_grid_x)
|
||||
if n_elements(geocode_grid_y) gt 0 then cfg.geocode_grid_y = double(geocode_grid_y)
|
||||
if n_elements(imported_hh_root) gt 0 then cfg.imported_hh_root = imported_hh_root
|
||||
if n_elements(imported_hv_root) gt 0 then cfg.imported_hv_root = imported_hv_root
|
||||
|
||||
cfg.import_root = cfg.out_dir + '\gf3_import'
|
||||
cfg.output_prefix = gf3_scene_token(cfg.input_meta_xml)
|
||||
|
||||
if n_elements(work_dir) eq 0 then cfg.work_dir = cfg.out_dir + '\work'
|
||||
if n_elements(temp_dir) eq 0 then cfg.temp_dir = cfg.out_dir + '\temp'
|
||||
|
||||
if n_elements(imported_hh_root) eq 0 then cfg.imported_hh_root = ''
|
||||
if n_elements(imported_hv_root) eq 0 then cfg.imported_hv_root = ''
|
||||
end
|
||||
|
||||
|
||||
pro gf3_prepare_dirs
|
||||
compile_opt idl2
|
||||
common gf3_pipeline_cfg, cfg
|
||||
|
||||
file_mkdir, cfg.work_dir
|
||||
file_mkdir, cfg.temp_dir
|
||||
file_mkdir, cfg.out_dir
|
||||
end
|
||||
|
||||
|
||||
pro gf3_start_batch
|
||||
compile_opt idl2
|
||||
common gf3_pipeline_cfg, cfg
|
||||
SARscape_Batch_Init, Temp_Directory=cfg.temp_dir
|
||||
end
|
||||
|
||||
|
||||
pro gf3_stop_batch
|
||||
compile_opt idl2
|
||||
SARscape_Batch_Exit
|
||||
end
|
||||
|
||||
|
||||
pro gf3_run_import, errmsg=errmsg
|
||||
compile_opt idl2
|
||||
common gf3_pipeline_cfg, cfg
|
||||
|
||||
errmsg = ''
|
||||
gf3_run_module, 'ImportGaofen3', 'gf3_pipeline_set_import_params', errmsg=errmsg, working_directory=cfg.work_dir
|
||||
end
|
||||
|
||||
|
||||
pro gf3_run_multilook, polarization, errmsg=errmsg
|
||||
compile_opt idl2
|
||||
common gf3_pipeline_cfg, cfg
|
||||
common gf3_pipeline_state, state
|
||||
|
||||
state = {current_input_root: gf3_base_root(polarization), current_output_root: gf3_stage_root(polarization, 'multilook')}
|
||||
errmsg = ''
|
||||
gf3_run_module, 'BaseMultilooking', 'gf3_pipeline_set_multilook_params', errmsg=errmsg, working_directory=cfg.work_dir
|
||||
end
|
||||
|
||||
|
||||
pro gf3_run_filter, polarization, errmsg=errmsg
|
||||
compile_opt idl2
|
||||
common gf3_pipeline_cfg, cfg
|
||||
common gf3_pipeline_state, state
|
||||
|
||||
state = {current_input_root: gf3_stage_root(polarization, 'multilook'), current_output_root: gf3_stage_root(polarization, 'filter')}
|
||||
errmsg = ''
|
||||
gf3_run_module, 'DespeckleConventionalSingle', 'gf3_pipeline_set_filter_params', errmsg=errmsg, working_directory=cfg.work_dir
|
||||
end
|
||||
|
||||
|
||||
pro gf3_run_geocode, polarization, errmsg=errmsg
|
||||
compile_opt idl2
|
||||
common gf3_pipeline_cfg, cfg
|
||||
common gf3_pipeline_state, state
|
||||
|
||||
state = {current_input_root: gf3_stage_root(polarization, 'filter'), current_output_root: gf3_stage_root(polarization, 'geocode')}
|
||||
errmsg = ''
|
||||
gf3_run_module, 'BasicGeocoding', 'gf3_pipeline_set_geocode_params', errmsg=errmsg, working_directory=cfg.work_dir
|
||||
end
|
||||
|
||||
|
||||
pro gf3_run_single_pol_pipeline, polarization, do_import=do_import, errmsg=errmsg
|
||||
compile_opt idl2
|
||||
|
||||
errmsg = ''
|
||||
if keyword_set(do_import) then begin
|
||||
gf3_run_import, errmsg=errmsg
|
||||
if errmsg ne '' then return
|
||||
gf3_refresh_import_roots
|
||||
endif
|
||||
|
||||
gf3_run_multilook, polarization, errmsg=errmsg
|
||||
if errmsg ne '' then return
|
||||
|
||||
gf3_run_filter, polarization, errmsg=errmsg
|
||||
if errmsg ne '' then return
|
||||
|
||||
gf3_run_geocode, polarization, errmsg=errmsg
|
||||
end
|
||||
|
||||
|
||||
function gf3_should_run_pol, polarizations, polarization
|
||||
compile_opt idl2
|
||||
|
||||
if n_elements(polarizations) eq 0 then return, 1
|
||||
|
||||
wanted = strupcase(strtrim(polarizations, 2))
|
||||
pol = strupcase(strtrim(polarization, 2))
|
||||
|
||||
for i = 0, n_elements(wanted) - 1 do begin
|
||||
if wanted[i] eq 'ALL' then return, 1
|
||||
if wanted[i] eq pol then return, 1
|
||||
endfor
|
||||
|
||||
return, 0
|
||||
end
|
||||
|
||||
|
||||
pro gf3_sarscape_run, input_meta_xml=input_meta_xml, out_dir=out_dir, dem_file=dem_file, $
|
||||
polarizations=polarizations, skip_import=skip_import, work_dir=work_dir, temp_dir=temp_dir, $
|
||||
range_looks=range_looks, azimuth_looks=azimuth_looks, filter_type=filter_type, $
|
||||
filter_window=filter_window, geocode_grid_x=geocode_grid_x, geocode_grid_y=geocode_grid_y
|
||||
|
||||
compile_opt idl2
|
||||
|
||||
gf3_configure, input_meta_xml=input_meta_xml, out_dir=out_dir, dem_file=dem_file, $
|
||||
work_dir=work_dir, temp_dir=temp_dir, range_looks=range_looks, $
|
||||
azimuth_looks=azimuth_looks, filter_type=filter_type, filter_window=filter_window, $
|
||||
geocode_grid_x=geocode_grid_x, geocode_grid_y=geocode_grid_y
|
||||
|
||||
gf3_prepare_dirs
|
||||
gf3_start_batch
|
||||
|
||||
errmsg = ''
|
||||
|
||||
if ~keyword_set(skip_import) then begin
|
||||
gf3_run_import, errmsg=errmsg
|
||||
if errmsg eq '' then gf3_refresh_import_roots
|
||||
endif else begin
|
||||
gf3_refresh_import_roots
|
||||
endelse
|
||||
|
||||
if errmsg eq '' && gf3_should_run_pol(polarizations, 'HH') then gf3_run_single_pol_pipeline, 'HH', errmsg=errmsg
|
||||
if errmsg eq '' && gf3_should_run_pol(polarizations, 'HV') then gf3_run_single_pol_pipeline, 'HV', errmsg=errmsg
|
||||
|
||||
gf3_stop_batch
|
||||
if errmsg ne '' then print, 'Pipeline stopped: ' + errmsg
|
||||
end
|
||||
|
||||
|
||||
pro gf3_sarscape_pipeline
|
||||
compile_opt idl2
|
||||
common gf3_pipeline_cfg, cfg
|
||||
|
||||
gf3_init_default_cfg
|
||||
gf3_prepare_dirs
|
||||
gf3_start_batch
|
||||
|
||||
errmsg = ''
|
||||
gf3_run_import, errmsg=errmsg
|
||||
if errmsg eq '' then gf3_refresh_import_roots
|
||||
if errmsg eq '' then gf3_run_single_pol_pipeline, 'HH', errmsg=errmsg
|
||||
if errmsg eq '' then gf3_run_single_pol_pipeline, 'HV', errmsg=errmsg
|
||||
|
||||
gf3_stop_batch
|
||||
if errmsg ne '' then print, 'Pipeline stopped: ' + errmsg
|
||||
end
|
||||
|
||||
|
||||
pro gf3_sarscape_import_only
|
||||
compile_opt idl2
|
||||
|
||||
gf3_init_default_cfg
|
||||
gf3_prepare_dirs
|
||||
gf3_start_batch
|
||||
errmsg = ''
|
||||
gf3_run_import, errmsg=errmsg
|
||||
gf3_stop_batch
|
||||
if errmsg ne '' then print, 'Import stopped: ' + errmsg
|
||||
end
|
||||
|
||||
|
||||
pro gf3_sarscape_hh_only
|
||||
compile_opt idl2
|
||||
|
||||
gf3_init_default_cfg
|
||||
gf3_prepare_dirs
|
||||
gf3_start_batch
|
||||
errmsg = ''
|
||||
gf3_run_single_pol_pipeline, 'HH', do_import=1, errmsg=errmsg
|
||||
gf3_stop_batch
|
||||
if errmsg ne '' then print, 'HH pipeline stopped: ' + errmsg
|
||||
end
|
||||
|
||||
|
||||
pro gf3_sarscape_hv_only
|
||||
compile_opt idl2
|
||||
|
||||
gf3_init_default_cfg
|
||||
gf3_prepare_dirs
|
||||
gf3_start_batch
|
||||
errmsg = ''
|
||||
gf3_run_single_pol_pipeline, 'HV', do_import=1, errmsg=errmsg
|
||||
gf3_stop_batch
|
||||
if errmsg ne '' then print, 'HV pipeline stopped: ' + errmsg
|
||||
end
|
||||
@@ -0,0 +1,8 @@
|
||||
@echo off
|
||||
set IDLRT=C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlrt.exe
|
||||
set SAV=C:\Users\Administrator\Desktop\GF3_Test\GF3_ENVI_IDL\gf3_sarscape_cli.sav
|
||||
set META=C:\Users\Administrator\Desktop\GF3_Test\GF3_scene_20260513\GF3_SYC_FSI_051370_E124.6_N51.7_20260513_L1A_HHHV_L10007355955.meta.xml
|
||||
set OUT=C:\Users\Administrator\Desktop\GF3_Test\GF3_cli_test_out_20260513
|
||||
set DEM=D:\DEM\COPDEM_GLO30_China_4326_DEM
|
||||
|
||||
"%IDLRT%" "%SAV%" -args "%META%" "%OUT%" "%DEM%" "HH,HV"
|
||||
@@ -0,0 +1,768 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"compress/gzip"
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
//go:embed assets/gf3_sarscape_cli.sav
|
||||
var embedded embed.FS
|
||||
|
||||
type config struct {
|
||||
configPath string `json:"-"`
|
||||
input string `json:"input"`
|
||||
output string `json:"output"`
|
||||
dem string `json:"dem"`
|
||||
polarizations string `json:"polarizations"`
|
||||
idlrt string `json:"idlrt"`
|
||||
keepExtracted bool `json:"keep_extracted"`
|
||||
}
|
||||
|
||||
type scene struct {
|
||||
metaPath string
|
||||
archivePath string
|
||||
name string
|
||||
}
|
||||
|
||||
type sceneFailure struct {
|
||||
name string
|
||||
input string
|
||||
err error
|
||||
}
|
||||
|
||||
type batchError struct {
|
||||
failures []sceneFailure
|
||||
}
|
||||
|
||||
func (e batchError) Error() string {
|
||||
return fmt.Sprintf("batch completed with %d failed scene(s)", len(e.failures))
|
||||
}
|
||||
|
||||
type completedProduct struct {
|
||||
polarization string
|
||||
path string
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg := parseFlags()
|
||||
if len(os.Args) == 1 {
|
||||
var err error
|
||||
cfg, err = promptConfig(cfg)
|
||||
if err != nil {
|
||||
fmt.Printf("ERROR: %v\n", err)
|
||||
waitForEnter()
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if err := run(cfg); err != nil {
|
||||
fmt.Printf("ERROR: %v\n", err)
|
||||
if len(os.Args) == 1 {
|
||||
waitForEnter()
|
||||
}
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := saveConfig(cfg.configPath, cfg); err != nil {
|
||||
fmt.Printf("WARNING: save config: %v\n", err)
|
||||
}
|
||||
if len(os.Args) == 1 {
|
||||
fmt.Println("Done.")
|
||||
waitForEnter()
|
||||
}
|
||||
}
|
||||
|
||||
func parseFlags() config {
|
||||
cfg := defaultConfig()
|
||||
configPath := preScanConfigPath()
|
||||
cfg.configPath = configPath
|
||||
_ = loadConfig(configPath, &cfg)
|
||||
|
||||
flag.StringVar(&cfg.configPath, "config", configPath, "Config file path")
|
||||
flag.StringVar(&cfg.input, "input", cfg.input, "GF-3 input: .meta.xml, .tar.gz, or directory")
|
||||
flag.StringVar(&cfg.output, "output", cfg.output, "Output directory")
|
||||
flag.StringVar(&cfg.dem, "dem", cfg.dem, "SARscape DEM file")
|
||||
flag.StringVar(&cfg.polarizations, "pol", cfg.polarizations, "Polarizations: HH, HV, or HH,HV")
|
||||
flag.StringVar(&cfg.idlrt, "idlrt", cfg.idlrt, "Path to idlrt.exe")
|
||||
flag.BoolVar(&cfg.keepExtracted, "keep-extracted", cfg.keepExtracted, "Keep extracted archives under output/.gf3_extract")
|
||||
flag.Parse()
|
||||
return cfg
|
||||
}
|
||||
|
||||
func defaultConfig() config {
|
||||
return config{
|
||||
dem: `D:\DEM\COPDEM_GLO30_China_4326_DEM`,
|
||||
polarizations: "HH,HV",
|
||||
idlrt: defaultIDLRT(),
|
||||
keepExtracted: true,
|
||||
}
|
||||
}
|
||||
|
||||
func preScanConfigPath() string {
|
||||
defaultPath := defaultConfigPath()
|
||||
for i := 1; i < len(os.Args)-1; i++ {
|
||||
if os.Args[i] == "-config" {
|
||||
return os.Args[i+1]
|
||||
}
|
||||
}
|
||||
return defaultPath
|
||||
}
|
||||
|
||||
func defaultConfigPath() string {
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
return "gf3wrapper.json"
|
||||
}
|
||||
return filepath.Join(filepath.Dir(exe), "gf3wrapper.json")
|
||||
}
|
||||
|
||||
func loadConfig(path string, cfg *config) error {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var stored struct {
|
||||
IDLRTPath string `json:"idlrt_path"`
|
||||
DemFile string `json:"dem_file"`
|
||||
Polarizations string `json:"polarizations"`
|
||||
LastInput string `json:"last_input"`
|
||||
LastOutput string `json:"last_output"`
|
||||
IDLRT string `json:"idlrt"`
|
||||
Dem string `json:"dem"`
|
||||
Input string `json:"input"`
|
||||
Output string `json:"output"`
|
||||
KeepExtracted *bool `json:"keep_extracted"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &stored); err != nil {
|
||||
return err
|
||||
}
|
||||
if stored.IDLRTPath != "" {
|
||||
cfg.idlrt = stored.IDLRTPath
|
||||
} else if stored.IDLRT != "" {
|
||||
cfg.idlrt = stored.IDLRT
|
||||
}
|
||||
if stored.DemFile != "" {
|
||||
cfg.dem = stored.DemFile
|
||||
} else if stored.Dem != "" {
|
||||
cfg.dem = stored.Dem
|
||||
}
|
||||
if stored.Polarizations != "" {
|
||||
cfg.polarizations = stored.Polarizations
|
||||
}
|
||||
if stored.LastInput != "" {
|
||||
cfg.input = stored.LastInput
|
||||
} else if stored.Input != "" {
|
||||
cfg.input = stored.Input
|
||||
}
|
||||
if stored.LastOutput != "" {
|
||||
cfg.output = stored.LastOutput
|
||||
} else if stored.Output != "" {
|
||||
cfg.output = stored.Output
|
||||
}
|
||||
if stored.KeepExtracted != nil {
|
||||
cfg.keepExtracted = *stored.KeepExtracted
|
||||
}
|
||||
cfg.configPath = path
|
||||
return nil
|
||||
}
|
||||
|
||||
func saveConfig(path string, cfg config) error {
|
||||
if path == "" {
|
||||
path = defaultConfigPath()
|
||||
}
|
||||
stored := struct {
|
||||
IDLRTPath string `json:"idlrt_path"`
|
||||
DemFile string `json:"dem_file"`
|
||||
Polarizations string `json:"polarizations"`
|
||||
LastInput string `json:"last_input"`
|
||||
LastOutput string `json:"last_output"`
|
||||
KeepExtracted bool `json:"keep_extracted"`
|
||||
}{
|
||||
IDLRTPath: cfg.idlrt,
|
||||
DemFile: cfg.dem,
|
||||
Polarizations: cfg.polarizations,
|
||||
LastInput: cfg.input,
|
||||
LastOutput: cfg.output,
|
||||
KeepExtracted: cfg.keepExtracted,
|
||||
}
|
||||
data, err := json.MarshalIndent(stored, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, data, 0o644)
|
||||
}
|
||||
|
||||
func promptConfig(defaults config) (config, error) {
|
||||
reader := bufio.NewReader(os.Stdin)
|
||||
cfg := defaults
|
||||
|
||||
fmt.Println("GF-3 SARscape Wrapper")
|
||||
fmt.Println()
|
||||
|
||||
cfg.input = prompt(reader, "Input .tar.gz / .meta.xml / folder", cfg.input)
|
||||
cfg.output = prompt(reader, "Output folder", cfg.output)
|
||||
cfg.dem = prompt(reader, "DEM file", cfg.dem)
|
||||
cfg.polarizations = prompt(reader, "Polarizations", cfg.polarizations)
|
||||
cfg.idlrt = prompt(reader, "IDL Runtime", cfg.idlrt)
|
||||
|
||||
if cfg.input == "" || cfg.output == "" || cfg.dem == "" {
|
||||
return cfg, errors.New("input, output, and DEM are required")
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func prompt(reader *bufio.Reader, label, defaultValue string) string {
|
||||
if defaultValue != "" {
|
||||
fmt.Printf("%s [%s]: ", label, defaultValue)
|
||||
} else {
|
||||
fmt.Printf("%s: ", label)
|
||||
}
|
||||
text, _ := reader.ReadString('\n')
|
||||
text = strings.TrimSpace(text)
|
||||
if text == "" {
|
||||
return defaultValue
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
func waitForEnter() {
|
||||
fmt.Println()
|
||||
fmt.Print("Press Enter to exit...")
|
||||
_, _ = bufio.NewReader(os.Stdin).ReadString('\n')
|
||||
}
|
||||
|
||||
func run(cfg config) error {
|
||||
if cfg.input == "" || cfg.output == "" || cfg.dem == "" {
|
||||
return errors.New("required flags: -input, -output, -dem")
|
||||
}
|
||||
if _, err := os.Stat(cfg.idlrt); err != nil {
|
||||
return fmt.Errorf("idlrt not found: %s: %w", cfg.idlrt, err)
|
||||
}
|
||||
if _, err := os.Stat(cfg.dem); err != nil {
|
||||
return fmt.Errorf("DEM not found: %s: %w", cfg.dem, err)
|
||||
}
|
||||
if err := os.MkdirAll(cfg.output, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
scenes, err := discoverScenes(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(scenes) == 0 {
|
||||
return fmt.Errorf("no *.meta.xml scenes found in %s", cfg.input)
|
||||
}
|
||||
|
||||
failures := make([]sceneFailure, 0)
|
||||
processed := 0
|
||||
skipped := 0
|
||||
savPath := ""
|
||||
for _, sc := range scenes {
|
||||
sceneOut := filepath.Join(cfg.output, sc.name)
|
||||
if products, ok := completedSceneOutputs(sceneOut, sc.name, cfg.polarizations); ok {
|
||||
recordSceneSkip(sc, sceneOut, products)
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
if err := os.MkdirAll(sceneOut, 0o755); err != nil {
|
||||
recordSceneFailure(&failures, sc, err)
|
||||
continue
|
||||
}
|
||||
if sc.archivePath != "" {
|
||||
extractDir := filepath.Join(sceneOut, ".gf3_extract")
|
||||
fmt.Printf("Extracting %s\n", sc.archivePath)
|
||||
if err := extractTarGz(sc.archivePath, extractDir); err != nil {
|
||||
recordSceneFailure(&failures, sc, fmt.Errorf("failed to extract archive before processing scene %s: %w", sc.archivePath, err))
|
||||
continue
|
||||
}
|
||||
extractedScenes, err := findMetaXML(extractDir)
|
||||
if err != nil {
|
||||
recordSceneFailure(&failures, sc, err)
|
||||
continue
|
||||
}
|
||||
if len(extractedScenes) == 0 {
|
||||
recordSceneFailure(&failures, sc, fmt.Errorf("no *.meta.xml found in archive: %s", sc.archivePath))
|
||||
continue
|
||||
}
|
||||
sc.metaPath = extractedScenes[0].metaPath
|
||||
sc.name = sceneName(sc.metaPath)
|
||||
}
|
||||
if savPath == "" {
|
||||
var err error
|
||||
savPath, err = materializeSAV(cfg.output)
|
||||
if err != nil {
|
||||
recordSceneFailure(&failures, sc, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err := runScene(cfg, savPath, sc, sceneOut); err != nil {
|
||||
recordSceneFailure(&failures, sc, err)
|
||||
continue
|
||||
}
|
||||
processed++
|
||||
}
|
||||
printBatchSummary(len(scenes), processed, skipped, failures)
|
||||
if len(failures) > 0 {
|
||||
return batchError{failures: failures}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sceneInput(sc scene) string {
|
||||
if sc.archivePath != "" {
|
||||
return sc.archivePath
|
||||
}
|
||||
if sc.metaPath != "" {
|
||||
return sc.metaPath
|
||||
}
|
||||
return sc.name
|
||||
}
|
||||
|
||||
func recordSceneFailure(failures *[]sceneFailure, sc scene, err error) {
|
||||
input := sceneInput(sc)
|
||||
|
||||
failure := sceneFailure{name: sc.name, input: input, err: err}
|
||||
*failures = append(*failures, failure)
|
||||
|
||||
fmt.Printf("FAILED %s\n", sc.name)
|
||||
fmt.Printf("Input %s\n", input)
|
||||
fmt.Printf("Reason: %v\n", err)
|
||||
fmt.Println("Continuing with next scene.")
|
||||
}
|
||||
|
||||
func recordSceneSkip(sc scene, sceneOut string, products []completedProduct) {
|
||||
parts := make([]string, 0, len(products))
|
||||
for _, product := range products {
|
||||
parts = append(parts, product.polarization+"="+product.path)
|
||||
}
|
||||
|
||||
fmt.Printf("SKIPPED %s\n", sc.name)
|
||||
fmt.Printf("Input %s\n", sceneInput(sc))
|
||||
fmt.Printf("Output already complete %s\n", sceneOut)
|
||||
fmt.Printf("Products %s\n", strings.Join(parts, "; "))
|
||||
}
|
||||
|
||||
func printBatchSummary(total, processed, skipped int, failures []sceneFailure) {
|
||||
fmt.Println()
|
||||
fmt.Printf("Batch summary: %d processed, %d skipped, %d failed, %d total\n", processed, skipped, len(failures), total)
|
||||
if len(failures) == 0 {
|
||||
return
|
||||
}
|
||||
fmt.Println("Failed scenes:")
|
||||
for _, failure := range failures {
|
||||
fmt.Printf("- %s (%s): %v\n", failure.name, failure.input, failure.err)
|
||||
}
|
||||
}
|
||||
|
||||
func materializeSAV(outputDir string) (string, error) {
|
||||
data, err := embedded.ReadFile("assets/gf3_sarscape_cli.sav")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
runtimeDir := filepath.Join(outputDir, ".gf3_runtime")
|
||||
if err := os.MkdirAll(runtimeDir, 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
savPath := filepath.Join(runtimeDir, "gf3_sarscape_cli.sav")
|
||||
if err := os.WriteFile(savPath, data, 0o644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return savPath, nil
|
||||
}
|
||||
|
||||
func discoverScenes(cfg config) ([]scene, error) {
|
||||
info, err := os.Stat(cfg.input)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return discoverScenesInDir(cfg.input, cfg.output)
|
||||
}
|
||||
lower := strings.ToLower(cfg.input)
|
||||
if strings.HasSuffix(lower, ".meta.xml") {
|
||||
return []scene{{metaPath: cfg.input, name: sceneName(cfg.input)}}, nil
|
||||
}
|
||||
if strings.HasSuffix(lower, ".tar.gz") || strings.HasSuffix(lower, ".tgz") {
|
||||
return []scene{{archivePath: cfg.input, name: safeName(trimArchiveExt(filepath.Base(cfg.input)))}}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("unsupported input type: %s", cfg.input)
|
||||
}
|
||||
|
||||
func discoverScenesInDir(inputDir, outputDir string) ([]scene, error) {
|
||||
_ = outputDir
|
||||
scenesByName := make(map[string]scene)
|
||||
|
||||
err := filepath.WalkDir(inputDir, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
if d.Name() == ".gf3_extract" {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
lower := strings.ToLower(d.Name())
|
||||
switch {
|
||||
case strings.HasSuffix(lower, ".meta.xml"):
|
||||
sc := scene{metaPath: path, name: sceneName(path)}
|
||||
key := strings.ToLower(sc.name)
|
||||
if _, ok := scenesByName[key]; !ok {
|
||||
scenesByName[key] = sc
|
||||
}
|
||||
case strings.HasSuffix(lower, ".tar.gz") || strings.HasSuffix(lower, ".tgz"):
|
||||
sc := scene{archivePath: path, name: safeName(trimArchiveExt(filepath.Base(path)))}
|
||||
scenesByName[strings.ToLower(sc.name)] = sc
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
scenes := make([]scene, 0, len(scenesByName))
|
||||
for _, sc := range scenesByName {
|
||||
scenes = append(scenes, sc)
|
||||
}
|
||||
sort.Slice(scenes, func(i, j int) bool {
|
||||
return strings.ToLower(scenes[i].name) < strings.ToLower(scenes[j].name)
|
||||
})
|
||||
return scenes, nil
|
||||
}
|
||||
|
||||
func findMetaXML(root string) ([]scene, error) {
|
||||
var scenes []scene
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if strings.HasSuffix(strings.ToLower(d.Name()), ".meta.xml") {
|
||||
scenes = append(scenes, scene{metaPath: path, name: sceneName(path)})
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return scenes, err
|
||||
}
|
||||
|
||||
func uniqueScenes(scenes []scene) []scene {
|
||||
seen := make(map[string]bool, len(scenes))
|
||||
out := make([]scene, 0, len(scenes))
|
||||
for _, sc := range scenes {
|
||||
key := filepath.Clean(sc.metaPath)
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, sc)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func runScene(cfg config, savPath string, sc scene, sceneOut string) error {
|
||||
fmt.Printf("Processing %s\n", sc.metaPath)
|
||||
fmt.Printf("Output %s\n", sceneOut)
|
||||
|
||||
cmd := exec.Command(cfg.idlrt, savPath, "-args", sc.metaPath, sceneOut, cfg.dem, cfg.polarizations)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
cmd.Dir = sceneOut
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- cmd.Wait()
|
||||
}()
|
||||
|
||||
ticker := time.NewTicker(2 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case err := <-done:
|
||||
fmt.Printf("\r%-100s\r", "")
|
||||
if err != nil {
|
||||
return fmt.Errorf("scene failed %s: %w", sc.metaPath, err)
|
||||
}
|
||||
fmt.Println("Completed", sc.name)
|
||||
return nil
|
||||
case <-ticker.C:
|
||||
fmt.Printf("\r%s", progressLine(sceneOut, sc.name, cfg.polarizations))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func progressLine(sceneOut, sceneNameValue, polarizations string) string {
|
||||
stage, done, total := progressStage(sceneOut, sceneNameValue, polarizations)
|
||||
width := 28
|
||||
filled := 0
|
||||
if total > 0 {
|
||||
filled = done * width / total
|
||||
}
|
||||
if filled > width {
|
||||
filled = width
|
||||
}
|
||||
bar := strings.Repeat("#", filled) + strings.Repeat("-", width-filled)
|
||||
return fmt.Sprintf("[%s] %2d/%2d %s", bar, done, total, stage)
|
||||
}
|
||||
|
||||
func progressStage(sceneOut, sceneNameValue, polarizations string) (string, int, int) {
|
||||
pols := requestedPolarizations(polarizations)
|
||||
checks := make([]struct {
|
||||
label string
|
||||
path string
|
||||
}, 0, len(pols)*4)
|
||||
|
||||
for _, pol := range pols {
|
||||
checks = append(checks, struct {
|
||||
label string
|
||||
path string
|
||||
}{"import " + pol, filepath.Join(sceneOut, "*_"+pol+"_slc.sml")})
|
||||
}
|
||||
|
||||
for _, pol := range pols {
|
||||
lower := strings.ToLower(pol)
|
||||
prefix := sceneNameValue + "_" + lower
|
||||
checks = append(checks,
|
||||
struct {
|
||||
label string
|
||||
path string
|
||||
}{"multilook " + pol, filepath.Join(sceneOut, prefix+"_ml.sml")},
|
||||
struct {
|
||||
label string
|
||||
path string
|
||||
}{"filter " + pol, filepath.Join(sceneOut, prefix+"_filt.sml")},
|
||||
struct {
|
||||
label string
|
||||
path string
|
||||
}{"geocode " + pol, filepath.Join(sceneOut, prefix+"_geo.sml")},
|
||||
)
|
||||
}
|
||||
|
||||
done := 0
|
||||
for _, check := range checks {
|
||||
if matched(check.path) {
|
||||
done++
|
||||
continue
|
||||
}
|
||||
return check.label, done, len(checks)
|
||||
}
|
||||
return "finishing", done, len(checks)
|
||||
}
|
||||
|
||||
func completedSceneOutputs(sceneOut, sceneNameValue, polarizations string) ([]completedProduct, bool) {
|
||||
info, err := os.Stat(sceneOut)
|
||||
if err != nil || !info.IsDir() {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
pols := requestedPolarizations(polarizations)
|
||||
products := make([]completedProduct, 0, len(pols))
|
||||
for _, pol := range pols {
|
||||
path, ok := completedGeoProduct(sceneOut, sceneNameValue, pol)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
products = append(products, completedProduct{polarization: pol, path: path})
|
||||
}
|
||||
return products, true
|
||||
}
|
||||
|
||||
func completedGeoProduct(sceneOut, sceneNameValue, polarization string) (string, bool) {
|
||||
lower := strings.ToLower(polarization)
|
||||
exact := filepath.Join(sceneOut, sceneNameValue+"_"+lower+"_geo.sml")
|
||||
if completeGeoSML(exact) {
|
||||
return exact, true
|
||||
}
|
||||
|
||||
matches, err := filepath.Glob(filepath.Join(sceneOut, "*_"+lower+"_geo.sml"))
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
sort.Strings(matches)
|
||||
for _, match := range matches {
|
||||
if completeGeoSML(match) {
|
||||
return match, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func completeGeoSML(path string) bool {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil || info.IsDir() || info.Size() == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
dataPath := strings.TrimSuffix(path, ".sml")
|
||||
dataInfo, err := os.Stat(dataPath)
|
||||
return err == nil && !dataInfo.IsDir() && dataInfo.Size() > 0
|
||||
}
|
||||
|
||||
func requestedPolarizations(value string) []string {
|
||||
parts := strings.Split(value, ",")
|
||||
out := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
pol := strings.ToUpper(strings.TrimSpace(part))
|
||||
if pol == "" {
|
||||
continue
|
||||
}
|
||||
if pol == "ALL" {
|
||||
return []string{"HH", "HV"}
|
||||
}
|
||||
out = append(out, pol)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return []string{"HH", "HV"}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func matched(pattern string) bool {
|
||||
matches, err := filepath.Glob(pattern)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return len(matches) > 0
|
||||
}
|
||||
|
||||
func extractTarGz(src, dst string) error {
|
||||
dst = filepath.Clean(dst)
|
||||
tmpDst := dst + ".tmp"
|
||||
|
||||
if err := os.RemoveAll(tmpDst); err != nil {
|
||||
return fmt.Errorf("remove stale temporary extraction directory %s: %w", tmpDst, err)
|
||||
}
|
||||
if err := os.MkdirAll(tmpDst, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := extractTarGzToDir(src, tmpDst); err != nil {
|
||||
_ = os.RemoveAll(tmpDst)
|
||||
return err
|
||||
}
|
||||
if err := os.RemoveAll(dst); err != nil {
|
||||
_ = os.RemoveAll(tmpDst)
|
||||
return fmt.Errorf("replace extraction directory %s: %w", dst, err)
|
||||
}
|
||||
if err := os.Rename(tmpDst, dst); err != nil {
|
||||
_ = os.RemoveAll(tmpDst)
|
||||
return fmt.Errorf("move completed extraction into %s: %w", dst, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func extractTarGzToDir(src, dst string) error {
|
||||
f, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
gz, err := gzip.NewReader(f)
|
||||
if err != nil {
|
||||
return archiveReadError(src, "", err)
|
||||
}
|
||||
defer gz.Close()
|
||||
|
||||
tr := tar.NewReader(gz)
|
||||
var currentEntry string
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return archiveReadError(src, currentEntry, err)
|
||||
}
|
||||
currentEntry = hdr.Name
|
||||
target := filepath.Join(dst, filepath.Clean(hdr.Name))
|
||||
if !strings.HasPrefix(target, filepath.Clean(dst)+string(os.PathSeparator)) && filepath.Clean(target) != filepath.Clean(dst) {
|
||||
return fmt.Errorf("archive entry escapes destination: %s", hdr.Name)
|
||||
}
|
||||
switch hdr.Typeflag {
|
||||
case tar.TypeDir:
|
||||
if err := os.MkdirAll(target, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
case tar.TypeReg:
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
out, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, copyErr := io.Copy(out, tr)
|
||||
closeErr := out.Close()
|
||||
if copyErr != nil {
|
||||
return archiveReadError(src, hdr.Name, copyErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func archiveReadError(src, entry string, err error) error {
|
||||
detail := src
|
||||
if info, statErr := os.Stat(src); statErr == nil {
|
||||
detail = fmt.Sprintf("%s (%d bytes)", src, info.Size())
|
||||
}
|
||||
if entry != "" {
|
||||
detail = fmt.Sprintf("%s while reading %s", detail, entry)
|
||||
}
|
||||
if errors.Is(err, io.ErrUnexpectedEOF) || strings.Contains(strings.ToLower(err.Error()), "unexpected eof") {
|
||||
return fmt.Errorf("archive appears truncated or incomplete: %s: %w; re-copy or re-download the .tar.gz", detail, err)
|
||||
}
|
||||
return fmt.Errorf("read archive %s: %w", detail, err)
|
||||
}
|
||||
|
||||
func sceneName(metaPath string) string {
|
||||
base := filepath.Base(metaPath)
|
||||
base = strings.TrimSuffix(base, ".meta.xml")
|
||||
base = strings.TrimSuffix(base, ".META.XML")
|
||||
return safeName(base)
|
||||
}
|
||||
|
||||
func safeName(s string) string {
|
||||
replacer := strings.NewReplacer(" ", "_", ":", "_", "/", "_", "\\", "_")
|
||||
return replacer.Replace(s)
|
||||
}
|
||||
|
||||
func trimArchiveExt(name string) string {
|
||||
lower := strings.ToLower(name)
|
||||
switch {
|
||||
case strings.HasSuffix(lower, ".tar.gz"):
|
||||
return name[:len(name)-7]
|
||||
case strings.HasSuffix(lower, ".tgz"):
|
||||
return name[:len(name)-4]
|
||||
default:
|
||||
return strings.TrimSuffix(name, filepath.Ext(name))
|
||||
}
|
||||
}
|
||||
|
||||
func defaultIDLRT() string {
|
||||
if v := os.Getenv("IDLRT_PATH"); v != "" {
|
||||
return v
|
||||
}
|
||||
return `C:\Program Files\Harris\ENVI56\IDL88\bin\bin.x86_64\idlrt.exe`
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCompletedSceneOutputsRequiresGeoSMLAndDataForAllPolarizations(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
sceneName := "GF3_TEST_SCENE"
|
||||
|
||||
writeFile(t, filepath.Join(dir, sceneName+"_hh_geo.sml"), "hh sml")
|
||||
writeFile(t, filepath.Join(dir, sceneName+"_hh_geo"), "hh data")
|
||||
writeFile(t, filepath.Join(dir, sceneName+"_hv_geo.sml"), "hv sml")
|
||||
writeFile(t, filepath.Join(dir, sceneName+"_hv_geo"), "hv data")
|
||||
|
||||
products, ok := completedSceneOutputs(dir, sceneName, "HH,HV")
|
||||
if !ok {
|
||||
t.Fatal("expected completed scene outputs")
|
||||
}
|
||||
if len(products) != 2 {
|
||||
t.Fatalf("expected 2 completed products, got %d", len(products))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompletedSceneOutputsRejectsMissingDataFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
sceneName := "GF3_TEST_SCENE"
|
||||
|
||||
writeFile(t, filepath.Join(dir, sceneName+"_hh_geo.sml"), "hh sml")
|
||||
writeFile(t, filepath.Join(dir, sceneName+"_hv_geo.sml"), "hv sml")
|
||||
writeFile(t, filepath.Join(dir, sceneName+"_hv_geo"), "hv data")
|
||||
|
||||
if _, ok := completedSceneOutputs(dir, sceneName, "HH,HV"); ok {
|
||||
t.Fatal("expected incomplete scene when one geocoded data file is missing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompletedSceneOutputsRequiresRequestedPolarizations(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
sceneName := "GF3_TEST_SCENE"
|
||||
|
||||
writeFile(t, filepath.Join(dir, sceneName+"_hh_geo.sml"), "hh sml")
|
||||
writeFile(t, filepath.Join(dir, sceneName+"_hh_geo"), "hh data")
|
||||
|
||||
if _, ok := completedSceneOutputs(dir, sceneName, "HH,HV"); ok {
|
||||
t.Fatal("expected incomplete scene when HV output is missing")
|
||||
}
|
||||
|
||||
if _, ok := completedSceneOutputs(dir, sceneName, "HH"); !ok {
|
||||
t.Fatal("expected completed scene for HH-only request")
|
||||
}
|
||||
}
|
||||
|
||||
func writeFile(t *testing.T, path, value string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(value), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user