Harmon/GF3_L1A_To_L2_pipeline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Handle corrupt GF3 archives in batch runs

Browse Source
This commit is contained in:
GF3 Pipeline Bot
2026-05-30 01:46:04 +08:00
parent 5c24c0e64c
commit ff41cf3ced
6 changed files with 125 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/gf3_sarscape_wrapper/README.md
+2
View File
@@ -39,6 +39,8 @@ Batch directory:
If `gf3wrapper.json` exists next to the executable, omitted `-input`, `-output`, `-dem`, and `-pol` values are loaded from that file.
In batch directory mode, a failed scene does not stop the whole run. Invalid or truncated archives, missing metadata, and SARscape failures are reported with the scene input path and reason, then the wrapper continues with the next scene. A final summary lists succeeded and failed scenes, and the process exits non-zero when any scene failed.
## Notes
- The server still needs ENVI, IDL Runtime, and SARscape installed and licensed.
tools/gf3_sarscape_wrapper/main.go
+103 -9
View File
@@ -38,6 +38,20 @@ type scene struct {
name string
}
type sceneFailure struct {
name string
input string
err error
}
type batchError struct {
failures []sceneFailure
}
func (e batchError) Error() string {
return fmt.Sprintf("batch completed with %d failed scene(s)", len(e.failures))
}
func main() {
cfg := parseFlags()
if len(os.Args) == 1 {
@@ -251,34 +265,76 @@ func run(cfg config) error {
return fmt.Errorf("no *.meta.xml scenes found in %s", cfg.input)
}
failures := make([]sceneFailure, 0)
succeeded := 0
for _, sc := range scenes {
sceneOut := filepath.Join(cfg.output, sc.name)
if err := os.MkdirAll(sceneOut, 0o755); err != nil {
return err
recordSceneFailure(&failures, sc, err)
continue
}
if sc.archivePath != "" {
extractDir := filepath.Join(sceneOut, ".gf3_extract")
fmt.Printf("Extracting %s\n", sc.archivePath)
if err := extractTarGz(sc.archivePath, extractDir); err != nil {
return err
recordSceneFailure(&failures, sc, fmt.Errorf("failed to extract archive before processing scene %s: %w", sc.archivePath, err))
continue
}
extractedScenes, err := findMetaXML(extractDir)
if err != nil {
return err
recordSceneFailure(&failures, sc, err)
continue
}
if len(extractedScenes) == 0 {
return fmt.Errorf("no *.meta.xml found in archive: %s", sc.archivePath)
recordSceneFailure(&failures, sc, fmt.Errorf("no *.meta.xml found in archive: %s", sc.archivePath))
continue
}
sc.metaPath = extractedScenes[0].metaPath
sc.name = sceneName(sc.metaPath)
}
if err := runScene(cfg, savPath, sc, sceneOut); err != nil {
return err
recordSceneFailure(&failures, sc, err)
continue
}
succeeded++
}
printBatchSummary(len(scenes), succeeded, failures)
if len(failures) > 0 {
return batchError{failures: failures}
}
return nil
}
func recordSceneFailure(failures *[]sceneFailure, sc scene, err error) {
input := sc.metaPath
if sc.archivePath != "" {
input = sc.archivePath
}
if input == "" {
input = sc.name
}
failure := sceneFailure{name: sc.name, input: input, err: err}
*failures = append(*failures, failure)
fmt.Printf("FAILED %s\n", sc.name)
fmt.Printf("Input %s\n", input)
fmt.Printf("Reason: %v\n", err)
fmt.Println("Continuing with next scene.")
}
func printBatchSummary(total, succeeded int, failures []sceneFailure) {
fmt.Println()
fmt.Printf("Batch summary: %d succeeded, %d failed, %d total\n", succeeded, len(failures), total)
if len(failures) == 0 {
return
}
fmt.Println("Failed scenes:")
for _, failure := range failures {
fmt.Printf("- %s (%s): %v\n", failure.name, failure.input, failure.err)
}
}
func materializeSAV(outputDir string) (string, error) {
data, err := embedded.ReadFile("assets/gf3_sarscape_cli.sav")
if err != nil {
@@ -508,9 +564,31 @@ func matched(pattern string) bool {
}
func extractTarGz(src, dst string) error {
if err := os.MkdirAll(dst, 0o755); err != nil {
dst = filepath.Clean(dst)
tmpDst := dst + ".tmp"
if err := os.RemoveAll(tmpDst); err != nil {
return fmt.Errorf("remove stale temporary extraction directory %s: %w", tmpDst, err)
}
if err := os.MkdirAll(tmpDst, 0o755); err != nil {
return err
}
if err := extractTarGzToDir(src, tmpDst); err != nil {
_ = os.RemoveAll(tmpDst)
return err
}
if err := os.RemoveAll(dst); err != nil {
_ = os.RemoveAll(tmpDst)
return fmt.Errorf("replace extraction directory %s: %w", dst, err)
}
if err := os.Rename(tmpDst, dst); err != nil {
_ = os.RemoveAll(tmpDst)
return fmt.Errorf("move completed extraction into %s: %w", dst, err)
}
return nil
}
func extractTarGzToDir(src, dst string) error {
f, err := os.Open(src)
if err != nil {
return err
@@ -519,19 +597,21 @@ func extractTarGz(src, dst string) error {
gz, err := gzip.NewReader(f)
if err != nil {
return err
return archiveReadError(src, "", err)
}
defer gz.Close()
tr := tar.NewReader(gz)
var currentEntry string
for {
hdr, err := tr.Next()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return err
return archiveReadError(src, currentEntry, err)
}
currentEntry = hdr.Name
target := filepath.Join(dst, filepath.Clean(hdr.Name))
if !strings.HasPrefix(target, filepath.Clean(dst)+string(os.PathSeparator)) && filepath.Clean(target) != filepath.Clean(dst) {
return fmt.Errorf("archive entry escapes destination: %s", hdr.Name)
@@ -552,7 +632,7 @@ func extractTarGz(src, dst string) error {
_, copyErr := io.Copy(out, tr)
closeErr := out.Close()
if copyErr != nil {
return copyErr
return archiveReadError(src, hdr.Name, copyErr)
}
if closeErr != nil {
return closeErr
@@ -562,6 +642,20 @@ func extractTarGz(src, dst string) error {
return nil
}
func archiveReadError(src, entry string, err error) error {
detail := src
if info, statErr := os.Stat(src); statErr == nil {
detail = fmt.Sprintf("%s (%d bytes)", src, info.Size())
}
if entry != "" {
detail = fmt.Sprintf("%s while reading %s", detail, entry)
}
if errors.Is(err, io.ErrUnexpectedEOF) || strings.Contains(strings.ToLower(err.Error()), "unexpected eof") {
return fmt.Errorf("archive appears truncated or incomplete: %s: %w; re-copy or re-download the .tar.gz", detail, err)
}
return fmt.Errorf("read archive %s: %w", detail, err)
}
func sceneName(metaPath string) string {
base := filepath.Base(metaPath)
base = strings.TrimSuffix(base, ".meta.xml")