diff --git a/README.md b/README.md index 002b2b7..ad81466 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,13 @@ OutputRoot/ restore.log ``` +恢复时会尽量去掉去重打包时增加的名称前缀： + +- `data/scene__<原始数据名>/` 只作为 bundle 内部存储目录，恢复后不会在 `master/` 或 `slave/` 下额外保留这一层目录。 +- `master/` 和 `slave/` 下直接放数据文件；如果数据文件名为 `scene__<原始文件名>`，会恢复为 `<原始文件名>`。 +- `orbit/orbit__<原始轨道文件名>` 会恢复为 `Task_*/orbit/<原始轨道文件名>`。 +- 如果 `pairs.json` 中提供了 `master_orbit_source_path` / `slave_orbit_source_path`，轨道文件会优先使用这些原始路径的文件名部分。 + Task 目录名优先使用： 1. `task_alias` diff --git a/main.go b/main.go index 63548c0..7e09d77 100644 --- a/main.go +++ b/main.go @@ -47,6 +47,10 @@ type pair struct { SlaveData string `json:"slave_data"` MasterOrbit string `json:"master_orbit"` SlaveOrbit string `json:"slave_orbit"` + MasterSourcePath string `json:"master_source_path"` + SlaveSourcePath string `json:"slave_source_path"` + MasterOrbitSource string `json:"master_orbit_source_path"` + SlaveOrbitSource string `json:"slave_orbit_source_path"` MasterImagingDate string `json:"master_imaging_date"` SlaveImagingDate string `json:"slave_imaging_date"` TimeBaselineDays int `json:"time_baseline_days"` @@ -60,6 +64,10 @@ type pairMetadata struct { SlaveData string `json:"slave_data"` MasterOrbit string `json:"master_orbit"` SlaveOrbit string `json:"slave_orbit"` + MasterSourcePath string `json:"master_source_path,omitempty"` + SlaveSourcePath string `json:"slave_source_path,omitempty"` + MasterOrbitSource string `json:"master_orbit_source_path,omitempty"` + SlaveOrbitSource string `json:"slave_orbit_source_path,omitempty"` MasterImagingDate string `json:"master_imaging_date"` SlaveImagingDate string `json:"slave_imaging_date"` TimeBaselineDays int `json:"time_baseline_days"` @@ -522,10 +530,10 @@ func restorePair(cfg config, inputRoot string, outputRoot string, p pair, logger } }() - if err := copyDirContents(masterSource, filepath.Join(tempDir, "master")); err != nil { + if err := copyDataTree(masterSource, filepath.Join(tempDir, "master"), p.MasterData, p.MasterSourcePath); err != nil { return nil, fmt.Errorf("copy master data: %w", err) } - if err := copyDirContents(slaveSource, filepath.Join(tempDir, "slave")); err != nil { + if err := copyDataTree(slaveSource, filepath.Join(tempDir, "slave"), p.SlaveData, p.SlaveSourcePath); err != nil { return nil, fmt.Errorf("copy slave data: %w", err) } @@ -539,6 +547,10 @@ func restorePair(cfg config, inputRoot string, outputRoot string, p pair, logger SlaveData: p.SlaveData, MasterOrbit: p.MasterOrbit, SlaveOrbit: p.SlaveOrbit, + MasterSourcePath: p.MasterSourcePath, + SlaveSourcePath: p.SlaveSourcePath, + MasterOrbitSource: p.MasterOrbitSource, + SlaveOrbitSource: p.SlaveOrbitSource, MasterImagingDate: p.MasterImagingDate, SlaveImagingDate: p.SlaveImagingDate, TimeBaselineDays: p.TimeBaselineDays, @@ -574,15 +586,23 @@ func taskDirectoryName(p pair) (string, error) { func copyOrbitFiles(inputRoot string, tempDir string, taskDirName string, p pair, logger *log.Logger) []string { var warnings []string for _, entry := range []struct { - label string - value string + label string + value string + sourcePath string }{ - {label: "master_orbit", value: p.MasterOrbit}, - {label: "slave_orbit", value: p.SlaveOrbit}, + {label: "master_orbit", value: p.MasterOrbit, sourcePath: p.MasterOrbitSource}, + {label: "slave_orbit", value: p.SlaveOrbit, sourcePath: p.SlaveOrbitSource}, } { if strings.TrimSpace(entry.value) == "" { continue } + destName, err := restoredOrbitFileName(entry.value, entry.sourcePath) + if err != nil { + warning := fmt.Sprintf("task %s %s cannot determine restored file name: %v", taskDirName, entry.label, err) + warnings = append(warnings, warning) + logger.Printf("warning %s", warning) + continue + } source, err := safeJoin(inputRoot, entry.value) if err != nil { warning := fmt.Sprintf("task %s %s invalid: %v", taskDirName, entry.label, err) @@ -610,7 +630,7 @@ func copyOrbitFiles(inputRoot string, tempDir string, taskDirName string, p pair logger.Printf("warning %s", warning) continue } - if err := copyFile(source, filepath.Join(orbitDir, filepath.Base(source)), stat.Mode()); err != nil { + if err := copyFile(source, filepath.Join(orbitDir, destName), stat.Mode()); err != nil { warning := fmt.Sprintf("task %s cannot copy %s %s: %v", taskDirName, entry.label, entry.value, err) warnings = append(warnings, warning) logger.Printf("warning %s", warning) @@ -623,15 +643,20 @@ func copyOrbitFiles(inputRoot string, tempDir string, taskDirName string, p pair func orbitWarnings(inputRoot string, p pair, taskDirName string) []string { var warnings []string for _, entry := range []struct { - label string - value string + label string + value string + sourcePath string }{ - {label: "master_orbit", value: p.MasterOrbit}, - {label: "slave_orbit", value: p.SlaveOrbit}, + {label: "master_orbit", value: p.MasterOrbit, sourcePath: p.MasterOrbitSource}, + {label: "slave_orbit", value: p.SlaveOrbit, sourcePath: p.SlaveOrbitSource}, } { if strings.TrimSpace(entry.value) == "" { continue } + if _, err := restoredOrbitFileName(entry.value, entry.sourcePath); err != nil { + warnings = append(warnings, fmt.Sprintf("task %s %s cannot determine restored file name: %v", taskDirName, entry.label, err)) + continue + } source, err := safeJoin(inputRoot, entry.value) if err != nil { warnings = append(warnings, fmt.Sprintf("task %s %s invalid: %v", taskDirName, entry.label, err)) @@ -646,7 +671,158 @@ func orbitWarnings(inputRoot string, p pair, taskDirName string) []string { return warnings } +func copyDataTree(sourceDir string, destRoleDir string, bundlePath string, sourcePath string) error { + rootName, hasRootName, err := restoredDataRootName(bundlePath, sourcePath) + if err != nil { + return err + } + if hasRootName { + if nestedSource, ok, err := singleNestedDataRoot(sourceDir, rootName); err != nil { + return err + } else if ok { + sourceDir = nestedSource + } + } + return copyDirContentsRenamed(sourceDir, destRoleDir, restoredDataEntryName) +} + +func singleNestedDataRoot(sourceDir string, rootName string) (string, bool, error) { + entries, err := os.ReadDir(sourceDir) + if err != nil { + return "", false, err + } + if len(entries) != 1 { + return "", false, nil + } + entry := entries[0] + if !entry.IsDir() { + return "", false, nil + } + if restoredDataEntryName(entry.Name()) != rootName { + return "", false, nil + } + return filepath.Join(sourceDir, entry.Name()), true, nil +} + +func restoredDataRootName(bundlePath string, sourcePath string) (string, bool, error) { + if name, ok, err := originalNameFromSourcePath(sourcePath); err != nil { + return "", false, err + } else if ok { + return name, true, nil + } + + name := pathLeafName(bundlePath) + if stripped := stripBundlePrefix(name, "scene"); stripped != name { + if err := validateRestoredLeafName(stripped); err != nil { + return "", false, err + } + return stripped, true, nil + } + return "", false, nil +} + +func restoredDataEntryName(name string) string { + return stripBundlePrefix(name, "scene") +} + +func restoredOrbitFileName(bundlePath string, sourcePath string) (string, error) { + if name, ok, err := originalNameFromSourcePath(sourcePath); err != nil { + return "", err + } else if ok { + return name, nil + } + + name := stripBundlePrefix(pathLeafName(bundlePath), "orbit") + if err := validateRestoredLeafName(name); err != nil { + return "", err + } + return name, nil +} + +func originalNameFromSourcePath(sourcePath string) (string, bool, error) { + name := pathLeafName(sourcePath) + if name == "" { + return "", false, nil + } + switch strings.ToLower(name) { + case "master", "slave", "data", "orbit": + return "", false, nil + } + if err := validateRestoredLeafName(name); err != nil { + return "", false, err + } + return name, true, nil +} + +func stripBundlePrefix(name string, prefix string) string { + rest, ok := strings.CutPrefix(name, prefix+"_") + if !ok { + return name + } + hashEnd := strings.IndexByte(rest, '_') + if hashEnd <= 0 || hashEnd == len(rest)-1 { + return name + } + if !looksLikeHashToken(rest[:hashEnd]) { + return name + } + return rest[hashEnd+1:] +} + +func looksLikeHashToken(value string) bool { + if len(value) < 6 { + return false + } + for _, r := range value { + switch { + case r >= '0' && r <= '9': + case r >= 'a' && r <= 'f': + case r >= 'A' && r <= 'F': + default: + return false + } + } + return true +} + +func pathLeafName(value string) string { + value = strings.TrimSpace(value) + value = strings.TrimRight(value, `/\`) + if value == "" { + return "" + } + index := strings.LastIndexAny(value, `/\`) + if index >= 0 { + return value[index+1:] + } + return value +} + +func validateRestoredLeafName(name string) error { + if name == "" || name == "." || name == ".." { + return fmt.Errorf("invalid empty file name") + } + if strings.ContainsAny(name, `<>:"/\|?*`) { + return fmt.Errorf("invalid file name %q", name) + } + if strings.HasSuffix(name, " ") || strings.HasSuffix(name, ".") { + return fmt.Errorf("invalid file name %q", name) + } + for _, r := range name { + if r < 32 { + return fmt.Errorf("invalid file name %q", name) + } + } + return nil +} + func copyDirContents(sourceDir string, destDir string) error { + return copyDirContentsRenamed(sourceDir, destDir, func(name string) string { + return name + }) +} + +func copyDirContentsRenamed(sourceDir string, destDir string, renameEntry func(string) string) error { sourceEntries, err := os.ReadDir(sourceDir) if err != nil { return err @@ -656,7 +832,11 @@ func copyDirContents(sourceDir string, destDir string) error { } for _, entry := range sourceEntries { sourcePath := filepath.Join(sourceDir, entry.Name()) - destPath := filepath.Join(destDir, entry.Name()) + destName := renameEntry(entry.Name()) + if err := validateRestoredLeafName(destName); err != nil { + return err + } + destPath := filepath.Join(destDir, destName) info, err := entry.Info() if err != nil { return err @@ -666,7 +846,7 @@ func copyDirContents(sourceDir string, destDir string) error { case mode&os.ModeSymlink != 0: return fmt.Errorf("symlinks are not supported: %s", sourcePath) case info.IsDir(): - if err := copyDirContents(sourcePath, destPath); err != nil { + if err := copyDirContentsRenamed(sourcePath, destPath, renameEntry); err != nil { return err } case mode.IsRegular(): diff --git a/main_test.go b/main_test.go index 83f76c7..47a5792 100644 --- a/main_test.go +++ b/main_test.go @@ -59,6 +59,95 @@ func TestRunRestoresPair(t *testing.T) { } } +func TestRunRestoresOriginalDataAndOrbitNames(t *testing.T) { + root := t.TempDir() + input := filepath.Join(root, "bundle") + output := filepath.Join(root, "out") + + masterName := "LT1B_MONO_SYC_STRIP1_019663_E131.0_N44.8_20251010_SLC_HH_S2A_0000898998" + slaveName := "LT1A_MONO_SYC_STRIP1_019664_E131.0_N44.8_20251022_SLC_HH_S2A_0000899001" + masterBundleName := "scene_e1e6ef658c_" + masterName + slaveBundleName := "scene_abc123def4_" + slaveName + + writeTestFile(t, filepath.Join(input, "data", masterBundleName, "scene_e1e6ef658c_"+masterName+".tiff"), "master") + writeTestFile(t, filepath.Join(input, "data", slaveBundleName, "scene_abc123def4_"+slaveName+".tiff"), "slave") + writeTestFile(t, filepath.Join(input, "orbit", "orbit_0123abcd_LT1A_GpsData_GAS_C_20230508.txt"), "master orbit") + writeTestFile(t, filepath.Join(input, "orbit", "orbit_abcd1234_LT1B_GpsData_GAS_C_20230508.txt"), "slave orbit") + writeTestJSON(t, filepath.Join(input, pairsFileName), pairsDocument{ + Pairs: []pair{{ + PairID: "pair_0001", + TaskName: "Task_20251010_20251022", + MasterData: filepath.ToSlash(filepath.Join("data", masterBundleName)), + SlaveData: filepath.ToSlash(filepath.Join("data", slaveBundleName)), + MasterOrbit: "orbit/orbit_0123abcd_LT1A_GpsData_GAS_C_20230508.txt", + SlaveOrbit: "orbit/orbit_abcd1234_LT1B_GpsData_GAS_C_20230508.txt", + }}, + }) + + result, err := run(config{inputRoot: input, outputRoot: output, skipExisting: true, logWriter: io.Discard}) + if err != nil { + t.Fatalf("run failed: %v", err) + } + if result.report.Restored != 1 { + t.Fatalf("restored = %d, want 1", result.report.Restored) + } + + task := filepath.Join(output, "Task_20251010_20251022") + for _, path := range []string{ + filepath.Join(task, "master", masterName+".tiff"), + filepath.Join(task, "slave", slaveName+".tiff"), + filepath.Join(task, "orbit", "LT1A_GpsData_GAS_C_20230508.txt"), + filepath.Join(task, "orbit", "LT1B_GpsData_GAS_C_20230508.txt"), + } { + if _, err := os.Stat(path); err != nil { + t.Fatalf("expected %s to exist: %v", path, err) + } + } + for _, path := range []string{ + filepath.Join(task, "master", masterBundleName), + filepath.Join(task, "slave", slaveBundleName), + filepath.Join(task, "master", "scene_e1e6ef658c_"+masterName+".tiff"), + filepath.Join(task, "slave", "scene_abc123def4_"+slaveName+".tiff"), + filepath.Join(task, "orbit", "orbit_0123abcd_LT1A_GpsData_GAS_C_20230508.txt"), + filepath.Join(task, "orbit", "orbit_abcd1234_LT1B_GpsData_GAS_C_20230508.txt"), + } { + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("expected %s to be absent, stat err=%v", path, err) + } + } +} + +func TestRunCollapsesNestedPrefixedDataRoot(t *testing.T) { + root := t.TempDir() + input := filepath.Join(root, "bundle") + output := filepath.Join(root, "out") + + originalName := "LT1B_MONO_SYC_STRIP1_019663_E131.0_N44.8_20251010_SLC_HH_S2A_0000898998" + bundleName := "scene_e1e6ef658c_" + originalName + writeTestFile(t, filepath.Join(input, "data", bundleName, bundleName, "m.txt"), "master") + writeTestFile(t, filepath.Join(input, "data", "scene_abc123def4_slave", "s.txt"), "slave") + writeTestJSON(t, filepath.Join(input, pairsFileName), pairsDocument{ + Pairs: []pair{{ + PairID: "pair_0001", + TaskName: "Task_20251010_20251022", + MasterData: filepath.ToSlash(filepath.Join("data", bundleName)), + SlaveData: "data/scene_abc123def4_slave", + }}, + }) + + if _, err := run(config{inputRoot: input, outputRoot: output, skipExisting: true, logWriter: io.Discard}); err != nil { + t.Fatalf("run failed: %v", err) + } + + task := filepath.Join(output, "Task_20251010_20251022") + if _, err := os.Stat(filepath.Join(task, "master", "m.txt")); err != nil { + t.Fatalf("expected nested data root to be collapsed: %v", err) + } + if _, err := os.Stat(filepath.Join(task, "master", originalName)); !os.IsNotExist(err) { + t.Fatalf("expected duplicated nested data root to be absent, stat err=%v", err) + } +} + func TestRunSkipsExistingCompleteTask(t *testing.T) { root := t.TempDir() input := filepath.Join(root, "bundle")