From 9d69a4afd1bb18333212dce43cd613c43106fb89 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 18 Apr 2026 20:47:35 +0800 Subject: [PATCH] Fix move duplicate handling with hash verification --- README.md | 4 + main_test.go | 312 +++++++++++++++++++++++++++++++++++++++++++++++++++ watcher.go | 301 ++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 600 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 1b5f043..62e7520 100644 --- a/README.md +++ b/README.md @@ -13,12 +13,14 @@ The program supports two rule types at the same time: Copies or moves files to a `zip` directory Skips when the `zip` directory already has a file with the same name Skips when the `unzip` directory already has an extracted folder with the same name + In `move` mode, if the existing archive or extracted folder is verified as the same content, the source file is removed - Generic file rule Watches other file types by extension list Only needs `UNC source path + local target path` Does not use `unzip` Supports two dedupe modes + In `move` mode, duplicate files are removed from the source only after hash verification ## Generic File Dedupe @@ -28,6 +30,7 @@ The generic file rule supports these dedupe modes: This is the default mode If the target directory already has a file with the same name and the same size, the file is skipped If the target directory already has a file with the same name but a different size, it is treated as a conflict and skipped without overwrite + In `move` mode, the program still verifies SHA-256 before deleting the source duplicate - `Same name and same hash` When the target directory already has a file with the same name, the program computes SHA-256 for source and target @@ -112,5 +115,6 @@ When you open the page again, the last saved settings are loaded automatically. - In `copy` mode, source files remain in the UNC path - In `move` mode, the program copies first and then removes the source file +- In `move` mode, if a duplicate is detected, the source file is removed only after hash/content verification succeeds - The generic file rule never overwrites an existing target file when a conflict is detected - The archive rule only handles `.tar.gz` diff --git a/main_test.go b/main_test.go index 3fe79d4..91df152 100644 --- a/main_test.go +++ b/main_test.go @@ -1,9 +1,12 @@ package main import ( + "archive/tar" + "compress/gzip" "os" "path/filepath" "reflect" + "sort" "testing" "time" ) @@ -140,6 +143,265 @@ func TestEvaluateGenericDuplicateNameHash(t *testing.T) { } } +func TestTransferFileMoveRemovesSource(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + sourcePath := filepath.Join(dir, "source.csv") + targetPath := filepath.Join(dir, "target.csv") + want := []byte("move-me") + + if err := os.WriteFile(sourcePath, want, 0o644); err != nil { + t.Fatalf("write source error = %v", err) + } + + if err := transferFile(sourcePath, targetPath, "move"); err != nil { + t.Fatalf("transferFile() error = %v", err) + } + + if _, err := os.Stat(sourcePath); !os.IsNotExist(err) { + t.Fatalf("expected source to be removed after move, stat err = %v", err) + } + + got, err := os.ReadFile(targetPath) + if err != nil { + t.Fatalf("read target error = %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("target content = %q, want %q", got, want) + } +} + +func TestProcessGenericCandidateMoveDuplicateDeletesSourceAfterHashVerification(t *testing.T) { + t.Parallel() + + rootDir := t.TempDir() + sourceDir := filepath.Join(rootDir, "source") + targetDir := filepath.Join(rootDir, "target") + if err := os.MkdirAll(sourceDir, 0o755); err != nil { + t.Fatalf("mkdir source error = %v", err) + } + if err := os.MkdirAll(targetDir, 0o755); err != nil { + t.Fatalf("mkdir target error = %v", err) + } + + sourcePath := filepath.Join(sourceDir, "dup.csv") + targetPath := filepath.Join(targetDir, "dup.csv") + data := []byte("same-content") + + if err := os.WriteFile(sourcePath, data, 0o644); err != nil { + t.Fatalf("write source error = %v", err) + } + if err := os.WriteFile(targetPath, data, 0o644); err != nil { + t.Fatalf("write target error = %v", err) + } + + cfg := config{ + mode: "move", + generic: genericRule{ + targetDir: targetDir, + dedupeMode: genericDedupeNameSize, + }, + } + + decision, handled, err := processGenericCandidate(cfg, sourcePath) + if err != nil { + t.Fatalf("processGenericCandidate() error = %v", err) + } + if !handled { + t.Fatalf("expected duplicate move to be treated as handled") + } + if decision == "" { + t.Fatalf("expected non-empty decision") + } + if _, err := os.Stat(sourcePath); !os.IsNotExist(err) { + t.Fatalf("expected source to be removed, stat err = %v", err) + } +} + +func TestProcessGenericCandidateMoveDuplicateKeepsSourceWhenHashDiffers(t *testing.T) { + t.Parallel() + + rootDir := t.TempDir() + sourceDir := filepath.Join(rootDir, "source") + targetDir := filepath.Join(rootDir, "target") + if err := os.MkdirAll(sourceDir, 0o755); err != nil { + t.Fatalf("mkdir source error = %v", err) + } + if err := os.MkdirAll(targetDir, 0o755); err != nil { + t.Fatalf("mkdir target error = %v", err) + } + + sourcePath := filepath.Join(sourceDir, "dup.csv") + targetPath := filepath.Join(targetDir, "dup.csv") + + if err := os.WriteFile(sourcePath, []byte("abcd"), 0o644); err != nil { + t.Fatalf("write source error = %v", err) + } + if err := os.WriteFile(targetPath, []byte("wxyz"), 0o644); err != nil { + t.Fatalf("write target error = %v", err) + } + + cfg := config{ + mode: "move", + generic: genericRule{ + targetDir: targetDir, + dedupeMode: genericDedupeNameSize, + }, + } + + _, handled, err := processGenericCandidate(cfg, sourcePath) + if err != nil { + t.Fatalf("processGenericCandidate() error = %v", err) + } + if handled { + t.Fatalf("expected hash mismatch not to be treated as handled duplicate") + } + if _, err := os.Stat(sourcePath); err != nil { + t.Fatalf("expected source to remain, stat err = %v", err) + } +} + +func TestProcessArchiveCandidateMoveDuplicateZipDeletesSource(t *testing.T) { + t.Parallel() + + rootDir := t.TempDir() + sourceDir := filepath.Join(rootDir, "source") + zipDir := filepath.Join(rootDir, "zip") + unzipDir := filepath.Join(rootDir, "unzip") + for _, dir := range []string{sourceDir, zipDir, unzipDir} { + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir %q error = %v", dir, err) + } + } + + sourcePath := filepath.Join(sourceDir, "demo.tar.gz") + targetPath := filepath.Join(zipDir, "demo.tar.gz") + data := []byte("archive-binary") + + if err := os.WriteFile(sourcePath, data, 0o644); err != nil { + t.Fatalf("write source archive error = %v", err) + } + if err := os.WriteFile(targetPath, data, 0o644); err != nil { + t.Fatalf("write target archive error = %v", err) + } + + cfg := config{ + mode: "move", + archive: archiveRule{ + zipDir: zipDir, + unzipDir: unzipDir, + }, + } + + decision, handled, err := processArchiveCandidate(cfg, sourcePath) + if err != nil { + t.Fatalf("processArchiveCandidate() error = %v", err) + } + if !handled { + t.Fatalf("expected duplicate archive move to be treated as handled") + } + if decision == "" { + t.Fatalf("expected non-empty decision") + } + if _, err := os.Stat(sourcePath); !os.IsNotExist(err) { + t.Fatalf("expected source archive to be removed, stat err = %v", err) + } +} + +func TestProcessArchiveCandidateMoveDuplicateUnzipDeletesSource(t *testing.T) { + t.Parallel() + + rootDir := t.TempDir() + sourceDir := filepath.Join(rootDir, "source") + zipDir := filepath.Join(rootDir, "zip") + unzipDir := filepath.Join(rootDir, "unzip") + for _, dir := range []string{sourceDir, zipDir, unzipDir} { + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir %q error = %v", dir, err) + } + } + + sourcePath := filepath.Join(sourceDir, "demo.tar.gz") + unzipTarget := filepath.Join(unzipDir, "demo") + files := map[string]string{ + "a.txt": "hello", + "nested/b.json": `{"ok":true}`, + "nested/c/data": "payload", + } + + if err := writeTarGz(sourcePath, "demo", files); err != nil { + t.Fatalf("write tar.gz error = %v", err) + } + if err := writeExtractedDir(unzipTarget, files); err != nil { + t.Fatalf("write extracted dir error = %v", err) + } + + cfg := config{ + mode: "move", + archive: archiveRule{ + zipDir: zipDir, + unzipDir: unzipDir, + }, + } + + decision, handled, err := processArchiveCandidate(cfg, sourcePath) + if err != nil { + t.Fatalf("processArchiveCandidate() error = %v", err) + } + if !handled { + t.Fatalf("expected matching unzip directory to be treated as handled duplicate") + } + if decision == "" { + t.Fatalf("expected non-empty decision") + } + if _, err := os.Stat(sourcePath); !os.IsNotExist(err) { + t.Fatalf("expected source archive to be removed, stat err = %v", err) + } +} + +func TestProcessArchiveCandidateMoveDuplicateUnzipKeepsSourceWhenContentDiffers(t *testing.T) { + t.Parallel() + + rootDir := t.TempDir() + sourceDir := filepath.Join(rootDir, "source") + zipDir := filepath.Join(rootDir, "zip") + unzipDir := filepath.Join(rootDir, "unzip") + for _, dir := range []string{sourceDir, zipDir, unzipDir} { + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir %q error = %v", dir, err) + } + } + + sourcePath := filepath.Join(sourceDir, "demo.tar.gz") + unzipTarget := filepath.Join(unzipDir, "demo") + if err := writeTarGz(sourcePath, "demo", map[string]string{"a.txt": "hello"}); err != nil { + t.Fatalf("write tar.gz error = %v", err) + } + if err := writeExtractedDir(unzipTarget, map[string]string{"a.txt": "DIFF"}); err != nil { + t.Fatalf("write extracted dir error = %v", err) + } + + cfg := config{ + mode: "move", + archive: archiveRule{ + zipDir: zipDir, + unzipDir: unzipDir, + }, + } + + _, handled, err := processArchiveCandidate(cfg, sourcePath) + if err != nil { + t.Fatalf("processArchiveCandidate() error = %v", err) + } + if handled { + t.Fatalf("expected different unzip content not to be treated as handled duplicate") + } + if _, err := os.Stat(sourcePath); err != nil { + t.Fatalf("expected source archive to remain, stat err = %v", err) + } +} + func TestSaveAndLoadSettings(t *testing.T) { t.Parallel() @@ -212,3 +474,53 @@ func TestLoadLegacySettings(t *testing.T) { t.Fatalf("expected legacy settings to default to name_size dedupe") } } + +func writeTarGz(targetPath, rootName string, files map[string]string) error { + file, err := os.Create(targetPath) + if err != nil { + return err + } + defer file.Close() + + gzipWriter := gzip.NewWriter(file) + defer gzipWriter.Close() + + tarWriter := tar.NewWriter(gzipWriter) + defer tarWriter.Close() + + paths := make([]string, 0, len(files)) + for name := range files { + paths = append(paths, name) + } + sort.Strings(paths) + + for _, name := range paths { + body := []byte(files[name]) + header := &tar.Header{ + Name: filepath.ToSlash(filepath.Join(rootName, name)), + Mode: 0o644, + Size: int64(len(body)), + } + if err := tarWriter.WriteHeader(header); err != nil { + return err + } + if _, err := tarWriter.Write(body); err != nil { + return err + } + } + + return nil +} + +func writeExtractedDir(root string, files map[string]string) error { + for name, body := range files { + targetPath := filepath.Join(root, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(targetPath), 0o755); err != nil { + return err + } + if err := os.WriteFile(targetPath, []byte(body), 0o644); err != nil { + return err + } + } + return nil +} diff --git a/watcher.go b/watcher.go index 8ac3f85..588f4f7 100644 --- a/watcher.go +++ b/watcher.go @@ -1,15 +1,26 @@ package main import ( + "archive/tar" + "compress/gzip" "crypto/sha256" "fmt" "io" "os" + "path" "path/filepath" + "sort" "strings" "time" ) +type contentRecord struct { + path string + kind byte + size int64 + sum [32]byte +} + func ensureExistingDir(path string) error { info, err := os.Stat(path) if err != nil { @@ -162,11 +173,38 @@ func processArchiveCandidate(cfg config, sourcePath string) (string, bool, error zipTarget := filepath.Join(cfg.archive.zipDir, fileName) if pathExists(zipTarget) { + if cfg.mode == "move" { + same, err := filesHaveSameHash(sourcePath, zipTarget) + if err != nil { + return "", false, err + } + if same { + if err := os.Remove(sourcePath); err != nil { + return "", false, fmt.Errorf("remove duplicate archive source: %w", err) + } + return "已跳过传输，zip 目录已有同名压缩包且哈希一致，源文件已删除", true, nil + } + return "已跳过，zip 目录中存在同名压缩包但哈希不同", false, nil + } return "已跳过，zip 目录中存在同名压缩包", false, nil } - unzipTarget := filepath.Join(cfg.archive.unzipDir, extractedDirName(fileName)) + extractedDir := extractedDirName(fileName) + unzipTarget := filepath.Join(cfg.archive.unzipDir, extractedDir) if isDir(unzipTarget) { + if cfg.mode == "move" { + same, err := archiveMatchesDirectory(sourcePath, unzipTarget, extractedDir) + if err != nil { + return "", false, err + } + if same { + if err := os.Remove(sourcePath); err != nil { + return "", false, fmt.Errorf("remove duplicate archive source: %w", err) + } + return "已跳过传输，unzip 目录已有同名解压目录且内容一致，源文件已删除", true, nil + } + return "已跳过，unzip 目录中存在同名解压目录但内容不同", false, nil + } return "已跳过，unzip 目录中存在同名解压目录", false, nil } @@ -185,6 +223,10 @@ func processGenericCandidate(cfg config, sourcePath string) (string, bool, error targetPath := filepath.Join(cfg.generic.targetDir, fileName) if pathExists(targetPath) { + if cfg.mode == "move" { + return handleGenericDuplicateMove(sourcePath, targetPath, cfg.generic.dedupeMode) + } + decision, duplicate, err := evaluateGenericDuplicate(sourcePath, targetPath, cfg.generic.dedupeMode) if err != nil { return "", false, err @@ -206,40 +248,70 @@ func processGenericCandidate(cfg config, sourcePath string) (string, bool, error } func evaluateGenericDuplicate(sourcePath, targetPath, dedupeMode string) (string, bool, error) { + decision, duplicate, _, err := analyzeGenericDuplicate(sourcePath, targetPath, dedupeMode) + return decision, duplicate, err +} + +func analyzeGenericDuplicate(sourcePath, targetPath, dedupeMode string) (string, bool, bool, error) { sourceInfo, err := os.Stat(sourcePath) if err != nil { - return "", false, fmt.Errorf("stat source: %w", err) + return "", false, false, fmt.Errorf("stat source: %w", err) } targetInfo, err := os.Stat(targetPath) if err != nil { - return "", false, fmt.Errorf("stat target: %w", err) + return "", false, false, fmt.Errorf("stat target: %w", err) } switch dedupeMode { case genericDedupeNameHash: if sourceInfo.Size() != targetInfo.Size() { - return "已跳过，同名文件已存在但大小不同", false, nil + return "已跳过，同名文件已存在但大小不同", false, false, nil } same, err := filesHaveSameHash(sourcePath, targetPath) if err != nil { - return "", false, err + return "", false, false, err } if same { - return "已跳过，同名文件已存在且哈希相同", true, nil + return "已跳过，同名文件已存在且哈希相同", true, true, nil } - return "已跳过，同名文件已存在但哈希不同", false, nil + return "已跳过，同名文件已存在但哈希不同", false, true, nil case genericDedupeNameSize: fallthrough default: if sourceInfo.Size() == targetInfo.Size() { - return "已跳过，同名文件已存在且大小相同", true, nil + return "已跳过，同名文件已存在且大小相同", true, false, nil } - return "已跳过，同名文件已存在但大小不同", false, nil + return "已跳过，同名文件已存在但大小不同", false, false, nil } } +func handleGenericDuplicateMove(sourcePath, targetPath, dedupeMode string) (string, bool, error) { + decision, duplicate, hashVerified, err := analyzeGenericDuplicate(sourcePath, targetPath, dedupeMode) + if err != nil { + return "", false, err + } + if !duplicate { + return decision, false, nil + } + + if !hashVerified { + same, err := filesHaveSameHash(sourcePath, targetPath) + if err != nil { + return "", false, err + } + if !same { + return "已跳过，同名文件大小相同但哈希不同", false, nil + } + } + + if err := os.Remove(sourcePath); err != nil { + return "", false, fmt.Errorf("remove duplicate generic source: %w", err) + } + return "已跳过传输，目标目录已有同名文件且哈希一致，源文件已删除", true, nil +} + func filesHaveSameHash(sourcePath, targetPath string) (bool, error) { sourceHash, err := fileSHA256(sourcePath) if err != nil { @@ -254,6 +326,192 @@ func filesHaveSameHash(sourcePath, targetPath string) (bool, error) { return sourceHash == targetHash, nil } +func archiveMatchesDirectory(sourcePath, targetPath, archiveRoot string) (bool, error) { + sourceRecords, err := archiveContentRecords(sourcePath, archiveRoot) + if err != nil { + return false, err + } + + targetRecords, err := directoryContentRecords(targetPath) + if err != nil { + return false, err + } + + if len(sourceRecords) != len(targetRecords) { + return false, nil + } + + for i := range sourceRecords { + if sourceRecords[i] != targetRecords[i] { + return false, nil + } + } + + return true, nil +} + +func archiveContentRecords(archivePath, archiveRoot string) ([]contentRecord, error) { + file, err := os.Open(archivePath) + if err != nil { + return nil, fmt.Errorf("open archive: %w", err) + } + defer file.Close() + + gzipReader, err := gzip.NewReader(file) + if err != nil { + return nil, fmt.Errorf("open gzip reader: %w", err) + } + defer gzipReader.Close() + + records := make(map[string]contentRecord) + tarReader := tar.NewReader(gzipReader) + + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("read archive entry: %w", err) + } + + name := normalizeArchiveRecordPath(header.Name, archiveRoot) + switch header.Typeflag { + case tar.TypeReg, tar.TypeRegA, tar.TypeGNUSparse: + if name == "" { + return nil, fmt.Errorf("archive entry %q resolves to empty path", header.Name) + } + + sum, err := readerSHA256(tarReader) + if err != nil { + return nil, fmt.Errorf("hash archive entry %q: %w", header.Name, err) + } + records[name] = contentRecord{ + path: name, + kind: 'F', + size: header.Size, + sum: sum, + } + + case tar.TypeSymlink: + if name == "" { + return nil, fmt.Errorf("archive symlink %q resolves to empty path", header.Name) + } + + records[name] = contentRecord{ + path: name, + kind: 'L', + sum: sha256.Sum256([]byte(filepath.ToSlash(header.Linkname))), + } + + case tar.TypeDir, tar.TypeXHeader, tar.TypeXGlobalHeader, tar.TypeGNULongName, tar.TypeGNULongLink: + continue + + default: + return nil, fmt.Errorf("unsupported archive entry type for %q: %d", header.Name, header.Typeflag) + } + } + + return sortedContentRecords(records), nil +} + +func directoryContentRecords(root string) ([]contentRecord, error) { + records := make(map[string]contentRecord) + + err := filepath.Walk(root, func(currentPath string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + return walkErr + } + if currentPath == root { + return nil + } + + relativePath, err := filepath.Rel(root, currentPath) + if err != nil { + return fmt.Errorf("build relative path for %q: %w", currentPath, err) + } + relativePath = filepath.ToSlash(relativePath) + + if info.Mode()&os.ModeSymlink != 0 { + linkTarget, err := os.Readlink(currentPath) + if err != nil { + return fmt.Errorf("read symlink %q: %w", currentPath, err) + } + records[relativePath] = contentRecord{ + path: relativePath, + kind: 'L', + sum: sha256.Sum256([]byte(filepath.ToSlash(linkTarget))), + } + return nil + } + if info.IsDir() { + return nil + } + if !info.Mode().IsRegular() { + return fmt.Errorf("unsupported directory entry %q", currentPath) + } + + sum, err := fileSHA256(currentPath) + if err != nil { + return fmt.Errorf("hash directory file %q: %w", currentPath, err) + } + records[relativePath] = contentRecord{ + path: relativePath, + kind: 'F', + size: info.Size(), + sum: sum, + } + return nil + }) + if err != nil { + return nil, err + } + + return sortedContentRecords(records), nil +} + +func normalizeArchiveRecordPath(name, archiveRoot string) string { + cleanPath := strings.TrimSpace(filepath.ToSlash(name)) + cleanPath = strings.TrimPrefix(cleanPath, "./") + cleanPath = strings.TrimPrefix(cleanPath, "/") + cleanPath = path.Clean(cleanPath) + + if cleanPath == "." { + return "" + } + + normalizedRoot := path.Clean(strings.Trim(filepath.ToSlash(archiveRoot), "/")) + if normalizedRoot != "." && normalizedRoot != "" { + if cleanPath == normalizedRoot { + return "" + } + prefix := normalizedRoot + "/" + if strings.HasPrefix(cleanPath, prefix) { + cleanPath = strings.TrimPrefix(cleanPath, prefix) + } + } + + if cleanPath == "." { + return "" + } + return cleanPath +} + +func sortedContentRecords(records map[string]contentRecord) []contentRecord { + items := make([]contentRecord, 0, len(records)) + for _, record := range records { + items = append(items, record) + } + + sort.Slice(items, func(i, j int) bool { + if items[i].path == items[j].path { + return items[i].kind < items[j].kind + } + return items[i].path < items[j].path + }) + return items +} + func fileSHA256(path string) ([32]byte, error) { file, err := os.Open(path) if err != nil { @@ -261,8 +519,12 @@ func fileSHA256(path string) ([32]byte, error) { } defer file.Close() + return readerSHA256(file) +} + +func readerSHA256(reader io.Reader) ([32]byte, error) { hasher := sha256.New() - if _, err := io.Copy(hasher, file); err != nil { + if _, err := io.Copy(hasher, reader); err != nil { return [32]byte{}, err } @@ -288,22 +550,27 @@ func transferFile(sourcePath, targetPath, mode string) error { if err != nil { return fmt.Errorf("open source: %w", err) } - defer source.Close() target, err := os.OpenFile(tempPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644) if err != nil { + _ = source.Close() return fmt.Errorf("create temp target: %w", err) } - copyErr := copyFileContents(target, source) - closeErr := target.Close() - if copyErr != nil { + if err := copyFileContents(target, source); err != nil { + _ = target.Close() + _ = source.Close() _ = os.Remove(tempPath) - return copyErr + return err } - if closeErr != nil { + if err := target.Close(); err != nil { + _ = source.Close() _ = os.Remove(tempPath) - return fmt.Errorf("close temp target: %w", closeErr) + return fmt.Errorf("close temp target: %w", err) + } + if err := source.Close(); err != nil { + _ = os.Remove(tempPath) + return fmt.Errorf("close source: %w", err) } if err := os.Rename(tempPath, targetPath); err != nil {