Fix move duplicate handling with hash verification
This commit is contained in:
@@ -13,12 +13,14 @@ The program supports two rule types at the same time:
|
|||||||
Copies or moves files to a `zip` directory
|
Copies or moves files to a `zip` directory
|
||||||
Skips when the `zip` directory already has a file with the same name
|
Skips when the `zip` directory already has a file with the same name
|
||||||
Skips when the `unzip` directory already has an extracted folder with the same name
|
Skips when the `unzip` directory already has an extracted folder with the same name
|
||||||
|
In `move` mode, if the existing archive or extracted folder is verified as the same content, the source file is removed
|
||||||
|
|
||||||
- Generic file rule
|
- Generic file rule
|
||||||
Watches other file types by extension list
|
Watches other file types by extension list
|
||||||
Only needs `UNC source path + local target path`
|
Only needs `UNC source path + local target path`
|
||||||
Does not use `unzip`
|
Does not use `unzip`
|
||||||
Supports two dedupe modes
|
Supports two dedupe modes
|
||||||
|
In `move` mode, duplicate files are removed from the source only after hash verification
|
||||||
|
|
||||||
## Generic File Dedupe
|
## Generic File Dedupe
|
||||||
|
|
||||||
@@ -28,6 +30,7 @@ The generic file rule supports these dedupe modes:
|
|||||||
This is the default mode
|
This is the default mode
|
||||||
If the target directory already has a file with the same name and the same size, the file is skipped
|
If the target directory already has a file with the same name and the same size, the file is skipped
|
||||||
If the target directory already has a file with the same name but a different size, it is treated as a conflict and skipped without overwrite
|
If the target directory already has a file with the same name but a different size, it is treated as a conflict and skipped without overwrite
|
||||||
|
In `move` mode, the program still verifies SHA-256 before deleting the source duplicate
|
||||||
|
|
||||||
- `Same name and same hash`
|
- `Same name and same hash`
|
||||||
When the target directory already has a file with the same name, the program computes SHA-256 for source and target
|
When the target directory already has a file with the same name, the program computes SHA-256 for source and target
|
||||||
@@ -112,5 +115,6 @@ When you open the page again, the last saved settings are loaded automatically.
|
|||||||
|
|
||||||
- In `copy` mode, source files remain in the UNC path
|
- In `copy` mode, source files remain in the UNC path
|
||||||
- In `move` mode, the program copies first and then removes the source file
|
- In `move` mode, the program copies first and then removes the source file
|
||||||
|
- In `move` mode, if a duplicate is detected, the source file is removed only after hash/content verification succeeds
|
||||||
- The generic file rule never overwrites an existing target file when a conflict is detected
|
- The generic file rule never overwrites an existing target file when a conflict is detected
|
||||||
- The archive rule only handles `.tar.gz`
|
- The archive rule only handles `.tar.gz`
|
||||||
|
|||||||
+312
@@ -1,9 +1,12 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"archive/tar"
|
||||||
|
"compress/gzip"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"sort"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@@ -140,6 +143,265 @@ func TestEvaluateGenericDuplicateNameHash(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestTransferFileMoveRemovesSource(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
sourcePath := filepath.Join(dir, "source.csv")
|
||||||
|
targetPath := filepath.Join(dir, "target.csv")
|
||||||
|
want := []byte("move-me")
|
||||||
|
|
||||||
|
if err := os.WriteFile(sourcePath, want, 0o644); err != nil {
|
||||||
|
t.Fatalf("write source error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := transferFile(sourcePath, targetPath, "move"); err != nil {
|
||||||
|
t.Fatalf("transferFile() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := os.Stat(sourcePath); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("expected source to be removed after move, stat err = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := os.ReadFile(targetPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read target error = %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("target content = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessGenericCandidateMoveDuplicateDeletesSourceAfterHashVerification(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
rootDir := t.TempDir()
|
||||||
|
sourceDir := filepath.Join(rootDir, "source")
|
||||||
|
targetDir := filepath.Join(rootDir, "target")
|
||||||
|
if err := os.MkdirAll(sourceDir, 0o755); err != nil {
|
||||||
|
t.Fatalf("mkdir source error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(targetDir, 0o755); err != nil {
|
||||||
|
t.Fatalf("mkdir target error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sourcePath := filepath.Join(sourceDir, "dup.csv")
|
||||||
|
targetPath := filepath.Join(targetDir, "dup.csv")
|
||||||
|
data := []byte("same-content")
|
||||||
|
|
||||||
|
if err := os.WriteFile(sourcePath, data, 0o644); err != nil {
|
||||||
|
t.Fatalf("write source error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(targetPath, data, 0o644); err != nil {
|
||||||
|
t.Fatalf("write target error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := config{
|
||||||
|
mode: "move",
|
||||||
|
generic: genericRule{
|
||||||
|
targetDir: targetDir,
|
||||||
|
dedupeMode: genericDedupeNameSize,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
decision, handled, err := processGenericCandidate(cfg, sourcePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("processGenericCandidate() error = %v", err)
|
||||||
|
}
|
||||||
|
if !handled {
|
||||||
|
t.Fatalf("expected duplicate move to be treated as handled")
|
||||||
|
}
|
||||||
|
if decision == "" {
|
||||||
|
t.Fatalf("expected non-empty decision")
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(sourcePath); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("expected source to be removed, stat err = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessGenericCandidateMoveDuplicateKeepsSourceWhenHashDiffers(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
rootDir := t.TempDir()
|
||||||
|
sourceDir := filepath.Join(rootDir, "source")
|
||||||
|
targetDir := filepath.Join(rootDir, "target")
|
||||||
|
if err := os.MkdirAll(sourceDir, 0o755); err != nil {
|
||||||
|
t.Fatalf("mkdir source error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(targetDir, 0o755); err != nil {
|
||||||
|
t.Fatalf("mkdir target error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sourcePath := filepath.Join(sourceDir, "dup.csv")
|
||||||
|
targetPath := filepath.Join(targetDir, "dup.csv")
|
||||||
|
|
||||||
|
if err := os.WriteFile(sourcePath, []byte("abcd"), 0o644); err != nil {
|
||||||
|
t.Fatalf("write source error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(targetPath, []byte("wxyz"), 0o644); err != nil {
|
||||||
|
t.Fatalf("write target error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := config{
|
||||||
|
mode: "move",
|
||||||
|
generic: genericRule{
|
||||||
|
targetDir: targetDir,
|
||||||
|
dedupeMode: genericDedupeNameSize,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, handled, err := processGenericCandidate(cfg, sourcePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("processGenericCandidate() error = %v", err)
|
||||||
|
}
|
||||||
|
if handled {
|
||||||
|
t.Fatalf("expected hash mismatch not to be treated as handled duplicate")
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(sourcePath); err != nil {
|
||||||
|
t.Fatalf("expected source to remain, stat err = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessArchiveCandidateMoveDuplicateZipDeletesSource(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
rootDir := t.TempDir()
|
||||||
|
sourceDir := filepath.Join(rootDir, "source")
|
||||||
|
zipDir := filepath.Join(rootDir, "zip")
|
||||||
|
unzipDir := filepath.Join(rootDir, "unzip")
|
||||||
|
for _, dir := range []string{sourceDir, zipDir, unzipDir} {
|
||||||
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
|
t.Fatalf("mkdir %q error = %v", dir, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sourcePath := filepath.Join(sourceDir, "demo.tar.gz")
|
||||||
|
targetPath := filepath.Join(zipDir, "demo.tar.gz")
|
||||||
|
data := []byte("archive-binary")
|
||||||
|
|
||||||
|
if err := os.WriteFile(sourcePath, data, 0o644); err != nil {
|
||||||
|
t.Fatalf("write source archive error = %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(targetPath, data, 0o644); err != nil {
|
||||||
|
t.Fatalf("write target archive error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := config{
|
||||||
|
mode: "move",
|
||||||
|
archive: archiveRule{
|
||||||
|
zipDir: zipDir,
|
||||||
|
unzipDir: unzipDir,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
decision, handled, err := processArchiveCandidate(cfg, sourcePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("processArchiveCandidate() error = %v", err)
|
||||||
|
}
|
||||||
|
if !handled {
|
||||||
|
t.Fatalf("expected duplicate archive move to be treated as handled")
|
||||||
|
}
|
||||||
|
if decision == "" {
|
||||||
|
t.Fatalf("expected non-empty decision")
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(sourcePath); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("expected source archive to be removed, stat err = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessArchiveCandidateMoveDuplicateUnzipDeletesSource(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
rootDir := t.TempDir()
|
||||||
|
sourceDir := filepath.Join(rootDir, "source")
|
||||||
|
zipDir := filepath.Join(rootDir, "zip")
|
||||||
|
unzipDir := filepath.Join(rootDir, "unzip")
|
||||||
|
for _, dir := range []string{sourceDir, zipDir, unzipDir} {
|
||||||
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
|
t.Fatalf("mkdir %q error = %v", dir, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sourcePath := filepath.Join(sourceDir, "demo.tar.gz")
|
||||||
|
unzipTarget := filepath.Join(unzipDir, "demo")
|
||||||
|
files := map[string]string{
|
||||||
|
"a.txt": "hello",
|
||||||
|
"nested/b.json": `{"ok":true}`,
|
||||||
|
"nested/c/data": "payload",
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := writeTarGz(sourcePath, "demo", files); err != nil {
|
||||||
|
t.Fatalf("write tar.gz error = %v", err)
|
||||||
|
}
|
||||||
|
if err := writeExtractedDir(unzipTarget, files); err != nil {
|
||||||
|
t.Fatalf("write extracted dir error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := config{
|
||||||
|
mode: "move",
|
||||||
|
archive: archiveRule{
|
||||||
|
zipDir: zipDir,
|
||||||
|
unzipDir: unzipDir,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
decision, handled, err := processArchiveCandidate(cfg, sourcePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("processArchiveCandidate() error = %v", err)
|
||||||
|
}
|
||||||
|
if !handled {
|
||||||
|
t.Fatalf("expected matching unzip directory to be treated as handled duplicate")
|
||||||
|
}
|
||||||
|
if decision == "" {
|
||||||
|
t.Fatalf("expected non-empty decision")
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(sourcePath); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("expected source archive to be removed, stat err = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessArchiveCandidateMoveDuplicateUnzipKeepsSourceWhenContentDiffers(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
rootDir := t.TempDir()
|
||||||
|
sourceDir := filepath.Join(rootDir, "source")
|
||||||
|
zipDir := filepath.Join(rootDir, "zip")
|
||||||
|
unzipDir := filepath.Join(rootDir, "unzip")
|
||||||
|
for _, dir := range []string{sourceDir, zipDir, unzipDir} {
|
||||||
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
|
t.Fatalf("mkdir %q error = %v", dir, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sourcePath := filepath.Join(sourceDir, "demo.tar.gz")
|
||||||
|
unzipTarget := filepath.Join(unzipDir, "demo")
|
||||||
|
if err := writeTarGz(sourcePath, "demo", map[string]string{"a.txt": "hello"}); err != nil {
|
||||||
|
t.Fatalf("write tar.gz error = %v", err)
|
||||||
|
}
|
||||||
|
if err := writeExtractedDir(unzipTarget, map[string]string{"a.txt": "DIFF"}); err != nil {
|
||||||
|
t.Fatalf("write extracted dir error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := config{
|
||||||
|
mode: "move",
|
||||||
|
archive: archiveRule{
|
||||||
|
zipDir: zipDir,
|
||||||
|
unzipDir: unzipDir,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, handled, err := processArchiveCandidate(cfg, sourcePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("processArchiveCandidate() error = %v", err)
|
||||||
|
}
|
||||||
|
if handled {
|
||||||
|
t.Fatalf("expected different unzip content not to be treated as handled duplicate")
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(sourcePath); err != nil {
|
||||||
|
t.Fatalf("expected source archive to remain, stat err = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestSaveAndLoadSettings(t *testing.T) {
|
func TestSaveAndLoadSettings(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -212,3 +474,53 @@ func TestLoadLegacySettings(t *testing.T) {
|
|||||||
t.Fatalf("expected legacy settings to default to name_size dedupe")
|
t.Fatalf("expected legacy settings to default to name_size dedupe")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func writeTarGz(targetPath, rootName string, files map[string]string) error {
|
||||||
|
file, err := os.Create(targetPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
gzipWriter := gzip.NewWriter(file)
|
||||||
|
defer gzipWriter.Close()
|
||||||
|
|
||||||
|
tarWriter := tar.NewWriter(gzipWriter)
|
||||||
|
defer tarWriter.Close()
|
||||||
|
|
||||||
|
paths := make([]string, 0, len(files))
|
||||||
|
for name := range files {
|
||||||
|
paths = append(paths, name)
|
||||||
|
}
|
||||||
|
sort.Strings(paths)
|
||||||
|
|
||||||
|
for _, name := range paths {
|
||||||
|
body := []byte(files[name])
|
||||||
|
header := &tar.Header{
|
||||||
|
Name: filepath.ToSlash(filepath.Join(rootName, name)),
|
||||||
|
Mode: 0o644,
|
||||||
|
Size: int64(len(body)),
|
||||||
|
}
|
||||||
|
if err := tarWriter.WriteHeader(header); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := tarWriter.Write(body); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeExtractedDir(root string, files map[string]string) error {
|
||||||
|
for name, body := range files {
|
||||||
|
targetPath := filepath.Join(root, filepath.FromSlash(name))
|
||||||
|
if err := os.MkdirAll(filepath.Dir(targetPath), 0o755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(targetPath, []byte(body), 0o644); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
+284
-17
@@ -1,15 +1,26 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"archive/tar"
|
||||||
|
"compress/gzip"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"path"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type contentRecord struct {
|
||||||
|
path string
|
||||||
|
kind byte
|
||||||
|
size int64
|
||||||
|
sum [32]byte
|
||||||
|
}
|
||||||
|
|
||||||
func ensureExistingDir(path string) error {
|
func ensureExistingDir(path string) error {
|
||||||
info, err := os.Stat(path)
|
info, err := os.Stat(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -162,11 +173,38 @@ func processArchiveCandidate(cfg config, sourcePath string) (string, bool, error
|
|||||||
zipTarget := filepath.Join(cfg.archive.zipDir, fileName)
|
zipTarget := filepath.Join(cfg.archive.zipDir, fileName)
|
||||||
|
|
||||||
if pathExists(zipTarget) {
|
if pathExists(zipTarget) {
|
||||||
|
if cfg.mode == "move" {
|
||||||
|
same, err := filesHaveSameHash(sourcePath, zipTarget)
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
if same {
|
||||||
|
if err := os.Remove(sourcePath); err != nil {
|
||||||
|
return "", false, fmt.Errorf("remove duplicate archive source: %w", err)
|
||||||
|
}
|
||||||
|
return "已跳过传输,zip 目录已有同名压缩包且哈希一致,源文件已删除", true, nil
|
||||||
|
}
|
||||||
|
return "已跳过,zip 目录中存在同名压缩包但哈希不同", false, nil
|
||||||
|
}
|
||||||
return "已跳过,zip 目录中存在同名压缩包", false, nil
|
return "已跳过,zip 目录中存在同名压缩包", false, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
unzipTarget := filepath.Join(cfg.archive.unzipDir, extractedDirName(fileName))
|
extractedDir := extractedDirName(fileName)
|
||||||
|
unzipTarget := filepath.Join(cfg.archive.unzipDir, extractedDir)
|
||||||
if isDir(unzipTarget) {
|
if isDir(unzipTarget) {
|
||||||
|
if cfg.mode == "move" {
|
||||||
|
same, err := archiveMatchesDirectory(sourcePath, unzipTarget, extractedDir)
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
if same {
|
||||||
|
if err := os.Remove(sourcePath); err != nil {
|
||||||
|
return "", false, fmt.Errorf("remove duplicate archive source: %w", err)
|
||||||
|
}
|
||||||
|
return "已跳过传输,unzip 目录已有同名解压目录且内容一致,源文件已删除", true, nil
|
||||||
|
}
|
||||||
|
return "已跳过,unzip 目录中存在同名解压目录但内容不同", false, nil
|
||||||
|
}
|
||||||
return "已跳过,unzip 目录中存在同名解压目录", false, nil
|
return "已跳过,unzip 目录中存在同名解压目录", false, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -185,6 +223,10 @@ func processGenericCandidate(cfg config, sourcePath string) (string, bool, error
|
|||||||
targetPath := filepath.Join(cfg.generic.targetDir, fileName)
|
targetPath := filepath.Join(cfg.generic.targetDir, fileName)
|
||||||
|
|
||||||
if pathExists(targetPath) {
|
if pathExists(targetPath) {
|
||||||
|
if cfg.mode == "move" {
|
||||||
|
return handleGenericDuplicateMove(sourcePath, targetPath, cfg.generic.dedupeMode)
|
||||||
|
}
|
||||||
|
|
||||||
decision, duplicate, err := evaluateGenericDuplicate(sourcePath, targetPath, cfg.generic.dedupeMode)
|
decision, duplicate, err := evaluateGenericDuplicate(sourcePath, targetPath, cfg.generic.dedupeMode)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", false, err
|
return "", false, err
|
||||||
@@ -206,40 +248,70 @@ func processGenericCandidate(cfg config, sourcePath string) (string, bool, error
|
|||||||
}
|
}
|
||||||
|
|
||||||
func evaluateGenericDuplicate(sourcePath, targetPath, dedupeMode string) (string, bool, error) {
|
func evaluateGenericDuplicate(sourcePath, targetPath, dedupeMode string) (string, bool, error) {
|
||||||
|
decision, duplicate, _, err := analyzeGenericDuplicate(sourcePath, targetPath, dedupeMode)
|
||||||
|
return decision, duplicate, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func analyzeGenericDuplicate(sourcePath, targetPath, dedupeMode string) (string, bool, bool, error) {
|
||||||
sourceInfo, err := os.Stat(sourcePath)
|
sourceInfo, err := os.Stat(sourcePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", false, fmt.Errorf("stat source: %w", err)
|
return "", false, false, fmt.Errorf("stat source: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
targetInfo, err := os.Stat(targetPath)
|
targetInfo, err := os.Stat(targetPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", false, fmt.Errorf("stat target: %w", err)
|
return "", false, false, fmt.Errorf("stat target: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
switch dedupeMode {
|
switch dedupeMode {
|
||||||
case genericDedupeNameHash:
|
case genericDedupeNameHash:
|
||||||
if sourceInfo.Size() != targetInfo.Size() {
|
if sourceInfo.Size() != targetInfo.Size() {
|
||||||
return "已跳过,同名文件已存在但大小不同", false, nil
|
return "已跳过,同名文件已存在但大小不同", false, false, nil
|
||||||
}
|
}
|
||||||
same, err := filesHaveSameHash(sourcePath, targetPath)
|
same, err := filesHaveSameHash(sourcePath, targetPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", false, err
|
return "", false, false, err
|
||||||
}
|
}
|
||||||
if same {
|
if same {
|
||||||
return "已跳过,同名文件已存在且哈希相同", true, nil
|
return "已跳过,同名文件已存在且哈希相同", true, true, nil
|
||||||
}
|
}
|
||||||
return "已跳过,同名文件已存在但哈希不同", false, nil
|
return "已跳过,同名文件已存在但哈希不同", false, true, nil
|
||||||
|
|
||||||
case genericDedupeNameSize:
|
case genericDedupeNameSize:
|
||||||
fallthrough
|
fallthrough
|
||||||
default:
|
default:
|
||||||
if sourceInfo.Size() == targetInfo.Size() {
|
if sourceInfo.Size() == targetInfo.Size() {
|
||||||
return "已跳过,同名文件已存在且大小相同", true, nil
|
return "已跳过,同名文件已存在且大小相同", true, false, nil
|
||||||
}
|
}
|
||||||
return "已跳过,同名文件已存在但大小不同", false, nil
|
return "已跳过,同名文件已存在但大小不同", false, false, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func handleGenericDuplicateMove(sourcePath, targetPath, dedupeMode string) (string, bool, error) {
|
||||||
|
decision, duplicate, hashVerified, err := analyzeGenericDuplicate(sourcePath, targetPath, dedupeMode)
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
if !duplicate {
|
||||||
|
return decision, false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if !hashVerified {
|
||||||
|
same, err := filesHaveSameHash(sourcePath, targetPath)
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
if !same {
|
||||||
|
return "已跳过,同名文件大小相同但哈希不同", false, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.Remove(sourcePath); err != nil {
|
||||||
|
return "", false, fmt.Errorf("remove duplicate generic source: %w", err)
|
||||||
|
}
|
||||||
|
return "已跳过传输,目标目录已有同名文件且哈希一致,源文件已删除", true, nil
|
||||||
|
}
|
||||||
|
|
||||||
func filesHaveSameHash(sourcePath, targetPath string) (bool, error) {
|
func filesHaveSameHash(sourcePath, targetPath string) (bool, error) {
|
||||||
sourceHash, err := fileSHA256(sourcePath)
|
sourceHash, err := fileSHA256(sourcePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -254,6 +326,192 @@ func filesHaveSameHash(sourcePath, targetPath string) (bool, error) {
|
|||||||
return sourceHash == targetHash, nil
|
return sourceHash == targetHash, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func archiveMatchesDirectory(sourcePath, targetPath, archiveRoot string) (bool, error) {
|
||||||
|
sourceRecords, err := archiveContentRecords(sourcePath, archiveRoot)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
targetRecords, err := directoryContentRecords(targetPath)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(sourceRecords) != len(targetRecords) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := range sourceRecords {
|
||||||
|
if sourceRecords[i] != targetRecords[i] {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func archiveContentRecords(archivePath, archiveRoot string) ([]contentRecord, error) {
|
||||||
|
file, err := os.Open(archivePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("open archive: %w", err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
gzipReader, err := gzip.NewReader(file)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("open gzip reader: %w", err)
|
||||||
|
}
|
||||||
|
defer gzipReader.Close()
|
||||||
|
|
||||||
|
records := make(map[string]contentRecord)
|
||||||
|
tarReader := tar.NewReader(gzipReader)
|
||||||
|
|
||||||
|
for {
|
||||||
|
header, err := tarReader.Next()
|
||||||
|
if err == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read archive entry: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
name := normalizeArchiveRecordPath(header.Name, archiveRoot)
|
||||||
|
switch header.Typeflag {
|
||||||
|
case tar.TypeReg, tar.TypeRegA, tar.TypeGNUSparse:
|
||||||
|
if name == "" {
|
||||||
|
return nil, fmt.Errorf("archive entry %q resolves to empty path", header.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
sum, err := readerSHA256(tarReader)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("hash archive entry %q: %w", header.Name, err)
|
||||||
|
}
|
||||||
|
records[name] = contentRecord{
|
||||||
|
path: name,
|
||||||
|
kind: 'F',
|
||||||
|
size: header.Size,
|
||||||
|
sum: sum,
|
||||||
|
}
|
||||||
|
|
||||||
|
case tar.TypeSymlink:
|
||||||
|
if name == "" {
|
||||||
|
return nil, fmt.Errorf("archive symlink %q resolves to empty path", header.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
records[name] = contentRecord{
|
||||||
|
path: name,
|
||||||
|
kind: 'L',
|
||||||
|
sum: sha256.Sum256([]byte(filepath.ToSlash(header.Linkname))),
|
||||||
|
}
|
||||||
|
|
||||||
|
case tar.TypeDir, tar.TypeXHeader, tar.TypeXGlobalHeader, tar.TypeGNULongName, tar.TypeGNULongLink:
|
||||||
|
continue
|
||||||
|
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("unsupported archive entry type for %q: %d", header.Name, header.Typeflag)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return sortedContentRecords(records), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func directoryContentRecords(root string) ([]contentRecord, error) {
|
||||||
|
records := make(map[string]contentRecord)
|
||||||
|
|
||||||
|
err := filepath.Walk(root, func(currentPath string, info os.FileInfo, walkErr error) error {
|
||||||
|
if walkErr != nil {
|
||||||
|
return walkErr
|
||||||
|
}
|
||||||
|
if currentPath == root {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
relativePath, err := filepath.Rel(root, currentPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("build relative path for %q: %w", currentPath, err)
|
||||||
|
}
|
||||||
|
relativePath = filepath.ToSlash(relativePath)
|
||||||
|
|
||||||
|
if info.Mode()&os.ModeSymlink != 0 {
|
||||||
|
linkTarget, err := os.Readlink(currentPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("read symlink %q: %w", currentPath, err)
|
||||||
|
}
|
||||||
|
records[relativePath] = contentRecord{
|
||||||
|
path: relativePath,
|
||||||
|
kind: 'L',
|
||||||
|
sum: sha256.Sum256([]byte(filepath.ToSlash(linkTarget))),
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if info.IsDir() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !info.Mode().IsRegular() {
|
||||||
|
return fmt.Errorf("unsupported directory entry %q", currentPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
sum, err := fileSHA256(currentPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("hash directory file %q: %w", currentPath, err)
|
||||||
|
}
|
||||||
|
records[relativePath] = contentRecord{
|
||||||
|
path: relativePath,
|
||||||
|
kind: 'F',
|
||||||
|
size: info.Size(),
|
||||||
|
sum: sum,
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return sortedContentRecords(records), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeArchiveRecordPath(name, archiveRoot string) string {
|
||||||
|
cleanPath := strings.TrimSpace(filepath.ToSlash(name))
|
||||||
|
cleanPath = strings.TrimPrefix(cleanPath, "./")
|
||||||
|
cleanPath = strings.TrimPrefix(cleanPath, "/")
|
||||||
|
cleanPath = path.Clean(cleanPath)
|
||||||
|
|
||||||
|
if cleanPath == "." {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
normalizedRoot := path.Clean(strings.Trim(filepath.ToSlash(archiveRoot), "/"))
|
||||||
|
if normalizedRoot != "." && normalizedRoot != "" {
|
||||||
|
if cleanPath == normalizedRoot {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
prefix := normalizedRoot + "/"
|
||||||
|
if strings.HasPrefix(cleanPath, prefix) {
|
||||||
|
cleanPath = strings.TrimPrefix(cleanPath, prefix)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if cleanPath == "." {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return cleanPath
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortedContentRecords(records map[string]contentRecord) []contentRecord {
|
||||||
|
items := make([]contentRecord, 0, len(records))
|
||||||
|
for _, record := range records {
|
||||||
|
items = append(items, record)
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Slice(items, func(i, j int) bool {
|
||||||
|
if items[i].path == items[j].path {
|
||||||
|
return items[i].kind < items[j].kind
|
||||||
|
}
|
||||||
|
return items[i].path < items[j].path
|
||||||
|
})
|
||||||
|
return items
|
||||||
|
}
|
||||||
|
|
||||||
func fileSHA256(path string) ([32]byte, error) {
|
func fileSHA256(path string) ([32]byte, error) {
|
||||||
file, err := os.Open(path)
|
file, err := os.Open(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -261,8 +519,12 @@ func fileSHA256(path string) ([32]byte, error) {
|
|||||||
}
|
}
|
||||||
defer file.Close()
|
defer file.Close()
|
||||||
|
|
||||||
|
return readerSHA256(file)
|
||||||
|
}
|
||||||
|
|
||||||
|
func readerSHA256(reader io.Reader) ([32]byte, error) {
|
||||||
hasher := sha256.New()
|
hasher := sha256.New()
|
||||||
if _, err := io.Copy(hasher, file); err != nil {
|
if _, err := io.Copy(hasher, reader); err != nil {
|
||||||
return [32]byte{}, err
|
return [32]byte{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -288,22 +550,27 @@ func transferFile(sourcePath, targetPath, mode string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("open source: %w", err)
|
return fmt.Errorf("open source: %w", err)
|
||||||
}
|
}
|
||||||
defer source.Close()
|
|
||||||
|
|
||||||
target, err := os.OpenFile(tempPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644)
|
target, err := os.OpenFile(tempPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
_ = source.Close()
|
||||||
return fmt.Errorf("create temp target: %w", err)
|
return fmt.Errorf("create temp target: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
copyErr := copyFileContents(target, source)
|
if err := copyFileContents(target, source); err != nil {
|
||||||
closeErr := target.Close()
|
_ = target.Close()
|
||||||
if copyErr != nil {
|
_ = source.Close()
|
||||||
_ = os.Remove(tempPath)
|
_ = os.Remove(tempPath)
|
||||||
return copyErr
|
return err
|
||||||
}
|
}
|
||||||
if closeErr != nil {
|
if err := target.Close(); err != nil {
|
||||||
|
_ = source.Close()
|
||||||
_ = os.Remove(tempPath)
|
_ = os.Remove(tempPath)
|
||||||
return fmt.Errorf("close temp target: %w", closeErr)
|
return fmt.Errorf("close temp target: %w", err)
|
||||||
|
}
|
||||||
|
if err := source.Close(); err != nil {
|
||||||
|
_ = os.Remove(tempPath)
|
||||||
|
return fmt.Errorf("close source: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := os.Rename(tempPath, targetPath); err != nil {
|
if err := os.Rename(tempPath, targetPath); err != nil {
|
||||||
|
|||||||
Reference in New Issue
Block a user