Checkpoint production workflow updates

This commit is contained in:
2026-06-30 15:25:29 +08:00
parent 19ae3ec37f
commit 9c80b95385
66 changed files with 7639 additions and 267 deletions
+38
View File
@@ -0,0 +1,38 @@
# -*- mode: python ; coding: utf-8 -*-
a = Analysis(
['lt1_data_sync_gui.py'],
pathex=[],
binaries=[('C:\\ProgramData\\anaconda3\\envs\\InSAR\\Library\\bin\\tcl86t.dll', '.'), ('C:\\ProgramData\\anaconda3\\envs\\InSAR\\Library\\bin\\tk86t.dll', '.'), ('C:\\ProgramData\\anaconda3\\envs\\InSAR\\Library\\bin\\libcrypto-3-x64.dll', '.'), ('C:\\ProgramData\\anaconda3\\envs\\InSAR\\Library\\bin\\liblzma.dll', '.'), ('C:\\ProgramData\\anaconda3\\envs\\InSAR\\Library\\bin\\libbz2.dll', '.')],
datas=[],
hiddenimports=[],
hookspath=[],
hooksconfig={},
runtime_hooks=[],
excludes=[],
noarchive=False,
optimize=0,
)
pyz = PYZ(a.pure)
exe = EXE(
pyz,
a.scripts,
a.binaries,
a.datas,
[],
name='LT1DataSync',
debug=False,
bootloader_ignore_signals=False,
strip=False,
upx=True,
upx_exclude=[],
runtime_tmpdir=None,
console=False,
disable_windowed_traceback=False,
argv_emulation=False,
target_arch=None,
codesign_identity=None,
entitlements_file=None,
)
+189
View File
@@ -0,0 +1,189 @@
# LT1AssetTool
独立 EXE 工具,只做三件事:
1. 扫描服务器资产路径,列出服务器现在有哪些资产,生成一个随身 JSON。
2. 扫描一个或多个 UNC 路径,读取随身 JSON,服务器已有资产不参与复制,剩下的复制到一个或多个指定路径。
3. 将一个或多个磁盘路径下的资产复制或剪切到服务器资产路径,并提示重新执行第 1 步。
## 资产识别
当前只识别平铺文件:
- `LT1*.tar.gz`
- `LT1*.tgz`
- `LT1*.tar`
- `LT1*.zip`
- `LT1*.txt`
判断是否已有资产使用:
```text
资产类型 + 文件名 + 文件大小
```
## 第 1 步:扫描服务器资产路径
输入:
```text
服务器资产路径:
D:\LuTan1_Image_Pool_Zip
D:\LT1_data_lsarorbit
服务器资产 JSON 保存为:
E:\server_assets.json
```
输出 JSON 示例:
```json
{
"schema": "lt1_asset_inventory.v2",
"generated_at": "2026-06-22 16:00:00",
"roots": [
"D:\\LuTan1_Image_Pool_Zip",
"D:\\LT1_data_lsarorbit"
],
"asset_count": 2,
"assets": [
{
"kind": "lt1_archive",
"name": "LT1A_xxx.tar.gz",
"path": "D:\\LuTan1_Image_Pool_Zip\\LT1A_xxx.tar.gz",
"size": 123456789,
"mtime": 1782100000.0
},
{
"kind": "lt1_orbit",
"name": "LT1A_GpsData_GAS_C_20240101.txt",
"path": "D:\\LT1_data_lsarorbit\\LT1A_GpsData_GAS_C_20240101.txt",
"size": 345678,
"mtime": 1782100100.0
}
]
}
```
以后就带着这个 JSON 去内网机器。
## 第 2 步:从 UNC 补拷缺失资产
输入:
```text
读取服务器资产 JSON
E:\server_assets.json
UNC 源路径:
\\server01\lt1_archives
\\server02\lt1_orbits
复制目标路径:
E:\LT1_TRANSFER
F:\LT1_TRANSFER
```
规则:
- JSON 中已有且大小一致:跳过
- JSON 中有同名资产但大小不同:报告冲突,不复制
- JSON 中没有:复制到指定目标路径
- 多个目标路径时,选择第一个空间足够的路径
- 如果目标路径已有同名文件且大小一致:跳过
- 复制过程先写 `.part`,完成并校验大小后再改名
## 第 3 步:磁盘导入服务器
输入:
```text
磁盘资产路径:
E:\LT1_TRANSFER
F:\LT1_TRANSFER
服务器资产路径:
D:\LuTan1_Image_Pool_Zip
D:\LT1_data_lsarorbit
```
可选:
```text
剪切到服务器
```
规则:
- 服务器已有且大小一致:跳过
- 服务器有同名资产但大小不同:报告冲突,不覆盖
- 服务器没有:复制或剪切到服务器路径
- 多个服务器路径时,选择第一个空间足够的路径
执行第 3 步之后,回到服务器重新执行第 1 步,生成新的随身 JSON。
## 报告和日志
需要指定 `报告/日志目录`,例如:
```text
E:\LT1AssetToolReports
```
工具会生成:
```text
E:\LT1AssetToolReports\
logs\
run_YYYYMMDD_HHMMSS.log
reports\
unc_copy_report_YYYYMMDD_HHMMSS.csv
disk_import_report_YYYYMMDD_HHMMSS.csv
```
CSV 可以用 Excel 打开。
## 命令行
扫描服务器:
```powershell
python lt1_data_sync_cli.py scan-server `
--report-dir "E:\LT1AssetToolReports" `
--server-roots "D:\LuTan1_Image_Pool_Zip;D:\LT1_data_lsarorbit" `
--output-json "E:\server_assets.json"
```
从 UNC 复制缺失资产:
```powershell
python lt1_data_sync_cli.py copy-unc `
--report-dir "E:\LT1AssetToolReports" `
--server-json "E:\server_assets.json" `
--unc-roots "\\server01\lt1_archives;\\server02\lt1_orbits" `
--targets "E:\LT1_TRANSFER;F:\LT1_TRANSFER" `
--execute
```
磁盘导入服务器:
```powershell
python lt1_data_sync_cli.py import-disk `
--report-dir "E:\LT1AssetToolReports" `
--disk-roots "E:\LT1_TRANSFER;F:\LT1_TRANSFER" `
--server-roots "D:\LuTan1_Image_Pool_Zip;D:\LT1_data_lsarorbit" `
--execute
```
## 打包 EXE
```powershell
cd D:\Code\Insar_management_system_v2\tools\lt1_data_sync
.\build_exe.ps1 -Python "C:\ProgramData\anaconda3\envs\InSAR\python.exe"
```
输出:
```text
dist\LT1DataSync.exe
```
+37
View File
@@ -0,0 +1,37 @@
param(
[string]$Python = "python"
)
$ErrorActionPreference = "Stop"
$ToolDir = Split-Path -Parent $MyInvocation.MyCommand.Path
Set-Location $ToolDir
try {
& $Python --version
} catch {
throw "Python was not found. Install Python 3.10+ or pass -Python with a full python.exe path."
}
$PythonExe = (Get-Command $Python).Source
$EnvRoot = Split-Path -Parent $PythonExe
$CondaBin = Join-Path $EnvRoot "Library\bin"
$ExtraArgs = @()
foreach ($DllName in @("tcl86t.dll", "tk86t.dll", "libcrypto-3-x64.dll", "liblzma.dll", "libbz2.dll")) {
$DllPath = Join-Path $CondaBin $DllName
if (Test-Path $DllPath) {
$ExtraArgs += "--add-binary"
$ExtraArgs += "$DllPath;."
}
}
& $Python -m pip install --upgrade pyinstaller
& $Python -m PyInstaller `
--noconfirm `
--onefile `
--windowed `
--name LT1DataSync `
@ExtraArgs `
lt1_data_sync_gui.py
Write-Host ""
Write-Host "EXE built at: $ToolDir\dist\LT1DataSync.exe"
+88
View File
@@ -0,0 +1,88 @@
from __future__ import annotations
import argparse
from sync_core import (
FileLogger,
copy_unc_missing_assets,
import_disk_assets_to_server,
normalize_path,
parse_path_list,
scan_assets,
stamp_text,
write_inventory,
)
def add_common(parser: argparse.ArgumentParser) -> None:
parser.add_argument("--report-dir", required=True)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="LT1 asset transfer tool")
sub = parser.add_subparsers(dest="command", required=True)
scan = sub.add_parser("scan-server", help="Scan server asset paths and write a JSON inventory")
add_common(scan)
scan.add_argument("--server-roots", required=True, help="Server asset paths separated by semicolon/newline")
scan.add_argument("--output-json", required=True)
copy = sub.add_parser("copy-unc", help="Copy UNC assets that are missing from server inventory")
add_common(copy)
copy.add_argument("--server-json", required=True)
copy.add_argument("--unc-roots", required=True, help="UNC source paths separated by semicolon/newline")
copy.add_argument("--targets", required=True, help="Copy target paths separated by semicolon/newline")
copy.add_argument("--execute", action="store_true")
imp = sub.add_parser("import-disk", help="Copy or move disk assets into server asset paths")
add_common(imp)
imp.add_argument("--disk-roots", required=True, help="Disk asset paths separated by semicolon/newline")
imp.add_argument("--server-roots", required=True, help="Server asset paths separated by semicolon/newline")
imp.add_argument("--execute", action="store_true")
imp.add_argument("--move", action="store_true")
return parser.parse_args()
def main() -> int:
args = parse_args()
report_dir = normalize_path(args.report_dir)
logger = FileLogger(report_dir / "logs" / f"cli_{stamp_text()}.log", print)
if args.command == "scan-server":
roots = parse_path_list(args.server_roots)
assets = scan_assets(roots, log=logger)
write_inventory(normalize_path(args.output_json), assets, [str(root) for root in roots])
logger(f"服务器资产 JSON 已生成:{args.output_json},资产 {len(assets)}")
return 0
if args.command == "copy-unc":
report = copy_unc_missing_assets(
parse_path_list(args.unc_roots),
normalize_path(args.server_json),
parse_path_list(args.targets),
report_dir / "reports",
execute=args.execute,
log=logger,
)
failed = sum(1 for item in report if item.action == "failed")
return 1 if failed else 0
if args.command == "import-disk":
report = import_disk_assets_to_server(
parse_path_list(args.disk_roots),
parse_path_list(args.server_roots),
report_dir / "reports",
execute=args.execute,
move=args.move,
log=logger,
)
failed = sum(1 for item in report if item.action == "failed")
logger("导入后请重新执行 scan-server,生成新的随身 JSON。")
return 1 if failed else 0
raise ValueError(f"Unsupported command: {args.command}")
if __name__ == "__main__":
raise SystemExit(main())
+327
View File
@@ -0,0 +1,327 @@
from __future__ import annotations
import json
import queue
import threading
from pathlib import Path
from tkinter import BooleanVar, StringVar, Tk, filedialog, messagebox, ttk
from tkinter.scrolledtext import ScrolledText
from sync_core import (
FileLogger,
copy_unc_missing_assets,
import_disk_assets_to_server,
normalize_path,
parse_path_list,
scan_assets,
stamp_text,
write_inventory,
)
APP_DIR = Path(__file__).resolve().parent
CONFIG_PATH = APP_DIR / "config.json"
DEFAULT_CONFIG = {
"server_asset_roots": r"D:\LuTan1_Image_Pool_Zip" + "\n" + r"D:\LT1_data_lsarorbit",
"inventory_output": "",
"server_inventory_json": "",
"unc_roots": "",
"unc_copy_targets": "",
"disk_roots": "",
"import_server_roots": r"D:\LuTan1_Image_Pool_Zip" + "\n" + r"D:\LT1_data_lsarorbit",
"report_dir": "",
"move_on_import": False,
}
class LT1AssetTool:
def __init__(self, root: Tk) -> None:
self.root = root
self.root.title("LT1 资产搬运工具")
self.root.geometry("1120x820")
self.log_queue: queue.Queue[str] = queue.Queue()
self.worker: threading.Thread | None = None
self.text_widgets: dict[str, ScrolledText] = {}
config = self.load_config()
self.vars = {
"server_asset_roots": StringVar(value=config["server_asset_roots"]),
"inventory_output": StringVar(value=config["inventory_output"]),
"server_inventory_json": StringVar(value=config["server_inventory_json"]),
"unc_roots": StringVar(value=config["unc_roots"]),
"unc_copy_targets": StringVar(value=config["unc_copy_targets"]),
"disk_roots": StringVar(value=config["disk_roots"]),
"import_server_roots": StringVar(value=config["import_server_roots"]),
"report_dir": StringVar(value=config["report_dir"]),
}
self.move_on_import = BooleanVar(value=bool(config["move_on_import"]))
self.summary_text = StringVar(value="准备就绪")
self.build_ui()
self.root.after(100, self.drain_logs)
def load_config(self) -> dict:
if not CONFIG_PATH.exists():
return dict(DEFAULT_CONFIG)
try:
payload = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
return {**DEFAULT_CONFIG, **payload}
except Exception:
return dict(DEFAULT_CONFIG)
def sync_text_vars(self) -> None:
for key, widget in self.text_widgets.items():
self.vars[key].set(widget.get("1.0", "end").strip())
def save_config(self) -> None:
self.sync_text_vars()
payload = {key: var.get() for key, var in self.vars.items()}
payload["move_on_import"] = self.move_on_import.get()
CONFIG_PATH.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
def build_ui(self) -> None:
outer = ttk.Frame(self.root, padding=12)
outer.pack(fill="both", expand=True)
common = ttk.LabelFrame(outer, text="公共输出", padding=10)
common.pack(fill="x")
self.add_path_row(common, 0, "报告/日志目录", "report_dir")
scan_box = ttk.LabelFrame(outer, text="1. 扫描服务器资产路径,生成随身 JSON", padding=10)
scan_box.pack(fill="x", pady=(10, 0))
self.add_multi_path_row(scan_box, 0, "服务器资产路径(多个用分号或换行)", "server_asset_roots")
self.add_file_save_row(scan_box, 1, "服务器资产 JSON 保存为", "inventory_output")
ttk.Button(scan_box, text="执行 1:扫描服务器并生成 JSON", command=lambda: self.start_worker(self.run_scan_server)).grid(
row=2,
column=1,
sticky="w",
pady=(8, 0),
)
unc_box = ttk.LabelFrame(outer, text="2. 扫描 UNC 多路径,按服务器 JSON 跳过已有,剩余复制到指定路径", padding=10)
unc_box.pack(fill="x", pady=(10, 0))
self.add_file_row(unc_box, 0, "读取服务器资产 JSON", "server_inventory_json")
self.add_multi_path_row(unc_box, 1, "UNC 源路径(多个用分号或换行)", "unc_roots")
self.add_multi_path_row(unc_box, 2, "复制目标路径(多个用分号或换行)", "unc_copy_targets")
ttk.Button(unc_box, text="预览 2:只生成复制报告", command=lambda: self.start_worker(lambda logger: self.run_unc_copy(logger, execute=False))).grid(
row=3,
column=1,
sticky="w",
pady=(8, 0),
)
ttk.Button(unc_box, text="执行 2:复制缺失资产", command=lambda: self.start_worker(lambda logger: self.run_unc_copy(logger, execute=True))).grid(
row=3,
column=1,
sticky="w",
padx=(180, 0),
pady=(8, 0),
)
import_box = ttk.LabelFrame(outer, text="3. 将指定磁盘路径资产复制或剪切到服务器资产路径", padding=10)
import_box.pack(fill="x", pady=(10, 0))
self.add_multi_path_row(import_box, 0, "磁盘资产路径(多个用分号或换行)", "disk_roots")
self.add_multi_path_row(import_box, 1, "服务器资产路径(多个用分号或换行)", "import_server_roots")
ttk.Checkbutton(import_box, text="剪切到服务器(不勾选则复制)", variable=self.move_on_import).grid(row=2, column=1, sticky="w")
ttk.Button(import_box, text="预览 3:只生成导入报告", command=lambda: self.start_worker(lambda logger: self.run_import(logger, execute=False))).grid(
row=3,
column=1,
sticky="w",
pady=(8, 0),
)
ttk.Button(import_box, text="执行 3:导入到服务器", command=lambda: self.start_worker(lambda logger: self.run_import(logger, execute=True))).grid(
row=3,
column=1,
sticky="w",
padx=(180, 0),
pady=(8, 0),
)
ttk.Label(import_box, text="执行 3 后,请回到服务器重新执行 1,生成新的随身 JSON。", foreground="#92400e").grid(
row=4,
column=1,
sticky="w",
pady=(8, 0),
)
control = ttk.Frame(outer)
control.pack(fill="x", pady=10)
ttk.Button(control, text="保存配置", command=self.handle_save_config).pack(side="left")
ttk.Label(control, textvariable=self.summary_text, font=("Microsoft YaHei UI", 10, "bold")).pack(side="left", padx=16)
log_box = ttk.LabelFrame(outer, text="日志", padding=10)
log_box.pack(fill="both", expand=True)
self.log_view = ScrolledText(log_box, height=14, wrap="word")
self.log_view.pack(fill="both", expand=True)
def add_path_row(self, frame: ttk.Frame, row: int, label: str, key: str) -> None:
ttk.Label(frame, text=label).grid(row=row, column=0, sticky="w", padx=(0, 8), pady=4)
ttk.Entry(frame, textvariable=self.vars[key], width=105).grid(row=row, column=1, sticky="ew", pady=4)
ttk.Button(frame, text="选择", command=lambda: self.choose_dir(key)).grid(row=row, column=2, padx=(8, 0), pady=4)
frame.columnconfigure(1, weight=1)
def add_file_row(self, frame: ttk.Frame, row: int, label: str, key: str) -> None:
ttk.Label(frame, text=label).grid(row=row, column=0, sticky="w", padx=(0, 8), pady=4)
ttk.Entry(frame, textvariable=self.vars[key], width=105).grid(row=row, column=1, sticky="ew", pady=4)
ttk.Button(frame, text="选择", command=lambda: self.choose_file(key)).grid(row=row, column=2, padx=(8, 0), pady=4)
frame.columnconfigure(1, weight=1)
def add_file_save_row(self, frame: ttk.Frame, row: int, label: str, key: str) -> None:
ttk.Label(frame, text=label).grid(row=row, column=0, sticky="w", padx=(0, 8), pady=4)
ttk.Entry(frame, textvariable=self.vars[key], width=105).grid(row=row, column=1, sticky="ew", pady=4)
ttk.Button(frame, text="选择", command=lambda: self.choose_save_file(key)).grid(row=row, column=2, padx=(8, 0), pady=4)
frame.columnconfigure(1, weight=1)
def add_multi_path_row(self, frame: ttk.Frame, row: int, label: str, key: str) -> None:
ttk.Label(frame, text=label).grid(row=row, column=0, sticky="nw", padx=(0, 8), pady=4)
text = ScrolledText(frame, width=105, height=3, wrap="none")
text.insert("1.0", self.vars[key].get())
text.grid(row=row, column=1, sticky="ew", pady=4)
self.text_widgets[key] = text
ttk.Button(frame, text="追加目录", command=lambda: self.append_dir(key)).grid(row=row, column=2, padx=(8, 0), pady=4, sticky="n")
frame.columnconfigure(1, weight=1)
def choose_dir(self, key: str) -> None:
selected = filedialog.askdirectory()
if selected:
self.vars[key].set(selected)
def append_dir(self, key: str) -> None:
selected = filedialog.askdirectory()
if not selected:
return
widget = self.text_widgets.get(key)
if not widget:
self.vars[key].set(selected)
return
current = widget.get("1.0", "end").strip()
widget.delete("1.0", "end")
widget.insert("1.0", f"{current}\n{selected}" if current else selected)
def choose_file(self, key: str) -> None:
selected = filedialog.askopenfilename(filetypes=[("JSON 文件", "*.json"), ("所有文件", "*.*")])
if selected:
self.vars[key].set(selected)
def choose_save_file(self, key: str) -> None:
selected = filedialog.asksaveasfilename(
defaultextension=".json",
filetypes=[("JSON 文件", "*.json"), ("所有文件", "*.*")],
initialfile=f"server_assets_{stamp_text()}.json",
)
if selected:
self.vars[key].set(selected)
def report_dir(self) -> Path:
text = self.vars["report_dir"].get().strip()
if not text:
raise ValueError("请指定报告/日志目录")
return normalize_path(text)
def logger(self) -> FileLogger:
return FileLogger(self.report_dir() / "logs" / f"run_{stamp_text()}.log", self.log)
def handle_save_config(self) -> None:
self.save_config()
messagebox.showinfo("已保存", f"配置已保存到:{CONFIG_PATH}")
def log(self, message: str) -> None:
self.log_queue.put(message)
def drain_logs(self) -> None:
while True:
try:
message = self.log_queue.get_nowait()
except queue.Empty:
break
self.log_view.insert("end", message + "\n")
self.log_view.see("end")
self.root.after(100, self.drain_logs)
def start_worker(self, action) -> None:
if self.worker and self.worker.is_alive():
messagebox.showwarning("正在运行", "已有任务正在运行。")
return
self.save_config()
self.log_view.delete("1.0", "end")
self.worker = threading.Thread(target=self.run_action, args=(action,), daemon=True)
self.worker.start()
def run_action(self, action) -> None:
try:
action(self.logger())
except Exception as exc:
self.log(f"任务失败:{exc}")
self.root.after(0, lambda: messagebox.showerror("任务失败", str(exc)))
def run_scan_server(self, logger: FileLogger) -> None:
roots = parse_path_list(self.vars["server_asset_roots"].get())
if not roots:
raise ValueError("请填写服务器资产路径")
output = self.vars["inventory_output"].get().strip()
if not output:
raise ValueError("请指定服务器资产 JSON 保存路径")
logger("开始扫描服务器资产路径")
assets = scan_assets(roots, log=logger)
write_inventory(normalize_path(output), assets, [str(root) for root in roots])
self.root.after(0, lambda: self.summary_text.set(f"服务器资产 {len(assets)} 个,JSON 已生成"))
logger(f"完成:{output}")
def run_unc_copy(self, logger: FileLogger, *, execute: bool) -> None:
inventory = self.vars["server_inventory_json"].get().strip()
unc_roots = parse_path_list(self.vars["unc_roots"].get())
targets = parse_path_list(self.vars["unc_copy_targets"].get())
if not inventory:
raise ValueError("请指定服务器资产 JSON")
if not unc_roots:
raise ValueError("请填写 UNC 源路径")
if not targets:
raise ValueError("请填写复制目标路径")
logger("开始扫描 UNC 并按服务器 JSON 跳过已有资产")
report = copy_unc_missing_assets(
unc_roots,
normalize_path(inventory),
targets,
self.report_dir() / "reports",
execute=execute,
log=logger,
)
copied = sum(1 for item in report if item.action == "copied")
planned = sum(1 for item in report if item.action == "planned")
skipped = sum(1 for item in report if item.action == "skip")
failed = sum(1 for item in report if item.action == "failed")
text = f"UNC 处理完成:copied={copied}, planned={planned}, skipped={skipped}, failed={failed}"
self.root.after(0, lambda: self.summary_text.set(text))
logger(text)
def run_import(self, logger: FileLogger, *, execute: bool) -> None:
disk_roots = parse_path_list(self.vars["disk_roots"].get())
server_roots = parse_path_list(self.vars["import_server_roots"].get())
if not disk_roots:
raise ValueError("请填写磁盘资产路径")
if not server_roots:
raise ValueError("请填写服务器资产路径")
logger("开始从磁盘导入资产到服务器")
report = import_disk_assets_to_server(
disk_roots,
server_roots,
self.report_dir() / "reports",
execute=execute,
move=self.move_on_import.get(),
log=logger,
)
imported = sum(1 for item in report if item.action in {"copied", "moved"})
planned = sum(1 for item in report if item.action == "planned")
skipped = sum(1 for item in report if item.action == "skip")
failed = sum(1 for item in report if item.action == "failed")
text = f"磁盘导入完成:imported={imported}, planned={planned}, skipped={skipped}, failed={failed}。请重新执行 1。"
self.root.after(0, lambda: self.summary_text.set(text))
logger(text)
def main() -> None:
root = Tk()
LT1AssetTool(root)
root.mainloop()
if __name__ == "__main__":
main()
+316
View File
@@ -0,0 +1,316 @@
from __future__ import annotations
import csv
import json
import os
import shutil
from dataclasses import asdict, dataclass
from datetime import datetime
from pathlib import Path
from typing import Callable, Iterable
LT1_ARCHIVE_SUFFIXES = (".tar.gz", ".tgz", ".tar", ".zip")
LT1_ORBIT_SUFFIXES = (".txt",)
ProgressCallback = Callable[[str], None]
def now_text() -> str:
return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
def stamp_text() -> str:
return datetime.now().strftime("%Y%m%d_%H%M%S")
def normalize_path(value: str) -> Path:
return Path(value.strip().strip('"')).expanduser()
def parse_path_list(value: str) -> list[Path]:
paths: list[Path] = []
for line in str(value or "").replace(";", "\n").splitlines():
text = line.strip().strip('"')
if text:
paths.append(normalize_path(text))
return paths
def safe_mkdir(path: Path) -> None:
path.mkdir(parents=True, exist_ok=True)
def is_asset_file(path: Path) -> bool:
lower = path.name.lower()
if lower.startswith("lt1") and lower.endswith(LT1_ARCHIVE_SUFFIXES):
return True
if lower.startswith("lt1") and lower.endswith(LT1_ORBIT_SUFFIXES):
return True
return False
def classify_asset(path: Path) -> str:
lower = path.name.lower()
if lower.endswith(LT1_ARCHIVE_SUFFIXES):
return "lt1_archive"
if lower.endswith(LT1_ORBIT_SUFFIXES):
return "lt1_orbit"
return "unknown"
@dataclass(frozen=True)
class AssetRecord:
kind: str
name: str
path: str
size: int
mtime: float
@dataclass(frozen=True)
class CopyRecord:
name: str
kind: str
source_path: str
target_path: str
size: int
action: str
reason: str
class FileLogger:
def __init__(self, path: Path, ui_log: ProgressCallback | None = None) -> None:
self.path = path
self.ui_log = ui_log
safe_mkdir(path.parent)
def __call__(self, message: str) -> None:
line = f"{now_text()} {message}"
with self.path.open("a", encoding="utf-8") as handle:
handle.write(line + "\n")
if self.ui_log:
self.ui_log(line)
def scan_assets(paths: Iterable[Path], *, log: ProgressCallback | None = None) -> list[AssetRecord]:
records: list[AssetRecord] = []
for root in paths:
if not root.exists():
if log:
log(f"路径不存在,跳过:{root}")
continue
if not root.is_dir():
if log:
log(f"不是目录,跳过:{root}")
continue
count = 0
for entry in root.iterdir():
if not entry.is_file() or not is_asset_file(entry):
continue
stat = entry.stat()
records.append(
AssetRecord(
kind=classify_asset(entry),
name=entry.name,
path=str(entry),
size=stat.st_size,
mtime=stat.st_mtime,
)
)
count += 1
if log:
log(f"扫描完成:{root},资产 {count}")
records.sort(key=lambda item: (item.kind, item.name.lower(), item.path.lower()))
return records
def asset_key(record: AssetRecord) -> tuple[str, str]:
return record.kind, record.name.lower()
def build_asset_index(records: Iterable[AssetRecord]) -> dict[tuple[str, str], AssetRecord]:
index: dict[tuple[str, str], AssetRecord] = {}
for record in records:
key = asset_key(record)
if key not in index:
index[key] = record
return index
def read_inventory(path: Path) -> list[AssetRecord]:
payload = json.loads(path.read_text(encoding="utf-8"))
records: list[AssetRecord] = []
for item in payload.get("assets", payload.get("files", [])):
try:
records.append(
AssetRecord(
kind=str(item.get("kind") or ""),
name=str(item.get("name") or ""),
path=str(item.get("path") or ""),
size=int(item.get("size") or 0),
mtime=float(item.get("mtime") or 0),
)
)
except (TypeError, ValueError):
continue
return [item for item in records if item.kind and item.name]
def write_inventory(path: Path, records: list[AssetRecord], roots: list[str]) -> None:
safe_mkdir(path.parent)
payload = {
"schema": "lt1_asset_inventory.v2",
"generated_at": now_text(),
"roots": roots,
"asset_count": len(records),
"assets": [asdict(item) for item in records],
}
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
def write_csv(path: Path, records: list[CopyRecord]) -> None:
safe_mkdir(path.parent)
with path.open("w", newline="", encoding="utf-8-sig") as handle:
writer = csv.DictWriter(
handle,
fieldnames=["name", "kind", "source_path", "target_path", "size", "action", "reason"],
)
writer.writeheader()
for record in records:
writer.writerow(asdict(record))
def choose_target_root(target_roots: list[Path], filename: str, size: int) -> tuple[Path | None, str]:
for root in target_roots:
dest = root / filename
if dest.exists():
try:
if dest.stat().st_size == size:
return None, f"目标已存在且大小一致:{dest}"
return None, f"目标已存在但大小不同:{dest}"
except OSError:
return None, f"无法读取目标文件状态:{dest}"
for root in target_roots:
try:
safe_mkdir(root)
free_bytes = shutil.disk_usage(root).free
if free_bytes > size:
return root, "选择第一个空间足够的目标路径"
except OSError:
continue
return None, "没有空间足够的目标路径"
def copy_file_atomic(source: Path, target: Path, *, move: bool = False) -> None:
safe_mkdir(target.parent)
part = target.with_name(target.name + ".part")
if part.exists():
part.unlink()
if target.exists():
raise FileExistsError(str(target))
if move:
shutil.copy2(source, part)
if part.stat().st_size != source.stat().st_size:
raise OSError(f"复制后大小不一致:{source} -> {target}")
os.replace(part, target)
source.unlink()
else:
shutil.copy2(source, part)
if part.stat().st_size != source.stat().st_size:
raise OSError(f"复制后大小不一致:{source} -> {target}")
os.replace(part, target)
def copy_unc_missing_assets(
unc_roots: list[Path],
server_inventory_json: Path,
target_roots: list[Path],
report_dir: Path,
*,
execute: bool,
log: ProgressCallback | None = None,
) -> list[CopyRecord]:
server_assets = read_inventory(server_inventory_json)
server_index = build_asset_index(server_assets)
source_assets = scan_assets(unc_roots, log=log)
report: list[CopyRecord] = []
for asset in source_assets:
server_asset = server_index.get(asset_key(asset))
if server_asset and server_asset.size == asset.size:
report.append(
CopyRecord(asset.name, asset.kind, asset.path, server_asset.path, asset.size, "skip", "服务器清单已有且大小一致")
)
continue
if server_asset and server_asset.size != asset.size:
report.append(
CopyRecord(asset.name, asset.kind, asset.path, server_asset.path, asset.size, "conflict", "服务器清单有同名资产但大小不同")
)
continue
target_root, reason = choose_target_root(target_roots, asset.name, asset.size)
if target_root is None:
report.append(CopyRecord(asset.name, asset.kind, asset.path, "", asset.size, "skip", reason))
continue
target_path = target_root / asset.name
if not execute:
report.append(CopyRecord(asset.name, asset.kind, asset.path, str(target_path), asset.size, "planned", reason))
continue
try:
copy_file_atomic(Path(asset.path), target_path)
report.append(CopyRecord(asset.name, asset.kind, asset.path, str(target_path), asset.size, "copied", reason))
if log:
log(f"已复制:{asset.name} -> {target_path}")
except Exception as exc:
report.append(CopyRecord(asset.name, asset.kind, asset.path, str(target_path), asset.size, "failed", str(exc)))
if log:
log(f"复制失败:{asset.name}{exc}")
write_csv(report_dir / f"unc_copy_report_{stamp_text()}.csv", report)
return report
def import_disk_assets_to_server(
disk_roots: list[Path],
server_asset_roots: list[Path],
report_dir: Path,
*,
execute: bool,
move: bool,
log: ProgressCallback | None = None,
) -> list[CopyRecord]:
source_assets = scan_assets(disk_roots, log=log)
server_assets = scan_assets(server_asset_roots, log=log)
server_index = build_asset_index(server_assets)
report: list[CopyRecord] = []
for asset in source_assets:
existing = server_index.get(asset_key(asset))
if existing and existing.size == asset.size:
report.append(CopyRecord(asset.name, asset.kind, asset.path, existing.path, asset.size, "skip", "服务器已存在且大小一致"))
continue
if existing and existing.size != asset.size:
report.append(CopyRecord(asset.name, asset.kind, asset.path, existing.path, asset.size, "conflict", "服务器有同名资产但大小不同"))
continue
target_root, reason = choose_target_root(server_asset_roots, asset.name, asset.size)
if target_root is None:
report.append(CopyRecord(asset.name, asset.kind, asset.path, "", asset.size, "skip", reason))
continue
target_path = target_root / asset.name
if not execute:
report.append(CopyRecord(asset.name, asset.kind, asset.path, str(target_path), asset.size, "planned", reason))
continue
try:
copy_file_atomic(Path(asset.path), target_path, move=move)
action = "moved" if move else "copied"
report.append(CopyRecord(asset.name, asset.kind, asset.path, str(target_path), asset.size, action, reason))
if log:
log(f"{('剪切' if move else '复制')}{asset.name} -> {target_path}")
except Exception as exc:
report.append(CopyRecord(asset.name, asset.kind, asset.path, str(target_path), asset.size, "failed", str(exc)))
if log:
log(f"导入失败:{asset.name}{exc}")
write_csv(report_dir / f"disk_import_report_{stamp_text()}.csv", report)
return report
+88
View File
@@ -0,0 +1,88 @@
from __future__ import annotations
import tempfile
from pathlib import Path
from sync_core import (
copy_unc_missing_assets,
import_disk_assets_to_server,
parse_path_list,
read_inventory,
scan_assets,
write_inventory,
)
def write_file(path: Path, content: bytes) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(content)
def test_scan_server_writes_inventory_json() -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
server = root / "server"
output = root / "server_assets.json"
write_file(server / "LT1A_EXIST.tar.gz", b"same")
assets = scan_assets([server])
write_inventory(output, assets, [str(server)])
loaded = read_inventory(output)
assert len(loaded) == 1
assert loaded[0].name == "LT1A_EXIST.tar.gz"
def test_unc_copy_uses_server_json_to_skip_existing() -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
server = root / "server"
unc = root / "unc"
target = root / "disk_a"
report_dir = root / "reports"
inventory = root / "server_assets.json"
write_file(server / "LT1A_EXIST.tar.gz", b"same")
write_file(unc / "LT1A_EXIST.tar.gz", b"same")
write_file(unc / "LT1A_MISSING.tar.gz", b"new")
write_inventory(inventory, scan_assets([server]), [str(server)])
report = copy_unc_missing_assets([unc], inventory, [target], report_dir, execute=True)
assert (target / "LT1A_MISSING.tar.gz").read_bytes() == b"new"
assert not (target / "LT1A_EXIST.tar.gz").exists()
assert sum(1 for item in report if item.action == "skip") == 1
assert sum(1 for item in report if item.action == "copied") == 1
def test_import_disk_copies_or_moves_to_server_and_skips_existing() -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
disk = root / "disk"
server = root / "server"
report_dir = root / "reports"
write_file(server / "LT1A_EXIST.tar.gz", b"same")
write_file(disk / "LT1A_EXIST.tar.gz", b"same")
write_file(disk / "LT1A_NEW.tar.gz", b"new")
report = import_disk_assets_to_server([disk], [server], report_dir, execute=True, move=True)
assert (server / "LT1A_NEW.tar.gz").read_bytes() == b"new"
assert not (disk / "LT1A_NEW.tar.gz").exists()
assert sum(1 for item in report if item.action == "skip") == 1
assert sum(1 for item in report if item.action == "moved") == 1
def test_parse_path_list_accepts_semicolon_and_newline() -> None:
paths = parse_path_list(r"E:\;F:\Data" + "\n" + r"\\server\share")
assert len(paths) == 3
assert str(paths[1]) == r"F:\Data"
assert "server" in str(paths[2])
if __name__ == "__main__":
test_scan_server_writes_inventory_json()
test_unc_copy_uses_server_json_to_skip_existing()
test_import_disk_copies_or_moves_to_server_and_skips_existing()
test_parse_path_list_accepts_semicolon_and_newline()
print("ok")