From 8151bd8630612da7df2ac1422fb5b595c69ac15a Mon Sep 17 00:00:00 2001 From: Harmon Date: Mon, 15 Jun 2026 15:46:28 +0800 Subject: [PATCH] feat: filter GF3 production by scene date --- backend/app/routers/monitor.py | 56 ++++++++- .../gf3_sarscape_production_service.py | 32 +++++ backend/app/services/job_handlers.py | 1 + ...SCAPE_NATIVE_TO_GEOTIFF_DESIGN_20260530.md | 9 ++ frontend/src/DataMonitorPanel.jsx | 109 +++++++++++++++++- 5 files changed, 205 insertions(+), 2 deletions(-) diff --git a/backend/app/routers/monitor.py b/backend/app/routers/monitor.py index 21fbb89..12a9ef3 100644 --- a/backend/app/routers/monitor.py +++ b/backend/app/routers/monitor.py @@ -74,6 +74,7 @@ class GF3SarscapeSyncRequest(BaseModel): class GF3SarscapeProduceRequest(BaseModel): max_archives_per_run: Optional[int] = Field(default=None, ge=0) + selected_dates: List[str] = [] auto_standardize: Optional[bool] = None clean_after_success: Optional[bool] = None force_standardize: bool = False @@ -250,6 +251,16 @@ async def run_gf3_sarscape_produce( raise HTTPException(status_code=400, detail="GF3_SARSCAPE_DEM_PATH or GF3_GEO_DEM_PATH is not configured.") options = request_data or GF3SarscapeProduceRequest() + selected_dates = [] + for raw_date in options.selected_dates or []: + text = str(raw_date or "").strip() + if not text: + continue + normalized = text.replace("-", "").replace("_", "") + if len(normalized) != 8 or not normalized.isdigit(): + raise HTTPException(status_code=400, detail=f"Invalid GF3 scene date: {raw_date}") + if normalized not in selected_dates: + selected_dates.append(normalized) task_type = "GF3_SARSCAPE_PRODUCE" task_name = "GF3 SARscape production" auto_standardize = settings.GF3_SARSCAPE_AUTO_STANDARDIZE if options.auto_standardize is None else bool(options.auto_standardize) @@ -265,6 +276,7 @@ async def run_gf3_sarscape_produce( "polarizations": settings.GF3_SARSCAPE_POLARIZATIONS, "archive_exts": split_env_paths(settings.GF3_ARCHIVE_EXTS), "max_archives_per_run": int(options.max_archives_per_run or 0), + "selected_dates": selected_dates, "timeout_seconds": int(settings.GF3_SARSCAPE_PRODUCE_TIMEOUT_SECONDS or 0), "keep_extracted": bool(settings.GF3_SARSCAPE_KEEP_EXTRACTED), "auto_standardize": bool(auto_standardize), @@ -279,13 +291,55 @@ async def run_gf3_sarscape_produce( task_id = await task_service.create_task(task_type, task_name, params=payload) await job_queue_service.create_job(task_type, payload=payload, task_id=task_id) return { - "message": "GF3 SARscape production task submitted", + "message": ( + f"GF3 SARscape production task submitted for {', '.join(selected_dates)}" + if selected_dates + else "GF3 SARscape production task submitted" + ), "task_id": task_id, } except ValueError as e: raise HTTPException(status_code=409, detail=str(e)) +@router.get("/monitor/gf3-sarscape-dates") +async def list_gf3_sarscape_dates(admin_user: AuthUserORM = Depends(_require_admin)): + """ + List available GF3 SARscape source dates from configured raw archive roots. + """ + from ..services.gf3_sarscape_production_service import discover_gf3_sarscape_inputs + + gf3_archive_source_dirs = MONITOR_CONFIG.get("gf3_archive_source_dirs") or [] + if not gf3_archive_source_dirs: + raise HTTPException(status_code=400, detail="GF3_ARCHIVE_SOURCE_DIRS is not configured.") + + discovery = discover_gf3_sarscape_inputs( + gf3_archive_source_dirs, + archive_exts=split_env_paths(settings.GF3_ARCHIVE_EXTS), + ) + by_date: dict[str, dict[str, object]] = {} + undated = 0 + for item in discovery.get("inputs") or []: + scene_name = str(item.get("scene_name") or "") + date_text = str(item.get("scene_date") or "") + if not date_text: + undated += 1 + continue + bucket = by_date.setdefault(date_text, {"date": date_text, "scene_count": 0, "scenes": []}) + bucket["scene_count"] = int(bucket.get("scene_count") or 0) + 1 + scenes = bucket.get("scenes") + if isinstance(scenes, list) and len(scenes) < 20: + scenes.append(scene_name) + + dates = sorted(by_date.values(), key=lambda item: str(item.get("date") or ""), reverse=True) + return { + "dates": dates, + "input_count": discovery.get("input_count") or 0, + "undated_count": undated, + "missing_roots": discovery.get("missing_roots") or [], + } + + @router.post("/monitor/gf3-sarscape-clean", status_code=202) async def run_gf3_sarscape_clean( request_data: GF3SarscapeCleanRequest | None = None, diff --git a/backend/app/services/gf3_sarscape_production_service.py b/backend/app/services/gf3_sarscape_production_service.py index 8ba6c06..be42782 100644 --- a/backend/app/services/gf3_sarscape_production_service.py +++ b/backend/app/services/gf3_sarscape_production_service.py @@ -90,6 +90,25 @@ def _date_from_scene_name(scene_name: str) -> str | None: return match.group(1) if match else None +def _normalize_scene_date(value: Any) -> str | None: + text = _clean_text(value) + if not text: + return None + match = re.search(r"(20\d{2})[-_]?(\d{2})[-_]?(\d{2})", text) + if not match: + return None + return "".join(match.groups()) + + +def _normalize_scene_dates(values: list[str] | tuple[str, ...] | None) -> set[str]: + dates: set[str] = set() + for value in values or []: + normalized = _normalize_scene_date(value) + if normalized: + dates.add(normalized) + return dates + + def _resolve_existing_dirs(values: list[str] | tuple[str, ...] | None) -> tuple[list[Path], list[str]]: roots: list[Path] = [] missing: list[str] = [] @@ -228,6 +247,7 @@ def discover_gf3_sarscape_inputs( { "path": str(resolved), "scene_name": _scene_name_from_input(path), + "scene_date": _date_from_scene_name(_scene_name_from_input(path)), "ext": ext, "source_root": str(root), } @@ -431,6 +451,7 @@ def run_gf3_sarscape_production( polarizations: str | None = None, archive_exts: list[str] | None = None, max_archives_per_run: int | None = None, + selected_dates: list[str] | None = None, timeout_seconds: int | None = None, keep_extracted: bool | None = None, log_callback: LogCallback | None = None, @@ -455,6 +476,13 @@ def run_gf3_sarscape_production( ext_config = archive_exts if archive_exts is not None else split_env_paths(settings.GF3_ARCHIVE_EXTS) discovery = discover_gf3_sarscape_inputs(source_dirs, archive_exts=ext_config) inputs = discovery.get("inputs") or [] + selected_date_set = _normalize_scene_dates(selected_dates) + if selected_date_set: + inputs = [ + item + for item in inputs + if _normalize_scene_date(item.get("scene_date") or item.get("scene_name")) in selected_date_set + ] max_to_process = int(max_archives_per_run or 0) timeout = int(timeout_seconds or 0) keep = bool(settings.GF3_SARSCAPE_KEEP_EXTRACTED if keep_extracted is None else keep_extracted) @@ -466,6 +494,8 @@ def run_gf3_sarscape_production( _emit_log(log_callback, "INFO", f"GF3 SARscape wrapper: {wrapper_path}") _emit_log(log_callback, "INFO", f"GF3 SARscape DEM: {dem}") _emit_log(log_callback, "INFO", f"GF3 SARscape polarizations: {pol_text}") + if selected_date_set: + _emit_log(log_callback, "INFO", f"GF3 SARscape selected dates: {', '.join(sorted(selected_date_set))}") if not inputs: _emit_progress(progress_callback, 100, "GF3 SARscape production found no supported inputs.") @@ -477,6 +507,7 @@ def run_gf3_sarscape_production( "failed_count": 0, "deferred_count": 0, "missing_roots": discovery.get("missing_roots") or [], + "selected_dates": sorted(selected_date_set), "results": [], } @@ -642,6 +673,7 @@ def run_gf3_sarscape_production( "deferred_count": deferred, "native_root": str(native_root_path), "missing_roots": discovery.get("missing_roots") or [], + "selected_dates": sorted(selected_date_set), "results": results, } diff --git a/backend/app/services/job_handlers.py b/backend/app/services/job_handlers.py index 3c4b481..94edf67 100644 --- a/backend/app/services/job_handlers.py +++ b/backend/app/services/job_handlers.py @@ -4051,6 +4051,7 @@ async def _handle_gf3_sarscape_produce(job: SystemJobORM) -> None: polarizations=payload.get("polarizations"), archive_exts=payload.get("archive_exts") or [], max_archives_per_run=payload.get("max_archives_per_run"), + selected_dates=payload.get("selected_dates") or [], timeout_seconds=payload.get("timeout_seconds"), keep_extracted=payload.get("keep_extracted"), log_callback=_log_cb, diff --git a/docs/GF3_SARSCAPE_NATIVE_TO_GEOTIFF_DESIGN_20260530.md b/docs/GF3_SARSCAPE_NATIVE_TO_GEOTIFF_DESIGN_20260530.md index 228292d..34158ae 100644 --- a/docs/GF3_SARSCAPE_NATIVE_TO_GEOTIFF_DESIGN_20260530.md +++ b/docs/GF3_SARSCAPE_NATIVE_TO_GEOTIFF_DESIGN_20260530.md @@ -490,3 +490,12 @@ Skip production when either condition is true: - Standardized L2 output already exists in `GF3_STORAGE_DIRS//` with `gf3_standard_manifest.json` status `DONE`/`PARTIAL` and every requested polarization has a valid `*_L2.tif`. This prevents reprocessing when native intermediates were cleaned but registered/standardized results still exist. The standardized L2 and registered assets are the durable result layer; source archives on UNC should not be reprocessed unless the operator explicitly removes or invalidates the existing result. + +## 2026-06-15 Date-Scoped Production + +GF3 SARscape production now supports an optional scene-date filter. + +- `GET /api/monitor/gf3-sarscape-dates` scans `GF3_ARCHIVE_SOURCE_DIRS`, groups wrapper-supported raw archives by the `YYYYMMDD` date embedded in the GF3 scene name, and returns scene counts per date. +- `POST /api/monitor/gf3-sarscape-produce` accepts `selected_dates: ["YYYYMMDD"]`. When omitted or empty, production keeps the previous all-date behavior. +- The frontend monitor panel exposes a date selector in the GF3 SARscape production section. Operators can select one image date before starting production, or leave it as all dates. +- The existing duplicate-result preflight still runs after date filtering. A selected date will not reprocess scenes whose standardized L2 result or complete native `_geo` outputs already exist. diff --git a/frontend/src/DataMonitorPanel.jsx b/frontend/src/DataMonitorPanel.jsx index 279daf2..769ff1e 100644 --- a/frontend/src/DataMonitorPanel.jsx +++ b/frontend/src/DataMonitorPanel.jsx @@ -37,6 +37,14 @@ const DEFAULT_UNPACK_CONFIG = { const toArray = (value) => (Array.isArray(value) ? value : []); +const formatGf3Date = (value) => { + const text = String(value || '').replace(/\D/g, ''); + if (text.length !== 8) { + return value || ''; + } + return `${text.slice(0, 4)}-${text.slice(4, 6)}-${text.slice(6, 8)}`; +}; + const createUnpackRunOptions = (config = DEFAULT_UNPACK_CONFIG) => ({ max_files_per_run: String(config?.max_files_per_run ?? 0), max_runtime_minutes: String(config?.max_runtime_minutes ?? 0), @@ -81,6 +89,9 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled const [gf3SarscapeSyncLoading, setGf3SarscapeSyncLoading] = useState(false); const [gf3SarscapeCleanLoading, setGf3SarscapeCleanLoading] = useState(false); const [gf3ScanLoading, setGf3ScanLoading] = useState(false); + const [gf3DateLoading, setGf3DateLoading] = useState(false); + const [gf3SarscapeDates, setGf3SarscapeDates] = useState([]); + const [gf3SelectedDate, setGf3SelectedDate] = useState(''); const [gf3Message, setGf3Message] = useState(''); const logEndRef = useRef(null); @@ -321,6 +332,49 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled const canRunGf3SarscapeClean = !readOnly && configLoaded && hasGf3SarscapeNativeDirs && hasGf3StorageDirs; const canOpenUnpackDialog = !readOnly && unpackConfig.source_dirs.length > 0; + useEffect(() => { + if (!enabled || !configLoaded || !hasGf3ArchiveSourceDirs) { + setGf3SarscapeDates([]); + setGf3SelectedDate(''); + return; + } + + let canceled = false; + const fetchGf3Dates = async () => { + setGf3DateLoading(true); + try { + const res = await fetch(`${apiEndpoint}/monitor/gf3-sarscape-dates`, { credentials: 'include' }); + const data = await parseJsonSafe(res, {}); + if (canceled) { + return; + } + if (!res.ok) { + throw new Error(data?.detail || `HTTP ${res.status}`); + } + const nextDates = toArray(data?.dates); + setGf3SarscapeDates(nextDates); + setGf3SelectedDate((prev) => ( + prev && nextDates.some((item) => String(item?.date || '') === prev) ? prev : '' + )); + } catch (err) { + if (!canceled) { + console.error('Failed to fetch GF3 SARscape dates:', err); + setGf3SarscapeDates([]); + } + } finally { + if (!canceled) { + setGf3DateLoading(false); + } + } + }; + + fetchGf3Dates(); + + return () => { + canceled = true; + }; + }, [apiEndpoint, enabled, configLoaded, hasGf3ArchiveSourceDirs]); + const handleS1Run = async () => { if (readOnly) { setS1Message('当前账户为只读模式，无法触发 Sentinel-1 任务。'); @@ -480,13 +534,14 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled setGf3SarscapeProduceLoading(true); setGf3Message('GF3 SARscape 生产链路启动中...'); try { + const payload = gf3SelectedDate ? { selected_dates: [gf3SelectedDate] } : {}; const res = await fetch(`${apiEndpoint}/monitor/gf3-sarscape-produce`, { method: 'POST', credentials: 'include', headers: { 'Content-Type': 'application/json', }, - body: JSON.stringify({}), + body: JSON.stringify(payload), }); const data = await parseJsonSafe(res, {}); if (res.ok) { @@ -507,6 +562,32 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled } }; + const handleRefreshGf3Dates = async () => { + if (!hasGf3ArchiveSourceDirs) { + setGf3Message('GF3_ARCHIVE_SOURCE_DIRS 未配置。'); + return; + } + setGf3DateLoading(true); + setGf3Message('正在刷新 GF3 影像日期...'); + try { + const res = await fetch(`${apiEndpoint}/monitor/gf3-sarscape-dates`, { credentials: 'include' }); + const data = await parseJsonSafe(res, {}); + if (!res.ok) { + throw new Error(data?.detail || `HTTP ${res.status}`); + } + const nextDates = toArray(data?.dates); + setGf3SarscapeDates(nextDates); + setGf3SelectedDate((prev) => ( + prev && nextDates.some((item) => String(item?.date || '') === prev) ? prev : '' + )); + setGf3Message(`GF3 影像日期已刷新：${nextDates.length} 个日期。`); + } catch (err) { + setGf3Message(`失败：${err.message || '未知错误'}`); + } finally { + setGf3DateLoading(false); + } + }; + const handleGf3SarscapeSync = async () => { if (readOnly) { setGf3Message('当前账户为只读模式，无法触发 GF3 SARscape 标准化。'); @@ -902,6 +983,32 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
SARscape DEM{config.gf3_sarscape_dem_path || '未配置'}
极化{config.gf3_sarscape_polarizations || 'HH,HV'}
Legacy GDAL{gf3LegacyGdalEnabled ? '启用' : '关闭'}
+
+ 影像日期 + + + + +
{gf3LegacyGdalEnabled && (