feat: filter GF3 production by scene date
This commit is contained in:
@@ -74,6 +74,7 @@ class GF3SarscapeSyncRequest(BaseModel):
|
||||
|
||||
class GF3SarscapeProduceRequest(BaseModel):
|
||||
max_archives_per_run: Optional[int] = Field(default=None, ge=0)
|
||||
selected_dates: List[str] = []
|
||||
auto_standardize: Optional[bool] = None
|
||||
clean_after_success: Optional[bool] = None
|
||||
force_standardize: bool = False
|
||||
@@ -250,6 +251,16 @@ async def run_gf3_sarscape_produce(
|
||||
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_DEM_PATH or GF3_GEO_DEM_PATH is not configured.")
|
||||
|
||||
options = request_data or GF3SarscapeProduceRequest()
|
||||
selected_dates = []
|
||||
for raw_date in options.selected_dates or []:
|
||||
text = str(raw_date or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
normalized = text.replace("-", "").replace("_", "")
|
||||
if len(normalized) != 8 or not normalized.isdigit():
|
||||
raise HTTPException(status_code=400, detail=f"Invalid GF3 scene date: {raw_date}")
|
||||
if normalized not in selected_dates:
|
||||
selected_dates.append(normalized)
|
||||
task_type = "GF3_SARSCAPE_PRODUCE"
|
||||
task_name = "GF3 SARscape production"
|
||||
auto_standardize = settings.GF3_SARSCAPE_AUTO_STANDARDIZE if options.auto_standardize is None else bool(options.auto_standardize)
|
||||
@@ -265,6 +276,7 @@ async def run_gf3_sarscape_produce(
|
||||
"polarizations": settings.GF3_SARSCAPE_POLARIZATIONS,
|
||||
"archive_exts": split_env_paths(settings.GF3_ARCHIVE_EXTS),
|
||||
"max_archives_per_run": int(options.max_archives_per_run or 0),
|
||||
"selected_dates": selected_dates,
|
||||
"timeout_seconds": int(settings.GF3_SARSCAPE_PRODUCE_TIMEOUT_SECONDS or 0),
|
||||
"keep_extracted": bool(settings.GF3_SARSCAPE_KEEP_EXTRACTED),
|
||||
"auto_standardize": bool(auto_standardize),
|
||||
@@ -279,13 +291,55 @@ async def run_gf3_sarscape_produce(
|
||||
task_id = await task_service.create_task(task_type, task_name, params=payload)
|
||||
await job_queue_service.create_job(task_type, payload=payload, task_id=task_id)
|
||||
return {
|
||||
"message": "GF3 SARscape production task submitted",
|
||||
"message": (
|
||||
f"GF3 SARscape production task submitted for {', '.join(selected_dates)}"
|
||||
if selected_dates
|
||||
else "GF3 SARscape production task submitted"
|
||||
),
|
||||
"task_id": task_id,
|
||||
}
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=409, detail=str(e))
|
||||
|
||||
|
||||
@router.get("/monitor/gf3-sarscape-dates")
|
||||
async def list_gf3_sarscape_dates(admin_user: AuthUserORM = Depends(_require_admin)):
|
||||
"""
|
||||
List available GF3 SARscape source dates from configured raw archive roots.
|
||||
"""
|
||||
from ..services.gf3_sarscape_production_service import discover_gf3_sarscape_inputs
|
||||
|
||||
gf3_archive_source_dirs = MONITOR_CONFIG.get("gf3_archive_source_dirs") or []
|
||||
if not gf3_archive_source_dirs:
|
||||
raise HTTPException(status_code=400, detail="GF3_ARCHIVE_SOURCE_DIRS is not configured.")
|
||||
|
||||
discovery = discover_gf3_sarscape_inputs(
|
||||
gf3_archive_source_dirs,
|
||||
archive_exts=split_env_paths(settings.GF3_ARCHIVE_EXTS),
|
||||
)
|
||||
by_date: dict[str, dict[str, object]] = {}
|
||||
undated = 0
|
||||
for item in discovery.get("inputs") or []:
|
||||
scene_name = str(item.get("scene_name") or "")
|
||||
date_text = str(item.get("scene_date") or "")
|
||||
if not date_text:
|
||||
undated += 1
|
||||
continue
|
||||
bucket = by_date.setdefault(date_text, {"date": date_text, "scene_count": 0, "scenes": []})
|
||||
bucket["scene_count"] = int(bucket.get("scene_count") or 0) + 1
|
||||
scenes = bucket.get("scenes")
|
||||
if isinstance(scenes, list) and len(scenes) < 20:
|
||||
scenes.append(scene_name)
|
||||
|
||||
dates = sorted(by_date.values(), key=lambda item: str(item.get("date") or ""), reverse=True)
|
||||
return {
|
||||
"dates": dates,
|
||||
"input_count": discovery.get("input_count") or 0,
|
||||
"undated_count": undated,
|
||||
"missing_roots": discovery.get("missing_roots") or [],
|
||||
}
|
||||
|
||||
|
||||
@router.post("/monitor/gf3-sarscape-clean", status_code=202)
|
||||
async def run_gf3_sarscape_clean(
|
||||
request_data: GF3SarscapeCleanRequest | None = None,
|
||||
|
||||
@@ -90,6 +90,25 @@ def _date_from_scene_name(scene_name: str) -> str | None:
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
def _normalize_scene_date(value: Any) -> str | None:
|
||||
text = _clean_text(value)
|
||||
if not text:
|
||||
return None
|
||||
match = re.search(r"(20\d{2})[-_]?(\d{2})[-_]?(\d{2})", text)
|
||||
if not match:
|
||||
return None
|
||||
return "".join(match.groups())
|
||||
|
||||
|
||||
def _normalize_scene_dates(values: list[str] | tuple[str, ...] | None) -> set[str]:
|
||||
dates: set[str] = set()
|
||||
for value in values or []:
|
||||
normalized = _normalize_scene_date(value)
|
||||
if normalized:
|
||||
dates.add(normalized)
|
||||
return dates
|
||||
|
||||
|
||||
def _resolve_existing_dirs(values: list[str] | tuple[str, ...] | None) -> tuple[list[Path], list[str]]:
|
||||
roots: list[Path] = []
|
||||
missing: list[str] = []
|
||||
@@ -228,6 +247,7 @@ def discover_gf3_sarscape_inputs(
|
||||
{
|
||||
"path": str(resolved),
|
||||
"scene_name": _scene_name_from_input(path),
|
||||
"scene_date": _date_from_scene_name(_scene_name_from_input(path)),
|
||||
"ext": ext,
|
||||
"source_root": str(root),
|
||||
}
|
||||
@@ -431,6 +451,7 @@ def run_gf3_sarscape_production(
|
||||
polarizations: str | None = None,
|
||||
archive_exts: list[str] | None = None,
|
||||
max_archives_per_run: int | None = None,
|
||||
selected_dates: list[str] | None = None,
|
||||
timeout_seconds: int | None = None,
|
||||
keep_extracted: bool | None = None,
|
||||
log_callback: LogCallback | None = None,
|
||||
@@ -455,6 +476,13 @@ def run_gf3_sarscape_production(
|
||||
ext_config = archive_exts if archive_exts is not None else split_env_paths(settings.GF3_ARCHIVE_EXTS)
|
||||
discovery = discover_gf3_sarscape_inputs(source_dirs, archive_exts=ext_config)
|
||||
inputs = discovery.get("inputs") or []
|
||||
selected_date_set = _normalize_scene_dates(selected_dates)
|
||||
if selected_date_set:
|
||||
inputs = [
|
||||
item
|
||||
for item in inputs
|
||||
if _normalize_scene_date(item.get("scene_date") or item.get("scene_name")) in selected_date_set
|
||||
]
|
||||
max_to_process = int(max_archives_per_run or 0)
|
||||
timeout = int(timeout_seconds or 0)
|
||||
keep = bool(settings.GF3_SARSCAPE_KEEP_EXTRACTED if keep_extracted is None else keep_extracted)
|
||||
@@ -466,6 +494,8 @@ def run_gf3_sarscape_production(
|
||||
_emit_log(log_callback, "INFO", f"GF3 SARscape wrapper: {wrapper_path}")
|
||||
_emit_log(log_callback, "INFO", f"GF3 SARscape DEM: {dem}")
|
||||
_emit_log(log_callback, "INFO", f"GF3 SARscape polarizations: {pol_text}")
|
||||
if selected_date_set:
|
||||
_emit_log(log_callback, "INFO", f"GF3 SARscape selected dates: {', '.join(sorted(selected_date_set))}")
|
||||
|
||||
if not inputs:
|
||||
_emit_progress(progress_callback, 100, "GF3 SARscape production found no supported inputs.")
|
||||
@@ -477,6 +507,7 @@ def run_gf3_sarscape_production(
|
||||
"failed_count": 0,
|
||||
"deferred_count": 0,
|
||||
"missing_roots": discovery.get("missing_roots") or [],
|
||||
"selected_dates": sorted(selected_date_set),
|
||||
"results": [],
|
||||
}
|
||||
|
||||
@@ -642,6 +673,7 @@ def run_gf3_sarscape_production(
|
||||
"deferred_count": deferred,
|
||||
"native_root": str(native_root_path),
|
||||
"missing_roots": discovery.get("missing_roots") or [],
|
||||
"selected_dates": sorted(selected_date_set),
|
||||
"results": results,
|
||||
}
|
||||
|
||||
|
||||
@@ -4051,6 +4051,7 @@ async def _handle_gf3_sarscape_produce(job: SystemJobORM) -> None:
|
||||
polarizations=payload.get("polarizations"),
|
||||
archive_exts=payload.get("archive_exts") or [],
|
||||
max_archives_per_run=payload.get("max_archives_per_run"),
|
||||
selected_dates=payload.get("selected_dates") or [],
|
||||
timeout_seconds=payload.get("timeout_seconds"),
|
||||
keep_extracted=payload.get("keep_extracted"),
|
||||
log_callback=_log_cb,
|
||||
|
||||
@@ -490,3 +490,12 @@ Skip production when either condition is true:
|
||||
- Standardized L2 output already exists in `GF3_STORAGE_DIRS/<imaging_date>/<scene_name>` with `gf3_standard_manifest.json` status `DONE`/`PARTIAL` and every requested polarization has a valid `*_L2.tif`.
|
||||
|
||||
This prevents reprocessing when native intermediates were cleaned but registered/standardized results still exist. The standardized L2 and registered assets are the durable result layer; source archives on UNC should not be reprocessed unless the operator explicitly removes or invalidates the existing result.
|
||||
|
||||
## 2026-06-15 Date-Scoped Production
|
||||
|
||||
GF3 SARscape production now supports an optional scene-date filter.
|
||||
|
||||
- `GET /api/monitor/gf3-sarscape-dates` scans `GF3_ARCHIVE_SOURCE_DIRS`, groups wrapper-supported raw archives by the `YYYYMMDD` date embedded in the GF3 scene name, and returns scene counts per date.
|
||||
- `POST /api/monitor/gf3-sarscape-produce` accepts `selected_dates: ["YYYYMMDD"]`. When omitted or empty, production keeps the previous all-date behavior.
|
||||
- The frontend monitor panel exposes a date selector in the GF3 SARscape production section. Operators can select one image date before starting production, or leave it as all dates.
|
||||
- The existing duplicate-result preflight still runs after date filtering. A selected date will not reprocess scenes whose standardized L2 result or complete native `_geo` outputs already exist.
|
||||
|
||||
@@ -37,6 +37,14 @@ const DEFAULT_UNPACK_CONFIG = {
|
||||
|
||||
const toArray = (value) => (Array.isArray(value) ? value : []);
|
||||
|
||||
const formatGf3Date = (value) => {
|
||||
const text = String(value || '').replace(/\D/g, '');
|
||||
if (text.length !== 8) {
|
||||
return value || '';
|
||||
}
|
||||
return `${text.slice(0, 4)}-${text.slice(4, 6)}-${text.slice(6, 8)}`;
|
||||
};
|
||||
|
||||
const createUnpackRunOptions = (config = DEFAULT_UNPACK_CONFIG) => ({
|
||||
max_files_per_run: String(config?.max_files_per_run ?? 0),
|
||||
max_runtime_minutes: String(config?.max_runtime_minutes ?? 0),
|
||||
@@ -81,6 +89,9 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
|
||||
const [gf3SarscapeSyncLoading, setGf3SarscapeSyncLoading] = useState(false);
|
||||
const [gf3SarscapeCleanLoading, setGf3SarscapeCleanLoading] = useState(false);
|
||||
const [gf3ScanLoading, setGf3ScanLoading] = useState(false);
|
||||
const [gf3DateLoading, setGf3DateLoading] = useState(false);
|
||||
const [gf3SarscapeDates, setGf3SarscapeDates] = useState([]);
|
||||
const [gf3SelectedDate, setGf3SelectedDate] = useState('');
|
||||
const [gf3Message, setGf3Message] = useState('');
|
||||
const logEndRef = useRef(null);
|
||||
|
||||
@@ -321,6 +332,49 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
|
||||
const canRunGf3SarscapeClean = !readOnly && configLoaded && hasGf3SarscapeNativeDirs && hasGf3StorageDirs;
|
||||
const canOpenUnpackDialog = !readOnly && unpackConfig.source_dirs.length > 0;
|
||||
|
||||
useEffect(() => {
|
||||
if (!enabled || !configLoaded || !hasGf3ArchiveSourceDirs) {
|
||||
setGf3SarscapeDates([]);
|
||||
setGf3SelectedDate('');
|
||||
return;
|
||||
}
|
||||
|
||||
let canceled = false;
|
||||
const fetchGf3Dates = async () => {
|
||||
setGf3DateLoading(true);
|
||||
try {
|
||||
const res = await fetch(`${apiEndpoint}/monitor/gf3-sarscape-dates`, { credentials: 'include' });
|
||||
const data = await parseJsonSafe(res, {});
|
||||
if (canceled) {
|
||||
return;
|
||||
}
|
||||
if (!res.ok) {
|
||||
throw new Error(data?.detail || `HTTP ${res.status}`);
|
||||
}
|
||||
const nextDates = toArray(data?.dates);
|
||||
setGf3SarscapeDates(nextDates);
|
||||
setGf3SelectedDate((prev) => (
|
||||
prev && nextDates.some((item) => String(item?.date || '') === prev) ? prev : ''
|
||||
));
|
||||
} catch (err) {
|
||||
if (!canceled) {
|
||||
console.error('Failed to fetch GF3 SARscape dates:', err);
|
||||
setGf3SarscapeDates([]);
|
||||
}
|
||||
} finally {
|
||||
if (!canceled) {
|
||||
setGf3DateLoading(false);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
fetchGf3Dates();
|
||||
|
||||
return () => {
|
||||
canceled = true;
|
||||
};
|
||||
}, [apiEndpoint, enabled, configLoaded, hasGf3ArchiveSourceDirs]);
|
||||
|
||||
const handleS1Run = async () => {
|
||||
if (readOnly) {
|
||||
setS1Message('当前账户为只读模式,无法触发 Sentinel-1 任务。');
|
||||
@@ -480,13 +534,14 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
|
||||
setGf3SarscapeProduceLoading(true);
|
||||
setGf3Message('GF3 SARscape 生产链路启动中...');
|
||||
try {
|
||||
const payload = gf3SelectedDate ? { selected_dates: [gf3SelectedDate] } : {};
|
||||
const res = await fetch(`${apiEndpoint}/monitor/gf3-sarscape-produce`, {
|
||||
method: 'POST',
|
||||
credentials: 'include',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({}),
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
const data = await parseJsonSafe(res, {});
|
||||
if (res.ok) {
|
||||
@@ -507,6 +562,32 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
|
||||
}
|
||||
};
|
||||
|
||||
const handleRefreshGf3Dates = async () => {
|
||||
if (!hasGf3ArchiveSourceDirs) {
|
||||
setGf3Message('GF3_ARCHIVE_SOURCE_DIRS 未配置。');
|
||||
return;
|
||||
}
|
||||
setGf3DateLoading(true);
|
||||
setGf3Message('正在刷新 GF3 影像日期...');
|
||||
try {
|
||||
const res = await fetch(`${apiEndpoint}/monitor/gf3-sarscape-dates`, { credentials: 'include' });
|
||||
const data = await parseJsonSafe(res, {});
|
||||
if (!res.ok) {
|
||||
throw new Error(data?.detail || `HTTP ${res.status}`);
|
||||
}
|
||||
const nextDates = toArray(data?.dates);
|
||||
setGf3SarscapeDates(nextDates);
|
||||
setGf3SelectedDate((prev) => (
|
||||
prev && nextDates.some((item) => String(item?.date || '') === prev) ? prev : ''
|
||||
));
|
||||
setGf3Message(`GF3 影像日期已刷新:${nextDates.length} 个日期。`);
|
||||
} catch (err) {
|
||||
setGf3Message(`失败:${err.message || '未知错误'}`);
|
||||
} finally {
|
||||
setGf3DateLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleGf3SarscapeSync = async () => {
|
||||
if (readOnly) {
|
||||
setGf3Message('当前账户为只读模式,无法触发 GF3 SARscape 标准化。');
|
||||
@@ -902,6 +983,32 @@ const DataMonitorPanel = ({ apiEndpoint, onTaskStart, readOnly = false, enabled
|
||||
<div style={rowStyle}><span style={labelStyle}>SARscape DEM</span><span style={{ wordBreak: 'break-all' }}>{config.gf3_sarscape_dem_path || '未配置'}</span></div>
|
||||
<div style={rowStyle}><span style={labelStyle}>极化</span><span>{config.gf3_sarscape_polarizations || 'HH,HV'}</span></div>
|
||||
<div style={rowStyle}><span style={labelStyle}>Legacy GDAL</span><span>{gf3LegacyGdalEnabled ? '启用' : '关闭'}</span></div>
|
||||
<div style={rowStyle}>
|
||||
<span style={labelStyle}>影像日期</span>
|
||||
<span style={{ display: 'flex', gap: '8px', alignItems: 'center', flexWrap: 'wrap' }}>
|
||||
<select
|
||||
value={gf3SelectedDate}
|
||||
onChange={(event) => setGf3SelectedDate(event.target.value)}
|
||||
disabled={gf3DateLoading || gf3SarscapeProduceLoading || readOnly || !canRunGf3SarscapeProduce}
|
||||
style={{ minWidth: '180px', padding: '4px 6px' }}
|
||||
>
|
||||
<option value="">全部可用日期</option>
|
||||
{gf3SarscapeDates.map((item) => (
|
||||
<option key={item.date} value={item.date}>
|
||||
{formatGf3Date(item.date)} ({item.scene_count || 0})
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
<button
|
||||
type="button"
|
||||
onClick={handleRefreshGf3Dates}
|
||||
disabled={gf3DateLoading || readOnly || !hasGf3ArchiveSourceDirs}
|
||||
style={actionBtnStyle(gf3DateLoading, readOnly || !hasGf3ArchiveSourceDirs)}
|
||||
>
|
||||
{gf3DateLoading ? '加载中...' : '刷新日期'}
|
||||
</button>
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
<div style={{ display: 'flex', gap: '10px', flexWrap: 'wrap' }}>
|
||||
{gf3LegacyGdalEnabled && (
|
||||
|
||||
Reference in New Issue
Block a user