Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: filter GF3 production by scene date

Browse Source
This commit is contained in:
Harmon
2026-06-15 15:46:28 +08:00
parent 2f86c5a8cb
commit 8151bd8630
5 changed files with 205 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/routers/monitor.py
+55 -1
View File
@@ -74,6 +74,7 @@ class GF3SarscapeSyncRequest(BaseModel):
class GF3SarscapeProduceRequest(BaseModel):
max_archives_per_run: Optional[int] = Field(default=None, ge=0)
selected_dates: List[str] = []
auto_standardize: Optional[bool] = None
clean_after_success: Optional[bool] = None
force_standardize: bool = False
@@ -250,6 +251,16 @@ async def run_gf3_sarscape_produce(
raise HTTPException(status_code=400, detail="GF3_SARSCAPE_DEM_PATH or GF3_GEO_DEM_PATH is not configured.")
options = request_data or GF3SarscapeProduceRequest()
selected_dates = []
for raw_date in options.selected_dates or []:
text = str(raw_date or "").strip()
if not text:
continue
normalized = text.replace("-", "").replace("_", "")
if len(normalized) != 8 or not normalized.isdigit():
raise HTTPException(status_code=400, detail=f"Invalid GF3 scene date: {raw_date}")
if normalized not in selected_dates:
selected_dates.append(normalized)
task_type = "GF3_SARSCAPE_PRODUCE"
task_name = "GF3 SARscape production"
auto_standardize = settings.GF3_SARSCAPE_AUTO_STANDARDIZE if options.auto_standardize is None else bool(options.auto_standardize)
@@ -265,6 +276,7 @@ async def run_gf3_sarscape_produce(
"polarizations": settings.GF3_SARSCAPE_POLARIZATIONS,
"archive_exts": split_env_paths(settings.GF3_ARCHIVE_EXTS),
"max_archives_per_run": int(options.max_archives_per_run or 0),
"selected_dates": selected_dates,
"timeout_seconds": int(settings.GF3_SARSCAPE_PRODUCE_TIMEOUT_SECONDS or 0),
"keep_extracted": bool(settings.GF3_SARSCAPE_KEEP_EXTRACTED),
"auto_standardize": bool(auto_standardize),
@@ -279,13 +291,55 @@ async def run_gf3_sarscape_produce(
task_id = await task_service.create_task(task_type, task_name, params=payload)
await job_queue_service.create_job(task_type, payload=payload, task_id=task_id)
return {
"message": "GF3 SARscape production task submitted",
"message": (
f"GF3 SARscape production task submitted for {', '.join(selected_dates)}"
if selected_dates
else "GF3 SARscape production task submitted"
),
"task_id": task_id,
}
except ValueError as e:
raise HTTPException(status_code=409, detail=str(e))
@router.get("/monitor/gf3-sarscape-dates")
async def list_gf3_sarscape_dates(admin_user: AuthUserORM = Depends(_require_admin)):
"""
List available GF3 SARscape source dates from configured raw archive roots.
"""
from ..services.gf3_sarscape_production_service import discover_gf3_sarscape_inputs
gf3_archive_source_dirs = MONITOR_CONFIG.get("gf3_archive_source_dirs") or []
if not gf3_archive_source_dirs:
raise HTTPException(status_code=400, detail="GF3_ARCHIVE_SOURCE_DIRS is not configured.")
discovery = discover_gf3_sarscape_inputs(
gf3_archive_source_dirs,
archive_exts=split_env_paths(settings.GF3_ARCHIVE_EXTS),
)
by_date: dict[str, dict[str, object]] = {}
undated = 0
for item in discovery.get("inputs") or []:
scene_name = str(item.get("scene_name") or "")
date_text = str(item.get("scene_date") or "")
if not date_text:
undated += 1
continue
bucket = by_date.setdefault(date_text, {"date": date_text, "scene_count": 0, "scenes": []})
bucket["scene_count"] = int(bucket.get("scene_count") or 0) + 1
scenes = bucket.get("scenes")
if isinstance(scenes, list) and len(scenes) < 20:
scenes.append(scene_name)
dates = sorted(by_date.values(), key=lambda item: str(item.get("date") or ""), reverse=True)
return {
"dates": dates,
"input_count": discovery.get("input_count") or 0,
"undated_count": undated,
"missing_roots": discovery.get("missing_roots") or [],
}
@router.post("/monitor/gf3-sarscape-clean", status_code=202)
async def run_gf3_sarscape_clean(
request_data: GF3SarscapeCleanRequest | None = None,
backend/app/services/gf3_sarscape_production_service.py
+32
View File
@@ -90,6 +90,25 @@ def _date_from_scene_name(scene_name: str) -> str | None:
return match.group(1) if match else None
def _normalize_scene_date(value: Any) -> str | None:
text = _clean_text(value)
if not text:
return None
match = re.search(r"(20\d{2})[-_]?(\d{2})[-_]?(\d{2})", text)
if not match:
return None
return "".join(match.groups())
def _normalize_scene_dates(values: list[str] | tuple[str, ...] | None) -> set[str]:
dates: set[str] = set()
for value in values or []:
normalized = _normalize_scene_date(value)
if normalized:
dates.add(normalized)
return dates
def _resolve_existing_dirs(values: list[str] | tuple[str, ...] | None) -> tuple[list[Path], list[str]]:
roots: list[Path] = []
missing: list[str] = []
@@ -228,6 +247,7 @@ def discover_gf3_sarscape_inputs(
{
"path": str(resolved),
"scene_name": _scene_name_from_input(path),
"scene_date": _date_from_scene_name(_scene_name_from_input(path)),
"ext": ext,
"source_root": str(root),
}
@@ -431,6 +451,7 @@ def run_gf3_sarscape_production(
polarizations: str | None = None,
archive_exts: list[str] | None = None,
max_archives_per_run: int | None = None,
selected_dates: list[str] | None = None,
timeout_seconds: int | None = None,
keep_extracted: bool | None = None,
log_callback: LogCallback | None = None,
@@ -455,6 +476,13 @@ def run_gf3_sarscape_production(
ext_config = archive_exts if archive_exts is not None else split_env_paths(settings.GF3_ARCHIVE_EXTS)
discovery = discover_gf3_sarscape_inputs(source_dirs, archive_exts=ext_config)
inputs = discovery.get("inputs") or []
selected_date_set = _normalize_scene_dates(selected_dates)
if selected_date_set:
inputs = [
item
for item in inputs
if _normalize_scene_date(item.get("scene_date") or item.get("scene_name")) in selected_date_set
]
max_to_process = int(max_archives_per_run or 0)
timeout = int(timeout_seconds or 0)
keep = bool(settings.GF3_SARSCAPE_KEEP_EXTRACTED if keep_extracted is None else keep_extracted)
@@ -466,6 +494,8 @@ def run_gf3_sarscape_production(
_emit_log(log_callback, "INFO", f"GF3 SARscape wrapper: {wrapper_path}")
_emit_log(log_callback, "INFO", f"GF3 SARscape DEM: {dem}")
_emit_log(log_callback, "INFO", f"GF3 SARscape polarizations: {pol_text}")
if selected_date_set:
_emit_log(log_callback, "INFO", f"GF3 SARscape selected dates: {', '.join(sorted(selected_date_set))}")
if not inputs:
_emit_progress(progress_callback, 100, "GF3 SARscape production found no supported inputs.")
@@ -477,6 +507,7 @@ def run_gf3_sarscape_production(
"failed_count": 0,
"deferred_count": 0,
"missing_roots": discovery.get("missing_roots") or [],
"selected_dates": sorted(selected_date_set),
"results": [],
}
@@ -642,6 +673,7 @@ def run_gf3_sarscape_production(
"deferred_count": deferred,
"native_root": str(native_root_path),
"missing_roots": discovery.get("missing_roots") or [],
"selected_dates": sorted(selected_date_set),
"results": results,
}
backend/app/services/job_handlers.py
+1
View File
@@ -4051,6 +4051,7 @@ async def _handle_gf3_sarscape_produce(job: SystemJobORM) -> None:
polarizations=payload.get("polarizations"),
archive_exts=payload.get("archive_exts") or [],
max_archives_per_run=payload.get("max_archives_per_run"),
selected_dates=payload.get("selected_dates") or [],
timeout_seconds=payload.get("timeout_seconds"),
keep_extracted=payload.get("keep_extracted"),
log_callback=_log_cb,