600 lines
20 KiB
Python
600 lines
20 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
import re
|
|
from dataclasses import dataclass
|
|
from typing import Any, Dict, Iterable, List, Optional
|
|
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from .. import database
|
|
from ..config import settings, split_env_paths
|
|
from ..models import ManagedRootORM, PathInventoryORM, ScanCursorORM
|
|
|
|
|
|
_SLUG_RE = re.compile(r"[^a-z0-9]+")
|
|
_WINDOWS_DRIVE_RE = re.compile(r"^[a-zA-Z]:[\\/]")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RootSpec:
|
|
root_code: str
|
|
root_role: str
|
|
display_name: str
|
|
path: str
|
|
path_kind: str
|
|
source_kind: str
|
|
source_ref: Optional[str]
|
|
scan_mode: str
|
|
owner_engine: Optional[str]
|
|
metadata_json: Dict[str, Any]
|
|
|
|
|
|
def _new_session() -> AsyncSession:
|
|
if database.AsyncSessionLocal is None:
|
|
raise RuntimeError("Database session factory is not initialized.")
|
|
return database.AsyncSessionLocal()
|
|
|
|
|
|
def _slugify(value: str, *, max_len: int = 96) -> str:
|
|
text = _SLUG_RE.sub("_", str(value or "").strip().lower()).strip("_")
|
|
return (text or "item")[:max_len]
|
|
|
|
|
|
def _normalize_root_path(path: str) -> str:
|
|
text = str(path or "").strip()
|
|
if not text:
|
|
return ""
|
|
if text.startswith("\\\\"):
|
|
return os.path.normpath(text)
|
|
if _WINDOWS_DRIVE_RE.match(text):
|
|
return os.path.normpath(text)
|
|
if text.startswith("/"):
|
|
return text.replace("\\", "/")
|
|
return os.path.normpath(os.path.abspath(text))
|
|
|
|
|
|
def _detect_path_kind(path: str) -> str:
|
|
text = str(path or "").strip()
|
|
if text.startswith("\\\\"):
|
|
return "unc"
|
|
if _WINDOWS_DRIVE_RE.match(text):
|
|
return "windows"
|
|
if text.startswith("/mnt/"):
|
|
return "wsl_mount"
|
|
if text.startswith("/"):
|
|
return "posix"
|
|
return "relative"
|
|
|
|
|
|
def _build_root_code(root_role: str, source_ref: Optional[str]) -> str:
|
|
role_part = _slugify(root_role, max_len=48)
|
|
source_part = _slugify(source_ref or "manual", max_len=40)
|
|
return f"{role_part}__{source_part}"[:96]
|
|
|
|
|
|
def _cursor_type_for_scan_mode(scan_mode: str) -> str:
|
|
mapping = {
|
|
"archive_walk": "archive_walk",
|
|
"scene_directory": "directory_walk",
|
|
"manifest_tree": "manifest_tree",
|
|
"file_pool": "file_pool",
|
|
"workspace": "workspace",
|
|
"quarantine": "directory_walk",
|
|
"directory_walk": "directory_walk",
|
|
}
|
|
return mapping.get(str(scan_mode or "").strip().lower(), "directory_walk")
|
|
|
|
|
|
def _iter_multi_root_specs(
|
|
*,
|
|
env_var: str,
|
|
paths: Iterable[str],
|
|
root_role: str,
|
|
display_prefix: str,
|
|
scan_mode: str,
|
|
owner_engine: Optional[str] = None,
|
|
) -> List[RootSpec]:
|
|
specs: List[RootSpec] = []
|
|
deduped: List[str] = []
|
|
for raw_path in paths:
|
|
normalized_path = _normalize_root_path(raw_path)
|
|
if normalized_path and normalized_path not in deduped:
|
|
deduped.append(normalized_path)
|
|
|
|
for idx, path in enumerate(deduped, start=1):
|
|
source_ref = f"{env_var}[{idx}]"
|
|
specs.append(
|
|
RootSpec(
|
|
root_code=_build_root_code(root_role, source_ref),
|
|
root_role=root_role,
|
|
display_name=f"{display_prefix} {idx}",
|
|
path=path,
|
|
path_kind=_detect_path_kind(path),
|
|
source_kind="env",
|
|
source_ref=source_ref,
|
|
scan_mode=scan_mode,
|
|
owner_engine=owner_engine,
|
|
metadata_json={
|
|
"env_var": env_var,
|
|
"env_index": idx,
|
|
"imported_from": "settings",
|
|
},
|
|
)
|
|
)
|
|
return specs
|
|
|
|
|
|
def _iter_single_root_specs(
|
|
*,
|
|
env_var: str,
|
|
path: Optional[str],
|
|
root_role: str,
|
|
display_name: str,
|
|
scan_mode: str,
|
|
owner_engine: Optional[str] = None,
|
|
) -> List[RootSpec]:
|
|
normalized_path = _normalize_root_path(path or "")
|
|
if not normalized_path:
|
|
return []
|
|
source_ref = env_var
|
|
return [
|
|
RootSpec(
|
|
root_code=_build_root_code(root_role, source_ref),
|
|
root_role=root_role,
|
|
display_name=display_name,
|
|
path=normalized_path,
|
|
path_kind=_detect_path_kind(normalized_path),
|
|
source_kind="env",
|
|
source_ref=source_ref,
|
|
scan_mode=scan_mode,
|
|
owner_engine=owner_engine,
|
|
metadata_json={
|
|
"env_var": env_var,
|
|
"imported_from": "settings",
|
|
},
|
|
)
|
|
]
|
|
|
|
|
|
def _build_root_specs_from_settings() -> List[RootSpec]:
|
|
specs: List[RootSpec] = []
|
|
specs.extend(
|
|
_iter_multi_root_specs(
|
|
env_var="UNPACK_SOURCE_DIRS",
|
|
paths=split_env_paths(settings.UNPACK_SOURCE_DIRS),
|
|
root_role="archive_inbox",
|
|
display_prefix="Archive Inbox",
|
|
scan_mode="archive_walk",
|
|
)
|
|
)
|
|
specs.extend(
|
|
_iter_multi_root_specs(
|
|
env_var="INSAR_STORAGE_DIRS",
|
|
paths=split_env_paths(settings.INSAR_STORAGE_DIRS),
|
|
root_role="source_pool_radar",
|
|
display_prefix="Radar Source Pool",
|
|
scan_mode="scene_directory",
|
|
)
|
|
)
|
|
specs.extend(
|
|
_iter_multi_root_specs(
|
|
env_var="MONITOR_RADAR_DIRS",
|
|
paths=split_env_paths(settings.MONITOR_RADAR_DIRS),
|
|
root_role="legacy_scan_root_radar",
|
|
display_prefix="Legacy Radar Scan Root",
|
|
scan_mode="directory_walk",
|
|
)
|
|
)
|
|
specs.extend(
|
|
_iter_multi_root_specs(
|
|
env_var="MONITOR_DINSAR_DIRS",
|
|
paths=split_env_paths(settings.MONITOR_DINSAR_DIRS),
|
|
root_role="legacy_scan_root_dinsar",
|
|
display_prefix="Legacy D-InSAR Scan Root",
|
|
scan_mode="directory_walk",
|
|
)
|
|
)
|
|
specs.extend(
|
|
_iter_multi_root_specs(
|
|
env_var="GF3_SOURCE_DIRS",
|
|
paths=split_env_paths(settings.GF3_SOURCE_DIRS),
|
|
root_role="source_pool_gf3_input",
|
|
display_prefix="GF3 Input Pool",
|
|
scan_mode="scene_directory",
|
|
)
|
|
)
|
|
specs.extend(
|
|
_iter_multi_root_specs(
|
|
env_var="GF3_STORAGE_DIRS",
|
|
paths=split_env_paths(settings.GF3_STORAGE_DIRS),
|
|
root_role="source_pool_gf3_output",
|
|
display_prefix="GF3 Output Pool",
|
|
scan_mode="scene_directory",
|
|
)
|
|
)
|
|
specs.extend(
|
|
_iter_single_root_specs(
|
|
env_var="MONITOR_ORBIT_DIR",
|
|
path=settings.MONITOR_ORBIT_DIR,
|
|
root_role="orbit_source",
|
|
display_name="Orbit Source",
|
|
scan_mode="file_pool",
|
|
)
|
|
)
|
|
specs.extend(
|
|
_iter_single_root_specs(
|
|
env_var="ORBIT_POOL_ENVI",
|
|
path=settings.ORBIT_POOL_ENVI,
|
|
root_role="orbit_pool_envi",
|
|
display_name="ENVI Orbit Pool",
|
|
scan_mode="file_pool",
|
|
owner_engine="envi",
|
|
)
|
|
)
|
|
specs.extend(
|
|
_iter_single_root_specs(
|
|
env_var="ORBIT_POOL_ISCE2",
|
|
path=settings.ORBIT_POOL_ISCE2,
|
|
root_role="orbit_pool_isce2",
|
|
display_name="ISCE2 Orbit Pool",
|
|
scan_mode="file_pool",
|
|
owner_engine="isce2",
|
|
)
|
|
)
|
|
specs.extend(
|
|
_iter_single_root_specs(
|
|
env_var="ORBIT_POOL_LANDSAR",
|
|
path=settings.ORBIT_POOL_LANDSAR,
|
|
root_role="orbit_pool_landsar",
|
|
display_name="LandSAR Orbit Pool",
|
|
scan_mode="file_pool",
|
|
owner_engine="landsar",
|
|
)
|
|
)
|
|
specs.extend(
|
|
_iter_single_root_specs(
|
|
env_var="RESULT_PUBLISH_ROOT",
|
|
path=settings.RESULT_PUBLISH_ROOT,
|
|
root_role="result_publish_root",
|
|
display_name="Result Publish Root",
|
|
scan_mode="manifest_tree",
|
|
)
|
|
)
|
|
specs.extend(
|
|
_iter_single_root_specs(
|
|
env_var="DINSAR_PRODUCT_DIR",
|
|
path=settings.DINSAR_PRODUCT_DIR,
|
|
root_role="publish_root_dinsar",
|
|
display_name="D-InSAR Publish Root",
|
|
scan_mode="manifest_tree",
|
|
owner_engine="dinsar",
|
|
)
|
|
)
|
|
specs.extend(
|
|
_iter_single_root_specs(
|
|
env_var="PSINSAR_PRODUCT_DIR",
|
|
path=settings.PSINSAR_PRODUCT_DIR,
|
|
root_role="publish_root_psinsar",
|
|
display_name="PS-InSAR Publish Root",
|
|
scan_mode="manifest_tree",
|
|
owner_engine="psinsar",
|
|
)
|
|
)
|
|
specs.extend(
|
|
_iter_single_root_specs(
|
|
env_var="RESULT_QUARANTINE_ROOT",
|
|
path=settings.RESULT_QUARANTINE_ROOT,
|
|
root_role="quarantine_root",
|
|
display_name="Result Quarantine Root",
|
|
scan_mode="quarantine",
|
|
)
|
|
)
|
|
specs.extend(
|
|
_iter_single_root_specs(
|
|
env_var="ISCE2_WORK_ROOT",
|
|
path=settings.ISCE2_WORK_ROOT,
|
|
root_role="work_root_isce2",
|
|
display_name="ISCE2 Work Root",
|
|
scan_mode="workspace",
|
|
owner_engine="isce2",
|
|
)
|
|
)
|
|
specs.extend(
|
|
_iter_single_root_specs(
|
|
env_var="TIMESERIES_WORK_ROOT",
|
|
path=settings.TIMESERIES_WORK_ROOT,
|
|
root_role="work_root_timeseries",
|
|
display_name="Timeseries Work Root",
|
|
scan_mode="workspace",
|
|
owner_engine="timeseries",
|
|
)
|
|
)
|
|
specs.extend(
|
|
_iter_single_root_specs(
|
|
env_var="IDL_WORKER_RUNTIME_DIR",
|
|
path=settings.IDL_WORKER_RUNTIME_DIR,
|
|
root_role="work_root_idl",
|
|
display_name="IDL Runtime Root",
|
|
scan_mode="workspace",
|
|
owner_engine="idl",
|
|
)
|
|
)
|
|
return specs
|
|
|
|
|
|
class RootRegistryService:
|
|
async def _ensure_default_cursor(self, db: AsyncSession, root: ManagedRootORM) -> Optional[str]:
|
|
cursor_result = await db.execute(
|
|
select(ScanCursorORM).where(
|
|
ScanCursorORM.root_ref_id == root.id,
|
|
ScanCursorORM.cursor_key == "default",
|
|
)
|
|
)
|
|
cursor = cursor_result.scalar_one_or_none()
|
|
desired_cursor_type = _cursor_type_for_scan_mode(root.scan_mode)
|
|
if cursor is None:
|
|
cursor = ScanCursorORM(
|
|
root_ref_id=root.id,
|
|
cursor_key="default",
|
|
cursor_type=desired_cursor_type,
|
|
scan_scope="root",
|
|
status="IDLE",
|
|
)
|
|
db.add(cursor)
|
|
return "created"
|
|
|
|
changed = False
|
|
if cursor.cursor_type != desired_cursor_type:
|
|
cursor.cursor_type = desired_cursor_type
|
|
changed = True
|
|
if cursor.scan_scope != "root":
|
|
cursor.scan_scope = "root"
|
|
changed = True
|
|
return "updated" if changed else None
|
|
|
|
async def sync_from_settings(self, db: Optional[AsyncSession] = None) -> Dict[str, Any]:
|
|
generated_session = db is None
|
|
if generated_session:
|
|
db = _new_session()
|
|
|
|
assert db is not None
|
|
|
|
try:
|
|
specs = _build_root_specs_from_settings()
|
|
existing_result = await db.execute(
|
|
select(ManagedRootORM).where(ManagedRootORM.source_kind == "env")
|
|
)
|
|
existing_rows = existing_result.scalars().all()
|
|
existing_by_code = {row.root_code: row for row in existing_rows}
|
|
|
|
created = 0
|
|
updated = 0
|
|
disabled = 0
|
|
cursor_created = 0
|
|
cursor_updated = 0
|
|
synced_codes: set[str] = set()
|
|
|
|
for spec in specs:
|
|
synced_codes.add(spec.root_code)
|
|
exists_flag = os.path.exists(spec.path)
|
|
row = existing_by_code.get(spec.root_code)
|
|
|
|
if row is None:
|
|
row = ManagedRootORM(
|
|
root_code=spec.root_code,
|
|
root_role=spec.root_role,
|
|
display_name=spec.display_name,
|
|
path=spec.path,
|
|
path_kind=spec.path_kind,
|
|
source_kind=spec.source_kind,
|
|
source_ref=spec.source_ref,
|
|
scan_mode=spec.scan_mode,
|
|
owner_engine=spec.owner_engine,
|
|
enabled=True,
|
|
exists_flag=exists_flag,
|
|
metadata_json=spec.metadata_json,
|
|
)
|
|
db.add(row)
|
|
await db.flush()
|
|
created += 1
|
|
else:
|
|
changed = False
|
|
for field_name, value in (
|
|
("root_role", spec.root_role),
|
|
("display_name", spec.display_name),
|
|
("path", spec.path),
|
|
("path_kind", spec.path_kind),
|
|
("source_kind", spec.source_kind),
|
|
("source_ref", spec.source_ref),
|
|
("scan_mode", spec.scan_mode),
|
|
("owner_engine", spec.owner_engine),
|
|
("exists_flag", exists_flag),
|
|
("metadata_json", spec.metadata_json),
|
|
):
|
|
if getattr(row, field_name) != value:
|
|
setattr(row, field_name, value)
|
|
changed = True
|
|
if not row.enabled:
|
|
row.enabled = True
|
|
changed = True
|
|
if changed:
|
|
updated += 1
|
|
|
|
cursor_change = await self._ensure_default_cursor(db, row)
|
|
if cursor_change == "created":
|
|
cursor_created += 1
|
|
elif cursor_change == "updated":
|
|
cursor_updated += 1
|
|
|
|
for row in existing_rows:
|
|
if row.root_code in synced_codes:
|
|
continue
|
|
if row.enabled:
|
|
row.enabled = False
|
|
disabled += 1
|
|
|
|
await db.commit()
|
|
|
|
summary = await self.get_summary(db)
|
|
return {
|
|
"synced": len(specs),
|
|
"created": created,
|
|
"updated": updated,
|
|
"disabled": disabled,
|
|
"cursor_created_or_updated": cursor_created + cursor_updated,
|
|
"summary": summary,
|
|
}
|
|
except Exception:
|
|
await db.rollback()
|
|
raise
|
|
finally:
|
|
if generated_session:
|
|
await db.close()
|
|
|
|
async def list_roots(self, db: AsyncSession, *, include_disabled: bool = False) -> List[Dict[str, Any]]:
|
|
stmt = select(ManagedRootORM).order_by(
|
|
ManagedRootORM.root_role.asc(),
|
|
ManagedRootORM.display_name.asc(),
|
|
ManagedRootORM.id.asc(),
|
|
)
|
|
if not include_disabled:
|
|
stmt = stmt.where(ManagedRootORM.enabled == True) # noqa: E712
|
|
root_result = await db.execute(stmt)
|
|
roots = root_result.scalars().all()
|
|
root_ids = [root.id for root in roots]
|
|
|
|
cursor_by_root_id: Dict[int, ScanCursorORM] = {}
|
|
inventory_counts: Dict[int, int] = {}
|
|
if root_ids:
|
|
cursor_result = await db.execute(
|
|
select(ScanCursorORM)
|
|
.where(ScanCursorORM.root_ref_id.in_(root_ids))
|
|
.where(ScanCursorORM.cursor_key == "default")
|
|
)
|
|
cursor_by_root_id = {
|
|
item.root_ref_id: item
|
|
for item in cursor_result.scalars().all()
|
|
}
|
|
|
|
inventory_result = await db.execute(
|
|
select(PathInventoryORM.root_ref_id, func.count(PathInventoryORM.id))
|
|
.where(PathInventoryORM.root_ref_id.in_(root_ids))
|
|
.where(PathInventoryORM.status != "REMOVED")
|
|
.group_by(PathInventoryORM.root_ref_id)
|
|
)
|
|
inventory_counts = {
|
|
int(root_ref_id): int(count or 0)
|
|
for root_ref_id, count in inventory_result.all()
|
|
}
|
|
|
|
payload: List[Dict[str, Any]] = []
|
|
for root in roots:
|
|
cursor = cursor_by_root_id.get(root.id)
|
|
payload.append(
|
|
{
|
|
"id": root.id,
|
|
"root_code": root.root_code,
|
|
"root_role": root.root_role,
|
|
"display_name": root.display_name,
|
|
"path": root.path,
|
|
"path_kind": root.path_kind,
|
|
"source_kind": root.source_kind,
|
|
"source_ref": root.source_ref,
|
|
"scan_mode": root.scan_mode,
|
|
"owner_engine": root.owner_engine,
|
|
"enabled": bool(root.enabled),
|
|
"exists_flag": bool(root.exists_flag),
|
|
"metadata_json": root.metadata_json,
|
|
"created_at": root.created_at,
|
|
"updated_at": root.updated_at,
|
|
"inventory_count": inventory_counts.get(root.id, 0),
|
|
"cursor": (
|
|
{
|
|
"id": cursor.id,
|
|
"cursor_key": cursor.cursor_key,
|
|
"cursor_type": cursor.cursor_type,
|
|
"scan_scope": cursor.scan_scope,
|
|
"status": cursor.status,
|
|
"last_scan_started_at": cursor.last_scan_started_at,
|
|
"last_scan_finished_at": cursor.last_scan_finished_at,
|
|
"last_seen_mtime": cursor.last_seen_mtime,
|
|
"last_seen_entry_count": cursor.last_seen_entry_count,
|
|
"last_seen_fingerprint": cursor.last_seen_fingerprint,
|
|
"last_error": cursor.last_error,
|
|
"updated_at": cursor.updated_at,
|
|
}
|
|
if cursor is not None
|
|
else None
|
|
),
|
|
}
|
|
)
|
|
return payload
|
|
|
|
async def get_summary(self, db: AsyncSession) -> Dict[str, Any]:
|
|
total_roots = int(
|
|
(await db.execute(select(func.count(ManagedRootORM.id)))).scalar_one() or 0
|
|
)
|
|
enabled_roots = int(
|
|
(
|
|
await db.execute(
|
|
select(func.count(ManagedRootORM.id)).where(ManagedRootORM.enabled == True) # noqa: E712
|
|
)
|
|
).scalar_one()
|
|
or 0
|
|
)
|
|
existing_roots = int(
|
|
(
|
|
await db.execute(
|
|
select(func.count(ManagedRootORM.id)).where(ManagedRootORM.exists_flag == True) # noqa: E712
|
|
)
|
|
).scalar_one()
|
|
or 0
|
|
)
|
|
total_cursors = int(
|
|
(await db.execute(select(func.count(ScanCursorORM.id)))).scalar_one() or 0
|
|
)
|
|
total_inventory = int(
|
|
(
|
|
await db.execute(
|
|
select(func.count(PathInventoryORM.id)).where(PathInventoryORM.status != "REMOVED")
|
|
)
|
|
).scalar_one()
|
|
or 0
|
|
)
|
|
|
|
role_rows = await db.execute(
|
|
select(
|
|
ManagedRootORM.root_role,
|
|
func.count(ManagedRootORM.id),
|
|
)
|
|
.group_by(ManagedRootORM.root_role)
|
|
.order_by(ManagedRootORM.root_role.asc())
|
|
)
|
|
role_counts = {
|
|
str(role): int(count or 0)
|
|
for role, count in role_rows.all()
|
|
}
|
|
|
|
return {
|
|
"total_roots": total_roots,
|
|
"enabled_roots": enabled_roots,
|
|
"existing_roots": existing_roots,
|
|
"missing_roots": max(0, enabled_roots - existing_roots),
|
|
"total_cursors": total_cursors,
|
|
"total_inventory_items": total_inventory,
|
|
"role_counts": role_counts,
|
|
}
|
|
|
|
async def get_status(self, db: AsyncSession, *, include_disabled: bool = False) -> Dict[str, Any]:
|
|
roots = await self.list_roots(db, include_disabled=include_disabled)
|
|
summary = await self.get_summary(db)
|
|
return {
|
|
"summary": summary,
|
|
"roots": roots,
|
|
}
|
|
|
|
|
|
root_registry_service = RootRegistryService()
|