from __future__ import annotations import os from typing import Any, Dict, List, Optional, Set from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from ..models import ResultAssetORM, ResultProductORM from .dinsar_engine_matrix import is_current_dinsar_engine DINSAR_CATALOG_NAME = "dinsar" PRESERVED_DIR_NAMES = {"assets", "preview", "current"} PRESERVED_FILE_NAMES = { "manifest.json", "execution_manifest.json", ".dinsar_run.json", ".dinsar_pair.json", "task_manifest.json", } def _norm_path(value: Any) -> str: text = str(value or "").strip() if not text: return "" return os.path.normcase(os.path.normpath(os.path.abspath(text))) def _is_within(child: str, parent: str) -> bool: child_norm = _norm_path(child) parent_norm = _norm_path(parent) if not child_norm or not parent_norm: return False try: return os.path.commonpath([child_norm, parent_norm]) == parent_norm except ValueError: return False def _path_size(path: str) -> int: if not os.path.exists(path): return 0 if os.path.isfile(path): try: return int(os.path.getsize(path) or 0) except OSError: return 0 total = 0 for root, _dirs, files in os.walk(path): for name in files: file_path = os.path.join(root, name) try: total += int(os.path.getsize(file_path) or 0) except OSError: continue return total def _required_assets_ok(assets: List[ResultAssetORM]) -> tuple[bool, List[Dict[str, Any]]]: missing: List[Dict[str, Any]] = [] for asset in assets: if not asset.is_required: continue path = _norm_path(asset.absolute_path) exists = bool(path and os.path.exists(path)) if not exists: missing.append( { "asset_id": asset.id, "asset_role": asset.asset_role, "asset_name": asset.asset_name, "absolute_path": asset.absolute_path, } ) return len(missing) == 0, missing def _is_preserved_path(path: str, product: ResultProductORM, asset_paths: Set[str]) -> bool: normalized = _norm_path(path) if not normalized: return True if normalized in asset_paths: return True if any(_is_within(asset_path, normalized) for asset_path in asset_paths): return True for preserve in [ product.publish_dir, product.manifest_path, product.preview_path, product.primary_asset_path, ]: preserve_norm = _norm_path(preserve) if preserve_norm and normalized == preserve_norm: return True if preserve_norm and _is_within(preserve_norm, normalized): return True name = os.path.basename(normalized) if name in PRESERVED_FILE_NAMES or name in PRESERVED_DIR_NAMES: return True return False class DinsarIntermediateCleanupService: async def build_product_plan( self, db: AsyncSession, *, product_db_id: int, ) -> Optional[Dict[str, Any]]: product_result = await db.execute( select(ResultProductORM).where( ResultProductORM.id == product_db_id, ResultProductORM.catalog_name == DINSAR_CATALOG_NAME, ) ) product = product_result.scalar_one_or_none() if product is None: return None asset_result = await db.execute( select(ResultAssetORM) .where(ResultAssetORM.product_ref_id == product.id) .order_by(ResultAssetORM.asset_role.asc(), ResultAssetORM.id.asc()) ) assets = asset_result.scalars().all() asset_paths = {_norm_path(asset.absolute_path) for asset in assets if _norm_path(asset.absolute_path)} required_ok, missing_required_assets = _required_assets_ok(assets) manifest_exists = bool(product.manifest_path and os.path.isfile(_norm_path(product.manifest_path))) current_engine = is_current_dinsar_engine(product.engine_code) blockers: List[str] = [] if not current_engine: blockers.append("legacy_or_unknown_engine") if not manifest_exists: blockers.append("manifest_missing") if not required_ok: blockers.append("required_assets_missing") candidates: List[Dict[str, Any]] = [] seen: Set[str] = set() def add_candidate(path: Any, reason: str) -> None: normalized = _norm_path(path) if not normalized or normalized in seen: return seen.add(normalized) if _is_preserved_path(normalized, product, asset_paths): return publish_dir = _norm_path(product.publish_dir) native_output_dir = _norm_path(product.native_output_dir) if publish_dir and normalized == publish_dir: return if publish_dir and not _is_within(normalized, publish_dir) and normalized != native_output_dir: blockers.append(f"candidate_outside_publish_dir:{normalized}") return candidates.append( { "path": normalized, "reason": reason, "exists": os.path.exists(normalized), "is_dir": os.path.isdir(normalized), "size_bytes": _path_size(normalized), } ) native_output_dir = _norm_path(product.native_output_dir) if native_output_dir: add_candidate(native_output_dir, "native_output_dir") publish_dir = _norm_path(product.publish_dir) if publish_dir: add_candidate(os.path.join(publish_dir, "native"), "managed_native_dir") for name in ("work", "tmp", "temp", "intermediate", "landsar_input", "landsar_output"): add_candidate(os.path.join(publish_dir, name), f"managed_{name}_dir") deletable = len(blockers) == 0 total_size = sum(int(item.get("size_bytes") or 0) for item in candidates if item.get("exists")) return { "schema": "insar.dinsar-intermediate-cleanup-plan/v1", "dry_run": True, "deletable": deletable, "product": { "id": product.id, "product_id": product.product_id, "pair_key": product.pair_key, "run_key": product.run_key, "engine_code": product.engine_code, "profile_code": product.profile_code, "publish_dir": product.publish_dir, "manifest_path": product.manifest_path, "native_output_dir": product.native_output_dir, }, "checks": { "manifest_exists": manifest_exists, "required_assets_ok": required_ok, "current_engine": current_engine, "missing_required_assets": missing_required_assets, }, "blockers": blockers, "candidates": candidates, "candidate_count": len(candidates), "total_size_bytes": total_size, "preserve": { "directories": sorted(PRESERVED_DIR_NAMES), "files": sorted(PRESERVED_FILE_NAMES), "asset_count": len(asset_paths), }, } async def build_pair_plan( self, db: AsyncSession, *, pair_key: str, ) -> Dict[str, Any]: normalized_pair_key = str(pair_key or "").strip() result = await db.execute( select(ResultProductORM) .where( ResultProductORM.catalog_name == DINSAR_CATALOG_NAME, ResultProductORM.pair_key == normalized_pair_key, ) .order_by(ResultProductORM.published_at.desc().nullslast(), ResultProductORM.id.desc()) ) products = result.scalars().all() product_plans: List[Dict[str, Any]] = [] for product in products: plan = await self.build_product_plan(db, product_db_id=int(product.id)) if plan is not None: product_plans.append(plan) return { "schema": "insar.dinsar-intermediate-cleanup-pair-plan/v1", "dry_run": True, "pair_key": normalized_pair_key, "product_count": len(product_plans), "deletable_product_count": sum(1 for item in product_plans if item.get("deletable")), "total_size_bytes": sum(int(item.get("total_size_bytes") or 0) for item in product_plans), "products": product_plans, } dinsar_intermediate_cleanup_service = DinsarIntermediateCleanupService()