from __future__ import annotations import hashlib import json import os import re import shutil import zipfile from dataclasses import dataclass from datetime import datetime, timedelta from pathlib import Path from typing import Any, Dict, Iterable, List, Optional from sqlalchemy import func, or_, select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import selectinload from .. import database from ..config import settings from ..models import ( AuthUserORM, DinsarResultORM, RadarDataORM, ResultAssetORM, ResultDeliveryItemORM, ResultDeliveryRequestORM, ResultProductORM, SARSceneGeoORM, ) from .dinsar_read_service import dinsar_read_service from .job_queue_service import job_queue_service from .task_service import task_service JOB_TYPE_RESULT_DELIVERY_BUILD = "RESULT_DELIVERY_BUILD" TASK_TYPE_RESULT_DELIVERY_BUILD = "RESULT_DELIVERY_BUILD" DELIVERY_STATUS_PENDING = "PENDING" DELIVERY_STATUS_RUNNING = "RUNNING" DELIVERY_STATUS_READY = "READY" DELIVERY_STATUS_FAILED = "FAILED" DELIVERY_STATUS_CANCELLED = "CANCELLED" DELIVERY_STATUS_EXPIRED = "EXPIRED" ITEM_STATUS_PENDING = "PENDING" ITEM_STATUS_COPIED = "COPIED" ITEM_STATUS_FAILED = "FAILED" ITEM_STATUS_SKIPPED = "SKIPPED" CHANNEL_DINSAR = "dinsar" CHANNEL_LT1_ORTHO = "lt1_ortho" CHANNEL_GF3_ORTHO = "gf3_ortho" CHANNEL_SBAS = "sbas" CHANNEL_S1_ORTHO = "s1_ortho" SUPPORTED_READY_CHANNELS = {CHANNEL_DINSAR, CHANNEL_LT1_ORTHO, CHANNEL_GF3_ORTHO} LT1_ANALYSIS_ENGINE = "lt_gamma" LT1_ANALYSIS_PROFILE = "lt1_gamma_geocoded_mli" GF3_STANDARD_SOURCE_FORMAT = "GF3_SARSCAPE_L2" GF3_NATIVE_PREVIEW_SOURCE_FORMAT = "GF3_SARSCAPE_NATIVE_PREVIEW" GF3_DELIVERABLE_SOURCE_FORMATS = {GF3_STANDARD_SOURCE_FORMAT, GF3_NATIVE_PREVIEW_SOURCE_FORMAT} PACKAGE_MODE_DIRECTORY = "directory" PACKAGE_MODE_ZIP = "zip" _ASSOCIATED_EXTENSIONS = ("", ".hdr", ".sml", ".xml", ".aux.xml", ".prj") _INVALID_SEGMENT_RE = re.compile(r'[<>:"/\\|?*\x00-\x1F]+') _SAFE_ID_RE = re.compile(r"[^0-9A-Za-z_-]+") @dataclass(frozen=True) class DeliverySource: display_name: str source_path: str product: Optional[ResultProductORM] = None compat_row: Optional[DinsarResultORM] = None source_asset_id: Optional[int] = None source_radar_data_id: Optional[int] = None source_scene_geo_id: Optional[int] = None product_id: Optional[str] = None task_name: Optional[str] = None source_kind: str = "result_product" def _new_session() -> AsyncSession: if database.AsyncSessionLocal is None: database.init_db() if database.AsyncSessionLocal is None: raise RuntimeError("Database session factory is not initialized.") return database.AsyncSessionLocal() def _utcnow() -> datetime: return datetime.utcnow() def _normalize_channel(value: str) -> str: channel = str(value or "").strip().lower() if not channel: raise ValueError("channel is required") return channel def _normalize_package_mode(value: str) -> str: mode = str(value or PACKAGE_MODE_DIRECTORY).strip().lower() if mode not in {PACKAGE_MODE_DIRECTORY, PACKAGE_MODE_ZIP}: raise ValueError("package_mode must be directory or zip") return mode def _normalize_int_ids(values: Optional[Iterable[Any]], *, max_count: int) -> List[int]: normalized: List[int] = [] seen = set() for raw in values or []: try: value = int(raw) except (TypeError, ValueError): continue if value <= 0 or value in seen: continue seen.add(value) normalized.append(value) if len(normalized) > max_count: raise ValueError(f"selected item count exceeds max limit ({max_count})") return normalized def _sanitize_segment(value: Any, *, default: str = "item", max_len: int = 120) -> str: text = str(value or "").strip() text = _INVALID_SEGMENT_RE.sub("_", text).strip(" .") text = _SAFE_ID_RE.sub("_", text) return (text or default)[:max_len] def _same_file_size(left: str, right: str) -> bool: try: return os.path.getsize(left) == os.path.getsize(right) except OSError: return False def _file_size(path: str) -> int: try: return int(os.path.getsize(path)) except OSError: return 0 def _sha256_file(path: str) -> str: digest = hashlib.sha256() with open(path, "rb") as stream: for chunk in iter(lambda: stream.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def _write_json(path: str, payload: Dict[str, Any]) -> None: os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "w", encoding="utf-8") as stream: json.dump(payload, stream, ensure_ascii=False, indent=2, default=str) def _path_within(parent: str, child: str) -> bool: parent_path = Path(parent).resolve() child_path = Path(child).resolve() try: child_path.relative_to(parent_path) return True except ValueError: return False def _iter_associated_files(source_path: str) -> List[str]: source = os.path.normpath(source_path) if os.path.isdir(source): files: List[str] = [] for root, _dirs, names in os.walk(source): for name in names: files.append(os.path.join(root, name)) return sorted(files) if not os.path.isfile(source): return [] src_dir = os.path.dirname(source) base_name = os.path.basename(source) files: List[str] = [] for ext in _ASSOCIATED_EXTENSIONS: candidate = os.path.join(src_dir, base_name + ext) if ext else source if os.path.isfile(candidate) and candidate not in files: files.append(candidate) return files def _delivery_root() -> str: root = str(settings.RESULT_DELIVERY_ROOT or "").strip() if not root: raise ValueError("RESULT_DELIVERY_ROOT is not configured") return os.path.normpath(os.path.abspath(root)) def _owner_dir(root: str, username: str) -> str: return os.path.join(root, _sanitize_segment(username, default="user", max_len=64)) def _make_delivery_id(username: str) -> str: stamp = _utcnow().strftime("%Y%m%d%H%M%S") digest = hashlib.sha1(f"{username}|{stamp}|{os.urandom(8).hex()}".encode("utf-8")).hexdigest()[:10] return f"rd_{stamp}_{digest}" def _resolve_source_path(product: ResultProductORM, assets: List[ResultAssetORM]) -> tuple[str, Optional[int]]: primary_assets = [ asset for asset in assets if asset.exists_flag and (asset.is_primary or str(asset.asset_role or "").lower() in {"disp", "primary_geotiff"}) ] primary_assets.sort(key=lambda item: (not item.is_primary, item.id)) for asset in primary_assets: path = str(asset.absolute_path or "").strip() if path: return path, asset.id for path in (product.primary_asset_path, product.source_primary_path, product.publish_dir): text = str(path or "").strip() if text: return text, None return "", None def _scene_product_id(scene: SARSceneGeoORM) -> str: return f"sar_scene_geo:{scene.id}" def _scene_display_name(scene: SARSceneGeoORM, radar: RadarDataORM) -> str: for value in ( radar.product_unique_id, radar.unique_id, radar.source_product_token, os.path.basename(str(radar.file_path or "").rstrip("/\\")), _scene_product_id(scene), ): text = str(value or "").strip() if text: return text return f"scene_{scene.id}" def _radar_display_name(radar: RadarDataORM) -> str: for value in ( radar.product_unique_id, radar.unique_id, radar.source_product_token, os.path.basename(str(radar.file_path or "").rstrip("/\\")), f"radar_{radar.id}", ): text = str(value or "").strip() if text: return text return f"radar_{radar.id}" def _manifest_path_for_scene(scene: SARSceneGeoORM) -> Optional[str]: metadata = scene.analysis_metadata_json if isinstance(scene.analysis_metadata_json, dict) else {} text = str(metadata.get("manifest_path") or "").strip() if text: return text if scene.analysis_dir: return os.path.join(scene.analysis_dir, "manifest.json") return None def _quality_path_for_scene(scene: SARSceneGeoORM) -> Optional[str]: if scene.analysis_dir: return os.path.join(scene.analysis_dir, "quality.json") return None def _catalog_file_size(path: Any) -> int: text = str(path or "").strip() return _file_size(text) if text else 0 class ResultDeliveryService: def channels(self) -> List[Dict[str, Any]]: return [ { "key": CHANNEL_DINSAR, "group": "InSAR 成果", "label": "D-InSAR 结果", "state": "ready", "state_text": "可交付", "description": "已登记 D-InSAR catalog，可创建后台交付包并下载到本地。", }, { "key": CHANNEL_SBAS, "group": "InSAR 成果", "label": "SBAS-InSAR 结果", "state": "planned", "state_text": "目录可查", "description": "SBAS 结果 catalog 已有基础能力，本阶段暂不开放交付打包。", }, { "key": CHANNEL_LT1_ORTHO, "group": "正射成果", "label": "LT-1 正射结果", "state": "ready", "state_text": "可交付", "description": "服务器生产的 LT-1 分析就绪正射 GeoTIFF 已接入交付，可打包下载到本地。", }, { "key": CHANNEL_S1_ORTHO, "group": "正射成果", "label": "Sentinel-1 正射结果", "state": "placeholder", "state_text": "待接入", "description": "Sentinel-1 正射生产尚未接入，当前只保留交付通道占位。", }, { "key": CHANNEL_GF3_ORTHO, "group": "正射成果", "label": "GF3 SARscape _geo", "state": "ready", "state_text": "可交付", "description": "已登记的 GF3 SARscape 标准化正射成品可直接创建交付包。", }, ] async def _resolve_dinsar_sources( self, db: AsyncSession, *, product_ids: List[int], compat_result_ids: List[int], ) -> List[DeliverySource]: sources: List[DeliverySource] = [] seen_products = set() if product_ids: result = await db.execute( select(ResultProductORM) .options(selectinload(ResultProductORM.assets)) .where( ResultProductORM.id.in_(product_ids), ResultProductORM.catalog_name == CHANNEL_DINSAR, ResultProductORM.status.in_(["READY", "PUBLISHED", "OK"]), ) .order_by(ResultProductORM.id.asc()) ) for product in result.scalars().unique().all(): assets = list(product.assets or []) source_path, source_asset_id = _resolve_source_path(product, assets) if not source_path: continue seen_products.add(int(product.id)) sources.append( DeliverySource( display_name=dinsar_read_service.get_display_name(product), source_path=source_path, product=product, compat_row=None, source_asset_id=source_asset_id, product_id=product.product_id, task_name=product.task_alias or product.task_name, source_kind="dinsar_product", ) ) if compat_result_ids: records = await dinsar_read_service.list_compat_records_by_ids( db, compat_result_ids=compat_result_ids, ) product_lookup_ids = [record.product.id for record in records if record.product is not None] assets_by_product: Dict[int, List[ResultAssetORM]] = {} if product_lookup_ids: asset_result = await db.execute( select(ResultAssetORM) .where(ResultAssetORM.product_ref_id.in_(product_lookup_ids)) .order_by(ResultAssetORM.id.asc()) ) for asset in asset_result.scalars().all(): assets_by_product.setdefault(int(asset.product_ref_id), []).append(asset) for record in records: product = record.product if int(product.id) in seen_products: continue assets = assets_by_product.get(int(product.id), []) source_path, source_asset_id = _resolve_source_path(product, assets) if not source_path and record.compat_row is not None: source_path = str(record.compat_row.file_path or "").strip() if not source_path: continue seen_products.add(int(product.id)) sources.append( DeliverySource( display_name=record.display_name, source_path=source_path, product=product, compat_row=record.compat_row, source_asset_id=source_asset_id, product_id=product.product_id, task_name=product.task_alias or product.task_name, source_kind="dinsar_product", ) ) return sources async def _resolve_lt1_sources( self, db: AsyncSession, *, item_ids: List[int], product_ids: List[int], ) -> List[DeliverySource]: selected_ids = item_ids or product_ids if not selected_ids: return [] result = await db.execute( select(SARSceneGeoORM, RadarDataORM) .join(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id) .where( SARSceneGeoORM.id.in_(selected_ids), SARSceneGeoORM.status == "DONE", SARSceneGeoORM.analysis_tif_path.isnot(None), SARSceneGeoORM.analysis_engine == LT1_ANALYSIS_ENGINE, SARSceneGeoORM.analysis_profile == LT1_ANALYSIS_PROFILE, ) .order_by(SARSceneGeoORM.id.asc()) ) sources: List[DeliverySource] = [] for scene, radar in result.all(): analysis_dir = str(scene.analysis_dir or "").strip() analysis_tif = str(scene.analysis_tif_path or "").strip() source_path = analysis_dir if analysis_dir and os.path.isdir(analysis_dir) else analysis_tif if not source_path: continue product_id = _scene_product_id(scene) display_name = _scene_display_name(scene, radar) sources.append( DeliverySource( display_name=display_name, source_path=source_path, source_radar_data_id=int(radar.id) if radar.id is not None else None, source_scene_geo_id=int(scene.id) if scene.id is not None else None, product_id=product_id, task_name=display_name, source_kind="lt1_scene_geo", ) ) return sources async def _resolve_gf3_sources( self, db: AsyncSession, *, item_ids: List[int], ) -> List[DeliverySource]: if not item_ids: return [] result = await db.execute( select(RadarDataORM, SARSceneGeoORM) .outerjoin(SARSceneGeoORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id) .where( RadarDataORM.id.in_(item_ids), RadarDataORM.source_format.in_(GF3_DELIVERABLE_SOURCE_FORMATS), RadarDataORM.geocoded_flag.is_(True), ) .order_by(RadarDataORM.id.asc()) ) sources: List[DeliverySource] = [] for radar, scene in result.all(): source_path = "" source_scene_geo_id = None if scene is not None and scene.analysis_tif_path: analysis_dir = str(scene.analysis_dir or "").strip() analysis_tif = str(scene.analysis_tif_path or "").strip() source_path = analysis_dir if analysis_dir and os.path.isdir(analysis_dir) else analysis_tif source_scene_geo_id = int(scene.id) if scene.id is not None else None if not source_path: source_path = str(radar.file_path or "").strip() if not source_path: continue display_name = _radar_display_name(radar) sources.append( DeliverySource( display_name=display_name, source_path=source_path, source_radar_data_id=int(radar.id) if radar.id is not None else None, source_scene_geo_id=source_scene_geo_id, product_id=f"gf3_sarscape:{radar.id}", task_name=display_name, source_kind="gf3_radar_data", ) ) return sources async def _resolve_sources_for_channel( self, db: AsyncSession, *, channel: str, product_ids: List[int], compat_result_ids: List[int], item_ids: List[int], ) -> List[DeliverySource]: if channel == CHANNEL_DINSAR: return await self._resolve_dinsar_sources( db, product_ids=product_ids, compat_result_ids=compat_result_ids, ) if channel == CHANNEL_LT1_ORTHO: return await self._resolve_lt1_sources(db, item_ids=item_ids, product_ids=product_ids) if channel == CHANNEL_GF3_ORTHO: return await self._resolve_gf3_sources(db, item_ids=item_ids or product_ids) return [] async def list_channel_catalog( self, db: AsyncSession, *, channel: str, limit: int = 100, offset: int = 0, query: Optional[str] = None, ) -> Dict[str, Any]: channel = _normalize_channel(channel) safe_limit = min(500, max(1, int(limit or 100))) safe_offset = max(0, int(offset or 0)) query_text = str(query or "").strip() if channel == CHANNEL_LT1_ORTHO: filters = [ SARSceneGeoORM.status == "DONE", SARSceneGeoORM.analysis_tif_path.isnot(None), SARSceneGeoORM.analysis_engine == LT1_ANALYSIS_ENGINE, SARSceneGeoORM.analysis_profile == LT1_ANALYSIS_PROFILE, ] if query_text: like = f"%{query_text}%" filters.append( or_( RadarDataORM.product_unique_id.ilike(like), RadarDataORM.unique_id.ilike(like), RadarDataORM.file_path.ilike(like), RadarDataORM.imaging_date.ilike(like), ) ) total_result = await db.execute( select(func.count(SARSceneGeoORM.id)) .join(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id) .where(*filters) ) rows_result = await db.execute( select(SARSceneGeoORM, RadarDataORM) .join(RadarDataORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id) .where(*filters) .order_by(SARSceneGeoORM.updated_at.desc().nullslast(), SARSceneGeoORM.id.desc()) .limit(safe_limit) .offset(safe_offset) ) items = [] for scene, radar in rows_result.all(): display_name = _scene_display_name(scene, radar) manifest_path = _manifest_path_for_scene(scene) quality_path = _quality_path_for_scene(scene) items.append( { "id": scene.id, "item_id": scene.id, "source_kind": "lt1_scene_geo", "product_id": _scene_product_id(scene), "catalog_name": "sar_scene_geo", "product_family": "lt1_analysis_ready_geotiff", "product_type": "analysis_ready_geotiff", "display_name": display_name, "task_name": display_name, "status": scene.status, "health_status": "OK", "engine_code": scene.analysis_engine, "profile_code": scene.analysis_profile, "radar_data_id": radar.id, "scene_geo_id": scene.id, "imaging_date": radar.imaging_date, "polarization": radar.polarization, "pixel_size_m": scene.pixel_size_m, "backscatter_unit": scene.analysis_backscatter_unit, "publish_dir": scene.analysis_dir, "manifest_path": manifest_path, "quality_path": quality_path, "primary_asset_path": scene.analysis_tif_path, "file_size": _catalog_file_size(scene.analysis_tif_path), "summary": { "radar_data_id": radar.id, "source_asset_ids": [radar.source_product_ref_id] if radar.source_product_ref_id else [], "imaging_date": radar.imaging_date, "polarization": radar.polarization, "pixel_size_m": scene.pixel_size_m, }, "produced_at": scene.updated_at.isoformat() if scene.updated_at else None, "published_at": scene.updated_at.isoformat() if scene.updated_at else None, } ) return { "items": items, "total": int(total_result.scalar_one() or 0), "limit": safe_limit, "offset": safe_offset, } if channel == CHANNEL_GF3_ORTHO: filters = [ RadarDataORM.source_format.in_(GF3_DELIVERABLE_SOURCE_FORMATS), RadarDataORM.geocoded_flag.is_(True), ] if query_text: like = f"%{query_text}%" filters.append( or_( RadarDataORM.product_unique_id.ilike(like), RadarDataORM.unique_id.ilike(like), RadarDataORM.file_path.ilike(like), RadarDataORM.imaging_date.ilike(like), ) ) total_result = await db.execute(select(func.count(RadarDataORM.id)).where(*filters)) rows_result = await db.execute( select(RadarDataORM, SARSceneGeoORM) .outerjoin(SARSceneGeoORM, SARSceneGeoORM.radar_data_id == RadarDataORM.id) .where(*filters) .order_by(RadarDataORM.acquisition_start_time_utc.desc().nullslast(), RadarDataORM.id.desc()) .limit(safe_limit) .offset(safe_offset) ) items = [] for radar, scene in rows_result.all(): display_name = _radar_display_name(radar) primary_path = ( str(scene.analysis_tif_path or "").strip() if scene is not None and scene.analysis_tif_path else str(radar.file_path or "").strip() ) publish_dir = ( str(scene.analysis_dir or "").strip() if scene is not None and scene.analysis_dir else str(radar.file_path or "").strip() ) items.append( { "id": radar.id, "item_id": radar.id, "source_kind": "gf3_radar_data", "product_id": f"gf3_sarscape:{radar.id}", "catalog_name": "radar_data", "product_family": "gf3_ortho", "product_type": "sarscape_l2_geotiff" if radar.source_format == GF3_STANDARD_SOURCE_FORMAT else "sarscape_native_geo", "display_name": display_name, "task_name": display_name, "status": "READY", "health_status": "OK", "engine_code": (scene.analysis_engine if scene is not None else None) or "gf3_sarscape", "profile_code": ( (scene.analysis_profile if scene is not None else None) or ("gf3_standard_geotiff" if radar.source_format == GF3_STANDARD_SOURCE_FORMAT else "gf3_native_geo") ), "source_format": radar.source_format, "radar_data_id": radar.id, "scene_geo_id": scene.id if scene is not None else None, "imaging_date": radar.imaging_date, "polarization": radar.polarization, "pixel_size_m": scene.pixel_size_m if scene is not None else None, "backscatter_unit": scene.analysis_backscatter_unit if scene is not None else None, "publish_dir": publish_dir, "manifest_path": _manifest_path_for_scene(scene) if scene is not None else None, "primary_asset_path": primary_path, "file_size": _catalog_file_size(primary_path), "summary": { "radar_data_id": radar.id, "source_asset_ids": [radar.source_product_ref_id] if radar.source_product_ref_id else [], "imaging_date": radar.imaging_date, "polarization": radar.polarization, }, "produced_at": ( scene.updated_at.isoformat() if scene is not None and scene.updated_at else None ), "published_at": ( scene.updated_at.isoformat() if scene is not None and scene.updated_at else None ), } ) return { "items": items, "total": int(total_result.scalar_one() or 0), "limit": safe_limit, "offset": safe_offset, } if channel == CHANNEL_DINSAR: total = await dinsar_read_service.count_compat_records(db) records = await dinsar_read_service.list_compat_records(db, limit=safe_limit, offset=safe_offset) items = [] for record in records: product = record.product compat_row = record.compat_row items.append( { "id": int(compat_row.id if compat_row is not None else product.id), "item_id": int(compat_row.id if compat_row is not None else product.id), "source_kind": "dinsar_result", "product_id": product.product_id, "catalog_name": product.catalog_name, "product_family": product.product_family, "product_type": product.product_type, "display_name": record.display_name, "task_name": product.task_alias or product.task_name, "status": product.status, "health_status": product.health_status, "engine_code": product.engine_code, "profile_code": product.profile_code, "pair_key": product.pair_key, "publish_dir": product.publish_dir, "manifest_path": product.manifest_path, "primary_asset_path": product.primary_asset_path or product.source_primary_path, "file_size": _catalog_file_size(product.primary_asset_path or product.source_primary_path), "produced_at": product.produced_at.isoformat() if product.produced_at else None, "published_at": product.published_at.isoformat() if product.published_at else None, } ) return {"items": items, "total": total, "limit": safe_limit, "offset": safe_offset} return {"items": [], "total": 0, "limit": safe_limit, "offset": safe_offset} async def create_delivery( self, db: AsyncSession, *, user: AuthUserORM, channel: str, product_ids: Optional[List[int]] = None, compat_result_ids: Optional[List[int]] = None, item_ids: Optional[List[int]] = None, package_mode: str = PACKAGE_MODE_DIRECTORY, include_checksums: Optional[bool] = None, ) -> ResultDeliveryRequestORM: channel = _normalize_channel(channel) if channel not in SUPPORTED_READY_CHANNELS: raise ValueError(f"{channel} delivery is not connected yet") package_mode = _normalize_package_mode(package_mode) max_items = max(1, int(settings.RESULT_DELIVERY_MAX_ITEMS or 500)) normalized_product_ids = _normalize_int_ids(product_ids, max_count=max_items) normalized_compat_ids = _normalize_int_ids(compat_result_ids, max_count=max_items) normalized_item_ids = _normalize_int_ids(item_ids, max_count=max_items) if not normalized_product_ids and not normalized_compat_ids and not normalized_item_ids: raise ValueError("select at least one result") if len(normalized_product_ids) + len(normalized_compat_ids) + len(normalized_item_ids) > max_items: raise ValueError(f"selected item count exceeds max limit ({max_items})") sources = await self._resolve_sources_for_channel( db, channel=channel, product_ids=normalized_product_ids, compat_result_ids=normalized_compat_ids, item_ids=normalized_item_ids, ) if not sources: raise ValueError("selected results do not have deliverable files") if len(sources) > max_items: raise ValueError(f"resolved result count exceeds max limit ({max_items})") delivery_root = _delivery_root() delivery_id = _make_delivery_id(str(user.username or "user")) delivery_dir = os.path.join(_owner_dir(delivery_root, str(user.username or "user")), delivery_id) expires_at = _utcnow() + timedelta(days=max(1, int(settings.RESULT_DELIVERY_RETENTION_DAYS or 7))) request_json = { "channel": channel, "product_ids": normalized_product_ids, "compat_result_ids": normalized_compat_ids, "item_ids": normalized_item_ids, "package_mode": package_mode, "include_checksums": ( bool(settings.RESULT_DELIVERY_CHECKSUM_ENABLED) if include_checksums is None else bool(include_checksums) ), } delivery = ResultDeliveryRequestORM( delivery_id=delivery_id, owner_user_id=user.id, owner_username=str(user.username or "unknown"), channel=channel, status=DELIVERY_STATUS_PENDING, package_mode=package_mode, item_count=len(sources), total_bytes=0, copied_bytes=0, delivery_root=delivery_root, delivery_dir=delivery_dir, zip_path=(f"{delivery_dir}.zip" if package_mode == PACKAGE_MODE_ZIP else None), expires_at=expires_at, request_json=request_json, summary_json={ "source_count": len(sources), "created_message": "Delivery package queued.", }, ) db.add(delivery) await db.flush() task_type = f"{TASK_TYPE_RESULT_DELIVERY_BUILD}_{delivery_id}" task_id = await task_service.create_task( task_type, f"成果交付包生成: {delivery_id}", params={ "delivery_id": delivery_id, "channel": channel, "item_count": len(sources), "package_mode": package_mode, }, db=db, ) job_id = await job_queue_service.create_job( JOB_TYPE_RESULT_DELIVERY_BUILD, payload={"delivery_id": delivery_id}, task_id=task_id, max_attempts=1, db=db, ) delivery.task_id = task_id delivery.job_id = job_id await db.flush() return delivery async def list_deliveries( self, db: AsyncSession, *, user: AuthUserORM, include_all: bool = False, include_items: bool = False, item_limit: Optional[int] = None, limit: int = 50, offset: int = 0, ) -> Dict[str, Any]: safe_limit = min(200, max(1, int(limit or 50))) safe_offset = max(0, int(offset or 0)) stmt = select(ResultDeliveryRequestORM) if include_items: stmt = stmt.options(selectinload(ResultDeliveryRequestORM.items)) count_stmt = select(func.count(ResultDeliveryRequestORM.id)) if not include_all: owner_filter = ResultDeliveryRequestORM.owner_user_id == user.id stmt = stmt.where(owner_filter) count_stmt = count_stmt.where(owner_filter) stmt = ( stmt.order_by(ResultDeliveryRequestORM.created_at.desc(), ResultDeliveryRequestORM.id.desc()) .offset(safe_offset) .limit(safe_limit) ) result = await db.execute(stmt) total_result = await db.execute(count_stmt) deliveries = result.scalars().unique().all() if include_items else result.scalars().all() items = [ self.serialize_delivery(item, include_items=include_items, item_limit=item_limit) for item in deliveries ] total = int(total_result.scalar_one() or 0) return { "items": items, "total": total, "limit": safe_limit, "offset": safe_offset, "has_more": safe_offset + len(items) < total, } async def get_delivery( self, db: AsyncSession, *, delivery_id: str, user: AuthUserORM, include_items: bool = True, ) -> Optional[ResultDeliveryRequestORM]: stmt = select(ResultDeliveryRequestORM).where( ResultDeliveryRequestORM.delivery_id == str(delivery_id or "").strip() ) if include_items: stmt = stmt.options(selectinload(ResultDeliveryRequestORM.items)) if str(user.role or "").lower() != "admin": stmt = stmt.where(ResultDeliveryRequestORM.owner_user_id == user.id) result = await db.execute(stmt) return result.scalars().unique().one_or_none() def serialize_delivery( self, delivery: ResultDeliveryRequestORM, *, include_items: bool = False, item_limit: Optional[int] = None, ) -> Dict[str, Any]: items = list(delivery.items or []) if include_items else [] if item_limit is not None: items = items[: max(0, int(item_limit))] summary = delivery.summary_json if isinstance(delivery.summary_json, dict) else {} payload = { "id": delivery.id, "delivery_id": delivery.delivery_id, "owner_user_id": delivery.owner_user_id, "owner_username": delivery.owner_username, "channel": delivery.channel, "status": delivery.status, "package_mode": delivery.package_mode, "item_count": delivery.item_count, "total_bytes": delivery.total_bytes, "copied_bytes": delivery.copied_bytes, "delivery_dir": delivery.delivery_dir, "zip_path": delivery.zip_path, "manifest_path": delivery.manifest_path, "expires_at": delivery.expires_at, "task_id": delivery.task_id, "job_id": delivery.job_id, "error_message": delivery.error_message, "summary": summary, "created_at": delivery.created_at, "updated_at": delivery.updated_at, "started_at": delivery.started_at, "completed_at": delivery.completed_at, "download_urls": { "manifest": f"/api/result-deliveries/{delivery.delivery_id}/manifest", "archive": ( f"/api/result-deliveries/{delivery.delivery_id}/archive/download" if delivery.zip_path else None ), }, } if include_items: payload["items"] = [self.serialize_item(item, delivery_id=delivery.delivery_id) for item in items] return payload def serialize_item(self, item: ResultDeliveryItemORM, *, delivery_id: str) -> Dict[str, Any]: return { "id": item.id, "delivery_id": item.delivery_id, "source_product_id": item.source_product_id, "source_result_id": item.source_result_id, "source_asset_id": item.source_asset_id, "source_radar_data_id": item.source_radar_data_id, "source_scene_geo_id": item.source_scene_geo_id, "display_name": item.display_name, "relative_path": item.relative_path, "file_size": item.file_size, "checksum_sha256": item.checksum_sha256, "status": item.status, "error_message": item.error_message, "download_url": ( f"/api/result-deliveries/{delivery_id}/files/{item.id}/download" if item.status == ITEM_STATUS_COPIED else None ), } async def build_delivery(self, delivery_id: str) -> Dict[str, Any]: async with _new_session() as db: result = await db.execute( select(ResultDeliveryRequestORM).where(ResultDeliveryRequestORM.delivery_id == delivery_id) ) delivery = result.scalar_one_or_none() if delivery is None: raise ValueError(f"delivery not found: {delivery_id}") if delivery.status in {DELIVERY_STATUS_READY, DELIVERY_STATUS_CANCELLED, DELIVERY_STATUS_EXPIRED}: return self.serialize_delivery(delivery) delivery.status = DELIVERY_STATUS_RUNNING delivery.started_at = _utcnow() delivery.error_message = None await db.commit() try: summary = await self._build_delivery_files(delivery_id) except Exception as exc: async with _new_session() as db: result = await db.execute( select(ResultDeliveryRequestORM).where(ResultDeliveryRequestORM.delivery_id == delivery_id) ) delivery = result.scalar_one_or_none() if delivery is not None: delivery.status = DELIVERY_STATUS_FAILED delivery.error_message = str(exc) delivery.completed_at = _utcnow() delivery.summary_json = { **(delivery.summary_json if isinstance(delivery.summary_json, dict) else {}), "error": str(exc), } await db.commit() if delivery.task_id: await task_service.update_task(delivery.task_id, status="FAILED", progress=100, message=str(exc)) raise async with _new_session() as db: result = await db.execute( select(ResultDeliveryRequestORM) .options(selectinload(ResultDeliveryRequestORM.items)) .where(ResultDeliveryRequestORM.delivery_id == delivery_id) ) delivery = result.scalars().unique().one_or_none() if delivery is None: raise ValueError(f"delivery not found: {delivery_id}") available_files = int(summary.get("copied_files", 0) or 0) + int(summary.get("skipped_files", 0) or 0) delivery.status = DELIVERY_STATUS_READY if available_files > 0 else DELIVERY_STATUS_FAILED delivery.completed_at = _utcnow() delivery.error_message = summary.get("error_message") delivery.summary_json = summary await db.commit() if delivery.task_id: if delivery.status == DELIVERY_STATUS_READY: await task_service.update_task( delivery.task_id, status="COMPLETED", progress=100, message=( f"成果交付包已生成: files={summary.get('copied_files', 0)}, " f"bytes={summary.get('copied_bytes', 0)}" ), ) else: await task_service.update_task( delivery.task_id, status="FAILED", progress=100, message=summary.get("error_message") or "成果交付包生成失败", ) return self.serialize_delivery(delivery, include_items=True) async def _build_delivery_files(self, delivery_id: str) -> Dict[str, Any]: async with _new_session() as db: result = await db.execute( select(ResultDeliveryRequestORM).where(ResultDeliveryRequestORM.delivery_id == delivery_id) ) delivery = result.scalar_one() request_json = delivery.request_json if isinstance(delivery.request_json, dict) else {} max_items = int(settings.RESULT_DELIVERY_MAX_ITEMS or 500) sources = await self._resolve_sources_for_channel( db, channel=delivery.channel, product_ids=_normalize_int_ids(request_json.get("product_ids"), max_count=max_items), compat_result_ids=_normalize_int_ids(request_json.get("compat_result_ids"), max_count=max_items), item_ids=_normalize_int_ids(request_json.get("item_ids"), max_count=max_items), ) task_id = delivery.task_id if not sources: raise ValueError("delivery has no deliverable sources") include_checksums = bool( request_json.get("include_checksums") if "include_checksums" in request_json else settings.RESULT_DELIVERY_CHECKSUM_ENABLED ) delivery_dir = os.path.normpath(os.path.abspath(delivery.delivery_dir)) root = os.path.normpath(os.path.abspath(delivery.delivery_root)) if not _path_within(root, delivery_dir): raise ValueError("delivery directory is outside RESULT_DELIVERY_ROOT") os.makedirs(delivery_dir, exist_ok=True) copied_files = 0 skipped_files = 0 failed_items = 0 copied_bytes = 0 total_bytes = 0 checksum_lines: List[str] = [] item_payloads: List[Dict[str, Any]] = [] manifest_items: List[Dict[str, Any]] = [] total_sources = len(sources) for source_index, source in enumerate(sources, start=1): folder = _sanitize_segment( source.task_name or source.display_name or source.product_id, default=f"product_{source_index}", ) source_files = _iter_associated_files(source.source_path) if not source_files: failed_items += 1 item_payloads.append( { "delivery_id": delivery_id, "source_product_id": source.product.id if source.product else None, "source_result_id": source.compat_row.id if source.compat_row else None, "source_asset_id": source.source_asset_id, "source_radar_data_id": source.source_radar_data_id, "source_scene_geo_id": source.source_scene_geo_id, "display_name": source.display_name, "source_path": source.source_path, "relative_path": None, "file_size": 0, "checksum_sha256": None, "status": ITEM_STATUS_FAILED, "error_message": "source file not found", } ) continue for file_index, source_file in enumerate(source_files, start=1): if os.path.isdir(source.source_path): relative_under_source = os.path.relpath(source_file, source.source_path) relative_path = os.path.join(folder, relative_under_source) else: relative_path = os.path.join(folder, os.path.basename(source_file)) target_path = os.path.join(delivery_dir, relative_path) size = _file_size(source_file) total_bytes += size status = ITEM_STATUS_COPIED error_message = None checksum = None try: os.makedirs(os.path.dirname(target_path), exist_ok=True) if os.path.isfile(target_path) and _same_file_size(source_file, target_path): skipped_files += 1 else: shutil.copy2(source_file, target_path) copied_files += 1 copied_bytes += size if include_checksums and os.path.isfile(target_path): checksum = _sha256_file(target_path) checksum_lines.append(f"{checksum} {relative_path.replace(os.sep, '/')}") except OSError as exc: status = ITEM_STATUS_FAILED error_message = str(exc) failed_items += 1 if task_id and (copied_files + skipped_files + failed_items) % 10 == 0: progress = min(95, int(((source_index - 1) / max(1, total_sources)) * 100)) await task_service.update_task( task_id, progress=progress, message=f"成果交付包生成中: {source_index}/{total_sources}", ) item_payload = { "delivery_id": delivery_id, "source_product_id": source.product.id if source.product else None, "source_result_id": source.compat_row.id if source.compat_row else None, "source_asset_id": source.source_asset_id if file_index == 1 else None, "source_radar_data_id": source.source_radar_data_id, "source_scene_geo_id": source.source_scene_geo_id, "display_name": source.display_name, "source_path": source_file, "relative_path": relative_path, "file_size": size, "checksum_sha256": checksum, "status": status, "error_message": error_message, } item_payloads.append(item_payload) manifest_items.append( { "display_name": source.display_name, "product_id": source.product_id or (source.product.product_id if source.product else None), "product_ref_id": source.product.id if source.product else None, "source_result_id": source.compat_row.id if source.compat_row else None, "source_radar_data_id": source.source_radar_data_id, "source_scene_geo_id": source.source_scene_geo_id, "source_kind": source.source_kind, "relative_path": relative_path.replace(os.sep, "/"), "file_size": size, "checksum_sha256": checksum, "status": status, "error_message": error_message, } ) checksums_path = os.path.join(delivery_dir, "checksums.sha256") if include_checksums: with open(checksums_path, "w", encoding="utf-8") as stream: stream.write("\n".join(checksum_lines)) if checksum_lines: stream.write("\n") manifest_path = os.path.join(delivery_dir, "manifest.json") summary = { "delivery_id": delivery_id, "channel": delivery.channel, "package_mode": delivery.package_mode, "source_count": total_sources, "file_count": len(item_payloads), "copied_files": copied_files, "skipped_files": skipped_files, "failed_items": failed_items, "total_bytes": total_bytes, "copied_bytes": copied_bytes, "include_checksums": include_checksums, "manifest_path": manifest_path, "checksums_path": checksums_path if include_checksums else None, "created_at": _utcnow().isoformat(timespec="seconds"), } if failed_items: summary["error_message"] = f"{failed_items} files failed during delivery build" _write_json( manifest_path, { "schema_version": "insar.result-delivery/v1", "summary": summary, "items": manifest_items, }, ) zip_path = None if delivery.package_mode == PACKAGE_MODE_ZIP: if total_bytes > int(settings.RESULT_DELIVERY_ZIP_MAX_BYTES or 0): raise ValueError( f"delivery size exceeds zip limit: {total_bytes} > {settings.RESULT_DELIVERY_ZIP_MAX_BYTES}" ) zip_path = str(delivery.zip_path or f"{delivery_dir}.zip") with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED, allowZip64=True) as archive: for root_dir, _dirs, files in os.walk(delivery_dir): for name in files: path = os.path.join(root_dir, name) archive.write(path, os.path.relpath(path, delivery_dir)) async with _new_session() as db: result = await db.execute( select(ResultDeliveryRequestORM).where(ResultDeliveryRequestORM.delivery_id == delivery_id) ) delivery_row = result.scalar_one() await db.execute( ResultDeliveryItemORM.__table__.delete().where( ResultDeliveryItemORM.delivery_id == delivery_id ) ) for payload in item_payloads: db.add(ResultDeliveryItemORM(**payload)) delivery_row.total_bytes = total_bytes delivery_row.copied_bytes = copied_bytes delivery_row.item_count = len(item_payloads) delivery_row.manifest_path = manifest_path delivery_row.zip_path = zip_path or delivery_row.zip_path await db.commit() return summary async def resolve_manifest_path( self, db: AsyncSession, *, delivery_id: str, user: AuthUserORM, ) -> str: delivery = await self.get_delivery(db, delivery_id=delivery_id, user=user, include_items=False) if delivery is None: raise ValueError("delivery not found") if delivery.status != DELIVERY_STATUS_READY: raise ValueError("delivery is not ready") path = str(delivery.manifest_path or "").strip() if not path or not os.path.isfile(path): raise ValueError("manifest file is missing") if not _path_within(delivery.delivery_root, path): raise ValueError("manifest path is outside delivery root") return path async def resolve_archive_path( self, db: AsyncSession, *, delivery_id: str, user: AuthUserORM, ) -> str: delivery = await self.get_delivery(db, delivery_id=delivery_id, user=user, include_items=False) if delivery is None: raise ValueError("delivery not found") if delivery.status != DELIVERY_STATUS_READY: raise ValueError("delivery is not ready") path = str(delivery.zip_path or "").strip() if not path or not os.path.isfile(path): raise ValueError("zip archive is not available") if not _path_within(delivery.delivery_root, path): raise ValueError("zip path is outside delivery root") return path async def resolve_item_path( self, db: AsyncSession, *, delivery_id: str, item_id: int, user: AuthUserORM, ) -> str: delivery = await self.get_delivery(db, delivery_id=delivery_id, user=user, include_items=False) if delivery is None: raise ValueError("delivery not found") if delivery.status != DELIVERY_STATUS_READY: raise ValueError("delivery is not ready") result = await db.execute( select(ResultDeliveryItemORM).where( ResultDeliveryItemORM.id == int(item_id), ResultDeliveryItemORM.delivery_id == delivery_id, ) ) item = result.scalar_one_or_none() if item is None: raise ValueError("delivery item not found") if item.status != ITEM_STATUS_COPIED: raise ValueError("delivery item is not available") relative_path = str(item.relative_path or "").strip() if not relative_path: raise ValueError("delivery item path is missing") path = os.path.normpath(os.path.abspath(os.path.join(delivery.delivery_dir, relative_path))) if not os.path.isfile(path): raise ValueError("delivery item file is missing") if not _path_within(delivery.delivery_dir, path): raise ValueError("delivery item path is outside delivery directory") return path result_delivery_service = ResultDeliveryService()