Add AOI upload search and resilient downloads
This commit is contained in:
@@ -1,10 +1,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
|
||||
|
||||
from sentinel_orbit_downloader.api.deps import get_db_path
|
||||
from sentinel_orbit_downloader.api.schemas import JobSceneResponse, JobSummary, SearchJobCreateRequest, SearchJobCreateResponse
|
||||
from sentinel_orbit_downloader.services.aoi import bbox_to_wkt
|
||||
from sentinel_orbit_downloader.api.schemas import (
|
||||
JobSceneResponse,
|
||||
JobSummary,
|
||||
SearchJobCreateRequest,
|
||||
SearchJobCreateResponse,
|
||||
UploadedAoiResponse,
|
||||
)
|
||||
from sentinel_orbit_downloader.services.aoi import bbox_to_wkt, parse_uploaded_aoi
|
||||
from sentinel_orbit_downloader.services.asf_client import SearchParams, query_scenes
|
||||
from sentinel_orbit_downloader.services.project_store import (
|
||||
create_search_job,
|
||||
@@ -21,6 +27,32 @@ from sentinel_orbit_downloader.services.region_catalog import get_region, region
|
||||
router = APIRouter(prefix="/search-jobs", tags=["search-jobs"])
|
||||
|
||||
|
||||
@router.post("/upload-aoi", response_model=UploadedAoiResponse)
|
||||
async def upload_search_aoi_endpoint(
|
||||
files: list[UploadFile] = File(...),
|
||||
) -> UploadedAoiResponse:
|
||||
uploads: list[tuple[str, bytes]] = []
|
||||
for file in files:
|
||||
if not file.filename:
|
||||
continue
|
||||
content = await file.read()
|
||||
uploads.append((file.filename, content))
|
||||
|
||||
try:
|
||||
parsed = parse_uploaded_aoi(uploads)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
return UploadedAoiResponse(
|
||||
name=parsed.name,
|
||||
source_format=parsed.source_format,
|
||||
geometry_type=parsed.geometry_type,
|
||||
feature_count=parsed.feature_count,
|
||||
bbox=parsed.bbox,
|
||||
wkt=parsed.wkt,
|
||||
)
|
||||
|
||||
|
||||
@router.post("", response_model=SearchJobCreateResponse)
|
||||
def create_search_job_endpoint(
|
||||
request: SearchJobCreateRequest,
|
||||
@@ -102,8 +134,8 @@ def resolve_search_aoi(request: SearchJobCreateRequest, db_path) -> dict[str, st
|
||||
if request.wkt:
|
||||
return {
|
||||
"aoi_wkt": request.wkt.strip(),
|
||||
"source_type": "wkt",
|
||||
"source_value": request.wkt.strip(),
|
||||
"source_type": "aoi_file" if request.source_name else "wkt",
|
||||
"source_value": request.source_name.strip() if request.source_name else request.wkt.strip(),
|
||||
"region_adcode": None,
|
||||
"region_name": None,
|
||||
}
|
||||
|
||||
@@ -121,11 +121,21 @@ class JobDetailResponse(BaseModel):
|
||||
events: list[JobEventResponse] = Field(default_factory=list)
|
||||
|
||||
|
||||
class UploadedAoiResponse(BaseModel):
|
||||
name: str
|
||||
source_format: str
|
||||
geometry_type: str
|
||||
feature_count: int
|
||||
bbox: list[float] = Field(default_factory=list)
|
||||
wkt: str
|
||||
|
||||
|
||||
class SearchJobCreateRequest(BaseModel):
|
||||
project_id: int | None = None
|
||||
bbox: str | None = None
|
||||
wkt: str | None = None
|
||||
region_adcode: str | None = None
|
||||
source_name: str | None = None
|
||||
start: str
|
||||
end: str
|
||||
processing_level: str = "SLC"
|
||||
@@ -158,7 +168,7 @@ class DownloadJobCreateRequest(BaseModel):
|
||||
manifest_path: str | None = None
|
||||
skip_data: bool = False
|
||||
skip_orbits: bool = False
|
||||
processes: int = Field(default=2, ge=1)
|
||||
processes: int = Field(default=3, ge=1)
|
||||
ignore_orbit_errors: bool = False
|
||||
username: str | None = None
|
||||
password: str | None = None
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import getpass
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
@@ -10,7 +11,6 @@ from sentinel_orbit_downloader.core.config import DEFAULT_DB_PATH
|
||||
from sentinel_orbit_downloader.services.aoi import aoi_file_to_wkt, bbox_to_wkt
|
||||
from sentinel_orbit_downloader.services.asf_client import (
|
||||
SearchParams,
|
||||
build_session,
|
||||
product_size_mb,
|
||||
query_scenes,
|
||||
scene_name_from_product,
|
||||
@@ -19,7 +19,11 @@ from sentinel_orbit_downloader.services.asf_client import (
|
||||
from sentinel_orbit_downloader.services.orbit_client import (
|
||||
collect_orbit_scenes,
|
||||
download_orbits,
|
||||
scene_names_from_results,
|
||||
)
|
||||
from sentinel_orbit_downloader.services.download_manager import (
|
||||
DownloadJobRequest,
|
||||
execute_download_job,
|
||||
queue_download_job,
|
||||
)
|
||||
from sentinel_orbit_downloader.services.project_store import (
|
||||
create_project,
|
||||
@@ -112,8 +116,8 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
download_parser.add_argument(
|
||||
"--processes",
|
||||
type=positive_int,
|
||||
default=2,
|
||||
help="parallel ASF download processes",
|
||||
default=3,
|
||||
help="retry attempts for each product download",
|
||||
)
|
||||
download_parser.add_argument(
|
||||
"--skip-data",
|
||||
@@ -418,6 +422,15 @@ def cmd_download(args: argparse.Namespace) -> int:
|
||||
results = perform_query(args, aoi_meta["aoi_wkt"])
|
||||
save_query_results(args.db, job.id, results)
|
||||
record_job_event(args.db, job.id, "info", f"Query returned {len(results)} scene(s)")
|
||||
update_search_job(
|
||||
args.db,
|
||||
job.id,
|
||||
status="running",
|
||||
result_count=len(results),
|
||||
data_dir=str(data_dir),
|
||||
orbit_dir=str(orbit_dir),
|
||||
manifest_path=str(manifest),
|
||||
)
|
||||
|
||||
print_results(results)
|
||||
write_manifest(results, manifest)
|
||||
@@ -437,51 +450,31 @@ def cmd_download(args: argparse.Namespace) -> int:
|
||||
print(f"Saved job: {job.id}")
|
||||
return 0
|
||||
|
||||
if not args.skip_data:
|
||||
data_dir.mkdir(parents=True, exist_ok=True)
|
||||
session = build_session(
|
||||
username=args.username,
|
||||
password=args.password,
|
||||
ask_password=args.ask_password,
|
||||
)
|
||||
print(f"Downloading {len(results)} product(s) to {data_dir} ...")
|
||||
results.download(path=str(data_dir), session=session, processes=args.processes)
|
||||
print("Product download finished.")
|
||||
record_job_event(args.db, job.id, "info", f"Product download finished into {data_dir}")
|
||||
record_product_downloads(args.db, job.id, results, data_dir)
|
||||
else:
|
||||
record_job_event(args.db, job.id, "info", "Product download skipped by request")
|
||||
record_product_downloads(args.db, job.id, results, None)
|
||||
password = args.password
|
||||
if args.ask_password and not args.skip_data:
|
||||
password = getpass.getpass("Earthdata password: ")
|
||||
|
||||
orbit_failures = 0
|
||||
if not args.skip_orbits:
|
||||
scenes = scene_names_from_results(results)
|
||||
print(f"Downloading orbit files for {len(scenes)} scene(s) to {orbit_dir} ...")
|
||||
orbit_results = download_orbits(scenes, orbit_dir)
|
||||
orbit_failures = summarize_orbit_results(orbit_results)
|
||||
record_orbit_downloads(args.db, job.id, orbit_results)
|
||||
record_job_event(
|
||||
args.db,
|
||||
job.id,
|
||||
"info",
|
||||
f"Orbit download finished: {len(orbit_results) - orbit_failures} ok, {orbit_failures} failed",
|
||||
)
|
||||
else:
|
||||
record_job_event(args.db, job.id, "info", "Orbit download skipped by request")
|
||||
|
||||
final_status = "partial_failed" if orbit_failures else "succeeded"
|
||||
update_search_job(
|
||||
args.db,
|
||||
job.id,
|
||||
status=final_status,
|
||||
result_count=len(results),
|
||||
data_dir=str(data_dir),
|
||||
orbit_dir=str(orbit_dir),
|
||||
manifest_path=str(manifest),
|
||||
manager_request = DownloadJobRequest(
|
||||
job_id=job.id,
|
||||
data_dir=data_dir,
|
||||
orbit_dir=orbit_dir,
|
||||
manifest_path=manifest,
|
||||
skip_data=args.skip_data,
|
||||
skip_orbits=args.skip_orbits,
|
||||
processes=args.processes,
|
||||
ignore_orbit_errors=args.ignore_orbit_errors,
|
||||
username=args.username,
|
||||
password=password,
|
||||
)
|
||||
queue_download_job(args.db, manager_request)
|
||||
print(f"Downloading {len(results)} scene(s) with resume and validation support ...")
|
||||
execute_download_job(args.db, manager_request)
|
||||
|
||||
finished_job = get_job(args.db, job.id)
|
||||
final_status = finished_job.status if finished_job is not None else "failed"
|
||||
print(f"Download job finished: {final_status}")
|
||||
print(f"Saved job: {job.id}")
|
||||
if orbit_failures and not args.ignore_orbit_errors:
|
||||
if final_status in {"failed", "partial_failed"}:
|
||||
return 1
|
||||
return 0
|
||||
except Exception as exc:
|
||||
|
||||
@@ -1,8 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from io import BytesIO
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable, Sequence
|
||||
import zipfile
|
||||
|
||||
import shapefile
|
||||
from pyproj import CRS, Transformer
|
||||
from shapely import make_valid
|
||||
from shapely.geometry import shape as shapely_shape
|
||||
from shapely.geometry.base import BaseGeometry
|
||||
from shapely.ops import unary_union
|
||||
from shapely.ops import transform as shapely_transform
|
||||
|
||||
|
||||
GEOJSON_SUFFIXES = {".geojson", ".json"}
|
||||
TEXT_WKT_SUFFIXES = {".wkt", ".txt"}
|
||||
SHAPEFILE_PART_SUFFIXES = {".shp", ".shx", ".dbf", ".prj", ".cpg"}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParsedAOI:
|
||||
name: str
|
||||
wkt: str
|
||||
source_format: str
|
||||
geometry_type: str
|
||||
feature_count: int
|
||||
bbox: list[float]
|
||||
|
||||
|
||||
def bbox_to_wkt(value: str) -> str:
|
||||
@@ -22,54 +48,285 @@ def bbox_to_wkt(value: str) -> str:
|
||||
|
||||
|
||||
def aoi_file_to_wkt(path: Path) -> str:
|
||||
text = path.read_text(encoding="utf-8").strip()
|
||||
suffix = path.suffix.lower()
|
||||
if suffix in GEOJSON_SUFFIXES:
|
||||
return parse_geojson_bytes(path.name, path.read_bytes()).wkt
|
||||
if suffix == ".zip":
|
||||
return parse_zip_bytes(path.name, path.read_bytes()).wkt
|
||||
if suffix == ".shp":
|
||||
return parse_shapefile_paths(path).wkt
|
||||
|
||||
text = decode_text_bytes(path.read_bytes()).strip()
|
||||
if not text:
|
||||
raise ValueError(f"AOI file is empty: {path}")
|
||||
if text[0] != "{":
|
||||
if suffix in TEXT_WKT_SUFFIXES:
|
||||
return text
|
||||
|
||||
data = json.loads(text)
|
||||
geometry = geojson_geometry(data)
|
||||
return geometry_to_wkt(geometry)
|
||||
if text[0] in "{[":
|
||||
return parse_geojson_text(path.name, text).wkt
|
||||
return text
|
||||
|
||||
|
||||
def geojson_geometry(data: dict[str, Any]) -> dict[str, Any]:
|
||||
def parse_uploaded_aoi(files: Sequence[tuple[str, bytes]]) -> ParsedAOI:
|
||||
cleaned = [(Path(name).name or name, content) for name, content in files if content]
|
||||
if not cleaned:
|
||||
raise ValueError("No AOI files were provided")
|
||||
|
||||
suffixes = {Path(name).suffix.lower() for name, _ in cleaned}
|
||||
if len(cleaned) == 1:
|
||||
name, content = cleaned[0]
|
||||
suffix = Path(name).suffix.lower()
|
||||
if suffix in GEOJSON_SUFFIXES:
|
||||
return parse_geojson_bytes(name, content)
|
||||
if suffix == ".zip":
|
||||
return parse_zip_bytes(name, content)
|
||||
if suffix == ".shp":
|
||||
return parse_shapefile_uploads(cleaned)
|
||||
if suffix in TEXT_WKT_SUFFIXES:
|
||||
return parse_wkt_text(name, content)
|
||||
|
||||
if suffixes & SHAPEFILE_PART_SUFFIXES:
|
||||
return parse_shapefile_uploads(cleaned)
|
||||
|
||||
raise ValueError(
|
||||
"Unsupported AOI upload. Use .geojson/.json, .zip shapefile, "
|
||||
"or shapefile parts (.shp/.shx/.dbf)."
|
||||
)
|
||||
|
||||
|
||||
def parse_wkt_text(name: str, content: bytes) -> ParsedAOI:
|
||||
text = decode_text_bytes(content).strip()
|
||||
if not text:
|
||||
raise ValueError(f"AOI file is empty: {name}")
|
||||
return ParsedAOI(
|
||||
name=Path(name).stem or name,
|
||||
wkt=text,
|
||||
source_format="wkt",
|
||||
geometry_type="WKT",
|
||||
feature_count=1,
|
||||
bbox=[],
|
||||
)
|
||||
|
||||
|
||||
def parse_geojson_bytes(name: str, content: bytes) -> ParsedAOI:
|
||||
text = decode_text_bytes(content)
|
||||
return parse_geojson_text(name, text)
|
||||
|
||||
|
||||
def parse_geojson_text(name: str, text: str) -> ParsedAOI:
|
||||
stripped = text.strip()
|
||||
if not stripped:
|
||||
raise ValueError(f"AOI file is empty: {name}")
|
||||
try:
|
||||
data = json.loads(stripped)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"Invalid GeoJSON: {exc}") from exc
|
||||
geometry, feature_count = geojson_to_geometry(data)
|
||||
return build_parsed_aoi(
|
||||
name=Path(name).stem or name,
|
||||
source_format="geojson",
|
||||
geometry=geometry,
|
||||
feature_count=feature_count,
|
||||
)
|
||||
|
||||
|
||||
def parse_zip_bytes(name: str, content: bytes) -> ParsedAOI:
|
||||
try:
|
||||
archive = zipfile.ZipFile(BytesIO(content))
|
||||
except zipfile.BadZipFile as exc:
|
||||
raise ValueError(f"Invalid zip file: {name}") from exc
|
||||
|
||||
with archive:
|
||||
file_names = [member for member in archive.namelist() if not member.endswith("/")]
|
||||
geojson_members = [member for member in file_names if Path(member).suffix.lower() in GEOJSON_SUFFIXES]
|
||||
if geojson_members:
|
||||
member = geojson_members[0]
|
||||
parsed = parse_geojson_bytes(Path(member).name, archive.read(member))
|
||||
return ParsedAOI(
|
||||
name=Path(name).stem or parsed.name,
|
||||
wkt=parsed.wkt,
|
||||
source_format="geojson-zip",
|
||||
geometry_type=parsed.geometry_type,
|
||||
feature_count=parsed.feature_count,
|
||||
bbox=parsed.bbox,
|
||||
)
|
||||
|
||||
shape_members = [
|
||||
(Path(member).name, archive.read(member))
|
||||
for member in file_names
|
||||
if Path(member).suffix.lower() in SHAPEFILE_PART_SUFFIXES
|
||||
]
|
||||
if shape_members:
|
||||
parsed = parse_shapefile_uploads(shape_members)
|
||||
return ParsedAOI(
|
||||
name=Path(name).stem or parsed.name,
|
||||
wkt=parsed.wkt,
|
||||
source_format="shapefile-zip",
|
||||
geometry_type=parsed.geometry_type,
|
||||
feature_count=parsed.feature_count,
|
||||
bbox=parsed.bbox,
|
||||
)
|
||||
|
||||
raise ValueError("Zip file does not contain GeoJSON or shapefile content")
|
||||
|
||||
|
||||
def parse_shapefile_paths(shp_path: Path) -> ParsedAOI:
|
||||
base = shp_path.with_suffix("")
|
||||
uploads: list[tuple[str, bytes]] = [(shp_path.name, shp_path.read_bytes())]
|
||||
for suffix in (".shx", ".dbf", ".prj", ".cpg"):
|
||||
candidate = Path(f"{base}{suffix}")
|
||||
if candidate.exists():
|
||||
uploads.append((candidate.name, candidate.read_bytes()))
|
||||
return parse_shapefile_uploads(uploads)
|
||||
|
||||
|
||||
def parse_shapefile_uploads(files: Sequence[tuple[str, bytes]]) -> ParsedAOI:
|
||||
groups: dict[str, dict[str, bytes]] = {}
|
||||
display_names: dict[str, str] = {}
|
||||
for name, content in files:
|
||||
suffix = Path(name).suffix.lower()
|
||||
if suffix not in SHAPEFILE_PART_SUFFIXES:
|
||||
continue
|
||||
stem = Path(name).stem
|
||||
key = stem.lower()
|
||||
groups.setdefault(key, {})[suffix] = content
|
||||
display_names[key] = stem
|
||||
|
||||
shp_groups = [key for key, parts in groups.items() if ".shp" in parts]
|
||||
if not shp_groups:
|
||||
raise ValueError("Shapefile upload requires at least a .shp file")
|
||||
if len(shp_groups) > 1:
|
||||
names = ", ".join(display_names[key] for key in shp_groups)
|
||||
raise ValueError(f"Multiple shapefiles were uploaded at once: {names}")
|
||||
|
||||
key = shp_groups[0]
|
||||
parts = groups[key]
|
||||
reader = shapefile.Reader(
|
||||
shp=BytesIO(parts[".shp"]),
|
||||
shx=BytesIO(parts[".shx"]) if ".shx" in parts else None,
|
||||
dbf=BytesIO(parts[".dbf"]) if ".dbf" in parts else None,
|
||||
)
|
||||
try:
|
||||
geometries = [shapely_shape(shape.__geo_interface__) for shape in reader.shapes()]
|
||||
finally:
|
||||
close = getattr(reader, "close", None)
|
||||
if callable(close):
|
||||
close()
|
||||
|
||||
if not geometries:
|
||||
raise ValueError("Shapefile has no geometry records")
|
||||
geometry = combine_geometries(geometries)
|
||||
geometry = reproject_shapefile_geometry(geometry, parts.get(".prj"))
|
||||
return build_parsed_aoi(
|
||||
name=display_names[key],
|
||||
source_format="shapefile",
|
||||
geometry=geometry,
|
||||
feature_count=len(geometries),
|
||||
)
|
||||
|
||||
|
||||
def geojson_to_geometry(data: dict[str, Any]) -> tuple[BaseGeometry, int]:
|
||||
kind = data.get("type")
|
||||
if kind == "FeatureCollection":
|
||||
features = data.get("features") or []
|
||||
if not features:
|
||||
raise ValueError("GeoJSON FeatureCollection has no features")
|
||||
return geojson_geometry(features[0])
|
||||
geometries = []
|
||||
for feature in features:
|
||||
geometry = feature.get("geometry") if isinstance(feature, dict) else None
|
||||
if geometry:
|
||||
geometries.append(shapely_shape(geometry))
|
||||
if not geometries:
|
||||
raise ValueError("GeoJSON FeatureCollection has no geometry")
|
||||
return combine_geometries(geometries), len(geometries)
|
||||
|
||||
if kind == "Feature":
|
||||
geometry = data.get("geometry")
|
||||
if not geometry:
|
||||
raise ValueError("GeoJSON Feature has no geometry")
|
||||
return normalize_geometry(shapely_shape(geometry)), 1
|
||||
|
||||
if kind:
|
||||
return normalize_geometry(shapely_shape(data)), 1
|
||||
|
||||
raise ValueError("Invalid GeoJSON: missing type")
|
||||
|
||||
|
||||
def combine_geometries(geometries: Iterable[BaseGeometry]) -> BaseGeometry:
|
||||
items = [normalize_geometry(geometry) for geometry in geometries if not geometry.is_empty]
|
||||
if not items:
|
||||
raise ValueError("AOI has no valid geometry")
|
||||
geometry = items[0] if len(items) == 1 else unary_union(items)
|
||||
return normalize_geometry(geometry)
|
||||
|
||||
|
||||
def normalize_geometry(geometry: BaseGeometry) -> BaseGeometry:
|
||||
if geometry.is_empty:
|
||||
raise ValueError("AOI geometry is empty")
|
||||
if not geometry.is_valid:
|
||||
repaired = make_valid(geometry)
|
||||
if not repaired.is_empty:
|
||||
geometry = repaired
|
||||
if geometry.is_empty:
|
||||
raise ValueError("AOI geometry is empty")
|
||||
return geometry
|
||||
|
||||
|
||||
def reproject_shapefile_geometry(geometry: BaseGeometry, prj_content: bytes | None) -> BaseGeometry:
|
||||
if not prj_content:
|
||||
validate_lon_lat_bounds(geometry)
|
||||
return geometry
|
||||
if kind in {"Polygon", "MultiPolygon", "Point", "MultiPoint", "LineString"}:
|
||||
return data
|
||||
raise ValueError(f"unsupported GeoJSON type: {kind}")
|
||||
|
||||
prj_text = decode_text_bytes(prj_content).strip()
|
||||
if not prj_text:
|
||||
validate_lon_lat_bounds(geometry)
|
||||
return geometry
|
||||
|
||||
try:
|
||||
source_crs = CRS.from_wkt(prj_text)
|
||||
except Exception as exc:
|
||||
raise ValueError(f"Unable to read shapefile .prj CRS: {exc}") from exc
|
||||
|
||||
target_crs = CRS.from_epsg(4326)
|
||||
if source_crs.equals(target_crs):
|
||||
validate_lon_lat_bounds(geometry)
|
||||
return geometry
|
||||
|
||||
transformer = Transformer.from_crs(source_crs, target_crs, always_xy=True)
|
||||
transformed = shapely_transform(transformer.transform, geometry)
|
||||
validate_lon_lat_bounds(transformed)
|
||||
return normalize_geometry(transformed)
|
||||
|
||||
|
||||
def geometry_to_wkt(geometry: dict[str, Any]) -> str:
|
||||
kind = geometry.get("type")
|
||||
coordinates = geometry.get("coordinates")
|
||||
if kind == "Point":
|
||||
lon, lat = coordinates
|
||||
return f"POINT({lon} {lat})"
|
||||
if kind == "MultiPoint":
|
||||
return "MULTIPOINT(" + ",".join(f"({lon} {lat})" for lon, lat in coordinates) + ")"
|
||||
if kind == "LineString":
|
||||
return "LINESTRING(" + coords_to_text(coordinates) + ")"
|
||||
if kind == "Polygon":
|
||||
return "POLYGON(" + ",".join(f"({coords_to_text(ring)})" for ring in coordinates) + ")"
|
||||
if kind == "MultiPolygon":
|
||||
polygons = []
|
||||
for polygon in coordinates:
|
||||
rings = ",".join(f"({coords_to_text(ring)})" for ring in polygon)
|
||||
polygons.append(f"({rings})")
|
||||
return "MULTIPOLYGON(" + ",".join(polygons) + ")"
|
||||
raise ValueError(f"unsupported GeoJSON geometry type: {kind}")
|
||||
def validate_lon_lat_bounds(geometry: BaseGeometry) -> None:
|
||||
min_x, min_y, max_x, max_y = geometry.bounds
|
||||
if min_x < -180 or max_x > 180 or min_y < -90 or max_y > 90:
|
||||
raise ValueError(
|
||||
"AOI coordinates are outside longitude/latitude bounds. "
|
||||
"For shapefiles, include a valid .prj file or convert the data to WGS84/EPSG:4326 first."
|
||||
)
|
||||
|
||||
|
||||
def coords_to_text(coords: Iterable[Sequence[float]]) -> str:
|
||||
return ",".join(f"{lon} {lat}" for lon, lat, *_ in coords)
|
||||
def build_parsed_aoi(
|
||||
*,
|
||||
name: str,
|
||||
source_format: str,
|
||||
geometry: BaseGeometry,
|
||||
feature_count: int,
|
||||
) -> ParsedAOI:
|
||||
geometry = normalize_geometry(geometry)
|
||||
bounds = list(geometry.bounds) if geometry.bounds else []
|
||||
return ParsedAOI(
|
||||
name=name,
|
||||
wkt=geometry.wkt,
|
||||
source_format=source_format,
|
||||
geometry_type=geometry.geom_type,
|
||||
feature_count=feature_count,
|
||||
bbox=[float(value) for value in bounds],
|
||||
)
|
||||
|
||||
|
||||
def decode_text_bytes(content: bytes) -> str:
|
||||
for encoding in ("utf-8-sig", "utf-8", "gb18030"):
|
||||
try:
|
||||
return content.decode(encoding)
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
return content.decode("latin-1")
|
||||
|
||||
@@ -2,9 +2,12 @@ from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
import time
|
||||
from typing import Sequence
|
||||
import zipfile
|
||||
|
||||
import asf_search as asf
|
||||
from asf_search.download.download import strip_auth_if_aws
|
||||
|
||||
from sentinel_orbit_downloader.services.asf_client import build_session, credentials_available, scene_name_from_product
|
||||
from sentinel_orbit_downloader.services.orbit_client import download_orbits
|
||||
@@ -40,6 +43,17 @@ class DownloadJobPaths:
|
||||
manifest_path: Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProductDownloadResult:
|
||||
scene_id: str
|
||||
status: str
|
||||
path: Path | None
|
||||
url: str | None
|
||||
bytes_total: float | None
|
||||
bytes_done: float | None
|
||||
error_message: str | None = None
|
||||
|
||||
|
||||
def resolve_download_paths(db_path: Path, request: DownloadJobRequest) -> DownloadJobPaths:
|
||||
job = get_job(db_path, request.job_id)
|
||||
if job is None:
|
||||
@@ -163,7 +177,7 @@ def execute_download_job(db_path: Path, request: DownloadJobRequest) -> None:
|
||||
record_product_skips(db_path, request.job_id, scene_ids)
|
||||
else:
|
||||
results = asf.granule_search(scene_ids)
|
||||
products = {scene_name_from_product(product): product for product in results}
|
||||
products = select_data_products(results, scene_ids, job.processing_level)
|
||||
ordered_products = [products[scene_id] for scene_id in scene_ids if scene_id in products]
|
||||
missing_scene_ids = [scene_id for scene_id in scene_ids if scene_id not in products]
|
||||
if missing_scene_ids:
|
||||
@@ -193,19 +207,52 @@ def execute_download_job(db_path: Path, request: DownloadJobRequest) -> None:
|
||||
path=None,
|
||||
)
|
||||
paths.data_dir.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
session = build_session(username=request.username, password=request.password)
|
||||
results.download(path=str(paths.data_dir), session=session, processes=request.processes)
|
||||
session = build_session(username=request.username, password=request.password)
|
||||
product_attempts = max(1, min(5, request.processes))
|
||||
product_results: list[ProductDownloadResult] = []
|
||||
for index, product in enumerate(ordered_products, start=1):
|
||||
scene_id = scene_name_from_product(product) or f"scene #{index}"
|
||||
record_job_event(
|
||||
db_path,
|
||||
request.job_id,
|
||||
"info",
|
||||
f"Product download finished into {paths.data_dir}",
|
||||
f"Product download {index}/{len(ordered_products)} started: {scene_id}",
|
||||
)
|
||||
except Exception as exc:
|
||||
product_error = str(exc)
|
||||
record_job_event(db_path, request.job_id, "error", f"Product download error: {exc}")
|
||||
record_product_downloads(db_path, request.job_id, ordered_products, paths.data_dir, product_error)
|
||||
result = download_product_with_retries(
|
||||
product,
|
||||
paths.data_dir,
|
||||
session,
|
||||
attempts=product_attempts,
|
||||
)
|
||||
product_results.append(result)
|
||||
record_product_download_result(db_path, request.job_id, result)
|
||||
if result.status == "succeeded":
|
||||
record_job_event(
|
||||
db_path,
|
||||
request.job_id,
|
||||
"info",
|
||||
f"Product download {index}/{len(ordered_products)} succeeded: {scene_id}",
|
||||
)
|
||||
else:
|
||||
record_job_event(
|
||||
db_path,
|
||||
request.job_id,
|
||||
"warning",
|
||||
f"Product download {index}/{len(ordered_products)} failed: {scene_id}; {result.error_message}",
|
||||
)
|
||||
|
||||
product_failures = sum(1 for result in product_results if result.status == "failed") + len(
|
||||
missing_scene_ids
|
||||
)
|
||||
product_successes = sum(1 for result in product_results if result.status == "succeeded")
|
||||
if product_failures:
|
||||
product_error = f"Product download failed for {product_failures} of {len(scene_ids)} scene(s)"
|
||||
record_job_event(
|
||||
db_path,
|
||||
request.job_id,
|
||||
"info",
|
||||
f"Product download finished: {product_successes} ok, {product_failures} failed",
|
||||
)
|
||||
|
||||
orbit_failures = 0
|
||||
if request.skip_orbits:
|
||||
@@ -233,12 +280,25 @@ def execute_download_job(db_path: Path, request: DownloadJobRequest) -> None:
|
||||
)
|
||||
|
||||
product_failures = count_download_failures(db_path, request.job_id, "product")
|
||||
product_successes = count_download_status(db_path, request.job_id, "product", "succeeded")
|
||||
effective_orbit_failures = 0 if request.ignore_orbit_errors else orbit_failures
|
||||
final_status = "succeeded"
|
||||
if product_failures or effective_orbit_failures:
|
||||
final_status = "partial_failed"
|
||||
if product_failures and (request.skip_orbits or effective_orbit_failures == len(scene_ids)):
|
||||
|
||||
if not request.skip_data and product_failures and product_successes == 0:
|
||||
final_status = "failed"
|
||||
elif request.skip_data and not request.skip_orbits and effective_orbit_failures == len(scene_ids):
|
||||
final_status = "failed"
|
||||
|
||||
error_message = None
|
||||
if final_status != "succeeded":
|
||||
error_message = summarize_failure(
|
||||
product_failures=product_failures if not request.skip_data else 0,
|
||||
orbit_failures=effective_orbit_failures if not request.skip_orbits else 0,
|
||||
scene_count=len(scene_ids),
|
||||
fallback=product_error,
|
||||
)
|
||||
|
||||
update_search_job(
|
||||
db_path,
|
||||
@@ -248,7 +308,7 @@ def execute_download_job(db_path: Path, request: DownloadJobRequest) -> None:
|
||||
data_dir=str(paths.data_dir),
|
||||
orbit_dir=str(paths.orbit_dir),
|
||||
manifest_path=str(paths.manifest_path),
|
||||
error_message=product_error,
|
||||
error_message=error_message,
|
||||
)
|
||||
except Exception as exc:
|
||||
update_search_job(
|
||||
@@ -306,6 +366,21 @@ def record_product_downloads(
|
||||
)
|
||||
|
||||
|
||||
def record_product_download_result(db_path: Path, job_id: int, result: ProductDownloadResult) -> None:
|
||||
record_download_result(
|
||||
db_path,
|
||||
job_id=job_id,
|
||||
scene_id=result.scene_id,
|
||||
kind="product",
|
||||
status=result.status,
|
||||
path=str(result.path) if result.path is not None else None,
|
||||
url=result.url,
|
||||
bytes_total=result.bytes_total,
|
||||
bytes_done=result.bytes_done,
|
||||
error_message=result.error_message,
|
||||
)
|
||||
|
||||
|
||||
def record_product_skips(db_path: Path, job_id: int, scene_ids: Sequence[str]) -> None:
|
||||
for scene_id in scene_ids:
|
||||
record_download_result(
|
||||
@@ -342,6 +417,209 @@ def expected_product_path(product, data_dir: Path) -> Path:
|
||||
return data_dir / f"{scene}.zip"
|
||||
|
||||
|
||||
def select_data_products(results, scene_ids: Sequence[str], processing_level: str) -> dict[str, object]:
|
||||
wanted = set(scene_ids)
|
||||
selected: dict[str, object] = {}
|
||||
selected_rank: dict[str, int] = {}
|
||||
for product in results:
|
||||
scene_id = scene_name_from_product(product)
|
||||
if scene_id not in wanted:
|
||||
continue
|
||||
rank = data_product_rank(product, processing_level)
|
||||
if rank is None:
|
||||
continue
|
||||
current_rank = selected_rank.get(scene_id)
|
||||
if current_rank is None or rank < current_rank:
|
||||
selected[scene_id] = product
|
||||
selected_rank[scene_id] = rank
|
||||
return selected
|
||||
|
||||
|
||||
def data_product_rank(product, processing_level: str) -> int | None:
|
||||
props = product.properties
|
||||
product_level = string_value(props.get("processingLevel"))
|
||||
file_name = string_value(props.get("fileName")) or ""
|
||||
url = string_value(props.get("url")) or ""
|
||||
lower_name = file_name.lower()
|
||||
lower_url = url.lower()
|
||||
|
||||
if product_level and product_level.upper().startswith("METADATA"):
|
||||
return None
|
||||
if lower_name.endswith(".iso.xml") or lower_url.endswith(".iso.xml"):
|
||||
return None
|
||||
if processing_level and product_level and product_level.upper() == processing_level.upper():
|
||||
return 0
|
||||
if lower_name.endswith(".zip") or lower_url.endswith(".zip"):
|
||||
return 1
|
||||
return None
|
||||
|
||||
|
||||
def download_product_with_retries(product, data_dir: Path, session, attempts: int = 3) -> ProductDownloadResult:
|
||||
scene_id = scene_name_from_product(product) or "unknown_scene"
|
||||
props = product.properties
|
||||
expected = expected_product_path(product, data_dir)
|
||||
bytes_total = float(props["bytes"]) if props.get("bytes") is not None else None
|
||||
expected_size = int(props["bytes"]) if props.get("bytes") is not None else None
|
||||
url = product_download_url(product)
|
||||
|
||||
if file_complete(expected, expected_size):
|
||||
return ProductDownloadResult(
|
||||
scene_id=scene_id,
|
||||
status="succeeded",
|
||||
path=expected,
|
||||
url=url,
|
||||
bytes_total=bytes_total,
|
||||
bytes_done=float(expected.stat().st_size),
|
||||
)
|
||||
|
||||
last_error = "Downloaded file not found at expected path"
|
||||
for attempt in range(1, max(1, attempts) + 1):
|
||||
try:
|
||||
if url is None:
|
||||
raise ValueError("Product has no downloadable URL")
|
||||
download_product_file(url, expected, session, expected_size)
|
||||
if file_complete(expected, expected_size):
|
||||
return ProductDownloadResult(
|
||||
scene_id=scene_id,
|
||||
status="succeeded",
|
||||
path=expected,
|
||||
url=url,
|
||||
bytes_total=bytes_total,
|
||||
bytes_done=float(expected.stat().st_size),
|
||||
)
|
||||
if expected.exists():
|
||||
last_error = (
|
||||
f"Downloaded file size mismatch: got {expected.stat().st_size}, expected {expected_size}"
|
||||
)
|
||||
else:
|
||||
last_error = "Downloaded file not found at expected path"
|
||||
except Exception as exc:
|
||||
last_error = str(exc) or exc.__class__.__name__
|
||||
|
||||
if attempt < attempts:
|
||||
time.sleep(min(15, attempt * 3))
|
||||
|
||||
part_path = partial_product_path(expected)
|
||||
return ProductDownloadResult(
|
||||
scene_id=scene_id,
|
||||
status="failed",
|
||||
path=None,
|
||||
url=url,
|
||||
bytes_total=bytes_total,
|
||||
bytes_done=partial_size(expected, part_path),
|
||||
error_message=last_error,
|
||||
)
|
||||
|
||||
|
||||
def product_download_url(product) -> str | None:
|
||||
try:
|
||||
urls = product.get_urls()
|
||||
except Exception:
|
||||
urls = []
|
||||
if urls:
|
||||
return str(urls[0])
|
||||
return string_value(product.properties.get("url"))
|
||||
|
||||
|
||||
def download_product_file(url: str, final_path: Path, session, expected_size: int | None) -> None:
|
||||
final_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if file_complete(final_path, expected_size):
|
||||
return
|
||||
|
||||
part_path = prepare_partial_file(final_path, expected_size)
|
||||
if validate_product_file(part_path, expected_size) is None:
|
||||
part_path.replace(final_path)
|
||||
return
|
||||
|
||||
resume_from = part_path.stat().st_size if part_path.exists() else 0
|
||||
headers = {"Range": f"bytes={resume_from}-"} if resume_from else {}
|
||||
response = session.get(
|
||||
url,
|
||||
stream=True,
|
||||
headers=headers,
|
||||
hooks={"response": strip_auth_if_aws},
|
||||
timeout=(30, 120),
|
||||
)
|
||||
try:
|
||||
response.raise_for_status()
|
||||
|
||||
if resume_from and response.status_code != 206:
|
||||
part_path.unlink(missing_ok=True)
|
||||
resume_from = 0
|
||||
elif resume_from:
|
||||
validate_content_range(response.headers.get("Content-Range"), resume_from)
|
||||
|
||||
mode = "ab" if resume_from else "wb"
|
||||
with part_path.open(mode) as handle:
|
||||
for chunk in response.iter_content(chunk_size=1024 * 1024):
|
||||
if chunk:
|
||||
handle.write(chunk)
|
||||
finally:
|
||||
response.close()
|
||||
|
||||
validation_error = validate_product_file(part_path, expected_size)
|
||||
if validation_error is not None:
|
||||
raise IOError(validation_error)
|
||||
|
||||
part_path.replace(final_path)
|
||||
|
||||
|
||||
def prepare_partial_file(final_path: Path, expected_size: int | None) -> Path:
|
||||
part_path = partial_product_path(final_path)
|
||||
|
||||
if final_path.exists() and not file_complete(final_path, expected_size):
|
||||
final_size = final_path.stat().st_size
|
||||
if expected_size is not None and final_size < expected_size and not part_path.exists():
|
||||
final_path.replace(part_path)
|
||||
else:
|
||||
final_path.unlink()
|
||||
|
||||
if part_path.exists() and expected_size is not None:
|
||||
part_size = part_path.stat().st_size
|
||||
if part_size > expected_size or (
|
||||
part_size == expected_size and validate_product_file(part_path, expected_size)
|
||||
):
|
||||
part_path.unlink()
|
||||
|
||||
return part_path
|
||||
|
||||
|
||||
def partial_product_path(final_path: Path) -> Path:
|
||||
return final_path.with_name(f"{final_path.name}.part")
|
||||
|
||||
|
||||
def file_complete(path: Path, expected_size: int | None) -> bool:
|
||||
return validate_product_file(path, expected_size) is None
|
||||
|
||||
|
||||
def validate_product_file(path: Path, expected_size: int | None) -> str | None:
|
||||
if not path.exists() or not path.is_file():
|
||||
return "Downloaded file not found at expected path"
|
||||
actual_size = path.stat().st_size
|
||||
if expected_size is not None and actual_size != expected_size:
|
||||
return f"Downloaded file size mismatch: got {actual_size}, expected {expected_size}"
|
||||
lower_name = path.name.lower()
|
||||
if (lower_name.endswith(".zip") or lower_name.endswith(".zip.part")) and not zipfile.is_zipfile(path):
|
||||
return "Downloaded ZIP failed integrity check"
|
||||
return None
|
||||
|
||||
|
||||
def validate_content_range(content_range: str | None, expected_start: int) -> None:
|
||||
if not content_range:
|
||||
return
|
||||
prefix = f"bytes {expected_start}-"
|
||||
if not content_range.startswith(prefix):
|
||||
raise IOError(f"Unexpected resume range from server: {content_range}")
|
||||
|
||||
|
||||
def partial_size(final_path: Path, part_path: Path) -> float | None:
|
||||
if final_path.exists():
|
||||
return float(final_path.stat().st_size)
|
||||
if part_path.exists():
|
||||
return float(part_path.stat().st_size)
|
||||
return None
|
||||
|
||||
|
||||
def string_value(value) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
@@ -349,6 +627,10 @@ def string_value(value) -> str | None:
|
||||
|
||||
|
||||
def count_download_failures(db_path: Path, job_id: int, kind: str) -> int:
|
||||
return count_download_status(db_path, job_id, kind, "failed")
|
||||
|
||||
|
||||
def count_download_status(db_path: Path, job_id: int, kind: str, status: str) -> int:
|
||||
from sentinel_orbit_downloader.storage.db import connect
|
||||
|
||||
if kind == "product":
|
||||
@@ -361,10 +643,27 @@ def count_download_failures(db_path: Path, job_id: int, kind: str) -> int:
|
||||
with connect(db_path) as connection:
|
||||
row = connection.execute(
|
||||
f"""
|
||||
SELECT COUNT(*) AS failure_count
|
||||
SELECT COUNT(*) AS status_count
|
||||
FROM job_scenes
|
||||
WHERE job_id = ? AND {column} = 'failed'
|
||||
WHERE job_id = ? AND {column} = ?
|
||||
""",
|
||||
(job_id,),
|
||||
(job_id, status),
|
||||
).fetchone()
|
||||
return int(row["failure_count"])
|
||||
return int(row["status_count"])
|
||||
|
||||
|
||||
def summarize_failure(
|
||||
*,
|
||||
product_failures: int,
|
||||
orbit_failures: int,
|
||||
scene_count: int,
|
||||
fallback: str | None,
|
||||
) -> str:
|
||||
parts = []
|
||||
if product_failures:
|
||||
parts.append(f"Product failed for {product_failures}/{scene_count} scene(s)")
|
||||
if orbit_failures:
|
||||
parts.append(f"Orbit failed for {orbit_failures}/{scene_count} scene(s)")
|
||||
if parts:
|
||||
return "; ".join(parts)
|
||||
return fallback or "Download finished with failures"
|
||||
|
||||
@@ -88,4 +88,7 @@ def download_orbits(scenes: Sequence[str], orbit_dir: Path) -> list[OrbitResult]
|
||||
except (InvalidSceneError, OrbitNotFoundError) as exc:
|
||||
message = str(exc) or exc.__class__.__name__
|
||||
results.append(OrbitResult(scene=scene, path=None, error=message))
|
||||
except Exception as exc:
|
||||
message = str(exc) or exc.__class__.__name__
|
||||
results.append(OrbitResult(scene=scene, path=None, error=message))
|
||||
return results
|
||||
|
||||
@@ -73,6 +73,25 @@ const api = {
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
},
|
||||
async uploadSearchAoi(files) {
|
||||
const formData = new FormData();
|
||||
for (const file of files) {
|
||||
formData.append("files", file);
|
||||
}
|
||||
const response = await fetch("/api/search-jobs/upload-aoi", {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
if (!response.ok) {
|
||||
let detail = response.statusText;
|
||||
try {
|
||||
const payload = await response.json();
|
||||
detail = payload.detail || JSON.stringify(payload);
|
||||
} catch (_) {}
|
||||
throw new Error(detail);
|
||||
}
|
||||
return response.json();
|
||||
},
|
||||
createDownloadJob(payload) {
|
||||
return this.request("/api/download-jobs", {
|
||||
method: "POST",
|
||||
@@ -98,6 +117,7 @@ const state = {
|
||||
regions: [],
|
||||
selectedRegion: null,
|
||||
selectedRegionDetail: null,
|
||||
uploadedAoi: null,
|
||||
earthdataSettings: null,
|
||||
jobs: [],
|
||||
selectedJobId: null,
|
||||
@@ -119,6 +139,9 @@ const el = {
|
||||
regionKeyword: document.getElementById("regionKeyword"),
|
||||
regionList: document.getElementById("regionList"),
|
||||
regionDetailCard: document.getElementById("regionDetailCard"),
|
||||
aoiMode: document.getElementById("aoiMode"),
|
||||
uploadAoiFiles: document.getElementById("uploadAoiFiles"),
|
||||
uploadAoiSummary: document.getElementById("uploadAoiSummary"),
|
||||
startDate: document.getElementById("startDate"),
|
||||
endDate: document.getElementById("endDate"),
|
||||
maxResults: document.getElementById("maxResults"),
|
||||
@@ -236,6 +259,10 @@ function currentProject() {
|
||||
return state.projects.find((project) => project.id === state.selectedProjectId) || null;
|
||||
}
|
||||
|
||||
function currentAoiMode() {
|
||||
return el.aoiMode.value || "region";
|
||||
}
|
||||
|
||||
function autoFillProjectPaths(project) {
|
||||
if (!project || !project.data_root) {
|
||||
return;
|
||||
@@ -352,6 +379,22 @@ function renderEarthdataSummary() {
|
||||
);
|
||||
}
|
||||
|
||||
function renderUploadedAoiSummary() {
|
||||
el.uploadAoiSummary.innerHTML = "";
|
||||
const uploadedAoi = state.uploadedAoi;
|
||||
if (!uploadedAoi) {
|
||||
el.uploadAoiSummary.appendChild(makeChip("上传 AOI：未解析"));
|
||||
return;
|
||||
}
|
||||
el.uploadAoiSummary.appendChild(makeChip(`AOI ${uploadedAoi.name}`));
|
||||
el.uploadAoiSummary.appendChild(makeChip(`格式 ${uploadedAoi.source_format}`));
|
||||
el.uploadAoiSummary.appendChild(makeChip(`几何 ${uploadedAoi.geometry_type}`));
|
||||
el.uploadAoiSummary.appendChild(makeChip(`要素 ${uploadedAoi.feature_count}`));
|
||||
if (Array.isArray(uploadedAoi.bbox) && uploadedAoi.bbox.length === 4) {
|
||||
el.uploadAoiSummary.appendChild(makeChip(`范围 ${formatBBox(uploadedAoi.bbox)}`));
|
||||
}
|
||||
}
|
||||
|
||||
function renderJobFilterSummary() {
|
||||
el.jobFilterSummary.innerHTML = "";
|
||||
const project = currentProject();
|
||||
@@ -515,8 +558,16 @@ function updateSelectionSummary() {
|
||||
if (project) {
|
||||
el.selectionSummary.appendChild(makeChip(`项目 ${project.name}`));
|
||||
}
|
||||
if (state.selectedRegion) {
|
||||
el.selectionSummary.appendChild(makeChip(`行政区 ${state.selectedRegion.label}`));
|
||||
if (currentAoiMode() === "upload") {
|
||||
el.selectionSummary.appendChild(makeChip("AOI 来源 上传文件"));
|
||||
if (state.uploadedAoi) {
|
||||
el.selectionSummary.appendChild(makeChip(`上传 AOI ${state.uploadedAoi.name}`));
|
||||
}
|
||||
} else {
|
||||
el.selectionSummary.appendChild(makeChip("AOI 来源 行政区"));
|
||||
if (state.selectedRegion) {
|
||||
el.selectionSummary.appendChild(makeChip(`行政区 ${state.selectedRegion.label}`));
|
||||
}
|
||||
}
|
||||
el.selectionSummary.appendChild(makeChip(`时间 ${el.startDate.value} - ${el.endDate.value}`));
|
||||
el.selectionSummary.appendChild(makeChip(`产品 ${el.processingLevel.value}`));
|
||||
@@ -614,6 +665,25 @@ async function loadSelectedJobDetail() {
|
||||
updateSelectionSummary();
|
||||
}
|
||||
|
||||
async function uploadAoi() {
|
||||
const files = Array.from(el.uploadAoiFiles.files || []);
|
||||
if (!files.length) {
|
||||
throw new Error("请先选择 AOI 文件");
|
||||
}
|
||||
state.uploadedAoi = await api.uploadSearchAoi(files);
|
||||
renderUploadedAoiSummary();
|
||||
updateSelectionSummary();
|
||||
setMessage(`AOI 已解析：${state.uploadedAoi.name}`, "success");
|
||||
}
|
||||
|
||||
function clearUploadedAoi() {
|
||||
state.uploadedAoi = null;
|
||||
el.uploadAoiFiles.value = "";
|
||||
renderUploadedAoiSummary();
|
||||
updateSelectionSummary();
|
||||
setMessage("上传 AOI 已清空", "info");
|
||||
}
|
||||
|
||||
async function importCatalog() {
|
||||
const payload = {
|
||||
source_path: el.catalogPath.value.trim() || DEFAULT_CATALOG_PATH,
|
||||
@@ -669,13 +739,9 @@ async function createSearchJob() {
|
||||
if (!state.selectedProjectId) {
|
||||
throw new Error("请先选择项目");
|
||||
}
|
||||
if (!state.selectedRegion) {
|
||||
throw new Error("请先选择行政区");
|
||||
}
|
||||
|
||||
const payload = {
|
||||
project_id: state.selectedProjectId,
|
||||
region_adcode: state.selectedRegion.adcode,
|
||||
start: el.startDate.value,
|
||||
end: el.endDate.value,
|
||||
processing_level: el.processingLevel.value,
|
||||
@@ -685,9 +751,23 @@ async function createSearchJob() {
|
||||
relative_orbit: numberOrNull(el.relativeOrbit.value),
|
||||
max_results: Number(el.maxResults.value || 20),
|
||||
manifest_path: emptyToNull(el.manifestPath.value),
|
||||
raw_region: el.rawRegion.checked,
|
||||
};
|
||||
|
||||
if (currentAoiMode() === "upload") {
|
||||
if (!state.uploadedAoi) {
|
||||
throw new Error("请先上传并解析 AOI 文件");
|
||||
}
|
||||
payload.wkt = state.uploadedAoi.wkt;
|
||||
payload.source_name = state.uploadedAoi.name;
|
||||
payload.raw_region = false;
|
||||
} else {
|
||||
if (!state.selectedRegion) {
|
||||
throw new Error("请先选择行政区");
|
||||
}
|
||||
payload.region_adcode = state.selectedRegion.adcode;
|
||||
payload.raw_region = el.rawRegion.checked;
|
||||
}
|
||||
|
||||
const response = await api.createSearchJob(payload);
|
||||
state.selectedJobId = response.job.id;
|
||||
state.selectedJobDetail = {
|
||||
@@ -822,6 +902,11 @@ function bindEvents() {
|
||||
document.getElementById("createProjectBtn").addEventListener("click", runAction(createProject, "正在创建项目"));
|
||||
document.getElementById("refreshProjectsBtn").addEventListener("click", runAction(loadProjects, "正在刷新项目"));
|
||||
document.getElementById("searchRegionsBtn").addEventListener("click", runAction(loadRegions, "正在查询行政区"));
|
||||
document.getElementById("uploadAoiBtn").addEventListener("click", runAction(uploadAoi, "正在解析上传 AOI"));
|
||||
document.getElementById("clearAoiBtn").addEventListener("click", clearUploadedAoi);
|
||||
document.getElementById("aoiMode").addEventListener("change", () => {
|
||||
updateSelectionSummary();
|
||||
});
|
||||
document.getElementById("clearRegionsBtn").addEventListener("click", () => {
|
||||
state.regions = [];
|
||||
state.selectedRegion = null;
|
||||
@@ -848,6 +933,7 @@ async function bootstrap() {
|
||||
if (!el.catalogPath.value) {
|
||||
el.catalogPath.value = DEFAULT_CATALOG_PATH;
|
||||
}
|
||||
renderUploadedAoiSummary();
|
||||
el.healthDetail.textContent = window.location.origin;
|
||||
setMessage("正在初始化工作台", "info");
|
||||
await loadHealth();
|
||||
|
||||
@@ -95,6 +95,27 @@
|
||||
<h2>检索参数</h2>
|
||||
<button class="primary" id="runSearchBtn" type="button">发起检索</button>
|
||||
</div>
|
||||
<div class="grid two">
|
||||
<div class="field">
|
||||
<label for="aoiMode">AOI 来源</label>
|
||||
<select id="aoiMode">
|
||||
<option value="region" selected>行政区</option>
|
||||
<option value="upload">上传 AOI 文件</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="field">
|
||||
<label for="uploadAoiFiles">AOI 文件</label>
|
||||
<input id="uploadAoiFiles" type="file" multiple accept=".geojson,.json,.zip,.shp,.shx,.dbf,.prj,.wkt,.txt">
|
||||
</div>
|
||||
</div>
|
||||
<div class="toolbar">
|
||||
<button class="ghost" id="uploadAoiBtn" type="button">解析上传 AOI</button>
|
||||
<button class="ghost" id="clearAoiBtn" type="button">清除上传 AOI</button>
|
||||
</div>
|
||||
<div class="chip-row" id="uploadAoiSummary"></div>
|
||||
<div class="note">
|
||||
支持 `.geojson`、`.json`、`.wkt`、`.txt`、zip shapefile，或同时选择 `.shp`、`.shx`、`.dbf`、`.prj` 等文件。带 `.prj` 的 shapefile 会自动转为 WGS84，经纬度 GeoJSON 会自动合并成一个 AOI 范围。
|
||||
</div>
|
||||
<div class="grid three">
|
||||
<div class="field">
|
||||
<label for="startDate">开始日期</label>
|
||||
@@ -193,8 +214,8 @@
|
||||
</select>
|
||||
</label>
|
||||
<div class="field">
|
||||
<label for="downloadProcesses">并发数</label>
|
||||
<input id="downloadProcesses" type="number" min="1" value="2">
|
||||
<label for="downloadProcesses">重试次数</label>
|
||||
<input id="downloadProcesses" type="number" min="1" max="5" value="3">
|
||||
</div>
|
||||
</div>
|
||||
<div class="toggle-row">
|
||||
@@ -219,7 +240,7 @@
|
||||
</div>
|
||||
<div class="chip-row" id="earthdataSummary"></div>
|
||||
<div class="note">
|
||||
下载原始 Sentinel-1 产品必须提供 Earthdata 用户名和密码。你可以手填后直接下载，也可以先保存到本地配置。当前阶段只下载原始产品包和匹配的 EOF 轨道文件，暂不包含 GeoTIFF 处理。
|
||||
下载原始 Sentinel-1 产品必须提供 Earthdata 用户名和密码。你可以手填后直接下载，也可以先保存到本地配置。产品下载支持完整文件跳过、.part 断点续传和下载后校验。当前阶段只下载原始产品包和匹配的 EOF 轨道文件，暂不包含 GeoTIFF 处理。
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
||||
Reference in New Issue
Block a user