Harmon/insar-management-system-v2
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: sync repository with current workspace state

Browse Source
This commit is contained in:
Harmon
2026-05-07 11:16:40 +08:00
parent 39cf92044f
commit 69d15bffcb
71 changed files with 3382 additions and 6776 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/dinsar_production_service.py
+116
View File
@@ -27,6 +27,7 @@ from .workflow_service import workflow_service
TASK_TYPE_DINSAR_PRODUCTION = "IDL_RUN_DINSAR"
TASK_TYPE_ISCE2_DINSAR_PRODUCTION = "ISCE2_RUN"
TASK_TYPE_PYINT_DINSAR_PRODUCTION = "PYINT_RUN"
RUN_STATUS_PENDING = "PENDING"
RUN_STATUS_RUNNING = "RUNNING"
RUN_STATUS_COMPLETED = "COMPLETED"
@@ -75,6 +76,8 @@ def _task_type_for_engine(engine_code: str) -> str:
return TASK_TYPE_DINSAR_PRODUCTION
if normalized == "isce2":
return TASK_TYPE_ISCE2_DINSAR_PRODUCTION
if normalized in {"pyint", "gamma"}:
return TASK_TYPE_PYINT_DINSAR_PRODUCTION
raise ValueError(f"Unsupported engine for D-InSAR production run: {engine_code}")
@@ -84,6 +87,8 @@ def _workflow_name_for_engine(engine_code: str) -> str:
return "dinsar_sarscape_production"
if normalized == "isce2":
return "dinsar_isce2_production"
if normalized in {"pyint", "gamma"}:
return "dinsar_pyint_gamma_production"
raise ValueError(f"Unsupported engine for D-InSAR production run: {engine_code}")
@@ -93,6 +98,8 @@ def _workflow_step_name_for_engine(engine_code: str) -> str:
return RUNS_STEP_NAME
if normalized == "isce2":
return "Execute ISCE2 D-InSAR items"
if normalized in {"pyint", "gamma"}:
return "Execute PyINT/Gamma D-InSAR items"
raise ValueError(f"Unsupported engine for D-InSAR production run: {engine_code}")
@@ -329,6 +336,82 @@ def _execution_dir(item: DinsarProductionRunItemORM, run_key: str) -> str:
return os.path.join(item.results_root_dir, "runs", run_key)
def _first_text(*values: Any) -> str:
for value in values:
text = str(value or "").strip()
if text:
return text
return ""
def _read_json_if_exists(path: str) -> Dict[str, Any]:
text = str(path or "").strip()
if not text or not os.path.isfile(text):
return {}
try:
with open(text, "r", encoding="utf-8") as fp:
payload = json.load(fp)
return payload if isinstance(payload, dict) else {}
except Exception:
return {}
def _maybe_join(base: str, *parts: str) -> str:
text = str(base or "").strip()
if not text:
return ""
return os.path.normpath(os.path.join(text, *parts))
def _build_output_paths(
*,
engine_code: str,
item: DinsarProductionRunItemORM,
run_key: str,
output_dir: str,
manifest_path: Optional[str] = None,
) -> Dict[str, Any]:
run_dir = os.path.normpath(str(output_dir or _execution_dir(item, run_key)))
native_dir = _maybe_join(run_dir, "native")
paths: Dict[str, Any] = {
"run_dir": run_dir,
"native_dir": native_dir,
"assets_dir": _maybe_join(run_dir, "assets"),
"quality_dir": _maybe_join(run_dir, "quality"),
"manifest_path": str(manifest_path or "").strip(),
}
if str(engine_code or "").strip().lower() in {"pyint", "gamma"}:
pair_key = _first_text(item.pair_key, os.path.basename(os.path.dirname(os.path.dirname(run_dir))))
project_name = f"{pair_key}_{run_key}" if pair_key and run_key else ""
work_root = _maybe_join(settings.PYINT_WORK_ROOT, pair_key, run_key)
project_dir = _maybe_join(work_root, project_name) if project_name else ""
summary_payload = _read_json_if_exists(_maybe_join(native_dir, "pyint_run_summary.json"))
summary_project_dir = _first_text(summary_payload.get("project_dir"))
project_dir = summary_project_dir or project_dir
master_date = _first_text(summary_payload.get("master_date"))
slave_date = _first_text(summary_payload.get("slave_date"))
pair_name = f"{master_date}-{slave_date}" if master_date and slave_date else ""
ifgrams_dir = _maybe_join(project_dir, "ifgrams", pair_name) if pair_name else _maybe_join(project_dir, "ifgrams")
paths.update(
{
"work_dir": work_root,
"project_dir": project_dir,
"ifgrams_dir": ifgrams_dir,
"reflatten_dir": _maybe_join(run_dir, "gamma_reflatten"),
"native_reflatten_dir": _maybe_join(native_dir, "reflatten"),
"pyint_summary_path": _maybe_join(native_dir, "pyint_run_summary.json"),
"stdout_log": _maybe_join(work_root, "pyint.stdout.log"),
"stderr_log": _maybe_join(work_root, "pyint.stderr.log"),
}
)
return paths
def _sanitize_pointer_fragment(value: str, default: str) -> str:
text = _SAFE_POINTER_RE.sub("_", str(value or "").strip()).strip("._")
return text or default
@@ -616,6 +699,7 @@ class DinsarProductionService:
)
result = await db.execute(stmt)
runs = result.scalars().all()
run_ids = [run.run_id for run in runs if run.run_id]
pending_reconcile = [
run
for run in runs
@@ -636,6 +720,15 @@ class DinsarProductionService:
) or changed
if changed:
await db.commit()
items_by_run_id: Dict[str, List[DinsarProductionRunItemORM]] = {}
if run_ids:
items_result = await db.execute(
select(DinsarProductionRunItemORM)
.where(DinsarProductionRunItemORM.run_id.in_(run_ids))
.order_by(DinsarProductionRunItemORM.order_index.asc(), DinsarProductionRunItemORM.id.asc())
)
for item in items_result.scalars().all():
items_by_run_id.setdefault(item.run_id, []).append(item)
return {
"runs": [
{
@@ -656,6 +749,29 @@ class DinsarProductionService:
"completed_items": run.completed_items,
"failed_items": run.failed_items,
"skipped_items": run.skipped_items,
"items": [
{
"task_name": item.task_name,
"task_alias": item.task_alias,
"pair_key": item.pair_key,
"status": item.status,
"current_step": item.current_step,
"latest_run_key": item.latest_run_key,
"latest_output_dir": item.latest_output_dir,
"latest_manifest_path": item.latest_manifest_path,
"last_error": item.last_error,
"paths": _build_output_paths(
engine_code=run.engine_code,
item=item,
run_key=str(item.latest_run_key or ""),
output_dir=str(item.latest_output_dir or _execution_dir(item, str(item.latest_run_key or ""))),
manifest_path=item.latest_manifest_path,
)
if item.latest_run_key
else {},
}
for item in items_by_run_id.get(run.run_id, [])[:5]
],
}
for run in runs
],
backend/app/services/job_handlers.py
+81 -1
View File
@@ -2058,6 +2058,25 @@ async def _handle_queued_engine_run(
pair_index = max(0, int(event.get("pair_index") or 0))
task_label = str(event.get("task_alias") or event.get("task_name") or "").strip()
if event_type == "log":
level = str(event.get("level") or "INFO").strip().upper()
if level not in {"DEBUG", "INFO", "WARNING", "ERROR"}:
level = "INFO"
source = str(event.get("source") or "").strip()
message = str(event.get("message") or "").strip()
if not message:
continue
label = task_label or str(progress_state.get("pair_label") or "").strip() or "pair"
prefix = f"{engine_title} {pair_index}/{pair_total} {label}"
if source:
prefix = f"{prefix} {source}"
await task_service.add_log(
job.task_id,
level,
f"{prefix}: {message}",
)
continue
if event_type == "pair_started":
progress = min(
90,
@@ -2496,7 +2515,22 @@ async def _run_wsl_dinsar_production_controller(
if event is None:
return
event_type = str(event.get("event") or "").strip().lower()
if event_type == "pair_started":
if event_type == "log":
level = str(event.get("level") or "INFO").strip().upper()
if level not in {"DEBUG", "INFO", "WARNING", "ERROR"}:
level = "INFO"
source = str(event.get("source") or "").strip()
message = str(event.get("message") or "").strip()
if message:
prefix = f"[{item_index}/{total_items}] {engine_title} {item_label}"
if source:
prefix = f"{prefix} {source}"
await task_service.add_log(
job.task_id,
level,
f"{prefix}: {message}",
)
elif event_type == "pair_started":
progress_state["message"] = (
f"[{engine_code}/{run.profile_code}] Running "
f"{item_index}/{total_items}: {item_label}"
@@ -2892,6 +2926,52 @@ async def _handle_isce2_run(job: SystemJobORM) -> None:
async def _handle_pyint_run(job: SystemJobORM) -> None:
production_run_id = str((job.payload or {}).get("production_run_id") or "").strip()
if production_run_id:
try:
await _run_wsl_dinsar_production_controller(
job,
engine_code="pyint",
engine_title="PyINT/Gamma",
fallback_timeout_seconds=settings.PYINT_DEFAULT_TIMEOUT_SECONDS,
)
except Exception as exc:
latest_message = f"PyINT/Gamma D-InSAR production controller failed: {exc}"
try:
async with AsyncSessionLocal() as db:
run = await dinsar_production_service.get_run(production_run_id, db)
if run is not None and str(run.status or "").strip().upper() not in {"COMPLETED", "FAILED", "CANCELLED"}:
summary_payload = dict(run.summary_json or {})
summary_payload["controller_error"] = str(exc)
await dinsar_production_service.finalize_run(
run,
db=db,
status="FAILED",
summary_payload=summary_payload,
latest_message=latest_message,
)
dinsar_production_service.append_run_log(
run.run_id,
f"[controller-failed] {exc}",
)
except Exception:
pass
try:
current_task = await task_service.get_task(job.task_id)
if current_task and current_task.status not in {"COMPLETED", "FAILED", "CANCELLED"}:
await task_service.add_log(job.task_id, "ERROR", latest_message)
await task_service.update_task(
job.task_id,
status="FAILED",
progress=100,
message=latest_message,
)
except Exception:
pass
raise
return
await _handle_queued_engine_run(
job,
engine_title="PyINT",
backend/app/services/pyint_input_assets_service.py
+1
View File
@@ -316,6 +316,7 @@ def get_pyint_dem_summary() -> Dict[str, Any]:
"hdr_exists": bool(prepared_dem_info.get("hdr_exists")),
"vrt_exists": bool(prepared_dem_info.get("vrt_exists")),
},
"configured_resolution_m": float(getattr(settings, "PYINT_DEM_RESOLUTION_M", 30.0) or 30.0),
"opentopo_dem_type": opentopo_dem_type,
"opentopo_api_key_configured": bool(opentopo_api_key),
"status": status,
backend/app/services/pyint_service.py
+270 -18
View File
@@ -4,6 +4,8 @@ from __future__ import annotations
import os
import re
import shlex
import math
import defusedxml.ElementTree as ET
from dataclasses import dataclass, field
from datetime import datetime
from pathlib import Path
@@ -11,12 +13,83 @@ from typing import Any, Dict, Iterable, List, Optional
from ..config import get_env_text, read_bool_env, settings
from .dinsar_naming import PAIR_META_FILENAME, build_fallback_pair_key, find_json_sidecar
from .wsl_service import run_wsl_command
from .wsl_service import run_wsl_exec
LT1_INPUT_GLOBS = ("LT1*.tar.gz", "LT1*.tiff")
DEFAULT_RANGE_LOOKS = 2
DEFAULT_AZIMUTH_LOOKS = 2
DEFAULT_DEM_RESOLUTION_M = 30.0
DEFAULT_UNWRAP_COH_THRESHOLD = 0.05
DEFAULT_PRODUCT_COH_THRESHOLD = 0.20
DEFAULT_REFERENCE_MODE = "none"
DEFAULT_REFERENCE_COH_THRESHOLD = 0.30
DEFAULT_DERAMP_MODE = "none"
DEFAULT_DERAMP_COH_THRESHOLD = 0.30
DEFAULT_GEO_INTERP = "1"
DEFAULT_ATMCOR_ENABLED = False
DEFAULT_ATMCOR_USE_FOR_DISP = False
DEFAULT_REFLATTEN_ENABLED = True
DEFAULT_REFLATTEN_MODEL = "plane"
DEFAULT_REFLATTEN_COH_THRESHOLD = 0.70
DEFAULT_REFLATTEN_FALLBACK_COH_THRESHOLD = 0.20
DEFAULT_REFLATTEN_RANGE_STEP = 32
DEFAULT_REFLATTEN_AZIMUTH_STEP = 32
DEM_OVERSAMPLING_MIN = 0.25
DEM_OVERSAMPLING_MAX = 16.0
REFERENCE_MODE_CHOICES = {"none", "coh_median"}
DERAMP_MODE_CHOICES = {"none", "plane"}
REFLATTEN_MODEL_CHOICES = {"plane", "linear", "quadratic"}
def _read_default_target_grid_size_m() -> int:
for name in ("PYINT_DEFAULT_TARGET_GRID_SIZE_M",):
text = str(get_env_text(name, "") or "").strip()
if not text:
continue
try:
value = float(text)
except (TypeError, ValueError):
continue
if value > 0:
return int(value)
return 0
def _read_float_env(names: Iterable[str], default: float) -> float:
for name in names:
text = str(get_env_text(name, "") or "").strip()
if not text:
continue
try:
value = float(text)
except (TypeError, ValueError):
continue
if math.isfinite(value):
return value
return float(default)
DEFAULT_TARGET_GRID_SIZE_M = _read_default_target_grid_size_m()
TARGET_GRID_SIZE_MIN_M = 0
TARGET_GRID_SIZE_MAX_M = 100
DEFAULT_DEM_RESOLUTION_M = _read_float_env(("PYINT_DEM_RESOLUTION_M",), DEFAULT_DEM_RESOLUTION_M)
DEFAULT_UNWRAP_COH_THRESHOLD = _read_float_env(
("PYINT_UNWRAP_COH_THRESHOLD",),
DEFAULT_UNWRAP_COH_THRESHOLD,
)
DEFAULT_PRODUCT_COH_THRESHOLD = _read_float_env(
("PYINT_PRODUCT_COH_THRESHOLD", "PYINT_COHERENCE_MASK_THRESHOLD"),
DEFAULT_PRODUCT_COH_THRESHOLD,
)
DEFAULT_REFERENCE_COH_THRESHOLD = _read_float_env(
("PYINT_REFERENCE_COH_THRESHOLD",),
DEFAULT_REFERENCE_COH_THRESHOLD,
)
DEFAULT_DERAMP_COH_THRESHOLD = _read_float_env(
("PYINT_DERAMP_COH_THRESHOLD",),
DEFAULT_DERAMP_COH_THRESHOLD,
)
DEFAULT_PARALLEL_WORKERS = 1
MAX_LOOKS = 32
MAX_PARALLEL_WORKERS = 16
@@ -76,6 +149,180 @@ def normalize_date_text(value: Any) -> str:
return ""
def _local_xml_tag_name(tag: Any) -> str:
text = str(tag or "")
return text.split("}")[-1] if "}" in text else text
def _read_xml_first_parameter(xml_file: str, names: Iterable[str]) -> Optional[str]:
path = os.path.normpath(str(xml_file or "").strip())
if not path or not os.path.isfile(path):
return None
wanted = {str(name or "").strip().lower() for name in names if str(name or "").strip()}
if not wanted:
return None
try:
tree = ET.parse(path)
root = tree.getroot()
except Exception:
return None
for elem in root.iter():
local_name = _local_xml_tag_name(elem.tag).lower()
if local_name in wanted and elem.text and str(elem.text).strip():
return str(elem.text).strip()
return None
def _read_scene_geometry_metadata(metadata_path: str) -> Dict[str, Any]:
source = os.path.normpath(str(metadata_path or "").strip())
range_spacing = _read_xml_first_parameter(
source,
("PixelSpacingRg", "columnSpacing", "slantRange", "range_pixel_spacing"),
)
azimuth_spacing = _read_xml_first_parameter(
source,
("PixelSpacingAz", "rowSpacing", "projectedSpacingAzimuth", "azimuth_pixel_spacing"),
)
incidence_angle = _read_xml_first_parameter(
source,
("IncidenceAngle", "incidence_angle"),
)
if not all((range_spacing, azimuth_spacing, incidence_angle)):
raise ValueError(f"Cannot read range/azimuth spacing and incidence angle from: {source}")
return {
"source": source,
"range_pixel_spacing_m": float(range_spacing),
"azimuth_pixel_spacing_m": float(azimuth_spacing),
"incidence_angle_deg": float(incidence_angle),
}
def _scene_geometry_metadata_candidates(directory: str, patterns: Iterable[str]) -> List[str]:
root = os.path.normpath(str(directory or "").strip())
if not root or not os.path.isdir(root):
return []
candidates: List[str] = []
for pattern in patterns:
candidates.extend(str(path) for path in Path(root).glob(pattern) if path.is_file())
return [
os.path.normpath(path)
for path in sorted(
set(candidates),
key=lambda item: (0 if item.lower().endswith(".sml") else 1, item.lower()),
)
]
def resolve_scene_geometry_metadata_files(scene_dir: str) -> List[str]:
return _scene_geometry_metadata_candidates(
scene_dir,
(
"*.sml",
"*.SML",
"*.meta.xml",
"*.META.XML",
),
)
def resolve_scene_geometry_metadata_file(scene_dir: str) -> str:
candidates = resolve_scene_geometry_metadata_files(scene_dir)
return candidates[0] if candidates else ""
def calculate_looks_from_scene_metadata(
*,
master_metadata: str,
slave_metadata: str,
target_resolution_m: float,
) -> Dict[str, Any]:
target_resolution = float(target_resolution_m)
if target_resolution <= 0:
raise ValueError("target_resolution_m must be greater than 0")
master = _read_scene_geometry_metadata(master_metadata)
slave = _read_scene_geometry_metadata(slave_metadata)
avg_azimuth = (
float(master["azimuth_pixel_spacing_m"]) + float(slave["azimuth_pixel_spacing_m"])
) / 2.0
master_ground_range = float(master["range_pixel_spacing_m"]) / math.sin(
math.radians(float(master["incidence_angle_deg"]))
)
slave_ground_range = float(slave["range_pixel_spacing_m"]) / math.sin(
math.radians(float(slave["incidence_angle_deg"]))
)
avg_ground_range = (master_ground_range + slave_ground_range) / 2.0
range_ratio = target_resolution / avg_ground_range
azimuth_ratio = target_resolution / avg_azimuth
range_looks = max(1, int(math.floor(range_ratio + 0.5)))
azimuth_looks = max(1, int(math.floor(azimuth_ratio + 0.5)))
return {
"mode": "target_grid_size",
"target_resolution_m": target_resolution,
"range_looks": range_looks,
"azimuth_looks": azimuth_looks,
"avg_ground_range_spacing_m": avg_ground_range,
"avg_azimuth_spacing_m": avg_azimuth,
"range_look_ratio": range_ratio,
"azimuth_look_ratio": azimuth_ratio,
"resolved_ground_range_spacing_m": avg_ground_range * range_looks,
"resolved_azimuth_spacing_m": avg_azimuth * azimuth_looks,
"master": master,
"slave": slave,
}
def calculate_looks_from_task_dir(task_dir: str, target_resolution_m: float) -> Dict[str, Any]:
task_root = os.path.normpath(str(task_dir or "").strip())
master_candidates = resolve_scene_geometry_metadata_files(os.path.join(task_root, "master"))
slave_candidates = resolve_scene_geometry_metadata_files(os.path.join(task_root, "slave"))
if not master_candidates or not slave_candidates:
raise ValueError(f"Cannot find SML/meta XML metadata under task: {task_root}")
errors: List[str] = []
for master_metadata in master_candidates:
for slave_metadata in slave_candidates:
try:
return calculate_looks_from_scene_metadata(
master_metadata=master_metadata,
slave_metadata=slave_metadata,
target_resolution_m=target_resolution_m,
)
except Exception as exc:
errors.append(f"{os.path.basename(master_metadata)} + {os.path.basename(slave_metadata)}: {exc}")
detail = "; ".join(errors[:3]) if errors else "unknown metadata parsing error"
raise ValueError(f"Cannot calculate looks from task metadata under {task_root}: {detail}")
def calculate_dem_oversampling(
*,
dem_resolution_m: float,
target_grid_size_m: float,
) -> Dict[str, Any]:
dem_resolution = float(dem_resolution_m or 0.0)
target_grid = float(target_grid_size_m or 0.0)
if not math.isfinite(dem_resolution) or dem_resolution <= 0:
dem_resolution = DEFAULT_DEM_RESOLUTION_M
raw_factor = dem_resolution / target_grid if math.isfinite(target_grid) and target_grid > 0 else None
oversampling = 1.0
actual_grid = dem_resolution / oversampling if oversampling > 0 else dem_resolution
mismatch_ratio = abs(actual_grid - target_grid) / target_grid if target_grid > 0 else None
return {
"mode": "gamma_dem_oversampling",
"dem_resolution_m": dem_resolution,
"target_grid_size_m": target_grid,
"raw_oversampling": raw_factor,
"oversampling": oversampling,
"actual_grid_size_m": actual_grid,
"mismatch_ratio": mismatch_ratio,
"min_oversampling": DEM_OVERSAMPLING_MIN,
"max_oversampling": DEM_OVERSAMPLING_MAX,
}
def slugify_text(value: Any, *, default: str = "item", max_len: int = 96) -> str:
text = _SAFE_TEXT_RE.sub("_", str(value or "").strip()).strip("._")
if not text:
@@ -278,7 +525,14 @@ def _gamma_prefix(gamma_env_script_wsl: str) -> str:
script = str(gamma_env_script_wsl or "").strip()
if not script:
return ""
return f". {quote_shell(script)} >/dev/null 2>&1 && "
return f". {quote_shell(script)} >/dev/null 2>&1 || exit 1; "
def _pyint_path_prefix(pyint_home_wsl: str) -> str:
home = str(pyint_home_wsl or "").strip().rstrip("/")
if not home:
return ""
return f"export PATH={quote_shell(home + '/pyint')}:\"$PATH\" && "
def check_pyint_environment(
@@ -320,7 +574,10 @@ def check_pyint_environment(
def add(name: str, ok: bool, detail: str = "", skipped: bool = False) -> None:
checks.append(PyintCheck(name=name, ok=ok, detail=detail, skipped=skipped))
rc, out, err = run_wsl_command("echo pyint_alive", distro=distro_value, timeout=15)
def run_check(command: str, timeout: int = 30):
return run_wsl_exec(["bash", "-lc", command], distro=distro_value, timeout=timeout)
rc, out, err = run_check("echo pyint_alive", timeout=15)
wsl_ok = rc == 0 and "pyint_alive" in out
add("WSL distro", wsl_ok, out or err or distro_value)
@@ -331,17 +588,15 @@ def check_pyint_environment(
message=f"WSL distro is unavailable: {distro_value}",
)
rc, out, err = run_wsl_command(
rc, out, err = run_check(
f"{quote_shell(python_value)} --version",
distro=distro_value,
timeout=15,
)
add("WSL Python", rc == 0, out or err or python_value)
if pyint_home_wsl:
rc, out, err = run_wsl_command(
rc, out, err = run_check(
f"test -d {quote_shell(pyint_home_wsl)} && echo ok",
distro=distro_value,
timeout=10,
)
add("PYINT_HOME", rc == 0 and "ok" in out, pyint_home_wsl or err)
@@ -349,9 +604,8 @@ def check_pyint_environment(
add("PYINT_HOME", False, "PYINT_HOME is empty")
if pyint_app_wsl:
rc, out, err = run_wsl_command(
rc, out, err = run_check(
f"test -f {quote_shell(pyint_app_wsl)} && echo ok",
distro=distro_value,
timeout=10,
)
add("pyintApp.py", rc == 0 and "ok" in out, pyint_app_wsl or err)
@@ -367,17 +621,15 @@ def check_pyint_environment(
if not path_text:
add(name, False, f"{name} is empty")
continue
rc, out, err = run_wsl_command(
rc, out, err = run_check(
f"test -d {quote_shell(path_text)} && test -w {quote_shell(path_text)} && echo ok",
distro=distro_value,
timeout=10,
)
add(name, rc == 0 and "ok" in out, path_text or err)
if gamma_env_wsl:
rc, out, err = run_wsl_command(
rc, out, err = run_check(
f"test -f {quote_shell(gamma_env_wsl)} && echo ok",
distro=distro_value,
timeout=10,
)
add("GAMMA env script", rc == 0 and "ok" in out, gamma_env_wsl or err)
@@ -385,16 +637,16 @@ def check_pyint_environment(
add("GAMMA env script", True, "Not configured; using current PATH", skipped=True)
gamma_prefix = _gamma_prefix(gamma_env_wsl)
pyint_prefix = _pyint_path_prefix(pyint_home_wsl)
for name, command_name in (
("GAMMA LT1 import", "LT1_import_SLC_from_zipfiles1"),
("GAMMA geocode_back", "geocode_back"),
):
rc, out, err = run_wsl_command(
gamma_prefix + f"command -v {quote_shell(command_name)}",
distro=distro_value,
rc, out, err = run_check(
gamma_prefix + pyint_prefix + f"command -v {quote_shell(command_name)} >/dev/null 2>&1 && echo ok",
timeout=10,
)
add(name, rc == 0 and bool(out.strip()), out or err or command_name)
add(name, rc == 0 and "ok" in out, out or err or command_name)
helper_path = (
Path(__file__).resolve().parent.parent
@@ -412,7 +664,7 @@ def check_pyint_environment(
+ gamma_prefix
+ f"{quote_shell(python_value)} {quote_shell(pyint_app_wsl)} -h >/dev/null"
)
rc, out, err = run_wsl_command(smoke_cmd, distro=distro_value, timeout=60)
rc, out, err = run_check(smoke_cmd, timeout=60)
add("PyINT smoke test", rc == 0, out or err or "pyintApp.py -h")
else:
add("PyINT smoke test", True, "Skipped", skipped=True)
backend/app/services/wsl_service.py
+103 -1
View File
@@ -11,8 +11,10 @@ from __future__ import annotations
import os
import shutil
import subprocess
import threading
import time
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Sequence, Tuple
from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
# ---------------------------------------------------------------------------
@@ -101,6 +103,79 @@ def _run_windows_command(
)
def _run_windows_command_stream(
args: List[str],
timeout: int = 30,
env: Optional[Dict[str, str]] = None,
stdout_callback: Optional[Callable[[str], None]] = None,
stderr_callback: Optional[Callable[[str], None]] = None,
) -> Tuple[int, str, str]:
proc_env = os.environ.copy()
if env:
proc_env.update(env)
stdout_parts: List[str] = []
stderr_parts: List[str] = []
try:
proc = subprocess.Popen(
args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=False,
env=proc_env,
)
except FileNotFoundError:
return -2, "", "wsl.exe not found"
except Exception as exc:
return -3, "", str(exc)
def _drain(stream: Any, parts: List[str], callback: Optional[Callable[[str], None]]) -> None:
for raw_line in iter(stream.readline, b""):
text = _decode_subprocess_output(raw_line)
if not text:
continue
parts.append(text)
if callback:
try:
callback(text)
except Exception:
pass
threads = [
threading.Thread(target=_drain, args=(proc.stdout, stdout_parts, stdout_callback), daemon=True),
threading.Thread(target=_drain, args=(proc.stderr, stderr_parts, stderr_callback), daemon=True),
]
for thread in threads:
thread.start()
timed_out = False
deadline = time.monotonic() + max(1, int(timeout or 30))
while proc.poll() is None:
if time.monotonic() >= deadline:
timed_out = True
try:
proc.kill()
except Exception:
pass
break
time.sleep(0.2)
try:
returncode = proc.wait(timeout=10)
except subprocess.TimeoutExpired:
returncode = -1
for thread in threads:
thread.join(timeout=5)
stdout = "\n".join(stdout_parts)
stderr = "\n".join(stderr_parts)
if timed_out:
timeout_text = f"command timed out ({timeout}s)"
stderr = f"{stderr}\n{timeout_text}".strip()
return -1, stdout, stderr
return returncode, stdout, stderr
def run_wsl_command(
cmd: str,
distro: Optional[str] = None,
@@ -127,6 +202,33 @@ def run_wsl_command(
return -3, "", str(exc)
def run_wsl_command_stream(
cmd: str,
distro: Optional[str] = None,
timeout: int = 30,
env: Optional[Dict[str, str]] = None,
stdout_callback: Optional[Callable[[str], None]] = None,
stderr_callback: Optional[Callable[[str], None]] = None,
) -> Tuple[int, str, str]:
"""Run a WSL bash command and stream decoded stdout/stderr lines to callbacks."""
wsl_exe = _find_wsl_executable()
if not wsl_exe:
return -2, "", "wsl.exe not found"
wsl_args = [wsl_exe]
if distro:
wsl_args += ["-d", distro]
wsl_args += ["bash", "-lc", cmd]
return _run_windows_command_stream(
wsl_args,
timeout=timeout,
env=env,
stdout_callback=stdout_callback,
stderr_callback=stderr_callback,
)
def run_wsl_exec(
argv: Sequence[str],
distro: Optional[str] = None,