Rewrite the gateway around health, retries, slicing and a Windows service
Scanner - manifest lookup is now depth-bounded and prunes numeric zoom directories, so a scan no longer walks the whole tile payload - rescans only report a version as updated when something actually changed - overlapping runs are refused instead of interleaving their bookkeeping Hot path - token authorisation loads only that token grants and throttles the last_used_at write instead of touching it on every tile - SQLite runs in WAL with a busy timeout and a real connection pool - the catalog lookup used by /tiles no longer loads every basemap Security - the admin surface requires ADMIN_TOKEN (bearer) or ADMIN_USER/ADMIN_PASSWORD (basic), and the service refuses to bind off-host without credentials - CORS is an explicit allow-list; newly minted tokens never reach a redirect URL - client tokens are redacted from the access log Operations - resource health inspection flags a moved or deleted tile root before a 404 - pushes retry with backoff and can be re-queued by hand; the schedule lives in the database so a restart does not lose it - access log plus /metrics counters Image slicing - tile_jobs drive the GDAL bundled with QGIS via scripts/gdal2tiles-qgis.bat - layer_sources remembers what a layer is made of: submit one source to append a flight, or all of them to rebuild the layer as a VRT mosaic with no tile seams; the console exposes both, including a one-click rebuild - slicing is available to integrations through a narrow slice token scope (submit, poll, idempotent by code, completion callback) Console - rebuilt around a fixed rail and single-view navigation with health and outstanding work leading, instead of one long scrolling page Windows service - native install/start/stop/status with crash recovery, auto-start and a rotating log file, so no NSSM wrapper is needed - start-all.bat and stop-all.bat are removed: they killed the process by port with taskkill, which fought the service restart policy Verified with 49 tests and live runs against real tile data and real GDAL.
This commit is contained in:
@@ -0,0 +1,49 @@
|
||||
@echo off
|
||||
setlocal
|
||||
chcp 65001 >nul 2>&1
|
||||
title Build Basemap Pusher
|
||||
|
||||
set "ROOT=%~dp0..\\"
|
||||
pushd "%ROOT%" >nul 2>&1
|
||||
set "ROOT=%CD%\"
|
||||
popd
|
||||
call "%ROOT%scripts\load-env.bat" "%ROOT%config\service.env"
|
||||
|
||||
REM Keep the toolchain caches inside the repo, the same way run-api.bat does.
|
||||
REM Without this, go build uses the global module cache and reaches out to
|
||||
REM proxy.golang.org, which is blocked on many internal networks.
|
||||
if "%GOCACHE%"=="" set "GOCACHE=%ROOT%.gocache"
|
||||
if "%GOMODCACHE%"=="" set "GOMODCACHE=%ROOT%.gomodcache"
|
||||
if "%GOTMPDIR%"=="" set "GOTMPDIR=%ROOT%.gotmp"
|
||||
if "%GOPATH%"=="" set "GOPATH=%ROOT%.gopath"
|
||||
if "%GOPROXY%"=="" set "GOPROXY=https://goproxy.cn,direct"
|
||||
if "%GOSUMDB%"=="" set "GOSUMDB=off"
|
||||
|
||||
set "OUT=%ROOT%bin\basemap-pusher.exe"
|
||||
if not exist "%ROOT%bin" mkdir "%ROOT%bin" >nul 2>&1
|
||||
|
||||
echo ========================================
|
||||
echo Building Basemap Pusher
|
||||
echo ========================================
|
||||
echo Output: %OUT%
|
||||
echo.
|
||||
|
||||
pushd "%ROOT%services\api-go"
|
||||
go build -o "%OUT%" .\cmd\api
|
||||
set "BUILD_EXIT=%ERRORLEVEL%"
|
||||
popd
|
||||
|
||||
if not "%BUILD_EXIT%"=="0" (
|
||||
echo.
|
||||
echo [FAIL] Build failed with exit code %BUILD_EXIT%.
|
||||
) else (
|
||||
echo.
|
||||
echo [OK] Built %OUT%
|
||||
echo.
|
||||
echo Install as a Windows service ^(elevated shell^):
|
||||
echo "%OUT%" --env-file "%ROOT%config\service.env" install
|
||||
echo "%OUT%" start
|
||||
)
|
||||
|
||||
if "%NO_PAUSE%"=="" pause
|
||||
exit /b %BUILD_EXIT%
|
||||
@@ -0,0 +1,35 @@
|
||||
@echo off
|
||||
setlocal enableextensions
|
||||
REM Run gdal2tiles.py from the GDAL bundled with QGIS.
|
||||
REM
|
||||
REM QGIS ships GDAL but its Python bindings need the OSGeo4W environment
|
||||
REM (GDAL_DATA, PROJ_LIB, PYTHONHOME). Running gdal2tiles.exe from a bare
|
||||
REM shell fails to import osgeo, so this wrapper loads o4w_env.bat first.
|
||||
REM
|
||||
REM Override auto-detection with: set GDAL2TILES_QGIS_ROOT=C:\path\to\QGIS
|
||||
|
||||
set "QGIS_ROOT=%GDAL2TILES_QGIS_ROOT%"
|
||||
if not "%QGIS_ROOT%"=="" goto have_root
|
||||
|
||||
for /f "delims=" %%D in ('dir /b /ad /o-n "%ProgramFiles%\QGIS *" 2^>nul') do (
|
||||
if not defined QGIS_ROOT set "QGIS_ROOT=%ProgramFiles%\%%D"
|
||||
)
|
||||
|
||||
:have_root
|
||||
if "%QGIS_ROOT%"=="" (
|
||||
echo gdal2tiles wrapper: QGIS not found. Set GDAL2TILES_QGIS_ROOT to the QGIS install directory. 1>&2
|
||||
exit /b 2
|
||||
)
|
||||
|
||||
if not exist "%QGIS_ROOT%\bin\o4w_env.bat" (
|
||||
echo gdal2tiles wrapper: "%QGIS_ROOT%\bin\o4w_env.bat" is missing; is this a QGIS install? 1>&2
|
||||
exit /b 2
|
||||
)
|
||||
if not exist "%QGIS_ROOT%\apps\Python312\Scripts\gdal2tiles.exe" (
|
||||
echo gdal2tiles wrapper: gdal2tiles.exe not found under "%QGIS_ROOT%\apps\Python312\Scripts". 1>&2
|
||||
exit /b 2
|
||||
)
|
||||
|
||||
call "%QGIS_ROOT%\bin\o4w_env.bat" >nul 2>&1
|
||||
"%QGIS_ROOT%\apps\Python312\Scripts\gdal2tiles.exe" %*
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,33 @@
|
||||
@echo off
|
||||
setlocal enableextensions
|
||||
REM Run gdalbuildvrt from the GDAL bundled with QGIS.
|
||||
REM
|
||||
REM QGIS ships GDAL but its Python bindings need the OSGeo4W environment
|
||||
REM (GDAL_DATA, PROJ_LIB). Loading o4w_env.bat first keeps PROJ working.
|
||||
REM
|
||||
REM Override auto-detection with: set GDAL2TILES_QGIS_ROOT=C:\path\to\QGIS
|
||||
|
||||
set "QGIS_ROOT=%GDAL2TILES_QGIS_ROOT%"
|
||||
if not "%QGIS_ROOT%"=="" goto have_root
|
||||
|
||||
for /f "delims=" %%D in ('dir /b /ad /o-n "%ProgramFiles%\QGIS *" 2^>nul') do (
|
||||
if not defined QGIS_ROOT set "QGIS_ROOT=%ProgramFiles%\%%D"
|
||||
)
|
||||
|
||||
:have_root
|
||||
if "%QGIS_ROOT%"=="" (
|
||||
echo gdalbuildvrt wrapper: QGIS not found. Set GDAL2TILES_QGIS_ROOT to the QGIS install directory. 1>&2
|
||||
exit /b 2
|
||||
)
|
||||
if not exist "%QGIS_ROOT%\bin\o4w_env.bat" (
|
||||
echo gdalbuildvrt wrapper: "%QGIS_ROOT%\bin\o4w_env.bat" is missing; is this a QGIS install? 1>&2
|
||||
exit /b 2
|
||||
)
|
||||
if not exist "%QGIS_ROOT%\bin\gdalbuildvrt.exe" (
|
||||
echo gdalbuildvrt wrapper: gdalbuildvrt.exe not found under "%QGIS_ROOT%\bin". 1>&2
|
||||
exit /b 2
|
||||
)
|
||||
|
||||
call "%QGIS_ROOT%\bin\o4w_env.bat" >nul 2>&1
|
||||
"%QGIS_ROOT%\bin\gdalbuildvrt.exe" %*
|
||||
exit /b %ERRORLEVEL%
|
||||
+3
-3
@@ -1,7 +1,7 @@
|
||||
@echo off
|
||||
setlocal
|
||||
chcp 65001 >nul 2>&1
|
||||
title Map Asset Gateway
|
||||
title Basemap Pusher
|
||||
|
||||
set "ROOT=%~dp0..\\"
|
||||
call "%ROOT%scripts\load-env.bat" "%ROOT%config\service.env"
|
||||
@@ -17,12 +17,12 @@ if "%API_PORT%"=="" set "API_PORT=8910"
|
||||
if "%API_BASE_URL%"=="" set "API_BASE_URL=http://127.0.0.1:%API_PORT%"
|
||||
if "%TILE_PUBLIC_BASE_URL%"=="" set "TILE_PUBLIC_BASE_URL=%API_BASE_URL%"
|
||||
if "%VECTOR_PUBLIC_BASE_URL%"=="" set "VECTOR_PUBLIC_BASE_URL=%API_BASE_URL%"
|
||||
if "%SQLITE_PATH%"=="" set "SQLITE_PATH=%ROOT%data\map-asset-gateway.db"
|
||||
if "%SQLITE_PATH%"=="" set "SQLITE_PATH=%ROOT%data\basemap-pusher.db"
|
||||
if "%BASEMAP_SCAN_ROOT%"=="" set "BASEMAP_SCAN_ROOT=%ROOT%data\tiles"
|
||||
if "%VECTOR_SCAN_ROOT%"=="" set "VECTOR_SCAN_ROOT=%ROOT%data\geojson"
|
||||
|
||||
echo ========================================
|
||||
echo Map Asset Gateway
|
||||
echo Basemap Pusher
|
||||
echo ========================================
|
||||
echo.
|
||||
echo Env file: %ROOT%config\service.env
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
@echo off
|
||||
setlocal
|
||||
chcp 65001 >nul 2>&1
|
||||
title Basemap Pusher Service
|
||||
|
||||
set "ROOT=%~dp0..\"
|
||||
pushd "%ROOT%" >nul 2>&1
|
||||
set "ROOT=%CD%\"
|
||||
popd
|
||||
|
||||
set "EXE=%ROOT%bin\basemap-pusher.exe"
|
||||
set "ENVFILE=%ROOT%config\service.env"
|
||||
|
||||
if not exist "%EXE%" (
|
||||
echo Service binary not found: %EXE%
|
||||
echo Build it first: scripts\build-api.bat
|
||||
if "%NO_PAUSE%"=="" pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
set "VERB=%~1"
|
||||
if "%VERB%"=="" set "VERB=status"
|
||||
|
||||
if /i "%VERB%"=="install" goto do_install
|
||||
if /i "%VERB%"=="uninstall" goto do_uninstall
|
||||
if /i "%VERB%"=="start" goto do_start
|
||||
if /i "%VERB%"=="stop" goto do_stop
|
||||
if /i "%VERB%"=="restart" goto do_restart
|
||||
if /i "%VERB%"=="status" goto do_status
|
||||
|
||||
echo Usage: service.bat ^<install^|uninstall^|start^|stop^|restart^|status^>
|
||||
exit /b 2
|
||||
|
||||
:do_install
|
||||
"%EXE%" --env-file "%ENVFILE%" install
|
||||
goto finish
|
||||
|
||||
:do_uninstall
|
||||
"%EXE%" uninstall
|
||||
goto finish
|
||||
|
||||
:do_start
|
||||
"%EXE%" start
|
||||
goto finish
|
||||
|
||||
:do_stop
|
||||
"%EXE%" stop
|
||||
goto finish
|
||||
|
||||
:do_restart
|
||||
"%EXE%" stop
|
||||
"%EXE%" start
|
||||
goto finish
|
||||
|
||||
:do_status
|
||||
"%EXE%" status
|
||||
goto finish
|
||||
|
||||
:finish
|
||||
set "RC=%ERRORLEVEL%"
|
||||
if not "%RC%"=="0" (
|
||||
echo.
|
||||
echo [FAIL] exit code %RC%.
|
||||
echo install / uninstall / start / stop need an elevated shell ^(Run as Administrator^).
|
||||
)
|
||||
if "%NO_PAUSE%"=="" pause
|
||||
exit /b %RC%
|
||||
Reference in New Issue
Block a user