Basemap Pusher: catalog and distribution gateway for existing tiles
A Go service that scans existing basemap tile directories and vector files, catalogues them in SQLite, issues scoped access tokens, and serves tiles and downloads to downstream systems. It deliberately does not generate or process raster data: GIS software produces the tiles, this service catalogues, authorises and distributes them. Capabilities - scans basemap tile directories (manifest-based or raw z/x/y) and vector assets (.geojson, .json, .zip, shapefiles with sidecars) - token auth with per-resource grants, expiry and coarse scopes (read, slice) - resource health inspection flags a moved or deleted tile root before a client hits a 404 - downstream push with exponential-backoff retries and manual re-queue - image slicing jobs driven by the GDAL bundled with QGIS, including VRT mosaicking so successive drone flights accumulate into one seamless layer - a narrow, non-admin slicing API for external callers: submit, poll, idempotent by job code, completion callback - ops console covering health, ledger, tokens, pushes, scans, slicing and service info - native Windows service install/start/stop with crash recovery, auto-start and a rotating log file, so no NSSM or WinSW wrapper is needed Verified with 49 tests plus live runs against real tile data and real GDAL.
This commit is contained in:
@@ -0,0 +1,49 @@
|
||||
@echo off
|
||||
setlocal
|
||||
chcp 65001 >nul 2>&1
|
||||
title Build Basemap Pusher
|
||||
|
||||
set "ROOT=%~dp0..\\"
|
||||
pushd "%ROOT%" >nul 2>&1
|
||||
set "ROOT=%CD%\"
|
||||
popd
|
||||
call "%ROOT%scripts\load-env.bat" "%ROOT%config\service.env"
|
||||
|
||||
REM Keep the toolchain caches inside the repo, the same way run-api.bat does.
|
||||
REM Without this, go build uses the global module cache and reaches out to
|
||||
REM proxy.golang.org, which is blocked on many internal networks.
|
||||
if "%GOCACHE%"=="" set "GOCACHE=%ROOT%.gocache"
|
||||
if "%GOMODCACHE%"=="" set "GOMODCACHE=%ROOT%.gomodcache"
|
||||
if "%GOTMPDIR%"=="" set "GOTMPDIR=%ROOT%.gotmp"
|
||||
if "%GOPATH%"=="" set "GOPATH=%ROOT%.gopath"
|
||||
if "%GOPROXY%"=="" set "GOPROXY=https://goproxy.cn,direct"
|
||||
if "%GOSUMDB%"=="" set "GOSUMDB=off"
|
||||
|
||||
set "OUT=%ROOT%bin\basemap-pusher.exe"
|
||||
if not exist "%ROOT%bin" mkdir "%ROOT%bin" >nul 2>&1
|
||||
|
||||
echo ========================================
|
||||
echo Building Basemap Pusher
|
||||
echo ========================================
|
||||
echo Output: %OUT%
|
||||
echo.
|
||||
|
||||
pushd "%ROOT%services\api-go"
|
||||
go build -o "%OUT%" .\cmd\api
|
||||
set "BUILD_EXIT=%ERRORLEVEL%"
|
||||
popd
|
||||
|
||||
if not "%BUILD_EXIT%"=="0" (
|
||||
echo.
|
||||
echo [FAIL] Build failed with exit code %BUILD_EXIT%.
|
||||
) else (
|
||||
echo.
|
||||
echo [OK] Built %OUT%
|
||||
echo.
|
||||
echo Install as a Windows service ^(elevated shell^):
|
||||
echo "%OUT%" --env-file "%ROOT%config\service.env" install
|
||||
echo "%OUT%" start
|
||||
)
|
||||
|
||||
if "%NO_PAUSE%"=="" pause
|
||||
exit /b %BUILD_EXIT%
|
||||
@@ -0,0 +1,35 @@
|
||||
@echo off
|
||||
setlocal enableextensions
|
||||
REM Run gdal2tiles.py from the GDAL bundled with QGIS.
|
||||
REM
|
||||
REM QGIS ships GDAL but its Python bindings need the OSGeo4W environment
|
||||
REM (GDAL_DATA, PROJ_LIB, PYTHONHOME). Running gdal2tiles.exe from a bare
|
||||
REM shell fails to import osgeo, so this wrapper loads o4w_env.bat first.
|
||||
REM
|
||||
REM Override auto-detection with: set GDAL2TILES_QGIS_ROOT=C:\path\to\QGIS
|
||||
|
||||
set "QGIS_ROOT=%GDAL2TILES_QGIS_ROOT%"
|
||||
if not "%QGIS_ROOT%"=="" goto have_root
|
||||
|
||||
for /f "delims=" %%D in ('dir /b /ad /o-n "%ProgramFiles%\QGIS *" 2^>nul') do (
|
||||
if not defined QGIS_ROOT set "QGIS_ROOT=%ProgramFiles%\%%D"
|
||||
)
|
||||
|
||||
:have_root
|
||||
if "%QGIS_ROOT%"=="" (
|
||||
echo gdal2tiles wrapper: QGIS not found. Set GDAL2TILES_QGIS_ROOT to the QGIS install directory. 1>&2
|
||||
exit /b 2
|
||||
)
|
||||
|
||||
if not exist "%QGIS_ROOT%\bin\o4w_env.bat" (
|
||||
echo gdal2tiles wrapper: "%QGIS_ROOT%\bin\o4w_env.bat" is missing; is this a QGIS install? 1>&2
|
||||
exit /b 2
|
||||
)
|
||||
if not exist "%QGIS_ROOT%\apps\Python312\Scripts\gdal2tiles.exe" (
|
||||
echo gdal2tiles wrapper: gdal2tiles.exe not found under "%QGIS_ROOT%\apps\Python312\Scripts". 1>&2
|
||||
exit /b 2
|
||||
)
|
||||
|
||||
call "%QGIS_ROOT%\bin\o4w_env.bat" >nul 2>&1
|
||||
"%QGIS_ROOT%\apps\Python312\Scripts\gdal2tiles.exe" %*
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,33 @@
|
||||
@echo off
|
||||
setlocal enableextensions
|
||||
REM Run gdalbuildvrt from the GDAL bundled with QGIS.
|
||||
REM
|
||||
REM QGIS ships GDAL but its Python bindings need the OSGeo4W environment
|
||||
REM (GDAL_DATA, PROJ_LIB). Loading o4w_env.bat first keeps PROJ working.
|
||||
REM
|
||||
REM Override auto-detection with: set GDAL2TILES_QGIS_ROOT=C:\path\to\QGIS
|
||||
|
||||
set "QGIS_ROOT=%GDAL2TILES_QGIS_ROOT%"
|
||||
if not "%QGIS_ROOT%"=="" goto have_root
|
||||
|
||||
for /f "delims=" %%D in ('dir /b /ad /o-n "%ProgramFiles%\QGIS *" 2^>nul') do (
|
||||
if not defined QGIS_ROOT set "QGIS_ROOT=%ProgramFiles%\%%D"
|
||||
)
|
||||
|
||||
:have_root
|
||||
if "%QGIS_ROOT%"=="" (
|
||||
echo gdalbuildvrt wrapper: QGIS not found. Set GDAL2TILES_QGIS_ROOT to the QGIS install directory. 1>&2
|
||||
exit /b 2
|
||||
)
|
||||
if not exist "%QGIS_ROOT%\bin\o4w_env.bat" (
|
||||
echo gdalbuildvrt wrapper: "%QGIS_ROOT%\bin\o4w_env.bat" is missing; is this a QGIS install? 1>&2
|
||||
exit /b 2
|
||||
)
|
||||
if not exist "%QGIS_ROOT%\bin\gdalbuildvrt.exe" (
|
||||
echo gdalbuildvrt wrapper: gdalbuildvrt.exe not found under "%QGIS_ROOT%\bin". 1>&2
|
||||
exit /b 2
|
||||
)
|
||||
|
||||
call "%QGIS_ROOT%\bin\o4w_env.bat" >nul 2>&1
|
||||
"%QGIS_ROOT%\bin\gdalbuildvrt.exe" %*
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,18 @@
|
||||
@echo off
|
||||
set "ENV_FILE=%~1"
|
||||
if "%ENV_FILE%"=="" set "ENV_FILE=%~dp0..\config\service.env"
|
||||
if not exist "%ENV_FILE%" exit /b 0
|
||||
|
||||
for /f "usebackq eol=# tokens=1,* delims==" %%A in ("%ENV_FILE%") do (
|
||||
if not "%%~A"=="" call :set_if_missing "%%~A" "%%~B"
|
||||
)
|
||||
exit /b 0
|
||||
|
||||
:set_if_missing
|
||||
call set "__current=%%%~1%%"
|
||||
if defined __current goto :clear_and_exit
|
||||
set "%~1=%~2"
|
||||
|
||||
:clear_and_exit
|
||||
set "__current="
|
||||
exit /b 0
|
||||
@@ -0,0 +1,41 @@
|
||||
@echo off
|
||||
setlocal
|
||||
chcp 65001 >nul 2>&1
|
||||
title Basemap Pusher
|
||||
|
||||
set "ROOT=%~dp0..\\"
|
||||
call "%ROOT%scripts\load-env.bat" "%ROOT%config\service.env"
|
||||
|
||||
if "%GOCACHE%"=="" set "GOCACHE=%ROOT%.gocache"
|
||||
if "%GOMODCACHE%"=="" set "GOMODCACHE=%ROOT%.gomodcache"
|
||||
if "%GOTMPDIR%"=="" set "GOTMPDIR=%ROOT%.gotmp"
|
||||
if "%GOPATH%"=="" set "GOPATH=%ROOT%.gopath"
|
||||
if "%GOPROXY%"=="" set "GOPROXY=https://goproxy.cn,direct"
|
||||
if "%GOSUMDB%"=="" set "GOSUMDB=off"
|
||||
if "%API_HOST%"=="" set "API_HOST=127.0.0.1"
|
||||
if "%API_PORT%"=="" set "API_PORT=8910"
|
||||
if "%API_BASE_URL%"=="" set "API_BASE_URL=http://127.0.0.1:%API_PORT%"
|
||||
if "%TILE_PUBLIC_BASE_URL%"=="" set "TILE_PUBLIC_BASE_URL=%API_BASE_URL%"
|
||||
if "%VECTOR_PUBLIC_BASE_URL%"=="" set "VECTOR_PUBLIC_BASE_URL=%API_BASE_URL%"
|
||||
if "%SQLITE_PATH%"=="" set "SQLITE_PATH=%ROOT%data\basemap-pusher.db"
|
||||
if "%BASEMAP_SCAN_ROOT%"=="" set "BASEMAP_SCAN_ROOT=%ROOT%data\tiles"
|
||||
if "%VECTOR_SCAN_ROOT%"=="" set "VECTOR_SCAN_ROOT=%ROOT%data\geojson"
|
||||
|
||||
echo ========================================
|
||||
echo Basemap Pusher
|
||||
echo ========================================
|
||||
echo.
|
||||
echo Env file: %ROOT%config\service.env
|
||||
echo Listen: %API_BASE_URL%
|
||||
echo Tile base: %TILE_PUBLIC_BASE_URL%
|
||||
echo Vector base: %VECTOR_PUBLIC_BASE_URL%
|
||||
echo SQLite: %SQLITE_PATH%
|
||||
echo Scan root: %BASEMAP_SCAN_ROOT%
|
||||
echo Vector root: %VECTOR_SCAN_ROOT%
|
||||
echo.
|
||||
|
||||
pushd "%ROOT%services\api-go"
|
||||
go run .\cmd\api
|
||||
popd
|
||||
|
||||
if "%NO_PAUSE%"=="" pause
|
||||
@@ -0,0 +1,67 @@
|
||||
@echo off
|
||||
setlocal
|
||||
chcp 65001 >nul 2>&1
|
||||
title Basemap Pusher Service
|
||||
|
||||
set "ROOT=%~dp0..\"
|
||||
pushd "%ROOT%" >nul 2>&1
|
||||
set "ROOT=%CD%\"
|
||||
popd
|
||||
|
||||
set "EXE=%ROOT%bin\basemap-pusher.exe"
|
||||
set "ENVFILE=%ROOT%config\service.env"
|
||||
|
||||
if not exist "%EXE%" (
|
||||
echo Service binary not found: %EXE%
|
||||
echo Build it first: scripts\build-api.bat
|
||||
if "%NO_PAUSE%"=="" pause
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
set "VERB=%~1"
|
||||
if "%VERB%"=="" set "VERB=status"
|
||||
|
||||
if /i "%VERB%"=="install" goto do_install
|
||||
if /i "%VERB%"=="uninstall" goto do_uninstall
|
||||
if /i "%VERB%"=="start" goto do_start
|
||||
if /i "%VERB%"=="stop" goto do_stop
|
||||
if /i "%VERB%"=="restart" goto do_restart
|
||||
if /i "%VERB%"=="status" goto do_status
|
||||
|
||||
echo Usage: service.bat ^<install^|uninstall^|start^|stop^|restart^|status^>
|
||||
exit /b 2
|
||||
|
||||
:do_install
|
||||
"%EXE%" --env-file "%ENVFILE%" install
|
||||
goto finish
|
||||
|
||||
:do_uninstall
|
||||
"%EXE%" uninstall
|
||||
goto finish
|
||||
|
||||
:do_start
|
||||
"%EXE%" start
|
||||
goto finish
|
||||
|
||||
:do_stop
|
||||
"%EXE%" stop
|
||||
goto finish
|
||||
|
||||
:do_restart
|
||||
"%EXE%" stop
|
||||
"%EXE%" start
|
||||
goto finish
|
||||
|
||||
:do_status
|
||||
"%EXE%" status
|
||||
goto finish
|
||||
|
||||
:finish
|
||||
set "RC=%ERRORLEVEL%"
|
||||
if not "%RC%"=="0" (
|
||||
echo.
|
||||
echo [FAIL] exit code %RC%.
|
||||
echo install / uninstall / start / stop need an elevated shell ^(Run as Administrator^).
|
||||
)
|
||||
if "%NO_PAUSE%"=="" pause
|
||||
exit /b %RC%
|
||||
Reference in New Issue
Block a user