Rewrite the gateway around health, retries, slicing and a Windows service

Scanner
- manifest lookup is now depth-bounded and prunes numeric zoom directories, so
  a scan no longer walks the whole tile payload
- rescans only report a version as updated when something actually changed
- overlapping runs are refused instead of interleaving their bookkeeping

Hot path
- token authorisation loads only that token grants and throttles the
  last_used_at write instead of touching it on every tile
- SQLite runs in WAL with a busy timeout and a real connection pool
- the catalog lookup used by /tiles no longer loads every basemap

Security
- the admin surface requires ADMIN_TOKEN (bearer) or ADMIN_USER/ADMIN_PASSWORD
  (basic), and the service refuses to bind off-host without credentials
- CORS is an explicit allow-list; newly minted tokens never reach a redirect URL
- client tokens are redacted from the access log

Operations
- resource health inspection flags a moved or deleted tile root before a 404
- pushes retry with backoff and can be re-queued by hand; the schedule lives in
  the database so a restart does not lose it
- access log plus /metrics counters

Image slicing
- tile_jobs drive the GDAL bundled with QGIS via scripts/gdal2tiles-qgis.bat
- layer_sources remembers what a layer is made of: submit one source to append
  a flight, or all of them to rebuild the layer as a VRT mosaic with no tile
  seams; the console exposes both, including a one-click rebuild
- slicing is available to integrations through a narrow slice token scope
  (submit, poll, idempotent by code, completion callback)

Console
- rebuilt around a fixed rail and single-view navigation with health and
  outstanding work leading, instead of one long scrolling page

Windows service
- native install/start/stop/status with crash recovery, auto-start and a
  rotating log file, so no NSSM wrapper is needed
- start-all.bat and stop-all.bat are removed: they killed the process by port
  with taskkill, which fought the service restart policy

Verified with 49 tests and live runs against real tile data and real GDAL.
This commit is contained in:
2026-10-16 10:35:30 +08:00
parent 8faf25b436
commit 73ad03fa59
49 changed files with 8668 additions and 1817 deletions
+136 -18
View File
@@ -2,36 +2,111 @@ package main
import (
"context"
"errors"
"fmt"
"log"
"net/http"
"os"
"os/signal"
"path/filepath"
"strconv"
"strings"
"syscall"
"time"
"map-asset-gateway/api-go/internal/basemap"
"map-asset-gateway/api-go/internal/basemapapi"
"map-asset-gateway/api-go/internal/config"
"tile-server/api-go/internal/basemap"
"tile-server/api-go/internal/basemapapi"
"tile-server/api-go/internal/config"
)
const shutdownGrace = 15 * time.Second
type cliArgs struct {
command string
envFile string
}
func main() {
args, err := parseArgs(os.Args[1:])
if err != nil {
log.Fatalf("parse arguments: %v", err)
}
if args.envFile != "" {
if err := os.Setenv("APP_ENV_FILE", args.envFile); err != nil {
log.Fatalf("set APP_ENV_FILE: %v", err)
}
}
// Service management verbs (install / uninstall / start / stop / status).
if handled, err := handleServiceArgs(args.command, args.envFile); handled {
if err != nil {
log.Fatalf("%v", err)
}
return
}
// When the SCM launched us there is no console and no SIGINT, so the service
// handler owns the lifecycle instead of the signal handler.
if runAsWindowsService() {
return
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if err := run(ctx); err != nil {
log.Fatalf("%v", err)
}
}
func parseArgs(argv []string) (cliArgs, error) {
var args cliArgs
for i := 0; i < len(argv); i++ {
switch argv[i] {
case "--env-file":
if i+1 >= len(argv) {
return args, errors.New("--env-file requires a path")
}
args.envFile = argv[i+1]
i++
default:
if args.command == "" {
args.command = argv[i]
}
}
}
return args, nil
}
// run boots the service and blocks until ctx is cancelled or the listener dies.
// It owns every resource it creates.
func run(ctx context.Context) error {
cfg, err := config.Load()
if err != nil {
log.Fatalf("load config: %v", err)
return fmt.Errorf("load config: %w", err)
}
// The admin surface can create tokens that read every basemap, so it must
// never be reachable off-host without credentials.
if cfg.BindsToNonLoopback() && !cfg.HasAdminCredentials() {
return fmt.Errorf("refusing to listen on %s without admin credentials: set ADMIN_TOKEN or ADMIN_USER/ADMIN_PASSWORD", cfg.ListenAddr())
}
store, err := basemap.OpenStore(basemap.StoreConfig{
SQLitePath: cfg.SQLitePath,
ScanRoot: cfg.BasemapScanRoot,
VectorScanRoot: cfg.VectorScanRoot,
APIBaseURL: cfg.APIBaseURL,
TileBaseURL: cfg.TilePublicBaseURL,
VectorBaseURL: cfg.VectorPublicBaseURL,
AutoScanOnStart: cfg.AutoScanOnStart,
SQLitePath: cfg.SQLitePath,
ScanRoot: cfg.BasemapScanRoot,
VectorScanRoot: cfg.VectorScanRoot,
APIBaseURL: cfg.APIBaseURL,
TileBaseURL: cfg.TilePublicBaseURL,
VectorBaseURL: cfg.VectorPublicBaseURL,
AutoScanOnStart: cfg.AutoScanOnStart,
TokenDefaultTTL: cfg.TokenDefaultTTL,
TileJobCommand: cfg.TileJobCommand,
TileJobVRTCommand: cfg.TileJobVRTCommand,
TileJobLogDir: cfg.TileJobLogDir,
HealthCheckInterval: cfg.HealthCheckInterval,
})
if err != nil {
log.Fatalf("open sqlite store: %v", err)
return fmt.Errorf("open sqlite store: %w", err)
}
defer func() {
if err := store.Close(); err != nil {
@@ -50,6 +125,7 @@ func main() {
SQLitePath: cfg.SQLitePath,
ScanRoot: cfg.BasemapScanRoot,
VectorScanRoot: cfg.VectorScanRoot,
TileJobCommand: cfg.TileJobCommand,
StartedAt: time.Now().UTC(),
}
@@ -68,10 +144,21 @@ func main() {
}
}
server, err := basemapapi.New(store, systemInfo)
// Health inspection, push retries and the tile queue run in the background.
store.StartBackgroundWorkers()
server, err := basemapapi.New(store, systemInfo, basemapapi.Options{
Admin: basemapapi.AdminAuth{
Token: cfg.AdminToken,
User: cfg.AdminUser,
Password: cfg.AdminPassword,
},
CORSAllowedOrigins: cfg.CORSAllowedOrigins,
})
if err != nil {
log.Fatalf("init api server: %v", err)
return fmt.Errorf("init api server: %w", err)
}
httpServer := &http.Server{
Addr: cfg.ListenAddr(),
Handler: server.Handler(),
@@ -79,22 +166,53 @@ func main() {
}
cleanupPID, err := writePIDFile(strings.TrimSpace(os.Getenv("RS_API_PID_FILE")))
if err != nil {
log.Fatalf("write pid file: %v", err)
return fmt.Errorf("write pid file: %w", err)
}
if cleanupPID != nil {
defer cleanupPID()
}
log.Printf("map asset gateway listening on http://%s", cfg.ListenAddr())
log.Printf("api base url: %s", cfg.APIBaseURL)
log.Printf("tile public base url: %s", cfg.TilePublicBaseURL)
log.Printf("vector public base url: %s", cfg.VectorPublicBaseURL)
log.Printf("sqlite path: %s", cfg.SQLitePath)
log.Printf("scan root: %s", cfg.BasemapScanRoot)
log.Printf("vector scan root: %s", cfg.VectorScanRoot)
if cfg.HasAdminCredentials() {
log.Printf("admin auth: enabled")
} else {
log.Printf("admin auth: disabled (loopback only)")
}
if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Fatalf("listen and serve: %v", err)
serveErr := make(chan error, 1)
go func() {
log.Printf("basemap pusher listening on http://%s", cfg.ListenAddr())
if err := httpServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
serveErr <- err
return
}
serveErr <- nil
}()
select {
case err := <-serveErr:
if err != nil {
return fmt.Errorf("listen and serve: %w", err)
}
return nil
case <-ctx.Done():
log.Printf("shutdown signal received, draining connections")
shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownGrace)
defer cancel()
if err := httpServer.Shutdown(shutdownCtx); err != nil {
log.Printf("graceful shutdown incomplete: %v", err)
_ = httpServer.Close()
}
if err := <-serveErr; err != nil {
log.Printf("server stopped with error: %v", err)
}
log.Printf("shutdown complete")
return nil
}
}