Rewrite the gateway around health, retries, slicing and a Windows service

Scanner - manifest lookup is now depth-bounded and prunes numeric zoom directories, so a scan no longer walks the whole tile payload - rescans only report a version as updated when something actually changed - overlapping runs are refused instead of interleaving their bookkeeping Hot path - token authorisation loads only that token grants and throttles the last_used_at write instead of touching it on every tile - SQLite runs in WAL with a busy timeout and a real connection pool - the catalog lookup used by /tiles no longer loads every basemap Security - the admin surface requires ADMIN_TOKEN (bearer) or ADMIN_USER/ADMIN_PASSWORD (basic), and the service refuses to bind off-host without credentials - CORS is an explicit allow-list; newly minted tokens never reach a redirect URL - client tokens are redacted from the access log Operations - resource health inspection flags a moved or deleted tile root before a 404 - pushes retry with backoff and can be re-queued by hand; the schedule lives in the database so a restart does not lose it - access log plus /metrics counters Image slicing - tile_jobs drive the GDAL bundled with QGIS via scripts/gdal2tiles-qgis.bat - layer_sources remembers what a layer is made of: submit one source to append a flight, or all of them to rebuild the layer as a VRT mosaic with no tile seams; the console exposes both, including a one-click rebuild - slicing is available to integrations through a narrow slice token scope (submit, poll, idempotent by code, completion callback) Console - rebuilt around a fixed rail and single-view navigation with health and outstanding work leading, instead of one long scrolling page Windows service - native install/start/stop/status with crash recovery, auto-start and a rotating log file, so no NSSM wrapper is needed - start-all.bat and stop-all.bat are removed: they killed the process by port with taskkill, which fought the service restart policy Verified with 49 tests and live runs against real tile data and real GDAL.
2026-10-16 10:35:30 +08:00
parent 8faf25b436
commit 73ad03fa59
49 changed files with 8668 additions and 1817 deletions
@@ -0,0 +1,279 @@
+package basemapapi
+
+import (
+	"context"
+	"encoding/base64"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"tile-server/api-go/internal/basemap"
+)
+
+func newTestStore(t *testing.T, root string) *basemap.Store {
+	t.Helper()
+	tileRoot := filepath.Join(root, "tiles")
+	writeFile(t, filepath.Join(tileRoot, "google_image", "0", "0", "0.webp"), "tile")
+	vectorRoot := filepath.Join(root, "vectors")
+
+	store, err := basemap.OpenStore(basemap.StoreConfig{
+		SQLitePath:     filepath.Join(root, "basemap.db"),
+		ScanRoot:       tileRoot,
+		VectorScanRoot: vectorRoot,
+		APIBaseURL:     "http://127.0.0.1:8910",
+		TileBaseURL:    "http://127.0.0.1:8910",
+		VectorBaseURL:  "http://127.0.0.1:8910",
+	})
+	if err != nil {
+		t.Fatalf("open store: %v", err)
+	}
+	t.Cleanup(func() { _ = store.Close() })
+
+	if _, err := store.RunScan(context.Background(), "local-basemaps"); err != nil {
+		t.Fatalf("run scan: %v", err)
+	}
+	return store
+}
+
+func writeFile(t *testing.T, path, body string) {
+	t.Helper()
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		t.Fatalf("mkdir: %v", err)
+	}
+	if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
+		t.Fatalf("write file: %v", err)
+	}
+}
+
+func newTestServer(t *testing.T, store *basemap.Store, options Options) *httptest.Server {
+	t.Helper()
+	server, err := New(store, basemap.SystemInfo{ServiceName: "basemap-pusher"}, options)
+	if err != nil {
+		t.Fatalf("new server: %v", err)
+	}
+	httpServer := httptest.NewServer(server.Handler())
+	t.Cleanup(httpServer.Close)
+	return httpServer
+}
+
+func basicAuthHeader(user, password string) string {
+	return "Basic " + base64.StdEncoding.EncodeToString([]byte(user+":"+password))
+}
+
+func TestAdminRequiresBasicAuth(t *testing.T) {
+	store := newTestStore(t, t.TempDir())
+	httpServer := newTestServer(t, store, Options{Admin: AdminAuth{User: "ops", Password: "s3cret"}})
+
+	for _, path := range []string{"/api/v1/admin/basemaps", "/admin/"} {
+		response, err := http.Get(httpServer.URL + path)
+		if err != nil {
+			t.Fatalf("unauthorized GET %s: %v", path, err)
+		}
+		response.Body.Close()
+		if response.StatusCode != http.StatusUnauthorized {
+			t.Fatalf("expected 401 for anonymous %s, got %d", path, response.StatusCode)
+		}
+
+		request, err := http.NewRequest(http.MethodGet, httpServer.URL+path, nil)
+		if err != nil {
+			t.Fatalf("build request: %v", err)
+		}
+		request.Header.Set("Authorization", basicAuthHeader("ops", "s3cret"))
+		response, err = http.DefaultClient.Do(request)
+		if err != nil {
+			t.Fatalf("authorized GET %s: %v", path, err)
+		}
+		response.Body.Close()
+		if response.StatusCode != http.StatusOK {
+			t.Fatalf("expected 200 for authorized %s, got %d", path, response.StatusCode)
+		}
+	}
+}
+
+func TestAdminRejectsWrongBasicAuth(t *testing.T) {
+	store := newTestStore(t, t.TempDir())
+	httpServer := newTestServer(t, store, Options{Admin: AdminAuth{User: "ops", Password: "s3cret"}})
+
+	request, _ := http.NewRequest(http.MethodGet, httpServer.URL+"/api/v1/admin/basemaps", nil)
+	request.Header.Set("Authorization", basicAuthHeader("ops", "wrong"))
+	response, err := http.DefaultClient.Do(request)
+	if err != nil {
+		t.Fatalf("request: %v", err)
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusUnauthorized {
+		t.Fatalf("expected 401, got %d", response.StatusCode)
+	}
+}
+
+func TestAdminAcceptsBearerToken(t *testing.T) {
+	store := newTestStore(t, t.TempDir())
+	httpServer := newTestServer(t, store, Options{Admin: AdminAuth{Token: "admin-token"}})
+
+	request, _ := http.NewRequest(http.MethodGet, httpServer.URL+"/api/v1/admin/basemaps", nil)
+	request.Header.Set("Authorization", "Bearer admin-token")
+	response, err := http.DefaultClient.Do(request)
+	if err != nil {
+		t.Fatalf("request: %v", err)
+	}
+	response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		t.Fatalf("expected 200 with bearer token, got %d", response.StatusCode)
+	}
+
+	request, _ = http.NewRequest(http.MethodGet, httpServer.URL+"/api/v1/admin/basemaps", nil)
+	request.Header.Set("Authorization", "Bearer nope")
+	response, err = http.DefaultClient.Do(request)
+	if err != nil {
+		t.Fatalf("request: %v", err)
+	}
+	response.Body.Close()
+	if response.StatusCode != http.StatusUnauthorized {
+		t.Fatalf("expected 401 with a wrong bearer token, got %d", response.StatusCode)
+	}
+}
+
+func TestAdminOpenWhenUnconfigured(t *testing.T) {
+	store := newTestStore(t, t.TempDir())
+	httpServer := newTestServer(t, store, Options{})
+
+	response, err := http.Get(httpServer.URL + "/api/v1/admin/basemaps")
+	if err != nil {
+		t.Fatalf("request: %v", err)
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		t.Fatalf("expected the admin API to be open without credentials, got %d", response.StatusCode)
+	}
+}
+
+func TestCatalogStillRequiresTokenEvenWithAdminAuth(t *testing.T) {
+	store := newTestStore(t, t.TempDir())
+	httpServer := newTestServer(t, store, Options{Admin: AdminAuth{User: "ops", Password: "s3cret"}})
+
+	response, err := http.Get(httpServer.URL + "/api/v1/catalog/basemaps")
+	if err != nil {
+		t.Fatalf("request: %v", err)
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusUnauthorized {
+		t.Fatalf("expected the catalog to still require a service token, got %d", response.StatusCode)
+	}
+}
+
+func TestCORSAllowlist(t *testing.T) {
+	store := newTestStore(t, t.TempDir())
+	httpServer := newTestServer(t, store, Options{CORSAllowedOrigins: []string{"https://maps.example.com"}})
+
+	allowed, _ := http.NewRequest(http.MethodGet, httpServer.URL+"/health", nil)
+	allowed.Header.Set("Origin", "https://maps.example.com")
+	response, err := http.DefaultClient.Do(allowed)
+	if err != nil {
+		t.Fatalf("allowed origin: %v", err)
+	}
+	response.Body.Close()
+	if got := response.Header.Get("Access-Control-Allow-Origin"); got != "https://maps.example.com" {
+		t.Fatalf("expected the allowlisted origin to be reflected, got %q", got)
+	}
+
+	blocked, _ := http.NewRequest(http.MethodGet, httpServer.URL+"/health", nil)
+	blocked.Header.Set("Origin", "https://evil.example.com")
+	response, err = http.DefaultClient.Do(blocked)
+	if err != nil {
+		t.Fatalf("blocked origin: %v", err)
+	}
+	response.Body.Close()
+	if got := response.Header.Get("Access-Control-Allow-Origin"); got != "" {
+		t.Fatalf("expected no CORS header for an unlisted origin, got %q", got)
+	}
+}
+
+func TestCreateTokenFormKeepsTokenOutOfURL(t *testing.T) {
+	store := newTestStore(t, t.TempDir())
+	httpServer := newTestServer(t, store, Options{})
+
+	form := url.Values{}
+	form.Set("name", "console")
+	form.Set("basemap_codes", "google_image")
+	request, err := http.NewRequest(http.MethodPost, httpServer.URL+"/admin/tokens", strings.NewReader(form.Encode()))
+	if err != nil {
+		t.Fatalf("build request: %v", err)
+	}
+	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	client := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
+	response, err := client.Do(request)
+	if err != nil {
+		t.Fatalf("post form: %v", err)
+	}
+	defer response.Body.Close()
+	body, _ := io.ReadAll(response.Body)
+
+	if location := response.Header.Get("Location"); strings.Contains(location, "bmp_") {
+		t.Fatalf("token leaked into the redirect URL: %s", location)
+	}
+	if !strings.Contains(string(body), "bmp_") {
+		t.Fatalf("expected the new token to be rendered in the response body")
+	}
+}
+
+func TestMetricsEndpointCountsTileMisses(t *testing.T) {
+	store := newTestStore(t, t.TempDir())
+	httpServer := newTestServer(t, store, Options{})
+
+	created, err := store.CreateToken(context.Background(), basemap.CreateTokenInput{
+		Name:         "metrics",
+		BasemapCodes: []string{"google_image"},
+	})
+	if err != nil {
+		t.Fatalf("create token: %v", err)
+	}
+
+	response, err := http.Get(httpServer.URL + "/tiles/google_image/current/9/9/9.png?token=" + created.Token)
+	if err != nil {
+		t.Fatalf("tile request: %v", err)
+	}
+	response.Body.Close()
+	if response.StatusCode != http.StatusNotFound {
+		t.Fatalf("expected 404 for a missing tile, got %d", response.StatusCode)
+	}
+
+	response, err = http.Get(httpServer.URL + "/metrics")
+	if err != nil {
+		t.Fatalf("metrics request: %v", err)
+	}
+	defer response.Body.Close()
+	body, _ := io.ReadAll(response.Body)
+	text := string(body)
+	for _, expected := range []string{
+		"basemap_pusher_requests_total",
+		"basemap_pusher_tile_requests_total 1",
+		"basemap_pusher_tile_misses_total 1",
+	} {
+		if !strings.Contains(text, expected) {
+			t.Fatalf("expected /metrics to contain %q, got:\n%s", expected, text)
+		}
+	}
+}
+
+func TestScanErrorStatusMapping(t *testing.T) {
+	cases := []struct {
+		err  error
+		want int
+	}{
+		{basemap.ErrScanInProgress, http.StatusConflict},
+		{basemap.ErrScanSourceNotFound, http.StatusNotFound},
+		{basemap.ErrScanSourceDisabled, http.StatusBadRequest},
+		{io.ErrUnexpectedEOF, http.StatusInternalServerError},
+	}
+	for _, tc := range cases {
+		if got := scanErrorStatus(tc.err); got != tc.want {
+			t.Fatalf("scanErrorStatus(%v) = %d, want %d", tc.err, got, tc.want)
+		}
+	}
+}