Harmon/map-asset-gateway
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rewrite the gateway around health, retries, slicing and a Windows service

Browse Source 
Scanner
- manifest lookup is now depth-bounded and prunes numeric zoom directories, so
  a scan no longer walks the whole tile payload
- rescans only report a version as updated when something actually changed
- overlapping runs are refused instead of interleaving their bookkeeping

Hot path
- token authorisation loads only that token grants and throttles the
  last_used_at write instead of touching it on every tile
- SQLite runs in WAL with a busy timeout and a real connection pool
- the catalog lookup used by /tiles no longer loads every basemap

Security
- the admin surface requires ADMIN_TOKEN (bearer) or ADMIN_USER/ADMIN_PASSWORD
  (basic), and the service refuses to bind off-host without credentials
- CORS is an explicit allow-list; newly minted tokens never reach a redirect URL
- client tokens are redacted from the access log

Operations
- resource health inspection flags a moved or deleted tile root before a 404
- pushes retry with backoff and can be re-queued by hand; the schedule lives in
  the database so a restart does not lose it
- access log plus /metrics counters

Image slicing
- tile_jobs drive the GDAL bundled with QGIS via scripts/gdal2tiles-qgis.bat
- layer_sources remembers what a layer is made of: submit one source to append
  a flight, or all of them to rebuild the layer as a VRT mosaic with no tile
  seams; the console exposes both, including a one-click rebuild
- slicing is available to integrations through a narrow slice token scope
  (submit, poll, idempotent by code, completion callback)

Console
- rebuilt around a fixed rail and single-view navigation with health and
  outstanding work leading, instead of one long scrolling page

Windows service
- native install/start/stop/status with crash recovery, auto-start and a
  rotating log file, so no NSSM wrapper is needed
- start-all.bat and stop-all.bat are removed: they killed the process by port
  with taskkill, which fought the service restart policy

Verified with 49 tests and live runs against real tile data and real GDAL.
This commit is contained in:
Harmon
2026-10-16 10:35:30 +08:00
parent 8faf25b436
commit 73ad03fa59
49 changed files with 8668 additions and 1817 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/api-go/internal/basemap/store.go
+206 -14
View File
@@ -11,37 +11,97 @@ import (
"errors"
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
_ "modernc.org/sqlite"
)
const (
defaultScanSourceCode = "local-basemaps"
defaultManifestName = "manifest.json"
readPermission = "read"
defaultScanSourceCode = "local-basemaps"
defaultManifestName = "manifest.json"
readPermission = "read"
defaultVectorSourceCode = "local-vectors"
)
const (
// sqliteMaxOpenConns lets concurrent tile reads overlap instead of being
// serialized behind a single connection.
sqliteMaxOpenConns = 8
)
// sqliteDSN tunes each pooled connection. busy_timeout makes concurrent
// writers wait for the lock instead of failing immediately.
func sqliteDSN(path string) string {
params := url.Values{}
params.Add("_pragma", "busy_timeout(5000)")
params.Add("_pragma", "foreign_keys(1)")
params.Add("_pragma", "synchronous(NORMAL)")
return "file:" + filepath.ToSlash(path) + "?" + params.Encode()
}
var (
// ErrScanInProgress is returned when a scan is already running.
ErrScanInProgress = errors.New("scan already in progress")
// ErrScanSourceNotFound is returned when a scan source code is unknown.
ErrScanSourceNotFound = errors.New("scan source not found")
// ErrScanSourceDisabled is returned when a scan source is turned off.
ErrScanSourceDisabled = errors.New("scan source disabled")
)
// scanSourceError keeps the operator-facing message stable while still matching
// errors.Is against a sentinel.
type scanSourceError struct {
sentinel error
code string
reason string
}
func (e *scanSourceError) Error() string {
return fmt.Sprintf("scan source %q %s", e.code, e.reason)
}
func (e *scanSourceError) Is(target error) bool {
return target == e.sentinel
}
type StoreConfig struct {
SQLitePath string
ScanRoot string
VectorScanRoot string
APIBaseURL string
TileBaseURL string
VectorBaseURL string
AutoScanOnStart bool
SQLitePath string
ScanRoot string
VectorScanRoot string
APIBaseURL string
TileBaseURL string
VectorBaseURL string
AutoScanOnStart bool
TokenDefaultTTL time.Duration
TileJobCommand string
TileJobVRTCommand string
TileJobLogDir string
HealthCheckInterval time.Duration
}
type Store struct {
db *sql.DB
cfg StoreConfig
httpClient *http.Client
// scanMu serializes scans so two runs cannot interleave their read-then-write
// bookkeeping.
scanMu sync.Mutex
// pushWG tracks detached push callbacks so Close can drain them.
pushWG sync.WaitGroup
bgOnce sync.Once
bgCancel context.CancelFunc
bgCtx context.Context
bgWG sync.WaitGroup
}
type CreatedToken struct {
@@ -66,13 +126,21 @@ func OpenStore(cfg StoreConfig) (*Store, error) {
return nil, fmt.Errorf("create sqlite dir: %w", err)
}
db, err := sql.Open("sqlite", cfg.SQLitePath)
db, err := sql.Open("sqlite", sqliteDSN(cfg.SQLitePath))
if err != nil {
return nil, fmt.Errorf("open sqlite: %w", err)
}
db.SetConnMaxLifetime(10 * time.Minute)
db.SetMaxOpenConns(1)
db.SetMaxIdleConns(1)
db.SetConnMaxLifetime(30 * time.Minute)
db.SetConnMaxIdleTime(5 * time.Minute)
db.SetMaxOpenConns(sqliteMaxOpenConns)
db.SetMaxIdleConns(sqliteMaxOpenConns)
// WAL keeps tile reads from blocking on scans/pushes. It is persisted in the
// database file, so enabling it once is enough; a filesystem that cannot
// support WAL (some network shares) must not stop the service from starting.
if _, err := db.Exec("PRAGMA journal_mode=WAL;"); err != nil {
log.Printf("enable sqlite WAL skipped: %v", err)
}
store := &Store{
db: db,
@@ -109,6 +177,13 @@ func (s *Store) Close() error {
if s == nil || s.db == nil {
return nil
}
// Stop the periodic loops, then let detached push callbacks finish before
// the pool goes away.
if s.bgCancel != nil {
s.bgCancel()
}
s.bgWG.Wait()
s.pushWG.Wait()
return s.db.Close()
}
@@ -142,6 +217,9 @@ func (s *Store) bootstrap(ctx context.Context) error {
bbox_json TEXT NOT NULL DEFAULT '',
attribution TEXT NOT NULL DEFAULT '',
metadata_json TEXT NOT NULL DEFAULT '',
health TEXT NOT NULL DEFAULT '',
health_checked_at TEXT NOT NULL DEFAULT '',
health_detail TEXT NOT NULL DEFAULT '',
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
UNIQUE(basemap_id, version),
@@ -188,6 +266,7 @@ func (s *Store) bootstrap(ctx context.Context) error {
name TEXT NOT NULL,
token_hash TEXT NOT NULL UNIQUE,
token_prefix TEXT NOT NULL,
scopes TEXT NOT NULL DEFAULT 'read',
status TEXT NOT NULL,
expires_at TEXT,
last_used_at TEXT,
@@ -213,6 +292,8 @@ func (s *Store) bootstrap(ctx context.Context) error {
response_status INTEGER,
response_body TEXT NOT NULL DEFAULT '',
error_message TEXT NOT NULL DEFAULT '',
attempt_count INTEGER NOT NULL DEFAULT 0,
next_attempt_at TEXT,
pushed_at TEXT NOT NULL,
finished_at TEXT,
FOREIGN KEY (target_system_id) REFERENCES target_systems(id) ON DELETE CASCADE,
@@ -231,6 +312,9 @@ func (s *Store) bootstrap(ctx context.Context) error {
checksum TEXT NOT NULL DEFAULT '',
download_url TEXT NOT NULL,
metadata_json TEXT NOT NULL DEFAULT '',
health TEXT NOT NULL DEFAULT '',
health_checked_at TEXT NOT NULL DEFAULT '',
health_detail TEXT NOT NULL DEFAULT '',
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);`,
@@ -252,15 +336,55 @@ func (s *Store) bootstrap(ctx context.Context) error {
response_status INTEGER,
response_body TEXT NOT NULL DEFAULT '',
error_message TEXT NOT NULL DEFAULT '',
attempt_count INTEGER NOT NULL DEFAULT 0,
next_attempt_at TEXT,
pushed_at TEXT NOT NULL,
finished_at TEXT,
FOREIGN KEY (target_system_id) REFERENCES target_systems(id) ON DELETE CASCADE,
FOREIGN KEY (vector_asset_id) REFERENCES vector_assets(id) ON DELETE CASCADE
);`,
`CREATE TABLE IF NOT EXISTS tile_jobs (
id TEXT PRIMARY KEY,
code TEXT NOT NULL UNIQUE,
name TEXT NOT NULL,
source_path TEXT NOT NULL,
source_count INTEGER NOT NULL DEFAULT 1,
sources_json TEXT NOT NULL DEFAULT '',
target_code TEXT NOT NULL,
output_root TEXT NOT NULL,
zoom_min INTEGER NOT NULL DEFAULT 0,
zoom_max INTEGER NOT NULL DEFAULT 0,
tile_size INTEGER NOT NULL DEFAULT 256,
tile_format TEXT NOT NULL DEFAULT 'webp',
status TEXT NOT NULL,
progress TEXT NOT NULL DEFAULT '',
message TEXT NOT NULL DEFAULT '',
log_path TEXT NOT NULL DEFAULT '',
callback_url TEXT NOT NULL DEFAULT '',
callback_status TEXT NOT NULL DEFAULT '',
callback_attempts INTEGER NOT NULL DEFAULT 0,
callback_next_attempt_at TEXT,
callback_message TEXT NOT NULL DEFAULT '',
requested_by TEXT NOT NULL DEFAULT '',
created_at TEXT NOT NULL,
started_at TEXT,
finished_at TEXT
);`,
`CREATE INDEX IF NOT EXISTS idx_tile_jobs_status ON tile_jobs(status, created_at);`,
`CREATE INDEX IF NOT EXISTS idx_tile_jobs_target ON tile_jobs(target_code);`,
`CREATE TABLE IF NOT EXISTS layer_sources (
id TEXT PRIMARY KEY,
target_code TEXT NOT NULL,
source_path TEXT NOT NULL,
added_at TEXT NOT NULL,
UNIQUE(target_code, source_path)
);`,
`CREATE INDEX IF NOT EXISTS idx_layer_sources_target ON layer_sources(target_code);`,
`CREATE INDEX IF NOT EXISTS idx_versions_basemap ON basemap_versions(basemap_id);`,
`CREATE INDEX IF NOT EXISTS idx_versions_source ON basemap_versions(scan_source_id);`,
`CREATE INDEX IF NOT EXISTS idx_grants_token ON token_grants(token_id);`,
`CREATE INDEX IF NOT EXISTS idx_push_target ON push_records(target_system_id);`,
`CREATE INDEX IF NOT EXISTS idx_vector_file_name ON vector_assets(file_name COLLATE NOCASE);`,
`CREATE INDEX IF NOT EXISTS idx_vector_grants_token ON vector_token_grants(token_id);`,
`CREATE INDEX IF NOT EXISTS idx_vector_push_target ON vector_push_records(target_system_id);`,
}
@@ -270,9 +394,77 @@ func (s *Store) bootstrap(ctx context.Context) error {
return fmt.Errorf("bootstrap sqlite: %w", err)
}
}
// CREATE TABLE IF NOT EXISTS never evolves an existing database, so columns
// added after the first release are patched in here.
migrations := []struct {
table string
column string
definition string
}{
{"basemap_versions", "health", "TEXT NOT NULL DEFAULT ''"},
{"basemap_versions", "health_checked_at", "TEXT NOT NULL DEFAULT ''"},
{"basemap_versions", "health_detail", "TEXT NOT NULL DEFAULT ''"},
{"vector_assets", "health", "TEXT NOT NULL DEFAULT ''"},
{"vector_assets", "health_checked_at", "TEXT NOT NULL DEFAULT ''"},
{"vector_assets", "health_detail", "TEXT NOT NULL DEFAULT ''"},
{"push_records", "attempt_count", "INTEGER NOT NULL DEFAULT 0"},
{"push_records", "next_attempt_at", "TEXT"},
{"vector_push_records", "attempt_count", "INTEGER NOT NULL DEFAULT 0"},
{"vector_push_records", "next_attempt_at", "TEXT"},
{"service_tokens", "scopes", "TEXT NOT NULL DEFAULT 'read'"},
{"tile_jobs", "source_count", "INTEGER NOT NULL DEFAULT 1"},
{"tile_jobs", "sources_json", "TEXT NOT NULL DEFAULT ''"},
{"tile_jobs", "callback_url", "TEXT NOT NULL DEFAULT ''"},
{"tile_jobs", "callback_status", "TEXT NOT NULL DEFAULT ''"},
{"tile_jobs", "callback_attempts", "INTEGER NOT NULL DEFAULT 0"},
{"tile_jobs", "callback_next_attempt_at", "TEXT"},
{"tile_jobs", "callback_message", "TEXT NOT NULL DEFAULT ''"},
}
for _, migration := range migrations {
if err := s.ensureColumn(ctx, migration.table, migration.column, migration.definition); err != nil {
return fmt.Errorf("bootstrap migrate %s.%s: %w", migration.table, migration.column, err)
}
}
// These indexes reference migrated columns, so they run after the patch.
for _, statement := range []string{
`CREATE INDEX IF NOT EXISTS idx_push_next_attempt ON push_records(next_attempt_at);`,
`CREATE INDEX IF NOT EXISTS idx_vector_push_next_attempt ON vector_push_records(next_attempt_at);`,
} {
if _, err := s.db.ExecContext(ctx, statement); err != nil {
return fmt.Errorf("bootstrap index: %w", err)
}
}
return nil
}
// ensureColumn adds a column when an existing table is missing it. PRAGMA
// table_info is the portable way to test SQLite column presence.
func (s *Store) ensureColumn(ctx context.Context, table, column, definition string) error {
rows, err := s.db.QueryContext(ctx, "PRAGMA table_info("+table+")")
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var cid, notNull, primaryKey int
var name, columnType string
var defaultValue sql.NullString
if err := rows.Scan(&cid, &name, &columnType, &notNull, &defaultValue, &primaryKey); err != nil {
return err
}
if name == column {
return nil
}
}
if err := rows.Err(); err != nil {
return err
}
_, err = s.db.ExecContext(ctx, "ALTER TABLE "+table+" ADD COLUMN "+column+" "+definition)
return err
}
func (s *Store) AutoScanEnabled() bool {
return s.cfg.AutoScanOnStart
}