Rewrite the gateway around health, retries, slicing and a Windows service

Scanner
- manifest lookup is now depth-bounded and prunes numeric zoom directories, so
  a scan no longer walks the whole tile payload
- rescans only report a version as updated when something actually changed
- overlapping runs are refused instead of interleaving their bookkeeping

Hot path
- token authorisation loads only that token grants and throttles the
  last_used_at write instead of touching it on every tile
- SQLite runs in WAL with a busy timeout and a real connection pool
- the catalog lookup used by /tiles no longer loads every basemap

Security
- the admin surface requires ADMIN_TOKEN (bearer) or ADMIN_USER/ADMIN_PASSWORD
  (basic), and the service refuses to bind off-host without credentials
- CORS is an explicit allow-list; newly minted tokens never reach a redirect URL
- client tokens are redacted from the access log

Operations
- resource health inspection flags a moved or deleted tile root before a 404
- pushes retry with backoff and can be re-queued by hand; the schedule lives in
  the database so a restart does not lose it
- access log plus /metrics counters

Image slicing
- tile_jobs drive the GDAL bundled with QGIS via scripts/gdal2tiles-qgis.bat
- layer_sources remembers what a layer is made of: submit one source to append
  a flight, or all of them to rebuild the layer as a VRT mosaic with no tile
  seams; the console exposes both, including a one-click rebuild
- slicing is available to integrations through a narrow slice token scope
  (submit, poll, idempotent by code, completion callback)

Console
- rebuilt around a fixed rail and single-view navigation with health and
  outstanding work leading, instead of one long scrolling page

Windows service
- native install/start/stop/status with crash recovery, auto-start and a
  rotating log file, so no NSSM wrapper is needed
- start-all.bat and stop-all.bat are removed: they killed the process by port
  with taskkill, which fought the service restart policy

Verified with 49 tests and live runs against real tile data and real GDAL.
This commit is contained in:
2026-10-16 10:35:30 +08:00
parent 8faf25b436
commit 73ad03fa59
49 changed files with 8668 additions and 1817 deletions
+145 -16
View File
@@ -12,10 +12,15 @@ import (
"path/filepath"
"sort"
"strings"
"time"
"map-asset-gateway/api-go/internal/uid"
"tile-server/api-go/internal/uid"
)
// lastUsedUpdateInterval throttles service_tokens.last_used_at writes so tile
// traffic does not issue one UPDATE per tile request.
const lastUsedUpdateInterval = time.Minute
var ErrBasemapVersionNotFound = errors.New("basemap version not found")
func (s *Store) CreateToken(ctx context.Context, input CreateTokenInput) (CreatedToken, error) {
@@ -23,11 +28,16 @@ func (s *Store) CreateToken(ctx context.Context, input CreateTokenInput) (Create
if name == "" {
return CreatedToken{}, errors.New("token name is required")
}
if len(input.BasemapCodes) == 0 {
if len(input.VectorCodes) == 0 {
return CreatedToken{}, errors.New("at least one basemap code or vector code is required")
// A token needs at least one capability: read grants, or the slice scope.
hasSliceScope := false
for _, value := range input.Scopes {
if strings.EqualFold(strings.TrimSpace(value), slicePermission) {
hasSliceScope = true
}
}
if len(input.BasemapCodes) == 0 && len(input.VectorCodes) == 0 && !hasSliceScope {
return CreatedToken{}, errors.New("at least one basemap code, vector code, or the slice scope is required")
}
resolvedVectorAssets := map[string]VectorAsset{}
for _, ref := range input.VectorCodes {
@@ -54,15 +64,21 @@ func (s *Store) CreateToken(ctx context.Context, input CreateTokenInput) (Create
}
now := nowUTC()
// Fall back to the configured default lifetime so tokens do not live forever
// unless an explicit expiry was supplied.
if input.ExpiresAt == nil && s.cfg.TokenDefaultTTL > 0 {
expiresAt := now.Add(s.cfg.TokenDefaultTTL)
input.ExpiresAt = &expiresAt
}
tokenID := uid.New()
prefix := token
if len(prefix) > 16 {
prefix = prefix[:16]
}
_, err = tx.ExecContext(ctx, `
INSERT INTO service_tokens (id, name, token_hash, token_prefix, status, expires_at, last_used_at, created_at)
VALUES (?, ?, ?, ?, 'active', ?, NULL, ?)
`, tokenID, name, tokenHash, prefix, nullableTime(input.ExpiresAt), toRFC3339(now))
INSERT INTO service_tokens (id, name, token_hash, token_prefix, scopes, status, expires_at, last_used_at, created_at)
VALUES (?, ?, ?, ?, ?, 'active', ?, NULL, ?)
`, tokenID, name, tokenHash, prefix, normalizeTokenScopes(input.Scopes), nullableTime(input.ExpiresAt), toRFC3339(now))
if err != nil {
return CreatedToken{}, err
}
@@ -75,7 +91,7 @@ func (s *Store) CreateToken(ctx context.Context, input CreateTokenInput) (Create
}
}
if len(normalizedCodes) == 0 {
if len(resolvedVectorAssets) == 0 {
if len(resolvedVectorAssets) == 0 && !hasSliceScope {
return CreatedToken{}, errors.New("no valid basemap codes provided")
}
}
@@ -129,7 +145,7 @@ func (s *Store) CreateToken(ctx context.Context, input CreateTokenInput) (Create
func (s *Store) ListTokens(ctx context.Context) ([]ServiceToken, error) {
rows, err := s.db.QueryContext(ctx, `
SELECT id, name, token_hash, token_prefix, status, expires_at, last_used_at, created_at
SELECT id, name, token_hash, token_prefix, scopes, status, expires_at, last_used_at, created_at
FROM service_tokens
ORDER BY created_at DESC
`)
@@ -226,6 +242,69 @@ func (s *Store) loadAllVectorTokenGrants(ctx context.Context) (map[string][]Vect
return result, rows.Err()
}
func (s *Store) loadTokenGrants(ctx context.Context, tokenID string) ([]TokenGrant, error) {
rows, err := s.db.QueryContext(ctx, `
SELECT
g.id,
g.token_id,
g.basemap_id,
b.code,
g.basemap_version_id,
v.version,
g.permission,
g.created_at
FROM token_grants g
JOIN basemaps b ON b.id = g.basemap_id
LEFT JOIN basemap_versions v ON v.id = g.basemap_version_id
WHERE g.token_id = ?
ORDER BY g.created_at DESC
`, tokenID)
if err != nil {
return nil, err
}
defer rows.Close()
var items []TokenGrant
for rows.Next() {
item, err := scanTokenGrant(rows)
if err != nil {
return nil, err
}
items = append(items, item)
}
return items, rows.Err()
}
func (s *Store) loadVectorTokenGrants(ctx context.Context, tokenID string) ([]VectorTokenGrant, error) {
rows, err := s.db.QueryContext(ctx, `
SELECT
g.id,
g.token_id,
g.vector_asset_id,
v.code,
g.permission,
g.created_at
FROM vector_token_grants g
JOIN vector_assets v ON v.id = g.vector_asset_id
WHERE g.token_id = ?
ORDER BY g.created_at DESC
`, tokenID)
if err != nil {
return nil, err
}
defer rows.Close()
var items []VectorTokenGrant
for rows.Next() {
item, err := scanVectorTokenGrant(rows)
if err != nil {
return nil, err
}
items = append(items, item)
}
return items, rows.Err()
}
func (s *Store) DisableToken(ctx context.Context, tokenID string) error {
result, err := s.db.ExecContext(ctx, `UPDATE service_tokens SET status = 'disabled' WHERE id = ?`, strings.TrimSpace(tokenID))
if err != nil {
@@ -243,7 +322,7 @@ func (s *Store) DisableToken(ctx context.Context, tokenID string) error {
func (s *Store) getTokenByID(ctx context.Context, tokenID string) (ServiceToken, error) {
row := s.db.QueryRowContext(ctx, `
SELECT id, name, token_hash, token_prefix, status, expires_at, last_used_at, created_at
SELECT id, name, token_hash, token_prefix, scopes, status, expires_at, last_used_at, created_at
FROM service_tokens
WHERE id = ?
`, tokenID)
@@ -273,7 +352,7 @@ func (s *Store) AuthorizeToken(ctx context.Context, rawToken string) (TokenAuth,
tokenHash := hex.EncodeToString(sum[:])
row := s.db.QueryRowContext(ctx, `
SELECT id, name, token_hash, token_prefix, status, expires_at, last_used_at, created_at
SELECT id, name, token_hash, token_prefix, scopes, status, expires_at, last_used_at, created_at
FROM service_tokens
WHERE token_hash = ?
`, tokenHash)
@@ -291,16 +370,16 @@ func (s *Store) AuthorizeToken(ctx context.Context, rawToken string) (TokenAuth,
return TokenAuth{}, errors.New("token is expired")
}
grantsByToken, err := s.loadAllTokenGrants(ctx)
grants, err := s.loadTokenGrants(ctx, item.ID)
if err != nil {
return TokenAuth{}, err
}
item.Grants = grantsByToken[item.ID]
vectorGrantsByToken, err := s.loadAllVectorTokenGrants(ctx)
item.Grants = grants
vectorGrants, err := s.loadVectorTokenGrants(ctx, item.ID)
if err != nil {
return TokenAuth{}, err
}
item.VectorGrants = vectorGrantsByToken[item.ID]
item.VectorGrants = vectorGrants
permissions := map[string]map[string]VersionScope{}
for _, grant := range item.Grants {
group, ok := permissions[grant.Permission]
@@ -329,14 +408,64 @@ func (s *Store) AuthorizeToken(ctx context.Context, rawToken string) (TokenAuth,
}
group[grant.VectorCode] = struct{}{}
}
_, _ = s.db.ExecContext(ctx, `UPDATE service_tokens SET last_used_at = ? WHERE id = ?`, toRFC3339(nowUTC()), item.ID)
if item.LastUsedAt == nil || nowUTC().Sub(*item.LastUsedAt) >= lastUsedUpdateInterval {
_, _ = s.db.ExecContext(ctx, `UPDATE service_tokens SET last_used_at = ? WHERE id = ?`, toRFC3339(nowUTC()), item.ID)
}
return TokenAuth{
Token: item,
Permissions: permissions,
VectorPermissions: vectorPermissions,
Scopes: scopeSet(item.Scopes),
}, nil
}
// Token scopes are coarse capabilities layered on top of per-resource grants.
// Every token can read; "slice" additionally allows submitting tile jobs.
const slicePermission = "slice"
var allowedTokenScopes = map[string]struct{}{
readPermission: {},
slicePermission: {},
}
// normalizeTokenScopes keeps only known scopes and always includes read, so the
// stored column is a stable, sorted, comma-separated list.
func normalizeTokenScopes(values []string) string {
scopes := map[string]struct{}{readPermission: {}}
for _, value := range values {
scope := strings.ToLower(strings.TrimSpace(value))
if _, ok := allowedTokenScopes[scope]; ok {
scopes[scope] = struct{}{}
}
}
return strings.Join(sortedKeys(scopes), ",")
}
func splitTokenScopes(value string) []string {
parts := strings.Split(value, ",")
scopes := make([]string, 0, len(parts))
for _, part := range parts {
scope := strings.ToLower(strings.TrimSpace(part))
if scope != "" {
scopes = append(scopes, scope)
}
}
return scopes
}
func scopeSet(values []string) map[string]struct{} {
scopes := make(map[string]struct{}, len(values))
for _, value := range values {
scopes[value] = struct{}{}
}
return scopes
}
// CanSlice reports whether the token may submit tile jobs.
func (auth TokenAuth) CanSlice() bool {
_, ok := auth.Scopes[slicePermission]
return ok
}
func (auth TokenAuth) CanReadBasemap(code string, version string) bool {
group, ok := auth.Permissions[readPermission]
if !ok {