Rewrite the gateway around health, retries, slicing and a Windows service
Scanner - manifest lookup is now depth-bounded and prunes numeric zoom directories, so a scan no longer walks the whole tile payload - rescans only report a version as updated when something actually changed - overlapping runs are refused instead of interleaving their bookkeeping Hot path - token authorisation loads only that token grants and throttles the last_used_at write instead of touching it on every tile - SQLite runs in WAL with a busy timeout and a real connection pool - the catalog lookup used by /tiles no longer loads every basemap Security - the admin surface requires ADMIN_TOKEN (bearer) or ADMIN_USER/ADMIN_PASSWORD (basic), and the service refuses to bind off-host without credentials - CORS is an explicit allow-list; newly minted tokens never reach a redirect URL - client tokens are redacted from the access log Operations - resource health inspection flags a moved or deleted tile root before a 404 - pushes retry with backoff and can be re-queued by hand; the schedule lives in the database so a restart does not lose it - access log plus /metrics counters Image slicing - tile_jobs drive the GDAL bundled with QGIS via scripts/gdal2tiles-qgis.bat - layer_sources remembers what a layer is made of: submit one source to append a flight, or all of them to rebuild the layer as a VRT mosaic with no tile seams; the console exposes both, including a one-click rebuild - slicing is available to integrations through a narrow slice token scope (submit, poll, idempotent by code, completion callback) Console - rebuilt around a fixed rail and single-view navigation with health and outstanding work leading, instead of one long scrolling page Windows service - native install/start/stop/status with crash recovery, auto-start and a rotating log file, so no NSSM wrapper is needed - start-all.bat and stop-all.bat are removed: they killed the process by port with taskkill, which fought the service restart policy Verified with 49 tests and live runs against real tile data and real GDAL.
This commit is contained in:
@@ -12,7 +12,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"map-asset-gateway/api-go/internal/uid"
|
||||
"tile-server/api-go/internal/uid"
|
||||
)
|
||||
|
||||
type scannedVersion struct {
|
||||
@@ -34,13 +34,30 @@ type scannedVersion struct {
|
||||
IsDefault bool
|
||||
}
|
||||
|
||||
// scanCounts reports what a scan actually changed, so operators can tell a real
|
||||
// update from a no-op rescan.
|
||||
type scanCounts struct {
|
||||
Scanned int
|
||||
Added int
|
||||
Updated int
|
||||
Removed int
|
||||
Unchanged int
|
||||
}
|
||||
|
||||
func (s *Store) RunScan(ctx context.Context, sourceCode string) (ScanRun, error) {
|
||||
// A scan reads the current catalog and then rewrites it, so two overlapping
|
||||
// runs could delete each other's rows. Refuse instead of interleaving.
|
||||
if !s.scanMu.TryLock() {
|
||||
return ScanRun{}, ErrScanInProgress
|
||||
}
|
||||
defer s.scanMu.Unlock()
|
||||
|
||||
source, err := s.GetScanSourceByCode(ctx, sourceCode)
|
||||
if err != nil {
|
||||
return ScanRun{}, err
|
||||
}
|
||||
if !source.Enabled {
|
||||
return ScanRun{}, fmt.Errorf("scan source %q is disabled", sourceCode)
|
||||
return ScanRun{}, &scanSourceError{sentinel: ErrScanSourceDisabled, code: sourceCode, reason: "is disabled"}
|
||||
}
|
||||
|
||||
startedAt := nowUTC()
|
||||
@@ -58,21 +75,25 @@ func (s *Store) RunScan(ctx context.Context, sourceCode string) (ScanRun, error)
|
||||
}
|
||||
|
||||
func (s *Store) runScan(ctx context.Context, runID string, source ScanSource, startedAt time.Time) (ScanRun, error) {
|
||||
fail := func(err error) (ScanRun, error) {
|
||||
return s.finishScanRun(ctx, runID, source, startedAt, "failed", nil, scanCounts{}, err)
|
||||
}
|
||||
|
||||
rootInfo, err := os.Stat(source.RootPath)
|
||||
if err != nil {
|
||||
return s.finishScanRun(ctx, runID, source, startedAt, "failed", nil, 0, 0, 0, 0, fmt.Errorf("stat scan root: %w", err))
|
||||
return fail(fmt.Errorf("stat scan root: %w", err))
|
||||
}
|
||||
if !rootInfo.IsDir() {
|
||||
return s.finishScanRun(ctx, runID, source, startedAt, "failed", nil, 0, 0, 0, 0, fmt.Errorf("scan root %s is not a directory", source.RootPath))
|
||||
return fail(fmt.Errorf("scan root %s is not a directory", source.RootPath))
|
||||
}
|
||||
|
||||
manifestItems, coveredRoots, err := s.findManifestVersions(source)
|
||||
if err != nil {
|
||||
return s.finishScanRun(ctx, runID, source, startedAt, "failed", nil, 0, 0, 0, 0, err)
|
||||
return fail(err)
|
||||
}
|
||||
fallbackItems, err := s.findDirectTileRoots(source, coveredRoots)
|
||||
if err != nil {
|
||||
return s.finishScanRun(ctx, runID, source, startedAt, "failed", nil, 0, 0, 0, 0, err)
|
||||
return fail(err)
|
||||
}
|
||||
|
||||
allItems := append(manifestItems, fallbackItems...)
|
||||
@@ -83,11 +104,12 @@ func (s *Store) runScan(ctx context.Context, runID string, source ScanSource, st
|
||||
return allItems[i].BasemapCode < allItems[j].BasemapCode
|
||||
})
|
||||
|
||||
addedCount, updatedCount, removedCount, err := s.persistScanResult(ctx, source, allItems)
|
||||
counts, err := s.persistScanResult(ctx, source, allItems)
|
||||
if err != nil {
|
||||
return s.finishScanRun(ctx, runID, source, startedAt, "failed", allItems, len(allItems), addedCount, updatedCount, removedCount, err)
|
||||
return s.finishScanRun(ctx, runID, source, startedAt, "failed", allItems, scanCounts{Scanned: len(allItems)}, err)
|
||||
}
|
||||
return s.finishScanRun(ctx, runID, source, startedAt, "succeeded", allItems, len(allItems), addedCount, updatedCount, removedCount, nil)
|
||||
counts.Scanned = len(allItems)
|
||||
return s.finishScanRun(ctx, runID, source, startedAt, "succeeded", allItems, counts, nil)
|
||||
}
|
||||
|
||||
func (s *Store) finishScanRun(
|
||||
@@ -97,20 +119,18 @@ func (s *Store) finishScanRun(
|
||||
startedAt time.Time,
|
||||
status string,
|
||||
items []scannedVersion,
|
||||
scannedCount int,
|
||||
addedCount int,
|
||||
updatedCount int,
|
||||
removedCount int,
|
||||
counts scanCounts,
|
||||
runErr error,
|
||||
) (ScanRun, error) {
|
||||
finishedAt := nowUTC()
|
||||
summary := map[string]any{
|
||||
"source_code": source.Code,
|
||||
"scan_root": source.RootPath,
|
||||
"scanned_count": scannedCount,
|
||||
"added_count": addedCount,
|
||||
"updated_count": updatedCount,
|
||||
"removed_count": removedCount,
|
||||
"source_code": source.Code,
|
||||
"scan_root": source.RootPath,
|
||||
"scanned_count": counts.Scanned,
|
||||
"added_count": counts.Added,
|
||||
"updated_count": counts.Updated,
|
||||
"removed_count": counts.Removed,
|
||||
"unchanged_count": counts.Unchanged,
|
||||
}
|
||||
if runErr != nil {
|
||||
summary["error"] = runErr.Error()
|
||||
@@ -127,7 +147,7 @@ func (s *Store) finishScanRun(
|
||||
UPDATE scan_runs
|
||||
SET status = ?, scanned_count = ?, added_count = ?, updated_count = ?, removed_count = ?, summary_json = ?, finished_at = ?
|
||||
WHERE id = ?
|
||||
`, status, scannedCount, addedCount, updatedCount, removedCount, writeJSON(summary), toRFC3339(finishedAt), runID)
|
||||
`, status, counts.Scanned, counts.Added, counts.Updated, counts.Removed, writeJSON(summary), toRFC3339(finishedAt), runID)
|
||||
if updateErr != nil {
|
||||
if runErr != nil {
|
||||
return ScanRun{}, fmt.Errorf("%v; update scan run: %w", runErr, updateErr)
|
||||
@@ -140,10 +160,10 @@ func (s *Store) finishScanRun(
|
||||
ScanSourceID: source.ID,
|
||||
SourceCode: source.Code,
|
||||
Status: status,
|
||||
ScannedCount: scannedCount,
|
||||
AddedCount: addedCount,
|
||||
UpdatedCount: updatedCount,
|
||||
RemovedCount: removedCount,
|
||||
ScannedCount: counts.Scanned,
|
||||
AddedCount: counts.Added,
|
||||
UpdatedCount: counts.Updated,
|
||||
RemovedCount: counts.Removed,
|
||||
Summary: summary,
|
||||
StartedAt: startedAt,
|
||||
FinishedAt: finishedAt,
|
||||
@@ -154,6 +174,11 @@ func (s *Store) finishScanRun(
|
||||
return run, nil
|
||||
}
|
||||
|
||||
// manifestScanMaxDepth bounds how deep the manifest lookup descends. Manifest
|
||||
// based basemaps place manifest.json at the basemap root or one version folder
|
||||
// below it, so anything deeper is tile payload that must not be walked.
|
||||
const manifestScanMaxDepth = 4
|
||||
|
||||
func (s *Store) findManifestVersions(source ScanSource) ([]scannedVersion, map[string]struct{}, error) {
|
||||
manifestName := strings.TrimSpace(source.ManifestName)
|
||||
if manifestName == "" {
|
||||
@@ -162,69 +187,103 @@ func (s *Store) findManifestVersions(source ScanSource) ([]scannedVersion, map[s
|
||||
|
||||
var items []scannedVersion
|
||||
coveredRoots := map[string]struct{}{}
|
||||
err := filepath.WalkDir(source.RootPath, func(path string, entry os.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
|
||||
var walk func(dir string, depth int) error
|
||||
walk = func(dir string, depth int) error {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if !strings.EqualFold(entry.Name(), manifestName) {
|
||||
|
||||
// A directory holding a manifest is a basemap root: record it and stop,
|
||||
// so the tile payload underneath (the default "tiles" subtree) is never
|
||||
// traversed.
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() || !strings.EqualFold(entry.Name(), manifestName) {
|
||||
continue
|
||||
}
|
||||
manifestPath := filepath.Join(dir, entry.Name())
|
||||
manifest, err := LoadManifest(manifestPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tileRoot, err := resolveTileRoot(dir, manifest.RootPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
format, minZoom, maxZoom, found, err := inspectTileRoot(tileRoot, manifest.TileFormat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if found {
|
||||
manifest.TileFormat = format
|
||||
manifest.MinZoom = minZoom
|
||||
manifest.MaxZoom = maxZoom
|
||||
}
|
||||
|
||||
if relPath, err := filepath.Rel(source.RootPath, tileRoot); err == nil {
|
||||
first := strings.Split(filepath.ToSlash(relPath), "/")[0]
|
||||
if first != "" && first != "." {
|
||||
coveredRoots[first] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
items = append(items, scannedVersion{
|
||||
BasemapCode: manifest.Code,
|
||||
Name: defaultText(manifest.Name, manifest.Code),
|
||||
Type: defaultText(manifest.Type, "xyz"),
|
||||
Status: defaultText(manifest.Status, "ready"),
|
||||
Description: manifest.Description,
|
||||
Version: defaultText(manifest.Version, "current"),
|
||||
Manifest: manifestPath,
|
||||
TileRoot: tileRoot,
|
||||
TileFormat: normalizeTileFormat(manifest.TileFormat),
|
||||
TileScheme: defaultText(manifest.TileScheme, "xyz"),
|
||||
MinZoom: manifest.MinZoom,
|
||||
MaxZoom: manifest.MaxZoom,
|
||||
BBox: manifest.BBox,
|
||||
Attribution: manifest.Attribution,
|
||||
Metadata: manifest.Metadata,
|
||||
IsDefault: manifest.IsDefault,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
manifest, err := LoadManifest(path)
|
||||
if err != nil {
|
||||
return err
|
||||
if depth >= manifestScanMaxDepth {
|
||||
return nil
|
||||
}
|
||||
tileRoot, err := resolveTileRoot(filepath.Dir(path), manifest.RootPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
format, minZoom, maxZoom, found, err := inspectTileRoot(tileRoot, manifest.TileFormat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if found {
|
||||
manifest.TileFormat = format
|
||||
manifest.MinZoom = minZoom
|
||||
manifest.MaxZoom = maxZoom
|
||||
}
|
||||
|
||||
relPath, err := filepath.Rel(source.RootPath, tileRoot)
|
||||
if err == nil {
|
||||
first := strings.Split(filepath.ToSlash(relPath), "/")[0]
|
||||
if first != "" && first != "." {
|
||||
coveredRoots[first] = struct{}{}
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() || isNumericDirName(entry.Name()) {
|
||||
continue
|
||||
}
|
||||
if err := walk(filepath.Join(dir, entry.Name()), depth+1); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
items = append(items, scannedVersion{
|
||||
BasemapCode: manifest.Code,
|
||||
Name: defaultText(manifest.Name, manifest.Code),
|
||||
Type: defaultText(manifest.Type, "xyz"),
|
||||
Status: defaultText(manifest.Status, "ready"),
|
||||
Description: manifest.Description,
|
||||
Version: defaultText(manifest.Version, "current"),
|
||||
Manifest: path,
|
||||
TileRoot: tileRoot,
|
||||
TileFormat: normalizeTileFormat(manifest.TileFormat),
|
||||
TileScheme: defaultText(manifest.TileScheme, "xyz"),
|
||||
MinZoom: manifest.MinZoom,
|
||||
MaxZoom: manifest.MaxZoom,
|
||||
BBox: manifest.BBox,
|
||||
Attribution: manifest.Attribution,
|
||||
Metadata: manifest.Metadata,
|
||||
IsDefault: manifest.IsDefault,
|
||||
})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
}
|
||||
|
||||
if err := walk(source.RootPath, 0); err != nil {
|
||||
return nil, nil, fmt.Errorf("walk manifests: %w", err)
|
||||
}
|
||||
return items, coveredRoots, nil
|
||||
}
|
||||
|
||||
// isNumericDirName reports whether the directory name is a bare zoom level such
|
||||
// as "0" or "14". Raw tile roots nest z/x/y under the basemap directory, so
|
||||
// pruning these keeps the scan clear of the tile payload.
|
||||
func isNumericDirName(name string) bool {
|
||||
if name == "" {
|
||||
return false
|
||||
}
|
||||
for _, r := range name {
|
||||
if r < '0' || r > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Store) findDirectTileRoots(source ScanSource, skip map[string]struct{}) ([]scannedVersion, error) {
|
||||
entries, err := os.ReadDir(source.RootPath)
|
||||
if err != nil {
|
||||
@@ -352,57 +411,114 @@ func resolveTileRoot(baseDir, relativePath string) (string, error) {
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
func (s *Store) persistScanResult(ctx context.Context, source ScanSource, items []scannedVersion) (int, int, int, error) {
|
||||
// existingVersion is a snapshot of a stored version, used to detect whether a
|
||||
// rescan actually changed anything.
|
||||
type existingVersion struct {
|
||||
Name string
|
||||
Type string
|
||||
Status string
|
||||
Description string
|
||||
ManifestPath string
|
||||
TileRootPath string
|
||||
TileFormat string
|
||||
TileScheme string
|
||||
MinZoom int
|
||||
MaxZoom int
|
||||
IsDefault bool
|
||||
BBoxJSON string
|
||||
Attribution string
|
||||
MetadataJSON string
|
||||
}
|
||||
|
||||
func (s *Store) persistScanResult(ctx context.Context, source ScanSource, items []scannedVersion) (scanCounts, error) {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, 0, 0, err
|
||||
return scanCounts{}, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
existing := map[string]struct{}{}
|
||||
existing := map[string]existingVersion{}
|
||||
rows, err := tx.QueryContext(ctx, `
|
||||
SELECT b.code, v.version
|
||||
SELECT
|
||||
b.code,
|
||||
v.version,
|
||||
b.name,
|
||||
b.type,
|
||||
b.status,
|
||||
b.description,
|
||||
v.manifest_path,
|
||||
v.tile_root_path,
|
||||
v.tile_format,
|
||||
v.tile_scheme,
|
||||
v.min_zoom,
|
||||
v.max_zoom,
|
||||
v.is_default,
|
||||
v.bbox_json,
|
||||
v.attribution,
|
||||
v.metadata_json
|
||||
FROM basemap_versions v
|
||||
JOIN basemaps b ON b.id = v.basemap_id
|
||||
WHERE v.scan_source_id = ?
|
||||
`, source.ID)
|
||||
if err != nil {
|
||||
return 0, 0, 0, err
|
||||
return scanCounts{}, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var code string
|
||||
var version string
|
||||
if err := rows.Scan(&code, &version); err != nil {
|
||||
var item existingVersion
|
||||
var isDefault int
|
||||
if err := rows.Scan(
|
||||
&code,
|
||||
&version,
|
||||
&item.Name,
|
||||
&item.Type,
|
||||
&item.Status,
|
||||
&item.Description,
|
||||
&item.ManifestPath,
|
||||
&item.TileRootPath,
|
||||
&item.TileFormat,
|
||||
&item.TileScheme,
|
||||
&item.MinZoom,
|
||||
&item.MaxZoom,
|
||||
&isDefault,
|
||||
&item.BBoxJSON,
|
||||
&item.Attribution,
|
||||
&item.MetadataJSON,
|
||||
); err != nil {
|
||||
rows.Close()
|
||||
return 0, 0, 0, err
|
||||
return scanCounts{}, err
|
||||
}
|
||||
existing[code+"::"+version] = struct{}{}
|
||||
item.IsDefault = isDefault == 1
|
||||
existing[code+"::"+version] = item
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return 0, 0, 0, err
|
||||
return scanCounts{}, err
|
||||
}
|
||||
|
||||
now := nowUTC()
|
||||
found := map[string]struct{}{}
|
||||
touchedBasemaps := map[string]struct{}{}
|
||||
addedCount := 0
|
||||
updatedCount := 0
|
||||
counts := scanCounts{}
|
||||
for _, item := range items {
|
||||
key := item.BasemapCode + "::" + item.Version
|
||||
found[key] = struct{}{}
|
||||
if _, exists := existing[key]; exists {
|
||||
updatedCount++
|
||||
if prev, exists := existing[key]; exists {
|
||||
if scannedVersionMatches(item, prev) {
|
||||
counts.Unchanged++
|
||||
} else {
|
||||
counts.Updated++
|
||||
}
|
||||
} else {
|
||||
addedCount++
|
||||
counts.Added++
|
||||
}
|
||||
if err := s.upsertScannedVersionTx(ctx, tx, source, item, now); err != nil {
|
||||
return 0, 0, 0, err
|
||||
return scanCounts{}, err
|
||||
}
|
||||
touchedBasemaps[item.BasemapCode] = struct{}{}
|
||||
}
|
||||
|
||||
removedCount := 0
|
||||
for key := range existing {
|
||||
if _, ok := found[key]; ok {
|
||||
continue
|
||||
@@ -412,22 +528,41 @@ func (s *Store) persistScanResult(ctx context.Context, source ScanSource, items
|
||||
continue
|
||||
}
|
||||
if err := s.deleteVersionBySourceTx(ctx, tx, source.ID, parts[0], parts[1]); err != nil {
|
||||
return 0, 0, 0, err
|
||||
return scanCounts{}, err
|
||||
}
|
||||
touchedBasemaps[parts[0]] = struct{}{}
|
||||
removedCount++
|
||||
counts.Removed++
|
||||
}
|
||||
|
||||
for code := range touchedBasemaps {
|
||||
if err := s.normalizeBasemapDefaultsTx(ctx, tx, code, now); err != nil {
|
||||
return 0, 0, 0, err
|
||||
return scanCounts{}, err
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return 0, 0, 0, err
|
||||
return scanCounts{}, err
|
||||
}
|
||||
return addedCount, updatedCount, removedCount, nil
|
||||
return counts, nil
|
||||
}
|
||||
|
||||
// scannedVersionMatches reports whether a freshly scanned version is identical
|
||||
// to what is already stored, so an unchanged rescan is not reported as updated.
|
||||
func scannedVersionMatches(item scannedVersion, prev existingVersion) bool {
|
||||
return item.Name == prev.Name &&
|
||||
item.Type == prev.Type &&
|
||||
item.Status == prev.Status &&
|
||||
item.Description == prev.Description &&
|
||||
item.Manifest == prev.ManifestPath &&
|
||||
item.TileRoot == prev.TileRootPath &&
|
||||
normalizeTileFormat(item.TileFormat) == prev.TileFormat &&
|
||||
item.TileScheme == prev.TileScheme &&
|
||||
item.MinZoom == prev.MinZoom &&
|
||||
item.MaxZoom == prev.MaxZoom &&
|
||||
item.IsDefault == prev.IsDefault &&
|
||||
writeJSON(item.BBox) == prev.BBoxJSON &&
|
||||
item.Attribution == prev.Attribution &&
|
||||
writeJSON(item.Metadata) == prev.MetadataJSON
|
||||
}
|
||||
|
||||
func (s *Store) upsertScannedVersionTx(ctx context.Context, tx *sql.Tx, source ScanSource, item scannedVersion, now time.Time) error {
|
||||
|
||||
Reference in New Issue
Block a user