Rewrite the gateway around health, retries, slicing and a Windows service

Scanner - manifest lookup is now depth-bounded and prunes numeric zoom directories, so a scan no longer walks the whole tile payload - rescans only report a version as updated when something actually changed - overlapping runs are refused instead of interleaving their bookkeeping Hot path - token authorisation loads only that token grants and throttles the last_used_at write instead of touching it on every tile - SQLite runs in WAL with a busy timeout and a real connection pool - the catalog lookup used by /tiles no longer loads every basemap Security - the admin surface requires ADMIN_TOKEN (bearer) or ADMIN_USER/ADMIN_PASSWORD (basic), and the service refuses to bind off-host without credentials - CORS is an explicit allow-list; newly minted tokens never reach a redirect URL - client tokens are redacted from the access log Operations - resource health inspection flags a moved or deleted tile root before a 404 - pushes retry with backoff and can be re-queued by hand; the schedule lives in the database so a restart does not lose it - access log plus /metrics counters Image slicing - tile_jobs drive the GDAL bundled with QGIS via scripts/gdal2tiles-qgis.bat - layer_sources remembers what a layer is made of: submit one source to append a flight, or all of them to rebuild the layer as a VRT mosaic with no tile seams; the console exposes both, including a one-click rebuild - slicing is available to integrations through a narrow slice token scope (submit, poll, idempotent by code, completion callback) Console - rebuilt around a fixed rail and single-view navigation with health and outstanding work leading, instead of one long scrolling page Windows service - native install/start/stop/status with crash recovery, auto-start and a rotating log file, so no NSSM wrapper is needed - start-all.bat and stop-all.bat are removed: they killed the process by port with taskkill, which fought the service restart policy Verified with 49 tests and live runs against real tile data and real GDAL.
2026-10-16 10:35:30 +08:00
parent 8faf25b436
commit 73ad03fa59
49 changed files with 8668 additions and 1817 deletions
@@ -18,7 +18,7 @@ import (
 	"strings"
 	"time"

-	"map-asset-gateway/api-go/internal/uid"
+	"tile-server/api-go/internal/uid"
 )

 type VectorDownloadDescriptor struct {
@@ -73,6 +73,20 @@ func (s *Store) GetVectorAssetByRef(ctx context.Context, ref string) (VectorAsse
 	if err != nil {
 		return VectorAsset{}, err
 	}
+
+	// The compatibility routes resolve by ref on every request, so try the
+	// indexed single-row lookups before falling back to a full table scan.
+	if item, ok, err := s.getVectorAssetByLookup(ctx, "code = ?", normalizeBasemapCode(cleaned)); err != nil {
+		return VectorAsset{}, err
+	} else if ok {
+		return item, nil
+	}
+	if item, ok, err := s.getVectorAssetByLookup(ctx, "file_name = ? COLLATE NOCASE", path.Base(cleaned)); err != nil {
+		return VectorAsset{}, err
+	} else if ok {
+		return item, nil
+	}
+
 	items, err := s.ListVectorAssets(ctx)
 	if err != nil {
 		return VectorAsset{}, err
@@ -85,16 +99,49 @@ func (s *Store) GetVectorAssetByRef(ctx context.Context, ref string) (VectorAsse
 	return VectorAsset{}, fmt.Errorf("vector asset %q not found", ref)
 }

+// getVectorAssetByLookup runs a single indexed lookup. where must be a trusted
+// internal literal, never user input.
+func (s *Store) getVectorAssetByLookup(ctx context.Context, where, value string) (VectorAsset, bool, error) {
+	value = strings.TrimSpace(value)
+	if value == "" {
+		return VectorAsset{}, false, nil
+	}
+	query := fmt.Sprintf(`
+		SELECT id, code, name, format, status, description, file_path, file_name, file_size, checksum, download_url, metadata_json, created_at, updated_at
+		FROM vector_assets
+		WHERE %s
+		LIMIT 1
+	`, where)
+	item, err := scanVectorAsset(s.db.QueryRowContext(ctx, query, value))
+	if err != nil {
+		if errors.Is(err, sql.ErrNoRows) {
+			return VectorAsset{}, false, nil
+		}
+		return VectorAsset{}, false, err
+	}
+	return item, true, nil
+}
+
 func (s *Store) RunVectorScan(ctx context.Context) (map[string]any, error) {
 	root := strings.TrimSpace(s.cfg.VectorScanRoot)
 	if root == "" {
 		return nil, nil
 	}
+	if !s.scanMu.TryLock() {
+		return nil, ErrScanInProgress
+	}
+	defer s.scanMu.Unlock()
+
 	if err := os.MkdirAll(root, 0o755); err != nil {
 		return nil, err
 	}

-	items, err := scanVectorRoot(root)
+	hashCache, err := s.loadVectorHashCache(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	items, err := scanVectorRoot(root, hashCache)
 	if err != nil {
 		return nil, err
 	}
@@ -105,32 +152,22 @@ func (s *Store) RunVectorScan(ctx context.Context) (map[string]any, error) {
 	}
 	defer tx.Rollback()

-	rows, err := tx.QueryContext(ctx, `SELECT code FROM vector_assets`)
+	existing, err := loadExistingVectors(ctx, tx)
 	if err != nil {
 		return nil, err
 	}
-	existing := map[string]struct{}{}
-	for rows.Next() {
-		var code string
-		if err := rows.Scan(&code); err != nil {
-			rows.Close()
-			return nil, err
-		}
-		existing[code] = struct{}{}
-	}
-	rows.Close()
-	if err := rows.Err(); err != nil {
-		return nil, err
-	}

 	found := map[string]struct{}{}
-	added := 0
-	updated := 0
+	added, updated, unchanged := 0, 0, 0
 	now := nowUTC()
 	for _, item := range items {
 		found[item.Code] = struct{}{}
-		if _, ok := existing[item.Code]; ok {
-			updated++
+		if prev, ok := existing[item.Code]; ok {
+			if s.scannedVectorMatches(item, prev) {
+				unchanged++
+			} else {
+				updated++
+			}
 		} else {
 			added++
 		}
@@ -154,14 +191,117 @@ func (s *Store) RunVectorScan(ctx context.Context) (map[string]any, error) {
 		return nil, err
 	}
 	return map[string]any{
-		"scan_root":     root,
-		"scanned_count": len(items),
-		"added_count":   added,
-		"updated_count": updated,
-		"removed_count": removed,
+		"scan_root":       root,
+		"scanned_count":   len(items),
+		"added_count":     added,
+		"updated_count":   updated,
+		"unchanged_count": unchanged,
+		"removed_count":   removed,
 	}, nil
 }

+// existingVector is a snapshot of a stored asset, used to tell a real update
+// from a no-op rescan.
+type existingVector struct {
+	Name         string
+	Format       string
+	Status       string
+	Description  string
+	FilePath     string
+	FileName     string
+	FileSize     int64
+	Checksum     string
+	DownloadURL  string
+	MetadataJSON string
+}
+
+func loadExistingVectors(ctx context.Context, tx *sql.Tx) (map[string]existingVector, error) {
+	rows, err := tx.QueryContext(ctx, `
+        SELECT code, name, format, status, description, file_path, file_name, file_size, checksum, download_url, metadata_json
+        FROM vector_assets
+    `)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	items := map[string]existingVector{}
+	for rows.Next() {
+		var code string
+		var item existingVector
+		if err := rows.Scan(
+			&code,
+			&item.Name,
+			&item.Format,
+			&item.Status,
+			&item.Description,
+			&item.FilePath,
+			&item.FileName,
+			&item.FileSize,
+			&item.Checksum,
+			&item.DownloadURL,
+			&item.MetadataJSON,
+		); err != nil {
+			return nil, err
+		}
+		items[code] = item
+	}
+	return items, rows.Err()
+}
+
+func (s *Store) scannedVectorMatches(item scannedVector, prev existingVector) bool {
+	return item.Name == prev.Name &&
+		item.Format == prev.Format &&
+		item.Status == prev.Status &&
+		item.Description == prev.Description &&
+		item.FilePath == prev.FilePath &&
+		item.FileName == prev.FileName &&
+		item.FileSize == prev.FileSize &&
+		item.Checksum == prev.Checksum &&
+		s.buildVectorDownloadURL(item.Code) == prev.DownloadURL &&
+		writeJSON(item.Metadata) == prev.MetadataJSON
+}
+
+// vectorHashEntry lets an unchanged file reuse the checksum recorded by the
+// previous scan instead of re-reading the whole file on every startup.
+type vectorHashEntry struct {
+	Size     int64
+	ModTime  int64
+	Checksum string
+}
+
+func (s *Store) loadVectorHashCache(ctx context.Context) (map[string]vectorHashEntry, error) {
+	rows, err := s.db.QueryContext(ctx, `
+        SELECT file_path, checksum, metadata_json
+        FROM vector_assets
+    `)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	cache := map[string]vectorHashEntry{}
+	for rows.Next() {
+		var filePath, checksum, metadataJSON string
+		if err := rows.Scan(&filePath, &checksum, &metadataJSON); err != nil {
+			return nil, err
+		}
+		if strings.TrimSpace(checksum) == "" {
+			continue
+		}
+		var meta struct {
+			HashSize  int64 `json:"hash_size"`
+			HashMTime int64 `json:"hash_mtime"`
+		}
+		_ = readMaybeJSON(metadataJSON, &meta)
+		if meta.HashSize == 0 && meta.HashMTime == 0 {
+			continue
+		}
+		cache[filePath] = vectorHashEntry{Size: meta.HashSize, ModTime: meta.HashMTime, Checksum: checksum}
+	}
+	return cache, rows.Err()
+}
+
 type scannedVector struct {
 	Code        string
 	Name        string
@@ -175,7 +315,7 @@ type scannedVector struct {
 	Metadata    map[string]any
 }

-func scanVectorRoot(root string) ([]scannedVector, error) {
+func scanVectorRoot(root string, hashCache map[string]vectorHashEntry) ([]scannedVector, error) {
 	shapefiles := map[string]string{}
 	var items []scannedVector
 	err := filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error {
@@ -188,7 +328,7 @@ func scanVectorRoot(root string) ([]scannedVector, error) {
 		ext := strings.ToLower(filepath.Ext(entry.Name()))
 		switch ext {
 		case ".geojson", ".json", ".zip":
-			item, err := buildScannedVector(root, path, ext)
+			item, err := buildScannedVector(root, path, ext, hashCache)
 			if err != nil {
 				return err
 			}
@@ -205,19 +345,51 @@ func scanVectorRoot(root string) ([]scannedVector, error) {

 	for _, base := range sortedKeysStringMap(shapefiles) {
 		path := shapefiles[base]
-		item, err := buildScannedVector(root, path, ".shp")
+		item, err := buildScannedVector(root, path, ".shp", hashCache)
 		if err != nil {
 			return nil, err
 		}
 		items = append(items, item)
 	}

+	items = disambiguateVectorCodes(items)
 	sort.Slice(items, func(i, j int) bool {
 		return items[i].Code < items[j].Code
 	})
 	return items, nil
 }

+// disambiguateVectorCodes keeps both assets when two different relative paths
+// normalize to the same code (for example "a/b.geojson" and "a-b.geojson").
+// The suffix comes from the relative path, so it is stable across scans.
+func disambiguateVectorCodes(items []scannedVector) []scannedVector {
+	byCode := map[string][]int{}
+	for i, item := range items {
+		byCode[item.Code] = append(byCode[item.Code], i)
+	}
+	for code, indexes := range byCode {
+		if len(indexes) < 2 {
+			continue
+		}
+		for _, index := range indexes {
+			sum := sha256.Sum256([]byte(scannedVectorRelPath(items[index])))
+			items[index].Code = code + "-" + hex.EncodeToString(sum[:4])
+		}
+	}
+	return items
+}
+
+func scannedVectorRelPath(item scannedVector) string {
+	if item.Metadata == nil {
+		return ""
+	}
+	value, ok := item.Metadata["relative_path"].(string)
+	if !ok {
+		return ""
+	}
+	return value
+}
+
 func sortedKeysStringMap(values map[string]string) []string {
 	keys := make([]string, 0, len(values))
 	for key := range values {
@@ -227,7 +399,7 @@ func sortedKeysStringMap(values map[string]string) []string {
 	return keys
 }

-func buildScannedVector(root, path, ext string) (scannedVector, error) {
+func buildScannedVector(root, path, ext string, hashCache map[string]vectorHashEntry) (scannedVector, error) {
 	absPath, err := filepath.Abs(path)
 	if err != nil {
 		return scannedVector{}, err
@@ -245,10 +417,24 @@ func buildScannedVector(root, path, ext string) (scannedVector, error) {
 	format := vectorFormatFromExt(ext)
 	fileName := filepath.Base(absPath)
 	fileSize := info.Size()
-	checksum, err := hashVector(absPath, ext)
+
+	hashSize, hashMTime, err := vectorHashStamp(absPath, ext)
 	if err != nil {
 		return scannedVector{}, err
 	}
+	checksum := ""
+	if cached, ok := hashCache[absPath]; ok &&
+		(hashSize > 0 || hashMTime > 0) &&
+		cached.Size == hashSize && cached.ModTime == hashMTime {
+		checksum = cached.Checksum
+	}
+	if checksum == "" {
+		checksum, err = hashVector(absPath, ext)
+		if err != nil {
+			return scannedVector{}, err
+		}
+	}
+
 	if format == "shapefile" {
 		fileName = name + ".zip"
 	}
@@ -263,10 +449,37 @@ func buildScannedVector(root, path, ext string) (scannedVector, error) {
 		Checksum: checksum,
 		Metadata: map[string]any{
 			"relative_path": filepath.ToSlash(rel),
+			"hash_size":     hashSize,
+			"hash_mtime":    hashMTime,
 		},
 	}, nil
 }

+// vectorHashStamp summarizes the inputs that feed the checksum, so an unchanged
+// file can reuse the previous checksum instead of being rehashed every startup.
+// Shapefiles fold in every sidecar so a changed .dbf/.shx also invalidates it.
+func vectorHashStamp(absPath, ext string) (int64, int64, error) {
+	if strings.EqualFold(ext, ".shp") {
+		var totalSize, maxModTime int64
+		for _, sibling := range shapefilePackageFiles(absPath) {
+			info, err := os.Stat(sibling)
+			if err != nil || info.IsDir() {
+				continue
+			}
+			totalSize += info.Size()
+			if modTime := info.ModTime().Unix(); modTime > maxModTime {
+				maxModTime = modTime
+			}
+		}
+		return totalSize, maxModTime, nil
+	}
+	info, err := os.Stat(absPath)
+	if err != nil {
+		return 0, 0, err
+	}
+	return info.Size(), info.ModTime().Unix(), nil
+}
+
 func vectorCodeFromRel(rel, ext string) string {
 	rel = filepath.ToSlash(rel)
 	rel = strings.TrimSuffix(rel, strings.ToLower(filepath.Ext(rel)))