Basemap Pusher: catalog and distribution gateway for existing tiles
A Go service that scans existing basemap tile directories and vector files, catalogues them in SQLite, issues scoped access tokens, and serves tiles and downloads to downstream systems. It deliberately does not generate or process raster data: GIS software produces the tiles, this service catalogues, authorises and distributes them. Capabilities - scans basemap tile directories (manifest-based or raw z/x/y) and vector assets (.geojson, .json, .zip, shapefiles with sidecars) - token auth with per-resource grants, expiry and coarse scopes (read, slice) - resource health inspection flags a moved or deleted tile root before a client hits a 404 - downstream push with exponential-backoff retries and manual re-queue - image slicing jobs driven by the GDAL bundled with QGIS, including VRT mosaicking so successive drone flights accumulate into one seamless layer - a narrow, non-admin slicing API for external callers: submit, poll, idempotent by job code, completion callback - ops console covering health, ledger, tokens, pushes, scans, slicing and service info - native Windows service install/start/stop with crash recovery, auto-start and a rotating log file, so no NSSM or WinSW wrapper is needed Verified with 49 tests plus live runs against real tile data and real GDAL.
This commit is contained in:
@@ -0,0 +1,664 @@
|
||||
package basemap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"mime"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"tile-server/api-go/internal/uid"
|
||||
)
|
||||
|
||||
// lastUsedUpdateInterval throttles service_tokens.last_used_at writes so tile
|
||||
// traffic does not issue one UPDATE per tile request.
|
||||
const lastUsedUpdateInterval = time.Minute
|
||||
|
||||
var ErrBasemapVersionNotFound = errors.New("basemap version not found")
|
||||
|
||||
func (s *Store) CreateToken(ctx context.Context, input CreateTokenInput) (CreatedToken, error) {
|
||||
name := strings.TrimSpace(input.Name)
|
||||
if name == "" {
|
||||
return CreatedToken{}, errors.New("token name is required")
|
||||
}
|
||||
// A token needs at least one capability: read grants, or the slice scope.
|
||||
hasSliceScope := false
|
||||
for _, value := range input.Scopes {
|
||||
if strings.EqualFold(strings.TrimSpace(value), slicePermission) {
|
||||
hasSliceScope = true
|
||||
}
|
||||
}
|
||||
if len(input.BasemapCodes) == 0 && len(input.VectorCodes) == 0 && !hasSliceScope {
|
||||
return CreatedToken{}, errors.New("at least one basemap code, vector code, or the slice scope is required")
|
||||
}
|
||||
|
||||
resolvedVectorAssets := map[string]VectorAsset{}
|
||||
for _, ref := range input.VectorCodes {
|
||||
ref = strings.TrimSpace(ref)
|
||||
if ref == "" {
|
||||
continue
|
||||
}
|
||||
asset, err := s.GetVectorAssetByRef(ctx, ref)
|
||||
if err != nil {
|
||||
return CreatedToken{}, err
|
||||
}
|
||||
resolvedVectorAssets[asset.Code] = asset
|
||||
}
|
||||
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return CreatedToken{}, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
token, tokenHash, err := s.generateTokenSecret()
|
||||
if err != nil {
|
||||
return CreatedToken{}, err
|
||||
}
|
||||
|
||||
now := nowUTC()
|
||||
// Fall back to the configured default lifetime so tokens do not live forever
|
||||
// unless an explicit expiry was supplied.
|
||||
if input.ExpiresAt == nil && s.cfg.TokenDefaultTTL > 0 {
|
||||
expiresAt := now.Add(s.cfg.TokenDefaultTTL)
|
||||
input.ExpiresAt = &expiresAt
|
||||
}
|
||||
tokenID := uid.New()
|
||||
prefix := token
|
||||
if len(prefix) > 16 {
|
||||
prefix = prefix[:16]
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `
|
||||
INSERT INTO service_tokens (id, name, token_hash, token_prefix, scopes, status, expires_at, last_used_at, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, 'active', ?, NULL, ?)
|
||||
`, tokenID, name, tokenHash, prefix, normalizeTokenScopes(input.Scopes), nullableTime(input.ExpiresAt), toRFC3339(now))
|
||||
if err != nil {
|
||||
return CreatedToken{}, err
|
||||
}
|
||||
|
||||
normalizedCodes := map[string]struct{}{}
|
||||
for _, code := range input.BasemapCodes {
|
||||
normalized := normalizeBasemapCode(code)
|
||||
if normalized != "" {
|
||||
normalizedCodes[normalized] = struct{}{}
|
||||
}
|
||||
}
|
||||
if len(normalizedCodes) == 0 {
|
||||
if len(resolvedVectorAssets) == 0 && !hasSliceScope {
|
||||
return CreatedToken{}, errors.New("no valid basemap codes provided")
|
||||
}
|
||||
}
|
||||
|
||||
for _, code := range sortedKeys(normalizedCodes) {
|
||||
var basemapID string
|
||||
err := tx.QueryRowContext(ctx, `SELECT id FROM basemaps WHERE code = ?`, code).Scan(&basemapID)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return CreatedToken{}, fmt.Errorf("basemap %q not found", code)
|
||||
}
|
||||
return CreatedToken{}, err
|
||||
}
|
||||
grantID := uid.New()
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO token_grants (id, token_id, basemap_id, basemap_version_id, permission, created_at)
|
||||
VALUES (?, ?, ?, NULL, ?, ?)
|
||||
`, grantID, tokenID, basemapID, readPermission, toRFC3339(now)); err != nil {
|
||||
return CreatedToken{}, err
|
||||
}
|
||||
}
|
||||
|
||||
vectorCodes := make([]string, 0, len(resolvedVectorAssets))
|
||||
for code := range resolvedVectorAssets {
|
||||
vectorCodes = append(vectorCodes, code)
|
||||
}
|
||||
sort.Strings(vectorCodes)
|
||||
for _, code := range vectorCodes {
|
||||
asset := resolvedVectorAssets[code]
|
||||
grantID := uid.New()
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO vector_token_grants (id, token_id, vector_asset_id, permission, created_at)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`, grantID, tokenID, asset.ID, readPermission, toRFC3339(now)); err != nil {
|
||||
return CreatedToken{}, err
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return CreatedToken{}, err
|
||||
}
|
||||
meta, err := s.getTokenByID(ctx, tokenID)
|
||||
if err != nil {
|
||||
return CreatedToken{}, err
|
||||
}
|
||||
return CreatedToken{
|
||||
Meta: meta,
|
||||
Token: token,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Store) ListTokens(ctx context.Context) ([]ServiceToken, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT id, name, token_hash, token_prefix, scopes, status, expires_at, last_used_at, created_at
|
||||
FROM service_tokens
|
||||
ORDER BY created_at DESC
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var items []ServiceToken
|
||||
for rows.Next() {
|
||||
item, _, err := scanServiceToken(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
grantsByToken, err := s.loadAllTokenGrants(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
vectorGrantsByToken, err := s.loadAllVectorTokenGrants(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range items {
|
||||
items[i].Grants = grantsByToken[items[i].ID]
|
||||
items[i].VectorGrants = vectorGrantsByToken[items[i].ID]
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (s *Store) loadAllTokenGrants(ctx context.Context) (map[string][]TokenGrant, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT
|
||||
g.id,
|
||||
g.token_id,
|
||||
g.basemap_id,
|
||||
b.code,
|
||||
g.basemap_version_id,
|
||||
v.version,
|
||||
g.permission,
|
||||
g.created_at
|
||||
FROM token_grants g
|
||||
JOIN basemaps b ON b.id = g.basemap_id
|
||||
LEFT JOIN basemap_versions v ON v.id = g.basemap_version_id
|
||||
ORDER BY g.created_at DESC
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
result := map[string][]TokenGrant{}
|
||||
for rows.Next() {
|
||||
item, err := scanTokenGrant(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result[item.TokenID] = append(result[item.TokenID], item)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) loadAllVectorTokenGrants(ctx context.Context) (map[string][]VectorTokenGrant, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT
|
||||
g.id,
|
||||
g.token_id,
|
||||
g.vector_asset_id,
|
||||
v.code,
|
||||
g.permission,
|
||||
g.created_at
|
||||
FROM vector_token_grants g
|
||||
JOIN vector_assets v ON v.id = g.vector_asset_id
|
||||
ORDER BY g.created_at DESC
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
result := map[string][]VectorTokenGrant{}
|
||||
for rows.Next() {
|
||||
item, err := scanVectorTokenGrant(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result[item.TokenID] = append(result[item.TokenID], item)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) loadTokenGrants(ctx context.Context, tokenID string) ([]TokenGrant, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT
|
||||
g.id,
|
||||
g.token_id,
|
||||
g.basemap_id,
|
||||
b.code,
|
||||
g.basemap_version_id,
|
||||
v.version,
|
||||
g.permission,
|
||||
g.created_at
|
||||
FROM token_grants g
|
||||
JOIN basemaps b ON b.id = g.basemap_id
|
||||
LEFT JOIN basemap_versions v ON v.id = g.basemap_version_id
|
||||
WHERE g.token_id = ?
|
||||
ORDER BY g.created_at DESC
|
||||
`, tokenID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var items []TokenGrant
|
||||
for rows.Next() {
|
||||
item, err := scanTokenGrant(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) loadVectorTokenGrants(ctx context.Context, tokenID string) ([]VectorTokenGrant, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT
|
||||
g.id,
|
||||
g.token_id,
|
||||
g.vector_asset_id,
|
||||
v.code,
|
||||
g.permission,
|
||||
g.created_at
|
||||
FROM vector_token_grants g
|
||||
JOIN vector_assets v ON v.id = g.vector_asset_id
|
||||
WHERE g.token_id = ?
|
||||
ORDER BY g.created_at DESC
|
||||
`, tokenID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var items []VectorTokenGrant
|
||||
for rows.Next() {
|
||||
item, err := scanVectorTokenGrant(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) DisableToken(ctx context.Context, tokenID string) error {
|
||||
result, err := s.db.ExecContext(ctx, `UPDATE service_tokens SET status = 'disabled' WHERE id = ?`, strings.TrimSpace(tokenID))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
count, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if count == 0 {
|
||||
return fmt.Errorf("token %q not found", tokenID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) getTokenByID(ctx context.Context, tokenID string) (ServiceToken, error) {
|
||||
row := s.db.QueryRowContext(ctx, `
|
||||
SELECT id, name, token_hash, token_prefix, scopes, status, expires_at, last_used_at, created_at
|
||||
FROM service_tokens
|
||||
WHERE id = ?
|
||||
`, tokenID)
|
||||
item, _, err := scanServiceToken(row)
|
||||
if err != nil {
|
||||
return ServiceToken{}, err
|
||||
}
|
||||
grants, err := s.loadAllTokenGrants(ctx)
|
||||
if err != nil {
|
||||
return ServiceToken{}, err
|
||||
}
|
||||
item.Grants = grants[item.ID]
|
||||
vectorGrants, err := s.loadAllVectorTokenGrants(ctx)
|
||||
if err != nil {
|
||||
return ServiceToken{}, err
|
||||
}
|
||||
item.VectorGrants = vectorGrants[item.ID]
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (s *Store) AuthorizeToken(ctx context.Context, rawToken string) (TokenAuth, error) {
|
||||
rawToken = strings.TrimSpace(rawToken)
|
||||
if rawToken == "" {
|
||||
return TokenAuth{}, errors.New("missing token")
|
||||
}
|
||||
sum := sha256.Sum256([]byte(rawToken))
|
||||
tokenHash := hex.EncodeToString(sum[:])
|
||||
|
||||
row := s.db.QueryRowContext(ctx, `
|
||||
SELECT id, name, token_hash, token_prefix, scopes, status, expires_at, last_used_at, created_at
|
||||
FROM service_tokens
|
||||
WHERE token_hash = ?
|
||||
`, tokenHash)
|
||||
item, _, err := scanServiceToken(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return TokenAuth{}, errors.New("invalid token")
|
||||
}
|
||||
return TokenAuth{}, err
|
||||
}
|
||||
if item.Status != "active" {
|
||||
return TokenAuth{}, errors.New("token is disabled")
|
||||
}
|
||||
if item.ExpiresAt != nil && item.ExpiresAt.Before(nowUTC()) {
|
||||
return TokenAuth{}, errors.New("token is expired")
|
||||
}
|
||||
|
||||
grants, err := s.loadTokenGrants(ctx, item.ID)
|
||||
if err != nil {
|
||||
return TokenAuth{}, err
|
||||
}
|
||||
item.Grants = grants
|
||||
vectorGrants, err := s.loadVectorTokenGrants(ctx, item.ID)
|
||||
if err != nil {
|
||||
return TokenAuth{}, err
|
||||
}
|
||||
item.VectorGrants = vectorGrants
|
||||
permissions := map[string]map[string]VersionScope{}
|
||||
for _, grant := range item.Grants {
|
||||
group, ok := permissions[grant.Permission]
|
||||
if !ok {
|
||||
group = map[string]VersionScope{}
|
||||
permissions[grant.Permission] = group
|
||||
}
|
||||
scope := group[grant.BasemapCode]
|
||||
if grant.BasemapVersion == nil {
|
||||
scope.AllVersions = true
|
||||
scope.Versions = nil
|
||||
} else if !scope.AllVersions {
|
||||
if scope.Versions == nil {
|
||||
scope.Versions = map[string]struct{}{}
|
||||
}
|
||||
scope.Versions[*grant.BasemapVersion] = struct{}{}
|
||||
}
|
||||
group[grant.BasemapCode] = scope
|
||||
}
|
||||
vectorPermissions := map[string]map[string]struct{}{}
|
||||
for _, grant := range item.VectorGrants {
|
||||
group, ok := vectorPermissions[grant.Permission]
|
||||
if !ok {
|
||||
group = map[string]struct{}{}
|
||||
vectorPermissions[grant.Permission] = group
|
||||
}
|
||||
group[grant.VectorCode] = struct{}{}
|
||||
}
|
||||
if item.LastUsedAt == nil || nowUTC().Sub(*item.LastUsedAt) >= lastUsedUpdateInterval {
|
||||
_, _ = s.db.ExecContext(ctx, `UPDATE service_tokens SET last_used_at = ? WHERE id = ?`, toRFC3339(nowUTC()), item.ID)
|
||||
}
|
||||
return TokenAuth{
|
||||
Token: item,
|
||||
Permissions: permissions,
|
||||
VectorPermissions: vectorPermissions,
|
||||
Scopes: scopeSet(item.Scopes),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Token scopes are coarse capabilities layered on top of per-resource grants.
|
||||
// Every token can read; "slice" additionally allows submitting tile jobs.
|
||||
const slicePermission = "slice"
|
||||
|
||||
var allowedTokenScopes = map[string]struct{}{
|
||||
readPermission: {},
|
||||
slicePermission: {},
|
||||
}
|
||||
|
||||
// normalizeTokenScopes keeps only known scopes and always includes read, so the
|
||||
// stored column is a stable, sorted, comma-separated list.
|
||||
func normalizeTokenScopes(values []string) string {
|
||||
scopes := map[string]struct{}{readPermission: {}}
|
||||
for _, value := range values {
|
||||
scope := strings.ToLower(strings.TrimSpace(value))
|
||||
if _, ok := allowedTokenScopes[scope]; ok {
|
||||
scopes[scope] = struct{}{}
|
||||
}
|
||||
}
|
||||
return strings.Join(sortedKeys(scopes), ",")
|
||||
}
|
||||
|
||||
func splitTokenScopes(value string) []string {
|
||||
parts := strings.Split(value, ",")
|
||||
scopes := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
scope := strings.ToLower(strings.TrimSpace(part))
|
||||
if scope != "" {
|
||||
scopes = append(scopes, scope)
|
||||
}
|
||||
}
|
||||
return scopes
|
||||
}
|
||||
|
||||
func scopeSet(values []string) map[string]struct{} {
|
||||
scopes := make(map[string]struct{}, len(values))
|
||||
for _, value := range values {
|
||||
scopes[value] = struct{}{}
|
||||
}
|
||||
return scopes
|
||||
}
|
||||
|
||||
// CanSlice reports whether the token may submit tile jobs.
|
||||
func (auth TokenAuth) CanSlice() bool {
|
||||
_, ok := auth.Scopes[slicePermission]
|
||||
return ok
|
||||
}
|
||||
func (auth TokenAuth) CanReadBasemap(code string, version string) bool {
|
||||
group, ok := auth.Permissions[readPermission]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
scope, ok := group[normalizeBasemapCode(code)]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if scope.AllVersions {
|
||||
return true
|
||||
}
|
||||
if version == "" {
|
||||
return len(scope.Versions) > 0
|
||||
}
|
||||
_, ok = scope.Versions[strings.TrimSpace(version)]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (s *Store) FilterCatalog(auth TokenAuth, basemaps []Basemap) []Basemap {
|
||||
filtered := make([]Basemap, 0, len(basemaps))
|
||||
for _, basemap := range basemaps {
|
||||
if !auth.CanReadBasemap(basemap.Code, "") {
|
||||
continue
|
||||
}
|
||||
copyItem := basemap
|
||||
copyItem.Versions = nil
|
||||
copyItem.Default = nil
|
||||
for _, version := range basemap.Versions {
|
||||
if auth.CanReadBasemap(basemap.Code, version.Version) {
|
||||
copyItem.Versions = append(copyItem.Versions, version)
|
||||
if version.IsDefault {
|
||||
copyVersion := version
|
||||
copyItem.Default = ©Version
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(copyItem.Versions) == 0 {
|
||||
continue
|
||||
}
|
||||
if copyItem.Default == nil {
|
||||
copyVersion := copyItem.Versions[0]
|
||||
copyItem.Default = ©Version
|
||||
}
|
||||
filtered = append(filtered, copyItem)
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
func (s *Store) ResolveTile(ctx context.Context, auth TokenAuth, basemapCode, version, tilePath string) (TileDescriptor, error) {
|
||||
basemapCode = normalizeBasemapCode(basemapCode)
|
||||
version = strings.TrimSpace(version)
|
||||
tilePath = filepath.Clean(strings.TrimPrefix(strings.TrimSpace(tilePath), "/"))
|
||||
if tilePath == "." || tilePath == "" {
|
||||
return TileDescriptor{}, errors.New("tile path is required")
|
||||
}
|
||||
if !auth.CanReadBasemap(basemapCode, version) {
|
||||
return TileDescriptor{}, errors.New("token has no access to this basemap")
|
||||
}
|
||||
|
||||
row := s.db.QueryRowContext(ctx, `
|
||||
SELECT
|
||||
v.id,
|
||||
v.basemap_id,
|
||||
b.code,
|
||||
v.version,
|
||||
v.status,
|
||||
v.is_default,
|
||||
v.manifest_path,
|
||||
v.tile_root_path,
|
||||
v.url_template,
|
||||
v.tile_format,
|
||||
v.tile_scheme,
|
||||
v.min_zoom,
|
||||
v.max_zoom,
|
||||
v.bbox_json,
|
||||
v.attribution,
|
||||
v.metadata_json,
|
||||
v.created_at,
|
||||
v.updated_at
|
||||
FROM basemap_versions v
|
||||
JOIN basemaps b ON b.id = v.basemap_id
|
||||
WHERE b.code = ? AND v.version = ?
|
||||
`, basemapCode, version)
|
||||
versionRow, err := scanBasemapVersionRow(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return TileDescriptor{}, fmt.Errorf("%w: %s/%s", ErrBasemapVersionNotFound, basemapCode, version)
|
||||
}
|
||||
return TileDescriptor{}, err
|
||||
}
|
||||
versionItem := decodeBasemapVersion(versionRow)
|
||||
basemap, err := s.GetBasemapByCode(ctx, basemapCode)
|
||||
if err != nil {
|
||||
return TileDescriptor{}, err
|
||||
}
|
||||
|
||||
filePath, err := safeJoin(versionItem.TileRootPath, tilePath)
|
||||
if err != nil {
|
||||
return TileDescriptor{}, err
|
||||
}
|
||||
info, err := os.Stat(filePath)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return TileDescriptor{}, fmt.Errorf("tile %s not found", tilePath)
|
||||
}
|
||||
return TileDescriptor{}, err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return TileDescriptor{}, errors.New("tile path is a directory")
|
||||
}
|
||||
contentType := mime.TypeByExtension(filepath.Ext(filePath))
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
return TileDescriptor{
|
||||
FilePath: filePath,
|
||||
ContentType: contentType,
|
||||
Basemap: basemap,
|
||||
Version: versionItem,
|
||||
Token: auth.Token,
|
||||
RelativePath: tilePath,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Store) ResolveDefaultTile(ctx context.Context, auth TokenAuth, basemapCode, tilePath string) (TileDescriptor, error) {
|
||||
basemapCode = normalizeBasemapCode(basemapCode)
|
||||
if basemapCode == "" {
|
||||
return TileDescriptor{}, errors.New("basemap code is required")
|
||||
}
|
||||
if !auth.CanReadBasemap(basemapCode, "") {
|
||||
return TileDescriptor{}, errors.New("token has no access to this basemap")
|
||||
}
|
||||
version, err := s.GetDefaultBasemapVersion(ctx, basemapCode)
|
||||
if err != nil {
|
||||
return TileDescriptor{}, err
|
||||
}
|
||||
if !auth.CanReadBasemap(basemapCode, version.Version) {
|
||||
return TileDescriptor{}, errors.New("token has no access to the default basemap version")
|
||||
}
|
||||
return s.ResolveTile(ctx, auth, basemapCode, version.Version, tilePath)
|
||||
}
|
||||
|
||||
func (s *Store) GetDefaultBasemapVersion(ctx context.Context, basemapCode string) (BasemapVersion, error) {
|
||||
basemapCode = normalizeBasemapCode(basemapCode)
|
||||
row := s.db.QueryRowContext(ctx, `
|
||||
SELECT
|
||||
v.id,
|
||||
v.basemap_id,
|
||||
b.code,
|
||||
v.version,
|
||||
v.status,
|
||||
v.is_default,
|
||||
v.manifest_path,
|
||||
v.tile_root_path,
|
||||
v.url_template,
|
||||
v.tile_format,
|
||||
v.tile_scheme,
|
||||
v.min_zoom,
|
||||
v.max_zoom,
|
||||
v.bbox_json,
|
||||
v.attribution,
|
||||
v.metadata_json,
|
||||
v.created_at,
|
||||
v.updated_at
|
||||
FROM basemap_versions v
|
||||
JOIN basemaps b ON b.id = v.basemap_id
|
||||
WHERE b.code = ?
|
||||
ORDER BY v.is_default DESC, v.version ASC
|
||||
LIMIT 1
|
||||
`, basemapCode)
|
||||
versionRow, err := scanBasemapVersionRow(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return BasemapVersion{}, fmt.Errorf("basemap %q has no versions", basemapCode)
|
||||
}
|
||||
return BasemapVersion{}, err
|
||||
}
|
||||
return decodeBasemapVersion(versionRow), nil
|
||||
}
|
||||
|
||||
func safeJoin(root, relative string) (string, error) {
|
||||
root = filepath.Clean(root)
|
||||
if root == "." || root == "" {
|
||||
return "", errors.New("invalid tile root")
|
||||
}
|
||||
target := filepath.Clean(filepath.Join(root, relative))
|
||||
rel, err := filepath.Rel(root, target)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
||||
return "", errors.New("invalid tile path")
|
||||
}
|
||||
return target, nil
|
||||
}
|
||||
@@ -0,0 +1,353 @@
|
||||
package basemap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"tile-server/api-go/internal/uid"
|
||||
)
|
||||
|
||||
func (s *Store) EnsureScanSource(ctx context.Context, input CreateScanSourceInput) (ScanSource, error) {
|
||||
code := normalizeBasemapCode(input.Code)
|
||||
if code == "" {
|
||||
return ScanSource{}, errors.New("scan source code is required")
|
||||
}
|
||||
name := strings.TrimSpace(input.Name)
|
||||
if name == "" {
|
||||
name = code
|
||||
}
|
||||
rootPath := strings.TrimSpace(input.RootPath)
|
||||
if rootPath == "" {
|
||||
return ScanSource{}, errors.New("scan source root path is required")
|
||||
}
|
||||
absRoot, err := filepath.Abs(rootPath)
|
||||
if err != nil {
|
||||
return ScanSource{}, fmt.Errorf("resolve root path: %w", err)
|
||||
}
|
||||
manifestName := strings.TrimSpace(input.ManifestName)
|
||||
if manifestName == "" {
|
||||
manifestName = defaultManifestName
|
||||
}
|
||||
|
||||
now := nowUTC()
|
||||
id := uid.Deterministic("scan-source", code)
|
||||
_, err = s.db.ExecContext(ctx, `
|
||||
INSERT INTO scan_sources (
|
||||
id, code, name, root_path, manifest_name, enabled, metadata_json, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, 1, '', ?, ?)
|
||||
ON CONFLICT(code) DO UPDATE SET
|
||||
name = excluded.name,
|
||||
root_path = excluded.root_path,
|
||||
manifest_name = excluded.manifest_name,
|
||||
enabled = 1,
|
||||
updated_at = excluded.updated_at
|
||||
`, id, code, name, absRoot, manifestName, toRFC3339(now), toRFC3339(now))
|
||||
if err != nil {
|
||||
return ScanSource{}, fmt.Errorf("upsert scan source: %w", err)
|
||||
}
|
||||
return s.GetScanSourceByCode(ctx, code)
|
||||
}
|
||||
|
||||
func (s *Store) GetScanSourceByCode(ctx context.Context, code string) (ScanSource, error) {
|
||||
row := s.db.QueryRowContext(ctx, `
|
||||
SELECT id, code, name, root_path, manifest_name, enabled, metadata_json, created_at, updated_at
|
||||
FROM scan_sources
|
||||
WHERE code = ?
|
||||
`, normalizeBasemapCode(code))
|
||||
item, err := scanScanSource(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return ScanSource{}, &scanSourceError{sentinel: ErrScanSourceNotFound, code: code, reason: "not found"}
|
||||
}
|
||||
return ScanSource{}, err
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (s *Store) ListScanSources(ctx context.Context) ([]ScanSource, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT id, code, name, root_path, manifest_name, enabled, metadata_json, created_at, updated_at
|
||||
FROM scan_sources
|
||||
ORDER BY code
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var items []ScanSource
|
||||
for rows.Next() {
|
||||
item, err := scanScanSource(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) ListBasemaps(ctx context.Context) ([]Basemap, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT id, code, name, type, status, description, created_at, updated_at
|
||||
FROM basemaps
|
||||
ORDER BY code
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var items []Basemap
|
||||
indexByID := map[string]int{}
|
||||
for rows.Next() {
|
||||
item, err := scanBasemap(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
indexByID[item.ID] = len(items)
|
||||
items = append(items, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
versionRows, err := s.db.QueryContext(ctx, `
|
||||
SELECT
|
||||
v.id,
|
||||
v.basemap_id,
|
||||
b.code,
|
||||
v.version,
|
||||
v.status,
|
||||
v.is_default,
|
||||
v.manifest_path,
|
||||
v.tile_root_path,
|
||||
v.url_template,
|
||||
v.tile_format,
|
||||
v.tile_scheme,
|
||||
v.min_zoom,
|
||||
v.max_zoom,
|
||||
v.bbox_json,
|
||||
v.attribution,
|
||||
v.metadata_json,
|
||||
v.created_at,
|
||||
v.updated_at
|
||||
FROM basemap_versions v
|
||||
JOIN basemaps b ON b.id = v.basemap_id
|
||||
ORDER BY b.code, v.version
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer versionRows.Close()
|
||||
|
||||
for versionRows.Next() {
|
||||
row, err := scanBasemapVersionRow(versionRows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
index, ok := indexByID[row.BasemapID]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
version := decodeBasemapVersion(row)
|
||||
items[index].Versions = append(items[index].Versions, version)
|
||||
if version.IsDefault {
|
||||
copyValue := version
|
||||
items[index].Default = ©Value
|
||||
}
|
||||
}
|
||||
if err := versionRows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (s *Store) GetBasemapByCode(ctx context.Context, code string) (Basemap, error) {
|
||||
normalized := normalizeBasemapCode(code)
|
||||
row := s.db.QueryRowContext(ctx, `
|
||||
SELECT id, code, name, type, status, description, created_at, updated_at
|
||||
FROM basemaps
|
||||
WHERE code = ?
|
||||
`, normalized)
|
||||
item, err := scanBasemap(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Basemap{}, fmt.Errorf("basemap %q not found", code)
|
||||
}
|
||||
return Basemap{}, err
|
||||
}
|
||||
|
||||
versionRows, err := s.db.QueryContext(ctx, `
|
||||
SELECT
|
||||
v.id,
|
||||
v.basemap_id,
|
||||
b.code,
|
||||
v.version,
|
||||
v.status,
|
||||
v.is_default,
|
||||
v.manifest_path,
|
||||
v.tile_root_path,
|
||||
v.url_template,
|
||||
v.tile_format,
|
||||
v.tile_scheme,
|
||||
v.min_zoom,
|
||||
v.max_zoom,
|
||||
v.bbox_json,
|
||||
v.attribution,
|
||||
v.metadata_json,
|
||||
v.created_at,
|
||||
v.updated_at
|
||||
FROM basemap_versions v
|
||||
JOIN basemaps b ON b.id = v.basemap_id
|
||||
WHERE v.basemap_id = ?
|
||||
ORDER BY v.version
|
||||
`, item.ID)
|
||||
if err != nil {
|
||||
return Basemap{}, err
|
||||
}
|
||||
defer versionRows.Close()
|
||||
|
||||
for versionRows.Next() {
|
||||
row, err := scanBasemapVersionRow(versionRows)
|
||||
if err != nil {
|
||||
return Basemap{}, err
|
||||
}
|
||||
version := decodeBasemapVersion(row)
|
||||
item.Versions = append(item.Versions, version)
|
||||
if version.IsDefault {
|
||||
copyValue := version
|
||||
item.Default = ©Value
|
||||
}
|
||||
}
|
||||
if err := versionRows.Err(); err != nil {
|
||||
return Basemap{}, err
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (s *Store) SetDefaultVersion(ctx context.Context, basemapCode, version string) error {
|
||||
basemapCode = normalizeBasemapCode(basemapCode)
|
||||
version = strings.TrimSpace(version)
|
||||
if basemapCode == "" || version == "" {
|
||||
return errors.New("basemap code and version are required")
|
||||
}
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
var basemapID string
|
||||
if err := tx.QueryRowContext(ctx, `SELECT id FROM basemaps WHERE code = ?`, basemapCode).Scan(&basemapID); err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return fmt.Errorf("basemap %q not found", basemapCode)
|
||||
}
|
||||
return err
|
||||
}
|
||||
result, err := tx.ExecContext(ctx, `
|
||||
UPDATE basemap_versions
|
||||
SET is_default = CASE WHEN version = ? THEN 1 ELSE 0 END, updated_at = ?
|
||||
WHERE basemap_id = ?
|
||||
`, version, toRFC3339(nowUTC()), basemapID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
count, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if count == 0 {
|
||||
return fmt.Errorf("version %q not found for basemap %q", version, basemapCode)
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `UPDATE basemaps SET updated_at = ? WHERE id = ?`, toRFC3339(nowUTC()), basemapID); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (s *Store) ListScanRuns(ctx context.Context, limit int) ([]ScanRun, error) {
|
||||
if limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT
|
||||
r.id,
|
||||
r.scan_source_id,
|
||||
s.code,
|
||||
r.status,
|
||||
r.scanned_count,
|
||||
r.added_count,
|
||||
r.updated_count,
|
||||
r.removed_count,
|
||||
r.summary_json,
|
||||
r.started_at,
|
||||
r.finished_at
|
||||
FROM scan_runs r
|
||||
JOIN scan_sources s ON s.id = r.scan_source_id
|
||||
ORDER BY r.started_at DESC
|
||||
LIMIT ?
|
||||
`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var items []ScanRun
|
||||
for rows.Next() {
|
||||
item, err := scanScanRun(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) ListPushRecords(ctx context.Context, limit int) ([]PushRecord, error) {
|
||||
if limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT
|
||||
p.id,
|
||||
p.target_system_id,
|
||||
t.code,
|
||||
p.basemap_version_id,
|
||||
b.code,
|
||||
v.version,
|
||||
p.status,
|
||||
p.request_json,
|
||||
p.response_status,
|
||||
p.response_body,
|
||||
p.error_message,
|
||||
p.attempt_count,
|
||||
p.next_attempt_at,
|
||||
p.pushed_at,
|
||||
p.finished_at
|
||||
FROM push_records p
|
||||
JOIN target_systems t ON t.id = p.target_system_id
|
||||
JOIN basemap_versions v ON v.id = p.basemap_version_id
|
||||
JOIN basemaps b ON b.id = v.basemap_id
|
||||
ORDER BY p.pushed_at DESC
|
||||
LIMIT ?
|
||||
`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var items []PushRecord
|
||||
for rows.Next() {
|
||||
item, err := scanPushRecord(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package basemap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestInsarCompatibilityHelpers(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
tileRoot := filepath.Join(root, "tiles")
|
||||
vectorRoot := filepath.Join(root, "vectors")
|
||||
sqlitePath := filepath.Join(root, "basemap.db")
|
||||
|
||||
if err := os.MkdirAll(filepath.Join(tileRoot, "google_image", "0", "0"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir tile root: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(tileRoot, "google_image", "0", "0", "0.webp"), []byte("tile"), 0o644); err != nil {
|
||||
t.Fatalf("write tile: %v", err)
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(filepath.Join(vectorRoot, "landuse"), 0o755); err != nil {
|
||||
t.Fatalf("mkdir vector root: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(vectorRoot, "全国行政区.geojson"), []byte(`{"type":"FeatureCollection","features":[]}`), 0o644); err != nil {
|
||||
t.Fatalf("write geojson: %v", err)
|
||||
}
|
||||
for _, name := range []string{"parcel.shp", "parcel.dbf", "parcel.shx"} {
|
||||
if err := os.WriteFile(filepath.Join(vectorRoot, "landuse", name), []byte(name), 0o644); err != nil {
|
||||
t.Fatalf("write shapefile part %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
store, err := OpenStore(StoreConfig{
|
||||
SQLitePath: sqlitePath,
|
||||
ScanRoot: tileRoot,
|
||||
VectorScanRoot: vectorRoot,
|
||||
APIBaseURL: "http://127.0.0.1:8910",
|
||||
TileBaseURL: "http://127.0.0.1:8910",
|
||||
VectorBaseURL: "http://127.0.0.1:8910",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := store.Close(); err != nil {
|
||||
t.Fatalf("close store: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
if _, err := store.RunScan(ctx, defaultScanSourceCode); err != nil {
|
||||
t.Fatalf("run basemap scan: %v", err)
|
||||
}
|
||||
if _, err := store.RunVectorScan(ctx); err != nil {
|
||||
t.Fatalf("run vector scan: %v", err)
|
||||
}
|
||||
|
||||
created, err := store.CreateToken(ctx, CreateTokenInput{
|
||||
Name: "insar-v2",
|
||||
BasemapCodes: []string{"google_image"},
|
||||
VectorCodes: []string{"全国行政区.geojson", "landuse/parcel.shp"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create token: %v", err)
|
||||
}
|
||||
|
||||
auth, err := store.AuthorizeToken(ctx, created.Token)
|
||||
if err != nil {
|
||||
t.Fatalf("authorize token: %v", err)
|
||||
}
|
||||
|
||||
tile, err := store.ResolveDefaultTile(ctx, auth, "google_image", "0/0/0.webp")
|
||||
if err != nil {
|
||||
t.Fatalf("resolve default tile: %v", err)
|
||||
}
|
||||
if tile.Version.Version != "current" {
|
||||
t.Fatalf("unexpected default version: %s", tile.Version.Version)
|
||||
}
|
||||
|
||||
geojson, err := store.ResolveVectorDownloadByRef(ctx, auth, "全国行政区.geojson", "geojson")
|
||||
if err != nil {
|
||||
t.Fatalf("resolve geojson by path: %v", err)
|
||||
}
|
||||
if geojson.Asset.Format != "geojson" {
|
||||
t.Fatalf("unexpected geojson format: %s", geojson.Asset.Format)
|
||||
}
|
||||
if filepath.Base(geojson.FilePath) != "全国行政区.geojson" {
|
||||
t.Fatalf("unexpected geojson file: %s", geojson.FilePath)
|
||||
}
|
||||
|
||||
shp, err := store.ResolveVectorDownloadByRef(ctx, auth, "landuse/parcel.shp", "shapefile")
|
||||
if err != nil {
|
||||
t.Fatalf("resolve shapefile by path: %v", err)
|
||||
}
|
||||
if !shp.Zipped {
|
||||
t.Fatal("expected shapefile download to be zipped")
|
||||
}
|
||||
if shp.FileName != "parcel.zip" {
|
||||
t.Fatalf("unexpected shapefile name: %s", shp.FileName)
|
||||
}
|
||||
if len(shp.PackageFiles) != 3 {
|
||||
t.Fatalf("unexpected shapefile package size: %d", len(shp.PackageFiles))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package basemap
|
||||
|
||||
import "context"
|
||||
|
||||
func (s *Store) Dashboard(ctx context.Context, info SystemInfo, flash string, createdToken string) (DashboardData, error) {
|
||||
basemaps, err := s.ListBasemaps(ctx)
|
||||
if err != nil {
|
||||
return DashboardData{}, err
|
||||
}
|
||||
vectorAssets, err := s.ListVectorAssets(ctx)
|
||||
if err != nil {
|
||||
return DashboardData{}, err
|
||||
}
|
||||
scanSources, err := s.ListScanSources(ctx)
|
||||
if err != nil {
|
||||
return DashboardData{}, err
|
||||
}
|
||||
scanRuns, err := s.ListScanRuns(ctx, 10)
|
||||
if err != nil {
|
||||
return DashboardData{}, err
|
||||
}
|
||||
targetSystems, err := s.ListTargetSystems(ctx)
|
||||
if err != nil {
|
||||
return DashboardData{}, err
|
||||
}
|
||||
tokens, err := s.ListTokens(ctx)
|
||||
if err != nil {
|
||||
return DashboardData{}, err
|
||||
}
|
||||
pushRecords, err := s.ListPushRecords(ctx, 10)
|
||||
if err != nil {
|
||||
return DashboardData{}, err
|
||||
}
|
||||
vectorPushRecords, err := s.ListVectorPushRecords(ctx, 10)
|
||||
if err != nil {
|
||||
return DashboardData{}, err
|
||||
}
|
||||
versionHealth, err := s.loadVersionHealth(ctx)
|
||||
if err != nil {
|
||||
return DashboardData{}, err
|
||||
}
|
||||
vectorHealth, err := s.loadVectorHealth(ctx)
|
||||
if err != nil {
|
||||
return DashboardData{}, err
|
||||
}
|
||||
tileJobs, err := s.ListTileJobs(ctx, 10)
|
||||
if err != nil {
|
||||
return DashboardData{}, err
|
||||
}
|
||||
layers, err := s.ListLayerInventories(ctx)
|
||||
if err != nil {
|
||||
return DashboardData{}, err
|
||||
}
|
||||
return DashboardData{
|
||||
SystemInfo: info,
|
||||
Basemaps: basemaps,
|
||||
VectorAssets: vectorAssets,
|
||||
ScanSources: scanSources,
|
||||
ScanRuns: scanRuns,
|
||||
TargetSystems: targetSystems,
|
||||
Tokens: tokens,
|
||||
PushRecords: pushRecords,
|
||||
VectorPushRecords: vectorPushRecords,
|
||||
Flash: flash,
|
||||
CreatedToken: createdToken,
|
||||
VersionHealth: versionHealth,
|
||||
VectorHealth: vectorHealth,
|
||||
TileJobs: tileJobs,
|
||||
Layers: layers,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,872 @@
|
||||
package basemap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCheckHealthFlagsMissingPaths(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
tileRoot := filepath.Join(root, "tiles")
|
||||
vectorRoot := filepath.Join(root, "vectors")
|
||||
writeTestFile(t, filepath.Join(tileRoot, "google_image", "0", "0", "0.webp"), "tile")
|
||||
writeTestFile(t, filepath.Join(vectorRoot, "boundary.geojson"), `{"type":"FeatureCollection","features":[]}`)
|
||||
|
||||
store := openTestStore(t, StoreConfig{
|
||||
SQLitePath: filepath.Join(root, "basemap.db"),
|
||||
ScanRoot: tileRoot,
|
||||
VectorScanRoot: vectorRoot,
|
||||
APIBaseURL: "http://127.0.0.1:8910",
|
||||
TileBaseURL: "http://127.0.0.1:8910",
|
||||
VectorBaseURL: "http://127.0.0.1:8910",
|
||||
})
|
||||
if _, err := store.RunScan(ctx, defaultScanSourceCode); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if _, err := store.RunVectorScan(ctx); err != nil {
|
||||
t.Fatalf("vector scan: %v", err)
|
||||
}
|
||||
|
||||
report, err := store.CheckHealth(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("check health: %v", err)
|
||||
}
|
||||
if report.VersionsMissing != 0 || report.VectorsMissing != 0 {
|
||||
t.Fatalf("expected everything healthy, got %+v", report)
|
||||
}
|
||||
|
||||
// A directory that disappears must be reported before the first 404.
|
||||
if err := os.RemoveAll(filepath.Join(tileRoot, "google_image")); err != nil {
|
||||
t.Fatalf("remove tile root: %v", err)
|
||||
}
|
||||
if err := os.Remove(filepath.Join(vectorRoot, "boundary.geojson")); err != nil {
|
||||
t.Fatalf("remove vector: %v", err)
|
||||
}
|
||||
|
||||
report, err = store.CheckHealth(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("check health after removal: %v", err)
|
||||
}
|
||||
if report.VersionsMissing != 1 || report.VectorsMissing != 1 {
|
||||
t.Fatalf("expected one missing of each, got %+v", report)
|
||||
}
|
||||
|
||||
versionHealth, err := store.loadVersionHealth(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("load version health: %v", err)
|
||||
}
|
||||
assetHealth, err := store.loadVectorHealth(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("load vector health: %v", err)
|
||||
}
|
||||
foundMissing := false
|
||||
for _, health := range versionHealth {
|
||||
if health.Health == healthMissing {
|
||||
foundMissing = true
|
||||
}
|
||||
}
|
||||
if !foundMissing {
|
||||
t.Fatalf("expected a missing version verdict, got %+v", versionHealth)
|
||||
}
|
||||
foundMissing = false
|
||||
for _, health := range assetHealth {
|
||||
if health.Health == healthMissing {
|
||||
foundMissing = true
|
||||
}
|
||||
}
|
||||
if !foundMissing {
|
||||
t.Fatalf("expected a missing vector verdict, got %+v", assetHealth)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPushRetrySucceedsAfterBackoff(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
tileRoot := filepath.Join(root, "tiles")
|
||||
writeTestFile(t, filepath.Join(tileRoot, "google_image", "0", "0", "0.webp"), "tile")
|
||||
|
||||
var attempts int
|
||||
callback := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
if attempts < 2 {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer callback.Close()
|
||||
|
||||
store := openTestStore(t, StoreConfig{
|
||||
SQLitePath: filepath.Join(root, "basemap.db"),
|
||||
ScanRoot: tileRoot,
|
||||
APIBaseURL: "http://127.0.0.1:8910",
|
||||
TileBaseURL: "http://127.0.0.1:8910",
|
||||
VectorBaseURL: "http://127.0.0.1:8910",
|
||||
})
|
||||
if _, err := store.RunScan(ctx, defaultScanSourceCode); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if _, err := store.CreateTargetSystem(ctx, CreateTargetSystemInput{Code: "downstream", CallbackURL: callback.URL}); err != nil {
|
||||
t.Fatalf("create target: %v", err)
|
||||
}
|
||||
record, err := store.PushBasemapVersion(ctx, "downstream", "google_image", "current")
|
||||
if err != nil {
|
||||
t.Fatalf("push: %v", err)
|
||||
}
|
||||
|
||||
// First attempt fails, so the record must be parked in retrying with a
|
||||
// scheduled next attempt rather than being dropped as failed.
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
var parked bool
|
||||
for time.Now().Before(deadline) {
|
||||
records, err := store.ListPushRecords(ctx, 5)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
for _, item := range records {
|
||||
if item.ID == record.ID && item.Status == pushStatusRetrying && item.NextAttemptAt != nil {
|
||||
parked = true
|
||||
}
|
||||
}
|
||||
if parked {
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
if !parked {
|
||||
t.Fatal("expected the failed push to be rescheduled rather than dropped")
|
||||
}
|
||||
|
||||
// Make the retry due immediately, then run the dispatcher.
|
||||
if _, err := store.db.ExecContext(ctx, `UPDATE push_records SET next_attempt_at = ? WHERE id = ?`, toRFC3339(nowUTC().Add(-time.Minute)), record.ID); err != nil {
|
||||
t.Fatalf("force due: %v", err)
|
||||
}
|
||||
if err := store.dispatchPushRetries(ctx); err != nil {
|
||||
t.Fatalf("dispatch retries: %v", err)
|
||||
}
|
||||
|
||||
records, err := store.ListPushRecords(ctx, 5)
|
||||
if err != nil {
|
||||
t.Fatalf("list after retry: %v", err)
|
||||
}
|
||||
var final PushRecord
|
||||
for _, item := range records {
|
||||
if item.ID == record.ID {
|
||||
final = item
|
||||
}
|
||||
}
|
||||
if final.Status != pushStatusSucceeded {
|
||||
t.Fatalf("expected the retry to succeed, got status=%q message=%q", final.Status, final.ErrorMessage)
|
||||
}
|
||||
if final.AttemptCount != 2 {
|
||||
t.Fatalf("expected 2 attempts, got %d", final.AttemptCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManualRetryRequeuesFailedPush(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
tileRoot := filepath.Join(root, "tiles")
|
||||
writeTestFile(t, filepath.Join(tileRoot, "google_image", "0", "0", "0.webp"), "tile")
|
||||
|
||||
callback := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusBadGateway)
|
||||
}))
|
||||
defer callback.Close()
|
||||
|
||||
store := openTestStore(t, StoreConfig{
|
||||
SQLitePath: filepath.Join(root, "basemap.db"),
|
||||
ScanRoot: tileRoot,
|
||||
APIBaseURL: "http://127.0.0.1:8910",
|
||||
TileBaseURL: "http://127.0.0.1:8910",
|
||||
VectorBaseURL: "http://127.0.0.1:8910",
|
||||
})
|
||||
if _, err := store.RunScan(ctx, defaultScanSourceCode); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if _, err := store.CreateTargetSystem(ctx, CreateTargetSystemInput{Code: "downstream", CallbackURL: callback.URL}); err != nil {
|
||||
t.Fatalf("create target: %v", err)
|
||||
}
|
||||
record, err := store.PushBasemapVersion(ctx, "downstream", "google_image", "current")
|
||||
if err != nil {
|
||||
t.Fatalf("push: %v", err)
|
||||
}
|
||||
|
||||
// Exhaust the retry budget so the record lands in failed.
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
records, _ := store.ListPushRecords(ctx, 5)
|
||||
for _, item := range records {
|
||||
if item.ID == record.ID && item.Status == pushStatusFailed {
|
||||
goto failed
|
||||
}
|
||||
}
|
||||
if _, err := store.db.ExecContext(ctx, `UPDATE push_records SET next_attempt_at = ? WHERE id = ? AND status = ?`, toRFC3339(nowUTC().Add(-time.Minute)), record.ID, pushStatusRetrying); err != nil {
|
||||
t.Fatalf("force due: %v", err)
|
||||
}
|
||||
_ = store.dispatchPushRetries(ctx)
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
|
||||
failed:
|
||||
records, err := store.ListPushRecords(ctx, 5)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
var failedRecord PushRecord
|
||||
for _, item := range records {
|
||||
if item.ID == record.ID {
|
||||
failedRecord = item
|
||||
}
|
||||
}
|
||||
if failedRecord.Status != pushStatusFailed {
|
||||
t.Fatalf("expected a failed record, got %q", failedRecord.Status)
|
||||
}
|
||||
if failedRecord.AttemptCount != pushMaxAttempts {
|
||||
t.Fatalf("expected %d attempts, got %d", pushMaxAttempts, failedRecord.AttemptCount)
|
||||
}
|
||||
|
||||
if err := store.RetryPush(ctx, "basemap", record.ID); err != nil {
|
||||
t.Fatalf("manual retry: %v", err)
|
||||
}
|
||||
records, err = store.ListPushRecords(ctx, 5)
|
||||
if err != nil {
|
||||
t.Fatalf("list after manual retry: %v", err)
|
||||
}
|
||||
for _, item := range records {
|
||||
if item.ID != record.ID {
|
||||
continue
|
||||
}
|
||||
if item.AttemptCount != 0 {
|
||||
t.Fatalf("expected the manual retry to reset the attempt counter, got %d", item.AttemptCount)
|
||||
}
|
||||
if item.Status == pushStatusSucceeded {
|
||||
t.Fatal("a failing target must not report success")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateTileJobValidation(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
tileRoot := filepath.Join(root, "tiles")
|
||||
source := filepath.Join(root, "input.tif")
|
||||
writeTestFile(t, source, "fake raster")
|
||||
|
||||
store := openTestStore(t, StoreConfig{
|
||||
SQLitePath: filepath.Join(root, "basemap.db"),
|
||||
ScanRoot: tileRoot,
|
||||
APIBaseURL: "http://127.0.0.1:8910",
|
||||
TileBaseURL: "http://127.0.0.1:8910",
|
||||
VectorBaseURL: "http://127.0.0.1:8910",
|
||||
})
|
||||
|
||||
if _, err := store.CreateTileJob(ctx, CreateTileJobInput{TargetCode: "x", SourcePath: ""}); err == nil {
|
||||
t.Fatal("expected a missing source path to be rejected")
|
||||
}
|
||||
if _, err := store.CreateTileJob(ctx, CreateTileJobInput{SourcePath: source}); err == nil {
|
||||
t.Fatal("expected a missing target code to be rejected")
|
||||
}
|
||||
if _, err := store.CreateTileJob(ctx, CreateTileJobInput{SourcePath: source, TargetCode: "x", ZoomMin: 5, ZoomMax: 2}); err == nil {
|
||||
t.Fatal("expected an inverted zoom range to be rejected")
|
||||
}
|
||||
if _, err := store.CreateTileJob(ctx, CreateTileJobInput{SourcePath: filepath.Join(root, "missing.tif"), TargetCode: "x"}); err == nil {
|
||||
t.Fatal("expected a missing source file to be rejected")
|
||||
}
|
||||
|
||||
job, err := store.CreateTileJob(ctx, CreateTileJobInput{SourcePath: source, TargetCode: "demo", ZoomMin: 0, ZoomMax: 3})
|
||||
if err != nil {
|
||||
t.Fatalf("create job: %v", err)
|
||||
}
|
||||
if job.Status != tileJobPending {
|
||||
t.Fatalf("expected a pending job, got %q", job.Status)
|
||||
}
|
||||
if job.TileSize != 256 || job.TileFormat != "webp" {
|
||||
t.Fatalf("expected defaults to be applied, got %d/%s", job.TileSize, job.TileFormat)
|
||||
}
|
||||
if !strings.HasSuffix(job.OutputRoot, filepath.Join("tiles", "demo")) {
|
||||
t.Fatalf("unexpected output root %q", job.OutputRoot)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTileJobCommandArgsSubstitution(t *testing.T) {
|
||||
job := TileJob{
|
||||
SourcePath: "/data/in.tif",
|
||||
OutputRoot: "/data/out",
|
||||
ZoomMin: 1,
|
||||
ZoomMax: 5,
|
||||
TileSize: 512,
|
||||
TileFormat: "png",
|
||||
TargetCode: "demo",
|
||||
}
|
||||
args := tileJobCommandArgs(`gdal2tiles.py --xyz --zoom={zmin}-{zmax} --tilesize={tilesize} --format={format} {input} {output}`, job, job.SourcePath)
|
||||
want := []string{"gdal2tiles.py", "--xyz", "--zoom=1-5", "--tilesize=512", "--format=png", "/data/in.tif", "/data/out"}
|
||||
if len(args) != len(want) {
|
||||
t.Fatalf("expected %d args, got %v", len(want), args)
|
||||
}
|
||||
for i := range want {
|
||||
if args[i] != want[i] {
|
||||
t.Fatalf("arg %d = %q, want %q", i, args[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitCommandLineKeepsQuotedSegments(t *testing.T) {
|
||||
parts := splitCommandLine(`cmd /c mkdir "C:\Program Files\tiles\0"`)
|
||||
if len(parts) != 4 {
|
||||
t.Fatalf("expected 4 parts, got %v", parts)
|
||||
}
|
||||
if parts[3] != `C:\Program Files\tiles\0` {
|
||||
t.Fatalf("quoted segment not preserved: %q", parts[3])
|
||||
}
|
||||
}
|
||||
|
||||
func TestTileJobRunsCommandThenScans(t *testing.T) {
|
||||
if runtime.GOOS != "windows" {
|
||||
t.Skip("the stub slicer below uses cmd.exe")
|
||||
}
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
tileRoot := filepath.Join(root, "tiles")
|
||||
source := filepath.Join(root, "input.tif")
|
||||
writeTestFile(t, source, "fake raster")
|
||||
if err := os.MkdirAll(tileRoot, 0o755); err != nil {
|
||||
t.Fatalf("mkdir tile root: %v", err)
|
||||
}
|
||||
|
||||
store := openTestStore(t, StoreConfig{
|
||||
SQLitePath: filepath.Join(root, "basemap.db"),
|
||||
ScanRoot: tileRoot,
|
||||
APIBaseURL: "http://127.0.0.1:8910",
|
||||
TileBaseURL: "http://127.0.0.1:8910",
|
||||
VectorBaseURL: "http://127.0.0.1:8910",
|
||||
// Stand-in for gdal2tiles: lay down a z/x tree the scanner can read.
|
||||
TileJobCommand: `cmd /c mkdir "{output}\0\0"`,
|
||||
})
|
||||
if _, err := store.RunScan(ctx, defaultScanSourceCode); err != nil {
|
||||
t.Fatalf("initial scan: %v", err)
|
||||
}
|
||||
|
||||
job, err := store.CreateTileJob(ctx, CreateTileJobInput{
|
||||
SourcePath: source,
|
||||
TargetCode: "demo",
|
||||
ZoomMin: 0,
|
||||
ZoomMax: 0,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create job: %v", err)
|
||||
}
|
||||
store.runTileJob(ctx, job)
|
||||
|
||||
tileJobs, err := store.ListTileJobs(ctx, 5)
|
||||
if err != nil {
|
||||
t.Fatalf("list jobs: %v", err)
|
||||
}
|
||||
if len(tileJobs) != 1 {
|
||||
t.Fatalf("expected 1 job, got %d", len(tileJobs))
|
||||
}
|
||||
if tileJobs[0].Status != tileJobSucceeded {
|
||||
t.Fatalf("expected success, got %q (%s)", tileJobs[0].Status, tileJobs[0].Message)
|
||||
}
|
||||
|
||||
basemaps, err := store.ListBasemaps(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list basemaps: %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, item := range basemaps {
|
||||
if item.Code == "demo" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("expected the sliced basemap to be scanned in, got %+v", basemaps)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTileJobFailureIsRecorded(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
tileRoot := filepath.Join(root, "tiles")
|
||||
source := filepath.Join(root, "input.tif")
|
||||
writeTestFile(t, source, "fake raster")
|
||||
|
||||
store := openTestStore(t, StoreConfig{
|
||||
SQLitePath: filepath.Join(root, "basemap.db"),
|
||||
ScanRoot: tileRoot,
|
||||
APIBaseURL: "http://127.0.0.1:8910",
|
||||
TileBaseURL: "http://127.0.0.1:8910",
|
||||
VectorBaseURL: "http://127.0.0.1:8910",
|
||||
TileJobCommand: "definitely-not-a-real-binary {input} {output}",
|
||||
})
|
||||
job, err := store.CreateTileJob(ctx, CreateTileJobInput{SourcePath: source, TargetCode: "demo"})
|
||||
if err != nil {
|
||||
t.Fatalf("create job: %v", err)
|
||||
}
|
||||
store.runTileJob(ctx, job)
|
||||
|
||||
jobs, err := store.ListTileJobs(ctx, 5)
|
||||
if err != nil {
|
||||
t.Fatalf("list jobs: %v", err)
|
||||
}
|
||||
if jobs[0].Status != tileJobFailed {
|
||||
t.Fatalf("expected failure, got %q", jobs[0].Status)
|
||||
}
|
||||
if jobs[0].Message == "" {
|
||||
t.Fatal("expected the failure reason to be recorded")
|
||||
}
|
||||
t.Logf("DEBUG job=%+v", jobs[0])
|
||||
if jobs[0].LogPath == "" {
|
||||
t.Fatal("expected a log path to be recorded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryPushRejectsUnknownKind(t *testing.T) {
|
||||
store := openTestStore(t, StoreConfig{SQLitePath: filepath.Join(t.TempDir(), "basemap.db")})
|
||||
if err := store.RetryPush(context.Background(), "nonsense", "x"); err == nil {
|
||||
t.Fatal("expected an unknown kind to be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetTargetSystemByIDMissing(t *testing.T) {
|
||||
store := openTestStore(t, StoreConfig{SQLitePath: filepath.Join(t.TempDir(), "basemap.db")})
|
||||
_, err := store.GetTargetSystemByID(context.Background(), "nope")
|
||||
if err == nil || errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("expected a not-found error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBootstrapMigratesLegacySchema(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
dbPath := filepath.Join(root, "legacy.db")
|
||||
|
||||
// A database created before the health and retry columns existed.
|
||||
legacy, err := sql.Open("sqlite", "file:"+filepath.ToSlash(dbPath))
|
||||
if err != nil {
|
||||
t.Fatalf("open legacy db: %v", err)
|
||||
}
|
||||
legacyDDL := []string{
|
||||
`CREATE TABLE basemap_versions (
|
||||
id TEXT PRIMARY KEY,
|
||||
basemap_id TEXT NOT NULL,
|
||||
scan_source_id TEXT,
|
||||
version TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
is_default INTEGER NOT NULL DEFAULT 0,
|
||||
manifest_path TEXT NOT NULL DEFAULT '',
|
||||
tile_root_path TEXT NOT NULL,
|
||||
url_template TEXT NOT NULL,
|
||||
tile_format TEXT NOT NULL,
|
||||
tile_scheme TEXT NOT NULL,
|
||||
min_zoom INTEGER NOT NULL DEFAULT 0,
|
||||
max_zoom INTEGER NOT NULL DEFAULT 0,
|
||||
bbox_json TEXT NOT NULL DEFAULT '',
|
||||
attribution TEXT NOT NULL DEFAULT '',
|
||||
metadata_json TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);`,
|
||||
`CREATE TABLE push_records (
|
||||
id TEXT PRIMARY KEY,
|
||||
target_system_id TEXT NOT NULL,
|
||||
basemap_version_id TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
request_json TEXT NOT NULL DEFAULT '',
|
||||
response_status INTEGER,
|
||||
response_body TEXT NOT NULL DEFAULT '',
|
||||
error_message TEXT NOT NULL DEFAULT '',
|
||||
pushed_at TEXT NOT NULL,
|
||||
finished_at TEXT
|
||||
);`,
|
||||
`INSERT INTO basemap_versions (id, basemap_id, version, status, tile_root_path, url_template, tile_format, tile_scheme, created_at, updated_at)
|
||||
VALUES ('v1', 'b1', 'current', 'ready', '/tiles/demo', '/tiles/demo/current/{z}/{x}/{y}.png', 'png', 'xyz', '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z');`,
|
||||
}
|
||||
for _, statement := range legacyDDL {
|
||||
if _, err := legacy.Exec(statement); err != nil {
|
||||
legacy.Close()
|
||||
t.Fatalf("seed legacy schema: %v", err)
|
||||
}
|
||||
}
|
||||
if err := legacy.Close(); err != nil {
|
||||
t.Fatalf("close legacy db: %v", err)
|
||||
}
|
||||
|
||||
store := openTestStore(t, StoreConfig{SQLitePath: dbPath, HealthCheckInterval: time.Minute})
|
||||
|
||||
// The pre-existing row must survive and gain default health values.
|
||||
var health, detail string
|
||||
if err := store.db.QueryRow(`SELECT health, health_detail FROM basemap_versions WHERE id = 'v1'`).Scan(&health, &detail); err != nil {
|
||||
t.Fatalf("legacy row lost or columns missing: %v", err)
|
||||
}
|
||||
if health != "" {
|
||||
t.Fatalf("expected a blank default health, got %q", health)
|
||||
}
|
||||
|
||||
var attempts int
|
||||
var nextAttempt sql.NullString
|
||||
if err := store.db.QueryRow(`SELECT attempt_count, next_attempt_at FROM push_records LIMIT 1`).Scan(&attempts, &nextAttempt); err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
t.Fatalf("retry columns missing on push_records: %v", err)
|
||||
}
|
||||
|
||||
if err := store.ensureColumn(context.Background(), "basemap_versions", "health", "TEXT NOT NULL DEFAULT ''"); err != nil {
|
||||
t.Fatalf("ensureColumn must be idempotent: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTokenScopesNormalization(t *testing.T) {
|
||||
if got := normalizeTokenScopes(nil); got != "read" {
|
||||
t.Fatalf("expected read by default, got %q", got)
|
||||
}
|
||||
if got := normalizeTokenScopes([]string{"slice"}); got != "read,slice" {
|
||||
t.Fatalf("expected read,slice, got %q", got)
|
||||
}
|
||||
if got := normalizeTokenScopes([]string{"SLICE", "bogus", " read "}); got != "read,slice" {
|
||||
t.Fatalf("expected unknown scopes to be dropped, got %q", got)
|
||||
}
|
||||
|
||||
store := openTestStore(t, StoreConfig{SQLitePath: filepath.Join(t.TempDir(), "basemap.db")})
|
||||
token, err := store.CreateToken(context.Background(), CreateTokenInput{
|
||||
Name: "slicer",
|
||||
Scopes: []string{"slice"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create token: %v", err)
|
||||
}
|
||||
auth, err := store.AuthorizeToken(context.Background(), token.Token)
|
||||
if err != nil {
|
||||
t.Fatalf("authorize: %v", err)
|
||||
}
|
||||
if !auth.CanSlice() {
|
||||
t.Fatal("expected the token to carry the slice scope after a round trip")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateTileJobIsIdempotentByCode(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
tileRoot := filepath.Join(root, "tiles")
|
||||
source := filepath.Join(root, "input.tif")
|
||||
writeTestFile(t, source, "fake raster")
|
||||
|
||||
store := openTestStore(t, StoreConfig{
|
||||
SQLitePath: filepath.Join(root, "basemap.db"),
|
||||
ScanRoot: tileRoot,
|
||||
APIBaseURL: "http://127.0.0.1:8910",
|
||||
TileBaseURL: "http://127.0.0.1:8910",
|
||||
VectorBaseURL: "http://127.0.0.1:8910",
|
||||
})
|
||||
|
||||
input := CreateTileJobInput{Code: "repeat-me", SourcePath: source, TargetCode: "demo"}
|
||||
first, err := store.CreateTileJob(ctx, input)
|
||||
if err != nil {
|
||||
t.Fatalf("first submit: %v", err)
|
||||
}
|
||||
second, err := store.CreateTileJob(ctx, input)
|
||||
if err != nil {
|
||||
t.Fatalf("second submit must be accepted: %v", err)
|
||||
}
|
||||
if first.ID != second.ID {
|
||||
t.Fatalf("expected the same job back, got %s then %s", first.ID, second.ID)
|
||||
}
|
||||
|
||||
jobs, err := store.ListTileJobs(ctx, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if len(jobs) != 1 {
|
||||
t.Fatalf("expected exactly one job, got %d", len(jobs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetTileJob(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
source := filepath.Join(root, "input.tif")
|
||||
writeTestFile(t, source, "fake raster")
|
||||
|
||||
store := openTestStore(t, StoreConfig{
|
||||
SQLitePath: filepath.Join(root, "basemap.db"),
|
||||
ScanRoot: filepath.Join(root, "tiles"),
|
||||
APIBaseURL: "http://127.0.0.1:8910",
|
||||
TileBaseURL: "http://127.0.0.1:8910",
|
||||
VectorBaseURL: "http://127.0.0.1:8910",
|
||||
})
|
||||
job, err := store.CreateTileJob(ctx, CreateTileJobInput{SourcePath: source, TargetCode: "demo"})
|
||||
if err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
found, err := store.GetTileJob(ctx, job.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if found.Code != job.Code {
|
||||
t.Fatalf("unexpected job %+v", found)
|
||||
}
|
||||
if _, err := store.GetTileJob(ctx, "nope"); err == nil {
|
||||
t.Fatal("expected a missing job to error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTileJobCallbackFiresAndRetries(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
tileRoot := filepath.Join(root, "tiles")
|
||||
source := filepath.Join(root, "input.tif")
|
||||
writeTestFile(t, source, "fake raster")
|
||||
|
||||
var attempts int
|
||||
received := make(chan map[string]any, 1)
|
||||
callback := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts++
|
||||
if attempts < 2 {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
var payload map[string]any
|
||||
_ = json.NewDecoder(r.Body).Decode(&payload)
|
||||
select {
|
||||
case received <- payload:
|
||||
default:
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer callback.Close()
|
||||
|
||||
store := openTestStore(t, StoreConfig{
|
||||
SQLitePath: filepath.Join(root, "basemap.db"),
|
||||
ScanRoot: tileRoot,
|
||||
APIBaseURL: "http://127.0.0.1:8910",
|
||||
TileBaseURL: "http://127.0.0.1:8910",
|
||||
VectorBaseURL: "http://127.0.0.1:8910",
|
||||
TileJobCommand: "definitely-not-a-real-binary {input} {output}",
|
||||
})
|
||||
if err := os.MkdirAll(tileRoot, 0o755); err != nil {
|
||||
t.Fatalf("mkdir: %v", err)
|
||||
}
|
||||
|
||||
job, err := store.CreateTileJob(ctx, CreateTileJobInput{
|
||||
SourcePath: source,
|
||||
TargetCode: "demo",
|
||||
CallbackURL: callback.URL,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
store.runTileJob(ctx, job)
|
||||
|
||||
// The first delivery fails, so the callback must be parked with a backoff.
|
||||
if err := store.dispatchTileJobCallbacks(ctx); err != nil {
|
||||
t.Fatalf("dispatch: %v", err)
|
||||
}
|
||||
jobs, err := store.ListTileJobs(ctx, 5)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if jobs[0].CallbackStatus != pushStatusRetrying {
|
||||
t.Fatalf("expected the callback to be retrying, got %q", jobs[0].CallbackStatus)
|
||||
}
|
||||
if jobs[0].CallbackNextAttemptAt == nil {
|
||||
t.Fatal("expected a next attempt to be scheduled")
|
||||
}
|
||||
|
||||
if _, err := store.db.ExecContext(ctx, `UPDATE tile_jobs SET callback_next_attempt_at = ? WHERE id = ?`, toRFC3339(nowUTC().Add(-time.Minute)), job.ID); err != nil {
|
||||
t.Fatalf("force due: %v", err)
|
||||
}
|
||||
if err := store.dispatchTileJobCallbacks(ctx); err != nil {
|
||||
t.Fatalf("dispatch retry: %v", err)
|
||||
}
|
||||
|
||||
jobs, _ = store.ListTileJobs(ctx, 5)
|
||||
if jobs[0].CallbackStatus != pushStatusSucceeded {
|
||||
t.Fatalf("expected the callback to succeed, got %q (%s)", jobs[0].CallbackStatus, jobs[0].CallbackMessage)
|
||||
}
|
||||
select {
|
||||
case payload := <-received:
|
||||
if payload["event"] != "tile.job.finished" {
|
||||
t.Fatalf("unexpected callback payload: %v", payload)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("callback payload never arrived")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTileJobCallbackSkippedWhenNoURL(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
tileRoot := filepath.Join(root, "tiles")
|
||||
source := filepath.Join(root, "input.tif")
|
||||
writeTestFile(t, source, "fake raster")
|
||||
if err := os.MkdirAll(tileRoot, 0o755); err != nil {
|
||||
t.Fatalf("mkdir: %v", err)
|
||||
}
|
||||
|
||||
store := openTestStore(t, StoreConfig{
|
||||
SQLitePath: filepath.Join(root, "basemap.db"),
|
||||
ScanRoot: tileRoot,
|
||||
APIBaseURL: "http://127.0.0.1:8910",
|
||||
TileBaseURL: "http://127.0.0.1:8910",
|
||||
VectorBaseURL: "http://127.0.0.1:8910",
|
||||
TileJobCommand: "definitely-not-a-real-binary {input} {output}",
|
||||
})
|
||||
job, err := store.CreateTileJob(ctx, CreateTileJobInput{SourcePath: source, TargetCode: "demo"})
|
||||
if err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
store.runTileJob(ctx, job)
|
||||
|
||||
jobs, _ := store.ListTileJobs(ctx, 5)
|
||||
if jobs[0].CallbackStatus != "" {
|
||||
t.Fatalf("expected no callback scheduling, got %q", jobs[0].CallbackStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTileJobTemplateExpandsInputList(t *testing.T) {
|
||||
job := TileJob{
|
||||
SourcePaths: []string{`C:\a\in1.tif`, `C:\b\in2.tif`},
|
||||
TargetCode: "demo",
|
||||
OutputRoot: `C:\out`,
|
||||
}
|
||||
got := tileJobVRTCommandArgs(`gdalbuildvrt -resolution highest {output_vrt} {inputs}`, job, `C:\work\m.vrt`)
|
||||
want := []string{"gdalbuildvrt", "-resolution", "highest", `C:\work\m.vrt`, `C:\a\in1.tif`, `C:\b\in2.tif`}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("got %v, want %v", got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("arg %d = %q, want %q", i, got[i], want[i])
|
||||
}
|
||||
}
|
||||
|
||||
// The slicer still gets one input path (the VRT) when mosaicking.
|
||||
slicer := tileJobCommandArgs(`gdal2tiles.py --exclude {input} {output}`, job, `C:\work\m.vrt`)
|
||||
if slicer[2] != `C:\work\m.vrt` {
|
||||
t.Fatalf("expected the VRT to be the slicer input, got %v", slicer)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTileJobMosaicsMultipleSources(t *testing.T) {
|
||||
if runtime.GOOS != "windows" {
|
||||
t.Skip("the stub commands below use cmd.exe")
|
||||
}
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
tileRoot := filepath.Join(root, "tiles")
|
||||
if err := os.MkdirAll(tileRoot, 0o755); err != nil {
|
||||
t.Fatalf("mkdir: %v", err)
|
||||
}
|
||||
first := filepath.Join(root, "flight-a.tif")
|
||||
second := filepath.Join(root, "flight-b.tif")
|
||||
writeTestFile(t, first, "fake raster a")
|
||||
writeTestFile(t, second, "fake raster b")
|
||||
|
||||
store := openTestStore(t, StoreConfig{
|
||||
SQLitePath: filepath.Join(root, "basemap.db"),
|
||||
ScanRoot: tileRoot,
|
||||
APIBaseURL: "http://127.0.0.1:8910",
|
||||
TileBaseURL: "http://127.0.0.1:8910",
|
||||
VectorBaseURL: "http://127.0.0.1:8910",
|
||||
// Stand-ins: the mosaic step writes the VRT, the slicer lays down a z tree.
|
||||
TileJobVRTCommand: `cmd /c echo vrt > {output_vrt}`,
|
||||
TileJobCommand: `cmd /c mkdir "{output}\0\0"`,
|
||||
})
|
||||
|
||||
job, err := store.CreateTileJob(ctx, CreateTileJobInput{
|
||||
SourcePaths: []string{first, second},
|
||||
TargetCode: "uav-2026",
|
||||
ZoomMin: 0,
|
||||
ZoomMax: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create job: %v", err)
|
||||
}
|
||||
if job.SourceCount != 2 || len(job.SourcePaths) != 2 {
|
||||
t.Fatalf("expected two sources on the job, got %d / %v", job.SourceCount, job.SourcePaths)
|
||||
}
|
||||
|
||||
store.runTileJob(ctx, job)
|
||||
|
||||
jobs, err := store.ListTileJobs(ctx, 5)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if jobs[0].Status != tileJobSucceeded {
|
||||
t.Fatalf("expected success, got %q (%s)", jobs[0].Status, jobs[0].Message)
|
||||
}
|
||||
if jobs[0].SourceCount != 2 {
|
||||
t.Fatalf("expected source_count 2 after a round trip, got %d", jobs[0].SourceCount)
|
||||
}
|
||||
|
||||
// The mosaic step ran: its VRT sits beside the job log.
|
||||
vrt := filepath.Join(filepath.Dir(jobs[0].LogPath), job.Code+".vrt")
|
||||
if _, err := os.Stat(vrt); err != nil {
|
||||
t.Fatalf("expected the mosaic VRT at %s: %v", vrt, err)
|
||||
}
|
||||
|
||||
// Both flights are recorded against the layer.
|
||||
sources, err := store.ListLayerSources(ctx, "uav-2026")
|
||||
if err != nil {
|
||||
t.Fatalf("list layer sources: %v", err)
|
||||
}
|
||||
if len(sources) != 2 {
|
||||
t.Fatalf("expected 2 recorded sources, got %d", len(sources))
|
||||
}
|
||||
|
||||
// Adding the same image twice does not duplicate the registry entry.
|
||||
if err := store.AddLayerSources(ctx, "uav-2026", []string{first}); err != nil {
|
||||
t.Fatalf("re-add: %v", err)
|
||||
}
|
||||
sources, _ = store.ListLayerSources(ctx, "uav-2026")
|
||||
if len(sources) != 2 {
|
||||
t.Fatalf("expected re-adding to be a no-op, got %d entries", len(sources))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTileJobSingleSourceSkipsMosaic(t *testing.T) {
|
||||
if runtime.GOOS != "windows" {
|
||||
t.Skip("the stub command below uses cmd.exe")
|
||||
}
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
tileRoot := filepath.Join(root, "tiles")
|
||||
if err := os.MkdirAll(tileRoot, 0o755); err != nil {
|
||||
t.Fatalf("mkdir: %v", err)
|
||||
}
|
||||
source := filepath.Join(root, "flight-a.tif")
|
||||
writeTestFile(t, source, "fake raster")
|
||||
|
||||
store := openTestStore(t, StoreConfig{
|
||||
SQLitePath: filepath.Join(root, "basemap.db"),
|
||||
ScanRoot: tileRoot,
|
||||
APIBaseURL: "http://127.0.0.1:8910",
|
||||
TileBaseURL: "http://127.0.0.1:8910",
|
||||
VectorBaseURL: "http://127.0.0.1:8910",
|
||||
// A failing mosaic command proves the single-source path never calls it.
|
||||
TileJobVRTCommand: `definitely-not-a-real-binary {output_vrt} {inputs}`,
|
||||
TileJobCommand: `cmd /c mkdir "{output}\0\0"`,
|
||||
})
|
||||
|
||||
job, err := store.CreateTileJob(ctx, CreateTileJobInput{SourcePath: source, TargetCode: "uav-2026"})
|
||||
if err != nil {
|
||||
t.Fatalf("create job: %v", err)
|
||||
}
|
||||
if job.SourceCount != 1 {
|
||||
t.Fatalf("expected source_count 1, got %d", job.SourceCount)
|
||||
}
|
||||
store.runTileJob(ctx, job)
|
||||
|
||||
jobs, _ := store.ListTileJobs(ctx, 5)
|
||||
if jobs[0].Status != tileJobSucceeded {
|
||||
t.Fatalf("expected the single source to slice directly, got %q (%s)", jobs[0].Status, jobs[0].Message)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package basemap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Availability verdicts recorded on catalog rows.
|
||||
const (
|
||||
healthOK = "ok"
|
||||
healthMissing = "missing"
|
||||
healthUnknown = "unknown"
|
||||
)
|
||||
|
||||
// HealthReport summarizes one inspection pass.
|
||||
type HealthReport struct {
|
||||
CheckedAt time.Time `json:"checked_at"`
|
||||
VersionsChecked int `json:"versions_checked"`
|
||||
VersionsMissing int `json:"versions_missing"`
|
||||
VectorsChecked int `json:"vectors_checked"`
|
||||
VectorsMissing int `json:"vectors_missing"`
|
||||
}
|
||||
|
||||
// CheckHealth stats every catalogued path and records whether it is still
|
||||
// there, so a directory that was moved or deleted surfaces before a client
|
||||
// hits a 404. It shares scanMu with the scanners because both rewrite catalog
|
||||
// rows.
|
||||
func (s *Store) CheckHealth(ctx context.Context) (HealthReport, error) {
|
||||
s.scanMu.Lock()
|
||||
defer s.scanMu.Unlock()
|
||||
|
||||
checkedAt := nowUTC()
|
||||
report := HealthReport{CheckedAt: checkedAt}
|
||||
|
||||
type verdict struct {
|
||||
id string
|
||||
health string
|
||||
detail string
|
||||
}
|
||||
|
||||
versions, err := s.collectHealth(ctx, `SELECT id, tile_root_path FROM basemap_versions`, true)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
vectors, err := s.collectHealth(ctx, `SELECT id, file_path FROM vector_assets`, false)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return report, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
stamp := toRFC3339(checkedAt)
|
||||
for _, item := range versions {
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
UPDATE basemap_versions SET health = ?, health_checked_at = ?, health_detail = ? WHERE id = ?
|
||||
`, item.health, stamp, item.detail, item.id); err != nil {
|
||||
return report, err
|
||||
}
|
||||
report.VersionsChecked++
|
||||
if item.health != healthOK {
|
||||
report.VersionsMissing++
|
||||
}
|
||||
}
|
||||
for _, item := range vectors {
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
UPDATE vector_assets SET health = ?, health_checked_at = ?, health_detail = ? WHERE id = ?
|
||||
`, item.health, stamp, item.detail, item.id); err != nil {
|
||||
return report, err
|
||||
}
|
||||
report.VectorsChecked++
|
||||
if item.health != healthOK {
|
||||
report.VectorsMissing++
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return report, err
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
type healthVerdict struct {
|
||||
id string
|
||||
health string
|
||||
detail string
|
||||
}
|
||||
|
||||
func (s *Store) collectHealth(ctx context.Context, query string, wantDir bool) ([]healthVerdict, error) {
|
||||
rows, err := s.db.QueryContext(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var verdicts []healthVerdict
|
||||
for rows.Next() {
|
||||
var id, path string
|
||||
if err := rows.Scan(&id, &path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
health, detail := inspectPath(path, wantDir)
|
||||
verdicts = append(verdicts, healthVerdict{id: id, health: health, detail: detail})
|
||||
}
|
||||
return verdicts, rows.Err()
|
||||
}
|
||||
|
||||
func inspectPath(path string, wantDir bool) (string, string) {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return healthMissing, "路径不存在"
|
||||
}
|
||||
return healthUnknown, err.Error()
|
||||
}
|
||||
if wantDir && !info.IsDir() {
|
||||
return healthMissing, "路径不是目录"
|
||||
}
|
||||
if !wantDir && info.IsDir() {
|
||||
return healthMissing, "路径是目录"
|
||||
}
|
||||
return healthOK, ""
|
||||
}
|
||||
|
||||
// The console reads health through these maps so the row scanners on the tile
|
||||
// request path stay untouched.
|
||||
func (s *Store) loadVersionHealth(ctx context.Context) (map[string]ResourceHealth, error) {
|
||||
return s.loadHealth(ctx, `SELECT id, health, health_detail, health_checked_at FROM basemap_versions`)
|
||||
}
|
||||
|
||||
func (s *Store) loadVectorHealth(ctx context.Context) (map[string]ResourceHealth, error) {
|
||||
return s.loadHealth(ctx, `SELECT id, health, health_detail, health_checked_at FROM vector_assets`)
|
||||
}
|
||||
|
||||
func (s *Store) loadHealth(ctx context.Context, query string) (map[string]ResourceHealth, error) {
|
||||
rows, err := s.db.QueryContext(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
result := map[string]ResourceHealth{}
|
||||
for rows.Next() {
|
||||
var id, health, detail, checkedAt string
|
||||
if err := rows.Scan(&id, &health, &detail, &checkedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result[id] = ResourceHealth{Health: health, Detail: detail, CheckedAt: parseTime(checkedAt)}
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package basemap
|
||||
|
||||
func defaultText(value, fallback string) string {
|
||||
if value != "" {
|
||||
return value
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package basemap
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func LoadManifest(path string) (Manifest, error) {
|
||||
content, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return Manifest{}, err
|
||||
}
|
||||
|
||||
var item Manifest
|
||||
if err := json.Unmarshal(content, &item); err != nil {
|
||||
return Manifest{}, fmt.Errorf("decode manifest %s: %w", path, err)
|
||||
}
|
||||
|
||||
item.Code = normalizeCode(item.Code)
|
||||
item.Name = strings.TrimSpace(item.Name)
|
||||
item.Type = defaultText(strings.TrimSpace(item.Type), "xyz")
|
||||
item.Description = strings.TrimSpace(item.Description)
|
||||
item.Version = strings.TrimSpace(item.Version)
|
||||
item.Status = defaultText(strings.TrimSpace(item.Status), "ready")
|
||||
item.TileFormat = normalizeTileFormat(item.TileFormat)
|
||||
item.TileScheme = defaultText(strings.TrimSpace(item.TileScheme), "xyz")
|
||||
item.Attribution = strings.TrimSpace(item.Attribution)
|
||||
item.RootPath = strings.TrimSpace(item.RootPath)
|
||||
if item.RootPath == "" {
|
||||
item.RootPath = "tiles"
|
||||
}
|
||||
if item.MaxZoom < item.MinZoom {
|
||||
return Manifest{}, fmt.Errorf("manifest %s has invalid zoom range", path)
|
||||
}
|
||||
if len(item.BBox) != 0 && len(item.BBox) != 4 {
|
||||
return Manifest{}, fmt.Errorf("manifest %s has invalid bbox", path)
|
||||
}
|
||||
if item.Code == "" {
|
||||
return Manifest{}, fmt.Errorf("manifest %s missing code", path)
|
||||
}
|
||||
if item.Name == "" {
|
||||
item.Name = item.Code
|
||||
}
|
||||
if item.Version == "" {
|
||||
item.Version = filepath.Base(filepath.Dir(path))
|
||||
}
|
||||
if item.Metadata == nil {
|
||||
item.Metadata = map[string]any{}
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func normalizeCode(value string) string {
|
||||
value = strings.TrimSpace(strings.ToLower(value))
|
||||
if value == "" {
|
||||
return ""
|
||||
}
|
||||
var builder strings.Builder
|
||||
lastDash := false
|
||||
for _, r := range value {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z':
|
||||
builder.WriteRune(r)
|
||||
lastDash = false
|
||||
case r >= '0' && r <= '9':
|
||||
builder.WriteRune(r)
|
||||
lastDash = false
|
||||
case r == '_' || r == '-' || r == '.' || r == ' ':
|
||||
if builder.Len() > 0 && !lastDash {
|
||||
builder.WriteByte('-')
|
||||
lastDash = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return strings.Trim(builder.String(), "-")
|
||||
}
|
||||
|
||||
func normalizeTileFormat(value string) string {
|
||||
value = strings.TrimSpace(strings.ToLower(value))
|
||||
switch value {
|
||||
case "jpg", "jpeg":
|
||||
return "jpg"
|
||||
case "png", "webp", "pbf":
|
||||
return value
|
||||
case "":
|
||||
return "png"
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
package basemap
|
||||
|
||||
import "time"
|
||||
|
||||
type SystemInfo struct {
|
||||
ServiceName string `json:"service_name"`
|
||||
ServiceVersion string `json:"service_version"`
|
||||
Commit string `json:"commit"`
|
||||
Env string `json:"env"`
|
||||
APIBaseURL string `json:"api_base_url"`
|
||||
TileBaseURL string `json:"tile_base_url"`
|
||||
VectorBaseURL string `json:"vector_base_url"`
|
||||
SQLitePath string `json:"sqlite_path"`
|
||||
ScanRoot string `json:"scan_root"`
|
||||
VectorScanRoot string `json:"vector_scan_root"`
|
||||
TileJobCommand string `json:"tile_job_command,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
}
|
||||
|
||||
type Basemap struct {
|
||||
ID string `json:"id"`
|
||||
Code string `json:"code"`
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
Status string `json:"status"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Default *BasemapVersion `json:"default_version,omitempty"`
|
||||
Versions []BasemapVersion `json:"versions,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type BasemapVersion struct {
|
||||
ID string `json:"id"`
|
||||
BasemapID string `json:"basemap_id"`
|
||||
BasemapCode string `json:"basemap_code"`
|
||||
Version string `json:"version"`
|
||||
Status string `json:"status"`
|
||||
IsDefault bool `json:"is_default"`
|
||||
ManifestPath string `json:"manifest_path"`
|
||||
TileRootPath string `json:"tile_root_path"`
|
||||
URLTemplate string `json:"url_template"`
|
||||
TileFormat string `json:"tile_format"`
|
||||
TileScheme string `json:"tile_scheme"`
|
||||
MinZoom int `json:"min_zoom"`
|
||||
MaxZoom int `json:"max_zoom"`
|
||||
BBox []float64 `json:"bbox,omitempty"`
|
||||
Attribution string `json:"attribution,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type ScanSource struct {
|
||||
ID string `json:"id"`
|
||||
Code string `json:"code"`
|
||||
Name string `json:"name"`
|
||||
RootPath string `json:"root_path"`
|
||||
ManifestName string `json:"manifest_name"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type ScanRun struct {
|
||||
ID string `json:"id"`
|
||||
ScanSourceID string `json:"scan_source_id"`
|
||||
SourceCode string `json:"source_code"`
|
||||
Status string `json:"status"`
|
||||
ScannedCount int `json:"scanned_count"`
|
||||
AddedCount int `json:"added_count"`
|
||||
UpdatedCount int `json:"updated_count"`
|
||||
RemovedCount int `json:"removed_count"`
|
||||
Summary map[string]any `json:"summary,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
FinishedAt time.Time `json:"finished_at"`
|
||||
}
|
||||
|
||||
type TargetSystem struct {
|
||||
ID string `json:"id"`
|
||||
Code string `json:"code"`
|
||||
Name string `json:"name"`
|
||||
CallbackURL string `json:"callback_url"`
|
||||
CallbackMethod string `json:"callback_method"`
|
||||
CallbackHeaders map[string]string `json:"callback_headers,omitempty"`
|
||||
Enabled bool `json:"enabled"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type ServiceToken struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
TokenPrefix string `json:"token_prefix"`
|
||||
Scopes []string `json:"scopes"`
|
||||
Status string `json:"status"`
|
||||
ExpiresAt *time.Time `json:"expires_at,omitempty"`
|
||||
LastUsedAt *time.Time `json:"last_used_at,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Grants []TokenGrant `json:"grants,omitempty"`
|
||||
VectorGrants []VectorTokenGrant `json:"vector_grants,omitempty"`
|
||||
}
|
||||
|
||||
type TokenGrant struct {
|
||||
ID string `json:"id"`
|
||||
TokenID string `json:"token_id"`
|
||||
BasemapID string `json:"basemap_id"`
|
||||
BasemapCode string `json:"basemap_code"`
|
||||
BasemapVersionID *string `json:"basemap_version_id,omitempty"`
|
||||
BasemapVersion *string `json:"basemap_version,omitempty"`
|
||||
Permission string `json:"permission"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type PushRecord struct {
|
||||
ID string `json:"id"`
|
||||
TargetSystemID string `json:"target_system_id"`
|
||||
TargetSystemCode string `json:"target_system_code"`
|
||||
BasemapVersionID string `json:"basemap_version_id"`
|
||||
BasemapCode string `json:"basemap_code"`
|
||||
BasemapVersion string `json:"basemap_version"`
|
||||
Status string `json:"status"`
|
||||
Request map[string]any `json:"request,omitempty"`
|
||||
ResponseStatus *int `json:"response_status,omitempty"`
|
||||
ResponseBody string `json:"response_body,omitempty"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
AttemptCount int `json:"attempt_count"`
|
||||
NextAttemptAt *time.Time `json:"next_attempt_at,omitempty"`
|
||||
PushedAt time.Time `json:"pushed_at"`
|
||||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
||||
}
|
||||
|
||||
type VectorAsset struct {
|
||||
ID string `json:"id"`
|
||||
Code string `json:"code"`
|
||||
Name string `json:"name"`
|
||||
Format string `json:"format"`
|
||||
Status string `json:"status"`
|
||||
Description string `json:"description,omitempty"`
|
||||
FilePath string `json:"file_path"`
|
||||
FileName string `json:"file_name"`
|
||||
FileSize int64 `json:"file_size"`
|
||||
Checksum string `json:"checksum"`
|
||||
DownloadURL string `json:"download_url"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type VectorTokenGrant struct {
|
||||
ID string `json:"id"`
|
||||
TokenID string `json:"token_id"`
|
||||
VectorID string `json:"vector_id"`
|
||||
VectorCode string `json:"vector_code"`
|
||||
Permission string `json:"permission"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type VectorPushRecord struct {
|
||||
ID string `json:"id"`
|
||||
TargetSystemID string `json:"target_system_id"`
|
||||
TargetSystemCode string `json:"target_system_code"`
|
||||
VectorAssetID string `json:"vector_asset_id"`
|
||||
VectorCode string `json:"vector_code"`
|
||||
Status string `json:"status"`
|
||||
Request map[string]any `json:"request,omitempty"`
|
||||
ResponseStatus *int `json:"response_status,omitempty"`
|
||||
ResponseBody string `json:"response_body,omitempty"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
AttemptCount int `json:"attempt_count"`
|
||||
NextAttemptAt *time.Time `json:"next_attempt_at,omitempty"`
|
||||
PushedAt time.Time `json:"pushed_at"`
|
||||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
||||
}
|
||||
|
||||
type Manifest struct {
|
||||
Code string `json:"code"`
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
Description string `json:"description"`
|
||||
Version string `json:"version"`
|
||||
Status string `json:"status"`
|
||||
TileFormat string `json:"tile_format"`
|
||||
TileScheme string `json:"tile_scheme"`
|
||||
MinZoom int `json:"min_zoom"`
|
||||
MaxZoom int `json:"max_zoom"`
|
||||
BBox []float64 `json:"bbox"`
|
||||
Attribution string `json:"attribution"`
|
||||
RootPath string `json:"root_path"`
|
||||
IsDefault bool `json:"is_default"`
|
||||
Metadata map[string]any `json:"metadata"`
|
||||
}
|
||||
|
||||
type CreateScanSourceInput struct {
|
||||
Code string `json:"code"`
|
||||
Name string `json:"name"`
|
||||
RootPath string `json:"root_path"`
|
||||
ManifestName string `json:"manifest_name"`
|
||||
}
|
||||
|
||||
type CreateTargetSystemInput struct {
|
||||
Code string `json:"code"`
|
||||
Name string `json:"name"`
|
||||
CallbackURL string `json:"callback_url"`
|
||||
CallbackMethod string `json:"callback_method"`
|
||||
CallbackHeaders map[string]string `json:"callback_headers"`
|
||||
}
|
||||
|
||||
type CreateTokenInput struct {
|
||||
Name string
|
||||
BasemapCodes []string
|
||||
VectorCodes []string
|
||||
Scopes []string
|
||||
ExpiresAt *time.Time
|
||||
}
|
||||
|
||||
type TokenAuth struct {
|
||||
Token ServiceToken
|
||||
// permission -> basemap_code -> version scope
|
||||
Permissions map[string]map[string]VersionScope
|
||||
VectorPermissions map[string]map[string]struct{}
|
||||
// Scopes are coarse capabilities beyond resource grants, e.g. "slice".
|
||||
Scopes map[string]struct{}
|
||||
}
|
||||
|
||||
type VersionScope struct {
|
||||
AllVersions bool
|
||||
Versions map[string]struct{}
|
||||
}
|
||||
|
||||
// ResourceHealth is the availability verdict recorded by the health inspector.
|
||||
// It is kept separate from Status: Status is a catalog fact, Health is whether
|
||||
// the bytes are still on disk right now.
|
||||
type ResourceHealth struct {
|
||||
Health string `json:"health"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
CheckedAt time.Time `json:"checked_at"`
|
||||
}
|
||||
|
||||
// LayerInventory is one layer and the imagery it is built from, ordered for
|
||||
// deterministic rendering in the console.
|
||||
type LayerInventory struct {
|
||||
TargetCode string `json:"target_code"`
|
||||
Sources []LayerSource `json:"sources"`
|
||||
// ZoomMin/ZoomMax/TileFormat mirror the layer's current basemap version so
|
||||
// a rebuild can reuse the range it is already published at.
|
||||
ZoomMin int `json:"zoom_min"`
|
||||
ZoomMax int `json:"zoom_max"`
|
||||
TileFormat string `json:"tile_format"`
|
||||
}
|
||||
|
||||
type DashboardData struct {
|
||||
SystemInfo SystemInfo
|
||||
Basemaps []Basemap
|
||||
VectorAssets []VectorAsset
|
||||
ScanSources []ScanSource
|
||||
ScanRuns []ScanRun
|
||||
TargetSystems []TargetSystem
|
||||
Tokens []ServiceToken
|
||||
PushRecords []PushRecord
|
||||
VectorPushRecords []VectorPushRecord
|
||||
Flash string
|
||||
CreatedToken string
|
||||
VersionHealth map[string]ResourceHealth
|
||||
VectorHealth map[string]ResourceHealth
|
||||
TileJobs []TileJob
|
||||
Layers []LayerInventory
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package basemap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func writeTestFile(t *testing.T, path, body string) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
t.Fatalf("mkdir %s: %v", filepath.Dir(path), err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||
t.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func manifestBody(code, name string) string {
|
||||
return `{"code":"` + code + `","name":"` + name + `","type":"xyz","version":"current","tile_format":"png","root_path":"tiles","is_default":true}`
|
||||
}
|
||||
|
||||
// The manifest lookup must not descend into the tile payload: a manifest nested
|
||||
// inside another basemap's "tiles" subtree must be ignored.
|
||||
func TestFindManifestVersionsPrunesTilePayload(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
writeTestFile(t, filepath.Join(root, "demo", "manifest.json"), manifestBody("demo", "Demo"))
|
||||
writeTestFile(t, filepath.Join(root, "demo", "tiles", "0", "0", "0.png"), "tile")
|
||||
writeTestFile(t, filepath.Join(root, "demo", "tiles", "nested", "manifest.json"), manifestBody("nested", "Nested"))
|
||||
|
||||
store := &Store{}
|
||||
items, covered, err := store.findManifestVersions(ScanSource{RootPath: root, ManifestName: "manifest.json"})
|
||||
if err != nil {
|
||||
t.Fatalf("find manifests: %v", err)
|
||||
}
|
||||
if len(items) != 1 {
|
||||
t.Fatalf("expected 1 manifest, got %d: %+v", len(items), items)
|
||||
}
|
||||
if items[0].BasemapCode != "demo" {
|
||||
t.Fatalf("expected code demo, got %q", items[0].BasemapCode)
|
||||
}
|
||||
if _, ok := covered["demo"]; !ok {
|
||||
t.Fatalf("expected demo to be marked as covered, got %v", covered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsNumericDirName(t *testing.T) {
|
||||
for _, name := range []string{"0", "14", "007"} {
|
||||
if !isNumericDirName(name) {
|
||||
t.Fatalf("expected %q to be numeric", name)
|
||||
}
|
||||
}
|
||||
for _, name := range []string{"", "gaode-image", "2026-04", "tiles", "1a"} {
|
||||
if isNumericDirName(name) {
|
||||
t.Fatalf("expected %q to be non-numeric", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteDSNEnablesWALAndForeignKeys(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
store, err := OpenStore(StoreConfig{SQLitePath: filepath.Join(root, "basemap.db")})
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
var mode string
|
||||
if err := store.db.QueryRow("PRAGMA journal_mode;").Scan(&mode); err != nil {
|
||||
t.Fatalf("read journal_mode: %v", err)
|
||||
}
|
||||
if !strings.EqualFold(mode, "wal") {
|
||||
t.Fatalf("expected WAL journal mode, got %q", mode)
|
||||
}
|
||||
|
||||
var foreignKeys int
|
||||
if err := store.db.QueryRow("PRAGMA foreign_keys;").Scan(&foreignKeys); err != nil {
|
||||
t.Fatalf("read foreign_keys: %v", err)
|
||||
}
|
||||
if foreignKeys != 1 {
|
||||
t.Fatalf("expected foreign_keys=1, got %d", foreignKeys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthorizeTokenThrottlesLastUsedWrite(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
tileRoot := filepath.Join(root, "tiles")
|
||||
writeTestFile(t, filepath.Join(tileRoot, "google_image", "0", "0", "0.webp"), "tile")
|
||||
|
||||
store, err := OpenStore(StoreConfig{
|
||||
SQLitePath: filepath.Join(root, "basemap.db"),
|
||||
ScanRoot: tileRoot,
|
||||
APIBaseURL: "http://127.0.0.1:8910",
|
||||
TileBaseURL: "http://127.0.0.1:8910",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
if _, err := store.RunScan(ctx, defaultScanSourceCode); err != nil {
|
||||
t.Fatalf("run scan: %v", err)
|
||||
}
|
||||
created, err := store.CreateToken(ctx, CreateTokenInput{Name: "throttle", BasemapCodes: []string{"google_image"}})
|
||||
if err != nil {
|
||||
t.Fatalf("create token: %v", err)
|
||||
}
|
||||
|
||||
// Mark the token as unused long ago so the next authorization refreshes it.
|
||||
stale := toRFC3339(nowUTC().Add(-10 * time.Minute))
|
||||
if _, err := store.db.ExecContext(ctx, `UPDATE service_tokens SET last_used_at = ? WHERE id = ?`, stale, created.Meta.ID); err != nil {
|
||||
t.Fatalf("seed stale last_used_at: %v", err)
|
||||
}
|
||||
if _, err := store.AuthorizeToken(ctx, created.Token); err != nil {
|
||||
t.Fatalf("authorize token: %v", err)
|
||||
}
|
||||
first := readLastUsedAt(t, store, created.Meta.ID)
|
||||
if first == stale {
|
||||
t.Fatalf("expected last_used_at to be refreshed, still %q", first)
|
||||
}
|
||||
|
||||
if _, err := store.AuthorizeToken(ctx, created.Token); err != nil {
|
||||
t.Fatalf("authorize token again: %v", err)
|
||||
}
|
||||
second := readLastUsedAt(t, store, created.Meta.ID)
|
||||
if second != first {
|
||||
t.Fatalf("expected last_used_at to be throttled, got %q then %q", first, second)
|
||||
}
|
||||
}
|
||||
|
||||
func readLastUsedAt(t *testing.T, store *Store, tokenID string) string {
|
||||
t.Helper()
|
||||
var value string
|
||||
if err := store.db.QueryRow(`SELECT last_used_at FROM service_tokens WHERE id = ?`, tokenID).Scan(&value); err != nil {
|
||||
t.Fatalf("read last_used_at: %v", err)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func TestGetBasemapByCodeLoadsVersions(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
tileRoot := filepath.Join(root, "tiles")
|
||||
writeTestFile(t, filepath.Join(tileRoot, "google_image", "0", "0", "0.webp"), "tile")
|
||||
writeTestFile(t, filepath.Join(tileRoot, "google_image", "1", "0", "0.webp"), "tile")
|
||||
|
||||
store, err := OpenStore(StoreConfig{
|
||||
SQLitePath: filepath.Join(root, "basemap.db"),
|
||||
ScanRoot: tileRoot,
|
||||
APIBaseURL: "http://127.0.0.1:8910",
|
||||
TileBaseURL: "http://127.0.0.1:8910",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
if _, err := store.RunScan(ctx, defaultScanSourceCode); err != nil {
|
||||
t.Fatalf("run scan: %v", err)
|
||||
}
|
||||
|
||||
item, err := store.GetBasemapByCode(ctx, "google_image")
|
||||
if err != nil {
|
||||
t.Fatalf("get basemap: %v", err)
|
||||
}
|
||||
if item.Code != "google-image" {
|
||||
t.Fatalf("unexpected code: %q", item.Code)
|
||||
}
|
||||
if len(item.Versions) != 1 {
|
||||
t.Fatalf("expected 1 version, got %d", len(item.Versions))
|
||||
}
|
||||
if item.Default == nil || item.Default.Version != "current" {
|
||||
t.Fatalf("expected default version current, got %+v", item.Default)
|
||||
}
|
||||
if item.Default.MinZoom != 0 || item.Default.MaxZoom != 1 {
|
||||
t.Fatalf("unexpected zoom range: %d-%d", item.Default.MinZoom, item.Default.MaxZoom)
|
||||
}
|
||||
if _, err := store.GetBasemapByCode(ctx, "missing"); err == nil {
|
||||
t.Fatal("expected missing basemap to error")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,496 @@
|
||||
package basemap
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"tile-server/api-go/internal/uid"
|
||||
)
|
||||
|
||||
const pushCallbackTimeout = 30 * time.Second
|
||||
|
||||
// Delivery outcomes. retrying is a transient state: the record stays queued
|
||||
// until next_attempt_at elapses.
|
||||
const (
|
||||
pushStatusSucceeded = "succeeded"
|
||||
pushStatusFailed = "failed"
|
||||
pushStatusRetrying = "retrying"
|
||||
|
||||
// pushMaxAttempts caps how many times one record is delivered.
|
||||
pushMaxAttempts = 3
|
||||
// pushRetryBase is the first backoff step; it doubles per attempt.
|
||||
pushRetryBase = 30 * time.Second
|
||||
)
|
||||
|
||||
func (s *Store) CreateTargetSystem(ctx context.Context, input CreateTargetSystemInput) (TargetSystem, error) {
|
||||
code := normalizeBasemapCode(input.Code)
|
||||
if code == "" {
|
||||
return TargetSystem{}, errors.New("target system code is required")
|
||||
}
|
||||
name := strings.TrimSpace(input.Name)
|
||||
if name == "" {
|
||||
name = code
|
||||
}
|
||||
callbackURL := strings.TrimSpace(input.CallbackURL)
|
||||
if callbackURL == "" {
|
||||
return TargetSystem{}, errors.New("callback url is required")
|
||||
}
|
||||
method := strings.ToUpper(strings.TrimSpace(input.CallbackMethod))
|
||||
if method == "" {
|
||||
method = http.MethodPost
|
||||
}
|
||||
if input.CallbackHeaders == nil {
|
||||
input.CallbackHeaders = map[string]string{}
|
||||
}
|
||||
|
||||
now := nowUTC()
|
||||
id := uid.Deterministic("target-system", code)
|
||||
_, err := s.db.ExecContext(ctx, `
|
||||
INSERT INTO target_systems (
|
||||
id, code, name, callback_url, callback_method, callback_headers_json, enabled, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?)
|
||||
ON CONFLICT(code) DO UPDATE SET
|
||||
name = excluded.name,
|
||||
callback_url = excluded.callback_url,
|
||||
callback_method = excluded.callback_method,
|
||||
callback_headers_json = excluded.callback_headers_json,
|
||||
enabled = 1,
|
||||
updated_at = excluded.updated_at
|
||||
`, id, code, name, callbackURL, method, writeJSON(input.CallbackHeaders), toRFC3339(now), toRFC3339(now))
|
||||
if err != nil {
|
||||
return TargetSystem{}, err
|
||||
}
|
||||
return s.GetTargetSystemByCode(ctx, code)
|
||||
}
|
||||
|
||||
func (s *Store) GetTargetSystemByCode(ctx context.Context, code string) (TargetSystem, error) {
|
||||
row := s.db.QueryRowContext(ctx, `
|
||||
SELECT id, code, name, callback_url, callback_method, callback_headers_json, enabled, created_at, updated_at
|
||||
FROM target_systems
|
||||
WHERE code = ?
|
||||
`, normalizeBasemapCode(code))
|
||||
item, err := scanTargetSystem(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return TargetSystem{}, fmt.Errorf("target system %q not found", code)
|
||||
}
|
||||
return TargetSystem{}, err
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (s *Store) ListTargetSystems(ctx context.Context) ([]TargetSystem, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT id, code, name, callback_url, callback_method, callback_headers_json, enabled, created_at, updated_at
|
||||
FROM target_systems
|
||||
ORDER BY code
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var items []TargetSystem
|
||||
for rows.Next() {
|
||||
item, err := scanTargetSystem(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
// PushBasemapVersion queues a downstream notification. The HTTP callback runs
|
||||
// off the request path so a slow target cannot block the caller; the returned
|
||||
// record starts as "pending" and is updated once the callback finishes.
|
||||
func (s *Store) PushBasemapVersion(ctx context.Context, targetCode, basemapCode, version string) (PushRecord, error) {
|
||||
target, err := s.GetTargetSystemByCode(ctx, targetCode)
|
||||
if err != nil {
|
||||
return PushRecord{}, err
|
||||
}
|
||||
if !target.Enabled {
|
||||
return PushRecord{}, fmt.Errorf("target system %q is disabled", target.Code)
|
||||
}
|
||||
basemapCode = normalizeBasemapCode(basemapCode)
|
||||
version = strings.TrimSpace(version)
|
||||
|
||||
row := s.db.QueryRowContext(ctx, `
|
||||
SELECT
|
||||
v.id,
|
||||
v.basemap_id,
|
||||
b.code,
|
||||
v.version,
|
||||
v.status,
|
||||
v.is_default,
|
||||
v.manifest_path,
|
||||
v.tile_root_path,
|
||||
v.url_template,
|
||||
v.tile_format,
|
||||
v.tile_scheme,
|
||||
v.min_zoom,
|
||||
v.max_zoom,
|
||||
v.bbox_json,
|
||||
v.attribution,
|
||||
v.metadata_json,
|
||||
v.created_at,
|
||||
v.updated_at
|
||||
FROM basemap_versions v
|
||||
JOIN basemaps b ON b.id = v.basemap_id
|
||||
WHERE b.code = ? AND v.version = ?
|
||||
`, basemapCode, version)
|
||||
versionRow, err := scanBasemapVersionRow(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return PushRecord{}, fmt.Errorf("basemap version %s/%s not found", basemapCode, version)
|
||||
}
|
||||
return PushRecord{}, err
|
||||
}
|
||||
versionItem := decodeBasemapVersion(versionRow)
|
||||
basemapItem, err := s.GetBasemapByCode(ctx, basemapCode)
|
||||
if err != nil {
|
||||
return PushRecord{}, err
|
||||
}
|
||||
|
||||
payload := map[string]any{
|
||||
"event": "basemap.published",
|
||||
"service": strings.TrimSpace(s.cfg.APIBaseURL),
|
||||
"basemap": basemapItem,
|
||||
"version": versionItem,
|
||||
"pushed_at": toRFC3339(nowUTC()),
|
||||
"targetCode": target.Code,
|
||||
}
|
||||
record := PushRecord{
|
||||
ID: uid.New(),
|
||||
TargetSystemID: target.ID,
|
||||
TargetSystemCode: target.Code,
|
||||
BasemapVersionID: versionItem.ID,
|
||||
BasemapCode: basemapItem.Code,
|
||||
BasemapVersion: versionItem.Version,
|
||||
Status: "pending",
|
||||
Request: payload,
|
||||
PushedAt: nowUTC(),
|
||||
}
|
||||
if _, err := s.db.ExecContext(ctx, `
|
||||
INSERT INTO push_records (
|
||||
id, target_system_id, basemap_version_id, status, request_json, response_status, response_body, error_message, pushed_at, finished_at
|
||||
) VALUES (?, ?, ?, ?, ?, NULL, '', '', ?, NULL)
|
||||
`, record.ID, record.TargetSystemID, record.BasemapVersionID, record.Status, writeJSON(record.Request), toRFC3339(record.PushedAt)); err != nil {
|
||||
return PushRecord{}, err
|
||||
}
|
||||
|
||||
s.enqueuePush("push_records", record.ID, target, payload)
|
||||
return record, nil
|
||||
}
|
||||
|
||||
func (s *Store) ListVectorPushRecords(ctx context.Context, limit int) ([]VectorPushRecord, error) {
|
||||
if limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT
|
||||
p.id,
|
||||
p.target_system_id,
|
||||
t.code,
|
||||
p.vector_asset_id,
|
||||
v.code,
|
||||
p.status,
|
||||
p.request_json,
|
||||
p.response_status,
|
||||
p.response_body,
|
||||
p.error_message,
|
||||
p.attempt_count,
|
||||
p.next_attempt_at,
|
||||
p.pushed_at,
|
||||
p.finished_at
|
||||
FROM vector_push_records p
|
||||
JOIN target_systems t ON t.id = p.target_system_id
|
||||
JOIN vector_assets v ON v.id = p.vector_asset_id
|
||||
ORDER BY p.pushed_at DESC
|
||||
LIMIT ?
|
||||
`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var items []VectorPushRecord
|
||||
for rows.Next() {
|
||||
item, err := scanVectorPushRecord(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
// PushVectorAsset queues a downstream notification, mirroring
|
||||
// PushBasemapVersion.
|
||||
func (s *Store) PushVectorAsset(ctx context.Context, targetCode, vectorCode string) (VectorPushRecord, error) {
|
||||
target, err := s.GetTargetSystemByCode(ctx, targetCode)
|
||||
if err != nil {
|
||||
return VectorPushRecord{}, err
|
||||
}
|
||||
if !target.Enabled {
|
||||
return VectorPushRecord{}, fmt.Errorf("target system %q is disabled", target.Code)
|
||||
}
|
||||
vectorCode = normalizeBasemapCode(vectorCode)
|
||||
asset, err := s.GetVectorAssetByCode(ctx, vectorCode)
|
||||
if err != nil {
|
||||
return VectorPushRecord{}, err
|
||||
}
|
||||
|
||||
payload := map[string]any{
|
||||
"event": "vector.published",
|
||||
"service": strings.TrimSpace(s.cfg.APIBaseURL),
|
||||
"vector": asset,
|
||||
"pushed_at": toRFC3339(nowUTC()),
|
||||
"targetCode": target.Code,
|
||||
}
|
||||
record := VectorPushRecord{
|
||||
ID: uid.New(),
|
||||
TargetSystemID: target.ID,
|
||||
TargetSystemCode: target.Code,
|
||||
VectorAssetID: asset.ID,
|
||||
VectorCode: asset.Code,
|
||||
Status: "pending",
|
||||
Request: payload,
|
||||
PushedAt: nowUTC(),
|
||||
}
|
||||
if _, err := s.db.ExecContext(ctx, `
|
||||
INSERT INTO vector_push_records (
|
||||
id, target_system_id, vector_asset_id, status, request_json, response_status, response_body, error_message, pushed_at, finished_at
|
||||
) VALUES (?, ?, ?, ?, ?, NULL, '', '', ?, NULL)
|
||||
`, record.ID, record.TargetSystemID, record.VectorAssetID, record.Status, writeJSON(record.Request), toRFC3339(record.PushedAt)); err != nil {
|
||||
return VectorPushRecord{}, err
|
||||
}
|
||||
|
||||
s.enqueuePush("vector_push_records", record.ID, target, payload)
|
||||
return record, nil
|
||||
}
|
||||
|
||||
// enqueuePush performs the first delivery attempt off the request path. A
|
||||
// failure inside the retry budget is rescheduled by dispatchPushRetries.
|
||||
func (s *Store) enqueuePush(table, pushID string, target TargetSystem, payload map[string]any) {
|
||||
s.pushWG.Add(1)
|
||||
go func() {
|
||||
defer s.pushWG.Done()
|
||||
s.attemptPush(table, pushID, 1, target, payload)
|
||||
}()
|
||||
}
|
||||
|
||||
// attemptPush performs one delivery attempt and records the outcome. attempt is
|
||||
// the 1-based attempt number; table is a trusted internal literal.
|
||||
func (s *Store) attemptPush(table, pushID string, attempt int, target TargetSystem, payload map[string]any) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), pushCallbackTimeout)
|
||||
defer cancel()
|
||||
|
||||
status, body, err := s.deliverCallback(ctx, target, payload)
|
||||
errorMessage := ""
|
||||
if err != nil {
|
||||
errorMessage = err.Error()
|
||||
}
|
||||
var responseStatus any
|
||||
if status != 0 {
|
||||
responseStatus = status
|
||||
}
|
||||
|
||||
if err != nil && attempt < pushMaxAttempts {
|
||||
// Stay queued: the dispatcher picks it up once the backoff elapses.
|
||||
next := nowUTC().Add(pushRetryDelay(attempt))
|
||||
if _, updateErr := s.db.ExecContext(context.Background(), fmt.Sprintf(`
|
||||
UPDATE %s
|
||||
SET status = ?, attempt_count = ?, next_attempt_at = ?, response_status = ?, response_body = ?,
|
||||
error_message = ?, finished_at = NULL
|
||||
WHERE id = ?
|
||||
`, table), pushStatusRetrying, attempt, toRFC3339(next), responseStatus, body, errorMessage, pushID); updateErr != nil {
|
||||
log.Printf("reschedule push %s/%s: %v", table, pushID, updateErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
recordStatus := pushStatusSucceeded
|
||||
if err != nil {
|
||||
recordStatus = pushStatusFailed
|
||||
}
|
||||
if _, updateErr := s.db.ExecContext(context.Background(), fmt.Sprintf(`
|
||||
UPDATE %s
|
||||
SET status = ?, attempt_count = ?, next_attempt_at = NULL, response_status = ?, response_body = ?,
|
||||
error_message = ?, finished_at = ?
|
||||
WHERE id = ?
|
||||
`, table), recordStatus, attempt, responseStatus, body, errorMessage, toRFC3339(nowUTC()), pushID); updateErr != nil {
|
||||
log.Printf("record push %s/%s: %v", table, pushID, updateErr)
|
||||
}
|
||||
}
|
||||
|
||||
// pushRetryDelay backs off exponentially: 30s, 60s, ...
|
||||
func pushRetryDelay(attempt int) time.Duration {
|
||||
if attempt < 1 {
|
||||
attempt = 1
|
||||
}
|
||||
return pushRetryBase * time.Duration(1<<(attempt-1))
|
||||
}
|
||||
|
||||
// dispatchPushRetries finds attempts whose backoff has elapsed and runs them.
|
||||
func (s *Store) dispatchPushRetries(ctx context.Context) error {
|
||||
for _, table := range []string{"push_records", "vector_push_records"} {
|
||||
due, err := s.dueRetries(ctx, table, 5)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, item := range due {
|
||||
target, err := s.GetTargetSystemByID(ctx, item.targetID)
|
||||
if err != nil {
|
||||
s.failRetry(table, item.id, fmt.Sprintf("target system is gone: %v", err))
|
||||
continue
|
||||
}
|
||||
if !target.Enabled {
|
||||
s.failRetry(table, item.id, "target system is disabled")
|
||||
continue
|
||||
}
|
||||
s.attemptPush(table, item.id, item.attempt+1, target, item.payload)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type duePush struct {
|
||||
id string
|
||||
attempt int
|
||||
targetID string
|
||||
payload map[string]any
|
||||
}
|
||||
|
||||
func (s *Store) dueRetries(ctx context.Context, table string, limit int) ([]duePush, error) {
|
||||
now := toRFC3339(nowUTC())
|
||||
rows, err := s.db.QueryContext(ctx, fmt.Sprintf(`
|
||||
SELECT id, attempt_count, target_system_id, request_json
|
||||
FROM %s
|
||||
WHERE status = ? AND (next_attempt_at IS NULL OR next_attempt_at <= ?)
|
||||
ORDER BY next_attempt_at
|
||||
LIMIT ?
|
||||
`, table), pushStatusRetrying, now, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var items []duePush
|
||||
for rows.Next() {
|
||||
var item duePush
|
||||
var requestJSON string
|
||||
if err := rows.Scan(&item.id, &item.attempt, &item.targetID, &requestJSON); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(requestJSON) != "" {
|
||||
if err := readMaybeJSON(requestJSON, &item.payload); err != nil {
|
||||
item.payload = nil
|
||||
}
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) failRetry(table, pushID, message string) {
|
||||
if _, err := s.db.ExecContext(context.Background(), fmt.Sprintf(`
|
||||
UPDATE %s SET status = ?, error_message = ?, next_attempt_at = NULL, finished_at = ? WHERE id = ?
|
||||
`, table), pushStatusFailed, message, toRFC3339(nowUTC()), pushID); err != nil {
|
||||
log.Printf("fail push %s/%s: %v", table, pushID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// GetTargetSystemByID looks a target up by primary key for retry dispatch.
|
||||
func (s *Store) GetTargetSystemByID(ctx context.Context, id string) (TargetSystem, error) {
|
||||
row := s.db.QueryRowContext(ctx, `
|
||||
SELECT id, code, name, callback_url, callback_method, callback_headers_json, enabled, created_at, updated_at
|
||||
FROM target_systems
|
||||
WHERE id = ?
|
||||
`, strings.TrimSpace(id))
|
||||
item, err := scanTargetSystem(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return TargetSystem{}, fmt.Errorf("target system %q not found", id)
|
||||
}
|
||||
return TargetSystem{}, err
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
// RetryPush re-queues a finished push record. kind is "basemap" or "vector".
|
||||
func (s *Store) RetryPush(ctx context.Context, kind, recordID string) error {
|
||||
table := "push_records"
|
||||
if kind == "vector" {
|
||||
table = "vector_push_records"
|
||||
} else if kind != "basemap" {
|
||||
return fmt.Errorf("unknown push kind %q", kind)
|
||||
}
|
||||
result, err := s.db.ExecContext(ctx, fmt.Sprintf(`
|
||||
UPDATE %s
|
||||
SET status = ?, attempt_count = 0, next_attempt_at = ?, finished_at = NULL, error_message = ''
|
||||
WHERE id = ?
|
||||
`, table), pushStatusRetrying, toRFC3339(nowUTC()), strings.TrimSpace(recordID))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
affected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected == 0 {
|
||||
return fmt.Errorf("push record %q not found", recordID)
|
||||
}
|
||||
// Kick the dispatcher so a manual retry does not wait for the next tick.
|
||||
s.pushWG.Add(1)
|
||||
go func() {
|
||||
defer s.pushWG.Done()
|
||||
_ = s.dispatchPushRetries(context.Background())
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// deliverCallback performs the downstream HTTP call for a target system.
|
||||
func (s *Store) deliverCallback(ctx context.Context, target TargetSystem, payload map[string]any) (int, string, error) {
|
||||
return s.postJSON(ctx, target.CallbackMethod, target.CallbackURL, target.CallbackHeaders, payload)
|
||||
}
|
||||
|
||||
// postJSON sends a JSON payload and returns the status code plus a truncated
|
||||
// response body. Shared by push callbacks and tile-job completion callbacks.
|
||||
func (s *Store) postJSON(ctx context.Context, method, url string, headers map[string]string, payload map[string]any) (int, string, error) {
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, method, url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
for key, value := range headers {
|
||||
request.Header.Set(key, value)
|
||||
}
|
||||
|
||||
response, err := s.httpClient.Do(request)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
|
||||
responseBody := limitedReadAll(response.Body, 256*1024)
|
||||
// Drain whatever is left so the connection can be reused.
|
||||
_, _ = io.Copy(io.Discard, response.Body)
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
return response.StatusCode, responseBody, fmt.Errorf("target returned status %d", response.StatusCode)
|
||||
}
|
||||
return response.StatusCode, responseBody, nil
|
||||
}
|
||||
@@ -0,0 +1,397 @@
|
||||
package basemap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func openTestStore(t *testing.T, cfg StoreConfig) *Store {
|
||||
t.Helper()
|
||||
store, err := OpenStore(cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := store.Close(); err != nil {
|
||||
t.Fatalf("close store: %v", err)
|
||||
}
|
||||
})
|
||||
return store
|
||||
}
|
||||
|
||||
func TestRunScanRefusesConcurrentRuns(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
tileRoot := filepath.Join(root, "tiles")
|
||||
writeTestFile(t, filepath.Join(tileRoot, "google_image", "0", "0", "0.webp"), "tile")
|
||||
vectorRoot := filepath.Join(root, "vectors")
|
||||
|
||||
store := openTestStore(t, StoreConfig{
|
||||
SQLitePath: filepath.Join(root, "basemap.db"),
|
||||
ScanRoot: tileRoot,
|
||||
VectorScanRoot: vectorRoot,
|
||||
APIBaseURL: "http://127.0.0.1:8910",
|
||||
TileBaseURL: "http://127.0.0.1:8910",
|
||||
VectorBaseURL: "http://127.0.0.1:8910",
|
||||
})
|
||||
|
||||
store.scanMu.Lock()
|
||||
if _, err := store.RunScan(ctx, defaultScanSourceCode); !errors.Is(err, ErrScanInProgress) {
|
||||
store.scanMu.Unlock()
|
||||
t.Fatalf("expected ErrScanInProgress, got %v", err)
|
||||
}
|
||||
if _, err := store.RunVectorScan(ctx); !errors.Is(err, ErrScanInProgress) {
|
||||
store.scanMu.Unlock()
|
||||
t.Fatalf("expected ErrScanInProgress from vector scan, got %v", err)
|
||||
}
|
||||
store.scanMu.Unlock()
|
||||
|
||||
if _, err := store.RunScan(ctx, defaultScanSourceCode); err != nil {
|
||||
t.Fatalf("run scan after unlock: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunScanReportsUnknownSource(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
store := openTestStore(t, StoreConfig{SQLitePath: filepath.Join(root, "basemap.db")})
|
||||
|
||||
if _, err := store.RunScan(ctx, "does-not-exist"); !errors.Is(err, ErrScanSourceNotFound) {
|
||||
t.Fatalf("expected ErrScanSourceNotFound, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRescanOnlyCountsRealChanges(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
tileRoot := filepath.Join(root, "tiles")
|
||||
writeTestFile(t, filepath.Join(tileRoot, "google_image", "0", "0", "0.webp"), "tile")
|
||||
|
||||
store := openTestStore(t, StoreConfig{
|
||||
SQLitePath: filepath.Join(root, "basemap.db"),
|
||||
ScanRoot: tileRoot,
|
||||
APIBaseURL: "http://127.0.0.1:8910",
|
||||
TileBaseURL: "http://127.0.0.1:8910",
|
||||
VectorBaseURL: "http://127.0.0.1:8910",
|
||||
})
|
||||
|
||||
first, err := store.RunScan(ctx, defaultScanSourceCode)
|
||||
if err != nil {
|
||||
t.Fatalf("first scan: %v", err)
|
||||
}
|
||||
if first.AddedCount != 1 {
|
||||
t.Fatalf("expected 1 added on first scan, got %d", first.AddedCount)
|
||||
}
|
||||
|
||||
second, err := store.RunScan(ctx, defaultScanSourceCode)
|
||||
if err != nil {
|
||||
t.Fatalf("second scan: %v", err)
|
||||
}
|
||||
if second.UpdatedCount != 0 {
|
||||
t.Fatalf("expected 0 updates on an unchanged rescan, got %d", second.UpdatedCount)
|
||||
}
|
||||
if second.AddedCount != 0 || second.RemovedCount != 0 {
|
||||
t.Fatalf("expected no additions/removals, got +%d/-%d", second.AddedCount, second.RemovedCount)
|
||||
}
|
||||
if got := second.Summary["unchanged_count"]; got != 1 {
|
||||
t.Fatalf("expected unchanged_count=1, got %v", got)
|
||||
}
|
||||
|
||||
// A new zoom level is a genuine change and must be reported as an update.
|
||||
writeTestFile(t, filepath.Join(tileRoot, "google_image", "1", "0", "0.webp"), "tile")
|
||||
third, err := store.RunScan(ctx, defaultScanSourceCode)
|
||||
if err != nil {
|
||||
t.Fatalf("third scan: %v", err)
|
||||
}
|
||||
if third.UpdatedCount != 1 {
|
||||
t.Fatalf("expected 1 update after adding a zoom level, got %d", third.UpdatedCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVectorCodeCollisionKeepsBothAssets(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
vectorRoot := filepath.Join(root, "vectors")
|
||||
writeTestFile(t, filepath.Join(vectorRoot, "a", "b.geojson"), `{"type":"FeatureCollection","features":[]}`)
|
||||
writeTestFile(t, filepath.Join(vectorRoot, "a-b.geojson"), `{"type":"FeatureCollection","features":[]}`)
|
||||
|
||||
store := openTestStore(t, StoreConfig{
|
||||
SQLitePath: filepath.Join(root, "basemap.db"),
|
||||
VectorScanRoot: vectorRoot,
|
||||
APIBaseURL: "http://127.0.0.1:8910",
|
||||
VectorBaseURL: "http://127.0.0.1:8910",
|
||||
})
|
||||
|
||||
if _, err := store.RunVectorScan(ctx); err != nil {
|
||||
t.Fatalf("first vector scan: %v", err)
|
||||
}
|
||||
assets, err := store.ListVectorAssets(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list assets: %v", err)
|
||||
}
|
||||
if len(assets) != 2 {
|
||||
t.Fatalf("expected both colliding assets to survive, got %d", len(assets))
|
||||
}
|
||||
if assets[0].Code == assets[1].Code {
|
||||
t.Fatalf("expected distinct codes, both are %q", assets[0].Code)
|
||||
}
|
||||
firstCodes := []string{assets[0].Code, assets[1].Code}
|
||||
|
||||
if _, err := store.RunVectorScan(ctx); err != nil {
|
||||
t.Fatalf("second vector scan: %v", err)
|
||||
}
|
||||
assets, err = store.ListVectorAssets(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list assets after rescan: %v", err)
|
||||
}
|
||||
if len(assets) != 2 {
|
||||
t.Fatalf("rescan dropped an asset, now %d", len(assets))
|
||||
}
|
||||
secondCodes := map[string]bool{assets[0].Code: true, assets[1].Code: true}
|
||||
for _, code := range firstCodes {
|
||||
if !secondCodes[code] {
|
||||
t.Fatalf("disambiguated code %q was not stable across rescans", code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestVectorScanReusesChecksumUntilFileChanges(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
vectorRoot := filepath.Join(root, "vectors")
|
||||
file := filepath.Join(vectorRoot, "boundary.geojson")
|
||||
writeTestFile(t, file, `{"type":"FeatureCollection","features":[]}`)
|
||||
|
||||
store := openTestStore(t, StoreConfig{
|
||||
SQLitePath: filepath.Join(root, "basemap.db"),
|
||||
VectorScanRoot: vectorRoot,
|
||||
APIBaseURL: "http://127.0.0.1:8910",
|
||||
VectorBaseURL: "http://127.0.0.1:8910",
|
||||
})
|
||||
|
||||
if _, err := store.RunVectorScan(ctx); err != nil {
|
||||
t.Fatalf("first scan: %v", err)
|
||||
}
|
||||
first, err := store.GetVectorAssetByCode(ctx, "boundary")
|
||||
if err != nil {
|
||||
t.Fatalf("get asset: %v", err)
|
||||
}
|
||||
if first.Metadata["hash_size"] == nil || first.Metadata["hash_mtime"] == nil {
|
||||
t.Fatalf("expected hash stamp in metadata, got %v", first.Metadata)
|
||||
}
|
||||
|
||||
second, err := store.RunVectorScan(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("second scan: %v", err)
|
||||
}
|
||||
if second["unchanged_count"] != 1 {
|
||||
t.Fatalf("expected unchanged_count=1, got %v", second["unchanged_count"])
|
||||
}
|
||||
reused, err := store.GetVectorAssetByCode(ctx, "boundary")
|
||||
if err != nil {
|
||||
t.Fatalf("get asset after rescan: %v", err)
|
||||
}
|
||||
if reused.Checksum != first.Checksum {
|
||||
t.Fatalf("checksum changed without a file change: %q -> %q", first.Checksum, reused.Checksum)
|
||||
}
|
||||
|
||||
// Rewriting the file with different content must invalidate the cached hash.
|
||||
writeTestFile(t, file, `{"type":"FeatureCollection","features":[{"type":"Feature"}]}`)
|
||||
if _, err := store.RunVectorScan(ctx); err != nil {
|
||||
t.Fatalf("third scan: %v", err)
|
||||
}
|
||||
changed, err := store.GetVectorAssetByCode(ctx, "boundary")
|
||||
if err != nil {
|
||||
t.Fatalf("get asset after change: %v", err)
|
||||
}
|
||||
if changed.Checksum == first.Checksum {
|
||||
t.Fatal("expected checksum to change after the file changed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateTokenAppliesDefaultTTL(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
tileRoot := filepath.Join(root, "tiles")
|
||||
writeTestFile(t, filepath.Join(tileRoot, "google_image", "0", "0", "0.webp"), "tile")
|
||||
|
||||
store := openTestStore(t, StoreConfig{
|
||||
SQLitePath: filepath.Join(root, "basemap.db"),
|
||||
ScanRoot: tileRoot,
|
||||
APIBaseURL: "http://127.0.0.1:8910",
|
||||
TileBaseURL: "http://127.0.0.1:8910",
|
||||
VectorBaseURL: "http://127.0.0.1:8910",
|
||||
TokenDefaultTTL: time.Hour,
|
||||
})
|
||||
if _, err := store.RunScan(ctx, defaultScanSourceCode); err != nil {
|
||||
t.Fatalf("run scan: %v", err)
|
||||
}
|
||||
|
||||
created, err := store.CreateToken(ctx, CreateTokenInput{Name: "ttl", BasemapCodes: []string{"google_image"}})
|
||||
if err != nil {
|
||||
t.Fatalf("create token: %v", err)
|
||||
}
|
||||
if created.Meta.ExpiresAt == nil {
|
||||
t.Fatal("expected a default expiry to be applied")
|
||||
}
|
||||
remaining := time.Until(*created.Meta.ExpiresAt)
|
||||
if remaining < 55*time.Minute || remaining > 65*time.Minute {
|
||||
t.Fatalf("expected ~1h expiry, got %s", remaining)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPushRunsAsynchronously(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
tileRoot := filepath.Join(root, "tiles")
|
||||
writeTestFile(t, filepath.Join(tileRoot, "google_image", "0", "0", "0.webp"), "tile")
|
||||
|
||||
received := make(chan map[string]any, 1)
|
||||
callback := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var payload map[string]any
|
||||
_ = json.NewDecoder(r.Body).Decode(&payload)
|
||||
select {
|
||||
case received <- payload:
|
||||
default:
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer callback.Close()
|
||||
|
||||
store := openTestStore(t, StoreConfig{
|
||||
SQLitePath: filepath.Join(root, "basemap.db"),
|
||||
ScanRoot: tileRoot,
|
||||
APIBaseURL: "http://127.0.0.1:8910",
|
||||
TileBaseURL: "http://127.0.0.1:8910",
|
||||
VectorBaseURL: "http://127.0.0.1:8910",
|
||||
})
|
||||
if _, err := store.RunScan(ctx, defaultScanSourceCode); err != nil {
|
||||
t.Fatalf("run scan: %v", err)
|
||||
}
|
||||
if _, err := store.CreateTargetSystem(ctx, CreateTargetSystemInput{
|
||||
Code: "downstream",
|
||||
CallbackURL: callback.URL,
|
||||
}); err != nil {
|
||||
t.Fatalf("create target: %v", err)
|
||||
}
|
||||
|
||||
record, err := store.PushBasemapVersion(ctx, "downstream", "google_image", "current")
|
||||
if err != nil {
|
||||
t.Fatalf("push: %v", err)
|
||||
}
|
||||
if record.Status != "pending" {
|
||||
t.Fatalf("expected the push to be queued as pending, got %q", record.Status)
|
||||
}
|
||||
|
||||
payload := waitForPayload(t, received)
|
||||
if payload["event"] != "basemap.published" {
|
||||
t.Fatalf("unexpected callback payload: %v", payload)
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
records, err := store.ListPushRecords(ctx, 5)
|
||||
if err != nil {
|
||||
t.Fatalf("list push records: %v", err)
|
||||
}
|
||||
done := false
|
||||
for _, item := range records {
|
||||
if item.ID == record.ID && item.Status == "succeeded" {
|
||||
done = true
|
||||
}
|
||||
}
|
||||
if done {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("push record %s never reached succeeded: %+v", record.ID, records)
|
||||
}
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func waitForPayload(t *testing.T, received <-chan map[string]any) map[string]any {
|
||||
t.Helper()
|
||||
select {
|
||||
case payload := <-received:
|
||||
return payload
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("callback was never invoked")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloseWaitsForQueuedPush(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
root := t.TempDir()
|
||||
tileRoot := filepath.Join(root, "tiles")
|
||||
writeTestFile(t, filepath.Join(tileRoot, "google_image", "0", "0", "0.webp"), "tile")
|
||||
|
||||
started := make(chan struct{}, 1)
|
||||
release := make(chan struct{})
|
||||
callback := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
select {
|
||||
case started <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
<-release
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer callback.Close()
|
||||
|
||||
store, err := OpenStore(StoreConfig{
|
||||
SQLitePath: filepath.Join(root, "basemap.db"),
|
||||
ScanRoot: tileRoot,
|
||||
APIBaseURL: "http://127.0.0.1:8910",
|
||||
TileBaseURL: "http://127.0.0.1:8910",
|
||||
VectorBaseURL: "http://127.0.0.1:8910",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
if _, err := store.RunScan(ctx, defaultScanSourceCode); err != nil {
|
||||
t.Fatalf("run scan: %v", err)
|
||||
}
|
||||
if _, err := store.CreateTargetSystem(ctx, CreateTargetSystemInput{Code: "downstream", CallbackURL: callback.URL}); err != nil {
|
||||
t.Fatalf("create target: %v", err)
|
||||
}
|
||||
if _, err := store.PushBasemapVersion(ctx, "downstream", "google_image", "current"); err != nil {
|
||||
t.Fatalf("push: %v", err)
|
||||
}
|
||||
|
||||
<-started
|
||||
closed := make(chan struct{})
|
||||
go func() {
|
||||
_ = store.Close()
|
||||
close(closed)
|
||||
}()
|
||||
select {
|
||||
case <-closed:
|
||||
t.Fatal("Close returned while a push callback was still in flight")
|
||||
case <-time.After(150 * time.Millisecond):
|
||||
}
|
||||
close(release)
|
||||
select {
|
||||
case <-closed:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Close did not return after the push finished")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanRootDirectoryIsNotRequiredAtOpen(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
store := openTestStore(t, StoreConfig{SQLitePath: filepath.Join(root, "basemap.db")})
|
||||
if store.ScanRoot() != "" {
|
||||
t.Fatalf("expected empty scan root, got %q", store.ScanRoot())
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "basemap.db")); err != nil {
|
||||
t.Fatalf("expected the database file to exist: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,713 @@
|
||||
package basemap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"tile-server/api-go/internal/uid"
|
||||
)
|
||||
|
||||
type scannedVersion struct {
|
||||
BasemapCode string
|
||||
Name string
|
||||
Type string
|
||||
Status string
|
||||
Description string
|
||||
Version string
|
||||
Manifest string
|
||||
TileRoot string
|
||||
TileFormat string
|
||||
TileScheme string
|
||||
MinZoom int
|
||||
MaxZoom int
|
||||
BBox []float64
|
||||
Attribution string
|
||||
Metadata map[string]any
|
||||
IsDefault bool
|
||||
}
|
||||
|
||||
// scanCounts reports what a scan actually changed, so operators can tell a real
|
||||
// update from a no-op rescan.
|
||||
type scanCounts struct {
|
||||
Scanned int
|
||||
Added int
|
||||
Updated int
|
||||
Removed int
|
||||
Unchanged int
|
||||
}
|
||||
|
||||
func (s *Store) RunScan(ctx context.Context, sourceCode string) (ScanRun, error) {
|
||||
// A scan reads the current catalog and then rewrites it, so two overlapping
|
||||
// runs could delete each other's rows. Refuse instead of interleaving.
|
||||
if !s.scanMu.TryLock() {
|
||||
return ScanRun{}, ErrScanInProgress
|
||||
}
|
||||
defer s.scanMu.Unlock()
|
||||
|
||||
source, err := s.GetScanSourceByCode(ctx, sourceCode)
|
||||
if err != nil {
|
||||
return ScanRun{}, err
|
||||
}
|
||||
if !source.Enabled {
|
||||
return ScanRun{}, &scanSourceError{sentinel: ErrScanSourceDisabled, code: sourceCode, reason: "is disabled"}
|
||||
}
|
||||
|
||||
startedAt := nowUTC()
|
||||
runID := uid.New()
|
||||
if _, err := s.db.ExecContext(ctx, `
|
||||
INSERT INTO scan_runs (
|
||||
id, scan_source_id, status, scanned_count, added_count, updated_count, removed_count, summary_json, started_at, finished_at
|
||||
) VALUES (?, ?, 'running', 0, 0, 0, 0, '', ?, ?)
|
||||
`, runID, source.ID, toRFC3339(startedAt), toRFC3339(startedAt)); err != nil {
|
||||
return ScanRun{}, fmt.Errorf("create scan run: %w", err)
|
||||
}
|
||||
|
||||
run, finalErr := s.runScan(ctx, runID, source, startedAt)
|
||||
return run, finalErr
|
||||
}
|
||||
|
||||
func (s *Store) runScan(ctx context.Context, runID string, source ScanSource, startedAt time.Time) (ScanRun, error) {
|
||||
fail := func(err error) (ScanRun, error) {
|
||||
return s.finishScanRun(ctx, runID, source, startedAt, "failed", nil, scanCounts{}, err)
|
||||
}
|
||||
|
||||
rootInfo, err := os.Stat(source.RootPath)
|
||||
if err != nil {
|
||||
return fail(fmt.Errorf("stat scan root: %w", err))
|
||||
}
|
||||
if !rootInfo.IsDir() {
|
||||
return fail(fmt.Errorf("scan root %s is not a directory", source.RootPath))
|
||||
}
|
||||
|
||||
manifestItems, coveredRoots, err := s.findManifestVersions(source)
|
||||
if err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
fallbackItems, err := s.findDirectTileRoots(source, coveredRoots)
|
||||
if err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
|
||||
allItems := append(manifestItems, fallbackItems...)
|
||||
sort.Slice(allItems, func(i, j int) bool {
|
||||
if allItems[i].BasemapCode == allItems[j].BasemapCode {
|
||||
return allItems[i].Version < allItems[j].Version
|
||||
}
|
||||
return allItems[i].BasemapCode < allItems[j].BasemapCode
|
||||
})
|
||||
|
||||
counts, err := s.persistScanResult(ctx, source, allItems)
|
||||
if err != nil {
|
||||
return s.finishScanRun(ctx, runID, source, startedAt, "failed", allItems, scanCounts{Scanned: len(allItems)}, err)
|
||||
}
|
||||
counts.Scanned = len(allItems)
|
||||
return s.finishScanRun(ctx, runID, source, startedAt, "succeeded", allItems, counts, nil)
|
||||
}
|
||||
|
||||
func (s *Store) finishScanRun(
|
||||
ctx context.Context,
|
||||
runID string,
|
||||
source ScanSource,
|
||||
startedAt time.Time,
|
||||
status string,
|
||||
items []scannedVersion,
|
||||
counts scanCounts,
|
||||
runErr error,
|
||||
) (ScanRun, error) {
|
||||
finishedAt := nowUTC()
|
||||
summary := map[string]any{
|
||||
"source_code": source.Code,
|
||||
"scan_root": source.RootPath,
|
||||
"scanned_count": counts.Scanned,
|
||||
"added_count": counts.Added,
|
||||
"updated_count": counts.Updated,
|
||||
"removed_count": counts.Removed,
|
||||
"unchanged_count": counts.Unchanged,
|
||||
}
|
||||
if runErr != nil {
|
||||
summary["error"] = runErr.Error()
|
||||
}
|
||||
if len(items) > 0 {
|
||||
codes := map[string]struct{}{}
|
||||
for _, item := range items {
|
||||
codes[item.BasemapCode] = struct{}{}
|
||||
}
|
||||
summary["basemap_codes"] = sortedKeys(codes)
|
||||
}
|
||||
|
||||
_, updateErr := s.db.ExecContext(ctx, `
|
||||
UPDATE scan_runs
|
||||
SET status = ?, scanned_count = ?, added_count = ?, updated_count = ?, removed_count = ?, summary_json = ?, finished_at = ?
|
||||
WHERE id = ?
|
||||
`, status, counts.Scanned, counts.Added, counts.Updated, counts.Removed, writeJSON(summary), toRFC3339(finishedAt), runID)
|
||||
if updateErr != nil {
|
||||
if runErr != nil {
|
||||
return ScanRun{}, fmt.Errorf("%v; update scan run: %w", runErr, updateErr)
|
||||
}
|
||||
return ScanRun{}, updateErr
|
||||
}
|
||||
|
||||
run := ScanRun{
|
||||
ID: runID,
|
||||
ScanSourceID: source.ID,
|
||||
SourceCode: source.Code,
|
||||
Status: status,
|
||||
ScannedCount: counts.Scanned,
|
||||
AddedCount: counts.Added,
|
||||
UpdatedCount: counts.Updated,
|
||||
RemovedCount: counts.Removed,
|
||||
Summary: summary,
|
||||
StartedAt: startedAt,
|
||||
FinishedAt: finishedAt,
|
||||
}
|
||||
if runErr != nil {
|
||||
return run, runErr
|
||||
}
|
||||
return run, nil
|
||||
}
|
||||
|
||||
// manifestScanMaxDepth bounds how deep the manifest lookup descends. Manifest
|
||||
// based basemaps place manifest.json at the basemap root or one version folder
|
||||
// below it, so anything deeper is tile payload that must not be walked.
|
||||
const manifestScanMaxDepth = 4
|
||||
|
||||
func (s *Store) findManifestVersions(source ScanSource) ([]scannedVersion, map[string]struct{}, error) {
|
||||
manifestName := strings.TrimSpace(source.ManifestName)
|
||||
if manifestName == "" {
|
||||
manifestName = defaultManifestName
|
||||
}
|
||||
|
||||
var items []scannedVersion
|
||||
coveredRoots := map[string]struct{}{}
|
||||
|
||||
var walk func(dir string, depth int) error
|
||||
walk = func(dir string, depth int) error {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// A directory holding a manifest is a basemap root: record it and stop,
|
||||
// so the tile payload underneath (the default "tiles" subtree) is never
|
||||
// traversed.
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() || !strings.EqualFold(entry.Name(), manifestName) {
|
||||
continue
|
||||
}
|
||||
manifestPath := filepath.Join(dir, entry.Name())
|
||||
manifest, err := LoadManifest(manifestPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tileRoot, err := resolveTileRoot(dir, manifest.RootPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
format, minZoom, maxZoom, found, err := inspectTileRoot(tileRoot, manifest.TileFormat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if found {
|
||||
manifest.TileFormat = format
|
||||
manifest.MinZoom = minZoom
|
||||
manifest.MaxZoom = maxZoom
|
||||
}
|
||||
|
||||
if relPath, err := filepath.Rel(source.RootPath, tileRoot); err == nil {
|
||||
first := strings.Split(filepath.ToSlash(relPath), "/")[0]
|
||||
if first != "" && first != "." {
|
||||
coveredRoots[first] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
items = append(items, scannedVersion{
|
||||
BasemapCode: manifest.Code,
|
||||
Name: defaultText(manifest.Name, manifest.Code),
|
||||
Type: defaultText(manifest.Type, "xyz"),
|
||||
Status: defaultText(manifest.Status, "ready"),
|
||||
Description: manifest.Description,
|
||||
Version: defaultText(manifest.Version, "current"),
|
||||
Manifest: manifestPath,
|
||||
TileRoot: tileRoot,
|
||||
TileFormat: normalizeTileFormat(manifest.TileFormat),
|
||||
TileScheme: defaultText(manifest.TileScheme, "xyz"),
|
||||
MinZoom: manifest.MinZoom,
|
||||
MaxZoom: manifest.MaxZoom,
|
||||
BBox: manifest.BBox,
|
||||
Attribution: manifest.Attribution,
|
||||
Metadata: manifest.Metadata,
|
||||
IsDefault: manifest.IsDefault,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
if depth >= manifestScanMaxDepth {
|
||||
return nil
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() || isNumericDirName(entry.Name()) {
|
||||
continue
|
||||
}
|
||||
if err := walk(filepath.Join(dir, entry.Name()), depth+1); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := walk(source.RootPath, 0); err != nil {
|
||||
return nil, nil, fmt.Errorf("walk manifests: %w", err)
|
||||
}
|
||||
return items, coveredRoots, nil
|
||||
}
|
||||
|
||||
// isNumericDirName reports whether the directory name is a bare zoom level such
|
||||
// as "0" or "14". Raw tile roots nest z/x/y under the basemap directory, so
|
||||
// pruning these keeps the scan clear of the tile payload.
|
||||
func isNumericDirName(name string) bool {
|
||||
if name == "" {
|
||||
return false
|
||||
}
|
||||
for _, r := range name {
|
||||
if r < '0' || r > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Store) findDirectTileRoots(source ScanSource, skip map[string]struct{}) ([]scannedVersion, error) {
|
||||
entries, err := os.ReadDir(source.RootPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read scan root: %w", err)
|
||||
}
|
||||
|
||||
var items []scannedVersion
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
name := entry.Name()
|
||||
if _, exists := skip[name]; exists {
|
||||
continue
|
||||
}
|
||||
|
||||
rootPath := filepath.Join(source.RootPath, name)
|
||||
format, minZoom, maxZoom, found, err := inspectTileRoot(rootPath, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
|
||||
code := normalizeBasemapCode(name)
|
||||
if code == "" {
|
||||
continue
|
||||
}
|
||||
items = append(items, scannedVersion{
|
||||
BasemapCode: code,
|
||||
Name: name,
|
||||
Type: "xyz",
|
||||
Status: "ready",
|
||||
Version: "current",
|
||||
TileRoot: rootPath,
|
||||
TileFormat: format,
|
||||
TileScheme: "xyz",
|
||||
MinZoom: minZoom,
|
||||
MaxZoom: maxZoom,
|
||||
Metadata: map[string]any{
|
||||
"discovered_from": "directory_scan",
|
||||
},
|
||||
IsDefault: true,
|
||||
})
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func inspectTileRoot(rootPath string, hintFormat string) (string, int, int, bool, error) {
|
||||
entries, err := os.ReadDir(rootPath)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return "", 0, 0, false, nil
|
||||
}
|
||||
return "", 0, 0, false, fmt.Errorf("read tile root %s: %w", rootPath, err)
|
||||
}
|
||||
|
||||
var zooms []int
|
||||
format := normalizeTileFormat(hintFormat)
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
zoom, err := strconv.Atoi(entry.Name())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
zooms = append(zooms, zoom)
|
||||
if format == "" || format == "png" {
|
||||
detected, found, err := detectTileFormat(filepath.Join(rootPath, entry.Name()))
|
||||
if err != nil {
|
||||
return "", 0, 0, false, err
|
||||
}
|
||||
if found {
|
||||
format = detected
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(zooms) == 0 {
|
||||
return "", 0, 0, false, nil
|
||||
}
|
||||
sort.Ints(zooms)
|
||||
if format == "" {
|
||||
format = "png"
|
||||
}
|
||||
return format, zooms[0], zooms[len(zooms)-1], true, nil
|
||||
}
|
||||
|
||||
func detectTileFormat(zoomPath string) (string, bool, error) {
|
||||
xEntries, err := os.ReadDir(zoomPath)
|
||||
if err != nil {
|
||||
return "", false, fmt.Errorf("read zoom path %s: %w", zoomPath, err)
|
||||
}
|
||||
for _, xEntry := range xEntries {
|
||||
if !xEntry.IsDir() {
|
||||
continue
|
||||
}
|
||||
tileEntries, err := os.ReadDir(filepath.Join(zoomPath, xEntry.Name()))
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
for _, tileEntry := range tileEntries {
|
||||
if tileEntry.IsDir() {
|
||||
continue
|
||||
}
|
||||
ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(tileEntry.Name())), ".")
|
||||
if ext == "" {
|
||||
continue
|
||||
}
|
||||
return normalizeTileFormat(ext), true, nil
|
||||
}
|
||||
}
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
func resolveTileRoot(baseDir, relativePath string) (string, error) {
|
||||
if strings.TrimSpace(relativePath) == "" {
|
||||
relativePath = "tiles"
|
||||
}
|
||||
if filepath.IsAbs(relativePath) {
|
||||
return filepath.Clean(relativePath), nil
|
||||
}
|
||||
resolved := filepath.Clean(filepath.Join(baseDir, relativePath))
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
// existingVersion is a snapshot of a stored version, used to detect whether a
|
||||
// rescan actually changed anything.
|
||||
type existingVersion struct {
|
||||
Name string
|
||||
Type string
|
||||
Status string
|
||||
Description string
|
||||
ManifestPath string
|
||||
TileRootPath string
|
||||
TileFormat string
|
||||
TileScheme string
|
||||
MinZoom int
|
||||
MaxZoom int
|
||||
IsDefault bool
|
||||
BBoxJSON string
|
||||
Attribution string
|
||||
MetadataJSON string
|
||||
}
|
||||
|
||||
func (s *Store) persistScanResult(ctx context.Context, source ScanSource, items []scannedVersion) (scanCounts, error) {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return scanCounts{}, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
existing := map[string]existingVersion{}
|
||||
rows, err := tx.QueryContext(ctx, `
|
||||
SELECT
|
||||
b.code,
|
||||
v.version,
|
||||
b.name,
|
||||
b.type,
|
||||
b.status,
|
||||
b.description,
|
||||
v.manifest_path,
|
||||
v.tile_root_path,
|
||||
v.tile_format,
|
||||
v.tile_scheme,
|
||||
v.min_zoom,
|
||||
v.max_zoom,
|
||||
v.is_default,
|
||||
v.bbox_json,
|
||||
v.attribution,
|
||||
v.metadata_json
|
||||
FROM basemap_versions v
|
||||
JOIN basemaps b ON b.id = v.basemap_id
|
||||
WHERE v.scan_source_id = ?
|
||||
`, source.ID)
|
||||
if err != nil {
|
||||
return scanCounts{}, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var code string
|
||||
var version string
|
||||
var item existingVersion
|
||||
var isDefault int
|
||||
if err := rows.Scan(
|
||||
&code,
|
||||
&version,
|
||||
&item.Name,
|
||||
&item.Type,
|
||||
&item.Status,
|
||||
&item.Description,
|
||||
&item.ManifestPath,
|
||||
&item.TileRootPath,
|
||||
&item.TileFormat,
|
||||
&item.TileScheme,
|
||||
&item.MinZoom,
|
||||
&item.MaxZoom,
|
||||
&isDefault,
|
||||
&item.BBoxJSON,
|
||||
&item.Attribution,
|
||||
&item.MetadataJSON,
|
||||
); err != nil {
|
||||
rows.Close()
|
||||
return scanCounts{}, err
|
||||
}
|
||||
item.IsDefault = isDefault == 1
|
||||
existing[code+"::"+version] = item
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return scanCounts{}, err
|
||||
}
|
||||
|
||||
now := nowUTC()
|
||||
found := map[string]struct{}{}
|
||||
touchedBasemaps := map[string]struct{}{}
|
||||
counts := scanCounts{}
|
||||
for _, item := range items {
|
||||
key := item.BasemapCode + "::" + item.Version
|
||||
found[key] = struct{}{}
|
||||
if prev, exists := existing[key]; exists {
|
||||
if scannedVersionMatches(item, prev) {
|
||||
counts.Unchanged++
|
||||
} else {
|
||||
counts.Updated++
|
||||
}
|
||||
} else {
|
||||
counts.Added++
|
||||
}
|
||||
if err := s.upsertScannedVersionTx(ctx, tx, source, item, now); err != nil {
|
||||
return scanCounts{}, err
|
||||
}
|
||||
touchedBasemaps[item.BasemapCode] = struct{}{}
|
||||
}
|
||||
|
||||
for key := range existing {
|
||||
if _, ok := found[key]; ok {
|
||||
continue
|
||||
}
|
||||
parts := strings.SplitN(key, "::", 2)
|
||||
if len(parts) != 2 {
|
||||
continue
|
||||
}
|
||||
if err := s.deleteVersionBySourceTx(ctx, tx, source.ID, parts[0], parts[1]); err != nil {
|
||||
return scanCounts{}, err
|
||||
}
|
||||
touchedBasemaps[parts[0]] = struct{}{}
|
||||
counts.Removed++
|
||||
}
|
||||
|
||||
for code := range touchedBasemaps {
|
||||
if err := s.normalizeBasemapDefaultsTx(ctx, tx, code, now); err != nil {
|
||||
return scanCounts{}, err
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return scanCounts{}, err
|
||||
}
|
||||
return counts, nil
|
||||
}
|
||||
|
||||
// scannedVersionMatches reports whether a freshly scanned version is identical
|
||||
// to what is already stored, so an unchanged rescan is not reported as updated.
|
||||
func scannedVersionMatches(item scannedVersion, prev existingVersion) bool {
|
||||
return item.Name == prev.Name &&
|
||||
item.Type == prev.Type &&
|
||||
item.Status == prev.Status &&
|
||||
item.Description == prev.Description &&
|
||||
item.Manifest == prev.ManifestPath &&
|
||||
item.TileRoot == prev.TileRootPath &&
|
||||
normalizeTileFormat(item.TileFormat) == prev.TileFormat &&
|
||||
item.TileScheme == prev.TileScheme &&
|
||||
item.MinZoom == prev.MinZoom &&
|
||||
item.MaxZoom == prev.MaxZoom &&
|
||||
item.IsDefault == prev.IsDefault &&
|
||||
writeJSON(item.BBox) == prev.BBoxJSON &&
|
||||
item.Attribution == prev.Attribution &&
|
||||
writeJSON(item.Metadata) == prev.MetadataJSON
|
||||
}
|
||||
|
||||
func (s *Store) upsertScannedVersionTx(ctx context.Context, tx *sql.Tx, source ScanSource, item scannedVersion, now time.Time) error {
|
||||
basemapID := uid.Deterministic("basemap", item.BasemapCode)
|
||||
versionID := uid.Deterministic("basemap-version", item.BasemapCode, item.Version)
|
||||
|
||||
_, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO basemaps (id, code, name, type, status, description, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(code) DO UPDATE SET
|
||||
name = excluded.name,
|
||||
type = excluded.type,
|
||||
status = excluded.status,
|
||||
description = excluded.description,
|
||||
updated_at = excluded.updated_at
|
||||
`, basemapID, item.BasemapCode, item.Name, item.Type, item.Status, item.Description, toRFC3339(now), toRFC3339(now))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
isDefault := 0
|
||||
if item.IsDefault {
|
||||
isDefault = 1
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `
|
||||
INSERT INTO basemap_versions (
|
||||
id, basemap_id, scan_source_id, version, status, is_default, manifest_path, tile_root_path, url_template,
|
||||
tile_format, tile_scheme, min_zoom, max_zoom, bbox_json, attribution, metadata_json, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(basemap_id, version) DO UPDATE SET
|
||||
scan_source_id = excluded.scan_source_id,
|
||||
status = excluded.status,
|
||||
is_default = CASE WHEN excluded.is_default = 1 THEN 1 ELSE basemap_versions.is_default END,
|
||||
manifest_path = excluded.manifest_path,
|
||||
tile_root_path = excluded.tile_root_path,
|
||||
url_template = excluded.url_template,
|
||||
tile_format = excluded.tile_format,
|
||||
tile_scheme = excluded.tile_scheme,
|
||||
min_zoom = excluded.min_zoom,
|
||||
max_zoom = excluded.max_zoom,
|
||||
bbox_json = excluded.bbox_json,
|
||||
attribution = excluded.attribution,
|
||||
metadata_json = excluded.metadata_json,
|
||||
updated_at = excluded.updated_at
|
||||
`, versionID, basemapID, source.ID, item.Version, item.Status, isDefault, item.Manifest, item.TileRoot, s.buildURLTemplate(item.BasemapCode, item.Version, item.TileFormat), item.TileFormat, item.TileScheme, item.MinZoom, item.MaxZoom, writeJSON(item.BBox), item.Attribution, writeJSON(item.Metadata), toRFC3339(now), toRFC3339(now))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if item.IsDefault {
|
||||
_, err = tx.ExecContext(ctx, `
|
||||
UPDATE basemap_versions
|
||||
SET is_default = CASE WHEN version = ? THEN 1 ELSE 0 END, updated_at = ?
|
||||
WHERE basemap_id = ?
|
||||
`, item.Version, toRFC3339(now), basemapID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) deleteVersionBySourceTx(ctx context.Context, tx *sql.Tx, sourceID, basemapCode, version string) error {
|
||||
result, err := tx.ExecContext(ctx, `
|
||||
DELETE FROM basemap_versions
|
||||
WHERE scan_source_id = ? AND version = ? AND basemap_id = (SELECT id FROM basemaps WHERE code = ?)
|
||||
`, sourceID, version, basemapCode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := result.RowsAffected(); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `
|
||||
DELETE FROM basemaps
|
||||
WHERE code = ? AND NOT EXISTS (
|
||||
SELECT 1 FROM basemap_versions WHERE basemap_id = basemaps.id
|
||||
)
|
||||
`, basemapCode)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) normalizeBasemapDefaultsTx(ctx context.Context, tx *sql.Tx, basemapCode string, now time.Time) error {
|
||||
var basemapID string
|
||||
err := tx.QueryRowContext(ctx, `SELECT id FROM basemaps WHERE code = ?`, basemapCode).Scan(&basemapID)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
rows, err := tx.QueryContext(ctx, `
|
||||
SELECT version, is_default
|
||||
FROM basemap_versions
|
||||
WHERE basemap_id = ?
|
||||
ORDER BY is_default DESC, version ASC
|
||||
`, basemapID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
type versionFlag struct {
|
||||
Version string
|
||||
IsDefault bool
|
||||
}
|
||||
var versions []versionFlag
|
||||
for rows.Next() {
|
||||
var item versionFlag
|
||||
var isDefault int
|
||||
if err := rows.Scan(&item.Version, &isDefault); err != nil {
|
||||
return err
|
||||
}
|
||||
item.IsDefault = isDefault == 1
|
||||
versions = append(versions, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(versions) == 0 {
|
||||
return nil
|
||||
}
|
||||
defaultVersion := versions[0].Version
|
||||
for _, item := range versions {
|
||||
if item.IsDefault {
|
||||
defaultVersion = item.Version
|
||||
break
|
||||
}
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
UPDATE basemap_versions
|
||||
SET is_default = CASE WHEN version = ? THEN 1 ELSE 0 END, updated_at = ?
|
||||
WHERE basemap_id = ?
|
||||
`, defaultVersion, toRFC3339(now), basemapID); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `UPDATE basemaps SET updated_at = ? WHERE id = ?`, toRFC3339(now), basemapID)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) buildURLTemplate(code, version, format string) string {
|
||||
base := trimURL(s.cfg.TileBaseURL)
|
||||
if base == "" {
|
||||
base = trimURL(s.cfg.APIBaseURL)
|
||||
}
|
||||
format = normalizeTileFormat(format)
|
||||
return fmt.Sprintf("%s/tiles/%s/%s/{z}/{x}/{y}.%s", base, code, version, format)
|
||||
}
|
||||
@@ -0,0 +1,556 @@
|
||||
package basemap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/base32"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultScanSourceCode = "local-basemaps"
|
||||
defaultManifestName = "manifest.json"
|
||||
readPermission = "read"
|
||||
defaultVectorSourceCode = "local-vectors"
|
||||
)
|
||||
|
||||
const (
|
||||
// sqliteMaxOpenConns lets concurrent tile reads overlap instead of being
|
||||
// serialized behind a single connection.
|
||||
sqliteMaxOpenConns = 8
|
||||
)
|
||||
|
||||
// sqliteDSN tunes each pooled connection. busy_timeout makes concurrent
|
||||
// writers wait for the lock instead of failing immediately.
|
||||
func sqliteDSN(path string) string {
|
||||
params := url.Values{}
|
||||
params.Add("_pragma", "busy_timeout(5000)")
|
||||
params.Add("_pragma", "foreign_keys(1)")
|
||||
params.Add("_pragma", "synchronous(NORMAL)")
|
||||
return "file:" + filepath.ToSlash(path) + "?" + params.Encode()
|
||||
}
|
||||
|
||||
var (
|
||||
// ErrScanInProgress is returned when a scan is already running.
|
||||
ErrScanInProgress = errors.New("scan already in progress")
|
||||
// ErrScanSourceNotFound is returned when a scan source code is unknown.
|
||||
ErrScanSourceNotFound = errors.New("scan source not found")
|
||||
// ErrScanSourceDisabled is returned when a scan source is turned off.
|
||||
ErrScanSourceDisabled = errors.New("scan source disabled")
|
||||
)
|
||||
|
||||
// scanSourceError keeps the operator-facing message stable while still matching
|
||||
// errors.Is against a sentinel.
|
||||
type scanSourceError struct {
|
||||
sentinel error
|
||||
code string
|
||||
reason string
|
||||
}
|
||||
|
||||
func (e *scanSourceError) Error() string {
|
||||
return fmt.Sprintf("scan source %q %s", e.code, e.reason)
|
||||
}
|
||||
|
||||
func (e *scanSourceError) Is(target error) bool {
|
||||
return target == e.sentinel
|
||||
}
|
||||
|
||||
type StoreConfig struct {
|
||||
SQLitePath string
|
||||
ScanRoot string
|
||||
VectorScanRoot string
|
||||
APIBaseURL string
|
||||
TileBaseURL string
|
||||
VectorBaseURL string
|
||||
AutoScanOnStart bool
|
||||
TokenDefaultTTL time.Duration
|
||||
TileJobCommand string
|
||||
TileJobVRTCommand string
|
||||
TileJobLogDir string
|
||||
HealthCheckInterval time.Duration
|
||||
}
|
||||
|
||||
type Store struct {
|
||||
db *sql.DB
|
||||
cfg StoreConfig
|
||||
httpClient *http.Client
|
||||
|
||||
// scanMu serializes scans so two runs cannot interleave their read-then-write
|
||||
// bookkeeping.
|
||||
scanMu sync.Mutex
|
||||
// pushWG tracks detached push callbacks so Close can drain them.
|
||||
pushWG sync.WaitGroup
|
||||
|
||||
bgOnce sync.Once
|
||||
bgCancel context.CancelFunc
|
||||
bgCtx context.Context
|
||||
bgWG sync.WaitGroup
|
||||
}
|
||||
|
||||
type CreatedToken struct {
|
||||
Meta ServiceToken `json:"meta"`
|
||||
Token string `json:"token"`
|
||||
}
|
||||
|
||||
type TileDescriptor struct {
|
||||
FilePath string
|
||||
ContentType string
|
||||
Basemap Basemap
|
||||
Version BasemapVersion
|
||||
Token ServiceToken
|
||||
RelativePath string
|
||||
}
|
||||
|
||||
func OpenStore(cfg StoreConfig) (*Store, error) {
|
||||
if strings.TrimSpace(cfg.SQLitePath) == "" {
|
||||
return nil, errors.New("sqlite path is required")
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(cfg.SQLitePath), 0o755); err != nil {
|
||||
return nil, fmt.Errorf("create sqlite dir: %w", err)
|
||||
}
|
||||
|
||||
db, err := sql.Open("sqlite", sqliteDSN(cfg.SQLitePath))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open sqlite: %w", err)
|
||||
}
|
||||
db.SetConnMaxLifetime(30 * time.Minute)
|
||||
db.SetConnMaxIdleTime(5 * time.Minute)
|
||||
db.SetMaxOpenConns(sqliteMaxOpenConns)
|
||||
db.SetMaxIdleConns(sqliteMaxOpenConns)
|
||||
|
||||
// WAL keeps tile reads from blocking on scans/pushes. It is persisted in the
|
||||
// database file, so enabling it once is enough; a filesystem that cannot
|
||||
// support WAL (some network shares) must not stop the service from starting.
|
||||
if _, err := db.Exec("PRAGMA journal_mode=WAL;"); err != nil {
|
||||
log.Printf("enable sqlite WAL skipped: %v", err)
|
||||
}
|
||||
|
||||
store := &Store{
|
||||
db: db,
|
||||
cfg: cfg,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 15 * time.Second,
|
||||
},
|
||||
}
|
||||
if err := store.bootstrap(context.Background()); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(cfg.ScanRoot) != "" {
|
||||
if _, err := store.EnsureScanSource(context.Background(), CreateScanSourceInput{
|
||||
Code: defaultScanSourceCode,
|
||||
Name: "Local Basemaps",
|
||||
RootPath: cfg.ScanRoot,
|
||||
ManifestName: defaultManifestName,
|
||||
}); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(cfg.VectorScanRoot) != "" {
|
||||
if err := os.MkdirAll(cfg.VectorScanRoot, 0o755); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, fmt.Errorf("create vector scan root: %w", err)
|
||||
}
|
||||
}
|
||||
return store, nil
|
||||
}
|
||||
|
||||
func (s *Store) Close() error {
|
||||
if s == nil || s.db == nil {
|
||||
return nil
|
||||
}
|
||||
// Stop the periodic loops, then let detached push callbacks finish before
|
||||
// the pool goes away.
|
||||
if s.bgCancel != nil {
|
||||
s.bgCancel()
|
||||
}
|
||||
s.bgWG.Wait()
|
||||
s.pushWG.Wait()
|
||||
return s.db.Close()
|
||||
}
|
||||
|
||||
func (s *Store) bootstrap(ctx context.Context) error {
|
||||
statements := []string{
|
||||
`PRAGMA foreign_keys = ON;`,
|
||||
`CREATE TABLE IF NOT EXISTS basemaps (
|
||||
id TEXT PRIMARY KEY,
|
||||
code TEXT NOT NULL UNIQUE,
|
||||
name TEXT NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
description TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);`,
|
||||
`CREATE TABLE IF NOT EXISTS basemap_versions (
|
||||
id TEXT PRIMARY KEY,
|
||||
basemap_id TEXT NOT NULL,
|
||||
scan_source_id TEXT,
|
||||
version TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
is_default INTEGER NOT NULL DEFAULT 0,
|
||||
manifest_path TEXT NOT NULL DEFAULT '',
|
||||
tile_root_path TEXT NOT NULL,
|
||||
url_template TEXT NOT NULL,
|
||||
tile_format TEXT NOT NULL,
|
||||
tile_scheme TEXT NOT NULL,
|
||||
min_zoom INTEGER NOT NULL DEFAULT 0,
|
||||
max_zoom INTEGER NOT NULL DEFAULT 0,
|
||||
bbox_json TEXT NOT NULL DEFAULT '',
|
||||
attribution TEXT NOT NULL DEFAULT '',
|
||||
metadata_json TEXT NOT NULL DEFAULT '',
|
||||
health TEXT NOT NULL DEFAULT '',
|
||||
health_checked_at TEXT NOT NULL DEFAULT '',
|
||||
health_detail TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
UNIQUE(basemap_id, version),
|
||||
FOREIGN KEY (basemap_id) REFERENCES basemaps(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (scan_source_id) REFERENCES scan_sources(id) ON DELETE SET NULL
|
||||
);`,
|
||||
`CREATE TABLE IF NOT EXISTS scan_sources (
|
||||
id TEXT PRIMARY KEY,
|
||||
code TEXT NOT NULL UNIQUE,
|
||||
name TEXT NOT NULL,
|
||||
root_path TEXT NOT NULL,
|
||||
manifest_name TEXT NOT NULL,
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
metadata_json TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);`,
|
||||
`CREATE TABLE IF NOT EXISTS scan_runs (
|
||||
id TEXT PRIMARY KEY,
|
||||
scan_source_id TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
scanned_count INTEGER NOT NULL DEFAULT 0,
|
||||
added_count INTEGER NOT NULL DEFAULT 0,
|
||||
updated_count INTEGER NOT NULL DEFAULT 0,
|
||||
removed_count INTEGER NOT NULL DEFAULT 0,
|
||||
summary_json TEXT NOT NULL DEFAULT '',
|
||||
started_at TEXT NOT NULL,
|
||||
finished_at TEXT NOT NULL,
|
||||
FOREIGN KEY (scan_source_id) REFERENCES scan_sources(id) ON DELETE CASCADE
|
||||
);`,
|
||||
`CREATE TABLE IF NOT EXISTS target_systems (
|
||||
id TEXT PRIMARY KEY,
|
||||
code TEXT NOT NULL UNIQUE,
|
||||
name TEXT NOT NULL,
|
||||
callback_url TEXT NOT NULL,
|
||||
callback_method TEXT NOT NULL,
|
||||
callback_headers_json TEXT NOT NULL DEFAULT '',
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);`,
|
||||
`CREATE TABLE IF NOT EXISTS service_tokens (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
token_hash TEXT NOT NULL UNIQUE,
|
||||
token_prefix TEXT NOT NULL,
|
||||
scopes TEXT NOT NULL DEFAULT 'read',
|
||||
status TEXT NOT NULL,
|
||||
expires_at TEXT,
|
||||
last_used_at TEXT,
|
||||
created_at TEXT NOT NULL
|
||||
);`,
|
||||
`CREATE TABLE IF NOT EXISTS token_grants (
|
||||
id TEXT PRIMARY KEY,
|
||||
token_id TEXT NOT NULL,
|
||||
basemap_id TEXT NOT NULL,
|
||||
basemap_version_id TEXT,
|
||||
permission TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
FOREIGN KEY (token_id) REFERENCES service_tokens(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (basemap_id) REFERENCES basemaps(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (basemap_version_id) REFERENCES basemap_versions(id) ON DELETE CASCADE
|
||||
);`,
|
||||
`CREATE TABLE IF NOT EXISTS push_records (
|
||||
id TEXT PRIMARY KEY,
|
||||
target_system_id TEXT NOT NULL,
|
||||
basemap_version_id TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
request_json TEXT NOT NULL DEFAULT '',
|
||||
response_status INTEGER,
|
||||
response_body TEXT NOT NULL DEFAULT '',
|
||||
error_message TEXT NOT NULL DEFAULT '',
|
||||
attempt_count INTEGER NOT NULL DEFAULT 0,
|
||||
next_attempt_at TEXT,
|
||||
pushed_at TEXT NOT NULL,
|
||||
finished_at TEXT,
|
||||
FOREIGN KEY (target_system_id) REFERENCES target_systems(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (basemap_version_id) REFERENCES basemap_versions(id) ON DELETE CASCADE
|
||||
);`,
|
||||
`CREATE TABLE IF NOT EXISTS vector_assets (
|
||||
id TEXT PRIMARY KEY,
|
||||
code TEXT NOT NULL UNIQUE,
|
||||
name TEXT NOT NULL,
|
||||
format TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
description TEXT NOT NULL DEFAULT '',
|
||||
file_path TEXT NOT NULL,
|
||||
file_name TEXT NOT NULL,
|
||||
file_size INTEGER NOT NULL DEFAULT 0,
|
||||
checksum TEXT NOT NULL DEFAULT '',
|
||||
download_url TEXT NOT NULL,
|
||||
metadata_json TEXT NOT NULL DEFAULT '',
|
||||
health TEXT NOT NULL DEFAULT '',
|
||||
health_checked_at TEXT NOT NULL DEFAULT '',
|
||||
health_detail TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);`,
|
||||
`CREATE TABLE IF NOT EXISTS vector_token_grants (
|
||||
id TEXT PRIMARY KEY,
|
||||
token_id TEXT NOT NULL,
|
||||
vector_asset_id TEXT NOT NULL,
|
||||
permission TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
FOREIGN KEY (token_id) REFERENCES service_tokens(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (vector_asset_id) REFERENCES vector_assets(id) ON DELETE CASCADE
|
||||
);`,
|
||||
`CREATE TABLE IF NOT EXISTS vector_push_records (
|
||||
id TEXT PRIMARY KEY,
|
||||
target_system_id TEXT NOT NULL,
|
||||
vector_asset_id TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
request_json TEXT NOT NULL DEFAULT '',
|
||||
response_status INTEGER,
|
||||
response_body TEXT NOT NULL DEFAULT '',
|
||||
error_message TEXT NOT NULL DEFAULT '',
|
||||
attempt_count INTEGER NOT NULL DEFAULT 0,
|
||||
next_attempt_at TEXT,
|
||||
pushed_at TEXT NOT NULL,
|
||||
finished_at TEXT,
|
||||
FOREIGN KEY (target_system_id) REFERENCES target_systems(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (vector_asset_id) REFERENCES vector_assets(id) ON DELETE CASCADE
|
||||
);`,
|
||||
`CREATE TABLE IF NOT EXISTS tile_jobs (
|
||||
id TEXT PRIMARY KEY,
|
||||
code TEXT NOT NULL UNIQUE,
|
||||
name TEXT NOT NULL,
|
||||
source_path TEXT NOT NULL,
|
||||
source_count INTEGER NOT NULL DEFAULT 1,
|
||||
sources_json TEXT NOT NULL DEFAULT '',
|
||||
target_code TEXT NOT NULL,
|
||||
output_root TEXT NOT NULL,
|
||||
zoom_min INTEGER NOT NULL DEFAULT 0,
|
||||
zoom_max INTEGER NOT NULL DEFAULT 0,
|
||||
tile_size INTEGER NOT NULL DEFAULT 256,
|
||||
tile_format TEXT NOT NULL DEFAULT 'webp',
|
||||
status TEXT NOT NULL,
|
||||
progress TEXT NOT NULL DEFAULT '',
|
||||
message TEXT NOT NULL DEFAULT '',
|
||||
log_path TEXT NOT NULL DEFAULT '',
|
||||
callback_url TEXT NOT NULL DEFAULT '',
|
||||
callback_status TEXT NOT NULL DEFAULT '',
|
||||
callback_attempts INTEGER NOT NULL DEFAULT 0,
|
||||
callback_next_attempt_at TEXT,
|
||||
callback_message TEXT NOT NULL DEFAULT '',
|
||||
requested_by TEXT NOT NULL DEFAULT '',
|
||||
created_at TEXT NOT NULL,
|
||||
started_at TEXT,
|
||||
finished_at TEXT
|
||||
);`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_tile_jobs_status ON tile_jobs(status, created_at);`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_tile_jobs_target ON tile_jobs(target_code);`,
|
||||
`CREATE TABLE IF NOT EXISTS layer_sources (
|
||||
id TEXT PRIMARY KEY,
|
||||
target_code TEXT NOT NULL,
|
||||
source_path TEXT NOT NULL,
|
||||
added_at TEXT NOT NULL,
|
||||
UNIQUE(target_code, source_path)
|
||||
);`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_layer_sources_target ON layer_sources(target_code);`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_versions_basemap ON basemap_versions(basemap_id);`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_versions_source ON basemap_versions(scan_source_id);`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_grants_token ON token_grants(token_id);`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_push_target ON push_records(target_system_id);`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_vector_file_name ON vector_assets(file_name COLLATE NOCASE);`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_vector_grants_token ON vector_token_grants(token_id);`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_vector_push_target ON vector_push_records(target_system_id);`,
|
||||
}
|
||||
|
||||
for _, statement := range statements {
|
||||
if _, err := s.db.ExecContext(ctx, statement); err != nil {
|
||||
return fmt.Errorf("bootstrap sqlite: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// CREATE TABLE IF NOT EXISTS never evolves an existing database, so columns
|
||||
// added after the first release are patched in here.
|
||||
migrations := []struct {
|
||||
table string
|
||||
column string
|
||||
definition string
|
||||
}{
|
||||
{"basemap_versions", "health", "TEXT NOT NULL DEFAULT ''"},
|
||||
{"basemap_versions", "health_checked_at", "TEXT NOT NULL DEFAULT ''"},
|
||||
{"basemap_versions", "health_detail", "TEXT NOT NULL DEFAULT ''"},
|
||||
{"vector_assets", "health", "TEXT NOT NULL DEFAULT ''"},
|
||||
{"vector_assets", "health_checked_at", "TEXT NOT NULL DEFAULT ''"},
|
||||
{"vector_assets", "health_detail", "TEXT NOT NULL DEFAULT ''"},
|
||||
{"push_records", "attempt_count", "INTEGER NOT NULL DEFAULT 0"},
|
||||
{"push_records", "next_attempt_at", "TEXT"},
|
||||
{"vector_push_records", "attempt_count", "INTEGER NOT NULL DEFAULT 0"},
|
||||
{"vector_push_records", "next_attempt_at", "TEXT"},
|
||||
{"service_tokens", "scopes", "TEXT NOT NULL DEFAULT 'read'"},
|
||||
{"tile_jobs", "source_count", "INTEGER NOT NULL DEFAULT 1"},
|
||||
{"tile_jobs", "sources_json", "TEXT NOT NULL DEFAULT ''"},
|
||||
{"tile_jobs", "callback_url", "TEXT NOT NULL DEFAULT ''"},
|
||||
{"tile_jobs", "callback_status", "TEXT NOT NULL DEFAULT ''"},
|
||||
{"tile_jobs", "callback_attempts", "INTEGER NOT NULL DEFAULT 0"},
|
||||
{"tile_jobs", "callback_next_attempt_at", "TEXT"},
|
||||
{"tile_jobs", "callback_message", "TEXT NOT NULL DEFAULT ''"},
|
||||
}
|
||||
for _, migration := range migrations {
|
||||
if err := s.ensureColumn(ctx, migration.table, migration.column, migration.definition); err != nil {
|
||||
return fmt.Errorf("bootstrap migrate %s.%s: %w", migration.table, migration.column, err)
|
||||
}
|
||||
}
|
||||
// These indexes reference migrated columns, so they run after the patch.
|
||||
for _, statement := range []string{
|
||||
`CREATE INDEX IF NOT EXISTS idx_push_next_attempt ON push_records(next_attempt_at);`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_vector_push_next_attempt ON vector_push_records(next_attempt_at);`,
|
||||
} {
|
||||
if _, err := s.db.ExecContext(ctx, statement); err != nil {
|
||||
return fmt.Errorf("bootstrap index: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensureColumn adds a column when an existing table is missing it. PRAGMA
|
||||
// table_info is the portable way to test SQLite column presence.
|
||||
func (s *Store) ensureColumn(ctx context.Context, table, column, definition string) error {
|
||||
rows, err := s.db.QueryContext(ctx, "PRAGMA table_info("+table+")")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var cid, notNull, primaryKey int
|
||||
var name, columnType string
|
||||
var defaultValue sql.NullString
|
||||
if err := rows.Scan(&cid, &name, &columnType, ¬Null, &defaultValue, &primaryKey); err != nil {
|
||||
return err
|
||||
}
|
||||
if name == column {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.db.ExecContext(ctx, "ALTER TABLE "+table+" ADD COLUMN "+column+" "+definition)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) AutoScanEnabled() bool {
|
||||
return s.cfg.AutoScanOnStart
|
||||
}
|
||||
|
||||
func (s *Store) ScanRoot() string {
|
||||
return s.cfg.ScanRoot
|
||||
}
|
||||
|
||||
func (s *Store) VectorScanRoot() string {
|
||||
return s.cfg.VectorScanRoot
|
||||
}
|
||||
|
||||
func (s *Store) generateTokenSecret() (string, string, error) {
|
||||
var raw [32]byte
|
||||
if _, err := rand.Read(raw[:]); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
encoder := base32.StdEncoding.WithPadding(base32.NoPadding)
|
||||
secret := strings.ToLower(encoder.EncodeToString(raw[:]))
|
||||
token := "bmp_" + secret
|
||||
sum := sha256.Sum256([]byte(token))
|
||||
return token, hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
func normalizeBasemapCode(value string) string {
|
||||
return normalizeCode(value)
|
||||
}
|
||||
|
||||
func nowUTC() time.Time {
|
||||
return time.Now().UTC().Round(time.Second)
|
||||
}
|
||||
|
||||
func toRFC3339(t time.Time) string {
|
||||
return t.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
func nullableTime(t *time.Time) any {
|
||||
if t == nil || t.IsZero() {
|
||||
return nil
|
||||
}
|
||||
return toRFC3339(*t)
|
||||
}
|
||||
|
||||
func scanNullString(value string) sql.NullString {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return sql.NullString{}
|
||||
}
|
||||
return sql.NullString{String: value, Valid: true}
|
||||
}
|
||||
|
||||
func readMaybeJSON[T any](value string, target *T) error {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return nil
|
||||
}
|
||||
return json.Unmarshal([]byte(value), target)
|
||||
}
|
||||
|
||||
func writeJSON(value any) string {
|
||||
if value == nil {
|
||||
return ""
|
||||
}
|
||||
encoded, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(encoded)
|
||||
}
|
||||
|
||||
func sortedKeys(values map[string]struct{}) []string {
|
||||
keys := make([]string, 0, len(values))
|
||||
for key := range values {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
func trimURL(value string) string {
|
||||
return strings.TrimRight(strings.TrimSpace(value), "/")
|
||||
}
|
||||
|
||||
func limitedReadAll(body io.Reader, limit int64) string {
|
||||
content, err := io.ReadAll(io.LimitReader(body, limit))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(content)
|
||||
}
|
||||
@@ -0,0 +1,337 @@
|
||||
package basemap
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type basemapVersionRow struct {
|
||||
ID string
|
||||
BasemapID string
|
||||
BasemapCode string
|
||||
Version string
|
||||
Status string
|
||||
IsDefault bool
|
||||
ManifestPath string
|
||||
TileRootPath string
|
||||
URLTemplate string
|
||||
TileFormat string
|
||||
TileScheme string
|
||||
MinZoom int
|
||||
MaxZoom int
|
||||
BBoxJSON string
|
||||
Attribution string
|
||||
MetadataJSON string
|
||||
CreatedAt string
|
||||
UpdatedAt string
|
||||
}
|
||||
|
||||
func parseTime(value string) time.Time {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return time.Time{}
|
||||
}
|
||||
parsed, err := time.Parse(time.RFC3339, value)
|
||||
if err != nil {
|
||||
return time.Time{}
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func scanBasemapVersionRow(scanner interface {
|
||||
Scan(dest ...any) error
|
||||
}) (basemapVersionRow, error) {
|
||||
var row basemapVersionRow
|
||||
var isDefault int
|
||||
if err := scanner.Scan(
|
||||
&row.ID,
|
||||
&row.BasemapID,
|
||||
&row.BasemapCode,
|
||||
&row.Version,
|
||||
&row.Status,
|
||||
&isDefault,
|
||||
&row.ManifestPath,
|
||||
&row.TileRootPath,
|
||||
&row.URLTemplate,
|
||||
&row.TileFormat,
|
||||
&row.TileScheme,
|
||||
&row.MinZoom,
|
||||
&row.MaxZoom,
|
||||
&row.BBoxJSON,
|
||||
&row.Attribution,
|
||||
&row.MetadataJSON,
|
||||
&row.CreatedAt,
|
||||
&row.UpdatedAt,
|
||||
); err != nil {
|
||||
return basemapVersionRow{}, err
|
||||
}
|
||||
row.IsDefault = isDefault == 1
|
||||
return row, nil
|
||||
}
|
||||
|
||||
func decodeBasemapVersion(row basemapVersionRow) BasemapVersion {
|
||||
item := BasemapVersion{
|
||||
ID: row.ID,
|
||||
BasemapID: row.BasemapID,
|
||||
BasemapCode: row.BasemapCode,
|
||||
Version: row.Version,
|
||||
Status: row.Status,
|
||||
IsDefault: row.IsDefault,
|
||||
ManifestPath: row.ManifestPath,
|
||||
TileRootPath: row.TileRootPath,
|
||||
URLTemplate: row.URLTemplate,
|
||||
TileFormat: row.TileFormat,
|
||||
TileScheme: row.TileScheme,
|
||||
MinZoom: row.MinZoom,
|
||||
MaxZoom: row.MaxZoom,
|
||||
Attribution: row.Attribution,
|
||||
CreatedAt: parseTime(row.CreatedAt),
|
||||
UpdatedAt: parseTime(row.UpdatedAt),
|
||||
}
|
||||
if strings.TrimSpace(row.BBoxJSON) != "" {
|
||||
_ = readMaybeJSON(row.BBoxJSON, &item.BBox)
|
||||
}
|
||||
if strings.TrimSpace(row.MetadataJSON) != "" {
|
||||
_ = readMaybeJSON(row.MetadataJSON, &item.Metadata)
|
||||
}
|
||||
if item.Metadata == nil {
|
||||
item.Metadata = map[string]any{}
|
||||
}
|
||||
return item
|
||||
}
|
||||
|
||||
func scanBasemap(scanner interface {
|
||||
Scan(dest ...any) error
|
||||
}) (Basemap, error) {
|
||||
var item Basemap
|
||||
var createdAt string
|
||||
var updatedAt string
|
||||
if err := scanner.Scan(
|
||||
&item.ID,
|
||||
&item.Code,
|
||||
&item.Name,
|
||||
&item.Type,
|
||||
&item.Status,
|
||||
&item.Description,
|
||||
&createdAt,
|
||||
&updatedAt,
|
||||
); err != nil {
|
||||
return Basemap{}, err
|
||||
}
|
||||
item.CreatedAt = parseTime(createdAt)
|
||||
item.UpdatedAt = parseTime(updatedAt)
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func scanScanSource(scanner interface {
|
||||
Scan(dest ...any) error
|
||||
}) (ScanSource, error) {
|
||||
var item ScanSource
|
||||
var enabled int
|
||||
var metadataJSON string
|
||||
var createdAt string
|
||||
var updatedAt string
|
||||
if err := scanner.Scan(
|
||||
&item.ID,
|
||||
&item.Code,
|
||||
&item.Name,
|
||||
&item.RootPath,
|
||||
&item.ManifestName,
|
||||
&enabled,
|
||||
&metadataJSON,
|
||||
&createdAt,
|
||||
&updatedAt,
|
||||
); err != nil {
|
||||
return ScanSource{}, err
|
||||
}
|
||||
item.Enabled = enabled == 1
|
||||
item.CreatedAt = parseTime(createdAt)
|
||||
item.UpdatedAt = parseTime(updatedAt)
|
||||
if metadataJSON != "" {
|
||||
_ = readMaybeJSON(metadataJSON, &item.Metadata)
|
||||
}
|
||||
if item.Metadata == nil {
|
||||
item.Metadata = map[string]any{}
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func scanScanRun(scanner interface {
|
||||
Scan(dest ...any) error
|
||||
}) (ScanRun, error) {
|
||||
var item ScanRun
|
||||
var summaryJSON string
|
||||
var startedAt string
|
||||
var finishedAt string
|
||||
if err := scanner.Scan(
|
||||
&item.ID,
|
||||
&item.ScanSourceID,
|
||||
&item.SourceCode,
|
||||
&item.Status,
|
||||
&item.ScannedCount,
|
||||
&item.AddedCount,
|
||||
&item.UpdatedCount,
|
||||
&item.RemovedCount,
|
||||
&summaryJSON,
|
||||
&startedAt,
|
||||
&finishedAt,
|
||||
); err != nil {
|
||||
return ScanRun{}, err
|
||||
}
|
||||
item.StartedAt = parseTime(startedAt)
|
||||
item.FinishedAt = parseTime(finishedAt)
|
||||
if summaryJSON != "" {
|
||||
_ = readMaybeJSON(summaryJSON, &item.Summary)
|
||||
}
|
||||
if item.Summary == nil {
|
||||
item.Summary = map[string]any{}
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func scanTargetSystem(scanner interface {
|
||||
Scan(dest ...any) error
|
||||
}) (TargetSystem, error) {
|
||||
var item TargetSystem
|
||||
var enabled int
|
||||
var headersJSON string
|
||||
var createdAt string
|
||||
var updatedAt string
|
||||
if err := scanner.Scan(
|
||||
&item.ID,
|
||||
&item.Code,
|
||||
&item.Name,
|
||||
&item.CallbackURL,
|
||||
&item.CallbackMethod,
|
||||
&headersJSON,
|
||||
&enabled,
|
||||
&createdAt,
|
||||
&updatedAt,
|
||||
); err != nil {
|
||||
return TargetSystem{}, err
|
||||
}
|
||||
item.Enabled = enabled == 1
|
||||
item.CreatedAt = parseTime(createdAt)
|
||||
item.UpdatedAt = parseTime(updatedAt)
|
||||
if headersJSON != "" {
|
||||
_ = readMaybeJSON(headersJSON, &item.CallbackHeaders)
|
||||
}
|
||||
if item.CallbackHeaders == nil {
|
||||
item.CallbackHeaders = map[string]string{}
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func scanServiceToken(scanner interface {
|
||||
Scan(dest ...any) error
|
||||
}) (ServiceToken, string, error) {
|
||||
var item ServiceToken
|
||||
var tokenHash string
|
||||
var scopes string
|
||||
var expiresAt sql.NullString
|
||||
var lastUsedAt sql.NullString
|
||||
var createdAt string
|
||||
if err := scanner.Scan(
|
||||
&item.ID,
|
||||
&item.Name,
|
||||
&tokenHash,
|
||||
&item.TokenPrefix,
|
||||
&scopes,
|
||||
&item.Status,
|
||||
&expiresAt,
|
||||
&lastUsedAt,
|
||||
&createdAt,
|
||||
); err != nil {
|
||||
return ServiceToken{}, "", err
|
||||
}
|
||||
item.CreatedAt = parseTime(createdAt)
|
||||
item.Scopes = splitTokenScopes(scopes)
|
||||
if expiresAt.Valid {
|
||||
value := parseTime(expiresAt.String)
|
||||
item.ExpiresAt = &value
|
||||
}
|
||||
if lastUsedAt.Valid {
|
||||
value := parseTime(lastUsedAt.String)
|
||||
item.LastUsedAt = &value
|
||||
}
|
||||
return item, tokenHash, nil
|
||||
}
|
||||
|
||||
func scanTokenGrant(scanner interface {
|
||||
Scan(dest ...any) error
|
||||
}) (TokenGrant, error) {
|
||||
var item TokenGrant
|
||||
var versionID sql.NullString
|
||||
var version sql.NullString
|
||||
var createdAt string
|
||||
if err := scanner.Scan(
|
||||
&item.ID,
|
||||
&item.TokenID,
|
||||
&item.BasemapID,
|
||||
&item.BasemapCode,
|
||||
&versionID,
|
||||
&version,
|
||||
&item.Permission,
|
||||
&createdAt,
|
||||
); err != nil {
|
||||
return TokenGrant{}, err
|
||||
}
|
||||
if versionID.Valid {
|
||||
item.BasemapVersionID = &versionID.String
|
||||
}
|
||||
if version.Valid {
|
||||
item.BasemapVersion = &version.String
|
||||
}
|
||||
item.CreatedAt = parseTime(createdAt)
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func scanPushRecord(scanner interface {
|
||||
Scan(dest ...any) error
|
||||
}) (PushRecord, error) {
|
||||
var item PushRecord
|
||||
var requestJSON string
|
||||
var responseStatus sql.NullInt64
|
||||
var nextAttemptAt sql.NullString
|
||||
var finishedAt sql.NullString
|
||||
var pushedAt string
|
||||
if err := scanner.Scan(
|
||||
&item.ID,
|
||||
&item.TargetSystemID,
|
||||
&item.TargetSystemCode,
|
||||
&item.BasemapVersionID,
|
||||
&item.BasemapCode,
|
||||
&item.BasemapVersion,
|
||||
&item.Status,
|
||||
&requestJSON,
|
||||
&responseStatus,
|
||||
&item.ResponseBody,
|
||||
&item.ErrorMessage,
|
||||
&item.AttemptCount,
|
||||
&nextAttemptAt,
|
||||
&pushedAt,
|
||||
&finishedAt,
|
||||
); err != nil {
|
||||
return PushRecord{}, err
|
||||
}
|
||||
if requestJSON != "" {
|
||||
_ = readMaybeJSON(requestJSON, &item.Request)
|
||||
}
|
||||
if item.Request == nil {
|
||||
item.Request = map[string]any{}
|
||||
}
|
||||
if responseStatus.Valid {
|
||||
value := int(responseStatus.Int64)
|
||||
item.ResponseStatus = &value
|
||||
}
|
||||
if nextAttemptAt.Valid {
|
||||
value := parseTime(nextAttemptAt.String)
|
||||
item.NextAttemptAt = &value
|
||||
}
|
||||
item.PushedAt = parseTime(pushedAt)
|
||||
if finishedAt.Valid {
|
||||
value := parseTime(finishedAt.String)
|
||||
item.FinishedAt = &value
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
@@ -0,0 +1,714 @@
|
||||
package basemap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"tile-server/api-go/internal/uid"
|
||||
)
|
||||
|
||||
// Tile job statuses.
|
||||
const (
|
||||
tileJobPending = "pending"
|
||||
tileJobRunning = "running"
|
||||
tileJobSucceeded = "succeeded"
|
||||
tileJobFailed = "failed"
|
||||
)
|
||||
|
||||
// defaultTileJobCommand slices a raster into XYZ tiles. {input} is the source
|
||||
// GeoTIFF, {output} the directory the gateway scans.
|
||||
const defaultTileJobCommand = "gdal2tiles.py --xyz --zoom={zmin}-{zmax} --tilesize={tilesize} --exclude {input} {output}"
|
||||
|
||||
// defaultTileJobVRTCommand mosaics a layer's sources into one virtual raster
|
||||
// so overlapping flights composite per pixel instead of per tile.
|
||||
const defaultTileJobVRTCommand = "gdalbuildvrt -resolution highest {output_vrt} {inputs}"
|
||||
|
||||
// tileJobColumns keeps every read path in step with the struct.
|
||||
const tileJobColumns = `id, code, name, source_path, source_count, sources_json, target_code, output_root, zoom_min, zoom_max,
|
||||
tile_size, tile_format, status, progress, message, log_path, callback_url, callback_status,
|
||||
callback_attempts, callback_next_attempt_at, callback_message, requested_by, created_at, started_at, finished_at`
|
||||
|
||||
type TileJob struct {
|
||||
ID string `json:"id"`
|
||||
Code string `json:"code"`
|
||||
Name string `json:"name"`
|
||||
SourcePath string `json:"source_path"`
|
||||
SourceCount int `json:"source_count"`
|
||||
SourcePaths []string `json:"source_paths,omitempty"`
|
||||
TargetCode string `json:"target_code"`
|
||||
OutputRoot string `json:"output_root"`
|
||||
ZoomMin int `json:"zoom_min"`
|
||||
ZoomMax int `json:"zoom_max"`
|
||||
TileSize int `json:"tile_size"`
|
||||
TileFormat string `json:"tile_format"`
|
||||
Status string `json:"status"`
|
||||
Progress string `json:"progress"`
|
||||
Message string `json:"message"`
|
||||
LogPath string `json:"log_path"`
|
||||
CallbackURL string `json:"callback_url,omitempty"`
|
||||
CallbackStatus string `json:"callback_status,omitempty"`
|
||||
CallbackAttempts int `json:"callback_attempts,omitempty"`
|
||||
CallbackMessage string `json:"callback_message,omitempty"`
|
||||
CallbackNextAttemptAt *time.Time `json:"callback_next_attempt_at,omitempty"`
|
||||
RequestedBy string `json:"requested_by"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
||||
}
|
||||
|
||||
type CreateTileJobInput struct {
|
||||
Code string `json:"code"`
|
||||
Name string `json:"name"`
|
||||
SourcePath string `json:"source_path"`
|
||||
SourcePaths []string `json:"source_paths"`
|
||||
TargetCode string `json:"target_code"`
|
||||
ZoomMin int `json:"zoom_min"`
|
||||
ZoomMax int `json:"zoom_max"`
|
||||
TileSize int `json:"tile_size"`
|
||||
TileFormat string `json:"tile_format"`
|
||||
CallbackURL string `json:"callback_url"`
|
||||
RequestedBy string `json:"requested_by"`
|
||||
}
|
||||
|
||||
// CreateTileJob validates the request and queues it. Slicing itself runs in the
|
||||
// background so a long gdal2tiles run cannot block the console.
|
||||
func (s *Store) CreateTileJob(ctx context.Context, input CreateTileJobInput) (TileJob, error) {
|
||||
sources, err := resolveTileJobSources(input)
|
||||
if err != nil {
|
||||
return TileJob{}, err
|
||||
}
|
||||
absSource := sources[0]
|
||||
|
||||
targetCode := normalizeBasemapCode(input.TargetCode)
|
||||
if targetCode == "" {
|
||||
return TileJob{}, errors.New("target code is required")
|
||||
}
|
||||
scanRoot := strings.TrimSpace(s.cfg.ScanRoot)
|
||||
if scanRoot == "" {
|
||||
return TileJob{}, errors.New("basemap scan root is not configured")
|
||||
}
|
||||
outputRoot := filepath.Join(scanRoot, targetCode)
|
||||
|
||||
if input.ZoomMin < 0 || input.ZoomMax < 0 || input.ZoomMax < input.ZoomMin {
|
||||
return TileJob{}, errors.New("invalid zoom range")
|
||||
}
|
||||
if input.ZoomMax > 24 {
|
||||
return TileJob{}, errors.New("zoom must be 24 or lower")
|
||||
}
|
||||
tileSize := input.TileSize
|
||||
if tileSize <= 0 {
|
||||
tileSize = 256
|
||||
}
|
||||
tileFormat := strings.ToLower(strings.TrimSpace(input.TileFormat))
|
||||
if tileFormat == "" {
|
||||
tileFormat = "webp"
|
||||
}
|
||||
code := normalizeBasemapCode(input.Code)
|
||||
if code == "" {
|
||||
code = normalizeBasemapCode(targetCode + "-" + nowUTC().Format("20060102150405"))
|
||||
}
|
||||
name := strings.TrimSpace(input.Name)
|
||||
if name == "" {
|
||||
name = filepath.Base(absSource)
|
||||
}
|
||||
|
||||
now := nowUTC()
|
||||
job := TileJob{
|
||||
ID: uid.New(),
|
||||
Code: code,
|
||||
Name: name,
|
||||
SourcePath: absSource,
|
||||
SourceCount: len(sources),
|
||||
SourcePaths: sources,
|
||||
TargetCode: targetCode,
|
||||
OutputRoot: outputRoot,
|
||||
ZoomMin: input.ZoomMin,
|
||||
ZoomMax: input.ZoomMax,
|
||||
TileSize: tileSize,
|
||||
TileFormat: tileFormat,
|
||||
Status: tileJobPending,
|
||||
RequestedBy: strings.TrimSpace(input.RequestedBy),
|
||||
CreatedAt: now,
|
||||
}
|
||||
job.CallbackURL = strings.TrimSpace(input.CallbackURL)
|
||||
|
||||
// A caller-supplied code makes submission idempotent: sending the same code
|
||||
// again returns the job that already exists instead of queueing a second copy.
|
||||
if existing, ok, err := s.getTileJobByCode(ctx, job.Code); err != nil {
|
||||
return TileJob{}, err
|
||||
} else if ok {
|
||||
return existing, nil
|
||||
}
|
||||
|
||||
if _, err := s.db.ExecContext(ctx, `
|
||||
INSERT INTO tile_jobs (
|
||||
id, code, name, source_path, source_count, sources_json, target_code, output_root, zoom_min, zoom_max,
|
||||
tile_size, tile_format, status, progress, message, log_path, callback_url, requested_by, created_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '', '', '', ?, ?, ?)
|
||||
`, job.ID, job.Code, job.Name, job.SourcePath, job.SourceCount, writeJSON(job.SourcePaths), job.TargetCode, job.OutputRoot,
|
||||
job.ZoomMin, job.ZoomMax, job.TileSize, job.TileFormat, job.Status,
|
||||
job.CallbackURL, job.RequestedBy, toRFC3339(now)); err != nil {
|
||||
if strings.Contains(err.Error(), "UNIQUE") {
|
||||
if existing, ok, getErr := s.getTileJobByCode(ctx, job.Code); getErr == nil && ok {
|
||||
return existing, nil
|
||||
}
|
||||
}
|
||||
return TileJob{}, err
|
||||
}
|
||||
// Remember what this layer is made of so it can be rebuilt later.
|
||||
if err := s.AddLayerSources(ctx, job.TargetCode, sources); err != nil {
|
||||
log.Printf("record layer sources for %s: %v", job.TargetCode, err)
|
||||
}
|
||||
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (s *Store) ListTileJobs(ctx context.Context, limit int) ([]TileJob, error) {
|
||||
if limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, "SELECT "+tileJobColumns+`
|
||||
FROM tile_jobs
|
||||
ORDER BY created_at DESC
|
||||
LIMIT ?
|
||||
`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var items []TileJob
|
||||
for rows.Next() {
|
||||
job, err := scanTileJob(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, job)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func scanTileJob(scanner interface {
|
||||
Scan(dest ...any) error
|
||||
}) (TileJob, error) {
|
||||
var job TileJob
|
||||
var createdAt string
|
||||
var sourcesJSON string
|
||||
var startedAt, finishedAt, callbackNextAttemptAt sql.NullString
|
||||
if err := scanner.Scan(
|
||||
&job.ID, &job.Code, &job.Name, &job.SourcePath, &job.SourceCount, &sourcesJSON, &job.TargetCode, &job.OutputRoot,
|
||||
&job.ZoomMin, &job.ZoomMax, &job.TileSize, &job.TileFormat, &job.Status,
|
||||
&job.Progress, &job.Message, &job.LogPath, &job.CallbackURL, &job.CallbackStatus, &job.CallbackAttempts,
|
||||
&callbackNextAttemptAt, &job.CallbackMessage, &job.RequestedBy, &createdAt,
|
||||
&startedAt, &finishedAt,
|
||||
); err != nil {
|
||||
return TileJob{}, err
|
||||
}
|
||||
job.CreatedAt = parseTime(createdAt)
|
||||
if strings.TrimSpace(sourcesJSON) != "" {
|
||||
_ = readMaybeJSON(sourcesJSON, &job.SourcePaths)
|
||||
}
|
||||
if len(job.SourcePaths) == 0 && job.SourcePath != "" {
|
||||
job.SourcePaths = []string{job.SourcePath}
|
||||
}
|
||||
if startedAt.Valid {
|
||||
value := parseTime(startedAt.String)
|
||||
job.StartedAt = &value
|
||||
}
|
||||
if finishedAt.Valid {
|
||||
value := parseTime(finishedAt.String)
|
||||
job.FinishedAt = &value
|
||||
}
|
||||
if callbackNextAttemptAt.Valid {
|
||||
value := parseTime(callbackNextAttemptAt.String)
|
||||
job.CallbackNextAttemptAt = &value
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
|
||||
// GetTileJob returns one job by id so a caller can poll a submission.
|
||||
func (s *Store) GetTileJob(ctx context.Context, id string) (TileJob, error) {
|
||||
row := s.db.QueryRowContext(ctx, "SELECT "+tileJobColumns+" FROM tile_jobs WHERE id = ?", strings.TrimSpace(id))
|
||||
job, err := scanTileJob(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return TileJob{}, fmt.Errorf("tile job %q not found", id)
|
||||
}
|
||||
return TileJob{}, err
|
||||
}
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (s *Store) getTileJobByCode(ctx context.Context, code string) (TileJob, bool, error) {
|
||||
row := s.db.QueryRowContext(ctx, "SELECT "+tileJobColumns+" FROM tile_jobs WHERE code = ?", code)
|
||||
job, err := scanTileJob(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return TileJob{}, false, nil
|
||||
}
|
||||
return TileJob{}, false, err
|
||||
}
|
||||
return job, true, nil
|
||||
}
|
||||
|
||||
// runTileJob executes one queued job and records the outcome verbatim.
|
||||
func (s *Store) runTileJob(ctx context.Context, job TileJob) {
|
||||
startedAt := nowUTC()
|
||||
if _, err := s.db.ExecContext(ctx, `
|
||||
UPDATE tile_jobs SET status = ?, started_at = ?, message = '' WHERE id = ?
|
||||
`, tileJobRunning, toRFC3339(startedAt), job.ID); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
logPath, _, runErr := s.executeTileJob(ctx, job)
|
||||
|
||||
status := tileJobSucceeded
|
||||
message := ""
|
||||
if runErr != nil {
|
||||
status = tileJobFailed
|
||||
message = runErr.Error()
|
||||
}
|
||||
finishedAt := nowUTC()
|
||||
if _, err := s.db.ExecContext(context.Background(), `
|
||||
UPDATE tile_jobs SET status = ?, message = ?, log_path = ?, progress = ?, finished_at = ? WHERE id = ?
|
||||
`, status, message, logPath, message, toRFC3339(finishedAt), job.ID); err != nil {
|
||||
return
|
||||
}
|
||||
// Tell the submitting system the job is done, if it asked to be told.
|
||||
if strings.TrimSpace(job.CallbackURL) != "" {
|
||||
s.queueTileJobCallback(context.Background(), job.ID)
|
||||
}
|
||||
|
||||
// A sliced basemap only becomes visible after the scan picks it up.
|
||||
if status == tileJobSucceeded && strings.TrimSpace(s.cfg.ScanRoot) != "" {
|
||||
if _, err := s.RunScan(context.Background(), defaultScanSourceCode); err != nil {
|
||||
if _, updateErr := s.db.ExecContext(context.Background(), `
|
||||
UPDATE tile_jobs SET message = ? WHERE id = ?
|
||||
`, "切片完成,但扫描入库失败:"+err.Error(), job.ID); updateErr != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// executeTileJob builds the input (mosaicking when several sources were
|
||||
// submitted), runs the slicer and captures its output.
|
||||
func (s *Store) executeTileJob(ctx context.Context, job TileJob) (string, []string, error) {
|
||||
if err := os.MkdirAll(job.OutputRoot, 0o755); err != nil {
|
||||
return "", nil, fmt.Errorf(
|
||||
"create output root %s: %w (the account running the service needs write access to the basemap scan root)",
|
||||
job.OutputRoot, err)
|
||||
}
|
||||
logDir := strings.TrimSpace(s.cfg.TileJobLogDir)
|
||||
if logDir == "" {
|
||||
// Keep logs beside the data: a system temp directory is not always
|
||||
// writable for the service account, and this stays under the scan root
|
||||
// without introducing a zoom directory the scanner would pick up.
|
||||
logDir = filepath.Join(filepath.Dir(job.OutputRoot), ".tile-jobs")
|
||||
}
|
||||
if err := os.MkdirAll(logDir, 0o755); err != nil {
|
||||
return "", nil, fmt.Errorf("create log dir: %w", err)
|
||||
}
|
||||
logPath := filepath.Join(logDir, job.Code+".log")
|
||||
logFile, err := os.Create(logPath)
|
||||
if err != nil {
|
||||
return logPath, nil, fmt.Errorf("create job log: %w", err)
|
||||
}
|
||||
defer logFile.Close()
|
||||
|
||||
// One source slices directly; several are mosaicked first so overlapping
|
||||
// flights composite per pixel instead of per tile.
|
||||
input := job.SourcePath
|
||||
if len(job.SourcePaths) > 1 {
|
||||
vrtPath := filepath.Join(logDir, job.Code+".vrt")
|
||||
if err := s.buildTileJobMosaic(ctx, job, vrtPath, logFile); err != nil {
|
||||
return logPath, nil, err
|
||||
}
|
||||
input = vrtPath
|
||||
}
|
||||
|
||||
template := strings.TrimSpace(s.cfg.TileJobCommand)
|
||||
if template == "" {
|
||||
template = defaultTileJobCommand
|
||||
}
|
||||
resolved := tileJobCommandArgs(template, job, input)
|
||||
if len(resolved) == 0 {
|
||||
return logPath, nil, errors.New("tile job command template is empty")
|
||||
}
|
||||
|
||||
command := exec.CommandContext(ctx, resolved[0], resolved[1:]...)
|
||||
command.Stdout = logFile
|
||||
command.Stderr = logFile
|
||||
if err := command.Run(); err != nil {
|
||||
return logPath, resolved, fmt.Errorf("%s: %w", strings.Join(resolved, " "), err)
|
||||
}
|
||||
return logPath, resolved, nil
|
||||
}
|
||||
|
||||
// buildTileJobMosaic runs gdalbuildvrt over the job's sources.
|
||||
func (s *Store) buildTileJobMosaic(ctx context.Context, job TileJob, vrtPath string, logFile *os.File) error {
|
||||
template := strings.TrimSpace(s.cfg.TileJobVRTCommand)
|
||||
if template == "" {
|
||||
template = defaultTileJobVRTCommand
|
||||
}
|
||||
resolved := tileJobVRTCommandArgs(template, job, vrtPath)
|
||||
if len(resolved) == 0 {
|
||||
return errors.New("tile job vrt command template is empty")
|
||||
}
|
||||
command := exec.CommandContext(ctx, resolved[0], resolved[1:]...)
|
||||
command.Stdout = logFile
|
||||
command.Stderr = logFile
|
||||
if err := command.Run(); err != nil {
|
||||
return fmt.Errorf("build mosaic: %s: %w", strings.Join(resolved, " "), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// resolveTileJobSources validates every submitted source and returns absolute
|
||||
// paths. SourcePaths wins over the single SourcePath field.
|
||||
func resolveTileJobSources(input CreateTileJobInput) ([]string, error) {
|
||||
raw := input.SourcePaths
|
||||
if len(raw) == 0 && strings.TrimSpace(input.SourcePath) != "" {
|
||||
raw = []string{input.SourcePath}
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil, errors.New("source path is required")
|
||||
}
|
||||
|
||||
seen := map[string]struct{}{}
|
||||
sources := make([]string, 0, len(raw))
|
||||
for _, candidate := range raw {
|
||||
candidate = strings.TrimSpace(candidate)
|
||||
if candidate == "" {
|
||||
continue
|
||||
}
|
||||
abs, err := filepath.Abs(candidate)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve source path %q: %w", candidate, err)
|
||||
}
|
||||
info, err := os.Stat(abs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("source path %q is not readable: %w", candidate, err)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil, fmt.Errorf("source path %q must be a file, not a directory", candidate)
|
||||
}
|
||||
if _, dup := seen[abs]; dup {
|
||||
continue
|
||||
}
|
||||
seen[abs] = struct{}{}
|
||||
sources = append(sources, abs)
|
||||
}
|
||||
if len(sources) == 0 {
|
||||
return nil, errors.New("source path is required")
|
||||
}
|
||||
return sources, nil
|
||||
}
|
||||
|
||||
// tileJobCommandArgs resolves the slicer template. Quoted segments are kept
|
||||
// together so paths with spaces survive.
|
||||
func tileJobCommandArgs(template string, job TileJob, input string) []string {
|
||||
return expandTileJobTemplate(template, map[string]string{
|
||||
"{input}": input,
|
||||
"{output}": job.OutputRoot,
|
||||
"{zmin}": fmt.Sprintf("%d", job.ZoomMin),
|
||||
"{zmax}": fmt.Sprintf("%d", job.ZoomMax),
|
||||
"{tilesize}": fmt.Sprintf("%d", job.TileSize),
|
||||
"{format}": job.TileFormat,
|
||||
"{code}": job.TargetCode,
|
||||
}, nil)
|
||||
}
|
||||
|
||||
// tileJobVRTCommandArgs resolves the mosaic template. {inputs} expands into one
|
||||
// argument per source raster.
|
||||
func tileJobVRTCommandArgs(template string, job TileJob, vrtPath string) []string {
|
||||
return expandTileJobTemplate(template, map[string]string{
|
||||
"{output_vrt}": vrtPath,
|
||||
"{output}": job.OutputRoot,
|
||||
"{code}": job.TargetCode,
|
||||
}, map[string][]string{"{inputs}": job.SourcePaths})
|
||||
}
|
||||
|
||||
// expandTileJobTemplate splits the template and substitutes placeholders. A
|
||||
// token that is exactly a list placeholder expands into several arguments.
|
||||
func expandTileJobTemplate(template string, scalars map[string]string, lists map[string][]string) []string {
|
||||
parts := splitCommandLine(template)
|
||||
args := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
if values, ok := lists[part]; ok {
|
||||
args = append(args, values...)
|
||||
continue
|
||||
}
|
||||
for key, value := range scalars {
|
||||
part = strings.ReplaceAll(part, key, value)
|
||||
}
|
||||
args = append(args, part)
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
// splitCommandLine splits on whitespace while honouring double quotes.
|
||||
func splitCommandLine(value string) []string {
|
||||
var parts []string
|
||||
var current strings.Builder
|
||||
inQuotes := false
|
||||
for _, r := range value {
|
||||
switch {
|
||||
case r == '"':
|
||||
inQuotes = !inQuotes
|
||||
case (r == ' ' || r == '\t' || r == '\n' || r == '\r') && !inQuotes:
|
||||
if current.Len() > 0 {
|
||||
parts = append(parts, current.String())
|
||||
current.Reset()
|
||||
}
|
||||
default:
|
||||
current.WriteRune(r)
|
||||
}
|
||||
}
|
||||
if current.Len() > 0 {
|
||||
parts = append(parts, current.String())
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
// dispatchTileJobs runs queued jobs one at a time, oldest first.
|
||||
func (s *Store) dispatchTileJobs(ctx context.Context) {
|
||||
for {
|
||||
job, ok, err := s.nextPendingTileJob(ctx)
|
||||
if err != nil || !ok {
|
||||
return
|
||||
}
|
||||
s.runTileJob(ctx, job)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Store) nextPendingTileJob(ctx context.Context) (TileJob, bool, error) {
|
||||
row := s.db.QueryRowContext(ctx, "SELECT "+tileJobColumns+`
|
||||
FROM tile_jobs
|
||||
WHERE status = ?
|
||||
ORDER BY created_at
|
||||
LIMIT 1
|
||||
`, tileJobPending)
|
||||
job, err := scanTileJob(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return TileJob{}, false, nil
|
||||
}
|
||||
return TileJob{}, false, err
|
||||
}
|
||||
return job, true, nil
|
||||
}
|
||||
|
||||
// Callback delivery for tile jobs. A finished job can notify the submitting
|
||||
// system; failures back off on the same schedule as push callbacks.
|
||||
const tileCallbackMaxAttempts = 3
|
||||
|
||||
func (s *Store) queueTileJobCallback(ctx context.Context, jobID string) {
|
||||
if _, err := s.db.ExecContext(ctx, `
|
||||
UPDATE tile_jobs
|
||||
SET callback_status = ?, callback_attempts = 0, callback_next_attempt_at = ?, callback_message = ''
|
||||
WHERE id = ?
|
||||
`, pushStatusRetrying, toRFC3339(nowUTC()), jobID); err != nil {
|
||||
log.Printf("queue tile job callback %s: %v", jobID, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Store) dispatchTileJobCallbacks(ctx context.Context) error {
|
||||
rows, err := s.db.QueryContext(ctx, "SELECT "+tileJobColumns+`
|
||||
FROM tile_jobs
|
||||
WHERE callback_url <> '' AND callback_status = ?
|
||||
AND (callback_next_attempt_at IS NULL OR callback_next_attempt_at <= ?)
|
||||
ORDER BY callback_next_attempt_at
|
||||
LIMIT 5
|
||||
`, pushStatusRetrying, toRFC3339(nowUTC()))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var jobs []TileJob
|
||||
for rows.Next() {
|
||||
job, err := scanTileJob(rows)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
jobs = append(jobs, job)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, job := range jobs {
|
||||
s.attemptTileJobCallback(job, job.CallbackAttempts+1)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) attemptTileJobCallback(job TileJob, attempt int) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), pushCallbackTimeout)
|
||||
defer cancel()
|
||||
|
||||
payload := map[string]any{
|
||||
"event": "tile.job.finished",
|
||||
"job": job,
|
||||
"delivered": toRFC3339(nowUTC()),
|
||||
}
|
||||
_, _, err := s.postJSON(ctx, http.MethodPost, job.CallbackURL, nil, payload)
|
||||
message := ""
|
||||
if err != nil {
|
||||
message = err.Error()
|
||||
}
|
||||
|
||||
if err != nil && attempt < tileCallbackMaxAttempts {
|
||||
next := nowUTC().Add(pushRetryDelay(attempt))
|
||||
if _, updateErr := s.db.ExecContext(context.Background(), `
|
||||
UPDATE tile_jobs
|
||||
SET callback_status = ?, callback_attempts = ?, callback_next_attempt_at = ?, callback_message = ?
|
||||
WHERE id = ?
|
||||
`, pushStatusRetrying, attempt, toRFC3339(next), message, job.ID); updateErr != nil {
|
||||
log.Printf("reschedule tile job callback %s: %v", job.ID, updateErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
status := pushStatusSucceeded
|
||||
if err != nil {
|
||||
status = pushStatusFailed
|
||||
}
|
||||
if _, updateErr := s.db.ExecContext(context.Background(), `
|
||||
UPDATE tile_jobs
|
||||
SET callback_status = ?, callback_attempts = ?, callback_next_attempt_at = NULL, callback_message = ?
|
||||
WHERE id = ?
|
||||
`, status, attempt, message, job.ID); updateErr != nil {
|
||||
log.Printf("record tile job callback %s: %v", job.ID, updateErr)
|
||||
}
|
||||
}
|
||||
|
||||
// LayerSource records one raster that belongs to a layer. The layer's tile
|
||||
// directory is the union of its sources, so adding a source and re-slicing
|
||||
// accumulates imagery instead of replacing the layer.
|
||||
type LayerSource struct {
|
||||
ID string `json:"id"`
|
||||
TargetCode string `json:"target_code"`
|
||||
SourcePath string `json:"source_path"`
|
||||
AddedAt time.Time `json:"added_at"`
|
||||
}
|
||||
|
||||
// AddLayerSources records sources against a layer. Re-adding a known path is a
|
||||
// no-op, which keeps repeated submissions idempotent.
|
||||
func (s *Store) AddLayerSources(ctx context.Context, targetCode string, paths []string) error {
|
||||
code := normalizeBasemapCode(targetCode)
|
||||
if code == "" {
|
||||
return errors.New("target code is required")
|
||||
}
|
||||
for _, path := range paths {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := s.db.ExecContext(ctx, `
|
||||
INSERT INTO layer_sources (id, target_code, source_path, added_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(target_code, source_path) DO NOTHING
|
||||
`, uid.New(), code, path, toRFC3339(nowUTC())); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListLayerSources returns a layer's source inventory in the order it was added.
|
||||
func (s *Store) ListLayerSources(ctx context.Context, targetCode string) ([]LayerSource, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT id, target_code, source_path, added_at
|
||||
FROM layer_sources
|
||||
WHERE target_code = ?
|
||||
ORDER BY added_at
|
||||
`, normalizeBasemapCode(targetCode))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var items []LayerSource
|
||||
for rows.Next() {
|
||||
var item LayerSource
|
||||
var addedAt string
|
||||
if err := rows.Scan(&item.ID, &item.TargetCode, &item.SourcePath, &addedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item.AddedAt = parseTime(addedAt)
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
// ListLayerInventories returns every layer with its sources, ordered by layer
|
||||
// code so the console renders stably.
|
||||
func (s *Store) ListLayerInventories(ctx context.Context) ([]LayerInventory, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT id, target_code, source_path, added_at
|
||||
FROM layer_sources
|
||||
ORDER BY target_code, added_at
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
byCode := map[string][]LayerSource{}
|
||||
var codes []string
|
||||
for rows.Next() {
|
||||
var item LayerSource
|
||||
var addedAt string
|
||||
if err := rows.Scan(&item.ID, &item.TargetCode, &item.SourcePath, &addedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item.AddedAt = parseTime(addedAt)
|
||||
if _, seen := byCode[item.TargetCode]; !seen {
|
||||
codes = append(codes, item.TargetCode)
|
||||
}
|
||||
byCode[item.TargetCode] = append(byCode[item.TargetCode], item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
items := make([]LayerInventory, 0, len(codes))
|
||||
for _, code := range codes {
|
||||
item := LayerInventory{TargetCode: code, Sources: byCode[code], ZoomMin: 0, ZoomMax: 18, TileFormat: "webp"}
|
||||
if minZoom, maxZoom, format, ok := s.layerPublishedRange(ctx, code); ok {
|
||||
item.ZoomMin, item.ZoomMax, item.TileFormat = minZoom, maxZoom, format
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// layerPublishedRange returns the zoom range and format the layer is currently
|
||||
// published at, so a rebuild keeps the same range instead of guessing.
|
||||
func (s *Store) layerPublishedRange(ctx context.Context, targetCode string) (int, int, string, bool) {
|
||||
var minZoom, maxZoom int
|
||||
var format string
|
||||
err := s.db.QueryRowContext(ctx, `
|
||||
SELECT v.min_zoom, v.max_zoom, v.tile_format
|
||||
FROM basemap_versions v
|
||||
JOIN basemaps b ON b.id = v.basemap_id
|
||||
WHERE b.code = ?
|
||||
ORDER BY v.is_default DESC, v.version
|
||||
LIMIT 1
|
||||
`, normalizeBasemapCode(targetCode)).Scan(&minZoom, &maxZoom, &format)
|
||||
if err != nil {
|
||||
return 0, 0, "", false
|
||||
}
|
||||
return minZoom, maxZoom, format, true
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package basemap
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
)
|
||||
|
||||
func scanVectorAsset(scanner interface {
|
||||
Scan(dest ...any) error
|
||||
}) (VectorAsset, error) {
|
||||
var item VectorAsset
|
||||
var metadataJSON string
|
||||
var createdAt string
|
||||
var updatedAt string
|
||||
if err := scanner.Scan(
|
||||
&item.ID,
|
||||
&item.Code,
|
||||
&item.Name,
|
||||
&item.Format,
|
||||
&item.Status,
|
||||
&item.Description,
|
||||
&item.FilePath,
|
||||
&item.FileName,
|
||||
&item.FileSize,
|
||||
&item.Checksum,
|
||||
&item.DownloadURL,
|
||||
&metadataJSON,
|
||||
&createdAt,
|
||||
&updatedAt,
|
||||
); err != nil {
|
||||
return VectorAsset{}, err
|
||||
}
|
||||
item.CreatedAt = parseTime(createdAt)
|
||||
item.UpdatedAt = parseTime(updatedAt)
|
||||
if metadataJSON != "" {
|
||||
_ = readMaybeJSON(metadataJSON, &item.Metadata)
|
||||
}
|
||||
if item.Metadata == nil {
|
||||
item.Metadata = map[string]any{}
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func scanVectorTokenGrant(scanner interface {
|
||||
Scan(dest ...any) error
|
||||
}) (VectorTokenGrant, error) {
|
||||
var item VectorTokenGrant
|
||||
var createdAt string
|
||||
if err := scanner.Scan(
|
||||
&item.ID,
|
||||
&item.TokenID,
|
||||
&item.VectorID,
|
||||
&item.VectorCode,
|
||||
&item.Permission,
|
||||
&createdAt,
|
||||
); err != nil {
|
||||
return VectorTokenGrant{}, err
|
||||
}
|
||||
item.CreatedAt = parseTime(createdAt)
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func scanVectorPushRecord(scanner interface {
|
||||
Scan(dest ...any) error
|
||||
}) (VectorPushRecord, error) {
|
||||
var item VectorPushRecord
|
||||
var requestJSON string
|
||||
var responseStatus sql.NullInt64
|
||||
var nextAttemptAt sql.NullString
|
||||
var finishedAt sql.NullString
|
||||
var pushedAt string
|
||||
if err := scanner.Scan(
|
||||
&item.ID,
|
||||
&item.TargetSystemID,
|
||||
&item.TargetSystemCode,
|
||||
&item.VectorAssetID,
|
||||
&item.VectorCode,
|
||||
&item.Status,
|
||||
&requestJSON,
|
||||
&responseStatus,
|
||||
&item.ResponseBody,
|
||||
&item.ErrorMessage,
|
||||
&item.AttemptCount,
|
||||
&nextAttemptAt,
|
||||
&pushedAt,
|
||||
&finishedAt,
|
||||
); err != nil {
|
||||
return VectorPushRecord{}, err
|
||||
}
|
||||
if requestJSON != "" {
|
||||
_ = readMaybeJSON(requestJSON, &item.Request)
|
||||
}
|
||||
if item.Request == nil {
|
||||
item.Request = map[string]any{}
|
||||
}
|
||||
if responseStatus.Valid {
|
||||
value := int(responseStatus.Int64)
|
||||
item.ResponseStatus = &value
|
||||
}
|
||||
if nextAttemptAt.Valid {
|
||||
value := parseTime(nextAttemptAt.String)
|
||||
item.NextAttemptAt = &value
|
||||
}
|
||||
item.PushedAt = parseTime(pushedAt)
|
||||
if finishedAt.Valid {
|
||||
value := parseTime(finishedAt.String)
|
||||
item.FinishedAt = &value
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
@@ -0,0 +1,768 @@
|
||||
package basemap
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"tile-server/api-go/internal/uid"
|
||||
)
|
||||
|
||||
type VectorDownloadDescriptor struct {
|
||||
Asset VectorAsset
|
||||
FilePath string
|
||||
ContentType string
|
||||
FileName string
|
||||
Zipped bool
|
||||
PackageFiles []string
|
||||
}
|
||||
|
||||
func (s *Store) ListVectorAssets(ctx context.Context) ([]VectorAsset, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT id, code, name, format, status, description, file_path, file_name, file_size, checksum, download_url, metadata_json, created_at, updated_at
|
||||
FROM vector_assets
|
||||
ORDER BY code
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var items []VectorAsset
|
||||
for rows.Next() {
|
||||
item, err := scanVectorAsset(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) GetVectorAssetByCode(ctx context.Context, code string) (VectorAsset, error) {
|
||||
row := s.db.QueryRowContext(ctx, `
|
||||
SELECT id, code, name, format, status, description, file_path, file_name, file_size, checksum, download_url, metadata_json, created_at, updated_at
|
||||
FROM vector_assets
|
||||
WHERE code = ?
|
||||
`, normalizeBasemapCode(code))
|
||||
item, err := scanVectorAsset(row)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return VectorAsset{}, fmt.Errorf("vector asset %q not found", code)
|
||||
}
|
||||
return VectorAsset{}, err
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (s *Store) GetVectorAssetByRef(ctx context.Context, ref string) (VectorAsset, error) {
|
||||
cleaned, err := cleanVectorAssetRef(ref)
|
||||
if err != nil {
|
||||
return VectorAsset{}, err
|
||||
}
|
||||
|
||||
// The compatibility routes resolve by ref on every request, so try the
|
||||
// indexed single-row lookups before falling back to a full table scan.
|
||||
if item, ok, err := s.getVectorAssetByLookup(ctx, "code = ?", normalizeBasemapCode(cleaned)); err != nil {
|
||||
return VectorAsset{}, err
|
||||
} else if ok {
|
||||
return item, nil
|
||||
}
|
||||
if item, ok, err := s.getVectorAssetByLookup(ctx, "file_name = ? COLLATE NOCASE", path.Base(cleaned)); err != nil {
|
||||
return VectorAsset{}, err
|
||||
} else if ok {
|
||||
return item, nil
|
||||
}
|
||||
|
||||
items, err := s.ListVectorAssets(ctx)
|
||||
if err != nil {
|
||||
return VectorAsset{}, err
|
||||
}
|
||||
for _, item := range items {
|
||||
if vectorAssetMatchesRef(item, cleaned) {
|
||||
return item, nil
|
||||
}
|
||||
}
|
||||
return VectorAsset{}, fmt.Errorf("vector asset %q not found", ref)
|
||||
}
|
||||
|
||||
// getVectorAssetByLookup runs a single indexed lookup. where must be a trusted
|
||||
// internal literal, never user input.
|
||||
func (s *Store) getVectorAssetByLookup(ctx context.Context, where, value string) (VectorAsset, bool, error) {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return VectorAsset{}, false, nil
|
||||
}
|
||||
query := fmt.Sprintf(`
|
||||
SELECT id, code, name, format, status, description, file_path, file_name, file_size, checksum, download_url, metadata_json, created_at, updated_at
|
||||
FROM vector_assets
|
||||
WHERE %s
|
||||
LIMIT 1
|
||||
`, where)
|
||||
item, err := scanVectorAsset(s.db.QueryRowContext(ctx, query, value))
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return VectorAsset{}, false, nil
|
||||
}
|
||||
return VectorAsset{}, false, err
|
||||
}
|
||||
return item, true, nil
|
||||
}
|
||||
|
||||
func (s *Store) RunVectorScan(ctx context.Context) (map[string]any, error) {
|
||||
root := strings.TrimSpace(s.cfg.VectorScanRoot)
|
||||
if root == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if !s.scanMu.TryLock() {
|
||||
return nil, ErrScanInProgress
|
||||
}
|
||||
defer s.scanMu.Unlock()
|
||||
|
||||
if err := os.MkdirAll(root, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
hashCache, err := s.loadVectorHashCache(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
items, err := scanVectorRoot(root, hashCache)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
existing, err := loadExistingVectors(ctx, tx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
found := map[string]struct{}{}
|
||||
added, updated, unchanged := 0, 0, 0
|
||||
now := nowUTC()
|
||||
for _, item := range items {
|
||||
found[item.Code] = struct{}{}
|
||||
if prev, ok := existing[item.Code]; ok {
|
||||
if s.scannedVectorMatches(item, prev) {
|
||||
unchanged++
|
||||
} else {
|
||||
updated++
|
||||
}
|
||||
} else {
|
||||
added++
|
||||
}
|
||||
if err := s.upsertVectorAssetTx(ctx, tx, item, now); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
removed := 0
|
||||
for code := range existing {
|
||||
if _, ok := found[code]; ok {
|
||||
continue
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `DELETE FROM vector_assets WHERE code = ?`, code); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
removed++
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{
|
||||
"scan_root": root,
|
||||
"scanned_count": len(items),
|
||||
"added_count": added,
|
||||
"updated_count": updated,
|
||||
"unchanged_count": unchanged,
|
||||
"removed_count": removed,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// existingVector is a snapshot of a stored asset, used to tell a real update
|
||||
// from a no-op rescan.
|
||||
type existingVector struct {
|
||||
Name string
|
||||
Format string
|
||||
Status string
|
||||
Description string
|
||||
FilePath string
|
||||
FileName string
|
||||
FileSize int64
|
||||
Checksum string
|
||||
DownloadURL string
|
||||
MetadataJSON string
|
||||
}
|
||||
|
||||
func loadExistingVectors(ctx context.Context, tx *sql.Tx) (map[string]existingVector, error) {
|
||||
rows, err := tx.QueryContext(ctx, `
|
||||
SELECT code, name, format, status, description, file_path, file_name, file_size, checksum, download_url, metadata_json
|
||||
FROM vector_assets
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
items := map[string]existingVector{}
|
||||
for rows.Next() {
|
||||
var code string
|
||||
var item existingVector
|
||||
if err := rows.Scan(
|
||||
&code,
|
||||
&item.Name,
|
||||
&item.Format,
|
||||
&item.Status,
|
||||
&item.Description,
|
||||
&item.FilePath,
|
||||
&item.FileName,
|
||||
&item.FileSize,
|
||||
&item.Checksum,
|
||||
&item.DownloadURL,
|
||||
&item.MetadataJSON,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items[code] = item
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) scannedVectorMatches(item scannedVector, prev existingVector) bool {
|
||||
return item.Name == prev.Name &&
|
||||
item.Format == prev.Format &&
|
||||
item.Status == prev.Status &&
|
||||
item.Description == prev.Description &&
|
||||
item.FilePath == prev.FilePath &&
|
||||
item.FileName == prev.FileName &&
|
||||
item.FileSize == prev.FileSize &&
|
||||
item.Checksum == prev.Checksum &&
|
||||
s.buildVectorDownloadURL(item.Code) == prev.DownloadURL &&
|
||||
writeJSON(item.Metadata) == prev.MetadataJSON
|
||||
}
|
||||
|
||||
// vectorHashEntry lets an unchanged file reuse the checksum recorded by the
|
||||
// previous scan instead of re-reading the whole file on every startup.
|
||||
type vectorHashEntry struct {
|
||||
Size int64
|
||||
ModTime int64
|
||||
Checksum string
|
||||
}
|
||||
|
||||
func (s *Store) loadVectorHashCache(ctx context.Context) (map[string]vectorHashEntry, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT file_path, checksum, metadata_json
|
||||
FROM vector_assets
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
cache := map[string]vectorHashEntry{}
|
||||
for rows.Next() {
|
||||
var filePath, checksum, metadataJSON string
|
||||
if err := rows.Scan(&filePath, &checksum, &metadataJSON); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(checksum) == "" {
|
||||
continue
|
||||
}
|
||||
var meta struct {
|
||||
HashSize int64 `json:"hash_size"`
|
||||
HashMTime int64 `json:"hash_mtime"`
|
||||
}
|
||||
_ = readMaybeJSON(metadataJSON, &meta)
|
||||
if meta.HashSize == 0 && meta.HashMTime == 0 {
|
||||
continue
|
||||
}
|
||||
cache[filePath] = vectorHashEntry{Size: meta.HashSize, ModTime: meta.HashMTime, Checksum: checksum}
|
||||
}
|
||||
return cache, rows.Err()
|
||||
}
|
||||
|
||||
type scannedVector struct {
|
||||
Code string
|
||||
Name string
|
||||
Format string
|
||||
Status string
|
||||
Description string
|
||||
FilePath string
|
||||
FileName string
|
||||
FileSize int64
|
||||
Checksum string
|
||||
Metadata map[string]any
|
||||
}
|
||||
|
||||
func scanVectorRoot(root string, hashCache map[string]vectorHashEntry) ([]scannedVector, error) {
|
||||
shapefiles := map[string]string{}
|
||||
var items []scannedVector
|
||||
err := filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
ext := strings.ToLower(filepath.Ext(entry.Name()))
|
||||
switch ext {
|
||||
case ".geojson", ".json", ".zip":
|
||||
item, err := buildScannedVector(root, path, ext, hashCache)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
items = append(items, item)
|
||||
case ".shp":
|
||||
base := strings.TrimSuffix(path, filepath.Ext(path))
|
||||
shapefiles[base] = path
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, base := range sortedKeysStringMap(shapefiles) {
|
||||
path := shapefiles[base]
|
||||
item, err := buildScannedVector(root, path, ".shp", hashCache)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
|
||||
items = disambiguateVectorCodes(items)
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
return items[i].Code < items[j].Code
|
||||
})
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// disambiguateVectorCodes keeps both assets when two different relative paths
|
||||
// normalize to the same code (for example "a/b.geojson" and "a-b.geojson").
|
||||
// The suffix comes from the relative path, so it is stable across scans.
|
||||
func disambiguateVectorCodes(items []scannedVector) []scannedVector {
|
||||
byCode := map[string][]int{}
|
||||
for i, item := range items {
|
||||
byCode[item.Code] = append(byCode[item.Code], i)
|
||||
}
|
||||
for code, indexes := range byCode {
|
||||
if len(indexes) < 2 {
|
||||
continue
|
||||
}
|
||||
for _, index := range indexes {
|
||||
sum := sha256.Sum256([]byte(scannedVectorRelPath(items[index])))
|
||||
items[index].Code = code + "-" + hex.EncodeToString(sum[:4])
|
||||
}
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func scannedVectorRelPath(item scannedVector) string {
|
||||
if item.Metadata == nil {
|
||||
return ""
|
||||
}
|
||||
value, ok := item.Metadata["relative_path"].(string)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func sortedKeysStringMap(values map[string]string) []string {
|
||||
keys := make([]string, 0, len(values))
|
||||
for key := range values {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
func buildScannedVector(root, path, ext string, hashCache map[string]vectorHashEntry) (scannedVector, error) {
|
||||
absPath, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return scannedVector{}, err
|
||||
}
|
||||
info, err := os.Stat(absPath)
|
||||
if err != nil {
|
||||
return scannedVector{}, err
|
||||
}
|
||||
rel, err := filepath.Rel(root, absPath)
|
||||
if err != nil {
|
||||
return scannedVector{}, err
|
||||
}
|
||||
code := vectorCodeFromRel(rel, ext)
|
||||
name := strings.TrimSuffix(filepath.Base(absPath), filepath.Ext(absPath))
|
||||
format := vectorFormatFromExt(ext)
|
||||
fileName := filepath.Base(absPath)
|
||||
fileSize := info.Size()
|
||||
|
||||
hashSize, hashMTime, err := vectorHashStamp(absPath, ext)
|
||||
if err != nil {
|
||||
return scannedVector{}, err
|
||||
}
|
||||
checksum := ""
|
||||
if cached, ok := hashCache[absPath]; ok &&
|
||||
(hashSize > 0 || hashMTime > 0) &&
|
||||
cached.Size == hashSize && cached.ModTime == hashMTime {
|
||||
checksum = cached.Checksum
|
||||
}
|
||||
if checksum == "" {
|
||||
checksum, err = hashVector(absPath, ext)
|
||||
if err != nil {
|
||||
return scannedVector{}, err
|
||||
}
|
||||
}
|
||||
|
||||
if format == "shapefile" {
|
||||
fileName = name + ".zip"
|
||||
}
|
||||
return scannedVector{
|
||||
Code: code,
|
||||
Name: name,
|
||||
Format: format,
|
||||
Status: "ready",
|
||||
FilePath: absPath,
|
||||
FileName: fileName,
|
||||
FileSize: fileSize,
|
||||
Checksum: checksum,
|
||||
Metadata: map[string]any{
|
||||
"relative_path": filepath.ToSlash(rel),
|
||||
"hash_size": hashSize,
|
||||
"hash_mtime": hashMTime,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// vectorHashStamp summarizes the inputs that feed the checksum, so an unchanged
|
||||
// file can reuse the previous checksum instead of being rehashed every startup.
|
||||
// Shapefiles fold in every sidecar so a changed .dbf/.shx also invalidates it.
|
||||
func vectorHashStamp(absPath, ext string) (int64, int64, error) {
|
||||
if strings.EqualFold(ext, ".shp") {
|
||||
var totalSize, maxModTime int64
|
||||
for _, sibling := range shapefilePackageFiles(absPath) {
|
||||
info, err := os.Stat(sibling)
|
||||
if err != nil || info.IsDir() {
|
||||
continue
|
||||
}
|
||||
totalSize += info.Size()
|
||||
if modTime := info.ModTime().Unix(); modTime > maxModTime {
|
||||
maxModTime = modTime
|
||||
}
|
||||
}
|
||||
return totalSize, maxModTime, nil
|
||||
}
|
||||
info, err := os.Stat(absPath)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
return info.Size(), info.ModTime().Unix(), nil
|
||||
}
|
||||
|
||||
func vectorCodeFromRel(rel, ext string) string {
|
||||
rel = filepath.ToSlash(rel)
|
||||
rel = strings.TrimSuffix(rel, strings.ToLower(filepath.Ext(rel)))
|
||||
rel = strings.TrimSuffix(rel, filepath.Ext(rel))
|
||||
rel = strings.ReplaceAll(rel, "/", "-")
|
||||
code := normalizeBasemapCode(rel)
|
||||
if code != "" {
|
||||
return code
|
||||
}
|
||||
sum := sha256.Sum256([]byte(rel))
|
||||
return "vector-" + hex.EncodeToString(sum[:6])
|
||||
}
|
||||
|
||||
func vectorFormatFromExt(ext string) string {
|
||||
switch strings.ToLower(ext) {
|
||||
case ".geojson", ".json":
|
||||
return "geojson"
|
||||
case ".shp":
|
||||
return "shapefile"
|
||||
case ".zip":
|
||||
return "shpzip"
|
||||
default:
|
||||
return strings.TrimPrefix(strings.ToLower(ext), ".")
|
||||
}
|
||||
}
|
||||
|
||||
func hashVector(path, ext string) (string, error) {
|
||||
sum := sha256.New()
|
||||
if strings.EqualFold(ext, ".shp") {
|
||||
for _, sibling := range shapefilePackageFiles(path) {
|
||||
info, err := os.Stat(sibling)
|
||||
if err != nil || info.IsDir() {
|
||||
continue
|
||||
}
|
||||
if _, err := io.WriteString(sum, filepath.Base(sibling)); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := hashFileInto(sum, sibling); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
return hex.EncodeToString(sum.Sum(nil)), nil
|
||||
}
|
||||
if err := hashFileInto(sum, path); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(sum.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func hashFileInto(writer io.Writer, path string) error {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
_, err = io.Copy(writer, file)
|
||||
return err
|
||||
}
|
||||
|
||||
func shapefilePackageFiles(path string) []string {
|
||||
base := strings.TrimSuffix(path, filepath.Ext(path))
|
||||
candidates := []string{".shp", ".dbf", ".shx", ".prj", ".cpg", ".qix", ".sbn", ".sbx"}
|
||||
files := make([]string, 0, len(candidates))
|
||||
for _, ext := range candidates {
|
||||
file := base + ext
|
||||
if _, err := os.Stat(file); err == nil {
|
||||
files = append(files, file)
|
||||
}
|
||||
}
|
||||
sort.Strings(files)
|
||||
return files
|
||||
}
|
||||
|
||||
func (s *Store) upsertVectorAssetTx(ctx context.Context, tx *sql.Tx, item scannedVector, now time.Time) error {
|
||||
id := uid.Deterministic("vector-asset", item.Code)
|
||||
_, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO vector_assets (
|
||||
id, code, name, format, status, description, file_path, file_name, file_size, checksum, download_url, metadata_json, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(code) DO UPDATE SET
|
||||
name = excluded.name,
|
||||
format = excluded.format,
|
||||
status = excluded.status,
|
||||
description = excluded.description,
|
||||
file_path = excluded.file_path,
|
||||
file_name = excluded.file_name,
|
||||
file_size = excluded.file_size,
|
||||
checksum = excluded.checksum,
|
||||
download_url = excluded.download_url,
|
||||
metadata_json = excluded.metadata_json,
|
||||
updated_at = excluded.updated_at
|
||||
`, id, item.Code, item.Name, item.Format, item.Status, item.Description, item.FilePath, item.FileName, item.FileSize, item.Checksum, s.buildVectorDownloadURL(item.Code), writeJSON(item.Metadata), toRFC3339(now), toRFC3339(now))
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) buildVectorDownloadURL(code string) string {
|
||||
base := trimURL(s.cfg.VectorBaseURL)
|
||||
if base == "" {
|
||||
base = trimURL(s.cfg.APIBaseURL)
|
||||
}
|
||||
return fmt.Sprintf("%s/vectors/%s/download", base, code)
|
||||
}
|
||||
|
||||
func (auth TokenAuth) CanReadVector(code string) bool {
|
||||
group, ok := auth.VectorPermissions[readPermission]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
_, ok = group[normalizeBasemapCode(code)]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (s *Store) FilterVectors(auth TokenAuth, items []VectorAsset) []VectorAsset {
|
||||
filtered := make([]VectorAsset, 0, len(items))
|
||||
for _, item := range items {
|
||||
if auth.CanReadVector(item.Code) {
|
||||
filtered = append(filtered, item)
|
||||
}
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
func (s *Store) ResolveVectorDownload(ctx context.Context, auth TokenAuth, code string) (VectorDownloadDescriptor, error) {
|
||||
code = normalizeBasemapCode(code)
|
||||
if !auth.CanReadVector(code) {
|
||||
return VectorDownloadDescriptor{}, errors.New("token has no access to this vector asset")
|
||||
}
|
||||
item, err := s.GetVectorAssetByCode(ctx, code)
|
||||
if err != nil {
|
||||
return VectorDownloadDescriptor{}, err
|
||||
}
|
||||
return buildVectorDownloadDescriptor(item)
|
||||
}
|
||||
|
||||
func (s *Store) ResolveVectorDownloadByRef(ctx context.Context, auth TokenAuth, ref string, allowedFormats ...string) (VectorDownloadDescriptor, error) {
|
||||
item, err := s.GetVectorAssetByRef(ctx, ref)
|
||||
if err != nil {
|
||||
return VectorDownloadDescriptor{}, err
|
||||
}
|
||||
if !auth.CanReadVector(item.Code) {
|
||||
return VectorDownloadDescriptor{}, errors.New("token has no access to this vector asset")
|
||||
}
|
||||
if !vectorFormatAllowed(item.Format, allowedFormats) {
|
||||
return VectorDownloadDescriptor{}, fmt.Errorf("vector asset %q is not available in requested format", ref)
|
||||
}
|
||||
return buildVectorDownloadDescriptor(item)
|
||||
}
|
||||
|
||||
func buildVectorDownloadDescriptor(item VectorAsset) (VectorDownloadDescriptor, error) {
|
||||
switch item.Format {
|
||||
case "geojson":
|
||||
contentType := mime.TypeByExtension(filepath.Ext(item.FilePath))
|
||||
if contentType == "" {
|
||||
contentType = "application/geo+json"
|
||||
}
|
||||
return VectorDownloadDescriptor{
|
||||
Asset: item,
|
||||
FilePath: item.FilePath,
|
||||
FileName: item.FileName,
|
||||
ContentType: contentType,
|
||||
}, nil
|
||||
case "shapefile":
|
||||
files := shapefilePackageFiles(item.FilePath)
|
||||
if len(files) == 0 {
|
||||
return VectorDownloadDescriptor{}, errors.New("shapefile package is missing sidecar files")
|
||||
}
|
||||
return VectorDownloadDescriptor{
|
||||
Asset: item,
|
||||
FileName: item.FileName,
|
||||
ContentType: "application/zip",
|
||||
Zipped: true,
|
||||
PackageFiles: files,
|
||||
}, nil
|
||||
case "shpzip":
|
||||
return VectorDownloadDescriptor{
|
||||
Asset: item,
|
||||
FilePath: item.FilePath,
|
||||
FileName: item.FileName,
|
||||
ContentType: "application/zip",
|
||||
}, nil
|
||||
default:
|
||||
contentType := mime.TypeByExtension(filepath.Ext(item.FilePath))
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
return VectorDownloadDescriptor{
|
||||
Asset: item,
|
||||
FilePath: item.FilePath,
|
||||
FileName: item.FileName,
|
||||
ContentType: contentType,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func cleanVectorAssetRef(value string) (string, error) {
|
||||
value = strings.TrimSpace(strings.ReplaceAll(value, "\\", "/"))
|
||||
value = strings.TrimPrefix(value, "/")
|
||||
if value == "" {
|
||||
return "", errors.New("vector asset path is required")
|
||||
}
|
||||
cleaned := path.Clean(value)
|
||||
if cleaned == "." || cleaned == "/" || cleaned == ".." || strings.HasPrefix(cleaned, "../") {
|
||||
return "", errors.New("invalid vector asset path")
|
||||
}
|
||||
return strings.TrimPrefix(cleaned, "./"), nil
|
||||
}
|
||||
|
||||
func vectorAssetMatchesRef(item VectorAsset, ref string) bool {
|
||||
for _, candidate := range vectorAssetRefs(item) {
|
||||
if strings.EqualFold(candidate, ref) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func vectorAssetRefs(item VectorAsset) []string {
|
||||
values := []string{item.Code}
|
||||
if relativePath := vectorRelativePath(item); relativePath != "" {
|
||||
values = append(values, relativePath)
|
||||
}
|
||||
if item.FileName != "" {
|
||||
values = append(values, strings.ReplaceAll(item.FileName, "\\", "/"))
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func vectorRelativePath(item VectorAsset) string {
|
||||
if item.Metadata == nil {
|
||||
return ""
|
||||
}
|
||||
value, ok := item.Metadata["relative_path"].(string)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
cleaned, err := cleanVectorAssetRef(value)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return cleaned
|
||||
}
|
||||
|
||||
func vectorFormatAllowed(format string, allowed []string) bool {
|
||||
if len(allowed) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, value := range allowed {
|
||||
if strings.EqualFold(strings.TrimSpace(value), format) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func WriteVectorZip(w http.ResponseWriter, descriptor VectorDownloadDescriptor) error {
|
||||
archive := zip.NewWriter(w)
|
||||
for _, path := range descriptor.PackageFiles {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
file.Close()
|
||||
return err
|
||||
}
|
||||
header, err := zip.FileInfoHeader(info)
|
||||
if err != nil {
|
||||
file.Close()
|
||||
return err
|
||||
}
|
||||
header.Name = filepath.Base(path)
|
||||
header.Method = zip.Deflate
|
||||
writer, err := archive.CreateHeader(header)
|
||||
if err != nil {
|
||||
file.Close()
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(writer, file); err != nil {
|
||||
file.Close()
|
||||
return err
|
||||
}
|
||||
file.Close()
|
||||
}
|
||||
return archive.Close()
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package basemap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// pushRetryInterval is how often due retries are looked for.
|
||||
pushRetryInterval = 15 * time.Second
|
||||
// tileJobInterval is how often the queue is checked for new jobs.
|
||||
tileJobInterval = 2 * time.Second
|
||||
)
|
||||
|
||||
// StartBackgroundWorkers launches the periodic maintenance loops: resource
|
||||
// health inspection, push retries and the tile slicing queue. Close stops them.
|
||||
// Calling it more than once has no additional effect.
|
||||
func (s *Store) StartBackgroundWorkers() {
|
||||
s.bgOnce.Do(func() {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
s.bgCtx = ctx
|
||||
s.bgCancel = cancel
|
||||
|
||||
s.bgWG.Add(1)
|
||||
go func() {
|
||||
defer s.bgWG.Done()
|
||||
s.healthLoop(ctx)
|
||||
}()
|
||||
|
||||
s.bgWG.Add(1)
|
||||
go func() {
|
||||
defer s.bgWG.Done()
|
||||
s.pushRetryLoop(ctx)
|
||||
}()
|
||||
|
||||
s.bgWG.Add(1)
|
||||
go func() {
|
||||
defer s.bgWG.Done()
|
||||
s.tileJobLoop(ctx)
|
||||
}()
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Store) healthLoop(ctx context.Context) {
|
||||
// Prime the catalog immediately so a resource that vanished since the last
|
||||
// scan shows up without waiting a full interval.
|
||||
if _, err := s.CheckHealth(ctx); err != nil && ctx.Err() == nil {
|
||||
log.Printf("health check failed: %v", err)
|
||||
}
|
||||
interval := s.cfg.HealthCheckInterval
|
||||
if interval <= 0 {
|
||||
return
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if _, err := s.CheckHealth(ctx); err != nil && ctx.Err() == nil {
|
||||
log.Printf("health check failed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Store) pushRetryLoop(ctx context.Context) {
|
||||
ticker := time.NewTicker(pushRetryInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := s.dispatchPushRetries(ctx); err != nil && ctx.Err() == nil {
|
||||
log.Printf("push retry dispatch failed: %v", err)
|
||||
}
|
||||
if err := s.dispatchTileJobCallbacks(ctx); err != nil && ctx.Err() == nil {
|
||||
log.Printf("tile job callback dispatch failed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Store) tileJobLoop(ctx context.Context) {
|
||||
ticker := time.NewTicker(tileJobInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
s.dispatchTileJobs(ctx)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user