Initial import of map-asset-gateway
This commit is contained in:
@@ -0,0 +1,578 @@
|
||||
package basemap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"map-asset-gateway/api-go/internal/uid"
|
||||
)
|
||||
|
||||
type scannedVersion struct {
|
||||
BasemapCode string
|
||||
Name string
|
||||
Type string
|
||||
Status string
|
||||
Description string
|
||||
Version string
|
||||
Manifest string
|
||||
TileRoot string
|
||||
TileFormat string
|
||||
TileScheme string
|
||||
MinZoom int
|
||||
MaxZoom int
|
||||
BBox []float64
|
||||
Attribution string
|
||||
Metadata map[string]any
|
||||
IsDefault bool
|
||||
}
|
||||
|
||||
func (s *Store) RunScan(ctx context.Context, sourceCode string) (ScanRun, error) {
|
||||
source, err := s.GetScanSourceByCode(ctx, sourceCode)
|
||||
if err != nil {
|
||||
return ScanRun{}, err
|
||||
}
|
||||
if !source.Enabled {
|
||||
return ScanRun{}, fmt.Errorf("scan source %q is disabled", sourceCode)
|
||||
}
|
||||
|
||||
startedAt := nowUTC()
|
||||
runID := uid.New()
|
||||
if _, err := s.db.ExecContext(ctx, `
|
||||
INSERT INTO scan_runs (
|
||||
id, scan_source_id, status, scanned_count, added_count, updated_count, removed_count, summary_json, started_at, finished_at
|
||||
) VALUES (?, ?, 'running', 0, 0, 0, 0, '', ?, ?)
|
||||
`, runID, source.ID, toRFC3339(startedAt), toRFC3339(startedAt)); err != nil {
|
||||
return ScanRun{}, fmt.Errorf("create scan run: %w", err)
|
||||
}
|
||||
|
||||
run, finalErr := s.runScan(ctx, runID, source, startedAt)
|
||||
return run, finalErr
|
||||
}
|
||||
|
||||
func (s *Store) runScan(ctx context.Context, runID string, source ScanSource, startedAt time.Time) (ScanRun, error) {
|
||||
rootInfo, err := os.Stat(source.RootPath)
|
||||
if err != nil {
|
||||
return s.finishScanRun(ctx, runID, source, startedAt, "failed", nil, 0, 0, 0, 0, fmt.Errorf("stat scan root: %w", err))
|
||||
}
|
||||
if !rootInfo.IsDir() {
|
||||
return s.finishScanRun(ctx, runID, source, startedAt, "failed", nil, 0, 0, 0, 0, fmt.Errorf("scan root %s is not a directory", source.RootPath))
|
||||
}
|
||||
|
||||
manifestItems, coveredRoots, err := s.findManifestVersions(source)
|
||||
if err != nil {
|
||||
return s.finishScanRun(ctx, runID, source, startedAt, "failed", nil, 0, 0, 0, 0, err)
|
||||
}
|
||||
fallbackItems, err := s.findDirectTileRoots(source, coveredRoots)
|
||||
if err != nil {
|
||||
return s.finishScanRun(ctx, runID, source, startedAt, "failed", nil, 0, 0, 0, 0, err)
|
||||
}
|
||||
|
||||
allItems := append(manifestItems, fallbackItems...)
|
||||
sort.Slice(allItems, func(i, j int) bool {
|
||||
if allItems[i].BasemapCode == allItems[j].BasemapCode {
|
||||
return allItems[i].Version < allItems[j].Version
|
||||
}
|
||||
return allItems[i].BasemapCode < allItems[j].BasemapCode
|
||||
})
|
||||
|
||||
addedCount, updatedCount, removedCount, err := s.persistScanResult(ctx, source, allItems)
|
||||
if err != nil {
|
||||
return s.finishScanRun(ctx, runID, source, startedAt, "failed", allItems, len(allItems), addedCount, updatedCount, removedCount, err)
|
||||
}
|
||||
return s.finishScanRun(ctx, runID, source, startedAt, "succeeded", allItems, len(allItems), addedCount, updatedCount, removedCount, nil)
|
||||
}
|
||||
|
||||
func (s *Store) finishScanRun(
|
||||
ctx context.Context,
|
||||
runID string,
|
||||
source ScanSource,
|
||||
startedAt time.Time,
|
||||
status string,
|
||||
items []scannedVersion,
|
||||
scannedCount int,
|
||||
addedCount int,
|
||||
updatedCount int,
|
||||
removedCount int,
|
||||
runErr error,
|
||||
) (ScanRun, error) {
|
||||
finishedAt := nowUTC()
|
||||
summary := map[string]any{
|
||||
"source_code": source.Code,
|
||||
"scan_root": source.RootPath,
|
||||
"scanned_count": scannedCount,
|
||||
"added_count": addedCount,
|
||||
"updated_count": updatedCount,
|
||||
"removed_count": removedCount,
|
||||
}
|
||||
if runErr != nil {
|
||||
summary["error"] = runErr.Error()
|
||||
}
|
||||
if len(items) > 0 {
|
||||
codes := map[string]struct{}{}
|
||||
for _, item := range items {
|
||||
codes[item.BasemapCode] = struct{}{}
|
||||
}
|
||||
summary["basemap_codes"] = sortedKeys(codes)
|
||||
}
|
||||
|
||||
_, updateErr := s.db.ExecContext(ctx, `
|
||||
UPDATE scan_runs
|
||||
SET status = ?, scanned_count = ?, added_count = ?, updated_count = ?, removed_count = ?, summary_json = ?, finished_at = ?
|
||||
WHERE id = ?
|
||||
`, status, scannedCount, addedCount, updatedCount, removedCount, writeJSON(summary), toRFC3339(finishedAt), runID)
|
||||
if updateErr != nil {
|
||||
if runErr != nil {
|
||||
return ScanRun{}, fmt.Errorf("%v; update scan run: %w", runErr, updateErr)
|
||||
}
|
||||
return ScanRun{}, updateErr
|
||||
}
|
||||
|
||||
run := ScanRun{
|
||||
ID: runID,
|
||||
ScanSourceID: source.ID,
|
||||
SourceCode: source.Code,
|
||||
Status: status,
|
||||
ScannedCount: scannedCount,
|
||||
AddedCount: addedCount,
|
||||
UpdatedCount: updatedCount,
|
||||
RemovedCount: removedCount,
|
||||
Summary: summary,
|
||||
StartedAt: startedAt,
|
||||
FinishedAt: finishedAt,
|
||||
}
|
||||
if runErr != nil {
|
||||
return run, runErr
|
||||
}
|
||||
return run, nil
|
||||
}
|
||||
|
||||
func (s *Store) findManifestVersions(source ScanSource) ([]scannedVersion, map[string]struct{}, error) {
|
||||
manifestName := strings.TrimSpace(source.ManifestName)
|
||||
if manifestName == "" {
|
||||
manifestName = defaultManifestName
|
||||
}
|
||||
|
||||
var items []scannedVersion
|
||||
coveredRoots := map[string]struct{}{}
|
||||
err := filepath.WalkDir(source.RootPath, func(path string, entry os.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if !strings.EqualFold(entry.Name(), manifestName) {
|
||||
return nil
|
||||
}
|
||||
|
||||
manifest, err := LoadManifest(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tileRoot, err := resolveTileRoot(filepath.Dir(path), manifest.RootPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
format, minZoom, maxZoom, found, err := inspectTileRoot(tileRoot, manifest.TileFormat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if found {
|
||||
manifest.TileFormat = format
|
||||
manifest.MinZoom = minZoom
|
||||
manifest.MaxZoom = maxZoom
|
||||
}
|
||||
|
||||
relPath, err := filepath.Rel(source.RootPath, tileRoot)
|
||||
if err == nil {
|
||||
first := strings.Split(filepath.ToSlash(relPath), "/")[0]
|
||||
if first != "" && first != "." {
|
||||
coveredRoots[first] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
items = append(items, scannedVersion{
|
||||
BasemapCode: manifest.Code,
|
||||
Name: defaultText(manifest.Name, manifest.Code),
|
||||
Type: defaultText(manifest.Type, "xyz"),
|
||||
Status: defaultText(manifest.Status, "ready"),
|
||||
Description: manifest.Description,
|
||||
Version: defaultText(manifest.Version, "current"),
|
||||
Manifest: path,
|
||||
TileRoot: tileRoot,
|
||||
TileFormat: normalizeTileFormat(manifest.TileFormat),
|
||||
TileScheme: defaultText(manifest.TileScheme, "xyz"),
|
||||
MinZoom: manifest.MinZoom,
|
||||
MaxZoom: manifest.MaxZoom,
|
||||
BBox: manifest.BBox,
|
||||
Attribution: manifest.Attribution,
|
||||
Metadata: manifest.Metadata,
|
||||
IsDefault: manifest.IsDefault,
|
||||
})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("walk manifests: %w", err)
|
||||
}
|
||||
return items, coveredRoots, nil
|
||||
}
|
||||
|
||||
func (s *Store) findDirectTileRoots(source ScanSource, skip map[string]struct{}) ([]scannedVersion, error) {
|
||||
entries, err := os.ReadDir(source.RootPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read scan root: %w", err)
|
||||
}
|
||||
|
||||
var items []scannedVersion
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
name := entry.Name()
|
||||
if _, exists := skip[name]; exists {
|
||||
continue
|
||||
}
|
||||
|
||||
rootPath := filepath.Join(source.RootPath, name)
|
||||
format, minZoom, maxZoom, found, err := inspectTileRoot(rootPath, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
|
||||
code := normalizeBasemapCode(name)
|
||||
if code == "" {
|
||||
continue
|
||||
}
|
||||
items = append(items, scannedVersion{
|
||||
BasemapCode: code,
|
||||
Name: name,
|
||||
Type: "xyz",
|
||||
Status: "ready",
|
||||
Version: "current",
|
||||
TileRoot: rootPath,
|
||||
TileFormat: format,
|
||||
TileScheme: "xyz",
|
||||
MinZoom: minZoom,
|
||||
MaxZoom: maxZoom,
|
||||
Metadata: map[string]any{
|
||||
"discovered_from": "directory_scan",
|
||||
},
|
||||
IsDefault: true,
|
||||
})
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func inspectTileRoot(rootPath string, hintFormat string) (string, int, int, bool, error) {
|
||||
entries, err := os.ReadDir(rootPath)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return "", 0, 0, false, nil
|
||||
}
|
||||
return "", 0, 0, false, fmt.Errorf("read tile root %s: %w", rootPath, err)
|
||||
}
|
||||
|
||||
var zooms []int
|
||||
format := normalizeTileFormat(hintFormat)
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
zoom, err := strconv.Atoi(entry.Name())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
zooms = append(zooms, zoom)
|
||||
if format == "" || format == "png" {
|
||||
detected, found, err := detectTileFormat(filepath.Join(rootPath, entry.Name()))
|
||||
if err != nil {
|
||||
return "", 0, 0, false, err
|
||||
}
|
||||
if found {
|
||||
format = detected
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(zooms) == 0 {
|
||||
return "", 0, 0, false, nil
|
||||
}
|
||||
sort.Ints(zooms)
|
||||
if format == "" {
|
||||
format = "png"
|
||||
}
|
||||
return format, zooms[0], zooms[len(zooms)-1], true, nil
|
||||
}
|
||||
|
||||
func detectTileFormat(zoomPath string) (string, bool, error) {
|
||||
xEntries, err := os.ReadDir(zoomPath)
|
||||
if err != nil {
|
||||
return "", false, fmt.Errorf("read zoom path %s: %w", zoomPath, err)
|
||||
}
|
||||
for _, xEntry := range xEntries {
|
||||
if !xEntry.IsDir() {
|
||||
continue
|
||||
}
|
||||
tileEntries, err := os.ReadDir(filepath.Join(zoomPath, xEntry.Name()))
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
for _, tileEntry := range tileEntries {
|
||||
if tileEntry.IsDir() {
|
||||
continue
|
||||
}
|
||||
ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(tileEntry.Name())), ".")
|
||||
if ext == "" {
|
||||
continue
|
||||
}
|
||||
return normalizeTileFormat(ext), true, nil
|
||||
}
|
||||
}
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
func resolveTileRoot(baseDir, relativePath string) (string, error) {
|
||||
if strings.TrimSpace(relativePath) == "" {
|
||||
relativePath = "tiles"
|
||||
}
|
||||
if filepath.IsAbs(relativePath) {
|
||||
return filepath.Clean(relativePath), nil
|
||||
}
|
||||
resolved := filepath.Clean(filepath.Join(baseDir, relativePath))
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
func (s *Store) persistScanResult(ctx context.Context, source ScanSource, items []scannedVersion) (int, int, int, error) {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, 0, 0, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
existing := map[string]struct{}{}
|
||||
rows, err := tx.QueryContext(ctx, `
|
||||
SELECT b.code, v.version
|
||||
FROM basemap_versions v
|
||||
JOIN basemaps b ON b.id = v.basemap_id
|
||||
WHERE v.scan_source_id = ?
|
||||
`, source.ID)
|
||||
if err != nil {
|
||||
return 0, 0, 0, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var code string
|
||||
var version string
|
||||
if err := rows.Scan(&code, &version); err != nil {
|
||||
rows.Close()
|
||||
return 0, 0, 0, err
|
||||
}
|
||||
existing[code+"::"+version] = struct{}{}
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return 0, 0, 0, err
|
||||
}
|
||||
|
||||
now := nowUTC()
|
||||
found := map[string]struct{}{}
|
||||
touchedBasemaps := map[string]struct{}{}
|
||||
addedCount := 0
|
||||
updatedCount := 0
|
||||
for _, item := range items {
|
||||
key := item.BasemapCode + "::" + item.Version
|
||||
found[key] = struct{}{}
|
||||
if _, exists := existing[key]; exists {
|
||||
updatedCount++
|
||||
} else {
|
||||
addedCount++
|
||||
}
|
||||
if err := s.upsertScannedVersionTx(ctx, tx, source, item, now); err != nil {
|
||||
return 0, 0, 0, err
|
||||
}
|
||||
touchedBasemaps[item.BasemapCode] = struct{}{}
|
||||
}
|
||||
|
||||
removedCount := 0
|
||||
for key := range existing {
|
||||
if _, ok := found[key]; ok {
|
||||
continue
|
||||
}
|
||||
parts := strings.SplitN(key, "::", 2)
|
||||
if len(parts) != 2 {
|
||||
continue
|
||||
}
|
||||
if err := s.deleteVersionBySourceTx(ctx, tx, source.ID, parts[0], parts[1]); err != nil {
|
||||
return 0, 0, 0, err
|
||||
}
|
||||
touchedBasemaps[parts[0]] = struct{}{}
|
||||
removedCount++
|
||||
}
|
||||
|
||||
for code := range touchedBasemaps {
|
||||
if err := s.normalizeBasemapDefaultsTx(ctx, tx, code, now); err != nil {
|
||||
return 0, 0, 0, err
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return 0, 0, 0, err
|
||||
}
|
||||
return addedCount, updatedCount, removedCount, nil
|
||||
}
|
||||
|
||||
func (s *Store) upsertScannedVersionTx(ctx context.Context, tx *sql.Tx, source ScanSource, item scannedVersion, now time.Time) error {
|
||||
basemapID := uid.Deterministic("basemap", item.BasemapCode)
|
||||
versionID := uid.Deterministic("basemap-version", item.BasemapCode, item.Version)
|
||||
|
||||
_, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO basemaps (id, code, name, type, status, description, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(code) DO UPDATE SET
|
||||
name = excluded.name,
|
||||
type = excluded.type,
|
||||
status = excluded.status,
|
||||
description = excluded.description,
|
||||
updated_at = excluded.updated_at
|
||||
`, basemapID, item.BasemapCode, item.Name, item.Type, item.Status, item.Description, toRFC3339(now), toRFC3339(now))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
isDefault := 0
|
||||
if item.IsDefault {
|
||||
isDefault = 1
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `
|
||||
INSERT INTO basemap_versions (
|
||||
id, basemap_id, scan_source_id, version, status, is_default, manifest_path, tile_root_path, url_template,
|
||||
tile_format, tile_scheme, min_zoom, max_zoom, bbox_json, attribution, metadata_json, created_at, updated_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(basemap_id, version) DO UPDATE SET
|
||||
scan_source_id = excluded.scan_source_id,
|
||||
status = excluded.status,
|
||||
is_default = CASE WHEN excluded.is_default = 1 THEN 1 ELSE basemap_versions.is_default END,
|
||||
manifest_path = excluded.manifest_path,
|
||||
tile_root_path = excluded.tile_root_path,
|
||||
url_template = excluded.url_template,
|
||||
tile_format = excluded.tile_format,
|
||||
tile_scheme = excluded.tile_scheme,
|
||||
min_zoom = excluded.min_zoom,
|
||||
max_zoom = excluded.max_zoom,
|
||||
bbox_json = excluded.bbox_json,
|
||||
attribution = excluded.attribution,
|
||||
metadata_json = excluded.metadata_json,
|
||||
updated_at = excluded.updated_at
|
||||
`, versionID, basemapID, source.ID, item.Version, item.Status, isDefault, item.Manifest, item.TileRoot, s.buildURLTemplate(item.BasemapCode, item.Version, item.TileFormat), item.TileFormat, item.TileScheme, item.MinZoom, item.MaxZoom, writeJSON(item.BBox), item.Attribution, writeJSON(item.Metadata), toRFC3339(now), toRFC3339(now))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if item.IsDefault {
|
||||
_, err = tx.ExecContext(ctx, `
|
||||
UPDATE basemap_versions
|
||||
SET is_default = CASE WHEN version = ? THEN 1 ELSE 0 END, updated_at = ?
|
||||
WHERE basemap_id = ?
|
||||
`, item.Version, toRFC3339(now), basemapID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) deleteVersionBySourceTx(ctx context.Context, tx *sql.Tx, sourceID, basemapCode, version string) error {
|
||||
result, err := tx.ExecContext(ctx, `
|
||||
DELETE FROM basemap_versions
|
||||
WHERE scan_source_id = ? AND version = ? AND basemap_id = (SELECT id FROM basemaps WHERE code = ?)
|
||||
`, sourceID, version, basemapCode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := result.RowsAffected(); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `
|
||||
DELETE FROM basemaps
|
||||
WHERE code = ? AND NOT EXISTS (
|
||||
SELECT 1 FROM basemap_versions WHERE basemap_id = basemaps.id
|
||||
)
|
||||
`, basemapCode)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) normalizeBasemapDefaultsTx(ctx context.Context, tx *sql.Tx, basemapCode string, now time.Time) error {
|
||||
var basemapID string
|
||||
err := tx.QueryRowContext(ctx, `SELECT id FROM basemaps WHERE code = ?`, basemapCode).Scan(&basemapID)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
rows, err := tx.QueryContext(ctx, `
|
||||
SELECT version, is_default
|
||||
FROM basemap_versions
|
||||
WHERE basemap_id = ?
|
||||
ORDER BY is_default DESC, version ASC
|
||||
`, basemapID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
type versionFlag struct {
|
||||
Version string
|
||||
IsDefault bool
|
||||
}
|
||||
var versions []versionFlag
|
||||
for rows.Next() {
|
||||
var item versionFlag
|
||||
var isDefault int
|
||||
if err := rows.Scan(&item.Version, &isDefault); err != nil {
|
||||
return err
|
||||
}
|
||||
item.IsDefault = isDefault == 1
|
||||
versions = append(versions, item)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(versions) == 0 {
|
||||
return nil
|
||||
}
|
||||
defaultVersion := versions[0].Version
|
||||
for _, item := range versions {
|
||||
if item.IsDefault {
|
||||
defaultVersion = item.Version
|
||||
break
|
||||
}
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
UPDATE basemap_versions
|
||||
SET is_default = CASE WHEN version = ? THEN 1 ELSE 0 END, updated_at = ?
|
||||
WHERE basemap_id = ?
|
||||
`, defaultVersion, toRFC3339(now), basemapID); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `UPDATE basemaps SET updated_at = ? WHERE id = ?`, toRFC3339(now), basemapID)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) buildURLTemplate(code, version, format string) string {
|
||||
base := trimURL(s.cfg.TileBaseURL)
|
||||
if base == "" {
|
||||
base = trimURL(s.cfg.APIBaseURL)
|
||||
}
|
||||
format = normalizeTileFormat(format)
|
||||
return fmt.Sprintf("%s/tiles/%s/%s/{z}/{x}/{y}.%s", base, code, version, format)
|
||||
}
|
||||
Reference in New Issue
Block a user