Simplify file cache metadata and CI benchmarks

.github/workflows/build-push.yaml

@@ -4,6 +4,10 @@ on:
   push:
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   lint-test:
     runs-on: ubuntu-24.04

docs/caching.md

@@ -27,15 +27,16 @@ responses are not stored.
 files under `cache.root`. It is different from
 `iiif.image.max_derivative_bytes`, which limits one generated response before it
 can be returned or cached. A cache write can temporarily exceed `cache.max_bytes`
-before eviction runs, and metadata sidecar files are not counted toward the
-target.
+before eviction runs. When size eviction runs, Triplet removes the oldest
+derivative payload files first based on payload file modification time; reads
+do not refresh cache age.
 
-`cache.max_age` is based on when Triplet wrote the derivative entry, not when it
-was last requested. When a cached derivative is older than `max_age`, Triplet
-removes it and treats the request as a cache miss. Expired entries are also
-removed opportunistically when new entries are written. Set `max_age: 0` or omit
-it to keep derivative files until size eviction, manual deletion, invalidation,
-or cache-key changes make them unused.
+`cache.max_age` is based on the derivative payload file modification time, not
+when it was last requested. When a cached derivative is older than `max_age`,
+Triplet removes it and treats the request as a cache miss. Expired entries are
+also removed opportunistically when new entries are written. Set `max_age: 0`
+or omit it to keep derivative files until size eviction, manual deletion,
+invalidation, or cache-key changes make them unused.
 
 ### Derivative invalidation
 
@@ -51,6 +52,15 @@ The invalidation route is protected by a bearer token and optional caller CIDR
 checks. See [Authorization](authorization.md#cache-invalidation-route) for the
 route configuration and caller requirements.
 
+```sh
+curl -X POST \
+  -H "Authorization: Bearer ${TRIPLET_IMAGE_CACHE_INVALIDATION_TOKEN}" \
+  "https://iiif.example.edu/iiif/3/https%3A%2F%2Frepo.example.edu%2Fsystem%2Ffiles%2Fimage.tif/cache/invalidate"
+```
+
+The identifier in the request path must be URI-encoded exactly as it appears in
+the IIIF Image API request path segment.
+
 ### Source version metadata
 
 Derivative cache keys include a source version so Triplet does not reuse old

internal/cache/file.go

@@ -12,18 +12,22 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
 	"sync"
 	"time"
 )
 
-// FileStore stores cache entries as files under Root. Bytes go in <hash>,
-// metadata in <hash>.meta. Keys are hashed (SHA-256) so they can contain any
-// characters and still be safe filenames.
+// FileStore stores cache entries as files under Root. Bytes go in <hash>.
+// When storeContentType is enabled, metadata goes in <hash>.meta. Keys are
+// hashed (SHA-256) so they can contain any characters and still be safe
+// filenames.
 type FileStore struct {
 	Root string
 
+	storeContentType bool
+
 	// MaxBytes optionally bounds total cache size; when exceeded, the
-	// least-recently-modified entries are evicted on the next Put.
+	// oldest payload files are evicted on the next Put based on mtime.
 	MaxBytes int64
 
 	// MaxAge optionally bounds how long entries remain usable after Put.
@@ -33,6 +37,8 @@ type FileStore struct {
 	mu sync.Mutex
 }
 
+const tempFilePrefix = ".tmp-"
+
 // NewFileStore constructs a FileStore. Root is created if it does not exist.
 func NewFileStore(root string, maxBytes int64) (*FileStore, error) {
 	abs, err := filepath.Abs(root)
@@ -42,7 +48,7 @@ func NewFileStore(root string, maxBytes int64) (*FileStore, error) {
 	if err := os.MkdirAll(abs, 0o750); err != nil {
 		return nil, fmt.Errorf("cache file mkdir: %w", err)
 	}
-	return &FileStore{Root: abs, MaxBytes: maxBytes}, nil
+	return &FileStore{Root: abs, storeContentType: true, MaxBytes: maxBytes}, nil
 }
 
 // NewFileStoreWithMaxAge constructs a FileStore with size and age eviction.
@@ -55,42 +61,67 @@ func NewFileStoreWithMaxAge(root string, maxBytes int64, maxAge time.Duration) (
 	return store, nil
 }
 
+// NewPayloadFileStoreWithMaxAge constructs a payload-only FileStore. It stores
+// content type out of band in the caller and uses payload file metadata for
+// age and size accounting.
+func NewPayloadFileStoreWithMaxAge(root string, maxBytes int64, maxAge time.Duration) (*FileStore, error) {
+	store, err := NewFileStore(root, maxBytes)
+	if err != nil {
+		return nil, err
+	}
+	store.storeContentType = false
+	store.MaxAge = maxAge
+	return store, nil
+}
+
 type fileMeta struct {
-	ContentType string    `json:"content_type"`
-	Size        int64     `json:"size"`
-	StoredAt    time.Time `json:"stored_at"`
+	ContentType string `json:"content_type"`
 }
 
 // Get implements Store.
 func (s *FileStore) Get(_ context.Context, key string) (io.ReadCloser, Entry, error) {
 	dataPath, metaPath := s.paths(key)
-	mb, err := os.ReadFile(metaPath)
+	contentType := ""
+	if s.storeContentType {
+		mb, err := os.ReadFile(metaPath)
+		if err != nil {
+			if errors.Is(err, fs.ErrNotExist) {
+				return nil, Entry{}, ErrMiss
+			}
+			return nil, Entry{}, err
+		}
+		var m fileMeta
+		if err := json.Unmarshal(mb, &m); err != nil {
+			return nil, Entry{}, fmt.Errorf("cache meta: %w", err)
+		}
+		contentType = m.ContentType
+	}
+	f, err := os.Open(dataPath)
 	if err != nil {
 		if errors.Is(err, fs.ErrNotExist) {
 			return nil, Entry{}, ErrMiss
 		}
 		return nil, Entry{}, err
 	}
-	var m fileMeta
-	if err := json.Unmarshal(mb, &m); err != nil {
-		return nil, Entry{}, fmt.Errorf("cache meta: %w", err)
+	info, err := f.Stat()
+	if err != nil {
+		_ = f.Close()
+		return nil, Entry{}, err
 	}
-	if s.expired(m.StoredAt, time.Now()) {
+	storedAt := info.ModTime()
+	if s.expired(storedAt, time.Now()) {
+		_ = f.Close()
 		_ = os.Remove(dataPath)
-		_ = os.Remove(metaPath)
-		return nil, Entry{}, ErrMiss
-	}
-	f, err := os.Open(dataPath)
-	if err != nil {
-		if errors.Is(err, fs.ErrNotExist) {
-			return nil, Entry{}, ErrMiss
+		if s.storeContentType {
+			_ = os.Remove(metaPath)
 		}
-		return nil, Entry{}, err
+		return nil, Entry{}, ErrMiss
 	}
-	// Refresh mtime so eviction LRU treats this as recently used.
-	now := time.Now()
-	_ = os.Chtimes(dataPath, now, now)
-	return f, Entry(m), nil
+	return f, Entry{
+		ContentType: contentType,
+		Size:        info.Size(),
+		StoredAt:    storedAt,
+	}, nil
 }
 
 // Put implements Store.
@@ -104,7 +135,7 @@ func (s *FileStore) Put(_ context.Context, key, contentType string, value io.Rea
 		return err
 	}
 	tmpName := tmp.Name()
-	n, err := io.Copy(tmp, value)
+	_, err = io.Copy(tmp, value)
 	if err != nil {
 		_ = tmp.Close()
 		_ = os.Remove(tmpName)
@@ -114,29 +145,44 @@ func (s *FileStore) Put(_ context.Context, key, contentType string, value io.Rea
 		_ = os.Remove(tmpName)
 		return err
 	}
+	if s.storeContentType {
+		if err := s.installWithMeta(tmpName, dataPath, metaPath, contentType); err != nil {
+			return err
+		}
+	} else if err := os.Rename(tmpName, dataPath); err != nil {
+		_ = os.Remove(tmpName)
+		return err
+	}
+	if s.MaxAge > 0 || s.MaxBytes > 0 {
+		s.evict()
+	}
+	return nil
+}
+
+func (s *FileStore) installWithMeta(tmpName, dataPath, metaPath, contentType string) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
 	if err := os.Rename(tmpName, dataPath); err != nil {
 		_ = os.Remove(tmpName)
 		return err
 	}
-	storedAt := time.Now()
-	_ = os.Chtimes(dataPath, storedAt, storedAt)
-	meta := fileMeta{ContentType: contentType, Size: n, StoredAt: storedAt}
+	meta := fileMeta{ContentType: contentType}
 	mb, _ := json.Marshal(meta)
 	if err := os.WriteFile(metaPath, mb, 0o640); err != nil {
+		_ = os.Remove(dataPath)
 		return err
 	}
-	_ = os.Chtimes(metaPath, storedAt, storedAt)
-	if s.MaxAge > 0 || s.MaxBytes > 0 {
-		s.evict()
-	}
 	return nil
 }
 
 // Delete implements Store.
 func (s *FileStore) Delete(_ context.Context, key string) error {
 	dataPath, metaPath := s.paths(key)
 	_ = os.Remove(dataPath)
-	_ = os.Remove(metaPath)
+	if s.storeContentType {
+		_ = os.Remove(metaPath)
+	}
 	return nil
 }
 
@@ -154,57 +200,61 @@ func (s *FileStore) evict() {
 	defer s.mu.Unlock()
 	var total int64
 	type entry struct {
-		path     string
-		metaPath string
-		size     int64
-		mod      time.Time
-		storedAt time.Time
+		path    string
+		size    int64
+		modTime time.Time
 	}
 	var entries []entry
 	now := time.Now()
 	_ = filepath.WalkDir(s.Root, func(p string, d fs.DirEntry, err error) error {
 		if err != nil || d.IsDir() {
 			return nil
 		}
-		if filepath.Ext(p) == ".meta" {
+		if filepath.Ext(p) == ".meta" || strings.HasPrefix(d.Name(), tempFilePrefix) {
 			return nil
 		}
 		info, err := d.Info()
 		if err != nil {
 			return nil
 		}
-		storedAt := info.ModTime()
 		metaPath := p + ".meta"
-		if mb, err := os.ReadFile(metaPath); err == nil {
-			var m fileMeta
-			if err := json.Unmarshal(mb, &m); err == nil && !m.StoredAt.IsZero() {
-				storedAt = m.StoredAt
+		if s.storeContentType {
+			if _, err := os.Stat(metaPath); err != nil {
+				if errors.Is(err, fs.ErrNotExist) {
+					_ = os.Remove(p)
+					return nil
+				}
+				return nil
 			}
 		}
-		if s.expired(storedAt, now) {
+		if s.expired(info.ModTime(), now) {
 			_ = os.Remove(p)
-			_ = os.Remove(metaPath)
+			if s.storeContentType {
+				_ = os.Remove(metaPath)
+			}
 			return nil
 		}
-		entries = append(entries, entry{path: p, metaPath: metaPath, size: info.Size(), mod: info.ModTime(), storedAt: storedAt})
+		entries = append(entries, entry{path: p, size: info.Size(), modTime: info.ModTime()})
 		total += info.Size()
 		return nil
 	})
 	if s.MaxBytes <= 0 || total <= s.MaxBytes {
 		return
 	}
 	sort.Slice(entries, func(i, j int) bool {
-		if entries[i].mod.Equal(entries[j].mod) {
+		if entries[i].modTime.Equal(entries[j].modTime) {
 			return entries[i].path < entries[j].path
 		}
-		return entries[i].mod.Before(entries[j].mod)
+		return entries[i].modTime.Before(entries[j].modTime)
 	})
 	for _, e := range entries {
 		if total <= s.MaxBytes {
 			return
 		}
 		_ = os.Remove(e.path)
-		_ = os.Remove(e.metaPath)
+		if s.storeContentType {
+			_ = os.Remove(e.path + ".meta")
+		}
 		total -= e.size
 	}
 }