feat: persist embedding metadata

- add schema versioning and migrate vault/cache metadata columns

- store n_tokens and truncated on vault rows and cache entries

- assert metadata persistence in unittest coverage

- document the remote parser refactor plan for later implementation

Author: andinux

remote-embedding-parser-refactor-plan.md

@@ -0,0 +1,170 @@
+# Remote Embedding Parser Refactor Plan
+
+This note captures the preferred implementation plan for later work. The goal is
+to make vectors.space response parsing directly testable without turning the
+parser into public API or giving production logic a test-only name.
+
+## Goal
+
+Refactor `src/dbmem-rembed.c` so the JSON response parsing currently embedded in
+`dbmem_remote_compute_embedding()` lives in a reusable internal function:
+
+```c
+int dbmem_remote_parse_embedding_response(...);
+```
+
+The function should be production logic, used by `dbmem_remote_compute_embedding()`
+and also callable from `test/unittest.c` via a manual forward declaration.
+
+Do not expose it in `sqlite-memory.h` or another public header.
+
+## Preferred Shape
+
+Use a normal internal production name, not a test-only name:
+
+```c
+int dbmem_remote_parse_embedding_response(
+    const char *json,
+    size_t json_len,
+    float **embedding,
+    size_t *embedding_capacity,
+    jsmntok_t **tokens,
+    int *tokens_capacity,
+    embedding_result_t *result,
+    char *err_msg,
+    size_t err_msg_len
+);
+```
+
+This keeps ownership explicit while avoiding exposure of `dbmem_remote_engine_t`
+or a new parser-state struct in test code.
+
+## Production Usage
+
+`dbmem_remote_compute_embedding()` keeps responsibility for:
+
+- request construction
+- HTTP transport
+- HTTP status handling
+- context error propagation
+- aggregate remote-engine stats
+
+After receiving a successful HTTP 200 response, it calls:
+
+```c
+char err_msg[DBMEM_ERRBUF_SIZE] = {0};
+int rc = dbmem_remote_parse_embedding_response(
+    engine->data,
+    engine->data_size,
+    &engine->embedding,
+    &engine->embedding_capacity,
+    &engine->tokens,
+    &engine->tokens_capacity,
+    result,
+    err_msg,
+    sizeof(err_msg)
+);
+
+if (rc != 0) {
+    dbmem_context_set_error(engine->context, err_msg);
+    return -1;
+}
+
+engine->total_tokens_processed += result->n_tokens;
+engine->total_embeddings_generated++;
+return 0;
+```
+
+## Parser Responsibility
+
+`dbmem_remote_parse_embedding_response()` should own:
+
+- parsing JSON with `jsmn`
+- allocating/growing the token buffer
+- locating top-level `output_dimension`
+- locating `data[0].embedding`
+- allocating/growing the embedding buffer
+- parsing embedding floats
+- reading `data[0].truncated`
+- reading token metadata from `usage`
+- filling `embedding_result_t`
+
+Token count priority should remain:
+
+1. `usage.exact_prompt_tokens`
+2. `usage.estimated_prompt_tokens`
+3. `usage.prompt_tokens`
+4. `0` if none are present
+
+## Unit Test Usage
+
+`test/unittest.c` can manually forward-declare the function under the relevant
+test guards:
+
+```c
+#if defined(TEST_SQLITE_EXTENSION) && !defined(DBMEM_OMIT_REMOTE_ENGINE)
+int dbmem_remote_parse_embedding_response(
+    const char *json,
+    size_t json_len,
+    float **embedding,
+    size_t *embedding_capacity,
+    jsmntok_t **tokens,
+    int *tokens_capacity,
+    embedding_result_t *result,
+    char *err_msg,
+    size_t err_msg_len
+);
+#endif
+```
+
+Tests create local buffers:
+
+```c
+float *embedding = NULL;
+size_t embedding_capacity = 0;
+jsmntok_t *tokens = NULL;
+int tokens_capacity = 0;
+embedding_result_t result = {0};
+char err_msg[1024] = {0};
+```
+
+Then call the parser with static JSON fixtures and free the buffers afterward:
+
+```c
+dbmemory_free(embedding);
+dbmemory_free(tokens);
+```
+
+## Fixture Tests To Add Later
+
+Recommended deterministic cases:
+
+- exact token count is preferred over estimated and prompt token counts
+- estimated token count is used when exact token count is absent
+- prompt token count is used when exact and estimated token counts are absent
+- missing usage object leaves `result.n_tokens == 0`
+- `data[0].truncated: false` maps to `result.truncated == false`
+- `data[0].truncated: true` maps to `result.truncated == true`
+- embedding float array is parsed correctly
+- output dimension is parsed correctly
+- missing `data`
+- missing `embedding`
+- empty embedding array
+- invalid top-level response shape
+
+Also decide whether the parser should reject mismatches between
+`output_dimension` and the embedding array length. Failing fast is likely safer,
+because a dimension mismatch can break later vector initialization/search.
+
+## Why This Plan
+
+This approach avoids:
+
+- live network dependence for parser correctness tests
+- exposing parser internals as public API
+- duplicating parser behavior in test-only code
+- coupling tests to `dbmem_remote_engine_t`
+- adding a new internal header before it is needed
+
+The e2e test discussion can proceed separately, especially around whether token
+metadata should become persisted product state or remain parser-only metadata.

src/sqlite-memory.c

@@ -60,6 +60,9 @@ SQLITE_EXTENSION_INIT1
 #define DBMEM_SETTINGS_KEY_EMBEDDING_CACHE      "embedding_cache"
 #define DBMEM_SETTINGS_KEY_CACHE_MAX_ENTRIES    "cache_max_entries"
 #define DBMEM_SETTINGS_KEY_SEARCH_OVERSAMPLE    "search_oversample"
+#define DBMEM_SETTINGS_KEY_SCHEMA_VERSION       "schema_version"
+
+#define DBMEM_SCHEMA_VERSION                    2
 
 // default values from https://docs.openclaw.ai/concepts/memory
 #define DEFAULT_CHARS_PER_TOKEN                 4       // Approximate number of characters per token (GPT ≈ 4, Claude ≈ 3.5)
@@ -358,6 +361,105 @@ void dbmem_settings_load (sqlite3 *db, dbmem_context *ctx) {
 
 // MARK: - Database -
 
+static bool dbmem_database_column_exists (sqlite3 *db, const char *table, const char *column, int *out_rc) {
+    char sql[256];
+    snprintf(sql, sizeof(sql), "PRAGMA table_info(%s);", table);
+
+    sqlite3_stmt *vm = NULL;
+    int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL);
+    if (rc != SQLITE_OK) {
+        if (out_rc) *out_rc = rc;
+        return false;
+    }
+
+    bool exists = false;
+    while ((rc = sqlite3_step(vm)) == SQLITE_ROW) {
+        const char *name = (const char *)sqlite3_column_text(vm, 1);
+        if (name && strcmp(name, column) == 0) {
+            exists = true;
+            break;
+        }
+    }
+
+    if (rc == SQLITE_DONE || rc == SQLITE_ROW) rc = SQLITE_OK;
+    sqlite3_finalize(vm);
+    if (out_rc) *out_rc = rc;
+    return exists;
+}
+
+static int dbmem_database_add_column_if_missing (sqlite3 *db, const char *table, const char *column, const char *alter_sql) {
+    int rc = SQLITE_OK;
+    if (dbmem_database_column_exists(db, table, column, &rc)) return SQLITE_OK;
+    if (rc != SQLITE_OK) return rc;
+    return sqlite3_exec(db, alter_sql, NULL, NULL, NULL);
+}
+
+static int dbmem_database_schema_version (sqlite3 *db, int *version) {
+    static const char *sql = "SELECT value FROM dbmem_settings WHERE key=?1 LIMIT 1;";
+
+    *version = 0;
+
+    sqlite3_stmt *vm = NULL;
+    int rc = sqlite3_prepare_v2(db, sql, -1, &vm, NULL);
+    if (rc != SQLITE_OK) goto cleanup;
+
+    rc = sqlite3_bind_text(vm, 1, DBMEM_SETTINGS_KEY_SCHEMA_VERSION, -1, SQLITE_STATIC);
+    if (rc != SQLITE_OK) goto cleanup;
+
+    rc = sqlite3_step(vm);
+    if (rc == SQLITE_ROW) {
+        *version = sqlite3_column_int(vm, 0);
+        rc = SQLITE_OK;
+    } else if (rc == SQLITE_DONE) {
+        rc = SQLITE_OK;
+    }
+
+cleanup:
+    if (vm) sqlite3_finalize(vm);
+    return rc;
+}
+
+static int dbmem_database_set_schema_version (sqlite3 *db, int version) {
+    return dbmem_settings_write_int(db, DBMEM_SETTINGS_KEY_SCHEMA_VERSION, version);
+}
+
+static int dbmem_database_migrate_v1_to_v2 (sqlite3 *db) {
+    int rc = dbmem_database_add_column_if_missing(db, "dbmem_vault", "n_tokens",
+        "ALTER TABLE dbmem_vault ADD COLUMN n_tokens INTEGER NOT NULL DEFAULT 0;");
+    if (rc != SQLITE_OK) return rc;
+
+    rc = dbmem_database_add_column_if_missing(db, "dbmem_vault", "truncated",
+        "ALTER TABLE dbmem_vault ADD COLUMN truncated INTEGER NOT NULL DEFAULT 0;");
+    if (rc != SQLITE_OK) return rc;
+
+    rc = dbmem_database_add_column_if_missing(db, "dbmem_cache", "n_tokens",
+        "ALTER TABLE dbmem_cache ADD COLUMN n_tokens INTEGER NOT NULL DEFAULT 0;");
+    if (rc != SQLITE_OK) return rc;
+
+    return dbmem_database_add_column_if_missing(db, "dbmem_cache", "truncated",
+        "ALTER TABLE dbmem_cache ADD COLUMN truncated INTEGER NOT NULL DEFAULT 0;");
+}
+
+static int dbmem_database_migrate (sqlite3 *db) {
+    int version = 0;
+    int rc = dbmem_database_schema_version(db, &version);
+    if (rc != SQLITE_OK) return rc;
+
+    if (version > DBMEM_SCHEMA_VERSION) return SQLITE_MISMATCH;
+    if (version <= 0) version = 1;
+
+    if (version < 2) {
+        rc = dbmem_database_migrate_v1_to_v2(db);
+        if (rc != SQLITE_OK) return rc;
+        version = 2;
+        rc = dbmem_database_set_schema_version(db, version);
+        if (rc != SQLITE_OK) return rc;
+    }
+
+    if (version != DBMEM_SCHEMA_VERSION) return SQLITE_MISMATCH;
+    return SQLITE_OK;
+}
+
 static int dbmem_database_init (sqlite3 *db) {
     const char *sql = "CREATE TABLE IF NOT EXISTS dbmem_settings (key TEXT PRIMARY KEY, value TEXT);";
     int rc = sqlite3_exec(db, sql, NULL, NULL, NULL);
@@ -367,14 +469,17 @@ static int dbmem_database_init (sqlite3 *db) {
     rc = sqlite3_exec(db, sql, NULL, NULL, NULL);
     if (rc != SQLITE_OK) return rc;
 
-    sql = "CREATE TABLE IF NOT EXISTS dbmem_vault (hash TEXT NOT NULL, seq INTEGER NOT NULL, embedding BLOB NOT NULL, offset INTEGER NOT NULL, length INTEGER NOT NULL, PRIMARY KEY (hash, seq));";
+    sql = "CREATE TABLE IF NOT EXISTS dbmem_vault (hash TEXT NOT NULL, seq INTEGER NOT NULL, embedding BLOB NOT NULL, offset INTEGER NOT NULL, length INTEGER NOT NULL, n_tokens INTEGER NOT NULL DEFAULT 0, truncated INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (hash, seq));";
     rc = sqlite3_exec(db, sql, NULL, NULL, NULL);
     if (rc != SQLITE_OK) return rc;
 
-    sql = "CREATE TABLE IF NOT EXISTS dbmem_cache (text_hash TEXT NOT NULL, provider TEXT NOT NULL, model TEXT NOT NULL, embedding BLOB NOT NULL, dimension INTEGER NOT NULL, PRIMARY KEY (text_hash, provider, model));";
+    sql = "CREATE TABLE IF NOT EXISTS dbmem_cache (text_hash TEXT NOT NULL, provider TEXT NOT NULL, model TEXT NOT NULL, embedding BLOB NOT NULL, dimension INTEGER NOT NULL, n_tokens INTEGER NOT NULL DEFAULT 0, truncated INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (text_hash, provider, model));";
     rc = sqlite3_exec(db, sql, NULL, NULL, NULL);
     if (rc != SQLITE_OK) return rc;
 
+    rc = dbmem_database_migrate(db);
+    if (rc != SQLITE_OK) return rc;
+
     sql = "CREATE VIRTUAL TABLE IF NOT EXISTS dbmem_vault_fts USING fts5 (content, hash UNINDEXED, seq UNINDEXED, context UNINDEXED);";
     rc = sqlite3_exec(db, sql, NULL, NULL, NULL);
     if (rc != SQLITE_OK) {
@@ -495,7 +600,7 @@ static int dbmem_database_add_entry (dbmem_context *ctx, sqlite3 *db, uint64_t h
 }
 
 static int dbmem_database_add_chunk (dbmem_context *ctx, embedding_result_t *result, size_t offset, size_t length, size_t index) {
-    static const char *sql = "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length) VALUES (?1, ?2, ?3, ?4, ?5);";
+    static const char *sql = "INSERT INTO dbmem_vault (hash, seq, embedding, offset, length, n_tokens, truncated) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7);";
 
     sqlite3_stmt *vm = NULL;
     int rc = sqlite3_prepare_v2(ctx->db, sql, -1, &vm, NULL);
@@ -515,6 +620,12 @@ static int dbmem_database_add_chunk (dbmem_context *ctx, embedding_result_t *res
 
     rc = sqlite3_bind_int64(vm, 5, (sqlite3_int64)length);
     if (rc != SQLITE_OK) goto cleanup;
+
+    rc = sqlite3_bind_int(vm, 6, result->n_tokens);
+    if (rc != SQLITE_OK) goto cleanup;
+
+    rc = sqlite3_bind_int(vm, 7, result->truncated ? 1 : 0);
+    if (rc != SQLITE_OK) goto cleanup;
 
     rc = sqlite3_step(vm);
     if (rc == SQLITE_DONE) rc = SQLITE_OK;
@@ -1267,7 +1378,7 @@ static void dbmem_dump_embeding (const embedding_result_t *result) {
 // MARK: - Embedding Cache -
 
 static bool dbmem_cache_lookup (dbmem_context *ctx, uint64_t text_hash, embedding_result_t *result) {
-    static const char *sql = "SELECT embedding, dimension FROM dbmem_cache WHERE text_hash=?1 AND provider=?2 AND model=?3 LIMIT 1;";
+    static const char *sql = "SELECT embedding, dimension, n_tokens, truncated FROM dbmem_cache WHERE text_hash=?1 AND provider=?2 AND model=?3 LIMIT 1;";
 
     if (!ctx->provider || !ctx->model) return false;
 
@@ -1300,8 +1411,8 @@ static bool dbmem_cache_lookup (dbmem_context *ctx, uint64_t text_hash, embeddin
     memcpy(ctx->cache_buffer, blob, blob_bytes);
     result->embedding = ctx->cache_buffer;
     result->n_embd = dimension;
-    result->n_tokens = 0;
-    result->truncated = false;
+    result->n_tokens = sqlite3_column_int(vm, 2);
+    result->truncated = sqlite3_column_int(vm, 3) != 0;
     found = true;
 
 cleanup:
@@ -1337,7 +1448,7 @@ static void dbmem_cache_evict (dbmem_context *ctx) {
 }
 
 static void dbmem_cache_store (dbmem_context *ctx, uint64_t text_hash, const embedding_result_t *result) {
-    static const char *sql = "INSERT OR REPLACE INTO dbmem_cache (text_hash, provider, model, embedding, dimension) VALUES (?1, ?2, ?3, ?4, ?5);";
+    static const char *sql = "INSERT OR REPLACE INTO dbmem_cache (text_hash, provider, model, embedding, dimension, n_tokens, truncated) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7);";
 
     if (!ctx->provider || !ctx->model) return;
 
@@ -1350,6 +1461,8 @@ static void dbmem_cache_store (dbmem_context *ctx, uint64_t text_hash, const emb
     sqlite3_bind_text(vm, 3, ctx->model, -1, SQLITE_STATIC);
     sqlite3_bind_blob(vm, 4, result->embedding, result->n_embd * (int)sizeof(float), SQLITE_STATIC);
     sqlite3_bind_int(vm, 5, result->n_embd);
+    sqlite3_bind_int(vm, 6, result->n_tokens);
+    sqlite3_bind_int(vm, 7, result->truncated ? 1 : 0);
 
     sqlite3_step(vm);
 

src/sqlite-memory.h

@@ -26,7 +26,7 @@
 extern "C" {
 #endif
 
-#define SQLITE_DBMEMORY_VERSION "1.1.0"
+#define SQLITE_DBMEMORY_VERSION "1.2.0"
 
 // public API
 SQLITE_DBMEMORY_API int sqlite3_memory_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi);