From 6ed59d77bc5e6889d6a21d99e8005b9212f00e90 Mon Sep 17 00:00:00 2001 From: Faysal Aberkane Date: Fri, 14 Aug 2026 09:52:28 +0100 Subject: [PATCH] Add precise dry-run index preview - Track file fingerprints after successful index updates - Report added, updated, and deleted files without writes - Preserve path-only previews until the manifest exists --- README.md | 2 + src/cocoindex_code/cli.py | 43 ++++- src/cocoindex_code/index_changes.py | 239 ++++++++++++++++++++++++++++ src/cocoindex_code/project.py | 12 +- src/cocoindex_code/shared.py | 1 + tests/test_cli_helpers.py | 9 ++ tests/test_e2e.py | 32 ++++ tests/test_index_changes.py | 97 +++++++++++ tests/test_project_indexing.py | 9 ++ 9 files changed, 437 insertions(+), 7 deletions(-) create mode 100644 src/cocoindex_code/index_changes.py create mode 100644 tests/test_index_changes.py diff --git a/README.md b/README.md index 4748373..f84a2cf 100644 --- a/README.md +++ b/README.md @@ -233,6 +233,7 @@ You can also use the CLI directly — useful for manual control, running indexin ```bash ccc init # initialize project (creates settings) +ccc index --dry # preview added, updated, and deleted files ccc index # build the index ccc search "authentication logic" # search! ``` @@ -246,6 +247,7 @@ The background daemon starts automatically on first use. | Command | Description | |---------|-------------| | `ccc init` | Initialize a project — creates settings files, adds `.cocoindex_code/` to `.gitignore` | +| `ccc index --dry` | Preview added, updated, and deleted files | | `ccc index` | Build or update the index (auto-inits if needed). Shows streaming progress. | | `ccc search ` | Semantic search across the codebase | | `ccc grep [path]` | Structural code search by example (no index needed) | diff --git a/src/cocoindex_code/cli.py b/src/cocoindex_code/cli.py index 88164a3..8a0a8b1 100644 --- a/src/cocoindex_code/cli.py +++ b/src/cocoindex_code/cli.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import functools import os import sys @@ -645,14 +646,46 @@ def init( @app.command() @_catch_daemon_start_error -def index() -> None: +def index( + dry: bool = _typer.Option( + False, + "--dry", + help="List files to add, update, or delete without indexing.", + ), +) -> None: """Create/update index for the codebase.""" + project_root = require_project_root(auto_init=not dry) + print_project_header(str(project_root)) + if dry: + from .index_changes import find_index_changes + + changes = asyncio.run(find_index_changes(project_root)) + _print_index_changes(changes.added, changes.updated, changes.deleted) + return + from . import client as _client - project_root = str(require_project_root(auto_init=True)) - print_project_header(project_root) - _run_index_with_progress(project_root) - print_index_stats(_client.project_status(project_root)) + _run_index_with_progress(str(project_root)) + print_index_stats(_client.project_status(str(project_root))) + + +def _print_index_changes( + added: tuple[str, ...], + updated: tuple[str, ...] | None, + deleted: tuple[str, ...], +) -> None: + _typer.echo(f"Files to add ({len(added)}):") + for path in added: + _typer.echo(f" {path}") + if updated is None: + _typer.echo("Files to update: unavailable until the next successful index") + else: + _typer.echo(f"Files to update ({len(updated)}):") + for path in updated: + _typer.echo(f" {path}") + _typer.echo(f"Files to delete ({len(deleted)}):") + for path in deleted: + _typer.echo(f" {path}") @app.command() diff --git a/src/cocoindex_code/index_changes.py b/src/cocoindex_code/index_changes.py new file mode 100644 index 0000000..af9bdb5 --- /dev/null +++ b/src/cocoindex_code/index_changes.py @@ -0,0 +1,239 @@ +"""Read-only preview of files changed since the last index.""" + +from __future__ import annotations + +import asyncio +import sqlite3 +from collections.abc import Collection, Iterator, Sequence +from dataclasses import dataclass +from pathlib import Path, PurePath +from typing import NamedTuple + +import cocoindex as coco +from cocoindex.connectorkits.fingerprint import fingerprint_bytes +from cocoindex.connectors import localfs +from cocoindex.connectors import sqlite as coco_sqlite +from cocoindex.inspect import iter_stable_paths_by_name + +from .file_walk import build_matcher, iter_included_files +from .settings import cocoindex_db_path, load_project_settings, target_sqlite_db_path +from .shared import APP_NAME, CODEBASE_DIR, SQLITE_DB + +FILE_MANIFEST_APP_NAME = f"{APP_NAME}FileManifest" +_FILE_MANIFEST_TABLE = "code_file_manifest" +_CREATE_FILE_MANIFEST_TABLE = f""" +CREATE TABLE IF NOT EXISTS {_FILE_MANIFEST_TABLE} ( + path TEXT PRIMARY KEY NOT NULL, + fingerprint BLOB NOT NULL +) +""" + + +@dataclass(frozen=True) +class IndexChanges: + added: tuple[str, ...] + updated: tuple[str, ...] | None + deleted: tuple[str, ...] + + +class _FileManifestAction(NamedTuple): + path: str + fingerprint: bytes | None + + +class _FileManifestHandler(coco.TargetHandler[bytes, bytes]): + def __init__(self) -> None: + self._sink = coco.TargetActionSink[_FileManifestAction, None].from_fn(self._apply) + + @staticmethod + def _apply( + context_provider: coco.ContextProvider, + actions: Sequence[_FileManifestAction], + /, + ) -> None: + db = context_provider.get(SQLITE_DB) + with db.transaction() as conn: + conn.execute(_CREATE_FILE_MANIFEST_TABLE) + conn.executemany( + f"DELETE FROM {_FILE_MANIFEST_TABLE} WHERE path = ?", + [(action.path,) for action in actions if action.fingerprint is None], + ) + conn.executemany( + f""" + INSERT INTO {_FILE_MANIFEST_TABLE} (path, fingerprint) + VALUES (?, ?) + ON CONFLICT (path) DO UPDATE SET fingerprint = excluded.fingerprint + """, + [ + (action.path, action.fingerprint) + for action in actions + if action.fingerprint is not None + ], + ) + + def reconcile( + self, + key: coco.StableKey, + desired_state: bytes | coco.NonExistenceType, + prev_possible_records: Collection[bytes], + prev_may_be_missing: bool, + /, + ) -> coco.TargetReconcileOutput[_FileManifestAction, bytes] | None: + assert isinstance(key, str) + if coco.is_non_existence(desired_state): + if not prev_possible_records and not prev_may_be_missing: + return None + return coco.TargetReconcileOutput( + action=_FileManifestAction(key, None), + sink=self._sink, + tracking_record=coco.NON_EXISTENCE, + ) + + if not prev_may_be_missing and all( + previous == desired_state for previous in prev_possible_records + ): + return None + + return coco.TargetReconcileOutput( + action=_FileManifestAction(key, desired_state), + sink=self._sink, + tracking_record=desired_state, + ) + + +_FILE_MANIFEST_PROVIDER = coco.register_root_target_states_provider( + "cocoindex_code/file_manifest", + _FileManifestHandler(), +) + + +@coco.fn(memo=True) +async def _track_file(file: localfs.File) -> None: + path = file.file_path.path.as_posix() + coco.declare_target_state( + _FILE_MANIFEST_PROVIDER.target_state(path, await file.content_fingerprint()) + ) + + +@coco.fn +async def _build_file_manifest() -> None: + project_root = coco.use_context(CODEBASE_DIR) + settings = load_project_settings(project_root) + matcher = build_matcher( + project_root, + settings.include_patterns, + settings.exclude_patterns, + settings.max_file_size, + ) + files = localfs.walk_dir( + CODEBASE_DIR, + recursive=True, + path_matcher=matcher, + ) + await coco.mount_each( + coco.component_subpath(coco.Symbol("track_file")), + _track_file, + files.items(), + ) + + +def create_file_manifest_app(env: coco.Environment) -> coco.App[[], None]: + """Create the app that persists source fingerprints after indexing.""" + return coco.App( + coco.AppConfig(name=FILE_MANIFEST_APP_NAME, environment=env), + _build_file_manifest, + ) + + +def prepare_file_manifest(db: coco_sqlite.ManagedConnection) -> None: + """Create the fingerprint table before the first manifest update.""" + with db.transaction() as conn: + conn.execute(_CREATE_FILE_MANIFEST_TABLE) + + +async def find_index_changes(project_root: Path) -> IndexChanges: + """Return files the next index would add, update, or delete.""" + indexed = await asyncio.to_thread(_indexed_fingerprints, project_root) + if indexed is not None: + matched = await asyncio.to_thread(_matched_fingerprints, project_root) + matched_paths = set(matched) + indexed_paths = set(indexed) + return IndexChanges( + added=tuple(sorted(matched_paths - indexed_paths)), + updated=tuple( + sorted( + path for path in matched_paths & indexed_paths if matched[path] != indexed[path] + ) + ), + deleted=tuple(sorted(indexed_paths - matched_paths)), + ) + + matched_paths, indexed_paths = await asyncio.gather( + asyncio.to_thread(_matched_paths, project_root), + _indexed_paths(project_root), + ) + return IndexChanges( + added=tuple(sorted(matched_paths - indexed_paths)), + updated=None if indexed_paths else (), + deleted=tuple(sorted(indexed_paths - matched_paths)), + ) + + +def _matched_fingerprints(project_root: Path) -> dict[str, bytes]: + fingerprints: dict[str, bytes] = {} + for absolute_path, relative_path in _iter_matched_files(project_root): + try: + fingerprints[relative_path.as_posix()] = fingerprint_bytes(absolute_path.read_bytes()) + except OSError: + continue + return fingerprints + + +def _matched_paths(project_root: Path) -> set[str]: + return {relative_path.as_posix() for _, relative_path in _iter_matched_files(project_root)} + + +def _iter_matched_files(project_root: Path) -> Iterator[tuple[Path, PurePath]]: + settings = load_project_settings(project_root) + matcher = build_matcher( + project_root, + settings.include_patterns, + settings.exclude_patterns, + settings.max_file_size, + ) + return iter_included_files(project_root, project_root, matcher) + + +def _indexed_fingerprints(project_root: Path) -> dict[str, bytes] | None: + db_path = target_sqlite_db_path(project_root) + if not db_path.exists(): + return None + + try: + conn = sqlite3.connect(f"{db_path.resolve().as_uri()}?mode=ro", uri=True) + try: + rows = conn.execute(f"SELECT path, fingerprint FROM {_FILE_MANIFEST_TABLE}").fetchall() + finally: + conn.close() + except sqlite3.OperationalError: + return None + return {path: fingerprint for path, fingerprint in rows} + + +async def _indexed_paths(project_root: Path) -> set[str]: + db_path = cocoindex_db_path(project_root) + if not db_path.exists(): + return set() + + env = coco.Environment(coco.Settings.from_env(db_path)) + paths: set[str] = set() + async for item in iter_stable_paths_by_name(env, APP_NAME): + parts = coco.StablePath(item.path).parts() + if ( + len(parts) == 2 + and isinstance(parts[0], coco.Symbol) + and parts[0].name == "process_file" + and isinstance(parts[1], str) + ): + paths.add(parts[1]) + return paths diff --git a/src/cocoindex_code/project.py b/src/cocoindex_code/project.py index 9849f77..12c42a5 100644 --- a/src/cocoindex_code/project.py +++ b/src/cocoindex_code/project.py @@ -13,6 +13,7 @@ from cocoindex.connectors import sqlite as coco_sqlite from .chunking import CHUNKER_REGISTRY, ChunkerFn +from .index_changes import create_file_manifest_app, prepare_file_manifest from .indexer import indexer_main from .protocol import ( IndexingProgress, @@ -34,6 +35,7 @@ target_sqlite_db_path as _target_sqlite_db_path, ) from .shared import ( + APP_NAME, CODEBASE_DIR, EMBEDDER, INDEXING_EMBED_PARAMS, @@ -49,6 +51,7 @@ class Project: _env: coco.Environment _app: coco.App[[], None] + _file_manifest_app: coco.App[[], None] _project_root: Path _index_lock: asyncio.Lock _clear_mps_cache_after_index: bool @@ -115,6 +118,7 @@ async def _run_index_inner( if on_progress is not None: on_progress(progress) await asyncio.sleep(0.1) + await self._file_manifest_app.update() finally: try: if self._clear_mps_cache_after_index: @@ -315,9 +319,12 @@ async def create( settings = coco.Settings.from_env(cocoindex_db) + target_db = coco_sqlite.connect(str(target_sqlite_db), load_vec=True) + prepare_file_manifest(target_db) + context = coco.ContextProvider() context.provide(CODEBASE_DIR, project_root) - context.provide(SQLITE_DB, coco_sqlite.connect(str(target_sqlite_db), load_vec=True)) + context.provide(SQLITE_DB, target_db) context.provide(EMBEDDER, embedder) context.provide(INDEXING_EMBED_PARAMS, dict(indexing_params)) context.provide(QUERY_EMBED_PARAMS, dict(query_params)) @@ -326,7 +333,7 @@ async def create( env = coco.Environment(settings, context_provider=context) app = coco.App( coco.AppConfig( - name="CocoIndexCode", + name=APP_NAME, environment=env, ), indexer_main, @@ -335,6 +342,7 @@ async def create( result = Project.__new__(Project) result._env = env result._app = app + result._file_manifest_app = create_file_manifest_app(env) result._project_root = project_root result._index_lock = asyncio.Lock() result._clear_mps_cache_after_index = clear_mps_cache_after_index diff --git a/src/cocoindex_code/shared.py b/src/cocoindex_code/shared.py index 08a71ae..166d093 100644 --- a/src/cocoindex_code/shared.py +++ b/src/cocoindex_code/shared.py @@ -24,6 +24,7 @@ logger = logging.getLogger(__name__) +APP_NAME = "CocoIndexCode" SBERT_PREFIX = "sbert/" DEFAULT_LITELLM_MIN_INTERVAL_MS = 5 diff --git a/tests/test_cli_helpers.py b/tests/test_cli_helpers.py index 461c68c..c2f94bc 100644 --- a/tests/test_cli_helpers.py +++ b/tests/test_cli_helpers.py @@ -6,6 +6,8 @@ from pathlib import Path import pytest +from click import unstyle +from typer.testing import CliRunner from cocoindex_code import cli from cocoindex_code.cli import ( @@ -198,6 +200,13 @@ def test_resolve_default_path_outside_project( assert result is None +def test_index_help_includes_dry_option() -> None: + result = CliRunner().invoke(cli.app, ["index", "--help"], catch_exceptions=False) + + assert result.exit_code == 0 + assert "--dry" in unstyle(result.output) + + # --------------------------------------------------------------------------- # .gitignore helpers # --------------------------------------------------------------------------- diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 17894f0..59ae196 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -28,6 +28,7 @@ load_user_settings, save_project_settings, save_user_settings, + target_sqlite_db_path, user_settings_path, ) @@ -223,6 +224,37 @@ def test_session_incremental_index(e2e_project: Path) -> None: assert "app.js" in result.output +def test_session_index_dry_lists_changes_without_indexing(e2e_project: Path) -> None: + runner.invoke(app, ["init"], catch_exceptions=False) + result = runner.invoke(app, ["index"], catch_exceptions=False) + assert result.exit_code == 0, result.output + + (e2e_project / "app.js").write_text(SAMPLE_APP_JS) + (e2e_project / "empty.py").write_text("") + (e2e_project / "main.py").write_text(SAMPLE_MAIN_PY.replace("World", "Nautilus")) + (e2e_project / "utils.py").unlink() + + first = runner.invoke(app, ["index", "--dry"], catch_exceptions=False) + second = runner.invoke(app, ["index", "--dry"], catch_exceptions=False) + + assert first.exit_code == 0, first.output + assert first.output == second.output + assert "Files to add (2):\n app.js\n empty.py\n" in first.output + assert "Files to update (1):\n main.py\n" in first.output + assert "Files to delete (1):\n utils.py\n" in first.output + + conn = coco_sqlite.connect(str(target_sqlite_db_path(e2e_project)), load_vec=True) + try: + with conn.readonly() as db: + indexed_paths = { + row[0] for row in db.execute("SELECT DISTINCT file_path FROM code_chunks_vec") + } + finally: + conn.close() + + assert indexed_paths == {"lib/database.py", "main.py", "utils.py"} + + def test_session_reset_databases(e2e_project: Path) -> None: """Init → index → search → reset (dbs only) → re-index → search works again.""" runner.invoke(app, ["init"], catch_exceptions=False) diff --git a/tests/test_index_changes.py b/tests/test_index_changes.py new file mode 100644 index 0000000..a3fe81d --- /dev/null +++ b/tests/test_index_changes.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +from pathlib import Path + +import cocoindex as coco +import pytest +from cocoindex.connectors import sqlite as coco_sqlite + +from cocoindex_code import index_changes +from cocoindex_code.settings import ProjectSettings, save_project_settings + + +@pytest.mark.asyncio +async def test_find_index_changes_without_existing_index(tmp_path: Path) -> None: + save_project_settings( + tmp_path, + ProjectSettings( + include_patterns=["**/*.py"], + exclude_patterns=[], + max_file_size=15, + ), + ) + (tmp_path / "main.py").write_text("print('main')\n") + (tmp_path / "oversized.py").write_text("x" * 16) + (tmp_path / "notes.txt").write_text("not indexed\n") + + changes = await index_changes.find_index_changes(tmp_path) + + assert changes.added == ("main.py",) + assert changes.updated == () + assert changes.deleted == () + + +@pytest.mark.asyncio +async def test_find_index_changes_compares_persisted_paths( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + save_project_settings( + tmp_path, + ProjectSettings(include_patterns=["**/*.py"], exclude_patterns=[]), + ) + (tmp_path / "existing.py").write_text("print('existing')\n") + (tmp_path / "new.py").write_text("print('new')\n") + + async def indexed_paths(_project_root: Path) -> set[str]: + return {"deleted.py", "existing.py"} + + monkeypatch.setattr(index_changes, "_indexed_paths", indexed_paths) + + changes = await index_changes.find_index_changes(tmp_path) + + assert changes.added == ("new.py",) + assert changes.updated is None + assert changes.deleted == ("deleted.py",) + + +@pytest.mark.asyncio +async def test_find_index_changes_compares_manifest_fingerprints(tmp_path: Path) -> None: + save_project_settings( + tmp_path, + ProjectSettings(include_patterns=["**/*.py"], exclude_patterns=[]), + ) + (tmp_path / "modified.py").write_text("value = 1\n") + (tmp_path / "deleted.py").write_text("value = 2\n") + + context = coco.ContextProvider() + context.provide(index_changes.CODEBASE_DIR, tmp_path) + target_db = coco_sqlite.connect( + str(index_changes.target_sqlite_db_path(tmp_path)), + load_vec=True, + ) + index_changes.prepare_file_manifest(target_db) + context.provide(index_changes.SQLITE_DB, target_db) + env = coco.Environment( + coco.Settings.from_env(index_changes.cocoindex_db_path(tmp_path)), + context_provider=context, + ) + app = index_changes.create_file_manifest_app(env) + await app.update() + data_path = index_changes.target_sqlite_db_path(tmp_path) + data_before = data_path.read_bytes() + + (tmp_path / "modified.py").write_text("value = 3\n") + (tmp_path / "deleted.py").unlink() + (tmp_path / "added.py").write_text("value = 4\n") + + first = await index_changes.find_index_changes(tmp_path) + second = await index_changes.find_index_changes(tmp_path) + data_after = data_path.read_bytes() + target_db.close() + + assert first == second + assert first.added == ("added.py",) + assert first.updated == ("modified.py",) + assert first.deleted == ("deleted.py",) + assert data_after == data_before diff --git a/tests/test_project_indexing.py b/tests/test_project_indexing.py index 0db593a..8d28289 100644 --- a/tests/test_project_indexing.py +++ b/tests/test_project_indexing.py @@ -4,6 +4,7 @@ import asyncio from collections.abc import AsyncIterator, Callable +from types import SimpleNamespace from typing import Any, cast from unittest.mock import AsyncMock @@ -56,6 +57,11 @@ def __init__( Any, _ControlledApp(_WatchHandle(on_enter=on_enter, on_exit=on_exit, release=release)), ) + self.file_manifest_update = AsyncMock() + self._file_manifest_app = cast( + Any, + SimpleNamespace(update=self.file_manifest_update), + ) self._index_lock = asyncio.Lock() self._clear_mps_cache_after_index = clear_mps_cache_after_index self._initial_index_done = asyncio.Event() @@ -93,6 +99,8 @@ def exit() -> None: release.set() await asyncio.gather(first_task, second_task) assert active == 0 + first.file_manifest_update.assert_awaited_once_with() + second.file_manifest_update.assert_awaited_once_with() async def test_concurrent_initial_index_requests_share_one_background_task() -> None: @@ -117,6 +125,7 @@ def enter() -> None: release.set() await project.wait_for_indexing_done() assert execution_count == 1 + project.file_manifest_update.assert_awaited_once_with() async def test_mps_allocator_cache_is_cleared_after_indexing(