Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion src/android_watcher/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -488,6 +488,13 @@ def mark_successful_run(self, when: datetime) -> None:
# Tables carried in a seed; run_state's seed_date marker is appended separately.
_SEED_TABLES = ("snapshots", "seen_feed_items", "http_cache")

# Columns held back from the seed. content_text is the full page body, a local
# runtime cache used only to diff a page against its prior version; it is not
# needed to detect changes (content_hash does that) and shipping it would bloat
# the seed past GitHub's 100MB file limit and the wheel. Seeded rows import it
# empty and self-heal on the first change per page.
_SEED_EXCLUDED_COLS = frozenset({"content_text"})

def seed_date(self) -> str | None:
"""The date the imported baseline was generated, or None if unseeded."""
row = self._conn.execute("SELECT value FROM run_state WHERE key = 'seed_date'").fetchone()
Expand All @@ -503,7 +510,7 @@ def export_seed_sql(self, seed_date: str) -> str:
for table in self._SEED_TABLES:
rows = self._conn.execute(f"SELECT * FROM {table}").fetchall()
for row in rows:
cols = row.keys()
cols = [c for c in row.keys() if c not in self._SEED_EXCLUDED_COLS]
vals = ", ".join(_sql_str(row[c]) for c in cols)
lines.append(f"INSERT OR IGNORE INTO {table} ({', '.join(cols)}) VALUES ({vals});")
lines.append(
Expand Down
36 changes: 36 additions & 0 deletions tests/test_seed.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,42 @@ def _store(tmp_path, name="state.db"):
return store


def test_seed_export_excludes_content_text(tmp_path):
"""The seed must not carry full page bodies (content_text).

content_text is a local runtime cache used to diff a page against its prior
version; it is not needed to detect changes (content_hash does that) and it
bloats the shipped seed past GitHub's 100MB file limit and the wheel. On a
seeded install it imports empty and self-heals on the first change per page.
"""
src = _store(tmp_path, "src.db")
big_body = "UNIQUE_BODY_MARKER " * 1000
src.upsert_snapshot(
"s1",
"https://developer.android.com/x",
signal_type="sitemap",
content_hash="h",
lastmod="2026-06-20",
excerpt="short excerpt",
content_text=big_body,
)
sql = src.export_seed_sql("2026-06-21")
src.close()

# The full body must not appear anywhere in the seed.
assert "UNIQUE_BODY_MARKER" not in sql
assert "content_text" not in sql

# Round-trip still works: hash + excerpt survive, content_text imports empty.
dst = _store(tmp_path, "dst.db")
dst.import_seed_sql(sql)
snap = dst.get_snapshot("s1", "https://developer.android.com/x")
assert snap is not None
assert snap.content_hash == "h"
assert snap.excerpt == "short excerpt"
assert snap.content_text == ""


def test_export_import_roundtrip(tmp_path):
src = _store(tmp_path, "src.db")
src.upsert_snapshot(
Expand Down
Loading