diff --git a/src/android_watcher/store.py b/src/android_watcher/store.py index 284fc80..af3c8b8 100644 --- a/src/android_watcher/store.py +++ b/src/android_watcher/store.py @@ -488,6 +488,13 @@ def mark_successful_run(self, when: datetime) -> None: # Tables carried in a seed; run_state's seed_date marker is appended separately. _SEED_TABLES = ("snapshots", "seen_feed_items", "http_cache") + # Columns held back from the seed. content_text is the full page body, a local + # runtime cache used only to diff a page against its prior version; it is not + # needed to detect changes (content_hash does that) and shipping it would bloat + # the seed past GitHub's 100MB file limit and the wheel. Seeded rows import it + # empty and self-heal on the first change per page. + _SEED_EXCLUDED_COLS = frozenset({"content_text"}) + def seed_date(self) -> str | None: """The date the imported baseline was generated, or None if unseeded.""" row = self._conn.execute("SELECT value FROM run_state WHERE key = 'seed_date'").fetchone() @@ -503,7 +510,7 @@ def export_seed_sql(self, seed_date: str) -> str: for table in self._SEED_TABLES: rows = self._conn.execute(f"SELECT * FROM {table}").fetchall() for row in rows: - cols = row.keys() + cols = [c for c in row.keys() if c not in self._SEED_EXCLUDED_COLS] vals = ", ".join(_sql_str(row[c]) for c in cols) lines.append(f"INSERT OR IGNORE INTO {table} ({', '.join(cols)}) VALUES ({vals});") lines.append( diff --git a/tests/test_seed.py b/tests/test_seed.py index 39dcb5a..392699c 100644 --- a/tests/test_seed.py +++ b/tests/test_seed.py @@ -23,6 +23,42 @@ def _store(tmp_path, name="state.db"): return store +def test_seed_export_excludes_content_text(tmp_path): + """The seed must not carry full page bodies (content_text). + + content_text is a local runtime cache used to diff a page against its prior + version; it is not needed to detect changes (content_hash does that) and it + bloats the shipped seed past GitHub's 100MB file limit and the wheel. On a + seeded install it imports empty and self-heals on the first change per page. + """ + src = _store(tmp_path, "src.db") + big_body = "UNIQUE_BODY_MARKER " * 1000 + src.upsert_snapshot( + "s1", + "https://developer.android.com/x", + signal_type="sitemap", + content_hash="h", + lastmod="2026-06-20", + excerpt="short excerpt", + content_text=big_body, + ) + sql = src.export_seed_sql("2026-06-21") + src.close() + + # The full body must not appear anywhere in the seed. + assert "UNIQUE_BODY_MARKER" not in sql + assert "content_text" not in sql + + # Round-trip still works: hash + excerpt survive, content_text imports empty. + dst = _store(tmp_path, "dst.db") + dst.import_seed_sql(sql) + snap = dst.get_snapshot("s1", "https://developer.android.com/x") + assert snap is not None + assert snap.content_hash == "h" + assert snap.excerpt == "short excerpt" + assert snap.content_text == "" + + def test_export_import_roundtrip(tmp_path): src = _store(tmp_path, "src.db") src.upsert_snapshot(