From ab9c2a7baa462c08ede3cbc3b0b31d83073638b6 Mon Sep 17 00:00:00 2001 From: guillaumemichel Date: Fri, 3 Jul 2026 15:46:03 +0200 Subject: [PATCH 1/5] fix(blockstore): don't activate bloom cache if build incomplete --- CHANGELOG.md | 13 + blockstore/allkeyschanwitherr_test.go | 527 ++++++++++++++++++++++++++ blockstore/blockstore.go | 65 +++- blockstore/bloom_cache.go | 30 +- blockstore/idstore.go | 11 +- blockstore/twoqueue_cache.go | 9 +- blockstore/validating_blockstore.go | 6 + 7 files changed, 649 insertions(+), 12 deletions(-) create mode 100644 blockstore/allkeyschanwitherr_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index f6acdf791..3acdaa533 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,19 @@ The following emojis are used to highlight certain changes: ### Fixed +- `blockstore`: the Bloom filter cache no longer activates after an incomplete +build. Previously, if `AllKeysChan` enumeration was truncated by a +mid-iteration datastore error (which was only logged, never propagated) or by a +cancelled context, the cache treated the closed channel as a complete build and +activated a Bloom filter holding only a subset of the stored CIDs. It then +answered "not present" conclusively for blocks that exist but were never +indexed, reporting present blocks as missing (`Has` returns false, +`Get`/`GetSize`/`View` return not-found, `DeleteBlock` becomes a silent no-op). +The build now activates the filter only when enumeration is known to have +completed; otherwise the cache degrades to correct pass-through. This also +fixes a race where a cancelled build could still mark the filter active. +[#NNNN](https://github.com/ipfs/boxo/pull/NNNN) + ### Security diff --git a/blockstore/allkeyschanwitherr_test.go b/blockstore/allkeyschanwitherr_test.go new file mode 100644 index 000000000..fc7c9142a --- /dev/null +++ b/blockstore/allkeyschanwitherr_test.go @@ -0,0 +1,527 @@ +package blockstore + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + blocks "github.com/ipfs/go-block-format" + cid "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + dsq "github.com/ipfs/go-datastore/query" + syncds "github.com/ipfs/go-datastore/sync" + ipld "github.com/ipfs/go-ipld-format" +) + +var errInjected = errors.New("simulated mid-iteration datastore error") + +// errAfterDS wraps a datastore and injects a mid-iteration query error after +// `after` entries have been streamed, simulating an I/O or iterator error +// partway through enumeration (a documented possibility: see dsq.Result.Error). +type errAfterDS struct { + ds.Batching + after int +} + +func (d *errAfterDS) Query(ctx context.Context, q dsq.Query) (dsq.Results, error) { + res, err := d.Batching.Query(ctx, q) + if err != nil { + return nil, err + } + return dsq.ResultsWithContext(q, func(ctx context.Context, out chan<- dsq.Result) { + defer res.Close() + n := 0 + for { + r, ok := res.NextSync() + if !ok { + return + } + if r.Error != nil { + out <- r + return + } + if n >= d.after { + out <- dsq.Result{Error: errInjected} + return + } + select { + case <-ctx.Done(): + return + case out <- r: + } + n++ + } + }), nil +} + +// cleanChanErrFnBS is a Blockstore whose allKeysChanWithErr returns a channel +// that closes normally, yet reports a non-nil enumeration error. It isolates +// the core invariant: the Bloom build must trust the reported error, not the +// fact that the channel closed. +type cleanChanErrFnBS struct { + Blockstore + enumErr error +} + +func (f *cleanChanErrFnBS) allKeysChanWithErr(ctx context.Context) (<-chan cid.Cid, func() error, error) { + ch, err := f.AllKeysChan(ctx) + if err != nil { + return nil, nil, err + } + return ch, func() error { return f.enumErr }, nil +} + +func TestAllKeysChanWithErrClean(t *testing.T) { + bs, keys := newBlockStoreWithKeys(t, nil, 100) + + e := bs.(allKeysChanWithErrer) + ch, errFn, err := e.allKeysChanWithErr(t.Context()) + if err != nil { + t.Fatal(err) + } + got := collect(ch) + if err := errFn(); err != nil { + t.Fatalf("errFn returned error on clean enumeration: %v", err) + } + expectMatches(t, keys, got) +} + +func TestAllKeysChanWithErrMidIteration(t *testing.T) { + const total = 100 + const after = 10 + + under := syncds.MutexWrap(ds.NewMapDatastore()) + bs := NewBlockstore(&errAfterDS{Batching: under, after: after}) + for i := range total { + if err := bs.Put(t.Context(), blocks.NewBlock(fmt.Appendf(nil, "data %d", i))); err != nil { + t.Fatal(err) + } + } + + e := bs.(allKeysChanWithErrer) + ch, errFn, err := e.allKeysChanWithErr(t.Context()) + if err != nil { + t.Fatal(err) + } + got := collect(ch) + + iterErr := errFn() + if iterErr == nil { + t.Fatal("expected errFn to report the mid-iteration error, got nil") + } + if !errors.Is(iterErr, errInjected) { + t.Fatalf("expected wrapped errInjected, got: %v", iterErr) + } + if len(got) >= total { + t.Fatalf("expected partial enumeration (< %d keys), got %d", total, len(got)) + } +} + +func TestAllKeysChanWithErrContextCancel(t *testing.T) { + // More keys than the channel buffer so the producer is forced to block on + // a send once we stop draining, making the cancellation deterministic. + n := 2*dsq.KeysOnlyBufSize + 10 + bs, _ := newBlockStoreWithKeys(t, nil, n) + + e := bs.(allKeysChanWithErrer) + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + ch, errFn, err := e.allKeysChanWithErr(ctx) + if err != nil { + t.Fatal(err) + } + + <-ch // ensure enumeration started, then stop draining + cancel() + + if iterErr := errFn(); !errors.Is(iterErr, context.Canceled) { + t.Fatalf("expected context.Canceled, got: %v", iterErr) + } +} + +func TestBloomBuildDoesNotActivateOnIncompleteEnumeration(t *testing.T) { + under, keys := newBlockStoreWithKeys(t, nil, 100) + enumErr := errors.New("incomplete enumeration") + fake := &cleanChanErrFnBS{Blockstore: under, enumErr: enumErr} + + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + t.Cleanup(cancel) + + cachedbs, err := testBloomCached(ctx, fake) + if err != nil { + t.Fatal(err) + } + + waitErr := cachedbs.Wait(ctx) + if waitErr == nil { + t.Fatal("expected Wait to return the build error, got nil") + } + if !errors.Is(waitErr, enumErr) { + t.Fatalf("expected wrapped enumErr, got: %v", waitErr) + } + if cachedbs.BloomActive() { + t.Fatal("bloom must not be active after an incomplete build") + } + + // Despite the failed build, the cache must answer correctly by falling + // through to the underlying store. + for _, k := range keys { + has, err := cachedbs.Has(t.Context(), k) + if err != nil { + t.Fatal(err) + } + if !has { + t.Fatalf("present block %s reported missing after incomplete build", k) + } + } + absent := blocks.NewBlock([]byte("definitely-absent")).Cid() + if has, err := cachedbs.Has(t.Context(), absent); err != nil || has { + t.Fatalf("absent block reported present (has=%v err=%v)", has, err) + } + + // Get / GetSize must also pass through to the underlying store, and + // DeleteBlock must actually reach it (not be skipped as a no-op). + if blk, err := cachedbs.Get(t.Context(), keys[0]); err != nil || blk == nil { + t.Fatalf("Get of present block failed under inactive bloom (blk=%v err=%v)", blk, err) + } + if sz, err := cachedbs.GetSize(t.Context(), keys[0]); err != nil || sz < 0 { + t.Fatalf("GetSize of present block failed under inactive bloom (sz=%d err=%v)", sz, err) + } + delKey := keys[len(keys)-1] + if err := cachedbs.DeleteBlock(t.Context(), delKey); err != nil { + t.Fatal(err) + } + if has, err := cachedbs.Has(t.Context(), delKey); err != nil || has { + t.Fatalf("DeleteBlock under inactive bloom did not delete (has=%v err=%v)", has, err) + } +} + +func TestBloomBuildMidIterationErrorEndToEnd(t *testing.T) { + const total = 1000 + const after = 100 + + under := syncds.MutexWrap(ds.NewMapDatastore()) + bs := NewBlockstore(&errAfterDS{Batching: under, after: after}) + keys := make([]cid.Cid, 0, total) + for i := range total { + b := blocks.NewBlock(fmt.Appendf(nil, "data %d", i)) + if err := bs.Put(t.Context(), b); err != nil { + t.Fatal(err) + } + keys = append(keys, b.Cid()) + } + + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + t.Cleanup(cancel) + + cachedbs, err := testBloomCached(ctx, bs) + if err != nil { + t.Fatal(err) + } + + waitErr := cachedbs.Wait(ctx) + if waitErr == nil { + t.Fatal("expected Wait to return the incomplete-build error, got nil") + } + if !errors.Is(waitErr, errInjected) { + t.Fatalf("expected wrapped errInjected, got: %v", waitErr) + } + if cachedbs.BloomActive() { + t.Fatal("bloom must not be active after a mid-iteration enumeration error") + } + + // All present blocks must still be reported present (pass-through). + for _, k := range keys { + has, err := cachedbs.Has(t.Context(), k) + if err != nil { + t.Fatal(err) + } + if !has { + t.Fatalf("present block %s reported missing", k) + } + } +} + +func TestBloomBuildCleanPathActivates(t *testing.T) { + under, keys := newBlockStoreWithKeys(t, nil, 200) + + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + t.Cleanup(cancel) + + cachedbs, err := testBloomCached(ctx, under) + if err != nil { + t.Fatal(err) + } + + if err := cachedbs.Wait(ctx); err != nil { + t.Fatalf("clean build returned error: %v", err) + } + if !cachedbs.BloomActive() { + t.Fatal("bloom should be active after a clean build") + } + + for _, k := range keys { + if has, err := cachedbs.Has(t.Context(), k); err != nil || !has { + t.Fatalf("present block reported missing (has=%v err=%v)", has, err) + } + } + absent := blocks.NewBlock([]byte("nope-not-here")).Cid() + if has, err := cachedbs.Has(t.Context(), absent); err != nil || has { + t.Fatalf("absent block reported present (has=%v err=%v)", has, err) + } +} + +func TestAllKeysChanWithErrForwardsThroughCacheStack(t *testing.T) { + const total = 100 + const after = 10 + + under := syncds.MutexWrap(ds.NewMapDatastore()) + bs := NewBlockstore(&errAfterDS{Batching: under, after: after}) + for i := range total { + if err := bs.Put(t.Context(), blocks.NewBlock(fmt.Appendf(nil, "data %d", i))); err != nil { + t.Fatal(err) + } + } + + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + t.Cleanup(cancel) + + // Default opts wrap in BOTH a two-queue cache and a Bloom filter, so the + // capability must traverse bloomcache -> tqcache -> base blockstore. + cbs, err := CachedBlockstore(ctx, bs, DefaultCacheOpts()) + if err != nil { + t.Fatal(err) + } + + e, ok := cbs.(allKeysChanWithErrer) + if !ok { + t.Fatal("CachedBlockstore result does not implement allKeysChanWithErrer") + } + ch, errFn, err := e.allKeysChanWithErr(ctx) + if err != nil { + t.Fatal(err) + } + _ = collect(ch) + if iterErr := errFn(); !errors.Is(iterErr, errInjected) { + t.Fatalf("expected errInjected to propagate through the cache stack, got: %v", iterErr) + } +} + +// legacyBS implements only the Blockstore interface (no allKeysChanWithErrer), +// to exercise the best-effort fallback in allKeysChanWithErrFor. It must NOT +// embed *blockstore, or the capability method would be promoted and the +// fallback branch would never run. +type legacyBS struct{ inner Blockstore } + +func (b *legacyBS) DeleteBlock(ctx context.Context, k cid.Cid) error { + return b.inner.DeleteBlock(ctx, k) +} +func (b *legacyBS) Has(ctx context.Context, k cid.Cid) (bool, error) { return b.inner.Has(ctx, k) } +func (b *legacyBS) Get(ctx context.Context, k cid.Cid) (blocks.Block, error) { + return b.inner.Get(ctx, k) +} + +func (b *legacyBS) GetSize(ctx context.Context, k cid.Cid) (int, error) { + return b.inner.GetSize(ctx, k) +} +func (b *legacyBS) Put(ctx context.Context, bl blocks.Block) error { return b.inner.Put(ctx, bl) } +func (b *legacyBS) PutMany(ctx context.Context, bls []blocks.Block) error { + return b.inner.PutMany(ctx, bls) +} + +func (b *legacyBS) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + return b.inner.AllKeysChan(ctx) +} + +func TestAllKeysChanWithErrFallbackForLegacyBlockstore(t *testing.T) { + under, keys := newBlockStoreWithKeys(t, nil, 100) + legacy := &legacyBS{inner: under} + + // Guard the premise: legacyBS must NOT advertise the capability, else the + // fallback branch is not exercised. + if _, ok := Blockstore(legacy).(allKeysChanWithErrer); ok { + t.Fatal("legacyBS unexpectedly implements allKeysChanWithErrer") + } + + // The fallback yields a no-op errFn and a full enumeration. + ch, errFn, err := allKeysChanWithErrFor(t.Context(), legacy) + if err != nil { + t.Fatal(err) + } + got := collect(ch) + if err := errFn(); err != nil { + t.Fatalf("fallback errFn must return nil, got: %v", err) + } + expectMatches(t, keys, got) + + // Over a fully-enumerable legacy store, the bloom still activates. + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + t.Cleanup(cancel) + + cachedbs, err := testBloomCached(ctx, legacy) + if err != nil { + t.Fatal(err) + } + if err := cachedbs.Wait(ctx); err != nil { + t.Fatalf("clean build over legacy store returned error: %v", err) + } + if !cachedbs.BloomActive() { + t.Fatal("bloom should activate over a fully-enumerable legacy store") + } + for _, k := range keys { + if has, err := cachedbs.Has(t.Context(), k); err != nil || !has { + t.Fatalf("present block reported missing (has=%v err=%v)", has, err) + } + } +} + +func TestAllKeysChanWithErrForwardingWrappers(t *testing.T) { + const total = 100 + const after = 10 + + newBaseWithErr := func() Blockstore { + under := syncds.MutexWrap(ds.NewMapDatastore()) + bs := NewBlockstore(&errAfterDS{Batching: under, after: after}) + for i := range total { + if err := bs.Put(t.Context(), blocks.NewBlock(fmt.Appendf(nil, "data %d", i))); err != nil { + t.Fatal(err) + } + } + return bs + } + + wrappers := map[string]func(Blockstore) Blockstore{ + "idstore": func(bs Blockstore) Blockstore { return NewIdStore(bs) }, + "ValidatingBlockstore": func(bs Blockstore) Blockstore { return &ValidatingBlockstore{Blockstore: bs} }, + } + + for name, wrap := range wrappers { + t.Run(name, func(t *testing.T) { + w := wrap(newBaseWithErr()) + e, ok := w.(allKeysChanWithErrer) + if !ok { + t.Fatalf("%s does not implement allKeysChanWithErrer", name) + } + ch, errFn, err := e.allKeysChanWithErr(t.Context()) + if err != nil { + t.Fatal(err) + } + _ = collect(ch) + if iterErr := errFn(); !errors.Is(iterErr, errInjected) { + t.Fatalf("%s did not propagate enumeration error, got: %v", name, iterErr) + } + }) + } +} + +func TestAllKeysChanWithErrSkipsUnparseableKey(t *testing.T) { + raw := syncds.MutexWrap(ds.NewMapDatastore()) + bs := NewBlockstore(raw) + + const total = 50 + keys := make([]cid.Cid, 0, total) + for i := range total { + b := blocks.NewBlock(fmt.Appendf(nil, "data %d", i)) + if err := bs.Put(t.Context(), b); err != nil { + t.Fatal(err) + } + keys = append(keys, b.Cid()) + } + + // Inject a datastore key under the block prefix that is not valid base32, + // so BinaryFromDsKey fails to parse it during enumeration. + badKey := BlockPrefix.ChildString("this-is-not-base32") + if err := raw.Put(t.Context(), badKey, []byte("garbage")); err != nil { + t.Fatal(err) + } + + e := bs.(allKeysChanWithErrer) + ch, errFn, err := e.allKeysChanWithErr(t.Context()) + if err != nil { + t.Fatal(err) + } + got := collect(ch) + + // The unparseable key is skipped, NOT treated as a truncating error: it + // cannot correspond to a retrievable block, so it must not disable the + // completeness signal. + if err := errFn(); err != nil { + t.Fatalf("unparseable key must not be reported as an enumeration error, got: %v", err) + } + expectMatches(t, keys, got) + + // The bloom filter still activates over such a store. + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + t.Cleanup(cancel) + + cachedbs, err := testBloomCached(ctx, bs) + if err != nil { + t.Fatal(err) + } + if err := cachedbs.Wait(ctx); err != nil { + t.Fatalf("build over store with an unparseable key returned error: %v", err) + } + if !cachedbs.BloomActive() { + t.Fatal("bloom should activate despite an unparseable datastore key") + } +} + +// incompleteViewerBS reports an enumeration error (so the bloom build stays +// inactive) AND implements Viewer (so bloomcache wires up its viewer path), +// exercising bloomcache.View's pass-through gating while inactive. +type incompleteViewerBS struct { + Blockstore + enumErr error +} + +func (f *incompleteViewerBS) allKeysChanWithErr(ctx context.Context) (<-chan cid.Cid, func() error, error) { + ch, err := f.AllKeysChan(ctx) + if err != nil { + return nil, nil, err + } + return ch, func() error { return f.enumErr }, nil +} + +func (f *incompleteViewerBS) View(ctx context.Context, k cid.Cid, callback func([]byte) error) error { + blk, err := f.Get(ctx, k) + if err != nil { + return err + } + return callback(blk.RawData()) +} + +func TestBloomViewPassThroughWhenInactive(t *testing.T) { + under, keys := newBlockStoreWithKeys(t, nil, 50) + fake := &incompleteViewerBS{Blockstore: under, enumErr: errors.New("incomplete enumeration")} + + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + t.Cleanup(cancel) + + cachedbs, err := testBloomCached(ctx, fake) + if err != nil { + t.Fatal(err) + } + if err := cachedbs.Wait(ctx); err == nil { + t.Fatal("expected incomplete-build error") + } + if cachedbs.BloomActive() { + t.Fatal("bloom must be inactive") + } + + // View passes through to the underlying viewer for a present block. + var got []byte + if err := cachedbs.View(t.Context(), keys[0], func(b []byte) error { got = append(got, b...); return nil }); err != nil { + t.Fatalf("View of present block failed under inactive bloom: %v", err) + } + if len(got) == 0 { + t.Fatal("View returned empty data for a present block") + } + + // View of an absent block returns not-found. + absent := blocks.NewBlock([]byte("absent-view")).Cid() + if err := cachedbs.View(t.Context(), absent, func(b []byte) error { return nil }); !ipld.IsNotFound(err) { + t.Fatalf("View of absent block: expected not-found, got %v", err) + } +} diff --git a/blockstore/blockstore.go b/blockstore/blockstore.go index 32cb66865..828d99271 100644 --- a/blockstore/blockstore.go +++ b/blockstore/blockstore.go @@ -5,6 +5,7 @@ package blockstore import ( "context" "errors" + "fmt" "sync" "sync/atomic" @@ -59,6 +60,13 @@ type Blockstore interface { // // When underlying blockstore is operating on Multihash and codec information // is not preserved, returned CIDs will use Raw (0x55) codec. + // + // If enumeration fails partway (for example an I/O error mid-iteration or a + // cancelled context), the channel may be closed early without warning. + // Callers MUST NOT assume the returned error reflects enumeration + // completeness: it is only guaranteed to cover query setup, not + // mid-iteration failures. A consumer that requires a complete enumeration + // (such as building a Bloom filter) cannot rely on this method alone. AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) } @@ -284,18 +292,52 @@ func (bs *blockstore) DeleteBlock(ctx context.Context, k cid.Cid) error { // // AllKeysChan respects context. func (bs *blockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + ch, _, err := bs.allKeysChanWithErr(ctx) + return ch, err +} + +// allKeysChanWithErrer is an optional capability a Blockstore may implement +// alongside AllKeysChan. allKeysChanWithErr behaves like AllKeysChan, but +// additionally returns a function that, once the returned channel has been +// fully drained, reports any error that terminated enumeration early. +// +// The reported error is nil if enumeration ran to completion without a +// mid-iteration error or cancellation; a non-nil error means enumeration was +// truncated and the delivered keys must not be treated as the complete set. +// Datastore keys that cannot be parsed as a block key are skipped without being +// treated as an error: such a key cannot correspond to a retrievable block, so +// omitting it cannot cause a Bloom-filter false negative. +// +// The function blocks until enumeration finishes, so it must be called only +// after the channel has been drained; calling it earlier deadlocks the caller +// once the producer fills the channel buffer. +type allKeysChanWithErrer interface { + allKeysChanWithErr(ctx context.Context) (<-chan cid.Cid, func() error, error) +} + +var _ allKeysChanWithErrer = (*blockstore)(nil) + +// allKeysChanWithErr implements [allKeysChanWithErrer]. The returned function +// reports any error that ended key enumeration early (nil if every key was +// delivered) and must be called only after the channel has been drained. +func (bs *blockstore) allKeysChanWithErr(ctx context.Context) (<-chan cid.Cid, func() error, error) { // KeysOnly, because that would be _a lot_ of data. q := dsq.Query{KeysOnly: true} res, err := bs.datastore.Query(ctx, q) if err != nil { - return nil, err + return nil, nil, err } output := make(chan cid.Cid, dsq.KeysOnlyBufSize) + var iterErr error + done := make(chan struct{}) go func() { defer func() { res.Close() // ensure exit (signals early exit, too) close(output) + // done is closed after output so a reader that drains output and + // then calls the returned func observes the fully-written iterErr. + close(done) }() for { @@ -304,26 +346,43 @@ func (bs *blockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { return } if e.Error != nil { - logger.Errorf("blockstore.AllKeysChan got err: %s", e.Error) + iterErr = fmt.Errorf("blockstore.AllKeysChan iteration error: %w", e.Error) + logger.Error(iterErr) return } // need to convert to key.Key using key.KeyFromDsKey. bk, err := dshelp.BinaryFromDsKey(ds.RawKey(e.Key)) if err != nil { + // A key that cannot be parsed as a block key cannot correspond + // to a retrievable block, so skipping it (rather than treating + // it as a truncating error) cannot cause a Bloom false negative. logger.Warnf("error parsing key from binary: %s", err) continue } k := cid.NewCidV1(cid.Raw, bk) select { case <-ctx.Done(): + iterErr = ctx.Err() return case output <- k: } } }() - return output, nil + return output, func() error { <-done; return iterErr }, nil +} + +// allKeysChanWithErrFor returns bs.allKeysChanWithErr when bs implements +// [allKeysChanWithErrer]. Otherwise it falls back to [Blockstore.AllKeysChan] +// with a no-op error function, preserving best-effort behavior for blockstores +// that cannot report enumeration errors. +func allKeysChanWithErrFor(ctx context.Context, bs Blockstore) (<-chan cid.Cid, func() error, error) { + if e, ok := bs.(allKeysChanWithErrer); ok { + return e.allKeysChanWithErr(ctx) + } + ch, err := bs.AllKeysChan(ctx) + return ch, func() error { return nil }, err } // NewGCLocker returns a default implementation of diff --git a/blockstore/bloom_cache.go b/blockstore/bloom_cache.go index 8561cfd04..e90013d37 100644 --- a/blockstore/bloom_cache.go +++ b/blockstore/bloom_cache.go @@ -79,8 +79,9 @@ type bloomcache struct { } var ( - _ Blockstore = (*bloomcache)(nil) - _ Viewer = (*bloomcache)(nil) + _ Blockstore = (*bloomcache)(nil) + _ Viewer = (*bloomcache)(nil) + _ allKeysChanWithErrer = (*bloomcache)(nil) ) func (b *bloomcache) BloomActive() bool { @@ -104,15 +105,30 @@ func (b *bloomcache) build(ctx context.Context) error { }() defer close(b.buildChan) - ch, err := b.blockstore.AllKeysChan(ctx) + ch, errFn, err := allKeysChanWithErrFor(ctx, b.blockstore) if err != nil { - b.buildErr = fmt.Errorf("AllKeysChan failed in bloomcache rebuild with: %v", err) + b.buildErr = fmt.Errorf("AllKeysChan failed in bloomcache build with: %w", err) return b.buildErr } for { select { case key, ok := <-ch: if !ok { + // A closed channel alone does not prove the enumeration was + // complete: it could have been truncated by a mid-iteration + // error or a cancelled context. Trust the filter only if every + // key was delivered, otherwise the "not in bloom" answer would + // be a false negative for blocks that exist but were never + // indexed. + // + // If the wrapped store does not implement allKeysChanWithErrer, + // errFn is a no-op returning nil (see allKeysChanWithErrFor) and + // the filter is treated as complete, preserving the pre-existing + // best-effort behavior for such stores. + if err := errFn(); err != nil { + b.buildErr = fmt.Errorf("bloomcache build incomplete, not activating filter: %w", err) + return b.buildErr + } atomic.StoreInt32(&b.active, 1) return nil } @@ -124,6 +140,12 @@ func (b *bloomcache) build(ctx context.Context) error { } } +// allKeysChanWithErr forwards the error-reporting enumeration to the wrapped +// blockstore, so the completeness signal survives the cache stack. +func (b *bloomcache) allKeysChanWithErr(ctx context.Context) (<-chan cid.Cid, func() error, error) { + return allKeysChanWithErrFor(ctx, b.blockstore) +} + func (b *bloomcache) DeleteBlock(ctx context.Context, k cid.Cid) error { if has, ok := b.hasCached(k); ok && !has { return nil diff --git a/blockstore/idstore.go b/blockstore/idstore.go index 875d67320..49867d6c4 100644 --- a/blockstore/idstore.go +++ b/blockstore/idstore.go @@ -16,9 +16,10 @@ type idstore struct { } var ( - _ Blockstore = (*idstore)(nil) - _ Viewer = (*idstore)(nil) - _ io.Closer = (*idstore)(nil) + _ Blockstore = (*idstore)(nil) + _ Viewer = (*idstore)(nil) + _ io.Closer = (*idstore)(nil) + _ allKeysChanWithErrer = (*idstore)(nil) ) func NewIdStore(bs Blockstore) Blockstore { @@ -113,6 +114,10 @@ func (b *idstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { return b.bs.AllKeysChan(ctx) } +func (b *idstore) allKeysChanWithErr(ctx context.Context) (<-chan cid.Cid, func() error, error) { + return allKeysChanWithErrFor(ctx, b.bs) +} + func (b *idstore) Close() error { if c, ok := b.bs.(io.Closer); ok { return c.Close() diff --git a/blockstore/twoqueue_cache.go b/blockstore/twoqueue_cache.go index e0a35d414..5cc46bdc8 100644 --- a/blockstore/twoqueue_cache.go +++ b/blockstore/twoqueue_cache.go @@ -42,8 +42,9 @@ type tqcache struct { } var ( - _ Blockstore = (*tqcache)(nil) - _ Viewer = (*tqcache)(nil) + _ Blockstore = (*tqcache)(nil) + _ Viewer = (*tqcache)(nil) + _ allKeysChanWithErrer = (*tqcache)(nil) ) func newTwoQueueCachedBS(ctx context.Context, bs Blockstore, lruSize int) (*tqcache, error) { @@ -398,6 +399,10 @@ func (b *tqcache) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { return b.blockstore.AllKeysChan(ctx) } +func (b *tqcache) allKeysChanWithErr(ctx context.Context) (<-chan cid.Cid, func() error, error) { + return allKeysChanWithErrFor(ctx, b.blockstore) +} + func (b *tqcache) GCLock(ctx context.Context) Unlocker { return b.blockstore.(GCBlockstore).GCLock(ctx) } diff --git a/blockstore/validating_blockstore.go b/blockstore/validating_blockstore.go index cc013c9eb..8ce5f6202 100644 --- a/blockstore/validating_blockstore.go +++ b/blockstore/validating_blockstore.go @@ -12,6 +12,12 @@ type ValidatingBlockstore struct { Blockstore } +var _ allKeysChanWithErrer = (*ValidatingBlockstore)(nil) + +func (bs *ValidatingBlockstore) allKeysChanWithErr(ctx context.Context) (<-chan cid.Cid, func() error, error) { + return allKeysChanWithErrFor(ctx, bs.Blockstore) +} + func (bs *ValidatingBlockstore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) { block, err := bs.Blockstore.Get(ctx, c) if err != nil { From d6353b14fb76e675e1132699b8187026ee5aab04 Mon Sep 17 00:00:00 2001 From: guillaumemichel Date: Fri, 3 Jul 2026 15:49:52 +0200 Subject: [PATCH 2/5] docs(CHANGELOG): link to PR number --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3acdaa533..ff03272be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,7 +35,7 @@ indexed, reporting present blocks as missing (`Has` returns false, The build now activates the filter only when enumeration is known to have completed; otherwise the cache degrades to correct pass-through. This also fixes a race where a cancelled build could still mark the filter active. -[#NNNN](https://github.com/ipfs/boxo/pull/NNNN) +[#1183](https://github.com/ipfs/boxo/pull/1183) ### Security From e6a0e4fe5073e9528064564f8861a2402d9aa035 Mon Sep 17 00:00:00 2001 From: guillaumemichel Date: Fri, 3 Jul 2026 17:35:57 +0200 Subject: [PATCH 3/5] feat(blockstore): expose BloomCacheStatus and add live bloom filter Rebuild --- CHANGELOG.md | 15 +- blockstore/allkeyschanwitherr_test.go | 38 ++-- blockstore/blockstore.go | 60 +++--- blockstore/bloom_cache.go | 115 ++++++++-- blockstore/bloom_cache_test.go | 4 +- blockstore/caching.go | 51 +++++ blockstore/idstore.go | 4 +- blockstore/rebuild_test.go | 298 ++++++++++++++++++++++++++ blockstore/twoqueue_cache.go | 4 +- blockstore/validating_blockstore.go | 4 +- 10 files changed, 513 insertions(+), 80 deletions(-) create mode 100644 blockstore/rebuild_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 33418f444..7ab0d2bb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,16 @@ The following emojis are used to highlight certain changes: - `gateway`: `GET`/`HEAD /ipfs/bafkqaaa?format=raw` now always returns `200` with an empty body, so probing clients keep marking the gateway as functional even when its backend cannot serve identity CIDs. `bitswap/network/httpnet` sends this [trustless gateway probe](https://specs.ipfs.tech/http-gateways/trustless-gateway/#dedicated-probe-paths) to check providers, and a failed probe drops the provider. Exported as `gateway.EmptyIdentityCID`. [#1179](https://github.com/ipfs/boxo/pull/1179) - `path`: added `NewPathFromURI`, which accepts native IPFS URIs (`ipfs://cid`, `ipns://name`, `ipld://cid`, and the schemeless `ipfs:`/`ipns:`/`ipld:` forms) and rewrites them to canonical content paths, so values copied from browsers and other tools parse as-is. `NewPath` stays strict and still rejects URI-shaped input, leaving untrusted parsing such as DNSLink records unchanged. [#1182](https://github.com/ipfs/boxo/pull/1182) +- `blockstore`: `CachedBlockstore` now returns a value implementing the +exported `BloomCacheStatus` interface (`Wait`/`BloomActive`/`Rebuild`) when a +Bloom filter is configured, so callers can wait for and observe the result of +the asynchronous filter build (previously these methods were unreachable on the +unexported cache type), and retry a failed build with `Rebuild`. While a +rebuild runs the filter is inactive (lookups fall through to the underlying +blockstore, so results stay correct but unaccelerated) and it is activated +again only on a complete enumeration. A new optional `AllKeysChanWithErrer` +capability lets a `Blockstore` report an error that truncates `AllKeysChan` +enumeration. [#MMMM](https://github.com/ipfs/boxo/pull/MMMM) ### Changed @@ -34,8 +44,9 @@ answered "not present" conclusively for blocks that exist but were never indexed, reporting present blocks as missing (`Has` returns false, `Get`/`GetSize`/`View` return not-found, `DeleteBlock` becomes a silent no-op). The build now activates the filter only when enumeration is known to have -completed; otherwise the cache degrades to correct pass-through. This also -fixes a race where a cancelled build could still mark the filter active. +completed; otherwise it records the error (observable via +`BloomCacheStatus.Wait`) and the cache degrades to correct pass-through. This +also fixes a race where a cancelled build could still mark the filter active. [#1183](https://github.com/ipfs/boxo/pull/1183) ### Security diff --git a/blockstore/allkeyschanwitherr_test.go b/blockstore/allkeyschanwitherr_test.go index fc7c9142a..c3e3cfc91 100644 --- a/blockstore/allkeyschanwitherr_test.go +++ b/blockstore/allkeyschanwitherr_test.go @@ -65,7 +65,7 @@ type cleanChanErrFnBS struct { enumErr error } -func (f *cleanChanErrFnBS) allKeysChanWithErr(ctx context.Context) (<-chan cid.Cid, func() error, error) { +func (f *cleanChanErrFnBS) AllKeysChanWithErr(ctx context.Context) (<-chan cid.Cid, func() error, error) { ch, err := f.AllKeysChan(ctx) if err != nil { return nil, nil, err @@ -76,8 +76,8 @@ func (f *cleanChanErrFnBS) allKeysChanWithErr(ctx context.Context) (<-chan cid.C func TestAllKeysChanWithErrClean(t *testing.T) { bs, keys := newBlockStoreWithKeys(t, nil, 100) - e := bs.(allKeysChanWithErrer) - ch, errFn, err := e.allKeysChanWithErr(t.Context()) + e := bs.(AllKeysChanWithErrer) + ch, errFn, err := e.AllKeysChanWithErr(t.Context()) if err != nil { t.Fatal(err) } @@ -100,8 +100,8 @@ func TestAllKeysChanWithErrMidIteration(t *testing.T) { } } - e := bs.(allKeysChanWithErrer) - ch, errFn, err := e.allKeysChanWithErr(t.Context()) + e := bs.(AllKeysChanWithErrer) + ch, errFn, err := e.AllKeysChanWithErr(t.Context()) if err != nil { t.Fatal(err) } @@ -125,10 +125,10 @@ func TestAllKeysChanWithErrContextCancel(t *testing.T) { n := 2*dsq.KeysOnlyBufSize + 10 bs, _ := newBlockStoreWithKeys(t, nil, n) - e := bs.(allKeysChanWithErrer) + e := bs.(AllKeysChanWithErrer) ctx, cancel := context.WithCancel(t.Context()) t.Cleanup(cancel) - ch, errFn, err := e.allKeysChanWithErr(ctx) + ch, errFn, err := e.AllKeysChanWithErr(ctx) if err != nil { t.Fatal(err) } @@ -295,11 +295,11 @@ func TestAllKeysChanWithErrForwardsThroughCacheStack(t *testing.T) { t.Fatal(err) } - e, ok := cbs.(allKeysChanWithErrer) + e, ok := cbs.(AllKeysChanWithErrer) if !ok { - t.Fatal("CachedBlockstore result does not implement allKeysChanWithErrer") + t.Fatal("CachedBlockstore result does not implement AllKeysChanWithErrer") } - ch, errFn, err := e.allKeysChanWithErr(ctx) + ch, errFn, err := e.AllKeysChanWithErr(ctx) if err != nil { t.Fatal(err) } @@ -309,7 +309,7 @@ func TestAllKeysChanWithErrForwardsThroughCacheStack(t *testing.T) { } } -// legacyBS implements only the Blockstore interface (no allKeysChanWithErrer), +// legacyBS implements only the Blockstore interface (no AllKeysChanWithErrer), // to exercise the best-effort fallback in allKeysChanWithErrFor. It must NOT // embed *blockstore, or the capability method would be promoted and the // fallback branch would never run. @@ -341,8 +341,8 @@ func TestAllKeysChanWithErrFallbackForLegacyBlockstore(t *testing.T) { // Guard the premise: legacyBS must NOT advertise the capability, else the // fallback branch is not exercised. - if _, ok := Blockstore(legacy).(allKeysChanWithErrer); ok { - t.Fatal("legacyBS unexpectedly implements allKeysChanWithErrer") + if _, ok := Blockstore(legacy).(AllKeysChanWithErrer); ok { + t.Fatal("legacyBS unexpectedly implements AllKeysChanWithErrer") } // The fallback yields a no-op errFn and a full enumeration. @@ -400,11 +400,11 @@ func TestAllKeysChanWithErrForwardingWrappers(t *testing.T) { for name, wrap := range wrappers { t.Run(name, func(t *testing.T) { w := wrap(newBaseWithErr()) - e, ok := w.(allKeysChanWithErrer) + e, ok := w.(AllKeysChanWithErrer) if !ok { - t.Fatalf("%s does not implement allKeysChanWithErrer", name) + t.Fatalf("%s does not implement AllKeysChanWithErrer", name) } - ch, errFn, err := e.allKeysChanWithErr(t.Context()) + ch, errFn, err := e.AllKeysChanWithErr(t.Context()) if err != nil { t.Fatal(err) } @@ -437,8 +437,8 @@ func TestAllKeysChanWithErrSkipsUnparseableKey(t *testing.T) { t.Fatal(err) } - e := bs.(allKeysChanWithErrer) - ch, errFn, err := e.allKeysChanWithErr(t.Context()) + e := bs.(AllKeysChanWithErrer) + ch, errFn, err := e.AllKeysChanWithErr(t.Context()) if err != nil { t.Fatal(err) } @@ -476,7 +476,7 @@ type incompleteViewerBS struct { enumErr error } -func (f *incompleteViewerBS) allKeysChanWithErr(ctx context.Context) (<-chan cid.Cid, func() error, error) { +func (f *incompleteViewerBS) AllKeysChanWithErr(ctx context.Context) (<-chan cid.Cid, func() error, error) { ch, err := f.AllKeysChan(ctx) if err != nil { return nil, nil, err diff --git a/blockstore/blockstore.go b/blockstore/blockstore.go index 828d99271..3e12ea5c3 100644 --- a/blockstore/blockstore.go +++ b/blockstore/blockstore.go @@ -65,11 +65,32 @@ type Blockstore interface { // cancelled context), the channel may be closed early without warning. // Callers MUST NOT assume the returned error reflects enumeration // completeness: it is only guaranteed to cover query setup, not - // mid-iteration failures. A consumer that requires a complete enumeration - // (such as building a Bloom filter) cannot rely on this method alone. + // mid-iteration failures. Consumers that require a complete enumeration + // (such as building a Bloom filter) should check whether the blockstore + // implements [AllKeysChanWithErrer]. AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) } +// AllKeysChanWithErrer is an optional capability a Blockstore may implement +// alongside AllKeysChan. AllKeysChanWithErr behaves like +// [Blockstore.AllKeysChan], but additionally returns a function that, once the +// returned channel has been fully drained, reports any error that terminated +// enumeration early. +// +// The reported error is nil if enumeration ran to completion without a +// mid-iteration error or cancellation; a non-nil error means enumeration was +// truncated and the delivered keys must not be treated as the complete set. +// Datastore keys that cannot be parsed as a block key are skipped without being +// treated as an error: such a key cannot correspond to a retrievable block, so +// omitting it cannot cause a Bloom-filter false negative. +// +// The function blocks until enumeration finishes, so it must be called only +// after the channel has been drained; calling it earlier deadlocks the caller +// once the producer fills the channel buffer. +type AllKeysChanWithErrer interface { + AllKeysChanWithErr(ctx context.Context) (<-chan cid.Cid, func() error, error) +} + // Viewer can be implemented by blockstores that offer zero-copy access to // values. // @@ -292,35 +313,16 @@ func (bs *blockstore) DeleteBlock(ctx context.Context, k cid.Cid) error { // // AllKeysChan respects context. func (bs *blockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { - ch, _, err := bs.allKeysChanWithErr(ctx) + ch, _, err := bs.AllKeysChanWithErr(ctx) return ch, err } -// allKeysChanWithErrer is an optional capability a Blockstore may implement -// alongside AllKeysChan. allKeysChanWithErr behaves like AllKeysChan, but -// additionally returns a function that, once the returned channel has been -// fully drained, reports any error that terminated enumeration early. -// -// The reported error is nil if enumeration ran to completion without a -// mid-iteration error or cancellation; a non-nil error means enumeration was -// truncated and the delivered keys must not be treated as the complete set. -// Datastore keys that cannot be parsed as a block key are skipped without being -// treated as an error: such a key cannot correspond to a retrievable block, so -// omitting it cannot cause a Bloom-filter false negative. -// -// The function blocks until enumeration finishes, so it must be called only -// after the channel has been drained; calling it earlier deadlocks the caller -// once the producer fills the channel buffer. -type allKeysChanWithErrer interface { - allKeysChanWithErr(ctx context.Context) (<-chan cid.Cid, func() error, error) -} - -var _ allKeysChanWithErrer = (*blockstore)(nil) +var _ AllKeysChanWithErrer = (*blockstore)(nil) -// allKeysChanWithErr implements [allKeysChanWithErrer]. The returned function +// AllKeysChanWithErr implements [AllKeysChanWithErrer]. The returned function // reports any error that ended key enumeration early (nil if every key was // delivered) and must be called only after the channel has been drained. -func (bs *blockstore) allKeysChanWithErr(ctx context.Context) (<-chan cid.Cid, func() error, error) { +func (bs *blockstore) AllKeysChanWithErr(ctx context.Context) (<-chan cid.Cid, func() error, error) { // KeysOnly, because that would be _a lot_ of data. q := dsq.Query{KeysOnly: true} res, err := bs.datastore.Query(ctx, q) @@ -373,13 +375,13 @@ func (bs *blockstore) allKeysChanWithErr(ctx context.Context) (<-chan cid.Cid, f return output, func() error { <-done; return iterErr }, nil } -// allKeysChanWithErrFor returns bs.allKeysChanWithErr when bs implements -// [allKeysChanWithErrer]. Otherwise it falls back to [Blockstore.AllKeysChan] +// allKeysChanWithErrFor returns bs.AllKeysChanWithErr when bs implements +// [AllKeysChanWithErrer]. Otherwise it falls back to [Blockstore.AllKeysChan] // with a no-op error function, preserving best-effort behavior for blockstores // that cannot report enumeration errors. func allKeysChanWithErrFor(ctx context.Context, bs Blockstore) (<-chan cid.Cid, func() error, error) { - if e, ok := bs.(allKeysChanWithErrer); ok { - return e.allKeysChanWithErr(ctx) + if e, ok := bs.(AllKeysChanWithErrer); ok { + return e.AllKeysChanWithErr(ctx) } ch, err := bs.AllKeysChan(ctx) return ch, func() error { return nil }, err diff --git a/blockstore/bloom_cache.go b/blockstore/bloom_cache.go index e90013d37..8b2fd9456 100644 --- a/blockstore/bloom_cache.go +++ b/blockstore/bloom_cache.go @@ -3,6 +3,7 @@ package blockstore import ( "context" "fmt" + "sync" "sync/atomic" "time" @@ -23,13 +24,15 @@ func bloomCached(ctx context.Context, bs Blockstore, bloomSize, hashCount int) ( } bc := &bloomcache{ blockstore: bs, - bloom: bl, + bloomSize: bloomSize, + hashCount: hashCount, hits: metrics.NewCtx(ctx, "bloom.hits_total", "Number of cache hits in bloom cache").Counter(), total: metrics.NewCtx(ctx, "bloom_total", "Total number of requests to bloom cache").Counter(), buildChan: make(chan struct{}), } + bc.bloom.Store(bl) if v, ok := bs.(Viewer); ok { bc.viewer = v } @@ -55,7 +58,7 @@ func bloomCached(ctx context.Context, bs Blockstore, bloomSize, hashCount int) ( case <-ctx.Done(): return case <-t.C: - fill.Set(bc.bloom.FillRatioTS()) + fill.Set(bc.bloom.Load().FillRatioTS()) } } } @@ -66,7 +69,15 @@ func bloomCached(ctx context.Context, bs Blockstore, bloomSize, hashCount int) ( type bloomcache struct { active int32 - bloom *bloom.Bloom + // bloom is the live filter. It is swapped atomically by Rebuild, so all + // accesses go through Load. + bloom atomic.Pointer[bloom.Bloom] + bloomSize int + hashCount int + + // buildMu serializes the initial build and any Rebuild, so at most one + // enumeration populates the filter at a time. + buildMu sync.Mutex buildErr error buildChan chan struct{} @@ -81,7 +92,8 @@ type bloomcache struct { var ( _ Blockstore = (*bloomcache)(nil) _ Viewer = (*bloomcache)(nil) - _ allKeysChanWithErrer = (*bloomcache)(nil) + _ AllKeysChanWithErrer = (*bloomcache)(nil) + _ BloomCacheStatus = (*bloomcache)(nil) ) func (b *bloomcache) BloomActive() bool { @@ -105,10 +117,74 @@ func (b *bloomcache) build(ctx context.Context) error { }() defer close(b.buildChan) + b.buildMu.Lock() + defer b.buildMu.Unlock() + + if err := b.populate(ctx, b.bloom.Load()); err != nil { + b.buildErr = err + return err + } + atomic.StoreInt32(&b.active, 1) + return nil +} + +// Rebuild discards the current Bloom filter and rebuilds it from a full +// AllKeysChan enumeration. It is meant to retry after a failed initial build +// (observable via [BloomCacheStatus]). +// +// While the rebuild runs, the filter is inactive: lookups fall through to the +// underlying blockstore (correct, but without Bloom acceleration) and writes +// populate the new filter. On a complete enumeration the filter is activated; +// if enumeration is truncated or ctx is cancelled, the filter is left inactive +// and the error is returned. +// +// Rebuild serializes with the initial build and concurrent Rebuild calls: it +// waits for any in-progress build to finish before starting (that wait does +// not observe ctx), then honors ctx for the enumeration. It does not update +// Wait, which reports only the initial build; observe a rebuild via this +// method's return value and BloomActive. Reliable rebuild while the store is +// written concurrently assumes a snapshot-consistent datastore enumeration +// (see rebuildLocked). +func (b *bloomcache) Rebuild(ctx context.Context) error { + b.buildMu.Lock() + defer b.buildMu.Unlock() + + if err := ctx.Err(); err != nil { + return err + } + fresh, err := bloom.New(float64(b.bloomSize), float64(b.hashCount)) + if err != nil { + return err + } + // Deactivate and swap in the empty filter before enumerating: reads degrade + // to correct pass-through and concurrent writes land in the new filter, + // exactly as during the initial build. + // + // Correctness against a Put that races this swap relies on the underlying + // datastore's enumeration reflecting every write that completed before the + // Query below was issued. Snapshotting backends (the in-memory MapDatastore, + // MutexWrap, LevelDB, Badger) provide this. A backend whose enumeration is + // not a point-in-time snapshot (such as flatfs's lazy directory walk) may + // instead leave a block written concurrently with a rebuild as a transient + // false negative until the next rebuild: the bloom-pointer atomic orders + // only the filter swap, not datastore visibility. + atomic.StoreInt32(&b.active, 0) + b.bloom.Store(fresh) + + if err := b.populate(ctx, fresh); err != nil { + return err + } + atomic.StoreInt32(&b.active, 1) + return nil +} + +// populate enumerates every key in the underlying blockstore into target. It +// returns a non-nil error if the enumeration did not run to completion, in +// which case the caller must not treat target as authoritative. +func (b *bloomcache) populate(ctx context.Context, target *bloom.Bloom) error { ch, errFn, err := allKeysChanWithErrFor(ctx, b.blockstore) if err != nil { - b.buildErr = fmt.Errorf("AllKeysChan failed in bloomcache build with: %w", err) - return b.buildErr + return fmt.Errorf("AllKeysChan failed in bloomcache build with: %w", err) } for { select { @@ -121,31 +197,22 @@ func (b *bloomcache) build(ctx context.Context) error { // be a false negative for blocks that exist but were never // indexed. // - // If the wrapped store does not implement allKeysChanWithErrer, + // If the wrapped store does not implement AllKeysChanWithErrer, // errFn is a no-op returning nil (see allKeysChanWithErrFor) and // the filter is treated as complete, preserving the pre-existing // best-effort behavior for such stores. if err := errFn(); err != nil { - b.buildErr = fmt.Errorf("bloomcache build incomplete, not activating filter: %w", err) - return b.buildErr + return fmt.Errorf("bloomcache build incomplete, not activating filter: %w", err) } - atomic.StoreInt32(&b.active, 1) return nil } - b.bloom.AddTS(key.Hash()) // Use binary key, the more compact the better + target.AddTS(key.Hash()) // Use binary key, the more compact the better case <-ctx.Done(): - b.buildErr = ctx.Err() - return b.buildErr + return ctx.Err() } } } -// allKeysChanWithErr forwards the error-reporting enumeration to the wrapped -// blockstore, so the completeness signal survives the cache stack. -func (b *bloomcache) allKeysChanWithErr(ctx context.Context) (<-chan cid.Cid, func() error, error) { - return allKeysChanWithErrFor(ctx, b.blockstore) -} - func (b *bloomcache) DeleteBlock(ctx context.Context, k cid.Cid) error { if has, ok := b.hasCached(k); ok && !has { return nil @@ -165,7 +232,7 @@ func (b *bloomcache) hasCached(k cid.Cid) (has bool, ok bool) { return false, false } if b.BloomActive() { - blr := b.bloom.HasTS(k.Hash()) + blr := b.bloom.Load().HasTS(k.Hash()) if !blr { // not contained in bloom is only conclusive answer bloom gives b.hits.Inc() return false, true @@ -217,7 +284,7 @@ func (b *bloomcache) Put(ctx context.Context, bl blocks.Block) error { // See comment in PutMany err := b.blockstore.Put(ctx, bl) if err == nil { - b.bloom.AddTS(bl.Cid().Hash()) + b.bloom.Load().AddTS(bl.Cid().Hash()) } return err } @@ -232,7 +299,7 @@ func (b *bloomcache) PutMany(ctx context.Context, bs []blocks.Block) error { return err } for _, bl := range bs { - b.bloom.AddTS(bl.Cid().Hash()) + b.bloom.Load().AddTS(bl.Cid().Hash()) } return nil } @@ -241,6 +308,10 @@ func (b *bloomcache) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { return b.blockstore.AllKeysChan(ctx) } +func (b *bloomcache) AllKeysChanWithErr(ctx context.Context) (<-chan cid.Cid, func() error, error) { + return allKeysChanWithErrFor(ctx, b.blockstore) +} + func (b *bloomcache) GCLock(ctx context.Context) Unlocker { return b.blockstore.(GCBlockstore).GCLock(ctx) } diff --git a/blockstore/bloom_cache_test.go b/blockstore/bloom_cache_test.go index 1aba6329c..7e053cd4f 100644 --- a/blockstore/bloom_cache_test.go +++ b/blockstore/bloom_cache_test.go @@ -41,7 +41,7 @@ func TestPutManyAddsToBloom(t *testing.T) { } if err := cachedbs.Wait(ctx); err != nil { - t.Fatalf("Failed while waiting for the filter to build: %d", cachedbs.bloom.ElementsAdded()) + t.Fatalf("Failed while waiting for the filter to build: %d", cachedbs.bloom.Load().ElementsAdded()) } block1 := blocks.NewBlock([]byte("foo")) @@ -110,7 +110,7 @@ func TestHasIsBloomCached(t *testing.T) { } if err := cachedbs.Wait(ctx); err != nil { - t.Fatalf("Failed while waiting for the filter to build: %d", cachedbs.bloom.ElementsAdded()) + t.Fatalf("Failed while waiting for the filter to build: %d", cachedbs.bloom.Load().ElementsAdded()) } cacheFails := 0 diff --git a/blockstore/caching.go b/blockstore/caching.go index c80f5780f..6d1471b86 100644 --- a/blockstore/caching.go +++ b/blockstore/caching.go @@ -24,8 +24,59 @@ func DefaultCacheOpts() CacheOpts { } } +// BloomCacheStatus may be implemented by the Blockstore returned from +// [CachedBlockstore] when a Bloom filter is configured +// (HasBloomFilterSize > 0). It lets callers observe the initial, asynchronous +// Bloom filter build: +// +// cbs, err := CachedBlockstore(ctx, bs, opts) +// if err != nil { +// // handle err +// } +// if s, ok := cbs.(BloomCacheStatus); ok { +// if err := s.Wait(ctx); err != nil { +// // The filter is not active: the blockstore still answers +// // correctly, but without Bloom-filter acceleration. +// } +// } +type BloomCacheStatus interface { + // Wait blocks until the initial Bloom filter build finishes, or until ctx + // is done, and returns that build's error, if any. It reflects only the + // initial build: a later Rebuild does not update it, so after calling + // Rebuild use that method's return value and BloomActive instead. A non-nil + // initial-build error means the filter is not active; the blockstore still + // returns correct answers, just without Bloom acceleration. + Wait(ctx context.Context) error + + // BloomActive reports whether the Bloom filter finished building + // successfully and is being used to answer negative lookups. + BloomActive() bool + + // Rebuild discards the current Bloom filter and rebuilds it from a full + // enumeration of the blockstore, returning the build error if any. It is + // meant to retry after a failed initial build (Wait returned an error). + // While it runs, the filter is inactive and lookups fall through to the + // underlying blockstore, so results stay correct but unaccelerated; on a + // failed rebuild the filter is left inactive. + // + // Rebuild serializes with the initial build and concurrent Rebuild calls; + // it waits for any in-progress build to finish before starting (that wait + // does not observe ctx), then honors ctx for the enumeration itself. + // + // Reliable rebuild while the store is written concurrently assumes the + // datastore's enumeration reflects all writes that completed before it + // began; a backend without that property (e.g. a lazy directory walk) may + // leave a block written during the rebuild as a transient false negative + // until the next rebuild. + Rebuild(ctx context.Context) error +} + // CachedBlockstore returns a blockstore wrapped in an TwoQueueCache and // then in a bloom filter cache, if the options indicate it. +// +// When a Bloom filter is configured, the returned Blockstore implements +// [BloomCacheStatus], which can be used to wait for and check the result of +// the initial Bloom filter build. func CachedBlockstore( ctx context.Context, bs Blockstore, diff --git a/blockstore/idstore.go b/blockstore/idstore.go index 49867d6c4..107232f85 100644 --- a/blockstore/idstore.go +++ b/blockstore/idstore.go @@ -19,7 +19,7 @@ var ( _ Blockstore = (*idstore)(nil) _ Viewer = (*idstore)(nil) _ io.Closer = (*idstore)(nil) - _ allKeysChanWithErrer = (*idstore)(nil) + _ AllKeysChanWithErrer = (*idstore)(nil) ) func NewIdStore(bs Blockstore) Blockstore { @@ -114,7 +114,7 @@ func (b *idstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { return b.bs.AllKeysChan(ctx) } -func (b *idstore) allKeysChanWithErr(ctx context.Context) (<-chan cid.Cid, func() error, error) { +func (b *idstore) AllKeysChanWithErr(ctx context.Context) (<-chan cid.Cid, func() error, error) { return allKeysChanWithErrFor(ctx, b.bs) } diff --git a/blockstore/rebuild_test.go b/blockstore/rebuild_test.go new file mode 100644 index 000000000..56d23675b --- /dev/null +++ b/blockstore/rebuild_test.go @@ -0,0 +1,298 @@ +package blockstore + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + "testing" + "time" + + blocks "github.com/ipfs/go-block-format" + cid "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + dsq "github.com/ipfs/go-datastore/query" + syncds "github.com/ipfs/go-datastore/sync" +) + +// toggleErrDS injects a mid-iteration enumeration error only while fail is set, +// so a test can fail an initial build and then let a Rebuild succeed. +type toggleErrDS struct { + ds.Batching + after int + fail atomic.Bool +} + +func (d *toggleErrDS) Query(ctx context.Context, q dsq.Query) (dsq.Results, error) { + res, err := d.Batching.Query(ctx, q) + if err != nil { + return nil, err + } + if !d.fail.Load() { + return res, nil + } + return dsq.ResultsWithContext(q, func(ctx context.Context, out chan<- dsq.Result) { + defer res.Close() + n := 0 + for { + r, ok := res.NextSync() + if !ok { + return + } + if r.Error != nil { + out <- r + return + } + if n >= d.after { + out <- dsq.Result{Error: errInjected} + return + } + select { + case <-ctx.Done(): + return + case out <- r: + } + n++ + } + }), nil +} + +// hookDS invokes hook once, after emitting `at` entries of an enumeration. The +// underlying MapDatastore materializes query results eagerly, so a block +// written by the hook is NOT seen by the in-flight enumeration — it can only +// reach the new filter through the concurrent write path. +type hookDS struct { + ds.Batching + at int + hook func() +} + +func (d *hookDS) Query(ctx context.Context, q dsq.Query) (dsq.Results, error) { + res, err := d.Batching.Query(ctx, q) + if err != nil { + return nil, err + } + return dsq.ResultsWithContext(q, func(ctx context.Context, out chan<- dsq.Result) { + defer res.Close() + n := 0 + for { + r, ok := res.NextSync() + if !ok { + return + } + if n == d.at && d.hook != nil { + d.hook() + } + select { + case <-ctx.Done(): + return + case out <- r: + } + n++ + } + }), nil +} + +func TestRebuildActivatesAfterFailedBuild(t *testing.T) { + under := syncds.MutexWrap(ds.NewMapDatastore()) + tds := &toggleErrDS{Batching: under, after: 10} + tds.fail.Store(true) + bs := NewBlockstore(tds) + + const total = 300 + keys := make([]cid.Cid, 0, total) + for i := range total { + b := blocks.NewBlock(fmt.Appendf(nil, "data %d", i)) + if err := bs.Put(bg, b); err != nil { + t.Fatal(err) + } + keys = append(keys, b.Cid()) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + cachedbs, err := testBloomCached(ctx, bs) + if err != nil { + t.Fatal(err) + } + + // The initial build fails: filter inactive, but reads remain correct. + if err := cachedbs.Wait(ctx); err == nil { + t.Fatal("expected initial build to fail") + } + if cachedbs.BloomActive() { + t.Fatal("filter must be inactive after a failed build") + } + for _, k := range keys { + if has, err := cachedbs.Has(bg, k); err != nil || !has { + t.Fatalf("present block reported missing while inactive (has=%v err=%v)", has, err) + } + } + + // Fix the datastore and rebuild. + tds.fail.Store(false) + if err := cachedbs.Rebuild(ctx); err != nil { + t.Fatalf("rebuild failed: %v", err) + } + if !cachedbs.BloomActive() { + t.Fatal("filter must be active after a successful rebuild") + } + // Wait reflects only the initial build and is intentionally not updated by + // Rebuild; callers observe a rebuild via its return value and BloomActive. + if err := cachedbs.Wait(ctx); err == nil { + t.Fatal("Wait should still report the initial build error after a Rebuild") + } + for _, k := range keys { + if has, err := cachedbs.Has(bg, k); err != nil || !has { + t.Fatalf("present block reported missing after rebuild (has=%v err=%v)", has, err) + } + } + absent := blocks.NewBlock([]byte("absent-after-rebuild")).Cid() + if has, err := cachedbs.Has(bg, absent); err != nil || has { + t.Fatalf("absent block reported present after rebuild (has=%v err=%v)", has, err) + } +} + +func TestRebuildFailureLeavesCorrectPassThrough(t *testing.T) { + under := syncds.MutexWrap(ds.NewMapDatastore()) + tds := &toggleErrDS{Batching: under, after: 10} // fail starts false: clean build + bs := NewBlockstore(tds) + + const total = 200 + keys := make([]cid.Cid, 0, total) + for i := range total { + b := blocks.NewBlock(fmt.Appendf(nil, "data %d", i)) + if err := bs.Put(bg, b); err != nil { + t.Fatal(err) + } + keys = append(keys, b.Cid()) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + cachedbs, err := testBloomCached(ctx, bs) + if err != nil { + t.Fatal(err) + } + if err := cachedbs.Wait(ctx); err != nil { + t.Fatalf("initial build failed: %v", err) + } + if !cachedbs.BloomActive() { + t.Fatal("filter should be active after a clean build") + } + + // A rebuild that fails leaves the filter inactive (not active on partial + // data), and reads stay correct via pass-through. + tds.fail.Store(true) + if err := cachedbs.Rebuild(ctx); err == nil { + t.Fatal("expected rebuild to fail") + } + if cachedbs.BloomActive() { + t.Fatal("filter must be inactive after a failed rebuild") + } + for _, k := range keys { + if has, err := cachedbs.Has(bg, k); err != nil || !has { + t.Fatalf("present block reported missing after failed rebuild (has=%v err=%v)", has, err) + } + } + absent := blocks.NewBlock([]byte("absent-after-failed-rebuild")).Cid() + if has, err := cachedbs.Has(bg, absent); err != nil || has { + t.Fatalf("absent block reported present after failed rebuild (has=%v err=%v)", has, err) + } +} + +func TestRebuildContextCancelledIsNonDestructive(t *testing.T) { + under, _ := newBlockStoreWithKeys(t, nil, 50) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + cachedbs, err := testBloomCached(ctx, under) + if err != nil { + t.Fatal(err) + } + if err := cachedbs.Wait(ctx); err != nil { + t.Fatalf("initial build failed: %v", err) + } + + cctx, ccancel := context.WithCancel(context.Background()) + ccancel() // already cancelled + if err := cachedbs.Rebuild(cctx); !errors.Is(err, context.Canceled) { + t.Fatalf("expected context.Canceled, got: %v", err) + } + // An immediately-cancelled rebuild bails before touching the filter. + if !cachedbs.BloomActive() { + t.Fatal("an immediately-cancelled rebuild must not deactivate the filter") + } +} + +func TestRebuildCapturesConcurrentPut(t *testing.T) { + under := syncds.MutexWrap(ds.NewMapDatastore()) + hds := &hookDS{Batching: under, at: 25} + bs := NewBlockstore(hds) + + const total = 50 + for i := range total { + if err := bs.Put(bg, blocks.NewBlock(fmt.Appendf(nil, "data %d", i))); err != nil { + t.Fatal(err) + } + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + cachedbs, err := testBloomCached(ctx, bs) + if err != nil { + t.Fatal(err) + } + if err := cachedbs.Wait(ctx); err != nil { + t.Fatalf("initial build failed: %v", err) + } + + // Arm a hook (after the initial build) that writes a new block partway + // through the rebuild enumeration. The eager query snapshot will not see + // it, so it can only land in the new filter via the concurrent write path. + concurrent := blocks.NewBlock([]byte("written-during-rebuild")) + hds.hook = func() { + if err := cachedbs.Put(bg, concurrent); err != nil { + t.Errorf("concurrent put failed: %v", err) + } + } + + if err := cachedbs.Rebuild(ctx); err != nil { + t.Fatalf("rebuild failed: %v", err) + } + if !cachedbs.BloomActive() { + t.Fatal("filter must be active after rebuild") + } + if has, err := cachedbs.Has(bg, concurrent.Cid()); err != nil || !has { + t.Fatalf("block written during rebuild reported missing (has=%v err=%v)", has, err) + } +} + +func TestBloomCacheStatusRebuildReachable(t *testing.T) { + under, _ := newBlockStoreWithKeys(t, nil, 10) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + cbs, err := CachedBlockstore(ctx, under, DefaultCacheOpts()) + if err != nil { + t.Fatal(err) + } + s, ok := cbs.(BloomCacheStatus) + if !ok { + t.Fatal("CachedBlockstore result does not implement BloomCacheStatus") + } + if err := s.Wait(ctx); err != nil { + t.Fatalf("initial build via BloomCacheStatus: %v", err) + } + if err := s.Rebuild(ctx); err != nil { + t.Fatalf("rebuild via BloomCacheStatus: %v", err) + } + if !s.BloomActive() { + t.Fatal("expected filter active after rebuild") + } +} diff --git a/blockstore/twoqueue_cache.go b/blockstore/twoqueue_cache.go index 5cc46bdc8..99e67a22e 100644 --- a/blockstore/twoqueue_cache.go +++ b/blockstore/twoqueue_cache.go @@ -44,7 +44,7 @@ type tqcache struct { var ( _ Blockstore = (*tqcache)(nil) _ Viewer = (*tqcache)(nil) - _ allKeysChanWithErrer = (*tqcache)(nil) + _ AllKeysChanWithErrer = (*tqcache)(nil) ) func newTwoQueueCachedBS(ctx context.Context, bs Blockstore, lruSize int) (*tqcache, error) { @@ -399,7 +399,7 @@ func (b *tqcache) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { return b.blockstore.AllKeysChan(ctx) } -func (b *tqcache) allKeysChanWithErr(ctx context.Context) (<-chan cid.Cid, func() error, error) { +func (b *tqcache) AllKeysChanWithErr(ctx context.Context) (<-chan cid.Cid, func() error, error) { return allKeysChanWithErrFor(ctx, b.blockstore) } diff --git a/blockstore/validating_blockstore.go b/blockstore/validating_blockstore.go index 8ce5f6202..2e7480dc4 100644 --- a/blockstore/validating_blockstore.go +++ b/blockstore/validating_blockstore.go @@ -12,9 +12,9 @@ type ValidatingBlockstore struct { Blockstore } -var _ allKeysChanWithErrer = (*ValidatingBlockstore)(nil) +var _ AllKeysChanWithErrer = (*ValidatingBlockstore)(nil) -func (bs *ValidatingBlockstore) allKeysChanWithErr(ctx context.Context) (<-chan cid.Cid, func() error, error) { +func (bs *ValidatingBlockstore) AllKeysChanWithErr(ctx context.Context) (<-chan cid.Cid, func() error, error) { return allKeysChanWithErrFor(ctx, bs.Blockstore) } From 33858eeab863db46cb25872c4b43434d0716d4dd Mon Sep 17 00:00:00 2001 From: guillaumemichel Date: Sat, 4 Jul 2026 05:30:45 +0200 Subject: [PATCH 4/5] docs: update PR reference --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ab0d2bb1..f96a183cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,7 @@ rebuild runs the filter is inactive (lookups fall through to the underlying blockstore, so results stay correct but unaccelerated) and it is activated again only on a complete enumeration. A new optional `AllKeysChanWithErrer` capability lets a `Blockstore` report an error that truncates `AllKeysChan` -enumeration. [#MMMM](https://github.com/ipfs/boxo/pull/MMMM) +enumeration. [#1184](https://github.com/ipfs/boxo/pull/1184) ### Changed From 551d07437f528530168617d08cd976d7e8fd2fbb Mon Sep 17 00:00:00 2001 From: guillaumemichel Date: Sat, 4 Jul 2026 22:11:02 +0200 Subject: [PATCH 5/5] refactor(blockstore): use atomic.Bool for bloomcache.active --- blockstore/bloom_cache.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/blockstore/bloom_cache.go b/blockstore/bloom_cache.go index 8b2fd9456..46eca98c4 100644 --- a/blockstore/bloom_cache.go +++ b/blockstore/bloom_cache.go @@ -67,7 +67,7 @@ func bloomCached(ctx context.Context, bs Blockstore, bloomSize, hashCount int) ( } type bloomcache struct { - active int32 + active atomic.Bool // bloom is the live filter. It is swapped atomically by Rebuild, so all // accesses go through Load. @@ -97,7 +97,7 @@ var ( ) func (b *bloomcache) BloomActive() bool { - return atomic.LoadInt32(&b.active) != 0 + return b.active.Load() } func (b *bloomcache) Wait(ctx context.Context) error { @@ -124,7 +124,7 @@ func (b *bloomcache) build(ctx context.Context) error { b.buildErr = err return err } - atomic.StoreInt32(&b.active, 1) + b.active.Store(true) return nil } @@ -168,13 +168,13 @@ func (b *bloomcache) Rebuild(ctx context.Context) error { // instead leave a block written concurrently with a rebuild as a transient // false negative until the next rebuild: the bloom-pointer atomic orders // only the filter swap, not datastore visibility. - atomic.StoreInt32(&b.active, 0) + b.active.Store(false) b.bloom.Store(fresh) if err := b.populate(ctx, fresh); err != nil { return err } - atomic.StoreInt32(&b.active, 1) + b.active.Store(true) return nil }